[llvm] [AArch64][SVE] Allow scalable factor-3 interleaved accesses (PR #200424)
Harry Ramsey via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 15 02:26:53 PDT 2026
https://github.com/Harry-Ramsey updated https://github.com/llvm/llvm-project/pull/200424
>From 365c462dfd0a0394f957e92ff7a231d1705f40cc Mon Sep 17 00:00:00 2001
From: Harry Ramsey <harry.ramsey at arm.com>
Date: Mon, 6 Jul 2026 15:55:59 +0000
Subject: [PATCH 1/4] [AArch64][ISel] Enable masked interleaved stores for
splat values
Enable masked interleaved store combine to recognise splat values and
split them into SVE component vectors. This allows the existing stN
lowering path to handle masked stores where the stored value is a splat
rather than an explicit vector.interleave result.
---
.../scalable_masked_interleaved_stores.ll | 72 +++++++++++++++++++
1 file changed, 72 insertions(+)
diff --git a/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll b/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
index d3cd9bf08cc0a..bdc25bba711e1 100644
--- a/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
+++ b/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
@@ -282,3 +282,75 @@ define void @foo_st2_nxv8i8_trunc(<vscale x 4 x i1> %mask, <vscale x 4 x i16> %v
call void @llvm.masked.store.nxv8i8.p0(<vscale x 8 x i8> %trunc.value, ptr %p, i32 1, <vscale x 8 x i1> %interleaved.mask)
ret void
}
+
+define void @foo_st2_nxv16i8_zeroinitializer(<vscale x 16 x i1> %mask, ptr %p) {
+; CHECK-LABEL: foo_st2_nxv16i8_zeroinitializer:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: zip2 p1.b, p0.b, p0.b
+; CHECK-NEXT: zip1 p0.b, p0.b, p0.b
+; CHECK-NEXT: st1b { z0.b }, p1, [x0, #1, mul vl]
+; CHECK-NEXT: st1b { z0.b }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask)
+ call void @llvm.masked.store.nxv32i8.p0(<vscale x 32 x i8> zeroinitializer, ptr %p, i32 1, <vscale x 32 x i1> %interleaved.mask)
+ ret void
+}
+
+define void @foo_st4_nxv16i8_zeroinitializer(<vscale x 16 x i1> %mask, ptr %p) {
+; CHECK-LABEL: foo_st4_nxv16i8_zeroinitializer:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: zip2 p1.b, p0.b, p0.b
+; CHECK-NEXT: zip1 p0.b, p0.b, p0.b
+; CHECK-NEXT: zip2 p2.b, p1.b, p1.b
+; CHECK-NEXT: zip1 p1.b, p1.b, p1.b
+; CHECK-NEXT: zip2 p3.b, p0.b, p0.b
+; CHECK-NEXT: st1b { z0.b }, p2, [x0, #3, mul vl]
+; CHECK-NEXT: zip1 p0.b, p0.b, p0.b
+; CHECK-NEXT: st1b { z0.b }, p1, [x0, #2, mul vl]
+; CHECK-NEXT: st1b { z0.b }, p3, [x0, #1, mul vl]
+; CHECK-NEXT: st1b { z0.b }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 64 x i1> @llvm.vector.interleave4.nxv64i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask)
+ call void @llvm.masked.store.nxv64i8.p0(<vscale x 64 x i8> zeroinitializer, ptr %p, i32 1, <vscale x 64 x i1> %interleaved.mask)
+ ret void
+}
+
+define void @foo_st2_nxv16i8_splat(<vscale x 16 x i1> %mask, ptr %p) {
+; CHECK-LABEL: foo_st2_nxv16i8_splat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov z0.b, #1 // =0x1
+; CHECK-NEXT: zip2 p1.b, p0.b, p0.b
+; CHECK-NEXT: zip1 p0.b, p0.b, p0.b
+; CHECK-NEXT: st1b { z0.b }, p1, [x0, #1, mul vl]
+; CHECK-NEXT: st1b { z0.b }, p0, [x0]
+; CHECK-NEXT: ret
+ %base = insertelement <vscale x 32 x i8> poison, i8 1, i32 0
+ %interleaved.value = shufflevector <vscale x 32 x i8> %base, <vscale x 32 x i8> poison, <vscale x 32 x i32> zeroinitializer
+ %interleaved.mask = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask)
+ call void @llvm.masked.store.nxv32i8.p0(<vscale x 32 x i8> %interleaved.value, ptr %p, i32 1, <vscale x 32 x i1> %interleaved.mask)
+ ret void
+}
+
+define void @foo_st4_nxv16i8_splat(<vscale x 16 x i1> %mask, ptr %p) {
+; CHECK-LABEL: foo_st4_nxv16i8_splat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: zip2 p1.b, p0.b, p0.b
+; CHECK-NEXT: mov z0.b, #1 // =0x1
+; CHECK-NEXT: zip1 p0.b, p0.b, p0.b
+; CHECK-NEXT: zip2 p2.b, p1.b, p1.b
+; CHECK-NEXT: zip1 p1.b, p1.b, p1.b
+; CHECK-NEXT: zip2 p3.b, p0.b, p0.b
+; CHECK-NEXT: st1b { z0.b }, p2, [x0, #3, mul vl]
+; CHECK-NEXT: zip1 p0.b, p0.b, p0.b
+; CHECK-NEXT: st1b { z0.b }, p1, [x0, #2, mul vl]
+; CHECK-NEXT: st1b { z0.b }, p3, [x0, #1, mul vl]
+; CHECK-NEXT: st1b { z0.b }, p0, [x0]
+; CHECK-NEXT: ret
+ %base = insertelement <vscale x 64 x i8> poison, i8 1, i32 0
+ %interleaved.value = shufflevector <vscale x 64 x i8> %base, <vscale x 64 x i8> poison, <vscale x 64 x i32> zeroinitializer
+ %interleaved.mask = call <vscale x 64 x i1> @llvm.vector.interleave4.nxv64i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask)
+ call void @llvm.masked.store.nxv64i8.p0(<vscale x 64 x i8> %interleaved.value, ptr %p, i32 1, <vscale x 64 x i1> %interleaved.mask)
+ ret void
+}
>From b130d00f840077403c4eb37b1ce98514d719a831 Mon Sep 17 00:00:00 2001
From: Harry Ramsey <harry.ramsey at arm.com>
Date: Mon, 6 Jul 2026 15:56:09 +0000
Subject: [PATCH 2/4] fixup! [AArch64][ISel] Enable masked interleaved stores
for splat values
---
.../Target/AArch64/AArch64ISelLowering.cpp | 59 ++++++++++++++++---
.../scalable_masked_interleaved_stores.ll | 40 ++++---------
2 files changed, 63 insertions(+), 36 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index f33d953eec747..151e6bd212f0d 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -27441,6 +27441,43 @@ isSequentialConcatOfVectorInterleave(SDNode *N, SmallVectorImpl<SDValue> &Ops) {
return true;
}
+static bool isSplatVectorInterleaveOps(SelectionDAG &DAG, SDLoc DL,
+ SDValue WideValue,
+ SmallVectorImpl<SDValue> &Ops) {
+ EVT WideVT = WideValue.getValueType();
+ if (!WideVT.isScalableVector())
+ return false;
+
+ SDValue SplatValue = DAG.getSplatValue(WideValue);
+ if (!SplatValue)
+ return false;
+
+ TypeSize WideSize = WideVT.getSizeInBits();
+ if (!WideSize.isKnownMultipleOf(128))
+ return false;
+
+ unsigned NumParts = WideSize.getKnownMinValue() / 128;
+ if (NumParts != 2 && NumParts != 3 && NumParts != 4)
+ return false;
+
+ ElementCount WideEC = WideVT.getVectorElementCount();
+ if (!WideEC.isKnownMultipleOf(NumParts))
+ return false;
+
+ EVT SubVecTy =
+ EVT::getVectorVT(*DAG.getContext(), WideVT.getVectorElementType(),
+ WideEC.divideCoefficientBy(NumParts));
+
+ if (SubVecTy.getSizeInBits().getKnownMinValue() != 128 ||
+ !DAG.getTargetLoweringInfo().isTypeLegal(SubVecTy))
+ return false;
+
+ SDValue NarrowSplat =
+ DAG.getNode(ISD::SPLAT_VECTOR, DL, SubVecTy, SplatValue);
+ Ops.append(NumParts, NarrowSplat);
+ return true;
+}
+
static SDValue getNarrowMaskForInterleavedOps(SelectionDAG &DAG, SDLoc &DL,
SDValue WideMask,
unsigned RequiredNumParts) {
@@ -27479,17 +27516,24 @@ static SDValue performInterleavedMaskedStoreCombine(
MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);
SDValue WideValue = MST->getValue();
- // Bail out if the stored value has an unexpected number of uses, since we'll
- // have to perform manual interleaving and may as well just use normal masked
- // stores. Also, discard masked stores that are truncating or indexed.
- if (!WideValue.hasOneUse() || !ISD::isNormalMaskedStore(MST) ||
- !MST->isSimple() || !MST->getOffset().isUndef())
+ // Discard masked stores that are truncating or indexed.
+ if (!ISD::isNormalMaskedStore(MST) || !MST->isSimple() ||
+ !MST->getOffset().isUndef())
return SDValue();
+ SDLoc DL(N);
SmallVector<SDValue, 4> ValueInterleaveOps;
- if (!isSequentialConcatOfVectorInterleave(WideValue.getNode(),
- ValueInterleaveOps))
+ if (isSequentialConcatOfVectorInterleave(WideValue.getNode(),
+ ValueInterleaveOps)) {
+ // Bail out if the stored value has an unexpected number of uses, since
+ // we'll have to perform manual interleaving and may as well just use normal
+ // masked stores.
+ if (!WideValue.hasOneUse())
+ return SDValue();
+ } else if (!isSplatVectorInterleaveOps(DAG, DL, WideValue,
+ ValueInterleaveOps)) {
return SDValue();
+ }
unsigned NumParts = ValueInterleaveOps.size();
if (NumParts != 2 && NumParts != 4)
@@ -27503,7 +27547,6 @@ static SDValue performInterleavedMaskedStoreCombine(
!DAG.getTargetLoweringInfo().isTypeLegal(SubVecTy))
return SDValue();
- SDLoc DL(N);
SDValue NarrowMask =
getNarrowMaskForInterleavedOps(DAG, DL, MST->getMask(), NumParts);
if (!NarrowMask)
diff --git a/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll b/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
index bdc25bba711e1..87833306571c2 100644
--- a/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
+++ b/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
@@ -287,10 +287,8 @@ define void @foo_st2_nxv16i8_zeroinitializer(<vscale x 16 x i1> %mask, ptr %p) {
; CHECK-LABEL: foo_st2_nxv16i8_zeroinitializer:
; CHECK: // %bb.0:
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: zip2 p1.b, p0.b, p0.b
-; CHECK-NEXT: zip1 p0.b, p0.b, p0.b
-; CHECK-NEXT: st1b { z0.b }, p1, [x0, #1, mul vl]
-; CHECK-NEXT: st1b { z0.b }, p0, [x0]
+; CHECK-NEXT: mov z1.d, z0.d
+; CHECK-NEXT: st2b { z0.b, z1.b }, p0, [x0]
; CHECK-NEXT: ret
%interleaved.mask = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask)
call void @llvm.masked.store.nxv32i8.p0(<vscale x 32 x i8> zeroinitializer, ptr %p, i32 1, <vscale x 32 x i1> %interleaved.mask)
@@ -301,16 +299,10 @@ define void @foo_st4_nxv16i8_zeroinitializer(<vscale x 16 x i1> %mask, ptr %p) {
; CHECK-LABEL: foo_st4_nxv16i8_zeroinitializer:
; CHECK: // %bb.0:
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: zip2 p1.b, p0.b, p0.b
-; CHECK-NEXT: zip1 p0.b, p0.b, p0.b
-; CHECK-NEXT: zip2 p2.b, p1.b, p1.b
-; CHECK-NEXT: zip1 p1.b, p1.b, p1.b
-; CHECK-NEXT: zip2 p3.b, p0.b, p0.b
-; CHECK-NEXT: st1b { z0.b }, p2, [x0, #3, mul vl]
-; CHECK-NEXT: zip1 p0.b, p0.b, p0.b
-; CHECK-NEXT: st1b { z0.b }, p1, [x0, #2, mul vl]
-; CHECK-NEXT: st1b { z0.b }, p3, [x0, #1, mul vl]
-; CHECK-NEXT: st1b { z0.b }, p0, [x0]
+; CHECK-NEXT: mov z1.d, z0.d
+; CHECK-NEXT: mov z2.d, z0.d
+; CHECK-NEXT: mov z3.d, z0.d
+; CHECK-NEXT: st4b { z0.b - z3.b }, p0, [x0]
; CHECK-NEXT: ret
%interleaved.mask = call <vscale x 64 x i1> @llvm.vector.interleave4.nxv64i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask)
call void @llvm.masked.store.nxv64i8.p0(<vscale x 64 x i8> zeroinitializer, ptr %p, i32 1, <vscale x 64 x i1> %interleaved.mask)
@@ -321,10 +313,8 @@ define void @foo_st2_nxv16i8_splat(<vscale x 16 x i1> %mask, ptr %p) {
; CHECK-LABEL: foo_st2_nxv16i8_splat:
; CHECK: // %bb.0:
; CHECK-NEXT: mov z0.b, #1 // =0x1
-; CHECK-NEXT: zip2 p1.b, p0.b, p0.b
-; CHECK-NEXT: zip1 p0.b, p0.b, p0.b
-; CHECK-NEXT: st1b { z0.b }, p1, [x0, #1, mul vl]
-; CHECK-NEXT: st1b { z0.b }, p0, [x0]
+; CHECK-NEXT: mov z1.d, z0.d
+; CHECK-NEXT: st2b { z0.b, z1.b }, p0, [x0]
; CHECK-NEXT: ret
%base = insertelement <vscale x 32 x i8> poison, i8 1, i32 0
%interleaved.value = shufflevector <vscale x 32 x i8> %base, <vscale x 32 x i8> poison, <vscale x 32 x i32> zeroinitializer
@@ -336,17 +326,11 @@ define void @foo_st2_nxv16i8_splat(<vscale x 16 x i1> %mask, ptr %p) {
define void @foo_st4_nxv16i8_splat(<vscale x 16 x i1> %mask, ptr %p) {
; CHECK-LABEL: foo_st4_nxv16i8_splat:
; CHECK: // %bb.0:
-; CHECK-NEXT: zip2 p1.b, p0.b, p0.b
; CHECK-NEXT: mov z0.b, #1 // =0x1
-; CHECK-NEXT: zip1 p0.b, p0.b, p0.b
-; CHECK-NEXT: zip2 p2.b, p1.b, p1.b
-; CHECK-NEXT: zip1 p1.b, p1.b, p1.b
-; CHECK-NEXT: zip2 p3.b, p0.b, p0.b
-; CHECK-NEXT: st1b { z0.b }, p2, [x0, #3, mul vl]
-; CHECK-NEXT: zip1 p0.b, p0.b, p0.b
-; CHECK-NEXT: st1b { z0.b }, p1, [x0, #2, mul vl]
-; CHECK-NEXT: st1b { z0.b }, p3, [x0, #1, mul vl]
-; CHECK-NEXT: st1b { z0.b }, p0, [x0]
+; CHECK-NEXT: mov z1.d, z0.d
+; CHECK-NEXT: mov z2.d, z0.d
+; CHECK-NEXT: mov z3.d, z0.d
+; CHECK-NEXT: st4b { z0.b - z3.b }, p0, [x0]
; CHECK-NEXT: ret
%base = insertelement <vscale x 64 x i8> poison, i8 1, i32 0
%interleaved.value = shufflevector <vscale x 64 x i8> %base, <vscale x 64 x i8> poison, <vscale x 64 x i32> zeroinitializer
>From 5146b84d18a753c858394eafa9fc2855d27f5ae0 Mon Sep 17 00:00:00 2001
From: Harry Ramsey <harry.ramsey at arm.com>
Date: Mon, 13 Jul 2026 15:13:30 +0000
Subject: [PATCH 3/4] [AArch64][ISel] Support i1 masked interleaved loads and
stores
Lower predicate value interleaved memory operations through full-width
integer containers. Extend predicates before st2/st4 and reconstruct
them with comparisons after ld2/ld4.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 55 ++++++--
.../scalable_masked_deinterleaved_loads.ll | 120 ++++++++++++++++++
.../scalable_masked_interleaved_stores.ll | 112 ++++++++++++++++
3 files changed, 276 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 151e6bd212f0d..a76b428c0fde5 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -27508,6 +27508,38 @@ static SDValue getNarrowMaskForInterleavedOps(SelectionDAG &DAG, SDLoc &DL,
WideMask->getOperand(0));
}
+static bool isSupportedSVEInterleavedMemSubvectorType(EVT VT,
+ SelectionDAG &DAG) {
+ return VT.isScalableVector() &&
+ (VT.getSizeInBits().getKnownMinValue() == 128 ||
+ (VT.isVectorOf(MVT::i1) && VT != MVT::nxv1i1)) &&
+ DAG.getTargetLoweringInfo().isTypeLegal(VT);
+}
+
+static EVT getSVEInterleavedMemContainerType(EVT VT) {
+ return VT.isVectorOf(MVT::i1) ? getPromotedVTForPredicate(VT) : VT;
+}
+
+static SDValue convertToSVEInterleavedMemContainer(SDValue Value, SDLoc DL,
+ SelectionDAG &DAG) {
+ EVT ContainerVT = getSVEInterleavedMemContainerType(Value.getValueType());
+ if (Value.getValueType() == ContainerVT)
+ return Value;
+ return DAG.getNode(ISD::ZERO_EXTEND, DL, ContainerVT, Value);
+}
+
+static SDValue convertFromSVEInterleavedMemContainer(SDValue Value, EVT VT,
+ SDLoc DL,
+ SelectionDAG &DAG) {
+ if (Value.getValueType() == VT)
+ return Value;
+ assert(VT.isVectorOf(MVT::i1) &&
+ Value.getValueType() == getPromotedVTForPredicate(VT) &&
+ "Unexpected SVE interleaved memory container conversion");
+ return DAG.getSetCC(DL, VT, Value,
+ DAG.getConstant(0, DL, Value.getValueType()), ISD::SETNE);
+}
+
static SDValue performInterleavedMaskedStoreCombine(
SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG) {
if (!DCI.isBeforeLegalize())
@@ -27542,9 +27574,7 @@ static SDValue performInterleavedMaskedStoreCombine(
// At the moment we're unlikely to see a fixed-width vector interleave as
// we usually generate shuffles instead.
EVT SubVecTy = ValueInterleaveOps[0].getValueType();
- if (!SubVecTy.isScalableVT() ||
- SubVecTy.getSizeInBits().getKnownMinValue() != 128 ||
- !DAG.getTargetLoweringInfo().isTypeLegal(SubVecTy))
+ if (!isSupportedSVEInterleavedMemSubvectorType(SubVecTy, DAG))
return SDValue();
SDValue NarrowMask =
@@ -27552,6 +27582,9 @@ static SDValue performInterleavedMaskedStoreCombine(
if (!NarrowMask)
return SDValue();
+ for (SDValue &Value : ValueInterleaveOps)
+ Value = convertToSVEInterleavedMemContainer(Value, DL, DAG);
+
const Intrinsic::ID IID =
NumParts == 2 ? Intrinsic::aarch64_sve_st2 : Intrinsic::aarch64_sve_st4;
SmallVector<SDValue, 8> NewStOps;
@@ -30449,9 +30482,7 @@ static SDValue performVectorDeinterleaveCombine(
// At the moment we're unlikely to see a fixed-width vector deinterleave as
// we usually generate shuffles instead.
unsigned MinNumElements = SubVecTy.getVectorMinNumElements();
- if (!SubVecTy.isScalableVector() ||
- SubVecTy.getSizeInBits().getKnownMinValue() != 128 ||
- !DAG.getTargetLoweringInfo().isTypeLegal(SubVecTy))
+ if (!isSupportedSVEInterleavedMemSubvectorType(SubVecTy, DAG))
return SDValue();
// Make sure each input operand is the correct extract_subvector of the same
@@ -30487,6 +30518,8 @@ static SDValue performVectorDeinterleaveCombine(
if (!NarrowMask)
return SDValue();
+ EVT LoadVT = getSVEInterleavedMemContainerType(SubVecTy);
+
const Intrinsic::ID IID = NumParts == 2 ? Intrinsic::aarch64_sve_ld2_sret
: Intrinsic::aarch64_sve_ld4_sret;
SDValue NewLdOps[] = {MaskedLoad->getChain(),
@@ -30494,17 +30527,17 @@ static SDValue performVectorDeinterleaveCombine(
MaskedLoad->getBasePtr()};
SDValue Res;
if (NumParts == 2)
- Res = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL,
- {SubVecTy, SubVecTy, MVT::Other}, NewLdOps);
+ Res = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, {LoadVT, LoadVT, MVT::Other},
+ NewLdOps);
else
Res = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL,
- {SubVecTy, SubVecTy, SubVecTy, SubVecTy, MVT::Other},
- NewLdOps);
+ {LoadVT, LoadVT, LoadVT, LoadVT, MVT::Other}, NewLdOps);
// We can now generate a structured load!
SmallVector<SDValue, 4> ResOps(NumParts);
for (unsigned Idx = 0; Idx < NumParts; Idx++)
- ResOps[Idx] = SDValue(Res.getNode(), Idx);
+ ResOps[Idx] = convertFromSVEInterleavedMemContainer(
+ SDValue(Res.getNode(), Idx), SubVecTy, DL, DAG);
// Replace uses of the original chain result with the new chain result.
DAG.ReplaceAllUsesOfValueWith(SDValue(MaskedLoad, 1),
diff --git a/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll b/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll
index ee2d482d9ffb5..ca5bae6791cf6 100644
--- a/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll
+++ b/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll
@@ -1,6 +1,62 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+define { <vscale x 2 x i1>, <vscale x 2 x i1> } @foo_ld2_nxv2i1(<vscale x 2 x i1> %mask, ptr %p) {
+; CHECK-LABEL: foo_ld2_nxv2i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ld2d { z0.d, z1.d }, p0/z, [x0]
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: cmpne p0.d, p1/z, z0.d, #0
+; CHECK-NEXT: cmpne p1.d, p1/z, z1.d, #0
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 4 x i1> @llvm.vector.interleave2.nxv4i1(<vscale x 2 x i1> %mask, <vscale x 2 x i1> %mask)
+ %wide.masked.vec = call <vscale x 4 x i1> @llvm.masked.load.nxv4i1.p0(ptr %p, i32 1, <vscale x 4 x i1> %interleaved.mask, <vscale x 4 x i1> poison)
+ %deinterleaved.vec = call { <vscale x 2 x i1>, <vscale x 2 x i1> } @llvm.vector.deinterleave2.nxv4i1(<vscale x 4 x i1> %wide.masked.vec)
+ ret { <vscale x 2 x i1>, <vscale x 2 x i1> } %deinterleaved.vec
+}
+
+define { <vscale x 4 x i1>, <vscale x 4 x i1> } @foo_ld2_nxv4i1(<vscale x 4 x i1> %mask, ptr %p) {
+; CHECK-LABEL: foo_ld2_nxv4i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ld2w { z0.s, z1.s }, p0/z, [x0]
+; CHECK-NEXT: ptrue p1.s
+; CHECK-NEXT: cmpne p0.s, p1/z, z0.s, #0
+; CHECK-NEXT: cmpne p1.s, p1/z, z1.s, #0
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 8 x i1> @llvm.vector.interleave2.nxv8i1(<vscale x 4 x i1> %mask, <vscale x 4 x i1> %mask)
+ %wide.masked.vec = call <vscale x 8 x i1> @llvm.masked.load.nxv8i1.p0(ptr %p, i32 1, <vscale x 8 x i1> %interleaved.mask, <vscale x 8 x i1> poison)
+ %deinterleaved.vec = call { <vscale x 4 x i1>, <vscale x 4 x i1> } @llvm.vector.deinterleave2.nxv8i1(<vscale x 8 x i1> %wide.masked.vec)
+ ret { <vscale x 4 x i1>, <vscale x 4 x i1> } %deinterleaved.vec
+}
+
+define { <vscale x 8 x i1>, <vscale x 8 x i1> } @foo_ld2_nxv8i1(<vscale x 8 x i1> %mask, ptr %p) {
+; CHECK-LABEL: foo_ld2_nxv8i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ld2h { z0.h, z1.h }, p0/z, [x0]
+; CHECK-NEXT: ptrue p1.h
+; CHECK-NEXT: cmpne p0.h, p1/z, z0.h, #0
+; CHECK-NEXT: cmpne p1.h, p1/z, z1.h, #0
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 16 x i1> @llvm.vector.interleave2.nxv16i1(<vscale x 8 x i1> %mask, <vscale x 8 x i1> %mask)
+ %wide.masked.vec = call <vscale x 16 x i1> @llvm.masked.load.nxv16i1.p0(ptr %p, i32 1, <vscale x 16 x i1> %interleaved.mask, <vscale x 16 x i1> poison)
+ %deinterleaved.vec = call { <vscale x 8 x i1>, <vscale x 8 x i1> } @llvm.vector.deinterleave2.nxv16i1(<vscale x 16 x i1> %wide.masked.vec)
+ ret { <vscale x 8 x i1>, <vscale x 8 x i1> } %deinterleaved.vec
+}
+
+define { <vscale x 16 x i1>, <vscale x 16 x i1> } @foo_ld2_nxv16i1(<vscale x 16 x i1> %mask, ptr %p) {
+; CHECK-LABEL: foo_ld2_nxv16i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ld2b { z0.b, z1.b }, p0/z, [x0]
+; CHECK-NEXT: ptrue p1.b
+; CHECK-NEXT: cmpne p0.b, p1/z, z0.b, #0
+; CHECK-NEXT: cmpne p1.b, p1/z, z1.b, #0
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask)
+ %wide.masked.vec = call <vscale x 32 x i1> @llvm.masked.load.nxv32i1.p0(ptr %p, i32 1, <vscale x 32 x i1> %interleaved.mask, <vscale x 32 x i1> poison)
+ %deinterleaved.vec = call { <vscale x 16 x i1>, <vscale x 16 x i1> } @llvm.vector.deinterleave2.nxv32i1(<vscale x 32 x i1> %wide.masked.vec)
+ ret { <vscale x 16 x i1>, <vscale x 16 x i1> } %deinterleaved.vec
+}
+
define { <vscale x 16 x i8>, <vscale x 16 x i8> } @foo_ld2_nxv16i8(<vscale x 16 x i1> %mask, ptr %p) {
; CHECK-LABEL: foo_ld2_nxv16i8:
; CHECK: // %bb.0:
@@ -45,6 +101,70 @@ define { <vscale x 2 x double>, <vscale x 2 x double> } @foo_ld2_nxv2f64(<vscale
ret { <vscale x 2 x double>, <vscale x 2 x double> } %deinterleaved.vec
}
+define { <vscale x 2 x i1>, <vscale x 2 x i1>, <vscale x 2 x i1>, <vscale x 2 x i1> } @foo_ld4_nxv2i1(<vscale x 2 x i1> %mask, ptr %p) {
+; CHECK-LABEL: foo_ld4_nxv2i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ld4d { z0.d - z3.d }, p0/z, [x0]
+; CHECK-NEXT: ptrue p3.d
+; CHECK-NEXT: cmpne p0.d, p3/z, z0.d, #0
+; CHECK-NEXT: cmpne p1.d, p3/z, z1.d, #0
+; CHECK-NEXT: cmpne p2.d, p3/z, z2.d, #0
+; CHECK-NEXT: cmpne p3.d, p3/z, z3.d, #0
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 8 x i1> @llvm.vector.interleave4.nxv8i1(<vscale x 2 x i1> %mask, <vscale x 2 x i1> %mask, <vscale x 2 x i1> %mask, <vscale x 2 x i1> %mask)
+ %wide.masked.vec = call <vscale x 8 x i1> @llvm.masked.load.nxv8i1.p0(ptr %p, i32 1, <vscale x 8 x i1> %interleaved.mask, <vscale x 8 x i1> poison)
+ %deinterleaved.vec = call { <vscale x 2 x i1>, <vscale x 2 x i1>, <vscale x 2 x i1>, <vscale x 2 x i1> } @llvm.vector.deinterleave4.nxv8i1(<vscale x 8 x i1> %wide.masked.vec)
+ ret { <vscale x 2 x i1>, <vscale x 2 x i1>, <vscale x 2 x i1>, <vscale x 2 x i1> } %deinterleaved.vec
+}
+
+define { <vscale x 4 x i1>, <vscale x 4 x i1>, <vscale x 4 x i1>, <vscale x 4 x i1> } @foo_ld4_nxv4i1(<vscale x 4 x i1> %mask, ptr %p) {
+; CHECK-LABEL: foo_ld4_nxv4i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ld4w { z0.s - z3.s }, p0/z, [x0]
+; CHECK-NEXT: ptrue p3.s
+; CHECK-NEXT: cmpne p0.s, p3/z, z0.s, #0
+; CHECK-NEXT: cmpne p1.s, p3/z, z1.s, #0
+; CHECK-NEXT: cmpne p2.s, p3/z, z2.s, #0
+; CHECK-NEXT: cmpne p3.s, p3/z, z3.s, #0
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 16 x i1> @llvm.vector.interleave4.nxv16i1(<vscale x 4 x i1> %mask, <vscale x 4 x i1> %mask, <vscale x 4 x i1> %mask, <vscale x 4 x i1> %mask)
+ %wide.masked.vec = call <vscale x 16 x i1> @llvm.masked.load.nxv16i1.p0(ptr %p, i32 1, <vscale x 16 x i1> %interleaved.mask, <vscale x 16 x i1> poison)
+ %deinterleaved.vec = call { <vscale x 4 x i1>, <vscale x 4 x i1>, <vscale x 4 x i1>, <vscale x 4 x i1> } @llvm.vector.deinterleave4.nxv16i1(<vscale x 16 x i1> %wide.masked.vec)
+ ret { <vscale x 4 x i1>, <vscale x 4 x i1>, <vscale x 4 x i1>, <vscale x 4 x i1> } %deinterleaved.vec
+}
+
+define { <vscale x 8 x i1>, <vscale x 8 x i1>, <vscale x 8 x i1>, <vscale x 8 x i1> } @foo_ld4_nxv8i1(<vscale x 8 x i1> %mask, ptr %p) {
+; CHECK-LABEL: foo_ld4_nxv8i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ld4h { z0.h - z3.h }, p0/z, [x0]
+; CHECK-NEXT: ptrue p3.h
+; CHECK-NEXT: cmpne p0.h, p3/z, z0.h, #0
+; CHECK-NEXT: cmpne p1.h, p3/z, z1.h, #0
+; CHECK-NEXT: cmpne p2.h, p3/z, z2.h, #0
+; CHECK-NEXT: cmpne p3.h, p3/z, z3.h, #0
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 32 x i1> @llvm.vector.interleave4.nxv32i1(<vscale x 8 x i1> %mask, <vscale x 8 x i1> %mask, <vscale x 8 x i1> %mask, <vscale x 8 x i1> %mask)
+ %wide.masked.vec = call <vscale x 32 x i1> @llvm.masked.load.nxv32i1.p0(ptr %p, i32 1, <vscale x 32 x i1> %interleaved.mask, <vscale x 32 x i1> poison)
+ %deinterleaved.vec = call { <vscale x 8 x i1>, <vscale x 8 x i1>, <vscale x 8 x i1>, <vscale x 8 x i1> } @llvm.vector.deinterleave4.nxv32i1(<vscale x 32 x i1> %wide.masked.vec)
+ ret { <vscale x 8 x i1>, <vscale x 8 x i1>, <vscale x 8 x i1>, <vscale x 8 x i1> } %deinterleaved.vec
+}
+
+define { <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1> } @foo_ld4_nxv16i1(<vscale x 16 x i1> %mask, ptr %p) {
+; CHECK-LABEL: foo_ld4_nxv16i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ld4b { z0.b - z3.b }, p0/z, [x0]
+; CHECK-NEXT: ptrue p3.b
+; CHECK-NEXT: cmpne p0.b, p3/z, z0.b, #0
+; CHECK-NEXT: cmpne p1.b, p3/z, z1.b, #0
+; CHECK-NEXT: cmpne p2.b, p3/z, z2.b, #0
+; CHECK-NEXT: cmpne p3.b, p3/z, z3.b, #0
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 64 x i1> @llvm.vector.interleave4.nxv64i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask)
+ %wide.masked.vec = call <vscale x 64 x i1> @llvm.masked.load.nxv64i1.p0(ptr %p, i32 1, <vscale x 64 x i1> %interleaved.mask, <vscale x 64 x i1> poison)
+ %deinterleaved.vec = call { <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1> } @llvm.vector.deinterleave4.nxv64i1(<vscale x 64 x i1> %wide.masked.vec)
+ ret { <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1> } %deinterleaved.vec
+}
+
define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @foo_ld4_nxv16i8(<vscale x 16 x i1> %mask, ptr %p) {
; CHECK-LABEL: foo_ld4_nxv16i8:
; CHECK: // %bb.0:
diff --git a/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll b/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
index 87833306571c2..ed3f2f772787a 100644
--- a/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
+++ b/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
@@ -1,6 +1,58 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+define void @foo_st2_nxv2i1(<vscale x 2 x i1> %mask, <vscale x 2 x i1> %val1, <vscale x 2 x i1> %val2, ptr %p) {
+; CHECK-LABEL: foo_st2_nxv2i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov z1.d, p2/z, #1 // =0x1
+; CHECK-NEXT: mov z0.d, p1/z, #1 // =0x1
+; CHECK-NEXT: st2d { z0.d, z1.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 4 x i1> @llvm.vector.interleave2.nxv4i1(<vscale x 2 x i1> %mask, <vscale x 2 x i1> %mask)
+ %interleaved.value = call <vscale x 4 x i1> @llvm.vector.interleave2.nxv4i1(<vscale x 2 x i1> %val1, <vscale x 2 x i1> %val2)
+ call void @llvm.masked.store.nxv4i1.p0(<vscale x 4 x i1> %interleaved.value, ptr %p, i32 1, <vscale x 4 x i1> %interleaved.mask)
+ ret void
+}
+
+define void @foo_st2_nxv4i1(<vscale x 4 x i1> %mask, <vscale x 4 x i1> %val1, <vscale x 4 x i1> %val2, ptr %p) {
+; CHECK-LABEL: foo_st2_nxv4i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov z1.s, p2/z, #1 // =0x1
+; CHECK-NEXT: mov z0.s, p1/z, #1 // =0x1
+; CHECK-NEXT: st2w { z0.s, z1.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 8 x i1> @llvm.vector.interleave2.nxv8i1(<vscale x 4 x i1> %mask, <vscale x 4 x i1> %mask)
+ %interleaved.value = call <vscale x 8 x i1> @llvm.vector.interleave2.nxv8i1(<vscale x 4 x i1> %val1, <vscale x 4 x i1> %val2)
+ call void @llvm.masked.store.nxv8i1.p0(<vscale x 8 x i1> %interleaved.value, ptr %p, i32 1, <vscale x 8 x i1> %interleaved.mask)
+ ret void
+}
+
+define void @foo_st2_nxv8i1(<vscale x 8 x i1> %mask, <vscale x 8 x i1> %val1, <vscale x 8 x i1> %val2, ptr %p) {
+; CHECK-LABEL: foo_st2_nxv8i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov z1.h, p2/z, #1 // =0x1
+; CHECK-NEXT: mov z0.h, p1/z, #1 // =0x1
+; CHECK-NEXT: st2h { z0.h, z1.h }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 16 x i1> @llvm.vector.interleave2.nxv16i1(<vscale x 8 x i1> %mask, <vscale x 8 x i1> %mask)
+ %interleaved.value = call <vscale x 16 x i1> @llvm.vector.interleave2.nxv16i1(<vscale x 8 x i1> %val1, <vscale x 8 x i1> %val2)
+ call void @llvm.masked.store.nxv16i1.p0(<vscale x 16 x i1> %interleaved.value, ptr %p, i32 1, <vscale x 16 x i1> %interleaved.mask)
+ ret void
+}
+
+define void @foo_st2_nxv16i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %val1, <vscale x 16 x i1> %val2, ptr %p) {
+; CHECK-LABEL: foo_st2_nxv16i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov z1.b, p2/z, #1 // =0x1
+; CHECK-NEXT: mov z0.b, p1/z, #1 // =0x1
+; CHECK-NEXT: st2b { z0.b, z1.b }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask)
+ %interleaved.value = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> %val1, <vscale x 16 x i1> %val2)
+ call void @llvm.masked.store.nxv32i1.p0(<vscale x 32 x i1> %interleaved.value, ptr %p, i32 1, <vscale x 32 x i1> %interleaved.mask)
+ ret void
+}
+
define void @foo_st2_nxv16i8(<vscale x 16 x i1> %mask, <vscale x 16 x i8> %val1, <vscale x 16 x i8> %val2, ptr %p) {
; CHECK-LABEL: foo_st2_nxv16i8:
; CHECK: // %bb.0:
@@ -68,6 +120,66 @@ define void @foo_st4_nxv16i8(<vscale x 16 x i1> %mask, <vscale x 16 x i8> %val1,
ret void
}
+define void @foo_st4_nxv2i1(<vscale x 2 x i1> %mask, <vscale x 2 x i1> %v, ptr %p) {
+; CHECK-LABEL: foo_st4_nxv2i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov z0.d, p1/z, #1 // =0x1
+; CHECK-NEXT: mov z1.d, z0.d
+; CHECK-NEXT: mov z2.d, z0.d
+; CHECK-NEXT: mov z3.d, z0.d
+; CHECK-NEXT: st4d { z0.d - z3.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 8 x i1> @llvm.vector.interleave4.nxv8i1(<vscale x 2 x i1> %mask, <vscale x 2 x i1> %mask, <vscale x 2 x i1> %mask, <vscale x 2 x i1> %mask)
+ %interleaved.value = call <vscale x 8 x i1> @llvm.vector.interleave4.nxv8i1(<vscale x 2 x i1> %v, <vscale x 2 x i1> %v, <vscale x 2 x i1> %v, <vscale x 2 x i1> %v)
+ call void @llvm.masked.store.nxv8i1.p0(<vscale x 8 x i1> %interleaved.value, ptr %p, i32 1, <vscale x 8 x i1> %interleaved.mask)
+ ret void
+}
+
+define void @foo_st4_nxv4i1(<vscale x 4 x i1> %mask, <vscale x 4 x i1> %v, ptr %p) {
+; CHECK-LABEL: foo_st4_nxv4i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov z0.s, p1/z, #1 // =0x1
+; CHECK-NEXT: mov z1.d, z0.d
+; CHECK-NEXT: mov z2.d, z0.d
+; CHECK-NEXT: mov z3.d, z0.d
+; CHECK-NEXT: st4w { z0.s - z3.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 16 x i1> @llvm.vector.interleave4.nxv16i1(<vscale x 4 x i1> %mask, <vscale x 4 x i1> %mask, <vscale x 4 x i1> %mask, <vscale x 4 x i1> %mask)
+ %interleaved.value = call <vscale x 16 x i1> @llvm.vector.interleave4.nxv16i1(<vscale x 4 x i1> %v, <vscale x 4 x i1> %v, <vscale x 4 x i1> %v, <vscale x 4 x i1> %v)
+ call void @llvm.masked.store.nxv16i1.p0(<vscale x 16 x i1> %interleaved.value, ptr %p, i32 1, <vscale x 16 x i1> %interleaved.mask)
+ ret void
+}
+
+define void @foo_st4_nxv8i1(<vscale x 8 x i1> %mask, <vscale x 8 x i1> %v, ptr %p) {
+; CHECK-LABEL: foo_st4_nxv8i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov z0.h, p1/z, #1 // =0x1
+; CHECK-NEXT: mov z1.d, z0.d
+; CHECK-NEXT: mov z2.d, z0.d
+; CHECK-NEXT: mov z3.d, z0.d
+; CHECK-NEXT: st4h { z0.h - z3.h }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 32 x i1> @llvm.vector.interleave4.nxv32i1(<vscale x 8 x i1> %mask, <vscale x 8 x i1> %mask, <vscale x 8 x i1> %mask, <vscale x 8 x i1> %mask)
+ %interleaved.value = call <vscale x 32 x i1> @llvm.vector.interleave4.nxv32i1(<vscale x 8 x i1> %v, <vscale x 8 x i1> %v, <vscale x 8 x i1> %v, <vscale x 8 x i1> %v)
+ call void @llvm.masked.store.nxv32i1.p0(<vscale x 32 x i1> %interleaved.value, ptr %p, i32 1, <vscale x 32 x i1> %interleaved.mask)
+ ret void
+}
+
+define void @foo_st4_nxv16i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %v, ptr %p) {
+; CHECK-LABEL: foo_st4_nxv16i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov z0.b, p1/z, #1 // =0x1
+; CHECK-NEXT: mov z1.d, z0.d
+; CHECK-NEXT: mov z2.d, z0.d
+; CHECK-NEXT: mov z3.d, z0.d
+; CHECK-NEXT: st4b { z0.b - z3.b }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 64 x i1> @llvm.vector.interleave4.nxv64i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask)
+ %interleaved.value = call <vscale x 64 x i1> @llvm.vector.interleave4.nxv64i1(<vscale x 16 x i1> %v, <vscale x 16 x i1> %v, <vscale x 16 x i1> %v, <vscale x 16 x i1> %v)
+ call void @llvm.masked.store.nxv64i1.p0(<vscale x 64 x i1> %interleaved.value, ptr %p, i32 1, <vscale x 64 x i1> %interleaved.mask)
+ ret void
+}
+
define void @foo_st4_nxv8i16(<vscale x 8 x i1> %mask, <vscale x 8 x i16> %val1, <vscale x 8 x i16> %val2, <vscale x 8 x i16> %val3, <vscale x 8 x i16> %val4, ptr %p) {
; CHECK-LABEL: foo_st4_nxv8i16:
; CHECK: // %bb.0:
>From edf74703729888284918f9d6b914f4158371aa5b Mon Sep 17 00:00:00 2001
From: Harry Ramsey <harry.ramsey at arm.com>
Date: Tue, 14 Jul 2026 22:48:50 +0000
Subject: [PATCH 4/4] [AArch64][ISel] Enable factor-3 scalable interleaved
memory costs
Allow scalable interleaved memory operations with factor 3 in the
AArch64 TTI cost model.
SelectionDAG can lower explicit scalable vector.interleave3 and
vector.deinterleave3 patterns for legal packed SVE vector types. For
vector.interleave3 values declared in seperate blocks must be
deinterleaved first before being used.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 87 +-
.../AArch64/AArch64TargetTransformInfo.cpp | 16 +-
.../scalable_masked_deinterleaved_loads.ll | 11 +
.../scalable_masked_interleaved_stores.ll | 14 +
.../AArch64/force-target-instruction-cost.ll | 58 +-
.../AArch64/sve-interleaved-accesses.ll | 40 +-
.../AArch64/sve-tail-folding-option.ll | 2457 +++++++++++++++--
7 files changed, 2349 insertions(+), 334 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index a76b428c0fde5..db4d066d2cd67 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -27540,6 +27540,38 @@ static SDValue convertFromSVEInterleavedMemContainer(SDValue Value, EVT VT,
DAG.getConstant(0, DL, Value.getValueType()), ISD::SETNE);
}
+static bool deinterleaveInterleavedValueForSVESt3(
+ SDValue WideValue, SDLoc DL, SelectionDAG &DAG,
+ SmallVectorImpl<SDValue> &Ops) {
+ constexpr unsigned Factor = 3;
+ EVT WideVT = WideValue.getValueType();
+ if (!WideVT.isScalableVector())
+ return false;
+
+ ElementCount WideEC = WideVT.getVectorElementCount();
+ if (!WideEC.isKnownMultipleOf(Factor))
+ return false;
+
+ EVT SubVecTy = EVT::getVectorVT(*DAG.getContext(),
+ WideVT.getVectorElementType(),
+ WideEC.divideCoefficientBy(Factor));
+ if (!isSupportedSVEInterleavedMemSubvectorType(SubVecTy, DAG))
+ return false;
+
+ SmallVector<SDValue, 3> SubVecs;
+ for (unsigned I = 0; I != Factor; ++I)
+ SubVecs.push_back(DAG.getNode(
+ ISD::EXTRACT_SUBVECTOR, DL, SubVecTy, WideValue,
+ DAG.getVectorIdxConstant(I * SubVecTy.getVectorMinNumElements(), DL)));
+
+ SmallVector<EVT, 3> ResultVTs(Factor, SubVecTy);
+ SDValue Deinterleaved = DAG.getNode(
+ ISD::VECTOR_DEINTERLEAVE, DL, DAG.getVTList(ResultVTs), SubVecs);
+ for (unsigned I = 0; I != Factor; ++I)
+ Ops.push_back(Deinterleaved.getValue(I));
+ return true;
+}
+
static SDValue performInterleavedMaskedStoreCombine(
SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG) {
if (!DCI.isBeforeLegalize())
@@ -27564,11 +27596,16 @@ static SDValue performInterleavedMaskedStoreCombine(
return SDValue();
} else if (!isSplatVectorInterleaveOps(DAG, DL, WideValue,
ValueInterleaveOps)) {
- return SDValue();
+ // A vector.interleave3 defined in another basic block is unknown here
+ // because SelectionDAGs are local to a basic block. Deinterleave the wide
+ // value back into legal operands so it can be used by st3.
+ if (!deinterleaveInterleavedValueForSVESt3(WideValue, DL, DAG,
+ ValueInterleaveOps))
+ return SDValue();
}
unsigned NumParts = ValueInterleaveOps.size();
- if (NumParts != 2 && NumParts != 4)
+ if (NumParts != 2 && NumParts != 3 && NumParts != 4)
return SDValue();
// At the moment we're unlikely to see a fixed-width vector interleave as
@@ -27585,8 +27622,18 @@ static SDValue performInterleavedMaskedStoreCombine(
for (SDValue &Value : ValueInterleaveOps)
Value = convertToSVEInterleavedMemContainer(Value, DL, DAG);
- const Intrinsic::ID IID =
- NumParts == 2 ? Intrinsic::aarch64_sve_st2 : Intrinsic::aarch64_sve_st4;
+ Intrinsic::ID IID;
+ switch (NumParts) {
+ case 2:
+ IID = Intrinsic::aarch64_sve_st2;
+ break;
+ case 3:
+ IID = Intrinsic::aarch64_sve_st3;
+ break;
+ case 4:
+ IID = Intrinsic::aarch64_sve_st4;
+ break;
+ }
SmallVector<SDValue, 8> NewStOps;
NewStOps.append({MST->getChain(), DAG.getConstant(IID, DL, MVT::i32)});
NewStOps.append(ValueInterleaveOps);
@@ -30474,7 +30521,7 @@ static SDValue performVectorDeinterleaveCombine(
return SDValue();
unsigned NumParts = N->getNumOperands();
- if (NumParts != 2 && NumParts != 4)
+ if (NumParts != 2 && NumParts != 3 && NumParts != 4)
return SDValue();
EVT SubVecTy = N->getValueType(0);
@@ -30520,18 +30567,36 @@ static SDValue performVectorDeinterleaveCombine(
EVT LoadVT = getSVEInterleavedMemContainerType(SubVecTy);
- const Intrinsic::ID IID = NumParts == 2 ? Intrinsic::aarch64_sve_ld2_sret
- : Intrinsic::aarch64_sve_ld4_sret;
+ Intrinsic::ID IID;
+ switch (NumParts) {
+ case 2:
+ IID = Intrinsic::aarch64_sve_ld2_sret;
+ break;
+ case 3:
+ IID = Intrinsic::aarch64_sve_ld3_sret;
+ break;
+ case 4:
+ IID = Intrinsic::aarch64_sve_ld4_sret;
+ break;
+ }
SDValue NewLdOps[] = {MaskedLoad->getChain(),
DAG.getConstant(IID, DL, MVT::i32), NarrowMask,
MaskedLoad->getBasePtr()};
SDValue Res;
- if (NumParts == 2)
- Res = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, {LoadVT, LoadVT, MVT::Other},
- NewLdOps);
- else
+ switch (NumParts) {
+ case 2:
+ Res = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL,
+ {LoadVT, LoadVT, MVT::Other}, NewLdOps);
+ break;
+ case 3:
+ Res = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL,
+ {LoadVT, LoadVT, LoadVT, MVT::Other}, NewLdOps);
+ break;
+ case 4:
Res = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL,
{LoadVT, LoadVT, LoadVT, LoadVT, MVT::Other}, NewLdOps);
+ break;
+ }
// We can now generate a structured load!
SmallVector<SDValue, 4> ResOps(NumParts);
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 9e1a0960617a5..bc2752d07deb8 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -5414,12 +5414,16 @@ InstructionCost AArch64TTIImpl::getInterleavedMemoryOpCost(
if (VecTy->isScalableTy() && !ST->hasSVE())
return InstructionCost::getInvalid();
- // Scalable VFs will emit vector.[de]interleave intrinsics, and currently we
- // only have lowering for power-of-2 factors.
- // TODO: Add lowering for vector.[de]interleave3 intrinsics and support in
- // InterleavedAccessPass for ld3/st3
- if (VecTy->isScalableTy() && !isPowerOf2_32(Factor))
- return InstructionCost::getInvalid();
+ // Scalable VFs emit vector.[de]interleave intrinsics, for which the target
+ // supports factors up to the maximum supported interleave factor.
+ if (VecTy->isScalableTy()) {
+ if (Factor > TLI->getMaxSupportedInterleaveFactor())
+ return InstructionCost::getInvalid();
+
+ if (Factor == 3 &&
+ DL.getTypeSizeInBits(VecTy).getKnownMinValue() < Factor * 128)
+ return InstructionCost::getInvalid();
+ }
// Vectorization for masked interleaved accesses is only enabled for scalable
// VF.
diff --git a/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll b/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll
index ca5bae6791cf6..f6630b645861b 100644
--- a/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll
+++ b/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll
@@ -101,6 +101,17 @@ define { <vscale x 2 x double>, <vscale x 2 x double> } @foo_ld2_nxv2f64(<vscale
ret { <vscale x 2 x double>, <vscale x 2 x double> } %deinterleaved.vec
}
+define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @foo_ld3_nxv16i8(<vscale x 16 x i1> %mask, ptr %p) {
+; CHECK-LABEL: foo_ld3_nxv16i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ld3b { z0.b - z2.b }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 48 x i1> @llvm.vector.interleave3.nxv48i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask)
+ %wide.masked.vec = call <vscale x 48 x i8> @llvm.masked.load.nxv48i8(ptr %p, i32 1, <vscale x 48 x i1> %interleaved.mask, <vscale x 48 x i8> poison)
+ %deinterleaved.vec = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.vector.deinterleave3.nxv48i8(<vscale x 48 x i8> %wide.masked.vec)
+ ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %deinterleaved.vec
+}
+
define { <vscale x 2 x i1>, <vscale x 2 x i1>, <vscale x 2 x i1>, <vscale x 2 x i1> } @foo_ld4_nxv2i1(<vscale x 2 x i1> %mask, ptr %p) {
; CHECK-LABEL: foo_ld4_nxv2i1:
; CHECK: // %bb.0:
diff --git a/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll b/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
index ed3f2f772787a..cfdab2485f3e6 100644
--- a/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
+++ b/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
@@ -105,6 +105,20 @@ define void @foo_st2_nxv2i64(<vscale x 2 x i1> %mask, <vscale x 2 x i64> %val1,
ret void
}
+define void @foo_st3_nxv16i8(<vscale x 16 x i1> %mask, <vscale x 16 x i8> %val1, <vscale x 16 x i8> %val2, <vscale x 16 x i8> %val3, ptr %p) {
+; CHECK-LABEL: foo_st3_nxv16i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-NEXT: st3b { z0.b - z2.b }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 48 x i1> @llvm.vector.interleave3.nxv48i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask)
+ %interleaved.value = call <vscale x 48 x i8> @llvm.vector.interleave3.nxv48i8(<vscale x 16 x i8> %val1, <vscale x 16 x i8> %val2, <vscale x 16 x i8> %val3)
+ call void @llvm.masked.store.nxv48i8.p0(<vscale x 48 x i8> %interleaved.value, ptr %p, i32 1, <vscale x 48 x i1> %interleaved.mask)
+ ret void
+}
+
define void @foo_st4_nxv16i8(<vscale x 16 x i1> %mask, <vscale x 16 x i8> %val1, <vscale x 16 x i8> %val2, <vscale x 16 x i8> %val3, <vscale x 16 x i8> %val4, ptr %p) {
; CHECK-LABEL: foo_st4_nxv16i8:
; CHECK: // %bb.0:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
index 07fc18e46c4fe..8fa200573e2b9 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
@@ -439,27 +439,40 @@ define void @interleave_group(ptr %dst) #1 {
; COST1-NEXT: [[ITER_CHECK:.*:]]
; COST1-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; COST1: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; COST1-NEXT: br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; COST1-NEXT: [[TMP24:%.*]] = call i64 @llvm.vscale.i64()
+; COST1-NEXT: [[TMP25:%.*]] = shl nuw i64 [[TMP24]], 5
+; COST1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 101, [[TMP25]]
+; COST1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; COST1: [[VECTOR_PH]]:
+; COST1-NEXT: [[TMP26:%.*]] = shl nuw i64 [[TMP24]], 4
+; COST1-NEXT: [[TMP27:%.*]] = shl nuw i64 [[TMP26]], 1
+; COST1-NEXT: [[N_MOD_VF:%.*]] = urem i64 101, [[TMP27]]
+; COST1-NEXT: [[N_VEC:%.*]] = sub i64 101, [[N_MOD_VF]]
; COST1-NEXT: br label %[[VECTOR_BODY:.*]]
; COST1: [[VECTOR_BODY]]:
; COST1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COST1-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 16
+; COST1-NEXT: [[TMP28:%.*]] = add i64 [[TMP26]], 0
+; COST1-NEXT: [[TMP5:%.*]] = mul i64 [[TMP28]], 1
+; COST1-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], [[TMP5]]
; COST1-NEXT: [[TMP1:%.*]] = mul i64 [[INDEX]], 3
; COST1-NEXT: [[TMP2:%.*]] = mul i64 [[TMP0]], 3
; COST1-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP1]]
; COST1-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP2]]
-; COST1-NEXT: store <48 x i8> zeroinitializer, ptr [[TMP3]], align 1
-; COST1-NEXT: store <48 x i8> zeroinitializer, ptr [[TMP4]], align 1
-; COST1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
-; COST1-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
-; COST1-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; COST1-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 48 x i8> @llvm.vector.interleave3.nxv48i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer)
+; COST1-NEXT: store <vscale x 48 x i8> [[INTERLEAVED_VEC]], ptr [[TMP3]], align 1
+; COST1-NEXT: [[INTERLEAVED_VEC1:%.*]] = call <vscale x 48 x i8> @llvm.vector.interleave3.nxv48i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer)
+; COST1-NEXT: store <vscale x 48 x i8> [[INTERLEAVED_VEC1]], ptr [[TMP4]], align 1
+; COST1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP27]]
+; COST1-NEXT: [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COST1-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
; COST1: [[MIDDLE_BLOCK]]:
-; COST1-NEXT: br i1 false, [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; COST1-NEXT: [[CMP_N:%.*]] = icmp eq i64 101, [[N_VEC]]
+; COST1-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; COST1: [[VEC_EPILOG_ITER_CHECK]]:
-; COST1-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
+; COST1-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
+; COST1-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
; COST1: [[VEC_EPILOG_PH]]:
-; COST1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 96, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; COST1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; COST1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[BC_RESUME_VAL]], i64 0
; COST1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COST1-NEXT: [[INDUCTION:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3>
@@ -509,23 +522,32 @@ define void @interleave_group(ptr %dst) #1 {
; COST10-NEXT: [[ITER_CHECK:.*:]]
; COST10-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; COST10: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; COST10-NEXT: br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; COST10-NEXT: [[TMP21:%.*]] = call i64 @llvm.vscale.i64()
+; COST10-NEXT: [[TMP22:%.*]] = shl nuw i64 [[TMP21]], 4
+; COST10-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 101, [[TMP22]]
+; COST10-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; COST10: [[VECTOR_PH]]:
+; COST10-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP21]], 4
+; COST10-NEXT: [[N_MOD_VF:%.*]] = urem i64 101, [[TMP2]]
+; COST10-NEXT: [[N_VEC:%.*]] = sub i64 101, [[N_MOD_VF]]
; COST10-NEXT: br label %[[VECTOR_BODY:.*]]
; COST10: [[VECTOR_BODY]]:
; COST10-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COST10-NEXT: [[TMP0:%.*]] = mul i64 [[INDEX]], 3
; COST10-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP0]]
-; COST10-NEXT: store <48 x i8> zeroinitializer, ptr [[TMP1]], align 1
-; COST10-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; COST10-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
-; COST10-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; COST10-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 48 x i8> @llvm.vector.interleave3.nxv48i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer)
+; COST10-NEXT: store <vscale x 48 x i8> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 1
+; COST10-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; COST10-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COST10-NEXT: br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
; COST10: [[MIDDLE_BLOCK]]:
-; COST10-NEXT: br i1 false, [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; COST10-NEXT: [[CMP_N:%.*]] = icmp eq i64 101, [[N_VEC]]
+; COST10-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; COST10: [[VEC_EPILOG_ITER_CHECK]]:
-; COST10-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
+; COST10-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
+; COST10-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
; COST10: [[VEC_EPILOG_PH]]:
-; COST10-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 96, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; COST10-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; COST10-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[BC_RESUME_VAL]], i64 0
; COST10-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COST10-NEXT: [[INDUCTION:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3>
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
index 6d65b4e597075..a48218d0efcdc 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
@@ -1320,8 +1320,6 @@ end:
; dst[i].z = a[i].z << b[i].z;
; }
;
-; TODO: Support scalable interleave groups once we can also codegen
-; @llvm.[de]interleave3
%struct.xyz = type { i32, i32, i32 }
define void @interleave_deinterleave_factor3(ptr writeonly noalias %dst, ptr readonly %a, ptr readonly %b) {
@@ -1335,36 +1333,28 @@ define void @interleave_deinterleave_factor3(ptr writeonly noalias %dst, ptr rea
; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP2]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP2]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP3]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds [[STRUCT_XYZ:%.*]], ptr [[A:%.*]], <vscale x 4 x i64> [[VEC_IND]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[WIDE_GEP]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
-; CHECK-NEXT: [[WIDE_GEP1:%.*]] = getelementptr inbounds [[STRUCT_XYZ]], ptr [[B:%.*]], <vscale x 4 x i64> [[VEC_IND]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[WIDE_GEP1]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_XYZ:%.*]], ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 12 x i32>, ptr [[TMP3]], align 4
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave3.nxv12i32(<vscale x 12 x i32> [[WIDE_VEC]])
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
+; CHECK-NEXT: [[WIDE_MASKED_GATHER5:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1
+; CHECK-NEXT: [[WIDE_MASKED_GATHER10:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 2
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [[STRUCT_XYZ]], ptr [[B:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_VEC1:%.*]] = load <vscale x 12 x i32>, ptr [[TMP8]], align 4
+; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave3.nxv12i32(<vscale x 12 x i32> [[WIDE_VEC1]])
+; CHECK-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 0
+; CHECK-NEXT: [[WIDE_MASKED_GATHER7:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 1
+; CHECK-NEXT: [[WIDE_MASKED_GATHER12:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 2
; CHECK-NEXT: [[TMP4:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_GATHER2]], [[WIDE_MASKED_GATHER]]
-; CHECK-NEXT: [[WIDE_GEP3:%.*]] = getelementptr inbounds [[STRUCT_XYZ]], ptr [[DST:%.*]], <vscale x 4 x i64> [[VEC_IND]]
-; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP4]], <vscale x 4 x ptr> align 4 [[WIDE_GEP3]], <vscale x 4 x i1> splat (i1 true))
-; CHECK-NEXT: [[WIDE_GEP4:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[WIDE_GEP]], i64 4
-; CHECK-NEXT: [[WIDE_MASKED_GATHER5:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[WIDE_GEP4]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
-; CHECK-NEXT: [[WIDE_GEP6:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[WIDE_GEP1]], i64 4
-; CHECK-NEXT: [[WIDE_MASKED_GATHER7:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[WIDE_GEP6]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds [[STRUCT_XYZ]], ptr [[DST:%.*]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP5:%.*]] = sub nsw <vscale x 4 x i32> [[WIDE_MASKED_GATHER5]], [[WIDE_MASKED_GATHER7]]
-; CHECK-NEXT: [[WIDE_GEP8:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[WIDE_GEP3]], i64 4
-; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP5]], <vscale x 4 x ptr> align 4 [[WIDE_GEP8]], <vscale x 4 x i1> splat (i1 true))
-; CHECK-NEXT: [[WIDE_GEP9:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[WIDE_GEP]], i64 8
-; CHECK-NEXT: [[WIDE_MASKED_GATHER10:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[WIDE_GEP9]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
-; CHECK-NEXT: [[WIDE_GEP11:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[WIDE_GEP1]], i64 8
-; CHECK-NEXT: [[WIDE_MASKED_GATHER12:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[WIDE_GEP11]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
; CHECK-NEXT: [[TMP6:%.*]] = shl <vscale x 4 x i32> [[WIDE_MASKED_GATHER10]], [[WIDE_MASKED_GATHER12]]
-; CHECK-NEXT: [[WIDE_GEP13:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[WIDE_GEP3]], i64 8
-; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP6]], <vscale x 4 x ptr> align 4 [[WIDE_GEP13]], <vscale x 4 x i1> splat (i1 true))
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 12 x i32> @llvm.vector.interleave3.nxv12i32(<vscale x 4 x i32> [[TMP4]], <vscale x 4 x i32> [[TMP5]], <vscale x 4 x i32> [[TMP6]])
+; CHECK-NEXT: store <vscale x 12 x i32> [[INTERLEAVED_VEC]], ptr [[TMP12]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
; CHECK: middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-option.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-option.ll
index fb17b9a80d09b..994cd150214aa 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-option.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-option.ll
@@ -1,9 +1,10 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -sve-tail-folding=disabled -S | FileCheck %s -check-prefix=CHECK-NOTF
; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -sve-tail-folding=default -S | FileCheck %s -check-prefix=CHECK-NOTF
; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -S | FileCheck %s -check-prefix=CHECK-NOTF
-; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -sve-tail-folding=all -S | FileCheck %s -check-prefix=CHECK-TF
+; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -sve-tail-folding=all -S | FileCheck %s -check-prefix=CHECK-TF-ALL
; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -sve-tail-folding=simple+reductions+recurrences+reverse -S | FileCheck %s -check-prefix=CHECK-TF
-; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -S -mcpu=neoverse-v1 -sve-tail-folding=default+reductions+recurrences+reverse | FileCheck %s -check-prefix=CHECK-TF
+; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -S -mcpu=neoverse-v1 -sve-tail-folding=default+reductions+recurrences+reverse | FileCheck %s -check-prefix=CHECK-TF-DEFAULT
; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -sve-tail-folding=all+noreductions -S | FileCheck %s -check-prefix=CHECK-TF-NORED
; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -sve-tail-folding=all+norecurrences -S | FileCheck %s -check-prefix=CHECK-TF-NOREC
; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -sve-tail-folding=all+noreverse -S | FileCheck %s -check-prefix=CHECK-TF-NOREV
@@ -15,61 +16,277 @@
target triple = "aarch64-unknown-linux-gnu"
define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 {
-; CHECK-NOTF-LABEL: @simple_memset(
-; CHECK-NOTF: vector.ph:
-; CHECK-NOTF: %[[INSERT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %val, i64 0
-; CHECK-NOTF: %[[SPLAT:.*]] = shufflevector <vscale x 4 x i32> %[[INSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NOTF: vector.body:
-; CHECK-NOTF-NOT: %{{.*}} = phi <vscale x 4 x i1>
-; CHECK-NOTF: store <vscale x 4 x i32> %[[SPLAT]], ptr
-
-; CHECK-TF-NORED-LABEL: @simple_memset(
-; CHECK-TF-NORED: vector.ph:
-; CHECK-TF-NORED: %[[INSERT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %val, i64 0
-; CHECK-TF-NORED: %[[SPLAT:.*]] = shufflevector <vscale x 4 x i32> %[[INSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-TF-NORED: vector.body:
-; CHECK-TF-NORED: %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
-; CHECK-TF-NORED: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> %[[SPLAT]], {{.*}} %[[ACTIVE_LANE_MASK]]
-
-; CHECK-TF-NOREC-LABEL: @simple_memset(
-; CHECK-TF-NOREC: vector.ph:
-; CHECK-TF-NOREC: %[[INSERT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %val, i64 0
-; CHECK-TF-NOREC: %[[SPLAT:.*]] = shufflevector <vscale x 4 x i32> %[[INSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-TF-NOREC: vector.body:
-; CHECK-TF-NOREC: %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
-; CHECK-TF-NOREC: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> %[[SPLAT]], {{.*}} %[[ACTIVE_LANE_MASK]]
-
-; CHECK-TF-NOREV-LABEL: @simple_memset(
-; CHECK-TF-NOREV: vector.ph:
-; CHECK-TF-NOREV: %[[INSERT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %val, i64 0
-; CHECK-TF-NOREV: %[[SPLAT:.*]] = shufflevector <vscale x 4 x i32> %[[INSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-TF-NOREV: vector.body:
-; CHECK-TF-NOREV: %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
-; CHECK-TF-NOREV: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> %[[SPLAT]], {{.*}} %[[ACTIVE_LANE_MASK]]
-
-; CHECK-TF-LABEL: @simple_memset(
-; CHECK-TF: vector.ph:
-; CHECK-TF: %[[INSERT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %val, i64 0
-; CHECK-TF: %[[SPLAT:.*]] = shufflevector <vscale x 4 x i32> %[[INSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-TF: vector.body:
-; CHECK-TF: %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
-; CHECK-TF: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> %[[SPLAT]], {{.*}} %[[ACTIVE_LANE_MASK]]
-
-; CHECK-TF-ONLYRED-LABEL: @simple_memset(
-; CHECK-TF-ONLYRED: vector.ph:
-; CHECK-TF-ONLYRED: %[[INSERT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %val, i64 0
-; CHECK-TF-ONLYRED: %[[SPLAT:.*]] = shufflevector <vscale x 4 x i32> %[[INSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-TF-ONLYRED: vector.body:
-; CHECK-TF-ONLYRED-NOT: %{{.*}} = phi <vscale x 4 x i1>
-; CHECK-TF-ONLYRED: store <vscale x 4 x i32> %[[SPLAT]], ptr
-
-; CHECK-NEOVERSE-V1-LABEL: @simple_memset(
-; CHECK-NEOVERSE-V1: vector.ph:
-; CHECK-NEOVERSE-V1: %[[INSERT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %val, i64 0
-; CHECK-NEOVERSE-V1: %[[SPLAT:.*]] = shufflevector <vscale x 4 x i32> %[[INSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEOVERSE-V1: vector.body:
-; CHECK-NEOVERSE-V1: %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
-; CHECK-NEOVERSE-V1: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> %[[SPLAT]], {{.*}} %[[ACTIVE_LANE_MASK]]
+; CHECK-NOTF-LABEL: define void @simple_memset(
+; CHECK-NOTF-SAME: i32 [[VAL:%.*]], ptr [[PTR:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NOTF-NEXT: [[ENTRY:.*]]:
+; CHECK-NOTF-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; CHECK-NOTF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NOTF-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NOTF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[UMAX]], [[TMP1]]
+; CHECK-NOTF-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NOTF: [[VECTOR_PH]]:
+; CHECK-NOTF-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NOTF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[UMAX]], [[TMP2]]
+; CHECK-NOTF-NEXT: [[N_VEC:%.*]] = sub i64 [[UMAX]], [[N_MOD_VF]]
+; CHECK-NOTF-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL]], i64 0
+; CHECK-NOTF-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NOTF-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-NOTF: [[VECTOR_BODY]]:
+; CHECK-NOTF-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NOTF-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX1]]
+; CHECK-NOTF-NEXT: store <vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr [[TMP3]], align 4
+; CHECK-NOTF-NEXT: [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], [[TMP2]]
+; CHECK-NOTF-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT2]], [[N_VEC]]
+; CHECK-NOTF-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NOTF: [[MIDDLE_BLOCK]]:
+; CHECK-NOTF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[UMAX]], [[N_VEC]]
+; CHECK-NOTF-NEXT: br i1 [[CMP_N]], label %[[WHILE_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NOTF: [[SCALAR_PH]]:
+; CHECK-NOTF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NOTF-NEXT: br label %[[WHILE_BODY:.*]]
+; CHECK-NOTF: [[WHILE_BODY]]:
+; CHECK-NOTF-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], %[[WHILE_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NOTF-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX]]
+; CHECK-NOTF-NEXT: store i32 [[VAL]], ptr [[GEP]], align 4
+; CHECK-NOTF-NEXT: [[INDEX_NEXT]] = add nsw i64 [[INDEX]], 1
+; CHECK-NOTF-NEXT: [[CMP10:%.*]] = icmp ult i64 [[INDEX_NEXT]], [[N]]
+; CHECK-NOTF-NEXT: br i1 [[CMP10]], label %[[WHILE_BODY]], label %[[WHILE_END_LOOPEXIT]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NOTF: [[WHILE_END_LOOPEXIT]]:
+; CHECK-NOTF-NEXT: ret void
+;
+; CHECK-TF-ALL-LABEL: define void @simple_memset(
+; CHECK-TF-ALL-SAME: i32 [[VAL:%.*]], ptr [[PTR:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-TF-ALL-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-ALL-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; CHECK-TF-ALL-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF-ALL: [[VECTOR_PH]]:
+; CHECK-TF-ALL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-ALL-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-ALL-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]])
+; CHECK-TF-ALL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL]], i64 0
+; CHECK-TF-ALL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-TF-ALL-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-ALL: [[VECTOR_BODY]]:
+; CHECK-TF-ALL-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ALL-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ALL-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX1]]
+; CHECK-TF-ALL-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP2]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-ALL-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP1]]
+; CHECK-TF-ALL-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT2]], i64 [[UMAX]])
+; CHECK-TF-ALL-NEXT: [[TMP3:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-ALL-NEXT: [[TMP4:%.*]] = xor i1 [[TMP3]], true
+; CHECK-TF-ALL-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-TF-ALL: [[MIDDLE_BLOCK]]:
+; CHECK-TF-ALL-NEXT: br label %[[WHILE_END_LOOPEXIT:.*]]
+; CHECK-TF-ALL: [[WHILE_END_LOOPEXIT]]:
+; CHECK-TF-ALL-NEXT: ret void
+;
+; CHECK-TF-LABEL: define void @simple_memset(
+; CHECK-TF-SAME: i32 [[VAL:%.*]], ptr [[PTR:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-TF-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; CHECK-TF-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF: [[VECTOR_PH]]:
+; CHECK-TF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]])
+; CHECK-TF-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL]], i64 0
+; CHECK-TF-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-TF-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF: [[VECTOR_BODY]]:
+; CHECK-TF-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX1]]
+; CHECK-TF-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP2]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP1]]
+; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT2]], i64 [[UMAX]])
+; CHECK-TF-NEXT: [[TMP3:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NEXT: [[TMP4:%.*]] = xor i1 [[TMP3]], true
+; CHECK-TF-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-TF: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NEXT: br label %[[WHILE_END_LOOPEXIT:.*]]
+; CHECK-TF: [[WHILE_END_LOOPEXIT]]:
+; CHECK-TF-NEXT: ret void
+;
+; CHECK-TF-DEFAULT-LABEL: define void @simple_memset(
+; CHECK-TF-DEFAULT-SAME: i32 [[VAL:%.*]], ptr [[PTR:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-TF-DEFAULT-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-DEFAULT-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; CHECK-TF-DEFAULT-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF-DEFAULT: [[VECTOR_PH]]:
+; CHECK-TF-DEFAULT-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-DEFAULT-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-DEFAULT-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]])
+; CHECK-TF-DEFAULT-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL]], i64 0
+; CHECK-TF-DEFAULT-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-TF-DEFAULT-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-DEFAULT: [[VECTOR_BODY]]:
+; CHECK-TF-DEFAULT-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-DEFAULT-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-DEFAULT-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX1]]
+; CHECK-TF-DEFAULT-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP2]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-DEFAULT-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP1]]
+; CHECK-TF-DEFAULT-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT2]], i64 [[UMAX]])
+; CHECK-TF-DEFAULT-NEXT: [[TMP3:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-DEFAULT-NEXT: [[TMP4:%.*]] = xor i1 [[TMP3]], true
+; CHECK-TF-DEFAULT-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-TF-DEFAULT: [[MIDDLE_BLOCK]]:
+; CHECK-TF-DEFAULT-NEXT: br label %[[WHILE_END_LOOPEXIT:.*]]
+; CHECK-TF-DEFAULT: [[WHILE_END_LOOPEXIT]]:
+; CHECK-TF-DEFAULT-NEXT: ret void
+;
+; CHECK-TF-NORED-LABEL: define void @simple_memset(
+; CHECK-TF-NORED-SAME: i32 [[VAL:%.*]], ptr [[PTR:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-TF-NORED-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-NORED-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; CHECK-TF-NORED-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF-NORED: [[VECTOR_PH]]:
+; CHECK-TF-NORED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NORED-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NORED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]])
+; CHECK-TF-NORED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL]], i64 0
+; CHECK-TF-NORED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-TF-NORED-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-NORED: [[VECTOR_BODY]]:
+; CHECK-TF-NORED-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NORED-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NORED-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX1]]
+; CHECK-TF-NORED-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP2]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NORED-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP1]]
+; CHECK-TF-NORED-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT2]], i64 [[UMAX]])
+; CHECK-TF-NORED-NEXT: [[TMP3:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NORED-NEXT: [[TMP4:%.*]] = xor i1 [[TMP3]], true
+; CHECK-TF-NORED-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-TF-NORED: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NORED-NEXT: br label %[[WHILE_END_LOOPEXIT:.*]]
+; CHECK-TF-NORED: [[WHILE_END_LOOPEXIT]]:
+; CHECK-TF-NORED-NEXT: ret void
+;
+; CHECK-TF-NOREC-LABEL: define void @simple_memset(
+; CHECK-TF-NOREC-SAME: i32 [[VAL:%.*]], ptr [[PTR:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-TF-NOREC-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-NOREC-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; CHECK-TF-NOREC-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF-NOREC: [[VECTOR_PH]]:
+; CHECK-TF-NOREC-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NOREC-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NOREC-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]])
+; CHECK-TF-NOREC-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL]], i64 0
+; CHECK-TF-NOREC-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-TF-NOREC-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-NOREC: [[VECTOR_BODY]]:
+; CHECK-TF-NOREC-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREC-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREC-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX1]]
+; CHECK-TF-NOREC-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP2]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NOREC-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP1]]
+; CHECK-TF-NOREC-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT2]], i64 [[UMAX]])
+; CHECK-TF-NOREC-NEXT: [[TMP3:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NOREC-NEXT: [[TMP4:%.*]] = xor i1 [[TMP3]], true
+; CHECK-TF-NOREC-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-TF-NOREC: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NOREC-NEXT: br label %[[WHILE_END_LOOPEXIT:.*]]
+; CHECK-TF-NOREC: [[WHILE_END_LOOPEXIT]]:
+; CHECK-TF-NOREC-NEXT: ret void
+;
+; CHECK-TF-NOREV-LABEL: define void @simple_memset(
+; CHECK-TF-NOREV-SAME: i32 [[VAL:%.*]], ptr [[PTR:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-TF-NOREV-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-NOREV-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; CHECK-TF-NOREV-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF-NOREV: [[VECTOR_PH]]:
+; CHECK-TF-NOREV-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NOREV-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NOREV-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]])
+; CHECK-TF-NOREV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL]], i64 0
+; CHECK-TF-NOREV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-TF-NOREV-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-NOREV: [[VECTOR_BODY]]:
+; CHECK-TF-NOREV-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREV-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREV-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX1]]
+; CHECK-TF-NOREV-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP2]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NOREV-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP1]]
+; CHECK-TF-NOREV-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT2]], i64 [[UMAX]])
+; CHECK-TF-NOREV-NEXT: [[TMP3:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NOREV-NEXT: [[TMP4:%.*]] = xor i1 [[TMP3]], true
+; CHECK-TF-NOREV-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-TF-NOREV: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NOREV-NEXT: br label %[[WHILE_END_LOOPEXIT:.*]]
+; CHECK-TF-NOREV: [[WHILE_END_LOOPEXIT]]:
+; CHECK-TF-NOREV-NEXT: ret void
+;
+; CHECK-TF-ONLYRED-LABEL: define void @simple_memset(
+; CHECK-TF-ONLYRED-SAME: i32 [[VAL:%.*]], ptr [[PTR:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-TF-ONLYRED-NEXT: [[ENTRY:.*]]:
+; CHECK-TF-ONLYRED-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; CHECK-TF-ONLYRED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-ONLYRED-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-ONLYRED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[UMAX]], [[TMP1]]
+; CHECK-TF-ONLYRED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF-ONLYRED: [[VECTOR_PH]]:
+; CHECK-TF-ONLYRED-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-ONLYRED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[UMAX]], [[TMP2]]
+; CHECK-TF-ONLYRED-NEXT: [[N_VEC:%.*]] = sub i64 [[UMAX]], [[N_MOD_VF]]
+; CHECK-TF-ONLYRED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL]], i64 0
+; CHECK-TF-ONLYRED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-TF-ONLYRED-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-ONLYRED: [[VECTOR_BODY]]:
+; CHECK-TF-ONLYRED-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ONLYRED-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX1]]
+; CHECK-TF-ONLYRED-NEXT: store <vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr [[TMP3]], align 4
+; CHECK-TF-ONLYRED-NEXT: [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], [[TMP2]]
+; CHECK-TF-ONLYRED-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT2]], [[N_VEC]]
+; CHECK-TF-ONLYRED-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-TF-ONLYRED: [[MIDDLE_BLOCK]]:
+; CHECK-TF-ONLYRED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[UMAX]], [[N_VEC]]
+; CHECK-TF-ONLYRED-NEXT: br i1 [[CMP_N]], label %[[WHILE_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-TF-ONLYRED: [[SCALAR_PH]]:
+; CHECK-TF-ONLYRED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-TF-ONLYRED-NEXT: br label %[[WHILE_BODY:.*]]
+; CHECK-TF-ONLYRED: [[WHILE_BODY]]:
+; CHECK-TF-ONLYRED-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], %[[WHILE_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-TF-ONLYRED-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX]]
+; CHECK-TF-ONLYRED-NEXT: store i32 [[VAL]], ptr [[GEP]], align 4
+; CHECK-TF-ONLYRED-NEXT: [[INDEX_NEXT]] = add nsw i64 [[INDEX]], 1
+; CHECK-TF-ONLYRED-NEXT: [[CMP10:%.*]] = icmp ult i64 [[INDEX_NEXT]], [[N]]
+; CHECK-TF-ONLYRED-NEXT: br i1 [[CMP10]], label %[[WHILE_BODY]], label %[[WHILE_END_LOOPEXIT]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-TF-ONLYRED: [[WHILE_END_LOOPEXIT]]:
+; CHECK-TF-ONLYRED-NEXT: ret void
+;
+; CHECK-NEOVERSE-V1-LABEL: define void @simple_memset(
+; CHECK-NEOVERSE-V1-SAME: i32 [[VAL:%.*]], ptr [[PTR:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEOVERSE-V1-NEXT: [[ENTRY:.*:]]
+; CHECK-NEOVERSE-V1-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; CHECK-NEOVERSE-V1-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-NEOVERSE-V1: [[VECTOR_PH]]:
+; CHECK-NEOVERSE-V1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEOVERSE-V1-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEOVERSE-V1-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]])
+; CHECK-NEOVERSE-V1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL]], i64 0
+; CHECK-NEOVERSE-V1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEOVERSE-V1-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-NEOVERSE-V1: [[VECTOR_BODY]]:
+; CHECK-NEOVERSE-V1-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEOVERSE-V1-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX1]]
+; CHECK-NEOVERSE-V1-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP2]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-NEOVERSE-V1-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP1]]
+; CHECK-NEOVERSE-V1-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT2]], i64 [[UMAX]])
+; CHECK-NEOVERSE-V1-NEXT: [[TMP3:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEOVERSE-V1-NEXT: [[TMP4:%.*]] = xor i1 [[TMP3]], true
+; CHECK-NEOVERSE-V1-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEOVERSE-V1: [[MIDDLE_BLOCK]]:
+; CHECK-NEOVERSE-V1-NEXT: br label %[[WHILE_END_LOOPEXIT:.*]]
+; CHECK-NEOVERSE-V1: [[WHILE_END_LOOPEXIT]]:
+; CHECK-NEOVERSE-V1-NEXT: ret void
+;
+
+
+
+
+
+
entry:
br label %while.body
@@ -87,69 +304,306 @@ while.end.loopexit:
}
define float @fadd_red_fast(ptr noalias nocapture readonly %a, i64 %n) #0 {
-; CHECK-NOTF-LABEL: @fadd_red_fast
-; CHECK-NOTF: vector.body:
-; CHECK-NOTF-NOT: %{{.*}} = phi <vscale x 4 x i1>
-; CHECK-NOTF: %[[LOAD:.*]] = load <vscale x 4 x float>
-; CHECK-NOTF: %[[ADD:.*]] = fadd fast <vscale x 4 x float> %[[LOAD]]
-; CHECK-NOTF: middle.block:
-; CHECK-NOTF-NEXT: call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %[[ADD]])
-
-; CHECK-TF-NORED-LABEL: @fadd_red_fast
-; CHECK-TF-NORED: vector.body:
-; CHECK-TF-NORED-NOT: %{{.*}} = phi <vscale x 4 x i1>
-; CHECK-TF-NORED: %[[LOAD:.*]] = load <vscale x 4 x float>
-; CHECK-TF-NORED: %[[ADD:.*]] = fadd fast <vscale x 4 x float> %[[LOAD]]
-; CHECK-TF-NORED: middle.block:
-; CHECK-TF-NORED-NEXT: call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %[[ADD]])
-
-; CHECK-TF-NOREC-LABEL: @fadd_red_fast
-; CHECK-TF-NOREC: vector.body:
-; CHECK-TF-NOREC: %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
-; CHECK-TF-NOREC: %[[VEC_PHI:.*]] = phi <vscale x 4 x float>
-; CHECK-TF-NOREC: %[[LOAD:.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0({{.*}} %[[ACTIVE_LANE_MASK]]
-; CHECK-TF-NOREC: %[[ADD:.*]] = fadd fast <vscale x 4 x float> %[[LOAD]]
-; CHECK-TF-NOREC: %[[SEL:.*]] = select fast <vscale x 4 x i1> %[[ACTIVE_LANE_MASK]], <vscale x 4 x float> %[[ADD]], <vscale x 4 x float> %[[VEC_PHI]]
-; CHECK-TF-NOREC: middle.block:
-; CHECK-TF-NOREC-NEXT: call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %[[SEL]])
-
-; CHECK-TF-NOREV-LABEL: @fadd_red_fast
-; CHECK-TF-NOREV: vector.body:
-; CHECK-TF-NOREV: %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
-; CHECK-TF-NOREV: %[[VEC_PHI:.*]] = phi <vscale x 4 x float>
-; CHECK-TF-NOREV: %[[LOAD:.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0({{.*}} %[[ACTIVE_LANE_MASK]]
-; CHECK-TF-NOREV: %[[ADD:.*]] = fadd fast <vscale x 4 x float> %[[LOAD]]
-; CHECK-TF-NOREV: %[[SEL:.*]] = select fast <vscale x 4 x i1> %[[ACTIVE_LANE_MASK]], <vscale x 4 x float> %[[ADD]], <vscale x 4 x float> %[[VEC_PHI]]
-; CHECK-TF-NOREV: middle.block:
-; CHECK-TF-NOREV-NEXT: call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %[[SEL]])
-
-; CHECK-TF-LABEL: @fadd_red_fast
-; CHECK-TF: vector.body:
-; CHECK-TF: %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
-; CHECK-TF: %[[VEC_PHI:.*]] = phi <vscale x 4 x float>
-; CHECK-TF: %[[LOAD:.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0({{.*}} %[[ACTIVE_LANE_MASK]]
-; CHECK-TF: %[[ADD:.*]] = fadd fast <vscale x 4 x float> %[[LOAD]]
-; CHECK-TF: %[[SEL:.*]] = select fast <vscale x 4 x i1> %[[ACTIVE_LANE_MASK]], <vscale x 4 x float> %[[ADD]], <vscale x 4 x float> %[[VEC_PHI]]
-; CHECK-TF: middle.block:
-; CHECK-TF-NEXT: call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %[[SEL]])
-
-; CHECK-TF-ONLYRED-LABEL: @fadd_red_fast
-; CHECK-TF-ONLYRED: vector.body:
-; CHECK-TF-ONLYRED: %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
-; CHECK-TF-ONLYRED: %[[VEC_PHI:.*]] = phi <vscale x 4 x float>
-; CHECK-TF-ONLYRED: %[[LOAD:.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0({{.*}} %[[ACTIVE_LANE_MASK]]
-; CHECK-TF-ONLYRED: %[[ADD:.*]] = fadd fast <vscale x 4 x float> %[[LOAD]]
-; CHECK-TF-ONLYRED: %[[SEL:.*]] = select fast <vscale x 4 x i1> %[[ACTIVE_LANE_MASK]], <vscale x 4 x float> %[[ADD]], <vscale x 4 x float> %[[VEC_PHI]]
-; CHECK-TF-ONLYRED: middle.block:
-; CHECK-TF-ONLYRED-NEXT: call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %[[SEL]])
-
-; CHECK-NEOVERSE-V1-LABEL: @fadd_red_fast
-; CHECK-NEOVERSE-V1: vector.body:
-; CHECK-NEOVERSE-V1-NOT: %{{.*}} = phi <vscale x 4 x i1>
-; CHECK-NEOVERSE-V1: %[[LOAD:.*]] = load <vscale x 4 x float>
-; CHECK-NEOVERSE-V1: %[[ADD:.*]] = fadd fast <vscale x 4 x float> %[[LOAD]]
-; CHECK-NEOVERSE-V1: middle.block:
-; CHECK-NEOVERSE-V1-NEXT: call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %[[ADD]])
+; CHECK-NOTF-LABEL: define float @fadd_red_fast(
+; CHECK-NOTF-SAME: ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NOTF-NEXT: [[ENTRY:.*]]:
+; CHECK-NOTF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NOTF-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NOTF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NOTF-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NOTF: [[VECTOR_PH]]:
+; CHECK-NOTF-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NOTF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
+; CHECK-NOTF-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NOTF-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-NOTF: [[VECTOR_BODY]]:
+; CHECK-NOTF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NOTF-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NOTF-NEXT: [[TMP3:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-NOTF-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP3]], align 4
+; CHECK-NOTF-NEXT: [[TMP4]] = fadd fast <vscale x 4 x float> [[WIDE_LOAD]], [[VEC_PHI]]
+; CHECK-NOTF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-NOTF-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NOTF-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NOTF: [[MIDDLE_BLOCK]]:
+; CHECK-NOTF-NEXT: [[TMP6:%.*]] = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> [[TMP4]])
+; CHECK-NOTF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NOTF-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-NOTF: [[SCALAR_PH]]:
+; CHECK-NOTF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NOTF-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; CHECK-NOTF-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-NOTF: [[FOR_BODY]]:
+; CHECK-NOTF-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NOTF-NEXT: [[SUM_07:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]
+; CHECK-NOTF-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
+; CHECK-NOTF-NEXT: [[TMP7:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-NOTF-NEXT: [[ADD]] = fadd fast float [[TMP7]], [[SUM_07]]
+; CHECK-NOTF-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NOTF-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NOTF-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NOTF: [[FOR_END]]:
+; CHECK-NOTF-NEXT: [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], %[[FOR_BODY]] ], [ [[TMP6]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NOTF-NEXT: ret float [[ADD_LCSSA]]
+;
+; CHECK-TF-ALL-LABEL: define float @fadd_red_fast(
+; CHECK-TF-ALL-SAME: ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-ALL-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-ALL-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF-ALL: [[VECTOR_PH]]:
+; CHECK-TF-ALL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-ALL-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-ALL-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-ALL-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-ALL: [[VECTOR_BODY]]:
+; CHECK-TF-ALL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ALL-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ALL-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ALL-NEXT: [[TMP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-TF-ALL-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr align 4 [[TMP2]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> poison)
+; CHECK-TF-ALL-NEXT: [[TMP3:%.*]] = fadd fast <vscale x 4 x float> [[WIDE_MASKED_LOAD]], [[VEC_PHI]]
+; CHECK-TF-ALL-NEXT: [[TMP4]] = select fast <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> [[TMP3]], <vscale x 4 x float> [[VEC_PHI]]
+; CHECK-TF-ALL-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-ALL-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-ALL-NEXT: [[TMP5:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-ALL-NEXT: [[TMP6:%.*]] = xor i1 [[TMP5]], true
+; CHECK-TF-ALL-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-TF-ALL: [[MIDDLE_BLOCK]]:
+; CHECK-TF-ALL-NEXT: [[TMP7:%.*]] = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> [[TMP4]])
+; CHECK-TF-ALL-NEXT: br label %[[FOR_END:.*]]
+; CHECK-TF-ALL: [[FOR_END]]:
+; CHECK-TF-ALL-NEXT: ret float [[TMP7]]
+;
+; CHECK-TF-LABEL: define float @fadd_red_fast(
+; CHECK-TF-SAME: ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF: [[VECTOR_PH]]:
+; CHECK-TF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF: [[VECTOR_BODY]]:
+; CHECK-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[TMP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr align 4 [[TMP2]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> poison)
+; CHECK-TF-NEXT: [[TMP3:%.*]] = fadd fast <vscale x 4 x float> [[WIDE_MASKED_LOAD]], [[VEC_PHI]]
+; CHECK-TF-NEXT: [[TMP4]] = select fast <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> [[TMP3]], <vscale x 4 x float> [[VEC_PHI]]
+; CHECK-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-NEXT: [[TMP5:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NEXT: [[TMP6:%.*]] = xor i1 [[TMP5]], true
+; CHECK-TF-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-TF: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NEXT: [[TMP7:%.*]] = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> [[TMP4]])
+; CHECK-TF-NEXT: br label %[[FOR_END:.*]]
+; CHECK-TF: [[FOR_END]]:
+; CHECK-TF-NEXT: ret float [[TMP7]]
+;
+; CHECK-TF-DEFAULT-LABEL: define float @fadd_red_fast(
+; CHECK-TF-DEFAULT-SAME: ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-DEFAULT-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-DEFAULT-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF-DEFAULT: [[VECTOR_PH]]:
+; CHECK-TF-DEFAULT-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-DEFAULT-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-DEFAULT-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-DEFAULT-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-DEFAULT: [[VECTOR_BODY]]:
+; CHECK-TF-DEFAULT-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-DEFAULT-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-DEFAULT-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-DEFAULT-NEXT: [[TMP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-TF-DEFAULT-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr align 4 [[TMP2]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> poison)
+; CHECK-TF-DEFAULT-NEXT: [[TMP3:%.*]] = fadd fast <vscale x 4 x float> [[WIDE_MASKED_LOAD]], [[VEC_PHI]]
+; CHECK-TF-DEFAULT-NEXT: [[TMP4]] = select fast <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> [[TMP3]], <vscale x 4 x float> [[VEC_PHI]]
+; CHECK-TF-DEFAULT-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-DEFAULT-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-DEFAULT-NEXT: [[TMP5:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-DEFAULT-NEXT: [[TMP6:%.*]] = xor i1 [[TMP5]], true
+; CHECK-TF-DEFAULT-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-TF-DEFAULT: [[MIDDLE_BLOCK]]:
+; CHECK-TF-DEFAULT-NEXT: [[TMP7:%.*]] = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> [[TMP4]])
+; CHECK-TF-DEFAULT-NEXT: br label %[[FOR_END:.*]]
+; CHECK-TF-DEFAULT: [[FOR_END]]:
+; CHECK-TF-DEFAULT-NEXT: ret float [[TMP7]]
+;
+; CHECK-TF-NORED-LABEL: define float @fadd_red_fast(
+; CHECK-TF-NORED-SAME: ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NORED-NEXT: [[ENTRY:.*]]:
+; CHECK-TF-NORED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NORED-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NORED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-TF-NORED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF-NORED: [[VECTOR_PH]]:
+; CHECK-TF-NORED-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NORED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
+; CHECK-TF-NORED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-TF-NORED-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-NORED: [[VECTOR_BODY]]:
+; CHECK-TF-NORED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NORED-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NORED-NEXT: [[TMP3:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-TF-NORED-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP3]], align 4
+; CHECK-TF-NORED-NEXT: [[TMP4]] = fadd fast <vscale x 4 x float> [[WIDE_LOAD]], [[VEC_PHI]]
+; CHECK-TF-NORED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-TF-NORED-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-NORED-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-TF-NORED: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NORED-NEXT: [[TMP6:%.*]] = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> [[TMP4]])
+; CHECK-TF-NORED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-TF-NORED-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-TF-NORED: [[SCALAR_PH]]:
+; CHECK-TF-NORED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-TF-NORED-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; CHECK-TF-NORED-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-TF-NORED: [[FOR_BODY]]:
+; CHECK-TF-NORED-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-TF-NORED-NEXT: [[SUM_07:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]
+; CHECK-TF-NORED-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
+; CHECK-TF-NORED-NEXT: [[TMP7:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-TF-NORED-NEXT: [[ADD]] = fadd fast float [[TMP7]], [[SUM_07]]
+; CHECK-TF-NORED-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-TF-NORED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-TF-NORED-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-TF-NORED: [[FOR_END]]:
+; CHECK-TF-NORED-NEXT: [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], %[[FOR_BODY]] ], [ [[TMP6]], %[[MIDDLE_BLOCK]] ]
+; CHECK-TF-NORED-NEXT: ret float [[ADD_LCSSA]]
+;
+; CHECK-TF-NOREC-LABEL: define float @fadd_red_fast(
+; CHECK-TF-NOREC-SAME: ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NOREC-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-NOREC-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF-NOREC: [[VECTOR_PH]]:
+; CHECK-TF-NOREC-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NOREC-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NOREC-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-NOREC-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-NOREC: [[VECTOR_BODY]]:
+; CHECK-TF-NOREC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREC-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREC-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREC-NEXT: [[TMP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-TF-NOREC-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr align 4 [[TMP2]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> poison)
+; CHECK-TF-NOREC-NEXT: [[TMP3:%.*]] = fadd fast <vscale x 4 x float> [[WIDE_MASKED_LOAD]], [[VEC_PHI]]
+; CHECK-TF-NOREC-NEXT: [[TMP4]] = select fast <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> [[TMP3]], <vscale x 4 x float> [[VEC_PHI]]
+; CHECK-TF-NOREC-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-NOREC-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-NOREC-NEXT: [[TMP5:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NOREC-NEXT: [[TMP6:%.*]] = xor i1 [[TMP5]], true
+; CHECK-TF-NOREC-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-TF-NOREC: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NOREC-NEXT: [[TMP7:%.*]] = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> [[TMP4]])
+; CHECK-TF-NOREC-NEXT: br label %[[FOR_END:.*]]
+; CHECK-TF-NOREC: [[FOR_END]]:
+; CHECK-TF-NOREC-NEXT: ret float [[TMP7]]
+;
+; CHECK-TF-NOREV-LABEL: define float @fadd_red_fast(
+; CHECK-TF-NOREV-SAME: ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NOREV-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-NOREV-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF-NOREV: [[VECTOR_PH]]:
+; CHECK-TF-NOREV-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NOREV-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NOREV-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-NOREV-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-NOREV: [[VECTOR_BODY]]:
+; CHECK-TF-NOREV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREV-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREV-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREV-NEXT: [[TMP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-TF-NOREV-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr align 4 [[TMP2]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> poison)
+; CHECK-TF-NOREV-NEXT: [[TMP3:%.*]] = fadd fast <vscale x 4 x float> [[WIDE_MASKED_LOAD]], [[VEC_PHI]]
+; CHECK-TF-NOREV-NEXT: [[TMP4]] = select fast <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> [[TMP3]], <vscale x 4 x float> [[VEC_PHI]]
+; CHECK-TF-NOREV-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-NOREV-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-NOREV-NEXT: [[TMP5:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NOREV-NEXT: [[TMP6:%.*]] = xor i1 [[TMP5]], true
+; CHECK-TF-NOREV-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-TF-NOREV: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NOREV-NEXT: [[TMP7:%.*]] = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> [[TMP4]])
+; CHECK-TF-NOREV-NEXT: br label %[[FOR_END:.*]]
+; CHECK-TF-NOREV: [[FOR_END]]:
+; CHECK-TF-NOREV-NEXT: ret float [[TMP7]]
+;
+; CHECK-TF-ONLYRED-LABEL: define float @fadd_red_fast(
+; CHECK-TF-ONLYRED-SAME: ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-ONLYRED-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-ONLYRED-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF-ONLYRED: [[VECTOR_PH]]:
+; CHECK-TF-ONLYRED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-ONLYRED-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-ONLYRED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-ONLYRED-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-ONLYRED: [[VECTOR_BODY]]:
+; CHECK-TF-ONLYRED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ONLYRED-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ONLYRED-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ONLYRED-NEXT: [[TMP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-TF-ONLYRED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr align 4 [[TMP2]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> poison)
+; CHECK-TF-ONLYRED-NEXT: [[TMP3:%.*]] = fadd fast <vscale x 4 x float> [[WIDE_MASKED_LOAD]], [[VEC_PHI]]
+; CHECK-TF-ONLYRED-NEXT: [[TMP4]] = select fast <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> [[TMP3]], <vscale x 4 x float> [[VEC_PHI]]
+; CHECK-TF-ONLYRED-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-ONLYRED-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-ONLYRED-NEXT: [[TMP5:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-ONLYRED-NEXT: [[TMP6:%.*]] = xor i1 [[TMP5]], true
+; CHECK-TF-ONLYRED-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-TF-ONLYRED: [[MIDDLE_BLOCK]]:
+; CHECK-TF-ONLYRED-NEXT: [[TMP7:%.*]] = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> [[TMP4]])
+; CHECK-TF-ONLYRED-NEXT: br label %[[FOR_END:.*]]
+; CHECK-TF-ONLYRED: [[FOR_END]]:
+; CHECK-TF-ONLYRED-NEXT: ret float [[TMP7]]
+;
+; CHECK-NEOVERSE-V1-LABEL: define float @fadd_red_fast(
+; CHECK-NEOVERSE-V1-SAME: ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEOVERSE-V1-NEXT: [[ENTRY:.*]]:
+; CHECK-NEOVERSE-V1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEOVERSE-V1-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEOVERSE-V1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NEOVERSE-V1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEOVERSE-V1: [[VECTOR_PH]]:
+; CHECK-NEOVERSE-V1-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEOVERSE-V1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
+; CHECK-NEOVERSE-V1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEOVERSE-V1-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-NEOVERSE-V1: [[VECTOR_BODY]]:
+; CHECK-NEOVERSE-V1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEOVERSE-V1-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP3:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEOVERSE-V1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP3]], align 4
+; CHECK-NEOVERSE-V1-NEXT: [[TMP4]] = fadd fast <vscale x 4 x float> [[WIDE_LOAD]], [[VEC_PHI]]
+; CHECK-NEOVERSE-V1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEOVERSE-V1-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEOVERSE-V1: [[MIDDLE_BLOCK]]:
+; CHECK-NEOVERSE-V1-NEXT: [[TMP6:%.*]] = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> [[TMP4]])
+; CHECK-NEOVERSE-V1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEOVERSE-V1-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-NEOVERSE-V1: [[SCALAR_PH]]:
+; CHECK-NEOVERSE-V1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEOVERSE-V1-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; CHECK-NEOVERSE-V1-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-NEOVERSE-V1: [[FOR_BODY]]:
+; CHECK-NEOVERSE-V1-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEOVERSE-V1-NEXT: [[SUM_07:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEOVERSE-V1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP7:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-NEOVERSE-V1-NEXT: [[ADD]] = fadd fast float [[TMP7]], [[SUM_07]]
+; CHECK-NEOVERSE-V1-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEOVERSE-V1-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEOVERSE-V1-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEOVERSE-V1: [[FOR_END]]:
+; CHECK-NEOVERSE-V1-NEXT: [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], %[[FOR_BODY]] ], [ [[TMP6]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEOVERSE-V1-NEXT: ret float [[ADD_LCSSA]]
+;
+
+
+
+
+
+
entry:
br label %for.body
@@ -169,96 +623,407 @@ for.end:
}
define void @add_recur(ptr noalias %dst, ptr noalias %src, i64 %n) #0 {
-; CHECK-NOTF-LABEL: @add_recur
-; CHECK-NOTF: entry:
-; CHECK-NOTF: %[[PRE:.*]] = load i32, ptr %src, align 4
-; CHECK-NOTF: vector.ph:
-; CHECK-NOTF: %[[RECUR_INIT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %[[PRE]]
-; CHECK-NOTF: vector.body:
-; CHECK-NOTF-NOT: %{{.*}} = phi <vscale x 4 x i1>
-; CHECK-NOTF: %[[VECTOR_RECUR:.*]] = phi <vscale x 4 x i32> [ %[[RECUR_INIT]], %vector.ph ], [ %[[LOAD:.*]], %vector.body ]
-; CHECK-NOTF: %[[LOAD]] = load <vscale x 4 x i32>
-; CHECK-NOTF: %[[SPLICE:.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> %[[VECTOR_RECUR]], <vscale x 4 x i32> %[[LOAD]], i32 1)
-; CHECK-NOTF: %[[ADD:.*]] = add nsw <vscale x 4 x i32> %[[LOAD]], %[[SPLICE]]
-; CHECK-NOTF: store <vscale x 4 x i32> %[[ADD]]
-
-; CHECK-TF-NORED-LABEL: @add_recur
-; CHECK-TF-NORED: entry:
-; CHECK-TF-NORED: %[[PRE:.*]] = load i32, ptr %src, align 4
-; CHECK-TF-NORED: vector.ph:
-; CHECK-TF-NORED: %[[RECUR_INIT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %[[PRE]]
-; CHECK-TF-NORED: vector.body:
-; CHECK-TF-NORED: %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
-; CHECK-TF-NORED: %[[VECTOR_RECUR:.*]] = phi <vscale x 4 x i32> [ %[[RECUR_INIT]], %vector.ph ], [ %[[LOAD:.*]], %vector.body ]
-; CHECK-TF-NORED: %[[LOAD]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0({{.*}} %[[ACTIVE_LANE_MASK]]
-; CHECK-TF-NORED: %[[SPLICE:.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> %[[VECTOR_RECUR]], <vscale x 4 x i32> %[[LOAD]], i32 1)
-; CHECK-TF-NORED: %[[ADD:.*]] = add nsw <vscale x 4 x i32> %[[LOAD]], %[[SPLICE]]
-; CHECK-TF-NORED: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> %[[ADD]], {{.*}} <vscale x 4 x i1> %[[ACTIVE_LANE_MASK]])
-
-; CHECK-TF-NOREC-LABEL: @add_recur
-; CHECK-TF-NOREC: entry:
-; CHECK-TF-NOREC: %[[PRE:.*]] = load i32, ptr %src, align 4
-; CHECK-TF-NOREC: vector.ph:
-; CHECK-TF-NOREC: %[[RECUR_INIT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %[[PRE]]
-; CHECK-TF-NOREC: vector.body:
-; CHECK-TF-NOREC-NOT: %{{.*}} = phi <vscale x 4 x i1>
-; CHECK-TF-NOREC: %[[VECTOR_RECUR:.*]] = phi <vscale x 4 x i32> [ %[[RECUR_INIT]], %vector.ph ], [ %[[LOAD:.*]], %vector.body ]
-; CHECK-TF-NOREC: %[[LOAD]] = load <vscale x 4 x i32>
-; CHECK-TF-NOREC: %[[SPLICE:.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> %[[VECTOR_RECUR]], <vscale x 4 x i32> %[[LOAD]], i32 1)
-; CHECK-TF-NOREC: %[[ADD:.*]] = add nsw <vscale x 4 x i32> %[[LOAD]], %[[SPLICE]]
-; CHECK-TF-NOREC: store <vscale x 4 x i32> %[[ADD]]
-
-; CHECK-TF-NOREV-LABEL: @add_recur
-; CHECK-TF-NOREV: entry:
-; CHECK-TF-NOREV: %[[PRE:.*]] = load i32, ptr %src, align 4
-; CHECK-TF-NOREV: vector.ph:
-; CHECK-TF-NOREV: %[[RECUR_INIT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %[[PRE]]
-; CHECK-TF-NOREV: vector.body:
-; CHECK-TF-NOREV: %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
-; CHECK-TF-NOREV: %[[VECTOR_RECUR:.*]] = phi <vscale x 4 x i32> [ %[[RECUR_INIT]], %vector.ph ], [ %[[LOAD:.*]], %vector.body ]
-; CHECK-TF-NOREV: %[[LOAD]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0({{.*}} %[[ACTIVE_LANE_MASK]]
-; CHECK-TF-NOREV: %[[SPLICE:.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> %[[VECTOR_RECUR]], <vscale x 4 x i32> %[[LOAD]], i32 1)
-; CHECK-TF-NOREV: %[[ADD:.*]] = add nsw <vscale x 4 x i32> %[[LOAD]], %[[SPLICE]]
-; CHECK-TF-NOREV: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> %[[ADD]], {{.*}} <vscale x 4 x i1> %[[ACTIVE_LANE_MASK]])
-
-; CHECK-TF-LABEL: @add_recur
-; CHECK-TF: entry:
-; CHECK-TF: %[[PRE:.*]] = load i32, ptr %src, align 4
-; CHECK-TF: vector.ph:
-; CHECK-TF: %[[RECUR_INIT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %[[PRE]]
-; CHECK-TF: vector.body:
-; CHECK-TF: %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
-; CHECK-TF: %[[VECTOR_RECUR:.*]] = phi <vscale x 4 x i32> [ %[[RECUR_INIT]], %vector.ph ], [ %[[LOAD:.*]], %vector.body ]
-; CHECK-TF: %[[LOAD]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0({{.*}} %[[ACTIVE_LANE_MASK]]
-; CHECK-TF: %[[SPLICE:.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> %[[VECTOR_RECUR]], <vscale x 4 x i32> %[[LOAD]], i32 1)
-; CHECK-TF: %[[ADD:.*]] = add nsw <vscale x 4 x i32> %[[LOAD]], %[[SPLICE]]
-; CHECK-TF: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> %[[ADD]], {{.*}} <vscale x 4 x i1> %[[ACTIVE_LANE_MASK]])
-
-; CHECK-TF-ONLYRED-LABEL: @add_recur
-; CHECK-TF-ONLYRED: entry:
-; CHECK-TF-ONLYRED: %[[PRE:.*]] = load i32, ptr %src, align 4
-; CHECK-TF-ONLYRED: vector.ph:
-; CHECK-TF-ONLYRED: %[[RECUR_INIT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %[[PRE]]
-; CHECK-TF-ONLYRED: vector.body:
-; CHECK-TF-ONLYRED-NOT: %{{.*}} = phi <vscale x 4 x i1>
-; CHECK-TF-ONLYRED: %[[VECTOR_RECUR:.*]] = phi <vscale x 4 x i32> [ %[[RECUR_INIT]], %vector.ph ], [ %[[LOAD:.*]], %vector.body ]
-; CHECK-TF-ONLYRED: %[[LOAD]] = load <vscale x 4 x i32>
-; CHECK-TF-ONLYRED: %[[SPLICE:.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> %[[VECTOR_RECUR]], <vscale x 4 x i32> %[[LOAD]], i32 1)
-; CHECK-TF-ONLYRED: %[[ADD:.*]] = add nsw <vscale x 4 x i32> %[[LOAD]], %[[SPLICE]]
-; CHECK-TF-ONLYRED: store <vscale x 4 x i32> %[[ADD]]
-
-; CHECK-NEOVERSE-V1-LABEL: @add_recur
-; CHECK-NEOVERSE-V1: entry:
-; CHECK-NEOVERSE-V1: %[[PRE:.*]] = load i32, ptr %src, align 4
-; CHECK-NEOVERSE-V1: vector.ph:
-; CHECK-NEOVERSE-V1: %[[RECUR_INIT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %[[PRE]]
-; CHECK-NEOVERSE-V1: vector.body:
-; CHECK-NEOVERSE-V1-NOT: %{{.*}} = phi <vscale x 4 x i1>
-; CHECK-NEOVERSE-V1: %[[VECTOR_RECUR:.*]] = phi <vscale x 4 x i32> [ %[[RECUR_INIT]], %vector.ph ], [ %[[LOAD:.*]], %vector.body ]
-; CHECK-NEOVERSE-V1: %[[LOAD]] = load <vscale x 4 x i32>
-; CHECK-NEOVERSE-V1: %[[SPLICE:.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> %[[VECTOR_RECUR]], <vscale x 4 x i32> %[[LOAD]], i32 1)
-; CHECK-NEOVERSE-V1: %[[ADD:.*]] = add nsw <vscale x 4 x i32> %[[LOAD]], %[[SPLICE]]
-; CHECK-NEOVERSE-V1: store <vscale x 4 x i32> %[[ADD]]
+; CHECK-NOTF-LABEL: define void @add_recur(
+; CHECK-NOTF-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NOTF-NEXT: [[ENTRY:.*]]:
+; CHECK-NOTF-NEXT: [[DOTPRE:%.*]] = load i32, ptr [[SRC]], align 4
+; CHECK-NOTF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NOTF-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NOTF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NOTF-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NOTF: [[VECTOR_PH]]:
+; CHECK-NOTF-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NOTF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
+; CHECK-NOTF-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NOTF-NEXT: [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NOTF-NEXT: [[TMP4:%.*]] = mul nuw i32 [[TMP3]], 4
+; CHECK-NOTF-NEXT: [[TMP5:%.*]] = sub i32 [[TMP4]], 1
+; CHECK-NOTF-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[DOTPRE]], i32 [[TMP5]]
+; CHECK-NOTF-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-NOTF: [[VECTOR_BODY]]:
+; CHECK-NOTF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NOTF-NEXT: [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NOTF-NEXT: [[TMP6:%.*]] = add nuw nsw i64 [[INDEX]], 1
+; CHECK-NOTF-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP6]]
+; CHECK-NOTF-NEXT: [[WIDE_LOAD]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
+; CHECK-NOTF-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR]], <vscale x 4 x i32> [[WIDE_LOAD]], i32 1)
+; CHECK-NOTF-NEXT: [[TMP9:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_LOAD]], [[TMP8]]
+; CHECK-NOTF-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NOTF-NEXT: store <vscale x 4 x i32> [[TMP9]], ptr [[TMP10]], align 4
+; CHECK-NOTF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-NOTF-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NOTF-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NOTF: [[MIDDLE_BLOCK]]:
+; CHECK-NOTF-NEXT: [[TMP12:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NOTF-NEXT: [[TMP13:%.*]] = mul nuw i32 [[TMP12]], 4
+; CHECK-NOTF-NEXT: [[TMP14:%.*]] = sub i32 [[TMP13]], 1
+; CHECK-NOTF-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_LOAD]], i32 [[TMP14]]
+; CHECK-NOTF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NOTF-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-NOTF: [[SCALAR_PH]]:
+; CHECK-NOTF-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ [[DOTPRE]], %[[ENTRY]] ]
+; CHECK-NOTF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NOTF-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-NOTF: [[FOR_BODY]]:
+; CHECK-NOTF-NEXT: [[TMP15:%.*]] = phi i32 [ [[TMP16:%.*]], %[[FOR_BODY]] ], [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ]
+; CHECK-NOTF-NEXT: [[I_010:%.*]] = phi i64 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NOTF-NEXT: [[ADD]] = add nuw nsw i64 [[I_010]], 1
+; CHECK-NOTF-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[ADD]]
+; CHECK-NOTF-NEXT: [[TMP16]] = load i32, ptr [[ARRAYIDX1]], align 4
+; CHECK-NOTF-NEXT: [[ADD2:%.*]] = add nsw i32 [[TMP16]], [[TMP15]]
+; CHECK-NOTF-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[I_010]]
+; CHECK-NOTF-NEXT: store i32 [[ADD2]], ptr [[ARRAYIDX3]], align 4
+; CHECK-NOTF-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[ADD]], [[N]]
+; CHECK-NOTF-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NOTF: [[FOR_END]]:
+; CHECK-NOTF-NEXT: ret void
+;
+; CHECK-TF-ALL-LABEL: define void @add_recur(
+; CHECK-TF-ALL-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-ALL-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-ALL-NEXT: [[DOTPRE:%.*]] = load i32, ptr [[SRC]], align 4
+; CHECK-TF-ALL-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF-ALL: [[VECTOR_PH]]:
+; CHECK-TF-ALL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-ALL-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-ALL-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-ALL-NEXT: [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-TF-ALL-NEXT: [[TMP3:%.*]] = mul nuw i32 [[TMP2]], 4
+; CHECK-TF-ALL-NEXT: [[TMP4:%.*]] = sub i32 [[TMP3]], 1
+; CHECK-TF-ALL-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[DOTPRE]], i32 [[TMP4]]
+; CHECK-TF-ALL-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-ALL: [[VECTOR_BODY]]:
+; CHECK-TF-ALL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ALL-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ALL-NEXT: [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_MASKED_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ALL-NEXT: [[TMP5:%.*]] = add nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP5]]
+; CHECK-TF-ALL-NEXT: [[WIDE_MASKED_LOAD]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
+; CHECK-TF-ALL-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], i32 1)
+; CHECK-TF-ALL-NEXT: [[TMP8:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], [[TMP7]]
+; CHECK-TF-ALL-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-TF-ALL-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP8]], ptr align 4 [[TMP9]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-ALL-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-ALL-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-ALL-NEXT: [[TMP10:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-ALL-NEXT: [[TMP11:%.*]] = xor i1 [[TMP10]], true
+; CHECK-TF-ALL-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-TF-ALL: [[MIDDLE_BLOCK]]:
+; CHECK-TF-ALL-NEXT: br label %[[FOR_END:.*]]
+; CHECK-TF-ALL: [[FOR_END]]:
+; CHECK-TF-ALL-NEXT: ret void
+;
+; CHECK-TF-LABEL: define void @add_recur(
+; CHECK-TF-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-NEXT: [[DOTPRE:%.*]] = load i32, ptr [[SRC]], align 4
+; CHECK-TF-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF: [[VECTOR_PH]]:
+; CHECK-TF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-NEXT: [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-TF-NEXT: [[TMP3:%.*]] = mul nuw i32 [[TMP2]], 4
+; CHECK-TF-NEXT: [[TMP4:%.*]] = sub i32 [[TMP3]], 1
+; CHECK-TF-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[DOTPRE]], i32 [[TMP4]]
+; CHECK-TF-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF: [[VECTOR_BODY]]:
+; CHECK-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_MASKED_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[TMP5:%.*]] = add nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP5]]
+; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
+; CHECK-TF-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], i32 1)
+; CHECK-TF-NEXT: [[TMP8:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], [[TMP7]]
+; CHECK-TF-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-TF-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP8]], ptr align 4 [[TMP9]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-NEXT: [[TMP10:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NEXT: [[TMP11:%.*]] = xor i1 [[TMP10]], true
+; CHECK-TF-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-TF: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NEXT: br label %[[FOR_END:.*]]
+; CHECK-TF: [[FOR_END]]:
+; CHECK-TF-NEXT: ret void
+;
+; CHECK-TF-DEFAULT-LABEL: define void @add_recur(
+; CHECK-TF-DEFAULT-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-DEFAULT-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-DEFAULT-NEXT: [[DOTPRE:%.*]] = load i32, ptr [[SRC]], align 4
+; CHECK-TF-DEFAULT-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF-DEFAULT: [[VECTOR_PH]]:
+; CHECK-TF-DEFAULT-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-DEFAULT-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-DEFAULT-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-DEFAULT-NEXT: [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-TF-DEFAULT-NEXT: [[TMP3:%.*]] = mul nuw i32 [[TMP2]], 4
+; CHECK-TF-DEFAULT-NEXT: [[TMP4:%.*]] = sub i32 [[TMP3]], 1
+; CHECK-TF-DEFAULT-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[DOTPRE]], i32 [[TMP4]]
+; CHECK-TF-DEFAULT-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-DEFAULT: [[VECTOR_BODY]]:
+; CHECK-TF-DEFAULT-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-DEFAULT-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-DEFAULT-NEXT: [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_MASKED_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-DEFAULT-NEXT: [[TMP5:%.*]] = add nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-DEFAULT-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP5]]
+; CHECK-TF-DEFAULT-NEXT: [[WIDE_MASKED_LOAD]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
+; CHECK-TF-DEFAULT-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], i32 1)
+; CHECK-TF-DEFAULT-NEXT: [[TMP8:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], [[TMP7]]
+; CHECK-TF-DEFAULT-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-TF-DEFAULT-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP8]], ptr align 4 [[TMP9]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-DEFAULT-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-DEFAULT-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-DEFAULT-NEXT: [[TMP10:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-DEFAULT-NEXT: [[TMP11:%.*]] = xor i1 [[TMP10]], true
+; CHECK-TF-DEFAULT-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-TF-DEFAULT: [[MIDDLE_BLOCK]]:
+; CHECK-TF-DEFAULT-NEXT: br label %[[FOR_END:.*]]
+; CHECK-TF-DEFAULT: [[FOR_END]]:
+; CHECK-TF-DEFAULT-NEXT: ret void
+;
+; CHECK-TF-NORED-LABEL: define void @add_recur(
+; CHECK-TF-NORED-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NORED-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-NORED-NEXT: [[DOTPRE:%.*]] = load i32, ptr [[SRC]], align 4
+; CHECK-TF-NORED-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF-NORED: [[VECTOR_PH]]:
+; CHECK-TF-NORED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NORED-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NORED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-NORED-NEXT: [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-TF-NORED-NEXT: [[TMP3:%.*]] = mul nuw i32 [[TMP2]], 4
+; CHECK-TF-NORED-NEXT: [[TMP4:%.*]] = sub i32 [[TMP3]], 1
+; CHECK-TF-NORED-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[DOTPRE]], i32 [[TMP4]]
+; CHECK-TF-NORED-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-NORED: [[VECTOR_BODY]]:
+; CHECK-TF-NORED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NORED-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NORED-NEXT: [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_MASKED_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NORED-NEXT: [[TMP5:%.*]] = add nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-NORED-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP5]]
+; CHECK-TF-NORED-NEXT: [[WIDE_MASKED_LOAD]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
+; CHECK-TF-NORED-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], i32 1)
+; CHECK-TF-NORED-NEXT: [[TMP8:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], [[TMP7]]
+; CHECK-TF-NORED-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-TF-NORED-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP8]], ptr align 4 [[TMP9]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NORED-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-NORED-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-NORED-NEXT: [[TMP10:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NORED-NEXT: [[TMP11:%.*]] = xor i1 [[TMP10]], true
+; CHECK-TF-NORED-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-TF-NORED: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NORED-NEXT: br label %[[FOR_END:.*]]
+; CHECK-TF-NORED: [[FOR_END]]:
+; CHECK-TF-NORED-NEXT: ret void
+;
+; CHECK-TF-NOREC-LABEL: define void @add_recur(
+; CHECK-TF-NOREC-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NOREC-NEXT: [[ENTRY:.*]]:
+; CHECK-TF-NOREC-NEXT: [[DOTPRE:%.*]] = load i32, ptr [[SRC]], align 4
+; CHECK-TF-NOREC-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NOREC-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NOREC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-TF-NOREC-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF-NOREC: [[VECTOR_PH]]:
+; CHECK-TF-NOREC-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NOREC-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
+; CHECK-TF-NOREC-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-TF-NOREC-NEXT: [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-TF-NOREC-NEXT: [[TMP4:%.*]] = mul nuw i32 [[TMP3]], 4
+; CHECK-TF-NOREC-NEXT: [[TMP5:%.*]] = sub i32 [[TMP4]], 1
+; CHECK-TF-NOREC-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[DOTPRE]], i32 [[TMP5]]
+; CHECK-TF-NOREC-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-NOREC: [[VECTOR_BODY]]:
+; CHECK-TF-NOREC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREC-NEXT: [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREC-NEXT: [[TMP6:%.*]] = add nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-NOREC-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP6]]
+; CHECK-TF-NOREC-NEXT: [[WIDE_LOAD]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
+; CHECK-TF-NOREC-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR]], <vscale x 4 x i32> [[WIDE_LOAD]], i32 1)
+; CHECK-TF-NOREC-NEXT: [[TMP9:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_LOAD]], [[TMP8]]
+; CHECK-TF-NOREC-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-TF-NOREC-NEXT: store <vscale x 4 x i32> [[TMP9]], ptr [[TMP10]], align 4
+; CHECK-TF-NOREC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-TF-NOREC-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-NOREC-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-TF-NOREC: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NOREC-NEXT: [[TMP12:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-TF-NOREC-NEXT: [[TMP13:%.*]] = mul nuw i32 [[TMP12]], 4
+; CHECK-TF-NOREC-NEXT: [[TMP14:%.*]] = sub i32 [[TMP13]], 1
+; CHECK-TF-NOREC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_LOAD]], i32 [[TMP14]]
+; CHECK-TF-NOREC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-TF-NOREC-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-TF-NOREC: [[SCALAR_PH]]:
+; CHECK-TF-NOREC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ [[DOTPRE]], %[[ENTRY]] ]
+; CHECK-TF-NOREC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-TF-NOREC-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-TF-NOREC: [[FOR_BODY]]:
+; CHECK-TF-NOREC-NEXT: [[TMP15:%.*]] = phi i32 [ [[TMP16:%.*]], %[[FOR_BODY]] ], [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ]
+; CHECK-TF-NOREC-NEXT: [[I_010:%.*]] = phi i64 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-TF-NOREC-NEXT: [[ADD]] = add nuw nsw i64 [[I_010]], 1
+; CHECK-TF-NOREC-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[ADD]]
+; CHECK-TF-NOREC-NEXT: [[TMP16]] = load i32, ptr [[ARRAYIDX1]], align 4
+; CHECK-TF-NOREC-NEXT: [[ADD2:%.*]] = add nsw i32 [[TMP16]], [[TMP15]]
+; CHECK-TF-NOREC-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[I_010]]
+; CHECK-TF-NOREC-NEXT: store i32 [[ADD2]], ptr [[ARRAYIDX3]], align 4
+; CHECK-TF-NOREC-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[ADD]], [[N]]
+; CHECK-TF-NOREC-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-TF-NOREC: [[FOR_END]]:
+; CHECK-TF-NOREC-NEXT: ret void
+;
+; CHECK-TF-NOREV-LABEL: define void @add_recur(
+; CHECK-TF-NOREV-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NOREV-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-NOREV-NEXT: [[DOTPRE:%.*]] = load i32, ptr [[SRC]], align 4
+; CHECK-TF-NOREV-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF-NOREV: [[VECTOR_PH]]:
+; CHECK-TF-NOREV-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NOREV-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NOREV-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-NOREV-NEXT: [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-TF-NOREV-NEXT: [[TMP3:%.*]] = mul nuw i32 [[TMP2]], 4
+; CHECK-TF-NOREV-NEXT: [[TMP4:%.*]] = sub i32 [[TMP3]], 1
+; CHECK-TF-NOREV-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[DOTPRE]], i32 [[TMP4]]
+; CHECK-TF-NOREV-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-NOREV: [[VECTOR_BODY]]:
+; CHECK-TF-NOREV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREV-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREV-NEXT: [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_MASKED_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREV-NEXT: [[TMP5:%.*]] = add nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-NOREV-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP5]]
+; CHECK-TF-NOREV-NEXT: [[WIDE_MASKED_LOAD]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
+; CHECK-TF-NOREV-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], i32 1)
+; CHECK-TF-NOREV-NEXT: [[TMP8:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], [[TMP7]]
+; CHECK-TF-NOREV-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-TF-NOREV-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP8]], ptr align 4 [[TMP9]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NOREV-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-NOREV-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-NOREV-NEXT: [[TMP10:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NOREV-NEXT: [[TMP11:%.*]] = xor i1 [[TMP10]], true
+; CHECK-TF-NOREV-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-TF-NOREV: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NOREV-NEXT: br label %[[FOR_END:.*]]
+; CHECK-TF-NOREV: [[FOR_END]]:
+; CHECK-TF-NOREV-NEXT: ret void
+;
+; CHECK-TF-ONLYRED-LABEL: define void @add_recur(
+; CHECK-TF-ONLYRED-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-ONLYRED-NEXT: [[ENTRY:.*]]:
+; CHECK-TF-ONLYRED-NEXT: [[DOTPRE:%.*]] = load i32, ptr [[SRC]], align 4
+; CHECK-TF-ONLYRED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-ONLYRED-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-ONLYRED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-TF-ONLYRED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF-ONLYRED: [[VECTOR_PH]]:
+; CHECK-TF-ONLYRED-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-ONLYRED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
+; CHECK-TF-ONLYRED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-TF-ONLYRED-NEXT: [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-TF-ONLYRED-NEXT: [[TMP4:%.*]] = mul nuw i32 [[TMP3]], 4
+; CHECK-TF-ONLYRED-NEXT: [[TMP5:%.*]] = sub i32 [[TMP4]], 1
+; CHECK-TF-ONLYRED-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[DOTPRE]], i32 [[TMP5]]
+; CHECK-TF-ONLYRED-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-ONLYRED: [[VECTOR_BODY]]:
+; CHECK-TF-ONLYRED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ONLYRED-NEXT: [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ONLYRED-NEXT: [[TMP6:%.*]] = add nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-ONLYRED-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP6]]
+; CHECK-TF-ONLYRED-NEXT: [[WIDE_LOAD]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
+; CHECK-TF-ONLYRED-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR]], <vscale x 4 x i32> [[WIDE_LOAD]], i32 1)
+; CHECK-TF-ONLYRED-NEXT: [[TMP9:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_LOAD]], [[TMP8]]
+; CHECK-TF-ONLYRED-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-TF-ONLYRED-NEXT: store <vscale x 4 x i32> [[TMP9]], ptr [[TMP10]], align 4
+; CHECK-TF-ONLYRED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-TF-ONLYRED-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-ONLYRED-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-TF-ONLYRED: [[MIDDLE_BLOCK]]:
+; CHECK-TF-ONLYRED-NEXT: [[TMP12:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-TF-ONLYRED-NEXT: [[TMP13:%.*]] = mul nuw i32 [[TMP12]], 4
+; CHECK-TF-ONLYRED-NEXT: [[TMP14:%.*]] = sub i32 [[TMP13]], 1
+; CHECK-TF-ONLYRED-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_LOAD]], i32 [[TMP14]]
+; CHECK-TF-ONLYRED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-TF-ONLYRED-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-TF-ONLYRED: [[SCALAR_PH]]:
+; CHECK-TF-ONLYRED-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ [[DOTPRE]], %[[ENTRY]] ]
+; CHECK-TF-ONLYRED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-TF-ONLYRED-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-TF-ONLYRED: [[FOR_BODY]]:
+; CHECK-TF-ONLYRED-NEXT: [[TMP15:%.*]] = phi i32 [ [[TMP16:%.*]], %[[FOR_BODY]] ], [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ]
+; CHECK-TF-ONLYRED-NEXT: [[I_010:%.*]] = phi i64 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-TF-ONLYRED-NEXT: [[ADD]] = add nuw nsw i64 [[I_010]], 1
+; CHECK-TF-ONLYRED-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[ADD]]
+; CHECK-TF-ONLYRED-NEXT: [[TMP16]] = load i32, ptr [[ARRAYIDX1]], align 4
+; CHECK-TF-ONLYRED-NEXT: [[ADD2:%.*]] = add nsw i32 [[TMP16]], [[TMP15]]
+; CHECK-TF-ONLYRED-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[I_010]]
+; CHECK-TF-ONLYRED-NEXT: store i32 [[ADD2]], ptr [[ARRAYIDX3]], align 4
+; CHECK-TF-ONLYRED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[ADD]], [[N]]
+; CHECK-TF-ONLYRED-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-TF-ONLYRED: [[FOR_END]]:
+; CHECK-TF-ONLYRED-NEXT: ret void
+;
+; CHECK-NEOVERSE-V1-LABEL: define void @add_recur(
+; CHECK-NEOVERSE-V1-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEOVERSE-V1-NEXT: [[ENTRY:.*]]:
+; CHECK-NEOVERSE-V1-NEXT: [[DOTPRE:%.*]] = load i32, ptr [[SRC]], align 4
+; CHECK-NEOVERSE-V1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEOVERSE-V1-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEOVERSE-V1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NEOVERSE-V1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEOVERSE-V1: [[VECTOR_PH]]:
+; CHECK-NEOVERSE-V1-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEOVERSE-V1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
+; CHECK-NEOVERSE-V1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEOVERSE-V1-NEXT: [[TMP4:%.*]] = mul nuw i32 [[TMP3]], 4
+; CHECK-NEOVERSE-V1-NEXT: [[TMP5:%.*]] = sub i32 [[TMP4]], 1
+; CHECK-NEOVERSE-V1-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[DOTPRE]], i32 [[TMP5]]
+; CHECK-NEOVERSE-V1-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-NEOVERSE-V1: [[VECTOR_BODY]]:
+; CHECK-NEOVERSE-V1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEOVERSE-V1-NEXT: [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP6:%.*]] = add nuw nsw i64 [[INDEX]], 1
+; CHECK-NEOVERSE-V1-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP6]]
+; CHECK-NEOVERSE-V1-NEXT: [[WIDE_LOAD]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
+; CHECK-NEOVERSE-V1-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR]], <vscale x 4 x i32> [[WIDE_LOAD]], i32 1)
+; CHECK-NEOVERSE-V1-NEXT: [[TMP9:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_LOAD]], [[TMP8]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEOVERSE-V1-NEXT: store <vscale x 4 x i32> [[TMP9]], ptr [[TMP10]], align 4
+; CHECK-NEOVERSE-V1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEOVERSE-V1-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEOVERSE-V1: [[MIDDLE_BLOCK]]:
+; CHECK-NEOVERSE-V1-NEXT: [[TMP12:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEOVERSE-V1-NEXT: [[TMP13:%.*]] = mul nuw i32 [[TMP12]], 4
+; CHECK-NEOVERSE-V1-NEXT: [[TMP14:%.*]] = sub i32 [[TMP13]], 1
+; CHECK-NEOVERSE-V1-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_LOAD]], i32 [[TMP14]]
+; CHECK-NEOVERSE-V1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEOVERSE-V1-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-NEOVERSE-V1: [[SCALAR_PH]]:
+; CHECK-NEOVERSE-V1-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ [[DOTPRE]], %[[ENTRY]] ]
+; CHECK-NEOVERSE-V1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEOVERSE-V1-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-NEOVERSE-V1: [[FOR_BODY]]:
+; CHECK-NEOVERSE-V1-NEXT: [[TMP15:%.*]] = phi i32 [ [[TMP16:%.*]], %[[FOR_BODY]] ], [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ]
+; CHECK-NEOVERSE-V1-NEXT: [[I_010:%.*]] = phi i64 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEOVERSE-V1-NEXT: [[ADD]] = add nuw nsw i64 [[I_010]], 1
+; CHECK-NEOVERSE-V1-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[ADD]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP16]] = load i32, ptr [[ARRAYIDX1]], align 4
+; CHECK-NEOVERSE-V1-NEXT: [[ADD2:%.*]] = add nsw i32 [[TMP16]], [[TMP15]]
+; CHECK-NEOVERSE-V1-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[I_010]]
+; CHECK-NEOVERSE-V1-NEXT: store i32 [[ADD2]], ptr [[ARRAYIDX3]], align 4
+; CHECK-NEOVERSE-V1-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[ADD]], [[N]]
+; CHECK-NEOVERSE-V1-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEOVERSE-V1: [[FOR_END]]:
+; CHECK-NEOVERSE-V1-NEXT: ret void
+;
+
+
+
+
+
+
entry:
%.pre = load i32, ptr %src, align 4
@@ -281,41 +1046,701 @@ for.end:
}
define void @interleave(ptr noalias %dst, ptr noalias %src, i64 %n) #0 {
-; CHECK-NOTF-LABEL: @interleave(
-; CHECK-NOTF: vector.body:
-; CHECK-NOTF: %[[LOAD:.*]] = load <8 x float>, ptr
-; CHECK-NOTF: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NOTF: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-
-; CHECK-TF-LABEL: @interleave(
-; CHECK-TF: vector.body:
-; CHECK-TF: %[[LOAD:.*]] = load <8 x float>, ptr
-; CHECK-TF: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-TF: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-
-; CHECK-TF-NORED-LABEL: @interleave(
-; CHECK-TF-NORED: vector.body:
-; CHECK-TF-NORED: %[[LOAD:.*]] = load <8 x float>, ptr
-; CHECK-TF-NORED: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-TF-NORED: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-
-; CHECK-TF-NOREC-LABEL: @interleave(
-; CHECK-TF-NOREC: vector.body:
-; CHECK-TF-NOREC: %[[LOAD:.*]] = load <8 x float>, ptr
-; CHECK-TF-NOREC: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-TF-NOREC: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-
-; CHECK-TF-NOREV-LABEL: @interleave(
-; CHECK-TF-NOREV: vector.body:
-; CHECK-TF-NOREV: %[[LOAD:.*]] = load <8 x float>, ptr
-; CHECK-TF-NOREV: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-TF-NOREV: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-
-; CHECK-NEOVERSE-V1-LABEL: @interleave(
-; CHECK-NEOVERSE-V1: vector.body:
-; CHECK-NEOVERSE-V1: %[[LOAD:.*]] = load <8 x float>, ptr
-; CHECK-NEOVERSE-V1: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEOVERSE-V1: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NOTF-LABEL: define void @interleave(
+; CHECK-NOTF-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NOTF-NEXT: [[ENTRY:.*]]:
+; CHECK-NOTF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NOTF-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-NOTF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NOTF-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NOTF: [[VECTOR_PH]]:
+; CHECK-NOTF-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NOTF-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-NOTF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-NOTF-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NOTF-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-NOTF: [[VECTOR_BODY]]:
+; CHECK-NOTF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NOTF-NEXT: [[TMP4:%.*]] = add i64 [[TMP2]], 0
+; CHECK-NOTF-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 1
+; CHECK-NOTF-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], [[TMP5]]
+; CHECK-NOTF-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[INDEX]], 1
+; CHECK-NOTF-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP6]], 1
+; CHECK-NOTF-NEXT: [[TMP9:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP7]]
+; CHECK-NOTF-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP8]]
+; CHECK-NOTF-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 8 x float>, ptr [[TMP9]], align 4
+; CHECK-NOTF-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC]])
+; CHECK-NOTF-NEXT: [[TMP11:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
+; CHECK-NOTF-NEXT: [[TMP12:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 1
+; CHECK-NOTF-NEXT: [[WIDE_VEC1:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
+; CHECK-NOTF-NEXT: [[STRIDED_VEC2:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC1]])
+; CHECK-NOTF-NEXT: [[TMP13:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC2]], 0
+; CHECK-NOTF-NEXT: [[TMP14:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC2]], 1
+; CHECK-NOTF-NEXT: [[TMP15:%.*]] = mul nuw nsw i64 [[INDEX]], 3
+; CHECK-NOTF-NEXT: [[TMP16:%.*]] = mul nuw nsw i64 [[TMP6]], 3
+; CHECK-NOTF-NEXT: [[TMP17:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP15]]
+; CHECK-NOTF-NEXT: [[TMP18:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP16]]
+; CHECK-NOTF-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP11]], <vscale x 4 x float> [[TMP12]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-NOTF-NEXT: store <vscale x 12 x float> [[INTERLEAVED_VEC]], ptr [[TMP17]], align 4
+; CHECK-NOTF-NEXT: [[INTERLEAVED_VEC3:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP13]], <vscale x 4 x float> [[TMP14]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-NOTF-NEXT: store <vscale x 12 x float> [[INTERLEAVED_VEC3]], ptr [[TMP18]], align 4
+; CHECK-NOTF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-NOTF-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NOTF-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NOTF: [[MIDDLE_BLOCK]]:
+; CHECK-NOTF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NOTF-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-NOTF: [[SCALAR_PH]]:
+; CHECK-NOTF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NOTF-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-NOTF: [[FOR_BODY]]:
+; CHECK-NOTF-NEXT: [[I_021:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NOTF-NEXT: [[MUL:%.*]] = shl nuw nsw i64 [[I_021]], 1
+; CHECK-NOTF-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[MUL]]
+; CHECK-NOTF-NEXT: [[TMP20:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-NOTF-NEXT: [[MUL1:%.*]] = mul nuw nsw i64 [[I_021]], 3
+; CHECK-NOTF-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[MUL1]]
+; CHECK-NOTF-NEXT: store float [[TMP20]], ptr [[ARRAYIDX2]], align 4
+; CHECK-NOTF-NEXT: [[ADD:%.*]] = or disjoint i64 [[MUL]], 1
+; CHECK-NOTF-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[ADD]]
+; CHECK-NOTF-NEXT: [[TMP21:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-NOTF-NEXT: [[ADD6:%.*]] = add nuw nsw i64 [[MUL1]], 1
+; CHECK-NOTF-NEXT: [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD6]]
+; CHECK-NOTF-NEXT: store float [[TMP21]], ptr [[ARRAYIDX7]], align 4
+; CHECK-NOTF-NEXT: [[ADD9:%.*]] = add nuw nsw i64 [[MUL1]], 2
+; CHECK-NOTF-NEXT: [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD9]]
+; CHECK-NOTF-NEXT: store float 3.000000e+00, ptr [[ARRAYIDX10]], align 4
+; CHECK-NOTF-NEXT: [[INC]] = add nuw nsw i64 [[I_021]], 1
+; CHECK-NOTF-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
+; CHECK-NOTF-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NOTF: [[FOR_END]]:
+; CHECK-NOTF-NEXT: ret void
+;
+; CHECK-TF-ALL-LABEL: define void @interleave(
+; CHECK-TF-ALL-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-ALL-NEXT: [[ENTRY:.*]]:
+; CHECK-TF-ALL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-ALL-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-TF-ALL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-TF-ALL-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF-ALL: [[VECTOR_PH]]:
+; CHECK-TF-ALL-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-ALL-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-TF-ALL-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-TF-ALL-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-TF-ALL-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-ALL: [[VECTOR_BODY]]:
+; CHECK-TF-ALL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ALL-NEXT: [[TMP4:%.*]] = add i64 [[TMP2]], 0
+; CHECK-TF-ALL-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 1
+; CHECK-TF-ALL-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], [[TMP5]]
+; CHECK-TF-ALL-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-ALL-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP6]], 1
+; CHECK-TF-ALL-NEXT: [[TMP9:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP7]]
+; CHECK-TF-ALL-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP8]]
+; CHECK-TF-ALL-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 8 x float>, ptr [[TMP9]], align 4
+; CHECK-TF-ALL-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC]])
+; CHECK-TF-ALL-NEXT: [[TMP11:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
+; CHECK-TF-ALL-NEXT: [[TMP12:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 1
+; CHECK-TF-ALL-NEXT: [[WIDE_VEC1:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
+; CHECK-TF-ALL-NEXT: [[STRIDED_VEC2:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC1]])
+; CHECK-TF-ALL-NEXT: [[TMP13:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC2]], 0
+; CHECK-TF-ALL-NEXT: [[TMP14:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC2]], 1
+; CHECK-TF-ALL-NEXT: [[TMP15:%.*]] = mul nuw nsw i64 [[INDEX]], 3
+; CHECK-TF-ALL-NEXT: [[TMP16:%.*]] = mul nuw nsw i64 [[TMP6]], 3
+; CHECK-TF-ALL-NEXT: [[TMP17:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP15]]
+; CHECK-TF-ALL-NEXT: [[TMP18:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP16]]
+; CHECK-TF-ALL-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP11]], <vscale x 4 x float> [[TMP12]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-TF-ALL-NEXT: store <vscale x 12 x float> [[INTERLEAVED_VEC]], ptr [[TMP17]], align 4
+; CHECK-TF-ALL-NEXT: [[INTERLEAVED_VEC3:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP13]], <vscale x 4 x float> [[TMP14]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-TF-ALL-NEXT: store <vscale x 12 x float> [[INTERLEAVED_VEC3]], ptr [[TMP18]], align 4
+; CHECK-TF-ALL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-TF-ALL-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-ALL-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-TF-ALL: [[MIDDLE_BLOCK]]:
+; CHECK-TF-ALL-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-TF-ALL-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-TF-ALL: [[SCALAR_PH]]:
+; CHECK-TF-ALL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-TF-ALL-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-TF-ALL: [[FOR_BODY]]:
+; CHECK-TF-ALL-NEXT: [[I_021:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-TF-ALL-NEXT: [[MUL:%.*]] = shl nuw nsw i64 [[I_021]], 1
+; CHECK-TF-ALL-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[MUL]]
+; CHECK-TF-ALL-NEXT: [[TMP20:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-TF-ALL-NEXT: [[MUL1:%.*]] = mul nuw nsw i64 [[I_021]], 3
+; CHECK-TF-ALL-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[MUL1]]
+; CHECK-TF-ALL-NEXT: store float [[TMP20]], ptr [[ARRAYIDX2]], align 4
+; CHECK-TF-ALL-NEXT: [[ADD:%.*]] = or disjoint i64 [[MUL]], 1
+; CHECK-TF-ALL-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[ADD]]
+; CHECK-TF-ALL-NEXT: [[TMP21:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-TF-ALL-NEXT: [[ADD6:%.*]] = add nuw nsw i64 [[MUL1]], 1
+; CHECK-TF-ALL-NEXT: [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD6]]
+; CHECK-TF-ALL-NEXT: store float [[TMP21]], ptr [[ARRAYIDX7]], align 4
+; CHECK-TF-ALL-NEXT: [[ADD9:%.*]] = add nuw nsw i64 [[MUL1]], 2
+; CHECK-TF-ALL-NEXT: [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD9]]
+; CHECK-TF-ALL-NEXT: store float 3.000000e+00, ptr [[ARRAYIDX10]], align 4
+; CHECK-TF-ALL-NEXT: [[INC]] = add nuw nsw i64 [[I_021]], 1
+; CHECK-TF-ALL-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
+; CHECK-TF-ALL-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-TF-ALL: [[FOR_END]]:
+; CHECK-TF-ALL-NEXT: ret void
+;
+; CHECK-TF-LABEL: define void @interleave(
+; CHECK-TF-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NEXT: [[ENTRY:.*]]:
+; CHECK-TF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-TF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-TF-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF: [[VECTOR_PH]]:
+; CHECK-TF-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-TF-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF: [[VECTOR_BODY]]:
+; CHECK-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[TMP4:%.*]] = add i64 [[TMP2]], 0
+; CHECK-TF-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 1
+; CHECK-TF-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], [[TMP5]]
+; CHECK-TF-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP6]], 1
+; CHECK-TF-NEXT: [[TMP9:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP7]]
+; CHECK-TF-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP8]]
+; CHECK-TF-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 8 x float>, ptr [[TMP9]], align 4
+; CHECK-TF-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC]])
+; CHECK-TF-NEXT: [[TMP11:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
+; CHECK-TF-NEXT: [[TMP12:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 1
+; CHECK-TF-NEXT: [[WIDE_VEC1:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
+; CHECK-TF-NEXT: [[STRIDED_VEC2:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC1]])
+; CHECK-TF-NEXT: [[TMP13:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC2]], 0
+; CHECK-TF-NEXT: [[TMP14:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC2]], 1
+; CHECK-TF-NEXT: [[TMP15:%.*]] = mul nuw nsw i64 [[INDEX]], 3
+; CHECK-TF-NEXT: [[TMP16:%.*]] = mul nuw nsw i64 [[TMP6]], 3
+; CHECK-TF-NEXT: [[TMP17:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP15]]
+; CHECK-TF-NEXT: [[TMP18:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP16]]
+; CHECK-TF-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP11]], <vscale x 4 x float> [[TMP12]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-TF-NEXT: store <vscale x 12 x float> [[INTERLEAVED_VEC]], ptr [[TMP17]], align 4
+; CHECK-TF-NEXT: [[INTERLEAVED_VEC3:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP13]], <vscale x 4 x float> [[TMP14]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-TF-NEXT: store <vscale x 12 x float> [[INTERLEAVED_VEC3]], ptr [[TMP18]], align 4
+; CHECK-TF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-TF-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-TF: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-TF-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-TF: [[SCALAR_PH]]:
+; CHECK-TF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-TF-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-TF: [[FOR_BODY]]:
+; CHECK-TF-NEXT: [[I_021:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-TF-NEXT: [[MUL:%.*]] = shl nuw nsw i64 [[I_021]], 1
+; CHECK-TF-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[MUL]]
+; CHECK-TF-NEXT: [[TMP20:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-TF-NEXT: [[MUL1:%.*]] = mul nuw nsw i64 [[I_021]], 3
+; CHECK-TF-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[MUL1]]
+; CHECK-TF-NEXT: store float [[TMP20]], ptr [[ARRAYIDX2]], align 4
+; CHECK-TF-NEXT: [[ADD:%.*]] = or disjoint i64 [[MUL]], 1
+; CHECK-TF-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[ADD]]
+; CHECK-TF-NEXT: [[TMP21:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-TF-NEXT: [[ADD6:%.*]] = add nuw nsw i64 [[MUL1]], 1
+; CHECK-TF-NEXT: [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD6]]
+; CHECK-TF-NEXT: store float [[TMP21]], ptr [[ARRAYIDX7]], align 4
+; CHECK-TF-NEXT: [[ADD9:%.*]] = add nuw nsw i64 [[MUL1]], 2
+; CHECK-TF-NEXT: [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD9]]
+; CHECK-TF-NEXT: store float 3.000000e+00, ptr [[ARRAYIDX10]], align 4
+; CHECK-TF-NEXT: [[INC]] = add nuw nsw i64 [[I_021]], 1
+; CHECK-TF-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
+; CHECK-TF-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-TF: [[FOR_END]]:
+; CHECK-TF-NEXT: ret void
+;
+; CHECK-TF-DEFAULT-LABEL: define void @interleave(
+; CHECK-TF-DEFAULT-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-DEFAULT-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-TF-DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-TF-DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK-TF-DEFAULT: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-TF-DEFAULT-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-DEFAULT-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-TF-DEFAULT-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-TF-DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF-DEFAULT: [[VECTOR_PH]]:
+; CHECK-TF-DEFAULT-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-DEFAULT-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-TF-DEFAULT-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-TF-DEFAULT-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-TF-DEFAULT-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-DEFAULT: [[VECTOR_BODY]]:
+; CHECK-TF-DEFAULT-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-DEFAULT-NEXT: [[TMP4:%.*]] = add i64 [[TMP2]], 0
+; CHECK-TF-DEFAULT-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 1
+; CHECK-TF-DEFAULT-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], [[TMP5]]
+; CHECK-TF-DEFAULT-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-DEFAULT-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP6]], 1
+; CHECK-TF-DEFAULT-NEXT: [[TMP9:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP7]]
+; CHECK-TF-DEFAULT-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP8]]
+; CHECK-TF-DEFAULT-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 8 x float>, ptr [[TMP9]], align 4
+; CHECK-TF-DEFAULT-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC]])
+; CHECK-TF-DEFAULT-NEXT: [[TMP11:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
+; CHECK-TF-DEFAULT-NEXT: [[TMP12:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 1
+; CHECK-TF-DEFAULT-NEXT: [[WIDE_VEC2:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
+; CHECK-TF-DEFAULT-NEXT: [[STRIDED_VEC3:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC2]])
+; CHECK-TF-DEFAULT-NEXT: [[TMP13:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC3]], 0
+; CHECK-TF-DEFAULT-NEXT: [[TMP14:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC3]], 1
+; CHECK-TF-DEFAULT-NEXT: [[TMP15:%.*]] = mul nuw nsw i64 [[INDEX]], 3
+; CHECK-TF-DEFAULT-NEXT: [[TMP16:%.*]] = mul nuw nsw i64 [[TMP6]], 3
+; CHECK-TF-DEFAULT-NEXT: [[TMP17:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP15]]
+; CHECK-TF-DEFAULT-NEXT: [[TMP18:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP16]]
+; CHECK-TF-DEFAULT-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP11]], <vscale x 4 x float> [[TMP12]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-TF-DEFAULT-NEXT: store <vscale x 12 x float> [[INTERLEAVED_VEC]], ptr [[TMP17]], align 4
+; CHECK-TF-DEFAULT-NEXT: [[INTERLEAVED_VEC4:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP13]], <vscale x 4 x float> [[TMP14]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-TF-DEFAULT-NEXT: store <vscale x 12 x float> [[INTERLEAVED_VEC4]], ptr [[TMP18]], align 4
+; CHECK-TF-DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-TF-DEFAULT-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-DEFAULT-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-TF-DEFAULT: [[MIDDLE_BLOCK]]:
+; CHECK-TF-DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-TF-DEFAULT-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK-TF-DEFAULT: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-TF-DEFAULT-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
+; CHECK-TF-DEFAULT-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF6:![0-9]+]]
+; CHECK-TF-DEFAULT: [[VEC_EPILOG_PH]]:
+; CHECK-TF-DEFAULT-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-TF-DEFAULT-NEXT: [[N_MOD_VF5:%.*]] = urem i64 [[N]], 4
+; CHECK-TF-DEFAULT-NEXT: [[N_VEC6:%.*]] = sub i64 [[N]], [[N_MOD_VF5]]
+; CHECK-TF-DEFAULT-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-TF-DEFAULT: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-TF-DEFAULT-NEXT: [[INDEX7:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT12:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-TF-DEFAULT-NEXT: [[TMP20:%.*]] = shl nuw nsw i64 [[INDEX7]], 1
+; CHECK-TF-DEFAULT-NEXT: [[TMP21:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP20]]
+; CHECK-TF-DEFAULT-NEXT: [[WIDE_VEC8:%.*]] = load <8 x float>, ptr [[TMP21]], align 4
+; CHECK-TF-DEFAULT-NEXT: [[STRIDED_VEC9:%.*]] = shufflevector <8 x float> [[WIDE_VEC8]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-TF-DEFAULT-NEXT: [[STRIDED_VEC10:%.*]] = shufflevector <8 x float> [[WIDE_VEC8]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-TF-DEFAULT-NEXT: [[TMP22:%.*]] = mul nuw nsw i64 [[INDEX7]], 3
+; CHECK-TF-DEFAULT-NEXT: [[TMP23:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP22]]
+; CHECK-TF-DEFAULT-NEXT: [[TMP24:%.*]] = shufflevector <4 x float> [[STRIDED_VEC9]], <4 x float> [[STRIDED_VEC10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-TF-DEFAULT-NEXT: [[TMP25:%.*]] = shufflevector <8 x float> [[TMP24]], <8 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float undef, float undef, float undef, float undef>, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; CHECK-TF-DEFAULT-NEXT: [[INTERLEAVED_VEC11:%.*]] = shufflevector <12 x float> [[TMP25]], <12 x float> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; CHECK-TF-DEFAULT-NEXT: store <12 x float> [[INTERLEAVED_VEC11]], ptr [[TMP23]], align 4
+; CHECK-TF-DEFAULT-NEXT: [[INDEX_NEXT12]] = add nuw i64 [[INDEX7]], 4
+; CHECK-TF-DEFAULT-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT12]], [[N_VEC6]]
+; CHECK-TF-DEFAULT-NEXT: br i1 [[TMP26]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-TF-DEFAULT: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-TF-DEFAULT-NEXT: [[CMP_N13:%.*]] = icmp eq i64 [[N]], [[N_VEC6]]
+; CHECK-TF-DEFAULT-NEXT: br i1 [[CMP_N13]], label %[[FOR_END]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK-TF-DEFAULT: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-TF-DEFAULT-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC6]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-TF-DEFAULT-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-TF-DEFAULT: [[FOR_BODY]]:
+; CHECK-TF-DEFAULT-NEXT: [[I_021:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; CHECK-TF-DEFAULT-NEXT: [[MUL:%.*]] = shl nuw nsw i64 [[I_021]], 1
+; CHECK-TF-DEFAULT-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[MUL]]
+; CHECK-TF-DEFAULT-NEXT: [[TMP27:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-TF-DEFAULT-NEXT: [[MUL1:%.*]] = mul nuw nsw i64 [[I_021]], 3
+; CHECK-TF-DEFAULT-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[MUL1]]
+; CHECK-TF-DEFAULT-NEXT: store float [[TMP27]], ptr [[ARRAYIDX2]], align 4
+; CHECK-TF-DEFAULT-NEXT: [[ADD:%.*]] = or disjoint i64 [[MUL]], 1
+; CHECK-TF-DEFAULT-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[ADD]]
+; CHECK-TF-DEFAULT-NEXT: [[TMP28:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-TF-DEFAULT-NEXT: [[ADD6:%.*]] = add nuw nsw i64 [[MUL1]], 1
+; CHECK-TF-DEFAULT-NEXT: [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD6]]
+; CHECK-TF-DEFAULT-NEXT: store float [[TMP28]], ptr [[ARRAYIDX7]], align 4
+; CHECK-TF-DEFAULT-NEXT: [[ADD9:%.*]] = add nuw nsw i64 [[MUL1]], 2
+; CHECK-TF-DEFAULT-NEXT: [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD9]]
+; CHECK-TF-DEFAULT-NEXT: store float 3.000000e+00, ptr [[ARRAYIDX10]], align 4
+; CHECK-TF-DEFAULT-NEXT: [[INC]] = add nuw nsw i64 [[I_021]], 1
+; CHECK-TF-DEFAULT-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
+; CHECK-TF-DEFAULT-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-TF-DEFAULT: [[FOR_END]]:
+; CHECK-TF-DEFAULT-NEXT: ret void
+;
+; CHECK-TF-NORED-LABEL: define void @interleave(
+; CHECK-TF-NORED-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NORED-NEXT: [[ENTRY:.*]]:
+; CHECK-TF-NORED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NORED-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-TF-NORED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-TF-NORED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF-NORED: [[VECTOR_PH]]:
+; CHECK-TF-NORED-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NORED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-TF-NORED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-TF-NORED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-TF-NORED-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-NORED: [[VECTOR_BODY]]:
+; CHECK-TF-NORED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NORED-NEXT: [[TMP4:%.*]] = add i64 [[TMP2]], 0
+; CHECK-TF-NORED-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 1
+; CHECK-TF-NORED-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], [[TMP5]]
+; CHECK-TF-NORED-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-NORED-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP6]], 1
+; CHECK-TF-NORED-NEXT: [[TMP9:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP7]]
+; CHECK-TF-NORED-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP8]]
+; CHECK-TF-NORED-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 8 x float>, ptr [[TMP9]], align 4
+; CHECK-TF-NORED-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC]])
+; CHECK-TF-NORED-NEXT: [[TMP11:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
+; CHECK-TF-NORED-NEXT: [[TMP12:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 1
+; CHECK-TF-NORED-NEXT: [[WIDE_VEC1:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
+; CHECK-TF-NORED-NEXT: [[STRIDED_VEC2:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC1]])
+; CHECK-TF-NORED-NEXT: [[TMP13:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC2]], 0
+; CHECK-TF-NORED-NEXT: [[TMP14:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC2]], 1
+; CHECK-TF-NORED-NEXT: [[TMP15:%.*]] = mul nuw nsw i64 [[INDEX]], 3
+; CHECK-TF-NORED-NEXT: [[TMP16:%.*]] = mul nuw nsw i64 [[TMP6]], 3
+; CHECK-TF-NORED-NEXT: [[TMP17:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP15]]
+; CHECK-TF-NORED-NEXT: [[TMP18:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP16]]
+; CHECK-TF-NORED-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP11]], <vscale x 4 x float> [[TMP12]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-TF-NORED-NEXT: store <vscale x 12 x float> [[INTERLEAVED_VEC]], ptr [[TMP17]], align 4
+; CHECK-TF-NORED-NEXT: [[INTERLEAVED_VEC3:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP13]], <vscale x 4 x float> [[TMP14]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-TF-NORED-NEXT: store <vscale x 12 x float> [[INTERLEAVED_VEC3]], ptr [[TMP18]], align 4
+; CHECK-TF-NORED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-TF-NORED-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-NORED-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-TF-NORED: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NORED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-TF-NORED-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-TF-NORED: [[SCALAR_PH]]:
+; CHECK-TF-NORED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-TF-NORED-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-TF-NORED: [[FOR_BODY]]:
+; CHECK-TF-NORED-NEXT: [[I_021:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-TF-NORED-NEXT: [[MUL:%.*]] = shl nuw nsw i64 [[I_021]], 1
+; CHECK-TF-NORED-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[MUL]]
+; CHECK-TF-NORED-NEXT: [[TMP20:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-TF-NORED-NEXT: [[MUL1:%.*]] = mul nuw nsw i64 [[I_021]], 3
+; CHECK-TF-NORED-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[MUL1]]
+; CHECK-TF-NORED-NEXT: store float [[TMP20]], ptr [[ARRAYIDX2]], align 4
+; CHECK-TF-NORED-NEXT: [[ADD:%.*]] = or disjoint i64 [[MUL]], 1
+; CHECK-TF-NORED-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[ADD]]
+; CHECK-TF-NORED-NEXT: [[TMP21:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-TF-NORED-NEXT: [[ADD6:%.*]] = add nuw nsw i64 [[MUL1]], 1
+; CHECK-TF-NORED-NEXT: [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD6]]
+; CHECK-TF-NORED-NEXT: store float [[TMP21]], ptr [[ARRAYIDX7]], align 4
+; CHECK-TF-NORED-NEXT: [[ADD9:%.*]] = add nuw nsw i64 [[MUL1]], 2
+; CHECK-TF-NORED-NEXT: [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD9]]
+; CHECK-TF-NORED-NEXT: store float 3.000000e+00, ptr [[ARRAYIDX10]], align 4
+; CHECK-TF-NORED-NEXT: [[INC]] = add nuw nsw i64 [[I_021]], 1
+; CHECK-TF-NORED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
+; CHECK-TF-NORED-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-TF-NORED: [[FOR_END]]:
+; CHECK-TF-NORED-NEXT: ret void
+;
+; CHECK-TF-NOREC-LABEL: define void @interleave(
+; CHECK-TF-NOREC-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NOREC-NEXT: [[ENTRY:.*]]:
+; CHECK-TF-NOREC-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NOREC-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-TF-NOREC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-TF-NOREC-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF-NOREC: [[VECTOR_PH]]:
+; CHECK-TF-NOREC-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NOREC-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-TF-NOREC-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-TF-NOREC-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-TF-NOREC-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-NOREC: [[VECTOR_BODY]]:
+; CHECK-TF-NOREC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREC-NEXT: [[TMP4:%.*]] = add i64 [[TMP2]], 0
+; CHECK-TF-NOREC-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 1
+; CHECK-TF-NOREC-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], [[TMP5]]
+; CHECK-TF-NOREC-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-NOREC-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP6]], 1
+; CHECK-TF-NOREC-NEXT: [[TMP9:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP7]]
+; CHECK-TF-NOREC-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP8]]
+; CHECK-TF-NOREC-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 8 x float>, ptr [[TMP9]], align 4
+; CHECK-TF-NOREC-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC]])
+; CHECK-TF-NOREC-NEXT: [[TMP11:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
+; CHECK-TF-NOREC-NEXT: [[TMP12:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 1
+; CHECK-TF-NOREC-NEXT: [[WIDE_VEC1:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
+; CHECK-TF-NOREC-NEXT: [[STRIDED_VEC2:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC1]])
+; CHECK-TF-NOREC-NEXT: [[TMP13:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC2]], 0
+; CHECK-TF-NOREC-NEXT: [[TMP14:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC2]], 1
+; CHECK-TF-NOREC-NEXT: [[TMP15:%.*]] = mul nuw nsw i64 [[INDEX]], 3
+; CHECK-TF-NOREC-NEXT: [[TMP16:%.*]] = mul nuw nsw i64 [[TMP6]], 3
+; CHECK-TF-NOREC-NEXT: [[TMP17:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP15]]
+; CHECK-TF-NOREC-NEXT: [[TMP18:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP16]]
+; CHECK-TF-NOREC-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP11]], <vscale x 4 x float> [[TMP12]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-TF-NOREC-NEXT: store <vscale x 12 x float> [[INTERLEAVED_VEC]], ptr [[TMP17]], align 4
+; CHECK-TF-NOREC-NEXT: [[INTERLEAVED_VEC3:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP13]], <vscale x 4 x float> [[TMP14]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-TF-NOREC-NEXT: store <vscale x 12 x float> [[INTERLEAVED_VEC3]], ptr [[TMP18]], align 4
+; CHECK-TF-NOREC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-TF-NOREC-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-NOREC-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-TF-NOREC: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NOREC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-TF-NOREC-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-TF-NOREC: [[SCALAR_PH]]:
+; CHECK-TF-NOREC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-TF-NOREC-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-TF-NOREC: [[FOR_BODY]]:
+; CHECK-TF-NOREC-NEXT: [[I_021:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-TF-NOREC-NEXT: [[MUL:%.*]] = shl nuw nsw i64 [[I_021]], 1
+; CHECK-TF-NOREC-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[MUL]]
+; CHECK-TF-NOREC-NEXT: [[TMP20:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-TF-NOREC-NEXT: [[MUL1:%.*]] = mul nuw nsw i64 [[I_021]], 3
+; CHECK-TF-NOREC-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[MUL1]]
+; CHECK-TF-NOREC-NEXT: store float [[TMP20]], ptr [[ARRAYIDX2]], align 4
+; CHECK-TF-NOREC-NEXT: [[ADD:%.*]] = or disjoint i64 [[MUL]], 1
+; CHECK-TF-NOREC-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[ADD]]
+; CHECK-TF-NOREC-NEXT: [[TMP21:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-TF-NOREC-NEXT: [[ADD6:%.*]] = add nuw nsw i64 [[MUL1]], 1
+; CHECK-TF-NOREC-NEXT: [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD6]]
+; CHECK-TF-NOREC-NEXT: store float [[TMP21]], ptr [[ARRAYIDX7]], align 4
+; CHECK-TF-NOREC-NEXT: [[ADD9:%.*]] = add nuw nsw i64 [[MUL1]], 2
+; CHECK-TF-NOREC-NEXT: [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD9]]
+; CHECK-TF-NOREC-NEXT: store float 3.000000e+00, ptr [[ARRAYIDX10]], align 4
+; CHECK-TF-NOREC-NEXT: [[INC]] = add nuw nsw i64 [[I_021]], 1
+; CHECK-TF-NOREC-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
+; CHECK-TF-NOREC-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-TF-NOREC: [[FOR_END]]:
+; CHECK-TF-NOREC-NEXT: ret void
+;
+; CHECK-TF-NOREV-LABEL: define void @interleave(
+; CHECK-TF-NOREV-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NOREV-NEXT: [[ENTRY:.*]]:
+; CHECK-TF-NOREV-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NOREV-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-TF-NOREV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-TF-NOREV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF-NOREV: [[VECTOR_PH]]:
+; CHECK-TF-NOREV-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NOREV-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-TF-NOREV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-TF-NOREV-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-TF-NOREV-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-NOREV: [[VECTOR_BODY]]:
+; CHECK-TF-NOREV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREV-NEXT: [[TMP4:%.*]] = add i64 [[TMP2]], 0
+; CHECK-TF-NOREV-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 1
+; CHECK-TF-NOREV-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], [[TMP5]]
+; CHECK-TF-NOREV-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-NOREV-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP6]], 1
+; CHECK-TF-NOREV-NEXT: [[TMP9:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP7]]
+; CHECK-TF-NOREV-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP8]]
+; CHECK-TF-NOREV-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 8 x float>, ptr [[TMP9]], align 4
+; CHECK-TF-NOREV-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC]])
+; CHECK-TF-NOREV-NEXT: [[TMP11:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
+; CHECK-TF-NOREV-NEXT: [[TMP12:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 1
+; CHECK-TF-NOREV-NEXT: [[WIDE_VEC1:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
+; CHECK-TF-NOREV-NEXT: [[STRIDED_VEC2:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC1]])
+; CHECK-TF-NOREV-NEXT: [[TMP13:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC2]], 0
+; CHECK-TF-NOREV-NEXT: [[TMP14:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC2]], 1
+; CHECK-TF-NOREV-NEXT: [[TMP15:%.*]] = mul nuw nsw i64 [[INDEX]], 3
+; CHECK-TF-NOREV-NEXT: [[TMP16:%.*]] = mul nuw nsw i64 [[TMP6]], 3
+; CHECK-TF-NOREV-NEXT: [[TMP17:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP15]]
+; CHECK-TF-NOREV-NEXT: [[TMP18:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP16]]
+; CHECK-TF-NOREV-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP11]], <vscale x 4 x float> [[TMP12]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-TF-NOREV-NEXT: store <vscale x 12 x float> [[INTERLEAVED_VEC]], ptr [[TMP17]], align 4
+; CHECK-TF-NOREV-NEXT: [[INTERLEAVED_VEC3:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP13]], <vscale x 4 x float> [[TMP14]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-TF-NOREV-NEXT: store <vscale x 12 x float> [[INTERLEAVED_VEC3]], ptr [[TMP18]], align 4
+; CHECK-TF-NOREV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-TF-NOREV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-NOREV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-TF-NOREV: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NOREV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-TF-NOREV-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-TF-NOREV: [[SCALAR_PH]]:
+; CHECK-TF-NOREV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-TF-NOREV-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-TF-NOREV: [[FOR_BODY]]:
+; CHECK-TF-NOREV-NEXT: [[I_021:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-TF-NOREV-NEXT: [[MUL:%.*]] = shl nuw nsw i64 [[I_021]], 1
+; CHECK-TF-NOREV-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[MUL]]
+; CHECK-TF-NOREV-NEXT: [[TMP20:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-TF-NOREV-NEXT: [[MUL1:%.*]] = mul nuw nsw i64 [[I_021]], 3
+; CHECK-TF-NOREV-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[MUL1]]
+; CHECK-TF-NOREV-NEXT: store float [[TMP20]], ptr [[ARRAYIDX2]], align 4
+; CHECK-TF-NOREV-NEXT: [[ADD:%.*]] = or disjoint i64 [[MUL]], 1
+; CHECK-TF-NOREV-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[ADD]]
+; CHECK-TF-NOREV-NEXT: [[TMP21:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-TF-NOREV-NEXT: [[ADD6:%.*]] = add nuw nsw i64 [[MUL1]], 1
+; CHECK-TF-NOREV-NEXT: [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD6]]
+; CHECK-TF-NOREV-NEXT: store float [[TMP21]], ptr [[ARRAYIDX7]], align 4
+; CHECK-TF-NOREV-NEXT: [[ADD9:%.*]] = add nuw nsw i64 [[MUL1]], 2
+; CHECK-TF-NOREV-NEXT: [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD9]]
+; CHECK-TF-NOREV-NEXT: store float 3.000000e+00, ptr [[ARRAYIDX10]], align 4
+; CHECK-TF-NOREV-NEXT: [[INC]] = add nuw nsw i64 [[I_021]], 1
+; CHECK-TF-NOREV-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
+; CHECK-TF-NOREV-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-TF-NOREV: [[FOR_END]]:
+; CHECK-TF-NOREV-NEXT: ret void
+;
+; CHECK-TF-ONLYRED-LABEL: define void @interleave(
+; CHECK-TF-ONLYRED-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-ONLYRED-NEXT: [[ENTRY:.*]]:
+; CHECK-TF-ONLYRED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-ONLYRED-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-TF-ONLYRED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-TF-ONLYRED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF-ONLYRED: [[VECTOR_PH]]:
+; CHECK-TF-ONLYRED-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-ONLYRED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-TF-ONLYRED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-TF-ONLYRED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-TF-ONLYRED-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-ONLYRED: [[VECTOR_BODY]]:
+; CHECK-TF-ONLYRED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ONLYRED-NEXT: [[TMP4:%.*]] = add i64 [[TMP2]], 0
+; CHECK-TF-ONLYRED-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 1
+; CHECK-TF-ONLYRED-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], [[TMP5]]
+; CHECK-TF-ONLYRED-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-ONLYRED-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP6]], 1
+; CHECK-TF-ONLYRED-NEXT: [[TMP9:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP7]]
+; CHECK-TF-ONLYRED-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP8]]
+; CHECK-TF-ONLYRED-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 8 x float>, ptr [[TMP9]], align 4
+; CHECK-TF-ONLYRED-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC]])
+; CHECK-TF-ONLYRED-NEXT: [[TMP11:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
+; CHECK-TF-ONLYRED-NEXT: [[TMP12:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 1
+; CHECK-TF-ONLYRED-NEXT: [[WIDE_VEC1:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
+; CHECK-TF-ONLYRED-NEXT: [[STRIDED_VEC2:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC1]])
+; CHECK-TF-ONLYRED-NEXT: [[TMP13:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC2]], 0
+; CHECK-TF-ONLYRED-NEXT: [[TMP14:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC2]], 1
+; CHECK-TF-ONLYRED-NEXT: [[TMP15:%.*]] = mul nuw nsw i64 [[INDEX]], 3
+; CHECK-TF-ONLYRED-NEXT: [[TMP16:%.*]] = mul nuw nsw i64 [[TMP6]], 3
+; CHECK-TF-ONLYRED-NEXT: [[TMP17:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP15]]
+; CHECK-TF-ONLYRED-NEXT: [[TMP18:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP16]]
+; CHECK-TF-ONLYRED-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP11]], <vscale x 4 x float> [[TMP12]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-TF-ONLYRED-NEXT: store <vscale x 12 x float> [[INTERLEAVED_VEC]], ptr [[TMP17]], align 4
+; CHECK-TF-ONLYRED-NEXT: [[INTERLEAVED_VEC3:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP13]], <vscale x 4 x float> [[TMP14]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-TF-ONLYRED-NEXT: store <vscale x 12 x float> [[INTERLEAVED_VEC3]], ptr [[TMP18]], align 4
+; CHECK-TF-ONLYRED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-TF-ONLYRED-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-ONLYRED-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-TF-ONLYRED: [[MIDDLE_BLOCK]]:
+; CHECK-TF-ONLYRED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-TF-ONLYRED-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-TF-ONLYRED: [[SCALAR_PH]]:
+; CHECK-TF-ONLYRED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-TF-ONLYRED-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-TF-ONLYRED: [[FOR_BODY]]:
+; CHECK-TF-ONLYRED-NEXT: [[I_021:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-TF-ONLYRED-NEXT: [[MUL:%.*]] = shl nuw nsw i64 [[I_021]], 1
+; CHECK-TF-ONLYRED-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[MUL]]
+; CHECK-TF-ONLYRED-NEXT: [[TMP20:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-TF-ONLYRED-NEXT: [[MUL1:%.*]] = mul nuw nsw i64 [[I_021]], 3
+; CHECK-TF-ONLYRED-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[MUL1]]
+; CHECK-TF-ONLYRED-NEXT: store float [[TMP20]], ptr [[ARRAYIDX2]], align 4
+; CHECK-TF-ONLYRED-NEXT: [[ADD:%.*]] = or disjoint i64 [[MUL]], 1
+; CHECK-TF-ONLYRED-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[ADD]]
+; CHECK-TF-ONLYRED-NEXT: [[TMP21:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-TF-ONLYRED-NEXT: [[ADD6:%.*]] = add nuw nsw i64 [[MUL1]], 1
+; CHECK-TF-ONLYRED-NEXT: [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD6]]
+; CHECK-TF-ONLYRED-NEXT: store float [[TMP21]], ptr [[ARRAYIDX7]], align 4
+; CHECK-TF-ONLYRED-NEXT: [[ADD9:%.*]] = add nuw nsw i64 [[MUL1]], 2
+; CHECK-TF-ONLYRED-NEXT: [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD9]]
+; CHECK-TF-ONLYRED-NEXT: store float 3.000000e+00, ptr [[ARRAYIDX10]], align 4
+; CHECK-TF-ONLYRED-NEXT: [[INC]] = add nuw nsw i64 [[I_021]], 1
+; CHECK-TF-ONLYRED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
+; CHECK-TF-ONLYRED-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-TF-ONLYRED: [[FOR_END]]:
+; CHECK-TF-ONLYRED-NEXT: ret void
+;
+; CHECK-NEOVERSE-V1-LABEL: define void @interleave(
+; CHECK-NEOVERSE-V1-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEOVERSE-V1-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-NEOVERSE-V1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEOVERSE-V1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK-NEOVERSE-V1: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-NEOVERSE-V1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEOVERSE-V1-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-NEOVERSE-V1-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NEOVERSE-V1-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEOVERSE-V1: [[VECTOR_PH]]:
+; CHECK-NEOVERSE-V1-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEOVERSE-V1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-NEOVERSE-V1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-NEOVERSE-V1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEOVERSE-V1-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-NEOVERSE-V1: [[VECTOR_BODY]]:
+; CHECK-NEOVERSE-V1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP4:%.*]] = add i64 [[TMP2]], 0
+; CHECK-NEOVERSE-V1-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 1
+; CHECK-NEOVERSE-V1-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], [[TMP5]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[INDEX]], 1
+; CHECK-NEOVERSE-V1-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP6]], 1
+; CHECK-NEOVERSE-V1-NEXT: [[TMP9:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP7]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP8]]
+; CHECK-NEOVERSE-V1-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 8 x float>, ptr [[TMP9]], align 4
+; CHECK-NEOVERSE-V1-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC]])
+; CHECK-NEOVERSE-V1-NEXT: [[TMP11:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
+; CHECK-NEOVERSE-V1-NEXT: [[TMP12:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 1
+; CHECK-NEOVERSE-V1-NEXT: [[WIDE_VEC2:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
+; CHECK-NEOVERSE-V1-NEXT: [[STRIDED_VEC3:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC2]])
+; CHECK-NEOVERSE-V1-NEXT: [[TMP13:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC3]], 0
+; CHECK-NEOVERSE-V1-NEXT: [[TMP14:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC3]], 1
+; CHECK-NEOVERSE-V1-NEXT: [[TMP15:%.*]] = mul nuw nsw i64 [[INDEX]], 3
+; CHECK-NEOVERSE-V1-NEXT: [[TMP16:%.*]] = mul nuw nsw i64 [[TMP6]], 3
+; CHECK-NEOVERSE-V1-NEXT: [[TMP17:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP15]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP18:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP16]]
+; CHECK-NEOVERSE-V1-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP11]], <vscale x 4 x float> [[TMP12]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-NEOVERSE-V1-NEXT: store <vscale x 12 x float> [[INTERLEAVED_VEC]], ptr [[TMP17]], align 4
+; CHECK-NEOVERSE-V1-NEXT: [[INTERLEAVED_VEC4:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP13]], <vscale x 4 x float> [[TMP14]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-NEOVERSE-V1-NEXT: store <vscale x 12 x float> [[INTERLEAVED_VEC4]], ptr [[TMP18]], align 4
+; CHECK-NEOVERSE-V1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEOVERSE-V1-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEOVERSE-V1: [[MIDDLE_BLOCK]]:
+; CHECK-NEOVERSE-V1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEOVERSE-V1-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK-NEOVERSE-V1: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEOVERSE-V1-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
+; CHECK-NEOVERSE-V1-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF8:![0-9]+]]
+; CHECK-NEOVERSE-V1: [[VEC_EPILOG_PH]]:
+; CHECK-NEOVERSE-V1-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEOVERSE-V1-NEXT: [[N_MOD_VF5:%.*]] = urem i64 [[N]], 4
+; CHECK-NEOVERSE-V1-NEXT: [[N_VEC6:%.*]] = sub i64 [[N]], [[N_MOD_VF5]]
+; CHECK-NEOVERSE-V1-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-NEOVERSE-V1: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-NEOVERSE-V1-NEXT: [[INDEX7:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT12:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP20:%.*]] = shl nuw nsw i64 [[INDEX7]], 1
+; CHECK-NEOVERSE-V1-NEXT: [[TMP21:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP20]]
+; CHECK-NEOVERSE-V1-NEXT: [[WIDE_VEC8:%.*]] = load <8 x float>, ptr [[TMP21]], align 4
+; CHECK-NEOVERSE-V1-NEXT: [[STRIDED_VEC9:%.*]] = shufflevector <8 x float> [[WIDE_VEC8]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEOVERSE-V1-NEXT: [[STRIDED_VEC10:%.*]] = shufflevector <8 x float> [[WIDE_VEC8]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEOVERSE-V1-NEXT: [[TMP22:%.*]] = mul nuw nsw i64 [[INDEX7]], 3
+; CHECK-NEOVERSE-V1-NEXT: [[TMP23:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP22]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP24:%.*]] = shufflevector <4 x float> [[STRIDED_VEC9]], <4 x float> [[STRIDED_VEC10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEOVERSE-V1-NEXT: [[TMP25:%.*]] = shufflevector <8 x float> [[TMP24]], <8 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float undef, float undef, float undef, float undef>, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEOVERSE-V1-NEXT: [[INTERLEAVED_VEC11:%.*]] = shufflevector <12 x float> [[TMP25]], <12 x float> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; CHECK-NEOVERSE-V1-NEXT: store <12 x float> [[INTERLEAVED_VEC11]], ptr [[TMP23]], align 4
+; CHECK-NEOVERSE-V1-NEXT: [[INDEX_NEXT12]] = add nuw i64 [[INDEX7]], 4
+; CHECK-NEOVERSE-V1-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT12]], [[N_VEC6]]
+; CHECK-NEOVERSE-V1-NEXT: br i1 [[TMP26]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEOVERSE-V1: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEOVERSE-V1-NEXT: [[CMP_N13:%.*]] = icmp eq i64 [[N]], [[N_VEC6]]
+; CHECK-NEOVERSE-V1-NEXT: br i1 [[CMP_N13]], label %[[FOR_END]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK-NEOVERSE-V1: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-NEOVERSE-V1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC6]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-NEOVERSE-V1-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-NEOVERSE-V1: [[FOR_BODY]]:
+; CHECK-NEOVERSE-V1-NEXT: [[I_021:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; CHECK-NEOVERSE-V1-NEXT: [[MUL:%.*]] = shl nuw nsw i64 [[I_021]], 1
+; CHECK-NEOVERSE-V1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[MUL]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP27:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-NEOVERSE-V1-NEXT: [[MUL1:%.*]] = mul nuw nsw i64 [[I_021]], 3
+; CHECK-NEOVERSE-V1-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[MUL1]]
+; CHECK-NEOVERSE-V1-NEXT: store float [[TMP27]], ptr [[ARRAYIDX2]], align 4
+; CHECK-NEOVERSE-V1-NEXT: [[ADD:%.*]] = or disjoint i64 [[MUL]], 1
+; CHECK-NEOVERSE-V1-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[ADD]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP28:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-NEOVERSE-V1-NEXT: [[ADD6:%.*]] = add nuw nsw i64 [[MUL1]], 1
+; CHECK-NEOVERSE-V1-NEXT: [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD6]]
+; CHECK-NEOVERSE-V1-NEXT: store float [[TMP28]], ptr [[ARRAYIDX7]], align 4
+; CHECK-NEOVERSE-V1-NEXT: [[ADD9:%.*]] = add nuw nsw i64 [[MUL1]], 2
+; CHECK-NEOVERSE-V1-NEXT: [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD9]]
+; CHECK-NEOVERSE-V1-NEXT: store float 3.000000e+00, ptr [[ARRAYIDX10]], align 4
+; CHECK-NEOVERSE-V1-NEXT: [[INC]] = add nuw nsw i64 [[I_021]], 1
+; CHECK-NEOVERSE-V1-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
+; CHECK-NEOVERSE-V1-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEOVERSE-V1: [[FOR_END]]:
+; CHECK-NEOVERSE-V1-NEXT: ret void
+;
+
+
+
+
+
entry:
br label %for.body
@@ -346,35 +1771,420 @@ for.end:
}
define void @reverse(ptr noalias %dst, ptr noalias %src) #0 {
-; CHECK-NOTF-LABEL: @reverse(
-; CHECK-NOTF: vector.body:
-; CHECK-NOTF-NOT: %{{.*}} = phi <vscale x 4 x i1>
-; CHECK-NOTF: %[[LOAD:.*]] = load <vscale x 2 x double>, ptr
-; CHECK-NOTF: %{{.*}} = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> %{{.*}})
-
-; CHECK-TF-NOREV-LABEL: @reverse(
-; CHECK-TF-NOREV: vector.body:
-; CHECK-TF-NOREV-NOT: %{{.*}} = phi <vscale x 4 x i1>
-; CHECK-TF-NOREV: %[[LOAD:.*]] = load <vscale x 2 x double>, ptr
-; CHECK-TF-NOREV: %{{.*}} = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> %{{.*}})
-
-; CHECK-TF-LABEL: @reverse(
-; CHECK-TF: vector.body:
-; CHECK-TF: %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 2 x i1>
-; CHECK-TF: %[[REVERSE_MASK:.*]] = call <vscale x 2 x i1> @llvm.vector.reverse.nxv2i1(<vscale x 2 x i1> %[[ACTIVE_LANE_MASK]])
-; CHECK-TF: %[[MASKED_LOAD:.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0({{.*}} <vscale x 2 x i1> %reverse
-
-; CHECK-TF-NORED-LABEL: @reverse(
-; CHECK-TF-NORED: vector.body:
-; CHECK-TF-NORED: %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 2 x i1>
-; CHECK-TF-NORED: %[[REVERSE_MASK:.*]] = call <vscale x 2 x i1> @llvm.vector.reverse.nxv2i1(<vscale x 2 x i1> %[[ACTIVE_LANE_MASK]])
-; CHECK-TF-NORED: %[[MASKED_LOAD:.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0({{.*}} <vscale x 2 x i1> %reverse
-
-; CHECK-TF-NOREC-LABEL: @reverse(
-; CHECK-TF-NOREC: vector.body:
-; CHECK-TF-NOREC: %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 2 x i1>
-; CHECK-TF-NOREC: %[[REVERSE_MASK:.*]] = call <vscale x 2 x i1> @llvm.vector.reverse.nxv2i1(<vscale x 2 x i1> %[[ACTIVE_LANE_MASK]])
-; CHECK-TF-NOREC: %[[MASKED_LOAD:.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0({{.*}} <vscale x 2 x i1> %reverse
+; CHECK-NOTF-LABEL: define void @reverse(
+; CHECK-NOTF-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-NOTF-NEXT: [[ENTRY:.*]]:
+; CHECK-NOTF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NOTF-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NOTF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
+; CHECK-NOTF-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NOTF: [[VECTOR_PH]]:
+; CHECK-NOTF-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-NOTF-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-NOTF-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
+; CHECK-NOTF-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NOTF-NEXT: [[TMP4:%.*]] = sub i64 1023, [[N_VEC]]
+; CHECK-NOTF-NEXT: [[TMP5:%.*]] = shl i64 [[N_VEC]], 3
+; CHECK-NOTF-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP5]]
+; CHECK-NOTF-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-NOTF: [[VECTOR_BODY]]:
+; CHECK-NOTF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NOTF-NEXT: [[TMP7:%.*]] = sub i64 1023, [[INDEX]]
+; CHECK-NOTF-NEXT: [[TMP8:%.*]] = shl i64 [[INDEX]], 3
+; CHECK-NOTF-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP8]]
+; CHECK-NOTF-NEXT: [[TMP9:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[TMP7]]
+; CHECK-NOTF-NEXT: [[TMP10:%.*]] = sub nuw nsw i64 [[TMP2]], 1
+; CHECK-NOTF-NEXT: [[TMP11:%.*]] = sub i64 0, [[TMP10]]
+; CHECK-NOTF-NEXT: [[TMP12:%.*]] = getelementptr inbounds double, ptr [[TMP9]], i64 [[TMP11]]
+; CHECK-NOTF-NEXT: [[TMP13:%.*]] = sub i64 [[TMP11]], [[TMP2]]
+; CHECK-NOTF-NEXT: [[TMP14:%.*]] = getelementptr inbounds double, ptr [[TMP9]], i64 [[TMP13]]
+; CHECK-NOTF-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x double>, ptr [[TMP12]], align 8
+; CHECK-NOTF-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 2 x double>, ptr [[TMP14]], align 8
+; CHECK-NOTF-NEXT: [[TMP15:%.*]] = fadd <vscale x 2 x double> [[WIDE_LOAD]], splat (double 1.000000e+00)
+; CHECK-NOTF-NEXT: [[TMP16:%.*]] = fadd <vscale x 2 x double> [[WIDE_LOAD1]], splat (double 1.000000e+00)
+; CHECK-NOTF-NEXT: [[REVERSE:%.*]] = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> [[TMP15]])
+; CHECK-NOTF-NEXT: [[REVERSE2:%.*]] = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> [[TMP16]])
+; CHECK-NOTF-NEXT: [[TMP17:%.*]] = getelementptr double, ptr [[NEXT_GEP]], i64 [[TMP2]]
+; CHECK-NOTF-NEXT: store <vscale x 2 x double> [[REVERSE]], ptr [[NEXT_GEP]], align 8
+; CHECK-NOTF-NEXT: store <vscale x 2 x double> [[REVERSE2]], ptr [[TMP17]], align 8
+; CHECK-NOTF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-NOTF-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NOTF-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NOTF: [[MIDDLE_BLOCK]]:
+; CHECK-NOTF-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NOTF-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-NOTF: [[SCALAR_PH]]:
+; CHECK-NOTF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ 1023, %[[ENTRY]] ]
+; CHECK-NOTF-NEXT: [[BC_RESUME_VAL3:%.*]] = phi ptr [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ [[DST]], %[[ENTRY]] ]
+; CHECK-NOTF-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-NOTF: [[FOR_BODY]]:
+; CHECK-NOTF-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NOTF-NEXT: [[INDVARS_PTR:%.*]] = phi ptr [ [[BC_RESUME_VAL3]], %[[SCALAR_PH]] ], [ [[INDVARS_PTR_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NOTF-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[INDVARS_IV]]
+; CHECK-NOTF-NEXT: [[TMP19:%.*]] = load double, ptr [[ARRAYIDX]], align 8
+; CHECK-NOTF-NEXT: [[ADD:%.*]] = fadd double [[TMP19]], 1.000000e+00
+; CHECK-NOTF-NEXT: store double [[ADD]], ptr [[INDVARS_PTR]], align 8
+; CHECK-NOTF-NEXT: [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], -1
+; CHECK-NOTF-NEXT: [[INDVARS_PTR_NEXT]] = getelementptr inbounds double, ptr [[INDVARS_PTR]], i64 1
+; CHECK-NOTF-NEXT: [[CMP_NOT:%.*]] = icmp eq i64 [[INDVARS_IV]], 0
+; CHECK-NOTF-NEXT: br i1 [[CMP_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NOTF: [[FOR_END]]:
+; CHECK-NOTF-NEXT: ret void
+;
+; CHECK-TF-ALL-LABEL: define void @reverse(
+; CHECK-TF-ALL-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-TF-ALL-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-ALL-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF-ALL: [[VECTOR_PH]]:
+; CHECK-TF-ALL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-ALL-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-TF-ALL-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1024)
+; CHECK-TF-ALL-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-ALL: [[VECTOR_BODY]]:
+; CHECK-TF-ALL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ALL-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ALL-NEXT: [[TMP2:%.*]] = sub i64 1023, [[INDEX]]
+; CHECK-TF-ALL-NEXT: [[TMP3:%.*]] = shl i64 [[INDEX]], 3
+; CHECK-TF-ALL-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP3]]
+; CHECK-TF-ALL-NEXT: [[TMP4:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[TMP2]]
+; CHECK-TF-ALL-NEXT: [[TMP5:%.*]] = sub nuw nsw i64 [[TMP1]], 1
+; CHECK-TF-ALL-NEXT: [[TMP6:%.*]] = sub i64 0, [[TMP5]]
+; CHECK-TF-ALL-NEXT: [[TMP7:%.*]] = getelementptr double, ptr [[TMP4]], i64 [[TMP6]]
+; CHECK-TF-ALL-NEXT: [[REVERSE:%.*]] = call <vscale x 2 x i1> @llvm.vector.reverse.nxv2i1(<vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-ALL-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> [[REVERSE]], <vscale x 2 x double> poison)
+; CHECK-TF-ALL-NEXT: [[TMP8:%.*]] = fadd <vscale x 2 x double> [[WIDE_MASKED_LOAD]], splat (double 1.000000e+00)
+; CHECK-TF-ALL-NEXT: [[REVERSE1:%.*]] = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> [[TMP8]])
+; CHECK-TF-ALL-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[REVERSE1]], ptr align 8 [[NEXT_GEP]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-ALL-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-ALL-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1024)
+; CHECK-TF-ALL-NEXT: [[TMP9:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-ALL-NEXT: [[TMP10:%.*]] = xor i1 [[TMP9]], true
+; CHECK-TF-ALL-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-TF-ALL: [[MIDDLE_BLOCK]]:
+; CHECK-TF-ALL-NEXT: br label %[[FOR_END:.*]]
+; CHECK-TF-ALL: [[FOR_END]]:
+; CHECK-TF-ALL-NEXT: ret void
+;
+; CHECK-TF-LABEL: define void @reverse(
+; CHECK-TF-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF: [[VECTOR_PH]]:
+; CHECK-TF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1024)
+; CHECK-TF-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF: [[VECTOR_BODY]]:
+; CHECK-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[TMP2:%.*]] = sub i64 1023, [[INDEX]]
+; CHECK-TF-NEXT: [[TMP3:%.*]] = shl i64 [[INDEX]], 3
+; CHECK-TF-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP3]]
+; CHECK-TF-NEXT: [[TMP4:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[TMP2]]
+; CHECK-TF-NEXT: [[TMP5:%.*]] = sub nuw nsw i64 [[TMP1]], 1
+; CHECK-TF-NEXT: [[TMP6:%.*]] = sub i64 0, [[TMP5]]
+; CHECK-TF-NEXT: [[TMP7:%.*]] = getelementptr double, ptr [[TMP4]], i64 [[TMP6]]
+; CHECK-TF-NEXT: [[REVERSE:%.*]] = call <vscale x 2 x i1> @llvm.vector.reverse.nxv2i1(<vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> [[REVERSE]], <vscale x 2 x double> poison)
+; CHECK-TF-NEXT: [[TMP8:%.*]] = fadd <vscale x 2 x double> [[WIDE_MASKED_LOAD]], splat (double 1.000000e+00)
+; CHECK-TF-NEXT: [[REVERSE1:%.*]] = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> [[TMP8]])
+; CHECK-TF-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[REVERSE1]], ptr align 8 [[NEXT_GEP]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1024)
+; CHECK-TF-NEXT: [[TMP9:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NEXT: [[TMP10:%.*]] = xor i1 [[TMP9]], true
+; CHECK-TF-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-TF: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NEXT: br label %[[FOR_END:.*]]
+; CHECK-TF: [[FOR_END]]:
+; CHECK-TF-NEXT: ret void
+;
+; CHECK-TF-DEFAULT-LABEL: define void @reverse(
+; CHECK-TF-DEFAULT-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-TF-DEFAULT-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-DEFAULT-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF-DEFAULT: [[VECTOR_PH]]:
+; CHECK-TF-DEFAULT-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-DEFAULT-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-TF-DEFAULT-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1024)
+; CHECK-TF-DEFAULT-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-DEFAULT: [[VECTOR_BODY]]:
+; CHECK-TF-DEFAULT-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-DEFAULT-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-DEFAULT-NEXT: [[TMP2:%.*]] = sub i64 1023, [[INDEX]]
+; CHECK-TF-DEFAULT-NEXT: [[TMP3:%.*]] = shl i64 [[INDEX]], 3
+; CHECK-TF-DEFAULT-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP3]]
+; CHECK-TF-DEFAULT-NEXT: [[TMP4:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[TMP2]]
+; CHECK-TF-DEFAULT-NEXT: [[TMP5:%.*]] = sub nuw nsw i64 [[TMP1]], 1
+; CHECK-TF-DEFAULT-NEXT: [[TMP6:%.*]] = sub i64 0, [[TMP5]]
+; CHECK-TF-DEFAULT-NEXT: [[TMP7:%.*]] = getelementptr double, ptr [[TMP4]], i64 [[TMP6]]
+; CHECK-TF-DEFAULT-NEXT: [[REVERSE:%.*]] = call <vscale x 2 x i1> @llvm.vector.reverse.nxv2i1(<vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-DEFAULT-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> [[REVERSE]], <vscale x 2 x double> poison)
+; CHECK-TF-DEFAULT-NEXT: [[TMP8:%.*]] = fadd <vscale x 2 x double> [[WIDE_MASKED_LOAD]], splat (double 1.000000e+00)
+; CHECK-TF-DEFAULT-NEXT: [[REVERSE1:%.*]] = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> [[TMP8]])
+; CHECK-TF-DEFAULT-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[REVERSE1]], ptr align 8 [[NEXT_GEP]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-DEFAULT-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-DEFAULT-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1024)
+; CHECK-TF-DEFAULT-NEXT: [[TMP9:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-DEFAULT-NEXT: [[TMP10:%.*]] = xor i1 [[TMP9]], true
+; CHECK-TF-DEFAULT-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-TF-DEFAULT: [[MIDDLE_BLOCK]]:
+; CHECK-TF-DEFAULT-NEXT: br label %[[FOR_END:.*]]
+; CHECK-TF-DEFAULT: [[FOR_END]]:
+; CHECK-TF-DEFAULT-NEXT: ret void
+;
+; CHECK-TF-NORED-LABEL: define void @reverse(
+; CHECK-TF-NORED-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NORED-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-NORED-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF-NORED: [[VECTOR_PH]]:
+; CHECK-TF-NORED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NORED-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-TF-NORED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1024)
+; CHECK-TF-NORED-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-NORED: [[VECTOR_BODY]]:
+; CHECK-TF-NORED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NORED-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NORED-NEXT: [[TMP2:%.*]] = sub i64 1023, [[INDEX]]
+; CHECK-TF-NORED-NEXT: [[TMP3:%.*]] = shl i64 [[INDEX]], 3
+; CHECK-TF-NORED-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP3]]
+; CHECK-TF-NORED-NEXT: [[TMP4:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[TMP2]]
+; CHECK-TF-NORED-NEXT: [[TMP5:%.*]] = sub nuw nsw i64 [[TMP1]], 1
+; CHECK-TF-NORED-NEXT: [[TMP6:%.*]] = sub i64 0, [[TMP5]]
+; CHECK-TF-NORED-NEXT: [[TMP7:%.*]] = getelementptr double, ptr [[TMP4]], i64 [[TMP6]]
+; CHECK-TF-NORED-NEXT: [[REVERSE:%.*]] = call <vscale x 2 x i1> @llvm.vector.reverse.nxv2i1(<vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NORED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> [[REVERSE]], <vscale x 2 x double> poison)
+; CHECK-TF-NORED-NEXT: [[TMP8:%.*]] = fadd <vscale x 2 x double> [[WIDE_MASKED_LOAD]], splat (double 1.000000e+00)
+; CHECK-TF-NORED-NEXT: [[REVERSE1:%.*]] = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> [[TMP8]])
+; CHECK-TF-NORED-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[REVERSE1]], ptr align 8 [[NEXT_GEP]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NORED-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-NORED-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1024)
+; CHECK-TF-NORED-NEXT: [[TMP9:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NORED-NEXT: [[TMP10:%.*]] = xor i1 [[TMP9]], true
+; CHECK-TF-NORED-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-TF-NORED: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NORED-NEXT: br label %[[FOR_END:.*]]
+; CHECK-TF-NORED: [[FOR_END]]:
+; CHECK-TF-NORED-NEXT: ret void
+;
+; CHECK-TF-NOREC-LABEL: define void @reverse(
+; CHECK-TF-NOREC-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NOREC-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-NOREC-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF-NOREC: [[VECTOR_PH]]:
+; CHECK-TF-NOREC-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NOREC-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-TF-NOREC-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1024)
+; CHECK-TF-NOREC-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-NOREC: [[VECTOR_BODY]]:
+; CHECK-TF-NOREC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREC-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREC-NEXT: [[TMP2:%.*]] = sub i64 1023, [[INDEX]]
+; CHECK-TF-NOREC-NEXT: [[TMP3:%.*]] = shl i64 [[INDEX]], 3
+; CHECK-TF-NOREC-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP3]]
+; CHECK-TF-NOREC-NEXT: [[TMP4:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[TMP2]]
+; CHECK-TF-NOREC-NEXT: [[TMP5:%.*]] = sub nuw nsw i64 [[TMP1]], 1
+; CHECK-TF-NOREC-NEXT: [[TMP6:%.*]] = sub i64 0, [[TMP5]]
+; CHECK-TF-NOREC-NEXT: [[TMP7:%.*]] = getelementptr double, ptr [[TMP4]], i64 [[TMP6]]
+; CHECK-TF-NOREC-NEXT: [[REVERSE:%.*]] = call <vscale x 2 x i1> @llvm.vector.reverse.nxv2i1(<vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NOREC-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> [[REVERSE]], <vscale x 2 x double> poison)
+; CHECK-TF-NOREC-NEXT: [[TMP8:%.*]] = fadd <vscale x 2 x double> [[WIDE_MASKED_LOAD]], splat (double 1.000000e+00)
+; CHECK-TF-NOREC-NEXT: [[REVERSE1:%.*]] = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> [[TMP8]])
+; CHECK-TF-NOREC-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[REVERSE1]], ptr align 8 [[NEXT_GEP]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NOREC-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-NOREC-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1024)
+; CHECK-TF-NOREC-NEXT: [[TMP9:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NOREC-NEXT: [[TMP10:%.*]] = xor i1 [[TMP9]], true
+; CHECK-TF-NOREC-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-TF-NOREC: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NOREC-NEXT: br label %[[FOR_END:.*]]
+; CHECK-TF-NOREC: [[FOR_END]]:
+; CHECK-TF-NOREC-NEXT: ret void
+;
+; CHECK-TF-NOREV-LABEL: define void @reverse(
+; CHECK-TF-NOREV-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NOREV-NEXT: [[ENTRY:.*]]:
+; CHECK-TF-NOREV-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NOREV-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NOREV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
+; CHECK-TF-NOREV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF-NOREV: [[VECTOR_PH]]:
+; CHECK-TF-NOREV-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-TF-NOREV-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-TF-NOREV-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
+; CHECK-TF-NOREV-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-TF-NOREV-NEXT: [[TMP4:%.*]] = sub i64 1023, [[N_VEC]]
+; CHECK-TF-NOREV-NEXT: [[TMP5:%.*]] = shl i64 [[N_VEC]], 3
+; CHECK-TF-NOREV-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP5]]
+; CHECK-TF-NOREV-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-NOREV: [[VECTOR_BODY]]:
+; CHECK-TF-NOREV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREV-NEXT: [[TMP7:%.*]] = sub i64 1023, [[INDEX]]
+; CHECK-TF-NOREV-NEXT: [[TMP8:%.*]] = shl i64 [[INDEX]], 3
+; CHECK-TF-NOREV-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP8]]
+; CHECK-TF-NOREV-NEXT: [[TMP9:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[TMP7]]
+; CHECK-TF-NOREV-NEXT: [[TMP10:%.*]] = sub nuw nsw i64 [[TMP2]], 1
+; CHECK-TF-NOREV-NEXT: [[TMP11:%.*]] = sub i64 0, [[TMP10]]
+; CHECK-TF-NOREV-NEXT: [[TMP12:%.*]] = getelementptr inbounds double, ptr [[TMP9]], i64 [[TMP11]]
+; CHECK-TF-NOREV-NEXT: [[TMP13:%.*]] = sub i64 [[TMP11]], [[TMP2]]
+; CHECK-TF-NOREV-NEXT: [[TMP14:%.*]] = getelementptr inbounds double, ptr [[TMP9]], i64 [[TMP13]]
+; CHECK-TF-NOREV-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x double>, ptr [[TMP12]], align 8
+; CHECK-TF-NOREV-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 2 x double>, ptr [[TMP14]], align 8
+; CHECK-TF-NOREV-NEXT: [[TMP15:%.*]] = fadd <vscale x 2 x double> [[WIDE_LOAD]], splat (double 1.000000e+00)
+; CHECK-TF-NOREV-NEXT: [[TMP16:%.*]] = fadd <vscale x 2 x double> [[WIDE_LOAD1]], splat (double 1.000000e+00)
+; CHECK-TF-NOREV-NEXT: [[REVERSE:%.*]] = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> [[TMP15]])
+; CHECK-TF-NOREV-NEXT: [[REVERSE2:%.*]] = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> [[TMP16]])
+; CHECK-TF-NOREV-NEXT: [[TMP17:%.*]] = getelementptr double, ptr [[NEXT_GEP]], i64 [[TMP2]]
+; CHECK-TF-NOREV-NEXT: store <vscale x 2 x double> [[REVERSE]], ptr [[NEXT_GEP]], align 8
+; CHECK-TF-NOREV-NEXT: store <vscale x 2 x double> [[REVERSE2]], ptr [[TMP17]], align 8
+; CHECK-TF-NOREV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-TF-NOREV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-NOREV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-TF-NOREV: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NOREV-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-TF-NOREV-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-TF-NOREV: [[SCALAR_PH]]:
+; CHECK-TF-NOREV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ 1023, %[[ENTRY]] ]
+; CHECK-TF-NOREV-NEXT: [[BC_RESUME_VAL3:%.*]] = phi ptr [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ [[DST]], %[[ENTRY]] ]
+; CHECK-TF-NOREV-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-TF-NOREV: [[FOR_BODY]]:
+; CHECK-TF-NOREV-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-TF-NOREV-NEXT: [[INDVARS_PTR:%.*]] = phi ptr [ [[BC_RESUME_VAL3]], %[[SCALAR_PH]] ], [ [[INDVARS_PTR_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-TF-NOREV-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[INDVARS_IV]]
+; CHECK-TF-NOREV-NEXT: [[TMP19:%.*]] = load double, ptr [[ARRAYIDX]], align 8
+; CHECK-TF-NOREV-NEXT: [[ADD:%.*]] = fadd double [[TMP19]], 1.000000e+00
+; CHECK-TF-NOREV-NEXT: store double [[ADD]], ptr [[INDVARS_PTR]], align 8
+; CHECK-TF-NOREV-NEXT: [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], -1
+; CHECK-TF-NOREV-NEXT: [[INDVARS_PTR_NEXT]] = getelementptr inbounds double, ptr [[INDVARS_PTR]], i64 1
+; CHECK-TF-NOREV-NEXT: [[CMP_NOT:%.*]] = icmp eq i64 [[INDVARS_IV]], 0
+; CHECK-TF-NOREV-NEXT: br i1 [[CMP_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-TF-NOREV: [[FOR_END]]:
+; CHECK-TF-NOREV-NEXT: ret void
+;
+; CHECK-TF-ONLYRED-LABEL: define void @reverse(
+; CHECK-TF-ONLYRED-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-TF-ONLYRED-NEXT: [[ENTRY:.*]]:
+; CHECK-TF-ONLYRED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-ONLYRED-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-ONLYRED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
+; CHECK-TF-ONLYRED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF-ONLYRED: [[VECTOR_PH]]:
+; CHECK-TF-ONLYRED-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-TF-ONLYRED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-TF-ONLYRED-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
+; CHECK-TF-ONLYRED-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-TF-ONLYRED-NEXT: [[TMP4:%.*]] = sub i64 1023, [[N_VEC]]
+; CHECK-TF-ONLYRED-NEXT: [[TMP5:%.*]] = shl i64 [[N_VEC]], 3
+; CHECK-TF-ONLYRED-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP5]]
+; CHECK-TF-ONLYRED-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-ONLYRED: [[VECTOR_BODY]]:
+; CHECK-TF-ONLYRED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ONLYRED-NEXT: [[TMP7:%.*]] = sub i64 1023, [[INDEX]]
+; CHECK-TF-ONLYRED-NEXT: [[TMP8:%.*]] = shl i64 [[INDEX]], 3
+; CHECK-TF-ONLYRED-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP8]]
+; CHECK-TF-ONLYRED-NEXT: [[TMP9:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[TMP7]]
+; CHECK-TF-ONLYRED-NEXT: [[TMP10:%.*]] = sub nuw nsw i64 [[TMP2]], 1
+; CHECK-TF-ONLYRED-NEXT: [[TMP11:%.*]] = sub i64 0, [[TMP10]]
+; CHECK-TF-ONLYRED-NEXT: [[TMP12:%.*]] = getelementptr inbounds double, ptr [[TMP9]], i64 [[TMP11]]
+; CHECK-TF-ONLYRED-NEXT: [[TMP13:%.*]] = sub i64 [[TMP11]], [[TMP2]]
+; CHECK-TF-ONLYRED-NEXT: [[TMP14:%.*]] = getelementptr inbounds double, ptr [[TMP9]], i64 [[TMP13]]
+; CHECK-TF-ONLYRED-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x double>, ptr [[TMP12]], align 8
+; CHECK-TF-ONLYRED-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 2 x double>, ptr [[TMP14]], align 8
+; CHECK-TF-ONLYRED-NEXT: [[TMP15:%.*]] = fadd <vscale x 2 x double> [[WIDE_LOAD]], splat (double 1.000000e+00)
+; CHECK-TF-ONLYRED-NEXT: [[TMP16:%.*]] = fadd <vscale x 2 x double> [[WIDE_LOAD1]], splat (double 1.000000e+00)
+; CHECK-TF-ONLYRED-NEXT: [[REVERSE:%.*]] = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> [[TMP15]])
+; CHECK-TF-ONLYRED-NEXT: [[REVERSE2:%.*]] = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> [[TMP16]])
+; CHECK-TF-ONLYRED-NEXT: [[TMP17:%.*]] = getelementptr double, ptr [[NEXT_GEP]], i64 [[TMP2]]
+; CHECK-TF-ONLYRED-NEXT: store <vscale x 2 x double> [[REVERSE]], ptr [[NEXT_GEP]], align 8
+; CHECK-TF-ONLYRED-NEXT: store <vscale x 2 x double> [[REVERSE2]], ptr [[TMP17]], align 8
+; CHECK-TF-ONLYRED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-TF-ONLYRED-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-ONLYRED-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-TF-ONLYRED: [[MIDDLE_BLOCK]]:
+; CHECK-TF-ONLYRED-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-TF-ONLYRED-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-TF-ONLYRED: [[SCALAR_PH]]:
+; CHECK-TF-ONLYRED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ 1023, %[[ENTRY]] ]
+; CHECK-TF-ONLYRED-NEXT: [[BC_RESUME_VAL3:%.*]] = phi ptr [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ [[DST]], %[[ENTRY]] ]
+; CHECK-TF-ONLYRED-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-TF-ONLYRED: [[FOR_BODY]]:
+; CHECK-TF-ONLYRED-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-TF-ONLYRED-NEXT: [[INDVARS_PTR:%.*]] = phi ptr [ [[BC_RESUME_VAL3]], %[[SCALAR_PH]] ], [ [[INDVARS_PTR_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-TF-ONLYRED-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[INDVARS_IV]]
+; CHECK-TF-ONLYRED-NEXT: [[TMP19:%.*]] = load double, ptr [[ARRAYIDX]], align 8
+; CHECK-TF-ONLYRED-NEXT: [[ADD:%.*]] = fadd double [[TMP19]], 1.000000e+00
+; CHECK-TF-ONLYRED-NEXT: store double [[ADD]], ptr [[INDVARS_PTR]], align 8
+; CHECK-TF-ONLYRED-NEXT: [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], -1
+; CHECK-TF-ONLYRED-NEXT: [[INDVARS_PTR_NEXT]] = getelementptr inbounds double, ptr [[INDVARS_PTR]], i64 1
+; CHECK-TF-ONLYRED-NEXT: [[CMP_NOT:%.*]] = icmp eq i64 [[INDVARS_IV]], 0
+; CHECK-TF-ONLYRED-NEXT: br i1 [[CMP_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-TF-ONLYRED: [[FOR_END]]:
+; CHECK-TF-ONLYRED-NEXT: ret void
+;
+; CHECK-NEOVERSE-V1-LABEL: define void @reverse(
+; CHECK-NEOVERSE-V1-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-NEOVERSE-V1-NEXT: [[ENTRY:.*]]:
+; CHECK-NEOVERSE-V1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEOVERSE-V1-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEOVERSE-V1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
+; CHECK-NEOVERSE-V1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEOVERSE-V1: [[VECTOR_PH]]:
+; CHECK-NEOVERSE-V1-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-NEOVERSE-V1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-NEOVERSE-V1-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
+; CHECK-NEOVERSE-V1-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP4:%.*]] = sub i64 1023, [[N_VEC]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP5:%.*]] = shl i64 [[N_VEC]], 3
+; CHECK-NEOVERSE-V1-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP5]]
+; CHECK-NEOVERSE-V1-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-NEOVERSE-V1: [[VECTOR_BODY]]:
+; CHECK-NEOVERSE-V1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP7:%.*]] = sub i64 1023, [[INDEX]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP8:%.*]] = shl i64 [[INDEX]], 3
+; CHECK-NEOVERSE-V1-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP8]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP9:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[TMP7]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP10:%.*]] = sub nuw nsw i64 [[TMP2]], 1
+; CHECK-NEOVERSE-V1-NEXT: [[TMP11:%.*]] = sub i64 0, [[TMP10]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP12:%.*]] = getelementptr inbounds double, ptr [[TMP9]], i64 [[TMP11]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP13:%.*]] = sub i64 [[TMP11]], [[TMP2]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP14:%.*]] = getelementptr inbounds double, ptr [[TMP9]], i64 [[TMP13]]
+; CHECK-NEOVERSE-V1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x double>, ptr [[TMP12]], align 8
+; CHECK-NEOVERSE-V1-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 2 x double>, ptr [[TMP14]], align 8
+; CHECK-NEOVERSE-V1-NEXT: [[TMP15:%.*]] = fadd <vscale x 2 x double> [[WIDE_LOAD]], splat (double 1.000000e+00)
+; CHECK-NEOVERSE-V1-NEXT: [[TMP16:%.*]] = fadd <vscale x 2 x double> [[WIDE_LOAD1]], splat (double 1.000000e+00)
+; CHECK-NEOVERSE-V1-NEXT: [[REVERSE:%.*]] = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> [[TMP15]])
+; CHECK-NEOVERSE-V1-NEXT: [[REVERSE2:%.*]] = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> [[TMP16]])
+; CHECK-NEOVERSE-V1-NEXT: [[TMP17:%.*]] = getelementptr double, ptr [[NEXT_GEP]], i64 [[TMP2]]
+; CHECK-NEOVERSE-V1-NEXT: store <vscale x 2 x double> [[REVERSE]], ptr [[NEXT_GEP]], align 8
+; CHECK-NEOVERSE-V1-NEXT: store <vscale x 2 x double> [[REVERSE2]], ptr [[TMP17]], align 8
+; CHECK-NEOVERSE-V1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEOVERSE-V1-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEOVERSE-V1: [[MIDDLE_BLOCK]]:
+; CHECK-NEOVERSE-V1-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEOVERSE-V1-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-NEOVERSE-V1: [[SCALAR_PH]]:
+; CHECK-NEOVERSE-V1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ 1023, %[[ENTRY]] ]
+; CHECK-NEOVERSE-V1-NEXT: [[BC_RESUME_VAL3:%.*]] = phi ptr [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ [[DST]], %[[ENTRY]] ]
+; CHECK-NEOVERSE-V1-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-NEOVERSE-V1: [[FOR_BODY]]:
+; CHECK-NEOVERSE-V1-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEOVERSE-V1-NEXT: [[INDVARS_PTR:%.*]] = phi ptr [ [[BC_RESUME_VAL3]], %[[SCALAR_PH]] ], [ [[INDVARS_PTR_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEOVERSE-V1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[INDVARS_IV]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP19:%.*]] = load double, ptr [[ARRAYIDX]], align 8
+; CHECK-NEOVERSE-V1-NEXT: [[ADD:%.*]] = fadd double [[TMP19]], 1.000000e+00
+; CHECK-NEOVERSE-V1-NEXT: store double [[ADD]], ptr [[INDVARS_PTR]], align 8
+; CHECK-NEOVERSE-V1-NEXT: [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], -1
+; CHECK-NEOVERSE-V1-NEXT: [[INDVARS_PTR_NEXT]] = getelementptr inbounds double, ptr [[INDVARS_PTR]], i64 1
+; CHECK-NEOVERSE-V1-NEXT: [[CMP_NOT:%.*]] = icmp eq i64 [[INDVARS_IV]], 0
+; CHECK-NEOVERSE-V1-NEXT: br i1 [[CMP_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEOVERSE-V1: [[FOR_END]]:
+; CHECK-NEOVERSE-V1-NEXT: ret void
+;
+
+
+
+
entry:
br label %for.body
@@ -402,3 +2212,102 @@ attributes #0 = { "target-features"="+sve" }
!2 = !{!"llvm.loop.vectorize.scalable.enable", i1 true}
!3 = !{!"llvm.loop.interleave.count", i32 1}
!4 = !{!"llvm.loop.vectorize.enable", i1 true}
+;.
+; CHECK-NOTF: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK-NOTF: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-NOTF: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK-NOTF: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+; CHECK-NOTF: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK-NOTF: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
+; CHECK-NOTF: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
+; CHECK-NOTF: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}
+; CHECK-NOTF: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
+; CHECK-NOTF: [[LOOP9]] = distinct !{[[LOOP9]], [[META2]], [[META1]]}
+; CHECK-NOTF: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]}
+; CHECK-NOTF: [[LOOP11]] = distinct !{[[LOOP11]], [[META2]], [[META1]]}
+;.
+; CHECK-TF-ALL: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK-TF-ALL: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-TF-ALL: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK-TF-ALL: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
+; CHECK-TF-ALL: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK-TF-ALL: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]}
+; CHECK-TF-ALL: [[LOOP6]] = distinct !{[[LOOP6]], [[META2]], [[META1]]}
+; CHECK-TF-ALL: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]], [[META2]]}
+;.
+; CHECK-TF: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK-TF: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-TF: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK-TF: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
+; CHECK-TF: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK-TF: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]}
+; CHECK-TF: [[LOOP6]] = distinct !{[[LOOP6]], [[META2]], [[META1]]}
+; CHECK-TF: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]], [[META2]]}
+;.
+; CHECK-TF-DEFAULT: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK-TF-DEFAULT: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-TF-DEFAULT: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK-TF-DEFAULT: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
+; CHECK-TF-DEFAULT: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK-TF-DEFAULT: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]}
+; CHECK-TF-DEFAULT: [[PROF6]] = !{!"branch_weights", i32 4, i32 12}
+; CHECK-TF-DEFAULT: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]], [[META2]]}
+; CHECK-TF-DEFAULT: [[LOOP8]] = distinct !{[[LOOP8]], [[META2]], [[META1]]}
+; CHECK-TF-DEFAULT: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]], [[META2]]}
+;.
+; CHECK-TF-NORED: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK-TF-NORED: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-TF-NORED: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK-TF-NORED: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
+; CHECK-TF-NORED: [[LOOP4]] = distinct !{[[LOOP4]], [[META2]], [[META1]]}
+; CHECK-TF-NORED: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]}
+; CHECK-TF-NORED: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
+; CHECK-TF-NORED: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}
+; CHECK-TF-NORED: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
+;.
+; CHECK-TF-NOREC: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK-TF-NOREC: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-TF-NOREC: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK-TF-NOREC: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
+; CHECK-TF-NOREC: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK-TF-NOREC: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
+; CHECK-TF-NOREC: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
+; CHECK-TF-NOREC: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}
+; CHECK-TF-NOREC: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
+;.
+; CHECK-TF-NOREV: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK-TF-NOREV: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-TF-NOREV: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK-TF-NOREV: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
+; CHECK-TF-NOREV: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK-TF-NOREV: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]}
+; CHECK-TF-NOREV: [[LOOP6]] = distinct !{[[LOOP6]], [[META2]], [[META1]]}
+; CHECK-TF-NOREV: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]], [[META2]]}
+; CHECK-TF-NOREV: [[LOOP8]] = distinct !{[[LOOP8]], [[META2]], [[META1]]}
+;.
+; CHECK-TF-ONLYRED: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK-TF-ONLYRED: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-TF-ONLYRED: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK-TF-ONLYRED: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+; CHECK-TF-ONLYRED: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK-TF-ONLYRED: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]}
+; CHECK-TF-ONLYRED: [[LOOP6]] = distinct !{[[LOOP6]], [[META2]], [[META1]]}
+; CHECK-TF-ONLYRED: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]], [[META2]]}
+; CHECK-TF-ONLYRED: [[LOOP8]] = distinct !{[[LOOP8]], [[META2]], [[META1]]}
+; CHECK-TF-ONLYRED: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]], [[META2]]}
+; CHECK-TF-ONLYRED: [[LOOP10]] = distinct !{[[LOOP10]], [[META2]], [[META1]]}
+;.
+; CHECK-NEOVERSE-V1: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK-NEOVERSE-V1: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-NEOVERSE-V1: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK-NEOVERSE-V1: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
+; CHECK-NEOVERSE-V1: [[LOOP4]] = distinct !{[[LOOP4]], [[META2]], [[META1]]}
+; CHECK-NEOVERSE-V1: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]}
+; CHECK-NEOVERSE-V1: [[LOOP6]] = distinct !{[[LOOP6]], [[META2]], [[META1]]}
+; CHECK-NEOVERSE-V1: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]], [[META2]]}
+; CHECK-NEOVERSE-V1: [[PROF8]] = !{!"branch_weights", i32 4, i32 12}
+; CHECK-NEOVERSE-V1: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]], [[META2]]}
+; CHECK-NEOVERSE-V1: [[LOOP10]] = distinct !{[[LOOP10]], [[META2]], [[META1]]}
+; CHECK-NEOVERSE-V1: [[LOOP11]] = distinct !{[[LOOP11]], [[META1]], [[META2]]}
+; CHECK-NEOVERSE-V1: [[LOOP12]] = distinct !{[[LOOP12]], [[META2]], [[META1]]}
+;.
More information about the llvm-commits
mailing list