[clang] [llvm] [DRAFT] Re-vectorisation in LV (PR #203885)
Gaëtan Bossu via cfe-commits
cfe-commits at lists.llvm.org
Wed Jul 29 06:18:08 PDT 2026
https://github.com/gbossu updated https://github.com/llvm/llvm-project/pull/203885
>From 54c12d3f1abb828b975432e84b878a1cd40d197f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ga=C3=ABtan=20Bossu?= <gaetan.bossu at arm.com>
Date: Fri, 29 May 2026 12:35:40 +0000
Subject: [PATCH 1/7] [AArch64] New quad structure ld/st intrinsics with nxv1i1
predicates
The predicate type for those now is the expected nxv1i1.
We might or might not push that change upstream.
---
llvm/include/llvm/IR/IntrinsicsAArch64.td | 39 +++++++
.../Target/AArch64/AArch64ISelDAGToDAG.cpp | 26 +++--
.../AArch64/AArch64TargetTransformInfo.cpp | 6 +
.../sve2p1-intrinsics-multivec-loads.ll | 88 +++++++++++++++
.../sve2p1-intrinsics-multivec-stores.ll | 104 ++++++++++++++++++
5 files changed, 256 insertions(+), 7 deletions(-)
diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td
index ba0d7c02bf427..b128aad50350e 100644
--- a/llvm/include/llvm/IR/IntrinsicsAArch64.td
+++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td
@@ -1068,17 +1068,30 @@ let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.".
: DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>],
[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyptr_ty],
[IntrReadMem, IntrArgMemOnly]>;
+ class AdvSIMD_2Vec_PredLoadQ_Intrinsic
+ : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>],
+ [llvm_nxv1i1_ty, llvm_ptr_ty],
+ [IntrReadMem, IntrArgMemOnly]>;
class AdvSIMD_3Vec_PredLoad_Intrinsic
: DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>],
[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyptr_ty],
[IntrReadMem, IntrArgMemOnly]>;
+ class AdvSIMD_3Vec_PredLoadQ_Intrinsic
+ : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>],
+ [llvm_nxv1i1_ty, llvm_ptr_ty],
+ [IntrReadMem, IntrArgMemOnly]>;
class AdvSIMD_4Vec_PredLoad_Intrinsic
: DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>,
LLVMMatchType<0>],
[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyptr_ty],
[IntrReadMem, IntrArgMemOnly]>;
+ class AdvSIMD_4Vec_PredLoadQ_Intrinsic
+ : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>,
+ LLVMMatchType<0>],
+ [llvm_nxv1i1_ty, llvm_ptr_ty],
+ [IntrReadMem, IntrArgMemOnly]>;
class AdvSIMD_1Vec_PredLoad_WriteFFR_Intrinsic
: DefaultAttrsIntrinsic<[llvm_anyvector_ty],
@@ -1096,12 +1109,22 @@ let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.".
[llvm_anyvector_ty, LLVMMatchType<0>,
LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyptr_ty],
[IntrArgMemOnly, NoCapture<ArgIndex<3>>]>;
+ class AdvSIMD_2Vec_PredStoreQ_Intrinsic
+ : DefaultAttrsIntrinsic<[],
+ [llvm_anyvector_ty, LLVMMatchType<0>,
+ llvm_nxv1i1_ty, llvm_ptr_ty],
+ [IntrArgMemOnly, NoCapture<ArgIndex<3>>]>;
class AdvSIMD_3Vec_PredStore_Intrinsic
: DefaultAttrsIntrinsic<[],
[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>,
LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyptr_ty],
[IntrArgMemOnly, NoCapture<ArgIndex<4>>]>;
+ class AdvSIMD_3Vec_PredStoreQ_Intrinsic
+ : DefaultAttrsIntrinsic<[],
+ [llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>,
+ llvm_nxv1i1_ty, llvm_ptr_ty],
+ [IntrArgMemOnly, NoCapture<ArgIndex<4>>]>;
class AdvSIMD_4Vec_PredStore_Intrinsic
: DefaultAttrsIntrinsic<[],
@@ -1109,6 +1132,12 @@ let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.".
LLVMMatchType<0>,
LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyptr_ty],
[IntrArgMemOnly, NoCapture<ArgIndex<5>>]>;
+ class AdvSIMD_4Vec_PredStoreQ_Intrinsic
+ : DefaultAttrsIntrinsic<[],
+ [llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>,
+ LLVMMatchType<0>,
+ llvm_nxv1i1_ty, llvm_ptr_ty],
+ [IntrArgMemOnly, NoCapture<ArgIndex<5>>]>;
class AdvSIMD_SVE_Index_Intrinsic<list<IntrinsicProperty> Attrs = []>
: DefaultAttrsIntrinsic<[llvm_anyvector_ty],
@@ -2915,10 +2944,20 @@ def int_aarch64_sve_ld2q_sret : AdvSIMD_2Vec_PredLoad_Intrinsic;
def int_aarch64_sve_ld3q_sret : AdvSIMD_3Vec_PredLoad_Intrinsic;
def int_aarch64_sve_ld4q_sret : AdvSIMD_4Vec_PredLoad_Intrinsic;
+// TODO: Remove old intrinsics that take a wrong predicate type.
+def int_aarch64_sve_ld2q_qpred : AdvSIMD_2Vec_PredLoadQ_Intrinsic;
+def int_aarch64_sve_ld3q_qpred : AdvSIMD_3Vec_PredLoadQ_Intrinsic;
+def int_aarch64_sve_ld4q_qpred : AdvSIMD_4Vec_PredLoadQ_Intrinsic;
+
def int_aarch64_sve_st2q : AdvSIMD_2Vec_PredStore_Intrinsic;
def int_aarch64_sve_st3q : AdvSIMD_3Vec_PredStore_Intrinsic;
def int_aarch64_sve_st4q : AdvSIMD_4Vec_PredStore_Intrinsic;
+// TODO: Remove old intrinsics that take a wrong predicate type.
+def int_aarch64_sve_st2q_qpred : AdvSIMD_2Vec_PredStoreQ_Intrinsic;
+def int_aarch64_sve_st3q_qpred : AdvSIMD_3Vec_PredStoreQ_Intrinsic;
+def int_aarch64_sve_st4q_qpred : AdvSIMD_4Vec_PredStoreQ_Intrinsic;
+
//
// SVE2.1 - Contiguous stores to multiple consecutive vectors
//
diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index b83ff4aa550c3..ed0ed3f97e9bc 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -5496,15 +5496,18 @@ void AArch64DAGToDAGISel::Select(SDNode *Node) {
case Intrinsic::aarch64_ld64b:
SelectLoad(Node, 8, AArch64::LD64B, AArch64::x8sub_0);
return;
- case Intrinsic::aarch64_sve_ld2q_sret: {
+ case Intrinsic::aarch64_sve_ld2q_sret:
+ case Intrinsic::aarch64_sve_ld2q_qpred: {
SelectPredicatedLoad(Node, 2, 4, AArch64::LD2Q_IMM, AArch64::LD2Q, true);
return;
}
- case Intrinsic::aarch64_sve_ld3q_sret: {
+ case Intrinsic::aarch64_sve_ld3q_sret:
+ case Intrinsic::aarch64_sve_ld3q_qpred: {
SelectPredicatedLoad(Node, 3, 4, AArch64::LD3Q_IMM, AArch64::LD3Q, true);
return;
}
- case Intrinsic::aarch64_sve_ld4q_sret: {
+ case Intrinsic::aarch64_sve_ld4q_sret:
+ case Intrinsic::aarch64_sve_ld4q_qpred: {
SelectPredicatedLoad(Node, 4, 4, AArch64::LD4Q_IMM, AArch64::LD4Q, true);
return;
}
@@ -7021,15 +7024,18 @@ void AArch64DAGToDAGISel::Select(SDNode *Node) {
}
break;
}
- case Intrinsic::aarch64_sve_st2q: {
+ case Intrinsic::aarch64_sve_st2q:
+ case Intrinsic::aarch64_sve_st2q_qpred: {
SelectPredicatedStore(Node, 2, 4, AArch64::ST2Q, AArch64::ST2Q_IMM);
return;
}
- case Intrinsic::aarch64_sve_st3q: {
+ case Intrinsic::aarch64_sve_st3q:
+ case Intrinsic::aarch64_sve_st3q_qpred: {
SelectPredicatedStore(Node, 3, 4, AArch64::ST3Q, AArch64::ST3Q_IMM);
return;
}
- case Intrinsic::aarch64_sve_st4q: {
+ case Intrinsic::aarch64_sve_st4q:
+ case Intrinsic::aarch64_sve_st4q_qpred: {
SelectPredicatedStore(Node, 4, 4, AArch64::ST4Q, AArch64::ST4Q_IMM);
return;
}
@@ -7704,7 +7710,7 @@ static EVT getPackedVectorTypeFromPredicateType(LLVMContext &Ctx, EVT PredVT,
return EVT();
if (PredVT != MVT::nxv16i1 && PredVT != MVT::nxv8i1 &&
- PredVT != MVT::nxv4i1 && PredVT != MVT::nxv2i1)
+ PredVT != MVT::nxv4i1 && PredVT != MVT::nxv2i1 && PredVT != MVT::nxv1i1)
return EVT();
ElementCount EC = PredVT.getVectorElementCount();
@@ -7770,23 +7776,29 @@ static EVT getMemVTFromNode(LLVMContext &Ctx, SDNode *Root) {
Ctx, Root->getOperand(2)->getValueType(0), /*NumVec=*/1);
case Intrinsic::aarch64_sve_ld2_sret:
case Intrinsic::aarch64_sve_ld2q_sret:
+ case Intrinsic::aarch64_sve_ld2q_qpred:
return getPackedVectorTypeFromPredicateType(
Ctx, Root->getOperand(2)->getValueType(0), /*NumVec=*/2);
case Intrinsic::aarch64_sve_st2q:
+ case Intrinsic::aarch64_sve_st2q_qpred:
return getPackedVectorTypeFromPredicateType(
Ctx, Root->getOperand(4)->getValueType(0), /*NumVec=*/2);
case Intrinsic::aarch64_sve_ld3_sret:
case Intrinsic::aarch64_sve_ld3q_sret:
+ case Intrinsic::aarch64_sve_ld3q_qpred:
return getPackedVectorTypeFromPredicateType(
Ctx, Root->getOperand(2)->getValueType(0), /*NumVec=*/3);
case Intrinsic::aarch64_sve_st3q:
+ case Intrinsic::aarch64_sve_st3q_qpred:
return getPackedVectorTypeFromPredicateType(
Ctx, Root->getOperand(5)->getValueType(0), /*NumVec=*/3);
case Intrinsic::aarch64_sve_ld4_sret:
case Intrinsic::aarch64_sve_ld4q_sret:
+ case Intrinsic::aarch64_sve_ld4q_qpred:
return getPackedVectorTypeFromPredicateType(
Ctx, Root->getOperand(2)->getValueType(0), /*NumVec=*/4);
case Intrinsic::aarch64_sve_st4q:
+ case Intrinsic::aarch64_sve_st4q_qpred:
return getPackedVectorTypeFromPredicateType(
Ctx, Root->getOperand(6)->getValueType(0), /*NumVec=*/4);
case Intrinsic::aarch64_sve_ld1udq:
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 730bec428a38e..52f9f51582c94 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -1717,10 +1717,13 @@ static SVEIntrinsicInfo constructSVEIntrinsicInfo(IntrinsicInst &II) {
case Intrinsic::aarch64_sve_ld1uwq:
case Intrinsic::aarch64_sve_ld2_sret:
case Intrinsic::aarch64_sve_ld2q_sret:
+ case Intrinsic::aarch64_sve_ld2q_qpred:
case Intrinsic::aarch64_sve_ld3_sret:
case Intrinsic::aarch64_sve_ld3q_sret:
+ case Intrinsic::aarch64_sve_ld3q_qpred:
case Intrinsic::aarch64_sve_ld4_sret:
case Intrinsic::aarch64_sve_ld4q_sret:
+ case Intrinsic::aarch64_sve_ld4q_qpred:
case Intrinsic::aarch64_sve_ldff1:
case Intrinsic::aarch64_sve_ldff1_gather:
case Intrinsic::aarch64_sve_ldff1_gather_index:
@@ -1775,12 +1778,15 @@ static SVEIntrinsicInfo constructSVEIntrinsicInfo(IntrinsicInst &II) {
return SVEIntrinsicInfo::defaultVoidOp(1);
case Intrinsic::aarch64_sve_st2:
case Intrinsic::aarch64_sve_st2q:
+ case Intrinsic::aarch64_sve_st2q_qpred:
return SVEIntrinsicInfo::defaultVoidOp(2);
case Intrinsic::aarch64_sve_st3:
case Intrinsic::aarch64_sve_st3q:
+ case Intrinsic::aarch64_sve_st3q_qpred:
return SVEIntrinsicInfo::defaultVoidOp(3);
case Intrinsic::aarch64_sve_st4:
case Intrinsic::aarch64_sve_st4q:
+ case Intrinsic::aarch64_sve_st4q_qpred:
return SVEIntrinsicInfo::defaultVoidOp(4);
}
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-multivec-loads.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-multivec-loads.ll
index 393cdeb6f1b37..b38814e1d03b2 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-multivec-loads.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-multivec-loads.ll
@@ -17,6 +17,31 @@ define { <vscale x 16 x i8>, <vscale x 16 x i8> } @ld2q_si_i8_off16(<vscale x 16
ret { <vscale x 16 x i8>, <vscale x 16 x i8> } %res
}
+define { <vscale x 16 x i8>, <vscale x 16 x i8> } @ld2q_si_i8_off16_qpred(<vscale x 1 x i1> %pg, ptr %addr ) {
+; CHECK-LABEL: ld2q_si_i8_off16_qpred:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ld2q { z0.q, z1.q }, p0/z, [x0, #-16, mul vl]
+; CHECK-NEXT: ret
+ %base = getelementptr <vscale x 16 x i8>, ptr %addr, i64 -16
+ %base_ptr = bitcast ptr %base to ptr
+ %res = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld2q.qpred.nxv16i8(<vscale x 1 x i1> %pg, ptr %base_ptr);
+ ret { <vscale x 16 x i8>, <vscale x 16 x i8> } %res
+}
+
+define { <vscale x 16 x i8>, <vscale x 16 x i8> } @ld2q_si_i8_off16_qpred_true(ptr %addr ) {
+; CHECK-LABEL: ld2q_si_i8_off16_qpred_true:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: ld2q { z0.q, z1.q }, p0/z, [x0, #-16, mul vl]
+; CHECK-NEXT: ret
+ %base = getelementptr <vscale x 16 x i8>, ptr %addr, i64 -16
+ %base_ptr = bitcast ptr %base to ptr
+ %pred.elem = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv16i1(<vscale x 16 x i1> splat (i1 true))
+ %pred = call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> %pred.elem)
+ %res = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld2q.qpred.nxv16i8(<vscale x 1 x i1> %pred, ptr %base_ptr);
+ ret { <vscale x 16 x i8>, <vscale x 16 x i8> } %res
+}
+
define { <vscale x 16 x i8>, <vscale x 16 x i8> } @ld2q_si_i8_off14(<vscale x 16 x i1> %pg, ptr %addr ) {
; CHECK-LABEL: ld2q_si_i8_off14:
; CHECK: // %bb.0:
@@ -58,6 +83,20 @@ define { <vscale x 8 x i16>, <vscale x 8 x i16> } @ld2q_si_i16(<vscale x 8 x i1>
ret { <vscale x 8 x i16>, <vscale x 8 x i16> } %res
}
+define { <vscale x 8 x i16>, <vscale x 8 x i16> } @ld2q_si_i16_qpred_true(ptr %addr) {
+; CHECK-LABEL: ld2q_si_i16_qpred_true:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: ld2q { z0.q, z1.q }, p0/z, [x0, #-16, mul vl]
+; CHECK-NEXT: ret
+ %base = getelementptr <vscale x 8 x i16>, ptr %addr, i64 -16
+ %base_ptr = bitcast ptr %base to ptr
+ %pred.elem = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv8i1(<vscale x 8 x i1> splat (i1 true))
+ %pred = call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> %pred.elem)
+ %res = call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld2q.qpred.nxv8i16(<vscale x 1 x i1> %pred, ptr %base_ptr);
+ ret { <vscale x 8 x i16>, <vscale x 8 x i16> } %res
+}
+
define { <vscale x 8 x i16>, <vscale x 8 x i16> } @ld2q_ss_i16(<vscale x 8 x i1> %pg, ptr %addr, i64 %a) {
; CHECK-LABEL: ld2q_ss_i16:
; CHECK: // %bb.0:
@@ -87,6 +126,19 @@ define { <vscale x 4 x i32>, <vscale x 4 x i32> } @ld2q_si_i32(<vscale x 4 x i1>
ret { <vscale x 4 x i32>, <vscale x 4 x i32> } %res
}
+define { <vscale x 4 x i32>, <vscale x 4 x i32> } @ld2q_si_i32_qpred_true(ptr %addr ) {
+; CHECK-LABEL: ld2q_si_i32_qpred_true:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: ld2q { z0.q, z1.q }, p0/z, [x0, #-16, mul vl]
+; CHECK-NEXT: ret
+ %base = getelementptr <vscale x 4 x i32>, ptr %addr, i64 -16
+ %pred.elem = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv4i1(<vscale x 4 x i1> splat (i1 true))
+ %pred = call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> %pred.elem)
+ %res = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld2q.qpred.nxv4i32(<vscale x 1 x i1> %pred, ptr %base);
+ ret { <vscale x 4 x i32>, <vscale x 4 x i32> } %res
+}
+
define { <vscale x 4 x i32>, <vscale x 4 x i32> } @ld2q_ss_i32(<vscale x 4 x i1> %pg, ptr %addr, i64 %a) {
; CHECK-LABEL: ld2q_ss_i32:
; CHECK: // %bb.0:
@@ -117,6 +169,20 @@ define { <vscale x 2 x i64>, <vscale x 2 x i64> } @ld2q_si_i64(<vscale x 2 x i1>
ret { <vscale x 2 x i64>, <vscale x 2 x i64> } %res
}
+define { <vscale x 2 x i64>, <vscale x 2 x i64> } @ld2q_si_i64_qpred_true(ptr %addr ) {
+; CHECK-LABEL: ld2q_si_i64_qpred_true:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: ld2q { z0.q, z1.q }, p0/z, [x0, #-16, mul vl]
+; CHECK-NEXT: ret
+ %base = getelementptr <vscale x 2 x i64>, ptr %addr, i64 -16
+ %base_ptr = bitcast ptr %base to ptr
+ %pred.elem = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv2i1(<vscale x 2 x i1> splat (i1 true))
+ %pred = call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> %pred.elem)
+ %res = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld2q.qpred.nxv2i64(<vscale x 1 x i1> %pred, ptr %base_ptr);
+ ret { <vscale x 2 x i64>, <vscale x 2 x i64> } %res
+}
+
define { <vscale x 2 x i64>, <vscale x 2 x i64> } @ld2q_ss_i64(<vscale x 2 x i1> %pg, ptr %addr, i64 %a) {
; CHECK-LABEL: ld2q_ss_i64:
; CHECK: // %bb.0:
@@ -268,6 +334,17 @@ define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @ld3q_si_i
ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res
}
+define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @ld3q_si_i8_off24_qpred(<vscale x 1 x i1> %pg, ptr %addr ) {
+; CHECK-LABEL: ld3q_si_i8_off24_qpred:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ld3q { z0.q - z2.q }, p0/z, [x0, #-24, mul vl]
+; CHECK-NEXT: ret
+ %base = getelementptr <vscale x 16 x i8>, ptr %addr, i64 -24
+ %base_ptr = bitcast ptr %base to ptr
+ %res = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld3q.qpred.nxv16i8(<vscale x 1 x i1> %pg, ptr %base_ptr);
+ ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res
+}
+
define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @ld3q_si_i8_off21(<vscale x 16 x i1> %pg, ptr %addr ) {
; CHECK-LABEL: ld3q_si_i8_off21:
; CHECK: // %bb.0:
@@ -519,6 +596,17 @@ define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 1
ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res
}
+define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @ld4q_si_i8_off32_qpred(<vscale x 1 x i1> %pg, ptr %addr ) {
+; CHECK-LABEL: ld4q_si_i8_off32_qpred:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ld4q { z0.q - z3.q }, p0/z, [x0, #-32, mul vl]
+; CHECK-NEXT: ret
+ %base = getelementptr <vscale x 16 x i8>, ptr %addr, i64 -32
+ %base_ptr = bitcast ptr %base to ptr
+ %res = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld4q.qpred.nxv16i8(<vscale x 1 x i1> %pg, ptr %base_ptr);
+ ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res
+}
+
define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @ld4q_si_i8_off28(<vscale x 16 x i1> %pg, ptr %addr ) {
; CHECK-LABEL: ld4q_si_i8_off28:
; CHECK: // %bb.0:
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-multivec-stores.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-multivec-stores.ll
index 262fd054b3cad..3336e099f8951 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-multivec-stores.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-multivec-stores.ll
@@ -19,6 +19,33 @@ define void @st2q_ss_i8(<vscale x 16 x i8> %v0, <vscale x 16 x i8> %v1, <vscale
ret void
}
+define void @st2q_ss_i8_qpred(<vscale x 16 x i8> %v0, <vscale x 16 x i8> %v1, <vscale x 1 x i1> %pred, ptr %addr, i64 %offset) {
+; CHECK-LABEL: st2q_ss_i8_qpred:
+; CHECK: // %bb.0:
+; CHECK-NEXT: st2q { z0.q, z1.q }, p0, [x0, x1, lsl #4]
+; CHECK-NEXT: ret
+ %1 = getelementptr i128, ptr %addr, i64 %offset
+ call void @llvm.aarch64.sve.st2q.qpred.nxv16i8(<vscale x 16 x i8> %v0, <vscale x 16 x i8> %v1 ,
+ <vscale x 1 x i1> %pred,
+ ptr %1)
+ ret void
+}
+
+define void @st2q_ss_i8_qpred_true(<vscale x 16 x i8> %v0, <vscale x 16 x i8> %v1, ptr %addr, i64 %offset) {
+; CHECK-LABEL: st2q_ss_i8_qpred_true:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: st2q { z0.q, z1.q }, p0, [x0, x1, lsl #4]
+; CHECK-NEXT: ret
+ %1 = getelementptr i128, ptr %addr, i64 %offset
+ %pred.elem = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv16i1(<vscale x 16 x i1> splat (i1 true))
+ %pred = call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> %pred.elem)
+ call void @llvm.aarch64.sve.st2q.qpred.nxv16i8(<vscale x 16 x i8> %v0, <vscale x 16 x i8> %v1 ,
+ <vscale x 1 x i1> %pred,
+ ptr %1)
+ ret void
+}
+
define void @st2q_ss_i16(<vscale x 8 x i16> %v0, <vscale x 8 x i16> %v1, <vscale x 8 x i1> %pred, ptr %addr, i64 %offset) {
; CHECK-LABEL: st2q_ss_i16:
; CHECK: // %bb.0:
@@ -32,6 +59,22 @@ define void @st2q_ss_i16(<vscale x 8 x i16> %v0, <vscale x 8 x i16> %v1, <vscale
ret void
}
+define void @st2q_ss_i16_qpred_true(<vscale x 8 x i16> %v0, <vscale x 8 x i16> %v1, ptr %addr, i64 %offset) {
+; CHECK-LABEL: st2q_ss_i16_qpred_true:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: st2q { z0.q, z1.q }, p0, [x0, x1, lsl #4]
+; CHECK-NEXT: ret
+ %1 = getelementptr i128, ptr %addr, i64 %offset
+ %pred.elem = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv8i1(<vscale x 8 x i1> splat (i1 true))
+ %pred = call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> %pred.elem)
+ call void @llvm.aarch64.sve.st2q.qpred.nxv8i16(<vscale x 8 x i16> %v0,
+ <vscale x 8 x i16> %v1,
+ <vscale x 1 x i1> %pred,
+ ptr %1)
+ ret void
+}
+
define void @st2q_ss_i32(<vscale x 4 x i32> %v0, <vscale x 4 x i32> %v1, <vscale x 4 x i1> %pred, ptr %addr, i64 %offset) {
; CHECK-LABEL: st2q_ss_i32:
; CHECK: // %bb.0:
@@ -45,6 +88,22 @@ define void @st2q_ss_i32(<vscale x 4 x i32> %v0, <vscale x 4 x i32> %v1, <vscale
ret void
}
+define void @st2q_ss_i32_qpred_true(<vscale x 4 x i32> %v0, <vscale x 4 x i32> %v1, ptr %addr, i64 %offset) {
+; CHECK-LABEL: st2q_ss_i32_qpred_true:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: st2q { z0.q, z1.q }, p0, [x0, x1, lsl #4]
+; CHECK-NEXT: ret
+ %1 = getelementptr i128, ptr %addr, i64 %offset
+ %pred.elem = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv4i1(<vscale x 4 x i1> splat (i1 true))
+ %pred = call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> %pred.elem)
+ call void @llvm.aarch64.sve.st2q.qpred.nxv4i32(<vscale x 4 x i32> %v0,
+ <vscale x 4 x i32> %v1,
+ <vscale x 1 x i1> %pred,
+ ptr %1)
+ ret void
+}
+
define void @st2q_ss_i64(<vscale x 2 x i64> %v0, <vscale x 2 x i64> %v1, <vscale x 2 x i1> %pred, ptr %addr, i64 %offset) {
; CHECK-LABEL: st2q_ss_i64:
; CHECK: // %bb.0:
@@ -58,6 +117,22 @@ define void @st2q_ss_i64(<vscale x 2 x i64> %v0, <vscale x 2 x i64> %v1, <vscale
ret void
}
+define void @st2q_ss_i64_qpred_true(<vscale x 2 x i64> %v0, <vscale x 2 x i64> %v1, ptr %addr, i64 %offset) {
+; CHECK-LABEL: st2q_ss_i64_qpred_true:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: st2q { z0.q, z1.q }, p0, [x0, x1, lsl #4]
+; CHECK-NEXT: ret
+ %1 = getelementptr i128, ptr %addr, i64 %offset
+ %pred.elem = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv2i1(<vscale x 2 x i1> splat (i1 true))
+ %pred = call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> %pred.elem)
+ call void @llvm.aarch64.sve.st2q.qpred.nxv2i64(<vscale x 2 x i64> %v0,
+ <vscale x 2 x i64> %v1,
+ <vscale x 1 x i1> %pred,
+ ptr %1)
+ ret void
+}
+
define void @st2q_ss_f16(<vscale x 8 x half> %v0, <vscale x 8 x half> %v1, <vscale x 8 x i1> %pred, ptr %addr, i64 %offset) {
; CHECK-LABEL: st2q_ss_f16:
; CHECK: // %bb.0:
@@ -246,6 +321,20 @@ define void @st3q_ss_i8(<vscale x 16 x i8> %v0, <vscale x 16 x i8> %v1, <vscale
ret void
}
+define void @st3q_ss_i8_qpred(<vscale x 16 x i8> %v0, <vscale x 16 x i8> %v1, <vscale x 16 x i8> %v2, <vscale x 1 x i1> %pred, ptr %addr, i64 %offset) {
+; CHECK-LABEL: st3q_ss_i8_qpred:
+; CHECK: // %bb.0:
+; CHECK-NEXT: st3q { z0.q - z2.q }, p0, [x0, x1, lsl #4]
+; CHECK-NEXT: ret
+ %1 = getelementptr i128, ptr %addr, i64 %offset
+ call void @llvm.aarch64.sve.st3q.qpred.nxv16i8(<vscale x 16 x i8>%v0,
+ <vscale x 16 x i8> %v1,
+ <vscale x 16 x i8> %v2,
+ <vscale x 1 x i1> %pred,
+ ptr %1)
+ ret void
+}
+
define void @st3q_ss_i16(<vscale x 8 x i16> %v0, <vscale x 8 x i16> %v1, <vscale x 8 x i16> %v2, <vscale x 8 x i1> %pred, ptr %addr, i64 %offset) {
; CHECK-LABEL: st3q_ss_i16:
; CHECK: // %bb.0:
@@ -488,6 +577,21 @@ define void @st4q_ss_i8(<vscale x 16 x i8> %v0, <vscale x 16 x i8> %v1, <vscale
ret void
}
+define void @st4q_ss_i8_qpred(<vscale x 16 x i8> %v0, <vscale x 16 x i8> %v1, <vscale x 16 x i8> %v2,<vscale x 16 x i8> %v3, <vscale x 1 x i1> %pred, ptr %addr, i64 %offset) {
+; CHECK-LABEL: st4q_ss_i8_qpred:
+; CHECK: // %bb.0:
+; CHECK-NEXT: st4q { z0.q - z3.q }, p0, [x0, x1, lsl #4]
+; CHECK-NEXT: ret
+ %1 = getelementptr i128, ptr %addr, i64 %offset
+ call void @llvm.aarch64.sve.st4q.qpred.nxv16i8(<vscale x 16 x i8>%v0,
+ <vscale x 16 x i8> %v1,
+ <vscale x 16 x i8> %v2,
+ <vscale x 16 x i8> %v3,
+ <vscale x 1 x i1> %pred,
+ ptr %1)
+ ret void
+}
+
define void @st4q_ss_i16(<vscale x 8 x i16> %v0, <vscale x 8 x i16> %v1, <vscale x 8 x i16> %v2, <vscale x 8 x i16> %v3, <vscale x 8 x i1> %pred, ptr %addr, i64 %offset) {
; CHECK-LABEL: st4q_ss_i16:
; CHECK: // %bb.0:
>From 5db0250381557816d5b89999afc6d89620b8dbd8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ga=C3=ABtan=20Bossu?= <gaetan.bossu at arm.com>
Date: Thu, 4 Jun 2026 17:16:12 +0000
Subject: [PATCH 2/7] [AArch64] Emit generic LLVM intrinsics for NEON min/max
This updates the clang lowering of NEON min/max builtins to use generic
LLVM intrinsics instead of target-specific ones.
Integer types will be lowered using llvm.smin, llvm.smax, llvm.umin or
llvm.umax. FP operations will be lowered using llvm.minimum or
llvm.maximum.
Currently, FP "number" operations like FMINNM/FMAXNM are unchanged and
will rely on the existing intrinsics. But I think they could eventually
get lowered to llvm.minnum and llvm.maxnum.
Note that the LLVM intrinsics like aarch64.neon.umax.xxxx() are not
removed and are still used in ISel tests.
WIP: CIR Needs updating as well.
---
.../include/clang/Basic/AArch64CodeGenUtils.h | 40 +++-
clang/lib/CodeGen/CodeGenFunction.h | 11 +-
clang/lib/CodeGen/TargetBuiltins/ARM.cpp | 47 ++---
clang/test/CodeGen/AArch64/neon/intrinsics.c | 188 ++++++++++++------
.../CodeGen/AArch64/v8.2a-fp16-intrinsics.c | 4 +-
.../CodeGen/AArch64/v8.2a-neon-intrinsics.c | 44 ++--
.../test/CodeGen/arm-v8.2a-neon-intrinsics.c | 8 +-
clang/test/CodeGen/arm_neon_intrinsics.c | 56 +++---
8 files changed, 244 insertions(+), 154 deletions(-)
diff --git a/clang/include/clang/Basic/AArch64CodeGenUtils.h b/clang/include/clang/Basic/AArch64CodeGenUtils.h
index f64a41df63cf8..e7c00c3360003 100644
--- a/clang/include/clang/Basic/AArch64CodeGenUtils.h
+++ b/clang/include/clang/Basic/AArch64CodeGenUtils.h
@@ -36,10 +36,11 @@ enum {
VectorizeArgTypes = (1 << 4),
InventFloatType = (1 << 5),
- UnsignedAlts = (1 << 6),
+ UnsignedAlts = (1 << 6), ///< Alternate intrinsic for signed types
+ FloatAlts = (1 << 7), ///< Alternate intrinsic for floating-point types
- Use64BitVectors = (1 << 7),
- Use128BitVectors = (1 << 8),
+ Use64BitVectors = (1 << 8),
+ Use128BitVectors = (1 << 9),
Vectorize1ArgType = Add1ArgType | VectorizeArgTypes,
VectorRet = AddRetType | VectorizeRetType,
@@ -54,6 +55,7 @@ struct ARMVectorIntrinsicInfo {
unsigned BuiltinID;
unsigned LLVMIntrinsic;
unsigned AltLLVMIntrinsic;
+ unsigned FloatLLVMIntrinsic;
uint64_t TypeModifier;
bool operator<(unsigned RHSBuiltinID) const {
@@ -64,17 +66,33 @@ struct ARMVectorIntrinsicInfo {
}
};
+/// Builtin to be custom-lowered in EmitCommonNeonBuiltinExpr().
#define NEONMAP0(NameBase) \
- {#NameBase, NEON::BI__builtin_neon_##NameBase, 0, 0, 0}
+ { #NameBase, NEON::BI__builtin_neon_##NameBase, 0, 0, 0, 0 }
+/// Construct a builtin->intrinsic mapping.
#define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
- {#NameBase, NEON::BI__builtin_neon_##NameBase, \
- llvm::Intrinsic::LLVMIntrinsic, 0, TypeModifier}
+ { \
+ #NameBase, NEON::BI__builtin_neon_##NameBase, \
+ llvm::Intrinsic::LLVMIntrinsic, 0, 0, TypeModifier \
+ }
+/// Construct a builtin->intrinsic mapping with 2 alternatives.
#define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier) \
- {#NameBase, NEON::BI__builtin_neon_##NameBase, \
- llvm::Intrinsic::LLVMIntrinsic, llvm::Intrinsic::AltLLVMIntrinsic, \
- TypeModifier}
+ { \
+ #NameBase, NEON::BI__builtin_neon_##NameBase, \
+ llvm::Intrinsic::LLVMIntrinsic, llvm::Intrinsic::AltLLVMIntrinsic, 0, \
+ TypeModifier \
+ }
+
+/// Construct a builtin->intrinsic mapping with 3 alternatives.
+#define NEONMAP3(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, \
+ FloatLLVMIntrinsic, TypeModifier) \
+ { \
+ #NameBase, NEON::BI__builtin_neon_##NameBase, \
+ llvm::Intrinsic::LLVMIntrinsic, llvm::Intrinsic::AltLLVMIntrinsic, \
+ llvm::Intrinsic::FloatLLVMIntrinsic, TypeModifier \
+ }
// clang-format off
const inline ARMVectorIntrinsicInfo AArch64SIMDIntrinsicMap [] = {
@@ -233,6 +251,10 @@ const inline ARMVectorIntrinsicInfo AArch64SIMDIntrinsicMap [] = {
NEONMAP1(vld1q_x2_v, aarch64_neon_ld1x2, 0),
NEONMAP1(vld1q_x3_v, aarch64_neon_ld1x3, 0),
NEONMAP1(vld1q_x4_v, aarch64_neon_ld1x4, 0),
+ NEONMAP3(vmax_v, umax, smax, maximum, Add1ArgType | UnsignedAlts | FloatAlts),
+ NEONMAP3(vmaxq_v, umax, smax, maximum, Add1ArgType | UnsignedAlts | FloatAlts),
+ NEONMAP3(vmin_v, umin, smin, minimum, Add1ArgType | UnsignedAlts | FloatAlts),
+ NEONMAP3(vminq_v, umin, smin, minimum, Add1ArgType | UnsignedAlts | FloatAlts),
NEONMAP1(vmmlaq_f16_f16, aarch64_neon_fmmla, 0),
NEONMAP1(vmmlaq_f32_f16, aarch64_neon_fmmla, 0),
NEONMAP1(vmmlaq_s32, aarch64_neon_smmla, 0),
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 6bb9f285ebcfd..7a47a65614762 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -4827,12 +4827,11 @@ class CodeGenFunction : public CodeGenTypeCache {
llvm::Value *EmitCMSEClearRecord(llvm::Value *V, llvm::ArrayType *ATy,
QualType RTy);
- llvm::Value *
- EmitCommonNeonBuiltinExpr(unsigned BuiltinID, unsigned LLVMIntrinsic,
- unsigned AltLLVMIntrinsic, const char *NameHint,
- unsigned Modifier, const CallExpr *E,
- SmallVectorImpl<llvm::Value *> &Ops, Address PtrOp0,
- Address PtrOp1, llvm::Triple::ArchType Arch);
+ llvm::Value *EmitCommonNeonBuiltinExpr(
+ unsigned BuiltinID, unsigned LLVMIntrinsic, unsigned AltLLVMIntrinsic,
+ unsigned FloatLLVMIntrinsic, const char *NameHint, unsigned Modifier,
+ const CallExpr *E, SmallVectorImpl<llvm::Value *> &Ops, Address PtrOp0,
+ Address PtrOp1, llvm::Triple::ArchType Arch);
llvm::Function *LookupNeonLLVMIntrinsic(unsigned IntrinsicID,
unsigned Modifier, llvm::Type *ArgTy,
diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
index ece8ff21561cf..ec0bdee6a78fa 100644
--- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
@@ -722,14 +722,14 @@ static const ARMVectorIntrinsicInfo ARMSIMDIntrinsicMap [] = {
NEONMAP1(vld4q_dup_v, arm_neon_vld4dup, 0),
NEONMAP1(vld4q_lane_v, arm_neon_vld4lane, 0),
NEONMAP1(vld4q_v, arm_neon_vld4, 0),
- NEONMAP2(vmax_v, arm_neon_vmaxu, arm_neon_vmaxs, Add1ArgType | UnsignedAlts),
+ NEONMAP3(vmax_v, umax, smax, maximum, Add1ArgType | UnsignedAlts | FloatAlts),
NEONMAP1(vmaxnm_v, arm_neon_vmaxnm, Add1ArgType),
NEONMAP1(vmaxnmq_v, arm_neon_vmaxnm, Add1ArgType),
- NEONMAP2(vmaxq_v, arm_neon_vmaxu, arm_neon_vmaxs, Add1ArgType | UnsignedAlts),
- NEONMAP2(vmin_v, arm_neon_vminu, arm_neon_vmins, Add1ArgType | UnsignedAlts),
+ NEONMAP3(vmaxq_v, umax, smax, maximum, Add1ArgType | UnsignedAlts | FloatAlts),
+ NEONMAP3(vmin_v, umin, smin, minimum, Add1ArgType | UnsignedAlts | FloatAlts),
NEONMAP1(vminnm_v, arm_neon_vminnm, Add1ArgType),
NEONMAP1(vminnmq_v, arm_neon_vminnm, Add1ArgType),
- NEONMAP2(vminq_v, arm_neon_vminu, arm_neon_vmins, Add1ArgType | UnsignedAlts),
+ NEONMAP3(vminq_v, umin, smin, minimum, Add1ArgType | UnsignedAlts | FloatAlts),
NEONMAP1(vmmlaq_s32, arm_neon_smmla, 0),
NEONMAP1(vmmlaq_u32, arm_neon_ummla, 0),
NEONMAP0(vmovl_v),
@@ -1004,6 +1004,7 @@ static const std::pair<unsigned, unsigned> NEONEquivalentIntrinsicMap[] = {
{ NEON::BI__builtin_neon_vstl1q_lane_f64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
{ NEON::BI__builtin_neon_vstl1q_lane_p64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
};
+// clang-format on
#undef NEONMAP0
#undef NEONMAP1
@@ -1012,11 +1013,11 @@ static const std::pair<unsigned, unsigned> NEONEquivalentIntrinsicMap[] = {
#define SVEMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
{ \
#NameBase, SVE::BI__builtin_sve_##NameBase, Intrinsic::LLVMIntrinsic, 0, \
- TypeModifier \
+ 0, TypeModifier \
}
#define SVEMAP2(NameBase, TypeModifier) \
- { #NameBase, SVE::BI__builtin_sve_##NameBase, 0, 0, TypeModifier }
+ { #NameBase, SVE::BI__builtin_sve_##NameBase, 0, 0, 0, TypeModifier }
static const ARMVectorIntrinsicInfo AArch64SVEIntrinsicMap[] = {
#define GET_SVE_LLVM_INTRINSIC_MAP
#include "clang/Basic/arm_sve_builtin_cg.inc"
@@ -1030,11 +1031,11 @@ static const ARMVectorIntrinsicInfo AArch64SVEIntrinsicMap[] = {
#define SMEMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
{ \
#NameBase, SME::BI__builtin_sme_##NameBase, Intrinsic::LLVMIntrinsic, 0, \
- TypeModifier \
+ 0, TypeModifier \
}
#define SMEMAP2(NameBase, TypeModifier) \
- { #NameBase, SME::BI__builtin_sme_##NameBase, 0, 0, TypeModifier }
+ { #NameBase, SME::BI__builtin_sme_##NameBase, 0, 0, 0, TypeModifier }
static const ARMVectorIntrinsicInfo AArch64SMEIntrinsicMap[] = {
#define GET_SME_LLVM_INTRINSIC_MAP
#include "clang/Basic/arm_sme_builtin_cg.inc"
@@ -1183,9 +1184,9 @@ static Value *EmitCommonNeonSISDBuiltinExpr(
Value *CodeGenFunction::EmitCommonNeonBuiltinExpr(
unsigned BuiltinID, unsigned LLVMIntrinsic, unsigned AltLLVMIntrinsic,
- const char *NameHint, unsigned Modifier, const CallExpr *E,
- SmallVectorImpl<llvm::Value *> &Ops, Address PtrOp0, Address PtrOp1,
- llvm::Triple::ArchType Arch) {
+ unsigned FloatLLVMIntrinsic, const char *NameHint, unsigned Modifier,
+ const CallExpr *E, SmallVectorImpl<llvm::Value *> &Ops, Address PtrOp0,
+ Address PtrOp1, llvm::Triple::ArchType Arch) {
// Extract the trailing immediate argument that encodes the type discriminator
// for this overloaded intrinsic.
@@ -1218,6 +1219,8 @@ Value *CodeGenFunction::EmitCommonNeonBuiltinExpr(
unsigned Int = LLVMIntrinsic;
if ((Modifier & UnsignedAlts) && !Usgn)
Int = AltLLVMIntrinsic;
+ if ((Modifier & FloatAlts) && Floating)
+ Int = FloatLLVMIntrinsic;
switch (BuiltinID) {
default: break;
@@ -2725,7 +2728,8 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
if (Builtin)
return EmitCommonNeonBuiltinExpr(
Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
- Builtin->NameHint, Builtin->TypeModifier, E, Ops, PtrOp0, PtrOp1, Arch);
+ Builtin->FloatLLVMIntrinsic, Builtin->NameHint, Builtin->TypeModifier,
+ E, Ops, PtrOp0, PtrOp1, Arch);
unsigned Int;
switch (BuiltinID) {
@@ -5418,7 +5422,8 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
if (Builtin)
return EmitCommonNeonBuiltinExpr(
Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
- Builtin->NameHint, Builtin->TypeModifier, E, Ops,
+ Builtin->FloatLLVMIntrinsic, Builtin->NameHint, Builtin->TypeModifier,
+ E, Ops,
/*never use addresses*/ Address::invalid(), Address::invalid(), Arch);
if (Value *V = EmitAArch64TblBuiltinExpr(*this, BuiltinID, E, Ops, Arch))
@@ -6199,24 +6204,12 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
Int = usgn ? Intrinsic::aarch64_neon_umull : Intrinsic::aarch64_neon_smull;
if (Type.isPoly()) Int = Intrinsic::aarch64_neon_pmull;
return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull");
- case NEON::BI__builtin_neon_vmax_v:
- case NEON::BI__builtin_neon_vmaxq_v:
- // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
- Int = usgn ? Intrinsic::aarch64_neon_umax : Intrinsic::aarch64_neon_smax;
- if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmax;
- return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmax");
case NEON::BI__builtin_neon_vmaxh_f16: {
- Int = Intrinsic::aarch64_neon_fmax;
+ Int = Intrinsic::maximum;
return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vmax");
}
- case NEON::BI__builtin_neon_vmin_v:
- case NEON::BI__builtin_neon_vminq_v:
- // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
- Int = usgn ? Intrinsic::aarch64_neon_umin : Intrinsic::aarch64_neon_smin;
- if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmin;
- return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmin");
case NEON::BI__builtin_neon_vminh_f16: {
- Int = Intrinsic::aarch64_neon_fmin;
+ Int = Intrinsic::minimum;
return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vmin");
}
case NEON::BI__builtin_neon_vabd_v:
diff --git a/clang/test/CodeGen/AArch64/neon/intrinsics.c b/clang/test/CodeGen/AArch64/neon/intrinsics.c
index f3c1014a5613d..354641852523a 100644
--- a/clang/test/CodeGen/AArch64/neon/intrinsics.c
+++ b/clang/test/CodeGen/AArch64/neon/intrinsics.c
@@ -1459,7 +1459,7 @@ int8x8_t test_vmax_s8(int8x8_t v1, int8x8_t v2) {
// CIR: cir.call_llvm_intrinsic "aarch64.neon.smax" %{{.*}}, %{{.*}} : (!cir.vector<8 x !s8i>, !cir.vector<8 x !s8i>) -> !cir.vector<8 x !s8i>
// LLVM-SAME: <8 x i8> {{.*}} [[V1:%.*]], <8 x i8> noundef [[V2:%.*]]) {{.*}} {
- // LLVM: [[VMAX_V_I:%.*]] = call <8 x i8> @llvm.aarch64.neon.smax.v8i8(<8 x i8> [[V1]], <8 x i8> [[V2]])
+ // LLVM: [[VMAX_V_I:%.*]] = call <8 x i8> @llvm.smax.v8i8(<8 x i8> [[V1]], <8 x i8> [[V2]])
// LLVM: ret <8 x i8> [[VMAX_V_I]]
return vmax_s8(v1, v2);
}
@@ -1474,8 +1474,10 @@ int16x4_t test_vmax_s16(int16x4_t v1, int16x4_t v2) {
// LLVM: [[TMP1:%.*]] = bitcast <4 x i16> [[V2]] to <8 x i8>
// LLVM: [[VMAX_V_I:%.*]] = bitcast <8 x i8> [[TMP0]] to <4 x i16>
// LLVM: [[VMAX_V1_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <4 x i16>
- // LLVM: [[VMAX_V2_I:%.*]] = call <4 x i16> @llvm.aarch64.neon.smax.v4i16(<4 x i16> [[VMAX_V_I]], <4 x i16> [[VMAX_V1_I]])
- // LLVM: ret <4 x i16> [[VMAX_V2_I]]
+ // LLVM: [[VMAX_V2_I:%.*]] = call <4 x i16> @llvm.smax.v4i16(<4 x i16> [[VMAX_V_I]], <4 x i16> [[VMAX_V1_I]])
+ // LLVM: [[VMAX_V3_I:%.*]] = bitcast <4 x i16> [[VMAX_V2_I]] to <8 x i8>
+ // LLVM: [[TMP2:%.*]] = bitcast <8 x i8> [[VMAX_V3_I]] to <4 x i16>
+ // LLVM: ret <4 x i16> [[TMP2]]
return vmax_s16(v1, v2);
}
@@ -1489,8 +1491,10 @@ int32x2_t test_vmax_s32(int32x2_t v1, int32x2_t v2) {
// LLVM: [[TMP1:%.*]] = bitcast <2 x i32> [[V2]] to <8 x i8>
// LLVM: [[VMAX_V_I:%.*]] = bitcast <8 x i8> [[TMP0]] to <2 x i32>
// LLVM: [[VMAX_V1_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <2 x i32>
- // LLVM: [[VMAX_V2_I:%.*]] = call <2 x i32> @llvm.aarch64.neon.smax.v2i32(<2 x i32> [[VMAX_V_I]], <2 x i32> [[VMAX_V1_I]])
- // LLVM: ret <2 x i32> [[VMAX_V2_I]]
+ // LLVM: [[VMAX_V2_I:%.*]] = call <2 x i32> @llvm.smax.v2i32(<2 x i32> [[VMAX_V_I]], <2 x i32> [[VMAX_V1_I]])
+ // LLVM: [[VMAX_V3_I:%.*]] = bitcast <2 x i32> [[VMAX_V2_I]] to <8 x i8>
+ // LLVM: [[TMP2:%.*]] = bitcast <8 x i8> [[VMAX_V3_I]] to <2 x i32>
+ // LLVM: ret <2 x i32> [[TMP2]]
return vmax_s32(v1, v2);
}
@@ -1500,7 +1504,7 @@ uint8x8_t test_vmax_u8(uint8x8_t v1, uint8x8_t v2) {
// CIR: cir.call_llvm_intrinsic "aarch64.neon.umax" %{{.*}}, %{{.*}} : (!cir.vector<8 x !u8i>, !cir.vector<8 x !u8i>) -> !cir.vector<8 x !u8i>
// LLVM-SAME: <8 x i8> {{.*}} [[V1:%.*]], <8 x i8> noundef [[V2:%.*]]) {{.*}} {
- // LLVM: [[VMAX_V_I:%.*]] = call <8 x i8> @llvm.aarch64.neon.umax.v8i8(<8 x i8> [[V1]], <8 x i8> [[V2]])
+ // LLVM: [[VMAX_V_I:%.*]] = call <8 x i8> @llvm.umax.v8i8(<8 x i8> [[V1]], <8 x i8> [[V2]])
// LLVM: ret <8 x i8> [[VMAX_V_I]]
return vmax_u8(v1, v2);
}
@@ -1515,8 +1519,10 @@ uint16x4_t test_vmax_u16(uint16x4_t v1, uint16x4_t v2) {
// LLVM: [[TMP1:%.*]] = bitcast <4 x i16> [[V2]] to <8 x i8>
// LLVM: [[VMAX_V_I:%.*]] = bitcast <8 x i8> [[TMP0]] to <4 x i16>
// LLVM: [[VMAX_V1_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <4 x i16>
- // LLVM: [[VMAX_V2_I:%.*]] = call <4 x i16> @llvm.aarch64.neon.umax.v4i16(<4 x i16> [[VMAX_V_I]], <4 x i16> [[VMAX_V1_I]])
- // LLVM: ret <4 x i16> [[VMAX_V2_I]]
+ // LLVM: [[VMAX_V2_I:%.*]] = call <4 x i16> @llvm.umax.v4i16(<4 x i16> [[VMAX_V_I]], <4 x i16> [[VMAX_V1_I]])
+ // LLVM: [[VMAX_V3_I:%.*]] = bitcast <4 x i16> [[VMAX_V2_I]] to <8 x i8>
+ // LLVM: [[TMP2:%.*]] = bitcast <8 x i8> [[VMAX_V3_I]] to <4 x i16>
+ // LLVM: ret <4 x i16> [[TMP2]]
return vmax_u16(v1, v2);
}
@@ -1530,8 +1536,10 @@ uint32x2_t test_vmax_u32(uint32x2_t v1, uint32x2_t v2) {
// LLVM: [[TMP1:%.*]] = bitcast <2 x i32> [[V2]] to <8 x i8>
// LLVM: [[VMAX_V_I:%.*]] = bitcast <8 x i8> [[TMP0]] to <2 x i32>
// LLVM: [[VMAX_V1_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <2 x i32>
- // LLVM: [[VMAX_V2_I:%.*]] = call <2 x i32> @llvm.aarch64.neon.umax.v2i32(<2 x i32> [[VMAX_V_I]], <2 x i32> [[VMAX_V1_I]])
- // LLVM: ret <2 x i32> [[VMAX_V2_I]]
+ // LLVM: [[VMAX_V2_I:%.*]] = call <2 x i32> @llvm.umax.v2i32(<2 x i32> [[VMAX_V_I]], <2 x i32> [[VMAX_V1_I]])
+ // LLVM: [[VMAX_V3_I:%.*]] = bitcast <2 x i32> [[VMAX_V2_I]] to <8 x i8>
+ // LLVM: [[TMP2:%.*]] = bitcast <8 x i8> [[VMAX_V3_I]] to <2 x i32>
+ // LLVM: ret <2 x i32> [[TMP2]]
return vmax_u32(v1, v2);
}
@@ -1541,7 +1549,7 @@ int8x16_t test_vmaxq_s8(int8x16_t v1, int8x16_t v2) {
// CIR: cir.call_llvm_intrinsic "aarch64.neon.smax" %{{.*}}, %{{.*}} : (!cir.vector<16 x !s8i>, !cir.vector<16 x !s8i>) -> !cir.vector<16 x !s8i>
// LLVM-SAME: <16 x i8> {{.*}} [[V1:%.*]], <16 x i8> noundef [[V2:%.*]]) {{.*}} {
- // LLVM: [[VMAXQ_V_I:%.*]] = call <16 x i8> @llvm.aarch64.neon.smax.v16i8(<16 x i8> [[V1]], <16 x i8> [[V2]])
+ // LLVM: [[VMAXQ_V_I:%.*]] = call <16 x i8> @llvm.smax.v16i8(<16 x i8> [[V1]], <16 x i8> [[V2]])
// LLVM: ret <16 x i8> [[VMAXQ_V_I]]
return vmaxq_s8(v1, v2);
}
@@ -1556,8 +1564,10 @@ int16x8_t test_vmaxq_s16(int16x8_t v1, int16x8_t v2) {
// LLVM: [[TMP1:%.*]] = bitcast <8 x i16> [[V2]] to <16 x i8>
// LLVM: [[VMAXQ_V_I:%.*]] = bitcast <16 x i8> [[TMP0]] to <8 x i16>
// LLVM: [[VMAXQ_V1_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <8 x i16>
- // LLVM: [[VMAXQ_V2_I:%.*]] = call <8 x i16> @llvm.aarch64.neon.smax.v8i16(<8 x i16> [[VMAXQ_V_I]], <8 x i16> [[VMAXQ_V1_I]])
- // LLVM: ret <8 x i16> [[VMAXQ_V2_I]]
+ // LLVM: [[VMAXQ_V2_I:%.*]] = call <8 x i16> @llvm.smax.v8i16(<8 x i16> [[VMAXQ_V_I]], <8 x i16> [[VMAXQ_V1_I]])
+ // LLVM: [[VMAXQ_V3_I:%.*]] = bitcast <8 x i16> [[VMAXQ_V2_I]] to <16 x i8>
+ // LLVM: [[TMP2:%.*]] = bitcast <16 x i8> [[VMAXQ_V3_I]] to <8 x i16>
+ // LLVM: ret <8 x i16> [[TMP2]]
return vmaxq_s16(v1, v2);
}
@@ -1571,8 +1581,10 @@ int32x4_t test_vmaxq_s32(int32x4_t v1, int32x4_t v2) {
// LLVM: [[TMP1:%.*]] = bitcast <4 x i32> [[V2]] to <16 x i8>
// LLVM: [[VMAXQ_V_I:%.*]] = bitcast <16 x i8> [[TMP0]] to <4 x i32>
// LLVM: [[VMAXQ_V1_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <4 x i32>
- // LLVM: [[VMAXQ_V2_I:%.*]] = call <4 x i32> @llvm.aarch64.neon.smax.v4i32(<4 x i32> [[VMAXQ_V_I]], <4 x i32> [[VMAXQ_V1_I]])
- // LLVM: ret <4 x i32> [[VMAXQ_V2_I]]
+ // LLVM: [[VMAXQ_V2_I:%.*]] = call <4 x i32> @llvm.smax.v4i32(<4 x i32> [[VMAXQ_V_I]], <4 x i32> [[VMAXQ_V1_I]])
+ // LLVM: [[VMAXQ_V3_I:%.*]] = bitcast <4 x i32> [[VMAXQ_V2_I]] to <16 x i8>
+ // LLVM: [[TMP2:%.*]] = bitcast <16 x i8> [[VMAXQ_V3_I]] to <4 x i32>
+ // LLVM: ret <4 x i32> [[TMP2]]
return vmaxq_s32(v1, v2);
}
@@ -1582,7 +1594,7 @@ uint8x16_t test_vmaxq_u8(uint8x16_t v1, uint8x16_t v2) {
// CIR: cir.call_llvm_intrinsic "aarch64.neon.umax" %{{.*}}, %{{.*}} : (!cir.vector<16 x !u8i>, !cir.vector<16 x !u8i>) -> !cir.vector<16 x !u8i>
// LLVM-SAME: <16 x i8> {{.*}} [[V1:%.*]], <16 x i8> noundef [[V2:%.*]]) {{.*}} {
- // LLVM: [[VMAXQ_V_I:%.*]] = call <16 x i8> @llvm.aarch64.neon.umax.v16i8(<16 x i8> [[V1]], <16 x i8> [[V2]])
+ // LLVM: [[VMAXQ_V_I:%.*]] = call <16 x i8> @llvm.umax.v16i8(<16 x i8> [[V1]], <16 x i8> [[V2]])
// LLVM: ret <16 x i8> [[VMAXQ_V_I]]
return vmaxq_u8(v1, v2);
}
@@ -1597,8 +1609,10 @@ uint16x8_t test_vmaxq_u16(uint16x8_t v1, uint16x8_t v2) {
// LLVM: [[TMP1:%.*]] = bitcast <8 x i16> [[V2]] to <16 x i8>
// LLVM: [[VMAXQ_V_I:%.*]] = bitcast <16 x i8> [[TMP0]] to <8 x i16>
// LLVM: [[VMAXQ_V1_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <8 x i16>
- // LLVM: [[VMAXQ_V2_I:%.*]] = call <8 x i16> @llvm.aarch64.neon.umax.v8i16(<8 x i16> [[VMAXQ_V_I]], <8 x i16> [[VMAXQ_V1_I]])
- // LLVM: ret <8 x i16> [[VMAXQ_V2_I]]
+ // LLVM: [[VMAXQ_V2_I:%.*]] = call <8 x i16> @llvm.umax.v8i16(<8 x i16> [[VMAXQ_V_I]], <8 x i16> [[VMAXQ_V1_I]])
+ // LLVM: [[VMAXQ_V3_I:%.*]] = bitcast <8 x i16> [[VMAXQ_V2_I]] to <16 x i8>
+ // LLVM: [[TMP2:%.*]] = bitcast <16 x i8> [[VMAXQ_V3_I]] to <8 x i16>
+ // LLVM: ret <8 x i16> [[TMP2]]
return vmaxq_u16(v1, v2);
}
@@ -1612,8 +1626,10 @@ uint32x4_t test_vmaxq_u32(uint32x4_t v1, uint32x4_t v2) {
// LLVM: [[TMP1:%.*]] = bitcast <4 x i32> [[V2]] to <16 x i8>
// LLVM: [[VMAXQ_V_I:%.*]] = bitcast <16 x i8> [[TMP0]] to <4 x i32>
// LLVM: [[VMAXQ_V1_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <4 x i32>
- // LLVM: [[VMAXQ_V2_I:%.*]] = call <4 x i32> @llvm.aarch64.neon.umax.v4i32(<4 x i32> [[VMAXQ_V_I]], <4 x i32> [[VMAXQ_V1_I]])
- // LLVM: ret <4 x i32> [[VMAXQ_V2_I]]
+ // LLVM: [[VMAXQ_V2_I:%.*]] = call <4 x i32> @llvm.umax.v4i32(<4 x i32> [[VMAXQ_V_I]], <4 x i32> [[VMAXQ_V1_I]])
+ // LLVM: [[VMAXQ_V3_I:%.*]] = bitcast <4 x i32> [[VMAXQ_V2_I]] to <16 x i8>
+ // LLVM: [[TMP2:%.*]] = bitcast <16 x i8> [[VMAXQ_V3_I]] to <4 x i32>
+ // LLVM: ret <4 x i32> [[TMP2]]
return vmaxq_u32(v1, v2);
}
@@ -1629,8 +1645,11 @@ float32x2_t test_vmax_f32(float32x2_t v1, float32x2_t v2) {
// LLVM: [[TMP3:%.*]] = bitcast <2 x i32> [[TMP1]] to <8 x i8>
// LLVM: [[VMAX_V_I:%.*]] = bitcast <8 x i8> [[TMP2]] to <2 x float>
// LLVM: [[VMAX_V1_I:%.*]] = bitcast <8 x i8> [[TMP3]] to <2 x float>
- // LLVM: [[VMAX_V2_I:%.*]] = call <2 x float> @llvm.aarch64.neon.fmax.v2f32(<2 x float> [[VMAX_V_I]], <2 x float> [[VMAX_V1_I]])
- // LLVM: ret <2 x float> [[VMAX_V2_I]]
+ // LLVM: [[VMAX_V2_I:%.*]] = call <2 x float> @llvm.maximum.v2f32(<2 x float> [[VMAX_V_I]], <2 x float> [[VMAX_V1_I]])
+ // LLVM: [[VMAX_V3_I:%.*]] = bitcast <2 x float> [[VMAX_V2_I]] to <8 x i8>
+ // LLVM: [[TMP4:%.*]] = bitcast <8 x i8> [[VMAX_V3_I]] to <2 x i32>
+ // LLVM: [[TMP5:%.*]] = bitcast <2 x i32> [[TMP4]] to <2 x float>
+ // LLVM: ret <2 x float> [[TMP5]]
return vmax_f32(v1, v2);
}
@@ -1640,8 +1659,17 @@ float64x1_t test_vmax_f64(float64x1_t v1, float64x1_t v2) {
// CIR: cir.call_llvm_intrinsic "aarch64.neon.fmax" %{{.*}}, %{{.*}} : (!cir.vector<1 x !cir.double>, !cir.vector<1 x !cir.double>) -> !cir.vector<1 x !cir.double>
// LLVM-SAME: <1 x double> {{.*}} [[V1:%.*]], <1 x double> noundef [[V2:%.*]]) {{.*}} {
- // LLVM: [[VMAX_V_I:%.*]] = call <1 x double> @llvm.aarch64.neon.fmax.v1f64(<1 x double> {{.*}}, <1 x double> {{.*}})
- // LLVM: ret <1 x double> [[VMAX_V_I]]
+ // LLVM: [[TMP0:%.*]] = bitcast <1 x double> [[V1]] to i64
+ // LLVM: [[TMP1:%.*]] = bitcast <1 x double> [[V2]] to i64
+ // LLVM: [[TMP2:%.*]] = bitcast <1 x i64> {{.*}} to <8 x i8>
+ // LLVM: [[TMP3:%.*]] = bitcast <1 x i64> {{.*}} to <8 x i8>
+ // LLVM: [[VMAX_V_I:%.*]] = bitcast <8 x i8> [[TMP2]] to <1 x double>
+ // LLVM: [[VMAX_V1_I:%.*]] = bitcast <8 x i8> [[TMP3]] to <1 x double>
+ // LLVM: [[VMAX_V2_I:%.*]] = call <1 x double> @llvm.maximum.v1f64(<1 x double> [[VMAX_V_I]], <1 x double> [[VMAX_V1_I]])
+ // LLVM: [[VMAX_V3_I:%.*]] = bitcast <1 x double> [[VMAX_V2_I]] to <8 x i8>
+ // LLVM: [[TMP4:%.*]] = bitcast <8 x i8> [[VMAX_V3_I]] to i64
+ // LLVM: [[TMP5:%.*]] = bitcast <1 x i64> {{.*}} to <1 x double>
+ // LLVM: ret <1 x double> [[TMP5]]
return vmax_f64(v1, v2);
}
@@ -1657,8 +1685,11 @@ float32x4_t test_vmaxq_f32(float32x4_t v1, float32x4_t v2) {
// LLVM: [[TMP3:%.*]] = bitcast <4 x i32> [[TMP1]] to <16 x i8>
// LLVM: [[VMAXQ_V_I:%.*]] = bitcast <16 x i8> [[TMP2]] to <4 x float>
// LLVM: [[VMAXQ_V1_I:%.*]] = bitcast <16 x i8> [[TMP3]] to <4 x float>
- // LLVM: [[VMAXQ_V2_I:%.*]] = call <4 x float> @llvm.aarch64.neon.fmax.v4f32(<4 x float> [[VMAXQ_V_I]], <4 x float> [[VMAXQ_V1_I]])
- // LLVM: ret <4 x float> [[VMAXQ_V2_I]]
+ // LLVM: [[VMAXQ_V2_I:%.*]] = call <4 x float> @llvm.maximum.v4f32(<4 x float> [[VMAXQ_V_I]], <4 x float> [[VMAXQ_V1_I]])
+ // LLVM: [[VMAXQ_V3_I:%.*]] = bitcast <4 x float> [[VMAXQ_V2_I]] to <16 x i8>
+ // LLVM: [[TMP4:%.*]] = bitcast <16 x i8> [[VMAXQ_V3_I]] to <4 x i32>
+ // LLVM: [[TMP5:%.*]] = bitcast <4 x i32> [[TMP4]] to <4 x float>
+ // LLVM: ret <4 x float> [[TMP5]]
return vmaxq_f32(v1, v2);
}
@@ -1674,8 +1705,11 @@ float64x2_t test_vmaxq_f64(float64x2_t v1, float64x2_t v2) {
// LLVM: [[TMP3:%.*]] = bitcast <2 x i64> [[TMP1]] to <16 x i8>
// LLVM: [[VMAXQ_V_I:%.*]] = bitcast <16 x i8> [[TMP2]] to <2 x double>
// LLVM: [[VMAXQ_V1_I:%.*]] = bitcast <16 x i8> [[TMP3]] to <2 x double>
- // LLVM: [[VMAXQ_V2_I:%.*]] = call <2 x double> @llvm.aarch64.neon.fmax.v2f64(<2 x double> [[VMAXQ_V_I]], <2 x double> [[VMAXQ_V1_I]])
- // LLVM: ret <2 x double> [[VMAXQ_V2_I]]
+ // LLVM: [[VMAXQ_V2_I:%.*]] = call <2 x double> @llvm.maximum.v2f64(<2 x double> [[VMAXQ_V_I]], <2 x double> [[VMAXQ_V1_I]])
+ // LLVM: [[VMAXQ_V3_I:%.*]] = bitcast <2 x double> [[VMAXQ_V2_I]] to <16 x i8>
+ // LLVM: [[TMP4:%.*]] = bitcast <16 x i8> [[VMAXQ_V3_I]] to <2 x i64>
+ // LLVM: [[TMP5:%.*]] = bitcast <2 x i64> [[TMP4]] to <2 x double>
+ // LLVM: ret <2 x double> [[TMP5]]
return vmaxq_f64(v1, v2);
}
@@ -2058,7 +2092,7 @@ int8x8_t test_vmin_s8(int8x8_t v1, int8x8_t v2) {
// CIR: cir.call_llvm_intrinsic "aarch64.neon.smin" %{{.*}}, %{{.*}} : (!cir.vector<8 x !s8i>, !cir.vector<8 x !s8i>) -> !cir.vector<8 x !s8i>
// LLVM-SAME: <8 x i8> {{.*}} [[V1:%.*]], <8 x i8> noundef [[V2:%.*]]) {{.*}} {
- // LLVM: [[VMIN_V_I:%.*]] = call <8 x i8> @llvm.aarch64.neon.smin.v8i8(<8 x i8> [[V1]], <8 x i8> [[V2]])
+ // LLVM: [[VMIN_V_I:%.*]] = call <8 x i8> @llvm.smin.v8i8(<8 x i8> [[V1]], <8 x i8> [[V2]])
// LLVM: ret <8 x i8> [[VMIN_V_I]]
return vmin_s8(v1, v2);
}
@@ -2073,8 +2107,10 @@ int16x4_t test_vmin_s16(int16x4_t v1, int16x4_t v2) {
// LLVM: [[TMP1:%.*]] = bitcast <4 x i16> [[V2]] to <8 x i8>
// LLVM: [[VMIN_V_I:%.*]] = bitcast <8 x i8> [[TMP0]] to <4 x i16>
// LLVM: [[VMIN_V1_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <4 x i16>
- // LLVM: [[VMIN_V2_I:%.*]] = call <4 x i16> @llvm.aarch64.neon.smin.v4i16(<4 x i16> [[VMIN_V_I]], <4 x i16> [[VMIN_V1_I]])
- // LLVM: ret <4 x i16> [[VMIN_V2_I]]
+ // LLVM: [[VMIN_V2_I:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[VMIN_V_I]], <4 x i16> [[VMIN_V1_I]])
+ // LLVM: [[VMIN_V3_I:%.*]] = bitcast <4 x i16> [[VMIN_V2_I]] to <8 x i8>
+ // LLVM: [[TMP2:%.*]] = bitcast <8 x i8> [[VMIN_V3_I]] to <4 x i16>
+ // LLVM: ret <4 x i16> [[TMP2]]
return vmin_s16(v1, v2);
}
@@ -2088,8 +2124,10 @@ int32x2_t test_vmin_s32(int32x2_t v1, int32x2_t v2) {
// LLVM: [[TMP1:%.*]] = bitcast <2 x i32> [[V2]] to <8 x i8>
// LLVM: [[VMIN_V_I:%.*]] = bitcast <8 x i8> [[TMP0]] to <2 x i32>
// LLVM: [[VMIN_V1_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <2 x i32>
- // LLVM: [[VMIN_V2_I:%.*]] = call <2 x i32> @llvm.aarch64.neon.smin.v2i32(<2 x i32> [[VMIN_V_I]], <2 x i32> [[VMIN_V1_I]])
- // LLVM: ret <2 x i32> [[VMIN_V2_I]]
+ // LLVM: [[VMIN_V2_I:%.*]] = call <2 x i32> @llvm.smin.v2i32(<2 x i32> [[VMIN_V_I]], <2 x i32> [[VMIN_V1_I]])
+ // LLVM: [[VMIN_V3_I:%.*]] = bitcast <2 x i32> [[VMIN_V2_I]] to <8 x i8>
+ // LLVM: [[TMP2:%.*]] = bitcast <8 x i8> [[VMIN_V3_I]] to <2 x i32>
+ // LLVM: ret <2 x i32> [[TMP2]]
return vmin_s32(v1, v2);
}
@@ -2099,7 +2137,7 @@ uint8x8_t test_vmin_u8(uint8x8_t v1, uint8x8_t v2) {
// CIR: cir.call_llvm_intrinsic "aarch64.neon.umin" %{{.*}}, %{{.*}} : (!cir.vector<8 x !u8i>, !cir.vector<8 x !u8i>) -> !cir.vector<8 x !u8i>
// LLVM-SAME: <8 x i8> {{.*}} [[V1:%.*]], <8 x i8> noundef [[V2:%.*]]) {{.*}} {
- // LLVM: [[VMIN_V_I:%.*]] = call <8 x i8> @llvm.aarch64.neon.umin.v8i8(<8 x i8> [[V1]], <8 x i8> [[V2]])
+ // LLVM: [[VMIN_V_I:%.*]] = call <8 x i8> @llvm.umin.v8i8(<8 x i8> [[V1]], <8 x i8> [[V2]])
// LLVM: ret <8 x i8> [[VMIN_V_I]]
return vmin_u8(v1, v2);
}
@@ -2114,8 +2152,10 @@ uint16x4_t test_vmin_u16(uint16x4_t v1, uint16x4_t v2) {
// LLVM: [[TMP1:%.*]] = bitcast <4 x i16> [[V2]] to <8 x i8>
// LLVM: [[VMIN_V_I:%.*]] = bitcast <8 x i8> [[TMP0]] to <4 x i16>
// LLVM: [[VMIN_V1_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <4 x i16>
- // LLVM: [[VMIN_V2_I:%.*]] = call <4 x i16> @llvm.aarch64.neon.umin.v4i16(<4 x i16> [[VMIN_V_I]], <4 x i16> [[VMIN_V1_I]])
- // LLVM: ret <4 x i16> [[VMIN_V2_I]]
+ // LLVM: [[VMIN_V2_I:%.*]] = call <4 x i16> @llvm.umin.v4i16(<4 x i16> [[VMIN_V_I]], <4 x i16> [[VMIN_V1_I]])
+ // LLVM: [[VMIN_V3_I:%.*]] = bitcast <4 x i16> [[VMIN_V2_I]] to <8 x i8>
+ // LLVM: [[TMP2:%.*]] = bitcast <8 x i8> [[VMIN_V3_I]] to <4 x i16>
+ // LLVM: ret <4 x i16> [[TMP2]]
return vmin_u16(v1, v2);
}
@@ -2129,8 +2169,10 @@ uint32x2_t test_vmin_u32(uint32x2_t v1, uint32x2_t v2) {
// LLVM: [[TMP1:%.*]] = bitcast <2 x i32> [[V2]] to <8 x i8>
// LLVM: [[VMIN_V_I:%.*]] = bitcast <8 x i8> [[TMP0]] to <2 x i32>
// LLVM: [[VMIN_V1_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <2 x i32>
- // LLVM: [[VMIN_V2_I:%.*]] = call <2 x i32> @llvm.aarch64.neon.umin.v2i32(<2 x i32> [[VMIN_V_I]], <2 x i32> [[VMIN_V1_I]])
- // LLVM: ret <2 x i32> [[VMIN_V2_I]]
+ // LLVM: [[VMIN_V2_I:%.*]] = call <2 x i32> @llvm.umin.v2i32(<2 x i32> [[VMIN_V_I]], <2 x i32> [[VMIN_V1_I]])
+ // LLVM: [[VMIN_V3_I:%.*]] = bitcast <2 x i32> [[VMIN_V2_I]] to <8 x i8>
+ // LLVM: [[TMP2:%.*]] = bitcast <8 x i8> [[VMIN_V3_I]] to <2 x i32>
+ // LLVM: ret <2 x i32> [[TMP2]]
return vmin_u32(v1, v2);
}
@@ -2140,7 +2182,7 @@ int8x16_t test_vminq_s8(int8x16_t v1, int8x16_t v2) {
// CIR: cir.call_llvm_intrinsic "aarch64.neon.smin" %{{.*}}, %{{.*}} : (!cir.vector<16 x !s8i>, !cir.vector<16 x !s8i>) -> !cir.vector<16 x !s8i>
// LLVM-SAME: <16 x i8> {{.*}} [[V1:%.*]], <16 x i8> noundef [[V2:%.*]]) {{.*}} {
- // LLVM: [[VMINQ_V_I:%.*]] = call <16 x i8> @llvm.aarch64.neon.smin.v16i8(<16 x i8> [[V1]], <16 x i8> [[V2]])
+ // LLVM: [[VMINQ_V_I:%.*]] = call <16 x i8> @llvm.smin.v16i8(<16 x i8> [[V1]], <16 x i8> [[V2]])
// LLVM: ret <16 x i8> [[VMINQ_V_I]]
return vminq_s8(v1, v2);
}
@@ -2155,8 +2197,10 @@ int16x8_t test_vminq_s16(int16x8_t v1, int16x8_t v2) {
// LLVM: [[TMP1:%.*]] = bitcast <8 x i16> [[V2]] to <16 x i8>
// LLVM: [[VMINQ_V_I:%.*]] = bitcast <16 x i8> [[TMP0]] to <8 x i16>
// LLVM: [[VMINQ_V1_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <8 x i16>
- // LLVM: [[VMINQ_V2_I:%.*]] = call <8 x i16> @llvm.aarch64.neon.smin.v8i16(<8 x i16> [[VMINQ_V_I]], <8 x i16> [[VMINQ_V1_I]])
- // LLVM: ret <8 x i16> [[VMINQ_V2_I]]
+ // LLVM: [[VMINQ_V2_I:%.*]] = call <8 x i16> @llvm.smin.v8i16(<8 x i16> [[VMINQ_V_I]], <8 x i16> [[VMINQ_V1_I]])
+ // LLVM: [[VMINQ_V3_I:%.*]] = bitcast <8 x i16> [[VMINQ_V2_I]] to <16 x i8>
+ // LLVM: [[TMP2:%.*]] = bitcast <16 x i8> [[VMINQ_V3_I]] to <8 x i16>
+ // LLVM: ret <8 x i16> [[TMP2]]
return vminq_s16(v1, v2);
}
@@ -2170,8 +2214,10 @@ int32x4_t test_vminq_s32(int32x4_t v1, int32x4_t v2) {
// LLVM: [[TMP1:%.*]] = bitcast <4 x i32> [[V2]] to <16 x i8>
// LLVM: [[VMINQ_V_I:%.*]] = bitcast <16 x i8> [[TMP0]] to <4 x i32>
// LLVM: [[VMINQ_V1_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <4 x i32>
- // LLVM: [[VMINQ_V2_I:%.*]] = call <4 x i32> @llvm.aarch64.neon.smin.v4i32(<4 x i32> [[VMINQ_V_I]], <4 x i32> [[VMINQ_V1_I]])
- // LLVM: ret <4 x i32> [[VMINQ_V2_I]]
+ // LLVM: [[VMINQ_V2_I:%.*]] = call <4 x i32> @llvm.smin.v4i32(<4 x i32> [[VMINQ_V_I]], <4 x i32> [[VMINQ_V1_I]])
+ // LLVM: [[VMINQ_V3_I:%.*]] = bitcast <4 x i32> [[VMINQ_V2_I]] to <16 x i8>
+ // LLVM: [[TMP2:%.*]] = bitcast <16 x i8> [[VMINQ_V3_I]] to <4 x i32>
+ // LLVM: ret <4 x i32> [[TMP2]]
return vminq_s32(v1, v2);
}
@@ -2181,7 +2227,7 @@ uint8x16_t test_vminq_u8(uint8x16_t v1, uint8x16_t v2) {
// CIR: cir.call_llvm_intrinsic "aarch64.neon.umin" %{{.*}}, %{{.*}} : (!cir.vector<16 x !u8i>, !cir.vector<16 x !u8i>) -> !cir.vector<16 x !u8i>
// LLVM-SAME: <16 x i8> {{.*}} [[V1:%.*]], <16 x i8> noundef [[V2:%.*]]) {{.*}} {
- // LLVM: [[VMINQ_V_I:%.*]] = call <16 x i8> @llvm.aarch64.neon.umin.v16i8(<16 x i8> [[V1]], <16 x i8> [[V2]])
+ // LLVM: [[VMINQ_V_I:%.*]] = call <16 x i8> @llvm.umin.v16i8(<16 x i8> [[V1]], <16 x i8> [[V2]])
// LLVM: ret <16 x i8> [[VMINQ_V_I]]
return vminq_u8(v1, v2);
}
@@ -2196,8 +2242,10 @@ uint16x8_t test_vminq_u16(uint16x8_t v1, uint16x8_t v2) {
// LLVM: [[TMP1:%.*]] = bitcast <8 x i16> [[V2]] to <16 x i8>
// LLVM: [[VMINQ_V_I:%.*]] = bitcast <16 x i8> [[TMP0]] to <8 x i16>
// LLVM: [[VMINQ_V1_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <8 x i16>
- // LLVM: [[VMINQ_V2_I:%.*]] = call <8 x i16> @llvm.aarch64.neon.umin.v8i16(<8 x i16> [[VMINQ_V_I]], <8 x i16> [[VMINQ_V1_I]])
- // LLVM: ret <8 x i16> [[VMINQ_V2_I]]
+ // LLVM: [[VMINQ_V2_I:%.*]] = call <8 x i16> @llvm.umin.v8i16(<8 x i16> [[VMINQ_V_I]], <8 x i16> [[VMINQ_V1_I]])
+ // LLVM: [[VMINQ_V3_I:%.*]] = bitcast <8 x i16> [[VMINQ_V2_I]] to <16 x i8>
+ // LLVM: [[TMP2:%.*]] = bitcast <16 x i8> [[VMINQ_V3_I]] to <8 x i16>
+ // LLVM: ret <8 x i16> [[TMP2]]
return vminq_u16(v1, v2);
}
@@ -2212,8 +2260,10 @@ uint32x4_t test_vminq_u32(uint32x4_t v1, uint32x4_t v2) {
// LLVM: [[TMP1:%.*]] = bitcast <4 x i32> [[V2]] to <16 x i8>
// LLVM: [[VMINQ_V_I:%.*]] = bitcast <16 x i8> [[TMP0]] to <4 x i32>
// LLVM: [[VMINQ_V1_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <4 x i32>
- // LLVM: [[VMINQ_V2_I:%.*]] = call <4 x i32> @llvm.aarch64.neon.umin.v4i32(<4 x i32> [[VMINQ_V_I]], <4 x i32> [[VMINQ_V1_I]])
- // LLVM: ret <4 x i32> [[VMINQ_V2_I]]
+ // LLVM: [[VMINQ_V2_I:%.*]] = call <4 x i32> @llvm.umin.v4i32(<4 x i32> [[VMINQ_V_I]], <4 x i32> [[VMINQ_V1_I]])
+ // LLVM: [[VMINQ_V3_I:%.*]] = bitcast <4 x i32> [[VMINQ_V2_I]] to <16 x i8>
+ // LLVM: [[TMP2:%.*]] = bitcast <16 x i8> [[VMINQ_V3_I]] to <4 x i32>
+ // LLVM: ret <4 x i32> [[TMP2]]
return vminq_u32(v1, v2);
}
@@ -2229,8 +2279,11 @@ float32x2_t test_vmin_f32(float32x2_t v1, float32x2_t v2) {
// LLVM: [[TMP3:%.*]] = bitcast <2 x i32> [[TMP1]] to <8 x i8>
// LLVM: [[VMIN_V_I:%.*]] = bitcast <8 x i8> [[TMP2]] to <2 x float>
// LLVM: [[VMIN_V1_I:%.*]] = bitcast <8 x i8> [[TMP3]] to <2 x float>
- // LLVM: [[VMIN_V2_I:%.*]] = call <2 x float> @llvm.aarch64.neon.fmin.v2f32(<2 x float> [[VMIN_V_I]], <2 x float> [[VMIN_V1_I]])
- // LLVM: ret <2 x float> [[VMIN_V2_I]]
+ // LLVM: [[VMIN_V2_I:%.*]] = call <2 x float> @llvm.minimum.v2f32(<2 x float> [[VMIN_V_I]], <2 x float> [[VMIN_V1_I]])
+ // LLVM: [[VMIN_V3_I:%.*]] = bitcast <2 x float> [[VMIN_V2_I]] to <8 x i8>
+ // LLVM: [[TMP4:%.*]] = bitcast <8 x i8> [[VMIN_V3_I]] to <2 x i32>
+ // LLVM: [[TMP5:%.*]] = bitcast <2 x i32> [[TMP4]] to <2 x float>
+ // LLVM: ret <2 x float> [[TMP5]]
return vmin_f32(v1, v2);
}
@@ -2240,10 +2293,17 @@ float64x1_t test_vmin_f64(float64x1_t v1, float64x1_t v2) {
// CIR: cir.call_llvm_intrinsic "aarch64.neon.fmin" %{{.*}}, %{{.*}} : (!cir.vector<1 x !cir.double>, !cir.vector<1 x !cir.double>) -> !cir.vector<1 x !cir.double>
// LLVM-SAME: <1 x double> {{.*}} [[V1:%.*]], <1 x double> noundef [[V2:%.*]]) {{.*}} {
- // LLVM: [[VMIN_V_I:%.*]] = bitcast <8 x i8> {{.*}} to <1 x double>
- // LLVM: [[VMIN_V1_I:%.*]] = bitcast <8 x i8> {{.*}} to <1 x double>
- // LLVM: [[VMIN_V2_I:%.*]] = call <1 x double> @llvm.aarch64.neon.fmin.v1f64(<1 x double> [[VMIN_V_I]], <1 x double> [[VMIN_V1_I]])
- // LLVM: ret <1 x double> [[VMIN_V2_I]]
+ // LLVM: [[TMP0:%.*]] = bitcast <1 x double> [[V1]] to i64
+ // LLVM: [[TMP1:%.*]] = bitcast <1 x double> [[V2]] to i64
+ // LLVM: [[TMP2:%.*]] = bitcast <1 x i64> {{.*}} to <8 x i8>
+ // LLVM: [[TMP3:%.*]] = bitcast <1 x i64> {{.*}} to <8 x i8>
+ // LLVM: [[VMIN_V_I:%.*]] = bitcast <8 x i8> [[TMP2]] to <1 x double>
+ // LLVM: [[VMIN_V1_I:%.*]] = bitcast <8 x i8> [[TMP3]] to <1 x double>
+ // LLVM: [[VMIN_V2_I:%.*]] = call <1 x double> @llvm.minimum.v1f64(<1 x double> [[VMIN_V_I]], <1 x double> [[VMIN_V1_I]])
+ // LLVM: [[VMIN_V3_I:%.*]] = bitcast <1 x double> [[VMIN_V2_I]] to <8 x i8>
+ // LLVM: [[TMP4:%.*]] = bitcast <8 x i8> [[VMIN_V3_I]] to i64
+ // LLVM: [[TMP5:%.*]] = bitcast <1 x i64> {{.*}} to <1 x double>
+ // LLVM: ret <1 x double> [[TMP5]]
return vmin_f64(v1, v2);
}
@@ -2259,8 +2319,11 @@ float32x4_t test_vminq_f32(float32x4_t v1, float32x4_t v2) {
// LLVM: [[TMP3:%.*]] = bitcast <4 x i32> [[TMP1]] to <16 x i8>
// LLVM: [[VMINQ_V_I:%.*]] = bitcast <16 x i8> [[TMP2]] to <4 x float>
// LLVM: [[VMINQ_V1_I:%.*]] = bitcast <16 x i8> [[TMP3]] to <4 x float>
- // LLVM: [[VMINQ_V2_I:%.*]] = call <4 x float> @llvm.aarch64.neon.fmin.v4f32(<4 x float> [[VMINQ_V_I]], <4 x float> [[VMINQ_V1_I]])
- // LLVM: ret <4 x float> [[VMINQ_V2_I]]
+ // LLVM: [[VMINQ_V2_I:%.*]] = call <4 x float> @llvm.minimum.v4f32(<4 x float> [[VMINQ_V_I]], <4 x float> [[VMINQ_V1_I]])
+ // LLVM: [[VMINQ_V3_I:%.*]] = bitcast <4 x float> [[VMINQ_V2_I]] to <16 x i8>
+ // LLVM: [[TMP4:%.*]] = bitcast <16 x i8> [[VMINQ_V3_I]] to <4 x i32>
+ // LLVM: [[TMP5:%.*]] = bitcast <4 x i32> [[TMP4]] to <4 x float>
+ // LLVM: ret <4 x float> [[TMP5]]
return vminq_f32(v1, v2);
}
@@ -2276,8 +2339,11 @@ float64x2_t test_vminq_f64(float64x2_t v1, float64x2_t v2) {
// LLVM: [[TMP3:%.*]] = bitcast <2 x i64> [[TMP1]] to <16 x i8>
// LLVM: [[VMINQ_V_I:%.*]] = bitcast <16 x i8> [[TMP2]] to <2 x double>
// LLVM: [[VMINQ_V1_I:%.*]] = bitcast <16 x i8> [[TMP3]] to <2 x double>
- // LLVM: [[VMINQ_V2_I:%.*]] = call <2 x double> @llvm.aarch64.neon.fmin.v2f64(<2 x double> [[VMINQ_V_I]], <2 x double> [[VMINQ_V1_I]])
- // LLVM: ret <2 x double> [[VMINQ_V2_I]]
+ // LLVM: [[VMINQ_V2_I:%.*]] = call <2 x double> @llvm.minimum.v2f64(<2 x double> [[VMINQ_V_I]], <2 x double> [[VMINQ_V1_I]])
+ // LLVM: [[VMINQ_V3_I:%.*]] = bitcast <2 x double> [[VMINQ_V2_I]] to <16 x i8>
+ // LLVM: [[TMP4:%.*]] = bitcast <16 x i8> [[VMINQ_V3_I]] to <2 x i64>
+ // LLVM: [[TMP5:%.*]] = bitcast <2 x i64> [[TMP4]] to <2 x double>
+ // LLVM: ret <2 x double> [[TMP5]]
return vminq_f64(v1, v2);
}
@@ -2305,11 +2371,9 @@ float64x1_t test_vminnm_f64(float64x1_t v1, float64x1_t v2) {
// LLVM-SAME: <1 x double> {{.*}} [[V1:%.*]], <1 x double> noundef [[V2:%.*]]) {{.*}} {
// LLVM: [[TMP0:%.*]] = bitcast <1 x double> [[V1]] to i64
- // LLVM: [[__P0_ADDR_I_SROA_0_0_VEC_INSERT:%.*]] = insertelement <1 x i64> undef, i64 [[TMP0]], i32 0
// LLVM: [[TMP1:%.*]] = bitcast <1 x double> [[V2]] to i64
- // LLVM: [[__P1_ADDR_I_SROA_0_0_VEC_INSERT:%.*]] = insertelement <1 x i64> undef, i64 [[TMP1]], i32 0
- // LLVM: [[TMP2:%.*]] = bitcast <1 x i64> [[__P0_ADDR_I_SROA_0_0_VEC_INSERT]] to <8 x i8>
- // LLVM: [[TMP3:%.*]] = bitcast <1 x i64> [[__P1_ADDR_I_SROA_0_0_VEC_INSERT]] to <8 x i8>
+ // LLVM: [[TMP2:%.*]] = bitcast <1 x i64> {{.*}} to <8 x i8>
+ // LLVM: [[TMP3:%.*]] = bitcast <1 x i64> {{.*}} to <8 x i8>
// LLVM: [[VMINNM_I:%.*]] = bitcast <8 x i8> [[TMP2]] to <1 x double>
// LLVM: [[VMINNM1_I:%.*]] = bitcast <8 x i8> [[TMP3]] to <1 x double>
// LLVM: [[VMINNM2_I:%.*]] = call <1 x double> @llvm.aarch64.neon.fminnm.v1f64(<1 x double> [[VMINNM_I]], <1 x double> [[VMINNM1_I]])
diff --git a/clang/test/CodeGen/AArch64/v8.2a-fp16-intrinsics.c b/clang/test/CodeGen/AArch64/v8.2a-fp16-intrinsics.c
index ed6246238bf8e..f19f946b70c7c 100644
--- a/clang/test/CodeGen/AArch64/v8.2a-fp16-intrinsics.c
+++ b/clang/test/CodeGen/AArch64/v8.2a-fp16-intrinsics.c
@@ -508,7 +508,7 @@ int64_t test_vcvth_n_u64_f16(float16_t a) {
}
// CHECK-LABEL: test_vmaxh_f16
-// CHECK: [[MAX:%.*]] = call half @llvm.aarch64.neon.fmax.f16(half %a, half %b)
+// CHECK: [[MAX:%.*]] = call half @llvm.maximum.f16(half %a, half %b)
// CHECK: ret half [[MAX]]
float16_t test_vmaxh_f16(float16_t a, float16_t b) {
return vmaxh_f16(a, b);
@@ -522,7 +522,7 @@ float16_t test_vmaxnmh_f16(float16_t a, float16_t b) {
}
// CHECK-LABEL: test_vminh_f16
-// CHECK: [[MIN:%.*]] = call half @llvm.aarch64.neon.fmin.f16(half %a, half %b)
+// CHECK: [[MIN:%.*]] = call half @llvm.minimum.f16(half %a, half %b)
// CHECK: ret half [[MIN]]
float16_t test_vminh_f16(float16_t a, float16_t b) {
return vminh_f16(a, b);
diff --git a/clang/test/CodeGen/AArch64/v8.2a-neon-intrinsics.c b/clang/test/CodeGen/AArch64/v8.2a-neon-intrinsics.c
index e8f1eead2a0d5..f340d37042b51 100644
--- a/clang/test/CodeGen/AArch64/v8.2a-neon-intrinsics.c
+++ b/clang/test/CodeGen/AArch64/v8.2a-neon-intrinsics.c
@@ -1167,10 +1167,13 @@ float16x8_t test_vdivq_f16(float16x8_t a, float16x8_t b) {
// CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x half> [[B]] to <4 x i16>
// CHECK-NEXT: [[TMP2:%.*]] = bitcast <4 x i16> [[TMP0]] to <8 x i8>
// CHECK-NEXT: [[TMP3:%.*]] = bitcast <4 x i16> [[TMP1]] to <8 x i8>
-// CHECK-NEXT: [[VMAX_I:%.*]] = bitcast <8 x i8> [[TMP2]] to <4 x half>
-// CHECK-NEXT: [[VMAX1_I:%.*]] = bitcast <8 x i8> [[TMP3]] to <4 x half>
-// CHECK-NEXT: [[VMAX2_I:%.*]] = call <4 x half> @llvm.aarch64.neon.fmax.v4f16(<4 x half> [[VMAX_I]], <4 x half> [[VMAX1_I]])
-// CHECK-NEXT: ret <4 x half> [[VMAX2_I]]
+// CHECK-NEXT: [[VMAX_V_I:%.*]] = bitcast <8 x i8> [[TMP2]] to <4 x half>
+// CHECK-NEXT: [[VMAX_V1_I:%.*]] = bitcast <8 x i8> [[TMP3]] to <4 x half>
+// CHECK-NEXT: [[VMAX_V2_I:%.*]] = call <4 x half> @llvm.maximum.v4f16(<4 x half> [[VMAX_V_I]], <4 x half> [[VMAX_V1_I]])
+// CHECK-NEXT: [[VMAX_V3_I:%.*]] = bitcast <4 x half> [[VMAX_V2_I]] to <8 x i8>
+// CHECK-NEXT: [[TMP4:%.*]] = bitcast <8 x i8> [[VMAX_V3_I]] to <4 x i16>
+// CHECK-NEXT: [[TMP5:%.*]] = bitcast <4 x i16> [[TMP4]] to <4 x half>
+// CHECK-NEXT: ret <4 x half> [[TMP5]]
//
float16x4_t test_vmax_f16(float16x4_t a, float16x4_t b) {
return vmax_f16(a, b);
@@ -1183,10 +1186,13 @@ float16x4_t test_vmax_f16(float16x4_t a, float16x4_t b) {
// CHECK-NEXT: [[TMP1:%.*]] = bitcast <8 x half> [[B]] to <8 x i16>
// CHECK-NEXT: [[TMP2:%.*]] = bitcast <8 x i16> [[TMP0]] to <16 x i8>
// CHECK-NEXT: [[TMP3:%.*]] = bitcast <8 x i16> [[TMP1]] to <16 x i8>
-// CHECK-NEXT: [[VMAX_I:%.*]] = bitcast <16 x i8> [[TMP2]] to <8 x half>
-// CHECK-NEXT: [[VMAX1_I:%.*]] = bitcast <16 x i8> [[TMP3]] to <8 x half>
-// CHECK-NEXT: [[VMAX2_I:%.*]] = call <8 x half> @llvm.aarch64.neon.fmax.v8f16(<8 x half> [[VMAX_I]], <8 x half> [[VMAX1_I]])
-// CHECK-NEXT: ret <8 x half> [[VMAX2_I]]
+// CHECK-NEXT: [[VMAXQ_V_I:%.*]] = bitcast <16 x i8> [[TMP2]] to <8 x half>
+// CHECK-NEXT: [[VMAXQ_V1_I:%.*]] = bitcast <16 x i8> [[TMP3]] to <8 x half>
+// CHECK-NEXT: [[VMAXQ_V2_I:%.*]] = call <8 x half> @llvm.maximum.v8f16(<8 x half> [[VMAXQ_V_I]], <8 x half> [[VMAXQ_V1_I]])
+// CHECK-NEXT: [[VMAXQ_V3_I:%.*]] = bitcast <8 x half> [[VMAXQ_V2_I]] to <16 x i8>
+// CHECK-NEXT: [[TMP4:%.*]] = bitcast <16 x i8> [[VMAXQ_V3_I]] to <8 x i16>
+// CHECK-NEXT: [[TMP5:%.*]] = bitcast <8 x i16> [[TMP4]] to <8 x half>
+// CHECK-NEXT: ret <8 x half> [[TMP5]]
//
float16x8_t test_vmaxq_f16(float16x8_t a, float16x8_t b) {
return vmaxq_f16(a, b);
@@ -1231,10 +1237,13 @@ float16x8_t test_vmaxnmq_f16(float16x8_t a, float16x8_t b) {
// CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x half> [[B]] to <4 x i16>
// CHECK-NEXT: [[TMP2:%.*]] = bitcast <4 x i16> [[TMP0]] to <8 x i8>
// CHECK-NEXT: [[TMP3:%.*]] = bitcast <4 x i16> [[TMP1]] to <8 x i8>
-// CHECK-NEXT: [[VMIN_I:%.*]] = bitcast <8 x i8> [[TMP2]] to <4 x half>
-// CHECK-NEXT: [[VMIN1_I:%.*]] = bitcast <8 x i8> [[TMP3]] to <4 x half>
-// CHECK-NEXT: [[VMIN2_I:%.*]] = call <4 x half> @llvm.aarch64.neon.fmin.v4f16(<4 x half> [[VMIN_I]], <4 x half> [[VMIN1_I]])
-// CHECK-NEXT: ret <4 x half> [[VMIN2_I]]
+// CHECK-NEXT: [[VMIN_V_I:%.*]] = bitcast <8 x i8> [[TMP2]] to <4 x half>
+// CHECK-NEXT: [[VMIN_V1_I:%.*]] = bitcast <8 x i8> [[TMP3]] to <4 x half>
+// CHECK-NEXT: [[VMIN_V2_I:%.*]] = call <4 x half> @llvm.minimum.v4f16(<4 x half> [[VMIN_V_I]], <4 x half> [[VMIN_V1_I]])
+// CHECK-NEXT: [[VMIN_V3_I:%.*]] = bitcast <4 x half> [[VMIN_V2_I]] to <8 x i8>
+// CHECK-NEXT: [[TMP4:%.*]] = bitcast <8 x i8> [[VMIN_V3_I]] to <4 x i16>
+// CHECK-NEXT: [[TMP5:%.*]] = bitcast <4 x i16> [[TMP4]] to <4 x half>
+// CHECK-NEXT: ret <4 x half> [[TMP5]]
//
float16x4_t test_vmin_f16(float16x4_t a, float16x4_t b) {
return vmin_f16(a, b);
@@ -1247,10 +1256,13 @@ float16x4_t test_vmin_f16(float16x4_t a, float16x4_t b) {
// CHECK-NEXT: [[TMP1:%.*]] = bitcast <8 x half> [[B]] to <8 x i16>
// CHECK-NEXT: [[TMP2:%.*]] = bitcast <8 x i16> [[TMP0]] to <16 x i8>
// CHECK-NEXT: [[TMP3:%.*]] = bitcast <8 x i16> [[TMP1]] to <16 x i8>
-// CHECK-NEXT: [[VMIN_I:%.*]] = bitcast <16 x i8> [[TMP2]] to <8 x half>
-// CHECK-NEXT: [[VMIN1_I:%.*]] = bitcast <16 x i8> [[TMP3]] to <8 x half>
-// CHECK-NEXT: [[VMIN2_I:%.*]] = call <8 x half> @llvm.aarch64.neon.fmin.v8f16(<8 x half> [[VMIN_I]], <8 x half> [[VMIN1_I]])
-// CHECK-NEXT: ret <8 x half> [[VMIN2_I]]
+// CHECK-NEXT: [[VMINQ_V_I:%.*]] = bitcast <16 x i8> [[TMP2]] to <8 x half>
+// CHECK-NEXT: [[VMINQ_V1_I:%.*]] = bitcast <16 x i8> [[TMP3]] to <8 x half>
+// CHECK-NEXT: [[VMINQ_V2_I:%.*]] = call <8 x half> @llvm.minimum.v8f16(<8 x half> [[VMINQ_V_I]], <8 x half> [[VMINQ_V1_I]])
+// CHECK-NEXT: [[VMINQ_V3_I:%.*]] = bitcast <8 x half> [[VMINQ_V2_I]] to <16 x i8>
+// CHECK-NEXT: [[TMP4:%.*]] = bitcast <16 x i8> [[VMINQ_V3_I]] to <8 x i16>
+// CHECK-NEXT: [[TMP5:%.*]] = bitcast <8 x i16> [[TMP4]] to <8 x half>
+// CHECK-NEXT: ret <8 x half> [[TMP5]]
//
float16x8_t test_vminq_f16(float16x8_t a, float16x8_t b) {
return vminq_f16(a, b);
diff --git a/clang/test/CodeGen/arm-v8.2a-neon-intrinsics.c b/clang/test/CodeGen/arm-v8.2a-neon-intrinsics.c
index 17d4eef1f7631..ec728dd9b1393 100644
--- a/clang/test/CodeGen/arm-v8.2a-neon-intrinsics.c
+++ b/clang/test/CodeGen/arm-v8.2a-neon-intrinsics.c
@@ -1139,7 +1139,7 @@ uint16x8_t test_vcvtq_n_u16_f16(float16x8_t a) {
// CHECK-NEXT: [[TMP3:%.*]] = bitcast <4 x i16> [[TMP1]] to <8 x i8>
// CHECK-NEXT: [[VMAX_V_I:%.*]] = bitcast <8 x i8> [[TMP2]] to <4 x half>
// CHECK-NEXT: [[VMAX_V1_I:%.*]] = bitcast <8 x i8> [[TMP3]] to <4 x half>
-// CHECK-NEXT: [[VMAX_V2_I:%.*]] = call <4 x half> @llvm.arm.neon.vmaxs.v4f16(<4 x half> [[VMAX_V_I]], <4 x half> [[VMAX_V1_I]])
+// CHECK-NEXT: [[VMAX_V2_I:%.*]] = call <4 x half> @llvm.maximum.v4f16(<4 x half> [[VMAX_V_I]], <4 x half> [[VMAX_V1_I]])
// CHECK-NEXT: [[VMAX_V3_I:%.*]] = bitcast <4 x half> [[VMAX_V2_I]] to <8 x i8>
// CHECK-NEXT: [[TMP4:%.*]] = bitcast <8 x i8> [[VMAX_V3_I]] to <4 x i16>
// CHECK-NEXT: [[TMP5:%.*]] = bitcast <4 x i16> [[TMP4]] to <4 x half>
@@ -1158,7 +1158,7 @@ float16x4_t test_vmax_f16(float16x4_t a, float16x4_t b) {
// CHECK-NEXT: [[TMP3:%.*]] = bitcast <8 x i16> [[TMP1]] to <16 x i8>
// CHECK-NEXT: [[VMAXQ_V_I:%.*]] = bitcast <16 x i8> [[TMP2]] to <8 x half>
// CHECK-NEXT: [[VMAXQ_V1_I:%.*]] = bitcast <16 x i8> [[TMP3]] to <8 x half>
-// CHECK-NEXT: [[VMAXQ_V2_I:%.*]] = call <8 x half> @llvm.arm.neon.vmaxs.v8f16(<8 x half> [[VMAXQ_V_I]], <8 x half> [[VMAXQ_V1_I]])
+// CHECK-NEXT: [[VMAXQ_V2_I:%.*]] = call <8 x half> @llvm.maximum.v8f16(<8 x half> [[VMAXQ_V_I]], <8 x half> [[VMAXQ_V1_I]])
// CHECK-NEXT: [[VMAXQ_V3_I:%.*]] = bitcast <8 x half> [[VMAXQ_V2_I]] to <16 x i8>
// CHECK-NEXT: [[TMP4:%.*]] = bitcast <16 x i8> [[VMAXQ_V3_I]] to <8 x i16>
// CHECK-NEXT: [[TMP5:%.*]] = bitcast <8 x i16> [[TMP4]] to <8 x half>
@@ -1215,7 +1215,7 @@ float16x8_t test_vmaxnmq_f16(float16x8_t a, float16x8_t b) {
// CHECK-NEXT: [[TMP3:%.*]] = bitcast <4 x i16> [[TMP1]] to <8 x i8>
// CHECK-NEXT: [[VMIN_V_I:%.*]] = bitcast <8 x i8> [[TMP2]] to <4 x half>
// CHECK-NEXT: [[VMIN_V1_I:%.*]] = bitcast <8 x i8> [[TMP3]] to <4 x half>
-// CHECK-NEXT: [[VMIN_V2_I:%.*]] = call <4 x half> @llvm.arm.neon.vmins.v4f16(<4 x half> [[VMIN_V_I]], <4 x half> [[VMIN_V1_I]])
+// CHECK-NEXT: [[VMIN_V2_I:%.*]] = call <4 x half> @llvm.minimum.v4f16(<4 x half> [[VMIN_V_I]], <4 x half> [[VMIN_V1_I]])
// CHECK-NEXT: [[VMIN_V3_I:%.*]] = bitcast <4 x half> [[VMIN_V2_I]] to <8 x i8>
// CHECK-NEXT: [[TMP4:%.*]] = bitcast <8 x i8> [[VMIN_V3_I]] to <4 x i16>
// CHECK-NEXT: [[TMP5:%.*]] = bitcast <4 x i16> [[TMP4]] to <4 x half>
@@ -1234,7 +1234,7 @@ float16x4_t test_vmin_f16(float16x4_t a, float16x4_t b) {
// CHECK-NEXT: [[TMP3:%.*]] = bitcast <8 x i16> [[TMP1]] to <16 x i8>
// CHECK-NEXT: [[VMINQ_V_I:%.*]] = bitcast <16 x i8> [[TMP2]] to <8 x half>
// CHECK-NEXT: [[VMINQ_V1_I:%.*]] = bitcast <16 x i8> [[TMP3]] to <8 x half>
-// CHECK-NEXT: [[VMINQ_V2_I:%.*]] = call <8 x half> @llvm.arm.neon.vmins.v8f16(<8 x half> [[VMINQ_V_I]], <8 x half> [[VMINQ_V1_I]])
+// CHECK-NEXT: [[VMINQ_V2_I:%.*]] = call <8 x half> @llvm.minimum.v8f16(<8 x half> [[VMINQ_V_I]], <8 x half> [[VMINQ_V1_I]])
// CHECK-NEXT: [[VMINQ_V3_I:%.*]] = bitcast <8 x half> [[VMINQ_V2_I]] to <16 x i8>
// CHECK-NEXT: [[TMP4:%.*]] = bitcast <16 x i8> [[VMINQ_V3_I]] to <8 x i16>
// CHECK-NEXT: [[TMP5:%.*]] = bitcast <8 x i16> [[TMP4]] to <8 x half>
diff --git a/clang/test/CodeGen/arm_neon_intrinsics.c b/clang/test/CodeGen/arm_neon_intrinsics.c
index eb9fe126ff2a0..1a7b0e80755af 100644
--- a/clang/test/CodeGen/arm_neon_intrinsics.c
+++ b/clang/test/CodeGen/arm_neon_intrinsics.c
@@ -9156,7 +9156,7 @@ poly16x4x4_t test_vld4_lane_p16(poly16_t const * a, poly16x4x4_t b) {
// CHECK-LABEL: define <8 x i8> @test_vmax_s8(
// CHECK-SAME: <8 x i8> noundef [[A:%.*]], <8 x i8> noundef [[B:%.*]]) #[[ATTR0]] {
// CHECK-NEXT: [[ENTRY:.*:]]
-// CHECK-NEXT: [[VMAX_V_I:%.*]] = call <8 x i8> @llvm.arm.neon.vmaxs.v8i8(<8 x i8> [[A]], <8 x i8> [[B]])
+// CHECK-NEXT: [[VMAX_V_I:%.*]] = call <8 x i8> @llvm.smax.v8i8(<8 x i8> [[A]], <8 x i8> [[B]])
// CHECK-NEXT: ret <8 x i8> [[VMAX_V_I]]
//
int8x8_t test_vmax_s8(int8x8_t a, int8x8_t b) {
@@ -9170,7 +9170,7 @@ int8x8_t test_vmax_s8(int8x8_t a, int8x8_t b) {
// CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i16> [[B]] to <8 x i8>
// CHECK-NEXT: [[VMAX_V_I:%.*]] = bitcast <8 x i8> [[TMP0]] to <4 x i16>
// CHECK-NEXT: [[VMAX_V1_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <4 x i16>
-// CHECK-NEXT: [[VMAX_V2_I:%.*]] = call <4 x i16> @llvm.arm.neon.vmaxs.v4i16(<4 x i16> [[VMAX_V_I]], <4 x i16> [[VMAX_V1_I]])
+// CHECK-NEXT: [[VMAX_V2_I:%.*]] = call <4 x i16> @llvm.smax.v4i16(<4 x i16> [[VMAX_V_I]], <4 x i16> [[VMAX_V1_I]])
// CHECK-NEXT: [[VMAX_V3_I:%.*]] = bitcast <4 x i16> [[VMAX_V2_I]] to <8 x i8>
// CHECK-NEXT: [[TMP2:%.*]] = bitcast <8 x i8> [[VMAX_V3_I]] to <4 x i16>
// CHECK-NEXT: ret <4 x i16> [[TMP2]]
@@ -9186,7 +9186,7 @@ int16x4_t test_vmax_s16(int16x4_t a, int16x4_t b) {
// CHECK-NEXT: [[TMP1:%.*]] = bitcast <2 x i32> [[B]] to <8 x i8>
// CHECK-NEXT: [[VMAX_V_I:%.*]] = bitcast <8 x i8> [[TMP0]] to <2 x i32>
// CHECK-NEXT: [[VMAX_V1_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <2 x i32>
-// CHECK-NEXT: [[VMAX_V2_I:%.*]] = call <2 x i32> @llvm.arm.neon.vmaxs.v2i32(<2 x i32> [[VMAX_V_I]], <2 x i32> [[VMAX_V1_I]])
+// CHECK-NEXT: [[VMAX_V2_I:%.*]] = call <2 x i32> @llvm.smax.v2i32(<2 x i32> [[VMAX_V_I]], <2 x i32> [[VMAX_V1_I]])
// CHECK-NEXT: [[VMAX_V3_I:%.*]] = bitcast <2 x i32> [[VMAX_V2_I]] to <8 x i8>
// CHECK-NEXT: [[TMP2:%.*]] = bitcast <8 x i8> [[VMAX_V3_I]] to <2 x i32>
// CHECK-NEXT: ret <2 x i32> [[TMP2]]
@@ -9198,7 +9198,7 @@ int32x2_t test_vmax_s32(int32x2_t a, int32x2_t b) {
// CHECK-LABEL: define <8 x i8> @test_vmax_u8(
// CHECK-SAME: <8 x i8> noundef [[A:%.*]], <8 x i8> noundef [[B:%.*]]) #[[ATTR0]] {
// CHECK-NEXT: [[ENTRY:.*:]]
-// CHECK-NEXT: [[VMAX_V_I:%.*]] = call <8 x i8> @llvm.arm.neon.vmaxu.v8i8(<8 x i8> [[A]], <8 x i8> [[B]])
+// CHECK-NEXT: [[VMAX_V_I:%.*]] = call <8 x i8> @llvm.umax.v8i8(<8 x i8> [[A]], <8 x i8> [[B]])
// CHECK-NEXT: ret <8 x i8> [[VMAX_V_I]]
//
uint8x8_t test_vmax_u8(uint8x8_t a, uint8x8_t b) {
@@ -9212,7 +9212,7 @@ uint8x8_t test_vmax_u8(uint8x8_t a, uint8x8_t b) {
// CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i16> [[B]] to <8 x i8>
// CHECK-NEXT: [[VMAX_V_I:%.*]] = bitcast <8 x i8> [[TMP0]] to <4 x i16>
// CHECK-NEXT: [[VMAX_V1_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <4 x i16>
-// CHECK-NEXT: [[VMAX_V2_I:%.*]] = call <4 x i16> @llvm.arm.neon.vmaxu.v4i16(<4 x i16> [[VMAX_V_I]], <4 x i16> [[VMAX_V1_I]])
+// CHECK-NEXT: [[VMAX_V2_I:%.*]] = call <4 x i16> @llvm.umax.v4i16(<4 x i16> [[VMAX_V_I]], <4 x i16> [[VMAX_V1_I]])
// CHECK-NEXT: [[VMAX_V3_I:%.*]] = bitcast <4 x i16> [[VMAX_V2_I]] to <8 x i8>
// CHECK-NEXT: [[TMP2:%.*]] = bitcast <8 x i8> [[VMAX_V3_I]] to <4 x i16>
// CHECK-NEXT: ret <4 x i16> [[TMP2]]
@@ -9228,7 +9228,7 @@ uint16x4_t test_vmax_u16(uint16x4_t a, uint16x4_t b) {
// CHECK-NEXT: [[TMP1:%.*]] = bitcast <2 x i32> [[B]] to <8 x i8>
// CHECK-NEXT: [[VMAX_V_I:%.*]] = bitcast <8 x i8> [[TMP0]] to <2 x i32>
// CHECK-NEXT: [[VMAX_V1_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <2 x i32>
-// CHECK-NEXT: [[VMAX_V2_I:%.*]] = call <2 x i32> @llvm.arm.neon.vmaxu.v2i32(<2 x i32> [[VMAX_V_I]], <2 x i32> [[VMAX_V1_I]])
+// CHECK-NEXT: [[VMAX_V2_I:%.*]] = call <2 x i32> @llvm.umax.v2i32(<2 x i32> [[VMAX_V_I]], <2 x i32> [[VMAX_V1_I]])
// CHECK-NEXT: [[VMAX_V3_I:%.*]] = bitcast <2 x i32> [[VMAX_V2_I]] to <8 x i8>
// CHECK-NEXT: [[TMP2:%.*]] = bitcast <8 x i8> [[VMAX_V3_I]] to <2 x i32>
// CHECK-NEXT: ret <2 x i32> [[TMP2]]
@@ -9246,7 +9246,7 @@ uint32x2_t test_vmax_u32(uint32x2_t a, uint32x2_t b) {
// CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x i32> [[TMP1]] to <8 x i8>
// CHECK-NEXT: [[VMAX_V_I:%.*]] = bitcast <8 x i8> [[TMP2]] to <2 x float>
// CHECK-NEXT: [[VMAX_V1_I:%.*]] = bitcast <8 x i8> [[TMP3]] to <2 x float>
-// CHECK-NEXT: [[VMAX_V2_I:%.*]] = call <2 x float> @llvm.arm.neon.vmaxs.v2f32(<2 x float> [[VMAX_V_I]], <2 x float> [[VMAX_V1_I]])
+// CHECK-NEXT: [[VMAX_V2_I:%.*]] = call <2 x float> @llvm.maximum.v2f32(<2 x float> [[VMAX_V_I]], <2 x float> [[VMAX_V1_I]])
// CHECK-NEXT: [[VMAX_V3_I:%.*]] = bitcast <2 x float> [[VMAX_V2_I]] to <8 x i8>
// CHECK-NEXT: [[TMP4:%.*]] = bitcast <8 x i8> [[VMAX_V3_I]] to <2 x i32>
// CHECK-NEXT: [[TMP5:%.*]] = bitcast <2 x i32> [[TMP4]] to <2 x float>
@@ -9259,7 +9259,7 @@ float32x2_t test_vmax_f32(float32x2_t a, float32x2_t b) {
// CHECK-LABEL: define <16 x i8> @test_vmaxq_s8(
// CHECK-SAME: <16 x i8> noundef [[A:%.*]], <16 x i8> noundef [[B:%.*]]) #[[ATTR0]] {
// CHECK-NEXT: [[ENTRY:.*:]]
-// CHECK-NEXT: [[VMAXQ_V_I:%.*]] = call <16 x i8> @llvm.arm.neon.vmaxs.v16i8(<16 x i8> [[A]], <16 x i8> [[B]])
+// CHECK-NEXT: [[VMAXQ_V_I:%.*]] = call <16 x i8> @llvm.smax.v16i8(<16 x i8> [[A]], <16 x i8> [[B]])
// CHECK-NEXT: ret <16 x i8> [[VMAXQ_V_I]]
//
int8x16_t test_vmaxq_s8(int8x16_t a, int8x16_t b) {
@@ -9273,7 +9273,7 @@ int8x16_t test_vmaxq_s8(int8x16_t a, int8x16_t b) {
// CHECK-NEXT: [[TMP1:%.*]] = bitcast <8 x i16> [[B]] to <16 x i8>
// CHECK-NEXT: [[VMAXQ_V_I:%.*]] = bitcast <16 x i8> [[TMP0]] to <8 x i16>
// CHECK-NEXT: [[VMAXQ_V1_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <8 x i16>
-// CHECK-NEXT: [[VMAXQ_V2_I:%.*]] = call <8 x i16> @llvm.arm.neon.vmaxs.v8i16(<8 x i16> [[VMAXQ_V_I]], <8 x i16> [[VMAXQ_V1_I]])
+// CHECK-NEXT: [[VMAXQ_V2_I:%.*]] = call <8 x i16> @llvm.smax.v8i16(<8 x i16> [[VMAXQ_V_I]], <8 x i16> [[VMAXQ_V1_I]])
// CHECK-NEXT: [[VMAXQ_V3_I:%.*]] = bitcast <8 x i16> [[VMAXQ_V2_I]] to <16 x i8>
// CHECK-NEXT: [[TMP2:%.*]] = bitcast <16 x i8> [[VMAXQ_V3_I]] to <8 x i16>
// CHECK-NEXT: ret <8 x i16> [[TMP2]]
@@ -9289,7 +9289,7 @@ int16x8_t test_vmaxq_s16(int16x8_t a, int16x8_t b) {
// CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[B]] to <16 x i8>
// CHECK-NEXT: [[VMAXQ_V_I:%.*]] = bitcast <16 x i8> [[TMP0]] to <4 x i32>
// CHECK-NEXT: [[VMAXQ_V1_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <4 x i32>
-// CHECK-NEXT: [[VMAXQ_V2_I:%.*]] = call <4 x i32> @llvm.arm.neon.vmaxs.v4i32(<4 x i32> [[VMAXQ_V_I]], <4 x i32> [[VMAXQ_V1_I]])
+// CHECK-NEXT: [[VMAXQ_V2_I:%.*]] = call <4 x i32> @llvm.smax.v4i32(<4 x i32> [[VMAXQ_V_I]], <4 x i32> [[VMAXQ_V1_I]])
// CHECK-NEXT: [[VMAXQ_V3_I:%.*]] = bitcast <4 x i32> [[VMAXQ_V2_I]] to <16 x i8>
// CHECK-NEXT: [[TMP2:%.*]] = bitcast <16 x i8> [[VMAXQ_V3_I]] to <4 x i32>
// CHECK-NEXT: ret <4 x i32> [[TMP2]]
@@ -9301,7 +9301,7 @@ int32x4_t test_vmaxq_s32(int32x4_t a, int32x4_t b) {
// CHECK-LABEL: define <16 x i8> @test_vmaxq_u8(
// CHECK-SAME: <16 x i8> noundef [[A:%.*]], <16 x i8> noundef [[B:%.*]]) #[[ATTR0]] {
// CHECK-NEXT: [[ENTRY:.*:]]
-// CHECK-NEXT: [[VMAXQ_V_I:%.*]] = call <16 x i8> @llvm.arm.neon.vmaxu.v16i8(<16 x i8> [[A]], <16 x i8> [[B]])
+// CHECK-NEXT: [[VMAXQ_V_I:%.*]] = call <16 x i8> @llvm.umax.v16i8(<16 x i8> [[A]], <16 x i8> [[B]])
// CHECK-NEXT: ret <16 x i8> [[VMAXQ_V_I]]
//
uint8x16_t test_vmaxq_u8(uint8x16_t a, uint8x16_t b) {
@@ -9315,7 +9315,7 @@ uint8x16_t test_vmaxq_u8(uint8x16_t a, uint8x16_t b) {
// CHECK-NEXT: [[TMP1:%.*]] = bitcast <8 x i16> [[B]] to <16 x i8>
// CHECK-NEXT: [[VMAXQ_V_I:%.*]] = bitcast <16 x i8> [[TMP0]] to <8 x i16>
// CHECK-NEXT: [[VMAXQ_V1_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <8 x i16>
-// CHECK-NEXT: [[VMAXQ_V2_I:%.*]] = call <8 x i16> @llvm.arm.neon.vmaxu.v8i16(<8 x i16> [[VMAXQ_V_I]], <8 x i16> [[VMAXQ_V1_I]])
+// CHECK-NEXT: [[VMAXQ_V2_I:%.*]] = call <8 x i16> @llvm.umax.v8i16(<8 x i16> [[VMAXQ_V_I]], <8 x i16> [[VMAXQ_V1_I]])
// CHECK-NEXT: [[VMAXQ_V3_I:%.*]] = bitcast <8 x i16> [[VMAXQ_V2_I]] to <16 x i8>
// CHECK-NEXT: [[TMP2:%.*]] = bitcast <16 x i8> [[VMAXQ_V3_I]] to <8 x i16>
// CHECK-NEXT: ret <8 x i16> [[TMP2]]
@@ -9331,7 +9331,7 @@ uint16x8_t test_vmaxq_u16(uint16x8_t a, uint16x8_t b) {
// CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[B]] to <16 x i8>
// CHECK-NEXT: [[VMAXQ_V_I:%.*]] = bitcast <16 x i8> [[TMP0]] to <4 x i32>
// CHECK-NEXT: [[VMAXQ_V1_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <4 x i32>
-// CHECK-NEXT: [[VMAXQ_V2_I:%.*]] = call <4 x i32> @llvm.arm.neon.vmaxu.v4i32(<4 x i32> [[VMAXQ_V_I]], <4 x i32> [[VMAXQ_V1_I]])
+// CHECK-NEXT: [[VMAXQ_V2_I:%.*]] = call <4 x i32> @llvm.umax.v4i32(<4 x i32> [[VMAXQ_V_I]], <4 x i32> [[VMAXQ_V1_I]])
// CHECK-NEXT: [[VMAXQ_V3_I:%.*]] = bitcast <4 x i32> [[VMAXQ_V2_I]] to <16 x i8>
// CHECK-NEXT: [[TMP2:%.*]] = bitcast <16 x i8> [[VMAXQ_V3_I]] to <4 x i32>
// CHECK-NEXT: ret <4 x i32> [[TMP2]]
@@ -9349,7 +9349,7 @@ uint32x4_t test_vmaxq_u32(uint32x4_t a, uint32x4_t b) {
// CHECK-NEXT: [[TMP3:%.*]] = bitcast <4 x i32> [[TMP1]] to <16 x i8>
// CHECK-NEXT: [[VMAXQ_V_I:%.*]] = bitcast <16 x i8> [[TMP2]] to <4 x float>
// CHECK-NEXT: [[VMAXQ_V1_I:%.*]] = bitcast <16 x i8> [[TMP3]] to <4 x float>
-// CHECK-NEXT: [[VMAXQ_V2_I:%.*]] = call <4 x float> @llvm.arm.neon.vmaxs.v4f32(<4 x float> [[VMAXQ_V_I]], <4 x float> [[VMAXQ_V1_I]])
+// CHECK-NEXT: [[VMAXQ_V2_I:%.*]] = call <4 x float> @llvm.maximum.v4f32(<4 x float> [[VMAXQ_V_I]], <4 x float> [[VMAXQ_V1_I]])
// CHECK-NEXT: [[VMAXQ_V3_I:%.*]] = bitcast <4 x float> [[VMAXQ_V2_I]] to <16 x i8>
// CHECK-NEXT: [[TMP4:%.*]] = bitcast <16 x i8> [[VMAXQ_V3_I]] to <4 x i32>
// CHECK-NEXT: [[TMP5:%.*]] = bitcast <4 x i32> [[TMP4]] to <4 x float>
@@ -9362,7 +9362,7 @@ float32x4_t test_vmaxq_f32(float32x4_t a, float32x4_t b) {
// CHECK-LABEL: define <8 x i8> @test_vmin_s8(
// CHECK-SAME: <8 x i8> noundef [[A:%.*]], <8 x i8> noundef [[B:%.*]]) #[[ATTR0]] {
// CHECK-NEXT: [[ENTRY:.*:]]
-// CHECK-NEXT: [[VMIN_V_I:%.*]] = call <8 x i8> @llvm.arm.neon.vmins.v8i8(<8 x i8> [[A]], <8 x i8> [[B]])
+// CHECK-NEXT: [[VMIN_V_I:%.*]] = call <8 x i8> @llvm.smin.v8i8(<8 x i8> [[A]], <8 x i8> [[B]])
// CHECK-NEXT: ret <8 x i8> [[VMIN_V_I]]
//
int8x8_t test_vmin_s8(int8x8_t a, int8x8_t b) {
@@ -9376,7 +9376,7 @@ int8x8_t test_vmin_s8(int8x8_t a, int8x8_t b) {
// CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i16> [[B]] to <8 x i8>
// CHECK-NEXT: [[VMIN_V_I:%.*]] = bitcast <8 x i8> [[TMP0]] to <4 x i16>
// CHECK-NEXT: [[VMIN_V1_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <4 x i16>
-// CHECK-NEXT: [[VMIN_V2_I:%.*]] = call <4 x i16> @llvm.arm.neon.vmins.v4i16(<4 x i16> [[VMIN_V_I]], <4 x i16> [[VMIN_V1_I]])
+// CHECK-NEXT: [[VMIN_V2_I:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[VMIN_V_I]], <4 x i16> [[VMIN_V1_I]])
// CHECK-NEXT: [[VMIN_V3_I:%.*]] = bitcast <4 x i16> [[VMIN_V2_I]] to <8 x i8>
// CHECK-NEXT: [[TMP2:%.*]] = bitcast <8 x i8> [[VMIN_V3_I]] to <4 x i16>
// CHECK-NEXT: ret <4 x i16> [[TMP2]]
@@ -9392,7 +9392,7 @@ int16x4_t test_vmin_s16(int16x4_t a, int16x4_t b) {
// CHECK-NEXT: [[TMP1:%.*]] = bitcast <2 x i32> [[B]] to <8 x i8>
// CHECK-NEXT: [[VMIN_V_I:%.*]] = bitcast <8 x i8> [[TMP0]] to <2 x i32>
// CHECK-NEXT: [[VMIN_V1_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <2 x i32>
-// CHECK-NEXT: [[VMIN_V2_I:%.*]] = call <2 x i32> @llvm.arm.neon.vmins.v2i32(<2 x i32> [[VMIN_V_I]], <2 x i32> [[VMIN_V1_I]])
+// CHECK-NEXT: [[VMIN_V2_I:%.*]] = call <2 x i32> @llvm.smin.v2i32(<2 x i32> [[VMIN_V_I]], <2 x i32> [[VMIN_V1_I]])
// CHECK-NEXT: [[VMIN_V3_I:%.*]] = bitcast <2 x i32> [[VMIN_V2_I]] to <8 x i8>
// CHECK-NEXT: [[TMP2:%.*]] = bitcast <8 x i8> [[VMIN_V3_I]] to <2 x i32>
// CHECK-NEXT: ret <2 x i32> [[TMP2]]
@@ -9404,7 +9404,7 @@ int32x2_t test_vmin_s32(int32x2_t a, int32x2_t b) {
// CHECK-LABEL: define <8 x i8> @test_vmin_u8(
// CHECK-SAME: <8 x i8> noundef [[A:%.*]], <8 x i8> noundef [[B:%.*]]) #[[ATTR0]] {
// CHECK-NEXT: [[ENTRY:.*:]]
-// CHECK-NEXT: [[VMIN_V_I:%.*]] = call <8 x i8> @llvm.arm.neon.vminu.v8i8(<8 x i8> [[A]], <8 x i8> [[B]])
+// CHECK-NEXT: [[VMIN_V_I:%.*]] = call <8 x i8> @llvm.umin.v8i8(<8 x i8> [[A]], <8 x i8> [[B]])
// CHECK-NEXT: ret <8 x i8> [[VMIN_V_I]]
//
uint8x8_t test_vmin_u8(uint8x8_t a, uint8x8_t b) {
@@ -9418,7 +9418,7 @@ uint8x8_t test_vmin_u8(uint8x8_t a, uint8x8_t b) {
// CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i16> [[B]] to <8 x i8>
// CHECK-NEXT: [[VMIN_V_I:%.*]] = bitcast <8 x i8> [[TMP0]] to <4 x i16>
// CHECK-NEXT: [[VMIN_V1_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <4 x i16>
-// CHECK-NEXT: [[VMIN_V2_I:%.*]] = call <4 x i16> @llvm.arm.neon.vminu.v4i16(<4 x i16> [[VMIN_V_I]], <4 x i16> [[VMIN_V1_I]])
+// CHECK-NEXT: [[VMIN_V2_I:%.*]] = call <4 x i16> @llvm.umin.v4i16(<4 x i16> [[VMIN_V_I]], <4 x i16> [[VMIN_V1_I]])
// CHECK-NEXT: [[VMIN_V3_I:%.*]] = bitcast <4 x i16> [[VMIN_V2_I]] to <8 x i8>
// CHECK-NEXT: [[TMP2:%.*]] = bitcast <8 x i8> [[VMIN_V3_I]] to <4 x i16>
// CHECK-NEXT: ret <4 x i16> [[TMP2]]
@@ -9434,7 +9434,7 @@ uint16x4_t test_vmin_u16(uint16x4_t a, uint16x4_t b) {
// CHECK-NEXT: [[TMP1:%.*]] = bitcast <2 x i32> [[B]] to <8 x i8>
// CHECK-NEXT: [[VMIN_V_I:%.*]] = bitcast <8 x i8> [[TMP0]] to <2 x i32>
// CHECK-NEXT: [[VMIN_V1_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <2 x i32>
-// CHECK-NEXT: [[VMIN_V2_I:%.*]] = call <2 x i32> @llvm.arm.neon.vminu.v2i32(<2 x i32> [[VMIN_V_I]], <2 x i32> [[VMIN_V1_I]])
+// CHECK-NEXT: [[VMIN_V2_I:%.*]] = call <2 x i32> @llvm.umin.v2i32(<2 x i32> [[VMIN_V_I]], <2 x i32> [[VMIN_V1_I]])
// CHECK-NEXT: [[VMIN_V3_I:%.*]] = bitcast <2 x i32> [[VMIN_V2_I]] to <8 x i8>
// CHECK-NEXT: [[TMP2:%.*]] = bitcast <8 x i8> [[VMIN_V3_I]] to <2 x i32>
// CHECK-NEXT: ret <2 x i32> [[TMP2]]
@@ -9452,7 +9452,7 @@ uint32x2_t test_vmin_u32(uint32x2_t a, uint32x2_t b) {
// CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x i32> [[TMP1]] to <8 x i8>
// CHECK-NEXT: [[VMIN_V_I:%.*]] = bitcast <8 x i8> [[TMP2]] to <2 x float>
// CHECK-NEXT: [[VMIN_V1_I:%.*]] = bitcast <8 x i8> [[TMP3]] to <2 x float>
-// CHECK-NEXT: [[VMIN_V2_I:%.*]] = call <2 x float> @llvm.arm.neon.vmins.v2f32(<2 x float> [[VMIN_V_I]], <2 x float> [[VMIN_V1_I]])
+// CHECK-NEXT: [[VMIN_V2_I:%.*]] = call <2 x float> @llvm.minimum.v2f32(<2 x float> [[VMIN_V_I]], <2 x float> [[VMIN_V1_I]])
// CHECK-NEXT: [[VMIN_V3_I:%.*]] = bitcast <2 x float> [[VMIN_V2_I]] to <8 x i8>
// CHECK-NEXT: [[TMP4:%.*]] = bitcast <8 x i8> [[VMIN_V3_I]] to <2 x i32>
// CHECK-NEXT: [[TMP5:%.*]] = bitcast <2 x i32> [[TMP4]] to <2 x float>
@@ -9465,7 +9465,7 @@ float32x2_t test_vmin_f32(float32x2_t a, float32x2_t b) {
// CHECK-LABEL: define <16 x i8> @test_vminq_s8(
// CHECK-SAME: <16 x i8> noundef [[A:%.*]], <16 x i8> noundef [[B:%.*]]) #[[ATTR0]] {
// CHECK-NEXT: [[ENTRY:.*:]]
-// CHECK-NEXT: [[VMINQ_V_I:%.*]] = call <16 x i8> @llvm.arm.neon.vmins.v16i8(<16 x i8> [[A]], <16 x i8> [[B]])
+// CHECK-NEXT: [[VMINQ_V_I:%.*]] = call <16 x i8> @llvm.smin.v16i8(<16 x i8> [[A]], <16 x i8> [[B]])
// CHECK-NEXT: ret <16 x i8> [[VMINQ_V_I]]
//
int8x16_t test_vminq_s8(int8x16_t a, int8x16_t b) {
@@ -9479,7 +9479,7 @@ int8x16_t test_vminq_s8(int8x16_t a, int8x16_t b) {
// CHECK-NEXT: [[TMP1:%.*]] = bitcast <8 x i16> [[B]] to <16 x i8>
// CHECK-NEXT: [[VMINQ_V_I:%.*]] = bitcast <16 x i8> [[TMP0]] to <8 x i16>
// CHECK-NEXT: [[VMINQ_V1_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <8 x i16>
-// CHECK-NEXT: [[VMINQ_V2_I:%.*]] = call <8 x i16> @llvm.arm.neon.vmins.v8i16(<8 x i16> [[VMINQ_V_I]], <8 x i16> [[VMINQ_V1_I]])
+// CHECK-NEXT: [[VMINQ_V2_I:%.*]] = call <8 x i16> @llvm.smin.v8i16(<8 x i16> [[VMINQ_V_I]], <8 x i16> [[VMINQ_V1_I]])
// CHECK-NEXT: [[VMINQ_V3_I:%.*]] = bitcast <8 x i16> [[VMINQ_V2_I]] to <16 x i8>
// CHECK-NEXT: [[TMP2:%.*]] = bitcast <16 x i8> [[VMINQ_V3_I]] to <8 x i16>
// CHECK-NEXT: ret <8 x i16> [[TMP2]]
@@ -9495,7 +9495,7 @@ int16x8_t test_vminq_s16(int16x8_t a, int16x8_t b) {
// CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[B]] to <16 x i8>
// CHECK-NEXT: [[VMINQ_V_I:%.*]] = bitcast <16 x i8> [[TMP0]] to <4 x i32>
// CHECK-NEXT: [[VMINQ_V1_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <4 x i32>
-// CHECK-NEXT: [[VMINQ_V2_I:%.*]] = call <4 x i32> @llvm.arm.neon.vmins.v4i32(<4 x i32> [[VMINQ_V_I]], <4 x i32> [[VMINQ_V1_I]])
+// CHECK-NEXT: [[VMINQ_V2_I:%.*]] = call <4 x i32> @llvm.smin.v4i32(<4 x i32> [[VMINQ_V_I]], <4 x i32> [[VMINQ_V1_I]])
// CHECK-NEXT: [[VMINQ_V3_I:%.*]] = bitcast <4 x i32> [[VMINQ_V2_I]] to <16 x i8>
// CHECK-NEXT: [[TMP2:%.*]] = bitcast <16 x i8> [[VMINQ_V3_I]] to <4 x i32>
// CHECK-NEXT: ret <4 x i32> [[TMP2]]
@@ -9507,7 +9507,7 @@ int32x4_t test_vminq_s32(int32x4_t a, int32x4_t b) {
// CHECK-LABEL: define <16 x i8> @test_vminq_u8(
// CHECK-SAME: <16 x i8> noundef [[A:%.*]], <16 x i8> noundef [[B:%.*]]) #[[ATTR0]] {
// CHECK-NEXT: [[ENTRY:.*:]]
-// CHECK-NEXT: [[VMINQ_V_I:%.*]] = call <16 x i8> @llvm.arm.neon.vminu.v16i8(<16 x i8> [[A]], <16 x i8> [[B]])
+// CHECK-NEXT: [[VMINQ_V_I:%.*]] = call <16 x i8> @llvm.umin.v16i8(<16 x i8> [[A]], <16 x i8> [[B]])
// CHECK-NEXT: ret <16 x i8> [[VMINQ_V_I]]
//
uint8x16_t test_vminq_u8(uint8x16_t a, uint8x16_t b) {
@@ -9521,7 +9521,7 @@ uint8x16_t test_vminq_u8(uint8x16_t a, uint8x16_t b) {
// CHECK-NEXT: [[TMP1:%.*]] = bitcast <8 x i16> [[B]] to <16 x i8>
// CHECK-NEXT: [[VMINQ_V_I:%.*]] = bitcast <16 x i8> [[TMP0]] to <8 x i16>
// CHECK-NEXT: [[VMINQ_V1_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <8 x i16>
-// CHECK-NEXT: [[VMINQ_V2_I:%.*]] = call <8 x i16> @llvm.arm.neon.vminu.v8i16(<8 x i16> [[VMINQ_V_I]], <8 x i16> [[VMINQ_V1_I]])
+// CHECK-NEXT: [[VMINQ_V2_I:%.*]] = call <8 x i16> @llvm.umin.v8i16(<8 x i16> [[VMINQ_V_I]], <8 x i16> [[VMINQ_V1_I]])
// CHECK-NEXT: [[VMINQ_V3_I:%.*]] = bitcast <8 x i16> [[VMINQ_V2_I]] to <16 x i8>
// CHECK-NEXT: [[TMP2:%.*]] = bitcast <16 x i8> [[VMINQ_V3_I]] to <8 x i16>
// CHECK-NEXT: ret <8 x i16> [[TMP2]]
@@ -9537,7 +9537,7 @@ uint16x8_t test_vminq_u16(uint16x8_t a, uint16x8_t b) {
// CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[B]] to <16 x i8>
// CHECK-NEXT: [[VMINQ_V_I:%.*]] = bitcast <16 x i8> [[TMP0]] to <4 x i32>
// CHECK-NEXT: [[VMINQ_V1_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <4 x i32>
-// CHECK-NEXT: [[VMINQ_V2_I:%.*]] = call <4 x i32> @llvm.arm.neon.vminu.v4i32(<4 x i32> [[VMINQ_V_I]], <4 x i32> [[VMINQ_V1_I]])
+// CHECK-NEXT: [[VMINQ_V2_I:%.*]] = call <4 x i32> @llvm.umin.v4i32(<4 x i32> [[VMINQ_V_I]], <4 x i32> [[VMINQ_V1_I]])
// CHECK-NEXT: [[VMINQ_V3_I:%.*]] = bitcast <4 x i32> [[VMINQ_V2_I]] to <16 x i8>
// CHECK-NEXT: [[TMP2:%.*]] = bitcast <16 x i8> [[VMINQ_V3_I]] to <4 x i32>
// CHECK-NEXT: ret <4 x i32> [[TMP2]]
@@ -9555,7 +9555,7 @@ uint32x4_t test_vminq_u32(uint32x4_t a, uint32x4_t b) {
// CHECK-NEXT: [[TMP3:%.*]] = bitcast <4 x i32> [[TMP1]] to <16 x i8>
// CHECK-NEXT: [[VMINQ_V_I:%.*]] = bitcast <16 x i8> [[TMP2]] to <4 x float>
// CHECK-NEXT: [[VMINQ_V1_I:%.*]] = bitcast <16 x i8> [[TMP3]] to <4 x float>
-// CHECK-NEXT: [[VMINQ_V2_I:%.*]] = call <4 x float> @llvm.arm.neon.vmins.v4f32(<4 x float> [[VMINQ_V_I]], <4 x float> [[VMINQ_V1_I]])
+// CHECK-NEXT: [[VMINQ_V2_I:%.*]] = call <4 x float> @llvm.minimum.v4f32(<4 x float> [[VMINQ_V_I]], <4 x float> [[VMINQ_V1_I]])
// CHECK-NEXT: [[VMINQ_V3_I:%.*]] = bitcast <4 x float> [[VMINQ_V2_I]] to <16 x i8>
// CHECK-NEXT: [[TMP4:%.*]] = bitcast <16 x i8> [[VMINQ_V3_I]] to <4 x i32>
// CHECK-NEXT: [[TMP5:%.*]] = bitcast <4 x i32> [[TMP4]] to <4 x float>
>From e5f6e868c9f7798a89a8ae4edf80cbb5e1c70e60 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ga=C3=ABtan=20Bossu?= <gaetan.bossu at arm.com>
Date: Wed, 6 May 2026 07:47:19 +0000
Subject: [PATCH 3/7] [AArch64][SDAG] Legalise nxv1 gather/scatter nodes
WIP Needs extra checks but otherwise could go upstream.
---
.../SelectionDAG/LegalizeVectorTypes.cpp | 20 +++++++++----------
.../AArch64/sve-masked-gather-64b-scaled.ll | 17 ++++++++++++++++
.../AArch64/sve-masked-gather-64b-unscaled.ll | 17 ++++++++++++++++
.../test/CodeGen/AArch64/sve-masked-gather.ll | 16 +++++++++++++++
.../CodeGen/AArch64/sve-masked-scatter.ll | 15 ++++++++++++++
5 files changed, 74 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 5350be412176d..806d1b2d2fc92 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -6971,27 +6971,25 @@ SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) {
EVT MaskVT = Mask.getValueType();
SDValue PassThru = GetWidenedVector(N->getPassThru());
SDValue Scale = N->getScale();
- unsigned NumElts = WideVT.getVectorNumElements();
+ ElementCount WideEC = WideVT.getVectorElementCount();
SDLoc dl(N);
// The mask should be widened as well
EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
- MaskVT.getVectorElementType(),
- WideVT.getVectorNumElements());
+ MaskVT.getVectorElementType(), WideEC);
Mask = ModifyToType(Mask, WideMaskVT, true);
// Widen the Index operand
SDValue Index = N->getIndex();
- EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
- Index.getValueType().getScalarType(),
- NumElts);
+ EVT WideIndexVT = EVT::getVectorVT(
+ *DAG.getContext(), Index.getValueType().getScalarType(), WideEC);
Index = ModifyToType(Index, WideIndexVT);
SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index,
Scale };
// Widen the MemoryType
EVT WideMemVT = EVT::getVectorVT(*DAG.getContext(),
- N->getMemoryVT().getScalarType(), NumElts);
+ N->getMemoryVT().getScalarType(), WideEC);
SDValue Res = DAG.getMaskedGather(DAG.getVTList(WideVT, MVT::Other),
WideMemVT, dl, Ops, N->getMemOperand(),
N->getIndexType(), N->getExtensionType());
@@ -8318,23 +8316,23 @@ SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) {
if (OpNo == 1) {
DataOp = GetWidenedVector(DataOp);
- unsigned NumElts = DataOp.getValueType().getVectorNumElements();
+ ElementCount WideEC = DataOp.getValueType().getVectorElementCount();
// Widen index.
EVT IndexVT = Index.getValueType();
EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
- IndexVT.getVectorElementType(), NumElts);
+ IndexVT.getVectorElementType(), WideEC);
Index = ModifyToType(Index, WideIndexVT);
// The mask should be widened as well.
EVT MaskVT = Mask.getValueType();
EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
- MaskVT.getVectorElementType(), NumElts);
+ MaskVT.getVectorElementType(), WideEC);
Mask = ModifyToType(Mask, WideMaskVT, true);
// Widen the MemoryType
WideMemVT = EVT::getVectorVT(*DAG.getContext(),
- MSC->getMemoryVT().getScalarType(), NumElts);
+ MSC->getMemoryVT().getScalarType(), WideEC);
} else if (OpNo == 4) {
// Just widen the index. It's allowed to have extra elements.
Index = GetWidenedVector(Index);
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-scaled.ll b/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-scaled.ll
index 624541766c957..72b8496c8d8b7 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-scaled.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-scaled.ll
@@ -96,6 +96,23 @@ define <vscale x 2 x i64> @masked_sgather_nxv2i32(ptr %base, <vscale x 2 x i64>
ret <vscale x 2 x i64> %vals.sext
}
+define <vscale x 2 x i64> @masked_gather_nxv1i64(ptr %base, <vscale x 2 x i64> %wide.offsets, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv1i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, z0.d, lsl #3]
+; CHECK-NEXT: ret
+ %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(
+ <vscale x 2 x i64> %wide.offsets, i64 0)
+ %ptrs = getelementptr i64, ptr %base, <vscale x 1 x i64> %offsets
+ %r = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64(
+ <vscale x 1 x ptr> %ptrs, i32 8, <vscale x 1 x i1> %mask, <vscale x 1 x i64> poison)
+ %r.legal = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.nxv1i64(
+ <vscale x 2 x i64> poison, <vscale x 1 x i64> %r, i64 0)
+ ret <vscale x 2 x i64> %r.legal
+}
+
declare <vscale x 2 x i16> @llvm.masked.gather.nxv2i16(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x i16>)
declare <vscale x 2 x i32> @llvm.masked.gather.nxv2i32(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x i32>)
declare <vscale x 2 x i64> @llvm.masked.gather.nxv2i64(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x i64>)
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-unscaled.ll b/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-unscaled.ll
index 0ed4dd1e4136e..493c81f51145a 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-unscaled.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-unscaled.ll
@@ -127,6 +127,23 @@ define <vscale x 2 x i64> @masked_sgather_nxv2i32(ptr %base, <vscale x 2 x i64>
ret <vscale x 2 x i64> %vals.sext
}
+define <vscale x 2 x i64> @masked_gather_nxv1i64(ptr %base, <vscale x 2 x i64> %wide.offsets, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv1i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, z0.d]
+; CHECK-NEXT: ret
+ %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(
+ <vscale x 2 x i64> %wide.offsets, i64 0)
+ %ptrs = getelementptr i8, ptr %base, <vscale x 1 x i64> %offsets
+ %r = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64(
+ <vscale x 1 x ptr> %ptrs, i32 8, <vscale x 1 x i1> %mask, <vscale x 1 x i64> poison)
+ %r.legal = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.nxv1i64(
+ <vscale x 2 x i64> poison, <vscale x 1 x i64> %r, i64 0)
+ ret <vscale x 2 x i64> %r.legal
+}
+
declare <vscale x 2 x i8> @llvm.masked.gather.nxv2i8(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x i8>)
declare <vscale x 2 x i16> @llvm.masked.gather.nxv2i16(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x i16>)
declare <vscale x 2 x i32> @llvm.masked.gather.nxv2i32(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x i32>)
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-gather.ll b/llvm/test/CodeGen/AArch64/sve-masked-gather.ll
index 5a7865e92415f..354377147a19d 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-gather.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-gather.ll
@@ -96,6 +96,22 @@ define <vscale x 2 x i64> @masked_sgather_nxv2i16(<vscale x 2 x ptr> %ptrs, <vsc
ret <vscale x 2 x i64> %vals.sext
}
+define <vscale x 2 x i64> @masked_gather_nxv1i64(<vscale x 2 x ptr> %wide.ptrs, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv1i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [z0.d]
+; CHECK-NEXT: ret
+ %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(
+ <vscale x 2 x ptr> %wide.ptrs, i64 0)
+ %r = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64(
+ <vscale x 1 x ptr> %ptrs, i32 8, <vscale x 1 x i1> %mask, <vscale x 1 x i64> poison)
+ %r.legal = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.nxv1i64(
+ <vscale x 2 x i64> poison, <vscale x 1 x i64> %r, i64 0)
+ ret <vscale x 2 x i64> %r.legal
+}
+
define <vscale x 2 x i64> @masked_sgather_nxv2i32(<vscale x 2 x ptr> %ptrs, <vscale x 2 x i1> %mask) {
; CHECK-LABEL: masked_sgather_nxv2i32:
; CHECK: // %bb.0:
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-scatter.ll b/llvm/test/CodeGen/AArch64/sve-masked-scatter.ll
index 8f2cbbdb55636..410bacc98b57a 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-scatter.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-scatter.ll
@@ -73,6 +73,21 @@ define void @masked_scatter_nxv2f64(<vscale x 2 x double> %data, <vscale x 2 x p
ret void
}
+define void @masked_scatter_nxv1i64(<vscale x 2 x i64> %data.wide, <vscale x 2 x ptr> %wide.ptrs, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_scatter_nxv1i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: st1d { z0.d }, p0, [z1.d]
+; CHECK-NEXT: ret
+ %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(
+ <vscale x 2 x ptr> %wide.ptrs, i64 0)
+ %data = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(
+ <vscale x 2 x i64> %data.wide, i64 0)
+ call void @llvm.masked.scatter.nxv1i64(<vscale x 1 x i64> %data, <vscale x 1 x ptr> %ptrs, i32 8, <vscale x 1 x i1> %mask)
+ ret void
+}
+
define void @masked_scatter_splat_constant_pointer (<vscale x 4 x i1> %pg) {
; CHECK-LABEL: masked_scatter_splat_constant_pointer:
; CHECK: // %bb.0: // %vector.body
>From 0044a6e014b7dd28dee64621a7b75754d61b20b9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ga=C3=ABtan=20Bossu?= <gaetan.bossu at arm.com>
Date: Fri, 5 Jun 2026 07:55:40 +0000
Subject: [PATCH 4/7] [AArch64][SDAG] Improve lowering of narrow (de)interleave
intrinsics
We previously promoted the narrow types, while the operation could
become a nop or a single instruction by re-interpreting the narrow types
as their packed equivalent.
WIP this needs better integration now that the following are merged
upstream:
https://github.com/llvm/llvm-project/pull/154338
https://github.com/llvm/llvm-project/pull/175585
---
.../Target/AArch64/AArch64ISelLowering.cpp | 103 ++++++++++++++++--
.../scalable_masked_interleaved_stores.ll | 8 +-
.../AArch64/sve-vector-deinterleave.ll | 24 ++++
.../CodeGen/AArch64/sve-vector-interleave.ll | 80 ++++++++------
4 files changed, 165 insertions(+), 50 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index efebe835fa530..250608a484467 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -269,6 +269,53 @@ static inline bool isUnpackedType(EVT VT, SelectionDAG &DAG) {
return Res;
}
+/// Reinterpret \p Op as a packed VT with the same element type.
+/// E.g. nxv8i8 -> nxv16i8 where every odd element is undefined
+/// nxv4f16 -> nxv8f16
+static SDValue getReinterpretAsPackedVT(SDValue Op, SelectionDAG &DAG) {
+ SDLoc DL(Op);
+ EVT InVT = Op.getValueType();
+ EVT PackedVTForSameElt = getPackedSVEVectorVT(InVT.getVectorElementType());
+ EVT PackedVTForSameLen = getPackedSVEVectorVT(InVT.getVectorElementCount());
+
+ if (DAG.getTargetLoweringInfo().isTypeLegal(InVT)) {
+ if (isPackedVectorType(InVT, DAG))
+ return Op;
+
+ // Legal unpacked types can be immediately reinterpreted as their
+ // packed type.
+ return DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL, PackedVTForSameElt,
+ Op);
+ }
+
+ // Illegal unpacked types need to be extended first to be made legal.
+ assert(InVT.isInteger() && "Expected integer vector");
+ return DAG.getBitcast(PackedVTForSameElt,
+ DAG.getAnyExtOrTrunc(Op, DL, PackedVTForSameLen));
+}
+
+/// Reinterpret \p Op as an unpacked VT with the same element type.
+/// E.g. nxv16i8 -> nxv8i8 where every odd element is undefined
+/// nxv8f16 -> nxv4f16
+static SDValue getReinterpretAsUnpackedVT(SDValue Op, EVT UnpackedVT,
+ SelectionDAG &DAG) {
+ SDLoc DL(Op);
+ EVT InVT = Op.getValueType();
+ assert(DAG.getTargetLoweringInfo().isTypeLegal(InVT) &&
+ isPackedVectorType(InVT, DAG));
+ assert(!DAG.getTargetLoweringInfo().isTypeLegal(UnpackedVT) ||
+ !isPackedVectorType(UnpackedVT, DAG));
+ assert(InVT.getVectorElementType() == UnpackedVT.getVectorElementType());
+
+ if (DAG.getTargetLoweringInfo().isTypeLegal(UnpackedVT))
+ return DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL, UnpackedVT, Op);
+
+ EVT PackedVTForDstLen =
+ getPackedSVEVectorVT(UnpackedVT.getVectorElementCount());
+ return DAG.getAnyExtOrTrunc(DAG.getBitcast(PackedVTForDstLen, Op), DL,
+ UnpackedVT);
+}
+
// Returns true for ####_MERGE_PASSTHRU opcodes, whose operands have a leading
// predicate and end with a passthru value matching the result type.
static bool isMergePassthruOpcode(unsigned Opc) {
@@ -21844,6 +21891,32 @@ static SDValue performConcatVectorsCombine(SDNode *N,
return DAG.getNode(AArch64ISD::TRN1, DL, VT, Op0MoreElems, Op1MoreElems);
}
+ // Interleave elements of narrow vectors into a packed legal type.
+ // This maps to trn1 after re-interpreting the outputs as packed types where
+ // odd elements are undefined.
+ //
+ // E.g. nxv4i16, nxv4i16 = VECTOR_INTERLEAVE(nxv4i16 V0, nxv4i16 V1)
+ // nxv8i16 = CONCAT_VECTORS interleaved.0, interleaved.1
+ // <=> nxv8i16 = trn1(nxv8i16 (packed_reinterpret V0),
+ // nxv8i16 (packed_reinterpret V1))
+ if (N->getNumOperands() == 2 && N0Opc == ISD::VECTOR_INTERLEAVE &&
+ N0.getNode() == N1.getNode() && N0->getNumOperands() == 2 &&
+ DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
+ isPackedVectorType(VT, DAG)) {
+ assert(VT.isScalableVector() && "VECTOR_INTERLEAVE for a fixed vector?");
+ assert(VT.isSimple() && "VECTOR_INTERLEAVE of extended types?");
+
+ if (N0->getOperand(1).isUndef())
+ return getReinterpretAsPackedVT(N0->getOperand(0), DAG);
+
+ SDValue ReinterpretedV0 = getReinterpretAsPackedVT(N0->getOperand(0), DAG);
+ SDValue ReinterpretedV1 = getReinterpretAsPackedVT(N0->getOperand(1), DAG);
+ SDValue IDVal =
+ DAG.getTargetConstant(Intrinsic::aarch64_sve_trn1, DL, MVT::i64);
+ SDValue Ops[] = {IDVal, ReinterpretedV0, ReinterpretedV1};
+ return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, Ops);
+ }
+
if (VT.isScalableVector())
return SDValue();
@@ -29486,14 +29559,7 @@ static SDValue performVectorDeinterleaveCombine(
return SDValue();
EVT SubVecTy = N->getValueType(0);
-
- // At the moment we're unlikely to see a fixed-width vector deinterleave as
- // we usually generate shuffles instead.
unsigned MinNumElements = SubVecTy.getVectorMinNumElements();
- if (!SubVecTy.isScalableVector() ||
- SubVecTy.getSizeInBits().getKnownMinValue() != 128 ||
- !DAG.getTargetLoweringInfo().isTypeLegal(SubVecTy))
- return SDValue();
// Make sure each input operand is the correct extract_subvector of the same
// wider vector.
@@ -29507,6 +29573,28 @@ static SDValue performVectorDeinterleaveCombine(
return SDValue();
}
+ SDValue WideSrc = Op0->getOperand(0);
+ EVT WideSrcTy = WideSrc.getValueType();
+ SDLoc DL(N);
+
+ // Look for "extract even elements" from legal packed types
+ if (!isa<MaskedLoadSDNode>(WideSrc) && NumParts == 2 &&
+ DAG.getTargetLoweringInfo().isTypeLegal(WideSrcTy) &&
+ isPackedVectorType(WideSrcTy, DAG) && !N->hasAnyUseOfValue(1)) {
+
+ return DAG.getMergeValues(
+ {getReinterpretAsUnpackedVT(WideSrc, SubVecTy, DAG),
+ DAG.getPOISON(SubVecTy)},
+ DL);
+ }
+
+ // At the moment we're unlikely to see a fixed-width vector deinterleave as
+ // we usually generate shuffles instead.
+ if (!SubVecTy.isScalableVector() ||
+ SubVecTy.getSizeInBits().getKnownMinValue() != 128 ||
+ !DAG.getTargetLoweringInfo().isTypeLegal(SubVecTy))
+ return SDValue();
+
// Normal loads are currently already handled by the InterleavedAccessPass so
// we don't expect to see them here. Bail out if the masked load has an
// unexpected number of uses, since we want to avoid a situation where we have
@@ -29522,7 +29610,6 @@ static SDValue performVectorDeinterleaveCombine(
return SDValue();
// Now prove that the mask is an interleave of identical masks.
- SDLoc DL(N);
SDValue NarrowMask =
getNarrowMaskForInterleavedOps(DAG, DL, MaskedLoad->getMask(), NumParts);
if (!NarrowMask)
diff --git a/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll b/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
index d3cd9bf08cc0a..e042a48f81b40 100644
--- a/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
+++ b/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
@@ -255,10 +255,8 @@ define void @foo_st2_nxv16i8_bad_mask3(<vscale x 32 x i1> %mask, <vscale x 16 x
define void @foo_st2_nxv8i8(<vscale x 8 x i1> %mask, <vscale x 8 x i8> %val1, <vscale x 8 x i8> %val2, ptr %p) {
; CHECK-LABEL: foo_st2_nxv8i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: zip2 z2.h, z0.h, z1.h
-; CHECK-NEXT: zip1 z0.h, z0.h, z1.h
+; CHECK-NEXT: trn1 z0.b, z0.b, z1.b
; CHECK-NEXT: trn1 p0.b, p0.b, p0.b
-; CHECK-NEXT: uzp1 z0.b, z0.b, z2.b
; CHECK-NEXT: st1b { z0.b }, p0, [x0]
; CHECK-NEXT: ret
%interleaved.mask = call <vscale x 16 x i1> @llvm.vector.interleave2.nxv16i1(<vscale x 8 x i1> %mask, <vscale x 8 x i1> %mask)
@@ -270,10 +268,8 @@ define void @foo_st2_nxv8i8(<vscale x 8 x i1> %mask, <vscale x 8 x i8> %val1, <v
define void @foo_st2_nxv8i8_trunc(<vscale x 4 x i1> %mask, <vscale x 4 x i16> %val1, <vscale x 4 x i16> %val2, ptr %p) {
; CHECK-LABEL: foo_st2_nxv8i8_trunc:
; CHECK: // %bb.0:
-; CHECK-NEXT: zip2 z2.s, z0.s, z1.s
-; CHECK-NEXT: zip1 z0.s, z0.s, z1.s
+; CHECK-NEXT: trn1 z0.h, z0.h, z1.h
; CHECK-NEXT: trn1 p0.h, p0.h, p0.h
-; CHECK-NEXT: uzp1 z0.h, z0.h, z2.h
; CHECK-NEXT: st1b { z0.h }, p0, [x0]
; CHECK-NEXT: ret
%interleaved.mask = call <vscale x 8 x i1> @llvm.vector.interleave2.nxv8i1(<vscale x 4 x i1> %mask, <vscale x 4 x i1> %mask)
diff --git a/llvm/test/CodeGen/AArch64/sve-vector-deinterleave.ll b/llvm/test/CodeGen/AArch64/sve-vector-deinterleave.ll
index f0d4e86752dec..a595fbba0a97e 100644
--- a/llvm/test/CodeGen/AArch64/sve-vector-deinterleave.ll
+++ b/llvm/test/CodeGen/AArch64/sve-vector-deinterleave.ll
@@ -79,6 +79,18 @@ define {<vscale x 2 x float>, <vscale x 2 x float>} @vector_deinterleave_nxv2f32
ret {<vscale x 2 x float>, <vscale x 2 x float>} %retval
}
+define void @vector_deinterleave_even_nxv4f32(<vscale x 4 x float> %vec, ptr %a) {
+; CHECK-LABEL: vector_deinterleave_even_nxv4f32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: st1w { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %res = call {<vscale x 2 x float>, <vscale x 2 x float>} @llvm.vector.deinterleave2.nxv4f32(<vscale x 4 x float> %vec)
+ %res.even = extractvalue { <vscale x 2 x float>, <vscale x 2 x float> } %res, 0
+ store <vscale x 2 x float> %res.even, ptr %a
+ ret void
+}
+
define {<vscale x 4 x float>, <vscale x 4 x float>} @vector_deinterleave_nxv4f32_nxv8f32(<vscale x 8 x float> %vec) {
; SVE-LABEL: vector_deinterleave_nxv4f32_nxv8f32:
; SVE: // %bb.0:
@@ -215,6 +227,18 @@ define {<vscale x 4 x i32>, <vscale x 4 x i32>} @vector_deinterleave_nxv4i32_nxv
ret {<vscale x 4 x i32>, <vscale x 4 x i32>} %retval
}
+define void @vector_deinterleave_even_nxv4i32(<vscale x 4 x i32> %vec, ptr %a) {
+; CHECK-LABEL: vector_deinterleave_even_nxv4i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: st1w { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %res = call {<vscale x 2 x i32>, <vscale x 2 x i32>} @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %vec)
+ %res.even = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %res, 0
+ store <vscale x 2 x i32> %res.even, ptr %a
+ ret void
+}
+
define {<vscale x 2 x i64>, <vscale x 2 x i64>} @vector_deinterleave_nxv2i64_nxv4i64(<vscale x 4 x i64> %vec) {
; SVE-LABEL: vector_deinterleave_nxv2i64_nxv4i64:
; SVE: // %bb.0:
diff --git a/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll b/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll
index 81a0cc242fc11..c68aca8473988 100644
--- a/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll
+++ b/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll
@@ -149,6 +149,27 @@ define <vscale x 16 x i16> @interleave2_nxv16i16(<vscale x 8 x i16> %vec0, <vsca
ret <vscale x 16 x i16> %retval
}
+define <vscale x 16 x i8> @interleave2_nxv16i8(<vscale x 8 x i16> %vec0, <vscale x 8 x i16> %vec1) {
+; CHECK-LABEL: interleave2_nxv16i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: trn1 z0.b, z0.b, z1.b
+; CHECK-NEXT: ret
+ %leg0 = trunc <vscale x 8 x i16> %vec0 to <vscale x 8 x i8>
+ %leg1 = trunc <vscale x 8 x i16> %vec1 to <vscale x 8 x i8>
+ %retval = call <vscale x 16 x i8> @llvm.vector.interleave2.nxv16i8(<vscale x 8 x i8> %leg0, <vscale x 8 x i8> %leg1)
+ ret <vscale x 16 x i8> %retval
+}
+
+define <vscale x 16 x i8> @interleave2_nxv16i8_with_poison(<vscale x 8 x i16> %vec0, <vscale x 8 x i16> %vec1) {
+; CHECK-LABEL: interleave2_nxv16i8_with_poison:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ret
+ %leg0 = trunc <vscale x 8 x i16> %vec0 to <vscale x 8 x i8>
+ %leg1 = trunc <vscale x 8 x i16> %vec1 to <vscale x 8 x i8>
+ %retval = call <vscale x 16 x i8> @llvm.vector.interleave2.nxv16i8(<vscale x 8 x i8> %leg0, <vscale x 8 x i8> poison)
+ ret <vscale x 16 x i8> %retval
+}
+
define <vscale x 8 x i32> @interleave2_nxv8i32(<vscale x 4 x i32> %vec0, <vscale x 4 x i32> %vec1) {
; SVE-LABEL: interleave2_nxv8i32:
; SVE: // %bb.0:
@@ -165,6 +186,17 @@ define <vscale x 8 x i32> @interleave2_nxv8i32(<vscale x 4 x i32> %vec0, <vscale
ret <vscale x 8 x i32> %retval
}
+define <vscale x 8 x i16> @interleave2_nxv8i16(<vscale x 4 x i32> %vec0, <vscale x 4 x i32> %vec1) {
+; CHECK-LABEL: interleave2_nxv8i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: trn1 z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+ %leg0 = trunc <vscale x 4 x i32> %vec0 to <vscale x 4 x i16>
+ %leg1 = trunc <vscale x 4 x i32> %vec1 to <vscale x 4 x i16>
+ %retval = call <vscale x 8 x i16> @llvm.vector.interleave2.nxv8i16(<vscale x 4 x i16> %leg0, <vscale x 4 x i16> %leg1)
+ ret <vscale x 8 x i16> %retval
+}
+
define <vscale x 4 x i64> @interleave2_nxv4i64(<vscale x 2 x i64> %vec0, <vscale x 2 x i64> %vec1) {
; SVE-LABEL: interleave2_nxv4i64:
; SVE: // %bb.0:
@@ -710,52 +742,28 @@ define <vscale x 8 x i64> @interleave2_nxv8i64(<vscale x 4 x i64> %vec0, <vscale
; Promote illegal type size
define <vscale x 16 x i8> @interleave2_nxv8i8(<vscale x 8 x i8> %vec0, <vscale x 8 x i8> %vec1) {
-; SVE-LABEL: interleave2_nxv8i8:
-; SVE: // %bb.0:
-; SVE-NEXT: zip2 z2.h, z0.h, z1.h
-; SVE-NEXT: zip1 z0.h, z0.h, z1.h
-; SVE-NEXT: uzp1 z0.b, z0.b, z2.b
-; SVE-NEXT: ret
-;
-; SME2-LABEL: interleave2_nxv8i8:
-; SME2: // %bb.0:
-; SME2-NEXT: zip { z0.h, z1.h }, z0.h, z1.h
-; SME2-NEXT: uzp1 z0.b, z0.b, z1.b
-; SME2-NEXT: ret
+; CHECK-LABEL: interleave2_nxv8i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: trn1 z0.b, z0.b, z1.b
+; CHECK-NEXT: ret
%retval = call <vscale x 16 x i8> @llvm.vector.interleave2.nxv16i8(<vscale x 8 x i8> %vec0, <vscale x 8 x i8> %vec1)
ret <vscale x 16 x i8> %retval
}
define <vscale x 8 x i16> @interleave2_nxv4i16(<vscale x 4 x i16> %vec0, <vscale x 4 x i16> %vec1) {
-; SVE-LABEL: interleave2_nxv4i16:
-; SVE: // %bb.0:
-; SVE-NEXT: zip2 z2.s, z0.s, z1.s
-; SVE-NEXT: zip1 z0.s, z0.s, z1.s
-; SVE-NEXT: uzp1 z0.h, z0.h, z2.h
-; SVE-NEXT: ret
-;
-; SME2-LABEL: interleave2_nxv4i16:
-; SME2: // %bb.0:
-; SME2-NEXT: zip { z0.s, z1.s }, z0.s, z1.s
-; SME2-NEXT: uzp1 z0.h, z0.h, z1.h
-; SME2-NEXT: ret
+; CHECK-LABEL: interleave2_nxv4i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: trn1 z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
%retval = call <vscale x 8 x i16> @llvm.vector.interleave2.nxv8i16(<vscale x 4 x i16> %vec0, <vscale x 4 x i16> %vec1)
ret <vscale x 8 x i16> %retval
}
define <vscale x 4 x i32> @interleave2_nxv2i32(<vscale x 2 x i32> %vec0, <vscale x 2 x i32> %vec1) {
-; SVE-LABEL: interleave2_nxv2i32:
-; SVE: // %bb.0:
-; SVE-NEXT: zip2 z2.d, z0.d, z1.d
-; SVE-NEXT: zip1 z0.d, z0.d, z1.d
-; SVE-NEXT: uzp1 z0.s, z0.s, z2.s
-; SVE-NEXT: ret
-;
-; SME2-LABEL: interleave2_nxv2i32:
-; SME2: // %bb.0:
-; SME2-NEXT: zip { z0.d, z1.d }, z0.d, z1.d
-; SME2-NEXT: uzp1 z0.s, z0.s, z1.s
-; SME2-NEXT: ret
+; CHECK-LABEL: interleave2_nxv2i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: trn1 z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
%retval = call <vscale x 4 x i32> @llvm.vector.interleave2.nxv4i32(<vscale x 2 x i32> %vec0, <vscale x 2 x i32> %vec1)
ret <vscale x 4 x i32> %retval
}
>From 60f52545a0b27a47fd699c1d24b23c33981aa2fc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ga=C3=ABtan=20Bossu?= <gaetan.bossu at arm.com>
Date: Wed, 10 Jun 2026 13:59:21 +0000
Subject: [PATCH 5/7] [LV] Fix unsafe canVectorizeWithIfConvert() checks
When allowExtraAnalysis is true, we might not have a canonical loop form
to work with.
Opened upstream as https://github.com/llvm/llvm-project/pull/203273
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 3 ++
.../Vectorize/LoopVectorizationLegality.cpp | 7 ++--
.../allowExtraAnalysis-invalid-cfg.ll | 34 +++++++++++++++++++
3 files changed, 41 insertions(+), 3 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/allowExtraAnalysis-invalid-cfg.ll
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 60e15b2a5bd82..69035dac80c42 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -2963,7 +2963,10 @@ bool LoopAccessInfo::blockNeedsPredication(const BasicBlock *BB,
assert(TheLoop->contains(BB) && "Unknown block used");
// Blocks that do not dominate the latch need predication.
+ // Treat blocks in loops with no unique latch as always needing predication.
const BasicBlock *Latch = TheLoop->getLoopLatch();
+ if (!Latch)
+ return true;
return !DT->dominates(BB, Latch);
}
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 214b5cff8a03a..69ca29885f384 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1470,12 +1470,13 @@ bool LoopVectorizationLegality::canVectorizeWithIfConvert() {
continue;
auto *CurrI = dyn_cast<Instruction>(CurrV);
+ BasicBlock *LoopPred = TheLoop->getLoopPredecessor();
if (!CurrI || !TheLoop->contains(CurrI)) {
// If operands from outside the loop may be poison then Ptr may also
// be poison.
- if (!isGuaranteedNotToBePoison(CurrV, AC,
- TheLoop->getLoopPredecessor()
- ->getTerminator()
+ if (!LoopPred ||
+ !isGuaranteedNotToBePoison(CurrV, AC,
+ LoopPred->getTerminator()
->getIterator(),
DT))
return false;
diff --git a/llvm/test/Transforms/LoopVectorize/allowExtraAnalysis-invalid-cfg.ll b/llvm/test/Transforms/LoopVectorize/allowExtraAnalysis-invalid-cfg.ll
new file mode 100644
index 0000000000000..3b562ec67a26a
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/allowExtraAnalysis-invalid-cfg.ll
@@ -0,0 +1,34 @@
+; RUN: opt -S -passes=loop-vectorize -pass-remarks-output=%t.yaml < %s | FileCheck %s
+
+; Enabling remarks forces extra legality analysis on loops with unexpected CFG.
+; Ensure LV does not crash due to the loop having multiple latches.
+; A normal branch gets canonicalized to a unique latch before LV runs, so use
+; indirectbr to keep the invalid CFG shape.
+
+; CHECK-LABEL: define void @multiple_latches_indirectbr(
+define void @multiple_latches_indirectbr() {
+entry:
+ indirectbr ptr null, [label %loop]
+
+loop:
+ br i1 false, label %bb, label %loop
+
+bb:
+ br label %loop
+}
+
+; CHECK-LABEL: define void @multiple_latches_and_predecessors_indirectbr(
+define void @multiple_latches_and_predecessors_indirectbr() {
+entry:
+ indirectbr ptr null, [label %loop, label %side]
+
+side:
+ br label %loop
+
+loop:
+ %x = load i64, ptr null, align 8
+ br i1 false, label %back, label %loop
+
+back:
+ br label %loop
+}
>From 9a7e6c873971b5c833a8f4368c18af4790272b94 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ga=C3=ABtan=20Bossu?= <gaetan.bossu at arm.com>
Date: Fri, 16 Jan 2026 17:49:38 +0000
Subject: [PATCH 6/7] llvm: Add scalable re-vectorization support
Teach LoopVectorize/VPlan to form scalable re-vectorized operations and add
AArch64 SVE cost modelling and lowering support for the resulting patterns.
Additional context will be given in an RFC.
---
llvm/include/llvm/Analysis/IVDescriptors.h | 21 +-
.../llvm/Analysis/TargetTransformInfo.h | 39 +
.../llvm/Analysis/TargetTransformInfoImpl.h | 40 +
llvm/include/llvm/Analysis/VectorUtils.h | 9 +-
llvm/include/llvm/CodeGen/BasicTTIImpl.h | 30 +
llvm/include/llvm/CodeGen/ISDOpcodes.h | 54 +
llvm/include/llvm/CodeGen/SelectionDAG.h | 8 +
llvm/include/llvm/CodeGen/SelectionDAGNodes.h | 23 +
llvm/include/llvm/CodeGen/TargetLowering.h | 14 +-
.../llvm/CodeGenTypes/MachineValueType.h | 4 +
llvm/include/llvm/IR/IRBuilder.h | 22 +-
llvm/include/llvm/IR/Intrinsics.h | 6 +-
llvm/include/llvm/IR/Intrinsics.td | 63 +
llvm/include/llvm/IR/VectorTypeUtils.h | 24 +
.../include/llvm/Target/TargetSelectionDAG.td | 8 +
.../include/llvm/Transforms/Utils/LoopUtils.h | 9 +-
.../Vectorize/LoopVectorizationLegality.h | 5 +
llvm/lib/Analysis/IVDescriptors.cpp | 42 +-
llvm/lib/Analysis/Loads.cpp | 2 +
llvm/lib/Analysis/TargetTransformInfo.cpp | 49 +
llvm/lib/Analysis/VectorUtils.cpp | 34 +-
llvm/lib/CodeGen/InterleavedAccessPass.cpp | 51 +-
llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 43 +-
.../SelectionDAG/LegalizeIntegerTypes.cpp | 88 +
llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h | 4 +
.../SelectionDAG/LegalizeVectorTypes.cpp | 22 +-
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 85 +-
.../SelectionDAG/SelectionDAGBuilder.cpp | 193 +-
.../SelectionDAG/SelectionDAGBuilder.h | 11 +-
.../SelectionDAG/SelectionDAGDumper.cpp | 42 +-
llvm/lib/CodeGen/ValueTypes.cpp | 9 +
llvm/lib/IR/Constants.cpp | 6 +-
llvm/lib/IR/IRBuilder.cpp | 48 +-
llvm/lib/IR/Intrinsics.cpp | 19 +-
llvm/lib/IR/Verifier.cpp | 46 +
llvm/lib/Target/AArch64/AArch64.td | 6 +
.../Target/AArch64/AArch64ISelLowering.cpp | 833 ++++++-
llvm/lib/Target/AArch64/AArch64ISelLowering.h | 28 +-
.../lib/Target/AArch64/AArch64SVEInstrInfo.td | 16 +
.../AArch64/AArch64TargetTransformInfo.cpp | 634 +++++-
.../AArch64/AArch64TargetTransformInfo.h | 61 +
.../AArch64/AArch64VectorIntrinsicMappings.td | 221 ++
llvm/lib/Target/AArch64/CMakeLists.txt | 1 +
llvm/lib/Target/AArch64/SMEInstrFormats.td | 20 +-
llvm/lib/Target/AArch64/SVEInstrFormats.td | 148 +-
llvm/lib/Target/RISCV/RISCVISelLowering.h | 13 +-
.../Target/RISCV/RISCVInterleavedAccess.cpp | 11 +-
llvm/lib/Transforms/Utils/LoopUtils.cpp | 86 +-
.../Vectorize/LoopVectorizationLegality.cpp | 79 +-
.../Vectorize/LoopVectorizationPlanner.cpp | 61 +-
.../Vectorize/LoopVectorizationPlanner.h | 3 +-
.../Transforms/Vectorize/LoopVectorize.cpp | 166 +-
.../Transforms/Vectorize/SLPVectorizer.cpp | 3 +
.../Transforms/Vectorize/VPRecipeBuilder.h | 2 +
llvm/lib/Transforms/Vectorize/VPlan.cpp | 37 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 62 +-
llvm/lib/Transforms/Vectorize/VPlanHelpers.h | 11 +
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 403 +++-
.../Transforms/Vectorize/VPlanTransforms.cpp | 13 +-
.../Transforms/Vectorize/VPlanTransforms.h | 3 +-
.../Analysis/CostModel/AArch64/masked_ldst.ll | 4 +-
.../Analysis/CostModel/AArch64/sve-arith.ll | 4 +-
.../Analysis/CostModel/AArch64/sve-cmpsel.ll | 2 +
.../Analysis/CostModel/AArch64/sve-fptoi.ll | 12 +-
.../Analysis/CostModel/AArch64/sve-ldst.ll | 4 +-
.../AArch64/sve-segment-gather-scatter.ll | 54 +
.../AArch64/sve-segmented-shuffle.ll | 164 ++
.../scalable_masked_deinterleaved_loads.ll | 24 +
.../scalable_masked_interleaved_stores.ll | 8 +-
llvm/test/CodeGen/AArch64/sve-int-arith.ll | 122 +-
.../CodeGen/AArch64/sve-masked-ldst-nonext.ll | 406 ++++
.../CodeGen/AArch64/sve-masked-ldst-sext.ll | 26 +
.../CodeGen/AArch64/sve-masked-ldst-trunc.ll | 12 +
.../sve-masked-segment-gather-scatter.ll | 175 ++
llvm/test/CodeGen/AArch64/sve-select.ll | 30 +
llvm/test/CodeGen/AArch64/sve-stepvector.ll | 67 +
.../CodeGen/AArch64/sve-vector-broadcast.ll | 324 +++
.../sve-vector-deinterleave-segments.ll | 346 +++
.../AArch64/sve-vector-interleave-segments.ll | 240 +++
llvm/test/CodeGen/AArch64/sve-vector-splat.ll | 14 +-
.../CodeGen/AArch64/sve-vector-stretch.ll | 136 ++
.../AArch64/sve2p1-partial-reduction-add.ll | 150 ++
.../sve2p1-partial-reduction-min-max.ll | 231 ++
.../AArch64/sve2p1-segmented-shuffles.ll | 1906 +++++++++++++++++
.../AArch64/sve2p2-segmented-shuffles.ll | 212 ++
.../scalable-deinterleave-intrinsics.ll | 113 +
.../AArch64/sve-deinterleave-segments.ll | 176 ++
.../AArch64/sve-interleave-segments.ll | 85 +
.../LoopVectorize/AArch64/invalid-costs.ll | 3 +-
.../LoopVectorize/AArch64/reduction-sdot.ll | 76 +
.../LoopVectorize/AArch64/reduction-udot.ll | 278 +++
.../LoopVectorize/AArch64/revec-bitcasts.ll | 202 ++
.../AArch64/revec-conversions.ll | 172 ++
.../AArch64/revec-gather-force-vf.ll | 54 +
.../LoopVectorize/AArch64/revec-gather.ll | 238 ++
.../LoopVectorize/AArch64/revec-ifconvert.ll | 215 ++
.../revec-interleaved-ldst-different-types.ll | 206 ++
.../AArch64/revec-interleaved-loads.ll | 441 ++++
.../AArch64/revec-interleaved-stores.ll | 360 ++++
.../LoopVectorize/AArch64/revec-inv-store.ll | 80 +
.../LoopVectorize/AArch64/revec-invalid.ll | 72 +
.../LoopVectorize/AArch64/revec-liveins.ll | 269 +++
.../AArch64/revec-mul-invariant-op.ll | 28 +
.../LoopVectorize/AArch64/revec-neon-64bit.ll | 1036 +++++++++
.../AArch64/revec-neon-extract-narrow.ll | 105 +
.../AArch64/revec-neon-force-vf.ll | 30 +
.../LoopVectorize/AArch64/revec-neon-mmla.ll | 74 +
.../AArch64/revec-neon-shr-narrow.ll | 180 ++
.../AArch64/revec-neon-usrhadd.ll | 60 +
.../LoopVectorize/AArch64/revec-neon.ll | 1033 +++++++++
.../AArch64/revec-nontemporal-load.ll | 28 +
.../AArch64/revec-partial-reduction.ll | 90 +
.../LoopVectorize/AArch64/revec-pixel-neon.ll | 159 ++
.../AArch64/revec-predication.ll | 227 ++
.../LoopVectorize/AArch64/revec-scatter.ll | 231 ++
.../LoopVectorize/AArch64/revec-select.ll | 197 ++
.../AArch64/revec-shufflevector-ic-vf.ll | 286 +++
.../AArch64/revec-shufflevector-ifconvert.ll | 28 +
.../revec-shufflevector-sve-version.ll | 117 +
.../AArch64/revec-shufflevector.ll | 370 ++++
.../LoopVectorize/AArch64/scalable-alloca.ll | 1 -
.../LoopVectorize/AArch64/scalable-call.ll | 9 +-
.../AArch64/type-shrinkage-zext-costs.ll | 4 +-
.../AArch64/vector-loop-64bit.ll | 134 ++
.../AArch64/vector-loop-vf4-uf1.ll | 300 +++
.../AArch64/vector-loop-vf4-uf2.ll | 334 +++
.../LoopVectorize/reduction-add-inloop.ll | 144 ++
.../Transforms/LoopVectorize/reduction-add.ll | 246 +++
.../LoopVectorize/reduction-fadd.ll | 266 +++
.../LoopVectorize/reduction-min-max.ll | 497 +++++
.../revec-scalable-first-order-recurrence.ll | 259 +++
...vector-pixel-neon-innermost-no-epilogue.ll | 107 +
.../masked-segment-gather-scatter-invalid.ll | 37 +
.../masked-segment-gather-scatter-valid.ll | 15 +
llvm/unittests/Analysis/IVDescriptorsTest.cpp | 103 +
.../Transforms/Vectorize/VPlanHCFGTest.cpp | 3 +-
136 files changed, 18102 insertions(+), 443 deletions(-)
create mode 100644 llvm/lib/Target/AArch64/AArch64VectorIntrinsicMappings.td
create mode 100644 llvm/test/Analysis/CostModel/AArch64/sve-segment-gather-scatter.ll
create mode 100644 llvm/test/Analysis/CostModel/AArch64/sve-segmented-shuffle.ll
create mode 100644 llvm/test/CodeGen/AArch64/sve-masked-segment-gather-scatter.ll
create mode 100644 llvm/test/CodeGen/AArch64/sve-vector-broadcast.ll
create mode 100644 llvm/test/CodeGen/AArch64/sve-vector-deinterleave-segments.ll
create mode 100644 llvm/test/CodeGen/AArch64/sve-vector-interleave-segments.ll
create mode 100644 llvm/test/CodeGen/AArch64/sve-vector-stretch.ll
create mode 100644 llvm/test/CodeGen/AArch64/sve2p1-partial-reduction-add.ll
create mode 100644 llvm/test/CodeGen/AArch64/sve2p1-partial-reduction-min-max.ll
create mode 100644 llvm/test/CodeGen/AArch64/sve2p1-segmented-shuffles.ll
create mode 100644 llvm/test/CodeGen/AArch64/sve2p2-segmented-shuffles.ll
create mode 100644 llvm/test/Transforms/InterleavedAccess/AArch64/sve-deinterleave-segments.ll
create mode 100644 llvm/test/Transforms/InterleavedAccess/AArch64/sve-interleave-segments.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/reduction-sdot.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/reduction-udot.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/revec-bitcasts.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/revec-conversions.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/revec-gather-force-vf.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/revec-gather.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/revec-ifconvert.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/revec-interleaved-ldst-different-types.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/revec-interleaved-loads.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/revec-interleaved-stores.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/revec-inv-store.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/revec-invalid.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/revec-liveins.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/revec-mul-invariant-op.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/revec-neon-64bit.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/revec-neon-extract-narrow.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/revec-neon-force-vf.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/revec-neon-mmla.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/revec-neon-shr-narrow.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/revec-neon-usrhadd.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/revec-neon.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/revec-nontemporal-load.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/revec-partial-reduction.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/revec-pixel-neon.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/revec-predication.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/revec-scatter.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/revec-select.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/revec-shufflevector-ic-vf.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/revec-shufflevector-ifconvert.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/revec-shufflevector-sve-version.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/revec-shufflevector.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/vector-loop-64bit.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/vector-loop-vf4-uf1.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/vector-loop-vf4-uf2.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/reduction-add-inloop.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/reduction-add.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/reduction-fadd.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/reduction-min-max.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/revec-scalable-first-order-recurrence.ll
create mode 100644 llvm/test/Transforms/PhaseOrdering/AArch64/vector-pixel-neon-innermost-no-epilogue.ll
create mode 100644 llvm/test/Verifier/masked-segment-gather-scatter-invalid.ll
create mode 100644 llvm/test/Verifier/masked-segment-gather-scatter-valid.ll
diff --git a/llvm/include/llvm/Analysis/IVDescriptors.h b/llvm/include/llvm/Analysis/IVDescriptors.h
index bad372421dfae..5a1c22d32ba41 100644
--- a/llvm/include/llvm/Analysis/IVDescriptors.h
+++ b/llvm/include/llvm/Analysis/IVDescriptors.h
@@ -30,6 +30,7 @@ class ScalarEvolution;
class SCEV;
class SCEVPredicate;
class StoreInst;
+class TargetTransformInfo;
/// These are the kinds of recurrences that we support.
enum class RecurKind {
@@ -68,6 +69,8 @@ enum class RecurKind {
FindLast, ///< FindLast reduction with select(cmp(),x,y) where x and y
///< are an integer type, one is the current recurrence value,
///< and the other is an arbitrary value.
+ TargetIntAccumulation, ///< A custom operation that accumulates integers in
+ ///< its first operand.
// clang-format on
// TODO: Any_of and FindLast reduction need not be restricted to integer type
// only.
@@ -154,10 +157,10 @@ class RecurrenceDescriptor {
/// advances the instruction pointer 'I' from the compare instruction to the
/// select instruction and stores this pointer in 'PatternLastInst' member of
/// the returned struct.
- LLVM_ABI static InstDesc isRecurrenceInstr(Loop *L, PHINode *Phi,
- Instruction *I, RecurKind Kind,
- InstDesc &Prev,
- ScalarEvolution *SE);
+ LLVM_ABI static InstDesc
+ isRecurrenceInstr(Loop *L, PHINode *Phi, Instruction *I, RecurKind Kind,
+ InstDesc &Prev, ScalarEvolution *SE,
+ const TargetTransformInfo *TTI = nullptr);
/// Returns true if instruction I has multiple uses in Insts
LLVM_ABI static bool hasMultipleUsesOf(Instruction *I,
@@ -201,7 +204,8 @@ class RecurrenceDescriptor {
AddReductionVar(PHINode *Phi, RecurKind Kind, Loop *TheLoop,
RecurrenceDescriptor &RedDes, DemandedBits *DB = nullptr,
AssumptionCache *AC = nullptr, DominatorTree *DT = nullptr,
- ScalarEvolution *SE = nullptr);
+ ScalarEvolution *SE = nullptr,
+ const TargetTransformInfo *TTI = nullptr);
/// Returns true if Phi is a reduction in TheLoop. The RecurrenceDescriptor
/// is returned in RedDes. If either \p DB is non-null or \p AC and \p DT are
@@ -211,7 +215,8 @@ class RecurrenceDescriptor {
LLVM_ABI static bool
isReductionPHI(PHINode *Phi, Loop *TheLoop, RecurrenceDescriptor &RedDes,
DemandedBits *DB = nullptr, AssumptionCache *AC = nullptr,
- DominatorTree *DT = nullptr, ScalarEvolution *SE = nullptr);
+ DominatorTree *DT = nullptr, ScalarEvolution *SE = nullptr,
+ const TargetTransformInfo *TTI = nullptr);
/// Returns true if Phi is a fixed-order recurrence. A fixed-order recurrence
/// is a non-reduction recurrence relation in which the value of the
@@ -299,6 +304,10 @@ class RecurrenceDescriptor {
return isFindLastRecurrenceKind(Kind) || isFindIVRecurrenceKind(Kind);
}
+ static bool isTargetRecurrenceKind(RecurKind Kind) {
+ return Kind == RecurKind::TargetIntAccumulation;
+ }
+
/// Returns the type of the recurrence. This type can be narrower than the
/// actual type of the Phi if the recurrence has been type-promoted.
Type *getRecurrenceType() const { return RecurrenceType; }
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 7d58473b81265..4ba7b2b029d24 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -53,6 +53,7 @@ class CondBrInst;
class Function;
class GlobalValue;
class InstCombiner;
+class IRBuilderBase;
class OptimizationRemarkEmitter;
class InterleavedAccessInfo;
class IntrinsicInst;
@@ -932,6 +933,13 @@ class TargetTransformInfo {
LLVM_ABI bool forceScalarizeMaskedScatter(VectorType *Type,
Align Alignment) const;
+ /// Return true if the target supports masked scatter of <vscale x SegmentTy>.
+ LLVM_ABI bool isLegalMaskedSegmentScatter(Type *DataType,
+ Align Alignment) const;
+ /// Return true if the target supports masked gather of <vscale x SegmentTy>.
+ LLVM_ABI bool isLegalMaskedSegmentGather(Type *DataType,
+ Align Alignment) const;
+
/// Return true if the target supports masked compress store.
LLVM_ABI bool isLegalMaskedCompressStore(Type *DataType,
Align Alignment) const;
@@ -1036,6 +1044,24 @@ class TargetTransformInfo {
/// convention.
LLVM_ABI bool useFastCCForInternalCall(Function &F) const;
+ /// Return true if the target intrinsic can be (re)vectorized.
+ /// The return type as well as all arguments are expected to be widened,
+ /// unless isTargetIntrinsicWithScalarOpAtArg() specifies otherwise.
+ LLVM_ABI bool isTargetIntrinsicVectorizable(Intrinsic::ID ID) const;
+
+ /// Return true if the intrinsic \p ID is supported for a \p RK
+ /// reduction pattern.
+ LLVM_ABI bool isSupportedTargetRecurrence(Intrinsic::ID ID,
+ RecurKind RK) const;
+
+ /// Return the (re)vectorized intrinsic call after the types and values have
+ /// been widened. This is only called for intrinsics that have passed
+ /// isTargetIntrinsicVectorizable() checks.
+ LLVM_ABI Instruction *
+ vectorizeTargetIntrinsic(Intrinsic::ID VectorIID, ArrayRef<Type *> TysForDecl,
+ ArrayRef<Value *> WideArgs, IRBuilderBase &Builder,
+ const Instruction &OrigInst) const;
+
/// Identifies if the vector form of the intrinsic has a scalar operand.
LLVM_ABI bool isTargetIntrinsicWithScalarOpAtArg(Intrinsic::ID ID,
unsigned ScalarOpdIdx) const;
@@ -1556,6 +1582,11 @@ class TargetTransformInfo {
VectorType *SubTp = nullptr, ArrayRef<const Value *> Args = {},
const Instruction *CxtI = nullptr) const;
+ /// \return the cost of a segmented shuffle, i.e. llvm.segmented.shuffle.
+ LLVM_ABI InstructionCost getSegmentedShuffleCost(
+ ShuffleKind Kind, VectorType *VTy, ArrayRef<int> Mask,
+ TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput) const;
+
/// Represents a hint about the context in which a cast is used.
///
/// For zext/sext, the context of the cast is the operand, which must be a
@@ -1711,6 +1742,14 @@ class TargetTransformInfo {
TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput,
bool UseMaskForCond = false, bool UseMaskForGaps = false) const;
+ /// Similar to \p getInterleavedMemoryOpCost, but for scalable types only,
+ /// where <MinEc x EltTy> segments are (de)interleaved instead of elements.
+ LLVM_ABI InstructionCost getSegmentInterleavedMemoryOpCost(
+ unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
+ Align Alignment, unsigned AddressSpace,
+ TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput,
+ bool UseMaskForCond = false, bool UseMaskForGaps = false) const;
+
/// A helper function to determine the type of reduction algorithm used
/// for a given \p Opcode and set of FastMathFlags \p FMF.
static bool requiresOrderedReduction(std::optional<FastMathFlags> FMF) {
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 89fd4a1e7628e..5dd6bd683effe 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -393,6 +393,16 @@ class LLVM_ABI TargetTransformInfoImplBase {
return false;
}
+ virtual bool isLegalMaskedSegmentGather(Type *DataType,
+ Align Alignment) const {
+ return false;
+ }
+
+ virtual bool isLegalMaskedSegmentScatter(Type *DataType,
+ Align Alignment) const {
+ return false;
+ }
+
virtual bool forceScalarizeMaskedGather(VectorType *DataType,
Align Alignment) const {
return false;
@@ -481,6 +491,22 @@ class LLVM_ABI TargetTransformInfoImplBase {
virtual bool useFastCCForInternalCall(Function &F) const { return true; }
+ virtual bool isTargetIntrinsicVectorizable(Intrinsic::ID ID) const {
+ return false;
+ }
+
+ virtual bool isSupportedTargetRecurrence(Intrinsic::ID ID,
+ RecurKind RK) const {
+ return false;
+ }
+
+ virtual Instruction *
+ vectorizeTargetIntrinsic(Intrinsic::ID VectorIID, ArrayRef<Type *> TysForDecl,
+ ArrayRef<Value *> WideArgs, IRBuilderBase &Builder,
+ const Instruction &OrigInst) const {
+ llvm_unreachable("Not implemented");
+ }
+
virtual bool isTargetIntrinsicWithScalarOpAtArg(Intrinsic::ID ID,
unsigned ScalarOpdIdx) const {
return false;
@@ -777,6 +803,13 @@ class LLVM_ABI TargetTransformInfoImplBase {
return 1;
}
+ virtual InstructionCost
+ getSegmentedShuffleCost(TTI::ShuffleKind Kind, VectorType *VTy,
+ ArrayRef<int> Mask,
+ TTI::TargetCostKind CostKind) const {
+ return InstructionCost::getInvalid();
+ }
+
virtual InstructionCost getCastInstrCost(unsigned Opcode, Type *Dst,
Type *Src, TTI::CastContextHint CCH,
TTI::TargetCostKind CostKind,
@@ -911,6 +944,13 @@ class LLVM_ABI TargetTransformInfoImplBase {
return 1;
}
+ virtual InstructionCost getSegmentInterleavedMemoryOpCost(
+ unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
+ Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
+ bool UseMaskForCond, bool UseMaskForGaps) const {
+ return InstructionCost::getInvalid();
+ }
+
virtual InstructionCost
getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
TTI::TargetCostKind CostKind) const {
diff --git a/llvm/include/llvm/Analysis/VectorUtils.h b/llvm/include/llvm/Analysis/VectorUtils.h
index 04c2041513aff..dcdc4a8ef5d1f 100644
--- a/llvm/include/llvm/Analysis/VectorUtils.h
+++ b/llvm/include/llvm/Analysis/VectorUtils.h
@@ -176,7 +176,8 @@ LLVM_ABI bool isVectorIntrinsicWithStructReturnOverloadAtField(
/// For the input call instruction it finds mapping intrinsic and returns
/// its intrinsic ID, in case it does not found it return not_intrinsic.
LLVM_ABI Intrinsic::ID
-getVectorIntrinsicIDForCall(const CallInst *CI, const TargetLibraryInfo *TLI);
+getVectorIntrinsicIDForCall(const CallInst *CI, const TargetLibraryInfo *TLI,
+ const TargetTransformInfo *TTI = nullptr);
/// Returns the corresponding factor of llvm.vector.interleaveN intrinsics.
LLVM_ABI unsigned getInterleaveIntrinsicFactor(Intrinsic::ID ID);
@@ -184,6 +185,12 @@ LLVM_ABI unsigned getInterleaveIntrinsicFactor(Intrinsic::ID ID);
/// Returns the corresponding factor of llvm.vector.deinterleaveN intrinsics.
LLVM_ABI unsigned getDeinterleaveIntrinsicFactor(Intrinsic::ID ID);
+/// Whether the intrinsic corresponds to llvm.vector.interleave.segmentsN.
+LLVM_ABI bool isInterleaveSegmentsIntrinsic(Intrinsic::ID ID);
+
+/// Whether the intrinsic corresponds to llvm.vector.deinterleave.segmentsN.
+LLVM_ABI bool isDeinterleaveSegmentsIntrinsic(Intrinsic::ID ID);
+
/// Given a deinterleaveN intrinsic, return the (narrow) vector type of each
/// factor.
LLVM_ABI VectorType *getDeinterleavedVectorType(IntrinsicInst *DI);
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index e4b6bf51c7a4e..4fa787564649f 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -1970,6 +1970,25 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
VarMask, Alignment, I),
CostKind);
}
+ case Intrinsic::masked_segment_scatter: {
+ const Value *Mask = Args[2];
+ bool VarMask = !isa<Constant>(Mask);
+ Align Alignment = I->getParamAlign(1).valueOrOne();
+ return thisT()->getMemIntrinsicInstrCost(
+ MemIntrinsicCostAttributes(Intrinsic::masked_segment_scatter,
+ ICA.getArgTypes()[0], Args[1], VarMask,
+ Alignment, I),
+ CostKind);
+ }
+ case Intrinsic::masked_segment_gather: {
+ const Value *Mask = Args[1];
+ bool VarMask = !isa<Constant>(Mask);
+ Align Alignment = I->getParamAlign(0).valueOrOne();
+ return thisT()->getMemIntrinsicInstrCost(
+ MemIntrinsicCostAttributes(Intrinsic::masked_segment_gather, RetTy,
+ Args[0], VarMask, Alignment, I),
+ CostKind);
+ }
case Intrinsic::masked_compressstore: {
const Value *Data = Args[0];
const Value *Mask = Args[2];
@@ -2075,6 +2094,17 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
IID, RetTy, {Args[0]->getType(), Args[1]->getType()}, FMF, I, 1);
return getTypeBasedIntrinsicInstrCost(Attrs, CostKind);
}
+ case Intrinsic::vector_segmented_shuffle: {
+ auto *VTy = cast<VectorType>(RetTy);
+ if (Args.size() == 3 && isa<Constant>(Args[2])) {
+ SmallVector<int, 8> Mask;
+ ShuffleVectorInst::getShuffleMask(cast<Constant>(Args[2]), Mask);
+ return thisT()->getSegmentedShuffleCost(
+ TargetTransformInfo::SK_PermuteTwoSrc, VTy, Mask, CostKind);
+ }
+ return thisT()->getSegmentedShuffleCost(
+ TargetTransformInfo::SK_PermuteTwoSrc, VTy, {}, CostKind);
+ }
case Intrinsic::fshl:
case Intrinsic::fshr: {
const Value *X = Args[0];
diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h
index 11099df775e38..dc59d060b7c8d 100644
--- a/llvm/include/llvm/CodeGen/ISDOpcodes.h
+++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h
@@ -623,6 +623,18 @@ enum NodeType {
/// Result[I][J] = CONCAT_VECTORS(...)[I + N * J]
VECTOR_DEINTERLEAVE,
+ /// VECTOR_DEINTERLEAVE_SEGMENTS(VEC1, VEC2, ...)
+ /// Returns N vectors from N input vectors, where N is the factor to
+ /// deinterleave. All input and output vectors must have the same type
+ /// and must be scalable.
+ ///
+ /// As opposed to VECTOR_DEINTERLEAVE, this variant deinterleaves fixed-size
+ /// segments (subvectors) instead of elements. The size of the segments
+ /// corresponds to the minimum number of elements of the inputs.
+ /// e.g. VECTOR_DEINTERLEAVE_SEGMENTS nxv4i32 Src1, nxv4i32 Src2
+ /// will deinterleave v4i32 segments.
+ VECTOR_DEINTERLEAVE_SEGMENTS,
+
/// VECTOR_INTERLEAVE(VEC1, VEC2, ...) - Returns N vectors from N input
/// vectors, where N is the factor to interleave. All input and
/// output vectors must have the same type.
@@ -634,6 +646,33 @@ enum NodeType {
/// Result[J] = EXTRACT_SUBVECTOR(Interleaved, J * getVectorMinNumElements())
VECTOR_INTERLEAVE,
+ /// VECTOR_INTERLEAVE_SEGMENTS(VEC1, VEC2, ...)
+ /// Returns N vectors from N input vectors, where N is the factor to
+ /// interleave. All input and output vectors must have the same type
+ /// and must be scalable.
+ ///
+ /// As opposed to VECTOR_INTERLEAVE, this variant interleaves fixed-size
+ /// segments (subvectors) instead of elements. The size of the segments
+ /// corresponds to the minimum number of elements of the inputs.
+ /// e.g. VECTOR_INTERLEAVE_SEGMENTS nxv4i32 Src1, nxv4i32 Src2
+ /// will interleave v4i32 segments.
+ VECTOR_INTERLEAVE_SEGMENTS,
+
+ /// VECTOR_BROADCAST(SRC_SUBVEC)
+ /// Duplicate a vector in a larger vector. The element count of the result
+ /// type is expected to be a multiple of the input vector's.
+ VECTOR_BROADCAST,
+
+ /// VECTOR_STRETCH(SRC_VEC)
+ /// Duplicate each element of SRC_VEC in place the same number of times until
+ /// matching the vector width of the output type. The width of the output type
+ /// is expected to be a compile-time-known multiple of the input vector.
+ /// The element type of the input and output types must also be the same.
+ ///
+ /// VECTOR_STRETCH(nxv2i1 {true, false}) -> nxv8i1
+ /// yields nxv8i1 {true, true, true, true, false, false, false, false}
+ VECTOR_STRETCH,
+
/// VECTOR_REVERSE(VECTOR) - Returns a vector, of the same type as VECTOR,
/// whose elements are shuffled using the following algorithm:
/// RESULT[i] = VECTOR[VECTOR.ElementCount - 1 - i]
@@ -648,6 +687,13 @@ enum NodeType {
/// in terms of the element size of VEC1/VEC2, not in terms of bytes.
VECTOR_SHUFFLE,
+ /// VECTOR_SEGMENTED_SHUFFLE(VEC1, VEC2) - Returns a vector of the same type
+ /// as VEC1/VEC2. The node also contains a mask, whose length must be a known
+ /// multiple of the vector length. That length determines the segment size.
+ /// VEC1 and VEC2 will be divided into multiple segments of that size, and
+ /// the mask will be applied on each of them to form the output segments.
+ VECTOR_SEGMENTED_SHUFFLE,
+
/// VECTOR_SPLICE_LEFT(VEC1, VEC2, OFFSET) - Shifts CONCAT_VECTORS(VEC1, VEC2)
/// left by OFFSET elements and returns the lower half.
VECTOR_SPLICE_LEFT,
@@ -1526,6 +1572,14 @@ enum NodeType {
VECREDUCE_UMAX,
VECREDUCE_UMIN,
+ // PARTIAL_REDUCE_xxxx(Accumulator, Source)
+ // Reduce a scalable vector to a fixed-length one and accumulate it.
+ PARTIAL_REDUCE_ADD,
+ PARTIAL_REDUCE_SMAX,
+ PARTIAL_REDUCE_SMIN,
+ PARTIAL_REDUCE_UMAX,
+ PARTIAL_REDUCE_UMIN,
+
/// PARTIAL_REDUCE_[U|S]MLA(Accumulator, Input1, Input2)
/// The partial reduction nodes sign or zero extend Input1 and Input2
/// (with the extension kind noted below) to the element type of
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index 8da46aa0e1411..c1847262e8da4 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -889,6 +889,14 @@ class SelectionDAG {
LLVM_ABI SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1,
SDValue N2, ArrayRef<int> Mask);
+ /// Return an ISD::VECTOR_SEGMENTED_SHUFFLE node.
+ /// The types of \p N1 and \p N2 must match VT, which is expected to be a
+ /// scalable vector. The length of \p Mask must be a known multiple of VT's
+ /// ElementCount.
+ LLVM_ABI SDValue getVectorSegmentedShuffle(EVT VT, const SDLoc &DL,
+ SDValue N1, SDValue N2,
+ ArrayRef<int> Mask);
+
/// Return an ISD::BUILD_VECTOR node. The number of elements in VT,
/// which must be a vector type, must match the number of operands in Ops.
/// The operands must have the same type as (or, for integers, a type wider
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
index cc7225c3c82cf..b38966fc5a1d9 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -1812,6 +1812,29 @@ class ShuffleVectorSDNode : public SDNode {
}
};
+/// This SDNode is used to implement the code generator
+/// support for llvm.vector.segmented.shuffle
+/// TODO-REVEC: Share some code with ShuffleVectorSDNode.
+class SegmentedShuffleVectorSDNode : public SDNode {
+ // The memory for Mask is owned by the SelectionDAG's OperandAllocator, and
+ // is freed when the SelectionDAG object is destroyed.
+ ArrayRef<int> Mask;
+
+protected:
+ friend class SelectionDAG;
+
+ SegmentedShuffleVectorSDNode(SDVTList VTs, unsigned Order, const DebugLoc &DL,
+ ArrayRef<int> M)
+ : SDNode(ISD::VECTOR_SEGMENTED_SHUFFLE, Order, DL, VTs), Mask(M) {}
+
+public:
+ ArrayRef<int> getMask() const { return Mask; }
+
+ static bool classof(const SDNode *N) {
+ return N->getOpcode() == ISD::VECTOR_SEGMENTED_SHUFFLE;
+ }
+};
+
class ConstantSDNode : public SDNode {
friend class SelectionDAG;
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 82c47cce0f522..31014ffd0e949 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -3344,9 +3344,12 @@ class LLVM_ABI TargetLoweringBase {
/// \p DI represents the deinterleaveN intrinsic.
/// \p GapMask is a mask with zeros for components / fields that may not be
/// accessed.
- virtual bool lowerDeinterleaveIntrinsicToLoad(Instruction *Load, Value *Mask,
- IntrinsicInst *DI,
- const APInt &GapMask) const {
+ /// \p DeinterleaveSegments Whether to deinterleave <MinEC x EltTy> segments
+ /// instead of elements.
+ virtual bool
+ lowerDeinterleaveIntrinsicToLoad(Instruction *Load, Value *Mask,
+ IntrinsicInst *DI, const APInt &GapMask,
+ bool DeinterleaveSegments) const {
return false;
}
@@ -3360,9 +3363,12 @@ class LLVM_ABI TargetLoweringBase {
/// component being interwoven) mask. Can be nullptr, in which case the
/// result is uncondiitional.
/// \p InterleaveValues contains the interleaved values.
+ /// \p InterleaveSegments Whether to interleave <MinEC x EltTy> segments
+ /// instead of elements.
virtual bool
lowerInterleaveIntrinsicToStore(Instruction *Store, Value *Mask,
- ArrayRef<Value *> InterleaveValues) const {
+ ArrayRef<Value *> InterleaveValues,
+ bool InterleaveSegments) const {
return false;
}
diff --git a/llvm/include/llvm/CodeGenTypes/MachineValueType.h b/llvm/include/llvm/CodeGenTypes/MachineValueType.h
index 92f3b79588996..46026396189f0 100644
--- a/llvm/include/llvm/CodeGenTypes/MachineValueType.h
+++ b/llvm/include/llvm/CodeGenTypes/MachineValueType.h
@@ -538,6 +538,10 @@ namespace llvm {
/// to a concrete value type.
LLVM_ABI static MVT getVT(Type *Ty, bool HandleUnknown = false);
+ /// Get a vector of equal size as \p VT which has \p p NewEC elements.
+ /// \pre VT's element count is a multiple of NewEC.
+ LLVM_ABI static MVT getSameSizeVT(MVT VT, ElementCount NewEC);
+
/// Returns an APFloat semantics tag appropriate for the value type. If this
/// is a vector type, the element semantics are returned.
LLVM_ABI const fltSemantics &getFltSemantics() const;
diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h
index 99e4b748425c0..cab131d3d84de 100644
--- a/llvm/include/llvm/IR/IRBuilder.h
+++ b/llvm/include/llvm/IR/IRBuilder.h
@@ -869,12 +869,13 @@ class IRBuilderBase {
LLVM_ABI CallInst *CreateMaskedGather(Type *Ty, Value *Ptrs, Align Alignment,
Value *Mask = nullptr,
Value *PassThru = nullptr,
- const Twine &Name = "");
+ const Twine &Name = "",
+ bool GatherSegments = false);
/// Create a call to Masked Scatter intrinsic
LLVM_ABI CallInst *CreateMaskedScatter(Value *Val, Value *Ptrs,
- Align Alignment,
- Value *Mask = nullptr);
+ Align Alignment, Value *Mask = nullptr,
+ bool ScatterSegments = false);
/// Create a call to Masked Expand Load intrinsic
LLVM_ABI CallInst *CreateMaskedExpandLoad(Type *Ty, Value *Ptr,
@@ -2678,8 +2679,21 @@ class IRBuilderBase {
return CreateShuffleVector(V, PoisonValue::get(V->getType()), Mask, Name);
}
+ Instruction *CreateSegmentedShuffleVector(Value *V1, Value *V2,
+ ArrayRef<int> Mask,
+ const Twine &Name = "") {
+ auto *VTy = cast<ScalableVectorType>(V1->getType());
+ auto *SegmentTy =
+ FixedVectorType::get(VTy->getElementType(), VTy->getMinNumElements());
+ Constant *MaskV =
+ ShuffleVectorInst::convertShuffleMaskForBitcode(Mask, SegmentTy);
+ return CreateIntrinsic(Intrinsic::vector_segmented_shuffle,
+ {VTy, MaskV->getType()}, {V1, V2, MaskV});
+ }
+
LLVM_ABI Value *CreateVectorInterleave(ArrayRef<Value *> Ops,
- const Twine &Name = "");
+ const Twine &Name = "",
+ bool InterleaveSegments = false);
Value *CreateExtractValue(Value *Agg, ArrayRef<unsigned> Idxs,
const Twine &Name = "") {
diff --git a/llvm/include/llvm/IR/Intrinsics.h b/llvm/include/llvm/IR/Intrinsics.h
index af97285fcee70..39b51d34671ca 100644
--- a/llvm/include/llvm/IR/Intrinsics.h
+++ b/llvm/include/llvm/IR/Intrinsics.h
@@ -294,11 +294,13 @@ LLVM_ABI bool isSignatureValid(Function *F,
LLVM_ABI std::optional<Function *> remangleIntrinsicFunction(Function *F);
/// Returns the corresponding llvm.vector.interleaveN intrinsic for factor N.
-LLVM_ABI Intrinsic::ID getInterleaveIntrinsicID(unsigned Factor);
+LLVM_ABI Intrinsic::ID
+getInterleaveIntrinsicID(unsigned Factor, bool InterleaveSegments = false);
/// Returns the corresponding llvm.vector.deinterleaveN intrinsic for factor
/// N.
-LLVM_ABI Intrinsic::ID getDeinterleaveIntrinsicID(unsigned Factor);
+LLVM_ABI Intrinsic::ID
+getDeinterleaveIntrinsicID(unsigned Factor, bool DeinterleaveSegments = false);
/// Print the argument info for the arguments with ArgInfo.
LLVM_ABI void printImmArg(ID IID, unsigned ArgIdx, raw_ostream &OS,
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index 0f9859d730182..1f364eaf1e12f 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -2667,6 +2667,16 @@ def int_masked_scatter:
LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>],
[IntrWriteMem]>;
+def int_masked_segment_gather:
+ DefaultAttrsIntrinsic<[llvm_anyvector_ty],
+ [llvm_anyvector_ty, LLVMType<nxv1i1>, LLVMMatchType<0>],
+ [IntrReadMem]>;
+
+def int_masked_segment_scatter:
+ DefaultAttrsIntrinsic<[],
+ [llvm_anyvector_ty, llvm_anyvector_ty, LLVMType<nxv1i1>],
+ [IntrWriteMem]>;
+
def int_masked_expandload:
DefaultAttrsIntrinsic<[llvm_anyvector_ty],
[llvm_ptr_ty, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
@@ -3011,6 +3021,33 @@ foreach n = 2...8 in {
[IntrNoMem, IntrSpeculatable]>;
}
+foreach n = {2, 4} in {
+ def int_vector_interleave_segments#n :
+ DefaultAttrsIntrinsic<[llvm_anyvector_ty],
+ !listsplat(LLVMOneNthElementsVectorType<0, n>, n),
+ [IntrNoMem]>;
+
+ def int_vector_deinterleave_segments#n :
+ DefaultAttrsIntrinsic<!listsplat(LLVMOneNthElementsVectorType<0, n>, n),
+ [llvm_anyvector_ty],
+ [IntrNoMem]>;
+}
+
+// vector_broadcast( SrcVector )
+// Broadcast a vector in a larger one.
+// This is the equivalent of a vector splat for vector types.
+def int_vector_broadcast : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
+ [llvm_anyvector_ty],
+ [IntrNoMem, IntrSpeculatable]>;
+
+// vector_stretch( SrcVector )
+// Duplicate each element of SrcVector in place the same number of times until
+// matching the vector width of the output type.
+// The width of the output type is expected to be a multiple of the input vector.
+def int_vector_stretch : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
+ [llvm_anyvector_ty],
+ [IntrNoMem, IntrSpeculatable]>;
+
//===-------------- Intrinsics to perform partial reduction ---------------===//
def int_vector_partial_reduce_add : DefaultAttrsIntrinsic<[LLVMMatchType<0>],
@@ -3023,6 +3060,32 @@ def int_vector_partial_reduce_fadd : DefaultAttrsIntrinsic<[LLVMMatchType<0>],
[llvm_anyfloat_ty, llvm_anyfloat_ty],
[IntrNoMem]>;
+def int_vector_partial_reduce_smax : DefaultAttrsIntrinsic<[LLVMMatchType<0>],
+ [llvm_anyvector_ty, llvm_anyvector_ty],
+ [IntrNoMem, IntrSpeculatable]>;
+def int_vector_partial_reduce_smin : DefaultAttrsIntrinsic<[LLVMMatchType<0>],
+ [llvm_anyvector_ty, llvm_anyvector_ty],
+ [IntrNoMem, IntrSpeculatable]>;
+def int_vector_partial_reduce_umax : DefaultAttrsIntrinsic<[LLVMMatchType<0>],
+ [llvm_anyvector_ty, llvm_anyvector_ty],
+ [IntrNoMem, IntrSpeculatable]>;
+def int_vector_partial_reduce_umin : DefaultAttrsIntrinsic<[LLVMMatchType<0>],
+ [llvm_anyvector_ty, llvm_anyvector_ty],
+ [IntrNoMem, IntrSpeculatable]>;
+
+//===--------------------- Segmented shufflevector -----------------------===//
+// This allows to re-vectorise shufflevector instructions.
+
+// vector_segmented_shuffle( Src0, Src1, Mask )
+// Arbitrary segmented shufflevector. The length of the segment is determined
+// by the length of the mask.
+def int_vector_segmented_shuffle :
+ DefaultAttrsIntrinsic<[llvm_anyvector_ty],
+ [LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyvector_ty],
+ [IntrNoMem, IntrSpeculatable]>;
+// TODO-REVEC: Add an IntrinsicProperty for constant vector operands.
+// TODO-REVEC: Enforce that the mask operand has i32 elements.
+
//===----------------- Pointer Authentication Intrinsics ------------------===//
//
diff --git a/llvm/include/llvm/IR/VectorTypeUtils.h b/llvm/include/llvm/IR/VectorTypeUtils.h
index e3d7fadad6089..6db83dd416a4e 100644
--- a/llvm/include/llvm/IR/VectorTypeUtils.h
+++ b/llvm/include/llvm/IR/VectorTypeUtils.h
@@ -20,6 +20,22 @@ namespace llvm {
inline Type *toVectorTy(Type *Scalar, ElementCount EC) {
if (Scalar->isVoidTy() || Scalar->isMetadataTy() || EC.isScalar())
return Scalar;
+
+ if (auto *VTy = dyn_cast<VectorType>(Scalar)) {
+ if (EC.isScalar())
+ return Scalar;
+
+ assert(!(VTy->getElementCount().isScalable() && EC.isScalable()) &&
+ "Attempt to create <vscale x vscale x N x elt>!");
+
+ if (auto *FVTy = dyn_cast<FixedVectorType>(VTy))
+ return VectorType::get(VTy->getElementType(),
+ EC * FVTy->getNumElements());
+
+ return VectorType::get(VTy->getElementType(),
+ VTy->getElementCount() * EC.getKnownMinValue());
+ }
+
return VectorType::get(Scalar, EC);
}
@@ -27,6 +43,14 @@ inline Type *toVectorTy(Type *Scalar, unsigned VF) {
return toVectorTy(Scalar, ElementCount::getFixed(VF));
}
+/// Returns the ElementCount if Ty is a vector type, and 1 otherwise.
+inline ElementCount getElementCount(Type *Ty) {
+ // TODO: Support vectorized structs?
+ if (auto *VTy = dyn_cast<VectorType>(Ty))
+ return VTy->getElementCount();
+ return ElementCount::getFixed(1);
+}
+
/// A helper for converting structs of scalar types to structs of vector types.
/// Note:
/// - If \p EC is scalar, \p StructTy is returned unchanged
diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td
index ea152bd130e9a..aa32b4b382896 100644
--- a/llvm/include/llvm/Target/TargetSelectionDAG.td
+++ b/llvm/include/llvm/Target/TargetSelectionDAG.td
@@ -911,6 +911,14 @@ def vector_insert_subvec : SDNode<"ISD::INSERT_SUBVECTOR",
def extract_subvector : SDNode<"ISD::EXTRACT_SUBVECTOR", SDTSubVecExtract, []>;
def insert_subvector : SDNode<"ISD::INSERT_SUBVECTOR", SDTSubVecInsert, []>;
+def vector_broadcast : SDNode<"ISD::VECTOR_BROADCAST",
+ SDTypeProfile<1, 1, [SDTCisVec<1>, SDTCisVec<0>]>,
+ []>;
+
+def vector_stretch : SDNode<"ISD::VECTOR_STRETCH",
+ SDTypeProfile<1, 1, [SDTCisSubVecOfVec<1, 0>]>,
+ []>;
+
def find_last_active
: SDNode<"ISD::VECTOR_FIND_LAST_ACTIVE",
SDTypeProfile<1, 1, [SDTCisInt<0>, SDTCisVec<1>]>, []>;
diff --git a/llvm/include/llvm/Transforms/Utils/LoopUtils.h b/llvm/include/llvm/Transforms/Utils/LoopUtils.h
index 9fa675856ba3f..6d534feab709c 100644
--- a/llvm/include/llvm/Transforms/Utils/LoopUtils.h
+++ b/llvm/include/llvm/Transforms/Utils/LoopUtils.h
@@ -471,7 +471,8 @@ LLVM_ABI bool canHoistLoad(LoadInst &LI, AAResults *AA, DominatorTree *DT,
/// Returns the llvm.vector.reduce intrinsic that corresponds to the recurrence
/// kind.
-LLVM_ABI constexpr Intrinsic::ID getReductionIntrinsicID(RecurKind RK);
+LLVM_ABI Intrinsic::ID getReductionIntrinsicID(RecurKind RK, Type *Ty);
+
/// Returns the llvm.vector.reduce min/max intrinsic that corresponds to the
/// intrinsic op.
LLVM_ABI Intrinsic::ID getMinMaxReductionIntrinsicID(Intrinsic::ID IID);
@@ -493,6 +494,9 @@ LLVM_ABI RecurKind getMinMaxReductionRecurKind(Intrinsic::ID RdxID);
/// Returns the comparison predicate used when expanding a min/max reduction.
LLVM_ABI CmpInst::Predicate getMinMaxReductionPredicate(RecurKind RK);
+/// Whether RdxID corresponds to a llvm.vector.partial.reduce.xxxx intrinsic.
+LLVM_ABI bool isPartialReductionIntrinsicID(Intrinsic::ID RdxID);
+
/// Given information about an @llvm.vector.reduce.* intrinsic, return
/// the identity value for the reduction.
LLVM_ABI Value *getReductionIdentity(Intrinsic::ID RdxID, Type *Ty,
@@ -534,7 +538,8 @@ LLVM_ABI Value *getShuffleReduction(IRBuilderBase &Builder, Value *Src,
/// additional information supplied in \p RdxKind.
/// Fast-math-flags are propagated using the IRBuilder's setting.
LLVM_ABI Value *createSimpleReduction(IRBuilderBase &B, Value *Src,
- RecurKind RdxKind);
+ RecurKind RdxKind,
+ Type *ReducedTy = nullptr);
/// Overloaded function to generate vector-predication intrinsics for
/// reduction.
LLVM_ABI Value *createSimpleReduction(IRBuilderBase &B, Value *Src,
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index 3b565f2a48ff8..abf754d6b2e86 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -48,6 +48,9 @@ class TargetLibraryInfo;
class TargetTransformInfo;
class Type;
+extern cl::opt<bool> VectorizeVectorLoops;
+extern cl::opt<bool> InterleaveRevecLoops;
+
/// Utility class for getting and setting loop vectorizer hints in the form
/// of loop metadata.
/// This class keeps a number of loop annotations locally (as member variables)
@@ -511,6 +514,8 @@ class LoopVectorizationLegality {
return CountableExitingBlocks;
}
+ bool LoopContainsVectors = false;
+
private:
/// Return true if the pre-header, exiting and latch blocks of \p Lp and all
/// its nested loops are considered legal for vectorization. These legal
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index 9d30928d1751c..7c62a572b7563 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -16,6 +16,7 @@
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
#include "llvm/Analysis/ScalarEvolutionPatternMatch.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Instructions.h"
@@ -56,6 +57,7 @@ bool RecurrenceDescriptor::isIntegerRecurrenceKind(RecurKind Kind) {
case RecurKind::AnyOf:
case RecurKind::FindIV:
case RecurKind::FindLast:
+ case RecurKind::TargetIntAccumulation:
return true;
}
return false;
@@ -263,10 +265,11 @@ hasRequiredFastMathFlags(FPMathOperator *FPOp, RecurKind &RK) {
static RecurrenceDescriptor getMinMaxRecurrence(PHINode *Phi, Loop *TheLoop,
ScalarEvolution *SE) {
Type *Ty = Phi->getType();
+ Type *ScalarTy = Ty->getScalarType();
BasicBlock *Latch = TheLoop->getLoopLatch();
if (Phi->getNumIncomingValues() != 2 ||
Phi->getParent() != TheLoop->getHeader() ||
- (!Ty->isIntegerTy() && !Ty->isFloatingPointTy()) || !Latch)
+ (!ScalarTy->isIntegerTy() && !ScalarTy->isFloatingPointTy()) || !Latch)
return {};
auto GetMinMaxRK = [](Value *V, Value *&A, Value *&B) -> RecurKind {
@@ -438,7 +441,7 @@ static bool isFindLastLikePhi(PHINode *Phi, PHINode *HeaderPhi,
bool RecurrenceDescriptor::AddReductionVar(
PHINode *Phi, RecurKind Kind, Loop *TheLoop, RecurrenceDescriptor &RedDes,
DemandedBits *DB, AssumptionCache *AC, DominatorTree *DT,
- ScalarEvolution *SE) {
+ ScalarEvolution *SE, const TargetTransformInfo *TTI) {
if (Phi->getNumIncomingValues() != 2)
return false;
@@ -604,8 +607,14 @@ bool RecurrenceDescriptor::AddReductionVar(
return false;
// Reductions of instructions such as Div, and Sub is only possible if the
- // LHS is the reduction variable.
- if (!Cur->isCommutative() && !IsAPhi && !isa<SelectInst>(Cur) &&
+ // LHS is the reduction variable. The only 3-input commutative instruction
+ // that is handled safely is fmuladd. Note that because it is a CallInst,
+ // it has an extra operand: the function address.
+ const bool IsSafelyCommutative =
+ Cur->isCommutative() &&
+ (!isa<IntrinsicInst>(Cur) || Cur->getNumOperands() == 3 ||
+ isFMulAddIntrinsic(Cur));
+ if (!IsSafelyCommutative && !IsAPhi && !isa<SelectInst>(Cur) &&
!isa<ICmpInst>(Cur) && !isa<FCmpInst>(Cur) &&
!VisitedInsts.count(dyn_cast<Instruction>(Cur->getOperand(0))))
return false;
@@ -614,7 +623,8 @@ bool RecurrenceDescriptor::AddReductionVar(
// the starting value (the Phi or an AND instruction if the Phi has been
// type-promoted).
if (Cur != Start) {
- ReduxDesc = isRecurrenceInstr(TheLoop, Phi, Cur, Kind, ReduxDesc, SE);
+ ReduxDesc =
+ isRecurrenceInstr(TheLoop, Phi, Cur, Kind, ReduxDesc, SE, TTI);
ExactFPMathInst = ExactFPMathInst == nullptr
? ReduxDesc.getExactFPMathInst()
: ExactFPMathInst;
@@ -984,10 +994,9 @@ RecurrenceDescriptor::isConditionalRdxPattern(Instruction *I) {
return InstDesc(true, I);
}
-RecurrenceDescriptor::InstDesc
-RecurrenceDescriptor::isRecurrenceInstr(Loop *L, PHINode *OrigPhi,
- Instruction *I, RecurKind Kind,
- InstDesc &Prev, ScalarEvolution *SE) {
+RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr(
+ Loop *L, PHINode *OrigPhi, Instruction *I, RecurKind Kind, InstDesc &Prev,
+ ScalarEvolution *SE, const TargetTransformInfo *TTI) {
assert(Prev.getRecKind() == RecurKind::None || Prev.getRecKind() == Kind);
switch (I->getOpcode()) {
default:
@@ -1037,6 +1046,10 @@ RecurrenceDescriptor::isRecurrenceInstr(Loop *L, PHINode *OrigPhi,
if (isFMulAddIntrinsic(I))
return InstDesc(Kind == RecurKind::FMulAdd, I,
I->hasAllowReassoc() ? nullptr : I);
+ if (auto *II = dyn_cast<IntrinsicInst>(I))
+ return InstDesc(
+ TTI && TTI->isSupportedTargetRecurrence(II->getIntrinsicID(), Kind),
+ I);
return InstDesc(false, I);
}
}
@@ -1059,7 +1072,8 @@ bool RecurrenceDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop,
RecurrenceDescriptor &RedDes,
DemandedBits *DB, AssumptionCache *AC,
DominatorTree *DT,
- ScalarEvolution *SE) {
+ ScalarEvolution *SE,
+ const TargetTransformInfo *TTI) {
if (AddReductionVar(Phi, RecurKind::Add, TheLoop, RedDes, DB, AC, DT, SE)) {
LLVM_DEBUG(dbgs() << "Found an ADD reduction PHI." << *Phi << "\n");
return true;
@@ -1133,6 +1147,13 @@ bool RecurrenceDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop,
return true;
}
+ if (AddReductionVar(Phi, RecurKind::TargetIntAccumulation, TheLoop, RedDes,
+ DB, AC, DT, SE, TTI)) {
+ LLVM_DEBUG(dbgs() << "Found a TargetIntAccumulation reduction PHI." << *Phi
+ << "\n");
+ return true;
+ }
+
// Not a reduction of known type.
return false;
}
@@ -1231,6 +1252,7 @@ unsigned RecurrenceDescriptor::getOpcode(RecurKind Kind) {
return Instruction::Sub;
case RecurKind::AddChainWithSubs:
case RecurKind::Add:
+ case RecurKind::TargetIntAccumulation:
return Instruction::Add;
case RecurKind::Mul:
return Instruction::Mul;
diff --git a/llvm/lib/Analysis/Loads.cpp b/llvm/lib/Analysis/Loads.cpp
index 1aed126079829..df128824a7799 100644
--- a/llvm/lib/Analysis/Loads.cpp
+++ b/llvm/lib/Analysis/Loads.cpp
@@ -312,6 +312,8 @@ bool llvm::isDereferenceableAndAlignedInLoop(
LoadInst *LI, Loop *L, ScalarEvolution &SE, DominatorTree &DT,
AssumptionCache *AC, SmallVectorImpl<const SCEVPredicate *> *Predicates) {
auto &DL = LI->getDataLayout();
+ if (!DL.getTypeStoreSize(LI->getType()).isFixed())
+ return false;
Value *Ptr = LI->getPointerOperand();
const SCEV *PtrSCEV = SE.getSCEV(Ptr);
APInt EltSize(DL.getIndexTypeSizeInBits(Ptr->getType()),
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 856950ccae595..e63e21076f1c2 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -523,6 +523,16 @@ bool TargetTransformInfo::isLegalMaskedScatter(Type *DataType,
return TTIImpl->isLegalMaskedScatter(DataType, Alignment);
}
+bool TargetTransformInfo::isLegalMaskedSegmentScatter(Type *DataType,
+ Align Alignment) const {
+ return TTIImpl->isLegalMaskedSegmentScatter(DataType, Alignment);
+}
+
+bool TargetTransformInfo::isLegalMaskedSegmentGather(Type *DataType,
+ Align Alignment) const {
+ return TTIImpl->isLegalMaskedSegmentGather(DataType, Alignment);
+}
+
bool TargetTransformInfo::forceScalarizeMaskedGather(VectorType *DataType,
Align Alignment) const {
return TTIImpl->forceScalarizeMaskedGather(DataType, Alignment);
@@ -629,6 +639,24 @@ bool TargetTransformInfo::useFastCCForInternalCall(Function &F) const {
return TTIImpl->useFastCCForInternalCall(F);
}
+bool TargetTransformInfo::isTargetIntrinsicVectorizable(
+ Intrinsic::ID ID) const {
+ return TTIImpl->isTargetIntrinsicVectorizable(ID);
+}
+
+bool TargetTransformInfo::isSupportedTargetRecurrence(Intrinsic::ID ID,
+ RecurKind RK) const {
+ return TTIImpl->isSupportedTargetRecurrence(ID, RK);
+}
+
+Instruction *TargetTransformInfo::vectorizeTargetIntrinsic(
+ Intrinsic::ID VectorIID, ArrayRef<Type *> TysForDecl,
+ ArrayRef<Value *> WideArgs, IRBuilderBase &Builder,
+ const Instruction &OrigInst) const {
+ return TTIImpl->vectorizeTargetIntrinsic(VectorIID, TysForDecl, WideArgs,
+ Builder, OrigInst);
+}
+
bool TargetTransformInfo::isTargetIntrinsicWithScalarOpAtArg(
Intrinsic::ID ID, unsigned ScalarOpdIdx) const {
return TTIImpl->isTargetIntrinsicWithScalarOpAtArg(ID, ScalarOpdIdx);
@@ -1049,6 +1077,16 @@ InstructionCost TargetTransformInfo::getShuffleCost(
return Cost;
}
+InstructionCost TargetTransformInfo::getSegmentedShuffleCost(
+ ShuffleKind Kind, VectorType *VTy, ArrayRef<int> Mask,
+ TTI::TargetCostKind CostKind) const {
+ assert(VTy->getElementCount().isKnownMultipleOf(Mask.size()));
+ InstructionCost Cost =
+ TTIImpl->getSegmentedShuffleCost(Kind, VTy, Mask, CostKind);
+ assert(Cost >= 0 && "TTI should not produce negative costs!");
+ return Cost;
+}
+
TargetTransformInfo::PartialReductionExtendKind
TargetTransformInfo::getPartialReductionExtendKind(Instruction *I) {
if (auto *Cast = dyn_cast<CastInst>(I))
@@ -1261,6 +1299,17 @@ InstructionCost TargetTransformInfo::getInterleavedMemoryOpCost(
return Cost;
}
+InstructionCost TargetTransformInfo::getSegmentInterleavedMemoryOpCost(
+ unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
+ Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
+ bool UseMaskForCond, bool UseMaskForGaps) const {
+ InstructionCost Cost = TTIImpl->getSegmentInterleavedMemoryOpCost(
+ Opcode, VecTy, Factor, Indices, Alignment, AddressSpace, CostKind,
+ UseMaskForCond, UseMaskForGaps);
+ assert(Cost >= 0 && "TTI should not produce negative costs!");
+ return Cost;
+}
+
InstructionCost
TargetTransformInfo::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
TTI::TargetCostKind CostKind) const {
diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp
index 0929f55ae68b2..6ab1ca219631a 100644
--- a/llvm/lib/Analysis/VectorUtils.cpp
+++ b/llvm/lib/Analysis/VectorUtils.cpp
@@ -233,12 +233,18 @@ bool llvm::isVectorIntrinsicWithStructReturnOverloadAtField(
/// Returns intrinsic ID for call.
/// For the input call instruction it finds mapping intrinsic and returns
/// its ID, in case it does not found it return not_intrinsic.
-Intrinsic::ID llvm::getVectorIntrinsicIDForCall(const CallInst *CI,
- const TargetLibraryInfo *TLI) {
+Intrinsic::ID
+llvm::getVectorIntrinsicIDForCall(const CallInst *CI,
+ const TargetLibraryInfo *TLI,
+ const TargetTransformInfo *TTI) {
Intrinsic::ID ID = getIntrinsicForCallSite(*CI, TLI);
if (ID == Intrinsic::not_intrinsic)
return Intrinsic::not_intrinsic;
+ if (TTI && Intrinsic::isTargetIntrinsic(ID) &&
+ TTI->isTargetIntrinsicVectorizable(ID))
+ return ID;
+
if (isTriviallyVectorizable(ID) || ID == Intrinsic::lifetime_start ||
ID == Intrinsic::lifetime_end || ID == Intrinsic::assume ||
ID == Intrinsic::experimental_noalias_scope_decl ||
@@ -250,10 +256,12 @@ Intrinsic::ID llvm::getVectorIntrinsicIDForCall(const CallInst *CI,
unsigned llvm::getInterleaveIntrinsicFactor(Intrinsic::ID ID) {
switch (ID) {
case Intrinsic::vector_interleave2:
+ case Intrinsic::vector_interleave_segments2:
return 2;
case Intrinsic::vector_interleave3:
return 3;
case Intrinsic::vector_interleave4:
+ case Intrinsic::vector_interleave_segments4:
return 4;
case Intrinsic::vector_interleave5:
return 5;
@@ -271,10 +279,12 @@ unsigned llvm::getInterleaveIntrinsicFactor(Intrinsic::ID ID) {
unsigned llvm::getDeinterleaveIntrinsicFactor(Intrinsic::ID ID) {
switch (ID) {
case Intrinsic::vector_deinterleave2:
+ case Intrinsic::vector_deinterleave_segments2:
return 2;
case Intrinsic::vector_deinterleave3:
return 3;
case Intrinsic::vector_deinterleave4:
+ case Intrinsic::vector_deinterleave_segments4:
return 4;
case Intrinsic::vector_deinterleave5:
return 5;
@@ -289,6 +299,26 @@ unsigned llvm::getDeinterleaveIntrinsicFactor(Intrinsic::ID ID) {
}
}
+bool llvm::isInterleaveSegmentsIntrinsic(Intrinsic::ID ID) {
+ switch (ID) {
+ case Intrinsic::vector_interleave_segments2:
+ case Intrinsic::vector_interleave_segments4:
+ return true;
+ default:
+ return false;
+ }
+}
+
+bool llvm::isDeinterleaveSegmentsIntrinsic(Intrinsic::ID ID) {
+ switch (ID) {
+ case Intrinsic::vector_deinterleave_segments2:
+ case Intrinsic::vector_deinterleave_segments4:
+ return true;
+ default:
+ return false;
+ }
+}
+
VectorType *llvm::getDeinterleavedVectorType(IntrinsicInst *DI) {
[[maybe_unused]] unsigned Factor =
getDeinterleaveIntrinsicFactor(DI->getIntrinsicID());
diff --git a/llvm/lib/CodeGen/InterleavedAccessPass.cpp b/llvm/lib/CodeGen/InterleavedAccessPass.cpp
index 5498ce1d81647..30fd0f9821a06 100644
--- a/llvm/lib/CodeGen/InterleavedAccessPass.cpp
+++ b/llvm/lib/CodeGen/InterleavedAccessPass.cpp
@@ -270,7 +270,8 @@ static Value *getMaskOperand(IntrinsicInst *II) {
// (2) Some mask effectively skips a certain field, and this element is a mask
// in which inactive lanes represent fields that are skipped (i.e. "gaps").
static std::pair<Value *, APInt> getMask(Value *WideMask, unsigned Factor,
- ElementCount LeafValueEC);
+ ElementCount LeafValueEC,
+ bool AccessSegments = false);
static std::pair<Value *, APInt> getMask(Value *WideMask, unsigned Factor,
VectorType *LeafValueTy) {
@@ -584,12 +585,13 @@ static void getGapMask(const Constant &MaskConst, unsigned Factor,
}
static std::pair<Value *, APInt> getMask(Value *WideMask, unsigned Factor,
- ElementCount LeafValueEC) {
+ ElementCount LeafValueEC,
+ bool AccessSegments) {
auto GapMask = APInt::getAllOnes(Factor);
if (auto *IMI = dyn_cast<IntrinsicInst>(WideMask)) {
if (unsigned F = getInterleaveIntrinsicFactor(IMI->getIntrinsicID());
- F && F == Factor) {
+ F && F == Factor && !AccessSegments) {
Value *RefArg = nullptr;
// Check if all the intrinsic arguments are the same, except those that
// are zeros, which we mark as gaps in the gap mask.
@@ -610,15 +612,22 @@ static std::pair<Value *, APInt> getMask(Value *WideMask, unsigned Factor,
// nullptr.
return {RefArg ? RefArg : IMI->getArgOperand(0), GapMask};
}
+
+ // Check if the wide mask is stretched from a narrow mask with the right EC.
+ if (IMI->getIntrinsicID() == Intrinsic::vector_stretch &&
+ cast<VectorType>(IMI->getArgOperand(0)->getType())->getElementCount() ==
+ LeafValueEC) {
+ return {IMI->getArgOperand(0), GapMask};
+ }
}
// Masks that are assembled from bitwise AND.
if (auto *AndOp = dyn_cast<BinaryOperator>(WideMask);
AndOp && AndOp->getOpcode() == Instruction::And) {
auto [MaskLHS, GapMaskLHS] =
- getMask(AndOp->getOperand(0), Factor, LeafValueEC);
+ getMask(AndOp->getOperand(0), Factor, LeafValueEC, AccessSegments);
auto [MaskRHS, GapMaskRHS] =
- getMask(AndOp->getOperand(1), Factor, LeafValueEC);
+ getMask(AndOp->getOperand(1), Factor, LeafValueEC, AccessSegments);
if (!MaskLHS || !MaskRHS)
return {nullptr, GapMask};
// Using IRBuilder here so that any trivial constants could be folded right
@@ -632,7 +641,7 @@ static std::pair<Value *, APInt> getMask(Value *WideMask, unsigned Factor,
// All-ones or all-zeros mask.
return {ConstantVector::getSplat(LeafValueEC, Splat), GapMask};
- if (LeafValueEC.isFixed()) {
+ if (LeafValueEC.isFixed() && !AccessSegments) {
unsigned LeafMaskLen = LeafValueEC.getFixedValue();
// First, check if we use a gap mask to skip some of the factors / fields.
getGapMask(*ConstMask, Factor, LeafMaskLen, GapMask);
@@ -654,6 +663,10 @@ static std::pair<Value *, APInt> getMask(Value *WideMask, unsigned Factor,
}
}
+ // Abort if we are (de)interleaving segments
+ if (AccessSegments)
+ return {nullptr, GapMask};
+
if (auto *SVI = dyn_cast<ShuffleVectorInst>(WideMask)) {
Type *Op1Ty = SVI->getOperand(1)->getType();
if (!isa<FixedVectorType>(Op1Ty))
@@ -696,6 +709,8 @@ bool InterleavedAccessImpl::lowerDeinterleaveIntrinsic(
const unsigned Factor = getDeinterleaveIntrinsicFactor(DI->getIntrinsicID());
assert(Factor && "unexpected deinterleave intrinsic");
+ bool DeinterleaveSegments =
+ isDeinterleaveSegmentsIntrinsic(DI->getIntrinsicID());
Value *Mask = nullptr;
auto GapMask = APInt::getAllOnes(Factor);
@@ -712,8 +727,13 @@ bool InterleavedAccessImpl::lowerDeinterleaveIntrinsic(
return false;
// Check mask operand. Handle both all-true/false and interleaved mask.
+ ElementCount NarrowMaskEC =
+ getDeinterleavedVectorType(DI)->getElementCount();
+ if (DeinterleaveSegments)
+ NarrowMaskEC =
+ NarrowMaskEC.divideCoefficientBy(NarrowMaskEC.getKnownMinValue());
std::tie(Mask, GapMask) =
- getMask(getMaskOperand(II), Factor, getDeinterleavedVectorType(DI));
+ getMask(getMaskOperand(II), Factor, NarrowMaskEC, DeinterleaveSegments);
if (!Mask)
return false;
@@ -725,7 +745,8 @@ bool InterleavedAccessImpl::lowerDeinterleaveIntrinsic(
}
// Try and match this with target specific intrinsics.
- if (!TLI->lowerDeinterleaveIntrinsicToLoad(LoadedVal, Mask, DI, GapMask))
+ if (!TLI->lowerDeinterleaveIntrinsicToLoad(LoadedVal, Mask, DI, GapMask,
+ DeinterleaveSegments))
return false;
DeadInsts.insert(DI);
@@ -747,6 +768,8 @@ bool InterleavedAccessImpl::lowerInterleaveIntrinsic(
SmallVector<Value *, 8> InterleaveValues(IntII->args());
const unsigned Factor = getInterleaveIntrinsicFactor(IntII->getIntrinsicID());
assert(Factor && "unexpected interleave intrinsic");
+ bool InterleaveSegments =
+ isInterleaveSegmentsIntrinsic(IntII->getIntrinsicID());
Value *Mask = nullptr;
if (II) {
@@ -755,9 +778,14 @@ bool InterleavedAccessImpl::lowerInterleaveIntrinsic(
return false;
// Check mask operand. Handle both all-true/false and interleaved mask.
APInt GapMask(Factor, 0);
+
+ ElementCount NarrowMaskEC =
+ cast<VectorType>(InterleaveValues[0]->getType())->getElementCount();
+ if (InterleaveSegments)
+ NarrowMaskEC =
+ NarrowMaskEC.divideCoefficientBy(NarrowMaskEC.getKnownMinValue());
std::tie(Mask, GapMask) =
- getMask(getMaskOperand(II), Factor,
- cast<VectorType>(InterleaveValues[0]->getType()));
+ getMask(getMaskOperand(II), Factor, NarrowMaskEC, InterleaveSegments);
if (!Mask)
return false;
// We haven't supported gap mask if it's interleaving using intrinsics. Yet
@@ -777,7 +805,8 @@ bool InterleavedAccessImpl::lowerInterleaveIntrinsic(
}
// Try and match this with target specific intrinsics.
- if (!TLI->lowerInterleaveIntrinsicToStore(StoredBy, Mask, InterleaveValues))
+ if (!TLI->lowerInterleaveIntrinsicToStore(StoredBy, Mask, InterleaveValues,
+ InterleaveSegments))
return false;
// We now have a target-specific store, so delete the old one.
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index dcd986d8fe047..bca567239d45a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -3768,7 +3768,30 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
Results.push_back(TLI.expandVectorSplice(Node, DAG));
break;
}
- case ISD::VECTOR_DEINTERLEAVE: {
+ case ISD::VECTOR_STRETCH: {
+ ElementCount InEC =
+ Node->getOperand(0).getValueType().getVectorElementCount();
+ EVT OutVT = Node->getValueType(0);
+ ElementCount OutEC = OutVT.getVectorElementCount();
+ unsigned StretchFactor = OutEC.getKnownScalarFactor(InEC);
+ if (StretchFactor <= 2 || !isPowerOf2_32(StretchFactor))
+ break;
+ // TODO-REVEC: If we are to upstream VECTOR_STRETCH, SDAG needs to handle
+ // legalisation for non power-of-2 types as well. We also need to ensure
+ // that the target supports VECTOR_STRETCH of factor 2, otherwise this
+ // transformation does not help.
+
+ // Expand one VECTOR_STRETCH into two by stretching to (InEC * 2) first.
+ EVT IntermediateVT = EVT::getVectorVT(
+ *DAG.getContext(), OutVT.getVectorElementType(), InEC * 2);
+ SDValue ToMidVT = DAG.getNode(ISD::VECTOR_STRETCH, dl, IntermediateVT,
+ Node->getOperand(0));
+ SDValue ToOutVT = DAG.getNode(ISD::VECTOR_STRETCH, dl, OutVT, ToMidVT);
+ Results.push_back(ToOutVT);
+ break;
+ }
+ case ISD::VECTOR_DEINTERLEAVE:
+ case ISD::VECTOR_DEINTERLEAVE_SEGMENTS: {
unsigned Factor = Node->getNumOperands();
if (Factor <= 2 || Factor % 2 != 0)
break;
@@ -3777,23 +3800,23 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
SmallVector<EVT> HalfVTs(Factor / 2, VecVT);
// Deinterleave at Factor/2 so each result contains two factors interleaved:
// a0b0 c0d0 a1b1 c1d1 -> [a0c0 b0d0] [a1c1 b1d1]
- SDValue L = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, dl, HalfVTs,
+ SDValue L = DAG.getNode(Node->getOpcode(), dl, HalfVTs,
ArrayRef(Ops).take_front(Factor / 2));
- SDValue R = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, dl, HalfVTs,
+ SDValue R = DAG.getNode(Node->getOpcode(), dl, HalfVTs,
ArrayRef(Ops).take_back(Factor / 2));
Results.resize(Factor);
// Deinterleave the 2 factors out:
// [a0c0 a1c1] [b0d0 b1d1] -> a0a1 b0b1 c0c1 d0d1
for (unsigned I = 0; I < Factor / 2; I++) {
- SDValue Deinterleave =
- DAG.getNode(ISD::VECTOR_DEINTERLEAVE, dl, {VecVT, VecVT},
- {L.getValue(I), R.getValue(I)});
+ SDValue Deinterleave = DAG.getNode(Node->getOpcode(), dl, {VecVT, VecVT},
+ {L.getValue(I), R.getValue(I)});
Results[I] = Deinterleave.getValue(0);
Results[I + Factor / 2] = Deinterleave.getValue(1);
}
break;
}
- case ISD::VECTOR_INTERLEAVE: {
+ case ISD::VECTOR_INTERLEAVE:
+ case ISD::VECTOR_INTERLEAVE_SEGMENTS: {
unsigned Factor = Node->getNumOperands();
if (Factor <= 2 || Factor % 2 != 0)
break;
@@ -3804,15 +3827,15 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
// a0a1 b0b1 c0c1 d0d1 -> [a0c0 b0d0] [a1c1 b1d1]
for (unsigned I = 0; I < Factor / 2; I++) {
SDValue Interleave =
- DAG.getNode(ISD::VECTOR_INTERLEAVE, dl, {VecVT, VecVT},
+ DAG.getNode(Node->getOpcode(), dl, {VecVT, VecVT},
{Node->getOperand(I), Node->getOperand(I + Factor / 2)});
LOps.push_back(Interleave.getValue(0));
ROps.push_back(Interleave.getValue(1));
}
// Interleave at Factor/2:
// [a0c0 b0d0] [a1c1 b1d1] -> a0b0 c0d0 a1b1 c1d1
- SDValue L = DAG.getNode(ISD::VECTOR_INTERLEAVE, dl, HalfVTs, LOps);
- SDValue R = DAG.getNode(ISD::VECTOR_INTERLEAVE, dl, HalfVTs, ROps);
+ SDValue L = DAG.getNode(Node->getOpcode(), dl, HalfVTs, LOps);
+ SDValue R = DAG.getNode(Node->getOpcode(), dl, HalfVTs, ROps);
for (unsigned I = 0; I < Factor / 2; I++)
Results.push_back(L.getValue(I));
for (unsigned I = 0; I < Factor / 2; I++)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index 86617b6dd33de..23d82e55c9d6a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -139,10 +139,18 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
case ISD::VECTOR_SPLICE_RIGHT:
Res = PromoteIntRes_VECTOR_SPLICE(N);
break;
+ case ISD::VECTOR_BROADCAST:
+ Res = PromoteIntRes_VECTOR_BROADCAST(N);
+ break;
case ISD::VECTOR_INTERLEAVE:
case ISD::VECTOR_DEINTERLEAVE:
+ case ISD::VECTOR_INTERLEAVE_SEGMENTS:
+ case ISD::VECTOR_DEINTERLEAVE_SEGMENTS:
Res = PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(N);
return;
+ case ISD::VECTOR_SEGMENTED_SHUFFLE:
+ Res = PromoteIntRes_VECTOR_SEGMENTED_SHUFFLE(N);
+ break;
case ISD::INSERT_VECTOR_ELT:
Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break;
case ISD::BUILD_VECTOR:
@@ -2211,6 +2219,16 @@ bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
case ISD::PARTIAL_REDUCE_SUMLA:
Res = PromoteIntOp_PARTIAL_REDUCE_MLA(N);
break;
+ case ISD::PARTIAL_REDUCE_ADD:
+ case ISD::PARTIAL_REDUCE_SMAX:
+ case ISD::PARTIAL_REDUCE_SMIN:
+ case ISD::PARTIAL_REDUCE_UMAX:
+ case ISD::PARTIAL_REDUCE_UMIN:
+ Res = PromoteIntOp_PARTIAL_REDUCE_ToFixed(N, OpNo);
+ break;
+ case ISD::VECTOR_BROADCAST:
+ Res = PromoteIntOp_VECTOR_BROADCAST(N);
+ break;
case ISD::LOOP_DEPENDENCE_RAW_MASK:
case ISD::LOOP_DEPENDENCE_WAR_MASK:
Res = PromoteIntOp_LOOP_DEPENDENCE_MASK(N);
@@ -3098,6 +3116,54 @@ SDValue DAGTypeLegalizer::PromoteIntOp_LOOP_DEPENDENCE_MASK(SDNode *N) {
return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
}
+SDValue DAGTypeLegalizer::PromoteIntOp_VECTOR_BROADCAST(SDNode *N) {
+ SDLoc DL(N);
+ SDValue Src = GetPromotedInteger(N->getOperand(0));
+ EVT SrcVT = Src.getValueType();
+ EVT OrigVT = N->getValueType(0);
+ EVT NewVT = EVT::getVectorVT(*DAG.getContext(), SrcVT.getVectorElementType(),
+ OrigVT.getVectorElementCount());
+ SDValue Res = DAG.getNode(ISD::VECTOR_BROADCAST, DL, NewVT, Src);
+ return DAG.getNode(ISD::TRUNCATE, DL, OrigVT, Res);
+}
+
+SDValue DAGTypeLegalizer::PromoteIntOp_PARTIAL_REDUCE_ToFixed(SDNode *N,
+ unsigned OpNo) {
+ assert(OpNo == 1 && "Only expected to widen the source vector");
+ SDLoc DL(N);
+ SDValue Src;
+ unsigned ExtOpc;
+ switch (N->getOpcode()) {
+ case ISD::PARTIAL_REDUCE_ADD:
+ Src = GetPromotedInteger(N->getOperand(1));
+ ExtOpc = ISD::ANY_EXTEND;
+ break;
+ case ISD::PARTIAL_REDUCE_SMAX:
+ case ISD::PARTIAL_REDUCE_SMIN:
+ Src = SExtPromotedInteger(N->getOperand(1));
+ ExtOpc = ISD::SIGN_EXTEND;
+ break;
+ case ISD::PARTIAL_REDUCE_UMAX:
+ case ISD::PARTIAL_REDUCE_UMIN:
+ Src = ZExtPromotedInteger(N->getOperand(1));
+ ExtOpc = ISD::ZERO_EXTEND;
+ break;
+ default:
+ llvm_unreachable("Unexpected partial recution opcode.");
+ }
+ EVT SrcVT = Src.getValueType();
+ EVT VT = EVT::getVectorVT(*DAG.getContext(), SrcVT.getVectorElementType(),
+ SrcVT.getVectorMinNumElements());
+ SDValue Acc;
+ if (ISD::NodeType BaseOpc = ISD::getVecReduceBaseOpcode(N->getOpcode());
+ DAG.isIdentityElement(BaseOpc, SDNodeFlags(), N->getOperand(0), 0))
+ Acc = DAG.getIdentityElement(BaseOpc, DL, VT, SDNodeFlags());
+ else
+ Acc = DAG.getNode(ExtOpc, DL, VT, N->getOperand(0));
+ SDValue Res = DAG.getNode(N->getOpcode(), DL, VT, Acc, Src);
+ return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), Res);
+}
+
//===----------------------------------------------------------------------===//
// Integer Result Expansion
//===----------------------------------------------------------------------===//
@@ -6148,6 +6214,19 @@ SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SPLICE(SDNode *N) {
return DAG.getNode(N->getOpcode(), dl, OutVT, V0, V1, N->getOperand(2));
}
+SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_BROADCAST(SDNode *N) {
+ SDLoc DL(N);
+
+ EVT OutVT = N->getValueType(0);
+ EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
+ assert(NOutVT.isVector() && "This type must be promoted to a vector type");
+ EVT NInVT = N->getOperand(0).getValueType().changeVectorElementType(
+ *DAG.getContext(), NOutVT.getVectorElementType());
+
+ SDValue Op = DAG.getNode(ISD::ANY_EXTEND, DL, NInVT, N->getOperand(0));
+ return DAG.getNode(N->getOpcode(), DL, NOutVT, Op);
+}
+
SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(SDNode *N) {
SDLoc DL(N);
unsigned Factor = N->getNumOperands();
@@ -6165,6 +6244,15 @@ SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(SDNode *N
return SDValue();
}
+SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SEGMENTED_SHUFFLE(SDNode *N) {
+ SDLoc DL(N);
+ auto *SSV = cast<SegmentedShuffleVectorSDNode>(N);
+ SDValue V0 = GetPromotedInteger(N->getOperand(0));
+ SDValue V1 = GetPromotedInteger(N->getOperand(1));
+ EVT OutVT = V0.getValueType();
+ return DAG.getVectorSegmentedShuffle(OutVT, DL, V0, V1, SSV->getMask());
+}
+
SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) {
EVT OutVT = N->getValueType(0);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index 450eba435cc0b..be2cd96d6f650 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -292,7 +292,9 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue PromoteIntRes_VECTOR_REVERSE(SDNode *N);
SDValue PromoteIntRes_VECTOR_SHUFFLE(SDNode *N);
SDValue PromoteIntRes_VECTOR_SPLICE(SDNode *N);
+ SDValue PromoteIntRes_VECTOR_BROADCAST(SDNode *N);
SDValue PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(SDNode *N);
+ SDValue PromoteIntRes_VECTOR_SEGMENTED_SHUFFLE(SDNode *N);
SDValue PromoteIntRes_BUILD_VECTOR(SDNode *N);
SDValue PromoteIntRes_ScalarOp(SDNode *N);
SDValue PromoteIntRes_STEP_VECTOR(SDNode *N);
@@ -423,6 +425,8 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue PromoteIntOp_UnaryBooleanVectorOp(SDNode *N, unsigned OpNo);
SDValue PromoteIntOp_GET_ACTIVE_LANE_MASK(SDNode *N);
SDValue PromoteIntOp_PARTIAL_REDUCE_MLA(SDNode *N);
+ SDValue PromoteIntOp_PARTIAL_REDUCE_ToFixed(SDNode *N, unsigned OpNo);
+ SDValue PromoteIntOp_VECTOR_BROADCAST(SDNode *N);
SDValue PromoteIntOp_LOOP_DEPENDENCE_MASK(SDNode *N);
SDValue PromoteIntOp_MaskedBinOp(SDNode *N, unsigned OpNo);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 806d1b2d2fc92..6db1f3bfadc27 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -1403,9 +1403,11 @@ void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
SplitVecRes_VECTOR_SPLICE(N, Lo, Hi);
break;
case ISD::VECTOR_DEINTERLEAVE:
+ case ISD::VECTOR_DEINTERLEAVE_SEGMENTS:
SplitVecRes_VECTOR_DEINTERLEAVE(N);
return;
case ISD::VECTOR_INTERLEAVE:
+ case ISD::VECTOR_INTERLEAVE_SEGMENTS:
SplitVecRes_VECTOR_INTERLEAVE(N);
return;
case ISD::VAARG:
@@ -3699,6 +3701,8 @@ void DAGTypeLegalizer::SplitVecRes_GET_ACTIVE_LANE_MASK(SDNode *N, SDValue &Lo,
}
void DAGTypeLegalizer::SplitVecRes_VECTOR_DEINTERLEAVE(SDNode *N) {
+ assert(N->getOpcode() == ISD::VECTOR_DEINTERLEAVE ||
+ N->getOpcode() == ISD::VECTOR_DEINTERLEAVE_SEGMENTS);
unsigned Factor = N->getNumOperands();
SmallVector<SDValue, 8> Ops(Factor * 2);
@@ -3712,16 +3716,18 @@ void DAGTypeLegalizer::SplitVecRes_VECTOR_DEINTERLEAVE(SDNode *N) {
SmallVector<EVT, 8> VTs(Factor, Ops[0].getValueType());
SDLoc DL(N);
- SDValue ResLo = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL, VTs,
- ArrayRef(Ops).slice(0, Factor));
- SDValue ResHi = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL, VTs,
- ArrayRef(Ops).slice(Factor, Factor));
+ SDValue ResLo =
+ DAG.getNode(N->getOpcode(), DL, VTs, ArrayRef(Ops).slice(0, Factor));
+ SDValue ResHi =
+ DAG.getNode(N->getOpcode(), DL, VTs, ArrayRef(Ops).slice(Factor, Factor));
for (unsigned i = 0; i != Factor; ++i)
SetSplitVector(SDValue(N, i), ResLo.getValue(i), ResHi.getValue(i));
}
void DAGTypeLegalizer::SplitVecRes_VECTOR_INTERLEAVE(SDNode *N) {
+ assert(N->getOpcode() == ISD::VECTOR_INTERLEAVE ||
+ N->getOpcode() == ISD::VECTOR_INTERLEAVE_SEGMENTS);
unsigned Factor = N->getNumOperands();
SmallVector<SDValue, 8> Ops(Factor * 2);
@@ -3735,10 +3741,10 @@ void DAGTypeLegalizer::SplitVecRes_VECTOR_INTERLEAVE(SDNode *N) {
SmallVector<EVT, 8> VTs(Factor, Ops[0].getValueType());
SDLoc DL(N);
- SDValue Res[] = {DAG.getNode(ISD::VECTOR_INTERLEAVE, DL, VTs,
- ArrayRef(Ops).slice(0, Factor)),
- DAG.getNode(ISD::VECTOR_INTERLEAVE, DL, VTs,
- ArrayRef(Ops).slice(Factor, Factor))};
+ SDValue Res[] = {
+ DAG.getNode(N->getOpcode(), DL, VTs, ArrayRef(Ops).slice(0, Factor)),
+ DAG.getNode(N->getOpcode(), DL, VTs,
+ ArrayRef(Ops).slice(Factor, Factor))};
for (unsigned i = 0; i != Factor; ++i) {
unsigned IdxLo = 2 * i;
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 75d550801315b..a3301157faf55 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -465,6 +465,7 @@ ISD::NodeType ISD::getVecReduceBaseOpcode(unsigned VecReduceOpcode) {
return ISD::FMUL;
case ISD::VECREDUCE_ADD:
case ISD::VP_REDUCE_ADD:
+ case ISD::PARTIAL_REDUCE_ADD:
return ISD::ADD;
case ISD::VECREDUCE_MUL:
case ISD::VP_REDUCE_MUL:
@@ -480,15 +481,19 @@ ISD::NodeType ISD::getVecReduceBaseOpcode(unsigned VecReduceOpcode) {
return ISD::XOR;
case ISD::VECREDUCE_SMAX:
case ISD::VP_REDUCE_SMAX:
+ case ISD::PARTIAL_REDUCE_SMAX:
return ISD::SMAX;
case ISD::VECREDUCE_SMIN:
case ISD::VP_REDUCE_SMIN:
+ case ISD::PARTIAL_REDUCE_SMIN:
return ISD::SMIN;
case ISD::VECREDUCE_UMAX:
case ISD::VP_REDUCE_UMAX:
+ case ISD::PARTIAL_REDUCE_UMAX:
return ISD::UMAX;
case ISD::VECREDUCE_UMIN:
case ISD::VP_REDUCE_UMIN:
+ case ISD::PARTIAL_REDUCE_UMIN:
return ISD::UMIN;
case ISD::VECREDUCE_FMAX:
case ISD::VP_REDUCE_FMAX:
@@ -2421,6 +2426,47 @@ SDValue SelectionDAG::getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1,
return V;
}
+SDValue SelectionDAG::getVectorSegmentedShuffle(EVT VT, const SDLoc &DL,
+ SDValue N1, SDValue N2,
+ ArrayRef<int> Mask) {
+ assert(VT == N1.getValueType() && VT == N2.getValueType() &&
+ "Invalid VECTOR_SHUFFLE");
+ const unsigned SegmentSize = Mask.size();
+ assert(VT.getVectorElementCount().isKnownMultipleOf(SegmentSize) &&
+ "VECTOR_SEGMENTED_SHUFFLE's element count is not a multiple of the "
+ "mask length!");
+
+ // TODO-REVEC: Some canonicalisation like in getVectorShuffle().
+
+ SDVTList VTs = getVTList(VT);
+ FoldingSetNodeID ID;
+ SDValue Ops[2] = {N1, N2};
+ AddNodeIDNode(ID, ISD::VECTOR_SEGMENTED_SHUFFLE, VTs, Ops);
+ for (unsigned Idx = 0; Idx != SegmentSize; ++Idx)
+ ID.AddInteger(Mask[Idx]);
+
+ void *IP = nullptr;
+ if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
+ return SDValue(E, 0);
+
+ // Allocate the mask array for the node out of the BumpPtrAllocator, since
+ // SDNode doesn't have access to it. This memory will be "leaked" when
+ // the node is deallocated, but recovered when the NodeAllocator is released.
+ int *MaskAlloc = OperandAllocator.Allocate<int>(SegmentSize);
+ llvm::copy(Mask, MaskAlloc);
+
+ auto *N = newSDNode<SegmentedShuffleVectorSDNode>(
+ VTs, DL.getIROrder(), DL.getDebugLoc(),
+ ArrayRef<int>(MaskAlloc, SegmentSize));
+ createOperands(N, Ops);
+
+ CSEMap.InsertNode(N, IP);
+ InsertNode(N);
+ SDValue V = SDValue(N, 0);
+ NewSDValueDbgMsg(V, "Creating new node: ", this);
+ return V;
+}
+
SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) {
EVT VT = SV.getValueType(0);
SmallVector<int, 8> MaskVec(SV.getMask());
@@ -8958,6 +9004,21 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
}
break;
}
+ case ISD::VECTOR_BROADCAST:
+ case ISD::VECTOR_STRETCH: {
+ [[maybe_unused]] EVT InputVT = N1.getValueType();
+ assert(InputVT.isVector() && VT.isVector() &&
+ "Expected the input and output of the "
+ "VECTOR_BROADCAST/VECTOR_STRETCH node to be "
+ "vectors!");
+ assert(VT.getVectorElementCount().hasKnownScalarFactor(
+ InputVT.getVectorElementCount()) &&
+ "Expected the element count of the output of the "
+ "VECTOR_BROADCAST/VECTOR_STRETCH "
+ "node to be a positive integer multiple of the element count of the "
+ "source operand!");
+ break;
+ }
case ISD::BITCAST:
// Fold bit_convert nodes from a type to themselves.
if (N1.getValueType() == VT)
@@ -8986,6 +9047,24 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
break;
}
+ case ISD::PARTIAL_REDUCE_ADD:
+ case ISD::PARTIAL_REDUCE_SMAX:
+ case ISD::PARTIAL_REDUCE_SMIN:
+ case ISD::PARTIAL_REDUCE_UMAX:
+ case ISD::PARTIAL_REDUCE_UMIN: {
+ [[maybe_unused]] EVT AccVT = N1.getValueType();
+ [[maybe_unused]] EVT InputVT = N2.getValueType();
+ assert(InputVT.isVector() &&
+ "Expected the second operand of the PARTIAL_REDUCE_xxxx "
+ "node to be of vector type!");
+ assert(VT.isVector() && VT == AccVT &&
+ "Expected the first operand of the PARTIAL_REDUCE_xxxx node to "
+ "have the same type as its result!");
+ assert(InputVT.getVectorElementType() == AccVT.getVectorElementType() &&
+ "Expected the element type of the input and accumulator operands of "
+ "the PARTIAL_REDUCE_xxxx node to be identical!");
+ break;
+ }
case ISD::PARTIAL_REDUCE_UMLA:
case ISD::PARTIAL_REDUCE_SMLA:
case ISD::PARTIAL_REDUCE_SUMLA:
@@ -14968,9 +15047,11 @@ SDValue SelectionDAG::getIdentityElement(unsigned Opcode, const SDLoc &DL,
case ISD::UMIN:
return getAllOnesConstant(DL, VT);
case ISD::SMAX:
- return getConstant(APInt::getSignedMinValue(VT.getSizeInBits()), DL, VT);
+ return getConstant(APInt::getSignedMinValue(VT.getScalarSizeInBits()), DL,
+ VT);
case ISD::SMIN:
- return getConstant(APInt::getSignedMaxValue(VT.getSizeInBits()), DL, VT);
+ return getConstant(APInt::getSignedMaxValue(VT.getScalarSizeInBits()), DL,
+ VT);
case ISD::FADD:
// If flags allow, prefer positive zero since it's generally cheaper
// to materialize on most targets.
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index eca5bb1598ae0..a25b3cd0ba636 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -4359,6 +4359,25 @@ void SelectionDAGBuilder::visitShuffleVector(const User &I) {
setValue(&I, DAG.getBuildVector(VT, DL, Ops));
}
+void SelectionDAGBuilder::visitVectorSegmentedShuffle(const CallInst &I) {
+ SDValue Src1 = getValue(I.getOperand(0));
+ SDValue Src2 = getValue(I.getOperand(1));
+
+ assert(isa<Constant>(I.getOperand(2)) &&
+ "vector.segmented.shuffle must have a constant mask.");
+ auto *MaskOp = cast<Constant>(I.getOperand(2));
+ SmallVector<int, 16> Mask;
+ ShuffleVectorInst::getShuffleMask(MaskOp, Mask);
+
+ SDLoc DL = getCurSDLoc();
+ const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+ EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
+ assert(VT == Src1.getValueType());
+
+ setValue(&I, DAG.getVectorSegmentedShuffle(VT, DL, Src1, Src2, Mask));
+ return;
+}
+
void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {
ArrayRef<unsigned> Indices = I.getIndices();
const Value *Op0 = I.getOperand(0);
@@ -5092,6 +5111,69 @@ void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
setValue(&I, Scatter);
}
+void SelectionDAGBuilder::visitMaskedSegmentGather(const CallInst &I) {
+ SDLoc DL = getCurSDLoc();
+
+ // @llvm.masked.segment.gather.*(Ptrs, Mask, Src0)
+ const Value *Ptr = I.getArgOperand(0);
+ SDValue Ptrs = getValue(Ptr);
+ SDValue Mask = getValue(I.getArgOperand(1));
+ SDValue Src0 = getValue(I.getArgOperand(2));
+
+ const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+ EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
+ Align Alignment = I.getParamAlign(0).valueOrOne();
+ unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
+ const MDNode *Ranges = getRangeMetadata(I);
+
+ MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
+ MachinePointerInfo(AS), MachineMemOperand::MOLoad,
+ LocationSize::beforeOrAfterPointer(), Alignment, I.getAAMetadata(),
+ Ranges);
+
+ SDValue Ops[] = {
+ DAG.getRoot(),
+ DAG.getTargetConstant(Intrinsic::masked_segment_gather, DL,
+ TLI.getPointerTy(DAG.getDataLayout())),
+ Ptrs, Mask, Src0};
+ SDValue Gather = DAG.getMemIntrinsicNode(
+ ISD::INTRINSIC_W_CHAIN, DL, DAG.getVTList(VT, MVT::Other), Ops, VT,
+ MMO);
+
+ PendingLoads.push_back(Gather.getValue(1));
+ setValue(&I, Gather);
+}
+
+void SelectionDAGBuilder::visitMaskedSegmentScatter(const CallInst &I) {
+ SDLoc DL = getCurSDLoc();
+
+ // @llvm.masked.segment.scatter.*(Data, Ptrs, Mask)
+ SDValue Data = getValue(I.getArgOperand(0));
+ const Value *Ptr = I.getArgOperand(1);
+ SDValue Ptrs = getValue(Ptr);
+ SDValue Mask = getValue(I.getArgOperand(2));
+
+ const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+ EVT VT = Data.getValueType();
+ Align Alignment = I.getParamAlign(1).valueOrOne();
+ unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
+
+ MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
+ MachinePointerInfo(AS), MachineMemOperand::MOStore,
+ LocationSize::beforeOrAfterPointer(), Alignment, I.getAAMetadata());
+
+ SDValue Ops[] = {
+ getMemoryRoot(),
+ DAG.getTargetConstant(Intrinsic::masked_segment_scatter, DL,
+ TLI.getPointerTy(DAG.getDataLayout())),
+ Data, Ptrs, Mask};
+ SDValue Scatter = DAG.getMemIntrinsicNode(
+ ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Ops, VT, MMO);
+
+ DAG.setRoot(Scatter);
+ setValue(&I, Scatter);
+}
+
void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
SDLoc sdl = getCurSDLoc();
@@ -6919,12 +7001,18 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
case Intrinsic::masked_gather:
visitMaskedGather(I);
return;
+ case Intrinsic::masked_segment_gather:
+ visitMaskedSegmentGather(I);
+ return;
case Intrinsic::masked_load:
visitMaskedLoad(I);
return;
case Intrinsic::masked_scatter:
visitMaskedScatter(I);
return;
+ case Intrinsic::masked_segment_scatter:
+ visitMaskedSegmentScatter(I);
+ return;
case Intrinsic::masked_store:
visitMaskedStore(I);
return;
@@ -8340,6 +8428,12 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
return;
}
case Intrinsic::vector_partial_reduce_add: {
+
+ // Use PARTIAL_REDUCE_ADD when going from scalable to fixed.
+ // Otherwise, use PARTIAL_REDUCE_UMLA to allow further combining.
+ if (visitPartialReduceToFixed(I, ISD::PARTIAL_REDUCE_ADD))
+ return;
+
SDValue Acc = getValue(I.getOperand(0));
SDValue Input = getValue(I.getOperand(1));
setValue(&I,
@@ -8355,6 +8449,18 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
Input, DAG.getConstantFP(1.0, sdl, Input.getValueType())));
return;
}
+ case Intrinsic::vector_partial_reduce_smax:
+ visitPartialReduceToFixed(I, ISD::PARTIAL_REDUCE_SMAX);
+ return;
+ case Intrinsic::vector_partial_reduce_smin:
+ visitPartialReduceToFixed(I, ISD::PARTIAL_REDUCE_SMIN);
+ return;
+ case Intrinsic::vector_partial_reduce_umax:
+ visitPartialReduceToFixed(I, ISD::PARTIAL_REDUCE_UMAX);
+ return;
+ case Intrinsic::vector_partial_reduce_umin:
+ visitPartialReduceToFixed(I, ISD::PARTIAL_REDUCE_UMIN);
+ return;
case Intrinsic::experimental_cttz_elts: {
SDValue Op = getValue(I.getOperand(0));
EVT OpVT = Op.getValueType();
@@ -8447,12 +8553,18 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
case Intrinsic::vector_interleave2:
visitVectorInterleave(I, 2);
return;
+ case Intrinsic::vector_interleave_segments2:
+ visitVectorInterleave(I, 2, true);
+ return;
case Intrinsic::vector_interleave3:
visitVectorInterleave(I, 3);
return;
case Intrinsic::vector_interleave4:
visitVectorInterleave(I, 4);
return;
+ case Intrinsic::vector_interleave_segments4:
+ visitVectorInterleave(I, 4, true);
+ return;
case Intrinsic::vector_interleave5:
visitVectorInterleave(I, 5);
return;
@@ -8468,12 +8580,18 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
case Intrinsic::vector_deinterleave2:
visitVectorDeinterleave(I, 2);
return;
+ case Intrinsic::vector_deinterleave_segments2:
+ visitVectorDeinterleave(I, 2, true);
+ return;
case Intrinsic::vector_deinterleave3:
visitVectorDeinterleave(I, 3);
return;
case Intrinsic::vector_deinterleave4:
visitVectorDeinterleave(I, 4);
return;
+ case Intrinsic::vector_deinterleave_segments4:
+ visitVectorDeinterleave(I, 4, true);
+ return;
case Intrinsic::vector_deinterleave5:
visitVectorDeinterleave(I, 5);
return;
@@ -8486,6 +8604,22 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
case Intrinsic::vector_deinterleave8:
visitVectorDeinterleave(I, 8);
return;
+ case Intrinsic::vector_broadcast: {
+ SDValue Vec = getValue(I.getOperand(0));
+ EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
+ setValue(&I, DAG.getNode(ISD::VECTOR_BROADCAST, sdl, ResultVT, Vec));
+ return;
+ }
+
+ case Intrinsic::vector_segmented_shuffle:
+ visitVectorSegmentedShuffle(I);
+ return;
+ case Intrinsic::vector_stretch: {
+ SDValue Vec = getValue(I.getOperand(0));
+ EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
+ setValue(&I, DAG.getNode(ISD::VECTOR_STRETCH, sdl, ResultVT, Vec));
+ return;
+ }
case Intrinsic::experimental_vector_compress:
setValue(&I, DAG.getNode(ISD::VECTOR_COMPRESS, sdl,
getValue(I.getArgOperand(0)).getValueType(),
@@ -11291,6 +11425,21 @@ void SelectionDAGBuilder::visitVectorReduce(const CallInst &I,
setValue(&I, Res);
}
+bool SelectionDAGBuilder::visitPartialReduceToFixed(const CallInst &I,
+ unsigned ISDOpcode) {
+ auto *AccTy = cast<VectorType>(I.getOperand(0)->getType());
+ auto *SrcTy = cast<VectorType>(I.getOperand(1)->getType());
+ SDValue Acc = getValue(I.getOperand(0));
+ SDValue Input = getValue(I.getOperand(1));
+ if (AccTy->getElementCount().isScalable() !=
+ SrcTy->getElementCount().isScalable()) {
+ SDLoc DL = getCurSDLoc();
+ setValue(&I, DAG.getNode(ISDOpcode, DL, Acc.getValueType(), Acc, Input));
+ return true;
+ };
+ return false;
+}
+
/// Returns an AttributeList representing the attributes applied to the return
/// value of the given call.
static AttributeList getReturnAttrs(TargetLowering::CallLoweringInfo &CLI) {
@@ -12920,7 +13069,8 @@ void SelectionDAGBuilder::visitVectorReverse(const CallInst &I) {
}
void SelectionDAGBuilder::visitVectorDeinterleave(const CallInst &I,
- unsigned Factor) {
+ unsigned Factor,
+ bool DeinterleaveSegments) {
auto DL = getCurSDLoc();
SDValue InVec = getValue(I.getOperand(0));
@@ -12931,6 +13081,34 @@ void SelectionDAGBuilder::visitVectorDeinterleave(const CallInst &I,
EVT OutVT = ValueVTs[0];
unsigned OutNumElts = OutVT.getVectorMinNumElements();
+ // Rewrite VECTOR_DEINTERLEAVE_SEGMENTS as VECTOR_DEINTERLEAVE for legal
+ // types.
+ const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+ if (DeinterleaveSegments && Factor == 2 &&
+ TLI.isOperationLegalOrCustom(ISD::VECTOR_DEINTERLEAVE,
+ InVec.getValueType())) {
+ EVT InVT = InVec.getValueType();
+ assert(InVT.isScalableVector() &&
+ "VECTOR_DEINTERLEAVE_SEGMENTS for a fixed vector?");
+ assert(InVT.isSimple() &&
+ "VECTOR_DEINTERLEAVE_SEGMENTS of extended types?");
+ MVT VTForDeinterleave =
+ MVT::getSameSizeVT(InVT.getSimpleVT(), ElementCount::getScalable(2));
+ SDValue InBitcast = DAG.getBitcast(VTForDeinterleave, InVec);
+ SDValue Res = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL,
+ {VTForDeinterleave, VTForDeinterleave},
+ {InBitcast, InBitcast});
+ SDValue Even = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT,
+ DAG.getBitcast(InVT, Res.getValue(0)),
+ DAG.getVectorIdxConstant(0, DL));
+ SDValue Odd = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT,
+ DAG.getBitcast(InVT, Res.getValue(1)),
+ DAG.getVectorIdxConstant(0, DL));
+ SDValue Merged = DAG.getMergeValues({Even, Odd}, getCurSDLoc());
+ setValue(&I, Merged);
+ return;
+ }
+
SmallVector<SDValue, 4> SubVecs(Factor);
for (unsigned i = 0; i != Factor; ++i) {
assert(ValueVTs[i] == OutVT && "Expected VTs to be the same");
@@ -12950,13 +13128,15 @@ void SelectionDAGBuilder::visitVectorDeinterleave(const CallInst &I,
return;
}
- SDValue Res = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL,
- DAG.getVTList(ValueVTs), SubVecs);
+ unsigned Opc = DeinterleaveSegments ? ISD::VECTOR_DEINTERLEAVE_SEGMENTS
+ : ISD::VECTOR_DEINTERLEAVE;
+ SDValue Res = DAG.getNode(Opc, DL, DAG.getVTList(ValueVTs), SubVecs);
setValue(&I, Res);
}
void SelectionDAGBuilder::visitVectorInterleave(const CallInst &I,
- unsigned Factor) {
+ unsigned Factor,
+ bool InterleaveSegments) {
auto DL = getCurSDLoc();
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
EVT InVT = getValue(I.getOperand(0)).getValueType();
@@ -12980,8 +13160,9 @@ void SelectionDAGBuilder::visitVectorInterleave(const CallInst &I,
}
SmallVector<EVT, 8> ValueVTs(Factor, InVT);
- SDValue Res =
- DAG.getNode(ISD::VECTOR_INTERLEAVE, DL, DAG.getVTList(ValueVTs), InVecs);
+ unsigned Opc = InterleaveSegments ? ISD::VECTOR_INTERLEAVE_SEGMENTS
+ : ISD::VECTOR_INTERLEAVE;
+ SDValue Res = DAG.getNode(Opc, DL, DAG.getVTList(ValueVTs), InVecs);
SmallVector<SDValue, 8> Results(Factor);
for (unsigned i = 0; i < Factor; ++i)
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
index 21aac333a73cd..f76d43f1a23d2 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
@@ -607,6 +607,7 @@ class SelectionDAGBuilder {
void visitExtractElement(const User &I);
void visitInsertElement(const User &I);
void visitShuffleVector(const User &I);
+ void visitVectorSegmentedShuffle(const CallInst &I);
void visitExtractValue(const ExtractValueInst &I);
void visitInsertValue(const InsertValueInst &I);
@@ -622,6 +623,8 @@ class SelectionDAGBuilder {
void visitMaskedStore(const CallInst &I, bool IsCompressing = false);
void visitMaskedGather(const CallInst &I);
void visitMaskedScatter(const CallInst &I);
+ void visitMaskedSegmentGather(const CallInst &I);
+ void visitMaskedSegmentScatter(const CallInst &I);
void visitAtomicCmpXchg(const AtomicCmpXchgInst &I);
void visitAtomicRMW(const AtomicRMWInst &I);
void visitFence(const FenceInst &I);
@@ -685,10 +688,14 @@ class SelectionDAGBuilder {
void visitGCResult(const GCResultInst &I);
void visitVectorReduce(const CallInst &I, unsigned Intrinsic);
+ bool visitPartialReduceToFixed(const CallInst &I, unsigned ISDOpcode);
void visitVectorReverse(const CallInst &I);
void visitVectorSplice(const CallInst &I);
- void visitVectorInterleave(const CallInst &I, unsigned Factor);
- void visitVectorDeinterleave(const CallInst &I, unsigned Factor);
+ void visitVectorInterleave(const CallInst &I, unsigned Factor,
+ bool InterleaveSegments = false);
+ void visitVectorDeinterleave(const CallInst &I, unsigned Factor,
+ bool DeinterleaveSegments = false);
+ void visitNamedVectorShuffle(const CallInst &I);
void visitStepVector(const CallInst &I);
void visitUserOp1(const Instruction &I) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
index 7b500fc863b72..a57f9a6007df5 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
@@ -354,8 +354,13 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
case ISD::EXTRACT_SUBVECTOR: return "extract_subvector";
case ISD::VECTOR_DEINTERLEAVE: return "vector_deinterleave";
case ISD::VECTOR_INTERLEAVE: return "vector_interleave";
+ case ISD::VECTOR_DEINTERLEAVE_SEGMENTS: return "vector_deinterleave_segments";
+ case ISD::VECTOR_INTERLEAVE_SEGMENTS: return "vector_interleave_segments";
+ case ISD::VECTOR_BROADCAST: return "vector_broadcast";
+ case ISD::VECTOR_STRETCH: return "vector_stretch";
case ISD::SCALAR_TO_VECTOR: return "scalar_to_vector";
case ISD::VECTOR_SHUFFLE: return "vector_shuffle";
+ case ISD::VECTOR_SEGMENTED_SHUFFLE: return "vector_segmented_shuffle";
case ISD::VECTOR_SPLICE_LEFT: return "vector_splice_left";
case ISD::VECTOR_SPLICE_RIGHT: return "vector_splice_right";
case ISD::SPLAT_VECTOR: return "splat_vector";
@@ -603,6 +608,16 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
case ISD::GET_ACTIVE_LANE_MASK:
return "get_active_lane_mask";
+ case ISD::PARTIAL_REDUCE_ADD:
+ return "partial_reduce_scalable";
+ case ISD::PARTIAL_REDUCE_SMAX:
+ return "partial_reduce_smax";
+ case ISD::PARTIAL_REDUCE_SMIN:
+ return "partial_reduce_smin";
+ case ISD::PARTIAL_REDUCE_UMAX:
+ return "partial_reduce_umax";
+ case ISD::PARTIAL_REDUCE_UMIN:
+ return "partial_reduce_umin";
case ISD::PARTIAL_REDUCE_UMLA:
return "partial_reduce_umla";
case ISD::PARTIAL_REDUCE_SMLA:
@@ -777,6 +792,19 @@ void SDNode::print_details(raw_ostream &OS, const SelectionDAG *G) const {
OS << Idx;
}
OS << ">";
+ } else if (const auto *SSVN = dyn_cast<SegmentedShuffleVectorSDNode>(this)) {
+ ArrayRef<int> Mask = SSVN->getMask();
+ OS << "<";
+ for (unsigned i = 0, e = Mask.size(); i != e; ++i) {
+ int Idx = Mask[i];
+ if (i)
+ OS << ",";
+ if (Idx < 0)
+ OS << "u";
+ else
+ OS << Idx;
+ }
+ OS << ">";
} else if (const ConstantSDNode *CSDN = dyn_cast<ConstantSDNode>(this)) {
OS << '<' << CSDN->getAPIntValue() << '>';
} else if (const ConstantFPSDNode *CSDN = dyn_cast<ConstantFPSDNode>(this)) {
@@ -790,7 +818,7 @@ void SDNode::print_details(raw_ostream &OS, const SelectionDAG *G) const {
OS << ")>";
}
} else if (const GlobalAddressSDNode *GADN =
- dyn_cast<GlobalAddressSDNode>(this)) {
+ dyn_cast<GlobalAddressSDNode>(this)) {
int64_t offset = GADN->getOffset();
OS << '<';
GADN->getGlobal()->printAsOperand(OS);
@@ -807,7 +835,8 @@ void SDNode::print_details(raw_ostream &OS, const SelectionDAG *G) const {
OS << "<" << JTDN->getIndex() << ">";
if (unsigned int TF = JTDN->getTargetFlags())
OS << " [TF=" << TF << ']';
- } else if (const ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(this)){
+ } else if (const ConstantPoolSDNode *CP =
+ dyn_cast<ConstantPoolSDNode>(this)) {
int offset = CP->getOffset();
if (CP->isMachineConstantPoolEntry())
OS << "<" << *CP->getMachineCPVal() << ">";
@@ -833,7 +862,7 @@ void SDNode::print_details(raw_ostream &OS, const SelectionDAG *G) const {
OS << ' ' << printReg(R->getReg(),
G ? G->getSubtarget().getRegisterInfo() : nullptr);
} else if (const ExternalSymbolSDNode *ES =
- dyn_cast<ExternalSymbolSDNode>(this)) {
+ dyn_cast<ExternalSymbolSDNode>(this)) {
OS << "'" << ES->getSymbol() << "'";
if (unsigned int TF = ES->getTargetFlags())
OS << " [TF=" << TF << ']';
@@ -849,8 +878,7 @@ void SDNode::print_details(raw_ostream &OS, const SelectionDAG *G) const {
OS << "<null>";
} else if (const VTSDNode *N = dyn_cast<VTSDNode>(this)) {
OS << ":" << N->getVT();
- }
- else if (const LoadSDNode *LD = dyn_cast<LoadSDNode>(this)) {
+ } else if (const LoadSDNode *LD = dyn_cast<LoadSDNode>(this)) {
OS << "<";
printMemOperand(OS, *LD->getMemOperand(), G);
@@ -970,7 +998,7 @@ void SDNode::print_details(raw_ostream &OS, const SelectionDAG *G) const {
}
OS << ">";
} else if (const BlockAddressSDNode *BA =
- dyn_cast<BlockAddressSDNode>(this)) {
+ dyn_cast<BlockAddressSDNode>(this)) {
int64_t offset = BA->getOffset();
OS << "<";
BA->getBlockAddress()->getFunction()->printAsOperand(OS, false);
@@ -984,7 +1012,7 @@ void SDNode::print_details(raw_ostream &OS, const SelectionDAG *G) const {
if (unsigned int TF = BA->getTargetFlags())
OS << " [TF=" << TF << ']';
} else if (const AddrSpaceCastSDNode *ASC =
- dyn_cast<AddrSpaceCastSDNode>(this)) {
+ dyn_cast<AddrSpaceCastSDNode>(this)) {
OS << '['
<< ASC->getSrcAddressSpace()
<< " -> "
diff --git a/llvm/lib/CodeGen/ValueTypes.cpp b/llvm/lib/CodeGen/ValueTypes.cpp
index e74068e22f4cd..b16bf40ae9d88 100644
--- a/llvm/lib/CodeGen/ValueTypes.cpp
+++ b/llvm/lib/CodeGen/ValueTypes.cpp
@@ -295,6 +295,15 @@ MVT MVT::getVT(Type *Ty, bool HandleUnknown){
}
}
+MVT MVT::getSameSizeVT(MVT VT, ElementCount NewEC) {
+ ElementCount OrigEC = VT.getVectorElementCount();
+ assert(OrigEC.isKnownMultipleOf(NewEC));
+ unsigned Factor = OrigEC.getKnownScalarFactor(NewEC);
+ MVT EltTy = VT.getVectorElementType();
+ MVT IntTy = MVT::getIntegerVT(EltTy.getSizeInBits() * Factor);
+ return MVT::getVectorVT(IntTy, NewEC);
+}
+
/// getEVT - Return the value type corresponding to the specified type.
/// If HandleUnknown is true, unknown types are returned as Other, otherwise
/// they are invalid.
diff --git a/llvm/lib/IR/Constants.cpp b/llvm/lib/IR/Constants.cpp
index cadb48b1a649b..6d10f12c34a60 100644
--- a/llvm/lib/IR/Constants.cpp
+++ b/llvm/lib/IR/Constants.cpp
@@ -2892,10 +2892,12 @@ Constant *ConstantExpr::getIntrinsicIdentity(Intrinsic::ID ID, Type *Ty) {
return Constant::getAllOnesValue(Ty);
case Intrinsic::smax:
return Constant::getIntegerValue(
- Ty, APInt::getSignedMinValue(Ty->getIntegerBitWidth()));
+ Ty,
+ APInt::getSignedMinValue(Ty->getScalarType()->getIntegerBitWidth()));
case Intrinsic::smin:
return Constant::getIntegerValue(
- Ty, APInt::getSignedMaxValue(Ty->getIntegerBitWidth()));
+ Ty,
+ APInt::getSignedMaxValue(Ty->getScalarType()->getIntegerBitWidth()));
default:
return nullptr;
}
diff --git a/llvm/lib/IR/IRBuilder.cpp b/llvm/lib/IR/IRBuilder.cpp
index a52c3db9dad97..140e0f605f83b 100644
--- a/llvm/lib/IR/IRBuilder.cpp
+++ b/llvm/lib/IR/IRBuilder.cpp
@@ -30,6 +30,7 @@
#include "llvm/IR/Statepoint.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Value.h"
+#include "llvm/IR/VectorTypeUtils.h"
#include "llvm/Support/Casting.h"
#include <cassert>
#include <cstdint>
@@ -629,10 +630,11 @@ CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
/// \p Name - name of the result variable
CallInst *IRBuilderBase::CreateMaskedGather(Type *Ty, Value *Ptrs,
Align Alignment, Value *Mask,
- Value *PassThru,
- const Twine &Name) {
+ Value *PassThru, const Twine &Name,
+ bool GatherSegments) {
auto *VecTy = cast<VectorType>(Ty);
- ElementCount NumElts = VecTy->getElementCount();
+ ElementCount NumElts =
+ GatherSegments ? ElementCount::getScalable(1) : VecTy->getElementCount();
auto *PtrsTy = cast<VectorType>(Ptrs->getType());
assert(NumElts == PtrsTy->getElementCount() && "Element count mismatch");
@@ -647,8 +649,9 @@ CallInst *IRBuilderBase::CreateMaskedGather(Type *Ty, Value *Ptrs,
// We specify only one type when we create this intrinsic. Types of other
// arguments are derived from this type.
- CallInst *CI = CreateMaskedIntrinsic(Intrinsic::masked_gather, Ops,
- OverloadedTypes, Name);
+ Intrinsic::ID IID = GatherSegments ? Intrinsic::masked_segment_gather
+ : Intrinsic::masked_gather;
+ CallInst *CI = CreateMaskedIntrinsic(IID, Ops, OverloadedTypes, Name);
CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), Alignment));
return CI;
}
@@ -661,10 +664,12 @@ CallInst *IRBuilderBase::CreateMaskedGather(Type *Ty, Value *Ptrs,
/// \p Mask - vector of booleans which indicates what vector lanes should
/// be accessed in memory
CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs,
- Align Alignment, Value *Mask) {
+ Align Alignment, Value *Mask,
+ bool ScatterSegments) {
auto *PtrsTy = cast<VectorType>(Ptrs->getType());
auto *DataTy = cast<VectorType>(Data->getType());
- ElementCount NumElts = PtrsTy->getElementCount();
+ ElementCount NumElts = ScatterSegments ? ElementCount::getScalable(1)
+ : PtrsTy->getElementCount();
if (!Mask)
Mask = getAllOnesMask(NumElts);
@@ -674,8 +679,9 @@ CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs,
// We specify only one type when we create this intrinsic. Types of other
// arguments are derived from this type.
- CallInst *CI =
- CreateMaskedIntrinsic(Intrinsic::masked_scatter, Ops, OverloadedTypes);
+ Intrinsic::ID IID = ScatterSegments ? Intrinsic::masked_segment_scatter
+ : Intrinsic::masked_scatter;
+ CallInst *CI = CreateMaskedIntrinsic(IID, Ops, OverloadedTypes);
CI->addParamAttr(1, Attribute::getWithAlignment(CI->getContext(), Alignment));
return CI;
}
@@ -1250,6 +1256,24 @@ Value *IRBuilderBase::CreateVectorSplat(ElementCount EC, Value *V,
const Twine &Name) {
assert(EC.isNonZero() && "Cannot splat to an empty vector!");
+ if (V->getType()->isVectorTy()) {
+ auto *VectorTy = cast<VectorType>(V->getType());
+ assert(!isa<ScalableVectorType>(VectorTy));
+
+ // If the value was already a constant splat, just recreate it with a
+ // bigger element count.
+ if (auto *VectorConstant = dyn_cast<Constant>(V);
+ VectorConstant && VectorConstant->getSplatValue()) {
+ ElementCount ActualEC =
+ EC.multiplyCoefficientBy(VectorTy->getElementCount().getFixedValue());
+ return CreateVectorSplat(ActualEC, VectorConstant->getSplatValue(), Name);
+ }
+
+ auto *WideVectorTy = toVectorTy(VectorTy, EC);
+ return CreateIntrinsic(Intrinsic::vector_broadcast,
+ {WideVectorTy, VectorTy}, {V}, {}, Name);
+ }
+
// First insert it into a poison vector so we can shuffle it.
Value *Poison = PoisonValue::get(VectorType::get(V->getType(), EC));
V = CreateInsertElement(Poison, V, getInt64(0), Name + ".splatinsert");
@@ -1261,7 +1285,8 @@ Value *IRBuilderBase::CreateVectorSplat(ElementCount EC, Value *V,
}
Value *IRBuilderBase::CreateVectorInterleave(ArrayRef<Value *> Ops,
- const Twine &Name) {
+ const Twine &Name,
+ bool InterleaveSegments) {
assert(Ops.size() >= 2 && Ops.size() <= 8 &&
"Unexpected number of operands to interleave");
@@ -1275,7 +1300,8 @@ Value *IRBuilderBase::CreateVectorInterleave(ArrayRef<Value *> Ops,
}
#endif
- unsigned IID = Intrinsic::getInterleaveIntrinsicID(Ops.size());
+ unsigned IID =
+ Intrinsic::getInterleaveIntrinsicID(Ops.size(), InterleaveSegments);
auto *SubvecTy = cast<VectorType>(Ops[0]->getType());
Type *DestTy = VectorType::get(SubvecTy->getElementType(),
SubvecTy->getElementCount() * Ops.size());
diff --git a/llvm/lib/IR/Intrinsics.cpp b/llvm/lib/IR/Intrinsics.cpp
index ac845fdb03005..7b7a017c9c49f 100644
--- a/llvm/lib/IR/Intrinsics.cpp
+++ b/llvm/lib/IR/Intrinsics.cpp
@@ -1334,14 +1334,29 @@ static InterleaveIntrinsic InterleaveIntrinsics[] = {
{Intrinsic::vector_interleave6, Intrinsic::vector_deinterleave6},
{Intrinsic::vector_interleave7, Intrinsic::vector_deinterleave7},
{Intrinsic::vector_interleave8, Intrinsic::vector_deinterleave8},
+ {Intrinsic::vector_interleave_segments2,
+ Intrinsic::vector_deinterleave_segments2},
+ {Intrinsic::not_intrinsic, Intrinsic::not_intrinsic},
+ {Intrinsic::vector_interleave_segments4,
+ Intrinsic::vector_deinterleave_segments4},
};
-Intrinsic::ID Intrinsic::getInterleaveIntrinsicID(unsigned Factor) {
+Intrinsic::ID Intrinsic::getInterleaveIntrinsicID(unsigned Factor,
+ bool InterleaveSegments) {
+ if (InterleaveSegments) {
+ assert((Factor == 2 || Factor == 4) && "Unexpected factor");
+ return InterleaveIntrinsics[Factor + 5].Interleave;
+ }
assert(Factor >= 2 && Factor <= 8 && "Unexpected factor");
return InterleaveIntrinsics[Factor - 2].Interleave;
}
-Intrinsic::ID Intrinsic::getDeinterleaveIntrinsicID(unsigned Factor) {
+Intrinsic::ID Intrinsic::getDeinterleaveIntrinsicID(unsigned Factor,
+ bool DeinterleaveSegments) {
+ if (DeinterleaveSegments) {
+ assert((Factor == 2 || Factor == 4) && "Unexpected factor");
+ return InterleaveIntrinsics[Factor + 5].Deinterleave;
+ }
assert(Factor >= 2 && Factor <= 8 && "Unexpected factor");
return InterleaveIntrinsics[Factor - 2].Deinterleave;
}
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index c9639d1420bfc..b6f4236d70849 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -6093,6 +6093,38 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) {
"const x86_amx is not allowed in argument!");
}
+ auto CheckMaskedSegmentMemOpTypes = [&](Type *DataTy, Type *PtrTy,
+ Type *MaskTy,
+ StringRef IntrinsicName) {
+ auto *DataVTy = dyn_cast<VectorType>(DataTy);
+ Check(
+ DataVTy && DataVTy->getElementCount().isScalable(),
+ (Twine(IntrinsicName) + ": data type must be a scalable vector").str(),
+ Call);
+
+ auto *PtrVTy = dyn_cast<VectorType>(PtrTy);
+ Check(PtrVTy && PtrVTy->getElementCount() == ElementCount::getScalable(1),
+ (Twine(IntrinsicName) + ": pointer type must be <vscale x 1 x ptr>")
+ .str(),
+ Call);
+ if (PtrVTy)
+ Check(
+ PtrVTy->getElementType()->isPointerTy(),
+ (Twine(IntrinsicName) + ": pointer vector elements must be pointers")
+ .str(),
+ Call);
+
+ auto *MaskVTy = dyn_cast<VectorType>(MaskTy);
+ Check(
+ MaskVTy && MaskVTy->getElementCount() == ElementCount::getScalable(1),
+ (Twine(IntrinsicName) + ": mask type must be <vscale x 1 x i1>").str(),
+ Call);
+ if (MaskVTy)
+ Check(MaskVTy->getElementType()->isIntegerTy(1),
+ (Twine(IntrinsicName) + ": mask vector elements must be i1").str(),
+ Call);
+ };
+
switch (ID) {
default:
break;
@@ -6703,6 +6735,20 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) {
"masked_store: vector mask must be same length as value", Call);
break;
}
+
+ case Intrinsic::masked_segment_gather: {
+ CheckMaskedSegmentMemOpTypes(
+ Call.getType(), Call.getArgOperand(0)->getType(),
+ Call.getArgOperand(1)->getType(), "masked_segment_gather");
+ break;
+ }
+ case Intrinsic::masked_segment_scatter: {
+ CheckMaskedSegmentMemOpTypes(
+ Call.getArgOperand(0)->getType(), Call.getArgOperand(1)->getType(),
+ Call.getArgOperand(2)->getType(), "masked_segment_scatter");
+ break;
+ }
+
case Intrinsic::experimental_guard: {
Check(isa<CallInst>(Call), "experimental_guard cannot be invoked", Call);
Check(Call.countOperandBundlesOfType(LLVMContext::OB_deopt) == 1,
diff --git a/llvm/lib/Target/AArch64/AArch64.td b/llvm/lib/Target/AArch64/AArch64.td
index bdb80e7ffdd6d..9bff9d7d02a0d 100644
--- a/llvm/lib/Target/AArch64/AArch64.td
+++ b/llvm/lib/Target/AArch64/AArch64.td
@@ -50,6 +50,12 @@ def AArch64InstrInfo : InstrInfo;
include "AArch64SystemOperands.td"
+//===----------------------------------------------------------------------===//
+// Vector Intrinsic Re-vectorization Mappings (NEON to SVE)
+//===----------------------------------------------------------------------===//
+
+include "AArch64VectorIntrinsicMappings.td"
+
//===----------------------------------------------------------------------===//
// AArch64 Processors supported.
//
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 250608a484467..4a8f57d20e7ad 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -92,6 +92,7 @@
#include "llvm/Target/TargetMachine.h"
#include "llvm/Target/TargetOptions.h"
#include "llvm/TargetParser/Triple.h"
+#include "llvm/Transforms/Utils/LoopUtils.h"
#include <algorithm>
#include <bitset>
#include <cassert>
@@ -1215,6 +1216,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setTargetDAGCombine(ISD::TRUNCATE);
setTargetDAGCombine(ISD::LOAD);
+ setTargetDAGCombine(ISD::MLOAD);
setTargetDAGCombine(ISD::MSTORE);
setTargetDAGCombine(ISD::MUL);
@@ -1449,6 +1451,15 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::VECREDUCE_OR, MVT::v2i64, Custom);
setOperationAction(ISD::VECREDUCE_XOR, MVT::v2i64, Custom);
+ // Reduction from SVE to NEON 64bit/128bit -> lower to "qv" intrinsics.
+ unsigned ReduceToNEONOps[] = {
+ ISD::PARTIAL_REDUCE_ADD, ISD::PARTIAL_REDUCE_SMAX,
+ ISD::PARTIAL_REDUCE_SMIN, ISD::PARTIAL_REDUCE_UMAX,
+ ISD::PARTIAL_REDUCE_UMIN};
+ for (MVT VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::v8i8,
+ MVT::v4i16, MVT::v2i32})
+ setOperationAction(ReduceToNEONOps, VT, Custom);
+
setOperationAction(ISD::ANY_EXTEND, MVT::v4i32, Legal);
setTruncStoreAction(MVT::v2i32, MVT::v2i16, Expand);
// Likewise, narrowing and extending vector loads/stores aren't handled
@@ -1641,6 +1652,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
Custom);
setOperationAction(ISD::VECTOR_FIND_LAST_ACTIVE, VT, Legal);
setOperationAction(ISD::GET_ACTIVE_LANE_MASK, VT, Legal);
+ setOperationAction(ISD::VECTOR_STRETCH, VT, Custom);
}
if (Subtarget->isSVEorStreamingSVEAvailable() &&
@@ -1698,7 +1710,10 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom);
+ setOperationAction(ISD::VECTOR_DEINTERLEAVE_SEGMENTS, VT, Custom);
setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
+ setOperationAction(ISD::VECTOR_INTERLEAVE_SEGMENTS, VT, Custom);
+ setOperationAction(ISD::VECTOR_SEGMENTED_SHUFFLE, VT, Custom);
setOperationAction(ISD::UMUL_LOHI, VT, Expand);
setOperationAction(ISD::SMUL_LOHI, VT, Expand);
@@ -1871,7 +1886,10 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::VECTOR_SPLICE_LEFT, VT, Custom);
setOperationAction(ISD::VECTOR_SPLICE_RIGHT, VT, Custom);
setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom);
+ setOperationAction(ISD::VECTOR_DEINTERLEAVE_SEGMENTS, VT, Custom);
setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
+ setOperationAction(ISD::VECTOR_INTERLEAVE_SEGMENTS, VT, Custom);
+ setOperationAction(ISD::VECTOR_SEGMENTED_SHUFFLE, VT, Custom);
setOperationAction(ISD::SELECT_CC, VT, Expand);
setOperationAction(ISD::FREM, VT, Expand);
@@ -1923,7 +1941,10 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::SELECT_CC, VT, Expand);
setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom);
+ setOperationAction(ISD::VECTOR_DEINTERLEAVE_SEGMENTS, VT, Custom);
setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
+ setOperationAction(ISD::VECTOR_INTERLEAVE_SEGMENTS, VT, Custom);
+ setOperationAction(ISD::VECTOR_SEGMENTED_SHUFFLE, VT, Custom);
setOperationAction(ISD::VECTOR_SPLICE_LEFT, VT, Custom);
setOperationAction(ISD::VECTOR_SPLICE_RIGHT, VT, Custom);
}
@@ -6436,6 +6457,154 @@ SDValue LowerVectorMatch(SDValue Op, SelectionDAG &DAG) {
return DAG.getNode(ISD::TRUNCATE, DL, ResVT, Match);
}
+/// Pack a <vscale x 1 x i1> mask into the lo bits of a <vscale x 2 x i1> mask.
+static SDValue getPacked64bSegmentsMask(SDValue Mask, SDLoc DL,
+ SelectionDAG &DAG) {
+ assert(Mask.getValueType() == MVT::nxv1i1);
+ return DAG.getInsertSubvector(DL, DAG.getConstant(0, DL, MVT::nxv2i1), Mask,
+ 0);
+}
+
+/// Pack a scalable 64b vector like <vscale x 4 x i16> into the lo bits of a
+/// <vscale x 2 x i64> vector.
+static SDValue getPacked64bSegments(SDValue Val, SDLoc DL, SelectionDAG &DAG) {
+ EVT VT = Val.getValueType();
+ EVT PackedVT = getPackedSVEVectorVT(VT.getVectorElementType());
+ SDValue LegalPackedVal =
+ DAG.getInsertSubvector(DL, DAG.getUNDEF(PackedVT), Val, 0);
+ return DAG.getBitcast(MVT::nxv2i64, LegalPackedVal);
+}
+
+static SDValue getFromPacked64bSegments(EVT VT, SDValue PackedSegments,
+ SDLoc DL, SelectionDAG &DAG) {
+ EVT PackedVT = getPackedSVEVectorVT(VT.getVectorElementType());
+ return DAG.getExtractSubvector(DL, VT,
+ DAG.getBitcast(PackedVT, PackedSegments), 0);
+}
+
+static SDValue getPackedQSegmentAddresses(SDValue Ptrs, SDLoc DL,
+ SelectionDAG &DAG) {
+ SDValue WidePtrs = getPacked64bSegments(Ptrs, DL, DAG);
+ // LD1Q/ST1Q read even 64-bit address lanes, so zip the lo addresses with
+ // themselves. Semantically, a zip with undef works as well, but this would
+ // introduce fake register dependencies.
+ return DAG.getNode(AArch64ISD::ZIP1, DL, MVT::nxv2i64, WidePtrs, WidePtrs);
+}
+
+static SDValue lowerMaskedSegmentGather(SDValue Chain, SDValue Ptrs,
+ MachineMemOperand *MMO, SDValue Mask,
+ SDValue PassThru, SDLoc DL,
+ SelectionDAG &DAG) {
+ EVT VT = PassThru.getValueType();
+ unsigned SegmentSizeInBits = VT.getSizeInBits().getKnownMinValue();
+ assert(SegmentSizeInBits == 64U || SegmentSizeInBits == 128U);
+
+ if (SegmentSizeInBits == 64U) {
+ EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
+ SDValue Base = DAG.getConstant(0, DL, PtrVT);
+ SDValue Scale = DAG.getTargetConstant(1, DL, PtrVT);
+ SDValue Ops[] = {Chain,
+ getPacked64bSegments(PassThru, DL, DAG),
+ getPacked64bSegmentsMask(Mask, DL, DAG),
+ Base,
+ getPacked64bSegments(Ptrs, DL, DAG),
+ Scale};
+ SDValue Gather = DAG.getMaskedGather(
+ DAG.getVTList(MVT::nxv2i64, MVT::Other), MVT::nxv2i64, DL, Ops, MMO,
+ ISD::SIGNED_SCALED, ISD::NON_EXTLOAD);
+ SDValue Res = getFromPacked64bSegments(VT, Gather, DL, DAG);
+ return DAG.getMergeValues({Res, Gather.getValue(1)}, DL);
+ }
+
+ SDValue Addrs = getPackedQSegmentAddresses(Ptrs, DL, DAG);
+ SDValue LoadOps[] = {
+ Chain,
+ DAG.getTargetConstant(Intrinsic::aarch64_sve_ld1q_gather_scalar_offset,
+ DL, MVT::i64),
+ Mask, Addrs, DAG.getConstant(0, DL, MVT::i64)};
+ SDValue Load = DAG.getMemIntrinsicNode(
+ ISD::INTRINSIC_W_CHAIN, DL, DAG.getVTList(VT, MVT::Other), LoadOps, VT,
+ MMO);
+ SDValue LoadChain = Load.getValue(1);
+
+ if (PassThru->isUndef() || isZerosVector(PassThru.getNode()))
+ return DAG.getMergeValues({Load, LoadChain}, DL);
+
+ EVT MaskVT = VT.changeVectorElementType(*DAG.getContext(), MVT::i1);
+ SDValue DataMask = DAG.getNode(ISD::VECTOR_STRETCH, DL, MaskVT, Mask);
+ SDValue Select = DAG.getSelect(DL, VT, DataMask, Load, PassThru);
+ return DAG.getMergeValues({Select, LoadChain}, DL);
+}
+
+static SDValue lowerMaskedSegmentScatter(SDValue Chain, SDValue Data,
+ SDValue Ptrs, MachineMemOperand *MMO,
+ SDValue Mask, SDLoc DL,
+ SelectionDAG &DAG) {
+ EVT VT = Data.getValueType();
+ unsigned SegmentSizeInBits = VT.getSizeInBits().getKnownMinValue();
+ assert(SegmentSizeInBits == 64U || SegmentSizeInBits == 128U);
+
+ if (SegmentSizeInBits == 64) {
+ EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
+ SDValue Base = DAG.getConstant(0, DL, PtrVT);
+ SDValue Scale = DAG.getTargetConstant(1, DL, PtrVT);
+ SDValue Ops[] = {Chain,
+ getPacked64bSegments(Data, DL, DAG),
+ getPacked64bSegmentsMask(Mask, DL, DAG),
+ Base,
+ getPacked64bSegments(Ptrs, DL, DAG),
+ Scale};
+ return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), MVT::nxv2i64, DL,
+ Ops, MMO, ISD::SIGNED_SCALED, false);
+ }
+
+ SDValue Addrs = getPackedQSegmentAddresses(Ptrs, DL, DAG);
+ SDValue StoreOps[] = {
+ Chain,
+ DAG.getTargetConstant(Intrinsic::aarch64_sve_st1q_scatter_scalar_offset,
+ DL, MVT::i64),
+ Data,
+ Mask,
+ Addrs,
+ DAG.getConstant(0, DL, MVT::i64)};
+ return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL,
+ DAG.getVTList(MVT::Other), StoreOps, VT, MMO);
+}
+
+static SDValue performMaskedSegmentIntrinsicCombine(
+ SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG) {
+ // TODO-REVEC: Instead of this pre-legalisation combine, it would make sense
+ // to introduce new SDNodes for segmented gather/scatter, just like normal
+ // gather/scatter. It would facilitate MMO handling.
+ if (!DCI.isBeforeLegalize())
+ return SDValue();
+
+ SDLoc DL(N);
+ Intrinsic::ID IntNo = N->getConstantOperandVal(1);
+
+ if (IntNo == Intrinsic::masked_segment_gather) {
+ auto *MemIntr = cast<MemIntrinsicSDNode>(N);
+ SDValue Chain = N->getOperand(0);
+ SDValue Ptrs = N->getOperand(2);
+ SDValue Mask = N->getOperand(3);
+ SDValue PassThru = N->getOperand(4);
+ return lowerMaskedSegmentGather(Chain, Ptrs, MemIntr->getMemOperand(), Mask,
+ PassThru, DL, DAG);
+ }
+
+ if (IntNo == Intrinsic::masked_segment_scatter) {
+ auto *MemIntr = cast<MemIntrinsicSDNode>(N);
+ SDValue Chain = N->getOperand(0);
+ SDValue Data = N->getOperand(2);
+ SDValue Ptrs = N->getOperand(3);
+ SDValue Mask = N->getOperand(4);
+ return lowerMaskedSegmentScatter(Chain, Data, Ptrs, MemIntr->getMemOperand(),
+ Mask, DL, DAG);
+ }
+
+ llvm_unreachable("Exected masked_segment_gather or masked_segment_scatter");
+}
+
SDValue AArch64TargetLowering::LowerINTRINSIC_VOID(SDValue Op,
SelectionDAG &DAG) const {
unsigned IntNo = Op.getConstantOperandVal(1);
@@ -8449,6 +8618,8 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
return LowerZERO_EXTEND_VECTOR_INREG(Op, DAG);
case ISD::VECTOR_SHUFFLE:
return LowerVECTOR_SHUFFLE(Op, DAG);
+ case ISD::VECTOR_SEGMENTED_SHUFFLE:
+ return LowerVECTOR_SEGMENTED_SHUFFLE(Op, DAG);
case ISD::SPLAT_VECTOR:
return LowerSPLAT_VECTOR(Op, DAG);
case ISD::EXTRACT_SUBVECTOR:
@@ -8552,6 +8723,16 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
case ISD::VECREDUCE_MUL:
case ISD::VECREDUCE_FMUL:
return LowerVECREDUCE_MUL(Op, DAG);
+ case ISD::PARTIAL_REDUCE_ADD:
+ return LowerPARTIAL_REDUCE_ToFixed(Op, DAG, Intrinsic::aarch64_sve_addqv);
+ case ISD::PARTIAL_REDUCE_SMAX:
+ return LowerPARTIAL_REDUCE_ToFixed(Op, DAG, Intrinsic::aarch64_sve_smaxqv);
+ case ISD::PARTIAL_REDUCE_SMIN:
+ return LowerPARTIAL_REDUCE_ToFixed(Op, DAG, Intrinsic::aarch64_sve_sminqv);
+ case ISD::PARTIAL_REDUCE_UMAX:
+ return LowerPARTIAL_REDUCE_ToFixed(Op, DAG, Intrinsic::aarch64_sve_umaxqv);
+ case ISD::PARTIAL_REDUCE_UMIN:
+ return LowerPARTIAL_REDUCE_ToFixed(Op, DAG, Intrinsic::aarch64_sve_uminqv);
case ISD::ATOMIC_LOAD_AND:
return LowerATOMIC_LOAD_AND(Op, DAG);
case ISD::DYNAMIC_STACKALLOC:
@@ -8627,10 +8808,16 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
case ISD::VECTOR_SPLICE_LEFT:
case ISD::VECTOR_SPLICE_RIGHT:
return LowerVECTOR_SPLICE(Op, DAG);
+ case ISD::VECTOR_STRETCH:
+ return LowerVECTOR_STRETCH(Op, DAG);
case ISD::VECTOR_DEINTERLEAVE:
return LowerVECTOR_DEINTERLEAVE(Op, DAG);
+ case ISD::VECTOR_DEINTERLEAVE_SEGMENTS:
+ return LowerVECTOR_DEINTERLEAVE_SEGMENTS(Op, DAG);
case ISD::VECTOR_INTERLEAVE:
return LowerVECTOR_INTERLEAVE(Op, DAG);
+ case ISD::VECTOR_INTERLEAVE_SEGMENTS:
+ return LowerVECTOR_INTERLEAVE_SEGMENTS(Op, DAG);
case ISD::GET_ACTIVE_LANE_MASK:
return LowerGET_ACTIVE_LANE_MASK(Op, DAG);
case ISD::LRINT:
@@ -12693,6 +12880,24 @@ SDValue AArch64TargetLowering::LowerVECTOR_SPLICE(SDValue Op,
return SDValue();
}
+SDValue AArch64TargetLowering::LowerVECTOR_STRETCH(SDValue Op,
+ SelectionDAG &DAG) const {
+ EVT Ty = Op.getValueType();
+ assert(Ty.isScalableVector() && Ty.getVectorElementType() == MVT::i1 &&
+ "Only expected predicate registers for VECTOR_STRETCH");
+
+ ElementCount InEC = Op.getOperand(0).getValueType().getVectorElementCount();
+ ElementCount OutEC = Op.getValueType().getVectorElementCount();
+ unsigned StretchFactor = OutEC.getKnownScalarFactor(InEC);
+
+ // Expand until we reach a stretch factor of 2.
+ if (StretchFactor != 2)
+ return SDValue();
+
+ // Otherwise this is legal.
+ return Op;
+}
+
SDValue AArch64TargetLowering::LowerSELECT_CC(SDValue Op,
SelectionDAG &DAG) const {
ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get();
@@ -14589,13 +14794,12 @@ static bool isWideDUPMask(ArrayRef<int> M, EVT VT, unsigned BlockSize,
// check if an EXT instruction can handle the shuffle mask when the
// vector sources of the shuffle are different.
-static bool isEXTMask(ArrayRef<int> M, EVT VT, bool &ReverseEXT,
+static bool isEXTMask(ArrayRef<int> M, unsigned NumElts, bool &ReverseEXT,
unsigned &Imm) {
// Look for the first non-undef element.
const int *FirstRealElt = find_if(M, [](int Elt) { return Elt >= 0; });
// Benefit from APInt to handle overflow when calculating expected element.
- unsigned NumElts = VT.getVectorNumElements();
unsigned MaskBits = APInt(32, NumElts * 2).logBase2();
APInt ExpectedElt = APInt(MaskBits, *FirstRealElt + 1, /*isSigned=*/false,
/*implicitTrunc=*/true);
@@ -14691,8 +14895,8 @@ static bool isEXTMaskWithSplat(ArrayRef<int> M, EVT VT, unsigned SplatOperand,
/// isZIP_v_undef_Mask - Special case of isZIPMask for canonical form of
/// "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
/// Mask is e.g., <0, 0, 1, 1> instead of <0, 4, 1, 5>.
-static bool isZIP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
- unsigned NumElts = VT.getVectorNumElements();
+static bool isZIP_v_undef_Mask(ArrayRef<int> M, unsigned NumElts,
+ unsigned &WhichResult) {
if (NumElts % 2 != 0)
return false;
WhichResult = (M[0] == 0 ? 0 : 1);
@@ -14710,8 +14914,9 @@ static bool isZIP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
/// isUZP_v_undef_Mask - Special case of isUZPMask for canonical form of
/// "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
/// Mask is e.g., <0, 2, 0, 2> instead of <0, 2, 4, 6>,
-static bool isUZP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
- unsigned Half = VT.getVectorNumElements() / 2;
+static bool isUZP_v_undef_Mask(ArrayRef<int> M, unsigned NumElts,
+ unsigned &WhichResult) {
+ unsigned Half = NumElts / 2;
WhichResult = (M[0] == 0 ? 0 : 1);
for (unsigned j = 0; j != 2; ++j) {
unsigned Idx = WhichResult;
@@ -14729,8 +14934,8 @@ static bool isUZP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
/// isTRN_v_undef_Mask - Special case of isTRNMask for canonical form of
/// "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
/// Mask is e.g., <0, 0, 2, 2> instead of <0, 4, 2, 6>.
-static bool isTRN_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
- unsigned NumElts = VT.getVectorNumElements();
+static bool isTRN_v_undef_Mask(ArrayRef<int> M, unsigned NumElts,
+ unsigned &WhichResult) {
if (NumElts % 2 != 0)
return false;
WhichResult = (M[0] == 0 ? 0 : 1);
@@ -15408,7 +15613,7 @@ SDValue AArch64TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
bool ReverseEXT = false;
unsigned Imm;
- if (isEXTMask(ShuffleMask, VT, ReverseEXT, Imm)) {
+ if (isEXTMask(ShuffleMask, NumElts, ReverseEXT, Imm)) {
if (ReverseEXT)
std::swap(V1, V2);
Imm *= getExtFactor(V1);
@@ -15437,15 +15642,15 @@ SDValue AArch64TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
OperandOrder == 0 ? V2 : V1);
}
- if (isZIP_v_undef_Mask(ShuffleMask, VT, WhichResult)) {
+ if (isZIP_v_undef_Mask(ShuffleMask, NumElts, WhichResult)) {
unsigned Opc = (WhichResult == 0) ? AArch64ISD::ZIP1 : AArch64ISD::ZIP2;
return DAG.getNode(Opc, DL, V1.getValueType(), V1, V1);
}
- if (isUZP_v_undef_Mask(ShuffleMask, VT, WhichResult)) {
+ if (isUZP_v_undef_Mask(ShuffleMask, NumElts, WhichResult)) {
unsigned Opc = (WhichResult == 0) ? AArch64ISD::UZP1 : AArch64ISD::UZP2;
return DAG.getNode(Opc, DL, V1.getValueType(), V1, V1);
}
- if (isTRN_v_undef_Mask(ShuffleMask, VT, WhichResult)) {
+ if (isTRN_v_undef_Mask(ShuffleMask, NumElts, WhichResult)) {
unsigned Opc = (WhichResult == 0) ? AArch64ISD::TRN1 : AArch64ISD::TRN2;
return DAG.getNode(Opc, DL, V1.getValueType(), V1, V1);
}
@@ -15521,6 +15726,136 @@ SDValue AArch64TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
return GenerateTBL(Op, ShuffleMask, DAG);
}
+SDValue
+AArch64TargetLowering::LowerVECTOR_SEGMENTED_SHUFFLE(SDValue Op,
+ SelectionDAG &DAG) const {
+ SDLoc DL(Op);
+ EVT VT = Op.getValueType();
+ const unsigned PackedEltSize =
+ getPackedSVEVectorVT(VT.getVectorElementCount()).getScalarSizeInBits();
+
+ auto *SSVN = cast<SegmentedShuffleVectorSDNode>(Op.getNode());
+
+ ArrayRef<int> ShuffleMask = SSVN->getMask();
+ const unsigned SegmentSize = ShuffleMask.size();
+ SDValue V1 = Op.getOperand(0);
+ SDValue V2 = Op.getOperand(1);
+
+ // TODO-REVEC: Support segments other than quads.
+ assert(V1.getValueType() == VT &&
+ "Unexpected VECTOR_SEGMENTED_SHUFFLE type!");
+ assert(SegmentSize == VT.getVectorMinNumElements() &&
+ "Unexpected VECTOR_SEGMENTED_SHUFFLE mask size!");
+
+ // Try to find some known shuffle masks before falling back to tblq.
+ std::optional<Intrinsic::ID> IID;
+ std::optional<unsigned> Imm;
+ bool Reverse = false;
+ unsigned OperandOrder = 0;
+ unsigned WhichResult;
+ if (ShuffleVectorSDNode::isSplatMask(ShuffleMask)) {
+ int SplatIdx = ShuffleVectorSDNode::getSplatMaskIndex(ShuffleMask);
+ return DAG.getNode(
+ ISD::INTRINSIC_WO_CHAIN, DL, VT,
+ {DAG.getTargetConstant(Intrinsic::aarch64_sve_dup_laneq, DL, MVT::i64),
+ V1, DAG.getTargetConstant(SplatIdx, DL, MVT::i64)});
+ }
+ if (ShuffleVectorInst::isReverseMask(ShuffleMask, SegmentSize)) {
+ // SVE2p1 has no generic REVx Z.Q, so emulate with REVD Z.Q.
+ SDValue Rev = DAG.getNode(
+ ISD::INTRINSIC_WO_CHAIN, DL, VT,
+ {DAG.getTargetConstant(Intrinsic::aarch64_sve_revd, DL, MVT::i64), V1,
+ DAG.getConstant(1, DL,
+ VT.changeElementType(*DAG.getContext(), MVT::i1)),
+ V1});
+ if (PackedEltSize != 64U) {
+ Intrinsic::ID RevIID;
+ switch (PackedEltSize) {
+ case 8U:
+ RevIID = Intrinsic::aarch64_sve_revb;
+ break;
+ case 16U:
+ RevIID = Intrinsic::aarch64_sve_revh;
+ break;
+ case 32U:
+ RevIID = Intrinsic::aarch64_sve_revw;
+ break;
+ default:
+ llvm_unreachable("Unexpected element size for REV SEGMENTED_SHUFFLE");
+ }
+ Rev = getSVESafeBitCast(MVT::nxv2i64, Rev, DAG);
+ Rev = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::nxv2i64,
+ {DAG.getTargetConstant(RevIID, DL, MVT::i64), Rev,
+ DAG.getConstant(1, DL, MVT::nxv2i1), Rev});
+ Rev = getSVESafeBitCast(VT, Rev, DAG);
+ }
+ return Rev;
+ }
+ if (isZIPMask(ShuffleMask, SegmentSize, WhichResult, OperandOrder) ||
+ isZIP_v_undef_Mask(ShuffleMask, SegmentSize, WhichResult))
+ IID = (WhichResult == 0) ? Intrinsic::aarch64_sve_zipq1
+ : Intrinsic::aarch64_sve_zipq2;
+ else if (isUZPMask(ShuffleMask, SegmentSize, WhichResult) ||
+ isUZP_v_undef_Mask(ShuffleMask, SegmentSize, WhichResult))
+ IID = (WhichResult == 0) ? Intrinsic::aarch64_sve_uzpq1
+ : Intrinsic::aarch64_sve_uzpq2;
+ else if (isTRNMask(ShuffleMask, SegmentSize, WhichResult, OperandOrder) ||
+ isTRN_v_undef_Mask(ShuffleMask, SegmentSize, WhichResult))
+ IID = (WhichResult == 0) ? Intrinsic::aarch64_sve_trn1
+ : Intrinsic::aarch64_sve_trn2;
+ else if (isEXTMask(ShuffleMask, SegmentSize, Reverse, WhichResult)) {
+ IID = Intrinsic::aarch64_sve_extq;
+ Imm = WhichResult;
+ }
+
+ if (IID) {
+ assert(!(Reverse && OperandOrder != 0) && "Reversing operands twice");
+ if (Reverse || OperandOrder != 0)
+ std::swap(V1, V2);
+ SDValue IDVal = DAG.getTargetConstant(*IID, DL, MVT::i64);
+ SmallVector<SDValue, 4> Ops = {IDVal, V1, V2};
+ if (Imm)
+ Ops.push_back(DAG.getTargetConstant(*Imm, DL, MVT::i32));
+ return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, Ops);
+ }
+
+ // Fallback to tblq
+ SmallVector<SDValue, 8> MaskIndices;
+ for (int Val : ShuffleMask) {
+ MaskIndices.push_back(DAG.getConstant(Val, DL, MVT::i64));
+ }
+ EVT IdxVT =
+ getPackedSVEVectorVT(VT.getVectorElementCount()).getVectorElementType();
+ EVT FixedMaskVT =
+ EVT::getVectorVT(*DAG.getContext(), IdxVT, VT.getVectorMinNumElements());
+ SDValue SegmentMask = DAG.getBuildVector(FixedMaskVT, DL, MaskIndices);
+ EVT MaskVT = VT.changeVectorElementType(*DAG.getContext(), IdxVT);
+ SDValue SegmentedMask = DAG.getNode(
+ AArch64ISD::DUPLANE128, DL, MaskVT,
+ DAG.getInsertSubvector(DL, DAG.getUNDEF(MaskVT), SegmentMask, 0),
+ DAG.getTargetConstant(0, DL, MVT::i64));
+
+ SDValue TblqIDVal =
+ DAG.getTargetConstant(Intrinsic::aarch64_sve_tblq, DL, MVT::i64);
+ SDValue Tbl = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, TblqIDVal, V1,
+ SegmentedMask);
+
+ // Handle 2-input shuffles using an extra tbxq
+ // Res = tbxq(tblq(V1, ShuffleMask), V2, ShuffleMask - splat(SegmentSize))
+ if (any_of(ShuffleMask, [SegmentSize](int EltIdx) {
+ return EltIdx >= int(SegmentSize);
+ })) {
+ SDValue TbxqIDVal =
+ DAG.getTargetConstant(Intrinsic::aarch64_sve_tbxq, DL, MVT::i64);
+ SDValue SegmentedMaskForV2 =
+ DAG.getNode(ISD::SUB, DL, MaskVT, SegmentedMask,
+ DAG.getConstant(SegmentSize, DL, MaskVT));
+ Tbl = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, TbxqIDVal, Tbl, V2,
+ SegmentedMaskForV2);
+ }
+ return Tbl;
+}
+
SDValue AArch64TargetLowering::LowerSPLAT_VECTOR(SDValue Op,
SelectionDAG &DAG) const {
EVT VT = Op.getValueType();
@@ -17223,18 +17558,38 @@ bool AArch64TargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const {
isREVMask(M, EltSize, NumElts, 64) ||
isREVMask(M, EltSize, NumElts, 32) ||
isREVMask(M, EltSize, NumElts, 16) ||
- isEXTMask(M, VT, DummyBool, DummyUnsigned) ||
+ isEXTMask(M, NumElts, DummyBool, DummyUnsigned) ||
isSingletonEXTMask(M, VT, DummyUnsigned) ||
isTRNMask(M, NumElts, DummyUnsigned, DummyUnsigned) ||
isUZPMask(M, NumElts, DummyUnsigned) ||
isZIPMask(M, NumElts, DummyUnsigned, DummyUnsigned) ||
- isTRN_v_undef_Mask(M, VT, DummyUnsigned) ||
- isUZP_v_undef_Mask(M, VT, DummyUnsigned) ||
- isZIP_v_undef_Mask(M, VT, DummyUnsigned) ||
+ isTRN_v_undef_Mask(M, NumElts, DummyUnsigned) ||
+ isUZP_v_undef_Mask(M, NumElts, DummyUnsigned) ||
+ isZIP_v_undef_Mask(M, NumElts, DummyUnsigned) ||
isINSMask(M, NumElts, DummyBool, DummyInt) ||
isConcatMask(M, VT, VT.getSizeInBits() == 128));
}
+bool AArch64TargetLowering::isSegmentedShuffleMaskSupported(
+ ArrayRef<int> M) const {
+ bool DummyBool = false;
+ unsigned DummyUint;
+
+ unsigned SegmentSize = M.size();
+ if (ShuffleVectorSDNode::isSplatMask(M) ||
+ ShuffleVectorInst::isReverseMask(M, SegmentSize) ||
+ isZIPMask(M, SegmentSize, DummyUint, DummyUint) ||
+ isZIP_v_undef_Mask(M, SegmentSize, DummyUint) ||
+ isUZPMask(M, SegmentSize, DummyUint) ||
+ isUZP_v_undef_Mask(M, SegmentSize, DummyUint) ||
+ isTRNMask(M, SegmentSize, DummyUint, DummyUint) ||
+ isTRN_v_undef_Mask(M, SegmentSize, DummyUint) ||
+ isEXTMask(M, SegmentSize, DummyBool, DummyUint))
+
+ return true;
+ return false;
+}
+
bool AArch64TargetLowering::isVectorClearMaskLegal(ArrayRef<int> M,
EVT VT) const {
// Just delegate to the generic legality, clear masks aren't special.
@@ -17688,6 +18043,48 @@ SDValue AArch64TargetLowering::LowerVECREDUCE_MUL(SDValue Op,
return DAG.getExtractVectorElt(DL, Op.getValueType(), Src, 0);
}
+SDValue AArch64TargetLowering::LowerPARTIAL_REDUCE_ToFixed(
+ SDValue Op, SelectionDAG &DAG, Intrinsic::ID ToIID) const {
+ EVT VT = Op.getValueType();
+ [[maybe_unused]] EVT SrcVT = Op->getOperand(1).getValueType();
+ assert(SrcVT.isScalableVector() &&
+ SrcVT.getSizeInBits().getKnownMinValue() == AArch64::SVEBitsPerBlock);
+
+ SDValue Acc = Op->getOperand(0);
+ SDValue Src = Op->getOperand(1);
+ SDLoc DL(Op);
+ unsigned BaseOpc = ISD::getVecReduceBaseOpcode(Op->getOpcode());
+
+ // Reducing a legal SVE vector to NEON 64bit.
+ if (VT.is64BitVector()) {
+ EVT NVT = EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(),
+ VT.getVectorNumElements() * 2);
+ SDValue Acc128 = DAG.getIdentityElement(BaseOpc, DL, NVT, SDNodeFlags());
+ SDValue Res128 =
+ DAG.getNode(Op->getOpcode(), DL, NVT, Acc128, Op->getOperand(1));
+ SDValue ResLo = DAG.getExtractSubvector(DL, VT, Res128, 0);
+ SDValue ResHi =
+ DAG.getExtractSubvector(DL, VT, Res128, VT.getVectorNumElements());
+ SDValue Res = DAG.getNode(BaseOpc, DL, VT, ResLo, ResHi);
+ if (!DAG.isIdentityElement(BaseOpc, SDNodeFlags(), Acc, 0))
+ Res = DAG.getNode(BaseOpc, DL, VT, Res, Acc);
+ return Res;
+ }
+
+ SDValue ID = DAG.getTargetConstant(ToIID, DL, MVT::i64);
+ SDValue VPg = getPTrue(
+ DAG, DL,
+ Src.getValueType().changeVectorElementType(*DAG.getContext(), MVT::i1),
+ AArch64SVEPredPattern::all);
+ SDValue Res =
+ DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Op.getValueType(), ID, VPg, Src);
+
+ // Handle non-neutral accumulators.
+ if (!DAG.isIdentityElement(BaseOpc, SDNodeFlags(), Acc, 0))
+ Res = DAG.getNode(BaseOpc, DL, VT, Acc, Res);
+ return Res;
+}
+
SDValue AArch64TargetLowering::LowerATOMIC_LOAD_AND(SDValue Op,
SelectionDAG &DAG) const {
auto &Subtarget = DAG.getSubtarget<AArch64Subtarget>();
@@ -18592,13 +18989,22 @@ AArch64TargetLowering::getTargetMMOFlags(const Instruction &I) const {
}
bool AArch64TargetLowering::isLegalInterleavedAccessType(
- VectorType *VecTy, const DataLayout &DL, bool &UseScalable) const {
+ VectorType *VecTy, const DataLayout &DL, bool &UseScalable,
+ bool AccessSegments) const {
unsigned ElSize = DL.getTypeSizeInBits(VecTy->getElementType());
auto EC = VecTy->getElementCount();
unsigned MinElts = EC.getKnownMinValue();
UseScalable = false;
+ // Segment (de)interleaving requires SVE2p1 and segments of 128 bits.
+ if (AccessSegments) {
+ assert(EC.isScalable() &&
+ "Expected a scalable type for (de)interleaving segments.");
+ UseScalable = true;
+ return Subtarget->hasSVE2p1() && (MinElts * ElSize) == 128u;
+ }
+
if (isa<FixedVectorType>(VecTy) && !Subtarget->isNeonAvailable() &&
(!Subtarget->useSVEForFixedLengthVectors() ||
!getSVEPredPatternFromNumElements(MinElts)))
@@ -18667,15 +19073,21 @@ static ScalableVectorType *getSVEContainerIRType(FixedVectorType *VTy) {
}
static Function *getStructuredLoadFunction(Module *M, unsigned Factor,
- bool Scalable, Type *LDVTy,
- Type *PtrTy) {
+ bool Scalable, bool Quad,
+ Type *LDVTy, Type *PtrTy) {
assert(Factor >= 2 && Factor <= 4 && "Invalid interleave factor");
static const Intrinsic::ID SVELoads[3] = {Intrinsic::aarch64_sve_ld2_sret,
Intrinsic::aarch64_sve_ld3_sret,
Intrinsic::aarch64_sve_ld4_sret};
+ static const Intrinsic::ID SVEQuadLoads[3] = {
+ Intrinsic::aarch64_sve_ld2q_qpred, Intrinsic::aarch64_sve_ld3q_qpred,
+ Intrinsic::aarch64_sve_ld4q_qpred};
static const Intrinsic::ID NEONLoads[3] = {Intrinsic::aarch64_neon_ld2,
Intrinsic::aarch64_neon_ld3,
Intrinsic::aarch64_neon_ld4};
+ if (Quad)
+ return Intrinsic::getOrInsertDeclaration(M, SVEQuadLoads[Factor - 2],
+ {LDVTy});
if (Scalable)
return Intrinsic::getOrInsertDeclaration(M, SVELoads[Factor - 2],
{LDVTy, PtrTy});
@@ -18685,15 +19097,21 @@ static Function *getStructuredLoadFunction(Module *M, unsigned Factor,
}
static Function *getStructuredStoreFunction(Module *M, unsigned Factor,
- bool Scalable, Type *STVTy,
- Type *PtrTy) {
+ bool Scalable, bool Quad,
+ Type *STVTy, Type *PtrTy) {
assert(Factor >= 2 && Factor <= 4 && "Invalid interleave factor");
static const Intrinsic::ID SVEStores[3] = {Intrinsic::aarch64_sve_st2,
Intrinsic::aarch64_sve_st3,
Intrinsic::aarch64_sve_st4};
+ static const Intrinsic::ID SVEQuadStores[3] = {
+ Intrinsic::aarch64_sve_st2q_qpred, Intrinsic::aarch64_sve_st3q_qpred,
+ Intrinsic::aarch64_sve_st4q_qpred};
static const Intrinsic::ID NEONStores[3] = {Intrinsic::aarch64_neon_st2,
Intrinsic::aarch64_neon_st3,
Intrinsic::aarch64_neon_st4};
+ if (Quad)
+ return Intrinsic::getOrInsertDeclaration(M, SVEQuadStores[Factor - 2],
+ {STVTy});
if (Scalable)
return Intrinsic::getOrInsertDeclaration(M, SVEStores[Factor - 2],
{STVTy, PtrTy});
@@ -18777,8 +19195,8 @@ bool AArch64TargetLowering::lowerInterleavedLoad(
Type *PredTy = VectorType::get(Type::getInt1Ty(LDVTy->getContext()),
LDVTy->getElementCount());
- Function *LdNFunc = getStructuredLoadFunction(LI->getModule(), Factor,
- UseScalable, LDVTy, PtrTy);
+ Function *LdNFunc = getStructuredLoadFunction(
+ LI->getModule(), Factor, UseScalable, /*Quad=*/false, LDVTy, PtrTy);
// Holds sub-vectors extracted from the load intrinsic return values. The
// sub-vectors are associated with the shufflevector instructions they will
@@ -19004,8 +19422,8 @@ bool AArch64TargetLowering::lowerInterleavedStore(Instruction *Store,
Type *PredTy = VectorType::get(Type::getInt1Ty(STVTy->getContext()),
STVTy->getElementCount());
- Function *StNFunc = getStructuredStoreFunction(SI->getModule(), Factor,
- UseScalable, STVTy, PtrTy);
+ Function *StNFunc = getStructuredStoreFunction(
+ SI->getModule(), Factor, UseScalable, /*Quad=*/false, STVTy, PtrTy);
Value *PTrue = nullptr;
if (UseScalable) {
@@ -19075,18 +19493,14 @@ bool AArch64TargetLowering::lowerInterleavedStore(Instruction *Store,
}
bool AArch64TargetLowering::lowerDeinterleaveIntrinsicToLoad(
- Instruction *Load, Value *Mask, IntrinsicInst *DI,
- const APInt &GapMask) const {
+ Instruction *Load, Value *Mask, IntrinsicInst *DI, const APInt &GapMask,
+ bool DeinterleaveSegments) const {
const unsigned Factor = getDeinterleaveIntrinsicFactor(DI->getIntrinsicID());
assert(GapMask.getBitWidth() == Factor);
if (Factor != 2 && Factor != 3 && Factor != 4) {
LLVM_DEBUG(dbgs() << "Matching ld2, ld3 and ld4 patterns failed\n");
return false;
}
- auto *LI = dyn_cast<LoadInst>(Load);
- if (!LI)
- return false;
- assert(!Mask && "Unexpected mask on a load\n");
// Gap mask is currently not supported.
if (!GapMask.isAllOnes())
@@ -19094,9 +19508,9 @@ bool AArch64TargetLowering::lowerDeinterleaveIntrinsicToLoad(
VectorType *VTy = getDeinterleavedVectorType(DI);
- const DataLayout &DL = LI->getModule()->getDataLayout();
+ const DataLayout &DL = Load->getModule()->getDataLayout();
bool UseScalable;
- if (!isLegalInterleavedAccessType(VTy, DL, UseScalable))
+ if (!isLegalInterleavedAccessType(VTy, DL, UseScalable, DeinterleaveSegments))
return false;
// TODO: Add support for using SVE instructions with fixed types later, using
@@ -19104,24 +19518,63 @@ bool AArch64TargetLowering::lowerDeinterleaveIntrinsicToLoad(
if (UseScalable && !VTy->isScalableTy())
return false;
+ // Masking requires SVE instructions.
+ if (Mask && !UseScalable)
+ return false;
+
+ // Do not bother with all-false predicates, the load will get optimised away.
+ using namespace llvm::PatternMatch;
+ if (Mask && match(Mask, m_ZeroOrPoison()))
+ return false;
+
unsigned NumLoads = getNumInterleavedAccesses(VTy, DL, UseScalable);
VectorType *LdTy =
VectorType::get(VTy->getElementType(),
VTy->getElementCount().divideCoefficientBy(NumLoads));
- Type *PtrTy = LI->getPointerOperandType();
- Function *LdNFunc = getStructuredLoadFunction(LI->getModule(), Factor,
- UseScalable, LdTy, PtrTy);
+ // TODO: Split masks for illegal loads.
+ if (NumLoads > 1 && Mask)
+ return false;
- IRBuilder<> Builder(LI);
+ if (!isa<LoadInst>(Load) &&
+ !match(Load, m_Intrinsic<Intrinsic::masked_load>(
+ m_Value(), m_Value(),
+ m_CombineOr(m_Poison(), m_Zero()))))
+ return false;
+
+ Value *BaseAddr = Load->getOperand(0);
+ Type *PtrTy = BaseAddr->getType();
+ Function *LdNFunc =
+ getStructuredLoadFunction(Load->getModule(), Factor, UseScalable,
+ DeinterleaveSegments, LdTy, PtrTy);
+
+ IRBuilder<> Builder(Load);
Value *Pred = nullptr;
- if (UseScalable)
- Pred =
- Builder.CreateVectorSplat(LdTy->getElementCount(), Builder.getTrue());
- Value *BaseAddr = LI->getPointerOperand();
+ if (UseScalable) {
+ if (Mask)
+ Pred = Mask;
+ else if (DeinterleaveSegments) {
+ // Create a ptrue based on the element type, but then reinterpret it as
+ // nxv1i1. This avoids generating splats of nxv1i1 type, which aren't
+ // natively supported.
+ Pred =
+ Builder.CreateVectorSplat(LdTy->getElementCount(), Builder.getTrue());
+ Pred = Builder.CreateIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool,
+ {Pred->getType()}, {Pred});
+ auto *QuadMaskTy =
+ ScalableVectorType::get(Type::getInt1Ty(Builder.getContext()), 1);
+ Pred = Builder.CreateIntrinsic(Intrinsic::aarch64_sve_convert_from_svbool,
+ {QuadMaskTy}, {Pred});
+ } else {
+ Pred =
+ Builder.CreateVectorSplat(LdTy->getElementCount(), Builder.getTrue());
+ }
+ }
+
Value *Result = nullptr;
if (NumLoads > 1) {
+ assert(!Mask && "Expected legal LdTy when masking is used.");
// Create multiple legal small ldN.
SmallVector<Value *, 4> ExtractedLdValues(Factor, PoisonValue::get(VTy));
for (unsigned I = 0; I < NumLoads; ++I) {
@@ -19159,23 +19612,19 @@ bool AArch64TargetLowering::lowerDeinterleaveIntrinsicToLoad(
}
bool AArch64TargetLowering::lowerInterleaveIntrinsicToStore(
- Instruction *Store, Value *Mask,
- ArrayRef<Value *> InterleavedValues) const {
+ Instruction *Store, Value *Mask, ArrayRef<Value *> InterleavedValues,
+ bool InterleaveSegments) const {
unsigned Factor = InterleavedValues.size();
if (Factor != 2 && Factor != 3 && Factor != 4) {
LLVM_DEBUG(dbgs() << "Matching st2, st3 and st4 patterns failed\n");
return false;
}
- StoreInst *SI = dyn_cast<StoreInst>(Store);
- if (!SI)
- return false;
- assert(!Mask && "Unexpected mask on plain store");
VectorType *VTy = cast<VectorType>(InterleavedValues[0]->getType());
- const DataLayout &DL = SI->getModule()->getDataLayout();
+ const DataLayout &DL = Store->getModule()->getDataLayout();
bool UseScalable;
- if (!isLegalInterleavedAccessType(VTy, DL, UseScalable))
+ if (!isLegalInterleavedAccessType(VTy, DL, UseScalable, InterleaveSegments))
return false;
// TODO: Add support for using SVE instructions with fixed types later, using
@@ -19183,24 +19632,58 @@ bool AArch64TargetLowering::lowerInterleaveIntrinsicToStore(
if (UseScalable && !VTy->isScalableTy())
return false;
+ // Masking requires SVE instructions.
+ if (Mask && !UseScalable)
+ return false;
+
+ // Do not bother with all-false predicates, the store will get optimised away.
+ using namespace llvm::PatternMatch;
+ if (Mask && match(Mask, m_ZeroOrPoison()))
+ return false;
+
unsigned NumStores = getNumInterleavedAccesses(VTy, DL, UseScalable);
+ // TODO: Split masks for illegal stores.
+ if (NumStores > 1 && Mask)
+ return false;
+
+ if (!isa<StoreInst>(Store) &&
+ !match(Store, m_Intrinsic<Intrinsic::masked_store>()))
+ return false;
+
VectorType *StTy =
VectorType::get(VTy->getElementType(),
VTy->getElementCount().divideCoefficientBy(NumStores));
- Type *PtrTy = SI->getPointerOperandType();
- Function *StNFunc = getStructuredStoreFunction(SI->getModule(), Factor,
- UseScalable, StTy, PtrTy);
+ Value *BaseAddr = Store->getOperand(1);
+ Type *PtrTy = BaseAddr->getType();
+ Function *StNFunc = getStructuredStoreFunction(
+ Store->getModule(), Factor, UseScalable, InterleaveSegments, StTy, PtrTy);
- IRBuilder<> Builder(SI);
+ IRBuilder<> Builder(Store);
- Value *BaseAddr = SI->getPointerOperand();
Value *Pred = nullptr;
- if (UseScalable)
- Pred =
- Builder.CreateVectorSplat(StTy->getElementCount(), Builder.getTrue());
+ if (UseScalable) {
+ if (Mask)
+ Pred = Mask;
+ else if (InterleaveSegments) {
+ // Create a ptrue based on the element type, but then reinterpret it as
+ // nxv1i1. This avoids generating splats of nxv1i1 type, which aren't
+ // natively supported.
+ Pred =
+ Builder.CreateVectorSplat(StTy->getElementCount(), Builder.getTrue());
+ Pred = Builder.CreateIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool,
+ {Pred->getType()}, {Pred});
+ auto *QuadMaskTy =
+ ScalableVectorType::get(Type::getInt1Ty(Builder.getContext()), 1);
+ Pred = Builder.CreateIntrinsic(Intrinsic::aarch64_sve_convert_from_svbool,
+ {QuadMaskTy}, {Pred});
+ } else {
+ Pred =
+ Builder.CreateVectorSplat(StTy->getElementCount(), Builder.getTrue());
+ }
+ }
auto ExtractedValues = InterleavedValues;
SmallVector<Value *, 4> StoreOperands(InterleavedValues);
@@ -19210,6 +19693,7 @@ bool AArch64TargetLowering::lowerInterleaveIntrinsicToStore(
for (unsigned I = 0; I < NumStores; ++I) {
Value *Address = BaseAddr;
if (NumStores > 1) {
+ assert(!Mask && "Expected legal StTy when masking is used.");
Value *Offset = Builder.getInt64(I * Factor);
Address = Builder.CreateGEP(StTy, BaseAddr, {Offset});
Value *Idx =
@@ -21917,6 +22401,35 @@ static SDValue performConcatVectorsCombine(SDNode *N,
return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, Ops);
}
+ // Rewrite VECTOR_INTERLEAVE_SEGMENTS(V0, V1) when the result type is packed.
+ // This means V0 and V1 are unpacked and can be re-interpreted as vectors with
+ // double the elements but every odd element is undefined.
+ //
+ // Interleaving 64-bit segments into a packed type means that within each
+ // output quad, the lo and hi elements will come from V0 and V1 respectively.
+ // After reinterpreting V0 and V1, this corresponds to uzpq1.
+ //
+ // E.g. nxv4i16, nxv4i16 = VECTOR_INTERLEAVE_SEGMENTS(nxv4i16 V0, nxv4i16 V1)
+ // nxv8i16 = CONCAT_VECTORS interleaved.0, interleaved.1
+ // <=> nxv8i16 = uzpq1(nxv8i16 (packed_reinterpret V0),
+ // nxv8i16 (packed_reinterpret V1))
+ if (DAG.getSubtarget<AArch64Subtarget>().hasSVE2p1() &&
+ N->getNumOperands() == 2 && N0Opc == ISD::VECTOR_INTERLEAVE_SEGMENTS &&
+ N0.getNode() == N1.getNode() && N0->getNumOperands() == 2 &&
+ DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
+ isPackedVectorType(VT, DAG)) {
+ assert(VT.isScalableVector() &&
+ "VECTOR_INTERLEAVE_SEGMENTS for a fixed vector?");
+ assert(VT.isSimple() && "VECTOR_INTERLEAVE_SEGMENTS of extended types?");
+
+ SDValue ReinterpretedV0 = getReinterpretAsPackedVT(N0->getOperand(0), DAG);
+ SDValue ReinterpretedV1 = getReinterpretAsPackedVT(N0->getOperand(1), DAG);
+ SDValue IDVal =
+ DAG.getTargetConstant(Intrinsic::aarch64_sve_uzpq1, DL, MVT::i64);
+ SDValue Ops[] = {IDVal, ReinterpretedV0, ReinterpretedV1};
+ return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, Ops);
+ }
+
if (VT.isScalableVector())
return SDValue();
@@ -22173,10 +22686,48 @@ static SDValue performConcatVectorsCombine(SDNode *N,
static SDValue
performExtractSubvectorCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
SelectionDAG &DAG) {
+ EVT VT = N->getValueType(0);
+ EVT InVT = N->getOperand(0).getValueType();
+
+ // Detect doubleword de-interleave of which the lo half is extracted
+ // E.g. t4: nxv2i64,nxv2i64 = vector_deinterleave t3, t3
+ // t5: nxv4i32 = bitcast t4
+ // t7: nxv2i32 = extract_subvector t5, Constant:i64<0>
+ // Transform to:
+ // t10: nxv4i32 = zipq1 (bitcast t3 to nxv4i32), (...)
+ // t11: nxv2i32 = unpacked_reinterpret t10
+ if (DAG.getSubtarget<AArch64Subtarget>().hasSVE2p1() &&
+ VT.getSizeInBits() == TypeSize(64, /*Scalable=*/true) &&
+ InVT.getSizeInBits() == TypeSize(128, /*Scalable=*/true) &&
+ DAG.getTargetLoweringInfo().isTypeLegal(InVT) &&
+ isNullConstant(N->getOperand(1))) {
+ SDValue InV = N->getOperand(0);
+
+ if (InV.hasOneUse() && InV->getOpcode() == ISD::BITCAST) {
+ SDValue OrigV = InV->getOperand(0);
+ if (OrigV.hasOneUse() &&
+ OrigV.getValueType().getScalarSizeInBits() == 64 &&
+ OrigV->getOpcode() == ISD::VECTOR_DEINTERLEAVE &&
+ OrigV->getNumOperands() == 2) {
+ // Now we know we are deinterleaving doublewords from 2 sources, and
+ // later extracting the lo half. This is equivalent to a ZIPQ1 for even
+ // doublewords, and ZIPQ2 otherwise.
+ Intrinsic::ID IID = OrigV.getResNo() == 0
+ ? Intrinsic::aarch64_sve_zipq1
+ : Intrinsic::aarch64_sve_zipq2;
+ SDLoc DL(N);
+ SDValue IDVal = DAG.getTargetConstant(IID, DL, MVT::i64);
+ SDValue Src = DAG.getBitcast(InVT, OrigV->getOperand(0));
+ SDValue Zipq =
+ DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, InVT, IDVal, Src, Src);
+ return getReinterpretAsUnpackedVT(Zipq, VT, DAG);
+ }
+ }
+ }
+
if (DCI.isBeforeLegalizeOps())
return SDValue();
- EVT VT = N->getValueType(0);
SDValue V = N->getOperand(0);
// Fixed-length splats get legalized to a scalable splat + fixed-length
@@ -26716,6 +27267,66 @@ static SDValue performInterleavedMaskedStoreCombine(
return DAG.getNode(ISD::INTRINSIC_VOID, DL, MVT::Other, NewStOps);
}
+/// Traverse a chain of VECTOR_STRETCH nodes until the source has an
+/// element count greater than \p MinSupportedEC.
+/// Returns the last traversed VECTOR_STRETCH or SDValue().
+static SDValue traverseVectorStretch(
+ SDValue V, ElementCount MinSupportedEC = ElementCount::getScalable(1)) {
+ SDValue StretchedNode = V;
+ while (StretchedNode->getOpcode() == ISD::VECTOR_STRETCH &&
+ ElementCount::isKnownGE(
+ StretchedNode.getOperand(0).getValueType().getVectorElementCount(),
+ MinSupportedEC)) {
+ StretchedNode = StretchedNode.getOperand(0);
+ }
+ if (StretchedNode != V)
+ return StretchedNode;
+ return SDValue();
+}
+
+static SDValue performMLOADCombine(SDNode *N,
+ TargetLowering::DAGCombinerInfo &DCI,
+ SelectionDAG &DAG,
+ const AArch64Subtarget *Subtarget) {
+ auto *MLD = cast<MaskedLoadSDNode>(N);
+ SDValue Mask = MLD->getMask();
+ SDLoc DL(N);
+
+ // We'll try to fold vector_stretch by using a larger element type.
+ // This cannot work if the memory and register sizes do not match 1:1.
+ if (MLD->isExpandingLoad() ||
+ MLD->getExtensionType() != llvm::ISD::NON_EXTLOAD)
+ return SDValue();
+
+ // No point in folding vector_stretch if the to-be-introduced bitcast
+ // is not a noop.
+ if (!Subtarget->isLittleEndian())
+ return SDValue();
+
+ // Only look at legal masked_loads
+ if (!DCI.isAfterLegalizeDAG())
+ return SDValue();
+ assert((MLD->getPassThru()->isUndef() ||
+ isZerosVector(MLD->getPassThru().getNode())) &&
+ "expected passthru value to have been lowered");
+
+ if (SDValue StretchedMask = traverseVectorStretch(
+ Mask, /*MinSupportedEC=*/ElementCount::getScalable(2))) {
+ ElementCount NewEC = StretchedMask.getValueType().getVectorElementCount();
+ MVT MidVT = MVT::getSameSizeVT(MLD->getSimpleValueType(0), NewEC);
+ SDValue WiderElementLoad = DAG.getMaskedLoad(
+ MidVT, DL, MLD->getChain(), MLD->getBasePtr(), MLD->getOffset(),
+ StretchedMask, DAG.getConstant(0, DL, MidVT), MidVT,
+ MLD->getMemOperand(), MLD->getAddressingMode(),
+ MLD->getExtensionType());
+ SDValue Cast = DAG.getBitcast(N->getValueType(0), WiderElementLoad);
+ SDValue LoadChain(WiderElementLoad.getNode(), 1);
+ return DAG.getMergeValues({Cast, LoadChain}, DL);
+ }
+
+ return SDValue();
+}
+
static SDValue performMSTORECombine(SDNode *N,
TargetLowering::DAGCombinerInfo &DCI,
SelectionDAG &DAG,
@@ -26772,6 +27383,21 @@ static SDValue performMSTORECombine(SDNode *N,
MST->getMemoryVT(), MST->getMemOperand(),
MST->getAddressingMode(), true);
}
+ } else if (!MST->isCompressingStore() && Subtarget->isLittleEndian() &&
+ DCI.isAfterLegalizeDAG()) {
+ // Try to fold vector_stretch by using a larger element type.
+ // Note: There is no point in folding vector_stretch if the to-be-introduced
+ // bitcast is not a noop.
+
+ if (SDValue StretchedMask = traverseVectorStretch(
+ Mask, /*MinSupportedEC=*/ElementCount::getScalable(2))) {
+ ElementCount NewEC = StretchedMask.getValueType().getVectorElementCount();
+ MVT MidVT = MVT::getSameSizeVT(Value.getSimpleValueType(), NewEC);
+ SDValue Cast = DAG.getBitcast(MidVT, Value);
+ return DAG.getMaskedStore(MST->getChain(), DL, Cast, MST->getBasePtr(),
+ MST->getOffset(), StretchedMask, MidVT,
+ MST->getMemOperand(), MST->getAddressingMode());
+ }
}
return SDValue();
@@ -28416,6 +29042,24 @@ static SDValue performVSelectCombine(SDNode *N,
if (SDValue R = performVselectPowCombine(N, DCI))
return R;
+ if (Subtarget->isLittleEndian()) {
+ // Try to fold vector_stretch by using a larger element type.
+ // Note: There is no point in folding vector_stretch if the to-be-introduced
+ // bitcasts aren't noops.
+
+ if (SDValue StretchedMask = traverseVectorStretch(
+ N0, /*MinSupportedEC=*/ElementCount::getScalable(2))) {
+ ElementCount NewEC = StretchedMask.getValueType().getVectorElementCount();
+ MVT MidVT = MVT::getSameSizeVT(ResVT.getSimpleVT(), NewEC);
+ SDValue IfTrueCast = DAG.getBitcast(MidVT, IfTrue);
+ SDValue IfFalseCast = DAG.getBitcast(MidVT, IfFalse);
+ SDValue WiderEltSelect =
+ DAG.getNode(ISD::VSELECT, SDLoc(N), MidVT, StretchedMask, IfTrueCast,
+ IfFalseCast);
+ return DAG.getBitcast(ResVT, WiderEltSelect);
+ }
+ }
+
EVT CmpVT = N0.getOperand(0).getValueType();
if (N0.getOpcode() != ISD::SETCC ||
CCVT.getVectorElementCount() != ElementCount::getFixed(1) ||
@@ -29873,6 +30517,8 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
return performSETCCCombine(N, DCI, DAG);
case ISD::LOAD:
return performLOADCombine(N, DCI, DAG, Subtarget);
+ case ISD::MLOAD:
+ return performMLOADCombine(N, DCI, DAG, Subtarget);
case ISD::STORE:
return performSTORECombine(N, DCI, DAG, Subtarget);
case ISD::MSTORE:
@@ -29959,6 +30605,9 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
case ISD::INTRINSIC_VOID:
case ISD::INTRINSIC_W_CHAIN:
switch (N->getConstantOperandVal(1)) {
+ case Intrinsic::masked_segment_gather:
+ case Intrinsic::masked_segment_scatter:
+ return performMaskedSegmentIntrinsicCombine(N, DCI, DAG);
case Intrinsic::aarch64_sve_prfb_gather_scalar_offset:
return combineSVEPrefetchVecBaseImmOff(N, DAG, 1 /*=ScalarSizeInBytes*/);
case Intrinsic::aarch64_sve_prfh_gather_scalar_offset:
@@ -33078,6 +33727,30 @@ AArch64TargetLowering::LowerVECTOR_DEINTERLEAVE(SDValue Op,
return DAG.getMergeValues({Even, Odd}, DL);
}
+SDValue AArch64TargetLowering::LowerVECTOR_DEINTERLEAVE_SEGMENTS(
+ SDValue Op, SelectionDAG &DAG) const {
+ SDLoc DL(Op);
+ EVT OpVT = Op.getValueType();
+ assert(OpVT.isScalableVector() &&
+ "Expected scalable vector in LowerVECTOR_DEINTERLEAVE_SEGMENTS.");
+
+ // Expand until we reach a deinterleave factor of 2.
+ if (Op->getNumOperands() != 2)
+ return SDValue();
+
+ assert(OpVT.getSizeInBits().getKnownMinValue() == AArch64::SVEBitsPerBlock &&
+ "VECTOR_DEINTERLEAVE_SEGMENTS only expected for legal SVE vectors.");
+ SDValue Even =
+ DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, OpVT,
+ DAG.getConstant(Intrinsic::aarch64_sve_uzp1q, DL, MVT::i64),
+ Op.getOperand(0), Op.getOperand(1));
+ SDValue Odd =
+ DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, OpVT,
+ DAG.getConstant(Intrinsic::aarch64_sve_uzp2q, DL, MVT::i64),
+ Op.getOperand(0), Op.getOperand(1));
+ return DAG.getMergeValues({Even, Odd}, DL);
+}
+
SDValue AArch64TargetLowering::LowerVECTOR_INTERLEAVE(SDValue Op,
SelectionDAG &DAG) const {
SDLoc DL(Op);
@@ -33188,6 +33861,28 @@ SDValue AArch64TargetLowering::LowerVECTOR_INTERLEAVE(SDValue Op,
return DAG.getMergeValues({Lo, Hi}, DL);
}
+SDValue AArch64TargetLowering::LowerVECTOR_INTERLEAVE_SEGMENTS(
+ SDValue Op, SelectionDAG &DAG) const {
+ SDLoc DL(Op);
+ EVT OpVT = Op.getValueType();
+ assert(OpVT.isScalableVector() &&
+ "Expected scalable vector in LowerVECTOR_INTERLEAVE_SEGMENTS.");
+
+ // Expand until we reach an interleave factor of 2.
+ if (Op->getNumOperands() != 2)
+ return SDValue();
+
+ SDValue Lo =
+ DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, OpVT,
+ DAG.getConstant(Intrinsic::aarch64_sve_zip1q, DL, MVT::i64),
+ Op.getOperand(0), Op.getOperand(1));
+ SDValue Hi =
+ DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, OpVT,
+ DAG.getConstant(Intrinsic::aarch64_sve_zip2q, DL, MVT::i64),
+ Op.getOperand(0), Op.getOperand(1));
+ return DAG.getMergeValues({Lo, Hi}, DL);
+}
+
SDValue AArch64TargetLowering::LowerVECTOR_HISTOGRAM(SDValue Op,
SelectionDAG &DAG) const {
// FIXME: Maybe share some code with LowerMGather/Scatter?
@@ -33563,7 +34258,7 @@ SDValue AArch64TargetLowering::LowerFixedLengthVECTOR_SHUFFLEToSVE(
bool ReverseEXT = false;
unsigned Imm;
- if (isEXTMask(ShuffleMask, VT, ReverseEXT, Imm) &&
+ if (isEXTMask(ShuffleMask, VT.getVectorNumElements(), ReverseEXT, Imm) &&
Imm == VT.getVectorNumElements() - 1) {
if (ReverseEXT)
std::swap(Op1, Op2);
@@ -33576,8 +34271,9 @@ SDValue AArch64TargetLowering::LowerFixedLengthVECTOR_SHUFFLEToSVE(
}
unsigned EltSize = VT.getScalarSizeInBits();
+ unsigned NumElts = VT.getVectorNumElements();
for (unsigned BlockSize : {64U, 32U, 16U}) {
- if (isREVMask(ShuffleMask, EltSize, VT.getVectorNumElements(), BlockSize)) {
+ if (isREVMask(ShuffleMask, EltSize, NumElts, BlockSize)) {
unsigned RevOp;
if (EltSize == 8)
RevOp = AArch64ISD::BSWAP_MERGE_PASSTHRU;
@@ -33598,7 +34294,7 @@ SDValue AArch64TargetLowering::LowerFixedLengthVECTOR_SHUFFLEToSVE(
}
if (Subtarget->hasSVE2p1() && EltSize == 64 &&
- isREVMask(ShuffleMask, EltSize, VT.getVectorNumElements(), 128)) {
+ isREVMask(ShuffleMask, EltSize, NumElts, 128)) {
SDValue Pg = getPredicateForVector(DAG, DL, VT);
SDValue Revd = DAG.getNode(AArch64ISD::REVD_MERGE_PASSTHRU, DL, ContainerVT,
Pg, Op1, DAG.getPOISON(ContainerVT));
@@ -33607,8 +34303,7 @@ SDValue AArch64TargetLowering::LowerFixedLengthVECTOR_SHUFFLEToSVE(
unsigned WhichResult;
unsigned OperandOrder;
- if (isZIPMask(ShuffleMask, VT.getVectorNumElements(), WhichResult,
- OperandOrder) &&
+ if (isZIPMask(ShuffleMask, NumElts, WhichResult, OperandOrder) &&
WhichResult == 0) {
SDValue ZIP = DAG.getNode(AArch64ISD::ZIP1, DL, ContainerVT,
OperandOrder == 0 ? Op1 : Op2,
@@ -33616,8 +34311,7 @@ SDValue AArch64TargetLowering::LowerFixedLengthVECTOR_SHUFFLEToSVE(
return convertFromScalableVector(DAG, VT, ZIP);
}
- if (isTRNMask(ShuffleMask, VT.getVectorNumElements(), WhichResult,
- OperandOrder)) {
+ if (isTRNMask(ShuffleMask, NumElts, WhichResult, OperandOrder)) {
unsigned Opc = (WhichResult == 0) ? AArch64ISD::TRN1 : AArch64ISD::TRN2;
SDValue TRN =
DAG.getNode(Opc, DL, ContainerVT, OperandOrder == 0 ? Op1 : Op2,
@@ -33625,11 +34319,11 @@ SDValue AArch64TargetLowering::LowerFixedLengthVECTOR_SHUFFLEToSVE(
return convertFromScalableVector(DAG, VT, TRN);
}
- if (isZIP_v_undef_Mask(ShuffleMask, VT, WhichResult) && WhichResult == 0)
+ if (isZIP_v_undef_Mask(ShuffleMask, NumElts, WhichResult) && WhichResult == 0)
return convertFromScalableVector(
DAG, VT, DAG.getNode(AArch64ISD::ZIP1, DL, ContainerVT, Op1, Op1));
- if (isTRN_v_undef_Mask(ShuffleMask, VT, WhichResult)) {
+ if (isTRN_v_undef_Mask(ShuffleMask, NumElts, WhichResult)) {
unsigned Opc = (WhichResult == 0) ? AArch64ISD::TRN1 : AArch64ISD::TRN2;
return convertFromScalableVector(
DAG, VT, DAG.getNode(Opc, DL, ContainerVT, Op1, Op1));
@@ -33677,11 +34371,12 @@ SDValue AArch64TargetLowering::LowerFixedLengthVECTOR_SHUFFLEToSVE(
DAG, VT, DAG.getNode(Opc, DL, ContainerVT, Op1, Op2));
}
- if (isZIP_v_undef_Mask(ShuffleMask, VT, WhichResult) && WhichResult != 0)
+ if (isZIP_v_undef_Mask(ShuffleMask, NumElts, WhichResult) &&
+ WhichResult != 0)
return convertFromScalableVector(
DAG, VT, DAG.getNode(AArch64ISD::ZIP2, DL, ContainerVT, Op1, Op1));
- if (isUZP_v_undef_Mask(ShuffleMask, VT, WhichResult)) {
+ if (isUZP_v_undef_Mask(ShuffleMask, NumElts, WhichResult)) {
unsigned Opc = (WhichResult == 0) ? AArch64ISD::UZP1 : AArch64ISD::UZP2;
return convertFromScalableVector(
DAG, VT, DAG.getNode(Opc, DL, ContainerVT, Op1, Op1));
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 7e4c4e1ba25ff..4c81d0469a50b 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -159,6 +159,9 @@ class AArch64TargetLowering : public TargetLowering {
/// should be stack expanded.
bool isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const override;
+ /// Return true if the given segmented shuffle mask can be codegen'd directly
+ bool isSegmentedShuffleMaskSupported(ArrayRef<int> M) const;
+
/// Similar to isShuffleMaskLegal. Return true is the given 'select with zero'
/// shuffle mask can be codegen'd directly.
bool isVectorClearMaskLegal(ArrayRef<int> M, EVT VT) const override;
@@ -237,13 +240,14 @@ class AArch64TargetLowering : public TargetLowering {
ShuffleVectorInst *SVI, unsigned Factor,
const APInt &GapMask) const override;
- bool lowerDeinterleaveIntrinsicToLoad(Instruction *Load, Value *Mask,
- IntrinsicInst *DI,
- const APInt &GapMask) const override;
+ bool
+ lowerDeinterleaveIntrinsicToLoad(Instruction *Load, Value *Mask,
+ IntrinsicInst *DI, const APInt &GapMask,
+ bool DeinterleaveSegments) const override;
- bool lowerInterleaveIntrinsicToStore(
- Instruction *Store, Value *Mask,
- ArrayRef<Value *> InterleaveValues) const override;
+ bool lowerInterleaveIntrinsicToStore(Instruction *Store, Value *Mask,
+ ArrayRef<Value *> InterleaveValues,
+ bool InterleaveSegments) const override;
bool isLegalAddImmediate(int64_t) const override;
bool isLegalAddScalableImmediate(int64_t) const override;
@@ -510,8 +514,11 @@ class AArch64TargetLowering : public TargetLowering {
/// Returns true if \p VecTy is a legal interleaved access type. This
/// function checks the vector element type and the overall width of the
/// vector.
+ /// \p AccessSegments Whether segments of multiple elements are
+ /// (de)interleaved, as opposed to single elements.
bool isLegalInterleavedAccessType(VectorType *VecTy, const DataLayout &DL,
- bool &UseScalable) const;
+ bool &UseScalable,
+ bool AccessSegments = false) const;
/// Returns the number of interleaved accesses that will be generated when
/// lowering accesses of the given type.
@@ -746,16 +753,21 @@ class AArch64TargetLowering : public TargetLowering {
SDValue LowerEXTEND_VECTOR_INREG(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerZERO_EXTEND_VECTOR_INREG(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const;
+ SDValue LowerVECTOR_SEGMENTED_SHUFFLE(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerSPLAT_VECTOR(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerDUPQLane(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerToPredicatedOp(SDValue Op, SelectionDAG &DAG,
unsigned NewOp) const;
SDValue LowerToScalableOp(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerVECTOR_SPLICE(SDValue Op, SelectionDAG &DAG) const;
+ SDValue LowerVECTOR_STRETCH(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerINSERT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerVECTOR_DEINTERLEAVE(SDValue Op, SelectionDAG &DAG) const;
+ SDValue LowerVECTOR_DEINTERLEAVE_SEGMENTS(SDValue Op,
+ SelectionDAG &DAG) const;
SDValue LowerVECTOR_INTERLEAVE(SDValue Op, SelectionDAG &DAG) const;
+ SDValue LowerVECTOR_INTERLEAVE_SEGMENTS(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerVECTOR_HISTOGRAM(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerPARTIAL_REDUCE_MLA(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerGET_ACTIVE_LANE_MASK(SDValue Op, SelectionDAG &DAG) const;
@@ -787,6 +799,8 @@ class AArch64TargetLowering : public TargetLowering {
SDValue LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerVECREDUCE(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerVECREDUCE_MUL(SDValue Op, SelectionDAG &DAG) const;
+ SDValue LowerPARTIAL_REDUCE_ToFixed(SDValue Op, SelectionDAG &DAG,
+ Intrinsic::ID ToIID) const;
SDValue LowerATOMIC_LOAD_AND(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerWindowsDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerInlineDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 63f5a680bbf1a..05e53e5a2083e 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -1985,6 +1985,22 @@ let Predicates = [HasSVE_or_SME] in {
defm TRN1_PPP : sve_int_perm_bin_perm_pp<0b100, "trn1", AArch64trn1, int_aarch64_sve_trn1_b16, int_aarch64_sve_trn1_b32, int_aarch64_sve_trn1_b64>;
defm TRN2_PPP : sve_int_perm_bin_perm_pp<0b101, "trn2", AArch64trn2, int_aarch64_sve_trn2_b16, int_aarch64_sve_trn2_b32, int_aarch64_sve_trn2_b64>;
+ // Stretch predicates.
+ def : Pat<(nxv2i1 (vector_stretch nxv1i1:$Ps)),
+ (TRN1_PPP_D PPR:$Ps, PPR:$Ps)>;
+ def : Pat<(nxv4i1 (vector_stretch nxv2i1:$Ps)),
+ (TRN1_PPP_S PPR:$Ps, PPR:$Ps)>;
+ def : Pat<(nxv8i1 (vector_stretch nxv4i1:$Ps)),
+ (TRN1_PPP_H PPR:$Ps, PPR:$Ps)>;
+ def : Pat<(nxv16i1 (vector_stretch nxv8i1:$Ps)),
+ (TRN1_PPP_B PPR:$Ps, PPR:$Ps)>;
+
+ // Unpack then stretch predicates
+ def : Pat<(nxv2i1 (vector_stretch (extract_subvector nxv2i1:$Ps, (i64 0)))),
+ (ZIP1_PPP_D PPR:$Ps, PPR:$Ps)>;
+ def : Pat<(nxv2i1 (vector_stretch (extract_subvector nxv2i1:$Ps, (i64 1)))),
+ (ZIP2_PPP_D PPR:$Ps, PPR:$Ps)>;
+
// Extract lo/hi halves of legal predicate types.
def : Pat<(nxv1i1 (extract_subvector nxv2i1:$Ps, (i64 0))),
(PUNPKLO_PP PPR:$Ps)>;
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 52f9f51582c94..b6a5a2443997c 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -34,6 +34,73 @@ using namespace llvm::PatternMatch;
#define DEBUG_TYPE "aarch64tti"
+// Define the struct for the generated intrinsic mapping table
+namespace {
+struct NEONToSVEIntrinsicMapping {
+ StringRef NEONIntrinsic;
+ StringRef SVEIntrinsic;
+ StringRef ArgMappings; // Comma-separated string: "T,Z,A:0,A:1"
+ bool isCustom() const { return SVEIntrinsic.empty() || ArgMappings.empty(); }
+};
+
+/// Specifies how to build an argument for an SVE intrinsic, possibly from
+/// another argument of an equivalent NEON intrinsic.
+class ArgMapping {
+public:
+ enum MappingKind { True, Zero, Poison, Arg, Imm };
+
+ ArgMapping(StringRef Spec) {
+ if (Spec == "T") {
+ Kind = True;
+ } else if (Spec == "Z") {
+ Kind = Zero;
+ } else if (Spec == "P") {
+ Kind = Poison;
+ } else if (Spec.starts_with("A:")) {
+ Kind = Arg;
+ unsigned Idx;
+ if (Spec.substr(2).getAsInteger(10, Idx))
+ llvm_unreachable("Invalid argument index in mapping!");
+ Index = Idx;
+ } else if (Spec.starts_with("I:")) {
+ Kind = Imm;
+ unsigned Idx;
+ if (Spec.substr(2).getAsInteger(10, Idx))
+ llvm_unreachable("Invalid argument index in mapping!");
+ Index = Idx;
+ } else {
+ llvm_unreachable("Unknown argument mapping kind!");
+ }
+ }
+
+ MappingKind getKind() const { return Kind; }
+
+ unsigned getArgIndex() const {
+ assert((Kind == Arg || Kind == Imm) &&
+ "getArgIndex called on non-Arg mapping");
+ return *Index;
+ }
+
+private:
+ MappingKind Kind;
+ std::optional<unsigned> Index;
+};
+
+// Parse comma-separated argument mapping string into ArgMapping objects
+static SmallVector<ArgMapping, 4> parseArgMappings(StringRef ArgMappingsStr) {
+ SmallVector<ArgMapping, 4> Result;
+ SmallVector<StringRef, 4> Specs;
+ ArgMappingsStr.split(Specs, ',');
+ for (StringRef Spec : Specs)
+ Result.emplace_back(Spec);
+ return Result;
+}
+} // end anonymous namespace
+
+// Include generated vector intrinsic mapping table
+#define GET_IntrinsicMappingTable_IMPL
+#include "AArch64GenVectorIntrinsicMappings.inc"
+
static cl::opt<bool> EnableFalkorHWPFUnrollFix("enable-falkor-hwpf-unroll-fix",
cl::init(true), cl::Hidden);
@@ -597,7 +664,7 @@ static InstructionCost getHistogramCost(const AArch64Subtarget *ST,
// using ptrue with a specific VL.
if (VectorType *VTy = dyn_cast<VectorType>(BucketPtrsTy)) {
unsigned EC = VTy->getElementCount().getKnownMinValue();
- if (!isPowerOf2_64(EC) || !VTy->isScalableTy())
+ if (!isPowerOf2_64(EC) || !VTy->isScalableTy() || EC == 1)
return InstructionCost::getInvalid();
// HistCnt only supports 32b and 64b element types
@@ -624,9 +691,16 @@ AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
// sufficiently reliable.
auto *RetTy = ICA.getReturnType();
if (auto *VTy = dyn_cast<ScalableVectorType>(RetTy))
- if (VTy->getElementCount() == ElementCount::getScalable(1))
+ if (VTy->getElementCount() == ElementCount::getScalable(1) &&
+ ICA.getID() != Intrinsic::masked_load &&
+ ICA.getID() != Intrinsic::masked_store)
return InstructionCost::getInvalid();
+ auto IsAcceptableTyForREVEC = [](const Type *Ty) {
+ return Ty->getPrimitiveSizeInBits().getKnownMinValue() <=
+ AArch64::SVEBitsPerBlock;
+ };
+
switch (ICA.getID()) {
case Intrinsic::experimental_vector_histogram_add: {
InstructionCost HistCost = getHistogramCost(ST, ICA);
@@ -1191,7 +1265,82 @@ AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
}
break;
}
+ case Intrinsic::vector_broadcast: {
+ // Duplicating quads requires SVE2p1 or SME2p1.
+ return (ST->hasSVE2p1() || ST->hasSME2p1()) ? InstructionCost(1)
+ : InstructionCost::getInvalid();
+ }
+ case Intrinsic::vector_partial_reduce_add:
+ case Intrinsic::vector_partial_reduce_smax:
+ case Intrinsic::vector_partial_reduce_umax:
+ case Intrinsic::vector_partial_reduce_smin:
+ case Intrinsic::vector_partial_reduce_umin: {
+ // TODO-REVEC: Give sensible costs based on types.
+ // Partial reductions from scalable to fixed vectors require SVE2p1.
+ return ST->hasSVE2p1() ? InstructionCost(1) : InstructionCost::getInvalid();
+ }
+ case Intrinsic::vector_deinterleave_segments2: {
+ // UZP of 64-bit elements or smaller only requires SVE.
+ // TODO-REVEC: Actually allow segments of e.g. 32 bits. Currently codegen
+ // isn't amazing and requires +f64mm.
+ unsigned SegmentSizeInBits =
+ RetTy->getPrimitiveSizeInBits().getKnownMinValue();
+ return SegmentSizeInBits == (AArch64::SVEBitsPerBlock / 2) && ST->hasSVE()
+ ? InstructionCost(1)
+ : InstructionCost::getInvalid();
+ }
+ case Intrinsic::vector_interleave_segments2: {
+ // Interleaving of scalable 64-bit vectors is selected using uzpq1.
+ // Interleaving of scalable 128-bit vectors is selected using zipq1+zipq2
+ // Both require SVE2.1.
+ unsigned SegmentSizeInBits =
+ RetTy->getPrimitiveSizeInBits().getKnownMinValue();
+ return ST->hasSVE2p1() &&
+ (SegmentSizeInBits == (AArch64::SVEBitsPerBlock / 2) ||
+ SegmentSizeInBits == (AArch64::SVEBitsPerBlock))
+ ? InstructionCost(1)
+ : InstructionCost::getInvalid();
+ }
+ case Intrinsic::aarch64_neon_sshl:
+ case Intrinsic::aarch64_neon_ushl: {
+ // Re-vectorising NEON's non-saturating non-rounding bi-directional shl
+ // requires the use of two SVE bi-directional shl: one for the non-rounding
+ // part and one for the non-saturating part. (+masking)
+ if (!IsAcceptableTyForREVEC(RetTy))
+ return InstructionCost::getInvalid();
+ return RetTy->isScalableTy() ? InstructionCost(2) : InstructionCost(1);
+ }
+ case Intrinsic::aarch64_neon_addp:
+ case Intrinsic::aarch64_neon_faddp: {
+ // Re-vectorising NEON's ADDP requires prior deinterleaving because SVE does
+ // not have have a ADDPQ variant working like NEON within quads.
+ // Re-vectorising 64-bit NEON requires even more instructions.
+ if (!IsAcceptableTyForREVEC(RetTy))
+ return InstructionCost::getInvalid();
+ return RetTy->isScalableTy() ? InstructionCost(2) : InstructionCost(1);
+ }
+ case Intrinsic::aarch64_neon_tbl1:
+ case Intrinsic::aarch64_neon_tbl2:
+ case Intrinsic::aarch64_neon_tbl3:
+ case Intrinsic::aarch64_neon_tbl4: {
+ assert(ICA.getArgs().size() >= 2 &&
+ "Tbl needs at least one source and one mask");
+ if (!IsAcceptableTyForREVEC(RetTy) ||
+ !all_of(ICA.getArgTypes(), IsAcceptableTyForREVEC))
+ return InstructionCost::getInvalid();
+ const unsigned NumSrcs = ICA.getArgs().size() - 1;
+ return RetTy->isScalableTy() ? InstructionCost(NumSrcs)
+ : InstructionCost(1);
+ }
default:
+ // Only allow REVEC of NEON intrinsics when the types are at most scalable
+ // 128-bit vectors. This avoids crashes when forcing e.g. VF = vscale x 2.
+ if (isTargetIntrinsicVectorizable(ICA.getID()) && RetTy->isScalableTy()) {
+ return (IsAcceptableTyForREVEC(RetTy) &&
+ all_of(ICA.getArgTypes(), IsAcceptableTyForREVEC))
+ ? InstructionCost(1)
+ : InstructionCost::getInvalid();
+ }
break;
}
return BaseT::getIntrinsicInstrCost(ICA, CostKind);
@@ -3382,6 +3531,14 @@ InstructionCost AArch64TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
const Instruction *I) const {
int ISD = TLI->InstructionOpcodeToISD(Opcode);
assert(ISD && "Invalid opcode");
+
+ // TODO-REVEC: Some casts for nxv1 types are supported, but others like
+ // float<->int conversions aren't.
+ // Until codegen works for every type, let's be conservative.
+ if (auto *VTy = dyn_cast<VectorType>(Dst);
+ VTy && VTy->getElementCount() == ElementCount::getScalable(1))
+ return InstructionCost::getInvalid();
+
// If the cast is observable, and it is used by a widening instruction (e.g.,
// uaddl, saddw, etc.), it may be free.
if (I && I->hasOneUser()) {
@@ -4366,11 +4523,13 @@ InstructionCost AArch64TTIImpl::getArithmeticInstrCost(
// The code-generator is currently not able to handle scalable vectors
// of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting
- // it. This change will be removed when code-generation for these types is
- // sufficiently reliable.
+ // it until all instructions are vetted.
+ int ISD = TLI->InstructionOpcodeToISD(Opcode);
+ static const std::set<int> Vettedx1Ops = {ISD::ADD, ISD::SUB, ISD::MUL};
if (auto *VTy = dyn_cast<ScalableVectorType>(Ty))
if (VTy->getElementCount() == ElementCount::getScalable(1))
- return InstructionCost::getInvalid();
+ if (!Vettedx1Ops.count(ISD))
+ return InstructionCost::getInvalid();
// TODO: Handle more cost kinds.
if (CostKind != TTI::TCK_RecipThroughput)
@@ -4379,7 +4538,6 @@ InstructionCost AArch64TTIImpl::getArithmeticInstrCost(
// Legalize the type.
std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
- int ISD = TLI->InstructionOpcodeToISD(Opcode);
// Increase the cost for half and bfloat types if not architecturally
// supported.
@@ -4906,6 +5064,9 @@ AArch64TTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
case Intrinsic::masked_scatter:
case Intrinsic::masked_gather:
return getGatherScatterOpCost(MICA, CostKind);
+ case Intrinsic::masked_segment_scatter:
+ case Intrinsic::masked_segment_gather:
+ return getSegmentGatherScatterOpCost(MICA, CostKind);
case Intrinsic::masked_load:
case Intrinsic::masked_expandload:
case Intrinsic::masked_store:
@@ -4930,12 +5091,9 @@ AArch64TTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
if (VT->getElementType()->isIntegerTy(1))
return InstructionCost::getInvalid();
- // The code-generator is currently not able to handle scalable vectors
- // of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting
- // it. This change will be removed when code-generation for these types is
- // sufficiently reliable.
+ // <vscale x 1 x eltty> operations require mask adaptation
if (VT->getElementCount() == ElementCount::getScalable(1))
- return InstructionCost::getInvalid();
+ return LT.first + 1;
InstructionCost MemOpCost = LT.first;
if (MICA.getID() == Intrinsic::masked_expandload) {
@@ -5024,6 +5182,20 @@ AArch64TTIImpl::getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
return LT.first * MemOpCost * getMaxNumElements(LegalVF);
}
+InstructionCost AArch64TTIImpl::getSegmentGatherScatterOpCost(
+ const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const {
+ // TODO-REVEC: This needs proper costs.
+ Type *DataTy = MICA.getDataType();
+ unsigned SegmentSizeInBits =
+ DataTy->getPrimitiveSizeInBits().getKnownMinValue();
+ assert((SegmentSizeInBits == 64U || SegmentSizeInBits == 128U) &&
+ "Illegal segment size.");
+ unsigned SegmentsPerQuad = AArch64::SVEBitsPerBlock / SegmentSizeInBits;
+ unsigned VScaleEstimate = std::max(ST->getVScaleForTuning(), 2U);
+ return VScaleEstimate * SegmentsPerQuad;
+}
+
bool AArch64TTIImpl::useNeonVector(const Type *Ty) const {
return isa<FixedVectorType>(Ty) && !ST->useSVEForFixedLengthVectors();
}
@@ -5045,16 +5217,23 @@ InstructionCost AArch64TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Ty,
if (!LT.first.isValid())
return InstructionCost::getInvalid();
- // The code-generator is currently not able to handle scalable vectors
- // of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting
- // it. This change will be removed when code-generation for these types is
- // sufficiently reliable.
+ // <vscale x 1 x eltty> operations require crafting a new mask
+ if (auto *VTy = dyn_cast<ScalableVectorType>(Ty)) {
+ if (VTy->getElementCount() == ElementCount::getScalable(1)) {
+ Intrinsic::ID IID = Opcode == Instruction::Load ? Intrinsic::masked_load
+ : Intrinsic::masked_store;
+ return getMaskedMemoryOpCost(
+ MemIntrinsicCostAttributes(IID, Ty, Alignment, AddressSpace),
+ CostKind) +
+ 1;
+ }
+ }
+
// We also only support full register predicate loads and stores.
if (auto *VTy = dyn_cast<ScalableVectorType>(Ty))
- if (VTy->getElementCount() == ElementCount::getScalable(1) ||
- (VTy->getElementType()->isIntegerTy(1) &&
- !VTy->getElementCount().isKnownMultipleOf(
- ElementCount::getScalable(16))))
+ if (VTy->getElementType()->isIntegerTy(1) &&
+ !VTy->getElementCount().isKnownMultipleOf(
+ ElementCount::getScalable(16)))
return InstructionCost::getInvalid();
// TODO: consider latency as well for TCK_SizeAndLatency.
@@ -5167,6 +5346,26 @@ InstructionCost AArch64TTIImpl::getInterleavedMemoryOpCost(
UseMaskForCond, UseMaskForGaps);
}
+InstructionCost AArch64TTIImpl::getSegmentInterleavedMemoryOpCost(
+ unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
+ Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
+ bool UseMaskForCond, bool UseMaskForGaps) const {
+
+ auto *VecVTy = cast<VectorType>(VecTy);
+ auto *SubVecTy = VectorType::getOneNthElementsVectorType(VecVTy, Factor);
+
+ bool UseScalable;
+ if (!TLI->isLegalInterleavedAccessType(SubVecTy, DL, UseScalable,
+ /*AccessSegments=*/true))
+ return InstructionCost::getInvalid();
+
+ // The mask might need stretching inside the loop.
+ unsigned StretchCost = UseMaskForCond ? 1U : 0U;
+ return StretchCost +
+ getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices, Alignment,
+ AddressSpace, CostKind, UseMaskForCond,
+ UseMaskForGaps);
+}
InstructionCost
AArch64TTIImpl::getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) const {
InstructionCost Cost = 0;
@@ -5656,6 +5855,370 @@ bool AArch64TTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst,
return true;
}
+bool AArch64TTIImpl::isSupportedTargetRecurrence(Intrinsic::ID ID,
+ RecurKind RK) const {
+ switch (ID) {
+ case Intrinsic::aarch64_neon_udot:
+ case Intrinsic::aarch64_neon_sdot:
+ case Intrinsic::aarch64_neon_usdot:
+ return RK == RecurKind::TargetIntAccumulation;
+ default:
+ return false;
+ }
+}
+
+bool AArch64TTIImpl::isTargetIntrinsicWithScalarOpAtArg(
+ Intrinsic::ID ID, unsigned ScalarOpdIdx) const {
+ switch (ID) {
+ case Intrinsic::aarch64_neon_rshrn:
+ case Intrinsic::aarch64_neon_sqrshrn:
+ case Intrinsic::aarch64_neon_sqrshrun:
+ case Intrinsic::aarch64_neon_sqshrn:
+ case Intrinsic::aarch64_neon_sqshrun:
+ case Intrinsic::aarch64_neon_uqrshrn:
+ case Intrinsic::aarch64_neon_uqshrn:
+ return ScalarOpdIdx == 1;
+ case Intrinsic::aarch64_neon_vsli:
+ case Intrinsic::aarch64_neon_vsri:
+ return ScalarOpdIdx == 2;
+ default:
+ return BaseT::isTargetIntrinsicWithScalarOpAtArg(ID, ScalarOpdIdx);
+ }
+}
+
+bool AArch64TTIImpl::isTargetIntrinsicWithOverloadTypeAtArg(Intrinsic::ID ID,
+ int OpdIdx) const {
+ switch (ID) {
+ case Intrinsic::aarch64_neon_facge:
+ case Intrinsic::aarch64_neon_facgt:
+ case Intrinsic::aarch64_neon_fcvtzs:
+ case Intrinsic::aarch64_neon_fcvtzu:
+ return OpdIdx == -1 || OpdIdx == 0;
+ case Intrinsic::aarch64_sve_facge:
+ case Intrinsic::aarch64_sve_facgt:
+ return OpdIdx == 1;
+ case Intrinsic::aarch64_sve_fcvtzs:
+ case Intrinsic::aarch64_sve_fcvtzu:
+ return OpdIdx == -1 || OpdIdx == 2;
+ case Intrinsic::aarch64_sve_fcvtx_f32f64:
+ case Intrinsic::aarch64_sve_fcvtxnt_f32f64:
+ return false;
+ case Intrinsic::aarch64_sve_sqxtnb:
+ case Intrinsic::aarch64_sve_sqxtnt:
+ case Intrinsic::aarch64_sve_sqxtunb:
+ case Intrinsic::aarch64_sve_sqxtunt:
+ case Intrinsic::aarch64_sve_uqxtnb:
+ case Intrinsic::aarch64_sve_uqxtnt:
+ return OpdIdx == 0;
+ case Intrinsic::aarch64_sve_rshrnb:
+ case Intrinsic::aarch64_sve_sqrshrnb:
+ case Intrinsic::aarch64_sve_sqrshrunb:
+ case Intrinsic::aarch64_sve_sqshrnb:
+ case Intrinsic::aarch64_sve_sqshrunb:
+ case Intrinsic::aarch64_sve_uqrshrnb:
+ case Intrinsic::aarch64_sve_uqshrnb:
+ return OpdIdx == 0;
+ default:
+ // For all other intrinsics, it's so far okay to pretend only the return
+ // type is overloaded.
+ return BaseT::isTargetIntrinsicWithOverloadTypeAtArg(ID, OpdIdx);
+ }
+}
+
+SmallVector<Type *, 2>
+AArch64TTIImpl::computeTysForDecl(Intrinsic::ID ID, Type *RetTy,
+ ArrayRef<Value *> Args) const {
+ SmallVector<Type *, 2> Tys;
+ if (isTargetIntrinsicWithOverloadTypeAtArg(ID, -1))
+ Tys.push_back(RetTy);
+ for (auto [ArgIdx, Arg] : enumerate(Args))
+ if (isTargetIntrinsicWithOverloadTypeAtArg(ID, ArgIdx))
+ Tys.push_back(Arg->getType());
+ return Tys;
+}
+
+bool AArch64TTIImpl::isTargetIntrinsicVectorizable(Intrinsic::ID ID) const {
+
+ // Need SVE2.1 to consider revectorisation of NEON intrinsics
+ if (!ST->hasSVE2p1())
+ return false;
+
+ return lookupNEONToSVEMappingByNEONIntrinsic(Intrinsic::getBaseName(ID));
+}
+
+/// Generate a segmented UZP shuffle that can later be selected to UZPQ.
+/// \pre Src0 and Src1 are scalable vectors
+Instruction *getSegmentedUZP(Value *Src0, Value *Src1, bool EvenElts,
+ IRBuilderBase &Builder) {
+ assert(isa<ScalableVectorType>(Src0->getType()));
+ auto *VTy = cast<ScalableVectorType>(Src0->getType());
+ const unsigned SegmentSize = VTy->getMinNumElements();
+ SmallVector<int, 8> Mask(SegmentSize);
+ transform(seq<int>(SegmentSize), Mask.begin(),
+ [EvenElts](int Idx) { return EvenElts ? Idx * 2 : Idx * 2 + 1; });
+ return Builder.CreateSegmentedShuffleVector(Src0, Src1, Mask);
+}
+
+/// Concatenate the even words of \p Src with its odd words within each quad.
+/// I.e. generate UZPQ1 Src.w,, (EXT Src, 4).w
+Instruction *concatEvenThenOddWordsWithinQuads(Value *Src,
+ IRBuilderBase &Builder,
+ bool ConcatWords = false) {
+ assert(Src->getType()->getPrimitiveSizeInBits().getKnownMinValue() ==
+ AArch64::SVEBitsPerBlock &&
+ "Expected legal SVE vector.");
+ assert(!Src->getType()->getScalarType()->isIntegerTy(32) &&
+ "Unexpected nxv4i32 input");
+ Type *OrigTy = Src->getType();
+ Src = Builder.CreateBitCast(Src,
+ ScalableVectorType::get(Builder.getInt32Ty(), 4));
+ Value *ShiftOddToEven =
+ Builder.CreateVectorSpliceLeft(Src, PoisonValue::get(Src->getType()), 1);
+ Instruction *Res =
+ getSegmentedUZP(Src, ShiftOddToEven, /*EvenElts=*/true, Builder);
+ return cast<Instruction>(Builder.CreateBitCast(Res, OrigTy));
+}
+
+Instruction *AArch64TTIImpl::vectorizeTargetIntrinsic(
+ Intrinsic::ID VectorIID, ArrayRef<Type *> TysForDecl,
+ ArrayRef<Value *> WideArgs, IRBuilderBase &Builder,
+ const Instruction &OrigInst) const {
+
+ // Look up the mapping in the generated table
+ StringRef NEONIntrinsicName = Intrinsic::getBaseName(VectorIID);
+ const NEONToSVEIntrinsicMapping *Mapping =
+ lookupNEONToSVEMappingByNEONIntrinsic(NEONIntrinsicName);
+ if (!Mapping)
+ llvm_unreachable("Unimplemented intrinsic vectorisation!");
+
+ auto IsScaledNEONOrScalar = [](const Value *V) -> bool {
+ if (V->getType()->isIntegerTy())
+ return true;
+ auto *SVTy = dyn_cast<ScalableVectorType>(V->getType());
+ if (!SVTy)
+ return false;
+ unsigned FixedBits = SVTy->getMinNumElements() *
+ SVTy->getElementType()->getScalarSizeInBits();
+ return FixedBits == AArch64::SVEBitsPerBlock ||
+ FixedBits == (AArch64::SVEBitsPerBlock / 2);
+ };
+ auto ToSVETy = [&](Type *Ty) -> Type * {
+ if (!Ty->isVectorTy())
+ return Ty;
+ auto *SVTy = cast<ScalableVectorType>(Ty);
+ unsigned FixedBits = SVTy->getMinNumElements() *
+ SVTy->getElementType()->getScalarSizeInBits();
+ return toVectorTy(
+ Ty, ElementCount::getFixed(AArch64::SVEBitsPerBlock / FixedBits));
+ };
+
+ // For scaled 64-bit vectors, interleave/deinterleave even elements to makes
+ // them valid SVE types. This plays nicely for both int and fp types later
+ // during ISel.
+ auto ToSVEVal = [&](Value *V) -> Value * {
+ Type *LegalSVETy = ToSVETy(V->getType());
+ if (LegalSVETy == V->getType())
+ return V;
+ return Builder.CreateVectorInterleave({V, PoisonValue::get(V->getType())});
+ };
+ auto FromSVEInst = [&](Instruction *I, Type *ExpectedTy) -> Instruction * {
+ if (ExpectedTy == I->getType())
+ return I;
+ auto *RevecTy = cast<VectorType>(I->getType());
+ auto *ExpectedEltTy = cast<VectorType>(ExpectedTy)->getElementType();
+ if (RevecTy->getElementType() != ExpectedEltTy) {
+ assert(RevecTy->getElementType()->isIntegerTy(1) &&
+ "Only expected SVE predicate to be turned into NEON vector.");
+ Type *NeonQuadPredTy =
+ VectorType::get(ExpectedEltTy, RevecTy->getElementCount());
+ I = cast<Instruction>(Builder.CreateZExt(I, NeonQuadPredTy));
+ }
+ if (ExpectedTy == I->getType())
+ return I;
+ auto *Deinterleave = Builder.CreateIntrinsic(
+ Intrinsic::vector_deinterleave2, I->getType(), I);
+ return cast<Instruction>(Builder.CreateExtractValue(Deinterleave, {0U}));
+ };
+
+ auto Zero = [&](Type *Ty) { return Constant::getNullValue(ToSVETy(Ty)); };
+ auto Poison = [&](Type *Ty) { return PoisonValue::get(ToSVETy(Ty)); };
+ auto True = [&](Type *Ty) {
+ auto *VectorTy = cast<VectorType>(ToSVETy(Ty));
+ // SVE intrinsics that narrow the element type have a predicate type with
+ // half as many elements as their return type because every odd lane is
+ // zeroed/undefined.
+ bool IsNarrowing = any_of(WideArgs, [Ty](const Value *V) {
+ return V->getType()->getScalarSizeInBits() > Ty->getScalarSizeInBits();
+ });
+ if (IsNarrowing)
+ VectorTy = VectorType::getOneNthElementsVectorType(VectorTy, 2);
+ auto *PredTy = VectorType::get(IntegerType::get(Ty->getContext(), 1),
+ VectorTy->getElementCount());
+ return Constant::getAllOnesValue(PredTy);
+ };
+ auto Arg = [&](unsigned ArgIdx) {
+ assert(ArgIdx < WideArgs.size() && "Argument index out of range!");
+ Value *V = WideArgs[ArgIdx];
+ return ToSVEVal(V);
+ };
+ auto Imm = [&](unsigned ArgIdx) {
+ assert(ArgIdx < WideArgs.size() && "Argument index out of range!");
+ Value *V = WideArgs[ArgIdx];
+ assert(isa<Constant>(V) && cast<Constant>(V)->getSplatValue());
+ return cast<Constant>(V)->getSplatValue();
+ };
+
+ assert(all_of(WideArgs, IsScaledNEONOrScalar) &&
+ "Expected SVE-compatible values.");
+
+ // Custom lower some intrinsics.
+ switch (VectorIID) {
+ case Intrinsic::aarch64_neon_uaddlp:
+ case Intrinsic::aarch64_neon_saddlp: {
+ // ADDLP works with adjacent elements: scaled 64-bit NEON inputs cannot be
+ // legalised to SVE by interleaving. Instead, extend and use a normal ADDP.
+ Type *SrcTy = WideArgs[0]->getType();
+ if (SrcTy->getPrimitiveSizeInBits().getKnownMinValue() != 64U)
+ break;
+
+ Type *DstTy = TysForDecl[0];
+ Type *WideSrcTy = ToSVETy(DstTy);
+ const bool IsSigned = VectorIID == Intrinsic::aarch64_neon_saddlp;
+ Value *Src = IsSigned ? Builder.CreateSExt(WideArgs[0], WideSrcTy)
+ : Builder.CreateZExt(WideArgs[0], WideSrcTy);
+ return FromSVEInst(Builder.CreateIntrinsic(Intrinsic::aarch64_sve_addp,
+ {WideSrcTy},
+ {True(WideSrcTy), Src, Src}),
+ DstTy);
+ }
+ case Intrinsic::aarch64_neon_addp:
+ case Intrinsic::aarch64_neon_faddp: {
+ Type *Ty = TysForDecl[0];
+ const bool IsInt = VectorIID == Intrinsic::aarch64_neon_addp;
+ // ADDP works with adjacent elements: scaled 64-bit NEON inputs cannot be
+ // legalised to SVE by interleaving with undef elements.
+ // Instead, deinterleave and use a normal ADD.
+
+ // But if the initial NEON vector contained a single pair, we can
+ // directly use SVE's ADDP without prior UZPQ de-interleaving.
+ unsigned NeonVecSize = OrigInst.getType()->getPrimitiveSizeInBits();
+ unsigned PairSize = OrigInst.getType()->getScalarSizeInBits() * 2;
+ unsigned NumPairsPerSrc = NeonVecSize / PairSize;
+ if (Ty == ToSVETy(Ty) && NumPairsPerSrc == 1) {
+ Intrinsic::ID IID =
+ IsInt ? Intrinsic::aarch64_sve_addp : Intrinsic::aarch64_sve_faddp;
+ return Builder.CreateIntrinsic(IID, {Ty},
+ {True(Ty), WideArgs[0], WideArgs[1]});
+ }
+
+ // Within each segment, ensure we have all the pairwise adds from Args0
+ // followed by those of Add1.
+ Value *EvenElts =
+ getSegmentedUZP(WideArgs[0], WideArgs[1], /*EvenElts=*/true, Builder);
+ Value *OddElts =
+ getSegmentedUZP(WideArgs[0], WideArgs[1], /*EvenElts=*/false, Builder);
+ auto *Res =
+ cast<Instruction>(IsInt ? Builder.CreateAdd(EvenElts, OddElts)
+ : Builder.CreateFAdd(EvenElts, OddElts));
+
+ // The segment type might not match the original type (VF = vscale x 2),
+ // i.e. the original type is 64-bit but our segments are 128-bit.
+ // Then, ensure that within each 64-bit segment, the lo 32-bit are pairwise
+ // adds from Arg0 and the hi 32-bit are pairwise adds from Arg1.
+ if (NeonVecSize != Ty->getPrimitiveSizeInBits().getKnownMinValue()) {
+ assert(NeonVecSize == 64U && Ty == ToSVETy(Ty) &&
+ "Expected REVEC from NEON 64-bit to legal SVE");
+ Res = concatEvenThenOddWordsWithinQuads(Res, Builder);
+ }
+ return Res;
+ }
+ case Intrinsic::aarch64_neon_sshl:
+ case Intrinsic::aarch64_neon_ushl: {
+ // SVE does not have a "plain" non-saturating non-rounding shl.
+ // This means we need to use [su]rshl for positive shift amounts
+ // and [su]qshl for negative shift amounts (effectively a shift right).
+ const bool IsSignedShift = VectorIID == Intrinsic::aarch64_neon_sshl;
+ Intrinsic::ID SHLIID = IsSignedShift ? Intrinsic::aarch64_sve_srshl
+ : Intrinsic::aarch64_sve_urshl;
+ Intrinsic::ID SHRIID = IsSignedShift ? Intrinsic::aarch64_sve_sqshl
+ : Intrinsic::aarch64_sve_uqshl;
+ Type *ArgTy = TysForDecl[0];
+ Value *Arg0 = Arg(0);
+ Value *Arg1 = Arg(1);
+ Value *SHLMask = Builder.CreateICmpSLE(
+ Arg1, ConstantInt::get(ToSVETy(ArgTy), 0), "shl.mask");
+ Value *SHL = Builder.CreateIntrinsic(SHLIID, {ToSVETy(ArgTy)},
+ {SHLMask, Arg0, Arg1});
+ Value *SHRMask =
+ Builder.CreateIntrinsic(Intrinsic::ctlz, {SHLMask->getType()},
+ {SHLMask, Builder.getTrue()}, {}, "shr.mask");
+ return FromSVEInst(Builder.CreateIntrinsic(SHRIID, {ToSVETy(ArgTy)},
+ {SHRMask, SHL, Arg1}, {},
+ "wide.bidir.shl"),
+ ArgTy);
+ }
+ case Intrinsic::aarch64_neon_tbl1:
+ case Intrinsic::aarch64_neon_tbl2:
+ case Intrinsic::aarch64_neon_tbl3:
+ case Intrinsic::aarch64_neon_tbl4: {
+ Type *Ty = TysForDecl[0];
+ assert(OrigInst.getType()->getPrimitiveSizeInBits() ==
+ Ty->getPrimitiveSizeInBits().getKnownMinValue() &&
+ "Unexpected vscale x 2 REVEC");
+ Value *Src0 = Arg(0);
+ Value *Mask = Arg(WideArgs.size() - 1);
+ Instruction *Res = Builder.CreateIntrinsic(Intrinsic::aarch64_sve_tblq,
+ {Src0->getType()}, {Src0, Mask});
+ for (unsigned SrcIdx = 1; SrcIdx < WideArgs.size() - 1; ++SrcIdx) {
+ Value *Src = Arg(SrcIdx);
+ Mask = Builder.CreateSub(Mask, ConstantInt::get(Mask->getType(), 16));
+ Res = Builder.CreateIntrinsic(Intrinsic::aarch64_sve_tbxq,
+ {Src->getType()}, {Res, Src, Mask});
+ }
+ return FromSVEInst(Res, Ty);
+ }
+ }
+ assert(!Mapping->isCustom() &&
+ "NEON intrinsic was expected to be custom-lowered to SVE");
+
+ // Note: Contrary to NEON, SVE intrinsics usually have less overload types
+ // because argument types are inferred.
+ Type *Ty = TysForDecl[0];
+ SmallVector<Value *, 4> SVEArgs;
+
+ // Parse and apply argument mappings from the table
+ SmallVector<ArgMapping, 4> ArgMappings =
+ parseArgMappings(Mapping->ArgMappings);
+ for (const ArgMapping &AM : ArgMappings) {
+ switch (AM.getKind()) {
+ case ArgMapping::True:
+ SVEArgs.push_back(True(Ty));
+ break;
+ case ArgMapping::Zero:
+ SVEArgs.push_back(Zero(Ty));
+ break;
+ case ArgMapping::Poison:
+ SVEArgs.push_back(Poison(Ty));
+ break;
+ case ArgMapping::Arg:
+ SVEArgs.push_back(Arg(AM.getArgIndex()));
+ break;
+ case ArgMapping::Imm:
+ assert(AM.getArgIndex() < WideArgs.size() &&
+ "Argument index out of range!");
+ SVEArgs.push_back(Imm(AM.getArgIndex()));
+ break;
+ }
+ }
+
+ Intrinsic::ID SVEIntrinsicID =
+ Intrinsic::lookupIntrinsicID(Mapping->SVEIntrinsic);
+ auto TysForSVEDecl = computeTysForDecl(SVEIntrinsicID, ToSVETy(Ty), SVEArgs);
+ assert(SVEIntrinsicID != Intrinsic::not_intrinsic);
+ return FromSVEInst(
+ Builder.CreateIntrinsic(SVEIntrinsicID, TysForSVEDecl, SVEArgs), Ty);
+}
+
/// See if \p I should be considered for address type promotion. We check if \p
/// I is a sext with right type and used in memory accesses. If it used in a
/// "complex" getelementptr, we allow it to be promoted without finding other
@@ -6191,6 +6754,10 @@ AArch64TTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy,
"Expected the same scalar types");
std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(SrcTy);
+ if (SrcTy->getElementCount() == ElementCount::getScalable(1) ||
+ DstTy->getElementCount() == ElementCount::getScalable(1))
+ return InstructionCost::getInvalid();
+
// If we have a Mask, and the LT is being legalized somehow, split the Mask
// into smaller vectors and sum the cost of each shuffle.
if (!Mask.empty() && isa<FixedVectorType>(SrcTy) && LT.second.isVector() &&
@@ -6544,6 +7111,35 @@ AArch64TTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy,
Args, CxtI);
}
+InstructionCost
+AArch64TTIImpl::getSegmentedShuffleCost(TTI::ShuffleKind Kind, VectorType *VTy,
+ ArrayRef<int> ShuffleMask,
+ TTI::TargetCostKind CostKind) const {
+ const unsigned SegmentSize = ShuffleMask.size();
+
+ // Can only use instructions like zipq and other q variants if VTy can be seen
+ // as <vscale x SegmentTy> where SegmentTy is at most 128b.
+ if (!VTy->isScalableTy() ||
+ VTy->getElementCount().getKnownMinValue() != SegmentSize ||
+ VTy->getPrimitiveSizeInBits().getKnownMinValue() >
+ AArch64::SVEBitsPerBlock)
+ return InstructionCost::getInvalid();
+
+ // TODO-REVEC: Some shuffles only require SVE, but it's easier to just require
+ // sve2p1 for HVLA.
+ if (!ST->hasSVE2p1() && !ST->hasSME2p1())
+ return InstructionCost::getInvalid();
+
+ // TODO-REVEC: Give sensible costs based on types?
+ if (TLI->isSegmentedShuffleMaskSupported(ShuffleMask))
+ return InstructionCost(1);
+
+ bool HasTwoInputs = any_of(ShuffleMask, [SegmentSize](int EltIdx) {
+ return EltIdx >= int(SegmentSize);
+ });
+ return HasTwoInputs ? InstructionCost(3) : InstructionCost(2);
+}
+
static bool containsDecreasingPointers(Loop *TheLoop,
PredicatedScalarEvolution *PSE,
const DominatorTree &DT) {
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index 200b4c9fba196..c3a2e687de19e 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -125,6 +125,9 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
return ST->hasSVE();
}
+ bool isSupportedTargetRecurrence(Intrinsic::ID ID,
+ RecurKind RK) const override;
+
unsigned getNumberOfRegisters(unsigned ClassID) const override {
bool Vector = (ClassID == 1);
if (Vector) {
@@ -192,6 +195,10 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
InstructionCost getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const;
+ InstructionCost
+ getSegmentGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const;
+
bool isExtPartOfAvgExpr(const Instruction *ExtUser, Type *Dst,
Type *Src) const;
@@ -294,6 +301,24 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
bool getTgtMemIntrinsic(IntrinsicInst *Inst,
MemIntrinsicInfo &Info) const override;
+ bool isTargetIntrinsicWithScalarOpAtArg(Intrinsic::ID ID,
+ unsigned ScalarOpdIdx) const override;
+
+ bool isTargetIntrinsicWithOverloadTypeAtArg(Intrinsic::ID ID,
+ int OpdIdx) const override;
+
+ /// Compute the list of Types required to get the right declaration of
+ /// the overloaded \p ID intrinsic.
+ SmallVector<Type *, 2> computeTysForDecl(Intrinsic::ID ID, Type *RetTy,
+ ArrayRef<Value *> Args) const;
+
+ bool isTargetIntrinsicVectorizable(Intrinsic::ID ID) const override;
+
+ Instruction *
+ vectorizeTargetIntrinsic(Intrinsic::ID VectorIID, ArrayRef<Type *> TysForDecl,
+ ArrayRef<Value *> WideArgs, IRBuilderBase &Builder,
+ const Instruction &OrigInst) const override;
+
bool isElementTypeLegalForScalableVector(Type *Ty) const override {
if (Ty->isPointerTy())
return true;
@@ -308,6 +333,13 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
Ty->isIntegerTy(32) || Ty->isIntegerTy(64))
return true;
+ if (auto *FVTy = dyn_cast<FixedVectorType>(Ty))
+ return (FVTy->getElementType()->isIntegerTy(1) ||
+ DL.getTypeSizeInBits(FVTy) == AArch64::SVEBitsPerBlock ||
+ DL.getTypeSizeInBits(FVTy) == AArch64::SVEBitsPerBlock / 2) &&
+ FVTy->getNumElements() > 1 &&
+ isElementTypeLegalForScalableVector(FVTy->getElementType());
+
return false;
}
@@ -373,6 +405,25 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
return isLegalMaskedGatherScatter(DataType);
}
+ bool isLegalMaskedSegmentGatherScatter(Type *DataType) const {
+ if (!ST->isSVEAvailable() || !ST->hasSVE2p1())
+ return false;
+
+ TypeSize SegmentSizeInBits = DL.getTypeSizeInBits(DataType);
+ return SegmentSizeInBits == TypeSize(64U, true) ||
+ SegmentSizeInBits == TypeSize(128U, true);
+ }
+
+ bool isLegalMaskedSegmentScatter(Type *DataType,
+ Align Alignment) const override {
+ return isLegalMaskedSegmentGatherScatter(DataType);
+ }
+
+ bool isLegalMaskedSegmentGather(Type *DataType,
+ Align Alignment) const override {
+ return isLegalMaskedSegmentGatherScatter(DataType);
+ }
+
bool isLegalBroadcastLoad(Type *ElementTy,
ElementCount NumElements) const override {
// Return true if we can generate a `ld1r` splat load instruction.
@@ -446,6 +497,11 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
bool UseMaskForCond = false, bool UseMaskForGaps = false) const override;
+ InstructionCost getSegmentInterleavedMemoryOpCost(
+ unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
+ Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
+ bool UseMaskForCond = false, bool UseMaskForGaps = false) const override;
+
bool shouldConsiderAddressTypePromotion(
const Instruction &I,
bool &AllowPromotionWithoutCommonHeader) const override;
@@ -509,6 +565,11 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
VectorType *SubTp, ArrayRef<const Value *> Args = {},
const Instruction *CxtI = nullptr) const override;
+ InstructionCost
+ getSegmentedShuffleCost(TTI::ShuffleKind Kind, VectorType *VTy,
+ ArrayRef<int> Mask,
+ TTI::TargetCostKind CostKind) const override;
+
InstructionCost
getScalarizationOverhead(VectorType *Ty, const APInt &DemandedElts,
bool Insert, bool Extract,
diff --git a/llvm/lib/Target/AArch64/AArch64VectorIntrinsicMappings.td b/llvm/lib/Target/AArch64/AArch64VectorIntrinsicMappings.td
new file mode 100644
index 0000000000000..6d9254119eb97
--- /dev/null
+++ b/llvm/lib/Target/AArch64/AArch64VectorIntrinsicMappings.td
@@ -0,0 +1,221 @@
+//===- AArch64VectorIntrinsicMappings.td - NEON to SVE Mappings -*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the mapping from NEON intrinsics to their SVE equivalents
+// for re-vectorization purposes.
+//
+// Note: SearchableTable.td is already included by AArch64SystemOperands.td
+//===----------------------------------------------------------------------===//
+
+// Argument mapping specification
+// Encodes how to map NEON arguments to SVE arguments
+// Format: Comma-separated specifiers where:
+// 'T' = True predicate, 'Z' = Zero, 'P' = poison
+// 'A:N' = Argument N from NEON
+// 'I:N' = Scalarise a constant splat into an immediate
+
+// Base class for intrinsic re-vectorization mappings
+class IntrinsicMapping<string neon, string sve, string argMappings> {
+ string NEONIntrinsic = neon;
+ string SVEIntrinsic = sve;
+ string ArgMappings = argMappings;
+}
+class CustomIntrinsicMapping<string neon> : IntrinsicMapping<neon, "", ""> {}
+
+// Direct mappings - NEON arguments pass through directly to SVE
+// udot/sdot: SVE(arg0, arg1, arg2) = NEON(arg0, arg1, arg2)
+def : IntrinsicMapping<"llvm.aarch64.neon.udot", "llvm.aarch64.sve.udot",
+ "A:0,A:1,A:2">;
+def : IntrinsicMapping<"llvm.aarch64.neon.sdot", "llvm.aarch64.sve.sdot",
+ "A:0,A:1,A:2">;
+def : IntrinsicMapping<"llvm.aarch64.neon.usdot", "llvm.aarch64.sve.usdot",
+ "A:0,A:1,A:2">;
+
+// NEON -> SVE ABD (absolute difference). Requires an extra predicate.
+// uabd/sabd: SVE(pred_true, arg0, arg1) = NEON(arg0, arg1)
+def : IntrinsicMapping<"llvm.aarch64.neon.uabd", "llvm.aarch64.sve.uabd",
+ "T,A:0,A:1">;
+def : IntrinsicMapping<"llvm.aarch64.neon.sabd", "llvm.aarch64.sve.sabd",
+ "T,A:0,A:1">;
+def : IntrinsicMapping<"llvm.aarch64.neon.fabd", "llvm.aarch64.sve.fabd",
+ "T,A:0,A:1">;
+
+// NEON -> SVE abs/sqabs. Requires an extra predicate and twice the Z input.
+// Note that NEON has no fabs.
+def : IntrinsicMapping<"llvm.aarch64.neon.abs", "llvm.aarch64.sve.abs",
+ "A:0,T,A:0">;
+def : IntrinsicMapping<"llvm.aarch64.neon.sqabs", "llvm.aarch64.sve.sqabs",
+ "A:0,T,A:0">;
+
+// NEON -> SVE FACGE/FACGT (compare absolute value). Requires an extra predicate.
+def : IntrinsicMapping<"llvm.aarch64.neon.facge", "llvm.aarch64.sve.facge",
+ "T,A:0,A:1">;
+def : IntrinsicMapping<"llvm.aarch64.neon.facgt", "llvm.aarch64.sve.facgt",
+ "T,A:0,A:1">;
+
+// NEON -> FCVTZ
+// AFAIU, SVE has no "round to nearest" or "round towards +inf" instruction
+def : IntrinsicMapping<"llvm.aarch64.neon.fcvtzu", "llvm.aarch64.sve.fcvtzu",
+ "P,T,A:0">;
+def : IntrinsicMapping<"llvm.aarch64.neon.fcvtzs", "llvm.aarch64.sve.fcvtzs",
+ "P,T,A:0">;
+
+// NEON 8.5a -> SVE FRINT32
+// Not clear to me how those instructions differ from the normal FRINTX/FRINTZ?
+def : IntrinsicMapping<"llvm.aarch64.neon.frint32x", "llvm.aarch64.sve.frintx",
+ "P,T,A:0">;
+def : IntrinsicMapping<"llvm.aarch64.neon.frint32z", "llvm.aarch64.sve.frintz",
+ "P,T,A:0">;
+
+// NEON -> SVE FRECP/FRSQRT
+def : IntrinsicMapping<"llvm.aarch64.neon.frecpe", "llvm.aarch64.sve.frecpe.x",
+ "A:0">;
+def : IntrinsicMapping<"llvm.aarch64.neon.frecps", "llvm.aarch64.sve.frecps.x",
+ "A:0,A:1">;
+def : IntrinsicMapping<"llvm.aarch64.neon.frsqrte", "llvm.aarch64.sve.frsqrte.x",
+ "A:0">;
+def : IntrinsicMapping<"llvm.aarch64.neon.frsqrts", "llvm.aarch64.sve.frsqrts.x",
+ "A:0,A:1">;
+
+// Accumulator mappings - Add true predicate and zero accumulator before NEON arguments
+// uadalp/sadalp: SVE(pred_true, zero, arg0) = NEON(arg0)
+def : IntrinsicMapping<"llvm.aarch64.neon.uaddlp", "llvm.aarch64.sve.uadalp",
+ "T,Z,A:0">;
+def : IntrinsicMapping<"llvm.aarch64.neon.saddlp", "llvm.aarch64.sve.sadalp",
+ "T,Z,A:0">;
+
+// NEON -> SVE hadd/hsub. Requires an extra predicate.
+def : IntrinsicMapping<"llvm.aarch64.neon.uhadd", "llvm.aarch64.sve.uhadd",
+ "T,A:0,A:1">;
+def : IntrinsicMapping<"llvm.aarch64.neon.shadd", "llvm.aarch64.sve.shadd",
+ "T,A:0,A:1">;
+def : IntrinsicMapping<"llvm.aarch64.neon.uhsub", "llvm.aarch64.sve.uhsub",
+ "T,A:0,A:1">;
+def : IntrinsicMapping<"llvm.aarch64.neon.shsub", "llvm.aarch64.sve.shsub",
+ "T,A:0,A:1">;
+def : IntrinsicMapping<"llvm.aarch64.neon.urhadd", "llvm.aarch64.sve.urhadd",
+ "T,A:0,A:1">;
+def : IntrinsicMapping<"llvm.aarch64.neon.srhadd", "llvm.aarch64.sve.srhadd",
+ "T,A:0,A:1">;
+
+// NEON -> SVE qadd/qsub. Requires an extra predicate.
+def : IntrinsicMapping<"llvm.aarch64.neon.uqadd", "llvm.aarch64.sve.uqadd",
+ "T,A:0,A:1">;
+def : IntrinsicMapping<"llvm.aarch64.neon.sqadd", "llvm.aarch64.sve.sqadd",
+ "T,A:0,A:1">;
+def : IntrinsicMapping<"llvm.aarch64.neon.uqsub", "llvm.aarch64.sve.uqsub",
+ "T,A:0,A:1">;
+def : IntrinsicMapping<"llvm.aarch64.neon.sqsub", "llvm.aarch64.sve.sqsub",
+ "T,A:0,A:1">;
+
+// NEON -> SVE qdmul/qrdmul. Direct mapping.
+def : IntrinsicMapping<"llvm.aarch64.neon.sqdmulh", "llvm.aarch64.sve.sqdmulh",
+ "A:0,A:1">;
+def : IntrinsicMapping<"llvm.aarch64.neon.sqrdmulh", "llvm.aarch64.sve.sqrdmulh",
+ "A:0,A:1">;
+
+// Shift left (saturating, rounding, both)
+def : IntrinsicMapping<"llvm.aarch64.neon.sqshl", "llvm.aarch64.sve.sqshl",
+ "T,A:0,A:1">;
+def : IntrinsicMapping<"llvm.aarch64.neon.uqshl", "llvm.aarch64.sve.uqshl",
+ "T,A:0,A:1">;
+def : IntrinsicMapping<"llvm.aarch64.neon.srshl", "llvm.aarch64.sve.srshl",
+ "T,A:0,A:1">;
+def : IntrinsicMapping<"llvm.aarch64.neon.urshl", "llvm.aarch64.sve.urshl",
+ "T,A:0,A:1">;
+def : IntrinsicMapping<"llvm.aarch64.neon.sqrshl", "llvm.aarch64.sve.sqrshl",
+ "T,A:0,A:1">;
+def : IntrinsicMapping<"llvm.aarch64.neon.uqrshl", "llvm.aarch64.sve.uqrshl",
+ "T,A:0,A:1">;
+def : IntrinsicMapping<"llvm.aarch64.neon.sqshlu", "llvm.aarch64.sve.sqshlu",
+ "T,A:0,I:1">;
+
+// NEON's bidirectional shifts like neon.sshl or neon.ushl cannot be rewritten for
+// SVE directly because their equivalents always have rounding/saturation.
+def : CustomIntrinsicMapping<"llvm.aarch64.neon.sshl">;
+def : CustomIntrinsicMapping<"llvm.aarch64.neon.ushl">;
+
+// Shift right narrow
+def : IntrinsicMapping<"llvm.aarch64.neon.rshrn", "llvm.aarch64.sve.rshrnb",
+ "A:0,A:1">;
+def : IntrinsicMapping<"llvm.aarch64.neon.sqrshrn", "llvm.aarch64.sve.sqrshrnb",
+ "A:0,A:1">;
+def : IntrinsicMapping<"llvm.aarch64.neon.sqrshrun", "llvm.aarch64.sve.sqrshrunb",
+ "A:0,A:1">;
+def : IntrinsicMapping<"llvm.aarch64.neon.sqshrn", "llvm.aarch64.sve.sqshrnb",
+ "A:0,A:1">;
+def : IntrinsicMapping<"llvm.aarch64.neon.sqshrun", "llvm.aarch64.sve.sqshrunb",
+ "A:0,A:1">;
+def : IntrinsicMapping<"llvm.aarch64.neon.uqrshrn", "llvm.aarch64.sve.uqrshrnb",
+ "A:0,A:1">;
+def : IntrinsicMapping<"llvm.aarch64.neon.uqshrn", "llvm.aarch64.sve.uqshrnb",
+ "A:0,A:1">;
+
+// Shift and insert
+def : IntrinsicMapping<"llvm.aarch64.neon.vsli", "llvm.aarch64.sve.sli",
+ "A:0,A:1,A:2">;
+def : IntrinsicMapping<"llvm.aarch64.neon.vsri", "llvm.aarch64.sve.sri",
+ "A:0,A:1,A:2">;
+
+// Other shift instructions that aren't considered:
+// - widening shift left (SHLL, SHLL2, USHLL, USHLL2, SSHLL, SSHLL2): Intrinsics are not used
+// - SRSHR, SRSRA, USHR, URSHR: No intrinsics
+
+// SVE's MULL takes inputs where every second element is ignored whereas
+// NEON's MULL ignores the hi part of the quad.
+// This is compatible with the default handling for scaled 64-bit vectors: each element
+// gets interleaved with poison values.
+def : IntrinsicMapping<"llvm.aarch64.neon.smull", "llvm.aarch64.sve.smullb",
+ "A:0,A:1">;
+def : IntrinsicMapping<"llvm.aarch64.neon.umull", "llvm.aarch64.sve.umullb",
+ "A:0,A:1">;
+
+// SVE's ADDP add pairs from src0 into even indices and pairs from src1 into odd indices.
+// NEON's ADDP concatenates sources and inserts results "in order" in the result quad/double.
+def : CustomIntrinsicMapping<"llvm.aarch64.neon.addp">;
+def : CustomIntrinsicMapping<"llvm.aarch64.neon.faddp">;
+
+// i8 4x4 MMLA (Matrix multiply accumulate)
+// (Requires SVE+i8mm, i8mm is already required by NEON's MMLA so there's nothing extra to check)
+// Note that NEON has no fmmla but SVE has.
+def : IntrinsicMapping<"llvm.aarch64.neon.ummla", "llvm.aarch64.sve.ummla",
+ "A:0,A:1,A:2">;
+def : IntrinsicMapping<"llvm.aarch64.neon.smmla", "llvm.aarch64.sve.smmla",
+ "A:0,A:1,A:2">;
+def : IntrinsicMapping<"llvm.aarch64.neon.usmmla", "llvm.aarch64.sve.usmmla",
+ "A:0,A:1,A:2">;
+
+// Extract narrow
+def : IntrinsicMapping<"llvm.aarch64.neon.sqxtn", "llvm.aarch64.sve.sqxtnb",
+ "A:0">;
+def : IntrinsicMapping<"llvm.aarch64.neon.sqxtun", "llvm.aarch64.sve.sqxtunb",
+ "A:0">;
+def : IntrinsicMapping<"llvm.aarch64.neon.uqxtn", "llvm.aarch64.sve.uqxtnb",
+ "A:0">;
+def : IntrinsicMapping<"llvm.aarch64.neon.fcvtxn", "llvm.aarch64.sve.fcvtx.f32f64",
+ "P,T,A:0">;
+
+// NEON's tbl1 can be remapped directly to SVE's tblq, but tbl[2-4] require
+// extra tbxq instruction(s)
+def : CustomIntrinsicMapping<"llvm.aarch64.neon.tbl1">;
+def : CustomIntrinsicMapping<"llvm.aarch64.neon.tbl2">;
+def : CustomIntrinsicMapping<"llvm.aarch64.neon.tbl3">;
+def : CustomIntrinsicMapping<"llvm.aarch64.neon.tbl4">;
+
+// Searchable table definition
+def IntrinsicMappingTable : GenericTable {
+ let FilterClass = "IntrinsicMapping";
+ let CppTypeName = "NEONToSVEIntrinsicMapping";
+ let Fields = ["NEONIntrinsic", "SVEIntrinsic", "ArgMappings"];
+}
+
+// Search index for looking up by NEON intrinsic name
+def lookupNEONToSVEMappingByNEONIntrinsic : SearchIndex {
+ let Table = IntrinsicMappingTable;
+ let Key = ["NEONIntrinsic"];
+}
diff --git a/llvm/lib/Target/AArch64/CMakeLists.txt b/llvm/lib/Target/AArch64/CMakeLists.txt
index 7be0ca38ef4f3..445e46589ea99 100644
--- a/llvm/lib/Target/AArch64/CMakeLists.txt
+++ b/llvm/lib/Target/AArch64/CMakeLists.txt
@@ -27,6 +27,7 @@ tablegen(LLVM AArch64GenSDNodeInfo.inc -gen-sd-node-info)
tablegen(LLVM AArch64GenSubtargetInfo.inc -gen-subtarget)
tablegen(LLVM AArch64GenSystemOperands.inc -gen-searchable-tables)
tablegen(LLVM AArch64GenExegesis.inc -gen-exegesis)
+tablegen(LLVM AArch64GenVectorIntrinsicMappings.inc -gen-searchable-tables)
add_public_tablegen_target(AArch64CommonTableGen)
diff --git a/llvm/lib/Target/AArch64/SMEInstrFormats.td b/llvm/lib/Target/AArch64/SMEInstrFormats.td
index 771c4c1fb2b6e..fbf494d1d19b4 100644
--- a/llvm/lib/Target/AArch64/SMEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SMEInstrFormats.td
@@ -1595,15 +1595,17 @@ class sve2_int_perm_revd<string asm>
multiclass sve2_int_perm_revd<string asm, SDPatternOperator op> {
def NAME : sve2_int_perm_revd<asm>;
- def : SVE_1_Op_Passthru_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME)>;
- def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME)>;
- def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME)>;
- def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME)>;
-
- def : SVE_1_Op_Passthru_Pat<nxv8bf16, op, nxv8i1, nxv8bf16, !cast<Instruction>(NAME)>;
- def : SVE_1_Op_Passthru_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME)>;
- def : SVE_1_Op_Passthru_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME)>;
- def : SVE_1_Op_Passthru_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME)>;
+ foreach VT = [nxv16i8] in
+ def : SVE_1_Op_Passthru_Pat<VT, op, nxv16i1, VT, !cast<Instruction>(NAME)>;
+
+ foreach VT = [nxv8i16, nxv8f16, nxv8bf16] in
+ def : SVE_1_Op_Passthru_Pat<VT, op, nxv8i1, VT, !cast<Instruction>(NAME)>;
+
+ foreach VT = [nxv4i32, nxv4f16, nxv4f32, nxv4bf16] in
+ def : SVE_1_Op_Passthru_Pat<VT, op, nxv4i1, VT, !cast<Instruction>(NAME)>;
+
+ foreach VT = [nxv2i64, nxv2f16, nxv2f32, nxv2f64, nxv2bf16] in
+ def : SVE_1_Op_Passthru_Pat<VT, op, nxv2i1, VT, !cast<Instruction>(NAME)>;
}
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index 040962e801604..25012d40e6178 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -1500,12 +1500,21 @@ multiclass sve_int_perm_dup_i<string asm> {
// Duplicate an extracted vector element across a vector.
- def : Pat<(nxv16i8 (splat_vector (i32 (vector_extract (nxv16i8 ZPR:$vec), sve_elm_idx_extdup_b:$index)))),
- (!cast<Instruction>(NAME # _B) ZPR:$vec, sve_elm_idx_extdup_b:$index)>;
- def : Pat<(nxv16i8 (splat_vector (i32 (vector_extract (v16i8 V128:$vec), sve_elm_idx_extdup_b:$index)))),
- (!cast<Instruction>(NAME # _B) (SUBREG_TO_REG $vec, zsub), sve_elm_idx_extdup_b:$index)>;
- def : Pat<(nxv16i8 (splat_vector (i32 (vector_extract (v8i8 V64:$vec), sve_elm_idx_extdup_b:$index)))),
- (!cast<Instruction>(NAME # _B) (SUBREG_TO_REG $vec, dsub), sve_elm_idx_extdup_b:$index)>;
+ foreach VT = [nxv16i8] in {
+ def : Pat<(VT (splat_vector (i32 (vector_extract (SVEType<VT>.Packed ZPR:$vec), sve_elm_idx_extdup_b:$index)))),
+ (!cast<Instruction>(NAME # _B) ZPR:$vec, sve_elm_idx_extdup_b:$index)>;
+ def : Pat<(VT (splat_vector (i32 (vector_extract (SVEType<VT>.ZSub V128:$vec), sve_elm_idx_extdup_b:$index)))),
+ (!cast<Instruction>(NAME # _B) (SUBREG_TO_REG $vec, zsub), sve_elm_idx_extdup_b:$index)>;
+ def : Pat<(VT (splat_vector (i32 (vector_extract (SVEType<VT>.DSub V64:$vec), sve_elm_idx_extdup_b:$index)))),
+ (!cast<Instruction>(NAME # _B) (SUBREG_TO_REG $vec, dsub), sve_elm_idx_extdup_b:$index)>;
+
+ // Broadcast a whole 128-bit vector
+ def : Pat<(VT (vector_broadcast (SVEType<VT>.ZSub V128:$vec))),
+ (!cast<Instruction>(NAME # _Q) (SUBREG_TO_REG $vec, zsub), (i64 0))>;
+ // Broadcast a whole 64-bit vector
+ def : Pat<(VT (vector_broadcast (SVEType<VT>.DSub V64:$vec))),
+ (!cast<Instruction>(NAME # _D) (SUBREG_TO_REG $vec, dsub), (i64 0))>;
+ }
foreach VT = [nxv8i16, nxv2f16, nxv4f16, nxv8f16, nxv2bf16, nxv4bf16, nxv8bf16] in {
def : Pat<(VT (splat_vector (SVEType<VT>.EltAsScalar (vector_extract (SVEType<VT>.Packed ZPR:$vec), sve_elm_idx_extdup_h:$index)))),
@@ -1514,6 +1523,13 @@ multiclass sve_int_perm_dup_i<string asm> {
(!cast<Instruction>(NAME # _H) (SUBREG_TO_REG $vec, zsub), sve_elm_idx_extdup_h:$index)>;
def : Pat<(VT (splat_vector (SVEType<VT>.EltAsScalar (vector_extract (SVEType<VT>.DSub V64:$vec), sve_elm_idx_extdup_h:$index)))),
(!cast<Instruction>(NAME # _H) (SUBREG_TO_REG $vec, dsub), sve_elm_idx_extdup_h:$index)>;
+
+ // Broadcast a whole 128-bit vector
+ def : Pat<(VT (vector_broadcast (SVEType<VT>.ZSub V128:$vec))),
+ (!cast<Instruction>(NAME # _Q) (SUBREG_TO_REG $vec, zsub), (i64 0))>;
+ // Broadcast a whole 64-bit vector
+ def : Pat<(VT (vector_broadcast (SVEType<VT>.DSub V64:$vec))),
+ (!cast<Instruction>(NAME # _D) (SUBREG_TO_REG $vec, dsub), (i64 0))>;
}
foreach VT = [nxv4i32, nxv2f32, nxv4f32 ] in {
@@ -1523,6 +1539,13 @@ multiclass sve_int_perm_dup_i<string asm> {
(!cast<Instruction>(NAME # _S) (SUBREG_TO_REG $vec, zsub), sve_elm_idx_extdup_s:$index)>;
def : Pat<(VT (splat_vector (SVEType<VT>.EltAsScalar (vector_extract (SVEType<VT>.DSub V64:$vec), sve_elm_idx_extdup_s:$index)))),
(!cast<Instruction>(NAME # _S) (SUBREG_TO_REG $vec, dsub), sve_elm_idx_extdup_s:$index)>;
+
+ // Broadcast a whole 128-bit vector
+ def : Pat<(VT (vector_broadcast (SVEType<VT>.ZSub V128:$vec))),
+ (!cast<Instruction>(NAME # _Q) (SUBREG_TO_REG $vec, zsub), (i64 0))>;
+ // Broadcast a whole 64-bit vector
+ def : Pat<(VT (vector_broadcast (SVEType<VT>.DSub V64:$vec))),
+ (!cast<Instruction>(NAME # _D) (SUBREG_TO_REG $vec, dsub), (i64 0))>;
}
foreach VT = [nxv2i64, nxv2f64] in {
@@ -1532,6 +1555,13 @@ multiclass sve_int_perm_dup_i<string asm> {
(!cast<Instruction>(NAME # _D) (SUBREG_TO_REG $vec, zsub), sve_elm_idx_extdup_d:$index)>;
def : Pat<(VT (splat_vector (SVEType<VT>.EltAsScalar (vector_extract (SVEType<VT>.DSub V64:$vec), sve_elm_idx_extdup_d:$index)))),
(!cast<Instruction>(NAME # _D) (SUBREG_TO_REG $vec, dsub), sve_elm_idx_extdup_d:$index)>;
+
+ // Broadcast a whole 128-bit vector
+ def : Pat<(VT (vector_broadcast (SVEType<VT>.ZSub V128:$vec))),
+ (!cast<Instruction>(NAME # _Q) (SUBREG_TO_REG $vec, zsub), (i64 0))>;
+ // Broadcast a whole 64-bit vector
+ def : Pat<(VT (vector_broadcast (SVEType<VT>.DSub V64:$vec))),
+ (!cast<Instruction>(NAME # _D) (SUBREG_TO_REG $vec, dsub), (i64 0))>;
}
// When extracting from an unpacked vector the index must be scaled to account
@@ -1696,16 +1726,17 @@ multiclass sve2_int_perm_tbx<string asm, bits<2> opc, SDPatternOperator op> {
def _S : sve2_int_perm_tbx<0b10, opc, asm, ZPR32>;
def _D : sve2_int_perm_tbx<0b11, opc, asm, ZPR64>;
- def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;
- def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;
- def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;
- def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;
+ foreach VT = [nxv16i8] in
+ def : SVE_3_Op_Pat<VT, op, VT, VT, nxv16i8, !cast<Instruction>(NAME # _B)>;
+
+ foreach VT = [nxv8i16, nxv8f16, nxv8bf16] in
+ def : SVE_3_Op_Pat<VT, op, VT, VT, nxv8i16, !cast<Instruction>(NAME # _H)>;
- def : SVE_3_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>;
- def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>;
- def : SVE_3_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>;
+ foreach VT = [nxv4i32, nxv4f16, nxv4f32, nxv4bf16] in
+ def : SVE_3_Op_Pat<VT, op, VT, VT, nxv4i32, !cast<Instruction>(NAME # _S)>;
- def : SVE_3_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8bf16, nxv8i16, !cast<Instruction>(NAME # _H)>;
+ foreach VT = [nxv2i64, nxv2f16, nxv2f32, nxv2f64, nxv2bf16] in
+ def : SVE_3_Op_Pat<VT, op, VT, VT, nxv2i64, !cast<Instruction>(NAME # _D)>;
}
class sve_int_perm_reverse_z<bits<2> sz8_64, string asm, ZPRRegOp zprty>
@@ -7933,15 +7964,17 @@ multiclass sve_int_perm_rev_revw_z<string asm, SDPatternOperator op> {
multiclass sve_int_perm_rev_revd_z<string asm, SDPatternOperator op> {
def NAME : sve_int_perm_rev_z<0b00, 0b1110, asm, ZPR128>;
- defm : SVE_1_Op_PassthruUndefZero_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME)>;
- defm : SVE_1_Op_PassthruUndefZero_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME)>;
- defm : SVE_1_Op_PassthruUndefZero_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME)>;
- defm : SVE_1_Op_PassthruUndefZero_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME)>;
+ foreach VT = [nxv16i8] in
+ defm : SVE_1_Op_PassthruUndefZero_Pat<VT, op, nxv16i1, VT, !cast<Instruction>(NAME)>;
- defm : SVE_1_Op_PassthruUndefZero_Pat<nxv8bf16, op, nxv8i1, nxv8bf16, !cast<Instruction>(NAME)>;
- defm : SVE_1_Op_PassthruUndefZero_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME)>;
- defm : SVE_1_Op_PassthruUndefZero_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME)>;
- defm : SVE_1_Op_PassthruUndefZero_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME)>;
+ foreach VT = [nxv8i16, nxv8f16, nxv8bf16] in
+ defm : SVE_1_Op_PassthruUndefZero_Pat<VT, op, nxv8i1, VT, !cast<Instruction>(NAME)>;
+
+ foreach VT = [nxv4i32, nxv4f16, nxv4f32, nxv4bf16] in
+ defm : SVE_1_Op_PassthruUndefZero_Pat<VT, op, nxv4i1, VT, !cast<Instruction>(NAME)>;
+
+ foreach VT = [nxv2i64, nxv2f16, nxv2f32, nxv2f64, nxv2bf16] in
+ defm : SVE_1_Op_PassthruUndefZero_Pat<VT, op, nxv2i1, VT, !cast<Instruction>(NAME)>;
}
class sve_int_perm_cpy_r<bits<2> sz8_64, string asm, ZPRRegOp zprty,
@@ -10799,15 +10832,17 @@ multiclass sve2p1_dupq<string mnemonic, SDPatternOperator Op> {
let Inst{20} = index;
}
- def : SVE_2_Op_Imm_Pat<nxv16i8, Op, nxv16i8, i32, VectorIndexB32b_timm, !cast<Instruction>(NAME # _B)>;
- def : SVE_2_Op_Imm_Pat<nxv8i16, Op, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _H)>;
- def : SVE_2_Op_Imm_Pat<nxv4i32, Op, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _S)>;
- def : SVE_2_Op_Imm_Pat<nxv2i64, Op, nxv2i64, i32, VectorIndexD32b_timm, !cast<Instruction>(NAME # _D)>;
+ foreach VT = [nxv16i8] in
+ def : SVE_2_Op_Imm_Pat<VT, Op, VT, i32, VectorIndexB32b_timm, !cast<Instruction>(NAME # _B)>;
+
+ foreach VT = [nxv8i16, nxv8f16, nxv8bf16] in
+ def : SVE_2_Op_Imm_Pat<VT, Op, VT, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _H)>;
+
+ foreach VT = [nxv4i32, nxv4f16, nxv4f32, nxv4bf16] in
+ def : SVE_2_Op_Imm_Pat<VT, Op, VT, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _S)>;
- def : SVE_2_Op_Imm_Pat<nxv8f16, Op, nxv8f16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _H)>;
- def : SVE_2_Op_Imm_Pat<nxv4f32, Op, nxv4f32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _S)>;
- def : SVE_2_Op_Imm_Pat<nxv2f64, Op, nxv2f64, i32, VectorIndexD32b_timm, !cast<Instruction>(NAME # _D)>;
- def : SVE_2_Op_Imm_Pat<nxv8bf16, Op, nxv8bf16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _H)>;
+ foreach VT = [nxv2i64, nxv2f16, nxv2f32, nxv2f64, nxv2bf16] in
+ def : SVE_2_Op_Imm_Pat<VT, Op, VT, i32, VectorIndexD32b_timm, !cast<Instruction>(NAME # _D)>;
}
@@ -10833,15 +10868,18 @@ class sve2p1_extq<string mnemonic>
multiclass sve2p1_extq<string mnemonic, SDPatternOperator Op> {
def NAME : sve2p1_extq<mnemonic>;
- def : SVE_3_Op_Imm_Pat<nxv16i8, Op, nxv16i8, nxv16i8, i32, timm32_0_15, !cast<Instruction>(NAME)>;
- def : SVE_3_Op_Imm_Pat<nxv8i16, Op, nxv8i16, nxv8i16, i32, extq_timm32_0_7m2, !cast<Instruction>(NAME)>;
- def : SVE_3_Op_Imm_Pat<nxv4i32, Op, nxv4i32, nxv4i32, i32, extq_timm32_0_3m4, !cast<Instruction>(NAME)>;
- def : SVE_3_Op_Imm_Pat<nxv2i64, Op, nxv2i64, nxv2i64, i32, extq_timm32_0_1m8, !cast<Instruction>(NAME)>;
- def : SVE_3_Op_Imm_Pat<nxv8f16, Op, nxv8f16, nxv8f16, i32, extq_timm32_0_7m2, !cast<Instruction>(NAME)>;
- def : SVE_3_Op_Imm_Pat<nxv4f32, Op, nxv4f32, nxv4f32, i32, extq_timm32_0_3m4, !cast<Instruction>(NAME)>;
- def : SVE_3_Op_Imm_Pat<nxv2f64, Op, nxv2f64, nxv2f64, i32, extq_timm32_0_1m8, !cast<Instruction>(NAME)>;
- def : SVE_3_Op_Imm_Pat<nxv8bf16, Op, nxv8bf16, nxv8bf16, i32, extq_timm32_0_7m2, !cast<Instruction>(NAME)>;
+ foreach VT = [nxv16i8] in
+ def : SVE_3_Op_Imm_Pat<VT, Op, VT, VT, i32, timm32_0_15, !cast<Instruction>(NAME)>;
+
+ foreach VT = [nxv8i16, nxv8f16, nxv8bf16] in
+ def : SVE_3_Op_Imm_Pat<VT, Op, VT, VT, i32, extq_timm32_0_7m2, !cast<Instruction>(NAME)>;
+
+ foreach VT = [nxv4i32, nxv4f16, nxv4f32, nxv4bf16] in
+ def : SVE_3_Op_Imm_Pat<VT, Op, VT, VT, i32, extq_timm32_0_3m4, !cast<Instruction>(NAME)>;
+
+ foreach VT = [nxv2i64, nxv2f16, nxv2f32, nxv2f64, nxv2bf16] in
+ def : SVE_3_Op_Imm_Pat<VT, Op, VT, VT, i32, extq_timm32_0_1m8, !cast<Instruction>(NAME)>;
}
// SVE move predicate from vector
@@ -11044,16 +11082,17 @@ multiclass sve2p1_permute_vec_elems_q<bits<3> opc, string mnemonic,
def _S : sve2p1_permute_vec_elems_q<0b10, opc, mnemonic, ZPR32, ZPR32>;
def _D : sve2p1_permute_vec_elems_q<0b11, opc, mnemonic, ZPR64, ZPR64>;
- def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;
- def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;
- def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;
- def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;
+ foreach VT = [nxv16i8] in
+ def : SVE_2_Op_Pat<VT, op, VT, VT, !cast<Instruction>(NAME # _B)>;
- def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>;
- def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>;
- def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>;
+ foreach VT = [nxv8i16, nxv8f16, nxv8bf16] in
+ def : SVE_2_Op_Pat<VT, op, VT, VT, !cast<Instruction>(NAME # _H)>;
- def : SVE_2_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>;
+ foreach VT = [nxv4i32, nxv4f16, nxv4f32, nxv4bf16] in
+ def : SVE_2_Op_Pat<VT, op, VT, VT, !cast<Instruction>(NAME # _S)>;
+
+ foreach VT = [nxv2i64, nxv2f16, nxv2f32, nxv2f64, nxv2bf16] in
+ def : SVE_2_Op_Pat<VT, op, VT, VT, !cast<Instruction>(NAME # _D)>;
}
multiclass sve2p1_tblq<string mnemonic, SDPatternOperator op> {
@@ -11062,16 +11101,17 @@ multiclass sve2p1_tblq<string mnemonic, SDPatternOperator op> {
def _S : sve2p1_permute_vec_elems_q<0b10, 0b110, mnemonic, ZPR32, Z_s>;
def _D : sve2p1_permute_vec_elems_q<0b11, 0b110, mnemonic, ZPR64, Z_d>;
- def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;
- def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;
- def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;
- def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;
+ foreach VT = [nxv16i8] in
+ def : SVE_2_Op_Pat<VT, op, VT, nxv16i8, !cast<Instruction>(NAME # _B)>;
- def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>;
- def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>;
- def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>;
+ foreach VT = [nxv8i16, nxv8f16, nxv8bf16] in
+ def : SVE_2_Op_Pat<VT, op, VT, nxv8i16, !cast<Instruction>(NAME # _H)>;
- def : SVE_2_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8i16, !cast<Instruction>(NAME # _H)>;
+ foreach VT = [nxv4i32, nxv4f16, nxv4f32, nxv4bf16] in
+ def : SVE_2_Op_Pat<VT, op, VT, nxv4i32, !cast<Instruction>(NAME # _S)>;
+
+ foreach VT = [nxv2i64, nxv2f16, nxv2f32, nxv2f64, nxv2bf16] in
+ def : SVE_2_Op_Pat<VT, op, VT, nxv2i64, !cast<Instruction>(NAME # _D)>;
}
//===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h
index c1d31c944714c..a1b27a67efcaa 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -443,13 +443,14 @@ class RISCVTargetLowering : public TargetLowering {
ShuffleVectorInst *SVI, unsigned Factor,
const APInt &GapMask) const override;
- bool lowerDeinterleaveIntrinsicToLoad(Instruction *Load, Value *Mask,
- IntrinsicInst *DI,
- const APInt &GapMask) const override;
+ bool
+ lowerDeinterleaveIntrinsicToLoad(Instruction *Load, Value *Mask,
+ IntrinsicInst *DI, const APInt &GapMask,
+ bool DeinterleaveSegments) const override;
- bool lowerInterleaveIntrinsicToStore(
- Instruction *Store, Value *Mask,
- ArrayRef<Value *> InterleaveValues) const override;
+ bool lowerInterleaveIntrinsicToStore(Instruction *Store, Value *Mask,
+ ArrayRef<Value *> InterleaveValues,
+ bool InterleaveSegments) const override;
bool supportKCFIBundles() const override { return true; }
diff --git a/llvm/lib/Target/RISCV/RISCVInterleavedAccess.cpp b/llvm/lib/Target/RISCV/RISCVInterleavedAccess.cpp
index 7026541cbd53d..a98331d8f88a2 100644
--- a/llvm/lib/Target/RISCV/RISCVInterleavedAccess.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInterleavedAccess.cpp
@@ -355,11 +355,11 @@ bool RISCVTargetLowering::lowerInterleavedStore(Instruction *Store,
}
bool RISCVTargetLowering::lowerDeinterleaveIntrinsicToLoad(
- Instruction *Load, Value *Mask, IntrinsicInst *DI,
- const APInt &GapMask) const {
+ Instruction *Load, Value *Mask, IntrinsicInst *DI, const APInt &GapMask,
+ bool DeinterleaveSegments) const {
const unsigned Factor = getDeinterleaveIntrinsicFactor(DI->getIntrinsicID());
assert(GapMask.getBitWidth() == Factor);
- if (Factor > 8)
+ if (Factor > 8 || DeinterleaveSegments)
return false;
// We only support cases where the skipped fields are the trailing ones.
@@ -463,9 +463,10 @@ bool RISCVTargetLowering::lowerDeinterleaveIntrinsicToLoad(
}
bool RISCVTargetLowering::lowerInterleaveIntrinsicToStore(
- Instruction *Store, Value *Mask, ArrayRef<Value *> InterleaveValues) const {
+ Instruction *Store, Value *Mask, ArrayRef<Value *> InterleaveValues,
+ bool InterleaveSegments) const {
unsigned Factor = InterleaveValues.size();
- if (Factor > 8)
+ if (Factor > 8 || InterleaveSegments)
return false;
IRBuilder<> Builder(Store);
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index f0586e4f0f464..77242eb1ab568 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -1093,14 +1093,19 @@ bool llvm::hasIterationCountInvariantInParent(Loop *InnerLoop,
return true;
}
-constexpr Intrinsic::ID llvm::getReductionIntrinsicID(RecurKind RK) {
+Intrinsic::ID llvm::getReductionIntrinsicID(RecurKind RK, Type *Ty) {
+ // TODO-REVEC: Do not rely on partial.reduce intrinsics for scalable->fixed
+ // vector reductions. They enforce no restrictions on lane ordering and we do
+ // not want to change their semantics.
switch (RK) {
default:
llvm_unreachable("Unexpected recurrence kind");
case RecurKind::AddChainWithSubs:
case RecurKind::Sub:
case RecurKind::Add:
- return Intrinsic::vector_reduce_add;
+ case RecurKind::TargetIntAccumulation:
+ return Ty->isVectorTy() ? Intrinsic::vector_partial_reduce_add
+ : Intrinsic::vector_reduce_add;
case RecurKind::Mul:
return Intrinsic::vector_reduce_mul;
case RecurKind::And:
@@ -1117,13 +1122,17 @@ constexpr Intrinsic::ID llvm::getReductionIntrinsicID(RecurKind RK) {
case RecurKind::FMul:
return Intrinsic::vector_reduce_fmul;
case RecurKind::SMax:
- return Intrinsic::vector_reduce_smax;
+ return Ty->isVectorTy() ? Intrinsic::vector_partial_reduce_smax
+ : Intrinsic::vector_reduce_smax;
case RecurKind::SMin:
- return Intrinsic::vector_reduce_smin;
+ return Ty->isVectorTy() ? Intrinsic::vector_partial_reduce_smin
+ : Intrinsic::vector_reduce_smin;
case RecurKind::UMax:
- return Intrinsic::vector_reduce_umax;
+ return Ty->isVectorTy() ? Intrinsic::vector_partial_reduce_umax
+ : Intrinsic::vector_reduce_umax;
case RecurKind::UMin:
- return Intrinsic::vector_reduce_umin;
+ return Ty->isVectorTy() ? Intrinsic::vector_partial_reduce_umin
+ : Intrinsic::vector_reduce_umin;
case RecurKind::FMax:
case RecurKind::FMaxNum:
return Intrinsic::vector_reduce_fmax;
@@ -1210,12 +1219,16 @@ Intrinsic::ID llvm::getMinMaxReductionIntrinsicOp(Intrinsic::ID RdxID) {
default:
llvm_unreachable("Unknown min/max recurrence kind");
case Intrinsic::vector_reduce_umin:
+ case Intrinsic::vector_partial_reduce_umin:
return Intrinsic::umin;
case Intrinsic::vector_reduce_umax:
+ case Intrinsic::vector_partial_reduce_umax:
return Intrinsic::umax;
case Intrinsic::vector_reduce_smin:
+ case Intrinsic::vector_partial_reduce_smin:
return Intrinsic::smin;
case Intrinsic::vector_reduce_smax:
+ case Intrinsic::vector_partial_reduce_smax:
return Intrinsic::smax;
case Intrinsic::vector_reduce_fmin:
return Intrinsic::minnum;
@@ -1490,6 +1503,35 @@ Value *llvm::createAnyOfReduction(IRBuilderBase &Builder, Value *Src,
return Builder.CreateSelect(AnyOf, NewVal, InitVal, "rdx.select");
}
+bool llvm::isPartialReductionIntrinsicID(Intrinsic::ID RdxID) {
+ switch (RdxID) {
+ default:
+ llvm_unreachable("Expecting a reduction intrinsic");
+ case Intrinsic::vector_reduce_add:
+ case Intrinsic::vector_reduce_mul:
+ case Intrinsic::vector_reduce_or:
+ case Intrinsic::vector_reduce_xor:
+ case Intrinsic::vector_reduce_and:
+ case Intrinsic::vector_reduce_fadd:
+ case Intrinsic::vector_reduce_fmul:
+ case Intrinsic::vector_reduce_umax:
+ case Intrinsic::vector_reduce_umin:
+ case Intrinsic::vector_reduce_smin:
+ case Intrinsic::vector_reduce_smax:
+ case Intrinsic::vector_reduce_fmax:
+ case Intrinsic::vector_reduce_fmaximum:
+ case Intrinsic::vector_reduce_fmin:
+ case Intrinsic::vector_reduce_fminimum:
+ return false;
+ case Intrinsic::vector_partial_reduce_add:
+ case Intrinsic::vector_partial_reduce_smax:
+ case Intrinsic::vector_partial_reduce_umax:
+ case Intrinsic::vector_partial_reduce_smin:
+ case Intrinsic::vector_partial_reduce_umin:
+ return true;
+ }
+}
+
Value *llvm::getReductionIdentity(Intrinsic::ID RdxID, Type *Ty,
FastMathFlags Flags) {
bool Negative = false;
@@ -1507,10 +1549,16 @@ Value *llvm::getReductionIdentity(Intrinsic::ID RdxID, Type *Ty,
return ConstantExpr::getBinOpIdentity(Opc, Ty, false,
Flags.noSignedZeros());
}
+ case Intrinsic::vector_partial_reduce_add:
+ return Constant::getNullValue(Ty);
case Intrinsic::vector_reduce_umax:
case Intrinsic::vector_reduce_umin:
case Intrinsic::vector_reduce_smin:
- case Intrinsic::vector_reduce_smax: {
+ case Intrinsic::vector_reduce_smax:
+ case Intrinsic::vector_partial_reduce_smax:
+ case Intrinsic::vector_partial_reduce_umax:
+ case Intrinsic::vector_partial_reduce_smin:
+ case Intrinsic::vector_partial_reduce_umin: {
Intrinsic::ID ScalarID = getMinMaxReductionIntrinsicOp(RdxID);
return ConstantExpr::getIntrinsicIdentity(ScalarID, Ty);
}
@@ -1536,15 +1584,16 @@ Value *llvm::getRecurrenceIdentity(RecurKind K, Type *Tp, FastMathFlags FMF) {
assert((!(K == RecurKind::FMin || K == RecurKind::FMax) ||
(FMF.noNaNs() && FMF.noSignedZeros())) &&
"nnan, nsz is expected to be set for FP min/max reduction.");
- Intrinsic::ID RdxID = getReductionIntrinsicID(K);
+ Intrinsic::ID RdxID = getReductionIntrinsicID(K, Tp);
return getReductionIdentity(RdxID, Tp, FMF);
}
Value *llvm::createSimpleReduction(IRBuilderBase &Builder, Value *Src,
- RecurKind RdxKind) {
- auto *SrcVecEltTy = cast<VectorType>(Src->getType())->getElementType();
+ RecurKind RdxKind, Type *ReducedTy) {
+ if (!ReducedTy)
+ ReducedTy = cast<VectorType>(Src->getType())->getElementType();
auto getIdentity = [&]() {
- return getRecurrenceIdentity(RdxKind, SrcVecEltTy,
+ return getRecurrenceIdentity(RdxKind, ReducedTy,
Builder.getFastMathFlags());
};
switch (RdxKind) {
@@ -1567,7 +1616,12 @@ Value *llvm::createSimpleReduction(IRBuilderBase &Builder, Value *Src,
case RecurKind::FMaximum:
case RecurKind::FMinimumNum:
case RecurKind::FMaximumNum:
- return Builder.CreateUnaryIntrinsic(getReductionIntrinsicID(RdxKind), Src);
+ case RecurKind::TargetIntAccumulation: {
+ Intrinsic::ID IID = getReductionIntrinsicID(RdxKind, ReducedTy);
+ if (isPartialReductionIntrinsicID(IID))
+ return Builder.CreateIntrinsic(ReducedTy, IID, {getIdentity(), Src});
+ return Builder.CreateUnaryIntrinsic(IID, Src);
+ }
case RecurKind::FMulAdd:
case RecurKind::FAddChainWithSubs:
case RecurKind::FSub:
@@ -1585,11 +1639,11 @@ Value *llvm::createSimpleReduction(IRBuilderBase &Builder, Value *Src,
assert(!RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind) &&
!RecurrenceDescriptor::isFindRecurrenceKind(Kind) &&
"AnyOf and FindIV reductions are not supported.");
- Intrinsic::ID Id = getReductionIntrinsicID(Kind);
+ auto *EltTy = cast<VectorType>(Src->getType())->getElementType();
+ Intrinsic::ID Id = getReductionIntrinsicID(Kind, EltTy);
auto VPID = VPIntrinsic::getForIntrinsic(Id);
assert(VPReductionIntrinsic::isVPReduction(VPID) &&
"No VPIntrinsic for this reduction");
- auto *EltTy = cast<VectorType>(Src->getType())->getElementType();
Value *Iden = getRecurrenceIdentity(Kind, EltTy, Builder.getFastMathFlags());
Value *Ops[] = {Iden, Src, Mask, EVL};
return Builder.CreateIntrinsic(EltTy, VPID, Ops);
@@ -1613,11 +1667,11 @@ Value *llvm::createOrderedReduction(IRBuilderBase &Builder, RecurKind Kind,
assert(Src->getType()->isVectorTy() && "Expected a vector type");
assert(!Start->getType()->isVectorTy() && "Expected a scalar type");
- Intrinsic::ID Id = getReductionIntrinsicID(RecurKind::FAdd);
+ auto *EltTy = cast<VectorType>(Src->getType())->getElementType();
+ Intrinsic::ID Id = getReductionIntrinsicID(RecurKind::FAdd, EltTy);
auto VPID = VPIntrinsic::getForIntrinsic(Id);
assert(VPReductionIntrinsic::isVPReduction(VPID) &&
"No VPIntrinsic for this reduction");
- auto *EltTy = cast<VectorType>(Src->getType())->getElementType();
Value *Ops[] = {Start, Src, Mask, EVL};
return Builder.CreateIntrinsic(EltTy, VPID, Ops);
}
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 69ca29885f384..3f799964d5221 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -79,6 +79,16 @@ static cl::opt<bool> EnableHistogramVectorization(
"enable-histogram-loop-vectorization", cl::init(false), cl::Hidden,
cl::desc("Enables autovectorization of some loops containing histograms"));
+namespace llvm {
+cl::opt<bool>
+ VectorizeVectorLoops("vectorize-vector-loops", cl::init(false), cl::Hidden,
+ cl::desc("Allow interleaving or vectorization of "
+ "loops with vector instructions."));
+cl::opt<bool>
+ InterleaveRevecLoops("interleave-revec-loops", cl::init(true), cl::Hidden,
+ cl::desc("Allow IC>1 for re-vectorised loops"));
+} // namespace llvm
+
/// Maximum vectorization interleave count.
static const unsigned MaxInterleaveFactor = 16;
@@ -808,6 +818,10 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
if (auto *Phi = dyn_cast<PHINode>(&I)) {
Type *PhiTy = Phi->getType();
+ if (auto *FVTy = dyn_cast<FixedVectorType>(PhiTy);
+ FVTy && VectorizeVectorLoops) {
+ PhiTy = FVTy->getElementType();
+ }
// Check that this PHI type is allowed.
if (!PhiTy->isIntegerTy() && !PhiTy->isFloatingPointTy() &&
!PhiTy->isPointerTy()) {
@@ -837,9 +851,20 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
return false;
}
+ // TODO-REVEC: Support more RecurKind.
+ auto CanRevecReduction = [](const RecurrenceDescriptor &RedDes) {
+ return is_contained({RecurKind::Add, RecurKind::TargetIntAccumulation,
+ RecurKind::SMax, RecurKind::SMin, RecurKind::UMax,
+ RecurKind::UMin},
+ RedDes.getRecurrenceKind());
+ };
+
RecurrenceDescriptor RedDes;
if (RecurrenceDescriptor::isReductionPHI(Phi, TheLoop, RedDes, DB, AC, DT,
- PSE.getSE())) {
+ PSE.getSE(), TTI)) {
+ if (Phi->getType()->isVectorTy() &&
+ (!VectorizeVectorLoops || !CanRevecReduction(RedDes)))
+ return false;
Requirements->addExactFPMathInst(RedDes.getExactFPMathInst());
Reductions[Phi] = std::move(RedDes);
assert((!RedDes.hasUsesOutsideReductionChain() ||
@@ -896,7 +921,7 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
// * Have a vector version available.
auto *CI = dyn_cast<CallInst>(&I);
- if (CI && !getVectorIntrinsicIDForCall(CI, TLI) &&
+ if (CI && !getVectorIntrinsicIDForCall(CI, TLI, TTI) &&
!(CI->getCalledFunction() && TLI &&
(!VFDatabase::getMappings(*CI).empty() || isTLIScalarize(*TLI, *CI)))) {
// If the call is a recognized math libary call, it is likely that
@@ -930,7 +955,7 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
// them to be vectorized (i.e. loop invariant).
if (CI) {
auto *SE = PSE.getSE();
- Intrinsic::ID IntrinID = getVectorIntrinsicIDForCall(CI, TLI);
+ Intrinsic::ID IntrinID = getVectorIntrinsicIDForCall(CI, TLI, TTI);
for (unsigned Idx = 0; Idx < CI->arg_size(); ++Idx)
if (isVectorIntrinsicWithScalarOpAtArg(IntrinID, Idx, TTI)) {
if (!SE->isLoopInvariant(PSE.getSCEV(CI->getOperand(Idx)), TheLoop)) {
@@ -948,8 +973,13 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
if (CI && !VFDatabase::getMappings(*CI).empty())
VecCallVariantsFound = true;
- auto CanWidenInstructionTy = [](Instruction const &Inst) {
+ auto CanWidenInstructionTy = [TTI = TTI](Instruction const &Inst) {
Type *InstTy = Inst.getType();
+
+ if (InstTy->isVectorTy())
+ return VectorizeVectorLoops &&
+ TTI->isElementTypeLegalForScalableVector(InstTy);
+
if (!isa<StructType>(InstTy))
return canVectorizeTy(InstTy);
@@ -960,12 +990,24 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
all_of(Inst.users(), IsaPred<ExtractValueInst>);
};
+ auto CanWidenCast = [&](const Instruction &CastI) {
+ assert(isa<CastInst>(CastI));
+ assert(CanWidenInstructionTy(CastI) &&
+ "CanWidenInstructionTy was not checked beforehand.");
+ Type *FromTy = CastI.getOperand(0)->getType();
+ return VectorType::isValidElementType(FromTy) ||
+ (VectorizeVectorLoops &&
+ VectorType::isValidElementType(FromTy->getScalarType()));
+ };
+
+ if (isa<VectorType>(I.getType()) ||
+ any_of(I.operand_values(),
+ [](const Value *V) { return V->getType()->isVectorTy(); }))
+ LoopContainsVectors = true;
+
// Check that the instruction return type is vectorizable.
- // We can't vectorize casts from vector type to scalar type.
// Also, we can't vectorize extractelement instructions.
- if (!CanWidenInstructionTy(I) ||
- (isa<CastInst>(I) &&
- !VectorType::isValidElementType(I.getOperand(0)->getType())) ||
+ if (!CanWidenInstructionTy(I) || (isa<CastInst>(I) && !CanWidenCast(I)) ||
isa<ExtractElementInst>(I)) {
reportVectorizationFailure("Found unvectorizable type",
"instruction return type cannot be vectorized",
@@ -977,7 +1019,10 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
// Check that the stored type is vectorizable.
if (auto *ST = dyn_cast<StoreInst>(&I)) {
Type *T = ST->getValueOperand()->getType();
- if (!VectorType::isValidElementType(T)) {
+ bool CanWidenStoreType = VectorType::isValidElementType(T) ||
+ (isa<FixedVectorType>(T) && VectorizeVectorLoops &&
+ TTI->isElementTypeLegalForScalableVector(T));
+ if (!CanWidenStoreType) {
reportVectorizationFailure("Store instruction cannot be vectorized",
"CantVectorizeStore", ORE, TheLoop, ST);
return false;
@@ -987,7 +1032,8 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
// supported on the target.
if (ST->getMetadata(LLVMContext::MD_nontemporal)) {
// Arbitrarily try a vector of 2 elements.
- auto *VecTy = FixedVectorType::get(T, /*NumElts=*/2);
+ Type *VecTy =
+ T->isVectorTy() ? T : FixedVectorType::get(T, /*NumElts=*/2);
assert(VecTy && "did not find vectorized version of stored type");
if (!TTI->isLegalNTStore(VecTy, ST->getAlign())) {
reportVectorizationFailure(
@@ -1001,7 +1047,9 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
if (LD->getMetadata(LLVMContext::MD_nontemporal)) {
// For nontemporal loads, check that a nontemporal vector version is
// supported on the target (arbitrarily try a vector of 2 elements).
- auto *VecTy = FixedVectorType::get(I.getType(), /*NumElts=*/2);
+ Type *VecTy = I.getType()->isVectorTy()
+ ? I.getType()
+ : FixedVectorType::get(I.getType(), /*NumElts=*/2);
assert(VecTy && "did not find vectorized version of load type");
if (!TTI->isLegalNTLoad(VecTy, LD->getAlign())) {
reportVectorizationFailure(
@@ -1498,7 +1546,7 @@ bool LoopVectorizationLegality::canVectorizeWithIfConvert() {
// Pass the Predicates pointer to isDereferenceableAndAlignedInLoop so
// that it will consider loops that need guarding by SCEV checks. The
// vectoriser will generate these checks if we decide to vectorise.
- if (LI && !LI->getType()->isVectorTy() && !mustSuppressSpeculation(*LI) &&
+ if (LI && !mustSuppressSpeculation(*LI) &&
CanSpeculatePointerOp(LI->getPointerOperand()) &&
isDereferenceableAndAlignedInLoop(LI, TheLoop, SE, *DT, AC,
&Predicates))
@@ -1996,6 +2044,13 @@ bool LoopVectorizationLegality::canFoldTailByMasking() const {
LLVM_DEBUG(dbgs() << "LV: checking if tail can be folded by masking.\n");
+ // TODO-REVEC: Disable tail-folding for now: codegen of active.lane.mask for
+ // <vscale x 1 x i1> is pretty bad at the moment and it has an Invalid cost.
+ if (LoopContainsVectors) {
+ LLVM_DEBUG(dbgs() << "LV: Tail-folding disabled for REVEC.\n");
+ return false;
+ }
+
// The list of pointers that we can safely read and write to remains empty.
SmallPtrSet<Value *, 8> SafePointers;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
index c922ca7d32dbf..b40dd2ccbb05a 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
@@ -120,11 +120,13 @@ void LoopVectorizationUtils::reportVectorizationInfo(
void LoopVectorizationUtils::reportVectorization(OptimizationRemarkEmitter *ORE,
Loop *TheLoop,
ElementCount VFWidth,
- unsigned IC) {
+ unsigned IC,
+ bool ContainsVectors) {
LLVM_DEBUG(debugVectorizationMessage(
"Vectorizing: ", TheLoop->isInnermost() ? "innermost loop" : "outer loop",
nullptr));
- StringRef LoopType = TheLoop->isInnermost() ? "" : "outer ";
+ StringRef LoopType =
+ TheLoop->isInnermost() ? (ContainsVectors ? "vector " : "") : "outer ";
ORE->emit([&]() {
return OptimizationRemark(DEBUG_TYPE, "Vectorized", TheLoop->getStartLoc(),
TheLoop->getHeader())
@@ -154,10 +156,17 @@ bool VFSelectionContext::isLegalGatherOrScatter(Value *V,
return false;
auto *Ty = getLoadStoreType(V);
Align Align = getLoadStoreAlignment(V);
+ bool Revec = Ty->isVectorTy();
+
if (VF.isVector())
- Ty = VectorType::get(Ty, VF);
- return ForceTargetSupportsGatherScatterOps ||
- (LI && TTI.isLegalMaskedGather(Ty, Align)) ||
+ Ty = toVectorTy(Ty, VF);
+
+ if (ForceTargetSupportsGatherScatterOps)
+ return true;
+ if (Revec)
+ return (LI && TTI.isLegalMaskedSegmentGather(Ty, Align)) ||
+ (SI && TTI.isLegalMaskedSegmentScatter(Ty, Align));
+ return (LI && TTI.isLegalMaskedGather(Ty, Align)) ||
(SI && TTI.isLegalMaskedScatter(Ty, Align));
}
@@ -278,7 +287,8 @@ ElementCount VFSelectionContext::getMaximizedVFForTarget(
else
MaxPermissibleVFWithoutMaxBW.FixedVF = MaxVF;
- if (useMaxBandwidth(ComputeScalableMaxVF)) {
+ // REVEC: Avoid creating "illegal" scalable types by choosing VF > vscale x 1.
+ if (useMaxBandwidth(ComputeScalableMaxVF) && !Legal->LoopContainsVectors) {
auto MaxVectorElementCountMaxBW = ElementCount::get(
llvm::bit_floor(WidestRegister.getKnownMinValue() / SmallestType),
ComputeScalableMaxVF);
@@ -337,7 +347,9 @@ bool VFSelectionContext::isScalableVectorizationAllowed() {
// Disable scalable vectorization if the loop contains unsupported reductions.
if (!all_of(Legal->getReductionVars(), [&](const auto &Reduction) -> bool {
- return TTI.isLegalToVectorizeReduction(Reduction.second, MaxScalableVF);
+ return RecurrenceDescriptor::isTargetRecurrenceKind(
+ Reduction.second.getRecurrenceKind()) ||
+ TTI.isLegalToVectorizeReduction(Reduction.second, MaxScalableVF);
})) {
reportVectorizationInfo(
"Scalable vectorization not supported for the reduction "
@@ -349,6 +361,8 @@ bool VFSelectionContext::isScalableVectorizationAllowed() {
// Disable scalable vectorization if the loop contains any instructions
// with element types not supported for scalable vectors.
if (any_of(ElementTypesInLoop, [&](Type *Ty) {
+ if (isa<FixedVectorType>(Ty) && !VectorizeVectorLoops)
+ return false;
return !Ty->isVoidTy() && !TTI.isElementTypeLegalForScalableVector(Ty);
})) {
reportVectorizationInfo("Scalable vectorization is not supported "
@@ -411,6 +425,11 @@ FixedScalableVFPair VFSelectionContext::computeFeasibleMaxVF(
auto MaxSafeFixedVF = ElementCount::getFixed(MaxSafeElementsPowerOf2);
auto MaxSafeScalableVF = getMaxLegalScalableVF(MaxSafeElementsPowerOf2);
+ // A vector loop can only be interleaved or widened to a scalable vector loop.
+ // TODO-REVEC: Support fixed-length REVEC
+ if (Legal->LoopContainsVectors)
+ MaxSafeFixedVF = ElementCount::getFixed(1);
+
if (!Legal->isSafeForAnyVectorWidth())
MaxSafeElements = MaxSafeElementsPowerOf2;
@@ -426,9 +445,13 @@ FixedScalableVFPair VFSelectionContext::computeFeasibleMaxVF(
if (ElementCount::isKnownLE(UserVF, MaxSafeUserVF)) {
// If `VF=vscale x N` is safe, then so is `VF=N`
- if (UserVF.isScalable())
- return FixedScalableVFPair(
- ElementCount::getFixed(UserVF.getKnownMinValue()), UserVF);
+ // ... unless doing REVEC.
+ if (UserVF.isScalable()) {
+ auto UserVFAsFixed = ElementCount::getFixed(UserVF.getKnownMinValue());
+ return ElementCount::isKnownLE(UserVFAsFixed, MaxSafeFixedVF)
+ ? FixedScalableVFPair(UserVFAsFixed, UserVF)
+ : FixedScalableVFPair(MaxSafeFixedVF, UserVF);
+ }
return UserVF;
}
@@ -517,17 +540,19 @@ VFSelectionContext::getSmallestAndWidestTypes() const {
// for casts on the input operands of the recurrence.
MinWidth = std::min(
MinWidth,
- std::min(RdxDesc.getMinWidthCastToRecurrenceTypeInBits(),
- RdxDesc.getRecurrenceType()->getScalarSizeInBits()));
- MaxWidth = std::max(MaxWidth,
- RdxDesc.getRecurrenceType()->getScalarSizeInBits());
+ std::min<unsigned>(RdxDesc.getMinWidthCastToRecurrenceTypeInBits(),
+ DL.getTypeSizeInBits(RdxDesc.getRecurrenceType())
+ .getFixedValue()));
+ MaxWidth = std::max<unsigned>(
+ MaxWidth,
+ DL.getTypeSizeInBits(RdxDesc.getRecurrenceType()).getFixedValue());
}
} else {
for (Type *T : ElementTypesInLoop) {
- MinWidth = std::min<unsigned>(
- MinWidth, DL.getTypeSizeInBits(T->getScalarType()).getFixedValue());
- MaxWidth = std::max<unsigned>(
- MaxWidth, DL.getTypeSizeInBits(T->getScalarType()).getFixedValue());
+ MinWidth =
+ std::min<unsigned>(MinWidth, DL.getTypeSizeInBits(T).getFixedValue());
+ MaxWidth =
+ std::max<unsigned>(MaxWidth, DL.getTypeSizeInBits(T).getFixedValue());
}
}
return {MinWidth, MaxWidth};
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index dbbb5e75adc66..20a98fb343448 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -91,7 +91,8 @@ void reportVectorizationInfo(const StringRef Msg, const StringRef ORETag,
/// Report successful vectorization of the loop. In case an outer loop is
/// vectorized, prepend "outer" to the vectorization remark.
void reportVectorization(OptimizationRemarkEmitter *ORE, Loop *TheLoop,
- ElementCount VFWidth, unsigned IC);
+ ElementCount VFWidth, unsigned IC,
+ bool ContainsVectors);
} // namespace LoopVectorizationUtils
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index fcfe11b6e9de6..a14e0fd7a6de5 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -168,6 +168,8 @@ const char VerboseDebug[] = DEBUG_TYPE "-verbose";
#endif
STATISTIC(LoopsVectorized, "Number of loops vectorized");
+STATISTIC(VectorLoopsVectorized, "Number of vector loops revectorized (VF>1)");
+STATISTIC(VectorLoopsInterleaved, "Number of vector loops interleaved (IC>1)");
STATISTIC(LoopsAnalyzed, "Number of loops analyzed for vectorization");
STATISTIC(LoopsEpilogueVectorized, "Number of epilogues vectorized");
STATISTIC(LoopsEarlyExitVectorized, "Number of early exit loops vectorized");
@@ -1124,6 +1126,9 @@ class LoopVectorizationCostModel {
return;
}
+ assert(!Legal->LoopContainsVectors &&
+ "Predicating re-vectorised instructions isn't supported yet");
+
// Default to TTI preference, but allow command line override.
ChosenTailFoldingStyle = TTI.getPreferredTailFoldingStyle();
if (ForceTailFoldingStyle.getNumOccurrences())
@@ -2141,7 +2146,7 @@ LoopVectorizationCostModel::getVectorCallCost(CallInst *CI,
: ScalarCallCost * VF.getKnownMinValue() +
getScalarizationOverhead(CI, VF);
- if (getVectorIntrinsicIDForCall(CI, TLI)) {
+ if (getVectorIntrinsicIDForCall(CI, TLI, &TTI)) {
InstructionCost IntrinsicCost = getVectorIntrinsicCost(CI, VF);
return std::min(Cost, IntrinsicCost);
}
@@ -2157,7 +2162,7 @@ static Type *maybeVectorizeType(Type *Ty, ElementCount VF) {
InstructionCost
LoopVectorizationCostModel::getVectorIntrinsicCost(CallInst *CI,
ElementCount VF) const {
- Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
+ Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI, &TTI);
assert(ID && "Expected intrinsic call!");
Type *RetTy = maybeVectorizeType(CI->getType(), VF);
FastMathFlags FMF;
@@ -2414,7 +2419,7 @@ bool LoopVectorizationCostModel::isScalarWithPredication(Instruction *I,
return true;
auto *CI = cast<CallInst>(I);
// A vector intrinsic or library variant lowering avoids scalarization.
- return !getVectorIntrinsicIDForCall(CI, TLI) &&
+ return !getVectorIntrinsicIDForCall(CI, TLI, &TTI) &&
!hasVectorLibraryVariantFor(*CI, VF, isMaskRequired(CI), TLI);
}
case Instruction::Load:
@@ -3219,6 +3224,13 @@ void LoopVectorizationPlanner::emitInvalidCostRemarks(
})
.Case([](const VPReductionRecipe *R) {
return RecurrenceDescriptor::getOpcode(R->getRecurrenceKind());
+ })
+ .Case<VPWidenShuffleRecipe>(
+ [](const auto *R) { return Instruction::ShuffleVector; })
+ .Default([](const auto *R) -> unsigned {
+ LLVM_DEBUG(R->dump());
+ llvm_unreachable(
+ "Unhandled recipe type in emitInvalidCostRemarks");
});
// If the next recipe is different, or if there are no other pairs,
@@ -3247,8 +3259,21 @@ void LoopVectorizationPlanner::emitInvalidCostRemarks(
Name = CalledFn->getName();
}
OS << " call to " << Name;
- } else
- OS << " " << Instruction::getOpcodeName(Opcode);
+ } else {
+ // TODO-REVEC: This isn't great to handle VPInst-specific opcodes here,
+ // but I want to avoid a larger refactoring that would create conflicts.
+ switch (Opcode) {
+ case VPInstruction::ExtractVectors:
+ OS << " ExtractVectors";
+ break;
+ case VPInstruction::ConcatVectors:
+ OS << " ConcatVectors";
+ break;
+ default:
+ OS << " " << Instruction::getOpcodeName(Opcode);
+ break;
+ }
+ }
reportVectorizationInfo(OutString, "InvalidCost", ORE, OrigLoop, nullptr,
R->getDebugLoc());
Tail = Tail.drop_front(Subset.size());
@@ -3301,6 +3326,7 @@ static bool willGenerateVectors(VPlan &Plan, ElementCount VF,
case VPRecipeBase::VPBlendSC:
case VPRecipeBase::VPFirstOrderRecurrencePHISC:
case VPRecipeBase::VPHistogramSC:
+ case VPRecipeBase::VPWidenShuffleSC:
case VPRecipeBase::VPWidenPHISC:
case VPRecipeBase::VPWidenIntOrFpInductionSC:
case VPRecipeBase::VPWidenPointerInductionSC:
@@ -3674,6 +3700,9 @@ LoopVectorizationPlanner::selectInterleaveCount(VPlan &Plan, ElementCount VF,
if (Plan.hasEarlyExit())
return 1;
+ if (Legal->LoopContainsVectors && !InterleaveRevecLoops)
+ return 1;
+
const bool HasReductions =
any_of(Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
IsaPred<VPReductionPHIRecipe>);
@@ -4377,8 +4406,17 @@ LoopVectorizationCostModel::getInterleaveGroupCost(Instruction *I,
auto *VectorTy = cast<VectorType>(toVectorTy(ValTy, VF));
unsigned AS = getLoadStoreAddressSpace(InsertPos);
+ // REVEC: Can only handle ValTy segments due to how (de)interleave.segments
+ // intrinsics work, so expect VF=vscalex1
+ if (ValTy->isVectorTy() && VF.getKnownMinValue() != 1)
+ return InstructionCost::getInvalid();
+
+ // TODO-REVEC: Support reversing segments
+ if (ValTy->isVectorTy() && Group->isReverse())
+ return InstructionCost::getInvalid();
+
unsigned InterleaveFactor = Group->getFactor();
- auto *WideVecTy = VectorType::get(ValTy, VF * InterleaveFactor);
+ auto *WideVecTy = toVectorTy(ValTy, VF * InterleaveFactor);
// Holds the indices of existing members in the interleaved group.
SmallVector<unsigned, 4> Indices;
@@ -4390,12 +4428,19 @@ LoopVectorizationCostModel::getInterleaveGroupCost(Instruction *I,
bool UseMaskForGaps =
(Group->requiresScalarEpilogue() && !isEpilogueAllowed()) ||
(isa<StoreInst>(I) && !Group->isFull());
- InstructionCost Cost = TTI.getInterleavedMemoryOpCost(
- InsertPos->getOpcode(), WideVecTy, Group->getFactor(), Indices,
- Group->getAlign(), AS, Config.CostKind, isMaskRequired(I),
- UseMaskForGaps);
+ InstructionCost Cost =
+ ValTy->isVectorTy()
+ ? TTI.getSegmentInterleavedMemoryOpCost(
+ InsertPos->getOpcode(), WideVecTy, Group->getFactor(), Indices,
+ Group->getAlign(), AS, Config.CostKind, isMaskRequired(I),
+ UseMaskForGaps)
+ : TTI.getInterleavedMemoryOpCost(
+ InsertPos->getOpcode(), WideVecTy, Group->getFactor(), Indices,
+ Group->getAlign(), AS, Config.CostKind, isMaskRequired(I),
+ UseMaskForGaps);
if (Group->isReverse()) {
+ assert(!ValTy->isVectorTy() && "Cannot reverse segments yet.");
// TODO: Add support for reversed masked interleaved access.
assert(!isMaskRequired(I) &&
"Reverse masked interleaved access not supported.");
@@ -4486,6 +4531,10 @@ LoopVectorizationCostModel::getReductionPatternCost(Instruction *I,
? dyn_cast<Instruction>(RetI->getOperand(0))
: dyn_cast<Instruction>(RetI->getOperand(1));
+ // TODO-REVEC: Let's not bother with the legacy cost model.
+ if (I->getOperand(0)->getType()->isVectorTy())
+ return std::nullopt;
+
VectorTy = VectorType::get(I->getOperand(0)->getType(), VectorTy);
Instruction *Op0, *Op1;
@@ -4625,6 +4674,9 @@ LoopVectorizationCostModel::getScalarizationOverhead(Instruction *I,
if (VF.isScalar())
return 0;
+ assert(!isa<VectorType>(I->getType()) &&
+ "Unexpected vector type for fixed VF");
+
InstructionCost Cost = 0;
Type *RetTy = toVectorizedTy(I->getType(), VF);
if (!RetTy->isVoidTy() &&
@@ -5135,6 +5187,7 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I,
// fold away. We can generalize this for all operations using the notion
// of neutral elements. (TODO)
if (I->getOpcode() == Instruction::Mul &&
+ PSE.getSE()->isSCEVable(I->getOperand(0)->getType()) &&
((TheLoop->isLoopInvariant(I->getOperand(0)) &&
PSE.getSCEV(I->getOperand(0))->isOne()) ||
(TheLoop->isLoopInvariant(I->getOperand(1)) &&
@@ -5173,8 +5226,23 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I,
}
case Instruction::Select: {
SelectInst *SI = cast<SelectInst>(I);
- const SCEV *CondSCEV = SE->getSCEV(SI->getCondition());
- bool ScalarCond = (SE->isLoopInvariant(CondSCEV, TheLoop));
+ Type *CondTy = SI->getCondition()->getType();
+ assert(CondTy->isVectorTy() || SE->isSCEVable(CondTy));
+
+ // TODO-REVEC: Support Select instructions
+ // If the condition is already a vector, check whether we can
+ // re-vectorise it.
+ if (CondTy->isVectorTy() && !VF.isScalar()) {
+ assert(VF.isScalable() && "Unexpected fixed VF for re-vectorisation");
+ if (!TTI.isElementTypeLegalForScalableVector(CondTy))
+ return InstructionCost::getInvalid();
+ }
+
+ bool ScalarCond = false;
+ if (SE->isSCEVable(CondTy)) {
+ const SCEV *CondSCEV = SE->getSCEV(SI->getCondition());
+ ScalarCond = (SE->isLoopInvariant(CondSCEV, TheLoop));
+ }
const Value *Op0, *Op1;
using namespace llvm::PatternMatch;
@@ -5193,9 +5261,8 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I,
I);
}
- Type *CondTy = SI->getCondition()->getType();
if (!ScalarCond)
- CondTy = VectorType::get(CondTy, VF);
+ CondTy = toVectorTy(CondTy, VF);
CmpInst::Predicate Pred = CmpInst::BAD_ICMP_PREDICATE;
if (auto *Cmp = dyn_cast<CmpInst>(SI->getCondition()))
@@ -5344,6 +5411,11 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I,
case Instruction::Freeze:
return TTI::TCC_Free;
default:
+ // TODO-REVEC: If re-vectorising, we can expect new instruction opcodes to
+ // reach here. Be conservative.
+ if (I->getType()->isVectorTy() && !VF.isScalar())
+ return InstructionCost::getInvalid();
+
// This opcode is unknown. Assume that it is the same as 'mul'.
return TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy,
Config.CostKind);
@@ -6340,6 +6412,53 @@ VPHistogramRecipe *VPRecipeBuilder::widenIfHistogram(VPInstruction *VPI) {
VPI->getDebugLoc());
}
+VPSingleDefRecipe *VPRecipeBuilder::tryToWidenShuffle(VPInstruction *VPI) {
+ assert(VPI->getOpcode() == Instruction::ShuffleVector);
+ auto &SVI = *cast<ShuffleVectorInst>(VPI->getUnderlyingInstr());
+
+ ArrayRef<int> Mask = SVI.getShuffleMask();
+ auto Size = Mask.size();
+ unsigned NumSrcElts =
+ cast<FixedVectorType>(SVI.getOperand(0)->getType())->getNumElements();
+
+ VPValue *Src0 = VPI->getOperand(0);
+ VPValue *Src1 = VPI->getOperand(1);
+
+ // Detect "aligned" subvector extracts of half the size.
+ if (int Index; SVI.isExtractSubvectorMask(Index) && Size == NumSrcElts / 2 &&
+ Index % Size == 0) {
+ const unsigned Factor = NumSrcElts / Size;
+ Type *I32Ty = IntegerType::getInt32Ty(SVI.getContext());
+ VPValue *IndexV = Plan.getOrAddLiveIn(ConstantInt::get(I32Ty, Index));
+ VPValue *FactorV = Plan.getOrAddLiveIn(ConstantInt::get(I32Ty, Factor));
+ return new VPInstruction(VPInstruction::ExtractVectors,
+ {Src0, IndexV, FactorV});
+ }
+
+ // Detect concatenation shuffles, including concat with undef
+ // and concat with self.
+ bool IsFirstSrcShuffle =
+ all_of(Mask, [NumSrcElts](int Idx) { return Idx < int(NumSrcElts); });
+ if (SVI.isConcat() || (Size == NumSrcElts * 2 && IsFirstSrcShuffle &&
+ SVI.isIdentityWithPadding())) {
+ return new VPInstruction(VPInstruction::ConcatVectors, {Src0, Src1});
+ }
+ if (Size == NumSrcElts * 2 && IsFirstSrcShuffle &&
+ ShuffleVectorInst::isIdentityMask(Mask.take_front(NumSrcElts),
+ NumSrcElts) &&
+ ShuffleVectorInst::isIdentityMask(Mask.take_back(NumSrcElts),
+ NumSrcElts)) {
+ return new VPInstruction(VPInstruction::ConcatVectors, {Src0, Src0});
+ }
+
+ if (SVI.changesLength())
+ return nullptr;
+
+ return new VPWidenShuffleRecipe({Src0, Src1}, SVI.getShuffleMask(),
+ SVI.getType(), VPIRFlags(SVI),
+ cast<VPIRMetadata>(*VPI), VPI->getDebugLoc());
+}
+
bool VPRecipeBuilder::replaceWithFinalIfReductionStore(
VPInstruction *VPI, VPBuilder &FinalRedStoresBuilder) {
StoreInst *SI;
@@ -6476,6 +6595,9 @@ VPRecipeBuilder::tryToCreateWidenNonPhiRecipe(VPSingleDefRecipe *R,
VPI->getDebugLoc());
}
+ if (VPI->getOpcode() == Instruction::ShuffleVector)
+ return tryToWidenShuffle(VPI);
+
return tryToWiden(VPI);
}
@@ -6614,7 +6736,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
if (Plan->isOuterLoop()) {
for (ElementCount VF : Range)
Plan->addVF(VF);
- if (!VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(*Plan, *TLI))
+ if (!VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(*Plan, *TLI,
+ TTI))
return nullptr;
VPlanTransforms::optimizeInductionLiveOutUsers(*Plan, PSE,
/*FoldTail=*/false);
@@ -8216,16 +8339,23 @@ bool LoopVectorizePass::processLoop(Loop *L) {
ORE->emit([&]() {
return OptimizationRemark(LV_NAME, "Interleaved", L->getStartLoc(),
L->getHeader())
- << "interleaved loop (interleaved count: "
- << NV("InterleaveCount", IC) << ")";
+ << "interleaved " << (LVL.LoopContainsVectors ? "vector " : "")
+ << "loop (interleaved count: " << NV("InterleaveCount", IC) << ")";
});
} else {
// Report the vectorization decision.
- reportVectorization(ORE, L, VF.Width, IC);
+ reportVectorization(ORE, L, VF.Width, IC, LVL.LoopContainsVectors);
}
if (ORE->allowExtraAnalysis(LV_NAME))
checkMixedPrecision(L, ORE);
+ if (LVL.LoopContainsVectors) {
+ if (!VF.Width.isScalar())
+ ++VectorLoopsVectorized;
+ if (IC > 1)
+ ++VectorLoopsInterleaved;
+ }
+
// If we decided that it is *legal* to interleave or vectorize the loop, then
// do it.
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 99ed09cc684b9..98a3777c725f9 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -30620,6 +30620,7 @@ class HorizontalReduction {
case RecurKind::FMinNum:
case RecurKind::FMaximumNum:
case RecurKind::FMinimumNum:
+ case RecurKind::TargetIntAccumulation:
case RecurKind::None:
llvm_unreachable("Unexpected reduction kind for repeated scalar.");
}
@@ -30778,6 +30779,7 @@ class HorizontalReduction {
case RecurKind::FMinNum:
case RecurKind::FMaximumNum:
case RecurKind::FMinimumNum:
+ case RecurKind::TargetIntAccumulation:
case RecurKind::None:
llvm_unreachable("Unexpected reduction kind for repeated scalar.");
}
@@ -30883,6 +30885,7 @@ class HorizontalReduction {
case RecurKind::FMinNum:
case RecurKind::FMaximumNum:
case RecurKind::FMinimumNum:
+ case RecurKind::TargetIntAccumulation:
case RecurKind::None:
llvm_unreachable("Unexpected reduction kind for reused scalars.");
}
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index f2777f99b407a..60024e0e54134 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -70,6 +70,8 @@ class VPRecipeBuilder {
/// scalar loop.
VPHistogramRecipe *widenIfHistogram(VPInstruction *VPI);
+ VPSingleDefRecipe *tryToWidenShuffle(VPInstruction *VPI);
+
/// If \p VPI is a store of a reduction into an invariant address, delete it.
/// If it is the final store of a reduction result, a uniform store recipe
/// will be created for it in the middle block. Returns `true` if replacement
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 88a328ac7b27b..81198fb141924 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -299,11 +299,25 @@ Value *VPTransformState::get(const VPValue *Def, const VPLane &Lane) {
assert(hasVectorValue(Def));
auto *VecPart = Data.VPV2Vector[Def];
- if (!VecPart->getType()->isVectorTy()) {
+ // If VecPart's type is the same as the initial pre-vectorisation type,
+ // Def hasn't been vectorised. We are done.
+ Type *InitialTy = Def->getScalarType();
+ if (VecPart->getType() == InitialTy) {
assert(Lane.isFirstLane() && "cannot get lane > 0 for scalar");
return VecPart;
}
+
+ // Last case: Extract a lane of type InitialTy from a vectorised value.
// TODO: Cache created scalar values.
+ if (auto *InitVTy = dyn_cast<VectorType>(InitialTy)) {
+ assert(!InitialTy->isScalableTy() && "REVEC: Unexpected scalable vector");
+ int EC = InitVTy->getElementCount().getFixedValue();
+ // Shift the demanded InitVTy-typed lane into lane 0.
+ int NumLanesFromEnd = VF.getKnownMinValue() - Lane.getOffsetInLastSubvec();
+ auto *ShiftLastSubvec = Builder.CreateVectorSpliceRight(
+ VecPart, PoisonValue::get(VecPart->getType()), NumLanesFromEnd * EC);
+ return Builder.CreateExtractVector(InitialTy, ShiftLastSubvec, uint64_t(0));
+ }
Value *LaneV = Lane.getAsRuntimeExpr(Builder, VF);
auto *Extract = Builder.CreateExtractElement(VecPart, LaneV);
// set(Def, Extract, Instance);
@@ -328,7 +342,7 @@ Value *VPTransformState::get(const VPValue *Def, bool NeedsScalar) {
auto GetBroadcastInstrs = [this](Value *V) {
if (VF.isScalar())
return V;
- // Broadcast the scalar into all locations in the vector.
+ // Broadcast the value into all locations in the vector.
Value *Shuf = Builder.CreateVectorSplat(VF, V, "broadcast");
return Shuf;
};
@@ -348,6 +362,25 @@ Value *VPTransformState::get(const VPValue *Def, bool NeedsScalar) {
return VectorValue;
}
+Value *VPTransformState::getAndStretch(const VPValue *Def,
+ ElementCount StretchFactor) {
+ Value *V = get(Def);
+
+ if (StretchFactor.isScalar())
+ return V;
+
+ ElementCount InitialEC = getElementCount(V->getType());
+ if (InitialEC.isScalar())
+ return Builder.CreateVectorSplat(StretchFactor, V);
+
+ // Now we know that V is a vector value that needs to be stretched.
+ assert((!InitialEC.isScalable() || !StretchFactor.isScalable()) &&
+ "Trying to stretch a scalable value by a scalable count");
+ Type *ResultTy = toVectorTy(V->getType(), StretchFactor);
+ return Builder.CreateIntrinsic(Intrinsic::vector_stretch,
+ {ResultTy, V->getType()}, {V}, nullptr);
+}
+
void VPTransformState::setDebugLocFrom(DebugLoc DL) {
const DILocation *DIL = DL;
// When a FSDiscriminator is enabled, we don't need to add the multiply
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 46788a23eef99..237cc94004f70 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -443,6 +443,7 @@ class LLVM_ABI_FOR_TEST VPRecipeBase
VPWidenStoreEVLSC,
VPWidenStoreSC,
VPWidenSC,
+ VPWidenShuffleSC,
VPBlendSC,
VPHistogramSC,
// START: Phi-like recipes. Need to be kept together.
@@ -639,6 +640,7 @@ class VPSingleDefRecipe : public VPRecipeBase, public VPSingleDefValue {
case VPRecipeBase::VPWidenIntrinsicSC:
case VPRecipeBase::VPWidenMemIntrinsicSC:
case VPRecipeBase::VPWidenSC:
+ case VPRecipeBase::VPWidenShuffleSC:
case VPRecipeBase::VPBlendSC:
case VPRecipeBase::VPPredInstPHISC:
case VPRecipeBase::VPCurrentIterationPHISC:
@@ -1134,6 +1136,7 @@ struct VPRecipeWithIRFlags : public VPSingleDefRecipe, public VPIRFlags {
R->getVPRecipeID() == VPRecipeBase::VPWidenCastSC ||
R->getVPRecipeID() == VPRecipeBase::VPWidenIntrinsicSC ||
R->getVPRecipeID() == VPRecipeBase::VPWidenMemIntrinsicSC ||
+ R->getVPRecipeID() == VPRecipeBase::VPWidenShuffleSC ||
R->getVPRecipeID() == VPRecipeBase::VPReductionSC ||
R->getVPRecipeID() == VPRecipeBase::VPReductionEVLSC ||
R->getVPRecipeID() == VPRecipeBase::VPReplicateSC ||
@@ -1277,6 +1280,16 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
// part if it is scalar. In the latter case, the recipe will be removed
// during unrolling.
ExtractPenultimateElement,
+ // ExtractVectors(VectorOp, Index, DeinterleaveFactor):
+ // Extract multiple (if VF>1) subvectors from its first operand into a
+ // result vector. This is equivalent to deinterleave.segments intrinsics.
+ ExtractVectors,
+ // ConcatVectors(Vec0, Vec1):
+ // Concatenate VF*2 subvectors from two sources. This will interleave
+ // subvectors (of Vec0's initial type) from both sources using
+ // interleave.segments intrinsics.
+ ConcatVectors,
+ // TODO-REVEC: ExtractElements
LogicalAnd, // Non-poison propagating logical And.
LogicalOr, // Non-poison propagating logical Or.
NumActiveLanes, // Counts the number of active lanes in a mask.
@@ -1932,7 +1945,7 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
}
/// Helper function to produce the widened intrinsic call.
- CallInst *createVectorCall(VPTransformState &State);
+ Instruction *createVectorCall(VPTransformState &State);
public:
VPWidenIntrinsicRecipe(CallInst &CI, Intrinsic::ID VectorIntrinsicID,
@@ -2179,6 +2192,50 @@ class VPHistogramRecipe : public VPRecipeBase, public VPIRMetadata {
#endif
};
+/// Widen a known shufflevector pattern using llvm.vector.segmented.shuffle.
+class VPWidenShuffleRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
+
+public:
+ VPWidenShuffleRecipe(ArrayRef<VPValue *> Operands, ArrayRef<int> ShuffleMask,
+ Type *OriginalType, const VPIRFlags &Flags = {},
+ const VPIRMetadata &Metadata = {},
+ DebugLoc DL = DebugLoc::getUnknown())
+ : VPRecipeWithIRFlags(VPRecipeBase::VPWidenShuffleSC, Operands,
+ OriginalType, Flags, DL),
+ VPIRMetadata(Metadata), ShuffleMask(ShuffleMask) {
+ assert(Operands.size() == 2);
+ }
+
+ ~VPWidenShuffleRecipe() override = default;
+
+ VPWidenShuffleRecipe *clone() override {
+ return new VPWidenShuffleRecipe(operands(), getShuffleMask(),
+ getScalarType(), *this, *this,
+ getDebugLoc());
+ }
+
+ VP_CLASSOF_IMPL(VPRecipeBase::VPWidenShuffleSC);
+
+ void execute(VPTransformState &State) override;
+
+ InstructionCost computeCost(ElementCount VF,
+ VPCostContext &Ctx) const override;
+
+ unsigned getOpcode() const { return Instruction::ShuffleVector; }
+
+ /// Return the shuffle mask that is applied within each segment.
+ ArrayRef<int> getShuffleMask() const { return ShuffleMask; }
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+ /// Print the recipe
+ void printRecipe(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const override;
+#endif
+
+private:
+ SmallVector<int, 8> ShuffleMask;
+};
+
/// A recipe for handling GEP instructions.
class LLVM_ABI_FOR_TEST VPWidenGEPRecipe : public VPRecipeWithIRFlags {
Type *SourceElementTy;
@@ -2829,6 +2886,9 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe, public VPIRFlags {
: VPHeaderPHIRecipe(VPRecipeBase::VPReductionPHISC, Phi, &Start),
VPIRFlags(Flags), Kind(Kind), Style(Style),
HasUsesOutsideReductionChain(HasUsesOutsideReductionChain) {
+ assert((!getScalarType()->isVectorTy() || !isPartialReduction()) &&
+ "Unexpected partial reduction for vector PHI. Did a wide vector "
+ "type pass legality checks?");
addOperand(&BackedgeValue);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
index 9ed27e3899eee..a7d2f69f6ab4e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
@@ -161,6 +161,13 @@ class VPLane {
return Lane;
}
+ /// Returns the offset of this lane from the start of the last
+ /// <N x ElTy> subvector.
+ unsigned getOffsetInLastSubvec() const {
+ assert(LaneKind == Kind::ScalableLast);
+ return Lane;
+ }
+
/// Returns an expression describing the lane index that can be used at
/// runtime.
Value *getAsRuntimeExpr(IRBuilderBase &Builder, const ElementCount &VF) const;
@@ -214,6 +221,10 @@ struct VPTransformState {
/// Get the generated Value for a given VPValue and given Part and Lane.
Value *get(const VPValue *Def, const VPLane &Lane);
+ /// Get the generated Value for \p Def, stretched by \p StretchFactor.
+ /// If the latter is 1, then this is equivalent to get(Def).
+ Value *getAndStretch(const VPValue *Def, ElementCount StretchFactor);
+
bool hasVectorValue(const VPValue *Def) {
return Data.VPV2Vector.contains(Def);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index badcd40daf46b..dbf188bb7d899 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -95,7 +95,8 @@ bool VPRecipeBase::mayWriteToMemory() const {
case VPWidenLoadSC:
case VPWidenPHISC:
case VPWidenPointerInductionSC:
- case VPWidenSC: {
+ case VPWidenSC:
+ case VPWidenShuffleSC: {
const Instruction *I =
dyn_cast_or_null<Instruction>(getVPSingleValue()->getUnderlyingValue());
(void)I;
@@ -147,7 +148,8 @@ bool VPRecipeBase::mayReadFromMemory() const {
case VPWidenIntOrFpInductionSC:
case VPWidenPHISC:
case VPWidenPointerInductionSC:
- case VPWidenSC: {
+ case VPWidenSC:
+ case VPWidenShuffleSC: {
const Instruction *I =
dyn_cast_or_null<Instruction>(getVPSingleValue()->getUnderlyingValue());
(void)I;
@@ -470,6 +472,8 @@ Type *llvm::computeScalarTypeForInstruction(unsigned Opcode,
return Type::getVoidTy(Ctx);
case Instruction::ICmp:
case Instruction::FCmp:
+ AssertOperandType(1, Op0Ty);
+ return CmpInst::makeCmpResultType(Op0Ty);
case VPInstruction::ActiveLaneMask:
AssertOperandType(1, Op0Ty);
return IntegerType::get(Ctx, 1);
@@ -500,12 +504,22 @@ Type *llvm::computeScalarTypeForInstruction(unsigned Opcode,
return StructTy->getTypeAtIndex(
cast<VPConstantInt>(Operands[1])->getZExtValue());
}
+ case VPInstruction::ExtractVectors: {
+ auto *BaseTy = cast<FixedVectorType>(Op0Ty);
+ unsigned Factor = cast<VPConstantInt>(Operands[2])->getZExtValue();
+ return VectorType::getOneNthElementsVectorType(BaseTy, Factor);
+ }
+ case VPInstruction::ConcatVectors: {
+ auto *BaseTy = cast<FixedVectorType>(Op0Ty);
+ return VectorType::getDoubleElementsVectorType(BaseTy);
+ }
case VPInstruction::FirstActiveLane:
case VPInstruction::LastActiveLane:
case VPInstruction::NumActiveLanes:
case VPInstruction::IncomingAliasMask:
case Instruction::Load:
case Instruction::Alloca:
+ case Instruction::ShuffleVector:
llvm_unreachable("type must be passed explicitly");
case Instruction::Call:
return getCalledFunction(Operands)->getReturnType();
@@ -534,9 +548,10 @@ Type *VPReplicateRecipe::computeScalarType(const Instruction *I,
ArrayRef<VPValue *> Operands) {
unsigned Opcode = I->getOpcode();
if (Instruction::isCast(Opcode) ||
- is_contained(ArrayRef<unsigned>({Instruction::ExtractValue,
- Instruction::Load, Instruction::Alloca}),
- Opcode))
+ is_contained(
+ ArrayRef<unsigned>({Instruction::ExtractValue, Instruction::Load,
+ Instruction::Alloca, Instruction::ShuffleVector}),
+ Opcode))
return I->getType();
return computeScalarTypeForInstruction(Opcode, Operands);
}
@@ -593,9 +608,11 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
case Instruction::ICmp:
case Instruction::FCmp:
case Instruction::ExtractElement:
+ case Instruction::ShuffleVector:
case Instruction::Store:
case VPInstruction::BranchOnCount:
case VPInstruction::BranchOnTwoConds:
+ case VPInstruction::ConcatVectors:
case VPInstruction::FirstOrderRecurrenceSplice:
case VPInstruction::LogicalAnd:
case VPInstruction::LogicalOr:
@@ -607,6 +624,7 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
case Instruction::InsertElement:
case Instruction::Select:
case VPInstruction::ActiveLaneMask:
+ case VPInstruction::ExtractVectors:
case VPInstruction::ReductionStartVector:
return 3;
case Instruction::Call:
@@ -718,9 +736,10 @@ Value *VPInstruction::generate(VPTransformState &State) {
}
case Instruction::Select: {
bool OnlyFirstLaneUsed = vputils::onlyFirstLaneUsed(this);
- Value *Cond =
- State.get(getOperand(0),
- OnlyFirstLaneUsed || vputils::isSingleScalar(getOperand(0)));
+ ElementCount InitialEC = getElementCount(getScalarType());
+ Value *Cond = OnlyFirstLaneUsed || vputils::isSingleScalar(getOperand(0))
+ ? State.get(getOperand(0), true)
+ : State.getAndStretch(getOperand(0), InitialEC);
Value *Op1 = State.get(getOperand(1), OnlyFirstLaneUsed);
Value *Op2 = State.get(getOperand(2), OnlyFirstLaneUsed);
return Builder.CreateSelectFMF(Cond, Op1, Op2, getFastMathFlags(), Name);
@@ -771,10 +790,11 @@ Value *VPInstruction::generate(VPTransformState &State) {
// v3 = vector(v1(3), v2(0, 1, 2))
auto *V1 = State.get(getOperand(0));
- if (!V1->getType()->isVectorTy())
+ if (State.VF.isScalar())
return V1;
Value *V2 = State.get(getOperand(1));
- return Builder.CreateVectorSpliceRight(V1, V2, 1, Name);
+ int InitialEC = getElementCount(getScalarType()).getFixedValue();
+ return Builder.CreateVectorSpliceRight(V1, V2, InitialEC, Name);
}
case VPInstruction::CalculateTripCountMinusVF: {
Value *ScalarTC = State.get(getOperand(0), VPLane(0));
@@ -856,11 +876,19 @@ Value *VPInstruction::generate(VPTransformState &State) {
Builder.setFastMathFlags(getFastMathFlags());
// If this start vector is scaled then it should produce a vector with fewer
// elements than the VF.
- ElementCount VF = State.VF.divideCoefficientBy(
- cast<VPConstantInt>(getOperand(2))->getZExtValue());
- auto *Iden = Builder.CreateVectorSplat(VF, State.get(getOperand(1), true));
- return Builder.CreateInsertElement(Iden, State.get(getOperand(0), true),
- Builder.getInt32(0));
+ unsigned ScaleFactor = cast<VPConstantInt>(getOperand(2))->getZExtValue();
+ ElementCount EC = State.VF.divideCoefficientBy(ScaleFactor);
+ auto *Iden = Builder.CreateVectorSplat(EC, State.get(getOperand(1), true));
+
+ // Now that we have our splat of neutral elements, insert the initial
+ // value(s) in it.
+ Value *InitialVal = State.get(getOperand(0), true);
+ assert(getElementCount(InitialVal->getType()).isFixed() &&
+ "Re-vectorising a scalable vector type.");
+ if (InitialVal->getType()->isVectorTy())
+ return Builder.CreateInsertVector(Iden->getType(), Iden, InitialVal,
+ Builder.getInt64(0));
+ return Builder.CreateInsertElement(Iden, InitialVal, Builder.getInt32(0));
}
case VPInstruction::ComputeReductionResult: {
RecurKind RK = getRecurKind();
@@ -907,7 +935,8 @@ Value *VPInstruction::generate(VPTransformState &State) {
// TODO: Support in-order reductions based on the recurrence descriptor.
// All ops in the reduction inherit fast-math-flags from the recurrence
// descriptor.
- ReducedPartRdx = createSimpleReduction(Builder, ReducedPartRdx, RK);
+ ReducedPartRdx =
+ createSimpleReduction(Builder, ReducedPartRdx, RK, getScalarType());
}
return ReducedPartRdx;
@@ -931,6 +960,34 @@ Value *VPInstruction::generate(VPTransformState &State) {
Res->setName(Name);
return Res;
}
+ case VPInstruction::ExtractVectors: {
+ assert(State.VF == ElementCount::getScalable(1) &&
+ "Can only use .segments intrinsics for VF=vscale x 1");
+ unsigned Factor = cast<VPConstantInt>(getOperand(2))->getZExtValue();
+ Intrinsic::ID IID = Intrinsic::getDeinterleaveIntrinsicID(
+ Factor, /*DeinterleaveSegments=*/true);
+ Value *InVector = State.get(getOperand(0));
+ Value *DeIntVectors =
+ State.Builder.CreateIntrinsic(IID, {InVector->getType()}, {InVector});
+
+ unsigned StartEltIdx = cast<VPConstantInt>(getOperand(1))->getZExtValue();
+ unsigned InitialSubVecSize =
+ cast<FixedVectorType>(getScalarType())->getNumElements();
+ assert(StartEltIdx % InitialSubVecSize == 0 &&
+ "Cannot do unaligned extracts");
+ return State.Builder.CreateExtractValue(DeIntVectors,
+ StartEltIdx / InitialSubVecSize);
+ }
+ case VPInstruction::ConcatVectors: {
+ assert(State.VF == ElementCount::getScalable(1) &&
+ "Can only use .segments intrinsics for VF=vscale x 1");
+ Intrinsic::ID IID = Intrinsic::getInterleaveIntrinsicID(
+ /*Factor=*/2, /*InterleaveSegments=*/true);
+ Type *RevecTy = toVectorTy(getScalarType(), State.VF);
+ Value *Concat = State.Builder.CreateIntrinsic(
+ IID, {RevecTy}, {State.get(getOperand(0)), State.get(getOperand(1))});
+ return Concat;
+ }
case VPInstruction::LogicalAnd: {
Value *A = State.get(getOperand(0));
Value *B = State.get(getOperand(1));
@@ -1240,7 +1297,7 @@ InstructionCost VPRecipeWithIRFlags::getCostForRecipeWithOpcode(
Type *CondTy = getOperand(0)->getScalarType();
if (!IsScalarCond && VF.isVector())
- CondTy = VectorType::get(CondTy, VF);
+ CondTy = toVectorTy(CondTy, VF);
llvm::CmpPredicate Pred;
if (!match(getOperand(0), m_Cmp(Pred, m_VPValue(), m_VPValue())))
@@ -1299,6 +1356,34 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
return Ctx.TTI.getVectorInstrCost(Instruction::ExtractElement, VecTy,
Ctx.CostKind);
}
+ case VPInstruction::ExtractVectors: {
+ // TODO-REVEC: deinterleave.segments intrinsics require VF=vscale x 1, but
+ // we could just bitcast the vector types if their fixed part is <= 64b.
+ if (VF != ElementCount::getScalable(1))
+ return InstructionCost::getInvalid();
+
+ unsigned Factor = cast<VPConstantInt>(getOperand(2))->getZExtValue();
+ Intrinsic::ID IID = Intrinsic::getDeinterleaveIntrinsicID(
+ Factor, /*DeinterleaveSegments=*/true);
+
+ Type *InVTy = toVectorTy(getOperand(0)->getScalarType(), VF);
+ Type *ExtVTy = toVectorTy(getScalarType(), VF);
+ IntrinsicCostAttributes Attrs(IID, ExtVTy, {InVTy});
+ return Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);
+ }
+ case VPInstruction::ConcatVectors: {
+ // interleave.segments intrinsics require VF=vscale x 1
+ if (VF != ElementCount::getScalable(1))
+ return InstructionCost::getInvalid();
+
+ Intrinsic::ID IID = Intrinsic::getInterleaveIntrinsicID(
+ /*Factor=*/2, /*InterleaveSegments=*/true);
+
+ Type *InVTy = toVectorTy(getOperand(0)->getScalarType(), VF);
+ Type *ExtVTy = toVectorTy(getScalarType(), VF);
+ IntrinsicCostAttributes Attrs(IID, ExtVTy, {InVTy});
+ return Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);
+ }
case VPInstruction::AnyOf: {
auto *VecTy = toVectorTy(this->getScalarType(), VF);
return Ctx.TTI.getArithmeticReductionCost(
@@ -1363,6 +1448,22 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
{ArgTy, ArgTy});
return Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);
}
+ case VPInstruction::ComputeReductionResult: {
+ Type *ResTy = getScalarType();
+ auto *SrcTy = toVectorTy(getOperand(0)->getScalarType(), VF);
+ // TODO-REVEC: We should probably query the cost for all reduction types,
+ // but in the case of vector partial reductions, it is mainly used
+ // to query the target whether it supports this operation.
+ if (ResTy->isVectorTy()) {
+ RecurKind RK = getRecurKind();
+ Intrinsic::ID IID = getReductionIntrinsicID(RK, ResTy);
+ assert(isPartialReductionIntrinsicID(IID) &&
+ "Reduction to a vector requires a partial.reduce intrinsic.");
+ IntrinsicCostAttributes ICA(IID, ResTy, {ResTy, SrcTy});
+ return Ctx.TTI.getIntrinsicInstrCost(ICA, Ctx.CostKind);
+ }
+ return 0;
+ }
case VPInstruction::ExplicitVectorLength: {
Type *Arg0Ty = getOperand(0)->getScalarType();
Type *I32Ty = Type::getInt32Ty(Ctx.LLVMCtx);
@@ -1438,6 +1539,8 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
}
}
+// Whether the instruction turns a VFxInitialTy type back into InitialTy.
+// Note InitialTy might already be a vector if re-vectorising.
bool VPInstruction::isVectorToScalar() const {
return getOpcode() == VPInstruction::ExtractLastLane ||
getOpcode() == VPInstruction::ExtractPenultimateElement ||
@@ -1480,10 +1583,14 @@ void VPInstruction::execute(VPTransformState &State) {
bool GeneratesPerFirstLaneOnly = canGenerateScalarForFirstLane() &&
(vputils::onlyFirstLaneUsed(this) ||
isVectorToScalar() || isSingleScalar());
- assert((((GeneratedValue->getType()->isVectorTy() ||
- GeneratedValue->getType()->isStructTy()) ==
- !GeneratesPerFirstLaneOnly) ||
- State.VF.isScalar()) &&
+ // Note: When re-vectorising, our initial values might be of vector type.
+ Type *InitialTy = getScalarType();
+ ElementCount NewEC = getElementCount(GeneratedValue->getType());
+ ElementCount InitialEC = getElementCount(InitialTy);
+ bool GotWidened = ElementCount::isKnownGT(NewEC, InitialEC) ||
+ (NewEC.isScalable() && !InitialEC.isScalable()) ||
+ GeneratedValue->getType()->isStructTy();
+ assert(((GotWidened == !GeneratesPerFirstLaneOnly) || State.VF.isScalar()) &&
"scalar value but not only first lane defined");
State.set(this, GeneratedValue,
/*IsScalar*/ GeneratesPerFirstLaneOnly);
@@ -1519,9 +1626,11 @@ bool VPInstruction::opcodeMayReadOrWriteFromMemory() const {
case VPInstruction::BuildVector:
case VPInstruction::CalculateTripCountMinusVF:
case VPInstruction::CanonicalIVIncrementForPart:
+ case VPInstruction::ConcatVectors:
case VPInstruction::ExtractLane:
case VPInstruction::ExtractLastLane:
case VPInstruction::ExtractLastPart:
+ case VPInstruction::ExtractVectors:
case VPInstruction::ExtractPenultimateElement:
case VPInstruction::ActiveLaneMask:
case VPInstruction::IncomingAliasMask:
@@ -1679,6 +1788,9 @@ void VPInstruction::printRecipe(raw_ostream &O, const Twine &Indent,
case VPInstruction::BuildVector:
O << "buildvector";
break;
+ case VPInstruction::ConcatVectors:
+ O << "concat-vectors";
+ break;
case VPInstruction::ExitingIVValue:
O << "exiting-iv-value";
break;
@@ -1697,6 +1809,9 @@ void VPInstruction::printRecipe(raw_ostream &O, const Twine &Indent,
case VPInstruction::ExtractPenultimateElement:
O << "extract-penultimate-element";
break;
+ case VPInstruction::ExtractVectors:
+ O << "extract-vectors";
+ break;
case VPInstruction::ComputeReductionResult:
O << "compute-reduction-result";
break;
@@ -2072,7 +2187,7 @@ void VPWidenCallRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
}
#endif
-CallInst *VPWidenIntrinsicRecipe::createVectorCall(VPTransformState &State) {
+Instruction *VPWidenIntrinsicRecipe::createVectorCall(VPTransformState &State) {
assert(State.VF.isVector() && "not widening");
SmallVector<Type *, 2> TysForDecl;
@@ -2104,27 +2219,36 @@ CallInst *VPWidenIntrinsicRecipe::createVectorCall(VPTransformState &State) {
}
// Use vector version of the intrinsic.
- Module *M = State.Builder.GetInsertBlock()->getModule();
- Function *VectorF =
- Intrinsic::getOrInsertDeclaration(M, VectorIntrinsicID, TysForDecl);
- assert(VectorF &&
- "Can't retrieve vector intrinsic or vector-predication intrinsics.");
+ Instruction *V = nullptr;
+ if (Intrinsic::isTargetIntrinsic(VectorIntrinsicID)) {
+ V = State.TTI->vectorizeTargetIntrinsic(VectorIntrinsicID, TysForDecl, Args,
+ State.Builder,
+ *getUnderlyingInstr());
+ assert(V && "Target did not vectorise intrinsic call.");
+ // TODO-REVEC: Properly retain metadata and flags somehow.
+ } else {
+ Module *M = State.Builder.GetInsertBlock()->getModule();
+ Function *VectorF =
+ Intrinsic::getOrInsertDeclaration(M, VectorIntrinsicID, TysForDecl);
+ assert(VectorF &&
+ "Can't retrieve vector intrinsic or vector-predication intrinsics.");
- auto *CI = cast_or_null<CallInst>(getUnderlyingValue());
- SmallVector<OperandBundleDef, 1> OpBundles;
- if (CI)
- CI->getOperandBundlesAsDefs(OpBundles);
+ auto *CI = cast_or_null<CallInst>(getUnderlyingValue());
+ SmallVector<OperandBundleDef, 1> OpBundles;
+ if (CI)
+ CI->getOperandBundlesAsDefs(OpBundles);
- CallInst *V = State.Builder.CreateCall(VectorF, Args, OpBundles);
+ V = State.Builder.CreateCall(VectorF, Args, OpBundles);
+ applyFlags(*V);
+ }
- applyFlags(*V);
applyMetadata(*V);
return V;
}
void VPWidenIntrinsicRecipe::execute(VPTransformState &State) {
- CallInst *V = createVectorCall(State);
+ Instruction *V = createVectorCall(State);
if (!V->getType()->isVoidTy())
State.set(this, V);
}
@@ -2214,7 +2338,7 @@ void VPWidenIntrinsicRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
#endif
void VPWidenMemIntrinsicRecipe::execute(VPTransformState &State) {
- CallInst *MemI = createVectorCall(State);
+ auto *MemI = cast<CallInst>(createVectorCall(State));
MemI->addParamAttr(
0, Attribute::getWithAlignment(MemI->getContext(), Alignment));
State.set(this, MemI);
@@ -2299,6 +2423,19 @@ InstructionCost VPHistogramRecipe::computeCost(ElementCount VF,
Ctx.TTI.getArithmeticInstrCost(Opcode, VTy, Ctx.CostKind);
}
+void VPWidenShuffleRecipe::execute(VPTransformState &State) {
+ Value *WideShuffle = State.Builder.CreateSegmentedShuffleVector(
+ State.get(getOperand(0)), State.get(getOperand(1)), getShuffleMask());
+ State.set(this, WideShuffle);
+}
+
+InstructionCost VPWidenShuffleRecipe::computeCost(ElementCount VF,
+ VPCostContext &Ctx) const {
+ auto *DstTy = cast<VectorType>(toVectorTy(getScalarType(), VF));
+ return Ctx.TTI.getSegmentedShuffleCost(TargetTransformInfo::SK_PermuteTwoSrc,
+ DstTy, getShuffleMask(), Ctx.CostKind);
+}
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
void VPHistogramRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const {
@@ -2318,6 +2455,28 @@ void VPHistogramRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
Mask->printAsOperand(O, SlotTracker);
}
}
+
+void VPWidenShuffleRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const {
+ O << Indent << "WIDEN-SHUFFLE ";
+ printAsOperand(O, SlotTracker);
+ O << " = shuffle<";
+
+ bool FirstElt = true;
+ for (int Elt : getShuffleMask()) {
+ if (FirstElt)
+ FirstElt = false;
+ else
+ O << ", ";
+ O << Elt;
+ }
+ O << ">";
+
+ getOperand(0)->printAsOperand(O, SlotTracker);
+ O << ", ";
+ getOperand(1)->printAsOperand(O, SlotTracker);
+}
+
#endif
VPIRFlags::FastMathFlagsTy::FastMathFlagsTy(const FastMathFlags &FMF) {
@@ -2515,6 +2674,9 @@ static void printRecurrenceKind(raw_ostream &OS, const RecurKind &Kind) {
case RecurKind::FindLast:
OS << "find-last";
break;
+ case RecurKind::TargetIntAccumulation:
+ OS << "target-int-accumulation";
+ break;
}
}
@@ -2661,7 +2823,11 @@ void VPWidenRecipe::execute(VPTransformState &State) {
}
case Instruction::Select: {
VPValue *CondOp = getOperand(0);
- Value *Cond = State.get(CondOp, vputils::isSingleScalar(CondOp));
+ // Select supports CondTy which is either i1 or an i1 vector of matching
+ // size, so be careful if the original type is an i1 vector.
+ Value *Cond =
+ State.get(CondOp, vputils::isSingleScalar(CondOp) &&
+ CondOp->getScalarType()->isIntegerTy(1));
Value *Op0 = State.get(getOperand(1));
Value *Op1 = State.get(getOperand(2));
Value *Sel = State.Builder.CreateSelect(Cond, Op0, Op1);
@@ -2683,7 +2849,7 @@ void VPWidenRecipe::execute(VPTransformState &State) {
#if !defined(NDEBUG)
// Verify that VPlan type inference results agree with the type of the
// generated values.
- assert(VectorType::get(this->getScalarType(), State.VF) ==
+ assert(toVectorTy(this->getScalarType(), State.VF) ==
State.get(this)->getType() &&
"inferred type and type from generated instructions do not match");
#endif
@@ -2741,7 +2907,7 @@ void VPWidenCastRecipe::execute(VPTransformState &State) {
auto &Builder = State.Builder;
/// Vectorize casts.
assert(State.VF.isVector() && "Not vectorizing?");
- Type *DestTy = VectorType::get(getScalarType(), State.VF);
+ Type *DestTy = toVectorTy(getScalarType(), State.VF);
VPValue *Op = getOperand(0);
Value *A = State.get(Op);
Value *Cast = Builder.CreateCast(Instruction::CastOps(Opcode), A, DestTy);
@@ -3197,7 +3363,8 @@ void VPReductionRecipe::execute(VPTransformState &State) {
assert(isInLoop() &&
"The reduction must either be ordered, partial or in-loop");
Value *PrevInChain = State.get(getChainOp(), /*IsScalar*/ true);
- NewRed = createSimpleReduction(State.Builder, NewVecOp, Kind);
+ NewRed = createSimpleReduction(State.Builder, NewVecOp, Kind,
+ PrevInChain->getType());
if (RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind))
NextInChain = createMinMaxOp(State.Builder, Kind, NewRed, PrevInChain);
else
@@ -3862,6 +4029,12 @@ InstructionCost VPReplicateRecipe::computeCost(ElementCount VF,
return Ctx.TTI.getInsertExtractValueCost(getOpcode(), Ctx.CostKind);
}
+ // TODO-REVEC: We might have allowed unsupported instructions like
+ // shufflevector to be replicated. This isn't always supported, so be
+ // conservative.
+ if (UI->getType()->isVectorTy() && !VF.isScalar())
+ return InstructionCost::getInvalid();
+
return Ctx.getLegacyCost(UI, VF);
}
@@ -3972,6 +4145,12 @@ InstructionCost VPWidenMemoryRecipe::computeCost(ElementCount VF,
unsigned Opcode = IsLoad ? Instruction::Load : Instruction::Store;
if (!Consecutive) {
+ // REVEC: Can only handle ValTy segments due to how segment.gather and
+ // segment.scatter intrinsics work, so expect VF=vscalex1
+ const bool Revec = ScalarTy->isVectorTy();
+ if (Revec && VF.getKnownMinValue() != 1)
+ return InstructionCost::getInvalid();
+
// TODO: Using the original IR may not be accurate.
// Currently, ARM will use the underlying IR to calculate gather/scatter
// instruction cost.
@@ -3995,8 +4174,12 @@ InstructionCost VPWidenMemoryRecipe::computeCost(ElementCount VF,
if (!vputils::isSingleScalar(getAddr()))
PtrTy = toVectorTy(PtrTy, VF);
- unsigned IID = isa<VPWidenLoadRecipe>(R) ? Intrinsic::masked_gather
- : isa<VPWidenStoreRecipe>(R) ? Intrinsic::masked_scatter
+ unsigned IID = isa<VPWidenLoadRecipe>(R)
+ ? (Revec ? Intrinsic::masked_segment_gather
+ : Intrinsic::masked_gather)
+ : isa<VPWidenStoreRecipe>(R)
+ ? (Revec ? Intrinsic::masked_segment_scatter
+ : Intrinsic::masked_scatter)
: isa<VPWidenLoadEVLRecipe>(R) ? Intrinsic::vp_gather
: Intrinsic::vp_scatter;
return Ctx.TTI.getAddressComputationCost(PtrTy, nullptr, nullptr,
@@ -4025,19 +4208,23 @@ InstructionCost VPWidenMemoryRecipe::computeCost(ElementCount VF,
void VPWidenLoadRecipe::execute(VPTransformState &State) {
Type *ScalarDataTy = getScalarType();
- auto *DataTy = VectorType::get(ScalarDataTy, State.VF);
+ auto *DataTy = toVectorTy(ScalarDataTy, State.VF);
bool CreateGather = !isConsecutive();
auto &Builder = State.Builder;
Value *Mask = nullptr;
- if (auto *VPMask = getMask())
- Mask = State.get(VPMask);
+ if (auto *VPMask = getMask()) {
+ ElementCount ScretchFactor = CreateGather ? ElementCount::getFixed(1)
+ : getElementCount(ScalarDataTy);
+ Mask = State.getAndStretch(VPMask, ScretchFactor);
+ }
Value *Addr = State.get(getAddr(), /*IsScalar*/ !CreateGather);
Value *NewLI;
if (CreateGather) {
- NewLI = Builder.CreateMaskedGather(DataTy, Addr, Alignment, Mask, nullptr,
- "wide.masked.gather");
+ NewLI = Builder.CreateMaskedGather(
+ DataTy, Addr, Alignment, Mask, nullptr, "wide.masked.gather",
+ /*GatherSegments=*/ScalarDataTy->isVectorTy());
} else if (Mask) {
NewLI =
Builder.CreateMaskedLoad(DataTy, Addr, Alignment, Mask,
@@ -4120,18 +4307,24 @@ void VPWidenLoadEVLRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
void VPWidenStoreRecipe::execute(VPTransformState &State) {
VPValue *StoredVPValue = getStoredValue();
bool CreateScatter = !isConsecutive();
+ Type *InitialDataTy = getOperand(1)->getScalarType();
auto &Builder = State.Builder;
Value *Mask = nullptr;
- if (auto *VPMask = getMask())
- Mask = State.get(VPMask);
+ if (auto *VPMask = getMask()) {
+ ElementCount ScretchFactor = CreateScatter ? ElementCount::getFixed(1)
+ : getElementCount(InitialDataTy);
+ Mask = State.getAndStretch(VPMask, ScretchFactor);
+ }
Value *StoredVal = State.get(StoredVPValue);
Value *Addr = State.get(getAddr(), /*IsScalar*/ !CreateScatter);
Instruction *NewSI = nullptr;
if (CreateScatter)
- NewSI = Builder.CreateMaskedScatter(StoredVal, Addr, Alignment, Mask);
+ NewSI = Builder.CreateMaskedScatter(
+ StoredVal, Addr, Alignment, Mask,
+ /*ScatterSegments=*/InitialDataTy->isVectorTy());
else if (Mask)
NewSI = Builder.CreateMaskedStore(StoredVal, Addr, Alignment, Mask);
else
@@ -4204,18 +4397,22 @@ void VPWidenStoreEVLRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
#endif
static Value *createBitOrPointerCast(IRBuilderBase &Builder, Value *V,
- VectorType *DstVTy, const DataLayout &DL) {
+ VectorType *DstVTy, const DataLayout &DL,
+ bool AllowDifferentElementCount = false) {
// Verify that V is a vector type with same number of elements as DstVTy.
auto VF = DstVTy->getElementCount();
auto *SrcVecTy = cast<VectorType>(V->getType());
- assert(VF == SrcVecTy->getElementCount() && "Vector dimensions do not match");
+ assert((VF == SrcVecTy->getElementCount() || AllowDifferentElementCount) &&
+ "Vector dimensions do not match");
Type *SrcElemTy = SrcVecTy->getElementType();
Type *DstElemTy = DstVTy->getElementType();
- assert((DL.getTypeSizeInBits(SrcElemTy) == DL.getTypeSizeInBits(DstElemTy)) &&
+ assert((DL.getTypeSizeInBits(SrcElemTy) == DL.getTypeSizeInBits(DstElemTy) ||
+ AllowDifferentElementCount) &&
"Vector elements must have same size");
// Do a direct cast if element types are castable.
- if (CastInst::isBitOrNoopPointerCastable(SrcElemTy, DstElemTy, DL)) {
+ if (CastInst::isBitOrNoopPointerCastable(SrcElemTy, DstElemTy, DL) ||
+ VF != SrcVecTy->getElementCount()) {
return Builder.CreateBitOrPointerCast(V, DstVTy);
}
// V cannot be directly casted to desired vector type.
@@ -4236,7 +4433,7 @@ static Value *createBitOrPointerCast(IRBuilderBase &Builder, Value *V,
/// Return a vector containing interleaved elements from multiple
/// smaller input vectors.
static Value *interleaveVectors(IRBuilderBase &Builder, ArrayRef<Value *> Vals,
- const Twine &Name) {
+ const Twine &Name, bool InterleaveSegments) {
unsigned Factor = Vals.size();
assert(Factor > 1 && "Tried to interleave invalid number of vectors");
@@ -4250,9 +4447,12 @@ static Value *interleaveVectors(IRBuilderBase &Builder, ArrayRef<Value *> Vals,
// must use intrinsics to interleave.
if (VecTy->isScalableTy()) {
assert(Factor <= 8 && "Unsupported interleave factor for scalable vectors");
- return Builder.CreateVectorInterleave(Vals, Name);
+ return Builder.CreateVectorInterleave(Vals, Name, InterleaveSegments);
}
+ assert(!InterleaveSegments &&
+ "REVEC: Unexpected fixed VF for vector interleaving.");
+
// Fixed length. Start by concatenating all vectors into a wide vector.
Value *WideVec = concatenateVectors(Builder, Vals);
@@ -4299,23 +4499,41 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
// Prepare for the vector type of the interleaved load/store.
Type *ScalarTy = getLoadStoreType(Instr);
unsigned InterleaveFactor = Group->getFactor();
- auto *VecTy = VectorType::get(ScalarTy, State.VF * InterleaveFactor);
+ auto *VecTy =
+ cast<VectorType>(toVectorTy(ScalarTy, State.VF * InterleaveFactor));
+
+ assert((!ScalarTy->isVectorTy() || !Group->isReverse()) &&
+ "REVEC: Unexpected reverse interleave group.");
VPValue *BlockInMask = getMask();
VPValue *Addr = getAddr();
Value *ResAddr = State.get(Addr, VPLane(0));
- auto CreateGroupMask = [&BlockInMask, &State,
- &InterleaveFactor](Value *MaskForGaps) -> Value * {
+ auto CreateGroupMask = [&BlockInMask, &State, &InterleaveFactor,
+ ScalarTy](Value *MaskForGaps) -> Value * {
if (State.VF.isScalable()) {
assert(!MaskForGaps && "Interleaved groups with gaps are not supported.");
assert(InterleaveFactor <= 8 &&
"Unsupported deinterleave factor for scalable vectors");
+
+ // If the original type is a vector, then the mask needs to be stretched
+ // by the number of elements and by the interleave factor. We can use
+ // vector.stretch all the way instead of a combination of vector.stretch
+ // and vector.interleave.
+ if (ScalarTy->isVectorTy()) {
+ ElementCount StretchFactor =
+ cast<VectorType>(ScalarTy)->getElementCount() * InterleaveFactor;
+ return State.getAndStretch(BlockInMask, StretchFactor);
+ }
+
auto *ResBlockInMask = State.get(BlockInMask);
SmallVector<Value *> Ops(InterleaveFactor, ResBlockInMask);
- return interleaveVectors(State.Builder, Ops, "interleaved.mask");
+ return interleaveVectors(State.Builder, Ops, "interleaved.mask", false);
}
+ assert(!ScalarTy->isVectorTy() &&
+ "REVEC: Unexpected fixed VF for vector interleaving.");
+
if (!BlockInMask)
return MaskForGaps;
@@ -4360,7 +4578,8 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
assert(InterleaveFactor <= 8 &&
"Unsupported deinterleave factor for scalable vectors");
NewLoad = State.Builder.CreateIntrinsic(
- Intrinsic::getDeinterleaveIntrinsicID(InterleaveFactor),
+ Intrinsic::getDeinterleaveIntrinsicID(InterleaveFactor,
+ ScalarTy->isVectorTy()),
NewLoad->getType(), NewLoad,
/*FMFSource=*/nullptr, "strided.vec");
}
@@ -4390,9 +4609,11 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
// If this member has different type, cast the result type.
if (Member->getType() != ScalarTy) {
- VectorType *OtherVTy = VectorType::get(Member->getType(), State.VF);
- StridedVec =
- createBitOrPointerCast(State.Builder, StridedVec, OtherVTy, DL);
+ VectorType *OtherVTy =
+ cast<VectorType>(toVectorTy(Member->getType(), State.VF));
+ StridedVec = createBitOrPointerCast(
+ State.Builder, StridedVec, OtherVTy, DL,
+ /*AllowDifferentElementCount=*/ScalarTy->isVectorTy());
}
if (Group->isReverse())
@@ -4405,7 +4626,8 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
}
// The sub vector type for current instruction.
- auto *SubVT = VectorType::get(ScalarTy, State.VF);
+ auto *SubVT =
+ VectorType::getOneNthElementsVectorType(VecTy, InterleaveFactor);
// Vectorize the interleaved store group.
Value *MaskForGaps =
@@ -4437,15 +4659,19 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
// If this member has different type, cast it to a unified type.
if (StoredVec->getType() != SubVT)
- StoredVec = createBitOrPointerCast(State.Builder, StoredVec, SubVT, DL);
+ StoredVec = createBitOrPointerCast(
+ State.Builder, StoredVec, SubVT, DL,
+ /*AllowDifferentElementCount=*/ScalarTy->isVectorTy());
StoredVecs.push_back(StoredVec);
}
// Interleave all the smaller vectors into one wider vector.
- Value *IVec = interleaveVectors(State.Builder, StoredVecs, "interleaved.vec");
+ Value *IVec = interleaveVectors(State.Builder, StoredVecs, "interleaved.vec",
+ ScalarTy->isVectorTy());
Instruction *NewStoreInstr;
if (BlockInMask || MaskForGaps) {
+ assert(!ScalarTy->isVectorTy() && "REVEC: unexpected mask for segment.");
Value *GroupMask = CreateGroupMask(MaskForGaps);
NewStoreInstr = State.Builder.CreateMaskedStore(
IVec, ResAddr, Group->getAlign(), GroupMask);
@@ -4515,7 +4741,8 @@ void VPInterleaveEVLRecipe::execute(VPTransformState &State) {
Value *GroupMask = nullptr;
if (VPValue *BlockInMask = getMask()) {
SmallVector<Value *> Ops(InterleaveFactor, State.get(BlockInMask));
- GroupMask = interleaveVectors(State.Builder, Ops, "interleaved.mask");
+ GroupMask =
+ interleaveVectors(State.Builder, Ops, "interleaved.mask", false);
} else {
GroupMask =
State.Builder.CreateVectorSplat(WideVF, State.Builder.getTrue());
@@ -4586,7 +4813,8 @@ void VPInterleaveEVLRecipe::execute(VPTransformState &State) {
}
// Interleave all the smaller vectors into one wider vector.
- Value *IVec = interleaveVectors(State.Builder, StoredVecs, "interleaved.vec");
+ Value *IVec =
+ interleaveVectors(State.Builder, StoredVecs, "interleaved.vec", false);
CallInst *NewStore =
State.Builder.CreateIntrinsic(Type::getVoidTy(Ctx), Intrinsic::vp_store,
{IVec, ResAddr, GroupMask, InterleaveEVL});
@@ -4648,8 +4876,17 @@ InstructionCost VPInterleaveBase::computeCost(ElementCount VF,
unsigned AS =
cast<PointerType>(getAddr()->getScalarType())->getAddressSpace();
+ // REVEC: Can only handle ValTy segments due to how (de)interleave.segments
+ // intrinsics work, so expect VF=vscalex1
+ if (ValTy->isVectorTy() && VF.getKnownMinValue() != 1)
+ return InstructionCost::getInvalid();
+
+ // TODO-REVEC: Support reversing segments
+ if (ValTy->isVectorTy() && IG->isReverse())
+ return InstructionCost::getInvalid();
+
unsigned InterleaveFactor = IG->getFactor();
- auto *WideVecTy = VectorType::get(ValTy, VF * InterleaveFactor);
+ auto *WideVecTy = toVectorTy(ValTy, VF * InterleaveFactor);
// Holds the indices of existing members in the interleaved group.
SmallVector<unsigned, 4> Indices;
@@ -4658,13 +4895,19 @@ InstructionCost VPInterleaveBase::computeCost(ElementCount VF,
Indices.push_back(IF);
// Calculate the cost of the whole interleaved group.
- InstructionCost Cost = Ctx.TTI.getInterleavedMemoryOpCost(
- InsertPos->getOpcode(), WideVecTy, IG->getFactor(), Indices,
- IG->getAlign(), AS, Ctx.CostKind, getMask(), NeedsMaskForGaps);
+ InstructionCost Cost =
+ ValTy->isVectorTy()
+ ? Ctx.TTI.getSegmentInterleavedMemoryOpCost(
+ InsertPos->getOpcode(), WideVecTy, IG->getFactor(), Indices,
+ IG->getAlign(), AS, Ctx.CostKind, getMask(), NeedsMaskForGaps)
+ : Ctx.TTI.getInterleavedMemoryOpCost(
+ InsertPos->getOpcode(), WideVecTy, IG->getFactor(), Indices,
+ IG->getAlign(), AS, Ctx.CostKind, getMask(), NeedsMaskForGaps);
if (!IG->isReverse())
return Cost;
+ assert(!ValTy->isVectorTy() && "Cannot reverse segments yet.");
return Cost + IG->getNumMembers() *
Ctx.TTI.getShuffleCost(TargetTransformInfo::SK_Reverse,
VectorTy, VectorTy, {}, Ctx.CostKind,
@@ -4723,7 +4966,7 @@ void VPFirstOrderRecurrencePHIRecipe::execute(VPTransformState &State) {
Type *VecTy = State.VF.isScalar()
? VectorInit->getType()
- : VectorType::get(VectorInit->getType(), State.VF);
+ : toVectorTy(VectorInit->getType(), State.VF);
BasicBlock *VectorPH =
State.CFG.VPBB2IRBB.at(getParent()->getCFGPredecessor(0));
@@ -4732,10 +4975,16 @@ void VPFirstOrderRecurrencePHIRecipe::execute(VPTransformState &State) {
auto *One = ConstantInt::get(IdxTy, 1);
IRBuilder<>::InsertPointGuard Guard(Builder);
Builder.SetInsertPoint(VectorPH->getTerminator());
- auto *RuntimeVF = getRuntimeVF(Builder, IdxTy, State.VF);
- auto *LastIdx = Builder.CreateSub(RuntimeVF, One);
- VectorInit = Builder.CreateInsertElement(
- PoisonValue::get(VecTy), VectorInit, LastIdx, "vector.recur.init");
+ if (VectorInit->getType()->isVectorTy()) {
+ // InsertVector does not support a runtime index, so let's just broadcast
+ // the initial value everywhere. Only the last subvector will be used.
+ VectorInit = State.get(getStartValue());
+ } else {
+ auto *RuntimeVF = getRuntimeVF(Builder, IdxTy, State.VF);
+ auto *LastIdx = Builder.CreateSub(RuntimeVF, One);
+ VectorInit = Builder.CreateInsertElement(
+ PoisonValue::get(VecTy), VectorInit, LastIdx, "vector.recur.init");
+ }
}
// Create a phi node for the new recurrence.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 5df097628ba7f..300c3caa55e65 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -49,7 +49,7 @@ using namespace VPlanPatternMatch;
using namespace SCEVPatternMatch;
bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
- VPlan &Plan, const TargetLibraryInfo &TLI) {
+ VPlan &Plan, const TargetLibraryInfo &TLI, const TargetTransformInfo &TTI) {
ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> RPOT(
Plan.getVectorLoopRegion());
@@ -91,7 +91,7 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
Ingredient.operands(), *VPI,
Ingredient.getDebugLoc(), GEP);
} else if (CallInst *CI = dyn_cast<CallInst>(Inst)) {
- Intrinsic::ID VectorID = getVectorIntrinsicIDForCall(CI, &TLI);
+ Intrinsic::ID VectorID = getVectorIntrinsicIDForCall(CI, &TLI, &TTI);
if (VectorID == Intrinsic::not_intrinsic)
return false;
@@ -6705,6 +6705,9 @@ getScaledReductions(VPReductionPHIRecipe *RedPhiR, VPCostContext &CostCtx,
SmallVector<VPPartialReductionChain> Chain;
RecurKind RK = RedPhiR->getRecurrenceKind();
Type *PhiType = RedPhiR->getScalarType();
+ // REVEC: Avoid partial reductions
+ if (PhiType->isVectorTy())
+ return std::nullopt;
TypeSize PHISize = PhiType->getPrimitiveSizeInBits();
// Work backwards from the ExitValue examining each reduction operation.
@@ -7043,7 +7046,8 @@ static CallWideningDecision decideCallWidening(VPInstruction &VPI,
auto *CalledFn = cast<Function>(
VPI.getOperand(VPI.getNumOperandsWithoutMask() - 1)->getLiveInIRValue());
Type *ResultTy = VPI.getScalarType();
- Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, &CostCtx.TLI);
+ Intrinsic::ID ID =
+ getVectorIntrinsicIDForCall(CI, &CostCtx.TLI, &CostCtx.TTI);
bool MaskRequired = CostCtx.isMaskRequired(CI);
// Pseudo intrinsics (assume, lifetime, ...) are always scalarized.
@@ -7103,7 +7107,8 @@ void VPlanTransforms::makeCallWideningDecisions(VPlan &Plan, VFRange &Range,
VPSingleDefRecipe *Replacement = nullptr;
switch (Decision.Kind) {
case CallWideningDecision::KindTy::Intrinsic: {
- Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, &CostCtx.TLI);
+ Intrinsic::ID ID =
+ getVectorIntrinsicIDForCall(CI, &CostCtx.TLI, &CostCtx.TTI);
Type *ResultTy = VPI->getScalarType();
Replacement = new VPWidenIntrinsicRecipe(*CI, ID, Ops, ResultTy, *VPI,
*VPI, VPI->getDebugLoc());
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 961931798b9c8..e07e56c471d86 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -216,7 +216,8 @@ struct VPlanTransforms {
/// to a wide recipe if needed.
LLVM_ABI_FOR_TEST static bool
tryToConvertVPInstructionsToVPRecipes(VPlan &Plan,
- const TargetLibraryInfo &TLI);
+ const TargetLibraryInfo &TLI,
+ const TargetTransformInfo &TTI);
/// Try to legalize reductions with multiple in-loop uses. Currently only
/// strict and non-strict min/max reductions used by FindLastIV reductions are
diff --git a/llvm/test/Analysis/CostModel/AArch64/masked_ldst.ll b/llvm/test/Analysis/CostModel/AArch64/masked_ldst.ll
index c1ee229ae8e2c..2718a6204aa68 100644
--- a/llvm/test/Analysis/CostModel/AArch64/masked_ldst.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/masked_ldst.ll
@@ -72,8 +72,9 @@ define void @scalable() {
; CHECK-NEXT: Cost Model: Found costs of 1 for: %nxv2f32 = call <vscale x 2 x float> @llvm.masked.load.nxv2f32.p0(ptr align 8 undef, <vscale x 2 x i1> undef, <vscale x 2 x float> undef)
; CHECK-NEXT: Cost Model: Found costs of 1 for: %nxv4f32 = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr align 8 undef, <vscale x 4 x i1> undef, <vscale x 4 x float> undef)
; CHECK-NEXT: Cost Model: Found costs of 1 for: %nxv2f64 = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 undef, <vscale x 2 x i1> undef, <vscale x 2 x double> undef)
-; CHECK-NEXT: Cost Model: Found costs of Invalid for: %nxv1i64 = call <vscale x 1 x i64> @llvm.masked.load.nxv1i64.p0(ptr align 8 undef, <vscale x 1 x i1> undef, <vscale x 1 x i64> undef)
+; CHECK-NEXT: Cost Model: Found costs of 2 for: %nxv1i64 = call <vscale x 1 x i64> @llvm.masked.load.nxv1i64.p0(ptr align 8 undef, <vscale x 1 x i1> undef, <vscale x 1 x i64> undef)
; CHECK-NEXT: Cost Model: Found costs of 4 for: %nxv4i64 = call <vscale x 4 x i64> @llvm.masked.load.nxv4i64.p0(ptr align 8 undef, <vscale x 4 x i1> undef, <vscale x 4 x i64> undef)
+; CHECK-NEXT: Cost Model: Found costs of 2 for: %nxv1i32 = call <vscale x 1 x i32> @llvm.masked.load.nxv1i32.p0(ptr align 8 undef, <vscale x 1 x i1> undef, <vscale x 1 x i32> undef)
; CHECK-NEXT: Cost Model: Found costs of 8 for: %nxv32f16 = call <vscale x 32 x half> @llvm.masked.load.nxv32f16.p0(ptr align 8 undef, <vscale x 32 x i1> undef, <vscale x 32 x half> undef)
; CHECK-NEXT: Cost Model: Found costs of Invalid for: %nxv4i1 = call <vscale x 4 x i1> @llvm.masked.load.nxv4i1.p0(ptr align 16 undef, <vscale x 4 x i1> undef, <vscale x 4 x i1> undef)
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
@@ -102,6 +103,7 @@ entry:
; A couple of examples of illegal scalable types
%nxv1i64 = call <vscale x 1 x i64> @llvm.masked.load.nxv1i64.p0(ptr undef, i32 8, <vscale x 1 x i1> undef, <vscale x 1 x i64> undef)
%nxv4i64 = call <vscale x 4 x i64> @llvm.masked.load.nxv4i64.p0(ptr undef, i32 8, <vscale x 4 x i1> undef, <vscale x 4 x i64> undef)
+ %nxv1i32 = call <vscale x 1 x i32> @llvm.masked.load.nxv1i32.p0(ptr undef, i32 8, <vscale x 1 x i1> undef, <vscale x 1 x i32> undef)
%nxv32f16 = call <vscale x 32 x half> @llvm.masked.load.nxv32f16.p0(ptr undef, i32 8, <vscale x 32 x i1> undef, <vscale x 32 x half> undef)
; Types that are legal, but for which we have no masked load/store lowering
diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-arith.ll b/llvm/test/Analysis/CostModel/AArch64/sve-arith.ll
index 75af1df08594f..48d10366a822d 100644
--- a/llvm/test/Analysis/CostModel/AArch64/sve-arith.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/sve-arith.ll
@@ -43,7 +43,7 @@ define void @scalable_mul() #0 {
; CHECK-NEXT: Cost Model: Found costs of 1 for: %mul_nxv8i16 = mul <vscale x 8 x i16> undef, undef
; CHECK-NEXT: Cost Model: Found costs of 1 for: %mul_nxv4i32 = mul <vscale x 4 x i32> undef, undef
; CHECK-NEXT: Cost Model: Found costs of 1 for: %mul_nxv2i64 = mul <vscale x 2 x i64> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of Invalid for: %mul_nxv1i64 = mul <vscale x 1 x i64> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of 1 for: %mul_nxv1i64 = mul <vscale x 1 x i64> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
entry:
@@ -62,7 +62,7 @@ define void @scalable_add() #0 {
; CHECK-NEXT: Cost Model: Found costs of 1 for: %add_nxv8i16 = add <vscale x 8 x i16> undef, undef
; CHECK-NEXT: Cost Model: Found costs of 1 for: %add_nxv4i32 = add <vscale x 4 x i32> undef, undef
; CHECK-NEXT: Cost Model: Found costs of 1 for: %add_nxv2i64 = add <vscale x 2 x i64> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of Invalid for: %add_nxv1i64 = add <vscale x 1 x i64> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of 1 for: %add_nxv1i64 = add <vscale x 1 x i64> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
entry:
diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-cmpsel.ll b/llvm/test/Analysis/CostModel/AArch64/sve-cmpsel.ll
index 5e15920ce8a39..4cff25f786686 100644
--- a/llvm/test/Analysis/CostModel/AArch64/sve-cmpsel.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/sve-cmpsel.ll
@@ -32,12 +32,14 @@ define <vscale x 4 x i1> @cmp_nxv4i64() {
; Check icmp for legal predicate vectors.
define void @cmp_legal_pred() {
; CHECK-LABEL: 'cmp_legal_pred'
+; CHECK-NEXT: Cost Model: Found costs of 1 for: %v = icmp ne <vscale x 1 x i1> undef, undef
; CHECK-NEXT: Cost Model: Found costs of 1 for: %1 = icmp ne <vscale x 2 x i1> undef, undef
; CHECK-NEXT: Cost Model: Found costs of 1 for: %2 = icmp ne <vscale x 4 x i1> undef, undef
; CHECK-NEXT: Cost Model: Found costs of 1 for: %3 = icmp ne <vscale x 8 x i1> undef, undef
; CHECK-NEXT: Cost Model: Found costs of 1 for: %4 = icmp ne <vscale x 16 x i1> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
+ %v = icmp ne <vscale x 1 x i1> undef, undef
%1 = icmp ne <vscale x 2 x i1> undef, undef
%2 = icmp ne <vscale x 4 x i1> undef, undef
%3 = icmp ne <vscale x 8 x i1> undef, undef
diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-fptoi.ll b/llvm/test/Analysis/CostModel/AArch64/sve-fptoi.ll
index ef2ee6e725030..c3b463126c2c6 100644
--- a/llvm/test/Analysis/CostModel/AArch64/sve-fptoi.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/sve-fptoi.ll
@@ -18,12 +18,12 @@ define void @sve-fptoi() {
; CHECK-NEXT: Cost Model: Found costs of Invalid for: %nv1f32_to_ui16 = fptoui <vscale x 1 x float> poison to <vscale x 1 x i16>
; CHECK-NEXT: Cost Model: Found costs of Invalid for: %nv1f32_to_si64 = fptosi <vscale x 1 x float> poison to <vscale x 1 x i64>
; CHECK-NEXT: Cost Model: Found costs of Invalid for: %nv1f32_to_ui64 = fptoui <vscale x 1 x float> poison to <vscale x 1 x i64>
-; CHECK-NEXT: Cost Model: Found costs of 1 for: %nv1f64_to_si8 = fptosi <vscale x 1 x double> poison to <vscale x 1 x i8>
-; CHECK-NEXT: Cost Model: Found costs of 1 for: %nv1f64_to_ui8 = fptoui <vscale x 1 x double> poison to <vscale x 1 x i8>
-; CHECK-NEXT: Cost Model: Found costs of 1 for: %nv1f64_to_si16 = fptosi <vscale x 1 x double> poison to <vscale x 1 x i16>
-; CHECK-NEXT: Cost Model: Found costs of 1 for: %nv1f64_to_ui16 = fptoui <vscale x 1 x double> poison to <vscale x 1 x i16>
-; CHECK-NEXT: Cost Model: Found costs of 1 for: %nv1f64_to_si32 = fptosi <vscale x 1 x double> poison to <vscale x 1 x i32>
-; CHECK-NEXT: Cost Model: Found costs of 1 for: %nv1f64_to_ui32 = fptoui <vscale x 1 x double> poison to <vscale x 1 x i32>
+; CHECK-NEXT: Cost Model: Found costs of Invalid for: %nv1f64_to_si8 = fptosi <vscale x 1 x double> poison to <vscale x 1 x i8>
+; CHECK-NEXT: Cost Model: Found costs of Invalid for: %nv1f64_to_ui8 = fptoui <vscale x 1 x double> poison to <vscale x 1 x i8>
+; CHECK-NEXT: Cost Model: Found costs of Invalid for: %nv1f64_to_si16 = fptosi <vscale x 1 x double> poison to <vscale x 1 x i16>
+; CHECK-NEXT: Cost Model: Found costs of Invalid for: %nv1f64_to_ui16 = fptoui <vscale x 1 x double> poison to <vscale x 1 x i16>
+; CHECK-NEXT: Cost Model: Found costs of Invalid for: %nv1f64_to_si32 = fptosi <vscale x 1 x double> poison to <vscale x 1 x i32>
+; CHECK-NEXT: Cost Model: Found costs of Invalid for: %nv1f64_to_ui32 = fptoui <vscale x 1 x double> poison to <vscale x 1 x i32>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %nv2f16_to_si8 = fptosi <vscale x 2 x half> poison to <vscale x 2 x i8>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %nv2f16_to_ui8 = fptoui <vscale x 2 x half> poison to <vscale x 2 x i8>
; CHECK-NEXT: Cost Model: Found costs of 1 for: %nv2f16_to_si32 = fptosi <vscale x 2 x half> poison to <vscale x 2 x i32>
diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-ldst.ll b/llvm/test/Analysis/CostModel/AArch64/sve-ldst.ll
index 1df82b583a449..bc99f21978280 100644
--- a/llvm/test/Analysis/CostModel/AArch64/sve-ldst.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/sve-ldst.ll
@@ -8,7 +8,7 @@ define void @scalable_loads() {
; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %res.nxv8i8 = load <vscale x 8 x i8>, ptr undef, align 8
; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %res.nxv16i8 = load <vscale x 16 x i8>, ptr undef, align 16
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:4 SizeLat:2 for: %res.nxv32i8 = load <vscale x 32 x i8>, ptr undef, align 32
-; CHECK-NEXT: Cost Model: Found costs of RThru:Invalid CodeSize:Invalid Lat:4 SizeLat:Invalid for: %res.nxv1i64 = load <vscale x 1 x i64>, ptr undef, align 8
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:3 for: %res.nxv1i64 = load <vscale x 1 x i64>, ptr undef, align 8
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:4 SizeLat:2 for: %res.nxv32i1 = load <vscale x 32 x i1>, ptr undef, align 4
; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %res.nxv16i1 = load <vscale x 16 x i1>, ptr undef, align 2
; CHECK-NEXT: Cost Model: Found costs of RThru:Invalid CodeSize:Invalid Lat:4 SizeLat:Invalid for: %res.nxv4i1 = load <vscale x 4 x i1>, ptr undef, align 1
@@ -29,7 +29,7 @@ define void @scalable_stores() {
; CHECK-NEXT: Cost Model: Found costs of 1 for: store <vscale x 8 x i8> undef, ptr undef, align 8
; CHECK-NEXT: Cost Model: Found costs of 1 for: store <vscale x 16 x i8> undef, ptr undef, align 16
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:1 SizeLat:2 for: store <vscale x 32 x i8> undef, ptr undef, align 32
-; CHECK-NEXT: Cost Model: Found costs of Invalid for: store <vscale x 1 x i64> undef, ptr undef, align 8
+; CHECK-NEXT: Cost Model: Found costs of 3 for: store <vscale x 1 x i64> undef, ptr undef, align 8
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:1 SizeLat:2 for: store <vscale x 32 x i1> undef, ptr undef, align 4
; CHECK-NEXT: Cost Model: Found costs of 1 for: store <vscale x 16 x i1> undef, ptr undef, align 2
; CHECK-NEXT: Cost Model: Found costs of Invalid for: store <vscale x 4 x i1> undef, ptr undef, align 1
diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-segment-gather-scatter.ll b/llvm/test/Analysis/CostModel/AArch64/sve-segment-gather-scatter.ll
new file mode 100644
index 0000000000000..62e8d255f3fe1
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/AArch64/sve-segment-gather-scatter.ll
@@ -0,0 +1,54 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output < %s | FileCheck %s --check-prefix=CHECK-VSCALE-2
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -sve-vscale-for-tuning=2 < %s | FileCheck %s --check-prefix=CHECK-VSCALE-2
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mcpu=neoverse-v1 < %s | FileCheck %s --check-prefix=CHECK-VSCALE-2
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -sve-vscale-for-tuning=4 < %s | FileCheck %s --check-prefix=CHECK-VSCALE-4
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple="aarch64--linux-gnu"
+
+define void @masked_segment_gather_i16(<vscale x 1 x ptr> %ptrs, <vscale x 1 x i1> %mask) #0 {
+; CHECK-VSCALE-2-LABEL: 'masked_segment_gather_i16'
+; CHECK-VSCALE-2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %res.nxv4i16 = call <vscale x 4 x i16> @llvm.masked.segment.gather.nxv4i16.nxv1p0(<vscale x 1 x ptr> align 8 %ptrs, <vscale x 1 x i1> %mask, <vscale x 4 x i16> zeroinitializer)
+; CHECK-VSCALE-2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %res.nxv2i32 = call <vscale x 2 x i32> @llvm.masked.segment.gather.nxv2i32.nxv1p0(<vscale x 1 x ptr> align 8 %ptrs, <vscale x 1 x i1> %mask, <vscale x 2 x i32> zeroinitializer)
+; CHECK-VSCALE-2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %res.nxv8i16 = call <vscale x 8 x i16> @llvm.masked.segment.gather.nxv8i16.nxv1p0(<vscale x 1 x ptr> align 16 %ptrs, <vscale x 1 x i1> %mask, <vscale x 8 x i16> zeroinitializer)
+; CHECK-VSCALE-2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %res.nxv4i32 = call <vscale x 4 x i32> @llvm.masked.segment.gather.nxv4i32.nxv1p0(<vscale x 1 x ptr> align 16 %ptrs, <vscale x 1 x i1> %mask, <vscale x 4 x i32> zeroinitializer)
+; CHECK-VSCALE-2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-VSCALE-4-LABEL: 'masked_segment_gather_i16'
+; CHECK-VSCALE-4-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %res.nxv4i16 = call <vscale x 4 x i16> @llvm.masked.segment.gather.nxv4i16.nxv1p0(<vscale x 1 x ptr> align 8 %ptrs, <vscale x 1 x i1> %mask, <vscale x 4 x i16> zeroinitializer)
+; CHECK-VSCALE-4-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %res.nxv2i32 = call <vscale x 2 x i32> @llvm.masked.segment.gather.nxv2i32.nxv1p0(<vscale x 1 x ptr> align 8 %ptrs, <vscale x 1 x i1> %mask, <vscale x 2 x i32> zeroinitializer)
+; CHECK-VSCALE-4-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %res.nxv8i16 = call <vscale x 8 x i16> @llvm.masked.segment.gather.nxv8i16.nxv1p0(<vscale x 1 x ptr> align 16 %ptrs, <vscale x 1 x i1> %mask, <vscale x 8 x i16> zeroinitializer)
+; CHECK-VSCALE-4-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %res.nxv4i32 = call <vscale x 4 x i32> @llvm.masked.segment.gather.nxv4i32.nxv1p0(<vscale x 1 x ptr> align 16 %ptrs, <vscale x 1 x i1> %mask, <vscale x 4 x i32> zeroinitializer)
+; CHECK-VSCALE-4-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+ %res.nxv4i16 = call <vscale x 4 x i16> @llvm.masked.segment.gather.nxv4i16.nxv1p0(<vscale x 1 x ptr> align 8 %ptrs, <vscale x 1 x i1> %mask, <vscale x 4 x i16> zeroinitializer)
+ %res.nxv2i32 = call <vscale x 2 x i32> @llvm.masked.segment.gather.nxv2i32.nxv1p0(<vscale x 1 x ptr> align 8 %ptrs, <vscale x 1 x i1> %mask, <vscale x 2 x i32> zeroinitializer)
+ %res.nxv8i16 = call <vscale x 8 x i16> @llvm.masked.segment.gather.nxv8i16.nxv1p0(<vscale x 1 x ptr> align 16 %ptrs, <vscale x 1 x i1> %mask, <vscale x 8 x i16> zeroinitializer)
+ %res.nxv4i32 = call <vscale x 4 x i32> @llvm.masked.segment.gather.nxv4i32.nxv1p0(<vscale x 1 x ptr> align 16 %ptrs, <vscale x 1 x i1> %mask, <vscale x 4 x i32> zeroinitializer)
+ ret void
+}
+
+define void @masked_segment_scatter_i16(<vscale x 4 x i16> %nxv4i16, <vscale x 8 x i16> %nxv8i16, <vscale x 2 x i32> %nxv2i32, <vscale x 4 x i32> %nxv4i32, <vscale x 1 x ptr> %ptrs, <vscale x 1 x i1> %mask) #0 {
+; CHECK-VSCALE-2-LABEL: 'masked_segment_scatter_i16'
+; CHECK-VSCALE-2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.segment.scatter.nxv4i16.nxv1p0(<vscale x 4 x i16> %nxv4i16, <vscale x 1 x ptr> align 8 %ptrs, <vscale x 1 x i1> %mask)
+; CHECK-VSCALE-2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.segment.scatter.nxv2i32.nxv1p0(<vscale x 2 x i32> %nxv2i32, <vscale x 1 x ptr> align 8 %ptrs, <vscale x 1 x i1> %mask)
+; CHECK-VSCALE-2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.segment.scatter.nxv8i16.nxv1p0(<vscale x 8 x i16> %nxv8i16, <vscale x 1 x ptr> align 16 %ptrs, <vscale x 1 x i1> %mask)
+; CHECK-VSCALE-2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.segment.scatter.nxv4i32.nxv1p0(<vscale x 4 x i32> %nxv4i32, <vscale x 1 x ptr> align 16 %ptrs, <vscale x 1 x i1> %mask)
+; CHECK-VSCALE-2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-VSCALE-4-LABEL: 'masked_segment_scatter_i16'
+; CHECK-VSCALE-4-NEXT: Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.segment.scatter.nxv4i16.nxv1p0(<vscale x 4 x i16> %nxv4i16, <vscale x 1 x ptr> align 8 %ptrs, <vscale x 1 x i1> %mask)
+; CHECK-VSCALE-4-NEXT: Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.segment.scatter.nxv2i32.nxv1p0(<vscale x 2 x i32> %nxv2i32, <vscale x 1 x ptr> align 8 %ptrs, <vscale x 1 x i1> %mask)
+; CHECK-VSCALE-4-NEXT: Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.segment.scatter.nxv8i16.nxv1p0(<vscale x 8 x i16> %nxv8i16, <vscale x 1 x ptr> align 16 %ptrs, <vscale x 1 x i1> %mask)
+; CHECK-VSCALE-4-NEXT: Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.segment.scatter.nxv4i32.nxv1p0(<vscale x 4 x i32> %nxv4i32, <vscale x 1 x ptr> align 16 %ptrs, <vscale x 1 x i1> %mask)
+; CHECK-VSCALE-4-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+ call void @llvm.masked.segment.scatter.nxv4i16.nxv1p0(<vscale x 4 x i16> %nxv4i16, <vscale x 1 x ptr> align 8 %ptrs, <vscale x 1 x i1> %mask)
+ call void @llvm.masked.segment.scatter.nxv2i32.nxv1p0(<vscale x 2 x i32> %nxv2i32, <vscale x 1 x ptr> align 8 %ptrs, <vscale x 1 x i1> %mask)
+ call void @llvm.masked.segment.scatter.nxv8i16.nxv1p0(<vscale x 8 x i16> %nxv8i16, <vscale x 1 x ptr> align 16 %ptrs, <vscale x 1 x i1> %mask)
+ call void @llvm.masked.segment.scatter.nxv4i32.nxv1p0(<vscale x 4 x i32> %nxv4i32, <vscale x 1 x ptr> align 16 %ptrs, <vscale x 1 x i1> %mask)
+ ret void
+}
+
+attributes #0 = { "target-features"="+sve2p1" }
diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-segmented-shuffle.ll b/llvm/test/Analysis/CostModel/AArch64/sve-segmented-shuffle.ll
new file mode 100644
index 0000000000000..01762d569266c
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/AArch64/sve-segmented-shuffle.ll
@@ -0,0 +1,164 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
+
+; Check getSegmentedShuffleCost
+
+; RUN: opt -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mtriple=aarch64--linux-gnu -mattr=sve < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SVE1
+; RUN: opt -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mtriple=aarch64--linux-gnu -mattr=sve2p1 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SVE2p1
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+
+define void @shuffle_known_64() {
+; CHECK-SVE1-LABEL: 'shuffle_known_64'
+; CHECK-SVE1-NEXT: Cost Model: Found costs of Invalid for: %dupq = call <vscale x 8 x i8> @llvm.vector.segmented.shuffle.nxv8i8.v8i32(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef, <8 x i32> splat (i32 5))
+; CHECK-SVE1-NEXT: Cost Model: Found costs of Invalid for: %rev.q = call <vscale x 8 x i8> @llvm.vector.segmented.shuffle.nxv8i8.v8i32(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+; CHECK-SVE1-NEXT: Cost Model: Found costs of Invalid for: %zip1 = call <vscale x 8 x i8> @llvm.vector.segmented.shuffle.nxv8i8.v8i32(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>)
+; CHECK-SVE1-NEXT: Cost Model: Found costs of Invalid for: %zip2 = call <vscale x 8 x i8> @llvm.vector.segmented.shuffle.nxv8i8.v8i32(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef, <8 x i32> <i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>)
+; CHECK-SVE1-NEXT: Cost Model: Found costs of Invalid for: %uzp1 = call <vscale x 8 x i8> @llvm.vector.segmented.shuffle.nxv8i8.v8i32(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>)
+; CHECK-SVE1-NEXT: Cost Model: Found costs of Invalid for: %uzp2 = call <vscale x 8 x i8> @llvm.vector.segmented.shuffle.nxv8i8.v8i32(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>)
+; CHECK-SVE1-NEXT: Cost Model: Found costs of Invalid for: %trn1 = call <vscale x 8 x i8> @llvm.vector.segmented.shuffle.nxv8i8.v8i32(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>)
+; CHECK-SVE1-NEXT: Cost Model: Found costs of Invalid for: %trn2 = call <vscale x 8 x i8> @llvm.vector.segmented.shuffle.nxv8i8.v8i32(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>)
+; CHECK-SVE1-NEXT: Cost Model: Found costs of Invalid for: %ext = call <vscale x 8 x i8> @llvm.vector.segmented.shuffle.nxv8i8.v8i32(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef, <8 x i32> <i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12>)
+; CHECK-SVE1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-SVE2p1-LABEL: 'shuffle_known_64'
+; CHECK-SVE2p1-NEXT: Cost Model: Found costs of 1 for: %dupq = call <vscale x 8 x i8> @llvm.vector.segmented.shuffle.nxv8i8.v8i32(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef, <8 x i32> splat (i32 5))
+; CHECK-SVE2p1-NEXT: Cost Model: Found costs of 1 for: %rev.q = call <vscale x 8 x i8> @llvm.vector.segmented.shuffle.nxv8i8.v8i32(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+; CHECK-SVE2p1-NEXT: Cost Model: Found costs of 1 for: %zip1 = call <vscale x 8 x i8> @llvm.vector.segmented.shuffle.nxv8i8.v8i32(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>)
+; CHECK-SVE2p1-NEXT: Cost Model: Found costs of 1 for: %zip2 = call <vscale x 8 x i8> @llvm.vector.segmented.shuffle.nxv8i8.v8i32(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef, <8 x i32> <i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>)
+; CHECK-SVE2p1-NEXT: Cost Model: Found costs of 1 for: %uzp1 = call <vscale x 8 x i8> @llvm.vector.segmented.shuffle.nxv8i8.v8i32(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>)
+; CHECK-SVE2p1-NEXT: Cost Model: Found costs of 1 for: %uzp2 = call <vscale x 8 x i8> @llvm.vector.segmented.shuffle.nxv8i8.v8i32(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>)
+; CHECK-SVE2p1-NEXT: Cost Model: Found costs of 1 for: %trn1 = call <vscale x 8 x i8> @llvm.vector.segmented.shuffle.nxv8i8.v8i32(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>)
+; CHECK-SVE2p1-NEXT: Cost Model: Found costs of 1 for: %trn2 = call <vscale x 8 x i8> @llvm.vector.segmented.shuffle.nxv8i8.v8i32(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>)
+; CHECK-SVE2p1-NEXT: Cost Model: Found costs of 1 for: %ext = call <vscale x 8 x i8> @llvm.vector.segmented.shuffle.nxv8i8.v8i32(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef, <8 x i32> <i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12>)
+; CHECK-SVE2p1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-NETX: Cost Model: Found an estimated cost of 0 for instruction: ret void
+ %dupq = call <vscale x 8 x i8> @llvm.vector.segmented.shuffle.nxv8i8.v8i32(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef,
+ <8 x i32> splat (i32 5))
+ %rev.q = call <vscale x 8 x i8> @llvm.vector.segmented.shuffle.nxv8i8.v8i32(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef,
+ <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+ %zip1 = call <vscale x 8 x i8> @llvm.vector.segmented.shuffle.nxv8i8.v8i32(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef,
+ <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>)
+ %zip2 = call <vscale x 8 x i8> @llvm.vector.segmented.shuffle.nxv8i8.v8i32(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef,
+ <8 x i32> <i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>)
+ %uzp1 = call <vscale x 8 x i8> @llvm.vector.segmented.shuffle.nxv8i8.v8i32(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef,
+ <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>)
+ %uzp2 = call <vscale x 8 x i8> @llvm.vector.segmented.shuffle.nxv8i8.v8i32(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef,
+ <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>)
+ %trn1 = call <vscale x 8 x i8> @llvm.vector.segmented.shuffle.nxv8i8.v8i32(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef,
+ <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>)
+ %trn2 = call <vscale x 8 x i8> @llvm.vector.segmented.shuffle.nxv8i8.v8i32(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef,
+ <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>)
+ %ext = call <vscale x 8 x i8> @llvm.vector.segmented.shuffle.nxv8i8.v8i32(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef,
+ <8 x i32> <i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12>)
+ ret void
+}
+
+define void @shuffle_known_128() {
+; CHECK-SVE1-LABEL: 'shuffle_known_128'
+; CHECK-SVE1-NEXT: Cost Model: Found costs of Invalid for: %dupq = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <16 x i32> splat (i32 5))
+; CHECK-SVE1-NEXT: Cost Model: Found costs of Invalid for: %rev.q = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+; CHECK-SVE1-NEXT: Cost Model: Found costs of Invalid for: %zip1 = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23>)
+; CHECK-SVE1-NEXT: Cost Model: Found costs of Invalid for: %zip2 = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <16 x i32> <i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>)
+; CHECK-SVE1-NEXT: Cost Model: Found costs of Invalid for: %uzp1 = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>)
+; CHECK-SVE1-NEXT: Cost Model: Found costs of Invalid for: %uzp2 = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>)
+; CHECK-SVE1-NEXT: Cost Model: Found costs of Invalid for: %trn1 = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>)
+; CHECK-SVE1-NEXT: Cost Model: Found costs of Invalid for: %trn2 = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <16 x i32> <i32 1, i32 17, i32 3, i32 19, i32 5, i32 21, i32 7, i32 23, i32 9, i32 25, i32 11, i32 27, i32 13, i32 29, i32 15, i32 31>)
+; CHECK-SVE1-NEXT: Cost Model: Found costs of Invalid for: %ext = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <16 x i32> <i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20>)
+; CHECK-SVE1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-SVE2p1-LABEL: 'shuffle_known_128'
+; CHECK-SVE2p1-NEXT: Cost Model: Found costs of 1 for: %dupq = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <16 x i32> splat (i32 5))
+; CHECK-SVE2p1-NEXT: Cost Model: Found costs of 1 for: %rev.q = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+; CHECK-SVE2p1-NEXT: Cost Model: Found costs of 1 for: %zip1 = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23>)
+; CHECK-SVE2p1-NEXT: Cost Model: Found costs of 1 for: %zip2 = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <16 x i32> <i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>)
+; CHECK-SVE2p1-NEXT: Cost Model: Found costs of 1 for: %uzp1 = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>)
+; CHECK-SVE2p1-NEXT: Cost Model: Found costs of 1 for: %uzp2 = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>)
+; CHECK-SVE2p1-NEXT: Cost Model: Found costs of 1 for: %trn1 = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>)
+; CHECK-SVE2p1-NEXT: Cost Model: Found costs of 1 for: %trn2 = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <16 x i32> <i32 1, i32 17, i32 3, i32 19, i32 5, i32 21, i32 7, i32 23, i32 9, i32 25, i32 11, i32 27, i32 13, i32 29, i32 15, i32 31>)
+; CHECK-SVE2p1-NEXT: Cost Model: Found costs of 1 for: %ext = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <16 x i32> <i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20>)
+; CHECK-SVE2p1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-NETX: Cost Model: Found an estimated cost of 0 for instruction: ret void
+ %dupq = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef,
+ <16 x i32> splat (i32 5))
+ %rev.q = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef,
+ <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+ %zip1 = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef,
+ <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23>)
+ %zip2 = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef,
+ <16 x i32> <i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>)
+ %uzp1 = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef,
+ <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>)
+ %uzp2 = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef,
+ <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>)
+ %trn1 = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef,
+ <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>)
+ %trn2 = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef,
+ <16 x i32> <i32 1, i32 17, i32 3, i32 19, i32 5, i32 21, i32 7, i32 23, i32 9, i32 25, i32 11, i32 27, i32 13, i32 29, i32 15, i32 31>)
+ %ext = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef,
+ <16 x i32> <i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20>)
+ ret void
+}
+
+; The segment length needs to match the minimum element count,
+; otherwise we cannot use the sve2p1 q instructions.
+define void @shuffle_known_128_segment_64() {
+; CHECK-LABEL: 'shuffle_known_128_segment_64'
+; CHECK-NEXT: Cost Model: Found costs of Invalid for: %dupq = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v8i32(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <8 x i32> splat (i32 5))
+; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-NETX: Cost Model: Found an estimated cost of 0 for instruction: ret void
+ %dupq = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v8i32(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef,
+ <8 x i32> splat (i32 5))
+ ret void
+}
+
+define void @shuffle_known_256() {
+; CHECK-LABEL: 'shuffle_known_256'
+; CHECK-NEXT: Cost Model: Found costs of Invalid for: %dupq = call <vscale x 32 x i8> @llvm.vector.segmented.shuffle.nxv32i8.v32i32(<vscale x 32 x i8> undef, <vscale x 32 x i8> undef, <32 x i32> splat (i32 5))
+; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-NETX: Cost Model: Found an estimated cost of 0 for instruction: ret void
+ %dupq = call <vscale x 32 x i8> @llvm.vector.segmented.shuffle.nxv32i8.v32i32(<vscale x 32 x i8> undef, <vscale x 32 x i8> undef,
+ <32 x i32> splat (i32 5))
+ ret void
+}
+
+define void @shuffle_arbitrary_64() {
+; CHECK-SVE1-LABEL: 'shuffle_arbitrary_64'
+; CHECK-SVE1-NEXT: Cost Model: Found costs of Invalid for: %one_src = call <vscale x 8 x i8> @llvm.vector.segmented.shuffle.nxv8i8.v8i32(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef, <8 x i32> <i32 7, i32 4, i32 4, i32 4, i32 3, i32 2, i32 1, i32 0>)
+; CHECK-SVE1-NEXT: Cost Model: Found costs of Invalid for: %two_src = call <vscale x 8 x i8> @llvm.vector.segmented.shuffle.nxv8i8.v8i32(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef, <8 x i32> <i32 12, i32 4, i32 4, i32 4, i32 3, i32 2, i32 1, i32 0>)
+; CHECK-SVE1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-SVE2p1-LABEL: 'shuffle_arbitrary_64'
+; CHECK-SVE2p1-NEXT: Cost Model: Found costs of 2 for: %one_src = call <vscale x 8 x i8> @llvm.vector.segmented.shuffle.nxv8i8.v8i32(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef, <8 x i32> <i32 7, i32 4, i32 4, i32 4, i32 3, i32 2, i32 1, i32 0>)
+; CHECK-SVE2p1-NEXT: Cost Model: Found costs of 3 for: %two_src = call <vscale x 8 x i8> @llvm.vector.segmented.shuffle.nxv8i8.v8i32(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef, <8 x i32> <i32 12, i32 4, i32 4, i32 4, i32 3, i32 2, i32 1, i32 0>)
+; CHECK-SVE2p1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-NETX: Cost Model: Found an estimated cost of 0 for instruction: ret void
+ %one_src = call <vscale x 8 x i8> @llvm.vector.segmented.shuffle.nxv8i8.v8i32(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef,
+ <8 x i32> <i32 7, i32 4, i32 4, i32 4, i32 3, i32 2, i32 1, i32 0>)
+ %two_src = call <vscale x 8 x i8> @llvm.vector.segmented.shuffle.nxv8i8.v8i32(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef,
+ <8 x i32> <i32 12, i32 4, i32 4, i32 4, i32 3, i32 2, i32 1, i32 0>)
+ ret void
+}
+
+define void @shuffle_arbitrary_128() {
+; CHECK-SVE1-LABEL: 'shuffle_arbitrary_128'
+; CHECK-SVE1-NEXT: Cost Model: Found costs of Invalid for: %one_src = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <16 x i32> <i32 15, i32 12, i32 10, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+; CHECK-SVE1-NEXT: Cost Model: Found costs of Invalid for: %two_src = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <16 x i32> <i32 20, i32 12, i32 10, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+; CHECK-SVE1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-SVE2p1-LABEL: 'shuffle_arbitrary_128'
+; CHECK-SVE2p1-NEXT: Cost Model: Found costs of 2 for: %one_src = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <16 x i32> <i32 15, i32 12, i32 10, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+; CHECK-SVE2p1-NEXT: Cost Model: Found costs of 3 for: %two_src = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <16 x i32> <i32 20, i32 12, i32 10, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+; CHECK-SVE2p1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-NETX: Cost Model: Found an estimated cost of 0 for instruction: ret void
+ %one_src = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef,
+ <16 x i32> <i32 15, i32 12, i32 10, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+ %two_src = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef,
+ <16 x i32> <i32 20, i32 12, i32 10, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+ ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll b/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll
index ee2d482d9ffb5..8294e96f28bad 100644
--- a/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll
+++ b/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll
@@ -12,6 +12,30 @@ define { <vscale x 16 x i8>, <vscale x 16 x i8> } @foo_ld2_nxv16i8(<vscale x 16
ret { <vscale x 16 x i8>, <vscale x 16 x i8> } %deinterleaved.vec
}
+define { <vscale x 16 x i8>, <vscale x 16 x i8> } @foo_ld2_nxv16i8_all_true(ptr %p) {
+; CHECK-LABEL: foo_ld2_nxv16i8_all_true:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: ld2b { z0.b, z1.b }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %wide.masked.vec = call <vscale x 32 x i8> @llvm.masked.load.nxv32i8(ptr %p, i32 1, <vscale x 32 x i1> splat (i1 true), <vscale x 32 x i8> poison)
+ %deinterleaved.vec = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.vector.deinterleave2.nxv32i8(<vscale x 32 x i8> %wide.masked.vec)
+ ret { <vscale x 16 x i8>, <vscale x 16 x i8> } %deinterleaved.vec
+}
+
+; TODO: This gets combined by the VECTOR_DEINTERLEAVE DAGCombiner,
+; even though the InterleavedAccess pass bails out.
+define { <vscale x 16 x i8>, <vscale x 16 x i8> } @foo_ld2_nxv16i8_all_false(ptr %p) {
+; CHECK-LABEL: foo_ld2_nxv16i8_all_false:
+; CHECK: // %bb.0:
+; CHECK-NEXT: pfalse p0.b
+; CHECK-NEXT: ld2b { z0.b, z1.b }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %wide.masked.vec = call <vscale x 32 x i8> @llvm.masked.load.nxv32i8(ptr %p, i32 1, <vscale x 32 x i1> splat (i1 false), <vscale x 32 x i8> poison)
+ %deinterleaved.vec = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.vector.deinterleave2.nxv32i8(<vscale x 32 x i8> %wide.masked.vec)
+ ret { <vscale x 16 x i8>, <vscale x 16 x i8> } %deinterleaved.vec
+}
+
define { <vscale x 8 x i16>, <vscale x 8 x i16> } @foo_ld2_nxv8i16(<vscale x 8 x i1> %mask, ptr %p) {
; CHECK-LABEL: foo_ld2_nxv8i16:
; CHECK: // %bb.0:
diff --git a/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll b/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
index e042a48f81b40..92281bf2599f4 100644
--- a/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
+++ b/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
@@ -159,10 +159,10 @@ define void @foo_st2_nxv16i8_all_false_mask(<vscale x 16 x i8> %val1, <vscale x
define void @foo_st2_nxv16i8_all_true_mask(<vscale x 16 x i8> %val1, <vscale x 16 x i8> %val2, ptr %p) {
; CHECK-LABEL: foo_st2_nxv16i8_all_true_mask:
; CHECK: // %bb.0:
-; CHECK-NEXT: zip2 z2.b, z0.b, z1.b
-; CHECK-NEXT: zip1 z0.b, z0.b, z1.b
-; CHECK-NEXT: str z2, [x0, #1, mul vl]
-; CHECK-NEXT: str z0, [x0]
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: st2b { z0.b, z1.b }, p0, [x0]
; CHECK-NEXT: ret
%interleaved.value = call <vscale x 32 x i8> @llvm.vector.interleave2.nxv32i8(<vscale x 16 x i8> %val1, <vscale x 16 x i8> %val2)
call void @llvm.masked.store.nxv32i8.p0(<vscale x 32 x i8> %interleaved.value, ptr %p, i32 1, <vscale x 32 x i1> splat(i1 1))
diff --git a/llvm/test/CodeGen/AArch64/sve-int-arith.ll b/llvm/test/CodeGen/AArch64/sve-int-arith.ll
index c59b1d430ff4f..776e2d30190aa 100644
--- a/llvm/test/CodeGen/AArch64/sve-int-arith.ll
+++ b/llvm/test/CodeGen/AArch64/sve-int-arith.ll
@@ -45,6 +45,16 @@ define <vscale x 16 x i8> @add_i8_zero(<vscale x 16 x i8> %a) {
ret <vscale x 16 x i8> %res
}
+define <vscale x 1 x i64> @add_nxv1i64(<vscale x 1 x i64> %a, <vscale x 1 x i64> %b) {
+; CHECK-LABEL: add_nxv1i64:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: add z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+entry:
+ %c = add <vscale x 1 x i64> %a, %b
+ ret <vscale x 1 x i64> %c
+}
+
define <vscale x 1 x i32> @add_nxv1i32(<vscale x 1 x i32> %a, <vscale x 1 x i32> %b) {
; CHECK-LABEL: add_nxv1i32:
; CHECK: // %bb.0: // %entry
@@ -55,6 +65,26 @@ entry:
ret <vscale x 1 x i32> %c
}
+define <vscale x 1 x i16> @add_nxv1i16(<vscale x 1 x i16> %a, <vscale x 1 x i16> %b) {
+; CHECK-LABEL: add_nxv1i16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: add z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+entry:
+ %c = add <vscale x 1 x i16> %a, %b
+ ret <vscale x 1 x i16> %c
+}
+
+define <vscale x 1 x i8> @add_nxv1i8(<vscale x 1 x i8> %a, <vscale x 1 x i8> %b) {
+; CHECK-LABEL: add_nxv1i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: add z0.b, z0.b, z1.b
+; CHECK-NEXT: ret
+entry:
+ %c = add <vscale x 1 x i8> %a, %b
+ ret <vscale x 1 x i8> %c
+}
+
define <vscale x 2 x i64> @sub_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: sub_i64:
; CHECK: // %bb.0:
@@ -99,6 +129,46 @@ define <vscale x 16 x i8> @sub_i8_zero(<vscale x 16 x i8> %a) {
ret <vscale x 16 x i8> %res
}
+define <vscale x 1 x i64> @sub_nxv1i64(<vscale x 1 x i64> %a, <vscale x 1 x i64> %b) {
+; CHECK-LABEL: sub_nxv1i64:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: sub z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+entry:
+ %c = sub <vscale x 1 x i64> %a, %b
+ ret <vscale x 1 x i64> %c
+}
+
+define <vscale x 1 x i32> @sub_nxv1i32(<vscale x 1 x i32> %a, <vscale x 1 x i32> %b) {
+; CHECK-LABEL: sub_nxv1i32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: sub z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+entry:
+ %c = sub <vscale x 1 x i32> %a, %b
+ ret <vscale x 1 x i32> %c
+}
+
+define <vscale x 1 x i16> @sub_nxv1i16(<vscale x 1 x i16> %a, <vscale x 1 x i16> %b) {
+; CHECK-LABEL: sub_nxv1i16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: sub z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+entry:
+ %c = sub <vscale x 1 x i16> %a, %b
+ ret <vscale x 1 x i16> %c
+}
+
+define <vscale x 1 x i8> @sub_nxv1i8(<vscale x 1 x i8> %a, <vscale x 1 x i8> %b) {
+; CHECK-LABEL: sub_nxv1i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: sub z0.b, z0.b, z1.b
+; CHECK-NEXT: ret
+entry:
+ %c = sub <vscale x 1 x i8> %a, %b
+ ret <vscale x 1 x i8> %c
+}
+
define <vscale x 16 x i8> @abs_nxv16i8(<vscale x 16 x i8> %a) {
; CHECK-LABEL: abs_nxv16i8:
; CHECK: // %bb.0:
@@ -337,6 +407,50 @@ define <vscale x 16 x i8> @uqsub_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b
ret <vscale x 16 x i8> %res
}
+define <vscale x 1 x i64> @mul_nxv1i64(<vscale x 1 x i64> %a, <vscale x 1 x i64> %b) {
+; CHECK-LABEL: mul_nxv1i64:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: mul z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+entry:
+ %c = mul <vscale x 1 x i64> %a, %b
+ ret <vscale x 1 x i64> %c
+}
+
+define <vscale x 1 x i32> @mul_nxv1i32(<vscale x 1 x i32> %a, <vscale x 1 x i32> %b) {
+; CHECK-LABEL: mul_nxv1i32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: mul z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
+entry:
+ %c = mul <vscale x 1 x i32> %a, %b
+ ret <vscale x 1 x i32> %c
+}
+
+define <vscale x 1 x i16> @mul_nxv1i16(<vscale x 1 x i16> %a, <vscale x 1 x i16> %b) {
+; CHECK-LABEL: mul_nxv1i16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: mul z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: ret
+entry:
+ %c = mul <vscale x 1 x i16> %a, %b
+ ret <vscale x 1 x i16> %c
+}
+
+define <vscale x 1 x i8> @mul_nxv1i8(<vscale x 1 x i8> %a, <vscale x 1 x i8> %b) {
+; CHECK-LABEL: mul_nxv1i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: mul z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT: ret
+entry:
+ %c = mul <vscale x 1 x i8> %a, %b
+ ret <vscale x 1 x i8> %c
+}
+
define <vscale x 16 x i8> @mad_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c) {
; CHECK-LABEL: mad_i8:
; CHECK: // %bb.0:
@@ -767,7 +881,7 @@ define void @mad_in_loop(ptr %dst, ptr %src1, ptr %src2, i32 %n) {
; CHECK-LABEL: mad_in_loop:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: cmp w3, #1
-; CHECK-NEXT: b.lt .LBB70_3
+; CHECK-NEXT: b.lt .LBB81_3
; CHECK-NEXT: // %bb.1: // %for.body.preheader
; CHECK-NEXT: mov w9, w3
; CHECK-NEXT: mov z0.s, #1 // =0x1
@@ -775,7 +889,7 @@ define void @mad_in_loop(ptr %dst, ptr %src1, ptr %src2, i32 %n) {
; CHECK-NEXT: whilelo p1.s, xzr, x9
; CHECK-NEXT: mov x8, xzr
; CHECK-NEXT: cntw x10
-; CHECK-NEXT: .LBB70_2: // %vector.body
+; CHECK-NEXT: .LBB81_2: // %vector.body
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
; CHECK-NEXT: ld1w { z1.s }, p1/z, [x1, x8, lsl #2]
; CHECK-NEXT: ld1w { z2.s }, p1/z, [x2, x8, lsl #2]
@@ -783,8 +897,8 @@ define void @mad_in_loop(ptr %dst, ptr %src1, ptr %src2, i32 %n) {
; CHECK-NEXT: st1w { z1.s }, p1, [x0, x8, lsl #2]
; CHECK-NEXT: add x8, x8, x10
; CHECK-NEXT: whilelo p1.s, x8, x9
-; CHECK-NEXT: b.mi .LBB70_2
-; CHECK-NEXT: .LBB70_3: // %for.cond.cleanup
+; CHECK-NEXT: b.mi .LBB81_2
+; CHECK-NEXT: .LBB81_3: // %for.cond.cleanup
; CHECK-NEXT: ret
entry:
%cmp9 = icmp sgt i32 %n, 0
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-ldst-nonext.ll b/llvm/test/CodeGen/AArch64/sve-masked-ldst-nonext.ll
index 800bcc1231460..773db5d3f09a2 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-ldst-nonext.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-ldst-nonext.ll
@@ -4,6 +4,16 @@
; Masked Loads
;
+define <vscale x 1 x i64> @masked_load_nxv1i64(ptr %a, <vscale x 1 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_load_nxv1i64:
+; CHECK: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %load = call <vscale x 1 x i64> @llvm.masked.load.nxv1i64(ptr %a, i32 8, <vscale x 1 x i1> %mask, <vscale x 1 x i64> poison)
+ ret <vscale x 1 x i64> %load
+}
+
define <vscale x 2 x i64> @masked_load_nxv2i64(ptr %a, <vscale x 2 x i1> %mask) nounwind {
; CHECK-LABEL: masked_load_nxv2i64:
; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
@@ -12,6 +22,19 @@ define <vscale x 2 x i64> @masked_load_nxv2i64(ptr %a, <vscale x 2 x i1> %mask)
ret <vscale x 2 x i64> %load
}
+; TODO: This is sub-optimal, we should use an extending ld1w z.d.
+define <vscale x 1 x i32> @masked_load_nxv1i32(ptr %a, <vscale x 1 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_load_nxv1i32:
+; CHECK: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: uzp1 p1.d, p1.d, p1.d
+; CHECK-NEXT: uzp1 p0.s, p0.s, p1.s
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %load = call <vscale x 1 x i32> @llvm.masked.load.nxv1i32(ptr %a, i32 4, <vscale x 1 x i1> %mask, <vscale x 1 x i32> poison)
+ ret <vscale x 1 x i32> %load
+}
+
define <vscale x 4 x i32> @masked_load_nxv4i32(ptr %a, <vscale x 4 x i1> %mask) nounwind {
; CHECK-LABEL: masked_load_nxv4i32:
; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]
@@ -20,6 +43,45 @@ define <vscale x 4 x i32> @masked_load_nxv4i32(ptr %a, <vscale x 4 x i1> %mask)
ret <vscale x 4 x i32> %load
}
+define <vscale x 4 x i32> @masked_load_nxv4i32_i64_mask(ptr %a, <vscale x 2 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_load_nxv4i32_i64_mask:
+; CHECK: ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %mask.stretched = call <vscale x 4 x i1> @llvm.vector.stretch.nxv4i1.nxv2i1(<vscale x 2 x i1> %mask)
+ %load = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32(ptr %a, i32 4, <vscale x 4 x i1> %mask.stretched, <vscale x 4 x i32> poison)
+ ret <vscale x 4 x i32> %load
+}
+
+define <vscale x 4 x i32> @masked_load_nxv4i32_i64_mask_icmp(ptr %a, <vscale x 2 x i64> %v1.legalabi, <vscale x 2 x i64> %v2.legalabi) nounwind {
+; CHECK-LABEL: masked_load_nxv4i32_i64_mask_icmp:
+; CHECK: ptrue p0.d
+; CHECK-NEXT: cmpeq p1.d, p0/z, z0.d, z1.d
+; CHECK-NEXT: zip1 p0.d, p1.d, p1.d
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %v1 = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(<vscale x 2 x i64> %v1.legalabi, i64 0)
+ %v2 = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(<vscale x 2 x i64> %v2.legalabi, i64 0)
+ %mask = icmp eq <vscale x 1 x i64> %v1, %v2
+ %mask.stretched = call <vscale x 4 x i1> @llvm.vector.stretch.nxv4i1.nxv2i1(<vscale x 1 x i1> %mask)
+ %load = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32(ptr %a, i32 4, <vscale x 4 x i1> %mask.stretched, <vscale x 4 x i32> poison)
+ ret <vscale x 4 x i32> %load
+}
+
+; TODO: This is sub-optimal, we should use an extending ld1h z.d.
+define <vscale x 1 x i16> @masked_load_nxv1i16(ptr %a, <vscale x 1 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_load_nxv1i16:
+; CHECK: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: uzp1 p1.d, p1.d, p1.d
+; CHECK-NEXT: uzp1 p0.s, p0.s, p1.s
+; CHECK-NEXT: uzp1 p1.s, p1.s, p1.s
+; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h
+; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %load = call <vscale x 1 x i16> @llvm.masked.load.nxv1i16(ptr %a, i32 2, <vscale x 1 x i1> %mask, <vscale x 1 x i16> poison)
+ ret <vscale x 1 x i16> %load
+}
+
define <vscale x 8 x i16> @masked_load_nxv8i16(ptr %a, <vscale x 8 x i1> %mask) nounwind {
; CHECK-LABEL: masked_load_nxv8i16:
; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]
@@ -28,6 +90,51 @@ define <vscale x 8 x i16> @masked_load_nxv8i16(ptr %a, <vscale x 8 x i1> %mask)
ret <vscale x 8 x i16> %load
}
+define <vscale x 8 x i16> @masked_load_nxv8i16_i64_mask(ptr %a, <vscale x 2 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_load_nxv8i16_i64_mask:
+; CHECK: ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %mask.stretched = call <vscale x 8 x i1> @llvm.vector.stretch.nxv8i1.nxv2i1(<vscale x 2 x i1> %mask)
+ %load = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16(ptr %a, i32 2, <vscale x 8 x i1> %mask.stretched, <vscale x 8 x i16> poison)
+ ret <vscale x 8 x i16> %load
+}
+
+define <vscale x 8 x i16> @masked_load_nxv8i16_i64_mask_multi_stretch(ptr %a, <vscale x 2 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_load_nxv8i16_i64_mask_multi_stretch:
+; CHECK: ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %mask.stretched = call <vscale x 4 x i1> @llvm.vector.stretch.nxv4i1.nxv2i1(<vscale x 2 x i1> %mask)
+ %mask.stretched2 = call <vscale x 8 x i1> @llvm.vector.stretch.nxv8i1.nxv4i1(<vscale x 4 x i1> %mask.stretched)
+ %load = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16(ptr %a, i32 2, <vscale x 8 x i1> %mask.stretched2, <vscale x 8 x i16> poison)
+ ret <vscale x 8 x i16> %load
+}
+
+define <vscale x 8 x i16> @masked_load_nxv8i16_i32_mask(ptr %a, <vscale x 4 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_load_nxv8i16_i32_mask:
+; CHECK: ld1w { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %mask.stretched = call <vscale x 8 x i1> @llvm.vector.stretch.nxv8i1.nxv4i1(<vscale x 4 x i1> %mask)
+ %load = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16(ptr %a, i32 2, <vscale x 8 x i1> %mask.stretched, <vscale x 8 x i16> poison)
+ ret <vscale x 8 x i16> %load
+}
+
+; TODO: This is sub-optimal, we should use an extending ld1b z.d.
+define <vscale x 1 x i8> @masked_load_nxv1i8(ptr %a, <vscale x 1 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_load_nxv1i8:
+; CHECK: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: uzp1 p1.d, p1.d, p1.d
+; CHECK-NEXT: uzp1 p0.s, p0.s, p1.s
+; CHECK-NEXT: uzp1 p1.s, p1.s, p1.s
+; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h
+; CHECK-NEXT: uzp1 p1.h, p1.h, p1.h
+; CHECK-NEXT: uzp1 p0.b, p0.b, p1.b
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %load = call <vscale x 1 x i8> @llvm.masked.load.nxv1i8(ptr %a, i32 1, <vscale x 1 x i1> %mask, <vscale x 1 x i8> poison)
+ ret <vscale x 1 x i8> %load
+}
+
define <vscale x 16 x i8> @masked_load_nxv16i8(ptr %a, <vscale x 16 x i1> %mask) nounwind {
; CHECK-LABEL: masked_load_nxv16i8:
; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
@@ -36,6 +143,43 @@ define <vscale x 16 x i8> @masked_load_nxv16i8(ptr %a, <vscale x 16 x i1> %mask)
ret <vscale x 16 x i8> %load
}
+define <vscale x 16 x i8> @masked_load_nxv16i8_i64_mask(ptr %a, <vscale x 2 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_load_nxv16i8_i64_mask:
+; CHECK: ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %mask.stretched = call <vscale x 16 x i1> @llvm.vector.stretch.nxv16i1.nxv2i1(<vscale x 2 x i1> %mask)
+ %load = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8(ptr %a, i32 1, <vscale x 16 x i1> %mask.stretched, <vscale x 16 x i8> poison)
+ ret <vscale x 16 x i8> %load
+}
+
+define <vscale x 16 x i8> @masked_load_nxv16i8_i32_mask(ptr %a, <vscale x 4 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_load_nxv16i8_i32_mask:
+; CHECK: ld1w { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %mask.stretched = call <vscale x 16 x i1> @llvm.vector.stretch.nxv16i1.nxv4i1(<vscale x 4 x i1> %mask)
+ %load = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8(ptr %a, i32 1, <vscale x 16 x i1> %mask.stretched, <vscale x 16 x i8> poison)
+ ret <vscale x 16 x i8> %load
+}
+
+define <vscale x 16 x i8> @masked_load_nxv16i8_i16_mask(ptr %a, <vscale x 8 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_load_nxv16i8_i16_mask:
+; CHECK: ld1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %mask.stretched = call <vscale x 16 x i1> @llvm.vector.stretch.nxv16i1.nxv8i1(<vscale x 8 x i1> %mask)
+ %load = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8(ptr %a, i32 1, <vscale x 16 x i1> %mask.stretched, <vscale x 16 x i8> poison)
+ ret <vscale x 16 x i8> %load
+}
+
+define <vscale x 1 x double> @masked_load_nxv1f64(ptr %a, <vscale x 1 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_load_nxv1f64:
+; CHECK: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %load = call <vscale x 1 x double> @llvm.masked.load.nxv1f64(ptr %a, i32 8, <vscale x 1 x i1> %mask, <vscale x 1 x double> poison)
+ ret <vscale x 1 x double> %load
+}
+
define <vscale x 2 x double> @masked_load_nxv2f64(ptr %a, <vscale x 2 x i1> %mask) nounwind {
; CHECK-LABEL: masked_load_nxv2f64:
; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
@@ -44,6 +188,16 @@ define <vscale x 2 x double> @masked_load_nxv2f64(ptr %a, <vscale x 2 x i1> %mas
ret <vscale x 2 x double> %load
}
+define <vscale x 1 x float> @masked_load_nxv1f32(ptr %a, <vscale x 1 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_load_nxv1f32:
+; CHECK: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: ld1w { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %load = call <vscale x 1 x float> @llvm.masked.load.nxv1f32(ptr %a, i32 4, <vscale x 1 x i1> %mask, <vscale x 1 x float> poison)
+ ret <vscale x 1 x float> %load
+}
+
define <vscale x 2 x float> @masked_load_nxv2f32(ptr %a, <vscale x 2 x i1> %mask) nounwind {
; CHECK-LABEL: masked_load_nxv2f32:
; CHECK-NEXT: ld1w { z0.d }, p0/z, [x0]
@@ -52,6 +206,16 @@ define <vscale x 2 x float> @masked_load_nxv2f32(ptr %a, <vscale x 2 x i1> %mask
ret <vscale x 2 x float> %load
}
+define <vscale x 1 x half> @masked_load_nxv1f16(ptr %a, <vscale x 1 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_load_nxv1f16:
+; CHECK: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: ld1h { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %load = call <vscale x 1 x half> @llvm.masked.load.nxv1f16(ptr %a, i32 2, <vscale x 1 x i1> %mask, <vscale x 1 x half> poison)
+ ret <vscale x 1 x half> %load
+}
+
define <vscale x 2 x half> @masked_load_nxv2f16(ptr %a, <vscale x 2 x i1> %mask) nounwind {
; CHECK-LABEL: masked_load_nxv2f16:
; CHECK-NEXT: ld1h { z0.d }, p0/z, [x0]
@@ -60,6 +224,16 @@ define <vscale x 2 x half> @masked_load_nxv2f16(ptr %a, <vscale x 2 x i1> %mask)
ret <vscale x 2 x half> %load
}
+define <vscale x 1 x bfloat> @masked_load_nxv1bf16(ptr %a, <vscale x 1 x i1> %mask) nounwind #0 {
+; CHECK-LABEL: masked_load_nxv1bf16:
+; CHECK: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: ld1h { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %load = call <vscale x 1 x bfloat> @llvm.masked.load.nxv1bf16(ptr %a, i32 2, <vscale x 1 x i1> %mask, <vscale x 1 x bfloat> poison)
+ ret <vscale x 1 x bfloat> %load
+}
+
define <vscale x 2 x bfloat> @masked_load_nxv2bf16(ptr %a, <vscale x 2 x i1> %mask) nounwind #0 {
; CHECK-LABEL: masked_load_nxv2bf16:
; CHECK-NEXT: ld1h { z0.d }, p0/z, [x0]
@@ -126,10 +300,30 @@ define <vscale x 2 x i16> @masked_load_nxv2i16(ptr noalias %in, <vscale x 2 x i1
ret <vscale x 2 x i16> %wide.load
}
+define <vscale x 4 x i16> @masked_load_nxv4i16_i64_mask(ptr noalias %a, <vscale x 2 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_load_nxv4i16_i64_mask:
+; CHECK: trn1 p0.s, p0.s, p0.s
+; CHECK-NEXT: ld1h { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %mask.stretched = call <vscale x 4 x i1> @llvm.vector.stretch.nxv4i1.nxv2i1(<vscale x 2 x i1> %mask)
+ %load = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16(ptr %a, i32 2, <vscale x 4 x i1> %mask.stretched, <vscale x 4 x i16> poison)
+ ret <vscale x 4 x i16> %load
+}
+
;
; Masked Stores
;
+define void @masked_store_nxv1i64(ptr %a, <vscale x 1 x i64> %val, <vscale x 1 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_store_nxv1i64:
+; CHECK: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: st1d { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ call void @llvm.masked.store.nxv1i64(<vscale x 1 x i64> %val, ptr %a, i32 8, <vscale x 1 x i1> %mask)
+ ret void
+}
+
define void @masked_store_nxv2i64(ptr %a, <vscale x 2 x i64> %val, <vscale x 2 x i1> %mask) nounwind {
; CHECK-LABEL: masked_store_nxv2i64:
; CHECK-NEXT: st1d { z0.d }, p0, [x0]
@@ -138,6 +332,27 @@ define void @masked_store_nxv2i64(ptr %a, <vscale x 2 x i64> %val, <vscale x 2 x
ret void
}
+; TODO: This is sub-optimal, we should use a truncating store from d to s
+define void @masked_store_nxv1i32(ptr %a, <vscale x 1 x i32> %val, <vscale x 1 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_store_nxv1i32:
+; CHECK: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: uzp1 p1.d, p1.d, p1.d
+; CHECK-NEXT: uzp1 p0.s, p0.s, p1.s
+; CHECK-NEXT: st1w { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+ call void @llvm.masked.store.nxv1i32(<vscale x 1 x i32> %val, ptr %a, i32 4, <vscale x 1 x i1> %mask)
+ ret void
+}
+
+define void @masked_store_nxv2i32(ptr %a, <vscale x 2 x i32> %val, <vscale x 2 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_store_nxv2i32:
+; CHECK: st1w { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ call void @llvm.masked.store.nxv2i32(<vscale x 2 x i32> %val, ptr %a, i32 4, <vscale x 2 x i1> %mask)
+ ret void
+}
+
define void @masked_store_nxv4i32(ptr %a, <vscale x 4 x i32> %val, <vscale x 4 x i1> %mask) nounwind {
; CHECK-LABEL: masked_store_nxv4i32:
; CHECK-NEXT: st1w { z0.s }, p0, [x0]
@@ -146,6 +361,30 @@ define void @masked_store_nxv4i32(ptr %a, <vscale x 4 x i32> %val, <vscale x 4 x
ret void
}
+define void @masked_store_nxv4i32_i64_mask(ptr %a, <vscale x 4 x i32> %val, <vscale x 2 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_store_nxv4i32_i64_mask:
+; CHECK: st1d { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %mask.stretched = call <vscale x 4 x i1> @llvm.vector.stretch.nxv4i1.nxv2i1(<vscale x 2 x i1> %mask)
+ call void @llvm.masked.store.nxv4i32(<vscale x 4 x i32> %val, ptr %a, i32 4, <vscale x 4 x i1> %mask.stretched)
+ ret void
+}
+
+; TODO: This is sub-optimal, we should use a truncating store from d to h
+define void @masked_store_nxv1i16(ptr %a, <vscale x 1 x i16> %val, <vscale x 1 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_store_nxv1i16:
+; CHECK: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: uzp1 p1.d, p1.d, p1.d
+; CHECK-NEXT: uzp1 p0.s, p0.s, p1.s
+; CHECK-NEXT: uzp1 p1.s, p1.s, p1.s
+; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h
+; CHECK-NEXT: st1h { z0.h }, p0, [x0]
+; CHECK-NEXT: ret
+ call void @llvm.masked.store.nxv1i16(<vscale x 1 x i16> %val, ptr %a, i32 2, <vscale x 1 x i1> %mask)
+ ret void
+}
+
define void @masked_store_nxv8i16(ptr %a, <vscale x 8 x i16> %val, <vscale x 8 x i1> %mask) nounwind {
; CHECK-LABEL: masked_store_nxv8i16:
; CHECK-NEXT: st1h { z0.h }, p0, [x0]
@@ -154,6 +393,41 @@ define void @masked_store_nxv8i16(ptr %a, <vscale x 8 x i16> %val, <vscale x 8 x
ret void
}
+define void @masked_store_nxv8i16_i64_mask(ptr %a, <vscale x 8 x i16> %val, <vscale x 2 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_store_nxv8i16_i64_mask:
+; CHECK: st1d { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %mask.stretched = call <vscale x 8 x i1> @llvm.vector.stretch.nxv8i1.nxv2i1(<vscale x 2 x i1> %mask)
+ call void @llvm.masked.store.nxv8i16(<vscale x 8 x i16> %val, ptr %a, i32 2, <vscale x 8 x i1> %mask.stretched)
+ ret void
+}
+
+define void @masked_store_nxv8i16_i32_mask(ptr %a, <vscale x 8 x i16> %val, <vscale x 4 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_store_nxv8i16_i32_mask:
+; CHECK: st1w { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %mask.stretched = call <vscale x 8 x i1> @llvm.vector.stretch.nxv8i1.nxv4i1(<vscale x 4 x i1> %mask)
+ call void @llvm.masked.store.nxv8i16(<vscale x 8 x i16> %val, ptr %a, i32 2, <vscale x 8 x i1> %mask.stretched)
+ ret void
+}
+
+; TODO: This is sub-optimal, we should use a truncating store from d to b
+define void @masked_store_nxv1i8(ptr %a, <vscale x 1 x i8> %val, <vscale x 1 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_store_nxv1i8:
+; CHECK: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: uzp1 p1.d, p1.d, p1.d
+; CHECK-NEXT: uzp1 p0.s, p0.s, p1.s
+; CHECK-NEXT: uzp1 p1.s, p1.s, p1.s
+; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h
+; CHECK-NEXT: uzp1 p1.h, p1.h, p1.h
+; CHECK-NEXT: uzp1 p0.b, p0.b, p1.b
+; CHECK-NEXT: st1b { z0.b }, p0, [x0]
+; CHECK-NEXT: ret
+ call void @llvm.masked.store.nxv1i8(<vscale x 1 x i8> %val, ptr %a, i32 1, <vscale x 1 x i1> %mask)
+ ret void
+}
+
define void @masked_store_nxv16i8(ptr %a, <vscale x 16 x i8> %val, <vscale x 16 x i1> %mask) nounwind {
; CHECK-LABEL: masked_store_nxv16i8:
; CHECK-NEXT: st1b { z0.b }, p0, [x0]
@@ -162,6 +436,43 @@ define void @masked_store_nxv16i8(ptr %a, <vscale x 16 x i8> %val, <vscale x 16
ret void
}
+define void @masked_store_nxv16i8_i64_mask(ptr %a, <vscale x 16 x i8> %val, <vscale x 2 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_store_nxv16i8_i64_mask:
+; CHECK: st1d { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %mask.stretched = call <vscale x 16 x i1> @llvm.vector.stretch.nxv16i1.nxv2i1(<vscale x 2 x i1> %mask)
+ call void @llvm.masked.store.nxv16i8(<vscale x 16 x i8> %val, ptr %a, i32 1, <vscale x 16 x i1> %mask.stretched)
+ ret void
+}
+
+define void @masked_store_nxv16i8_i32_mask(ptr %a, <vscale x 16 x i8> %val, <vscale x 4 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_store_nxv16i8_i32_mask:
+; CHECK: st1w { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %mask.stretched = call <vscale x 16 x i1> @llvm.vector.stretch.nxv16i1.nxv4i1(<vscale x 4 x i1> %mask)
+ call void @llvm.masked.store.nxv16i8(<vscale x 16 x i8> %val, ptr %a, i32 1, <vscale x 16 x i1> %mask.stretched)
+ ret void
+}
+
+define void @masked_store_nxv16i8_i16_mask(ptr %a, <vscale x 16 x i8> %val, <vscale x 8 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_store_nxv16i8_i16_mask:
+; CHECK: st1h { z0.h }, p0, [x0]
+; CHECK-NEXT: ret
+ %mask.stretched = call <vscale x 16 x i1> @llvm.vector.stretch.nxv16i1.nxv8i1(<vscale x 8 x i1> %mask)
+ call void @llvm.masked.store.nxv16i8(<vscale x 16 x i8> %val, ptr %a, i32 1, <vscale x 16 x i1> %mask.stretched)
+ ret void
+}
+
+define void @masked_store_nxv1f64(ptr %a, <vscale x 1 x double> %val, <vscale x 1 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_store_nxv1f64:
+; CHECK: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: st1d { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ call void @llvm.masked.store.nxv1f64(<vscale x 1 x double> %val, ptr %a, i32 8, <vscale x 1 x i1> %mask)
+ ret void
+}
+
define void @masked_store_nxv2f64(ptr %a, <vscale x 2 x double> %val, <vscale x 2 x i1> %mask) nounwind {
; CHECK-LABEL: masked_store_nxv2f64:
; CHECK-NEXT: st1d { z0.d }, p0, [x0]
@@ -170,6 +481,16 @@ define void @masked_store_nxv2f64(ptr %a, <vscale x 2 x double> %val, <vscale x
ret void
}
+define void @masked_store_nxv1f32(ptr %a, <vscale x 1 x float> %val, <vscale x 1 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_store_nxv1f32:
+; CHECK: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: st1w { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ call void @llvm.masked.store.nxv1f32(<vscale x 1 x float> %val, ptr %a, i32 4, <vscale x 1 x i1> %mask)
+ ret void
+}
+
define void @masked_store_nxv2f32(ptr %a, <vscale x 2 x float> %val, <vscale x 2 x i1> %mask) nounwind {
; CHECK-LABEL: masked_store_nxv2f32:
; CHECK-NEXT: st1w { z0.d }, p0, [x0]
@@ -178,6 +499,16 @@ define void @masked_store_nxv2f32(ptr %a, <vscale x 2 x float> %val, <vscale x 2
ret void
}
+define void @masked_store_nxv1f16(ptr %a, <vscale x 1 x half> %val, <vscale x 1 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_store_nxv1f16:
+; CHECK: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: st1h { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ call void @llvm.masked.store.nxv1f16(<vscale x 1 x half> %val, ptr %a, i32 4, <vscale x 1 x i1> %mask)
+ ret void
+}
+
define void @masked_store_nxv2f16(ptr %a, <vscale x 2 x half> %val, <vscale x 2 x i1> %mask) nounwind {
; CHECK-LABEL: masked_store_nxv2f16:
; CHECK-NEXT: st1h { z0.d }, p0, [x0]
@@ -194,6 +525,15 @@ define void @masked_store_nxv4f32(ptr %a, <vscale x 4 x float> %val, <vscale x 4
ret void
}
+define void @masked_store_nxv4f32_i64_mask(ptr %a, <vscale x 4 x float> %val, <vscale x 2 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_store_nxv4f32_i64_mask:
+; CHECK: st1d { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %mask.stretched = call <vscale x 4 x i1> @llvm.vector.stretch.nxv4i1.nxv2i1(<vscale x 2 x i1> %mask)
+ call void @llvm.masked.store.nxv4f32(<vscale x 4 x float> %val, ptr %a, i32 4, <vscale x 4 x i1> %mask.stretched)
+ ret void
+}
+
define void @masked_store_nxv4f16(ptr %a, <vscale x 4 x half> %val, <vscale x 4 x i1> %mask) nounwind {
; CHECK-LABEL: masked_store_nxv4f16:
; CHECK-NEXT: st1h { z0.s }, p0, [x0]
@@ -210,6 +550,34 @@ define void @masked_store_nxv8f16(ptr %a, <vscale x 8 x half> %val, <vscale x 8
ret void
}
+define void @masked_store_nxv8f16_i64_mask(ptr %a, <vscale x 8 x half> %val, <vscale x 2 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_store_nxv8f16_i64_mask:
+; CHECK: st1d { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %mask.stretched = call <vscale x 8 x i1> @llvm.vector.stretch.nxv8i1.nxv2i1(<vscale x 2 x i1> %mask)
+ call void @llvm.masked.store.nxv8f16(<vscale x 8 x half> %val, ptr %a, i32 2, <vscale x 8 x i1> %mask.stretched)
+ ret void
+}
+
+define void @masked_store_nxv8f16_i32_mask(ptr %a, <vscale x 8 x half> %val, <vscale x 4 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_store_nxv8f16_i32_mask:
+; CHECK: st1w { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %mask.stretched = call <vscale x 8 x i1> @llvm.vector.stretch.nxv8i1.nxv4i1(<vscale x 4 x i1> %mask)
+ call void @llvm.masked.store.nxv8f16(<vscale x 8 x half> %val, ptr %a, i32 2, <vscale x 8 x i1> %mask.stretched)
+ ret void
+}
+
+define void @masked_store_nxv1bf16(ptr %a, <vscale x 1 x bfloat> %val, <vscale x 1 x i1> %mask) nounwind #0 {
+; CHECK-LABEL: masked_store_nxv1bf16:
+; CHECK: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: st1h { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ call void @llvm.masked.store.nxv1bf16(<vscale x 1 x bfloat> %val, ptr %a, i32 2, <vscale x 1 x i1> %mask)
+ ret void
+}
+
define void @masked_store_nxv2bf16(ptr %a, <vscale x 2 x bfloat> %val, <vscale x 2 x i1> %mask) nounwind #0 {
; CHECK-LABEL: masked_store_nxv2bf16:
; CHECK-NEXT: st1h { z0.d }, p0, [x0]
@@ -234,12 +602,40 @@ define void @masked_store_nxv8bf16(ptr %a, <vscale x 8 x bfloat> %val, <vscale x
ret void
}
+define void @masked_store_nxv8bf16_i64_mask(ptr %a, <vscale x 8 x bfloat> %val, <vscale x 2 x i1> %mask) nounwind #0 {
+; CHECK-LABEL: masked_store_nxv8bf16_i64_mask:
+; CHECK: st1d { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %mask.stretched = call <vscale x 8 x i1> @llvm.vector.stretch.nxv8i1.nxv2i1(<vscale x 2 x i1> %mask)
+ call void @llvm.masked.store.nxv8bf16(<vscale x 8 x bfloat> %val, ptr %a, i32 2, <vscale x 8 x i1> %mask.stretched)
+ ret void
+}
+
+define void @masked_store_nxv8bf16_i32_mask(ptr %a, <vscale x 8 x bfloat> %val, <vscale x 4 x i1> %mask) nounwind #0 {
+; CHECK-LABEL: masked_store_nxv8bf16_i32_mask:
+; CHECK: st1w { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %mask.stretched = call <vscale x 8 x i1> @llvm.vector.stretch.nxv8i1.nxv4i1(<vscale x 4 x i1> %mask)
+ call void @llvm.masked.store.nxv8bf16(<vscale x 8 x bfloat> %val, ptr %a, i32 2, <vscale x 8 x i1> %mask.stretched)
+ ret void
+}
+
;
; Masked load store of pointer data type
;
; Pointer of integer type
+define <vscale x 1 x ptr> @masked.load.nxv1p0(ptr %vector_ptr, <vscale x 1 x i1> %mask) nounwind {
+; CHECK-LABEL: masked.load.nxv1p0:
+; CHECK: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %v = call <vscale x 1 x ptr> @llvm.masked.load.nxv1p0(ptr %vector_ptr, i32 8, <vscale x 1 x i1> %mask, <vscale x 1 x ptr> poison)
+ ret <vscale x 1 x ptr> %v
+}
+
define <vscale x 2 x ptr> @masked.load.nxv2p0i8(ptr %vector_ptr, <vscale x 2 x i1> %mask) nounwind {
; CHECK-LABEL: masked.load.nxv2p0i8:
; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
@@ -302,6 +698,16 @@ define <vscale x 2 x ptr> @masked.load.nxv2p0f64(ptr %vector_ptr, <vscale x 2 x
; Pointer of array type
+define void @masked.store.nxv1p0(<vscale x 1 x ptr> %data, ptr %vector_ptr, <vscale x 1 x i1> %mask) nounwind {
+; CHECK-LABEL: masked.store.nxv1p0:
+; CHECK: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: st1d { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ call void @llvm.masked.store.nxv1p0(<vscale x 1 x ptr> %data, ptr %vector_ptr, i32 8, <vscale x 1 x i1> %mask)
+ ret void
+}
+
define void @masked.store.nxv2p0a64i16(<vscale x 2 x ptr> %data, ptr %vector_ptr, <vscale x 2 x i1> %mask) nounwind {
; CHECK-LABEL: masked.store.nxv2p0a64i16:
; CHECK-NEXT: st1d { z0.d }, p0, [x0]
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-ldst-sext.ll b/llvm/test/CodeGen/AArch64/sve-masked-ldst-sext.ll
index 5c506ab20e6f4..b7f3528be07dc 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-ldst-sext.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-ldst-sext.ll
@@ -15,6 +15,19 @@ define <vscale x 2 x i64> @masked_sload_nxv2i8(ptr %a, <vscale x 2 x i1> %mask)
ret <vscale x 2 x i64> %ext
}
+; Stretched masks cannot be folded for extending loads.
+define <vscale x 2 x i64> @masked_sload_nxv2i8_i128_mask(ptr %a, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_sload_nxv2i8_i128_mask:
+; CHECK: // %bb.0:
+; CHECK-NEXT: trn1 p0.d, p0.d, p0.d
+; CHECK-NEXT: ld1sb { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %mask.stretched = call <vscale x 2 x i1> @llvm.vector.stretch.nxv2i1.nxv1i1(<vscale x 1 x i1> %mask)
+ %load = call <vscale x 2 x i8> @llvm.masked.load.nxv2i8(ptr %a, i32 1, <vscale x 2 x i1> %mask.stretched, <vscale x 2 x i8> poison)
+ %ext = sext <vscale x 2 x i8> %load to <vscale x 2 x i64>
+ ret <vscale x 2 x i64> %ext
+}
+
define <vscale x 2 x i64> @masked_sload_nxv2i16(ptr %a, <vscale x 2 x i1> %mask) {
; CHECK-LABEL: masked_sload_nxv2i16:
; CHECK: // %bb.0:
@@ -45,6 +58,19 @@ define <vscale x 4 x i32> @masked_sload_nxv4i8(ptr %a, <vscale x 4 x i1> %mask)
ret <vscale x 4 x i32> %ext
}
+; Stretched masks cannot be folded for extending loads.
+define <vscale x 4 x i32> @masked_sload_nxv4i8_i64_mask(ptr %a, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_sload_nxv4i8_i64_mask:
+; CHECK: // %bb.0:
+; CHECK-NEXT: trn1 p0.s, p0.s, p0.s
+; CHECK-NEXT: ld1sb { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %mask.stretched = call <vscale x 4 x i1> @llvm.vector.stretch.nxv4i1.nxv2i1(<vscale x 2 x i1> %mask)
+ %load = call <vscale x 4 x i8> @llvm.masked.load.nxv4i8(ptr %a, i32 1, <vscale x 4 x i1> %mask.stretched, <vscale x 4 x i8> poison)
+ %ext = sext <vscale x 4 x i8> %load to <vscale x 4 x i32>
+ ret <vscale x 4 x i32> %ext
+}
+
define <vscale x 4 x i32> @masked_sload_nxv4i16(ptr %a, <vscale x 4 x i1> %mask) {
; CHECK-LABEL: masked_sload_nxv4i16:
; CHECK: // %bb.0:
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-ldst-trunc.ll b/llvm/test/CodeGen/AArch64/sve-masked-ldst-trunc.ll
index 63e7d9e27eab9..29b4ce841a393 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-ldst-trunc.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-ldst-trunc.ll
@@ -40,6 +40,18 @@ define void @masked_trunc_store_nxv4i8(ptr %a, <vscale x 4 x i32> %val, ptr %b,
ret void
}
+; Stretched masks cannot be folded for truncating stores.
+define void @masked_trunc_store_nxv4i8_i64_mask(ptr %a, <vscale x 4 x i32> %val, ptr %b, <vscale x 2 x i1> %mask) nounwind {
+; CHECK-LABEL: masked_trunc_store_nxv4i8_i64_mask:
+; CHECK: trn1 p0.s, p0.s, p0.s
+; CHECK-NEXT: st1b { z0.s }, p0, [x1]
+; CHECK-NEXT: ret
+ %mask.stretched = call <vscale x 4 x i1> @llvm.vector.stretch.nxv4i1.nxv2i1(<vscale x 2 x i1> %mask)
+ %trunc = trunc <vscale x 4 x i32> %val to <vscale x 4 x i8>
+ call void @llvm.masked.store.nxv4i8(<vscale x 4 x i8> %trunc, ptr %b, i32 4, <vscale x 4 x i1> %mask.stretched)
+ ret void
+}
+
define void @masked_trunc_store_nxv4i16(ptr %a, <vscale x 4 x i32> %val, ptr %b, <vscale x 4 x i1> %mask) nounwind {
; CHECK-LABEL: masked_trunc_store_nxv4i16:
; CHECK-NEXT: st1h { z0.s }, p0, [x1]
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-segment-gather-scatter.ll b/llvm/test/CodeGen/AArch64/sve-masked-segment-gather-scatter.ll
new file mode 100644
index 0000000000000..bd4df0df174b2
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-masked-segment-gather-scatter.ll
@@ -0,0 +1,175 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve2p1 -asm-verbose=0 < %s | FileCheck %s
+
+define <vscale x 2 x i64> @segment_gather_64_i32(<vscale x 2 x ptr> %wide.ptrs, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: segment_gather_64_i32:
+; CHECK: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [z0.d]
+; CHECK-NEXT: uunpklo z0.d, z0.s
+; CHECK-NEXT: ret
+ %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(
+ <vscale x 2 x ptr> %wide.ptrs, i64 0)
+ %r = call <vscale x 2 x i32> @llvm.masked.segment.gather.nxv2i32.nxv1p0(
+ <vscale x 1 x ptr> align 8 %ptrs, <vscale x 1 x i1> %mask,
+ <vscale x 2 x i32> poison)
+ %r.legal = zext <vscale x 2 x i32> %r to <vscale x 2 x i64>
+ ret <vscale x 2 x i64> %r.legal
+}
+
+define <vscale x 4 x i32> @segment_gather_64_i16(<vscale x 2 x ptr> %wide.ptrs, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: segment_gather_64_i16:
+; CHECK: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [z0.d]
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: ret
+ %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(
+ <vscale x 2 x ptr> %wide.ptrs, i64 0)
+ %r = call <vscale x 4 x i16> @llvm.masked.segment.gather.nxv4i16.nxv1p0(
+ <vscale x 1 x ptr> align 8 %ptrs, <vscale x 1 x i1> %mask,
+ <vscale x 4 x i16> poison)
+ %r.legal = zext <vscale x 4 x i16> %r to <vscale x 4 x i32>
+ ret <vscale x 4 x i32> %r.legal
+}
+
+define <vscale x 4 x i32> @segment_gather_64_i16_load_ptrs(ptr %ptrs.addr, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: segment_gather_64_i16_load_ptrs:
+; CHECK: rdvl x8, #1
+; CHECK-NEXT: pfalse p2.b
+; CHECK-NEXT: lsr x8, x8, #4
+; CHECK-NEXT: uzp1 p0.d, p0.d, p2.d
+; CHECK-NEXT: whilelo p1.d, xzr, x8
+; CHECK-NEXT: ld1d { z0.d }, p1/z, [x0]
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [z0.d]
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: ret
+ %ptrs = load <vscale x 1 x ptr>, ptr %ptrs.addr
+ %r = call <vscale x 4 x i16> @llvm.masked.segment.gather.nxv4i16.nxv1p0(
+ <vscale x 1 x ptr> align 8 %ptrs, <vscale x 1 x i1> %mask,
+ <vscale x 4 x i16> poison)
+ %r.legal = zext <vscale x 4 x i16> %r to <vscale x 4 x i32>
+ ret <vscale x 4 x i32> %r.legal
+}
+
+define <vscale x 4 x half> @segment_gather_64_f16(<vscale x 2 x ptr> %wide.ptrs, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: segment_gather_64_f16:
+; CHECK: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [z0.d]
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: ret
+ %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(
+ <vscale x 2 x ptr> %wide.ptrs, i64 0)
+ %r = call <vscale x 4 x half> @llvm.masked.segment.gather.nxv4f16.nxv1p0(
+ <vscale x 1 x ptr> align 8 %ptrs, <vscale x 1 x i1> %mask,
+ <vscale x 4 x half> poison)
+ ret <vscale x 4 x half> %r
+}
+
+define <vscale x 4 x half> @segment_gather_64_f16_load_ptrs(ptr %ptrs.addr, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: segment_gather_64_f16_load_ptrs:
+; CHECK: rdvl x8, #1
+; CHECK-NEXT: pfalse p2.b
+; CHECK-NEXT: lsr x8, x8, #4
+; CHECK-NEXT: uzp1 p0.d, p0.d, p2.d
+; CHECK-NEXT: whilelo p1.d, xzr, x8
+; CHECK-NEXT: ld1d { z0.d }, p1/z, [x0]
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [z0.d]
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: ret
+ %ptrs = load <vscale x 1 x ptr>, ptr %ptrs.addr
+ %r = call <vscale x 4 x half> @llvm.masked.segment.gather.nxv4f16.nxv1p0(
+ <vscale x 1 x ptr> align 8 %ptrs, <vscale x 1 x i1> %mask,
+ <vscale x 4 x half> poison)
+ ret <vscale x 4 x half> %r
+}
+
+define void @segment_scatter_64_i32(<vscale x 2 x i64> %wide.data, <vscale x 2 x ptr> %wide.ptrs, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: segment_scatter_64_i32:
+; CHECK: pfalse p1.b
+; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: st1d { z0.d }, p0, [z1.d]
+; CHECK-NEXT: ret
+ %data = trunc <vscale x 2 x i64> %wide.data to <vscale x 2 x i32>
+ %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(
+ <vscale x 2 x ptr> %wide.ptrs, i64 0)
+ call void @llvm.masked.segment.scatter.nxv2i32.nxv1p0(
+ <vscale x 2 x i32> %data, <vscale x 1 x ptr> align 8 %ptrs,
+ <vscale x 1 x i1> %mask)
+ ret void
+}
+
+define void @segment_scatter_64_i16(<vscale x 4 x i32> %wide.data, <vscale x 2 x ptr> %wide.ptrs, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: segment_scatter_64_i16:
+; CHECK: pfalse p1.b
+; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: st1d { z0.d }, p0, [z1.d]
+; CHECK-NEXT: ret
+ %data = trunc <vscale x 4 x i32> %wide.data to <vscale x 4 x i16>
+ %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(
+ <vscale x 2 x ptr> %wide.ptrs, i64 0)
+ call void @llvm.masked.segment.scatter.nxv4i16.nxv1p0(
+ <vscale x 4 x i16> %data, <vscale x 1 x ptr> align 8 %ptrs,
+ <vscale x 1 x i1> %mask)
+ ret void
+}
+
+define void @segment_scatter_64_f16(<vscale x 4 x half> %data, <vscale x 2 x ptr> %wide.ptrs, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: segment_scatter_64_f16:
+; CHECK: pfalse p1.b
+; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: st1d { z0.d }, p0, [z1.d]
+; CHECK-NEXT: ret
+ %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(
+ <vscale x 2 x ptr> %wide.ptrs, i64 0)
+ call void @llvm.masked.segment.scatter.nxv4f16.nxv1p0(
+ <vscale x 4 x half> %data, <vscale x 1 x ptr> align 8 %ptrs,
+ <vscale x 1 x i1> %mask)
+ ret void
+}
+
+define <vscale x 4 x i32> @segment_gather_128_i32(<vscale x 2 x ptr> %wide.ptrs, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: segment_gather_128_i32:
+; CHECK: zip1 z0.d, z0.d, z0.d
+; CHECK-NEXT: ld1q { z0.q }, p0/z, [z0.d]
+; CHECK-NEXT: ret
+ %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(
+ <vscale x 2 x ptr> %wide.ptrs, i64 0)
+ %r = call <vscale x 4 x i32> @llvm.masked.segment.gather.nxv4i32.nxv1p0(
+ <vscale x 1 x ptr> align 16 %ptrs, <vscale x 1 x i1> %mask,
+ <vscale x 4 x i32> zeroinitializer)
+ ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x i32> @segment_gather_128_i32_load_ptrs(ptr %ptrs.addr, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: segment_gather_128_i32_load_ptrs:
+; CHECK: rdvl x8, #1
+; CHECK-NEXT: lsr x8, x8, #4
+; CHECK-NEXT: whilelo p1.d, xzr, x8
+; CHECK-NEXT: ld1d { z0.d }, p1/z, [x0]
+; CHECK-NEXT: zip1 z0.d, z0.d, z0.d
+; CHECK-NEXT: ld1q { z0.q }, p0/z, [z0.d]
+; CHECK-NEXT: ret
+ %ptrs = load <vscale x 1 x ptr>, ptr %ptrs.addr
+ %r = call <vscale x 4 x i32> @llvm.masked.segment.gather.nxv4i32.nxv1p0(
+ <vscale x 1 x ptr> align 16 %ptrs, <vscale x 1 x i1> %mask,
+ <vscale x 4 x i32> zeroinitializer)
+ ret <vscale x 4 x i32> %r
+}
+
+define void @segment_scatter_128_i32(<vscale x 4 x i32> %data, <vscale x 2 x ptr> %wide.ptrs, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: segment_scatter_128_i32:
+; CHECK: zip1 z1.d, z1.d, z1.d
+; CHECK-NEXT: st1q { z0.q }, p0, [z1.d]
+; CHECK-NEXT: ret
+ %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(
+ <vscale x 2 x ptr> %wide.ptrs, i64 0)
+ call void @llvm.masked.segment.scatter.nxv4i32.nxv1p0(
+ <vscale x 4 x i32> %data, <vscale x 1 x ptr> align 16 %ptrs,
+ <vscale x 1 x i1> %mask)
+ ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/sve-select.ll b/llvm/test/CodeGen/AArch64/sve-select.ll
index 513613b741cd4..7262e6514cbc2 100644
--- a/llvm/test/CodeGen/AArch64/sve-select.ll
+++ b/llvm/test/CodeGen/AArch64/sve-select.ll
@@ -277,6 +277,36 @@ define <vscale x 16 x i8> @sel_nxv16i8(<vscale x 16 x i1> %p, <vscale x 16 x i8>
ret <vscale x 16 x i8> %sel
}
+define <vscale x 16 x i8> @sel_nxv16i8_i64_mask(<vscale x 2 x i1> %mask, <vscale x 16 x i8> %dst, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: sel_nxv16i8_i64_mask:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov z0.d, p0/m, z1.d
+; CHECK-NEXT: ret
+ %p.stretched = call <vscale x 16 x i1> @llvm.vector.stretch.nxv16i1.nxv2i1(<vscale x 2 x i1> %mask)
+ %sel = select <vscale x 16 x i1> %p.stretched, <vscale x 16 x i8> %a, <vscale x 16 x i8> %dst
+ ret <vscale x 16 x i8> %sel
+}
+
+define <vscale x 16 x i8> @sel_nxv16i8_i32_mask(<vscale x 4 x i1> %mask, <vscale x 16 x i8> %dst, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: sel_nxv16i8_i32_mask:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov z0.s, p0/m, z1.s
+; CHECK-NEXT: ret
+ %p.stretched = call <vscale x 16 x i1> @llvm.vector.stretch.nxv16i1.nxv4i1(<vscale x 4 x i1> %mask)
+ %sel = select <vscale x 16 x i1> %p.stretched, <vscale x 16 x i8> %a, <vscale x 16 x i8> %dst
+ ret <vscale x 16 x i8> %sel
+}
+
+define <vscale x 16 x i8> @sel_nxv16i8_i16_mask(<vscale x 8 x i1> %mask, <vscale x 16 x i8> %dst, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: sel_nxv16i8_i16_mask:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov z0.h, p0/m, z1.h
+; CHECK-NEXT: ret
+ %p.stretched = call <vscale x 16 x i1> @llvm.vector.stretch.nxv16i1.nxv8i1(<vscale x 8 x i1> %mask)
+ %sel = select <vscale x 16 x i1> %p.stretched, <vscale x 16 x i8> %a, <vscale x 16 x i8> %dst
+ ret <vscale x 16 x i8> %sel
+}
+
define <vscale x 8 x i16> @sel_nxv8i16(<vscale x 8 x i1> %p, <vscale x 8 x i16> %dst, <vscale x 8 x i16> %a) {
; CHECK-LABEL: sel_nxv8i16:
; CHECK: // %bb.0:
diff --git a/llvm/test/CodeGen/AArch64/sve-stepvector.ll b/llvm/test/CodeGen/AArch64/sve-stepvector.ll
index 3cff76f4d81ad..35b0946e0667e 100644
--- a/llvm/test/CodeGen/AArch64/sve-stepvector.ll
+++ b/llvm/test/CodeGen/AArch64/sve-stepvector.ll
@@ -184,6 +184,19 @@ define <vscale x 4 x i32> @add_stepvector_nxv4i32_1(i32 %data) {
ret <vscale x 4 x i32> %4
}
+define <vscale x 2 x i32> @add_stepvector_nxv2i32_1(i32 %data) {
+; CHECK-LABEL: add_stepvector_nxv2i32_1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT: index z0.d, x0, #1
+; CHECK-NEXT: ret
+ %1 = insertelement <vscale x 2 x i32> poison, i32 %data, i32 0
+ %2 = shufflevector <vscale x 2 x i32> %1, <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
+ %3 = call <vscale x 2 x i32> @llvm.stepvector.nxv2i32()
+ %4 = add <vscale x 2 x i32> %3, %2
+ ret <vscale x 2 x i32> %4
+}
+
define <vscale x 4 x i32> @multiple_use_stepvector_nxv4i32_1(i32 %data) {
; CHECK-LABEL: multiple_use_stepvector_nxv4i32_1:
; CHECK: // %bb.0:
@@ -214,6 +227,60 @@ define <vscale x 2 x i64> @add_stepvector_nxv2i64_1(i64 %data) {
ret <vscale x 2 x i64> %4
}
+define <vscale x 2 x i64> @add_stepvector_nxv2i64_1_from_i32(i32 %data) {
+; CHECK-LABEL: add_stepvector_nxv2i64_1_from_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT: index z0.d, x0, #1
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: sxtw z0.d, p0/m, z0.d
+; CHECK-NEXT: ret
+ %1 = insertelement <vscale x 2 x i32> poison, i32 %data, i32 0
+ %splat.data = shufflevector <vscale x 2 x i32> %1, <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
+ %3 = call <vscale x 2 x i32> @llvm.stepvector.nxv2i32()
+ %based = add <vscale x 2 x i32> %3, %splat.data
+ %4 = sext <vscale x 2 x i32> %based to <vscale x 2 x i64>
+ ret <vscale x 2 x i64> %4
+}
+
+define <vscale x 1 x i64> @add_stepvector_nxv1i64_1_simple(i64 %data) {
+; CHECK-LABEL: add_stepvector_nxv1i64_1_simple:
+; CHECK: // %bb.0:
+; CHECK-NEXT: index z0.d, #0, #1
+; CHECK-NEXT: ret
+ %3 = call <vscale x 1 x i64> @llvm.stepvector.nxv1i64()
+ ret <vscale x 1 x i64> %3
+}
+
+define <vscale x 1 x i1> @add_stepvector_nxv1i64_1_cmp(i64 %data) {
+; CHECK-LABEL: add_stepvector_nxv1i64_1_cmp:
+; CHECK: // %bb.0:
+; CHECK-NEXT: index z0.d, #0, #1
+; CHECK-NEXT: mov z1.d, x0
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: cmpne p1.d, p0/z, z0.d, z1.d
+; CHECK-NEXT: punpklo p0.h, p1.b
+; CHECK-NEXT: ret
+ %1 = insertelement <vscale x 1 x i64> poison, i64 %data, i32 0
+ %splat.data = shufflevector <vscale x 1 x i64> %1, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+ %3 = call <vscale x 1 x i64> @llvm.stepvector.nxv1i64()
+ %4 = icmp ne <vscale x 1 x i64> %3, %splat.data
+ ret <vscale x 1 x i1> %4
+}
+
+define <vscale x 1 x i64> @add_stepvector_nxv1i64_4(i64 %data) {
+; CHECK-LABEL: add_stepvector_nxv1i64_4:
+; CHECK: // %bb.0:
+; CHECK-NEXT: index z0.d, x0, #4
+; CHECK-NEXT: ret
+ %1 = insertelement <vscale x 1 x i64> poison, i64 %data, i32 0
+ %2 = shufflevector <vscale x 1 x i64> %1, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+ %3 = call <vscale x 1 x i64> @llvm.stepvector.nxv1i64()
+ %scaled = mul <vscale x 1 x i64> %3, splat (i64 4)
+ %4 = add <vscale x 1 x i64> %2, %scaled
+ ret <vscale x 1 x i64> %4
+}
+
define <vscale x 2 x i64> @multiple_use_stepvector_nxv2i64_1(i64 %data) {
; CHECK-LABEL: multiple_use_stepvector_nxv2i64_1:
; CHECK: // %bb.0:
diff --git a/llvm/test/CodeGen/AArch64/sve-vector-broadcast.ll b/llvm/test/CodeGen/AArch64/sve-vector-broadcast.ll
new file mode 100644
index 0000000000000..d9b8c008ec757
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-vector-broadcast.ll
@@ -0,0 +1,324 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+
+define <vscale x 16 x i8> @broadcast_quad_i8(<16 x i8> %a) {
+; CHECK-LABEL: broadcast_quad_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: mov z0.q, q0
+; CHECK-NEXT: ret
+ %out = call <vscale x 16 x i8> @llvm.vector.broadcast.nxv16i8.v16i8(<16 x i8> %a)
+ ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 16 x i8> @broadcast_double_i8(<8 x i8> %a) {
+; CHECK-LABEL: broadcast_double_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: mov z0.d, d0
+; CHECK-NEXT: ret
+ %out = call <vscale x 16 x i8> @llvm.vector.broadcast.nxv16i8.v8i8(<8 x i8> %a)
+ ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @broadcast_double_i8_to_double_sve(<8 x i8> %a) {
+; CHECK-LABEL: broadcast_double_i8_to_double_sve:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: mov z0.q, q0
+; CHECK-NEXT: sxtb z0.h, p0/m, z0.h
+; CHECK-NEXT: ret
+ %out = call <vscale x 8 x i8> @llvm.vector.broadcast.nxv8i8.v8i8(<8 x i8> %a)
+ %out.legal = sext <vscale x 8 x i8> %out to <vscale x 8 x i16>
+ ret <vscale x 8 x i16> %out.legal
+}
+
+define <vscale x 8 x i16> @broadcast_quad_i16(<8 x i16> %a) {
+; CHECK-LABEL: broadcast_quad_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: mov z0.q, q0
+; CHECK-NEXT: ret
+ %out = call <vscale x 8 x i16> @llvm.vector.broadcast.nxv8i16.v8i16(<8 x i16> %a)
+ ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 8 x i16> @broadcast_double_i16(<4 x i16> %a) {
+; CHECK-LABEL: broadcast_double_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: mov z0.d, d0
+; CHECK-NEXT: ret
+ %out = call <vscale x 8 x i16> @llvm.vector.broadcast.nxv8i16.v4i16(<4 x i16> %a)
+ ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @broadcast_double_i16_to_double_sve(<4 x i16> %a) {
+; CHECK-LABEL: broadcast_double_i16_to_double_sve:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: mov z0.q, q0
+; CHECK-NEXT: and z0.s, z0.s, #0xffff
+; CHECK-NEXT: ret
+ %out = call <vscale x 4 x i16> @llvm.vector.broadcast.nxv4i16.v4i16(<4 x i16> %a)
+ %out.legal = zext <vscale x 4 x i16> %out to <vscale x 4 x i32>
+ ret <vscale x 4 x i32> %out.legal
+}
+
+define <vscale x 4 x i32> @broadcast_quad_i32(<4 x i32> %a) {
+; CHECK-LABEL: broadcast_quad_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: mov z0.q, q0
+; CHECK-NEXT: ret
+ %out = call <vscale x 4 x i32> @llvm.vector.broadcast.nxv4i32.v4i32(<4 x i32> %a)
+ ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @broadcast_quad_i64(<2 x i64> %a) {
+; CHECK-LABEL: broadcast_quad_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: mov z0.q, q0
+; CHECK-NEXT: ret
+ %out = call <vscale x 2 x i64> @llvm.vector.broadcast.nxv2i64.v2i64(<2 x i64> %a)
+ ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 8 x half> @broadcast_quad_f16(<8 x half> %a) {
+; CHECK-LABEL: broadcast_quad_f16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: mov z0.q, q0
+; CHECK-NEXT: ret
+ %out = call <vscale x 8 x half> @llvm.vector.broadcast.nxv8f16(<8 x half> %a)
+ ret <vscale x 8 x half> %out
+}
+
+define <vscale x 8 x half> @broadcast_double_f16(<4 x half> %a) {
+; CHECK-LABEL: broadcast_double_f16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: mov z0.d, d0
+; CHECK-NEXT: ret
+ %out = call <vscale x 8 x half> @llvm.vector.broadcast.nxv8f16(<4 x half> %a)
+ ret <vscale x 8 x half> %out
+}
+
+define <vscale x 4 x half> @broadcast_double_f16_to_double_sve(<4 x half> %a) {
+; CHECK-LABEL: broadcast_double_f16_to_double_sve:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: mov z0.d, d0
+; CHECK-NEXT: ret
+ %out = call <vscale x 4 x half> @llvm.vector.broadcast.nxv4f16(<4 x half> %a)
+ ret <vscale x 4 x half> %out
+}
+
+; TODO: Support broadcasts from 32-bit vec
+; define <vscale x 8 x half> @broadcast_single_f16(<2 x half> %a) {
+; %out = call <vscale x 8 x half> @llvm.vector.broadcast.nxv8f16(<2 x half> %a)
+; ret <vscale x 8 x half> %out
+; }
+
+define <vscale x 8 x bfloat> @broadcast_quad_bf16(<8 x bfloat> %a) #0 {
+; CHECK-LABEL: broadcast_quad_bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: mov z0.q, q0
+; CHECK-NEXT: ret
+ %out = call <vscale x 8 x bfloat> @llvm.vector.broadcast.nxv8bf16.v8bf16(<8 x bfloat> %a)
+ ret <vscale x 8 x bfloat> %out
+}
+
+define <vscale x 4 x float> @broadcast_quad_f32(<4 x float> %a) {
+; CHECK-LABEL: broadcast_quad_f32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: mov z0.q, q0
+; CHECK-NEXT: ret
+ %out = call <vscale x 4 x float> @llvm.vector.broadcast.nxv4f32.v4f32(<4 x float> %a)
+ ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @broadcast_quad_f64(<2 x double> %a) {
+; CHECK-LABEL: broadcast_quad_f64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: mov z0.q, q0
+; CHECK-NEXT: ret
+ %out = call <vscale x 2 x double> @llvm.vector.broadcast.nxv2f64.v2f64(<2 x double> %a)
+ ret <vscale x 2 x double> %out
+}
+
+; Predicates
+
+define <vscale x 16 x i1> @broadcast_v16i1_to_nxv16i1(<16 x i8> %a) {
+; CHECK-LABEL: broadcast_v16i1_to_nxv16i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: mov z0.q, q0
+; CHECK-NEXT: and z0.b, z0.b, #0x1
+; CHECK-NEXT: cmpne p0.b, p0/z, z0.b, #0
+; CHECK-NEXT: ret
+ %a.legal = trunc <16 x i8> %a to <16 x i1>
+ %out = call <vscale x 16 x i1> @llvm.vector.broadcast.nxv16i1.v16i1(<16 x i1> %a.legal)
+ ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @broadcast_v8i1_to_nxv16i1(<8 x i8> %a) {
+; CHECK-LABEL: broadcast_v8i1_to_nxv16i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: mov z0.d, d0
+; CHECK-NEXT: and z0.b, z0.b, #0x1
+; CHECK-NEXT: cmpne p0.b, p0/z, z0.b, #0
+; CHECK-NEXT: ret
+ %a.legal = trunc <8 x i8> %a to <8 x i1>
+ %out = call <vscale x 16 x i1> @llvm.vector.broadcast.nxv16i1.v8i1(<8 x i1> %a.legal)
+ ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 8 x i1> @broadcast_v8i1_double_to_nxv8i1(<8 x i8> %a) {
+; CHECK-LABEL: broadcast_v8i1_double_to_nxv8i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: mov z0.q, q0
+; CHECK-NEXT: and z0.h, z0.h, #0x1
+; CHECK-NEXT: cmpne p0.h, p0/z, z0.h, #0
+; CHECK-NEXT: ret
+ %a.legal = trunc <8 x i8> %a to <8 x i1>
+ %out = call <vscale x 8 x i1> @llvm.vector.broadcast.nxv8i1.v8i1(<8 x i1> %a.legal)
+ ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 8 x i1> @broadcast_v8i1_quad_to_nxv8i1(<8 x i16> %a) {
+; CHECK-LABEL: broadcast_v8i1_quad_to_nxv8i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: mov z0.q, q0
+; CHECK-NEXT: and z0.h, z0.h, #0x1
+; CHECK-NEXT: cmpne p0.h, p0/z, z0.h, #0
+; CHECK-NEXT: ret
+ %a.legal = trunc <8 x i16> %a to <8 x i1>
+ %out = call <vscale x 8 x i1> @llvm.vector.broadcast.nxv8i1.v8i1(<8 x i1> %a.legal)
+ ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 8 x i1> @broadcast_v4i1_double_to_nxv8i1(<4 x i16> %a) {
+; CHECK-LABEL: broadcast_v4i1_double_to_nxv8i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: mov z0.d, d0
+; CHECK-NEXT: and z0.h, z0.h, #0x1
+; CHECK-NEXT: cmpne p0.h, p0/z, z0.h, #0
+; CHECK-NEXT: ret
+ %a.legal = trunc <4 x i16> %a to <4 x i1>
+ %out = call <vscale x 8 x i1> @llvm.vector.broadcast.nxv8i1.v4i1(<4 x i1> %a.legal)
+ ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 8 x i1> @broadcast_v4i1_quad_to_nxv8i1(<4 x i32> %a) {
+; CHECK-LABEL: broadcast_v4i1_quad_to_nxv8i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: xtn v0.4h, v0.4s
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: mov z0.d, d0
+; CHECK-NEXT: and z0.h, z0.h, #0x1
+; CHECK-NEXT: cmpne p0.h, p0/z, z0.h, #0
+; CHECK-NEXT: ret
+ %a.legal = trunc <4 x i32> %a to <4 x i1>
+ %out = call <vscale x 8 x i1> @llvm.vector.broadcast.nxv8i1.v4i1(<4 x i1> %a.legal)
+ ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 4 x i1> @broadcast_v4i1_double_to_nxv4i1(<4 x i16> %a) {
+; CHECK-LABEL: broadcast_v4i1_double_to_nxv4i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: mov z0.q, q0
+; CHECK-NEXT: and z0.s, z0.s, #0x1
+; CHECK-NEXT: cmpne p0.s, p0/z, z0.s, #0
+; CHECK-NEXT: ret
+ %a.legal = trunc <4 x i16> %a to <4 x i1>
+ %out = call <vscale x 4 x i1> @llvm.vector.broadcast.nxv4i1.v4i1(<4 x i1> %a.legal)
+ ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 4 x i1> @broadcast_v4i1_quad_to_nxv4i1(<4 x i32> %a) {
+; CHECK-LABEL: broadcast_v4i1_quad_to_nxv4i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: mov z0.q, q0
+; CHECK-NEXT: and z0.s, z0.s, #0x1
+; CHECK-NEXT: cmpne p0.s, p0/z, z0.s, #0
+; CHECK-NEXT: ret
+ %a.legal = trunc <4 x i32> %a to <4 x i1>
+ %out = call <vscale x 4 x i1> @llvm.vector.broadcast.nxv4i1.v4i1(<4 x i1> %a.legal)
+ ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 4 x i1> @broadcast_v2i1_double_to_nxv4i1(<2 x i32> %a) {
+; CHECK-LABEL: broadcast_v2i1_double_to_nxv4i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: mov z0.d, d0
+; CHECK-NEXT: and z0.s, z0.s, #0x1
+; CHECK-NEXT: cmpne p0.s, p0/z, z0.s, #0
+; CHECK-NEXT: ret
+ %a.legal = trunc <2 x i32> %a to <2 x i1>
+ %out = call <vscale x 4 x i1> @llvm.vector.broadcast.nxv4i1.v2i1(<2 x i1> %a.legal)
+ ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 4 x i1> @broadcast_v2i1_quad_to_nxv4i1(<2 x i64> %a) {
+; CHECK-LABEL: broadcast_v2i1_quad_to_nxv4i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: xtn v0.2s, v0.2d
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: mov z0.d, d0
+; CHECK-NEXT: and z0.s, z0.s, #0x1
+; CHECK-NEXT: cmpne p0.s, p0/z, z0.s, #0
+; CHECK-NEXT: ret
+ %a.legal = trunc <2 x i64> %a to <2 x i1>
+ %out = call <vscale x 4 x i1> @llvm.vector.broadcast.nxv4i1.v2i1(<2 x i1> %a.legal)
+ ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 2 x i1> @broadcast_v2i1_double_to_nxv2i1(<2 x i32> %a) {
+; CHECK-LABEL: broadcast_v2i1_double_to_nxv2i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ushll v0.2d, v0.2s, #0
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: mov z0.q, q0
+; CHECK-NEXT: and z0.d, z0.d, #0x1
+; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0
+; CHECK-NEXT: ret
+ %a.legal = trunc <2 x i32> %a to <2 x i1>
+ %out = call <vscale x 2 x i1> @llvm.vector.broadcast.nxv2i1.v2i1(<2 x i1> %a.legal)
+ ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 2 x i1> @broadcast_v2i1_quad_to_nxv2i1(<2 x i64> %a) {
+; CHECK-LABEL: broadcast_v2i1_quad_to_nxv2i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: mov z0.q, q0
+; CHECK-NEXT: and z0.d, z0.d, #0x1
+; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0
+; CHECK-NEXT: ret
+ %a.legal = trunc <2 x i64> %a to <2 x i1>
+ %out = call <vscale x 2 x i1> @llvm.vector.broadcast.nxv2i1.v2i1(<2 x i1> %a.legal)
+ ret <vscale x 2 x i1> %out
+}
diff --git a/llvm/test/CodeGen/AArch64/sve-vector-deinterleave-segments.ll b/llvm/test/CodeGen/AArch64/sve-vector-deinterleave-segments.ll
new file mode 100644
index 0000000000000..3c51cfd645c87
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-vector-deinterleave-segments.ll
@@ -0,0 +1,346 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mattr=+sve,+f64mm | FileCheck %s --check-prefixes=CHECK,CHECK-SVE
+; RUN: llc < %s -mattr=+sve2p1,+f64mm | FileCheck %s --check-prefixes=CHECK,CHECK-SVE2p1
+
+target triple = "aarch64-unknown-linux-gnu"
+
+; TODO: To upstream the deinterleave.segments intrinsics, we'll need more tests:
+; - Predicate types
+; - Illegal types that need splitting like <vscale x 4 x i64>
+; - "packed" types like <vscale x 4 x half>
+
+define {<vscale x 8 x half>, <vscale x 8 x half>} @vector_deinterleave_nxv8f16_nxv16f16(<vscale x 16 x half> %vec) {
+; CHECK-LABEL: vector_deinterleave_nxv8f16_nxv16f16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uzp1 z2.q, z0.q, z1.q
+; CHECK-NEXT: uzp2 z1.q, z0.q, z1.q
+; CHECK-NEXT: mov z0.d, z2.d
+; CHECK-NEXT: ret
+ %retval = call {<vscale x 8 x half>, <vscale x 8 x half>} @llvm.vector.deinterleave.segments2.nxv16f16(<vscale x 16 x half> %vec)
+ ret {<vscale x 8 x half>, <vscale x 8 x half>} %retval
+}
+
+define {<vscale x 4 x float>, <vscale x 4 x float>} @vector_deinterleave_nxv4f32_nxv8f32(<vscale x 8 x float> %vec) {
+; CHECK-LABEL: vector_deinterleave_nxv4f32_nxv8f32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uzp1 z2.q, z0.q, z1.q
+; CHECK-NEXT: uzp2 z1.q, z0.q, z1.q
+; CHECK-NEXT: mov z0.d, z2.d
+; CHECK-NEXT: ret
+ %retval = call {<vscale x 4 x float>, <vscale x 4 x float>} @llvm.vector.deinterleave.segments2.nxv8f32(<vscale x 8 x float> %vec)
+ ret {<vscale x 4 x float>, <vscale x 4 x float>} %retval
+}
+
+define {<vscale x 2 x double>, <vscale x 2 x double>} @vector_deinterleave_nxv2f64_nxv4f64(<vscale x 4 x double> %vec) {
+; CHECK-LABEL: vector_deinterleave_nxv2f64_nxv4f64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uzp1 z2.q, z0.q, z1.q
+; CHECK-NEXT: uzp2 z1.q, z0.q, z1.q
+; CHECK-NEXT: mov z0.d, z2.d
+; CHECK-NEXT: ret
+ %retval = call {<vscale x 2 x double>, <vscale x 2 x double>} @llvm.vector.deinterleave.segments2.nxv4f64(<vscale x 4 x double> %vec)
+ ret {<vscale x 2 x double>, <vscale x 2 x double>} %retval
+}
+
+define {<vscale x 8 x bfloat>, <vscale x 8 x bfloat>} @vector_deinterleave_nxv8bf16_nxv16bf16(<vscale x 16 x bfloat> %vec) {
+; CHECK-LABEL: vector_deinterleave_nxv8bf16_nxv16bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uzp1 z2.q, z0.q, z1.q
+; CHECK-NEXT: uzp2 z1.q, z0.q, z1.q
+; CHECK-NEXT: mov z0.d, z2.d
+; CHECK-NEXT: ret
+ %retval = call {<vscale x 8 x bfloat>, <vscale x 8 x bfloat>} @llvm.vector.deinterleave.segments2.nxv16bf16(<vscale x 16 x bfloat> %vec)
+ ret {<vscale x 8 x bfloat>, <vscale x 8 x bfloat>} %retval
+}
+
+; Integers
+
+define {<vscale x 16 x i8>, <vscale x 16 x i8>} @vector_deinterleave_nxv16i8_nxv32i8(<vscale x 32 x i8> %vec) {
+; CHECK-LABEL: vector_deinterleave_nxv16i8_nxv32i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uzp1 z2.q, z0.q, z1.q
+; CHECK-NEXT: uzp2 z1.q, z0.q, z1.q
+; CHECK-NEXT: mov z0.d, z2.d
+; CHECK-NEXT: ret
+ %retval = call {<vscale x 16 x i8>, <vscale x 16 x i8>} @llvm.vector.deinterleave.segments2.nxv32i8(<vscale x 32 x i8> %vec)
+ ret {<vscale x 16 x i8>, <vscale x 16 x i8>} %retval
+}
+
+define {<vscale x 8 x i16>, <vscale x 8 x i16>} @vector_deinterleave_nxv8i16_nxv16i16(<vscale x 16 x i16> %vec) {
+; CHECK-LABEL: vector_deinterleave_nxv8i16_nxv16i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uzp1 z2.q, z0.q, z1.q
+; CHECK-NEXT: uzp2 z1.q, z0.q, z1.q
+; CHECK-NEXT: mov z0.d, z2.d
+; CHECK-NEXT: ret
+ %retval = call {<vscale x 8 x i16>, <vscale x 8 x i16>} @llvm.vector.deinterleave.segments2.nxv16i16(<vscale x 16 x i16> %vec)
+ ret {<vscale x 8 x i16>, <vscale x 8 x i16>} %retval
+}
+
+; Deinterleaving into 2 64-bit vectors
+define <vscale x 4 x i32> @vector_deinterleave_nxv4i16_nxv8i16(<vscale x 8 x i16> %vec) {
+; CHECK-SVE-LABEL: vector_deinterleave_nxv4i16_nxv8i16:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: uzp2 z1.d, z0.d, z0.d
+; CHECK-SVE-NEXT: uzp1 z0.d, z0.d, z0.d
+; CHECK-SVE-NEXT: uunpklo z1.s, z1.h
+; CHECK-SVE-NEXT: uunpklo z0.s, z0.h
+; CHECK-SVE-NEXT: sub z0.s, z0.s, z1.s
+; CHECK-SVE-NEXT: and z0.s, z0.s, #0xffff
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SVE2p1-LABEL: vector_deinterleave_nxv4i16_nxv8i16:
+; CHECK-SVE2p1: // %bb.0:
+; CHECK-SVE2p1-NEXT: zipq2 z1.h, z0.h, z0.h
+; CHECK-SVE2p1-NEXT: zipq1 z0.h, z0.h, z0.h
+; CHECK-SVE2p1-NEXT: sub z0.s, z0.s, z1.s
+; CHECK-SVE2p1-NEXT: and z0.s, z0.s, #0xffff
+; CHECK-SVE2p1-NEXT: ret
+ %retval = call {<vscale x 4 x i16>, <vscale x 4 x i16>} @llvm.vector.deinterleave.segments2.nxv8i16(<vscale x 8 x i16> %vec)
+ %res.even = extractvalue {<vscale x 4 x i16>, <vscale x 4 x i16>} %retval, 0
+ %res.odd = extractvalue {<vscale x 4 x i16>, <vscale x 4 x i16>} %retval, 1
+ %res = sub <vscale x 4 x i16> %res.even, %res.odd
+ %res2 = zext <vscale x 4 x i16> %res to <vscale x 4 x i32>
+ ret <vscale x 4 x i32> %res2
+}
+
+; Deinterleaving into 2 32-bit vectors
+define <vscale x 2 x i64> @vector_deinterleave_nxv2i16_nxv4i16(<vscale x 4 x i32> %vec) {
+; CHECK-LABEL: vector_deinterleave_nxv2i16_nxv4i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uunpkhi z1.d, z0.s
+; CHECK-NEXT: uunpklo z0.d, z0.s
+; CHECK-NEXT: uzp2 z2.q, z0.q, z1.q
+; CHECK-NEXT: uzp1 z0.q, z0.q, z1.q
+; CHECK-NEXT: sub z0.d, z0.d, z2.d
+; CHECK-NEXT: and z0.d, z0.d, #0xffff
+; CHECK-NEXT: ret
+ %vec2 = trunc <vscale x 4 x i32> %vec to <vscale x 4 x i16>
+ %retval = call {<vscale x 2 x i16>, <vscale x 2 x i16>} @llvm.vector.deinterleave.segments2.nxv4i16(<vscale x 4 x i16> %vec2)
+ %res.even = extractvalue {<vscale x 2 x i16>, <vscale x 2 x i16>} %retval, 0
+ %res.odd = extractvalue {<vscale x 2 x i16>, <vscale x 2 x i16>} %retval, 1
+ %res = sub <vscale x 2 x i16> %res.even, %res.odd
+ %res2 = zext <vscale x 2 x i16> %res to <vscale x 2 x i64>
+ ret <vscale x 2 x i64> %res2
+}
+
+define {<vscale x 4 x i32>, <vscale x 4 x i32>} @vector_deinterleave_nxv4i32_nxvv8i32(<vscale x 8 x i32> %vec) {
+; CHECK-LABEL: vector_deinterleave_nxv4i32_nxvv8i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uzp1 z2.q, z0.q, z1.q
+; CHECK-NEXT: uzp2 z1.q, z0.q, z1.q
+; CHECK-NEXT: mov z0.d, z2.d
+; CHECK-NEXT: ret
+ %retval = call {<vscale x 4 x i32>, <vscale x 4 x i32>} @llvm.vector.deinterleave.segments2.nxv8i32(<vscale x 8 x i32> %vec)
+ ret {<vscale x 4 x i32>, <vscale x 4 x i32>} %retval
+}
+
+; Deinterleaving into 2 64-bit vectors
+define <vscale x 2 x i64> @vector_deinterleave_nxv2i32_nxv4i32(<vscale x 4 x i32> %vec) {
+; CHECK-SVE-LABEL: vector_deinterleave_nxv2i32_nxv4i32:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: uzp2 z1.d, z0.d, z0.d
+; CHECK-SVE-NEXT: uzp1 z0.d, z0.d, z0.d
+; CHECK-SVE-NEXT: ptrue p0.d
+; CHECK-SVE-NEXT: uunpklo z1.d, z1.s
+; CHECK-SVE-NEXT: uunpklo z0.d, z0.s
+; CHECK-SVE-NEXT: sub z0.d, z0.d, z1.d
+; CHECK-SVE-NEXT: sxtw z0.d, p0/m, z0.d
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SVE2p1-LABEL: vector_deinterleave_nxv2i32_nxv4i32:
+; CHECK-SVE2p1: // %bb.0:
+; CHECK-SVE2p1-NEXT: zipq2 z1.s, z0.s, z0.s
+; CHECK-SVE2p1-NEXT: zipq1 z0.s, z0.s, z0.s
+; CHECK-SVE2p1-NEXT: ptrue p0.d
+; CHECK-SVE2p1-NEXT: sub z0.d, z0.d, z1.d
+; CHECK-SVE2p1-NEXT: sxtw z0.d, p0/m, z0.d
+; CHECK-SVE2p1-NEXT: ret
+ %retval = call {<vscale x 2 x i32>, <vscale x 2 x i32>} @llvm.vector.deinterleave.segments2.nxv4i32(<vscale x 4 x i32> %vec)
+ %res.even = extractvalue {<vscale x 2 x i32>, <vscale x 2 x i32>} %retval, 0
+ %res.odd = extractvalue {<vscale x 2 x i32>, <vscale x 2 x i32>} %retval, 1
+ %res = sub <vscale x 2 x i32> %res.even, %res.odd
+ %res2 = sext <vscale x 2 x i32> %res to <vscale x 2 x i64>
+ ret <vscale x 2 x i64> %res2
+}
+
+define {<vscale x 2 x i64>, <vscale x 2 x i64>} @vector_deinterleave_nxv2i64_nxv4i64(<vscale x 4 x i64> %vec) {
+; CHECK-LABEL: vector_deinterleave_nxv2i64_nxv4i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uzp1 z2.q, z0.q, z1.q
+; CHECK-NEXT: uzp2 z1.q, z0.q, z1.q
+; CHECK-NEXT: mov z0.d, z2.d
+; CHECK-NEXT: ret
+ %retval = call {<vscale x 2 x i64>, <vscale x 2 x i64>} @llvm.vector.deinterleave.segments2.nxv4i64(<vscale x 4 x i64> %vec)
+ ret {<vscale x 2 x i64>, <vscale x 2 x i64>} %retval
+}
+
+define {<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>} @vector_deinterleave_nxv16i8_nxv64i8(<vscale x 64 x i8> %vec) {
+; CHECK-LABEL: vector_deinterleave_nxv16i8_nxv64i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uzp1 z4.q, z2.q, z3.q
+; CHECK-NEXT: uzp1 z5.q, z0.q, z1.q
+; CHECK-NEXT: uzp2 z3.q, z2.q, z3.q
+; CHECK-NEXT: uzp2 z6.q, z0.q, z1.q
+; CHECK-NEXT: uzp1 z0.q, z5.q, z4.q
+; CHECK-NEXT: uzp2 z2.q, z5.q, z4.q
+; CHECK-NEXT: uzp1 z1.q, z6.q, z3.q
+; CHECK-NEXT: uzp2 z3.q, z6.q, z3.q
+; CHECK-NEXT: ret
+ %retval = call {<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>} @llvm.vector.deinterleave.segments4.nxv64i8(<vscale x 64 x i8> %vec)
+ ret {<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>} %retval
+}
+
+define {<vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>} @vector_deinterleave_nxv8i16_nxv32i16(<vscale x 32 x i16> %vec) {
+; CHECK-LABEL: vector_deinterleave_nxv8i16_nxv32i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uzp1 z4.q, z2.q, z3.q
+; CHECK-NEXT: uzp1 z5.q, z0.q, z1.q
+; CHECK-NEXT: uzp2 z3.q, z2.q, z3.q
+; CHECK-NEXT: uzp2 z6.q, z0.q, z1.q
+; CHECK-NEXT: uzp1 z0.q, z5.q, z4.q
+; CHECK-NEXT: uzp2 z2.q, z5.q, z4.q
+; CHECK-NEXT: uzp1 z1.q, z6.q, z3.q
+; CHECK-NEXT: uzp2 z3.q, z6.q, z3.q
+; CHECK-NEXT: ret
+ %retval = call {<vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>} @llvm.vector.deinterleave.segments4.nxv32i16(<vscale x 32 x i16> %vec)
+ ret {<vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>} %retval
+}
+
+define {<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>} @vector_deinterleave_nxv4i32_nxv16i32(<vscale x 16 x i32> %vec) {
+; CHECK-LABEL: vector_deinterleave_nxv4i32_nxv16i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uzp1 z4.q, z2.q, z3.q
+; CHECK-NEXT: uzp1 z5.q, z0.q, z1.q
+; CHECK-NEXT: uzp2 z3.q, z2.q, z3.q
+; CHECK-NEXT: uzp2 z6.q, z0.q, z1.q
+; CHECK-NEXT: uzp1 z0.q, z5.q, z4.q
+; CHECK-NEXT: uzp2 z2.q, z5.q, z4.q
+; CHECK-NEXT: uzp1 z1.q, z6.q, z3.q
+; CHECK-NEXT: uzp2 z3.q, z6.q, z3.q
+; CHECK-NEXT: ret
+ %retval = call {<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>} @llvm.vector.deinterleave.segments4.nxv16i32(<vscale x 16 x i32> %vec)
+ ret {<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>} %retval
+}
+
+define {<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>} @vector_deinterleave_nxv2i64_nxv8i64(<vscale x 8 x i64> %vec) {
+; CHECK-LABEL: vector_deinterleave_nxv2i64_nxv8i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uzp1 z4.q, z2.q, z3.q
+; CHECK-NEXT: uzp1 z5.q, z0.q, z1.q
+; CHECK-NEXT: uzp2 z3.q, z2.q, z3.q
+; CHECK-NEXT: uzp2 z6.q, z0.q, z1.q
+; CHECK-NEXT: uzp1 z0.q, z5.q, z4.q
+; CHECK-NEXT: uzp2 z2.q, z5.q, z4.q
+; CHECK-NEXT: uzp1 z1.q, z6.q, z3.q
+; CHECK-NEXT: uzp2 z3.q, z6.q, z3.q
+; CHECK-NEXT: ret
+ %retval = call {<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>} @llvm.vector.deinterleave.segments4.nxv8i64(<vscale x 8 x i64> %vec)
+ ret {<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>} %retval
+}
+
+
+define {<vscale x 2 x half>, <vscale x 2 x half>} @vector_deinterleave_nxv2f16_nxv4f16(<vscale x 4 x half> %vec) {
+; CHECK-LABEL: vector_deinterleave_nxv2f16_nxv4f16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-2
+; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x40, 0x1e, 0x22 // sp + 16 + 16 * VG
+; CHECK-NEXT: .cfi_offset w29, -16
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: st1h { z0.s }, p0, [sp, #3, mul vl]
+; CHECK-NEXT: ld1w { z0.d }, p1/z, [sp, #3, mul vl]
+; CHECK-NEXT: uzp1 z1.d, z0.d, z0.d
+; CHECK-NEXT: uzp2 z0.d, z0.d, z0.d
+; CHECK-NEXT: st1w { z1.d }, p1, [sp, #2, mul vl]
+; CHECK-NEXT: st1w { z0.d }, p1, [sp, #1, mul vl]
+; CHECK-NEXT: ld1h { z0.d }, p1/z, [sp, #4, mul vl]
+; CHECK-NEXT: ld1h { z1.d }, p1/z, [sp, #2, mul vl]
+; CHECK-NEXT: addvl sp, sp, #2
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+ %retval = call {<vscale x 2 x half>, <vscale x 2 x half>} @llvm.vector.deinterleave.segments2.nxv4f16(<vscale x 4 x half> %vec)
+ ret {<vscale x 2 x half>, <vscale x 2 x half>} %retval
+}
+
+define {<vscale x 4 x half>, <vscale x 4 x half>} @vector_deinterleave_nxv4f16_nxv8f16(<vscale x 8 x half> %vec) {
+; CHECK-SVE-LABEL: vector_deinterleave_nxv4f16_nxv8f16:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: uzp1 z1.d, z0.d, z0.d
+; CHECK-SVE-NEXT: uzp2 z2.d, z0.d, z0.d
+; CHECK-SVE-NEXT: uunpklo z0.s, z1.h
+; CHECK-SVE-NEXT: uunpklo z1.s, z2.h
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SVE2p1-LABEL: vector_deinterleave_nxv4f16_nxv8f16:
+; CHECK-SVE2p1: // %bb.0:
+; CHECK-SVE2p1-NEXT: zipq1 z2.h, z0.h, z0.h
+; CHECK-SVE2p1-NEXT: zipq2 z1.h, z0.h, z0.h
+; CHECK-SVE2p1-NEXT: mov z0.d, z2.d
+; CHECK-SVE2p1-NEXT: ret
+ %retval = call {<vscale x 4 x half>, <vscale x 4 x half>} @llvm.vector.deinterleave.segments2.nxv8f16(<vscale x 8 x half> %vec)
+ ret {<vscale x 4 x half>, <vscale x 4 x half>} %retval
+}
+
+define {<vscale x 2 x float>, <vscale x 2 x float>} @vector_deinterleave_nxv2f32_nxv4f32(<vscale x 4 x float> %vec) {
+; CHECK-SVE-LABEL: vector_deinterleave_nxv2f32_nxv4f32:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: uzp1 z1.d, z0.d, z0.d
+; CHECK-SVE-NEXT: uzp2 z2.d, z0.d, z0.d
+; CHECK-SVE-NEXT: uunpklo z0.d, z1.s
+; CHECK-SVE-NEXT: uunpklo z1.d, z2.s
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SVE2p1-LABEL: vector_deinterleave_nxv2f32_nxv4f32:
+; CHECK-SVE2p1: // %bb.0:
+; CHECK-SVE2p1-NEXT: zipq1 z2.s, z0.s, z0.s
+; CHECK-SVE2p1-NEXT: zipq2 z1.s, z0.s, z0.s
+; CHECK-SVE2p1-NEXT: mov z0.d, z2.d
+; CHECK-SVE2p1-NEXT: ret
+ %retval = call {<vscale x 2 x float>, <vscale x 2 x float>} @llvm.vector.deinterleave.segments2.nxv4f32(<vscale x 4 x float> %vec)
+ ret {<vscale x 2 x float>, <vscale x 2 x float>} %retval
+}
+
+define {<vscale x 2 x bfloat>, <vscale x 2 x bfloat>} @vector_deinterleave_nxv2bf16_nxv4bf16(<vscale x 4 x bfloat> %vec) {
+; CHECK-LABEL: vector_deinterleave_nxv2bf16_nxv4bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-2
+; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x40, 0x1e, 0x22 // sp + 16 + 16 * VG
+; CHECK-NEXT: .cfi_offset w29, -16
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: st1h { z0.s }, p0, [sp, #3, mul vl]
+; CHECK-NEXT: ld1w { z0.d }, p1/z, [sp, #3, mul vl]
+; CHECK-NEXT: uzp1 z1.d, z0.d, z0.d
+; CHECK-NEXT: uzp2 z0.d, z0.d, z0.d
+; CHECK-NEXT: st1w { z1.d }, p1, [sp, #2, mul vl]
+; CHECK-NEXT: st1w { z0.d }, p1, [sp, #1, mul vl]
+; CHECK-NEXT: ld1h { z0.d }, p1/z, [sp, #4, mul vl]
+; CHECK-NEXT: ld1h { z1.d }, p1/z, [sp, #2, mul vl]
+; CHECK-NEXT: addvl sp, sp, #2
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+ %retval = call {<vscale x 2 x bfloat>, <vscale x 2 x bfloat>} @llvm.vector.deinterleave.segments2.nxv4bf16(<vscale x 4 x bfloat> %vec)
+ ret {<vscale x 2 x bfloat>, <vscale x 2 x bfloat>} %retval
+}
+
+define {<vscale x 4 x bfloat>, <vscale x 4 x bfloat>} @vector_deinterleave_nxv4bf16_nxv8bf16(<vscale x 8 x bfloat> %vec) {
+; CHECK-SVE-LABEL: vector_deinterleave_nxv4bf16_nxv8bf16:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: uzp1 z1.d, z0.d, z0.d
+; CHECK-SVE-NEXT: uzp2 z2.d, z0.d, z0.d
+; CHECK-SVE-NEXT: uunpklo z0.s, z1.h
+; CHECK-SVE-NEXT: uunpklo z1.s, z2.h
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SVE2p1-LABEL: vector_deinterleave_nxv4bf16_nxv8bf16:
+; CHECK-SVE2p1: // %bb.0:
+; CHECK-SVE2p1-NEXT: zipq1 z2.h, z0.h, z0.h
+; CHECK-SVE2p1-NEXT: zipq2 z1.h, z0.h, z0.h
+; CHECK-SVE2p1-NEXT: mov z0.d, z2.d
+; CHECK-SVE2p1-NEXT: ret
+ %retval = call {<vscale x 4 x bfloat>, <vscale x 4 x bfloat>} @llvm.vector.deinterleave.segments2.nxv8bf16(<vscale x 8 x bfloat> %vec)
+ ret {<vscale x 4 x bfloat>, <vscale x 4 x bfloat>} %retval
+}
diff --git a/llvm/test/CodeGen/AArch64/sve-vector-interleave-segments.ll b/llvm/test/CodeGen/AArch64/sve-vector-interleave-segments.ll
new file mode 100644
index 0000000000000..196d31a884d8c
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-vector-interleave-segments.ll
@@ -0,0 +1,240 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mattr=+sve2p1,+f64mm | FileCheck %s
+
+target triple = "aarch64-unknown-linux-gnu"
+
+; TODO: To upstream the deinterleave.segments intrinsics, we'll need more tests:
+; - Predicate types
+; - Illegal types that need splitting like <vscale x 4 x i64>
+; - "packed" types like <vscale x 4 x half>
+
+
+define <vscale x 16 x half> @interleave2_nxv16f16(<vscale x 8 x half> %vec0, <vscale x 8 x half> %vec1) {
+; CHECK-LABEL: interleave2_nxv16f16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: zip1 z2.q, z0.q, z1.q
+; CHECK-NEXT: zip2 z1.q, z0.q, z1.q
+; CHECK-NEXT: mov z0.d, z2.d
+; CHECK-NEXT: ret
+ %retval = call <vscale x 16 x half> @llvm.vector.interleave.segments2.nxv16f16(<vscale x 8 x half> %vec0, <vscale x 8 x half> %vec1)
+ ret <vscale x 16 x half> %retval
+}
+
+define <vscale x 8 x float> @interleave2_nxv8f32(<vscale x 4 x float> %vec0, <vscale x 4 x float> %vec1) {
+; CHECK-LABEL: interleave2_nxv8f32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: zip1 z2.q, z0.q, z1.q
+; CHECK-NEXT: zip2 z1.q, z0.q, z1.q
+; CHECK-NEXT: mov z0.d, z2.d
+; CHECK-NEXT: ret
+ %retval = call <vscale x 8 x float> @llvm.vector.interleave.segments2.nxv8f32(<vscale x 4 x float> %vec0, <vscale x 4 x float> %vec1)
+ ret <vscale x 8 x float> %retval
+}
+
+define <vscale x 8 x half> @interleave2_nxv8f16(<vscale x 4 x half> %vec0, <vscale x 4 x half> %vec1) {
+; CHECK-LABEL: interleave2_nxv8f16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uzpq1 z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+ %retval = call <vscale x 8 x half> @llvm.vector.interleave.segments2.nxv8f16(<vscale x 4 x half> %vec0, <vscale x 4 x half> %vec1)
+ ret <vscale x 8 x half> %retval
+}
+
+define <vscale x 4 x double> @interleave2_nxv4f64(<vscale x 2 x double> %vec0, <vscale x 2 x double> %vec1) {
+; CHECK-LABEL: interleave2_nxv4f64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: zip1 z2.q, z0.q, z1.q
+; CHECK-NEXT: zip2 z1.q, z0.q, z1.q
+; CHECK-NEXT: mov z0.d, z2.d
+; CHECK-NEXT: ret
+ %retval = call <vscale x 4 x double>@llvm.vector.interleave.segments2.nxv4f64(<vscale x 2 x double> %vec0, <vscale x 2 x double> %vec1)
+ ret <vscale x 4 x double> %retval
+}
+
+define <vscale x 4 x float> @interleave2_nxv4f32(<vscale x 2 x float> %vec0, <vscale x 2 x float> %vec1) {
+; CHECK-LABEL: interleave2_nxv4f32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uzpq1 z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+ %retval = call <vscale x 4 x float> @llvm.vector.interleave.segments2.nxv4f32(<vscale x 2 x float> %vec0, <vscale x 2 x float> %vec1)
+ ret <vscale x 4 x float> %retval
+}
+
+define <vscale x 16 x bfloat> @interleave2_nxv16bf16(<vscale x 8 x bfloat> %vec0, <vscale x 8 x bfloat> %vec1) {
+; CHECK-LABEL: interleave2_nxv16bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: zip1 z2.q, z0.q, z1.q
+; CHECK-NEXT: zip2 z1.q, z0.q, z1.q
+; CHECK-NEXT: mov z0.d, z2.d
+; CHECK-NEXT: ret
+ %retval = call <vscale x 16 x bfloat> @llvm.vector.interleave.segments2.nxv16bf16(<vscale x 8 x bfloat> %vec0, <vscale x 8 x bfloat> %vec1)
+ ret <vscale x 16 x bfloat> %retval
+}
+
+; Integers
+
+define <vscale x 32 x i8> @interleave2_nxv32i8(<vscale x 16 x i8> %vec0, <vscale x 16 x i8> %vec1) {
+; CHECK-LABEL: interleave2_nxv32i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: zip1 z2.q, z0.q, z1.q
+; CHECK-NEXT: zip2 z1.q, z0.q, z1.q
+; CHECK-NEXT: mov z0.d, z2.d
+; CHECK-NEXT: ret
+ %retval = call <vscale x 32 x i8> @llvm.vector.interleave.segments2.nxv32i8(<vscale x 16 x i8> %vec0, <vscale x 16 x i8> %vec1)
+ ret <vscale x 32 x i8> %retval
+}
+
+define <vscale x 16 x i16> @interleave2_nxv16i16(<vscale x 8 x i16> %vec0, <vscale x 8 x i16> %vec1) {
+; CHECK-LABEL: interleave2_nxv16i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: zip1 z2.q, z0.q, z1.q
+; CHECK-NEXT: zip2 z1.q, z0.q, z1.q
+; CHECK-NEXT: mov z0.d, z2.d
+; CHECK-NEXT: ret
+ %retval = call <vscale x 16 x i16> @llvm.vector.interleave.segments2.nxv16i16(<vscale x 8 x i16> %vec0, <vscale x 8 x i16> %vec1)
+ ret <vscale x 16 x i16> %retval
+}
+
+define <vscale x 16 x i8> @interleave2_nxv16i8(<vscale x 8 x i16> %vec0, <vscale x 8 x i16> %vec1) {
+; CHECK-LABEL: interleave2_nxv16i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uzpq1 z0.b, z0.b, z1.b
+; CHECK-NEXT: ret
+ %leg0 = trunc <vscale x 8 x i16> %vec0 to <vscale x 8 x i8>
+ %leg1 = trunc <vscale x 8 x i16> %vec1 to <vscale x 8 x i8>
+ %retval = call <vscale x 16 x i8> @llvm.vector.interleave.segments2.nxv16i8(<vscale x 8 x i8> %leg0, <vscale x 8 x i8> %leg1)
+ ret <vscale x 16 x i8> %retval
+}
+
+define <vscale x 8 x i32> @interleave2_nxv8i32(<vscale x 4 x i32> %vec0, <vscale x 4 x i32> %vec1) {
+; CHECK-LABEL: interleave2_nxv8i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: zip1 z2.q, z0.q, z1.q
+; CHECK-NEXT: zip2 z1.q, z0.q, z1.q
+; CHECK-NEXT: mov z0.d, z2.d
+; CHECK-NEXT: ret
+ %retval = call <vscale x 8 x i32> @llvm.vector.interleave.segments2.nxv8i32(<vscale x 4 x i32> %vec0, <vscale x 4 x i32> %vec1)
+ ret <vscale x 8 x i32> %retval
+}
+
+define <vscale x 8 x i16> @interleave2_nxv8i16(<vscale x 4 x i32> %vec0, <vscale x 4 x i32> %vec1) {
+; CHECK-LABEL: interleave2_nxv8i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uzpq1 z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+ %leg0 = trunc <vscale x 4 x i32> %vec0 to <vscale x 4 x i16>
+ %leg1 = trunc <vscale x 4 x i32> %vec1 to <vscale x 4 x i16>
+ %retval = call <vscale x 8 x i16> @llvm.vector.interleave.segments2.nxv8i16(<vscale x 4 x i16> %leg0, <vscale x 4 x i16> %leg1)
+ ret <vscale x 8 x i16> %retval
+}
+
+define <vscale x 4 x i64> @interleave2_nxv4i64(<vscale x 2 x i64> %vec0, <vscale x 2 x i64> %vec1) {
+; CHECK-LABEL: interleave2_nxv4i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: zip1 z2.q, z0.q, z1.q
+; CHECK-NEXT: zip2 z1.q, z0.q, z1.q
+; CHECK-NEXT: mov z0.d, z2.d
+; CHECK-NEXT: ret
+ %retval = call <vscale x 4 x i64> @llvm.vector.interleave.segments2.nxv4i64(<vscale x 2 x i64> %vec0, <vscale x 2 x i64> %vec1)
+ ret <vscale x 4 x i64> %retval
+}
+
+define <vscale x 4 x i32> @interleave2_nxv4i32(<vscale x 2 x i64> %vec0, <vscale x 2 x i64> %vec1) {
+; CHECK-LABEL: interleave2_nxv4i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uzpq1 z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+ %leg0 = trunc <vscale x 2 x i64> %vec0 to <vscale x 2 x i32>
+ %leg1 = trunc <vscale x 2 x i64> %vec1 to <vscale x 2 x i32>
+ %retval = call <vscale x 4 x i32> @llvm.vector.interleave.segments2.nxv4i32(<vscale x 2 x i32> %leg0, <vscale x 2 x i32> %leg1)
+ ret <vscale x 4 x i32> %retval
+}
+
+define <vscale x 64 x i8> @interleave4_nxv16i8(<vscale x 16 x i8> %vec0, <vscale x 16 x i8> %vec1, <vscale x 16 x i8> %vec2, <vscale x 16 x i8> %vec3) {
+; CHECK-LABEL: interleave4_nxv16i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: zip1 z4.q, z1.q, z3.q
+; CHECK-NEXT: zip1 z5.q, z0.q, z2.q
+; CHECK-NEXT: zip2 z3.q, z1.q, z3.q
+; CHECK-NEXT: zip2 z6.q, z0.q, z2.q
+; CHECK-NEXT: zip1 z0.q, z5.q, z4.q
+; CHECK-NEXT: zip2 z1.q, z5.q, z4.q
+; CHECK-NEXT: zip1 z2.q, z6.q, z3.q
+; CHECK-NEXT: zip2 z3.q, z6.q, z3.q
+; CHECK-NEXT: ret
+ %retval = call <vscale x 64 x i8> @llvm.vector.interleave.segments4.nxv16i8(<vscale x 16 x i8> %vec0, <vscale x 16 x i8> %vec1, <vscale x 16 x i8> %vec2, <vscale x 16 x i8> %vec3)
+ ret <vscale x 64 x i8> %retval
+}
+
+define <vscale x 32 x i16> @interleave4_nxv8i16(<vscale x 8 x i16> %vec0, <vscale x 8 x i16> %vec1, <vscale x 8 x i16> %vec2, <vscale x 8 x i16> %vec3) {
+; CHECK-LABEL: interleave4_nxv8i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: zip1 z4.q, z1.q, z3.q
+; CHECK-NEXT: zip1 z5.q, z0.q, z2.q
+; CHECK-NEXT: zip2 z3.q, z1.q, z3.q
+; CHECK-NEXT: zip2 z6.q, z0.q, z2.q
+; CHECK-NEXT: zip1 z0.q, z5.q, z4.q
+; CHECK-NEXT: zip2 z1.q, z5.q, z4.q
+; CHECK-NEXT: zip1 z2.q, z6.q, z3.q
+; CHECK-NEXT: zip2 z3.q, z6.q, z3.q
+; CHECK-NEXT: ret
+ %retval = call <vscale x 32 x i16> @llvm.vector.interleave.segments4.nxv32i16(<vscale x 8 x i16> %vec0, <vscale x 8 x i16> %vec1, <vscale x 8 x i16> %vec2, <vscale x 8 x i16> %vec3)
+ ret <vscale x 32 x i16> %retval
+}
+
+define <vscale x 16 x i32> @interleave4_nxv4i32(<vscale x 4 x i32> %vec0, <vscale x 4 x i32> %vec1, <vscale x 4 x i32> %vec2, <vscale x 4 x i32> %vec3) {
+; CHECK-LABEL: interleave4_nxv4i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: zip1 z4.q, z1.q, z3.q
+; CHECK-NEXT: zip1 z5.q, z0.q, z2.q
+; CHECK-NEXT: zip2 z3.q, z1.q, z3.q
+; CHECK-NEXT: zip2 z6.q, z0.q, z2.q
+; CHECK-NEXT: zip1 z0.q, z5.q, z4.q
+; CHECK-NEXT: zip2 z1.q, z5.q, z4.q
+; CHECK-NEXT: zip1 z2.q, z6.q, z3.q
+; CHECK-NEXT: zip2 z3.q, z6.q, z3.q
+; CHECK-NEXT: ret
+ %retval = call <vscale x 16 x i32> @llvm.vector.interleave.segments4.nxv4i32(<vscale x 4 x i32> %vec0, <vscale x 4 x i32> %vec1, <vscale x 4 x i32> %vec2, <vscale x 4 x i32> %vec3)
+ ret <vscale x 16 x i32> %retval
+}
+
+define <vscale x 8 x i64> @interleave4_nxv8i64(<vscale x 2 x i64> %vec0, <vscale x 2 x i64> %vec1, <vscale x 2 x i64> %vec2, <vscale x 2 x i64> %vec3) {
+; CHECK-LABEL: interleave4_nxv8i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: zip1 z4.q, z1.q, z3.q
+; CHECK-NEXT: zip1 z5.q, z0.q, z2.q
+; CHECK-NEXT: zip2 z3.q, z1.q, z3.q
+; CHECK-NEXT: zip2 z6.q, z0.q, z2.q
+; CHECK-NEXT: zip1 z0.q, z5.q, z4.q
+; CHECK-NEXT: zip2 z1.q, z5.q, z4.q
+; CHECK-NEXT: zip1 z2.q, z6.q, z3.q
+; CHECK-NEXT: zip2 z3.q, z6.q, z3.q
+; CHECK-NEXT: ret
+ %retval = call <vscale x 8 x i64> @llvm.vector.interleave.segments4.nxv8i64(<vscale x 2 x i64> %vec0, <vscale x 2 x i64> %vec1, <vscale x 2 x i64> %vec2, <vscale x 2 x i64> %vec3)
+ ret <vscale x 8 x i64> %retval
+}
+
+; define <vscale x 4 x half> @interleave2_nxv4f16(<vscale x 2 x half> %vec0, <vscale x 2 x half> %vec1) {
+; %retval = call <vscale x 4 x half> @llvm.vector.interleave.segments2.nxv4f16(<vscale x 2 x half> %vec0, <vscale x 2 x half> %vec1)
+; ret <vscale x 4 x half> %retval
+; }
+
+; define <vscale x 8 x half> @interleave2_nxv8f16(<vscale x 4 x half> %vec0, <vscale x 4 x half> %vec1) {
+; %retval = call <vscale x 8 x half> @llvm.vector.interleave.segments2.nxv8f16(<vscale x 4 x half> %vec0, <vscale x 4 x half> %vec1)
+; ret <vscale x 8 x half> %retval
+; }
+
+; define <vscale x 4 x float> @interleave2_nxv4f32(<vscale x 2 x float> %vec0, <vscale x 2 x float> %vec1) {
+; %retval = call <vscale x 4 x float> @llvm.vector.interleave.segments2.nxv4f32(<vscale x 2 x float> %vec0, <vscale x 2 x float> %vec1)
+; ret <vscale x 4 x float> %retval
+; }
+
+; define <vscale x 4 x bfloat> @interleave2_nxv4bf16(<vscale x 2 x bfloat> %vec0, <vscale x 2 x bfloat> %vec1) {
+; %retval = call <vscale x 4 x bfloat> @llvm.vector.interleave.segments2.nxv4bf16(<vscale x 2 x bfloat> %vec0, <vscale x 2 x bfloat> %vec1)
+; ret <vscale x 4 x bfloat> %retval
+; }
+
+; define <vscale x 8 x bfloat> @interleave2_nxv8bf16(<vscale x 4 x bfloat> %vec0, <vscale x 4 x bfloat> %vec1) {
+; %retval = call <vscale x 8 x bfloat> @llvm.vector.interleave.segments2.nxv8bf16(<vscale x 4 x bfloat> %vec0, <vscale x 4 x bfloat> %vec1)
+; ret <vscale x 8 x bfloat> %retval
+; }
diff --git a/llvm/test/CodeGen/AArch64/sve-vector-splat.ll b/llvm/test/CodeGen/AArch64/sve-vector-splat.ll
index 1ceaa5ad27734..bc2c2c85441fa 100644
--- a/llvm/test/CodeGen/AArch64/sve-vector-splat.ll
+++ b/llvm/test/CodeGen/AArch64/sve-vector-splat.ll
@@ -43,6 +43,16 @@ define <vscale x 2 x i64> @sve_splat_2xi64(i64 %val) {
ret <vscale x 2 x i64> %splat
}
+define <vscale x 1 x i64> @sve_splat_1xi64(i64 %val) {
+; CHECK-LABEL: sve_splat_1xi64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov z0.d, x0
+; CHECK-NEXT: ret
+ %ins = insertelement <vscale x 1 x i64> poison, i64 %val, i32 0
+ %splat = shufflevector <vscale x 1 x i64> %ins, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+ ret <vscale x 1 x i64> %splat
+}
+
define <vscale x 16 x i8> @sve_splat_16xi8_imm() {
; CHECK-LABEL: sve_splat_16xi8_imm:
; CHECK: // %bb.0:
@@ -869,8 +879,8 @@ define <vscale x 2 x double> @splat_nxv2f64_imm_out_of_range() {
; CHECK-LABEL: splat_nxv2f64_imm_out_of_range:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: adrp x8, .LCPI96_0
-; CHECK-NEXT: add x8, x8, :lo12:.LCPI96_0
+; CHECK-NEXT: adrp x8, .LCPI97_0
+; CHECK-NEXT: add x8, x8, :lo12:.LCPI97_0
; CHECK-NEXT: ld1rd { z0.d }, p0/z, [x8]
; CHECK-NEXT: ret
ret <vscale x 2 x double> splat(double 3.33)
diff --git a/llvm/test/CodeGen/AArch64/sve-vector-stretch.ll b/llvm/test/CodeGen/AArch64/sve-vector-stretch.ll
new file mode 100644
index 0000000000000..af0c953b5d85b
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-vector-stretch.ll
@@ -0,0 +1,136 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme --force-streaming < %s | FileCheck %s
+
+
+; From <vscale x 1 x i1> predicates
+
+define <vscale x 2 x i1> @broadcast_pred_i64_from_i128(<vscale x 1 x i1> %a) {
+; CHECK-LABEL: broadcast_pred_i64_from_i128:
+; CHECK: // %bb.0:
+; CHECK-NEXT: trn1 p0.d, p0.d, p0.d
+; CHECK-NEXT: ret
+ %out = call <vscale x 2 x i1> @llvm.vector.stretch.nxv2i1.nxv1i1(<vscale x 1 x i1> %a)
+ ret <vscale x 2 x i1> %out
+}
+
+; "Unpack" an i64 predicate to a i128 one before stretching its bits back to a i64 predicate.
+define <vscale x 2 x i1> @widen_then_broadcast_pred_i64_from_i64_lo(<vscale x 2 x i1> %a) {
+; CHECK-LABEL: widen_then_broadcast_pred_i64_from_i64_lo:
+; CHECK: // %bb.0:
+; CHECK-NEXT: zip1 p0.d, p0.d, p0.d
+; CHECK-NEXT: ret
+ %widen = call <vscale x 1 x i1> @llvm.vector.extract.nxv1i1.nxv2i1(<vscale x 2 x i1> %a, i64 0)
+ %stretch = call <vscale x 2 x i1> @llvm.vector.stretch.nxv2i1.nxv1i1(<vscale x 1 x i1> %widen)
+ ret <vscale x 2 x i1> %stretch
+}
+
+define <vscale x 2 x i1> @widen_upper_then_broadcast_pred_i64_from_i64_lo(<vscale x 2 x i1> %a) {
+; CHECK-LABEL: widen_upper_then_broadcast_pred_i64_from_i64_lo:
+; CHECK: // %bb.0:
+; CHECK-NEXT: zip2 p0.d, p0.d, p0.d
+; CHECK-NEXT: ret
+ %widen = call <vscale x 1 x i1> @llvm.vector.extract.nxv1i1.nxv2i1(<vscale x 2 x i1> %a, i64 1)
+ %stretch = call <vscale x 2 x i1> @llvm.vector.stretch.nxv2i1.nxv1i1(<vscale x 1 x i1> %widen)
+ ret <vscale x 2 x i1> %stretch
+}
+
+define <vscale x 4 x i1> @broadcast_pred_i32_from_i128(<vscale x 1 x i1> %a) {
+; CHECK-LABEL: broadcast_pred_i32_from_i128:
+; CHECK: // %bb.0:
+; CHECK-NEXT: trn1 p0.d, p0.d, p0.d
+; CHECK-NEXT: trn1 p0.s, p0.s, p0.s
+; CHECK-NEXT: ret
+ %out = call <vscale x 4 x i1> @llvm.vector.stretch.nxv4i1.nxv1i1(<vscale x 1 x i1> %a)
+ ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 8 x i1> @broadcast_pred_i16_from_i128(<vscale x 1 x i1> %a) {
+; CHECK-LABEL: broadcast_pred_i16_from_i128:
+; CHECK: // %bb.0:
+; CHECK-NEXT: trn1 p0.d, p0.d, p0.d
+; CHECK-NEXT: trn1 p0.s, p0.s, p0.s
+; CHECK-NEXT: trn1 p0.h, p0.h, p0.h
+; CHECK-NEXT: ret
+ %out = call <vscale x 8 x i1> @llvm.vector.stretch.nxv8i1.nxv1i1(<vscale x 1 x i1> %a)
+ ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 16 x i1> @broadcast_pred_i8_from_i128(<vscale x 1 x i1> %a) {
+; CHECK-LABEL: broadcast_pred_i8_from_i128:
+; CHECK: // %bb.0:
+; CHECK-NEXT: trn1 p0.d, p0.d, p0.d
+; CHECK-NEXT: trn1 p0.s, p0.s, p0.s
+; CHECK-NEXT: trn1 p0.h, p0.h, p0.h
+; CHECK-NEXT: trn1 p0.b, p0.b, p0.b
+; CHECK-NEXT: ret
+ %out = call <vscale x 16 x i1> @llvm.vector.stretch.nxv16i1.nxv1i1(<vscale x 1 x i1> %a)
+ ret <vscale x 16 x i1> %out
+}
+
+
+; From <vscale x 2 x i1> predicates
+
+define <vscale x 4 x i1> @broadcast_pred_i32_from_i64(<vscale x 2 x i1> %a) {
+; CHECK-LABEL: broadcast_pred_i32_from_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: trn1 p0.s, p0.s, p0.s
+; CHECK-NEXT: ret
+ %out = call <vscale x 4 x i1> @llvm.vector.stretch.nxv4i1.nxv2i1(<vscale x 2 x i1> %a)
+ ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 8 x i1> @broadcast_pred_i16_from_i64(<vscale x 2 x i1> %a) {
+; CHECK-LABEL: broadcast_pred_i16_from_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: trn1 p0.s, p0.s, p0.s
+; CHECK-NEXT: trn1 p0.h, p0.h, p0.h
+; CHECK-NEXT: ret
+ %out = call <vscale x 8 x i1> @llvm.vector.stretch.nxv8i1.nxv2i1(<vscale x 2 x i1> %a)
+ ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 16 x i1> @broadcast_pred_i8_from_i64(<vscale x 2 x i1> %a) {
+; CHECK-LABEL: broadcast_pred_i8_from_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: trn1 p0.s, p0.s, p0.s
+; CHECK-NEXT: trn1 p0.h, p0.h, p0.h
+; CHECK-NEXT: trn1 p0.b, p0.b, p0.b
+; CHECK-NEXT: ret
+ %out = call <vscale x 16 x i1> @llvm.vector.stretch.nxv16i1.nxv2i1(<vscale x 2 x i1> %a)
+ ret <vscale x 16 x i1> %out
+}
+
+
+; From <vscale x 4 x i1> predicates
+
+define <vscale x 8 x i1> @broadcast_pred_i16_from_i32(<vscale x 4 x i1> %a) {
+; CHECK-LABEL: broadcast_pred_i16_from_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: trn1 p0.h, p0.h, p0.h
+; CHECK-NEXT: ret
+ %out = call <vscale x 8 x i1> @llvm.vector.stretch.nxv8i1.nxv4i1(<vscale x 4 x i1> %a)
+ ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 16 x i1> @broadcast_pred_i8_from_i32(<vscale x 4 x i1> %a) {
+; CHECK-LABEL: broadcast_pred_i8_from_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: trn1 p0.h, p0.h, p0.h
+; CHECK-NEXT: trn1 p0.b, p0.b, p0.b
+; CHECK-NEXT: ret
+ %out = call <vscale x 16 x i1> @llvm.vector.stretch.nxv16i1.nxv4i1(<vscale x 4 x i1> %a)
+ ret <vscale x 16 x i1> %out
+}
+
+
+; From <vscale x 8 x i1> predicates
+
+define <vscale x 16 x i1> @broadcast_pred_i8_from_i16(<vscale x 8 x i1> %a) {
+; CHECK-LABEL: broadcast_pred_i8_from_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: trn1 p0.b, p0.b, p0.b
+; CHECK-NEXT: ret
+ %out = call <vscale x 16 x i1> @llvm.vector.stretch.nxv16i1.nxv8i1(<vscale x 8 x i1> %a)
+ ret <vscale x 16 x i1> %out
+}
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-partial-reduction-add.ll b/llvm/test/CodeGen/AArch64/sve2p1-partial-reduction-add.ll
new file mode 100644
index 0000000000000..265f27650e73f
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve2p1-partial-reduction-add.ll
@@ -0,0 +1,150 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc -mtriple=aarch64 -mattr=+sve2p1 -o - %s | FileCheck %s
+
+define <16 x i8> @partial_reduce_scalable_zero_v16i8(<16 x i8> %accumulator, <vscale x 16 x i8> %0) #0 {
+; CHECK-LABEL: partial_reduce_scalable_zero_v16i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: addqv v0.16b, p0, z1.b
+; CHECK-NEXT: ret
+entry:
+ %partial.reduce = call <16 x i8> @llvm.vector.partial.reduce.add.v16i8.v16i8.nxv16i8(<16 x i8> zeroinitializer, <vscale x 16 x i8> %0)
+ ret <16 x i8> %partial.reduce
+}
+
+define <8 x i8> @partial_reduce_scalable_zero_nxv16i8_v8i8(<8 x i8> %accumulator, <vscale x 16 x i8> %0) #0 {
+; CHECK-LABEL: partial_reduce_scalable_zero_nxv16i8_v8i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: addqv v0.16b, p0, z1.b
+; CHECK-NEXT: mov d1, v0.d[1]
+; CHECK-NEXT: add v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: ret
+entry:
+ %partial.reduce = call <8 x i8> @llvm.vector.partial.reduce.add.v8i8.v8i8.nxv16i8(<8 x i8> zeroinitializer, <vscale x 16 x i8> %0)
+ ret <8 x i8> %partial.reduce
+}
+
+define <8 x i8> @partial_reduce_scalable_zero_nxv8i8_v8i8(<8 x i8> %accumulator, <vscale x 16 x i8> %0) #0 {
+; CHECK-LABEL: partial_reduce_scalable_zero_nxv8i8_v8i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: uunpklo z0.h, z1.b
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: addqv v0.8h, p0, z0.h
+; CHECK-NEXT: xtn v0.8b, v0.8h
+; CHECK-NEXT: ret
+entry:
+ %1 = call <vscale x 8 x i8> @llvm.vector.extract.nxv8i8.nxv16i8(<vscale x 16 x i8> %0, i64 0)
+ %partial.reduce = call <8 x i8> @llvm.vector.partial.reduce.add.v8i8.v8i8.nxv8i8(<8 x i8> zeroinitializer, <vscale x 8 x i8> %1)
+ ret <8 x i8> %partial.reduce
+}
+
+define <16 x i8> @partial_reduce_scalable_acc_v16i8(<16 x i8> %accumulator, <vscale x 16 x i8> %0) #0 {
+; CHECK-LABEL: partial_reduce_scalable_acc_v16i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: addqv v1.16b, p0, z1.b
+; CHECK-NEXT: add v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: ret
+entry:
+ %partial.reduce = call <16 x i8> @llvm.vector.partial.reduce.add.v16i8.v16i8.nxv16i8(<16 x i8> %accumulator, <vscale x 16 x i8> %0)
+ ret <16 x i8> %partial.reduce
+}
+
+define <8 x i8> @partial_reduce_scalable_acc_nxv16i8_v8i8(<8 x i8> %accumulator, <vscale x 16 x i8> %0) #0 {
+; CHECK-LABEL: partial_reduce_scalable_acc_nxv16i8_v8i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: addqv v1.16b, p0, z1.b
+; CHECK-NEXT: mov d2, v1.d[1]
+; CHECK-NEXT: add v0.8b, v1.8b, v0.8b
+; CHECK-NEXT: add v0.8b, v0.8b, v2.8b
+; CHECK-NEXT: ret
+entry:
+ %partial.reduce = call <8 x i8> @llvm.vector.partial.reduce.add.v8i8.v8i8.nxv16i8(<8 x i8> %accumulator, <vscale x 16 x i8> %0)
+ ret <8 x i8> %partial.reduce
+}
+
+define <8 x i8> @partial_reduce_scalable_acc_nxv8i8_v8i8(<8 x i8> %accumulator, <vscale x 16 x i8> %0) #0 {
+; CHECK-LABEL: partial_reduce_scalable_acc_nxv8i8_v8i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: uunpklo z1.h, z1.b
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: addqv v1.8h, p0, z1.h
+; CHECK-NEXT: uaddw v0.8h, v1.8h, v0.8b
+; CHECK-NEXT: xtn v0.8b, v0.8h
+; CHECK-NEXT: ret
+entry:
+ %1 = call <vscale x 8 x i8> @llvm.vector.extract.nxv8i8.nxv16i8(<vscale x 16 x i8> %0, i64 0)
+ %partial.reduce = call <8 x i8> @llvm.vector.partial.reduce.add.v8i8.v8i8.nxv8i8(<8 x i8> %accumulator, <vscale x 8 x i8> %1)
+ ret <8 x i8> %partial.reduce
+}
+
+define <8 x i16> @partial_reduce_scalable_zero_v8i16(<8 x i16> %accumulator, <vscale x 8 x i16> %0) #0 {
+; CHECK-LABEL: partial_reduce_scalable_zero_v8i16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: addqv v0.8h, p0, z1.h
+; CHECK-NEXT: ret
+entry:
+ %partial.reduce = call <8 x i16> @llvm.vector.partial.reduce.add.v8i16.v8i16.nxv8i16(<8 x i16> zeroinitializer, <vscale x 8 x i16> %0)
+ ret <8 x i16> %partial.reduce
+}
+
+define <8 x i16> @partial_reduce_scalable_acc_v8i16(<8 x i16> %accumulator, <vscale x 8 x i16> %0) #0 {
+; CHECK-LABEL: partial_reduce_scalable_acc_v8i16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: addqv v1.8h, p0, z1.h
+; CHECK-NEXT: add v0.8h, v0.8h, v1.8h
+; CHECK-NEXT: ret
+entry:
+ %partial.reduce = call <8 x i16> @llvm.vector.partial.reduce.add.v8i16.v8i16.nxv8i16(<8 x i16> %accumulator, <vscale x 8 x i16> %0)
+ ret <8 x i16> %partial.reduce
+}
+
+define <4 x i32> @partial_reduce_scalable_zero_v4i32(<4 x i32> %accumulator, <vscale x 4 x i32> %0) #0 {
+; CHECK-LABEL: partial_reduce_scalable_zero_v4i32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: addqv v0.4s, p0, z1.s
+; CHECK-NEXT: ret
+entry:
+ %partial.reduce = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v4i32.nxv4i32(<4 x i32> zeroinitializer, <vscale x 4 x i32> %0)
+ ret <4 x i32> %partial.reduce
+}
+
+define <4 x i32> @partial_reduce_scalable_acc_v4i32(<4 x i32> %accumulator, <vscale x 4 x i32> %0) #0 {
+; CHECK-LABEL: partial_reduce_scalable_acc_v4i32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: addqv v1.4s, p0, z1.s
+; CHECK-NEXT: add v0.4s, v0.4s, v1.4s
+; CHECK-NEXT: ret
+entry:
+ %partial.reduce = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v4i32.nxv4i32(<4 x i32> %accumulator, <vscale x 4 x i32> %0)
+ ret <4 x i32> %partial.reduce
+}
+
+define <2 x i64> @partial_reduce_scalable_zero_v2i64(<2 x i64> %accumulator, <vscale x 2 x i64> %0) #0 {
+; CHECK-LABEL: partial_reduce_scalable_zero_v2i64:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: addqv v0.2d, p0, z1.d
+; CHECK-NEXT: ret
+entry:
+ %partial.reduce = call <2 x i64> @llvm.vector.partial.reduce.add.v2i64.v2i64.nxv2i64(<2 x i64> zeroinitializer, <vscale x 2 x i64> %0)
+ ret <2 x i64> %partial.reduce
+}
+
+define <2 x i64> @partial_reduce_scalable_acc_v2i64(<2 x i64> %accumulator, <vscale x 2 x i64> %0) #0 {
+; CHECK-LABEL: partial_reduce_scalable_acc_v2i64:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: addqv v1.2d, p0, z1.d
+; CHECK-NEXT: add v0.2d, v0.2d, v1.2d
+; CHECK-NEXT: ret
+entry:
+ %partial.reduce = call <2 x i64> @llvm.vector.partial.reduce.add.v2i64.v2i64.nxv2i64(<2 x i64> %accumulator, <vscale x 2 x i64> %0)
+ ret <2 x i64> %partial.reduce
+}
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-partial-reduction-min-max.ll b/llvm/test/CodeGen/AArch64/sve2p1-partial-reduction-min-max.ll
new file mode 100644
index 0000000000000..1d0f2c0b3de3e
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve2p1-partial-reduction-min-max.ll
@@ -0,0 +1,231 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64 -mattr=+sve2p1 -o - %s | FileCheck %s
+
+define <16 x i8> @partial_reduce_smax_scalable_zero_v16i8(<16 x i8> %accumulator, <vscale x 16 x i8> %0) #0 {
+; CHECK-LABEL: partial_reduce_smax_scalable_zero_v16i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: smaxqv v0.16b, p0, z1.b
+; CHECK-NEXT: ret
+entry:
+ %partial.reduce = call <16 x i8> @llvm.vector.partial.reduce.smax.v16i8.v16i8.nxv16i8(<16 x i8> splat (i8 -128), <vscale x 16 x i8> %0)
+ ret <16 x i8> %partial.reduce
+}
+
+define <16 x i8> @partial_reduce_smax_scalable_acc_v16i8(<16 x i8> %accumulator, <vscale x 16 x i8> %0) #0 {
+; CHECK-LABEL: partial_reduce_smax_scalable_acc_v16i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: smaxqv v1.16b, p0, z1.b
+; CHECK-NEXT: smax v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: ret
+entry:
+ %partial.reduce = call <16 x i8> @llvm.vector.partial.reduce.smax.v16i8.v16i8.nxv16i8(<16 x i8> %accumulator, <vscale x 16 x i8> %0)
+ ret <16 x i8> %partial.reduce
+}
+
+define <8 x i8> @partial_reduce_smax_scalable_neutral_nxv16i8_v8i8(<8 x i8> %accumulator, <vscale x 16 x i8> %0) #0 {
+; CHECK-LABEL: partial_reduce_smax_scalable_neutral_nxv16i8_v8i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: smaxqv v0.16b, p0, z1.b
+; CHECK-NEXT: mov d1, v0.d[1]
+; CHECK-NEXT: smax v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: ret
+entry:
+ %partial.reduce = call <8 x i8> @llvm.vector.partial.reduce.smax.v8i8.v8i8.nxv16i8(<8 x i8> splat (i8 -128), <vscale x 16 x i8> %0)
+ ret <8 x i8> %partial.reduce
+}
+
+define <8 x i8> @partial_reduce_smax_scalable_acc_nxv16i8_v8i8(<8 x i8> %accumulator, <vscale x 16 x i8> %0) #0 {
+; CHECK-LABEL: partial_reduce_smax_scalable_acc_nxv16i8_v8i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: smaxqv v1.16b, p0, z1.b
+; CHECK-NEXT: mov d2, v1.d[1]
+; CHECK-NEXT: smax v1.8b, v1.8b, v2.8b
+; CHECK-NEXT: smax v0.8b, v1.8b, v0.8b
+; CHECK-NEXT: ret
+entry:
+ %partial.reduce = call <8 x i8> @llvm.vector.partial.reduce.smax.v8i8.v8i8.nxv16i8(<8 x i8> %accumulator, <vscale x 16 x i8> %0)
+ ret <8 x i8> %partial.reduce
+}
+
+define <8 x i8> @partial_reduce_smax_scalable_neutral_nxv8i8_v8i8(<8 x i8> %accumulator, <vscale x 8 x i16> %0) #0 {
+; CHECK-LABEL: partial_reduce_smax_scalable_neutral_nxv8i8_v8i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: sxtb z1.h, p0/m, z1.h
+; CHECK-NEXT: smaxqv v0.8h, p0, z1.h
+; CHECK-NEXT: xtn v0.8b, v0.8h
+; CHECK-NEXT: ret
+entry:
+ %leg = trunc <vscale x 8 x i16> %0 to <vscale x 8 x i8>
+ %partial.reduce = call <8 x i8> @llvm.vector.partial.reduce.smax.v8i8.v8i8.nxv8i8(<8 x i8> splat (i8 -128), <vscale x 8 x i8> %leg)
+ ret <8 x i8> %partial.reduce
+}
+
+define <8 x i8> @partial_reduce_smax_scalable_acc_nxv8i8_v8i8(<8 x i8> %accumulator, <vscale x 8 x i16> %0) #0 {
+; CHECK-LABEL: partial_reduce_smax_scalable_acc_nxv8i8_v8i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: sshll v0.8h, v0.8b, #0
+; CHECK-NEXT: sxtb z1.h, p0/m, z1.h
+; CHECK-NEXT: smaxqv v1.8h, p0, z1.h
+; CHECK-NEXT: smax v0.8h, v0.8h, v1.8h
+; CHECK-NEXT: xtn v0.8b, v0.8h
+; CHECK-NEXT: ret
+entry:
+ %leg = trunc <vscale x 8 x i16> %0 to <vscale x 8 x i8>
+ %partial.reduce = call <8 x i8> @llvm.vector.partial.reduce.smax.v8i8.v8i8.nxv8i8(<8 x i8> %accumulator, <vscale x 8 x i8> %leg)
+ ret <8 x i8> %partial.reduce
+}
+
+define <8 x i16> @partial_reduce_smax_scalable_zero_v8i16(<8 x i16> %accumulator, <vscale x 8 x i16> %0) #0 {
+; CHECK-LABEL: partial_reduce_smax_scalable_zero_v8i16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: smaxqv v0.8h, p0, z1.h
+; CHECK-NEXT: ret
+entry:
+ %partial.reduce = call <8 x i16> @llvm.vector.partial.reduce.smax.v8i16.v8i16.nxv8i16(<8 x i16> splat (i16 -32768), <vscale x 8 x i16> %0)
+ ret <8 x i16> %partial.reduce
+}
+
+define <4 x i32> @partial_reduce_smax_scalable_zero_v4i32(<4 x i32> %accumulator, <vscale x 4 x i32> %0) #0 {
+; CHECK-LABEL: partial_reduce_smax_scalable_zero_v4i32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: smaxqv v0.4s, p0, z1.s
+; CHECK-NEXT: ret
+entry:
+ %partial.reduce = call <4 x i32> @llvm.vector.partial.reduce.smax.v4i32.v4i32.nxv4i32(<4 x i32> splat (i32 -2147483648), <vscale x 4 x i32> %0)
+ ret <4 x i32> %partial.reduce
+}
+
+define <2 x i64> @partial_reduce_smax_scalable_zero_v2i64(<2 x i64> %accumulator, <vscale x 2 x i64> %0) #0 {
+; CHECK-LABEL: partial_reduce_smax_scalable_zero_v2i64:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: smaxqv v0.2d, p0, z1.d
+; CHECK-NEXT: ret
+entry:
+ %partial.reduce = call <2 x i64> @llvm.vector.partial.reduce.smax.v2i64.v2i64.nxv2i64(<2 x i64> splat (i64 -9223372036854775808), <vscale x 2 x i64> %0)
+ ret <2 x i64> %partial.reduce
+}
+
+; The lowering is the same for umax/smin/umin, so just test them for one type.
+
+define <16 x i8> @partial_reduce_umax_scalable_zero_v16i8(<16 x i8> %accumulator, <vscale x 16 x i8> %0) #0 {
+; CHECK-LABEL: partial_reduce_umax_scalable_zero_v16i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: umaxqv v0.16b, p0, z1.b
+; CHECK-NEXT: ret
+entry:
+ %partial.reduce = call <16 x i8> @llvm.vector.partial.reduce.umax.v16i8.v16i8.nxv16i8(<16 x i8> zeroinitializer, <vscale x 16 x i8> %0)
+ ret <16 x i8> %partial.reduce
+}
+
+define <8 x i8> @partial_reduce_umax_scalable_neutral_nxv16i8_v8i8(<8 x i8> %accumulator, <vscale x 16 x i8> %0) #0 {
+; CHECK-LABEL: partial_reduce_umax_scalable_neutral_nxv16i8_v8i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: umaxqv v0.16b, p0, z1.b
+; CHECK-NEXT: mov d1, v0.d[1]
+; CHECK-NEXT: umax v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: ret
+entry:
+ %partial.reduce = call <8 x i8> @llvm.vector.partial.reduce.umax.v8i8.v8i8.nxv16i8(<8 x i8> splat (i8 0), <vscale x 16 x i8> %0)
+ ret <8 x i8> %partial.reduce
+}
+
+define <8 x i8> @partial_reduce_umax_scalable_neutral_nxv8i8_v8i8(<8 x i8> %accumulator, <vscale x 8 x i16> %0) #0 {
+; CHECK-LABEL: partial_reduce_umax_scalable_neutral_nxv8i8_v8i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: and z1.h, z1.h, #0xff
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: umaxqv v0.8h, p0, z1.h
+; CHECK-NEXT: xtn v0.8b, v0.8h
+; CHECK-NEXT: ret
+entry:
+ %leg = trunc <vscale x 8 x i16> %0 to <vscale x 8 x i8>
+ %partial.reduce = call <8 x i8> @llvm.vector.partial.reduce.umax.v8i8.v8i8.nxv8i8(<8 x i8> splat (i8 0), <vscale x 8 x i8> %leg)
+ ret <8 x i8> %partial.reduce
+}
+
+define <16 x i8> @partial_reduce_smin_scalable_zero_v16i8(<16 x i8> %accumulator, <vscale x 16 x i8> %0) #0 {
+; CHECK-LABEL: partial_reduce_smin_scalable_zero_v16i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: sminqv v0.16b, p0, z1.b
+; CHECK-NEXT: ret
+entry:
+ %partial.reduce = call <16 x i8> @llvm.vector.partial.reduce.smin.v16i8.v16i8.nxv16i8(<16 x i8> splat (i8 127), <vscale x 16 x i8> %0)
+ ret <16 x i8> %partial.reduce
+}
+
+define <8 x i8> @partial_reduce_smin_scalable_neutral_nxv16i8_v8i8(<8 x i8> %accumulator, <vscale x 16 x i8> %0) #0 {
+; CHECK-LABEL: partial_reduce_smin_scalable_neutral_nxv16i8_v8i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: sminqv v0.16b, p0, z1.b
+; CHECK-NEXT: mov d1, v0.d[1]
+; CHECK-NEXT: smin v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: ret
+entry:
+ %partial.reduce = call <8 x i8> @llvm.vector.partial.reduce.smin.v8i8.v8i8.nxv16i8(<8 x i8> splat (i8 127), <vscale x 16 x i8> %0)
+ ret <8 x i8> %partial.reduce
+}
+
+define <8 x i8> @partial_reduce_smin_scalable_neutral_nxv8i8_v8i8(<8 x i8> %accumulator, <vscale x 8 x i16> %0) #0 {
+; CHECK-LABEL: partial_reduce_smin_scalable_neutral_nxv8i8_v8i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: sxtb z1.h, p0/m, z1.h
+; CHECK-NEXT: sminqv v0.8h, p0, z1.h
+; CHECK-NEXT: xtn v0.8b, v0.8h
+; CHECK-NEXT: ret
+entry:
+ %leg = trunc <vscale x 8 x i16> %0 to <vscale x 8 x i8>
+ %partial.reduce = call <8 x i8> @llvm.vector.partial.reduce.smin.v8i8.v8i8.nxv8i8(<8 x i8> splat (i8 127), <vscale x 8 x i8> %leg)
+ ret <8 x i8> %partial.reduce
+}
+
+define <16 x i8> @partial_reduce_umin_scalable_zero_v16i8(<16 x i8> %accumulator, <vscale x 16 x i8> %0) #0 {
+; CHECK-LABEL: partial_reduce_umin_scalable_zero_v16i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: uminqv v0.16b, p0, z1.b
+; CHECK-NEXT: ret
+entry:
+ %partial.reduce = call <16 x i8> @llvm.vector.partial.reduce.umin.v16i8.v16i8.nxv16i8(<16 x i8> splat (i8 -1), <vscale x 16 x i8> %0)
+ ret <16 x i8> %partial.reduce
+}
+
+define <8 x i8> @partial_reduce_umin_scalable_neutral_nxv16i8_v8i8(<8 x i8> %accumulator, <vscale x 16 x i8> %0) #0 {
+; CHECK-LABEL: partial_reduce_umin_scalable_neutral_nxv16i8_v8i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: uminqv v0.16b, p0, z1.b
+; CHECK-NEXT: mov d1, v0.d[1]
+; CHECK-NEXT: umin v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: ret
+entry:
+ %partial.reduce = call <8 x i8> @llvm.vector.partial.reduce.umin.v8i8.v8i8.nxv16i8(<8 x i8> splat (i8 -1), <vscale x 16 x i8> %0)
+ ret <8 x i8> %partial.reduce
+}
+
+define <8 x i8> @partial_reduce_umin_scalable_neutral_nxv8i8_v8i8(<8 x i8> %accumulator, <vscale x 8 x i16> %0) #0 {
+; CHECK-LABEL: partial_reduce_umin_scalable_neutral_nxv8i8_v8i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: and z1.h, z1.h, #0xff
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: uminqv v0.8h, p0, z1.h
+; CHECK-NEXT: xtn v0.8b, v0.8h
+; CHECK-NEXT: ret
+entry:
+ %leg = trunc <vscale x 8 x i16> %0 to <vscale x 8 x i8>
+ %partial.reduce = call <8 x i8> @llvm.vector.partial.reduce.umin.v8i8.v8i8.nxv8i8(<8 x i8> splat (i8 -1), <vscale x 8 x i8> %leg)
+ ret <8 x i8> %partial.reduce
+}
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-segmented-shuffles.ll b/llvm/test/CodeGen/AArch64/sve2p1-segmented-shuffles.ll
new file mode 100644
index 0000000000000..73643d06229f4
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve2p1-segmented-shuffles.ll
@@ -0,0 +1,1906 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64 -mattr=+sve2p1 -o - %s | FileCheck %s
+
+; trn1
+
+define <vscale x 16 x i8> @trn1_v16i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1) {
+; CHECK-LABEL: trn1_v16i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: trn1 z0.b, z0.b, z1.b
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(
+ <vscale x 16 x i8> %0, <vscale x 16 x i8> %1, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>)
+ ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @trn1_v8i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1) {
+; CHECK-LABEL: trn1_v8i16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: trn1 z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 8 x i16> @llvm.vector.segmented.shuffle.nxv8i16.v8i32(
+ <vscale x 8 x i16> %0, <vscale x 8 x i16> %1, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>)
+ ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 8 x i16> @trn1_v8i8(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1) {
+; CHECK-LABEL: trn1_v8i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: trn1 z0.h, z0.h, z1.h
+; CHECK-NEXT: and z0.h, z0.h, #0xff
+; CHECK-NEXT: ret
+entry:
+ %leg0 = trunc <vscale x 8 x i16> %0 to <vscale x 8 x i8>
+ %leg1 = trunc <vscale x 8 x i16> %1 to <vscale x 8 x i8>
+ %res = call <vscale x 8 x i8> @llvm.vector.segmented.shuffle.nxv8i8.v8i32(
+ <vscale x 8 x i8> %leg0, <vscale x 8 x i8> %leg1, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>)
+ %res.leg = zext <vscale x 8 x i8> %res to <vscale x 8 x i16>
+ ret <vscale x 8 x i16> %res.leg
+}
+
+define <vscale x 8 x half> @trn1_v8f16(<vscale x 8 x half> %0, <vscale x 8 x half> %1) {
+; CHECK-LABEL: trn1_v8f16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: trn1 z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 8 x half> @llvm.vector.segmented.shuffle.nxv8f16.v8i32(
+ <vscale x 8 x half> %0, <vscale x 8 x half> %1, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>)
+ ret <vscale x 8 x half> %res
+}
+
+define <vscale x 8 x bfloat> @trn1_v8bf16(<vscale x 8 x bfloat> %0, <vscale x 8 x bfloat> %1) {
+; CHECK-LABEL: trn1_v8bf16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: trn1 z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 8 x bfloat> @llvm.vector.segmented.shuffle.nxv8bf16.v8i32(
+ <vscale x 8 x bfloat> %0, <vscale x 8 x bfloat> %1, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>)
+ ret <vscale x 8 x bfloat> %res
+}
+
+define <vscale x 4 x i32> @trn1_v4i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1) {
+; CHECK-LABEL: trn1_v4i32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: trn1 z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 4 x i32> @llvm.vector.segmented.shuffle.nxv4i32.v4i32(
+ <vscale x 4 x i32> %0, <vscale x 4 x i32> %1, <4 x i32> <i32 0, i32 4, i32 2, i32 6>)
+ ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @trn1_v4i16(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1) {
+; CHECK-LABEL: trn1_v4i16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: trn1 z0.s, z0.s, z0.s
+; CHECK-NEXT: and z0.s, z0.s, #0xffff
+; CHECK-NEXT: ret
+entry:
+ %leg0 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i16>
+ %leg1 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i16>
+ %res = call <vscale x 4 x i16> @llvm.vector.segmented.shuffle.nxv4i16.v4i32(
+ <vscale x 4 x i16> %leg0, <vscale x 4 x i16> %leg1, <4 x i32> <i32 0, i32 4, i32 2, i32 6>)
+ %res.leg = zext <vscale x 4 x i16> %res to <vscale x 4 x i32>
+ ret <vscale x 4 x i32> %res.leg
+}
+
+define <vscale x 4 x i32> @trn1_v4i8(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1) {
+; CHECK-LABEL: trn1_v4i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: trn1 z0.s, z0.s, z0.s
+; CHECK-NEXT: and z0.s, z0.s, #0xff
+; CHECK-NEXT: ret
+entry:
+ %leg0 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i8>
+ %leg1 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i8>
+ %res = call <vscale x 4 x i8> @llvm.vector.segmented.shuffle.nxv4i8.v4i32(
+ <vscale x 4 x i8> %leg0, <vscale x 4 x i8> %leg1, <4 x i32> <i32 0, i32 4, i32 2, i32 6>)
+ %res.leg = zext <vscale x 4 x i8> %res to <vscale x 4 x i32>
+ ret <vscale x 4 x i32> %res.leg
+}
+
+define <vscale x 4 x float> @trn1_v4f32(<vscale x 4 x float> %0, <vscale x 4 x float> %1) {
+; CHECK-LABEL: trn1_v4f32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: trn1 z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 4 x float> @llvm.vector.segmented.shuffle.nxv4f32.v4i32(
+ <vscale x 4 x float> %0, <vscale x 4 x float> %1, <4 x i32> <i32 0, i32 4, i32 2, i32 6>)
+ ret <vscale x 4 x float> %res
+}
+
+define <vscale x 4 x half> @trn1_v4f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1) {
+; CHECK-LABEL: trn1_v4f16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: trn1 z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 4 x half> @llvm.vector.segmented.shuffle.nxv4f16.v4i32(
+ <vscale x 4 x half> %0, <vscale x 4 x half> %1, <4 x i32> <i32 0, i32 4, i32 2, i32 6>)
+ ret <vscale x 4 x half> %res
+}
+
+define <vscale x 2 x i64> @trn1_uzp1_zip1_v2i64(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1) {
+; CHECK-LABEL: trn1_uzp1_zip1_v2i64:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: zipq1 z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 2 x i64> @llvm.vector.segmented.shuffle.nxv2i64.v2i32(
+ <vscale x 2 x i64> %0, <vscale x 2 x i64> %1, <2 x i32> <i32 0, i32 2>)
+ ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @trn1_uzp1_zip1_v2i32(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1) {
+; CHECK-LABEL: trn1_uzp1_zip1_v2i32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: zipq1 z0.d, z0.d, z0.d
+; CHECK-NEXT: and z0.d, z0.d, #0xffffffff
+; CHECK-NEXT: ret
+entry:
+ %leg0 = trunc <vscale x 2 x i64> %0 to <vscale x 2 x i32>
+ %leg1 = trunc <vscale x 2 x i64> %0 to <vscale x 2 x i32>
+ %res = call <vscale x 2 x i32> @llvm.vector.segmented.shuffle.nxv2i32.v2i32(
+ <vscale x 2 x i32> %leg0, <vscale x 2 x i32> %leg1, <2 x i32> <i32 0, i32 2>)
+ %res.leg = zext <vscale x 2 x i32> %res to <vscale x 2 x i64>
+ ret <vscale x 2 x i64> %res.leg
+}
+
+define <vscale x 2 x double> @trn1_uzp1_zip1_v2f64(<vscale x 2 x double> %0, <vscale x 2 x double> %1) {
+; CHECK-LABEL: trn1_uzp1_zip1_v2f64:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: zipq1 z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 2 x double> @llvm.vector.segmented.shuffle.nxv2f64.v2i32(
+ <vscale x 2 x double> %0, <vscale x 2 x double> %1, <2 x i32> <i32 0, i32 2>)
+ ret <vscale x 2 x double> %res
+}
+
+define <vscale x 2 x float> @trn1_uzp1_zip1_v2f32(<vscale x 2 x float> %0, <vscale x 2 x float> %1) {
+; CHECK-LABEL: trn1_uzp1_zip1_v2f32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: zipq1 z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 2 x float> @llvm.vector.segmented.shuffle.nxv2f32.v2i32(
+ <vscale x 2 x float> %0, <vscale x 2 x float> %1, <2 x i32> <i32 0, i32 2>)
+ ret <vscale x 2 x float> %res
+}
+
+define <vscale x 4 x float> @trn1_v_undef(<vscale x 4 x float> %0, <vscale x 4 x float> %1) {
+; CHECK-LABEL: trn1_v_undef:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: trn1 z0.s, z0.s, z0.s
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 4 x float> @llvm.vector.segmented.shuffle.nxv4f32.v4i32(
+ <vscale x 4 x float> %0, <vscale x 4 x float> poison, <4 x i32> <i32 0, i32 0, i32 2, i32 2>)
+ ret <vscale x 4 x float> %res
+}
+
+
+; trn2
+
+define <vscale x 16 x i8> @trn2_v16i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1) {
+; CHECK-LABEL: trn2_v16i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: trn2 z0.b, z0.b, z1.b
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(
+ <vscale x 16 x i8> %0, <vscale x 16 x i8> %1, <16 x i32> <i32 1, i32 17, i32 3, i32 19, i32 5, i32 21, i32 7, i32 23, i32 9, i32 25, i32 11, i32 27, i32 13, i32 29, i32 15, i32 31>)
+ ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @trn2_v8i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1) {
+; CHECK-LABEL: trn2_v8i16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: trn2 z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 8 x i16> @llvm.vector.segmented.shuffle.nxv8i16.v8i32(
+ <vscale x 8 x i16> %0, <vscale x 8 x i16> %1, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>)
+ ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 8 x i16> @trn2_v8i8(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1) {
+; CHECK-LABEL: trn2_v8i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: trn2 z0.h, z0.h, z1.h
+; CHECK-NEXT: and z0.h, z0.h, #0xff
+; CHECK-NEXT: ret
+entry:
+ %leg0 = trunc <vscale x 8 x i16> %0 to <vscale x 8 x i8>
+ %leg1 = trunc <vscale x 8 x i16> %1 to <vscale x 8 x i8>
+ %res = call <vscale x 8 x i8> @llvm.vector.segmented.shuffle.nxv8i8.v8i32(
+ <vscale x 8 x i8> %leg0, <vscale x 8 x i8> %leg1, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>)
+ %res.leg = zext <vscale x 8 x i8> %res to <vscale x 8 x i16>
+ ret <vscale x 8 x i16> %res.leg
+}
+
+define <vscale x 8 x half> @trn2_v8f16(<vscale x 8 x half> %0, <vscale x 8 x half> %1) {
+; CHECK-LABEL: trn2_v8f16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: trn2 z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 8 x half> @llvm.vector.segmented.shuffle.nxv8f16.v8i32(
+ <vscale x 8 x half> %0, <vscale x 8 x half> %1, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>)
+ ret <vscale x 8 x half> %res
+}
+
+define <vscale x 8 x bfloat> @trn2_v8bf16(<vscale x 8 x bfloat> %0, <vscale x 8 x bfloat> %1) {
+; CHECK-LABEL: trn2_v8bf16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: trn2 z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 8 x bfloat> @llvm.vector.segmented.shuffle.nxv8bf16.v8i32(
+ <vscale x 8 x bfloat> %0, <vscale x 8 x bfloat> %1, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>)
+ ret <vscale x 8 x bfloat> %res
+}
+
+define <vscale x 4 x i32> @trn2_v4i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1) {
+; CHECK-LABEL: trn2_v4i32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: trn2 z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 4 x i32> @llvm.vector.segmented.shuffle.nxv4i32.v4i32(
+ <vscale x 4 x i32> %0, <vscale x 4 x i32> %1, <4 x i32> <i32 1, i32 5, i32 3, i32 7>)
+ ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @trn2_v4i16(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1) {
+; CHECK-LABEL: trn2_v4i16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: trn2 z0.s, z0.s, z0.s
+; CHECK-NEXT: and z0.s, z0.s, #0xffff
+; CHECK-NEXT: ret
+entry:
+ %leg0 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i16>
+ %leg1 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i16>
+ %res = call <vscale x 4 x i16> @llvm.vector.segmented.shuffle.nxv4i16.v4i32(
+ <vscale x 4 x i16> %leg0, <vscale x 4 x i16> %leg1, <4 x i32> <i32 1, i32 5, i32 3, i32 7>)
+ %res.leg = zext <vscale x 4 x i16> %res to <vscale x 4 x i32>
+ ret <vscale x 4 x i32> %res.leg
+}
+
+define <vscale x 4 x i32> @trn2_v4i8(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1) {
+; CHECK-LABEL: trn2_v4i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: trn2 z0.s, z0.s, z0.s
+; CHECK-NEXT: and z0.s, z0.s, #0xff
+; CHECK-NEXT: ret
+entry:
+ %leg0 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i8>
+ %leg1 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i8>
+ %res = call <vscale x 4 x i8> @llvm.vector.segmented.shuffle.nxv4i8.v4i32(
+ <vscale x 4 x i8> %leg0, <vscale x 4 x i8> %leg1, <4 x i32> <i32 1, i32 5, i32 3, i32 7>)
+ %res.leg = zext <vscale x 4 x i8> %res to <vscale x 4 x i32>
+ ret <vscale x 4 x i32> %res.leg
+}
+
+define <vscale x 4 x float> @trn2_v4f32(<vscale x 4 x float> %0, <vscale x 4 x float> %1) {
+; CHECK-LABEL: trn2_v4f32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: trn2 z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 4 x float> @llvm.vector.segmented.shuffle.nxv4f32.v4i32(
+ <vscale x 4 x float> %0, <vscale x 4 x float> %1, <4 x i32> <i32 1, i32 5, i32 3, i32 7>)
+ ret <vscale x 4 x float> %res
+}
+
+define <vscale x 4 x half> @trn2_v4f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1) {
+; CHECK-LABEL: trn2_v4f16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: trn2 z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 4 x half> @llvm.vector.segmented.shuffle.nxv4f16.v4i32(
+ <vscale x 4 x half> %0, <vscale x 4 x half> %1, <4 x i32> <i32 1, i32 5, i32 3, i32 7>)
+ ret <vscale x 4 x half> %res
+}
+
+define <vscale x 2 x i64> @trn2_uzp2_zip2_v2i64(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1) {
+; CHECK-LABEL: trn2_uzp2_zip2_v2i64:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: zipq2 z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 2 x i64> @llvm.vector.segmented.shuffle.nxv2i64.v2i32(
+ <vscale x 2 x i64> %0, <vscale x 2 x i64> %1, <2 x i32> <i32 1, i32 3>)
+ ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @trn2_uzp2_zip2_v2i32(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1) {
+; CHECK-LABEL: trn2_uzp2_zip2_v2i32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: zipq2 z0.d, z0.d, z0.d
+; CHECK-NEXT: and z0.d, z0.d, #0xffffffff
+; CHECK-NEXT: ret
+entry:
+ %leg0 = trunc <vscale x 2 x i64> %0 to <vscale x 2 x i32>
+ %leg1 = trunc <vscale x 2 x i64> %0 to <vscale x 2 x i32>
+ %res = call <vscale x 2 x i32> @llvm.vector.segmented.shuffle.nxv2i32.v2i32(
+ <vscale x 2 x i32> %leg0, <vscale x 2 x i32> %leg1, <2 x i32> <i32 1, i32 3>)
+ %res.leg = zext <vscale x 2 x i32> %res to <vscale x 2 x i64>
+ ret <vscale x 2 x i64> %res.leg
+}
+
+define <vscale x 2 x double> @trn2_uzp2_zip2_v2f64(<vscale x 2 x double> %0, <vscale x 2 x double> %1) {
+; CHECK-LABEL: trn2_uzp2_zip2_v2f64:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: zipq2 z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 2 x double> @llvm.vector.segmented.shuffle.nxv2f64.v2i32(
+ <vscale x 2 x double> %0, <vscale x 2 x double> %1, <2 x i32> <i32 1, i32 3>)
+ ret <vscale x 2 x double> %res
+}
+
+define <vscale x 2 x float> @trn2_uzp2_zip2_v2f32(<vscale x 2 x float> %0, <vscale x 2 x float> %1) {
+; CHECK-LABEL: trn2_uzp2_zip2_v2f32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: zipq2 z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 2 x float> @llvm.vector.segmented.shuffle.nxv2f32.v2i32(
+ <vscale x 2 x float> %0, <vscale x 2 x float> %1, <2 x i32> <i32 1, i32 3>)
+ ret <vscale x 2 x float> %res
+}
+
+define <vscale x 4 x float> @trn2_v_undef(<vscale x 4 x float> %0, <vscale x 4 x float> %1) {
+; CHECK-LABEL: trn2_v_undef:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: trn2 z0.s, z0.s, z0.s
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 4 x float> @llvm.vector.segmented.shuffle.nxv4f32.v4i32(
+ <vscale x 4 x float> %0, <vscale x 4 x float> poison, <4 x i32> <i32 1, i32 1, i32 3, i32 3>)
+ ret <vscale x 4 x float> %res
+}
+
+
+
+; uzp1
+
+define <vscale x 16 x i8> @uzp1_v16i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1) {
+; CHECK-LABEL: uzp1_v16i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: uzpq1 z0.b, z0.b, z1.b
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(
+ <vscale x 16 x i8> %0, <vscale x 16 x i8> %1, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>)
+ ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @uzp1_v8i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1) {
+; CHECK-LABEL: uzp1_v8i16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: uzpq1 z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 8 x i16> @llvm.vector.segmented.shuffle.nxv8i16.v8i32(
+ <vscale x 8 x i16> %0, <vscale x 8 x i16> %1, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>)
+ ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 8 x i16> @uzp1_v8i8(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1) {
+; CHECK-LABEL: uzp1_v8i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: uzpq1 z0.h, z0.h, z1.h
+; CHECK-NEXT: and z0.h, z0.h, #0xff
+; CHECK-NEXT: ret
+entry:
+ %leg0 = trunc <vscale x 8 x i16> %0 to <vscale x 8 x i8>
+ %leg1 = trunc <vscale x 8 x i16> %1 to <vscale x 8 x i8>
+ %res = call <vscale x 8 x i8> @llvm.vector.segmented.shuffle.nxv8i8.v8i32(
+ <vscale x 8 x i8> %leg0, <vscale x 8 x i8> %leg1, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>)
+ %res.leg = zext <vscale x 8 x i8> %res to <vscale x 8 x i16>
+ ret <vscale x 8 x i16> %res.leg
+}
+
+define <vscale x 8 x half> @uzp1_v8f16(<vscale x 8 x half> %0, <vscale x 8 x half> %1) {
+; CHECK-LABEL: uzp1_v8f16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: uzpq1 z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 8 x half> @llvm.vector.segmented.shuffle.nxv8f16.v8i32(
+ <vscale x 8 x half> %0, <vscale x 8 x half> %1, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>)
+ ret <vscale x 8 x half> %res
+}
+
+define <vscale x 8 x bfloat> @uzp1_v8bf16(<vscale x 8 x bfloat> %0, <vscale x 8 x bfloat> %1) {
+; CHECK-LABEL: uzp1_v8bf16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: uzpq1 z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 8 x bfloat> @llvm.vector.segmented.shuffle.nxv8bf16.v8i32(
+ <vscale x 8 x bfloat> %0, <vscale x 8 x bfloat> %1, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>)
+ ret <vscale x 8 x bfloat> %res
+}
+
+define <vscale x 4 x i32> @uzp1_v4i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1) {
+; CHECK-LABEL: uzp1_v4i32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: uzpq1 z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 4 x i32> @llvm.vector.segmented.shuffle.nxv4i32.v4i32(
+ <vscale x 4 x i32> %0, <vscale x 4 x i32> %1, <4 x i32> <i32 0, i32 2, i32 4, i32 6>)
+ ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @uzp1_v4i16(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1) {
+; CHECK-LABEL: uzp1_v4i16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: uzpq1 z0.s, z0.s, z0.s
+; CHECK-NEXT: and z0.s, z0.s, #0xffff
+; CHECK-NEXT: ret
+entry:
+ %leg0 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i16>
+ %leg1 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i16>
+ %res = call <vscale x 4 x i16> @llvm.vector.segmented.shuffle.nxv4i16.v4i32(
+ <vscale x 4 x i16> %leg0, <vscale x 4 x i16> %leg1, <4 x i32> <i32 0, i32 2, i32 4, i32 6>)
+ %res.leg = zext <vscale x 4 x i16> %res to <vscale x 4 x i32>
+ ret <vscale x 4 x i32> %res.leg
+}
+
+define <vscale x 4 x i32> @uzp1_v4i8(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1) {
+; CHECK-LABEL: uzp1_v4i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: uzpq1 z0.s, z0.s, z0.s
+; CHECK-NEXT: and z0.s, z0.s, #0xff
+; CHECK-NEXT: ret
+entry:
+ %leg0 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i8>
+ %leg1 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i8>
+ %res = call <vscale x 4 x i8> @llvm.vector.segmented.shuffle.nxv4i8.v4i32(
+ <vscale x 4 x i8> %leg0, <vscale x 4 x i8> %leg1, <4 x i32> <i32 0, i32 2, i32 4, i32 6>)
+ %res.leg = zext <vscale x 4 x i8> %res to <vscale x 4 x i32>
+ ret <vscale x 4 x i32> %res.leg
+}
+
+define <vscale x 4 x float> @uzp1_v4f32(<vscale x 4 x float> %0, <vscale x 4 x float> %1) {
+; CHECK-LABEL: uzp1_v4f32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: uzpq1 z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 4 x float> @llvm.vector.segmented.shuffle.nxv4f32.v4i32(
+ <vscale x 4 x float> %0, <vscale x 4 x float> %1, <4 x i32> <i32 0, i32 2, i32 4, i32 6>)
+ ret <vscale x 4 x float> %res
+}
+
+define <vscale x 4 x half> @uzp1_v4f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1) {
+; CHECK-LABEL: uzp1_v4f16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: uzpq1 z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 4 x half> @llvm.vector.segmented.shuffle.nxv4f16.v4i32(
+ <vscale x 4 x half> %0, <vscale x 4 x half> %1, <4 x i32> <i32 0, i32 2, i32 4, i32 6>)
+ ret <vscale x 4 x half> %res
+}
+
+define <vscale x 4 x float> @uzp1_v_undef(<vscale x 4 x float> %0, <vscale x 4 x float> %1) {
+; CHECK-LABEL: uzp1_v_undef:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: uzpq1 z0.s, z0.s, z0.s
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 4 x float> @llvm.vector.segmented.shuffle.nxv4f32.v4i32(
+ <vscale x 4 x float> %0, <vscale x 4 x float> poison, <4 x i32> <i32 0, i32 2, i32 0, i32 2>)
+ ret <vscale x 4 x float> %res
+}
+
+
+
+; uzp2
+
+define <vscale x 16 x i8> @uzp2_v16i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1) {
+; CHECK-LABEL: uzp2_v16i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: uzpq2 z0.b, z0.b, z1.b
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(
+ <vscale x 16 x i8> %0, <vscale x 16 x i8> %1, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>)
+ ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @uzp2_v8i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1) {
+; CHECK-LABEL: uzp2_v8i16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: uzpq2 z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 8 x i16> @llvm.vector.segmented.shuffle.nxv8i16.v8i32(
+ <vscale x 8 x i16> %0, <vscale x 8 x i16> %1, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>)
+ ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 8 x i16> @uzp2_v8i8(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1) {
+; CHECK-LABEL: uzp2_v8i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: uzpq2 z0.h, z0.h, z1.h
+; CHECK-NEXT: and z0.h, z0.h, #0xff
+; CHECK-NEXT: ret
+entry:
+ %leg0 = trunc <vscale x 8 x i16> %0 to <vscale x 8 x i8>
+ %leg1 = trunc <vscale x 8 x i16> %1 to <vscale x 8 x i8>
+ %res = call <vscale x 8 x i8> @llvm.vector.segmented.shuffle.nxv8i8.v8i32(
+ <vscale x 8 x i8> %leg0, <vscale x 8 x i8> %leg1, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>)
+ %res.leg = zext <vscale x 8 x i8> %res to <vscale x 8 x i16>
+ ret <vscale x 8 x i16> %res.leg
+}
+
+define <vscale x 8 x half> @uzp2_v8f16(<vscale x 8 x half> %0, <vscale x 8 x half> %1) {
+; CHECK-LABEL: uzp2_v8f16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: uzpq2 z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 8 x half> @llvm.vector.segmented.shuffle.nxv8f16.v8i32(
+ <vscale x 8 x half> %0, <vscale x 8 x half> %1, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>)
+ ret <vscale x 8 x half> %res
+}
+
+define <vscale x 8 x bfloat> @uzp2_v8bf16(<vscale x 8 x bfloat> %0, <vscale x 8 x bfloat> %1) {
+; CHECK-LABEL: uzp2_v8bf16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: uzpq2 z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 8 x bfloat> @llvm.vector.segmented.shuffle.nxv8bf16.v8i32(
+ <vscale x 8 x bfloat> %0, <vscale x 8 x bfloat> %1, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>)
+ ret <vscale x 8 x bfloat> %res
+}
+
+define <vscale x 4 x i32> @uzp2_v4i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1) {
+; CHECK-LABEL: uzp2_v4i32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: uzpq2 z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 4 x i32> @llvm.vector.segmented.shuffle.nxv4i32.v4i32(
+ <vscale x 4 x i32> %0, <vscale x 4 x i32> %1, <4 x i32> <i32 1, i32 3, i32 5, i32 7>)
+ ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @uzp2_v4i16(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1) {
+; CHECK-LABEL: uzp2_v4i16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: uzpq2 z0.s, z0.s, z0.s
+; CHECK-NEXT: and z0.s, z0.s, #0xffff
+; CHECK-NEXT: ret
+entry:
+ %leg0 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i16>
+ %leg1 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i16>
+ %res = call <vscale x 4 x i16> @llvm.vector.segmented.shuffle.nxv4i16.v4i32(
+ <vscale x 4 x i16> %leg0, <vscale x 4 x i16> %leg1, <4 x i32> <i32 1, i32 3, i32 5, i32 7>)
+ %res.leg = zext <vscale x 4 x i16> %res to <vscale x 4 x i32>
+ ret <vscale x 4 x i32> %res.leg
+}
+
+define <vscale x 4 x i32> @uzp2_v4i8(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1) {
+; CHECK-LABEL: uzp2_v4i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: uzpq2 z0.s, z0.s, z0.s
+; CHECK-NEXT: and z0.s, z0.s, #0xff
+; CHECK-NEXT: ret
+entry:
+ %leg0 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i8>
+ %leg1 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i8>
+ %res = call <vscale x 4 x i8> @llvm.vector.segmented.shuffle.nxv4i8.v4i32(
+ <vscale x 4 x i8> %leg0, <vscale x 4 x i8> %leg1, <4 x i32> <i32 1, i32 3, i32 5, i32 7>)
+ %res.leg = zext <vscale x 4 x i8> %res to <vscale x 4 x i32>
+ ret <vscale x 4 x i32> %res.leg
+}
+
+define <vscale x 4 x float> @uzp2_v4f32(<vscale x 4 x float> %0, <vscale x 4 x float> %1) {
+; CHECK-LABEL: uzp2_v4f32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: uzpq2 z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 4 x float> @llvm.vector.segmented.shuffle.nxv4f32.v4i32(
+ <vscale x 4 x float> %0, <vscale x 4 x float> %1, <4 x i32> <i32 1, i32 3, i32 5, i32 7>)
+ ret <vscale x 4 x float> %res
+}
+
+define <vscale x 4 x half> @uzp2_v4f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1) {
+; CHECK-LABEL: uzp2_v4f16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: uzpq2 z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 4 x half> @llvm.vector.segmented.shuffle.nxv4f16.v4i32(
+ <vscale x 4 x half> %0, <vscale x 4 x half> %1, <4 x i32> <i32 1, i32 3, i32 5, i32 7>)
+ ret <vscale x 4 x half> %res
+}
+
+define <vscale x 4 x float> @uzp2_v_undef(<vscale x 4 x float> %0, <vscale x 4 x float> %1) {
+; CHECK-LABEL: uzp2_v_undef:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: uzpq2 z0.s, z0.s, z0.s
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 4 x float> @llvm.vector.segmented.shuffle.nxv4f32.v4i32(
+ <vscale x 4 x float> %0, <vscale x 4 x float> poison, <4 x i32> <i32 1, i32 3, i32 1, i32 3>)
+ ret <vscale x 4 x float> %res
+}
+
+
+
+; zip1
+
+define <vscale x 16 x i8> @zip1_v16i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1) {
+; CHECK-LABEL: zip1_v16i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: zipq1 z0.b, z0.b, z1.b
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(
+ <vscale x 16 x i8> %0, <vscale x 16 x i8> %1, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23>)
+ ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @zip1_v8i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1) {
+; CHECK-LABEL: zip1_v8i16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: zipq1 z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 8 x i16> @llvm.vector.segmented.shuffle.nxv8i16.v8i32(
+ <vscale x 8 x i16> %0, <vscale x 8 x i16> %1, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>)
+ ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 8 x i16> @zip1_v8i8(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1) {
+; CHECK-LABEL: zip1_v8i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: zipq1 z0.h, z0.h, z1.h
+; CHECK-NEXT: and z0.h, z0.h, #0xff
+; CHECK-NEXT: ret
+entry:
+ %leg0 = trunc <vscale x 8 x i16> %0 to <vscale x 8 x i8>
+ %leg1 = trunc <vscale x 8 x i16> %1 to <vscale x 8 x i8>
+ %res = call <vscale x 8 x i8> @llvm.vector.segmented.shuffle.nxv8i8.v8i32(
+ <vscale x 8 x i8> %leg0, <vscale x 8 x i8> %leg1, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>)
+ %res.leg = zext <vscale x 8 x i8> %res to <vscale x 8 x i16>
+ ret <vscale x 8 x i16> %res.leg
+}
+
+define <vscale x 8 x half> @zip1_v8f16(<vscale x 8 x half> %0, <vscale x 8 x half> %1) {
+; CHECK-LABEL: zip1_v8f16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: zipq1 z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 8 x half> @llvm.vector.segmented.shuffle.nxv8f16.v8i32(
+ <vscale x 8 x half> %0, <vscale x 8 x half> %1, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>)
+ ret <vscale x 8 x half> %res
+}
+
+define <vscale x 8 x bfloat> @zip1_v8bf16(<vscale x 8 x bfloat> %0, <vscale x 8 x bfloat> %1) {
+; CHECK-LABEL: zip1_v8bf16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: zipq1 z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 8 x bfloat> @llvm.vector.segmented.shuffle.nxv8bf16.v8i32(
+ <vscale x 8 x bfloat> %0, <vscale x 8 x bfloat> %1, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>)
+ ret <vscale x 8 x bfloat> %res
+}
+
+define <vscale x 4 x i32> @zip1_v4i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1) {
+; CHECK-LABEL: zip1_v4i32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: zipq1 z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 4 x i32> @llvm.vector.segmented.shuffle.nxv4i32.v4i32(
+ <vscale x 4 x i32> %0, <vscale x 4 x i32> %1, <4 x i32> <i32 0, i32 4, i32 1, i32 5>)
+ ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @zip1_v4i16(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1) {
+; CHECK-LABEL: zip1_v4i16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: zipq1 z0.s, z0.s, z0.s
+; CHECK-NEXT: and z0.s, z0.s, #0xffff
+; CHECK-NEXT: ret
+entry:
+ %leg0 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i16>
+ %leg1 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i16>
+ %res = call <vscale x 4 x i16> @llvm.vector.segmented.shuffle.nxv4i16.v4i32(
+ <vscale x 4 x i16> %leg0, <vscale x 4 x i16> %leg1, <4 x i32> <i32 0, i32 4, i32 1, i32 5>)
+ %res.leg = zext <vscale x 4 x i16> %res to <vscale x 4 x i32>
+ ret <vscale x 4 x i32> %res.leg
+}
+
+define <vscale x 4 x i32> @zip1_v4i8(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1) {
+; CHECK-LABEL: zip1_v4i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: zipq1 z0.s, z0.s, z0.s
+; CHECK-NEXT: and z0.s, z0.s, #0xff
+; CHECK-NEXT: ret
+entry:
+ %leg0 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i8>
+ %leg1 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i8>
+ %res = call <vscale x 4 x i8> @llvm.vector.segmented.shuffle.nxv4i8.v4i32(
+ <vscale x 4 x i8> %leg0, <vscale x 4 x i8> %leg1, <4 x i32> <i32 0, i32 4, i32 1, i32 5>)
+ %res.leg = zext <vscale x 4 x i8> %res to <vscale x 4 x i32>
+ ret <vscale x 4 x i32> %res.leg
+}
+
+define <vscale x 4 x float> @zip1_v4f32(<vscale x 4 x float> %0, <vscale x 4 x float> %1) {
+; CHECK-LABEL: zip1_v4f32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: zipq1 z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 4 x float> @llvm.vector.segmented.shuffle.nxv4f32.v4i32(
+ <vscale x 4 x float> %0, <vscale x 4 x float> %1, <4 x i32> <i32 0, i32 4, i32 1, i32 5>)
+ ret <vscale x 4 x float> %res
+}
+
+define <vscale x 4 x half> @zip1_v4f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1) {
+; CHECK-LABEL: zip1_v4f16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: zipq1 z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 4 x half> @llvm.vector.segmented.shuffle.nxv4f16.v4i32(
+ <vscale x 4 x half> %0, <vscale x 4 x half> %1, <4 x i32> <i32 0, i32 4, i32 1, i32 5>)
+ ret <vscale x 4 x half> %res
+}
+
+define <vscale x 4 x float> @zip1_v_undef(<vscale x 4 x float> %0, <vscale x 4 x float> %1) {
+; CHECK-LABEL: zip1_v_undef:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: zipq1 z0.s, z0.s, z0.s
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 4 x float> @llvm.vector.segmented.shuffle.nxv4f32.v4i32(
+ <vscale x 4 x float> %0, <vscale x 4 x float> poison, <4 x i32> <i32 0, i32 0, i32 1, i32 1>)
+ ret <vscale x 4 x float> %res
+}
+
+
+
+; zip2
+
+define <vscale x 16 x i8> @zip2_v16i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1) {
+; CHECK-LABEL: zip2_v16i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: zipq2 z0.b, z0.b, z1.b
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(
+ <vscale x 16 x i8> %0, <vscale x 16 x i8> %1, <16 x i32> <i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>)
+ ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @zip2_v8i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1) {
+; CHECK-LABEL: zip2_v8i16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: zipq2 z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 8 x i16> @llvm.vector.segmented.shuffle.nxv8i16.v8i32(
+ <vscale x 8 x i16> %0, <vscale x 8 x i16> %1, <8 x i32> <i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>)
+ ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 8 x i16> @zip2_v8i8(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1) {
+; CHECK-LABEL: zip2_v8i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: zipq2 z0.h, z0.h, z1.h
+; CHECK-NEXT: and z0.h, z0.h, #0xff
+; CHECK-NEXT: ret
+entry:
+ %leg0 = trunc <vscale x 8 x i16> %0 to <vscale x 8 x i8>
+ %leg1 = trunc <vscale x 8 x i16> %1 to <vscale x 8 x i8>
+ %res = call <vscale x 8 x i8> @llvm.vector.segmented.shuffle.nxv8i8.v8i32(
+ <vscale x 8 x i8> %leg0, <vscale x 8 x i8> %leg1, <8 x i32> <i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>)
+ %res.leg = zext <vscale x 8 x i8> %res to <vscale x 8 x i16>
+ ret <vscale x 8 x i16> %res.leg
+}
+
+define <vscale x 8 x half> @zip2_v8f16(<vscale x 8 x half> %0, <vscale x 8 x half> %1) {
+; CHECK-LABEL: zip2_v8f16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: zipq2 z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 8 x half> @llvm.vector.segmented.shuffle.nxv8f16.v8i32(
+ <vscale x 8 x half> %0, <vscale x 8 x half> %1, <8 x i32> <i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>)
+ ret <vscale x 8 x half> %res
+}
+
+define <vscale x 8 x bfloat> @zip2_v8bf16(<vscale x 8 x bfloat> %0, <vscale x 8 x bfloat> %1) {
+; CHECK-LABEL: zip2_v8bf16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: zipq2 z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 8 x bfloat> @llvm.vector.segmented.shuffle.nxv8bf16.v8i32(
+ <vscale x 8 x bfloat> %0, <vscale x 8 x bfloat> %1, <8 x i32> <i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>)
+ ret <vscale x 8 x bfloat> %res
+}
+
+define <vscale x 4 x i32> @zip2_v4i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1) {
+; CHECK-LABEL: zip2_v4i32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: zipq2 z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 4 x i32> @llvm.vector.segmented.shuffle.nxv4i32.v4i32(
+ <vscale x 4 x i32> %0, <vscale x 4 x i32> %1, <4 x i32> <i32 2, i32 6, i32 3, i32 7>)
+ ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @zip2_v4i16(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1) {
+; CHECK-LABEL: zip2_v4i16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: zipq2 z0.s, z0.s, z0.s
+; CHECK-NEXT: and z0.s, z0.s, #0xffff
+; CHECK-NEXT: ret
+entry:
+ %leg0 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i16>
+ %leg1 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i16>
+ %res = call <vscale x 4 x i16> @llvm.vector.segmented.shuffle.nxv4i16.v4i32(
+ <vscale x 4 x i16> %leg0, <vscale x 4 x i16> %leg1, <4 x i32> <i32 2, i32 6, i32 3, i32 7>)
+ %res.leg = zext <vscale x 4 x i16> %res to <vscale x 4 x i32>
+ ret <vscale x 4 x i32> %res.leg
+}
+
+define <vscale x 4 x i32> @zip2_v4i8(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1) {
+; CHECK-LABEL: zip2_v4i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: zipq2 z0.s, z0.s, z0.s
+; CHECK-NEXT: and z0.s, z0.s, #0xff
+; CHECK-NEXT: ret
+entry:
+ %leg0 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i8>
+ %leg1 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i8>
+ %res = call <vscale x 4 x i8> @llvm.vector.segmented.shuffle.nxv4i8.v4i32(
+ <vscale x 4 x i8> %leg0, <vscale x 4 x i8> %leg1, <4 x i32> <i32 2, i32 6, i32 3, i32 7>)
+ %res.leg = zext <vscale x 4 x i8> %res to <vscale x 4 x i32>
+ ret <vscale x 4 x i32> %res.leg
+}
+
+define <vscale x 4 x float> @zip2_v4f32(<vscale x 4 x float> %0, <vscale x 4 x float> %1) {
+; CHECK-LABEL: zip2_v4f32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: zipq2 z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 4 x float> @llvm.vector.segmented.shuffle.nxv4f32.v4i32(
+ <vscale x 4 x float> %0, <vscale x 4 x float> %1, <4 x i32> <i32 2, i32 6, i32 3, i32 7>)
+ ret <vscale x 4 x float> %res
+}
+
+define <vscale x 4 x half> @zip2_v4f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1) {
+; CHECK-LABEL: zip2_v4f16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: zipq2 z0.s, z0.s, z1.s
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 4 x half> @llvm.vector.segmented.shuffle.nxv4f16.v4i32(
+ <vscale x 4 x half> %0, <vscale x 4 x half> %1, <4 x i32> <i32 2, i32 6, i32 3, i32 7>)
+ ret <vscale x 4 x half> %res
+}
+
+define <vscale x 4 x float> @zip2_v_undef(<vscale x 4 x float> %0, <vscale x 4 x float> %1) {
+; CHECK-LABEL: zip2_v_undef:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: zipq2 z0.s, z0.s, z0.s
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 4 x float> @llvm.vector.segmented.shuffle.nxv4f32.v4i32(
+ <vscale x 4 x float> %0, <vscale x 4 x float> poison, <4 x i32> <i32 2, i32 2, i32 3, i32 3>)
+ ret <vscale x 4 x float> %res
+}
+
+
+
+; ext
+
+define <vscale x 16 x i8> @ext_v16i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1) {
+; CHECK-LABEL: ext_v16i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: extq z0.b, z0.b, z1.b, #5
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(
+ <vscale x 16 x i8> %0, <vscale x 16 x i8> %1, <16 x i32> <i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20>)
+ ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @ext_v8i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1) {
+; CHECK-LABEL: ext_v8i16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: extq z0.b, z0.b, z1.b, #10
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 8 x i16> @llvm.vector.segmented.shuffle.nxv8i16.v8i32(
+ <vscale x 8 x i16> %0, <vscale x 8 x i16> %1, <8 x i32> <i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12>)
+ ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 8 x i16> @ext_v8i8(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1) {
+; CHECK-LABEL: ext_v8i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: extq z0.b, z0.b, z1.b, #10
+; CHECK-NEXT: and z0.h, z0.h, #0xff
+; CHECK-NEXT: ret
+entry:
+ %leg0 = trunc <vscale x 8 x i16> %0 to <vscale x 8 x i8>
+ %leg1 = trunc <vscale x 8 x i16> %1 to <vscale x 8 x i8>
+ %res = call <vscale x 8 x i8> @llvm.vector.segmented.shuffle.nxv8i8.v8i32(
+ <vscale x 8 x i8> %leg0, <vscale x 8 x i8> %leg1, <8 x i32> <i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12>)
+ %res.leg = zext <vscale x 8 x i8> %res to <vscale x 8 x i16>
+ ret <vscale x 8 x i16> %res.leg
+}
+
+define <vscale x 8 x half> @ext_v8f16(<vscale x 8 x half> %0, <vscale x 8 x half> %1) {
+; CHECK-LABEL: ext_v8f16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: extq z0.b, z0.b, z1.b, #10
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 8 x half> @llvm.vector.segmented.shuffle.nxv8f16.v8i32(
+ <vscale x 8 x half> %0, <vscale x 8 x half> %1, <8 x i32> <i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12>)
+ ret <vscale x 8 x half> %res
+}
+
+define <vscale x 8 x bfloat> @ext_v8bf16(<vscale x 8 x bfloat> %0, <vscale x 8 x bfloat> %1) {
+; CHECK-LABEL: ext_v8bf16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: extq z0.b, z0.b, z1.b, #10
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 8 x bfloat> @llvm.vector.segmented.shuffle.nxv8bf16.v8i32(
+ <vscale x 8 x bfloat> %0, <vscale x 8 x bfloat> %1, <8 x i32> <i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12>)
+ ret <vscale x 8 x bfloat> %res
+}
+
+define <vscale x 4 x i32> @ext_v4i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1) {
+; CHECK-LABEL: ext_v4i32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: extq z0.b, z0.b, z1.b, #12
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 4 x i32> @llvm.vector.segmented.shuffle.nxv4i32.v4i32(
+ <vscale x 4 x i32> %0, <vscale x 4 x i32> %1, <4 x i32> <i32 3, i32 4, i32 5, i32 6>)
+ ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @ext_v4i16(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1) {
+; CHECK-LABEL: ext_v4i16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: extq z0.b, z0.b, z0.b, #12
+; CHECK-NEXT: and z0.s, z0.s, #0xffff
+; CHECK-NEXT: ret
+entry:
+ %leg0 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i16>
+ %leg1 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i16>
+ %res = call <vscale x 4 x i16> @llvm.vector.segmented.shuffle.nxv4i16.v4i32(
+ <vscale x 4 x i16> %leg0, <vscale x 4 x i16> %leg1, <4 x i32> <i32 3, i32 4, i32 5, i32 6>)
+ %res.leg = zext <vscale x 4 x i16> %res to <vscale x 4 x i32>
+ ret <vscale x 4 x i32> %res.leg
+}
+
+define <vscale x 4 x i32> @ext_v4i8(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1) {
+; CHECK-LABEL: ext_v4i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: extq z0.b, z0.b, z0.b, #12
+; CHECK-NEXT: and z0.s, z0.s, #0xff
+; CHECK-NEXT: ret
+entry:
+ %leg0 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i8>
+ %leg1 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i8>
+ %res = call <vscale x 4 x i8> @llvm.vector.segmented.shuffle.nxv4i8.v4i32(
+ <vscale x 4 x i8> %leg0, <vscale x 4 x i8> %leg1, <4 x i32> <i32 3, i32 4, i32 5, i32 6>)
+ %res.leg = zext <vscale x 4 x i8> %res to <vscale x 4 x i32>
+ ret <vscale x 4 x i32> %res.leg
+}
+
+define <vscale x 4 x float> @ext_v4f32(<vscale x 4 x float> %0, <vscale x 4 x float> %1) {
+; CHECK-LABEL: ext_v4f32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: extq z0.b, z0.b, z1.b, #12
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 4 x float> @llvm.vector.segmented.shuffle.nxv4f32.v4i32(
+ <vscale x 4 x float> %0, <vscale x 4 x float> %1, <4 x i32> <i32 3, i32 4, i32 5, i32 6>)
+ ret <vscale x 4 x float> %res
+}
+
+define <vscale x 4 x half> @ext_v4f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1) {
+; CHECK-LABEL: ext_v4f16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: extq z0.b, z0.b, z1.b, #12
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 4 x half> @llvm.vector.segmented.shuffle.nxv4f16.v4i32(
+ <vscale x 4 x half> %0, <vscale x 4 x half> %1, <4 x i32> <i32 3, i32 4, i32 5, i32 6>)
+ ret <vscale x 4 x half> %res
+}
+
+define <vscale x 2 x i64> @ext_v2i64(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1) {
+; CHECK-LABEL: ext_v2i64:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: extq z0.b, z0.b, z1.b, #8
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 2 x i64> @llvm.vector.segmented.shuffle.nxv2i64.v2i32(
+ <vscale x 2 x i64> %0, <vscale x 2 x i64> %1, <2 x i32> <i32 1, i32 2>)
+ ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @ext_v2i32(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1) {
+; CHECK-LABEL: ext_v2i32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: extq z0.b, z0.b, z0.b, #8
+; CHECK-NEXT: and z0.d, z0.d, #0xffffffff
+; CHECK-NEXT: ret
+entry:
+ %leg0 = trunc <vscale x 2 x i64> %0 to <vscale x 2 x i32>
+ %leg1 = trunc <vscale x 2 x i64> %0 to <vscale x 2 x i32>
+ %res = call <vscale x 2 x i32> @llvm.vector.segmented.shuffle.nxv2i32.v2i32(
+ <vscale x 2 x i32> %leg0, <vscale x 2 x i32> %leg1, <2 x i32> <i32 1, i32 2>)
+ %res.leg = zext <vscale x 2 x i32> %res to <vscale x 2 x i64>
+ ret <vscale x 2 x i64> %res.leg
+}
+
+define <vscale x 2 x double> @ext_v2f64(<vscale x 2 x double> %0, <vscale x 2 x double> %1) {
+; CHECK-LABEL: ext_v2f64:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: extq z0.b, z0.b, z1.b, #8
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 2 x double> @llvm.vector.segmented.shuffle.nxv2f64.v2i32(
+ <vscale x 2 x double> %0, <vscale x 2 x double> %1, <2 x i32> <i32 1, i32 2>)
+ ret <vscale x 2 x double> %res
+}
+
+define <vscale x 2 x float> @ext_v2f32(<vscale x 2 x float> %0, <vscale x 2 x float> %1) {
+; CHECK-LABEL: ext_v2f32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: extq z0.b, z0.b, z1.b, #8
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 2 x float> @llvm.vector.segmented.shuffle.nxv2f32.v2i32(
+ <vscale x 2 x float> %0, <vscale x 2 x float> %1, <2 x i32> <i32 1, i32 2>)
+ ret <vscale x 2 x float> %res
+}
+
+
+
+; dup
+
+define <vscale x 16 x i8> @dup_v16i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1) {
+; CHECK-LABEL: dup_v16i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: dupq z0.b, z0.b[7]
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(
+ <vscale x 16 x i8> %0, <vscale x 16 x i8> %1, <16 x i32> <i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7>)
+ ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @dup_v8i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1) {
+; CHECK-LABEL: dup_v8i16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: dupq z0.h, z0.h[7]
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 8 x i16> @llvm.vector.segmented.shuffle.nxv8i16.v8i32(
+ <vscale x 8 x i16> %0, <vscale x 8 x i16> %1, <8 x i32> <i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7>)
+ ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 8 x i16> @dup_v8i8(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1) {
+; CHECK-LABEL: dup_v8i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: dupq z0.h, z0.h[7]
+; CHECK-NEXT: and z0.h, z0.h, #0xff
+; CHECK-NEXT: ret
+entry:
+ %leg0 = trunc <vscale x 8 x i16> %0 to <vscale x 8 x i8>
+ %leg1 = trunc <vscale x 8 x i16> %1 to <vscale x 8 x i8>
+ %res = call <vscale x 8 x i8> @llvm.vector.segmented.shuffle.nxv8i8.v8i32(
+ <vscale x 8 x i8> %leg0, <vscale x 8 x i8> %leg1, <8 x i32> <i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7>)
+ %res.leg = zext <vscale x 8 x i8> %res to <vscale x 8 x i16>
+ ret <vscale x 8 x i16> %res.leg
+}
+
+define <vscale x 8 x half> @dup_v8f16(<vscale x 8 x half> %0, <vscale x 8 x half> %1) {
+; CHECK-LABEL: dup_v8f16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: dupq z0.h, z0.h[7]
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 8 x half> @llvm.vector.segmented.shuffle.nxv8f16.v8i32(
+ <vscale x 8 x half> %0, <vscale x 8 x half> %1, <8 x i32> <i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7>)
+ ret <vscale x 8 x half> %res
+}
+
+define <vscale x 8 x bfloat> @dup_v8bf16(<vscale x 8 x bfloat> %0, <vscale x 8 x bfloat> %1) {
+; CHECK-LABEL: dup_v8bf16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: dupq z0.h, z0.h[7]
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 8 x bfloat> @llvm.vector.segmented.shuffle.nxv8bf16.v8i32(
+ <vscale x 8 x bfloat> %0, <vscale x 8 x bfloat> %1, <8 x i32> <i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7>)
+ ret <vscale x 8 x bfloat> %res
+}
+
+define <vscale x 4 x i32> @dup_v4i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1) {
+; CHECK-LABEL: dup_v4i32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: dupq z0.s, z0.s[2]
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 4 x i32> @llvm.vector.segmented.shuffle.nxv4i32.v4i32(
+ <vscale x 4 x i32> %0, <vscale x 4 x i32> %1, <4 x i32> <i32 2, i32 2, i32 2, i32 2>)
+ ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @dup_v4i16(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1) {
+; CHECK-LABEL: dup_v4i16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: dupq z0.s, z0.s[2]
+; CHECK-NEXT: and z0.s, z0.s, #0xffff
+; CHECK-NEXT: ret
+entry:
+ %leg0 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i16>
+ %leg1 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i16>
+ %res = call <vscale x 4 x i16> @llvm.vector.segmented.shuffle.nxv4i16.v4i32(
+ <vscale x 4 x i16> %leg0, <vscale x 4 x i16> %leg1, <4 x i32> <i32 2, i32 2, i32 2, i32 2>)
+ %res.leg = zext <vscale x 4 x i16> %res to <vscale x 4 x i32>
+ ret <vscale x 4 x i32> %res.leg
+}
+
+define <vscale x 4 x i32> @dup_v4i8(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1) {
+; CHECK-LABEL: dup_v4i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: dupq z0.s, z0.s[2]
+; CHECK-NEXT: and z0.s, z0.s, #0xff
+; CHECK-NEXT: ret
+entry:
+ %leg0 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i8>
+ %leg1 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i8>
+ %res = call <vscale x 4 x i8> @llvm.vector.segmented.shuffle.nxv4i8.v4i32(
+ <vscale x 4 x i8> %leg0, <vscale x 4 x i8> %leg1, <4 x i32> <i32 2, i32 2, i32 2, i32 2>)
+ %res.leg = zext <vscale x 4 x i8> %res to <vscale x 4 x i32>
+ ret <vscale x 4 x i32> %res.leg
+}
+
+define <vscale x 4 x float> @dup_v4f32(<vscale x 4 x float> %0, <vscale x 4 x float> %1) {
+; CHECK-LABEL: dup_v4f32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: dupq z0.s, z0.s[2]
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 4 x float> @llvm.vector.segmented.shuffle.nxv4f32.v4i32(
+ <vscale x 4 x float> %0, <vscale x 4 x float> %1, <4 x i32> <i32 2, i32 2, i32 2, i32 2>)
+ ret <vscale x 4 x float> %res
+}
+
+define <vscale x 4 x half> @dup_v4f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1) {
+; CHECK-LABEL: dup_v4f16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: dupq z0.s, z0.s[2]
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 4 x half> @llvm.vector.segmented.shuffle.nxv4f16.v4i32(
+ <vscale x 4 x half> %0, <vscale x 4 x half> %1, <4 x i32> <i32 2, i32 2, i32 2, i32 2>)
+ ret <vscale x 4 x half> %res
+}
+
+define <vscale x 2 x i64> @dup_v2i64(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1) {
+; CHECK-LABEL: dup_v2i64:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: dupq z0.d, z0.d[1]
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 2 x i64> @llvm.vector.segmented.shuffle.nxv2i64.v2i32(
+ <vscale x 2 x i64> %0, <vscale x 2 x i64> %1, <2 x i32> <i32 1, i32 1>)
+ ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @dup_v2i32(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1) {
+; CHECK-LABEL: dup_v2i32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: dupq z0.d, z0.d[1]
+; CHECK-NEXT: and z0.d, z0.d, #0xffffffff
+; CHECK-NEXT: ret
+entry:
+ %leg0 = trunc <vscale x 2 x i64> %0 to <vscale x 2 x i32>
+ %leg1 = trunc <vscale x 2 x i64> %0 to <vscale x 2 x i32>
+ %res = call <vscale x 2 x i32> @llvm.vector.segmented.shuffle.nxv2i32.v2i32(
+ <vscale x 2 x i32> %leg0, <vscale x 2 x i32> %leg1, <2 x i32> <i32 1, i32 1>)
+ %res.leg = zext <vscale x 2 x i32> %res to <vscale x 2 x i64>
+ ret <vscale x 2 x i64> %res.leg
+}
+
+define <vscale x 2 x double> @dup_v2f64(<vscale x 2 x double> %0, <vscale x 2 x double> %1) {
+; CHECK-LABEL: dup_v2f64:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: dupq z0.d, z0.d[1]
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 2 x double> @llvm.vector.segmented.shuffle.nxv2f64.v2i32(
+ <vscale x 2 x double> %0, <vscale x 2 x double> %1, <2 x i32> <i32 1, i32 1>)
+ ret <vscale x 2 x double> %res
+}
+
+define <vscale x 2 x float> @dup_v2f32(<vscale x 2 x float> %0, <vscale x 2 x float> %1) {
+; CHECK-LABEL: dup_v2f32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: dupq z0.d, z0.d[1]
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 2 x float> @llvm.vector.segmented.shuffle.nxv2f32.v2i32(
+ <vscale x 2 x float> %0, <vscale x 2 x float> %1, <2 x i32> <i32 1, i32 1>)
+ ret <vscale x 2 x float> %res
+}
+
+
+
+; rev
+
+define <vscale x 16 x i8> @rev_v16i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1) {
+; CHECK-LABEL: rev_v16i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: revd z0.q, p0/m, z0.q
+; CHECK-NEXT: revb z0.d, p1/m, z0.d
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(
+ <vscale x 16 x i8> %0, <vscale x 16 x i8> %1, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+ ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @rev_v8i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1) {
+; CHECK-LABEL: rev_v8i16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: revd z0.q, p0/m, z0.q
+; CHECK-NEXT: revh z0.d, p1/m, z0.d
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 8 x i16> @llvm.vector.segmented.shuffle.nxv8i16.v8i32(
+ <vscale x 8 x i16> %0, <vscale x 8 x i16> %1, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+ ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 8 x i16> @rev_v8i8(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1) {
+; CHECK-LABEL: rev_v8i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: revd z0.q, p0/m, z0.q
+; CHECK-NEXT: revh z0.d, p1/m, z0.d
+; CHECK-NEXT: and z0.h, z0.h, #0xff
+; CHECK-NEXT: ret
+entry:
+ %leg0 = trunc <vscale x 8 x i16> %0 to <vscale x 8 x i8>
+ %leg1 = trunc <vscale x 8 x i16> %1 to <vscale x 8 x i8>
+ %res = call <vscale x 8 x i8> @llvm.vector.segmented.shuffle.nxv8i8.v8i32(
+ <vscale x 8 x i8> %leg0, <vscale x 8 x i8> %leg1, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+ %res.leg = zext <vscale x 8 x i8> %res to <vscale x 8 x i16>
+ ret <vscale x 8 x i16> %res.leg
+}
+
+define <vscale x 8 x half> @rev_v8f16(<vscale x 8 x half> %0, <vscale x 8 x half> %1) {
+; CHECK-LABEL: rev_v8f16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: revd z0.q, p0/m, z0.q
+; CHECK-NEXT: revh z0.d, p1/m, z0.d
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 8 x half> @llvm.vector.segmented.shuffle.nxv8f16.v8i32(
+ <vscale x 8 x half> %0, <vscale x 8 x half> %1, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+ ret <vscale x 8 x half> %res
+}
+
+define <vscale x 8 x bfloat> @rev_v8bf16(<vscale x 8 x bfloat> %0, <vscale x 8 x bfloat> %1) {
+; CHECK-LABEL: rev_v8bf16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: revd z0.q, p0/m, z0.q
+; CHECK-NEXT: revh z0.d, p1/m, z0.d
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 8 x bfloat> @llvm.vector.segmented.shuffle.nxv8bf16.v8i32(
+ <vscale x 8 x bfloat> %0, <vscale x 8 x bfloat> %1, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+ ret <vscale x 8 x bfloat> %res
+}
+
+define <vscale x 4 x i32> @rev_v4i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1) {
+; CHECK-LABEL: rev_v4i32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: revd z0.q, p0/m, z0.q
+; CHECK-NEXT: revw z0.d, p1/m, z0.d
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 4 x i32> @llvm.vector.segmented.shuffle.nxv4i32.v4i32(
+ <vscale x 4 x i32> %0, <vscale x 4 x i32> %1, <4 x i32> <i32 3, i32 2, i32 1, i32 0>)
+ ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @rev_v4i16(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1) {
+; CHECK-LABEL: rev_v4i16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: revd z0.q, p0/m, z0.q
+; CHECK-NEXT: revw z0.d, p1/m, z0.d
+; CHECK-NEXT: and z0.s, z0.s, #0xffff
+; CHECK-NEXT: ret
+entry:
+ %leg0 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i16>
+ %leg1 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i16>
+ %res = call <vscale x 4 x i16> @llvm.vector.segmented.shuffle.nxv4i16.v4i32(
+ <vscale x 4 x i16> %leg0, <vscale x 4 x i16> %leg1, <4 x i32> <i32 3, i32 2, i32 1, i32 0>)
+ %res.leg = zext <vscale x 4 x i16> %res to <vscale x 4 x i32>
+ ret <vscale x 4 x i32> %res.leg
+}
+
+define <vscale x 4 x i32> @rev_v4i8(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1) {
+; CHECK-LABEL: rev_v4i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: revd z0.q, p0/m, z0.q
+; CHECK-NEXT: revw z0.d, p1/m, z0.d
+; CHECK-NEXT: and z0.s, z0.s, #0xff
+; CHECK-NEXT: ret
+entry:
+ %leg0 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i8>
+ %leg1 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i8>
+ %res = call <vscale x 4 x i8> @llvm.vector.segmented.shuffle.nxv4i8.v4i32(
+ <vscale x 4 x i8> %leg0, <vscale x 4 x i8> %leg1, <4 x i32> <i32 3, i32 2, i32 1, i32 0>)
+ %res.leg = zext <vscale x 4 x i8> %res to <vscale x 4 x i32>
+ ret <vscale x 4 x i32> %res.leg
+}
+
+define <vscale x 4 x float> @rev_v4f32(<vscale x 4 x float> %0, <vscale x 4 x float> %1) {
+; CHECK-LABEL: rev_v4f32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: revd z0.q, p0/m, z0.q
+; CHECK-NEXT: revw z0.d, p1/m, z0.d
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 4 x float> @llvm.vector.segmented.shuffle.nxv4f32.v4i32(
+ <vscale x 4 x float> %0, <vscale x 4 x float> %1, <4 x i32> <i32 3, i32 2, i32 1, i32 0>)
+ ret <vscale x 4 x float> %res
+}
+
+define <vscale x 4 x half> @rev_v4f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1) {
+; CHECK-LABEL: rev_v4f16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: revd z0.q, p0/m, z0.q
+; CHECK-NEXT: revw z0.d, p1/m, z0.d
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 4 x half> @llvm.vector.segmented.shuffle.nxv4f16.v4i32(
+ <vscale x 4 x half> %0, <vscale x 4 x half> %1, <4 x i32> <i32 3, i32 2, i32 1, i32 0>)
+ ret <vscale x 4 x half> %res
+}
+
+define <vscale x 2 x i64> @rev_v2i64(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1) {
+; CHECK-LABEL: rev_v2i64:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: revd z0.q, p0/m, z0.q
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 2 x i64> @llvm.vector.segmented.shuffle.nxv2i64.v2i32(
+ <vscale x 2 x i64> %0, <vscale x 2 x i64> %1, <2 x i32> <i32 1, i32 0>)
+ ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @rev_v2i32(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1) {
+; CHECK-LABEL: rev_v2i32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: revd z0.q, p0/m, z0.q
+; CHECK-NEXT: and z0.d, z0.d, #0xffffffff
+; CHECK-NEXT: ret
+entry:
+ %leg0 = trunc <vscale x 2 x i64> %0 to <vscale x 2 x i32>
+ %leg1 = trunc <vscale x 2 x i64> %0 to <vscale x 2 x i32>
+ %res = call <vscale x 2 x i32> @llvm.vector.segmented.shuffle.nxv2i32.v2i32(
+ <vscale x 2 x i32> %leg0, <vscale x 2 x i32> %leg1, <2 x i32> <i32 1, i32 0>)
+ %res.leg = zext <vscale x 2 x i32> %res to <vscale x 2 x i64>
+ ret <vscale x 2 x i64> %res.leg
+}
+
+define <vscale x 2 x double> @rev_v2f64(<vscale x 2 x double> %0, <vscale x 2 x double> %1) {
+; CHECK-LABEL: rev_v2f64:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: revd z0.q, p0/m, z0.q
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 2 x double> @llvm.vector.segmented.shuffle.nxv2f64.v2i32(
+ <vscale x 2 x double> %0, <vscale x 2 x double> %1, <2 x i32> <i32 1, i32 0>)
+ ret <vscale x 2 x double> %res
+}
+
+define <vscale x 2 x float> @rev_v2f32(<vscale x 2 x float> %0, <vscale x 2 x float> %1) {
+; CHECK-LABEL: rev_v2f32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: revd z0.q, p0/m, z0.q
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 2 x float> @llvm.vector.segmented.shuffle.nxv2f32.v2i32(
+ <vscale x 2 x float> %0, <vscale x 2 x float> %1, <2 x i32> <i32 1, i32 0>)
+ ret <vscale x 2 x float> %res
+}
+
+
+
+; tbl1; arbitrary 2-input shuffle
+
+define <vscale x 16 x i8> @tbl1_v16i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1) {
+; CHECK-LABEL: tbl1_v16i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: adrp x8, .LCPI116_0
+; CHECK-NEXT: add x8, x8, :lo12:.LCPI116_0
+; CHECK-NEXT: ld1rqb { z1.b }, p0/z, [x8]
+; CHECK-NEXT: tblq z0.b, { z0.b }, z1.b
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(
+ <vscale x 16 x i8> %0, <vscale x 16 x i8> %1, <16 x i32> <i32 15, i32 14, i32 10, i32 12, i32 11, i32 10, i32 9, i32 8, i32 10, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+ ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @tbl1_v8i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1) {
+; CHECK-LABEL: tbl1_v8i16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: adrp x8, .LCPI117_0
+; CHECK-NEXT: add x8, x8, :lo12:.LCPI117_0
+; CHECK-NEXT: ld1rqh { z1.h }, p0/z, [x8]
+; CHECK-NEXT: tblq z0.h, { z0.h }, z1.h
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 8 x i16> @llvm.vector.segmented.shuffle.nxv8i16.v8i32(
+ <vscale x 8 x i16> %0, <vscale x 8 x i16> %1, <8 x i32> <i32 7, i32 6, i32 3, i32 4, i32 3, i32 2, i32 1, i32 0>)
+ ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 8 x i16> @tbl1_v8i8(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1) {
+; CHECK-LABEL: tbl1_v8i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: adrp x8, .LCPI118_0
+; CHECK-NEXT: add x8, x8, :lo12:.LCPI118_0
+; CHECK-NEXT: ld1rqh { z1.h }, p0/z, [x8]
+; CHECK-NEXT: tblq z0.h, { z0.h }, z1.h
+; CHECK-NEXT: and z0.h, z0.h, #0xff
+; CHECK-NEXT: ret
+entry:
+ %leg0 = trunc <vscale x 8 x i16> %0 to <vscale x 8 x i8>
+ %leg1 = trunc <vscale x 8 x i16> %1 to <vscale x 8 x i8>
+ %res = call <vscale x 8 x i8> @llvm.vector.segmented.shuffle.nxv8i8.v8i32(
+ <vscale x 8 x i8> %leg0, <vscale x 8 x i8> %leg1, <8 x i32> <i32 7, i32 6, i32 3, i32 4, i32 3, i32 2, i32 1, i32 0>)
+ %res.leg = zext <vscale x 8 x i8> %res to <vscale x 8 x i16>
+ ret <vscale x 8 x i16> %res.leg
+}
+
+define <vscale x 8 x half> @tbl1_v8f16(<vscale x 8 x half> %0, <vscale x 8 x half> %1) {
+; CHECK-LABEL: tbl1_v8f16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: adrp x8, .LCPI119_0
+; CHECK-NEXT: add x8, x8, :lo12:.LCPI119_0
+; CHECK-NEXT: ld1rqh { z1.h }, p0/z, [x8]
+; CHECK-NEXT: tblq z0.h, { z0.h }, z1.h
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 8 x half> @llvm.vector.segmented.shuffle.nxv8f16.v8i32(
+ <vscale x 8 x half> %0, <vscale x 8 x half> %1, <8 x i32> <i32 7, i32 6, i32 5, i32 3, i32 3, i32 2, i32 1, i32 0>)
+ ret <vscale x 8 x half> %res
+}
+
+define <vscale x 8 x bfloat> @tbl1_v8bf16(<vscale x 8 x bfloat> %0, <vscale x 8 x bfloat> %1) {
+; CHECK-LABEL: tbl1_v8bf16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: adrp x8, .LCPI120_0
+; CHECK-NEXT: add x8, x8, :lo12:.LCPI120_0
+; CHECK-NEXT: ld1rqh { z1.h }, p0/z, [x8]
+; CHECK-NEXT: tblq z0.h, { z0.h }, z1.h
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 8 x bfloat> @llvm.vector.segmented.shuffle.nxv8bf16.v8i32(
+ <vscale x 8 x bfloat> %0, <vscale x 8 x bfloat> %1, <8 x i32> <i32 7, i32 6, i32 5, i32 3, i32 3, i32 2, i32 1, i32 0>)
+ ret <vscale x 8 x bfloat> %res
+}
+
+define <vscale x 4 x i32> @tbl1_v4i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1) {
+; CHECK-LABEL: tbl1_v4i32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: adrp x8, .LCPI121_0
+; CHECK-NEXT: add x8, x8, :lo12:.LCPI121_0
+; CHECK-NEXT: ld1rqw { z1.s }, p0/z, [x8]
+; CHECK-NEXT: tblq z0.s, { z0.s }, z1.s
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 4 x i32> @llvm.vector.segmented.shuffle.nxv4i32.v4i32(
+ <vscale x 4 x i32> %0, <vscale x 4 x i32> %1, <4 x i32> <i32 3, i32 3, i32 1, i32 0>)
+ ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @tbl1_v4i16(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1) {
+; CHECK-LABEL: tbl1_v4i16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: adrp x8, .LCPI122_0
+; CHECK-NEXT: add x8, x8, :lo12:.LCPI122_0
+; CHECK-NEXT: ld1rqw { z1.s }, p0/z, [x8]
+; CHECK-NEXT: tblq z0.s, { z0.s }, z1.s
+; CHECK-NEXT: and z0.s, z0.s, #0xffff
+; CHECK-NEXT: ret
+entry:
+ %leg0 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i16>
+ %leg1 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i16>
+ %res = call <vscale x 4 x i16> @llvm.vector.segmented.shuffle.nxv4i16.v4i32(
+ <vscale x 4 x i16> %leg0, <vscale x 4 x i16> %leg1, <4 x i32> <i32 3, i32 3, i32 1, i32 0>)
+ %res.leg = zext <vscale x 4 x i16> %res to <vscale x 4 x i32>
+ ret <vscale x 4 x i32> %res.leg
+}
+
+define <vscale x 4 x i32> @tbl1_v4i8(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1) {
+; CHECK-LABEL: tbl1_v4i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: adrp x8, .LCPI123_0
+; CHECK-NEXT: add x8, x8, :lo12:.LCPI123_0
+; CHECK-NEXT: ld1rqw { z1.s }, p0/z, [x8]
+; CHECK-NEXT: tblq z0.s, { z0.s }, z1.s
+; CHECK-NEXT: and z0.s, z0.s, #0xff
+; CHECK-NEXT: ret
+entry:
+ %leg0 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i8>
+ %leg1 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i8>
+ %res = call <vscale x 4 x i8> @llvm.vector.segmented.shuffle.nxv4i8.v4i32(
+ <vscale x 4 x i8> %leg0, <vscale x 4 x i8> %leg1, <4 x i32> <i32 3, i32 3, i32 1, i32 0>)
+ %res.leg = zext <vscale x 4 x i8> %res to <vscale x 4 x i32>
+ ret <vscale x 4 x i32> %res.leg
+}
+
+define <vscale x 4 x float> @tbl1_v4f32(<vscale x 4 x float> %0, <vscale x 4 x float> %1) {
+; CHECK-LABEL: tbl1_v4f32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: adrp x8, .LCPI124_0
+; CHECK-NEXT: add x8, x8, :lo12:.LCPI124_0
+; CHECK-NEXT: ld1rqw { z1.s }, p0/z, [x8]
+; CHECK-NEXT: tblq z0.s, { z0.s }, z1.s
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 4 x float> @llvm.vector.segmented.shuffle.nxv4f32.v4i32(
+ <vscale x 4 x float> %0, <vscale x 4 x float> %1, <4 x i32> <i32 3, i32 3, i32 1, i32 0>)
+ ret <vscale x 4 x float> %res
+}
+
+define <vscale x 4 x half> @tbl1_v4f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1) {
+; CHECK-LABEL: tbl1_v4f16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: adrp x8, .LCPI125_0
+; CHECK-NEXT: add x8, x8, :lo12:.LCPI125_0
+; CHECK-NEXT: ld1rqw { z1.s }, p0/z, [x8]
+; CHECK-NEXT: tblq z0.s, { z0.s }, z1.s
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 4 x half> @llvm.vector.segmented.shuffle.nxv4f16.v4i32(
+ <vscale x 4 x half> %0, <vscale x 4 x half> %1, <4 x i32> <i32 3, i32 3, i32 1, i32 0>)
+ ret <vscale x 4 x half> %res
+}
+
+
+
+; tbl2; arbitrary 1-input shuffle
+
+define <vscale x 16 x i8> @tbl2_v16i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1) {
+; CHECK-LABEL: tbl2_v16i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: adrp x8, .LCPI126_0
+; CHECK-NEXT: add x8, x8, :lo12:.LCPI126_0
+; CHECK-NEXT: ld1rqb { z2.b }, p0/z, [x8]
+; CHECK-NEXT: movprfx z3, z2
+; CHECK-NEXT: sub z3.b, z3.b, #16 // =0x10
+; CHECK-NEXT: tblq z0.b, { z0.b }, z2.b
+; CHECK-NEXT: tbxq z0.b, z1.b, z3.b
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(
+ <vscale x 16 x i8> %0, <vscale x 16 x i8> %1, <16 x i32> <i32 5, i32 6, i32 2, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20>)
+ ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @tbl2_v8i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1) {
+; CHECK-LABEL: tbl2_v8i16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: adrp x8, .LCPI127_0
+; CHECK-NEXT: add x8, x8, :lo12:.LCPI127_0
+; CHECK-NEXT: ld1rqh { z2.h }, p0/z, [x8]
+; CHECK-NEXT: movprfx z3, z2
+; CHECK-NEXT: sub z3.h, z3.h, #8 // =0x8
+; CHECK-NEXT: tblq z0.h, { z0.h }, z2.h
+; CHECK-NEXT: tbxq z0.h, z1.h, z3.h
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 8 x i16> @llvm.vector.segmented.shuffle.nxv8i16.v8i32(
+ <vscale x 8 x i16> %0, <vscale x 8 x i16> %1, <8 x i32> <i32 5, i32 6, i32 2, i32 8, i32 9, i32 10, i32 11, i32 12>)
+ ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 8 x i16> @tbl2_v8i8(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1) {
+; CHECK-LABEL: tbl2_v8i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: adrp x8, .LCPI128_0
+; CHECK-NEXT: add x8, x8, :lo12:.LCPI128_0
+; CHECK-NEXT: ld1rqh { z2.h }, p0/z, [x8]
+; CHECK-NEXT: movprfx z3, z2
+; CHECK-NEXT: sub z3.h, z3.h, #8 // =0x8
+; CHECK-NEXT: tblq z0.h, { z0.h }, z2.h
+; CHECK-NEXT: tbxq z0.h, z1.h, z3.h
+; CHECK-NEXT: and z0.h, z0.h, #0xff
+; CHECK-NEXT: ret
+entry:
+ %leg0 = trunc <vscale x 8 x i16> %0 to <vscale x 8 x i8>
+ %leg1 = trunc <vscale x 8 x i16> %1 to <vscale x 8 x i8>
+ %res = call <vscale x 8 x i8> @llvm.vector.segmented.shuffle.nxv8i8.v8i32(
+ <vscale x 8 x i8> %leg0, <vscale x 8 x i8> %leg1, <8 x i32> <i32 5, i32 6, i32 2, i32 8, i32 9, i32 10, i32 11, i32 12>)
+ %res.leg = zext <vscale x 8 x i8> %res to <vscale x 8 x i16>
+ ret <vscale x 8 x i16> %res.leg
+}
+
+define <vscale x 8 x half> @tbl2_v8f16(<vscale x 8 x half> %0, <vscale x 8 x half> %1) {
+; CHECK-LABEL: tbl2_v8f16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: adrp x8, .LCPI129_0
+; CHECK-NEXT: add x8, x8, :lo12:.LCPI129_0
+; CHECK-NEXT: ld1rqh { z2.h }, p0/z, [x8]
+; CHECK-NEXT: movprfx z3, z2
+; CHECK-NEXT: sub z3.h, z3.h, #8 // =0x8
+; CHECK-NEXT: tblq z0.h, { z0.h }, z2.h
+; CHECK-NEXT: tbxq z0.h, z1.h, z3.h
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 8 x half> @llvm.vector.segmented.shuffle.nxv8f16.v8i32(
+ <vscale x 8 x half> %0, <vscale x 8 x half> %1, <8 x i32> <i32 5, i32 6, i32 2, i32 8, i32 9, i32 10, i32 11, i32 12>)
+ ret <vscale x 8 x half> %res
+}
+
+define <vscale x 8 x bfloat> @tbl2_v8bf16(<vscale x 8 x bfloat> %0, <vscale x 8 x bfloat> %1) {
+; CHECK-LABEL: tbl2_v8bf16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: adrp x8, .LCPI130_0
+; CHECK-NEXT: add x8, x8, :lo12:.LCPI130_0
+; CHECK-NEXT: ld1rqh { z2.h }, p0/z, [x8]
+; CHECK-NEXT: movprfx z3, z2
+; CHECK-NEXT: sub z3.h, z3.h, #8 // =0x8
+; CHECK-NEXT: tblq z0.h, { z0.h }, z2.h
+; CHECK-NEXT: tbxq z0.h, z1.h, z3.h
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 8 x bfloat> @llvm.vector.segmented.shuffle.nxv8bf16.v8i32(
+ <vscale x 8 x bfloat> %0, <vscale x 8 x bfloat> %1, <8 x i32> <i32 5, i32 3, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12>)
+ ret <vscale x 8 x bfloat> %res
+}
+
+define <vscale x 4 x i32> @tbl2_v4i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1) {
+; CHECK-LABEL: tbl2_v4i32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: adrp x8, .LCPI131_0
+; CHECK-NEXT: add x8, x8, :lo12:.LCPI131_0
+; CHECK-NEXT: ld1rqw { z2.s }, p0/z, [x8]
+; CHECK-NEXT: movprfx z3, z2
+; CHECK-NEXT: sub z3.s, z3.s, #4 // =0x4
+; CHECK-NEXT: tblq z0.s, { z0.s }, z2.s
+; CHECK-NEXT: tbxq z0.s, z1.s, z3.s
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 4 x i32> @llvm.vector.segmented.shuffle.nxv4i32.v4i32(
+ <vscale x 4 x i32> %0, <vscale x 4 x i32> %1, <4 x i32> <i32 3, i32 3, i32 5, i32 6>)
+ ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @tbl2_v4i16(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1) {
+; CHECK-LABEL: tbl2_v4i16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: adrp x8, .LCPI132_0
+; CHECK-NEXT: add x8, x8, :lo12:.LCPI132_0
+; CHECK-NEXT: ld1rqw { z1.s }, p0/z, [x8]
+; CHECK-NEXT: movprfx z2, z1
+; CHECK-NEXT: sub z2.s, z2.s, #4 // =0x4
+; CHECK-NEXT: tblq z1.s, { z0.s }, z1.s
+; CHECK-NEXT: tbxq z1.s, z0.s, z2.s
+; CHECK-NEXT: movprfx z0, z1
+; CHECK-NEXT: and z0.s, z0.s, #0xffff
+; CHECK-NEXT: ret
+entry:
+ %leg0 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i16>
+ %leg1 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i16>
+ %res = call <vscale x 4 x i16> @llvm.vector.segmented.shuffle.nxv4i16.v4i32(
+ <vscale x 4 x i16> %leg0, <vscale x 4 x i16> %leg1, <4 x i32> <i32 3, i32 3, i32 5, i32 6>)
+ %res.leg = zext <vscale x 4 x i16> %res to <vscale x 4 x i32>
+ ret <vscale x 4 x i32> %res.leg
+}
+
+define <vscale x 4 x i32> @tbl2_v4i8(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1) {
+; CHECK-LABEL: tbl2_v4i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: adrp x8, .LCPI133_0
+; CHECK-NEXT: add x8, x8, :lo12:.LCPI133_0
+; CHECK-NEXT: ld1rqw { z1.s }, p0/z, [x8]
+; CHECK-NEXT: movprfx z2, z1
+; CHECK-NEXT: sub z2.s, z2.s, #4 // =0x4
+; CHECK-NEXT: tblq z1.s, { z0.s }, z1.s
+; CHECK-NEXT: tbxq z1.s, z0.s, z2.s
+; CHECK-NEXT: movprfx z0, z1
+; CHECK-NEXT: and z0.s, z0.s, #0xff
+; CHECK-NEXT: ret
+entry:
+ %leg0 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i8>
+ %leg1 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i8>
+ %res = call <vscale x 4 x i8> @llvm.vector.segmented.shuffle.nxv4i8.v4i32(
+ <vscale x 4 x i8> %leg0, <vscale x 4 x i8> %leg1, <4 x i32> <i32 3, i32 3, i32 5, i32 6>)
+ %res.leg = zext <vscale x 4 x i8> %res to <vscale x 4 x i32>
+ ret <vscale x 4 x i32> %res.leg
+}
+
+define <vscale x 4 x float> @tbl2_v4f32(<vscale x 4 x float> %0, <vscale x 4 x float> %1) {
+; CHECK-LABEL: tbl2_v4f32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: adrp x8, .LCPI134_0
+; CHECK-NEXT: add x8, x8, :lo12:.LCPI134_0
+; CHECK-NEXT: ld1rqw { z2.s }, p0/z, [x8]
+; CHECK-NEXT: movprfx z3, z2
+; CHECK-NEXT: sub z3.s, z3.s, #4 // =0x4
+; CHECK-NEXT: tblq z0.s, { z0.s }, z2.s
+; CHECK-NEXT: tbxq z0.s, z1.s, z3.s
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 4 x float> @llvm.vector.segmented.shuffle.nxv4f32.v4i32(
+ <vscale x 4 x float> %0, <vscale x 4 x float> %1, <4 x i32> <i32 3, i32 3, i32 5, i32 6>)
+ ret <vscale x 4 x float> %res
+}
+
+define <vscale x 4 x half> @tbl2_v4f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1) {
+; CHECK-LABEL: tbl2_v4f16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: adrp x8, .LCPI135_0
+; CHECK-NEXT: add x8, x8, :lo12:.LCPI135_0
+; CHECK-NEXT: ld1rqw { z2.s }, p0/z, [x8]
+; CHECK-NEXT: movprfx z3, z2
+; CHECK-NEXT: sub z3.s, z3.s, #4 // =0x4
+; CHECK-NEXT: tblq z0.s, { z0.s }, z2.s
+; CHECK-NEXT: tbxq z0.s, z1.s, z3.s
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 4 x half> @llvm.vector.segmented.shuffle.nxv4f16.v4i32(
+ <vscale x 4 x half> %0, <vscale x 4 x half> %1, <4 x i32> <i32 3, i32 3, i32 5, i32 6>)
+ ret <vscale x 4 x half> %res
+}
+
+define <vscale x 2 x i64> @tbl2_v2i64(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1) {
+; CHECK-LABEL: tbl2_v2i64:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: index z2.d, #0, #3
+; CHECK-NEXT: mov z2.q, q2
+; CHECK-NEXT: movprfx z3, z2
+; CHECK-NEXT: sub z3.d, z3.d, #2 // =0x2
+; CHECK-NEXT: tblq z0.d, { z0.d }, z2.d
+; CHECK-NEXT: tbxq z0.d, z1.d, z3.d
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 2 x i64> @llvm.vector.segmented.shuffle.nxv2i64.v2i32(
+ <vscale x 2 x i64> %0, <vscale x 2 x i64> %1, <2 x i32> <i32 0, i32 3>)
+ ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @tbl2_v2i32(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1) {
+; CHECK-LABEL: tbl2_v2i32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: index z1.d, #0, #3
+; CHECK-NEXT: mov z1.q, q1
+; CHECK-NEXT: movprfx z2, z1
+; CHECK-NEXT: sub z2.d, z2.d, #2 // =0x2
+; CHECK-NEXT: tblq z1.d, { z0.d }, z1.d
+; CHECK-NEXT: tbxq z1.d, z0.d, z2.d
+; CHECK-NEXT: movprfx z0, z1
+; CHECK-NEXT: and z0.d, z0.d, #0xffffffff
+; CHECK-NEXT: ret
+entry:
+ %leg0 = trunc <vscale x 2 x i64> %0 to <vscale x 2 x i32>
+ %leg1 = trunc <vscale x 2 x i64> %0 to <vscale x 2 x i32>
+ %res = call <vscale x 2 x i32> @llvm.vector.segmented.shuffle.nxv2i32.v2i32(
+ <vscale x 2 x i32> %leg0, <vscale x 2 x i32> %leg1, <2 x i32> <i32 0, i32 3>)
+ %res.leg = zext <vscale x 2 x i32> %res to <vscale x 2 x i64>
+ ret <vscale x 2 x i64> %res.leg
+}
+
+define <vscale x 2 x double> @tbl2_v2f64(<vscale x 2 x double> %0, <vscale x 2 x double> %1) {
+; CHECK-LABEL: tbl2_v2f64:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: index z2.d, #0, #3
+; CHECK-NEXT: mov z2.q, q2
+; CHECK-NEXT: movprfx z3, z2
+; CHECK-NEXT: sub z3.d, z3.d, #2 // =0x2
+; CHECK-NEXT: tblq z0.d, { z0.d }, z2.d
+; CHECK-NEXT: tbxq z0.d, z1.d, z3.d
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 2 x double> @llvm.vector.segmented.shuffle.nxv2f64.v2i32(
+ <vscale x 2 x double> %0, <vscale x 2 x double> %1, <2 x i32> <i32 0, i32 3>)
+ ret <vscale x 2 x double> %res
+}
+
+define <vscale x 2 x float> @tbl2_v2f32(<vscale x 2 x float> %0, <vscale x 2 x float> %1) {
+; CHECK-LABEL: tbl2_v2f32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: index z2.d, #0, #3
+; CHECK-NEXT: mov z2.q, q2
+; CHECK-NEXT: movprfx z3, z2
+; CHECK-NEXT: sub z3.d, z3.d, #2 // =0x2
+; CHECK-NEXT: tblq z0.d, { z0.d }, z2.d
+; CHECK-NEXT: tbxq z0.d, z1.d, z3.d
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 2 x float> @llvm.vector.segmented.shuffle.nxv2f32.v2i32(
+ <vscale x 2 x float> %0, <vscale x 2 x float> %1, <2 x i32> <i32 0, i32 3>)
+ ret <vscale x 2 x float> %res
+}
diff --git a/llvm/test/CodeGen/AArch64/sve2p2-segmented-shuffles.ll b/llvm/test/CodeGen/AArch64/sve2p2-segmented-shuffles.ll
new file mode 100644
index 0000000000000..bfa63002adf89
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve2p2-segmented-shuffles.ll
@@ -0,0 +1,212 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64 -mattr=+sve2p2 -o - %s | FileCheck %s
+
+; Test SVE2.2 instructions for segmented shuffles.
+
+
+
+; rev: Should use the zeroing variant.
+
+define <vscale x 16 x i8> @rev_v16i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1) {
+; CHECK-LABEL: rev_v16i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: revd z0.q, p0/z, z0.q
+; CHECK-NEXT: revb z0.d, p1/z, z0.d
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(
+ <vscale x 16 x i8> %0, <vscale x 16 x i8> %1, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+ ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @rev_v8i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1) {
+; CHECK-LABEL: rev_v8i16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: revd z0.q, p0/z, z0.q
+; CHECK-NEXT: revh z0.d, p1/z, z0.d
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 8 x i16> @llvm.vector.segmented.shuffle.nxv8i16.v8i32(
+ <vscale x 8 x i16> %0, <vscale x 8 x i16> %1, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+ ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 8 x i16> @rev_v8i8(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1) {
+; CHECK-LABEL: rev_v8i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: revd z0.q, p0/z, z0.q
+; CHECK-NEXT: revh z0.d, p1/z, z0.d
+; CHECK-NEXT: and z0.h, z0.h, #0xff
+; CHECK-NEXT: ret
+entry:
+ %leg0 = trunc <vscale x 8 x i16> %0 to <vscale x 8 x i8>
+ %leg1 = trunc <vscale x 8 x i16> %1 to <vscale x 8 x i8>
+ %res = call <vscale x 8 x i8> @llvm.vector.segmented.shuffle.nxv8i8.v8i32(
+ <vscale x 8 x i8> %leg0, <vscale x 8 x i8> %leg1, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+ %res.leg = zext <vscale x 8 x i8> %res to <vscale x 8 x i16>
+ ret <vscale x 8 x i16> %res.leg
+}
+
+define <vscale x 8 x half> @rev_v8f16(<vscale x 8 x half> %0, <vscale x 8 x half> %1) {
+; CHECK-LABEL: rev_v8f16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: revd z0.q, p0/z, z0.q
+; CHECK-NEXT: revh z0.d, p1/z, z0.d
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 8 x half> @llvm.vector.segmented.shuffle.nxv8f16.v8i32(
+ <vscale x 8 x half> %0, <vscale x 8 x half> %1, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+ ret <vscale x 8 x half> %res
+}
+
+define <vscale x 8 x bfloat> @rev_v8bf16(<vscale x 8 x bfloat> %0, <vscale x 8 x bfloat> %1) {
+; CHECK-LABEL: rev_v8bf16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: revd z0.q, p0/z, z0.q
+; CHECK-NEXT: revh z0.d, p1/z, z0.d
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 8 x bfloat> @llvm.vector.segmented.shuffle.nxv8bf16.v8i32(
+ <vscale x 8 x bfloat> %0, <vscale x 8 x bfloat> %1, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+ ret <vscale x 8 x bfloat> %res
+}
+
+define <vscale x 4 x i32> @rev_v4i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1) {
+; CHECK-LABEL: rev_v4i32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: revd z0.q, p0/z, z0.q
+; CHECK-NEXT: revw z0.d, p1/z, z0.d
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 4 x i32> @llvm.vector.segmented.shuffle.nxv4i32.v4i32(
+ <vscale x 4 x i32> %0, <vscale x 4 x i32> %1, <4 x i32> <i32 3, i32 2, i32 1, i32 0>)
+ ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @rev_v4i16(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1) {
+; CHECK-LABEL: rev_v4i16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: revd z0.q, p0/z, z0.q
+; CHECK-NEXT: revw z0.d, p1/z, z0.d
+; CHECK-NEXT: and z0.s, z0.s, #0xffff
+; CHECK-NEXT: ret
+entry:
+ %leg0 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i16>
+ %leg1 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i16>
+ %res = call <vscale x 4 x i16> @llvm.vector.segmented.shuffle.nxv4i16.v4i32(
+ <vscale x 4 x i16> %leg0, <vscale x 4 x i16> %leg1, <4 x i32> <i32 3, i32 2, i32 1, i32 0>)
+ %res.leg = zext <vscale x 4 x i16> %res to <vscale x 4 x i32>
+ ret <vscale x 4 x i32> %res.leg
+}
+
+define <vscale x 4 x i32> @rev_v4i8(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1) {
+; CHECK-LABEL: rev_v4i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: revd z0.q, p0/z, z0.q
+; CHECK-NEXT: revw z0.d, p1/z, z0.d
+; CHECK-NEXT: and z0.s, z0.s, #0xff
+; CHECK-NEXT: ret
+entry:
+ %leg0 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i8>
+ %leg1 = trunc <vscale x 4 x i32> %0 to <vscale x 4 x i8>
+ %res = call <vscale x 4 x i8> @llvm.vector.segmented.shuffle.nxv4i8.v4i32(
+ <vscale x 4 x i8> %leg0, <vscale x 4 x i8> %leg1, <4 x i32> <i32 3, i32 2, i32 1, i32 0>)
+ %res.leg = zext <vscale x 4 x i8> %res to <vscale x 4 x i32>
+ ret <vscale x 4 x i32> %res.leg
+}
+
+define <vscale x 4 x float> @rev_v4f32(<vscale x 4 x float> %0, <vscale x 4 x float> %1) {
+; CHECK-LABEL: rev_v4f32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: revd z0.q, p0/z, z0.q
+; CHECK-NEXT: revw z0.d, p1/z, z0.d
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 4 x float> @llvm.vector.segmented.shuffle.nxv4f32.v4i32(
+ <vscale x 4 x float> %0, <vscale x 4 x float> %1, <4 x i32> <i32 3, i32 2, i32 1, i32 0>)
+ ret <vscale x 4 x float> %res
+}
+
+define <vscale x 4 x half> @rev_v4f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1) {
+; CHECK-LABEL: rev_v4f16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: revd z0.q, p0/z, z0.q
+; CHECK-NEXT: revw z0.d, p1/z, z0.d
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 4 x half> @llvm.vector.segmented.shuffle.nxv4f16.v4i32(
+ <vscale x 4 x half> %0, <vscale x 4 x half> %1, <4 x i32> <i32 3, i32 2, i32 1, i32 0>)
+ ret <vscale x 4 x half> %res
+}
+
+define <vscale x 2 x i64> @rev_v2i64(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1) {
+; CHECK-LABEL: rev_v2i64:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: revd z0.q, p0/z, z0.q
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 2 x i64> @llvm.vector.segmented.shuffle.nxv2i64.v2i32(
+ <vscale x 2 x i64> %0, <vscale x 2 x i64> %1, <2 x i32> <i32 1, i32 0>)
+ ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @rev_v2i32(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1) {
+; CHECK-LABEL: rev_v2i32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: revd z0.q, p0/z, z0.q
+; CHECK-NEXT: and z0.d, z0.d, #0xffffffff
+; CHECK-NEXT: ret
+entry:
+ %leg0 = trunc <vscale x 2 x i64> %0 to <vscale x 2 x i32>
+ %leg1 = trunc <vscale x 2 x i64> %0 to <vscale x 2 x i32>
+ %res = call <vscale x 2 x i32> @llvm.vector.segmented.shuffle.nxv2i32.v2i32(
+ <vscale x 2 x i32> %leg0, <vscale x 2 x i32> %leg1, <2 x i32> <i32 1, i32 0>)
+ %res.leg = zext <vscale x 2 x i32> %res to <vscale x 2 x i64>
+ ret <vscale x 2 x i64> %res.leg
+}
+
+define <vscale x 2 x double> @rev_v2f64(<vscale x 2 x double> %0, <vscale x 2 x double> %1) {
+; CHECK-LABEL: rev_v2f64:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: revd z0.q, p0/z, z0.q
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 2 x double> @llvm.vector.segmented.shuffle.nxv2f64.v2i32(
+ <vscale x 2 x double> %0, <vscale x 2 x double> %1, <2 x i32> <i32 1, i32 0>)
+ ret <vscale x 2 x double> %res
+}
+
+define <vscale x 2 x float> @rev_v2f32(<vscale x 2 x float> %0, <vscale x 2 x float> %1) {
+; CHECK-LABEL: rev_v2f32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: revd z0.q, p0/z, z0.q
+; CHECK-NEXT: ret
+entry:
+ %res = call <vscale x 2 x float> @llvm.vector.segmented.shuffle.nxv2f32.v2i32(
+ <vscale x 2 x float> %0, <vscale x 2 x float> %1, <2 x i32> <i32 1, i32 0>)
+ ret <vscale x 2 x float> %res
+}
diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/scalable-deinterleave-intrinsics.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/scalable-deinterleave-intrinsics.ll
index c3b68f828d87c..117785ecfe4b2 100644
--- a/llvm/test/Transforms/InterleavedAccess/AArch64/scalable-deinterleave-intrinsics.ll
+++ b/llvm/test/Transforms/InterleavedAccess/AArch64/scalable-deinterleave-intrinsics.ll
@@ -18,6 +18,59 @@ define void @deinterleave_nxi8_factor2(ptr %ptr) #0 {
ret void
}
+define void @deinterleave_nxi8_factor2_masked(ptr %ptr, <vscale x 16 x i1> %mask) #0 {
+; CHECK-LABEL: define void @deinterleave_nxi8_factor2_masked
+; CHECK-SAME: (ptr [[PTR:%.*]], <vscale x 16 x i1> [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[MASK_WIDE:%.*]] = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> [[MASK]], <vscale x 16 x i1> [[MASK]])
+; CHECK-NEXT: [[LDN:%.*]] = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld2.sret.nxv16i8.p0(<vscale x 16 x i1> [[MASK]], ptr [[PTR]])
+; CHECK-NEXT: [[EXTRACT1:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[LDN]], 1
+; CHECK-NEXT: [[EXTRACT2:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[LDN]], 0
+; CHECK-NEXT: ret void
+;
+ %mask.wide = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask)
+ %load = call <vscale x 32 x i8> @llvm.masked.load.nxv32i8.p0(ptr %ptr, i32 8, <vscale x 32 x i1> %mask.wide, <vscale x 32 x i8> poison)
+ %deinterleave = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.vector.deinterleave2.nxv32i8(<vscale x 32 x i8> %load)
+ %extract1 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %deinterleave, 1
+ %extract2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %deinterleave, 0
+ ret void
+}
+
+define void @deinterleave_nxi8_factor2_masked_zero_passthru(ptr %ptr, <vscale x 16 x i1> %mask) #0 {
+; CHECK-LABEL: define void @deinterleave_nxi8_factor2_masked_zero_passthru
+; CHECK-SAME: (ptr [[PTR:%.*]], <vscale x 16 x i1> [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[MASK_WIDE:%.*]] = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> [[MASK]], <vscale x 16 x i1> [[MASK]])
+; CHECK-NEXT: [[LDN:%.*]] = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld2.sret.nxv16i8.p0(<vscale x 16 x i1> [[MASK]], ptr [[PTR]])
+; CHECK-NEXT: [[EXTRACT1:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[LDN]], 1
+; CHECK-NEXT: [[EXTRACT2:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[LDN]], 0
+; CHECK-NEXT: ret void
+;
+ %mask.wide = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask)
+ %load = call <vscale x 32 x i8> @llvm.masked.load.nxv32i8.p0(ptr %ptr, i32 8, <vscale x 32 x i1> %mask.wide, <vscale x 32 x i8> splat (i8 0))
+ %deinterleave = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.vector.deinterleave2.nxv32i8(<vscale x 32 x i8> %load)
+ %extract1 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %deinterleave, 1
+ %extract2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %deinterleave, 0
+ ret void
+}
+
+; Passthru is neither 0 nor poison, this isn't supported yet.
+define void @deinterleave_nxi8_factor2_masked_other_passthru(ptr %ptr, <vscale x 16 x i1> %mask) #0 {
+; CHECK-LABEL: define void @deinterleave_nxi8_factor2_masked_other_passthru
+; CHECK-SAME: (ptr [[PTR:%.*]], <vscale x 16 x i1> [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[MASK_WIDE:%.*]] = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> [[MASK]], <vscale x 16 x i1> [[MASK]])
+; CHECK-NEXT: [[LOAD:%.*]] = call <vscale x 32 x i8> @llvm.masked.load.nxv32i8.p0(ptr align 8 [[PTR]], <vscale x 32 x i1> [[MASK_WIDE]], <vscale x 32 x i8> splat (i8 5))
+; CHECK-NEXT: [[DEINTERLEAVE:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.vector.deinterleave2.nxv32i8(<vscale x 32 x i8> [[LOAD]])
+; CHECK-NEXT: [[EXTRACT1:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[DEINTERLEAVE]], 1
+; CHECK-NEXT: [[EXTRACT2:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[DEINTERLEAVE]], 0
+; CHECK-NEXT: ret void
+;
+ %mask.wide = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask)
+ %load = call <vscale x 32 x i8> @llvm.masked.load.nxv32i8.p0(ptr %ptr, i32 8, <vscale x 32 x i1> %mask.wide, <vscale x 32 x i8> splat (i8 5))
+ %deinterleave = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.vector.deinterleave2.nxv32i8(<vscale x 32 x i8> %load)
+ %extract1 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %deinterleave, 1
+ %extract2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %deinterleave, 0
+ ret void
+}
+
define void @deinterleave_nxi16_factor2(ptr %ptr) #0 {
; CHECK-LABEL: define void @deinterleave_nxi16_factor2
; CHECK-SAME: (ptr [[PTR:%.*]]) #[[ATTR0]] {
@@ -33,6 +86,19 @@ define void @deinterleave_nxi16_factor2(ptr %ptr) #0 {
ret void
}
+define void @deinterleave_nxi16_factor2_masked(ptr %ptr, <vscale x 8 x i1> %mask) #0 {
+; CHECK-LABEL: define void @deinterleave_nxi16_factor2_masked
+; CHECK-SAME: (ptr [[PTR:%.*]], <vscale x 8 x i1> [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[MASK_WIDE:%.*]] = call <vscale x 16 x i1> @llvm.vector.interleave2.nxv16i1(<vscale x 8 x i1> [[MASK]], <vscale x 8 x i1> [[MASK]])
+; CHECK-NEXT: [[LDN:%.*]] = call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld2.sret.nxv8i16.p0(<vscale x 8 x i1> [[MASK]], ptr [[PTR]])
+; CHECK-NEXT: ret void
+;
+ %mask.wide = call <vscale x 16 x i1> @llvm.vector.interleave2.nxv16i1(<vscale x 8 x i1> %mask, <vscale x 8 x i1> %mask)
+ %load = call <vscale x 16 x i16> @llvm.masked.load.nxv16i16.p0(ptr %ptr, i32 16, <vscale x 16 x i1> %mask.wide, <vscale x 16 x i16> poison)
+ %deinterleave = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.vector.deinterleave2.nxv16i16(<vscale x 16 x i16> %load)
+ ret void
+}
+
define void @deinterleave_nx8xi32_factor2(ptr %ptr) #0 {
; CHECK-LABEL: define void @deinterleave_nx8xi32_factor2
; CHECK-SAME: (ptr [[PTR:%.*]]) #[[ATTR0]] {
@@ -48,6 +114,23 @@ define void @deinterleave_nx8xi32_factor2(ptr %ptr) #0 {
ret void
}
+define void @deinterleave_nx8xi32_factor2_masked(ptr %ptr, <vscale x 4 x i1> %mask) #0 {
+; CHECK-LABEL: define void @deinterleave_nx8xi32_factor2_masked
+; CHECK-SAME: (ptr [[PTR:%.*]], <vscale x 4 x i1> [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[MASK_WIDE:%.*]] = call <vscale x 8 x i1> @llvm.vector.interleave2.nxv8i1(<vscale x 4 x i1> [[MASK]], <vscale x 4 x i1> [[MASK]])
+; CHECK-NEXT: [[LDN:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld2.sret.nxv4i32.p0(<vscale x 4 x i1> [[MASK]], ptr [[PTR]])
+; CHECK-NEXT: [[EXTRACT1:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN]], 0
+; CHECK-NEXT: [[EXTRACT2:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN]], 1
+; CHECK-NEXT: ret void
+;
+ %mask.wide = call <vscale x 8 x i1> @llvm.vector.interleave2.nxv8i1(<vscale x 4 x i1> %mask, <vscale x 4 x i1> %mask)
+ %load = call <vscale x 8 x i32> @llvm.masked.load.nxv8i32.p0(ptr %ptr, i32 32, <vscale x 8 x i1> %mask.wide, <vscale x 8 x i32> poison)
+ %deinterleave = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %load)
+ %extract1 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %deinterleave, 0
+ %extract2 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %deinterleave, 1
+ ret void
+}
+
define void @deinterleave_nxi64_factor2(ptr %ptr) #0 {
; CHECK-LABEL: define void @deinterleave_nxi64_factor2
; CHECK-SAME: (ptr [[PTR:%.*]]) #[[ATTR0]] {
@@ -63,6 +146,23 @@ define void @deinterleave_nxi64_factor2(ptr %ptr) #0 {
ret void
}
+define void @deinterleave_nxi64_factor2_masked(ptr %ptr, <vscale x 2 x i1> %mask) #0 {
+; CHECK-LABEL: define void @deinterleave_nxi64_factor2_masked
+; CHECK-SAME: (ptr [[PTR:%.*]], <vscale x 2 x i1> [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[MASK_WIDE:%.*]] = call <vscale x 4 x i1> @llvm.vector.interleave2.nxv4i1(<vscale x 2 x i1> [[MASK]], <vscale x 2 x i1> [[MASK]])
+; CHECK-NEXT: [[LDN:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld2.sret.nxv2i64.p0(<vscale x 2 x i1> [[MASK]], ptr [[PTR]])
+; CHECK-NEXT: [[EXTRACT1:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 0
+; CHECK-NEXT: [[EXTRACT2:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 1
+; CHECK-NEXT: ret void
+;
+ %mask.wide = call <vscale x 4 x i1> @llvm.vector.interleave2.nxv4i1(<vscale x 2 x i1> %mask, <vscale x 2 x i1> %mask)
+ %load = call <vscale x 4 x i64> @llvm.masked.load.nxv4i64.p0(ptr %ptr, i32 64, <vscale x 4 x i1> %mask.wide, <vscale x 4 x i64> poison)
+ %deinterleave = tail call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave2.nxv4i64(<vscale x 4 x i64> %load)
+ %extract1 = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } %deinterleave, 0
+ %extract2 = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } %deinterleave, 1
+ ret void
+}
+
define void @deinterleave_nxfloat_factor2(ptr %ptr) #0 {
; CHECK-LABEL: define void @deinterleave_nxfloat_factor2
; CHECK-SAME: (ptr [[PTR:%.*]]) #[[ATTR0]] {
@@ -119,6 +219,19 @@ define void @interleave_nxi8_factor2(ptr %ptr, <vscale x 16 x i8> %l, <vscale x
ret void
}
+define void @interleave_nxi8_factor2_masked(ptr %ptr, <vscale x 16 x i8> %l, <vscale x 16 x i8> %r, <vscale x 16 x i1> %mask) #0 {
+; CHECK-LABEL: define void @interleave_nxi8_factor2_masked
+; CHECK-SAME: (ptr [[PTR:%.*]], <vscale x 16 x i8> [[L:%.*]], <vscale x 16 x i8> [[R:%.*]], <vscale x 16 x i1> [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[MASK_WIDE:%.*]] = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> [[MASK]], <vscale x 16 x i1> [[MASK]])
+; CHECK-NEXT: call void @llvm.aarch64.sve.st2.nxv16i8.p0(<vscale x 16 x i8> [[L]], <vscale x 16 x i8> [[R]], <vscale x 16 x i1> [[MASK]], ptr [[PTR]])
+; CHECK-NEXT: ret void
+;
+ %interleave = tail call <vscale x 32 x i8> @llvm.vector.interleave2.nxv32i8(<vscale x 16 x i8> %l, <vscale x 16 x i8> %r)
+ %mask.wide = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask)
+ call void @llvm.masked.store.nxv32i8(<vscale x 32 x i8> %interleave, ptr %ptr, i32 8, <vscale x 32 x i1> %mask.wide)
+ ret void
+}
+
define void @interleave_nxi16_factor2(ptr %ptr, <vscale x 8 x i16> %l, <vscale x 8 x i16> %r) #0 {
; CHECK-LABEL: define void @interleave_nxi16_factor2
; CHECK-SAME: (ptr [[PTR:%.*]], <vscale x 8 x i16> [[L:%.*]], <vscale x 8 x i16> [[R:%.*]]) #[[ATTR0]] {
diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/sve-deinterleave-segments.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/sve-deinterleave-segments.ll
new file mode 100644
index 0000000000000..2e50384d7d0d0
--- /dev/null
+++ b/llvm/test/Transforms/InterleavedAccess/AArch64/sve-deinterleave-segments.ll
@@ -0,0 +1,176 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; RUN: opt < %s -passes=interleaved-access -mtriple=aarch64-linux-gnu -mattr=+sve2p1 -S | FileCheck %s
+
+; SVE2p1 supports quad deinterleaving, i.e. segments of 128 bits.
+; Verify we combine into ld2q/ld4q.
+
+; TODO-REVEC: Test all types.
+
+define void @deinterleave4(ptr %src) {
+; CHECK-LABEL: define void @deinterleave4
+; CHECK-SAME: (ptr [[SRC:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[TMP1:%.*]] = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv4i1(<vscale x 4 x i1> splat (i1 true))
+; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> [[TMP1]])
+; CHECK-NEXT: [[LDN:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld4q.qpred.nxv4i32(<vscale x 1 x i1> [[TMP2]], ptr [[SRC]])
+; CHECK-NEXT: ret void
+;
+ %load = load <vscale x 16 x i32>, ptr %src, align 4
+ %deinterleave = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave.segments4.nxv16i32(<vscale x 16 x i32> %load)
+ ret void
+}
+
+define void @deinterleave4_masked(ptr %src, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: define void @deinterleave4_masked
+; CHECK-SAME: (ptr [[SRC:%.*]], <vscale x 1 x i1> [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[MASK_WIDE:%.*]] = call <vscale x 16 x i1> @llvm.vector.stretch.nxv16i1.nxv1i1(<vscale x 1 x i1> [[MASK]])
+; CHECK-NEXT: [[LDN:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld4q.qpred.nxv4i32(<vscale x 1 x i1> [[MASK]], ptr [[SRC]])
+; CHECK-NEXT: ret void
+;
+ %mask.wide = call <vscale x 16 x i1> @llvm.vector.stretch.nxv16i1.nxv1i1(<vscale x 1 x i1> %mask)
+ %load = call <vscale x 16 x i32> @llvm.masked.load.nxv16i32.p0(ptr %src, i32 4, <vscale x 16 x i1> %mask.wide, <vscale x 16 x i32> poison)
+ %deinterleave = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave.segments4.nxv16i32(<vscale x 16 x i32> %load)
+ ret void
+}
+
+define void @deinterleave2_i64(ptr %src) {
+; CHECK-LABEL: define void @deinterleave2_i64
+; CHECK-SAME: (ptr [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv2i1(<vscale x 2 x i1> splat (i1 true))
+; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> [[TMP1]])
+; CHECK-NEXT: [[LDN:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld2q.qpred.nxv2i64(<vscale x 1 x i1> [[TMP2]], ptr [[SRC]])
+; CHECK-NEXT: ret void
+;
+ %load = load <vscale x 4 x i64>, ptr %src, align 4
+ %deinterleave = tail call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave.segments2.nxv8i32(<vscale x 4 x i64> %load)
+ ret void
+}
+
+define void @deinterleave2_f64(ptr %src) {
+; CHECK-LABEL: define void @deinterleave2_f64
+; CHECK-SAME: (ptr [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv2i1(<vscale x 2 x i1> splat (i1 true))
+; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> [[TMP1]])
+; CHECK-NEXT: [[LDN:%.*]] = call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld2q.qpred.nxv2f64(<vscale x 1 x i1> [[TMP2]], ptr [[SRC]])
+; CHECK-NEXT: ret void
+;
+ %load = load <vscale x 4 x double>, ptr %src, align 4
+ %deinterleave = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.vector.deinterleave.segments2.nxv8i32(<vscale x 4 x double> %load)
+ ret void
+}
+
+define void @deinterleave2_i32(ptr %src) {
+; CHECK-LABEL: define void @deinterleave2_i32
+; CHECK-SAME: (ptr [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv4i1(<vscale x 4 x i1> splat (i1 true))
+; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> [[TMP1]])
+; CHECK-NEXT: [[LDN:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld2q.qpred.nxv4i32(<vscale x 1 x i1> [[TMP2]], ptr [[SRC]])
+; CHECK-NEXT: ret void
+;
+ %load = load <vscale x 8 x i32>, ptr %src, align 4
+ %deinterleave = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave.segments2.nxv8i32(<vscale x 8 x i32> %load)
+ ret void
+}
+
+define void @deinterleave2_f32(ptr %src) {
+; CHECK-LABEL: define void @deinterleave2_f32
+; CHECK-SAME: (ptr [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv4i1(<vscale x 4 x i1> splat (i1 true))
+; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> [[TMP1]])
+; CHECK-NEXT: [[LDN:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld2q.qpred.nxv4f32(<vscale x 1 x i1> [[TMP2]], ptr [[SRC]])
+; CHECK-NEXT: ret void
+;
+ %load = load <vscale x 8 x float>, ptr %src, align 4
+ %deinterleave = tail call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave.segments2.nxv8i32(<vscale x 8 x float> %load)
+ ret void
+}
+
+define void @deinterleave2_i16(ptr %src) {
+; CHECK-LABEL: define void @deinterleave2_i16
+; CHECK-SAME: (ptr [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv8i1(<vscale x 8 x i1> splat (i1 true))
+; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> [[TMP1]])
+; CHECK-NEXT: [[LDN:%.*]] = call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld2q.qpred.nxv8i16(<vscale x 1 x i1> [[TMP2]], ptr [[SRC]])
+; CHECK-NEXT: ret void
+;
+ %load = load <vscale x 16 x i16>, ptr %src, align 4
+ %deinterleave = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.vector.deinterleave.segments2.nxv8i32(<vscale x 16 x i16> %load)
+ ret void
+}
+
+define void @deinterleave2_i16_masked(ptr %src, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: define void @deinterleave2_i16_masked
+; CHECK-SAME: (ptr [[SRC:%.*]], <vscale x 1 x i1> [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[MASK_WIDE:%.*]] = call <vscale x 16 x i1> @llvm.vector.stretch.nxv16i1.nxv1i1(<vscale x 1 x i1> [[MASK]])
+; CHECK-NEXT: [[LDN:%.*]] = call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld2q.qpred.nxv8i16(<vscale x 1 x i1> [[MASK]], ptr [[SRC]])
+; CHECK-NEXT: ret void
+;
+ %mask.wide = call <vscale x 16 x i1> @llvm.vector.stretch.nxv16i1.nxv1i1(<vscale x 1 x i1> %mask)
+ %load = call <vscale x 16 x i16> @llvm.masked.load.nxv16i16.p0(ptr %src, i32 16, <vscale x 16 x i1> %mask.wide, <vscale x 16 x i16> poison)
+ %deinterleave = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.vector.deinterleave.segments2.nxv8i32(<vscale x 16 x i16> %load)
+ ret void
+}
+
+define void @deinterleave2_f16(ptr %src) {
+; CHECK-LABEL: define void @deinterleave2_f16
+; CHECK-SAME: (ptr [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv8i1(<vscale x 8 x i1> splat (i1 true))
+; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> [[TMP1]])
+; CHECK-NEXT: [[LDN:%.*]] = call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld2q.qpred.nxv8f16(<vscale x 1 x i1> [[TMP2]], ptr [[SRC]])
+; CHECK-NEXT: ret void
+;
+ %load = load <vscale x 16 x half>, ptr %src, align 4
+ %deinterleave = tail call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.vector.deinterleave.segments2.nxv8i32(<vscale x 16 x half> %load)
+ ret void
+}
+
+define void @deinterleave2_bf16(ptr %src) {
+; CHECK-LABEL: define void @deinterleave2_bf16
+; CHECK-SAME: (ptr [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv8i1(<vscale x 8 x i1> splat (i1 true))
+; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> [[TMP1]])
+; CHECK-NEXT: [[LDN:%.*]] = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ld2q.qpred.nxv8bf16(<vscale x 1 x i1> [[TMP2]], ptr [[SRC]])
+; CHECK-NEXT: ret void
+;
+ %load = load <vscale x 16 x bfloat>, ptr %src, align 4
+ %deinterleave = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.vector.deinterleave.segments2.nxv8i32(<vscale x 16 x bfloat> %load)
+ ret void
+}
+
+define void @deinterleave2_i8(ptr %src) {
+; CHECK-LABEL: define void @deinterleave2_i8
+; CHECK-SAME: (ptr [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv16i1(<vscale x 16 x i1> splat (i1 true))
+; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> [[TMP1]])
+; CHECK-NEXT: [[LDN:%.*]] = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld2q.qpred.nxv16i8(<vscale x 1 x i1> [[TMP2]], ptr [[SRC]])
+; CHECK-NEXT: ret void
+;
+ %load = load <vscale x 32 x i8>, ptr %src, align 4
+ %deinterleave = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.vector.deinterleave.segments2.nxv8i32(<vscale x 32 x i8> %load)
+ ret void
+}
+
+; Cannot deinterleave segments of 64 bits
+define void @narrow_deinterleave2(ptr %src) {
+; CHECK-LABEL: define void @narrow_deinterleave2
+; CHECK-SAME: (ptr [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[SRC]], align 4
+; CHECK-NEXT: [[DEINTERLEAVE:%.*]] = tail call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave.segments2.nxv4i32(<vscale x 4 x i32> [[LOAD]])
+; CHECK-NEXT: ret void
+;
+ %load = load <vscale x 4 x i32>, ptr %src, align 4
+ %deinterleave = tail call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave.segments2.nxv4i32(<vscale x 4 x i32> %load)
+ ret void
+}
+
+; Cannot deinterleave segments of 256 bits
+define void @wide_deinterleave2(ptr %src) {
+; CHECK-LABEL: define void @wide_deinterleave2
+; CHECK-SAME: (ptr [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[LOAD:%.*]] = load <vscale x 16 x i32>, ptr [[SRC]], align 4
+; CHECK-NEXT: [[DEINTERLEAVE:%.*]] = tail call { <vscale x 8 x i32>, <vscale x 8 x i32> } @llvm.vector.deinterleave.segments2.nxv16i32(<vscale x 16 x i32> [[LOAD]])
+; CHECK-NEXT: ret void
+;
+ %load = load <vscale x 16 x i32>, ptr %src, align 4
+ %deinterleave = tail call { <vscale x 8 x i32>, <vscale x 8 x i32> } @llvm.vector.deinterleave.segments2.nxv16i32(<vscale x 16 x i32> %load)
+ ret void
+}
diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/sve-interleave-segments.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/sve-interleave-segments.ll
new file mode 100644
index 0000000000000..47eeec909cbf2
--- /dev/null
+++ b/llvm/test/Transforms/InterleavedAccess/AArch64/sve-interleave-segments.ll
@@ -0,0 +1,85 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; RUN: opt < %s -passes=interleaved-access -mtriple=aarch64-linux-gnu -mattr=+sve2p1 -S | FileCheck %s
+
+; SVE2p1 supports quad interleaving, i.e. segments of 128 bits.
+; Verify we combine into st2q/st4q.
+
+; TODO-REVEC: Test all types.
+
+define void @interleave4(ptr %dst, <vscale x 4 x i32> %v0, <vscale x 4 x i32> %v1, <vscale x 4 x i32> %v2, <vscale x 4 x i32> %v3) {
+; CHECK-LABEL: define void @interleave4
+; CHECK-SAME: (ptr [[DST:%.*]], <vscale x 4 x i32> [[V0:%.*]], <vscale x 4 x i32> [[V1:%.*]], <vscale x 4 x i32> [[V2:%.*]], <vscale x 4 x i32> [[V3:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[TMP1:%.*]] = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv4i1(<vscale x 4 x i1> splat (i1 true))
+; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> [[TMP1]])
+; CHECK-NEXT: call void @llvm.aarch64.sve.st4q.qpred.nxv4i32(<vscale x 4 x i32> [[V0]], <vscale x 4 x i32> [[V1]], <vscale x 4 x i32> [[V2]], <vscale x 4 x i32> [[V3]], <vscale x 1 x i1> [[TMP2]], ptr [[DST]])
+; CHECK-NEXT: ret void
+;
+ %interleaved.vec = call <vscale x 16 x i32> @llvm.vector.interleave.segments4.nxv16i32(<vscale x 4 x i32> %v0, <vscale x 4 x i32> %v1, <vscale x 4 x i32> %v2, <vscale x 4 x i32> %v3)
+ store <vscale x 16 x i32> %interleaved.vec, ptr %dst, align 4
+ ret void
+}
+
+define void @interleave4_masked(ptr %dst, <vscale x 4 x i32> %v0, <vscale x 4 x i32> %v1, <vscale x 4 x i32> %v2, <vscale x 4 x i32> %v3, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: define void @interleave4_masked
+; CHECK-SAME: (ptr [[DST:%.*]], <vscale x 4 x i32> [[V0:%.*]], <vscale x 4 x i32> [[V1:%.*]], <vscale x 4 x i32> [[V2:%.*]], <vscale x 4 x i32> [[V3:%.*]], <vscale x 1 x i1> [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[MASK_WIDE:%.*]] = call <vscale x 16 x i1> @llvm.vector.stretch.nxv16i1.nxv1i1(<vscale x 1 x i1> [[MASK]])
+; CHECK-NEXT: call void @llvm.aarch64.sve.st4q.qpred.nxv4i32(<vscale x 4 x i32> [[V0]], <vscale x 4 x i32> [[V1]], <vscale x 4 x i32> [[V2]], <vscale x 4 x i32> [[V3]], <vscale x 1 x i1> [[MASK]], ptr [[DST]])
+; CHECK-NEXT: ret void
+;
+ %interleaved.vec = call <vscale x 16 x i32> @llvm.vector.interleave.segments4.nxv16i32(<vscale x 4 x i32> %v0, <vscale x 4 x i32> %v1, <vscale x 4 x i32> %v2, <vscale x 4 x i32> %v3)
+ %mask.wide = call <vscale x 16 x i1> @llvm.vector.stretch.nxv32i1.nxv1i1(<vscale x 1 x i1> %mask)
+ call void @llvm.masked.store.nxv16i32(<vscale x 16 x i32> %interleaved.vec, ptr %dst, i32 4, <vscale x 16 x i1> %mask.wide)
+ ret void
+}
+
+; Cannot interleave segments of 64 bits.
+define void @narrow_interleave4(ptr %dst, <vscale x 2 x i32> %v0, <vscale x 2 x i32> %v1, <vscale x 2 x i32> %v2, <vscale x 2 x i32> %v3) {
+; CHECK-LABEL: define void @narrow_interleave4
+; CHECK-SAME: (ptr [[DST:%.*]], <vscale x 2 x i32> [[V0:%.*]], <vscale x 2 x i32> [[V1:%.*]], <vscale x 2 x i32> [[V2:%.*]], <vscale x 2 x i32> [[V3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 8 x i32> @llvm.vector.interleave.segments4.nxv8i32(<vscale x 2 x i32> [[V0]], <vscale x 2 x i32> [[V1]], <vscale x 2 x i32> [[V2]], <vscale x 2 x i32> [[V3]])
+; CHECK-NEXT: store <vscale x 8 x i32> [[INTERLEAVED_VEC]], ptr [[DST]], align 4
+; CHECK-NEXT: ret void
+;
+ %interleaved.vec = call <vscale x 8 x i32> @llvm.vector.interleave.segments4.nxv16i32(<vscale x 2 x i32> %v0, <vscale x 2 x i32> %v1, <vscale x 2 x i32> %v2, <vscale x 2 x i32> %v3)
+ store <vscale x 8 x i32> %interleaved.vec, ptr %dst, align 4
+ ret void
+}
+
+; Cannot interleave segments of 256 bits.
+define void @wide_interleave4(ptr %dst, <vscale x 8 x i32> %v0, <vscale x 8 x i32> %v1, <vscale x 8 x i32> %v2, <vscale x 8 x i32> %v3) {
+; CHECK-LABEL: define void @wide_interleave4
+; CHECK-SAME: (ptr [[DST:%.*]], <vscale x 8 x i32> [[V0:%.*]], <vscale x 8 x i32> [[V1:%.*]], <vscale x 8 x i32> [[V2:%.*]], <vscale x 8 x i32> [[V3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 32 x i32> @llvm.vector.interleave.segments4.nxv32i32(<vscale x 8 x i32> [[V0]], <vscale x 8 x i32> [[V1]], <vscale x 8 x i32> [[V2]], <vscale x 8 x i32> [[V3]])
+; CHECK-NEXT: store <vscale x 32 x i32> [[INTERLEAVED_VEC]], ptr [[DST]], align 4
+; CHECK-NEXT: ret void
+;
+ %interleaved.vec = call <vscale x 32 x i32> @llvm.vector.interleave.segments4.nxv32i32(<vscale x 8 x i32> %v0, <vscale x 8 x i32> %v1, <vscale x 8 x i32> %v2, <vscale x 8 x i32> %v3)
+ store <vscale x 32 x i32> %interleaved.vec, ptr %dst, align 4
+ ret void
+}
+
+define void @interleave2(ptr %dst, <vscale x 4 x i32> %v0, <vscale x 4 x i32> %v1, <vscale x 4 x i32> %v2, <vscale x 4 x i32> %v3) {
+; CHECK-LABEL: define void @interleave2
+; CHECK-SAME: (ptr [[DST:%.*]], <vscale x 4 x i32> [[V0:%.*]], <vscale x 4 x i32> [[V1:%.*]], <vscale x 4 x i32> [[V2:%.*]], <vscale x 4 x i32> [[V3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv4i1(<vscale x 4 x i1> splat (i1 true))
+; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> [[TMP1]])
+; CHECK-NEXT: call void @llvm.aarch64.sve.st2q.qpred.nxv4i32(<vscale x 4 x i32> [[V0]], <vscale x 4 x i32> [[V1]], <vscale x 1 x i1> [[TMP2]], ptr [[DST]])
+; CHECK-NEXT: ret void
+;
+ %interleaved.vec = call <vscale x 8 x i32> @llvm.vector.interleave.segments2.nxv8i32(<vscale x 4 x i32> %v0, <vscale x 4 x i32> %v1)
+ store <vscale x 8 x i32> %interleaved.vec, ptr %dst, align 4
+ ret void
+}
+
+define void @interleave2_nxi8_masked(ptr %ptr, <vscale x 16 x i8> %l, <vscale x 16 x i8> %r, <vscale x 1 x i1> %mask) #0 {
+; CHECK-LABEL: define void @interleave2_nxi8_masked
+; CHECK-SAME: (ptr [[PTR:%.*]], <vscale x 16 x i8> [[L:%.*]], <vscale x 16 x i8> [[R:%.*]], <vscale x 1 x i1> [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[MASK_WIDE:%.*]] = call <vscale x 32 x i1> @llvm.vector.stretch.nxv32i1.nxv1i1(<vscale x 1 x i1> [[MASK]])
+; CHECK-NEXT: call void @llvm.aarch64.sve.st2q.qpred.nxv16i8(<vscale x 16 x i8> [[L]], <vscale x 16 x i8> [[R]], <vscale x 1 x i1> [[MASK]], ptr [[PTR]])
+; CHECK-NEXT: ret void
+;
+ %interleave = tail call <vscale x 32 x i8> @llvm.vector.interleave.segments2.nxv32i8(<vscale x 16 x i8> %l, <vscale x 16 x i8> %r)
+ %mask.wide = call <vscale x 32 x i1> @llvm.vector.stretch.nxv32i1.nxv1i1(<vscale x 1 x i1> %mask)
+ call void @llvm.masked.store.nxv32i8(<vscale x 32 x i8> %interleave, ptr %ptr, i32 8, <vscale x 32 x i1> %mask.wide)
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/invalid-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/invalid-costs.ll
index 37b6362c12927..ffff56f4919e1 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/invalid-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/invalid-costs.ll
@@ -4,7 +4,8 @@
target triple = "arm64-apple-macosx"
-; REMARKS: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): load
+; REMARKS: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): sext
+; REMARKS: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): call to llvm.masked.sdiv
; Test case for https://github.com/llvm/llvm-project/issues/160792.
define void @replicate_sdiv_conditional(ptr noalias %a, ptr noalias %b, ptr noalias %c) #0 {
; CHECK-LABEL: define void @replicate_sdiv_conditional(
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/reduction-sdot.ll b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-sdot.ll
new file mode 100644
index 0000000000000..e1913c5a62313
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-sdot.ll
@@ -0,0 +1,76 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=1 -vectorize-vector-loops \
+; RUN: -mtriple=aarch64 -mattr=+sve2p1 -S | FileCheck %s --check-prefixes=CHECK-SVE2p1,CHECK-SVE2p1-out
+; RUN: opt < %s -passes=loop-vectorize \
+; RUN: -prefer-inloop-reductions -force-vector-interleave=1 -vectorize-vector-loops \
+; RUN: -mtriple=aarch64 -mattr=+sve2p1 -S | FileCheck %s --check-prefixes=CHECK-SVE2p1,CHECK-SVE2p1-inl
+
+; Test re-vectorisation of sdot-based reductions.
+; Note that inloop reductions are not support by LV for RK=TargetIntAccumulation.
+; As a consequence, -prefer-inloop-reductions has no effect, but it's still tested here.
+
+define <4 x i32> @reduction_sum_4xi32(ptr noalias nocapture %A) {
+; CHECK-SVE2p1-LABEL: @reduction_sum_4xi32(
+; CHECK-SVE2p1-NEXT: entry:
+; CHECK-SVE2p1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE2p1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 64, [[TMP0]]
+; CHECK-SVE2p1-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-SVE2p1: vector.ph:
+; CHECK-SVE2p1-NEXT: [[N_MOD_VF:%.*]] = urem i64 64, [[TMP0]]
+; CHECK-SVE2p1-NEXT: [[N_VEC:%.*]] = sub i64 64, [[N_MOD_VF]]
+; CHECK-SVE2p1-NEXT: [[TMP2:%.*]] = shl i64 [[N_VEC]], 2
+; CHECK-SVE2p1-NEXT: [[TMP3:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v4i32(<vscale x 4 x i32> zeroinitializer, <4 x i32> zeroinitializer, i64 0)
+; CHECK-SVE2p1-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK-SVE2p1: vector.body:
+; CHECK-SVE2p1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE2p1-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ [[TMP3]], [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE2p1-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 2
+; CHECK-SVE2p1-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[OFFSET_IDX]]
+; CHECK-SVE2p1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 4
+; CHECK-SVE2p1-NEXT: [[TMP5]] = call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.nxv4i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i8> [[WIDE_LOAD]], <vscale x 16 x i8> [[WIDE_LOAD]])
+; CHECK-SVE2p1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; CHECK-SVE2p1-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-SVE2p1-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-SVE2p1: middle.block:
+; CHECK-SVE2p1-NEXT: [[TMP7:%.*]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.nxv4i32(<4 x i32> zeroinitializer, <vscale x 4 x i32> [[TMP5]])
+; CHECK-SVE2p1-NEXT: [[CMP_N:%.*]] = icmp eq i64 64, [[N_VEC]]
+; CHECK-SVE2p1-NEXT: br i1 [[CMP_N]], label [[LOOP_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-SVE2p1: scalar.ph:
+; CHECK-SVE2p1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP2]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-SVE2p1-NEXT: [[BC_MERGE_RDX:%.*]] = phi <4 x i32> [ [[TMP7]], [[MIDDLE_BLOCK]] ], [ zeroinitializer, [[ENTRY]] ]
+; CHECK-SVE2p1-NEXT: br label [[SCALAR_LOOP:%.*]]
+; CHECK-SVE2p1: scalar.loop:
+; CHECK-SVE2p1-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[SCALAR_LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-SVE2p1-NEXT: [[SUM_02:%.*]] = phi <4 x i32> [ [[L7:%.*]], [[SCALAR_LOOP]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-SVE2p1-NEXT: [[L2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-SVE2p1-NEXT: [[L3:%.*]] = load <16 x i8>, ptr [[L2]], align 4
+; CHECK-SVE2p1-NEXT: [[L7]] = call <4 x i32> @llvm.aarch64.neon.sdot.v4i32.v16i8(<4 x i32> [[SUM_02]], <16 x i8> [[L3]], <16 x i8> [[L3]])
+; CHECK-SVE2p1-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 4
+; CHECK-SVE2p1-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-SVE2p1-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], 256
+; CHECK-SVE2p1-NEXT: br i1 [[EXITCOND]], label [[LOOP_EXIT]], label [[SCALAR_LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-SVE2p1: loop.exit:
+; CHECK-SVE2p1-NEXT: [[SUM_0_LCSSA:%.*]] = phi <4 x i32> [ [[L7]], [[SCALAR_LOOP]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]
+; CHECK-SVE2p1-NEXT: ret <4 x i32> [[SUM_0_LCSSA]]
+;
+entry:
+ br label %scalar.loop
+
+scalar.loop: ; preds = %entry, %scalar.loop
+ %indvars.iv = phi i64 [ %indvars.iv.next, %scalar.loop ], [ 0, %entry ]
+ %sum.02 = phi <4 x i32> [ %l7, %scalar.loop ], [ zeroinitializer, %entry ]
+ %l2 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
+ %l3 = load <16 x i8>, ptr %l2, align 4
+ %l7 = call <4 x i32> @llvm.aarch64.neon.sdot.v4i32.v16i8(<4 x i32> %sum.02, <16 x i8> %l3, <16 x i8> %l3)
+ %indvars.iv.next = add i64 %indvars.iv, 4
+ %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+ %exitcond = icmp eq i32 %lftr.wideiv, 256
+ br i1 %exitcond, label %loop.exit, label %scalar.loop
+
+loop.exit: ; preds = %scalar.loop
+ %sum.0.lcssa = phi <4 x i32> [ %l7, %scalar.loop ]
+ ret <4 x i32> %sum.0.lcssa
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK-SVE2p1-inl: {{.*}}
+; CHECK-SVE2p1-out: {{.*}}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/reduction-udot.ll b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-udot.ll
new file mode 100644
index 0000000000000..47537ebe635b1
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-udot.ll
@@ -0,0 +1,278 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=1 -vectorize-vector-loops \
+; RUN: -mtriple=aarch64 -mattr=+sve2 -S | FileCheck %s --check-prefixes=CHECK-SVE2
+; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=1 -vectorize-vector-loops \
+; RUN: -mtriple=aarch64 -mattr=+sve2p1 -S | FileCheck %s --check-prefixes=CHECK-SVE2p1,CHECK-SVE2p1-out
+; RUN: opt < %s -passes=loop-vectorize \
+; RUN: -prefer-inloop-reductions -force-vector-interleave=1 -vectorize-vector-loops \
+; RUN: -mtriple=aarch64 -mattr=+sve2p1 -S | FileCheck %s --check-prefixes=CHECK-SVE2p1,CHECK-SVE2p1-inl
+; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=2 -vectorize-vector-loops \
+; RUN: -mtriple=aarch64 -mattr=+sve2p1 -S | FileCheck %s --check-prefixes=CHECK-IC2
+
+; Test re-vectorisation of udot-based reductions.
+; Note that inloop reductions are not support by LV for RK=TargetIntAccumulation.
+; As a consequence, -prefer-inloop-reductions has no effect, but it's still tested here.
+
+define <4 x i32> @reduction_sum_4xi32(ptr noalias nocapture %A) {
+; CHECK-SVE2-LABEL: @reduction_sum_4xi32(
+; CHECK-SVE2-NEXT: entry:
+; CHECK-SVE2-NEXT: br label [[SCALAR_LOOP:%.*]]
+; CHECK-SVE2: scalar.loop:
+; CHECK-SVE2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[SCALAR_LOOP]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-SVE2-NEXT: [[SUM_02:%.*]] = phi <4 x i32> [ [[L7:%.*]], [[SCALAR_LOOP]] ], [ zeroinitializer, [[ENTRY]] ]
+; CHECK-SVE2-NEXT: [[L2:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDVARS_IV]]
+; CHECK-SVE2-NEXT: [[L3:%.*]] = load <16 x i8>, ptr [[L2]], align 4
+; CHECK-SVE2-NEXT: [[L7]] = call <4 x i32> @llvm.aarch64.neon.udot.v4i32.v16i8(<4 x i32> [[SUM_02]], <16 x i8> [[L3]], <16 x i8> [[L3]])
+; CHECK-SVE2-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 4
+; CHECK-SVE2-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-SVE2-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], 256
+; CHECK-SVE2-NEXT: br i1 [[EXITCOND]], label [[LOOP_EXIT:%.*]], label [[SCALAR_LOOP]]
+; CHECK-SVE2: loop.exit:
+; CHECK-SVE2-NEXT: [[SUM_0_LCSSA:%.*]] = phi <4 x i32> [ [[L7]], [[SCALAR_LOOP]] ]
+; CHECK-SVE2-NEXT: ret <4 x i32> [[SUM_0_LCSSA]]
+;
+; CHECK-SVE2p1-LABEL: @reduction_sum_4xi32(
+; CHECK-SVE2p1-NEXT: entry:
+; CHECK-SVE2p1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE2p1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 64, [[TMP0]]
+; CHECK-SVE2p1-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-SVE2p1: vector.ph:
+; CHECK-SVE2p1-NEXT: [[N_MOD_VF:%.*]] = urem i64 64, [[TMP0]]
+; CHECK-SVE2p1-NEXT: [[N_VEC:%.*]] = sub i64 64, [[N_MOD_VF]]
+; CHECK-SVE2p1-NEXT: [[TMP2:%.*]] = shl i64 [[N_VEC]], 2
+; CHECK-SVE2p1-NEXT: [[TMP3:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v4i32(<vscale x 4 x i32> zeroinitializer, <4 x i32> zeroinitializer, i64 0)
+; CHECK-SVE2p1-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK-SVE2p1: vector.body:
+; CHECK-SVE2p1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE2p1-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ [[TMP3]], [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE2p1-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 2
+; CHECK-SVE2p1-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[OFFSET_IDX]]
+; CHECK-SVE2p1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 4
+; CHECK-SVE2p1-NEXT: [[TMP5]] = call <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i8> [[WIDE_LOAD]], <vscale x 16 x i8> [[WIDE_LOAD]])
+; CHECK-SVE2p1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; CHECK-SVE2p1-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-SVE2p1-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-SVE2p1: middle.block:
+; CHECK-SVE2p1-NEXT: [[TMP7:%.*]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.nxv4i32(<4 x i32> zeroinitializer, <vscale x 4 x i32> [[TMP5]])
+; CHECK-SVE2p1-NEXT: [[CMP_N:%.*]] = icmp eq i64 64, [[N_VEC]]
+; CHECK-SVE2p1-NEXT: br i1 [[CMP_N]], label [[LOOP_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-SVE2p1: scalar.ph:
+; CHECK-SVE2p1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP2]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-SVE2p1-NEXT: [[BC_MERGE_RDX:%.*]] = phi <4 x i32> [ [[TMP7]], [[MIDDLE_BLOCK]] ], [ zeroinitializer, [[ENTRY]] ]
+; CHECK-SVE2p1-NEXT: br label [[SCALAR_LOOP:%.*]]
+; CHECK-SVE2p1: scalar.loop:
+; CHECK-SVE2p1-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[SCALAR_LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-SVE2p1-NEXT: [[SUM_02:%.*]] = phi <4 x i32> [ [[L7:%.*]], [[SCALAR_LOOP]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-SVE2p1-NEXT: [[L2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-SVE2p1-NEXT: [[L3:%.*]] = load <16 x i8>, ptr [[L2]], align 4
+; CHECK-SVE2p1-NEXT: [[L7]] = call <4 x i32> @llvm.aarch64.neon.udot.v4i32.v16i8(<4 x i32> [[SUM_02]], <16 x i8> [[L3]], <16 x i8> [[L3]])
+; CHECK-SVE2p1-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 4
+; CHECK-SVE2p1-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-SVE2p1-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], 256
+; CHECK-SVE2p1-NEXT: br i1 [[EXITCOND]], label [[LOOP_EXIT]], label [[SCALAR_LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-SVE2p1: loop.exit:
+; CHECK-SVE2p1-NEXT: [[SUM_0_LCSSA:%.*]] = phi <4 x i32> [ [[L7]], [[SCALAR_LOOP]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]
+; CHECK-SVE2p1-NEXT: ret <4 x i32> [[SUM_0_LCSSA]]
+;
+; CHECK-IC2-LABEL: @reduction_sum_4xi32(
+; CHECK-IC2-NEXT: entry:
+; CHECK-IC2-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-IC2-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-IC2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 64, [[TMP1]]
+; CHECK-IC2-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-IC2: vector.ph:
+; CHECK-IC2-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-IC2-NEXT: [[N_MOD_VF:%.*]] = urem i64 64, [[TMP3]]
+; CHECK-IC2-NEXT: [[N_VEC:%.*]] = sub i64 64, [[N_MOD_VF]]
+; CHECK-IC2-NEXT: [[TMP4:%.*]] = shl i64 [[N_VEC]], 2
+; CHECK-IC2-NEXT: [[TMP5:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v4i32(<vscale x 4 x i32> zeroinitializer, <4 x i32> zeroinitializer, i64 0)
+; CHECK-IC2-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK-IC2: vector.body:
+; CHECK-IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-IC2-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ [[TMP5]], [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ]
+; CHECK-IC2-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[VECTOR_BODY]] ]
+; CHECK-IC2-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 2
+; CHECK-IC2-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[OFFSET_IDX]]
+; CHECK-IC2-NEXT: [[TMP9:%.*]] = getelementptr inbounds <16 x i8>, ptr [[TMP6]], i64 [[TMP0]]
+; CHECK-IC2-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP6]], align 4
+; CHECK-IC2-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP9]], align 4
+; CHECK-IC2-NEXT: [[TMP10]] = call <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i8> [[WIDE_LOAD]], <vscale x 16 x i8> [[WIDE_LOAD]])
+; CHECK-IC2-NEXT: [[TMP11]] = call <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32> [[VEC_PHI1]], <vscale x 16 x i8> [[WIDE_LOAD2]], <vscale x 16 x i8> [[WIDE_LOAD2]])
+; CHECK-IC2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-IC2-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-IC2-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-IC2: middle.block:
+; CHECK-IC2-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[TMP11]], [[TMP10]]
+; CHECK-IC2-NEXT: [[TMP13:%.*]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.nxv4i32(<4 x i32> zeroinitializer, <vscale x 4 x i32> [[BIN_RDX]])
+; CHECK-IC2-NEXT: [[CMP_N:%.*]] = icmp eq i64 64, [[N_VEC]]
+; CHECK-IC2-NEXT: br i1 [[CMP_N]], label [[LOOP_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-IC2: scalar.ph:
+; CHECK-IC2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP4]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-IC2-NEXT: [[BC_MERGE_RDX:%.*]] = phi <4 x i32> [ [[TMP13]], [[MIDDLE_BLOCK]] ], [ zeroinitializer, [[ENTRY]] ]
+; CHECK-IC2-NEXT: br label [[SCALAR_LOOP:%.*]]
+; CHECK-IC2: scalar.loop:
+; CHECK-IC2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[SCALAR_LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-IC2-NEXT: [[SUM_02:%.*]] = phi <4 x i32> [ [[L7:%.*]], [[SCALAR_LOOP]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-IC2-NEXT: [[L2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-IC2-NEXT: [[L3:%.*]] = load <16 x i8>, ptr [[L2]], align 4
+; CHECK-IC2-NEXT: [[L7]] = call <4 x i32> @llvm.aarch64.neon.udot.v4i32.v16i8(<4 x i32> [[SUM_02]], <16 x i8> [[L3]], <16 x i8> [[L3]])
+; CHECK-IC2-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 4
+; CHECK-IC2-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-IC2-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], 256
+; CHECK-IC2-NEXT: br i1 [[EXITCOND]], label [[LOOP_EXIT]], label [[SCALAR_LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-IC2: loop.exit:
+; CHECK-IC2-NEXT: [[SUM_0_LCSSA:%.*]] = phi <4 x i32> [ [[L7]], [[SCALAR_LOOP]] ], [ [[TMP13]], [[MIDDLE_BLOCK]] ]
+; CHECK-IC2-NEXT: ret <4 x i32> [[SUM_0_LCSSA]]
+;
+entry:
+ br label %scalar.loop
+
+scalar.loop: ; preds = %entry, %scalar.loop
+ %indvars.iv = phi i64 [ %indvars.iv.next, %scalar.loop ], [ 0, %entry ]
+ %sum.02 = phi <4 x i32> [ %l7, %scalar.loop ], [ zeroinitializer, %entry ]
+ %l2 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
+ %l3 = load <16 x i8>, ptr %l2, align 4
+ %l7 = call <4 x i32> @llvm.aarch64.neon.udot.v4i32.v16i8(<4 x i32> %sum.02, <16 x i8> %l3, <16 x i8> %l3)
+ %indvars.iv.next = add i64 %indvars.iv, 4
+ %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+ %exitcond = icmp eq i32 %lftr.wideiv, 256
+ br i1 %exitcond, label %loop.exit, label %scalar.loop
+
+loop.exit: ; preds = %scalar.loop
+ %sum.0.lcssa = phi <4 x i32> [ %l7, %scalar.loop ]
+ ret <4 x i32> %sum.0.lcssa
+}
+
+define <4 x i32> @reduction_usdot_4xi32(ptr noalias nocapture %A) {
+; CHECK-SVE2-LABEL: @reduction_usdot_4xi32(
+; CHECK-SVE2-NEXT: entry:
+; CHECK-SVE2-NEXT: br label [[SCALAR_LOOP:%.*]]
+; CHECK-SVE2: scalar.loop:
+; CHECK-SVE2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[SCALAR_LOOP]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-SVE2-NEXT: [[SUM_02:%.*]] = phi <4 x i32> [ [[L7:%.*]], [[SCALAR_LOOP]] ], [ zeroinitializer, [[ENTRY]] ]
+; CHECK-SVE2-NEXT: [[L2:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDVARS_IV]]
+; CHECK-SVE2-NEXT: [[L3:%.*]] = load <16 x i8>, ptr [[L2]], align 4
+; CHECK-SVE2-NEXT: [[L7]] = call <4 x i32> @llvm.aarch64.neon.usdot.v4i32.v16i8(<4 x i32> [[SUM_02]], <16 x i8> [[L3]], <16 x i8> [[L3]])
+; CHECK-SVE2-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 4
+; CHECK-SVE2-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-SVE2-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], 256
+; CHECK-SVE2-NEXT: br i1 [[EXITCOND]], label [[LOOP_EXIT:%.*]], label [[SCALAR_LOOP]]
+; CHECK-SVE2: loop.exit:
+; CHECK-SVE2-NEXT: [[SUM_0_LCSSA:%.*]] = phi <4 x i32> [ [[L7]], [[SCALAR_LOOP]] ]
+; CHECK-SVE2-NEXT: ret <4 x i32> [[SUM_0_LCSSA]]
+;
+; CHECK-SVE2p1-LABEL: @reduction_usdot_4xi32(
+; CHECK-SVE2p1-NEXT: entry:
+; CHECK-SVE2p1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE2p1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 64, [[TMP0]]
+; CHECK-SVE2p1-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-SVE2p1: vector.ph:
+; CHECK-SVE2p1-NEXT: [[N_MOD_VF:%.*]] = urem i64 64, [[TMP0]]
+; CHECK-SVE2p1-NEXT: [[N_VEC:%.*]] = sub i64 64, [[N_MOD_VF]]
+; CHECK-SVE2p1-NEXT: [[TMP2:%.*]] = shl i64 [[N_VEC]], 2
+; CHECK-SVE2p1-NEXT: [[TMP3:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v4i32(<vscale x 4 x i32> zeroinitializer, <4 x i32> zeroinitializer, i64 0)
+; CHECK-SVE2p1-NEXT: br label [[SCALAR_LOOP:%.*]]
+; CHECK-SVE2p1: vector.body:
+; CHECK-SVE2p1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[SCALAR_LOOP]] ]
+; CHECK-SVE2p1-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ [[TMP3]], [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[SCALAR_LOOP]] ]
+; CHECK-SVE2p1-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 2
+; CHECK-SVE2p1-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[OFFSET_IDX]]
+; CHECK-SVE2p1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 4
+; CHECK-SVE2p1-NEXT: [[TMP5]] = call <vscale x 4 x i32> @llvm.aarch64.sve.usdot.nxv4i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i8> [[WIDE_LOAD]], <vscale x 16 x i8> [[WIDE_LOAD]])
+; CHECK-SVE2p1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; CHECK-SVE2p1-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-SVE2p1-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[SCALAR_LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-SVE2p1: middle.block:
+; CHECK-SVE2p1-NEXT: [[TMP7:%.*]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.nxv4i32(<4 x i32> zeroinitializer, <vscale x 4 x i32> [[TMP5]])
+; CHECK-SVE2p1-NEXT: [[CMP_N:%.*]] = icmp eq i64 64, [[N_VEC]]
+; CHECK-SVE2p1-NEXT: br i1 [[CMP_N]], label [[LOOP_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-SVE2p1: scalar.ph:
+; CHECK-SVE2p1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP2]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-SVE2p1-NEXT: [[BC_MERGE_RDX:%.*]] = phi <4 x i32> [ [[TMP7]], [[MIDDLE_BLOCK]] ], [ zeroinitializer, [[ENTRY]] ]
+; CHECK-SVE2p1-NEXT: br label [[SCALAR_LOOP1:%.*]]
+; CHECK-SVE2p1: scalar.loop:
+; CHECK-SVE2p1-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[SCALAR_LOOP1]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-SVE2p1-NEXT: [[SUM_02:%.*]] = phi <4 x i32> [ [[L7:%.*]], [[SCALAR_LOOP1]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-SVE2p1-NEXT: [[L2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-SVE2p1-NEXT: [[L3:%.*]] = load <16 x i8>, ptr [[L2]], align 4
+; CHECK-SVE2p1-NEXT: [[L7]] = call <4 x i32> @llvm.aarch64.neon.usdot.v4i32.v16i8(<4 x i32> [[SUM_02]], <16 x i8> [[L3]], <16 x i8> [[L3]])
+; CHECK-SVE2p1-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 4
+; CHECK-SVE2p1-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-SVE2p1-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], 256
+; CHECK-SVE2p1-NEXT: br i1 [[EXITCOND]], label [[LOOP_EXIT]], label [[SCALAR_LOOP1]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-SVE2p1: loop.exit:
+; CHECK-SVE2p1-NEXT: [[SUM_0_LCSSA:%.*]] = phi <4 x i32> [ [[L7]], [[SCALAR_LOOP1]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]
+; CHECK-SVE2p1-NEXT: ret <4 x i32> [[SUM_0_LCSSA]]
+;
+; CHECK-IC2-LABEL: @reduction_usdot_4xi32(
+; CHECK-IC2-NEXT: entry:
+; CHECK-IC2-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-IC2-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-IC2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 64, [[TMP1]]
+; CHECK-IC2-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-IC2: vector.ph:
+; CHECK-IC2-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-IC2-NEXT: [[N_MOD_VF:%.*]] = urem i64 64, [[TMP3]]
+; CHECK-IC2-NEXT: [[N_VEC:%.*]] = sub i64 64, [[N_MOD_VF]]
+; CHECK-IC2-NEXT: [[TMP4:%.*]] = shl i64 [[N_VEC]], 2
+; CHECK-IC2-NEXT: [[TMP5:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v4i32(<vscale x 4 x i32> zeroinitializer, <4 x i32> zeroinitializer, i64 0)
+; CHECK-IC2-NEXT: br label [[SCALAR_LOOP:%.*]]
+; CHECK-IC2: vector.body:
+; CHECK-IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[SCALAR_LOOP]] ]
+; CHECK-IC2-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ [[TMP5]], [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[SCALAR_LOOP]] ]
+; CHECK-IC2-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[SCALAR_LOOP]] ]
+; CHECK-IC2-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 2
+; CHECK-IC2-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[OFFSET_IDX]]
+; CHECK-IC2-NEXT: [[TMP9:%.*]] = getelementptr inbounds <16 x i8>, ptr [[TMP6]], i64 [[TMP0]]
+; CHECK-IC2-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP6]], align 4
+; CHECK-IC2-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP9]], align 4
+; CHECK-IC2-NEXT: [[TMP10]] = call <vscale x 4 x i32> @llvm.aarch64.sve.usdot.nxv4i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i8> [[WIDE_LOAD]], <vscale x 16 x i8> [[WIDE_LOAD]])
+; CHECK-IC2-NEXT: [[TMP11]] = call <vscale x 4 x i32> @llvm.aarch64.sve.usdot.nxv4i32(<vscale x 4 x i32> [[VEC_PHI1]], <vscale x 16 x i8> [[WIDE_LOAD2]], <vscale x 16 x i8> [[WIDE_LOAD2]])
+; CHECK-IC2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-IC2-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-IC2-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[SCALAR_LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-IC2: middle.block:
+; CHECK-IC2-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[TMP11]], [[TMP10]]
+; CHECK-IC2-NEXT: [[TMP13:%.*]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.nxv4i32(<4 x i32> zeroinitializer, <vscale x 4 x i32> [[BIN_RDX]])
+; CHECK-IC2-NEXT: [[CMP_N:%.*]] = icmp eq i64 64, [[N_VEC]]
+; CHECK-IC2-NEXT: br i1 [[CMP_N]], label [[LOOP_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-IC2: scalar.ph:
+; CHECK-IC2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP4]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-IC2-NEXT: [[BC_MERGE_RDX:%.*]] = phi <4 x i32> [ [[TMP13]], [[MIDDLE_BLOCK]] ], [ zeroinitializer, [[ENTRY]] ]
+; CHECK-IC2-NEXT: br label [[SCALAR_LOOP1:%.*]]
+; CHECK-IC2: scalar.loop:
+; CHECK-IC2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[SCALAR_LOOP1]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-IC2-NEXT: [[SUM_02:%.*]] = phi <4 x i32> [ [[L7:%.*]], [[SCALAR_LOOP1]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-IC2-NEXT: [[L2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-IC2-NEXT: [[L3:%.*]] = load <16 x i8>, ptr [[L2]], align 4
+; CHECK-IC2-NEXT: [[L7]] = call <4 x i32> @llvm.aarch64.neon.usdot.v4i32.v16i8(<4 x i32> [[SUM_02]], <16 x i8> [[L3]], <16 x i8> [[L3]])
+; CHECK-IC2-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 4
+; CHECK-IC2-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-IC2-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], 256
+; CHECK-IC2-NEXT: br i1 [[EXITCOND]], label [[LOOP_EXIT]], label [[SCALAR_LOOP1]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-IC2: loop.exit:
+; CHECK-IC2-NEXT: [[SUM_0_LCSSA:%.*]] = phi <4 x i32> [ [[L7]], [[SCALAR_LOOP1]] ], [ [[TMP13]], [[MIDDLE_BLOCK]] ]
+; CHECK-IC2-NEXT: ret <4 x i32> [[SUM_0_LCSSA]]
+;
+entry:
+ br label %scalar.loop
+
+scalar.loop: ; preds = %entry, %scalar.loop
+ %indvars.iv = phi i64 [ %indvars.iv.next, %scalar.loop ], [ 0, %entry ]
+ %sum.02 = phi <4 x i32> [ %l7, %scalar.loop ], [ zeroinitializer, %entry ]
+ %l2 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
+ %l3 = load <16 x i8>, ptr %l2, align 4
+ %l7 = call <4 x i32> @llvm.aarch64.neon.usdot.v4i32.v16i8(<4 x i32> %sum.02, <16 x i8> %l3, <16 x i8> %l3)
+ %indvars.iv.next = add i64 %indvars.iv, 4
+ %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+ %exitcond = icmp eq i32 %lftr.wideiv, 256
+ br i1 %exitcond, label %loop.exit, label %scalar.loop
+
+loop.exit: ; preds = %scalar.loop
+ %sum.0.lcssa = phi <4 x i32> [ %l7, %scalar.loop ]
+ ret <4 x i32> %sum.0.lcssa
+}
+
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK-SVE2p1-inl: {{.*}}
+; CHECK-SVE2p1-out: {{.*}}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-bitcasts.ll b/llvm/test/Transforms/LoopVectorize/AArch64/revec-bitcasts.ll
new file mode 100644
index 0000000000000..96f86f3939c0f
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/revec-bitcasts.ll
@@ -0,0 +1,202 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph" --version 6
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -vectorize-vector-loops -mtriple=aarch64 -mattr=+sve2p1 -S < %s | FileCheck %s
+
+define void @int16_to_int32(ptr noalias nocapture noundef writeonly %a, ptr readonly %b) {
+; CHECK-LABEL: define void @int16_to_int32(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr readonly [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP0]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <8 x i16>, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP2]], align 16
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <vscale x 8 x i16> [[WIDE_LOAD]] to <vscale x 4 x i32>
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds <8 x half>, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: store <vscale x 4 x i32> [[TMP3]], ptr [[TMP4]], align 16
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <8 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x i16>, ptr %arrayidx, align 16
+ %result = bitcast <8 x i16> %0 to <4 x i32>
+ %arrayidx2 = getelementptr inbounds <8 x half>, ptr %a, i64 %indvars.iv
+ store <4 x i32> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @int16_sext_to_int32(ptr noalias nocapture noundef writeonly %a, ptr readonly %b) {
+; CHECK-LABEL: define void @int16_sext_to_int32(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr readonly [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP0]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <4 x i16>, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP2]], align 16
+; CHECK-NEXT: [[TMP3:%.*]] = sext <vscale x 4 x i16> [[WIDE_LOAD]] to <vscale x 4 x i32>
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds <4 x i32>, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: store <vscale x 4 x i32> [[TMP3]], ptr [[TMP4]], align 16
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i16>, ptr %arrayidx, align 16
+ %result = sext <4 x i16> %0 to <4 x i32>
+ %arrayidx2 = getelementptr inbounds <4 x i32>, ptr %a, i64 %indvars.iv
+ store <4 x i32> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @int32_trunc_to_int16(ptr noalias nocapture noundef writeonly %a, ptr readonly %b) {
+; CHECK-LABEL: define void @int32_trunc_to_int16(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr readonly [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP0]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <4 x i32>, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP2]], align 16
+; CHECK-NEXT: [[TMP3:%.*]] = trunc <vscale x 4 x i32> [[WIDE_LOAD]] to <vscale x 4 x i16>
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds <4 x i16>, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: store <vscale x 4 x i16> [[TMP3]], ptr [[TMP4]], align 16
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i32>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i32>, ptr %arrayidx, align 16
+ %result = trunc <4 x i32> %0 to <4 x i16>
+ %arrayidx2 = getelementptr inbounds <4 x i16>, ptr %a, i64 %indvars.iv
+ store <4 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @scalar_to_int32v(ptr noalias nocapture noundef writeonly %a, ptr readonly %b) {
+; CHECK-LABEL: define void @scalar_to_int32v(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr readonly [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP:.*]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds <8 x i16>, ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP0:%.*]] = load i128, ptr [[ARRAYIDX]], align 16
+; CHECK-NEXT: [[RESULT:%.*]] = bitcast i128 [[TMP0]] to <4 x i32>
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds <8 x half>, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: store <4 x i32> [[RESULT]], ptr [[ARRAYIDX2]], align 16
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1024
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <8 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load i128, ptr %arrayidx, align 16
+ %result = bitcast i128 %0 to <4 x i32>
+ %arrayidx2 = getelementptr inbounds <8 x half>, ptr %a, i64 %indvars.iv
+ store <4 x i32> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @scalar_livein_to_int32v(ptr noalias nocapture noundef writeonly %a, ptr readonly %b, i128 %livein) {
+; CHECK-LABEL: define void @scalar_livein_to_int32v(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr readonly [[B:%.*]], i128 [[LIVEIN:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP:.*]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[RESULT:%.*]] = bitcast i128 [[LIVEIN]] to <4 x i32>
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds <8 x half>, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: store <4 x i32> [[RESULT]], ptr [[ARRAYIDX2]], align 16
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1024
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %result = bitcast i128 %livein to <4 x i32>
+ %arrayidx2 = getelementptr inbounds <8 x half>, ptr %a, i64 %indvars.iv
+ store <4 x i32> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-conversions.ll b/llvm/test/Transforms/LoopVectorize/AArch64/revec-conversions.ll
new file mode 100644
index 0000000000000..3faea8f79ab50
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/revec-conversions.ll
@@ -0,0 +1,172 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph" --version 6
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -vectorize-vector-loops -mtriple=aarch64 -mattr=+sve2p1 -S < %s | FileCheck %s
+
+define void @sint16_to_f16(ptr noalias nocapture noundef writeonly %a, ptr readonly %b) {
+; CHECK-LABEL: define void @sint16_to_f16(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr readonly [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP6]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP6]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <8 x i16>, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP2]], align 16
+; CHECK-NEXT: [[TMP3:%.*]] = sitofp <vscale x 8 x i16> [[WIDE_LOAD]] to <vscale x 8 x half>
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds <8 x half>, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: store <vscale x 8 x half> [[TMP3]], ptr [[TMP4]], align 16
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <8 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x i16>, ptr %arrayidx, align 16
+ %result = sitofp <8 x i16> %0 to <8 x half>
+ %arrayidx2 = getelementptr inbounds <8 x half>, ptr %a, i64 %indvars.iv
+ store <8 x half> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @sint32_to_f32(ptr noalias nocapture noundef writeonly %a, ptr readonly %b) {
+; CHECK-LABEL: define void @sint32_to_f32(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr readonly [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP6]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP6]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <8 x i16>, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP2]], align 16
+; CHECK-NEXT: [[TMP3:%.*]] = sitofp <vscale x 4 x i32> [[WIDE_LOAD]] to <vscale x 4 x float>
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds <8 x half>, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: store <vscale x 4 x float> [[TMP3]], ptr [[TMP4]], align 16
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <8 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i32>, ptr %arrayidx, align 16
+ %result = sitofp <4 x i32> %0 to <4 x float>
+ %arrayidx2 = getelementptr inbounds <8 x half>, ptr %a, i64 %indvars.iv
+ store <4 x float> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @f32_to_uint32(ptr noalias nocapture noundef writeonly %a, ptr readonly %b) {
+; CHECK-LABEL: define void @f32_to_uint32(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr readonly [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP0]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <8 x i16>, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP2]], align 16
+; CHECK-NEXT: [[TMP3:%.*]] = fptoui <vscale x 4 x float> [[WIDE_LOAD]] to <vscale x 4 x i32>
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds <8 x half>, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: store <vscale x 4 x i32> [[TMP3]], ptr [[TMP4]], align 16
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <8 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x float>, ptr %arrayidx, align 16
+ %result = fptoui <4 x float> %0 to <4 x i32>
+ %arrayidx2 = getelementptr inbounds <8 x half>, ptr %a, i64 %indvars.iv
+ store <4 x i32> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+; TODO-REVEC: extensions/truncations are not supported because only quad vector
+; types are considered legal so far.
+define void @sext_int32(ptr noalias nocapture noundef writeonly %a, ptr readonly %b) {
+; CHECK-LABEL: define void @sext_int32(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr readonly [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP:.*]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds <8 x i16>, ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i16>, ptr [[ARRAYIDX]], align 16
+; CHECK-NEXT: [[RESULT:%.*]] = sext <4 x i16> [[TMP0]] to <4 x i32>
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds <8 x half>, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: store <4 x i32> [[RESULT]], ptr [[ARRAYIDX2]], align 16
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1024
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <8 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i16>, ptr %arrayidx, align 16
+ %result = sext <4 x i16> %0 to <4 x i32>
+ %arrayidx2 = getelementptr inbounds <8 x half>, ptr %a, i64 %indvars.iv
+ store <4 x i32> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-gather-force-vf.ll b/llvm/test/Transforms/LoopVectorize/AArch64/revec-gather-force-vf.ll
new file mode 100644
index 0000000000000..0e56cdb72662a
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/revec-gather-force-vf.ll
@@ -0,0 +1,54 @@
+; RUN: opt -passes=loop-vectorize -vectorize-vector-loops -lv-strided-pointer-ivs \
+; RUN: -force-vector-interleave=1 -scalable-vectorization=on -force-vector-width=1 \
+; RUN: -mtriple=aarch64 -mattr=+sve2p1 -S < %s | FileCheck %s --check-prefixes=CHECK,VF1
+; RUN: opt -passes=loop-vectorize -vectorize-vector-loops -lv-strided-pointer-ivs \
+; RUN: -force-vector-interleave=1 -scalable-vectorization=on -force-vector-width=2 \
+; RUN: -mtriple=aarch64 -mattr=+sve2p1 -S < %s | FileCheck %s --check-prefixes=CHECK,VF2
+
+; Check that gathers cannot be re-vectorised for VF = vscale x 2.
+; This is a hard limitation of .segment intrinsics which expect types like <vscale x SegmentTy>.
+
+define void @gather_128(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b, i64 %stride) {
+; CHECK-LABEL: define void @gather_128(
+; CHECK: call <vscale x 8 x i16> @llvm.masked.segment.gather
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %offset = mul i64 %indvars.iv, %stride
+ %arrayidx0 = getelementptr inbounds i16, ptr %b, i64 %offset
+ %0 = load <8 x i16>, ptr %arrayidx0, align 16
+ %result = add <8 x i16> %0, splat (i16 1)
+ %arrayidx2 = getelementptr inbounds <8 x i16>, ptr %a, i64 %indvars.iv
+ store <8 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @gather_64(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b, i64 %stride) {
+; CHECK-LABEL: define void @gather_64(
+; VF1: call <vscale x 4 x i16> @llvm.masked.segment.gather
+; VF2-NOT: vector.body
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %offset = mul i64 %indvars.iv, %stride
+ %arrayidx0 = getelementptr inbounds i16, ptr %b, i64 %offset
+ %0 = load <4 x i16>, ptr %arrayidx0, align 8
+ %result = add <4 x i16> %0, splat (i16 1)
+ %arrayidx2 = getelementptr inbounds <4 x i16>, ptr %a, i64 %indvars.iv
+ store <4 x i16> %result, ptr %arrayidx2, align 8
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-gather.ll b/llvm/test/Transforms/LoopVectorize/AArch64/revec-gather.ll
new file mode 100644
index 0000000000000..70b5a107efd4d
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/revec-gather.ll
@@ -0,0 +1,238 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "scalar.ph:" --replace-value-regex "!llvm.loop ![0-9]+" --version 6
+; RUN: opt -passes=loop-vectorize -vectorize-vector-loops -lv-strided-pointer-ivs \
+; RUN: -force-vector-interleave=1 -scalable-vectorization=on -force-vector-width=1 \
+; RUN: -mtriple=aarch64 -mattr=+sve2p1 -S < %s | FileCheck %s
+
+
+define void @gather_128(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b, i64 %stride) {
+; CHECK-LABEL: define void @gather_128(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP8]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; CHECK: [[VECTOR_SCEVCHECK]]:
+; CHECK-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; CHECK-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP8]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 1 x i64> @llvm.stepvector.nxv1i64()
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[TMP8]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 1 x i64> [ [[TMP2]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i16, ptr [[B]], <vscale x 1 x i64> [[VEC_IND]]
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 8 x i16> @llvm.masked.segment.gather.nxv8i16.nxv1p0(<vscale x 1 x ptr> align 16 [[TMP4]], <vscale x 1 x i1> splat (i1 true), <vscale x 8 x i16> poison)
+; CHECK-NEXT: [[TMP5:%.*]] = add <vscale x 8 x i16> [[WIDE_MASKED_GATHER]], splat (i16 1)
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds <8 x i16>, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: store <vscale x 8 x i16> [[TMP5]], ptr [[TMP6]], align 16
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 1 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %offset = mul i64 %indvars.iv, %stride
+ %arrayidx0 = getelementptr inbounds i16, ptr %b, i64 %offset
+ %0 = load <8 x i16>, ptr %arrayidx0, align 16
+ %result = add <8 x i16> %0, splat (i16 1)
+ %arrayidx2 = getelementptr inbounds <8 x i16>, ptr %a, i64 %indvars.iv
+ store <8 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @gather_128_ptr_iv(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b, i64 %stride) {
+; CHECK-LABEL: define void @gather_128_ptr_iv(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[STRIDE]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT1:%.*]] = shufflevector <vscale x 1 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shl <vscale x 1 x i64> [[BROADCAST_SPLAT1]], splat (i64 1)
+; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP1]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <vscale x 1 x i64> [[BROADCAST_SPLAT]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[N_VEC]], [[TMP11]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP3]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[B]], %[[VECTOR_PH]] ], [ [[PTR_IND:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 1 x i64> @llvm.stepvector.nxv1i64()
+; CHECK-NEXT: [[TMP6:%.*]] = mul <vscale x 1 x i64> [[TMP5]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <vscale x 1 x i64> [[TMP6]]
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 8 x i16> @llvm.masked.segment.gather.nxv8i16.nxv1p0(<vscale x 1 x ptr> align 16 [[VECTOR_GEP]], <vscale x 1 x i1> splat (i1 true), <vscale x 8 x i16> poison)
+; CHECK-NEXT: [[TMP7:%.*]] = add <vscale x 8 x i16> [[WIDE_MASKED_GATHER]], splat (i16 1)
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds <8 x i16>, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: store <vscale x 8 x i16> [[TMP7]], ptr [[TMP8]], align 16
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <vscale x 1 x i64> [[BROADCAST_SPLAT]], i64 0
+; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP12]], [[TMP1]]
+; CHECK-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP9]]
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %addr.iv = phi ptr [ %b, %entry ], [ %addr.next, %for.body ]
+ %0 = load <8 x i16>, ptr %addr.iv, align 16
+ %result = add <8 x i16> %0, splat (i16 1)
+ %arrayidx2 = getelementptr inbounds <8 x i16>, ptr %a, i64 %indvars.iv
+ store <8 x i16> %result, ptr %arrayidx2, align 16
+ %addr.next = getelementptr inbounds i16, ptr %addr.iv, i64 %stride
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @gather_64(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b, i64 %stride) {
+; CHECK-LABEL: define void @gather_64(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; CHECK: [[VECTOR_SCEVCHECK]]:
+; CHECK-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; CHECK-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP0]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 1 x i64> @llvm.stepvector.nxv1i64()
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[TMP0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 1 x i64> [ [[TMP2]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i16, ptr [[B]], <vscale x 1 x i64> [[VEC_IND]]
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i16> @llvm.masked.segment.gather.nxv4i16.nxv1p0(<vscale x 1 x ptr> align 8 [[TMP4]], <vscale x 1 x i1> splat (i1 true), <vscale x 4 x i16> poison)
+; CHECK-NEXT: [[TMP5:%.*]] = add <vscale x 4 x i16> [[WIDE_MASKED_GATHER]], splat (i16 1)
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds <4 x i16>, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: store <vscale x 4 x i16> [[TMP5]], ptr [[TMP6]], align 8
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 1 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %offset = mul i64 %indvars.iv, %stride
+ %arrayidx0 = getelementptr inbounds i16, ptr %b, i64 %offset
+ %0 = load <4 x i16>, ptr %arrayidx0, align 8
+ %result = add <4 x i16> %0, splat (i16 1)
+ %arrayidx2 = getelementptr inbounds <4 x i16>, ptr %a, i64 %indvars.iv
+ store <4 x i16> %result, ptr %arrayidx2, align 8
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @gather_128_predication(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b, ptr nocapture noundef readonly %flags, i64 %cond, i64 %stride) {
+; CHECK-LABEL: define void @gather_128_predication(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], ptr noundef readonly captures(none) [[FLAGS:%.*]], i64 [[COND:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; CHECK: [[VECTOR_SCEVCHECK]]:
+; CHECK-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; CHECK-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP0]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[COND]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 1 x i64> @llvm.stepvector.nxv1i64()
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[TMP0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 1 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 1 x i64> [ [[TMP2]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[FLAGS]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 1 x i64>, ptr [[TMP4]], align 8
+; CHECK-NEXT: [[TMP5:%.*]] = icmp ne <vscale x 1 x i64> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i16, ptr [[B]], <vscale x 1 x i64> [[VEC_IND]]
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 8 x i16> @llvm.masked.segment.gather.nxv8i16.nxv1p0(<vscale x 1 x ptr> align 16 [[TMP6]], <vscale x 1 x i1> [[TMP5]], <vscale x 8 x i16> poison)
+; CHECK-NEXT: [[TMP7:%.*]] = call <vscale x 8 x i1> @llvm.vector.stretch.nxv8i1.nxv1i1(<vscale x 1 x i1> [[TMP5]])
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <vscale x 8 x i1> [[TMP7]], <vscale x 8 x i16> [[WIDE_MASKED_GATHER]], <vscale x 8 x i16> zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = add <vscale x 8 x i16> [[PREDPHI]], splat (i16 1)
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds <8 x i16>, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: store <vscale x 8 x i16> [[TMP8]], ptr [[TMP9]], align 16
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 1 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup:
+ ret void
+
+for.body:
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.inc ]
+ %flagidx = getelementptr inbounds i64, ptr %flags, i64 %indvars.iv
+ %scl.val = load i64, ptr %flagidx, align 8
+ %do.access = icmp ne i64 %scl.val, %cond
+ br i1 %do.access, label %if.then, label %for.inc
+
+if.then:
+ %offset = mul i64 %indvars.iv, %stride
+ %arrayidx0 = getelementptr inbounds i16, ptr %b, i64 %offset
+ %0 = load <8 x i16>, ptr %arrayidx0, align 16
+ br label %for.inc
+
+for.inc:
+ %1 = phi <8 x i16> [ zeroinitializer, %for.body ], [ %0, %if.then ]
+ %result = add <8 x i16> %1, splat (i16 1)
+ %arrayidx2 = getelementptr inbounds <8 x i16>, ptr %a, i64 %indvars.iv
+ store <8 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-ifconvert.ll b/llvm/test/Transforms/LoopVectorize/AArch64/revec-ifconvert.ll
new file mode 100644
index 0000000000000..c4413939f79a2
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/revec-ifconvert.ll
@@ -0,0 +1,215 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^scalar.ph:"
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -vectorize-vector-loops \
+; RUN: -sve-vscale-for-tuning=2 \
+; RUN: -mtriple=aarch64 -mattr=+sve2p1 -S < %s | FileCheck %s
+
+
+define void @cond_load_iv(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b, ptr nocapture noundef readonly %c, i64 %cond) {
+; CHECK-LABEL: @cond_load_iv(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP0]], i64 2)
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[UMAX]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP1]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[COND:%.*]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 1 x i64> @llvm.stepvector.nxv1i64()
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[TMP1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 1 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 1 x i64> [ [[TMP2]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds <8 x i16>, ptr [[B:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP4]], align 16
+; CHECK-NEXT: [[TMP5:%.*]] = icmp ne <vscale x 1 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr <8 x i16>, ptr [[C:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP7:%.*]] = call <vscale x 8 x i1> @llvm.vector.stretch.nxv8i1.nxv1i1(<vscale x 1 x i1> [[TMP5]])
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 16 [[TMP6]], <vscale x 8 x i1> [[TMP7]], <vscale x 8 x i16> poison)
+; CHECK-NEXT: [[TMP8:%.*]] = call <vscale x 8 x i1> @llvm.vector.stretch.nxv8i1.nxv1i1(<vscale x 1 x i1> [[TMP5]])
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <vscale x 8 x i1> [[TMP8]], <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], <vscale x 8 x i16> zeroinitializer
+; CHECK-NEXT: [[TMP9:%.*]] = add <vscale x 8 x i16> [[WIDE_LOAD]], [[PREDPHI]]
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds <8 x i16>, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: store <vscale x 8 x i16> [[TMP9]], ptr [[TMP10]], align 16
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 1 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.inc
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.inc ]
+ %arrayidx0 = getelementptr inbounds <8 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x i16>, ptr %arrayidx0, align 16
+ %do.access = icmp ne i64 %indvars.iv, %cond
+ br i1 %do.access, label %if.then, label %for.inc
+
+if.then:
+ %arrayidx1 = getelementptr inbounds <8 x i16>, ptr %c, i64 %indvars.iv
+ %pred.1 = load <8 x i16>, ptr %arrayidx1, align 16
+ br label %for.inc
+
+for.inc:
+ %1 = phi <8 x i16> [ zeroinitializer, %for.body ], [ %pred.1, %if.then ]
+ %result = add <8 x i16> %0, %1
+ %arrayidx2 = getelementptr inbounds <8 x i16>, ptr %a, i64 %indvars.iv
+ store <8 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+
+define void @conv_load_load(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b, ptr nocapture noundef readonly %c, ptr nocapture noundef readonly %flags, i64 %cond) {
+; CHECK-LABEL: @conv_load_load(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP11]], i64 2)
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[UMAX]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP12]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[COND:%.*]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <8 x i16>, ptr [[B:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP2]], align 16
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[FLAGS:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 1 x i64>, ptr [[TMP3]], align 8
+; CHECK-NEXT: [[TMP4:%.*]] = icmp ne <vscale x 1 x i64> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr <8 x i16>, ptr [[C:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 8 x i1> @llvm.vector.stretch.nxv8i1.nxv1i1(<vscale x 1 x i1> [[TMP4]])
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 16 [[TMP5]], <vscale x 8 x i1> [[TMP6]], <vscale x 8 x i16> poison)
+; CHECK-NEXT: [[TMP7:%.*]] = call <vscale x 8 x i1> @llvm.vector.stretch.nxv8i1.nxv1i1(<vscale x 1 x i1> [[TMP4]])
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <vscale x 8 x i1> [[TMP7]], <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], <vscale x 8 x i16> zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = add <vscale x 8 x i16> [[WIDE_LOAD]], [[PREDPHI]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds <8 x i16>, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: store <vscale x 8 x i16> [[TMP8]], ptr [[TMP9]], align 16
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP12]]
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.inc
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.inc ]
+ %arrayidx0 = getelementptr inbounds <8 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x i16>, ptr %arrayidx0, align 16
+ %flagidx = getelementptr inbounds i64, ptr %flags, i64 %indvars.iv
+ %scl.val = load i64, ptr %flagidx, align 8
+ %do.access = icmp ne i64 %scl.val, %cond
+ br i1 %do.access, label %if.then, label %for.inc
+
+if.then:
+ %arrayidx1 = getelementptr inbounds <8 x i16>, ptr %c, i64 %indvars.iv
+ %pred.1 = load <8 x i16>, ptr %arrayidx1, align 16
+ br label %for.inc
+
+for.inc:
+ %1 = phi <8 x i16> [ zeroinitializer, %for.body ], [ %pred.1, %if.then ]
+ %result = add <8 x i16> %0, %1
+ %arrayidx2 = getelementptr inbounds <8 x i16>, ptr %a, i64 %indvars.iv
+ store <8 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @conv_interleaved_loads_load(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b, ptr nocapture noundef readonly %c, ptr nocapture noundef readonly %flags, i64 %cond) {
+; CHECK-LABEL: @conv_interleaved_loads_load(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP0]], i64 2)
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[UMAX]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP1]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[COND:%.*]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <8 x i16>, ptr [[B:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP2]], align 16
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[FLAGS:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 1 x i64>, ptr [[TMP3]], align 8
+; CHECK-NEXT: [[TMP4:%.*]] = icmp ne <vscale x 1 x i64> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr [16 x i16], ptr [[C:%.*]], i64 [[INDEX]], i64 0
+; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 16 x i1> @llvm.vector.stretch.nxv16i1.nxv1i1(<vscale x 1 x i1> [[TMP4]])
+; CHECK-NEXT: [[WIDE_MASKED_VEC:%.*]] = call <vscale x 16 x i16> @llvm.masked.load.nxv16i16.p0(ptr align 16 [[TMP5]], <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i16> poison)
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.vector.deinterleave.segments2.nxv16i16(<vscale x 16 x i16> [[WIDE_MASKED_VEC]])
+; CHECK-NEXT: [[TMP7:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[STRIDED_VEC]], 0
+; CHECK-NEXT: [[TMP8:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[STRIDED_VEC]], 1
+; CHECK-NEXT: [[TMP9:%.*]] = add <vscale x 8 x i16> [[TMP7]], [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = call <vscale x 8 x i1> @llvm.vector.stretch.nxv8i1.nxv1i1(<vscale x 1 x i1> [[TMP4]])
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <vscale x 8 x i1> [[TMP10]], <vscale x 8 x i16> [[TMP9]], <vscale x 8 x i16> zeroinitializer
+; CHECK-NEXT: [[TMP11:%.*]] = add <vscale x 8 x i16> [[WIDE_LOAD]], [[PREDPHI]]
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds <8 x i16>, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: store <vscale x 8 x i16> [[TMP11]], ptr [[TMP12]], align 16
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
+; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.inc
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.inc ]
+ %arrayidx0 = getelementptr inbounds <8 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x i16>, ptr %arrayidx0, align 16
+ %flagidx = getelementptr inbounds i64, ptr %flags, i64 %indvars.iv
+ %scl.val = load i64, ptr %flagidx, align 8
+ %do.access = icmp ne i64 %scl.val, %cond
+ br i1 %do.access, label %if.then, label %for.inc
+
+if.then:
+ %arrayidx1 = getelementptr inbounds [16 x i16], ptr %c, i64 %indvars.iv, i64 0
+ %pred.1 = load <8 x i16>, ptr %arrayidx1, align 16
+ %arrayidx2 = getelementptr inbounds [16 x i16], ptr %c, i64 %indvars.iv, i64 8
+ %pred.2 = load <8 x i16>, ptr %arrayidx2, align 16
+ %pred.sum = add <8 x i16> %pred.1, %pred.2
+ br label %for.inc
+
+for.inc:
+ %1 = phi <8 x i16> [ zeroinitializer, %for.body ], [ %pred.sum, %if.then ]
+ %result = add <8 x i16> %0, %1
+ %arrayidx3 = getelementptr inbounds <8 x i16>, ptr %a, i64 %indvars.iv
+ store <8 x i16> %result, ptr %arrayidx3, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-interleaved-ldst-different-types.ll b/llvm/test/Transforms/LoopVectorize/AArch64/revec-interleaved-ldst-different-types.ll
new file mode 100644
index 0000000000000..6a7c101fc4d4c
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/revec-interleaved-ldst-different-types.ll
@@ -0,0 +1,206 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -vectorize-vector-loops \
+; RUN: -mtriple=aarch64 -mattr=+sve2p1 -S < %s | FileCheck %s
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -vectorize-vector-loops \
+; RUN: -tail-folding-policy=prefer-fold-tail \
+; RUN: -mtriple=aarch64 -mattr=+sve2p1 -S < %s | FileCheck %s
+
+; Test vector interleaved accesses with different types, and with reversing.
+
+; Test where interleave members have different types (different elt type but same elt count)
+define void @factor2_different_elttype(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @factor2_different_elttype(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP0]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [16 x i16], ptr [[B]], i64 [[INDEX]], i64 0
+; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 16 x i16>, ptr [[TMP2]], align 16
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.vector.deinterleave.segments2.nxv16i16(<vscale x 16 x i16> [[WIDE_VEC]])
+; CHECK-NEXT: [[TMP3:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[STRIDED_VEC]], 0
+; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[STRIDED_VEC]], 1
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast <vscale x 8 x i16> [[TMP4]] to <vscale x 8 x half>
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds [16 x i16], ptr [[A]], i64 [[INDEX]], i64 0
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast <vscale x 8 x i16> [[TMP3]] to <vscale x 8 x half>
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 16 x half> @llvm.vector.interleave.segments2.nxv16f16(<vscale x 8 x half> [[TMP8]], <vscale x 8 x half> [[TMP5]])
+; CHECK-NEXT: store <vscale x 16 x half> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 16
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[ARRAYIDX0:%.*]] = getelementptr inbounds [16 x i16], ptr [[B]], i64 [[INDVARS_IV]], i64 0
+; CHECK-NEXT: [[V0:%.*]] = load <8 x i16>, ptr [[ARRAYIDX0]], align 16
+; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds [16 x i16], ptr [[B]], i64 [[INDVARS_IV]], i64 8
+; CHECK-NEXT: [[V1:%.*]] = load <8 x half>, ptr [[ARRAYIDX1]], align 16
+; CHECK-NEXT: [[ARRAYIDX_OUT0:%.*]] = getelementptr inbounds [16 x i16], ptr [[A]], i64 [[INDVARS_IV]], i64 0
+; CHECK-NEXT: store <8 x i16> [[V0]], ptr [[ARRAYIDX_OUT0]], align 16
+; CHECK-NEXT: [[ARRAYIDX_OUT1:%.*]] = getelementptr inbounds [16 x i16], ptr [[A]], i64 [[INDVARS_IV]], i64 8
+; CHECK-NEXT: store <8 x half> [[V1]], ptr [[ARRAYIDX_OUT1]], align 16
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1024
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx0 = getelementptr inbounds [16 x i16], ptr %b, i64 %indvars.iv, i64 0
+ %v0 = load <8 x i16>, ptr %arrayidx0, align 16
+ %arrayidx1 = getelementptr inbounds [16 x i16], ptr %b, i64 %indvars.iv, i64 8
+ %v1 = load <8 x half>, ptr %arrayidx1, align 16
+
+ %arrayidx.out0 = getelementptr inbounds [16 x i16], ptr %a, i64 %indvars.iv, i64 0
+ store <8 x i16> %v0, ptr %arrayidx.out0, align 16
+ %arrayidx.out1 = getelementptr inbounds [16 x i16], ptr %a, i64 %indvars.iv, i64 8
+ store <8 x half> %v1, ptr %arrayidx.out1, align 16
+
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+; Test where interleave members have different types (different elt type and elt count)
+define void @factor2_different_elttype_eltcount(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @factor2_different_elttype_eltcount(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP0]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [16 x i16], ptr [[B]], i64 [[INDEX]], i64 0
+; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 16 x i16>, ptr [[TMP2]], align 16
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.vector.deinterleave.segments2.nxv16i16(<vscale x 16 x i16> [[WIDE_VEC]])
+; CHECK-NEXT: [[TMP3:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[STRIDED_VEC]], 0
+; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[STRIDED_VEC]], 1
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast <vscale x 8 x i16> [[TMP4]] to <vscale x 16 x i8>
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds [16 x i16], ptr [[A]], i64 [[INDEX]], i64 0
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast <vscale x 8 x i16> [[TMP3]] to <vscale x 16 x i8>
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 32 x i8> @llvm.vector.interleave.segments2.nxv32i8(<vscale x 16 x i8> [[TMP8]], <vscale x 16 x i8> [[TMP5]])
+; CHECK-NEXT: store <vscale x 32 x i8> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 16
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[ARRAYIDX0:%.*]] = getelementptr inbounds [16 x i16], ptr [[B]], i64 [[INDVARS_IV]], i64 0
+; CHECK-NEXT: [[V0:%.*]] = load <8 x i16>, ptr [[ARRAYIDX0]], align 16
+; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds [16 x i16], ptr [[B]], i64 [[INDVARS_IV]], i64 8
+; CHECK-NEXT: [[V1:%.*]] = load <16 x i8>, ptr [[ARRAYIDX1]], align 16
+; CHECK-NEXT: [[ARRAYIDX_OUT0:%.*]] = getelementptr inbounds [16 x i16], ptr [[A]], i64 [[INDVARS_IV]], i64 0
+; CHECK-NEXT: store <8 x i16> [[V0]], ptr [[ARRAYIDX_OUT0]], align 16
+; CHECK-NEXT: [[ARRAYIDX_OUT1:%.*]] = getelementptr inbounds [16 x i16], ptr [[A]], i64 [[INDVARS_IV]], i64 8
+; CHECK-NEXT: store <16 x i8> [[V1]], ptr [[ARRAYIDX_OUT1]], align 16
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1024
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx0 = getelementptr inbounds [16 x i16], ptr %b, i64 %indvars.iv, i64 0
+ %v0 = load <8 x i16>, ptr %arrayidx0, align 16
+ %arrayidx1 = getelementptr inbounds [16 x i16], ptr %b, i64 %indvars.iv, i64 8
+ %v1 = load <16 x i8>, ptr %arrayidx1, align 16
+
+ %arrayidx.out0 = getelementptr inbounds [16 x i16], ptr %a, i64 %indvars.iv, i64 0
+ store <8 x i16> %v0, ptr %arrayidx.out0, align 16
+ %arrayidx.out1 = getelementptr inbounds [16 x i16], ptr %a, i64 %indvars.iv, i64 8
+ store <16 x i8> %v1, ptr %arrayidx.out1, align 16
+
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+; Same as above, but with a decrementing IV
+define void @factor2_different_elttype_eltcount_reverse(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @factor2_different_elttype_eltcount_reverse(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP:.*]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 1024, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[ARRAYIDX0:%.*]] = getelementptr inbounds [16 x i16], ptr [[B]], i64 [[INDVARS_IV]], i64 0
+; CHECK-NEXT: [[V0:%.*]] = load <8 x i16>, ptr [[ARRAYIDX0]], align 16
+; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds [16 x i16], ptr [[B]], i64 [[INDVARS_IV]], i64 8
+; CHECK-NEXT: [[V1:%.*]] = load <16 x i8>, ptr [[ARRAYIDX1]], align 16
+; CHECK-NEXT: [[ARRAYIDX_OUT0:%.*]] = getelementptr inbounds [16 x i16], ptr [[A]], i64 [[INDVARS_IV]], i64 0
+; CHECK-NEXT: store <8 x i16> [[V0]], ptr [[ARRAYIDX_OUT0]], align 16
+; CHECK-NEXT: [[ARRAYIDX_OUT1:%.*]] = getelementptr inbounds [16 x i16], ptr [[A]], i64 [[INDVARS_IV]], i64 8
+; CHECK-NEXT: store <16 x i8> [[V1]], ptr [[ARRAYIDX_OUT1]], align 16
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], -1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 0
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 1024, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx0 = getelementptr inbounds [16 x i16], ptr %b, i64 %indvars.iv, i64 0
+ %v0 = load <8 x i16>, ptr %arrayidx0, align 16
+ %arrayidx1 = getelementptr inbounds [16 x i16], ptr %b, i64 %indvars.iv, i64 8
+ %v1 = load <16 x i8>, ptr %arrayidx1, align 16
+
+ %arrayidx.out0 = getelementptr inbounds [16 x i16], ptr %a, i64 %indvars.iv, i64 0
+ store <8 x i16> %v0, ptr %arrayidx.out0, align 16
+ %arrayidx.out1 = getelementptr inbounds [16 x i16], ptr %a, i64 %indvars.iv, i64 8
+ store <16 x i8> %v1, ptr %arrayidx.out1, align 16
+
+ %indvars.iv.next = add nsw i64 %indvars.iv, -1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 0
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-interleaved-loads.ll b/llvm/test/Transforms/LoopVectorize/AArch64/revec-interleaved-loads.ll
new file mode 100644
index 0000000000000..d72f08c53929c
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/revec-interleaved-loads.ll
@@ -0,0 +1,441 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -vectorize-vector-loops \
+; RUN: -mtriple=aarch64 -mattr=+sve2p1 -S < %s | FileCheck %s
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -vectorize-vector-loops \
+; RUN: -tail-folding-policy=prefer-fold-tail \
+; RUN: -mtriple=aarch64 -mattr=+sve2p1 -S < %s | FileCheck %s
+
+define void @ld2q(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @ld2q(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP10]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP10]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [16 x i16], ptr [[B]], i64 [[INDEX]], i64 0
+; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 16 x i16>, ptr [[TMP2]], align 16
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.vector.deinterleave.segments2.nxv16i16(<vscale x 16 x i16> [[WIDE_VEC]])
+; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[STRIDED_VEC]], 0
+; CHECK-NEXT: [[TMP6:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[STRIDED_VEC]], 1
+; CHECK-NEXT: [[TMP7:%.*]] = add <vscale x 8 x i16> [[TMP4]], [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds <8 x i16>, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: store <vscale x 8 x i16> [[TMP7]], ptr [[TMP8]], align 16
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[ARRAYIDX0:%.*]] = getelementptr inbounds [16 x i16], ptr [[B]], i64 [[INDVARS_IV]], i64 0
+; CHECK-NEXT: [[TMP0:%.*]] = load <8 x i16>, ptr [[ARRAYIDX0]], align 16
+; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds [16 x i16], ptr [[B]], i64 [[INDVARS_IV]], i64 8
+; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i16>, ptr [[ARRAYIDX1]], align 16
+; CHECK-NEXT: [[RESULT:%.*]] = add <8 x i16> [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds <8 x i16>, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: store <8 x i16> [[RESULT]], ptr [[ARRAYIDX2]], align 16
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1024
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx0 = getelementptr inbounds [16 x i16], ptr %b, i64 %indvars.iv, i64 0
+ %0 = load <8 x i16>, ptr %arrayidx0, align 16
+ %arrayidx1 = getelementptr inbounds [16 x i16], ptr %b, i64 %indvars.iv, i64 8
+ %1 = load <8 x i16>, ptr %arrayidx1, align 16
+ %result = add <8 x i16> %0, %1
+ %arrayidx2 = getelementptr inbounds <8 x i16>, ptr %a, i64 %indvars.iv
+ store <8 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+; TODO-REVEC:
+; Due to how (de)interleave.segments intrinsics work, we cannot have VF = vscale x 2.
+; And getSegmentInterleavedMemoryOpCost() forbids non-128bit segments.
+define void @ld2q_64bits(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @ld2q_64bits(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP:.*]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[ARRAYIDX0:%.*]] = getelementptr inbounds [8 x i16], ptr [[B]], i64 [[INDVARS_IV]], i64 0
+; CHECK-NEXT: [[TMP9:%.*]] = load <4 x i16>, ptr [[ARRAYIDX0]], align 16
+; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds [8 x i16], ptr [[B]], i64 [[INDVARS_IV]], i64 4
+; CHECK-NEXT: [[TMP10:%.*]] = load <4 x i16>, ptr [[ARRAYIDX1]], align 16
+; CHECK-NEXT: [[RESULT:%.*]] = add <4 x i16> [[TMP9]], [[TMP10]]
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds <4 x i16>, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: store <4 x i16> [[RESULT]], ptr [[ARRAYIDX2]], align 16
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1024
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx0 = getelementptr inbounds [8 x i16], ptr %b, i64 %indvars.iv, i64 0
+ %0 = load <4 x i16>, ptr %arrayidx0, align 16
+ %arrayidx1 = getelementptr inbounds [8 x i16], ptr %b, i64 %indvars.iv, i64 4
+ %1 = load <4 x i16>, ptr %arrayidx1, align 16
+ %result = add <4 x i16> %0, %1
+ %arrayidx2 = getelementptr inbounds <4 x i16>, ptr %a, i64 %indvars.iv
+ store <4 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @ld2q_reverse(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @ld2q_reverse(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP:.*]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 1024, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[ARRAYIDX0:%.*]] = getelementptr inbounds [16 x i16], ptr [[B]], i64 [[INDVARS_IV]], i64 0
+; CHECK-NEXT: [[TMP21:%.*]] = load <8 x i16>, ptr [[ARRAYIDX0]], align 16
+; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds [16 x i16], ptr [[B]], i64 [[INDVARS_IV]], i64 8
+; CHECK-NEXT: [[TMP22:%.*]] = load <8 x i16>, ptr [[ARRAYIDX1]], align 16
+; CHECK-NEXT: [[RESULT:%.*]] = add <8 x i16> [[TMP21]], [[TMP22]]
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds <8 x i16>, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: store <8 x i16> [[RESULT]], ptr [[ARRAYIDX2]], align 16
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], -1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 0
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 1024, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx0 = getelementptr inbounds [16 x i16], ptr %b, i64 %indvars.iv, i64 0
+ %0 = load <8 x i16>, ptr %arrayidx0, align 16
+ %arrayidx1 = getelementptr inbounds [16 x i16], ptr %b, i64 %indvars.iv, i64 8
+ %1 = load <8 x i16>, ptr %arrayidx1, align 16
+ %result = add <8 x i16> %0, %1
+ %arrayidx2 = getelementptr inbounds <8 x i16>, ptr %a, i64 %indvars.iv
+ store <8 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nsw i64 %indvars.iv, -1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 0
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @ld3q(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @ld3q(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP:.*]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[ARRAYIDX0:%.*]] = getelementptr inbounds [24 x i16], ptr [[B]], i64 [[INDVARS_IV]], i64 0
+; CHECK-NEXT: [[TMP0:%.*]] = load <8 x i16>, ptr [[ARRAYIDX0]], align 16
+; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds [24 x i16], ptr [[B]], i64 [[INDVARS_IV]], i64 8
+; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i16>, ptr [[ARRAYIDX1]], align 16
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds [24 x i16], ptr [[B]], i64 [[INDVARS_IV]], i64 16
+; CHECK-NEXT: [[TMP2:%.*]] = load <8 x i16>, ptr [[ARRAYIDX2]], align 16
+; CHECK-NEXT: [[TMP:%.*]] = add <8 x i16> [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[RESULT:%.*]] = add <8 x i16> [[TMP]], [[TMP2]]
+; CHECK-NEXT: [[ARRAYIDX_OUT:%.*]] = getelementptr inbounds <8 x i16>, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: store <8 x i16> [[RESULT]], ptr [[ARRAYIDX_OUT]], align 16
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1024
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx0 = getelementptr inbounds [24 x i16], ptr %b, i64 %indvars.iv, i64 0
+ %0 = load <8 x i16>, ptr %arrayidx0, align 16
+ %arrayidx1 = getelementptr inbounds [24 x i16], ptr %b, i64 %indvars.iv, i64 8
+ %1 = load <8 x i16>, ptr %arrayidx1, align 16
+ %arrayidx2 = getelementptr inbounds [24 x i16], ptr %b, i64 %indvars.iv, i64 16
+ %2 = load <8 x i16>, ptr %arrayidx2, align 16
+ %tmp = add <8 x i16> %0, %1
+ %result = add <8 x i16> %tmp, %2
+ %arrayidx.out = getelementptr inbounds <8 x i16>, ptr %a, i64 %indvars.iv
+ store <8 x i16> %result, ptr %arrayidx.out, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @ld4q(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @ld4q(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP0]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [32 x i16], ptr [[B]], i64 [[INDEX]], i64 0
+; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 32 x i16>, ptr [[TMP2]], align 16
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.vector.deinterleave.segments4.nxv32i16(<vscale x 32 x i16> [[WIDE_VEC]])
+; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[STRIDED_VEC]], 0
+; CHECK-NEXT: [[TMP6:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[STRIDED_VEC]], 1
+; CHECK-NEXT: [[TMP8:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[STRIDED_VEC]], 2
+; CHECK-NEXT: [[TMP10:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[STRIDED_VEC]], 3
+; CHECK-NEXT: [[TMP11:%.*]] = add <vscale x 8 x i16> [[TMP4]], [[TMP6]]
+; CHECK-NEXT: [[TMP12:%.*]] = add <vscale x 8 x i16> [[TMP11]], [[TMP8]]
+; CHECK-NEXT: [[TMP13:%.*]] = add <vscale x 8 x i16> [[TMP12]], [[TMP10]]
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds <8 x i16>, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: store <vscale x 8 x i16> [[TMP13]], ptr [[TMP14]], align 16
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[ARRAYIDX0:%.*]] = getelementptr inbounds [32 x i16], ptr [[B]], i64 [[INDVARS_IV]], i64 0
+; CHECK-NEXT: [[TMP16:%.*]] = load <8 x i16>, ptr [[ARRAYIDX0]], align 16
+; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds [32 x i16], ptr [[B]], i64 [[INDVARS_IV]], i64 8
+; CHECK-NEXT: [[TMP17:%.*]] = load <8 x i16>, ptr [[ARRAYIDX1]], align 16
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds [32 x i16], ptr [[B]], i64 [[INDVARS_IV]], i64 16
+; CHECK-NEXT: [[TMP18:%.*]] = load <8 x i16>, ptr [[ARRAYIDX2]], align 16
+; CHECK-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds [32 x i16], ptr [[B]], i64 [[INDVARS_IV]], i64 24
+; CHECK-NEXT: [[TMP19:%.*]] = load <8 x i16>, ptr [[ARRAYIDX3]], align 16
+; CHECK-NEXT: [[TMP:%.*]] = add <8 x i16> [[TMP16]], [[TMP17]]
+; CHECK-NEXT: [[TMP4:%.*]] = add <8 x i16> [[TMP]], [[TMP18]]
+; CHECK-NEXT: [[RESULT:%.*]] = add <8 x i16> [[TMP4]], [[TMP19]]
+; CHECK-NEXT: [[ARRAYIDX_OUT:%.*]] = getelementptr inbounds <8 x i16>, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: store <8 x i16> [[RESULT]], ptr [[ARRAYIDX_OUT]], align 16
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1024
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx0 = getelementptr inbounds [32 x i16], ptr %b, i64 %indvars.iv, i64 0
+ %0 = load <8 x i16>, ptr %arrayidx0, align 16
+ %arrayidx1 = getelementptr inbounds [32 x i16], ptr %b, i64 %indvars.iv, i64 8
+ %1 = load <8 x i16>, ptr %arrayidx1, align 16
+ %arrayidx2 = getelementptr inbounds [32 x i16], ptr %b, i64 %indvars.iv, i64 16
+ %2 = load <8 x i16>, ptr %arrayidx2, align 16
+ %arrayidx3 = getelementptr inbounds [32 x i16], ptr %b, i64 %indvars.iv, i64 24
+ %3 = load <8 x i16>, ptr %arrayidx3, align 16
+ %tmp = add <8 x i16> %0, %1
+ %tmp2 = add <8 x i16> %tmp, %2
+ %result = add <8 x i16> %tmp2, %3
+ %arrayidx.out = getelementptr inbounds <8 x i16>, ptr %a, i64 %indvars.iv
+ store <8 x i16> %result, ptr %arrayidx.out, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @ld4q_gap(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @ld4q_gap(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP0]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [32 x i16], ptr [[B]], i64 [[INDEX]], i64 0
+; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 32 x i16>, ptr [[TMP2]], align 16
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.vector.deinterleave.segments4.nxv32i16(<vscale x 32 x i16> [[WIDE_VEC]])
+; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[STRIDED_VEC]], 0
+; CHECK-NEXT: [[TMP6:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[STRIDED_VEC]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[STRIDED_VEC]], 3
+; CHECK-NEXT: [[TMP12:%.*]] = add <vscale x 8 x i16> [[TMP4]], [[TMP6]]
+; CHECK-NEXT: [[TMP13:%.*]] = add <vscale x 8 x i16> [[TMP12]], [[TMP10]]
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds <8 x i16>, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: store <vscale x 8 x i16> [[TMP13]], ptr [[TMP14]], align 16
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[ARRAYIDX0:%.*]] = getelementptr inbounds [32 x i16], ptr [[B]], i64 [[INDVARS_IV]], i64 0
+; CHECK-NEXT: [[TMP16:%.*]] = load <8 x i16>, ptr [[ARRAYIDX0]], align 16
+; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds [32 x i16], ptr [[B]], i64 [[INDVARS_IV]], i64 8
+; CHECK-NEXT: [[TMP17:%.*]] = load <8 x i16>, ptr [[ARRAYIDX1]], align 16
+; CHECK-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds [32 x i16], ptr [[B]], i64 [[INDVARS_IV]], i64 24
+; CHECK-NEXT: [[TMP19:%.*]] = load <8 x i16>, ptr [[ARRAYIDX3]], align 16
+; CHECK-NEXT: [[TMP:%.*]] = add <8 x i16> [[TMP16]], [[TMP17]]
+; CHECK-NEXT: [[RESULT:%.*]] = add <8 x i16> [[TMP]], [[TMP19]]
+; CHECK-NEXT: [[ARRAYIDX_OUT:%.*]] = getelementptr inbounds <8 x i16>, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: store <8 x i16> [[RESULT]], ptr [[ARRAYIDX_OUT]], align 16
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1024
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx0 = getelementptr inbounds [32 x i16], ptr %b, i64 %indvars.iv, i64 0
+ %0 = load <8 x i16>, ptr %arrayidx0, align 16
+ %arrayidx1 = getelementptr inbounds [32 x i16], ptr %b, i64 %indvars.iv, i64 8
+ %1 = load <8 x i16>, ptr %arrayidx1, align 16
+ %arrayidx3 = getelementptr inbounds [32 x i16], ptr %b, i64 %indvars.iv, i64 24
+ %3 = load <8 x i16>, ptr %arrayidx3, align 16
+ %tmp = add <8 x i16> %0, %1
+ %result = add <8 x i16> %tmp, %3
+ %arrayidx.out = getelementptr inbounds <8 x i16>, ptr %a, i64 %indvars.iv
+ store <8 x i16> %result, ptr %arrayidx.out, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @ld4q_gap_end(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @ld4q_gap_end(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 1024, [[TMP0]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP0]]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
+; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[TMP2]], i64 [[TMP0]], i64 [[N_MOD_VF]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[TMP3]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [32 x i16], ptr [[B]], i64 [[INDEX]], i64 0
+; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 32 x i16>, ptr [[TMP4]], align 16
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.vector.deinterleave.segments4.nxv32i16(<vscale x 32 x i16> [[WIDE_VEC]])
+; CHECK-NEXT: [[TMP6:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[STRIDED_VEC]], 0
+; CHECK-NEXT: [[TMP8:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[STRIDED_VEC]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[STRIDED_VEC]], 2
+; CHECK-NEXT: [[TMP11:%.*]] = add <vscale x 8 x i16> [[TMP6]], [[TMP8]]
+; CHECK-NEXT: [[TMP12:%.*]] = add <vscale x 8 x i16> [[TMP11]], [[TMP10]]
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds <8 x i16>, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: store <vscale x 8 x i16> [[TMP12]], ptr [[TMP13]], align 16
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP:.*]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[ARRAYIDX0:%.*]] = getelementptr inbounds [32 x i16], ptr [[B]], i64 [[INDVARS_IV]], i64 0
+; CHECK-NEXT: [[TMP15:%.*]] = load <8 x i16>, ptr [[ARRAYIDX0]], align 16
+; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds [32 x i16], ptr [[B]], i64 [[INDVARS_IV]], i64 8
+; CHECK-NEXT: [[TMP16:%.*]] = load <8 x i16>, ptr [[ARRAYIDX1]], align 16
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds [32 x i16], ptr [[B]], i64 [[INDVARS_IV]], i64 16
+; CHECK-NEXT: [[TMP17:%.*]] = load <8 x i16>, ptr [[ARRAYIDX2]], align 16
+; CHECK-NEXT: [[TMP:%.*]] = add <8 x i16> [[TMP15]], [[TMP16]]
+; CHECK-NEXT: [[RESULT:%.*]] = add <8 x i16> [[TMP]], [[TMP17]]
+; CHECK-NEXT: [[ARRAYIDX_OUT:%.*]] = getelementptr inbounds <8 x i16>, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: store <8 x i16> [[RESULT]], ptr [[ARRAYIDX_OUT]], align 16
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1024
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx0 = getelementptr inbounds [32 x i16], ptr %b, i64 %indvars.iv, i64 0
+ %0 = load <8 x i16>, ptr %arrayidx0, align 16
+ %arrayidx1 = getelementptr inbounds [32 x i16], ptr %b, i64 %indvars.iv, i64 8
+ %1 = load <8 x i16>, ptr %arrayidx1, align 16
+ %arrayidx2 = getelementptr inbounds [32 x i16], ptr %b, i64 %indvars.iv, i64 16
+ %2 = load <8 x i16>, ptr %arrayidx2, align 16
+ %tmp = add <8 x i16> %0, %1
+ %result = add <8 x i16> %tmp, %2
+ %arrayidx.out = getelementptr inbounds <8 x i16>, ptr %a, i64 %indvars.iv
+ store <8 x i16> %result, ptr %arrayidx.out, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
+; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
+; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}
+; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
+; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META2]], [[META1]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-interleaved-stores.ll b/llvm/test/Transforms/LoopVectorize/AArch64/revec-interleaved-stores.ll
new file mode 100644
index 0000000000000..fba8ef7e8fbd0
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/revec-interleaved-stores.ll
@@ -0,0 +1,360 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -vectorize-vector-loops \
+; RUN: -mtriple=aarch64 -mattr=+sve2p1 -S < %s | FileCheck %s
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -vectorize-vector-loops \
+; RUN: -tail-folding-policy=prefer-fold-tail \
+; RUN: -mtriple=aarch64 -mattr=+sve2p1 -S < %s | FileCheck %s
+
+define void @st2q(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @st2q(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP0]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <8 x i16>, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP2]], align 16
+; CHECK-NEXT: [[TMP3:%.*]] = add <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 1)
+; CHECK-NEXT: [[TMP4:%.*]] = add <vscale x 8 x i16> [[TMP3]], splat (i16 1)
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [16 x i16], ptr [[A]], i64 [[INDEX]], i64 0
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 16 x i16> @llvm.vector.interleave.segments2.nxv16i16(<vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]])
+; CHECK-NEXT: store <vscale x 16 x i16> [[INTERLEAVED_VEC]], ptr [[TMP5]], align 16
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[ARRAYIDX_IN:%.*]] = getelementptr inbounds <8 x i16>, ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP9:%.*]] = load <8 x i16>, ptr [[ARRAYIDX_IN]], align 16
+; CHECK-NEXT: [[V0:%.*]] = add <8 x i16> [[TMP9]], splat (i16 1)
+; CHECK-NEXT: [[V1:%.*]] = add <8 x i16> [[V0]], splat (i16 1)
+; CHECK-NEXT: [[ARRAYIDX_OUT0:%.*]] = getelementptr inbounds [16 x i16], ptr [[A]], i64 [[INDVARS_IV]], i64 0
+; CHECK-NEXT: store <8 x i16> [[V0]], ptr [[ARRAYIDX_OUT0]], align 16
+; CHECK-NEXT: [[ARRAYIDX_OUT1:%.*]] = getelementptr inbounds [16 x i16], ptr [[A]], i64 [[INDVARS_IV]], i64 8
+; CHECK-NEXT: store <8 x i16> [[V1]], ptr [[ARRAYIDX_OUT1]], align 16
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1024
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx.in = getelementptr inbounds <8 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x i16>, ptr %arrayidx.in, align 16
+ %v0 = add <8 x i16> %0, splat (i16 1)
+ %v1 = add <8 x i16> %v0, splat (i16 1)
+
+ %arrayidx.out0 = getelementptr inbounds [16 x i16], ptr %a, i64 %indvars.iv, i64 0
+ store <8 x i16> %v0, ptr %arrayidx.out0, align 16
+ %arrayidx.out1 = getelementptr inbounds [16 x i16], ptr %a, i64 %indvars.iv, i64 8
+ store <8 x i16> %v1, ptr %arrayidx.out1, align 16
+
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @st2q_reverse(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @st2q_reverse(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP:.*]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 1024, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[ARRAYIDX_IN:%.*]] = getelementptr inbounds <8 x i16>, ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP20:%.*]] = load <8 x i16>, ptr [[ARRAYIDX_IN]], align 16
+; CHECK-NEXT: [[V0:%.*]] = add <8 x i16> [[TMP20]], splat (i16 1)
+; CHECK-NEXT: [[V1:%.*]] = add <8 x i16> [[V0]], splat (i16 1)
+; CHECK-NEXT: [[ARRAYIDX_OUT0:%.*]] = getelementptr inbounds [16 x i16], ptr [[A]], i64 [[INDVARS_IV]], i64 0
+; CHECK-NEXT: store <8 x i16> [[V0]], ptr [[ARRAYIDX_OUT0]], align 16
+; CHECK-NEXT: [[ARRAYIDX_OUT1:%.*]] = getelementptr inbounds [16 x i16], ptr [[A]], i64 [[INDVARS_IV]], i64 8
+; CHECK-NEXT: store <8 x i16> [[V1]], ptr [[ARRAYIDX_OUT1]], align 16
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], -1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 0
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 1024, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx.in = getelementptr inbounds <8 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x i16>, ptr %arrayidx.in, align 16
+ %v0 = add <8 x i16> %0, splat (i16 1)
+ %v1 = add <8 x i16> %v0, splat (i16 1)
+
+ %arrayidx.out0 = getelementptr inbounds [16 x i16], ptr %a, i64 %indvars.iv, i64 0
+ store <8 x i16> %v0, ptr %arrayidx.out0, align 16
+ %arrayidx.out1 = getelementptr inbounds [16 x i16], ptr %a, i64 %indvars.iv, i64 8
+ store <8 x i16> %v1, ptr %arrayidx.out1, align 16
+
+ %indvars.iv.next = add nsw i64 %indvars.iv, -1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 0
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @st3q(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @st3q(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP:.*]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[ARRAYIDX_IN:%.*]] = getelementptr inbounds <8 x i16>, ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <8 x i16>, ptr [[ARRAYIDX_IN]], align 16
+; CHECK-NEXT: [[V0:%.*]] = add <8 x i16> [[TMP0]], splat (i16 1)
+; CHECK-NEXT: [[V1:%.*]] = add <8 x i16> [[V0]], splat (i16 1)
+; CHECK-NEXT: [[V2:%.*]] = add <8 x i16> [[V1]], splat (i16 1)
+; CHECK-NEXT: [[ARRAYIDX_OUT0:%.*]] = getelementptr inbounds [24 x i16], ptr [[A]], i64 [[INDVARS_IV]], i64 0
+; CHECK-NEXT: store <8 x i16> [[V0]], ptr [[ARRAYIDX_OUT0]], align 16
+; CHECK-NEXT: [[ARRAYIDX_OUT1:%.*]] = getelementptr inbounds [24 x i16], ptr [[A]], i64 [[INDVARS_IV]], i64 8
+; CHECK-NEXT: store <8 x i16> [[V1]], ptr [[ARRAYIDX_OUT1]], align 16
+; CHECK-NEXT: [[ARRAYIDX_OUT2:%.*]] = getelementptr inbounds [24 x i16], ptr [[A]], i64 [[INDVARS_IV]], i64 16
+; CHECK-NEXT: store <8 x i16> [[V2]], ptr [[ARRAYIDX_OUT2]], align 16
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1024
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx.in = getelementptr inbounds <8 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x i16>, ptr %arrayidx.in, align 16
+ %v0 = add <8 x i16> %0, splat (i16 1)
+ %v1 = add <8 x i16> %v0, splat (i16 1)
+ %v2 = add <8 x i16> %v1, splat (i16 1)
+
+ %arrayidx.out0 = getelementptr inbounds [24 x i16], ptr %a, i64 %indvars.iv, i64 0
+ store <8 x i16> %v0, ptr %arrayidx.out0, align 16
+ %arrayidx.out1 = getelementptr inbounds [24 x i16], ptr %a, i64 %indvars.iv, i64 8
+ store <8 x i16> %v1, ptr %arrayidx.out1, align 16
+ %arrayidx.out2 = getelementptr inbounds [24 x i16], ptr %a, i64 %indvars.iv, i64 16
+ store <8 x i16> %v2, ptr %arrayidx.out2, align 16
+
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @st4q(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @st4q(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP0]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <8 x i16>, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP2]], align 16
+; CHECK-NEXT: [[TMP3:%.*]] = add <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 1)
+; CHECK-NEXT: [[TMP4:%.*]] = add <vscale x 8 x i16> [[TMP3]], splat (i16 1)
+; CHECK-NEXT: [[TMP5:%.*]] = add <vscale x 8 x i16> [[TMP4]], splat (i16 1)
+; CHECK-NEXT: [[TMP6:%.*]] = add <vscale x 8 x i16> [[TMP5]], splat (i16 1)
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds [32 x i16], ptr [[A]], i64 [[INDEX]], i64 0
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 32 x i16> @llvm.vector.interleave.segments4.nxv32i16(<vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], <vscale x 8 x i16> [[TMP5]], <vscale x 8 x i16> [[TMP6]])
+; CHECK-NEXT: store <vscale x 32 x i16> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 16
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[ARRAYIDX_IN:%.*]] = getelementptr inbounds <8 x i16>, ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP13:%.*]] = load <8 x i16>, ptr [[ARRAYIDX_IN]], align 16
+; CHECK-NEXT: [[V0:%.*]] = add <8 x i16> [[TMP13]], splat (i16 1)
+; CHECK-NEXT: [[V1:%.*]] = add <8 x i16> [[V0]], splat (i16 1)
+; CHECK-NEXT: [[V2:%.*]] = add <8 x i16> [[V1]], splat (i16 1)
+; CHECK-NEXT: [[V3:%.*]] = add <8 x i16> [[V2]], splat (i16 1)
+; CHECK-NEXT: [[ARRAYIDX_OUT0:%.*]] = getelementptr inbounds [32 x i16], ptr [[A]], i64 [[INDVARS_IV]], i64 0
+; CHECK-NEXT: store <8 x i16> [[V0]], ptr [[ARRAYIDX_OUT0]], align 16
+; CHECK-NEXT: [[ARRAYIDX_OUT1:%.*]] = getelementptr inbounds [32 x i16], ptr [[A]], i64 [[INDVARS_IV]], i64 8
+; CHECK-NEXT: store <8 x i16> [[V1]], ptr [[ARRAYIDX_OUT1]], align 16
+; CHECK-NEXT: [[ARRAYIDX_OUT2:%.*]] = getelementptr inbounds [32 x i16], ptr [[A]], i64 [[INDVARS_IV]], i64 16
+; CHECK-NEXT: store <8 x i16> [[V2]], ptr [[ARRAYIDX_OUT2]], align 16
+; CHECK-NEXT: [[ARRAYIDX_OUT3:%.*]] = getelementptr inbounds [32 x i16], ptr [[A]], i64 [[INDVARS_IV]], i64 24
+; CHECK-NEXT: store <8 x i16> [[V3]], ptr [[ARRAYIDX_OUT3]], align 16
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1024
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx.in = getelementptr inbounds <8 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x i16>, ptr %arrayidx.in, align 16
+ %v0 = add <8 x i16> %0, splat (i16 1)
+ %v1 = add <8 x i16> %v0, splat (i16 1)
+ %v2 = add <8 x i16> %v1, splat (i16 1)
+ %v3 = add <8 x i16> %v2, splat (i16 1)
+
+ %arrayidx.out0 = getelementptr inbounds [32 x i16], ptr %a, i64 %indvars.iv, i64 0
+ store <8 x i16> %v0, ptr %arrayidx.out0, align 16
+ %arrayidx.out1 = getelementptr inbounds [32 x i16], ptr %a, i64 %indvars.iv, i64 8
+ store <8 x i16> %v1, ptr %arrayidx.out1, align 16
+ %arrayidx.out2 = getelementptr inbounds [32 x i16], ptr %a, i64 %indvars.iv, i64 16
+ store <8 x i16> %v2, ptr %arrayidx.out2, align 16
+ %arrayidx.out3 = getelementptr inbounds [32 x i16], ptr %a, i64 %indvars.iv, i64 24
+ store <8 x i16> %v3, ptr %arrayidx.out3, align 16
+
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+; Store interleaving with gaps isn't supported for REVEC
+define void @st4q_gap_mid(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @st4q_gap_mid(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP:.*]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[ARRAYIDX_IN:%.*]] = getelementptr inbounds <8 x i16>, ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <8 x i16>, ptr [[ARRAYIDX_IN]], align 16
+; CHECK-NEXT: [[V0:%.*]] = add <8 x i16> [[TMP0]], splat (i16 1)
+; CHECK-NEXT: [[V1:%.*]] = add <8 x i16> [[V0]], splat (i16 1)
+; CHECK-NEXT: [[V3:%.*]] = add <8 x i16> [[V1]], splat (i16 1)
+; CHECK-NEXT: [[ARRAYIDX_OUT0:%.*]] = getelementptr inbounds [32 x i16], ptr [[A]], i64 [[INDVARS_IV]], i64 0
+; CHECK-NEXT: store <8 x i16> [[V0]], ptr [[ARRAYIDX_OUT0]], align 16
+; CHECK-NEXT: [[ARRAYIDX_OUT1:%.*]] = getelementptr inbounds [32 x i16], ptr [[A]], i64 [[INDVARS_IV]], i64 8
+; CHECK-NEXT: store <8 x i16> [[V1]], ptr [[ARRAYIDX_OUT1]], align 16
+; CHECK-NEXT: [[ARRAYIDX_OUT3:%.*]] = getelementptr inbounds [32 x i16], ptr [[A]], i64 [[INDVARS_IV]], i64 24
+; CHECK-NEXT: store <8 x i16> [[V3]], ptr [[ARRAYIDX_OUT3]], align 16
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1024
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx.in = getelementptr inbounds <8 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x i16>, ptr %arrayidx.in, align 16
+ %v0 = add <8 x i16> %0, splat (i16 1)
+ %v1 = add <8 x i16> %v0, splat (i16 1)
+ %v3 = add <8 x i16> %v1, splat (i16 1)
+
+ %arrayidx.out0 = getelementptr inbounds [32 x i16], ptr %a, i64 %indvars.iv, i64 0
+ store <8 x i16> %v0, ptr %arrayidx.out0, align 16
+ %arrayidx.out1 = getelementptr inbounds [32 x i16], ptr %a, i64 %indvars.iv, i64 8
+ store <8 x i16> %v1, ptr %arrayidx.out1, align 16
+ %arrayidx.out3 = getelementptr inbounds [32 x i16], ptr %a, i64 %indvars.iv, i64 24
+ store <8 x i16> %v3, ptr %arrayidx.out3, align 16
+
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+; Store interleaving with gaps isn't supported for REVEC
+define void @st4q_gap_end(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @st4q_gap_end(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP:.*]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[ARRAYIDX_IN:%.*]] = getelementptr inbounds <8 x i16>, ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <8 x i16>, ptr [[ARRAYIDX_IN]], align 16
+; CHECK-NEXT: [[V0:%.*]] = add <8 x i16> [[TMP0]], splat (i16 1)
+; CHECK-NEXT: [[V1:%.*]] = add <8 x i16> [[V0]], splat (i16 1)
+; CHECK-NEXT: [[V2:%.*]] = add <8 x i16> [[V1]], splat (i16 1)
+; CHECK-NEXT: [[ARRAYIDX_OUT0:%.*]] = getelementptr inbounds [32 x i16], ptr [[A]], i64 [[INDVARS_IV]], i64 0
+; CHECK-NEXT: store <8 x i16> [[V0]], ptr [[ARRAYIDX_OUT0]], align 16
+; CHECK-NEXT: [[ARRAYIDX_OUT1:%.*]] = getelementptr inbounds [32 x i16], ptr [[A]], i64 [[INDVARS_IV]], i64 8
+; CHECK-NEXT: store <8 x i16> [[V1]], ptr [[ARRAYIDX_OUT1]], align 16
+; CHECK-NEXT: [[ARRAYIDX_OUT2:%.*]] = getelementptr inbounds [32 x i16], ptr [[A]], i64 [[INDVARS_IV]], i64 16
+; CHECK-NEXT: store <8 x i16> [[V2]], ptr [[ARRAYIDX_OUT2]], align 16
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1024
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx.in = getelementptr inbounds <8 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x i16>, ptr %arrayidx.in, align 16
+ %v0 = add <8 x i16> %0, splat (i16 1)
+ %v1 = add <8 x i16> %v0, splat (i16 1)
+ %v2 = add <8 x i16> %v1, splat (i16 1)
+
+ %arrayidx.out0 = getelementptr inbounds [32 x i16], ptr %a, i64 %indvars.iv, i64 0
+ store <8 x i16> %v0, ptr %arrayidx.out0, align 16
+ %arrayidx.out1 = getelementptr inbounds [32 x i16], ptr %a, i64 %indvars.iv, i64 8
+ store <8 x i16> %v1, ptr %arrayidx.out1, align 16
+ %arrayidx.out2 = getelementptr inbounds [32 x i16], ptr %a, i64 %indvars.iv, i64 16
+ store <8 x i16> %v2, ptr %arrayidx.out2, align 16
+
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-inv-store.ll b/llvm/test/Transforms/LoopVectorize/AArch64/revec-inv-store.ll
new file mode 100644
index 0000000000000..9897217320d7c
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/revec-inv-store.ll
@@ -0,0 +1,80 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^scalar.ph:"
+; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=1 -vectorize-vector-loops \
+; RUN: -scalable-vectorization=on -force-vector-width=1 \
+; RUN: -mtriple=aarch64 -mattr=+sve2 -S | FileCheck %s --check-prefixes=CHECK
+; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=1 -vectorize-vector-loops \
+; RUN: -scalable-vectorization=on -force-vector-width=1 \
+; RUN: -mtriple=aarch64 -mattr=+sve2p1 -S | FileCheck %s --check-prefixes=CHECK
+; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=2 -vectorize-vector-loops \
+; RUN: -mtriple=aarch64 -mattr=+sve2p1 -S | FileCheck %s --check-prefixes=CHECK-IC2
+
+; Test we can interleave without vectorizing
+; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=2 -force-vector-width=1 \
+; RUN: -vectorize-vector-loops -mtriple=aarch64 -mattr=+sve2p1 -S \
+; RUN: | FileCheck %s --check-prefixes=CHECK-IC2
+
+; Note that the IC=2 tests are not doing any interleaving or vectorization, but
+; still create a middle block and vector.body. This isn't a limitation coming
+; from REVEC. We'd get the same with a scalar invariant store.
+define void @inv_store_v4i32(ptr noalias %dst, ptr noalias readonly %src, i64 %N) #0 {
+; CHECK-LABEL: @inv_store_v4i32(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP0]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP0]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[ARRAYIDX]], align 16
+; CHECK-NEXT: [[TMP4:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> poison, i32 4)
+; CHECK-NEXT: [[LD:%.*]] = call <4 x i32> @llvm.vector.extract.v4i32.nxv4i32(<vscale x 4 x i32> [[TMP4]], i64 0)
+; CHECK-NEXT: store <4 x i32> [[LD]], ptr [[DST:%.*]], align 2
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_INC24:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+;
+; CHECK-IC2-LABEL: @inv_store_v4i32(
+; CHECK-IC2-NEXT: entry:
+; CHECK-IC2-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N:%.*]], 2
+; CHECK-IC2-NEXT: br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-IC2: vector.ph:
+; CHECK-IC2-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 2
+; CHECK-IC2-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-IC2-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK-IC2: vector.body:
+; CHECK-IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-IC2-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
+; CHECK-IC2-NEXT: [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC:%.*]], i64 [[TMP0]]
+; CHECK-IC2-NEXT: [[TMP2:%.*]] = load <4 x i32>, ptr [[TMP1]], align 16
+; CHECK-IC2-NEXT: store <4 x i32> [[TMP2]], ptr [[DST:%.*]], align 2
+; CHECK-IC2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-IC2-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-IC2-NEXT: br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-IC2: middle.block:
+; CHECK-IC2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-IC2-NEXT: br i1 [[CMP_N]], label [[FOR_INC24:%.*]], label [[VEC_EPILOG_PH]]
+; CHECK-IC2: scalar.ph:
+;
+entry:
+ br label %for.body14
+
+for.body14: ; preds = %for.body14, %entry
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body14 ]
+ %arrayidx = getelementptr inbounds <4 x i32>, ptr %src, i64 %indvars.iv
+ %ld = load <4 x i32>, ptr %arrayidx
+ store <4 x i32> %ld, ptr %dst, align 2
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, %N
+ br i1 %exitcond.not, label %for.inc24, label %for.body14
+
+for.inc24: ; preds = %for.body14, %for.body
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-invalid.ll b/llvm/test/Transforms/LoopVectorize/AArch64/revec-invalid.ll
new file mode 100644
index 0000000000000..21bb9605299f1
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/revec-invalid.ll
@@ -0,0 +1,72 @@
+; REQUIRES: asserts
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -vectorize-vector-loops \
+; RUN: -mtriple=aarch64 -mattr=+sve2p1 -debug-only=loop-vectorize \
+; RUN: -disable-output -S < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,VECTORIZE
+; RUN: opt -passes=loop-vectorize -force-vector-width=1 -force-vector-interleave=2 \
+; RUN: -vectorize-vector-loops -mtriple=aarch64 -mattr=+sve2p1 \
+; RUN: -debug-only=loop-vectorize -disable-output -S < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,INTERLEAVE
+
+; CHECK-LABEL: test_store_fcmp
+; VECTORIZE: Cost of Invalid for VF vscale x 1: WIDEN store vp<%6>, ir<%resi1>
+; INTERLEAVE: Executing best plan with VF=1, UF=2
+define void @test_store_fcmp(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <8 x half>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x half>, ptr %arrayidx, align 16
+ %resi1 = fcmp fast ule <8 x half> %0, zeroinitializer
+ %arrayidx2 = getelementptr inbounds <8 x i1>, ptr %a, i64 %indvars.iv
+ store <8 x i1> %resi1, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+; CHECK-LABEL: insertelement_cost
+; VECTORIZE: Cost of Invalid for VF vscale x 1: REPLICATE ir<%result> = insertelement ir<zeroinitializer>, ir<%0>, ir<0>
+; INTERLEAVE: Executing best plan with VF=1, UF=2
+define void @insertelement_cost(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds i16, ptr %b, i64 %indvars.iv
+ %0 = load i16, ptr %arrayidx, align 16
+ %result = insertelement <8 x i16> zeroinitializer, i16 %0, i32 0
+ %arrayidx2 = getelementptr inbounds <8 x i16>, ptr %a, i64 %indvars.iv
+ store <8 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+; CHECK-LABEL: extractelement_cost
+; CHECK: LV: Not vectorizing: Found unvectorizable type %result = extractelement <4 x i32> %0, i32 1
+define void @extractelement_cost(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i32>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i32>, ptr %arrayidx, align 16
+ %result = extractelement <4 x i32> %0, i32 1
+ %arrayidx2 = getelementptr inbounds i32, ptr %a, i64 %indvars.iv
+ store i32 %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-liveins.ll b/llvm/test/Transforms/LoopVectorize/AArch64/revec-liveins.ll
new file mode 100644
index 0000000000000..8d106b624eec6
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/revec-liveins.ll
@@ -0,0 +1,269 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -vectorize-vector-loops -mtriple=aarch64 -mattr=+sve2p1 -S < %s | FileCheck %s
+
+; Test how vector liveins (i.e. values defined outside the loop) are handled.
+
+define dso_local void @splat0(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define dso_local void @splat0(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP0]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <4 x i32>, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: store <vscale x 4 x i32> zeroinitializer, ptr [[TMP2]], align 16
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds <4 x i32>, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: store <4 x i32> zeroinitializer, ptr [[ARRAYIDX2]], align 16
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1024
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx2 = getelementptr inbounds <4 x i32>, ptr %a, i64 %indvars.iv
+ store <4 x i32> zeroinitializer, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define dso_local void @splat1(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define dso_local void @splat1(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP0]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <4 x i32>, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: store <vscale x 4 x i32> splat (i32 1), ptr [[TMP2]], align 16
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds <4 x i32>, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: store <4 x i32> splat (i32 1), ptr [[ARRAYIDX2]], align 16
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1024
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx2 = getelementptr inbounds <4 x i32>, ptr %a, i64 %indvars.iv
+ store <4 x i32> splat (i32 1), ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define dso_local void @constant_list(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define dso_local void @constant_list(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP0]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <4 x i32>, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[BROADCAST:%.*]] = call <vscale x 4 x i32> @llvm.vector.broadcast.nxv4i32.v4i32(<4 x i32> <i32 0, i32 1, i32 2, i32 3>)
+; CHECK-NEXT: store <vscale x 4 x i32> [[BROADCAST]], ptr [[TMP2]], align 16
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds <4 x i32>, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: store <4 x i32> <i32 0, i32 1, i32 2, i32 3>, ptr [[ARRAYIDX2]], align 16
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1024
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx2 = getelementptr inbounds <4 x i32>, ptr %a, i64 %indvars.iv
+ store <4 x i32> <i32 0, i32 1, i32 2, i32 3>, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define dso_local void @arbitrary_vec(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b, <4 x i32> %in) {
+; CHECK-LABEL: define dso_local void @arbitrary_vec(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], <4 x i32> [[IN:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP0]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT: [[BROADCAST:%.*]] = call <vscale x 4 x i32> @llvm.vector.broadcast.nxv4i32.v4i32(<4 x i32> [[IN]])
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <4 x i32>, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: store <vscale x 4 x i32> [[BROADCAST]], ptr [[TMP2]], align 16
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds <4 x i32>, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: store <4 x i32> [[IN]], ptr [[ARRAYIDX2]], align 16
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1024
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx2 = getelementptr inbounds <4 x i32>, ptr %a, i64 %indvars.iv
+ store <4 x i32> %in, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define dso_local void @uniform_inst_in_loop(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b, ptr nocapture noundef readonly %cst) {
+; CHECK-LABEL: define dso_local void @uniform_inst_in_loop(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], ptr noundef readonly captures(none) [[CST:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP0]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <4 x i32>, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP3:%.*]] = load <4 x i32>, ptr [[CST]], align 16
+; CHECK-NEXT: [[BROADCAST:%.*]] = call <vscale x 4 x i32> @llvm.vector.broadcast.nxv4i32.v4i32(<4 x i32> [[TMP3]])
+; CHECK-NEXT: store <vscale x 4 x i32> [[BROADCAST]], ptr [[TMP2]], align 16
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds <4 x i32>, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[UNIFORM:%.*]] = load <4 x i32>, ptr [[CST]], align 16
+; CHECK-NEXT: store <4 x i32> [[UNIFORM]], ptr [[ARRAYIDX2]], align 16
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1024
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx2 = getelementptr inbounds <4 x i32>, ptr %a, i64 %indvars.iv
+ %uniform = load <4 x i32>, ptr %cst
+ store <4 x i32> %uniform, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
+; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
+; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}
+; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
+; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META2]], [[META1]]}
+; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]}
+; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META2]], [[META1]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-mul-invariant-op.ll b/llvm/test/Transforms/LoopVectorize/AArch64/revec-mul-invariant-op.ll
new file mode 100644
index 0000000000000..32494ceff1396
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/revec-mul-invariant-op.ll
@@ -0,0 +1,28 @@
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=1 \
+; RUN: -scalable-vectorization=on -vectorize-vector-loops -mtriple=aarch64 \
+; RUN: -mattr=+sve2p1 -S < %s | FileCheck %s
+
+
+define void @vector_mul_invariant_vector_operand(ptr noalias %a, ptr noalias readonly %b, <8 x i16> %m) {
+; CHECK-LABEL: define void @vector_mul_invariant_vector_operand(
+; CHECK: vector.ph:
+; CHECK: call <vscale x 8 x i16> @llvm.vector.broadcast.nxv8i16.v8i16(<8 x i16> %m)
+; CHECK: vector.body:
+; CHECK: mul <vscale x 8 x i16>
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %bp = getelementptr inbounds <8 x i16>, ptr %b, i64 %iv
+ %v = load <8 x i16>, ptr %bp, align 16
+ %r = mul <8 x i16> %m, %v
+ %ap = getelementptr inbounds <8 x i16>, ptr %a, i64 %iv
+ store <8 x i16> %r, ptr %ap, align 16
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exit = icmp eq i64 %iv.next, 1024
+ br i1 %exit, label %done, label %loop
+
+done:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-neon-64bit.ll b/llvm/test/Transforms/LoopVectorize/AArch64/revec-neon-64bit.ll
new file mode 100644
index 0000000000000..4f133516a5c58
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/revec-neon-64bit.ll
@@ -0,0 +1,1036 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "scalar.ph:" --replace-value-regex "!llvm.loop ![0-9]+" --version 6
+; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=1 -vectorize-vector-loops \
+; RUN: -scalable-vectorization=on -force-vector-width=1 \
+; RUN: -mtriple=aarch64 -mattr=+sve2p1 -S | FileCheck %s --check-prefixes=CHECK,VScale1
+; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=1 -vectorize-vector-loops \
+; RUN: -scalable-vectorization=on -force-vector-width=2 \
+; RUN: -mtriple=aarch64 -mattr=+sve2p1 -S | FileCheck %s --check-prefixes=CHECK,VScale2
+
+; Test how 64-bit NEON intrinsic are re-vectorised using HVLA.
+
+define void @uabd(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; VScale1-LABEL: define void @uabd(
+; VScale1-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; VScale1-NEXT: [[ENTRY:.*:]]
+; VScale1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; VScale1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; VScale1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VScale1: [[VECTOR_PH]]:
+; VScale1-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP0]]
+; VScale1-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; VScale1-NEXT: br label %[[VECTOR_BODY:.*]]
+; VScale1: [[VECTOR_BODY]]:
+; VScale1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VScale1-NEXT: [[TMP2:%.*]] = getelementptr inbounds <4 x i16>, ptr [[B]], i64 [[INDEX]]
+; VScale1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP2]], align 16
+; VScale1-NEXT: [[TO_SVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.interleave2.nxv8i16(<vscale x 4 x i16> [[WIDE_LOAD]], <vscale x 4 x i16> poison)
+; VScale1-NEXT: [[TO_SVE1:%.*]] = call <vscale x 8 x i16> @llvm.vector.interleave2.nxv8i16(<vscale x 4 x i16> splat (i16 1), <vscale x 4 x i16> poison)
+; VScale1-NEXT: [[TMP3:%.*]] = call <vscale x 8 x i16> @llvm.aarch64.sve.uabd.nxv8i16(<vscale x 8 x i1> splat (i1 true), <vscale x 8 x i16> [[TO_SVE]], <vscale x 8 x i16> [[TO_SVE1]])
+; VScale1-NEXT: [[TMP6:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> [[TMP3]])
+; VScale1-NEXT: [[FROM_SVE:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[TMP6]], 0
+; VScale1-NEXT: [[TMP4:%.*]] = getelementptr inbounds <4 x i16>, ptr [[A]], i64 [[INDEX]]
+; VScale1-NEXT: store <vscale x 4 x i16> [[FROM_SVE]], ptr [[TMP4]], align 16
+; VScale1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; VScale1-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VScale1-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; VScale1: [[MIDDLE_BLOCK]]:
+; VScale1-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; VScale1-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; VScale1: [[SCALAR_PH]]:
+;
+; VScale2-LABEL: define void @uabd(
+; VScale2-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; VScale2-NEXT: [[ENTRY:.*:]]
+; VScale2-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; VScale2-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; VScale2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
+; VScale2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VScale2: [[VECTOR_PH]]:
+; VScale2-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 1
+; VScale2-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
+; VScale2-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; VScale2-NEXT: br label %[[VECTOR_BODY:.*]]
+; VScale2: [[VECTOR_BODY]]:
+; VScale2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VScale2-NEXT: [[TMP4:%.*]] = getelementptr inbounds <4 x i16>, ptr [[B]], i64 [[INDEX]]
+; VScale2-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP4]], align 16
+; VScale2-NEXT: [[TMP5:%.*]] = call <vscale x 8 x i16> @llvm.aarch64.sve.uabd.nxv8i16(<vscale x 8 x i1> splat (i1 true), <vscale x 8 x i16> [[WIDE_LOAD]], <vscale x 8 x i16> splat (i16 1))
+; VScale2-NEXT: [[TMP6:%.*]] = getelementptr inbounds <4 x i16>, ptr [[A]], i64 [[INDEX]]
+; VScale2-NEXT: store <vscale x 8 x i16> [[TMP5]], ptr [[TMP6]], align 16
+; VScale2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; VScale2-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VScale2-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; VScale2: [[MIDDLE_BLOCK]]:
+; VScale2-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; VScale2-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; VScale2: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i16>, ptr %arrayidx, align 16
+ %result = call <4 x i16> @llvm.aarch64.neon.uabd.v8i16(<4 x i16> %0, <4 x i16> splat (i16 1))
+ %arrayidx2 = getelementptr inbounds <4 x i16>, ptr %a, i64 %indvars.iv
+ store <4 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @sabd(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; VScale1-LABEL: define void @sabd(
+; VScale1-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
+; VScale1-NEXT: [[ENTRY:.*:]]
+; VScale1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; VScale1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; VScale1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VScale1: [[VECTOR_PH]]:
+; VScale1-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP0]]
+; VScale1-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; VScale1-NEXT: br label %[[VECTOR_BODY:.*]]
+; VScale1: [[VECTOR_BODY]]:
+; VScale1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VScale1-NEXT: [[TMP2:%.*]] = getelementptr inbounds <4 x i16>, ptr [[B]], i64 [[INDEX]]
+; VScale1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP2]], align 16
+; VScale1-NEXT: [[TO_SVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.interleave2.nxv8i16(<vscale x 4 x i16> [[WIDE_LOAD]], <vscale x 4 x i16> poison)
+; VScale1-NEXT: [[TO_SVE1:%.*]] = call <vscale x 8 x i16> @llvm.vector.interleave2.nxv8i16(<vscale x 4 x i16> splat (i16 1), <vscale x 4 x i16> poison)
+; VScale1-NEXT: [[TMP3:%.*]] = call <vscale x 8 x i16> @llvm.aarch64.sve.sabd.nxv8i16(<vscale x 8 x i1> splat (i1 true), <vscale x 8 x i16> [[TO_SVE]], <vscale x 8 x i16> [[TO_SVE1]])
+; VScale1-NEXT: [[TMP6:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> [[TMP3]])
+; VScale1-NEXT: [[FROM_SVE:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[TMP6]], 0
+; VScale1-NEXT: [[TMP4:%.*]] = getelementptr inbounds <4 x i16>, ptr [[A]], i64 [[INDEX]]
+; VScale1-NEXT: store <vscale x 4 x i16> [[FROM_SVE]], ptr [[TMP4]], align 16
+; VScale1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; VScale1-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VScale1-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; VScale1: [[MIDDLE_BLOCK]]:
+; VScale1-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; VScale1-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; VScale1: [[SCALAR_PH]]:
+;
+; VScale2-LABEL: define void @sabd(
+; VScale2-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
+; VScale2-NEXT: [[ENTRY:.*:]]
+; VScale2-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; VScale2-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; VScale2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
+; VScale2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VScale2: [[VECTOR_PH]]:
+; VScale2-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 1
+; VScale2-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
+; VScale2-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; VScale2-NEXT: br label %[[VECTOR_BODY:.*]]
+; VScale2: [[VECTOR_BODY]]:
+; VScale2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VScale2-NEXT: [[TMP4:%.*]] = getelementptr inbounds <4 x i16>, ptr [[B]], i64 [[INDEX]]
+; VScale2-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP4]], align 16
+; VScale2-NEXT: [[TMP5:%.*]] = call <vscale x 8 x i16> @llvm.aarch64.sve.sabd.nxv8i16(<vscale x 8 x i1> splat (i1 true), <vscale x 8 x i16> [[WIDE_LOAD]], <vscale x 8 x i16> splat (i16 1))
+; VScale2-NEXT: [[TMP6:%.*]] = getelementptr inbounds <4 x i16>, ptr [[A]], i64 [[INDEX]]
+; VScale2-NEXT: store <vscale x 8 x i16> [[TMP5]], ptr [[TMP6]], align 16
+; VScale2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; VScale2-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VScale2-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; VScale2: [[MIDDLE_BLOCK]]:
+; VScale2-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; VScale2-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; VScale2: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i16>, ptr %arrayidx, align 16
+ %result = call <4 x i16> @llvm.aarch64.neon.sabd.v8i16(<4 x i16> %0, <4 x i16> splat (i16 1))
+ %arrayidx2 = getelementptr inbounds <4 x i16>, ptr %a, i64 %indvars.iv
+ store <4 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @uaddlp(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; VScale1-LABEL: define void @uaddlp(
+; VScale1-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
+; VScale1-NEXT: [[ENTRY:.*:]]
+; VScale1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; VScale1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; VScale1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VScale1: [[VECTOR_PH]]:
+; VScale1-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP0]]
+; VScale1-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; VScale1-NEXT: br label %[[VECTOR_BODY:.*]]
+; VScale1: [[VECTOR_BODY]]:
+; VScale1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VScale1-NEXT: [[TMP2:%.*]] = getelementptr inbounds <4 x i16>, ptr [[B]], i64 [[INDEX]]
+; VScale1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP2]], align 16
+; VScale1-NEXT: [[TMP3:%.*]] = zext <vscale x 4 x i16> [[WIDE_LOAD]] to <vscale x 4 x i32>
+; VScale1-NEXT: [[TMP4:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.addp.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP3]])
+; VScale1-NEXT: [[TMP5:%.*]] = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> [[TMP4]])
+; VScale1-NEXT: [[TMP6:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } [[TMP5]], 0
+; VScale1-NEXT: [[TMP7:%.*]] = getelementptr inbounds <2 x i32>, ptr [[A]], i64 [[INDEX]]
+; VScale1-NEXT: store <vscale x 2 x i32> [[TMP6]], ptr [[TMP7]], align 16
+; VScale1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; VScale1-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VScale1-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; VScale1: [[MIDDLE_BLOCK]]:
+; VScale1-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; VScale1-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; VScale1: [[SCALAR_PH]]:
+;
+; VScale2-LABEL: define void @uaddlp(
+; VScale2-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
+; VScale2-NEXT: [[ENTRY:.*:]]
+; VScale2-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; VScale2-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; VScale2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
+; VScale2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VScale2: [[VECTOR_PH]]:
+; VScale2-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 1
+; VScale2-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
+; VScale2-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; VScale2-NEXT: br label %[[VECTOR_BODY:.*]]
+; VScale2: [[VECTOR_BODY]]:
+; VScale2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VScale2-NEXT: [[TMP4:%.*]] = getelementptr inbounds <4 x i16>, ptr [[B]], i64 [[INDEX]]
+; VScale2-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP4]], align 16
+; VScale2-NEXT: [[TMP5:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.uadalp.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> zeroinitializer, <vscale x 8 x i16> [[WIDE_LOAD]])
+; VScale2-NEXT: [[TMP6:%.*]] = getelementptr inbounds <2 x i32>, ptr [[A]], i64 [[INDEX]]
+; VScale2-NEXT: store <vscale x 4 x i32> [[TMP5]], ptr [[TMP6]], align 16
+; VScale2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; VScale2-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VScale2-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; VScale2: [[MIDDLE_BLOCK]]:
+; VScale2-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; VScale2-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; VScale2: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i16>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <2 x i32> @llvm.aarch64.neon.uaddlp.v2i32.v4i16(<4 x i16> %0)
+ %arrayidx2 = getelementptr inbounds <2 x i32>, ptr %a, i64 %indvars.iv
+ store <2 x i32> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @saddlp(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; VScale1-LABEL: define void @saddlp(
+; VScale1-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
+; VScale1-NEXT: [[ENTRY:.*:]]
+; VScale1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; VScale1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; VScale1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VScale1: [[VECTOR_PH]]:
+; VScale1-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP0]]
+; VScale1-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; VScale1-NEXT: br label %[[VECTOR_BODY:.*]]
+; VScale1: [[VECTOR_BODY]]:
+; VScale1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VScale1-NEXT: [[TMP2:%.*]] = getelementptr inbounds <4 x i16>, ptr [[B]], i64 [[INDEX]]
+; VScale1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP2]], align 16
+; VScale1-NEXT: [[TMP3:%.*]] = sext <vscale x 4 x i16> [[WIDE_LOAD]] to <vscale x 4 x i32>
+; VScale1-NEXT: [[TMP4:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.addp.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP3]])
+; VScale1-NEXT: [[TMP5:%.*]] = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> [[TMP4]])
+; VScale1-NEXT: [[TMP6:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } [[TMP5]], 0
+; VScale1-NEXT: [[TMP7:%.*]] = getelementptr inbounds <2 x i32>, ptr [[A]], i64 [[INDEX]]
+; VScale1-NEXT: store <vscale x 2 x i32> [[TMP6]], ptr [[TMP7]], align 16
+; VScale1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; VScale1-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VScale1-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; VScale1: [[MIDDLE_BLOCK]]:
+; VScale1-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; VScale1-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; VScale1: [[SCALAR_PH]]:
+;
+; VScale2-LABEL: define void @saddlp(
+; VScale2-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
+; VScale2-NEXT: [[ENTRY:.*:]]
+; VScale2-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; VScale2-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; VScale2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
+; VScale2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VScale2: [[VECTOR_PH]]:
+; VScale2-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 1
+; VScale2-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
+; VScale2-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; VScale2-NEXT: br label %[[VECTOR_BODY:.*]]
+; VScale2: [[VECTOR_BODY]]:
+; VScale2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VScale2-NEXT: [[TMP4:%.*]] = getelementptr inbounds <4 x i16>, ptr [[B]], i64 [[INDEX]]
+; VScale2-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP4]], align 16
+; VScale2-NEXT: [[TMP5:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.sadalp.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> zeroinitializer, <vscale x 8 x i16> [[WIDE_LOAD]])
+; VScale2-NEXT: [[TMP6:%.*]] = getelementptr inbounds <2 x i32>, ptr [[A]], i64 [[INDEX]]
+; VScale2-NEXT: store <vscale x 4 x i32> [[TMP5]], ptr [[TMP6]], align 16
+; VScale2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; VScale2-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VScale2-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; VScale2: [[MIDDLE_BLOCK]]:
+; VScale2-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; VScale2-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; VScale2: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i16>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <2 x i32> @llvm.aarch64.neon.saddlp.v2i32.v4i16(<4 x i16> %0)
+ %arrayidx2 = getelementptr inbounds <2 x i32>, ptr %a, i64 %indvars.iv
+ store <2 x i32> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @addp(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; VScale1-LABEL: define void @addp(
+; VScale1-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
+; VScale1-NEXT: [[ENTRY:.*:]]
+; VScale1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; VScale1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; VScale1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VScale1: [[VECTOR_PH]]:
+; VScale1-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP0]]
+; VScale1-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; VScale1-NEXT: br label %[[VECTOR_BODY:.*]]
+; VScale1: [[VECTOR_BODY]]:
+; VScale1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VScale1-NEXT: [[TMP2:%.*]] = getelementptr inbounds <2 x i32>, ptr [[B]], i64 [[INDEX]]
+; VScale1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x i32>, ptr [[TMP2]], align 16
+; VScale1-NEXT: [[TMP3:%.*]] = call <vscale x 2 x i32> @llvm.vector.segmented.shuffle.nxv2i32.v2i32(<vscale x 2 x i32> splat (i32 8), <vscale x 2 x i32> [[WIDE_LOAD]], <2 x i32> <i32 0, i32 2>)
+; VScale1-NEXT: [[TMP4:%.*]] = call <vscale x 2 x i32> @llvm.vector.segmented.shuffle.nxv2i32.v2i32(<vscale x 2 x i32> splat (i32 8), <vscale x 2 x i32> [[WIDE_LOAD]], <2 x i32> <i32 1, i32 3>)
+; VScale1-NEXT: [[FROM_SVE:%.*]] = add <vscale x 2 x i32> [[TMP3]], [[TMP4]]
+; VScale1-NEXT: [[TMP6:%.*]] = getelementptr inbounds <2 x i32>, ptr [[A]], i64 [[INDEX]]
+; VScale1-NEXT: store <vscale x 2 x i32> [[FROM_SVE]], ptr [[TMP6]], align 16
+; VScale1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; VScale1-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VScale1-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; VScale1: [[MIDDLE_BLOCK]]:
+; VScale1-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; VScale1-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; VScale1: [[SCALAR_PH]]:
+;
+; VScale2-LABEL: define void @addp(
+; VScale2-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
+; VScale2-NEXT: [[ENTRY:.*:]]
+; VScale2-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; VScale2-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; VScale2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
+; VScale2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VScale2: [[VECTOR_PH]]:
+; VScale2-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 1
+; VScale2-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
+; VScale2-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; VScale2-NEXT: br label %[[VECTOR_BODY:.*]]
+; VScale2: [[VECTOR_BODY]]:
+; VScale2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VScale2-NEXT: [[TMP4:%.*]] = getelementptr inbounds <2 x i32>, ptr [[B]], i64 [[INDEX]]
+; VScale2-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP4]], align 16
+; VScale2-NEXT: [[TMP5:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.addp.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> splat (i32 8), <vscale x 4 x i32> [[WIDE_LOAD]])
+; VScale2-NEXT: [[TMP8:%.*]] = getelementptr inbounds <2 x i32>, ptr [[A]], i64 [[INDEX]]
+; VScale2-NEXT: store <vscale x 4 x i32> [[TMP5]], ptr [[TMP8]], align 16
+; VScale2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; VScale2-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VScale2-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; VScale2: [[MIDDLE_BLOCK]]:
+; VScale2-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; VScale2-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; VScale2: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <2 x i32>, ptr %b, i64 %indvars.iv
+ %0 = load <2 x i32>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <2 x i32> @llvm.aarch64.neon.addp.v2i32(<2 x i32> splat (i32 8), <2 x i32> %0)
+ %arrayidx2 = getelementptr inbounds <2 x i32>, ptr %a, i64 %indvars.iv
+ store <2 x i32> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @addp_i16(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; VScale1-LABEL: define void @addp_i16(
+; VScale1-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
+; VScale1-NEXT: [[ENTRY:.*:]]
+; VScale1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; VScale1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; VScale1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VScale1: [[VECTOR_PH]]:
+; VScale1-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP0]]
+; VScale1-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; VScale1-NEXT: br label %[[VECTOR_BODY:.*]]
+; VScale1: [[VECTOR_BODY]]:
+; VScale1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VScale1-NEXT: [[TMP2:%.*]] = getelementptr inbounds <4 x i16>, ptr [[B]], i64 [[INDEX]]
+; VScale1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP2]], align 16
+; VScale1-NEXT: [[TMP3:%.*]] = call <vscale x 4 x i16> @llvm.vector.segmented.shuffle.nxv4i16.v4i32(<vscale x 4 x i16> splat (i16 8), <vscale x 4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>)
+; VScale1-NEXT: [[TMP4:%.*]] = call <vscale x 4 x i16> @llvm.vector.segmented.shuffle.nxv4i16.v4i32(<vscale x 4 x i16> splat (i16 8), <vscale x 4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>)
+; VScale1-NEXT: [[FROM_SVE:%.*]] = add <vscale x 4 x i16> [[TMP3]], [[TMP4]]
+; VScale1-NEXT: [[TMP10:%.*]] = getelementptr inbounds <4 x i16>, ptr [[A]], i64 [[INDEX]]
+; VScale1-NEXT: store <vscale x 4 x i16> [[FROM_SVE]], ptr [[TMP10]], align 16
+; VScale1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; VScale1-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VScale1-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; VScale1: [[MIDDLE_BLOCK]]:
+; VScale1-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; VScale1-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; VScale1: [[SCALAR_PH]]:
+;
+; VScale2-LABEL: define void @addp_i16(
+; VScale2-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
+; VScale2-NEXT: [[ENTRY:.*:]]
+; VScale2-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; VScale2-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; VScale2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
+; VScale2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VScale2: [[VECTOR_PH]]:
+; VScale2-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 1
+; VScale2-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
+; VScale2-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; VScale2-NEXT: br label %[[VECTOR_BODY:.*]]
+; VScale2: [[VECTOR_BODY]]:
+; VScale2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VScale2-NEXT: [[TMP4:%.*]] = getelementptr inbounds <4 x i16>, ptr [[B]], i64 [[INDEX]]
+; VScale2-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP4]], align 16
+; VScale2-NEXT: [[TMP5:%.*]] = call <vscale x 8 x i16> @llvm.vector.segmented.shuffle.nxv8i16.v8i32(<vscale x 8 x i16> splat (i16 8), <vscale x 8 x i16> [[WIDE_LOAD]], <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>)
+; VScale2-NEXT: [[TMP6:%.*]] = call <vscale x 8 x i16> @llvm.vector.segmented.shuffle.nxv8i16.v8i32(<vscale x 8 x i16> splat (i16 8), <vscale x 8 x i16> [[WIDE_LOAD]], <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>)
+; VScale2-NEXT: [[TMP11:%.*]] = add <vscale x 8 x i16> [[TMP5]], [[TMP6]]
+; VScale2-NEXT: [[TMP8:%.*]] = bitcast <vscale x 8 x i16> [[TMP11]] to <vscale x 4 x i32>
+; VScale2-NEXT: [[TMP9:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.left.nxv4i32(<vscale x 4 x i32> [[TMP8]], <vscale x 4 x i32> poison, i32 1)
+; VScale2-NEXT: [[TMP10:%.*]] = call <vscale x 4 x i32> @llvm.vector.segmented.shuffle.nxv4i32.v4i32(<vscale x 4 x i32> [[TMP8]], <vscale x 4 x i32> [[TMP9]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>)
+; VScale2-NEXT: [[TMP14:%.*]] = bitcast <vscale x 4 x i32> [[TMP10]] to <vscale x 8 x i16>
+; VScale2-NEXT: [[TMP12:%.*]] = getelementptr inbounds <4 x i16>, ptr [[A]], i64 [[INDEX]]
+; VScale2-NEXT: store <vscale x 8 x i16> [[TMP14]], ptr [[TMP12]], align 16
+; VScale2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; VScale2-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VScale2-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; VScale2: [[MIDDLE_BLOCK]]:
+; VScale2-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; VScale2-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; VScale2: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i16>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <4 x i16> @llvm.aarch64.neon.addp.v4i16(<4 x i16> splat (i16 8), <4 x i16> %0)
+ %arrayidx2 = getelementptr inbounds <4 x i16>, ptr %a, i64 %indvars.iv
+ store <4 x i16> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @addp_f16(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; VScale1-LABEL: define void @addp_f16(
+; VScale1-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
+; VScale1-NEXT: [[ENTRY:.*:]]
+; VScale1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; VScale1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; VScale1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VScale1: [[VECTOR_PH]]:
+; VScale1-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP0]]
+; VScale1-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; VScale1-NEXT: br label %[[VECTOR_BODY:.*]]
+; VScale1: [[VECTOR_BODY]]:
+; VScale1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VScale1-NEXT: [[TMP2:%.*]] = getelementptr inbounds <4 x half>, ptr [[B]], i64 [[INDEX]]
+; VScale1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x half>, ptr [[TMP2]], align 16
+; VScale1-NEXT: [[TMP3:%.*]] = call <vscale x 4 x half> @llvm.vector.segmented.shuffle.nxv4f16.v4i32(<vscale x 4 x half> splat (half 8.000000e+00), <vscale x 4 x half> [[WIDE_LOAD]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>)
+; VScale1-NEXT: [[TMP4:%.*]] = call <vscale x 4 x half> @llvm.vector.segmented.shuffle.nxv4f16.v4i32(<vscale x 4 x half> splat (half 8.000000e+00), <vscale x 4 x half> [[WIDE_LOAD]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>)
+; VScale1-NEXT: [[TMP5:%.*]] = fadd <vscale x 4 x half> [[TMP3]], [[TMP4]]
+; VScale1-NEXT: [[TMP6:%.*]] = getelementptr inbounds <4 x half>, ptr [[A]], i64 [[INDEX]]
+; VScale1-NEXT: store <vscale x 4 x half> [[TMP5]], ptr [[TMP6]], align 16
+; VScale1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; VScale1-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VScale1-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; VScale1: [[MIDDLE_BLOCK]]:
+; VScale1-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; VScale1-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; VScale1: [[SCALAR_PH]]:
+;
+; VScale2-LABEL: define void @addp_f16(
+; VScale2-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
+; VScale2-NEXT: [[ENTRY:.*:]]
+; VScale2-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; VScale2-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; VScale2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
+; VScale2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VScale2: [[VECTOR_PH]]:
+; VScale2-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 1
+; VScale2-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
+; VScale2-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; VScale2-NEXT: br label %[[VECTOR_BODY:.*]]
+; VScale2: [[VECTOR_BODY]]:
+; VScale2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VScale2-NEXT: [[TMP4:%.*]] = getelementptr inbounds <4 x half>, ptr [[B]], i64 [[INDEX]]
+; VScale2-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x half>, ptr [[TMP4]], align 16
+; VScale2-NEXT: [[TMP5:%.*]] = call <vscale x 8 x half> @llvm.vector.segmented.shuffle.nxv8f16.v8i32(<vscale x 8 x half> splat (half 8.000000e+00), <vscale x 8 x half> [[WIDE_LOAD]], <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>)
+; VScale2-NEXT: [[TMP6:%.*]] = call <vscale x 8 x half> @llvm.vector.segmented.shuffle.nxv8f16.v8i32(<vscale x 8 x half> splat (half 8.000000e+00), <vscale x 8 x half> [[WIDE_LOAD]], <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>)
+; VScale2-NEXT: [[TMP7:%.*]] = fadd <vscale x 8 x half> [[TMP5]], [[TMP6]]
+; VScale2-NEXT: [[TMP8:%.*]] = bitcast <vscale x 8 x half> [[TMP7]] to <vscale x 4 x i32>
+; VScale2-NEXT: [[TMP9:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.left.nxv4i32(<vscale x 4 x i32> [[TMP8]], <vscale x 4 x i32> poison, i32 1)
+; VScale2-NEXT: [[TMP10:%.*]] = call <vscale x 4 x i32> @llvm.vector.segmented.shuffle.nxv4i32.v4i32(<vscale x 4 x i32> [[TMP8]], <vscale x 4 x i32> [[TMP9]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>)
+; VScale2-NEXT: [[TMP11:%.*]] = bitcast <vscale x 4 x i32> [[TMP10]] to <vscale x 8 x half>
+; VScale2-NEXT: [[TMP12:%.*]] = getelementptr inbounds <4 x half>, ptr [[A]], i64 [[INDEX]]
+; VScale2-NEXT: store <vscale x 8 x half> [[TMP11]], ptr [[TMP12]], align 16
+; VScale2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; VScale2-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VScale2-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; VScale2: [[MIDDLE_BLOCK]]:
+; VScale2-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; VScale2-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; VScale2: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x half>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x half>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <4 x half> @llvm.aarch64.neon.addp.v4f16(<4 x half> splat (half 8.), <4 x half> %0)
+ %arrayidx2 = getelementptr inbounds <4 x half>, ptr %a, i64 %indvars.iv
+ store <4 x half> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @facge(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; VScale1-LABEL: define void @facge(
+; VScale1-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
+; VScale1-NEXT: [[ENTRY:.*:]]
+; VScale1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; VScale1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; VScale1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VScale1: [[VECTOR_PH]]:
+; VScale1-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP0]]
+; VScale1-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; VScale1-NEXT: br label %[[VECTOR_BODY:.*]]
+; VScale1: [[VECTOR_BODY]]:
+; VScale1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VScale1-NEXT: [[TMP2:%.*]] = getelementptr inbounds <2 x float>, ptr [[B]], i64 [[INDEX]]
+; VScale1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x float>, ptr [[TMP2]], align 16
+; VScale1-NEXT: [[TO_SVE:%.*]] = call <vscale x 4 x float> @llvm.vector.interleave2.nxv4f32(<vscale x 2 x float> [[WIDE_LOAD]], <vscale x 2 x float> poison)
+; VScale1-NEXT: [[TO_SVE1:%.*]] = call <vscale x 4 x float> @llvm.vector.interleave2.nxv4f32(<vscale x 2 x float> splat (float 1.000000e+00), <vscale x 2 x float> poison)
+; VScale1-NEXT: [[TMP3:%.*]] = call <vscale x 4 x i1> @llvm.aarch64.sve.facge.nxv4f32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x float> [[TO_SVE]], <vscale x 4 x float> [[TO_SVE1]])
+; VScale1-NEXT: [[TMP4:%.*]] = zext <vscale x 4 x i1> [[TMP3]] to <vscale x 4 x i32>
+; VScale1-NEXT: [[TMP7:%.*]] = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> [[TMP4]])
+; VScale1-NEXT: [[FROM_SVE:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } [[TMP7]], 0
+; VScale1-NEXT: [[TMP5:%.*]] = getelementptr inbounds <2 x i32>, ptr [[A]], i64 [[INDEX]]
+; VScale1-NEXT: store <vscale x 2 x i32> [[FROM_SVE]], ptr [[TMP5]], align 16
+; VScale1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; VScale1-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VScale1-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; VScale1: [[MIDDLE_BLOCK]]:
+; VScale1-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; VScale1-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; VScale1: [[SCALAR_PH]]:
+;
+; VScale2-LABEL: define void @facge(
+; VScale2-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
+; VScale2-NEXT: [[ENTRY:.*:]]
+; VScale2-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; VScale2-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; VScale2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
+; VScale2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VScale2: [[VECTOR_PH]]:
+; VScale2-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 1
+; VScale2-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
+; VScale2-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; VScale2-NEXT: br label %[[VECTOR_BODY:.*]]
+; VScale2: [[VECTOR_BODY]]:
+; VScale2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VScale2-NEXT: [[TMP4:%.*]] = getelementptr inbounds <2 x float>, ptr [[B]], i64 [[INDEX]]
+; VScale2-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP4]], align 16
+; VScale2-NEXT: [[TMP5:%.*]] = call <vscale x 4 x i1> @llvm.aarch64.sve.facge.nxv4f32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x float> splat (float 1.000000e+00))
+; VScale2-NEXT: [[TMP6:%.*]] = zext <vscale x 4 x i1> [[TMP5]] to <vscale x 4 x i32>
+; VScale2-NEXT: [[TMP7:%.*]] = getelementptr inbounds <2 x i32>, ptr [[A]], i64 [[INDEX]]
+; VScale2-NEXT: store <vscale x 4 x i32> [[TMP6]], ptr [[TMP7]], align 16
+; VScale2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; VScale2-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VScale2-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; VScale2: [[MIDDLE_BLOCK]]:
+; VScale2-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; VScale2-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; VScale2: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <2 x float>, ptr %b, i64 %indvars.iv
+ %0 = load <2 x float>, ptr %arrayidx, align 16
+ %result = call <2 x i32> @llvm.aarch64.neon.facge.v2i32.v2f32(<2 x float> %0, <2 x float> splat (float 1.0))
+ %arrayidx2 = getelementptr inbounds <2 x i32>, ptr %a, i64 %indvars.iv
+ store <2 x i32> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+
+define void @fcvtzs(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; VScale1-LABEL: define void @fcvtzs(
+; VScale1-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
+; VScale1-NEXT: [[ENTRY:.*:]]
+; VScale1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; VScale1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; VScale1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VScale1: [[VECTOR_PH]]:
+; VScale1-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP0]]
+; VScale1-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; VScale1-NEXT: br label %[[VECTOR_BODY:.*]]
+; VScale1: [[VECTOR_BODY]]:
+; VScale1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VScale1-NEXT: [[TMP2:%.*]] = getelementptr inbounds <4 x half>, ptr [[B]], i64 [[INDEX]]
+; VScale1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x half>, ptr [[TMP2]], align 16
+; VScale1-NEXT: [[TO_SVE:%.*]] = call <vscale x 8 x half> @llvm.vector.interleave2.nxv8f16(<vscale x 4 x half> [[WIDE_LOAD]], <vscale x 4 x half> poison)
+; VScale1-NEXT: [[TMP3:%.*]] = call <vscale x 8 x i16> @llvm.aarch64.sve.fcvtzs.nxv8i16.nxv8f16(<vscale x 8 x i16> poison, <vscale x 8 x i1> splat (i1 true), <vscale x 8 x half> [[TO_SVE]])
+; VScale1-NEXT: [[TMP6:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> [[TMP3]])
+; VScale1-NEXT: [[FROM_SVE:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[TMP6]], 0
+; VScale1-NEXT: [[TMP4:%.*]] = getelementptr inbounds <4 x i16>, ptr [[A]], i64 [[INDEX]]
+; VScale1-NEXT: store <vscale x 4 x i16> [[FROM_SVE]], ptr [[TMP4]], align 16
+; VScale1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; VScale1-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VScale1-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; VScale1: [[MIDDLE_BLOCK]]:
+; VScale1-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; VScale1-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; VScale1: [[SCALAR_PH]]:
+;
+; VScale2-LABEL: define void @fcvtzs(
+; VScale2-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
+; VScale2-NEXT: [[ENTRY:.*:]]
+; VScale2-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; VScale2-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; VScale2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
+; VScale2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VScale2: [[VECTOR_PH]]:
+; VScale2-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 1
+; VScale2-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
+; VScale2-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; VScale2-NEXT: br label %[[VECTOR_BODY:.*]]
+; VScale2: [[VECTOR_BODY]]:
+; VScale2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VScale2-NEXT: [[TMP4:%.*]] = getelementptr inbounds <4 x half>, ptr [[B]], i64 [[INDEX]]
+; VScale2-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x half>, ptr [[TMP4]], align 16
+; VScale2-NEXT: [[TMP5:%.*]] = call <vscale x 8 x i16> @llvm.aarch64.sve.fcvtzs.nxv8i16.nxv8f16(<vscale x 8 x i16> poison, <vscale x 8 x i1> splat (i1 true), <vscale x 8 x half> [[WIDE_LOAD]])
+; VScale2-NEXT: [[TMP6:%.*]] = getelementptr inbounds <4 x i16>, ptr [[A]], i64 [[INDEX]]
+; VScale2-NEXT: store <vscale x 8 x i16> [[TMP5]], ptr [[TMP6]], align 16
+; VScale2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; VScale2-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VScale2-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; VScale2: [[MIDDLE_BLOCK]]:
+; VScale2-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; VScale2-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; VScale2: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x half>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x half>, ptr %arrayidx, align 16
+ %result = call <4 x i16> @llvm.aarch64.neon.fcvtzs.v4i16.v4f16(<4 x half> %0)
+ %arrayidx2 = getelementptr inbounds <4 x i16>, ptr %a, i64 %indvars.iv
+ store <4 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @sshl(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; VScale1-LABEL: define void @sshl(
+; VScale1-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
+; VScale1-NEXT: [[ENTRY:.*:]]
+; VScale1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; VScale1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; VScale1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VScale1: [[VECTOR_PH]]:
+; VScale1-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP0]]
+; VScale1-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; VScale1-NEXT: br label %[[VECTOR_BODY:.*]]
+; VScale1: [[VECTOR_BODY]]:
+; VScale1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VScale1-NEXT: [[TMP2:%.*]] = getelementptr inbounds <2 x i32>, ptr [[B]], i64 [[INDEX]]
+; VScale1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x i32>, ptr [[TMP2]], align 16
+; VScale1-NEXT: [[TO_SVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.interleave2.nxv4i32(<vscale x 2 x i32> splat (i32 8), <vscale x 2 x i32> poison)
+; VScale1-NEXT: [[TO_SVE1:%.*]] = call <vscale x 4 x i32> @llvm.vector.interleave2.nxv4i32(<vscale x 2 x i32> [[WIDE_LOAD]], <vscale x 2 x i32> poison)
+; VScale1-NEXT: [[SHL_MASK:%.*]] = icmp sle <vscale x 4 x i32> [[TO_SVE1]], zeroinitializer
+; VScale1-NEXT: [[TMP3:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.srshl.nxv4i32(<vscale x 4 x i1> [[SHL_MASK]], <vscale x 4 x i32> [[TO_SVE]], <vscale x 4 x i32> [[TO_SVE1]])
+; VScale1-NEXT: [[SHR_MASK:%.*]] = call <vscale x 4 x i1> @llvm.ctlz.nxv4i1(<vscale x 4 x i1> [[SHL_MASK]], i1 true)
+; VScale1-NEXT: [[WIDE_BIDIR_SHL1:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.sqshl.nxv4i32(<vscale x 4 x i1> [[SHR_MASK]], <vscale x 4 x i32> [[TMP3]], <vscale x 4 x i32> [[TO_SVE1]])
+; VScale1-NEXT: [[TMP6:%.*]] = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> [[WIDE_BIDIR_SHL1]])
+; VScale1-NEXT: [[WIDE_BIDIR_SHL:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } [[TMP6]], 0
+; VScale1-NEXT: [[TMP4:%.*]] = getelementptr inbounds <2 x i32>, ptr [[A]], i64 [[INDEX]]
+; VScale1-NEXT: store <vscale x 2 x i32> [[WIDE_BIDIR_SHL]], ptr [[TMP4]], align 16
+; VScale1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; VScale1-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VScale1-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; VScale1: [[MIDDLE_BLOCK]]:
+; VScale1-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; VScale1-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; VScale1: [[SCALAR_PH]]:
+;
+; VScale2-LABEL: define void @sshl(
+; VScale2-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
+; VScale2-NEXT: [[ENTRY:.*:]]
+; VScale2-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; VScale2-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; VScale2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
+; VScale2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VScale2: [[VECTOR_PH]]:
+; VScale2-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 1
+; VScale2-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
+; VScale2-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; VScale2-NEXT: br label %[[VECTOR_BODY:.*]]
+; VScale2: [[VECTOR_BODY]]:
+; VScale2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VScale2-NEXT: [[TMP4:%.*]] = getelementptr inbounds <2 x i32>, ptr [[B]], i64 [[INDEX]]
+; VScale2-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP4]], align 16
+; VScale2-NEXT: [[SHL_MASK:%.*]] = icmp sle <vscale x 4 x i32> [[WIDE_LOAD]], zeroinitializer
+; VScale2-NEXT: [[TMP5:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.srshl.nxv4i32(<vscale x 4 x i1> [[SHL_MASK]], <vscale x 4 x i32> splat (i32 8), <vscale x 4 x i32> [[WIDE_LOAD]])
+; VScale2-NEXT: [[SHR_MASK:%.*]] = call <vscale x 4 x i1> @llvm.ctlz.nxv4i1(<vscale x 4 x i1> [[SHL_MASK]], i1 true)
+; VScale2-NEXT: [[WIDE_BIDIR_SHL:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.sqshl.nxv4i32(<vscale x 4 x i1> [[SHR_MASK]], <vscale x 4 x i32> [[TMP5]], <vscale x 4 x i32> [[WIDE_LOAD]])
+; VScale2-NEXT: [[TMP6:%.*]] = getelementptr inbounds <2 x i32>, ptr [[A]], i64 [[INDEX]]
+; VScale2-NEXT: store <vscale x 4 x i32> [[WIDE_BIDIR_SHL]], ptr [[TMP6]], align 16
+; VScale2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; VScale2-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VScale2-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; VScale2: [[MIDDLE_BLOCK]]:
+; VScale2-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; VScale2-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; VScale2: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <2 x i32>, ptr %b, i64 %indvars.iv
+ %0 = load <2 x i32>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <2 x i32> @llvm.aarch64.neon.sshl.v4i32(<2 x i32> splat (i32 8), <2 x i32> %0)
+ %arrayidx2 = getelementptr inbounds <2 x i32>, ptr %a, i64 %indvars.iv
+ store <2 x i32> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @vsli(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; VScale1-LABEL: define void @vsli(
+; VScale1-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
+; VScale1-NEXT: [[ENTRY:.*:]]
+; VScale1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; VScale1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; VScale1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VScale1: [[VECTOR_PH]]:
+; VScale1-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP0]]
+; VScale1-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; VScale1-NEXT: br label %[[VECTOR_BODY:.*]]
+; VScale1: [[VECTOR_BODY]]:
+; VScale1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VScale1-NEXT: [[TMP2:%.*]] = getelementptr inbounds <2 x i32>, ptr [[B]], i64 [[INDEX]]
+; VScale1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x i32>, ptr [[TMP2]], align 16
+; VScale1-NEXT: [[TO_SVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.interleave2.nxv4i32(<vscale x 2 x i32> splat (i32 1), <vscale x 2 x i32> poison)
+; VScale1-NEXT: [[TO_SVE1:%.*]] = call <vscale x 4 x i32> @llvm.vector.interleave2.nxv4i32(<vscale x 2 x i32> [[WIDE_LOAD]], <vscale x 2 x i32> poison)
+; VScale1-NEXT: [[TMP3:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.sli.nxv4i32(<vscale x 4 x i32> [[TO_SVE]], <vscale x 4 x i32> [[TO_SVE1]], i32 2)
+; VScale1-NEXT: [[TMP6:%.*]] = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> [[TMP3]])
+; VScale1-NEXT: [[FROM_SVE:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } [[TMP6]], 0
+; VScale1-NEXT: [[TMP4:%.*]] = getelementptr inbounds <2 x i32>, ptr [[A]], i64 [[INDEX]]
+; VScale1-NEXT: store <vscale x 2 x i32> [[FROM_SVE]], ptr [[TMP4]], align 16
+; VScale1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; VScale1-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VScale1-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; VScale1: [[MIDDLE_BLOCK]]:
+; VScale1-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; VScale1-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; VScale1: [[SCALAR_PH]]:
+;
+; VScale2-LABEL: define void @vsli(
+; VScale2-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
+; VScale2-NEXT: [[ENTRY:.*:]]
+; VScale2-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; VScale2-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; VScale2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
+; VScale2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VScale2: [[VECTOR_PH]]:
+; VScale2-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 1
+; VScale2-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
+; VScale2-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; VScale2-NEXT: br label %[[VECTOR_BODY:.*]]
+; VScale2: [[VECTOR_BODY]]:
+; VScale2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VScale2-NEXT: [[TMP4:%.*]] = getelementptr inbounds <2 x i32>, ptr [[B]], i64 [[INDEX]]
+; VScale2-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP4]], align 16
+; VScale2-NEXT: [[TMP5:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.sli.nxv4i32(<vscale x 4 x i32> splat (i32 1), <vscale x 4 x i32> [[WIDE_LOAD]], i32 2)
+; VScale2-NEXT: [[TMP6:%.*]] = getelementptr inbounds <2 x i32>, ptr [[A]], i64 [[INDEX]]
+; VScale2-NEXT: store <vscale x 4 x i32> [[TMP5]], ptr [[TMP6]], align 16
+; VScale2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; VScale2-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VScale2-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; VScale2: [[MIDDLE_BLOCK]]:
+; VScale2-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; VScale2-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; VScale2: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <2 x i32>, ptr %b, i64 %indvars.iv
+ %0 = load <2 x i32>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <2 x i32> @llvm.aarch64.neon.vsli.v4i32(<2 x i32> splat (i32 1), <2 x i32> %0, i32 2)
+ %arrayidx2 = getelementptr inbounds <2 x i32>, ptr %a, i64 %indvars.iv
+ store <2 x i32> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @smull(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @smull(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP0]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <2 x i32>, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x i32>, ptr [[TMP1]], align 16
+; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 4 x i32> @llvm.vector.interleave2.nxv4i32(<vscale x 2 x i32> splat (i32 8), <vscale x 2 x i32> poison)
+; CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 4 x i32> @llvm.vector.interleave2.nxv4i32(<vscale x 2 x i32> [[WIDE_LOAD]], <vscale x 2 x i32> poison)
+; CHECK-NEXT: [[TMP4:%.*]] = call <vscale x 2 x i64> @llvm.aarch64.sve.smullb.nxv2i64(<vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]])
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds <2 x i64>, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: store <vscale x 2 x i64> [[TMP4]], ptr [[TMP5]], align 16
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <2 x i32>, ptr %b, i64 %indvars.iv
+ %0 = load <2 x i32>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> splat (i32 8), <2 x i32> %0)
+ %arrayidx2 = getelementptr inbounds <2 x i64>, ptr %a, i64 %indvars.iv
+ store <2 x i64> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @umull(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @umull(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP0]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <2 x i32>, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x i32>, ptr [[TMP1]], align 16
+; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 4 x i32> @llvm.vector.interleave2.nxv4i32(<vscale x 2 x i32> splat (i32 8), <vscale x 2 x i32> poison)
+; CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 4 x i32> @llvm.vector.interleave2.nxv4i32(<vscale x 2 x i32> [[WIDE_LOAD]], <vscale x 2 x i32> poison)
+; CHECK-NEXT: [[TMP4:%.*]] = call <vscale x 2 x i64> @llvm.aarch64.sve.umullb.nxv2i64(<vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]])
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds <2 x i64>, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: store <vscale x 2 x i64> [[TMP4]], ptr [[TMP5]], align 16
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <2 x i32>, ptr %b, i64 %indvars.iv
+ %0 = load <2 x i32>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> splat (i32 8), <2 x i32> %0)
+ %arrayidx2 = getelementptr inbounds <2 x i64>, ptr %a, i64 %indvars.iv
+ store <2 x i64> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @tbl1(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b, <8 x i8> %mask) {
+; CHECK-LABEL: define void @tbl1(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], <8 x i8> [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP0]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT: [[BROADCAST:%.*]] = call <vscale x 8 x i8> @llvm.vector.broadcast.nxv8i8.v8i8(<8 x i8> [[MASK]])
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <16 x i8>, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP1]], align 16
+; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 16 x i8> @llvm.vector.interleave2.nxv16i8(<vscale x 8 x i8> [[BROADCAST]], <vscale x 8 x i8> poison)
+; CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 16 x i8> @llvm.aarch64.sve.tblq.nxv16i8(<vscale x 16 x i8> [[WIDE_LOAD]], <vscale x 16 x i8> [[TMP2]])
+; CHECK-NEXT: [[TMP4:%.*]] = call { <vscale x 8 x i8>, <vscale x 8 x i8> } @llvm.vector.deinterleave2.nxv16i8(<vscale x 16 x i8> [[TMP3]])
+; CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <vscale x 8 x i8>, <vscale x 8 x i8> } [[TMP4]], 0
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds <8 x i8>, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: store <vscale x 8 x i8> [[TMP5]], ptr [[TMP6]], align 8
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <16 x i8>, ptr %b, i64 %indvars.iv
+ %0 = load <16 x i8>, ptr %arrayidx, align 16
+ %res = tail call <8 x i8> @llvm.aarch64.neon.tbl1.v8i8(<16 x i8> %0, <8 x i8> %mask)
+ %arrayidx2 = getelementptr inbounds <8 x i8>, ptr %a, i64 %indvars.iv
+ store <8 x i8> %res, ptr %arrayidx2, align 8
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @tbl2(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b, <8 x i8> %mask) {
+; CHECK-LABEL: define void @tbl2(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], <8 x i8> [[MASK:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP0]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT: [[BROADCAST:%.*]] = call <vscale x 8 x i8> @llvm.vector.broadcast.nxv8i8.v8i8(<8 x i8> [[MASK]])
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <16 x i8>, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP1]], align 16
+; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 16 x i8> @llvm.vector.interleave2.nxv16i8(<vscale x 8 x i8> [[BROADCAST]], <vscale x 8 x i8> poison)
+; CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 16 x i8> @llvm.aarch64.sve.tblq.nxv16i8(<vscale x 16 x i8> [[WIDE_LOAD]], <vscale x 16 x i8> [[TMP2]])
+; CHECK-NEXT: [[TMP4:%.*]] = sub <vscale x 16 x i8> [[TMP2]], splat (i8 16)
+; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 16 x i8> @llvm.aarch64.sve.tbxq.nxv16i8(<vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> splat (i8 3), <vscale x 16 x i8> [[TMP4]])
+; CHECK-NEXT: [[TMP6:%.*]] = call { <vscale x 8 x i8>, <vscale x 8 x i8> } @llvm.vector.deinterleave2.nxv16i8(<vscale x 16 x i8> [[TMP5]])
+; CHECK-NEXT: [[TMP7:%.*]] = extractvalue { <vscale x 8 x i8>, <vscale x 8 x i8> } [[TMP6]], 0
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds <8 x i8>, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: store <vscale x 8 x i8> [[TMP7]], ptr [[TMP8]], align 8
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <16 x i8>, ptr %b, i64 %indvars.iv
+ %0 = load <16 x i8>, ptr %arrayidx, align 16
+ %res = tail call <8 x i8> @llvm.aarch64.neon.tbl2.v8i8(<16 x i8> %0, <16 x i8> splat (i8 3), <8 x i8> %mask)
+ %arrayidx2 = getelementptr inbounds <8 x i8>, ptr %a, i64 %indvars.iv
+ store <8 x i8> %res, ptr %arrayidx2, align 8
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-neon-extract-narrow.ll b/llvm/test/Transforms/LoopVectorize/AArch64/revec-neon-extract-narrow.ll
new file mode 100644
index 0000000000000..ae98d0fcfc6e9
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/revec-neon-extract-narrow.ll
@@ -0,0 +1,105 @@
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -vectorize-vector-loops \
+; RUN: -mtriple=aarch64 -mattr=+sve2p1 -S < %s | FileCheck %s
+
+; Test how NEON intrinsic are re-vectorised using HVLA.
+
+
+define void @sqxtn(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @sqxtn(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 8 x i16> @llvm.aarch64.sve.sqxtnb.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]])
+; CHECK: [[TMP4:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> [[WIDE_OP]])
+; CHECK: [[TMP5:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[TMP4]], 0
+; CHECK: store <vscale x 4 x i16> [[TMP5]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i32>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i32>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <4 x i16> @llvm.aarch64.neon.sqxtn.v4i16(<4 x i32> %0)
+ %arrayidx2 = getelementptr inbounds <4 x i16>, ptr %a, i64 %indvars.iv
+ store <4 x i16> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @sqxtun(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @sqxtun(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 8 x i16> @llvm.aarch64.sve.sqxtunb.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]])
+; CHECK: [[TMP4:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> [[WIDE_OP]])
+; CHECK: [[TMP5:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[TMP4]], 0
+; CHECK: store <vscale x 4 x i16> [[TMP5]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i32>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i32>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <4 x i16> @llvm.aarch64.neon.sqxtun.v4i16(<4 x i32> %0)
+ %arrayidx2 = getelementptr inbounds <4 x i16>, ptr %a, i64 %indvars.iv
+ store <4 x i16> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @uqxtn(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @uqxtn(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 8 x i16> @llvm.aarch64.sve.uqxtnb.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]])
+; CHECK: [[TMP4:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> [[WIDE_OP]])
+; CHECK: [[TMP5:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[TMP4]], 0
+; CHECK: store <vscale x 4 x i16> [[TMP5]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i32>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i32>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <4 x i16> @llvm.aarch64.neon.uqxtn.v4i16(<4 x i32> %0)
+ %arrayidx2 = getelementptr inbounds <4 x i16>, ptr %a, i64 %indvars.iv
+ store <4 x i16> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @fcvtxn(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @fcvtxn(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 2 x double>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 4 x float> @llvm.aarch64.sve.fcvtx.f32f64(<vscale x 4 x float> poison, <vscale x 2 x i1> splat (i1 true), <vscale x 2 x double> [[WIDE_LOAD]])
+; CHECK: [[TMP4:%.*]] = call { <vscale x 2 x float>, <vscale x 2 x float> } @llvm.vector.deinterleave2.nxv4f32(<vscale x 4 x float> [[WIDE_OP]])
+; CHECK: [[TMP5:%.*]] = extractvalue { <vscale x 2 x float>, <vscale x 2 x float> } [[TMP4]], 0
+; CHECK: store <vscale x 2 x float> [[TMP5]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <2 x double>, ptr %b, i64 %indvars.iv
+ %0 = load <2 x double>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <2 x float> @llvm.aarch64.neon.fcvtxn.v2f64(<2 x double> %0)
+ %arrayidx2 = getelementptr inbounds <2 x float>, ptr %a, i64 %indvars.iv
+ store <2 x float> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-neon-force-vf.ll b/llvm/test/Transforms/LoopVectorize/AArch64/revec-neon-force-vf.ll
new file mode 100644
index 0000000000000..fe33e77e1e239
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/revec-neon-force-vf.ll
@@ -0,0 +1,30 @@
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -vectorize-vector-loops \
+; RUN: -scalable-vectorization=on -force-vector-width=1 -mtriple=aarch64 -mattr=+sve2p1 \
+; RUN: -S < %s | FileCheck %s
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -vectorize-vector-loops \
+; RUN: -scalable-vectorization=on -force-vector-width=2 -mtriple=aarch64 -mattr=+sve2p1 \
+; RUN: -S < %s | FileCheck %s
+
+; Test that 128-bit NEON intrinsics cannot get re-vectorised with VF = vscale x 2
+
+define void @uabd(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @uabd(
+; CHECK: vector.body
+; CHECK: load <vscale x 8 x i16>
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <8 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x i16>, ptr %arrayidx, align 16
+ %result = call <8 x i16> @llvm.aarch64.neon.uabd.v8i16(<8 x i16> %0, <8 x i16> splat (i16 1))
+ %arrayidx2 = getelementptr inbounds <8 x i16>, ptr %a, i64 %indvars.iv
+ store <8 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-neon-mmla.ll b/llvm/test/Transforms/LoopVectorize/AArch64/revec-neon-mmla.ll
new file mode 100644
index 0000000000000..1fa8071946b20
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/revec-neon-mmla.ll
@@ -0,0 +1,74 @@
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -vectorize-vector-loops \
+; RUN: -mtriple=aarch64 -mattr=+sve2p1,+i8mm -S < %s | FileCheck %s
+
+; Test how NEON intrinsic are re-vectorised using HVLA.
+
+
+define void @smmla(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @smmla(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.smmla.nxv4i32(<vscale x 4 x i32> splat (i32 8), <vscale x 16 x i8> [[WIDE_LOAD]], <vscale x 16 x i8> [[WIDE_LOAD]])
+; CHECK: store <vscale x 4 x i32> [[WIDE_OP]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <16 x i8>, ptr %b, i64 %indvars.iv
+ %0 = load <16 x i8>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <4 x i32> @llvm.aarch64.neon.smmla.v4i32.v16i8(<4 x i32> splat (i32 8), <16 x i8> %0, <16 x i8> %0)
+ %arrayidx2 = getelementptr inbounds <4 x i32>, ptr %a, i64 %indvars.iv
+ store <4 x i32> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @ummla(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @ummla(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.ummla.nxv4i32(<vscale x 4 x i32> splat (i32 8), <vscale x 16 x i8> [[WIDE_LOAD]], <vscale x 16 x i8> [[WIDE_LOAD]])
+; CHECK: store <vscale x 4 x i32> [[WIDE_OP]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <16 x i8>, ptr %b, i64 %indvars.iv
+ %0 = load <16 x i8>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <4 x i32> @llvm.aarch64.neon.ummla.v4i32.v16i8(<4 x i32> splat (i32 8), <16 x i8> %0, <16 x i8> %0)
+ %arrayidx2 = getelementptr inbounds <4 x i32>, ptr %a, i64 %indvars.iv
+ store <4 x i32> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @usmmla(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @usmmla(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.usmmla.nxv4i32(<vscale x 4 x i32> splat (i32 8), <vscale x 16 x i8> [[WIDE_LOAD]], <vscale x 16 x i8> [[WIDE_LOAD]])
+; CHECK: store <vscale x 4 x i32> [[WIDE_OP]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <16 x i8>, ptr %b, i64 %indvars.iv
+ %0 = load <16 x i8>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <4 x i32> @llvm.aarch64.neon.usmmla.v4i32.v16i8(<4 x i32> splat (i32 8), <16 x i8> %0, <16 x i8> %0)
+ %arrayidx2 = getelementptr inbounds <4 x i32>, ptr %a, i64 %indvars.iv
+ store <4 x i32> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-neon-shr-narrow.ll b/llvm/test/Transforms/LoopVectorize/AArch64/revec-neon-shr-narrow.ll
new file mode 100644
index 0000000000000..c322d2e62b083
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/revec-neon-shr-narrow.ll
@@ -0,0 +1,180 @@
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -vectorize-vector-loops \
+; RUN: -mtriple=aarch64 -mattr=+sve2p1 -S < %s | FileCheck %s
+
+; Test how NEON intrinsic are re-vectorised using HVLA.
+
+
+define void @rshrn(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @rshrn(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>
+; CHECK: [[TMP3:%.*]] = call <vscale x 8 x i16> @llvm.aarch64.sve.rshrnb.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]], i32 1)
+; CHECK: [[TMP4:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> [[TMP3]])
+; CHECK: [[TMP5:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[TMP4]], 0
+; CHECK: store <vscale x 4 x i16> [[TMP5]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i32>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i32>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <4 x i16> @llvm.aarch64.neon.rshrn.v4i16(<4 x i32> %0, i32 1)
+ %arrayidx2 = getelementptr inbounds <4 x i16>, ptr %a, i64 %indvars.iv
+ store <4 x i16> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @sqrshrn(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @sqrshrn(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>
+; CHECK: [[TMP3:%.*]] = call <vscale x 8 x i16> @llvm.aarch64.sve.sqrshrnb.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]], i32 1)
+; CHECK: [[TMP4:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> [[TMP3]])
+; CHECK: [[TMP5:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[TMP4]], 0
+; CHECK: store <vscale x 4 x i16> [[TMP5]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i32>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i32>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <4 x i16> @llvm.aarch64.neon.sqrshrn.v4i16(<4 x i32> %0, i32 1)
+ %arrayidx2 = getelementptr inbounds <4 x i16>, ptr %a, i64 %indvars.iv
+ store <4 x i16> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @sqrshrun(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @sqrshrun(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>
+; CHECK: [[TMP3:%.*]] = call <vscale x 8 x i16> @llvm.aarch64.sve.sqrshrunb.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]], i32 1)
+; CHECK: [[TMP4:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> [[TMP3]])
+; CHECK: [[TMP5:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[TMP4]], 0
+; CHECK: store <vscale x 4 x i16> [[TMP5]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i32>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i32>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <4 x i16> @llvm.aarch64.neon.sqrshrun.v4i16(<4 x i32> %0, i32 1)
+ %arrayidx2 = getelementptr inbounds <4 x i16>, ptr %a, i64 %indvars.iv
+ store <4 x i16> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @sqshrn(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @sqshrn(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>
+; CHECK: [[TMP3:%.*]] = call <vscale x 8 x i16> @llvm.aarch64.sve.sqshrnb.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]], i32 1)
+; CHECK: [[TMP4:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> [[TMP3]])
+; CHECK: [[TMP5:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[TMP4]], 0
+; CHECK: store <vscale x 4 x i16> [[TMP5]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i32>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i32>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <4 x i16> @llvm.aarch64.neon.sqshrn.v4i16(<4 x i32> %0, i32 1)
+ %arrayidx2 = getelementptr inbounds <4 x i16>, ptr %a, i64 %indvars.iv
+ store <4 x i16> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @sqshrun(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @sqshrun(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>
+; CHECK: [[TMP3:%.*]] = call <vscale x 8 x i16> @llvm.aarch64.sve.sqshrunb.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]], i32 1)
+; CHECK: [[TMP4:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> [[TMP3]])
+; CHECK: [[TMP5:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[TMP4]], 0
+; CHECK: store <vscale x 4 x i16> [[TMP5]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i32>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i32>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <4 x i16> @llvm.aarch64.neon.sqshrun.v4i16(<4 x i32> %0, i32 1)
+ %arrayidx2 = getelementptr inbounds <4 x i16>, ptr %a, i64 %indvars.iv
+ store <4 x i16> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @uqrshrn(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @uqrshrn(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>
+; CHECK: [[TMP3:%.*]] = call <vscale x 8 x i16> @llvm.aarch64.sve.uqrshrnb.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]], i32 1)
+; CHECK: [[TMP4:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> [[TMP3]])
+; CHECK: [[TMP5:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[TMP4]], 0
+; CHECK: store <vscale x 4 x i16> [[TMP5]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i32>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i32>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <4 x i16> @llvm.aarch64.neon.uqrshrn.v4i16(<4 x i32> %0, i32 1)
+ %arrayidx2 = getelementptr inbounds <4 x i16>, ptr %a, i64 %indvars.iv
+ store <4 x i16> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @uqshrn(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @uqshrn(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>
+; CHECK: [[TMP3:%.*]] = call <vscale x 8 x i16> @llvm.aarch64.sve.uqshrnb.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]], i32 1)
+; CHECK: [[TMP4:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> [[TMP3]])
+; CHECK: [[TMP5:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[TMP4]], 0
+; CHECK: store <vscale x 4 x i16> [[TMP5]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i32>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i32>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <4 x i16> @llvm.aarch64.neon.uqshrn.v4i16(<4 x i32> %0, i32 1)
+ %arrayidx2 = getelementptr inbounds <4 x i16>, ptr %a, i64 %indvars.iv
+ store <4 x i16> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-neon-usrhadd.ll b/llvm/test/Transforms/LoopVectorize/AArch64/revec-neon-usrhadd.ll
new file mode 100644
index 0000000000000..1cd5940bb9f73
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/revec-neon-usrhadd.ll
@@ -0,0 +1,60 @@
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -vectorize-vector-loops -mtriple=aarch64 -mattr=+sve2p1 -S < %s | FileCheck %s
+
+; Test that NEON rounding halving add intrinsics (urhadd/srhadd) can be
+; successfully re-vectorized to SVE.
+
+define void @urhadd_v16i8(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b, ptr nocapture noundef readonly %c) {
+; CHECK-LABEL: define void @urhadd_v16i8(
+; CHECK: [[WIDE_LOAD_B:%.*]] = load <vscale x 16 x i8>
+; CHECK: [[WIDE_LOAD_C:%.*]] = load <vscale x 16 x i8>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 16 x i8> @llvm.aarch64.sve.urhadd.nxv16i8(<vscale x 16 x i1> splat (i1 true), <vscale x 16 x i8> [[WIDE_LOAD_B]], <vscale x 16 x i8> [[WIDE_LOAD_C]])
+; CHECK: store <vscale x 16 x i8> [[WIDE_OP]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <16 x i8>, ptr %b, i64 %indvars.iv
+ %0 = load <16 x i8>, ptr %arrayidx, align 16
+ %arrayidx2 = getelementptr inbounds <16 x i8>, ptr %c, i64 %indvars.iv
+ %1 = load <16 x i8>, ptr %arrayidx2, align 16
+ %result = call <16 x i8> @llvm.aarch64.neon.urhadd.v16i8(<16 x i8> %0, <16 x i8> %1)
+ %arrayidx3 = getelementptr inbounds <16 x i8>, ptr %a, i64 %indvars.iv
+ store <16 x i8> %result, ptr %arrayidx3, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+declare <16 x i8> @llvm.aarch64.neon.urhadd.v16i8(<16 x i8>, <16 x i8>)
+
+define void @srhadd_v16i8(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b, ptr nocapture noundef readonly %c) {
+; CHECK-LABEL: define void @srhadd_v16i8(
+; CHECK: [[WIDE_LOAD_B:%.*]] = load <vscale x 16 x i8>
+; CHECK: [[WIDE_LOAD_C:%.*]] = load <vscale x 16 x i8>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 16 x i8> @llvm.aarch64.sve.srhadd.nxv16i8(<vscale x 16 x i1> splat (i1 true), <vscale x 16 x i8> [[WIDE_LOAD_B]], <vscale x 16 x i8> [[WIDE_LOAD_C]])
+; CHECK: store <vscale x 16 x i8> [[WIDE_OP]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <16 x i8>, ptr %b, i64 %indvars.iv
+ %0 = load <16 x i8>, ptr %arrayidx, align 16
+ %arrayidx2 = getelementptr inbounds <16 x i8>, ptr %c, i64 %indvars.iv
+ %1 = load <16 x i8>, ptr %arrayidx2, align 16
+ %result = call <16 x i8> @llvm.aarch64.neon.srhadd.v16i8(<16 x i8> %0, <16 x i8> %1)
+ %arrayidx3 = getelementptr inbounds <16 x i8>, ptr %a, i64 %indvars.iv
+ store <16 x i8> %result, ptr %arrayidx3, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+declare <16 x i8> @llvm.aarch64.neon.srhadd.v16i8(<16 x i8>, <16 x i8>)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-neon.ll b/llvm/test/Transforms/LoopVectorize/AArch64/revec-neon.ll
new file mode 100644
index 0000000000000..fea326abc8e26
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/revec-neon.ll
@@ -0,0 +1,1033 @@
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -vectorize-vector-loops \
+; RUN: -mtriple=aarch64 -mattr=+sve2p1,+v8.6a -S < %s | FileCheck %s
+
+; Test how NEON intrinsic are re-vectorised using HVLA.
+
+define void @uabd(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @uabd(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 8 x i16> @llvm.aarch64.sve.uabd.nxv8i16(<vscale x 8 x i1> splat (i1 true), <vscale x 8 x i16> [[WIDE_LOAD]], <vscale x 8 x i16> splat (i16 1))
+; CHECK: store <vscale x 8 x i16> [[WIDE_OP]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <8 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x i16>, ptr %arrayidx, align 16
+ %result = call <8 x i16> @llvm.aarch64.neon.uabd.v8i16(<8 x i16> %0, <8 x i16> splat (i16 1))
+ %arrayidx2 = getelementptr inbounds <8 x i16>, ptr %a, i64 %indvars.iv
+ store <8 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @sabd(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @sabd(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 8 x i16> @llvm.aarch64.sve.sabd.nxv8i16(<vscale x 8 x i1> splat (i1 true), <vscale x 8 x i16> [[WIDE_LOAD]], <vscale x 8 x i16> splat (i16 1))
+; CHECK: store <vscale x 8 x i16> [[WIDE_OP]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <8 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x i16>, ptr %arrayidx, align 16
+ %result = call <8 x i16> @llvm.aarch64.neon.sabd.v8i16(<8 x i16> %0, <8 x i16> splat (i16 1))
+ %arrayidx2 = getelementptr inbounds <8 x i16>, ptr %a, i64 %indvars.iv
+ store <8 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @fabd(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @fabd(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 8 x half>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 8 x half> @llvm.aarch64.sve.fabd.nxv8f16(<vscale x 8 x i1> splat (i1 true), <vscale x 8 x half> [[WIDE_LOAD]], <vscale x 8 x half> splat (half 1.000000e+00))
+; CHECK: store <vscale x 8 x half> [[WIDE_OP]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <8 x half>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x half>, ptr %arrayidx, align 16
+ %result = call <8 x half> @llvm.aarch64.neon.fabd.v8f16(<8 x half> %0, <8 x half> splat (half 1.000000e+00))
+ %arrayidx2 = getelementptr inbounds <8 x half>, ptr %a, i64 %indvars.iv
+ store <8 x half> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @facge(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @facge(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 8 x half>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.facge.nxv8f16(<vscale x 8 x i1> splat (i1 true), <vscale x 8 x half> [[WIDE_LOAD]], <vscale x 8 x half> splat (half 1.000000e+00))
+; CHECK: [[TMP:%.*]] = zext <vscale x 8 x i1> [[WIDE_OP]] to <vscale x 8 x i16>
+; CHECK: store <vscale x 8 x i16> [[TMP]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <8 x half>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x half>, ptr %arrayidx, align 16
+ %result = call <8 x i16> @llvm.aarch64.neon.facge.v8i16.v8f16(<8 x half> %0, <8 x half> splat (half 1.000000e+00))
+ %arrayidx2 = getelementptr inbounds <8 x i16>, ptr %a, i64 %indvars.iv
+ store <8 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @facgt(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @facgt(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 8 x half>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.facgt.nxv8f16(<vscale x 8 x i1> splat (i1 true), <vscale x 8 x half> [[WIDE_LOAD]], <vscale x 8 x half> splat (half 1.000000e+00))
+; CHECK: [[TMP:%.*]] = zext <vscale x 8 x i1> [[WIDE_OP]] to <vscale x 8 x i16>
+; CHECK: store <vscale x 8 x i16> [[TMP]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <8 x half>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x half>, ptr %arrayidx, align 16
+ %result = call <8 x i16> @llvm.aarch64.neon.facgt.v8i16.v8f16(<8 x half> %0, <8 x half> splat (half 1.000000e+00))
+ %arrayidx2 = getelementptr inbounds <8 x i16>, ptr %a, i64 %indvars.iv
+ store <8 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @fcvtzs(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @fcvtzs(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 8 x half>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 8 x i16> @llvm.aarch64.sve.fcvtzs.nxv8i16.nxv8f16(<vscale x 8 x i16> poison, <vscale x 8 x i1> splat (i1 true), <vscale x 8 x half> [[WIDE_LOAD]])
+; CHECK: store <vscale x 8 x i16> [[WIDE_OP]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <8 x half>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x half>, ptr %arrayidx, align 16
+ %result = call <8 x i16> @llvm.aarch64.neon.fcvtzs.v8i16.v8f16(<8 x half> %0)
+ %arrayidx2 = getelementptr inbounds <8 x i16>, ptr %a, i64 %indvars.iv
+ store <8 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @fcvtzu(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @fcvtzu(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 8 x half>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 8 x i16> @llvm.aarch64.sve.fcvtzu.nxv8i16.nxv8f16(<vscale x 8 x i16> poison, <vscale x 8 x i1> splat (i1 true), <vscale x 8 x half> [[WIDE_LOAD]])
+; CHECK: store <vscale x 8 x i16> [[WIDE_OP]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <8 x half>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x half>, ptr %arrayidx, align 16
+ %result = call <8 x i16> @llvm.aarch64.neon.fcvtzu.v8i16.v8f16(<8 x half> %0)
+ %arrayidx2 = getelementptr inbounds <8 x i16>, ptr %a, i64 %indvars.iv
+ store <8 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @frintx(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @frintx(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 4 x float> @llvm.aarch64.sve.frintx.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x float> [[WIDE_LOAD]])
+; CHECK: store <vscale x 4 x float> [[WIDE_OP]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x float>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x float>, ptr %arrayidx, align 16
+ %result = call <4 x float> @llvm.aarch64.neon.frint32x.v4f32(<4 x float> %0)
+ %arrayidx2 = getelementptr inbounds <4 x float>, ptr %a, i64 %indvars.iv
+ store <4 x float> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @frintz(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @frintz(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 4 x float> @llvm.aarch64.sve.frintz.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x float> [[WIDE_LOAD]])
+; CHECK: store <vscale x 4 x float> [[WIDE_OP]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x float>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x float>, ptr %arrayidx, align 16
+ %result = call <4 x float> @llvm.aarch64.neon.frint32z.v4f32(<4 x float> %0)
+ %arrayidx2 = getelementptr inbounds <4 x float>, ptr %a, i64 %indvars.iv
+ store <4 x float> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @frecpe(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @frecpe(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 4 x float> @llvm.aarch64.sve.frecpe.x.nxv4f32(<vscale x 4 x float> [[WIDE_LOAD]])
+; CHECK: store <vscale x 4 x float> [[WIDE_OP]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x float>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x float>, ptr %arrayidx, align 16
+ %result = call <4 x float> @llvm.aarch64.neon.frecpe.v4f32(<4 x float> %0)
+ %arrayidx2 = getelementptr inbounds <4 x float>, ptr %a, i64 %indvars.iv
+ store <4 x float> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @frecps(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @frecps(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 4 x float> @llvm.aarch64.sve.frecps.x.nxv4f32(<vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x float> splat (float 1.000000e+00))
+; CHECK: store <vscale x 4 x float> [[WIDE_OP]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x float>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x float>, ptr %arrayidx, align 16
+ %result = call <4 x float> @llvm.aarch64.neon.frecps.v4f32(<4 x float> %0, <4 x float> splat (float 1.0))
+ %arrayidx2 = getelementptr inbounds <4 x float>, ptr %a, i64 %indvars.iv
+ store <4 x float> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @frsqrte(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @frsqrte(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 4 x float> @llvm.aarch64.sve.frsqrte.x.nxv4f32(<vscale x 4 x float> [[WIDE_LOAD]])
+; CHECK: store <vscale x 4 x float> [[WIDE_OP]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x float>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x float>, ptr %arrayidx, align 16
+ %result = call <4 x float> @llvm.aarch64.neon.frsqrte.v4f32(<4 x float> %0)
+ %arrayidx2 = getelementptr inbounds <4 x float>, ptr %a, i64 %indvars.iv
+ store <4 x float> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @frsqrts(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @frsqrts(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 4 x float> @llvm.aarch64.sve.frsqrts.x.nxv4f32(<vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x float> splat (float 1.000000e+00))
+; CHECK: store <vscale x 4 x float> [[WIDE_OP]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x float>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x float>, ptr %arrayidx, align 16
+ %result = call <4 x float> @llvm.aarch64.neon.frsqrts.v4f32(<4 x float> %0, <4 x float> splat (float 1.0))
+ %arrayidx2 = getelementptr inbounds <4 x float>, ptr %a, i64 %indvars.iv
+ store <4 x float> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @abs(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @abs(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 8 x i16> @llvm.aarch64.sve.abs.nxv8i16(<vscale x 8 x i16> [[WIDE_LOAD]], <vscale x 8 x i1> splat (i1 true), <vscale x 8 x i16> [[WIDE_LOAD]])
+; CHECK: store <vscale x 8 x i16> [[WIDE_OP]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <8 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x i16>, ptr %arrayidx, align 16
+ %result = call <8 x i16> @llvm.aarch64.neon.abs.v8i16(<8 x i16> %0)
+ %arrayidx2 = getelementptr inbounds <8 x i16>, ptr %a, i64 %indvars.iv
+ store <8 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @sqabs(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @sqabs(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 8 x i16> @llvm.aarch64.sve.sqabs.nxv8i16(<vscale x 8 x i16> [[WIDE_LOAD]], <vscale x 8 x i1> splat (i1 true), <vscale x 8 x i16> [[WIDE_LOAD]])
+; CHECK: store <vscale x 8 x i16> [[WIDE_OP]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <8 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x i16>, ptr %arrayidx, align 16
+ %result = call <8 x i16> @llvm.aarch64.neon.sqabs.v8i16(<8 x i16> %0)
+ %arrayidx2 = getelementptr inbounds <8 x i16>, ptr %a, i64 %indvars.iv
+ store <8 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @uaddlp(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @uaddlp(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.uadalp.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> zeroinitializer, <vscale x 8 x i16> [[WIDE_LOAD]])
+; CHECK: store <vscale x 4 x i32> [[WIDE_OP]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <8 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x i16>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <4 x i32> @llvm.aarch64.neon.uaddlp.v4i32.v8i16(<8 x i16> %0)
+ %arrayidx2 = getelementptr inbounds <4 x i32>, ptr %a, i64 %indvars.iv
+ store <4 x i32> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @saddlp(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @saddlp(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.sadalp.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> zeroinitializer, <vscale x 8 x i16> [[WIDE_LOAD]])
+; CHECK: store <vscale x 4 x i32> [[WIDE_OP]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <8 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x i16>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <4 x i32> @llvm.aarch64.neon.saddlp.v4i32.v8i16(<8 x i16> %0)
+ %arrayidx2 = getelementptr inbounds <4 x i32>, ptr %a, i64 %indvars.iv
+ store <4 x i32> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @udot(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @udot(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> [[WIDE_LOAD]], <vscale x 16 x i8> [[WIDE_LOAD]])
+; CHECK: store <vscale x 4 x i32> [[WIDE_OP]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <16 x i8>, ptr %b, i64 %indvars.iv
+ %0 = load <16 x i8>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <4 x i32> @llvm.aarch64.neon.udot.v4i32.v16i8(<4 x i32> zeroinitializer, <16 x i8> %0, <16 x i8> %0)
+ %arrayidx2 = getelementptr inbounds <4 x i32>, ptr %a, i64 %indvars.iv
+ store <4 x i32> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @sdot(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @sdot(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> [[WIDE_LOAD]], <vscale x 16 x i8> [[WIDE_LOAD]])
+; CHECK: store <vscale x 4 x i32> [[WIDE_OP]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <16 x i8>, ptr %b, i64 %indvars.iv
+ %0 = load <16 x i8>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <4 x i32> @llvm.aarch64.neon.sdot.v4i32.v16i8(<4 x i32> zeroinitializer, <16 x i8> %0, <16 x i8> %0)
+ %arrayidx2 = getelementptr inbounds <4 x i32>, ptr %a, i64 %indvars.iv
+ store <4 x i32> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @usdot(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @usdot(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.usdot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> [[WIDE_LOAD]], <vscale x 16 x i8> [[WIDE_LOAD]])
+; CHECK: store <vscale x 4 x i32> [[WIDE_OP]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <16 x i8>, ptr %b, i64 %indvars.iv
+ %0 = load <16 x i8>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <4 x i32> @llvm.aarch64.neon.usdot.v4i32.v16i8(<4 x i32> zeroinitializer, <16 x i8> %0, <16 x i8> %0)
+ %arrayidx2 = getelementptr inbounds <4 x i32>, ptr %a, i64 %indvars.iv
+ store <4 x i32> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @uhadd(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @uhadd(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.uhadd.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> splat (i32 1))
+; CHECK: store <vscale x 4 x i32> [[WIDE_OP]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i32>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i32>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <4 x i32> @llvm.aarch64.neon.uhadd.v4i32(<4 x i32> %0, <4 x i32> splat (i32 1))
+ %arrayidx2 = getelementptr inbounds <4 x i32>, ptr %a, i64 %indvars.iv
+ store <4 x i32> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @shadd(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @shadd(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.shadd.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> splat (i32 1))
+; CHECK: store <vscale x 4 x i32> [[WIDE_OP]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i32>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i32>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <4 x i32> @llvm.aarch64.neon.shadd.v4i32(<4 x i32> %0, <4 x i32> splat (i32 1))
+ %arrayidx2 = getelementptr inbounds <4 x i32>, ptr %a, i64 %indvars.iv
+ store <4 x i32> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @uhsub(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @uhsub(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.uhsub.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> splat (i32 1))
+; CHECK: store <vscale x 4 x i32> [[WIDE_OP]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i32>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i32>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <4 x i32> @llvm.aarch64.neon.uhsub.v4i32(<4 x i32> %0, <4 x i32> splat (i32 1))
+ %arrayidx2 = getelementptr inbounds <4 x i32>, ptr %a, i64 %indvars.iv
+ store <4 x i32> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @shsub(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @shsub(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.shsub.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> splat (i32 1))
+; CHECK: store <vscale x 4 x i32> [[WIDE_OP]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i32>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i32>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <4 x i32> @llvm.aarch64.neon.shsub.v4i32(<4 x i32> %0, <4 x i32> splat (i32 1))
+ %arrayidx2 = getelementptr inbounds <4 x i32>, ptr %a, i64 %indvars.iv
+ store <4 x i32> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @uqadd(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @uqadd(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.uqadd.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> splat (i32 1))
+; CHECK: store <vscale x 4 x i32> [[WIDE_OP]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i32>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i32>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <4 x i32> @llvm.aarch64.neon.uqadd.v4i32(<4 x i32> %0, <4 x i32> splat (i32 1))
+ %arrayidx2 = getelementptr inbounds <4 x i32>, ptr %a, i64 %indvars.iv
+ store <4 x i32> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @sqadd(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @sqadd(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.sqadd.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> splat (i32 1))
+; CHECK: store <vscale x 4 x i32> [[WIDE_OP]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i32>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i32>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <4 x i32> @llvm.aarch64.neon.sqadd.v4i32(<4 x i32> %0, <4 x i32> splat (i32 1))
+ %arrayidx2 = getelementptr inbounds <4 x i32>, ptr %a, i64 %indvars.iv
+ store <4 x i32> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @uqsub(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @uqsub(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.uqsub.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> splat (i32 1))
+; CHECK: store <vscale x 4 x i32> [[WIDE_OP]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i32>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i32>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <4 x i32> @llvm.aarch64.neon.uqsub.v4i32(<4 x i32> %0, <4 x i32> splat (i32 1))
+ %arrayidx2 = getelementptr inbounds <4 x i32>, ptr %a, i64 %indvars.iv
+ store <4 x i32> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @sqsub(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @sqsub(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.sqsub.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> splat (i32 1))
+; CHECK: store <vscale x 4 x i32> [[WIDE_OP]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i32>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i32>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <4 x i32> @llvm.aarch64.neon.sqsub.v4i32(<4 x i32> %0, <4 x i32> splat (i32 1))
+ %arrayidx2 = getelementptr inbounds <4 x i32>, ptr %a, i64 %indvars.iv
+ store <4 x i32> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @sqdmulh(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @sqdmulh(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.sqdmulh.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> splat (i32 3))
+; CHECK: store <vscale x 4 x i32> [[WIDE_OP]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i32>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i32>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <4 x i32> @llvm.aarch64.neon.sqdmulh.v4i32(<4 x i32> %0, <4 x i32> splat (i32 3))
+ %arrayidx2 = getelementptr inbounds <4 x i32>, ptr %a, i64 %indvars.iv
+ store <4 x i32> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @sqrdmulh(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @sqrdmulh(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.sqrdmulh.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> splat (i32 3))
+; CHECK: store <vscale x 4 x i32> [[WIDE_OP]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i32>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i32>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <4 x i32> @llvm.aarch64.neon.sqrdmulh.v4i32(<4 x i32> %0, <4 x i32> splat (i32 3))
+ %arrayidx2 = getelementptr inbounds <4 x i32>, ptr %a, i64 %indvars.iv
+ store <4 x i32> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @sqshl(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @sqshl(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.sqshl.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> splat (i32 3))
+; CHECK: store <vscale x 4 x i32> [[WIDE_OP]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i32>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i32>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <4 x i32> @llvm.aarch64.neon.sqshl.v4i32(<4 x i32> %0, <4 x i32> splat (i32 3))
+ %arrayidx2 = getelementptr inbounds <4 x i32>, ptr %a, i64 %indvars.iv
+ store <4 x i32> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @sqshlu(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @sqshlu(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.sqshlu.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[WIDE_LOAD]], i32 1)
+; CHECK: store <vscale x 4 x i32> [[WIDE_OP]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i32>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i32>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <4 x i32> @llvm.aarch64.neon.sqshlu.v4i32(<4 x i32> %0, <4 x i32> <i32 1, i32 1, i32 1, i32 1>)
+ %arrayidx2 = getelementptr inbounds <4 x i32>, ptr %a, i64 %indvars.iv
+ store <4 x i32> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @uqshl(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @uqshl(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.uqshl.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> splat (i32 3))
+; CHECK: store <vscale x 4 x i32> [[WIDE_OP]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i32>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i32>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <4 x i32> @llvm.aarch64.neon.uqshl.v4i32(<4 x i32> %0, <4 x i32> splat (i32 3))
+ %arrayidx2 = getelementptr inbounds <4 x i32>, ptr %a, i64 %indvars.iv
+ store <4 x i32> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @srshl(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @srshl(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.srshl.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> splat (i32 3))
+; CHECK: store <vscale x 4 x i32> [[WIDE_OP]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i32>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i32>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <4 x i32> @llvm.aarch64.neon.srshl.v4i32(<4 x i32> %0, <4 x i32> splat (i32 3))
+ %arrayidx2 = getelementptr inbounds <4 x i32>, ptr %a, i64 %indvars.iv
+ store <4 x i32> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @sqrshl(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @sqrshl(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.sqrshl.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> splat (i32 3))
+; CHECK: store <vscale x 4 x i32> [[WIDE_OP]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i32>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i32>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <4 x i32> @llvm.aarch64.neon.sqrshl.v4i32(<4 x i32> %0, <4 x i32> splat (i32 3))
+ %arrayidx2 = getelementptr inbounds <4 x i32>, ptr %a, i64 %indvars.iv
+ store <4 x i32> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @sshl(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @sshl(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>
+; CHECK: [[SHL_MASK:%.*]] = icmp sle <vscale x 4 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK: [[TMP3:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.srshl.nxv4i32(<vscale x 4 x i1> [[SHL_MASK]], <vscale x 4 x i32> splat (i32 8), <vscale x 4 x i32> [[WIDE_LOAD]])
+; CHECK: [[SHR_MASK:%.*]] = call <vscale x 4 x i1> @llvm.ctlz.nxv4i1(<vscale x 4 x i1> [[SHL_MASK]], i1 true)
+; CHECK: [[WIDE_BIDIR_SHL:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.sqshl.nxv4i32(<vscale x 4 x i1> [[SHR_MASK]], <vscale x 4 x i32> [[TMP3]], <vscale x 4 x i32> [[WIDE_LOAD]])
+; CHECK: store <vscale x 4 x i32> [[WIDE_BIDIR_SHL]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i32>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i32>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <4 x i32> @llvm.aarch64.neon.sshl.v4i32(<4 x i32> splat (i32 8), <4 x i32> %0)
+ %arrayidx2 = getelementptr inbounds <4 x i32>, ptr %a, i64 %indvars.iv
+ store <4 x i32> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @ushl(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @ushl(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>
+; CHECK: [[SHL_MASK:%.*]] = icmp sle <vscale x 4 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK: [[TMP3:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.urshl.nxv4i32(<vscale x 4 x i1> [[SHL_MASK]], <vscale x 4 x i32> splat (i32 8), <vscale x 4 x i32> [[WIDE_LOAD]])
+; CHECK: [[SHR_MASK:%.*]] = call <vscale x 4 x i1> @llvm.ctlz.nxv4i1(<vscale x 4 x i1> [[SHL_MASK]], i1 true)
+; CHECK: [[WIDE_BIDIR_SHL:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.uqshl.nxv4i32(<vscale x 4 x i1> [[SHR_MASK]], <vscale x 4 x i32> [[TMP3]], <vscale x 4 x i32> [[WIDE_LOAD]])
+; CHECK: store <vscale x 4 x i32> [[WIDE_BIDIR_SHL]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i32>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i32>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <4 x i32> @llvm.aarch64.neon.ushl.v4i32(<4 x i32> splat (i32 8), <4 x i32> %0)
+ %arrayidx2 = getelementptr inbounds <4 x i32>, ptr %a, i64 %indvars.iv
+ store <4 x i32> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @vsli(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @vsli(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.sli.nxv4i32(<vscale x 4 x i32> splat (i32 1), <vscale x 4 x i32> [[WIDE_LOAD]], i32 2)
+; CHECK: store <vscale x 4 x i32> [[WIDE_OP]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i32>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i32>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <4 x i32> @llvm.aarch64.neon.vsli.v4i32(<4 x i32> splat (i32 1), <4 x i32> %0, i32 2)
+ %arrayidx2 = getelementptr inbounds <4 x i32>, ptr %a, i64 %indvars.iv
+ store <4 x i32> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @vsri(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @vsri(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>
+; CHECK: [[WIDE_OP:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.sri.nxv4i32(<vscale x 4 x i32> splat (i32 1), <vscale x 4 x i32> [[WIDE_LOAD]], i32 2)
+; CHECK: store <vscale x 4 x i32> [[WIDE_OP]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i32>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i32>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <4 x i32> @llvm.aarch64.neon.vsri.v4i32(<4 x i32> splat (i32 1), <4 x i32> %0, i32 2)
+ %arrayidx2 = getelementptr inbounds <4 x i32>, ptr %a, i64 %indvars.iv
+ store <4 x i32> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @addp(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @addp(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>
+; CHECK: [[TMP3:%.*]] = call <vscale x 4 x i32> @llvm.vector.segmented.shuffle.nxv4i32.v4i32(<vscale x 4 x i32> splat (i32 8), <vscale x 4 x i32> [[WIDE_LOAD]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>)
+; CHECK: [[TMP4:%.*]] = call <vscale x 4 x i32> @llvm.vector.segmented.shuffle.nxv4i32.v4i32(<vscale x 4 x i32> splat (i32 8), <vscale x 4 x i32> [[WIDE_LOAD]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>)
+; CHECK: [[TMP5:%.*]] = add <vscale x 4 x i32> [[TMP3]], [[TMP4]]
+; CHECK: store <vscale x 4 x i32> [[TMP5]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i32>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i32>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <4 x i32> @llvm.aarch64.neon.addp.v4i32(<4 x i32> splat (i32 8), <4 x i32> %0)
+ %arrayidx2 = getelementptr inbounds <4 x i32>, ptr %a, i64 %indvars.iv
+ store <4 x i32> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @addp_64(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @addp_64(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>
+; CHECK: [[RES:%.*]] = call <vscale x 2 x i64> @llvm.aarch64.sve.addp.nxv2i64(<vscale x 2 x i1> splat (i1 true), <vscale x 2 x i64> splat (i64 8), <vscale x 2 x i64> [[WIDE_LOAD]])
+; CHECK: store <vscale x 2 x i64> [[RES]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <2 x i64>, ptr %b, i64 %indvars.iv
+ %0 = load <2 x i64>, ptr %arrayidx, align 16
+ %vpaddl1.i = tail call <2 x i64> @llvm.aarch64.neon.addp.v2i64(<2 x i64> splat (i64 8), <2 x i64> %0)
+ %arrayidx2 = getelementptr inbounds <2 x i64>, ptr %a, i64 %indvars.iv
+ store <2 x i64> %vpaddl1.i, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+
+define void @tbl1(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @tbl1(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>
+; CHECK: [[RES:%.*]] = call <vscale x 16 x i8> @llvm.aarch64.sve.tblq.nxv16i8(<vscale x 16 x i8> [[WIDE_LOAD]], <vscale x 16 x i8> [[WIDE_LOAD]])
+; CHECK: store <vscale x 16 x i8> [[RES]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <16 x i8>, ptr %b, i64 %indvars.iv
+ %0 = load <16 x i8>, ptr %arrayidx, align 16
+ %res = tail call <16 x i8> @llvm.aarch64.neon.tbl1.v16i8(<16 x i8> %0, <16 x i8> %0)
+ %arrayidx2 = getelementptr inbounds <16 x i8>, ptr %a, i64 %indvars.iv
+ store <16 x i8> %res, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @tbl2(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @tbl2(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>
+; CHECK: [[RES:%.*]] = call <vscale x 16 x i8> @llvm.aarch64.sve.tblq.nxv16i8(<vscale x 16 x i8> [[WIDE_LOAD]], <vscale x 16 x i8> [[WIDE_LOAD]])
+; CHECK: [[TBX_MASK:%.*]] = sub <vscale x 16 x i8> [[WIDE_LOAD]], splat (i8 16)
+; CHECK: [[RES2:%.*]] = call <vscale x 16 x i8> @llvm.aarch64.sve.tbxq.nxv16i8(<vscale x 16 x i8> [[RES]], <vscale x 16 x i8> splat (i8 4), <vscale x 16 x i8> [[TBX_MASK]])
+; CHECK: store <vscale x 16 x i8> [[RES2]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <16 x i8>, ptr %b, i64 %indvars.iv
+ %0 = load <16 x i8>, ptr %arrayidx, align 16
+ %res = tail call <16 x i8> @llvm.aarch64.neon.tbl2.v16i8(<16 x i8> %0, <16 x i8> splat (i8 4), <16 x i8> %0)
+ %arrayidx2 = getelementptr inbounds <16 x i8>, ptr %a, i64 %indvars.iv
+ store <16 x i8> %res, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @tbl3(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @tbl3(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>
+; CHECK: [[RES:%.*]] = call <vscale x 16 x i8> @llvm.aarch64.sve.tblq.nxv16i8(<vscale x 16 x i8> [[WIDE_LOAD]], <vscale x 16 x i8> [[WIDE_LOAD]])
+; CHECK: [[TBX_MASK:%.*]] = sub <vscale x 16 x i8> [[WIDE_LOAD]], splat (i8 16)
+; CHECK: [[RES2:%.*]] = call <vscale x 16 x i8> @llvm.aarch64.sve.tbxq.nxv16i8(<vscale x 16 x i8> [[RES]], <vscale x 16 x i8> splat (i8 4), <vscale x 16 x i8> [[TBX_MASK]])
+; CHECK: [[TBX_MASK2:%.*]] = sub <vscale x 16 x i8> [[TBX_MASK]], splat (i8 16)
+; CHECK: [[RES3:%.*]] = call <vscale x 16 x i8> @llvm.aarch64.sve.tbxq.nxv16i8(<vscale x 16 x i8> [[RES2]], <vscale x 16 x i8> splat (i8 6), <vscale x 16 x i8> [[TBX_MASK2]])
+; CHECK: store <vscale x 16 x i8> [[RES3]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <16 x i8>, ptr %b, i64 %indvars.iv
+ %0 = load <16 x i8>, ptr %arrayidx, align 16
+ %res = tail call <16 x i8> @llvm.aarch64.neon.tbl3.v16i8(<16 x i8> %0, <16 x i8> splat (i8 4), <16 x i8> splat (i8 6), <16 x i8> %0)
+ %arrayidx2 = getelementptr inbounds <16 x i8>, ptr %a, i64 %indvars.iv
+ store <16 x i8> %res, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-nontemporal-load.ll b/llvm/test/Transforms/LoopVectorize/AArch64/revec-nontemporal-load.ll
new file mode 100644
index 0000000000000..086a323d793a4
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/revec-nontemporal-load.ll
@@ -0,0 +1,28 @@
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=1 \
+; RUN: -scalable-vectorization=on -vectorize-vector-loops -mtriple=aarch64 \
+; RUN: -mattr=+sve2p1 -S < %s | FileCheck %s
+
+define void @nontemporal_vector_load_store(ptr noalias %a, ptr noalias readonly %b) {
+; CHECK-LABEL: define void @nontemporal_vector_load_store(
+; CHECK: vector.body:
+; CHECK: load <vscale x 8 x i16>, ptr {{.*}}, align 16, !nontemporal
+; CHECK: store <vscale x 8 x i16> {{.*}}, ptr {{.*}}, align 16, !nontemporal
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %bp = getelementptr inbounds <8 x i16>, ptr %b, i64 %iv
+ %v = load <8 x i16>, ptr %bp, align 16, !nontemporal !0
+ %r = add <8 x i16> %v, splat (i16 1)
+ %ap = getelementptr inbounds <8 x i16>, ptr %a, i64 %iv
+ store <8 x i16> %r, ptr %ap, align 16, !nontemporal !0
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exit = icmp eq i64 %iv.next, 1024
+ br i1 %exit, label %done, label %loop
+
+done:
+ ret void
+}
+
+!0 = !{i32 1}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-partial-reduction.ll b/llvm/test/Transforms/LoopVectorize/AArch64/revec-partial-reduction.ll
new file mode 100644
index 0000000000000..8e2c486ccd399
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/revec-partial-reduction.ll
@@ -0,0 +1,90 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "scalar.ph:" --replace-value-regex "!llvm.loop ![0-9]+" --version 6
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -vectorize-vector-loops -mtriple=aarch64 -mattr=+sve2p1 -S < %s | FileCheck %s
+
+; Ensure that vector reductions with zext/sext in their chains are not matched as
+; partial reductions.
+
+define <8 x i16> @partial_reduction_shape(i64 %0) {
+; CHECK-LABEL: define <8 x i16> @partial_reduction_shape(
+; CHECK-SAME: i64 [[TMP0:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[SCALAR_LOOP:.*]]
+; CHECK: [[SCALAR_LOOP]]:
+; CHECK-NEXT: [[INDVARS_IV_I_US_I:%.*]] = phi i64 [ [[TMP0]], %[[ENTRY]] ], [ [[INDVARS_IV_NEXT_I_US_I:%.*]], %[[SCALAR_LOOP]] ]
+; CHECK-NEXT: [[SUM_1235_I_US_I:%.*]] = phi <8 x i16> [ zeroinitializer, %[[ENTRY]] ], [ [[ADD_I_I_US_I:%.*]], %[[SCALAR_LOOP]] ]
+; CHECK-NEXT: [[INDVARS_IV_NEXT_I_US_I]] = add i64 [[INDVARS_IV_I_US_I]], 1
+; CHECK-NEXT: [[TMP1:%.*]] = load i8, ptr null, align 1
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <8 x i8> zeroinitializer, i8 [[TMP1]], i64 0
+; CHECK-NEXT: [[VMOVL_I_I_US_I:%.*]] = zext <8 x i8> [[TMP2]] to <8 x i16>
+; CHECK-NEXT: [[ADD_I_I_US_I]] = add <8 x i16> [[SUM_1235_I_US_I]], [[VMOVL_I_I_US_I]]
+; CHECK-NEXT: [[CMP93_I_US_I:%.*]] = icmp ugt i64 [[INDVARS_IV_I_US_I]], 0
+; CHECK-NEXT: br i1 [[CMP93_I_US_I]], label %[[SCALAR_LOOP]], label %[[LOOP_END:.*]]
+; CHECK: [[LOOP_END]]:
+; CHECK-NEXT: [[ADD_I_I_US_I_LCSSA:%.*]] = phi <8 x i16> [ [[ADD_I_I_US_I]], %[[SCALAR_LOOP]] ]
+; CHECK-NEXT: ret <8 x i16> [[ADD_I_I_US_I_LCSSA]]
+;
+entry:
+ br label %scalar.loop
+
+scalar.loop:
+ %indvars.iv.i.us.i = phi i64 [ %0, %entry ], [ %indvars.iv.next.i.us.i, %scalar.loop ]
+ %sum.1235.i.us.i = phi <8 x i16> [ zeroinitializer, %entry ], [ %add.i.i.us.i, %scalar.loop ]
+ %indvars.iv.next.i.us.i = add i64 %indvars.iv.i.us.i, 1
+ %1 = load i8, ptr null, align 1
+ %2 = insertelement <8 x i8> zeroinitializer, i8 %1, i64 0
+ %vmovl.i.i.us.i = zext <8 x i8> %2 to <8 x i16>
+ %add.i.i.us.i = add <8 x i16> %sum.1235.i.us.i, %vmovl.i.i.us.i
+ %cmp93.i.us.i = icmp ugt i64 %indvars.iv.i.us.i, 0
+ br i1 %cmp93.i.us.i, label %scalar.loop, label %loop.end
+
+loop.end:
+ ret <8 x i16> %add.i.i.us.i
+}
+
+
+define <8 x i16> @partial_reduction_vector_shape(i64 %0) {
+; CHECK-LABEL: define <8 x i16> @partial_reduction_vector_shape(
+; CHECK-SAME: i64 [[TMP0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP1:%.*]] = sub i64 1, [[TMP0]]
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], [[TMP2]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[TMP0]], [[N_VEC]]
+; CHECK-NEXT: [[TMP4:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v8i16(<vscale x 8 x i16> zeroinitializer, <8 x i16> zeroinitializer, i64 0)
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 8 x i16> [ [[TMP4]], %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = load <8 x i8>, ptr null, align 1
+; CHECK-NEXT: [[BROADCAST:%.*]] = call <vscale x 8 x i8> @llvm.vector.broadcast.nxv8i8.v8i8(<8 x i8> [[TMP7]])
+; CHECK-NEXT: [[TMP6:%.*]] = zext <vscale x 8 x i8> [[BROADCAST]] to <vscale x 8 x i16>
+; CHECK-NEXT: [[TMP8]] = add <vscale x 8 x i16> [[VEC_PHI]], [[TMP6]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP9:%.*]] = call <8 x i16> @llvm.vector.partial.reduce.add.v8i16.nxv8i16(<8 x i16> zeroinitializer, <vscale x 8 x i16> [[TMP8]])
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[LOOP_END:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %scalar.loop
+
+scalar.loop:
+ %indvars.iv.i.us.i = phi i64 [ %0, %entry ], [ %indvars.iv.next.i.us.i, %scalar.loop ]
+ %sum.1235.i.us.i = phi <8 x i16> [ zeroinitializer, %entry ], [ %add.i.i.us.i, %scalar.loop ]
+ %indvars.iv.next.i.us.i = add i64 %indvars.iv.i.us.i, 1
+ %2 = load <8 x i8>, ptr null, align 1
+ %vmovl.i.i.us.i = zext <8 x i8> %2 to <8 x i16>
+ %add.i.i.us.i = add <8 x i16> %sum.1235.i.us.i, %vmovl.i.i.us.i
+ %cmp93.i.us.i = icmp ugt i64 %indvars.iv.i.us.i, 0
+ br i1 %cmp93.i.us.i, label %scalar.loop, label %loop.end
+
+loop.end:
+ ret <8 x i16> %add.i.i.us.i
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-pixel-neon.ll b/llvm/test/Transforms/LoopVectorize/AArch64/revec-pixel-neon.ll
new file mode 100644
index 0000000000000..8546009068e3e
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/revec-pixel-neon.ll
@@ -0,0 +1,159 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=1 -vectorize-vector-loops \
+; RUN: -mtriple=aarch64 -mattr=+sve2p1 -S | FileCheck %s --check-prefixes=VECTORIZE
+
+define void @pixel_ssd(ptr noundef readonly captures(none) %pixuv1, ptr noundef readonly captures(none) %pixuv2, i32 noundef %width, i32 noundef %height, ptr noundef writeonly captures(none) %ssd_u, ptr noundef writeonly captures(none) %ssd_v) {
+; VECTORIZE-LABEL: define void @pixel_ssd(
+; VECTORIZE-SAME: ptr noundef readonly captures(none) [[PIXUV1:%.*]], ptr noundef readonly captures(none) [[PIXUV2:%.*]], i32 noundef [[WIDTH:%.*]], i32 noundef [[HEIGHT:%.*]], ptr noundef writeonly captures(none) [[SSD_U:%.*]], ptr noundef writeonly captures(none) [[SSD_V:%.*]]) #[[ATTR0:[0-9]+]] {
+; VECTORIZE-NEXT: [[ENTRY:.*]]:
+; VECTORIZE-NEXT: [[MUL:%.*]] = shl nsw i32 [[WIDTH]], 1
+; VECTORIZE-NEXT: [[IDX_EXT14:%.*]] = sext i32 [[MUL]] to i64
+; VECTORIZE-NEXT: [[TMP0:%.*]] = sext i32 [[WIDTH]] to i64
+; VECTORIZE-NEXT: [[SMAX:%.*]] = tail call i32 @llvm.smax.i32(i32 [[HEIGHT]], i32 1)
+; VECTORIZE-NEXT: [[SMAX1:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP0]], i64 8)
+; VECTORIZE-NEXT: [[TMP1:%.*]] = add nsw i64 [[SMAX1]], -1
+; VECTORIZE-NEXT: [[TMP9:%.*]] = lshr i64 [[TMP1]], 3
+; VECTORIZE-NEXT: [[TMP10:%.*]] = add nuw nsw i64 [[TMP9]], 1
+; VECTORIZE-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; VECTORIZE-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP4]], i64 3)
+; VECTORIZE-NEXT: br label %[[DO_BODY:.*]]
+; VECTORIZE: [[DO_BODY]]:
+; VECTORIZE-NEXT: [[H_0:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD18:%.*]], %[[DO_END:.*]] ]
+; VECTORIZE-NEXT: [[VSSD_V_0:%.*]] = phi <4 x i32> [ zeroinitializer, %[[ENTRY]] ], [ [[VDOT1_I_LCSSA:%.*]], %[[DO_END]] ]
+; VECTORIZE-NEXT: [[VSSD_U_0:%.*]] = phi <4 x i32> [ zeroinitializer, %[[ENTRY]] ], [ [[VDOT1_I44_LCSSA:%.*]], %[[DO_END]] ]
+; VECTORIZE-NEXT: [[PIXUV2_ADDR_0:%.*]] = phi ptr [ [[PIXUV2]], %[[ENTRY]] ], [ [[ADD_PTR17:%.*]], %[[DO_END]] ]
+; VECTORIZE-NEXT: [[PIXUV1_ADDR_0:%.*]] = phi ptr [ [[PIXUV1]], %[[ENTRY]] ], [ [[ADD_PTR15:%.*]], %[[DO_END]] ]
+; VECTORIZE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP10]], [[UMAX]]
+; VECTORIZE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VECTORIZE: [[VECTOR_PH]]:
+; VECTORIZE-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; VECTORIZE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP10]], [[TMP5]]
+; VECTORIZE-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP10]], [[N_MOD_VF]]
+; VECTORIZE-NEXT: [[TMP6:%.*]] = shl i64 [[N_VEC]], 3
+; VECTORIZE-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v4i32(<vscale x 4 x i32> zeroinitializer, <4 x i32> [[VSSD_V_0]], i64 0)
+; VECTORIZE-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v4i32(<vscale x 4 x i32> zeroinitializer, <4 x i32> [[VSSD_U_0]], i64 0)
+; VECTORIZE-NEXT: br label %[[VECTOR_BODY:.*]]
+; VECTORIZE: [[VECTOR_BODY]]:
+; VECTORIZE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VECTORIZE-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ [[TMP7]], %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
+; VECTORIZE-NEXT: [[VEC_PHI2:%.*]] = phi <vscale x 4 x i32> [ [[TMP8]], %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]
+; VECTORIZE-NEXT: [[INDVARS_IV1:%.*]] = shl i64 [[INDEX]], 3
+; VECTORIZE-NEXT: [[TMP20:%.*]] = shl nuw nsw i64 [[INDVARS_IV1]], 1
+; VECTORIZE-NEXT: [[ADD_PTR1:%.*]] = getelementptr inbounds nuw i8, ptr [[PIXUV1_ADDR_0]], i64 [[TMP20]]
+; VECTORIZE-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[ADD_PTR1]], align 1
+; VECTORIZE-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[PIXUV2_ADDR_0]], i64 [[TMP20]]
+; VECTORIZE-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 16 x i8>, ptr [[TMP11]], align 1
+; VECTORIZE-NEXT: [[TMP12:%.*]] = call <vscale x 16 x i8> @llvm.aarch64.sve.uabd.nxv16i8(<vscale x 16 x i1> splat (i1 true), <vscale x 16 x i8> [[WIDE_LOAD]], <vscale x 16 x i8> [[WIDE_LOAD3]])
+; VECTORIZE-NEXT: [[TMP13:%.*]] = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(<vscale x 16 x i8> [[TMP12]], <vscale x 16 x i8> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>)
+; VECTORIZE-NEXT: [[TMP14:%.*]] = call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(<vscale x 16 x i8> [[TMP12]], <vscale x 16 x i8> poison, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>)
+; VECTORIZE-NEXT: [[TMP15]] = call <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32> [[VEC_PHI2]], <vscale x 16 x i8> [[TMP13]], <vscale x 16 x i8> [[TMP13]])
+; VECTORIZE-NEXT: [[TMP16]] = call <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i8> [[TMP14]], <vscale x 16 x i8> [[TMP14]])
+; VECTORIZE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; VECTORIZE-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VECTORIZE-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; VECTORIZE: [[MIDDLE_BLOCK]]:
+; VECTORIZE-NEXT: [[TMP18:%.*]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.nxv4i32(<4 x i32> zeroinitializer, <vscale x 4 x i32> [[TMP16]])
+; VECTORIZE-NEXT: [[TMP19:%.*]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.nxv4i32(<4 x i32> zeroinitializer, <vscale x 4 x i32> [[TMP15]])
+; VECTORIZE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP10]], [[N_VEC]]
+; VECTORIZE-NEXT: br i1 [[CMP_N]], label %[[DO_END]], label %[[SCALAR_PH]]
+; VECTORIZE: [[SCALAR_PH]]:
+; VECTORIZE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ 0, %[[DO_BODY]] ]
+; VECTORIZE-NEXT: [[BC_MERGE_RDX:%.*]] = phi <4 x i32> [ [[TMP18]], %[[MIDDLE_BLOCK]] ], [ [[VSSD_V_0]], %[[DO_BODY]] ]
+; VECTORIZE-NEXT: [[BC_MERGE_RDX4:%.*]] = phi <4 x i32> [ [[TMP19]], %[[MIDDLE_BLOCK]] ], [ [[VSSD_U_0]], %[[DO_BODY]] ]
+; VECTORIZE-NEXT: br label %[[DO_BODY2:.*]]
+; VECTORIZE: [[DO_BODY2]]:
+; VECTORIZE-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[DO_BODY2]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; VECTORIZE-NEXT: [[VSSD_V_1:%.*]] = phi <4 x i32> [ [[VDOT1_I:%.*]], %[[DO_BODY2]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
+; VECTORIZE-NEXT: [[VSSD_U_1:%.*]] = phi <4 x i32> [ [[VDOT1_I44:%.*]], %[[DO_BODY2]] ], [ [[BC_MERGE_RDX4]], %[[SCALAR_PH]] ]
+; VECTORIZE-NEXT: [[TMP21:%.*]] = shl nuw nsw i64 [[INDVARS_IV]], 1
+; VECTORIZE-NEXT: [[ADD_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[PIXUV1_ADDR_0]], i64 [[TMP21]]
+; VECTORIZE-NEXT: [[TMP2:%.*]] = load <16 x i8>, ptr [[ADD_PTR]], align 1
+; VECTORIZE-NEXT: [[ADD_PTR7:%.*]] = getelementptr inbounds nuw i8, ptr [[PIXUV2_ADDR_0]], i64 [[TMP21]]
+; VECTORIZE-NEXT: [[TMP3:%.*]] = load <16 x i8>, ptr [[ADD_PTR7]], align 1
+; VECTORIZE-NEXT: [[VABD_I:%.*]] = tail call noundef <16 x i8> @llvm.aarch64.neon.uabd.v16i8(<16 x i8> [[TMP2]], <16 x i8> [[TMP3]])
+; VECTORIZE-NEXT: [[SHUFFLE_I:%.*]] = shufflevector <16 x i8> [[VABD_I]], <16 x i8> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; VECTORIZE-NEXT: [[SHUFFLE_I43:%.*]] = shufflevector <16 x i8> [[VABD_I]], <16 x i8> poison, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; VECTORIZE-NEXT: [[VDOT1_I44]] = tail call noundef <4 x i32> @llvm.aarch64.neon.udot.v4i32.v16i8(<4 x i32> [[VSSD_U_1]], <16 x i8> [[SHUFFLE_I]], <16 x i8> [[SHUFFLE_I]])
+; VECTORIZE-NEXT: [[VDOT1_I]] = tail call noundef <4 x i32> @llvm.aarch64.neon.udot.v4i32.v16i8(<4 x i32> [[VSSD_V_1]], <16 x i8> [[SHUFFLE_I43]], <16 x i8> [[SHUFFLE_I43]])
+; VECTORIZE-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 8
+; VECTORIZE-NEXT: [[CMP:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[TMP0]]
+; VECTORIZE-NEXT: br i1 [[CMP]], label %[[DO_BODY2]], label %[[DO_END]], !llvm.loop [[LOOP3:![0-9]+]]
+; VECTORIZE: [[DO_END]]:
+; VECTORIZE-NEXT: [[VDOT1_I44_LCSSA]] = phi <4 x i32> [ [[VDOT1_I44]], %[[DO_BODY2]] ], [ [[TMP19]], %[[MIDDLE_BLOCK]] ]
+; VECTORIZE-NEXT: [[VDOT1_I_LCSSA]] = phi <4 x i32> [ [[VDOT1_I]], %[[DO_BODY2]] ], [ [[TMP18]], %[[MIDDLE_BLOCK]] ]
+; VECTORIZE-NEXT: [[ADD_PTR15]] = getelementptr inbounds i8, ptr [[PIXUV1_ADDR_0]], i64 [[IDX_EXT14]]
+; VECTORIZE-NEXT: [[ADD_PTR17]] = getelementptr inbounds i8, ptr [[PIXUV2_ADDR_0]], i64 [[IDX_EXT14]]
+; VECTORIZE-NEXT: [[ADD18]] = add nuw nsw i32 [[H_0]], 1
+; VECTORIZE-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[ADD18]], [[SMAX]]
+; VECTORIZE-NEXT: br i1 [[EXITCOND_NOT]], label %[[DO_END21:.*]], label %[[DO_BODY]]
+; VECTORIZE: [[DO_END21]]:
+; VECTORIZE-NEXT: [[VDOT1_I44_LCSSA_LCSSA:%.*]] = phi <4 x i32> [ [[VDOT1_I44_LCSSA]], %[[DO_END]] ]
+; VECTORIZE-NEXT: [[VDOT1_I_LCSSA_LCSSA:%.*]] = phi <4 x i32> [ [[VDOT1_I_LCSSA]], %[[DO_END]] ]
+; VECTORIZE-NEXT: [[VADDVQ_U32_I45:%.*]] = tail call noundef i32 @llvm.aarch64.neon.uaddv.i32.v4i32(<4 x i32> [[VDOT1_I44_LCSSA_LCSSA]])
+; VECTORIZE-NEXT: [[DIV41:%.*]] = lshr i32 [[VADDVQ_U32_I45]], 1
+; VECTORIZE-NEXT: [[CONV:%.*]] = zext nneg i32 [[DIV41]] to i64
+; VECTORIZE-NEXT: store i64 [[CONV]], ptr [[SSD_U]], align 8
+; VECTORIZE-NEXT: [[VADDVQ_U32_I:%.*]] = tail call noundef i32 @llvm.aarch64.neon.uaddv.i32.v4i32(<4 x i32> [[VDOT1_I_LCSSA_LCSSA]])
+; VECTORIZE-NEXT: [[DIV2442:%.*]] = lshr i32 [[VADDVQ_U32_I]], 1
+; VECTORIZE-NEXT: [[CONV25:%.*]] = zext nneg i32 [[DIV2442]] to i64
+; VECTORIZE-NEXT: store i64 [[CONV25]], ptr [[SSD_V]], align 8
+; VECTORIZE-NEXT: ret void
+;
+entry:
+ %mul = shl nsw i32 %width, 1
+ %idx.ext14 = sext i32 %mul to i64
+ %0 = sext i32 %width to i64
+ %smax = tail call i32 @llvm.smax.i32(i32 %height, i32 1)
+ br label %do.body
+
+do.body: ; preds = %do.end, %entry
+ %h.0 = phi i32 [ 0, %entry ], [ %add18, %do.end ]
+ %vssd_v.0 = phi <4 x i32> [ zeroinitializer, %entry ], [ %vdot1.i, %do.end ]
+ %vssd_u.0 = phi <4 x i32> [ zeroinitializer, %entry ], [ %vdot1.i44, %do.end ]
+ %pixuv2.addr.0 = phi ptr [ %pixuv2, %entry ], [ %add.ptr17, %do.end ]
+ %pixuv1.addr.0 = phi ptr [ %pixuv1, %entry ], [ %add.ptr15, %do.end ]
+ br label %do.body2
+
+do.body2: ; preds = %do.body2, %do.body
+ %indvars.iv = phi i64 [ %indvars.iv.next, %do.body2 ], [ 0, %do.body ]
+ %vssd_v.1 = phi <4 x i32> [ %vdot1.i, %do.body2 ], [ %vssd_v.0, %do.body ]
+ %vssd_u.1 = phi <4 x i32> [ %vdot1.i44, %do.body2 ], [ %vssd_u.0, %do.body ]
+ %1 = shl nuw nsw i64 %indvars.iv, 1
+ %add.ptr = getelementptr inbounds nuw i8, ptr %pixuv1.addr.0, i64 %1
+ %2 = load <16 x i8>, ptr %add.ptr, align 1
+ %add.ptr7 = getelementptr inbounds nuw i8, ptr %pixuv2.addr.0, i64 %1
+ %3 = load <16 x i8>, ptr %add.ptr7, align 1
+ %vabd.i = tail call noundef <16 x i8> @llvm.aarch64.neon.uabd.v16i8(<16 x i8> %2, <16 x i8> %3)
+ %shuffle.i = shufflevector <16 x i8> %vabd.i, <16 x i8> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+ %shuffle.i43 = shufflevector <16 x i8> %vabd.i, <16 x i8> poison, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+ %vdot1.i44 = tail call noundef <4 x i32> @llvm.aarch64.neon.udot.v4i32.v16i8(<4 x i32> %vssd_u.1, <16 x i8> %shuffle.i, <16 x i8> %shuffle.i)
+ %vdot1.i = tail call noundef <4 x i32> @llvm.aarch64.neon.udot.v4i32.v16i8(<4 x i32> %vssd_v.1, <16 x i8> %shuffle.i43, <16 x i8> %shuffle.i43)
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 8
+ %cmp = icmp slt i64 %indvars.iv.next, %0
+ br i1 %cmp, label %do.body2, label %do.end
+
+do.end: ; preds = %do.body2
+ %add.ptr15 = getelementptr inbounds i8, ptr %pixuv1.addr.0, i64 %idx.ext14
+ %add.ptr17 = getelementptr inbounds i8, ptr %pixuv2.addr.0, i64 %idx.ext14
+ %add18 = add nuw nsw i32 %h.0, 1
+ %exitcond.not = icmp eq i32 %add18, %smax
+ br i1 %exitcond.not, label %do.end21, label %do.body
+
+do.end21: ; preds = %do.end
+ %vaddvq_u32.i45 = tail call noundef i32 @llvm.aarch64.neon.uaddv.i32.v4i32(<4 x i32> %vdot1.i44)
+ %div41 = lshr i32 %vaddvq_u32.i45, 1
+ %conv = zext nneg i32 %div41 to i64
+ store i64 %conv, ptr %ssd_u, align 8
+ %vaddvq_u32.i = tail call noundef i32 @llvm.aarch64.neon.uaddv.i32.v4i32(<4 x i32> %vdot1.i)
+ %div2442 = lshr i32 %vaddvq_u32.i, 1
+ %conv25 = zext nneg i32 %div2442 to i64
+ store i64 %conv25, ptr %ssd_v, align 8
+ ret void
+}
+
+;.
+; VECTORIZE: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; VECTORIZE: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; VECTORIZE: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; VECTORIZE: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-predication.ll b/llvm/test/Transforms/LoopVectorize/AArch64/revec-predication.ll
new file mode 100644
index 0000000000000..f8d0ea88763ce
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/revec-predication.ll
@@ -0,0 +1,227 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "scalar.ph:" --replace-value-regex "!llvm.loop ![0-9]+" --version 6
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -vectorize-vector-loops \
+; RUN: -mtriple=aarch64 -mattr=+sve2p1 -S < %s | FileCheck %s
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -vectorize-vector-loops \
+; RUN: -tail-folding-policy=prefer-fold-tail \
+; RUN: -mtriple=aarch64 -mattr=+sve2p1 -S < %s | FileCheck %s
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -vectorize-vector-loops \
+; RUN: -tail-folding-policy=must-fold-tail \
+; RUN: -mtriple=aarch64 -mattr=+sve2p1 -S < %s | FileCheck %s --check-prefix=FORCE-TF
+
+; Try to force predication with tail folding. This should be rejected in favour
+; of a scalar epilogue because re-vectorisation does not handle predication.
+define void @illegal_tail_folding_by_masking(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b, ptr nocapture noundef readonly %c) {
+; CHECK-LABEL: define void @illegal_tail_folding_by_masking(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], ptr noundef readonly captures(none) [[C:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP0]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <8 x i16>, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP2]], align 16
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds <8 x i16>, ptr [[C]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 8 x i16>, ptr [[TMP3]], align 16
+; CHECK-NEXT: [[TMP4:%.*]] = add <vscale x 8 x i16> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds <8 x i16>, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: store <vscale x 8 x i16> [[TMP4]], ptr [[TMP5]], align 16
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+; FORCE-TF-LABEL: define void @illegal_tail_folding_by_masking(
+; FORCE-TF-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], ptr noundef readonly captures(none) [[C:%.*]]) #[[ATTR0:[0-9]+]] {
+; FORCE-TF-NEXT: [[ENTRY:.*]]:
+; FORCE-TF-NEXT: br label %[[FOR_BODY:.*]]
+; FORCE-TF: [[FOR_COND_CLEANUP:.*]]:
+; FORCE-TF-NEXT: ret void
+; FORCE-TF: [[FOR_BODY]]:
+; FORCE-TF-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; FORCE-TF-NEXT: [[ARRAYIDX0:%.*]] = getelementptr inbounds <8 x i16>, ptr [[B]], i64 [[INDVARS_IV]]
+; FORCE-TF-NEXT: [[TMP0:%.*]] = load <8 x i16>, ptr [[ARRAYIDX0]], align 16
+; FORCE-TF-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds <8 x i16>, ptr [[C]], i64 [[INDVARS_IV]]
+; FORCE-TF-NEXT: [[TMP1:%.*]] = load <8 x i16>, ptr [[ARRAYIDX1]], align 16
+; FORCE-TF-NEXT: [[RESULT:%.*]] = add <8 x i16> [[TMP0]], [[TMP1]]
+; FORCE-TF-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds <8 x i16>, ptr [[A]], i64 [[INDVARS_IV]]
+; FORCE-TF-NEXT: store <8 x i16> [[RESULT]], ptr [[ARRAYIDX2]], align 16
+; FORCE-TF-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; FORCE-TF-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1024
+; FORCE-TF-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx0 = getelementptr inbounds <8 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x i16>, ptr %arrayidx0, align 16
+ %arrayidx1 = getelementptr inbounds <8 x i16>, ptr %c, i64 %indvars.iv
+ %1 = load <8 x i16>, ptr %arrayidx1, align 16
+ %result = add <8 x i16> %0, %1
+ %arrayidx2 = getelementptr inbounds <8 x i16>, ptr %a, i64 %indvars.iv
+ store <8 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @existing_predication(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b, ptr nocapture noundef readonly %c, i64 %cond) {
+; CHECK-LABEL: define void @existing_predication(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], ptr noundef readonly captures(none) [[C:%.*]], i64 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP11]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP11]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[COND]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 1 x i64> @llvm.stepvector.nxv1i64()
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[TMP11]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 1 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 1 x i64> [ [[TMP2]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds <8 x i16>, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP4]], align 16
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds <8 x i16>, ptr [[C]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 16
+; CHECK-NEXT: [[TMP6:%.*]] = add <vscale x 8 x i16> [[WIDE_LOAD]], [[WIDE_LOAD3]]
+; CHECK-NEXT: [[TMP7:%.*]] = icmp ne <vscale x 1 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr <8 x i16>, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 8 x i1> @llvm.vector.stretch.nxv8i1.nxv1i1(<vscale x 1 x i1> [[TMP7]])
+; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP6]], ptr align 16 [[TMP8]], <vscale x 8 x i1> [[TMP9]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 1 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+; FORCE-TF-LABEL: define void @existing_predication(
+; FORCE-TF-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], ptr noundef readonly captures(none) [[C:%.*]], i64 [[COND:%.*]]) #[[ATTR0]] {
+; FORCE-TF-NEXT: [[ENTRY:.*]]:
+; FORCE-TF-NEXT: br label %[[FOR_BODY:.*]]
+; FORCE-TF: [[FOR_COND_CLEANUP:.*]]:
+; FORCE-TF-NEXT: ret void
+; FORCE-TF: [[FOR_BODY]]:
+; FORCE-TF-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; FORCE-TF-NEXT: [[ARRAYIDX0:%.*]] = getelementptr inbounds <8 x i16>, ptr [[B]], i64 [[INDVARS_IV]]
+; FORCE-TF-NEXT: [[TMP0:%.*]] = load <8 x i16>, ptr [[ARRAYIDX0]], align 16
+; FORCE-TF-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds <8 x i16>, ptr [[C]], i64 [[INDVARS_IV]]
+; FORCE-TF-NEXT: [[TMP1:%.*]] = load <8 x i16>, ptr [[ARRAYIDX1]], align 16
+; FORCE-TF-NEXT: [[RESULT:%.*]] = add <8 x i16> [[TMP0]], [[TMP1]]
+; FORCE-TF-NEXT: [[DO_ACCESS:%.*]] = icmp ne i64 [[INDVARS_IV]], [[COND]]
+; FORCE-TF-NEXT: br i1 [[DO_ACCESS]], label %[[IF_THEN:.*]], label %[[FOR_INC]]
+; FORCE-TF: [[IF_THEN]]:
+; FORCE-TF-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds <8 x i16>, ptr [[A]], i64 [[INDVARS_IV]]
+; FORCE-TF-NEXT: store <8 x i16> [[RESULT]], ptr [[ARRAYIDX2]], align 16
+; FORCE-TF-NEXT: br label %[[FOR_INC]]
+; FORCE-TF: [[FOR_INC]]:
+; FORCE-TF-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; FORCE-TF-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1024
+; FORCE-TF-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.inc
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.inc ]
+ %arrayidx0 = getelementptr inbounds <8 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x i16>, ptr %arrayidx0, align 16
+ %arrayidx1 = getelementptr inbounds <8 x i16>, ptr %c, i64 %indvars.iv
+ %1 = load <8 x i16>, ptr %arrayidx1, align 16
+ %result = add <8 x i16> %0, %1
+ %do.access = icmp ne i64 %indvars.iv, %cond
+ br i1 %do.access, label %if.then, label %for.inc
+
+if.then:
+ %arrayidx2 = getelementptr inbounds <8 x i16>, ptr %a, i64 %indvars.iv
+ store <8 x i16> %result, ptr %arrayidx2, align 16
+ br label %for.inc
+
+for.inc:
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+; Predication should be disabled even if the loop does not define any vector
+; but takes on as input.
+define void @init_loop(ptr noalias nocapture noundef writeonly %a, <8 x i16> %v0, <8 x i16> %v1) {
+; CHECK-LABEL: define void @init_loop(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], <8 x i16> [[V0:%.*]], <8 x i16> [[V1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP0]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT: [[BROADCAST:%.*]] = call <vscale x 8 x i16> @llvm.vector.broadcast.nxv8i16.v8i16(<8 x i16> [[V0]])
+; CHECK-NEXT: [[BROADCAST1:%.*]] = call <vscale x 8 x i16> @llvm.vector.broadcast.nxv8i16.v8i16(<8 x i16> [[V1]])
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [16 x i16], ptr [[A]], i64 [[INDEX]], i64 0
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 16 x i16> @llvm.vector.interleave.segments2.nxv16i16(<vscale x 8 x i16> [[BROADCAST]], <vscale x 8 x i16> [[BROADCAST1]])
+; CHECK-NEXT: store <vscale x 16 x i16> [[INTERLEAVED_VEC]], ptr [[TMP2]], align 16
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+; FORCE-TF-LABEL: define void @init_loop(
+; FORCE-TF-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], <8 x i16> [[V0:%.*]], <8 x i16> [[V1:%.*]]) #[[ATTR0]] {
+; FORCE-TF-NEXT: [[ENTRY:.*]]:
+; FORCE-TF-NEXT: br label %[[FOR_BODY:.*]]
+; FORCE-TF: [[FOR_COND_CLEANUP:.*]]:
+; FORCE-TF-NEXT: ret void
+; FORCE-TF: [[FOR_BODY]]:
+; FORCE-TF-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; FORCE-TF-NEXT: [[ARRAYIDX_OUT0:%.*]] = getelementptr inbounds [16 x i16], ptr [[A]], i64 [[INDVARS_IV]], i64 0
+; FORCE-TF-NEXT: store <8 x i16> [[V0]], ptr [[ARRAYIDX_OUT0]], align 16
+; FORCE-TF-NEXT: [[ARRAYIDX_OUT1:%.*]] = getelementptr inbounds [16 x i16], ptr [[A]], i64 [[INDVARS_IV]], i64 8
+; FORCE-TF-NEXT: store <8 x i16> [[V1]], ptr [[ARRAYIDX_OUT1]], align 16
+; FORCE-TF-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; FORCE-TF-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1024
+; FORCE-TF-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx.out0 = getelementptr inbounds [16 x i16], ptr %a, i64 %indvars.iv, i64 0
+ store <8 x i16> %v0, ptr %arrayidx.out0, align 16
+ %arrayidx.out1 = getelementptr inbounds [16 x i16], ptr %a, i64 %indvars.iv, i64 8
+ store <8 x i16> %v1, ptr %arrayidx.out1, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-scatter.ll b/llvm/test/Transforms/LoopVectorize/AArch64/revec-scatter.ll
new file mode 100644
index 0000000000000..69642d90af06e
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/revec-scatter.ll
@@ -0,0 +1,231 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "scalar.ph:" --replace-value-regex "!llvm.loop ![0-9]+" --version 6
+; RUN: opt -passes=loop-vectorize -vectorize-vector-loops -lv-strided-pointer-ivs \
+; RUN: -force-vector-interleave=1 -scalable-vectorization=on -force-vector-width=1 \
+; RUN: -mtriple=aarch64 -mattr=+sve2p1 -S < %s | FileCheck %s
+
+
+define void @scatter_128(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b, i64 %stride) {
+; CHECK-LABEL: define void @scatter_128(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP8]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; CHECK: [[VECTOR_SCEVCHECK]]:
+; CHECK-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; CHECK-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP8]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP4:%.*]] = call <vscale x 1 x i64> @llvm.stepvector.nxv1i64()
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[TMP8]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 1 x i64> [ [[TMP4]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <8 x i16>, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = load <vscale x 8 x i16>, ptr [[TMP2]], align 16
+; CHECK-NEXT: [[TMP5:%.*]] = add <vscale x 8 x i16> [[WIDE_MASKED_GATHER]], splat (i16 1)
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i16, ptr [[A]], <vscale x 1 x i64> [[VEC_IND]]
+; CHECK-NEXT: call void @llvm.masked.segment.scatter.nxv8i16.nxv1p0(<vscale x 8 x i16> [[TMP5]], <vscale x 1 x ptr> align 16 [[TMP6]], <vscale x 1 x i1> splat (i1 true))
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 1 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx0 = getelementptr inbounds <8 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x i16>, ptr %arrayidx0, align 16
+ %result = add <8 x i16> %0, splat (i16 1)
+ %offset = mul i64 %indvars.iv, %stride
+ %arrayidx2 = getelementptr inbounds i16, ptr %a, i64 %offset
+ store <8 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @scatter_128_ptr_iv(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b, i64 %stride) {
+; CHECK-LABEL: define void @scatter_128_ptr_iv(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[STRIDE]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT1:%.*]] = shufflevector <vscale x 1 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shl <vscale x 1 x i64> [[BROADCAST_SPLAT1]], splat (i64 1)
+; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP1]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <vscale x 1 x i64> [[BROADCAST_SPLAT]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[N_VEC]], [[TMP11]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP3]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[A]], %[[VECTOR_PH]] ], [ [[PTR_IND:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 1 x i64> @llvm.stepvector.nxv1i64()
+; CHECK-NEXT: [[TMP6:%.*]] = mul <vscale x 1 x i64> [[TMP5]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <vscale x 1 x i64> [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds <8 x i16>, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = load <vscale x 8 x i16>, ptr [[TMP8]], align 16
+; CHECK-NEXT: [[TMP7:%.*]] = add <vscale x 8 x i16> [[WIDE_MASKED_GATHER]], splat (i16 1)
+; CHECK-NEXT: call void @llvm.masked.segment.scatter.nxv8i16.nxv1p0(<vscale x 8 x i16> [[TMP7]], <vscale x 1 x ptr> align 16 [[VECTOR_GEP]], <vscale x 1 x i1> splat (i1 true))
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <vscale x 1 x i64> [[BROADCAST_SPLAT]], i64 0
+; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP12]], [[TMP1]]
+; CHECK-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP9]]
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %addr.iv = phi ptr [ %a, %entry ], [ %addr.next, %for.body ]
+ %arrayidx0 = getelementptr inbounds <8 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x i16>, ptr %arrayidx0, align 16
+ %result = add <8 x i16> %0, splat (i16 1)
+ store <8 x i16> %result, ptr %addr.iv, align 16
+ %addr.next = getelementptr inbounds i16, ptr %addr.iv, i64 %stride
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+; TODO: Not supported because the two stores cannot be proven safe by LAA.
+; Store one quad at a[iv * 2 * stride]
+; and at a[iv * 2 * stride + stride]
+define void @scatter_two_128(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b, i64 %stride) {
+; CHECK-LABEL: define void @scatter_two_128(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP:.*]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[ARRAYIDX0:%.*]] = getelementptr inbounds <8 x i16>, ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <8 x i16>, ptr [[ARRAYIDX0]], align 16
+; CHECK-NEXT: [[RESULT:%.*]] = add <8 x i16> [[TMP0]], splat (i16 1)
+; CHECK-NEXT: [[OFFSET:%.*]] = mul i64 [[INDVARS_IV]], [[STRIDE]]
+; CHECK-NEXT: [[OFFSET2:%.*]] = mul i64 [[OFFSET]], 2
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 [[OFFSET2]]
+; CHECK-NEXT: store <8 x i16> [[RESULT]], ptr [[ARRAYIDX2]], align 16
+; CHECK-NEXT: [[OFFSET3:%.*]] = add i64 [[OFFSET2]], [[STRIDE]]
+; CHECK-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 [[OFFSET3]]
+; CHECK-NEXT: store <8 x i16> [[RESULT]], ptr [[ARRAYIDX3]], align 16
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1024
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx0 = getelementptr inbounds <8 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x i16>, ptr %arrayidx0, align 16
+ %result = add <8 x i16> %0, splat (i16 1)
+ %offset = mul i64 %indvars.iv, %stride
+ %offset2 = mul i64 %offset, 2
+ %arrayidx2 = getelementptr inbounds i16, ptr %a, i64 %offset2
+ store <8 x i16> %result, ptr %arrayidx2, align 16
+ %offset3 = add i64 %offset2, %stride
+ %arrayidx3 = getelementptr inbounds i16, ptr %a, i64 %offset3
+ store <8 x i16> %result, ptr %arrayidx3, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @scatter_128_predication(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b, ptr nocapture noundef readonly %flags, i64 %cond, i64 %stride) {
+; CHECK-LABEL: define void @scatter_128_predication(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], ptr noundef readonly captures(none) [[FLAGS:%.*]], i64 [[COND:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; CHECK: [[VECTOR_SCEVCHECK]]:
+; CHECK-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; CHECK-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP0]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[COND]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 1 x i64> @llvm.stepvector.nxv1i64()
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[TMP0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 1 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 1 x i64> [ [[TMP2]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds <8 x i16>, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 8 x i16>, ptr [[TMP6]], align 16
+; CHECK-NEXT: [[TMP7:%.*]] = add <vscale x 8 x i16> [[WIDE_LOAD1]], splat (i16 1)
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[FLAGS]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 1 x i64>, ptr [[TMP4]], align 8
+; CHECK-NEXT: [[TMP5:%.*]] = icmp ne <vscale x 1 x i64> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i16, ptr [[A]], <vscale x 1 x i64> [[VEC_IND]]
+; CHECK-NEXT: call void @llvm.masked.segment.scatter.nxv8i16.nxv1p0(<vscale x 8 x i16> [[TMP7]], <vscale x 1 x ptr> align 16 [[TMP8]], <vscale x 1 x i1> [[TMP5]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 1 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup:
+ ret void
+
+for.body:
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.inc ]
+ %arrayidx0 = getelementptr inbounds <8 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x i16>, ptr %arrayidx0, align 16
+ %result = add <8 x i16> %0, splat (i16 1)
+ %flagidx = getelementptr inbounds i64, ptr %flags, i64 %indvars.iv
+ %scl.val = load i64, ptr %flagidx, align 8
+ %do.access = icmp ne i64 %scl.val, %cond
+ br i1 %do.access, label %if.then, label %for.inc
+
+if.then:
+ %offset = mul i64 %indvars.iv, %stride
+ %arrayidx2 = getelementptr inbounds i16, ptr %a, i64 %offset
+ store <8 x i16> %result, ptr %arrayidx2, align 16
+ br label %for.inc
+
+for.inc:
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-select.ll b/llvm/test/Transforms/LoopVectorize/AArch64/revec-select.ll
new file mode 100644
index 0000000000000..5fcb8d8c83d88
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/revec-select.ll
@@ -0,0 +1,197 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph" --version 6
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -vectorize-vector-loops -mtriple=aarch64 -mattr=+sve2p1 -S < %s | FileCheck %s
+
+; Not vectorised because the condition is an unsupported type: <4 x i1>
+define void @select_vector_cond(ptr noalias writeonly %a, ptr readonly %b, ptr readonly %c, <4 x i1> %cond) {
+; CHECK-LABEL: define void @select_vector_cond(
+; CHECK-SAME: ptr noalias writeonly [[A:%.*]], ptr readonly [[B:%.*]], ptr readonly [[C:%.*]], <4 x i1> [[COND:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP0]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[ARRAYIDX_B:%.*]] = getelementptr inbounds <4 x i32>, ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[ARRAYIDX_C:%.*]] = getelementptr inbounds <4 x i32>, ptr [[C]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[ARRAYIDX_B]], align 16
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i32>, ptr [[ARRAYIDX_C]], align 16
+; CHECK-NEXT: [[BROADCAST:%.*]] = call <vscale x 4 x i1> @llvm.vector.broadcast.nxv4i1.v4i1(<4 x i1> [[COND]])
+; CHECK-NEXT: [[TMP4:%.*]] = select <vscale x 4 x i1> [[BROADCAST]], <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[WIDE_LOAD1]]
+; CHECK-NEXT: [[ARRAYIDX_A:%.*]] = getelementptr inbounds <4 x i32>, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: store <vscale x 4 x i32> [[TMP4]], ptr [[ARRAYIDX_A]], align 16
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDVARS_IV]], [[TMP0]]
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+
+ %arrayidx.b = getelementptr inbounds <4 x i32>, ptr %b, i64 %indvars.iv
+ %arrayidx.c = getelementptr inbounds <4 x i32>, ptr %c, i64 %indvars.iv
+ %v1 = load <4 x i32>, ptr %arrayidx.b
+ %v2 = load <4 x i32>, ptr %arrayidx.c
+ %res = select <4 x i1> %cond, <4 x i32> %v1, <4 x i32> %v2
+ %arrayidx.a = getelementptr inbounds <4 x i32>, ptr %a, i64 %indvars.iv
+ store <4 x i32> %res, ptr %arrayidx.a, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @test_fcmp_select(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b) {
+; CHECK-LABEL: define void @test_fcmp_select(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP0]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <8 x half>, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x half>, ptr [[TMP2]], align 16
+; CHECK-NEXT: [[TMP3:%.*]] = fcmp fast ule <vscale x 8 x half> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = select <vscale x 8 x i1> [[TMP3]], <vscale x 8 x half> [[WIDE_LOAD]], <vscale x 8 x half> zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds <8 x half>, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: store <vscale x 8 x half> [[TMP4]], ptr [[TMP5]], align 16
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <8 x half>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x half>, ptr %arrayidx, align 16
+ %resi1 = fcmp fast ule <8 x half> %0, zeroinitializer
+ %masked_res = select <8 x i1> %resi1, <8 x half> %0, <8 x half> zeroinitializer
+ %arrayidx2 = getelementptr inbounds <8 x half>, ptr %a, i64 %indvars.iv
+ store <8 x half> %masked_res, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @test_fcmp_chained_select(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b, <4 x i1> %cond) {
+; CHECK-LABEL: define void @test_fcmp_chained_select(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], <4 x i1> [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP0]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <4 x float>, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP2]], align 16
+; CHECK-NEXT: [[TMP3:%.*]] = fcmp fast ule <vscale x 4 x float> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[BROADCAST:%.*]] = call <vscale x 4 x i1> @llvm.vector.broadcast.nxv4i1.v4i1(<4 x i1> [[COND]])
+; CHECK-NEXT: [[TMP4:%.*]] = select <vscale x 4 x i1> [[BROADCAST]], <vscale x 4 x i1> zeroinitializer, <vscale x 4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = select <vscale x 4 x i1> [[TMP4]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x float> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds <4 x float>, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: store <vscale x 4 x float> [[TMP5]], ptr [[TMP6]], align 16
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x float>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x float>, ptr %arrayidx, align 16
+ %resi1 = fcmp fast ule <4 x float> %0, zeroinitializer
+ %resi1.masked = select <4 x i1> %cond, <4 x i1> zeroinitializer, <4 x i1> %resi1
+ %masked_res = select <4 x i1> %resi1.masked, <4 x float> %0, <4 x float> zeroinitializer
+ %arrayidx2 = getelementptr inbounds <4 x float>, ptr %a, i64 %indvars.iv
+ store <4 x float> %masked_res, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @select_scalar_cond(ptr noalias writeonly %a, ptr readonly %b, ptr readonly %c, i1 %cond) {
+; CHECK-LABEL: define void @select_scalar_cond(
+; CHECK-SAME: ptr noalias writeonly [[A:%.*]], ptr readonly [[B:%.*]], ptr readonly [[C:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP0]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <4 x i32>, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds <4 x i32>, ptr [[C]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP2]], align 16
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i32>, ptr [[TMP3]], align 16
+; CHECK-NEXT: [[TMP4:%.*]] = select i1 [[COND]], <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds <4 x i32>, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: store <vscale x 4 x i32> [[TMP4]], ptr [[TMP5]], align 16
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+
+ %arrayidx.b = getelementptr inbounds <4 x i32>, ptr %b, i64 %indvars.iv
+ %arrayidx.c = getelementptr inbounds <4 x i32>, ptr %c, i64 %indvars.iv
+ %v1 = load <4 x i32>, ptr %arrayidx.b
+ %v2 = load <4 x i32>, ptr %arrayidx.c
+ %res = select i1 %cond, <4 x i32> %v1, <4 x i32> %v2
+ %arrayidx.a = getelementptr inbounds <4 x i32>, ptr %a, i64 %indvars.iv
+ store <4 x i32> %res, ptr %arrayidx.a, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-shufflevector-ic-vf.ll b/llvm/test/Transforms/LoopVectorize/AArch64/revec-shufflevector-ic-vf.ll
new file mode 100644
index 0000000000000..d429f0449e8e5
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/revec-shufflevector-ic-vf.ll
@@ -0,0 +1,286 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph" --version 6
+; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=2 -vectorize-vector-loops \
+; RUN: -scalable-vectorization=on -force-vector-width=1 \
+; RUN: -mtriple=aarch64 -mattr=+sve2p1 -S | FileCheck %s --check-prefixes=CHECK,IC2,VScale1-IC2
+; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=1 -vectorize-vector-loops \
+; RUN: -scalable-vectorization=on -force-vector-width=2 \
+; RUN: -mtriple=aarch64 -mattr=+sve2p1 -S | FileCheck %s --check-prefixes=CHECK,VScale2-IC1
+; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=2 -vectorize-vector-loops \
+; RUN: -scalable-vectorization=on -force-vector-width=2 \
+; RUN: -mtriple=aarch64 -mattr=+sve2p1 -S | FileCheck %s --check-prefixes=CHECK,IC2,VScale2-IC2
+
+; Test how shufflevectors are re-vectorised for combinations of VF and IC.
+; In general, revec only supports VF = vscale x 1. But we want to make sure IC>1 works.
+; There are more tests in revec-shufflevector.ll for the standard VF = vscale x 1 case.
+
+
+define void @extract_shuffle_idx1(ptr noalias nocapture noundef writeonly %a, ptr readonly %b) {
+; IC2-LABEL: define void @extract_shuffle_idx1(
+; IC2-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr readonly [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; IC2-NEXT: [[ENTRY:.*:]]
+; IC2-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; IC2-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP4]], 1
+; IC2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
+; IC2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; IC2: [[VECTOR_PH]]:
+; IC2-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP4]], 1
+; IC2-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
+; IC2-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; IC2-NEXT: br label %[[VECTOR_BODY:.*]]
+; IC2: [[VECTOR_BODY]]:
+; IC2-NEXT: [[TMP0:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IC2-NEXT: [[TMP2:%.*]] = getelementptr inbounds <8 x i16>, ptr [[B]], i64 [[TMP0]]
+; IC2-NEXT: [[TMP7:%.*]] = getelementptr inbounds <8 x i16>, ptr [[TMP2]], i64 [[TMP4]]
+; IC2-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP2]], align 16
+; IC2-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 8 x i16>, ptr [[TMP7]], align 16
+; IC2-NEXT: [[TMP17:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave.segments2.nxv8i16(<vscale x 8 x i16> [[WIDE_LOAD]])
+; IC2-NEXT: [[TMP9:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[TMP17]], 1
+; IC2-NEXT: [[TMP10:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave.segments2.nxv8i16(<vscale x 8 x i16> [[WIDE_LOAD1]])
+; IC2-NEXT: [[TMP11:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[TMP10]], 1
+; IC2-NEXT: [[TMP8:%.*]] = getelementptr inbounds <4 x i16>, ptr [[A]], i64 [[TMP0]]
+; IC2-NEXT: [[TMP15:%.*]] = getelementptr inbounds <4 x i16>, ptr [[TMP8]], i64 [[TMP4]]
+; IC2-NEXT: store <vscale x 4 x i16> [[TMP9]], ptr [[TMP8]], align 16
+; IC2-NEXT: store <vscale x 4 x i16> [[TMP11]], ptr [[TMP15]], align 16
+; IC2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP0]], [[TMP3]]
+; IC2-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; IC2-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; IC2: [[MIDDLE_BLOCK]]:
+; IC2-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; IC2-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; IC2: [[SCALAR_PH]]:
+;
+; VScale2-IC1-LABEL: define void @extract_shuffle_idx1(
+; VScale2-IC1-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr readonly [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; VScale2-IC1-NEXT: [[ENTRY:.*:]]
+; VScale2-IC1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; VScale2-IC1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; VScale2-IC1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VScale2-IC1: [[VECTOR_PH]]:
+; VScale2-IC1-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP0]]
+; VScale2-IC1-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; VScale2-IC1-NEXT: br label %[[VECTOR_BODY:.*]]
+; VScale2-IC1: [[VECTOR_BODY]]:
+; VScale2-IC1-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VScale2-IC1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds <8 x i16>, ptr [[B]], i64 [[INDVARS_IV]]
+; VScale2-IC1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[ARRAYIDX]], align 16
+; VScale2-IC1-NEXT: [[TMP3:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave.segments2.nxv8i16(<vscale x 8 x i16> [[WIDE_LOAD]])
+; VScale2-IC1-NEXT: [[TMP4:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[TMP3]], 1
+; VScale2-IC1-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds <4 x i16>, ptr [[A]], i64 [[INDVARS_IV]]
+; VScale2-IC1-NEXT: store <vscale x 4 x i16> [[TMP4]], ptr [[ARRAYIDX2]], align 16
+; VScale2-IC1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDVARS_IV]], [[TMP0]]
+; VScale2-IC1-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VScale2-IC1-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; VScale2-IC1: [[MIDDLE_BLOCK]]:
+; VScale2-IC1-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; VScale2-IC1-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; VScale2-IC1: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <8 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x i16>, ptr %arrayidx, align 16
+ %result = shufflevector <8 x i16> %0, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+ %arrayidx2 = getelementptr inbounds <4 x i16>, ptr %a, i64 %indvars.iv
+ store <4 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @extract_shuffle_from_64bits(ptr noalias nocapture noundef writeonly %a, ptr readonly %b) {
+; CHECK-LABEL: define void @extract_shuffle_from_64bits(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr readonly [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP:.*]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds <4 x i16>, ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i16>, ptr [[ARRAYIDX]], align 16
+; CHECK-NEXT: [[RESULT:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds <2 x i16>, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: store <2 x i16> [[RESULT]], ptr [[ARRAYIDX2]], align 16
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1024
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i16>, ptr %arrayidx, align 16
+ %result = shufflevector <4 x i16> %0, <4 x i16> poison, <2 x i32> <i32 0, i32 1>
+ %arrayidx2 = getelementptr inbounds <2 x i16>, ptr %a, i64 %indvars.iv
+ store <2 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @unzip_even_same_src(ptr noalias nocapture noundef writeonly %a, ptr readonly %b) {
+; IC2-LABEL: define void @unzip_even_same_src(
+; IC2-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr readonly [[B:%.*]]) #[[ATTR0]] {
+; IC2-NEXT: [[ENTRY:.*:]]
+; IC2-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; IC2-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; IC2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
+; IC2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; IC2: [[VECTOR_PH]]:
+; IC2-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 1
+; IC2-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
+; IC2-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; IC2-NEXT: br label %[[VECTOR_BODY:.*]]
+; IC2: [[VECTOR_BODY]]:
+; IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IC2-NEXT: [[TMP4:%.*]] = getelementptr inbounds <8 x i16>, ptr [[B]], i64 [[INDEX]]
+; IC2-NEXT: [[TMP7:%.*]] = getelementptr inbounds <8 x i16>, ptr [[TMP4]], i64 [[TMP0]]
+; IC2-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP4]], align 16
+; IC2-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 8 x i16>, ptr [[TMP7]], align 16
+; IC2-NEXT: [[TMP8:%.*]] = call <vscale x 8 x i16> @llvm.vector.segmented.shuffle.nxv8i16.v8i32(<vscale x 8 x i16> [[WIDE_LOAD]], <vscale x 8 x i16> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 0, i32 2, i32 4, i32 6>)
+; IC2-NEXT: [[TMP9:%.*]] = call <vscale x 8 x i16> @llvm.vector.segmented.shuffle.nxv8i16.v8i32(<vscale x 8 x i16> [[WIDE_LOAD1]], <vscale x 8 x i16> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 0, i32 2, i32 4, i32 6>)
+; IC2-NEXT: [[TMP10:%.*]] = getelementptr inbounds <8 x i16>, ptr [[A]], i64 [[INDEX]]
+; IC2-NEXT: [[TMP13:%.*]] = getelementptr inbounds <8 x i16>, ptr [[TMP10]], i64 [[TMP0]]
+; IC2-NEXT: store <vscale x 8 x i16> [[TMP8]], ptr [[TMP10]], align 16
+; IC2-NEXT: store <vscale x 8 x i16> [[TMP9]], ptr [[TMP13]], align 16
+; IC2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; IC2-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; IC2-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; IC2: [[MIDDLE_BLOCK]]:
+; IC2-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; IC2-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; IC2: [[SCALAR_PH]]:
+;
+; VScale2-IC1-LABEL: define void @unzip_even_same_src(
+; VScale2-IC1-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr readonly [[B:%.*]]) #[[ATTR0]] {
+; VScale2-IC1-NEXT: [[ENTRY:.*:]]
+; VScale2-IC1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; VScale2-IC1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; VScale2-IC1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VScale2-IC1: [[VECTOR_PH]]:
+; VScale2-IC1-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP0]]
+; VScale2-IC1-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; VScale2-IC1-NEXT: br label %[[VECTOR_BODY:.*]]
+; VScale2-IC1: [[VECTOR_BODY]]:
+; VScale2-IC1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VScale2-IC1-NEXT: [[TMP2:%.*]] = getelementptr inbounds <8 x i16>, ptr [[B]], i64 [[INDEX]]
+; VScale2-IC1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP2]], align 16
+; VScale2-IC1-NEXT: [[TMP3:%.*]] = call <vscale x 8 x i16> @llvm.vector.segmented.shuffle.nxv8i16.v8i32(<vscale x 8 x i16> [[WIDE_LOAD]], <vscale x 8 x i16> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 0, i32 2, i32 4, i32 6>)
+; VScale2-IC1-NEXT: [[TMP4:%.*]] = getelementptr inbounds <8 x i16>, ptr [[A]], i64 [[INDEX]]
+; VScale2-IC1-NEXT: store <vscale x 8 x i16> [[TMP3]], ptr [[TMP4]], align 16
+; VScale2-IC1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; VScale2-IC1-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VScale2-IC1-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; VScale2-IC1: [[MIDDLE_BLOCK]]:
+; VScale2-IC1-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; VScale2-IC1-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; VScale2-IC1: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <8 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x i16>, ptr %arrayidx, align 16
+ %result = shufflevector <8 x i16> %0, <8 x i16> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 0, i32 2, i32 4, i32 6>
+ %arrayidx2 = getelementptr inbounds <8 x i16>, ptr %a, i64 %indvars.iv
+ store <8 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @unzip_even_same_src_64b(ptr noalias nocapture noundef writeonly %a, ptr readonly %b) {
+; IC2-LABEL: define void @unzip_even_same_src_64b(
+; IC2-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr readonly [[B:%.*]]) #[[ATTR0]] {
+; IC2-NEXT: [[ENTRY:.*:]]
+; IC2-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; IC2-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; IC2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
+; IC2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; IC2: [[VECTOR_PH]]:
+; IC2-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 1
+; IC2-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
+; IC2-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; IC2-NEXT: br label %[[VECTOR_BODY:.*]]
+; IC2: [[VECTOR_BODY]]:
+; IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IC2-NEXT: [[TMP4:%.*]] = getelementptr inbounds <4 x i16>, ptr [[B]], i64 [[INDEX]]
+; IC2-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i16>, ptr [[TMP4]], i64 [[TMP0]]
+; IC2-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP4]], align 16
+; IC2-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i16>, ptr [[TMP7]], align 16
+; IC2-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i16> @llvm.vector.segmented.shuffle.nxv4i16.v4i32(<vscale x 4 x i16> [[WIDE_LOAD]], <vscale x 4 x i16> poison, <4 x i32> <i32 0, i32 2, i32 0, i32 2>)
+; IC2-NEXT: [[TMP9:%.*]] = call <vscale x 4 x i16> @llvm.vector.segmented.shuffle.nxv4i16.v4i32(<vscale x 4 x i16> [[WIDE_LOAD1]], <vscale x 4 x i16> poison, <4 x i32> <i32 0, i32 2, i32 0, i32 2>)
+; IC2-NEXT: [[TMP10:%.*]] = getelementptr inbounds <4 x i16>, ptr [[A]], i64 [[INDEX]]
+; IC2-NEXT: [[TMP13:%.*]] = getelementptr inbounds <4 x i16>, ptr [[TMP10]], i64 [[TMP0]]
+; IC2-NEXT: store <vscale x 4 x i16> [[TMP8]], ptr [[TMP10]], align 16
+; IC2-NEXT: store <vscale x 4 x i16> [[TMP9]], ptr [[TMP13]], align 16
+; IC2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; IC2-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; IC2-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; IC2: [[MIDDLE_BLOCK]]:
+; IC2-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; IC2-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; IC2: [[SCALAR_PH]]:
+;
+; VScale2-IC1-LABEL: define void @unzip_even_same_src_64b(
+; VScale2-IC1-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr readonly [[B:%.*]]) #[[ATTR0]] {
+; VScale2-IC1-NEXT: [[ENTRY:.*:]]
+; VScale2-IC1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; VScale2-IC1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; VScale2-IC1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VScale2-IC1: [[VECTOR_PH]]:
+; VScale2-IC1-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP0]]
+; VScale2-IC1-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; VScale2-IC1-NEXT: br label %[[VECTOR_BODY:.*]]
+; VScale2-IC1: [[VECTOR_BODY]]:
+; VScale2-IC1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VScale2-IC1-NEXT: [[TMP2:%.*]] = getelementptr inbounds <4 x i16>, ptr [[B]], i64 [[INDEX]]
+; VScale2-IC1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP2]], align 16
+; VScale2-IC1-NEXT: [[TMP3:%.*]] = call <vscale x 4 x i16> @llvm.vector.segmented.shuffle.nxv4i16.v4i32(<vscale x 4 x i16> [[WIDE_LOAD]], <vscale x 4 x i16> poison, <4 x i32> <i32 0, i32 2, i32 0, i32 2>)
+; VScale2-IC1-NEXT: [[TMP4:%.*]] = getelementptr inbounds <4 x i16>, ptr [[A]], i64 [[INDEX]]
+; VScale2-IC1-NEXT: store <vscale x 4 x i16> [[TMP3]], ptr [[TMP4]], align 16
+; VScale2-IC1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; VScale2-IC1-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VScale2-IC1-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; VScale2-IC1: [[MIDDLE_BLOCK]]:
+; VScale2-IC1-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; VScale2-IC1-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; VScale2-IC1: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i16>, ptr %arrayidx, align 16
+ %result = shufflevector <4 x i16> %0, <4 x i16> poison, <4 x i32> <i32 0, i32 2, i32 0, i32 2>
+ %arrayidx2 = getelementptr inbounds <4 x i16>, ptr %a, i64 %indvars.iv
+ store <4 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; VScale1-IC2: {{.*}}
+; VScale2-IC2: {{.*}}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-shufflevector-ifconvert.ll b/llvm/test/Transforms/LoopVectorize/AArch64/revec-shufflevector-ifconvert.ll
new file mode 100644
index 0000000000000..4970f3d27f87f
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/revec-shufflevector-ifconvert.ll
@@ -0,0 +1,28 @@
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -vectorize-vector-loops -mtriple=aarch64 -mattr=+sve2p1 -S < %s | FileCheck %s
+
+; Make sure phi nodes that are operands of shufflevectors get correctly blended.
+
+define void @blend_simple_phi(ptr noalias %dst, i64 %conv4.i11, <8 x i16> %in) {
+; CHECK-LABEL: define void @blend_simple_phi(
+; CHECK: vector.ph:
+; CHECK: [[BROADCAST:%.*]] = call <vscale x 8 x i16> @llvm.vector.broadcast.nxv8i16.v8i16(<8 x i16> %in)
+; CHECK: segmented.shuffle.nxv8i16.v8i32(<vscale x 8 x i16> [[BROADCAST]]
+entry:
+ br label %do.body29.i
+
+do.body29.i: ; preds = %if.end57.i, %entry
+ %indvars.iv78.i = phi i64 [ 0, %entry ], [ %indvars.iv.next79.i, %if.end57.i ]
+ br label %if.end57.i
+
+if.end57.i: ; preds = %do.body29.i
+ %a030.1.i = phi <8 x i16> [ %in, %do.body29.i ]
+ %shuffle.i4.i.i44 = shufflevector <8 x i16> %a030.1.i, <8 x i16> %a030.1.i, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 0, i32 2, i32 4, i32 6>
+ %arrayidx2 = getelementptr inbounds <8 x i16>, ptr %dst, i64 %indvars.iv78.i
+ store <8 x i16> %shuffle.i4.i.i44, ptr %arrayidx2, align 2
+ %indvars.iv.next79.i = add i64 %indvars.iv78.i, 1
+ %cmp87.i = icmp slt i64 %indvars.iv78.i, %conv4.i11
+ br i1 %cmp87.i, label %do.body29.i, label %loopexit.i
+
+loopexit.i: ; preds = %if.end57.i
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-shufflevector-sve-version.ll b/llvm/test/Transforms/LoopVectorize/AArch64/revec-shufflevector-sve-version.ll
new file mode 100644
index 0000000000000..0b26c4db85dc6
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/revec-shufflevector-sve-version.ll
@@ -0,0 +1,117 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph" --version 6
+; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=1 -vectorize-vector-loops \
+; RUN: -scalable-vectorization=on -force-vector-width=1 \
+; RUN: -mtriple=aarch64 -mattr=+sve -S | FileCheck %s --check-prefixes=CHECK,SVE1
+; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=1 -vectorize-vector-loops \
+; RUN: -scalable-vectorization=on -force-vector-width=1 \
+; RUN: -mtriple=aarch64 -mattr=+sve2p1 -S | FileCheck %s --check-prefixes=CHECK,SVE2p1
+
+; Test how shufflevectors are re-vectorised depending on the SVE version.
+
+define void @extract_shuffle_idx1(ptr noalias nocapture noundef writeonly %a, ptr readonly %b) {
+; CHECK-LABEL: define void @extract_shuffle_idx1(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr readonly [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP0]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <8 x i16>, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP2]], align 16
+; CHECK-NEXT: [[TMP3:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave.segments2.nxv8i16(<vscale x 8 x i16> [[WIDE_LOAD]])
+; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[TMP3]], 1
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds <4 x i16>, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: store <vscale x 4 x i16> [[TMP4]], ptr [[TMP5]], align 16
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <8 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x i16>, ptr %arrayidx, align 16
+ %result = shufflevector <8 x i16> %0, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+ %arrayidx2 = getelementptr inbounds <4 x i16>, ptr %a, i64 %indvars.iv
+ store <4 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+
+define void @unzip_even_same_src(ptr noalias nocapture noundef writeonly %a, ptr readonly %b) {
+; SVE1-LABEL: define void @unzip_even_same_src(
+; SVE1-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr readonly [[B:%.*]]) #[[ATTR0]] {
+; SVE1-NEXT: [[ENTRY:.*]]:
+; SVE1-NEXT: br label %[[FOR_BODY:.*]]
+; SVE1: [[FOR_COND_CLEANUP:.*]]:
+; SVE1-NEXT: ret void
+; SVE1: [[FOR_BODY]]:
+; SVE1-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; SVE1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds <8 x i16>, ptr [[B]], i64 [[INDVARS_IV]]
+; SVE1-NEXT: [[TMP0:%.*]] = load <8 x i16>, ptr [[ARRAYIDX]], align 16
+; SVE1-NEXT: [[RESULT:%.*]] = shufflevector <8 x i16> [[TMP0]], <8 x i16> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 0, i32 2, i32 4, i32 6>
+; SVE1-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds <8 x i16>, ptr [[A]], i64 [[INDVARS_IV]]
+; SVE1-NEXT: store <8 x i16> [[RESULT]], ptr [[ARRAYIDX2]], align 16
+; SVE1-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; SVE1-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1024
+; SVE1-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+; SVE2p1-LABEL: define void @unzip_even_same_src(
+; SVE2p1-SAME: ptr noalias noundef writeonly captures(none) [[A:%.*]], ptr readonly [[B:%.*]]) #[[ATTR0]] {
+; SVE2p1-NEXT: [[ENTRY:.*:]]
+; SVE2p1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; SVE2p1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
+; SVE2p1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; SVE2p1: [[VECTOR_PH]]:
+; SVE2p1-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP0]]
+; SVE2p1-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; SVE2p1-NEXT: br label %[[VECTOR_BODY:.*]]
+; SVE2p1: [[VECTOR_BODY]]:
+; SVE2p1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SVE2p1-NEXT: [[TMP2:%.*]] = getelementptr inbounds <8 x i16>, ptr [[B]], i64 [[INDEX]]
+; SVE2p1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP2]], align 16
+; SVE2p1-NEXT: [[TMP3:%.*]] = call <vscale x 8 x i16> @llvm.vector.segmented.shuffle.nxv8i16.v8i32(<vscale x 8 x i16> [[WIDE_LOAD]], <vscale x 8 x i16> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 0, i32 2, i32 4, i32 6>)
+; SVE2p1-NEXT: [[TMP4:%.*]] = getelementptr inbounds <8 x i16>, ptr [[A]], i64 [[INDEX]]
+; SVE2p1-NEXT: store <vscale x 8 x i16> [[TMP3]], ptr [[TMP4]], align 16
+; SVE2p1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; SVE2p1-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; SVE2p1-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; SVE2p1: [[MIDDLE_BLOCK]]:
+; SVE2p1-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; SVE2p1-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; SVE2p1: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <8 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x i16>, ptr %arrayidx, align 16
+ %result = shufflevector <8 x i16> %0, <8 x i16> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 0, i32 2, i32 4, i32 6>
+ %arrayidx2 = getelementptr inbounds <8 x i16>, ptr %a, i64 %indvars.iv
+ store <8 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/revec-shufflevector.ll b/llvm/test/Transforms/LoopVectorize/AArch64/revec-shufflevector.ll
new file mode 100644
index 0000000000000..71cf5e18a9a40
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/revec-shufflevector.ll
@@ -0,0 +1,370 @@
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -vectorize-vector-loops -mtriple=aarch64 -mattr=+sve2p1 -S < %s | FileCheck %s
+
+; Test how shufflevectors are re-vectorised using HVLA.
+
+define void @unzip_even_same_src(ptr noalias nocapture noundef writeonly %a, ptr readonly %b) {
+; CHECK-LABEL: define void @unzip_even_same_src(
+; CHECK: vector.body:
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>
+; CHECK: call <vscale x 8 x i16> @llvm.vector.segmented.shuffle.nxv8i16.v8i32(<vscale x 8 x i16> [[WIDE_LOAD]], <vscale x 8 x i16> poison
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <8 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x i16>, ptr %arrayidx, align 16
+ %result = shufflevector <8 x i16> %0, <8 x i16> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 0, i32 2, i32 4, i32 6>
+ %arrayidx2 = getelementptr inbounds <8 x i16>, ptr %a, i64 %indvars.iv
+ store <8 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @unzip_odd(ptr noalias nocapture noundef writeonly %a, ptr readonly %b, ptr readonly %c) {
+; CHECK-LABEL: define void @unzip_odd(
+; CHECK: vector.body:
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>
+; CHECK: [[WIDE_LOAD2:%.*]] = load <vscale x 8 x i16>
+; CHECK: call <vscale x 8 x i16> @llvm.vector.segmented.shuffle.nxv8i16.v8i32(<vscale x 8 x i16> [[WIDE_LOAD]], <vscale x 8 x i16> [[WIDE_LOAD2]]
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <8 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x i16>, ptr %arrayidx, align 16
+ %arrayidx1 = getelementptr inbounds <8 x i16>, ptr %c, i64 %indvars.iv
+ %1 = load <8 x i16>, ptr %arrayidx1, align 16
+ %result = shufflevector <8 x i16> %0, <8 x i16> %1, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+ %arrayidx2 = getelementptr inbounds <8 x i16>, ptr %a, i64 %indvars.iv
+ store <8 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @trn1_single_src(ptr noalias nocapture noundef writeonly %a, ptr readonly %b) {
+; CHECK-LABEL: define void @trn1_single_src(
+; CHECK: vector.body:
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>
+; CHECK: call <vscale x 8 x i16> @llvm.vector.segmented.shuffle.nxv8i16.v8i32(<vscale x 8 x i16> [[WIDE_LOAD]], <vscale x 8 x i16> poison
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <8 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x i16>, ptr %arrayidx, align 16
+ %result = shufflevector <8 x i16> %0, <8 x i16> poison, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
+ %arrayidx2 = getelementptr inbounds <8 x i16>, ptr %a, i64 %indvars.iv
+ store <8 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @duplicate_element(ptr noalias nocapture noundef writeonly %a, ptr readonly %b) {
+; CHECK-LABEL: define void @duplicate_element(
+; CHECK: vector.body:
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>
+; CHECK: call <vscale x 8 x i16> @llvm.vector.segmented.shuffle.nxv8i16.v8i32(<vscale x 8 x i16> [[WIDE_LOAD]], <vscale x 8 x i16> poison
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <8 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x i16>, ptr %arrayidx, align 16
+ %result = shufflevector <8 x i16> %0, <8 x i16> poison, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+ %arrayidx2 = getelementptr inbounds <8 x i16>, ptr %a, i64 %indvars.iv
+ store <8 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @duplicate_element_64bit(ptr noalias nocapture noundef writeonly %a, ptr readonly %b) {
+; CHECK-LABEL: define void @duplicate_element_64bit(
+; CHECK: vector.body:
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>
+; CHECK: call <vscale x 4 x i16> @llvm.vector.segmented.shuffle.nxv4i16.v4i32(<vscale x 4 x i16> [[WIDE_LOAD]], <vscale x 4 x i16> poison, <4 x i32> splat (i32 3))
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i16>, ptr %arrayidx, align 16
+ %result = shufflevector <4 x i16> %0, <4 x i16> poison, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+ %arrayidx2 = getelementptr inbounds <4 x i16>, ptr %a, i64 %indvars.iv
+ store <4 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+; Cannot re-vectorise "duplicate element" shuffles that change size.
+define void @duplicate_element_64bit_to_128bit(ptr noalias nocapture noundef writeonly %a, ptr readonly %b) {
+; CHECK-LABEL: define void @duplicate_element_64bit_to_128bit(
+; CHECK-NOT: vector.body:
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i16>, ptr %arrayidx, align 16
+ %result = shufflevector <4 x i16> %0, <4 x i16> poison, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+ %arrayidx2 = getelementptr inbounds <8 x i16>, ptr %a, i64 %indvars.iv
+ store <8 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+; For uniform shuffles, the resulting code is found in the preheader.
+define void @duplicate_element_uniform(ptr noalias nocapture noundef writeonly %a, <8 x i16> %0) {
+; CHECK-LABEL: define void @duplicate_element_uniform(
+; CHECK: vector.ph:
+; CHECK: [[BROADCAST:%.*]] = call <vscale x 8 x i16> @llvm.vector.broadcast.nxv8i16.v8i16
+; CHECK: call <vscale x 8 x i16> @llvm.vector.segmented.shuffle.nxv8i16.v8i32(<vscale x 8 x i16> [[BROADCAST]], <vscale x 8 x i16> poison, <8 x i32> splat (i32 3))
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %result = shufflevector <8 x i16> %0, <8 x i16> poison, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+ %arrayidx2 = getelementptr inbounds <8 x i16>, ptr %a, i64 %indvars.iv
+ store <8 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @concat_shuffle(ptr noalias nocapture noundef writeonly %a, ptr readonly %b) {
+; CHECK-LABEL: define void @concat_shuffle(
+; CHECK: vector.body:
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>
+; CHECK: [[INT_SEGS:%.*]] = call <vscale x 8 x i16> @llvm.vector.interleave.segments2.nxv8i16(<vscale x 4 x i16> [[WIDE_LOAD]], <vscale x 4 x i16> splat (i16 1))
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i16>, ptr %arrayidx, align 16
+ %result = shufflevector <4 x i16> %0, <4 x i16> splat (i16 1), <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %arrayidx2 = getelementptr inbounds <8 x i16>, ptr %a, i64 %indvars.iv
+ store <8 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @concat_shuffle_undef(ptr noalias nocapture noundef writeonly %a, ptr readonly %b) {
+; CHECK-LABEL: define void @concat_shuffle_undef(
+; CHECK: vector.body:
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>
+; CHECK: [[INT_SEGS:%.*]] = call <vscale x 8 x i16> @llvm.vector.interleave.segments2.nxv8i16(<vscale x 4 x i16> [[WIDE_LOAD]], <vscale x 4 x i16> poison)
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i16>, ptr %arrayidx, align 16
+ %result = shufflevector <4 x i16> %0, <4 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %arrayidx2 = getelementptr inbounds <8 x i16>, ptr %a, i64 %indvars.iv
+ store <8 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @concat_shuffle_self(ptr noalias nocapture noundef writeonly %a, ptr readonly %b) {
+; CHECK-LABEL: define void @concat_shuffle_self(
+; CHECK: vector.body:
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>
+; CHECK: [[INT_SEGS:%.*]] = call <vscale x 8 x i16> @llvm.vector.interleave.segments2.nxv8i16(<vscale x 4 x i16> [[WIDE_LOAD]], <vscale x 4 x i16> [[WIDE_LOAD]])
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i16>, ptr %arrayidx, align 16
+ %result = shufflevector <4 x i16> %0, <4 x i16> poison, <8 x i32> <i32 0, i32 poison, i32 2, i32 3, i32 poison, i32 1, i32 2, i32 3>
+ %arrayidx2 = getelementptr inbounds <8 x i16>, ptr %a, i64 %indvars.iv
+ store <8 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @extract_shuffle_idx0(ptr noalias nocapture noundef writeonly %a, ptr readonly %b) {
+; CHECK-LABEL: define void @extract_shuffle_idx0(
+; CHECK: vector.body:
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>
+; CHECK: [[DEINT:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave.segments2.nxv8i16(<vscale x 8 x i16> [[WIDE_LOAD]])
+; CHECK: extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[DEINT]], 0
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <8 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x i16>, ptr %arrayidx, align 16
+ %result = shufflevector <8 x i16> %0, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ %arrayidx2 = getelementptr inbounds <4 x i16>, ptr %a, i64 %indvars.iv
+ store <4 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @extract_shuffle_idx1(ptr noalias nocapture noundef writeonly %a, ptr readonly %b) {
+; CHECK-LABEL: define void @extract_shuffle_idx1(
+; CHECK: vector.body:
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>
+; CHECK: [[DEINT:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave.segments2.nxv8i16(<vscale x 8 x i16> [[WIDE_LOAD]])
+; CHECK: extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[DEINT]], 1
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <8 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x i16>, ptr %arrayidx, align 16
+ %result = shufflevector <8 x i16> %0, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+ %arrayidx2 = getelementptr inbounds <4 x i16>, ptr %a, i64 %indvars.iv
+ store <4 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+; Cannot re-vectorise "unaligned" extracts
+define void @extract_shuffle_invalid_idx(ptr noalias nocapture noundef writeonly %a, ptr readonly %b) {
+; CHECK-LABEL: define void @extract_shuffle_invalid_idx(
+; CHECK-NOT: vector.body:
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <8 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x i16>, ptr %arrayidx, align 16
+ %result = shufflevector <8 x i16> %0, <8 x i16> poison, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+ %arrayidx2 = getelementptr inbounds <4 x i16>, ptr %a, i64 %indvars.iv
+ store <4 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+; Cannot re-vectorise "quarter" extracts
+define void @extract_shuffle_factor4(ptr noalias nocapture noundef writeonly %a, ptr readonly %b) {
+; CHECK-LABEL: define void @extract_shuffle_factor4(
+; CHECK-NOT: vector.body:
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <8 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x i16>, ptr %arrayidx, align 16
+ %result = shufflevector <8 x i16> %0, <8 x i16> poison, <2 x i32> <i32 0, i32 1>
+ %arrayidx2 = getelementptr inbounds <2 x i16>, ptr %a, i64 %indvars.iv
+ store <2 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+; Cannot re-vectorise extracts from 64-bit NEON
+define void @extract_shuffle_from_64bits(ptr noalias nocapture noundef writeonly %a, ptr readonly %b) {
+; CHECK-LABEL: define void @extract_shuffle_from_64bits(
+; CHECK-NOT: vector.body:
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <4 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <4 x i16>, ptr %arrayidx, align 16
+ %result = shufflevector <4 x i16> %0, <4 x i16> poison, <2 x i32> <i32 0, i32 1>
+ %arrayidx2 = getelementptr inbounds <2 x i16>, ptr %a, i64 %indvars.iv
+ store <2 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @arbitrary_shuffle(ptr noalias nocapture noundef writeonly %a, ptr readonly %b) {
+; CHECK-LABEL: define void @arbitrary_shuffle(
+; CHECK: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>
+; CHECK: call <vscale x 8 x i16> @llvm.vector.segmented.shuffle.nxv8i16.v8i32(<vscale x 8 x i16> [[WIDE_LOAD]], <vscale x 8 x i16> poison
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds <8 x i16>, ptr %b, i64 %indvars.iv
+ %0 = load <8 x i16>, ptr %arrayidx, align 16
+ %result = shufflevector <8 x i16> %0, <8 x i16> poison, <8 x i32> <i32 3, i32 2, i32 3, i32 6, i32 3, i32 1, i32 3, i32 0>
+ %arrayidx2 = getelementptr inbounds <8 x i16>, ptr %a, i64 %indvars.iv
+ store <8 x i16> %result, ptr %arrayidx2, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-alloca.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-alloca.ll
index 0ef03c58be97a..57f407cf99be5 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-alloca.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-alloca.ll
@@ -3,7 +3,6 @@
; CHECK-REMARKS: UserVF ignored because of invalid costs.
; CHECK-REMARKS: Recipe with invalid costs prevented vectorization at VF=(vscale x 1, vscale x 2): alloca
-; CHECK-REMARKS: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): store
define void @alloca(ptr %vla, i64 %N) {
; CHECK-LABEL: @alloca(
; CHECK-NOT: <vscale x
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-call.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-call.ll
index 0459b98f2eb1d..ba6cd9c318fe6 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-call.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-call.ll
@@ -101,9 +101,8 @@ for.end:
}
; CHECK-REMARKS: UserVF ignored because of invalid costs.
-; CHECK-REMARKS-NEXT: t.c:3:10: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): load
; CHECK-REMARKS-NEXT: t.c:3:20: Recipe with invalid costs prevented vectorization at VF=(vscale x 1, vscale x 2): call to llvm.sin
-; CHECK-REMARKS-NEXT: t.c:3:30: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): store
+; CHECK-REMARKS-NOT: Recipe with invalid costs
define void @vec_sin_no_mapping(ptr noalias nocapture %dst, ptr noalias nocapture readonly %src, i64 %n) {
; CHECK: @vec_sin_no_mapping
; CHECK: call fast <2 x float> @llvm.sin.v2f32
@@ -127,11 +126,10 @@ for.cond.cleanup:
}
; CHECK-REMARKS: UserVF ignored because of invalid costs.
-; CHECK-REMARKS-NEXT: t.c:3:10: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): load
; CHECK-REMARKS-NEXT: t.c:3:30: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): fadd
; CHECK-REMARKS-NEXT: t.c:3:30: Recipe with invalid costs prevented vectorization at VF=(vscale x 1, vscale x 2): call to llvm.sin
; CHECK-REMARKS-NEXT: t.c:3:20: Recipe with invalid costs prevented vectorization at VF=(vscale x 1, vscale x 2): call to llvm.sin
-; CHECK-REMARKS-NEXT: t.c:3:40: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): store
+; CHECK-REMARKS-NOT: Recipe with invalid costs
define void @vec_sin_no_mapping_ite(ptr noalias nocapture %dst, ptr noalias nocapture readonly %src, i64 %n) {
; CHECK: @vec_sin_no_mapping_ite
; CHECK-NOT: <vscale x
@@ -165,9 +163,8 @@ for.cond.cleanup:
}
; CHECK-REMARKS: UserVF ignored because of invalid costs.
-; CHECK-REMARKS-NEXT: t.c:3:10: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): load
; CHECK-REMARKS-NEXT: t.c:3:20: Recipe with invalid costs prevented vectorization at VF=(vscale x 1, vscale x 2): call to llvm.sin
-; CHECK-REMARKS-NEXT: t.c:3:30: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): store
+; CHECK-REMARKS-NOT: Recipe with invalid costs
define void @vec_sin_fixed_mapping(ptr noalias nocapture %dst, ptr noalias nocapture readonly %src, i64 %n) {
; CHECK: @vec_sin_fixed_mapping
; CHECK: call fast <2 x float> @llvm.sin.v2f32
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/type-shrinkage-zext-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/type-shrinkage-zext-costs.ll
index b5c8c922a24d3..53f91dedaa25f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/type-shrinkage-zext-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/type-shrinkage-zext-costs.ll
@@ -13,7 +13,7 @@ define void @zext_i8_i16(ptr noalias nocapture readonly %p, ptr noalias nocaptur
; CHECK-COST: Cost of 1 for VF 4: WIDEN-CAST ir<%conv> = zext ir<%0> to i16
; CHECK-COST: Cost of 1 for VF 8: WIDEN-CAST ir<%conv> = zext ir<%0> to i16
; CHECK-COST: Cost of 2 for VF 16: WIDEN-CAST ir<%conv> = zext ir<%0> to i16
-; CHECK-COST: Cost of 1 for VF vscale x 1: WIDEN-CAST ir<%conv> = zext ir<%0> to i16
+; CHECK-COST: Cost of Invalid for VF vscale x 1: WIDEN-CAST ir<%conv> = zext ir<%0> to i16
; CHECK-COST: Cost of 1 for VF vscale x 2: WIDEN-CAST ir<%conv> = zext ir<%0> to i16
; CHECK-COST: Cost of 1 for VF vscale x 4: WIDEN-CAST ir<%conv> = zext ir<%0> to i16
; CHECK-COST: Cost of 0 for VF vscale x 8: WIDEN-CAST ir<%conv> = zext ir<%0> to i16
@@ -92,7 +92,7 @@ define void @sext_i8_i16(ptr noalias nocapture readonly %p, ptr noalias nocaptur
; CHECK-COST: Cost of 1 for VF 4: WIDEN-CAST ir<%conv> = sext ir<%0> to i16
; CHECK-COST: Cost of 1 for VF 8: WIDEN-CAST ir<%conv> = sext ir<%0> to i16
; CHECK-COST: Cost of 2 for VF 16: WIDEN-CAST ir<%conv> = sext ir<%0> to i16
-; CHECK-COST: Cost of 1 for VF vscale x 1: WIDEN-CAST ir<%conv> = sext ir<%0> to i16
+; CHECK-COST: Cost of Invalid for VF vscale x 1: WIDEN-CAST ir<%conv> = sext ir<%0> to i16
; CHECK-COST: Cost of 1 for VF vscale x 2: WIDEN-CAST ir<%conv> = sext ir<%0> to i16
; CHECK-COST: Cost of 1 for VF vscale x 4: WIDEN-CAST ir<%conv> = sext ir<%0> to i16
; CHECK-COST: Cost of 0 for VF vscale x 8: WIDEN-CAST ir<%conv> = sext ir<%0> to i16
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/vector-loop-64bit.ll b/llvm/test/Transforms/LoopVectorize/AArch64/vector-loop-64bit.ll
new file mode 100644
index 0000000000000..75569363b60c3
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/vector-loop-64bit.ll
@@ -0,0 +1,134 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -mattr=+sve2 -passes=loop-vectorize -vectorize-vector-loops -S < %s | FileCheck %s -check-prefixes=VECTORIZE,VECTORIZE_SVE2
+; RUN: opt -mattr=+sve2p2 -passes=loop-vectorize -vectorize-vector-loops -S < %s | FileCheck %s -check-prefixes=VECTORIZE,VECTORIZE_SVE2p2
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32"
+target triple = "aarch64-unknown-linux-gnu"
+
+; A simple loop with 64-bit vectors.
+
+; Function Attrs: nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable
+define dso_local void @loop(ptr noalias nocapture noundef writeonly %dst, ptr nocapture noundef readonly %src1, ptr nocapture noundef readonly %src2, i32 noundef %count) local_unnamed_addr #0 {
+; VECTORIZE-LABEL: define dso_local void @loop(
+; VECTORIZE-SAME: ptr noalias noundef writeonly captures(none) [[DST:%.*]], ptr noundef readonly captures(none) [[SRC1:%.*]], ptr noundef readonly captures(none) [[SRC2:%.*]], i32 noundef [[COUNT:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; VECTORIZE-NEXT: [[ENTRY:.*:]]
+; VECTORIZE-NEXT: [[CMP9:%.*]] = icmp sgt i32 [[COUNT]], 0
+; VECTORIZE-NEXT: br i1 [[CMP9]], label %[[VECTOR_PH:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; VECTORIZE: [[VECTOR_PH]]:
+; VECTORIZE-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[COUNT]] to i64
+; VECTORIZE-NEXT: [[N_VEC:%.*]] = and i64 [[WIDE_TRIP_COUNT]], 2147483644
+; VECTORIZE-NEXT: [[TMP0:%.*]] = add nsw i64 [[N_VEC]], -2
+; VECTORIZE-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 1
+; VECTORIZE-NEXT: [[TMP2:%.*]] = add nuw i64 [[TMP1]], 1
+; VECTORIZE-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; VECTORIZE-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 2
+; VECTORIZE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP4]]
+; VECTORIZE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH1:.*]]
+; VECTORIZE: [[VECTOR_PH1]]:
+; VECTORIZE-NEXT: [[TMP10:%.*]] = shl nuw i64 [[TMP3]], 1
+; VECTORIZE-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP10]], 1
+; VECTORIZE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP6]]
+; VECTORIZE-NEXT: [[N_VEC2:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
+; VECTORIZE-NEXT: [[TMP7:%.*]] = shl i64 [[N_VEC2]], 1
+; VECTORIZE-NEXT: br label %[[VECTOR_BODY3:.*]]
+; VECTORIZE: [[VECTOR_BODY3]]:
+; VECTORIZE-NEXT: [[INDEX4:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT9:%.*]], %[[VECTOR_BODY3]] ]
+; VECTORIZE-NEXT: [[INDEX:%.*]] = shl i64 [[INDEX4]], 1
+; VECTORIZE-NEXT: [[TMP28:%.*]] = getelementptr inbounds float, ptr [[SRC1]], i64 [[INDEX]]
+; VECTORIZE-NEXT: [[TMP11:%.*]] = getelementptr inbounds <2 x float>, ptr [[TMP28]], i64 [[TMP10]]
+; VECTORIZE-NEXT: [[WIDE_LOAD5:%.*]] = load <vscale x 4 x float>, ptr [[TMP28]], align 4, !tbaa [[TBAA0:![0-9]+]]
+; VECTORIZE-NEXT: [[WIDE_LOAD6:%.*]] = load <vscale x 4 x float>, ptr [[TMP11]], align 4, !tbaa [[TBAA0]]
+; VECTORIZE-NEXT: [[TMP29:%.*]] = getelementptr inbounds float, ptr [[SRC2]], i64 [[INDEX]]
+; VECTORIZE-NEXT: [[TMP12:%.*]] = getelementptr inbounds <2 x float>, ptr [[TMP29]], i64 [[TMP10]]
+; VECTORIZE-NEXT: [[WIDE_LOAD7:%.*]] = load <vscale x 4 x float>, ptr [[TMP29]], align 4, !tbaa [[TBAA0]]
+; VECTORIZE-NEXT: [[WIDE_LOAD8:%.*]] = load <vscale x 4 x float>, ptr [[TMP12]], align 4, !tbaa [[TBAA0]]
+; VECTORIZE-NEXT: [[TMP16:%.*]] = fadd <vscale x 4 x float> [[WIDE_LOAD5]], [[WIDE_LOAD7]]
+; VECTORIZE-NEXT: [[TMP17:%.*]] = fadd <vscale x 4 x float> [[WIDE_LOAD6]], [[WIDE_LOAD8]]
+; VECTORIZE-NEXT: [[TMP18:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[INDEX]]
+; VECTORIZE-NEXT: [[TMP19:%.*]] = getelementptr inbounds <2 x float>, ptr [[TMP18]], i64 [[TMP10]]
+; VECTORIZE-NEXT: store <vscale x 4 x float> [[TMP16]], ptr [[TMP18]], align 4, !tbaa [[TBAA0]]
+; VECTORIZE-NEXT: store <vscale x 4 x float> [[TMP17]], ptr [[TMP19]], align 4, !tbaa [[TBAA0]]
+; VECTORIZE-NEXT: [[INDEX_NEXT9]] = add nuw i64 [[INDEX4]], [[TMP6]]
+; VECTORIZE-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT9]], [[N_VEC2]]
+; VECTORIZE-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY3]], !llvm.loop [[LOOP4:![0-9]+]]
+; VECTORIZE: [[MIDDLE_BLOCK]]:
+; VECTORIZE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC2]]
+; VECTORIZE-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; VECTORIZE: [[SCALAR_PH]]:
+; VECTORIZE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_PH]] ]
+; VECTORIZE-NEXT: br label %[[VECTOR_BODY:.*]]
+; VECTORIZE: [[VECTOR_BODY]]:
+; VECTORIZE-NEXT: [[INDEX1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VECTORIZE-NEXT: [[TMP23:%.*]] = getelementptr inbounds float, ptr [[SRC1]], i64 [[INDEX1]]
+; VECTORIZE-NEXT: [[WIDE_LOAD:%.*]] = load <2 x float>, ptr [[TMP23]], align 4, !tbaa [[TBAA0]]
+; VECTORIZE-NEXT: [[TMP24:%.*]] = getelementptr inbounds float, ptr [[SRC2]], i64 [[INDEX1]]
+; VECTORIZE-NEXT: [[WIDE_LOAD12:%.*]] = load <2 x float>, ptr [[TMP24]], align 4, !tbaa [[TBAA0]]
+; VECTORIZE-NEXT: [[TMP20:%.*]] = fadd <2 x float> [[WIDE_LOAD]], [[WIDE_LOAD12]]
+; VECTORIZE-NEXT: [[TMP26:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[INDEX1]]
+; VECTORIZE-NEXT: store <2 x float> [[TMP20]], ptr [[TMP26]], align 4, !tbaa [[TBAA0]]
+; VECTORIZE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX1]], 2
+; VECTORIZE-NEXT: [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VECTORIZE-NEXT: br i1 [[TMP32]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; VECTORIZE: [[FOR_COND_CLEANUP_LOOPEXIT]]:
+; VECTORIZE-NEXT: br label %[[FOR_COND_CLEANUP]]
+; VECTORIZE: [[FOR_COND_CLEANUP]]:
+; VECTORIZE-NEXT: ret void
+;
+entry:
+ %cmp9 = icmp sgt i32 %count, 0
+ br i1 %cmp9, label %vector.ph, label %for.cond.cleanup
+
+vector.ph: ; preds = %entry
+ %wide.trip.count = zext nneg i32 %count to i64
+ %n.vec = and i64 %wide.trip.count, 2147483644
+ br label %vector.body
+
+vector.body: ; preds = %vector.body, %vector.ph
+ %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+ %0 = getelementptr inbounds float, ptr %src1, i64 %index
+ %wide.load = load <2 x float>, ptr %0, align 4, !tbaa !6
+ %1 = getelementptr inbounds float, ptr %src2, i64 %index
+ %wide.load12 = load <2 x float>, ptr %1, align 4, !tbaa !6
+ %2 = fadd <2 x float> %wide.load, %wide.load12
+ %3 = getelementptr inbounds float, ptr %dst, i64 %index
+ store <2 x float> %2, ptr %3, align 4, !tbaa !6
+ %index.next = add nuw i64 %index, 2
+ %4 = icmp eq i64 %index.next, %n.vec
+ br i1 %4, label %for.cond.cleanup, label %vector.body, !llvm.loop !10
+
+for.cond.cleanup:
+ ret void
+}
+
+attributes #0 = { nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+fp-armv8,+neon,+outline-atomics,+v8a" }
+
+!6 = !{!7, !7, i64 0}
+!7 = !{!"float", !8, i64 0}
+!8 = !{!"omnipotent char", !9, i64 0}
+!9 = !{!"Simple C/C++ TBAA"}
+!10 = distinct !{!10, !11}
+!11 = !{!"llvm.loop.mustprogress"}
+;.
+; VECTORIZE_SVE2: [[TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
+; VECTORIZE_SVE2: [[META1]] = !{!"float", [[META2:![0-9]+]], i64 0}
+; VECTORIZE_SVE2: [[META2]] = !{!"omnipotent char", [[META3:![0-9]+]], i64 0}
+; VECTORIZE_SVE2: [[META3]] = !{!"Simple C/C++ TBAA"}
+; VECTORIZE_SVE2: [[LOOP4]] = distinct !{[[LOOP4]], [[META5:![0-9]+]], [[META6:![0-9]+]], [[META7:![0-9]+]]}
+; VECTORIZE_SVE2: [[META5]] = !{!"llvm.loop.mustprogress"}
+; VECTORIZE_SVE2: [[META6]] = !{!"llvm.loop.isvectorized", i32 1}
+; VECTORIZE_SVE2: [[META7]] = !{!"llvm.loop.unroll.runtime.disable"}
+; VECTORIZE_SVE2: [[LOOP8]] = distinct !{[[LOOP8]], [[META5]], [[META7]], [[META6]]}
+;.
+; VECTORIZE_SVE2p2: [[TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
+; VECTORIZE_SVE2p2: [[META1]] = !{!"float", [[META2:![0-9]+]], i64 0}
+; VECTORIZE_SVE2p2: [[META2]] = !{!"omnipotent char", [[META3:![0-9]+]], i64 0}
+; VECTORIZE_SVE2p2: [[META3]] = !{!"Simple C/C++ TBAA"}
+; VECTORIZE_SVE2p2: [[LOOP4]] = distinct !{[[LOOP4]], [[META5:![0-9]+]], [[META6:![0-9]+]], [[META7:![0-9]+]]}
+; VECTORIZE_SVE2p2: [[META5]] = !{!"llvm.loop.mustprogress"}
+; VECTORIZE_SVE2p2: [[META6]] = !{!"llvm.loop.isvectorized", i32 1}
+; VECTORIZE_SVE2p2: [[META7]] = !{!"llvm.loop.unroll.runtime.disable"}
+; VECTORIZE_SVE2p2: [[LOOP8]] = distinct !{[[LOOP8]], [[META5]], [[META7]], [[META6]]}
+;.
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; VECTORIZE_SVE2: {{.*}}
+; VECTORIZE_SVE2p2: {{.*}}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/vector-loop-vf4-uf1.ll b/llvm/test/Transforms/LoopVectorize/AArch64/vector-loop-vf4-uf1.ll
new file mode 100644
index 0000000000000..44291c836f670
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/vector-loop-vf4-uf1.ll
@@ -0,0 +1,300 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -mattr=+sve2 -passes=loop-vectorize -vectorize-vector-loops -force-vector-width=1 -S < %s | FileCheck %s -check-prefixes=CHECK,CHECK_SVE2
+; RUN: opt -mattr=+sve2p2 -passes=loop-vectorize -vectorize-vector-loops -force-vector-width=1 -S < %s | FileCheck %s -check-prefixes=CHECK,CHECK_SVE2p2
+; RUN: opt -mattr=+sve2 -passes=loop-vectorize -vectorize-vector-loops -S < %s | FileCheck %s -check-prefixes=VECTORIZE,VECTORIZE_SVE2
+; RUN: opt -mattr=+sve2p2 -passes=loop-vectorize -vectorize-vector-loops -S < %s | FileCheck %s -check-prefixes=VECTORIZE,VECTORIZE_SVE2p2
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32"
+target triple = "aarch64-unknown-linux-gnu"
+
+; Function Attrs: nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable
+define dso_local void @loop(ptr noalias nocapture noundef writeonly %dst, ptr nocapture noundef readonly %src1, ptr nocapture noundef readonly %src2, i32 noundef %count) local_unnamed_addr #0 {
+; CHECK-LABEL: define dso_local void @loop(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[DST:%.*]], ptr noundef readonly captures(none) [[SRC1:%.*]], ptr noundef readonly captures(none) [[SRC2:%.*]], i32 noundef [[COUNT:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[CMP9:%.*]] = icmp sgt i32 [[COUNT]], 0
+; CHECK-NEXT: br i1 [[CMP9]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[COUNT]] to i64
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[COUNT]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[FOR_BODY_PREHEADER13:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[FOR_BODY_PREHEADER13]]:
+; CHECK-NEXT: [[INDVARS_IV_PH:%.*]] = phi i64 [ 0, %[[FOR_BODY_PREHEADER]] ], [ [[N_VEC:%.*]], %[[MIDDLE_BLOCK:.*]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_VEC]] = and i64 [[WIDE_TRIP_COUNT]], 2147483644
+; CHECK-NEXT: [[TMP0:%.*]] = add nsw i64 [[N_VEC]], -4
+; CHECK-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 2
+; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
+; CHECK-NEXT: [[MIN_ITERS_CHECK2:%.*]] = icmp ult i64 [[TMP2]], 2
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK2]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH2:.*]]
+; CHECK: [[VECTOR_PH2]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 2
+; CHECK-NEXT: [[N_VEC4:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[N_VEC4]], 2
+; CHECK-NEXT: br label %[[VECTOR_BODY4:.*]]
+; CHECK: [[VECTOR_BODY4]]:
+; CHECK-NEXT: [[INDEX6:%.*]] = phi i64 [ 0, %[[VECTOR_PH2]] ], [ [[INDEX_NEXT7:%.*]], %[[VECTOR_BODY4]] ]
+; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX6]], 2
+; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[OFFSET_IDX]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds float, ptr [[SRC1]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[SRC1]], i64 [[TMP4]]
+; CHECK-NEXT: [[TMP7:%.*]] = load <4 x float>, ptr [[TMP5]], align 4, !tbaa [[TBAA6:![0-9]+]]
+; CHECK-NEXT: [[TMP8:%.*]] = load <4 x float>, ptr [[TMP6]], align 4, !tbaa [[TBAA6]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds float, ptr [[SRC2]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[SRC2]], i64 [[TMP4]]
+; CHECK-NEXT: [[TMP11:%.*]] = load <4 x float>, ptr [[TMP9]], align 4, !tbaa [[TBAA6]]
+; CHECK-NEXT: [[TMP12:%.*]] = load <4 x float>, ptr [[TMP10]], align 4, !tbaa [[TBAA6]]
+; CHECK-NEXT: [[TMP13:%.*]] = fadd <4 x float> [[TMP7]], [[TMP11]]
+; CHECK-NEXT: [[TMP14:%.*]] = fadd <4 x float> [[TMP8]], [[TMP12]]
+; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP4]]
+; CHECK-NEXT: store <4 x float> [[TMP13]], ptr [[TMP15]], align 4, !tbaa [[TBAA6]]
+; CHECK-NEXT: store <4 x float> [[TMP14]], ptr [[TMP16]], align 4, !tbaa [[TBAA6]]
+; CHECK-NEXT: [[INDEX_NEXT7]] = add nuw i64 [[INDEX6]], 2
+; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT7]], [[N_VEC4]]
+; CHECK-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK7:.*]], label %[[VECTOR_BODY4]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK7]]:
+; CHECK-NEXT: [[CMP_N8:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC4]]
+; CHECK-NEXT: br i1 [[CMP_N8]], label %[[MIDDLE_BLOCK]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP3]], %[[MIDDLE_BLOCK7]] ], [ 0, %[[VECTOR_PH]] ]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds float, ptr [[SRC1]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP18]], align 4, !tbaa [[TBAA6]]
+; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds float, ptr [[SRC2]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <4 x float>, ptr [[TMP19]], align 4, !tbaa [[TBAA6]]
+; CHECK-NEXT: [[TMP20:%.*]] = fadd <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD12]]
+; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x float> [[TMP20]], ptr [[TMP21]], align 4, !tbaa [[TBAA6]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY_PREHEADER13]]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT:.*]]:
+; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[INDVARS_IV_PH]], %[[FOR_BODY_PREHEADER13]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[SRC1]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP23:%.*]] = load float, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA6]]
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[SRC2]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP24:%.*]] = load float, ptr [[ARRAYIDX2]], align 4, !tbaa [[TBAA6]]
+; CHECK-NEXT: [[ADD:%.*]] = fadd float [[TMP23]], [[TMP24]]
+; CHECK-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: store float [[ADD]], ptr [[ARRAYIDX4]], align 4, !tbaa [[TBAA6]]
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+;
+; VECTORIZE-LABEL: define dso_local void @loop(
+; VECTORIZE-SAME: ptr noalias noundef writeonly captures(none) [[DST:%.*]], ptr noundef readonly captures(none) [[SRC1:%.*]], ptr noundef readonly captures(none) [[SRC2:%.*]], i32 noundef [[COUNT:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; VECTORIZE-NEXT: [[ENTRY:.*:]]
+; VECTORIZE-NEXT: [[CMP9:%.*]] = icmp sgt i32 [[COUNT]], 0
+; VECTORIZE-NEXT: br i1 [[CMP9]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; VECTORIZE: [[FOR_BODY_PREHEADER]]:
+; VECTORIZE-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[COUNT]] to i64
+; VECTORIZE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[COUNT]], 4
+; VECTORIZE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[FOR_BODY_PREHEADER13:.*]], label %[[VECTOR_PH:.*]]
+; VECTORIZE: [[FOR_BODY_PREHEADER13]]:
+; VECTORIZE-NEXT: [[INDVARS_IV_PH:%.*]] = phi i64 [ 0, %[[FOR_BODY_PREHEADER]] ], [ [[N_VEC:%.*]], %[[MIDDLE_BLOCK:.*]] ]
+; VECTORIZE-NEXT: br label %[[FOR_BODY:.*]]
+; VECTORIZE: [[VECTOR_PH]]:
+; VECTORIZE-NEXT: [[N_VEC]] = and i64 [[WIDE_TRIP_COUNT]], 2147483644
+; VECTORIZE-NEXT: [[TMP0:%.*]] = add nsw i64 [[N_VEC]], -4
+; VECTORIZE-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 2
+; VECTORIZE-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
+; VECTORIZE-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; VECTORIZE-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 1
+; VECTORIZE-NEXT: [[MIN_ITERS_CHECK2:%.*]] = icmp ult i64 [[TMP2]], [[TMP4]]
+; VECTORIZE-NEXT: br i1 [[MIN_ITERS_CHECK2]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH2:.*]]
+; VECTORIZE: [[VECTOR_PH2]]:
+; VECTORIZE-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP3]], 1
+; VECTORIZE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP6]]
+; VECTORIZE-NEXT: [[N_VEC4:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
+; VECTORIZE-NEXT: [[TMP7:%.*]] = shl i64 [[N_VEC4]], 2
+; VECTORIZE-NEXT: br label %[[VECTOR_BODY4:.*]]
+; VECTORIZE: [[VECTOR_BODY4]]:
+; VECTORIZE-NEXT: [[INDEX6:%.*]] = phi i64 [ 0, %[[VECTOR_PH2]] ], [ [[INDEX_NEXT11:%.*]], %[[VECTOR_BODY4]] ]
+; VECTORIZE-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX6]], 2
+; VECTORIZE-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[SRC1]], i64 [[OFFSET_IDX]]
+; VECTORIZE-NEXT: [[TMP9:%.*]] = getelementptr inbounds <4 x float>, ptr [[TMP10]], i64 [[TMP3]]
+; VECTORIZE-NEXT: [[WIDE_LOAD6:%.*]] = load <vscale x 4 x float>, ptr [[TMP10]], align 4, !tbaa [[TBAA6:![0-9]+]]
+; VECTORIZE-NEXT: [[WIDE_LOAD7:%.*]] = load <vscale x 4 x float>, ptr [[TMP9]], align 4, !tbaa [[TBAA6]]
+; VECTORIZE-NEXT: [[TMP15:%.*]] = getelementptr inbounds float, ptr [[SRC2]], i64 [[OFFSET_IDX]]
+; VECTORIZE-NEXT: [[TMP11:%.*]] = getelementptr inbounds <4 x float>, ptr [[TMP15]], i64 [[TMP3]]
+; VECTORIZE-NEXT: [[WIDE_LOAD8:%.*]] = load <vscale x 4 x float>, ptr [[TMP15]], align 4, !tbaa [[TBAA6]]
+; VECTORIZE-NEXT: [[WIDE_LOAD9:%.*]] = load <vscale x 4 x float>, ptr [[TMP11]], align 4, !tbaa [[TBAA6]]
+; VECTORIZE-NEXT: [[TMP16:%.*]] = fadd <vscale x 4 x float> [[WIDE_LOAD6]], [[WIDE_LOAD8]]
+; VECTORIZE-NEXT: [[TMP20:%.*]] = fadd <vscale x 4 x float> [[WIDE_LOAD7]], [[WIDE_LOAD9]]
+; VECTORIZE-NEXT: [[TMP22:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[OFFSET_IDX]]
+; VECTORIZE-NEXT: [[TMP17:%.*]] = getelementptr inbounds <4 x float>, ptr [[TMP22]], i64 [[TMP3]]
+; VECTORIZE-NEXT: store <vscale x 4 x float> [[TMP16]], ptr [[TMP22]], align 4, !tbaa [[TBAA6]]
+; VECTORIZE-NEXT: store <vscale x 4 x float> [[TMP20]], ptr [[TMP17]], align 4, !tbaa [[TBAA6]]
+; VECTORIZE-NEXT: [[INDEX_NEXT11]] = add nuw i64 [[INDEX6]], [[TMP6]]
+; VECTORIZE-NEXT: [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT11]], [[N_VEC4]]
+; VECTORIZE-NEXT: br i1 [[TMP27]], label %[[MIDDLE_BLOCK11:.*]], label %[[VECTOR_BODY4]], !llvm.loop [[LOOP10:![0-9]+]]
+; VECTORIZE: [[MIDDLE_BLOCK11]]:
+; VECTORIZE-NEXT: [[CMP_N12:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC4]]
+; VECTORIZE-NEXT: br i1 [[CMP_N12]], label %[[MIDDLE_BLOCK]], label %[[SCALAR_PH]]
+; VECTORIZE: [[SCALAR_PH]]:
+; VECTORIZE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP7]], %[[MIDDLE_BLOCK11]] ], [ 0, %[[VECTOR_PH]] ]
+; VECTORIZE-NEXT: br label %[[VECTOR_BODY:.*]]
+; VECTORIZE: [[VECTOR_BODY]]:
+; VECTORIZE-NEXT: [[INDEX:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VECTORIZE-NEXT: [[TMP28:%.*]] = getelementptr inbounds float, ptr [[SRC1]], i64 [[INDEX]]
+; VECTORIZE-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP28]], align 4, !tbaa [[TBAA6]]
+; VECTORIZE-NEXT: [[TMP29:%.*]] = getelementptr inbounds float, ptr [[SRC2]], i64 [[INDEX]]
+; VECTORIZE-NEXT: [[WIDE_LOAD12:%.*]] = load <4 x float>, ptr [[TMP29]], align 4, !tbaa [[TBAA6]]
+; VECTORIZE-NEXT: [[TMP30:%.*]] = fadd <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD12]]
+; VECTORIZE-NEXT: [[TMP31:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[INDEX]]
+; VECTORIZE-NEXT: store <4 x float> [[TMP30]], ptr [[TMP31]], align 4, !tbaa [[TBAA6]]
+; VECTORIZE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; VECTORIZE-NEXT: [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VECTORIZE-NEXT: br i1 [[TMP32]], label %[[MIDDLE_BLOCK]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; VECTORIZE: [[MIDDLE_BLOCK]]:
+; VECTORIZE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[WIDE_TRIP_COUNT]]
+; VECTORIZE-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY_PREHEADER13]]
+; VECTORIZE: [[FOR_COND_CLEANUP_LOOPEXIT:.*]]:
+; VECTORIZE-NEXT: br label %[[FOR_COND_CLEANUP]]
+; VECTORIZE: [[FOR_COND_CLEANUP]]:
+; VECTORIZE-NEXT: ret void
+; VECTORIZE: [[FOR_BODY]]:
+; VECTORIZE-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[INDVARS_IV_PH]], %[[FOR_BODY_PREHEADER13]] ]
+; VECTORIZE-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[SRC1]], i64 [[INDVARS_IV]]
+; VECTORIZE-NEXT: [[TMP33:%.*]] = load float, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA6]]
+; VECTORIZE-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[SRC2]], i64 [[INDVARS_IV]]
+; VECTORIZE-NEXT: [[TMP34:%.*]] = load float, ptr [[ARRAYIDX2]], align 4, !tbaa [[TBAA6]]
+; VECTORIZE-NEXT: [[ADD:%.*]] = fadd float [[TMP33]], [[TMP34]]
+; VECTORIZE-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[INDVARS_IV]]
+; VECTORIZE-NEXT: store float [[ADD]], ptr [[ARRAYIDX4]], align 4, !tbaa [[TBAA6]]
+; VECTORIZE-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; VECTORIZE-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; VECTORIZE-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+;
+entry:
+ %cmp9 = icmp sgt i32 %count, 0
+ br i1 %cmp9, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader: ; preds = %entry
+ %wide.trip.count = zext nneg i32 %count to i64
+ %min.iters.check = icmp ult i32 %count, 4
+ br i1 %min.iters.check, label %for.body.preheader13, label %vector.ph
+
+for.body.preheader13: ; preds = %middle.block, %for.body.preheader
+ %indvars.iv.ph = phi i64 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
+ br label %for.body
+
+vector.ph: ; preds = %for.body.preheader
+ %n.vec = and i64 %wide.trip.count, 2147483644
+ br label %vector.body
+
+vector.body: ; preds = %vector.body, %vector.ph
+ %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+ %0 = getelementptr inbounds float, ptr %src1, i64 %index
+ %wide.load = load <4 x float>, ptr %0, align 4, !tbaa !6
+ %1 = getelementptr inbounds float, ptr %src2, i64 %index
+ %wide.load12 = load <4 x float>, ptr %1, align 4, !tbaa !6
+ %2 = fadd <4 x float> %wide.load, %wide.load12
+ %3 = getelementptr inbounds float, ptr %dst, i64 %index
+ store <4 x float> %2, ptr %3, align 4, !tbaa !6
+ %index.next = add nuw i64 %index, 4
+ %4 = icmp eq i64 %index.next, %n.vec
+ br i1 %4, label %middle.block, label %vector.body, !llvm.loop !10
+
+middle.block: ; preds = %vector.body
+ %cmp.n = icmp eq i64 %n.vec, %wide.trip.count
+ br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader13
+
+for.cond.cleanup: ; preds = %for.body, %middle.block, %entry
+ ret void
+
+for.body: ; preds = %for.body.preheader13, %for.body
+ %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ %indvars.iv.ph, %for.body.preheader13 ]
+ %arrayidx = getelementptr inbounds float, ptr %src1, i64 %indvars.iv
+ %5 = load float, ptr %arrayidx, align 4, !tbaa !6
+ %arrayidx2 = getelementptr inbounds float, ptr %src2, i64 %indvars.iv
+ %6 = load float, ptr %arrayidx2, align 4, !tbaa !6
+ %add = fadd float %5, %6
+ %arrayidx4 = getelementptr inbounds float, ptr %dst, i64 %indvars.iv
+ store float %add, ptr %arrayidx4, align 4, !tbaa !6
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !llvm.loop !14
+}
+
+attributes #0 = { nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+fp-armv8,+neon,+outline-atomics,+v8a" }
+
+!llvm.module.flags = !{!0, !1, !2, !3, !4}
+!llvm.ident = !{!5}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 8, !"PIC Level", i32 2}
+!2 = !{i32 7, !"PIE Level", i32 2}
+!3 = !{i32 7, !"uwtable", i32 2}
+!4 = !{i32 7, !"frame-pointer", i32 1}
+!5 = !{!"clang version 20.0.0git"}
+!6 = !{!7, !7, i64 0}
+!7 = !{!"float", !8, i64 0}
+!8 = !{!"omnipotent char", !9, i64 0}
+!9 = !{!"Simple C/C++ TBAA"}
+!10 = distinct !{!10, !11}
+!11 = !{!"llvm.loop.mustprogress"}
+!12 = !{!"llvm.loop.unroll.runtime.disable"}
+!13 = !{!"llvm.loop.isvectorized", i32 1}
+!14 = distinct !{!14, !11, !12, !13}
+;.
+; CHECK_SVE2: [[TBAA6]] = !{[[META7:![0-9]+]], [[META7]], i64 0}
+; CHECK_SVE2: [[META7]] = !{!"float", [[META8:![0-9]+]], i64 0}
+; CHECK_SVE2: [[META8]] = !{!"omnipotent char", [[META9:![0-9]+]], i64 0}
+; CHECK_SVE2: [[META9]] = !{!"Simple C/C++ TBAA"}
+; CHECK_SVE2: [[LOOP10]] = distinct !{[[LOOP10]], [[META11:![0-9]+]], [[META12:![0-9]+]], [[META13:![0-9]+]]}
+; CHECK_SVE2: [[META11]] = !{!"llvm.loop.mustprogress"}
+; CHECK_SVE2: [[META12]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK_SVE2: [[META13]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK_SVE2: [[LOOP14]] = distinct !{[[LOOP14]], [[META11]], [[META12]]}
+; CHECK_SVE2: [[LOOP15]] = distinct !{[[LOOP15]], [[META11]], [[META13]], [[META12]]}
+;.
+; CHECK_SVE2p2: [[TBAA6]] = !{[[META7:![0-9]+]], [[META7]], i64 0}
+; CHECK_SVE2p2: [[META7]] = !{!"float", [[META8:![0-9]+]], i64 0}
+; CHECK_SVE2p2: [[META8]] = !{!"omnipotent char", [[META9:![0-9]+]], i64 0}
+; CHECK_SVE2p2: [[META9]] = !{!"Simple C/C++ TBAA"}
+; CHECK_SVE2p2: [[LOOP10]] = distinct !{[[LOOP10]], [[META11:![0-9]+]], [[META12:![0-9]+]], [[META13:![0-9]+]]}
+; CHECK_SVE2p2: [[META11]] = !{!"llvm.loop.mustprogress"}
+; CHECK_SVE2p2: [[META12]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK_SVE2p2: [[META13]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK_SVE2p2: [[LOOP14]] = distinct !{[[LOOP14]], [[META11]], [[META12]]}
+; CHECK_SVE2p2: [[LOOP15]] = distinct !{[[LOOP15]], [[META11]], [[META13]], [[META12]]}
+;.
+; VECTORIZE_SVE2: [[TBAA6]] = !{[[META7:![0-9]+]], [[META7]], i64 0}
+; VECTORIZE_SVE2: [[META7]] = !{!"float", [[META8:![0-9]+]], i64 0}
+; VECTORIZE_SVE2: [[META8]] = !{!"omnipotent char", [[META9:![0-9]+]], i64 0}
+; VECTORIZE_SVE2: [[META9]] = !{!"Simple C/C++ TBAA"}
+; VECTORIZE_SVE2: [[LOOP10]] = distinct !{[[LOOP10]], [[META11:![0-9]+]], [[META12:![0-9]+]], [[META13:![0-9]+]]}
+; VECTORIZE_SVE2: [[META11]] = !{!"llvm.loop.mustprogress"}
+; VECTORIZE_SVE2: [[META12]] = !{!"llvm.loop.isvectorized", i32 1}
+; VECTORIZE_SVE2: [[META13]] = !{!"llvm.loop.unroll.runtime.disable"}
+; VECTORIZE_SVE2: [[LOOP14]] = distinct !{[[LOOP14]], [[META11]], [[META13]], [[META12]]}
+; VECTORIZE_SVE2: [[LOOP15]] = distinct !{[[LOOP15]], [[META11]], [[META13]], [[META12]]}
+;.
+; VECTORIZE_SVE2p2: [[TBAA6]] = !{[[META7:![0-9]+]], [[META7]], i64 0}
+; VECTORIZE_SVE2p2: [[META7]] = !{!"float", [[META8:![0-9]+]], i64 0}
+; VECTORIZE_SVE2p2: [[META8]] = !{!"omnipotent char", [[META9:![0-9]+]], i64 0}
+; VECTORIZE_SVE2p2: [[META9]] = !{!"Simple C/C++ TBAA"}
+; VECTORIZE_SVE2p2: [[LOOP10]] = distinct !{[[LOOP10]], [[META11:![0-9]+]], [[META12:![0-9]+]], [[META13:![0-9]+]]}
+; VECTORIZE_SVE2p2: [[META11]] = !{!"llvm.loop.mustprogress"}
+; VECTORIZE_SVE2p2: [[META12]] = !{!"llvm.loop.isvectorized", i32 1}
+; VECTORIZE_SVE2p2: [[META13]] = !{!"llvm.loop.unroll.runtime.disable"}
+; VECTORIZE_SVE2p2: [[LOOP14]] = distinct !{[[LOOP14]], [[META11]], [[META13]], [[META12]]}
+; VECTORIZE_SVE2p2: [[LOOP15]] = distinct !{[[LOOP15]], [[META11]], [[META13]], [[META12]]}
+;.
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK_SVE2: {{.*}}
+; CHECK_SVE2p2: {{.*}}
+; VECTORIZE_SVE2: {{.*}}
+; VECTORIZE_SVE2p2: {{.*}}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/vector-loop-vf4-uf2.ll b/llvm/test/Transforms/LoopVectorize/AArch64/vector-loop-vf4-uf2.ll
new file mode 100644
index 0000000000000..2f0c2b97ea538
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/vector-loop-vf4-uf2.ll
@@ -0,0 +1,334 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -mattr=+sve2 -passes=loop-vectorize -vectorize-vector-loops -force-vector-width=1 -S < %s | FileCheck %s -check-prefixes=CHECK,CHECK_SVE2
+; RUN: opt -mattr=+sve2p2 -passes=loop-vectorize -vectorize-vector-loops -force-vector-width=1 -S < %s | FileCheck %s -check-prefixes=CHECK,CHECK_SVE2p2
+; RUN: opt -mattr=+sve2 -passes=loop-vectorize -vectorize-vector-loops -S < %s | FileCheck %s -check-prefixes=VECTORIZE,VECTORIZE_SVE2
+; RUN: opt -mattr=+sve2p2 -passes=loop-vectorize -vectorize-vector-loops -S < %s | FileCheck %s -check-prefixes=VECTORIZE,VECTORIZE_SVE2p2
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32"
+target triple = "aarch64-unknown-linux-gnu"
+
+; Function Attrs: nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable
+define dso_local void @loop(ptr noalias nocapture noundef writeonly %dst, ptr nocapture noundef readonly %src1, ptr nocapture noundef readonly %src2, i32 noundef %count) local_unnamed_addr #0 {
+; CHECK-LABEL: define dso_local void @loop(
+; CHECK-SAME: ptr noalias noundef writeonly captures(none) [[DST:%.*]], ptr noundef readonly captures(none) [[SRC1:%.*]], ptr noundef readonly captures(none) [[SRC2:%.*]], i32 noundef [[COUNT:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[CMP9:%.*]] = icmp sgt i32 [[COUNT]], 0
+; CHECK-NEXT: br i1 [[CMP9]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[COUNT]] to i64
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[COUNT]], 8
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[FOR_BODY_PREHEADER15:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[FOR_BODY_PREHEADER15]]:
+; CHECK-NEXT: [[INDVARS_IV_PH:%.*]] = phi i64 [ 0, %[[FOR_BODY_PREHEADER]] ], [ [[N_VEC:%.*]], %[[MIDDLE_BLOCK:.*]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_VEC]] = and i64 [[WIDE_TRIP_COUNT]], 2147483640
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds float, ptr [[SRC1]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 16
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP0]], align 4, !tbaa [[TBAA6:![0-9]+]]
+; CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <4 x float>, ptr [[TMP1]], align 4, !tbaa [[TBAA6]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds float, ptr [[SRC2]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 16
+; CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <4 x float>, ptr [[TMP2]], align 4, !tbaa [[TBAA6]]
+; CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <4 x float>, ptr [[TMP3]], align 4, !tbaa [[TBAA6]]
+; CHECK-NEXT: [[TMP4:%.*]] = fadd <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD13]]
+; CHECK-NEXT: [[TMP5:%.*]] = fadd <4 x float> [[WIDE_LOAD12]], [[WIDE_LOAD14]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP6]], i64 16
+; CHECK-NEXT: store <4 x float> [[TMP4]], ptr [[TMP6]], align 4, !tbaa [[TBAA6]]
+; CHECK-NEXT: store <4 x float> [[TMP5]], ptr [[TMP7]], align 4, !tbaa [[TBAA6]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY_PREHEADER15]]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT:.*]]:
+; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[INDVARS_IV_PH]], %[[FOR_BODY_PREHEADER15]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[SRC1]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP9:%.*]] = load float, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA6]]
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[SRC2]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP10:%.*]] = load float, ptr [[ARRAYIDX2]], align 4, !tbaa [[TBAA6]]
+; CHECK-NEXT: [[ADD:%.*]] = fadd float [[TMP9]], [[TMP10]]
+; CHECK-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: store float [[ADD]], ptr [[ARRAYIDX4]], align 4, !tbaa [[TBAA6]]
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+;
+; VECTORIZE_SVE2-LABEL: define dso_local void @loop(
+; VECTORIZE_SVE2-SAME: ptr noalias noundef writeonly captures(none) [[DST:%.*]], ptr noundef readonly captures(none) [[SRC1:%.*]], ptr noundef readonly captures(none) [[SRC2:%.*]], i32 noundef [[COUNT:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; VECTORIZE_SVE2-NEXT: [[ENTRY:.*:]]
+; VECTORIZE_SVE2-NEXT: [[CMP9:%.*]] = icmp sgt i32 [[COUNT]], 0
+; VECTORIZE_SVE2-NEXT: br i1 [[CMP9]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; VECTORIZE_SVE2: [[FOR_BODY_PREHEADER]]:
+; VECTORIZE_SVE2-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[COUNT]] to i64
+; VECTORIZE_SVE2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[COUNT]], 8
+; VECTORIZE_SVE2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[FOR_BODY_PREHEADER15:.*]], label %[[VECTOR_PH:.*]]
+; VECTORIZE_SVE2: [[FOR_BODY_PREHEADER15]]:
+; VECTORIZE_SVE2-NEXT: [[INDVARS_IV_PH:%.*]] = phi i64 [ 0, %[[FOR_BODY_PREHEADER]] ], [ [[N_VEC:%.*]], %[[MIDDLE_BLOCK:.*]] ]
+; VECTORIZE_SVE2-NEXT: br label %[[FOR_BODY:.*]]
+; VECTORIZE_SVE2: [[VECTOR_PH]]:
+; VECTORIZE_SVE2-NEXT: [[N_VEC]] = and i64 [[WIDE_TRIP_COUNT]], 2147483640
+; VECTORIZE_SVE2-NEXT: br label %[[VECTOR_BODY:.*]]
+; VECTORIZE_SVE2: [[VECTOR_BODY]]:
+; VECTORIZE_SVE2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VECTORIZE_SVE2-NEXT: [[TMP0:%.*]] = getelementptr inbounds float, ptr [[SRC1]], i64 [[INDEX]]
+; VECTORIZE_SVE2-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 16
+; VECTORIZE_SVE2-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP0]], align 4, !tbaa [[TBAA6:![0-9]+]]
+; VECTORIZE_SVE2-NEXT: [[WIDE_LOAD12:%.*]] = load <4 x float>, ptr [[TMP1]], align 4, !tbaa [[TBAA6]]
+; VECTORIZE_SVE2-NEXT: [[TMP2:%.*]] = getelementptr inbounds float, ptr [[SRC2]], i64 [[INDEX]]
+; VECTORIZE_SVE2-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 16
+; VECTORIZE_SVE2-NEXT: [[WIDE_LOAD13:%.*]] = load <4 x float>, ptr [[TMP2]], align 4, !tbaa [[TBAA6]]
+; VECTORIZE_SVE2-NEXT: [[WIDE_LOAD14:%.*]] = load <4 x float>, ptr [[TMP3]], align 4, !tbaa [[TBAA6]]
+; VECTORIZE_SVE2-NEXT: [[TMP4:%.*]] = fadd <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD13]]
+; VECTORIZE_SVE2-NEXT: [[TMP5:%.*]] = fadd <4 x float> [[WIDE_LOAD12]], [[WIDE_LOAD14]]
+; VECTORIZE_SVE2-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[INDEX]]
+; VECTORIZE_SVE2-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP6]], i64 16
+; VECTORIZE_SVE2-NEXT: store <4 x float> [[TMP4]], ptr [[TMP6]], align 4, !tbaa [[TBAA6]]
+; VECTORIZE_SVE2-NEXT: store <4 x float> [[TMP5]], ptr [[TMP7]], align 4, !tbaa [[TBAA6]]
+; VECTORIZE_SVE2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; VECTORIZE_SVE2-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VECTORIZE_SVE2-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; VECTORIZE_SVE2: [[MIDDLE_BLOCK]]:
+; VECTORIZE_SVE2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[WIDE_TRIP_COUNT]]
+; VECTORIZE_SVE2-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY_PREHEADER15]]
+; VECTORIZE_SVE2: [[FOR_COND_CLEANUP_LOOPEXIT:.*]]:
+; VECTORIZE_SVE2-NEXT: br label %[[FOR_COND_CLEANUP]]
+; VECTORIZE_SVE2: [[FOR_COND_CLEANUP]]:
+; VECTORIZE_SVE2-NEXT: ret void
+; VECTORIZE_SVE2: [[FOR_BODY]]:
+; VECTORIZE_SVE2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[INDVARS_IV_PH]], %[[FOR_BODY_PREHEADER15]] ]
+; VECTORIZE_SVE2-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[SRC1]], i64 [[INDVARS_IV]]
+; VECTORIZE_SVE2-NEXT: [[TMP9:%.*]] = load float, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA6]]
+; VECTORIZE_SVE2-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[SRC2]], i64 [[INDVARS_IV]]
+; VECTORIZE_SVE2-NEXT: [[TMP10:%.*]] = load float, ptr [[ARRAYIDX2]], align 4, !tbaa [[TBAA6]]
+; VECTORIZE_SVE2-NEXT: [[ADD:%.*]] = fadd float [[TMP9]], [[TMP10]]
+; VECTORIZE_SVE2-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[INDVARS_IV]]
+; VECTORIZE_SVE2-NEXT: store float [[ADD]], ptr [[ARRAYIDX4]], align 4, !tbaa [[TBAA6]]
+; VECTORIZE_SVE2-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; VECTORIZE_SVE2-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; VECTORIZE_SVE2-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+;
+; VECTORIZE_SVE2p2-LABEL: define dso_local void @loop(
+; VECTORIZE_SVE2p2-SAME: ptr noalias noundef writeonly captures(none) [[DST:%.*]], ptr noundef readonly captures(none) [[SRC1:%.*]], ptr noundef readonly captures(none) [[SRC2:%.*]], i32 noundef [[COUNT:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; VECTORIZE_SVE2p2-NEXT: [[ENTRY:.*:]]
+; VECTORIZE_SVE2p2-NEXT: [[CMP9:%.*]] = icmp sgt i32 [[COUNT]], 0
+; VECTORIZE_SVE2p2-NEXT: br i1 [[CMP9]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; VECTORIZE_SVE2p2: [[FOR_BODY_PREHEADER]]:
+; VECTORIZE_SVE2p2-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[COUNT]] to i64
+; VECTORIZE_SVE2p2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[COUNT]], 8
+; VECTORIZE_SVE2p2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[FOR_BODY_PREHEADER15:.*]], label %[[VECTOR_PH:.*]]
+; VECTORIZE_SVE2p2: [[FOR_BODY_PREHEADER15]]:
+; VECTORIZE_SVE2p2-NEXT: [[INDVARS_IV_PH:%.*]] = phi i64 [ 0, %[[FOR_BODY_PREHEADER]] ], [ [[N_VEC:%.*]], %[[MIDDLE_BLOCK:.*]] ]
+; VECTORIZE_SVE2p2-NEXT: br label %[[FOR_BODY:.*]]
+; VECTORIZE_SVE2p2: [[VECTOR_PH]]:
+; VECTORIZE_SVE2p2-NEXT: [[N_VEC]] = and i64 [[WIDE_TRIP_COUNT]], 2147483640
+; VECTORIZE_SVE2p2-NEXT: [[TMP0:%.*]] = add nsw i64 [[N_VEC]], -8
+; VECTORIZE_SVE2p2-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 3
+; VECTORIZE_SVE2p2-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
+; VECTORIZE_SVE2p2-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; VECTORIZE_SVE2p2-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP2]], [[TMP3]]
+; VECTORIZE_SVE2p2-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH2:.*]]
+; VECTORIZE_SVE2p2: [[VECTOR_PH2]]:
+; VECTORIZE_SVE2p2-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP3]]
+; VECTORIZE_SVE2p2-NEXT: [[N_VEC3:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
+; VECTORIZE_SVE2p2-NEXT: [[TMP4:%.*]] = shl i64 [[N_VEC3]], 3
+; VECTORIZE_SVE2p2-NEXT: br label %[[VECTOR_BODY4:.*]]
+; VECTORIZE_SVE2p2: [[VECTOR_BODY4]]:
+; VECTORIZE_SVE2p2-NEXT: [[INDEX5:%.*]] = phi i64 [ 0, %[[VECTOR_PH2]] ], [ [[INDEX_NEXT13:%.*]], %[[VECTOR_BODY4]] ]
+; VECTORIZE_SVE2p2-NEXT: [[TMP10:%.*]] = shl i64 [[INDEX5]], 3
+; VECTORIZE_SVE2p2-NEXT: [[TMP12:%.*]] = getelementptr inbounds float, ptr [[SRC1]], i64 [[TMP10]]
+; VECTORIZE_SVE2p2-NEXT: [[WIDE_VEC6:%.*]] = load <vscale x 8 x float>, ptr [[TMP12]], align 4, !tbaa [[TBAA6:![0-9]+]]
+; VECTORIZE_SVE2p2-NEXT: [[STRIDED_VEC7:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave.segments2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC6]])
+; VECTORIZE_SVE2p2-NEXT: [[TMP15:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC7]], 0
+; VECTORIZE_SVE2p2-NEXT: [[TMP16:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC7]], 1
+; VECTORIZE_SVE2p2-NEXT: [[TMP18:%.*]] = getelementptr inbounds float, ptr [[SRC2]], i64 [[TMP10]]
+; VECTORIZE_SVE2p2-NEXT: [[WIDE_VEC10:%.*]] = load <vscale x 8 x float>, ptr [[TMP18]], align 4, !tbaa [[TBAA6]]
+; VECTORIZE_SVE2p2-NEXT: [[STRIDED_VEC11:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave.segments2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC10]])
+; VECTORIZE_SVE2p2-NEXT: [[TMP21:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC11]], 0
+; VECTORIZE_SVE2p2-NEXT: [[TMP22:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC11]], 1
+; VECTORIZE_SVE2p2-NEXT: [[TMP24:%.*]] = fadd <vscale x 4 x float> [[TMP15]], [[TMP21]]
+; VECTORIZE_SVE2p2-NEXT: [[TMP26:%.*]] = fadd <vscale x 4 x float> [[TMP16]], [[TMP22]]
+; VECTORIZE_SVE2p2-NEXT: [[TMP28:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP10]]
+; VECTORIZE_SVE2p2-NEXT: [[INTERLEAVED_VEC12:%.*]] = call <vscale x 8 x float> @llvm.vector.interleave.segments2.nxv8f32(<vscale x 4 x float> [[TMP24]], <vscale x 4 x float> [[TMP26]])
+; VECTORIZE_SVE2p2-NEXT: store <vscale x 8 x float> [[INTERLEAVED_VEC12]], ptr [[TMP28]], align 4, !tbaa [[TBAA6]]
+; VECTORIZE_SVE2p2-NEXT: [[INDEX_NEXT13]] = add nuw i64 [[INDEX5]], [[TMP3]]
+; VECTORIZE_SVE2p2-NEXT: [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT13]], [[N_VEC3]]
+; VECTORIZE_SVE2p2-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK14:.*]], label %[[VECTOR_BODY4]], !llvm.loop [[LOOP10:![0-9]+]]
+; VECTORIZE_SVE2p2: [[MIDDLE_BLOCK14]]:
+; VECTORIZE_SVE2p2-NEXT: [[CMP_N15:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC3]]
+; VECTORIZE_SVE2p2-NEXT: br i1 [[CMP_N15]], label %[[MIDDLE_BLOCK]], label %[[SCALAR_PH]]
+; VECTORIZE_SVE2p2: [[SCALAR_PH]]:
+; VECTORIZE_SVE2p2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP4]], %[[MIDDLE_BLOCK14]] ], [ 0, %[[VECTOR_PH]] ]
+; VECTORIZE_SVE2p2-NEXT: br label %[[VECTOR_BODY:.*]]
+; VECTORIZE_SVE2p2: [[VECTOR_BODY]]:
+; VECTORIZE_SVE2p2-NEXT: [[INDEX:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VECTORIZE_SVE2p2-NEXT: [[TMP30:%.*]] = getelementptr inbounds float, ptr [[SRC1]], i64 [[INDEX]]
+; VECTORIZE_SVE2p2-NEXT: [[TMP31:%.*]] = getelementptr inbounds i8, ptr [[TMP30]], i64 16
+; VECTORIZE_SVE2p2-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP30]], align 4, !tbaa [[TBAA6]]
+; VECTORIZE_SVE2p2-NEXT: [[WIDE_LOAD12:%.*]] = load <4 x float>, ptr [[TMP31]], align 4, !tbaa [[TBAA6]]
+; VECTORIZE_SVE2p2-NEXT: [[TMP32:%.*]] = getelementptr inbounds float, ptr [[SRC2]], i64 [[INDEX]]
+; VECTORIZE_SVE2p2-NEXT: [[TMP33:%.*]] = getelementptr inbounds i8, ptr [[TMP32]], i64 16
+; VECTORIZE_SVE2p2-NEXT: [[WIDE_LOAD13:%.*]] = load <4 x float>, ptr [[TMP32]], align 4, !tbaa [[TBAA6]]
+; VECTORIZE_SVE2p2-NEXT: [[WIDE_LOAD14:%.*]] = load <4 x float>, ptr [[TMP33]], align 4, !tbaa [[TBAA6]]
+; VECTORIZE_SVE2p2-NEXT: [[TMP34:%.*]] = fadd <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD13]]
+; VECTORIZE_SVE2p2-NEXT: [[TMP35:%.*]] = fadd <4 x float> [[WIDE_LOAD12]], [[WIDE_LOAD14]]
+; VECTORIZE_SVE2p2-NEXT: [[TMP36:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[INDEX]]
+; VECTORIZE_SVE2p2-NEXT: [[TMP37:%.*]] = getelementptr inbounds i8, ptr [[TMP36]], i64 16
+; VECTORIZE_SVE2p2-NEXT: store <4 x float> [[TMP34]], ptr [[TMP36]], align 4, !tbaa [[TBAA6]]
+; VECTORIZE_SVE2p2-NEXT: store <4 x float> [[TMP35]], ptr [[TMP37]], align 4, !tbaa [[TBAA6]]
+; VECTORIZE_SVE2p2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; VECTORIZE_SVE2p2-NEXT: [[TMP38:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VECTORIZE_SVE2p2-NEXT: br i1 [[TMP38]], label %[[MIDDLE_BLOCK]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; VECTORIZE_SVE2p2: [[MIDDLE_BLOCK]]:
+; VECTORIZE_SVE2p2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[WIDE_TRIP_COUNT]]
+; VECTORIZE_SVE2p2-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY_PREHEADER15]]
+; VECTORIZE_SVE2p2: [[FOR_COND_CLEANUP_LOOPEXIT:.*]]:
+; VECTORIZE_SVE2p2-NEXT: br label %[[FOR_COND_CLEANUP]]
+; VECTORIZE_SVE2p2: [[FOR_COND_CLEANUP]]:
+; VECTORIZE_SVE2p2-NEXT: ret void
+; VECTORIZE_SVE2p2: [[FOR_BODY]]:
+; VECTORIZE_SVE2p2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[INDVARS_IV_PH]], %[[FOR_BODY_PREHEADER15]] ]
+; VECTORIZE_SVE2p2-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[SRC1]], i64 [[INDVARS_IV]]
+; VECTORIZE_SVE2p2-NEXT: [[TMP39:%.*]] = load float, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA6]]
+; VECTORIZE_SVE2p2-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[SRC2]], i64 [[INDVARS_IV]]
+; VECTORIZE_SVE2p2-NEXT: [[TMP40:%.*]] = load float, ptr [[ARRAYIDX2]], align 4, !tbaa [[TBAA6]]
+; VECTORIZE_SVE2p2-NEXT: [[ADD:%.*]] = fadd float [[TMP39]], [[TMP40]]
+; VECTORIZE_SVE2p2-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[INDVARS_IV]]
+; VECTORIZE_SVE2p2-NEXT: store float [[ADD]], ptr [[ARRAYIDX4]], align 4, !tbaa [[TBAA6]]
+; VECTORIZE_SVE2p2-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; VECTORIZE_SVE2p2-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; VECTORIZE_SVE2p2-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+;
+entry:
+ %cmp9 = icmp sgt i32 %count, 0
+ br i1 %cmp9, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader: ; preds = %entry
+ %wide.trip.count = zext nneg i32 %count to i64
+ %min.iters.check = icmp ult i32 %count, 8
+ br i1 %min.iters.check, label %for.body.preheader15, label %vector.ph
+
+for.body.preheader15: ; preds = %middle.block, %for.body.preheader
+ %indvars.iv.ph = phi i64 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
+ br label %for.body
+
+vector.ph: ; preds = %for.body.preheader
+ %n.vec = and i64 %wide.trip.count, 2147483640
+ br label %vector.body
+
+vector.body: ; preds = %vector.body, %vector.ph
+ %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+ %0 = getelementptr inbounds float, ptr %src1, i64 %index
+ %1 = getelementptr inbounds i8, ptr %0, i64 16
+ %wide.load = load <4 x float>, ptr %0, align 4, !tbaa !6
+ %wide.load12 = load <4 x float>, ptr %1, align 4, !tbaa !6
+ %2 = getelementptr inbounds float, ptr %src2, i64 %index
+ %3 = getelementptr inbounds i8, ptr %2, i64 16
+ %wide.load13 = load <4 x float>, ptr %2, align 4, !tbaa !6
+ %wide.load14 = load <4 x float>, ptr %3, align 4, !tbaa !6
+ %4 = fadd <4 x float> %wide.load, %wide.load13
+ %5 = fadd <4 x float> %wide.load12, %wide.load14
+ %6 = getelementptr inbounds float, ptr %dst, i64 %index
+ %7 = getelementptr inbounds i8, ptr %6, i64 16
+ store <4 x float> %4, ptr %6, align 4, !tbaa !6
+ store <4 x float> %5, ptr %7, align 4, !tbaa !6
+ %index.next = add nuw i64 %index, 8
+ %8 = icmp eq i64 %index.next, %n.vec
+ br i1 %8, label %middle.block, label %vector.body, !llvm.loop !10
+
+middle.block: ; preds = %vector.body
+ %cmp.n = icmp eq i64 %n.vec, %wide.trip.count
+ br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader15
+
+for.cond.cleanup: ; preds = %for.body, %middle.block, %entry
+ ret void
+
+for.body: ; preds = %for.body.preheader15, %for.body
+ %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ %indvars.iv.ph, %for.body.preheader15 ]
+ %arrayidx = getelementptr inbounds float, ptr %src1, i64 %indvars.iv
+ %9 = load float, ptr %arrayidx, align 4, !tbaa !6
+ %arrayidx2 = getelementptr inbounds float, ptr %src2, i64 %indvars.iv
+ %10 = load float, ptr %arrayidx2, align 4, !tbaa !6
+ %add = fadd float %9, %10
+ %arrayidx4 = getelementptr inbounds float, ptr %dst, i64 %indvars.iv
+ store float %add, ptr %arrayidx4, align 4, !tbaa !6
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !llvm.loop !14
+}
+
+attributes #0 = { nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+fp-armv8,+neon,+outline-atomics,+v8a" }
+
+!llvm.module.flags = !{!0, !1, !2, !3, !4}
+!llvm.ident = !{!5}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 8, !"PIC Level", i32 2}
+!2 = !{i32 7, !"PIE Level", i32 2}
+!3 = !{i32 7, !"uwtable", i32 2}
+!4 = !{i32 7, !"frame-pointer", i32 1}
+!5 = !{!"clang version 20.0.0git"}
+!6 = !{!7, !7, i64 0}
+!7 = !{!"float", !8, i64 0}
+!8 = !{!"omnipotent char", !9, i64 0}
+!9 = !{!"Simple C/C++ TBAA"}
+!10 = distinct !{!10, !11}
+!11 = !{!"llvm.loop.mustprogress"}
+!12 = !{!"llvm.loop.unroll.runtime.disable"}
+!13 = !{!"llvm.loop.isvectorized", i32 1}
+!14 = distinct !{!14, !11, !12, !13}
+;.
+; CHECK_SVE2: [[TBAA6]] = !{[[META7:![0-9]+]], [[META7]], i64 0}
+; CHECK_SVE2: [[META7]] = !{!"float", [[META8:![0-9]+]], i64 0}
+; CHECK_SVE2: [[META8]] = !{!"omnipotent char", [[META9:![0-9]+]], i64 0}
+; CHECK_SVE2: [[META9]] = !{!"Simple C/C++ TBAA"}
+; CHECK_SVE2: [[LOOP10]] = distinct !{[[LOOP10]], [[META11:![0-9]+]]}
+; CHECK_SVE2: [[META11]] = !{!"llvm.loop.mustprogress"}
+; CHECK_SVE2: [[LOOP12]] = distinct !{[[LOOP12]], [[META11]], [[META13:![0-9]+]], [[META14:![0-9]+]]}
+; CHECK_SVE2: [[META13]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK_SVE2: [[META14]] = !{!"llvm.loop.isvectorized", i32 1}
+;.
+; CHECK_SVE2p2: [[TBAA6]] = !{[[META7:![0-9]+]], [[META7]], i64 0}
+; CHECK_SVE2p2: [[META7]] = !{!"float", [[META8:![0-9]+]], i64 0}
+; CHECK_SVE2p2: [[META8]] = !{!"omnipotent char", [[META9:![0-9]+]], i64 0}
+; CHECK_SVE2p2: [[META9]] = !{!"Simple C/C++ TBAA"}
+; CHECK_SVE2p2: [[LOOP10]] = distinct !{[[LOOP10]], [[META11:![0-9]+]]}
+; CHECK_SVE2p2: [[META11]] = !{!"llvm.loop.mustprogress"}
+; CHECK_SVE2p2: [[LOOP12]] = distinct !{[[LOOP12]], [[META11]], [[META13:![0-9]+]], [[META14:![0-9]+]]}
+; CHECK_SVE2p2: [[META13]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK_SVE2p2: [[META14]] = !{!"llvm.loop.isvectorized", i32 1}
+;.
+; VECTORIZE_SVE2: [[TBAA6]] = !{[[META7:![0-9]+]], [[META7]], i64 0}
+; VECTORIZE_SVE2: [[META7]] = !{!"float", [[META8:![0-9]+]], i64 0}
+; VECTORIZE_SVE2: [[META8]] = !{!"omnipotent char", [[META9:![0-9]+]], i64 0}
+; VECTORIZE_SVE2: [[META9]] = !{!"Simple C/C++ TBAA"}
+; VECTORIZE_SVE2: [[LOOP10]] = distinct !{[[LOOP10]], [[META11:![0-9]+]]}
+; VECTORIZE_SVE2: [[META11]] = !{!"llvm.loop.mustprogress"}
+; VECTORIZE_SVE2: [[LOOP12]] = distinct !{[[LOOP12]], [[META11]], [[META13:![0-9]+]], [[META14:![0-9]+]]}
+; VECTORIZE_SVE2: [[META13]] = !{!"llvm.loop.unroll.runtime.disable"}
+; VECTORIZE_SVE2: [[META14]] = !{!"llvm.loop.isvectorized", i32 1}
+;.
+; VECTORIZE_SVE2p2: [[TBAA6]] = !{[[META7:![0-9]+]], [[META7]], i64 0}
+; VECTORIZE_SVE2p2: [[META7]] = !{!"float", [[META8:![0-9]+]], i64 0}
+; VECTORIZE_SVE2p2: [[META8]] = !{!"omnipotent char", [[META9:![0-9]+]], i64 0}
+; VECTORIZE_SVE2p2: [[META9]] = !{!"Simple C/C++ TBAA"}
+; VECTORIZE_SVE2p2: [[LOOP10]] = distinct !{[[LOOP10]], [[META11:![0-9]+]], [[META12:![0-9]+]], [[META13:![0-9]+]]}
+; VECTORIZE_SVE2p2: [[META11]] = !{!"llvm.loop.mustprogress"}
+; VECTORIZE_SVE2p2: [[META12]] = !{!"llvm.loop.isvectorized", i32 1}
+; VECTORIZE_SVE2p2: [[META13]] = !{!"llvm.loop.unroll.runtime.disable"}
+; VECTORIZE_SVE2p2: [[LOOP14]] = distinct !{[[LOOP14]], [[META11]], [[META13]], [[META12]]}
+; VECTORIZE_SVE2p2: [[LOOP15]] = distinct !{[[LOOP15]], [[META11]], [[META13]], [[META12]]}
+;.
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK_SVE2: {{.*}}
+; CHECK_SVE2p2: {{.*}}
+; VECTORIZE: {{.*}}
diff --git a/llvm/test/Transforms/LoopVectorize/reduction-add-inloop.ll b/llvm/test/Transforms/LoopVectorize/reduction-add-inloop.ll
new file mode 100644
index 0000000000000..e33298c55f17f
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/reduction-add-inloop.ll
@@ -0,0 +1,144 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -passes=loop-vectorize \
+; RUN: -prefer-inloop-reductions -vectorize-vector-loops \
+; RUN: -scalable-vectorization=on -force-vector-width=1 -force-vector-interleave=1 \
+; RUN: -mtriple=aarch64 -mattr=+sve2 -S | FileCheck %s --check-prefixes=CHECK,CHECK-SVE2
+; RUN: opt < %s -passes=loop-vectorize \
+; RUN: -prefer-inloop-reductions -vectorize-vector-loops \
+; RUN: -scalable-vectorization=on -force-vector-width=1 -force-vector-interleave=1 \
+; RUN: -mtriple=aarch64 -mattr=+sve2p1 -S | FileCheck %s --check-prefixes=CHECK,CHECK-SVE2p1
+; RUN: opt < %s -passes=loop-vectorize \
+; RUN: -prefer-inloop-reductions -vectorize-vector-loops \
+; RUN: -scalable-vectorization=on -force-vector-width=1 -force-vector-interleave=2 \
+; RUN: -mtriple=aarch64 -mattr=+sve2p1 -S | FileCheck %s --check-prefixes=CHECK-IC2
+
+define <4 x i32> @reduction_sum_4xi32(ptr noalias nocapture %A) {
+; CHECK-SVE2-LABEL: @reduction_sum_4xi32(
+; CHECK-SVE2-NEXT: entry:
+; CHECK-SVE2-NEXT: br label [[SCALAR_LOOP:%.*]]
+; CHECK-SVE2: scalar.loop:
+; CHECK-SVE2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[SCALAR_LOOP]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-SVE2-NEXT: [[SUM_02:%.*]] = phi <4 x i32> [ [[L7:%.*]], [[SCALAR_LOOP]] ], [ zeroinitializer, [[ENTRY]] ]
+; CHECK-SVE2-NEXT: [[L2:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDVARS_IV]]
+; CHECK-SVE2-NEXT: [[L3:%.*]] = load <4 x i32>, ptr [[L2]], align 4
+; CHECK-SVE2-NEXT: [[L7]] = add <4 x i32> [[SUM_02]], [[L3]]
+; CHECK-SVE2-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 4
+; CHECK-SVE2-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-SVE2-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], 256
+; CHECK-SVE2-NEXT: br i1 [[EXITCOND]], label [[LOOP_EXIT:%.*]], label [[SCALAR_LOOP]]
+; CHECK-SVE2: loop.exit:
+; CHECK-SVE2-NEXT: [[SUM_0_LCSSA:%.*]] = phi <4 x i32> [ [[L7]], [[SCALAR_LOOP]] ]
+; CHECK-SVE2-NEXT: ret <4 x i32> [[SUM_0_LCSSA]]
+;
+; CHECK-SVE2p1-LABEL: @reduction_sum_4xi32(
+; CHECK-SVE2p1-NEXT: entry:
+; CHECK-SVE2p1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE2p1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 64, [[TMP0]]
+; CHECK-SVE2p1-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-SVE2p1: vector.ph:
+; CHECK-SVE2p1-NEXT: [[N_MOD_VF:%.*]] = urem i64 64, [[TMP0]]
+; CHECK-SVE2p1-NEXT: [[N_VEC:%.*]] = sub i64 64, [[N_MOD_VF]]
+; CHECK-SVE2p1-NEXT: [[TMP2:%.*]] = shl i64 [[N_VEC]], 2
+; CHECK-SVE2p1-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK-SVE2p1: vector.body:
+; CHECK-SVE2p1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE2p1-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE2p1-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 2
+; CHECK-SVE2p1-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[OFFSET_IDX]]
+; CHECK-SVE2p1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP4]], align 4
+; CHECK-SVE2p1-NEXT: [[TMP5:%.*]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.nxv4i32(<4 x i32> zeroinitializer, <vscale x 4 x i32> [[WIDE_LOAD]])
+; CHECK-SVE2p1-NEXT: [[TMP7]] = add <4 x i32> [[VEC_PHI]], [[TMP5]]
+; CHECK-SVE2p1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; CHECK-SVE2p1-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-SVE2p1-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-SVE2p1: middle.block:
+; CHECK-SVE2p1-NEXT: [[CMP_N:%.*]] = icmp eq i64 64, [[N_VEC]]
+; CHECK-SVE2p1-NEXT: br i1 [[CMP_N]], label [[LOOP_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-SVE2p1: scalar.ph:
+; CHECK-SVE2p1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP2]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-SVE2p1-NEXT: [[BC_MERGE_RDX:%.*]] = phi <4 x i32> [ [[TMP7]], [[MIDDLE_BLOCK]] ], [ zeroinitializer, [[ENTRY]] ]
+; CHECK-SVE2p1-NEXT: br label [[SCALAR_LOOP:%.*]]
+; CHECK-SVE2p1: scalar.loop:
+; CHECK-SVE2p1-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[SCALAR_LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-SVE2p1-NEXT: [[SUM_02:%.*]] = phi <4 x i32> [ [[L7:%.*]], [[SCALAR_LOOP]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-SVE2p1-NEXT: [[L2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-SVE2p1-NEXT: [[L3:%.*]] = load <4 x i32>, ptr [[L2]], align 4
+; CHECK-SVE2p1-NEXT: [[L7]] = add <4 x i32> [[SUM_02]], [[L3]]
+; CHECK-SVE2p1-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 4
+; CHECK-SVE2p1-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-SVE2p1-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], 256
+; CHECK-SVE2p1-NEXT: br i1 [[EXITCOND]], label [[LOOP_EXIT]], label [[SCALAR_LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-SVE2p1: loop.exit:
+; CHECK-SVE2p1-NEXT: [[SUM_0_LCSSA:%.*]] = phi <4 x i32> [ [[L7]], [[SCALAR_LOOP]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]
+; CHECK-SVE2p1-NEXT: ret <4 x i32> [[SUM_0_LCSSA]]
+;
+; CHECK-IC2-LABEL: @reduction_sum_4xi32(
+; CHECK-IC2-NEXT: entry:
+; CHECK-IC2-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-IC2-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-IC2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 64, [[TMP1]]
+; CHECK-IC2-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-IC2: vector.ph:
+; CHECK-IC2-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-IC2-NEXT: [[N_MOD_VF:%.*]] = urem i64 64, [[TMP3]]
+; CHECK-IC2-NEXT: [[N_VEC:%.*]] = sub i64 64, [[N_MOD_VF]]
+; CHECK-IC2-NEXT: [[TMP4:%.*]] = shl i64 [[N_VEC]], 2
+; CHECK-IC2-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK-IC2: vector.body:
+; CHECK-IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-IC2-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ]
+; CHECK-IC2-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP14:%.*]], [[VECTOR_BODY]] ]
+; CHECK-IC2-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 2
+; CHECK-IC2-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[OFFSET_IDX]]
+; CHECK-IC2-NEXT: [[TMP9:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP6]], i64 [[TMP0]]
+; CHECK-IC2-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP6]], align 4
+; CHECK-IC2-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 4 x i32>, ptr [[TMP9]], align 4
+; CHECK-IC2-NEXT: [[TMP15:%.*]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.nxv4i32(<4 x i32> zeroinitializer, <vscale x 4 x i32> [[WIDE_LOAD]])
+; CHECK-IC2-NEXT: [[TMP10]] = add <4 x i32> [[VEC_PHI]], [[TMP15]]
+; CHECK-IC2-NEXT: [[TMP11:%.*]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.nxv4i32(<4 x i32> zeroinitializer, <vscale x 4 x i32> [[WIDE_LOAD2]])
+; CHECK-IC2-NEXT: [[TMP14]] = add <4 x i32> [[VEC_PHI1]], [[TMP11]]
+; CHECK-IC2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-IC2-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-IC2-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-IC2: middle.block:
+; CHECK-IC2-NEXT: [[TMP13:%.*]] = add <4 x i32> [[TMP14]], [[TMP10]]
+; CHECK-IC2-NEXT: [[CMP_N:%.*]] = icmp eq i64 64, [[N_VEC]]
+; CHECK-IC2-NEXT: br i1 [[CMP_N]], label [[LOOP_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-IC2: scalar.ph:
+; CHECK-IC2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP4]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-IC2-NEXT: [[BC_MERGE_RDX:%.*]] = phi <4 x i32> [ [[TMP13]], [[MIDDLE_BLOCK]] ], [ zeroinitializer, [[ENTRY]] ]
+; CHECK-IC2-NEXT: br label [[SCALAR_LOOP:%.*]]
+; CHECK-IC2: scalar.loop:
+; CHECK-IC2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[SCALAR_LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-IC2-NEXT: [[SUM_02:%.*]] = phi <4 x i32> [ [[L7:%.*]], [[SCALAR_LOOP]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-IC2-NEXT: [[L2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-IC2-NEXT: [[L3:%.*]] = load <4 x i32>, ptr [[L2]], align 4
+; CHECK-IC2-NEXT: [[L7]] = add <4 x i32> [[SUM_02]], [[L3]]
+; CHECK-IC2-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 4
+; CHECK-IC2-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-IC2-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], 256
+; CHECK-IC2-NEXT: br i1 [[EXITCOND]], label [[LOOP_EXIT]], label [[SCALAR_LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-IC2: loop.exit:
+; CHECK-IC2-NEXT: [[SUM_0_LCSSA:%.*]] = phi <4 x i32> [ [[L7]], [[SCALAR_LOOP]] ], [ [[TMP13]], [[MIDDLE_BLOCK]] ]
+; CHECK-IC2-NEXT: ret <4 x i32> [[SUM_0_LCSSA]]
+;
+entry:
+ br label %scalar.loop
+
+scalar.loop: ; preds = %entry, %scalar.loop
+ %indvars.iv = phi i64 [ %indvars.iv.next, %scalar.loop ], [ 0, %entry ]
+ %sum.02 = phi <4 x i32> [ %l7, %scalar.loop ], [ zeroinitializer, %entry ]
+ %l2 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
+ %l3 = load <4 x i32>, ptr %l2, align 4
+ %l7 = add <4 x i32> %sum.02, %l3
+ %indvars.iv.next = add i64 %indvars.iv, 4
+ %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+ %exitcond = icmp eq i32 %lftr.wideiv, 256
+ br i1 %exitcond, label %loop.exit, label %scalar.loop
+
+loop.exit: ; preds = %scalar.loop
+ %sum.0.lcssa = phi <4 x i32> [ %l7, %scalar.loop ]
+ ret <4 x i32> %sum.0.lcssa
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
diff --git a/llvm/test/Transforms/LoopVectorize/reduction-add.ll b/llvm/test/Transforms/LoopVectorize/reduction-add.ll
new file mode 100644
index 0000000000000..08fdeec9aec58
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/reduction-add.ll
@@ -0,0 +1,246 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "scalar.ph:" --replace-value-regex "!llvm.loop ![0-9]+" --version 6
+; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=1 -vectorize-vector-loops \
+; RUN: -mtriple=aarch64 -mattr=+sve2 -S | FileCheck %s --check-prefixes=CHECK,CHECK-SVE2
+; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=1 -vectorize-vector-loops \
+; RUN: -mtriple=aarch64 -mattr=+sve2p1 -S | FileCheck %s --check-prefixes=CHECK,CHECK-SVE2p1
+; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=2 -vectorize-vector-loops \
+; RUN: -mtriple=aarch64 -mattr=+sve2p1 -S | FileCheck %s --check-prefixes=CHECK-IC2
+
+; Test we can interleave reductions without vectorizing
+; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=2 -force-vector-width=1 \
+; RUN: -vectorize-vector-loops -mtriple=aarch64 -mattr=+sve2p1 -S \
+; RUN: | FileCheck %s --check-prefixes=CHECK-VF1-IC2
+
+define i32 @reduction_sum_single(ptr noalias nocapture %A) {
+; CHECK-LABEL: define i32 @reduction_sum_single(
+; CHECK-SAME: ptr noalias captures(none) [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 256, [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 256, [[TMP3]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 256, [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP4]], align 4
+; CHECK-NEXT: [[TMP5]] = add <vscale x 4 x i32> [[VEC_PHI]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP5]])
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 256, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[LOOP_EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+; CHECK-IC2-LABEL: define i32 @reduction_sum_single(
+; CHECK-IC2-SAME: ptr noalias captures(none) [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-IC2-NEXT: [[ENTRY:.*:]]
+; CHECK-IC2-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-IC2-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-IC2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 256, [[TMP1]]
+; CHECK-IC2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-IC2: [[VECTOR_PH]]:
+; CHECK-IC2-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-IC2-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP6]], 1
+; CHECK-IC2-NEXT: [[N_MOD_VF:%.*]] = urem i64 256, [[TMP3]]
+; CHECK-IC2-NEXT: [[N_VEC:%.*]] = sub i64 256, [[N_MOD_VF]]
+; CHECK-IC2-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-IC2: [[VECTOR_BODY]]:
+; CHECK-IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-IC2-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-IC2-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-IC2-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-IC2-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP4]], i64 [[TMP6]]
+; CHECK-IC2-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP4]], align 4
+; CHECK-IC2-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
+; CHECK-IC2-NEXT: [[TMP8]] = add <vscale x 4 x i32> [[VEC_PHI]], [[WIDE_LOAD]]
+; CHECK-IC2-NEXT: [[TMP9]] = add <vscale x 4 x i32> [[VEC_PHI1]], [[WIDE_LOAD2]]
+; CHECK-IC2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-IC2-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-IC2-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK-IC2: [[MIDDLE_BLOCK]]:
+; CHECK-IC2-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[TMP9]], [[TMP8]]
+; CHECK-IC2-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
+; CHECK-IC2-NEXT: [[CMP_N:%.*]] = icmp eq i64 256, [[N_VEC]]
+; CHECK-IC2-NEXT: br i1 [[CMP_N]], [[LOOP_EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK-IC2: [[SCALAR_PH]]:
+;
+; CHECK-VF1-IC2-LABEL: define i32 @reduction_sum_single(
+; CHECK-VF1-IC2-SAME: ptr noalias captures(none) [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-VF1-IC2-NEXT: [[ENTRY:.*:]]
+; CHECK-VF1-IC2-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-VF1-IC2: [[VECTOR_PH]]:
+; CHECK-VF1-IC2-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-VF1-IC2: [[VECTOR_BODY]]:
+; CHECK-VF1-IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1-IC2-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1-IC2-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1-IC2-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
+; CHECK-VF1-IC2-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-VF1-IC2-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP0]]
+; CHECK-VF1-IC2-NEXT: [[TMP3:%.*]] = load i32, ptr [[TMP1]], align 4
+; CHECK-VF1-IC2-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP2]], align 4
+; CHECK-VF1-IC2-NEXT: [[TMP5]] = add i32 [[VEC_PHI]], [[TMP3]]
+; CHECK-VF1-IC2-NEXT: [[TMP6]] = add i32 [[VEC_PHI1]], [[TMP4]]
+; CHECK-VF1-IC2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-VF1-IC2-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
+; CHECK-VF1-IC2-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK-VF1-IC2: [[MIDDLE_BLOCK]]:
+; CHECK-VF1-IC2-NEXT: [[BIN_RDX:%.*]] = add i32 [[TMP6]], [[TMP5]]
+; CHECK-VF1-IC2-NEXT: br label %[[LOOP_EXIT:.*]]
+; CHECK-VF1-IC2: [[LOOP_EXIT]]:
+; CHECK-VF1-IC2-NEXT: ret i32 [[BIN_RDX]]
+;
+entry:
+ br label %scalar.loop
+
+scalar.loop: ; preds = %entry, %scalar.loop
+ %indvars.iv = phi i64 [ %indvars.iv.next, %scalar.loop ], [ 0, %entry ]
+ %sum.02 = phi i32 [ %l7, %scalar.loop ], [ 0, %entry ]
+ %l2 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
+ %l3 = load i32, ptr %l2, align 4
+ %l7 = add i32 %sum.02, %l3
+ %indvars.iv.next = add i64 %indvars.iv, 1
+ %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+ %exitcond = icmp eq i32 %lftr.wideiv, 256
+ br i1 %exitcond, label %loop.exit, label %scalar.loop
+
+loop.exit: ; preds = %scalar.loop
+ %sum.0.lcssa = phi i32 [ %l7, %scalar.loop ]
+ ret i32 %sum.0.lcssa
+}
+
+define <4 x i32> @reduction_sum_4xi32(ptr noalias nocapture %A) {
+; CHECK-SVE2-LABEL: define <4 x i32> @reduction_sum_4xi32(
+; CHECK-SVE2-SAME: ptr noalias captures(none) [[A:%.*]]) #[[ATTR0]] {
+; CHECK-SVE2-NEXT: [[ENTRY:.*]]:
+; CHECK-SVE2-NEXT: br label %[[SCALAR_LOOP:.*]]
+; CHECK-SVE2: [[SCALAR_LOOP]]:
+; CHECK-SVE2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[SCALAR_LOOP]] ], [ 0, %[[ENTRY]] ]
+; CHECK-SVE2-NEXT: [[SUM_02:%.*]] = phi <4 x i32> [ [[L7:%.*]], %[[SCALAR_LOOP]] ], [ zeroinitializer, %[[ENTRY]] ]
+; CHECK-SVE2-NEXT: [[L2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-SVE2-NEXT: [[L3:%.*]] = load <4 x i32>, ptr [[L2]], align 4
+; CHECK-SVE2-NEXT: [[L7]] = add <4 x i32> [[SUM_02]], [[L3]]
+; CHECK-SVE2-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 4
+; CHECK-SVE2-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-SVE2-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], 256
+; CHECK-SVE2-NEXT: br i1 [[EXITCOND]], label %[[LOOP_EXIT:.*]], label %[[SCALAR_LOOP]]
+; CHECK-SVE2: [[LOOP_EXIT]]:
+; CHECK-SVE2-NEXT: [[SUM_0_LCSSA:%.*]] = phi <4 x i32> [ [[L7]], %[[SCALAR_LOOP]] ]
+; CHECK-SVE2-NEXT: ret <4 x i32> [[SUM_0_LCSSA]]
+;
+; CHECK-SVE2p1-LABEL: define <4 x i32> @reduction_sum_4xi32(
+; CHECK-SVE2p1-SAME: ptr noalias captures(none) [[A:%.*]]) #[[ATTR0]] {
+; CHECK-SVE2p1-NEXT: [[ENTRY:.*:]]
+; CHECK-SVE2p1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE2p1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 64, [[TMP0]]
+; CHECK-SVE2p1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-SVE2p1: [[VECTOR_PH]]:
+; CHECK-SVE2p1-NEXT: [[N_MOD_VF:%.*]] = urem i64 64, [[TMP0]]
+; CHECK-SVE2p1-NEXT: [[N_VEC:%.*]] = sub i64 64, [[N_MOD_VF]]
+; CHECK-SVE2p1-NEXT: [[TMP2:%.*]] = shl i64 [[N_VEC]], 2
+; CHECK-SVE2p1-NEXT: [[TMP3:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v4i32(<vscale x 4 x i32> zeroinitializer, <4 x i32> zeroinitializer, i64 0)
+; CHECK-SVE2p1-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-SVE2p1: [[VECTOR_BODY]]:
+; CHECK-SVE2p1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-SVE2p1-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ [[TMP3]], %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-SVE2p1-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 2
+; CHECK-SVE2p1-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[OFFSET_IDX]]
+; CHECK-SVE2p1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP4]], align 4
+; CHECK-SVE2p1-NEXT: [[TMP5]] = add <vscale x 4 x i32> [[VEC_PHI]], [[WIDE_LOAD]]
+; CHECK-SVE2p1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; CHECK-SVE2p1-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-SVE2p1-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK-SVE2p1: [[MIDDLE_BLOCK]]:
+; CHECK-SVE2p1-NEXT: [[TMP7:%.*]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.nxv4i32(<4 x i32> zeroinitializer, <vscale x 4 x i32> [[TMP5]])
+; CHECK-SVE2p1-NEXT: [[CMP_N:%.*]] = icmp eq i64 64, [[N_VEC]]
+; CHECK-SVE2p1-NEXT: br i1 [[CMP_N]], [[LOOP_EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK-SVE2p1: [[SCALAR_PH]]:
+;
+; CHECK-IC2-LABEL: define <4 x i32> @reduction_sum_4xi32(
+; CHECK-IC2-SAME: ptr noalias captures(none) [[A:%.*]]) #[[ATTR0]] {
+; CHECK-IC2-NEXT: [[ENTRY:.*:]]
+; CHECK-IC2-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-IC2-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-IC2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 64, [[TMP1]]
+; CHECK-IC2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-IC2: [[VECTOR_PH]]:
+; CHECK-IC2-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-IC2-NEXT: [[N_MOD_VF:%.*]] = urem i64 64, [[TMP3]]
+; CHECK-IC2-NEXT: [[N_VEC:%.*]] = sub i64 64, [[N_MOD_VF]]
+; CHECK-IC2-NEXT: [[TMP4:%.*]] = shl i64 [[N_VEC]], 2
+; CHECK-IC2-NEXT: [[TMP5:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v4i32(<vscale x 4 x i32> zeroinitializer, <4 x i32> zeroinitializer, i64 0)
+; CHECK-IC2-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-IC2: [[VECTOR_BODY]]:
+; CHECK-IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-IC2-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ [[TMP5]], %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-IC2-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-IC2-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 2
+; CHECK-IC2-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[OFFSET_IDX]]
+; CHECK-IC2-NEXT: [[TMP9:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP6]], i64 [[TMP0]]
+; CHECK-IC2-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP6]], align 4
+; CHECK-IC2-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 4 x i32>, ptr [[TMP9]], align 4
+; CHECK-IC2-NEXT: [[TMP10]] = add <vscale x 4 x i32> [[VEC_PHI]], [[WIDE_LOAD]]
+; CHECK-IC2-NEXT: [[TMP11]] = add <vscale x 4 x i32> [[VEC_PHI1]], [[WIDE_LOAD2]]
+; CHECK-IC2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-IC2-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-IC2-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK-IC2: [[MIDDLE_BLOCK]]:
+; CHECK-IC2-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[TMP11]], [[TMP10]]
+; CHECK-IC2-NEXT: [[TMP13:%.*]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.nxv4i32(<4 x i32> zeroinitializer, <vscale x 4 x i32> [[BIN_RDX]])
+; CHECK-IC2-NEXT: [[CMP_N:%.*]] = icmp eq i64 64, [[N_VEC]]
+; CHECK-IC2-NEXT: br i1 [[CMP_N]], [[LOOP_EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK-IC2: [[SCALAR_PH]]:
+;
+; CHECK-VF1-IC2-LABEL: define <4 x i32> @reduction_sum_4xi32(
+; CHECK-VF1-IC2-SAME: ptr noalias captures(none) [[A:%.*]]) #[[ATTR0]] {
+; CHECK-VF1-IC2-NEXT: [[ENTRY:.*:]]
+; CHECK-VF1-IC2-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-VF1-IC2: [[VECTOR_PH]]:
+; CHECK-VF1-IC2-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-VF1-IC2: [[VECTOR_BODY]]:
+; CHECK-VF1-IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1-IC2-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1-IC2-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1-IC2-NEXT: [[TMP0:%.*]] = shl i64 [[INDEX]], 2
+; CHECK-VF1-IC2-NEXT: [[TMP1:%.*]] = add i64 [[TMP0]], 4
+; CHECK-VF1-IC2-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP0]]
+; CHECK-VF1-IC2-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP1]]
+; CHECK-VF1-IC2-NEXT: [[TMP4:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-VF1-IC2-NEXT: [[TMP5:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
+; CHECK-VF1-IC2-NEXT: [[TMP6]] = add <4 x i32> [[VEC_PHI]], [[TMP4]]
+; CHECK-VF1-IC2-NEXT: [[TMP7]] = add <4 x i32> [[VEC_PHI1]], [[TMP5]]
+; CHECK-VF1-IC2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-VF1-IC2-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64
+; CHECK-VF1-IC2-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK-VF1-IC2: [[MIDDLE_BLOCK]]:
+; CHECK-VF1-IC2-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[TMP7]], [[TMP6]]
+; CHECK-VF1-IC2-NEXT: br label %[[LOOP_EXIT:.*]]
+; CHECK-VF1-IC2: [[LOOP_EXIT]]:
+; CHECK-VF1-IC2-NEXT: ret <4 x i32> [[BIN_RDX]]
+;
+entry:
+ br label %scalar.loop
+
+scalar.loop: ; preds = %entry, %scalar.loop
+ %indvars.iv = phi i64 [ %indvars.iv.next, %scalar.loop ], [ 0, %entry ]
+ %sum.02 = phi <4 x i32> [ %l7, %scalar.loop ], [ zeroinitializer, %entry ]
+ %l2 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
+ %l3 = load <4 x i32>, ptr %l2, align 4
+ %l7 = add <4 x i32> %sum.02, %l3
+ %indvars.iv.next = add i64 %indvars.iv, 4
+ %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+ %exitcond = icmp eq i32 %lftr.wideiv, 256
+ br i1 %exitcond, label %loop.exit, label %scalar.loop
+
+loop.exit: ; preds = %scalar.loop
+ %sum.0.lcssa = phi <4 x i32> [ %l7, %scalar.loop ]
+ ret <4 x i32> %sum.0.lcssa
+}
diff --git a/llvm/test/Transforms/LoopVectorize/reduction-fadd.ll b/llvm/test/Transforms/LoopVectorize/reduction-fadd.ll
new file mode 100644
index 0000000000000..975e0ed4e49ca
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/reduction-fadd.ll
@@ -0,0 +1,266 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "scalar.ph:" --replace-value-regex "!llvm.loop ![0-9]+" --version 6
+; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=1 -vectorize-vector-loops \
+; RUN: -mtriple=aarch64 -mattr=+sve2 -S | FileCheck %s --check-prefixes=CHECK
+; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=1 -vectorize-vector-loops \
+; RUN: -mtriple=aarch64 -mattr=+sve2p1 -S | FileCheck %s --check-prefixes=CHECK
+; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=2 -vectorize-vector-loops \
+; RUN: -mtriple=aarch64 -mattr=+sve2p1 -S | FileCheck %s --check-prefixes=CHECK-IC2
+
+; Test whether we can interleave reductions without vectorizing.
+; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=2 -force-vector-width=1 \
+; RUN: -vectorize-vector-loops -mtriple=aarch64 -mattr=+sve2p1 -S \
+; RUN: | FileCheck %s --check-prefixes=CHECK-VF1-IC2
+
+define float @reduction_fadd_single(ptr noalias nocapture %A) {
+; CHECK-LABEL: define float @reduction_fadd_single(
+; CHECK-SAME: ptr noalias captures(none) [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 256, [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 256, [[TMP2]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 256, [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP3]], align 4
+; CHECK-NEXT: [[TMP4]] = fadd fast <vscale x 4 x float> [[VEC_PHI]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP6:%.*]] = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> [[TMP4]])
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 256, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[LOOP_EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+; CHECK-IC2-LABEL: define float @reduction_fadd_single(
+; CHECK-IC2-SAME: ptr noalias captures(none) [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-IC2-NEXT: [[ENTRY:.*:]]
+; CHECK-IC2-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-IC2-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-IC2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 256, [[TMP1]]
+; CHECK-IC2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-IC2: [[VECTOR_PH]]:
+; CHECK-IC2-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-IC2-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-IC2-NEXT: [[N_MOD_VF:%.*]] = urem i64 256, [[TMP3]]
+; CHECK-IC2-NEXT: [[N_VEC:%.*]] = sub i64 256, [[N_MOD_VF]]
+; CHECK-IC2-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-IC2: [[VECTOR_BODY]]:
+; CHECK-IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-IC2-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-IC2-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-IC2-NEXT: [[TMP4:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-IC2-NEXT: [[TMP5:%.*]] = getelementptr inbounds float, ptr [[TMP4]], i64 [[TMP2]]
+; CHECK-IC2-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP4]], align 4
+; CHECK-IC2-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 4 x float>, ptr [[TMP5]], align 4
+; CHECK-IC2-NEXT: [[TMP6]] = fadd fast <vscale x 4 x float> [[VEC_PHI]], [[WIDE_LOAD]]
+; CHECK-IC2-NEXT: [[TMP7]] = fadd fast <vscale x 4 x float> [[VEC_PHI1]], [[WIDE_LOAD2]]
+; CHECK-IC2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-IC2-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-IC2-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK-IC2: [[MIDDLE_BLOCK]]:
+; CHECK-IC2-NEXT: [[BIN_RDX:%.*]] = fadd fast <vscale x 4 x float> [[TMP7]], [[TMP6]]
+; CHECK-IC2-NEXT: [[TMP9:%.*]] = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> [[BIN_RDX]])
+; CHECK-IC2-NEXT: [[CMP_N:%.*]] = icmp eq i64 256, [[N_VEC]]
+; CHECK-IC2-NEXT: br i1 [[CMP_N]], [[LOOP_EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK-IC2: [[SCALAR_PH]]:
+;
+; CHECK-VF1-IC2-LABEL: define float @reduction_fadd_single(
+; CHECK-VF1-IC2-SAME: ptr noalias captures(none) [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-VF1-IC2-NEXT: [[ENTRY:.*:]]
+; CHECK-VF1-IC2-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-VF1-IC2: [[VECTOR_PH]]:
+; CHECK-VF1-IC2-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-VF1-IC2: [[VECTOR_BODY]]:
+; CHECK-VF1-IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1-IC2-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1-IC2-NEXT: [[VEC_PHI1:%.*]] = phi float [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1-IC2-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
+; CHECK-VF1-IC2-NEXT: [[TMP1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-VF1-IC2-NEXT: [[TMP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP0]]
+; CHECK-VF1-IC2-NEXT: [[TMP3:%.*]] = load float, ptr [[TMP1]], align 4
+; CHECK-VF1-IC2-NEXT: [[TMP4:%.*]] = load float, ptr [[TMP2]], align 4
+; CHECK-VF1-IC2-NEXT: [[TMP5]] = fadd fast float [[VEC_PHI]], [[TMP3]]
+; CHECK-VF1-IC2-NEXT: [[TMP6]] = fadd fast float [[VEC_PHI1]], [[TMP4]]
+; CHECK-VF1-IC2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-VF1-IC2-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
+; CHECK-VF1-IC2-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK-VF1-IC2: [[MIDDLE_BLOCK]]:
+; CHECK-VF1-IC2-NEXT: [[BIN_RDX:%.*]] = fadd fast float [[TMP6]], [[TMP5]]
+; CHECK-VF1-IC2-NEXT: br label %[[LOOP_EXIT:.*]]
+; CHECK-VF1-IC2: [[LOOP_EXIT]]:
+; CHECK-VF1-IC2-NEXT: ret float [[BIN_RDX]]
+;
+entry:
+ br label %scalar.loop
+
+scalar.loop: ; preds = %entry, %scalar.loop
+ %indvars.iv = phi i64 [ %indvars.iv.next, %scalar.loop ], [ 0, %entry ]
+ %sum.02 = phi float [ %l7, %scalar.loop ], [ 0.000000e+00, %entry ]
+ %l2 = getelementptr inbounds float, ptr %A, i64 %indvars.iv
+ %l3 = load float, ptr %l2, align 4
+ %l7 = fadd fast float %sum.02, %l3
+ %indvars.iv.next = add i64 %indvars.iv, 1
+ %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+ %exitcond = icmp eq i32 %lftr.wideiv, 256
+ br i1 %exitcond, label %loop.exit, label %scalar.loop
+
+loop.exit: ; preds = %scalar.loop
+ %sum.0.lcssa = phi float [ %l7, %scalar.loop ]
+ ret float %sum.0.lcssa
+}
+
+define <4 x float> @reduction_fadd_4xf32(ptr noalias nocapture %A) {
+; CHECK-LABEL: define <4 x float> @reduction_fadd_4xf32(
+; CHECK-SAME: ptr noalias captures(none) [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[SCALAR_LOOP:.*]]
+; CHECK: [[SCALAR_LOOP]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[SCALAR_LOOP]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[SUM_02:%.*]] = phi <4 x float> [ [[L7:%.*]], %[[SCALAR_LOOP]] ], [ zeroinitializer, %[[ENTRY]] ]
+; CHECK-NEXT: [[L2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[L3:%.*]] = load <4 x float>, ptr [[L2]], align 4
+; CHECK-NEXT: [[L7]] = fadd <4 x float> [[SUM_02]], [[L3]]
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 4
+; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], 256
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[LOOP_EXIT:.*]], label %[[SCALAR_LOOP]]
+; CHECK: [[LOOP_EXIT]]:
+; CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi <4 x float> [ [[L7]], %[[SCALAR_LOOP]] ]
+; CHECK-NEXT: ret <4 x float> [[SUM_0_LCSSA]]
+;
+; CHECK-IC2-LABEL: define <4 x float> @reduction_fadd_4xf32(
+; CHECK-IC2-SAME: ptr noalias captures(none) [[A:%.*]]) #[[ATTR0]] {
+; CHECK-IC2-NEXT: [[ENTRY:.*]]:
+; CHECK-IC2-NEXT: br label %[[SCALAR_LOOP:.*]]
+; CHECK-IC2: [[SCALAR_LOOP]]:
+; CHECK-IC2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[SCALAR_LOOP]] ], [ 0, %[[ENTRY]] ]
+; CHECK-IC2-NEXT: [[SUM_02:%.*]] = phi <4 x float> [ [[L7:%.*]], %[[SCALAR_LOOP]] ], [ zeroinitializer, %[[ENTRY]] ]
+; CHECK-IC2-NEXT: [[L2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-IC2-NEXT: [[L3:%.*]] = load <4 x float>, ptr [[L2]], align 4
+; CHECK-IC2-NEXT: [[L7]] = fadd <4 x float> [[SUM_02]], [[L3]]
+; CHECK-IC2-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 4
+; CHECK-IC2-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-IC2-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], 256
+; CHECK-IC2-NEXT: br i1 [[EXITCOND]], label %[[LOOP_EXIT:.*]], label %[[SCALAR_LOOP]]
+; CHECK-IC2: [[LOOP_EXIT]]:
+; CHECK-IC2-NEXT: [[SUM_0_LCSSA:%.*]] = phi <4 x float> [ [[L7]], %[[SCALAR_LOOP]] ]
+; CHECK-IC2-NEXT: ret <4 x float> [[SUM_0_LCSSA]]
+;
+; CHECK-VF1-IC2-LABEL: define <4 x float> @reduction_fadd_4xf32(
+; CHECK-VF1-IC2-SAME: ptr noalias captures(none) [[A:%.*]]) #[[ATTR0]] {
+; CHECK-VF1-IC2-NEXT: [[ENTRY:.*]]:
+; CHECK-VF1-IC2-NEXT: br label %[[SCALAR_LOOP:.*]]
+; CHECK-VF1-IC2: [[SCALAR_LOOP]]:
+; CHECK-VF1-IC2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[SCALAR_LOOP]] ], [ 0, %[[ENTRY]] ]
+; CHECK-VF1-IC2-NEXT: [[SUM_02:%.*]] = phi <4 x float> [ [[L7:%.*]], %[[SCALAR_LOOP]] ], [ zeroinitializer, %[[ENTRY]] ]
+; CHECK-VF1-IC2-NEXT: [[L2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-VF1-IC2-NEXT: [[L3:%.*]] = load <4 x float>, ptr [[L2]], align 4
+; CHECK-VF1-IC2-NEXT: [[L7]] = fadd <4 x float> [[SUM_02]], [[L3]]
+; CHECK-VF1-IC2-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 4
+; CHECK-VF1-IC2-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-VF1-IC2-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], 256
+; CHECK-VF1-IC2-NEXT: br i1 [[EXITCOND]], label %[[LOOP_EXIT:.*]], label %[[SCALAR_LOOP]]
+; CHECK-VF1-IC2: [[LOOP_EXIT]]:
+; CHECK-VF1-IC2-NEXT: [[SUM_0_LCSSA:%.*]] = phi <4 x float> [ [[L7]], %[[SCALAR_LOOP]] ]
+; CHECK-VF1-IC2-NEXT: ret <4 x float> [[SUM_0_LCSSA]]
+;
+entry:
+ br label %scalar.loop
+
+scalar.loop: ; preds = %entry, %scalar.loop
+ %indvars.iv = phi i64 [ %indvars.iv.next, %scalar.loop ], [ 0, %entry ]
+ %sum.02 = phi <4 x float> [ %l7, %scalar.loop ], [ zeroinitializer, %entry ]
+ %l2 = getelementptr inbounds float, ptr %A, i64 %indvars.iv
+ %l3 = load <4 x float>, ptr %l2, align 4
+ %l7 = fadd <4 x float> %sum.02, %l3
+ %indvars.iv.next = add i64 %indvars.iv, 4
+ %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+ %exitcond = icmp eq i32 %lftr.wideiv, 256
+ br i1 %exitcond, label %loop.exit, label %scalar.loop
+
+loop.exit: ; preds = %scalar.loop
+ %sum.0.lcssa = phi <4 x float> [ %l7, %scalar.loop ]
+ ret <4 x float> %sum.0.lcssa
+}
+
+define <4 x float> @reduction_fadd_fast_4xf32(ptr noalias nocapture %A) {
+; CHECK-LABEL: define <4 x float> @reduction_fadd_fast_4xf32(
+; CHECK-SAME: ptr noalias captures(none) [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[SCALAR_LOOP:.*]]
+; CHECK: [[SCALAR_LOOP]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[SCALAR_LOOP]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[SUM_02:%.*]] = phi fast <4 x float> [ [[L7:%.*]], %[[SCALAR_LOOP]] ], [ zeroinitializer, %[[ENTRY]] ]
+; CHECK-NEXT: [[L2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[L3:%.*]] = load <4 x float>, ptr [[L2]], align 4
+; CHECK-NEXT: [[L7]] = fadd fast <4 x float> [[SUM_02]], [[L3]]
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 4
+; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], 256
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[LOOP_EXIT:.*]], label %[[SCALAR_LOOP]]
+; CHECK: [[LOOP_EXIT]]:
+; CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi fast <4 x float> [ [[L7]], %[[SCALAR_LOOP]] ]
+; CHECK-NEXT: ret <4 x float> [[SUM_0_LCSSA]]
+;
+; CHECK-IC2-LABEL: define <4 x float> @reduction_fadd_fast_4xf32(
+; CHECK-IC2-SAME: ptr noalias captures(none) [[A:%.*]]) #[[ATTR0]] {
+; CHECK-IC2-NEXT: [[ENTRY:.*]]:
+; CHECK-IC2-NEXT: br label %[[SCALAR_LOOP:.*]]
+; CHECK-IC2: [[SCALAR_LOOP]]:
+; CHECK-IC2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[SCALAR_LOOP]] ], [ 0, %[[ENTRY]] ]
+; CHECK-IC2-NEXT: [[SUM_02:%.*]] = phi fast <4 x float> [ [[L7:%.*]], %[[SCALAR_LOOP]] ], [ zeroinitializer, %[[ENTRY]] ]
+; CHECK-IC2-NEXT: [[L2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-IC2-NEXT: [[L3:%.*]] = load <4 x float>, ptr [[L2]], align 4
+; CHECK-IC2-NEXT: [[L7]] = fadd fast <4 x float> [[SUM_02]], [[L3]]
+; CHECK-IC2-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 4
+; CHECK-IC2-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-IC2-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], 256
+; CHECK-IC2-NEXT: br i1 [[EXITCOND]], label %[[LOOP_EXIT:.*]], label %[[SCALAR_LOOP]]
+; CHECK-IC2: [[LOOP_EXIT]]:
+; CHECK-IC2-NEXT: [[SUM_0_LCSSA:%.*]] = phi fast <4 x float> [ [[L7]], %[[SCALAR_LOOP]] ]
+; CHECK-IC2-NEXT: ret <4 x float> [[SUM_0_LCSSA]]
+;
+; CHECK-VF1-IC2-LABEL: define <4 x float> @reduction_fadd_fast_4xf32(
+; CHECK-VF1-IC2-SAME: ptr noalias captures(none) [[A:%.*]]) #[[ATTR0]] {
+; CHECK-VF1-IC2-NEXT: [[ENTRY:.*]]:
+; CHECK-VF1-IC2-NEXT: br label %[[SCALAR_LOOP:.*]]
+; CHECK-VF1-IC2: [[SCALAR_LOOP]]:
+; CHECK-VF1-IC2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[SCALAR_LOOP]] ], [ 0, %[[ENTRY]] ]
+; CHECK-VF1-IC2-NEXT: [[SUM_02:%.*]] = phi fast <4 x float> [ [[L7:%.*]], %[[SCALAR_LOOP]] ], [ zeroinitializer, %[[ENTRY]] ]
+; CHECK-VF1-IC2-NEXT: [[L2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-VF1-IC2-NEXT: [[L3:%.*]] = load <4 x float>, ptr [[L2]], align 4
+; CHECK-VF1-IC2-NEXT: [[L7]] = fadd fast <4 x float> [[SUM_02]], [[L3]]
+; CHECK-VF1-IC2-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 4
+; CHECK-VF1-IC2-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-VF1-IC2-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], 256
+; CHECK-VF1-IC2-NEXT: br i1 [[EXITCOND]], label %[[LOOP_EXIT:.*]], label %[[SCALAR_LOOP]]
+; CHECK-VF1-IC2: [[LOOP_EXIT]]:
+; CHECK-VF1-IC2-NEXT: [[SUM_0_LCSSA:%.*]] = phi fast <4 x float> [ [[L7]], %[[SCALAR_LOOP]] ]
+; CHECK-VF1-IC2-NEXT: ret <4 x float> [[SUM_0_LCSSA]]
+;
+entry:
+ br label %scalar.loop
+
+scalar.loop: ; preds = %entry, %scalar.loop
+ %indvars.iv = phi i64 [ %indvars.iv.next, %scalar.loop ], [ 0, %entry ]
+ %sum.02 = phi fast <4 x float> [ %l7, %scalar.loop ], [ zeroinitializer, %entry ]
+ %l2 = getelementptr inbounds float, ptr %A, i64 %indvars.iv
+ %l3 = load <4 x float>, ptr %l2, align 4
+ %l7 = fadd fast <4 x float> %sum.02, %l3
+ %indvars.iv.next = add i64 %indvars.iv, 4
+ %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+ %exitcond = icmp eq i32 %lftr.wideiv, 256
+ br i1 %exitcond, label %loop.exit, label %scalar.loop
+
+loop.exit: ; preds = %scalar.loop
+ %sum.0.lcssa = phi fast <4 x float> [ %l7, %scalar.loop ]
+ ret <4 x float> %sum.0.lcssa
+}
diff --git a/llvm/test/Transforms/LoopVectorize/reduction-min-max.ll b/llvm/test/Transforms/LoopVectorize/reduction-min-max.ll
new file mode 100644
index 0000000000000..1c0ba2be6b6c3
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/reduction-min-max.ll
@@ -0,0 +1,497 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "scalar.ph:" --replace-value-regex "!llvm.loop ![0-9]+" --version 6
+; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=1 -vectorize-vector-loops \
+; RUN: -mtriple=aarch64 -mattr=+sve2 -S | FileCheck %s --check-prefixes=CHECK,CHECK-SVE2
+; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=1 -vectorize-vector-loops \
+; RUN: -mtriple=aarch64 -mattr=+sve2p1 -S | FileCheck %s --check-prefixes=CHECK,CHECK-SVE2p1
+; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=2 -vectorize-vector-loops \
+; RUN: -mtriple=aarch64 -mattr=+sve2p1 -S | FileCheck %s --check-prefixes=CHECK-IC2
+
+; Test we can interleave reductions without vectorizing
+; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=2 -force-vector-width=1 \
+; RUN: -vectorize-vector-loops -mtriple=aarch64 -mattr=+sve2p1 -S \
+; RUN: | FileCheck %s --check-prefixes=CHECK-VF1-IC2
+
+define i32 @smax_intrinsic(ptr nocapture readonly %x) {
+; CHECK-LABEL: define i32 @smax_intrinsic(
+; CHECK-SAME: ptr readonly captures(none) [[X:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 2
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 1024, [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i32 [[TMP0]], 2
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP3]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 1024, [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ splat (i32 100), %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[X]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP4]], align 4
+; CHECK-NEXT: [[TMP5]] = call <vscale x 4 x i32> @llvm.smax.nxv4i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[WIDE_LOAD]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP3]]
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.smax.nxv4i32(<vscale x 4 x i32> [[TMP5]])
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 1024, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+; CHECK-IC2-LABEL: define i32 @smax_intrinsic(
+; CHECK-IC2-SAME: ptr readonly captures(none) [[X:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-IC2-NEXT: [[ENTRY:.*:]]
+; CHECK-IC2-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-IC2-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 3
+; CHECK-IC2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 1024, [[TMP1]]
+; CHECK-IC2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-IC2: [[VECTOR_PH]]:
+; CHECK-IC2-NEXT: [[TMP2:%.*]] = shl nuw i32 [[TMP0]], 2
+; CHECK-IC2-NEXT: [[TMP3:%.*]] = shl nuw i32 [[TMP2]], 1
+; CHECK-IC2-NEXT: [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP3]]
+; CHECK-IC2-NEXT: [[N_VEC:%.*]] = sub i32 1024, [[N_MOD_VF]]
+; CHECK-IC2-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-IC2: [[VECTOR_BODY]]:
+; CHECK-IC2-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-IC2-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ splat (i32 100), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-IC2-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ splat (i32 100), %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-IC2-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[X]], i32 [[INDEX]]
+; CHECK-IC2-NEXT: [[TMP6:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-IC2-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP4]], i64 [[TMP6]]
+; CHECK-IC2-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP4]], align 4
+; CHECK-IC2-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
+; CHECK-IC2-NEXT: [[TMP8]] = call <vscale x 4 x i32> @llvm.smax.nxv4i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[WIDE_LOAD]])
+; CHECK-IC2-NEXT: [[TMP9]] = call <vscale x 4 x i32> @llvm.smax.nxv4i32(<vscale x 4 x i32> [[VEC_PHI1]], <vscale x 4 x i32> [[WIDE_LOAD2]])
+; CHECK-IC2-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP3]]
+; CHECK-IC2-NEXT: [[TMP10:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-IC2-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK-IC2: [[MIDDLE_BLOCK]]:
+; CHECK-IC2-NEXT: [[RDX_MINMAX:%.*]] = call <vscale x 4 x i32> @llvm.smax.nxv4i32(<vscale x 4 x i32> [[TMP8]], <vscale x 4 x i32> [[TMP9]])
+; CHECK-IC2-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.smax.nxv4i32(<vscale x 4 x i32> [[RDX_MINMAX]])
+; CHECK-IC2-NEXT: [[CMP_N:%.*]] = icmp eq i32 1024, [[N_VEC]]
+; CHECK-IC2-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP:label %.*]], label %[[SCALAR_PH]]
+; CHECK-IC2: [[SCALAR_PH]]:
+;
+; CHECK-VF1-IC2-LABEL: define i32 @smax_intrinsic(
+; CHECK-VF1-IC2-SAME: ptr readonly captures(none) [[X:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-VF1-IC2-NEXT: [[ENTRY:.*:]]
+; CHECK-VF1-IC2-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-VF1-IC2: [[VECTOR_PH]]:
+; CHECK-VF1-IC2-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-VF1-IC2: [[VECTOR_BODY]]:
+; CHECK-VF1-IC2-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1-IC2-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 100, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1-IC2-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ 100, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1-IC2-NEXT: [[TMP0:%.*]] = add i32 [[INDEX]], 1
+; CHECK-VF1-IC2-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[X]], i32 [[INDEX]]
+; CHECK-VF1-IC2-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[X]], i32 [[TMP0]]
+; CHECK-VF1-IC2-NEXT: [[TMP3:%.*]] = load i32, ptr [[TMP1]], align 4
+; CHECK-VF1-IC2-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP2]], align 4
+; CHECK-VF1-IC2-NEXT: [[TMP5]] = tail call i32 @llvm.smax.i32(i32 [[VEC_PHI]], i32 [[TMP3]])
+; CHECK-VF1-IC2-NEXT: [[TMP6]] = tail call i32 @llvm.smax.i32(i32 [[VEC_PHI1]], i32 [[TMP4]])
+; CHECK-VF1-IC2-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; CHECK-VF1-IC2-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
+; CHECK-VF1-IC2-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK-VF1-IC2: [[MIDDLE_BLOCK]]:
+; CHECK-VF1-IC2-NEXT: [[RDX_MINMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[TMP5]], i32 [[TMP6]])
+; CHECK-VF1-IC2-NEXT: br label %[[FOR_COND_CLEANUP:.*]]
+; CHECK-VF1-IC2: [[FOR_COND_CLEANUP]]:
+; CHECK-VF1-IC2-NEXT: ret i32 [[RDX_MINMAX]]
+;
+entry:
+ br label %for.body
+
+for.body: ; preds = %entry, %for.body
+ %i.012 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+ %s.011 = phi i32 [ 100, %entry ], [ %1, %for.body ]
+ %arrayidx = getelementptr inbounds i32, ptr %x, i32 %i.012
+ %0 = load i32, ptr %arrayidx, align 4
+ %1 = tail call i32 @llvm.smax.i32(i32 %s.011, i32 %0)
+ %inc = add nuw nsw i32 %i.012, 1
+ %exitcond.not = icmp eq i32 %inc, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret i32 %1
+}
+
+define <4 x i32> @reduction_smax_4xi32(ptr noalias nocapture %A) {
+; CHECK-SVE2-LABEL: define <4 x i32> @reduction_smax_4xi32(
+; CHECK-SVE2-SAME: ptr noalias captures(none) [[A:%.*]]) #[[ATTR0]] {
+; CHECK-SVE2-NEXT: [[ENTRY:.*]]:
+; CHECK-SVE2-NEXT: br label %[[SCALAR_LOOP:.*]]
+; CHECK-SVE2: [[SCALAR_LOOP]]:
+; CHECK-SVE2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[SCALAR_LOOP]] ], [ 0, %[[ENTRY]] ]
+; CHECK-SVE2-NEXT: [[ACC_02:%.*]] = phi <4 x i32> [ [[L7:%.*]], %[[SCALAR_LOOP]] ], [ zeroinitializer, %[[ENTRY]] ]
+; CHECK-SVE2-NEXT: [[L2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-SVE2-NEXT: [[L3:%.*]] = load <4 x i32>, ptr [[L2]], align 4
+; CHECK-SVE2-NEXT: [[L7]] = tail call <4 x i32> @llvm.smax.v4i32(<4 x i32> [[ACC_02]], <4 x i32> [[L3]])
+; CHECK-SVE2-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 4
+; CHECK-SVE2-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-SVE2-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], 256
+; CHECK-SVE2-NEXT: br i1 [[EXITCOND]], label %[[LOOP_EXIT:.*]], label %[[SCALAR_LOOP]]
+; CHECK-SVE2: [[LOOP_EXIT]]:
+; CHECK-SVE2-NEXT: [[SUM_0_LCSSA:%.*]] = phi <4 x i32> [ [[L7]], %[[SCALAR_LOOP]] ]
+; CHECK-SVE2-NEXT: ret <4 x i32> [[SUM_0_LCSSA]]
+;
+; CHECK-SVE2p1-LABEL: define <4 x i32> @reduction_smax_4xi32(
+; CHECK-SVE2p1-SAME: ptr noalias captures(none) [[A:%.*]]) #[[ATTR0]] {
+; CHECK-SVE2p1-NEXT: [[ENTRY:.*:]]
+; CHECK-SVE2p1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE2p1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 64, [[TMP0]]
+; CHECK-SVE2p1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-SVE2p1: [[VECTOR_PH]]:
+; CHECK-SVE2p1-NEXT: [[N_MOD_VF:%.*]] = urem i64 64, [[TMP0]]
+; CHECK-SVE2p1-NEXT: [[N_VEC:%.*]] = sub i64 64, [[N_MOD_VF]]
+; CHECK-SVE2p1-NEXT: [[TMP2:%.*]] = shl i64 [[N_VEC]], 2
+; CHECK-SVE2p1-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-SVE2p1: [[VECTOR_BODY]]:
+; CHECK-SVE2p1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-SVE2p1-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-SVE2p1-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 2
+; CHECK-SVE2p1-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[OFFSET_IDX]]
+; CHECK-SVE2p1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP3]], align 4
+; CHECK-SVE2p1-NEXT: [[TMP4]] = call <vscale x 4 x i32> @llvm.smax.nxv4i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[WIDE_LOAD]])
+; CHECK-SVE2p1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; CHECK-SVE2p1-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-SVE2p1-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK-SVE2p1: [[MIDDLE_BLOCK]]:
+; CHECK-SVE2p1-NEXT: [[TMP6:%.*]] = call <4 x i32> @llvm.vector.partial.reduce.smax.v4i32.nxv4i32(<4 x i32> splat (i32 -2147483648), <vscale x 4 x i32> [[TMP4]])
+; CHECK-SVE2p1-NEXT: [[CMP_N:%.*]] = icmp eq i64 64, [[N_VEC]]
+; CHECK-SVE2p1-NEXT: br i1 [[CMP_N]], [[LOOP_EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK-SVE2p1: [[SCALAR_PH]]:
+;
+; CHECK-IC2-LABEL: define <4 x i32> @reduction_smax_4xi32(
+; CHECK-IC2-SAME: ptr noalias captures(none) [[A:%.*]]) #[[ATTR0]] {
+; CHECK-IC2-NEXT: [[ENTRY:.*:]]
+; CHECK-IC2-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-IC2-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-IC2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 64, [[TMP1]]
+; CHECK-IC2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-IC2: [[VECTOR_PH]]:
+; CHECK-IC2-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-IC2-NEXT: [[N_MOD_VF:%.*]] = urem i64 64, [[TMP3]]
+; CHECK-IC2-NEXT: [[N_VEC:%.*]] = sub i64 64, [[N_MOD_VF]]
+; CHECK-IC2-NEXT: [[TMP4:%.*]] = shl i64 [[N_VEC]], 2
+; CHECK-IC2-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-IC2: [[VECTOR_BODY]]:
+; CHECK-IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-IC2-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-IC2-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-IC2-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 2
+; CHECK-IC2-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[OFFSET_IDX]]
+; CHECK-IC2-NEXT: [[TMP8:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP5]], i64 [[TMP0]]
+; CHECK-IC2-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP5]], align 4
+; CHECK-IC2-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 4 x i32>, ptr [[TMP8]], align 4
+; CHECK-IC2-NEXT: [[TMP9]] = call <vscale x 4 x i32> @llvm.smax.nxv4i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[WIDE_LOAD]])
+; CHECK-IC2-NEXT: [[TMP10]] = call <vscale x 4 x i32> @llvm.smax.nxv4i32(<vscale x 4 x i32> [[VEC_PHI1]], <vscale x 4 x i32> [[WIDE_LOAD2]])
+; CHECK-IC2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-IC2-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-IC2-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK-IC2: [[MIDDLE_BLOCK]]:
+; CHECK-IC2-NEXT: [[RDX_MINMAX:%.*]] = call <vscale x 4 x i32> @llvm.smax.nxv4i32(<vscale x 4 x i32> [[TMP9]], <vscale x 4 x i32> [[TMP10]])
+; CHECK-IC2-NEXT: [[TMP12:%.*]] = call <4 x i32> @llvm.vector.partial.reduce.smax.v4i32.nxv4i32(<4 x i32> splat (i32 -2147483648), <vscale x 4 x i32> [[RDX_MINMAX]])
+; CHECK-IC2-NEXT: [[CMP_N:%.*]] = icmp eq i64 64, [[N_VEC]]
+; CHECK-IC2-NEXT: br i1 [[CMP_N]], [[LOOP_EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK-IC2: [[SCALAR_PH]]:
+;
+; CHECK-VF1-IC2-LABEL: define <4 x i32> @reduction_smax_4xi32(
+; CHECK-VF1-IC2-SAME: ptr noalias captures(none) [[A:%.*]]) #[[ATTR0]] {
+; CHECK-VF1-IC2-NEXT: [[ENTRY:.*:]]
+; CHECK-VF1-IC2-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-VF1-IC2: [[VECTOR_PH]]:
+; CHECK-VF1-IC2-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-VF1-IC2: [[VECTOR_BODY]]:
+; CHECK-VF1-IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1-IC2-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1-IC2-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1-IC2-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 2
+; CHECK-VF1-IC2-NEXT: [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 4
+; CHECK-VF1-IC2-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[OFFSET_IDX]]
+; CHECK-VF1-IC2-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP0]]
+; CHECK-VF1-IC2-NEXT: [[TMP3:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-VF1-IC2-NEXT: [[TMP4:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-VF1-IC2-NEXT: [[TMP5]] = tail call <4 x i32> @llvm.smax.v4i32(<4 x i32> [[VEC_PHI]], <4 x i32> [[TMP3]])
+; CHECK-VF1-IC2-NEXT: [[TMP6]] = tail call <4 x i32> @llvm.smax.v4i32(<4 x i32> [[VEC_PHI1]], <4 x i32> [[TMP4]])
+; CHECK-VF1-IC2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-VF1-IC2-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64
+; CHECK-VF1-IC2-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK-VF1-IC2: [[MIDDLE_BLOCK]]:
+; CHECK-VF1-IC2-NEXT: [[RDX_MINMAX:%.*]] = call <4 x i32> @llvm.smax.v4i32(<4 x i32> [[TMP5]], <4 x i32> [[TMP6]])
+; CHECK-VF1-IC2-NEXT: br label %[[LOOP_EXIT:.*]]
+; CHECK-VF1-IC2: [[LOOP_EXIT]]:
+; CHECK-VF1-IC2-NEXT: ret <4 x i32> [[RDX_MINMAX]]
+;
+entry:
+ br label %scalar.loop
+
+scalar.loop: ; preds = %entry, %scalar.loop
+ %indvars.iv = phi i64 [ %indvars.iv.next, %scalar.loop ], [ 0, %entry ]
+ %acc.02 = phi <4 x i32> [ %l7, %scalar.loop ], [ zeroinitializer, %entry ]
+ %l2 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
+ %l3 = load <4 x i32>, ptr %l2, align 4
+ %l7 = tail call <4 x i32> @llvm.smax.v4i32(<4 x i32> %acc.02, <4 x i32> %l3)
+ %indvars.iv.next = add i64 %indvars.iv, 4
+ %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+ %exitcond = icmp eq i32 %lftr.wideiv, 256
+ br i1 %exitcond, label %loop.exit, label %scalar.loop
+
+loop.exit: ; preds = %scalar.loop
+ %sum.0.lcssa = phi <4 x i32> [ %l7, %scalar.loop ]
+ ret <4 x i32> %sum.0.lcssa
+}
+
+define <4 x i32> @reduction_umin_4xi32(ptr noalias nocapture %A) {
+; CHECK-SVE2-LABEL: define <4 x i32> @reduction_umin_4xi32(
+; CHECK-SVE2-SAME: ptr noalias captures(none) [[A:%.*]]) #[[ATTR0]] {
+; CHECK-SVE2-NEXT: [[ENTRY:.*]]:
+; CHECK-SVE2-NEXT: br label %[[SCALAR_LOOP:.*]]
+; CHECK-SVE2: [[SCALAR_LOOP]]:
+; CHECK-SVE2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[SCALAR_LOOP]] ], [ 0, %[[ENTRY]] ]
+; CHECK-SVE2-NEXT: [[ACC_02:%.*]] = phi <4 x i32> [ [[L7:%.*]], %[[SCALAR_LOOP]] ], [ zeroinitializer, %[[ENTRY]] ]
+; CHECK-SVE2-NEXT: [[L2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-SVE2-NEXT: [[L3:%.*]] = load <4 x i32>, ptr [[L2]], align 4
+; CHECK-SVE2-NEXT: [[L7]] = tail call <4 x i32> @llvm.umin.v4i32(<4 x i32> [[ACC_02]], <4 x i32> [[L3]])
+; CHECK-SVE2-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 4
+; CHECK-SVE2-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-SVE2-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], 256
+; CHECK-SVE2-NEXT: br i1 [[EXITCOND]], label %[[LOOP_EXIT:.*]], label %[[SCALAR_LOOP]]
+; CHECK-SVE2: [[LOOP_EXIT]]:
+; CHECK-SVE2-NEXT: [[SUM_0_LCSSA:%.*]] = phi <4 x i32> [ [[L7]], %[[SCALAR_LOOP]] ]
+; CHECK-SVE2-NEXT: ret <4 x i32> [[SUM_0_LCSSA]]
+;
+; CHECK-SVE2p1-LABEL: define <4 x i32> @reduction_umin_4xi32(
+; CHECK-SVE2p1-SAME: ptr noalias captures(none) [[A:%.*]]) #[[ATTR0]] {
+; CHECK-SVE2p1-NEXT: [[ENTRY:.*:]]
+; CHECK-SVE2p1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE2p1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 64, [[TMP0]]
+; CHECK-SVE2p1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-SVE2p1: [[VECTOR_PH]]:
+; CHECK-SVE2p1-NEXT: [[N_MOD_VF:%.*]] = urem i64 64, [[TMP0]]
+; CHECK-SVE2p1-NEXT: [[N_VEC:%.*]] = sub i64 64, [[N_MOD_VF]]
+; CHECK-SVE2p1-NEXT: [[TMP2:%.*]] = shl i64 [[N_VEC]], 2
+; CHECK-SVE2p1-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-SVE2p1: [[VECTOR_BODY]]:
+; CHECK-SVE2p1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-SVE2p1-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-SVE2p1-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 2
+; CHECK-SVE2p1-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[OFFSET_IDX]]
+; CHECK-SVE2p1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP3]], align 4
+; CHECK-SVE2p1-NEXT: [[TMP4]] = call <vscale x 4 x i32> @llvm.umin.nxv4i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[WIDE_LOAD]])
+; CHECK-SVE2p1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
+; CHECK-SVE2p1-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-SVE2p1-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK-SVE2p1: [[MIDDLE_BLOCK]]:
+; CHECK-SVE2p1-NEXT: [[TMP6:%.*]] = call <4 x i32> @llvm.vector.partial.reduce.umin.v4i32.nxv4i32(<4 x i32> splat (i32 -1), <vscale x 4 x i32> [[TMP4]])
+; CHECK-SVE2p1-NEXT: [[CMP_N:%.*]] = icmp eq i64 64, [[N_VEC]]
+; CHECK-SVE2p1-NEXT: br i1 [[CMP_N]], [[LOOP_EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK-SVE2p1: [[SCALAR_PH]]:
+;
+; CHECK-IC2-LABEL: define <4 x i32> @reduction_umin_4xi32(
+; CHECK-IC2-SAME: ptr noalias captures(none) [[A:%.*]]) #[[ATTR0]] {
+; CHECK-IC2-NEXT: [[ENTRY:.*:]]
+; CHECK-IC2-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-IC2-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-IC2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 64, [[TMP1]]
+; CHECK-IC2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-IC2: [[VECTOR_PH]]:
+; CHECK-IC2-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-IC2-NEXT: [[N_MOD_VF:%.*]] = urem i64 64, [[TMP3]]
+; CHECK-IC2-NEXT: [[N_VEC:%.*]] = sub i64 64, [[N_MOD_VF]]
+; CHECK-IC2-NEXT: [[TMP4:%.*]] = shl i64 [[N_VEC]], 2
+; CHECK-IC2-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-IC2: [[VECTOR_BODY]]:
+; CHECK-IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-IC2-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-IC2-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-IC2-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 2
+; CHECK-IC2-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[OFFSET_IDX]]
+; CHECK-IC2-NEXT: [[TMP8:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP5]], i64 [[TMP0]]
+; CHECK-IC2-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP5]], align 4
+; CHECK-IC2-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 4 x i32>, ptr [[TMP8]], align 4
+; CHECK-IC2-NEXT: [[TMP9]] = call <vscale x 4 x i32> @llvm.umin.nxv4i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[WIDE_LOAD]])
+; CHECK-IC2-NEXT: [[TMP10]] = call <vscale x 4 x i32> @llvm.umin.nxv4i32(<vscale x 4 x i32> [[VEC_PHI1]], <vscale x 4 x i32> [[WIDE_LOAD2]])
+; CHECK-IC2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-IC2-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-IC2-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK-IC2: [[MIDDLE_BLOCK]]:
+; CHECK-IC2-NEXT: [[RDX_MINMAX:%.*]] = call <vscale x 4 x i32> @llvm.umin.nxv4i32(<vscale x 4 x i32> [[TMP9]], <vscale x 4 x i32> [[TMP10]])
+; CHECK-IC2-NEXT: [[TMP12:%.*]] = call <4 x i32> @llvm.vector.partial.reduce.umin.v4i32.nxv4i32(<4 x i32> splat (i32 -1), <vscale x 4 x i32> [[RDX_MINMAX]])
+; CHECK-IC2-NEXT: [[CMP_N:%.*]] = icmp eq i64 64, [[N_VEC]]
+; CHECK-IC2-NEXT: br i1 [[CMP_N]], [[LOOP_EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK-IC2: [[SCALAR_PH]]:
+;
+; CHECK-VF1-IC2-LABEL: define <4 x i32> @reduction_umin_4xi32(
+; CHECK-VF1-IC2-SAME: ptr noalias captures(none) [[A:%.*]]) #[[ATTR0]] {
+; CHECK-VF1-IC2-NEXT: [[ENTRY:.*:]]
+; CHECK-VF1-IC2-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-VF1-IC2: [[VECTOR_PH]]:
+; CHECK-VF1-IC2-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-VF1-IC2: [[VECTOR_BODY]]:
+; CHECK-VF1-IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1-IC2-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1-IC2-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1-IC2-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 2
+; CHECK-VF1-IC2-NEXT: [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 4
+; CHECK-VF1-IC2-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[OFFSET_IDX]]
+; CHECK-VF1-IC2-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP0]]
+; CHECK-VF1-IC2-NEXT: [[TMP3:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-VF1-IC2-NEXT: [[TMP4:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-VF1-IC2-NEXT: [[TMP5]] = tail call <4 x i32> @llvm.umin.v4i32(<4 x i32> [[VEC_PHI]], <4 x i32> [[TMP3]])
+; CHECK-VF1-IC2-NEXT: [[TMP6]] = tail call <4 x i32> @llvm.umin.v4i32(<4 x i32> [[VEC_PHI1]], <4 x i32> [[TMP4]])
+; CHECK-VF1-IC2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-VF1-IC2-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64
+; CHECK-VF1-IC2-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK-VF1-IC2: [[MIDDLE_BLOCK]]:
+; CHECK-VF1-IC2-NEXT: [[RDX_MINMAX:%.*]] = call <4 x i32> @llvm.umin.v4i32(<4 x i32> [[TMP5]], <4 x i32> [[TMP6]])
+; CHECK-VF1-IC2-NEXT: br label %[[LOOP_EXIT:.*]]
+; CHECK-VF1-IC2: [[LOOP_EXIT]]:
+; CHECK-VF1-IC2-NEXT: ret <4 x i32> [[RDX_MINMAX]]
+;
+entry:
+ br label %scalar.loop
+
+scalar.loop: ; preds = %entry, %scalar.loop
+ %indvars.iv = phi i64 [ %indvars.iv.next, %scalar.loop ], [ 0, %entry ]
+ %acc.02 = phi <4 x i32> [ %l7, %scalar.loop ], [ zeroinitializer, %entry ]
+ %l2 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
+ %l3 = load <4 x i32>, ptr %l2, align 4
+ %l7 = tail call <4 x i32> @llvm.umin.v4i32(<4 x i32> %acc.02, <4 x i32> %l3)
+ %indvars.iv.next = add i64 %indvars.iv, 4
+ %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+ %exitcond = icmp eq i32 %lftr.wideiv, 256
+ br i1 %exitcond, label %loop.exit, label %scalar.loop
+
+loop.exit: ; preds = %scalar.loop
+ %sum.0.lcssa = phi <4 x i32> [ %l7, %scalar.loop ]
+ ret <4 x i32> %sum.0.lcssa
+}
+
+; Verify 64-bit vectors are also supported.
+define <4 x i16> @reduction_smin_4xi16(ptr noalias nocapture %A) {
+; CHECK-SVE2-LABEL: define <4 x i16> @reduction_smin_4xi16(
+; CHECK-SVE2-SAME: ptr noalias captures(none) [[A:%.*]]) #[[ATTR0]] {
+; CHECK-SVE2-NEXT: [[ENTRY:.*]]:
+; CHECK-SVE2-NEXT: br label %[[SCALAR_LOOP:.*]]
+; CHECK-SVE2: [[SCALAR_LOOP]]:
+; CHECK-SVE2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[SCALAR_LOOP]] ], [ 0, %[[ENTRY]] ]
+; CHECK-SVE2-NEXT: [[ACC_02:%.*]] = phi <4 x i16> [ [[L7:%.*]], %[[SCALAR_LOOP]] ], [ zeroinitializer, %[[ENTRY]] ]
+; CHECK-SVE2-NEXT: [[L2:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-SVE2-NEXT: [[L3:%.*]] = load <4 x i16>, ptr [[L2]], align 4
+; CHECK-SVE2-NEXT: [[L7]] = tail call <4 x i16> @llvm.smax.v4i16(<4 x i16> [[ACC_02]], <4 x i16> [[L3]])
+; CHECK-SVE2-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 4
+; CHECK-SVE2-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 256
+; CHECK-SVE2-NEXT: br i1 [[EXITCOND]], label %[[LOOP_EXIT:.*]], label %[[SCALAR_LOOP]]
+; CHECK-SVE2: [[LOOP_EXIT]]:
+; CHECK-SVE2-NEXT: [[SUM_0_LCSSA:%.*]] = phi <4 x i16> [ [[L7]], %[[SCALAR_LOOP]] ]
+; CHECK-SVE2-NEXT: ret <4 x i16> [[SUM_0_LCSSA]]
+;
+; CHECK-SVE2p1-LABEL: define <4 x i16> @reduction_smin_4xi16(
+; CHECK-SVE2p1-SAME: ptr noalias captures(none) [[A:%.*]]) #[[ATTR0]] {
+; CHECK-SVE2p1-NEXT: [[ENTRY:.*:]]
+; CHECK-SVE2p1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE2p1-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-SVE2p1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 64, [[TMP1]]
+; CHECK-SVE2p1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-SVE2p1: [[VECTOR_PH]]:
+; CHECK-SVE2p1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-SVE2p1-NEXT: [[N_MOD_VF:%.*]] = urem i64 64, [[TMP3]]
+; CHECK-SVE2p1-NEXT: [[N_VEC:%.*]] = sub i64 64, [[N_MOD_VF]]
+; CHECK-SVE2p1-NEXT: [[TMP4:%.*]] = shl i64 [[N_VEC]], 2
+; CHECK-SVE2p1-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-SVE2p1: [[VECTOR_BODY]]:
+; CHECK-SVE2p1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-SVE2p1-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 8 x i16> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-SVE2p1-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 2
+; CHECK-SVE2p1-NEXT: [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 [[OFFSET_IDX]]
+; CHECK-SVE2p1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 4
+; CHECK-SVE2p1-NEXT: [[TMP6]] = call <vscale x 8 x i16> @llvm.smax.nxv8i16(<vscale x 8 x i16> [[VEC_PHI]], <vscale x 8 x i16> [[WIDE_LOAD]])
+; CHECK-SVE2p1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-SVE2p1-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-SVE2p1-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK-SVE2p1: [[MIDDLE_BLOCK]]:
+; CHECK-SVE2p1-NEXT: [[TMP8:%.*]] = call <4 x i16> @llvm.vector.partial.reduce.smax.v4i16.nxv8i16(<4 x i16> splat (i16 -32768), <vscale x 8 x i16> [[TMP6]])
+; CHECK-SVE2p1-NEXT: [[CMP_N:%.*]] = icmp eq i64 64, [[N_VEC]]
+; CHECK-SVE2p1-NEXT: br i1 [[CMP_N]], [[LOOP_EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK-SVE2p1: [[SCALAR_PH]]:
+;
+; CHECK-IC2-LABEL: define <4 x i16> @reduction_smin_4xi16(
+; CHECK-IC2-SAME: ptr noalias captures(none) [[A:%.*]]) #[[ATTR0]] {
+; CHECK-IC2-NEXT: [[ENTRY:.*:]]
+; CHECK-IC2-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-IC2-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-IC2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 64, [[TMP1]]
+; CHECK-IC2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-IC2: [[VECTOR_PH]]:
+; CHECK-IC2-NEXT: [[TMP7:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-IC2-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP7]], 1
+; CHECK-IC2-NEXT: [[N_MOD_VF:%.*]] = urem i64 64, [[TMP3]]
+; CHECK-IC2-NEXT: [[N_VEC:%.*]] = sub i64 64, [[N_MOD_VF]]
+; CHECK-IC2-NEXT: [[TMP4:%.*]] = shl i64 [[N_VEC]], 2
+; CHECK-IC2-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-IC2: [[VECTOR_BODY]]:
+; CHECK-IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-IC2-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 8 x i16> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-IC2-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 8 x i16> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-IC2-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 2
+; CHECK-IC2-NEXT: [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 [[OFFSET_IDX]]
+; CHECK-IC2-NEXT: [[TMP8:%.*]] = getelementptr inbounds <4 x i16>, ptr [[TMP5]], i64 [[TMP7]]
+; CHECK-IC2-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 4
+; CHECK-IC2-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 8 x i16>, ptr [[TMP8]], align 4
+; CHECK-IC2-NEXT: [[TMP9]] = call <vscale x 8 x i16> @llvm.smax.nxv8i16(<vscale x 8 x i16> [[VEC_PHI]], <vscale x 8 x i16> [[WIDE_LOAD]])
+; CHECK-IC2-NEXT: [[TMP10]] = call <vscale x 8 x i16> @llvm.smax.nxv8i16(<vscale x 8 x i16> [[VEC_PHI1]], <vscale x 8 x i16> [[WIDE_LOAD2]])
+; CHECK-IC2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-IC2-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-IC2-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK-IC2: [[MIDDLE_BLOCK]]:
+; CHECK-IC2-NEXT: [[RDX_MINMAX:%.*]] = call <vscale x 8 x i16> @llvm.smax.nxv8i16(<vscale x 8 x i16> [[TMP9]], <vscale x 8 x i16> [[TMP10]])
+; CHECK-IC2-NEXT: [[TMP12:%.*]] = call <4 x i16> @llvm.vector.partial.reduce.smax.v4i16.nxv8i16(<4 x i16> splat (i16 -32768), <vscale x 8 x i16> [[RDX_MINMAX]])
+; CHECK-IC2-NEXT: [[CMP_N:%.*]] = icmp eq i64 64, [[N_VEC]]
+; CHECK-IC2-NEXT: br i1 [[CMP_N]], [[LOOP_EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK-IC2: [[SCALAR_PH]]:
+;
+; CHECK-VF1-IC2-LABEL: define <4 x i16> @reduction_smin_4xi16(
+; CHECK-VF1-IC2-SAME: ptr noalias captures(none) [[A:%.*]]) #[[ATTR0]] {
+; CHECK-VF1-IC2-NEXT: [[ENTRY:.*:]]
+; CHECK-VF1-IC2-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-VF1-IC2: [[VECTOR_PH]]:
+; CHECK-VF1-IC2-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-VF1-IC2: [[VECTOR_BODY]]:
+; CHECK-VF1-IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1-IC2-NEXT: [[VEC_PHI:%.*]] = phi <4 x i16> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1-IC2-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i16> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1-IC2-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 2
+; CHECK-VF1-IC2-NEXT: [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 4
+; CHECK-VF1-IC2-NEXT: [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 [[OFFSET_IDX]]
+; CHECK-VF1-IC2-NEXT: [[TMP2:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 [[TMP0]]
+; CHECK-VF1-IC2-NEXT: [[TMP3:%.*]] = load <4 x i16>, ptr [[TMP1]], align 4
+; CHECK-VF1-IC2-NEXT: [[TMP4:%.*]] = load <4 x i16>, ptr [[TMP2]], align 4
+; CHECK-VF1-IC2-NEXT: [[TMP5]] = tail call <4 x i16> @llvm.smax.v4i16(<4 x i16> [[VEC_PHI]], <4 x i16> [[TMP3]])
+; CHECK-VF1-IC2-NEXT: [[TMP6]] = tail call <4 x i16> @llvm.smax.v4i16(<4 x i16> [[VEC_PHI1]], <4 x i16> [[TMP4]])
+; CHECK-VF1-IC2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-VF1-IC2-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64
+; CHECK-VF1-IC2-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK-VF1-IC2: [[MIDDLE_BLOCK]]:
+; CHECK-VF1-IC2-NEXT: [[RDX_MINMAX:%.*]] = call <4 x i16> @llvm.smax.v4i16(<4 x i16> [[TMP5]], <4 x i16> [[TMP6]])
+; CHECK-VF1-IC2-NEXT: br label %[[LOOP_EXIT:.*]]
+; CHECK-VF1-IC2: [[LOOP_EXIT]]:
+; CHECK-VF1-IC2-NEXT: ret <4 x i16> [[RDX_MINMAX]]
+;
+entry:
+ br label %scalar.loop
+
+scalar.loop: ; preds = %entry, %scalar.loop
+ %indvars.iv = phi i64 [ %indvars.iv.next, %scalar.loop ], [ 0, %entry ]
+ %acc.02 = phi <4 x i16> [ %l7, %scalar.loop ], [ zeroinitializer, %entry ]
+ %l2 = getelementptr inbounds i16, ptr %A, i64 %indvars.iv
+ %l3 = load <4 x i16>, ptr %l2, align 4
+ %l7 = tail call <4 x i16> @llvm.smax.v4i16(<4 x i16> %acc.02, <4 x i16> %l3)
+ %indvars.iv.next = add i64 %indvars.iv, 4
+ %exitcond = icmp eq i64 %indvars.iv.next, 256
+ br i1 %exitcond, label %loop.exit, label %scalar.loop
+
+loop.exit: ; preds = %scalar.loop
+ %sum.0.lcssa = phi <4 x i16> [ %l7, %scalar.loop ]
+ ret <4 x i16> %sum.0.lcssa
+}
diff --git a/llvm/test/Transforms/LoopVectorize/revec-scalable-first-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/revec-scalable-first-order-recurrence.ll
new file mode 100644
index 0000000000000..306ef996bc7c8
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/revec-scalable-first-order-recurrence.ll
@@ -0,0 +1,259 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "scalar.body:" --version 5
+; RUN: opt -passes=loop-vectorize -scalable-vectorization=on -force-vector-width=1 -force-vector-interleave=1 -force-target-supports-scalable-vectors=true -vectorize-vector-loops -S < %s | FileCheck %s --check-prefix=CHECK-UF1
+; RUN: opt -passes=loop-vectorize -scalable-vectorization=on -force-vector-width=1 -force-vector-interleave=2 -force-target-supports-scalable-vectors=true -vectorize-vector-loops -S < %s | FileCheck %s --check-prefix=CHECK-UF2
+
+; void recurrence_1(int *a, int *b, int n) {
+; for(int i = 0; i < n; i++)
+; b[i] = a[i] + a[i - 1]
+; }
+;
+define <4 x i32> @recurrence_1(ptr nocapture readonly %a, ptr nocapture noalias %b, i32 %n) {
+; CHECK-UF1-LABEL: define <4 x i32> @recurrence_1(
+; CHECK-UF1-SAME: ptr readonly captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i32 [[N:%.*]]) {
+; CHECK-UF1-NEXT: [[ENTRY:.*:]]
+; CHECK-UF1-NEXT: br label %[[FOR_PREHEADER:.*]]
+; CHECK-UF1: [[FOR_PREHEADER]]:
+; CHECK-UF1-NEXT: [[PRE_LOAD:%.*]] = load <4 x i32>, ptr [[A]], align 16
+; CHECK-UF1-NEXT: [[BROADCAST:%.*]] = call <vscale x 4 x i32> @llvm.vector.broadcast.nxv4i32.v4i32(<4 x i32> [[PRE_LOAD]])
+; CHECK-UF1-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-UF1-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-UF1-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
+; CHECK-UF1-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UF1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP4]]
+; CHECK-UF1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-UF1: [[VECTOR_PH]]:
+; CHECK-UF1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP4]]
+; CHECK-UF1-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
+; CHECK-UF1-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-UF1: [[VECTOR_BODY]]:
+; CHECK-UF1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-UF1-NEXT: [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i32> [ [[BROADCAST]], %[[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-UF1-NEXT: [[TMP9:%.*]] = add nuw nsw i64 [[INDEX]], 1
+; CHECK-UF1-NEXT: [[TMP10:%.*]] = getelementptr inbounds <4 x i32>, ptr [[A]], i64 [[TMP9]]
+; CHECK-UF1-NEXT: [[WIDE_LOAD]] = load <vscale x 4 x i32>, ptr [[TMP10]], align 16
+; CHECK-UF1-NEXT: [[TMP11:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR]], <vscale x 4 x i32> [[WIDE_LOAD]], i32 4)
+; CHECK-UF1-NEXT: [[TMP12:%.*]] = getelementptr inbounds <4 x i32>, ptr [[B]], i64 [[INDEX]]
+; CHECK-UF1-NEXT: [[TMP13:%.*]] = add <vscale x 4 x i32> [[WIDE_LOAD]], [[TMP11]]
+; CHECK-UF1-NEXT: store <vscale x 4 x i32> [[TMP13]], ptr [[TMP12]], align 16
+; CHECK-UF1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
+; CHECK-UF1-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-UF1-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-UF1: [[MIDDLE_BLOCK]]:
+; CHECK-UF1-NEXT: [[TMP15:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> poison, i32 4)
+; CHECK-UF1-NEXT: [[TMP16:%.*]] = call <4 x i32> @llvm.vector.extract.v4i32.nxv4i32(<vscale x 4 x i32> [[TMP15]], i64 0)
+; CHECK-UF1-NEXT: [[TMP17:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[TMP11]], <vscale x 4 x i32> poison, i32 4)
+; CHECK-UF1-NEXT: [[TMP18:%.*]] = call <4 x i32> @llvm.vector.extract.v4i32.nxv4i32(<vscale x 4 x i32> [[TMP17]], i64 0)
+; CHECK-UF1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
+; CHECK-UF1-NEXT: br i1 [[CMP_N]], [[FOR_EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK-UF1: [[SCALAR_PH]]:
+; CHECK-UF1-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi <4 x i32> [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ [[PRE_LOAD]], %[[FOR_PREHEADER]] ]
+; CHECK-UF1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_PREHEADER]] ]
+; CHECK-UF1-NEXT: br label %[[SCALAR_BODY:.*]]
+; CHECK-UF1: [[SCALAR_BODY]]:
+;
+; CHECK-UF2-LABEL: define <4 x i32> @recurrence_1(
+; CHECK-UF2-SAME: ptr readonly captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i32 [[N:%.*]]) {
+; CHECK-UF2-NEXT: [[ENTRY:.*:]]
+; CHECK-UF2-NEXT: br label %[[FOR_PREHEADER:.*]]
+; CHECK-UF2: [[FOR_PREHEADER]]:
+; CHECK-UF2-NEXT: [[PRE_LOAD:%.*]] = load <4 x i32>, ptr [[A]], align 16
+; CHECK-UF2-NEXT: [[BROADCAST:%.*]] = call <vscale x 4 x i32> @llvm.vector.broadcast.nxv4i32.v4i32(<4 x i32> [[PRE_LOAD]])
+; CHECK-UF2-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-UF2-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-UF2-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
+; CHECK-UF2-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UF2-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 1
+; CHECK-UF2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP4]]
+; CHECK-UF2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-UF2: [[VECTOR_PH]]:
+; CHECK-UF2-NEXT: [[TMP10:%.*]] = shl nuw i64 [[TMP3]], 1
+; CHECK-UF2-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP10]]
+; CHECK-UF2-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
+; CHECK-UF2-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-UF2: [[VECTOR_BODY]]:
+; CHECK-UF2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-UF2-NEXT: [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i32> [ [[BROADCAST]], %[[VECTOR_PH]] ], [ [[WIDE_LOAD3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-UF2-NEXT: [[TMP11:%.*]] = add nuw nsw i64 [[INDEX]], 1
+; CHECK-UF2-NEXT: [[TMP12:%.*]] = getelementptr inbounds <4 x i32>, ptr [[A]], i64 [[TMP11]]
+; CHECK-UF2-NEXT: [[TMP15:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP12]], i64 [[TMP3]]
+; CHECK-UF2-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP12]], align 16
+; CHECK-UF2-NEXT: [[WIDE_LOAD3]] = load <vscale x 4 x i32>, ptr [[TMP15]], align 16
+; CHECK-UF2-NEXT: [[TMP16:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR]], <vscale x 4 x i32> [[WIDE_LOAD]], i32 4)
+; CHECK-UF2-NEXT: [[TMP17:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[WIDE_LOAD3]], i32 4)
+; CHECK-UF2-NEXT: [[TMP18:%.*]] = getelementptr inbounds <4 x i32>, ptr [[B]], i64 [[INDEX]]
+; CHECK-UF2-NEXT: [[TMP19:%.*]] = add <vscale x 4 x i32> [[WIDE_LOAD]], [[TMP16]]
+; CHECK-UF2-NEXT: [[TMP20:%.*]] = add <vscale x 4 x i32> [[WIDE_LOAD3]], [[TMP17]]
+; CHECK-UF2-NEXT: [[TMP23:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP18]], i64 [[TMP3]]
+; CHECK-UF2-NEXT: store <vscale x 4 x i32> [[TMP19]], ptr [[TMP18]], align 16
+; CHECK-UF2-NEXT: store <vscale x 4 x i32> [[TMP20]], ptr [[TMP23]], align 16
+; CHECK-UF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
+; CHECK-UF2-NEXT: [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-UF2-NEXT: br i1 [[TMP24]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-UF2: [[MIDDLE_BLOCK]]:
+; CHECK-UF2-NEXT: [[TMP25:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD3]], <vscale x 4 x i32> poison, i32 4)
+; CHECK-UF2-NEXT: [[TMP26:%.*]] = call <4 x i32> @llvm.vector.extract.v4i32.nxv4i32(<vscale x 4 x i32> [[TMP25]], i64 0)
+; CHECK-UF2-NEXT: [[TMP27:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[TMP17]], <vscale x 4 x i32> poison, i32 4)
+; CHECK-UF2-NEXT: [[TMP28:%.*]] = call <4 x i32> @llvm.vector.extract.v4i32.nxv4i32(<vscale x 4 x i32> [[TMP27]], i64 0)
+; CHECK-UF2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
+; CHECK-UF2-NEXT: br i1 [[CMP_N]], [[FOR_EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK-UF2: [[SCALAR_PH]]:
+; CHECK-UF2-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi <4 x i32> [ [[TMP26]], %[[MIDDLE_BLOCK]] ], [ [[PRE_LOAD]], %[[FOR_PREHEADER]] ]
+; CHECK-UF2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_PREHEADER]] ]
+; CHECK-UF2-NEXT: br label %[[SCALAR_BODY:.*]]
+; CHECK-UF2: [[SCALAR_BODY]]:
+;
+entry:
+ br label %for.preheader
+
+for.preheader:
+ %pre_load = load <4 x i32>, ptr %a
+ br label %scalar.body
+
+scalar.body:
+ %0 = phi <4 x i32> [ %pre_load, %for.preheader ], [ %1, %scalar.body ]
+ %indvars.iv = phi i64 [ 0, %for.preheader ], [ %indvars.iv.next, %scalar.body ]
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %arrayidx32 = getelementptr inbounds <4 x i32>, ptr %a, i64 %indvars.iv.next
+ %1 = load <4 x i32>, ptr %arrayidx32
+ %arrayidx34 = getelementptr inbounds <4 x i32>, ptr %b, i64 %indvars.iv
+ %add35 = add <4 x i32> %1, %0
+ store <4 x i32> %add35, ptr %arrayidx34
+ %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+ %exitcond = icmp eq i32 %lftr.wideiv, %n
+ br i1 %exitcond, label %for.exit, label %scalar.body
+
+for.exit:
+ ret <4 x i32> %0
+}
+
+; A second order recurrence is represented as a chain or first order ones.
+; void recurrence_2(int *a, int *b, int n) {
+; for(int i = 0; i < n; i++)
+; b[i] = a[i] + a[i - 2]
+; }
+define <4 x i32> @recurrence_2(ptr nocapture readonly %a, ptr nocapture noalias %b, i32 %n) {
+; CHECK-UF1-LABEL: define <4 x i32> @recurrence_2(
+; CHECK-UF1-SAME: ptr readonly captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i32 [[N:%.*]]) {
+; CHECK-UF1-NEXT: [[ENTRY:.*:]]
+; CHECK-UF1-NEXT: br label %[[FOR_PREHEADER:.*]]
+; CHECK-UF1: [[FOR_PREHEADER]]:
+; CHECK-UF1-NEXT: [[PRE_LOAD:%.*]] = load <4 x i32>, ptr [[A]], align 16
+; CHECK-UF1-NEXT: [[BROADCAST:%.*]] = call <vscale x 4 x i32> @llvm.vector.broadcast.nxv4i32.v4i32(<4 x i32> [[PRE_LOAD]])
+; CHECK-UF1-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-UF1-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-UF1-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
+; CHECK-UF1-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UF1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP3]]
+; CHECK-UF1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-UF1: [[VECTOR_PH]]:
+; CHECK-UF1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP3]]
+; CHECK-UF1-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
+; CHECK-UF1-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-UF1: [[VECTOR_BODY]]:
+; CHECK-UF1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-UF1-NEXT: [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-UF1-NEXT: [[VECTOR_RECUR1:%.*]] = phi <vscale x 4 x i32> [ [[BROADCAST]], %[[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-UF1-NEXT: [[TMP5:%.*]] = add nuw nsw i64 [[INDEX]], 1
+; CHECK-UF1-NEXT: [[TMP6:%.*]] = getelementptr inbounds <4 x i32>, ptr [[A]], i64 [[TMP5]]
+; CHECK-UF1-NEXT: [[WIDE_LOAD]] = load <vscale x 4 x i32>, ptr [[TMP6]], align 16
+; CHECK-UF1-NEXT: [[TMP7]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR1]], <vscale x 4 x i32> [[WIDE_LOAD]], i32 4)
+; CHECK-UF1-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR]], <vscale x 4 x i32> [[TMP7]], i32 4)
+; CHECK-UF1-NEXT: [[TMP9:%.*]] = getelementptr inbounds <4 x i32>, ptr [[B]], i64 [[INDEX]]
+; CHECK-UF1-NEXT: [[TMP10:%.*]] = add <vscale x 4 x i32> [[WIDE_LOAD]], [[TMP8]]
+; CHECK-UF1-NEXT: store <vscale x 4 x i32> [[TMP10]], ptr [[TMP9]], align 16
+; CHECK-UF1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-UF1-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-UF1-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-UF1: [[MIDDLE_BLOCK]]:
+; CHECK-UF1-NEXT: [[TMP12:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[TMP7]], <vscale x 4 x i32> poison, i32 4)
+; CHECK-UF1-NEXT: [[TMP13:%.*]] = call <4 x i32> @llvm.vector.extract.v4i32.nxv4i32(<vscale x 4 x i32> [[TMP12]], i64 0)
+; CHECK-UF1-NEXT: [[TMP14:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> poison, i32 4)
+; CHECK-UF1-NEXT: [[TMP15:%.*]] = call <4 x i32> @llvm.vector.extract.v4i32.nxv4i32(<vscale x 4 x i32> [[TMP14]], i64 0)
+; CHECK-UF1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
+; CHECK-UF1-NEXT: br i1 [[CMP_N]], [[FOR_EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK-UF1: [[SCALAR_PH]]:
+; CHECK-UF1-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi <4 x i32> [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ zeroinitializer, %[[FOR_PREHEADER]] ]
+; CHECK-UF1-NEXT: [[SCALAR_RECUR_INIT2:%.*]] = phi <4 x i32> [ [[TMP15]], %[[MIDDLE_BLOCK]] ], [ [[PRE_LOAD]], %[[FOR_PREHEADER]] ]
+; CHECK-UF1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_PREHEADER]] ]
+; CHECK-UF1-NEXT: br label %[[SCALAR_BODY:.*]]
+; CHECK-UF1: [[SCALAR_BODY]]:
+;
+; CHECK-UF2-LABEL: define <4 x i32> @recurrence_2(
+; CHECK-UF2-SAME: ptr readonly captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i32 [[N:%.*]]) {
+; CHECK-UF2-NEXT: [[ENTRY:.*:]]
+; CHECK-UF2-NEXT: br label %[[FOR_PREHEADER:.*]]
+; CHECK-UF2: [[FOR_PREHEADER]]:
+; CHECK-UF2-NEXT: [[PRE_LOAD:%.*]] = load <4 x i32>, ptr [[A]], align 16
+; CHECK-UF2-NEXT: [[BROADCAST:%.*]] = call <vscale x 4 x i32> @llvm.vector.broadcast.nxv4i32.v4i32(<4 x i32> [[PRE_LOAD]])
+; CHECK-UF2-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-UF2-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-UF2-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
+; CHECK-UF2-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UF2-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 1
+; CHECK-UF2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP4]]
+; CHECK-UF2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-UF2: [[VECTOR_PH]]:
+; CHECK-UF2-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP3]], 1
+; CHECK-UF2-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP6]]
+; CHECK-UF2-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
+; CHECK-UF2-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-UF2: [[VECTOR_BODY]]:
+; CHECK-UF2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-UF2-NEXT: [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-UF2-NEXT: [[VECTOR_RECUR1:%.*]] = phi <vscale x 4 x i32> [ [[BROADCAST]], %[[VECTOR_PH]] ], [ [[WIDE_LOAD2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-UF2-NEXT: [[TMP7:%.*]] = add nuw nsw i64 [[INDEX]], 1
+; CHECK-UF2-NEXT: [[TMP8:%.*]] = getelementptr inbounds <4 x i32>, ptr [[A]], i64 [[TMP7]]
+; CHECK-UF2-NEXT: [[TMP11:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP8]], i64 [[TMP3]]
+; CHECK-UF2-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP8]], align 16
+; CHECK-UF2-NEXT: [[WIDE_LOAD2]] = load <vscale x 4 x i32>, ptr [[TMP11]], align 16
+; CHECK-UF2-NEXT: [[TMP9:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR1]], <vscale x 4 x i32> [[WIDE_LOAD]], i32 4)
+; CHECK-UF2-NEXT: [[TMP10]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[WIDE_LOAD2]], i32 4)
+; CHECK-UF2-NEXT: [[TMP14:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR]], <vscale x 4 x i32> [[TMP9]], i32 4)
+; CHECK-UF2-NEXT: [[TMP15:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[TMP9]], <vscale x 4 x i32> [[TMP10]], i32 4)
+; CHECK-UF2-NEXT: [[TMP16:%.*]] = getelementptr inbounds <4 x i32>, ptr [[B]], i64 [[INDEX]]
+; CHECK-UF2-NEXT: [[TMP17:%.*]] = add <vscale x 4 x i32> [[WIDE_LOAD]], [[TMP14]]
+; CHECK-UF2-NEXT: [[TMP18:%.*]] = add <vscale x 4 x i32> [[WIDE_LOAD2]], [[TMP15]]
+; CHECK-UF2-NEXT: [[TMP21:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP16]], i64 [[TMP3]]
+; CHECK-UF2-NEXT: store <vscale x 4 x i32> [[TMP17]], ptr [[TMP16]], align 16
+; CHECK-UF2-NEXT: store <vscale x 4 x i32> [[TMP18]], ptr [[TMP21]], align 16
+; CHECK-UF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
+; CHECK-UF2-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-UF2-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-UF2: [[MIDDLE_BLOCK]]:
+; CHECK-UF2-NEXT: [[TMP23:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[TMP10]], <vscale x 4 x i32> poison, i32 4)
+; CHECK-UF2-NEXT: [[TMP24:%.*]] = call <4 x i32> @llvm.vector.extract.v4i32.nxv4i32(<vscale x 4 x i32> [[TMP23]], i64 0)
+; CHECK-UF2-NEXT: [[TMP25:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD2]], <vscale x 4 x i32> poison, i32 4)
+; CHECK-UF2-NEXT: [[TMP26:%.*]] = call <4 x i32> @llvm.vector.extract.v4i32.nxv4i32(<vscale x 4 x i32> [[TMP25]], i64 0)
+; CHECK-UF2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
+; CHECK-UF2-NEXT: br i1 [[CMP_N]], [[FOR_EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK-UF2: [[SCALAR_PH]]:
+; CHECK-UF2-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi <4 x i32> [ [[TMP24]], %[[MIDDLE_BLOCK]] ], [ zeroinitializer, %[[FOR_PREHEADER]] ]
+; CHECK-UF2-NEXT: [[SCALAR_RECUR_INIT3:%.*]] = phi <4 x i32> [ [[TMP26]], %[[MIDDLE_BLOCK]] ], [ [[PRE_LOAD]], %[[FOR_PREHEADER]] ]
+; CHECK-UF2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_PREHEADER]] ]
+; CHECK-UF2-NEXT: br label %[[SCALAR_BODY:.*]]
+; CHECK-UF2: [[SCALAR_BODY]]:
+;
+entry:
+ br label %for.preheader
+
+for.preheader:
+ %pre_load = load <4 x i32>, ptr %a
+ br label %scalar.body
+
+scalar.body:
+ %prev.2 = phi <4 x i32> [ zeroinitializer, %for.preheader ], [ %prev.1, %scalar.body ]
+ %prev.1 = phi <4 x i32> [ %pre_load, %for.preheader ], [ %1, %scalar.body ]
+ %indvars.iv = phi i64 [ 0, %for.preheader ], [ %indvars.iv.next, %scalar.body ]
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %arrayidx32 = getelementptr inbounds <4 x i32>, ptr %a, i64 %indvars.iv.next
+ %1 = load <4 x i32>, ptr %arrayidx32
+ %arrayidx34 = getelementptr inbounds <4 x i32>, ptr %b, i64 %indvars.iv
+ %add36 = add <4 x i32> %1, %prev.2
+ store <4 x i32> %add36, ptr %arrayidx34
+ %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+ %exitcond = icmp eq i32 %lftr.wideiv, %n
+ br i1 %exitcond, label %for.exit, label %scalar.body
+
+for.exit:
+ ret <4 x i32> %prev.1
+}
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/vector-pixel-neon-innermost-no-epilogue.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/vector-pixel-neon-innermost-no-epilogue.ll
new file mode 100644
index 0000000000000..84711c8275cb4
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/vector-pixel-neon-innermost-no-epilogue.ll
@@ -0,0 +1,107 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -O3 -force-vector-interleave=1 -mtriple=aarch64 -mattr=+sve2p1 -vectorize-vector-loops -S < %s | FileCheck %s
+
+; This is a phase ordering test verifying the output of the -O3 opt pipeline.
+; We are expecting the loop to use SVE udot and uabd intrinsics instead of NEON
+
+; TODO-REVEC: Get rid of -force-vector-interleave=1
+
+define dso_local void @pixel_ssd.do.body2(<4 x i32> %vssd_v.0, <4 x i32> %vssd_u.0, ptr %pixuv1.addr.0, ptr %pixuv2.addr.0, i64 %0, ptr %ssd_u, ptr %ssd_v) #1 {
+; CHECK-LABEL: define dso_local void @pixel_ssd.do.body2(
+; CHECK-SAME: <4 x i32> [[VSSD_V_0:%.*]], <4 x i32> [[VSSD_U_0:%.*]], ptr readonly captures(none) [[PIXUV1_ADDR_0:%.*]], ptr readonly captures(none) [[PIXUV2_ADDR_0:%.*]], i64 [[TMP0:%.*]], ptr writeonly captures(none) [[SSD_U:%.*]], ptr writeonly captures(none) [[SSD_V:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[NEWFUNCROOT:.*]]:
+; CHECK-NEXT: [[SMAX:%.*]] = shl i64 [[TMP0]], 4
+; CHECK-NEXT: [[ASSUMP:%.*]] = icmp sgt i64 [[SMAX]], 31
+; CHECK-NEXT: tail call void @llvm.assume(i1 [[ASSUMP]])
+; CHECK-NEXT: [[TMP1:%.*]] = add nsw i64 [[SMAX]], -8
+; CHECK-NEXT: [[TMP2:%.*]] = lshr exact i64 [[TMP1]], 3
+; CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v4i32(<vscale x 4 x i32> zeroinitializer, <4 x i32> [[VSSD_V_0]], i64 0)
+; CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v4i32(<vscale x 4 x i32> zeroinitializer, <4 x i32> [[VSSD_U_0]], i64 0)
+; CHECK-NEXT: [[INVARIANT_OP:%.*]] = sub nuw i64 [[TMP2]], 1
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[NEWFUNCROOT]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ [[TMP3]], %[[NEWFUNCROOT]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ [[TMP4]], %[[NEWFUNCROOT]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP9:%.*]] = shl i64 [[INDEX]], 4
+; CHECK-NEXT: [[ADD_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[PIXUV1_ADDR_0]], i64 [[TMP9]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[ADD_PTR]], align 1
+; CHECK-NEXT: [[ADD_PTR7:%.*]] = getelementptr inbounds nuw i8, ptr [[PIXUV2_ADDR_0]], i64 [[TMP9]]
+; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[ADD_PTR7]], align 1
+; CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.uabd.u.nxv16i8(<vscale x 16 x i1> splat (i1 true), <vscale x 16 x i8> [[WIDE_LOAD]], <vscale x 16 x i8> [[WIDE_LOAD2]])
+; CHECK-NEXT: [[TMP8:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(<vscale x 16 x i8> [[TMP7]], <vscale x 16 x i8> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>)
+; CHECK-NEXT: [[TMP14:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.segmented.shuffle.nxv16i8.v16i32(<vscale x 16 x i8> [[TMP7]], <vscale x 16 x i8> poison, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>)
+; CHECK-NEXT: [[TMP10]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32> [[VEC_PHI1]], <vscale x 16 x i8> [[TMP8]], <vscale x 16 x i8> [[TMP8]])
+; CHECK-NEXT: [[TMP11]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i8> [[TMP14]], <vscale x 16 x i8> [[TMP14]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX]], [[INVARIANT_OP]]
+; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[VDOT1_I_1:%.*]] = tail call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.nxv4i32(<4 x i32> zeroinitializer, <vscale x 4 x i32> [[TMP11]])
+; CHECK-NEXT: [[VDOT1_I44_1:%.*]] = tail call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.nxv4i32(<4 x i32> zeroinitializer, <vscale x 4 x i32> [[TMP10]])
+; CHECK-NEXT: [[VADDVQ_U32_I45:%.*]] = tail call noundef i32 @llvm.aarch64.neon.uaddv.i32.v4i32(<4 x i32> [[VDOT1_I44_1]])
+; CHECK-NEXT: [[DIV41:%.*]] = lshr i32 [[VADDVQ_U32_I45]], 1
+; CHECK-NEXT: [[CONV:%.*]] = zext nneg i32 [[DIV41]] to i64
+; CHECK-NEXT: store i64 [[CONV]], ptr [[SSD_U]], align 8
+; CHECK-NEXT: [[VADDVQ_U32_I:%.*]] = tail call noundef i32 @llvm.aarch64.neon.uaddv.i32.v4i32(<4 x i32> [[VDOT1_I_1]])
+; CHECK-NEXT: [[DIV2442:%.*]] = lshr i32 [[VADDVQ_U32_I]], 1
+; CHECK-NEXT: [[CONV25:%.*]] = zext nneg i32 [[DIV2442]] to i64
+; CHECK-NEXT: store i64 [[CONV25]], ptr [[SSD_V]], align 8
+; CHECK-NEXT: ret void
+;
+newFuncRoot:
+ %tc = mul i64 %0, 16
+ %assump = icmp sge i64 %tc, 32
+ call void @llvm.assume(i1 %assump)
+ br label %do.body2
+
+do.body2: ; preds = %newFuncRoot, %do.body2
+ %indvars.iv = phi i64 [ %indvars.iv.next, %do.body2 ], [ 0, %newFuncRoot ]
+ %vssd_v.1 = phi <4 x i32> [ %vdot1.i, %do.body2 ], [ %vssd_v.0, %newFuncRoot ]
+ %vssd_u.1 = phi <4 x i32> [ %vdot1.i44, %do.body2 ], [ %vssd_u.0, %newFuncRoot ]
+ %1 = shl nuw nsw i64 %indvars.iv, 1
+ %add.ptr = getelementptr inbounds nuw i8, ptr %pixuv1.addr.0, i64 %1
+ %2 = load <16 x i8>, ptr %add.ptr, align 1
+ %add.ptr7 = getelementptr inbounds nuw i8, ptr %pixuv2.addr.0, i64 %1
+ %3 = load <16 x i8>, ptr %add.ptr7, align 1
+ %vabd.i = tail call noundef <16 x i8> @llvm.aarch64.neon.uabd.v16i8(<16 x i8> %2, <16 x i8> %3)
+ %shuffle.i = shufflevector <16 x i8> %vabd.i, <16 x i8> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+ %shuffle.i43 = shufflevector <16 x i8> %vabd.i, <16 x i8> poison, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+ %vdot1.i44 = tail call noundef <4 x i32> @llvm.aarch64.neon.udot.v4i32.v16i8(<4 x i32> %vssd_u.1, <16 x i8> %shuffle.i, <16 x i8> %shuffle.i)
+ %vdot1.i = tail call noundef <4 x i32> @llvm.aarch64.neon.udot.v4i32.v16i8(<4 x i32> %vssd_v.1, <16 x i8> %shuffle.i43, <16 x i8> %shuffle.i43)
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 8
+ %cmp = icmp slt i64 %indvars.iv.next, %tc
+ br i1 %cmp, label %do.body2, label %do.end.exitStub, !llvm.loop !5
+
+do.end.exitStub: ; preds = %do.body2
+ %vaddvq_u32.i45 = tail call noundef i32 @llvm.aarch64.neon.uaddv.i32.v4i32(<4 x i32> %vdot1.i44)
+ %div41 = lshr i32 %vaddvq_u32.i45, 1
+ %conv = zext nneg i32 %div41 to i64
+ store i64 %conv, ptr %ssd_u, align 8
+ %vaddvq_u32.i = tail call noundef i32 @llvm.aarch64.neon.uaddv.i32.v4i32(<4 x i32> %vdot1.i)
+ %div2442 = lshr i32 %vaddvq_u32.i, 1
+ %conv25 = zext nneg i32 %div2442 to i64
+ store i64 %conv25, ptr %ssd_v, align 8
+ ret void
+}
+
+declare void @llvm.assume(i1 noundef)
+
+attributes #0 = { nocallback nofree nosync nounwind willreturn memory(none) }
+attributes #1 = { vscale_range(2,2) }
+
+!llvm.module.flags = !{!0, !1, !2, !3, !4}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 8, !"PIC Level", i32 2}
+!2 = !{i32 7, !"PIE Level", i32 2}
+!3 = !{i32 7, !"uwtable", i32 2}
+!4 = !{i32 7, !"frame-pointer", i32 1}
+!5 = distinct !{!5, !6}
+!6 = !{!"llvm.loop.mustprogress"}
+;.
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META6:![0-9]+]], [[META7:![0-9]+]], [[META8:![0-9]+]]}
+; CHECK: [[META6]] = !{!"llvm.loop.mustprogress"}
+; CHECK: [[META7]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META8]] = !{!"llvm.loop.unroll.runtime.disable"}
+;.
diff --git a/llvm/test/Verifier/masked-segment-gather-scatter-invalid.ll b/llvm/test/Verifier/masked-segment-gather-scatter-invalid.ll
new file mode 100644
index 0000000000000..761682f1db2d7
--- /dev/null
+++ b/llvm/test/Verifier/masked-segment-gather-scatter-invalid.ll
@@ -0,0 +1,37 @@
+; RUN: not opt -passes=verify < %s 2>&1 | FileCheck %s
+
+define <4 x i32> @gather_fixed_data(<vscale x 1 x ptr> %ptrs, <vscale x 1 x i1> %mask, <4 x i32> %passthru) {
+; CHECK: masked_segment_gather: data type must be a scalable vector
+ %res = call <4 x i32> @llvm.masked.segment.gather.v4i32.nxv1p0(<vscale x 1 x ptr> align 4 %ptrs, <vscale x 1 x i1> %mask, <4 x i32> %passthru)
+ ret <4 x i32> %res
+}
+
+define <vscale x 4 x i32> @gather_wide_ptrs(<vscale x 4 x ptr> %ptrs, <vscale x 1 x i1> %mask, <vscale x 4 x i32> %passthru) {
+; CHECK: masked_segment_gather: pointer type must be <vscale x 1 x ptr>
+ %res = call <vscale x 4 x i32> @llvm.masked.segment.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 %ptrs, <vscale x 1 x i1> %mask, <vscale x 4 x i32> %passthru)
+ ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @gather_non_ptrs(<vscale x 1 x i32> %ptrs, <vscale x 1 x i1> %mask, <vscale x 4 x i32> %passthru) {
+; CHECK: masked_segment_gather: pointer vector elements must be pointers
+ %res = call <vscale x 4 x i32> @llvm.masked.segment.gather.nxv4i32.nxv1i32(<vscale x 1 x i32> %ptrs, <vscale x 1 x i1> %mask, <vscale x 4 x i32> %passthru)
+ ret <vscale x 4 x i32> %res
+}
+
+define void @scatter_fixed_data(<4 x i32> %value, <vscale x 1 x ptr> %ptrs, <vscale x 1 x i1> %mask) {
+; CHECK: masked_segment_scatter: data type must be a scalable vector
+ call void @llvm.masked.segment.scatter.v4i32.nxv1p0(<4 x i32> %value, <vscale x 1 x ptr> align 4 %ptrs, <vscale x 1 x i1> %mask)
+ ret void
+}
+
+define void @scatter_wide_ptrs(<vscale x 4 x i32> %value, <vscale x 4 x ptr> %ptrs, <vscale x 1 x i1> %mask) {
+; CHECK: masked_segment_scatter: pointer type must be <vscale x 1 x ptr>
+ call void @llvm.masked.segment.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> %value, <vscale x 4 x ptr> align 4 %ptrs, <vscale x 1 x i1> %mask)
+ ret void
+}
+
+define void @scatter_non_ptrs(<vscale x 4 x i32> %value, <vscale x 1 x i32> %ptrs, <vscale x 1 x i1> %mask) {
+; CHECK: masked_segment_scatter: pointer vector elements must be pointers
+ call void @llvm.masked.segment.scatter.nxv4i32.nxv1i32(<vscale x 4 x i32> %value, <vscale x 1 x i32> %ptrs, <vscale x 1 x i1> %mask)
+ ret void
+}
diff --git a/llvm/test/Verifier/masked-segment-gather-scatter-valid.ll b/llvm/test/Verifier/masked-segment-gather-scatter-valid.ll
new file mode 100644
index 0000000000000..848bd8075b3ad
--- /dev/null
+++ b/llvm/test/Verifier/masked-segment-gather-scatter-valid.ll
@@ -0,0 +1,15 @@
+; RUN: opt -passes=verify -S < %s | FileCheck %s
+
+define <vscale x 4 x i32> @valid_gather(<vscale x 1 x ptr> %ptrs, <vscale x 1 x i1> %mask, <vscale x 4 x i32> %passthru) {
+; CHECK-LABEL: @valid_gather(
+; CHECK: call <vscale x 4 x i32> @llvm.masked.segment.gather.nxv4i32.nxv1p0
+ %res = call <vscale x 4 x i32> @llvm.masked.segment.gather.nxv4i32.nxv1p0(<vscale x 1 x ptr> align 4 %ptrs, <vscale x 1 x i1> %mask, <vscale x 4 x i32> %passthru)
+ ret <vscale x 4 x i32> %res
+}
+
+define void @valid_scatter(<vscale x 4 x i32> %value, <vscale x 1 x ptr> %ptrs, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: @valid_scatter(
+; CHECK: call void @llvm.masked.segment.scatter.nxv4i32.nxv1p0
+ call void @llvm.masked.segment.scatter.nxv4i32.nxv1p0(<vscale x 4 x i32> %value, <vscale x 1 x ptr> align 4 %ptrs, <vscale x 1 x i1> %mask)
+ ret void
+}
diff --git a/llvm/unittests/Analysis/IVDescriptorsTest.cpp b/llvm/unittests/Analysis/IVDescriptorsTest.cpp
index a56f9800fed76..f6edcee928f9a 100644
--- a/llvm/unittests/Analysis/IVDescriptorsTest.cpp
+++ b/llvm/unittests/Analysis/IVDescriptorsTest.cpp
@@ -15,6 +15,7 @@
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Module.h"
#include "llvm/Support/SourceMgr.h"
+#include "llvm/Transforms/Utils/LoopUtils.h"
#include "gtest/gtest.h"
using namespace llvm;
@@ -383,3 +384,105 @@ TEST(IVDescriptorsTest, InvariantStoreNoSCEV) {
EXPECT_FALSE(IsRdxPhi);
});
}
+
+/// Test that vector add reductions are detected.
+TEST(IVDescriptorsTest, VectorIntAdd) {
+ LLVMContext Context;
+ std::unique_ptr<Module> M = parseIR(Context,
+ R"(define <4 x i32> @foo(ptr %A) {
+entry:
+ br label %loop
+
+loop: ; preds = %entry, %loop
+ %indvars.iv = phi i64 [ %indvars.iv.next, %loop ], [ 0, %entry ]
+ %sum.02 = phi <4 x i32> [ %l7, %loop ], [ zeroinitializer, %entry ]
+ %l2 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
+ %l3 = load <4 x i32>, ptr %l2, align 4
+ %l7 = add <4 x i32> %sum.02, %l3
+ %indvars.iv.next = add i64 %indvars.iv, 4
+ %exitcond = icmp eq i64 %indvars.iv.next, 256
+ br i1 %exitcond, label %loop.exit, label %loop
+
+loop.exit: ; preds = %loop
+ %sum.0.lcssa = phi <4 x i32> [ %l7, %loop ]
+ ret <4 x i32> %sum.0.lcssa
+})");
+
+ runWithLoopInfoAndSE(
+ *M, "foo", [&](Function &F, LoopInfo &LI, ScalarEvolution &SE) {
+ BasicBlock *Header = F.getEntryBlock().getSingleSuccessor();
+ assert(Header->getName() == "loop");
+ Loop *L = LI.getLoopFor(Header);
+ ASSERT_NE(L, nullptr);
+ PHINode &Phi = *std::next(Header->phis().begin());
+ assert(Phi.getName() == "sum.02");
+
+ RecurrenceDescriptor Rdx;
+ bool IsRdxPhi = RecurrenceDescriptor::isReductionPHI(
+ &Phi, L, Rdx, /*DB=*/nullptr, /*AC=*/nullptr, /*DT=*/nullptr, &SE,
+ /*TTI=*/nullptr);
+ EXPECT_TRUE(IsRdxPhi);
+ const RecurKind RK = Rdx.getRecurrenceKind();
+ EXPECT_EQ(RK, RecurKind::Add);
+
+ Type *Ty = Rdx.getRecurrenceType();
+ EXPECT_EQ(Ty, VectorType::get(IntegerType::get(Context, 32),
+ ElementCount::getFixed(4)));
+ EXPECT_EQ(Ty, Phi.getType());
+
+ // Identity is expected to be (<4 x i32> zeroinitializer).
+ EXPECT_EQ(getRecurrenceIdentity(RK, Ty, Rdx.getFastMathFlags()),
+ Constant::getNullValue(Ty));
+
+ EXPECT_EQ(Rdx.getRecurrenceStartValue(), Constant::getNullValue(Ty));
+ EXPECT_EQ(Rdx.getLoopExitInstr()->getName(), "l7");
+ });
+}
+
+/// Test that vector add reductions are detected when there is more than one
+/// reduction operation (add).
+TEST(IVDescriptorsTest, VectorIntAddTwice) {
+ LLVMContext Context;
+ std::unique_ptr<Module> M = parseIR(Context,
+ R"(define <4 x i32> @foo(ptr %A) {
+entry:
+ br label %loop
+
+loop: ; preds = %entry, %loop
+ %indvars.iv = phi i64 [ %indvars.iv.next, %loop ], [ 0, %entry ]
+ %sum.02 = phi <4 x i32> [ %l7, %loop ], [ zeroinitializer, %entry ]
+ %l2 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
+ %l3 = load <4 x i32>, ptr %l2, align 4
+ %l6 = add <4 x i32> %sum.02, %l3
+ %l7 = add <4 x i32> %l6, splat (i32 1)
+ %indvars.iv.next = add i64 %indvars.iv, 4
+ %exitcond = icmp eq i64 %indvars.iv.next, 256
+ br i1 %exitcond, label %loop.exit, label %loop
+
+loop.exit: ; preds = %loop
+ %sum.0.lcssa = phi <4 x i32> [ %l7, %loop ]
+ ret <4 x i32> %sum.0.lcssa
+})");
+
+ runWithLoopInfoAndSE(
+ *M, "foo", [&](Function &F, LoopInfo &LI, ScalarEvolution &SE) {
+ BasicBlock *Header = F.getEntryBlock().getSingleSuccessor();
+ Loop *L = LI.getLoopFor(Header);
+ PHINode &Phi = *std::next(Header->phis().begin());
+ assert(Phi.getName() == "sum.02");
+
+ RecurrenceDescriptor Rdx;
+ bool IsRdxPhi = RecurrenceDescriptor::isReductionPHI(
+ &Phi, L, Rdx, /*DB=*/nullptr, /*AC=*/nullptr, /*DT=*/nullptr, &SE);
+ EXPECT_TRUE(IsRdxPhi);
+ EXPECT_EQ(Rdx.getRecurrenceKind(), RecurKind::Add);
+
+ Type *Ty = Rdx.getRecurrenceType();
+ EXPECT_EQ(Ty, VectorType::get(IntegerType::get(Context, 32),
+ ElementCount::getFixed(4)));
+ EXPECT_EQ(Ty, Phi.getType());
+
+ EXPECT_EQ(Rdx.getRecurrenceStartValue(), Constant::getNullValue(Ty));
+ EXPECT_EQ(Rdx.getLoopExitInstr()->getName(), "l7");
+ });
+}
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
index d07c72c41de84..238c102d02bea 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
@@ -199,13 +199,14 @@ TEST_F(VPlanHCFGTest, testVPInstructionToVPRecipesInner) {
TargetLibraryInfoImpl TLII(M.getTargetTriple());
TargetLibraryInfo TLI(TLII);
+ TargetTransformInfo TTI(DL);
// Current VPlan construction doesn't add a terminator for top-level loop
// latches. Add it before running transform.
cast<VPBasicBlock>(Plan->getVectorLoopRegion()->getExiting())
->appendRecipe(new VPInstruction(
VPInstruction::BranchOnCond,
{Plan->getOrAddLiveIn(ConstantInt::getTrue(F->getContext()))}));
- VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(*Plan, TLI);
+ VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(*Plan, TLI, TTI);
VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
EXPECT_EQ(0u, Entry->getNumPredecessors());
>From fc8e2800e392b3e9512cbf0926ea18b4334c8de2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ga=C3=ABtan=20Bossu?= <gaetan.bossu at arm.com>
Date: Tue, 28 Jul 2026 16:47:47 +0000
Subject: [PATCH 7/7] Create new TargetRevectorizeInfo interface
The point is to move REVEC-specific functions to a new interface which
can handle these main tasks:
- Whether we can revectorise an intrinsic
- What the cost of re-vectorisation would be
- Performing the rewriting for a given VF and widened args
Assisted-by: Codex
---
.../llvm/Analysis/TargetRevectorizeInfo.h | 124 +++++
.../llvm/Analysis/TargetTransformInfo.h | 13 -
.../llvm/Analysis/TargetTransformInfoImpl.h | 11 -
llvm/include/llvm/Analysis/VectorUtils.h | 3 +-
llvm/include/llvm/Target/TargetMachine.h | 11 +
.../Vectorize/LoopVectorizationLegality.h | 23 +-
.../llvm/Transforms/Vectorize/LoopVectorize.h | 2 +
llvm/lib/Analysis/CMakeLists.txt | 1 +
llvm/lib/Analysis/TargetRevectorizeInfo.cpp | 90 ++++
llvm/lib/Analysis/TargetTransformInfo.cpp | 13 -
llvm/lib/Analysis/VectorUtils.cpp | 9 +-
llvm/lib/Passes/PassBuilder.cpp | 1 +
llvm/lib/Passes/PassRegistry.def | 3 +
.../Target/AArch64/AArch64TargetMachine.cpp | 8 +
.../lib/Target/AArch64/AArch64TargetMachine.h | 3 +
.../AArch64/AArch64TargetRevectorizeInfo.cpp | 479 ++++++++++++++++++
.../AArch64/AArch64TargetRevectorizeInfo.h | 43 ++
.../AArch64/AArch64TargetTransformInfo.cpp | 407 +--------------
.../AArch64/AArch64TargetTransformInfo.h | 12 -
llvm/lib/Target/AArch64/CMakeLists.txt | 1 +
llvm/lib/Target/TargetMachine.cpp | 14 +
.../Vectorize/LoopVectorizationLegality.cpp | 4 +-
.../Vectorize/LoopVectorizationPlanner.h | 19 +-
.../Transforms/Vectorize/LoopVectorize.cpp | 44 +-
llvm/lib/Transforms/Vectorize/VPlan.cpp | 7 +-
llvm/lib/Transforms/Vectorize/VPlanHelpers.h | 15 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 12 +-
.../Transforms/Vectorize/VPlanTransforms.cpp | 9 +-
.../Transforms/Vectorize/VPlanTransforms.h | 8 +-
29 files changed, 877 insertions(+), 512 deletions(-)
create mode 100644 llvm/include/llvm/Analysis/TargetRevectorizeInfo.h
create mode 100644 llvm/lib/Analysis/TargetRevectorizeInfo.cpp
create mode 100644 llvm/lib/Target/AArch64/AArch64TargetRevectorizeInfo.cpp
create mode 100644 llvm/lib/Target/AArch64/AArch64TargetRevectorizeInfo.h
diff --git a/llvm/include/llvm/Analysis/TargetRevectorizeInfo.h b/llvm/include/llvm/Analysis/TargetRevectorizeInfo.h
new file mode 100644
index 0000000000000..58199431d47be
--- /dev/null
+++ b/llvm/include/llvm/Analysis/TargetRevectorizeInfo.h
@@ -0,0 +1,124 @@
+//===- TargetRevectorizeInfo.h - Target revectorization info ----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_ANALYSIS_TARGETREVECTORIZEINFO_H
+#define LLVM_ANALYSIS_TARGETREVECTORIZEINFO_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/PassManager.h"
+#include "llvm/Support/Compiler.h"
+#include "llvm/Support/InstructionCost.h"
+#include <functional>
+
+namespace llvm {
+
+class Function;
+class Instruction;
+class IRBuilderBase;
+class TargetTransformInfo;
+class Type;
+class Value;
+
+/// The type-erased interface implemented by targets that support
+/// revectorization.
+class LLVM_ABI TargetRevectorizeInfoImplBase {
+protected:
+ const TargetTransformInfo &TTI;
+
+public:
+ explicit TargetRevectorizeInfoImplBase(const TargetTransformInfo &TTI)
+ : TTI(TTI) {}
+ virtual ~TargetRevectorizeInfoImplBase();
+
+ /// Return true if the target intrinsic can be revectorized.
+ virtual bool isTargetIntrinsicVectorizable(Intrinsic::ID ID) const {
+ return false;
+ }
+
+ virtual InstructionCost
+ getTargetIntrinsicVectorizationCost(Intrinsic::ID FromID, Type *WideRetTy,
+ ArrayRef<Type *> WideArgTys,
+ ElementCount VF) const;
+
+ virtual Instruction *vectorizeTargetIntrinsic(Intrinsic::ID FromID,
+ ArrayRef<Type *> TysForDecl,
+ ArrayRef<Value *> WideArgs,
+ ElementCount VF,
+ IRBuilderBase &Builder) const;
+};
+
+/// Provides access to target-specific revectorization hooks without exposing
+/// the target's subtarget type.
+class LLVM_ABI TargetRevectorizeInfo {
+public:
+ explicit TargetRevectorizeInfo(const TargetTransformInfo &TTI);
+ explicit TargetRevectorizeInfo(
+ std::unique_ptr<const TargetRevectorizeInfoImplBase> Impl);
+
+ TargetRevectorizeInfo(TargetRevectorizeInfo &&);
+ TargetRevectorizeInfo &operator=(TargetRevectorizeInfo &&);
+ ~TargetRevectorizeInfo();
+
+ bool invalidate(Function &F, const PreservedAnalyses &PA,
+ FunctionAnalysisManager::Invalidator &Inv);
+
+ /// Return whether the target intrinsic \p ID can be vectorised. This is meant
+ /// as an early legality check, and it should be checked further for a given
+ /// VF using \p getTargetIntrinsicVectorizationCost().
+ bool isTargetIntrinsicVectorizable(Intrinsic::ID ID) const;
+
+ /// Return the cost of vectorizing a target intrinsic with its argument types
+ /// and return type widened by \p VF. The cost might be invalid.
+ ///
+ /// \pre isTargetIntrinsicVectorizable(FromID)
+ InstructionCost
+ getTargetIntrinsicVectorizationCost(Intrinsic::ID FromID, Type *WideRetTy,
+ ArrayRef<Type *> WideArgTys,
+ ElementCount VF) const;
+
+ /// Return the vectorized intrinsic call after its operands have been
+ /// widened for a given \p VF.
+ ///
+ /// \p FromID The original intrinsic ID that need vectorizing.
+ /// \p TysForDecl The widened types used to overload FromID
+ /// \p WideArgs The arguments to the intrinsic widened by \p VF
+ Instruction *vectorizeTargetIntrinsic(Intrinsic::ID FromID,
+ ArrayRef<Type *> TysForDecl,
+ ArrayRef<Value *> WideArgs,
+ ElementCount VF,
+ IRBuilderBase &Builder) const;
+
+private:
+ std::unique_ptr<const TargetRevectorizeInfoImplBase> Impl;
+};
+
+using TRVI = TargetRevectorizeInfo;
+
+class LLVM_ABI TargetRevectorizeWrapper
+ : public AnalysisInfoMixin<TargetRevectorizeWrapper> {
+public:
+ using Result = TargetRevectorizeInfo;
+
+ TargetRevectorizeWrapper();
+ explicit TargetRevectorizeWrapper(
+ std::function<Result(const Function &, const TargetTransformInfo &)>
+ Callback);
+
+ Result run(Function &F, FunctionAnalysisManager &);
+
+private:
+ friend AnalysisInfoMixin<TargetRevectorizeWrapper>;
+ static AnalysisKey Key;
+
+ std::function<Result(const Function &, const TargetTransformInfo &)> Callback;
+};
+
+} // namespace llvm
+
+#endif // LLVM_ANALYSIS_TARGETREVECTORIZEINFO_H
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 4ba7b2b029d24..42b17a90c324c 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -1044,24 +1044,11 @@ class TargetTransformInfo {
/// convention.
LLVM_ABI bool useFastCCForInternalCall(Function &F) const;
- /// Return true if the target intrinsic can be (re)vectorized.
- /// The return type as well as all arguments are expected to be widened,
- /// unless isTargetIntrinsicWithScalarOpAtArg() specifies otherwise.
- LLVM_ABI bool isTargetIntrinsicVectorizable(Intrinsic::ID ID) const;
-
/// Return true if the intrinsic \p ID is supported for a \p RK
/// reduction pattern.
LLVM_ABI bool isSupportedTargetRecurrence(Intrinsic::ID ID,
RecurKind RK) const;
- /// Return the (re)vectorized intrinsic call after the types and values have
- /// been widened. This is only called for intrinsics that have passed
- /// isTargetIntrinsicVectorizable() checks.
- LLVM_ABI Instruction *
- vectorizeTargetIntrinsic(Intrinsic::ID VectorIID, ArrayRef<Type *> TysForDecl,
- ArrayRef<Value *> WideArgs, IRBuilderBase &Builder,
- const Instruction &OrigInst) const;
-
/// Identifies if the vector form of the intrinsic has a scalar operand.
LLVM_ABI bool isTargetIntrinsicWithScalarOpAtArg(Intrinsic::ID ID,
unsigned ScalarOpdIdx) const;
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 5dd6bd683effe..55300e3d06139 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -491,22 +491,11 @@ class LLVM_ABI TargetTransformInfoImplBase {
virtual bool useFastCCForInternalCall(Function &F) const { return true; }
- virtual bool isTargetIntrinsicVectorizable(Intrinsic::ID ID) const {
- return false;
- }
-
virtual bool isSupportedTargetRecurrence(Intrinsic::ID ID,
RecurKind RK) const {
return false;
}
- virtual Instruction *
- vectorizeTargetIntrinsic(Intrinsic::ID VectorIID, ArrayRef<Type *> TysForDecl,
- ArrayRef<Value *> WideArgs, IRBuilderBase &Builder,
- const Instruction &OrigInst) const {
- llvm_unreachable("Not implemented");
- }
-
virtual bool isTargetIntrinsicWithScalarOpAtArg(Intrinsic::ID ID,
unsigned ScalarOpdIdx) const {
return false;
diff --git a/llvm/include/llvm/Analysis/VectorUtils.h b/llvm/include/llvm/Analysis/VectorUtils.h
index dcdc4a8ef5d1f..c5e537924eed4 100644
--- a/llvm/include/llvm/Analysis/VectorUtils.h
+++ b/llvm/include/llvm/Analysis/VectorUtils.h
@@ -26,6 +26,7 @@
namespace llvm {
class TargetLibraryInfo;
+class TargetRevectorizeInfo;
class IntrinsicInst;
/// The Vector Function Database.
@@ -177,7 +178,7 @@ LLVM_ABI bool isVectorIntrinsicWithStructReturnOverloadAtField(
/// its intrinsic ID, in case it does not found it return not_intrinsic.
LLVM_ABI Intrinsic::ID
getVectorIntrinsicIDForCall(const CallInst *CI, const TargetLibraryInfo *TLI,
- const TargetTransformInfo *TTI = nullptr);
+ const TargetRevectorizeInfo *TRVI = nullptr);
/// Returns the corresponding factor of llvm.vector.interleaveN intrinsics.
LLVM_ABI unsigned getInterleaveIntrinsicFactor(Intrinsic::ID ID);
diff --git a/llvm/include/llvm/Target/TargetMachine.h b/llvm/include/llvm/Target/TargetMachine.h
index b5d804d8fe942..8db549dfd1349 100644
--- a/llvm/include/llvm/Target/TargetMachine.h
+++ b/llvm/include/llvm/Target/TargetMachine.h
@@ -58,6 +58,8 @@ class SMDiagnostic;
class SMRange;
class Target;
class TargetIRAnalysis;
+class TargetRevectorizeWrapper;
+class TargetRevectorizeInfo;
class TargetTransformInfo;
class TargetLoweringObjectFile;
class TargetPassConfig;
@@ -403,6 +405,15 @@ class LLVM_ABI TargetMachine {
/// corresponding to \p F.
virtual TargetTransformInfo getTargetTransformInfo(const Function &F) const;
+ /// Get a TargetRevectorizeWrapper appropriate for the target.
+ TargetRevectorizeWrapper getTargetRevectorizeAnalysis() const;
+
+ /// Return revectorization information specialized for the subtarget
+ /// corresponding to \p F.
+ virtual TargetRevectorizeInfo
+ getTargetRevectorizeInfo(const Function &F,
+ const TargetTransformInfo &TTI) const;
+
/// Allow the target to modify the pass pipeline.
// TODO: Populate all pass names by using <Target>PassRegistry.def.
virtual void registerPassBuilderCallbacks(PassBuilder &) {}
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index abf754d6b2e86..16ecc3fad0669 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -45,6 +45,7 @@ class OptimizationRemarkEmitter;
class PredicatedScalarEvolution;
class ProfileSummaryInfo;
class TargetLibraryInfo;
+class TargetRevectorizeInfo;
class TargetTransformInfo;
class Type;
@@ -278,14 +279,17 @@ enum class UncountableExitTrait { None, ReadOnly, ReadWrite };
/// induction variable and the different reduction variables.
class LoopVectorizationLegality {
public:
- LoopVectorizationLegality(
- Loop *L, PredicatedScalarEvolution &PSE, DominatorTree *DT,
- TargetTransformInfo *TTI, TargetLibraryInfo *TLI, Function *F,
- LoopAccessInfoManager &LAIs, LoopInfo *LI, OptimizationRemarkEmitter *ORE,
- LoopVectorizationRequirements *R, LoopVectorizeHints *H, DemandedBits *DB,
- AssumptionCache *AC, bool AllowRuntimeSCEVChecks, AAResults *AA)
- : TheLoop(L), LI(LI), PSE(PSE), TTI(TTI), TLI(TLI), DT(DT), LAIs(LAIs),
- ORE(ORE), Requirements(R), Hints(H), DB(DB), AC(AC),
+ LoopVectorizationLegality(Loop *L, PredicatedScalarEvolution &PSE,
+ DominatorTree *DT, TargetTransformInfo *TTI,
+ TargetRevectorizeInfo *TRVI, TargetLibraryInfo *TLI,
+ Function *F, LoopAccessInfoManager &LAIs,
+ LoopInfo *LI, OptimizationRemarkEmitter *ORE,
+ LoopVectorizationRequirements *R,
+ LoopVectorizeHints *H, DemandedBits *DB,
+ AssumptionCache *AC, bool AllowRuntimeSCEVChecks,
+ AAResults *AA)
+ : TheLoop(L), LI(LI), PSE(PSE), TTI(TTI), TRVI(TRVI), TLI(TLI), DT(DT),
+ LAIs(LAIs), ORE(ORE), Requirements(R), Hints(H), DB(DB), AC(AC),
AllowRuntimeSCEVChecks(AllowRuntimeSCEVChecks), AA(AA) {}
/// ReductionList contains the reduction descriptors for all
@@ -682,6 +686,9 @@ class LoopVectorizationLegality {
/// Target Transform Info.
TargetTransformInfo *TTI;
+ /// Target Revectorization Info.
+ TargetRevectorizeInfo *TRVI;
+
/// Target Library Info.
TargetLibraryInfo *TLI;
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h
index 4512e5664cfc4..e7df14b59abb8 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h
@@ -77,6 +77,7 @@ class OptimizationRemarkEmitter;
class ProfileSummaryInfo;
class ScalarEvolution;
class TargetLibraryInfo;
+class TargetRevectorizeInfo;
class TargetTransformInfo;
LLVM_ABI extern cl::opt<bool> EnableLoopInterleaving;
@@ -143,6 +144,7 @@ struct LoopVectorizePass : public OptionalPassInfoMixin<LoopVectorizePass> {
ScalarEvolution *SE;
LoopInfo *LI;
+ TargetRevectorizeInfo *TRVI;
TargetTransformInfo *TTI;
DominatorTree *DT;
std::function<BlockFrequencyInfo &()> GetBFI;
diff --git a/llvm/lib/Analysis/CMakeLists.txt b/llvm/lib/Analysis/CMakeLists.txt
index f3586c66cb056..5e87a412f5f8b 100644
--- a/llvm/lib/Analysis/CMakeLists.txt
+++ b/llvm/lib/Analysis/CMakeLists.txt
@@ -152,6 +152,7 @@ add_llvm_component_library(LLVMAnalysis
SyntheticCountsUtils.cpp
TFLiteUtils.cpp
TargetLibraryInfo.cpp
+ TargetRevectorizeInfo.cpp
TargetTransformInfo.cpp
TensorSpec.cpp
Trace.cpp
diff --git a/llvm/lib/Analysis/TargetRevectorizeInfo.cpp b/llvm/lib/Analysis/TargetRevectorizeInfo.cpp
new file mode 100644
index 0000000000000..825b7f6b5ea70
--- /dev/null
+++ b/llvm/lib/Analysis/TargetRevectorizeInfo.cpp
@@ -0,0 +1,90 @@
+//===- TargetRevectorizeInfo.cpp - Target revectorization info ------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Analysis/TargetRevectorizeInfo.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/IR/Function.h"
+#include "llvm/Support/ErrorHandling.h"
+
+using namespace llvm;
+
+TargetRevectorizeInfoImplBase::~TargetRevectorizeInfoImplBase() = default;
+
+InstructionCost
+TargetRevectorizeInfoImplBase::getTargetIntrinsicVectorizationCost(
+ Intrinsic::ID FromID, Type *WideRetTy, ArrayRef<Type *> WideArgTys,
+ ElementCount VF) const {
+ llvm_unreachable("Target intrinsic cost is not implemented");
+}
+
+Instruction *TargetRevectorizeInfoImplBase::vectorizeTargetIntrinsic(
+ Intrinsic::ID, ArrayRef<Type *>, ArrayRef<Value *>, ElementCount,
+ IRBuilderBase &) const {
+ llvm_unreachable("Target intrinsic vectorization is not implemented");
+}
+
+TargetRevectorizeInfo::TargetRevectorizeInfo(const TargetTransformInfo &TTI)
+ : TargetRevectorizeInfo(
+ std::make_unique<TargetRevectorizeInfoImplBase>(TTI)) {}
+
+TargetRevectorizeInfo::TargetRevectorizeInfo(
+ std::unique_ptr<const TargetRevectorizeInfoImplBase> Impl)
+ : Impl(std::move(Impl)) {
+ assert(this->Impl && "Expected a valid TargetRevectorizeInfo implementation");
+}
+
+TargetRevectorizeInfo::TargetRevectorizeInfo(TargetRevectorizeInfo &&) =
+ default;
+TargetRevectorizeInfo &
+TargetRevectorizeInfo::operator=(TargetRevectorizeInfo &&) = default;
+TargetRevectorizeInfo::~TargetRevectorizeInfo() = default;
+
+bool TargetRevectorizeInfo::invalidate(
+ Function &F, const PreservedAnalyses &PA,
+ FunctionAnalysisManager::Invalidator &Inv) {
+ // The implementation holds a reference to TTI (result of TargetIRAnalysis),
+ // so it gets invalided if TargetIRAnalysis itself is invalidated.
+ return Inv.invalidate<TargetIRAnalysis>(F, PA);
+}
+
+bool TargetRevectorizeInfo::isTargetIntrinsicVectorizable(
+ Intrinsic::ID ID) const {
+ return Impl->isTargetIntrinsicVectorizable(ID);
+}
+
+InstructionCost TargetRevectorizeInfo::getTargetIntrinsicVectorizationCost(
+ Intrinsic::ID FromID, Type *WideRetTy, ArrayRef<Type *> WideArgTys,
+ ElementCount VF) const {
+ return Impl->getTargetIntrinsicVectorizationCost(FromID, WideRetTy,
+ WideArgTys, VF);
+}
+
+Instruction *TargetRevectorizeInfo::vectorizeTargetIntrinsic(
+ Intrinsic::ID VectorIID, ArrayRef<Type *> TysForDecl,
+ ArrayRef<Value *> WideArgs, ElementCount VF, IRBuilderBase &Builder) const {
+ return Impl->vectorizeTargetIntrinsic(VectorIID, TysForDecl, WideArgs, VF,
+ Builder);
+}
+
+TargetRevectorizeWrapper::TargetRevectorizeWrapper()
+ : Callback([](const Function &, const TargetTransformInfo &TTI) {
+ return TargetRevectorizeInfo(TTI);
+ }) {}
+
+TargetRevectorizeWrapper::TargetRevectorizeWrapper(
+ std::function<Result(const Function &, const TargetTransformInfo &)>
+ Callback)
+ : Callback(std::move(Callback)) {}
+
+TargetRevectorizeWrapper::Result
+TargetRevectorizeWrapper::run(Function &F, FunctionAnalysisManager &AM) {
+ assert(!F.isIntrinsic() && "Should not request TRVI for intrinsics");
+ return Callback(F, AM.getResult<TargetIRAnalysis>(F));
+}
+
+AnalysisKey TargetRevectorizeWrapper::Key;
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index e63e21076f1c2..573a293ae0b93 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -639,24 +639,11 @@ bool TargetTransformInfo::useFastCCForInternalCall(Function &F) const {
return TTIImpl->useFastCCForInternalCall(F);
}
-bool TargetTransformInfo::isTargetIntrinsicVectorizable(
- Intrinsic::ID ID) const {
- return TTIImpl->isTargetIntrinsicVectorizable(ID);
-}
-
bool TargetTransformInfo::isSupportedTargetRecurrence(Intrinsic::ID ID,
RecurKind RK) const {
return TTIImpl->isSupportedTargetRecurrence(ID, RK);
}
-Instruction *TargetTransformInfo::vectorizeTargetIntrinsic(
- Intrinsic::ID VectorIID, ArrayRef<Type *> TysForDecl,
- ArrayRef<Value *> WideArgs, IRBuilderBase &Builder,
- const Instruction &OrigInst) const {
- return TTIImpl->vectorizeTargetIntrinsic(VectorIID, TysForDecl, WideArgs,
- Builder, OrigInst);
-}
-
bool TargetTransformInfo::isTargetIntrinsicWithScalarOpAtArg(
Intrinsic::ID ID, unsigned ScalarOpdIdx) const {
return TTIImpl->isTargetIntrinsicWithScalarOpAtArg(ID, ScalarOpdIdx);
diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp
index 6ab1ca219631a..970125fc3b4a3 100644
--- a/llvm/lib/Analysis/VectorUtils.cpp
+++ b/llvm/lib/Analysis/VectorUtils.cpp
@@ -18,6 +18,7 @@
#include "llvm/Analysis/LoopIterator.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
+#include "llvm/Analysis/TargetRevectorizeInfo.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/IR/Constants.h"
@@ -236,13 +237,15 @@ bool llvm::isVectorIntrinsicWithStructReturnOverloadAtField(
Intrinsic::ID
llvm::getVectorIntrinsicIDForCall(const CallInst *CI,
const TargetLibraryInfo *TLI,
- const TargetTransformInfo *TTI) {
+ const TargetRevectorizeInfo *TRVI) {
Intrinsic::ID ID = getIntrinsicForCallSite(*CI, TLI);
if (ID == Intrinsic::not_intrinsic)
return Intrinsic::not_intrinsic;
- if (TTI && Intrinsic::isTargetIntrinsic(ID) &&
- TTI->isTargetIntrinsicVectorizable(ID))
+ // REVEC: Keep the original Intrinsic::ID so far and let TRVI handle
+ // the actual rewriting.
+ if (TRVI && Intrinsic::isTargetIntrinsic(ID) &&
+ TRVI->isTargetIntrinsicVectorizable(ID))
return ID;
if (isTriviallyVectorizable(ID) || ID == Intrinsic::lifetime_start ||
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index f7db63ef8bf74..0badff4bf7509 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -75,6 +75,7 @@
#include "llvm/Analysis/StackSafetyAnalysis.h"
#include "llvm/Analysis/StructuralHash.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/Analysis/TargetRevectorizeInfo.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/TypeBasedAliasAnalysis.h"
#include "llvm/Analysis/UniformityAnalysis.h"
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index 8f70c7cefa408..814a0299f48fc 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -390,6 +390,9 @@ FUNCTION_ANALYSIS("ssp-layout", SSPLayoutAnalysis())
FUNCTION_ANALYSIS("stack-safety-local", StackSafetyAnalysis())
FUNCTION_ANALYSIS("target-ir",
TM ? TM->getTargetIRAnalysis() : TargetIRAnalysis())
+FUNCTION_ANALYSIS("target-revectorize",
+ TM ? TM->getTargetRevectorizeAnalysis()
+ : TargetRevectorizeWrapper())
FUNCTION_ANALYSIS("target-lib-info", TargetLibraryAnalysis())
FUNCTION_ANALYSIS("uniformity", UniformityInfoAnalysis())
FUNCTION_ANALYSIS("verify", VerifierAnalysis())
diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
index b8cc69f7569d6..cc3bca6ada19e 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
@@ -16,10 +16,12 @@
#include "AArch64MacroFusion.h"
#include "AArch64Subtarget.h"
#include "AArch64TargetObjectFile.h"
+#include "AArch64TargetRevectorizeInfo.h"
#include "AArch64TargetTransformInfo.h"
#include "MCTargetDesc/AArch64MCTargetDesc.h"
#include "TargetInfo/AArch64TargetInfo.h"
#include "llvm/ADT/StringExtras.h"
+#include "llvm/Analysis/TargetRevectorizeInfo.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/CodeGen/CSEConfigBase.h"
@@ -620,6 +622,12 @@ AArch64TargetMachine::getTargetTransformInfo(const Function &F) const {
return TargetTransformInfo(std::make_unique<AArch64TTIImpl>(this, F));
}
+TargetRevectorizeInfo AArch64TargetMachine::getTargetRevectorizeInfo(
+ const Function &F, const TargetTransformInfo &TTI) const {
+ return TargetRevectorizeInfo(
+ std::make_unique<AArch64RevectorizeInfoImpl>(TTI, *getSubtargetImpl(F)));
+}
+
TargetPassConfig *AArch64TargetMachine::createPassConfig(PassManagerBase &PM) {
return new AArch64PassConfig(*this, PM);
}
diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.h b/llvm/lib/Target/AArch64/AArch64TargetMachine.h
index 98eaecbb6d2af..897a93c8611f1 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetMachine.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.h
@@ -49,6 +49,9 @@ class AArch64TargetMachine : public CodeGenTargetMachineImpl {
void registerPassBuilderCallbacks(PassBuilder &PB) override;
TargetTransformInfo getTargetTransformInfo(const Function &F) const override;
+ TargetRevectorizeInfo
+ getTargetRevectorizeInfo(const Function &F,
+ const TargetTransformInfo &TTI) const override;
TargetLoweringObjectFile* getObjFileLowering() const override {
return TLOF.get();
diff --git a/llvm/lib/Target/AArch64/AArch64TargetRevectorizeInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetRevectorizeInfo.cpp
new file mode 100644
index 0000000000000..82f20e0623b07
--- /dev/null
+++ b/llvm/lib/Target/AArch64/AArch64TargetRevectorizeInfo.cpp
@@ -0,0 +1,479 @@
+//===- AArch64TargetRevectorizeInfo.cpp - AArch64 TRVI --------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "AArch64TargetRevectorizeInfo.h"
+#include "AArch64Subtarget.h"
+#include "Utils/AArch64BaseInfo.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicsAArch64.h"
+#include "llvm/IR/VectorTypeUtils.h"
+#include <optional>
+
+using namespace llvm;
+
+// Define the struct for the generated intrinsic mapping table
+namespace {
+struct NEONToSVEIntrinsicMapping {
+ StringRef NEONIntrinsic;
+ StringRef SVEIntrinsic;
+ StringRef ArgMappings; // Comma-separated string: "T,Z,A:0,A:1"
+ bool isCustom() const { return SVEIntrinsic.empty() || ArgMappings.empty(); }
+};
+
+/// Specifies how to build an argument for an SVE intrinsic, possibly from
+/// another argument of an equivalent NEON intrinsic.
+class ArgMapping {
+public:
+ enum MappingKind { True, Zero, Poison, Arg, Imm };
+
+ ArgMapping(StringRef Spec) {
+ if (Spec == "T") {
+ Kind = True;
+ } else if (Spec == "Z") {
+ Kind = Zero;
+ } else if (Spec == "P") {
+ Kind = Poison;
+ } else if (Spec.starts_with("A:")) {
+ Kind = Arg;
+ unsigned Idx;
+ if (Spec.substr(2).getAsInteger(10, Idx))
+ llvm_unreachable("Invalid argument index in mapping!");
+ Index = Idx;
+ } else if (Spec.starts_with("I:")) {
+ Kind = Imm;
+ unsigned Idx;
+ if (Spec.substr(2).getAsInteger(10, Idx))
+ llvm_unreachable("Invalid argument index in mapping!");
+ Index = Idx;
+ } else {
+ llvm_unreachable("Unknown argument mapping kind!");
+ }
+ }
+
+ MappingKind getKind() const { return Kind; }
+
+ unsigned getArgIndex() const {
+ assert((Kind == Arg || Kind == Imm) &&
+ "getArgIndex called on non-Arg mapping");
+ return *Index;
+ }
+
+private:
+ MappingKind Kind;
+ std::optional<unsigned> Index;
+};
+
+// Parse comma-separated argument mapping string into ArgMapping objects
+static SmallVector<ArgMapping, 4> parseArgMappings(StringRef ArgMappingsStr) {
+ SmallVector<ArgMapping, 4> Result;
+ SmallVector<StringRef, 4> Specs;
+ ArgMappingsStr.split(Specs, ',');
+ for (StringRef Spec : Specs)
+ Result.emplace_back(Spec);
+ return Result;
+}
+
+/// Generate a segmented UZP shuffle that can later be selected to UZPQ.
+/// \pre Src0 and Src1 are scalable vectors
+Instruction *getSegmentedUZP(Value *Src0, Value *Src1, bool EvenElts,
+ IRBuilderBase &Builder) {
+ assert(isa<ScalableVectorType>(Src0->getType()));
+ auto *VTy = cast<ScalableVectorType>(Src0->getType());
+ const unsigned SegmentSize = VTy->getMinNumElements();
+ SmallVector<int, 8> Mask(SegmentSize);
+ transform(seq<int>(SegmentSize), Mask.begin(),
+ [EvenElts](int Idx) { return EvenElts ? Idx * 2 : Idx * 2 + 1; });
+ return Builder.CreateSegmentedShuffleVector(Src0, Src1, Mask);
+}
+
+/// Concatenate the even words of \p Src with its odd words within each quad.
+/// I.e. generate UZPQ1 Src.w,, (EXT Src, 4).w
+Instruction *concatEvenThenOddWordsWithinQuads(Value *Src,
+ IRBuilderBase &Builder) {
+ assert(Src->getType()->getPrimitiveSizeInBits().getKnownMinValue() ==
+ AArch64::SVEBitsPerBlock &&
+ "Expected legal SVE vector.");
+ assert(!Src->getType()->getScalarType()->isIntegerTy(32) &&
+ "Unexpected nxv4i32 input");
+ Type *OrigTy = Src->getType();
+ Src = Builder.CreateBitCast(Src,
+ ScalableVectorType::get(Builder.getInt32Ty(), 4));
+ Value *ShiftOddToEven =
+ Builder.CreateVectorSpliceLeft(Src, PoisonValue::get(Src->getType()), 1);
+ Instruction *Res =
+ getSegmentedUZP(Src, ShiftOddToEven, /*EvenElts=*/true, Builder);
+ return cast<Instruction>(Builder.CreateBitCast(Res, OrigTy));
+}
+
+SmallVector<Type *, 2> computeTysForDecl(Intrinsic::ID ID, Type *RetTy,
+ ArrayRef<Value *> Args,
+ const TargetTransformInfo &TTI) {
+ SmallVector<Type *, 2> Tys;
+ if (TTI.isTargetIntrinsicWithOverloadTypeAtArg(ID, -1))
+ Tys.push_back(RetTy);
+ for (auto [ArgIdx, Arg] : enumerate(Args))
+ if (TTI.isTargetIntrinsicWithOverloadTypeAtArg(ID, ArgIdx))
+ Tys.push_back(Arg->getType());
+ return Tys;
+}
+
+/// A helper class to rewrite a widened NEON intrinsic to SVE.
+class IntrinsicRewriter {
+public:
+ IntrinsicRewriter(ArrayRef<Type *> TysForDecl, ArrayRef<Value *> WideArgs,
+ IRBuilderBase &Builder)
+ : TysForDecl(TysForDecl), WideArgs(WideArgs), Builder(Builder) {}
+
+ /// Automatically rewrite a widened intrinsic using a \p
+ /// NEONToSVEIntrinsicMapping.
+ Instruction *rewriteWithMapping(const NEONToSVEIntrinsicMapping &Mapping,
+ const TargetTransformInfo &TTI) {
+ assert(!Mapping.isCustom() &&
+ "NEON intrinsic was expected to be custom-lowered to SVE");
+
+ // DataTy governs the creation of zero/poison values and predicates. It is
+ // conveniently always the first "overload type" in TysForDecl.
+ Type *DataTy = TysForDecl[0];
+
+ // Parse and apply argument mappings from the table
+ SmallVector<ArgMapping, 4> ArgMappings =
+ parseArgMappings(Mapping.ArgMappings);
+ SmallVector<Value *, 4> SVEArgs;
+ for (const ArgMapping &AM : ArgMappings) {
+ switch (AM.getKind()) {
+ case ArgMapping::True:
+ SVEArgs.push_back(createTrue(DataTy));
+ break;
+ case ArgMapping::Zero:
+ SVEArgs.push_back(createZero(DataTy));
+ break;
+ case ArgMapping::Poison:
+ SVEArgs.push_back(createPoison(DataTy));
+ break;
+ case ArgMapping::Arg:
+ SVEArgs.push_back(createArg(AM.getArgIndex()));
+ break;
+ case ArgMapping::Imm:
+ SVEArgs.push_back(createImm(AM.getArgIndex()));
+ break;
+ }
+ }
+
+ Intrinsic::ID SVEIntrinsicID =
+ Intrinsic::lookupIntrinsicID(Mapping.SVEIntrinsic);
+ auto TysForSVEDecl = computeTysForDecl(
+ SVEIntrinsicID, getLegalizedSVETy(DataTy), SVEArgs, TTI);
+ assert(SVEIntrinsicID != Intrinsic::not_intrinsic);
+ return fromSVEInst(
+ Builder.CreateIntrinsic(SVEIntrinsicID, TysForSVEDecl, SVEArgs),
+ DataTy);
+ }
+
+ /// Custom rewrite a widened intrinsic without using a \p
+ /// NEONToSVEIntrinsicMapping.
+ Instruction *rewriteCustom(Intrinsic::ID IID, ElementCount VF) {
+ // Custom lower some intrinsics.
+ switch (IID) {
+ case Intrinsic::aarch64_neon_uaddlp:
+ case Intrinsic::aarch64_neon_saddlp: {
+ // ADDLP works with adjacent elements: scaled 64-bit NEON inputs cannot be
+ // legalised to SVE by interleaving. Instead, extend and use a normal
+ // ADDP.
+ Type *SrcTy = WideArgs[0]->getType();
+ if (SrcTy->getPrimitiveSizeInBits().getKnownMinValue() != 64U)
+ break;
+
+ Type *DstTy = TysForDecl[0];
+ Type *WideSrcTy = getLegalizedSVETy(DstTy);
+ const bool IsSigned = IID == Intrinsic::aarch64_neon_saddlp;
+ Value *Src = IsSigned ? Builder.CreateSExt(WideArgs[0], WideSrcTy)
+ : Builder.CreateZExt(WideArgs[0], WideSrcTy);
+ return fromSVEInst(
+ Builder.CreateIntrinsic(Intrinsic::aarch64_sve_addp, {WideSrcTy},
+ {createTrue(WideSrcTy), Src, Src}),
+ DstTy);
+ }
+ case Intrinsic::aarch64_neon_addp:
+ case Intrinsic::aarch64_neon_faddp: {
+ Type *Ty = TysForDecl[0];
+ const bool IsInt = IID == Intrinsic::aarch64_neon_addp;
+ // ADDP works with adjacent elements: scaled 64-bit NEON inputs cannot be
+ // legalised to SVE by interleaving with undef elements.
+ // Instead, deinterleave and use a normal ADD.
+
+ // But if the initial NEON vector contained a single pair, we can
+ // directly use SVE's ADDP without prior UZPQ de-interleaving.
+ ElementCount NumPairsPerVec = getElementCount(Ty).divideCoefficientBy(2);
+ unsigned NumPairsPerSrcVec =
+ ElementCount::get(NumPairsPerVec.getKnownMinValue() /
+ VF.getKnownMinValue(),
+ NumPairsPerVec.isScalable() != VF.isScalable())
+ .getFixedValue();
+ if (Ty == getLegalizedSVETy(Ty) && NumPairsPerSrcVec == 1) {
+ Intrinsic::ID IID =
+ IsInt ? Intrinsic::aarch64_sve_addp : Intrinsic::aarch64_sve_faddp;
+ return Builder.CreateIntrinsic(
+ IID, {Ty}, {createTrue(Ty), WideArgs[0], WideArgs[1]});
+ }
+
+ // Within each segment, ensure we have all the pairwise adds from Args0
+ // followed by those of Add1.
+ Value *EvenElts =
+ getSegmentedUZP(WideArgs[0], WideArgs[1], /*EvenElts=*/true, Builder);
+ Value *OddElts = getSegmentedUZP(WideArgs[0], WideArgs[1],
+ /*EvenElts=*/false, Builder);
+ auto *Res =
+ cast<Instruction>(IsInt ? Builder.CreateAdd(EvenElts, OddElts)
+ : Builder.CreateFAdd(EvenElts, OddElts));
+
+ // The segment type might not match the original type (VF = vscale x 2),
+ // i.e. the original type is 64-bit but our segments are 128-bit.
+ // Then, ensure that within each 64-bit segment, the lo 32-bit are
+ // pairwise adds from Arg0 and the hi 32-bit are pairwise adds from Arg1.
+ if (VF != ElementCount::getScalable(1)) {
+ assert(VF == ElementCount::getScalable(2) &&
+ Ty == getLegalizedSVETy(Ty) &&
+ "Expected REVEC from NEON 64-bit to legal SVE");
+ Res = concatEvenThenOddWordsWithinQuads(Res, Builder);
+ }
+ return Res;
+ }
+ case Intrinsic::aarch64_neon_sshl:
+ case Intrinsic::aarch64_neon_ushl: {
+ // SVE does not have a "plain" non-saturating non-rounding shl.
+ // This means we need to use [su]rshl for positive shift amounts
+ // and [su]qshl for negative shift amounts (effectively a shift right).
+ const bool IsSignedShift = IID == Intrinsic::aarch64_neon_sshl;
+ Intrinsic::ID SHLIID = IsSignedShift ? Intrinsic::aarch64_sve_srshl
+ : Intrinsic::aarch64_sve_urshl;
+ Intrinsic::ID SHRIID = IsSignedShift ? Intrinsic::aarch64_sve_sqshl
+ : Intrinsic::aarch64_sve_uqshl;
+ Type *ArgTy = TysForDecl[0];
+ Value *Arg0 = createArg(0);
+ Value *Arg1 = createArg(1);
+ Value *SHLMask = Builder.CreateICmpSLE(
+ Arg1, ConstantInt::get(getLegalizedSVETy(ArgTy), 0), "shl.mask");
+ Value *SHL = Builder.CreateIntrinsic(SHLIID, {getLegalizedSVETy(ArgTy)},
+ {SHLMask, Arg0, Arg1});
+ Value *SHRMask =
+ Builder.CreateIntrinsic(Intrinsic::ctlz, {SHLMask->getType()},
+ {SHLMask, Builder.getTrue()}, {}, "shr.mask");
+ return fromSVEInst(
+ Builder.CreateIntrinsic(SHRIID, {getLegalizedSVETy(ArgTy)},
+ {SHRMask, SHL, Arg1}, {}, "wide.bidir.shl"),
+ ArgTy);
+ }
+ case Intrinsic::aarch64_neon_tbl1:
+ case Intrinsic::aarch64_neon_tbl2:
+ case Intrinsic::aarch64_neon_tbl3:
+ case Intrinsic::aarch64_neon_tbl4: {
+ Type *Ty = TysForDecl[0];
+ assert(VF == ElementCount::getScalable(1) &&
+ "Unexpected vscale x 2 REVEC");
+ Value *Src0 = createArg(0);
+ Value *Mask = createArg(WideArgs.size() - 1);
+ Instruction *Res = Builder.CreateIntrinsic(
+ Intrinsic::aarch64_sve_tblq, {Src0->getType()}, {Src0, Mask});
+ for (unsigned SrcIdx = 1; SrcIdx < WideArgs.size() - 1; ++SrcIdx) {
+ Value *Src = createArg(SrcIdx);
+ Mask = Builder.CreateSub(Mask, ConstantInt::get(Mask->getType(), 16));
+ Res = Builder.CreateIntrinsic(Intrinsic::aarch64_sve_tbxq,
+ {Src->getType()}, {Res, Src, Mask});
+ }
+ return fromSVEInst(Res, Ty);
+ }
+ }
+ return nullptr;
+ }
+
+private:
+ ArrayRef<Type *> TysForDecl;
+ ArrayRef<Value *> WideArgs;
+ IRBuilderBase &Builder;
+
+ /// Get the type that \p Ty can be legalized to for SVE instructions.
+ Type *getLegalizedSVETy(Type *Ty) const {
+ if (!Ty->isVectorTy())
+ return Ty;
+ auto *SVTy = cast<ScalableVectorType>(Ty);
+ unsigned FixedBits = SVTy->getMinNumElements() *
+ SVTy->getElementType()->getScalarSizeInBits();
+ return toVectorTy(
+ Ty, ElementCount::getFixed(AArch64::SVEBitsPerBlock / FixedBits));
+ };
+
+ /// Turn a widened value into something that an SVE instruction can consume.
+ Value *toSVEVal(Value *V) {
+ Type *LegalSVETy = getLegalizedSVETy(V->getType());
+ if (LegalSVETy == V->getType())
+ return V;
+
+ // For scaled 64-bit vectors, interleave/deinterleave even elements to makes
+ // them valid SVE types. This plays nicely for both int and fp types later
+ // during ISel.
+ return Builder.CreateVectorInterleave({V, PoisonValue::get(V->getType())});
+ };
+
+ /// Turn a legalised widened value back to its expected widened type.
+ Instruction *fromSVEInst(Instruction *I, Type *ExpectedTy) {
+ if (ExpectedTy == I->getType())
+ return I;
+ auto *RevecTy = cast<VectorType>(I->getType());
+ auto *ExpectedEltTy = cast<VectorType>(ExpectedTy)->getElementType();
+ if (RevecTy->getElementType() != ExpectedEltTy) {
+ assert(RevecTy->getElementType()->isIntegerTy(1) &&
+ "Only expected SVE predicate to be turned into NEON vector.");
+ Type *NeonQuadPredTy =
+ VectorType::get(ExpectedEltTy, RevecTy->getElementCount());
+ I = cast<Instruction>(Builder.CreateZExt(I, NeonQuadPredTy));
+ }
+ if (ExpectedTy == I->getType())
+ return I;
+ auto *Deinterleave = Builder.CreateIntrinsic(
+ Intrinsic::vector_deinterleave2, I->getType(), I);
+ return cast<Instruction>(Builder.CreateExtractValue(Deinterleave, {0U}));
+ };
+
+ /// Create a zero-initialised value for the \p Ty widened type.
+ Value *createZero(Type *Ty) {
+ return Constant::getNullValue(getLegalizedSVETy(Ty));
+ }
+
+ /// Create a poison value for the \p Ty widened type.
+ Value *createPoison(Type *Ty) {
+ return PoisonValue::get(getLegalizedSVETy(Ty));
+ }
+
+ /// Create an all-true predicate for the \p Ty widened data type.
+ Value *createTrue(Type *Ty) {
+ auto *VectorTy = cast<VectorType>(getLegalizedSVETy(Ty));
+ // SVE intrinsics that narrow the element type have a predicate type with
+ // half as many elements as their return type because every odd lane is
+ // zeroed/undefined.
+ bool IsNarrowing = any_of(WideArgs, [Ty](const Value *V) {
+ return V->getType()->getScalarSizeInBits() > Ty->getScalarSizeInBits();
+ });
+ if (IsNarrowing)
+ VectorTy = VectorType::getOneNthElementsVectorType(VectorTy, 2);
+ auto *PredTy = VectorType::get(IntegerType::get(Ty->getContext(), 1),
+ VectorTy->getElementCount());
+ return Constant::getAllOnesValue(PredTy);
+ }
+
+ /// Create a legal SVE value for the argument at \p ArgIdx.
+ Value *createArg(unsigned ArgIdx) {
+ assert(ArgIdx < WideArgs.size() && "Argument index out of range!");
+ Value *V = WideArgs[ArgIdx];
+ return toSVEVal(V);
+ };
+
+ /// Derive a scalar predicate from the argument at \p ArgIdx.
+ Value *createImm(unsigned ArgIdx) {
+ assert(ArgIdx < WideArgs.size() && "Argument index out of range!");
+ Value *V = WideArgs[ArgIdx];
+ assert(isa<Constant>(V) && cast<Constant>(V)->getSplatValue());
+ return cast<Constant>(V)->getSplatValue();
+ };
+};
+} // end anonymous namespace
+
+// Include generated vector intrinsic mapping table
+#define GET_IntrinsicMappingTable_IMPL
+#include "AArch64GenVectorIntrinsicMappings.inc"
+
+bool AArch64RevectorizeInfoImpl::isTargetIntrinsicVectorizable(
+ Intrinsic::ID ID) const {
+ return ST.hasSVE2p1() &&
+ lookupNEONToSVEMappingByNEONIntrinsic(Intrinsic::getBaseName(ID));
+}
+
+Instruction *AArch64RevectorizeInfoImpl::vectorizeTargetIntrinsic(
+ Intrinsic::ID FromID, ArrayRef<Type *> TysForDecl,
+ ArrayRef<Value *> WideArgs, ElementCount VF, IRBuilderBase &Builder) const {
+
+ // Look up the mapping in the generated table
+ StringRef NEONIntrinsicName = Intrinsic::getBaseName(FromID);
+ const NEONToSVEIntrinsicMapping *Mapping =
+ lookupNEONToSVEMappingByNEONIntrinsic(NEONIntrinsicName);
+ if (!Mapping)
+ llvm_unreachable("Unimplemented intrinsic vectorisation!");
+
+ auto IsScaledNEONOrScalar = [](const Value *V) -> bool {
+ if (V->getType()->isIntegerTy())
+ return true;
+ auto *SVTy = dyn_cast<ScalableVectorType>(V->getType());
+ if (!SVTy)
+ return false;
+ unsigned FixedBits = SVTy->getMinNumElements() *
+ SVTy->getElementType()->getScalarSizeInBits();
+ return FixedBits == AArch64::SVEBitsPerBlock ||
+ FixedBits == (AArch64::SVEBitsPerBlock / 2);
+ };
+ assert(all_of(WideArgs, IsScaledNEONOrScalar) &&
+ "Expected SVE-compatible values.");
+
+ IntrinsicRewriter Rewriter(TysForDecl, WideArgs, Builder);
+ if (Instruction *I = Rewriter.rewriteCustom(FromID, VF))
+ return I;
+ return Rewriter.rewriteWithMapping(*Mapping, TTI);
+}
+
+InstructionCost AArch64RevectorizeInfoImpl::getTargetIntrinsicVectorizationCost(
+ Intrinsic::ID FromID, Type *WideRetTy, ArrayRef<Type *> WideArgTys,
+ ElementCount VF) const {
+ auto IsAcceptableTyForREVEC = [](const Type *Ty) {
+ return Ty->getPrimitiveSizeInBits().getKnownMinValue() <=
+ AArch64::SVEBitsPerBlock;
+ };
+
+ switch (FromID) {
+ case Intrinsic::aarch64_neon_sshl:
+ case Intrinsic::aarch64_neon_ushl: {
+ // Re-vectorising NEON's non-saturating non-rounding bi-directional shl
+ // requires the use of two SVE bi-directional shl: one for the non-rounding
+ // part and one for the non-saturating part. (+masking)
+ if (!IsAcceptableTyForREVEC(WideRetTy))
+ return InstructionCost::getInvalid();
+ return InstructionCost(2);
+ }
+ case Intrinsic::aarch64_neon_addp:
+ case Intrinsic::aarch64_neon_faddp: {
+ // Re-vectorising NEON's ADDP requires prior deinterleaving because SVE does
+ // not have have a ADDPQ variant working like NEON within quads.
+ // Re-vectorising 64-bit NEON requires even more instructions.
+ if (!IsAcceptableTyForREVEC(WideRetTy))
+ return InstructionCost::getInvalid();
+ return InstructionCost(2);
+ }
+ case Intrinsic::aarch64_neon_tbl1:
+ case Intrinsic::aarch64_neon_tbl2:
+ case Intrinsic::aarch64_neon_tbl3:
+ case Intrinsic::aarch64_neon_tbl4: {
+ assert(WideArgTys.size() >= 2 &&
+ "Tbl needs at least one source and one mask");
+ if (!IsAcceptableTyForREVEC(WideRetTy) ||
+ !all_of(WideArgTys, IsAcceptableTyForREVEC))
+ return InstructionCost::getInvalid();
+ const unsigned NumSrcs = WideArgTys.size() - 1;
+ return InstructionCost(NumSrcs);
+ }
+ default:
+ // Only allow REVEC of NEON intrinsics when the types are at most scalable
+ // 128-bit vectors. This avoids crashes when forcing e.g. VF = vscale x 2.
+ return (IsAcceptableTyForREVEC(WideRetTy) &&
+ all_of(WideArgTys, IsAcceptableTyForREVEC))
+ ? InstructionCost(1)
+ : InstructionCost::getInvalid();
+ }
+}
diff --git a/llvm/lib/Target/AArch64/AArch64TargetRevectorizeInfo.h b/llvm/lib/Target/AArch64/AArch64TargetRevectorizeInfo.h
new file mode 100644
index 0000000000000..2f2654fd77f7b
--- /dev/null
+++ b/llvm/lib/Target/AArch64/AArch64TargetRevectorizeInfo.h
@@ -0,0 +1,43 @@
+//===- AArch64TargetRevectorizeInfo.h - AArch64 TRVI -----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AARCH64_AARCH64TARGETREVECTORIZEINFO_H
+#define LLVM_LIB_TARGET_AARCH64_AARCH64TARGETREVECTORIZEINFO_H
+
+#include "llvm/Analysis/TargetRevectorizeInfo.h"
+
+namespace llvm {
+
+class AArch64Subtarget;
+
+class AArch64RevectorizeInfoImpl final : public TargetRevectorizeInfoImplBase {
+ // The target machine owns its subtargets and outlives function analyses.
+ const AArch64Subtarget &ST;
+
+public:
+ AArch64RevectorizeInfoImpl(const TargetTransformInfo &TTI,
+ const AArch64Subtarget &ST)
+ : TargetRevectorizeInfoImplBase(TTI), ST(ST) {}
+
+ bool isTargetIntrinsicVectorizable(Intrinsic::ID ID) const override;
+
+ Instruction *vectorizeTargetIntrinsic(Intrinsic::ID VectorIID,
+ ArrayRef<Type *> TysForDecl,
+ ArrayRef<Value *> WideArgs,
+ ElementCount VF,
+ IRBuilderBase &Builder) const override;
+
+ InstructionCost
+ getTargetIntrinsicVectorizationCost(Intrinsic::ID FromID, Type *WideRetTy,
+ ArrayRef<Type *> WideArgTys,
+ ElementCount VF) const override;
+};
+
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_AARCH64_AARCH64TARGETREVECTORIZEINFO_H
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index b6a5a2443997c..aca818c510ad5 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -10,6 +10,7 @@
#include "AArch64ExpandImm.h"
#include "AArch64PerfectShuffle.h"
#include "AArch64SMEAttributes.h"
+#include "AArch64TargetRevectorizeInfo.h"
#include "MCTargetDesc/AArch64AddressingModes.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/Analysis/LoopInfo.h"
@@ -33,74 +34,6 @@ using namespace llvm;
using namespace llvm::PatternMatch;
#define DEBUG_TYPE "aarch64tti"
-
-// Define the struct for the generated intrinsic mapping table
-namespace {
-struct NEONToSVEIntrinsicMapping {
- StringRef NEONIntrinsic;
- StringRef SVEIntrinsic;
- StringRef ArgMappings; // Comma-separated string: "T,Z,A:0,A:1"
- bool isCustom() const { return SVEIntrinsic.empty() || ArgMappings.empty(); }
-};
-
-/// Specifies how to build an argument for an SVE intrinsic, possibly from
-/// another argument of an equivalent NEON intrinsic.
-class ArgMapping {
-public:
- enum MappingKind { True, Zero, Poison, Arg, Imm };
-
- ArgMapping(StringRef Spec) {
- if (Spec == "T") {
- Kind = True;
- } else if (Spec == "Z") {
- Kind = Zero;
- } else if (Spec == "P") {
- Kind = Poison;
- } else if (Spec.starts_with("A:")) {
- Kind = Arg;
- unsigned Idx;
- if (Spec.substr(2).getAsInteger(10, Idx))
- llvm_unreachable("Invalid argument index in mapping!");
- Index = Idx;
- } else if (Spec.starts_with("I:")) {
- Kind = Imm;
- unsigned Idx;
- if (Spec.substr(2).getAsInteger(10, Idx))
- llvm_unreachable("Invalid argument index in mapping!");
- Index = Idx;
- } else {
- llvm_unreachable("Unknown argument mapping kind!");
- }
- }
-
- MappingKind getKind() const { return Kind; }
-
- unsigned getArgIndex() const {
- assert((Kind == Arg || Kind == Imm) &&
- "getArgIndex called on non-Arg mapping");
- return *Index;
- }
-
-private:
- MappingKind Kind;
- std::optional<unsigned> Index;
-};
-
-// Parse comma-separated argument mapping string into ArgMapping objects
-static SmallVector<ArgMapping, 4> parseArgMappings(StringRef ArgMappingsStr) {
- SmallVector<ArgMapping, 4> Result;
- SmallVector<StringRef, 4> Specs;
- ArgMappingsStr.split(Specs, ',');
- for (StringRef Spec : Specs)
- Result.emplace_back(Spec);
- return Result;
-}
-} // end anonymous namespace
-
-// Include generated vector intrinsic mapping table
-#define GET_IntrinsicMappingTable_IMPL
-#include "AArch64GenVectorIntrinsicMappings.inc"
-
static cl::opt<bool> EnableFalkorHWPFUnrollFix("enable-falkor-hwpf-unroll-fix",
cl::init(true), cl::Hidden);
@@ -696,11 +629,6 @@ AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
ICA.getID() != Intrinsic::masked_store)
return InstructionCost::getInvalid();
- auto IsAcceptableTyForREVEC = [](const Type *Ty) {
- return Ty->getPrimitiveSizeInBits().getKnownMinValue() <=
- AArch64::SVEBitsPerBlock;
- };
-
switch (ICA.getID()) {
case Intrinsic::experimental_vector_histogram_add: {
InstructionCost HistCost = getHistogramCost(ST, ICA);
@@ -1301,46 +1229,7 @@ AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
? InstructionCost(1)
: InstructionCost::getInvalid();
}
- case Intrinsic::aarch64_neon_sshl:
- case Intrinsic::aarch64_neon_ushl: {
- // Re-vectorising NEON's non-saturating non-rounding bi-directional shl
- // requires the use of two SVE bi-directional shl: one for the non-rounding
- // part and one for the non-saturating part. (+masking)
- if (!IsAcceptableTyForREVEC(RetTy))
- return InstructionCost::getInvalid();
- return RetTy->isScalableTy() ? InstructionCost(2) : InstructionCost(1);
- }
- case Intrinsic::aarch64_neon_addp:
- case Intrinsic::aarch64_neon_faddp: {
- // Re-vectorising NEON's ADDP requires prior deinterleaving because SVE does
- // not have have a ADDPQ variant working like NEON within quads.
- // Re-vectorising 64-bit NEON requires even more instructions.
- if (!IsAcceptableTyForREVEC(RetTy))
- return InstructionCost::getInvalid();
- return RetTy->isScalableTy() ? InstructionCost(2) : InstructionCost(1);
- }
- case Intrinsic::aarch64_neon_tbl1:
- case Intrinsic::aarch64_neon_tbl2:
- case Intrinsic::aarch64_neon_tbl3:
- case Intrinsic::aarch64_neon_tbl4: {
- assert(ICA.getArgs().size() >= 2 &&
- "Tbl needs at least one source and one mask");
- if (!IsAcceptableTyForREVEC(RetTy) ||
- !all_of(ICA.getArgTypes(), IsAcceptableTyForREVEC))
- return InstructionCost::getInvalid();
- const unsigned NumSrcs = ICA.getArgs().size() - 1;
- return RetTy->isScalableTy() ? InstructionCost(NumSrcs)
- : InstructionCost(1);
- }
default:
- // Only allow REVEC of NEON intrinsics when the types are at most scalable
- // 128-bit vectors. This avoids crashes when forcing e.g. VF = vscale x 2.
- if (isTargetIntrinsicVectorizable(ICA.getID()) && RetTy->isScalableTy()) {
- return (IsAcceptableTyForREVEC(RetTy) &&
- all_of(ICA.getArgTypes(), IsAcceptableTyForREVEC))
- ? InstructionCost(1)
- : InstructionCost::getInvalid();
- }
break;
}
return BaseT::getIntrinsicInstrCost(ICA, CostKind);
@@ -5925,300 +5814,6 @@ bool AArch64TTIImpl::isTargetIntrinsicWithOverloadTypeAtArg(Intrinsic::ID ID,
}
}
-SmallVector<Type *, 2>
-AArch64TTIImpl::computeTysForDecl(Intrinsic::ID ID, Type *RetTy,
- ArrayRef<Value *> Args) const {
- SmallVector<Type *, 2> Tys;
- if (isTargetIntrinsicWithOverloadTypeAtArg(ID, -1))
- Tys.push_back(RetTy);
- for (auto [ArgIdx, Arg] : enumerate(Args))
- if (isTargetIntrinsicWithOverloadTypeAtArg(ID, ArgIdx))
- Tys.push_back(Arg->getType());
- return Tys;
-}
-
-bool AArch64TTIImpl::isTargetIntrinsicVectorizable(Intrinsic::ID ID) const {
-
- // Need SVE2.1 to consider revectorisation of NEON intrinsics
- if (!ST->hasSVE2p1())
- return false;
-
- return lookupNEONToSVEMappingByNEONIntrinsic(Intrinsic::getBaseName(ID));
-}
-
-/// Generate a segmented UZP shuffle that can later be selected to UZPQ.
-/// \pre Src0 and Src1 are scalable vectors
-Instruction *getSegmentedUZP(Value *Src0, Value *Src1, bool EvenElts,
- IRBuilderBase &Builder) {
- assert(isa<ScalableVectorType>(Src0->getType()));
- auto *VTy = cast<ScalableVectorType>(Src0->getType());
- const unsigned SegmentSize = VTy->getMinNumElements();
- SmallVector<int, 8> Mask(SegmentSize);
- transform(seq<int>(SegmentSize), Mask.begin(),
- [EvenElts](int Idx) { return EvenElts ? Idx * 2 : Idx * 2 + 1; });
- return Builder.CreateSegmentedShuffleVector(Src0, Src1, Mask);
-}
-
-/// Concatenate the even words of \p Src with its odd words within each quad.
-/// I.e. generate UZPQ1 Src.w,, (EXT Src, 4).w
-Instruction *concatEvenThenOddWordsWithinQuads(Value *Src,
- IRBuilderBase &Builder,
- bool ConcatWords = false) {
- assert(Src->getType()->getPrimitiveSizeInBits().getKnownMinValue() ==
- AArch64::SVEBitsPerBlock &&
- "Expected legal SVE vector.");
- assert(!Src->getType()->getScalarType()->isIntegerTy(32) &&
- "Unexpected nxv4i32 input");
- Type *OrigTy = Src->getType();
- Src = Builder.CreateBitCast(Src,
- ScalableVectorType::get(Builder.getInt32Ty(), 4));
- Value *ShiftOddToEven =
- Builder.CreateVectorSpliceLeft(Src, PoisonValue::get(Src->getType()), 1);
- Instruction *Res =
- getSegmentedUZP(Src, ShiftOddToEven, /*EvenElts=*/true, Builder);
- return cast<Instruction>(Builder.CreateBitCast(Res, OrigTy));
-}
-
-Instruction *AArch64TTIImpl::vectorizeTargetIntrinsic(
- Intrinsic::ID VectorIID, ArrayRef<Type *> TysForDecl,
- ArrayRef<Value *> WideArgs, IRBuilderBase &Builder,
- const Instruction &OrigInst) const {
-
- // Look up the mapping in the generated table
- StringRef NEONIntrinsicName = Intrinsic::getBaseName(VectorIID);
- const NEONToSVEIntrinsicMapping *Mapping =
- lookupNEONToSVEMappingByNEONIntrinsic(NEONIntrinsicName);
- if (!Mapping)
- llvm_unreachable("Unimplemented intrinsic vectorisation!");
-
- auto IsScaledNEONOrScalar = [](const Value *V) -> bool {
- if (V->getType()->isIntegerTy())
- return true;
- auto *SVTy = dyn_cast<ScalableVectorType>(V->getType());
- if (!SVTy)
- return false;
- unsigned FixedBits = SVTy->getMinNumElements() *
- SVTy->getElementType()->getScalarSizeInBits();
- return FixedBits == AArch64::SVEBitsPerBlock ||
- FixedBits == (AArch64::SVEBitsPerBlock / 2);
- };
- auto ToSVETy = [&](Type *Ty) -> Type * {
- if (!Ty->isVectorTy())
- return Ty;
- auto *SVTy = cast<ScalableVectorType>(Ty);
- unsigned FixedBits = SVTy->getMinNumElements() *
- SVTy->getElementType()->getScalarSizeInBits();
- return toVectorTy(
- Ty, ElementCount::getFixed(AArch64::SVEBitsPerBlock / FixedBits));
- };
-
- // For scaled 64-bit vectors, interleave/deinterleave even elements to makes
- // them valid SVE types. This plays nicely for both int and fp types later
- // during ISel.
- auto ToSVEVal = [&](Value *V) -> Value * {
- Type *LegalSVETy = ToSVETy(V->getType());
- if (LegalSVETy == V->getType())
- return V;
- return Builder.CreateVectorInterleave({V, PoisonValue::get(V->getType())});
- };
- auto FromSVEInst = [&](Instruction *I, Type *ExpectedTy) -> Instruction * {
- if (ExpectedTy == I->getType())
- return I;
- auto *RevecTy = cast<VectorType>(I->getType());
- auto *ExpectedEltTy = cast<VectorType>(ExpectedTy)->getElementType();
- if (RevecTy->getElementType() != ExpectedEltTy) {
- assert(RevecTy->getElementType()->isIntegerTy(1) &&
- "Only expected SVE predicate to be turned into NEON vector.");
- Type *NeonQuadPredTy =
- VectorType::get(ExpectedEltTy, RevecTy->getElementCount());
- I = cast<Instruction>(Builder.CreateZExt(I, NeonQuadPredTy));
- }
- if (ExpectedTy == I->getType())
- return I;
- auto *Deinterleave = Builder.CreateIntrinsic(
- Intrinsic::vector_deinterleave2, I->getType(), I);
- return cast<Instruction>(Builder.CreateExtractValue(Deinterleave, {0U}));
- };
-
- auto Zero = [&](Type *Ty) { return Constant::getNullValue(ToSVETy(Ty)); };
- auto Poison = [&](Type *Ty) { return PoisonValue::get(ToSVETy(Ty)); };
- auto True = [&](Type *Ty) {
- auto *VectorTy = cast<VectorType>(ToSVETy(Ty));
- // SVE intrinsics that narrow the element type have a predicate type with
- // half as many elements as their return type because every odd lane is
- // zeroed/undefined.
- bool IsNarrowing = any_of(WideArgs, [Ty](const Value *V) {
- return V->getType()->getScalarSizeInBits() > Ty->getScalarSizeInBits();
- });
- if (IsNarrowing)
- VectorTy = VectorType::getOneNthElementsVectorType(VectorTy, 2);
- auto *PredTy = VectorType::get(IntegerType::get(Ty->getContext(), 1),
- VectorTy->getElementCount());
- return Constant::getAllOnesValue(PredTy);
- };
- auto Arg = [&](unsigned ArgIdx) {
- assert(ArgIdx < WideArgs.size() && "Argument index out of range!");
- Value *V = WideArgs[ArgIdx];
- return ToSVEVal(V);
- };
- auto Imm = [&](unsigned ArgIdx) {
- assert(ArgIdx < WideArgs.size() && "Argument index out of range!");
- Value *V = WideArgs[ArgIdx];
- assert(isa<Constant>(V) && cast<Constant>(V)->getSplatValue());
- return cast<Constant>(V)->getSplatValue();
- };
-
- assert(all_of(WideArgs, IsScaledNEONOrScalar) &&
- "Expected SVE-compatible values.");
-
- // Custom lower some intrinsics.
- switch (VectorIID) {
- case Intrinsic::aarch64_neon_uaddlp:
- case Intrinsic::aarch64_neon_saddlp: {
- // ADDLP works with adjacent elements: scaled 64-bit NEON inputs cannot be
- // legalised to SVE by interleaving. Instead, extend and use a normal ADDP.
- Type *SrcTy = WideArgs[0]->getType();
- if (SrcTy->getPrimitiveSizeInBits().getKnownMinValue() != 64U)
- break;
-
- Type *DstTy = TysForDecl[0];
- Type *WideSrcTy = ToSVETy(DstTy);
- const bool IsSigned = VectorIID == Intrinsic::aarch64_neon_saddlp;
- Value *Src = IsSigned ? Builder.CreateSExt(WideArgs[0], WideSrcTy)
- : Builder.CreateZExt(WideArgs[0], WideSrcTy);
- return FromSVEInst(Builder.CreateIntrinsic(Intrinsic::aarch64_sve_addp,
- {WideSrcTy},
- {True(WideSrcTy), Src, Src}),
- DstTy);
- }
- case Intrinsic::aarch64_neon_addp:
- case Intrinsic::aarch64_neon_faddp: {
- Type *Ty = TysForDecl[0];
- const bool IsInt = VectorIID == Intrinsic::aarch64_neon_addp;
- // ADDP works with adjacent elements: scaled 64-bit NEON inputs cannot be
- // legalised to SVE by interleaving with undef elements.
- // Instead, deinterleave and use a normal ADD.
-
- // But if the initial NEON vector contained a single pair, we can
- // directly use SVE's ADDP without prior UZPQ de-interleaving.
- unsigned NeonVecSize = OrigInst.getType()->getPrimitiveSizeInBits();
- unsigned PairSize = OrigInst.getType()->getScalarSizeInBits() * 2;
- unsigned NumPairsPerSrc = NeonVecSize / PairSize;
- if (Ty == ToSVETy(Ty) && NumPairsPerSrc == 1) {
- Intrinsic::ID IID =
- IsInt ? Intrinsic::aarch64_sve_addp : Intrinsic::aarch64_sve_faddp;
- return Builder.CreateIntrinsic(IID, {Ty},
- {True(Ty), WideArgs[0], WideArgs[1]});
- }
-
- // Within each segment, ensure we have all the pairwise adds from Args0
- // followed by those of Add1.
- Value *EvenElts =
- getSegmentedUZP(WideArgs[0], WideArgs[1], /*EvenElts=*/true, Builder);
- Value *OddElts =
- getSegmentedUZP(WideArgs[0], WideArgs[1], /*EvenElts=*/false, Builder);
- auto *Res =
- cast<Instruction>(IsInt ? Builder.CreateAdd(EvenElts, OddElts)
- : Builder.CreateFAdd(EvenElts, OddElts));
-
- // The segment type might not match the original type (VF = vscale x 2),
- // i.e. the original type is 64-bit but our segments are 128-bit.
- // Then, ensure that within each 64-bit segment, the lo 32-bit are pairwise
- // adds from Arg0 and the hi 32-bit are pairwise adds from Arg1.
- if (NeonVecSize != Ty->getPrimitiveSizeInBits().getKnownMinValue()) {
- assert(NeonVecSize == 64U && Ty == ToSVETy(Ty) &&
- "Expected REVEC from NEON 64-bit to legal SVE");
- Res = concatEvenThenOddWordsWithinQuads(Res, Builder);
- }
- return Res;
- }
- case Intrinsic::aarch64_neon_sshl:
- case Intrinsic::aarch64_neon_ushl: {
- // SVE does not have a "plain" non-saturating non-rounding shl.
- // This means we need to use [su]rshl for positive shift amounts
- // and [su]qshl for negative shift amounts (effectively a shift right).
- const bool IsSignedShift = VectorIID == Intrinsic::aarch64_neon_sshl;
- Intrinsic::ID SHLIID = IsSignedShift ? Intrinsic::aarch64_sve_srshl
- : Intrinsic::aarch64_sve_urshl;
- Intrinsic::ID SHRIID = IsSignedShift ? Intrinsic::aarch64_sve_sqshl
- : Intrinsic::aarch64_sve_uqshl;
- Type *ArgTy = TysForDecl[0];
- Value *Arg0 = Arg(0);
- Value *Arg1 = Arg(1);
- Value *SHLMask = Builder.CreateICmpSLE(
- Arg1, ConstantInt::get(ToSVETy(ArgTy), 0), "shl.mask");
- Value *SHL = Builder.CreateIntrinsic(SHLIID, {ToSVETy(ArgTy)},
- {SHLMask, Arg0, Arg1});
- Value *SHRMask =
- Builder.CreateIntrinsic(Intrinsic::ctlz, {SHLMask->getType()},
- {SHLMask, Builder.getTrue()}, {}, "shr.mask");
- return FromSVEInst(Builder.CreateIntrinsic(SHRIID, {ToSVETy(ArgTy)},
- {SHRMask, SHL, Arg1}, {},
- "wide.bidir.shl"),
- ArgTy);
- }
- case Intrinsic::aarch64_neon_tbl1:
- case Intrinsic::aarch64_neon_tbl2:
- case Intrinsic::aarch64_neon_tbl3:
- case Intrinsic::aarch64_neon_tbl4: {
- Type *Ty = TysForDecl[0];
- assert(OrigInst.getType()->getPrimitiveSizeInBits() ==
- Ty->getPrimitiveSizeInBits().getKnownMinValue() &&
- "Unexpected vscale x 2 REVEC");
- Value *Src0 = Arg(0);
- Value *Mask = Arg(WideArgs.size() - 1);
- Instruction *Res = Builder.CreateIntrinsic(Intrinsic::aarch64_sve_tblq,
- {Src0->getType()}, {Src0, Mask});
- for (unsigned SrcIdx = 1; SrcIdx < WideArgs.size() - 1; ++SrcIdx) {
- Value *Src = Arg(SrcIdx);
- Mask = Builder.CreateSub(Mask, ConstantInt::get(Mask->getType(), 16));
- Res = Builder.CreateIntrinsic(Intrinsic::aarch64_sve_tbxq,
- {Src->getType()}, {Res, Src, Mask});
- }
- return FromSVEInst(Res, Ty);
- }
- }
- assert(!Mapping->isCustom() &&
- "NEON intrinsic was expected to be custom-lowered to SVE");
-
- // Note: Contrary to NEON, SVE intrinsics usually have less overload types
- // because argument types are inferred.
- Type *Ty = TysForDecl[0];
- SmallVector<Value *, 4> SVEArgs;
-
- // Parse and apply argument mappings from the table
- SmallVector<ArgMapping, 4> ArgMappings =
- parseArgMappings(Mapping->ArgMappings);
- for (const ArgMapping &AM : ArgMappings) {
- switch (AM.getKind()) {
- case ArgMapping::True:
- SVEArgs.push_back(True(Ty));
- break;
- case ArgMapping::Zero:
- SVEArgs.push_back(Zero(Ty));
- break;
- case ArgMapping::Poison:
- SVEArgs.push_back(Poison(Ty));
- break;
- case ArgMapping::Arg:
- SVEArgs.push_back(Arg(AM.getArgIndex()));
- break;
- case ArgMapping::Imm:
- assert(AM.getArgIndex() < WideArgs.size() &&
- "Argument index out of range!");
- SVEArgs.push_back(Imm(AM.getArgIndex()));
- break;
- }
- }
-
- Intrinsic::ID SVEIntrinsicID =
- Intrinsic::lookupIntrinsicID(Mapping->SVEIntrinsic);
- auto TysForSVEDecl = computeTysForDecl(SVEIntrinsicID, ToSVETy(Ty), SVEArgs);
- assert(SVEIntrinsicID != Intrinsic::not_intrinsic);
- return FromSVEInst(
- Builder.CreateIntrinsic(SVEIntrinsicID, TysForSVEDecl, SVEArgs), Ty);
-}
-
/// See if \p I should be considered for address type promotion. We check if \p
/// I is a sext with right type and used in memory accesses. If it used in a
/// "complex" getelementptr, we allow it to be promoted without finding other
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index c3a2e687de19e..d727a53efdbe8 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -307,18 +307,6 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
bool isTargetIntrinsicWithOverloadTypeAtArg(Intrinsic::ID ID,
int OpdIdx) const override;
- /// Compute the list of Types required to get the right declaration of
- /// the overloaded \p ID intrinsic.
- SmallVector<Type *, 2> computeTysForDecl(Intrinsic::ID ID, Type *RetTy,
- ArrayRef<Value *> Args) const;
-
- bool isTargetIntrinsicVectorizable(Intrinsic::ID ID) const override;
-
- Instruction *
- vectorizeTargetIntrinsic(Intrinsic::ID VectorIID, ArrayRef<Type *> TysForDecl,
- ArrayRef<Value *> WideArgs, IRBuilderBase &Builder,
- const Instruction &OrigInst) const override;
-
bool isElementTypeLegalForScalableVector(Type *Ty) const override {
if (Ty->isPointerTy())
return true;
diff --git a/llvm/lib/Target/AArch64/CMakeLists.txt b/llvm/lib/Target/AArch64/CMakeLists.txt
index 445e46589ea99..5dbc7c1abcaa9 100644
--- a/llvm/lib/Target/AArch64/CMakeLists.txt
+++ b/llvm/lib/Target/AArch64/CMakeLists.txt
@@ -89,6 +89,7 @@ add_llvm_target(AArch64CodeGen
AArch64Subtarget.cpp
AArch64TargetMachine.cpp
AArch64TargetObjectFile.cpp
+ AArch64TargetRevectorizeInfo.cpp
AArch64TargetTransformInfo.cpp
SMEPeepholeOpt.cpp
SVEIntrinsicOpts.cpp
diff --git a/llvm/lib/Target/TargetMachine.cpp b/llvm/lib/Target/TargetMachine.cpp
index 9243cdad43e1e..10f82e9c17dbd 100644
--- a/llvm/lib/Target/TargetMachine.cpp
+++ b/llvm/lib/Target/TargetMachine.cpp
@@ -11,6 +11,7 @@
//===----------------------------------------------------------------------===//
#include "llvm/Target/TargetMachine.h"
+#include "llvm/Analysis/TargetRevectorizeInfo.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalValue.h"
@@ -289,6 +290,12 @@ TargetMachine::getTargetTransformInfo(const Function &F) const {
return TargetTransformInfo(F.getDataLayout());
}
+TargetRevectorizeInfo
+TargetMachine::getTargetRevectorizeInfo(const Function &,
+ const TargetTransformInfo &TTI) const {
+ return TargetRevectorizeInfo(TTI);
+}
+
void TargetMachine::getNameWithPrefix(SmallVectorImpl<char> &Name,
const GlobalValue *GV, Mangler &Mang,
bool MayAlwaysUsePrivate) const {
@@ -320,6 +327,13 @@ TargetIRAnalysis TargetMachine::getTargetIRAnalysis() const {
[this](const Function &F) { return this->getTargetTransformInfo(F); });
}
+TargetRevectorizeWrapper TargetMachine::getTargetRevectorizeAnalysis() const {
+ return TargetRevectorizeWrapper(
+ [this](const Function &F, const TargetTransformInfo &TTI) {
+ return getTargetRevectorizeInfo(F, TTI);
+ });
+}
+
std::pair<int, int> TargetMachine::parseBinutilsVersion(StringRef Version) {
if (Version == "none")
return {INT_MAX, INT_MAX}; // Make binutilsIsAtLeast() return true.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 3f799964d5221..0b63f20e10fc8 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -921,7 +921,7 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
// * Have a vector version available.
auto *CI = dyn_cast<CallInst>(&I);
- if (CI && !getVectorIntrinsicIDForCall(CI, TLI, TTI) &&
+ if (CI && !getVectorIntrinsicIDForCall(CI, TLI, TRVI) &&
!(CI->getCalledFunction() && TLI &&
(!VFDatabase::getMappings(*CI).empty() || isTLIScalarize(*TLI, *CI)))) {
// If the call is a recognized math libary call, it is likely that
@@ -955,7 +955,7 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
// them to be vectorized (i.e. loop invariant).
if (CI) {
auto *SE = PSE.getSE();
- Intrinsic::ID IntrinID = getVectorIntrinsicIDForCall(CI, TLI, TTI);
+ Intrinsic::ID IntrinID = getVectorIntrinsicIDForCall(CI, TLI, TRVI);
for (unsigned Idx = 0; Idx < CI->arg_size(); ++Idx)
if (isVectorIntrinsicWithScalarOpAtArg(IntrinID, Idx, TTI)) {
if (!SE->isLoopInvariant(PSE.getSCEV(CI->getOperand(Idx)), TheLoop)) {
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 20a98fb343448..656507e4f549c 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -35,6 +35,8 @@ class GeneratedRTChecks;
namespace llvm {
+class TargetRevectorizeInfo;
+
class LoopInfo;
class DominatorTree;
class LoopVectorizationLegality;
@@ -801,6 +803,9 @@ class LoopVectorizationPlanner {
/// Target Transform Info.
const TargetTransformInfo &TTI;
+ /// Target Revectorization Info.
+ const TargetRevectorizeInfo &TRVI;
+
/// The legality analysis.
LoopVectorizationLegality *Legal;
@@ -846,12 +851,14 @@ class LoopVectorizationPlanner {
public:
LoopVectorizationPlanner(
Loop *L, LoopInfo *LI, DominatorTree *DT, const TargetLibraryInfo *TLI,
- const TargetTransformInfo &TTI, LoopVectorizationLegality *Legal,
- LoopVectorizationCostModel &CM, VFSelectionContext &Config,
- InterleavedAccessInfo &IAI, PredicatedScalarEvolution &PSE,
- const LoopVectorizeHints &Hints, OptimizationRemarkEmitter *ORE)
- : OrigLoop(L), LI(LI), DT(DT), TLI(TLI), TTI(TTI), Legal(Legal), CM(CM),
- Config(Config), IAI(IAI), PSE(PSE), Hints(Hints), ORE(ORE) {}
+ const TargetTransformInfo &TTI, const TargetRevectorizeInfo &TRVI,
+ LoopVectorizationLegality *Legal, LoopVectorizationCostModel &CM,
+ VFSelectionContext &Config, InterleavedAccessInfo &IAI,
+ PredicatedScalarEvolution &PSE, const LoopVectorizeHints &Hints,
+ OptimizationRemarkEmitter *ORE)
+ : OrigLoop(L), LI(LI), DT(DT), TLI(TLI), TTI(TTI), TRVI(TRVI),
+ Legal(Legal), CM(CM), Config(Config), IAI(IAI), PSE(PSE), Hints(Hints),
+ ORE(ORE) {}
/// Build VPlans for the specified \p UserVF and \p UserIC if they are
/// non-zero or all applicable candidate VFs otherwise. If vectorization and
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index a14e0fd7a6de5..f20642f83cc33 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -95,6 +95,7 @@
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
#include "llvm/Analysis/ScalarEvolutionPatternMatch.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/Analysis/TargetRevectorizeInfo.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/Analysis/VectorUtils.h"
@@ -788,6 +789,7 @@ class LoopVectorizationCostModel {
PredicatedScalarEvolution &PSE, LoopInfo *LI,
LoopVectorizationLegality *Legal,
const TargetTransformInfo &TTI,
+ const TargetRevectorizeInfo &TRVI,
const TargetLibraryInfo *TLI, AssumptionCache *AC,
OptimizationRemarkEmitter *ORE,
std::function<BlockFrequencyInfo &()> GetBFI,
@@ -795,7 +797,7 @@ class LoopVectorizationCostModel {
InterleavedAccessInfo &IAI,
VFSelectionContext &Config)
: Config(Config), EpilogueLoweringStatus(SEL), TheLoop(L), PSE(PSE),
- LI(LI), Legal(Legal), TTI(TTI), TLI(TLI), AC(AC), ORE(ORE),
+ LI(LI), Legal(Legal), TTI(TTI), TRVI(TRVI), TLI(TLI), AC(AC), ORE(ORE),
GetBFI(GetBFI), TheFunction(F), Hints(Hints), InterleaveInfo(IAI) {}
/// \return An upper bound for the vectorization factors (both fixed and
@@ -1504,6 +1506,9 @@ class LoopVectorizationCostModel {
/// Vector target information.
const TargetTransformInfo &TTI;
+ /// Target revectorization information.
+ const TargetRevectorizeInfo &TRVI;
+
/// Target Library Info.
const TargetLibraryInfo *TLI;
@@ -2146,7 +2151,7 @@ LoopVectorizationCostModel::getVectorCallCost(CallInst *CI,
: ScalarCallCost * VF.getKnownMinValue() +
getScalarizationOverhead(CI, VF);
- if (getVectorIntrinsicIDForCall(CI, TLI, &TTI)) {
+ if (getVectorIntrinsicIDForCall(CI, TLI, &TRVI)) {
InstructionCost IntrinsicCost = getVectorIntrinsicCost(CI, VF);
return std::min(Cost, IntrinsicCost);
}
@@ -2162,7 +2167,7 @@ static Type *maybeVectorizeType(Type *Ty, ElementCount VF) {
InstructionCost
LoopVectorizationCostModel::getVectorIntrinsicCost(CallInst *CI,
ElementCount VF) const {
- Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI, &TTI);
+ Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI, &TRVI);
assert(ID && "Expected intrinsic call!");
Type *RetTy = maybeVectorizeType(CI->getType(), VF);
FastMathFlags FMF;
@@ -2419,7 +2424,7 @@ bool LoopVectorizationCostModel::isScalarWithPredication(Instruction *I,
return true;
auto *CI = cast<CallInst>(I);
// A vector intrinsic or library variant lowering avoids scalarization.
- return !getVectorIntrinsicIDForCall(CI, TLI, &TTI) &&
+ return !getVectorIntrinsicIDForCall(CI, TLI, &TRVI) &&
!hasVectorLibraryVariantFor(*CI, VF, isMaskRequired(CI), TLI);
}
case Instruction::Load:
@@ -3161,8 +3166,8 @@ void LoopVectorizationPlanner::emitInvalidCostRemarks(
if (VF.isScalar())
continue;
- VPCostContext CostCtx(CM.TTI, *CM.TLI, *Plan, CM, Config.CostKind, CM.PSE,
- OrigLoop);
+ VPCostContext CostCtx(CM.TTI, CM.TRVI, *CM.TLI, *Plan, CM,
+ Config.CostKind, CM.PSE, OrigLoop);
precomputeCosts(*Plan, VF, CostCtx);
auto Iter = vp_depth_first_deep(Plan->getVectorLoopRegion()->getEntry());
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Iter)) {
@@ -5823,8 +5828,8 @@ LoopVectorizationPlanner::precomputeCosts(VPlan &Plan, ElementCount VF,
InstructionCost LoopVectorizationPlanner::cost(VPlan &Plan, ElementCount VF,
VPRegisterUsage *RU) const {
- VPCostContext CostCtx(CM.TTI, *CM.TLI, Plan, CM, Config.CostKind, PSE,
- OrigLoop);
+ VPCostContext CostCtx(CM.TTI, CM.TRVI, *CM.TLI, Plan, CM, Config.CostKind,
+ PSE, OrigLoop);
InstructionCost Cost = precomputeCosts(Plan, VF, CostCtx);
// Now compute and add the VPlan-based cost.
@@ -6063,7 +6068,7 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
// Perform the actual loop transformation.
VPTransformState State(&TTI, BestVF, LI, DT, ILV.AC, ILV.Builder, &BestVPlan,
- OrigLoop->getParentLoop());
+ OrigLoop->getParentLoop(), &TRVI);
#ifdef EXPENSIVE_CHECKS
assert(DT->verify(DominatorTree::VerificationLevel::Fast));
@@ -6737,7 +6742,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
for (ElementCount VF : Range)
Plan->addVF(VF);
if (!VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(*Plan, *TLI,
- TTI))
+ TTI, &TRVI))
return nullptr;
VPlanTransforms::optimizeInductionLiveOutUsers(*Plan, PSE,
/*FoldTail=*/false);
@@ -6832,8 +6837,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
RUN_VPLAN_PASS(VPlanTransforms::createInLoopReductionRecipes, *Plan,
Range.Start);
- VPCostContext CostCtx(CM.TTI, *CM.TLI, *Plan, CM, Config.CostKind, CM.PSE,
- OrigLoop);
+ VPCostContext CostCtx(CM.TTI, CM.TRVI, *CM.TLI, *Plan, CM, Config.CostKind,
+ CM.PSE, OrigLoop);
RUN_VPLAN_PASS(VPlanTransforms::makeMemOpWideningDecisions, *Plan, Range,
RecipeBuilder);
@@ -7988,7 +7993,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// Check if it is legal to vectorize the loop.
LoopVectorizationRequirements Requirements;
- LoopVectorizationLegality LVL(L, PSE, DT, TTI, TLI, F, *LAIs, LI, ORE,
+ LoopVectorizationLegality LVL(L, PSE, DT, TTI, TRVI, TLI, F, *LAIs, LI, ORE,
&Requirements, &Hints, DB, AC,
/*AllowRuntimeSCEVChecks=*/!OptForSize, AA);
if (!LVL.canVectorize(EnableVPlanNativePath)) {
@@ -8121,11 +8126,11 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// Use the cost model.
VFSelectionContext Config(*TTI, &LVL, L, *F, PSE, DB, ORE, &Hints,
OptForSize);
- LoopVectorizationCostModel CM(SEL, L, PSE, LI, &LVL, *TTI, TLI, AC, ORE,
- GetBFI, F, &Hints, IAI, Config);
+ LoopVectorizationCostModel CM(SEL, L, PSE, LI, &LVL, *TTI, *TRVI, TLI, AC,
+ ORE, GetBFI, F, &Hints, IAI, Config);
// Use the planner for vectorization.
- LoopVectorizationPlanner LVP(L, LI, DT, TLI, *TTI, &LVL, CM, Config, IAI, PSE,
- Hints, ORE);
+ LoopVectorizationPlanner LVP(L, LI, DT, TLI, *TTI, *TRVI, &LVL, CM, Config,
+ IAI, PSE, Hints, ORE);
EpilogueLowering EpilogueTailLoweringStatus =
getEpilogueTailLowering(CM, L, ORE);
@@ -8192,8 +8197,8 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// Check if it is profitable to vectorize with runtime checks.
bool ForceVectorization =
Hints.getForce() == LoopVectorizeHints::FK_Enabled;
- VPCostContext CostCtx(CM.TTI, *CM.TLI, *BestPlanPtr, CM, Config.CostKind,
- CM.PSE, L);
+ VPCostContext CostCtx(CM.TTI, CM.TRVI, *CM.TLI, *BestPlanPtr, CM,
+ Config.CostKind, CM.PSE, L);
if (!ForceVectorization &&
!isOutsideLoopWorkProfitable(Checks, VF, L, PSE, CostCtx, *BestPlanPtr,
SEL, Config.getVScaleForTuning())) {
@@ -8518,6 +8523,7 @@ PreservedAnalyses LoopVectorizePass::run(Function &F,
return PreservedAnalyses::all();
SE = &AM.getResult<ScalarEvolutionAnalysis>(F);
TTI = &AM.getResult<TargetIRAnalysis>(F);
+ TRVI = &AM.getResult<TargetRevectorizeWrapper>(F);
DT = &AM.getResult<DominatorTreeAnalysis>(F);
TLI = &AM.getResult<TargetLibraryAnalysis>(F);
AC = &AM.getResult<AssumptionAnalysis>(F);
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 81198fb141924..7025491f02a33 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -274,9 +274,10 @@ VPTransformState::VPTransformState(const TargetTransformInfo *TTI,
ElementCount VF, LoopInfo *LI,
DominatorTree *DT, AssumptionCache *AC,
IRBuilderBase &Builder, VPlan *Plan,
- Loop *CurrentParentLoop)
- : TTI(TTI), VF(VF), CFG(DT), LI(LI), AC(AC), Builder(Builder), Plan(Plan),
- CurrentParentLoop(CurrentParentLoop), VPDT(*Plan) {}
+ Loop *CurrentParentLoop,
+ const TargetRevectorizeInfo *TRVI)
+ : TTI(TTI), TRVI(TRVI), VF(VF), CFG(DT), LI(LI), AC(AC), Builder(Builder),
+ Plan(Plan), CurrentParentLoop(CurrentParentLoop), VPDT(*Plan) {}
Value *VPTransformState::get(const VPValue *Def, const VPLane &Lane) {
if (isa<VPIRValue, VPSymbolicValue>(Def))
diff --git a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
index a7d2f69f6ab4e..d2ab6f8b12d87 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
@@ -28,6 +28,8 @@
namespace llvm {
+class TargetRevectorizeInfo;
+
class AssumptionCache;
class BasicBlock;
class CallInst;
@@ -198,11 +200,14 @@ class VPLane {
struct VPTransformState {
VPTransformState(const TargetTransformInfo *TTI, ElementCount VF,
LoopInfo *LI, DominatorTree *DT, AssumptionCache *AC,
- IRBuilderBase &Builder, VPlan *Plan,
- Loop *CurrentParentLoop);
+ IRBuilderBase &Builder, VPlan *Plan, Loop *CurrentParentLoop,
+ const TargetRevectorizeInfo *TRVI = nullptr);
/// Target Transform Info.
const TargetTransformInfo *TTI;
+ /// Target Revectorization Info.
+ const TargetRevectorizeInfo *TRVI;
+
/// The chosen Vectorization Factor of the loop being vectorized.
ElementCount VF;
@@ -332,6 +337,7 @@ struct VPTransformState {
/// Struct to hold various analysis needed for cost computations.
struct VPCostContext {
const TargetTransformInfo &TTI;
+ const TargetRevectorizeInfo &TRVI;
const TargetLibraryInfo &TLI;
LLVMContext &LLVMCtx;
LoopVectorizationCostModel &CM;
@@ -343,11 +349,12 @@ struct VPCostContext {
/// Number of predicated stores in the VPlan, computed on demand.
std::optional<unsigned> NumPredStores;
- VPCostContext(const TargetTransformInfo &TTI, const TargetLibraryInfo &TLI,
+ VPCostContext(const TargetTransformInfo &TTI,
+ const TargetRevectorizeInfo &TRVI, const TargetLibraryInfo &TLI,
const VPlan &Plan, LoopVectorizationCostModel &CM,
TargetTransformInfo::TargetCostKind CostKind,
PredicatedScalarEvolution &PSE, const Loop *L)
- : TTI(TTI), TLI(TLI), LLVMCtx(Plan.getContext()), CM(CM),
+ : TTI(TTI), TRVI(TRVI), TLI(TLI), LLVMCtx(Plan.getContext()), CM(CM),
CostKind(CostKind), PSE(PSE), L(L) {}
/// Return the cost for \p UI with \p VF using the legacy cost model as
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index dbf188bb7d899..d83586621859a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -24,6 +24,7 @@
#include "llvm/Analysis/IVDescriptors.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
+#include "llvm/Analysis/TargetRevectorizeInfo.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instruction.h"
@@ -2221,9 +2222,9 @@ Instruction *VPWidenIntrinsicRecipe::createVectorCall(VPTransformState &State) {
// Use vector version of the intrinsic.
Instruction *V = nullptr;
if (Intrinsic::isTargetIntrinsic(VectorIntrinsicID)) {
- V = State.TTI->vectorizeTargetIntrinsic(VectorIntrinsicID, TysForDecl, Args,
- State.Builder,
- *getUnderlyingInstr());
+ assert(State.TRVI && "missing target revectorization information");
+ V = State.TRVI->vectorizeTargetIntrinsic(VectorIntrinsicID, TysForDecl,
+ Args, State.VF, State.Builder);
assert(V && "Target did not vectorise intrinsic call.");
// TODO-REVEC: Properly retain metadata and flags somehow.
} else {
@@ -2288,6 +2289,11 @@ InstructionCost VPWidenIntrinsicRecipe::computeCallCost(
return toVectorTy(Op->getScalarType(), VF);
});
+ // REVEC: Query TRVI for vectorized target intrinsics instead of TTI.
+ if (Intrinsic::isTargetIntrinsic(ID))
+ return Ctx.TRVI.getTargetIntrinsicVectorizationCost(ID, RetTy, ParamTys,
+ VF);
+
// TODO: Rework TTI interface to avoid reliance on underlying IntrinsicInst.
IntrinsicCostAttributes CostAttrs(
ID, RetTy, Arguments, ParamTys, R.getFastMathFlags(),
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 300c3caa55e65..3ed855983f339 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -49,7 +49,8 @@ using namespace VPlanPatternMatch;
using namespace SCEVPatternMatch;
bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
- VPlan &Plan, const TargetLibraryInfo &TLI, const TargetTransformInfo &TTI) {
+ VPlan &Plan, const TargetLibraryInfo &TLI, const TargetTransformInfo &TTI,
+ const TargetRevectorizeInfo *TRVI) {
ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> RPOT(
Plan.getVectorLoopRegion());
@@ -91,7 +92,7 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
Ingredient.operands(), *VPI,
Ingredient.getDebugLoc(), GEP);
} else if (CallInst *CI = dyn_cast<CallInst>(Inst)) {
- Intrinsic::ID VectorID = getVectorIntrinsicIDForCall(CI, &TLI, &TTI);
+ Intrinsic::ID VectorID = getVectorIntrinsicIDForCall(CI, &TLI, TRVI);
if (VectorID == Intrinsic::not_intrinsic)
return false;
@@ -7047,7 +7048,7 @@ static CallWideningDecision decideCallWidening(VPInstruction &VPI,
VPI.getOperand(VPI.getNumOperandsWithoutMask() - 1)->getLiveInIRValue());
Type *ResultTy = VPI.getScalarType();
Intrinsic::ID ID =
- getVectorIntrinsicIDForCall(CI, &CostCtx.TLI, &CostCtx.TTI);
+ getVectorIntrinsicIDForCall(CI, &CostCtx.TLI, &CostCtx.TRVI);
bool MaskRequired = CostCtx.isMaskRequired(CI);
// Pseudo intrinsics (assume, lifetime, ...) are always scalarized.
@@ -7108,7 +7109,7 @@ void VPlanTransforms::makeCallWideningDecisions(VPlan &Plan, VFRange &Range,
switch (Decision.Kind) {
case CallWideningDecision::KindTy::Intrinsic: {
Intrinsic::ID ID =
- getVectorIntrinsicIDForCall(CI, &CostCtx.TLI, &CostCtx.TTI);
+ getVectorIntrinsicIDForCall(CI, &CostCtx.TLI, &CostCtx.TRVI);
Type *ResultTy = VPI->getScalarType();
Replacement = new VPWidenIntrinsicRecipe(*CI, ID, Ops, ResultTy, *VPI,
*VPI, VPI->getDebugLoc());
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index e07e56c471d86..7fd360d986788 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -33,6 +33,7 @@ class PHINode;
class ScalarEvolution;
class PredicatedScalarEvolution;
class TargetLibraryInfo;
+class TargetRevectorizeInfo;
class TargetTransformInfo;
class VPBuilder;
class VPRecipeBuilder;
@@ -214,10 +215,9 @@ struct VPlanTransforms {
/// Replaces the VPInstructions in \p Plan with corresponding
/// widen recipes. Returns false if any VPInstructions could not be converted
/// to a wide recipe if needed.
- LLVM_ABI_FOR_TEST static bool
- tryToConvertVPInstructionsToVPRecipes(VPlan &Plan,
- const TargetLibraryInfo &TLI,
- const TargetTransformInfo &TTI);
+ LLVM_ABI_FOR_TEST static bool tryToConvertVPInstructionsToVPRecipes(
+ VPlan &Plan, const TargetLibraryInfo &TLI, const TargetTransformInfo &TTI,
+ const TargetRevectorizeInfo *TRVI = nullptr);
/// Try to legalize reductions with multiple in-loop uses. Currently only
/// strict and non-strict min/max reductions used by FindLastIV reductions are
More information about the cfe-commits
mailing list