[llvm] 33ffce5 - [AArch64][SVE] Remove LD1/ST1 dependency on llvm.masked.load/store
Kerry McLaughlin via llvm-commits
llvm-commits at lists.llvm.org
Mon Apr 20 03:20:43 PDT 2020
Author: Kerry McLaughlin
Date: 2020-04-20T11:08:11+01:00
New Revision: 33ffce5414ec47e81016c70abdf0f0a7bff50076
URL: https://github.com/llvm/llvm-project/commit/33ffce5414ec47e81016c70abdf0f0a7bff50076
DIFF: https://github.com/llvm/llvm-project/commit/33ffce5414ec47e81016c70abdf0f0a7bff50076.diff
LOG: [AArch64][SVE] Remove LD1/ST1 dependency on llvm.masked.load/store
Summary:
The SVE masked load and store intrinsics introduced in D76688 rely on
common llvm.masked.load/store nodes. This patch creates new ISD nodes
for LD1(S) & ST1 to remove this dependency.
Additionally, this adds support for sign & zero extending
loads and truncating stores.
Reviewers: sdesmalen, efriedma, cameron.mcinally, c-rhodes, rengolin
Reviewed By: efriedma
Subscribers: tschuett, kristof.beyls, hiraditya, rkruppe, psnobl, danielkiss, andwar, cfe-commits, llvm-commits
Tags: #llvm
Differential Revision: https://reviews.llvm.org/D78204
Added:
llvm/test/CodeGen/AArch64/sve-intrinsics-ld1.ll
llvm/test/CodeGen/AArch64/sve-intrinsics-st1.ll
Modified:
llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
llvm/lib/Target/AArch64/AArch64ISelLowering.h
llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
Removed:
llvm/test/CodeGen/AArch64/sve-intrinsics-ldst1.ll
################################################################################
diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index 6e7896e9497e..bcf7436dc3a9 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -4646,9 +4646,13 @@ static EVT getMemVTFromNode(LLVMContext &Ctx, SDNode *Root) {
// For custom ISD nodes, we have to look at them individually to extract the
// type of the data moved to/from memory.
switch (Opcode) {
+ case AArch64ISD::LD1:
+ case AArch64ISD::LD1S:
case AArch64ISD::LDNF1:
case AArch64ISD::LDNF1S:
return cast<VTSDNode>(Root->getOperand(3))->getVT();
+ case AArch64ISD::ST1:
+ return cast<VTSDNode>(Root->getOperand(4))->getVT();
default:
break;
}
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 8ceb970efcf4..da96ecaaae25 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1415,6 +1415,8 @@ const char *AArch64TargetLowering::getTargetNodeName(unsigned Opcode) const {
case AArch64ISD::INSR: return "AArch64ISD::INSR";
case AArch64ISD::PTEST: return "AArch64ISD::PTEST";
case AArch64ISD::PTRUE: return "AArch64ISD::PTRUE";
+ case AArch64ISD::LD1: return "AArch64ISD::LD1";
+ case AArch64ISD::LD1S: return "AArch64ISD::LD1S";
case AArch64ISD::LDNF1: return "AArch64ISD::LDNF1";
case AArch64ISD::LDNF1S: return "AArch64ISD::LDNF1S";
case AArch64ISD::LDFF1: return "AArch64ISD::LDFF1";
@@ -1454,6 +1456,8 @@ const char *AArch64TargetLowering::getTargetNodeName(unsigned Opcode) const {
case AArch64ISD::GLDNT1_INDEX: return "AArch64ISD::GLDNT1_INDEX";
case AArch64ISD::GLDNT1S: return "AArch64ISD::GLDNT1S";
+ case AArch64ISD::ST1: return "AArch64ISD::ST1";
+
case AArch64ISD::SST1: return "AArch64ISD::SST1";
case AArch64ISD::SST1_SCALED: return "AArch64ISD::SST1_SCALED";
case AArch64ISD::SST1_SXTW: return "AArch64ISD::SST1_SXTW";
@@ -9041,7 +9045,6 @@ bool AArch64TargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
Info.align = Align(16);
Info.flags = MachineMemOperand::MOStore | MachineMemOperand::MOVolatile;
return true;
- case Intrinsic::aarch64_sve_ld1:
case Intrinsic::aarch64_sve_ldnt1: {
PointerType *PtrTy = cast<PointerType>(I.getArgOperand(1)->getType());
Info.opc = ISD::INTRINSIC_W_CHAIN;
@@ -9054,7 +9057,6 @@ bool AArch64TargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
Info.flags |= MachineMemOperand::MONonTemporal;
return true;
}
- case Intrinsic::aarch64_sve_st1:
case Intrinsic::aarch64_sve_stnt1: {
PointerType *PtrTy = cast<PointerType>(I.getArgOperand(2)->getType());
Info.opc = ISD::INTRINSIC_W_CHAIN;
@@ -10515,6 +10517,7 @@ static SDValue performSVEAndCombine(SDNode *N,
// SVE load instructions perform an implicit zero-extend, which makes them
// perfect candidates for combining.
switch (Src->getOpcode()) {
+ case AArch64ISD::LD1:
case AArch64ISD::LDNF1:
case AArch64ISD::LDFF1:
MemVT = cast<VTSDNode>(Src->getOperand(3))->getVT();
@@ -11581,7 +11584,33 @@ static MVT getSVEContainerType(EVT ContentTy) {
}
}
-static SDValue performLD1Combine(SDNode *N, SelectionDAG &DAG) {
+static SDValue performLD1Combine(SDNode *N, SelectionDAG &DAG, unsigned Opc) {
+ SDLoc DL(N);
+ EVT VT = N->getValueType(0);
+
+ if (VT.getSizeInBits().getKnownMinSize() > AArch64::SVEBitsPerBlock)
+ return SDValue();
+
+ EVT ContainerVT = VT;
+ if (ContainerVT.isInteger())
+ ContainerVT = getSVEContainerType(ContainerVT);
+
+ SDVTList VTs = DAG.getVTList(ContainerVT, MVT::Other);
+ SDValue Ops[] = { N->getOperand(0), // Chain
+ N->getOperand(2), // Pg
+ N->getOperand(3), // Base
+ DAG.getValueType(VT) };
+
+ SDValue Load = DAG.getNode(Opc, DL, VTs, Ops);
+ SDValue LoadChain = SDValue(Load.getNode(), 1);
+
+ if (ContainerVT.isInteger() && (VT != ContainerVT))
+ Load = DAG.getNode(ISD::TRUNCATE, DL, VT, Load.getValue(0));
+
+ return DAG.getMergeValues({ Load, LoadChain }, DL);
+}
+
+static SDValue performLDNT1Combine(SDNode *N, SelectionDAG &DAG) {
SDLoc DL(N);
EVT VT = N->getValueType(0);
EVT PtrTy = N->getOperand(3).getValueType();
@@ -11608,6 +11637,32 @@ static SDValue performLD1Combine(SDNode *N, SelectionDAG &DAG) {
static SDValue performST1Combine(SDNode *N, SelectionDAG &DAG) {
SDLoc DL(N);
+ SDValue Data = N->getOperand(2);
+ EVT DataVT = Data.getValueType();
+ EVT HwSrcVt = getSVEContainerType(DataVT);
+ SDValue InputVT = DAG.getValueType(DataVT);
+
+ if (DataVT.isFloatingPoint())
+ InputVT = DAG.getValueType(HwSrcVt);
+
+ SDValue SrcNew;
+ if (Data.getValueType().isFloatingPoint())
+ SrcNew = DAG.getNode(ISD::BITCAST, DL, HwSrcVt, Data);
+ else
+ SrcNew = DAG.getNode(ISD::ANY_EXTEND, DL, HwSrcVt, Data);
+
+ SDValue Ops[] = { N->getOperand(0), // Chain
+ SrcNew,
+ N->getOperand(4), // Base
+ N->getOperand(3), // Pg
+ InputVT
+ };
+
+ return DAG.getNode(AArch64ISD::ST1, DL, N->getValueType(0), Ops);
+}
+
+static SDValue performSTNT1Combine(SDNode *N, SelectionDAG &DAG) {
+ SDLoc DL(N);
SDValue Data = N->getOperand(2);
EVT DataVT = Data.getValueType();
@@ -11623,32 +11678,6 @@ static SDValue performST1Combine(SDNode *N, SelectionDAG &DAG) {
ISD::UNINDEXED, false, false);
}
-static SDValue performLDNF1Combine(SDNode *N, SelectionDAG &DAG, unsigned Opc) {
- SDLoc DL(N);
- EVT VT = N->getValueType(0);
-
- if (VT.getSizeInBits().getKnownMinSize() > AArch64::SVEBitsPerBlock)
- return SDValue();
-
- EVT ContainerVT = VT;
- if (ContainerVT.isInteger())
- ContainerVT = getSVEContainerType(ContainerVT);
-
- SDVTList VTs = DAG.getVTList(ContainerVT, MVT::Other);
- SDValue Ops[] = { N->getOperand(0), // Chain
- N->getOperand(2), // Pg
- N->getOperand(3), // Base
- DAG.getValueType(VT) };
-
- SDValue Load = DAG.getNode(Opc, DL, VTs, Ops);
- SDValue LoadChain = SDValue(Load.getNode(), 1);
-
- if (ContainerVT.isInteger() && (VT != ContainerVT))
- Load = DAG.getNode(ISD::TRUNCATE, DL, VT, Load.getValue(0));
-
- return DAG.getMergeValues({ Load, LoadChain }, DL);
-}
-
/// Replace a splat of zeros to a vector store by scalar stores of WZR/XZR. The
/// load store optimizer pass will merge them to store pair stores. This should
/// be better than a movi to create the vector zero followed by a vector store
@@ -12963,6 +12992,10 @@ performSignExtendInRegCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
unsigned NewOpc;
unsigned MemVTOpNum = 4;
switch (Opc) {
+ case AArch64ISD::LD1:
+ NewOpc = AArch64ISD::LD1S;
+ MemVTOpNum = 3;
+ break;
case AArch64ISD::LDNF1:
NewOpc = AArch64ISD::LDNF1S;
MemVTOpNum = 3;
@@ -13189,9 +13222,8 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
case Intrinsic::aarch64_neon_st3lane:
case Intrinsic::aarch64_neon_st4lane:
return performNEONPostLDSTCombine(N, DCI, DAG);
- case Intrinsic::aarch64_sve_ld1:
case Intrinsic::aarch64_sve_ldnt1:
- return performLD1Combine(N, DAG);
+ return performLDNT1Combine(N, DAG);
case Intrinsic::aarch64_sve_ldnt1_gather_scalar_offset:
return performGatherLoadCombine(N, DAG, AArch64ISD::GLDNT1);
case Intrinsic::aarch64_sve_ldnt1_gather:
@@ -13200,13 +13232,16 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
return performGatherLoadCombine(N, DAG, AArch64ISD::GLDNT1_INDEX);
case Intrinsic::aarch64_sve_ldnt1_gather_uxtw:
return performGatherLoadCombine(N, DAG, AArch64ISD::GLDNT1);
+ case Intrinsic::aarch64_sve_ld1:
+ return performLD1Combine(N, DAG, AArch64ISD::LD1);
case Intrinsic::aarch64_sve_ldnf1:
- return performLDNF1Combine(N, DAG, AArch64ISD::LDNF1);
+ return performLD1Combine(N, DAG, AArch64ISD::LDNF1);
case Intrinsic::aarch64_sve_ldff1:
- return performLDNF1Combine(N, DAG, AArch64ISD::LDFF1);
+ return performLD1Combine(N, DAG, AArch64ISD::LDFF1);
case Intrinsic::aarch64_sve_st1:
- case Intrinsic::aarch64_sve_stnt1:
return performST1Combine(N, DAG);
+ case Intrinsic::aarch64_sve_stnt1:
+ return performSTNT1Combine(N, DAG);
case Intrinsic::aarch64_sve_stnt1_scatter_scalar_offset:
return performScatterStoreCombine(N, DAG, AArch64ISD::SSTNT1);
case Intrinsic::aarch64_sve_stnt1_scatter_uxtw:
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 3b3fb40301e4..19b0188cdd22 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -226,6 +226,8 @@ enum NodeType : unsigned {
REINTERPRET_CAST,
+ LD1,
+ LD1S,
LDNF1,
LDNF1S,
LDFF1,
@@ -272,6 +274,8 @@ enum NodeType : unsigned {
GLDNT1_INDEX,
GLDNT1S,
+ ST1,
+
// Scatter store
SST1,
SST1_SCALED,
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index c046b0e18a37..dd1b72456c33 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -13,18 +13,23 @@
def SVE8BitLslImm : ComplexPattern<i32, 2, "SelectSVE8BitLslImm", [imm]>;
def SVELShiftImm64 : ComplexPattern<i32, 1, "SelectSVEShiftImm64<0, 64>", []>;
-// Non-faulting & first-faulting loads - node definitions
+// Contiguous loads - node definitions
//
-def SDT_AArch64_LDNF1 : SDTypeProfile<1, 3, [
+def SDT_AArch64_LD1 : SDTypeProfile<1, 3, [
SDTCisVec<0>, SDTCisVec<1>, SDTCisPtrTy<2>,
SDTCVecEltisVT<1,i1>, SDTCisSameNumEltsAs<0,1>
]>;
-def AArch64ldnf1 : SDNode<"AArch64ISD::LDNF1", SDT_AArch64_LDNF1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
-def AArch64ldff1 : SDNode<"AArch64ISD::LDFF1", SDT_AArch64_LDNF1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
+def AArch64ld1 : SDNode<"AArch64ISD::LD1", SDT_AArch64_LD1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue]>;
+def AArch64ld1s : SDNode<"AArch64ISD::LD1S", SDT_AArch64_LD1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue]>;
+
+// Non-faulting & first-faulting loads - node definitions
+//
+def AArch64ldnf1 : SDNode<"AArch64ISD::LDNF1", SDT_AArch64_LD1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
+def AArch64ldff1 : SDNode<"AArch64ISD::LDFF1", SDT_AArch64_LD1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
-def AArch64ldnf1s : SDNode<"AArch64ISD::LDNF1S", SDT_AArch64_LDNF1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
-def AArch64ldff1s : SDNode<"AArch64ISD::LDFF1S", SDT_AArch64_LDNF1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
+def AArch64ldnf1s : SDNode<"AArch64ISD::LDNF1S", SDT_AArch64_LD1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
+def AArch64ldff1s : SDNode<"AArch64ISD::LDFF1S", SDT_AArch64_LD1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue, SDNPOutGlue]>;
// Gather loads - node definitions
//
@@ -73,6 +78,15 @@ def AArch64ldff1s_gather_imm : SDNode<"AArch64ISD::GLDFF1S_IMM",
def AArch64ldnt1_gather : SDNode<"AArch64ISD::GLDNT1", SDT_AArch64_GATHER_VS, [SDNPHasChain, SDNPMayLoad]>;
def AArch64ldnt1s_gather : SDNode<"AArch64ISD::GLDNT1S", SDT_AArch64_GATHER_VS, [SDNPHasChain, SDNPMayLoad]>;
+// Contiguous stores - node definitions
+//
+def SDT_AArch64_ST1 : SDTypeProfile<0, 4, [
+ SDTCisVec<0>, SDTCisPtrTy<1>, SDTCisVec<2>,
+ SDTCVecEltisVT<2,i1>, SDTCisSameNumEltsAs<0,2>
+]>;
+
+def AArch64st1 : SDNode<"AArch64ISD::ST1", SDT_AArch64_ST1, [SDNPHasChain, SDNPMayStore]>;
+
// Scatter stores - node definitions
//
def SDT_AArch64_SCATTER_SV : SDTypeProfile<0, 5, [
@@ -1554,7 +1568,7 @@ multiclass sve_prefetch<SDPatternOperator prefetch, ValueType PredTy, Instructio
defm Pat_Load_P4 : unpred_load_predicate<nxv4i1, LDR_PXI>;
defm Pat_Load_P2 : unpred_load_predicate<nxv2i1, LDR_PXI>;
- multiclass ldnf1<Instruction I, ValueType Ty, SDPatternOperator Load, ValueType PredTy, ValueType MemVT> {
+ multiclass ld1<Instruction I, ValueType Ty, SDPatternOperator Load, ValueType PredTy, ValueType MemVT> {
// scalar + immediate (mul vl)
let AddedComplexity = 1 in {
def : Pat<(Ty (Load (PredTy PPR:$gp), (am_sve_indexed_s4 GPR64sp:$base, simm4s1:$offset), MemVT)),
@@ -1566,32 +1580,60 @@ multiclass sve_prefetch<SDPatternOperator prefetch, ValueType PredTy, Instructio
(I PPR:$gp, GPR64sp:$base, (i64 0))>;
}
+ // 2-element contiguous loads
+ defm : ld1<LD1B_D_IMM, nxv2i64, AArch64ld1, nxv2i1, nxv2i8>;
+ defm : ld1<LD1SB_D_IMM, nxv2i64, AArch64ld1s, nxv2i1, nxv2i8>;
+ defm : ld1<LD1H_D_IMM, nxv2i64, AArch64ld1, nxv2i1, nxv2i16>;
+ defm : ld1<LD1SH_D_IMM, nxv2i64, AArch64ld1s, nxv2i1, nxv2i16>;
+ defm : ld1<LD1W_D_IMM, nxv2i64, AArch64ld1, nxv2i1, nxv2i32>;
+ defm : ld1<LD1SW_D_IMM, nxv2i64, AArch64ld1s, nxv2i1, nxv2i32>;
+ defm : ld1<LD1D_IMM, nxv2i64, AArch64ld1, nxv2i1, nxv2i64>;
+ defm : ld1<LD1D_IMM, nxv2f64, AArch64ld1, nxv2i1, nxv2f64>;
+
+ // 4-element contiguous loads
+ defm : ld1<LD1B_S_IMM, nxv4i32, AArch64ld1, nxv4i1, nxv4i8>;
+ defm : ld1<LD1SB_S_IMM, nxv4i32, AArch64ld1s, nxv4i1, nxv4i8>;
+ defm : ld1<LD1H_S_IMM, nxv4i32, AArch64ld1, nxv4i1, nxv4i16>;
+ defm : ld1<LD1SH_S_IMM, nxv4i32, AArch64ld1s, nxv4i1, nxv4i16>;
+ defm : ld1<LD1W_IMM, nxv4i32, AArch64ld1, nxv4i1, nxv4i32>;
+ defm : ld1<LD1W_IMM, nxv4f32, AArch64ld1, nxv4i1, nxv4f32>;
+
+ // 8-element contiguous loads
+ defm : ld1<LD1B_H_IMM, nxv8i16, AArch64ld1, nxv8i1, nxv8i8>;
+ defm : ld1<LD1SB_H_IMM, nxv8i16, AArch64ld1s, nxv8i1, nxv8i8>;
+ defm : ld1<LD1H_IMM, nxv8i16, AArch64ld1, nxv8i1, nxv8i16>;
+ defm : ld1<LD1H_IMM, nxv8f16, AArch64ld1, nxv8i1, nxv8f16>;
+
+ // 16-element contiguous loads
+ defm : ld1<LD1B_IMM, nxv16i8, AArch64ld1, nxv16i1, nxv16i8>;
+
+
// 2-element contiguous non-faulting loads
- defm : ldnf1<LDNF1B_D_IMM, nxv2i64, AArch64ldnf1, nxv2i1, nxv2i8>;
- defm : ldnf1<LDNF1SB_D_IMM, nxv2i64, AArch64ldnf1s, nxv2i1, nxv2i8>;
- defm : ldnf1<LDNF1H_D_IMM, nxv2i64, AArch64ldnf1, nxv2i1, nxv2i16>;
- defm : ldnf1<LDNF1SH_D_IMM, nxv2i64, AArch64ldnf1s, nxv2i1, nxv2i16>;
- defm : ldnf1<LDNF1W_D_IMM, nxv2i64, AArch64ldnf1, nxv2i1, nxv2i32>;
- defm : ldnf1<LDNF1SW_D_IMM, nxv2i64, AArch64ldnf1s, nxv2i1, nxv2i32>;
- defm : ldnf1<LDNF1D_IMM, nxv2i64, AArch64ldnf1, nxv2i1, nxv2i64>;
- defm : ldnf1<LDNF1D_IMM, nxv2f64, AArch64ldnf1, nxv2i1, nxv2f64>;
+ defm : ld1<LDNF1B_D_IMM, nxv2i64, AArch64ldnf1, nxv2i1, nxv2i8>;
+ defm : ld1<LDNF1SB_D_IMM, nxv2i64, AArch64ldnf1s, nxv2i1, nxv2i8>;
+ defm : ld1<LDNF1H_D_IMM, nxv2i64, AArch64ldnf1, nxv2i1, nxv2i16>;
+ defm : ld1<LDNF1SH_D_IMM, nxv2i64, AArch64ldnf1s, nxv2i1, nxv2i16>;
+ defm : ld1<LDNF1W_D_IMM, nxv2i64, AArch64ldnf1, nxv2i1, nxv2i32>;
+ defm : ld1<LDNF1SW_D_IMM, nxv2i64, AArch64ldnf1s, nxv2i1, nxv2i32>;
+ defm : ld1<LDNF1D_IMM, nxv2i64, AArch64ldnf1, nxv2i1, nxv2i64>;
+ defm : ld1<LDNF1D_IMM, nxv2f64, AArch64ldnf1, nxv2i1, nxv2f64>;
// 4-element contiguous non-faulting loads
- defm : ldnf1<LDNF1B_S_IMM, nxv4i32, AArch64ldnf1, nxv4i1, nxv4i8>;
- defm : ldnf1<LDNF1SB_S_IMM, nxv4i32, AArch64ldnf1s, nxv4i1, nxv4i8>;
- defm : ldnf1<LDNF1H_S_IMM, nxv4i32, AArch64ldnf1, nxv4i1, nxv4i16>;
- defm : ldnf1<LDNF1SH_S_IMM, nxv4i32, AArch64ldnf1s, nxv4i1, nxv4i16>;
- defm : ldnf1<LDNF1W_IMM, nxv4i32, AArch64ldnf1, nxv4i1, nxv4i32>;
- defm : ldnf1<LDNF1W_IMM, nxv4f32, AArch64ldnf1, nxv4i1, nxv4f32>;
+ defm : ld1<LDNF1B_S_IMM, nxv4i32, AArch64ldnf1, nxv4i1, nxv4i8>;
+ defm : ld1<LDNF1SB_S_IMM, nxv4i32, AArch64ldnf1s, nxv4i1, nxv4i8>;
+ defm : ld1<LDNF1H_S_IMM, nxv4i32, AArch64ldnf1, nxv4i1, nxv4i16>;
+ defm : ld1<LDNF1SH_S_IMM, nxv4i32, AArch64ldnf1s, nxv4i1, nxv4i16>;
+ defm : ld1<LDNF1W_IMM, nxv4i32, AArch64ldnf1, nxv4i1, nxv4i32>;
+ defm : ld1<LDNF1W_IMM, nxv4f32, AArch64ldnf1, nxv4i1, nxv4f32>;
// 8-element contiguous non-faulting loads
- defm : ldnf1<LDNF1B_H_IMM, nxv8i16, AArch64ldnf1, nxv8i1, nxv8i8>;
- defm : ldnf1<LDNF1SB_H_IMM, nxv8i16, AArch64ldnf1s, nxv8i1, nxv8i8>;
- defm : ldnf1<LDNF1H_IMM, nxv8i16, AArch64ldnf1, nxv8i1, nxv8i16>;
- defm : ldnf1<LDNF1H_IMM, nxv8f16, AArch64ldnf1, nxv8i1, nxv8f16>;
+ defm : ld1<LDNF1B_H_IMM, nxv8i16, AArch64ldnf1, nxv8i1, nxv8i8>;
+ defm : ld1<LDNF1SB_H_IMM, nxv8i16, AArch64ldnf1s, nxv8i1, nxv8i8>;
+ defm : ld1<LDNF1H_IMM, nxv8i16, AArch64ldnf1, nxv8i1, nxv8i16>;
+ defm : ld1<LDNF1H_IMM, nxv8f16, AArch64ldnf1, nxv8i1, nxv8f16>;
// 16-element contiguous non-faulting loads
- defm : ldnf1<LDNF1B_IMM, nxv16i8, AArch64ldnf1, nxv16i1, nxv16i8>;
+ defm : ld1<LDNF1B_IMM, nxv16i8, AArch64ldnf1, nxv16i1, nxv16i8>;
multiclass ldff1<Instruction I, ValueType Ty, SDPatternOperator Load, ValueType PredTy, ValueType MemVT, ComplexPattern AddrCP> {
// reg + reg
@@ -1632,6 +1674,37 @@ multiclass sve_prefetch<SDPatternOperator prefetch, ValueType PredTy, Instructio
// 16-element contiguous first faulting loads
defm : ldff1<LDFF1B, nxv16i8, AArch64ldff1, nxv16i1, nxv16i8, am_sve_regreg_lsl0>;
+
+ multiclass st1<Instruction I, ValueType Ty, SDPatternOperator Store, ValueType PredTy, ValueType MemVT> {
+ // scalar + immediate (mul vl)
+ let AddedComplexity = 1 in {
+ def : Pat<(Store (Ty ZPR:$vec), (am_sve_indexed_s4 GPR64sp:$base, simm4s1:$offset), (PredTy PPR:$gp), MemVT),
+ (I ZPR:$vec, PPR:$gp, GPR64sp:$base, simm4s1:$offset)>;
+ }
+
+ // base
+ def : Pat<(Store (Ty ZPR:$vec), GPR64:$base, (PredTy PPR:$gp), MemVT),
+ (I ZPR:$vec, PPR:$gp, GPR64:$base, (i64 0))>;
+ }
+
+ // 2-element contiguous store
+ defm : st1<ST1B_D_IMM, nxv2i64, AArch64st1, nxv2i1, nxv2i8>;
+ defm : st1<ST1H_D_IMM, nxv2i64, AArch64st1, nxv2i1, nxv2i16>;
+ defm : st1<ST1W_D_IMM, nxv2i64, AArch64st1, nxv2i1, nxv2i32>;
+ defm : st1<ST1D_IMM, nxv2i64, AArch64st1, nxv2i1, nxv2i64>;
+
+ // 4-element contiguous store
+ defm : st1<ST1B_S_IMM, nxv4i32, AArch64st1, nxv4i1, nxv4i8>;
+ defm : st1<ST1H_S_IMM, nxv4i32, AArch64st1, nxv4i1, nxv4i16>;
+ defm : st1<ST1W_IMM, nxv4i32, AArch64st1, nxv4i1, nxv4i32>;
+
+ // 8-element contiguous store
+ defm : st1<ST1B_H_IMM, nxv8i16, AArch64st1, nxv8i1, nxv8i8>;
+ defm : st1<ST1H_IMM, nxv8i16, AArch64st1, nxv8i1, nxv8i16>;
+
+ // 16-element contiguous store
+ defm : st1<ST1B_IMM, nxv16i8, AArch64st1, nxv16i1, nxv16i8>;
+
}
let Predicates = [HasSVE2] in {
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-ld1.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-ld1.ll
new file mode 100644
index 000000000000..c1e69c725e6b
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-ld1.ll
@@ -0,0 +1,469 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;
+; LD1B
+;
+
+define <vscale x 16 x i8> @ld1b_i8(<vscale x 16 x i1> %pred, i8* %addr) {
+; CHECK-LABEL: ld1b_i8:
+; CHECK: ld1b { z0.b }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %res = call <vscale x 16 x i8> @llvm.aarch64.sve.ld1.nxv16i8(<vscale x 16 x i1> %pred, i8* %addr)
+ ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @ld1b_h(<vscale x 8 x i1> %pred, i8* %addr) {
+; CHECK-LABEL: ld1b_h:
+; CHECK: ld1b { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %load = call <vscale x 8 x i8> @llvm.aarch64.sve.ld1.nxv8i8(<vscale x 8 x i1> %pred, i8* %addr)
+ %res = zext <vscale x 8 x i8> %load to <vscale x 8 x i16>
+ ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 8 x i16> @ld1sb_h(<vscale x 8 x i1> %pred, i8* %addr) {
+; CHECK-LABEL: ld1sb_h:
+; CHECK: ld1sb { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %load = call <vscale x 8 x i8> @llvm.aarch64.sve.ld1.nxv8i8(<vscale x 8 x i1> %pred, i8* %addr)
+ %res = sext <vscale x 8 x i8> %load to <vscale x 8 x i16>
+ ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @ld1b_s(<vscale x 4 x i1> %pred, i8* %addr) {
+; CHECK-LABEL: ld1b_s:
+; CHECK: ld1b { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %load = call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.nxv4i8(<vscale x 4 x i1> %pred, i8* %addr)
+ %res = zext <vscale x 4 x i8> %load to <vscale x 4 x i32>
+ ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @ld1sb_s(<vscale x 4 x i1> %pred, i8* %addr) {
+; CHECK-LABEL: ld1sb_s:
+; CHECK: ld1sb { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %load = call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.nxv4i8(<vscale x 4 x i1> %pred, i8* %addr)
+ %res = sext <vscale x 4 x i8> %load to <vscale x 4 x i32>
+ ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @ld1b_d(<vscale x 2 x i1> %pred, i8* %addr) {
+; CHECK-LABEL: ld1b_d:
+; CHECK: ld1b { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %load = call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.nxv2i8(<vscale x 2 x i1> %pred, i8* %addr)
+ %res = zext <vscale x 2 x i8> %load to <vscale x 2 x i64>
+ ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @ld1sb_d(<vscale x 2 x i1> %pred, i8* %addr) {
+; CHECK-LABEL: ld1sb_d:
+; CHECK: ld1sb { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %load = call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.nxv2i8(<vscale x 2 x i1> %pred, i8* %addr)
+ %res = sext <vscale x 2 x i8> %load to <vscale x 2 x i64>
+ ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 16 x i8> @ld1b_upper_bound(<vscale x 16 x i1> %pg, i8* %a) {
+; CHECK-LABEL: ld1b_upper_bound:
+; CHECK: ld1b { z0.b }, p0/z, [x0, #7, mul vl]
+; CHECK-NEXT: ret
+ %base_scalable = bitcast i8* %a to <vscale x 16 x i8>*
+ %base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %base_scalable, i64 7
+ %base_scalar = bitcast <vscale x 16 x i8>* %base to i8*
+ %load = call <vscale x 16 x i8> @llvm.aarch64.sve.ld1.nxv16i8(<vscale x 16 x i1> %pg, i8* %base_scalar)
+ ret <vscale x 16 x i8> %load
+}
+
+define <vscale x 16 x i8> @ld1b_inbound(<vscale x 16 x i1> %pg, i8* %a) {
+; CHECK-LABEL: ld1b_inbound:
+; CHECK: ld1b { z0.b }, p0/z, [x0, #1, mul vl]
+; CHECK-NEXT: ret
+ %base_scalable = bitcast i8* %a to <vscale x 16 x i8>*
+ %base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %base_scalable, i64 1
+ %base_scalar = bitcast <vscale x 16 x i8>* %base to i8*
+ %load = call <vscale x 16 x i8> @llvm.aarch64.sve.ld1.nxv16i8(<vscale x 16 x i1> %pg, i8* %base_scalar)
+ ret <vscale x 16 x i8> %load
+}
+
+define <vscale x 4 x i32> @ld1b_s_inbound(<vscale x 4 x i1> %pg, i8* %a) {
+; CHECK-LABEL: ld1b_s_inbound:
+; CHECK: ld1b { z0.s }, p0/z, [x0, #7, mul vl]
+; CHECK-NEXT: ret
+ %base_scalable = bitcast i8* %a to <vscale x 4 x i8>*
+ %base = getelementptr <vscale x 4 x i8>, <vscale x 4 x i8>* %base_scalable, i64 7
+ %base_scalar = bitcast <vscale x 4 x i8>* %base to i8*
+ %load = call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.nxv4i8(<vscale x 4 x i1> %pg, i8* %base_scalar)
+ %res = zext <vscale x 4 x i8> %load to <vscale x 4 x i32>
+ ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @ld1sb_s_inbound(<vscale x 4 x i1> %pg, i8* %a) {
+; CHECK-LABEL: ld1sb_s_inbound:
+; CHECK: ld1sb { z0.s }, p0/z, [x0, #7, mul vl]
+; CHECK-NEXT: ret
+ %base_scalable = bitcast i8* %a to <vscale x 4 x i8>*
+ %base = getelementptr <vscale x 4 x i8>, <vscale x 4 x i8>* %base_scalable, i64 7
+ %base_scalar = bitcast <vscale x 4 x i8>* %base to i8*
+ %load = call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.nxv4i8(<vscale x 4 x i1> %pg, i8* %base_scalar)
+ %res = sext <vscale x 4 x i8> %load to <vscale x 4 x i32>
+ ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 16 x i8> @ld1b_lower_bound(<vscale x 16 x i1> %pg, i8* %a) {
+; CHECK-LABEL: ld1b_lower_bound:
+; CHECK: ld1b { z0.b }, p0/z, [x0, #-8, mul vl]
+; CHECK-NEXT: ret
+ %base_scalable = bitcast i8* %a to <vscale x 16 x i8>*
+ %base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %base_scalable, i64 -8
+ %base_scalar = bitcast <vscale x 16 x i8>* %base to i8*
+ %load = call <vscale x 16 x i8> @llvm.aarch64.sve.ld1.nxv16i8(<vscale x 16 x i1> %pg, i8* %base_scalar)
+ ret <vscale x 16 x i8> %load
+}
+
+define <vscale x 16 x i8> @ld1b_out_of_upper_bound(<vscale x 16 x i1> %pg, i8* %a) {
+; CHECK-LABEL: ld1b_out_of_upper_bound:
+; CHECK: rdvl x[[OFFSET:[0-9]+]], #8
+; CHECK-NEXT: add x[[BASE:[0-9]+]], x0, x[[OFFSET]]
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x[[BASE]]]
+; CHECK-NEXT: ret
+ %base_scalable = bitcast i8* %a to <vscale x 16 x i8>*
+ %base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %base_scalable, i64 8
+ %base_scalar = bitcast <vscale x 16 x i8>* %base to i8*
+ %load = call <vscale x 16 x i8> @llvm.aarch64.sve.ld1.nxv16i8(<vscale x 16 x i1> %pg, i8* %base_scalar)
+ ret <vscale x 16 x i8> %load
+}
+
+define <vscale x 16 x i8> @ld1b_out_of_lower_bound(<vscale x 16 x i1> %pg, i8* %a) {
+; CHECK-LABEL: ld1b_out_of_lower_bound:
+; CHECK: rdvl x[[OFFSET:[0-9]+]], #-9
+; CHECK-NEXT: add x[[BASE:[0-9]+]], x0, x[[OFFSET]]
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x[[BASE]]]
+; CHECK-NEXT: ret
+ %base_scalable = bitcast i8* %a to <vscale x 16 x i8>*
+ %base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %base_scalable, i64 -9
+ %base_scalar = bitcast <vscale x 16 x i8>* %base to i8*
+ %load = call <vscale x 16 x i8> @llvm.aarch64.sve.ld1.nxv16i8(<vscale x 16 x i1> %pg, i8* %base_scalar)
+ ret <vscale x 16 x i8> %load
+}
+
+;
+; LD1H
+;
+
+define <vscale x 8 x i16> @ld1h_i16(<vscale x 8 x i1> %pred, i16* %addr) {
+; CHECK-LABEL: ld1h_i16:
+; CHECK: ld1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %res = call <vscale x 8 x i16> @llvm.aarch64.sve.ld1.nxv8i16(<vscale x 8 x i1> %pred, i16* %addr)
+ ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 8 x half> @ld1h_f16(<vscale x 8 x i1> %pred, half* %addr) {
+; CHECK-LABEL: ld1h_f16:
+; CHECK: ld1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %res = call <vscale x 8 x half> @llvm.aarch64.sve.ld1.nxv8f16(<vscale x 8 x i1> %pred, half* %addr)
+ ret <vscale x 8 x half> %res
+}
+
+define <vscale x 4 x i32> @ld1h_s(<vscale x 4 x i1> %pred, i16* %addr) {
+; CHECK-LABEL: ld1h_s:
+; CHECK: ld1h { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %load = call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.nxv4i16(<vscale x 4 x i1> %pred, i16* %addr)
+ %res = zext <vscale x 4 x i16> %load to <vscale x 4 x i32>
+ ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @ld1sh_s(<vscale x 4 x i1> %pred, i16* %addr) {
+; CHECK-LABEL: ld1sh_s:
+; CHECK: ld1sh { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %load = call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.nxv4i16(<vscale x 4 x i1> %pred, i16* %addr)
+ %res = sext <vscale x 4 x i16> %load to <vscale x 4 x i32>
+ ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @ld1h_d(<vscale x 2 x i1> %pred, i16* %addr) {
+; CHECK-LABEL: ld1h_d:
+; CHECK: ld1h { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.nxv2i16(<vscale x 2 x i1> %pred, i16* %addr)
+ %res = zext <vscale x 2 x i16> %load to <vscale x 2 x i64>
+ ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @ld1sh_d(<vscale x 2 x i1> %pred, i16* %addr) {
+; CHECK-LABEL: ld1sh_d:
+; CHECK: ld1sh { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.nxv2i16(<vscale x 2 x i1> %pred, i16* %addr)
+ %res = sext <vscale x 2 x i16> %load to <vscale x 2 x i64>
+ ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 8 x i16> @ld1b_h_inbound(<vscale x 8 x i1> %pg, i8* %a) {
+; CHECK-LABEL: ld1b_h_inbound:
+; CHECK: ld1b { z0.h }, p0/z, [x0, #7, mul vl]
+; CHECK-NEXT: ret
+ %base_scalable = bitcast i8* %a to <vscale x 8 x i8>*
+ %base = getelementptr <vscale x 8 x i8>, <vscale x 8 x i8>* %base_scalable, i64 7
+ %base_scalar = bitcast <vscale x 8 x i8>* %base to i8*
+ %load = call <vscale x 8 x i8> @llvm.aarch64.sve.ld1.nxv8i8(<vscale x 8 x i1> %pg, i8* %base_scalar)
+ %res = zext <vscale x 8 x i8> %load to <vscale x 8 x i16>
+ ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 8 x i16> @ld1sb_h_inbound(<vscale x 8 x i1> %pg, i8* %a) {
+; CHECK-LABEL: ld1sb_h_inbound:
+; CHECK: ld1sb { z0.h }, p0/z, [x0, #7, mul vl]
+; CHECK-NEXT: ret
+ %base_scalable = bitcast i8* %a to <vscale x 8 x i8>*
+ %base = getelementptr <vscale x 8 x i8>, <vscale x 8 x i8>* %base_scalable, i64 7
+ %base_scalar = bitcast <vscale x 8 x i8>* %base to i8*
+ %load = call <vscale x 8 x i8> @llvm.aarch64.sve.ld1.nxv8i8(<vscale x 8 x i1> %pg, i8* %base_scalar)
+ %res = sext <vscale x 8 x i8> %load to <vscale x 8 x i16>
+ ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 8 x i16> @ld1h_inbound(<vscale x 8 x i1> %pg, i16* %a) {
+; CHECK-LABEL: ld1h_inbound:
+; CHECK: ld1h { z0.h }, p0/z, [x0, #1, mul vl]
+; CHECK-NEXT: ret
+ %base_scalable = bitcast i16* %a to <vscale x 8 x i16>*
+ %base = getelementptr <vscale x 8 x i16>, <vscale x 8 x i16>* %base_scalable, i64 1
+ %base_scalar = bitcast <vscale x 8 x i16>* %base to i16*
+ %load = call <vscale x 8 x i16> @llvm.aarch64.sve.ld1.nxv8i16(<vscale x 8 x i1> %pg, i16* %base_scalar)
+ ret <vscale x 8 x i16> %load
+}
+
+define <vscale x 4 x i32> @ld1h_s_inbound(<vscale x 4 x i1> %pg, i16* %a) {
+; CHECK-LABEL: ld1h_s_inbound:
+; CHECK: ld1h { z0.s }, p0/z, [x0, #7, mul vl]
+; CHECK-NEXT: ret
+ %base_scalable = bitcast i16* %a to <vscale x 4 x i16>*
+ %base = getelementptr <vscale x 4 x i16>, <vscale x 4 x i16>* %base_scalable, i64 7
+ %base_scalar = bitcast <vscale x 4 x i16>* %base to i16*
+ %load = call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.nxv4i16(<vscale x 4 x i1> %pg, i16* %base_scalar)
+ %res = zext <vscale x 4 x i16> %load to <vscale x 4 x i32>
+ ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @ld1sh_s_inbound(<vscale x 4 x i1> %pg, i16* %a) {
+; CHECK-LABEL: ld1sh_s_inbound:
+; CHECK: ld1sh { z0.s }, p0/z, [x0, #7, mul vl]
+; CHECK-NEXT: ret
+ %base_scalable = bitcast i16* %a to <vscale x 4 x i16>*
+ %base = getelementptr <vscale x 4 x i16>, <vscale x 4 x i16>* %base_scalable, i64 7
+ %base_scalar = bitcast <vscale x 4 x i16>* %base to i16*
+ %load = call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.nxv4i16(<vscale x 4 x i1> %pg, i16* %base_scalar)
+ %res = sext <vscale x 4 x i16> %load to <vscale x 4 x i32>
+ ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @ld1b_d_inbound(<vscale x 2 x i1> %pg, i8* %a) {
+; CHECK-LABEL: ld1b_d_inbound:
+; CHECK: ld1b { z0.d }, p0/z, [x0, #7, mul vl]
+; CHECK-NEXT: ret
+ %base_scalable = bitcast i8* %a to <vscale x 2 x i8>*
+ %base = getelementptr <vscale x 2 x i8>, <vscale x 2 x i8>* %base_scalable, i64 7
+ %base_scalar = bitcast <vscale x 2 x i8>* %base to i8*
+ %load = call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.nxv2i8(<vscale x 2 x i1> %pg, i8* %base_scalar)
+ %res = zext <vscale x 2 x i8> %load to <vscale x 2 x i64>
+ ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @ld1sb_d_inbound(<vscale x 2 x i1> %pg, i8* %a) {
+; CHECK-LABEL: ld1sb_d_inbound:
+; CHECK: ld1sb { z0.d }, p0/z, [x0, #7, mul vl]
+; CHECK-NEXT: ret
+ %base_scalable = bitcast i8* %a to <vscale x 2 x i8>*
+ %base = getelementptr <vscale x 2 x i8>, <vscale x 2 x i8>* %base_scalable, i64 7
+ %base_scalar = bitcast <vscale x 2 x i8>* %base to i8*
+ %load = call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.nxv2i8(<vscale x 2 x i1> %pg, i8* %base_scalar)
+ %res = sext <vscale x 2 x i8> %load to <vscale x 2 x i64>
+ ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @ld1h_d_inbound(<vscale x 2 x i1> %pg, i16* %a) {
+; CHECK-LABEL: ld1h_d_inbound:
+; CHECK: ld1h { z0.d }, p0/z, [x0, #7, mul vl]
+; CHECK-NEXT: ret
+ %base_scalable = bitcast i16* %a to <vscale x 2 x i16>*
+ %base = getelementptr <vscale x 2 x i16>, <vscale x 2 x i16>* %base_scalable, i64 7
+ %base_scalar = bitcast <vscale x 2 x i16>* %base to i16*
+ %load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.nxv2i16(<vscale x 2 x i1> %pg, i16* %base_scalar)
+ %res = zext <vscale x 2 x i16> %load to <vscale x 2 x i64>
+ ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @ld1sh_d_inbound(<vscale x 2 x i1> %pg, i16* %a) {
+; CHECK-LABEL: ld1sh_d_inbound:
+; CHECK: ld1sh { z0.d }, p0/z, [x0, #7, mul vl]
+; CHECK-NEXT: ret
+ %base_scalable = bitcast i16* %a to <vscale x 2 x i16>*
+ %base = getelementptr <vscale x 2 x i16>, <vscale x 2 x i16>* %base_scalable, i64 7
+ %base_scalar = bitcast <vscale x 2 x i16>* %base to i16*
+ %load = call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.nxv2i16(<vscale x 2 x i1> %pg, i16* %base_scalar)
+ %res = sext <vscale x 2 x i16> %load to <vscale x 2 x i64>
+ ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 8 x half> @ld1h_f16_inbound(<vscale x 8 x i1> %pg, half* %a) {
+; CHECK-LABEL: ld1h_f16_inbound:
+; CHECK: ld1h { z0.h }, p0/z, [x0, #1, mul vl]
+; CHECK-NEXT: ret
+ %base_scalable = bitcast half* %a to <vscale x 8 x half>*
+ %base = getelementptr <vscale x 8 x half>, <vscale x 8 x half>* %base_scalable, i64 1
+ %base_scalar = bitcast <vscale x 8 x half>* %base to half*
+ %load = call <vscale x 8 x half> @llvm.aarch64.sve.ld1.nxv8f16(<vscale x 8 x i1> %pg, half* %base_scalar)
+ ret <vscale x 8 x half> %load
+}
+
+;
+; LD1W
+;
+
+define <vscale x 4 x i32> @ld1w_i32(<vscale x 4 x i1> %pred, i32* %addr) {
+; CHECK-LABEL: ld1w_i32:
+; CHECK: ld1w { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %res = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.nxv4i32(<vscale x 4 x i1> %pred, i32* %addr)
+ ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x float> @ld1w_f32(<vscale x 4 x i1> %pred, float* %addr) {
+; CHECK-LABEL: ld1w_f32:
+; CHECK: ld1w { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %res = call <vscale x 4 x float> @llvm.aarch64.sve.ld1.nxv4f32(<vscale x 4 x i1> %pred, float* %addr)
+ ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x i64> @ld1w_d(<vscale x 2 x i1> %pred, i32* %addr) {
+; CHECK-LABEL: ld1w_d:
+; CHECK: ld1w { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.nxv2i32(<vscale x 2 x i1> %pred, i32* %addr)
+ %res = zext <vscale x 2 x i32> %load to <vscale x 2 x i64>
+ ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @ld1sw_d(<vscale x 2 x i1> %pred, i32* %addr) {
+; CHECK-LABEL: ld1sw_d:
+; CHECK: ld1sw { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.nxv2i32(<vscale x 2 x i1> %pred, i32* %addr)
+ %res = sext <vscale x 2 x i32> %load to <vscale x 2 x i64>
+ ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 4 x i32> @ld1w_inbound(<vscale x 4 x i1> %pg, i32* %a) {
+; CHECK-LABEL: ld1w_inbound:
+; CHECK: ld1w { z0.s }, p0/z, [x0, #7, mul vl]
+; CHECK-NEXT: ret
+ %base_scalable = bitcast i32* %a to <vscale x 4 x i32>*
+ %base = getelementptr <vscale x 4 x i32>, <vscale x 4 x i32>* %base_scalable, i64 7
+ %base_scalar = bitcast <vscale x 4 x i32>* %base to i32*
+ %load = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.nxv4i32(<vscale x 4 x i1> %pg, i32* %base_scalar)
+ ret <vscale x 4 x i32> %load
+}
+
+define <vscale x 4 x float> @ld1w_f32_inbound(<vscale x 4 x i1> %pg, float* %a) {
+; CHECK-LABEL: ld1w_f32_inbound:
+; CHECK: ld1w { z0.s }, p0/z, [x0, #7, mul vl]
+; CHECK-NEXT: ret
+ %base_scalable = bitcast float* %a to <vscale x 4 x float>*
+ %base = getelementptr <vscale x 4 x float>, <vscale x 4 x float>* %base_scalable, i64 7
+ %base_scalar = bitcast <vscale x 4 x float>* %base to float*
+ %load = call <vscale x 4 x float> @llvm.aarch64.sve.ld1.nxv4f32(<vscale x 4 x i1> %pg, float* %base_scalar)
+ ret <vscale x 4 x float> %load
+}
+
+;
+; LD1D
+;
+
+define <vscale x 2 x i64> @ld1d_i64(<vscale x 2 x i1> %pred, i64* %addr) {
+; CHECK-LABEL: ld1d_i64:
+; CHECK: ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %res = call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.nxv2i64(<vscale x 2 x i1> %pred,
+ i64* %addr)
+ ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x double> @ld1d_f64(<vscale x 2 x i1> %pred, double* %addr) {
+; CHECK-LABEL: ld1d_f64:
+; CHECK: ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %res = call <vscale x 2 x double> @llvm.aarch64.sve.ld1.nxv2f64(<vscale x 2 x i1> %pred,
+ double* %addr)
+ ret <vscale x 2 x double> %res
+}
+
+define <vscale x 2 x i64> @ld1d_inbound(<vscale x 2 x i1> %pg, i64* %a) {
+; CHECK-LABEL: ld1d_inbound:
+; CHECK: ld1d { z0.d }, p0/z, [x0, #1, mul vl]
+; CHECK-NEXT: ret
+ %base_scalable = bitcast i64* %a to <vscale x 2 x i64>*
+ %base = getelementptr <vscale x 2 x i64>, <vscale x 2 x i64>* %base_scalable, i64 1
+ %base_scalar = bitcast <vscale x 2 x i64>* %base to i64*
+ %load = call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.nxv2i64(<vscale x 2 x i1> %pg, i64* %base_scalar)
+ ret <vscale x 2 x i64> %load
+}
+
+define <vscale x 2 x i64> @ld1w_d_inbound(<vscale x 2 x i1> %pg, i32* %a) {
+; CHECK-LABEL: ld1w_d_inbound:
+; CHECK: ld1w { z0.d }, p0/z, [x0, #7, mul vl]
+; CHECK-NEXT: ret
+ %base_scalable = bitcast i32* %a to <vscale x 2 x i32>*
+ %base = getelementptr <vscale x 2 x i32>, <vscale x 2 x i32>* %base_scalable, i64 7
+ %base_scalar = bitcast <vscale x 2 x i32>* %base to i32*
+ %load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.nxv2i32(<vscale x 2 x i1> %pg, i32* %base_scalar)
+ %res = zext <vscale x 2 x i32> %load to <vscale x 2 x i64>
+ ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @ld1sw_d_inbound(<vscale x 2 x i1> %pg, i32* %a) {
+; CHECK-LABEL: ld1sw_d_inbound:
+; CHECK: ld1sw { z0.d }, p0/z, [x0, #7, mul vl]
+; CHECK-NEXT: ret
+ %base_scalable = bitcast i32* %a to <vscale x 2 x i32>*
+ %base = getelementptr <vscale x 2 x i32>, <vscale x 2 x i32>* %base_scalable, i64 7
+ %base_scalar = bitcast <vscale x 2 x i32>* %base to i32*
+ %load = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.nxv2i32(<vscale x 2 x i1> %pg, i32* %base_scalar)
+ %res = sext <vscale x 2 x i32> %load to <vscale x 2 x i64>
+ ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x double> @ld1d_f64_inbound(<vscale x 2 x i1> %pg, double* %a) {
+; CHECK-LABEL: ld1d_f64_inbound:
+; CHECK: ld1d { z0.d }, p0/z, [x0, #1, mul vl]
+; CHECK-NEXT: ret
+ %base_scalable = bitcast double* %a to <vscale x 2 x double>*
+ %base = getelementptr <vscale x 2 x double>, <vscale x 2 x double>* %base_scalable, i64 1
+ %base_scalar = bitcast <vscale x 2 x double>* %base to double*
+ %load = call <vscale x 2 x double> @llvm.aarch64.sve.ld1.nxv2f64(<vscale x 2 x i1> %pg, double* %base_scalar)
+ ret <vscale x 2 x double> %load
+}
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.ld1.nxv16i8(<vscale x 16 x i1>, i8*)
+
+declare <vscale x 8 x i8> @llvm.aarch64.sve.ld1.nxv8i8(<vscale x 8 x i1>, i8*)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.ld1.nxv8i16(<vscale x 8 x i1>, i16*)
+declare <vscale x 8 x half> @llvm.aarch64.sve.ld1.nxv8f16(<vscale x 8 x i1>, half*)
+
+declare <vscale x 4 x i8> @llvm.aarch64.sve.ld1.nxv4i8(<vscale x 4 x i1>, i8*)
+declare <vscale x 4 x i16> @llvm.aarch64.sve.ld1.nxv4i16(<vscale x 4 x i1>, i16*)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.ld1.nxv4i32(<vscale x 4 x i1>, i32*)
+declare <vscale x 4 x float> @llvm.aarch64.sve.ld1.nxv4f32(<vscale x 4 x i1>, float*)
+
+declare <vscale x 2 x i8> @llvm.aarch64.sve.ld1.nxv2i8(<vscale x 2 x i1>, i8*)
+declare <vscale x 2 x i16> @llvm.aarch64.sve.ld1.nxv2i16(<vscale x 2 x i1>, i16*)
+declare <vscale x 2 x i32> @llvm.aarch64.sve.ld1.nxv2i32(<vscale x 2 x i1>, i32*)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.ld1.nxv2i64(<vscale x 2 x i1>, i64*)
+declare <vscale x 2 x double> @llvm.aarch64.sve.ld1.nxv2f64(<vscale x 2 x i1>, double*)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-ldst1.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-ldst1.ll
deleted file mode 100644
index 1501647b0883..000000000000
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-ldst1.ll
+++ /dev/null
@@ -1,182 +0,0 @@
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
-
-;
-; LD1B
-;
-
-define <vscale x 16 x i8> @ld1b_i8(<vscale x 16 x i1> %pred, i8* %addr) {
-; CHECK-LABEL: ld1b_i8:
-; CHECK: ld1b { z0.b }, p0/z, [x0]
-; CHECK-NEXT: ret
- %res = call <vscale x 16 x i8> @llvm.aarch64.sve.ld1.nxv16i8(<vscale x 16 x i1> %pred,
- i8* %addr)
- ret <vscale x 16 x i8> %res
-}
-
-;
-; LD1H
-;
-
-define <vscale x 8 x i16> @ld1h_i16(<vscale x 8 x i1> %pred, i16* %addr) {
-; CHECK-LABEL: ld1h_i16:
-; CHECK: ld1h { z0.h }, p0/z, [x0]
-; CHECK-NEXT: ret
- %res = call <vscale x 8 x i16> @llvm.aarch64.sve.ld1.nxv8i16(<vscale x 8 x i1> %pred,
- i16* %addr)
- ret <vscale x 8 x i16> %res
-}
-
-define <vscale x 8 x half> @ld1h_f16(<vscale x 8 x i1> %pred, half* %addr) {
-; CHECK-LABEL: ld1h_f16:
-; CHECK: ld1h { z0.h }, p0/z, [x0]
-; CHECK-NEXT: ret
- %res = call <vscale x 8 x half> @llvm.aarch64.sve.ld1.nxv8f16(<vscale x 8 x i1> %pred,
- half* %addr)
- ret <vscale x 8 x half> %res
-}
-
-;
-; LD1W
-;
-
-define <vscale x 4 x i32> @ld1w_i32(<vscale x 4 x i1> %pred, i32* %addr) {
-; CHECK-LABEL: ld1w_i32:
-; CHECK: ld1w { z0.s }, p0/z, [x0]
-; CHECK-NEXT: ret
- %res = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.nxv4i32(<vscale x 4 x i1> %pred,
- i32* %addr)
- ret <vscale x 4 x i32> %res
-}
-
-define <vscale x 4 x float> @ld1w_f32(<vscale x 4 x i1> %pred, float* %addr) {
-; CHECK-LABEL: ld1w_f32:
-; CHECK: ld1w { z0.s }, p0/z, [x0]
-; CHECK-NEXT: ret
- %res = call <vscale x 4 x float> @llvm.aarch64.sve.ld1.nxv4f32(<vscale x 4 x i1> %pred,
- float* %addr)
- ret <vscale x 4 x float> %res
-}
-
-;
-; LD1D
-;
-
-define <vscale x 2 x i64> @ld1d_i64(<vscale x 2 x i1> %pred, i64* %addr) {
-; CHECK-LABEL: ld1d_i64:
-; CHECK: ld1d { z0.d }, p0/z, [x0]
-; CHECK-NEXT: ret
- %res = call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.nxv2i64(<vscale x 2 x i1> %pred,
- i64* %addr)
- ret <vscale x 2 x i64> %res
-}
-
-define <vscale x 2 x double> @ld1d_f64(<vscale x 2 x i1> %pred, double* %addr) {
-; CHECK-LABEL: ld1d_f64:
-; CHECK: ld1d { z0.d }, p0/z, [x0]
-; CHECK-NEXT: ret
- %res = call <vscale x 2 x double> @llvm.aarch64.sve.ld1.nxv2f64(<vscale x 2 x i1> %pred,
- double* %addr)
- ret <vscale x 2 x double> %res
-}
-
-;
-; ST1B
-;
-
-define void @st1b_i8(<vscale x 16 x i8> %data, <vscale x 16 x i1> %pred, i8* %addr) {
-; CHECK-LABEL: st1b_i8:
-; CHECK: st1b { z0.b }, p0, [x0]
-; CHECK-NEXT: ret
- call void @llvm.aarch64.sve.st1.nxv16i8(<vscale x 16 x i8> %data,
- <vscale x 16 x i1> %pred,
- i8* %addr)
- ret void
-}
-
-;
-; ST1H
-;
-
-define void @st1h_i16(<vscale x 8 x i16> %data, <vscale x 8 x i1> %pred, i16* %addr) {
-; CHECK-LABEL: st1h_i16:
-; CHECK: st1h { z0.h }, p0, [x0]
-; CHECK-NEXT: ret
- call void @llvm.aarch64.sve.st1.nxv8i16(<vscale x 8 x i16> %data,
- <vscale x 8 x i1> %pred,
- i16* %addr)
- ret void
-}
-
-define void @st1h_f16(<vscale x 8 x half> %data, <vscale x 8 x i1> %pred, half* %addr) {
-; CHECK-LABEL: st1h_f16:
-; CHECK: st1h { z0.h }, p0, [x0]
-; CHECK-NEXT: ret
- call void @llvm.aarch64.sve.st1.nxv8f16(<vscale x 8 x half> %data,
- <vscale x 8 x i1> %pred,
- half* %addr)
- ret void
-}
-
-;
-; ST1W
-;
-
-define void @st1w_i32(<vscale x 4 x i32> %data, <vscale x 4 x i1> %pred, i32* %addr) {
-; CHECK-LABEL: st1w_i32:
-; CHECK: st1w { z0.s }, p0, [x0]
-; CHECK-NEXT: ret
- call void @llvm.aarch64.sve.st1.nxv4i32(<vscale x 4 x i32> %data,
- <vscale x 4 x i1> %pred,
- i32* %addr)
- ret void
-}
-
-define void @st1w_f32(<vscale x 4 x float> %data, <vscale x 4 x i1> %pred, float* %addr) {
-; CHECK-LABEL: st1w_f32:
-; CHECK: st1w { z0.s }, p0, [x0]
-; CHECK-NEXT: ret
- call void @llvm.aarch64.sve.st1.nxv4f32(<vscale x 4 x float> %data,
- <vscale x 4 x i1> %pred,
- float* %addr)
- ret void
-}
-
-;
-; ST1D
-;
-
-define void @st1d_i64(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pred, i64* %addr) {
-; CHECK-LABEL: st1d_i64:
-; CHECK: st1d { z0.d }, p0, [x0]
-; CHECK-NEXT: ret
- call void @llvm.aarch64.sve.st1.nxv2i64(<vscale x 2 x i64> %data,
- <vscale x 2 x i1> %pred,
- i64* %addr)
- ret void
-}
-
-define void @st1d_f64(<vscale x 2 x double> %data, <vscale x 2 x i1> %pred, double* %addr) {
-; CHECK-LABEL: st1d_f64:
-; CHECK: st1d { z0.d }, p0, [x0]
-; CHECK-NEXT: ret
- call void @llvm.aarch64.sve.st1.nxv2f64(<vscale x 2 x double> %data,
- <vscale x 2 x i1> %pred,
- double* %addr)
- ret void
-}
-
-declare <vscale x 16 x i8> @llvm.aarch64.sve.ld1.nxv16i8(<vscale x 16 x i1>, i8*)
-declare <vscale x 8 x i16> @llvm.aarch64.sve.ld1.nxv8i16(<vscale x 8 x i1>, i16*)
-declare <vscale x 4 x i32> @llvm.aarch64.sve.ld1.nxv4i32(<vscale x 4 x i1>, i32*)
-declare <vscale x 2 x i64> @llvm.aarch64.sve.ld1.nxv2i64(<vscale x 2 x i1>, i64*)
-declare <vscale x 8 x half> @llvm.aarch64.sve.ld1.nxv8f16(<vscale x 8 x i1>, half*)
-declare <vscale x 4 x float> @llvm.aarch64.sve.ld1.nxv4f32(<vscale x 4 x i1>, float*)
-declare <vscale x 2 x double> @llvm.aarch64.sve.ld1.nxv2f64(<vscale x 2 x i1>, double*)
-
-declare void @llvm.aarch64.sve.st1.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i1>, i8*)
-declare void @llvm.aarch64.sve.st1.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>, i16*)
-declare void @llvm.aarch64.sve.st1.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, i32*)
-declare void @llvm.aarch64.sve.st1.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, i64*)
-declare void @llvm.aarch64.sve.st1.nxv8f16(<vscale x 8 x half>, <vscale x 8 x i1>, half*)
-declare void @llvm.aarch64.sve.st1.nxv4f32(<vscale x 4 x float>, <vscale x 4 x i1>, float*)
-declare void @llvm.aarch64.sve.st1.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i1>, double*)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-st1.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-st1.ll
new file mode 100644
index 000000000000..7fdc150f64fd
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-st1.ll
@@ -0,0 +1,367 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;
+; ST1B
+;
+
+define void @st1b_i8(<vscale x 16 x i8> %data, <vscale x 16 x i1> %pred, i8* %addr) {
+; CHECK-LABEL: st1b_i8:
+; CHECK: st1b { z0.b }, p0, [x0]
+; CHECK-NEXT: ret
+ call void @llvm.aarch64.sve.st1.nxv16i8(<vscale x 16 x i8> %data,
+ <vscale x 16 x i1> %pred,
+ i8* %addr)
+ ret void
+}
+
+define void @st1b_h(<vscale x 8 x i16> %data, <vscale x 8 x i1> %pred, i8* %addr) {
+; CHECK-LABEL: st1b_h:
+; CHECK: st1b { z0.h }, p0, [x0]
+; CHECK-NEXT: ret
+ %trunc = trunc <vscale x 8 x i16> %data to <vscale x 8 x i8>
+ call void @llvm.aarch64.sve.st1.nxv8i8(<vscale x 8 x i8> %trunc,
+ <vscale x 8 x i1> %pred,
+ i8* %addr)
+ ret void
+}
+
+define void @st1b_s(<vscale x 4 x i32> %data, <vscale x 4 x i1> %pred, i8* %addr) {
+; CHECK-LABEL: st1b_s:
+; CHECK: st1b { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %trunc = trunc <vscale x 4 x i32> %data to <vscale x 4 x i8>
+ call void @llvm.aarch64.sve.st1.nxv4i8(<vscale x 4 x i8> %trunc,
+ <vscale x 4 x i1> %pred,
+ i8* %addr)
+ ret void
+}
+
+define void @st1b_d(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pred, i8* %addr) {
+; CHECK-LABEL: st1b_d:
+; CHECK: st1b { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %trunc = trunc <vscale x 2 x i64> %data to <vscale x 2 x i8>
+ call void @llvm.aarch64.sve.st1.nxv2i8(<vscale x 2 x i8> %trunc,
+ <vscale x 2 x i1> %pred,
+ i8* %addr)
+ ret void
+}
+
+define void @st1b_upper_bound(<vscale x 16 x i8> %data, <vscale x 16 x i1> %pg, i8* %a) {
+; CHECK-LABEL: st1b_upper_bound:
+; CHECK: st1b { z0.b }, p0, [x0, #7, mul vl]
+; CHECK-NEXT: ret
+ %base_scalable = bitcast i8* %a to <vscale x 16 x i8>*
+ %base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %base_scalable, i64 7
+ %base_scalar = bitcast <vscale x 16 x i8>* %base to i8*
+ call void @llvm.aarch64.sve.st1.nxv16i8(<vscale x 16 x i8> %data, <vscale x 16 x i1> %pg, i8* %base_scalar)
+ ret void
+}
+
+define void @st1b_inbound(<vscale x 16 x i8> %data, <vscale x 16 x i1> %pg, i8* %a) {
+; CHECK-LABEL: st1b_inbound:
+; CHECK: st1b { z0.b }, p0, [x0, #1, mul vl]
+; CHECK-NEXT: ret
+ %base_scalable = bitcast i8* %a to <vscale x 16 x i8>*
+ %base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %base_scalable, i64 1
+ %base_scalar = bitcast <vscale x 16 x i8>* %base to i8*
+ call void @llvm.aarch64.sve.st1.nxv16i8(<vscale x 16 x i8> %data, <vscale x 16 x i1> %pg, i8* %base_scalar)
+ ret void
+}
+
+define void @st1b_lower_bound(<vscale x 16 x i8> %data, <vscale x 16 x i1> %pg, i8* %a) {
+; CHECK-LABEL: st1b_lower_bound:
+; CHECK: st1b { z0.b }, p0, [x0, #-8, mul vl]
+; CHECK-NEXT: ret
+ %base_scalable = bitcast i8* %a to <vscale x 16 x i8>*
+ %base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %base_scalable, i64 -8
+ %base_scalar = bitcast <vscale x 16 x i8>* %base to i8*
+ call void @llvm.aarch64.sve.st1.nxv16i8(<vscale x 16 x i8> %data, <vscale x 16 x i1> %pg, i8* %base_scalar)
+ ret void
+}
+
+define void @st1b_out_of_upper_bound(<vscale x 16 x i8> %data, <vscale x 16 x i1> %pg, i8* %a) {
+; CHECK-LABEL: st1b_out_of_upper_bound:
+; CHECK: rdvl x[[OFFSET:[0-9]+]], #8
+; CHECK: add x[[BASE:[0-9]+]], x0, x[[OFFSET]]
+; CHECK: st1b { z0.b }, p0, [x[[BASE]]]
+; CHECK-NEXT: ret
+ %base_scalable = bitcast i8* %a to <vscale x 16 x i8>*
+ %base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %base_scalable, i64 8
+ %base_scalar = bitcast <vscale x 16 x i8>* %base to i8*
+ call void @llvm.aarch64.sve.st1.nxv16i8(<vscale x 16 x i8> %data, <vscale x 16 x i1> %pg, i8* %base_scalar)
+ ret void
+}
+
+define void @st1b_out_of_lower_bound(<vscale x 16 x i8> %data, <vscale x 16 x i1> %pg, i8* %a) {
+; CHECK-LABEL: st1b_out_of_lower_bound:
+; CHECK: rdvl x[[OFFSET:[0-9]+]], #-9
+; CHECK: add x[[BASE:[0-9]+]], x0, x[[OFFSET]]
+; CHECK: st1b { z0.b }, p0, [x[[BASE]]]
+; CHECK-NEXT: ret
+ %base_scalable = bitcast i8* %a to <vscale x 16 x i8>*
+ %base = getelementptr <vscale x 16 x i8>, <vscale x 16 x i8>* %base_scalable, i64 -9
+ %base_scalar = bitcast <vscale x 16 x i8>* %base to i8*
+ call void @llvm.aarch64.sve.st1.nxv16i8(<vscale x 16 x i8> %data, <vscale x 16 x i1> %pg, i8* %base_scalar)
+ ret void
+}
+
+define void @st1b_s_inbound(<vscale x 4 x i32> %data, <vscale x 4 x i1> %pg, i8* %a) {
+; CHECK-LABEL: st1b_s_inbound:
+; CHECK: st1b { z0.s }, p0, [x0, #7, mul vl]
+; CHECK-NEXT: ret
+ %base_scalable = bitcast i8* %a to <vscale x 4 x i8>*
+ %base = getelementptr <vscale x 4 x i8>, <vscale x 4 x i8>* %base_scalable, i64 7
+ %base_scalar = bitcast <vscale x 4 x i8>* %base to i8*
+ %trunc = trunc <vscale x 4 x i32> %data to <vscale x 4 x i8>
+ call void @llvm.aarch64.sve.st1.nxv4i8(<vscale x 4 x i8> %trunc, <vscale x 4 x i1> %pg, i8* %base_scalar)
+ ret void
+}
+
+define void @st1b_h_inbound(<vscale x 8 x i16> %data, <vscale x 8 x i1> %pg, i8* %a) {
+; CHECK-LABEL: st1b_h_inbound:
+; CHECK: st1b { z0.h }, p0, [x0, #1, mul vl]
+; CHECK-NEXT: ret
+ %base_scalable = bitcast i8* %a to <vscale x 8 x i8>*
+ %base = getelementptr <vscale x 8 x i8>, <vscale x 8 x i8>* %base_scalable, i64 1
+ %base_scalar = bitcast <vscale x 8 x i8>* %base to i8*
+ %trunc = trunc <vscale x 8 x i16> %data to <vscale x 8 x i8>
+ call void @llvm.aarch64.sve.st1.nxv8i8(<vscale x 8 x i8> %trunc, <vscale x 8 x i1> %pg, i8* %base_scalar)
+ ret void
+}
+
+define void @st1b_d_inbound(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pg, i8* %a) {
+; CHECK-LABEL: st1b_d_inbound:
+; CHECK: st1b { z0.d }, p0, [x0, #-7, mul vl]
+; CHECK-NEXT: ret
+ %base_scalable = bitcast i8* %a to <vscale x 2 x i8>*
+ %base = getelementptr <vscale x 2 x i8>, <vscale x 2 x i8>* %base_scalable, i64 -7
+ %base_scalar = bitcast <vscale x 2 x i8>* %base to i8*
+ %trunc = trunc <vscale x 2 x i64> %data to <vscale x 2 x i8>
+ call void @llvm.aarch64.sve.st1.nxv2i8(<vscale x 2 x i8> %trunc, <vscale x 2 x i1> %pg, i8* %base_scalar)
+ ret void
+}
+
+;
+; ST1H
+;
+
+define void @st1h_i16(<vscale x 8 x i16> %data, <vscale x 8 x i1> %pred, i16* %addr) {
+; CHECK-LABEL: st1h_i16:
+; CHECK: st1h { z0.h }, p0, [x0]
+; CHECK-NEXT: ret
+ call void @llvm.aarch64.sve.st1.nxv8i16(<vscale x 8 x i16> %data,
+ <vscale x 8 x i1> %pred,
+ i16* %addr)
+ ret void
+}
+
+define void @st1h_f16(<vscale x 8 x half> %data, <vscale x 8 x i1> %pred, half* %addr) {
+; CHECK-LABEL: st1h_f16:
+; CHECK: st1h { z0.h }, p0, [x0]
+; CHECK-NEXT: ret
+ call void @llvm.aarch64.sve.st1.nxv8f16(<vscale x 8 x half> %data,
+ <vscale x 8 x i1> %pred,
+ half* %addr)
+ ret void
+}
+
+define void @st1h_s(<vscale x 4 x i32> %data, <vscale x 4 x i1> %pred, i16* %addr) {
+; CHECK-LABEL: st1h_s:
+; CHECK: st1h { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %trunc = trunc <vscale x 4 x i32> %data to <vscale x 4 x i16>
+ call void @llvm.aarch64.sve.st1.nxv4i16(<vscale x 4 x i16> %trunc,
+ <vscale x 4 x i1> %pred,
+ i16* %addr)
+ ret void
+}
+
+define void @st1h_d(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pred, i16* %addr) {
+; CHECK-LABEL: st1h_d:
+; CHECK: st1h { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %trunc = trunc <vscale x 2 x i64> %data to <vscale x 2 x i16>
+ call void @llvm.aarch64.sve.st1.nxv2i16(<vscale x 2 x i16> %trunc,
+ <vscale x 2 x i1> %pred,
+ i16* %addr)
+ ret void
+}
+
+define void @st1h_inbound(<vscale x 8 x i16> %data, <vscale x 8 x i1> %pg, i16* %a) {
+; CHECK-LABEL: st1h_inbound:
+; CHECK: st1h { z0.h }, p0, [x0, #-1, mul vl]
+; CHECK-NEXT: ret
+ %base_scalable = bitcast i16* %a to <vscale x 8 x i16>*
+ %base = getelementptr <vscale x 8 x i16>, <vscale x 8 x i16>* %base_scalable, i64 -1
+ %base_scalar = bitcast <vscale x 8 x i16>* %base to i16*
+ call void @llvm.aarch64.sve.st1.nxv8i16(<vscale x 8 x i16> %data, <vscale x 8 x i1> %pg, i16* %base_scalar)
+ ret void
+}
+
+define void @st1h_f16_inbound(<vscale x 8 x half> %data, <vscale x 8 x i1> %pg, half* %a) {
+; CHECK-LABEL: st1h_f16_inbound:
+; CHECK: st1h { z0.h }, p0, [x0, #-5, mul vl]
+; CHECK-NEXT: ret
+ %base_scalable = bitcast half* %a to <vscale x 8 x half>*
+ %base = getelementptr <vscale x 8 x half>, <vscale x 8 x half>* %base_scalable, i64 -5
+ %base_scalar = bitcast <vscale x 8 x half>* %base to half*
+ call void @llvm.aarch64.sve.st1.nxv8f16(<vscale x 8 x half> %data, <vscale x 8 x i1> %pg, half* %base_scalar)
+ ret void
+}
+
+define void @st1h_s_inbound(<vscale x 4 x i32> %data, <vscale x 4 x i1> %pg, i16* %a) {
+; CHECK-LABEL: st1h_s_inbound:
+; CHECK: st1h { z0.s }, p0, [x0, #2, mul vl]
+; CHECK-NEXT: ret
+ %base_scalable = bitcast i16* %a to <vscale x 4 x i16>*
+ %base = getelementptr <vscale x 4 x i16>, <vscale x 4 x i16>* %base_scalable, i64 2
+ %base_scalar = bitcast <vscale x 4 x i16>* %base to i16*
+ %trunc = trunc <vscale x 4 x i32> %data to <vscale x 4 x i16>
+ call void @llvm.aarch64.sve.st1.nxv4i16(<vscale x 4 x i16> %trunc, <vscale x 4 x i1> %pg, i16* %base_scalar)
+ ret void
+}
+
+define void @st1h_d_inbound(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pg, i16* %a) {
+; CHECK-LABEL: st1h_d_inbound:
+; CHECK: st1h { z0.d }, p0, [x0, #-4, mul vl]
+; CHECK-NEXT: ret
+ %base_scalable = bitcast i16* %a to <vscale x 2 x i16>*
+ %base = getelementptr <vscale x 2 x i16>, <vscale x 2 x i16>* %base_scalable, i64 -4
+ %base_scalar = bitcast <vscale x 2 x i16>* %base to i16*
+ %trunc = trunc <vscale x 2 x i64> %data to <vscale x 2 x i16>
+ call void @llvm.aarch64.sve.st1.nxv2i16(<vscale x 2 x i16> %trunc, <vscale x 2 x i1> %pg, i16* %base_scalar)
+ ret void
+}
+
+;
+; ST1W
+;
+
+define void @st1w_i32(<vscale x 4 x i32> %data, <vscale x 4 x i1> %pred, i32* %addr) {
+; CHECK-LABEL: st1w_i32:
+; CHECK: st1w { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+ call void @llvm.aarch64.sve.st1.nxv4i32(<vscale x 4 x i32> %data,
+ <vscale x 4 x i1> %pred,
+ i32* %addr)
+ ret void
+}
+
+define void @st1w_f32(<vscale x 4 x float> %data, <vscale x 4 x i1> %pred, float* %addr) {
+; CHECK-LABEL: st1w_f32:
+; CHECK: st1w { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+ call void @llvm.aarch64.sve.st1.nxv4f32(<vscale x 4 x float> %data,
+ <vscale x 4 x i1> %pred,
+ float* %addr)
+ ret void
+}
+
+define void @st1w_d(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pred, i32* %addr) {
+; CHECK-LABEL: st1w_d:
+; CHECK: st1w { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %trunc = trunc <vscale x 2 x i64> %data to <vscale x 2 x i32>
+ call void @llvm.aarch64.sve.st1.nxv2i32(<vscale x 2 x i32> %trunc,
+ <vscale x 2 x i1> %pred,
+ i32* %addr)
+ ret void
+}
+
+define void @st1w_inbound(<vscale x 4 x i32> %data, <vscale x 4 x i1> %pg, i32* %a) {
+; CHECK-LABEL: st1w_inbound:
+; CHECK: st1w { z0.s }, p0, [x0, #6, mul vl]
+; CHECK-NEXT: ret
+ %base_scalable = bitcast i32* %a to <vscale x 4 x i32>*
+ %base = getelementptr <vscale x 4 x i32>, <vscale x 4 x i32>* %base_scalable, i64 6
+ %base_scalar = bitcast <vscale x 4 x i32>* %base to i32*
+ call void @llvm.aarch64.sve.st1.nxv4i32(<vscale x 4 x i32> %data, <vscale x 4 x i1> %pg, i32* %base_scalar)
+ ret void
+}
+
+define void @st1w_f32_inbound(<vscale x 4 x float> %data, <vscale x 4 x i1> %pg, float* %a) {
+; CHECK-LABEL: st1w_f32_inbound:
+; CHECK: st1w { z0.s }, p0, [x0, #-1, mul vl]
+; CHECK-NEXT: ret
+ %base_scalable = bitcast float* %a to <vscale x 4 x float>*
+ %base = getelementptr <vscale x 4 x float>, <vscale x 4 x float>* %base_scalable, i64 -1
+ %base_scalar = bitcast <vscale x 4 x float>* %base to float*
+ call void @llvm.aarch64.sve.st1.nxv4f32(<vscale x 4 x float> %data, <vscale x 4 x i1> %pg, float* %base_scalar)
+ ret void
+}
+
+define void @st1w_d_inbound(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pg, i32* %a) {
+; CHECK-LABEL: st1w_d_inbound:
+; CHECK: st1w { z0.d }, p0, [x0, #1, mul vl]
+; CHECK-NEXT: ret
+ %base_scalable = bitcast i32* %a to <vscale x 2 x i32>*
+ %base = getelementptr <vscale x 2 x i32>, <vscale x 2 x i32>* %base_scalable, i64 1
+ %base_scalar = bitcast <vscale x 2 x i32>* %base to i32*
+ %trunc = trunc <vscale x 2 x i64> %data to <vscale x 2 x i32>
+ call void @llvm.aarch64.sve.st1.nxv2i32(<vscale x 2 x i32> %trunc, <vscale x 2 x i1> %pg, i32* %base_scalar)
+ ret void
+}
+
+;
+; ST1D
+;
+
+define void @st1d_i64(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pred, i64* %addr) {
+; CHECK-LABEL: st1d_i64:
+; CHECK: st1d { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ call void @llvm.aarch64.sve.st1.nxv2i64(<vscale x 2 x i64> %data,
+ <vscale x 2 x i1> %pred,
+ i64* %addr)
+ ret void
+}
+
+define void @st1d_f64(<vscale x 2 x double> %data, <vscale x 2 x i1> %pred, double* %addr) {
+; CHECK-LABEL: st1d_f64:
+; CHECK: st1d { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ call void @llvm.aarch64.sve.st1.nxv2f64(<vscale x 2 x double> %data,
+ <vscale x 2 x i1> %pred,
+ double* %addr)
+ ret void
+}
+
+define void @st1d_inbound(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pg, i64* %a) {
+; CHECK-LABEL: st1d_inbound:
+; CHECK: st1d { z0.d }, p0, [x0, #5, mul vl]
+; CHECK-NEXT: ret
+ %base_scalable = bitcast i64* %a to <vscale x 2 x i64>*
+ %base = getelementptr <vscale x 2 x i64>, <vscale x 2 x i64>* %base_scalable, i64 5
+ %base_scalar = bitcast <vscale x 2 x i64>* %base to i64*
+ call void @llvm.aarch64.sve.st1.nxv2i64(<vscale x 2 x i64> %data, <vscale x 2 x i1> %pg, i64* %base_scalar)
+ ret void
+}
+
+define void @st1d_f64_inbound(<vscale x 2 x double> %data, <vscale x 2 x i1> %pg, double* %a) {
+; CHECK-LABEL: st1d_f64_inbound:
+; CHECK: st1d { z0.d }, p0, [x0, #-8, mul vl]
+; CHECK-NEXT: ret
+ %base_scalable = bitcast double* %a to <vscale x 2 x double>*
+ %base = getelementptr <vscale x 2 x double>, <vscale x 2 x double>* %base_scalable, i64 -8
+ %base_scalar = bitcast <vscale x 2 x double>* %base to double*
+ call void @llvm.aarch64.sve.st1.nxv2f64(<vscale x 2 x double> %data, <vscale x 2 x i1> %pg, double* %base_scalar)
+ ret void
+}
+
+declare void @llvm.aarch64.sve.st1.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i1>, i8*)
+
+declare void @llvm.aarch64.sve.st1.nxv8i8(<vscale x 8 x i8>, <vscale x 8 x i1>, i8*)
+declare void @llvm.aarch64.sve.st1.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>, i16*)
+declare void @llvm.aarch64.sve.st1.nxv8f16(<vscale x 8 x half>, <vscale x 8 x i1>, half*)
+
+declare void @llvm.aarch64.sve.st1.nxv4i8(<vscale x 4 x i8>, <vscale x 4 x i1>, i8*)
+declare void @llvm.aarch64.sve.st1.nxv4i16(<vscale x 4 x i16>, <vscale x 4 x i1>, i16*)
+declare void @llvm.aarch64.sve.st1.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, i32*)
+declare void @llvm.aarch64.sve.st1.nxv4f32(<vscale x 4 x float>, <vscale x 4 x i1>, float*)
+
+declare void @llvm.aarch64.sve.st1.nxv2i8(<vscale x 2 x i8>, <vscale x 2 x i1>, i8*)
+declare void @llvm.aarch64.sve.st1.nxv2i16(<vscale x 2 x i16>, <vscale x 2 x i1>, i16*)
+declare void @llvm.aarch64.sve.st1.nxv2i32(<vscale x 2 x i32>, <vscale x 2 x i1>, i32*)
+declare void @llvm.aarch64.sve.st1.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, i64*)
+declare void @llvm.aarch64.sve.st1.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i1>, double*)
More information about the llvm-commits
mailing list