[llvm] [AArch64][SME2] Add stridedorcontiguous pseudos for multi-vector-stores (PR #211551)
Jacob Crawley via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 29 02:16:39 PDT 2026
https://github.com/jacob-crawley updated https://github.com/llvm/llvm-project/pull/211551
>From 04db6c55f256aaf1503beaf249ac80d25b67d7bc Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Thu, 23 Jul 2026 12:45:33 +0000
Subject: [PATCH 1/4] [AArch64][SME2] Enable strided register allocation for
multi-vector stores
Select strided-or-contiguous psuedos for SME2 multi-vector stores,
allowing register allocation to choose the most profitable tuple layout.
The pseudos can then be expanded to choose the appropriate contiguous
or strided instruction after allocation.
---
.../AArch64/AArch64ExpandPseudoInsts.cpp | 64 ++
.../Target/AArch64/AArch64ISelLowering.cpp | 12 -
llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 32 +
.../Target/AArch64/AArch64RegisterInfo.cpp | 29 +-
.../lib/Target/AArch64/AArch64SVEInstrInfo.td | 114 ++-
llvm/lib/Target/AArch64/SVEInstrFormats.td | 14 +-
.../regalloc-hint-movprfx-streaming.mir | 39 +-
.../CodeGen/AArch64/sve-ldst-multi-vec.mir | 36 +
.../AArch64/sve-multivector-load-stores.ll | 769 +++---------------
.../CodeGen/AArch64/sve-vector-interleave.ll | 4 +-
.../AArch64/sve2p1-intrinsics-stores.ll | 487 ++++++++++-
11 files changed, 916 insertions(+), 684 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
index 53f7fdaab5522..3965ff1783bc5 100644
--- a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
@@ -1840,6 +1840,38 @@ bool AArch64ExpandPseudoImpl::expandMI(MachineBasicBlock &MBB,
return expandMultiVecPseudo(
MBB, MBBI, AArch64::ZPR2RegClass, AArch64::ZPR2StridedRegClass,
AArch64::LDNT1D_2Z_IMM, AArch64::LDNT1D_2Z_STRIDED_IMM);
+ case AArch64::ST1B_2Z_IMM_PSEUDO:
+ return expandMultiVecPseudo(
+ MBB, MBBI, AArch64::ZPR2RegClass, AArch64::ZPR2StridedRegClass,
+ AArch64::ST1B_2Z_IMM, AArch64::ST1B_2Z_STRIDED_IMM);
+ case AArch64::ST1H_2Z_IMM_PSEUDO:
+ return expandMultiVecPseudo(
+ MBB, MBBI, AArch64::ZPR2RegClass, AArch64::ZPR2StridedRegClass,
+ AArch64::ST1H_2Z_IMM, AArch64::ST1H_2Z_STRIDED_IMM);
+ case AArch64::ST1W_2Z_IMM_PSEUDO:
+ return expandMultiVecPseudo(
+ MBB, MBBI, AArch64::ZPR2RegClass, AArch64::ZPR2StridedRegClass,
+ AArch64::ST1W_2Z_IMM, AArch64::ST1W_2Z_STRIDED_IMM);
+ case AArch64::ST1D_2Z_IMM_PSEUDO:
+ return expandMultiVecPseudo(
+ MBB, MBBI, AArch64::ZPR2RegClass, AArch64::ZPR2StridedRegClass,
+ AArch64::ST1D_2Z_IMM, AArch64::ST1D_2Z_STRIDED_IMM);
+ case AArch64::STNT1B_2Z_IMM_PSEUDO:
+ return expandMultiVecPseudo(
+ MBB, MBBI, AArch64::ZPR2RegClass, AArch64::ZPR2StridedRegClass,
+ AArch64::STNT1B_2Z_IMM, AArch64::STNT1B_2Z_STRIDED_IMM);
+ case AArch64::STNT1H_2Z_IMM_PSEUDO:
+ return expandMultiVecPseudo(
+ MBB, MBBI, AArch64::ZPR2RegClass, AArch64::ZPR2StridedRegClass,
+ AArch64::STNT1H_2Z_IMM, AArch64::STNT1H_2Z_STRIDED_IMM);
+ case AArch64::STNT1W_2Z_IMM_PSEUDO:
+ return expandMultiVecPseudo(
+ MBB, MBBI, AArch64::ZPR2RegClass, AArch64::ZPR2StridedRegClass,
+ AArch64::STNT1W_2Z_IMM, AArch64::STNT1W_2Z_STRIDED_IMM);
+ case AArch64::STNT1D_2Z_IMM_PSEUDO:
+ return expandMultiVecPseudo(
+ MBB, MBBI, AArch64::ZPR2RegClass, AArch64::ZPR2StridedRegClass,
+ AArch64::STNT1D_2Z_IMM, AArch64::STNT1D_2Z_STRIDED_IMM);
case AArch64::LD1B_2Z_PSEUDO:
return expandMultiVecPseudo(MBB, MBBI, AArch64::ZPR2RegClass,
AArch64::ZPR2StridedRegClass, AArch64::LD1B_2Z,
@@ -1904,6 +1936,38 @@ bool AArch64ExpandPseudoImpl::expandMI(MachineBasicBlock &MBB,
return expandMultiVecPseudo(
MBB, MBBI, AArch64::ZPR4RegClass, AArch64::ZPR4StridedRegClass,
AArch64::LDNT1D_4Z_IMM, AArch64::LDNT1D_4Z_STRIDED_IMM);
+ case AArch64::ST1B_4Z_IMM_PSEUDO:
+ return expandMultiVecPseudo(
+ MBB, MBBI, AArch64::ZPR4RegClass, AArch64::ZPR4StridedRegClass,
+ AArch64::ST1B_4Z_IMM, AArch64::ST1B_4Z_STRIDED_IMM);
+ case AArch64::ST1H_4Z_IMM_PSEUDO:
+ return expandMultiVecPseudo(
+ MBB, MBBI, AArch64::ZPR4RegClass, AArch64::ZPR4StridedRegClass,
+ AArch64::ST1H_4Z_IMM, AArch64::ST1H_4Z_STRIDED_IMM);
+ case AArch64::ST1W_4Z_IMM_PSEUDO:
+ return expandMultiVecPseudo(
+ MBB, MBBI, AArch64::ZPR4RegClass, AArch64::ZPR4StridedRegClass,
+ AArch64::ST1W_4Z_IMM, AArch64::ST1W_4Z_STRIDED_IMM);
+ case AArch64::ST1D_4Z_IMM_PSEUDO:
+ return expandMultiVecPseudo(
+ MBB, MBBI, AArch64::ZPR4RegClass, AArch64::ZPR4StridedRegClass,
+ AArch64::ST1D_4Z_IMM, AArch64::ST1D_4Z_STRIDED_IMM);
+ case AArch64::STNT1B_4Z_IMM_PSEUDO:
+ return expandMultiVecPseudo(
+ MBB, MBBI, AArch64::ZPR4RegClass, AArch64::ZPR4StridedRegClass,
+ AArch64::STNT1B_4Z_IMM, AArch64::STNT1B_4Z_STRIDED_IMM);
+ case AArch64::STNT1H_4Z_IMM_PSEUDO:
+ return expandMultiVecPseudo(
+ MBB, MBBI, AArch64::ZPR4RegClass, AArch64::ZPR4StridedRegClass,
+ AArch64::STNT1H_4Z_IMM, AArch64::STNT1H_4Z_STRIDED_IMM);
+ case AArch64::STNT1W_4Z_IMM_PSEUDO:
+ return expandMultiVecPseudo(
+ MBB, MBBI, AArch64::ZPR4RegClass, AArch64::ZPR4StridedRegClass,
+ AArch64::STNT1W_4Z_IMM, AArch64::STNT1W_4Z_STRIDED_IMM);
+ case AArch64::STNT1D_4Z_IMM_PSEUDO:
+ return expandMultiVecPseudo(
+ MBB, MBBI, AArch64::ZPR4RegClass, AArch64::ZPR4StridedRegClass,
+ AArch64::STNT1D_4Z_IMM, AArch64::STNT1D_4Z_STRIDED_IMM);
case AArch64::LD1B_4Z_PSEUDO:
return expandMultiVecPseudo(MBB, MBBI, AArch64::ZPR4RegClass,
AArch64::ZPR4StridedRegClass, AArch64::LD1B_4Z,
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 1c649261752df..c895ab1b37daa 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -141,12 +141,6 @@ static cl::opt<bool> EnableExtToTBL("aarch64-enable-ext-to-tbl", cl::Hidden,
cl::desc("Combine ext and trunc to TBL"),
cl::init(true));
-static cl::opt<bool> EnableSME2MultiVectorStoreLowering(
- "aarch64-enable-sme2-multivector-store-lowering", cl::Hidden,
- cl::desc("Enable lowering of oversized SVE stores to SME2 multi-vector "
- "operations."),
- cl::init(false));
-
// All of the XOR, OR and CMP use ALU ports, and data dependency will become the
// bottleneck after this transform on high end CPU. So this max leaf node
// limitation is guard cmp+ccmp will be profitable.
@@ -2108,12 +2102,6 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
(Subtarget->hasSME2() && Subtarget->isStreaming()))) {
for (unsigned Opcode : {ISD::LOAD, ISD::STORE}) {
- // FIXME: Remove this guard on SME2 once new register classes and
- // pseudos for multi-vector stores have been added.
- if (Opcode == ISD::STORE && Subtarget->hasSME2() &&
- Subtarget->isStreaming() && !EnableSME2MultiVectorStoreLowering)
- continue;
-
// 2x multi-vector load/stores
setOperationAction(Opcode, MVT::nxv32i8, Custom);
setOperationAction(Opcode, MVT::nxv16i16, Custom);
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index ec4fe39aafd80..a4e80a6db2814 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -3307,6 +3307,14 @@ unsigned AArch64InstrInfo::getLoadStoreImmIdx(unsigned Opc) {
case AArch64::STNT1W_2Z_STRIDED_IMM:
case AArch64::STNT1D_2Z_IMM:
case AArch64::STNT1D_2Z_STRIDED_IMM:
+ case AArch64::ST1B_2Z_IMM_PSEUDO:
+ case AArch64::ST1H_2Z_IMM_PSEUDO:
+ case AArch64::ST1W_2Z_IMM_PSEUDO:
+ case AArch64::ST1D_2Z_IMM_PSEUDO:
+ case AArch64::STNT1B_2Z_IMM_PSEUDO:
+ case AArch64::STNT1H_2Z_IMM_PSEUDO:
+ case AArch64::STNT1W_2Z_IMM_PSEUDO:
+ case AArch64::STNT1D_2Z_IMM_PSEUDO:
case AArch64::ST1B_4Z_IMM:
case AArch64::ST1B_4Z_STRIDED_IMM:
case AArch64::ST1H_4Z_IMM:
@@ -3335,6 +3343,14 @@ unsigned AArch64InstrInfo::getLoadStoreImmIdx(unsigned Opc) {
case AArch64::STNT1W_4Z_STRIDED_IMM:
case AArch64::STNT1D_4Z_IMM:
case AArch64::STNT1D_4Z_STRIDED_IMM:
+ case AArch64::ST1B_4Z_IMM_PSEUDO:
+ case AArch64::ST1H_4Z_IMM_PSEUDO:
+ case AArch64::ST1W_4Z_IMM_PSEUDO:
+ case AArch64::ST1D_4Z_IMM_PSEUDO:
+ case AArch64::STNT1B_4Z_IMM_PSEUDO:
+ case AArch64::STNT1H_4Z_IMM_PSEUDO:
+ case AArch64::STNT1W_4Z_IMM_PSEUDO:
+ case AArch64::STNT1D_4Z_IMM_PSEUDO:
return 3;
case AArch64::LDPDpost:
case AArch64::LDPDpre:
@@ -5053,6 +5069,14 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale,
case AArch64::STNT1W_2Z_STRIDED_IMM:
case AArch64::STNT1D_2Z_IMM:
case AArch64::STNT1D_2Z_STRIDED_IMM:
+ case AArch64::ST1B_2Z_IMM_PSEUDO:
+ case AArch64::ST1H_2Z_IMM_PSEUDO:
+ case AArch64::ST1W_2Z_IMM_PSEUDO:
+ case AArch64::ST1D_2Z_IMM_PSEUDO:
+ case AArch64::STNT1B_2Z_IMM_PSEUDO:
+ case AArch64::STNT1H_2Z_IMM_PSEUDO:
+ case AArch64::STNT1W_2Z_IMM_PSEUDO:
+ case AArch64::STNT1D_2Z_IMM_PSEUDO:
Scale = Width = TypeSize::getScalable(16 * 2);
MinOffset = -8;
MaxOffset = 7;
@@ -5117,6 +5141,14 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale,
case AArch64::STNT1W_4Z_STRIDED_IMM:
case AArch64::STNT1D_4Z_IMM:
case AArch64::STNT1D_4Z_STRIDED_IMM:
+ case AArch64::ST1B_4Z_IMM_PSEUDO:
+ case AArch64::ST1H_4Z_IMM_PSEUDO:
+ case AArch64::ST1W_4Z_IMM_PSEUDO:
+ case AArch64::ST1D_4Z_IMM_PSEUDO:
+ case AArch64::STNT1B_4Z_IMM_PSEUDO:
+ case AArch64::STNT1H_4Z_IMM_PSEUDO:
+ case AArch64::STNT1W_4Z_IMM_PSEUDO:
+ case AArch64::STNT1D_4Z_IMM_PSEUDO:
Scale = Width = TypeSize::getScalable(16 * 4);
MinOffset = -8;
MaxOffset = 7;
diff --git a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
index edca3e94a0afe..bdcd6bd8a6c07 100644
--- a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
@@ -1196,14 +1196,31 @@ bool AArch64RegisterInfo::getRegAllocationHints(
const AArch64InstrInfo *TII =
MF.getSubtarget<AArch64Subtarget>().getInstrInfo();
const MachineRegisterInfo &MRI = MF.getRegInfo();
+ const TargetRegisterClass *RegRC = MRI.getRegClass(VirtReg);
+ unsigned RegID = RegRC->getID();
bool ConsiderOnlyHints =
TargetRegisterInfo::getRegAllocationHints(VirtReg, Order, Hints, MF, VRM);
+ // Discard generic strided-tuple hints that would use an otherwise unused
+ // callee-saved Z register. Registers already defined or allocated do not
+ // introduce an additional spill.
+ if (ST.hasSME() && ST.isStreaming() &&
+ (RegID == AArch64::ZPR2StridedOrContiguousRegClassID ||
+ RegID == AArch64::ZPR4StridedOrContiguousRegClassID)) {
+ llvm::erase_if(Hints, [&](MCPhysReg Hint) {
+ for (const MCPhysReg *CSR = MRI.getCalleeSavedRegs(); *CSR; ++CSR) {
+ if (regsOverlap(Hint, *CSR) && MRI.def_empty(*CSR) &&
+ !Matrix->isPhysRegUsed(*CSR))
+ return true;
+ }
+ return false;
+ });
+ }
+
// For predicated SVE instructions where the inactive lanes are undef,
// pick a destination register that is not unique to avoid introducing
// a movprfx.
- const TargetRegisterClass *RegRC = MRI.getRegClass(VirtReg);
if (AArch64::ZPRRegClass.hasSubClassEq(RegRC)) {
for (const MachineOperand &DefOp : MRI.def_operands(VirtReg)) {
const MachineInstr &Def = *DefOp.getParent();
@@ -1266,8 +1283,7 @@ bool AArch64RegisterInfo::getRegAllocationHints(
return ConsiderOnlyHints;
if (!ST.hasSME() || !ST.isStreaming())
- return TargetRegisterInfo::getRegAllocationHints(VirtReg, Order, Hints, MF,
- VRM);
+ return ConsiderOnlyHints;
// The SVE calling convention preserves registers Z8-Z23. As a result, there
// are no ZPR2Strided or ZPR4Strided registers that do not overlap with the
@@ -1277,7 +1293,6 @@ bool AArch64RegisterInfo::getRegAllocationHints(
// COPY_INTO_TRANSPOSED_TUPLE pseudos, we want to favour reducing copy
// instructions over reducing the number of clobbered callee-save registers,
// so we add the strided registers as a hint.
- unsigned RegID = RegRC->getID();
if (RegID == AArch64::ZPR2StridedOrContiguousRegClassID ||
RegID == AArch64::ZPR4StridedOrContiguousRegClassID) {
@@ -1409,8 +1424,7 @@ bool AArch64RegisterInfo::getRegAllocationHints(
}
if (!Hints.empty())
- return TargetRegisterInfo::getRegAllocationHints(VirtReg, Order, Hints,
- MF, VRM);
+ return ConsiderOnlyHints;
}
}
@@ -1441,8 +1455,7 @@ bool AArch64RegisterInfo::getRegAllocationHints(
}
}
- return TargetRegisterInfo::getRegAllocationHints(VirtReg, Order, Hints, MF,
- VRM);
+ return ConsiderOnlyHints;
}
unsigned AArch64RegisterInfo::getLocalAddressRegister(
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 68e66e51dae63..6c00bfade217c 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -4586,36 +4586,36 @@ def ST1B_2Z : sve2p1_mem_cst_ss_2z<"st1b", 0b00, 0b0, ZZ_b_mul_r, GPR64sh
def ST1H_2Z : sve2p1_mem_cst_ss_2z<"st1h", 0b01, 0b0, ZZ_h_mul_r, GPR64shifted16>;
def ST1W_2Z : sve2p1_mem_cst_ss_2z<"st1w", 0b10, 0b0, ZZ_s_mul_r, GPR64shifted32>;
def ST1D_2Z : sve2p1_mem_cst_ss_2z<"st1d", 0b11, 0b0, ZZ_d_mul_r, GPR64shifted64>;
-defm ST1B_2Z_IMM : sve2p1_mem_cst_si_2z<"st1b", 0b00, 0b0, ZZ_b_mul_r>;
-defm ST1H_2Z_IMM : sve2p1_mem_cst_si_2z<"st1h", 0b01, 0b0, ZZ_h_mul_r>;
-defm ST1W_2Z_IMM : sve2p1_mem_cst_si_2z<"st1w", 0b10, 0b0, ZZ_s_mul_r>;
-defm ST1D_2Z_IMM : sve2p1_mem_cst_si_2z<"st1d", 0b11, 0b0, ZZ_d_mul_r>;
+defm ST1B_2Z_IMM : sve2p1_mem_cst_si_2z<"st1b", 0b00, 0b0, ZZ_b_mul_r, ZZ_b_strided_and_contiguous>;
+defm ST1H_2Z_IMM : sve2p1_mem_cst_si_2z<"st1h", 0b01, 0b0, ZZ_h_mul_r, ZZ_h_strided_and_contiguous>;
+defm ST1W_2Z_IMM : sve2p1_mem_cst_si_2z<"st1w", 0b10, 0b0, ZZ_s_mul_r, ZZ_s_strided_and_contiguous>;
+defm ST1D_2Z_IMM : sve2p1_mem_cst_si_2z<"st1d", 0b11, 0b0, ZZ_d_mul_r, ZZ_d_strided_and_contiguous>;
def STNT1B_2Z : sve2p1_mem_cst_ss_2z<"stnt1b", 0b00, 0b1, ZZ_b_mul_r, GPR64shifted8>;
def STNT1H_2Z : sve2p1_mem_cst_ss_2z<"stnt1h", 0b01, 0b1, ZZ_h_mul_r, GPR64shifted16>;
def STNT1W_2Z : sve2p1_mem_cst_ss_2z<"stnt1w", 0b10, 0b1, ZZ_s_mul_r, GPR64shifted32>;
def STNT1D_2Z : sve2p1_mem_cst_ss_2z<"stnt1d", 0b11, 0b1, ZZ_d_mul_r, GPR64shifted64>;
-defm STNT1B_2Z_IMM : sve2p1_mem_cst_si_2z<"stnt1b", 0b00, 0b1, ZZ_b_mul_r>;
-defm STNT1H_2Z_IMM : sve2p1_mem_cst_si_2z<"stnt1h", 0b01, 0b1, ZZ_h_mul_r>;
-defm STNT1W_2Z_IMM : sve2p1_mem_cst_si_2z<"stnt1w", 0b10, 0b1, ZZ_s_mul_r>;
-defm STNT1D_2Z_IMM : sve2p1_mem_cst_si_2z<"stnt1d", 0b11, 0b1, ZZ_d_mul_r>;
+defm STNT1B_2Z_IMM : sve2p1_mem_cst_si_2z<"stnt1b", 0b00, 0b1, ZZ_b_mul_r, ZZ_b_strided_and_contiguous>;
+defm STNT1H_2Z_IMM : sve2p1_mem_cst_si_2z<"stnt1h", 0b01, 0b1, ZZ_h_mul_r, ZZ_h_strided_and_contiguous>;
+defm STNT1W_2Z_IMM : sve2p1_mem_cst_si_2z<"stnt1w", 0b10, 0b1, ZZ_s_mul_r, ZZ_s_strided_and_contiguous>;
+defm STNT1D_2Z_IMM : sve2p1_mem_cst_si_2z<"stnt1d", 0b11, 0b1, ZZ_d_mul_r, ZZ_d_strided_and_contiguous>;
// Stores of four registers
def ST1B_4Z : sve2p1_mem_cst_ss_4z<"st1b", 0b00, 0b0, ZZZZ_b_mul_r, GPR64shifted8>;
def ST1H_4Z : sve2p1_mem_cst_ss_4z<"st1h", 0b01, 0b0, ZZZZ_h_mul_r, GPR64shifted16>;
def ST1W_4Z : sve2p1_mem_cst_ss_4z<"st1w", 0b10, 0b0, ZZZZ_s_mul_r, GPR64shifted32>;
def ST1D_4Z : sve2p1_mem_cst_ss_4z<"st1d", 0b11, 0b0, ZZZZ_d_mul_r, GPR64shifted64>;
-defm ST1B_4Z_IMM : sve2p1_mem_cst_si_4z<"st1b", 0b00, 0b0, ZZZZ_b_mul_r>;
-defm ST1H_4Z_IMM : sve2p1_mem_cst_si_4z<"st1h", 0b01, 0b0, ZZZZ_h_mul_r>;
-defm ST1W_4Z_IMM : sve2p1_mem_cst_si_4z<"st1w", 0b10, 0b0, ZZZZ_s_mul_r>;
-defm ST1D_4Z_IMM : sve2p1_mem_cst_si_4z<"st1d", 0b11, 0b0, ZZZZ_d_mul_r>;
+defm ST1B_4Z_IMM : sve2p1_mem_cst_si_4z<"st1b", 0b00, 0b0, ZZZZ_b_mul_r, ZZZZ_b_strided_and_contiguous>;
+defm ST1H_4Z_IMM : sve2p1_mem_cst_si_4z<"st1h", 0b01, 0b0, ZZZZ_h_mul_r, ZZZZ_h_strided_and_contiguous>;
+defm ST1W_4Z_IMM : sve2p1_mem_cst_si_4z<"st1w", 0b10, 0b0, ZZZZ_s_mul_r, ZZZZ_s_strided_and_contiguous>;
+defm ST1D_4Z_IMM : sve2p1_mem_cst_si_4z<"st1d", 0b11, 0b0, ZZZZ_d_mul_r, ZZZZ_d_strided_and_contiguous>;
def STNT1B_4Z : sve2p1_mem_cst_ss_4z<"stnt1b", 0b00, 0b1, ZZZZ_b_mul_r, GPR64shifted8>;
def STNT1H_4Z : sve2p1_mem_cst_ss_4z<"stnt1h", 0b01, 0b1, ZZZZ_h_mul_r, GPR64shifted16>;
def STNT1W_4Z : sve2p1_mem_cst_ss_4z<"stnt1w", 0b10, 0b1, ZZZZ_s_mul_r, GPR64shifted32>;
def STNT1D_4Z : sve2p1_mem_cst_ss_4z<"stnt1d", 0b11, 0b1, ZZZZ_d_mul_r, GPR64shifted64>;
-defm STNT1B_4Z_IMM : sve2p1_mem_cst_si_4z<"stnt1b", 0b00, 0b1, ZZZZ_b_mul_r>;
-defm STNT1H_4Z_IMM : sve2p1_mem_cst_si_4z<"stnt1h", 0b01, 0b1, ZZZZ_h_mul_r>;
-defm STNT1W_4Z_IMM : sve2p1_mem_cst_si_4z<"stnt1w", 0b10, 0b1, ZZZZ_s_mul_r>;
-defm STNT1D_4Z_IMM : sve2p1_mem_cst_si_4z<"stnt1d", 0b11, 0b1, ZZZZ_d_mul_r>;
+defm STNT1B_4Z_IMM : sve2p1_mem_cst_si_4z<"stnt1b", 0b00, 0b1, ZZZZ_b_mul_r, ZZZZ_b_strided_and_contiguous>;
+defm STNT1H_4Z_IMM : sve2p1_mem_cst_si_4z<"stnt1h", 0b01, 0b1, ZZZZ_h_mul_r, ZZZZ_h_strided_and_contiguous>;
+defm STNT1W_4Z_IMM : sve2p1_mem_cst_si_4z<"stnt1w", 0b10, 0b1, ZZZZ_s_mul_r, ZZZZ_s_strided_and_contiguous>;
+defm STNT1D_4Z_IMM : sve2p1_mem_cst_si_4z<"stnt1d", 0b11, 0b1, ZZZZ_d_mul_r, ZZZZ_d_strided_and_contiguous>;
multiclass store_pn_x2<ValueType Ty, SDPatternOperator Store,
Instruction RegImmInst> {
@@ -4697,6 +4697,88 @@ defm WHILELO_CXX : sve2p1_int_while_rr_pn<"whilelo", 0b110, AArch64whilelo_pn_fl
defm WHILELS_CXX : sve2p1_int_while_rr_pn<"whilels", 0b111, AArch64whilels_pn_flag>;
} // End HasSVE2p1_or_StreamingSME2
+multiclass store_pn_x2_sme2<ValueType Ty, SDPatternOperator Store,
+ Instruction RegImmPseudo> {
+ let AddedComplexity = 2 in {
+ // scalar + immediate (mul vl)
+ def : Pat<(Store Ty:$vec0, Ty:$vec1, aarch64svcount:$PNg,
+ (am_sve_indexed_s4 GPR64sp:$base, simm4s1:$offset)),
+ (RegImmPseudo
+ (REG_SEQUENCE ZPR2StridedOrContiguous,
+ Ty:$vec0, zsub0, Ty:$vec1, zsub1),
+ PNR:$PNg, GPR64:$base, simm4s1:$offset)>;
+ }
+
+ let AddedComplexity = 1 in
+ def : Pat<(Store Ty:$vec0, Ty:$vec1, aarch64svcount:$PNg, GPR64:$base),
+ (RegImmPseudo
+ (REG_SEQUENCE ZPR2StridedOrContiguous,
+ Ty:$vec0, zsub0, Ty:$vec1, zsub1),
+ PNR:$PNg, GPR64:$base, (i64 0))>;
+}
+
+multiclass store_pn_x4_sme2<ValueType Ty, SDPatternOperator Store,
+ Instruction RegImmPseudo> {
+ let AddedComplexity = 2 in {
+ // scalar + immediate (mul vl)
+ def : Pat<(Store Ty:$vec0, Ty:$vec1, Ty:$vec2, Ty:$vec3,
+ aarch64svcount:$PNg,
+ (am_sve_indexed_s4 GPR64sp:$base, simm4s1:$offset)),
+ (RegImmPseudo
+ (REG_SEQUENCE ZPR4StridedOrContiguous,
+ Ty:$vec0, zsub0, Ty:$vec1, zsub1,
+ Ty:$vec2, zsub2, Ty:$vec3, zsub3),
+ PNR:$PNg, GPR64:$base, simm4s1:$offset)>;
+ }
+
+ let AddedComplexity = 1 in
+ def : Pat<(Store Ty:$vec0, Ty:$vec1, Ty:$vec2, Ty:$vec3,
+ aarch64svcount:$PNg, GPR64:$base),
+ (RegImmPseudo
+ (REG_SEQUENCE ZPR4StridedOrContiguous,
+ Ty:$vec0, zsub0, Ty:$vec1, zsub1,
+ Ty:$vec2, zsub2, Ty:$vec3, zsub3),
+ PNR:$PNg, GPR64:$base, (i64 0))>;
+}
+
+let Predicates = [HasSME2] in {
+// Stores of 2 strided or contiguous vectors
+defm : store_pn_x2_sme2<nxv16i8, int_aarch64_sve_st1_pn_x2, ST1B_2Z_IMM_PSEUDO>;
+defm : store_pn_x2_sme2<nxv8i16, int_aarch64_sve_st1_pn_x2, ST1H_2Z_IMM_PSEUDO>;
+defm : store_pn_x2_sme2<nxv4i32, int_aarch64_sve_st1_pn_x2, ST1W_2Z_IMM_PSEUDO>;
+defm : store_pn_x2_sme2<nxv2i64, int_aarch64_sve_st1_pn_x2, ST1D_2Z_IMM_PSEUDO>;
+defm : store_pn_x2_sme2<nxv16i8, int_aarch64_sve_stnt1_pn_x2, STNT1B_2Z_IMM_PSEUDO>;
+defm : store_pn_x2_sme2<nxv8i16, int_aarch64_sve_stnt1_pn_x2, STNT1H_2Z_IMM_PSEUDO>;
+defm : store_pn_x2_sme2<nxv4i32, int_aarch64_sve_stnt1_pn_x2, STNT1W_2Z_IMM_PSEUDO>;
+defm : store_pn_x2_sme2<nxv2i64, int_aarch64_sve_stnt1_pn_x2, STNT1D_2Z_IMM_PSEUDO>;
+defm : store_pn_x2_sme2<nxv8f16, int_aarch64_sve_st1_pn_x2, ST1H_2Z_IMM_PSEUDO>;
+defm : store_pn_x2_sme2<nxv8bf16, int_aarch64_sve_st1_pn_x2, ST1H_2Z_IMM_PSEUDO>;
+defm : store_pn_x2_sme2<nxv4f32, int_aarch64_sve_st1_pn_x2, ST1W_2Z_IMM_PSEUDO>;
+defm : store_pn_x2_sme2<nxv2f64, int_aarch64_sve_st1_pn_x2, ST1D_2Z_IMM_PSEUDO>;
+defm : store_pn_x2_sme2<nxv8f16, int_aarch64_sve_stnt1_pn_x2, STNT1H_2Z_IMM_PSEUDO>;
+defm : store_pn_x2_sme2<nxv8bf16, int_aarch64_sve_stnt1_pn_x2, STNT1H_2Z_IMM_PSEUDO>;
+defm : store_pn_x2_sme2<nxv4f32, int_aarch64_sve_stnt1_pn_x2, STNT1W_2Z_IMM_PSEUDO>;
+defm : store_pn_x2_sme2<nxv2f64, int_aarch64_sve_stnt1_pn_x2, STNT1D_2Z_IMM_PSEUDO>;
+
+// Stores of 4 strided or contiguous vectors
+defm : store_pn_x4_sme2<nxv16i8, int_aarch64_sve_st1_pn_x4, ST1B_4Z_IMM_PSEUDO>;
+defm : store_pn_x4_sme2<nxv8i16, int_aarch64_sve_st1_pn_x4, ST1H_4Z_IMM_PSEUDO>;
+defm : store_pn_x4_sme2<nxv4i32, int_aarch64_sve_st1_pn_x4, ST1W_4Z_IMM_PSEUDO>;
+defm : store_pn_x4_sme2<nxv2i64, int_aarch64_sve_st1_pn_x4, ST1D_4Z_IMM_PSEUDO>;
+defm : store_pn_x4_sme2<nxv16i8, int_aarch64_sve_stnt1_pn_x4, STNT1B_4Z_IMM_PSEUDO>;
+defm : store_pn_x4_sme2<nxv8i16, int_aarch64_sve_stnt1_pn_x4, STNT1H_4Z_IMM_PSEUDO>;
+defm : store_pn_x4_sme2<nxv4i32, int_aarch64_sve_stnt1_pn_x4, STNT1W_4Z_IMM_PSEUDO>;
+defm : store_pn_x4_sme2<nxv2i64, int_aarch64_sve_stnt1_pn_x4, STNT1D_4Z_IMM_PSEUDO>;
+defm : store_pn_x4_sme2<nxv8f16, int_aarch64_sve_st1_pn_x4, ST1H_4Z_IMM_PSEUDO>;
+defm : store_pn_x4_sme2<nxv8bf16, int_aarch64_sve_st1_pn_x4, ST1H_4Z_IMM_PSEUDO>;
+defm : store_pn_x4_sme2<nxv4f32, int_aarch64_sve_st1_pn_x4, ST1W_4Z_IMM_PSEUDO>;
+defm : store_pn_x4_sme2<nxv2f64, int_aarch64_sve_st1_pn_x4, ST1D_4Z_IMM_PSEUDO>;
+defm : store_pn_x4_sme2<nxv8f16, int_aarch64_sve_stnt1_pn_x4, STNT1H_4Z_IMM_PSEUDO>;
+defm : store_pn_x4_sme2<nxv8bf16, int_aarch64_sve_stnt1_pn_x4, STNT1H_4Z_IMM_PSEUDO>;
+defm : store_pn_x4_sme2<nxv4f32, int_aarch64_sve_stnt1_pn_x4, STNT1W_4Z_IMM_PSEUDO>;
+defm : store_pn_x4_sme2<nxv2f64, int_aarch64_sve_stnt1_pn_x4, STNT1D_4Z_IMM_PSEUDO>;
+}
+
let Predicates = [HasSVE_or_SME] in {
// Aliases for existing SVE instructions for which predicate-as-counter are
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index b1f1e08c4adfa..1cf0b23fbb484 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -10418,11 +10418,16 @@ class sve2p1_mem_cst_si_2z<string mnemonic, bits<2> msz, bit n,
multiclass sve2p1_mem_cst_si_2z<string mnemonic, bits<2> msz, bit n,
- RegisterOperand vector_ty> {
+ RegisterOperand vector_ty,
+ RegisterOperand vector_pseudo_ty> {
def NAME : sve2p1_mem_cst_si_2z<mnemonic, msz, n, vector_ty>;
def : InstAlias<mnemonic # " $Zt, $PNg, [$Rn]",
(!cast<Instruction>(NAME) vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, 0), 1>;
+
+ let hasSideEffects = 0, mayStore = 1 in
+ def NAME # _PSEUDO : Pseudo<(outs), (ins vector_pseudo_ty:$Zt,
+ PNRAny_p8to15:$PNg, GPR64sp:$Rn, simm4s2:$imm4), []>;
}
@@ -10478,11 +10483,16 @@ class sve2p1_mem_cst_si_4z<string mnemonic, bits<2> msz, bit n,
multiclass sve2p1_mem_cst_si_4z<string mnemonic, bits<2> msz, bit n,
- RegisterOperand vector_ty> {
+ RegisterOperand vector_ty,
+ RegisterOperand vector_pseudo_ty> {
def NAME : sve2p1_mem_cst_si_4z<mnemonic, msz, n, vector_ty>;
def : InstAlias<mnemonic # " $Zt, $PNg, [$Rn]",
(!cast<Instruction>(NAME) vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn,0), 1>;
+
+ let hasSideEffects = 0, mayStore = 1 in
+ def NAME # _PSEUDO : Pseudo<(outs), (ins vector_pseudo_ty:$Zt,
+ PNRAny_p8to15:$PNg, GPR64sp:$Rn, simm4s4:$imm4), []>;
}
// SVE predicate count (predicate-as-counter)
diff --git a/llvm/test/CodeGen/AArch64/regalloc-hint-movprfx-streaming.mir b/llvm/test/CodeGen/AArch64/regalloc-hint-movprfx-streaming.mir
index 5e86b863fffe6..f451fa51530d4 100644
--- a/llvm/test/CodeGen/AArch64/regalloc-hint-movprfx-streaming.mir
+++ b/llvm/test/CodeGen/AArch64/regalloc-hint-movprfx-streaming.mir
@@ -1,5 +1,6 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -mtriple=aarch64 -mattr=+sme2 -force-streaming -start-before=greedy -stop-after=virtregrewriter %s -o - | FileCheck %s
+# RUN: llc -mtriple=aarch64 -mattr=+sme2 -force-streaming -start-before=greedy -stop-after=virtregrewriter -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=aarch64 -mattr=+sme2 -force-streaming -start-before=greedy -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=ASM
# Check the register allocator gets hints to reuse registers of one of its operands.
---
@@ -85,3 +86,39 @@ body: |
STR_ZXI %8, %1, 0
RET_ReallyLR
...
+
+# Check the load, arithmetic and store can retain the same strided tuple.
+---
+name: retain_strided_tuple_for_store
+tracksRegLiveness: true
+isSSA: false
+noVRegs: false
+body: |
+ bb.0.entry:
+ liveins: $x0
+
+ ; CHECK-LABEL: name: retain_strided_tuple_for_store
+ ; CHECK: liveins: $x0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: renamable $pn8 = PTRUE_C_B implicit $vg
+ ; CHECK-NEXT: renamable $z16_z24 = LD1B_2Z_IMM_PSEUDO renamable $pn8, renamable $x0, 0
+ ; CHECK-NEXT: renamable $z16 = ADD_ZI_B_PSEUDO renamable $z16, 5, 0
+ ; CHECK-NEXT: renamable $z24 = ADD_ZI_B_PSEUDO renamable $z24, 5, 0
+ ; CHECK-NEXT: ST1B_2Z_IMM_PSEUDO killed renamable $z16_z24, killed renamable $pn8, killed renamable $x0, 0
+ ; CHECK-NEXT: RET_ReallyLR
+
+ ; ASM-LABEL: retain_strided_tuple_for_store:
+ ; ASM-NOT: movprfx
+ ; ASM: ld1b { z16.b, z24.b }, pn8/z, [x0]
+ ; ASM-NEXT: add z16.b, z16.b, #5
+ ; ASM-NEXT: add z24.b, z24.b, #5
+ ; ASM-NEXT: st1b { z16.b, z24.b }, pn8, [x0]
+
+ %0:gpr64common = COPY $x0
+ %1:pnr_p8to15 = PTRUE_C_B implicit $vg
+ %2:zpr2stridedorcontiguous = LD1B_2Z_IMM_PSEUDO %1, %0, 0
+ %2.zsub0 = ADD_ZI_B_PSEUDO %2.zsub0, 5, 0
+ %2.zsub1 = ADD_ZI_B_PSEUDO %2.zsub1, 5, 0
+ ST1B_2Z_IMM_PSEUDO killed %2, killed %1, killed %0, 0
+ RET_ReallyLR
+...
diff --git a/llvm/test/CodeGen/AArch64/sve-ldst-multi-vec.mir b/llvm/test/CodeGen/AArch64/sve-ldst-multi-vec.mir
index 021af943015a6..92328da3f9228 100644
--- a/llvm/test/CodeGen/AArch64/sve-ldst-multi-vec.mir
+++ b/llvm/test/CodeGen/AArch64/sve-ldst-multi-vec.mir
@@ -115,6 +115,10 @@ body: |
; CHECK-NEXT: STNT1W_4Z_STRIDED_IMM renamable $z16_z20_z24_z28, renamable $pn8, $sp, 7
; CHECK-NEXT: STNT1D_4Z_STRIDED_IMM renamable $z16_z20_z24_z28, renamable $pn8, $sp, -8
; CHECK-NEXT: STNT1D_4Z_STRIDED_IMM renamable $z16_z20_z24_z28, renamable $pn8, $sp, 7
+ ; CHECK-NEXT: ST1B_2Z_IMM_PSEUDO renamable $z16_z17, renamable $pn8, $sp, -8
+ ; CHECK-NEXT: STNT1D_2Z_IMM_PSEUDO renamable $z16_z24, renamable $pn8, $sp, 7
+ ; CHECK-NEXT: ST1W_4Z_IMM_PSEUDO renamable $z16_z17_z18_z19, renamable $pn8, $sp, -8
+ ; CHECK-NEXT: STNT1H_4Z_IMM_PSEUDO renamable $z16_z20_z24_z28, renamable $pn8, $sp, 7
; CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 31, implicit $vg
; CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 1, implicit $vg
; CHECK-NEXT: early-clobber $sp, $fp = frame-destroy LDRXpost $sp, 16 :: (load (s64) from %stack.1)
@@ -219,6 +223,10 @@ body: |
; CHECK-OFFSET-NEXT: stnt1w { z16.s, z20.s, z24.s, z28.s }, pn8, [sp, #28, mul vl]
; CHECK-OFFSET-NEXT: stnt1d { z16.d, z20.d, z24.d, z28.d }, pn8, [sp, #-32, mul vl]
; CHECK-OFFSET-NEXT: stnt1d { z16.d, z20.d, z24.d, z28.d }, pn8, [sp, #28, mul vl]
+ ; CHECK-OFFSET-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #-16, mul vl]
+ ; CHECK-OFFSET-NEXT: stnt1d { z16.d, z24.d }, pn8, [sp, #14, mul vl]
+ ; CHECK-OFFSET-NEXT: st1w { z16.s - z19.s }, pn8, [sp, #-32, mul vl]
+ ; CHECK-OFFSET-NEXT: stnt1h { z16.h, z20.h, z24.h, z28.h }, pn8, [sp, #28, mul vl]
; CHECK-OFFSET-NEXT: addvl sp, sp, #31
; CHECK-OFFSET-NEXT: addvl sp, sp, #1
; CHECK-OFFSET-NEXT: ldr x29, [sp], #16
@@ -336,6 +344,12 @@ body: |
STNT1D_4Z_STRIDED_IMM renamable $z16_z20_z24_z28, renamable $pn8, %stack.0, -8
STNT1D_4Z_STRIDED_IMM renamable $z16_z20_z24_z28, renamable $pn8, %stack.0, 7
+ ; Store pseudos with contiguous and strided physical tuples.
+ ST1B_2Z_IMM_PSEUDO renamable $z16_z17, renamable $pn8, %stack.0, -8
+ STNT1D_2Z_IMM_PSEUDO renamable $z16_z24, renamable $pn8, %stack.0, 7
+ ST1W_4Z_IMM_PSEUDO renamable $z16_z17_z18_z19, renamable $pn8, %stack.0, -8
+ STNT1H_4Z_IMM_PSEUDO renamable $z16_z20_z24_z28, renamable $pn8, %stack.0, 7
+
RET_ReallyLR
...
---
@@ -544,6 +558,14 @@ body: |
; CHECK-NEXT: STNT1D_4Z_STRIDED_IMM renamable $z16_z20_z24_z28, renamable $pn8, killed $x8, -8
; CHECK-NEXT: $x8 = ADDVL_XXI $sp, 4, implicit $vg
; CHECK-NEXT: STNT1D_4Z_STRIDED_IMM renamable $z16_z20_z24_z28, renamable $pn8, killed $x8, 7
+ ; CHECK-NEXT: $x8 = ADDVL_XXI $sp, -2, implicit $vg
+ ; CHECK-NEXT: ST1B_2Z_IMM_PSEUDO renamable $z16_z17, renamable $pn8, killed $x8, -8
+ ; CHECK-NEXT: $x8 = ADDVL_XXI $sp, 2, implicit $vg
+ ; CHECK-NEXT: STNT1D_2Z_IMM_PSEUDO renamable $z16_z24, renamable $pn8, killed $x8, 7
+ ; CHECK-NEXT: $x8 = ADDVL_XXI $sp, -4, implicit $vg
+ ; CHECK-NEXT: ST1W_4Z_IMM_PSEUDO renamable $z16_z17_z18_z19, renamable $pn8, killed $x8, -8
+ ; CHECK-NEXT: $x8 = ADDVL_XXI $sp, 4, implicit $vg
+ ; CHECK-NEXT: STNT1H_4Z_IMM_PSEUDO renamable $z16_z20_z24_z28, renamable $pn8, killed $x8, 7
; CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 31, implicit $vg
; CHECK-NEXT: $sp = frame-destroy ADDVL_XXI $sp, 1, implicit $vg
; CHECK-NEXT: early-clobber $sp, $fp = frame-destroy LDRXpost $sp, 16 :: (load (s64) from %stack.1)
@@ -744,6 +766,14 @@ body: |
; CHECK-OFFSET-NEXT: stnt1d { z16.d, z20.d, z24.d, z28.d }, pn8, [x8, #-32, mul vl]
; CHECK-OFFSET-NEXT: addvl x8, sp, #4
; CHECK-OFFSET-NEXT: stnt1d { z16.d, z20.d, z24.d, z28.d }, pn8, [x8, #28, mul vl]
+ ; CHECK-OFFSET-NEXT: addvl x8, sp, #-2
+ ; CHECK-OFFSET-NEXT: st1b { z16.b, z17.b }, pn8, [x8, #-16, mul vl]
+ ; CHECK-OFFSET-NEXT: addvl x8, sp, #2
+ ; CHECK-OFFSET-NEXT: stnt1d { z16.d, z24.d }, pn8, [x8, #14, mul vl]
+ ; CHECK-OFFSET-NEXT: addvl x8, sp, #-4
+ ; CHECK-OFFSET-NEXT: st1w { z16.s - z19.s }, pn8, [x8, #-32, mul vl]
+ ; CHECK-OFFSET-NEXT: addvl x8, sp, #4
+ ; CHECK-OFFSET-NEXT: stnt1h { z16.h, z20.h, z24.h, z28.h }, pn8, [x8, #28, mul vl]
; CHECK-OFFSET-NEXT: addvl sp, sp, #31
; CHECK-OFFSET-NEXT: addvl sp, sp, #1
; CHECK-OFFSET-NEXT: ldr x29, [sp], #16
@@ -861,5 +891,11 @@ body: |
STNT1D_4Z_STRIDED_IMM renamable $z16_z20_z24_z28, renamable $pn8, %stack.0, -9
STNT1D_4Z_STRIDED_IMM renamable $z16_z20_z24_z28, renamable $pn8, %stack.0, 8
+ ; Store pseudos with out-of-range frame offsets.
+ ST1B_2Z_IMM_PSEUDO renamable $z16_z17, renamable $pn8, %stack.0, -9
+ STNT1D_2Z_IMM_PSEUDO renamable $z16_z24, renamable $pn8, %stack.0, 8
+ ST1W_4Z_IMM_PSEUDO renamable $z16_z17_z18_z19, renamable $pn8, %stack.0, -9
+ STNT1H_4Z_IMM_PSEUDO renamable $z16_z20_z24_z28, renamable $pn8, %stack.0, 8
+
RET_ReallyLR
...
diff --git a/llvm/test/CodeGen/AArch64/sve-multivector-load-stores.ll b/llvm/test/CodeGen/AArch64/sve-multivector-load-stores.ll
index 5b8a5d52b49fc..714b6a0874cd7 100644
--- a/llvm/test/CodeGen/AArch64/sve-multivector-load-stores.ll
+++ b/llvm/test/CodeGen/AArch64/sve-multivector-load-stores.ll
@@ -1,8 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -mattr=+sve-b16b16,+sve2p1 < %s -verify-machineinstrs | FileCheck %s --check-prefix=SVE2p1
; RUN: llc -mattr=+sve-b16b16,+sve2p1 -aarch64-enable-subreg-liveness-tracking < %s -verify-machineinstrs | FileCheck %s --check-prefix=SVE2p1-SL
-; RUN: llc -mattr=+sve-b16b16,+sme2 --force-streaming -aarch64-enable-sme2-multivector-store-lowering < %s -verify-machineinstrs | FileCheck %s --check-prefix=SME2
-; RUN: llc -mattr=+sve-b16b16,+sme2 --force-streaming < %s -verify-machineinstrs | FileCheck %s --check-prefix=SME2-NO-MV-STORES
+; RUN: llc -mattr=+sve-b16b16,+sme2 --force-streaming < %s -verify-machineinstrs | FileCheck %s --check-prefix=SME2
target triple = "aarch64-unknown-linux-gnu"
@@ -30,22 +29,10 @@ define void @load_store_2x_vectors_i8_r(ptr %addr) {
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.b
; SME2-NEXT: ld1b { z16.b, z24.b }, pn8/z, [x0]
-; SME2-NEXT: movprfx z1, z24
-; SME2-NEXT: add z1.b, z1.b, #5 // =0x5
-; SME2-NEXT: movprfx z0, z16
-; SME2-NEXT: add z0.b, z0.b, #5 // =0x5
-; SME2-NEXT: st1b { z0.b, z1.b }, pn8, [x0]
+; SME2-NEXT: add z24.b, z24.b, #5 // =0x5
+; SME2-NEXT: add z16.b, z16.b, #5 // =0x5
+; SME2-NEXT: st1b { z16.b, z24.b }, pn8, [x0]
; SME2-NEXT: ret
-;
-; SME2-NO-MV-STORES-LABEL: load_store_2x_vectors_i8_r:
-; SME2-NO-MV-STORES: // %bb.0:
-; SME2-NO-MV-STORES-NEXT: ptrue pn8.b
-; SME2-NO-MV-STORES-NEXT: ld1b { z16.b, z24.b }, pn8/z, [x0]
-; SME2-NO-MV-STORES-NEXT: add z24.b, z24.b, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: add z16.b, z16.b, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: str z24, [x0, #1, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z16, [x0]
-; SME2-NO-MV-STORES-NEXT: ret
%a = load <vscale x 32 x i8>, ptr %addr
%b = add <vscale x 32 x i8> %a, splat (i8 5)
store <vscale x 32 x i8> %b, ptr %addr
@@ -80,24 +67,10 @@ define void @load_store_2x_vectors_i8_rr(ptr %base, i64 %idx) {
; SME2-NEXT: ptrue pn8.b
; SME2-NEXT: add x8, x0, x1
; SME2-NEXT: ld1b { z16.b, z24.b }, pn8/z, [x0, x1]
-; SME2-NEXT: movprfx z1, z24
-; SME2-NEXT: add z1.b, z1.b, #5 // =0x5
-; SME2-NEXT: movprfx z0, z16
-; SME2-NEXT: add z0.b, z0.b, #5 // =0x5
-; SME2-NEXT: st1b { z0.b, z1.b }, pn8, [x8]
+; SME2-NEXT: add z24.b, z24.b, #5 // =0x5
+; SME2-NEXT: add z16.b, z16.b, #5 // =0x5
+; SME2-NEXT: st1b { z16.b, z24.b }, pn8, [x8]
; SME2-NEXT: ret
-;
-; SME2-NO-MV-STORES-LABEL: load_store_2x_vectors_i8_rr:
-; SME2-NO-MV-STORES: // %bb.0:
-; SME2-NO-MV-STORES-NEXT: ptrue pn8.b
-; SME2-NO-MV-STORES-NEXT: ptrue p0.b
-; SME2-NO-MV-STORES-NEXT: add x8, x0, x1
-; SME2-NO-MV-STORES-NEXT: ld1b { z16.b, z24.b }, pn8/z, [x0, x1]
-; SME2-NO-MV-STORES-NEXT: add z16.b, z16.b, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: add z24.b, z24.b, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: st1b { z16.b }, p0, [x0, x1]
-; SME2-NO-MV-STORES-NEXT: str z24, [x8, #1, mul vl]
-; SME2-NO-MV-STORES-NEXT: ret
%addr = getelementptr i8, ptr %base, i64 %idx
%a = load <vscale x 32 x i8>, ptr %addr
%b = add <vscale x 32 x i8> %a, splat (i8 5)
@@ -129,22 +102,10 @@ define void @load_store_2x_vectors_i16_r(ptr %addr) {
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.h
; SME2-NEXT: ld1h { z16.h, z24.h }, pn8/z, [x0]
-; SME2-NEXT: movprfx z1, z24
-; SME2-NEXT: add z1.h, z1.h, #5 // =0x5
-; SME2-NEXT: movprfx z0, z16
-; SME2-NEXT: add z0.h, z0.h, #5 // =0x5
-; SME2-NEXT: st1h { z0.h, z1.h }, pn8, [x0]
+; SME2-NEXT: add z24.h, z24.h, #5 // =0x5
+; SME2-NEXT: add z16.h, z16.h, #5 // =0x5
+; SME2-NEXT: st1h { z16.h, z24.h }, pn8, [x0]
; SME2-NEXT: ret
-;
-; SME2-NO-MV-STORES-LABEL: load_store_2x_vectors_i16_r:
-; SME2-NO-MV-STORES: // %bb.0:
-; SME2-NO-MV-STORES-NEXT: ptrue pn8.h
-; SME2-NO-MV-STORES-NEXT: ld1h { z16.h, z24.h }, pn8/z, [x0]
-; SME2-NO-MV-STORES-NEXT: add z24.h, z24.h, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: add z16.h, z16.h, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: str z24, [x0, #1, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z16, [x0]
-; SME2-NO-MV-STORES-NEXT: ret
%a = load <vscale x 16 x i16>, ptr %addr
%b = add <vscale x 16 x i16> %a, splat (i16 5)
store <vscale x 16 x i16> %b, ptr %addr
@@ -179,24 +140,10 @@ define void @load_store_2x_vectors_i16_rr(ptr %base, i64 %idx) {
; SME2-NEXT: ptrue pn8.h
; SME2-NEXT: add x8, x0, x1, lsl #1
; SME2-NEXT: ld1h { z16.h, z24.h }, pn8/z, [x0, x1, lsl #1]
-; SME2-NEXT: movprfx z1, z24
-; SME2-NEXT: add z1.h, z1.h, #5 // =0x5
-; SME2-NEXT: movprfx z0, z16
-; SME2-NEXT: add z0.h, z0.h, #5 // =0x5
-; SME2-NEXT: st1h { z0.h, z1.h }, pn8, [x8]
+; SME2-NEXT: add z24.h, z24.h, #5 // =0x5
+; SME2-NEXT: add z16.h, z16.h, #5 // =0x5
+; SME2-NEXT: st1h { z16.h, z24.h }, pn8, [x8]
; SME2-NEXT: ret
-;
-; SME2-NO-MV-STORES-LABEL: load_store_2x_vectors_i16_rr:
-; SME2-NO-MV-STORES: // %bb.0:
-; SME2-NO-MV-STORES-NEXT: ptrue pn8.h
-; SME2-NO-MV-STORES-NEXT: ptrue p0.h
-; SME2-NO-MV-STORES-NEXT: add x8, x0, x1, lsl #1
-; SME2-NO-MV-STORES-NEXT: ld1h { z16.h, z24.h }, pn8/z, [x0, x1, lsl #1]
-; SME2-NO-MV-STORES-NEXT: add z16.h, z16.h, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: add z24.h, z24.h, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: st1h { z16.h }, p0, [x0, x1, lsl #1]
-; SME2-NO-MV-STORES-NEXT: str z24, [x8, #1, mul vl]
-; SME2-NO-MV-STORES-NEXT: ret
%addr = getelementptr i16, ptr %base, i64 %idx
%a = load <vscale x 16 x i16>, ptr %addr
%b = add <vscale x 16 x i16> %a, splat (i16 5)
@@ -228,22 +175,10 @@ define void @load_store_2x_vectors_i32_r(ptr %addr) {
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.s
; SME2-NEXT: ld1w { z16.s, z24.s }, pn8/z, [x0]
-; SME2-NEXT: movprfx z1, z24
-; SME2-NEXT: add z1.s, z1.s, #5 // =0x5
-; SME2-NEXT: movprfx z0, z16
-; SME2-NEXT: add z0.s, z0.s, #5 // =0x5
-; SME2-NEXT: st1w { z0.s, z1.s }, pn8, [x0]
+; SME2-NEXT: add z24.s, z24.s, #5 // =0x5
+; SME2-NEXT: add z16.s, z16.s, #5 // =0x5
+; SME2-NEXT: st1w { z16.s, z24.s }, pn8, [x0]
; SME2-NEXT: ret
-;
-; SME2-NO-MV-STORES-LABEL: load_store_2x_vectors_i32_r:
-; SME2-NO-MV-STORES: // %bb.0:
-; SME2-NO-MV-STORES-NEXT: ptrue pn8.s
-; SME2-NO-MV-STORES-NEXT: ld1w { z16.s, z24.s }, pn8/z, [x0]
-; SME2-NO-MV-STORES-NEXT: add z24.s, z24.s, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: add z16.s, z16.s, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: str z24, [x0, #1, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z16, [x0]
-; SME2-NO-MV-STORES-NEXT: ret
%a = load <vscale x 8 x i32>, ptr %addr
%b = add <vscale x 8 x i32> %a, splat (i32 5)
store <vscale x 8 x i32> %b, ptr %addr
@@ -278,24 +213,10 @@ define void @load_store_2x_vectors_i32_rr(ptr %base, i64 %idx) {
; SME2-NEXT: ptrue pn8.s
; SME2-NEXT: add x8, x0, x1, lsl #2
; SME2-NEXT: ld1w { z16.s, z24.s }, pn8/z, [x0, x1, lsl #2]
-; SME2-NEXT: movprfx z1, z24
-; SME2-NEXT: add z1.s, z1.s, #5 // =0x5
-; SME2-NEXT: movprfx z0, z16
-; SME2-NEXT: add z0.s, z0.s, #5 // =0x5
-; SME2-NEXT: st1w { z0.s, z1.s }, pn8, [x8]
+; SME2-NEXT: add z24.s, z24.s, #5 // =0x5
+; SME2-NEXT: add z16.s, z16.s, #5 // =0x5
+; SME2-NEXT: st1w { z16.s, z24.s }, pn8, [x8]
; SME2-NEXT: ret
-;
-; SME2-NO-MV-STORES-LABEL: load_store_2x_vectors_i32_rr:
-; SME2-NO-MV-STORES: // %bb.0:
-; SME2-NO-MV-STORES-NEXT: ptrue pn8.s
-; SME2-NO-MV-STORES-NEXT: ptrue p0.s
-; SME2-NO-MV-STORES-NEXT: add x8, x0, x1, lsl #2
-; SME2-NO-MV-STORES-NEXT: ld1w { z16.s, z24.s }, pn8/z, [x0, x1, lsl #2]
-; SME2-NO-MV-STORES-NEXT: add z16.s, z16.s, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: add z24.s, z24.s, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: st1w { z16.s }, p0, [x0, x1, lsl #2]
-; SME2-NO-MV-STORES-NEXT: str z24, [x8, #1, mul vl]
-; SME2-NO-MV-STORES-NEXT: ret
%addr = getelementptr i32, ptr %base, i64 %idx
%a = load <vscale x 8 x i32>, ptr %addr
%b = add <vscale x 8 x i32> %a, splat (i32 5)
@@ -327,22 +248,10 @@ define void @load_store_2x_vectors_i64_r(ptr %addr) {
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.d
; SME2-NEXT: ld1d { z16.d, z24.d }, pn8/z, [x0]
-; SME2-NEXT: movprfx z1, z24
-; SME2-NEXT: add z1.d, z1.d, #5 // =0x5
-; SME2-NEXT: movprfx z0, z16
-; SME2-NEXT: add z0.d, z0.d, #5 // =0x5
-; SME2-NEXT: st1d { z0.d, z1.d }, pn8, [x0]
+; SME2-NEXT: add z24.d, z24.d, #5 // =0x5
+; SME2-NEXT: add z16.d, z16.d, #5 // =0x5
+; SME2-NEXT: st1d { z16.d, z24.d }, pn8, [x0]
; SME2-NEXT: ret
-;
-; SME2-NO-MV-STORES-LABEL: load_store_2x_vectors_i64_r:
-; SME2-NO-MV-STORES: // %bb.0:
-; SME2-NO-MV-STORES-NEXT: ptrue pn8.d
-; SME2-NO-MV-STORES-NEXT: ld1d { z16.d, z24.d }, pn8/z, [x0]
-; SME2-NO-MV-STORES-NEXT: add z24.d, z24.d, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: add z16.d, z16.d, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: str z24, [x0, #1, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z16, [x0]
-; SME2-NO-MV-STORES-NEXT: ret
%a = load <vscale x 4 x i64>, ptr %addr
%b = add <vscale x 4 x i64> %a, splat (i64 5)
store <vscale x 4 x i64> %b, ptr %addr
@@ -377,24 +286,10 @@ define void @load_store_2x_vectors_i64_rr(ptr %base, i64 %idx) {
; SME2-NEXT: ptrue pn8.d
; SME2-NEXT: add x8, x0, x1, lsl #3
; SME2-NEXT: ld1d { z16.d, z24.d }, pn8/z, [x0, x1, lsl #3]
-; SME2-NEXT: movprfx z1, z24
-; SME2-NEXT: add z1.d, z1.d, #5 // =0x5
-; SME2-NEXT: movprfx z0, z16
-; SME2-NEXT: add z0.d, z0.d, #5 // =0x5
-; SME2-NEXT: st1d { z0.d, z1.d }, pn8, [x8]
+; SME2-NEXT: add z24.d, z24.d, #5 // =0x5
+; SME2-NEXT: add z16.d, z16.d, #5 // =0x5
+; SME2-NEXT: st1d { z16.d, z24.d }, pn8, [x8]
; SME2-NEXT: ret
-;
-; SME2-NO-MV-STORES-LABEL: load_store_2x_vectors_i64_rr:
-; SME2-NO-MV-STORES: // %bb.0:
-; SME2-NO-MV-STORES-NEXT: ptrue pn8.d
-; SME2-NO-MV-STORES-NEXT: ptrue p0.d
-; SME2-NO-MV-STORES-NEXT: add x8, x0, x1, lsl #3
-; SME2-NO-MV-STORES-NEXT: ld1d { z16.d, z24.d }, pn8/z, [x0, x1, lsl #3]
-; SME2-NO-MV-STORES-NEXT: add z16.d, z16.d, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: add z24.d, z24.d, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: st1d { z16.d }, p0, [x0, x1, lsl #3]
-; SME2-NO-MV-STORES-NEXT: str z24, [x8, #1, mul vl]
-; SME2-NO-MV-STORES-NEXT: ret
%addr = getelementptr i64, ptr %base, i64 %idx
%a = load <vscale x 4 x i64>, ptr %addr
%b = add <vscale x 4 x i64> %a, splat (i64 5)
@@ -429,23 +324,10 @@ define void @load_store_2x_vectors_f16_r(ptr %addr) {
; SME2-NEXT: ptrue pn8.h
; SME2-NEXT: ptrue p0.h
; SME2-NEXT: ld1h { z16.h, z24.h }, pn8/z, [x0]
-; SME2-NEXT: movprfx z1, z24
-; SME2-NEXT: fadd z1.h, p0/m, z1.h, #1.0
-; SME2-NEXT: movprfx z0, z16
-; SME2-NEXT: fadd z0.h, p0/m, z0.h, #1.0
-; SME2-NEXT: st1h { z0.h, z1.h }, pn8, [x0]
+; SME2-NEXT: fadd z24.h, p0/m, z24.h, #1.0
+; SME2-NEXT: fadd z16.h, p0/m, z16.h, #1.0
+; SME2-NEXT: st1h { z16.h, z24.h }, pn8, [x0]
; SME2-NEXT: ret
-;
-; SME2-NO-MV-STORES-LABEL: load_store_2x_vectors_f16_r:
-; SME2-NO-MV-STORES: // %bb.0:
-; SME2-NO-MV-STORES-NEXT: ptrue pn8.h
-; SME2-NO-MV-STORES-NEXT: ptrue p0.h
-; SME2-NO-MV-STORES-NEXT: ld1h { z16.h, z24.h }, pn8/z, [x0]
-; SME2-NO-MV-STORES-NEXT: fadd z24.h, p0/m, z24.h, #1.0
-; SME2-NO-MV-STORES-NEXT: fadd z16.h, p0/m, z16.h, #1.0
-; SME2-NO-MV-STORES-NEXT: str z24, [x0, #1, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z16, [x0]
-; SME2-NO-MV-STORES-NEXT: ret
%a = load <vscale x 16 x half>, ptr %addr
%b = fadd <vscale x 16 x half> %a, splat (half 1.0)
store <vscale x 16 x half> %b, ptr %addr
@@ -482,24 +364,10 @@ define void @load_store_2x_vectors_f16_rr(ptr %base, i64 %idx) {
; SME2-NEXT: ptrue p0.h
; SME2-NEXT: add x8, x0, x1, lsl #1
; SME2-NEXT: ld1h { z16.h, z24.h }, pn8/z, [x0, x1, lsl #1]
-; SME2-NEXT: movprfx z1, z24
-; SME2-NEXT: fadd z1.h, p0/m, z1.h, #1.0
-; SME2-NEXT: movprfx z0, z16
-; SME2-NEXT: fadd z0.h, p0/m, z0.h, #1.0
-; SME2-NEXT: st1h { z0.h, z1.h }, pn8, [x8]
+; SME2-NEXT: fadd z24.h, p0/m, z24.h, #1.0
+; SME2-NEXT: fadd z16.h, p0/m, z16.h, #1.0
+; SME2-NEXT: st1h { z16.h, z24.h }, pn8, [x8]
; SME2-NEXT: ret
-;
-; SME2-NO-MV-STORES-LABEL: load_store_2x_vectors_f16_rr:
-; SME2-NO-MV-STORES: // %bb.0:
-; SME2-NO-MV-STORES-NEXT: ptrue pn8.h
-; SME2-NO-MV-STORES-NEXT: ptrue p0.h
-; SME2-NO-MV-STORES-NEXT: add x8, x0, x1, lsl #1
-; SME2-NO-MV-STORES-NEXT: ld1h { z16.h, z24.h }, pn8/z, [x0, x1, lsl #1]
-; SME2-NO-MV-STORES-NEXT: fadd z16.h, p0/m, z16.h, #1.0
-; SME2-NO-MV-STORES-NEXT: fadd z24.h, p0/m, z24.h, #1.0
-; SME2-NO-MV-STORES-NEXT: st1h { z16.h }, p0, [x0, x1, lsl #1]
-; SME2-NO-MV-STORES-NEXT: str z24, [x8, #1, mul vl]
-; SME2-NO-MV-STORES-NEXT: ret
%addr = getelementptr half, ptr %base, i64 %idx
%a = load <vscale x 16 x half>, ptr %addr
%b = fadd <vscale x 16 x half> %a, splat (half 1.0)
@@ -534,23 +402,10 @@ define void @load_store_2x_vectors_f32_r(ptr %addr) {
; SME2-NEXT: ptrue pn8.s
; SME2-NEXT: ptrue p0.s
; SME2-NEXT: ld1w { z16.s, z24.s }, pn8/z, [x0]
-; SME2-NEXT: movprfx z1, z24
-; SME2-NEXT: fadd z1.s, p0/m, z1.s, #1.0
-; SME2-NEXT: movprfx z0, z16
-; SME2-NEXT: fadd z0.s, p0/m, z0.s, #1.0
-; SME2-NEXT: st1w { z0.s, z1.s }, pn8, [x0]
+; SME2-NEXT: fadd z24.s, p0/m, z24.s, #1.0
+; SME2-NEXT: fadd z16.s, p0/m, z16.s, #1.0
+; SME2-NEXT: st1w { z16.s, z24.s }, pn8, [x0]
; SME2-NEXT: ret
-;
-; SME2-NO-MV-STORES-LABEL: load_store_2x_vectors_f32_r:
-; SME2-NO-MV-STORES: // %bb.0:
-; SME2-NO-MV-STORES-NEXT: ptrue pn8.s
-; SME2-NO-MV-STORES-NEXT: ptrue p0.s
-; SME2-NO-MV-STORES-NEXT: ld1w { z16.s, z24.s }, pn8/z, [x0]
-; SME2-NO-MV-STORES-NEXT: fadd z24.s, p0/m, z24.s, #1.0
-; SME2-NO-MV-STORES-NEXT: fadd z16.s, p0/m, z16.s, #1.0
-; SME2-NO-MV-STORES-NEXT: str z24, [x0, #1, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z16, [x0]
-; SME2-NO-MV-STORES-NEXT: ret
%a = load <vscale x 8 x float>, ptr %addr
%b = fadd <vscale x 8 x float> %a, splat (float 1.0)
store <vscale x 8 x float> %b, ptr %addr
@@ -587,24 +442,10 @@ define void @load_store_2x_vectors_f32_rr(ptr %base, i64 %idx) {
; SME2-NEXT: ptrue p0.s
; SME2-NEXT: add x8, x0, x1, lsl #2
; SME2-NEXT: ld1w { z16.s, z24.s }, pn8/z, [x0, x1, lsl #2]
-; SME2-NEXT: movprfx z1, z24
-; SME2-NEXT: fadd z1.s, p0/m, z1.s, #1.0
-; SME2-NEXT: movprfx z0, z16
-; SME2-NEXT: fadd z0.s, p0/m, z0.s, #1.0
-; SME2-NEXT: st1w { z0.s, z1.s }, pn8, [x8]
+; SME2-NEXT: fadd z24.s, p0/m, z24.s, #1.0
+; SME2-NEXT: fadd z16.s, p0/m, z16.s, #1.0
+; SME2-NEXT: st1w { z16.s, z24.s }, pn8, [x8]
; SME2-NEXT: ret
-;
-; SME2-NO-MV-STORES-LABEL: load_store_2x_vectors_f32_rr:
-; SME2-NO-MV-STORES: // %bb.0:
-; SME2-NO-MV-STORES-NEXT: ptrue pn8.s
-; SME2-NO-MV-STORES-NEXT: ptrue p0.s
-; SME2-NO-MV-STORES-NEXT: add x8, x0, x1, lsl #2
-; SME2-NO-MV-STORES-NEXT: ld1w { z16.s, z24.s }, pn8/z, [x0, x1, lsl #2]
-; SME2-NO-MV-STORES-NEXT: fadd z16.s, p0/m, z16.s, #1.0
-; SME2-NO-MV-STORES-NEXT: fadd z24.s, p0/m, z24.s, #1.0
-; SME2-NO-MV-STORES-NEXT: st1w { z16.s }, p0, [x0, x1, lsl #2]
-; SME2-NO-MV-STORES-NEXT: str z24, [x8, #1, mul vl]
-; SME2-NO-MV-STORES-NEXT: ret
%addr = getelementptr float, ptr %base, i64 %idx
%a = load <vscale x 8 x float>, ptr %addr
%b = fadd <vscale x 8 x float> %a, splat (float 1.0)
@@ -639,23 +480,10 @@ define void @load_store_2x_vectors_f64_r(ptr %addr) {
; SME2-NEXT: ptrue pn8.d
; SME2-NEXT: ptrue p0.d
; SME2-NEXT: ld1d { z16.d, z24.d }, pn8/z, [x0]
-; SME2-NEXT: movprfx z1, z24
-; SME2-NEXT: fadd z1.d, p0/m, z1.d, #1.0
-; SME2-NEXT: movprfx z0, z16
-; SME2-NEXT: fadd z0.d, p0/m, z0.d, #1.0
-; SME2-NEXT: st1d { z0.d, z1.d }, pn8, [x0]
+; SME2-NEXT: fadd z24.d, p0/m, z24.d, #1.0
+; SME2-NEXT: fadd z16.d, p0/m, z16.d, #1.0
+; SME2-NEXT: st1d { z16.d, z24.d }, pn8, [x0]
; SME2-NEXT: ret
-;
-; SME2-NO-MV-STORES-LABEL: load_store_2x_vectors_f64_r:
-; SME2-NO-MV-STORES: // %bb.0:
-; SME2-NO-MV-STORES-NEXT: ptrue pn8.d
-; SME2-NO-MV-STORES-NEXT: ptrue p0.d
-; SME2-NO-MV-STORES-NEXT: ld1d { z16.d, z24.d }, pn8/z, [x0]
-; SME2-NO-MV-STORES-NEXT: fadd z24.d, p0/m, z24.d, #1.0
-; SME2-NO-MV-STORES-NEXT: fadd z16.d, p0/m, z16.d, #1.0
-; SME2-NO-MV-STORES-NEXT: str z24, [x0, #1, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z16, [x0]
-; SME2-NO-MV-STORES-NEXT: ret
%a = load <vscale x 4 x double>, ptr %addr
%b = fadd <vscale x 4 x double> %a, splat (double 1.0)
store <vscale x 4 x double> %b, ptr %addr
@@ -692,24 +520,10 @@ define void @load_store_2x_vectors_f64_rr(ptr %base, i64 %idx) {
; SME2-NEXT: ptrue p0.d
; SME2-NEXT: add x8, x0, x1, lsl #3
; SME2-NEXT: ld1d { z16.d, z24.d }, pn8/z, [x0, x1, lsl #3]
-; SME2-NEXT: movprfx z1, z24
-; SME2-NEXT: fadd z1.d, p0/m, z1.d, #1.0
-; SME2-NEXT: movprfx z0, z16
-; SME2-NEXT: fadd z0.d, p0/m, z0.d, #1.0
-; SME2-NEXT: st1d { z0.d, z1.d }, pn8, [x8]
+; SME2-NEXT: fadd z24.d, p0/m, z24.d, #1.0
+; SME2-NEXT: fadd z16.d, p0/m, z16.d, #1.0
+; SME2-NEXT: st1d { z16.d, z24.d }, pn8, [x8]
; SME2-NEXT: ret
-;
-; SME2-NO-MV-STORES-LABEL: load_store_2x_vectors_f64_rr:
-; SME2-NO-MV-STORES: // %bb.0:
-; SME2-NO-MV-STORES-NEXT: ptrue pn8.d
-; SME2-NO-MV-STORES-NEXT: ptrue p0.d
-; SME2-NO-MV-STORES-NEXT: add x8, x0, x1, lsl #3
-; SME2-NO-MV-STORES-NEXT: ld1d { z16.d, z24.d }, pn8/z, [x0, x1, lsl #3]
-; SME2-NO-MV-STORES-NEXT: fadd z16.d, p0/m, z16.d, #1.0
-; SME2-NO-MV-STORES-NEXT: fadd z24.d, p0/m, z24.d, #1.0
-; SME2-NO-MV-STORES-NEXT: st1d { z16.d }, p0, [x0, x1, lsl #3]
-; SME2-NO-MV-STORES-NEXT: str z24, [x8, #1, mul vl]
-; SME2-NO-MV-STORES-NEXT: ret
%addr = getelementptr double, ptr %base, i64 %idx
%a = load <vscale x 4 x double>, ptr %addr
%b = fadd <vscale x 4 x double> %a, splat (double 1.0)
@@ -744,21 +558,10 @@ define void @load_store_2x_vectors_bf16_r(ptr %addr) {
; SME2-NEXT: ptrue pn8.h
; SME2-NEXT: fmov z0.h, #1.87500000
; SME2-NEXT: ld1h { z16.h, z24.h }, pn8/z, [x0]
-; SME2-NEXT: bfadd z1.h, z24.h, z0.h
-; SME2-NEXT: bfadd z0.h, z16.h, z0.h
-; SME2-NEXT: st1h { z0.h, z1.h }, pn8, [x0]
+; SME2-NEXT: bfadd z24.h, z24.h, z0.h
+; SME2-NEXT: bfadd z16.h, z16.h, z0.h
+; SME2-NEXT: st1h { z16.h, z24.h }, pn8, [x0]
; SME2-NEXT: ret
-;
-; SME2-NO-MV-STORES-LABEL: load_store_2x_vectors_bf16_r:
-; SME2-NO-MV-STORES: // %bb.0:
-; SME2-NO-MV-STORES-NEXT: ptrue pn8.h
-; SME2-NO-MV-STORES-NEXT: fmov z0.h, #1.87500000
-; SME2-NO-MV-STORES-NEXT: ld1h { z16.h, z24.h }, pn8/z, [x0]
-; SME2-NO-MV-STORES-NEXT: bfadd z1.h, z24.h, z0.h
-; SME2-NO-MV-STORES-NEXT: bfadd z0.h, z16.h, z0.h
-; SME2-NO-MV-STORES-NEXT: str z1, [x0, #1, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z0, [x0]
-; SME2-NO-MV-STORES-NEXT: ret
%a = load <vscale x 16 x bfloat>, ptr %addr
%b = fadd <vscale x 16 x bfloat> %a, splat (bfloat 1.0)
store <vscale x 16 x bfloat> %b, ptr %addr
@@ -796,23 +599,10 @@ define void @load_store_2x_vectors_bf16_rr(ptr %base, i64 %idx) {
; SME2-NEXT: fmov z0.h, #1.87500000
; SME2-NEXT: add x8, x0, x1, lsl #1
; SME2-NEXT: ld1h { z16.h, z24.h }, pn8/z, [x0, x1, lsl #1]
-; SME2-NEXT: bfadd z1.h, z24.h, z0.h
-; SME2-NEXT: bfadd z0.h, z16.h, z0.h
-; SME2-NEXT: st1h { z0.h, z1.h }, pn8, [x8]
+; SME2-NEXT: bfadd z24.h, z24.h, z0.h
+; SME2-NEXT: bfadd z16.h, z16.h, z0.h
+; SME2-NEXT: st1h { z16.h, z24.h }, pn8, [x8]
; SME2-NEXT: ret
-;
-; SME2-NO-MV-STORES-LABEL: load_store_2x_vectors_bf16_rr:
-; SME2-NO-MV-STORES: // %bb.0:
-; SME2-NO-MV-STORES-NEXT: ptrue pn8.h
-; SME2-NO-MV-STORES-NEXT: fmov z0.h, #1.87500000
-; SME2-NO-MV-STORES-NEXT: add x8, x0, x1, lsl #1
-; SME2-NO-MV-STORES-NEXT: ld1h { z16.h, z24.h }, pn8/z, [x0, x1, lsl #1]
-; SME2-NO-MV-STORES-NEXT: ptrue p0.h
-; SME2-NO-MV-STORES-NEXT: bfadd z1.h, z16.h, z0.h
-; SME2-NO-MV-STORES-NEXT: bfadd z0.h, z24.h, z0.h
-; SME2-NO-MV-STORES-NEXT: st1h { z1.h }, p0, [x0, x1, lsl #1]
-; SME2-NO-MV-STORES-NEXT: str z0, [x8, #1, mul vl]
-; SME2-NO-MV-STORES-NEXT: ret
%addr = getelementptr bfloat, ptr %base, i64 %idx
%a = load <vscale x 16 x bfloat>, ptr %addr
%b = fadd <vscale x 16 x bfloat> %a, splat (bfloat 1.0)
@@ -854,30 +644,12 @@ define void @load_store_4x_vectors_i8_r(ptr %addr) {
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.b
; SME2-NEXT: ld1b { z16.b, z20.b, z24.b, z28.b }, pn8/z, [x0]
-; SME2-NEXT: movprfx z3, z28
-; SME2-NEXT: add z3.b, z3.b, #5 // =0x5
-; SME2-NEXT: movprfx z2, z24
-; SME2-NEXT: add z2.b, z2.b, #5 // =0x5
-; SME2-NEXT: movprfx z1, z20
-; SME2-NEXT: add z1.b, z1.b, #5 // =0x5
-; SME2-NEXT: movprfx z0, z16
-; SME2-NEXT: add z0.b, z0.b, #5 // =0x5
-; SME2-NEXT: st1b { z0.b - z3.b }, pn8, [x0]
+; SME2-NEXT: add z28.b, z28.b, #5 // =0x5
+; SME2-NEXT: add z24.b, z24.b, #5 // =0x5
+; SME2-NEXT: add z20.b, z20.b, #5 // =0x5
+; SME2-NEXT: add z16.b, z16.b, #5 // =0x5
+; SME2-NEXT: st1b { z16.b, z20.b, z24.b, z28.b }, pn8, [x0]
; SME2-NEXT: ret
-;
-; SME2-NO-MV-STORES-LABEL: load_store_4x_vectors_i8_r:
-; SME2-NO-MV-STORES: // %bb.0:
-; SME2-NO-MV-STORES-NEXT: ptrue pn8.b
-; SME2-NO-MV-STORES-NEXT: ld1b { z16.b, z20.b, z24.b, z28.b }, pn8/z, [x0]
-; SME2-NO-MV-STORES-NEXT: add z28.b, z28.b, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: add z24.b, z24.b, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: add z20.b, z20.b, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: add z16.b, z16.b, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: str z28, [x0, #3, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z24, [x0, #2, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z20, [x0, #1, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z16, [x0]
-; SME2-NO-MV-STORES-NEXT: ret
%a = load <vscale x 64 x i8>, ptr %addr
%b = add <vscale x 64 x i8> %a, splat (i8 5)
store <vscale x 64 x i8> %b, ptr %addr
@@ -920,32 +692,12 @@ define void @load_store_4x_vectors_i8_rr(ptr %base, i64 %idx) {
; SME2-NEXT: ptrue pn8.b
; SME2-NEXT: add x8, x0, x1
; SME2-NEXT: ld1b { z16.b, z20.b, z24.b, z28.b }, pn8/z, [x0, x1]
-; SME2-NEXT: movprfx z3, z28
-; SME2-NEXT: add z3.b, z3.b, #5 // =0x5
-; SME2-NEXT: movprfx z2, z24
-; SME2-NEXT: add z2.b, z2.b, #5 // =0x5
-; SME2-NEXT: movprfx z1, z20
-; SME2-NEXT: add z1.b, z1.b, #5 // =0x5
-; SME2-NEXT: movprfx z0, z16
-; SME2-NEXT: add z0.b, z0.b, #5 // =0x5
-; SME2-NEXT: st1b { z0.b - z3.b }, pn8, [x8]
+; SME2-NEXT: add z28.b, z28.b, #5 // =0x5
+; SME2-NEXT: add z24.b, z24.b, #5 // =0x5
+; SME2-NEXT: add z20.b, z20.b, #5 // =0x5
+; SME2-NEXT: add z16.b, z16.b, #5 // =0x5
+; SME2-NEXT: st1b { z16.b, z20.b, z24.b, z28.b }, pn8, [x8]
; SME2-NEXT: ret
-;
-; SME2-NO-MV-STORES-LABEL: load_store_4x_vectors_i8_rr:
-; SME2-NO-MV-STORES: // %bb.0:
-; SME2-NO-MV-STORES-NEXT: ptrue pn8.b
-; SME2-NO-MV-STORES-NEXT: ptrue p0.b
-; SME2-NO-MV-STORES-NEXT: add x8, x0, x1
-; SME2-NO-MV-STORES-NEXT: ld1b { z16.b, z20.b, z24.b, z28.b }, pn8/z, [x0, x1]
-; SME2-NO-MV-STORES-NEXT: add z16.b, z16.b, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: add z28.b, z28.b, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: add z24.b, z24.b, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: add z20.b, z20.b, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: st1b { z16.b }, p0, [x0, x1]
-; SME2-NO-MV-STORES-NEXT: str z28, [x8, #3, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z24, [x8, #2, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z20, [x8, #1, mul vl]
-; SME2-NO-MV-STORES-NEXT: ret
%addr = getelementptr i8, ptr %base, i64 %idx
%a = load <vscale x 64 x i8>, ptr %addr
%b = add <vscale x 64 x i8> %a, splat (i8 5)
@@ -985,30 +737,12 @@ define void @load_store_4x_vectors_i16_r(ptr %addr) {
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.h
; SME2-NEXT: ld1h { z16.h, z20.h, z24.h, z28.h }, pn8/z, [x0]
-; SME2-NEXT: movprfx z3, z28
-; SME2-NEXT: add z3.h, z3.h, #5 // =0x5
-; SME2-NEXT: movprfx z2, z24
-; SME2-NEXT: add z2.h, z2.h, #5 // =0x5
-; SME2-NEXT: movprfx z1, z20
-; SME2-NEXT: add z1.h, z1.h, #5 // =0x5
-; SME2-NEXT: movprfx z0, z16
-; SME2-NEXT: add z0.h, z0.h, #5 // =0x5
-; SME2-NEXT: st1h { z0.h - z3.h }, pn8, [x0]
+; SME2-NEXT: add z28.h, z28.h, #5 // =0x5
+; SME2-NEXT: add z24.h, z24.h, #5 // =0x5
+; SME2-NEXT: add z20.h, z20.h, #5 // =0x5
+; SME2-NEXT: add z16.h, z16.h, #5 // =0x5
+; SME2-NEXT: st1h { z16.h, z20.h, z24.h, z28.h }, pn8, [x0]
; SME2-NEXT: ret
-;
-; SME2-NO-MV-STORES-LABEL: load_store_4x_vectors_i16_r:
-; SME2-NO-MV-STORES: // %bb.0:
-; SME2-NO-MV-STORES-NEXT: ptrue pn8.h
-; SME2-NO-MV-STORES-NEXT: ld1h { z16.h, z20.h, z24.h, z28.h }, pn8/z, [x0]
-; SME2-NO-MV-STORES-NEXT: add z28.h, z28.h, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: add z24.h, z24.h, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: add z20.h, z20.h, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: add z16.h, z16.h, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: str z28, [x0, #3, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z24, [x0, #2, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z20, [x0, #1, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z16, [x0]
-; SME2-NO-MV-STORES-NEXT: ret
%a = load <vscale x 32 x i16>, ptr %addr
%b = add <vscale x 32 x i16> %a, splat (i16 5)
store <vscale x 32 x i16> %b, ptr %addr
@@ -1051,32 +785,12 @@ define void @load_store_4x_vectors_i16_rr(ptr %base, i64 %idx) {
; SME2-NEXT: ptrue pn8.h
; SME2-NEXT: add x8, x0, x1, lsl #1
; SME2-NEXT: ld1h { z16.h, z20.h, z24.h, z28.h }, pn8/z, [x0, x1, lsl #1]
-; SME2-NEXT: movprfx z3, z28
-; SME2-NEXT: add z3.h, z3.h, #5 // =0x5
-; SME2-NEXT: movprfx z2, z24
-; SME2-NEXT: add z2.h, z2.h, #5 // =0x5
-; SME2-NEXT: movprfx z1, z20
-; SME2-NEXT: add z1.h, z1.h, #5 // =0x5
-; SME2-NEXT: movprfx z0, z16
-; SME2-NEXT: add z0.h, z0.h, #5 // =0x5
-; SME2-NEXT: st1h { z0.h - z3.h }, pn8, [x8]
+; SME2-NEXT: add z28.h, z28.h, #5 // =0x5
+; SME2-NEXT: add z24.h, z24.h, #5 // =0x5
+; SME2-NEXT: add z20.h, z20.h, #5 // =0x5
+; SME2-NEXT: add z16.h, z16.h, #5 // =0x5
+; SME2-NEXT: st1h { z16.h, z20.h, z24.h, z28.h }, pn8, [x8]
; SME2-NEXT: ret
-;
-; SME2-NO-MV-STORES-LABEL: load_store_4x_vectors_i16_rr:
-; SME2-NO-MV-STORES: // %bb.0:
-; SME2-NO-MV-STORES-NEXT: ptrue pn8.h
-; SME2-NO-MV-STORES-NEXT: ptrue p0.h
-; SME2-NO-MV-STORES-NEXT: add x8, x0, x1, lsl #1
-; SME2-NO-MV-STORES-NEXT: ld1h { z16.h, z20.h, z24.h, z28.h }, pn8/z, [x0, x1, lsl #1]
-; SME2-NO-MV-STORES-NEXT: add z16.h, z16.h, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: add z28.h, z28.h, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: add z24.h, z24.h, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: add z20.h, z20.h, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: st1h { z16.h }, p0, [x0, x1, lsl #1]
-; SME2-NO-MV-STORES-NEXT: str z28, [x8, #3, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z24, [x8, #2, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z20, [x8, #1, mul vl]
-; SME2-NO-MV-STORES-NEXT: ret
%addr = getelementptr i16, ptr %base, i64 %idx
%a = load <vscale x 32 x i16>, ptr %addr
%b = add <vscale x 32 x i16> %a, splat (i16 5)
@@ -1116,30 +830,12 @@ define void @load_store_4x_vectors_i32_r(ptr %addr) {
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.s
; SME2-NEXT: ld1w { z16.s, z20.s, z24.s, z28.s }, pn8/z, [x0]
-; SME2-NEXT: movprfx z3, z28
-; SME2-NEXT: add z3.s, z3.s, #5 // =0x5
-; SME2-NEXT: movprfx z2, z24
-; SME2-NEXT: add z2.s, z2.s, #5 // =0x5
-; SME2-NEXT: movprfx z1, z20
-; SME2-NEXT: add z1.s, z1.s, #5 // =0x5
-; SME2-NEXT: movprfx z0, z16
-; SME2-NEXT: add z0.s, z0.s, #5 // =0x5
-; SME2-NEXT: st1w { z0.s - z3.s }, pn8, [x0]
+; SME2-NEXT: add z28.s, z28.s, #5 // =0x5
+; SME2-NEXT: add z24.s, z24.s, #5 // =0x5
+; SME2-NEXT: add z20.s, z20.s, #5 // =0x5
+; SME2-NEXT: add z16.s, z16.s, #5 // =0x5
+; SME2-NEXT: st1w { z16.s, z20.s, z24.s, z28.s }, pn8, [x0]
; SME2-NEXT: ret
-;
-; SME2-NO-MV-STORES-LABEL: load_store_4x_vectors_i32_r:
-; SME2-NO-MV-STORES: // %bb.0:
-; SME2-NO-MV-STORES-NEXT: ptrue pn8.s
-; SME2-NO-MV-STORES-NEXT: ld1w { z16.s, z20.s, z24.s, z28.s }, pn8/z, [x0]
-; SME2-NO-MV-STORES-NEXT: add z28.s, z28.s, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: add z24.s, z24.s, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: add z20.s, z20.s, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: add z16.s, z16.s, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: str z28, [x0, #3, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z24, [x0, #2, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z20, [x0, #1, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z16, [x0]
-; SME2-NO-MV-STORES-NEXT: ret
%a = load <vscale x 16 x i32>, ptr %addr
%b = add <vscale x 16 x i32> %a, splat (i32 5)
store <vscale x 16 x i32> %b, ptr %addr
@@ -1182,32 +878,12 @@ define void @load_store_4x_vectors_i32_rr(ptr %base, i64 %idx) {
; SME2-NEXT: ptrue pn8.s
; SME2-NEXT: add x8, x0, x1, lsl #2
; SME2-NEXT: ld1w { z16.s, z20.s, z24.s, z28.s }, pn8/z, [x0, x1, lsl #2]
-; SME2-NEXT: movprfx z3, z28
-; SME2-NEXT: add z3.s, z3.s, #5 // =0x5
-; SME2-NEXT: movprfx z2, z24
-; SME2-NEXT: add z2.s, z2.s, #5 // =0x5
-; SME2-NEXT: movprfx z1, z20
-; SME2-NEXT: add z1.s, z1.s, #5 // =0x5
-; SME2-NEXT: movprfx z0, z16
-; SME2-NEXT: add z0.s, z0.s, #5 // =0x5
-; SME2-NEXT: st1w { z0.s - z3.s }, pn8, [x8]
+; SME2-NEXT: add z28.s, z28.s, #5 // =0x5
+; SME2-NEXT: add z24.s, z24.s, #5 // =0x5
+; SME2-NEXT: add z20.s, z20.s, #5 // =0x5
+; SME2-NEXT: add z16.s, z16.s, #5 // =0x5
+; SME2-NEXT: st1w { z16.s, z20.s, z24.s, z28.s }, pn8, [x8]
; SME2-NEXT: ret
-;
-; SME2-NO-MV-STORES-LABEL: load_store_4x_vectors_i32_rr:
-; SME2-NO-MV-STORES: // %bb.0:
-; SME2-NO-MV-STORES-NEXT: ptrue pn8.s
-; SME2-NO-MV-STORES-NEXT: ptrue p0.s
-; SME2-NO-MV-STORES-NEXT: add x8, x0, x1, lsl #2
-; SME2-NO-MV-STORES-NEXT: ld1w { z16.s, z20.s, z24.s, z28.s }, pn8/z, [x0, x1, lsl #2]
-; SME2-NO-MV-STORES-NEXT: add z16.s, z16.s, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: add z28.s, z28.s, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: add z24.s, z24.s, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: add z20.s, z20.s, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: st1w { z16.s }, p0, [x0, x1, lsl #2]
-; SME2-NO-MV-STORES-NEXT: str z28, [x8, #3, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z24, [x8, #2, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z20, [x8, #1, mul vl]
-; SME2-NO-MV-STORES-NEXT: ret
%addr = getelementptr i32, ptr %base, i64 %idx
%a = load <vscale x 16 x i32>, ptr %addr
%b = add <vscale x 16 x i32> %a, splat (i32 5)
@@ -1247,30 +923,12 @@ define void @load_store_4x_vectors_i64_r(ptr %addr) {
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.d
; SME2-NEXT: ld1d { z16.d, z20.d, z24.d, z28.d }, pn8/z, [x0]
-; SME2-NEXT: movprfx z3, z28
-; SME2-NEXT: add z3.d, z3.d, #5 // =0x5
-; SME2-NEXT: movprfx z2, z24
-; SME2-NEXT: add z2.d, z2.d, #5 // =0x5
-; SME2-NEXT: movprfx z1, z20
-; SME2-NEXT: add z1.d, z1.d, #5 // =0x5
-; SME2-NEXT: movprfx z0, z16
-; SME2-NEXT: add z0.d, z0.d, #5 // =0x5
-; SME2-NEXT: st1d { z0.d - z3.d }, pn8, [x0]
+; SME2-NEXT: add z28.d, z28.d, #5 // =0x5
+; SME2-NEXT: add z24.d, z24.d, #5 // =0x5
+; SME2-NEXT: add z20.d, z20.d, #5 // =0x5
+; SME2-NEXT: add z16.d, z16.d, #5 // =0x5
+; SME2-NEXT: st1d { z16.d, z20.d, z24.d, z28.d }, pn8, [x0]
; SME2-NEXT: ret
-;
-; SME2-NO-MV-STORES-LABEL: load_store_4x_vectors_i64_r:
-; SME2-NO-MV-STORES: // %bb.0:
-; SME2-NO-MV-STORES-NEXT: ptrue pn8.d
-; SME2-NO-MV-STORES-NEXT: ld1d { z16.d, z20.d, z24.d, z28.d }, pn8/z, [x0]
-; SME2-NO-MV-STORES-NEXT: add z28.d, z28.d, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: add z24.d, z24.d, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: add z20.d, z20.d, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: add z16.d, z16.d, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: str z28, [x0, #3, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z24, [x0, #2, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z20, [x0, #1, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z16, [x0]
-; SME2-NO-MV-STORES-NEXT: ret
%a = load <vscale x 8 x i64>, ptr %addr
%b = add <vscale x 8 x i64> %a, splat (i64 5)
store <vscale x 8 x i64> %b, ptr %addr
@@ -1313,32 +971,12 @@ define void @load_store_4x_vectors_i64_rr(ptr %base, i64 %idx) {
; SME2-NEXT: ptrue pn8.d
; SME2-NEXT: add x8, x0, x1, lsl #3
; SME2-NEXT: ld1d { z16.d, z20.d, z24.d, z28.d }, pn8/z, [x0, x1, lsl #3]
-; SME2-NEXT: movprfx z3, z28
-; SME2-NEXT: add z3.d, z3.d, #5 // =0x5
-; SME2-NEXT: movprfx z2, z24
-; SME2-NEXT: add z2.d, z2.d, #5 // =0x5
-; SME2-NEXT: movprfx z1, z20
-; SME2-NEXT: add z1.d, z1.d, #5 // =0x5
-; SME2-NEXT: movprfx z0, z16
-; SME2-NEXT: add z0.d, z0.d, #5 // =0x5
-; SME2-NEXT: st1d { z0.d - z3.d }, pn8, [x8]
+; SME2-NEXT: add z28.d, z28.d, #5 // =0x5
+; SME2-NEXT: add z24.d, z24.d, #5 // =0x5
+; SME2-NEXT: add z20.d, z20.d, #5 // =0x5
+; SME2-NEXT: add z16.d, z16.d, #5 // =0x5
+; SME2-NEXT: st1d { z16.d, z20.d, z24.d, z28.d }, pn8, [x8]
; SME2-NEXT: ret
-;
-; SME2-NO-MV-STORES-LABEL: load_store_4x_vectors_i64_rr:
-; SME2-NO-MV-STORES: // %bb.0:
-; SME2-NO-MV-STORES-NEXT: ptrue pn8.d
-; SME2-NO-MV-STORES-NEXT: ptrue p0.d
-; SME2-NO-MV-STORES-NEXT: add x8, x0, x1, lsl #3
-; SME2-NO-MV-STORES-NEXT: ld1d { z16.d, z20.d, z24.d, z28.d }, pn8/z, [x0, x1, lsl #3]
-; SME2-NO-MV-STORES-NEXT: add z16.d, z16.d, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: add z28.d, z28.d, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: add z24.d, z24.d, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: add z20.d, z20.d, #5 // =0x5
-; SME2-NO-MV-STORES-NEXT: st1d { z16.d }, p0, [x0, x1, lsl #3]
-; SME2-NO-MV-STORES-NEXT: str z28, [x8, #3, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z24, [x8, #2, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z20, [x8, #1, mul vl]
-; SME2-NO-MV-STORES-NEXT: ret
%addr = getelementptr i64, ptr %base, i64 %idx
%a = load <vscale x 8 x i64>, ptr %addr
%b = add <vscale x 8 x i64> %a, splat (i64 5)
@@ -1381,31 +1019,12 @@ define void @load_store_4x_vectors_f16_r(ptr %addr) {
; SME2-NEXT: ptrue pn8.h
; SME2-NEXT: ptrue p0.h
; SME2-NEXT: ld1h { z16.h, z20.h, z24.h, z28.h }, pn8/z, [x0]
-; SME2-NEXT: movprfx z3, z28
-; SME2-NEXT: fadd z3.h, p0/m, z3.h, #1.0
-; SME2-NEXT: movprfx z2, z24
-; SME2-NEXT: fadd z2.h, p0/m, z2.h, #1.0
-; SME2-NEXT: movprfx z1, z20
-; SME2-NEXT: fadd z1.h, p0/m, z1.h, #1.0
-; SME2-NEXT: movprfx z0, z16
-; SME2-NEXT: fadd z0.h, p0/m, z0.h, #1.0
-; SME2-NEXT: st1h { z0.h - z3.h }, pn8, [x0]
+; SME2-NEXT: fadd z28.h, p0/m, z28.h, #1.0
+; SME2-NEXT: fadd z24.h, p0/m, z24.h, #1.0
+; SME2-NEXT: fadd z20.h, p0/m, z20.h, #1.0
+; SME2-NEXT: fadd z16.h, p0/m, z16.h, #1.0
+; SME2-NEXT: st1h { z16.h, z20.h, z24.h, z28.h }, pn8, [x0]
; SME2-NEXT: ret
-;
-; SME2-NO-MV-STORES-LABEL: load_store_4x_vectors_f16_r:
-; SME2-NO-MV-STORES: // %bb.0:
-; SME2-NO-MV-STORES-NEXT: ptrue pn8.h
-; SME2-NO-MV-STORES-NEXT: ptrue p0.h
-; SME2-NO-MV-STORES-NEXT: ld1h { z16.h, z20.h, z24.h, z28.h }, pn8/z, [x0]
-; SME2-NO-MV-STORES-NEXT: fadd z28.h, p0/m, z28.h, #1.0
-; SME2-NO-MV-STORES-NEXT: fadd z24.h, p0/m, z24.h, #1.0
-; SME2-NO-MV-STORES-NEXT: fadd z20.h, p0/m, z20.h, #1.0
-; SME2-NO-MV-STORES-NEXT: fadd z16.h, p0/m, z16.h, #1.0
-; SME2-NO-MV-STORES-NEXT: str z28, [x0, #3, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z24, [x0, #2, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z20, [x0, #1, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z16, [x0]
-; SME2-NO-MV-STORES-NEXT: ret
%a = load <vscale x 32 x half>, ptr %addr
%b = fadd <vscale x 32 x half> %a, splat (half 1.0)
store <vscale x 32 x half> %b, ptr %addr
@@ -1450,32 +1069,12 @@ define void @load_store_4x_vectors_f16_rr(ptr %base, i64 %idx) {
; SME2-NEXT: ptrue p0.h
; SME2-NEXT: add x8, x0, x1, lsl #1
; SME2-NEXT: ld1h { z16.h, z20.h, z24.h, z28.h }, pn8/z, [x0, x1, lsl #1]
-; SME2-NEXT: movprfx z3, z28
-; SME2-NEXT: fadd z3.h, p0/m, z3.h, #1.0
-; SME2-NEXT: movprfx z2, z24
-; SME2-NEXT: fadd z2.h, p0/m, z2.h, #1.0
-; SME2-NEXT: movprfx z1, z20
-; SME2-NEXT: fadd z1.h, p0/m, z1.h, #1.0
-; SME2-NEXT: movprfx z0, z16
-; SME2-NEXT: fadd z0.h, p0/m, z0.h, #1.0
-; SME2-NEXT: st1h { z0.h - z3.h }, pn8, [x8]
+; SME2-NEXT: fadd z28.h, p0/m, z28.h, #1.0
+; SME2-NEXT: fadd z24.h, p0/m, z24.h, #1.0
+; SME2-NEXT: fadd z20.h, p0/m, z20.h, #1.0
+; SME2-NEXT: fadd z16.h, p0/m, z16.h, #1.0
+; SME2-NEXT: st1h { z16.h, z20.h, z24.h, z28.h }, pn8, [x8]
; SME2-NEXT: ret
-;
-; SME2-NO-MV-STORES-LABEL: load_store_4x_vectors_f16_rr:
-; SME2-NO-MV-STORES: // %bb.0:
-; SME2-NO-MV-STORES-NEXT: ptrue pn8.h
-; SME2-NO-MV-STORES-NEXT: ptrue p0.h
-; SME2-NO-MV-STORES-NEXT: add x8, x0, x1, lsl #1
-; SME2-NO-MV-STORES-NEXT: ld1h { z16.h, z20.h, z24.h, z28.h }, pn8/z, [x0, x1, lsl #1]
-; SME2-NO-MV-STORES-NEXT: fadd z16.h, p0/m, z16.h, #1.0
-; SME2-NO-MV-STORES-NEXT: fadd z28.h, p0/m, z28.h, #1.0
-; SME2-NO-MV-STORES-NEXT: fadd z24.h, p0/m, z24.h, #1.0
-; SME2-NO-MV-STORES-NEXT: fadd z20.h, p0/m, z20.h, #1.0
-; SME2-NO-MV-STORES-NEXT: st1h { z16.h }, p0, [x0, x1, lsl #1]
-; SME2-NO-MV-STORES-NEXT: str z28, [x8, #3, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z24, [x8, #2, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z20, [x8, #1, mul vl]
-; SME2-NO-MV-STORES-NEXT: ret
%addr = getelementptr half, ptr %base, i64 %idx
%a = load <vscale x 32 x half>, ptr %addr
%b = fadd <vscale x 32 x half> %a, splat (half 1.0)
@@ -1518,31 +1117,12 @@ define void @load_store_4x_vectors_f32_r(ptr %addr) {
; SME2-NEXT: ptrue pn8.s
; SME2-NEXT: ptrue p0.s
; SME2-NEXT: ld1w { z16.s, z20.s, z24.s, z28.s }, pn8/z, [x0]
-; SME2-NEXT: movprfx z3, z28
-; SME2-NEXT: fadd z3.s, p0/m, z3.s, #1.0
-; SME2-NEXT: movprfx z2, z24
-; SME2-NEXT: fadd z2.s, p0/m, z2.s, #1.0
-; SME2-NEXT: movprfx z1, z20
-; SME2-NEXT: fadd z1.s, p0/m, z1.s, #1.0
-; SME2-NEXT: movprfx z0, z16
-; SME2-NEXT: fadd z0.s, p0/m, z0.s, #1.0
-; SME2-NEXT: st1w { z0.s - z3.s }, pn8, [x0]
+; SME2-NEXT: fadd z28.s, p0/m, z28.s, #1.0
+; SME2-NEXT: fadd z24.s, p0/m, z24.s, #1.0
+; SME2-NEXT: fadd z20.s, p0/m, z20.s, #1.0
+; SME2-NEXT: fadd z16.s, p0/m, z16.s, #1.0
+; SME2-NEXT: st1w { z16.s, z20.s, z24.s, z28.s }, pn8, [x0]
; SME2-NEXT: ret
-;
-; SME2-NO-MV-STORES-LABEL: load_store_4x_vectors_f32_r:
-; SME2-NO-MV-STORES: // %bb.0:
-; SME2-NO-MV-STORES-NEXT: ptrue pn8.s
-; SME2-NO-MV-STORES-NEXT: ptrue p0.s
-; SME2-NO-MV-STORES-NEXT: ld1w { z16.s, z20.s, z24.s, z28.s }, pn8/z, [x0]
-; SME2-NO-MV-STORES-NEXT: fadd z28.s, p0/m, z28.s, #1.0
-; SME2-NO-MV-STORES-NEXT: fadd z24.s, p0/m, z24.s, #1.0
-; SME2-NO-MV-STORES-NEXT: fadd z20.s, p0/m, z20.s, #1.0
-; SME2-NO-MV-STORES-NEXT: fadd z16.s, p0/m, z16.s, #1.0
-; SME2-NO-MV-STORES-NEXT: str z28, [x0, #3, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z24, [x0, #2, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z20, [x0, #1, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z16, [x0]
-; SME2-NO-MV-STORES-NEXT: ret
%a = load <vscale x 16 x float>, ptr %addr
%b = fadd <vscale x 16 x float> %a, splat (float 1.0)
store <vscale x 16 x float> %b, ptr %addr
@@ -1587,32 +1167,12 @@ define void @load_store_4x_vectors_f32_rr(ptr %base, i64 %idx) {
; SME2-NEXT: ptrue p0.s
; SME2-NEXT: add x8, x0, x1, lsl #2
; SME2-NEXT: ld1w { z16.s, z20.s, z24.s, z28.s }, pn8/z, [x0, x1, lsl #2]
-; SME2-NEXT: movprfx z3, z28
-; SME2-NEXT: fadd z3.s, p0/m, z3.s, #1.0
-; SME2-NEXT: movprfx z2, z24
-; SME2-NEXT: fadd z2.s, p0/m, z2.s, #1.0
-; SME2-NEXT: movprfx z1, z20
-; SME2-NEXT: fadd z1.s, p0/m, z1.s, #1.0
-; SME2-NEXT: movprfx z0, z16
-; SME2-NEXT: fadd z0.s, p0/m, z0.s, #1.0
-; SME2-NEXT: st1w { z0.s - z3.s }, pn8, [x8]
+; SME2-NEXT: fadd z28.s, p0/m, z28.s, #1.0
+; SME2-NEXT: fadd z24.s, p0/m, z24.s, #1.0
+; SME2-NEXT: fadd z20.s, p0/m, z20.s, #1.0
+; SME2-NEXT: fadd z16.s, p0/m, z16.s, #1.0
+; SME2-NEXT: st1w { z16.s, z20.s, z24.s, z28.s }, pn8, [x8]
; SME2-NEXT: ret
-;
-; SME2-NO-MV-STORES-LABEL: load_store_4x_vectors_f32_rr:
-; SME2-NO-MV-STORES: // %bb.0:
-; SME2-NO-MV-STORES-NEXT: ptrue pn8.s
-; SME2-NO-MV-STORES-NEXT: ptrue p0.s
-; SME2-NO-MV-STORES-NEXT: add x8, x0, x1, lsl #2
-; SME2-NO-MV-STORES-NEXT: ld1w { z16.s, z20.s, z24.s, z28.s }, pn8/z, [x0, x1, lsl #2]
-; SME2-NO-MV-STORES-NEXT: fadd z16.s, p0/m, z16.s, #1.0
-; SME2-NO-MV-STORES-NEXT: fadd z28.s, p0/m, z28.s, #1.0
-; SME2-NO-MV-STORES-NEXT: fadd z24.s, p0/m, z24.s, #1.0
-; SME2-NO-MV-STORES-NEXT: fadd z20.s, p0/m, z20.s, #1.0
-; SME2-NO-MV-STORES-NEXT: st1w { z16.s }, p0, [x0, x1, lsl #2]
-; SME2-NO-MV-STORES-NEXT: str z28, [x8, #3, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z24, [x8, #2, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z20, [x8, #1, mul vl]
-; SME2-NO-MV-STORES-NEXT: ret
%addr = getelementptr float, ptr %base, i64 %idx
%a = load <vscale x 16 x float>, ptr %addr
%b = fadd <vscale x 16 x float> %a, splat (float 1.0)
@@ -1655,31 +1215,12 @@ define void @load_store_4x_vectors_f64_r(ptr %addr) {
; SME2-NEXT: ptrue pn8.d
; SME2-NEXT: ptrue p0.d
; SME2-NEXT: ld1d { z16.d, z20.d, z24.d, z28.d }, pn8/z, [x0]
-; SME2-NEXT: movprfx z3, z28
-; SME2-NEXT: fadd z3.d, p0/m, z3.d, #1.0
-; SME2-NEXT: movprfx z2, z24
-; SME2-NEXT: fadd z2.d, p0/m, z2.d, #1.0
-; SME2-NEXT: movprfx z1, z20
-; SME2-NEXT: fadd z1.d, p0/m, z1.d, #1.0
-; SME2-NEXT: movprfx z0, z16
-; SME2-NEXT: fadd z0.d, p0/m, z0.d, #1.0
-; SME2-NEXT: st1d { z0.d - z3.d }, pn8, [x0]
+; SME2-NEXT: fadd z28.d, p0/m, z28.d, #1.0
+; SME2-NEXT: fadd z24.d, p0/m, z24.d, #1.0
+; SME2-NEXT: fadd z20.d, p0/m, z20.d, #1.0
+; SME2-NEXT: fadd z16.d, p0/m, z16.d, #1.0
+; SME2-NEXT: st1d { z16.d, z20.d, z24.d, z28.d }, pn8, [x0]
; SME2-NEXT: ret
-;
-; SME2-NO-MV-STORES-LABEL: load_store_4x_vectors_f64_r:
-; SME2-NO-MV-STORES: // %bb.0:
-; SME2-NO-MV-STORES-NEXT: ptrue pn8.d
-; SME2-NO-MV-STORES-NEXT: ptrue p0.d
-; SME2-NO-MV-STORES-NEXT: ld1d { z16.d, z20.d, z24.d, z28.d }, pn8/z, [x0]
-; SME2-NO-MV-STORES-NEXT: fadd z28.d, p0/m, z28.d, #1.0
-; SME2-NO-MV-STORES-NEXT: fadd z24.d, p0/m, z24.d, #1.0
-; SME2-NO-MV-STORES-NEXT: fadd z20.d, p0/m, z20.d, #1.0
-; SME2-NO-MV-STORES-NEXT: fadd z16.d, p0/m, z16.d, #1.0
-; SME2-NO-MV-STORES-NEXT: str z28, [x0, #3, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z24, [x0, #2, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z20, [x0, #1, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z16, [x0]
-; SME2-NO-MV-STORES-NEXT: ret
%a = load <vscale x 8 x double>, ptr %addr
%b = fadd <vscale x 8 x double> %a, splat (double 1.0)
store <vscale x 8 x double> %b, ptr %addr
@@ -1724,32 +1265,12 @@ define void @load_store_4x_vectors_f64_rr(ptr %base, i64 %idx) {
; SME2-NEXT: ptrue p0.d
; SME2-NEXT: add x8, x0, x1, lsl #3
; SME2-NEXT: ld1d { z16.d, z20.d, z24.d, z28.d }, pn8/z, [x0, x1, lsl #3]
-; SME2-NEXT: movprfx z3, z28
-; SME2-NEXT: fadd z3.d, p0/m, z3.d, #1.0
-; SME2-NEXT: movprfx z2, z24
-; SME2-NEXT: fadd z2.d, p0/m, z2.d, #1.0
-; SME2-NEXT: movprfx z1, z20
-; SME2-NEXT: fadd z1.d, p0/m, z1.d, #1.0
-; SME2-NEXT: movprfx z0, z16
-; SME2-NEXT: fadd z0.d, p0/m, z0.d, #1.0
-; SME2-NEXT: st1d { z0.d - z3.d }, pn8, [x8]
+; SME2-NEXT: fadd z28.d, p0/m, z28.d, #1.0
+; SME2-NEXT: fadd z24.d, p0/m, z24.d, #1.0
+; SME2-NEXT: fadd z20.d, p0/m, z20.d, #1.0
+; SME2-NEXT: fadd z16.d, p0/m, z16.d, #1.0
+; SME2-NEXT: st1d { z16.d, z20.d, z24.d, z28.d }, pn8, [x8]
; SME2-NEXT: ret
-;
-; SME2-NO-MV-STORES-LABEL: load_store_4x_vectors_f64_rr:
-; SME2-NO-MV-STORES: // %bb.0:
-; SME2-NO-MV-STORES-NEXT: ptrue pn8.d
-; SME2-NO-MV-STORES-NEXT: ptrue p0.d
-; SME2-NO-MV-STORES-NEXT: add x8, x0, x1, lsl #3
-; SME2-NO-MV-STORES-NEXT: ld1d { z16.d, z20.d, z24.d, z28.d }, pn8/z, [x0, x1, lsl #3]
-; SME2-NO-MV-STORES-NEXT: fadd z16.d, p0/m, z16.d, #1.0
-; SME2-NO-MV-STORES-NEXT: fadd z28.d, p0/m, z28.d, #1.0
-; SME2-NO-MV-STORES-NEXT: fadd z24.d, p0/m, z24.d, #1.0
-; SME2-NO-MV-STORES-NEXT: fadd z20.d, p0/m, z20.d, #1.0
-; SME2-NO-MV-STORES-NEXT: st1d { z16.d }, p0, [x0, x1, lsl #3]
-; SME2-NO-MV-STORES-NEXT: str z28, [x8, #3, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z24, [x8, #2, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z20, [x8, #1, mul vl]
-; SME2-NO-MV-STORES-NEXT: ret
%addr = getelementptr double, ptr %base, i64 %idx
%a = load <vscale x 8 x double>, ptr %addr
%b = fadd <vscale x 8 x double> %a, splat (double 1.0)
@@ -1792,27 +1313,12 @@ define void @load_store_4x_vectors_bf16_r(ptr %addr) {
; SME2-NEXT: ptrue pn8.h
; SME2-NEXT: fmov z0.h, #1.87500000
; SME2-NEXT: ld1h { z16.h, z20.h, z24.h, z28.h }, pn8/z, [x0]
-; SME2-NEXT: bfadd z3.h, z28.h, z0.h
-; SME2-NEXT: bfadd z2.h, z24.h, z0.h
-; SME2-NEXT: bfadd z1.h, z20.h, z0.h
-; SME2-NEXT: bfadd z0.h, z16.h, z0.h
-; SME2-NEXT: st1h { z0.h - z3.h }, pn8, [x0]
+; SME2-NEXT: bfadd z28.h, z28.h, z0.h
+; SME2-NEXT: bfadd z24.h, z24.h, z0.h
+; SME2-NEXT: bfadd z20.h, z20.h, z0.h
+; SME2-NEXT: bfadd z16.h, z16.h, z0.h
+; SME2-NEXT: st1h { z16.h, z20.h, z24.h, z28.h }, pn8, [x0]
; SME2-NEXT: ret
-;
-; SME2-NO-MV-STORES-LABEL: load_store_4x_vectors_bf16_r:
-; SME2-NO-MV-STORES: // %bb.0:
-; SME2-NO-MV-STORES-NEXT: ptrue pn8.h
-; SME2-NO-MV-STORES-NEXT: fmov z0.h, #1.87500000
-; SME2-NO-MV-STORES-NEXT: ld1h { z16.h, z20.h, z24.h, z28.h }, pn8/z, [x0]
-; SME2-NO-MV-STORES-NEXT: bfadd z1.h, z28.h, z0.h
-; SME2-NO-MV-STORES-NEXT: bfadd z2.h, z24.h, z0.h
-; SME2-NO-MV-STORES-NEXT: str z1, [x0, #3, mul vl]
-; SME2-NO-MV-STORES-NEXT: bfadd z1.h, z20.h, z0.h
-; SME2-NO-MV-STORES-NEXT: bfadd z0.h, z16.h, z0.h
-; SME2-NO-MV-STORES-NEXT: str z2, [x0, #2, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z1, [x0, #1, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z0, [x0]
-; SME2-NO-MV-STORES-NEXT: ret
%a = load <vscale x 32 x bfloat>, ptr %addr
%b = fadd <vscale x 32 x bfloat> %a, splat (bfloat 1.0)
store <vscale x 32 x bfloat> %b, ptr %addr
@@ -1858,29 +1364,12 @@ define void @load_store_4x_vectors_bf16_rr(ptr %base, i64 %idx) {
; SME2-NEXT: fmov z0.h, #1.87500000
; SME2-NEXT: add x8, x0, x1, lsl #1
; SME2-NEXT: ld1h { z16.h, z20.h, z24.h, z28.h }, pn8/z, [x0, x1, lsl #1]
-; SME2-NEXT: bfadd z3.h, z28.h, z0.h
-; SME2-NEXT: bfadd z2.h, z24.h, z0.h
-; SME2-NEXT: bfadd z1.h, z20.h, z0.h
-; SME2-NEXT: bfadd z0.h, z16.h, z0.h
-; SME2-NEXT: st1h { z0.h - z3.h }, pn8, [x8]
+; SME2-NEXT: bfadd z28.h, z28.h, z0.h
+; SME2-NEXT: bfadd z24.h, z24.h, z0.h
+; SME2-NEXT: bfadd z20.h, z20.h, z0.h
+; SME2-NEXT: bfadd z16.h, z16.h, z0.h
+; SME2-NEXT: st1h { z16.h, z20.h, z24.h, z28.h }, pn8, [x8]
; SME2-NEXT: ret
-;
-; SME2-NO-MV-STORES-LABEL: load_store_4x_vectors_bf16_rr:
-; SME2-NO-MV-STORES: // %bb.0:
-; SME2-NO-MV-STORES-NEXT: ptrue pn8.h
-; SME2-NO-MV-STORES-NEXT: fmov z0.h, #1.87500000
-; SME2-NO-MV-STORES-NEXT: add x8, x0, x1, lsl #1
-; SME2-NO-MV-STORES-NEXT: ld1h { z16.h, z20.h, z24.h, z28.h }, pn8/z, [x0, x1, lsl #1]
-; SME2-NO-MV-STORES-NEXT: ptrue p0.h
-; SME2-NO-MV-STORES-NEXT: bfadd z1.h, z16.h, z0.h
-; SME2-NO-MV-STORES-NEXT: bfadd z2.h, z28.h, z0.h
-; SME2-NO-MV-STORES-NEXT: st1h { z1.h }, p0, [x0, x1, lsl #1]
-; SME2-NO-MV-STORES-NEXT: bfadd z1.h, z24.h, z0.h
-; SME2-NO-MV-STORES-NEXT: bfadd z0.h, z20.h, z0.h
-; SME2-NO-MV-STORES-NEXT: str z2, [x8, #3, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z1, [x8, #2, mul vl]
-; SME2-NO-MV-STORES-NEXT: str z0, [x8, #1, mul vl]
-; SME2-NO-MV-STORES-NEXT: ret
%addr = getelementptr bfloat, ptr %base, i64 %idx
%a = load <vscale x 32 x bfloat>, ptr %addr
%b = fadd <vscale x 32 x bfloat> %a, splat (bfloat 1.0)
diff --git a/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll b/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll
index dbffad1b630d8..81a0cc242fc11 100644
--- a/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll
+++ b/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc < %s -mattr=+sve -enable-subreg-liveness=true | FileCheck %s -check-prefixes=CHECK,SVE
; RUN: llc < %s -mattr=+sve,+sme2 -enable-subreg-liveness=true | FileCheck %s -check-prefixes=CHECK,SVE
-; RUN: llc < %s -mattr=+sme2 -force-streaming -enable-subreg-liveness=true -aarch64-enable-sme2-multivector-store-lowering=true | FileCheck %s -check-prefixes=CHECK,SME2,SME-ALL
-; RUN: llc < %s -mattr=+sme2 -force-streaming -aarch64-sve-vector-bits-min=256 -enable-subreg-liveness=true -aarch64-enable-sme2-multivector-store-lowering=true | FileCheck %s -check-prefixes=CHECK,SME2,SME2-256
+; RUN: llc < %s -mattr=+sme2 -force-streaming -enable-subreg-liveness=true | FileCheck %s -check-prefixes=CHECK,SME2,SME-ALL
+; RUN: llc < %s -mattr=+sme2 -force-streaming -aarch64-sve-vector-bits-min=256 -enable-subreg-liveness=true | FileCheck %s -check-prefixes=CHECK,SME2,SME2-256
target triple = "aarch64-unknown-linux-gnu"
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-stores.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-stores.ll
index f0e9efbe86ab2..9d642b25da634 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-stores.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-stores.ll
@@ -1,7 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=aarch64-linux-gnu -enable-misched=0 -mattr=+sve2p1 -enable-subreg-liveness=true < %s | FileCheck %s
-; RUN: llc -mtriple=aarch64-linux-gnu -enable-misched=0 -mattr=+sme2 -force-streaming -enable-subreg-liveness=true < %s | FileCheck %s
-; RUN: llc -mtriple=aarch64-linux-gnu -enable-misched=0 -mattr=+sme,+sve2p1 -force-streaming -enable-subreg-liveness=true < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -enable-misched=0 -mattr=+sve2p1 -enable-subreg-liveness=true -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -enable-misched=0 -mattr=+sme2 -force-streaming -enable-subreg-liveness=true -verify-machineinstrs < %s | FileCheck %s --check-prefix=SME2
+; RUN: llc -mtriple=aarch64-linux-gnu -enable-misched=0 -mattr=+sme,+sve2p1 -force-streaming -enable-subreg-liveness=true -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -enable-misched=0 -mattr=+sme2,+sve2p1 -force-streaming -enable-subreg-liveness=true -verify-machineinstrs < %s | FileCheck %s --check-prefix=SME2
; == Normal Multi-Vector Consecutive Stores ==
@@ -19,6 +20,20 @@ define void @st1_x2_i8(<vscale x 16 x i8> %unused, <vscale x 16 x i8> %zn0, <vsc
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
+;
+; SME2-LABEL: st1_x2_i8:
+; SME2: // %bb.0:
+; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; SME2-NEXT: addvl sp, sp, #-1
+; SME2-NEXT: mov z3.d, z2.d
+; SME2-NEXT: mov z2.d, z1.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: mov p8.b, p0.b
+; SME2-NEXT: st1b { z2.b, z3.b }, pn8, [x0]
+; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; SME2-NEXT: ret
call void @llvm.aarch64.sve.st1.pn.x2.nxv16i8(<vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, target("aarch64.svcount") %pn, ptr %ptr);
ret void
}
@@ -37,6 +52,20 @@ define void @st1_x2_i16(<vscale x 16 x i8> %unused, <vscale x 8 x i16> %zn0, <vs
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
+;
+; SME2-LABEL: st1_x2_i16:
+; SME2: // %bb.0:
+; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; SME2-NEXT: addvl sp, sp, #-1
+; SME2-NEXT: mov z3.d, z2.d
+; SME2-NEXT: mov z2.d, z1.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: mov p8.b, p0.b
+; SME2-NEXT: st1h { z2.h, z3.h }, pn8, [x0]
+; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; SME2-NEXT: ret
call void @llvm.aarch64.sve.st1.pn.x2.nxv8i16(<vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, target("aarch64.svcount") %pn, ptr %ptr);
ret void
}
@@ -55,6 +84,20 @@ define void @st1_x2_i32(<vscale x 16 x i8> %unused, <vscale x 4 x i32> %zn0, <vs
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
+;
+; SME2-LABEL: st1_x2_i32:
+; SME2: // %bb.0:
+; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; SME2-NEXT: addvl sp, sp, #-1
+; SME2-NEXT: mov z3.d, z2.d
+; SME2-NEXT: mov z2.d, z1.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: mov p8.b, p0.b
+; SME2-NEXT: st1w { z2.s, z3.s }, pn8, [x0]
+; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; SME2-NEXT: ret
call void @llvm.aarch64.sve.st1.pn.x2.nxv4i32(<vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1, target("aarch64.svcount") %pn, ptr %ptr);
ret void
}
@@ -73,6 +116,20 @@ define void @st1_x2_i64(<vscale x 16 x i8> %unused, <vscale x 2 x i64> %zn0, <vs
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
+;
+; SME2-LABEL: st1_x2_i64:
+; SME2: // %bb.0:
+; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; SME2-NEXT: addvl sp, sp, #-1
+; SME2-NEXT: mov z3.d, z2.d
+; SME2-NEXT: mov z2.d, z1.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: mov p8.b, p0.b
+; SME2-NEXT: st1d { z2.d, z3.d }, pn8, [x0]
+; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; SME2-NEXT: ret
call void @llvm.aarch64.sve.st1.pn.x2.nxv2i64(<vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1, target("aarch64.svcount") %pn, ptr %ptr);
ret void
}
@@ -91,6 +148,20 @@ define void @st1_x2_f16(<vscale x 16 x i8> %unused, <vscale x 8 x half> %zn0, <v
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
+;
+; SME2-LABEL: st1_x2_f16:
+; SME2: // %bb.0:
+; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; SME2-NEXT: addvl sp, sp, #-1
+; SME2-NEXT: mov z3.d, z2.d
+; SME2-NEXT: mov z2.d, z1.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: mov p8.b, p0.b
+; SME2-NEXT: st1h { z2.h, z3.h }, pn8, [x0]
+; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; SME2-NEXT: ret
call void @llvm.aarch64.sve.st1.pn.x2.nxv8f16(<vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1, target("aarch64.svcount") %pn, ptr %ptr);
ret void
}
@@ -109,6 +180,20 @@ define void @st1_x2_bf16(<vscale x 16 x i8> %unused, <vscale x 8 x bfloat> %zn0,
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
+;
+; SME2-LABEL: st1_x2_bf16:
+; SME2: // %bb.0:
+; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; SME2-NEXT: addvl sp, sp, #-1
+; SME2-NEXT: mov z3.d, z2.d
+; SME2-NEXT: mov z2.d, z1.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: mov p8.b, p0.b
+; SME2-NEXT: st1h { z2.h, z3.h }, pn8, [x0]
+; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; SME2-NEXT: ret
call void @llvm.aarch64.sve.st1.pn.x2.nxv8bf16(<vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1, target("aarch64.svcount") %pn, ptr %ptr);
ret void
}
@@ -127,6 +212,20 @@ define void @st1_x2_f32(<vscale x 16 x i8> %unused, <vscale x 4 x float> %zn0, <
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
+;
+; SME2-LABEL: st1_x2_f32:
+; SME2: // %bb.0:
+; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; SME2-NEXT: addvl sp, sp, #-1
+; SME2-NEXT: mov z3.d, z2.d
+; SME2-NEXT: mov z2.d, z1.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: mov p8.b, p0.b
+; SME2-NEXT: st1w { z2.s, z3.s }, pn8, [x0]
+; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; SME2-NEXT: ret
call void @llvm.aarch64.sve.st1.pn.x2.nxv4f32(<vscale x 4 x float> %zn0, <vscale x 4 x float> %zn1, target("aarch64.svcount") %pn, ptr %ptr);
ret void
}
@@ -145,6 +244,20 @@ define void @st1_x2_f64(<vscale x 16 x i8> %unused, <vscale x 2 x double> %zn0,
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
+;
+; SME2-LABEL: st1_x2_f64:
+; SME2: // %bb.0:
+; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; SME2-NEXT: addvl sp, sp, #-1
+; SME2-NEXT: mov z3.d, z2.d
+; SME2-NEXT: mov z2.d, z1.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: mov p8.b, p0.b
+; SME2-NEXT: st1d { z2.d, z3.d }, pn8, [x0]
+; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; SME2-NEXT: ret
call void @llvm.aarch64.sve.st1.pn.x2.nxv2f64(<vscale x 2 x double> %zn0, <vscale x 2 x double> %zn1, target("aarch64.svcount") %pn, ptr %ptr);
ret void
}
@@ -165,6 +278,22 @@ define void @st1_x4_i8(<vscale x 16 x i8> %unused, <vscale x 16 x i8> %zn0, <vsc
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
+;
+; SME2-LABEL: st1_x4_i8:
+; SME2: // %bb.0:
+; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; SME2-NEXT: addvl sp, sp, #-1
+; SME2-NEXT: mov z7.d, z4.d
+; SME2-NEXT: mov z6.d, z3.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: mov z5.d, z2.d
+; SME2-NEXT: mov z4.d, z1.d
+; SME2-NEXT: mov p8.b, p0.b
+; SME2-NEXT: st1b { z4.b - z7.b }, pn8, [x0]
+; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; SME2-NEXT: ret
call void @llvm.aarch64.sve.st1.pn.x4.nxv16i8(<vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3, target("aarch64.svcount") %pn, ptr %ptr);
ret void
}
@@ -185,6 +314,22 @@ define void @st1_x4_i16(<vscale x 16 x i8> %unused, <vscale x 8 x i16> %zn0, <vs
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
+;
+; SME2-LABEL: st1_x4_i16:
+; SME2: // %bb.0:
+; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; SME2-NEXT: addvl sp, sp, #-1
+; SME2-NEXT: mov z7.d, z4.d
+; SME2-NEXT: mov z6.d, z3.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: mov z5.d, z2.d
+; SME2-NEXT: mov z4.d, z1.d
+; SME2-NEXT: mov p8.b, p0.b
+; SME2-NEXT: st1h { z4.h - z7.h }, pn8, [x0]
+; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; SME2-NEXT: ret
call void @llvm.aarch64.sve.st1.pn.x4.nxv8i16(<vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3, target("aarch64.svcount") %pn, ptr %ptr);
ret void
}
@@ -205,6 +350,22 @@ define void @st1_x4_i32(<vscale x 16 x i8> %unused, <vscale x 4 x i32> %zn0, <vs
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
+;
+; SME2-LABEL: st1_x4_i32:
+; SME2: // %bb.0:
+; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; SME2-NEXT: addvl sp, sp, #-1
+; SME2-NEXT: mov z7.d, z4.d
+; SME2-NEXT: mov z6.d, z3.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: mov z5.d, z2.d
+; SME2-NEXT: mov z4.d, z1.d
+; SME2-NEXT: mov p8.b, p0.b
+; SME2-NEXT: st1w { z4.s - z7.s }, pn8, [x0]
+; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; SME2-NEXT: ret
call void @llvm.aarch64.sve.st1.pn.x4.nxv4i32(<vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3, target("aarch64.svcount") %pn, ptr %ptr);
ret void
}
@@ -225,6 +386,22 @@ define void @st1_x4_i64(<vscale x 16 x i8> %unused, <vscale x 2 x i64> %zn0, <vs
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
+;
+; SME2-LABEL: st1_x4_i64:
+; SME2: // %bb.0:
+; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; SME2-NEXT: addvl sp, sp, #-1
+; SME2-NEXT: mov z7.d, z4.d
+; SME2-NEXT: mov z6.d, z3.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: mov z5.d, z2.d
+; SME2-NEXT: mov z4.d, z1.d
+; SME2-NEXT: mov p8.b, p0.b
+; SME2-NEXT: st1d { z4.d - z7.d }, pn8, [x0]
+; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; SME2-NEXT: ret
call void @llvm.aarch64.sve.st1.pn.x4.nxv2i64(<vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1, <vscale x 2 x i64> %zn2, <vscale x 2 x i64> %zn3, target("aarch64.svcount") %pn, ptr %ptr);
ret void
}
@@ -245,6 +422,22 @@ define void @st1_x4_f16(<vscale x 16 x i8> %unused, <vscale x 8 x half> %zn0, <v
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
+;
+; SME2-LABEL: st1_x4_f16:
+; SME2: // %bb.0:
+; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; SME2-NEXT: addvl sp, sp, #-1
+; SME2-NEXT: mov z7.d, z4.d
+; SME2-NEXT: mov z6.d, z3.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: mov z5.d, z2.d
+; SME2-NEXT: mov z4.d, z1.d
+; SME2-NEXT: mov p8.b, p0.b
+; SME2-NEXT: st1h { z4.h - z7.h }, pn8, [x0]
+; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; SME2-NEXT: ret
call void @llvm.aarch64.sve.st1.pn.x4.nxv8f16(<vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3, target("aarch64.svcount") %pn, ptr %ptr);
ret void
}
@@ -265,6 +458,22 @@ define void @st1_x4_bf16(<vscale x 16 x i8> %unused, <vscale x 8 x bfloat> %zn0,
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
+;
+; SME2-LABEL: st1_x4_bf16:
+; SME2: // %bb.0:
+; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; SME2-NEXT: addvl sp, sp, #-1
+; SME2-NEXT: mov z7.d, z4.d
+; SME2-NEXT: mov z6.d, z3.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: mov z5.d, z2.d
+; SME2-NEXT: mov z4.d, z1.d
+; SME2-NEXT: mov p8.b, p0.b
+; SME2-NEXT: st1h { z4.h - z7.h }, pn8, [x0]
+; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; SME2-NEXT: ret
call void @llvm.aarch64.sve.st1.pn.x4.nxv8bf16(<vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3, target("aarch64.svcount") %pn, ptr %ptr);
ret void
}
@@ -285,6 +494,22 @@ define void @st1_x4_f32(<vscale x 16 x i8> %unused, <vscale x 4 x float> %zn0, <
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
+;
+; SME2-LABEL: st1_x4_f32:
+; SME2: // %bb.0:
+; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; SME2-NEXT: addvl sp, sp, #-1
+; SME2-NEXT: mov z7.d, z4.d
+; SME2-NEXT: mov z6.d, z3.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: mov z5.d, z2.d
+; SME2-NEXT: mov z4.d, z1.d
+; SME2-NEXT: mov p8.b, p0.b
+; SME2-NEXT: st1w { z4.s - z7.s }, pn8, [x0]
+; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; SME2-NEXT: ret
call void @llvm.aarch64.sve.st1.pn.x4.nxv4f32(<vscale x 4 x float> %zn0, <vscale x 4 x float> %zn1, <vscale x 4 x float> %zn2, <vscale x 4 x float> %zn3, target("aarch64.svcount") %pn, ptr %ptr);
ret void
}
@@ -305,6 +530,22 @@ define void @st1_x4_f64(<vscale x 16 x i8> %unused, <vscale x 2 x double> %zn0,
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
+;
+; SME2-LABEL: st1_x4_f64:
+; SME2: // %bb.0:
+; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; SME2-NEXT: addvl sp, sp, #-1
+; SME2-NEXT: mov z7.d, z4.d
+; SME2-NEXT: mov z6.d, z3.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: mov z5.d, z2.d
+; SME2-NEXT: mov z4.d, z1.d
+; SME2-NEXT: mov p8.b, p0.b
+; SME2-NEXT: st1d { z4.d - z7.d }, pn8, [x0]
+; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; SME2-NEXT: ret
call void @llvm.aarch64.sve.st1.pn.x4.nxv2f64(<vscale x 2 x double> %zn0, <vscale x 2 x double> %zn1, <vscale x 2 x double> %zn2, <vscale x 2 x double> %zn3, target("aarch64.svcount") %pn, ptr %ptr);
ret void
}
@@ -325,6 +566,20 @@ define void @stnt1_x2_i8(<vscale x 16 x i8> %unused, <vscale x 16 x i8> %zn0, <v
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
+;
+; SME2-LABEL: stnt1_x2_i8:
+; SME2: // %bb.0:
+; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; SME2-NEXT: addvl sp, sp, #-1
+; SME2-NEXT: mov z3.d, z2.d
+; SME2-NEXT: mov z2.d, z1.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: mov p8.b, p0.b
+; SME2-NEXT: stnt1b { z2.b, z3.b }, pn8, [x0]
+; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; SME2-NEXT: ret
call void @llvm.aarch64.sve.stnt1.pn.x2.nxv16i8(<vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, target("aarch64.svcount") %pn, ptr %ptr);
ret void
}
@@ -343,6 +598,20 @@ define void @stnt1_x2_i16(<vscale x 16 x i8> %unused, <vscale x 8 x i16> %zn0, <
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
+;
+; SME2-LABEL: stnt1_x2_i16:
+; SME2: // %bb.0:
+; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; SME2-NEXT: addvl sp, sp, #-1
+; SME2-NEXT: mov z3.d, z2.d
+; SME2-NEXT: mov z2.d, z1.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: mov p8.b, p0.b
+; SME2-NEXT: stnt1h { z2.h, z3.h }, pn8, [x0]
+; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; SME2-NEXT: ret
call void @llvm.aarch64.sve.stnt1.pn.x2.nxv8i16(<vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, target("aarch64.svcount") %pn, ptr %ptr);
ret void
}
@@ -361,6 +630,20 @@ define void @stnt1_x2_i32(<vscale x 16 x i8> %unused, <vscale x 4 x i32> %zn0, <
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
+;
+; SME2-LABEL: stnt1_x2_i32:
+; SME2: // %bb.0:
+; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; SME2-NEXT: addvl sp, sp, #-1
+; SME2-NEXT: mov z3.d, z2.d
+; SME2-NEXT: mov z2.d, z1.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: mov p8.b, p0.b
+; SME2-NEXT: stnt1w { z2.s, z3.s }, pn8, [x0]
+; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; SME2-NEXT: ret
call void @llvm.aarch64.sve.stnt1.pn.x2.nxv4i32(<vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1, target("aarch64.svcount") %pn, ptr %ptr);
ret void
}
@@ -379,6 +662,20 @@ define void @stnt1_x2_i64(<vscale x 16 x i8> %unused, <vscale x 2 x i64> %zn0, <
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
+;
+; SME2-LABEL: stnt1_x2_i64:
+; SME2: // %bb.0:
+; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; SME2-NEXT: addvl sp, sp, #-1
+; SME2-NEXT: mov z3.d, z2.d
+; SME2-NEXT: mov z2.d, z1.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: mov p8.b, p0.b
+; SME2-NEXT: stnt1d { z2.d, z3.d }, pn8, [x0]
+; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; SME2-NEXT: ret
call void @llvm.aarch64.sve.stnt1.pn.x2.nxv2i64(<vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1, target("aarch64.svcount") %pn, ptr %ptr);
ret void
}
@@ -397,6 +694,20 @@ define void @stnt1_x2_f16(<vscale x 16 x i8> %unused, <vscale x 8 x half> %zn0,
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
+;
+; SME2-LABEL: stnt1_x2_f16:
+; SME2: // %bb.0:
+; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; SME2-NEXT: addvl sp, sp, #-1
+; SME2-NEXT: mov z3.d, z2.d
+; SME2-NEXT: mov z2.d, z1.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: mov p8.b, p0.b
+; SME2-NEXT: stnt1h { z2.h, z3.h }, pn8, [x0]
+; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; SME2-NEXT: ret
call void @llvm.aarch64.sve.stnt1.pn.x2.nxv8f16(<vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1, target("aarch64.svcount") %pn, ptr %ptr);
ret void
}
@@ -415,6 +726,20 @@ define void @stnt1_x2_bf16(<vscale x 16 x i8> %unused, <vscale x 8 x bfloat> %zn
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
+;
+; SME2-LABEL: stnt1_x2_bf16:
+; SME2: // %bb.0:
+; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; SME2-NEXT: addvl sp, sp, #-1
+; SME2-NEXT: mov z3.d, z2.d
+; SME2-NEXT: mov z2.d, z1.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: mov p8.b, p0.b
+; SME2-NEXT: stnt1h { z2.h, z3.h }, pn8, [x0]
+; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; SME2-NEXT: ret
call void @llvm.aarch64.sve.stnt1.pn.x2.nxv8bf16(<vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1, target("aarch64.svcount") %pn, ptr %ptr);
ret void
}
@@ -433,6 +758,20 @@ define void @stnt1_x2_f32(<vscale x 16 x i8> %unused, <vscale x 4 x float> %zn0,
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
+;
+; SME2-LABEL: stnt1_x2_f32:
+; SME2: // %bb.0:
+; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; SME2-NEXT: addvl sp, sp, #-1
+; SME2-NEXT: mov z3.d, z2.d
+; SME2-NEXT: mov z2.d, z1.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: mov p8.b, p0.b
+; SME2-NEXT: stnt1w { z2.s, z3.s }, pn8, [x0]
+; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; SME2-NEXT: ret
call void @llvm.aarch64.sve.stnt1.pn.x2.nxv4f32(<vscale x 4 x float> %zn0, <vscale x 4 x float> %zn1, target("aarch64.svcount") %pn, ptr %ptr);
ret void
}
@@ -451,6 +790,20 @@ define void @stnt1_x2_f64(<vscale x 16 x i8> %unused, <vscale x 2 x double> %zn0
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
+;
+; SME2-LABEL: stnt1_x2_f64:
+; SME2: // %bb.0:
+; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; SME2-NEXT: addvl sp, sp, #-1
+; SME2-NEXT: mov z3.d, z2.d
+; SME2-NEXT: mov z2.d, z1.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: mov p8.b, p0.b
+; SME2-NEXT: stnt1d { z2.d, z3.d }, pn8, [x0]
+; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; SME2-NEXT: ret
call void @llvm.aarch64.sve.stnt1.pn.x2.nxv2f64(<vscale x 2 x double> %zn0, <vscale x 2 x double> %zn1, target("aarch64.svcount") %pn, ptr %ptr);
ret void
}
@@ -471,6 +824,22 @@ define void @stnt1_x4_i8(<vscale x 16 x i8> %unused, <vscale x 16 x i8> %zn0, <v
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
+;
+; SME2-LABEL: stnt1_x4_i8:
+; SME2: // %bb.0:
+; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; SME2-NEXT: addvl sp, sp, #-1
+; SME2-NEXT: mov z7.d, z4.d
+; SME2-NEXT: mov z6.d, z3.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: mov z5.d, z2.d
+; SME2-NEXT: mov z4.d, z1.d
+; SME2-NEXT: mov p8.b, p0.b
+; SME2-NEXT: stnt1b { z4.b - z7.b }, pn8, [x0]
+; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; SME2-NEXT: ret
call void @llvm.aarch64.sve.stnt1.pn.x4.nxv16i8(<vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3, target("aarch64.svcount") %pn, ptr %ptr);
ret void
}
@@ -491,6 +860,22 @@ define void @stnt1_x4_i16(<vscale x 16 x i8> %unused, <vscale x 8 x i16> %zn0, <
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
+;
+; SME2-LABEL: stnt1_x4_i16:
+; SME2: // %bb.0:
+; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; SME2-NEXT: addvl sp, sp, #-1
+; SME2-NEXT: mov z7.d, z4.d
+; SME2-NEXT: mov z6.d, z3.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: mov z5.d, z2.d
+; SME2-NEXT: mov z4.d, z1.d
+; SME2-NEXT: mov p8.b, p0.b
+; SME2-NEXT: stnt1h { z4.h - z7.h }, pn8, [x0]
+; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; SME2-NEXT: ret
call void @llvm.aarch64.sve.stnt1.pn.x4.nxv8i16(<vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3, target("aarch64.svcount") %pn, ptr %ptr);
ret void
}
@@ -511,6 +896,22 @@ define void @stnt1_x4_i32(<vscale x 16 x i8> %unused, <vscale x 4 x i32> %zn0, <
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
+;
+; SME2-LABEL: stnt1_x4_i32:
+; SME2: // %bb.0:
+; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; SME2-NEXT: addvl sp, sp, #-1
+; SME2-NEXT: mov z7.d, z4.d
+; SME2-NEXT: mov z6.d, z3.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: mov z5.d, z2.d
+; SME2-NEXT: mov z4.d, z1.d
+; SME2-NEXT: mov p8.b, p0.b
+; SME2-NEXT: stnt1w { z4.s - z7.s }, pn8, [x0]
+; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; SME2-NEXT: ret
call void @llvm.aarch64.sve.stnt1.pn.x4.nxv4i32(<vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3, target("aarch64.svcount") %pn, ptr %ptr);
ret void
}
@@ -531,6 +932,22 @@ define void @stnt1_x4_i64(<vscale x 16 x i8> %unused, <vscale x 2 x i64> %zn0, <
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
+;
+; SME2-LABEL: stnt1_x4_i64:
+; SME2: // %bb.0:
+; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; SME2-NEXT: addvl sp, sp, #-1
+; SME2-NEXT: mov z7.d, z4.d
+; SME2-NEXT: mov z6.d, z3.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: mov z5.d, z2.d
+; SME2-NEXT: mov z4.d, z1.d
+; SME2-NEXT: mov p8.b, p0.b
+; SME2-NEXT: stnt1d { z4.d - z7.d }, pn8, [x0]
+; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; SME2-NEXT: ret
call void @llvm.aarch64.sve.stnt1.pn.x4.nxv2i64(<vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1, <vscale x 2 x i64> %zn2, <vscale x 2 x i64> %zn3, target("aarch64.svcount") %pn, ptr %ptr);
ret void
}
@@ -551,6 +968,22 @@ define void @stnt1_x4_f16(<vscale x 16 x i8> %unused, <vscale x 8 x half> %zn0,
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
+;
+; SME2-LABEL: stnt1_x4_f16:
+; SME2: // %bb.0:
+; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; SME2-NEXT: addvl sp, sp, #-1
+; SME2-NEXT: mov z7.d, z4.d
+; SME2-NEXT: mov z6.d, z3.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: mov z5.d, z2.d
+; SME2-NEXT: mov z4.d, z1.d
+; SME2-NEXT: mov p8.b, p0.b
+; SME2-NEXT: stnt1h { z4.h - z7.h }, pn8, [x0]
+; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; SME2-NEXT: ret
call void @llvm.aarch64.sve.stnt1.pn.x4.nxv8f16(<vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3, target("aarch64.svcount") %pn, ptr %ptr);
ret void
}
@@ -571,6 +1004,22 @@ define void @stnt1_x4_bf16(<vscale x 16 x i8> %unused, <vscale x 8 x bfloat> %zn
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
+;
+; SME2-LABEL: stnt1_x4_bf16:
+; SME2: // %bb.0:
+; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; SME2-NEXT: addvl sp, sp, #-1
+; SME2-NEXT: mov z7.d, z4.d
+; SME2-NEXT: mov z6.d, z3.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: mov z5.d, z2.d
+; SME2-NEXT: mov z4.d, z1.d
+; SME2-NEXT: mov p8.b, p0.b
+; SME2-NEXT: stnt1h { z4.h - z7.h }, pn8, [x0]
+; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; SME2-NEXT: ret
call void @llvm.aarch64.sve.stnt1.pn.x4.nxv8bf16(<vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3, target("aarch64.svcount") %pn, ptr %ptr);
ret void
}
@@ -591,6 +1040,22 @@ define void @stnt1_x4_f32(<vscale x 16 x i8> %unused, <vscale x 4 x float> %zn0,
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
+;
+; SME2-LABEL: stnt1_x4_f32:
+; SME2: // %bb.0:
+; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; SME2-NEXT: addvl sp, sp, #-1
+; SME2-NEXT: mov z7.d, z4.d
+; SME2-NEXT: mov z6.d, z3.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: mov z5.d, z2.d
+; SME2-NEXT: mov z4.d, z1.d
+; SME2-NEXT: mov p8.b, p0.b
+; SME2-NEXT: stnt1w { z4.s - z7.s }, pn8, [x0]
+; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; SME2-NEXT: ret
call void @llvm.aarch64.sve.stnt1.pn.x4.nxv4f32(<vscale x 4 x float> %zn0, <vscale x 4 x float> %zn1, <vscale x 4 x float> %zn2, <vscale x 4 x float> %zn3, target("aarch64.svcount") %pn, ptr %ptr);
ret void
}
@@ -611,6 +1076,22 @@ define void @stnt1_x4_f64(<vscale x 16 x i8> %unused, <vscale x 2 x double> %zn0
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
+;
+; SME2-LABEL: stnt1_x4_f64:
+; SME2: // %bb.0:
+; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; SME2-NEXT: addvl sp, sp, #-1
+; SME2-NEXT: mov z7.d, z4.d
+; SME2-NEXT: mov z6.d, z3.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: mov z5.d, z2.d
+; SME2-NEXT: mov z4.d, z1.d
+; SME2-NEXT: mov p8.b, p0.b
+; SME2-NEXT: stnt1d { z4.d - z7.d }, pn8, [x0]
+; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; SME2-NEXT: ret
call void @llvm.aarch64.sve.stnt1.pn.x4.nxv2f64(<vscale x 2 x double> %zn0, <vscale x 2 x double> %zn1, <vscale x 2 x double> %zn2, <vscale x 2 x double> %zn3, target("aarch64.svcount") %pn, ptr %ptr);
ret void
}
>From 4c848fda595adab6f6095b9c1a940f321a9980ea Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Tue, 28 Jul 2026 09:09:09 +0000
Subject: [PATCH 2/4] rm hint
---
.../Target/AArch64/AArch64RegisterInfo.cpp | 29 +-
.../AArch64/sve2p1-intrinsics-stores.ll | 448 ++++++++++--------
2 files changed, 264 insertions(+), 213 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
index bdcd6bd8a6c07..edca3e94a0afe 100644
--- a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
@@ -1196,31 +1196,14 @@ bool AArch64RegisterInfo::getRegAllocationHints(
const AArch64InstrInfo *TII =
MF.getSubtarget<AArch64Subtarget>().getInstrInfo();
const MachineRegisterInfo &MRI = MF.getRegInfo();
- const TargetRegisterClass *RegRC = MRI.getRegClass(VirtReg);
- unsigned RegID = RegRC->getID();
bool ConsiderOnlyHints =
TargetRegisterInfo::getRegAllocationHints(VirtReg, Order, Hints, MF, VRM);
- // Discard generic strided-tuple hints that would use an otherwise unused
- // callee-saved Z register. Registers already defined or allocated do not
- // introduce an additional spill.
- if (ST.hasSME() && ST.isStreaming() &&
- (RegID == AArch64::ZPR2StridedOrContiguousRegClassID ||
- RegID == AArch64::ZPR4StridedOrContiguousRegClassID)) {
- llvm::erase_if(Hints, [&](MCPhysReg Hint) {
- for (const MCPhysReg *CSR = MRI.getCalleeSavedRegs(); *CSR; ++CSR) {
- if (regsOverlap(Hint, *CSR) && MRI.def_empty(*CSR) &&
- !Matrix->isPhysRegUsed(*CSR))
- return true;
- }
- return false;
- });
- }
-
// For predicated SVE instructions where the inactive lanes are undef,
// pick a destination register that is not unique to avoid introducing
// a movprfx.
+ const TargetRegisterClass *RegRC = MRI.getRegClass(VirtReg);
if (AArch64::ZPRRegClass.hasSubClassEq(RegRC)) {
for (const MachineOperand &DefOp : MRI.def_operands(VirtReg)) {
const MachineInstr &Def = *DefOp.getParent();
@@ -1283,7 +1266,8 @@ bool AArch64RegisterInfo::getRegAllocationHints(
return ConsiderOnlyHints;
if (!ST.hasSME() || !ST.isStreaming())
- return ConsiderOnlyHints;
+ return TargetRegisterInfo::getRegAllocationHints(VirtReg, Order, Hints, MF,
+ VRM);
// The SVE calling convention preserves registers Z8-Z23. As a result, there
// are no ZPR2Strided or ZPR4Strided registers that do not overlap with the
@@ -1293,6 +1277,7 @@ bool AArch64RegisterInfo::getRegAllocationHints(
// COPY_INTO_TRANSPOSED_TUPLE pseudos, we want to favour reducing copy
// instructions over reducing the number of clobbered callee-save registers,
// so we add the strided registers as a hint.
+ unsigned RegID = RegRC->getID();
if (RegID == AArch64::ZPR2StridedOrContiguousRegClassID ||
RegID == AArch64::ZPR4StridedOrContiguousRegClassID) {
@@ -1424,7 +1409,8 @@ bool AArch64RegisterInfo::getRegAllocationHints(
}
if (!Hints.empty())
- return ConsiderOnlyHints;
+ return TargetRegisterInfo::getRegAllocationHints(VirtReg, Order, Hints,
+ MF, VRM);
}
}
@@ -1455,7 +1441,8 @@ bool AArch64RegisterInfo::getRegAllocationHints(
}
}
- return ConsiderOnlyHints;
+ return TargetRegisterInfo::getRegAllocationHints(VirtReg, Order, Hints, MF,
+ VRM);
}
unsigned AArch64RegisterInfo::getLocalAddressRegister(
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-stores.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-stores.ll
index 9d642b25da634..c2ce411fa391e 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-stores.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-stores.ll
@@ -24,14 +24,15 @@ define void @st1_x2_i8(<vscale x 16 x i8> %unused, <vscale x 16 x i8> %zn0, <vsc
; SME2-LABEL: st1_x2_i8:
; SME2: // %bb.0:
; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; SME2-NEXT: addvl sp, sp, #-1
-; SME2-NEXT: mov z3.d, z2.d
-; SME2-NEXT: mov z2.d, z1.d
+; SME2-NEXT: addvl sp, sp, #-2
+; SME2-NEXT: str z9, [sp, #1, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z9.d, z2.d
; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; SME2-NEXT: mov p8.b, p0.b
-; SME2-NEXT: st1b { z2.b, z3.b }, pn8, [x0]
+; SME2-NEXT: st1b { z1.b, z9.b }, pn8, [x0]
+; SME2-NEXT: ldr z9, [sp, #1, mul vl] // 16-byte Folded Reload
; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
-; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: addvl sp, sp, #2
; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; SME2-NEXT: ret
call void @llvm.aarch64.sve.st1.pn.x2.nxv16i8(<vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, target("aarch64.svcount") %pn, ptr %ptr);
@@ -56,14 +57,15 @@ define void @st1_x2_i16(<vscale x 16 x i8> %unused, <vscale x 8 x i16> %zn0, <vs
; SME2-LABEL: st1_x2_i16:
; SME2: // %bb.0:
; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; SME2-NEXT: addvl sp, sp, #-1
-; SME2-NEXT: mov z3.d, z2.d
-; SME2-NEXT: mov z2.d, z1.d
+; SME2-NEXT: addvl sp, sp, #-2
+; SME2-NEXT: str z9, [sp, #1, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z9.d, z2.d
; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; SME2-NEXT: mov p8.b, p0.b
-; SME2-NEXT: st1h { z2.h, z3.h }, pn8, [x0]
+; SME2-NEXT: st1h { z1.h, z9.h }, pn8, [x0]
+; SME2-NEXT: ldr z9, [sp, #1, mul vl] // 16-byte Folded Reload
; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
-; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: addvl sp, sp, #2
; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; SME2-NEXT: ret
call void @llvm.aarch64.sve.st1.pn.x2.nxv8i16(<vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, target("aarch64.svcount") %pn, ptr %ptr);
@@ -88,14 +90,15 @@ define void @st1_x2_i32(<vscale x 16 x i8> %unused, <vscale x 4 x i32> %zn0, <vs
; SME2-LABEL: st1_x2_i32:
; SME2: // %bb.0:
; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; SME2-NEXT: addvl sp, sp, #-1
-; SME2-NEXT: mov z3.d, z2.d
-; SME2-NEXT: mov z2.d, z1.d
+; SME2-NEXT: addvl sp, sp, #-2
+; SME2-NEXT: str z9, [sp, #1, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z9.d, z2.d
; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; SME2-NEXT: mov p8.b, p0.b
-; SME2-NEXT: st1w { z2.s, z3.s }, pn8, [x0]
+; SME2-NEXT: st1w { z1.s, z9.s }, pn8, [x0]
+; SME2-NEXT: ldr z9, [sp, #1, mul vl] // 16-byte Folded Reload
; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
-; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: addvl sp, sp, #2
; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; SME2-NEXT: ret
call void @llvm.aarch64.sve.st1.pn.x2.nxv4i32(<vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1, target("aarch64.svcount") %pn, ptr %ptr);
@@ -120,14 +123,15 @@ define void @st1_x2_i64(<vscale x 16 x i8> %unused, <vscale x 2 x i64> %zn0, <vs
; SME2-LABEL: st1_x2_i64:
; SME2: // %bb.0:
; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; SME2-NEXT: addvl sp, sp, #-1
-; SME2-NEXT: mov z3.d, z2.d
-; SME2-NEXT: mov z2.d, z1.d
+; SME2-NEXT: addvl sp, sp, #-2
+; SME2-NEXT: str z9, [sp, #1, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z9.d, z2.d
; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; SME2-NEXT: mov p8.b, p0.b
-; SME2-NEXT: st1d { z2.d, z3.d }, pn8, [x0]
+; SME2-NEXT: st1d { z1.d, z9.d }, pn8, [x0]
+; SME2-NEXT: ldr z9, [sp, #1, mul vl] // 16-byte Folded Reload
; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
-; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: addvl sp, sp, #2
; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; SME2-NEXT: ret
call void @llvm.aarch64.sve.st1.pn.x2.nxv2i64(<vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1, target("aarch64.svcount") %pn, ptr %ptr);
@@ -152,14 +156,15 @@ define void @st1_x2_f16(<vscale x 16 x i8> %unused, <vscale x 8 x half> %zn0, <v
; SME2-LABEL: st1_x2_f16:
; SME2: // %bb.0:
; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; SME2-NEXT: addvl sp, sp, #-1
-; SME2-NEXT: mov z3.d, z2.d
-; SME2-NEXT: mov z2.d, z1.d
+; SME2-NEXT: addvl sp, sp, #-2
+; SME2-NEXT: str z9, [sp, #1, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z9.d, z2.d
; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; SME2-NEXT: mov p8.b, p0.b
-; SME2-NEXT: st1h { z2.h, z3.h }, pn8, [x0]
+; SME2-NEXT: st1h { z1.h, z9.h }, pn8, [x0]
+; SME2-NEXT: ldr z9, [sp, #1, mul vl] // 16-byte Folded Reload
; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
-; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: addvl sp, sp, #2
; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; SME2-NEXT: ret
call void @llvm.aarch64.sve.st1.pn.x2.nxv8f16(<vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1, target("aarch64.svcount") %pn, ptr %ptr);
@@ -184,14 +189,15 @@ define void @st1_x2_bf16(<vscale x 16 x i8> %unused, <vscale x 8 x bfloat> %zn0,
; SME2-LABEL: st1_x2_bf16:
; SME2: // %bb.0:
; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; SME2-NEXT: addvl sp, sp, #-1
-; SME2-NEXT: mov z3.d, z2.d
-; SME2-NEXT: mov z2.d, z1.d
+; SME2-NEXT: addvl sp, sp, #-2
+; SME2-NEXT: str z9, [sp, #1, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z9.d, z2.d
; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; SME2-NEXT: mov p8.b, p0.b
-; SME2-NEXT: st1h { z2.h, z3.h }, pn8, [x0]
+; SME2-NEXT: st1h { z1.h, z9.h }, pn8, [x0]
+; SME2-NEXT: ldr z9, [sp, #1, mul vl] // 16-byte Folded Reload
; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
-; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: addvl sp, sp, #2
; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; SME2-NEXT: ret
call void @llvm.aarch64.sve.st1.pn.x2.nxv8bf16(<vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1, target("aarch64.svcount") %pn, ptr %ptr);
@@ -216,14 +222,15 @@ define void @st1_x2_f32(<vscale x 16 x i8> %unused, <vscale x 4 x float> %zn0, <
; SME2-LABEL: st1_x2_f32:
; SME2: // %bb.0:
; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; SME2-NEXT: addvl sp, sp, #-1
-; SME2-NEXT: mov z3.d, z2.d
-; SME2-NEXT: mov z2.d, z1.d
+; SME2-NEXT: addvl sp, sp, #-2
+; SME2-NEXT: str z9, [sp, #1, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z9.d, z2.d
; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; SME2-NEXT: mov p8.b, p0.b
-; SME2-NEXT: st1w { z2.s, z3.s }, pn8, [x0]
+; SME2-NEXT: st1w { z1.s, z9.s }, pn8, [x0]
+; SME2-NEXT: ldr z9, [sp, #1, mul vl] // 16-byte Folded Reload
; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
-; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: addvl sp, sp, #2
; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; SME2-NEXT: ret
call void @llvm.aarch64.sve.st1.pn.x2.nxv4f32(<vscale x 4 x float> %zn0, <vscale x 4 x float> %zn1, target("aarch64.svcount") %pn, ptr %ptr);
@@ -248,14 +255,15 @@ define void @st1_x2_f64(<vscale x 16 x i8> %unused, <vscale x 2 x double> %zn0,
; SME2-LABEL: st1_x2_f64:
; SME2: // %bb.0:
; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; SME2-NEXT: addvl sp, sp, #-1
-; SME2-NEXT: mov z3.d, z2.d
-; SME2-NEXT: mov z2.d, z1.d
+; SME2-NEXT: addvl sp, sp, #-2
+; SME2-NEXT: str z9, [sp, #1, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z9.d, z2.d
; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; SME2-NEXT: mov p8.b, p0.b
-; SME2-NEXT: st1d { z2.d, z3.d }, pn8, [x0]
+; SME2-NEXT: st1d { z1.d, z9.d }, pn8, [x0]
+; SME2-NEXT: ldr z9, [sp, #1, mul vl] // 16-byte Folded Reload
; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
-; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: addvl sp, sp, #2
; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; SME2-NEXT: ret
call void @llvm.aarch64.sve.st1.pn.x2.nxv2f64(<vscale x 2 x double> %zn0, <vscale x 2 x double> %zn1, target("aarch64.svcount") %pn, ptr %ptr);
@@ -282,16 +290,19 @@ define void @st1_x4_i8(<vscale x 16 x i8> %unused, <vscale x 16 x i8> %zn0, <vsc
; SME2-LABEL: st1_x4_i8:
; SME2: // %bb.0:
; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; SME2-NEXT: addvl sp, sp, #-1
-; SME2-NEXT: mov z7.d, z4.d
-; SME2-NEXT: mov z6.d, z3.d
-; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: addvl sp, sp, #-3
+; SME2-NEXT: str z13, [sp, #1, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z13.d, z4.d
; SME2-NEXT: mov z5.d, z2.d
-; SME2-NEXT: mov z4.d, z1.d
+; SME2-NEXT: str z9, [sp, #2, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z9.d, z3.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; SME2-NEXT: mov p8.b, p0.b
-; SME2-NEXT: st1b { z4.b - z7.b }, pn8, [x0]
+; SME2-NEXT: st1b { z1.b, z5.b, z9.b, z13.b }, pn8, [x0]
+; SME2-NEXT: ldr z13, [sp, #1, mul vl] // 16-byte Folded Reload
+; SME2-NEXT: ldr z9, [sp, #2, mul vl] // 16-byte Folded Reload
; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
-; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: addvl sp, sp, #3
; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; SME2-NEXT: ret
call void @llvm.aarch64.sve.st1.pn.x4.nxv16i8(<vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3, target("aarch64.svcount") %pn, ptr %ptr);
@@ -318,16 +329,19 @@ define void @st1_x4_i16(<vscale x 16 x i8> %unused, <vscale x 8 x i16> %zn0, <vs
; SME2-LABEL: st1_x4_i16:
; SME2: // %bb.0:
; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; SME2-NEXT: addvl sp, sp, #-1
-; SME2-NEXT: mov z7.d, z4.d
-; SME2-NEXT: mov z6.d, z3.d
-; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: addvl sp, sp, #-3
+; SME2-NEXT: str z13, [sp, #1, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z13.d, z4.d
; SME2-NEXT: mov z5.d, z2.d
-; SME2-NEXT: mov z4.d, z1.d
+; SME2-NEXT: str z9, [sp, #2, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z9.d, z3.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; SME2-NEXT: mov p8.b, p0.b
-; SME2-NEXT: st1h { z4.h - z7.h }, pn8, [x0]
+; SME2-NEXT: st1h { z1.h, z5.h, z9.h, z13.h }, pn8, [x0]
+; SME2-NEXT: ldr z13, [sp, #1, mul vl] // 16-byte Folded Reload
+; SME2-NEXT: ldr z9, [sp, #2, mul vl] // 16-byte Folded Reload
; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
-; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: addvl sp, sp, #3
; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; SME2-NEXT: ret
call void @llvm.aarch64.sve.st1.pn.x4.nxv8i16(<vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3, target("aarch64.svcount") %pn, ptr %ptr);
@@ -354,16 +368,19 @@ define void @st1_x4_i32(<vscale x 16 x i8> %unused, <vscale x 4 x i32> %zn0, <vs
; SME2-LABEL: st1_x4_i32:
; SME2: // %bb.0:
; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; SME2-NEXT: addvl sp, sp, #-1
-; SME2-NEXT: mov z7.d, z4.d
-; SME2-NEXT: mov z6.d, z3.d
-; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: addvl sp, sp, #-3
+; SME2-NEXT: str z13, [sp, #1, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z13.d, z4.d
; SME2-NEXT: mov z5.d, z2.d
-; SME2-NEXT: mov z4.d, z1.d
+; SME2-NEXT: str z9, [sp, #2, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z9.d, z3.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; SME2-NEXT: mov p8.b, p0.b
-; SME2-NEXT: st1w { z4.s - z7.s }, pn8, [x0]
+; SME2-NEXT: st1w { z1.s, z5.s, z9.s, z13.s }, pn8, [x0]
+; SME2-NEXT: ldr z13, [sp, #1, mul vl] // 16-byte Folded Reload
+; SME2-NEXT: ldr z9, [sp, #2, mul vl] // 16-byte Folded Reload
; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
-; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: addvl sp, sp, #3
; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; SME2-NEXT: ret
call void @llvm.aarch64.sve.st1.pn.x4.nxv4i32(<vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3, target("aarch64.svcount") %pn, ptr %ptr);
@@ -390,16 +407,19 @@ define void @st1_x4_i64(<vscale x 16 x i8> %unused, <vscale x 2 x i64> %zn0, <vs
; SME2-LABEL: st1_x4_i64:
; SME2: // %bb.0:
; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; SME2-NEXT: addvl sp, sp, #-1
-; SME2-NEXT: mov z7.d, z4.d
-; SME2-NEXT: mov z6.d, z3.d
-; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: addvl sp, sp, #-3
+; SME2-NEXT: str z13, [sp, #1, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z13.d, z4.d
; SME2-NEXT: mov z5.d, z2.d
-; SME2-NEXT: mov z4.d, z1.d
+; SME2-NEXT: str z9, [sp, #2, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z9.d, z3.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; SME2-NEXT: mov p8.b, p0.b
-; SME2-NEXT: st1d { z4.d - z7.d }, pn8, [x0]
+; SME2-NEXT: st1d { z1.d, z5.d, z9.d, z13.d }, pn8, [x0]
+; SME2-NEXT: ldr z13, [sp, #1, mul vl] // 16-byte Folded Reload
+; SME2-NEXT: ldr z9, [sp, #2, mul vl] // 16-byte Folded Reload
; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
-; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: addvl sp, sp, #3
; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; SME2-NEXT: ret
call void @llvm.aarch64.sve.st1.pn.x4.nxv2i64(<vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1, <vscale x 2 x i64> %zn2, <vscale x 2 x i64> %zn3, target("aarch64.svcount") %pn, ptr %ptr);
@@ -426,16 +446,19 @@ define void @st1_x4_f16(<vscale x 16 x i8> %unused, <vscale x 8 x half> %zn0, <v
; SME2-LABEL: st1_x4_f16:
; SME2: // %bb.0:
; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; SME2-NEXT: addvl sp, sp, #-1
-; SME2-NEXT: mov z7.d, z4.d
-; SME2-NEXT: mov z6.d, z3.d
-; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: addvl sp, sp, #-3
+; SME2-NEXT: str z13, [sp, #1, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z13.d, z4.d
; SME2-NEXT: mov z5.d, z2.d
-; SME2-NEXT: mov z4.d, z1.d
+; SME2-NEXT: str z9, [sp, #2, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z9.d, z3.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; SME2-NEXT: mov p8.b, p0.b
-; SME2-NEXT: st1h { z4.h - z7.h }, pn8, [x0]
+; SME2-NEXT: st1h { z1.h, z5.h, z9.h, z13.h }, pn8, [x0]
+; SME2-NEXT: ldr z13, [sp, #1, mul vl] // 16-byte Folded Reload
+; SME2-NEXT: ldr z9, [sp, #2, mul vl] // 16-byte Folded Reload
; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
-; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: addvl sp, sp, #3
; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; SME2-NEXT: ret
call void @llvm.aarch64.sve.st1.pn.x4.nxv8f16(<vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3, target("aarch64.svcount") %pn, ptr %ptr);
@@ -462,16 +485,19 @@ define void @st1_x4_bf16(<vscale x 16 x i8> %unused, <vscale x 8 x bfloat> %zn0,
; SME2-LABEL: st1_x4_bf16:
; SME2: // %bb.0:
; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; SME2-NEXT: addvl sp, sp, #-1
-; SME2-NEXT: mov z7.d, z4.d
-; SME2-NEXT: mov z6.d, z3.d
-; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: addvl sp, sp, #-3
+; SME2-NEXT: str z13, [sp, #1, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z13.d, z4.d
; SME2-NEXT: mov z5.d, z2.d
-; SME2-NEXT: mov z4.d, z1.d
+; SME2-NEXT: str z9, [sp, #2, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z9.d, z3.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; SME2-NEXT: mov p8.b, p0.b
-; SME2-NEXT: st1h { z4.h - z7.h }, pn8, [x0]
+; SME2-NEXT: st1h { z1.h, z5.h, z9.h, z13.h }, pn8, [x0]
+; SME2-NEXT: ldr z13, [sp, #1, mul vl] // 16-byte Folded Reload
+; SME2-NEXT: ldr z9, [sp, #2, mul vl] // 16-byte Folded Reload
; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
-; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: addvl sp, sp, #3
; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; SME2-NEXT: ret
call void @llvm.aarch64.sve.st1.pn.x4.nxv8bf16(<vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3, target("aarch64.svcount") %pn, ptr %ptr);
@@ -498,16 +524,19 @@ define void @st1_x4_f32(<vscale x 16 x i8> %unused, <vscale x 4 x float> %zn0, <
; SME2-LABEL: st1_x4_f32:
; SME2: // %bb.0:
; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; SME2-NEXT: addvl sp, sp, #-1
-; SME2-NEXT: mov z7.d, z4.d
-; SME2-NEXT: mov z6.d, z3.d
-; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: addvl sp, sp, #-3
+; SME2-NEXT: str z13, [sp, #1, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z13.d, z4.d
; SME2-NEXT: mov z5.d, z2.d
-; SME2-NEXT: mov z4.d, z1.d
+; SME2-NEXT: str z9, [sp, #2, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z9.d, z3.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; SME2-NEXT: mov p8.b, p0.b
-; SME2-NEXT: st1w { z4.s - z7.s }, pn8, [x0]
+; SME2-NEXT: st1w { z1.s, z5.s, z9.s, z13.s }, pn8, [x0]
+; SME2-NEXT: ldr z13, [sp, #1, mul vl] // 16-byte Folded Reload
+; SME2-NEXT: ldr z9, [sp, #2, mul vl] // 16-byte Folded Reload
; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
-; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: addvl sp, sp, #3
; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; SME2-NEXT: ret
call void @llvm.aarch64.sve.st1.pn.x4.nxv4f32(<vscale x 4 x float> %zn0, <vscale x 4 x float> %zn1, <vscale x 4 x float> %zn2, <vscale x 4 x float> %zn3, target("aarch64.svcount") %pn, ptr %ptr);
@@ -534,16 +563,19 @@ define void @st1_x4_f64(<vscale x 16 x i8> %unused, <vscale x 2 x double> %zn0,
; SME2-LABEL: st1_x4_f64:
; SME2: // %bb.0:
; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; SME2-NEXT: addvl sp, sp, #-1
-; SME2-NEXT: mov z7.d, z4.d
-; SME2-NEXT: mov z6.d, z3.d
-; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: addvl sp, sp, #-3
+; SME2-NEXT: str z13, [sp, #1, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z13.d, z4.d
; SME2-NEXT: mov z5.d, z2.d
-; SME2-NEXT: mov z4.d, z1.d
+; SME2-NEXT: str z9, [sp, #2, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z9.d, z3.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; SME2-NEXT: mov p8.b, p0.b
-; SME2-NEXT: st1d { z4.d - z7.d }, pn8, [x0]
+; SME2-NEXT: st1d { z1.d, z5.d, z9.d, z13.d }, pn8, [x0]
+; SME2-NEXT: ldr z13, [sp, #1, mul vl] // 16-byte Folded Reload
+; SME2-NEXT: ldr z9, [sp, #2, mul vl] // 16-byte Folded Reload
; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
-; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: addvl sp, sp, #3
; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; SME2-NEXT: ret
call void @llvm.aarch64.sve.st1.pn.x4.nxv2f64(<vscale x 2 x double> %zn0, <vscale x 2 x double> %zn1, <vscale x 2 x double> %zn2, <vscale x 2 x double> %zn3, target("aarch64.svcount") %pn, ptr %ptr);
@@ -570,14 +602,15 @@ define void @stnt1_x2_i8(<vscale x 16 x i8> %unused, <vscale x 16 x i8> %zn0, <v
; SME2-LABEL: stnt1_x2_i8:
; SME2: // %bb.0:
; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; SME2-NEXT: addvl sp, sp, #-1
-; SME2-NEXT: mov z3.d, z2.d
-; SME2-NEXT: mov z2.d, z1.d
+; SME2-NEXT: addvl sp, sp, #-2
+; SME2-NEXT: str z9, [sp, #1, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z9.d, z2.d
; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; SME2-NEXT: mov p8.b, p0.b
-; SME2-NEXT: stnt1b { z2.b, z3.b }, pn8, [x0]
+; SME2-NEXT: stnt1b { z1.b, z9.b }, pn8, [x0]
+; SME2-NEXT: ldr z9, [sp, #1, mul vl] // 16-byte Folded Reload
; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
-; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: addvl sp, sp, #2
; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; SME2-NEXT: ret
call void @llvm.aarch64.sve.stnt1.pn.x2.nxv16i8(<vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, target("aarch64.svcount") %pn, ptr %ptr);
@@ -602,14 +635,15 @@ define void @stnt1_x2_i16(<vscale x 16 x i8> %unused, <vscale x 8 x i16> %zn0, <
; SME2-LABEL: stnt1_x2_i16:
; SME2: // %bb.0:
; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; SME2-NEXT: addvl sp, sp, #-1
-; SME2-NEXT: mov z3.d, z2.d
-; SME2-NEXT: mov z2.d, z1.d
+; SME2-NEXT: addvl sp, sp, #-2
+; SME2-NEXT: str z9, [sp, #1, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z9.d, z2.d
; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; SME2-NEXT: mov p8.b, p0.b
-; SME2-NEXT: stnt1h { z2.h, z3.h }, pn8, [x0]
+; SME2-NEXT: stnt1h { z1.h, z9.h }, pn8, [x0]
+; SME2-NEXT: ldr z9, [sp, #1, mul vl] // 16-byte Folded Reload
; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
-; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: addvl sp, sp, #2
; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; SME2-NEXT: ret
call void @llvm.aarch64.sve.stnt1.pn.x2.nxv8i16(<vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, target("aarch64.svcount") %pn, ptr %ptr);
@@ -634,14 +668,15 @@ define void @stnt1_x2_i32(<vscale x 16 x i8> %unused, <vscale x 4 x i32> %zn0, <
; SME2-LABEL: stnt1_x2_i32:
; SME2: // %bb.0:
; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; SME2-NEXT: addvl sp, sp, #-1
-; SME2-NEXT: mov z3.d, z2.d
-; SME2-NEXT: mov z2.d, z1.d
+; SME2-NEXT: addvl sp, sp, #-2
+; SME2-NEXT: str z9, [sp, #1, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z9.d, z2.d
; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; SME2-NEXT: mov p8.b, p0.b
-; SME2-NEXT: stnt1w { z2.s, z3.s }, pn8, [x0]
+; SME2-NEXT: stnt1w { z1.s, z9.s }, pn8, [x0]
+; SME2-NEXT: ldr z9, [sp, #1, mul vl] // 16-byte Folded Reload
; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
-; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: addvl sp, sp, #2
; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; SME2-NEXT: ret
call void @llvm.aarch64.sve.stnt1.pn.x2.nxv4i32(<vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1, target("aarch64.svcount") %pn, ptr %ptr);
@@ -666,14 +701,15 @@ define void @stnt1_x2_i64(<vscale x 16 x i8> %unused, <vscale x 2 x i64> %zn0, <
; SME2-LABEL: stnt1_x2_i64:
; SME2: // %bb.0:
; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; SME2-NEXT: addvl sp, sp, #-1
-; SME2-NEXT: mov z3.d, z2.d
-; SME2-NEXT: mov z2.d, z1.d
+; SME2-NEXT: addvl sp, sp, #-2
+; SME2-NEXT: str z9, [sp, #1, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z9.d, z2.d
; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; SME2-NEXT: mov p8.b, p0.b
-; SME2-NEXT: stnt1d { z2.d, z3.d }, pn8, [x0]
+; SME2-NEXT: stnt1d { z1.d, z9.d }, pn8, [x0]
+; SME2-NEXT: ldr z9, [sp, #1, mul vl] // 16-byte Folded Reload
; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
-; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: addvl sp, sp, #2
; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; SME2-NEXT: ret
call void @llvm.aarch64.sve.stnt1.pn.x2.nxv2i64(<vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1, target("aarch64.svcount") %pn, ptr %ptr);
@@ -698,14 +734,15 @@ define void @stnt1_x2_f16(<vscale x 16 x i8> %unused, <vscale x 8 x half> %zn0,
; SME2-LABEL: stnt1_x2_f16:
; SME2: // %bb.0:
; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; SME2-NEXT: addvl sp, sp, #-1
-; SME2-NEXT: mov z3.d, z2.d
-; SME2-NEXT: mov z2.d, z1.d
+; SME2-NEXT: addvl sp, sp, #-2
+; SME2-NEXT: str z9, [sp, #1, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z9.d, z2.d
; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; SME2-NEXT: mov p8.b, p0.b
-; SME2-NEXT: stnt1h { z2.h, z3.h }, pn8, [x0]
+; SME2-NEXT: stnt1h { z1.h, z9.h }, pn8, [x0]
+; SME2-NEXT: ldr z9, [sp, #1, mul vl] // 16-byte Folded Reload
; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
-; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: addvl sp, sp, #2
; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; SME2-NEXT: ret
call void @llvm.aarch64.sve.stnt1.pn.x2.nxv8f16(<vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1, target("aarch64.svcount") %pn, ptr %ptr);
@@ -730,14 +767,15 @@ define void @stnt1_x2_bf16(<vscale x 16 x i8> %unused, <vscale x 8 x bfloat> %zn
; SME2-LABEL: stnt1_x2_bf16:
; SME2: // %bb.0:
; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; SME2-NEXT: addvl sp, sp, #-1
-; SME2-NEXT: mov z3.d, z2.d
-; SME2-NEXT: mov z2.d, z1.d
+; SME2-NEXT: addvl sp, sp, #-2
+; SME2-NEXT: str z9, [sp, #1, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z9.d, z2.d
; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; SME2-NEXT: mov p8.b, p0.b
-; SME2-NEXT: stnt1h { z2.h, z3.h }, pn8, [x0]
+; SME2-NEXT: stnt1h { z1.h, z9.h }, pn8, [x0]
+; SME2-NEXT: ldr z9, [sp, #1, mul vl] // 16-byte Folded Reload
; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
-; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: addvl sp, sp, #2
; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; SME2-NEXT: ret
call void @llvm.aarch64.sve.stnt1.pn.x2.nxv8bf16(<vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1, target("aarch64.svcount") %pn, ptr %ptr);
@@ -762,14 +800,15 @@ define void @stnt1_x2_f32(<vscale x 16 x i8> %unused, <vscale x 4 x float> %zn0,
; SME2-LABEL: stnt1_x2_f32:
; SME2: // %bb.0:
; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; SME2-NEXT: addvl sp, sp, #-1
-; SME2-NEXT: mov z3.d, z2.d
-; SME2-NEXT: mov z2.d, z1.d
+; SME2-NEXT: addvl sp, sp, #-2
+; SME2-NEXT: str z9, [sp, #1, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z9.d, z2.d
; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; SME2-NEXT: mov p8.b, p0.b
-; SME2-NEXT: stnt1w { z2.s, z3.s }, pn8, [x0]
+; SME2-NEXT: stnt1w { z1.s, z9.s }, pn8, [x0]
+; SME2-NEXT: ldr z9, [sp, #1, mul vl] // 16-byte Folded Reload
; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
-; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: addvl sp, sp, #2
; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; SME2-NEXT: ret
call void @llvm.aarch64.sve.stnt1.pn.x2.nxv4f32(<vscale x 4 x float> %zn0, <vscale x 4 x float> %zn1, target("aarch64.svcount") %pn, ptr %ptr);
@@ -794,14 +833,15 @@ define void @stnt1_x2_f64(<vscale x 16 x i8> %unused, <vscale x 2 x double> %zn0
; SME2-LABEL: stnt1_x2_f64:
; SME2: // %bb.0:
; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; SME2-NEXT: addvl sp, sp, #-1
-; SME2-NEXT: mov z3.d, z2.d
-; SME2-NEXT: mov z2.d, z1.d
+; SME2-NEXT: addvl sp, sp, #-2
+; SME2-NEXT: str z9, [sp, #1, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z9.d, z2.d
; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; SME2-NEXT: mov p8.b, p0.b
-; SME2-NEXT: stnt1d { z2.d, z3.d }, pn8, [x0]
+; SME2-NEXT: stnt1d { z1.d, z9.d }, pn8, [x0]
+; SME2-NEXT: ldr z9, [sp, #1, mul vl] // 16-byte Folded Reload
; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
-; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: addvl sp, sp, #2
; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; SME2-NEXT: ret
call void @llvm.aarch64.sve.stnt1.pn.x2.nxv2f64(<vscale x 2 x double> %zn0, <vscale x 2 x double> %zn1, target("aarch64.svcount") %pn, ptr %ptr);
@@ -828,16 +868,19 @@ define void @stnt1_x4_i8(<vscale x 16 x i8> %unused, <vscale x 16 x i8> %zn0, <v
; SME2-LABEL: stnt1_x4_i8:
; SME2: // %bb.0:
; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; SME2-NEXT: addvl sp, sp, #-1
-; SME2-NEXT: mov z7.d, z4.d
-; SME2-NEXT: mov z6.d, z3.d
-; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: addvl sp, sp, #-3
+; SME2-NEXT: str z13, [sp, #1, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z13.d, z4.d
; SME2-NEXT: mov z5.d, z2.d
-; SME2-NEXT: mov z4.d, z1.d
+; SME2-NEXT: str z9, [sp, #2, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z9.d, z3.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; SME2-NEXT: mov p8.b, p0.b
-; SME2-NEXT: stnt1b { z4.b - z7.b }, pn8, [x0]
+; SME2-NEXT: stnt1b { z1.b, z5.b, z9.b, z13.b }, pn8, [x0]
+; SME2-NEXT: ldr z13, [sp, #1, mul vl] // 16-byte Folded Reload
+; SME2-NEXT: ldr z9, [sp, #2, mul vl] // 16-byte Folded Reload
; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
-; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: addvl sp, sp, #3
; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; SME2-NEXT: ret
call void @llvm.aarch64.sve.stnt1.pn.x4.nxv16i8(<vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3, target("aarch64.svcount") %pn, ptr %ptr);
@@ -864,16 +907,19 @@ define void @stnt1_x4_i16(<vscale x 16 x i8> %unused, <vscale x 8 x i16> %zn0, <
; SME2-LABEL: stnt1_x4_i16:
; SME2: // %bb.0:
; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; SME2-NEXT: addvl sp, sp, #-1
-; SME2-NEXT: mov z7.d, z4.d
-; SME2-NEXT: mov z6.d, z3.d
-; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: addvl sp, sp, #-3
+; SME2-NEXT: str z13, [sp, #1, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z13.d, z4.d
; SME2-NEXT: mov z5.d, z2.d
-; SME2-NEXT: mov z4.d, z1.d
+; SME2-NEXT: str z9, [sp, #2, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z9.d, z3.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; SME2-NEXT: mov p8.b, p0.b
-; SME2-NEXT: stnt1h { z4.h - z7.h }, pn8, [x0]
+; SME2-NEXT: stnt1h { z1.h, z5.h, z9.h, z13.h }, pn8, [x0]
+; SME2-NEXT: ldr z13, [sp, #1, mul vl] // 16-byte Folded Reload
+; SME2-NEXT: ldr z9, [sp, #2, mul vl] // 16-byte Folded Reload
; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
-; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: addvl sp, sp, #3
; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; SME2-NEXT: ret
call void @llvm.aarch64.sve.stnt1.pn.x4.nxv8i16(<vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3, target("aarch64.svcount") %pn, ptr %ptr);
@@ -900,16 +946,19 @@ define void @stnt1_x4_i32(<vscale x 16 x i8> %unused, <vscale x 4 x i32> %zn0, <
; SME2-LABEL: stnt1_x4_i32:
; SME2: // %bb.0:
; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; SME2-NEXT: addvl sp, sp, #-1
-; SME2-NEXT: mov z7.d, z4.d
-; SME2-NEXT: mov z6.d, z3.d
-; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: addvl sp, sp, #-3
+; SME2-NEXT: str z13, [sp, #1, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z13.d, z4.d
; SME2-NEXT: mov z5.d, z2.d
-; SME2-NEXT: mov z4.d, z1.d
+; SME2-NEXT: str z9, [sp, #2, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z9.d, z3.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; SME2-NEXT: mov p8.b, p0.b
-; SME2-NEXT: stnt1w { z4.s - z7.s }, pn8, [x0]
+; SME2-NEXT: stnt1w { z1.s, z5.s, z9.s, z13.s }, pn8, [x0]
+; SME2-NEXT: ldr z13, [sp, #1, mul vl] // 16-byte Folded Reload
+; SME2-NEXT: ldr z9, [sp, #2, mul vl] // 16-byte Folded Reload
; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
-; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: addvl sp, sp, #3
; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; SME2-NEXT: ret
call void @llvm.aarch64.sve.stnt1.pn.x4.nxv4i32(<vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3, target("aarch64.svcount") %pn, ptr %ptr);
@@ -936,16 +985,19 @@ define void @stnt1_x4_i64(<vscale x 16 x i8> %unused, <vscale x 2 x i64> %zn0, <
; SME2-LABEL: stnt1_x4_i64:
; SME2: // %bb.0:
; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; SME2-NEXT: addvl sp, sp, #-1
-; SME2-NEXT: mov z7.d, z4.d
-; SME2-NEXT: mov z6.d, z3.d
-; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: addvl sp, sp, #-3
+; SME2-NEXT: str z13, [sp, #1, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z13.d, z4.d
; SME2-NEXT: mov z5.d, z2.d
-; SME2-NEXT: mov z4.d, z1.d
+; SME2-NEXT: str z9, [sp, #2, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z9.d, z3.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; SME2-NEXT: mov p8.b, p0.b
-; SME2-NEXT: stnt1d { z4.d - z7.d }, pn8, [x0]
+; SME2-NEXT: stnt1d { z1.d, z5.d, z9.d, z13.d }, pn8, [x0]
+; SME2-NEXT: ldr z13, [sp, #1, mul vl] // 16-byte Folded Reload
+; SME2-NEXT: ldr z9, [sp, #2, mul vl] // 16-byte Folded Reload
; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
-; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: addvl sp, sp, #3
; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; SME2-NEXT: ret
call void @llvm.aarch64.sve.stnt1.pn.x4.nxv2i64(<vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1, <vscale x 2 x i64> %zn2, <vscale x 2 x i64> %zn3, target("aarch64.svcount") %pn, ptr %ptr);
@@ -972,16 +1024,19 @@ define void @stnt1_x4_f16(<vscale x 16 x i8> %unused, <vscale x 8 x half> %zn0,
; SME2-LABEL: stnt1_x4_f16:
; SME2: // %bb.0:
; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; SME2-NEXT: addvl sp, sp, #-1
-; SME2-NEXT: mov z7.d, z4.d
-; SME2-NEXT: mov z6.d, z3.d
-; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: addvl sp, sp, #-3
+; SME2-NEXT: str z13, [sp, #1, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z13.d, z4.d
; SME2-NEXT: mov z5.d, z2.d
-; SME2-NEXT: mov z4.d, z1.d
+; SME2-NEXT: str z9, [sp, #2, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z9.d, z3.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; SME2-NEXT: mov p8.b, p0.b
-; SME2-NEXT: stnt1h { z4.h - z7.h }, pn8, [x0]
+; SME2-NEXT: stnt1h { z1.h, z5.h, z9.h, z13.h }, pn8, [x0]
+; SME2-NEXT: ldr z13, [sp, #1, mul vl] // 16-byte Folded Reload
+; SME2-NEXT: ldr z9, [sp, #2, mul vl] // 16-byte Folded Reload
; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
-; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: addvl sp, sp, #3
; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; SME2-NEXT: ret
call void @llvm.aarch64.sve.stnt1.pn.x4.nxv8f16(<vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3, target("aarch64.svcount") %pn, ptr %ptr);
@@ -1008,16 +1063,19 @@ define void @stnt1_x4_bf16(<vscale x 16 x i8> %unused, <vscale x 8 x bfloat> %zn
; SME2-LABEL: stnt1_x4_bf16:
; SME2: // %bb.0:
; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; SME2-NEXT: addvl sp, sp, #-1
-; SME2-NEXT: mov z7.d, z4.d
-; SME2-NEXT: mov z6.d, z3.d
-; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: addvl sp, sp, #-3
+; SME2-NEXT: str z13, [sp, #1, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z13.d, z4.d
; SME2-NEXT: mov z5.d, z2.d
-; SME2-NEXT: mov z4.d, z1.d
+; SME2-NEXT: str z9, [sp, #2, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z9.d, z3.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; SME2-NEXT: mov p8.b, p0.b
-; SME2-NEXT: stnt1h { z4.h - z7.h }, pn8, [x0]
+; SME2-NEXT: stnt1h { z1.h, z5.h, z9.h, z13.h }, pn8, [x0]
+; SME2-NEXT: ldr z13, [sp, #1, mul vl] // 16-byte Folded Reload
+; SME2-NEXT: ldr z9, [sp, #2, mul vl] // 16-byte Folded Reload
; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
-; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: addvl sp, sp, #3
; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; SME2-NEXT: ret
call void @llvm.aarch64.sve.stnt1.pn.x4.nxv8bf16(<vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3, target("aarch64.svcount") %pn, ptr %ptr);
@@ -1044,16 +1102,19 @@ define void @stnt1_x4_f32(<vscale x 16 x i8> %unused, <vscale x 4 x float> %zn0,
; SME2-LABEL: stnt1_x4_f32:
; SME2: // %bb.0:
; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; SME2-NEXT: addvl sp, sp, #-1
-; SME2-NEXT: mov z7.d, z4.d
-; SME2-NEXT: mov z6.d, z3.d
-; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: addvl sp, sp, #-3
+; SME2-NEXT: str z13, [sp, #1, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z13.d, z4.d
; SME2-NEXT: mov z5.d, z2.d
-; SME2-NEXT: mov z4.d, z1.d
+; SME2-NEXT: str z9, [sp, #2, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z9.d, z3.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; SME2-NEXT: mov p8.b, p0.b
-; SME2-NEXT: stnt1w { z4.s - z7.s }, pn8, [x0]
+; SME2-NEXT: stnt1w { z1.s, z5.s, z9.s, z13.s }, pn8, [x0]
+; SME2-NEXT: ldr z13, [sp, #1, mul vl] // 16-byte Folded Reload
+; SME2-NEXT: ldr z9, [sp, #2, mul vl] // 16-byte Folded Reload
; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
-; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: addvl sp, sp, #3
; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; SME2-NEXT: ret
call void @llvm.aarch64.sve.stnt1.pn.x4.nxv4f32(<vscale x 4 x float> %zn0, <vscale x 4 x float> %zn1, <vscale x 4 x float> %zn2, <vscale x 4 x float> %zn3, target("aarch64.svcount") %pn, ptr %ptr);
@@ -1080,16 +1141,19 @@ define void @stnt1_x4_f64(<vscale x 16 x i8> %unused, <vscale x 2 x double> %zn0
; SME2-LABEL: stnt1_x4_f64:
; SME2: // %bb.0:
; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; SME2-NEXT: addvl sp, sp, #-1
-; SME2-NEXT: mov z7.d, z4.d
-; SME2-NEXT: mov z6.d, z3.d
-; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: addvl sp, sp, #-3
+; SME2-NEXT: str z13, [sp, #1, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z13.d, z4.d
; SME2-NEXT: mov z5.d, z2.d
-; SME2-NEXT: mov z4.d, z1.d
+; SME2-NEXT: str z9, [sp, #2, mul vl] // 16-byte Folded Spill
+; SME2-NEXT: mov z9.d, z3.d
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; SME2-NEXT: mov p8.b, p0.b
-; SME2-NEXT: stnt1d { z4.d - z7.d }, pn8, [x0]
+; SME2-NEXT: stnt1d { z1.d, z5.d, z9.d, z13.d }, pn8, [x0]
+; SME2-NEXT: ldr z13, [sp, #1, mul vl] // 16-byte Folded Reload
+; SME2-NEXT: ldr z9, [sp, #2, mul vl] // 16-byte Folded Reload
; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
-; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: addvl sp, sp, #3
; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; SME2-NEXT: ret
call void @llvm.aarch64.sve.stnt1.pn.x4.nxv2f64(<vscale x 2 x double> %zn0, <vscale x 2 x double> %zn1, <vscale x 2 x double> %zn2, <vscale x 2 x double> %zn3, target("aarch64.svcount") %pn, ptr %ptr);
>From 18da3d0c522d87def7affe2f68db0d7ae80c3443 Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Wed, 29 Jul 2026 09:01:53 +0000
Subject: [PATCH 3/4] Cleanup SVEInsrInfo.td and mir tests
---
.../lib/Target/AArch64/AArch64SVEInstrInfo.td | 8 ++---
.../regalloc-hint-movprfx-streaming.mir | 36 -------------------
.../CodeGen/AArch64/sve-ldst-multi-vec.mir | 1 -
3 files changed, 4 insertions(+), 41 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 6c00bfade217c..37b3908811380 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -4697,9 +4697,11 @@ defm WHILELO_CXX : sve2p1_int_while_rr_pn<"whilelo", 0b110, AArch64whilelo_pn_fl
defm WHILELS_CXX : sve2p1_int_while_rr_pn<"whilels", 0b111, AArch64whilels_pn_flag>;
} // End HasSVE2p1_or_StreamingSME2
+// Override the default contiguous-only patterns when SME2 is available,
+// allow register allocation to choose a strided or contiguous tuple.
multiclass store_pn_x2_sme2<ValueType Ty, SDPatternOperator Store,
Instruction RegImmPseudo> {
- let AddedComplexity = 2 in {
+ let AddedComplexity = 2 in
// scalar + immediate (mul vl)
def : Pat<(Store Ty:$vec0, Ty:$vec1, aarch64svcount:$PNg,
(am_sve_indexed_s4 GPR64sp:$base, simm4s1:$offset)),
@@ -4707,7 +4709,6 @@ multiclass store_pn_x2_sme2<ValueType Ty, SDPatternOperator Store,
(REG_SEQUENCE ZPR2StridedOrContiguous,
Ty:$vec0, zsub0, Ty:$vec1, zsub1),
PNR:$PNg, GPR64:$base, simm4s1:$offset)>;
- }
let AddedComplexity = 1 in
def : Pat<(Store Ty:$vec0, Ty:$vec1, aarch64svcount:$PNg, GPR64:$base),
@@ -4719,7 +4720,7 @@ multiclass store_pn_x2_sme2<ValueType Ty, SDPatternOperator Store,
multiclass store_pn_x4_sme2<ValueType Ty, SDPatternOperator Store,
Instruction RegImmPseudo> {
- let AddedComplexity = 2 in {
+ let AddedComplexity = 2 in
// scalar + immediate (mul vl)
def : Pat<(Store Ty:$vec0, Ty:$vec1, Ty:$vec2, Ty:$vec3,
aarch64svcount:$PNg,
@@ -4729,7 +4730,6 @@ multiclass store_pn_x4_sme2<ValueType Ty, SDPatternOperator Store,
Ty:$vec0, zsub0, Ty:$vec1, zsub1,
Ty:$vec2, zsub2, Ty:$vec3, zsub3),
PNR:$PNg, GPR64:$base, simm4s1:$offset)>;
- }
let AddedComplexity = 1 in
def : Pat<(Store Ty:$vec0, Ty:$vec1, Ty:$vec2, Ty:$vec3,
diff --git a/llvm/test/CodeGen/AArch64/regalloc-hint-movprfx-streaming.mir b/llvm/test/CodeGen/AArch64/regalloc-hint-movprfx-streaming.mir
index f451fa51530d4..1716d58d08a43 100644
--- a/llvm/test/CodeGen/AArch64/regalloc-hint-movprfx-streaming.mir
+++ b/llvm/test/CodeGen/AArch64/regalloc-hint-movprfx-streaming.mir
@@ -86,39 +86,3 @@ body: |
STR_ZXI %8, %1, 0
RET_ReallyLR
...
-
-# Check the load, arithmetic and store can retain the same strided tuple.
----
-name: retain_strided_tuple_for_store
-tracksRegLiveness: true
-isSSA: false
-noVRegs: false
-body: |
- bb.0.entry:
- liveins: $x0
-
- ; CHECK-LABEL: name: retain_strided_tuple_for_store
- ; CHECK: liveins: $x0
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: renamable $pn8 = PTRUE_C_B implicit $vg
- ; CHECK-NEXT: renamable $z16_z24 = LD1B_2Z_IMM_PSEUDO renamable $pn8, renamable $x0, 0
- ; CHECK-NEXT: renamable $z16 = ADD_ZI_B_PSEUDO renamable $z16, 5, 0
- ; CHECK-NEXT: renamable $z24 = ADD_ZI_B_PSEUDO renamable $z24, 5, 0
- ; CHECK-NEXT: ST1B_2Z_IMM_PSEUDO killed renamable $z16_z24, killed renamable $pn8, killed renamable $x0, 0
- ; CHECK-NEXT: RET_ReallyLR
-
- ; ASM-LABEL: retain_strided_tuple_for_store:
- ; ASM-NOT: movprfx
- ; ASM: ld1b { z16.b, z24.b }, pn8/z, [x0]
- ; ASM-NEXT: add z16.b, z16.b, #5
- ; ASM-NEXT: add z24.b, z24.b, #5
- ; ASM-NEXT: st1b { z16.b, z24.b }, pn8, [x0]
-
- %0:gpr64common = COPY $x0
- %1:pnr_p8to15 = PTRUE_C_B implicit $vg
- %2:zpr2stridedorcontiguous = LD1B_2Z_IMM_PSEUDO %1, %0, 0
- %2.zsub0 = ADD_ZI_B_PSEUDO %2.zsub0, 5, 0
- %2.zsub1 = ADD_ZI_B_PSEUDO %2.zsub1, 5, 0
- ST1B_2Z_IMM_PSEUDO killed %2, killed %1, killed %0, 0
- RET_ReallyLR
-...
diff --git a/llvm/test/CodeGen/AArch64/sve-ldst-multi-vec.mir b/llvm/test/CodeGen/AArch64/sve-ldst-multi-vec.mir
index 92328da3f9228..fa4451142ea58 100644
--- a/llvm/test/CodeGen/AArch64/sve-ldst-multi-vec.mir
+++ b/llvm/test/CodeGen/AArch64/sve-ldst-multi-vec.mir
@@ -891,7 +891,6 @@ body: |
STNT1D_4Z_STRIDED_IMM renamable $z16_z20_z24_z28, renamable $pn8, %stack.0, -9
STNT1D_4Z_STRIDED_IMM renamable $z16_z20_z24_z28, renamable $pn8, %stack.0, 8
- ; Store pseudos with out-of-range frame offsets.
ST1B_2Z_IMM_PSEUDO renamable $z16_z17, renamable $pn8, %stack.0, -9
STNT1D_2Z_IMM_PSEUDO renamable $z16_z24, renamable $pn8, %stack.0, 8
ST1W_4Z_IMM_PSEUDO renamable $z16_z17_z18_z19, renamable $pn8, %stack.0, -9
>From 1902cdbb3753b23b91b1967d270a41c9395903fd Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Wed, 29 Jul 2026 09:12:25 +0000
Subject: [PATCH 4/4] fix identation in TableGen pattern
---
.../lib/Target/AArch64/AArch64SVEInstrInfo.td | 32 +++++++++----------
1 file changed, 16 insertions(+), 16 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 37b3908811380..32ac606c89c8c 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -4701,14 +4701,14 @@ defm WHILELS_CXX : sve2p1_int_while_rr_pn<"whilels", 0b111, AArch64whilels_pn_fl
// allow register allocation to choose a strided or contiguous tuple.
multiclass store_pn_x2_sme2<ValueType Ty, SDPatternOperator Store,
Instruction RegImmPseudo> {
+ // scalar + immediate (mul vl)
let AddedComplexity = 2 in
- // scalar + immediate (mul vl)
- def : Pat<(Store Ty:$vec0, Ty:$vec1, aarch64svcount:$PNg,
- (am_sve_indexed_s4 GPR64sp:$base, simm4s1:$offset)),
- (RegImmPseudo
- (REG_SEQUENCE ZPR2StridedOrContiguous,
- Ty:$vec0, zsub0, Ty:$vec1, zsub1),
- PNR:$PNg, GPR64:$base, simm4s1:$offset)>;
+ def : Pat<(Store Ty:$vec0, Ty:$vec1, aarch64svcount:$PNg,
+ (am_sve_indexed_s4 GPR64sp:$base, simm4s1:$offset)),
+ (RegImmPseudo
+ (REG_SEQUENCE ZPR2StridedOrContiguous,
+ Ty:$vec0, zsub0, Ty:$vec1, zsub1),
+ PNR:$PNg, GPR64:$base, simm4s1:$offset)>;
let AddedComplexity = 1 in
def : Pat<(Store Ty:$vec0, Ty:$vec1, aarch64svcount:$PNg, GPR64:$base),
@@ -4721,15 +4721,15 @@ multiclass store_pn_x2_sme2<ValueType Ty, SDPatternOperator Store,
multiclass store_pn_x4_sme2<ValueType Ty, SDPatternOperator Store,
Instruction RegImmPseudo> {
let AddedComplexity = 2 in
- // scalar + immediate (mul vl)
- def : Pat<(Store Ty:$vec0, Ty:$vec1, Ty:$vec2, Ty:$vec3,
- aarch64svcount:$PNg,
- (am_sve_indexed_s4 GPR64sp:$base, simm4s1:$offset)),
- (RegImmPseudo
- (REG_SEQUENCE ZPR4StridedOrContiguous,
- Ty:$vec0, zsub0, Ty:$vec1, zsub1,
- Ty:$vec2, zsub2, Ty:$vec3, zsub3),
- PNR:$PNg, GPR64:$base, simm4s1:$offset)>;
+ // scalar + immediate (mul vl)
+ def : Pat<(Store Ty:$vec0, Ty:$vec1, Ty:$vec2, Ty:$vec3,
+ aarch64svcount:$PNg,
+ (am_sve_indexed_s4 GPR64sp:$base, simm4s1:$offset)),
+ (RegImmPseudo
+ (REG_SEQUENCE ZPR4StridedOrContiguous,
+ Ty:$vec0, zsub0, Ty:$vec1, zsub1,
+ Ty:$vec2, zsub2, Ty:$vec3, zsub3),
+ PNR:$PNg, GPR64:$base, simm4s1:$offset)>;
let AddedComplexity = 1 in
def : Pat<(Store Ty:$vec0, Ty:$vec1, Ty:$vec2, Ty:$vec3,
More information about the llvm-commits
mailing list