[llvm] [AArch64] Enable quad variant st1b/ld1b for callee-saved spills (PR #225100)
Kieran B via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 24 02:13:47 PDT 2026
https://github.com/kieroxide updated https://github.com/llvm/llvm-project/pull/225100
>From 00b55d85891e077c338bf02ac180fee8bde0a581 Mon Sep 17 00:00:00 2001
From: Kieran Bailey <kieran.bailey at arm.com>
Date: Mon, 21 Sep 2026 14:04:11 +0000
Subject: [PATCH 1/4] [AArch64] Enable quad variant st1b/ld1b for callee-saved
spills
Currently, only the pair variant is generated by callee-saved spills. Generate quad variants when:
- 4 consecutive registers (example: z8, z9, z10, z11)
- First register will be divisible by 4. (example: z8)
- Offset will be divisible by 4. (example: #4, mul vl)
- Offset in range -32 to 28
---
.../Target/AArch64/AArch64FrameLowering.cpp | 284 +++++++++++-----
.../AArch64/AArch64PrologueEpilogue.cpp | 2 +
.../AArch64/sme2-fp8-intrinsics-cvt.ll | 6 +-
.../AArch64/sme2-intrinsics-int-dots.ll | 48 +--
.../CodeGen/AArch64/sme2-intrinsics-ld1.ll | 320 ++++++------------
.../CodeGen/AArch64/sme2-intrinsics-ldnt1.ll | 320 ++++++------------
.../CodeGen/AArch64/sme2-intrinsics-qcvt.ll | 6 +-
.../CodeGen/AArch64/sme2-intrinsics-qrshr.ll | 6 +-
.../CodeGen/AArch64/sme2-intrinsics-vdot.ll | 24 +-
.../AArch64/sve-callee-save-restore-pairs.ll | 60 ++--
10 files changed, 482 insertions(+), 594 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
index d816c4285a87d..fc75f686f7866 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
@@ -430,7 +430,7 @@ bool AArch64FrameLowering::homogeneousPrologEpilog(
return false;
// If there are an odd number of GPRs before LR and FP in the CSRs list,
- // they will not be paired into one RegPairInfo, which is incompatible with
+ // they will not be paired into one RegGroupInfo, which is incompatible with
// the assumption made by the homogeneous prolog epilog pass.
const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs();
unsigned NumGPRs = 0;
@@ -1636,17 +1636,33 @@ static bool invalidateRegisterPairing(bool SpillExtendedVolatile,
namespace {
-struct RegPairInfo {
+struct RegGroupInfo {
Register Reg1;
Register Reg2;
+ Register Reg3;
+ Register Reg4;
int FrameIdx;
int Offset;
enum RegType { GPR, FPR64, FPR128, PPR, ZPR, VG } Type;
const TargetRegisterClass *RC;
- RegPairInfo() = default;
+ RegGroupInfo() = default;
- bool isPaired() const { return Reg2.isValid(); }
+ bool isPaired() const {
+ return Reg2.isValid() && !Reg3.isValid() && !Reg4.isValid();
+ }
+
+ bool isQuad() const {
+ return Reg2.isValid() && Reg3.isValid() && Reg4.isValid();
+ }
+
+ unsigned getNumRegs() const {
+ if (isQuad())
+ return 4;
+ if (isPaired())
+ return 2;
+ return 1;
+ }
bool isScalable() const { return Type == PPR || Type == ZPR; }
};
@@ -1685,7 +1701,7 @@ void computeCalleeSaveRegisterPairs(const AArch64FrameLowering &AFL,
MachineFunction &MF,
ArrayRef<CalleeSavedInfo> CSI,
const TargetRegisterInfo *TRI,
- SmallVectorImpl<RegPairInfo> &RegPairs,
+ SmallVectorImpl<RegGroupInfo> &RegPairs,
bool NeedsFrameRecord) {
if (CSI.empty())
@@ -1759,32 +1775,32 @@ void computeCalleeSaveRegisterPairs(const AArch64FrameLowering &AFL,
// When iterating backwards, the loop condition relies on unsigned wraparound.
for (unsigned i = FirstReg; i < Count; i += RegInc) {
- RegPairInfo RPI;
+ RegGroupInfo RPI;
RPI.Reg1 = CSI[i].getReg();
if (AArch64::GPR64RegClass.contains(RPI.Reg1)) {
- RPI.Type = RegPairInfo::GPR;
+ RPI.Type = RegGroupInfo::GPR;
RPI.RC = &AArch64::GPR64RegClass;
} else if (AArch64::FPR64RegClass.contains(RPI.Reg1)) {
- RPI.Type = RegPairInfo::FPR64;
+ RPI.Type = RegGroupInfo::FPR64;
RPI.RC = &AArch64::FPR64RegClass;
} else if (AArch64::FPR128RegClass.contains(RPI.Reg1)) {
- RPI.Type = RegPairInfo::FPR128;
+ RPI.Type = RegGroupInfo::FPR128;
RPI.RC = &AArch64::FPR128RegClass;
} else if (AArch64::ZPRRegClass.contains(RPI.Reg1)) {
- RPI.Type = RegPairInfo::ZPR;
+ RPI.Type = RegGroupInfo::ZPR;
RPI.RC = &AArch64::ZPRRegClass;
} else if (AArch64::PPRRegClass.contains(RPI.Reg1)) {
- RPI.Type = RegPairInfo::PPR;
+ RPI.Type = RegGroupInfo::PPR;
RPI.RC = &AArch64::PPRRegClass;
} else if (RPI.Reg1 == AArch64::VG) {
- RPI.Type = RegPairInfo::VG;
+ RPI.Type = RegGroupInfo::VG;
RPI.RC = &AArch64::FIXED_REGSRegClass;
} else {
llvm_unreachable("Unsupported register class.");
}
- int &ScalableByteOffset = RPI.Type == RegPairInfo::PPR && SplitPPRs
+ int &ScalableByteOffset = RPI.Type == RegGroupInfo::PPR && SplitPPRs
? PPRByteOffset
: ZPRByteOffset;
@@ -1806,37 +1822,57 @@ void computeCalleeSaveRegisterPairs(const AArch64FrameLowering &AFL,
bool PairFitsImmRange =
PairOffset / Scale >= -64 && PairOffset / Scale <= 63;
switch (RPI.Type) {
- case RegPairInfo::GPR:
+ case RegGroupInfo::GPR:
if (AArch64::GPR64RegClass.contains(NextReg) && PairFitsImmRange &&
!invalidateRegisterPairing(SpillExtendedVolatile, SpillCount,
RPI.Reg1, NextReg, IsWindows,
NeedsWinCFI, NeedsFrameRecord, TRI))
RPI.Reg2 = NextReg;
break;
- case RegPairInfo::FPR64:
+ case RegGroupInfo::FPR64:
if (AArch64::FPR64RegClass.contains(NextReg) && PairFitsImmRange &&
!invalidateRegisterPairing(SpillExtendedVolatile, SpillCount,
RPI.Reg1, NextReg, IsWindows,
NeedsWinCFI, NeedsFrameRecord, TRI))
RPI.Reg2 = NextReg;
break;
- case RegPairInfo::FPR128:
+ case RegGroupInfo::FPR128:
if (AArch64::FPR128RegClass.contains(NextReg) && PairFitsImmRange)
RPI.Reg2 = NextReg;
break;
- case RegPairInfo::PPR:
+ case RegGroupInfo::PPR:
break;
- case RegPairInfo::ZPR:
- if (!NeedsWinCFI && AFI->getPredicateRegForFillSpill() != 0 &&
- ((RPI.Reg1 - AArch64::Z0) & 1) == 0 && (NextReg == RPI.Reg1 + 1)) {
- // Calculate offset of register pair to see if pair instruction can be
- // used.
- int Offset = (ScalableByteOffset + StackFillDir * 2 * Scale) / Scale;
- if ((-16 <= Offset && Offset <= 14) && (Offset % 2 == 0))
- RPI.Reg2 = NextReg;
+ case RegGroupInfo::ZPR:
+ if (!NeedsWinCFI && AFI->getPredicateRegForFillSpill() != 0) {
+ if (unsigned(i + 2 * RegInc) < Count &&
+ unsigned(i + 3 * RegInc) < Count &&
+ (RPI.Reg1 - AArch64::Z0) % 4 == 0) {
+ auto Reg3 = CSI[i + RegInc * 2].getReg();
+ auto Reg4 = CSI[i + RegInc * 3].getReg();
+ bool Consecutive = (RPI.Reg1 + 1 == NextReg) &&
+ (NextReg + 1 == Reg3) && (Reg3 + 1 == Reg4);
+ int Offset =
+ (ScalableByteOffset + StackFillDir * 4 * Scale) / Scale;
+
+ if (Consecutive && (Offset % 4 == 0) &&
+ (-32 <= Offset && Offset <= 28)) {
+ RPI.Reg2 = NextReg;
+ RPI.Reg3 = Reg3;
+ RPI.Reg4 = Reg4;
+ }
+ }
+ if (!RPI.isQuad() && ((RPI.Reg1 - AArch64::Z0) & 1) == 0 &&
+ (NextReg == RPI.Reg1 + 1)) {
+ // Calculate offset of register pair to see if pair instruction can
+ // be used.
+ int Offset =
+ (ScalableByteOffset + StackFillDir * 2 * Scale) / Scale;
+ if ((-16 <= Offset && Offset <= 14) && (Offset % 2 == 0))
+ RPI.Reg2 = NextReg;
+ }
}
break;
- case RegPairInfo::VG:
+ case RegGroupInfo::VG:
break;
}
}
@@ -1851,6 +1887,17 @@ void computeCalleeSaveRegisterPairs(const AArch64FrameLowering &AFL,
(CSI[i].getFrameIdx() + RegInc == CSI[i + RegInc].getFrameIdx())) &&
"Out of order callee saved regs!");
+ assert((!RPI.isQuad() ||
+ ((CSI[i].getFrameIdx() + RegInc == CSI[i + RegInc].getFrameIdx()) &&
+ (CSI[i + RegInc].getFrameIdx() + RegInc ==
+ CSI[i + RegInc * 2].getFrameIdx()) &&
+ (CSI[i + RegInc * 2].getFrameIdx() + RegInc ==
+ CSI[i + RegInc * 3].getFrameIdx()))) &&
+ "Out of order callee saved regs!");
+
+ assert((!RPI.isQuad() || RPI.Type == RegGroupInfo::ZPR) &&
+ "Currently only ZPR's support four-register spills");
+
assert((!RPI.isPaired() || !NeedsFrameRecord || RPI.Reg2 != AArch64::FP ||
RPI.Reg1 == AArch64::LR) &&
"FrameRecord must be allocated together with LR");
@@ -1871,9 +1918,9 @@ void computeCalleeSaveRegisterPairs(const AArch64FrameLowering &AFL,
"Callee-save registers not saved as adjacent register pair!");
RPI.FrameIdx = CSI[i].getFrameIdx();
- if (IsWindows &&
- RPI.isPaired()) // RPI.FrameIdx must be the lower index of the pair
- RPI.FrameIdx = CSI[i + RegInc].getFrameIdx();
+ if (IsWindows)
+ // RPI.FrameIdx must be the lower index of the group
+ RPI.FrameIdx = CSI[i + RegInc * (RPI.getNumRegs() - 1)].getFrameIdx();
// Realign the scalable offset if necessary. This is relevant when spilling
// predicates on Windows.
@@ -1889,9 +1936,9 @@ void computeCalleeSaveRegisterPairs(const AArch64FrameLowering &AFL,
assert(OffsetPre % Scale == 0);
if (RPI.isScalable())
- ScalableByteOffset += StackFillDir * (RPI.isPaired() ? 2 * Scale : Scale);
+ ScalableByteOffset += StackFillDir * RPI.getNumRegs() * Scale;
else
- ByteOffset += StackFillDir * (RPI.isPaired() ? 2 * Scale : Scale);
+ ByteOffset += StackFillDir * RPI.getNumRegs() * Scale;
// Swift's async context is directly before FP, so allocate an extra
// 8 bytes for it.
@@ -1903,7 +1950,7 @@ void computeCalleeSaveRegisterPairs(const AArch64FrameLowering &AFL,
// Round up size of non-pair to pair size if we need to pad the
// callee-save area to ensure 16-byte alignment.
if (NeedGapToAlignStack && !IsWindows && !RPI.isScalable() &&
- RPI.Type != RegPairInfo::FPR128 && !RPI.isPaired() &&
+ RPI.Type != RegGroupInfo::FPR128 && !RPI.isPaired() &&
ByteOffset % 16 != 0) {
ByteOffset += 8 * StackFillDir;
assert(MFI.getObjectAlign(RPI.FrameIdx) <= Align(16));
@@ -1954,8 +2001,7 @@ void computeCalleeSaveRegisterPairs(const AArch64FrameLowering &AFL,
AFI->setCalleeSaveBaseToFrameRecordOffset(Offset);
RegPairs.push_back(RPI);
- if (RPI.isPaired())
- i += RegInc;
+ i += RegInc * (RPI.getNumRegs() - 1);
}
if (IsWindows) {
// If we need an alignment gap in the stack, align the topmost stack
@@ -1980,7 +2026,7 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
const AArch64InstrInfo &TII = *Subtarget.getInstrInfo();
bool NeedsWinCFI = needsWinCFI(MF);
DebugLoc DL;
- SmallVector<RegPairInfo, 8> RegPairs;
+ SmallVector<RegGroupInfo, 8> RegPairs;
computeCalleeSaveRegisterPairs(*this, MF, CSI, TRI, RegPairs, hasFP(MF));
@@ -2006,9 +2052,11 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
return true;
}
bool PTrueCreated = false;
- for (const RegPairInfo &RPI : llvm::reverse(RegPairs)) {
+ for (const RegGroupInfo &RPI : llvm::reverse(RegPairs)) {
Register Reg1 = RPI.Reg1;
Register Reg2 = RPI.Reg2;
+ Register Reg3 = RPI.Reg3;
+ Register Reg4 = RPI.Reg4;
unsigned StrOpc;
// Issue sequence of spills for cs regs. The first spill may be converted
@@ -2024,22 +2072,24 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
unsigned Size = TRI->getSpillSize(*RPI.RC);
Align Alignment = TRI->getSpillAlign(*RPI.RC);
switch (RPI.Type) {
- case RegPairInfo::GPR:
+ case RegGroupInfo::GPR:
StrOpc = RPI.isPaired() ? AArch64::STPXi : AArch64::STRXui;
break;
- case RegPairInfo::FPR64:
+ case RegGroupInfo::FPR64:
StrOpc = RPI.isPaired() ? AArch64::STPDi : AArch64::STRDui;
break;
- case RegPairInfo::FPR128:
+ case RegGroupInfo::FPR128:
StrOpc = RPI.isPaired() ? AArch64::STPQi : AArch64::STRQui;
break;
- case RegPairInfo::ZPR:
- StrOpc = RPI.isPaired() ? AArch64::ST1B_2Z_IMM : AArch64::STR_ZXI;
+ case RegGroupInfo::ZPR:
+ StrOpc = RPI.isQuad()
+ ? AArch64::ST1B_4Z_IMM
+ : (RPI.isPaired() ? AArch64::ST1B_2Z_IMM : AArch64::STR_ZXI);
break;
- case RegPairInfo::PPR:
+ case RegGroupInfo::PPR:
StrOpc = AArch64::STR_PXI;
break;
- case RegPairInfo::VG:
+ case RegGroupInfo::VG:
StrOpc = AArch64::STRXui;
break;
}
@@ -2088,11 +2138,19 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
LLVM_DEBUG({
dbgs() << "CSR spill: (" << printReg(Reg1, TRI);
- if (RPI.isPaired())
+ if (RPI.isPaired() || RPI.isQuad())
dbgs() << ", " << printReg(Reg2, TRI);
+ if (RPI.isQuad()) {
+ dbgs() << ", " << printReg(Reg3, TRI);
+ dbgs() << ", " << printReg(Reg4, TRI);
+ }
dbgs() << ") -> fi#(" << RPI.FrameIdx;
- if (RPI.isPaired())
+ if (RPI.isPaired() || RPI.isQuad())
dbgs() << ", " << RPI.FrameIdx + 1;
+ if (RPI.isQuad()) {
+ dbgs() << ", " << RPI.FrameIdx + 2;
+ dbgs() << ", " << RPI.FrameIdx + 3;
+ }
dbgs() << ")\n";
});
@@ -2104,12 +2162,20 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
// and not (x+1,x).
unsigned FrameIdxReg1 = RPI.FrameIdx;
unsigned FrameIdxReg2 = RPI.FrameIdx + 1;
+ unsigned FrameIdxReg3 = RPI.FrameIdx + 2;
+ unsigned FrameIdxReg4 = RPI.FrameIdx + 3;
+
if (isTargetWindows(MF) && RPI.isPaired()) {
std::swap(Reg1, Reg2);
std::swap(FrameIdxReg1, FrameIdxReg2);
+ } else if (isTargetWindows(MF) && RPI.isQuad()) {
+ std::swap(Reg1, Reg4);
+ std::swap(Reg2, Reg3);
+ std::swap(FrameIdxReg1, FrameIdxReg4);
+ std::swap(FrameIdxReg2, FrameIdxReg3);
}
- if (RPI.isPaired() && RPI.isScalable()) {
+ if ((RPI.isQuad() || RPI.isPaired()) && RPI.isScalable()) {
[[maybe_unused]] const AArch64Subtarget &Subtarget =
MF.getSubtarget<AArch64Subtarget>();
AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
@@ -2117,12 +2183,12 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
assert((PnReg != 0 && enableMultiVectorSpillFill(Subtarget, MF)) &&
"Expects SVE2.1 or SME2 target and a predicate register");
#ifdef EXPENSIVE_CHECKS
- auto IsPPR = [](const RegPairInfo &c) {
- return c.Reg1 == RegPairInfo::PPR;
+ auto IsPPR = [](const RegGroupInfo &c) {
+ return c.Type == RegGroupInfo::PPR;
};
auto PPRBegin = std::find_if(RegPairs.begin(), RegPairs.end(), IsPPR);
- auto IsZPR = [](const RegPairInfo &c) {
- return c.Type == RegPairInfo::ZPR;
+ auto IsZPR = [](const RegGroupInfo &c) {
+ return c.Type == RegGroupInfo::ZPR;
};
auto ZPRBegin = std::find_if(RegPairs.begin(), RegPairs.end(), IsZPR);
assert(!(PPRBegin < ZPRBegin) &&
@@ -2138,14 +2204,31 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
MBB.addLiveIn(Reg1);
if (!MRI.isReserved(Reg2))
MBB.addLiveIn(Reg2);
- MIB.addReg(/*PairRegs*/ AArch64::Z0_Z1 + (RPI.Reg1 - AArch64::Z0));
+ if (RPI.isQuad()) {
+ if (!MRI.isReserved(Reg3))
+ MBB.addLiveIn(Reg3);
+ if (!MRI.isReserved(Reg4))
+ MBB.addLiveIn(Reg4);
+ }
+ if (RPI.isPaired()) {
+ MIB.addReg(/*PairRegs*/ AArch64::Z0_Z1 + (RPI.Reg1 - AArch64::Z0));
+ } else if (RPI.isQuad()) {
+ MIB.addReg(/*QuadRegs*/ AArch64::Z0_Z1_Z2_Z3 +
+ (RPI.Reg1 - AArch64::Z0));
+ MIB.addMemOperand(MF.getMachineMemOperand(
+ MachinePointerInfo::getFixedStack(MF, FrameIdxReg4),
+ MachineMemOperand::MOStore, Size, Alignment));
+ MIB.addMemOperand(MF.getMachineMemOperand(
+ MachinePointerInfo::getFixedStack(MF, FrameIdxReg3),
+ MachineMemOperand::MOStore, Size, Alignment));
+ }
MIB.addMemOperand(MF.getMachineMemOperand(
MachinePointerInfo::getFixedStack(MF, FrameIdxReg2),
MachineMemOperand::MOStore, Size, Alignment));
MIB.addReg(PnReg);
MIB.addReg(AArch64::SP)
- .addImm(RPI.Offset / 2) // [sp, #imm*2*vscale],
- // where 2*vscale is implicit
+ .addImm(RPI.Offset / // [sp, #imm*size*vscale]
+ RPI.getNumRegs()) // where size*vscale is implicit
.setMIFlag(MachineInstr::FrameSetup);
MIB.addMemOperand(MF.getMachineMemOperand(
MachinePointerInfo::getFixedStack(MF, FrameIdxReg1),
@@ -2177,14 +2260,22 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
}
// Update the StackIDs of the SVE stack slots.
MachineFrameInfo &MFI = MF.getFrameInfo();
- if (RPI.Type == RegPairInfo::ZPR) {
+ if (RPI.Type == RegGroupInfo::ZPR) {
MFI.setStackID(FrameIdxReg1, TargetStackID::ScalableVector);
- if (RPI.isPaired())
+ if (RPI.isPaired() || RPI.isQuad())
MFI.setStackID(FrameIdxReg2, TargetStackID::ScalableVector);
- } else if (RPI.Type == RegPairInfo::PPR) {
+ if (RPI.isQuad()) {
+ MFI.setStackID(FrameIdxReg3, TargetStackID::ScalableVector);
+ MFI.setStackID(FrameIdxReg4, TargetStackID::ScalableVector);
+ }
+ } else if (RPI.Type == RegGroupInfo::PPR) {
MFI.setStackID(FrameIdxReg1, TargetStackID::ScalablePredicateVector);
- if (RPI.isPaired())
+ if (RPI.isPaired() || RPI.isQuad())
MFI.setStackID(FrameIdxReg2, TargetStackID::ScalablePredicateVector);
+ if (RPI.isQuad()) {
+ MFI.setStackID(FrameIdxReg3, TargetStackID::ScalablePredicateVector);
+ MFI.setStackID(FrameIdxReg4, TargetStackID::ScalablePredicateVector);
+ }
}
}
return true;
@@ -2197,7 +2288,7 @@ bool AArch64FrameLowering::restoreCalleeSavedRegisters(
const AArch64InstrInfo &TII =
*MF.getSubtarget<AArch64Subtarget>().getInstrInfo();
DebugLoc DL;
- SmallVector<RegPairInfo, 8> RegPairs;
+ SmallVector<RegGroupInfo, 8> RegPairs;
bool NeedsWinCFI = needsWinCFI(MF);
if (MBBI != MBB.end())
@@ -2215,19 +2306,25 @@ bool AArch64FrameLowering::restoreCalleeSavedRegisters(
}
// For performance reasons restore SVE register in increasing order
- auto IsPPR = [](const RegPairInfo &c) { return c.Type == RegPairInfo::PPR; };
+ auto IsPPR = [](const RegGroupInfo &c) {
+ return c.Type == RegGroupInfo::PPR;
+ };
auto PPRBegin = llvm::find_if(RegPairs, IsPPR);
auto PPREnd = std::find_if_not(PPRBegin, RegPairs.end(), IsPPR);
std::reverse(PPRBegin, PPREnd);
- auto IsZPR = [](const RegPairInfo &c) { return c.Type == RegPairInfo::ZPR; };
+ auto IsZPR = [](const RegGroupInfo &c) {
+ return c.Type == RegGroupInfo::ZPR;
+ };
auto ZPRBegin = llvm::find_if(RegPairs, IsZPR);
auto ZPREnd = std::find_if_not(ZPRBegin, RegPairs.end(), IsZPR);
std::reverse(ZPRBegin, ZPREnd);
bool PTrueCreated = false;
- for (const RegPairInfo &RPI : RegPairs) {
+ for (const RegGroupInfo &RPI : RegPairs) {
Register Reg1 = RPI.Reg1;
Register Reg2 = RPI.Reg2;
+ Register Reg3 = RPI.Reg3;
+ Register Reg4 = RPI.Reg4;
// Issue sequence of restores for cs regs. The last restore may be converted
// to a post-increment load later by emitEpilogue if the callee-save stack
@@ -2241,31 +2338,41 @@ bool AArch64FrameLowering::restoreCalleeSavedRegisters(
unsigned Size = TRI->getSpillSize(*RPI.RC);
Align Alignment = TRI->getSpillAlign(*RPI.RC);
switch (RPI.Type) {
- case RegPairInfo::GPR:
+ case RegGroupInfo::GPR:
LdrOpc = RPI.isPaired() ? AArch64::LDPXi : AArch64::LDRXui;
break;
- case RegPairInfo::FPR64:
+ case RegGroupInfo::FPR64:
LdrOpc = RPI.isPaired() ? AArch64::LDPDi : AArch64::LDRDui;
break;
- case RegPairInfo::FPR128:
+ case RegGroupInfo::FPR128:
LdrOpc = RPI.isPaired() ? AArch64::LDPQi : AArch64::LDRQui;
break;
- case RegPairInfo::ZPR:
- LdrOpc = RPI.isPaired() ? AArch64::LD1B_2Z_IMM : AArch64::LDR_ZXI;
+ case RegGroupInfo::ZPR:
+ LdrOpc = RPI.isQuad()
+ ? AArch64::LD1B_4Z_IMM
+ : (RPI.isPaired() ? AArch64::LD1B_2Z_IMM : AArch64::LDR_ZXI);
break;
- case RegPairInfo::PPR:
+ case RegGroupInfo::PPR:
LdrOpc = AArch64::LDR_PXI;
break;
- case RegPairInfo::VG:
+ case RegGroupInfo::VG:
continue;
}
LLVM_DEBUG({
dbgs() << "CSR restore: (" << printReg(Reg1, TRI);
- if (RPI.isPaired())
+ if (RPI.isPaired() || RPI.isQuad())
dbgs() << ", " << printReg(Reg2, TRI);
+ if (RPI.isQuad()) {
+ dbgs() << ", " << printReg(Reg3, TRI);
+ dbgs() << ", " << printReg(Reg4, TRI);
+ }
dbgs() << ") -> fi#(" << RPI.FrameIdx;
- if (RPI.isPaired())
+ if (RPI.isPaired() || RPI.isQuad())
dbgs() << ", " << RPI.FrameIdx + 1;
+ if (RPI.isQuad()) {
+ dbgs() << ", " << RPI.FrameIdx + 2;
+ dbgs() << ", " << RPI.FrameIdx + 3;
+ }
dbgs() << ")\n";
});
@@ -2274,13 +2381,23 @@ bool AArch64FrameLowering::restoreCalleeSavedRegisters(
// and not (x+1,x).
unsigned FrameIdxReg1 = RPI.FrameIdx;
unsigned FrameIdxReg2 = RPI.FrameIdx + 1;
- if (isTargetWindows(MF) && RPI.isPaired()) {
- std::swap(Reg1, Reg2);
- std::swap(FrameIdxReg1, FrameIdxReg2);
+ unsigned FrameIdxReg3 = RPI.FrameIdx + 2;
+ unsigned FrameIdxReg4 = RPI.FrameIdx + 3;
+
+ if (isTargetWindows(MF)) {
+ if (RPI.isPaired()) {
+ std::swap(Reg1, Reg2);
+ std::swap(FrameIdxReg1, FrameIdxReg2);
+ } else if (RPI.isQuad()) {
+ std::swap(Reg1, Reg4);
+ std::swap(Reg2, Reg3);
+ std::swap(FrameIdxReg1, FrameIdxReg4);
+ std::swap(FrameIdxReg2, FrameIdxReg3);
+ }
}
AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
- if (RPI.isPaired() && RPI.isScalable()) {
+ if ((RPI.isQuad() || RPI.isPaired()) && RPI.isScalable()) {
[[maybe_unused]] const AArch64Subtarget &Subtarget =
MF.getSubtarget<AArch64Subtarget>();
unsigned PnReg = AFI->getPredicateRegForFillSpill();
@@ -2296,15 +2413,26 @@ bool AArch64FrameLowering::restoreCalleeSavedRegisters(
.setMIFlags(MachineInstr::FrameDestroy);
}
MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII.get(LdrOpc));
- MIB.addReg(/*PairRegs*/ AArch64::Z0_Z1 + (RPI.Reg1 - AArch64::Z0),
- getDefRegState(true));
+ if (RPI.isPaired()) {
+ MIB.addReg(/*PairRegs*/ AArch64::Z0_Z1 + (RPI.Reg1 - AArch64::Z0),
+ getDefRegState(true));
+ } else if (RPI.isQuad()) {
+ MIB.addReg(/*QuadRegs*/ AArch64::Z0_Z1_Z2_Z3 + (RPI.Reg1 - AArch64::Z0),
+ getDefRegState(true));
+ MIB.addMemOperand(MF.getMachineMemOperand(
+ MachinePointerInfo::getFixedStack(MF, FrameIdxReg4),
+ MachineMemOperand::MOLoad, Size, Alignment));
+ MIB.addMemOperand(MF.getMachineMemOperand(
+ MachinePointerInfo::getFixedStack(MF, FrameIdxReg3),
+ MachineMemOperand::MOLoad, Size, Alignment));
+ }
MIB.addMemOperand(MF.getMachineMemOperand(
MachinePointerInfo::getFixedStack(MF, FrameIdxReg2),
MachineMemOperand::MOLoad, Size, Alignment));
MIB.addReg(PnReg);
MIB.addReg(AArch64::SP)
- .addImm(RPI.Offset / 2) // [sp, #imm*2*vscale]
- // where 2*vscale is implicit
+ .addImm(RPI.Offset / // [sp, #imm*size*vscale]
+ RPI.getNumRegs()) // where size*vscale is implicit
.setMIFlag(MachineInstr::FrameDestroy);
MIB.addMemOperand(MF.getMachineMemOperand(
MachinePointerInfo::getFixedStack(MF, FrameIdxReg1),
diff --git a/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp b/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp
index ceb7444be805b..f159216aff2ee 100644
--- a/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp
+++ b/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp
@@ -55,6 +55,8 @@ static bool isPartOfZPRCalleeSaves(MachineBasicBlock::iterator I) {
return false;
case AArch64::LD1B_2Z_IMM:
case AArch64::ST1B_2Z_IMM:
+ case AArch64::LD1B_4Z_IMM:
+ case AArch64::ST1B_4Z_IMM:
case AArch64::STR_ZXI:
case AArch64::LDR_ZXI:
case AArch64::PTRUE_C_B:
diff --git a/llvm/test/CodeGen/AArch64/sme2-fp8-intrinsics-cvt.ll b/llvm/test/CodeGen/AArch64/sme2-fp8-intrinsics-cvt.ll
index a689e7d09dbe5..ce35336221309 100644
--- a/llvm/test/CodeGen/AArch64/sme2-fp8-intrinsics-cvt.ll
+++ b/llvm/test/CodeGen/AArch64/sme2-fp8-intrinsics-cvt.ll
@@ -31,8 +31,7 @@ define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 1
; CHECK-NEXT: ptrue pn8.b
; CHECK-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; CHECK-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; CHECK-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; CHECK-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CHECK-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
; CHECK-NEXT: str z22, [sp, #8, mul vl] // 16-byte Folded Spill
; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0xc8, 0x00, 0x1e, 0x22 // sp + 16 + 72 * VG
; CHECK-NEXT: .cfi_offset w29, -16
@@ -52,8 +51,7 @@ define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 1
; CHECK-NEXT: fcvt z3.b, { z28.s - z31.s }
; CHECK-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; CHECK-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; CHECK-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; CHECK-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CHECK-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
; CHECK-NEXT: ldr z22, [sp, #8, mul vl] // 16-byte Folded Reload
; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; CHECK-NEXT: addvl sp, sp, #9
diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-int-dots.ll b/llvm/test/CodeGen/AArch64/sme2-intrinsics-int-dots.ll
index db9beb65f4700..46752cf52c20e 100644
--- a/llvm/test/CodeGen/AArch64/sme2-intrinsics-int-dots.ll
+++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-int-dots.ll
@@ -697,8 +697,7 @@ define void @udot_single_za32_u16_vg1x4_tuple(ptr %ptr, i64 %stride, <vscale x 8
; CHECK-NEXT: ptrue pn8.b
; CHECK-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; CHECK-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; CHECK-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; CHECK-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CHECK-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
; CHECK-NEXT: str z22, [sp, #8, mul vl] // 16-byte Folded Spill
; CHECK-NEXT: ptrue pn8.b
; CHECK-NEXT: add x10, x0, x1
@@ -715,8 +714,7 @@ define void @udot_single_za32_u16_vg1x4_tuple(ptr %ptr, i64 %stride, <vscale x 8
; CHECK-NEXT: udot za.s[w8, 0, vgx4], { z28.h - z31.h }, z0.h
; CHECK-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; CHECK-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; CHECK-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; CHECK-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CHECK-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
; CHECK-NEXT: ldr z22, [sp, #8, mul vl] // 16-byte Folded Reload
; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; CHECK-NEXT: addvl sp, sp, #9
@@ -878,8 +876,7 @@ define void @usdot_single_za32_u16_vg1x4_tuple(ptr %ptr, i64 %stride, <vscale x
; CHECK-NEXT: ptrue pn8.b
; CHECK-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; CHECK-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; CHECK-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; CHECK-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CHECK-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
; CHECK-NEXT: str z22, [sp, #8, mul vl] // 16-byte Folded Spill
; CHECK-NEXT: ptrue pn8.b
; CHECK-NEXT: add x10, x9, x1
@@ -895,8 +892,7 @@ define void @usdot_single_za32_u16_vg1x4_tuple(ptr %ptr, i64 %stride, <vscale x
; CHECK-NEXT: usdot za.s[w8, 0, vgx4], { z28.b - z31.b }, z0.b
; CHECK-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; CHECK-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; CHECK-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; CHECK-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CHECK-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
; CHECK-NEXT: ldr z22, [sp, #8, mul vl] // 16-byte Folded Reload
; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; CHECK-NEXT: addvl sp, sp, #9
@@ -1009,8 +1005,7 @@ define void @sdot_single_za32_u16_vg1x4_tuple(ptr %ptr, i64 %stride, <vscale x 8
; CHECK-NEXT: ptrue pn8.b
; CHECK-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; CHECK-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; CHECK-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; CHECK-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CHECK-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
; CHECK-NEXT: str z22, [sp, #8, mul vl] // 16-byte Folded Spill
; CHECK-NEXT: ptrue pn8.b
; CHECK-NEXT: add x10, x0, x1
@@ -1027,8 +1022,7 @@ define void @sdot_single_za32_u16_vg1x4_tuple(ptr %ptr, i64 %stride, <vscale x 8
; CHECK-NEXT: sdot za.s[w8, 0, vgx4], { z28.h - z31.h }, z0.h
; CHECK-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; CHECK-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; CHECK-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; CHECK-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CHECK-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
; CHECK-NEXT: ldr z22, [sp, #8, mul vl] // 16-byte Folded Reload
; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; CHECK-NEXT: addvl sp, sp, #9
@@ -1190,8 +1184,7 @@ define void @sudot_single_za32_u16_vg1x4_tuple(ptr %ptr, i64 %stride, <vscale x
; CHECK-NEXT: ptrue pn8.b
; CHECK-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; CHECK-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; CHECK-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; CHECK-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CHECK-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
; CHECK-NEXT: str z22, [sp, #8, mul vl] // 16-byte Folded Spill
; CHECK-NEXT: ptrue pn8.b
; CHECK-NEXT: add x10, x9, x1
@@ -1207,8 +1200,7 @@ define void @sudot_single_za32_u16_vg1x4_tuple(ptr %ptr, i64 %stride, <vscale x
; CHECK-NEXT: sudot za.s[w8, 0, vgx4], { z28.b - z31.b }, z0.b
; CHECK-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; CHECK-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; CHECK-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; CHECK-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CHECK-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
; CHECK-NEXT: ldr z22, [sp, #8, mul vl] // 16-byte Folded Reload
; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; CHECK-NEXT: addvl sp, sp, #9
@@ -1434,8 +1426,7 @@ define void @udot_form_4x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 16 x i8>
; CHECK-NEXT: ptrue pn8.b
; CHECK-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; CHECK-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; CHECK-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; CHECK-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CHECK-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
; CHECK-NEXT: str z22, [sp, #8, mul vl] // 16-byte Folded Spill
; CHECK-NEXT: ptrue pn8.b
; CHECK-NEXT: add x10, x9, x1
@@ -1451,8 +1442,7 @@ define void @udot_form_4x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 16 x i8>
; CHECK-NEXT: udot za.s[w8, 0, vgx4], { z28.b - z31.b }, z0.b[0]
; CHECK-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; CHECK-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; CHECK-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; CHECK-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CHECK-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
; CHECK-NEXT: ldr z22, [sp, #8, mul vl] // 16-byte Folded Reload
; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; CHECK-NEXT: str z0, [x0]
@@ -1734,8 +1724,7 @@ define void @usdot_form_4x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 16 x i8>
; CHECK-NEXT: ptrue pn8.b
; CHECK-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; CHECK-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; CHECK-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; CHECK-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CHECK-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
; CHECK-NEXT: str z22, [sp, #8, mul vl] // 16-byte Folded Spill
; CHECK-NEXT: ptrue pn8.b
; CHECK-NEXT: add x10, x9, x1
@@ -1751,8 +1740,7 @@ define void @usdot_form_4x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 16 x i8>
; CHECK-NEXT: usdot za.s[w8, 0, vgx4], { z28.b - z31.b }, z0.b[0]
; CHECK-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; CHECK-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; CHECK-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; CHECK-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CHECK-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
; CHECK-NEXT: ldr z22, [sp, #8, mul vl] // 16-byte Folded Reload
; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; CHECK-NEXT: str z0, [x0]
@@ -1984,8 +1972,7 @@ define void @sdot_form_4x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 16 x i8>
; CHECK-NEXT: ptrue pn8.b
; CHECK-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; CHECK-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; CHECK-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; CHECK-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CHECK-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
; CHECK-NEXT: str z22, [sp, #8, mul vl] // 16-byte Folded Spill
; CHECK-NEXT: ptrue pn8.b
; CHECK-NEXT: add x10, x9, x1
@@ -2001,8 +1988,7 @@ define void @sdot_form_4x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 16 x i8>
; CHECK-NEXT: sdot za.s[w8, 0, vgx4], { z28.b - z31.b }, z0.b[0]
; CHECK-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; CHECK-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; CHECK-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; CHECK-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CHECK-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
; CHECK-NEXT: ldr z22, [sp, #8, mul vl] // 16-byte Folded Reload
; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; CHECK-NEXT: str z0, [x0]
@@ -2234,8 +2220,7 @@ define void @sudot_form_4x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 16 x i8>
; CHECK-NEXT: ptrue pn8.b
; CHECK-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; CHECK-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; CHECK-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; CHECK-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CHECK-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
; CHECK-NEXT: str z22, [sp, #8, mul vl] // 16-byte Folded Spill
; CHECK-NEXT: ptrue pn8.b
; CHECK-NEXT: add x10, x9, x1
@@ -2251,8 +2236,7 @@ define void @sudot_form_4x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 16 x i8>
; CHECK-NEXT: sudot za.s[w8, 0, vgx4], { z28.b - z31.b }, z0.b[0]
; CHECK-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; CHECK-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; CHECK-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; CHECK-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CHECK-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
; CHECK-NEXT: ldr z22, [sp, #8, mul vl] // 16-byte Folded Reload
; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; CHECK-NEXT: str z0, [x0]
diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-ld1.ll b/llvm/test/CodeGen/AArch64/sme2-intrinsics-ld1.ll
index 5272e4e6788c7..748fc0f71b5b2 100644
--- a/llvm/test/CodeGen/AArch64/sme2-intrinsics-ld1.ll
+++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-ld1.ll
@@ -58,12 +58,9 @@ define <vscale x 32 x i8> @ld1_x2_i8_z0_z8(<vscale x 16 x i8> %unused, <vscale x
; STRIDED-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z14.b, z15.b }, pn8, [sp, #8, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z12.b, z13.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z10.b, z11.b }, pn8, [sp, #12, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z8.b, z9.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z12.b - z15.b }, pn8, [sp, #8, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z8.b - z11.b }, pn8, [sp, #12, mul vl] // 64-byte Folded Spill
; STRIDED-NEXT: mov p8.b, p0.b
; STRIDED-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; STRIDED-NEXT: str z22, [sp, #16, mul vl] // 16-byte Folded Spill
@@ -74,14 +71,11 @@ define <vscale x 32 x i8> @ld1_x2_i8_z0_z8(<vscale x 16 x i8> %unused, <vscale x
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z14.b, z15.b }, pn8/z, [sp, #8, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z12.b, z13.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: mov z1.d, z8.d
-; STRIDED-NEXT: ld1b { z10.b, z11.b }, pn8/z, [sp, #12, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z8.b, z9.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ld1b { z12.b - z15.b }, pn8/z, [sp, #8, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT: mov z1.d, z8.d
+; STRIDED-NEXT: ld1b { z8.b - z11.b }, pn8/z, [sp, #12, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; STRIDED-NEXT: addvl sp, sp, #17
; STRIDED-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -189,12 +183,9 @@ define <vscale x 32 x i8> @ld1_x2_i8_z0_z8_scalar(<vscale x 16 x i8> %unused, <v
; STRIDED-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z14.b, z15.b }, pn8, [sp, #8, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z12.b, z13.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z10.b, z11.b }, pn8, [sp, #12, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z8.b, z9.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z12.b - z15.b }, pn8, [sp, #8, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z8.b - z11.b }, pn8, [sp, #12, mul vl] // 64-byte Folded Spill
; STRIDED-NEXT: mov p8.b, p0.b
; STRIDED-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; STRIDED-NEXT: str z22, [sp, #16, mul vl] // 16-byte Folded Spill
@@ -205,14 +196,11 @@ define <vscale x 32 x i8> @ld1_x2_i8_z0_z8_scalar(<vscale x 16 x i8> %unused, <v
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z14.b, z15.b }, pn8/z, [sp, #8, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z12.b, z13.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: mov z1.d, z8.d
-; STRIDED-NEXT: ld1b { z10.b, z11.b }, pn8/z, [sp, #12, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z8.b, z9.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ld1b { z12.b - z15.b }, pn8/z, [sp, #8, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT: mov z1.d, z8.d
+; STRIDED-NEXT: ld1b { z8.b - z11.b }, pn8/z, [sp, #12, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; STRIDED-NEXT: addvl sp, sp, #17
; STRIDED-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -321,12 +309,9 @@ define <vscale x 16 x i16> @ld1_x2_i16_z0_z8(<vscale x 8 x i16> %unused, <vscale
; STRIDED-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z14.b, z15.b }, pn8, [sp, #8, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z12.b, z13.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z10.b, z11.b }, pn8, [sp, #12, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z8.b, z9.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z12.b - z15.b }, pn8, [sp, #8, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z8.b - z11.b }, pn8, [sp, #12, mul vl] // 64-byte Folded Spill
; STRIDED-NEXT: mov p8.b, p0.b
; STRIDED-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; STRIDED-NEXT: str z22, [sp, #16, mul vl] // 16-byte Folded Spill
@@ -337,14 +322,11 @@ define <vscale x 16 x i16> @ld1_x2_i16_z0_z8(<vscale x 8 x i16> %unused, <vscale
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z14.b, z15.b }, pn8/z, [sp, #8, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z12.b, z13.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: mov z1.d, z8.d
-; STRIDED-NEXT: ld1b { z10.b, z11.b }, pn8/z, [sp, #12, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z8.b, z9.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ld1b { z12.b - z15.b }, pn8/z, [sp, #8, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT: mov z1.d, z8.d
+; STRIDED-NEXT: ld1b { z8.b - z11.b }, pn8/z, [sp, #12, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; STRIDED-NEXT: addvl sp, sp, #17
; STRIDED-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -452,12 +434,9 @@ define <vscale x 16 x i16> @ld1_x2_i16_z0_z8_scalar(<vscale x 8 x i16> %unused,
; STRIDED-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z14.b, z15.b }, pn8, [sp, #8, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z12.b, z13.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z10.b, z11.b }, pn8, [sp, #12, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z8.b, z9.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z12.b - z15.b }, pn8, [sp, #8, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z8.b - z11.b }, pn8, [sp, #12, mul vl] // 64-byte Folded Spill
; STRIDED-NEXT: mov p8.b, p0.b
; STRIDED-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; STRIDED-NEXT: str z22, [sp, #16, mul vl] // 16-byte Folded Spill
@@ -468,14 +447,11 @@ define <vscale x 16 x i16> @ld1_x2_i16_z0_z8_scalar(<vscale x 8 x i16> %unused,
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z14.b, z15.b }, pn8/z, [sp, #8, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z12.b, z13.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: mov z1.d, z8.d
-; STRIDED-NEXT: ld1b { z10.b, z11.b }, pn8/z, [sp, #12, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z8.b, z9.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ld1b { z12.b - z15.b }, pn8/z, [sp, #8, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT: mov z1.d, z8.d
+; STRIDED-NEXT: ld1b { z8.b - z11.b }, pn8/z, [sp, #12, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; STRIDED-NEXT: addvl sp, sp, #17
; STRIDED-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -584,12 +560,9 @@ define <vscale x 8 x i32> @ld1_x2_i32_z0_z8(<vscale x 4 x i32> %unused, <vscale
; STRIDED-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z14.b, z15.b }, pn8, [sp, #8, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z12.b, z13.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z10.b, z11.b }, pn8, [sp, #12, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z8.b, z9.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z12.b - z15.b }, pn8, [sp, #8, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z8.b - z11.b }, pn8, [sp, #12, mul vl] // 64-byte Folded Spill
; STRIDED-NEXT: mov p8.b, p0.b
; STRIDED-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; STRIDED-NEXT: str z22, [sp, #16, mul vl] // 16-byte Folded Spill
@@ -600,14 +573,11 @@ define <vscale x 8 x i32> @ld1_x2_i32_z0_z8(<vscale x 4 x i32> %unused, <vscale
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z14.b, z15.b }, pn8/z, [sp, #8, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z12.b, z13.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: mov z1.d, z8.d
-; STRIDED-NEXT: ld1b { z10.b, z11.b }, pn8/z, [sp, #12, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z8.b, z9.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ld1b { z12.b - z15.b }, pn8/z, [sp, #8, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT: mov z1.d, z8.d
+; STRIDED-NEXT: ld1b { z8.b - z11.b }, pn8/z, [sp, #12, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; STRIDED-NEXT: addvl sp, sp, #17
; STRIDED-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -715,12 +685,9 @@ define <vscale x 8 x i32> @ld1_x2_i32_z0_z8_scalar(<vscale x 4 x i32> %unused, <
; STRIDED-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z14.b, z15.b }, pn8, [sp, #8, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z12.b, z13.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z10.b, z11.b }, pn8, [sp, #12, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z8.b, z9.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z12.b - z15.b }, pn8, [sp, #8, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z8.b - z11.b }, pn8, [sp, #12, mul vl] // 64-byte Folded Spill
; STRIDED-NEXT: mov p8.b, p0.b
; STRIDED-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; STRIDED-NEXT: str z22, [sp, #16, mul vl] // 16-byte Folded Spill
@@ -731,14 +698,11 @@ define <vscale x 8 x i32> @ld1_x2_i32_z0_z8_scalar(<vscale x 4 x i32> %unused, <
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z14.b, z15.b }, pn8/z, [sp, #8, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z12.b, z13.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: mov z1.d, z8.d
-; STRIDED-NEXT: ld1b { z10.b, z11.b }, pn8/z, [sp, #12, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z8.b, z9.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ld1b { z12.b - z15.b }, pn8/z, [sp, #8, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT: mov z1.d, z8.d
+; STRIDED-NEXT: ld1b { z8.b - z11.b }, pn8/z, [sp, #12, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; STRIDED-NEXT: addvl sp, sp, #17
; STRIDED-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -847,12 +811,9 @@ define <vscale x 4 x i64> @ld1_x2_i64_z0_z8(<vscale x 2 x i64> %unused, <vscale
; STRIDED-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z14.b, z15.b }, pn8, [sp, #8, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z12.b, z13.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z10.b, z11.b }, pn8, [sp, #12, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z8.b, z9.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z12.b - z15.b }, pn8, [sp, #8, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z8.b - z11.b }, pn8, [sp, #12, mul vl] // 64-byte Folded Spill
; STRIDED-NEXT: mov p8.b, p0.b
; STRIDED-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; STRIDED-NEXT: str z22, [sp, #16, mul vl] // 16-byte Folded Spill
@@ -863,14 +824,11 @@ define <vscale x 4 x i64> @ld1_x2_i64_z0_z8(<vscale x 2 x i64> %unused, <vscale
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z14.b, z15.b }, pn8/z, [sp, #8, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z12.b, z13.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: mov z1.d, z8.d
-; STRIDED-NEXT: ld1b { z10.b, z11.b }, pn8/z, [sp, #12, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z8.b, z9.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ld1b { z12.b - z15.b }, pn8/z, [sp, #8, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT: mov z1.d, z8.d
+; STRIDED-NEXT: ld1b { z8.b - z11.b }, pn8/z, [sp, #12, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; STRIDED-NEXT: addvl sp, sp, #17
; STRIDED-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -978,12 +936,9 @@ define <vscale x 4 x i64> @ld1_x2_i64_z0_z8_scalar(<vscale x 2 x i64> %unused, <
; STRIDED-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z14.b, z15.b }, pn8, [sp, #8, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z12.b, z13.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z10.b, z11.b }, pn8, [sp, #12, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z8.b, z9.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z12.b - z15.b }, pn8, [sp, #8, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z8.b - z11.b }, pn8, [sp, #12, mul vl] // 64-byte Folded Spill
; STRIDED-NEXT: mov p8.b, p0.b
; STRIDED-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; STRIDED-NEXT: str z22, [sp, #16, mul vl] // 16-byte Folded Spill
@@ -994,14 +949,11 @@ define <vscale x 4 x i64> @ld1_x2_i64_z0_z8_scalar(<vscale x 2 x i64> %unused, <
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z14.b, z15.b }, pn8/z, [sp, #8, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z12.b, z13.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: mov z1.d, z8.d
-; STRIDED-NEXT: ld1b { z10.b, z11.b }, pn8/z, [sp, #12, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z8.b, z9.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ld1b { z12.b - z15.b }, pn8/z, [sp, #8, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT: mov z1.d, z8.d
+; STRIDED-NEXT: ld1b { z8.b - z11.b }, pn8/z, [sp, #12, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; STRIDED-NEXT: addvl sp, sp, #17
; STRIDED-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -1112,12 +1064,9 @@ define <vscale x 64 x i8> @ld1_x4_i8_z0_z4_z8_z12(<vscale x 16 x i8> %unused, <v
; STRIDED-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z14.b, z15.b }, pn8, [sp, #8, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z12.b, z13.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z10.b, z11.b }, pn8, [sp, #12, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z8.b, z9.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z12.b - z15.b }, pn8, [sp, #8, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z8.b - z11.b }, pn8, [sp, #12, mul vl] // 64-byte Folded Spill
; STRIDED-NEXT: mov p8.b, p0.b
; STRIDED-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; STRIDED-NEXT: str z22, [sp, #16, mul vl] // 16-byte Folded Spill
@@ -1128,15 +1077,12 @@ define <vscale x 64 x i8> @ld1_x4_i8_z0_z4_z8_z12(<vscale x 16 x i8> %unused, <v
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z14.b, z15.b }, pn8/z, [sp, #8, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z10.b, z11.b }, pn8/z, [sp, #12, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: mov z2.d, z8.d
; STRIDED-NEXT: mov z3.d, z12.d
-; STRIDED-NEXT: ld1b { z12.b, z13.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z8.b, z9.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT: ld1b { z12.b - z15.b }, pn8/z, [sp, #8, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ld1b { z8.b - z11.b }, pn8/z, [sp, #12, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; STRIDED-NEXT: mov z1.d, z4.d
; STRIDED-NEXT: addvl sp, sp, #17
@@ -1255,12 +1201,9 @@ define <vscale x 64 x i8> @ld1_x4_i8_z0_z4_z8_z12_scalar(<vscale x 16 x i8> %unu
; STRIDED-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z14.b, z15.b }, pn8, [sp, #8, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z12.b, z13.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z10.b, z11.b }, pn8, [sp, #12, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z8.b, z9.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z12.b - z15.b }, pn8, [sp, #8, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z8.b - z11.b }, pn8, [sp, #12, mul vl] // 64-byte Folded Spill
; STRIDED-NEXT: mov p8.b, p0.b
; STRIDED-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; STRIDED-NEXT: str z22, [sp, #16, mul vl] // 16-byte Folded Spill
@@ -1271,15 +1214,12 @@ define <vscale x 64 x i8> @ld1_x4_i8_z0_z4_z8_z12_scalar(<vscale x 16 x i8> %unu
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z14.b, z15.b }, pn8/z, [sp, #8, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z10.b, z11.b }, pn8/z, [sp, #12, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: mov z2.d, z8.d
; STRIDED-NEXT: mov z3.d, z12.d
-; STRIDED-NEXT: ld1b { z12.b, z13.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z8.b, z9.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT: ld1b { z12.b - z15.b }, pn8/z, [sp, #8, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ld1b { z8.b - z11.b }, pn8/z, [sp, #12, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; STRIDED-NEXT: mov z1.d, z4.d
; STRIDED-NEXT: addvl sp, sp, #17
@@ -1399,12 +1339,9 @@ define <vscale x 32 x i16> @ld1_x4_i16_z0_z4_z8_z12(<vscale x 8 x i16> %unused,
; STRIDED-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z14.b, z15.b }, pn8, [sp, #8, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z12.b, z13.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z10.b, z11.b }, pn8, [sp, #12, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z8.b, z9.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z12.b - z15.b }, pn8, [sp, #8, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z8.b - z11.b }, pn8, [sp, #12, mul vl] // 64-byte Folded Spill
; STRIDED-NEXT: mov p8.b, p0.b
; STRIDED-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; STRIDED-NEXT: str z22, [sp, #16, mul vl] // 16-byte Folded Spill
@@ -1415,15 +1352,12 @@ define <vscale x 32 x i16> @ld1_x4_i16_z0_z4_z8_z12(<vscale x 8 x i16> %unused,
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z14.b, z15.b }, pn8/z, [sp, #8, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z10.b, z11.b }, pn8/z, [sp, #12, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: mov z2.d, z8.d
; STRIDED-NEXT: mov z3.d, z12.d
-; STRIDED-NEXT: ld1b { z12.b, z13.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z8.b, z9.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT: ld1b { z12.b - z15.b }, pn8/z, [sp, #8, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ld1b { z8.b - z11.b }, pn8/z, [sp, #12, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; STRIDED-NEXT: mov z1.d, z4.d
; STRIDED-NEXT: addvl sp, sp, #17
@@ -1543,12 +1477,9 @@ define <vscale x 32 x i16> @ld1_x4_i16_z0_z4_z8_z12_scalar(<vscale x 8 x i16> %u
; STRIDED-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z14.b, z15.b }, pn8, [sp, #8, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z12.b, z13.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z10.b, z11.b }, pn8, [sp, #12, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z8.b, z9.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z12.b - z15.b }, pn8, [sp, #8, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z8.b - z11.b }, pn8, [sp, #12, mul vl] // 64-byte Folded Spill
; STRIDED-NEXT: mov p8.b, p0.b
; STRIDED-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; STRIDED-NEXT: str z22, [sp, #16, mul vl] // 16-byte Folded Spill
@@ -1559,15 +1490,12 @@ define <vscale x 32 x i16> @ld1_x4_i16_z0_z4_z8_z12_scalar(<vscale x 8 x i16> %u
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z14.b, z15.b }, pn8/z, [sp, #8, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z10.b, z11.b }, pn8/z, [sp, #12, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: mov z2.d, z8.d
; STRIDED-NEXT: mov z3.d, z12.d
-; STRIDED-NEXT: ld1b { z12.b, z13.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z8.b, z9.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT: ld1b { z12.b - z15.b }, pn8/z, [sp, #8, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ld1b { z8.b - z11.b }, pn8/z, [sp, #12, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; STRIDED-NEXT: mov z1.d, z4.d
; STRIDED-NEXT: addvl sp, sp, #17
@@ -1687,12 +1615,9 @@ define <vscale x 16 x i32> @ld1_x4_i32_z0_z4_z8_z12(<vscale x 4 x i32> %unused,
; STRIDED-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z14.b, z15.b }, pn8, [sp, #8, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z12.b, z13.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z10.b, z11.b }, pn8, [sp, #12, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z8.b, z9.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z12.b - z15.b }, pn8, [sp, #8, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z8.b - z11.b }, pn8, [sp, #12, mul vl] // 64-byte Folded Spill
; STRIDED-NEXT: mov p8.b, p0.b
; STRIDED-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; STRIDED-NEXT: str z22, [sp, #16, mul vl] // 16-byte Folded Spill
@@ -1703,15 +1628,12 @@ define <vscale x 16 x i32> @ld1_x4_i32_z0_z4_z8_z12(<vscale x 4 x i32> %unused,
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z14.b, z15.b }, pn8/z, [sp, #8, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z10.b, z11.b }, pn8/z, [sp, #12, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: mov z2.d, z8.d
; STRIDED-NEXT: mov z3.d, z12.d
-; STRIDED-NEXT: ld1b { z12.b, z13.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z8.b, z9.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT: ld1b { z12.b - z15.b }, pn8/z, [sp, #8, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ld1b { z8.b - z11.b }, pn8/z, [sp, #12, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; STRIDED-NEXT: mov z1.d, z4.d
; STRIDED-NEXT: addvl sp, sp, #17
@@ -1831,12 +1753,9 @@ define <vscale x 16 x i32> @ld1_x4_i32_z0_z4_z8_z12_scalar(<vscale x 4 x i32> %u
; STRIDED-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z14.b, z15.b }, pn8, [sp, #8, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z12.b, z13.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z10.b, z11.b }, pn8, [sp, #12, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z8.b, z9.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z12.b - z15.b }, pn8, [sp, #8, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z8.b - z11.b }, pn8, [sp, #12, mul vl] // 64-byte Folded Spill
; STRIDED-NEXT: mov p8.b, p0.b
; STRIDED-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; STRIDED-NEXT: str z22, [sp, #16, mul vl] // 16-byte Folded Spill
@@ -1847,15 +1766,12 @@ define <vscale x 16 x i32> @ld1_x4_i32_z0_z4_z8_z12_scalar(<vscale x 4 x i32> %u
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z14.b, z15.b }, pn8/z, [sp, #8, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z10.b, z11.b }, pn8/z, [sp, #12, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: mov z2.d, z8.d
; STRIDED-NEXT: mov z3.d, z12.d
-; STRIDED-NEXT: ld1b { z12.b, z13.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z8.b, z9.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT: ld1b { z12.b - z15.b }, pn8/z, [sp, #8, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ld1b { z8.b - z11.b }, pn8/z, [sp, #12, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; STRIDED-NEXT: mov z1.d, z4.d
; STRIDED-NEXT: addvl sp, sp, #17
@@ -1975,12 +1891,9 @@ define <vscale x 8 x i64> @ld1_x4_i64_z0_z4_z8_z12(<vscale x 2 x i64> %unused, <
; STRIDED-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z14.b, z15.b }, pn8, [sp, #8, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z12.b, z13.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z10.b, z11.b }, pn8, [sp, #12, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z8.b, z9.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z12.b - z15.b }, pn8, [sp, #8, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z8.b - z11.b }, pn8, [sp, #12, mul vl] // 64-byte Folded Spill
; STRIDED-NEXT: mov p8.b, p0.b
; STRIDED-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; STRIDED-NEXT: str z22, [sp, #16, mul vl] // 16-byte Folded Spill
@@ -1991,15 +1904,12 @@ define <vscale x 8 x i64> @ld1_x4_i64_z0_z4_z8_z12(<vscale x 2 x i64> %unused, <
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z14.b, z15.b }, pn8/z, [sp, #8, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z10.b, z11.b }, pn8/z, [sp, #12, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: mov z2.d, z8.d
; STRIDED-NEXT: mov z3.d, z12.d
-; STRIDED-NEXT: ld1b { z12.b, z13.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z8.b, z9.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT: ld1b { z12.b - z15.b }, pn8/z, [sp, #8, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ld1b { z8.b - z11.b }, pn8/z, [sp, #12, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; STRIDED-NEXT: mov z1.d, z4.d
; STRIDED-NEXT: addvl sp, sp, #17
@@ -2119,12 +2029,9 @@ define <vscale x 8 x i64> @ld1_x4_i64_z0_z4_z8_z12_scalar(<vscale x 2 x i64> %un
; STRIDED-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z14.b, z15.b }, pn8, [sp, #8, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z12.b, z13.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z10.b, z11.b }, pn8, [sp, #12, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z8.b, z9.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z12.b - z15.b }, pn8, [sp, #8, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z8.b - z11.b }, pn8, [sp, #12, mul vl] // 64-byte Folded Spill
; STRIDED-NEXT: mov p8.b, p0.b
; STRIDED-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; STRIDED-NEXT: str z22, [sp, #16, mul vl] // 16-byte Folded Spill
@@ -2135,15 +2042,12 @@ define <vscale x 8 x i64> @ld1_x4_i64_z0_z4_z8_z12_scalar(<vscale x 2 x i64> %un
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z14.b, z15.b }, pn8/z, [sp, #8, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z10.b, z11.b }, pn8/z, [sp, #12, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: mov z2.d, z8.d
; STRIDED-NEXT: mov z3.d, z12.d
-; STRIDED-NEXT: ld1b { z12.b, z13.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z8.b, z9.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT: ld1b { z12.b - z15.b }, pn8/z, [sp, #8, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ld1b { z8.b - z11.b }, pn8/z, [sp, #12, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; STRIDED-NEXT: mov z1.d, z4.d
; STRIDED-NEXT: addvl sp, sp, #17
diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-ldnt1.ll b/llvm/test/CodeGen/AArch64/sme2-intrinsics-ldnt1.ll
index 1a4e27fa736d7..cf1a3902f55ca 100644
--- a/llvm/test/CodeGen/AArch64/sme2-intrinsics-ldnt1.ll
+++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-ldnt1.ll
@@ -11,12 +11,9 @@ define <vscale x 32 x i8> @ldnt1_x2_i8_z0_z8(<vscale x 16 x i8> %unused, <vscale
; STRIDED-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z14.b, z15.b }, pn8, [sp, #8, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z12.b, z13.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z10.b, z11.b }, pn8, [sp, #12, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z8.b, z9.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z12.b - z15.b }, pn8, [sp, #8, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z8.b - z11.b }, pn8, [sp, #12, mul vl] // 64-byte Folded Spill
; STRIDED-NEXT: mov p8.b, p0.b
; STRIDED-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; STRIDED-NEXT: str z22, [sp, #16, mul vl] // 16-byte Folded Spill
@@ -27,14 +24,11 @@ define <vscale x 32 x i8> @ldnt1_x2_i8_z0_z8(<vscale x 16 x i8> %unused, <vscale
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z14.b, z15.b }, pn8/z, [sp, #8, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z12.b, z13.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: mov z1.d, z8.d
-; STRIDED-NEXT: ld1b { z10.b, z11.b }, pn8/z, [sp, #12, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z8.b, z9.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ld1b { z12.b - z15.b }, pn8/z, [sp, #8, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT: mov z1.d, z8.d
+; STRIDED-NEXT: ld1b { z8.b - z11.b }, pn8/z, [sp, #12, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; STRIDED-NEXT: addvl sp, sp, #17
; STRIDED-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -95,12 +89,9 @@ define <vscale x 32 x i8> @ldnt1_x2_i8_z0_z8_scalar(<vscale x 16 x i8> %unused,
; STRIDED-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z14.b, z15.b }, pn8, [sp, #8, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z12.b, z13.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z10.b, z11.b }, pn8, [sp, #12, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z8.b, z9.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z12.b - z15.b }, pn8, [sp, #8, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z8.b - z11.b }, pn8, [sp, #12, mul vl] // 64-byte Folded Spill
; STRIDED-NEXT: mov p8.b, p0.b
; STRIDED-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; STRIDED-NEXT: str z22, [sp, #16, mul vl] // 16-byte Folded Spill
@@ -111,14 +102,11 @@ define <vscale x 32 x i8> @ldnt1_x2_i8_z0_z8_scalar(<vscale x 16 x i8> %unused,
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z14.b, z15.b }, pn8/z, [sp, #8, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z12.b, z13.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: mov z1.d, z8.d
-; STRIDED-NEXT: ld1b { z10.b, z11.b }, pn8/z, [sp, #12, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z8.b, z9.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ld1b { z12.b - z15.b }, pn8/z, [sp, #8, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT: mov z1.d, z8.d
+; STRIDED-NEXT: ld1b { z8.b - z11.b }, pn8/z, [sp, #12, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; STRIDED-NEXT: addvl sp, sp, #17
; STRIDED-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -180,12 +168,9 @@ define <vscale x 16 x i16> @ldnt1_x2_i16_z0_z8(<vscale x 8 x i16> %unused, <vsca
; STRIDED-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z14.b, z15.b }, pn8, [sp, #8, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z12.b, z13.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z10.b, z11.b }, pn8, [sp, #12, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z8.b, z9.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z12.b - z15.b }, pn8, [sp, #8, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z8.b - z11.b }, pn8, [sp, #12, mul vl] // 64-byte Folded Spill
; STRIDED-NEXT: mov p8.b, p0.b
; STRIDED-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; STRIDED-NEXT: str z22, [sp, #16, mul vl] // 16-byte Folded Spill
@@ -196,14 +181,11 @@ define <vscale x 16 x i16> @ldnt1_x2_i16_z0_z8(<vscale x 8 x i16> %unused, <vsca
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z14.b, z15.b }, pn8/z, [sp, #8, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z12.b, z13.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: mov z1.d, z8.d
-; STRIDED-NEXT: ld1b { z10.b, z11.b }, pn8/z, [sp, #12, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z8.b, z9.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ld1b { z12.b - z15.b }, pn8/z, [sp, #8, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT: mov z1.d, z8.d
+; STRIDED-NEXT: ld1b { z8.b - z11.b }, pn8/z, [sp, #12, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; STRIDED-NEXT: addvl sp, sp, #17
; STRIDED-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -264,12 +246,9 @@ define <vscale x 16 x i16> @ldnt1_x2_i16_z0_z8_scalar(<vscale x 8 x i16> %unused
; STRIDED-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z14.b, z15.b }, pn8, [sp, #8, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z12.b, z13.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z10.b, z11.b }, pn8, [sp, #12, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z8.b, z9.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z12.b - z15.b }, pn8, [sp, #8, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z8.b - z11.b }, pn8, [sp, #12, mul vl] // 64-byte Folded Spill
; STRIDED-NEXT: mov p8.b, p0.b
; STRIDED-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; STRIDED-NEXT: str z22, [sp, #16, mul vl] // 16-byte Folded Spill
@@ -280,14 +259,11 @@ define <vscale x 16 x i16> @ldnt1_x2_i16_z0_z8_scalar(<vscale x 8 x i16> %unused
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z14.b, z15.b }, pn8/z, [sp, #8, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z12.b, z13.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: mov z1.d, z8.d
-; STRIDED-NEXT: ld1b { z10.b, z11.b }, pn8/z, [sp, #12, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z8.b, z9.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ld1b { z12.b - z15.b }, pn8/z, [sp, #8, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT: mov z1.d, z8.d
+; STRIDED-NEXT: ld1b { z8.b - z11.b }, pn8/z, [sp, #12, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; STRIDED-NEXT: addvl sp, sp, #17
; STRIDED-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -349,12 +325,9 @@ define <vscale x 8 x i32> @ldnt1_x2_i32_z0_z8(<vscale x 4 x i32> %unused, <vscal
; STRIDED-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z14.b, z15.b }, pn8, [sp, #8, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z12.b, z13.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z10.b, z11.b }, pn8, [sp, #12, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z8.b, z9.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z12.b - z15.b }, pn8, [sp, #8, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z8.b - z11.b }, pn8, [sp, #12, mul vl] // 64-byte Folded Spill
; STRIDED-NEXT: mov p8.b, p0.b
; STRIDED-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; STRIDED-NEXT: str z22, [sp, #16, mul vl] // 16-byte Folded Spill
@@ -365,14 +338,11 @@ define <vscale x 8 x i32> @ldnt1_x2_i32_z0_z8(<vscale x 4 x i32> %unused, <vscal
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z14.b, z15.b }, pn8/z, [sp, #8, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z12.b, z13.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: mov z1.d, z8.d
-; STRIDED-NEXT: ld1b { z10.b, z11.b }, pn8/z, [sp, #12, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z8.b, z9.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ld1b { z12.b - z15.b }, pn8/z, [sp, #8, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT: mov z1.d, z8.d
+; STRIDED-NEXT: ld1b { z8.b - z11.b }, pn8/z, [sp, #12, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; STRIDED-NEXT: addvl sp, sp, #17
; STRIDED-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -433,12 +403,9 @@ define <vscale x 8 x i32> @ldnt1_x2_i32_z0_z8_scalar(<vscale x 4 x i32> %unused,
; STRIDED-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z14.b, z15.b }, pn8, [sp, #8, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z12.b, z13.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z10.b, z11.b }, pn8, [sp, #12, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z8.b, z9.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z12.b - z15.b }, pn8, [sp, #8, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z8.b - z11.b }, pn8, [sp, #12, mul vl] // 64-byte Folded Spill
; STRIDED-NEXT: mov p8.b, p0.b
; STRIDED-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; STRIDED-NEXT: str z22, [sp, #16, mul vl] // 16-byte Folded Spill
@@ -449,14 +416,11 @@ define <vscale x 8 x i32> @ldnt1_x2_i32_z0_z8_scalar(<vscale x 4 x i32> %unused,
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z14.b, z15.b }, pn8/z, [sp, #8, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z12.b, z13.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: mov z1.d, z8.d
-; STRIDED-NEXT: ld1b { z10.b, z11.b }, pn8/z, [sp, #12, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z8.b, z9.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ld1b { z12.b - z15.b }, pn8/z, [sp, #8, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT: mov z1.d, z8.d
+; STRIDED-NEXT: ld1b { z8.b - z11.b }, pn8/z, [sp, #12, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; STRIDED-NEXT: addvl sp, sp, #17
; STRIDED-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -518,12 +482,9 @@ define <vscale x 4 x i64> @ldnt1_x2_i64_z0_z8(<vscale x 2 x i64> %unused, <vscal
; STRIDED-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z14.b, z15.b }, pn8, [sp, #8, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z12.b, z13.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z10.b, z11.b }, pn8, [sp, #12, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z8.b, z9.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z12.b - z15.b }, pn8, [sp, #8, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z8.b - z11.b }, pn8, [sp, #12, mul vl] // 64-byte Folded Spill
; STRIDED-NEXT: mov p8.b, p0.b
; STRIDED-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; STRIDED-NEXT: str z22, [sp, #16, mul vl] // 16-byte Folded Spill
@@ -534,14 +495,11 @@ define <vscale x 4 x i64> @ldnt1_x2_i64_z0_z8(<vscale x 2 x i64> %unused, <vscal
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z14.b, z15.b }, pn8/z, [sp, #8, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z12.b, z13.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: mov z1.d, z8.d
-; STRIDED-NEXT: ld1b { z10.b, z11.b }, pn8/z, [sp, #12, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z8.b, z9.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ld1b { z12.b - z15.b }, pn8/z, [sp, #8, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT: mov z1.d, z8.d
+; STRIDED-NEXT: ld1b { z8.b - z11.b }, pn8/z, [sp, #12, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; STRIDED-NEXT: addvl sp, sp, #17
; STRIDED-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -602,12 +560,9 @@ define <vscale x 4 x i64> @ldnt1_x2_i64_z0_z8_scalar(<vscale x 2 x i64> %unused,
; STRIDED-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z14.b, z15.b }, pn8, [sp, #8, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z12.b, z13.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z10.b, z11.b }, pn8, [sp, #12, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z8.b, z9.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z12.b - z15.b }, pn8, [sp, #8, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z8.b - z11.b }, pn8, [sp, #12, mul vl] // 64-byte Folded Spill
; STRIDED-NEXT: mov p8.b, p0.b
; STRIDED-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; STRIDED-NEXT: str z22, [sp, #16, mul vl] // 16-byte Folded Spill
@@ -618,14 +573,11 @@ define <vscale x 4 x i64> @ldnt1_x2_i64_z0_z8_scalar(<vscale x 2 x i64> %unused,
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z14.b, z15.b }, pn8/z, [sp, #8, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z12.b, z13.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: mov z1.d, z8.d
-; STRIDED-NEXT: ld1b { z10.b, z11.b }, pn8/z, [sp, #12, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z8.b, z9.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ld1b { z12.b - z15.b }, pn8/z, [sp, #8, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT: mov z1.d, z8.d
+; STRIDED-NEXT: ld1b { z8.b - z11.b }, pn8/z, [sp, #12, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; STRIDED-NEXT: addvl sp, sp, #17
; STRIDED-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -687,12 +639,9 @@ define <vscale x 64 x i8> @ldnt1_x4_i8_z0_z4_z8_z12(<vscale x 16 x i8> %unused,
; STRIDED-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z14.b, z15.b }, pn8, [sp, #8, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z12.b, z13.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z10.b, z11.b }, pn8, [sp, #12, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z8.b, z9.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z12.b - z15.b }, pn8, [sp, #8, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z8.b - z11.b }, pn8, [sp, #12, mul vl] // 64-byte Folded Spill
; STRIDED-NEXT: mov p8.b, p0.b
; STRIDED-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; STRIDED-NEXT: str z22, [sp, #16, mul vl] // 16-byte Folded Spill
@@ -703,15 +652,12 @@ define <vscale x 64 x i8> @ldnt1_x4_i8_z0_z4_z8_z12(<vscale x 16 x i8> %unused,
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z14.b, z15.b }, pn8/z, [sp, #8, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z10.b, z11.b }, pn8/z, [sp, #12, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: mov z2.d, z8.d
; STRIDED-NEXT: mov z3.d, z12.d
-; STRIDED-NEXT: ld1b { z12.b, z13.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z8.b, z9.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT: ld1b { z12.b - z15.b }, pn8/z, [sp, #8, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ld1b { z8.b - z11.b }, pn8/z, [sp, #12, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; STRIDED-NEXT: mov z1.d, z4.d
; STRIDED-NEXT: addvl sp, sp, #17
@@ -781,12 +727,9 @@ define <vscale x 64 x i8> @ldnt1_x4_i8_z0_z4_z8_z12_scalar(<vscale x 16 x i8> %u
; STRIDED-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z14.b, z15.b }, pn8, [sp, #8, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z12.b, z13.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z10.b, z11.b }, pn8, [sp, #12, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z8.b, z9.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z12.b - z15.b }, pn8, [sp, #8, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z8.b - z11.b }, pn8, [sp, #12, mul vl] // 64-byte Folded Spill
; STRIDED-NEXT: mov p8.b, p0.b
; STRIDED-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; STRIDED-NEXT: str z22, [sp, #16, mul vl] // 16-byte Folded Spill
@@ -797,15 +740,12 @@ define <vscale x 64 x i8> @ldnt1_x4_i8_z0_z4_z8_z12_scalar(<vscale x 16 x i8> %u
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z14.b, z15.b }, pn8/z, [sp, #8, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z10.b, z11.b }, pn8/z, [sp, #12, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: mov z2.d, z8.d
; STRIDED-NEXT: mov z3.d, z12.d
-; STRIDED-NEXT: ld1b { z12.b, z13.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z8.b, z9.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT: ld1b { z12.b - z15.b }, pn8/z, [sp, #8, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ld1b { z8.b - z11.b }, pn8/z, [sp, #12, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; STRIDED-NEXT: mov z1.d, z4.d
; STRIDED-NEXT: addvl sp, sp, #17
@@ -876,12 +816,9 @@ define <vscale x 32 x i16> @ldnt1_x4_i16_z0_z4_z8_z12(<vscale x 8 x i16> %unused
; STRIDED-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z14.b, z15.b }, pn8, [sp, #8, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z12.b, z13.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z10.b, z11.b }, pn8, [sp, #12, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z8.b, z9.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z12.b - z15.b }, pn8, [sp, #8, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z8.b - z11.b }, pn8, [sp, #12, mul vl] // 64-byte Folded Spill
; STRIDED-NEXT: mov p8.b, p0.b
; STRIDED-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; STRIDED-NEXT: str z22, [sp, #16, mul vl] // 16-byte Folded Spill
@@ -892,15 +829,12 @@ define <vscale x 32 x i16> @ldnt1_x4_i16_z0_z4_z8_z12(<vscale x 8 x i16> %unused
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z14.b, z15.b }, pn8/z, [sp, #8, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z10.b, z11.b }, pn8/z, [sp, #12, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: mov z2.d, z8.d
; STRIDED-NEXT: mov z3.d, z12.d
-; STRIDED-NEXT: ld1b { z12.b, z13.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z8.b, z9.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT: ld1b { z12.b - z15.b }, pn8/z, [sp, #8, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ld1b { z8.b - z11.b }, pn8/z, [sp, #12, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; STRIDED-NEXT: mov z1.d, z4.d
; STRIDED-NEXT: addvl sp, sp, #17
@@ -970,12 +904,9 @@ define <vscale x 32 x i16> @ldnt1_x4_i16_z0_z4_z8_z12_scalar(<vscale x 8 x i16>
; STRIDED-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z14.b, z15.b }, pn8, [sp, #8, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z12.b, z13.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z10.b, z11.b }, pn8, [sp, #12, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z8.b, z9.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z12.b - z15.b }, pn8, [sp, #8, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z8.b - z11.b }, pn8, [sp, #12, mul vl] // 64-byte Folded Spill
; STRIDED-NEXT: mov p8.b, p0.b
; STRIDED-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; STRIDED-NEXT: str z22, [sp, #16, mul vl] // 16-byte Folded Spill
@@ -986,15 +917,12 @@ define <vscale x 32 x i16> @ldnt1_x4_i16_z0_z4_z8_z12_scalar(<vscale x 8 x i16>
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z14.b, z15.b }, pn8/z, [sp, #8, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z10.b, z11.b }, pn8/z, [sp, #12, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: mov z2.d, z8.d
; STRIDED-NEXT: mov z3.d, z12.d
-; STRIDED-NEXT: ld1b { z12.b, z13.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z8.b, z9.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT: ld1b { z12.b - z15.b }, pn8/z, [sp, #8, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ld1b { z8.b - z11.b }, pn8/z, [sp, #12, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; STRIDED-NEXT: mov z1.d, z4.d
; STRIDED-NEXT: addvl sp, sp, #17
@@ -1065,12 +993,9 @@ define <vscale x 16 x i32> @ldnt1_x4_i32_z0_z4_z8_z12(<vscale x 4 x i32> %unused
; STRIDED-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z14.b, z15.b }, pn8, [sp, #8, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z12.b, z13.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z10.b, z11.b }, pn8, [sp, #12, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z8.b, z9.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z12.b - z15.b }, pn8, [sp, #8, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z8.b - z11.b }, pn8, [sp, #12, mul vl] // 64-byte Folded Spill
; STRIDED-NEXT: mov p8.b, p0.b
; STRIDED-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; STRIDED-NEXT: str z22, [sp, #16, mul vl] // 16-byte Folded Spill
@@ -1081,15 +1006,12 @@ define <vscale x 16 x i32> @ldnt1_x4_i32_z0_z4_z8_z12(<vscale x 4 x i32> %unused
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z14.b, z15.b }, pn8/z, [sp, #8, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z10.b, z11.b }, pn8/z, [sp, #12, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: mov z2.d, z8.d
; STRIDED-NEXT: mov z3.d, z12.d
-; STRIDED-NEXT: ld1b { z12.b, z13.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z8.b, z9.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT: ld1b { z12.b - z15.b }, pn8/z, [sp, #8, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ld1b { z8.b - z11.b }, pn8/z, [sp, #12, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; STRIDED-NEXT: mov z1.d, z4.d
; STRIDED-NEXT: addvl sp, sp, #17
@@ -1159,12 +1081,9 @@ define <vscale x 16 x i32> @ldnt1_x4_i32_z0_z4_z8_z12_scalar(<vscale x 4 x i32>
; STRIDED-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z14.b, z15.b }, pn8, [sp, #8, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z12.b, z13.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z10.b, z11.b }, pn8, [sp, #12, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z8.b, z9.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z12.b - z15.b }, pn8, [sp, #8, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z8.b - z11.b }, pn8, [sp, #12, mul vl] // 64-byte Folded Spill
; STRIDED-NEXT: mov p8.b, p0.b
; STRIDED-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; STRIDED-NEXT: str z22, [sp, #16, mul vl] // 16-byte Folded Spill
@@ -1175,15 +1094,12 @@ define <vscale x 16 x i32> @ldnt1_x4_i32_z0_z4_z8_z12_scalar(<vscale x 4 x i32>
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z14.b, z15.b }, pn8/z, [sp, #8, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z10.b, z11.b }, pn8/z, [sp, #12, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: mov z2.d, z8.d
; STRIDED-NEXT: mov z3.d, z12.d
-; STRIDED-NEXT: ld1b { z12.b, z13.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z8.b, z9.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT: ld1b { z12.b - z15.b }, pn8/z, [sp, #8, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ld1b { z8.b - z11.b }, pn8/z, [sp, #12, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; STRIDED-NEXT: mov z1.d, z4.d
; STRIDED-NEXT: addvl sp, sp, #17
@@ -1254,12 +1170,9 @@ define <vscale x 8 x i64> @ldnt1_x4_i64_z0_z4_z8_z12(<vscale x 2 x i64> %unused,
; STRIDED-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z14.b, z15.b }, pn8, [sp, #8, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z12.b, z13.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z10.b, z11.b }, pn8, [sp, #12, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z8.b, z9.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z12.b - z15.b }, pn8, [sp, #8, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z8.b - z11.b }, pn8, [sp, #12, mul vl] // 64-byte Folded Spill
; STRIDED-NEXT: mov p8.b, p0.b
; STRIDED-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; STRIDED-NEXT: str z22, [sp, #16, mul vl] // 16-byte Folded Spill
@@ -1270,15 +1183,12 @@ define <vscale x 8 x i64> @ldnt1_x4_i64_z0_z4_z8_z12(<vscale x 2 x i64> %unused,
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z14.b, z15.b }, pn8/z, [sp, #8, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z10.b, z11.b }, pn8/z, [sp, #12, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: mov z2.d, z8.d
; STRIDED-NEXT: mov z3.d, z12.d
-; STRIDED-NEXT: ld1b { z12.b, z13.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z8.b, z9.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT: ld1b { z12.b - z15.b }, pn8/z, [sp, #8, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ld1b { z8.b - z11.b }, pn8/z, [sp, #12, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; STRIDED-NEXT: mov z1.d, z4.d
; STRIDED-NEXT: addvl sp, sp, #17
@@ -1348,12 +1258,9 @@ define <vscale x 8 x i64> @ldnt1_x4_i64_z0_z4_z8_z12_scalar(<vscale x 2 x i64> %
; STRIDED-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z14.b, z15.b }, pn8, [sp, #8, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z12.b, z13.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z10.b, z11.b }, pn8, [sp, #12, mul vl] // 32-byte Folded Spill
-; STRIDED-NEXT: st1b { z8.b, z9.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z12.b - z15.b }, pn8, [sp, #8, mul vl] // 64-byte Folded Spill
+; STRIDED-NEXT: st1b { z8.b - z11.b }, pn8, [sp, #12, mul vl] // 64-byte Folded Spill
; STRIDED-NEXT: mov p8.b, p0.b
; STRIDED-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; STRIDED-NEXT: str z22, [sp, #16, mul vl] // 16-byte Folded Spill
@@ -1364,15 +1271,12 @@ define <vscale x 8 x i64> @ldnt1_x4_i64_z0_z4_z8_z12_scalar(<vscale x 2 x i64> %
; STRIDED-NEXT: ptrue pn8.b
; STRIDED-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z14.b, z15.b }, pn8/z, [sp, #8, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z10.b, z11.b }, pn8/z, [sp, #12, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
; STRIDED-NEXT: mov z2.d, z8.d
; STRIDED-NEXT: mov z3.d, z12.d
-; STRIDED-NEXT: ld1b { z12.b, z13.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ld1b { z8.b, z9.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
-; STRIDED-NEXT: ldr z22, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT: ld1b { z12.b - z15.b }, pn8/z, [sp, #8, mul vl] // 64-byte Folded Reload
+; STRIDED-NEXT: ld1b { z8.b - z11.b }, pn8/z, [sp, #12, mul vl] // 64-byte Folded Reload
; STRIDED-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; STRIDED-NEXT: mov z1.d, z4.d
; STRIDED-NEXT: addvl sp, sp, #17
diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-qcvt.ll b/llvm/test/CodeGen/AArch64/sme2-intrinsics-qcvt.ll
index c637aab31f4d7..81cf871e75b76 100644
--- a/llvm/test/CodeGen/AArch64/sme2-intrinsics-qcvt.ll
+++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-qcvt.ll
@@ -53,8 +53,7 @@ define { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8
; CHECK-NEXT: ptrue pn8.b
; CHECK-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; CHECK-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; CHECK-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; CHECK-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CHECK-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
; CHECK-NEXT: str z22, [sp, #8, mul vl] // 16-byte Folded Spill
; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0xc8, 0x00, 0x1e, 0x22 // sp + 16 + 72 * VG
; CHECK-NEXT: .cfi_offset w29, -16
@@ -74,8 +73,7 @@ define { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8
; CHECK-NEXT: sqcvt z3.h, { z28.d - z31.d }
; CHECK-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; CHECK-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; CHECK-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; CHECK-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CHECK-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
; CHECK-NEXT: ldr z22, [sp, #8, mul vl] // 16-byte Folded Reload
; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; CHECK-NEXT: addvl sp, sp, #9
diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-qrshr.ll b/llvm/test/CodeGen/AArch64/sme2-intrinsics-qrshr.ll
index ed1b03bb209b3..546c44ac85c7e 100644
--- a/llvm/test/CodeGen/AArch64/sme2-intrinsics-qrshr.ll
+++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-qrshr.ll
@@ -93,8 +93,7 @@ define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 1
; CHECK-NEXT: ptrue pn8.b
; CHECK-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; CHECK-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; CHECK-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; CHECK-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CHECK-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
; CHECK-NEXT: str z22, [sp, #8, mul vl] // 16-byte Folded Spill
; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0xc8, 0x00, 0x1e, 0x22 // sp + 16 + 72 * VG
; CHECK-NEXT: .cfi_offset w29, -16
@@ -114,8 +113,7 @@ define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 1
; CHECK-NEXT: sqrshr z3.b, { z28.s - z31.s }, #32
; CHECK-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; CHECK-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; CHECK-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; CHECK-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CHECK-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
; CHECK-NEXT: ldr z22, [sp, #8, mul vl] // 16-byte Folded Reload
; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; CHECK-NEXT: addvl sp, sp, #9
diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-vdot.ll b/llvm/test/CodeGen/AArch64/sme2-intrinsics-vdot.ll
index ab4813c364db8..ef523535a0324 100644
--- a/llvm/test/CodeGen/AArch64/sme2-intrinsics-vdot.ll
+++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-vdot.ll
@@ -196,8 +196,7 @@ define void @svdot_form_4x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 8 x i16>
; CHECK-NEXT: ptrue pn8.b
; CHECK-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; CHECK-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; CHECK-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; CHECK-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CHECK-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
; CHECK-NEXT: str z22, [sp, #8, mul vl] // 16-byte Folded Spill
; CHECK-NEXT: ptrue pn8.b
; CHECK-NEXT: add x10, x9, x1
@@ -213,8 +212,7 @@ define void @svdot_form_4x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 8 x i16>
; CHECK-NEXT: svdot za.s[w8, 0, vgx4], { z28.b - z31.b }, z0.b[0]
; CHECK-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; CHECK-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; CHECK-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; CHECK-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CHECK-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
; CHECK-NEXT: ldr z22, [sp, #8, mul vl] // 16-byte Folded Reload
; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; CHECK-NEXT: str z0, [x0]
@@ -419,8 +417,7 @@ define void @uvdot_form_4x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 8 x i16>
; CHECK-NEXT: ptrue pn8.b
; CHECK-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; CHECK-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; CHECK-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; CHECK-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CHECK-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
; CHECK-NEXT: str z22, [sp, #8, mul vl] // 16-byte Folded Spill
; CHECK-NEXT: ptrue pn8.b
; CHECK-NEXT: add x10, x9, x1
@@ -436,8 +433,7 @@ define void @uvdot_form_4x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 8 x i16>
; CHECK-NEXT: uvdot za.s[w8, 0, vgx4], { z28.b - z31.b }, z0.b[0]
; CHECK-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; CHECK-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; CHECK-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; CHECK-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CHECK-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
; CHECK-NEXT: ldr z22, [sp, #8, mul vl] // 16-byte Folded Reload
; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; CHECK-NEXT: str z0, [x0]
@@ -554,8 +550,7 @@ define void @suvdot_form_4x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 8 x i16
; CHECK-NEXT: ptrue pn8.b
; CHECK-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; CHECK-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; CHECK-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; CHECK-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CHECK-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
; CHECK-NEXT: str z22, [sp, #8, mul vl] // 16-byte Folded Spill
; CHECK-NEXT: ptrue pn8.b
; CHECK-NEXT: add x10, x9, x1
@@ -571,8 +566,7 @@ define void @suvdot_form_4x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 8 x i16
; CHECK-NEXT: suvdot za.s[w8, 0, vgx4], { z28.b - z31.b }, z0.b[0]
; CHECK-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; CHECK-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; CHECK-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; CHECK-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CHECK-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
; CHECK-NEXT: ldr z22, [sp, #8, mul vl] // 16-byte Folded Reload
; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; CHECK-NEXT: str z0, [x0]
@@ -689,8 +683,7 @@ define void @usvdot_form_4x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 8 x i16
; CHECK-NEXT: ptrue pn8.b
; CHECK-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill
; CHECK-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; CHECK-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; CHECK-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CHECK-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
; CHECK-NEXT: str z22, [sp, #8, mul vl] // 16-byte Folded Spill
; CHECK-NEXT: ptrue pn8.b
; CHECK-NEXT: add x10, x9, x1
@@ -706,8 +699,7 @@ define void @usvdot_form_4x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 8 x i16
; CHECK-NEXT: usvdot za.s[w8, 0, vgx4], { z28.b - z31.b }, z0.b[0]
; CHECK-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
; CHECK-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; CHECK-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; CHECK-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CHECK-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
; CHECK-NEXT: ldr z22, [sp, #8, mul vl] // 16-byte Folded Reload
; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; CHECK-NEXT: str z0, [x0]
diff --git a/llvm/test/CodeGen/AArch64/sve-callee-save-restore-pairs.ll b/llvm/test/CodeGen/AArch64/sve-callee-save-restore-pairs.ll
index 813dde53a1d7c..57ea481bf5147 100644
--- a/llvm/test/CodeGen/AArch64/sve-callee-save-restore-pairs.ll
+++ b/llvm/test/CodeGen/AArch64/sve-callee-save-restore-pairs.ll
@@ -171,14 +171,10 @@ define void @fbyte(<vscale x 16 x i8> %v){
; SPLIT-PAIR-NEXT: sub sp, sp, #16
; SPLIT-PAIR-NEXT: addvl sp, sp, #-16
; SPLIT-PAIR-NEXT: ptrue pn8.b
-; SPLIT-PAIR-NEXT: st1b { z22.b, z23.b }, pn8, [sp] // 32-byte Folded Spill
-; SPLIT-PAIR-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; SPLIT-PAIR-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; SPLIT-PAIR-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
-; SPLIT-PAIR-NEXT: st1b { z14.b, z15.b }, pn8, [sp, #8, mul vl] // 32-byte Folded Spill
-; SPLIT-PAIR-NEXT: st1b { z12.b, z13.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
-; SPLIT-PAIR-NEXT: st1b { z10.b, z11.b }, pn8, [sp, #12, mul vl] // 32-byte Folded Spill
-; SPLIT-PAIR-NEXT: st1b { z8.b, z9.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; SPLIT-PAIR-NEXT: st1b { z20.b - z23.b }, pn8, [sp] // 64-byte Folded Spill
+; SPLIT-PAIR-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
+; SPLIT-PAIR-NEXT: st1b { z12.b - z15.b }, pn8, [sp, #8, mul vl] // 64-byte Folded Spill
+; SPLIT-PAIR-NEXT: st1b { z8.b - z11.b }, pn8, [sp, #12, mul vl] // 64-byte Folded Spill
; SPLIT-PAIR-NEXT: sub sp, sp, #16
; SPLIT-PAIR-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x30, 0x92, 0x2e, 0x00, 0x11, 0x90, 0x01, 0x1e, 0x22 // sp + 48 + 144 * VG
; SPLIT-PAIR-NEXT: .cfi_offset w30, -8
@@ -194,14 +190,10 @@ define void @fbyte(<vscale x 16 x i8> %v){
; SPLIT-PAIR-NEXT: bl my_func
; SPLIT-PAIR-NEXT: add sp, sp, #16
; SPLIT-PAIR-NEXT: ptrue pn8.b
-; SPLIT-PAIR-NEXT: ld1b { z22.b, z23.b }, pn8/z, [sp] // 32-byte Folded Reload
-; SPLIT-PAIR-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; SPLIT-PAIR-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; SPLIT-PAIR-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
-; SPLIT-PAIR-NEXT: ld1b { z14.b, z15.b }, pn8/z, [sp, #8, mul vl] // 32-byte Folded Reload
-; SPLIT-PAIR-NEXT: ld1b { z12.b, z13.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
-; SPLIT-PAIR-NEXT: ld1b { z10.b, z11.b }, pn8/z, [sp, #12, mul vl] // 32-byte Folded Reload
-; SPLIT-PAIR-NEXT: ld1b { z8.b, z9.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; SPLIT-PAIR-NEXT: ld1b { z20.b - z23.b }, pn8/z, [sp] // 64-byte Folded Reload
+; SPLIT-PAIR-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
+; SPLIT-PAIR-NEXT: ld1b { z12.b - z15.b }, pn8/z, [sp, #8, mul vl] // 64-byte Folded Reload
+; SPLIT-PAIR-NEXT: ld1b { z8.b - z11.b }, pn8/z, [sp, #12, mul vl] // 64-byte Folded Reload
; SPLIT-PAIR-NEXT: add sp, sp, #16
; SPLIT-PAIR-NEXT: addvl sp, sp, #16
; SPLIT-PAIR-NEXT: ldr p15, [sp, #4, mul vl] // 2-byte Reload
@@ -383,14 +375,10 @@ define void @fhalf(<vscale x 8 x half> %v) {
; SPLIT-PAIR-NEXT: sub sp, sp, #16
; SPLIT-PAIR-NEXT: addvl sp, sp, #-16
; SPLIT-PAIR-NEXT: ptrue pn8.b
-; SPLIT-PAIR-NEXT: st1b { z22.b, z23.b }, pn8, [sp] // 32-byte Folded Spill
-; SPLIT-PAIR-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; SPLIT-PAIR-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; SPLIT-PAIR-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
-; SPLIT-PAIR-NEXT: st1b { z14.b, z15.b }, pn8, [sp, #8, mul vl] // 32-byte Folded Spill
-; SPLIT-PAIR-NEXT: st1b { z12.b, z13.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
-; SPLIT-PAIR-NEXT: st1b { z10.b, z11.b }, pn8, [sp, #12, mul vl] // 32-byte Folded Spill
-; SPLIT-PAIR-NEXT: st1b { z8.b, z9.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; SPLIT-PAIR-NEXT: st1b { z20.b - z23.b }, pn8, [sp] // 64-byte Folded Spill
+; SPLIT-PAIR-NEXT: st1b { z16.b - z19.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
+; SPLIT-PAIR-NEXT: st1b { z12.b - z15.b }, pn8, [sp, #8, mul vl] // 64-byte Folded Spill
+; SPLIT-PAIR-NEXT: st1b { z8.b - z11.b }, pn8, [sp, #12, mul vl] // 64-byte Folded Spill
; SPLIT-PAIR-NEXT: sub sp, sp, #16
; SPLIT-PAIR-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x30, 0x92, 0x2e, 0x00, 0x11, 0x90, 0x01, 0x1e, 0x22 // sp + 48 + 144 * VG
; SPLIT-PAIR-NEXT: .cfi_offset w30, -8
@@ -406,14 +394,10 @@ define void @fhalf(<vscale x 8 x half> %v) {
; SPLIT-PAIR-NEXT: bl my_func
; SPLIT-PAIR-NEXT: add sp, sp, #16
; SPLIT-PAIR-NEXT: ptrue pn8.b
-; SPLIT-PAIR-NEXT: ld1b { z22.b, z23.b }, pn8/z, [sp] // 32-byte Folded Reload
-; SPLIT-PAIR-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
-; SPLIT-PAIR-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; SPLIT-PAIR-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
-; SPLIT-PAIR-NEXT: ld1b { z14.b, z15.b }, pn8/z, [sp, #8, mul vl] // 32-byte Folded Reload
-; SPLIT-PAIR-NEXT: ld1b { z12.b, z13.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
-; SPLIT-PAIR-NEXT: ld1b { z10.b, z11.b }, pn8/z, [sp, #12, mul vl] // 32-byte Folded Reload
-; SPLIT-PAIR-NEXT: ld1b { z8.b, z9.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; SPLIT-PAIR-NEXT: ld1b { z20.b - z23.b }, pn8/z, [sp] // 64-byte Folded Reload
+; SPLIT-PAIR-NEXT: ld1b { z16.b - z19.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
+; SPLIT-PAIR-NEXT: ld1b { z12.b - z15.b }, pn8/z, [sp, #8, mul vl] // 64-byte Folded Reload
+; SPLIT-PAIR-NEXT: ld1b { z8.b - z11.b }, pn8/z, [sp, #12, mul vl] // 64-byte Folded Reload
; SPLIT-PAIR-NEXT: add sp, sp, #16
; SPLIT-PAIR-NEXT: addvl sp, sp, #16
; SPLIT-PAIR-NEXT: ldr p15, [sp, #4, mul vl] // 2-byte Reload
@@ -1222,8 +1206,7 @@ define aarch64_sve_vector_pcs void @reorder_z_spills_splits_pair() {
; SPLIT-PAIR-NEXT: sub sp, sp, #16
; SPLIT-PAIR-NEXT: addvl sp, sp, #-6
; SPLIT-PAIR-NEXT: ptrue pn8.b
-; SPLIT-PAIR-NEXT: st1b { z22.b, z23.b }, pn8, [sp] // 32-byte Folded Spill
-; SPLIT-PAIR-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; SPLIT-PAIR-NEXT: st1b { z20.b - z23.b }, pn8, [sp] // 64-byte Folded Spill
; SPLIT-PAIR-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
; SPLIT-PAIR-NEXT: sub sp, sp, #16
; SPLIT-PAIR-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x30, 0x92, 0x2e, 0x00, 0x11, 0x38, 0x1e, 0x22 // sp + 48 + 56 * VG
@@ -1232,8 +1215,7 @@ define aarch64_sve_vector_pcs void @reorder_z_spills_splits_pair() {
; SPLIT-PAIR-NEXT: //NO_APP
; SPLIT-PAIR-NEXT: add sp, sp, #16
; SPLIT-PAIR-NEXT: ptrue pn8.b
-; SPLIT-PAIR-NEXT: ld1b { z22.b, z23.b }, pn8/z, [sp] // 32-byte Folded Reload
-; SPLIT-PAIR-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; SPLIT-PAIR-NEXT: ld1b { z20.b - z23.b }, pn8/z, [sp] // 64-byte Folded Reload
; SPLIT-PAIR-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
; SPLIT-PAIR-NEXT: add sp, sp, #16
; SPLIT-PAIR-NEXT: addvl sp, sp, #6
@@ -1308,8 +1290,7 @@ define aarch64_sve_vector_pcs void @reorder_z_spills() {
; SPLIT-PAIR-NEXT: addvl sp, sp, #-7
; SPLIT-PAIR-NEXT: ptrue pn8.b
; SPLIT-PAIR-NEXT: str z17, [sp, #6, mul vl] // 16-byte Folded Spill
-; SPLIT-PAIR-NEXT: st1b { z22.b, z23.b }, pn8, [sp] // 32-byte Folded Spill
-; SPLIT-PAIR-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; SPLIT-PAIR-NEXT: st1b { z20.b - z23.b }, pn8, [sp] // 64-byte Folded Spill
; SPLIT-PAIR-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
; SPLIT-PAIR-NEXT: sub sp, sp, #16
; SPLIT-PAIR-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x30, 0x92, 0x2e, 0x00, 0x11, 0xc0, 0x00, 0x1e, 0x22 // sp + 48 + 64 * VG
@@ -1319,8 +1300,7 @@ define aarch64_sve_vector_pcs void @reorder_z_spills() {
; SPLIT-PAIR-NEXT: add sp, sp, #16
; SPLIT-PAIR-NEXT: ptrue pn8.b
; SPLIT-PAIR-NEXT: ldr z17, [sp, #6, mul vl] // 16-byte Folded Reload
-; SPLIT-PAIR-NEXT: ld1b { z22.b, z23.b }, pn8/z, [sp] // 32-byte Folded Reload
-; SPLIT-PAIR-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; SPLIT-PAIR-NEXT: ld1b { z20.b - z23.b }, pn8/z, [sp] // 64-byte Folded Reload
; SPLIT-PAIR-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
; SPLIT-PAIR-NEXT: add sp, sp, #16
; SPLIT-PAIR-NEXT: addvl sp, sp, #7
>From c78ee880cbdf8d61a7d28fddc4eae5e572c9d5e9 Mon Sep 17 00:00:00 2001
From: Kieran Bailey <kieran.bailey at arm.com>
Date: Tue, 22 Sep 2026 14:31:02 +0000
Subject: [PATCH 2/4] Removed hardcoded offsets checks Refactor code style and
naming
---
.../Target/AArch64/AArch64FrameLowering.cpp | 412 +++++++++---------
1 file changed, 215 insertions(+), 197 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
index fc75f686f7866..3fd49d1e917eb 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
@@ -1634,6 +1634,20 @@ static bool invalidateRegisterPairing(bool SpillExtendedVolatile,
return false;
}
+static bool isValidMemOpOffset(const AArch64InstrInfo *TII, unsigned Opcode,
+ int Offset) {
+ int64_t MinOff, MaxOff;
+ TypeSize ScaleValue(0U, false), Width(0U, false);
+ if (!TII->getMemOpInfo(Opcode, ScaleValue, Width, MinOff, MaxOff))
+ return false;
+
+ if (Offset % ScaleValue.getKnownMinValue() != 0)
+ return false;
+
+ Offset /= ScaleValue.getKnownMinValue();
+ return Offset >= MinOff && Offset <= MaxOff;
+}
+
namespace {
struct RegGroupInfo {
@@ -1701,12 +1715,14 @@ void computeCalleeSaveRegisterPairs(const AArch64FrameLowering &AFL,
MachineFunction &MF,
ArrayRef<CalleeSavedInfo> CSI,
const TargetRegisterInfo *TRI,
- SmallVectorImpl<RegGroupInfo> &RegPairs,
+ SmallVectorImpl<RegGroupInfo> &RegGroups,
bool NeedsFrameRecord) {
if (CSI.empty())
return;
+ const AArch64InstrInfo *TII =
+ MF.getSubtarget<AArch64Subtarget>().getInstrInfo();
bool IsWindows = isTargetWindows(MF);
AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
unsigned StackHazardSize = getStackHazardSize(MF);
@@ -1775,44 +1791,44 @@ void computeCalleeSaveRegisterPairs(const AArch64FrameLowering &AFL,
// When iterating backwards, the loop condition relies on unsigned wraparound.
for (unsigned i = FirstReg; i < Count; i += RegInc) {
- RegGroupInfo RPI;
- RPI.Reg1 = CSI[i].getReg();
-
- if (AArch64::GPR64RegClass.contains(RPI.Reg1)) {
- RPI.Type = RegGroupInfo::GPR;
- RPI.RC = &AArch64::GPR64RegClass;
- } else if (AArch64::FPR64RegClass.contains(RPI.Reg1)) {
- RPI.Type = RegGroupInfo::FPR64;
- RPI.RC = &AArch64::FPR64RegClass;
- } else if (AArch64::FPR128RegClass.contains(RPI.Reg1)) {
- RPI.Type = RegGroupInfo::FPR128;
- RPI.RC = &AArch64::FPR128RegClass;
- } else if (AArch64::ZPRRegClass.contains(RPI.Reg1)) {
- RPI.Type = RegGroupInfo::ZPR;
- RPI.RC = &AArch64::ZPRRegClass;
- } else if (AArch64::PPRRegClass.contains(RPI.Reg1)) {
- RPI.Type = RegGroupInfo::PPR;
- RPI.RC = &AArch64::PPRRegClass;
- } else if (RPI.Reg1 == AArch64::VG) {
- RPI.Type = RegGroupInfo::VG;
- RPI.RC = &AArch64::FIXED_REGSRegClass;
+ RegGroupInfo RGI;
+ RGI.Reg1 = CSI[i].getReg();
+
+ if (AArch64::GPR64RegClass.contains(RGI.Reg1)) {
+ RGI.Type = RegGroupInfo::GPR;
+ RGI.RC = &AArch64::GPR64RegClass;
+ } else if (AArch64::FPR64RegClass.contains(RGI.Reg1)) {
+ RGI.Type = RegGroupInfo::FPR64;
+ RGI.RC = &AArch64::FPR64RegClass;
+ } else if (AArch64::FPR128RegClass.contains(RGI.Reg1)) {
+ RGI.Type = RegGroupInfo::FPR128;
+ RGI.RC = &AArch64::FPR128RegClass;
+ } else if (AArch64::ZPRRegClass.contains(RGI.Reg1)) {
+ RGI.Type = RegGroupInfo::ZPR;
+ RGI.RC = &AArch64::ZPRRegClass;
+ } else if (AArch64::PPRRegClass.contains(RGI.Reg1)) {
+ RGI.Type = RegGroupInfo::PPR;
+ RGI.RC = &AArch64::PPRRegClass;
+ } else if (RGI.Reg1 == AArch64::VG) {
+ RGI.Type = RegGroupInfo::VG;
+ RGI.RC = &AArch64::FIXED_REGSRegClass;
} else {
llvm_unreachable("Unsupported register class.");
}
- int &ScalableByteOffset = RPI.Type == RegGroupInfo::PPR && SplitPPRs
+ int &ScalableByteOffset = RGI.Type == RegGroupInfo::PPR && SplitPPRs
? PPRByteOffset
: ZPRByteOffset;
// Add the stack hazard size as we transition from GPR->FPR CSRs.
if (HasCSHazardPadding &&
(!LastReg || !AArch64InstrInfo::isFpOrNEON(LastReg)) &&
- AArch64InstrInfo::isFpOrNEON(RPI.Reg1))
+ AArch64InstrInfo::isFpOrNEON(RGI.Reg1))
ByteOffset += StackFillDir * StackHazardSize;
- LastReg = RPI.Reg1;
+ LastReg = RGI.Reg1;
bool NeedsWinCFI = AFL.needsWinCFI(MF);
- int Scale = TRI->getSpillSize(*RPI.RC);
+ int Scale = TRI->getSpillSize(*RGI.RC);
// Add the next reg to the pair if it is in the same register class.
if (unsigned(i + RegInc) < Count && !HasCSHazardPadding) {
MCRegister NextReg = CSI[i + RegInc].getReg();
@@ -1821,56 +1837,58 @@ void computeCalleeSaveRegisterPairs(const AArch64FrameLowering &AFL,
int PairOffset = IsWindows ? Aligned : Aligned + StackFillDir * 2 * Scale;
bool PairFitsImmRange =
PairOffset / Scale >= -64 && PairOffset / Scale <= 63;
- switch (RPI.Type) {
+ switch (RGI.Type) {
case RegGroupInfo::GPR:
if (AArch64::GPR64RegClass.contains(NextReg) && PairFitsImmRange &&
!invalidateRegisterPairing(SpillExtendedVolatile, SpillCount,
- RPI.Reg1, NextReg, IsWindows,
+ RGI.Reg1, NextReg, IsWindows,
NeedsWinCFI, NeedsFrameRecord, TRI))
- RPI.Reg2 = NextReg;
+ RGI.Reg2 = NextReg;
break;
case RegGroupInfo::FPR64:
if (AArch64::FPR64RegClass.contains(NextReg) && PairFitsImmRange &&
!invalidateRegisterPairing(SpillExtendedVolatile, SpillCount,
- RPI.Reg1, NextReg, IsWindows,
+ RGI.Reg1, NextReg, IsWindows,
NeedsWinCFI, NeedsFrameRecord, TRI))
- RPI.Reg2 = NextReg;
+ RGI.Reg2 = NextReg;
break;
case RegGroupInfo::FPR128:
if (AArch64::FPR128RegClass.contains(NextReg) && PairFitsImmRange)
- RPI.Reg2 = NextReg;
+ RGI.Reg2 = NextReg;
break;
case RegGroupInfo::PPR:
break;
case RegGroupInfo::ZPR:
- if (!NeedsWinCFI && AFI->getPredicateRegForFillSpill() != 0) {
- if (unsigned(i + 2 * RegInc) < Count &&
- unsigned(i + 3 * RegInc) < Count &&
- (RPI.Reg1 - AArch64::Z0) % 4 == 0) {
- auto Reg3 = CSI[i + RegInc * 2].getReg();
- auto Reg4 = CSI[i + RegInc * 3].getReg();
- bool Consecutive = (RPI.Reg1 + 1 == NextReg) &&
- (NextReg + 1 == Reg3) && (Reg3 + 1 == Reg4);
- int Offset =
- (ScalableByteOffset + StackFillDir * 4 * Scale) / Scale;
-
- if (Consecutive && (Offset % 4 == 0) &&
- (-32 <= Offset && Offset <= 28)) {
- RPI.Reg2 = NextReg;
- RPI.Reg3 = Reg3;
- RPI.Reg4 = Reg4;
- }
- }
- if (!RPI.isQuad() && ((RPI.Reg1 - AArch64::Z0) & 1) == 0 &&
- (NextReg == RPI.Reg1 + 1)) {
- // Calculate offset of register pair to see if pair instruction can
- // be used.
- int Offset =
- (ScalableByteOffset + StackFillDir * 2 * Scale) / Scale;
- if ((-16 <= Offset && Offset <= 14) && (Offset % 2 == 0))
- RPI.Reg2 = NextReg;
+ if (NeedsWinCFI || AFI->getPredicateRegForFillSpill() == 0)
+ break;
+ if (unsigned(i + 2 * RegInc) < Count &&
+ unsigned(i + 3 * RegInc) < Count &&
+ (RGI.Reg1 - AArch64::Z0) % 4 == 0) {
+ MCRegister Reg3 = CSI[i + RegInc * 2].getReg();
+ MCRegister Reg4 = CSI[i + RegInc * 3].getReg();
+ bool Consecutive = (RGI.Reg1 + 1 == NextReg) &&
+ (NextReg + 1 == Reg3) && (Reg3 + 1 == Reg4);
+
+ const int NumRegs = 4;
+ int Offset = (ScalableByteOffset + StackFillDir * NumRegs * Scale);
+
+ if (Consecutive &&
+ isValidMemOpOffset(TII, AArch64::LD1B_4Z_IMM, Offset)) {
+ RGI.Reg2 = NextReg;
+ RGI.Reg3 = Reg3;
+ RGI.Reg4 = Reg4;
}
}
+ if (!RGI.isQuad() && ((RGI.Reg1 - AArch64::Z0) & 1) == 0 &&
+ (NextReg == RGI.Reg1 + 1)) {
+ // Calculate offset of register pair to see if pair instruction can
+ // be used.
+ const int NumRegs = 2;
+ int Offset = (ScalableByteOffset + StackFillDir * NumRegs * Scale);
+ // Note: ST1B has the same offset constraints.
+ if (isValidMemOpOffset(TII, AArch64::LD1B_2Z_IMM, Offset))
+ RGI.Reg2 = NextReg;
+ }
break;
case RegGroupInfo::VG:
break;
@@ -1883,11 +1901,11 @@ void computeCalleeSaveRegisterPairs(const AArch64FrameLowering &AFL,
//
// The order of the registers in the list is controlled by
// getCalleeSavedRegs(), so they will always be in-order, as well.
- assert((!RPI.isPaired() ||
+ assert((!RGI.isPaired() ||
(CSI[i].getFrameIdx() + RegInc == CSI[i + RegInc].getFrameIdx())) &&
"Out of order callee saved regs!");
- assert((!RPI.isQuad() ||
+ assert((!RGI.isQuad() ||
((CSI[i].getFrameIdx() + RegInc == CSI[i + RegInc].getFrameIdx()) &&
(CSI[i + RegInc].getFrameIdx() + RegInc ==
CSI[i + RegInc * 2].getFrameIdx()) &&
@@ -1895,16 +1913,16 @@ void computeCalleeSaveRegisterPairs(const AArch64FrameLowering &AFL,
CSI[i + RegInc * 3].getFrameIdx()))) &&
"Out of order callee saved regs!");
- assert((!RPI.isQuad() || RPI.Type == RegGroupInfo::ZPR) &&
+ assert((!RGI.isQuad() || RGI.Type == RegGroupInfo::ZPR) &&
"Currently only ZPR's support four-register spills");
- assert((!RPI.isPaired() || !NeedsFrameRecord || RPI.Reg2 != AArch64::FP ||
- RPI.Reg1 == AArch64::LR) &&
+ assert((!RGI.isPaired() || !NeedsFrameRecord || RGI.Reg2 != AArch64::FP ||
+ RGI.Reg1 == AArch64::LR) &&
"FrameRecord must be allocated together with LR");
// Windows AAPCS has FP and LR reversed.
- assert((!RPI.isPaired() || !NeedsFrameRecord || RPI.Reg1 != AArch64::FP ||
- RPI.Reg2 == AArch64::LR) &&
+ assert((!RGI.isPaired() || !NeedsFrameRecord || RGI.Reg1 != AArch64::FP ||
+ RGI.Reg2 == AArch64::LR) &&
"FrameRecord must be allocated together with LR");
// MachO's compact unwind format relies on all registers being stored in
@@ -1912,56 +1930,56 @@ void computeCalleeSaveRegisterPairs(const AArch64FrameLowering &AFL,
assert((!produceCompactUnwindFrame(AFL, MF) ||
CC == CallingConv::PreserveMost || CC == CallingConv::PreserveAll ||
CC == CallingConv::CXX_FAST_TLS || CC == CallingConv::Win64 ||
- (RPI.isPaired() &&
- ((RPI.Reg1 == AArch64::LR && RPI.Reg2 == AArch64::FP) ||
- RPI.Reg1 + 1 == RPI.Reg2))) &&
+ (RGI.isPaired() &&
+ ((RGI.Reg1 == AArch64::LR && RGI.Reg2 == AArch64::FP) ||
+ RGI.Reg1 + 1 == RGI.Reg2))) &&
"Callee-save registers not saved as adjacent register pair!");
- RPI.FrameIdx = CSI[i].getFrameIdx();
+ RGI.FrameIdx = CSI[i].getFrameIdx();
if (IsWindows)
- // RPI.FrameIdx must be the lower index of the group
- RPI.FrameIdx = CSI[i + RegInc * (RPI.getNumRegs() - 1)].getFrameIdx();
+ // RGI.FrameIdx must be the lower index of the group
+ RGI.FrameIdx = CSI[i + RegInc * (RGI.getNumRegs() - 1)].getFrameIdx();
// Realign the scalable offset if necessary. This is relevant when spilling
// predicates on Windows.
- if (RPI.isScalable() && ScalableByteOffset % Scale != 0)
+ if (RGI.isScalable() && ScalableByteOffset % Scale != 0)
ScalableByteOffset = AlignOffset(ScalableByteOffset, Scale);
// Realign the fixed offset if necessary. This is relevant when spilling Q
// registers after spilling an odd amount of X registers.
- if (!RPI.isScalable() && ByteOffset % Scale != 0)
+ if (!RGI.isScalable() && ByteOffset % Scale != 0)
ByteOffset = AlignOffset(ByteOffset, Scale);
- int OffsetPre = RPI.isScalable() ? ScalableByteOffset : ByteOffset;
+ int OffsetPre = RGI.isScalable() ? ScalableByteOffset : ByteOffset;
assert(OffsetPre % Scale == 0);
- if (RPI.isScalable())
- ScalableByteOffset += StackFillDir * RPI.getNumRegs() * Scale;
+ if (RGI.isScalable())
+ ScalableByteOffset += StackFillDir * RGI.getNumRegs() * Scale;
else
- ByteOffset += StackFillDir * RPI.getNumRegs() * Scale;
+ ByteOffset += StackFillDir * RGI.getNumRegs() * Scale;
// Swift's async context is directly before FP, so allocate an extra
// 8 bytes for it.
if (NeedsFrameRecord && AFI->hasSwiftAsyncContext() &&
- ((!IsWindows && RPI.Reg2 == AArch64::FP) ||
- (IsWindows && RPI.Reg2 == AArch64::LR)))
+ ((!IsWindows && RGI.Reg2 == AArch64::FP) ||
+ (IsWindows && RGI.Reg2 == AArch64::LR)))
ByteOffset += StackFillDir * 8;
// Round up size of non-pair to pair size if we need to pad the
// callee-save area to ensure 16-byte alignment.
- if (NeedGapToAlignStack && !IsWindows && !RPI.isScalable() &&
- RPI.Type != RegGroupInfo::FPR128 && !RPI.isPaired() &&
+ if (NeedGapToAlignStack && !IsWindows && !RGI.isScalable() &&
+ RGI.Type != RegGroupInfo::FPR128 && !RGI.isPaired() &&
ByteOffset % 16 != 0) {
ByteOffset += 8 * StackFillDir;
- assert(MFI.getObjectAlign(RPI.FrameIdx) <= Align(16));
+ assert(MFI.getObjectAlign(RGI.FrameIdx) <= Align(16));
// A stack frame with a gap looks like this, bottom up:
// d9, d8. x21, gap, x20, x19.
// Set extra alignment on the x21 object to create the gap above it.
- MFI.setObjectAlignment(RPI.FrameIdx, Align(16));
+ MFI.setObjectAlignment(RGI.FrameIdx, Align(16));
NeedGapToAlignStack = false;
}
- int OffsetPost = RPI.isScalable() ? ScalableByteOffset : ByteOffset;
+ int OffsetPost = RGI.isScalable() ? ScalableByteOffset : ByteOffset;
assert(OffsetPost % Scale == 0);
// If filling top down (default), we want the offset after incrementing it.
// If filling bottom up (WinCFI) we need the original offset.
@@ -1970,20 +1988,20 @@ void computeCalleeSaveRegisterPairs(const AArch64FrameLowering &AFL,
// The FP, LR pair goes 8 bytes into our expanded 24-byte slot so that the
// Swift context can directly precede FP.
if (NeedsFrameRecord && AFI->hasSwiftAsyncContext() &&
- ((!IsWindows && RPI.Reg2 == AArch64::FP) ||
- (IsWindows && RPI.Reg2 == AArch64::LR)))
+ ((!IsWindows && RGI.Reg2 == AArch64::FP) ||
+ (IsWindows && RGI.Reg2 == AArch64::LR)))
Offset += 8;
- RPI.Offset = Offset / Scale;
+ RGI.Offset = Offset / Scale;
- assert((!RPI.isPaired() ||
- (!RPI.isScalable() && RPI.Offset >= -64 && RPI.Offset <= 63) ||
- (RPI.isScalable() && RPI.Offset >= -256 && RPI.Offset <= 255)) &&
+ assert((!RGI.isPaired() ||
+ (!RGI.isScalable() && RGI.Offset >= -64 && RGI.Offset <= 63) ||
+ (RGI.isScalable() && RGI.Offset >= -256 && RGI.Offset <= 255)) &&
"Offset out of bounds for LDP/STP immediate");
auto isFrameRecord = [&] {
- if (RPI.isPaired())
- return IsWindows ? RPI.Reg1 == AArch64::FP && RPI.Reg2 == AArch64::LR
- : RPI.Reg1 == AArch64::LR && RPI.Reg2 == AArch64::FP;
+ if (RGI.isPaired())
+ return IsWindows ? RGI.Reg1 == AArch64::FP && RGI.Reg2 == AArch64::LR
+ : RGI.Reg1 == AArch64::LR && RGI.Reg2 == AArch64::FP;
// Otherwise, look for the frame record as two unpaired registers. This is
// needed for -aarch64-stack-hazard-size=<val>, which disables register
// pairing (as the padding may be too large for the LDP/STP offset). Note:
@@ -1991,7 +2009,7 @@ void computeCalleeSaveRegisterPairs(const AArch64FrameLowering &AFL,
// and on other platforms current reg == FP, previous reg == LR. This
// works out as the correct pre-increment or post-increment offsets
// respectively.
- return i > 0 && RPI.Reg1 == AArch64::FP &&
+ return i > 0 && RGI.Reg1 == AArch64::FP &&
CSI[i - 1].getReg() == AArch64::LR;
};
@@ -2000,8 +2018,8 @@ void computeCalleeSaveRegisterPairs(const AArch64FrameLowering &AFL,
if (NeedsFrameRecord && isFrameRecord())
AFI->setCalleeSaveBaseToFrameRecordOffset(Offset);
- RegPairs.push_back(RPI);
- i += RegInc * (RPI.getNumRegs() - 1);
+ RegGroups.push_back(RGI);
+ i += RegInc * (RGI.getNumRegs() - 1);
}
if (IsWindows) {
// If we need an alignment gap in the stack, align the topmost stack
@@ -2011,9 +2029,9 @@ void computeCalleeSaveRegisterPairs(const AArch64FrameLowering &AFL,
// CSI, which goes top down), to create the gap above it.
if (AFI->hasCalleeSaveStackFreeSpace())
MFI.setObjectAlignment(CSI[0].getFrameIdx(), Align(16));
- // We iterated bottom up over the registers; flip RegPairs back to top
+ // We iterated bottom up over the registers; flip RegGroups back to top
// down order.
- std::reverse(RegPairs.begin(), RegPairs.end());
+ std::reverse(RegGroups.begin(), RegGroups.end());
}
}
@@ -2026,9 +2044,9 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
const AArch64InstrInfo &TII = *Subtarget.getInstrInfo();
bool NeedsWinCFI = needsWinCFI(MF);
DebugLoc DL;
- SmallVector<RegGroupInfo, 8> RegPairs;
+ SmallVector<RegGroupInfo, 8> RegGroups;
- computeCalleeSaveRegisterPairs(*this, MF, CSI, TRI, RegPairs, hasFP(MF));
+ computeCalleeSaveRegisterPairs(*this, MF, CSI, TRI, RegGroups, hasFP(MF));
MachineRegisterInfo &MRI = MF.getRegInfo();
// Refresh the reserved regs in case there are any potential changes since the
@@ -2039,24 +2057,24 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
auto MIB = BuildMI(MBB, MI, DL, TII.get(AArch64::HOM_Prolog))
.setMIFlag(MachineInstr::FrameSetup);
- for (auto &RPI : RegPairs) {
- MIB.addReg(RPI.Reg1);
- MIB.addReg(RPI.Reg2);
+ for (auto &RGI : RegGroups) {
+ MIB.addReg(RGI.Reg1);
+ MIB.addReg(RGI.Reg2);
// Update register live in.
- if (!MRI.isReserved(RPI.Reg1))
- MBB.addLiveIn(RPI.Reg1);
- if (RPI.isPaired() && !MRI.isReserved(RPI.Reg2))
- MBB.addLiveIn(RPI.Reg2);
+ if (!MRI.isReserved(RGI.Reg1))
+ MBB.addLiveIn(RGI.Reg1);
+ if (RGI.isPaired() && !MRI.isReserved(RGI.Reg2))
+ MBB.addLiveIn(RGI.Reg2);
}
return true;
}
bool PTrueCreated = false;
- for (const RegGroupInfo &RPI : llvm::reverse(RegPairs)) {
- Register Reg1 = RPI.Reg1;
- Register Reg2 = RPI.Reg2;
- Register Reg3 = RPI.Reg3;
- Register Reg4 = RPI.Reg4;
+ for (const RegGroupInfo &RGI : llvm::reverse(RegGroups)) {
+ Register Reg1 = RGI.Reg1;
+ Register Reg2 = RGI.Reg2;
+ Register Reg3 = RGI.Reg3;
+ Register Reg4 = RGI.Reg4;
unsigned StrOpc;
// Issue sequence of spills for cs regs. The first spill may be converted
@@ -2069,22 +2087,22 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
// Rationale: This sequence saves uop updates compared to a sequence of
// pre-increment spills like stp xi,xj,[sp,#-16]!
// Note: Similar rationale and sequence for restores in epilog.
- unsigned Size = TRI->getSpillSize(*RPI.RC);
- Align Alignment = TRI->getSpillAlign(*RPI.RC);
- switch (RPI.Type) {
+ unsigned Size = TRI->getSpillSize(*RGI.RC);
+ Align Alignment = TRI->getSpillAlign(*RGI.RC);
+ switch (RGI.Type) {
case RegGroupInfo::GPR:
- StrOpc = RPI.isPaired() ? AArch64::STPXi : AArch64::STRXui;
+ StrOpc = RGI.isPaired() ? AArch64::STPXi : AArch64::STRXui;
break;
case RegGroupInfo::FPR64:
- StrOpc = RPI.isPaired() ? AArch64::STPDi : AArch64::STRDui;
+ StrOpc = RGI.isPaired() ? AArch64::STPDi : AArch64::STRDui;
break;
case RegGroupInfo::FPR128:
- StrOpc = RPI.isPaired() ? AArch64::STPQi : AArch64::STRQui;
+ StrOpc = RGI.isPaired() ? AArch64::STPQi : AArch64::STRQui;
break;
case RegGroupInfo::ZPR:
- StrOpc = RPI.isQuad()
+ StrOpc = RGI.isQuad()
? AArch64::ST1B_4Z_IMM
- : (RPI.isPaired() ? AArch64::ST1B_2Z_IMM : AArch64::STR_ZXI);
+ : (RGI.isPaired() ? AArch64::ST1B_2Z_IMM : AArch64::STR_ZXI);
break;
case RegGroupInfo::PPR:
StrOpc = AArch64::STR_PXI;
@@ -2138,18 +2156,18 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
LLVM_DEBUG({
dbgs() << "CSR spill: (" << printReg(Reg1, TRI);
- if (RPI.isPaired() || RPI.isQuad())
+ if (RGI.isPaired() || RGI.isQuad())
dbgs() << ", " << printReg(Reg2, TRI);
- if (RPI.isQuad()) {
+ if (RGI.isQuad()) {
dbgs() << ", " << printReg(Reg3, TRI);
dbgs() << ", " << printReg(Reg4, TRI);
}
- dbgs() << ") -> fi#(" << RPI.FrameIdx;
- if (RPI.isPaired() || RPI.isQuad())
- dbgs() << ", " << RPI.FrameIdx + 1;
- if (RPI.isQuad()) {
- dbgs() << ", " << RPI.FrameIdx + 2;
- dbgs() << ", " << RPI.FrameIdx + 3;
+ dbgs() << ") -> fi#(" << RGI.FrameIdx;
+ if (RGI.isPaired() || RGI.isQuad())
+ dbgs() << ", " << RGI.FrameIdx + 1;
+ if (RGI.isQuad()) {
+ dbgs() << ", " << RGI.FrameIdx + 2;
+ dbgs() << ", " << RGI.FrameIdx + 3;
}
dbgs() << ")\n";
});
@@ -2160,22 +2178,22 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
// Windows unwind codes require consecutive registers if registers are
// paired. Make the switch here, so that the code below will save (x,x+1)
// and not (x+1,x).
- unsigned FrameIdxReg1 = RPI.FrameIdx;
- unsigned FrameIdxReg2 = RPI.FrameIdx + 1;
- unsigned FrameIdxReg3 = RPI.FrameIdx + 2;
- unsigned FrameIdxReg4 = RPI.FrameIdx + 3;
+ unsigned FrameIdxReg1 = RGI.FrameIdx;
+ unsigned FrameIdxReg2 = RGI.FrameIdx + 1;
+ unsigned FrameIdxReg3 = RGI.FrameIdx + 2;
+ unsigned FrameIdxReg4 = RGI.FrameIdx + 3;
- if (isTargetWindows(MF) && RPI.isPaired()) {
+ if (isTargetWindows(MF) && RGI.isPaired()) {
std::swap(Reg1, Reg2);
std::swap(FrameIdxReg1, FrameIdxReg2);
- } else if (isTargetWindows(MF) && RPI.isQuad()) {
+ } else if (isTargetWindows(MF) && RGI.isQuad()) {
std::swap(Reg1, Reg4);
std::swap(Reg2, Reg3);
std::swap(FrameIdxReg1, FrameIdxReg4);
std::swap(FrameIdxReg2, FrameIdxReg3);
}
- if ((RPI.isQuad() || RPI.isPaired()) && RPI.isScalable()) {
+ if ((RGI.isQuad() || RGI.isPaired()) && RGI.isScalable()) {
[[maybe_unused]] const AArch64Subtarget &Subtarget =
MF.getSubtarget<AArch64Subtarget>();
AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
@@ -2186,11 +2204,11 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
auto IsPPR = [](const RegGroupInfo &c) {
return c.Type == RegGroupInfo::PPR;
};
- auto PPRBegin = std::find_if(RegPairs.begin(), RegPairs.end(), IsPPR);
+ auto PPRBegin = std::find_if(RegGroups.begin(), RegGroups.end(), IsPPR);
auto IsZPR = [](const RegGroupInfo &c) {
return c.Type == RegGroupInfo::ZPR;
};
- auto ZPRBegin = std::find_if(RegPairs.begin(), RegPairs.end(), IsZPR);
+ auto ZPRBegin = std::find_if(RegGroups.begin(), RegGroups.end(), IsZPR);
assert(!(PPRBegin < ZPRBegin) &&
"Expected callee save predicate to be handled first");
#endif
@@ -2204,17 +2222,17 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
MBB.addLiveIn(Reg1);
if (!MRI.isReserved(Reg2))
MBB.addLiveIn(Reg2);
- if (RPI.isQuad()) {
+ if (RGI.isQuad()) {
if (!MRI.isReserved(Reg3))
MBB.addLiveIn(Reg3);
if (!MRI.isReserved(Reg4))
MBB.addLiveIn(Reg4);
}
- if (RPI.isPaired()) {
- MIB.addReg(/*PairRegs*/ AArch64::Z0_Z1 + (RPI.Reg1 - AArch64::Z0));
- } else if (RPI.isQuad()) {
+ if (RGI.isPaired()) {
+ MIB.addReg(/*PairRegs*/ AArch64::Z0_Z1 + (RGI.Reg1 - AArch64::Z0));
+ } else if (RGI.isQuad()) {
MIB.addReg(/*QuadRegs*/ AArch64::Z0_Z1_Z2_Z3 +
- (RPI.Reg1 - AArch64::Z0));
+ (RGI.Reg1 - AArch64::Z0));
MIB.addMemOperand(MF.getMachineMemOperand(
MachinePointerInfo::getFixedStack(MF, FrameIdxReg4),
MachineMemOperand::MOStore, Size, Alignment));
@@ -2227,8 +2245,8 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
MachineMemOperand::MOStore, Size, Alignment));
MIB.addReg(PnReg);
MIB.addReg(AArch64::SP)
- .addImm(RPI.Offset / // [sp, #imm*size*vscale]
- RPI.getNumRegs()) // where size*vscale is implicit
+ .addImm(RGI.Offset / // [sp, #imm*size*vscale]
+ RGI.getNumRegs()) // where size*vscale is implicit
.setMIFlag(MachineInstr::FrameSetup);
MIB.addMemOperand(MF.getMachineMemOperand(
MachinePointerInfo::getFixedStack(MF, FrameIdxReg1),
@@ -2239,7 +2257,7 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(StrOpc));
if (!MRI.isReserved(Reg1))
MBB.addLiveIn(Reg1);
- if (RPI.isPaired()) {
+ if (RGI.isPaired()) {
if (!MRI.isReserved(Reg2))
MBB.addLiveIn(Reg2);
MIB.addReg(Reg2, getPrologueDeath(MF, Reg2));
@@ -2249,7 +2267,7 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
}
MIB.addReg(Reg1, getPrologueDeath(MF, Reg1))
.addReg(AArch64::SP)
- .addImm(RPI.Offset) // [sp, #offset*vscale],
+ .addImm(RGI.Offset) // [sp, #offset*vscale],
// where factor*vscale is implicit
.setMIFlag(MachineInstr::FrameSetup);
MIB.addMemOperand(MF.getMachineMemOperand(
@@ -2260,19 +2278,19 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
}
// Update the StackIDs of the SVE stack slots.
MachineFrameInfo &MFI = MF.getFrameInfo();
- if (RPI.Type == RegGroupInfo::ZPR) {
+ if (RGI.Type == RegGroupInfo::ZPR) {
MFI.setStackID(FrameIdxReg1, TargetStackID::ScalableVector);
- if (RPI.isPaired() || RPI.isQuad())
+ if (RGI.isPaired() || RGI.isQuad())
MFI.setStackID(FrameIdxReg2, TargetStackID::ScalableVector);
- if (RPI.isQuad()) {
+ if (RGI.isQuad()) {
MFI.setStackID(FrameIdxReg3, TargetStackID::ScalableVector);
MFI.setStackID(FrameIdxReg4, TargetStackID::ScalableVector);
}
- } else if (RPI.Type == RegGroupInfo::PPR) {
+ } else if (RGI.Type == RegGroupInfo::PPR) {
MFI.setStackID(FrameIdxReg1, TargetStackID::ScalablePredicateVector);
- if (RPI.isPaired() || RPI.isQuad())
+ if (RGI.isPaired() || RGI.isQuad())
MFI.setStackID(FrameIdxReg2, TargetStackID::ScalablePredicateVector);
- if (RPI.isQuad()) {
+ if (RGI.isQuad()) {
MFI.setStackID(FrameIdxReg3, TargetStackID::ScalablePredicateVector);
MFI.setStackID(FrameIdxReg4, TargetStackID::ScalablePredicateVector);
}
@@ -2288,19 +2306,19 @@ bool AArch64FrameLowering::restoreCalleeSavedRegisters(
const AArch64InstrInfo &TII =
*MF.getSubtarget<AArch64Subtarget>().getInstrInfo();
DebugLoc DL;
- SmallVector<RegGroupInfo, 8> RegPairs;
+ SmallVector<RegGroupInfo, 8> RegGroups;
bool NeedsWinCFI = needsWinCFI(MF);
if (MBBI != MBB.end())
DL = MBBI->getDebugLoc();
- computeCalleeSaveRegisterPairs(*this, MF, CSI, TRI, RegPairs, hasFP(MF));
+ computeCalleeSaveRegisterPairs(*this, MF, CSI, TRI, RegGroups, hasFP(MF));
if (homogeneousPrologEpilog(MF, &MBB)) {
auto MIB = BuildMI(MBB, MBBI, DL, TII.get(AArch64::HOM_Epilog))
.setMIFlag(MachineInstr::FrameDestroy);
- for (auto &RPI : RegPairs) {
- MIB.addReg(RPI.Reg1, RegState::Define);
- MIB.addReg(RPI.Reg2, RegState::Define);
+ for (auto &RGI : RegGroups) {
+ MIB.addReg(RGI.Reg1, RegState::Define);
+ MIB.addReg(RGI.Reg2, RegState::Define);
}
return true;
}
@@ -2309,22 +2327,22 @@ bool AArch64FrameLowering::restoreCalleeSavedRegisters(
auto IsPPR = [](const RegGroupInfo &c) {
return c.Type == RegGroupInfo::PPR;
};
- auto PPRBegin = llvm::find_if(RegPairs, IsPPR);
- auto PPREnd = std::find_if_not(PPRBegin, RegPairs.end(), IsPPR);
+ auto PPRBegin = llvm::find_if(RegGroups, IsPPR);
+ auto PPREnd = std::find_if_not(PPRBegin, RegGroups.end(), IsPPR);
std::reverse(PPRBegin, PPREnd);
auto IsZPR = [](const RegGroupInfo &c) {
return c.Type == RegGroupInfo::ZPR;
};
- auto ZPRBegin = llvm::find_if(RegPairs, IsZPR);
- auto ZPREnd = std::find_if_not(ZPRBegin, RegPairs.end(), IsZPR);
+ auto ZPRBegin = llvm::find_if(RegGroups, IsZPR);
+ auto ZPREnd = std::find_if_not(ZPRBegin, RegGroups.end(), IsZPR);
std::reverse(ZPRBegin, ZPREnd);
bool PTrueCreated = false;
- for (const RegGroupInfo &RPI : RegPairs) {
- Register Reg1 = RPI.Reg1;
- Register Reg2 = RPI.Reg2;
- Register Reg3 = RPI.Reg3;
- Register Reg4 = RPI.Reg4;
+ for (const RegGroupInfo &RGI : RegGroups) {
+ Register Reg1 = RGI.Reg1;
+ Register Reg2 = RGI.Reg2;
+ Register Reg3 = RGI.Reg3;
+ Register Reg4 = RGI.Reg4;
// Issue sequence of restores for cs regs. The last restore may be converted
// to a post-increment load later by emitEpilogue if the callee-save stack
@@ -2335,22 +2353,22 @@ bool AArch64FrameLowering::restoreCalleeSavedRegisters(
// ldp x22, x21, [sp, #0] // addImm(+0)
// Note: see comment in spillCalleeSavedRegisters()
unsigned LdrOpc;
- unsigned Size = TRI->getSpillSize(*RPI.RC);
- Align Alignment = TRI->getSpillAlign(*RPI.RC);
- switch (RPI.Type) {
+ unsigned Size = TRI->getSpillSize(*RGI.RC);
+ Align Alignment = TRI->getSpillAlign(*RGI.RC);
+ switch (RGI.Type) {
case RegGroupInfo::GPR:
- LdrOpc = RPI.isPaired() ? AArch64::LDPXi : AArch64::LDRXui;
+ LdrOpc = RGI.isPaired() ? AArch64::LDPXi : AArch64::LDRXui;
break;
case RegGroupInfo::FPR64:
- LdrOpc = RPI.isPaired() ? AArch64::LDPDi : AArch64::LDRDui;
+ LdrOpc = RGI.isPaired() ? AArch64::LDPDi : AArch64::LDRDui;
break;
case RegGroupInfo::FPR128:
- LdrOpc = RPI.isPaired() ? AArch64::LDPQi : AArch64::LDRQui;
+ LdrOpc = RGI.isPaired() ? AArch64::LDPQi : AArch64::LDRQui;
break;
case RegGroupInfo::ZPR:
- LdrOpc = RPI.isQuad()
+ LdrOpc = RGI.isQuad()
? AArch64::LD1B_4Z_IMM
- : (RPI.isPaired() ? AArch64::LD1B_2Z_IMM : AArch64::LDR_ZXI);
+ : (RGI.isPaired() ? AArch64::LD1B_2Z_IMM : AArch64::LDR_ZXI);
break;
case RegGroupInfo::PPR:
LdrOpc = AArch64::LDR_PXI;
@@ -2360,18 +2378,18 @@ bool AArch64FrameLowering::restoreCalleeSavedRegisters(
}
LLVM_DEBUG({
dbgs() << "CSR restore: (" << printReg(Reg1, TRI);
- if (RPI.isPaired() || RPI.isQuad())
+ if (RGI.isPaired() || RGI.isQuad())
dbgs() << ", " << printReg(Reg2, TRI);
- if (RPI.isQuad()) {
+ if (RGI.isQuad()) {
dbgs() << ", " << printReg(Reg3, TRI);
dbgs() << ", " << printReg(Reg4, TRI);
}
- dbgs() << ") -> fi#(" << RPI.FrameIdx;
- if (RPI.isPaired() || RPI.isQuad())
- dbgs() << ", " << RPI.FrameIdx + 1;
- if (RPI.isQuad()) {
- dbgs() << ", " << RPI.FrameIdx + 2;
- dbgs() << ", " << RPI.FrameIdx + 3;
+ dbgs() << ") -> fi#(" << RGI.FrameIdx;
+ if (RGI.isPaired() || RGI.isQuad())
+ dbgs() << ", " << RGI.FrameIdx + 1;
+ if (RGI.isQuad()) {
+ dbgs() << ", " << RGI.FrameIdx + 2;
+ dbgs() << ", " << RGI.FrameIdx + 3;
}
dbgs() << ")\n";
});
@@ -2379,16 +2397,16 @@ bool AArch64FrameLowering::restoreCalleeSavedRegisters(
// Windows unwind codes require consecutive registers if registers are
// paired. Make the switch here, so that the code below will save (x,x+1)
// and not (x+1,x).
- unsigned FrameIdxReg1 = RPI.FrameIdx;
- unsigned FrameIdxReg2 = RPI.FrameIdx + 1;
- unsigned FrameIdxReg3 = RPI.FrameIdx + 2;
- unsigned FrameIdxReg4 = RPI.FrameIdx + 3;
+ unsigned FrameIdxReg1 = RGI.FrameIdx;
+ unsigned FrameIdxReg2 = RGI.FrameIdx + 1;
+ unsigned FrameIdxReg3 = RGI.FrameIdx + 2;
+ unsigned FrameIdxReg4 = RGI.FrameIdx + 3;
if (isTargetWindows(MF)) {
- if (RPI.isPaired()) {
+ if (RGI.isPaired()) {
std::swap(Reg1, Reg2);
std::swap(FrameIdxReg1, FrameIdxReg2);
- } else if (RPI.isQuad()) {
+ } else if (RGI.isQuad()) {
std::swap(Reg1, Reg4);
std::swap(Reg2, Reg3);
std::swap(FrameIdxReg1, FrameIdxReg4);
@@ -2397,7 +2415,7 @@ bool AArch64FrameLowering::restoreCalleeSavedRegisters(
}
AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
- if ((RPI.isQuad() || RPI.isPaired()) && RPI.isScalable()) {
+ if ((RGI.isQuad() || RGI.isPaired()) && RGI.isScalable()) {
[[maybe_unused]] const AArch64Subtarget &Subtarget =
MF.getSubtarget<AArch64Subtarget>();
unsigned PnReg = AFI->getPredicateRegForFillSpill();
@@ -2413,11 +2431,11 @@ bool AArch64FrameLowering::restoreCalleeSavedRegisters(
.setMIFlags(MachineInstr::FrameDestroy);
}
MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII.get(LdrOpc));
- if (RPI.isPaired()) {
- MIB.addReg(/*PairRegs*/ AArch64::Z0_Z1 + (RPI.Reg1 - AArch64::Z0),
+ if (RGI.isPaired()) {
+ MIB.addReg(/*PairRegs*/ AArch64::Z0_Z1 + (RGI.Reg1 - AArch64::Z0),
getDefRegState(true));
- } else if (RPI.isQuad()) {
- MIB.addReg(/*QuadRegs*/ AArch64::Z0_Z1_Z2_Z3 + (RPI.Reg1 - AArch64::Z0),
+ } else if (RGI.isQuad()) {
+ MIB.addReg(/*QuadRegs*/ AArch64::Z0_Z1_Z2_Z3 + (RGI.Reg1 - AArch64::Z0),
getDefRegState(true));
MIB.addMemOperand(MF.getMachineMemOperand(
MachinePointerInfo::getFixedStack(MF, FrameIdxReg4),
@@ -2431,8 +2449,8 @@ bool AArch64FrameLowering::restoreCalleeSavedRegisters(
MachineMemOperand::MOLoad, Size, Alignment));
MIB.addReg(PnReg);
MIB.addReg(AArch64::SP)
- .addImm(RPI.Offset / // [sp, #imm*size*vscale]
- RPI.getNumRegs()) // where size*vscale is implicit
+ .addImm(RGI.Offset / // [sp, #imm*size*vscale]
+ RGI.getNumRegs()) // where size*vscale is implicit
.setMIFlag(MachineInstr::FrameDestroy);
MIB.addMemOperand(MF.getMachineMemOperand(
MachinePointerInfo::getFixedStack(MF, FrameIdxReg1),
@@ -2441,7 +2459,7 @@ bool AArch64FrameLowering::restoreCalleeSavedRegisters(
insertSEH(MIB, TII, MachineInstr::FrameDestroy);
} else {
MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII.get(LdrOpc));
- if (RPI.isPaired()) {
+ if (RGI.isPaired()) {
MIB.addReg(Reg2, getDefRegState(true));
MIB.addMemOperand(MF.getMachineMemOperand(
MachinePointerInfo::getFixedStack(MF, FrameIdxReg2),
@@ -2449,7 +2467,7 @@ bool AArch64FrameLowering::restoreCalleeSavedRegisters(
}
MIB.addReg(Reg1, getDefRegState(true));
MIB.addReg(AArch64::SP)
- .addImm(RPI.Offset) // [sp, #offset*vscale]
+ .addImm(RGI.Offset) // [sp, #offset*vscale]
// where factor*vscale is implicit
.setMIFlag(MachineInstr::FrameDestroy);
MIB.addMemOperand(MF.getMachineMemOperand(
>From 5770645b66996b3b0bb141fbfc9afcdd55e329db Mon Sep 17 00:00:00 2001
From: Kieran Bailey <kieran.bailey at arm.com>
Date: Wed, 23 Sep 2026 14:47:15 +0000
Subject: [PATCH 3/4] Removed reordering of scalable reg groups on windows
Refactored code further
---
.../Target/AArch64/AArch64FrameLowering.cpp | 88 ++++++++-----------
.../win-nounwind-callee-save-restore-pairs.ll | 54 ++++++++++++
2 files changed, 90 insertions(+), 52 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/win-nounwind-callee-save-restore-pairs.ll
diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
index 3fd49d1e917eb..6afd7df16b187 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
@@ -1670,6 +1670,8 @@ struct RegGroupInfo {
return Reg2.isValid() && Reg3.isValid() && Reg4.isValid();
}
+ bool isGrouped() const { return isPaired() || isQuad(); }
+
unsigned getNumRegs() const {
if (isQuad())
return 4;
@@ -1901,17 +1903,13 @@ void computeCalleeSaveRegisterPairs(const AArch64FrameLowering &AFL,
//
// The order of the registers in the list is controlled by
// getCalleeSavedRegs(), so they will always be in-order, as well.
- assert((!RGI.isPaired() ||
- (CSI[i].getFrameIdx() + RegInc == CSI[i + RegInc].getFrameIdx())) &&
- "Out of order callee saved regs!");
- assert((!RGI.isQuad() ||
- ((CSI[i].getFrameIdx() + RegInc == CSI[i + RegInc].getFrameIdx()) &&
- (CSI[i + RegInc].getFrameIdx() + RegInc ==
- CSI[i + RegInc * 2].getFrameIdx()) &&
- (CSI[i + RegInc * 2].getFrameIdx() + RegInc ==
- CSI[i + RegInc * 3].getFrameIdx()))) &&
- "Out of order callee saved regs!");
+ bool RegsOrdered = all_of(seq(RGI.getNumRegs() - 1), [&](int RegOffset) {
+ return CSI[i + RegInc * RegOffset].getFrameIdx() + RegInc ==
+ CSI[i + RegInc * RegOffset + RegInc].getFrameIdx();
+ });
+ assert(RegsOrdered && "Out of order callee saved regs!");
+ (void)RegsOrdered;
assert((!RGI.isQuad() || RGI.Type == RegGroupInfo::ZPR) &&
"Currently only ZPR's support four-register spills");
@@ -1968,7 +1966,7 @@ void computeCalleeSaveRegisterPairs(const AArch64FrameLowering &AFL,
// Round up size of non-pair to pair size if we need to pad the
// callee-save area to ensure 16-byte alignment.
if (NeedGapToAlignStack && !IsWindows && !RGI.isScalable() &&
- RGI.Type != RegGroupInfo::FPR128 && !RGI.isPaired() &&
+ RGI.Type != RegGroupInfo::FPR128 && !RGI.isGrouped() &&
ByteOffset % 16 != 0) {
ByteOffset += 8 * StackFillDir;
assert(MFI.getObjectAlign(RGI.FrameIdx) <= Align(16));
@@ -2156,14 +2154,14 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
LLVM_DEBUG({
dbgs() << "CSR spill: (" << printReg(Reg1, TRI);
- if (RGI.isPaired() || RGI.isQuad())
+ if (RGI.isGrouped())
dbgs() << ", " << printReg(Reg2, TRI);
if (RGI.isQuad()) {
dbgs() << ", " << printReg(Reg3, TRI);
dbgs() << ", " << printReg(Reg4, TRI);
}
dbgs() << ") -> fi#(" << RGI.FrameIdx;
- if (RGI.isPaired() || RGI.isQuad())
+ if (RGI.isGrouped())
dbgs() << ", " << RGI.FrameIdx + 1;
if (RGI.isQuad()) {
dbgs() << ", " << RGI.FrameIdx + 2;
@@ -2175,25 +2173,14 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
assert((!isTargetWindows(MF) ||
!(Reg1 == AArch64::LR && Reg2 == AArch64::FP)) &&
"Windows unwdinding requires a consecutive (FP,LR) pair");
- // Windows unwind codes require consecutive registers if registers are
- // paired. Make the switch here, so that the code below will save (x,x+1)
- // and not (x+1,x).
unsigned FrameIdxReg1 = RGI.FrameIdx;
unsigned FrameIdxReg2 = RGI.FrameIdx + 1;
unsigned FrameIdxReg3 = RGI.FrameIdx + 2;
unsigned FrameIdxReg4 = RGI.FrameIdx + 3;
- if (isTargetWindows(MF) && RGI.isPaired()) {
- std::swap(Reg1, Reg2);
- std::swap(FrameIdxReg1, FrameIdxReg2);
- } else if (isTargetWindows(MF) && RGI.isQuad()) {
- std::swap(Reg1, Reg4);
- std::swap(Reg2, Reg3);
- std::swap(FrameIdxReg1, FrameIdxReg4);
- std::swap(FrameIdxReg2, FrameIdxReg3);
- }
-
- if ((RGI.isQuad() || RGI.isPaired()) && RGI.isScalable()) {
+ if ((RGI.isGrouped()) && RGI.isScalable()) {
+ assert(!NeedsWinCFI &&
+ "Scalable register groups are not supported by Windows WinCFI");
[[maybe_unused]] const AArch64Subtarget &Subtarget =
MF.getSubtarget<AArch64Subtarget>();
AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
@@ -2251,9 +2238,14 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
MIB.addMemOperand(MF.getMachineMemOperand(
MachinePointerInfo::getFixedStack(MF, FrameIdxReg1),
MachineMemOperand::MOStore, Size, Alignment));
- if (NeedsWinCFI)
- insertSEH(MIB, TII, MachineInstr::FrameSetup);
- } else { // The code when the pair of ZReg is not present
+ } else {
+ // Windows unwind codes require consecutive registers if registers are
+ // paired. Make the switch here, so that the code below will save (x,x+1)
+ // and not (x+1,x).
+ if (isTargetWindows(MF) && RGI.isPaired()) {
+ std::swap(Reg1, Reg2);
+ std::swap(FrameIdxReg1, FrameIdxReg2);
+ }
MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(StrOpc));
if (!MRI.isReserved(Reg1))
MBB.addLiveIn(Reg1);
@@ -2280,7 +2272,7 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
MachineFrameInfo &MFI = MF.getFrameInfo();
if (RGI.Type == RegGroupInfo::ZPR) {
MFI.setStackID(FrameIdxReg1, TargetStackID::ScalableVector);
- if (RGI.isPaired() || RGI.isQuad())
+ if (RGI.isGrouped())
MFI.setStackID(FrameIdxReg2, TargetStackID::ScalableVector);
if (RGI.isQuad()) {
MFI.setStackID(FrameIdxReg3, TargetStackID::ScalableVector);
@@ -2288,7 +2280,7 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
}
} else if (RGI.Type == RegGroupInfo::PPR) {
MFI.setStackID(FrameIdxReg1, TargetStackID::ScalablePredicateVector);
- if (RGI.isPaired() || RGI.isQuad())
+ if (RGI.isGrouped())
MFI.setStackID(FrameIdxReg2, TargetStackID::ScalablePredicateVector);
if (RGI.isQuad()) {
MFI.setStackID(FrameIdxReg3, TargetStackID::ScalablePredicateVector);
@@ -2378,14 +2370,14 @@ bool AArch64FrameLowering::restoreCalleeSavedRegisters(
}
LLVM_DEBUG({
dbgs() << "CSR restore: (" << printReg(Reg1, TRI);
- if (RGI.isPaired() || RGI.isQuad())
+ if (RGI.isGrouped())
dbgs() << ", " << printReg(Reg2, TRI);
if (RGI.isQuad()) {
dbgs() << ", " << printReg(Reg3, TRI);
dbgs() << ", " << printReg(Reg4, TRI);
}
dbgs() << ") -> fi#(" << RGI.FrameIdx;
- if (RGI.isPaired() || RGI.isQuad())
+ if (RGI.isGrouped())
dbgs() << ", " << RGI.FrameIdx + 1;
if (RGI.isQuad()) {
dbgs() << ", " << RGI.FrameIdx + 2;
@@ -2394,28 +2386,15 @@ bool AArch64FrameLowering::restoreCalleeSavedRegisters(
dbgs() << ")\n";
});
- // Windows unwind codes require consecutive registers if registers are
- // paired. Make the switch here, so that the code below will save (x,x+1)
- // and not (x+1,x).
unsigned FrameIdxReg1 = RGI.FrameIdx;
unsigned FrameIdxReg2 = RGI.FrameIdx + 1;
unsigned FrameIdxReg3 = RGI.FrameIdx + 2;
unsigned FrameIdxReg4 = RGI.FrameIdx + 3;
- if (isTargetWindows(MF)) {
- if (RGI.isPaired()) {
- std::swap(Reg1, Reg2);
- std::swap(FrameIdxReg1, FrameIdxReg2);
- } else if (RGI.isQuad()) {
- std::swap(Reg1, Reg4);
- std::swap(Reg2, Reg3);
- std::swap(FrameIdxReg1, FrameIdxReg4);
- std::swap(FrameIdxReg2, FrameIdxReg3);
- }
- }
-
AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
- if ((RGI.isQuad() || RGI.isPaired()) && RGI.isScalable()) {
+ if ((RGI.isGrouped()) && RGI.isScalable()) {
+ assert(!NeedsWinCFI &&
+ "Scalable register groups are not supported by Windows WinCFI");
[[maybe_unused]] const AArch64Subtarget &Subtarget =
MF.getSubtarget<AArch64Subtarget>();
unsigned PnReg = AFI->getPredicateRegForFillSpill();
@@ -2455,9 +2434,14 @@ bool AArch64FrameLowering::restoreCalleeSavedRegisters(
MIB.addMemOperand(MF.getMachineMemOperand(
MachinePointerInfo::getFixedStack(MF, FrameIdxReg1),
MachineMemOperand::MOLoad, Size, Alignment));
- if (NeedsWinCFI)
- insertSEH(MIB, TII, MachineInstr::FrameDestroy);
} else {
+ // Windows unwind codes require consecutive registers if registers are
+ // paired. Make the switch here, so that the code below will save (x,x+1)
+ // and not (x+1,x).
+ if (isTargetWindows(MF) && RGI.isPaired()) {
+ std::swap(Reg1, Reg2);
+ std::swap(FrameIdxReg1, FrameIdxReg2);
+ }
MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII.get(LdrOpc));
if (RGI.isPaired()) {
MIB.addReg(Reg2, getDefRegState(true));
diff --git a/llvm/test/CodeGen/AArch64/win-nounwind-callee-save-restore-pairs.ll b/llvm/test/CodeGen/AArch64/win-nounwind-callee-save-restore-pairs.ll
new file mode 100644
index 0000000000000..d2736c6273a67
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/win-nounwind-callee-save-restore-pairs.ll
@@ -0,0 +1,54 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=aarch64-pc-windows-msvc -mattr=+sve2p1 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK
+
+define aarch64_sve_vector_pcs void @pair() nounwind {
+; CHECK-LABEL: pair:
+; CHECK: // %bb.0:
+; CHECK-NEXT: addvl sp, sp, #-5
+; CHECK-NEXT: str p8, [sp] // 2-byte Spill
+; CHECK-NEXT: ptrue pn8.b
+; CHECK-NEXT: str z9, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: st1b { z10.b, z11.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; CHECK-NEXT: str z12, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: //APP
+; CHECK-NEXT: //NO_APP
+; CHECK-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ptrue pn8.b
+; CHECK-NEXT: ldr z9, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z12, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ld1b { z10.b, z11.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; CHECK-NEXT: ldr p8, [sp] // 2-byte Reload
+; CHECK-NEXT: addvl sp, sp, #5
+; CHECK-NEXT: ret
+ call void asm sideeffect "", "~{z9},~{z10},~{z12},~{z13}"()
+ ret void
+}
+
+
+define aarch64_sve_vector_pcs void @quad() nounwind {
+; CHECK-LABEL: quad:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: addvl sp, sp, #-8
+; CHECK-NEXT: str p8, [sp] // 2-byte Spill
+; CHECK-NEXT: ptrue pn8.b
+; CHECK-NEXT: str z8, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: st1b { z12.b - z15.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
+; CHECK-NEXT: str z9, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z10, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: //APP
+; CHECK-NEXT: //NO_APP
+; CHECK-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ptrue pn8.b
+; CHECK-NEXT: ldr z8, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z9, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z10, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ld1b { z12.b - z15.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
+; CHECK-NEXT: ldr p8, [sp] // 2-byte Reload
+; CHECK-NEXT: addvl sp, sp, #8
+; CHECK-NEXT: ret
+entry:
+ call void asm sideeffect "", "~{z8},~{z12},~{z13},~{z14},~{z15}"()
+ ret void
+}
>From 7e1e7b914b5268bb9a61d89ff55a2e39a37fbe87 Mon Sep 17 00:00:00 2001
From: Kieran Bailey <kieran.bailey at arm.com>
Date: Thu, 24 Sep 2026 09:12:05 +0000
Subject: [PATCH 4/4] Added review refactor suggestions
Fixed stale windows test checks, I can confirm the extra z11 in the previous test checks was stale.
---
.../Target/AArch64/AArch64FrameLowering.cpp | 26 ++++++-------------
.../win-nounwind-callee-save-restore-pairs.ll | 17 +++++-------
2 files changed, 14 insertions(+), 29 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
index 6afd7df16b187..443115c515a18 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
@@ -2178,7 +2178,7 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
unsigned FrameIdxReg3 = RGI.FrameIdx + 2;
unsigned FrameIdxReg4 = RGI.FrameIdx + 3;
- if ((RGI.isGrouped()) && RGI.isScalable()) {
+ if (RGI.isGrouped() && RGI.isScalable()) {
assert(!NeedsWinCFI &&
"Scalable register groups are not supported by Windows WinCFI");
[[maybe_unused]] const AArch64Subtarget &Subtarget =
@@ -2270,22 +2270,12 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
}
// Update the StackIDs of the SVE stack slots.
MachineFrameInfo &MFI = MF.getFrameInfo();
- if (RGI.Type == RegGroupInfo::ZPR) {
- MFI.setStackID(FrameIdxReg1, TargetStackID::ScalableVector);
- if (RGI.isGrouped())
- MFI.setStackID(FrameIdxReg2, TargetStackID::ScalableVector);
- if (RGI.isQuad()) {
- MFI.setStackID(FrameIdxReg3, TargetStackID::ScalableVector);
- MFI.setStackID(FrameIdxReg4, TargetStackID::ScalableVector);
- }
- } else if (RGI.Type == RegGroupInfo::PPR) {
- MFI.setStackID(FrameIdxReg1, TargetStackID::ScalablePredicateVector);
- if (RGI.isGrouped())
- MFI.setStackID(FrameIdxReg2, TargetStackID::ScalablePredicateVector);
- if (RGI.isQuad()) {
- MFI.setStackID(FrameIdxReg3, TargetStackID::ScalablePredicateVector);
- MFI.setStackID(FrameIdxReg4, TargetStackID::ScalablePredicateVector);
- }
+ for (unsigned I = 0; I < RGI.getNumRegs(); ++I) {
+ if (RGI.Type == RegGroupInfo::ZPR)
+ MFI.setStackID(RGI.FrameIdx + I, TargetStackID::ScalableVector);
+ else if (RGI.Type == RegGroupInfo::PPR)
+ MFI.setStackID(RGI.FrameIdx + I,
+ TargetStackID::ScalablePredicateVector);
}
}
return true;
@@ -2392,7 +2382,7 @@ bool AArch64FrameLowering::restoreCalleeSavedRegisters(
unsigned FrameIdxReg4 = RGI.FrameIdx + 3;
AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
- if ((RGI.isGrouped()) && RGI.isScalable()) {
+ if (RGI.isGrouped() && RGI.isScalable()) {
assert(!NeedsWinCFI &&
"Scalable register groups are not supported by Windows WinCFI");
[[maybe_unused]] const AArch64Subtarget &Subtarget =
diff --git a/llvm/test/CodeGen/AArch64/win-nounwind-callee-save-restore-pairs.ll b/llvm/test/CodeGen/AArch64/win-nounwind-callee-save-restore-pairs.ll
index d2736c6273a67..b6d18a0fe0501 100644
--- a/llvm/test/CodeGen/AArch64/win-nounwind-callee-save-restore-pairs.ll
+++ b/llvm/test/CodeGen/AArch64/win-nounwind-callee-save-restore-pairs.ll
@@ -4,28 +4,25 @@
define aarch64_sve_vector_pcs void @pair() nounwind {
; CHECK-LABEL: pair:
; CHECK: // %bb.0:
-; CHECK-NEXT: addvl sp, sp, #-5
+; CHECK-NEXT: addvl sp, sp, #-4
; CHECK-NEXT: str p8, [sp] // 2-byte Spill
; CHECK-NEXT: ptrue pn8.b
; CHECK-NEXT: str z9, [sp, #1, mul vl] // 16-byte Folded Spill
; CHECK-NEXT: st1b { z10.b, z11.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
-; CHECK-NEXT: str z12, [sp, #4, mul vl] // 16-byte Folded Spill
; CHECK-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
; CHECK-NEXT: //APP
; CHECK-NEXT: //NO_APP
; CHECK-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ptrue pn8.b
; CHECK-NEXT: ldr z9, [sp, #1, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT: ldr z12, [sp, #4, mul vl] // 16-byte Folded Reload
; CHECK-NEXT: ld1b { z10.b, z11.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
; CHECK-NEXT: ldr p8, [sp] // 2-byte Reload
-; CHECK-NEXT: addvl sp, sp, #5
+; CHECK-NEXT: addvl sp, sp, #4
; CHECK-NEXT: ret
- call void asm sideeffect "", "~{z9},~{z10},~{z12},~{z13}"()
+ call void asm sideeffect "", "~{z9},~{z10},~{z11}"()
ret void
}
-
define aarch64_sve_vector_pcs void @quad() nounwind {
; CHECK-LABEL: quad:
; CHECK: // %bb.0: // %entry
@@ -33,22 +30,20 @@ define aarch64_sve_vector_pcs void @quad() nounwind {
; CHECK-NEXT: str p8, [sp] // 2-byte Spill
; CHECK-NEXT: ptrue pn8.b
; CHECK-NEXT: str z8, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: st1b { z10.b, z11.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
; CHECK-NEXT: st1b { z12.b - z15.b }, pn8, [sp, #4, mul vl] // 64-byte Folded Spill
-; CHECK-NEXT: str z9, [sp, #2, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT: str z10, [sp, #3, mul vl] // 16-byte Folded Spill
; CHECK-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
; CHECK-NEXT: //APP
; CHECK-NEXT: //NO_APP
; CHECK-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ptrue pn8.b
; CHECK-NEXT: ldr z8, [sp, #1, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT: ldr z9, [sp, #2, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT: ldr z10, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ld1b { z10.b, z11.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
; CHECK-NEXT: ld1b { z12.b - z15.b }, pn8/z, [sp, #4, mul vl] // 64-byte Folded Reload
; CHECK-NEXT: ldr p8, [sp] // 2-byte Reload
; CHECK-NEXT: addvl sp, sp, #8
; CHECK-NEXT: ret
entry:
- call void asm sideeffect "", "~{z8},~{z12},~{z13},~{z14},~{z15}"()
+ call void asm sideeffect "", "~{z8},~{z10},~{z11},~{z12},~{z13},~{z14},~{z15}"()
ret void
}
More information about the llvm-commits
mailing list