[llvm] [AArch64LoadStoreOptimizer] Pair far-offset loads/stores via base-register adjustment (PR #223684)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 21 00:48:46 PDT 2026
https://github.com/whokeke updated https://github.com/llvm/llvm-project/pull/223684
>From a92093d2c4fc8247dec7e95e947a66d42d5177df Mon Sep 17 00:00:00 2001
From: whoiskk <hukeke2 at huawei.com>
Date: Mon, 7 Sep 2026 19:41:07 +0800
Subject: [PATCH] [AArch64LoadStoreOptimizer] Pair far-offset loads/stores via
base-register adjustment
LDP/STP take a 7-bit signed scaled immediate, so adjacent same-base
scaled loads/stores with offsets outside [-64, +63] cannot pair and
stay as two independent LDR/STR.
When the pair offset is out of range, insert an ADDXri to materialize
a scratch base and emit the pair off it with offset 0. Gated by
-aarch64-ldp-stp-base-adjust (default on), restricted to scaled
accesses and loop-invariant bases: base-adjust is net negative for
loop-variant bases, where the per-pair ADDXri cannot be hoisted and
LDP cannot overlap like independent LDRs.
For GPR64 loads the first destination register is reused as the
adjusted base (dead-by-construction, and 'ldp Rt1, Rt2, [Rt1]' is
legal on AArch64). Otherwise the scratch is scavenged from
caller-saved X9..X15, excluding X16/X17 (IP0/IP1), which the linker
may clobber via veneers/PLT entries, and using regsOverlap to reject
candidates aliasing Wn value registers. Scavenging failure is a
missed optimization, not a miscompile: the accesses are left unpaired.
Test: ldp-far-offset.ll, stp-far-offset.ll
---
.../AArch64/AArch64LoadStoreOptimizer.cpp | 228 +++++-
.../GlobalISel/split-offsets-for-stp.ll | 12 +-
llvm/test/CodeGen/AArch64/O3-pipeline.ll | 4 +
llvm/test/CodeGen/AArch64/aarch64-gep-opt.ll | 5 +-
.../AArch64/aarch64-ldst-opt-instr-ref.mir | 6 +
.../AArch64/aarch64-sve-fill-spill-pair.ll | 71 +-
llvm/test/CodeGen/AArch64/ldp-far-offset.ll | 291 +++++++
llvm/test/CodeGen/AArch64/ldst-opt.ll | 12 +-
.../CodeGen/AArch64/preserve-all-large-csr.ll | 12 +-
.../CodeGen/AArch64/sme-streaming-checkvl.ll | 74 +-
llvm/test/CodeGen/AArch64/stack-hazard.ll | 340 ++++-----
llvm/test/CodeGen/AArch64/stp-far-offset.ll | 176 +++++
...e-streaming-mode-fixed-length-fp-to-int.ll | 16 +-
...streaming-mode-fixed-length-int-extends.ll | 86 +--
...e-streaming-mode-fixed-length-int-to-fp.ll | 6 +-
...ing-mode-fixed-length-masked-expandload.ll | 38 +-
...streaming-mode-fixed-length-masked-load.ll | 38 +-
.../sve-streaming-mode-fixed-length-trunc.ll | 707 +++++++++---------
18 files changed, 1440 insertions(+), 682 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/ldp-far-offset.ll
create mode 100644 llvm/test/CodeGen/AArch64/stp-far-offset.ll
diff --git a/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp b/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
index 289552a8d9411..2f52fc0e80a24 100644
--- a/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
@@ -21,6 +21,7 @@
#include "AArch64MachineFunctionInfo.h"
#include "AArch64Subtarget.h"
#include "MCTargetDesc/AArch64AddressingModes.h"
+#include "llvm/ADT/BitVector.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
@@ -32,8 +33,10 @@
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/RegisterScavenging.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/MC/MCAsmInfo.h"
@@ -67,6 +70,10 @@ STATISTIC(NumConstOffsetFolded,
"Number of const offset of index address folded");
STATISTIC(NumUMOVFoldedToFPRStore,
"Number of UMOV + GPR stores folded to FPR stores");
+STATISTIC(NumBaseAdjustLdpCreated,
+ "Number of LDP created with base register adjustment");
+STATISTIC(NumBaseAdjustStpCreated,
+ "Number of STP created with base register adjustment");
DEBUG_COUNTER(RegRenamingCounter, DEBUG_TYPE "-reg-renaming",
"Controls which pairs are considered for renaming");
@@ -94,6 +101,12 @@ static cl::opt<unsigned> UMOVFoldLimit("aarch64-umov-fold-scan-limit",
static cl::opt<bool> EnableRenaming("aarch64-load-store-renaming",
cl::init(true), cl::Hidden);
+// Allow LDP/STP pairing for far-offset scaled loads/stores by inserting an
+// ADDXri to adjust the base register. Restricted to scaled accesses with
+// loop-invariant bases.
+static cl::opt<bool> EnableLdpStpBaseAdjust("aarch64-ldp-stp-base-adjust",
+ cl::init(true), cl::Hidden);
+
#define AARCH64_LOAD_STORE_OPT_NAME "AArch64 load / store optimization pass"
namespace {
@@ -115,6 +128,14 @@ using LdStPairFlags = struct LdStPairFlags {
// forward.
std::optional<MCPhysReg> RenameReg;
+ // If true, the LDP/STP offset is out of range and we need to insert an
+ // ADDXri to compute an adjusted base register before the pair.
+ bool RequiresBaseAdjust = false;
+ // Byte offset added to the base register (encoded in ADDXri).
+ int64_t BaseAdjustByteOffset = 0;
+ // Scratch register used as the adjusted base for the pair.
+ Register BaseAdjustScratchReg = AArch64::NoRegister;
+
LdStPairFlags() = default;
void setMergeForward(bool V = true) { MergeForward = V; }
@@ -126,13 +147,43 @@ using LdStPairFlags = struct LdStPairFlags {
void setRenameReg(MCPhysReg R) { RenameReg = R; }
void clearRenameReg() { RenameReg = std::nullopt; }
std::optional<MCPhysReg> getRenameReg() const { return RenameReg; }
+
+ void setRequiresBaseAdjust(bool V = true) { RequiresBaseAdjust = V; }
+ bool getRequiresBaseAdjust() const { return RequiresBaseAdjust; }
+ void setBaseAdjustByteOffset(int64_t V) { BaseAdjustByteOffset = V; }
+ int64_t getBaseAdjustByteOffset() const { return BaseAdjustByteOffset; }
+ void setBaseAdjustScratchReg(Register R) { BaseAdjustScratchReg = R; }
+ Register getBaseAdjustScratchReg() const { return BaseAdjustScratchReg; }
};
+// Check whether Val can be encoded as the immediate operand of an ADDXri
+// instruction (12-bit unsigned immediate with shift 0 or 12).
+static bool canEncodeAddXriImm(int64_t Val) {
+ if (Val < 0)
+ return false;
+ if (Val <= 0xFFF)
+ return true;
+ return (Val & 0xFFF) == 0 && (Val >> 12) <= 0xFFF;
+}
+
+// Byte scale of a pairable load/store for base-adjust purposes. SVE
+// fills/spills (LDR_ZXI/STR_ZXI) pair into Q-register LDP/STP under the
+// VLS-128 guard in isCandidateToMergeOrPair, where one vl unit is exactly
+// 16 bytes; getMemScale itself has no case for them and would assert.
+static int getBaseAdjustMemScale(const AArch64InstrInfo *TII,
+ const MachineInstr &MI) {
+ unsigned Opc = MI.getOpcode();
+ if (Opc == AArch64::LDR_ZXI || Opc == AArch64::STR_ZXI)
+ return 16;
+ return TII->getMemScale(MI);
+}
+
struct AArch64LoadStoreOpt {
AliasAnalysis *AA;
const AArch64InstrInfo *TII;
const TargetRegisterInfo *TRI;
const AArch64Subtarget *Subtarget;
+ MachineLoopInfo *MLI = nullptr;
// Track which register units have been modified and used.
LiveRegUnits ModifiedRegUnits, UsedRegUnits;
@@ -229,6 +280,11 @@ struct AArch64LoadStoreOpt {
// Replace a UMOV (lane 0) + GPR store with a direct FPR sub-register store.
bool tryToReplaceUMOVStore(MachineBasicBlock::iterator &MBBI);
+ // Returns true if the base register of MI is not modified anywhere in the
+ // enclosing loop (or MI is not in a loop). LSO runs post-RA with no SSA
+ // def-use chain, so scan the loop body for writes to the base physreg.
+ bool isBaseLoopInvariant(const MachineInstr &MI);
+
bool optimizeBlock(MachineBasicBlock &MBB, bool EnableNarrowZeroStOpt);
bool runOnMachineFunction(MachineFunction &MF);
@@ -243,6 +299,8 @@ struct AArch64LoadStoreOptLegacy : public MachineFunctionPass {
void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.addRequired<AAResultsWrapperPass>();
+ AU.addRequired<MachineLoopInfoWrapperPass>();
+ AU.addPreserved<MachineLoopInfoWrapperPass>();
MachineFunctionPass::getAnalysisUsage(AU);
}
@@ -1146,6 +1204,29 @@ AArch64LoadStoreOpt::mergePairedInsns(MachineBasicBlock::iterator I,
#endif
}
+ // --- Base register adjustment for far-offset LDP/STP ---
+ if (Flags.getRequiresBaseAdjust()) {
+ Register Scratch = Flags.getBaseAdjustScratchReg();
+ int64_t ByteOffset = Flags.getBaseAdjustByteOffset();
+ unsigned Imm12, Shift;
+ if (ByteOffset <= 0xFFF) {
+ Imm12 = ByteOffset;
+ Shift = 0;
+ } else {
+ Imm12 = ByteOffset >> 12;
+ Shift = 12;
+ }
+ MachineBasicBlock *AdjMBB = I->getParent();
+ DebugLoc AdjDL = I->getDebugLoc();
+ Register BaseReg = AArch64InstrInfo::getLdStBaseOp(*I).getReg();
+ BuildMI(*AdjMBB, I, AdjDL, TII->get(AArch64::ADDXri))
+ .addDef(Scratch)
+ .addUse(BaseReg)
+ .addImm(Imm12)
+ .addImm(Shift);
+ MergeForward = false;
+ }
+
// Insert our new paired instruction after whichever of the paired
// instructions MergeForward indicates.
MachineBasicBlock::iterator InsertionPoint = MergeForward ? Paired : I;
@@ -1154,6 +1235,17 @@ AArch64LoadStoreOpt::mergePairedInsns(MachineBasicBlock::iterator I,
const MachineOperand &BaseRegOp =
MergeForward ? AArch64InstrInfo::getLdStBaseOp(*Paired)
: AArch64InstrInfo::getLdStBaseOp(*I);
+ // If base adjustment is used, override the base register to the scratch
+ // register.
+ MachineOperand BaseRegOpForLDP = BaseRegOp;
+ if (Flags.getRequiresBaseAdjust()) {
+ Register Scratch = Flags.getBaseAdjustScratchReg();
+ BaseRegOpForLDP =
+ MachineOperand::CreateReg(Scratch, /*isDef=*/false, /*isImp=*/false,
+ /*isKill=*/false, /*isDead=*/false,
+ /*isUndef=*/false, /*isEarlyClobber=*/false,
+ /*isImplicit=*/false);
+ }
int Offset = AArch64InstrInfo::getLdStOffsetOp(*I).getImm();
int PairedOffset = AArch64InstrInfo::getLdStOffsetOp(*Paired).getImm();
@@ -1198,6 +1290,10 @@ AArch64LoadStoreOpt::mergePairedInsns(MachineBasicBlock::iterator I,
"Unscaled offset cannot be scaled.");
OffsetImm /= TII->getMemScale(*RtMI);
}
+ // With base adjustment the pair address is scratch + 0; the byte offset
+ // is carried by the ADDXri.
+ if (Flags.getRequiresBaseAdjust())
+ OffsetImm = 0;
// Construct the new instruction.
MachineInstrBuilder MIB;
@@ -1243,7 +1339,7 @@ AArch64LoadStoreOpt::mergePairedInsns(MachineBasicBlock::iterator I,
MIB.add(RegOp0)
.add(RegOp1)
- .add(BaseRegOp)
+ .add(BaseRegOpForLDP)
.addImm(OffsetImm)
.cloneMergedMemRefs({&*I, &*Paired})
.setMIFlags(I->mergeFlagsWith(*Paired));
@@ -2047,6 +2143,30 @@ AArch64LoadStoreOpt::findMatchingInsn(MachineBasicBlock::iterator I,
// Remember any instructions that read/write memory between FirstMI and MI.
SmallVector<MachineInstr *, 4> MemInsns;
+ // Scratch availability at FirstMI, computed lazily and at most once per
+ // call. ModifiedRegUnits/UsedRegUnits only cover the region between the
+ // paired insns, so the base-adjust path needs the scavenger for sound
+ // liveness at FirstMI; without it, no scratch is picked.
+ BitVector AvailableScratchRegs;
+ bool ScavengerReady = false;
+ bool ScavengerComputed = false;
+ auto computeScratchAvailability = [&] {
+ if (ScavengerComputed)
+ return;
+ ScavengerComputed = true;
+ const MachineFunction &MF = *FirstMI.getParent()->getParent();
+ if (MF.getRegInfo().tracksLiveness()) {
+ RegScavenger RS;
+ RS.enterBasicBlockEnd(*I->getParent());
+ // backward(I) stops at FirstMI, so the live set is taken between
+ // FirstMI and the next instruction; a store's Rt looks dead there,
+ // which the regsOverlap skip below compensates for.
+ RS.backward(I);
+ AvailableScratchRegs = RS.getRegsAvailable(&AArch64::GPR64commonRegClass);
+ ScavengerReady = true;
+ }
+ };
+
LLVM_DEBUG(dbgs() << "Find match for: "; FirstMI.dump());
for (unsigned Count = 0; MBBI != E && Count < Limit;
MBBI = next_nodbg(MBBI, E)) {
@@ -2146,15 +2266,74 @@ AArch64LoadStoreOpt::findMatchingInsn(MachineBasicBlock::iterator I,
// Pairwise instructions have a 7-bit signed offset field. Single
// insns have a 12-bit unsigned offset field. If the resultant
// immediate offset of merging these instructions is out of range for
- // a pairwise instruction, bail and keep looking.
+ // a pairwise instruction, try base register adjustment.
if (!inBoundsForPair(IsUnscaled, MinOffset, OffsetStride)) {
+ if (EnableLdpStpBaseAdjust) {
+ int MemScale = getBaseAdjustMemScale(TII, FirstMI);
+ int64_t ByteOffset =
+ static_cast<int64_t>(MinOffset) * (IsUnscaled ? 1 : MemScale);
+ if (ByteOffset >= 0 && canEncodeAddXriImm(ByteOffset)) {
+ Register Reg0 = getLdStRegOp(FirstMI).getReg();
+ Register Reg1 = getLdStRegOp(MI).getReg();
+ Register Scratch = AArch64::NoRegister;
+
+ // For GPR64 loads, reuse the first destination register as
+ // the adjusted base. It is dead-by-construction (about to
+ // be written by the load), so no liveness analysis is
+ // needed, and `ldp Rt1, Rt2, [Rt1]` is legal: the base is
+ // read before Rt1 is written (same model as
+ // `ldr x0, [x0, #8]`). X16/X17 are skipped here too; see
+ // the scavenger path below.
+ if (MayLoad && AArch64::GPR64RegClass.contains(Reg0) &&
+ !TRI->regsOverlap(Reg0, Reg1) && Reg0 != AArch64::X16 &&
+ Reg0 != AArch64::X17)
+ Scratch = Reg0;
+ else {
+ // Stores and sub-64-bit loads cannot reuse a dest/source
+ // register (it holds the value to store, or is not a
+ // 64-bit base), so scavenge from caller-saved temporaries
+ // x9-x15. X16/X17 (IP0/IP1) are excluded: the linker may
+ // clobber them with veneers/PLT entries, and holding a
+ // live adjusted base in IP0/IP1 across the add/ldp is
+ // unsound under CSPGO, where long-range branch veneers
+ // are common. If no register is free, leave the accesses
+ // unpaired; a missed optimization beats a miscompile.
+ computeScratchAvailability();
+ static const MCPhysReg ScratchCandidates[] = {
+ AArch64::X9, AArch64::X10, AArch64::X11, AArch64::X12,
+ AArch64::X13, AArch64::X14, AArch64::X15};
+ // Candidates are 64-bit while Reg0/Reg1 may be 32-bit (Wn)
+ // for sub-64-bit accesses, so skip by register overlap: an
+ // exact compare would miss that Xn aliases Wn and let the
+ // ADDXri clobber the stored value before the STP reads it.
+ if (ScavengerReady) {
+ for (MCPhysReg Reg : ScratchCandidates) {
+ if (TRI->regsOverlap(Reg, Reg0) ||
+ TRI->regsOverlap(Reg, Reg1))
+ continue;
+ if (AvailableScratchRegs[Reg]) {
+ Scratch = Reg;
+ break;
+ }
+ }
+ }
+ }
+ if (Scratch != AArch64::NoRegister) {
+ Flags.setRequiresBaseAdjust(true);
+ Flags.setBaseAdjustByteOffset(ByteOffset);
+ Flags.setBaseAdjustScratchReg(Scratch);
+ goto base_adjust_accepted;
+ }
+ }
+ }
LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits,
UsedRegUnits, TRI);
MemInsns.push_back(&MI);
- LLVM_DEBUG(dbgs() << "Offset doesn't fit in immediate, "
+ LLVM_DEBUG(dbgs() << "No base adjust opportunity, "
<< "keep looking.\n");
continue;
}
+ base_adjust_accepted:
// If the alignment requirements of the paired (scaled) instruction
// can't express the offset of the unscaled input, bail and keep
// looking.
@@ -2843,6 +3022,20 @@ bool AArch64LoadStoreOpt::tryToMergeZeroStInst(
return false;
}
+bool AArch64LoadStoreOpt::isBaseLoopInvariant(const MachineInstr &MI) {
+ if (!MLI)
+ return true;
+ const MachineLoop *L = MLI->getLoopFor(MI.getParent());
+ if (!L)
+ return true;
+ Register BaseReg = AArch64InstrInfo::getLdStBaseOp(MI).getReg();
+ for (const MachineBasicBlock *MBB : L->blocks())
+ for (const MachineInstr &LoopMI : *MBB)
+ if (LoopMI.modifiesRegister(BaseReg, TRI))
+ return false;
+ return true;
+}
+
// Find loads and stores that can be merged into a single load or store pair
// instruction.
bool AArch64LoadStoreOpt::tryToPairLdStInst(MachineBasicBlock::iterator &MBBI) {
@@ -2869,8 +3062,25 @@ bool AArch64LoadStoreOpt::tryToPairLdStInst(MachineBasicBlock::iterator &MBBI) {
// Allow one more for offset.
if (Offset > 0)
Offset -= OffsetStride;
- if (!inBoundsForPair(IsUnscaled, Offset, OffsetStride))
- return false;
+ if (!inBoundsForPair(IsUnscaled, Offset, OffsetStride)) {
+ // Base adjustment is restricted to scaled accesses: an unscaled far
+ // offset cannot occur in practice (LDUR's signed 9-bit range covers the
+ // LDP/STP pair range), and pairing an unscaled access whose byte offset
+ // is not a multiple of the scale would truncate it during the merge.
+ if (!EnableLdpStpBaseAdjust || IsUnscaled)
+ return false;
+ // Base-adjust is net negative for loop-variant bases: the per-pair
+ // ADDXri cannot be hoisted out of the loop.
+ if (!isBaseLoopInvariant(MI))
+ return false;
+ int MemScale = getBaseAdjustMemScale(TII, MI);
+ int64_t ByteOffset =
+ static_cast<int64_t>(Offset) * (IsUnscaled ? 1 : MemScale);
+ if (ByteOffset < 0 || !canEncodeAddXriImm(ByteOffset))
+ return false;
+ LLVM_DEBUG(dbgs() << "Offset " << Offset
+ << " out of LDP/STP range, may use base adjustment\n");
+ }
// Look ahead up to LdStLimit instructions for a pairable instruction.
LdStPairFlags Flags;
@@ -2914,6 +3124,12 @@ bool AArch64LoadStoreOpt::tryToPairLdStInst(MachineBasicBlock::iterator &MBBI) {
++NumPairCreated;
if (TII->hasUnscaledLdStOffset(MI))
++NumUnscaledPairCreated;
+ if (Flags.getRequiresBaseAdjust()) {
+ if (MI.mayLoad())
+ ++NumBaseAdjustLdpCreated;
+ else
+ ++NumBaseAdjustStpCreated;
+ }
MBBI = mergePairedInsns(MBBI, Paired, Flags);
// Collect liveness info for instructions between Prev and the new position
@@ -3333,6 +3549,7 @@ bool AArch64LoadStoreOptLegacy::runOnMachineFunction(MachineFunction &MF) {
return false;
AArch64LoadStoreOpt Impl;
Impl.AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
+ Impl.MLI = &getAnalysis<MachineLoopInfoWrapperPass>().getLI();
return Impl.runOnMachineFunction(MF);
}
@@ -3349,6 +3566,7 @@ AArch64LoadStoreOptPass::run(MachineFunction &MF,
Impl.AA = &MFAM.getResult<FunctionAnalysisManagerMachineFunctionProxy>(MF)
.getManager()
.getResult<AAManager>(MF.getFunction());
+ Impl.MLI = &MFAM.getResult<MachineLoopAnalysis>(MF);
bool Changed = Impl.runOnMachineFunction(MF);
if (!Changed)
return PreservedAnalyses::all();
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/split-offsets-for-stp.ll b/llvm/test/CodeGen/AArch64/GlobalISel/split-offsets-for-stp.ll
index 2c2d72ce6afd0..8cb1d610bd938 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/split-offsets-for-stp.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/split-offsets-for-stp.ll
@@ -256,9 +256,9 @@ define void @use_of_load_in_between(ptr %p, ptr %ldptr, ptr %ldptr2) {
define void @offset_legal_for_add_imm(ptr %p) {
; CHECK-NO-SPLIT-LABEL: offset_legal_for_add_imm:
; CHECK-NO-SPLIT: ; %bb.0:
-; CHECK-NO-SPLIT-NEXT: str xzr, [x0, #3200]
-; CHECK-NO-SPLIT-NEXT: str xzr, [x0, #3208]
+; CHECK-NO-SPLIT-NEXT: add x9, x0, #3200
; CHECK-NO-SPLIT-NEXT: str xzr, [x0, #3216]
+; CHECK-NO-SPLIT-NEXT: stp xzr, xzr, [x9]
; CHECK-NO-SPLIT-NEXT: ret
;
; CHECK-SPLIT-LABEL: offset_legal_for_add_imm:
@@ -302,10 +302,10 @@ define void @offset_illegal_for_add_imm(ptr %p) {
define void @offset_legal_for_add_imm_4_stores(ptr %p) {
; CHECK-NO-SPLIT-LABEL: offset_legal_for_add_imm_4_stores:
; CHECK-NO-SPLIT: ; %bb.0:
-; CHECK-NO-SPLIT-NEXT: str xzr, [x0, #3200]
-; CHECK-NO-SPLIT-NEXT: str xzr, [x0, #3208]
-; CHECK-NO-SPLIT-NEXT: str xzr, [x0, #3216]
-; CHECK-NO-SPLIT-NEXT: str xzr, [x0, #3224]
+; CHECK-NO-SPLIT-NEXT: add x9, x0, #3200
+; CHECK-NO-SPLIT-NEXT: stp xzr, xzr, [x9]
+; CHECK-NO-SPLIT-NEXT: add x9, x0, #3216
+; CHECK-NO-SPLIT-NEXT: stp xzr, xzr, [x9]
; CHECK-NO-SPLIT-NEXT: ret
;
; CHECK-SPLIT-LABEL: offset_legal_for_add_imm_4_stores:
diff --git a/llvm/test/CodeGen/AArch64/O3-pipeline.ll b/llvm/test/CodeGen/AArch64/O3-pipeline.ll
index 9b36359c339c8..8f41801325ee0 100644
--- a/llvm/test/CodeGen/AArch64/O3-pipeline.ll
+++ b/llvm/test/CodeGen/AArch64/O3-pipeline.ll
@@ -225,6 +225,8 @@
; CHECK-NEXT: Machine Copy Propagation Pass
; CHECK-NEXT: Post-RA pseudo instruction expansion pass
; CHECK-NEXT: AArch64 pseudo instruction expansion pass
+; CHECK-NEXT: MachineDominator Tree Construction
+; CHECK-NEXT: Machine Natural Loop Construction
; CHECK-NEXT: AArch64 load / store optimization pass
; CHECK-NEXT: Insert KCFI indirect call checks
; CHECK-NEXT: AArch64 speculation hardening pass
@@ -240,6 +242,8 @@
; CHECK-NEXT: Insert fentry calls
; CHECK-NEXT: Insert XRay ops
; CHECK-NEXT: Implement the 'patchable-function' attribute
+; CHECK-NEXT: MachineDominator Tree Construction
+; CHECK-NEXT: Machine Natural Loop Construction
; CHECK-NEXT: AArch64 load / store optimization pass
; CHECK-NEXT: Machine Copy Propagation Pass
; CHECK-NEXT: AArch64 Redundant Conditional Branch Elimination
diff --git a/llvm/test/CodeGen/AArch64/aarch64-gep-opt.ll b/llvm/test/CodeGen/AArch64/aarch64-gep-opt.ll
index d9cdac4946360..ebe05a06fc4b4 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-gep-opt.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-gep-opt.ll
@@ -71,9 +71,8 @@ exit:
ret void
}
; CHECK-LABEL: test_GEP_across_BB:
-; CHECK: ldr {{w[0-9]+}}, [{{x[0-9]+}}, #528]
-; CHECK: ldr {{w[0-9]+}}, [{{x[0-9]+}}, #532]
-; CHECK-NOT: add
+; CHECK: add {{x[0-9]+}}, {{x[0-9]+}}, #528
+; CHECK: ldp {{w[0-9]+}}, {{w[0-9]+}}, [{{x[0-9]+}}]
; CHECK: str {{w[0-9]+}}, [{{x[0-9]+}}, #532]
; CHECK: str {{w[0-9]+}}, [{{x[0-9]+}}, #528]
diff --git a/llvm/test/CodeGen/AArch64/aarch64-ldst-opt-instr-ref.mir b/llvm/test/CodeGen/AArch64/aarch64-ldst-opt-instr-ref.mir
index 5a6b4c75a6ecf..ef9dc831dc4ff 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-ldst-opt-instr-ref.mir
+++ b/llvm/test/CodeGen/AArch64/aarch64-ldst-opt-instr-ref.mir
@@ -69,8 +69,14 @@
!14 = distinct !DICompositeType(tag: DW_TAG_class_type, size: 128, identifier: "_ZTSN4llvm9StringRefE")
name: _ZNK4llvm9StringRef4sizeEv
+body: |
+ bb.0 (%ir-block.entry):
+ RET_ReallyLR
---
name: _ZNK4llvm9StringRef4dataEv
+body: |
+ bb.0 (%ir-block.entry):
+ RET_ReallyLR
...
name: _ZNK4llvm7Pattern5matchENS_9StringRefERKNS_9SourceMgrE
debugValueSubstitutions: []
diff --git a/llvm/test/CodeGen/AArch64/aarch64-sve-fill-spill-pair.ll b/llvm/test/CodeGen/AArch64/aarch64-sve-fill-spill-pair.ll
index 503ead4eba2db..fe6bdc512e252 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-sve-fill-spill-pair.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-sve-fill-spill-pair.ll
@@ -122,8 +122,8 @@ define void @nxv16i8_outside_range(ptr %ldptr, ptr %stptr) {
; CHECK: // %bb.0:
; CHECK-NEXT: ldr z0, [x0, #-65, mul vl]
; CHECK-NEXT: ldr z1, [x0, #-64, mul vl]
-; CHECK-NEXT: str z0, [x1, #64, mul vl]
-; CHECK-NEXT: str z1, [x1, #65, mul vl]
+; CHECK-NEXT: add x9, x1, #1024
+; CHECK-NEXT: stp q0, q1, [x9]
; CHECK-NEXT: ret
;
; CHECK-BE-LABEL: nxv16i8_outside_range:
@@ -149,8 +149,8 @@ define void @nxv16i8_outside_range(ptr %ldptr, ptr %stptr) {
; CHECK-LDPALIGNEDONLY: // %bb.0:
; CHECK-LDPALIGNEDONLY-NEXT: ldr z0, [x0, #-65, mul vl]
; CHECK-LDPALIGNEDONLY-NEXT: ldr z1, [x0, #-64, mul vl]
-; CHECK-LDPALIGNEDONLY-NEXT: str z0, [x1, #64, mul vl]
-; CHECK-LDPALIGNEDONLY-NEXT: str z1, [x1, #65, mul vl]
+; CHECK-LDPALIGNEDONLY-NEXT: add x9, x1, #1024
+; CHECK-LDPALIGNEDONLY-NEXT: stp q0, q1, [x9]
; CHECK-LDPALIGNEDONLY-NEXT: ret
;
; CHECK-STPALIGNEDONLY-LABEL: nxv16i8_outside_range:
@@ -281,3 +281,66 @@ define void @nxv2f64_32b_aligned(ptr %ldptr, ptr %stptr) {
store <vscale x 2 x double> %ld2, ptr %stptr2, align 32
ret void
}
+
+; SVE base-adjust: under VLS-128, SVE fills/spills pair into Q-register
+; LDP/STP. When the pair offset exceeds STPQi's 7-bit range (64 * 16 = 1024
+; bytes), base-adjust inserts an ADDXri to materialize a scratch base. Under
+; scalable or VLS-256, the VLS-128 guard in isCandidateToMergeOrPair rejects
+; pairing, so no base-adjust occurs and the accesses stay as ldr/str z.
+define void @sve_base_adjust_vls128_vs_scalable(ptr %ldptr, ptr %stptr) {
+; CHECK-LABEL: sve_base_adjust_vls128_vs_scalable:
+; CHECK: // %bb.0:
+; CHECK-NEXT: add x9, x0, #1024
+; CHECK-NEXT: ldp q0, q1, [x9]
+; CHECK-NEXT: add x9, x1, #1024
+; CHECK-NEXT: stp q0, q1, [x9]
+; CHECK-NEXT: ret
+;
+; CHECK-BE-LABEL: sve_base_adjust_vls128_vs_scalable:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rdvl x8, #1
+; CHECK-BE-NEXT: mov w9, #1024
+; CHECK-BE-NEXT: mov w10, #1040
+; CHECK-BE-NEXT: lsr x8, x8, #4
+; CHECK-BE-NEXT: ptrue p0.b
+; CHECK-BE-NEXT: mul x9, x8, x9
+; CHECK-BE-NEXT: mul x8, x8, x10
+; CHECK-BE-NEXT: ld1b { z0.b }, p0/z, [x0, x9]
+; CHECK-BE-NEXT: ld1b { z1.b }, p0/z, [x0, x8]
+; CHECK-BE-NEXT: st1b { z0.b }, p0, [x1, x9]
+; CHECK-BE-NEXT: st1b { z1.b }, p0, [x1, x8]
+; CHECK-BE-NEXT: ret
+;
+; CHECK-LDPALIGNEDONLY-LABEL: sve_base_adjust_vls128_vs_scalable:
+; CHECK-LDPALIGNEDONLY: // %bb.0:
+; CHECK-LDPALIGNEDONLY-NEXT: ldr z0, [x0, #64, mul vl]
+; CHECK-LDPALIGNEDONLY-NEXT: ldr z1, [x0, #65, mul vl]
+; CHECK-LDPALIGNEDONLY-NEXT: add x9, x1, #1024
+; CHECK-LDPALIGNEDONLY-NEXT: stp q0, q1, [x9]
+; CHECK-LDPALIGNEDONLY-NEXT: ret
+;
+; CHECK-STPALIGNEDONLY-LABEL: sve_base_adjust_vls128_vs_scalable:
+; CHECK-STPALIGNEDONLY: // %bb.0:
+; CHECK-STPALIGNEDONLY-NEXT: add x9, x0, #1024
+; CHECK-STPALIGNEDONLY-NEXT: ldp q0, q1, [x9]
+; CHECK-STPALIGNEDONLY-NEXT: str z0, [x1, #64, mul vl]
+; CHECK-STPALIGNEDONLY-NEXT: str z1, [x1, #65, mul vl]
+; CHECK-STPALIGNEDONLY-NEXT: ret
+;
+; CHECK-OFF-LABEL: sve_base_adjust_vls128_vs_scalable:
+; CHECK-OFF: // %bb.0:
+; CHECK-OFF-NEXT: ldr z0, [x0, #64, mul vl]
+; CHECK-OFF-NEXT: ldr z1, [x0, #65, mul vl]
+; CHECK-OFF-NEXT: str z0, [x1, #64, mul vl]
+; CHECK-OFF-NEXT: str z1, [x1, #65, mul vl]
+; CHECK-OFF-NEXT: ret
+ %ldptr1 = getelementptr inbounds nuw <vscale x 16 x i8>, ptr %ldptr, i64 64
+ %ldptr2 = getelementptr inbounds nuw <vscale x 16 x i8>, ptr %ldptr, i64 65
+ %stptr1 = getelementptr inbounds nuw <vscale x 16 x i8>, ptr %stptr, i64 64
+ %stptr2 = getelementptr inbounds nuw <vscale x 16 x i8>, ptr %stptr, i64 65
+ %ld1 = load <vscale x 16 x i8>, ptr %ldptr1, align 16
+ %ld2 = load <vscale x 16 x i8>, ptr %ldptr2, align 16
+ store <vscale x 16 x i8> %ld1, ptr %stptr1, align 16
+ store <vscale x 16 x i8> %ld2, ptr %stptr2, align 16
+ ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/ldp-far-offset.ll b/llvm/test/CodeGen/AArch64/ldp-far-offset.ll
new file mode 100644
index 0000000000000..c70916a6c978b
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/ldp-far-offset.ll
@@ -0,0 +1,291 @@
+; RUN: llc -mtriple=aarch64 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ENABLED
+; RUN: llc -mtriple=aarch64 -verify-machineinstrs -aarch64-ldp-stp-base-adjust=0 < %s | FileCheck %s --check-prefixes=CHECK,DISABLED
+
+define void @ldp_far_offset_i32(ptr %p) {
+; ENABLED-LABEL: ldp_far_offset_i32:
+; ENABLED: // %bb.0:
+; ENABLED-NEXT: add x[[TMP:[0-9]+]], x0, #400
+; ENABLED-NEXT: ldp w[[R0:[0-9]+]], w[[R1:[0-9]+]], [x[[TMP]]]
+; ENABLED-NEXT: str w[[R0]]
+; ENABLED-NEXT: str w[[R1]]
+; ENABLED-NEXT: ret
+;
+; DISABLED-LABEL: ldp_far_offset_i32:
+; DISABLED: // %bb.0:
+; DISABLED-NOT: ldp w
+; DISABLED-NOT: ldp x
+; DISABLED: ret
+ %gep0 = getelementptr i32, ptr %p, i64 100
+ %gep1 = getelementptr i32, ptr %p, i64 101
+ %v0 = load i32, ptr %gep0
+ %v1 = load i32, ptr %gep1
+ store volatile i32 %v0, ptr poison
+ store volatile i32 %v1, ptr poison
+ ret void
+}
+
+define void @ldp_far_offset_i64(ptr %p) {
+; ENABLED-LABEL: ldp_far_offset_i64:
+; ENABLED: // %bb.0:
+; ENABLED-NEXT: add x[[TMP:[0-9]+]], x0, #800
+; ENABLED-NEXT: ldp x[[R0:[0-9]+]], x[[R1:[0-9]+]], [x[[TMP]]]
+; ENABLED-NEXT: str x[[R0]]
+; ENABLED-NEXT: str x[[R1]]
+; ENABLED-NEXT: ret
+;
+; DISABLED-LABEL: ldp_far_offset_i64:
+; DISABLED: // %bb.0:
+; DISABLED-NOT: ldp w
+; DISABLED-NOT: ldp x
+; DISABLED: ret
+ %gep0 = getelementptr i64, ptr %p, i64 100
+ %gep1 = getelementptr i64, ptr %p, i64 101
+ %v0 = load i64, ptr %gep0
+ %v1 = load i64, ptr %gep1
+ store volatile i64 %v0, ptr poison
+ store volatile i64 %v1, ptr poison
+ ret void
+}
+
+; The program-order-first load is at the higher offset (101), so the reused
+; destination register lands in the second LDP destination slot (Rt2) and is
+; also the adjusted base: `add Rt2, base, #imm; ldp Rt0, Rt2, [Rt2]`. The
+; backreference `x[[TMP]]` for both the second destination and the base
+; makes this CHECK fail if a separate scratch is reintroduced.
+define void @ldp_far_offset_i64_swapped(ptr %p) {
+; ENABLED-LABEL: ldp_far_offset_i64_swapped:
+; ENABLED: // %bb.0:
+; ENABLED-NEXT: add x[[TMP:[0-9]+]], x0, #800
+; ENABLED-NEXT: ldp x[[R0:[0-9]+]], x[[TMP]], [x[[TMP]]]
+; ENABLED-NEXT: str x[[TMP]]
+; ENABLED-NEXT: str x[[R0]]
+; ENABLED-NEXT: ret
+;
+; DISABLED-LABEL: ldp_far_offset_i64_swapped:
+; DISABLED: // %bb.0:
+; DISABLED-NOT: ldp w
+; DISABLED-NOT: ldp x
+; DISABLED: ret
+ %gep_hi = getelementptr i64, ptr %p, i64 101
+ %gep_lo = getelementptr i64, ptr %p, i64 100
+ %v_hi = load i64, ptr %gep_hi
+ %v_lo = load i64, ptr %gep_lo
+ store volatile i64 %v_hi, ptr poison
+ store volatile i64 %v_lo, ptr poison
+ ret void
+}
+
+; Regression guard for the GPR64 dest-reuse path: when the load destination
+; is itself IP0/IP1 (X16/X17), reusing it as the adjusted base reintroduces
+; the linker-veneer hazard the scavenger path avoids. The dest-reuse path
+; must skip X16/X17 and fall through to the X9..X15 scavenger. The scratch
+; capture `(1[0-5]|[0-9])` cannot match 16 or 17.
+define void @ldp_far_offset_i64_x16_dst(ptr %p) {
+; ENABLED-LABEL: ldp_far_offset_i64_x16_dst:
+; ENABLED: // %bb.0:
+; ENABLED-NEXT: add x[[SCRATCH:(1[0-5]|[0-9])]], x0, #800
+; ENABLED-NEXT: ldp x16, x9, [x[[SCRATCH]]]
+; ENABLED: ret
+;
+; DISABLED-LABEL: ldp_far_offset_i64_x16_dst:
+; DISABLED: // %bb.0:
+; DISABLED-NOT: ldp w
+; DISABLED-NOT: ldp x
+; DISABLED: ret
+ %gep0 = getelementptr i64, ptr %p, i64 100
+ %gep1 = getelementptr i64, ptr %p, i64 101
+ %v0 = load i64, ptr %gep0
+ %v1 = load i64, ptr %gep1
+ %r0 = call i64 asm "", "={x16},0"(i64 %v0)
+ %r1 = call i64 asm "", "={x9},0"(i64 %v1)
+ store volatile i64 %r0, ptr poison
+ store volatile i64 %r1, ptr poison
+ ret void
+}
+
+define void @ldp_near_offset(ptr %p) {
+; Near offsets within LDP range always pair regardless of the option.
+; CHECK-LABEL: ldp_near_offset:
+; CHECK: // %bb.0:
+; CHECK-NOT: add
+; CHECK: ldp w{{[0-9]+}}, w{{[0-9]+}}, [x0, #40]
+; CHECK: ret
+ %gep0 = getelementptr i32, ptr %p, i64 10
+ %gep1 = getelementptr i32, ptr %p, i64 11
+ %v0 = load i32, ptr %gep0
+ %v1 = load i32, ptr %gep1
+ store volatile i32 %v0, ptr poison
+ store volatile i32 %v1, ptr poison
+ ret void
+}
+
+define void @ldp_far_offset_interleaved(ptr %p, ptr %q) {
+; Intervening load from a different base between two far-offset loads.
+; ENABLED-LABEL: ldp_far_offset_interleaved:
+; ENABLED: add x[[TMP:[0-9]+]], x0, #400
+; ENABLED: ldp w{{[0-9]+}}, w{{[0-9]+}}, [x[[TMP]]]
+;
+; DISABLED-LABEL: ldp_far_offset_interleaved:
+; DISABLED-NOT: ldp w
+; DISABLED-NOT: ldp x
+; DISABLED: ret
+ %gep0 = getelementptr i32, ptr %p, i64 100
+ %gep1 = getelementptr i32, ptr %p, i64 101
+ %v0 = load i32, ptr %gep0
+ %v2 = load volatile i32, ptr %q
+ %v1 = load i32, ptr %gep1
+ store volatile i32 %v0, ptr poison
+ store volatile i32 %v1, ptr poison
+ ret void
+}
+
+; Offset = 1024 * 4 = 4096 bytes = 0x1000
+; The ADDXri encodes as #1, LSL #12 which equals 4096.
+define void @ldp_far_offset_4k_aligned_i32(ptr %p) {
+; ENABLED-LABEL: ldp_far_offset_4k_aligned_i32:
+; ENABLED: add x[[TMP:[0-9]+]], x0, #1, lsl #12
+; ENABLED: ldp w{{[0-9]+}}, w{{[0-9]+}}, [x[[TMP]]]
+;
+; DISABLED-LABEL: ldp_far_offset_4k_aligned_i32:
+; DISABLED-NOT: ldp w
+; DISABLED-NOT: ldp x
+; DISABLED: ret
+ %gep0 = getelementptr i32, ptr %p, i64 1024
+ %gep1 = getelementptr i32, ptr %p, i64 1025
+ %v0 = load i32, ptr %gep0
+ %v1 = load i32, ptr %gep1
+ store volatile i32 %v0, ptr poison
+ store volatile i32 %v1, ptr poison
+ ret void
+}
+
+; Offset = 500 * 8 = 4000 bytes (fits in shift=0, <= 4095)
+define void @ldp_far_offset_large_i64(ptr %p) {
+; ENABLED-LABEL: ldp_far_offset_large_i64:
+; ENABLED: add x[[TMP:[0-9]+]], x0, #4000
+; ENABLED: ldp x{{[0-9]+}}, x{{[0-9]+}}, [x[[TMP]]]
+;
+; DISABLED-LABEL: ldp_far_offset_large_i64:
+; DISABLED-NOT: ldp w
+; DISABLED-NOT: ldp x
+; DISABLED: ret
+ %gep0 = getelementptr i64, ptr %p, i64 500
+ %gep1 = getelementptr i64, ptr %p, i64 501
+ %v0 = load i64, ptr %gep0
+ %v1 = load i64, ptr %gep1
+ store volatile i64 %v0, ptr poison
+ store volatile i64 %v1, ptr poison
+ ret void
+}
+
+define void @ldp_far_offset_q(ptr %p) {
+; ENABLED-LABEL: ldp_far_offset_q:
+; ENABLED: // %bb.0:
+; ENABLED-NEXT: add x[[TMP:[0-9]+]], x0, #1600
+; ENABLED-NEXT: ldp q0, q1, [x[[TMP]]]
+; ENABLED-NEXT: str q0, [x8]
+; ENABLED-NEXT: str q1, [x8]
+; ENABLED-NEXT: ret
+;
+; DISABLED-LABEL: ldp_far_offset_q:
+; DISABLED: // %bb.0:
+; DISABLED-NEXT: ldr q0, [x0, #1600]
+; DISABLED-NEXT: ldr q1, [x0, #1616]
+; DISABLED-NEXT: str q0, [x8]
+; DISABLED-NEXT: str q1, [x8]
+; DISABLED-NEXT: ret
+ %gep0 = getelementptr <2 x i64>, ptr %p, i64 100
+ %gep1 = getelementptr <2 x i64>, ptr %p, i64 101
+ %v0 = load <2 x i64>, ptr %gep0
+ %v1 = load <2 x i64>, ptr %gep1
+ store volatile <2 x i64> %v0, ptr poison
+ store volatile <2 x i64> %v1, ptr poison
+ ret void
+}
+
+; Offset = 256 * 16 = 4096 bytes = 0x1000
+; The ADDXri encodes as #1, LSL #12 which equals 4096.
+define void @ldp_far_offset_q_4k_aligned(ptr %p) {
+; ENABLED-LABEL: ldp_far_offset_q_4k_aligned:
+; ENABLED: // %bb.0:
+; ENABLED-NEXT: add x[[TMP:[0-9]+]], x0, #1, lsl #12
+; ENABLED-NEXT: ldp q0, q1, [x[[TMP]]]
+; ENABLED-NEXT: str q0, [x8]
+; ENABLED-NEXT: str q1, [x8]
+; ENABLED-NEXT: ret
+;
+; DISABLED-LABEL: ldp_far_offset_q_4k_aligned:
+; DISABLED: // %bb.0:
+; DISABLED-NEXT: ldr q0, [x0, #4096]
+; DISABLED-NEXT: ldr q1, [x0, #4112]
+; DISABLED-NEXT: str q0, [x8]
+; DISABLED-NEXT: str q1, [x8]
+; DISABLED-NEXT: ret
+ %gep0 = getelementptr <2 x i64>, ptr %p, i64 256
+ %gep1 = getelementptr <2 x i64>, ptr %p, i64 257
+ %v0 = load <2 x i64>, ptr %gep0
+ %v1 = load <2 x i64>, ptr %gep1
+ store volatile <2 x i64> %v0, ptr poison
+ store volatile <2 x i64> %v1, ptr poison
+ ret void
+}
+
+; Loop-invariant base inside a loop should still allow base-adjust.
+define void @ldp_far_offset_loop_invariant(ptr %p, i32 %n) {
+; ENABLED-LABEL: ldp_far_offset_loop_invariant:
+; ENABLED: add x[[TMP:[0-9]+]], x0, #400
+; ENABLED: ldp w{{[0-9]+}}, w{{[0-9]+}}, [x[[TMP]]]
+;
+; DISABLED-LABEL: ldp_far_offset_loop_invariant:
+; DISABLED-NOT: ldp w
+; DISABLED-NOT: ldp x
+; DISABLED: ret
+entry:
+ %gep0 = getelementptr i32, ptr %p, i64 100
+ %gep1 = getelementptr i32, ptr %p, i64 101
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %next, %loop ]
+ %v0 = load i32, ptr %gep0
+ %v1 = load i32, ptr %gep1
+ store volatile i32 %v0, ptr poison
+ store volatile i32 %v1, ptr poison
+ %next = add i32 %i, 1
+ %cond = icmp eq i32 %next, %n
+ br i1 %cond, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Loop-variant base should not allow base-adjust: the base pointer changes
+; each iteration (p+j), so the per-pair ADDXri would stay in the loop.
+; Expect 2x ldr even with -aarch64-ldp-stp-base-adjust=1.
+define void @ldp_far_offset_loop_variant(ptr %p, i32 %n) {
+; CHECK-LABEL: ldp_far_offset_loop_variant:
+; CHECK: // %bb.0:
+; CHECK-NOT: add x{{[0-9]+}}, x{{[0-9]+}}, #400
+; CHECK-NOT: ldp w
+; CHECK-NOT: ldp x
+; CHECK: ret
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %next, %loop ]
+ %idx = and i32 %i, 255
+ %gep_base = getelementptr i32, ptr %p, i32 %idx
+ %gep0 = getelementptr i32, ptr %gep_base, i64 100
+ %gep1 = getelementptr i32, ptr %gep_base, i64 101
+ %v0 = load i32, ptr %gep0
+ %v1 = load i32, ptr %gep1
+ store volatile i32 %v0, ptr poison
+ store volatile i32 %v1, ptr poison
+ %next = add i32 %i, 1
+ %cond = icmp eq i32 %next, %n
+ br i1 %cond, label %exit, label %loop
+
+exit:
+ ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/ldst-opt.ll b/llvm/test/CodeGen/AArch64/ldst-opt.ll
index e002f3fba9d5b..84d29aa585735 100644
--- a/llvm/test/CodeGen/AArch64/ldst-opt.ll
+++ b/llvm/test/CodeGen/AArch64/ldst-opt.ll
@@ -1888,10 +1888,10 @@ define void @merge_zr32_2_offset(ptr %p) {
;
; STRICTALIGN-LABEL: merge_zr32_2_offset:
; STRICTALIGN: // %bb.0: // %entry
-; STRICTALIGN-NEXT: str wzr, [x0, #504]
-; STRICTALIGN-NEXT: str wzr, [x0, #508]
-; STRICTALIGN-NEXT: str wzr, [x0, #512]
-; STRICTALIGN-NEXT: str wzr, [x0, #516]
+; STRICTALIGN-NEXT: add x9, x0, #504
+; STRICTALIGN-NEXT: stp wzr, wzr, [x9]
+; STRICTALIGN-NEXT: add x9, x0, #512
+; STRICTALIGN-NEXT: stp wzr, wzr, [x9]
; STRICTALIGN-NEXT: ret
entry:
%p0 = getelementptr i32, ptr %p, i32 126
@@ -1917,9 +1917,9 @@ define void @no_merge_zr32_2_offset(ptr %p) {
;
; STRICTALIGN-LABEL: no_merge_zr32_2_offset:
; STRICTALIGN: // %bb.0: // %entry
-; STRICTALIGN-NEXT: str wzr, [x0, #4096]
-; STRICTALIGN-NEXT: str wzr, [x0, #4100]
+; STRICTALIGN-NEXT: add x9, x0, #1, lsl #12 // =4096
; STRICTALIGN-NEXT: str wzr, [x0, #4104]
+; STRICTALIGN-NEXT: stp wzr, wzr, [x9]
; STRICTALIGN-NEXT: str wzr, [x0, #4108]
; STRICTALIGN-NEXT: ret
entry:
diff --git a/llvm/test/CodeGen/AArch64/preserve-all-large-csr.ll b/llvm/test/CodeGen/AArch64/preserve-all-large-csr.ll
index 6426483ebbce9..b2ef76e1b3d22 100644
--- a/llvm/test/CodeGen/AArch64/preserve-all-large-csr.ll
+++ b/llvm/test/CodeGen/AArch64/preserve-all-large-csr.ll
@@ -21,6 +21,8 @@ define preserve_allcc void @trigger_stack_spill() {
; CHECK-LABEL: trigger_stack_spill:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: stp q31, q30, [sp, #-544]! // 32-byte Folded Spill
+; CHECK-NEXT: stp x10, x9, [sp, #424] // 16-byte Folded Spill
+; CHECK-NEXT: add x9, sp, #520
; CHECK-NEXT: stp q29, q28, [sp, #32] // 32-byte Folded Spill
; CHECK-NEXT: stp q27, q26, [sp, #64] // 32-byte Folded Spill
; CHECK-NEXT: stp q25, q24, [sp, #96] // 32-byte Folded Spill
@@ -35,14 +37,12 @@ define preserve_allcc void @trigger_stack_spill() {
; CHECK-NEXT: str x15, [sp, #384] // 8-byte Spill
; CHECK-NEXT: stp x14, x13, [sp, #392] // 16-byte Folded Spill
; CHECK-NEXT: stp x12, x11, [sp, #408] // 16-byte Folded Spill
-; CHECK-NEXT: stp x10, x9, [sp, #424] // 16-byte Folded Spill
; CHECK-NEXT: stp x29, x30, [sp, #440] // 16-byte Folded Spill
; CHECK-NEXT: stp x28, x27, [sp, #456] // 16-byte Folded Spill
; CHECK-NEXT: stp x26, x25, [sp, #472] // 16-byte Folded Spill
; CHECK-NEXT: stp x24, x23, [sp, #488] // 16-byte Folded Spill
; CHECK-NEXT: stp x22, x21, [sp, #504] // 16-byte Folded Spill
-; CHECK-NEXT: str x20, [sp, #520] // 8-byte Spill
-; CHECK-NEXT: str x19, [sp, #528] // 8-byte Spill
+; CHECK-NEXT: stp x20, x19, [x9] // 16-byte Folded Spill
; CHECK-NEXT: sub sp, sp, #16
; CHECK-NEXT: .cfi_def_cfa_offset 560
; CHECK-NEXT: .cfi_offset w19, -16
@@ -91,11 +91,11 @@ define preserve_allcc void @trigger_stack_spill() {
; CHECK-NEXT: //APP
; CHECK-NEXT: //NO_APP
; CHECK-NEXT: add sp, sp, #16
+; CHECK-NEXT: add x19, sp, #520
; CHECK-NEXT: ldp x22, x21, [sp, #504] // 16-byte Folded Reload
-; CHECK-NEXT: ldr x19, [sp, #528] // 8-byte Reload
-; CHECK-NEXT: ldp x24, x23, [sp, #488] // 16-byte Folded Reload
-; CHECK-NEXT: ldr x20, [sp, #520] // 8-byte Reload
+; CHECK-NEXT: ldp x20, x19, [x19] // 16-byte Folded Reload
; CHECK-NEXT: ldr x15, [sp, #384] // 8-byte Reload
+; CHECK-NEXT: ldp x24, x23, [sp, #488] // 16-byte Folded Reload
; CHECK-NEXT: ldp x26, x25, [sp, #472] // 16-byte Folded Reload
; CHECK-NEXT: ldp x28, x27, [sp, #456] // 16-byte Folded Reload
; CHECK-NEXT: ldp x29, x30, [sp, #440] // 16-byte Folded Reload
diff --git a/llvm/test/CodeGen/AArch64/sme-streaming-checkvl.ll b/llvm/test/CodeGen/AArch64/sme-streaming-checkvl.ll
index 475cb2879da70..be3baa8efab7c 100644
--- a/llvm/test/CodeGen/AArch64/sme-streaming-checkvl.ll
+++ b/llvm/test/CodeGen/AArch64/sme-streaming-checkvl.ll
@@ -82,15 +82,15 @@ define void @foo_streaming_compatible_pass_arg(ptr %arg) #1 {
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: sub sp, sp, #1136
; CHECK-NEXT: .cfi_def_cfa_offset 1136
-; CHECK-NEXT: cntd x9
+; CHECK-NEXT: add x9, sp, #1088
+; CHECK-NEXT: add x10, sp, #1104
; CHECK-NEXT: stp d15, d14, [sp] // 16-byte Folded Spill
+; CHECK-NEXT: stp x29, x30, [x9] // 16-byte Folded Spill
+; CHECK-NEXT: cntd x9
; CHECK-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill
; CHECK-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT: str x29, [sp, #1088] // 8-byte Spill
-; CHECK-NEXT: str x30, [sp, #1096] // 8-byte Spill
-; CHECK-NEXT: str x9, [sp, #1104] // 8-byte Spill
-; CHECK-NEXT: str x28, [sp, #1112] // 8-byte Spill
+; CHECK-NEXT: stp x9, x28, [x10] // 16-byte Folded Spill
; CHECK-NEXT: str x19, [sp, #1120] // 8-byte Spill
; CHECK-NEXT: add x29, sp, #1088
; CHECK-NEXT: .cfi_def_cfa w29, 48
@@ -132,12 +132,12 @@ define void @foo_streaming_compatible_pass_arg(ptr %arg) #1 {
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: add sp, sp, #1024
; CHECK-NEXT: .cfi_def_cfa wsp, 1136
+; CHECK-NEXT: add x19, sp, #1112
+; CHECK-NEXT: add x30, sp, #1088
+; CHECK-NEXT: ldp x28, x19, [x19] // 16-byte Folded Reload
+; CHECK-NEXT: ldp x29, x30, [x30] // 16-byte Folded Reload
; CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT: ldr x19, [sp, #1120] // 8-byte Reload
; CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT: ldr x28, [sp, #1112] // 8-byte Reload
-; CHECK-NEXT: ldr x30, [sp, #1096] // 8-byte Reload
-; CHECK-NEXT: ldr x29, [sp, #1088] // 8-byte Reload
; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; CHECK-NEXT: ldp d15, d14, [sp] // 16-byte Folded Reload
; CHECK-NEXT: add sp, sp, #1136
@@ -168,15 +168,15 @@ define void @foo_streaming_pass_arg(ptr %arg) #0 {
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: sub sp, sp, #1120
; CHECK-NEXT: .cfi_def_cfa_offset 1120
-; CHECK-NEXT: cntd x9
+; CHECK-NEXT: add x9, sp, #1088
+; CHECK-NEXT: add x10, sp, #1104
; CHECK-NEXT: stp d15, d14, [sp] // 16-byte Folded Spill
+; CHECK-NEXT: stp x29, x30, [x9] // 16-byte Folded Spill
+; CHECK-NEXT: cntd x9
; CHECK-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill
; CHECK-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT: str x29, [sp, #1088] // 8-byte Spill
-; CHECK-NEXT: str x30, [sp, #1096] // 8-byte Spill
-; CHECK-NEXT: str x9, [sp, #1104] // 8-byte Spill
-; CHECK-NEXT: str x28, [sp, #1112] // 8-byte Spill
+; CHECK-NEXT: stp x9, x28, [x10] // 16-byte Folded Spill
; CHECK-NEXT: add x29, sp, #1088
; CHECK-NEXT: .cfi_def_cfa w29, 32
; CHECK-NEXT: .cfi_offset w28, -8
@@ -210,11 +210,11 @@ define void @foo_streaming_pass_arg(ptr %arg) #0 {
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: add sp, sp, #1024
; CHECK-NEXT: .cfi_def_cfa wsp, 1120
+; CHECK-NEXT: add x30, sp, #1088
; CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload
+; CHECK-NEXT: ldp x29, x30, [x30] // 16-byte Folded Reload
; CHECK-NEXT: ldr x28, [sp, #1112] // 8-byte Reload
; CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT: ldr x30, [sp, #1096] // 8-byte Reload
-; CHECK-NEXT: ldr x29, [sp, #1088] // 8-byte Reload
; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; CHECK-NEXT: ldp d15, d14, [sp] // 16-byte Folded Reload
; CHECK-NEXT: add sp, sp, #1120
@@ -316,17 +316,17 @@ define void @foo_streaming_compatible_retval(ptr %ptr) #1 {
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: sub sp, sp, #1136
; CHECK-NEXT: .cfi_def_cfa_offset 1136
-; CHECK-NEXT: cntd x9
+; CHECK-NEXT: add x9, sp, #1088
+; CHECK-NEXT: add x10, sp, #1104
; CHECK-NEXT: stp d15, d14, [sp] // 16-byte Folded Spill
+; CHECK-NEXT: stp x29, x30, [x9] // 16-byte Folded Spill
+; CHECK-NEXT: cntd x9
+; CHECK-NEXT: stp x9, x28, [x10] // 16-byte Folded Spill
+; CHECK-NEXT: add x9, sp, #1120
; CHECK-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill
; CHECK-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT: str x29, [sp, #1088] // 8-byte Spill
-; CHECK-NEXT: str x30, [sp, #1096] // 8-byte Spill
-; CHECK-NEXT: str x9, [sp, #1104] // 8-byte Spill
-; CHECK-NEXT: str x28, [sp, #1112] // 8-byte Spill
-; CHECK-NEXT: str x20, [sp, #1120] // 8-byte Spill
-; CHECK-NEXT: str x19, [sp, #1128] // 8-byte Spill
+; CHECK-NEXT: stp x20, x19, [x9] // 16-byte Folded Spill
; CHECK-NEXT: add x29, sp, #1088
; CHECK-NEXT: .cfi_def_cfa w29, 48
; CHECK-NEXT: .cfi_offset w19, -8
@@ -369,13 +369,13 @@ define void @foo_streaming_compatible_retval(ptr %ptr) #1 {
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: add sp, sp, #1024
; CHECK-NEXT: .cfi_def_cfa wsp, 1136
+; CHECK-NEXT: add x19, sp, #1120
+; CHECK-NEXT: add x30, sp, #1088
+; CHECK-NEXT: ldr x28, [sp, #1112] // 8-byte Reload
+; CHECK-NEXT: ldp x20, x19, [x19] // 16-byte Folded Reload
+; CHECK-NEXT: ldp x29, x30, [x30] // 16-byte Folded Reload
; CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT: ldr x19, [sp, #1128] // 8-byte Reload
; CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT: ldr x20, [sp, #1120] // 8-byte Reload
-; CHECK-NEXT: ldr x28, [sp, #1112] // 8-byte Reload
-; CHECK-NEXT: ldr x30, [sp, #1096] // 8-byte Reload
-; CHECK-NEXT: ldr x29, [sp, #1088] // 8-byte Reload
; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; CHECK-NEXT: ldp d15, d14, [sp] // 16-byte Folded Reload
; CHECK-NEXT: add sp, sp, #1136
@@ -407,15 +407,15 @@ define void @foo_streaming_retval(ptr %ptr) #0 {
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: sub sp, sp, #1136
; CHECK-NEXT: .cfi_def_cfa_offset 1136
-; CHECK-NEXT: cntd x9
+; CHECK-NEXT: add x9, sp, #1088
+; CHECK-NEXT: add x10, sp, #1104
; CHECK-NEXT: stp d15, d14, [sp] // 16-byte Folded Spill
+; CHECK-NEXT: stp x29, x30, [x9] // 16-byte Folded Spill
+; CHECK-NEXT: cntd x9
; CHECK-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill
; CHECK-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT: str x29, [sp, #1088] // 8-byte Spill
-; CHECK-NEXT: str x30, [sp, #1096] // 8-byte Spill
-; CHECK-NEXT: str x9, [sp, #1104] // 8-byte Spill
-; CHECK-NEXT: str x28, [sp, #1112] // 8-byte Spill
+; CHECK-NEXT: stp x9, x28, [x10] // 16-byte Folded Spill
; CHECK-NEXT: str x19, [sp, #1120] // 8-byte Spill
; CHECK-NEXT: add x29, sp, #1088
; CHECK-NEXT: .cfi_def_cfa w29, 48
@@ -451,12 +451,12 @@ define void @foo_streaming_retval(ptr %ptr) #0 {
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: add sp, sp, #1024
; CHECK-NEXT: .cfi_def_cfa wsp, 1136
+; CHECK-NEXT: add x19, sp, #1112
+; CHECK-NEXT: add x30, sp, #1088
+; CHECK-NEXT: ldp x28, x19, [x19] // 16-byte Folded Reload
+; CHECK-NEXT: ldp x29, x30, [x30] // 16-byte Folded Reload
; CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT: ldr x19, [sp, #1120] // 8-byte Reload
; CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT: ldr x28, [sp, #1112] // 8-byte Reload
-; CHECK-NEXT: ldr x30, [sp, #1096] // 8-byte Reload
-; CHECK-NEXT: ldr x29, [sp, #1088] // 8-byte Reload
; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; CHECK-NEXT: ldp d15, d14, [sp] // 16-byte Folded Reload
; CHECK-NEXT: add sp, sp, #1136
diff --git a/llvm/test/CodeGen/AArch64/stack-hazard.ll b/llvm/test/CodeGen/AArch64/stack-hazard.ll
index 663c34dd245d3..5843493cf3d4b 100644
--- a/llvm/test/CodeGen/AArch64/stack-hazard.ll
+++ b/llvm/test/CodeGen/AArch64/stack-hazard.ll
@@ -356,10 +356,10 @@ define i32 @csr_d8_allocd_framepointer(double %d) "aarch64_pstate_sm_compatible"
; CHECK1024-LABEL: csr_d8_allocd_framepointer:
; CHECK1024: // %bb.0: // %entry
; CHECK1024-NEXT: sub sp, sp, #1056
+; CHECK1024-NEXT: add x9, sp, #1032
; CHECK1024-NEXT: str d8, [sp] // 8-byte Spill
-; CHECK1024-NEXT: str x29, [sp, #1032] // 8-byte Spill
+; CHECK1024-NEXT: stp x29, x30, [x9] // 16-byte Folded Spill
; CHECK1024-NEXT: add x29, sp, #1032
-; CHECK1024-NEXT: str x30, [sp, #1040] // 8-byte Spill
; CHECK1024-NEXT: sub sp, sp, #1040
; CHECK1024-NEXT: .cfi_def_cfa w29, 24
; CHECK1024-NEXT: .cfi_offset w30, -16
@@ -370,9 +370,9 @@ define i32 @csr_d8_allocd_framepointer(double %d) "aarch64_pstate_sm_compatible"
; CHECK1024-NEXT: //NO_APP
; CHECK1024-NEXT: str d0, [sp, #1032]
; CHECK1024-NEXT: add sp, sp, #1040
-; CHECK1024-NEXT: ldr x30, [sp, #1040] // 8-byte Reload
-; CHECK1024-NEXT: ldr x29, [sp, #1032] // 8-byte Reload
+; CHECK1024-NEXT: add x30, sp, #1032
; CHECK1024-NEXT: ldr d8, [sp] // 8-byte Reload
+; CHECK1024-NEXT: ldp x29, x30, [x30] // 16-byte Folded Reload
; CHECK1024-NEXT: add sp, sp, #1056
; CHECK1024-NEXT: ret
entry:
@@ -549,18 +549,18 @@ define i32 @csr_x18_25_d8_15_allocdi64(i64 %d, double %e) "aarch64_pstate_sm_com
; CHECK1024-LABEL: csr_x18_25_d8_15_allocdi64:
; CHECK1024: // %bb.0: // %entry
; CHECK1024-NEXT: sub sp, sp, #1152
+; CHECK1024-NEXT: add x9, sp, #1088
; CHECK1024-NEXT: stp d15, d14, [sp] // 16-byte Folded Spill
+; CHECK1024-NEXT: stp x29, x25, [x9] // 16-byte Folded Spill
+; CHECK1024-NEXT: add x9, sp, #1104
+; CHECK1024-NEXT: stp x24, x23, [x9] // 16-byte Folded Spill
+; CHECK1024-NEXT: add x9, sp, #1120
+; CHECK1024-NEXT: stp x22, x21, [x9] // 16-byte Folded Spill
+; CHECK1024-NEXT: add x9, sp, #1136
; CHECK1024-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill
; CHECK1024-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill
; CHECK1024-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK1024-NEXT: str x29, [sp, #1088] // 8-byte Spill
-; CHECK1024-NEXT: str x25, [sp, #1096] // 8-byte Spill
-; CHECK1024-NEXT: str x24, [sp, #1104] // 8-byte Spill
-; CHECK1024-NEXT: str x23, [sp, #1112] // 8-byte Spill
-; CHECK1024-NEXT: str x22, [sp, #1120] // 8-byte Spill
-; CHECK1024-NEXT: str x21, [sp, #1128] // 8-byte Spill
-; CHECK1024-NEXT: str x20, [sp, #1136] // 8-byte Spill
-; CHECK1024-NEXT: str x19, [sp, #1144] // 8-byte Spill
+; CHECK1024-NEXT: stp x20, x19, [x9] // 16-byte Folded Spill
; CHECK1024-NEXT: sub sp, sp, #1056
; CHECK1024-NEXT: .cfi_def_cfa_offset 2208
; CHECK1024-NEXT: .cfi_offset w19, -8
@@ -588,16 +588,16 @@ define i32 @csr_x18_25_d8_15_allocdi64(i64 %d, double %e) "aarch64_pstate_sm_com
; CHECK1024-NEXT: str x8, [sp, #8]
; CHECK1024-NEXT: str d0, [sp, #1048]
; CHECK1024-NEXT: add sp, sp, #1056
+; CHECK1024-NEXT: add x19, sp, #1136
+; CHECK1024-NEXT: add x21, sp, #1120
+; CHECK1024-NEXT: add x23, sp, #1104
+; CHECK1024-NEXT: add x25, sp, #1088
+; CHECK1024-NEXT: ldp x20, x19, [x19] // 16-byte Folded Reload
+; CHECK1024-NEXT: ldp x22, x21, [x21] // 16-byte Folded Reload
+; CHECK1024-NEXT: ldp x24, x23, [x23] // 16-byte Folded Reload
+; CHECK1024-NEXT: ldp x29, x25, [x25] // 16-byte Folded Reload
; CHECK1024-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload
-; CHECK1024-NEXT: ldr x19, [sp, #1144] // 8-byte Reload
; CHECK1024-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload
-; CHECK1024-NEXT: ldr x20, [sp, #1136] // 8-byte Reload
-; CHECK1024-NEXT: ldr x21, [sp, #1128] // 8-byte Reload
-; CHECK1024-NEXT: ldr x22, [sp, #1120] // 8-byte Reload
-; CHECK1024-NEXT: ldr x23, [sp, #1112] // 8-byte Reload
-; CHECK1024-NEXT: ldr x24, [sp, #1104] // 8-byte Reload
-; CHECK1024-NEXT: ldr x25, [sp, #1096] // 8-byte Reload
-; CHECK1024-NEXT: ldr x29, [sp, #1088] // 8-byte Reload
; CHECK1024-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; CHECK1024-NEXT: ldp d15, d14, [sp] // 16-byte Folded Reload
; CHECK1024-NEXT: add sp, sp, #1152
@@ -765,21 +765,21 @@ define i32 @csr_x18_25_d8_15_allocdi64_locallystreaming(i64 %d, double %e) "aarc
; CHECK1024: // %bb.0: // %entry
; CHECK1024-NEXT: sub sp, sp, #1168
; CHECK1024-NEXT: .cfi_def_cfa_offset 1168
-; CHECK1024-NEXT: cntd x9
+; CHECK1024-NEXT: add x9, sp, #1088
+; CHECK1024-NEXT: add x10, sp, #1104
; CHECK1024-NEXT: stp d15, d14, [sp] // 16-byte Folded Spill
+; CHECK1024-NEXT: stp x29, x30, [x9] // 16-byte Folded Spill
+; CHECK1024-NEXT: cntd x9
+; CHECK1024-NEXT: stp x9, x25, [x10] // 16-byte Folded Spill
+; CHECK1024-NEXT: add x9, sp, #1120
+; CHECK1024-NEXT: stp x24, x23, [x9] // 16-byte Folded Spill
+; CHECK1024-NEXT: add x9, sp, #1136
+; CHECK1024-NEXT: stp x22, x21, [x9] // 16-byte Folded Spill
+; CHECK1024-NEXT: add x9, sp, #1152
; CHECK1024-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill
; CHECK1024-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill
; CHECK1024-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK1024-NEXT: str x29, [sp, #1088] // 8-byte Spill
-; CHECK1024-NEXT: str x30, [sp, #1096] // 8-byte Spill
-; CHECK1024-NEXT: str x9, [sp, #1104] // 8-byte Spill
-; CHECK1024-NEXT: str x25, [sp, #1112] // 8-byte Spill
-; CHECK1024-NEXT: str x24, [sp, #1120] // 8-byte Spill
-; CHECK1024-NEXT: str x23, [sp, #1128] // 8-byte Spill
-; CHECK1024-NEXT: str x22, [sp, #1136] // 8-byte Spill
-; CHECK1024-NEXT: str x21, [sp, #1144] // 8-byte Spill
-; CHECK1024-NEXT: str x20, [sp, #1152] // 8-byte Spill
-; CHECK1024-NEXT: str x19, [sp, #1160] // 8-byte Spill
+; CHECK1024-NEXT: stp x20, x19, [x9] // 16-byte Folded Spill
; CHECK1024-NEXT: .cfi_offset w19, -8
; CHECK1024-NEXT: .cfi_offset w20, -16
; CHECK1024-NEXT: .cfi_offset w21, -24
@@ -813,17 +813,17 @@ define i32 @csr_x18_25_d8_15_allocdi64_locallystreaming(i64 %d, double %e) "aarc
; CHECK1024-NEXT: mov w0, wzr
; CHECK1024-NEXT: add sp, sp, #1056
; CHECK1024-NEXT: .cfi_def_cfa_offset 1168
+; CHECK1024-NEXT: add x19, sp, #1152
+; CHECK1024-NEXT: add x21, sp, #1136
+; CHECK1024-NEXT: add x23, sp, #1120
+; CHECK1024-NEXT: add x30, sp, #1088
+; CHECK1024-NEXT: ldp x20, x19, [x19] // 16-byte Folded Reload
+; CHECK1024-NEXT: ldp x22, x21, [x21] // 16-byte Folded Reload
+; CHECK1024-NEXT: ldr x25, [sp, #1112] // 8-byte Reload
+; CHECK1024-NEXT: ldp x24, x23, [x23] // 16-byte Folded Reload
+; CHECK1024-NEXT: ldp x29, x30, [x30] // 16-byte Folded Reload
; CHECK1024-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload
-; CHECK1024-NEXT: ldr x19, [sp, #1160] // 8-byte Reload
; CHECK1024-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload
-; CHECK1024-NEXT: ldr x20, [sp, #1152] // 8-byte Reload
-; CHECK1024-NEXT: ldr x21, [sp, #1144] // 8-byte Reload
-; CHECK1024-NEXT: ldr x22, [sp, #1136] // 8-byte Reload
-; CHECK1024-NEXT: ldr x23, [sp, #1128] // 8-byte Reload
-; CHECK1024-NEXT: ldr x24, [sp, #1120] // 8-byte Reload
-; CHECK1024-NEXT: ldr x25, [sp, #1112] // 8-byte Reload
-; CHECK1024-NEXT: ldr x30, [sp, #1096] // 8-byte Reload
-; CHECK1024-NEXT: ldr x29, [sp, #1088] // 8-byte Reload
; CHECK1024-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; CHECK1024-NEXT: ldp d15, d14, [sp] // 16-byte Folded Reload
; CHECK1024-NEXT: add sp, sp, #1168
@@ -1531,14 +1531,14 @@ define i32 @svecc_csr_x18_25_d8_15_allocdi64(i64 %d, double %e, <vscale x 4 x i3
; CHECK1024-NOSPLITSVE-LABEL: svecc_csr_x18_25_d8_15_allocdi64:
; CHECK1024-NOSPLITSVE: // %bb.0: // %entry
; CHECK1024-NOSPLITSVE-NEXT: sub sp, sp, #1088
-; CHECK1024-NOSPLITSVE-NEXT: str x29, [sp, #1024] // 8-byte Spill
-; CHECK1024-NOSPLITSVE-NEXT: str x25, [sp, #1032] // 8-byte Spill
-; CHECK1024-NOSPLITSVE-NEXT: str x24, [sp, #1040] // 8-byte Spill
-; CHECK1024-NOSPLITSVE-NEXT: str x23, [sp, #1048] // 8-byte Spill
-; CHECK1024-NOSPLITSVE-NEXT: str x22, [sp, #1056] // 8-byte Spill
-; CHECK1024-NOSPLITSVE-NEXT: str x21, [sp, #1064] // 8-byte Spill
-; CHECK1024-NOSPLITSVE-NEXT: str x20, [sp, #1072] // 8-byte Spill
-; CHECK1024-NOSPLITSVE-NEXT: str x19, [sp, #1080] // 8-byte Spill
+; CHECK1024-NOSPLITSVE-NEXT: add x9, sp, #1024
+; CHECK1024-NOSPLITSVE-NEXT: stp x29, x25, [x9] // 16-byte Folded Spill
+; CHECK1024-NOSPLITSVE-NEXT: add x9, sp, #1040
+; CHECK1024-NOSPLITSVE-NEXT: stp x24, x23, [x9] // 16-byte Folded Spill
+; CHECK1024-NOSPLITSVE-NEXT: add x9, sp, #1056
+; CHECK1024-NOSPLITSVE-NEXT: stp x22, x21, [x9] // 16-byte Folded Spill
+; CHECK1024-NOSPLITSVE-NEXT: add x9, sp, #1072
+; CHECK1024-NOSPLITSVE-NEXT: stp x20, x19, [x9] // 16-byte Folded Spill
; CHECK1024-NOSPLITSVE-NEXT: addvl sp, sp, #-8
; CHECK1024-NOSPLITSVE-NEXT: str z15, [sp] // 16-byte Folded Spill
; CHECK1024-NOSPLITSVE-NEXT: str z14, [sp, #1, mul vl] // 16-byte Folded Spill
@@ -1584,14 +1584,14 @@ define i32 @svecc_csr_x18_25_d8_15_allocdi64(i64 %d, double %e, <vscale x 4 x i3
; CHECK1024-NOSPLITSVE-NEXT: ldr z9, [sp, #6, mul vl] // 16-byte Folded Reload
; CHECK1024-NOSPLITSVE-NEXT: ldr z8, [sp, #7, mul vl] // 16-byte Folded Reload
; CHECK1024-NOSPLITSVE-NEXT: addvl sp, sp, #8
-; CHECK1024-NOSPLITSVE-NEXT: ldr x19, [sp, #1080] // 8-byte Reload
-; CHECK1024-NOSPLITSVE-NEXT: ldr x20, [sp, #1072] // 8-byte Reload
-; CHECK1024-NOSPLITSVE-NEXT: ldr x21, [sp, #1064] // 8-byte Reload
-; CHECK1024-NOSPLITSVE-NEXT: ldr x22, [sp, #1056] // 8-byte Reload
-; CHECK1024-NOSPLITSVE-NEXT: ldr x23, [sp, #1048] // 8-byte Reload
-; CHECK1024-NOSPLITSVE-NEXT: ldr x24, [sp, #1040] // 8-byte Reload
-; CHECK1024-NOSPLITSVE-NEXT: ldr x25, [sp, #1032] // 8-byte Reload
-; CHECK1024-NOSPLITSVE-NEXT: ldr x29, [sp, #1024] // 8-byte Reload
+; CHECK1024-NOSPLITSVE-NEXT: add x19, sp, #1072
+; CHECK1024-NOSPLITSVE-NEXT: add x21, sp, #1056
+; CHECK1024-NOSPLITSVE-NEXT: add x23, sp, #1040
+; CHECK1024-NOSPLITSVE-NEXT: add x25, sp, #1024
+; CHECK1024-NOSPLITSVE-NEXT: ldp x20, x19, [x19] // 16-byte Folded Reload
+; CHECK1024-NOSPLITSVE-NEXT: ldp x22, x21, [x21] // 16-byte Folded Reload
+; CHECK1024-NOSPLITSVE-NEXT: ldp x24, x23, [x23] // 16-byte Folded Reload
+; CHECK1024-NOSPLITSVE-NEXT: ldp x29, x25, [x25] // 16-byte Folded Reload
; CHECK1024-NOSPLITSVE-NEXT: add sp, sp, #1088
; CHECK1024-NOSPLITSVE-NEXT: ret
;
@@ -1872,19 +1872,19 @@ define i32 @f128_libcall(fp128 %v0, fp128 %v1, fp128 %v2, fp128 %v3, i32 %a, i32
; CHECK1024: // %bb.0:
; CHECK1024-NEXT: sub sp, sp, #1152
; CHECK1024-NEXT: .cfi_def_cfa_offset 1152
-; CHECK1024-NEXT: cntd x9
+; CHECK1024-NEXT: add x9, sp, #1088
+; CHECK1024-NEXT: add x10, sp, #1104
; CHECK1024-NEXT: stp d15, d14, [sp] // 16-byte Folded Spill
+; CHECK1024-NEXT: stp x29, x30, [x9] // 16-byte Folded Spill
+; CHECK1024-NEXT: cntd x9
+; CHECK1024-NEXT: stp x9, x28, [x10] // 16-byte Folded Spill
+; CHECK1024-NEXT: add x9, sp, #1120
+; CHECK1024-NEXT: stp x22, x21, [x9] // 16-byte Folded Spill
+; CHECK1024-NEXT: add x9, sp, #1136
; CHECK1024-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill
; CHECK1024-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill
; CHECK1024-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK1024-NEXT: str x29, [sp, #1088] // 8-byte Spill
-; CHECK1024-NEXT: str x30, [sp, #1096] // 8-byte Spill
-; CHECK1024-NEXT: str x9, [sp, #1104] // 8-byte Spill
-; CHECK1024-NEXT: str x28, [sp, #1112] // 8-byte Spill
-; CHECK1024-NEXT: str x22, [sp, #1120] // 8-byte Spill
-; CHECK1024-NEXT: str x21, [sp, #1128] // 8-byte Spill
-; CHECK1024-NEXT: str x20, [sp, #1136] // 8-byte Spill
-; CHECK1024-NEXT: str x19, [sp, #1144] // 8-byte Spill
+; CHECK1024-NEXT: stp x20, x19, [x9] // 16-byte Folded Spill
; CHECK1024-NEXT: .cfi_offset w19, -8
; CHECK1024-NEXT: .cfi_offset w20, -16
; CHECK1024-NEXT: .cfi_offset w21, -24
@@ -1903,20 +1903,20 @@ define i32 @f128_libcall(fp128 %v0, fp128 %v1, fp128 %v2, fp128 %v3, i32 %a, i32
; CHECK1024-NEXT: .cfi_offset b15, -1152
; CHECK1024-NEXT: sub sp, sp, #1088
; CHECK1024-NEXT: .cfi_def_cfa_offset 2240
+; CHECK1024-NEXT: add x9, sp, #1056
; CHECK1024-NEXT: mov w19, w1
; CHECK1024-NEXT: mov w20, w0
-; CHECK1024-NEXT: str q3, [sp, #1072] // 16-byte Spill
-; CHECK1024-NEXT: str q2, [sp, #1056] // 16-byte Spill
-; CHECK1024-NEXT: str q1, [sp, #1040] // 16-byte Spill
-; CHECK1024-NEXT: str q0, [sp, #1024] // 16-byte Spill
+; CHECK1024-NEXT: stp q2, q3, [x9] // 32-byte Folded Spill
+; CHECK1024-NEXT: add x9, sp, #1024
+; CHECK1024-NEXT: stp q0, q1, [x9] // 32-byte Folded Spill
; CHECK1024-NEXT: bl __arm_sme_state
; CHECK1024-NEXT: mov x21, x0
; CHECK1024-NEXT: tbz w21, #0, .LBB27_2
; CHECK1024-NEXT: // %bb.1:
; CHECK1024-NEXT: smstop sm
; CHECK1024-NEXT: .LBB27_2:
-; CHECK1024-NEXT: ldr q0, [sp, #1024] // 16-byte Reload
-; CHECK1024-NEXT: ldr q1, [sp, #1040] // 16-byte Reload
+; CHECK1024-NEXT: add x9, sp, #1024
+; CHECK1024-NEXT: ldp q0, q1, [x9] // 32-byte Folded Reload
; CHECK1024-NEXT: bl __lttf2
; CHECK1024-NEXT: mov w22, w0
; CHECK1024-NEXT: tbz w21, #0, .LBB27_4
@@ -1927,8 +1927,8 @@ define i32 @f128_libcall(fp128 %v0, fp128 %v1, fp128 %v2, fp128 %v3, i32 %a, i32
; CHECK1024-NEXT: // %bb.5:
; CHECK1024-NEXT: smstop sm
; CHECK1024-NEXT: .LBB27_6:
-; CHECK1024-NEXT: ldr q0, [sp, #1056] // 16-byte Reload
-; CHECK1024-NEXT: ldr q1, [sp, #1072] // 16-byte Reload
+; CHECK1024-NEXT: add x9, sp, #1056
+; CHECK1024-NEXT: ldp q0, q1, [x9] // 32-byte Folded Reload
; CHECK1024-NEXT: bl __getf2
; CHECK1024-NEXT: tbz w21, #0, .LBB27_8
; CHECK1024-NEXT: // %bb.7:
@@ -1939,15 +1939,15 @@ define i32 @f128_libcall(fp128 %v0, fp128 %v1, fp128 %v2, fp128 %v3, i32 %a, i32
; CHECK1024-NEXT: csel w0, w20, w19, mi
; CHECK1024-NEXT: add sp, sp, #1088
; CHECK1024-NEXT: .cfi_def_cfa_offset 1152
+; CHECK1024-NEXT: add x19, sp, #1136
+; CHECK1024-NEXT: add x21, sp, #1120
+; CHECK1024-NEXT: add x30, sp, #1088
+; CHECK1024-NEXT: ldp x20, x19, [x19] // 16-byte Folded Reload
+; CHECK1024-NEXT: ldr x28, [sp, #1112] // 8-byte Reload
+; CHECK1024-NEXT: ldp x22, x21, [x21] // 16-byte Folded Reload
+; CHECK1024-NEXT: ldp x29, x30, [x30] // 16-byte Folded Reload
; CHECK1024-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload
-; CHECK1024-NEXT: ldr x19, [sp, #1144] // 8-byte Reload
; CHECK1024-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload
-; CHECK1024-NEXT: ldr x20, [sp, #1136] // 8-byte Reload
-; CHECK1024-NEXT: ldr x21, [sp, #1128] // 8-byte Reload
-; CHECK1024-NEXT: ldr x22, [sp, #1120] // 8-byte Reload
-; CHECK1024-NEXT: ldr x28, [sp, #1112] // 8-byte Reload
-; CHECK1024-NEXT: ldr x30, [sp, #1096] // 8-byte Reload
-; CHECK1024-NEXT: ldr x29, [sp, #1088] // 8-byte Reload
; CHECK1024-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; CHECK1024-NEXT: ldp d15, d14, [sp] // 16-byte Folded Reload
; CHECK1024-NEXT: add sp, sp, #1152
@@ -2236,14 +2236,14 @@ define i32 @svecc_call(<4 x i16> %P0, ptr %P1, i32 %P2, <vscale x 16 x i8> %P3,
; CHECK1024-NOSPLITSVE: // %bb.0: // %entry
; CHECK1024-NOSPLITSVE-NEXT: sub sp, sp, #1088
; CHECK1024-NOSPLITSVE-NEXT: .cfi_def_cfa_offset 1088
-; CHECK1024-NOSPLITSVE-NEXT: cntd x9
-; CHECK1024-NOSPLITSVE-NEXT: str x29, [sp, #1024] // 8-byte Spill
-; CHECK1024-NOSPLITSVE-NEXT: str x30, [sp, #1032] // 8-byte Spill
-; CHECK1024-NOSPLITSVE-NEXT: str x9, [sp, #1040] // 8-byte Spill
-; CHECK1024-NOSPLITSVE-NEXT: str x28, [sp, #1048] // 8-byte Spill
-; CHECK1024-NOSPLITSVE-NEXT: str x27, [sp, #1056] // 8-byte Spill
-; CHECK1024-NOSPLITSVE-NEXT: str x26, [sp, #1064] // 8-byte Spill
+; CHECK1024-NOSPLITSVE-NEXT: add x9, sp, #1024
+; CHECK1024-NOSPLITSVE-NEXT: add x10, sp, #1040
; CHECK1024-NOSPLITSVE-NEXT: str x19, [sp, #1072] // 8-byte Spill
+; CHECK1024-NOSPLITSVE-NEXT: stp x29, x30, [x9] // 16-byte Folded Spill
+; CHECK1024-NOSPLITSVE-NEXT: cntd x9
+; CHECK1024-NOSPLITSVE-NEXT: stp x9, x28, [x10] // 16-byte Folded Spill
+; CHECK1024-NOSPLITSVE-NEXT: add x9, sp, #1056
+; CHECK1024-NOSPLITSVE-NEXT: stp x27, x26, [x9] // 16-byte Folded Spill
; CHECK1024-NOSPLITSVE-NEXT: add x29, sp, #1024
; CHECK1024-NOSPLITSVE-NEXT: .cfi_def_cfa w29, 64
; CHECK1024-NOSPLITSVE-NEXT: .cfi_offset w19, -16
@@ -2349,12 +2349,12 @@ define i32 @svecc_call(<4 x i16> %P0, ptr %P1, i32 %P2, <vscale x 16 x i8> %P3,
; CHECK1024-NOSPLITSVE-NEXT: .cfi_restore z14
; CHECK1024-NOSPLITSVE-NEXT: .cfi_restore z15
; CHECK1024-NOSPLITSVE-NEXT: .cfi_def_cfa wsp, 1088
-; CHECK1024-NOSPLITSVE-NEXT: ldr x19, [sp, #1072] // 8-byte Reload
-; CHECK1024-NOSPLITSVE-NEXT: ldr x26, [sp, #1064] // 8-byte Reload
-; CHECK1024-NOSPLITSVE-NEXT: ldr x27, [sp, #1056] // 8-byte Reload
-; CHECK1024-NOSPLITSVE-NEXT: ldr x28, [sp, #1048] // 8-byte Reload
-; CHECK1024-NOSPLITSVE-NEXT: ldr x30, [sp, #1032] // 8-byte Reload
-; CHECK1024-NOSPLITSVE-NEXT: ldr x29, [sp, #1024] // 8-byte Reload
+; CHECK1024-NOSPLITSVE-NEXT: add x19, sp, #1064
+; CHECK1024-NOSPLITSVE-NEXT: add x27, sp, #1048
+; CHECK1024-NOSPLITSVE-NEXT: add x30, sp, #1024
+; CHECK1024-NOSPLITSVE-NEXT: ldp x26, x19, [x19] // 16-byte Folded Reload
+; CHECK1024-NOSPLITSVE-NEXT: ldp x28, x27, [x27] // 16-byte Folded Reload
+; CHECK1024-NOSPLITSVE-NEXT: ldp x29, x30, [x30] // 16-byte Folded Reload
; CHECK1024-NOSPLITSVE-NEXT: add sp, sp, #1088
; CHECK1024-NOSPLITSVE-NEXT: .cfi_def_cfa_offset 0
; CHECK1024-NOSPLITSVE-NEXT: .cfi_restore w19
@@ -2760,14 +2760,14 @@ define i32 @svecc_alloca_call(<4 x i16> %P0, ptr %P1, i32 %P2, <vscale x 16 x i8
; CHECK1024-NOSPLITSVE: // %bb.0: // %entry
; CHECK1024-NOSPLITSVE-NEXT: sub sp, sp, #1088
; CHECK1024-NOSPLITSVE-NEXT: .cfi_def_cfa_offset 1088
-; CHECK1024-NOSPLITSVE-NEXT: cntd x9
-; CHECK1024-NOSPLITSVE-NEXT: str x29, [sp, #1024] // 8-byte Spill
-; CHECK1024-NOSPLITSVE-NEXT: str x30, [sp, #1032] // 8-byte Spill
-; CHECK1024-NOSPLITSVE-NEXT: str x9, [sp, #1040] // 8-byte Spill
-; CHECK1024-NOSPLITSVE-NEXT: str x28, [sp, #1048] // 8-byte Spill
-; CHECK1024-NOSPLITSVE-NEXT: str x27, [sp, #1056] // 8-byte Spill
-; CHECK1024-NOSPLITSVE-NEXT: str x26, [sp, #1064] // 8-byte Spill
+; CHECK1024-NOSPLITSVE-NEXT: add x9, sp, #1024
+; CHECK1024-NOSPLITSVE-NEXT: add x10, sp, #1040
; CHECK1024-NOSPLITSVE-NEXT: str x19, [sp, #1072] // 8-byte Spill
+; CHECK1024-NOSPLITSVE-NEXT: stp x29, x30, [x9] // 16-byte Folded Spill
+; CHECK1024-NOSPLITSVE-NEXT: cntd x9
+; CHECK1024-NOSPLITSVE-NEXT: stp x9, x28, [x10] // 16-byte Folded Spill
+; CHECK1024-NOSPLITSVE-NEXT: add x9, sp, #1056
+; CHECK1024-NOSPLITSVE-NEXT: stp x27, x26, [x9] // 16-byte Folded Spill
; CHECK1024-NOSPLITSVE-NEXT: add x29, sp, #1024
; CHECK1024-NOSPLITSVE-NEXT: .cfi_def_cfa w29, 64
; CHECK1024-NOSPLITSVE-NEXT: .cfi_offset w19, -16
@@ -2872,12 +2872,12 @@ define i32 @svecc_alloca_call(<4 x i16> %P0, ptr %P1, i32 %P2, <vscale x 16 x i8
; CHECK1024-NOSPLITSVE-NEXT: .cfi_restore z14
; CHECK1024-NOSPLITSVE-NEXT: .cfi_restore z15
; CHECK1024-NOSPLITSVE-NEXT: .cfi_def_cfa wsp, 1088
-; CHECK1024-NOSPLITSVE-NEXT: ldr x19, [sp, #1072] // 8-byte Reload
-; CHECK1024-NOSPLITSVE-NEXT: ldr x26, [sp, #1064] // 8-byte Reload
-; CHECK1024-NOSPLITSVE-NEXT: ldr x27, [sp, #1056] // 8-byte Reload
-; CHECK1024-NOSPLITSVE-NEXT: ldr x28, [sp, #1048] // 8-byte Reload
-; CHECK1024-NOSPLITSVE-NEXT: ldr x30, [sp, #1032] // 8-byte Reload
-; CHECK1024-NOSPLITSVE-NEXT: ldr x29, [sp, #1024] // 8-byte Reload
+; CHECK1024-NOSPLITSVE-NEXT: add x19, sp, #1064
+; CHECK1024-NOSPLITSVE-NEXT: add x27, sp, #1048
+; CHECK1024-NOSPLITSVE-NEXT: add x30, sp, #1024
+; CHECK1024-NOSPLITSVE-NEXT: ldp x26, x19, [x19] // 16-byte Folded Reload
+; CHECK1024-NOSPLITSVE-NEXT: ldp x28, x27, [x27] // 16-byte Folded Reload
+; CHECK1024-NOSPLITSVE-NEXT: ldp x29, x30, [x30] // 16-byte Folded Reload
; CHECK1024-NOSPLITSVE-NEXT: add sp, sp, #1088
; CHECK1024-NOSPLITSVE-NEXT: .cfi_def_cfa_offset 0
; CHECK1024-NOSPLITSVE-NEXT: .cfi_restore w19
@@ -3064,9 +3064,9 @@ define void @call_with_doubles() "aarch64_pstate_sm_compatible" {
; CHECK1024-LABEL: call_with_doubles:
; CHECK1024: // %bb.0: // %entry
; CHECK1024-NEXT: sub sp, sp, #1056
+; CHECK1024-NEXT: add x9, sp, #1032
; CHECK1024-NEXT: str d8, [sp] // 8-byte Spill
-; CHECK1024-NEXT: str x29, [sp, #1032] // 8-byte Spill
-; CHECK1024-NEXT: str x30, [sp, #1040] // 8-byte Spill
+; CHECK1024-NEXT: stp x29, x30, [x9] // 16-byte Folded Spill
; CHECK1024-NEXT: sub sp, sp, #1024
; CHECK1024-NEXT: .cfi_def_cfa_offset 2080
; CHECK1024-NEXT: .cfi_offset w30, -16
@@ -3078,9 +3078,9 @@ define void @call_with_doubles() "aarch64_pstate_sm_compatible" {
; CHECK1024-NEXT: bl calld
; CHECK1024-NEXT: fmov d0, d8
; CHECK1024-NEXT: add sp, sp, #1024
-; CHECK1024-NEXT: ldr x30, [sp, #1040] // 8-byte Reload
-; CHECK1024-NEXT: ldr x29, [sp, #1032] // 8-byte Reload
+; CHECK1024-NEXT: add x30, sp, #1032
; CHECK1024-NEXT: ldr d8, [sp] // 8-byte Reload
+; CHECK1024-NEXT: ldp x29, x30, [x30] // 16-byte Folded Reload
; CHECK1024-NEXT: add sp, sp, #1056
; CHECK1024-NEXT: b calld
entry:
@@ -3351,17 +3351,17 @@ define i32 @vastate(i32 %x) "aarch64_inout_za" "aarch64_pstate_sm_enabled" "targ
; CHECK1024: // %bb.0: // %entry
; CHECK1024-NEXT: sub sp, sp, #1136
; CHECK1024-NEXT: .cfi_def_cfa_offset 1136
-; CHECK1024-NEXT: cntd x9
+; CHECK1024-NEXT: add x9, sp, #1088
+; CHECK1024-NEXT: add x10, sp, #1104
; CHECK1024-NEXT: stp d15, d14, [sp] // 16-byte Folded Spill
+; CHECK1024-NEXT: stp x29, x30, [x9] // 16-byte Folded Spill
+; CHECK1024-NEXT: cntd x9
+; CHECK1024-NEXT: stp x9, x28, [x10] // 16-byte Folded Spill
+; CHECK1024-NEXT: add x9, sp, #1120
; CHECK1024-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill
; CHECK1024-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill
; CHECK1024-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK1024-NEXT: str x29, [sp, #1088] // 8-byte Spill
-; CHECK1024-NEXT: str x30, [sp, #1096] // 8-byte Spill
-; CHECK1024-NEXT: str x9, [sp, #1104] // 8-byte Spill
-; CHECK1024-NEXT: str x28, [sp, #1112] // 8-byte Spill
-; CHECK1024-NEXT: str x20, [sp, #1120] // 8-byte Spill
-; CHECK1024-NEXT: str x19, [sp, #1128] // 8-byte Spill
+; CHECK1024-NEXT: stp x20, x19, [x9] // 16-byte Folded Spill
; CHECK1024-NEXT: add x29, sp, #1088
; CHECK1024-NEXT: .cfi_def_cfa w29, 48
; CHECK1024-NEXT: .cfi_offset w19, -8
@@ -3402,13 +3402,13 @@ define i32 @vastate(i32 %x) "aarch64_inout_za" "aarch64_pstate_sm_enabled" "targ
; CHECK1024-NEXT: msr TPIDR2_EL0, xzr
; CHECK1024-NEXT: sub sp, x29, #1088
; CHECK1024-NEXT: .cfi_def_cfa wsp, 1136
+; CHECK1024-NEXT: add x19, sp, #1120
+; CHECK1024-NEXT: add x30, sp, #1088
+; CHECK1024-NEXT: ldr x28, [sp, #1112] // 8-byte Reload
+; CHECK1024-NEXT: ldp x20, x19, [x19] // 16-byte Folded Reload
+; CHECK1024-NEXT: ldp x29, x30, [x30] // 16-byte Folded Reload
; CHECK1024-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload
-; CHECK1024-NEXT: ldr x19, [sp, #1128] // 8-byte Reload
; CHECK1024-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload
-; CHECK1024-NEXT: ldr x20, [sp, #1120] // 8-byte Reload
-; CHECK1024-NEXT: ldr x28, [sp, #1112] // 8-byte Reload
-; CHECK1024-NEXT: ldr x30, [sp, #1096] // 8-byte Reload
-; CHECK1024-NEXT: ldr x29, [sp, #1088] // 8-byte Reload
; CHECK1024-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; CHECK1024-NEXT: ldp d15, d14, [sp] // 16-byte Folded Reload
; CHECK1024-NEXT: add sp, sp, #1136
@@ -3498,11 +3498,11 @@ define i32 @sve_stack_object_and_vla(double %d, i64 %sz) "aarch64_pstate_sm_comp
; CHECK1024-LABEL: sve_stack_object_and_vla:
; CHECK1024: // %bb.0: // %entry
; CHECK1024-NEXT: sub sp, sp, #1056
-; CHECK1024-NEXT: str x29, [sp, #1024] // 8-byte Spill
+; CHECK1024-NEXT: add x9, sp, #1024
+; CHECK1024-NEXT: stp x29, x30, [x9] // 16-byte Folded Spill
+; CHECK1024-NEXT: add x9, sp, #1040
; CHECK1024-NEXT: add x29, sp, #1024
-; CHECK1024-NEXT: str x30, [sp, #1032] // 8-byte Spill
-; CHECK1024-NEXT: str x28, [sp, #1040] // 8-byte Spill
-; CHECK1024-NEXT: str x19, [sp, #1048] // 8-byte Spill
+; CHECK1024-NEXT: stp x28, x19, [x9] // 16-byte Folded Spill
; CHECK1024-NEXT: sub sp, sp, #1024
; CHECK1024-NEXT: addvl sp, sp, #-1
; CHECK1024-NEXT: mov x19, sp
@@ -3523,10 +3523,10 @@ define i32 @sve_stack_object_and_vla(double %d, i64 %sz) "aarch64_pstate_sm_comp
; CHECK1024-NEXT: bl bar
; CHECK1024-NEXT: mov w0, wzr
; CHECK1024-NEXT: sub sp, x29, #1024
-; CHECK1024-NEXT: ldr x19, [sp, #1048] // 8-byte Reload
-; CHECK1024-NEXT: ldr x28, [sp, #1040] // 8-byte Reload
-; CHECK1024-NEXT: ldr x30, [sp, #1032] // 8-byte Reload
-; CHECK1024-NEXT: ldr x29, [sp, #1024] // 8-byte Reload
+; CHECK1024-NEXT: add x19, sp, #1040
+; CHECK1024-NEXT: add x30, sp, #1024
+; CHECK1024-NEXT: ldp x28, x19, [x19] // 16-byte Folded Reload
+; CHECK1024-NEXT: ldp x29, x30, [x30] // 16-byte Folded Reload
; CHECK1024-NEXT: add sp, sp, #1056
; CHECK1024-NEXT: ret
entry:
@@ -3818,15 +3818,15 @@ define i32 @svecc_call_dynamic_alloca(<4 x i16> %P0, i32 %P1, i32 %P2, <vscale x
; CHECK1024-NOSPLITSVE: // %bb.0: // %entry
; CHECK1024-NOSPLITSVE-NEXT: sub sp, sp, #1088
; CHECK1024-NOSPLITSVE-NEXT: .cfi_def_cfa_offset 1088
+; CHECK1024-NOSPLITSVE-NEXT: add x9, sp, #1024
+; CHECK1024-NOSPLITSVE-NEXT: add x10, sp, #1040
+; CHECK1024-NOSPLITSVE-NEXT: stp x29, x30, [x9] // 16-byte Folded Spill
; CHECK1024-NOSPLITSVE-NEXT: cntd x9
-; CHECK1024-NOSPLITSVE-NEXT: str x29, [sp, #1024] // 8-byte Spill
-; CHECK1024-NOSPLITSVE-NEXT: str x30, [sp, #1032] // 8-byte Spill
-; CHECK1024-NOSPLITSVE-NEXT: str x9, [sp, #1040] // 8-byte Spill
-; CHECK1024-NOSPLITSVE-NEXT: str x28, [sp, #1048] // 8-byte Spill
-; CHECK1024-NOSPLITSVE-NEXT: str x27, [sp, #1056] // 8-byte Spill
-; CHECK1024-NOSPLITSVE-NEXT: str x26, [sp, #1064] // 8-byte Spill
-; CHECK1024-NOSPLITSVE-NEXT: str x20, [sp, #1072] // 8-byte Spill
-; CHECK1024-NOSPLITSVE-NEXT: str x19, [sp, #1080] // 8-byte Spill
+; CHECK1024-NOSPLITSVE-NEXT: stp x9, x28, [x10] // 16-byte Folded Spill
+; CHECK1024-NOSPLITSVE-NEXT: add x9, sp, #1056
+; CHECK1024-NOSPLITSVE-NEXT: stp x27, x26, [x9] // 16-byte Folded Spill
+; CHECK1024-NOSPLITSVE-NEXT: add x9, sp, #1072
+; CHECK1024-NOSPLITSVE-NEXT: stp x20, x19, [x9] // 16-byte Folded Spill
; CHECK1024-NOSPLITSVE-NEXT: add x29, sp, #1024
; CHECK1024-NOSPLITSVE-NEXT: .cfi_def_cfa w29, 64
; CHECK1024-NOSPLITSVE-NEXT: .cfi_offset w19, -8
@@ -3941,13 +3941,13 @@ define i32 @svecc_call_dynamic_alloca(<4 x i16> %P0, i32 %P1, i32 %P2, <vscale x
; CHECK1024-NOSPLITSVE-NEXT: .cfi_restore z15
; CHECK1024-NOSPLITSVE-NEXT: sub sp, x29, #1024
; CHECK1024-NOSPLITSVE-NEXT: .cfi_def_cfa wsp, 1088
-; CHECK1024-NOSPLITSVE-NEXT: ldr x19, [sp, #1080] // 8-byte Reload
-; CHECK1024-NOSPLITSVE-NEXT: ldr x20, [sp, #1072] // 8-byte Reload
-; CHECK1024-NOSPLITSVE-NEXT: ldr x26, [sp, #1064] // 8-byte Reload
-; CHECK1024-NOSPLITSVE-NEXT: ldr x27, [sp, #1056] // 8-byte Reload
+; CHECK1024-NOSPLITSVE-NEXT: add x19, sp, #1072
+; CHECK1024-NOSPLITSVE-NEXT: add x26, sp, #1056
+; CHECK1024-NOSPLITSVE-NEXT: add x30, sp, #1024
+; CHECK1024-NOSPLITSVE-NEXT: ldp x20, x19, [x19] // 16-byte Folded Reload
; CHECK1024-NOSPLITSVE-NEXT: ldr x28, [sp, #1048] // 8-byte Reload
-; CHECK1024-NOSPLITSVE-NEXT: ldr x30, [sp, #1032] // 8-byte Reload
-; CHECK1024-NOSPLITSVE-NEXT: ldr x29, [sp, #1024] // 8-byte Reload
+; CHECK1024-NOSPLITSVE-NEXT: ldp x27, x26, [x26] // 16-byte Folded Reload
+; CHECK1024-NOSPLITSVE-NEXT: ldp x29, x30, [x30] // 16-byte Folded Reload
; CHECK1024-NOSPLITSVE-NEXT: add sp, sp, #1088
; CHECK1024-NOSPLITSVE-NEXT: .cfi_def_cfa_offset 0
; CHECK1024-NOSPLITSVE-NEXT: .cfi_restore w19
@@ -4368,14 +4368,14 @@ define i32 @svecc_call_realign(<4 x i16> %P0, i32 %P1, i32 %P2, <vscale x 16 x i
; CHECK1024-NOSPLITSVE: // %bb.0: // %entry
; CHECK1024-NOSPLITSVE-NEXT: sub sp, sp, #1088
; CHECK1024-NOSPLITSVE-NEXT: .cfi_def_cfa_offset 1088
-; CHECK1024-NOSPLITSVE-NEXT: cntd x9
-; CHECK1024-NOSPLITSVE-NEXT: str x29, [sp, #1024] // 8-byte Spill
-; CHECK1024-NOSPLITSVE-NEXT: str x30, [sp, #1032] // 8-byte Spill
-; CHECK1024-NOSPLITSVE-NEXT: str x9, [sp, #1040] // 8-byte Spill
-; CHECK1024-NOSPLITSVE-NEXT: str x28, [sp, #1048] // 8-byte Spill
-; CHECK1024-NOSPLITSVE-NEXT: str x27, [sp, #1056] // 8-byte Spill
-; CHECK1024-NOSPLITSVE-NEXT: str x26, [sp, #1064] // 8-byte Spill
+; CHECK1024-NOSPLITSVE-NEXT: add x9, sp, #1024
+; CHECK1024-NOSPLITSVE-NEXT: add x10, sp, #1040
; CHECK1024-NOSPLITSVE-NEXT: str x19, [sp, #1072] // 8-byte Spill
+; CHECK1024-NOSPLITSVE-NEXT: stp x29, x30, [x9] // 16-byte Folded Spill
+; CHECK1024-NOSPLITSVE-NEXT: cntd x9
+; CHECK1024-NOSPLITSVE-NEXT: stp x9, x28, [x10] // 16-byte Folded Spill
+; CHECK1024-NOSPLITSVE-NEXT: add x9, sp, #1056
+; CHECK1024-NOSPLITSVE-NEXT: stp x27, x26, [x9] // 16-byte Folded Spill
; CHECK1024-NOSPLITSVE-NEXT: add x29, sp, #1024
; CHECK1024-NOSPLITSVE-NEXT: .cfi_def_cfa w29, 64
; CHECK1024-NOSPLITSVE-NEXT: .cfi_offset w19, -16
@@ -4482,12 +4482,12 @@ define i32 @svecc_call_realign(<4 x i16> %P0, i32 %P1, i32 %P2, <vscale x 16 x i
; CHECK1024-NOSPLITSVE-NEXT: .cfi_restore z15
; CHECK1024-NOSPLITSVE-NEXT: sub sp, x29, #1024
; CHECK1024-NOSPLITSVE-NEXT: .cfi_def_cfa wsp, 1088
-; CHECK1024-NOSPLITSVE-NEXT: ldr x19, [sp, #1072] // 8-byte Reload
-; CHECK1024-NOSPLITSVE-NEXT: ldr x26, [sp, #1064] // 8-byte Reload
-; CHECK1024-NOSPLITSVE-NEXT: ldr x27, [sp, #1056] // 8-byte Reload
-; CHECK1024-NOSPLITSVE-NEXT: ldr x28, [sp, #1048] // 8-byte Reload
-; CHECK1024-NOSPLITSVE-NEXT: ldr x30, [sp, #1032] // 8-byte Reload
-; CHECK1024-NOSPLITSVE-NEXT: ldr x29, [sp, #1024] // 8-byte Reload
+; CHECK1024-NOSPLITSVE-NEXT: add x19, sp, #1064
+; CHECK1024-NOSPLITSVE-NEXT: add x27, sp, #1048
+; CHECK1024-NOSPLITSVE-NEXT: add x30, sp, #1024
+; CHECK1024-NOSPLITSVE-NEXT: ldp x26, x19, [x19] // 16-byte Folded Reload
+; CHECK1024-NOSPLITSVE-NEXT: ldp x28, x27, [x27] // 16-byte Folded Reload
+; CHECK1024-NOSPLITSVE-NEXT: ldp x29, x30, [x30] // 16-byte Folded Reload
; CHECK1024-NOSPLITSVE-NEXT: add sp, sp, #1088
; CHECK1024-NOSPLITSVE-NEXT: .cfi_def_cfa_offset 0
; CHECK1024-NOSPLITSVE-NEXT: .cfi_restore w19
@@ -4861,14 +4861,14 @@ define i32 @svecc_call_dynamic_and_scalable_alloca(<4 x i16> %P0, i32 %P1, i32 %
; CHECK1024-NOSPLITSVE-LABEL: svecc_call_dynamic_and_scalable_alloca:
; CHECK1024-NOSPLITSVE: // %bb.0: // %entry
; CHECK1024-NOSPLITSVE-NEXT: sub sp, sp, #1088
-; CHECK1024-NOSPLITSVE-NEXT: str x29, [sp, #1024] // 8-byte Spill
-; CHECK1024-NOSPLITSVE-NEXT: add x29, sp, #1024
-; CHECK1024-NOSPLITSVE-NEXT: str x30, [sp, #1032] // 8-byte Spill
-; CHECK1024-NOSPLITSVE-NEXT: str x28, [sp, #1040] // 8-byte Spill
-; CHECK1024-NOSPLITSVE-NEXT: str x27, [sp, #1048] // 8-byte Spill
-; CHECK1024-NOSPLITSVE-NEXT: str x26, [sp, #1056] // 8-byte Spill
-; CHECK1024-NOSPLITSVE-NEXT: str x20, [sp, #1064] // 8-byte Spill
+; CHECK1024-NOSPLITSVE-NEXT: add x9, sp, #1024
; CHECK1024-NOSPLITSVE-NEXT: str x19, [sp, #1072] // 8-byte Spill
+; CHECK1024-NOSPLITSVE-NEXT: stp x29, x30, [x9] // 16-byte Folded Spill
+; CHECK1024-NOSPLITSVE-NEXT: add x9, sp, #1040
+; CHECK1024-NOSPLITSVE-NEXT: add x29, sp, #1024
+; CHECK1024-NOSPLITSVE-NEXT: stp x28, x27, [x9] // 16-byte Folded Spill
+; CHECK1024-NOSPLITSVE-NEXT: add x9, sp, #1056
+; CHECK1024-NOSPLITSVE-NEXT: stp x26, x20, [x9] // 16-byte Folded Spill
; CHECK1024-NOSPLITSVE-NEXT: addvl sp, sp, #-18
; CHECK1024-NOSPLITSVE-NEXT: str p15, [sp, #4, mul vl] // 2-byte Spill
; CHECK1024-NOSPLITSVE-NEXT: str p14, [sp, #5, mul vl] // 2-byte Spill
@@ -4966,13 +4966,13 @@ define i32 @svecc_call_dynamic_and_scalable_alloca(<4 x i16> %P0, i32 %P1, i32 %
; CHECK1024-NOSPLITSVE-NEXT: ldr p5, [sp, #14, mul vl] // 2-byte Reload
; CHECK1024-NOSPLITSVE-NEXT: ldr p4, [sp, #15, mul vl] // 2-byte Reload
; CHECK1024-NOSPLITSVE-NEXT: sub sp, x29, #1024
-; CHECK1024-NOSPLITSVE-NEXT: ldr x19, [sp, #1072] // 8-byte Reload
-; CHECK1024-NOSPLITSVE-NEXT: ldr x20, [sp, #1064] // 8-byte Reload
-; CHECK1024-NOSPLITSVE-NEXT: ldr x26, [sp, #1056] // 8-byte Reload
-; CHECK1024-NOSPLITSVE-NEXT: ldr x27, [sp, #1048] // 8-byte Reload
-; CHECK1024-NOSPLITSVE-NEXT: ldr x28, [sp, #1040] // 8-byte Reload
-; CHECK1024-NOSPLITSVE-NEXT: ldr x30, [sp, #1032] // 8-byte Reload
+; CHECK1024-NOSPLITSVE-NEXT: add x19, sp, #1064
+; CHECK1024-NOSPLITSVE-NEXT: add x26, sp, #1048
+; CHECK1024-NOSPLITSVE-NEXT: add x28, sp, #1032
+; CHECK1024-NOSPLITSVE-NEXT: ldp x20, x19, [x19] // 16-byte Folded Reload
; CHECK1024-NOSPLITSVE-NEXT: ldr x29, [sp, #1024] // 8-byte Reload
+; CHECK1024-NOSPLITSVE-NEXT: ldp x27, x26, [x26] // 16-byte Folded Reload
+; CHECK1024-NOSPLITSVE-NEXT: ldp x30, x28, [x28] // 16-byte Folded Reload
; CHECK1024-NOSPLITSVE-NEXT: add sp, sp, #1088
; CHECK1024-NOSPLITSVE-NEXT: ret
;
diff --git a/llvm/test/CodeGen/AArch64/stp-far-offset.ll b/llvm/test/CodeGen/AArch64/stp-far-offset.ll
new file mode 100644
index 0000000000000..a1af3a203b757
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/stp-far-offset.ll
@@ -0,0 +1,176 @@
+; RUN: llc -mtriple=aarch64 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ENABLED
+; RUN: llc -mtriple=aarch64 -verify-machineinstrs -aarch64-ldp-stp-base-adjust=0 < %s | FileCheck %s --check-prefixes=CHECK,DISABLED
+
+define void @stp_far_offset_i32(ptr %p, i32 %v0, i32 %v1) {
+; ENABLED-LABEL: stp_far_offset_i32:
+; ENABLED: // %bb.0:
+; ENABLED-NEXT: add x[[TMP:[0-9]+]], x0, #400
+; ENABLED-NEXT: stp w1, w2, [x[[TMP]]]
+; ENABLED-NEXT: ret
+;
+; DISABLED-LABEL: stp_far_offset_i32:
+; DISABLED: // %bb.0:
+; DISABLED-NOT: stp w
+; DISABLED-NOT: stp x
+; DISABLED: ret
+ %gep0 = getelementptr i32, ptr %p, i64 100
+ %gep1 = getelementptr i32, ptr %p, i64 101
+ store i32 %v0, ptr %gep0
+ store i32 %v1, ptr %gep1
+ ret void
+}
+
+define void @stp_far_offset_i64(ptr %p, i64 %v0, i64 %v1) {
+; ENABLED-LABEL: stp_far_offset_i64:
+; ENABLED: // %bb.0:
+; ENABLED-NEXT: add x[[TMP:[0-9]+]], x0, #800
+; ENABLED-NEXT: stp x1, x2, [x[[TMP]]]
+; ENABLED-NEXT: ret
+;
+; DISABLED-LABEL: stp_far_offset_i64:
+; DISABLED: // %bb.0:
+; DISABLED-NOT: stp w
+; DISABLED-NOT: stp x
+; DISABLED: ret
+ %gep0 = getelementptr i64, ptr %p, i64 100
+ %gep1 = getelementptr i64, ptr %p, i64 101
+ store i64 %v0, ptr %gep0
+ store i64 %v1, ptr %gep1
+ ret void
+}
+
+; Regression guard for the base-adjust scratch scavenger when the store
+; value registers are sub-64-bit (Wn): the scratch candidates are 64-bit
+; (X9..X15), so an exact register compare would miss that X9 aliases W9
+; and pick a scratch that clobbers the stored value before the STP reads
+; it. The inline-asm constraints pin the stored values to W9 and W8, and
+; the scratch capture `1[0-5]` cannot match 8 or 9, so this fails if the
+; aliasing-aware skip is ever dropped.
+define void @stp_far_offset_i32_w9w8_dst(ptr %p) {
+; ENABLED-LABEL: stp_far_offset_i32_w9w8_dst:
+; ENABLED: // %bb.0:
+; ENABLED-NEXT: add x[[SCRATCH:1[0-5]]], x0, #400
+; ENABLED: stp w9, w8, [x[[SCRATCH]]]
+; ENABLED: ret
+;
+; DISABLED-LABEL: stp_far_offset_i32_w9w8_dst:
+; DISABLED: // %bb.0:
+; DISABLED-NOT: stp w
+; DISABLED-NOT: stp x
+; DISABLED: ret
+ %v0 = call i32 asm "", "={w9}"()
+ %v1 = call i32 asm "", "={w8}"()
+ %gep0 = getelementptr i32, ptr %p, i64 100
+ %gep1 = getelementptr i32, ptr %p, i64 101
+ store i32 %v0, ptr %gep0
+ store i32 %v1, ptr %gep1
+ ret void
+}
+
+define void @stp_near_offset(ptr %p, i32 %v0, i32 %v1) {
+; Near offsets within STP range always pair regardless of the option.
+; CHECK-LABEL: stp_near_offset:
+; CHECK: // %bb.0:
+; CHECK-NOT: add
+; CHECK: stp w{{[0-9]+}}, w{{[0-9]+}}, [x0, #40]
+; CHECK: ret
+ %gep0 = getelementptr i32, ptr %p, i64 10
+ %gep1 = getelementptr i32, ptr %p, i64 11
+ store i32 %v0, ptr %gep0
+ store i32 %v1, ptr %gep1
+ ret void
+}
+
+define void @stp_far_offset_interleaved(ptr %p, i32 %v0, i32 %v1) {
+; Intervening non-aliasing operation between two far-offset stores.
+; ENABLED-LABEL: stp_far_offset_interleaved:
+; ENABLED: add x[[TMP:[0-9]+]], x0, #400
+; ENABLED: stp w{{[0-9]+}}, w{{[0-9]+}}, [x[[TMP]]]
+;
+; DISABLED-LABEL: stp_far_offset_interleaved:
+; DISABLED-NOT: stp w
+; DISABLED-NOT: stp x
+; DISABLED: ret
+ %gep0 = getelementptr i32, ptr %p, i64 100
+ %gep1 = getelementptr i32, ptr %p, i64 101
+ %stack = alloca i32
+ store i32 %v0, ptr %gep0
+ %v2 = load volatile i32, ptr %stack
+ store i32 %v1, ptr %gep1
+ ret void
+}
+
+; Offset = 1024 * 4 = 4096 bytes = 0x1000
+; The ADDXri encodes as #1, LSL #12 which equals 4096.
+define void @stp_far_offset_4k_aligned_i32(ptr %p, i32 %v0, i32 %v1) {
+; ENABLED-LABEL: stp_far_offset_4k_aligned_i32:
+; ENABLED: add x[[TMP:[0-9]+]], x0, #1, lsl #12
+; ENABLED: stp w{{[0-9]+}}, w{{[0-9]+}}, [x[[TMP]]]
+;
+; DISABLED-LABEL: stp_far_offset_4k_aligned_i32:
+; DISABLED-NOT: stp w
+; DISABLED-NOT: stp x
+; DISABLED: ret
+ %gep0 = getelementptr i32, ptr %p, i64 1024
+ %gep1 = getelementptr i32, ptr %p, i64 1025
+ store i32 %v0, ptr %gep0
+ store i32 %v1, ptr %gep1
+ ret void
+}
+
+; Offset = 500 * 8 = 4000 bytes (fits in shift=0, <= 4095)
+define void @stp_far_offset_large_i64(ptr %p, i64 %v0, i64 %v1) {
+; ENABLED-LABEL: stp_far_offset_large_i64:
+; ENABLED: add x[[TMP:[0-9]+]], x0, #4000
+; ENABLED: stp x{{[0-9]+}}, x{{[0-9]+}}, [x[[TMP]]]
+;
+; DISABLED-LABEL: stp_far_offset_large_i64:
+; DISABLED-NOT: stp w
+; DISABLED-NOT: stp x
+; DISABLED: ret
+ %gep0 = getelementptr i64, ptr %p, i64 500
+ %gep1 = getelementptr i64, ptr %p, i64 501
+ store i64 %v0, ptr %gep0
+ store i64 %v1, ptr %gep1
+ ret void
+}
+
+define void @stp_far_offset_q(ptr %p, <2 x i64> %v0, <2 x i64> %v1) {
+; ENABLED-LABEL: stp_far_offset_q:
+; ENABLED: // %bb.0:
+; ENABLED-NEXT: add x[[TMP:[0-9]+]], x0, #1600
+; ENABLED-NEXT: stp q0, q1, [x[[TMP]]]
+; ENABLED-NEXT: ret
+;
+; DISABLED-LABEL: stp_far_offset_q:
+; DISABLED: // %bb.0:
+; DISABLED-NEXT: str q0, [x0, #1600]
+; DISABLED-NEXT: str q1, [x0, #1616]
+; DISABLED-NEXT: ret
+ %gep0 = getelementptr <2 x i64>, ptr %p, i64 100
+ %gep1 = getelementptr <2 x i64>, ptr %p, i64 101
+ store <2 x i64> %v0, ptr %gep0
+ store <2 x i64> %v1, ptr %gep1
+ ret void
+}
+
+; Offset = 256 * 16 = 4096 bytes = 0x1000
+; The ADDXri encodes as #1, LSL #12 which equals 4096.
+define void @stp_far_offset_q_4k_aligned(ptr %p, <2 x i64> %v0, <2 x i64> %v1) {
+; ENABLED-LABEL: stp_far_offset_q_4k_aligned:
+; ENABLED: // %bb.0:
+; ENABLED-NEXT: add x[[TMP:[0-9]+]], x0, #1, lsl #12
+; ENABLED-NEXT: stp q0, q1, [x[[TMP]]]
+; ENABLED-NEXT: ret
+;
+; DISABLED-LABEL: stp_far_offset_q_4k_aligned:
+; DISABLED: // %bb.0:
+; DISABLED-NEXT: str q0, [x0, #4096]
+; DISABLED-NEXT: str q1, [x0, #4112]
+; DISABLED-NEXT: ret
+ %gep0 = getelementptr <2 x i64>, ptr %p, i64 256
+ %gep1 = getelementptr <2 x i64>, ptr %p, i64 257
+ store <2 x i64> %v0, ptr %gep0
+ store <2 x i64> %v1, ptr %gep1
+ ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-to-int.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-to-int.ll
index 76aa8e45ccda3..5d12d8c1feb82 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-to-int.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-to-int.ll
@@ -1462,11 +1462,11 @@ define void @fcvtzu_v16f64_v16i16(ptr %a, ptr %b) {
; NONEON-NOSVE-NEXT: strh w9, [sp, #282]
; NONEON-NOSVE-NEXT: strh w8, [sp, #280]
; NONEON-NOSVE-NEXT: ldp w8, w9, [sp, #224]
-; NONEON-NOSVE-NEXT: strh w8, [sp, #276]
-; NONEON-NOSVE-NEXT: ldr w8, [sp, #300]
; NONEON-NOSVE-NEXT: strh w9, [sp, #278]
-; NONEON-NOSVE-NEXT: strh w8, [sp, #274]
-; NONEON-NOSVE-NEXT: ldr w8, [sp, #296]
+; NONEON-NOSVE-NEXT: add x9, sp, #296
+; NONEON-NOSVE-NEXT: strh w8, [sp, #276]
+; NONEON-NOSVE-NEXT: ldp w8, w9, [x9]
+; NONEON-NOSVE-NEXT: strh w9, [sp, #274]
; NONEON-NOSVE-NEXT: strh w8, [sp, #272]
; NONEON-NOSVE-NEXT: ldp q1, q0, [sp, #256]
; NONEON-NOSVE-NEXT: stp q1, q0, [x1]
@@ -3163,11 +3163,11 @@ define void @fcvtzs_v16f64_v16i16(ptr %a, ptr %b) {
; NONEON-NOSVE-NEXT: strh w9, [sp, #282]
; NONEON-NOSVE-NEXT: strh w8, [sp, #280]
; NONEON-NOSVE-NEXT: ldp w8, w9, [sp, #224]
-; NONEON-NOSVE-NEXT: strh w8, [sp, #276]
-; NONEON-NOSVE-NEXT: ldr w8, [sp, #300]
; NONEON-NOSVE-NEXT: strh w9, [sp, #278]
-; NONEON-NOSVE-NEXT: strh w8, [sp, #274]
-; NONEON-NOSVE-NEXT: ldr w8, [sp, #296]
+; NONEON-NOSVE-NEXT: add x9, sp, #296
+; NONEON-NOSVE-NEXT: strh w8, [sp, #276]
+; NONEON-NOSVE-NEXT: ldp w8, w9, [x9]
+; NONEON-NOSVE-NEXT: strh w9, [sp, #274]
; NONEON-NOSVE-NEXT: strh w8, [sp, #272]
; NONEON-NOSVE-NEXT: ldp q1, q0, [sp, #256]
; NONEON-NOSVE-NEXT: stp q1, q0, [x1]
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-extends.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-extends.ll
index 8a14f65b3d1f7..44ece45b9e56e 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-extends.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-extends.ll
@@ -2317,9 +2317,7 @@ define void @zext_v32i8_v32i64(ptr %in, ptr %out) {
; NONEON-NOSVE-NEXT: .cfi_offset w30, -88
; NONEON-NOSVE-NEXT: .cfi_offset w29, -96
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT: str wzr, [sp, #300]
; NONEON-NOSVE-NEXT: str wzr, [sp, #292]
-; NONEON-NOSVE-NEXT: str wzr, [sp, #284]
; NONEON-NOSVE-NEXT: stp q1, q0, [sp, #16]
; NONEON-NOSVE-NEXT: ldrb w9, [sp, #46]
; NONEON-NOSVE-NEXT: ldrb w8, [sp, #47]
@@ -2353,19 +2351,19 @@ define void @zext_v32i8_v32i64(ptr %in, ptr %out) {
; NONEON-NOSVE-NEXT: add w9, w28, w28
; NONEON-NOSVE-NEXT: stp w8, wzr, [sp, #136]
; NONEON-NOSVE-NEXT: add w8, w27, w27
-; NONEON-NOSVE-NEXT: ldrb w4, [sp, #33]
-; NONEON-NOSVE-NEXT: and w8, w8, #0xff
; NONEON-NOSVE-NEXT: ldrb w16, [sp, #16]
-; NONEON-NOSVE-NEXT: ldrb w3, [sp, #34]
+; NONEON-NOSVE-NEXT: and w8, w8, #0xff
+; NONEON-NOSVE-NEXT: ldrb w18, [sp, #17]
+; NONEON-NOSVE-NEXT: ldrb w4, [sp, #33]
; NONEON-NOSVE-NEXT: stp w8, wzr, [sp, #112]
; NONEON-NOSVE-NEXT: and w8, w9, #0xff
; NONEON-NOSVE-NEXT: add w9, w26, w26
; NONEON-NOSVE-NEXT: stp w8, wzr, [sp, #120]
; NONEON-NOSVE-NEXT: add w8, w25, w25
-; NONEON-NOSVE-NEXT: ldrb w2, [sp, #35]
-; NONEON-NOSVE-NEXT: and w8, w8, #0xff
-; NONEON-NOSVE-NEXT: ldrb w18, [sp, #17]
; NONEON-NOSVE-NEXT: add w0, w16, w16
+; NONEON-NOSVE-NEXT: and w8, w8, #0xff
+; NONEON-NOSVE-NEXT: add w22, w18, w18
+; NONEON-NOSVE-NEXT: ldrb w3, [sp, #34]
; NONEON-NOSVE-NEXT: stp w8, wzr, [sp, #96]
; NONEON-NOSVE-NEXT: and w8, w9, #0xff
; NONEON-NOSVE-NEXT: add w9, w21, w21
@@ -2373,107 +2371,109 @@ define void @zext_v32i8_v32i64(ptr %in, ptr %out) {
; NONEON-NOSVE-NEXT: add w8, w24, w24
; NONEON-NOSVE-NEXT: and w23, w0, #0xff
; NONEON-NOSVE-NEXT: and w8, w8, #0xff
-; NONEON-NOSVE-NEXT: ldrb w0, [sp, #36]
-; NONEON-NOSVE-NEXT: add w22, w18, w18
+; NONEON-NOSVE-NEXT: ldrb w2, [sp, #35]
+; NONEON-NOSVE-NEXT: and w22, w22, #0xff
; NONEON-NOSVE-NEXT: stp w8, wzr, [sp, #80]
; NONEON-NOSVE-NEXT: and w8, w9, #0xff
; NONEON-NOSVE-NEXT: add w9, w19, w19
; NONEON-NOSVE-NEXT: stp w8, wzr, [sp, #88]
; NONEON-NOSVE-NEXT: add w8, w20, w20
-; NONEON-NOSVE-NEXT: ldrb w18, [sp, #37]
+; NONEON-NOSVE-NEXT: ldrb w10, [sp, #44]
; NONEON-NOSVE-NEXT: and w8, w8, #0xff
-; NONEON-NOSVE-NEXT: ldrb w17, [sp, #38]
-; NONEON-NOSVE-NEXT: ldrb w16, [sp, #39]
+; NONEON-NOSVE-NEXT: ldrb w11, [sp, #45]
+; NONEON-NOSVE-NEXT: ldrb w12, [sp, #42]
; NONEON-NOSVE-NEXT: stp w8, wzr, [sp, #64]
; NONEON-NOSVE-NEXT: and w8, w9, #0xff
; NONEON-NOSVE-NEXT: add w9, w6, w6
; NONEON-NOSVE-NEXT: stp w8, wzr, [sp, #72]
; NONEON-NOSVE-NEXT: add w8, w7, w7
-; NONEON-NOSVE-NEXT: ldrb w15, [sp, #40]
+; NONEON-NOSVE-NEXT: ldrb w13, [sp, #43]
; NONEON-NOSVE-NEXT: and w8, w8, #0xff
+; NONEON-NOSVE-NEXT: ldrb w15, [sp, #40]
; NONEON-NOSVE-NEXT: ldrb w14, [sp, #41]
-; NONEON-NOSVE-NEXT: ldrb w12, [sp, #42]
; NONEON-NOSVE-NEXT: stp w8, wzr, [sp, #48]
; NONEON-NOSVE-NEXT: and w8, w9, #0xff
; NONEON-NOSVE-NEXT: add w9, w4, w4
; NONEON-NOSVE-NEXT: stp w8, wzr, [sp, #56]
; NONEON-NOSVE-NEXT: add w8, w5, w5
-; NONEON-NOSVE-NEXT: ldrb w13, [sp, #43]
+; NONEON-NOSVE-NEXT: ldrb w17, [sp, #38]
; NONEON-NOSVE-NEXT: and w8, w8, #0xff
-; NONEON-NOSVE-NEXT: ldrb w10, [sp, #44]
-; NONEON-NOSVE-NEXT: ldrb w11, [sp, #45]
+; NONEON-NOSVE-NEXT: ldrb w16, [sp, #39]
+; NONEON-NOSVE-NEXT: ldrb w0, [sp, #36]
; NONEON-NOSVE-NEXT: str w8, [sp, #288]
; NONEON-NOSVE-NEXT: and w8, w9, #0xff
-; NONEON-NOSVE-NEXT: add w9, w2, w2
-; NONEON-NOSVE-NEXT: str w8, [sp, #296]
+; NONEON-NOSVE-NEXT: add x9, sp, #296
+; NONEON-NOSVE-NEXT: ldrb w18, [sp, #37]
+; NONEON-NOSVE-NEXT: stp w23, wzr, [sp, #160]
+; NONEON-NOSVE-NEXT: stp w22, wzr, [sp, #168]
+; NONEON-NOSVE-NEXT: stp w8, wzr, [x9]
; NONEON-NOSVE-NEXT: add w8, w3, w3
-; NONEON-NOSVE-NEXT: and w22, w22, #0xff
+; NONEON-NOSVE-NEXT: add w9, w2, w2
; NONEON-NOSVE-NEXT: and w8, w8, #0xff
-; NONEON-NOSVE-NEXT: stp w23, wzr, [sp, #160]
+; NONEON-NOSVE-NEXT: str wzr, [sp, #276]
; NONEON-NOSVE-NEXT: str w8, [sp, #272]
; NONEON-NOSVE-NEXT: and w8, w9, #0xff
-; NONEON-NOSVE-NEXT: add w9, w18, w18
-; NONEON-NOSVE-NEXT: str w8, [sp, #280]
+; NONEON-NOSVE-NEXT: add x9, sp, #280
+; NONEON-NOSVE-NEXT: stp w8, wzr, [x9]
; NONEON-NOSVE-NEXT: add w8, w0, w0
+; NONEON-NOSVE-NEXT: add w9, w18, w18
; NONEON-NOSVE-NEXT: and w8, w8, #0xff
-; NONEON-NOSVE-NEXT: stp w22, wzr, [sp, #168]
+; NONEON-NOSVE-NEXT: str wzr, [sp, #260]
; NONEON-NOSVE-NEXT: stp wzr, w8, [sp, #252]
; NONEON-NOSVE-NEXT: and w8, w9, #0xff
-; NONEON-NOSVE-NEXT: add w9, w16, w16
-; NONEON-NOSVE-NEXT: str w8, [sp, #264]
+; NONEON-NOSVE-NEXT: add x9, sp, #264
+; NONEON-NOSVE-NEXT: stp w8, wzr, [x9]
; NONEON-NOSVE-NEXT: add w8, w17, w17
+; NONEON-NOSVE-NEXT: add w9, w16, w16
; NONEON-NOSVE-NEXT: and w8, w8, #0xff
-; NONEON-NOSVE-NEXT: str wzr, [sp, #276]
+; NONEON-NOSVE-NEXT: ldp q1, q0, [sp, #144]
; NONEON-NOSVE-NEXT: stp w8, wzr, [sp, #240]
; NONEON-NOSVE-NEXT: and w8, w9, #0xff
; NONEON-NOSVE-NEXT: add w9, w14, w14
; NONEON-NOSVE-NEXT: str w8, [sp, #248]
; NONEON-NOSVE-NEXT: add w8, w15, w15
; NONEON-NOSVE-NEXT: and w8, w8, #0xff
-; NONEON-NOSVE-NEXT: str wzr, [sp, #268]
+; NONEON-NOSVE-NEXT: ldp q3, q2, [sp, #112]
; NONEON-NOSVE-NEXT: stp w8, wzr, [sp, #224]
; NONEON-NOSVE-NEXT: and w8, w9, #0xff
; NONEON-NOSVE-NEXT: add w9, w13, w13
; NONEON-NOSVE-NEXT: stp w8, wzr, [sp, #232]
; NONEON-NOSVE-NEXT: add w8, w12, w12
; NONEON-NOSVE-NEXT: and w8, w8, #0xff
-; NONEON-NOSVE-NEXT: str wzr, [sp, #260]
+; NONEON-NOSVE-NEXT: ldp q5, q4, [sp, #80]
; NONEON-NOSVE-NEXT: stp w8, wzr, [sp, #208]
; NONEON-NOSVE-NEXT: and w8, w9, #0xff
; NONEON-NOSVE-NEXT: add w9, w11, w11
; NONEON-NOSVE-NEXT: stp w8, wzr, [sp, #216]
; NONEON-NOSVE-NEXT: add w8, w10, w10
; NONEON-NOSVE-NEXT: and w8, w8, #0xff
-; NONEON-NOSVE-NEXT: ldp q1, q0, [sp, #144]
+; NONEON-NOSVE-NEXT: ldp q7, q6, [sp, #48]
; NONEON-NOSVE-NEXT: stp w8, wzr, [sp, #192]
; NONEON-NOSVE-NEXT: and w8, w9, #0xff
; NONEON-NOSVE-NEXT: stp w8, wzr, [sp, #200]
-; NONEON-NOSVE-NEXT: ldp q3, q2, [sp, #112]
+; NONEON-NOSVE-NEXT: ldp q17, q16, [sp, #272]
; NONEON-NOSVE-NEXT: ldp w9, w8, [sp, #8] // 8-byte Folded Reload
-; NONEON-NOSVE-NEXT: ldp q5, q4, [sp, #80]
-; NONEON-NOSVE-NEXT: ldp q7, q6, [sp, #48]
+; NONEON-NOSVE-NEXT: ldp q20, q19, [sp, #240]
+; NONEON-NOSVE-NEXT: ldp q23, q22, [sp, #208]
; NONEON-NOSVE-NEXT: add w8, w8, w8
; NONEON-NOSVE-NEXT: add w9, w9, w9
; NONEON-NOSVE-NEXT: and w8, w8, #0xff
-; NONEON-NOSVE-NEXT: ldp q17, q16, [sp, #272]
+; NONEON-NOSVE-NEXT: ldp x20, x19, [sp, #384] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT: stp w8, wzr, [sp, #176]
; NONEON-NOSVE-NEXT: and w8, w9, #0xff
; NONEON-NOSVE-NEXT: stp w8, wzr, [sp, #184]
-; NONEON-NOSVE-NEXT: ldp q20, q19, [sp, #240]
+; NONEON-NOSVE-NEXT: ldp x22, x21, [sp, #368] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT: ldp q18, q21, [sp, #176]
-; NONEON-NOSVE-NEXT: ldp q23, q22, [sp, #208]
; NONEON-NOSVE-NEXT: stp q0, q1, [x1]
-; NONEON-NOSVE-NEXT: ldp x20, x19, [sp, #384] // 16-byte Folded Reload
+; NONEON-NOSVE-NEXT: ldp x24, x23, [sp, #352] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT: stp q2, q3, [x1, #32]
-; NONEON-NOSVE-NEXT: ldp x22, x21, [sp, #368] // 16-byte Folded Reload
+; NONEON-NOSVE-NEXT: ldp x26, x25, [sp, #336] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT: stp q4, q5, [x1, #64]
-; NONEON-NOSVE-NEXT: ldp x24, x23, [sp, #352] // 16-byte Folded Reload
+; NONEON-NOSVE-NEXT: ldp x28, x27, [sp, #320] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT: stp q6, q7, [x1, #96]
-; NONEON-NOSVE-NEXT: ldp x26, x25, [sp, #336] // 16-byte Folded Reload
+; NONEON-NOSVE-NEXT: ldp x29, x30, [sp, #304] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT: stp q16, q17, [x1, #128]
-; NONEON-NOSVE-NEXT: ldp x28, x27, [sp, #320] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT: stp q19, q20, [x1, #160]
-; NONEON-NOSVE-NEXT: ldp x29, x30, [sp, #304] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT: stp q22, q23, [x1, #192]
; NONEON-NOSVE-NEXT: stp q21, q18, [x1, #224]
; NONEON-NOSVE-NEXT: add sp, sp, #400
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-to-fp.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-to-fp.ll
index 2c6ab7980b031..67587d653c8cd 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-to-fp.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-to-fp.ll
@@ -2826,11 +2826,11 @@ define void @scvtf_v16i32_v16f64(ptr %a, ptr %b) {
; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #208]
; NONEON-NOSVE-NEXT: scvtf d1, w9
; NONEON-NOSVE-NEXT: scvtf d0, w8
-; NONEON-NOSVE-NEXT: ldr w8, [sp, #268]
+; NONEON-NOSVE-NEXT: add x9, sp, #264
; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #224]
-; NONEON-NOSVE-NEXT: scvtf d1, w8
-; NONEON-NOSVE-NEXT: ldr w8, [sp, #264]
+; NONEON-NOSVE-NEXT: ldp w8, w9, [x9]
; NONEON-NOSVE-NEXT: ldp q4, q6, [sp, #208]
+; NONEON-NOSVE-NEXT: scvtf d1, w9
; NONEON-NOSVE-NEXT: scvtf d0, w8
; NONEON-NOSVE-NEXT: ldp w8, w9, [sp, #88]
; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #240]
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-expandload.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-expandload.ll
index 3b2a722d0dcdb..dcf899617cd7a 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-expandload.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-expandload.ll
@@ -3567,17 +3567,17 @@ define <8 x float> @masked_load_v8f32(ptr %src, <8 x i1> %mask) {
; NONEON-NOSVE-NEXT: .LBB10_3: // %cond.load1
; NONEON-NOSVE-NEXT: ldr s2, [x0], #4
; NONEON-NOSVE-NEXT: str q0, [sp, #400]
+; NONEON-NOSVE-NEXT: add x10, sp, #408
; NONEON-NOSVE-NEXT: str s2, [sp, #432]
; NONEON-NOSVE-NEXT: ldr q0, [sp, #432]
; NONEON-NOSVE-NEXT: str q0, [sp, #384]
-; NONEON-NOSVE-NEXT: ldr s0, [sp, #412]
-; NONEON-NOSVE-NEXT: str s0, [sp, #428]
-; NONEON-NOSVE-NEXT: ldr s0, [sp, #408]
-; NONEON-NOSVE-NEXT: str s0, [sp, #424]
-; NONEON-NOSVE-NEXT: ldr s0, [sp, #400]
-; NONEON-NOSVE-NEXT: str s0, [sp, #416]
+; NONEON-NOSVE-NEXT: ldp s0, s2, [x10]
+; NONEON-NOSVE-NEXT: add x10, sp, #424
+; NONEON-NOSVE-NEXT: stp s0, s2, [x10]
+; NONEON-NOSVE-NEXT: add x10, sp, #416
+; NONEON-NOSVE-NEXT: ldr s2, [sp, #400]
+; NONEON-NOSVE-NEXT: stp s2, s0, [x10]
; NONEON-NOSVE-NEXT: ldr s0, [sp, #384]
-; NONEON-NOSVE-NEXT: str s0, [sp, #420]
; NONEON-NOSVE-NEXT: ldr q0, [sp, #416]
; NONEON-NOSVE-NEXT: .LBB10_4: // %else2
; NONEON-NOSVE-NEXT: tbnz w8, #2, .LBB10_12
@@ -3610,31 +3610,31 @@ define <8 x float> @masked_load_v8f32(ptr %src, <8 x i1> %mask) {
; NONEON-NOSVE-NEXT: .LBB10_12: // %cond.load5
; NONEON-NOSVE-NEXT: ldr s2, [x0], #4
; NONEON-NOSVE-NEXT: str q0, [sp, #336]
+; NONEON-NOSVE-NEXT: add x10, sp, #360
; NONEON-NOSVE-NEXT: str s2, [sp, #368]
+; NONEON-NOSVE-NEXT: ldr s2, [sp, #348]
; NONEON-NOSVE-NEXT: ldr q0, [sp, #368]
; NONEON-NOSVE-NEXT: str q0, [sp, #320]
-; NONEON-NOSVE-NEXT: ldr s0, [sp, #348]
-; NONEON-NOSVE-NEXT: str s0, [sp, #364]
-; NONEON-NOSVE-NEXT: ldr s0, [sp, #340]
-; NONEON-NOSVE-NEXT: str s0, [sp, #356]
-; NONEON-NOSVE-NEXT: ldr s0, [sp, #336]
-; NONEON-NOSVE-NEXT: str s0, [sp, #352]
+; NONEON-NOSVE-NEXT: stp s0, s2, [x10]
+; NONEON-NOSVE-NEXT: add x10, sp, #336
+; NONEON-NOSVE-NEXT: ldp s0, s2, [x10]
+; NONEON-NOSVE-NEXT: add x10, sp, #352
+; NONEON-NOSVE-NEXT: stp s0, s2, [x10]
; NONEON-NOSVE-NEXT: ldr s0, [sp, #320]
-; NONEON-NOSVE-NEXT: str s0, [sp, #360]
; NONEON-NOSVE-NEXT: ldr q0, [sp, #352]
; NONEON-NOSVE-NEXT: tbz w8, #3, .LBB10_6
; NONEON-NOSVE-NEXT: .LBB10_13: // %cond.load9
; NONEON-NOSVE-NEXT: ldr s2, [x0], #4
; NONEON-NOSVE-NEXT: str q0, [sp, #256]
-; NONEON-NOSVE-NEXT: ldr x10, [sp, #256]
+; NONEON-NOSVE-NEXT: add x10, sp, #296
; NONEON-NOSVE-NEXT: str s2, [sp, #304]
+; NONEON-NOSVE-NEXT: ldr s2, [sp, #264]
; NONEON-NOSVE-NEXT: ldr q0, [sp, #304]
-; NONEON-NOSVE-NEXT: str x10, [sp, #288]
; NONEON-NOSVE-NEXT: str q0, [sp, #272]
-; NONEON-NOSVE-NEXT: ldr s0, [sp, #264]
-; NONEON-NOSVE-NEXT: str s0, [sp, #296]
+; NONEON-NOSVE-NEXT: stp s2, s0, [x10]
+; NONEON-NOSVE-NEXT: ldr x10, [sp, #256]
; NONEON-NOSVE-NEXT: ldr s0, [sp, #272]
-; NONEON-NOSVE-NEXT: str s0, [sp, #300]
+; NONEON-NOSVE-NEXT: str x10, [sp, #288]
; NONEON-NOSVE-NEXT: ldr q0, [sp, #288]
; NONEON-NOSVE-NEXT: tbz w8, #4, .LBB10_7
; NONEON-NOSVE-NEXT: .LBB10_14: // %cond.load13
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-load.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-load.ll
index 583dde21d054d..669bc0187f552 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-load.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-load.ll
@@ -2399,17 +2399,17 @@ define <8 x float> @masked_load_v8f32(ptr %src, <8 x i1> %mask) {
; NONEON-NOSVE-NEXT: .LBB10_3: // %cond.load1
; NONEON-NOSVE-NEXT: ldr s2, [x0, #4]
; NONEON-NOSVE-NEXT: str q0, [sp, #400]
+; NONEON-NOSVE-NEXT: add x10, sp, #408
; NONEON-NOSVE-NEXT: str s2, [sp, #432]
; NONEON-NOSVE-NEXT: ldr q0, [sp, #432]
; NONEON-NOSVE-NEXT: str q0, [sp, #384]
-; NONEON-NOSVE-NEXT: ldr s0, [sp, #412]
-; NONEON-NOSVE-NEXT: str s0, [sp, #428]
-; NONEON-NOSVE-NEXT: ldr s0, [sp, #408]
-; NONEON-NOSVE-NEXT: str s0, [sp, #424]
-; NONEON-NOSVE-NEXT: ldr s0, [sp, #400]
-; NONEON-NOSVE-NEXT: str s0, [sp, #416]
+; NONEON-NOSVE-NEXT: ldp s0, s2, [x10]
+; NONEON-NOSVE-NEXT: add x10, sp, #424
+; NONEON-NOSVE-NEXT: stp s0, s2, [x10]
+; NONEON-NOSVE-NEXT: add x10, sp, #416
+; NONEON-NOSVE-NEXT: ldr s2, [sp, #400]
+; NONEON-NOSVE-NEXT: stp s2, s0, [x10]
; NONEON-NOSVE-NEXT: ldr s0, [sp, #384]
-; NONEON-NOSVE-NEXT: str s0, [sp, #420]
; NONEON-NOSVE-NEXT: ldr q0, [sp, #416]
; NONEON-NOSVE-NEXT: .LBB10_4: // %else2
; NONEON-NOSVE-NEXT: tbnz w8, #2, .LBB10_12
@@ -2442,31 +2442,31 @@ define <8 x float> @masked_load_v8f32(ptr %src, <8 x i1> %mask) {
; NONEON-NOSVE-NEXT: .LBB10_12: // %cond.load4
; NONEON-NOSVE-NEXT: ldr s2, [x0, #8]
; NONEON-NOSVE-NEXT: str q0, [sp, #336]
+; NONEON-NOSVE-NEXT: add x10, sp, #360
; NONEON-NOSVE-NEXT: str s2, [sp, #368]
+; NONEON-NOSVE-NEXT: ldr s2, [sp, #348]
; NONEON-NOSVE-NEXT: ldr q0, [sp, #368]
; NONEON-NOSVE-NEXT: str q0, [sp, #320]
-; NONEON-NOSVE-NEXT: ldr s0, [sp, #348]
-; NONEON-NOSVE-NEXT: str s0, [sp, #364]
-; NONEON-NOSVE-NEXT: ldr s0, [sp, #340]
-; NONEON-NOSVE-NEXT: str s0, [sp, #356]
-; NONEON-NOSVE-NEXT: ldr s0, [sp, #336]
-; NONEON-NOSVE-NEXT: str s0, [sp, #352]
+; NONEON-NOSVE-NEXT: stp s0, s2, [x10]
+; NONEON-NOSVE-NEXT: add x10, sp, #336
+; NONEON-NOSVE-NEXT: ldp s0, s2, [x10]
+; NONEON-NOSVE-NEXT: add x10, sp, #352
+; NONEON-NOSVE-NEXT: stp s0, s2, [x10]
; NONEON-NOSVE-NEXT: ldr s0, [sp, #320]
-; NONEON-NOSVE-NEXT: str s0, [sp, #360]
; NONEON-NOSVE-NEXT: ldr q0, [sp, #352]
; NONEON-NOSVE-NEXT: tbz w8, #3, .LBB10_6
; NONEON-NOSVE-NEXT: .LBB10_13: // %cond.load7
; NONEON-NOSVE-NEXT: ldr s2, [x0, #12]
; NONEON-NOSVE-NEXT: str q0, [sp, #256]
-; NONEON-NOSVE-NEXT: ldr x10, [sp, #256]
+; NONEON-NOSVE-NEXT: add x10, sp, #296
; NONEON-NOSVE-NEXT: str s2, [sp, #304]
+; NONEON-NOSVE-NEXT: ldr s2, [sp, #264]
; NONEON-NOSVE-NEXT: ldr q0, [sp, #304]
-; NONEON-NOSVE-NEXT: str x10, [sp, #288]
; NONEON-NOSVE-NEXT: str q0, [sp, #272]
-; NONEON-NOSVE-NEXT: ldr s0, [sp, #264]
-; NONEON-NOSVE-NEXT: str s0, [sp, #296]
+; NONEON-NOSVE-NEXT: stp s2, s0, [x10]
+; NONEON-NOSVE-NEXT: ldr x10, [sp, #256]
; NONEON-NOSVE-NEXT: ldr s0, [sp, #272]
-; NONEON-NOSVE-NEXT: str s0, [sp, #300]
+; NONEON-NOSVE-NEXT: str x10, [sp, #288]
; NONEON-NOSVE-NEXT: ldr q0, [sp, #288]
; NONEON-NOSVE-NEXT: tbz w8, #4, .LBB10_7
; NONEON-NOSVE-NEXT: .LBB10_14: // %cond.load10
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-trunc.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-trunc.ll
index 6cc9175a1f478..2ff0fb312992e 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-trunc.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-trunc.ll
@@ -582,224 +582,226 @@ define void @trunc_v128i16_v128i8(ptr %in, ptr %out) nounwind {
; NONEON-NOSVE-NEXT: sub sp, sp, #800
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0, #32]
; NONEON-NOSVE-NEXT: str x1, [sp, #408] // 8-byte Spill
+; NONEON-NOSVE-NEXT: ldp q17, q16, [x0, #192]
+; NONEON-NOSVE-NEXT: ldp q23, q22, [x0, #224]
; NONEON-NOSVE-NEXT: ldp q3, q2, [x0]
; NONEON-NOSVE-NEXT: ldp q5, q4, [x0, #96]
; NONEON-NOSVE-NEXT: ldp q7, q6, [x0, #64]
-; NONEON-NOSVE-NEXT: ldp q17, q16, [x0, #192]
; NONEON-NOSVE-NEXT: ldp q19, q18, [x0, #160]
; NONEON-NOSVE-NEXT: ldp q21, q20, [x0, #128]
-; NONEON-NOSVE-NEXT: ldp q23, q22, [x0, #224]
; NONEON-NOSVE-NEXT: str q0, [sp, #592]
+; NONEON-NOSVE-NEXT: stp q17, q23, [sp, #432]
; NONEON-NOSVE-NEXT: ldrh w8, [sp, #606]
-; NONEON-NOSVE-NEXT: str q19, [sp, #496]
-; NONEON-NOSVE-NEXT: ldrh w10, [sp, #600]
+; NONEON-NOSVE-NEXT: ldrh w9, [sp, #432]
+; NONEON-NOSVE-NEXT: stp q22, q16, [sp, #464]
+; NONEON-NOSVE-NEXT: add w8, w8, w8
+; NONEON-NOSVE-NEXT: add x9, sp, #400
; NONEON-NOSVE-NEXT: stp q18, q20, [sp, #512]
+; NONEON-NOSVE-NEXT: str q19, [sp, #496]
+; NONEON-NOSVE-NEXT: stp q4, q6, [sp, #560]
+; NONEON-NOSVE-NEXT: stp q2, q1, [sp, #608]
+; NONEON-NOSVE-NEXT: stp q7, q21, [sp, #640]
+; NONEON-NOSVE-NEXT: str q5, [sp, #544]
+; NONEON-NOSVE-NEXT: str q3, [sp, #416]
+; NONEON-NOSVE-NEXT: stp w8, w9, [x9] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: add x9, sp, #392
+; NONEON-NOSVE-NEXT: str w8, [sp, #64] // 4-byte Spill
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #434]
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #436]
+; NONEON-NOSVE-NEXT: ldr w30, [sp, #64] // 4-byte Reload
+; NONEON-NOSVE-NEXT: stp w8, w10, [x9] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: add x9, sp, #384
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #438]
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #440]
+; NONEON-NOSVE-NEXT: stp w8, w10, [x9] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: add x9, sp, #376
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #442]
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #444]
+; NONEON-NOSVE-NEXT: stp w8, w10, [x9] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: add x9, sp, #368
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #446]
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #480]
+; NONEON-NOSVE-NEXT: stp w8, w10, [x9] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: add x9, sp, #360
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #482]
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #484]
+; NONEON-NOSVE-NEXT: stp w8, w10, [x9] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: add x9, sp, #352
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #486]
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #488]
+; NONEON-NOSVE-NEXT: stp w8, w10, [x9] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: add x9, sp, #344
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #490]
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #492]
+; NONEON-NOSVE-NEXT: stp w8, w10, [x9] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: add x9, sp, #336
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #494]
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #448]
+; NONEON-NOSVE-NEXT: stp w8, w10, [x9] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: add x9, sp, #328
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #450]
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #452]
+; NONEON-NOSVE-NEXT: stp w8, w10, [x9] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: add x9, sp, #320
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #454]
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #456]
+; NONEON-NOSVE-NEXT: stp w8, w10, [x9] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: add x9, sp, #312
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #458]
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #460]
+; NONEON-NOSVE-NEXT: stp w8, w10, [x9] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: add x9, sp, #304
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #462]
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #464]
+; NONEON-NOSVE-NEXT: stp w8, w10, [x9] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: add x9, sp, #296
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #466]
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #468]
+; NONEON-NOSVE-NEXT: stp w8, w10, [x9] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: add x9, sp, #288
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #470]
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #472]
+; NONEON-NOSVE-NEXT: stp w8, w10, [x9] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: add x9, sp, #280
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #474]
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #476]
+; NONEON-NOSVE-NEXT: stp w8, w10, [x9] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: add x9, sp, #272
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #478]
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #656]
+; NONEON-NOSVE-NEXT: stp w8, w10, [x9] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: add x9, sp, #264
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #658]
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #660]
+; NONEON-NOSVE-NEXT: stp w8, w10, [x9] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: add x9, sp, #256
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #662]
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #664]
+; NONEON-NOSVE-NEXT: stp w8, w10, [x9] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #666]
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #668]
; NONEON-NOSVE-NEXT: ldrh w11, [sp, #598]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #670]
; NONEON-NOSVE-NEXT: ldrh w12, [sp, #596]
-; NONEON-NOSVE-NEXT: add w8, w8, w8
-; NONEON-NOSVE-NEXT: stp q17, q23, [sp, #432]
; NONEON-NOSVE-NEXT: ldrh w13, [sp, #594]
-; NONEON-NOSVE-NEXT: str w8, [sp, #64] // 4-byte Spill
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #432]
; NONEON-NOSVE-NEXT: ldrh w14, [sp, #592]
-; NONEON-NOSVE-NEXT: stp q22, q16, [sp, #464]
-; NONEON-NOSVE-NEXT: ldr w30, [sp, #64] // 4-byte Reload
-; NONEON-NOSVE-NEXT: str w8, [sp, #404] // 4-byte Spill
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #434]
-; NONEON-NOSVE-NEXT: stp q4, q6, [sp, #560]
-; NONEON-NOSVE-NEXT: str w8, [sp, #400] // 4-byte Spill
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #436]
-; NONEON-NOSVE-NEXT: str q5, [sp, #544]
-; NONEON-NOSVE-NEXT: str w8, [sp, #396] // 4-byte Spill
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #438]
-; NONEON-NOSVE-NEXT: stp q2, q1, [sp, #608]
-; NONEON-NOSVE-NEXT: str w8, [sp, #392] // 4-byte Spill
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #440]
; NONEON-NOSVE-NEXT: ldrh w15, [sp, #638]
-; NONEON-NOSVE-NEXT: stp q7, q21, [sp, #640]
; NONEON-NOSVE-NEXT: ldrh w16, [sp, #636]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #248] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #528]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #530]
; NONEON-NOSVE-NEXT: ldrh w17, [sp, #634]
-; NONEON-NOSVE-NEXT: str w8, [sp, #388] // 4-byte Spill
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #442]
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #666]
-; NONEON-NOSVE-NEXT: str q3, [sp, #416]
; NONEON-NOSVE-NEXT: ldrh w18, [sp, #632]
; NONEON-NOSVE-NEXT: ldrh w0, [sp, #630]
-; NONEON-NOSVE-NEXT: str w8, [sp, #384] // 4-byte Spill
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #444]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #240] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #532]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #534]
; NONEON-NOSVE-NEXT: ldrh w1, [sp, #628]
; NONEON-NOSVE-NEXT: ldrh w2, [sp, #626]
; NONEON-NOSVE-NEXT: ldrh w3, [sp, #624]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #232] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #536]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #538]
; NONEON-NOSVE-NEXT: ldrh w4, [sp, #622]
-; NONEON-NOSVE-NEXT: str w8, [sp, #380] // 4-byte Spill
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #446]
; NONEON-NOSVE-NEXT: ldrh w5, [sp, #620]
; NONEON-NOSVE-NEXT: ldrh w6, [sp, #618]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #224] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #540]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #542]
; NONEON-NOSVE-NEXT: ldrh w7, [sp, #616]
; NONEON-NOSVE-NEXT: ldrh w19, [sp, #614]
-; NONEON-NOSVE-NEXT: str w8, [sp, #376] // 4-byte Spill
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #480]
; NONEON-NOSVE-NEXT: ldrh w20, [sp, #612]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #216] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #496]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #498]
; NONEON-NOSVE-NEXT: ldrh w21, [sp, #610]
; NONEON-NOSVE-NEXT: ldrh w22, [sp, #608]
; NONEON-NOSVE-NEXT: ldrh w23, [sp, #430]
-; NONEON-NOSVE-NEXT: str w8, [sp, #372] // 4-byte Spill
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #482]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #208] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #500]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #502]
; NONEON-NOSVE-NEXT: ldrh w24, [sp, #428]
; NONEON-NOSVE-NEXT: ldrh w25, [sp, #426]
; NONEON-NOSVE-NEXT: ldrh w26, [sp, #424]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #200] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #504]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #506]
; NONEON-NOSVE-NEXT: ldrh w27, [sp, #422]
-; NONEON-NOSVE-NEXT: str w8, [sp, #368] // 4-byte Spill
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #484]
; NONEON-NOSVE-NEXT: ldrh w28, [sp, #420]
; NONEON-NOSVE-NEXT: ldrh w29, [sp, #418]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #192] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #508]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #510]
+; NONEON-NOSVE-NEXT: ldrh w9, [sp, #604]
; NONEON-NOSVE-NEXT: strb w30, [sp, #767]
-; NONEON-NOSVE-NEXT: str w8, [sp, #364] // 4-byte Spill
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #486]
-; NONEON-NOSVE-NEXT: str w8, [sp, #360] // 4-byte Spill
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #488]
-; NONEON-NOSVE-NEXT: str w8, [sp, #356] // 4-byte Spill
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #490]
-; NONEON-NOSVE-NEXT: str w8, [sp, #352] // 4-byte Spill
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #492]
-; NONEON-NOSVE-NEXT: str w8, [sp, #348] // 4-byte Spill
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #494]
-; NONEON-NOSVE-NEXT: str w8, [sp, #344] // 4-byte Spill
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #448]
-; NONEON-NOSVE-NEXT: str w8, [sp, #340] // 4-byte Spill
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #450]
-; NONEON-NOSVE-NEXT: str w8, [sp, #336] // 4-byte Spill
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #452]
-; NONEON-NOSVE-NEXT: str w8, [sp, #332] // 4-byte Spill
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #454]
-; NONEON-NOSVE-NEXT: str w8, [sp, #328] // 4-byte Spill
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #456]
-; NONEON-NOSVE-NEXT: str w8, [sp, #324] // 4-byte Spill
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #458]
-; NONEON-NOSVE-NEXT: str w8, [sp, #320] // 4-byte Spill
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #460]
-; NONEON-NOSVE-NEXT: str w8, [sp, #316] // 4-byte Spill
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #462]
-; NONEON-NOSVE-NEXT: str w8, [sp, #312] // 4-byte Spill
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #464]
-; NONEON-NOSVE-NEXT: str w8, [sp, #308] // 4-byte Spill
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #466]
-; NONEON-NOSVE-NEXT: str w8, [sp, #304] // 4-byte Spill
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #468]
-; NONEON-NOSVE-NEXT: str w8, [sp, #300] // 4-byte Spill
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #470]
-; NONEON-NOSVE-NEXT: str w8, [sp, #296] // 4-byte Spill
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #472]
-; NONEON-NOSVE-NEXT: str w8, [sp, #292] // 4-byte Spill
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #474]
-; NONEON-NOSVE-NEXT: str w8, [sp, #288] // 4-byte Spill
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #476]
-; NONEON-NOSVE-NEXT: str w8, [sp, #284] // 4-byte Spill
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #478]
-; NONEON-NOSVE-NEXT: str w8, [sp, #280] // 4-byte Spill
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #656]
-; NONEON-NOSVE-NEXT: str w8, [sp, #276] // 4-byte Spill
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #658]
-; NONEON-NOSVE-NEXT: str w8, [sp, #272] // 4-byte Spill
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #660]
-; NONEON-NOSVE-NEXT: str w8, [sp, #268] // 4-byte Spill
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #662]
-; NONEON-NOSVE-NEXT: str w8, [sp, #264] // 4-byte Spill
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #664]
-; NONEON-NOSVE-NEXT: str w8, [sp, #260] // 4-byte Spill
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #668]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #252] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #670]
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #528]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #244] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #530]
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #532]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #236] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #534]
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #536]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #228] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #538]
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #540]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #220] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #542]
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #496]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #212] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #498]
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #500]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #204] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #502]
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #504]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #196] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #506]
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #508]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #188] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #510]
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #512]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #180] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #514]
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #516]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #172] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #518]
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #520]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #164] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #522]
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #524]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #156] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #526]
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #640]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #148] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #642]
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #644]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #140] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #646]
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #648]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #132] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #650]
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #652]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #124] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #654]
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #576]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #116] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #578]
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #580]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #108] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #582]
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #584]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #100] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #586]
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #588]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #92] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #590]
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #544]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #84] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #546]
-; NONEON-NOSVE-NEXT: ldrh w8, [sp, #548]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #76] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #550]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #184] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #512]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #514]
+; NONEON-NOSVE-NEXT: add w9, w9, w9
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #176] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #516]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #518]
+; NONEON-NOSVE-NEXT: strb w9, [sp, #766]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #168] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #520]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #522]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #160] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #524]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #526]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #152] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #640]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #642]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #144] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #644]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #646]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #136] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #648]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #650]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #128] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #652]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #654]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #120] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #576]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #578]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #112] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #580]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #582]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #104] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #584]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #586]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #96] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #588]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #590]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #88] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #544]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #546]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #80] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #548]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #550]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #72] // 8-byte Folded Spill
; NONEON-NOSVE-NEXT: ldrh w8, [sp, #552]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #68] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #554]
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #554]
+; NONEON-NOSVE-NEXT: str w8, [sp, #68] // 4-byte Spill
; NONEON-NOSVE-NEXT: ldrh w8, [sp, #556]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #56] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #558]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #56] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #558]
; NONEON-NOSVE-NEXT: ldrh w8, [sp, #560]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #48] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #562]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #48] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #562]
; NONEON-NOSVE-NEXT: ldrh w8, [sp, #564]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #40] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #566]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #40] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #566]
; NONEON-NOSVE-NEXT: ldrh w8, [sp, #568]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #32] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #570]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #32] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #570]
; NONEON-NOSVE-NEXT: ldrh w8, [sp, #572]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #24] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #574]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #24] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #574]
; NONEON-NOSVE-NEXT: ldrh w8, [sp, #416]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #16] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #16] // 8-byte Folded Spill
; NONEON-NOSVE-NEXT: ldrh w8, [sp, #602]
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #604]
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #600]
; NONEON-NOSVE-NEXT: add w8, w8, w8
-; NONEON-NOSVE-NEXT: add w9, w9, w9
; NONEON-NOSVE-NEXT: strb w8, [sp, #765]
; NONEON-NOSVE-NEXT: add w8, w10, w10
; NONEON-NOSVE-NEXT: strb w8, [sp, #764]
@@ -858,7 +860,6 @@ define void @trunc_v128i16_v128i8(ptr %in, ptr %out) nounwind {
; NONEON-NOSVE-NEXT: add w8, w29, w29
; NONEON-NOSVE-NEXT: strb w8, [sp, #737]
; NONEON-NOSVE-NEXT: ldr w8, [sp, #16] // 4-byte Reload
-; NONEON-NOSVE-NEXT: strb w9, [sp, #766]
; NONEON-NOSVE-NEXT: add w8, w8, w8
; NONEON-NOSVE-NEXT: strb w8, [sp, #736]
; NONEON-NOSVE-NEXT: ldr w8, [sp, #20] // 4-byte Reload
@@ -1489,96 +1490,96 @@ define void @trunc_v64i32_v64i8(ptr %in, ptr %out) nounwind {
; NONEON-NOSVE-NEXT: stp x20, x19, [sp, #80] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT: sub sp, sp, #480
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0, #96]
-; NONEON-NOSVE-NEXT: str x1, [sp, #152] // 8-byte Spill
+; NONEON-NOSVE-NEXT: add x9, sp, #312
; NONEON-NOSVE-NEXT: ldp q3, q2, [x0, #64]
-; NONEON-NOSVE-NEXT: ldp q17, q16, [x0, #128]
+; NONEON-NOSVE-NEXT: str x1, [sp, #152] // 8-byte Spill
; NONEON-NOSVE-NEXT: ldp q5, q4, [x0, #32]
; NONEON-NOSVE-NEXT: ldp q7, q6, [x0]
+; NONEON-NOSVE-NEXT: ldp q17, q16, [x0, #128]
; NONEON-NOSVE-NEXT: ldp q19, q18, [x0, #224]
; NONEON-NOSVE-NEXT: ldp q21, q20, [x0, #192]
; NONEON-NOSVE-NEXT: ldp q23, q22, [x0, #160]
-; NONEON-NOSVE-NEXT: stp q1, q0, [sp, #288]
-; NONEON-NOSVE-NEXT: ldr w8, [sp, #316]
+; NONEON-NOSVE-NEXT: stp q3, q17, [sp, #384]
; NONEON-NOSVE-NEXT: str q18, [sp, #208]
-; NONEON-NOSVE-NEXT: ldr w10, [sp, #304]
; NONEON-NOSVE-NEXT: stp q21, q19, [sp, #176]
-; NONEON-NOSVE-NEXT: ldr w11, [sp, #296]
-; NONEON-NOSVE-NEXT: ldr w12, [sp, #292]
-; NONEON-NOSVE-NEXT: add w20, w8, w8
; NONEON-NOSVE-NEXT: stp q20, q23, [sp, #224]
-; NONEON-NOSVE-NEXT: ldr w13, [sp, #288]
; NONEON-NOSVE-NEXT: stp q22, q16, [sp, #256]
-; NONEON-NOSVE-NEXT: ldr w22, [sp, #312]
-; NONEON-NOSVE-NEXT: stp q3, q17, [sp, #384]
-; NONEON-NOSVE-NEXT: ldr w9, [sp, #400]
-; NONEON-NOSVE-NEXT: ldr w8, [sp, #404]
; NONEON-NOSVE-NEXT: str q7, [sp, #160]
+; NONEON-NOSVE-NEXT: stp q1, q0, [sp, #288]
; NONEON-NOSVE-NEXT: stp q2, q4, [sp, #320]
-; NONEON-NOSVE-NEXT: ldr w18, [sp, #396]
-; NONEON-NOSVE-NEXT: ldr w0, [sp, #392]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #144] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldr w9, [sp, #408]
-; NONEON-NOSVE-NEXT: ldr w8, [sp, #412]
-; NONEON-NOSVE-NEXT: ldr w14, [sp, #332]
-; NONEON-NOSVE-NEXT: ldr w15, [sp, #328]
-; NONEON-NOSVE-NEXT: ldr w16, [sp, #324]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #136] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldr w9, [sp, #272]
-; NONEON-NOSVE-NEXT: ldr w8, [sp, #276]
-; NONEON-NOSVE-NEXT: ldr w17, [sp, #320]
-; NONEON-NOSVE-NEXT: ldr w1, [sp, #388]
-; NONEON-NOSVE-NEXT: ldr w2, [sp, #384]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #128] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldr w9, [sp, #280]
-; NONEON-NOSVE-NEXT: ldr w8, [sp, #284]
-; NONEON-NOSVE-NEXT: ldr w3, [sp, #348]
-; NONEON-NOSVE-NEXT: ldr w4, [sp, #344]
-; NONEON-NOSVE-NEXT: ldr w5, [sp, #340]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #120] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldr w6, [sp, #336]
+; NONEON-NOSVE-NEXT: ldr w11, [sp, #296]
; NONEON-NOSVE-NEXT: stp q6, q5, [sp, #352]
-; NONEON-NOSVE-NEXT: ldr w7, [sp, #380]
-; NONEON-NOSVE-NEXT: ldr w19, [sp, #376]
-; NONEON-NOSVE-NEXT: ldr w21, [sp, #372]
-; NONEON-NOSVE-NEXT: ldr w23, [sp, #368]
-; NONEON-NOSVE-NEXT: ldr w24, [sp, #364]
-; NONEON-NOSVE-NEXT: ldr w25, [sp, #360]
-; NONEON-NOSVE-NEXT: ldr w26, [sp, #356]
-; NONEON-NOSVE-NEXT: ldr w27, [sp, #352]
-; NONEON-NOSVE-NEXT: strb w20, [sp, #463]
-; NONEON-NOSVE-NEXT: add w20, w22, w22
-; NONEON-NOSVE-NEXT: strb w20, [sp, #462]
-; NONEON-NOSVE-NEXT: ldp w9, w8, [sp, #240]
+; NONEON-NOSVE-NEXT: ldp w22, w8, [x9]
+; NONEON-NOSVE-NEXT: add x9, sp, #400
; NONEON-NOSVE-NEXT: ldp w29, w28, [sp, #168]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #112] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldp w9, w8, [sp, #248]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #104] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldr w9, [sp, #256]
-; NONEON-NOSVE-NEXT: ldr w8, [sp, #260]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #96] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldr w9, [sp, #264]
-; NONEON-NOSVE-NEXT: ldr w8, [sp, #268]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #88] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldp w9, w8, [sp, #176]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #80] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldp w9, w8, [sp, #184]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #72] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldp w9, w8, [sp, #224]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #64] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldp w9, w8, [sp, #232]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #56] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldp w9, w8, [sp, #192]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #48] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldp w9, w8, [sp, #200]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #40] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldp w9, w8, [sp, #208]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #32] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldp w9, w8, [sp, #216]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #24] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldr w9, [sp, #300]
+; NONEON-NOSVE-NEXT: add w20, w8, w8
+; NONEON-NOSVE-NEXT: ldp w10, w8, [x9]
+; NONEON-NOSVE-NEXT: add x9, sp, #408
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #144] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldp w10, w8, [x9]
+; NONEON-NOSVE-NEXT: add x9, sp, #272
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #136] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldp w10, w8, [x9]
+; NONEON-NOSVE-NEXT: add x9, sp, #280
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #128] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldp w10, w8, [x9]
+; NONEON-NOSVE-NEXT: add x9, sp, #256
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #120] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldp w10, w8, [sp, #240]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #112] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldp w10, w8, [sp, #248]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #104] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldp w10, w8, [x9]
+; NONEON-NOSVE-NEXT: add x9, sp, #264
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #96] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldp w10, w8, [x9]
+; NONEON-NOSVE-NEXT: add x9, sp, #352
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #88] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldp w10, w8, [sp, #176]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #80] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldp w10, w8, [sp, #184]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #72] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldp w10, w8, [sp, #224]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #64] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldp w10, w8, [sp, #232]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #56] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldp w10, w8, [sp, #192]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #48] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldp w10, w8, [sp, #200]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #40] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldp w10, w8, [sp, #208]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #32] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldp w10, w8, [sp, #216]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #24] // 8-byte Folded Spill
; NONEON-NOSVE-NEXT: ldp w8, w30, [sp, #160]
; NONEON-NOSVE-NEXT: str w8, [sp, #20] // 4-byte Spill
-; NONEON-NOSVE-NEXT: ldr w8, [sp, #308]
+; NONEON-NOSVE-NEXT: ldp w27, w26, [x9]
+; NONEON-NOSVE-NEXT: add x9, sp, #360
+; NONEON-NOSVE-NEXT: ldp w25, w24, [x9]
+; NONEON-NOSVE-NEXT: add x9, sp, #368
+; NONEON-NOSVE-NEXT: ldp w23, w21, [x9]
+; NONEON-NOSVE-NEXT: add x9, sp, #376
+; NONEON-NOSVE-NEXT: ldp w19, w7, [x9]
+; NONEON-NOSVE-NEXT: add x9, sp, #336
+; NONEON-NOSVE-NEXT: ldp w6, w5, [x9]
+; NONEON-NOSVE-NEXT: add x9, sp, #344
+; NONEON-NOSVE-NEXT: ldp w4, w3, [x9]
+; NONEON-NOSVE-NEXT: add x9, sp, #384
+; NONEON-NOSVE-NEXT: ldp w2, w1, [x9]
+; NONEON-NOSVE-NEXT: add x9, sp, #392
+; NONEON-NOSVE-NEXT: ldp w0, w18, [x9]
+; NONEON-NOSVE-NEXT: add x9, sp, #320
+; NONEON-NOSVE-NEXT: ldp w17, w16, [x9]
+; NONEON-NOSVE-NEXT: add x9, sp, #328
+; NONEON-NOSVE-NEXT: ldp w15, w14, [x9]
+; NONEON-NOSVE-NEXT: add x9, sp, #288
+; NONEON-NOSVE-NEXT: ldp w13, w12, [x9]
+; NONEON-NOSVE-NEXT: add x9, sp, #304
+; NONEON-NOSVE-NEXT: ldp w10, w8, [x9]
+; NONEON-NOSVE-NEXT: ldr w9, [sp, #300]
+; NONEON-NOSVE-NEXT: strb w20, [sp, #463]
+; NONEON-NOSVE-NEXT: add w20, w22, w22
+; NONEON-NOSVE-NEXT: strb w20, [sp, #462]
; NONEON-NOSVE-NEXT: add w8, w8, w8
; NONEON-NOSVE-NEXT: strb w8, [sp, #461]
; NONEON-NOSVE-NEXT: add w8, w10, w10
@@ -2079,95 +2080,95 @@ define void @trunc_v64i32_v64i16(ptr %in, ptr %out) nounwind {
; NONEON-NOSVE-NEXT: stp x20, x19, [sp, #80] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT: sub sp, sp, #528
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0, #32]
-; NONEON-NOSVE-NEXT: mov x5, x1
-; NONEON-NOSVE-NEXT: ldp q17, q16, [x0, #192]
-; NONEON-NOSVE-NEXT: ldp q23, q22, [x0, #224]
+; NONEON-NOSVE-NEXT: add x9, sp, #328
; NONEON-NOSVE-NEXT: ldp q3, q2, [x0]
+; NONEON-NOSVE-NEXT: mov x5, x1
; NONEON-NOSVE-NEXT: ldp q5, q4, [x0, #96]
; NONEON-NOSVE-NEXT: ldp q7, q6, [x0, #64]
+; NONEON-NOSVE-NEXT: ldp q17, q16, [x0, #192]
; NONEON-NOSVE-NEXT: ldp q19, q18, [x0, #160]
; NONEON-NOSVE-NEXT: ldp q21, q20, [x0, #128]
-; NONEON-NOSVE-NEXT: str q0, [sp, #320]
-; NONEON-NOSVE-NEXT: ldr w8, [sp, #332]
+; NONEON-NOSVE-NEXT: ldp q23, q22, [x0, #224]
+; NONEON-NOSVE-NEXT: stp q4, q6, [sp, #288]
+; NONEON-NOSVE-NEXT: str q19, [sp, #224]
+; NONEON-NOSVE-NEXT: stp q18, q20, [sp, #240]
; NONEON-NOSVE-NEXT: stp q17, q23, [sp, #160]
-; NONEON-NOSVE-NEXT: ldr w10, [sp, #320]
; NONEON-NOSVE-NEXT: stp q22, q16, [sp, #192]
-; NONEON-NOSVE-NEXT: ldr w23, [sp, #328]
-; NONEON-NOSVE-NEXT: add w21, w8, w8
-; NONEON-NOSVE-NEXT: stp q18, q20, [sp, #240]
-; NONEON-NOSVE-NEXT: ldp w9, w8, [sp, #160]
-; NONEON-NOSVE-NEXT: stp q7, q21, [sp, #368]
-; NONEON-NOSVE-NEXT: str q19, [sp, #224]
-; NONEON-NOSVE-NEXT: ldr w29, [sp, #380]
-; NONEON-NOSVE-NEXT: ldr w30, [sp, #376]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #136] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldp w9, w8, [sp, #168]
-; NONEON-NOSVE-NEXT: stp q4, q6, [sp, #288]
+; NONEON-NOSVE-NEXT: str q5, [sp, #272]
+; NONEON-NOSVE-NEXT: str q3, [sp, #144]
; NONEON-NOSVE-NEXT: stp q2, q1, [sp, #336]
-; NONEON-NOSVE-NEXT: ldr w3, [sp, #300]
-; NONEON-NOSVE-NEXT: ldr w4, [sp, #296]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #128] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: stp q7, q21, [sp, #368]
; NONEON-NOSVE-NEXT: ldr w11, [sp, #360]
-; NONEON-NOSVE-NEXT: ldr w12, [sp, #356]
-; NONEON-NOSVE-NEXT: ldp w9, w8, [sp, #208]
-; NONEON-NOSVE-NEXT: ldr w13, [sp, #352]
-; NONEON-NOSVE-NEXT: ldr w14, [sp, #348]
-; NONEON-NOSVE-NEXT: ldr w15, [sp, #344]
-; NONEON-NOSVE-NEXT: str q3, [sp, #144]
-; NONEON-NOSVE-NEXT: ldr w16, [sp, #340]
-; NONEON-NOSVE-NEXT: ldr w17, [sp, #336]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #120] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldr w6, [sp, #292]
-; NONEON-NOSVE-NEXT: ldr w7, [sp, #288]
-; NONEON-NOSVE-NEXT: str q5, [sp, #272]
-; NONEON-NOSVE-NEXT: ldr w25, [sp, #316]
-; NONEON-NOSVE-NEXT: ldr w26, [sp, #312]
-; NONEON-NOSVE-NEXT: ldr w19, [sp, #284]
-; NONEON-NOSVE-NEXT: ldr w20, [sp, #280]
-; NONEON-NOSVE-NEXT: ldr w22, [sp, #276]
-; NONEON-NOSVE-NEXT: ldr w24, [sp, #272]
-; NONEON-NOSVE-NEXT: ldr w27, [sp, #308]
-; NONEON-NOSVE-NEXT: ldr w28, [sp, #304]
-; NONEON-NOSVE-NEXT: strh w21, [sp, #494]
-; NONEON-NOSVE-NEXT: add w21, w23, w23
-; NONEON-NOSVE-NEXT: strh w21, [sp, #492]
-; NONEON-NOSVE-NEXT: ldp w9, w8, [sp, #216]
+; NONEON-NOSVE-NEXT: str q0, [sp, #320]
+; NONEON-NOSVE-NEXT: ldp w23, w8, [x9]
+; NONEON-NOSVE-NEXT: add x9, sp, #384
; NONEON-NOSVE-NEXT: ldp w0, w18, [sp, #152]
+; NONEON-NOSVE-NEXT: add w21, w8, w8
+; NONEON-NOSVE-NEXT: ldp w10, w8, [sp, #160]
; NONEON-NOSVE-NEXT: ldp w2, w1, [sp, #144]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #112] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldp w9, w8, [sp, #176]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #104] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldp w9, w8, [sp, #184]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #96] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldp w9, w8, [sp, #192]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #88] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldp w9, w8, [sp, #200]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #80] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldr w9, [sp, #384]
-; NONEON-NOSVE-NEXT: ldr w8, [sp, #388]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #72] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldr w9, [sp, #392]
-; NONEON-NOSVE-NEXT: ldr w8, [sp, #396]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #64] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldr w9, [sp, #256]
-; NONEON-NOSVE-NEXT: ldr w8, [sp, #260]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #56] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldr w9, [sp, #264]
-; NONEON-NOSVE-NEXT: ldr w8, [sp, #268]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #48] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldp w9, w8, [sp, #224]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #40] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldp w9, w8, [sp, #232]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #32] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldp w9, w8, [sp, #240]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #24] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldp w9, w8, [sp, #248]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #16] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldr w9, [sp, #368]
-; NONEON-NOSVE-NEXT: ldr w8, [sp, #372]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #8] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT: ldr w8, [sp, #324]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #136] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldp w10, w8, [sp, #168]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #128] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldp w10, w8, [sp, #208]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #120] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldp w10, w8, [sp, #216]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #112] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldp w10, w8, [sp, #176]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #104] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldp w10, w8, [sp, #184]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #96] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldp w10, w8, [sp, #192]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #88] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldp w10, w8, [sp, #200]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #80] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldp w10, w8, [x9]
+; NONEON-NOSVE-NEXT: add x9, sp, #392
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #72] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldp w10, w8, [x9]
+; NONEON-NOSVE-NEXT: add x9, sp, #256
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #64] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldp w10, w8, [x9]
+; NONEON-NOSVE-NEXT: add x9, sp, #264
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #56] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldp w10, w8, [x9]
+; NONEON-NOSVE-NEXT: add x9, sp, #368
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #48] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldp w10, w8, [sp, #224]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #40] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldp w10, w8, [sp, #232]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #32] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldp w10, w8, [sp, #240]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #24] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldp w10, w8, [sp, #248]
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #16] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldp w10, w8, [x9]
+; NONEON-NOSVE-NEXT: add x9, sp, #376
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #8] // 8-byte Folded Spill
+; NONEON-NOSVE-NEXT: ldp w30, w29, [x9]
+; NONEON-NOSVE-NEXT: add x9, sp, #304
+; NONEON-NOSVE-NEXT: ldp w28, w27, [x9]
+; NONEON-NOSVE-NEXT: add x9, sp, #312
+; NONEON-NOSVE-NEXT: ldp w26, w25, [x9]
+; NONEON-NOSVE-NEXT: add x9, sp, #272
+; NONEON-NOSVE-NEXT: ldp w24, w22, [x9]
+; NONEON-NOSVE-NEXT: add x9, sp, #280
+; NONEON-NOSVE-NEXT: ldp w20, w19, [x9]
+; NONEON-NOSVE-NEXT: add x9, sp, #288
+; NONEON-NOSVE-NEXT: ldp w7, w6, [x9]
+; NONEON-NOSVE-NEXT: add x9, sp, #296
+; NONEON-NOSVE-NEXT: ldp w4, w3, [x9]
+; NONEON-NOSVE-NEXT: add x9, sp, #336
+; NONEON-NOSVE-NEXT: ldp w17, w16, [x9]
+; NONEON-NOSVE-NEXT: add x9, sp, #344
+; NONEON-NOSVE-NEXT: ldp w15, w14, [x9]
+; NONEON-NOSVE-NEXT: add x9, sp, #352
+; NONEON-NOSVE-NEXT: ldp w13, w12, [x9]
+; NONEON-NOSVE-NEXT: add x9, sp, #320
+; NONEON-NOSVE-NEXT: ldp w10, w8, [x9]
; NONEON-NOSVE-NEXT: ldr w9, [sp, #364]
+; NONEON-NOSVE-NEXT: strh w21, [sp, #494]
+; NONEON-NOSVE-NEXT: add w21, w23, w23
+; NONEON-NOSVE-NEXT: strh w21, [sp, #492]
; NONEON-NOSVE-NEXT: add w8, w8, w8
; NONEON-NOSVE-NEXT: strh w8, [sp, #490]
; NONEON-NOSVE-NEXT: add w8, w10, w10
@@ -3410,19 +3411,19 @@ define void @trunc_v32i64_v32i32(ptr %in, ptr %out) nounwind {
; NONEON-NOSVE-NEXT: ldr w5, [sp, #104]
; NONEON-NOSVE-NEXT: str w8, [sp, #380]
; NONEON-NOSVE-NEXT: add w8, w20, w20
-; NONEON-NOSVE-NEXT: ldr w13, [sp, #56]
+; NONEON-NOSVE-NEXT: ldr w29, [sp, #224]
; NONEON-NOSVE-NEXT: str w9, [sp, #376]
; NONEON-NOSVE-NEXT: add w9, w19, w19
-; NONEON-NOSVE-NEXT: ldr w10, [sp, #80]
+; NONEON-NOSVE-NEXT: ldr w30, [sp, #232]
; NONEON-NOSVE-NEXT: str w8, [sp, #372]
; NONEON-NOSVE-NEXT: add w8, w7, w7
-; NONEON-NOSVE-NEXT: ldr w11, [sp, #88]
+; NONEON-NOSVE-NEXT: ldr w13, [sp, #56]
; NONEON-NOSVE-NEXT: str w9, [sp, #368]
; NONEON-NOSVE-NEXT: add w9, w6, w6
-; NONEON-NOSVE-NEXT: ldr w29, [sp, #224]
+; NONEON-NOSVE-NEXT: ldr w10, [sp, #80]
; NONEON-NOSVE-NEXT: str w8, [sp, #316]
; NONEON-NOSVE-NEXT: add w8, w5, w5
-; NONEON-NOSVE-NEXT: ldr w30, [sp, #232]
+; NONEON-NOSVE-NEXT: ldr w11, [sp, #88]
; NONEON-NOSVE-NEXT: str w9, [sp, #312]
; NONEON-NOSVE-NEXT: add w9, w4, w4
; NONEON-NOSVE-NEXT: str w8, [sp, #308]
@@ -3433,45 +3434,45 @@ define void @trunc_v32i64_v32i32(ptr %in, ptr %out) nounwind {
; NONEON-NOSVE-NEXT: add w8, w17, w17
; NONEON-NOSVE-NEXT: str w9, [sp, #392]
; NONEON-NOSVE-NEXT: add w9, w16, w16
+; NONEON-NOSVE-NEXT: str w2, [sp, #364]
+; NONEON-NOSVE-NEXT: add w2, w30, w30
+; NONEON-NOSVE-NEXT: str w3, [sp, #360]
+; NONEON-NOSVE-NEXT: add w3, w29, w29
; NONEON-NOSVE-NEXT: str w8, [sp, #388]
; NONEON-NOSVE-NEXT: add w8, w15, w15
; NONEON-NOSVE-NEXT: str w9, [sp, #384]
; NONEON-NOSVE-NEXT: add w9, w14, w14
-; NONEON-NOSVE-NEXT: str w8, [sp, #284]
+; NONEON-NOSVE-NEXT: add x14, sp, #280
+; NONEON-NOSVE-NEXT: str w2, [sp, #356]
+; NONEON-NOSVE-NEXT: str w3, [sp, #352]
+; NONEON-NOSVE-NEXT: stp w9, w8, [x14]
; NONEON-NOSVE-NEXT: add w8, w13, w13
-; NONEON-NOSVE-NEXT: str w9, [sp, #280]
; NONEON-NOSVE-NEXT: add w9, w12, w12
-; NONEON-NOSVE-NEXT: str w8, [sp, #276]
+; NONEON-NOSVE-NEXT: add x12, sp, #272
+; NONEON-NOSVE-NEXT: stp w9, w8, [x12]
; NONEON-NOSVE-NEXT: add w8, w11, w11
-; NONEON-NOSVE-NEXT: str w9, [sp, #272]
; NONEON-NOSVE-NEXT: add w9, w10, w10
-; NONEON-NOSVE-NEXT: str w8, [sp, #300]
+; NONEON-NOSVE-NEXT: add x10, sp, #296
+; NONEON-NOSVE-NEXT: stp w9, w8, [x10]
+; NONEON-NOSVE-NEXT: add x10, sp, #288
+; NONEON-NOSVE-NEXT: ldp w8, w9, [sp, #8] // 8-byte Folded Reload
+; NONEON-NOSVE-NEXT: add w8, w8, w8
+; NONEON-NOSVE-NEXT: add w9, w9, w9
+; NONEON-NOSVE-NEXT: stp w9, w8, [x10]
+; NONEON-NOSVE-NEXT: ldp q1, q0, [sp, #336]
+; NONEON-NOSVE-NEXT: ldp q5, q2, [sp, #272]
; NONEON-NOSVE-NEXT: ldp q6, q3, [sp, #304]
-; NONEON-NOSVE-NEXT: str w9, [sp, #296]
; NONEON-NOSVE-NEXT: ldp q4, q7, [sp, #368]
-; NONEON-NOSVE-NEXT: str w2, [sp, #364]
-; NONEON-NOSVE-NEXT: add w2, w30, w30
-; NONEON-NOSVE-NEXT: str w3, [sp, #360]
-; NONEON-NOSVE-NEXT: add w3, w29, w29
-; NONEON-NOSVE-NEXT: str w2, [sp, #356]
+; NONEON-NOSVE-NEXT: stp q1, q0, [x1]
; NONEON-NOSVE-NEXT: ldp x20, x19, [sp, #480] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT: str w3, [sp, #352]
+; NONEON-NOSVE-NEXT: stp q2, q5, [x1, #96]
; NONEON-NOSVE-NEXT: ldp x22, x21, [sp, #464] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT: ldp w8, w9, [sp, #8] // 8-byte Folded Reload
-; NONEON-NOSVE-NEXT: ldp q1, q0, [sp, #336]
; NONEON-NOSVE-NEXT: ldp x24, x23, [sp, #448] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT: add w8, w8, w8
-; NONEON-NOSVE-NEXT: add w9, w9, w9
-; NONEON-NOSVE-NEXT: str w8, [sp, #292]
+; NONEON-NOSVE-NEXT: stp q4, q3, [x1, #32]
; NONEON-NOSVE-NEXT: ldp x26, x25, [sp, #432] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT: str w9, [sp, #288]
+; NONEON-NOSVE-NEXT: stp q7, q6, [x1, #64]
; NONEON-NOSVE-NEXT: ldp x28, x27, [sp, #416] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT: ldp q5, q2, [sp, #272]
-; NONEON-NOSVE-NEXT: stp q1, q0, [x1]
-; NONEON-NOSVE-NEXT: stp q4, q3, [x1, #32]
; NONEON-NOSVE-NEXT: ldp x29, x30, [sp, #400] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT: stp q7, q6, [x1, #64]
-; NONEON-NOSVE-NEXT: stp q2, q5, [x1, #96]
; NONEON-NOSVE-NEXT: add sp, sp, #496
; NONEON-NOSVE-NEXT: ret
%a = load <32 x i64>, ptr %in
More information about the llvm-commits
mailing list