[llvm] [AArch64] Allow LDR merge with same destination register by renaming (PR #71908)
Zhaoxuan Jiang via llvm-commits
llvm-commits at lists.llvm.org
Sat Nov 11 00:18:49 PST 2023
https://github.com/nocchijiang updated https://github.com/llvm/llvm-project/pull/71908
>From 5649cd6725dc34dc6af488f27e97eb7c98e8f0e1 Mon Sep 17 00:00:00 2001
From: Zhaoxuan Jiang <jiangzhaoxuan94 at gmail.com>
Date: Fri, 10 Nov 2023 16:23:38 +0800
Subject: [PATCH 1/2] [AArch64] Allow LDR merge with same destination register
by renaming
The patch is based on a reverted patch: https://reviews.llvm.org/D103597.
It was trying to rename registers before alias check, which is not safe
and causes miscompiles. This patch does 2 things:
1. Do the renaming with necessary checks passed, including alias check.
2. Rename the register for the instructions between the pairs and
combine the second load into the first. By doing so we can just check
the renamability between the pairs and avoid scanning unknown amount
of instructions before/after the pairs.
Necessary refactoring has been made in order to reuse as much code
possible with STR renaming.
---
.../AArch64/AArch64LoadStoreOptimizer.cpp | 272 ++++++++++++------
.../CodeGen/AArch64/arm64-variadic-aapcs.ll | 3 +-
llvm/test/CodeGen/AArch64/fexplog.ll | 100 +++----
llvm/test/CodeGen/AArch64/fpow.ll | 74 +++--
llvm/test/CodeGen/AArch64/frem.ll | 74 +++--
llvm/test/CodeGen/AArch64/fsincos.ll | 40 ++-
.../test/CodeGen/AArch64/ldrpre-ldr-merge.mir | 5 +-
llvm/test/CodeGen/AArch64/neon-dotreduce.ll | 42 ++-
.../CodeGen/AArch64/stp-opt-with-renaming.mir | 134 ++++++++-
.../CodeGen/AArch64/sve-fixed-length-fp128.ll | 17 +-
10 files changed, 471 insertions(+), 290 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp b/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
index 299ea43a539edb7..3827cef566f12ee 100644
--- a/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
@@ -816,12 +816,11 @@ AArch64LoadStoreOpt::mergeNarrowZeroStores(MachineBasicBlock::iterator I,
return NextI;
}
-// Apply Fn to all instructions between MI and the beginning of the block, until
-// a def for DefReg is reached. Returns true, iff Fn returns true for all
-// visited instructions. Stop after visiting Limit iterations.
-static bool forAllMIsUntilDef(MachineInstr &MI, MCPhysReg DefReg,
- const TargetRegisterInfo *TRI, unsigned Limit,
- std::function<bool(MachineInstr &, bool)> &Fn) {
+static bool forAllMIsUntil(MachineInstr &MI,
+ std::function<bool(MachineInstr &MI, bool)> Until,
+ MCPhysReg DefReg, const TargetRegisterInfo *TRI,
+ unsigned Limit,
+ std::function<bool(MachineInstr &, bool)> &Fn) {
auto MBB = MI.getParent();
for (MachineInstr &I :
instructionsWithoutDebug(MI.getReverseIterator(), MBB->instr_rend())) {
@@ -841,6 +840,17 @@ static bool forAllMIsUntilDef(MachineInstr &MI, MCPhysReg DefReg,
return true;
}
+// Apply Fn to all instructions between MI and the beginning of the block, until
+// a def for DefReg is reached. Returns true, iff Fn returns true for all
+// visited instructions. Stop after visiting Limit iterations.
+static bool forAllMIsUntilDef(MachineInstr &MI, MCPhysReg DefReg,
+ const TargetRegisterInfo *TRI, unsigned Limit,
+ std::function<bool(MachineInstr &, bool)> &Fn) {
+ return forAllMIsUntil(
+ MI, [](MachineInstr &, bool IsDef) { return IsDef; }, DefReg, TRI, Limit,
+ Fn);
+}
+
static void updateDefinedRegisters(MachineInstr &MI, LiveRegUnits &Units,
const TargetRegisterInfo *TRI) {
@@ -875,7 +885,7 @@ AArch64LoadStoreOpt::mergePairedInsns(MachineBasicBlock::iterator I,
bool MergeForward = Flags.getMergeForward();
std::optional<MCPhysReg> RenameReg = Flags.getRenameReg();
- if (MergeForward && RenameReg) {
+ if (RenameReg) {
MCRegister RegToRename = getLdStRegOp(*I).getReg();
DefinedInBB.addReg(*RenameReg);
@@ -892,7 +902,8 @@ AArch64LoadStoreOpt::mergePairedInsns(MachineBasicBlock::iterator I,
};
std::function<bool(MachineInstr &, bool)> UpdateMIs =
- [this, RegToRename, GetMatchingSubReg](MachineInstr &MI, bool IsDef) {
+ [this, RegToRename, GetMatchingSubReg, MergeForward](MachineInstr &MI,
+ bool IsDef) {
if (IsDef) {
bool SeenDef = false;
for (unsigned OpIdx = 0; OpIdx < MI.getNumOperands(); ++OpIdx) {
@@ -900,7 +911,8 @@ AArch64LoadStoreOpt::mergePairedInsns(MachineBasicBlock::iterator I,
// Rename the first explicit definition and all implicit
// definitions matching RegToRename.
if (MOP.isReg() && !MOP.isDebug() && MOP.getReg() &&
- (!SeenDef || (MOP.isDef() && MOP.isImplicit())) &&
+ (!MergeForward || !SeenDef ||
+ (MOP.isDef() && MOP.isImplicit())) &&
TRI->regsOverlap(MOP.getReg(), RegToRename)) {
assert((MOP.isImplicit() ||
(MOP.isRenamable() && !MOP.isEarlyClobber())) &&
@@ -938,20 +950,35 @@ AArch64LoadStoreOpt::mergePairedInsns(MachineBasicBlock::iterator I,
LLVM_DEBUG(dbgs() << "Renamed " << MI << "\n");
return true;
};
- forAllMIsUntilDef(*I, RegToRename, TRI, LdStLimit, UpdateMIs);
+ if (MergeForward)
+ forAllMIsUntilDef(*I, RegToRename, TRI, LdStLimit, UpdateMIs);
+ else
+ forAllMIsUntil(
+ *std::prev(Paired),
+ [&I](MachineInstr &MI, bool) { return &MI == &*I; }, RegToRename, TRI,
+ LdStLimit, UpdateMIs);
#if !defined(NDEBUG)
- // Make sure the register used for renaming is not used between the paired
- // instructions. That would trash the content before the new paired
- // instruction.
+ // For forward merging store:
+ // Make sure the register used for renaming is not used between the
+ // paired instructions. That would trash the content before the new
+ // paired instruction.
+ MCPhysReg RegToCheck = *RenameReg;
+ // For backward merging load:
+ // Make sure the register being renamed is not used between the
+ // paired instructions. That would trash the content after the new
+ // paired instruction.
+ if (!MergeForward)
+ RegToCheck = RegToRename;
for (auto &MI :
iterator_range<MachineInstrBundleIterator<llvm::MachineInstr>>(
- std::next(I), std::next(Paired)))
+ MergeForward ? std::next(I) : I,
+ MergeForward ? std::next(Paired) : Paired))
assert(all_of(MI.operands(),
- [this, &RenameReg](const MachineOperand &MOP) {
+ [this, RegToCheck](const MachineOperand &MOP) {
return !MOP.isReg() || MOP.isDebug() || !MOP.getReg() ||
MOP.isUndef() ||
- !TRI->regsOverlap(MOP.getReg(), *RenameReg);
+ !TRI->regsOverlap(MOP.getReg(), RegToCheck);
}) &&
"Rename register used between paired instruction, trashing the "
"content");
@@ -1397,6 +1424,38 @@ static bool areCandidatesToMergeOrPair(MachineInstr &FirstMI, MachineInstr &MI,
// FIXME: Can we also match a mixed sext/zext unscaled/scaled pair?
}
+static bool canRenameMOP(const MachineOperand &MOP,
+ const TargetRegisterInfo *TRI) {
+ if (MOP.isReg()) {
+ auto *RegClass = TRI->getMinimalPhysRegClass(MOP.getReg());
+ // Renaming registers with multiple disjunct sub-registers (e.g. the
+ // result of a LD3) means that all sub-registers are renamed, potentially
+ // impacting other instructions we did not check. Bail out.
+ // Note that this relies on the structure of the AArch64 register file. In
+ // particular, a subregister cannot be written without overwriting the
+ // whole register.
+ if (RegClass->HasDisjunctSubRegs) {
+ LLVM_DEBUG(
+ dbgs()
+ << " Cannot rename operands with multiple disjunct subregisters ("
+ << MOP << ")\n");
+ return false;
+ }
+
+ // We cannot rename arbitrary implicit-defs, the specific rule to rewrite
+ // them must be known. For example, in ORRWrs the implicit-def
+ // corresponds to the result register.
+ if (MOP.isImplicit() && MOP.isDef()) {
+ if (!isRewritableImplicitDef(MOP.getParent()->getOpcode()))
+ return false;
+ return TRI->isSuperOrSubRegisterEq(
+ MOP.getParent()->getOperand(0).getReg(), MOP.getReg());
+ }
+ }
+ return MOP.isImplicit() ||
+ (MOP.isRenamable() && !MOP.isEarlyClobber() && !MOP.isTied());
+}
+
static bool
canRenameUpToDef(MachineInstr &FirstMI, LiveRegUnits &UsedInBetween,
SmallPtrSetImpl<const TargetRegisterClass *> &RequiredClasses,
@@ -1406,10 +1465,6 @@ canRenameUpToDef(MachineInstr &FirstMI, LiveRegUnits &UsedInBetween,
// Check if we can find an unused register which we can use to rename
// the register used by the first load/store.
- auto *RegClass = TRI->getMinimalPhysRegClass(getLdStRegOp(FirstMI).getReg());
- MachineFunction &MF = *FirstMI.getParent()->getParent();
- if (!RegClass || !MF.getRegInfo().tracksLiveness())
- return false;
auto RegToRename = getLdStRegOp(FirstMI).getReg();
// For now, we only rename if the store operand gets killed at the store.
@@ -1423,36 +1478,6 @@ canRenameUpToDef(MachineInstr &FirstMI, LiveRegUnits &UsedInBetween,
LLVM_DEBUG(dbgs() << " Operand not killed at " << FirstMI << "\n");
return false;
}
- auto canRenameMOP = [TRI](const MachineOperand &MOP) {
- if (MOP.isReg()) {
- auto *RegClass = TRI->getMinimalPhysRegClass(MOP.getReg());
- // Renaming registers with multiple disjunct sub-registers (e.g. the
- // result of a LD3) means that all sub-registers are renamed, potentially
- // impacting other instructions we did not check. Bail out.
- // Note that this relies on the structure of the AArch64 register file. In
- // particular, a subregister cannot be written without overwriting the
- // whole register.
- if (RegClass->HasDisjunctSubRegs) {
- LLVM_DEBUG(
- dbgs()
- << " Cannot rename operands with multiple disjunct subregisters ("
- << MOP << ")\n");
- return false;
- }
-
- // We cannot rename arbitrary implicit-defs, the specific rule to rewrite
- // them must be known. For example, in ORRWrs the implicit-def
- // corresponds to the result register.
- if (MOP.isImplicit() && MOP.isDef()) {
- if (!isRewritableImplicitDef(MOP.getParent()->getOpcode()))
- return false;
- return TRI->isSuperOrSubRegisterEq(
- MOP.getParent()->getOperand(0).getReg(), MOP.getReg());
- }
- }
- return MOP.isImplicit() ||
- (MOP.isRenamable() && !MOP.isEarlyClobber() && !MOP.isTied());
- };
bool FoundDef = false;
@@ -1495,7 +1520,7 @@ canRenameUpToDef(MachineInstr &FirstMI, LiveRegUnits &UsedInBetween,
if (!MOP.isReg() || !MOP.isDef() || MOP.isDebug() || !MOP.getReg() ||
!TRI->regsOverlap(MOP.getReg(), RegToRename))
continue;
- if (!canRenameMOP(MOP)) {
+ if (!canRenameMOP(MOP, TRI)) {
LLVM_DEBUG(dbgs()
<< " Cannot rename " << MOP << " in " << MI << "\n");
return false;
@@ -1509,7 +1534,7 @@ canRenameUpToDef(MachineInstr &FirstMI, LiveRegUnits &UsedInBetween,
!TRI->regsOverlap(MOP.getReg(), RegToRename))
continue;
- if (!canRenameMOP(MOP)) {
+ if (!canRenameMOP(MOP, TRI)) {
LLVM_DEBUG(dbgs()
<< " Cannot rename " << MOP << " in " << MI << "\n");
return false;
@@ -1530,6 +1555,56 @@ canRenameUpToDef(MachineInstr &FirstMI, LiveRegUnits &UsedInBetween,
return true;
}
+// We want to merge the second load into the first by rewriting the usages of
+// the same reg between first (incl.) and second (excl.). We don't need to care
+// about any insns before FirstLoad or after SecondLoad.
+// 1. The second load writes new value into the same reg.
+// - The renaming is impossible to impact later use of the reg.
+// - The second load always trash the value written by the first load which
+// means the reg must be killed before the second load.
+// 2. The first load must be a def for the same reg so we don't need to look
+// into anything before it.
+static bool canRenameUntilSecondLoad(
+ MachineInstr &FirstLoad, MachineInstr &SecondLoad,
+ LiveRegUnits &UsedInBetween,
+ SmallPtrSetImpl<const TargetRegisterClass *> &RequiredClasses,
+ const TargetRegisterInfo *TRI) {
+ if (FirstLoad.isPseudo())
+ return false;
+
+ UsedInBetween.accumulate(FirstLoad);
+ auto RegToRename = getLdStRegOp(FirstLoad).getReg();
+ bool Success = std::all_of(
+ FirstLoad.getIterator(), SecondLoad.getIterator(),
+ [&](MachineBasicBlock::iterator MBBI) {
+ MachineInstr &MI = *MBBI;
+
+ LLVM_DEBUG(dbgs() << "Checking " << MI << "\n");
+ // Currently we do not try to rename across frame-setup instructions.
+ if (MI.getFlag(MachineInstr::FrameSetup)) {
+ LLVM_DEBUG(dbgs()
+ << " Cannot rename framesetup instructions currently ("
+ << MI << ")\n");
+ return false;
+ }
+
+ for (auto &MOP : MI.operands()) {
+ if (!MOP.isReg() || MOP.isDebug() || !MOP.getReg() ||
+ !TRI->regsOverlap(MOP.getReg(), RegToRename))
+ continue;
+ if (!canRenameMOP(MOP, TRI)) {
+ LLVM_DEBUG(dbgs()
+ << " Cannot rename " << MOP << " in " << MI << "\n");
+ return false;
+ }
+ RequiredClasses.insert(TRI->getMinimalPhysRegClass(MOP.getReg()));
+ }
+
+ return true;
+ });
+ return Success;
+}
+
// Check if we can find a physical register for renaming \p Reg. This register
// must:
// * not be defined already in \p DefinedInBB; DefinedInBB must contain all
@@ -1578,6 +1653,41 @@ static std::optional<MCPhysReg> tryToFindRegisterToRename(
return std::nullopt;
}
+// For store pairs: returns a register from FirstMI to the beginning of the
+// block that can be renamed.
+// For load pairs: returns a register from FirstMI to MI that can be renamed.
+static std::optional<MCPhysReg> findRenameRegForSameLdStRegPair(
+ std::optional<bool> MaybeCanRename, MachineInstr &FirstMI, MachineInstr &MI,
+ Register Reg, LiveRegUnits &DefinedInBB, LiveRegUnits &UsedInBetween,
+ SmallPtrSetImpl<const TargetRegisterClass *> &RequiredClasses,
+ const TargetRegisterInfo *TRI) {
+ std::optional<MCPhysReg> RenameReg;
+ if (!DebugCounter::shouldExecute(RegRenamingCounter))
+ return RenameReg;
+
+ auto *RegClass = TRI->getMinimalPhysRegClass(getLdStRegOp(FirstMI).getReg());
+ MachineFunction &MF = *FirstMI.getParent()->getParent();
+ if (!RegClass || !MF.getRegInfo().tracksLiveness())
+ return RenameReg;
+
+ const bool IsLoad = FirstMI.mayLoad();
+
+ if (!MaybeCanRename) {
+ if (IsLoad)
+ MaybeCanRename = {canRenameUntilSecondLoad(FirstMI, MI, UsedInBetween,
+ RequiredClasses, TRI)};
+ else
+ MaybeCanRename = {
+ canRenameUpToDef(FirstMI, UsedInBetween, RequiredClasses, TRI)};
+ }
+
+ if (*MaybeCanRename) {
+ RenameReg = tryToFindRegisterToRename(MF, Reg, DefinedInBB, UsedInBetween,
+ RequiredClasses, TRI);
+ }
+ return RenameReg;
+}
+
/// Scan the instructions looking for a load/store that can be combined with the
/// current instruction into a wider equivalent or a load/store pair.
MachineBasicBlock::iterator
@@ -1730,17 +1840,6 @@ AArch64LoadStoreOpt::findMatchingInsn(MachineBasicBlock::iterator I,
continue;
}
}
- // If the destination register of one load is the same register or a
- // sub/super register of the other load, bail and keep looking. A
- // load-pair instruction with both destination registers the same is
- // UNPREDICTABLE and will result in an exception.
- if (MayLoad &&
- TRI->isSuperOrSubRegisterEq(Reg, getLdStRegOp(MI).getReg())) {
- LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits,
- TRI);
- MemInsns.push_back(&MI);
- continue;
- }
// If the BaseReg has been modified, then we cannot do the optimization.
// For example, in the following pattern
@@ -1751,17 +1850,37 @@ AArch64LoadStoreOpt::findMatchingInsn(MachineBasicBlock::iterator I,
if (!ModifiedRegUnits.available(BaseReg))
return E;
+ const bool SameLoadReg = MayLoad && TRI->isSuperOrSubRegisterEq(
+ Reg, getLdStRegOp(MI).getReg());
+
// If the Rt of the second instruction was not modified or used between
// the two instructions and none of the instructions between the second
// and first alias with the second, we can combine the second into the
// first.
if (ModifiedRegUnits.available(getLdStRegOp(MI).getReg()) &&
- !(MI.mayLoad() &&
+ !(MI.mayLoad() && !SameLoadReg &&
!UsedRegUnits.available(getLdStRegOp(MI).getReg())) &&
!mayAlias(MI, MemInsns, AA)) {
+ // For pairs loading into the same reg, try to find a renaming
+ // opportunity to allow the renaming of Reg between FirstMI and MI
+ // and combine MI into FirstMI; otherwise bail and keep looking.
+ if (SameLoadReg) {
+ std::optional<MCPhysReg> RenameReg =
+ findRenameRegForSameLdStRegPair(MaybeCanRename, FirstMI, MI,
+ Reg, DefinedInBB, UsedInBetween,
+ RequiredClasses, TRI);
+ if (!RenameReg) {
+ LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits,
+ UsedRegUnits, TRI);
+ MemInsns.push_back(&MI);
+ continue;
+ }
+ Flags.setRenameReg(*RenameReg);
+ }
Flags.setMergeForward(false);
- Flags.clearRenameReg();
+ if (!SameLoadReg)
+ Flags.clearRenameReg();
return MBBI;
}
@@ -1779,22 +1898,13 @@ AArch64LoadStoreOpt::findMatchingInsn(MachineBasicBlock::iterator I,
return MBBI;
}
- if (DebugCounter::shouldExecute(RegRenamingCounter)) {
- if (!MaybeCanRename)
- MaybeCanRename = {canRenameUpToDef(FirstMI, UsedInBetween,
- RequiredClasses, TRI)};
-
- if (*MaybeCanRename) {
- std::optional<MCPhysReg> MaybeRenameReg =
- tryToFindRegisterToRename(*FirstMI.getParent()->getParent(),
- Reg, DefinedInBB, UsedInBetween,
- RequiredClasses, TRI);
- if (MaybeRenameReg) {
- Flags.setRenameReg(*MaybeRenameReg);
- Flags.setMergeForward(true);
- MBBIWithRenameReg = MBBI;
- }
- }
+ std::optional<MCPhysReg> RenameReg = findRenameRegForSameLdStRegPair(
+ MaybeCanRename, FirstMI, MI, Reg, DefinedInBB, UsedInBetween,
+ RequiredClasses, TRI);
+ if (RenameReg) {
+ Flags.setMergeForward(true);
+ Flags.setRenameReg(*RenameReg);
+ MBBIWithRenameReg = MBBI;
}
}
// Unable to combine these instructions due to interference in between.
diff --git a/llvm/test/CodeGen/AArch64/arm64-variadic-aapcs.ll b/llvm/test/CodeGen/AArch64/arm64-variadic-aapcs.ll
index de07ed1b5d7ec96..e688af7fdeca38a 100644
--- a/llvm/test/CodeGen/AArch64/arm64-variadic-aapcs.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-variadic-aapcs.ll
@@ -125,9 +125,8 @@ define dso_local void @test_va_copy() {
; CHECK: add x[[SRC:[0-9]+]], {{x[0-9]+}}, :lo12:var
-; CHECK: ldr [[BLOCKB:q[0-9]+]], [x[[SRC]], #16]
+; CHECK: ldp [[BLOCKA:q[0-9]+]], [[BLOCKB:q[0-9]+]], [x[[SRC]]]
; CHECK: add x[[DST:[0-9]+]], {{x[0-9]+}}, :lo12:second_list
-; CHECK: ldr [[BLOCKA:q[0-9]+]], [x[[SRC]]]
; CHECK: stp [[BLOCKA]], [[BLOCKB]], [x[[DST]]]
ret void
; CHECK: ret
diff --git a/llvm/test/CodeGen/AArch64/fexplog.ll b/llvm/test/CodeGen/AArch64/fexplog.ll
index be6040faa2836bf..26c0b68307b32b4 100644
--- a/llvm/test/CodeGen/AArch64/fexplog.ll
+++ b/llvm/test/CodeGen/AArch64/fexplog.ll
@@ -713,14 +713,12 @@ define <7 x half> @exp_v7f16(<7 x half> %a) {
; CHECK-GI-NEXT: ldr x30, [sp, #144] // 8-byte Folded Reload
; CHECK-GI-NEXT: ldp d11, d10, [sp, #112] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[1], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #48] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #32] // 32-byte Folded Reload
; CHECK-GI-NEXT: ldp d13, d12, [sp, #96] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[2], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[2], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[3], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[4], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp] // 32-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[4], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[5], v2.h[0]
; CHECK-GI-NEXT: mov v1.h[6], v0.h[0]
; CHECK-GI-NEXT: mov v1.h[7], v0.h[0]
@@ -963,14 +961,12 @@ define <8 x half> @exp_v8f16(<8 x half> %a) {
; CHECK-GI-NEXT: ldp d11, d10, [sp, #136] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldr d14, [sp, #112] // 8-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[1], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #64] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #48] // 32-byte Folded Reload
; CHECK-GI-NEXT: ldp d13, d12, [sp, #120] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[2], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #48] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[2], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[3], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[4], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #16] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #16] // 32-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[4], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[5], v2.h[0]
; CHECK-GI-NEXT: ldr q2, [sp] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[6], v2.h[0]
@@ -1994,14 +1990,12 @@ define <7 x half> @exp2_v7f16(<7 x half> %a) {
; CHECK-GI-NEXT: ldr x30, [sp, #144] // 8-byte Folded Reload
; CHECK-GI-NEXT: ldp d11, d10, [sp, #112] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[1], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #48] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #32] // 32-byte Folded Reload
; CHECK-GI-NEXT: ldp d13, d12, [sp, #96] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[2], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[2], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[3], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[4], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp] // 32-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[4], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[5], v2.h[0]
; CHECK-GI-NEXT: mov v1.h[6], v0.h[0]
; CHECK-GI-NEXT: mov v1.h[7], v0.h[0]
@@ -2244,14 +2238,12 @@ define <8 x half> @exp2_v8f16(<8 x half> %a) {
; CHECK-GI-NEXT: ldp d11, d10, [sp, #136] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldr d14, [sp, #112] // 8-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[1], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #64] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #48] // 32-byte Folded Reload
; CHECK-GI-NEXT: ldp d13, d12, [sp, #120] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[2], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #48] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[2], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[3], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[4], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #16] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #16] // 32-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[4], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[5], v2.h[0]
; CHECK-GI-NEXT: ldr q2, [sp] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[6], v2.h[0]
@@ -3275,14 +3267,12 @@ define <7 x half> @log_v7f16(<7 x half> %a) {
; CHECK-GI-NEXT: ldr x30, [sp, #144] // 8-byte Folded Reload
; CHECK-GI-NEXT: ldp d11, d10, [sp, #112] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[1], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #48] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #32] // 32-byte Folded Reload
; CHECK-GI-NEXT: ldp d13, d12, [sp, #96] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[2], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[2], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[3], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[4], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp] // 32-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[4], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[5], v2.h[0]
; CHECK-GI-NEXT: mov v1.h[6], v0.h[0]
; CHECK-GI-NEXT: mov v1.h[7], v0.h[0]
@@ -3525,14 +3515,12 @@ define <8 x half> @log_v8f16(<8 x half> %a) {
; CHECK-GI-NEXT: ldp d11, d10, [sp, #136] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldr d14, [sp, #112] // 8-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[1], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #64] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #48] // 32-byte Folded Reload
; CHECK-GI-NEXT: ldp d13, d12, [sp, #120] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[2], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #48] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[2], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[3], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[4], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #16] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #16] // 32-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[4], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[5], v2.h[0]
; CHECK-GI-NEXT: ldr q2, [sp] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[6], v2.h[0]
@@ -4556,14 +4544,12 @@ define <7 x half> @log2_v7f16(<7 x half> %a) {
; CHECK-GI-NEXT: ldr x30, [sp, #144] // 8-byte Folded Reload
; CHECK-GI-NEXT: ldp d11, d10, [sp, #112] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[1], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #48] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #32] // 32-byte Folded Reload
; CHECK-GI-NEXT: ldp d13, d12, [sp, #96] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[2], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[2], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[3], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[4], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp] // 32-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[4], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[5], v2.h[0]
; CHECK-GI-NEXT: mov v1.h[6], v0.h[0]
; CHECK-GI-NEXT: mov v1.h[7], v0.h[0]
@@ -4806,14 +4792,12 @@ define <8 x half> @log2_v8f16(<8 x half> %a) {
; CHECK-GI-NEXT: ldp d11, d10, [sp, #136] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldr d14, [sp, #112] // 8-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[1], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #64] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #48] // 32-byte Folded Reload
; CHECK-GI-NEXT: ldp d13, d12, [sp, #120] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[2], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #48] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[2], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[3], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[4], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #16] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #16] // 32-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[4], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[5], v2.h[0]
; CHECK-GI-NEXT: ldr q2, [sp] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[6], v2.h[0]
@@ -5837,14 +5821,12 @@ define <7 x half> @log10_v7f16(<7 x half> %a) {
; CHECK-GI-NEXT: ldr x30, [sp, #144] // 8-byte Folded Reload
; CHECK-GI-NEXT: ldp d11, d10, [sp, #112] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[1], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #48] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #32] // 32-byte Folded Reload
; CHECK-GI-NEXT: ldp d13, d12, [sp, #96] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[2], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[2], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[3], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[4], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp] // 32-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[4], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[5], v2.h[0]
; CHECK-GI-NEXT: mov v1.h[6], v0.h[0]
; CHECK-GI-NEXT: mov v1.h[7], v0.h[0]
@@ -6087,14 +6069,12 @@ define <8 x half> @log10_v8f16(<8 x half> %a) {
; CHECK-GI-NEXT: ldp d11, d10, [sp, #136] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldr d14, [sp, #112] // 8-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[1], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #64] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #48] // 32-byte Folded Reload
; CHECK-GI-NEXT: ldp d13, d12, [sp, #120] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[2], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #48] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[2], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[3], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[4], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #16] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #16] // 32-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[4], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[5], v2.h[0]
; CHECK-GI-NEXT: ldr q2, [sp] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[6], v2.h[0]
diff --git a/llvm/test/CodeGen/AArch64/fpow.ll b/llvm/test/CodeGen/AArch64/fpow.ll
index 1b3ae7a227924a3..7681ab510b3099f 100644
--- a/llvm/test/CodeGen/AArch64/fpow.ll
+++ b/llvm/test/CodeGen/AArch64/fpow.ll
@@ -267,14 +267,13 @@ define <4 x double> @pow_v4f64(<4 x double> %a, <4 x double> %b) {
; CHECK-GI-NEXT: fmov d1, d11
; CHECK-GI-NEXT: fmov d0, d9
; CHECK-GI-NEXT: bl pow
-; CHECK-GI-NEXT: ldr q2, [sp, #48] // 16-byte Folded Reload
-; CHECK-GI-NEXT: ldr q1, [sp, #16] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q3, q1, [sp, #16] // 32-byte Folded Reload
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT: ldr x30, [sp, #96] // 8-byte Folded Reload
+; CHECK-GI-NEXT: ldr q2, [sp, #48] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldp d9, d8, [sp, #80] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v2.d[1], v1.d[0]
-; CHECK-GI-NEXT: ldr q1, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldr x30, [sp, #96] // 8-byte Folded Reload
; CHECK-GI-NEXT: ldp d11, d10, [sp, #64] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mov v2.d[1], v3.d[0]
; CHECK-GI-NEXT: mov v1.d[1], v0.d[0]
; CHECK-GI-NEXT: mov v0.16b, v2.16b
; CHECK-GI-NEXT: add sp, sp, #112
@@ -715,14 +714,14 @@ define <7 x half> @pow_v7f16(<7 x half> %a, <7 x half> %b) {
; CHECK-SD-NEXT: fcvt s1, h1
; CHECK-SD-NEXT: bl powf
; CHECK-SD-NEXT: fcvt h0, s0
-; CHECK-SD-NEXT: ldr q1, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT: mov v0.h[1], v1.h[0]
+; CHECK-SD-NEXT: ldp q1, q2, [sp, #16] // 32-byte Folded Reload
+; CHECK-SD-NEXT: mov h1, v1.h[2]
+; CHECK-SD-NEXT: mov v0.h[1], v2.h[0]
+; CHECK-SD-NEXT: fcvt s1, h1
; CHECK-SD-NEXT: str q0, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NEXT: ldp q0, q1, [sp] // 32-byte Folded Reload
+; CHECK-SD-NEXT: ldr q0, [sp] // 16-byte Folded Reload
; CHECK-SD-NEXT: mov h0, v0.h[2]
-; CHECK-SD-NEXT: mov h1, v1.h[2]
; CHECK-SD-NEXT: fcvt s0, h0
-; CHECK-SD-NEXT: fcvt s1, h1
; CHECK-SD-NEXT: bl powf
; CHECK-SD-NEXT: fcvt h0, s0
; CHECK-SD-NEXT: ldr q1, [sp, #32] // 16-byte Folded Reload
@@ -858,14 +857,13 @@ define <7 x half> @pow_v7f16(<7 x half> %a, <7 x half> %b) {
; CHECK-GI-NEXT: fcvt s1, h0
; CHECK-GI-NEXT: fmov s0, s2
; CHECK-GI-NEXT: bl powf
-; CHECK-GI-NEXT: ldr q1, [sp, #64] // 16-byte Folded Reload
-; CHECK-GI-NEXT: ldr q2, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #16] // 32-byte Folded Reload
; CHECK-GI-NEXT: fcvt h0, s0
+; CHECK-GI-NEXT: ldr q1, [sp, #64] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldp d9, d8, [sp, #144] // 16-byte Folded Reload
-; CHECK-GI-NEXT: ldr x30, [sp, #160] // 8-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[1], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #16] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldp d11, d10, [sp, #128] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldr x30, [sp, #160] // 8-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[1], v3.h[0]
; CHECK-GI-NEXT: ldp d13, d12, [sp, #112] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldp d15, d14, [sp, #96] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[2], v2.h[0]
@@ -1020,14 +1018,14 @@ define <8 x half> @pow_v8f16(<8 x half> %a, <8 x half> %b) {
; CHECK-SD-NEXT: fcvt s1, h1
; CHECK-SD-NEXT: bl powf
; CHECK-SD-NEXT: fcvt h0, s0
-; CHECK-SD-NEXT: ldr q1, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT: mov v0.h[1], v1.h[0]
+; CHECK-SD-NEXT: ldp q1, q2, [sp, #16] // 32-byte Folded Reload
+; CHECK-SD-NEXT: mov h1, v1.h[2]
+; CHECK-SD-NEXT: mov v0.h[1], v2.h[0]
+; CHECK-SD-NEXT: fcvt s1, h1
; CHECK-SD-NEXT: str q0, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NEXT: ldp q0, q1, [sp] // 32-byte Folded Reload
+; CHECK-SD-NEXT: ldr q0, [sp] // 16-byte Folded Reload
; CHECK-SD-NEXT: mov h0, v0.h[2]
-; CHECK-SD-NEXT: mov h1, v1.h[2]
; CHECK-SD-NEXT: fcvt s0, h0
-; CHECK-SD-NEXT: fcvt s1, h1
; CHECK-SD-NEXT: bl powf
; CHECK-SD-NEXT: fcvt h0, s0
; CHECK-SD-NEXT: ldr q1, [sp, #32] // 16-byte Folded Reload
@@ -1175,22 +1173,20 @@ define <8 x half> @pow_v8f16(<8 x half> %a, <8 x half> %b) {
; CHECK-GI-NEXT: fcvt s1, h0
; CHECK-GI-NEXT: fmov s0, s2
; CHECK-GI-NEXT: bl powf
-; CHECK-GI-NEXT: ldr q1, [sp, #80] // 16-byte Folded Reload
-; CHECK-GI-NEXT: ldr q2, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #16] // 32-byte Folded Reload
; CHECK-GI-NEXT: fcvt h0, s0
+; CHECK-GI-NEXT: ldr q1, [sp, #80] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldp d9, d8, [sp, #160] // 16-byte Folded Reload
-; CHECK-GI-NEXT: ldr x30, [sp, #176] // 8-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[1], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #16] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldp d11, d10, [sp, #144] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldr x30, [sp, #176] // 8-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[1], v3.h[0]
; CHECK-GI-NEXT: ldp d13, d12, [sp, #128] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldp d15, d14, [sp, #112] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[2], v2.h[0]
; CHECK-GI-NEXT: ldr q2, [sp] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[3], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #48] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[4], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #64] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q3, q2, [sp, #48] // 32-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[4], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[5], v2.h[0]
; CHECK-GI-NEXT: ldr q2, [sp, #96] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[6], v2.h[0]
@@ -1225,14 +1221,13 @@ define <16 x half> @pow_v16f16(<16 x half> %a, <16 x half> %b) {
; CHECK-SD-NEXT: fcvt s0, h0
; CHECK-SD-NEXT: bl powf
; CHECK-SD-NEXT: fcvt h0, s0
-; CHECK-SD-NEXT: ldr q1, [sp, #64] // 16-byte Folded Reload
-; CHECK-SD-NEXT: mov v0.h[1], v1.h[0]
-; CHECK-SD-NEXT: ldr q1, [sp, #48] // 16-byte Folded Reload
+; CHECK-SD-NEXT: ldp q1, q2, [sp, #48] // 32-byte Folded Reload
; CHECK-SD-NEXT: mov h1, v1.h[2]
+; CHECK-SD-NEXT: mov v0.h[1], v2.h[0]
+; CHECK-SD-NEXT: fcvt s1, h1
; CHECK-SD-NEXT: str q0, [sp, #64] // 16-byte Folded Spill
; CHECK-SD-NEXT: ldr q0, [sp] // 16-byte Folded Reload
; CHECK-SD-NEXT: mov h0, v0.h[2]
-; CHECK-SD-NEXT: fcvt s1, h1
; CHECK-SD-NEXT: fcvt s0, h0
; CHECK-SD-NEXT: bl powf
; CHECK-SD-NEXT: fcvt h0, s0
@@ -1307,14 +1302,14 @@ define <16 x half> @pow_v16f16(<16 x half> %a, <16 x half> %b) {
; CHECK-SD-NEXT: fcvt s1, h1
; CHECK-SD-NEXT: bl powf
; CHECK-SD-NEXT: fcvt h0, s0
-; CHECK-SD-NEXT: ldr q1, [sp, #48] // 16-byte Folded Reload
-; CHECK-SD-NEXT: mov v0.h[1], v1.h[0]
+; CHECK-SD-NEXT: ldp q1, q2, [sp, #32] // 32-byte Folded Reload
+; CHECK-SD-NEXT: mov h1, v1.h[2]
+; CHECK-SD-NEXT: mov v0.h[1], v2.h[0]
+; CHECK-SD-NEXT: fcvt s1, h1
; CHECK-SD-NEXT: str q0, [sp, #48] // 16-byte Folded Spill
-; CHECK-SD-NEXT: ldp q0, q1, [sp, #16] // 32-byte Folded Reload
+; CHECK-SD-NEXT: ldr q0, [sp, #16] // 16-byte Folded Reload
; CHECK-SD-NEXT: mov h0, v0.h[2]
-; CHECK-SD-NEXT: mov h1, v1.h[2]
; CHECK-SD-NEXT: fcvt s0, h0
-; CHECK-SD-NEXT: fcvt s1, h1
; CHECK-SD-NEXT: bl powf
; CHECK-SD-NEXT: fcvt h0, s0
; CHECK-SD-NEXT: ldr q1, [sp, #48] // 16-byte Folded Reload
@@ -1578,9 +1573,8 @@ define <16 x half> @pow_v16f16(<16 x half> %a, <16 x half> %b) {
; CHECK-GI-NEXT: mov v3.h[3], v2.h[0]
; CHECK-GI-NEXT: ldr q2, [sp, #192] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[3], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #240] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v3.h[4], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #256] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q4, q2, [sp, #240] // 32-byte Folded Reload
+; CHECK-GI-NEXT: mov v3.h[4], v4.h[0]
; CHECK-GI-NEXT: mov v1.h[4], v2.h[0]
; CHECK-GI-NEXT: ldr q2, [sp, #176] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v3.h[5], v2.h[0]
diff --git a/llvm/test/CodeGen/AArch64/frem.ll b/llvm/test/CodeGen/AArch64/frem.ll
index e8077b94f2bf688..ed0f6c442ece170 100644
--- a/llvm/test/CodeGen/AArch64/frem.ll
+++ b/llvm/test/CodeGen/AArch64/frem.ll
@@ -269,14 +269,13 @@ define <4 x double> @frem_v4f64(<4 x double> %a, <4 x double> %b) {
; CHECK-GI-NEXT: fmov d1, d11
; CHECK-GI-NEXT: fmov d0, d9
; CHECK-GI-NEXT: bl fmod
-; CHECK-GI-NEXT: ldr q2, [sp, #48] // 16-byte Folded Reload
-; CHECK-GI-NEXT: ldr q1, [sp, #16] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q3, q1, [sp, #16] // 32-byte Folded Reload
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT: ldr x30, [sp, #96] // 8-byte Folded Reload
+; CHECK-GI-NEXT: ldr q2, [sp, #48] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldp d9, d8, [sp, #80] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v2.d[1], v1.d[0]
-; CHECK-GI-NEXT: ldr q1, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldr x30, [sp, #96] // 8-byte Folded Reload
; CHECK-GI-NEXT: ldp d11, d10, [sp, #64] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mov v2.d[1], v3.d[0]
; CHECK-GI-NEXT: mov v1.d[1], v0.d[0]
; CHECK-GI-NEXT: mov v0.16b, v2.16b
; CHECK-GI-NEXT: add sp, sp, #112
@@ -717,14 +716,14 @@ define <7 x half> @frem_v7f16(<7 x half> %a, <7 x half> %b) {
; CHECK-SD-NEXT: fcvt s1, h1
; CHECK-SD-NEXT: bl fmodf
; CHECK-SD-NEXT: fcvt h0, s0
-; CHECK-SD-NEXT: ldr q1, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT: mov v0.h[1], v1.h[0]
+; CHECK-SD-NEXT: ldp q1, q2, [sp, #16] // 32-byte Folded Reload
+; CHECK-SD-NEXT: mov h1, v1.h[2]
+; CHECK-SD-NEXT: mov v0.h[1], v2.h[0]
+; CHECK-SD-NEXT: fcvt s1, h1
; CHECK-SD-NEXT: str q0, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NEXT: ldp q0, q1, [sp] // 32-byte Folded Reload
+; CHECK-SD-NEXT: ldr q0, [sp] // 16-byte Folded Reload
; CHECK-SD-NEXT: mov h0, v0.h[2]
-; CHECK-SD-NEXT: mov h1, v1.h[2]
; CHECK-SD-NEXT: fcvt s0, h0
-; CHECK-SD-NEXT: fcvt s1, h1
; CHECK-SD-NEXT: bl fmodf
; CHECK-SD-NEXT: fcvt h0, s0
; CHECK-SD-NEXT: ldr q1, [sp, #32] // 16-byte Folded Reload
@@ -860,14 +859,13 @@ define <7 x half> @frem_v7f16(<7 x half> %a, <7 x half> %b) {
; CHECK-GI-NEXT: fcvt s1, h0
; CHECK-GI-NEXT: fmov s0, s2
; CHECK-GI-NEXT: bl fmodf
-; CHECK-GI-NEXT: ldr q1, [sp, #64] // 16-byte Folded Reload
-; CHECK-GI-NEXT: ldr q2, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #16] // 32-byte Folded Reload
; CHECK-GI-NEXT: fcvt h0, s0
+; CHECK-GI-NEXT: ldr q1, [sp, #64] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldp d9, d8, [sp, #144] // 16-byte Folded Reload
-; CHECK-GI-NEXT: ldr x30, [sp, #160] // 8-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[1], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #16] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldp d11, d10, [sp, #128] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldr x30, [sp, #160] // 8-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[1], v3.h[0]
; CHECK-GI-NEXT: ldp d13, d12, [sp, #112] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldp d15, d14, [sp, #96] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[2], v2.h[0]
@@ -1022,14 +1020,14 @@ define <8 x half> @frem_v8f16(<8 x half> %a, <8 x half> %b) {
; CHECK-SD-NEXT: fcvt s1, h1
; CHECK-SD-NEXT: bl fmodf
; CHECK-SD-NEXT: fcvt h0, s0
-; CHECK-SD-NEXT: ldr q1, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT: mov v0.h[1], v1.h[0]
+; CHECK-SD-NEXT: ldp q1, q2, [sp, #16] // 32-byte Folded Reload
+; CHECK-SD-NEXT: mov h1, v1.h[2]
+; CHECK-SD-NEXT: mov v0.h[1], v2.h[0]
+; CHECK-SD-NEXT: fcvt s1, h1
; CHECK-SD-NEXT: str q0, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NEXT: ldp q0, q1, [sp] // 32-byte Folded Reload
+; CHECK-SD-NEXT: ldr q0, [sp] // 16-byte Folded Reload
; CHECK-SD-NEXT: mov h0, v0.h[2]
-; CHECK-SD-NEXT: mov h1, v1.h[2]
; CHECK-SD-NEXT: fcvt s0, h0
-; CHECK-SD-NEXT: fcvt s1, h1
; CHECK-SD-NEXT: bl fmodf
; CHECK-SD-NEXT: fcvt h0, s0
; CHECK-SD-NEXT: ldr q1, [sp, #32] // 16-byte Folded Reload
@@ -1177,22 +1175,20 @@ define <8 x half> @frem_v8f16(<8 x half> %a, <8 x half> %b) {
; CHECK-GI-NEXT: fcvt s1, h0
; CHECK-GI-NEXT: fmov s0, s2
; CHECK-GI-NEXT: bl fmodf
-; CHECK-GI-NEXT: ldr q1, [sp, #80] // 16-byte Folded Reload
-; CHECK-GI-NEXT: ldr q2, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #16] // 32-byte Folded Reload
; CHECK-GI-NEXT: fcvt h0, s0
+; CHECK-GI-NEXT: ldr q1, [sp, #80] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldp d9, d8, [sp, #160] // 16-byte Folded Reload
-; CHECK-GI-NEXT: ldr x30, [sp, #176] // 8-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[1], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #16] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldp d11, d10, [sp, #144] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldr x30, [sp, #176] // 8-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[1], v3.h[0]
; CHECK-GI-NEXT: ldp d13, d12, [sp, #128] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldp d15, d14, [sp, #112] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[2], v2.h[0]
; CHECK-GI-NEXT: ldr q2, [sp] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[3], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #48] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[4], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #64] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q3, q2, [sp, #48] // 32-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[4], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[5], v2.h[0]
; CHECK-GI-NEXT: ldr q2, [sp, #96] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[6], v2.h[0]
@@ -1227,14 +1223,13 @@ define <16 x half> @frem_v16f16(<16 x half> %a, <16 x half> %b) {
; CHECK-SD-NEXT: fcvt s0, h0
; CHECK-SD-NEXT: bl fmodf
; CHECK-SD-NEXT: fcvt h0, s0
-; CHECK-SD-NEXT: ldr q1, [sp, #64] // 16-byte Folded Reload
-; CHECK-SD-NEXT: mov v0.h[1], v1.h[0]
-; CHECK-SD-NEXT: ldr q1, [sp, #48] // 16-byte Folded Reload
+; CHECK-SD-NEXT: ldp q1, q2, [sp, #48] // 32-byte Folded Reload
; CHECK-SD-NEXT: mov h1, v1.h[2]
+; CHECK-SD-NEXT: mov v0.h[1], v2.h[0]
+; CHECK-SD-NEXT: fcvt s1, h1
; CHECK-SD-NEXT: str q0, [sp, #64] // 16-byte Folded Spill
; CHECK-SD-NEXT: ldr q0, [sp] // 16-byte Folded Reload
; CHECK-SD-NEXT: mov h0, v0.h[2]
-; CHECK-SD-NEXT: fcvt s1, h1
; CHECK-SD-NEXT: fcvt s0, h0
; CHECK-SD-NEXT: bl fmodf
; CHECK-SD-NEXT: fcvt h0, s0
@@ -1309,14 +1304,14 @@ define <16 x half> @frem_v16f16(<16 x half> %a, <16 x half> %b) {
; CHECK-SD-NEXT: fcvt s1, h1
; CHECK-SD-NEXT: bl fmodf
; CHECK-SD-NEXT: fcvt h0, s0
-; CHECK-SD-NEXT: ldr q1, [sp, #48] // 16-byte Folded Reload
-; CHECK-SD-NEXT: mov v0.h[1], v1.h[0]
+; CHECK-SD-NEXT: ldp q1, q2, [sp, #32] // 32-byte Folded Reload
+; CHECK-SD-NEXT: mov h1, v1.h[2]
+; CHECK-SD-NEXT: mov v0.h[1], v2.h[0]
+; CHECK-SD-NEXT: fcvt s1, h1
; CHECK-SD-NEXT: str q0, [sp, #48] // 16-byte Folded Spill
-; CHECK-SD-NEXT: ldp q0, q1, [sp, #16] // 32-byte Folded Reload
+; CHECK-SD-NEXT: ldr q0, [sp, #16] // 16-byte Folded Reload
; CHECK-SD-NEXT: mov h0, v0.h[2]
-; CHECK-SD-NEXT: mov h1, v1.h[2]
; CHECK-SD-NEXT: fcvt s0, h0
-; CHECK-SD-NEXT: fcvt s1, h1
; CHECK-SD-NEXT: bl fmodf
; CHECK-SD-NEXT: fcvt h0, s0
; CHECK-SD-NEXT: ldr q1, [sp, #48] // 16-byte Folded Reload
@@ -1580,9 +1575,8 @@ define <16 x half> @frem_v16f16(<16 x half> %a, <16 x half> %b) {
; CHECK-GI-NEXT: mov v3.h[3], v2.h[0]
; CHECK-GI-NEXT: ldr q2, [sp, #192] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[3], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #240] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v3.h[4], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #256] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q4, q2, [sp, #240] // 32-byte Folded Reload
+; CHECK-GI-NEXT: mov v3.h[4], v4.h[0]
; CHECK-GI-NEXT: mov v1.h[4], v2.h[0]
; CHECK-GI-NEXT: ldr q2, [sp, #176] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v3.h[5], v2.h[0]
diff --git a/llvm/test/CodeGen/AArch64/fsincos.ll b/llvm/test/CodeGen/AArch64/fsincos.ll
index ec60fedeb47296b..9784b9ea9b65f58 100644
--- a/llvm/test/CodeGen/AArch64/fsincos.ll
+++ b/llvm/test/CodeGen/AArch64/fsincos.ll
@@ -713,14 +713,12 @@ define <7 x half> @sin_v7f16(<7 x half> %a) {
; CHECK-GI-NEXT: ldr x30, [sp, #144] // 8-byte Folded Reload
; CHECK-GI-NEXT: ldp d11, d10, [sp, #112] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[1], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #48] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #32] // 32-byte Folded Reload
; CHECK-GI-NEXT: ldp d13, d12, [sp, #96] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[2], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[2], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[3], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[4], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp] // 32-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[4], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[5], v2.h[0]
; CHECK-GI-NEXT: mov v1.h[6], v0.h[0]
; CHECK-GI-NEXT: mov v1.h[7], v0.h[0]
@@ -963,14 +961,12 @@ define <8 x half> @sin_v8f16(<8 x half> %a) {
; CHECK-GI-NEXT: ldp d11, d10, [sp, #136] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldr d14, [sp, #112] // 8-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[1], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #64] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #48] // 32-byte Folded Reload
; CHECK-GI-NEXT: ldp d13, d12, [sp, #120] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[2], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #48] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[2], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[3], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[4], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #16] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #16] // 32-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[4], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[5], v2.h[0]
; CHECK-GI-NEXT: ldr q2, [sp] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[6], v2.h[0]
@@ -1994,14 +1990,12 @@ define <7 x half> @cos_v7f16(<7 x half> %a) {
; CHECK-GI-NEXT: ldr x30, [sp, #144] // 8-byte Folded Reload
; CHECK-GI-NEXT: ldp d11, d10, [sp, #112] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[1], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #48] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #32] // 32-byte Folded Reload
; CHECK-GI-NEXT: ldp d13, d12, [sp, #96] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[2], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[2], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[3], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[4], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp] // 32-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[4], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[5], v2.h[0]
; CHECK-GI-NEXT: mov v1.h[6], v0.h[0]
; CHECK-GI-NEXT: mov v1.h[7], v0.h[0]
@@ -2244,14 +2238,12 @@ define <8 x half> @cos_v8f16(<8 x half> %a) {
; CHECK-GI-NEXT: ldp d11, d10, [sp, #136] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldr d14, [sp, #112] // 8-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[1], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #64] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #48] // 32-byte Folded Reload
; CHECK-GI-NEXT: ldp d13, d12, [sp, #120] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[2], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #48] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[2], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[3], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[4], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #16] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #16] // 32-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[4], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[5], v2.h[0]
; CHECK-GI-NEXT: ldr q2, [sp] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[6], v2.h[0]
diff --git a/llvm/test/CodeGen/AArch64/ldrpre-ldr-merge.mir b/llvm/test/CodeGen/AArch64/ldrpre-ldr-merge.mir
index b76570da781f5a6..8e29255189bf530 100644
--- a/llvm/test/CodeGen/AArch64/ldrpre-ldr-merge.mir
+++ b/llvm/test/CodeGen/AArch64/ldrpre-ldr-merge.mir
@@ -348,8 +348,6 @@ body: |
bb.0:
liveins: $s0, $s1, $x1
- ; The offset of the second load is not equal to the
- ; size of the destination register, and hence can’t be merged.
; CHECK-LABEL: name: 12-ldrspre-ldrsui-no-merge
; CHECK: liveins: $s0, $s1, $x1
@@ -445,8 +443,7 @@ body: |
; CHECK-LABEL: name: 15-ldrqpre-ldrqui-same-dst-reg-no-merge
; CHECK: liveins: $q0, $q1, $x1
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: early-clobber renamable $x1, dead renamable $q0 = LDRQpre renamable $x1, 32, implicit $w1 :: (load (s128))
- ; CHECK-NEXT: renamable $q0 = LDRQui renamable $x1, 1 :: (load (s128))
+ ; CHECK-NEXT: early-clobber $x1, dead $q2, renamable $q0 = LDPQpre renamable $x1, 2 :: (load (s128))
; CHECK-NEXT: STRQui renamable $q0, renamable $x1, 0 :: (store (s128))
; CHECK-NEXT: RET undef $lr
early-clobber renamable $x1, renamable $q0 = LDRQpre killed renamable $x1, 32 :: (load (s128))
diff --git a/llvm/test/CodeGen/AArch64/neon-dotreduce.ll b/llvm/test/CodeGen/AArch64/neon-dotreduce.ll
index 74048b8bee3329f..706aa4ad1b46653 100644
--- a/llvm/test/CodeGen/AArch64/neon-dotreduce.ll
+++ b/llvm/test/CodeGen/AArch64/neon-dotreduce.ll
@@ -1473,11 +1473,9 @@ define i32 @test_udot_v32i8(ptr nocapture readonly %a, ptr nocapture readonly %b
; CHECK-LABEL: test_udot_v32i8:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: ldr q1, [x0, #16]
-; CHECK-NEXT: ldr q2, [x1, #16]
-; CHECK-NEXT: udot v0.4s, v2.16b, v1.16b
-; CHECK-NEXT: ldr q1, [x0]
-; CHECK-NEXT: ldr q2, [x1]
+; CHECK-NEXT: ldp q1, q3, [x0]
+; CHECK-NEXT: ldp q2, q4, [x1]
+; CHECK-NEXT: udot v0.4s, v4.16b, v3.16b
; CHECK-NEXT: udot v0.4s, v2.16b, v1.16b
; CHECK-NEXT: addv s0, v0.4s
; CHECK-NEXT: fmov w8, s0
@@ -1499,9 +1497,8 @@ define i32 @test_udot_v32i8_nomla(ptr nocapture readonly %a1) {
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: movi v0.16b, #1
; CHECK-NEXT: movi v1.2d, #0000000000000000
-; CHECK-NEXT: ldr q2, [x0, #16]
-; CHECK-NEXT: udot v1.4s, v2.16b, v0.16b
-; CHECK-NEXT: ldr q2, [x0]
+; CHECK-NEXT: ldp q2, q3, [x0]
+; CHECK-NEXT: udot v1.4s, v3.16b, v0.16b
; CHECK-NEXT: udot v1.4s, v2.16b, v0.16b
; CHECK-NEXT: addv s0, v1.4s
; CHECK-NEXT: fmov w0, s0
@@ -1516,11 +1513,9 @@ define i32 @test_sdot_v32i8(ptr nocapture readonly %a, ptr nocapture readonly %b
; CHECK-LABEL: test_sdot_v32i8:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: ldr q1, [x0, #16]
-; CHECK-NEXT: ldr q2, [x1, #16]
-; CHECK-NEXT: sdot v0.4s, v2.16b, v1.16b
-; CHECK-NEXT: ldr q1, [x0]
-; CHECK-NEXT: ldr q2, [x1]
+; CHECK-NEXT: ldp q1, q3, [x0]
+; CHECK-NEXT: ldp q2, q4, [x1]
+; CHECK-NEXT: sdot v0.4s, v4.16b, v3.16b
; CHECK-NEXT: sdot v0.4s, v2.16b, v1.16b
; CHECK-NEXT: addv s0, v0.4s
; CHECK-NEXT: fmov w8, s0
@@ -2190,11 +2185,9 @@ define i32 @test_udot_v48i8(ptr nocapture readonly %a, ptr nocapture readonly %b
; CHECK-NEXT: ldr q1, [x0, #32]
; CHECK-NEXT: ldr q2, [x1, #32]
; CHECK-NEXT: udot v0.4s, v2.16b, v1.16b
-; CHECK-NEXT: ldr q1, [x0]
-; CHECK-NEXT: ldr q2, [x1]
-; CHECK-NEXT: udot v0.4s, v2.16b, v1.16b
-; CHECK-NEXT: ldr q1, [x0, #16]
-; CHECK-NEXT: ldr q2, [x1, #16]
+; CHECK-NEXT: ldp q3, q1, [x0]
+; CHECK-NEXT: ldp q4, q2, [x1]
+; CHECK-NEXT: udot v0.4s, v4.16b, v3.16b
; CHECK-NEXT: udot v0.4s, v2.16b, v1.16b
; CHECK-NEXT: addv s0, v0.4s
; CHECK-NEXT: fmov w8, s0
@@ -2218,9 +2211,8 @@ define i32 @test_udot_v48i8_nomla(ptr nocapture readonly %a1) {
; CHECK-NEXT: movi v1.2d, #0000000000000000
; CHECK-NEXT: ldr q2, [x0, #32]
; CHECK-NEXT: udot v1.4s, v2.16b, v0.16b
-; CHECK-NEXT: ldr q2, [x0]
-; CHECK-NEXT: udot v1.4s, v2.16b, v0.16b
-; CHECK-NEXT: ldr q2, [x0, #16]
+; CHECK-NEXT: ldp q3, q2, [x0]
+; CHECK-NEXT: udot v1.4s, v3.16b, v0.16b
; CHECK-NEXT: udot v1.4s, v2.16b, v0.16b
; CHECK-NEXT: addv s0, v1.4s
; CHECK-NEXT: fmov w0, s0
@@ -2238,11 +2230,9 @@ define i32 @test_sdot_v48i8(ptr nocapture readonly %a, ptr nocapture readonly %b
; CHECK-NEXT: ldr q1, [x0, #32]
; CHECK-NEXT: ldr q2, [x1, #32]
; CHECK-NEXT: sdot v0.4s, v2.16b, v1.16b
-; CHECK-NEXT: ldr q1, [x0]
-; CHECK-NEXT: ldr q2, [x1]
-; CHECK-NEXT: sdot v0.4s, v2.16b, v1.16b
-; CHECK-NEXT: ldr q1, [x0, #16]
-; CHECK-NEXT: ldr q2, [x1, #16]
+; CHECK-NEXT: ldp q3, q1, [x0]
+; CHECK-NEXT: ldp q4, q2, [x1]
+; CHECK-NEXT: sdot v0.4s, v4.16b, v3.16b
; CHECK-NEXT: sdot v0.4s, v2.16b, v1.16b
; CHECK-NEXT: addv s0, v0.4s
; CHECK-NEXT: fmov w8, s0
diff --git a/llvm/test/CodeGen/AArch64/stp-opt-with-renaming.mir b/llvm/test/CodeGen/AArch64/stp-opt-with-renaming.mir
index f1ca71680062b4d..15313602051378b 100644
--- a/llvm/test/CodeGen/AArch64/stp-opt-with-renaming.mir
+++ b/llvm/test/CodeGen/AArch64/stp-opt-with-renaming.mir
@@ -327,8 +327,7 @@ body: |
# CHECK: renamable $q7 = FADDv2f64 renamable $q7, renamable $q7, implicit $fpcr
# CHECK-NEXT: STRQui killed renamable $q7, renamable $x0, 100 :: (store (s128), align 4)
-# CHECK-NEXT: $q7 = LDRQui $x0, 0 :: (load (s128))
-# CHECK-NEXT: renamable $q9 = LDRQui $x0, 1 :: (load (s128))
+# CHECK-NEXT: $q7, renamable $q9 = LDPQi $x0, 0 :: (load (s128))
# CHECK-NEXT: STPQi killed renamable $q9, killed $q7, renamable $x0, 10 :: (store (s128), align 4)
# CHECK-NEXT: RET undef $lr
@@ -546,3 +545,134 @@ body: |
RET undef $lr
...
+
+# During ISel, the order of load/store pairs can be optimized and changed
+# so that only a single register is used. Due to this register reuse, LDP/STPs
+# are not generated. These tests check that LDP/STPs will be generated after
+# register renaming is attempted.
+...
+---
+#
+# CHECK-LABEL: name: ldst32
+# CHECK: renamable $q0, $q1 = LDPQi $sp, 0 :: (load (s128)), (load (s128), align 32)
+# CHECK-NEXT: STPQi killed renamable $q0, killed $q1, $sp, 2 :: (store (s128), align 32)
+# CHECK-NEXT: RET undef $lr
+#
+name: ldst32
+alignment: 4
+tracksRegLiveness: true
+frameInfo:
+ maxAlignment: 1
+ maxCallFrameSize: 0
+machineFunctionInfo:
+ hasRedZone: false
+body: |
+ bb.0.entry:
+ renamable $q0 = LDRQui $sp, 1 :: (load 16)
+ STRQui killed renamable $q0, $sp, 3 :: (store 16, basealign 32)
+ renamable $q0 = LDRQui $sp, 0 :: (load 16, align 32)
+ STRQui killed renamable $q0, $sp, 2 :: (store 16, align 32)
+ RET undef $lr
+
+...
+---
+#
+# CHECK-LABEL: name: ldst64
+# CHECK: renamable $q0, $q1 = LDPQi $sp, 0 :: (load (s128)), (load (s128), align 64)
+# CHECK-NEXT: STPQi killed renamable $q0, killed $q1, $sp, 2 :: (store (s128), align 64)
+# CHECK-NEXT: RET undef $lr
+#
+name: ldst64
+alignment: 4
+tracksRegLiveness: true
+frameInfo:
+ maxAlignment: 1
+ maxCallFrameSize: 0
+machineFunctionInfo:
+ hasRedZone: false
+body: |
+ bb.0.entry:
+ renamable $q0 = LDRQui $sp, 1 :: (load 16)
+ STRQui killed renamable $q0, $sp, 3 :: (store 16, basealign 64)
+ renamable $q0 = LDRQui $sp, 0 :: (load 16, align 64)
+ STRQui killed renamable $q0, $sp, 2 :: (store 16, align 64)
+ RET undef $lr
+
+...
+---
+#
+# CHECK-LABEL: name: ldst128
+# CHECK: renamable $q0, $q1 = LDPQi $sp, 0 :: (load (s128)), (load (s128), align 128)
+# CHECK-NEXT: STPQi killed renamable $q0, killed $q1, $sp, 2 :: (store (s128), align 128)
+# CHECK-NEXT: RET undef $lr
+#
+name: ldst128
+alignment: 4
+tracksRegLiveness: true
+frameInfo:
+ maxAlignment: 1
+ maxCallFrameSize: 0
+machineFunctionInfo:
+ hasRedZone: false
+body: |
+ bb.0.entry:
+ renamable $q0 = LDRQui $sp, 1 :: (load 16)
+ STRQui killed renamable $q0, $sp, 3 :: (store 16, basealign 128)
+ renamable $q0 = LDRQui $sp, 0 :: (load 16, align 128)
+ STRQui killed renamable $q0, $sp, 2 :: (store 16, align 128)
+ RET undef $lr
+
+...
+---
+#
+# CHECK-LABEL: name: ldst-no-reg-available
+# CHECK: liveins: $q1, $q2, $q3, $q4, $q5, $q6, $q7, $q8, $q9, $q10
+# CHECK: renamable $q0 = LDRQui $sp, 1 :: (load (s128))
+# CHECK-NEXT: STRQui killed renamable $q0, $sp, 3 :: (store (s128), align 32)
+# CHECK-NEXT: renamable $q0 = LDRQui $sp, 0 :: (load (s128), align 32)
+# CHECK-NEXT: STRQui killed renamable $q0, $sp, 2 :: (store (s128), align 32)
+# CHECK-NEXT: RET undef $lr
+#
+name: ldst-no-reg-available
+alignment: 4
+tracksRegLiveness: true
+frameInfo:
+ maxAlignment: 1
+ maxCallFrameSize: 0
+machineFunctionInfo:
+ hasRedZone: false
+body: |
+ bb.0.entry:
+ liveins: $q1, $q2, $q3, $q4, $q5, $q6, $q7, $q8, $q9, $q10
+ renamable $q0 = LDRQui $sp, 1 :: (load 16)
+ STRQui killed renamable $q0, $sp, 3 :: (store 16, basealign 32)
+ renamable $q0 = LDRQui $sp, 0 :: (load 16, align 32)
+ STRQui killed renamable $q0, $sp, 2 :: (store 16, align 32)
+ RET undef $lr
+
+...
+---
+#
+# CHECK-LABEL: name: ldst-basereg-modified
+# CHECK: renamable $q0, $q1 = LDPQi $sp, 0 :: (load (s128)), (load (s128), align 32)
+# CHECK-NEXT: STRQui killed $q1, $sp, 3 :: (store (s128), align 32)
+# CHECK-NEXT: $sp = ADDXri $sp, 16, 0
+# CHECK-NEXT: STRQui killed renamable $q0, $sp, 2 :: (store (s128), align 32)
+# CHECK-NEXT: RET undef $lr
+#
+name: ldst-basereg-modified
+alignment: 4
+tracksRegLiveness: true
+frameInfo:
+ maxAlignment: 1
+ maxCallFrameSize: 0
+machineFunctionInfo:
+ hasRedZone: false
+body: |
+ bb.0.entry:
+ renamable $q0 = LDRQui $sp, 1 :: (load 16)
+ STRQui killed renamable $q0, $sp, 3 :: (store 16, basealign 32)
+ renamable $q0 = LDRQui $sp, 0 :: (load 16, align 32)
+ $sp = ADDXri $sp, 16, 0
+ STRQui killed renamable $q0, $sp, 2 :: (store 16, align 32)
+ RET undef $lr
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp128.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp128.ll
index da7e772461e28bc..ffd8590ad47920c 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp128.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp128.ll
@@ -39,8 +39,7 @@ define void @fcvt_v4f64_v4f128(ptr %a, ptr %b) vscale_range(2,0) #0 {
; CHECK-NEXT: bl __extenddftf2
; CHECK-NEXT: ldr q1, [sp] // 16-byte Folded Reload
; CHECK-NEXT: stp q1, q0, [x19]
-; CHECK-NEXT: ldr q1, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT: ldr q0, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT: ldp q1, q0, [sp, #16] // 32-byte Folded Reload
; CHECK-NEXT: stp q0, q1, [x19, #32]
; CHECK-NEXT: addvl sp, sp, #2
; CHECK-NEXT: add sp, sp, #48
@@ -61,19 +60,15 @@ define void @fcvt_v4f128_v4f64(ptr %a, ptr %b) vscale_range(2,0) #0 {
; CHECK-NEXT: stp x30, x19, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT: addvl sp, sp, #-2
; CHECK-NEXT: sub sp, sp, #128
-; CHECK-NEXT: ldr q1, [x0, #64]
-; CHECK-NEXT: ldr q0, [x0, #80]
+; CHECK-NEXT: ldp q1, q0, [x0, #64]
; CHECK-NEXT: mov x19, x1
; CHECK-NEXT: stp q0, q1, [sp, #96] // 32-byte Folded Spill
-; CHECK-NEXT: ldr q1, [x0, #96]
-; CHECK-NEXT: ldr q0, [x0, #112]
+; CHECK-NEXT: ldp q1, q0, [x0, #96]
; CHECK-NEXT: stp q0, q1, [sp, #64] // 32-byte Folded Spill
-; CHECK-NEXT: ldr q1, [x0]
-; CHECK-NEXT: ldr q0, [x0, #16]
+; CHECK-NEXT: ldp q1, q0, [x0]
; CHECK-NEXT: stp q0, q1, [sp, #32] // 32-byte Folded Spill
-; CHECK-NEXT: ldr q0, [x0, #32]
-; CHECK-NEXT: str q0, [sp, #16] // 16-byte Folded Spill
-; CHECK-NEXT: ldr q0, [x0, #48]
+; CHECK-NEXT: ldp q1, q0, [x0, #32]
+; CHECK-NEXT: str q1, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT: bl __trunctfdf2
; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
; CHECK-NEXT: str q0, [sp] // 16-byte Folded Spill
>From d95829d6dacc07b4d749215241e79c68f67f5ea0 Mon Sep 17 00:00:00 2001
From: Zhaoxuan Jiang <jiangzhaoxuan94 at gmail.com>
Date: Sat, 11 Nov 2023 16:17:16 +0800
Subject: [PATCH 2/2] fix not renaming for implicit MOPs
---
.../AArch64/AArch64LoadStoreOptimizer.cpp | 15 ++++++++----
.../CodeGen/AArch64/stp-opt-with-renaming.mir | 24 +++++++++++++++++++
2 files changed, 34 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp b/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
index 3827cef566f12ee..0655e65ef891ebf 100644
--- a/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
@@ -939,11 +939,16 @@ AArch64LoadStoreOpt::mergePairedInsns(MachineBasicBlock::iterator I,
assert((MOP.isImplicit() ||
(MOP.isRenamable() && !MOP.isEarlyClobber())) &&
"Need renamable operands");
- const TargetRegisterClass *RC =
- MI.getRegClassConstraint(OpIdx, TII, TRI);
- if (!RC)
- continue;
- MOP.setReg(GetMatchingSubReg(RC));
+ Register MatchingReg;
+ if (const TargetRegisterClass *RC =
+ MI.getRegClassConstraint(OpIdx, TII, TRI))
+ MatchingReg = GetMatchingSubReg(RC);
+ else
+ MatchingReg = GetMatchingSubReg(
+ TRI->getMinimalPhysRegClass(MOP.getReg()));
+ if (MatchingReg == AArch64::NoRegister)
+ report_fatal_error("Cannot find matching regs for renaming");
+ MOP.setReg(MatchingReg);
}
}
}
diff --git a/llvm/test/CodeGen/AArch64/stp-opt-with-renaming.mir b/llvm/test/CodeGen/AArch64/stp-opt-with-renaming.mir
index 15313602051378b..44f127f78f7c042 100644
--- a/llvm/test/CodeGen/AArch64/stp-opt-with-renaming.mir
+++ b/llvm/test/CodeGen/AArch64/stp-opt-with-renaming.mir
@@ -676,3 +676,27 @@ body: |
$sp = ADDXri $sp, 16, 0
STRQui killed renamable $q0, $sp, 2 :: (store 16, align 32)
RET undef $lr
+
+...
+---
+#
+# CHECK-LABEL: name: ldr-dest-reg-implicit-killed
+# CHECK: renamable $q0, $q1 = LDPQi $sp, 0 :: (load (s128)), (load (s128), align 32)
+# CHECK-NEXT: STPSi $s1, renamable $s0, $sp, 9 :: (store (s32))
+# CHECK-NEXT: RET undef $lr
+#
+name: ldr-dest-reg-implicit-killed
+alignment: 4
+tracksRegLiveness: true
+frameInfo:
+ maxAlignment: 1
+ maxCallFrameSize: 0
+machineFunctionInfo:
+ hasRedZone: false
+body: |
+ bb.0.entry:
+ renamable $q0 = LDRQui $sp, 1 :: (load 16)
+ STRSui renamable $s0, $sp, 9, implicit killed $q0 :: (store (s32))
+ renamable $q0 = LDRQui $sp, 0 :: (load 16, align 32)
+ STRSui renamable $s0, $sp, 10, implicit killed $q0 :: (store (s32))
+ RET undef $lr
More information about the llvm-commits
mailing list