[llvm] 147c5d6 - [AArch64] Allow LDR merge with same destination register by renaming (#71908)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Nov 27 16:45:03 PST 2023
Author: Zhaoxuan Jiang
Date: 2023-11-23T08:21:27Z
New Revision: 147c5d6686b935ecd93f8fa0e2dcf38deb593890
URL: https://github.com/llvm/llvm-project/commit/147c5d6686b935ecd93f8fa0e2dcf38deb593890
DIFF: https://github.com/llvm/llvm-project/commit/147c5d6686b935ecd93f8fa0e2dcf38deb593890.diff
LOG: [AArch64] Allow LDR merge with same destination register by renaming (#71908)
The patch is based on a reverted patch:
https://reviews.llvm.org/D103597. It was trying to rename registers
before alias check, which is not safe and causes miscompiles. This patch
does 2 things:
1. Do the renaming with necessary checks passed, including alias check.
2. Rename the register for the instructions between the pairs and
combine the second load into the first. By doing so we can just check
the renamability between the pairs and avoid scanning unknown amount of
instructions before/after the pairs.
Necessary refactoring has been made in order to reuse as much code
possible with STR renaming.
Added:
Modified:
llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
llvm/test/CodeGen/AArch64/arm64-variadic-aapcs.ll
llvm/test/CodeGen/AArch64/fexplog.ll
llvm/test/CodeGen/AArch64/fpow.ll
llvm/test/CodeGen/AArch64/frem.ll
llvm/test/CodeGen/AArch64/fsincos.ll
llvm/test/CodeGen/AArch64/ldrpre-ldr-merge.mir
llvm/test/CodeGen/AArch64/neon-dotreduce.ll
llvm/test/CodeGen/AArch64/stp-opt-with-renaming.mir
llvm/test/CodeGen/AArch64/sve-fixed-length-fp128.ll
llvm/test/CodeGen/AArch64/vecreduce-add.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp b/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
index 7add127e21e3e55..a03a7f8737be0ed 100644
--- a/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
@@ -874,7 +874,7 @@ AArch64LoadStoreOpt::mergePairedInsns(MachineBasicBlock::iterator I,
bool MergeForward = Flags.getMergeForward();
std::optional<MCPhysReg> RenameReg = Flags.getRenameReg();
- if (MergeForward && RenameReg) {
+ if (RenameReg) {
MCRegister RegToRename = getLdStRegOp(*I).getReg();
DefinedInBB.addReg(*RenameReg);
@@ -891,7 +891,8 @@ AArch64LoadStoreOpt::mergePairedInsns(MachineBasicBlock::iterator I,
};
std::function<bool(MachineInstr &, bool)> UpdateMIs =
- [this, RegToRename, GetMatchingSubReg](MachineInstr &MI, bool IsDef) {
+ [this, RegToRename, GetMatchingSubReg, MergeForward](MachineInstr &MI,
+ bool IsDef) {
if (IsDef) {
bool SeenDef = false;
for (unsigned OpIdx = 0; OpIdx < MI.getNumOperands(); ++OpIdx) {
@@ -899,7 +900,8 @@ AArch64LoadStoreOpt::mergePairedInsns(MachineBasicBlock::iterator I,
// Rename the first explicit definition and all implicit
// definitions matching RegToRename.
if (MOP.isReg() && !MOP.isDebug() && MOP.getReg() &&
- (!SeenDef || (MOP.isDef() && MOP.isImplicit())) &&
+ (!MergeForward || !SeenDef ||
+ (MOP.isDef() && MOP.isImplicit())) &&
TRI->regsOverlap(MOP.getReg(), RegToRename)) {
assert((MOP.isImplicit() ||
(MOP.isRenamable() && !MOP.isEarlyClobber())) &&
@@ -926,31 +928,46 @@ AArch64LoadStoreOpt::mergePairedInsns(MachineBasicBlock::iterator I,
assert((MOP.isImplicit() ||
(MOP.isRenamable() && !MOP.isEarlyClobber())) &&
"Need renamable operands");
- const TargetRegisterClass *RC =
- MI.getRegClassConstraint(OpIdx, TII, TRI);
- if (!RC)
- continue;
- MOP.setReg(GetMatchingSubReg(RC));
+ Register MatchingReg;
+ if (const TargetRegisterClass *RC =
+ MI.getRegClassConstraint(OpIdx, TII, TRI))
+ MatchingReg = GetMatchingSubReg(RC);
+ else
+ MatchingReg = GetMatchingSubReg(
+ TRI->getMinimalPhysRegClass(MOP.getReg()));
+ assert(MatchingReg != AArch64::NoRegister &&
+ "Cannot find matching regs for renaming");
+ MOP.setReg(MatchingReg);
}
}
}
LLVM_DEBUG(dbgs() << "Renamed " << MI << "\n");
return true;
};
- forAllMIsUntilDef(*I, RegToRename, TRI, LdStLimit, UpdateMIs);
+ forAllMIsUntilDef(MergeForward ? *I : *std::prev(Paired), RegToRename, TRI,
+ LdStLimit, UpdateMIs);
#if !defined(NDEBUG)
- // Make sure the register used for renaming is not used between the paired
- // instructions. That would trash the content before the new paired
- // instruction.
+ // For forward merging store:
+ // Make sure the register used for renaming is not used between the
+ // paired instructions. That would trash the content before the new
+ // paired instruction.
+ MCPhysReg RegToCheck = *RenameReg;
+ // For backward merging load:
+ // Make sure the register being renamed is not used between the
+ // paired instructions. That would trash the content after the new
+ // paired instruction.
+ if (!MergeForward)
+ RegToCheck = RegToRename;
for (auto &MI :
iterator_range<MachineInstrBundleIterator<llvm::MachineInstr>>(
- std::next(I), std::next(Paired)))
+ MergeForward ? std::next(I) : I,
+ MergeForward ? std::next(Paired) : Paired))
assert(all_of(MI.operands(),
- [this, &RenameReg](const MachineOperand &MOP) {
+ [this, RegToCheck](const MachineOperand &MOP) {
return !MOP.isReg() || MOP.isDebug() || !MOP.getReg() ||
MOP.isUndef() ||
- !TRI->regsOverlap(MOP.getReg(), *RenameReg);
+ !TRI->regsOverlap(MOP.getReg(), RegToCheck);
}) &&
"Rename register used between paired instruction, trashing the "
"content");
@@ -1396,6 +1413,38 @@ static bool areCandidatesToMergeOrPair(MachineInstr &FirstMI, MachineInstr &MI,
// FIXME: Can we also match a mixed sext/zext unscaled/scaled pair?
}
+static bool canRenameMOP(const MachineOperand &MOP,
+ const TargetRegisterInfo *TRI) {
+ if (MOP.isReg()) {
+ auto *RegClass = TRI->getMinimalPhysRegClass(MOP.getReg());
+ // Renaming registers with multiple disjunct sub-registers (e.g. the
+ // result of a LD3) means that all sub-registers are renamed, potentially
+ // impacting other instructions we did not check. Bail out.
+ // Note that this relies on the structure of the AArch64 register file. In
+ // particular, a subregister cannot be written without overwriting the
+ // whole register.
+ if (RegClass->HasDisjunctSubRegs) {
+ LLVM_DEBUG(
+ dbgs()
+ << " Cannot rename operands with multiple disjunct subregisters ("
+ << MOP << ")\n");
+ return false;
+ }
+
+ // We cannot rename arbitrary implicit-defs, the specific rule to rewrite
+ // them must be known. For example, in ORRWrs the implicit-def
+ // corresponds to the result register.
+ if (MOP.isImplicit() && MOP.isDef()) {
+ if (!isRewritableImplicitDef(MOP.getParent()->getOpcode()))
+ return false;
+ return TRI->isSuperOrSubRegisterEq(
+ MOP.getParent()->getOperand(0).getReg(), MOP.getReg());
+ }
+ }
+ return MOP.isImplicit() ||
+ (MOP.isRenamable() && !MOP.isEarlyClobber() && !MOP.isTied());
+}
+
static bool
canRenameUpToDef(MachineInstr &FirstMI, LiveRegUnits &UsedInBetween,
SmallPtrSetImpl<const TargetRegisterClass *> &RequiredClasses,
@@ -1405,10 +1454,6 @@ canRenameUpToDef(MachineInstr &FirstMI, LiveRegUnits &UsedInBetween,
// Check if we can find an unused register which we can use to rename
// the register used by the first load/store.
- auto *RegClass = TRI->getMinimalPhysRegClass(getLdStRegOp(FirstMI).getReg());
- MachineFunction &MF = *FirstMI.getParent()->getParent();
- if (!RegClass || !MF.getRegInfo().tracksLiveness())
- return false;
auto RegToRename = getLdStRegOp(FirstMI).getReg();
// For now, we only rename if the store operand gets killed at the store.
@@ -1422,36 +1467,6 @@ canRenameUpToDef(MachineInstr &FirstMI, LiveRegUnits &UsedInBetween,
LLVM_DEBUG(dbgs() << " Operand not killed at " << FirstMI << "\n");
return false;
}
- auto canRenameMOP = [TRI](const MachineOperand &MOP) {
- if (MOP.isReg()) {
- auto *RegClass = TRI->getMinimalPhysRegClass(MOP.getReg());
- // Renaming registers with multiple disjunct sub-registers (e.g. the
- // result of a LD3) means that all sub-registers are renamed, potentially
- // impacting other instructions we did not check. Bail out.
- // Note that this relies on the structure of the AArch64 register file. In
- // particular, a subregister cannot be written without overwriting the
- // whole register.
- if (RegClass->HasDisjunctSubRegs) {
- LLVM_DEBUG(
- dbgs()
- << " Cannot rename operands with multiple disjunct subregisters ("
- << MOP << ")\n");
- return false;
- }
-
- // We cannot rename arbitrary implicit-defs, the specific rule to rewrite
- // them must be known. For example, in ORRWrs the implicit-def
- // corresponds to the result register.
- if (MOP.isImplicit() && MOP.isDef()) {
- if (!isRewritableImplicitDef(MOP.getParent()->getOpcode()))
- return false;
- return TRI->isSuperOrSubRegisterEq(
- MOP.getParent()->getOperand(0).getReg(), MOP.getReg());
- }
- }
- return MOP.isImplicit() ||
- (MOP.isRenamable() && !MOP.isEarlyClobber() && !MOP.isTied());
- };
bool FoundDef = false;
@@ -1494,7 +1509,7 @@ canRenameUpToDef(MachineInstr &FirstMI, LiveRegUnits &UsedInBetween,
if (!MOP.isReg() || !MOP.isDef() || MOP.isDebug() || !MOP.getReg() ||
!TRI->regsOverlap(MOP.getReg(), RegToRename))
continue;
- if (!canRenameMOP(MOP)) {
+ if (!canRenameMOP(MOP, TRI)) {
LLVM_DEBUG(dbgs()
<< " Cannot rename " << MOP << " in " << MI << "\n");
return false;
@@ -1508,7 +1523,7 @@ canRenameUpToDef(MachineInstr &FirstMI, LiveRegUnits &UsedInBetween,
!TRI->regsOverlap(MOP.getReg(), RegToRename))
continue;
- if (!canRenameMOP(MOP)) {
+ if (!canRenameMOP(MOP, TRI)) {
LLVM_DEBUG(dbgs()
<< " Cannot rename " << MOP << " in " << MI << "\n");
return false;
@@ -1529,6 +1544,56 @@ canRenameUpToDef(MachineInstr &FirstMI, LiveRegUnits &UsedInBetween,
return true;
}
+// We want to merge the second load into the first by rewriting the usages of
+// the same reg between first (incl.) and second (excl.). We don't need to care
+// about any insns before FirstLoad or after SecondLoad.
+// 1. The second load writes new value into the same reg.
+// - The renaming is impossible to impact later use of the reg.
+// - The second load always trash the value written by the first load which
+// means the reg must be killed before the second load.
+// 2. The first load must be a def for the same reg so we don't need to look
+// into anything before it.
+static bool canRenameUntilSecondLoad(
+ MachineInstr &FirstLoad, MachineInstr &SecondLoad,
+ LiveRegUnits &UsedInBetween,
+ SmallPtrSetImpl<const TargetRegisterClass *> &RequiredClasses,
+ const TargetRegisterInfo *TRI) {
+ if (FirstLoad.isPseudo())
+ return false;
+
+ UsedInBetween.accumulate(FirstLoad);
+ auto RegToRename = getLdStRegOp(FirstLoad).getReg();
+ bool Success = std::all_of(
+ FirstLoad.getIterator(), SecondLoad.getIterator(),
+ [&](MachineBasicBlock::iterator MBBI) {
+ MachineInstr &MI = *MBBI;
+
+ LLVM_DEBUG(dbgs() << "Checking " << MI << "\n");
+ // Currently we do not try to rename across frame-setup instructions.
+ if (MI.getFlag(MachineInstr::FrameSetup)) {
+ LLVM_DEBUG(dbgs()
+ << " Cannot rename framesetup instructions currently ("
+ << MI << ")\n");
+ return false;
+ }
+
+ for (auto &MOP : MI.operands()) {
+ if (!MOP.isReg() || MOP.isDebug() || !MOP.getReg() ||
+ !TRI->regsOverlap(MOP.getReg(), RegToRename))
+ continue;
+ if (!canRenameMOP(MOP, TRI)) {
+ LLVM_DEBUG(dbgs()
+ << " Cannot rename " << MOP << " in " << MI << "\n");
+ return false;
+ }
+ RequiredClasses.insert(TRI->getMinimalPhysRegClass(MOP.getReg()));
+ }
+
+ return true;
+ });
+ return Success;
+}
+
// Check if we can find a physical register for renaming \p Reg. This register
// must:
// * not be defined already in \p DefinedInBB; DefinedInBB must contain all
@@ -1577,6 +1642,41 @@ static std::optional<MCPhysReg> tryToFindRegisterToRename(
return std::nullopt;
}
+// For store pairs: returns a register from FirstMI to the beginning of the
+// block that can be renamed.
+// For load pairs: returns a register from FirstMI to MI that can be renamed.
+static std::optional<MCPhysReg> findRenameRegForSameLdStRegPair(
+ std::optional<bool> MaybeCanRename, MachineInstr &FirstMI, MachineInstr &MI,
+ Register Reg, LiveRegUnits &DefinedInBB, LiveRegUnits &UsedInBetween,
+ SmallPtrSetImpl<const TargetRegisterClass *> &RequiredClasses,
+ const TargetRegisterInfo *TRI) {
+ std::optional<MCPhysReg> RenameReg;
+ if (!DebugCounter::shouldExecute(RegRenamingCounter))
+ return RenameReg;
+
+ auto *RegClass = TRI->getMinimalPhysRegClass(getLdStRegOp(FirstMI).getReg());
+ MachineFunction &MF = *FirstMI.getParent()->getParent();
+ if (!RegClass || !MF.getRegInfo().tracksLiveness())
+ return RenameReg;
+
+ const bool IsLoad = FirstMI.mayLoad();
+
+ if (!MaybeCanRename) {
+ if (IsLoad)
+ MaybeCanRename = {canRenameUntilSecondLoad(FirstMI, MI, UsedInBetween,
+ RequiredClasses, TRI)};
+ else
+ MaybeCanRename = {
+ canRenameUpToDef(FirstMI, UsedInBetween, RequiredClasses, TRI)};
+ }
+
+ if (*MaybeCanRename) {
+ RenameReg = tryToFindRegisterToRename(MF, Reg, DefinedInBB, UsedInBetween,
+ RequiredClasses, TRI);
+ }
+ return RenameReg;
+}
+
/// Scan the instructions looking for a load/store that can be combined with the
/// current instruction into a wider equivalent or a load/store pair.
MachineBasicBlock::iterator
@@ -1729,17 +1829,6 @@ AArch64LoadStoreOpt::findMatchingInsn(MachineBasicBlock::iterator I,
continue;
}
}
- // If the destination register of one load is the same register or a
- // sub/super register of the other load, bail and keep looking. A
- // load-pair instruction with both destination registers the same is
- // UNPREDICTABLE and will result in an exception.
- if (MayLoad &&
- TRI->isSuperOrSubRegisterEq(Reg, getLdStRegOp(MI).getReg())) {
- LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits,
- TRI);
- MemInsns.push_back(&MI);
- continue;
- }
// If the BaseReg has been modified, then we cannot do the optimization.
// For example, in the following pattern
@@ -1750,17 +1839,37 @@ AArch64LoadStoreOpt::findMatchingInsn(MachineBasicBlock::iterator I,
if (!ModifiedRegUnits.available(BaseReg))
return E;
+ const bool SameLoadReg = MayLoad && TRI->isSuperOrSubRegisterEq(
+ Reg, getLdStRegOp(MI).getReg());
+
// If the Rt of the second instruction was not modified or used between
// the two instructions and none of the instructions between the second
// and first alias with the second, we can combine the second into the
// first.
if (ModifiedRegUnits.available(getLdStRegOp(MI).getReg()) &&
- !(MI.mayLoad() &&
+ !(MI.mayLoad() && !SameLoadReg &&
!UsedRegUnits.available(getLdStRegOp(MI).getReg())) &&
!mayAlias(MI, MemInsns, AA)) {
+ // For pairs loading into the same reg, try to find a renaming
+ // opportunity to allow the renaming of Reg between FirstMI and MI
+ // and combine MI into FirstMI; otherwise bail and keep looking.
+ if (SameLoadReg) {
+ std::optional<MCPhysReg> RenameReg =
+ findRenameRegForSameLdStRegPair(MaybeCanRename, FirstMI, MI,
+ Reg, DefinedInBB, UsedInBetween,
+ RequiredClasses, TRI);
+ if (!RenameReg) {
+ LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits,
+ UsedRegUnits, TRI);
+ MemInsns.push_back(&MI);
+ continue;
+ }
+ Flags.setRenameReg(*RenameReg);
+ }
Flags.setMergeForward(false);
- Flags.clearRenameReg();
+ if (!SameLoadReg)
+ Flags.clearRenameReg();
return MBBI;
}
@@ -1778,22 +1887,13 @@ AArch64LoadStoreOpt::findMatchingInsn(MachineBasicBlock::iterator I,
return MBBI;
}
- if (DebugCounter::shouldExecute(RegRenamingCounter)) {
- if (!MaybeCanRename)
- MaybeCanRename = {canRenameUpToDef(FirstMI, UsedInBetween,
- RequiredClasses, TRI)};
-
- if (*MaybeCanRename) {
- std::optional<MCPhysReg> MaybeRenameReg =
- tryToFindRegisterToRename(*FirstMI.getParent()->getParent(),
- Reg, DefinedInBB, UsedInBetween,
- RequiredClasses, TRI);
- if (MaybeRenameReg) {
- Flags.setRenameReg(*MaybeRenameReg);
- Flags.setMergeForward(true);
- MBBIWithRenameReg = MBBI;
- }
- }
+ std::optional<MCPhysReg> RenameReg = findRenameRegForSameLdStRegPair(
+ MaybeCanRename, FirstMI, MI, Reg, DefinedInBB, UsedInBetween,
+ RequiredClasses, TRI);
+ if (RenameReg) {
+ Flags.setMergeForward(true);
+ Flags.setRenameReg(*RenameReg);
+ MBBIWithRenameReg = MBBI;
}
}
// Unable to combine these instructions due to interference in between.
diff --git a/llvm/test/CodeGen/AArch64/arm64-variadic-aapcs.ll b/llvm/test/CodeGen/AArch64/arm64-variadic-aapcs.ll
index de07ed1b5d7ec96..e688af7fdeca38a 100644
--- a/llvm/test/CodeGen/AArch64/arm64-variadic-aapcs.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-variadic-aapcs.ll
@@ -125,9 +125,8 @@ define dso_local void @test_va_copy() {
; CHECK: add x[[SRC:[0-9]+]], {{x[0-9]+}}, :lo12:var
-; CHECK: ldr [[BLOCKB:q[0-9]+]], [x[[SRC]], #16]
+; CHECK: ldp [[BLOCKA:q[0-9]+]], [[BLOCKB:q[0-9]+]], [x[[SRC]]]
; CHECK: add x[[DST:[0-9]+]], {{x[0-9]+}}, :lo12:second_list
-; CHECK: ldr [[BLOCKA:q[0-9]+]], [x[[SRC]]]
; CHECK: stp [[BLOCKA]], [[BLOCKB]], [x[[DST]]]
ret void
; CHECK: ret
diff --git a/llvm/test/CodeGen/AArch64/fexplog.ll b/llvm/test/CodeGen/AArch64/fexplog.ll
index be6040faa2836bf..26c0b68307b32b4 100644
--- a/llvm/test/CodeGen/AArch64/fexplog.ll
+++ b/llvm/test/CodeGen/AArch64/fexplog.ll
@@ -713,14 +713,12 @@ define <7 x half> @exp_v7f16(<7 x half> %a) {
; CHECK-GI-NEXT: ldr x30, [sp, #144] // 8-byte Folded Reload
; CHECK-GI-NEXT: ldp d11, d10, [sp, #112] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[1], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #48] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #32] // 32-byte Folded Reload
; CHECK-GI-NEXT: ldp d13, d12, [sp, #96] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[2], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[2], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[3], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[4], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp] // 32-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[4], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[5], v2.h[0]
; CHECK-GI-NEXT: mov v1.h[6], v0.h[0]
; CHECK-GI-NEXT: mov v1.h[7], v0.h[0]
@@ -963,14 +961,12 @@ define <8 x half> @exp_v8f16(<8 x half> %a) {
; CHECK-GI-NEXT: ldp d11, d10, [sp, #136] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldr d14, [sp, #112] // 8-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[1], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #64] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #48] // 32-byte Folded Reload
; CHECK-GI-NEXT: ldp d13, d12, [sp, #120] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[2], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #48] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[2], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[3], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[4], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #16] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #16] // 32-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[4], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[5], v2.h[0]
; CHECK-GI-NEXT: ldr q2, [sp] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[6], v2.h[0]
@@ -1994,14 +1990,12 @@ define <7 x half> @exp2_v7f16(<7 x half> %a) {
; CHECK-GI-NEXT: ldr x30, [sp, #144] // 8-byte Folded Reload
; CHECK-GI-NEXT: ldp d11, d10, [sp, #112] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[1], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #48] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #32] // 32-byte Folded Reload
; CHECK-GI-NEXT: ldp d13, d12, [sp, #96] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[2], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[2], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[3], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[4], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp] // 32-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[4], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[5], v2.h[0]
; CHECK-GI-NEXT: mov v1.h[6], v0.h[0]
; CHECK-GI-NEXT: mov v1.h[7], v0.h[0]
@@ -2244,14 +2238,12 @@ define <8 x half> @exp2_v8f16(<8 x half> %a) {
; CHECK-GI-NEXT: ldp d11, d10, [sp, #136] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldr d14, [sp, #112] // 8-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[1], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #64] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #48] // 32-byte Folded Reload
; CHECK-GI-NEXT: ldp d13, d12, [sp, #120] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[2], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #48] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[2], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[3], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[4], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #16] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #16] // 32-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[4], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[5], v2.h[0]
; CHECK-GI-NEXT: ldr q2, [sp] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[6], v2.h[0]
@@ -3275,14 +3267,12 @@ define <7 x half> @log_v7f16(<7 x half> %a) {
; CHECK-GI-NEXT: ldr x30, [sp, #144] // 8-byte Folded Reload
; CHECK-GI-NEXT: ldp d11, d10, [sp, #112] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[1], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #48] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #32] // 32-byte Folded Reload
; CHECK-GI-NEXT: ldp d13, d12, [sp, #96] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[2], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[2], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[3], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[4], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp] // 32-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[4], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[5], v2.h[0]
; CHECK-GI-NEXT: mov v1.h[6], v0.h[0]
; CHECK-GI-NEXT: mov v1.h[7], v0.h[0]
@@ -3525,14 +3515,12 @@ define <8 x half> @log_v8f16(<8 x half> %a) {
; CHECK-GI-NEXT: ldp d11, d10, [sp, #136] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldr d14, [sp, #112] // 8-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[1], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #64] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #48] // 32-byte Folded Reload
; CHECK-GI-NEXT: ldp d13, d12, [sp, #120] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[2], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #48] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[2], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[3], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[4], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #16] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #16] // 32-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[4], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[5], v2.h[0]
; CHECK-GI-NEXT: ldr q2, [sp] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[6], v2.h[0]
@@ -4556,14 +4544,12 @@ define <7 x half> @log2_v7f16(<7 x half> %a) {
; CHECK-GI-NEXT: ldr x30, [sp, #144] // 8-byte Folded Reload
; CHECK-GI-NEXT: ldp d11, d10, [sp, #112] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[1], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #48] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #32] // 32-byte Folded Reload
; CHECK-GI-NEXT: ldp d13, d12, [sp, #96] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[2], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[2], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[3], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[4], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp] // 32-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[4], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[5], v2.h[0]
; CHECK-GI-NEXT: mov v1.h[6], v0.h[0]
; CHECK-GI-NEXT: mov v1.h[7], v0.h[0]
@@ -4806,14 +4792,12 @@ define <8 x half> @log2_v8f16(<8 x half> %a) {
; CHECK-GI-NEXT: ldp d11, d10, [sp, #136] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldr d14, [sp, #112] // 8-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[1], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #64] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #48] // 32-byte Folded Reload
; CHECK-GI-NEXT: ldp d13, d12, [sp, #120] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[2], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #48] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[2], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[3], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[4], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #16] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #16] // 32-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[4], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[5], v2.h[0]
; CHECK-GI-NEXT: ldr q2, [sp] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[6], v2.h[0]
@@ -5837,14 +5821,12 @@ define <7 x half> @log10_v7f16(<7 x half> %a) {
; CHECK-GI-NEXT: ldr x30, [sp, #144] // 8-byte Folded Reload
; CHECK-GI-NEXT: ldp d11, d10, [sp, #112] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[1], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #48] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #32] // 32-byte Folded Reload
; CHECK-GI-NEXT: ldp d13, d12, [sp, #96] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[2], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[2], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[3], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[4], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp] // 32-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[4], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[5], v2.h[0]
; CHECK-GI-NEXT: mov v1.h[6], v0.h[0]
; CHECK-GI-NEXT: mov v1.h[7], v0.h[0]
@@ -6087,14 +6069,12 @@ define <8 x half> @log10_v8f16(<8 x half> %a) {
; CHECK-GI-NEXT: ldp d11, d10, [sp, #136] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldr d14, [sp, #112] // 8-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[1], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #64] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #48] // 32-byte Folded Reload
; CHECK-GI-NEXT: ldp d13, d12, [sp, #120] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[2], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #48] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[2], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[3], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[4], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #16] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #16] // 32-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[4], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[5], v2.h[0]
; CHECK-GI-NEXT: ldr q2, [sp] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[6], v2.h[0]
diff --git a/llvm/test/CodeGen/AArch64/fpow.ll b/llvm/test/CodeGen/AArch64/fpow.ll
index 1b3ae7a227924a3..7681ab510b3099f 100644
--- a/llvm/test/CodeGen/AArch64/fpow.ll
+++ b/llvm/test/CodeGen/AArch64/fpow.ll
@@ -267,14 +267,13 @@ define <4 x double> @pow_v4f64(<4 x double> %a, <4 x double> %b) {
; CHECK-GI-NEXT: fmov d1, d11
; CHECK-GI-NEXT: fmov d0, d9
; CHECK-GI-NEXT: bl pow
-; CHECK-GI-NEXT: ldr q2, [sp, #48] // 16-byte Folded Reload
-; CHECK-GI-NEXT: ldr q1, [sp, #16] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q3, q1, [sp, #16] // 32-byte Folded Reload
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT: ldr x30, [sp, #96] // 8-byte Folded Reload
+; CHECK-GI-NEXT: ldr q2, [sp, #48] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldp d9, d8, [sp, #80] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v2.d[1], v1.d[0]
-; CHECK-GI-NEXT: ldr q1, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldr x30, [sp, #96] // 8-byte Folded Reload
; CHECK-GI-NEXT: ldp d11, d10, [sp, #64] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mov v2.d[1], v3.d[0]
; CHECK-GI-NEXT: mov v1.d[1], v0.d[0]
; CHECK-GI-NEXT: mov v0.16b, v2.16b
; CHECK-GI-NEXT: add sp, sp, #112
@@ -715,14 +714,14 @@ define <7 x half> @pow_v7f16(<7 x half> %a, <7 x half> %b) {
; CHECK-SD-NEXT: fcvt s1, h1
; CHECK-SD-NEXT: bl powf
; CHECK-SD-NEXT: fcvt h0, s0
-; CHECK-SD-NEXT: ldr q1, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT: mov v0.h[1], v1.h[0]
+; CHECK-SD-NEXT: ldp q1, q2, [sp, #16] // 32-byte Folded Reload
+; CHECK-SD-NEXT: mov h1, v1.h[2]
+; CHECK-SD-NEXT: mov v0.h[1], v2.h[0]
+; CHECK-SD-NEXT: fcvt s1, h1
; CHECK-SD-NEXT: str q0, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NEXT: ldp q0, q1, [sp] // 32-byte Folded Reload
+; CHECK-SD-NEXT: ldr q0, [sp] // 16-byte Folded Reload
; CHECK-SD-NEXT: mov h0, v0.h[2]
-; CHECK-SD-NEXT: mov h1, v1.h[2]
; CHECK-SD-NEXT: fcvt s0, h0
-; CHECK-SD-NEXT: fcvt s1, h1
; CHECK-SD-NEXT: bl powf
; CHECK-SD-NEXT: fcvt h0, s0
; CHECK-SD-NEXT: ldr q1, [sp, #32] // 16-byte Folded Reload
@@ -858,14 +857,13 @@ define <7 x half> @pow_v7f16(<7 x half> %a, <7 x half> %b) {
; CHECK-GI-NEXT: fcvt s1, h0
; CHECK-GI-NEXT: fmov s0, s2
; CHECK-GI-NEXT: bl powf
-; CHECK-GI-NEXT: ldr q1, [sp, #64] // 16-byte Folded Reload
-; CHECK-GI-NEXT: ldr q2, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #16] // 32-byte Folded Reload
; CHECK-GI-NEXT: fcvt h0, s0
+; CHECK-GI-NEXT: ldr q1, [sp, #64] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldp d9, d8, [sp, #144] // 16-byte Folded Reload
-; CHECK-GI-NEXT: ldr x30, [sp, #160] // 8-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[1], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #16] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldp d11, d10, [sp, #128] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldr x30, [sp, #160] // 8-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[1], v3.h[0]
; CHECK-GI-NEXT: ldp d13, d12, [sp, #112] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldp d15, d14, [sp, #96] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[2], v2.h[0]
@@ -1020,14 +1018,14 @@ define <8 x half> @pow_v8f16(<8 x half> %a, <8 x half> %b) {
; CHECK-SD-NEXT: fcvt s1, h1
; CHECK-SD-NEXT: bl powf
; CHECK-SD-NEXT: fcvt h0, s0
-; CHECK-SD-NEXT: ldr q1, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT: mov v0.h[1], v1.h[0]
+; CHECK-SD-NEXT: ldp q1, q2, [sp, #16] // 32-byte Folded Reload
+; CHECK-SD-NEXT: mov h1, v1.h[2]
+; CHECK-SD-NEXT: mov v0.h[1], v2.h[0]
+; CHECK-SD-NEXT: fcvt s1, h1
; CHECK-SD-NEXT: str q0, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NEXT: ldp q0, q1, [sp] // 32-byte Folded Reload
+; CHECK-SD-NEXT: ldr q0, [sp] // 16-byte Folded Reload
; CHECK-SD-NEXT: mov h0, v0.h[2]
-; CHECK-SD-NEXT: mov h1, v1.h[2]
; CHECK-SD-NEXT: fcvt s0, h0
-; CHECK-SD-NEXT: fcvt s1, h1
; CHECK-SD-NEXT: bl powf
; CHECK-SD-NEXT: fcvt h0, s0
; CHECK-SD-NEXT: ldr q1, [sp, #32] // 16-byte Folded Reload
@@ -1175,22 +1173,20 @@ define <8 x half> @pow_v8f16(<8 x half> %a, <8 x half> %b) {
; CHECK-GI-NEXT: fcvt s1, h0
; CHECK-GI-NEXT: fmov s0, s2
; CHECK-GI-NEXT: bl powf
-; CHECK-GI-NEXT: ldr q1, [sp, #80] // 16-byte Folded Reload
-; CHECK-GI-NEXT: ldr q2, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #16] // 32-byte Folded Reload
; CHECK-GI-NEXT: fcvt h0, s0
+; CHECK-GI-NEXT: ldr q1, [sp, #80] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldp d9, d8, [sp, #160] // 16-byte Folded Reload
-; CHECK-GI-NEXT: ldr x30, [sp, #176] // 8-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[1], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #16] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldp d11, d10, [sp, #144] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldr x30, [sp, #176] // 8-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[1], v3.h[0]
; CHECK-GI-NEXT: ldp d13, d12, [sp, #128] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldp d15, d14, [sp, #112] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[2], v2.h[0]
; CHECK-GI-NEXT: ldr q2, [sp] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[3], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #48] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[4], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #64] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q3, q2, [sp, #48] // 32-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[4], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[5], v2.h[0]
; CHECK-GI-NEXT: ldr q2, [sp, #96] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[6], v2.h[0]
@@ -1225,14 +1221,13 @@ define <16 x half> @pow_v16f16(<16 x half> %a, <16 x half> %b) {
; CHECK-SD-NEXT: fcvt s0, h0
; CHECK-SD-NEXT: bl powf
; CHECK-SD-NEXT: fcvt h0, s0
-; CHECK-SD-NEXT: ldr q1, [sp, #64] // 16-byte Folded Reload
-; CHECK-SD-NEXT: mov v0.h[1], v1.h[0]
-; CHECK-SD-NEXT: ldr q1, [sp, #48] // 16-byte Folded Reload
+; CHECK-SD-NEXT: ldp q1, q2, [sp, #48] // 32-byte Folded Reload
; CHECK-SD-NEXT: mov h1, v1.h[2]
+; CHECK-SD-NEXT: mov v0.h[1], v2.h[0]
+; CHECK-SD-NEXT: fcvt s1, h1
; CHECK-SD-NEXT: str q0, [sp, #64] // 16-byte Folded Spill
; CHECK-SD-NEXT: ldr q0, [sp] // 16-byte Folded Reload
; CHECK-SD-NEXT: mov h0, v0.h[2]
-; CHECK-SD-NEXT: fcvt s1, h1
; CHECK-SD-NEXT: fcvt s0, h0
; CHECK-SD-NEXT: bl powf
; CHECK-SD-NEXT: fcvt h0, s0
@@ -1307,14 +1302,14 @@ define <16 x half> @pow_v16f16(<16 x half> %a, <16 x half> %b) {
; CHECK-SD-NEXT: fcvt s1, h1
; CHECK-SD-NEXT: bl powf
; CHECK-SD-NEXT: fcvt h0, s0
-; CHECK-SD-NEXT: ldr q1, [sp, #48] // 16-byte Folded Reload
-; CHECK-SD-NEXT: mov v0.h[1], v1.h[0]
+; CHECK-SD-NEXT: ldp q1, q2, [sp, #32] // 32-byte Folded Reload
+; CHECK-SD-NEXT: mov h1, v1.h[2]
+; CHECK-SD-NEXT: mov v0.h[1], v2.h[0]
+; CHECK-SD-NEXT: fcvt s1, h1
; CHECK-SD-NEXT: str q0, [sp, #48] // 16-byte Folded Spill
-; CHECK-SD-NEXT: ldp q0, q1, [sp, #16] // 32-byte Folded Reload
+; CHECK-SD-NEXT: ldr q0, [sp, #16] // 16-byte Folded Reload
; CHECK-SD-NEXT: mov h0, v0.h[2]
-; CHECK-SD-NEXT: mov h1, v1.h[2]
; CHECK-SD-NEXT: fcvt s0, h0
-; CHECK-SD-NEXT: fcvt s1, h1
; CHECK-SD-NEXT: bl powf
; CHECK-SD-NEXT: fcvt h0, s0
; CHECK-SD-NEXT: ldr q1, [sp, #48] // 16-byte Folded Reload
@@ -1578,9 +1573,8 @@ define <16 x half> @pow_v16f16(<16 x half> %a, <16 x half> %b) {
; CHECK-GI-NEXT: mov v3.h[3], v2.h[0]
; CHECK-GI-NEXT: ldr q2, [sp, #192] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[3], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #240] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v3.h[4], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #256] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q4, q2, [sp, #240] // 32-byte Folded Reload
+; CHECK-GI-NEXT: mov v3.h[4], v4.h[0]
; CHECK-GI-NEXT: mov v1.h[4], v2.h[0]
; CHECK-GI-NEXT: ldr q2, [sp, #176] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v3.h[5], v2.h[0]
diff --git a/llvm/test/CodeGen/AArch64/frem.ll b/llvm/test/CodeGen/AArch64/frem.ll
index e8077b94f2bf688..ed0f6c442ece170 100644
--- a/llvm/test/CodeGen/AArch64/frem.ll
+++ b/llvm/test/CodeGen/AArch64/frem.ll
@@ -269,14 +269,13 @@ define <4 x double> @frem_v4f64(<4 x double> %a, <4 x double> %b) {
; CHECK-GI-NEXT: fmov d1, d11
; CHECK-GI-NEXT: fmov d0, d9
; CHECK-GI-NEXT: bl fmod
-; CHECK-GI-NEXT: ldr q2, [sp, #48] // 16-byte Folded Reload
-; CHECK-GI-NEXT: ldr q1, [sp, #16] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q3, q1, [sp, #16] // 32-byte Folded Reload
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT: ldr x30, [sp, #96] // 8-byte Folded Reload
+; CHECK-GI-NEXT: ldr q2, [sp, #48] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldp d9, d8, [sp, #80] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v2.d[1], v1.d[0]
-; CHECK-GI-NEXT: ldr q1, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldr x30, [sp, #96] // 8-byte Folded Reload
; CHECK-GI-NEXT: ldp d11, d10, [sp, #64] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mov v2.d[1], v3.d[0]
; CHECK-GI-NEXT: mov v1.d[1], v0.d[0]
; CHECK-GI-NEXT: mov v0.16b, v2.16b
; CHECK-GI-NEXT: add sp, sp, #112
@@ -717,14 +716,14 @@ define <7 x half> @frem_v7f16(<7 x half> %a, <7 x half> %b) {
; CHECK-SD-NEXT: fcvt s1, h1
; CHECK-SD-NEXT: bl fmodf
; CHECK-SD-NEXT: fcvt h0, s0
-; CHECK-SD-NEXT: ldr q1, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT: mov v0.h[1], v1.h[0]
+; CHECK-SD-NEXT: ldp q1, q2, [sp, #16] // 32-byte Folded Reload
+; CHECK-SD-NEXT: mov h1, v1.h[2]
+; CHECK-SD-NEXT: mov v0.h[1], v2.h[0]
+; CHECK-SD-NEXT: fcvt s1, h1
; CHECK-SD-NEXT: str q0, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NEXT: ldp q0, q1, [sp] // 32-byte Folded Reload
+; CHECK-SD-NEXT: ldr q0, [sp] // 16-byte Folded Reload
; CHECK-SD-NEXT: mov h0, v0.h[2]
-; CHECK-SD-NEXT: mov h1, v1.h[2]
; CHECK-SD-NEXT: fcvt s0, h0
-; CHECK-SD-NEXT: fcvt s1, h1
; CHECK-SD-NEXT: bl fmodf
; CHECK-SD-NEXT: fcvt h0, s0
; CHECK-SD-NEXT: ldr q1, [sp, #32] // 16-byte Folded Reload
@@ -860,14 +859,13 @@ define <7 x half> @frem_v7f16(<7 x half> %a, <7 x half> %b) {
; CHECK-GI-NEXT: fcvt s1, h0
; CHECK-GI-NEXT: fmov s0, s2
; CHECK-GI-NEXT: bl fmodf
-; CHECK-GI-NEXT: ldr q1, [sp, #64] // 16-byte Folded Reload
-; CHECK-GI-NEXT: ldr q2, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #16] // 32-byte Folded Reload
; CHECK-GI-NEXT: fcvt h0, s0
+; CHECK-GI-NEXT: ldr q1, [sp, #64] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldp d9, d8, [sp, #144] // 16-byte Folded Reload
-; CHECK-GI-NEXT: ldr x30, [sp, #160] // 8-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[1], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #16] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldp d11, d10, [sp, #128] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldr x30, [sp, #160] // 8-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[1], v3.h[0]
; CHECK-GI-NEXT: ldp d13, d12, [sp, #112] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldp d15, d14, [sp, #96] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[2], v2.h[0]
@@ -1022,14 +1020,14 @@ define <8 x half> @frem_v8f16(<8 x half> %a, <8 x half> %b) {
; CHECK-SD-NEXT: fcvt s1, h1
; CHECK-SD-NEXT: bl fmodf
; CHECK-SD-NEXT: fcvt h0, s0
-; CHECK-SD-NEXT: ldr q1, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT: mov v0.h[1], v1.h[0]
+; CHECK-SD-NEXT: ldp q1, q2, [sp, #16] // 32-byte Folded Reload
+; CHECK-SD-NEXT: mov h1, v1.h[2]
+; CHECK-SD-NEXT: mov v0.h[1], v2.h[0]
+; CHECK-SD-NEXT: fcvt s1, h1
; CHECK-SD-NEXT: str q0, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NEXT: ldp q0, q1, [sp] // 32-byte Folded Reload
+; CHECK-SD-NEXT: ldr q0, [sp] // 16-byte Folded Reload
; CHECK-SD-NEXT: mov h0, v0.h[2]
-; CHECK-SD-NEXT: mov h1, v1.h[2]
; CHECK-SD-NEXT: fcvt s0, h0
-; CHECK-SD-NEXT: fcvt s1, h1
; CHECK-SD-NEXT: bl fmodf
; CHECK-SD-NEXT: fcvt h0, s0
; CHECK-SD-NEXT: ldr q1, [sp, #32] // 16-byte Folded Reload
@@ -1177,22 +1175,20 @@ define <8 x half> @frem_v8f16(<8 x half> %a, <8 x half> %b) {
; CHECK-GI-NEXT: fcvt s1, h0
; CHECK-GI-NEXT: fmov s0, s2
; CHECK-GI-NEXT: bl fmodf
-; CHECK-GI-NEXT: ldr q1, [sp, #80] // 16-byte Folded Reload
-; CHECK-GI-NEXT: ldr q2, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #16] // 32-byte Folded Reload
; CHECK-GI-NEXT: fcvt h0, s0
+; CHECK-GI-NEXT: ldr q1, [sp, #80] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldp d9, d8, [sp, #160] // 16-byte Folded Reload
-; CHECK-GI-NEXT: ldr x30, [sp, #176] // 8-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[1], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #16] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldp d11, d10, [sp, #144] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldr x30, [sp, #176] // 8-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[1], v3.h[0]
; CHECK-GI-NEXT: ldp d13, d12, [sp, #128] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldp d15, d14, [sp, #112] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[2], v2.h[0]
; CHECK-GI-NEXT: ldr q2, [sp] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[3], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #48] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[4], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #64] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q3, q2, [sp, #48] // 32-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[4], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[5], v2.h[0]
; CHECK-GI-NEXT: ldr q2, [sp, #96] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[6], v2.h[0]
@@ -1227,14 +1223,13 @@ define <16 x half> @frem_v16f16(<16 x half> %a, <16 x half> %b) {
; CHECK-SD-NEXT: fcvt s0, h0
; CHECK-SD-NEXT: bl fmodf
; CHECK-SD-NEXT: fcvt h0, s0
-; CHECK-SD-NEXT: ldr q1, [sp, #64] // 16-byte Folded Reload
-; CHECK-SD-NEXT: mov v0.h[1], v1.h[0]
-; CHECK-SD-NEXT: ldr q1, [sp, #48] // 16-byte Folded Reload
+; CHECK-SD-NEXT: ldp q1, q2, [sp, #48] // 32-byte Folded Reload
; CHECK-SD-NEXT: mov h1, v1.h[2]
+; CHECK-SD-NEXT: mov v0.h[1], v2.h[0]
+; CHECK-SD-NEXT: fcvt s1, h1
; CHECK-SD-NEXT: str q0, [sp, #64] // 16-byte Folded Spill
; CHECK-SD-NEXT: ldr q0, [sp] // 16-byte Folded Reload
; CHECK-SD-NEXT: mov h0, v0.h[2]
-; CHECK-SD-NEXT: fcvt s1, h1
; CHECK-SD-NEXT: fcvt s0, h0
; CHECK-SD-NEXT: bl fmodf
; CHECK-SD-NEXT: fcvt h0, s0
@@ -1309,14 +1304,14 @@ define <16 x half> @frem_v16f16(<16 x half> %a, <16 x half> %b) {
; CHECK-SD-NEXT: fcvt s1, h1
; CHECK-SD-NEXT: bl fmodf
; CHECK-SD-NEXT: fcvt h0, s0
-; CHECK-SD-NEXT: ldr q1, [sp, #48] // 16-byte Folded Reload
-; CHECK-SD-NEXT: mov v0.h[1], v1.h[0]
+; CHECK-SD-NEXT: ldp q1, q2, [sp, #32] // 32-byte Folded Reload
+; CHECK-SD-NEXT: mov h1, v1.h[2]
+; CHECK-SD-NEXT: mov v0.h[1], v2.h[0]
+; CHECK-SD-NEXT: fcvt s1, h1
; CHECK-SD-NEXT: str q0, [sp, #48] // 16-byte Folded Spill
-; CHECK-SD-NEXT: ldp q0, q1, [sp, #16] // 32-byte Folded Reload
+; CHECK-SD-NEXT: ldr q0, [sp, #16] // 16-byte Folded Reload
; CHECK-SD-NEXT: mov h0, v0.h[2]
-; CHECK-SD-NEXT: mov h1, v1.h[2]
; CHECK-SD-NEXT: fcvt s0, h0
-; CHECK-SD-NEXT: fcvt s1, h1
; CHECK-SD-NEXT: bl fmodf
; CHECK-SD-NEXT: fcvt h0, s0
; CHECK-SD-NEXT: ldr q1, [sp, #48] // 16-byte Folded Reload
@@ -1580,9 +1575,8 @@ define <16 x half> @frem_v16f16(<16 x half> %a, <16 x half> %b) {
; CHECK-GI-NEXT: mov v3.h[3], v2.h[0]
; CHECK-GI-NEXT: ldr q2, [sp, #192] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[3], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #240] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v3.h[4], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #256] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q4, q2, [sp, #240] // 32-byte Folded Reload
+; CHECK-GI-NEXT: mov v3.h[4], v4.h[0]
; CHECK-GI-NEXT: mov v1.h[4], v2.h[0]
; CHECK-GI-NEXT: ldr q2, [sp, #176] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v3.h[5], v2.h[0]
diff --git a/llvm/test/CodeGen/AArch64/fsincos.ll b/llvm/test/CodeGen/AArch64/fsincos.ll
index ec60fedeb47296b..9784b9ea9b65f58 100644
--- a/llvm/test/CodeGen/AArch64/fsincos.ll
+++ b/llvm/test/CodeGen/AArch64/fsincos.ll
@@ -713,14 +713,12 @@ define <7 x half> @sin_v7f16(<7 x half> %a) {
; CHECK-GI-NEXT: ldr x30, [sp, #144] // 8-byte Folded Reload
; CHECK-GI-NEXT: ldp d11, d10, [sp, #112] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[1], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #48] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #32] // 32-byte Folded Reload
; CHECK-GI-NEXT: ldp d13, d12, [sp, #96] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[2], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[2], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[3], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[4], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp] // 32-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[4], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[5], v2.h[0]
; CHECK-GI-NEXT: mov v1.h[6], v0.h[0]
; CHECK-GI-NEXT: mov v1.h[7], v0.h[0]
@@ -963,14 +961,12 @@ define <8 x half> @sin_v8f16(<8 x half> %a) {
; CHECK-GI-NEXT: ldp d11, d10, [sp, #136] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldr d14, [sp, #112] // 8-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[1], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #64] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #48] // 32-byte Folded Reload
; CHECK-GI-NEXT: ldp d13, d12, [sp, #120] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[2], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #48] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[2], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[3], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[4], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #16] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #16] // 32-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[4], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[5], v2.h[0]
; CHECK-GI-NEXT: ldr q2, [sp] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[6], v2.h[0]
@@ -1994,14 +1990,12 @@ define <7 x half> @cos_v7f16(<7 x half> %a) {
; CHECK-GI-NEXT: ldr x30, [sp, #144] // 8-byte Folded Reload
; CHECK-GI-NEXT: ldp d11, d10, [sp, #112] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[1], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #48] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #32] // 32-byte Folded Reload
; CHECK-GI-NEXT: ldp d13, d12, [sp, #96] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[2], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[2], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[3], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[4], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp] // 32-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[4], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[5], v2.h[0]
; CHECK-GI-NEXT: mov v1.h[6], v0.h[0]
; CHECK-GI-NEXT: mov v1.h[7], v0.h[0]
@@ -2244,14 +2238,12 @@ define <8 x half> @cos_v8f16(<8 x half> %a) {
; CHECK-GI-NEXT: ldp d11, d10, [sp, #136] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldr d14, [sp, #112] // 8-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[1], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #64] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #48] // 32-byte Folded Reload
; CHECK-GI-NEXT: ldp d13, d12, [sp, #120] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[2], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #48] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[2], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[3], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v1.h[4], v2.h[0]
-; CHECK-GI-NEXT: ldr q2, [sp, #16] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp q2, q3, [sp, #16] // 32-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[4], v3.h[0]
; CHECK-GI-NEXT: mov v1.h[5], v2.h[0]
; CHECK-GI-NEXT: ldr q2, [sp] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v1.h[6], v2.h[0]
diff --git a/llvm/test/CodeGen/AArch64/ldrpre-ldr-merge.mir b/llvm/test/CodeGen/AArch64/ldrpre-ldr-merge.mir
index b76570da781f5a6..8e29255189bf530 100644
--- a/llvm/test/CodeGen/AArch64/ldrpre-ldr-merge.mir
+++ b/llvm/test/CodeGen/AArch64/ldrpre-ldr-merge.mir
@@ -348,8 +348,6 @@ body: |
bb.0:
liveins: $s0, $s1, $x1
- ; The offset of the second load is not equal to the
- ; size of the destination register, and hence can’t be merged.
; CHECK-LABEL: name: 12-ldrspre-ldrsui-no-merge
; CHECK: liveins: $s0, $s1, $x1
@@ -445,8 +443,7 @@ body: |
; CHECK-LABEL: name: 15-ldrqpre-ldrqui-same-dst-reg-no-merge
; CHECK: liveins: $q0, $q1, $x1
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: early-clobber renamable $x1, dead renamable $q0 = LDRQpre renamable $x1, 32, implicit $w1 :: (load (s128))
- ; CHECK-NEXT: renamable $q0 = LDRQui renamable $x1, 1 :: (load (s128))
+ ; CHECK-NEXT: early-clobber $x1, dead $q2, renamable $q0 = LDPQpre renamable $x1, 2 :: (load (s128))
; CHECK-NEXT: STRQui renamable $q0, renamable $x1, 0 :: (store (s128))
; CHECK-NEXT: RET undef $lr
early-clobber renamable $x1, renamable $q0 = LDRQpre killed renamable $x1, 32 :: (load (s128))
diff --git a/llvm/test/CodeGen/AArch64/neon-dotreduce.ll b/llvm/test/CodeGen/AArch64/neon-dotreduce.ll
index 74048b8bee3329f..706aa4ad1b46653 100644
--- a/llvm/test/CodeGen/AArch64/neon-dotreduce.ll
+++ b/llvm/test/CodeGen/AArch64/neon-dotreduce.ll
@@ -1473,11 +1473,9 @@ define i32 @test_udot_v32i8(ptr nocapture readonly %a, ptr nocapture readonly %b
; CHECK-LABEL: test_udot_v32i8:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: ldr q1, [x0, #16]
-; CHECK-NEXT: ldr q2, [x1, #16]
-; CHECK-NEXT: udot v0.4s, v2.16b, v1.16b
-; CHECK-NEXT: ldr q1, [x0]
-; CHECK-NEXT: ldr q2, [x1]
+; CHECK-NEXT: ldp q1, q3, [x0]
+; CHECK-NEXT: ldp q2, q4, [x1]
+; CHECK-NEXT: udot v0.4s, v4.16b, v3.16b
; CHECK-NEXT: udot v0.4s, v2.16b, v1.16b
; CHECK-NEXT: addv s0, v0.4s
; CHECK-NEXT: fmov w8, s0
@@ -1499,9 +1497,8 @@ define i32 @test_udot_v32i8_nomla(ptr nocapture readonly %a1) {
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: movi v0.16b, #1
; CHECK-NEXT: movi v1.2d, #0000000000000000
-; CHECK-NEXT: ldr q2, [x0, #16]
-; CHECK-NEXT: udot v1.4s, v2.16b, v0.16b
-; CHECK-NEXT: ldr q2, [x0]
+; CHECK-NEXT: ldp q2, q3, [x0]
+; CHECK-NEXT: udot v1.4s, v3.16b, v0.16b
; CHECK-NEXT: udot v1.4s, v2.16b, v0.16b
; CHECK-NEXT: addv s0, v1.4s
; CHECK-NEXT: fmov w0, s0
@@ -1516,11 +1513,9 @@ define i32 @test_sdot_v32i8(ptr nocapture readonly %a, ptr nocapture readonly %b
; CHECK-LABEL: test_sdot_v32i8:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: ldr q1, [x0, #16]
-; CHECK-NEXT: ldr q2, [x1, #16]
-; CHECK-NEXT: sdot v0.4s, v2.16b, v1.16b
-; CHECK-NEXT: ldr q1, [x0]
-; CHECK-NEXT: ldr q2, [x1]
+; CHECK-NEXT: ldp q1, q3, [x0]
+; CHECK-NEXT: ldp q2, q4, [x1]
+; CHECK-NEXT: sdot v0.4s, v4.16b, v3.16b
; CHECK-NEXT: sdot v0.4s, v2.16b, v1.16b
; CHECK-NEXT: addv s0, v0.4s
; CHECK-NEXT: fmov w8, s0
@@ -2190,11 +2185,9 @@ define i32 @test_udot_v48i8(ptr nocapture readonly %a, ptr nocapture readonly %b
; CHECK-NEXT: ldr q1, [x0, #32]
; CHECK-NEXT: ldr q2, [x1, #32]
; CHECK-NEXT: udot v0.4s, v2.16b, v1.16b
-; CHECK-NEXT: ldr q1, [x0]
-; CHECK-NEXT: ldr q2, [x1]
-; CHECK-NEXT: udot v0.4s, v2.16b, v1.16b
-; CHECK-NEXT: ldr q1, [x0, #16]
-; CHECK-NEXT: ldr q2, [x1, #16]
+; CHECK-NEXT: ldp q3, q1, [x0]
+; CHECK-NEXT: ldp q4, q2, [x1]
+; CHECK-NEXT: udot v0.4s, v4.16b, v3.16b
; CHECK-NEXT: udot v0.4s, v2.16b, v1.16b
; CHECK-NEXT: addv s0, v0.4s
; CHECK-NEXT: fmov w8, s0
@@ -2218,9 +2211,8 @@ define i32 @test_udot_v48i8_nomla(ptr nocapture readonly %a1) {
; CHECK-NEXT: movi v1.2d, #0000000000000000
; CHECK-NEXT: ldr q2, [x0, #32]
; CHECK-NEXT: udot v1.4s, v2.16b, v0.16b
-; CHECK-NEXT: ldr q2, [x0]
-; CHECK-NEXT: udot v1.4s, v2.16b, v0.16b
-; CHECK-NEXT: ldr q2, [x0, #16]
+; CHECK-NEXT: ldp q3, q2, [x0]
+; CHECK-NEXT: udot v1.4s, v3.16b, v0.16b
; CHECK-NEXT: udot v1.4s, v2.16b, v0.16b
; CHECK-NEXT: addv s0, v1.4s
; CHECK-NEXT: fmov w0, s0
@@ -2238,11 +2230,9 @@ define i32 @test_sdot_v48i8(ptr nocapture readonly %a, ptr nocapture readonly %b
; CHECK-NEXT: ldr q1, [x0, #32]
; CHECK-NEXT: ldr q2, [x1, #32]
; CHECK-NEXT: sdot v0.4s, v2.16b, v1.16b
-; CHECK-NEXT: ldr q1, [x0]
-; CHECK-NEXT: ldr q2, [x1]
-; CHECK-NEXT: sdot v0.4s, v2.16b, v1.16b
-; CHECK-NEXT: ldr q1, [x0, #16]
-; CHECK-NEXT: ldr q2, [x1, #16]
+; CHECK-NEXT: ldp q3, q1, [x0]
+; CHECK-NEXT: ldp q4, q2, [x1]
+; CHECK-NEXT: sdot v0.4s, v4.16b, v3.16b
; CHECK-NEXT: sdot v0.4s, v2.16b, v1.16b
; CHECK-NEXT: addv s0, v0.4s
; CHECK-NEXT: fmov w8, s0
diff --git a/llvm/test/CodeGen/AArch64/stp-opt-with-renaming.mir b/llvm/test/CodeGen/AArch64/stp-opt-with-renaming.mir
index f1ca71680062b4d..6168a02b3d95747 100644
--- a/llvm/test/CodeGen/AArch64/stp-opt-with-renaming.mir
+++ b/llvm/test/CodeGen/AArch64/stp-opt-with-renaming.mir
@@ -327,8 +327,7 @@ body: |
# CHECK: renamable $q7 = FADDv2f64 renamable $q7, renamable $q7, implicit $fpcr
# CHECK-NEXT: STRQui killed renamable $q7, renamable $x0, 100 :: (store (s128), align 4)
-# CHECK-NEXT: $q7 = LDRQui $x0, 0 :: (load (s128))
-# CHECK-NEXT: renamable $q9 = LDRQui $x0, 1 :: (load (s128))
+# CHECK-NEXT: $q7, renamable $q9 = LDPQi $x0, 0 :: (load (s128))
# CHECK-NEXT: STPQi killed renamable $q9, killed $q7, renamable $x0, 10 :: (store (s128), align 4)
# CHECK-NEXT: RET undef $lr
@@ -546,3 +545,158 @@ body: |
RET undef $lr
...
+
+# During ISel, the order of load/store pairs can be optimized and changed
+# so that only a single register is used. Due to this register reuse, LDP/STPs
+# are not generated. These tests check that LDP/STPs will be generated after
+# register renaming is attempted.
+
+---
+#
+# CHECK-LABEL: name: ldst32
+# CHECK: renamable $q0, $q1 = LDPQi $sp, 0 :: (load (s128)), (load (s128), align 32)
+# CHECK-NEXT: STPQi killed renamable $q0, killed $q1, $sp, 2 :: (store (s128), align 32)
+# CHECK-NEXT: RET undef $lr
+#
+name: ldst32
+alignment: 4
+tracksRegLiveness: true
+frameInfo:
+ maxAlignment: 1
+ maxCallFrameSize: 0
+machineFunctionInfo:
+ hasRedZone: false
+body: |
+ bb.0.entry:
+ renamable $q0 = LDRQui $sp, 1 :: (load 16)
+ STRQui killed renamable $q0, $sp, 3 :: (store 16, basealign 32)
+ renamable $q0 = LDRQui $sp, 0 :: (load 16, align 32)
+ STRQui killed renamable $q0, $sp, 2 :: (store 16, align 32)
+ RET undef $lr
+
+...
+---
+#
+# CHECK-LABEL: name: ldst64
+# CHECK: renamable $q0, $q1 = LDPQi $sp, 0 :: (load (s128)), (load (s128), align 64)
+# CHECK-NEXT: STPQi killed renamable $q0, killed $q1, $sp, 2 :: (store (s128), align 64)
+# CHECK-NEXT: RET undef $lr
+#
+name: ldst64
+alignment: 4
+tracksRegLiveness: true
+frameInfo:
+ maxAlignment: 1
+ maxCallFrameSize: 0
+machineFunctionInfo:
+ hasRedZone: false
+body: |
+ bb.0.entry:
+ renamable $q0 = LDRQui $sp, 1 :: (load 16)
+ STRQui killed renamable $q0, $sp, 3 :: (store 16, basealign 64)
+ renamable $q0 = LDRQui $sp, 0 :: (load 16, align 64)
+ STRQui killed renamable $q0, $sp, 2 :: (store 16, align 64)
+ RET undef $lr
+
+...
+---
+#
+# CHECK-LABEL: name: ldst128
+# CHECK: renamable $q0, $q1 = LDPQi $sp, 0 :: (load (s128)), (load (s128), align 128)
+# CHECK-NEXT: STPQi killed renamable $q0, killed $q1, $sp, 2 :: (store (s128), align 128)
+# CHECK-NEXT: RET undef $lr
+#
+name: ldst128
+alignment: 4
+tracksRegLiveness: true
+frameInfo:
+ maxAlignment: 1
+ maxCallFrameSize: 0
+machineFunctionInfo:
+ hasRedZone: false
+body: |
+ bb.0.entry:
+ renamable $q0 = LDRQui $sp, 1 :: (load 16)
+ STRQui killed renamable $q0, $sp, 3 :: (store 16, basealign 128)
+ renamable $q0 = LDRQui $sp, 0 :: (load 16, align 128)
+ STRQui killed renamable $q0, $sp, 2 :: (store 16, align 128)
+ RET undef $lr
+
+...
+---
+#
+# CHECK-LABEL: name: ldst-no-reg-available
+# CHECK: liveins: $q1, $q2, $q3, $q4, $q5, $q6, $q7, $q8, $q9, $q10
+# CHECK: renamable $q0 = LDRQui $sp, 1 :: (load (s128))
+# CHECK-NEXT: STRQui killed renamable $q0, $sp, 3 :: (store (s128), align 32)
+# CHECK-NEXT: renamable $q0 = LDRQui $sp, 0 :: (load (s128), align 32)
+# CHECK-NEXT: STRQui killed renamable $q0, $sp, 2 :: (store (s128), align 32)
+# CHECK-NEXT: RET undef $lr
+#
+name: ldst-no-reg-available
+alignment: 4
+tracksRegLiveness: true
+frameInfo:
+ maxAlignment: 1
+ maxCallFrameSize: 0
+machineFunctionInfo:
+ hasRedZone: false
+body: |
+ bb.0.entry:
+ liveins: $q1, $q2, $q3, $q4, $q5, $q6, $q7, $q8, $q9, $q10
+ renamable $q0 = LDRQui $sp, 1 :: (load 16)
+ STRQui killed renamable $q0, $sp, 3 :: (store 16, basealign 32)
+ renamable $q0 = LDRQui $sp, 0 :: (load 16, align 32)
+ STRQui killed renamable $q0, $sp, 2 :: (store 16, align 32)
+ RET undef $lr
+
+...
+---
+#
+# CHECK-LABEL: name: ldst-basereg-modified
+# CHECK: renamable $q0, $q1 = LDPQi $sp, 0 :: (load (s128)), (load (s128), align 32)
+# CHECK-NEXT: STRQui killed $q1, $sp, 3 :: (store (s128), align 32)
+# CHECK-NEXT: $sp = ADDXri $sp, 16, 0
+# CHECK-NEXT: STRQui killed renamable $q0, $sp, 2 :: (store (s128), align 32)
+# CHECK-NEXT: RET undef $lr
+#
+name: ldst-basereg-modified
+alignment: 4
+tracksRegLiveness: true
+frameInfo:
+ maxAlignment: 1
+ maxCallFrameSize: 0
+machineFunctionInfo:
+ hasRedZone: false
+body: |
+ bb.0.entry:
+ renamable $q0 = LDRQui $sp, 1 :: (load 16)
+ STRQui killed renamable $q0, $sp, 3 :: (store 16, basealign 32)
+ renamable $q0 = LDRQui $sp, 0 :: (load 16, align 32)
+ $sp = ADDXri $sp, 16, 0
+ STRQui killed renamable $q0, $sp, 2 :: (store 16, align 32)
+ RET undef $lr
+
+...
+---
+#
+# CHECK-LABEL: name: ldr-dest-reg-implicit-killed
+# CHECK: renamable $q0, $q1 = LDPQi $sp, 0 :: (load (s128)), (load (s128), align 32)
+# CHECK-NEXT: STPSi $s1, renamable $s0, $sp, 9 :: (store (s32))
+# CHECK-NEXT: RET undef $lr
+#
+name: ldr-dest-reg-implicit-killed
+alignment: 4
+tracksRegLiveness: true
+frameInfo:
+ maxAlignment: 1
+ maxCallFrameSize: 0
+machineFunctionInfo:
+ hasRedZone: false
+body: |
+ bb.0.entry:
+ renamable $q0 = LDRQui $sp, 1 :: (load 16)
+ STRSui renamable $s0, $sp, 9, implicit killed $q0 :: (store (s32))
+ renamable $q0 = LDRQui $sp, 0 :: (load 16, align 32)
+ STRSui renamable $s0, $sp, 10, implicit killed $q0 :: (store (s32))
+ RET undef $lr
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp128.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp128.ll
index 9d9d4a64a5d1f58..115f722986b5c60 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp128.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp128.ll
@@ -39,8 +39,7 @@ define void @fcvt_v4f64_v4f128(ptr %a, ptr %b) vscale_range(2,0) #0 {
; CHECK-NEXT: bl __extenddftf2
; CHECK-NEXT: ldr q1, [sp] // 16-byte Folded Reload
; CHECK-NEXT: stp q1, q0, [x19]
-; CHECK-NEXT: ldr q1, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT: ldr q0, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT: ldp q1, q0, [sp, #16] // 32-byte Folded Reload
; CHECK-NEXT: stp q0, q1, [x19, #32]
; CHECK-NEXT: addvl sp, sp, #2
; CHECK-NEXT: add sp, sp, #48
@@ -61,19 +60,15 @@ define void @fcvt_v4f128_v4f64(ptr %a, ptr %b) vscale_range(2,0) #0 {
; CHECK-NEXT: stp x30, x19, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT: sub sp, sp, #128
; CHECK-NEXT: addvl sp, sp, #-2
-; CHECK-NEXT: ldr q1, [x0, #64]
-; CHECK-NEXT: ldr q0, [x0, #80]
+; CHECK-NEXT: ldp q1, q0, [x0, #64]
; CHECK-NEXT: mov x19, x1
; CHECK-NEXT: stp q0, q1, [sp, #96] // 32-byte Folded Spill
-; CHECK-NEXT: ldr q1, [x0, #96]
-; CHECK-NEXT: ldr q0, [x0, #112]
+; CHECK-NEXT: ldp q1, q0, [x0, #96]
; CHECK-NEXT: stp q0, q1, [sp, #64] // 32-byte Folded Spill
-; CHECK-NEXT: ldr q1, [x0]
-; CHECK-NEXT: ldr q0, [x0, #16]
+; CHECK-NEXT: ldp q1, q0, [x0]
; CHECK-NEXT: stp q0, q1, [sp, #32] // 32-byte Folded Spill
-; CHECK-NEXT: ldr q0, [x0, #32]
-; CHECK-NEXT: str q0, [sp, #16] // 16-byte Folded Spill
-; CHECK-NEXT: ldr q0, [x0, #48]
+; CHECK-NEXT: ldp q1, q0, [x0, #32]
+; CHECK-NEXT: str q1, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT: bl __trunctfdf2
; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
; CHECK-NEXT: str q0, [sp] // 16-byte Folded Spill
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-add.ll b/llvm/test/CodeGen/AArch64/vecreduce-add.ll
index f788cd71f19465b..b24967d5a130ed1 100644
--- a/llvm/test/CodeGen/AArch64/vecreduce-add.ll
+++ b/llvm/test/CodeGen/AArch64/vecreduce-add.ll
@@ -3558,11 +3558,9 @@ define i32 @test_udot_v48i8(ptr %p1, ptr %p2) {
; CHECK-SD-DOT-NEXT: ldr q1, [x0, #32]
; CHECK-SD-DOT-NEXT: ldr q2, [x1, #32]
; CHECK-SD-DOT-NEXT: udot v0.4s, v2.16b, v1.16b
-; CHECK-SD-DOT-NEXT: ldr q1, [x0]
-; CHECK-SD-DOT-NEXT: ldr q2, [x1]
-; CHECK-SD-DOT-NEXT: udot v0.4s, v2.16b, v1.16b
-; CHECK-SD-DOT-NEXT: ldr q1, [x0, #16]
-; CHECK-SD-DOT-NEXT: ldr q2, [x1, #16]
+; CHECK-SD-DOT-NEXT: ldp q3, q1, [x0]
+; CHECK-SD-DOT-NEXT: ldp q4, q2, [x1]
+; CHECK-SD-DOT-NEXT: udot v0.4s, v4.16b, v3.16b
; CHECK-SD-DOT-NEXT: udot v0.4s, v2.16b, v1.16b
; CHECK-SD-DOT-NEXT: addv s0, v0.4s
; CHECK-SD-DOT-NEXT: fmov w0, s0
@@ -4052,11 +4050,9 @@ define i32 @test_sdot_v48i8(ptr %p1, ptr %p2) {
; CHECK-SD-DOT-NEXT: ldr q1, [x0, #32]
; CHECK-SD-DOT-NEXT: ldr q2, [x1, #32]
; CHECK-SD-DOT-NEXT: sdot v0.4s, v2.16b, v1.16b
-; CHECK-SD-DOT-NEXT: ldr q1, [x0]
-; CHECK-SD-DOT-NEXT: ldr q2, [x1]
-; CHECK-SD-DOT-NEXT: sdot v0.4s, v2.16b, v1.16b
-; CHECK-SD-DOT-NEXT: ldr q1, [x0, #16]
-; CHECK-SD-DOT-NEXT: ldr q2, [x1, #16]
+; CHECK-SD-DOT-NEXT: ldp q3, q1, [x0]
+; CHECK-SD-DOT-NEXT: ldp q4, q2, [x1]
+; CHECK-SD-DOT-NEXT: sdot v0.4s, v4.16b, v3.16b
; CHECK-SD-DOT-NEXT: sdot v0.4s, v2.16b, v1.16b
; CHECK-SD-DOT-NEXT: addv s0, v0.4s
; CHECK-SD-DOT-NEXT: fmov w0, s0
More information about the llvm-commits
mailing list