[llvm] [AArch64] Fold a constant offset into the frame index ADDXri (PR #218561)
Andrew Gaul via llvm-commits
llvm-commits at lists.llvm.org
Sat Sep 26 17:06:43 PDT 2026
https://github.com/gaul updated https://github.com/llvm/llvm-project/pull/218561
>From 17c82161a0b3e13d7455a10f2aaece9a0b68e1d3 Mon Sep 17 00:00:00 2001
From: Andrew Gaul <andrew at gaul.org>
Date: Mon, 24 Aug 2026 17:16:37 -0700
Subject: [PATCH 1/3] [AArch64] Add test coverage for constant offsets off a
frame index
Pre-commit test showing the current codegen for address computations
that add a constant offset to an alloca. An escaping interior pointer
currently materialises the frame index and the offset separately, and
where the same frame index is live at two offsets the materialised base
also occupies a callee-saved register for the length of the sequence.
Co-Authored-By: Claude Opus 5 (1M context) <noreply at anthropic.com>
---
.../CodeGen/AArch64/frameindex-offset-fold.ll | 133 ++++++++++++++++++
1 file changed, 133 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/frameindex-offset-fold.ll
diff --git a/llvm/test/CodeGen/AArch64/frameindex-offset-fold.ll b/llvm/test/CodeGen/AArch64/frameindex-offset-fold.ll
new file mode 100644
index 0000000000000..9db4d32e66e80
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/frameindex-offset-fold.ll
@@ -0,0 +1,133 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64 -verify-machineinstrs < %s | FileCheck %s
+
+; Address computations that add a constant offset to an alloca, in the shapes
+; that decide whether the offset can ride along on the frame index itself: a
+; small offset, one at the top of the unshifted ADD immediate, one past it, and
+; cases where the same frame index is live at more than one offset.
+
+declare void @use(ptr)
+
+define void @alloca_offset_8() {
+; CHECK-LABEL: alloca_offset_8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sub sp, sp, #80
+; CHECK-NEXT: str x30, [sp, #64] // 8-byte Spill
+; CHECK-NEXT: .cfi_def_cfa_offset 80
+; CHECK-NEXT: .cfi_offset w30, -16
+; CHECK-NEXT: mov x8, sp
+; CHECK-NEXT: add x0, x8, #8
+; CHECK-NEXT: bl use
+; CHECK-NEXT: ldr x30, [sp, #64] // 8-byte Reload
+; CHECK-NEXT: add sp, sp, #80
+; CHECK-NEXT: ret
+ %a = alloca [64 x i8], align 8
+ %p = getelementptr inbounds i8, ptr %a, i64 8
+ call void @use(ptr %p)
+ ret void
+}
+
+define void @alloca_offset_4095() {
+; CHECK-LABEL: alloca_offset_4095:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEXT: sub sp, sp, #2, lsl #12 // =8192
+; CHECK-NEXT: .cfi_def_cfa_offset 8208
+; CHECK-NEXT: .cfi_offset w30, -8
+; CHECK-NEXT: .cfi_offset w29, -16
+; CHECK-NEXT: mov x8, sp
+; CHECK-NEXT: add x0, x8, #4095
+; CHECK-NEXT: bl use
+; CHECK-NEXT: add sp, sp, #2, lsl #12 // =8192
+; CHECK-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT: ret
+ %a = alloca [8192 x i8], align 8
+ %p = getelementptr inbounds i8, ptr %a, i64 4095
+ call void @use(ptr %p)
+ ret void
+}
+
+; 4096 only encodes as an ADD immediate with "lsl #12".
+define void @alloca_offset_4096() {
+; CHECK-LABEL: alloca_offset_4096:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEXT: sub sp, sp, #2, lsl #12 // =8192
+; CHECK-NEXT: .cfi_def_cfa_offset 8208
+; CHECK-NEXT: .cfi_offset w30, -8
+; CHECK-NEXT: .cfi_offset w29, -16
+; CHECK-NEXT: mov x8, sp
+; CHECK-NEXT: add x0, x8, #1, lsl #12 // =4096
+; CHECK-NEXT: bl use
+; CHECK-NEXT: add sp, sp, #2, lsl #12 // =8192
+; CHECK-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT: ret
+ %a = alloca [8192 x i8], align 8
+ %p = getelementptr inbounds i8, ptr %a, i64 4096
+ call void @use(ptr %p)
+ ret void
+}
+
+; Two interior pointers off one alloca.
+define void @alloca_two_interior_ptrs() {
+; CHECK-LABEL: alloca_two_interior_ptrs:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sub sp, sp, #80
+; CHECK-NEXT: stp x30, x19, [sp, #64] // 16-byte Folded Spill
+; CHECK-NEXT: .cfi_def_cfa_offset 80
+; CHECK-NEXT: .cfi_offset w19, -8
+; CHECK-NEXT: .cfi_offset w30, -16
+; CHECK-NEXT: mov x19, sp
+; CHECK-NEXT: add x0, x19, #8
+; CHECK-NEXT: bl use
+; CHECK-NEXT: add x0, x19, #16
+; CHECK-NEXT: bl use
+; CHECK-NEXT: ldp x30, x19, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEXT: add sp, sp, #80
+; CHECK-NEXT: ret
+ %a = alloca [64 x i8], align 8
+ %p = getelementptr inbounds i8, ptr %a, i64 8
+ %q = getelementptr inbounds i8, ptr %a, i64 16
+ call void @use(ptr %p)
+ call void @use(ptr %q)
+ ret void
+}
+
+; The alloca is used both at offset 0 and at an interior offset.
+define void @alloca_base_and_interior() {
+; CHECK-LABEL: alloca_base_and_interior:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sub sp, sp, #80
+; CHECK-NEXT: stp x30, x19, [sp, #64] // 16-byte Folded Spill
+; CHECK-NEXT: .cfi_def_cfa_offset 80
+; CHECK-NEXT: .cfi_offset w19, -8
+; CHECK-NEXT: .cfi_offset w30, -16
+; CHECK-NEXT: mov x0, sp
+; CHECK-NEXT: mov x19, sp
+; CHECK-NEXT: bl use
+; CHECK-NEXT: add x0, x19, #8
+; CHECK-NEXT: bl use
+; CHECK-NEXT: ldp x30, x19, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEXT: add sp, sp, #80
+; CHECK-NEXT: ret
+ %a = alloca [64 x i8], align 8
+ %p = getelementptr inbounds i8, ptr %a, i64 8
+ call void @use(ptr %a)
+ call void @use(ptr %p)
+ ret void
+}
+
+; The offset is consumed by an addressing mode rather than escaping.
+define i64 @alloca_offset_load() {
+; CHECK-LABEL: alloca_offset_load:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sub sp, sp, #64
+; CHECK-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NEXT: ldr x0, [sp, #8]
+; CHECK-NEXT: add sp, sp, #64
+; CHECK-NEXT: ret
+ %a = alloca [64 x i8], align 8
+ %p = getelementptr inbounds i8, ptr %a, i64 8
+ %v = load i64, ptr %p
+ ret i64 %v
+}
>From 07efe3c96f5df3798a3161ebbaa1d664d310cf2c Mon Sep 17 00:00:00 2001
From: Andrew Gaul <andrew at gaul.org>
Date: Mon, 24 Aug 2026 17:17:12 -0700
Subject: [PATCH 2/3] [AArch64] Fold a constant offset into the frame index
ADDXri
A bare ISD::FrameIndex is selected to "ADDXri <FI>, 0" with a hardcoded
zero immediate, so an escaping interior pointer into an alloca costs two
instructions:
%0:gpr64sp = ADDXri %stack.1.b, 0, 0
%1:gpr64sp = nuw ADDXri killed %0, 8, 0
mov x8, sp
add x0, x8, #8
Nothing downstream cleans this up: AArch64MIPeepholeOpt coalesces ADDXrr
but not ADDXri chains. Memory operands already avoid it, because
SelectAddrModeIndexed folds FrameIndex + offset into the addressing mode,
so the same address used by a load is a single "ldr x0, [sp, #8]".
Select (add (FrameIndex FI), C) directly to ADDXri <FI>, C instead.
rewriteAArch64FrameIndex() already adds the immediate an ADDXri carries
to the resolved frame offset and hands the total to emitFrameOffset(),
so the offset costs nothing there. Beyond saving the second add, this
frequently saves a register too: several interior pointers off one frame
index no longer keep a materialised base alive, which in some of the
tests below drops a callee-saved register and its spill/reload.
The fold is restricted to offsets that fit the unshifted immediate.
rewriteAArch64FrameIndex() reads the immediate operand but ignores the
shift operand, so folding a shifted offset would silently drop the shift;
a larger offset keeps its own add, as before.
This makes a nonzero immediate on a frame index ADDXri reachable for the
first time, which canFoldIntoCSel() did not expect. It folds "add x, 1"
into a CSINC by reading the add's register operand, guarded only by that
immediate being 1, a combination a frame index ADDXri could not produce
while it was always built with a hardcoded zero. Check for a register
before reading one. There is no base register to build a CSINC from, so
the select stays a plain CSEL.
Measured with llc -O3 over the 5,244 translation units of an LLVM, clang
and clang-tools-extra build (39.7M instructions of AArch64 code), this
removes 100,123 instructions, 0.25%. 3,581 translation units change:
3,115 shrink and 265 grow, by 3,295 instructions in total, where the
shorter address sequences shift register allocation or block layout.
Fewer interior pointers need a live base register, so the stores and
loads of spilled values fall too: 12,831 fewer stur and 9,643 fewer ldur.
Found via armlint.
Co-Authored-By: Claude Opus 5 (1M context) <noreply at anthropic.com>
Co-Authored-By: Claude Fable 5.1 <noreply at anthropic.com>
Claude-Session: https://claude.ai/code/session_01RN4gpjgX8QWXRuQoBzYace
---
.../Target/AArch64/AArch64ISelDAGToDAG.cpp | 50 +++++
llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 6 +-
.../Atomics/aarch64-atomic-load-rcpc_immo.ll | 2 -
.../AArch64/GlobalISel/aapcs_vararg_frame.ll | 10 +-
.../GlobalISel/split-wide-shifts-multiway.ll | 16 +-
.../test/CodeGen/AArch64/arm64-abi-varargs.ll | 9 +-
llvm/test/CodeGen/AArch64/arm64-mte.ll | 9 +-
.../CodeGen/AArch64/arm64-variadic-aapcs.ll | 25 +--
.../AArch64/bitinsert-bitextract-fp.ll | 14 +-
.../CodeGen/AArch64/bitinsert-bitextract.ll | 84 ++++-----
llvm/test/CodeGen/AArch64/div-i256.ll | 95 +++++-----
.../AArch64/early-ifcvt-frame-index.mir | 49 +++++
llvm/test/CodeGen/AArch64/fcvt-i256.ll | 176 ++++++++----------
.../CodeGen/AArch64/frameindex-offset-fold.ll | 24 +--
.../AArch64/named-vector-shuffles-neon.ll | 3 +-
.../AArch64/named-vector-shuffles-sve.ll | 2 +-
.../AArch64/preserve_nonecc_varargs_aapcs.ll | 8 +-
llvm/test/CodeGen/AArch64/shift-i256.ll | 33 ++--
llvm/test/CodeGen/AArch64/soft-float-abi.ll | 3 +-
.../sve-extract-fixed-from-scalable-vector.ll | 22 +--
.../AArch64/sve-masked-ldst-alias-analysis.ll | 3 +-
llvm/test/CodeGen/AArch64/trampoline.ll | 9 +-
llvm/test/CodeGen/AArch64/vararg.ll | 40 ++--
...lar-shift-by-byte-multiple-legalization.ll | 16 +-
.../AArch64/wide-scalar-shift-legalization.ll | 3 +-
25 files changed, 371 insertions(+), 340 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/early-ifcvt-frame-index.mir
diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index 48c0e37151b9c..30eaca99b4a52 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -457,6 +457,8 @@ class AArch64DAGToDAGISel : public SelectionDAGISel {
bool tryFoldCselToFMaxMin(SDNode *N);
+ bool tryFoldFrameIndexOffset(SDNode *N);
+
// Include the pieces autogenerated from the target description.
#include "AArch64GenDAGISel.inc"
@@ -5191,6 +5193,49 @@ static SDValue getZeroRegister(SelectionDAG &DAG, SDLoc DL, EVT VT) {
VT == MVT::i32 ? AArch64::WZR : AArch64::XZR, VT);
}
+/// Fold a constant offset into the ADDXri that materialises a frame index:
+/// (add (FrameIndex FI), C) -> ADDXri <FI>, C
+///
+/// A bare frame index selects to "ADDXri <FI>, 0" (see the ISD::FrameIndex
+/// case below), and PrologEpilogInserter adds whatever immediate that
+/// instruction already carries to the resolved frame offset, so an interior
+/// pointer costs no more than the base pointer does. Selecting the add
+/// separately instead costs a second ADDXri, and frequently an extra live
+/// register too, since several interior pointers off one frame index all keep
+/// the materialised base alive.
+bool AArch64DAGToDAGISel::tryFoldFrameIndexOffset(SDNode *N) {
+ if (N->getValueType(0) != MVT::i64)
+ return false;
+
+ SDValue Base = N->getOperand(0);
+ if (Base.getOpcode() != ISD::FrameIndex)
+ return false;
+
+ auto *Off = dyn_cast<ConstantSDNode>(N->getOperand(1));
+ if (!Off)
+ return false;
+
+ // rewriteAArch64FrameIndex() reads the immediate operand of the ADDXri it
+ // resolves but ignores the shift operand, so only fold offsets that the
+ // unshifted immediate can hold. A larger offset keeps its own add, which
+ // is what would have been emitted anyway.
+ uint64_t Offset = Off->getZExtValue();
+ if (Offset > 0xFFF)
+ return false;
+
+ SDLoc DL(N);
+ const TargetLowering *TLI = getTargetLowering();
+ SDValue TFI =
+ CurDAG->getTargetFrameIndex(cast<FrameIndexSDNode>(Base)->getIndex(),
+ TLI->getPointerTy(CurDAG->getDataLayout()));
+ SDValue Ops[] = {
+ TFI, CurDAG->getTargetConstant(Offset, DL, MVT::i32),
+ CurDAG->getTargetConstant(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0),
+ DL, MVT::i32)};
+ CurDAG->SelectNodeTo(N, AArch64::ADDXri, MVT::i64, Ops);
+ return true;
+}
+
void AArch64DAGToDAGISel::Select(SDNode *Node) {
// If we have a custom node, we already have selected!
if (Node->isMachineOpcode()) {
@@ -5258,6 +5303,11 @@ void AArch64DAGToDAGISel::Select(SDNode *Node) {
return;
break;
+ case ISD::ADD:
+ if (tryFoldFrameIndexOffset(Node))
+ return;
+ break;
+
case ISD::EXTRACT_SUBVECTOR: {
if (trySelectCastScalableToFixedLengthVector(Node))
return;
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index 885b27677f786..7052137c07a65 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -1189,7 +1189,11 @@ static unsigned canFoldIntoCSel(const MachineRegisterInfo &MRI, unsigned VReg,
case AArch64::ADDXri:
case AArch64::ADDWri:
// add x, 1 -> csinc.
- if (!DefMI->getOperand(2).isImm() || DefMI->getOperand(2).getImm() != 1 ||
+ // An ADDXri that materialises a frame index holds a frame index rather
+ // than a register in operand 1, and it can carry any immediate the
+ // addressed object needs, so check for a register before reading one.
+ if (!DefMI->getOperand(1).isReg() || !DefMI->getOperand(2).isImm() ||
+ DefMI->getOperand(2).getImm() != 1 ||
DefMI->getOperand(3).getImm() != 0)
return 0;
SrcReg = DefMI->getOperand(1).getReg();
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc_immo.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc_immo.ll
index f29ae86b73842..58aebd691a30f 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc_immo.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc_immo.ll
@@ -958,7 +958,6 @@ define i32 @load_atomic_i32_from_gep() {
;
; SDAG-AVOIDLDAPUR-LABEL: load_atomic_i32_from_gep:
; SDAG-AVOIDLDAPUR: bl init
-; SDAG-AVOIDLDAPUR: add x8, x19, #4
; SDAG-AVOIDLDAPUR: ldapr w0, [x8]
;
; SDAG-NOAVOIDLDAPUR-LABEL: load_atomic_i32_from_gep:
@@ -978,7 +977,6 @@ define i64 @load_atomic_i64_from_gep() {
;
; SDAG-AVOIDLDAPUR-LABEL: load_atomic_i64_from_gep:
; SDAG-AVOIDLDAPUR: bl init
-; SDAG-AVOIDLDAPUR: add x8, x19, #8
; SDAG-AVOIDLDAPUR: ldapr x0, [x8]
;
; SDAG-NOAVOIDLDAPUR-LABEL: load_atomic_i64_from_gep:
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/aapcs_vararg_frame.ll b/llvm/test/CodeGen/AArch64/GlobalISel/aapcs_vararg_frame.ll
index 6ea520ddcd6cd..d3732e5c71bc5 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/aapcs_vararg_frame.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/aapcs_vararg_frame.ll
@@ -31,15 +31,13 @@ define void @va_used(i32 %count, half %f, ...) nounwind {
; CHECK: // %bb.0:
; CHECK-NEXT: sub sp, sp, #208
; CHECK-NEXT: mov x8, #-56 // =0xffffffffffffffc8
-; CHECK-NEXT: mov x9, sp
-; CHECK-NEXT: add x10, sp, #120
-; CHECK-NEXT: movk x8, #65424, lsl #32
-; CHECK-NEXT: add x9, x9, #112
+; CHECK-NEXT: add x9, sp, #112
; CHECK-NEXT: stp x2, x3, [sp, #128]
+; CHECK-NEXT: movk x8, #65424, lsl #32
+; CHECK-NEXT: str x1, [sp, #120]
; CHECK-NEXT: stp x9, x8, [sp, #192]
-; CHECK-NEXT: add x8, x10, #56
+; CHECK-NEXT: add x8, sp, #176
; CHECK-NEXT: add x9, sp, #208
-; CHECK-NEXT: str x1, [sp, #120]
; CHECK-NEXT: stp x4, x5, [sp, #144]
; CHECK-NEXT: stp x6, x7, [sp, #160]
; CHECK-NEXT: stp q1, q2, [sp]
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/split-wide-shifts-multiway.ll b/llvm/test/CodeGen/AArch64/GlobalISel/split-wide-shifts-multiway.ll
index ce1dae3cdfeff..8435323c7bda2 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/split-wide-shifts-multiway.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/split-wide-shifts-multiway.ll
@@ -16,16 +16,15 @@ define void @test_shl_i512(ptr %result, ptr %input, i32 %shift) {
; SDAG-NEXT: ldr q3, [x1, #32]
; SDAG-NEXT: stp x9, x8, [sp, #112]
; SDAG-NEXT: mov w8, w2
-; SDAG-NEXT: mov x9, sp
; SDAG-NEXT: lsr x10, x8, #3
-; SDAG-NEXT: add x9, x9, #64
; SDAG-NEXT: stp q0, q0, [sp]
-; SDAG-NEXT: stp q0, q0, [sp, #32]
; SDAG-NEXT: and x3, x8, #0x3f
-; SDAG-NEXT: and x10, x10, #0x38
-; SDAG-NEXT: stp q2, q3, [sp, #80]
+; SDAG-NEXT: stp q0, q0, [sp, #32]
; SDAG-NEXT: eor x3, x3, #0x3f
-; SDAG-NEXT: sub x10, x9, x10
+; SDAG-NEXT: and x9, x10, #0x38
+; SDAG-NEXT: add x10, sp, #64
+; SDAG-NEXT: stp q2, q3, [sp, #80]
+; SDAG-NEXT: sub x10, x10, x9
; SDAG-NEXT: str q1, [sp, #64]
; SDAG-NEXT: ldp x9, x11, [x10]
; SDAG-NEXT: ldp x13, x12, [x10, #16]
@@ -1109,11 +1108,10 @@ define void @test_shl_i1024(ptr %result, ptr %input, i32 %shift) {
; SDAG-NEXT: ldp x8, x9, [x1, #112]
; SDAG-NEXT: movi.2d v0, #0000000000000000
; SDAG-NEXT: ldp q1, q2, [x1]
-; SDAG-NEXT: mov x10, sp
+; SDAG-NEXT: add x10, sp, #128
; SDAG-NEXT: ldp q3, q4, [x1, #32]
-; SDAG-NEXT: add x10, x10, #128
-; SDAG-NEXT: ldp q5, q6, [x1, #64]
; SDAG-NEXT: mvn w4, w2
+; SDAG-NEXT: ldp q5, q6, [x1, #64]
; SDAG-NEXT: ldr q7, [x1, #96]
; SDAG-NEXT: stp x8, x9, [sp, #240]
; SDAG-NEXT: mov w8, w2
diff --git a/llvm/test/CodeGen/AArch64/arm64-abi-varargs.ll b/llvm/test/CodeGen/AArch64/arm64-abi-varargs.ll
index a1919444c3d48..e881dfec0c2b5 100644
--- a/llvm/test/CodeGen/AArch64/arm64-abi-varargs.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-abi-varargs.ll
@@ -14,8 +14,7 @@ define void @fn9(ptr %a1, i32 %a2, i32 %a3, i32 %a4, i32 %a5, i32 %a6, i32 %a7,
; CHECK-NEXT: stp w6, w5, [sp, #36]
; CHECK-NEXT: str w7, [sp, #32]
; CHECK-NEXT: str w8, [x0]
-; CHECK-NEXT: add x8, sp, #72
-; CHECK-NEXT: add x8, x8, #8
+; CHECK-NEXT: add x8, sp, #80
; CHECK-NEXT: ldr w9, [sp, #72]
; CHECK-NEXT: str w9, [sp, #20]
; CHECK-NEXT: ldr w9, [x8], #8
@@ -146,8 +145,7 @@ define void @foo(ptr %fmt, ...) nounwind {
; CHECK-NEXT: sub sp, sp, #48
; CHECK-NEXT: ldr w8, [sp, #48]
; CHECK-NEXT: str w8, [sp, #28]
-; CHECK-NEXT: add x8, sp, #48
-; CHECK-NEXT: add x8, x8, #23
+; CHECK-NEXT: add x8, sp, #71
; CHECK-NEXT: and x8, x8, #0xfffffffffffffff0
; CHECK-NEXT: add x9, x8, #16
; CHECK-NEXT: stp x9, x0, [sp, #32]
@@ -207,8 +205,7 @@ define void @foo2(ptr %fmt, ...) nounwind {
; CHECK-NEXT: sub sp, sp, #48
; CHECK-NEXT: ldr w8, [sp, #48]
; CHECK-NEXT: str w8, [sp, #28]
-; CHECK-NEXT: add x8, sp, #48
-; CHECK-NEXT: add x8, x8, #23
+; CHECK-NEXT: add x8, sp, #71
; CHECK-NEXT: and x8, x8, #0xfffffffffffffff0
; CHECK-NEXT: add x9, x8, #16
; CHECK-NEXT: stp x9, x0, [sp, #32]
diff --git a/llvm/test/CodeGen/AArch64/arm64-mte.ll b/llvm/test/CodeGen/AArch64/arm64-mte.ll
index d78f4eb830e10..a69800c309a7d 100644
--- a/llvm/test/CodeGen/AArch64/arm64-mte.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-mte.ll
@@ -199,8 +199,7 @@ entry:
%0 = call ptr @llvm.aarch64.ldg(ptr nonnull %arrayidx, ptr nonnull %arrayidx)
call void @llvm.lifetime.end.p0(i64 8192, ptr nonnull %s)
ret ptr %0
-; CHECK: mov [[T0:x[0-9]+]], sp
-; CHECK: add x0, [[T0]], #16
+; CHECK: add x0, sp, #16
; CHECK: ldg x0, [sp, #16]
}
@@ -224,8 +223,7 @@ entry:
%0 = call ptr @llvm.aarch64.ldg(ptr nonnull %arrayidx, ptr nonnull %arrayidx)
call void @llvm.lifetime.end.p0(i64 8192, ptr nonnull %s)
ret ptr %0
-; CHECK: mov [[T0:x[0-9]+]], sp
-; CHECK: add x0, [[T0]], #4080
+; CHECK: add x0, sp, #4080
; CHECK: ldg x0, [sp, #4080]
}
@@ -273,8 +271,7 @@ entry:
%0 = call ptr @llvm.aarch64.ldg(ptr nonnull %arrayidx, ptr nonnull %arrayidx)
call void @llvm.lifetime.end.p0(i64 8192, ptr nonnull %s)
ret ptr %0
-; CHECK: mov [[T0:x[0-9]+]], sp
-; CHECK: add x[[T1:[0-9]+]], [[T0]], #20
+; CHECK: add x[[T1:[0-9]+]], sp, #20
; CHECK-NEXT: ldg x[[T1]], [x[[T1]]]
}
diff --git a/llvm/test/CodeGen/AArch64/arm64-variadic-aapcs.ll b/llvm/test/CodeGen/AArch64/arm64-variadic-aapcs.ll
index 18b648146c470..c0e4906a33cf3 100644
--- a/llvm/test/CodeGen/AArch64/arm64-variadic-aapcs.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-variadic-aapcs.ll
@@ -22,15 +22,12 @@ define dso_local void @test_simple(i32 %n, ...) {
; ... omit middle ones ...
; CHECK-DAG: stp q6, q7, [sp, #
-; CHECK: str [[STACK_TOP]], [x[[VA_LIST]]]
+; __gr_top is [[GR_BASE]] + 56, which is the same address as __stack, so the
+; two share a register and store as a pair.
+; CHECK: stp [[STACK_TOP]], [[STACK_TOP]], [x[[VA_LIST]]]
-; CHECK: add [[GR_TOPTMP:x[0-9]+]], sp, #[[GR_BASE]]
-; CHECK: add [[GR_TOP:x[0-9]+]], [[GR_TOPTMP]], #56
-
-
-; CHECK: mov [[VR_TOPTMP:x[0-9]+]], sp
-; CHECK: add [[VR_TOP:x[0-9]+]], [[VR_TOPTMP]], #128
-; CHECK: stp [[GR_TOP]], [[VR_TOP]], [x[[VA_LIST]], #8]
+; CHECK: add [[VR_TOP:x[0-9]+]], sp, #128
+; CHECK: str [[VR_TOP]], [x[[VA_LIST]], #16]
; CHECK: mov [[GRVR:x[0-9]+]], #-56
; CHECK: movk [[GRVR]], #65408, lsl #32
@@ -57,14 +54,12 @@ define dso_local void @test_fewargs(i32 %n, i32 %n1, i32 %n2, float %m, ...) {
; ... omit middle ones ...
; CHECK-DAG: str q1, [sp]
-; CHECK: str [[STACK_TOP]], [x[[VA_LIST]]]
-
-; CHECK: add [[GR_TOPTMP:x[0-9]+]], sp, #[[GR_BASE]]
-; CHECK: add [[GR_TOP:x[0-9]+]], [[GR_TOPTMP]], #40
+; __gr_top is [[GR_BASE]] + 40, which is the same address as __stack, so the
+; two share a register and store as a pair.
+; CHECK: stp [[STACK_TOP]], [[STACK_TOP]], [x[[VA_LIST]]]
-; CHECK: mov [[VR_TOPTMP:x[0-9]+]], sp
-; CHECK: add [[VR_TOP:x[0-9]+]], [[VR_TOPTMP]], #112
-; CHECK: stp [[GR_TOP]], [[VR_TOP]], [x[[VA_LIST]], #8]
+; CHECK: add [[VR_TOP:x[0-9]+]], sp, #112
+; CHECK: str [[VR_TOP]], [x[[VA_LIST]], #16]
; CHECK: mov [[GRVR_OFFS:x[0-9]+]], #-40
; CHECK: movk [[GRVR_OFFS]], #65424, lsl #32
diff --git a/llvm/test/CodeGen/AArch64/bitinsert-bitextract-fp.ll b/llvm/test/CodeGen/AArch64/bitinsert-bitextract-fp.ll
index 2e996640ab43a..deb5087c503b3 100644
--- a/llvm/test/CodeGen/AArch64/bitinsert-bitextract-fp.ll
+++ b/llvm/test/CodeGen/AArch64/bitinsert-bitextract-fp.ll
@@ -273,23 +273,21 @@ define b231 @test_bitinsert_double_b231_var(b231 %base, double %val, i32 %off) {
; AARCH64: // %bb.0:
; AARCH64-NEXT: sub sp, sp, #128
; AARCH64-NEXT: .cfi_def_cfa_offset 128
-; AARCH64-NEXT: mov w8, w4
; AARCH64-NEXT: movi v1.2d, #0000000000000000
+; AARCH64-NEXT: mov w8, w4
; AARCH64-NEXT: mov x9, #-1 // =0xffffffffffffffff
-; AARCH64-NEXT: lsr x11, x8, #3
-; AARCH64-NEXT: mov x10, sp
+; AARCH64-NEXT: lsr x10, x8, #3
; AARCH64-NEXT: str x9, [sp, #96]
-; AARCH64-NEXT: add x10, x10, #32
+; AARCH64-NEXT: add x11, sp, #96
; AARCH64-NEXT: str xzr, [sp, #56]
; AARCH64-NEXT: and x15, x8, #0x3f
-; AARCH64-NEXT: and x9, x11, #0x18
-; AARCH64-NEXT: add x11, sp, #64
+; AARCH64-NEXT: and x9, x10, #0x18
+; AARCH64-NEXT: add x10, sp, #32
; AARCH64-NEXT: str d0, [sp, #32]
-; AARCH64-NEXT: add x11, x11, #32
; AARCH64-NEXT: stur q1, [sp, #40]
; AARCH64-NEXT: sub x10, x10, x9
-; AARCH64-NEXT: stp q1, q1, [sp]
; AARCH64-NEXT: sub x9, x11, x9
+; AARCH64-NEXT: stp q1, q1, [sp]
; AARCH64-NEXT: eor x15, x15, #0x3f
; AARCH64-NEXT: str xzr, [sp, #120]
; AARCH64-NEXT: stur q1, [sp, #104]
diff --git a/llvm/test/CodeGen/AArch64/bitinsert-bitextract.ll b/llvm/test/CodeGen/AArch64/bitinsert-bitextract.ll
index aa1664a3d84a9..755a5eec8d39c 100644
--- a/llvm/test/CodeGen/AArch64/bitinsert-bitextract.ll
+++ b/llvm/test/CodeGen/AArch64/bitinsert-bitextract.ll
@@ -229,66 +229,64 @@ define b231 @test_bitinsert_b231_var(b231 %base, i32 %val, i32 %off) {
; AARCH64-NEXT: .cfi_def_cfa_offset 128
; AARCH64-NEXT: mov w8, w4
; AARCH64-NEXT: movi v0.2d, #0000000000000000
-; AARCH64-NEXT: mov x9, sp
+; AARCH64-NEXT: mov w10, #-1 // =0xffffffff
; AARCH64-NEXT: str x8, [sp, #32]
; AARCH64-NEXT: mov w8, w5
-; AARCH64-NEXT: add x12, sp, #64
-; AARCH64-NEXT: lsr x10, x8, #3
-; AARCH64-NEXT: mov w11, #-1 // =0xffffffff
-; AARCH64-NEXT: add x9, x9, #32
-; AARCH64-NEXT: add x12, x12, #32
+; AARCH64-NEXT: add x11, sp, #32
+; AARCH64-NEXT: lsr x9, x8, #3
+; AARCH64-NEXT: add x12, sp, #96
; AARCH64-NEXT: str xzr, [sp, #56]
+; AARCH64-NEXT: str xzr, [sp, #120]
; AARCH64-NEXT: and x15, x8, #0x3f
-; AARCH64-NEXT: and x10, x10, #0x18
+; AARCH64-NEXT: and x9, x9, #0x18
; AARCH64-NEXT: stur q0, [sp, #40]
; AARCH64-NEXT: eor x15, x15, #0x3f
; AARCH64-NEXT: stp q0, q0, [sp]
-; AARCH64-NEXT: sub x9, x9, x10
-; AARCH64-NEXT: sub x10, x12, x10
-; AARCH64-NEXT: str xzr, [sp, #120]
+; AARCH64-NEXT: sub x11, x11, x9
+; AARCH64-NEXT: sub x9, x12, x9
; AARCH64-NEXT: stur q0, [sp, #104]
; AARCH64-NEXT: stp q0, q0, [sp, #64]
-; AARCH64-NEXT: str x11, [sp, #96]
-; AARCH64-NEXT: ldp x12, x16, [x10]
-; AARCH64-NEXT: ldp x11, x13, [x9]
+; AARCH64-NEXT: str x10, [sp, #96]
+; AARCH64-NEXT: ldp x12, x16, [x9]
+; AARCH64-NEXT: ldp x10, x13, [x11]
; AARCH64-NEXT: lsr x17, x12, #1
; AARCH64-NEXT: lsl x4, x16, x8
; AARCH64-NEXT: lsr x16, x16, #1
-; AARCH64-NEXT: lsr x14, x11, #1
+; AARCH64-NEXT: lsr x14, x10, #1
; AARCH64-NEXT: lsl x18, x13, x8
; AARCH64-NEXT: lsr x13, x13, #1
; AARCH64-NEXT: lsr x17, x17, x15
; AARCH64-NEXT: lsl x12, x12, x8
; AARCH64-NEXT: lsr x14, x14, x15
; AARCH64-NEXT: orr x17, x4, x17
-; AARCH64-NEXT: ldp x4, x10, [x10, #16]
+; AARCH64-NEXT: ldp x4, x9, [x9, #16]
; AARCH64-NEXT: orr x14, x18, x14
; AARCH64-NEXT: mvn w18, w5
; AARCH64-NEXT: bic x17, x1, x17
-; AARCH64-NEXT: ldp x5, x9, [x9, #16]
+; AARCH64-NEXT: ldp x5, x11, [x11, #16]
; AARCH64-NEXT: lsr x13, x13, x18
; AARCH64-NEXT: lsr x1, x4, #1
; AARCH64-NEXT: lsr x16, x16, x18
; AARCH64-NEXT: lsl x18, x4, x8
-; AARCH64-NEXT: lsl x10, x10, x8
+; AARCH64-NEXT: lsl x9, x9, x8
; AARCH64-NEXT: bic x12, x0, x12
; AARCH64-NEXT: lsr x4, x5, #1
; AARCH64-NEXT: lsr x1, x1, x15
; AARCH64-NEXT: lsl x6, x5, x8
-; AARCH64-NEXT: lsl x9, x9, x8
+; AARCH64-NEXT: lsl x11, x11, x8
; AARCH64-NEXT: orr x16, x18, x16
-; AARCH64-NEXT: lsl x8, x11, x8
+; AARCH64-NEXT: lsl x8, x10, x8
; AARCH64-NEXT: lsr x15, x4, x15
-; AARCH64-NEXT: orr x10, x10, x1
+; AARCH64-NEXT: orr x9, x9, x1
; AARCH64-NEXT: orr x13, x6, x13
-; AARCH64-NEXT: eor x10, x10, #0x7fffffffff
-; AARCH64-NEXT: bic x11, x2, x16
+; AARCH64-NEXT: eor x9, x9, #0x7fffffffff
+; AARCH64-NEXT: bic x10, x2, x16
; AARCH64-NEXT: orr x0, x12, x8
-; AARCH64-NEXT: orr x9, x9, x15
-; AARCH64-NEXT: and x10, x3, x10
+; AARCH64-NEXT: orr x11, x11, x15
+; AARCH64-NEXT: and x9, x3, x9
; AARCH64-NEXT: orr x1, x17, x14
-; AARCH64-NEXT: orr x2, x11, x13
-; AARCH64-NEXT: orr x3, x10, x9
+; AARCH64-NEXT: orr x2, x10, x13
+; AARCH64-NEXT: orr x3, x9, x11
; AARCH64-NEXT: add sp, sp, #128
; AARCH64-NEXT: ret
%result = bitinsert b231 %base, i32 %val, i32 %off
@@ -573,35 +571,33 @@ define b231 @test_bitinsert_val_b123_into_b231_var(b231 %base, b123 %val, i32 %o
; AARCH64-NEXT: movi v0.2d, #0000000000000000
; AARCH64-NEXT: mov x9, #576460752303423487 // =0x7ffffffffffffff
; AARCH64-NEXT: stp x4, x8, [sp, #32]
-; AARCH64-NEXT: mov x10, #-1 // =0xffffffffffffffff
; AARCH64-NEXT: mov w8, w6
+; AARCH64-NEXT: mov x10, #-1 // =0xffffffffffffffff
+; AARCH64-NEXT: lsr x11, x8, #3
; AARCH64-NEXT: stp x10, x9, [sp, #96]
-; AARCH64-NEXT: lsr x9, x8, #3
-; AARCH64-NEXT: mov x11, sp
-; AARCH64-NEXT: add x10, sp, #64
-; AARCH64-NEXT: add x11, x11, #32
+; AARCH64-NEXT: add x10, sp, #32
; AARCH64-NEXT: and x15, x8, #0x3f
-; AARCH64-NEXT: and x9, x9, #0x18
-; AARCH64-NEXT: add x10, x10, #32
+; AARCH64-NEXT: and x9, x11, #0x18
+; AARCH64-NEXT: add x11, sp, #96
; AARCH64-NEXT: stp q0, q0, [sp]
; AARCH64-NEXT: stp q0, q0, [sp, #48]
-; AARCH64-NEXT: sub x11, x11, x9
-; AARCH64-NEXT: sub x9, x10, x9
+; AARCH64-NEXT: sub x10, x10, x9
+; AARCH64-NEXT: sub x9, x11, x9
; AARCH64-NEXT: str q0, [sp, #112]
; AARCH64-NEXT: eor x15, x15, #0x3f
; AARCH64-NEXT: str q0, [sp, #80]
-; AARCH64-NEXT: ldp x10, x16, [x9]
-; AARCH64-NEXT: ldp x12, x13, [x11]
-; AARCH64-NEXT: ldp x5, x11, [x11, #16]
-; AARCH64-NEXT: lsr x17, x10, #1
+; AARCH64-NEXT: ldp x11, x16, [x9]
+; AARCH64-NEXT: ldp x12, x13, [x10]
+; AARCH64-NEXT: ldp x5, x10, [x10, #16]
+; AARCH64-NEXT: lsr x17, x11, #1
; AARCH64-NEXT: lsl x4, x16, x8
; AARCH64-NEXT: lsr x16, x16, #1
; AARCH64-NEXT: lsr x14, x12, #1
; AARCH64-NEXT: lsl x18, x13, x8
; AARCH64-NEXT: lsr x13, x13, #1
; AARCH64-NEXT: lsr x17, x17, x15
-; AARCH64-NEXT: lsl x11, x11, x8
; AARCH64-NEXT: lsl x10, x10, x8
+; AARCH64-NEXT: lsl x11, x11, x8
; AARCH64-NEXT: lsr x14, x14, x15
; AARCH64-NEXT: orr x17, x4, x17
; AARCH64-NEXT: ldp x4, x9, [x9, #16]
@@ -621,15 +617,15 @@ define b231 @test_bitinsert_val_b123_into_b231_var(b231 %base, b123 %val, i32 %o
; AARCH64-NEXT: lsr x15, x4, x15
; AARCH64-NEXT: orr x16, x18, x16
; AARCH64-NEXT: bic x12, x2, x16
-; AARCH64-NEXT: bic x10, x0, x10
+; AARCH64-NEXT: bic x11, x0, x11
; AARCH64-NEXT: orr x9, x9, x1
-; AARCH64-NEXT: orr x11, x11, x15
-; AARCH64-NEXT: orr x0, x10, x8
+; AARCH64-NEXT: orr x10, x10, x15
+; AARCH64-NEXT: orr x0, x11, x8
; AARCH64-NEXT: eor x9, x9, #0x7fffffffff
; AARCH64-NEXT: orr x1, x17, x14
; AARCH64-NEXT: orr x2, x12, x13
; AARCH64-NEXT: and x9, x3, x9
-; AARCH64-NEXT: orr x3, x9, x11
+; AARCH64-NEXT: orr x3, x9, x10
; AARCH64-NEXT: add sp, sp, #128
; AARCH64-NEXT: ret
%result = bitinsert b231 %base, b123 %val, i32 %off
diff --git a/llvm/test/CodeGen/AArch64/div-i256.ll b/llvm/test/CodeGen/AArch64/div-i256.ll
index 5f8d362c85bb2..d93e3801b055c 100644
--- a/llvm/test/CodeGen/AArch64/div-i256.ll
+++ b/llvm/test/CodeGen/AArch64/div-i256.ll
@@ -67,9 +67,8 @@ define i256 @udiv256(i256 %a, i256 %b) nounwind {
; CHECK-NEXT: sub sp, sp, #208
; CHECK-NEXT: mov w8, #255 // =0xff
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: add x9, sp, #64
+; CHECK-NEXT: add x9, sp, #96
; CHECK-NEXT: sub x11, x8, x15
-; CHECK-NEXT: add x9, x9, #32
; CHECK-NEXT: stp x3, x27, [sp, #120] // 8-byte Folded Spill
; CHECK-NEXT: lsr x8, x11, #3
; CHECK-NEXT: stp x26, x25, [sp, #144] // 16-byte Folded Spill
@@ -284,9 +283,8 @@ define i256 @sdiv256(i256 %a, i256 %b) nounwind {
; CHECK-NEXT: sub sp, sp, #208
; CHECK-NEXT: mov w13, #255 // =0xff
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: add x4, sp, #64
+; CHECK-NEXT: add x4, sp, #96
; CHECK-NEXT: sub x3, x13, x2
-; CHECK-NEXT: add x4, x4, #32
; CHECK-NEXT: stp x28, x27, [sp, #128] // 16-byte Folded Spill
; CHECK-NEXT: lsr x13, x3, #3
; CHECK-NEXT: stp x26, x25, [sp, #144] // 16-byte Folded Spill
@@ -486,9 +484,8 @@ define i256 @urem256(i256 %a, i256 %b) nounwind {
; CHECK-NEXT: sub sp, sp, #240
; CHECK-NEXT: mov w8, #255 // =0xff
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: add x14, sp, #80
+; CHECK-NEXT: add x14, sp, #112
; CHECK-NEXT: sub x13, x8, x12
-; CHECK-NEXT: add x14, x14, #32
; CHECK-NEXT: stp x29, x30, [sp, #144] // 16-byte Folded Spill
; CHECK-NEXT: lsr x8, x13, #3
; CHECK-NEXT: stp x28, x27, [sp, #160] // 16-byte Folded Spill
@@ -740,9 +737,8 @@ define i256 @srem256(i256 %a, i256 %b) nounwind {
; CHECK-NEXT: // %bb.2: // %udiv-bb1
; CHECK-NEXT: mov w9, #255 // =0xff
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: add x17, sp, #80
+; CHECK-NEXT: add x17, sp, #112
; CHECK-NEXT: sub x9, x9, x2
-; CHECK-NEXT: add x17, x17, #32
; CHECK-NEXT: stp x22, x23, [sp, #112]
; CHECK-NEXT: lsr x10, x9, #3
; CHECK-NEXT: stp x12, x11, [sp, #128]
@@ -929,20 +925,20 @@ define i256 @sdiv256_const(i256 %a) nounwind {
; CHECK-NEXT: eor x10, x1, x8
; CHECK-NEXT: subs x18, x9, x8
; CHECK-NEXT: eor x9, x2, x8
-; CHECK-NEXT: sbcs x0, x10, x8
+; CHECK-NEXT: sbcs x1, x10, x8
; CHECK-NEXT: eor x10, x3, x8
-; CHECK-NEXT: sbcs x1, x9, x8
+; CHECK-NEXT: sbcs x2, x9, x8
; CHECK-NEXT: clz x9, x18
-; CHECK-NEXT: clz x11, x0
+; CHECK-NEXT: clz x11, x1
; CHECK-NEXT: sbcs x5, x10, x8
-; CHECK-NEXT: clz x10, x1
+; CHECK-NEXT: clz x10, x2
; CHECK-NEXT: add x9, x9, #64
; CHECK-NEXT: add x10, x10, #64
; CHECK-NEXT: clz x12, x5
-; CHECK-NEXT: orr x14, x0, x5
+; CHECK-NEXT: orr x14, x1, x5
; CHECK-NEXT: csel x10, x12, x10, ne
-; CHECK-NEXT: cmp x0, #0
-; CHECK-NEXT: orr x12, x1, x5
+; CHECK-NEXT: cmp x1, #0
+; CHECK-NEXT: orr x12, x2, x5
; CHECK-NEXT: csel x9, x11, x9, ne
; CHECK-NEXT: cmp x12, #0
; CHECK-NEXT: mov w11, #253 // =0xfd
@@ -953,15 +949,15 @@ define i256 @sdiv256_const(i256 %a) nounwind {
; CHECK-NEXT: ngcs x11, xzr
; CHECK-NEXT: ngc x12, xzr
; CHECK-NEXT: cmp x13, x9
-; CHECK-NEXT: orr x13, x18, x1
+; CHECK-NEXT: orr x13, x18, x2
; CHECK-NEXT: ngcs xzr, x10
; CHECK-NEXT: orr x13, x13, x14
; CHECK-NEXT: ngcs xzr, x11
; CHECK-NEXT: ngcs xzr, x12
; CHECK-NEXT: ccmp x13, #0, #4, hs
; CHECK-NEXT: csel x13, xzr, x5, eq
-; CHECK-NEXT: csel x15, xzr, x1, eq
-; CHECK-NEXT: csel x2, xzr, x0, eq
+; CHECK-NEXT: csel x15, xzr, x2, eq
+; CHECK-NEXT: csel x0, xzr, x1, eq
; CHECK-NEXT: csel x14, xzr, x18, eq
; CHECK-NEXT: b.eq .LBB5_6
; CHECK-NEXT: // %bb.1: // %_udiv-special-cases
@@ -974,33 +970,32 @@ define i256 @sdiv256_const(i256 %a) nounwind {
; CHECK-NEXT: sub sp, sp, #192
; CHECK-NEXT: mov w13, #255 // =0xff
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: add x15, sp, #64
+; CHECK-NEXT: add x15, sp, #96
; CHECK-NEXT: sub x13, x13, x9
-; CHECK-NEXT: add x15, x15, #32
; CHECK-NEXT: stp x5, x25, [sp, #120] // 8-byte Folded Spill
+; CHECK-NEXT: adds x9, x9, #1
; CHECK-NEXT: lsr x14, x13, #3
; CHECK-NEXT: stp x24, x23, [sp, #144] // 16-byte Folded Spill
-; CHECK-NEXT: adds x9, x9, #1
-; CHECK-NEXT: stp x22, x21, [sp, #160] // 16-byte Folded Spill
; CHECK-NEXT: and x3, x13, #0x3f
+; CHECK-NEXT: stp x22, x21, [sp, #160] // 16-byte Folded Spill
; CHECK-NEXT: adcs x10, x10, xzr
+; CHECK-NEXT: mvn w4, w13
; CHECK-NEXT: and x14, x14, #0x18
; CHECK-NEXT: stp x20, x19, [sp, #176] // 16-byte Folded Spill
-; CHECK-NEXT: mvn w4, w13
-; CHECK-NEXT: stp x18, x0, [sp, #96]
-; CHECK-NEXT: sub x14, x15, x14
; CHECK-NEXT: eor x3, x3, #0x3f
-; CHECK-NEXT: str x1, [sp, #112]
+; CHECK-NEXT: stp x18, x1, [sp, #96]
+; CHECK-NEXT: sub x14, x15, x14
; CHECK-NEXT: adcs x11, x11, xzr
+; CHECK-NEXT: str x2, [sp, #112]
; CHECK-NEXT: stp q0, q0, [sp, #64]
-; CHECK-NEXT: ldp x16, x2, [x14, #8]
+; CHECK-NEXT: ldp x16, x0, [x14, #8]
; CHECK-NEXT: ldr x17, [x14, #24]
; CHECK-NEXT: ldr x14, [x14]
; CHECK-NEXT: lsl x17, x17, x13
; CHECK-NEXT: lsr x15, x16, #1
-; CHECK-NEXT: lsr x6, x2, #1
+; CHECK-NEXT: lsr x6, x0, #1
; CHECK-NEXT: lsr x7, x14, #1
-; CHECK-NEXT: lsl x2, x2, x13
+; CHECK-NEXT: lsl x0, x0, x13
; CHECK-NEXT: lsl x19, x16, x13
; CHECK-NEXT: lsr x4, x15, x4
; CHECK-NEXT: lsr x6, x6, x3
@@ -1009,38 +1004,38 @@ define i256 @sdiv256_const(i256 %a) nounwind {
; CHECK-NEXT: lsl x12, x14, x13
; CHECK-NEXT: cset w7, hs
; CHECK-NEXT: orr x16, x17, x6
-; CHECK-NEXT: orr x14, x2, x4
+; CHECK-NEXT: orr x14, x0, x4
; CHECK-NEXT: orr x13, x19, x3
; CHECK-NEXT: mov x17, xzr
; CHECK-NEXT: tbnz w7, #0, .LBB5_5
; CHECK-NEXT: // %bb.3: // %udiv-preheader
; CHECK-NEXT: lsr x6, x9, #3
-; CHECK-NEXT: stp x18, x0, [sp]
-; CHECK-NEXT: mov x0, sp
+; CHECK-NEXT: stp x18, x1, [sp]
+; CHECK-NEXT: mov x1, sp
; CHECK-NEXT: stp q0, q0, [sp, #32]
; CHECK-NEXT: mvn w7, w9
; CHECK-NEXT: mov x3, xzr
; CHECK-NEXT: and x18, x6, #0x18
-; CHECK-NEXT: stp x1, x5, [sp, #16]
+; CHECK-NEXT: stp x2, x5, [sp, #16]
; CHECK-NEXT: and x6, x9, #0x3f
-; CHECK-NEXT: add x0, x0, x18
+; CHECK-NEXT: add x1, x1, x18
; CHECK-NEXT: mov w18, #7 // =0x7
; CHECK-NEXT: eor x6, x6, #0x3f
-; CHECK-NEXT: ldp x1, x5, [x0, #16]
+; CHECK-NEXT: ldp x2, x5, [x1, #16]
; CHECK-NEXT: mov x4, xzr
-; CHECK-NEXT: ldp x21, x19, [x0]
-; CHECK-NEXT: mov x2, xzr
-; CHECK-NEXT: lsl x0, x1, #1
+; CHECK-NEXT: ldp x21, x19, [x1]
+; CHECK-NEXT: mov x0, xzr
+; CHECK-NEXT: lsl x1, x2, #1
; CHECK-NEXT: lsl x20, x5, #1
-; CHECK-NEXT: lsr x22, x1, x9
-; CHECK-NEXT: lsl x1, x19, #1
+; CHECK-NEXT: lsr x22, x2, x9
+; CHECK-NEXT: lsl x2, x19, #1
; CHECK-NEXT: lsr x24, x19, x9
; CHECK-NEXT: lsr x21, x21, x9
-; CHECK-NEXT: lsl x23, x0, x7
-; CHECK-NEXT: subs x0, x18, #1
+; CHECK-NEXT: lsl x23, x1, x7
+; CHECK-NEXT: subs x1, x18, #1
; CHECK-NEXT: lsl x20, x20, x6
-; CHECK-NEXT: lsl x6, x1, x6
-; CHECK-NEXT: ngcs x1, xzr
+; CHECK-NEXT: lsl x6, x2, x6
+; CHECK-NEXT: ngcs x2, xzr
; CHECK-NEXT: lsr x19, x5, x9
; CHECK-NEXT: ngcs x5, xzr
; CHECK-NEXT: orr x7, x20, x22
@@ -1055,12 +1050,12 @@ define i256 @sdiv256_const(i256 %a) nounwind {
; CHECK-NEXT: extr x19, x19, x7, #63
; CHECK-NEXT: extr x16, x16, x14, #63
; CHECK-NEXT: extr x14, x14, x13, #63
-; CHECK-NEXT: cmp x0, x22
-; CHECK-NEXT: sbcs xzr, x1, x23
+; CHECK-NEXT: cmp x1, x22
+; CHECK-NEXT: sbcs xzr, x2, x23
; CHECK-NEXT: orr x14, x4, x14
-; CHECK-NEXT: orr x16, x2, x16
+; CHECK-NEXT: orr x16, x0, x16
; CHECK-NEXT: sbcs xzr, x5, x24
-; CHECK-NEXT: mov x2, xzr
+; CHECK-NEXT: mov x0, xzr
; CHECK-NEXT: sbc x7, x6, x19
; CHECK-NEXT: asr x25, x7, #63
; CHECK-NEXT: and x7, x25, x18
@@ -1084,7 +1079,7 @@ define i256 @sdiv256_const(i256 %a) nounwind {
; CHECK-NEXT: cbnz x22, .LBB5_4
; CHECK-NEXT: .LBB5_5: // %udiv-loop-exit
; CHECK-NEXT: ldp x20, x19, [sp, #176] // 16-byte Folded Reload
-; CHECK-NEXT: extr x2, x13, x12, #63
+; CHECK-NEXT: extr x0, x13, x12, #63
; CHECK-NEXT: ldp x22, x21, [sp, #160] // 16-byte Folded Reload
; CHECK-NEXT: extr x15, x14, x13, #63
; CHECK-NEXT: ldp x24, x23, [sp, #144] // 16-byte Folded Reload
@@ -1094,7 +1089,7 @@ define i256 @sdiv256_const(i256 %a) nounwind {
; CHECK-NEXT: add sp, sp, #192
; CHECK-NEXT: .LBB5_6: // %udiv-end
; CHECK-NEXT: eor x9, x14, x8
-; CHECK-NEXT: eor x10, x2, x8
+; CHECK-NEXT: eor x10, x0, x8
; CHECK-NEXT: subs x0, x9, x8
; CHECK-NEXT: eor x9, x15, x8
; CHECK-NEXT: sbcs x1, x10, x8
diff --git a/llvm/test/CodeGen/AArch64/early-ifcvt-frame-index.mir b/llvm/test/CodeGen/AArch64/early-ifcvt-frame-index.mir
new file mode 100644
index 0000000000000..e1a4fe87534e7
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/early-ifcvt-frame-index.mir
@@ -0,0 +1,49 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=aarch64-- -run-pass=early-ifcvt -stress-early-ifcvt -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=aarch64-- -passes=early-ifcvt -stress-early-ifcvt %s -o - | FileCheck %s
+
+# An ADDXri that materialises a frame index holds a frame index, not a
+# register, in operand 1, and it carries the offset of the addressed object in
+# its immediate. canFoldIntoCSel() must not read operand 1 as a register when
+# that immediate happens to be 1, so this select stays a plain CSEL rather than
+# becoming a CSINC off a non-existent base register.
+
+---
+name: frame_index_offset_one
+tracksRegLiveness: true
+frameInfo:
+ maxAlignment: 8
+stack:
+ - { id: 0, size: 64, alignment: 8 }
+body: |
+ ; CHECK-LABEL: name: frame_index_offset_one
+ ; CHECK: bb.0.entry:
+ ; CHECK-NEXT: liveins: $x0, $w1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr32sp = COPY $w1
+ ; CHECK-NEXT: dead $wzr = SUBSWri [[COPY1]], 0, 0, implicit-def $nzcv
+ ; CHECK-NEXT: [[ADDXri:%[0-9]+]]:gpr64common = ADDXri %stack.0, 1, 0
+ ; CHECK-NEXT: [[CSELXr:%[0-9]+]]:gpr64 = CSELXr [[COPY]], [[ADDXri]], 13, implicit $nzcv
+ ; CHECK-NEXT: $x0 = COPY [[CSELXr]]
+ ; CHECK-NEXT: RET_ReallyLR implicit $x0
+ bb.0.entry:
+ successors: %bb.1, %bb.2
+ liveins: $x0, $w1
+
+ %0:gpr64 = COPY $x0
+ %1:gpr32sp = COPY $w1
+ dead $wzr = SUBSWri %1, 0, 0, implicit-def $nzcv
+ Bcc 13, %bb.2, implicit $nzcv
+ B %bb.1
+
+ bb.1:
+ successors: %bb.2
+
+ %2:gpr64common = ADDXri %stack.0, 1, 0
+
+ bb.2:
+ %3:gpr64 = PHI %2, %bb.1, %0, %bb.0
+ $x0 = COPY %3
+ RET_ReallyLR implicit $x0
+...
diff --git a/llvm/test/CodeGen/AArch64/fcvt-i256.ll b/llvm/test/CodeGen/AArch64/fcvt-i256.ll
index d7d9d04187506..7ff9328856b3b 100644
--- a/llvm/test/CodeGen/AArch64/fcvt-i256.ll
+++ b/llvm/test/CodeGen/AArch64/fcvt-i256.ll
@@ -56,44 +56,43 @@ define float @s256_to_f32(i256 %val) {
; CHECK-NEXT: movi v0.2d, #0000000000000000
; CHECK-NEXT: add x17, sp, #64
; CHECK-NEXT: sub w14, w14, w15
+; CHECK-NEXT: add w15, w15, #26
; CHECK-NEXT: stp x10, x13, [sp, #64]
-; CHECK-NEXT: mov x18, sp
; CHECK-NEXT: lsr x16, x14, #3
+; CHECK-NEXT: lsr x18, x15, #3
; CHECK-NEXT: stp x12, x11, [sp, #80]
; CHECK-NEXT: and x16, x16, #0x18
; CHECK-NEXT: stp q0, q0, [sp, #96]
; CHECK-NEXT: add x16, x17, x16
; CHECK-NEXT: ldp x16, x17, [x16]
; CHECK-NEXT: stp x10, x13, [sp, #32]
-; CHECK-NEXT: add w10, w15, #26
-; CHECK-NEXT: add x13, x18, #32
+; CHECK-NEXT: and x10, x18, #0x18
+; CHECK-NEXT: add x13, sp, #32
; CHECK-NEXT: stp q0, q0, [sp]
-; CHECK-NEXT: lsr x15, x10, #3
; CHECK-NEXT: stp x12, x11, [sp, #48]
-; CHECK-NEXT: and x18, x10, #0x3f
+; CHECK-NEXT: sub x10, x13, x10
+; CHECK-NEXT: and x18, x15, #0x3f
+; CHECK-NEXT: ldp x12, x11, [x10, #16]
; CHECK-NEXT: lsl x17, x17, #1
-; CHECK-NEXT: and x15, x15, #0x18
-; CHECK-NEXT: sub x12, x13, x15
-; CHECK-NEXT: ldp x13, x11, [x12, #16]
-; CHECK-NEXT: ldp x12, x15, [x12]
-; CHECK-NEXT: orr x11, x15, x11
-; CHECK-NEXT: orr x12, x12, x13
-; CHECK-NEXT: lsr x13, x15, #1
-; CHECK-NEXT: orr x11, x12, x11
-; CHECK-NEXT: mvn w15, w10
-; CHECK-NEXT: lsr x12, x12, #1
-; CHECK-NEXT: lsl x10, x11, x10
-; CHECK-NEXT: lsr x11, x13, x15
+; CHECK-NEXT: ldp x10, x13, [x10]
+; CHECK-NEXT: orr x11, x13, x11
+; CHECK-NEXT: orr x10, x10, x12
+; CHECK-NEXT: lsr x12, x13, #1
+; CHECK-NEXT: orr x11, x10, x11
+; CHECK-NEXT: mvn w13, w15
+; CHECK-NEXT: lsr x10, x10, #1
+; CHECK-NEXT: lsl x11, x11, x15
+; CHECK-NEXT: lsr x12, x12, x13
; CHECK-NEXT: eor x13, x18, #0x3f
; CHECK-NEXT: and x15, x14, #0x3f
-; CHECK-NEXT: lsr x12, x12, x13
+; CHECK-NEXT: lsr x10, x10, x13
; CHECK-NEXT: eor x13, x15, #0x3f
-; CHECK-NEXT: orr x10, x10, x11
-; CHECK-NEXT: lsr x11, x16, x14
+; CHECK-NEXT: orr x11, x11, x12
+; CHECK-NEXT: lsr x12, x16, x14
; CHECK-NEXT: lsl x13, x17, x13
-; CHECK-NEXT: orr x10, x10, x12
+; CHECK-NEXT: orr x10, x11, x10
; CHECK-NEXT: cmp x10, #0
-; CHECK-NEXT: orr x10, x13, x11
+; CHECK-NEXT: orr x10, x13, x12
; CHECK-NEXT: cset w11, ne
; CHECK-NEXT: orr x10, x10, x11
; CHECK-NEXT: .LBB0_5: // %itofp-sw-epilog
@@ -131,8 +130,7 @@ define float @s256_to_f32(i256 %val) {
; CHECK-NEXT: sub w9, w15, #232
; CHECK-NEXT: str x10, [sp, #160]
; CHECK-NEXT: lsr x11, x9, #3
-; CHECK-NEXT: add x10, sp, #128
-; CHECK-NEXT: add x10, x10, #32
+; CHECK-NEXT: add x10, sp, #160
; CHECK-NEXT: and x11, x11, #0x18
; CHECK-NEXT: stp q0, q0, [sp, #128]
; CHECK-NEXT: sub x10, x10, x11
@@ -189,12 +187,11 @@ define float @u256_to_f32(i256 %val) {
; CHECK-NEXT: add x13, sp, #64
; CHECK-NEXT: sub w10, w10, w11
; CHECK-NEXT: add w11, w11, #26
-; CHECK-NEXT: mov x15, sp
+; CHECK-NEXT: stp x0, x1, [sp, #64]
; CHECK-NEXT: lsr x12, x10, #3
; CHECK-NEXT: lsr x14, x11, #3
-; CHECK-NEXT: stp x0, x1, [sp, #64]
; CHECK-NEXT: stp x2, x3, [sp, #80]
-; CHECK-NEXT: add x15, x15, #32
+; CHECK-NEXT: add x15, sp, #32
; CHECK-NEXT: and x18, x11, #0x3f
; CHECK-NEXT: and x12, x12, #0x18
; CHECK-NEXT: stp q0, q0, [sp, #96]
@@ -202,8 +199,8 @@ define float @u256_to_f32(i256 %val) {
; CHECK-NEXT: add x12, x13, x12
; CHECK-NEXT: sub x14, x15, x14
; CHECK-NEXT: ldp x12, x13, [x12]
-; CHECK-NEXT: stp x0, x1, [sp, #32]
; CHECK-NEXT: stp q0, q0, [sp]
+; CHECK-NEXT: stp x0, x1, [sp, #32]
; CHECK-NEXT: stp x2, x3, [sp, #48]
; CHECK-NEXT: ldp x16, x15, [x14, #16]
; CHECK-NEXT: lsl x13, x13, #1
@@ -254,9 +251,8 @@ define float @u256_to_f32(i256 %val) {
; CHECK-NEXT: .LBB1_8: // %itofp-if-else
; CHECK-NEXT: movi v0.2d, #0000000000000000
; CHECK-NEXT: sub w9, w11, #232
-; CHECK-NEXT: add x11, sp, #128
+; CHECK-NEXT: add x11, sp, #160
; CHECK-NEXT: lsr x10, x9, #3
-; CHECK-NEXT: add x11, x11, #32
; CHECK-NEXT: str x0, [sp, #160]
; CHECK-NEXT: and x10, x10, #0x18
; CHECK-NEXT: stp q0, q0, [sp, #128]
@@ -285,22 +281,22 @@ define double @s256_to_f64(i256 %val) {
; CHECK-NEXT: asr x8, x3, #63
; CHECK-NEXT: eor x9, x0, x8
; CHECK-NEXT: eor x10, x1, x8
-; CHECK-NEXT: eor x11, x3, x8
-; CHECK-NEXT: subs x12, x9, x8
+; CHECK-NEXT: eor x12, x3, x8
+; CHECK-NEXT: subs x11, x9, x8
; CHECK-NEXT: eor x9, x2, x8
; CHECK-NEXT: sbcs x10, x10, x8
-; CHECK-NEXT: clz x14, x12
+; CHECK-NEXT: clz x14, x11
; CHECK-NEXT: sbcs x13, x9, x8
; CHECK-NEXT: add w14, w14, #64
-; CHECK-NEXT: sbcs x11, x11, x8
+; CHECK-NEXT: sbcs x12, x12, x8
; CHECK-NEXT: clz x8, x13
; CHECK-NEXT: add w8, w8, #64
-; CHECK-NEXT: clz x9, x11
+; CHECK-NEXT: clz x9, x12
; CHECK-NEXT: csel w8, w9, w8, ne
; CHECK-NEXT: clz x9, x10
; CHECK-NEXT: cmp x10, #0
; CHECK-NEXT: csel w9, w9, w14, ne
-; CHECK-NEXT: orr x14, x13, x11
+; CHECK-NEXT: orr x14, x13, x12
; CHECK-NEXT: add w9, w9, #128
; CHECK-NEXT: cmp x14, #0
; CHECK-NEXT: csel w14, w8, w9, ne
@@ -319,57 +315,56 @@ define double @s256_to_f64(i256 %val) {
; CHECK-NEXT: // %bb.4: // %itofp-sw-default
; CHECK-NEXT: mov w15, #201 // =0xc9
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: add x17, sp, #64
+; CHECK-NEXT: add w0, w14, #55
; CHECK-NEXT: sub w15, w15, w14
-; CHECK-NEXT: stp x12, x10, [sp, #64]
-; CHECK-NEXT: mov x1, sp
+; CHECK-NEXT: add x17, sp, #64
+; CHECK-NEXT: lsr x1, x0, #3
; CHECK-NEXT: lsr x16, x15, #3
-; CHECK-NEXT: stp x13, x11, [sp, #80]
+; CHECK-NEXT: stp x11, x10, [sp, #64]
+; CHECK-NEXT: and x2, x0, #0x3f
+; CHECK-NEXT: stp x13, x12, [sp, #80]
+; CHECK-NEXT: sub w14, w14, #202
; CHECK-NEXT: and x16, x16, #0x18
; CHECK-NEXT: stp q0, q0, [sp, #96]
; CHECK-NEXT: add x16, x17, x16
; CHECK-NEXT: ldp x18, x17, [x16, #8]
; CHECK-NEXT: ldr x16, [x16]
-; CHECK-NEXT: stp x12, x10, [sp, #32]
-; CHECK-NEXT: add w12, w14, #55
-; CHECK-NEXT: add x10, x1, #32
-; CHECK-NEXT: lsr x0, x12, #3
+; CHECK-NEXT: stp x11, x10, [sp, #32]
+; CHECK-NEXT: and x10, x1, #0x18
+; CHECK-NEXT: add x11, sp, #32
; CHECK-NEXT: stp q0, q0, [sp]
-; CHECK-NEXT: and x2, x12, #0x3f
-; CHECK-NEXT: stp x13, x11, [sp, #48]
+; CHECK-NEXT: sub x10, x11, x10
; CHECK-NEXT: and x1, x15, #0x3f
-; CHECK-NEXT: sub w14, w14, #202
-; CHECK-NEXT: and x0, x0, #0x18
+; CHECK-NEXT: stp x13, x12, [sp, #48]
; CHECK-NEXT: eor x1, x1, #0x3f
-; CHECK-NEXT: sub x10, x10, x0
-; CHECK-NEXT: ldp x13, x11, [x10, #16]
-; CHECK-NEXT: ldp x10, x0, [x10]
-; CHECK-NEXT: orr x11, x0, x11
-; CHECK-NEXT: orr x10, x10, x13
-; CHECK-NEXT: lsl x13, x17, #1
-; CHECK-NEXT: lsr x17, x0, #1
+; CHECK-NEXT: ldp x12, x11, [x10, #16]
+; CHECK-NEXT: ldp x10, x13, [x10]
+; CHECK-NEXT: orr x11, x13, x11
+; CHECK-NEXT: orr x10, x10, x12
+; CHECK-NEXT: lsr x13, x13, #1
+; CHECK-NEXT: lsl x12, x17, #1
; CHECK-NEXT: orr x11, x10, x11
-; CHECK-NEXT: mvn w0, w12
+; CHECK-NEXT: mvn w17, w0
; CHECK-NEXT: lsr x10, x10, #1
-; CHECK-NEXT: lsl x11, x11, x12
-; CHECK-NEXT: lsl x13, x13, x14
-; CHECK-NEXT: lsr x12, x17, x0
+; CHECK-NEXT: lsl x11, x11, x0
+; CHECK-NEXT: lsr x13, x13, x17
; CHECK-NEXT: eor x17, x2, #0x3f
; CHECK-NEXT: lsl x0, x18, #1
+; CHECK-NEXT: lsl x12, x12, x14
; CHECK-NEXT: lsr x10, x10, x17
+; CHECK-NEXT: orr x11, x11, x13
; CHECK-NEXT: lsr x17, x18, x15
-; CHECK-NEXT: orr x11, x11, x12
-; CHECK-NEXT: lsr x12, x16, x15
+; CHECK-NEXT: lsr x13, x16, x15
; CHECK-NEXT: lsl x14, x0, x1
; CHECK-NEXT: orr x10, x11, x10
; CHECK-NEXT: cmp x10, #0
-; CHECK-NEXT: orr x11, x14, x12
-; CHECK-NEXT: orr x10, x17, x13
+; CHECK-NEXT: orr x10, x17, x12
+; CHECK-NEXT: orr x11, x14, x13
; CHECK-NEXT: cset w12, ne
-; CHECK-NEXT: orr x12, x11, x12
+; CHECK-NEXT: orr x11, x11, x12
; CHECK-NEXT: .LBB2_5: // %itofp-sw-epilog
-; CHECK-NEXT: ubfx w11, w12, #2, #1
-; CHECK-NEXT: orr x11, x12, x11
+; CHECK-NEXT: ubfx w12, w11, #2, #1
+; CHECK-NEXT: orr x11, x11, x12
; CHECK-NEXT: adds x11, x11, #1
; CHECK-NEXT: adcs x10, x10, xzr
; CHECK-NEXT: extr x12, x10, x11, #2
@@ -397,21 +392,20 @@ define double @s256_to_f64(i256 %val) {
; CHECK-NEXT: .LBB2_8: // %itofp-if-else
; CHECK-NEXT: movi v0.2d, #0000000000000000
; CHECK-NEXT: sub w9, w14, #203
-; CHECK-NEXT: stp x12, x10, [sp, #160]
+; CHECK-NEXT: stp x11, x10, [sp, #160]
; CHECK-NEXT: lsr x10, x9, #3
-; CHECK-NEXT: add x12, sp, #128
-; CHECK-NEXT: stp x13, x11, [sp, #176]
-; CHECK-NEXT: add x12, x12, #32
+; CHECK-NEXT: add x11, sp, #160
+; CHECK-NEXT: stp x13, x12, [sp, #176]
; CHECK-NEXT: and x10, x10, #0x18
; CHECK-NEXT: stp q0, q0, [sp, #128]
-; CHECK-NEXT: sub x10, x12, x10
+; CHECK-NEXT: sub x10, x11, x10
; CHECK-NEXT: ldr x10, [x10]
; CHECK-NEXT: lsl x10, x10, x9
; CHECK-NEXT: lsr x11, x10, #32
; CHECK-NEXT: b .LBB2_6
; CHECK-NEXT: .LBB2_9: // %itofp-sw-bb
-; CHECK-NEXT: extr x10, x10, x12, #63
-; CHECK-NEXT: lsl x12, x12, #1
+; CHECK-NEXT: extr x10, x10, x11, #63
+; CHECK-NEXT: lsl x11, x11, #1
; CHECK-NEXT: b .LBB2_5
%result = sitofp i256 %val to double
ret double %result
@@ -461,26 +455,25 @@ define double @u256_to_f64(i256 %val) {
; CHECK-NEXT: lsr x16, x15, #3
; CHECK-NEXT: add x13, sp, #64
; CHECK-NEXT: lsr x12, x11, #3
-; CHECK-NEXT: mov x17, sp
; CHECK-NEXT: stp x0, x1, [sp, #64]
+; CHECK-NEXT: add x17, sp, #32
; CHECK-NEXT: stp x2, x3, [sp, #80]
-; CHECK-NEXT: add x17, x17, #32
; CHECK-NEXT: and x16, x16, #0x18
+; CHECK-NEXT: sub w10, w10, #202
; CHECK-NEXT: and x12, x12, #0x18
; CHECK-NEXT: stp q0, q0, [sp, #96]
; CHECK-NEXT: sub x16, x17, x16
; CHECK-NEXT: add x12, x13, x12
-; CHECK-NEXT: sub w10, w10, #202
; CHECK-NEXT: ldp x14, x13, [x12, #8]
; CHECK-NEXT: ldr x12, [x12]
+; CHECK-NEXT: stp q0, q0, [sp]
; CHECK-NEXT: stp x0, x1, [sp, #32]
; CHECK-NEXT: and x1, x11, #0x3f
-; CHECK-NEXT: stp q0, q0, [sp]
-; CHECK-NEXT: eor x1, x1, #0x3f
; CHECK-NEXT: stp x2, x3, [sp, #48]
; CHECK-NEXT: and x2, x15, #0x3f
; CHECK-NEXT: lsl x13, x13, #1
; CHECK-NEXT: ldp x18, x17, [x16, #16]
+; CHECK-NEXT: eor x1, x1, #0x3f
; CHECK-NEXT: ldp x16, x0, [x16]
; CHECK-NEXT: lsl x10, x13, x10
; CHECK-NEXT: orr x17, x0, x17
@@ -531,9 +524,8 @@ define double @u256_to_f64(i256 %val) {
; CHECK-NEXT: .LBB3_8: // %itofp-if-else
; CHECK-NEXT: movi v0.2d, #0000000000000000
; CHECK-NEXT: sub w9, w10, #203
-; CHECK-NEXT: add x11, sp, #128
+; CHECK-NEXT: add x11, sp, #160
; CHECK-NEXT: lsr x10, x9, #3
-; CHECK-NEXT: add x11, x11, #32
; CHECK-NEXT: stp x0, x1, [sp, #160]
; CHECK-NEXT: stp x2, x3, [sp, #176]
; CHECK-NEXT: and x10, x10, #0x18
@@ -593,9 +585,8 @@ define i256 @f32_to_s256(float %val) {
; CHECK-SD-NEXT: sub w10, w10, #150
; CHECK-SD-NEXT: str x11, [sp, #32]
; CHECK-SD-NEXT: lsr x11, x10, #3
-; CHECK-SD-NEXT: mov x12, sp
+; CHECK-SD-NEXT: add x12, sp, #32
; CHECK-SD-NEXT: str xzr, [sp, #56]
-; CHECK-SD-NEXT: add x12, x12, #32
; CHECK-SD-NEXT: and x16, x10, #0x3f
; CHECK-SD-NEXT: mvn w2, w10
; CHECK-SD-NEXT: and x11, x11, #0x18
@@ -817,9 +808,8 @@ define i256 @f32_to_u256(float %val) {
; CHECK-SD-NEXT: sub w9, w9, #150
; CHECK-SD-NEXT: str x8, [sp, #32]
; CHECK-SD-NEXT: lsr x8, x9, #3
-; CHECK-SD-NEXT: mov x10, sp
+; CHECK-SD-NEXT: add x10, sp, #32
; CHECK-SD-NEXT: str xzr, [sp, #56]
-; CHECK-SD-NEXT: add x10, x10, #32
; CHECK-SD-NEXT: and x13, x9, #0x3f
; CHECK-SD-NEXT: mvn w17, w9
; CHECK-SD-NEXT: and x8, x8, #0x18
@@ -955,9 +945,8 @@ define i256 @f64_to_s256(double %val) {
; CHECK-SD-NEXT: sub x10, x10, #1075
; CHECK-SD-NEXT: str x11, [sp, #32]
; CHECK-SD-NEXT: lsr x11, x10, #3
-; CHECK-SD-NEXT: mov x12, sp
+; CHECK-SD-NEXT: add x12, sp, #32
; CHECK-SD-NEXT: str xzr, [sp, #56]
-; CHECK-SD-NEXT: add x12, x12, #32
; CHECK-SD-NEXT: and x16, x10, #0x3f
; CHECK-SD-NEXT: mvn w2, w10
; CHECK-SD-NEXT: and x11, x11, #0x18
@@ -1178,9 +1167,8 @@ define i256 @f64_to_u256(double %val) {
; CHECK-SD-NEXT: sub x8, x8, #1075
; CHECK-SD-NEXT: str x9, [sp, #32]
; CHECK-SD-NEXT: lsr x9, x8, #3
-; CHECK-SD-NEXT: mov x10, sp
+; CHECK-SD-NEXT: add x10, sp, #32
; CHECK-SD-NEXT: str xzr, [sp, #56]
-; CHECK-SD-NEXT: add x10, x10, #32
; CHECK-SD-NEXT: and x13, x8, #0x3f
; CHECK-SD-NEXT: mvn w17, w8
; CHECK-SD-NEXT: and x9, x9, #0x18
@@ -1326,9 +1314,8 @@ define i256 @f32_to_s256_sat(float %val) {
; CHECK-SD-NEXT: sub w10, w10, #150
; CHECK-SD-NEXT: str x11, [sp, #32]
; CHECK-SD-NEXT: lsr x11, x10, #3
-; CHECK-SD-NEXT: mov x12, sp
+; CHECK-SD-NEXT: add x12, sp, #32
; CHECK-SD-NEXT: str xzr, [sp, #56]
-; CHECK-SD-NEXT: add x12, x12, #32
; CHECK-SD-NEXT: and x16, x10, #0x3f
; CHECK-SD-NEXT: mvn w2, w10
; CHECK-SD-NEXT: and x11, x11, #0x18
@@ -1584,9 +1571,8 @@ define i256 @f32_to_u256_sat(float %val) {
; CHECK-SD-NEXT: sub w8, w8, #150
; CHECK-SD-NEXT: str x9, [sp, #32]
; CHECK-SD-NEXT: lsr x9, x8, #3
-; CHECK-SD-NEXT: mov x10, sp
+; CHECK-SD-NEXT: add x10, sp, #32
; CHECK-SD-NEXT: str xzr, [sp, #56]
-; CHECK-SD-NEXT: add x10, x10, #32
; CHECK-SD-NEXT: and x13, x8, #0x3f
; CHECK-SD-NEXT: mvn w17, w8
; CHECK-SD-NEXT: and x9, x9, #0x18
@@ -1756,9 +1742,8 @@ define i256 @f64_to_s256_sat(double %val) {
; CHECK-SD-NEXT: sub x10, x10, #1075
; CHECK-SD-NEXT: str x11, [sp, #32]
; CHECK-SD-NEXT: lsr x11, x10, #3
-; CHECK-SD-NEXT: mov x12, sp
+; CHECK-SD-NEXT: add x12, sp, #32
; CHECK-SD-NEXT: str xzr, [sp, #56]
-; CHECK-SD-NEXT: add x12, x12, #32
; CHECK-SD-NEXT: and x16, x10, #0x3f
; CHECK-SD-NEXT: mvn w2, w10
; CHECK-SD-NEXT: and x11, x11, #0x18
@@ -2011,9 +1996,8 @@ define i256 @f64_to_u256_sat(double %val) {
; CHECK-SD-NEXT: sub x8, x8, #1075
; CHECK-SD-NEXT: str x9, [sp, #32]
; CHECK-SD-NEXT: lsr x9, x8, #3
-; CHECK-SD-NEXT: mov x10, sp
+; CHECK-SD-NEXT: add x10, sp, #32
; CHECK-SD-NEXT: str xzr, [sp, #56]
-; CHECK-SD-NEXT: add x10, x10, #32
; CHECK-SD-NEXT: and x13, x8, #0x3f
; CHECK-SD-NEXT: mvn w17, w8
; CHECK-SD-NEXT: and x9, x9, #0x18
diff --git a/llvm/test/CodeGen/AArch64/frameindex-offset-fold.ll b/llvm/test/CodeGen/AArch64/frameindex-offset-fold.ll
index 9db4d32e66e80..db72398c109ab 100644
--- a/llvm/test/CodeGen/AArch64/frameindex-offset-fold.ll
+++ b/llvm/test/CodeGen/AArch64/frameindex-offset-fold.ll
@@ -15,8 +15,7 @@ define void @alloca_offset_8() {
; CHECK-NEXT: str x30, [sp, #64] // 8-byte Spill
; CHECK-NEXT: .cfi_def_cfa_offset 80
; CHECK-NEXT: .cfi_offset w30, -16
-; CHECK-NEXT: mov x8, sp
-; CHECK-NEXT: add x0, x8, #8
+; CHECK-NEXT: add x0, sp, #8
; CHECK-NEXT: bl use
; CHECK-NEXT: ldr x30, [sp, #64] // 8-byte Reload
; CHECK-NEXT: add sp, sp, #80
@@ -35,8 +34,7 @@ define void @alloca_offset_4095() {
; CHECK-NEXT: .cfi_def_cfa_offset 8208
; CHECK-NEXT: .cfi_offset w30, -8
; CHECK-NEXT: .cfi_offset w29, -16
-; CHECK-NEXT: mov x8, sp
-; CHECK-NEXT: add x0, x8, #4095
+; CHECK-NEXT: add x0, sp, #4095
; CHECK-NEXT: bl use
; CHECK-NEXT: add sp, sp, #2, lsl #12 // =8192
; CHECK-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload
@@ -73,16 +71,14 @@ define void @alloca_two_interior_ptrs() {
; CHECK-LABEL: alloca_two_interior_ptrs:
; CHECK: // %bb.0:
; CHECK-NEXT: sub sp, sp, #80
-; CHECK-NEXT: stp x30, x19, [sp, #64] // 16-byte Folded Spill
+; CHECK-NEXT: str x30, [sp, #64] // 8-byte Spill
; CHECK-NEXT: .cfi_def_cfa_offset 80
-; CHECK-NEXT: .cfi_offset w19, -8
; CHECK-NEXT: .cfi_offset w30, -16
-; CHECK-NEXT: mov x19, sp
-; CHECK-NEXT: add x0, x19, #8
+; CHECK-NEXT: add x0, sp, #8
; CHECK-NEXT: bl use
-; CHECK-NEXT: add x0, x19, #16
+; CHECK-NEXT: add x0, sp, #16
; CHECK-NEXT: bl use
-; CHECK-NEXT: ldp x30, x19, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEXT: ldr x30, [sp, #64] // 8-byte Reload
; CHECK-NEXT: add sp, sp, #80
; CHECK-NEXT: ret
%a = alloca [64 x i8], align 8
@@ -98,16 +94,14 @@ define void @alloca_base_and_interior() {
; CHECK-LABEL: alloca_base_and_interior:
; CHECK: // %bb.0:
; CHECK-NEXT: sub sp, sp, #80
-; CHECK-NEXT: stp x30, x19, [sp, #64] // 16-byte Folded Spill
+; CHECK-NEXT: str x30, [sp, #64] // 8-byte Spill
; CHECK-NEXT: .cfi_def_cfa_offset 80
-; CHECK-NEXT: .cfi_offset w19, -8
; CHECK-NEXT: .cfi_offset w30, -16
; CHECK-NEXT: mov x0, sp
-; CHECK-NEXT: mov x19, sp
; CHECK-NEXT: bl use
-; CHECK-NEXT: add x0, x19, #8
+; CHECK-NEXT: add x0, sp, #8
; CHECK-NEXT: bl use
-; CHECK-NEXT: ldp x30, x19, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEXT: ldr x30, [sp, #64] // 8-byte Reload
; CHECK-NEXT: add sp, sp, #80
; CHECK-NEXT: ret
%a = alloca [64 x i8], align 8
diff --git a/llvm/test/CodeGen/AArch64/named-vector-shuffles-neon.ll b/llvm/test/CodeGen/AArch64/named-vector-shuffles-neon.ll
index dc7c3cbf9459c..a8805da2eaf6d 100644
--- a/llvm/test/CodeGen/AArch64/named-vector-shuffles-neon.ll
+++ b/llvm/test/CodeGen/AArch64/named-vector-shuffles-neon.ll
@@ -151,11 +151,10 @@ define <4 x i32> @splice_right_v4i32_variable_offset(<4 x i32> %a, <4 x i32> %b,
; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0
; CHECK-NEXT: ubfiz x8, x0, #2, #32
; CHECK-NEXT: mov w9, #16 // =0x10
-; CHECK-NEXT: mov x10, sp
; CHECK-NEXT: stp q0, q1, [sp]
; CHECK-NEXT: cmp x8, #16
; CHECK-NEXT: csel x8, x8, x9, lo
-; CHECK-NEXT: add x9, x10, #16
+; CHECK-NEXT: add x9, sp, #16
; CHECK-NEXT: sub x8, x9, x8
; CHECK-NEXT: ldr q0, [x8]
; CHECK-NEXT: add sp, sp, #32
diff --git a/llvm/test/CodeGen/AArch64/named-vector-shuffles-sve.ll b/llvm/test/CodeGen/AArch64/named-vector-shuffles-sve.ll
index 747c134913931..a3d268cd9608a 100644
--- a/llvm/test/CodeGen/AArch64/named-vector-shuffles-sve.ll
+++ b/llvm/test/CodeGen/AArch64/named-vector-shuffles-sve.ll
@@ -497,7 +497,7 @@ define <vscale x 16 x float> @splice_nxv16f32_16(<vscale x 16 x float> %a, <vsca
; CHECK-NEXT: str z5, [sp, #5, mul vl]
; CHECK-NEXT: str z6, [sp, #6, mul vl]
; CHECK-NEXT: ld1w { z0.s }, p0/z, [x9, x8, lsl #2]
-; CHECK-NEXT: add x8, x9, #64
+; CHECK-NEXT: add x8, sp, #64
; CHECK-NEXT: ldr z1, [x8, #1, mul vl]
; CHECK-NEXT: ldr z2, [x8, #2, mul vl]
; CHECK-NEXT: ldr z3, [x8, #3, mul vl]
diff --git a/llvm/test/CodeGen/AArch64/preserve_nonecc_varargs_aapcs.ll b/llvm/test/CodeGen/AArch64/preserve_nonecc_varargs_aapcs.ll
index a359343eacf79..6ccad8ecf6c0d 100644
--- a/llvm/test/CodeGen/AArch64/preserve_nonecc_varargs_aapcs.ll
+++ b/llvm/test/CodeGen/AArch64/preserve_nonecc_varargs_aapcs.ll
@@ -8,14 +8,12 @@ define preserve_nonecc i32 @callee(i32 %a1, i32 %a2, i32 %a3, i32 %a4, i32 %a5,
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: sub sp, sp, #192
; CHECK-NEXT: mov x8, #-24 // =0xffffffffffffffe8
-; CHECK-NEXT: mov x9, sp
-; CHECK-NEXT: add x10, sp, #136
+; CHECK-NEXT: add x9, sp, #128
+; CHECK-NEXT: add x10, sp, #192
; CHECK-NEXT: movk x8, #65408, lsl #32
-; CHECK-NEXT: add x9, x9, #128
; CHECK-NEXT: stp x6, x7, [sp, #144]
; CHECK-NEXT: stp x9, x8, [sp, #176]
-; CHECK-NEXT: add x9, x10, #24
-; CHECK-NEXT: add x10, sp, #192
+; CHECK-NEXT: add x9, sp, #160
; CHECK-NEXT: mov w8, #-24 // =0xffffffe8
; CHECK-NEXT: str x5, [sp, #136]
; CHECK-NEXT: stp q0, q1, [sp]
diff --git a/llvm/test/CodeGen/AArch64/shift-i256.ll b/llvm/test/CodeGen/AArch64/shift-i256.ll
index cde8144643575..c19062247909f 100644
--- a/llvm/test/CodeGen/AArch64/shift-i256.ll
+++ b/llvm/test/CodeGen/AArch64/shift-i256.ll
@@ -8,31 +8,30 @@ define i256 @shl_i256(i256 %a, i256 %amt) nounwind {
; CHECK-NEXT: sub sp, sp, #64
; CHECK-NEXT: movi v0.2d, #0000000000000000
; CHECK-NEXT: lsr x8, x4, #3
-; CHECK-NEXT: mov x9, sp
-; CHECK-NEXT: add x9, x9, #32
+; CHECK-NEXT: add x9, sp, #32
; CHECK-NEXT: stp x2, x3, [sp, #48]
-; CHECK-NEXT: mvn w14, w4
+; CHECK-NEXT: and x11, x4, #0x3f
+; CHECK-NEXT: mvn w12, w4
; CHECK-NEXT: and x8, x8, #0x18
; CHECK-NEXT: stp x0, x1, [sp, #32]
-; CHECK-NEXT: and x11, x4, #0x3f
-; CHECK-NEXT: sub x8, x9, x8
; CHECK-NEXT: eor x11, x11, #0x3f
+; CHECK-NEXT: sub x8, x9, x8
; CHECK-NEXT: stp q0, q0, [sp]
; CHECK-NEXT: ldp x10, x9, [x8]
-; CHECK-NEXT: ldp x12, x8, [x8, #16]
-; CHECK-NEXT: lsr x13, x9, #1
-; CHECK-NEXT: lsr x15, x10, #1
-; CHECK-NEXT: lsl x9, x9, x4
+; CHECK-NEXT: ldp x13, x8, [x8, #16]
+; CHECK-NEXT: lsr x14, x10, #1
+; CHECK-NEXT: lsl x15, x9, x4
+; CHECK-NEXT: lsr x9, x9, #1
+; CHECK-NEXT: lsr x16, x13, #1
+; CHECK-NEXT: lsl x13, x13, x4
; CHECK-NEXT: lsl x8, x8, x4
+; CHECK-NEXT: lsr x14, x14, x11
; CHECK-NEXT: lsl x0, x10, x4
-; CHECK-NEXT: lsr x13, x13, x14
-; CHECK-NEXT: lsr x14, x12, #1
-; CHECK-NEXT: lsr x15, x15, x11
-; CHECK-NEXT: lsl x12, x12, x4
-; CHECK-NEXT: lsr x11, x14, x11
-; CHECK-NEXT: orr x1, x9, x15
-; CHECK-NEXT: orr x2, x12, x13
-; CHECK-NEXT: orr x3, x8, x11
+; CHECK-NEXT: lsr x9, x9, x12
+; CHECK-NEXT: lsr x10, x16, x11
+; CHECK-NEXT: orr x1, x15, x14
+; CHECK-NEXT: orr x2, x13, x9
+; CHECK-NEXT: orr x3, x8, x10
; CHECK-NEXT: add sp, sp, #64
; CHECK-NEXT: ret
%r = shl i256 %a, %amt
diff --git a/llvm/test/CodeGen/AArch64/soft-float-abi.ll b/llvm/test/CodeGen/AArch64/soft-float-abi.ll
index 291c3875c2488..4546dab437dd9 100644
--- a/llvm/test/CodeGen/AArch64/soft-float-abi.ll
+++ b/llvm/test/CodeGen/AArch64/soft-float-abi.ll
@@ -98,10 +98,9 @@ define double @test20(i32 %a, ...) {
; CHECK-NEXT: sub sp, sp, #96
; CHECK-NEXT: .cfi_def_cfa_offset 96
; CHECK-NEXT: mov w8, #-56 // =0xffffffc8
-; CHECK-NEXT: add x10, sp, #8
; CHECK-NEXT: add x9, sp, #96
+; CHECK-NEXT: add x10, sp, #64
; CHECK-NEXT: str x8, [sp, #88]
-; CHECK-NEXT: add x10, x10, #56
; CHECK-NEXT: ldrsw x8, [sp, #88]
; CHECK-NEXT: stp x1, x2, [sp, #8]
; CHECK-NEXT: stp x3, x4, [sp, #24]
diff --git a/llvm/test/CodeGen/AArch64/sve-extract-fixed-from-scalable-vector.ll b/llvm/test/CodeGen/AArch64/sve-extract-fixed-from-scalable-vector.ll
index 3473b063e35ee..232a3fdae9096 100644
--- a/llvm/test/CodeGen/AArch64/sve-extract-fixed-from-scalable-vector.ll
+++ b/llvm/test/CodeGen/AArch64/sve-extract-fixed-from-scalable-vector.ll
@@ -67,10 +67,9 @@ define <2 x i16> @extract_v2i16_nxv32i16_8(<vscale x 32 x i16> %arg) {
; CHECK-NEXT: addvl sp, sp, #-8
; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0xc0, 0x00, 0x1e, 0x22 // sp + 16 + 64 * VG
; CHECK-NEXT: .cfi_offset w29, -16
-; CHECK-NEXT: mov x8, sp
; CHECK-NEXT: str z3, [sp, #3, mul vl]
+; CHECK-NEXT: add x8, sp, #32
; CHECK-NEXT: str z2, [sp, #2, mul vl]
-; CHECK-NEXT: add x8, x8, #32
; CHECK-NEXT: str z1, [sp, #1, mul vl]
; CHECK-NEXT: str z0, [sp]
; CHECK-NEXT: str z3, [sp, #7, mul vl]
@@ -78,8 +77,8 @@ define <2 x i16> @extract_v2i16_nxv32i16_8(<vscale x 32 x i16> %arg) {
; CHECK-NEXT: str z1, [sp, #5, mul vl]
; CHECK-NEXT: str z0, [sp, #4, mul vl]
; CHECK-NEXT: ld1 { v0.h }[0], [x8]
-; CHECK-NEXT: addvl x8, sp, #4
-; CHECK-NEXT: add x8, x8, #34
+; CHECK-NEXT: add x8, sp, #34
+; CHECK-NEXT: addvl x8, x8, #4
; CHECK-NEXT: ld1 { v0.h }[2], [x8]
; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-NEXT: addvl sp, sp, #8
@@ -210,10 +209,9 @@ define <4 x i3> @extract_v4i3_nxv32i3_16(<vscale x 32 x i3> %arg) {
; CHECK-NEXT: addvl sp, sp, #-8
; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0xc0, 0x00, 0x1e, 0x22 // sp + 16 + 64 * VG
; CHECK-NEXT: .cfi_offset w29, -16
-; CHECK-NEXT: mov x8, sp
+; CHECK-NEXT: add x8, sp, #16
; CHECK-NEXT: str z1, [sp, #1, mul vl]
; CHECK-NEXT: str z0, [sp]
-; CHECK-NEXT: add x8, x8, #16
; CHECK-NEXT: str z1, [sp, #3, mul vl]
; CHECK-NEXT: str z0, [sp, #2, mul vl]
; CHECK-NEXT: str z1, [sp, #5, mul vl]
@@ -221,14 +219,14 @@ define <4 x i3> @extract_v4i3_nxv32i3_16(<vscale x 32 x i3> %arg) {
; CHECK-NEXT: str z1, [sp, #7, mul vl]
; CHECK-NEXT: str z0, [sp, #6, mul vl]
; CHECK-NEXT: ld1 { v0.b }[0], [x8]
-; CHECK-NEXT: addvl x8, sp, #2
-; CHECK-NEXT: add x8, x8, #17
+; CHECK-NEXT: add x8, sp, #17
+; CHECK-NEXT: addvl x8, x8, #2
; CHECK-NEXT: ld1 { v0.b }[2], [x8]
-; CHECK-NEXT: addvl x8, sp, #4
-; CHECK-NEXT: add x8, x8, #18
+; CHECK-NEXT: add x8, sp, #18
+; CHECK-NEXT: addvl x8, x8, #4
; CHECK-NEXT: ld1 { v0.b }[4], [x8]
-; CHECK-NEXT: addvl x8, sp, #6
-; CHECK-NEXT: add x8, x8, #19
+; CHECK-NEXT: add x8, sp, #19
+; CHECK-NEXT: addvl x8, x8, #6
; CHECK-NEXT: ld1 { v0.b }[6], [x8]
; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-NEXT: addvl sp, sp, #8
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-ldst-alias-analysis.ll b/llvm/test/CodeGen/AArch64/sve-masked-ldst-alias-analysis.ll
index 6bca834340291..22623337bb31c 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-ldst-alias-analysis.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-ldst-alias-analysis.ll
@@ -9,12 +9,11 @@ define <vscale x 16 x i8> @reverse_masked_load() nounwind {
; CHECK-NEXT: sub sp, sp, #16
; CHECK-NEXT: ptrue p0.b, vl6
; CHECK-NEXT: mov w8, #1799 // =0x707
-; CHECK-NEXT: add x9, sp, #8
; CHECK-NEXT: strh w8, [sp, #12]
; CHECK-NEXT: mov w8, #117901063 // =0x7070707
; CHECK-NEXT: rev p0.b, p0.b
; CHECK-NEXT: str w8, [sp, #8]
-; CHECK-NEXT: add x8, x9, #6
+; CHECK-NEXT: add x8, sp, #14
; CHECK-NEXT: ld1b { z0.b }, p0/z, [x8, #-1, mul vl]
; CHECK-NEXT: add sp, sp, #16
; CHECK-NEXT: ret
diff --git a/llvm/test/CodeGen/AArch64/trampoline.ll b/llvm/test/CodeGen/AArch64/trampoline.ll
index 12e3738b13c5b..a42b5ac92d61e 100644
--- a/llvm/test/CodeGen/AArch64/trampoline.ll
+++ b/llvm/test/CodeGen/AArch64/trampoline.ll
@@ -80,6 +80,7 @@ define i64 @func1() {
; CHECK-LINUX-NEXT: add x0, sp, #8
; CHECK-LINUX-NEXT: ldr x8, [x8, :got_lo12:f]
; CHECK-LINUX-NEXT: movk w9, #54815, lsl #16
+; CHECK-LINUX-NEXT: add x1, sp, #20
; CHECK-LINUX-NEXT: str w9, [sp, #16]
; CHECK-LINUX-NEXT: add x9, sp, #56
; CHECK-LINUX-NEXT: stp x9, x8, [sp, #24]
@@ -88,8 +89,6 @@ define i64 @func1() {
; CHECK-LINUX-NEXT: movk x8, #177, lsl #32
; CHECK-LINUX-NEXT: movk x8, #22528, lsl #48
; CHECK-LINUX-NEXT: str x8, [sp, #8]
-; CHECK-LINUX-NEXT: add x8, sp, #8
-; CHECK-LINUX-NEXT: add x1, x8, #12
; CHECK-LINUX-NEXT: bl __clear_cache
; CHECK-LINUX-NEXT: ldr x30, [sp, #48] // 8-byte Reload
; CHECK-LINUX-NEXT: mov x0, xzr
@@ -111,14 +110,13 @@ define i64 @func1() {
; CHECK-PC-NEXT: mov w8, #544 // =0x220
; CHECK-PC-NEXT: add x0, sp, #8
; CHECK-PC-NEXT: movk w8, #54815, lsl #16
+; CHECK-PC-NEXT: add x1, sp, #20
; CHECK-PC-NEXT: str w8, [sp, #16]
; CHECK-PC-NEXT: mov x8, #143 // =0x8f
; CHECK-PC-NEXT: movk x8, #22528, lsl #16
; CHECK-PC-NEXT: movk x8, #177, lsl #32
; CHECK-PC-NEXT: movk x8, #22528, lsl #48
; CHECK-PC-NEXT: str x8, [sp, #8]
-; CHECK-PC-NEXT: add x8, sp, #8
-; CHECK-PC-NEXT: add x1, x8, #12
; CHECK-PC-NEXT: bl __clear_cache
; CHECK-PC-NEXT: mov x0, xzr
; CHECK-PC-NEXT: .seh_startepilogue
@@ -147,14 +145,13 @@ define i64 @func1() {
; CHECK-APPLE-NEXT: mov w8, #544 ; =0x220
; CHECK-APPLE-NEXT: mov x0, sp
; CHECK-APPLE-NEXT: movk w8, #54815, lsl #16
+; CHECK-APPLE-NEXT: add x1, sp, #12
; CHECK-APPLE-NEXT: str w8, [sp, #8]
; CHECK-APPLE-NEXT: mov x8, #143 ; =0x8f
; CHECK-APPLE-NEXT: movk x8, #22528, lsl #16
; CHECK-APPLE-NEXT: movk x8, #177, lsl #32
; CHECK-APPLE-NEXT: movk x8, #22528, lsl #48
; CHECK-APPLE-NEXT: str x8, [sp]
-; CHECK-APPLE-NEXT: mov x8, sp
-; CHECK-APPLE-NEXT: add x1, x8, #12
; CHECK-APPLE-NEXT: bl ___clear_cache
; CHECK-APPLE-NEXT: ldp x29, x30, [sp, #48] ; 16-byte Folded Reload
; CHECK-APPLE-NEXT: mov x0, xzr
diff --git a/llvm/test/CodeGen/AArch64/vararg.ll b/llvm/test/CodeGen/AArch64/vararg.ll
index 291eee2ddf706..30a8cf37ea23f 100644
--- a/llvm/test/CodeGen/AArch64/vararg.ll
+++ b/llvm/test/CodeGen/AArch64/vararg.ll
@@ -43,14 +43,12 @@ define i64 @vararg(...) #0 {
; CHECK-SD-NEXT: str w8, [x29, #20]
; CHECK-SD-NEXT: mov w8, #-64 // =0xffffffc0
; CHECK-SD-NEXT: str w8, [x29, #16]
-; CHECK-SD-NEXT: add x8, x29, #16
-; CHECK-SD-NEXT: stur x8, [x29, #-8]
-; CHECK-SD-NEXT: mov x8, sp
-; CHECK-SD-NEXT: add x8, x8, #128
+; CHECK-SD-NEXT: add x8, sp, #128
; CHECK-SD-NEXT: str x8, [x29, #8]
-; CHECK-SD-NEXT: sub x8, x29, #72
-; CHECK-SD-NEXT: add x8, x8, #64
+; CHECK-SD-NEXT: sub x8, x29, #8
; CHECK-SD-NEXT: str x8, [x29]
+; CHECK-SD-NEXT: add x8, x29, #16
+; CHECK-SD-NEXT: stur x8, [x29, #-8]
; CHECK-SD-NEXT: mov w8, #1 // =0x1
; CHECK-SD-NEXT: mov w0, w8
; CHECK-SD-NEXT: .cfi_def_cfa wsp, 224
@@ -135,14 +133,12 @@ define i64 @vararg_many_gpr(i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6
; CHECK-SD-NEXT: str w8, [x29, #20]
; CHECK-SD-NEXT: mov w8, #-8 // =0xfffffff8
; CHECK-SD-NEXT: str w8, [x29, #16]
-; CHECK-SD-NEXT: add x8, x29, #16
-; CHECK-SD-NEXT: stur x8, [x29, #-8]
-; CHECK-SD-NEXT: mov x8, sp
-; CHECK-SD-NEXT: add x8, x8, #128
+; CHECK-SD-NEXT: add x8, sp, #128
; CHECK-SD-NEXT: str x8, [x29, #8]
-; CHECK-SD-NEXT: sub x8, x29, #16
-; CHECK-SD-NEXT: add x8, x8, #8
+; CHECK-SD-NEXT: sub x8, x29, #8
; CHECK-SD-NEXT: str x8, [x29]
+; CHECK-SD-NEXT: add x8, x29, #16
+; CHECK-SD-NEXT: stur x8, [x29, #-8]
; CHECK-SD-NEXT: mov w8, #1 // =0x1
; CHECK-SD-NEXT: mov w0, w8
; CHECK-SD-NEXT: .cfi_def_cfa wsp, 160
@@ -220,14 +216,12 @@ define i64 @vararg_many_float(float %a1, float %a2, float %a3, float %a4, float
; CHECK-SD-NEXT: str w8, [x29, #20]
; CHECK-SD-NEXT: mov w8, #-64 // =0xffffffc0
; CHECK-SD-NEXT: str w8, [x29, #16]
-; CHECK-SD-NEXT: add x8, x29, #16
-; CHECK-SD-NEXT: stur x8, [x29, #-8]
-; CHECK-SD-NEXT: mov x8, sp
-; CHECK-SD-NEXT: add x8, x8, #16
+; CHECK-SD-NEXT: add x8, sp, #16
; CHECK-SD-NEXT: str x8, [x29, #8]
-; CHECK-SD-NEXT: add x8, sp, #24
-; CHECK-SD-NEXT: add x8, x8, #64
+; CHECK-SD-NEXT: add x8, sp, #88
; CHECK-SD-NEXT: str x8, [x29]
+; CHECK-SD-NEXT: add x8, x29, #16
+; CHECK-SD-NEXT: stur x8, [x29, #-8]
; CHECK-SD-NEXT: mov w8, #1 // =0x1
; CHECK-SD-NEXT: mov w0, w8
; CHECK-SD-NEXT: .cfi_def_cfa wsp, 112
@@ -310,14 +304,12 @@ define i64 @gpr1_fpr1(i32 %i, float %f, ...) #0 {
; CHECK-SD-NEXT: str w8, [x29, #20]
; CHECK-SD-NEXT: mov w8, #-56 // =0xffffffc8
; CHECK-SD-NEXT: str w8, [x29, #16]
-; CHECK-SD-NEXT: add x8, x29, #16
-; CHECK-SD-NEXT: stur x8, [x29, #-8]
-; CHECK-SD-NEXT: mov x8, sp
-; CHECK-SD-NEXT: add x8, x8, #112
+; CHECK-SD-NEXT: add x8, sp, #112
; CHECK-SD-NEXT: str x8, [x29, #8]
-; CHECK-SD-NEXT: sub x8, x29, #64
-; CHECK-SD-NEXT: add x8, x8, #56
+; CHECK-SD-NEXT: sub x8, x29, #8
; CHECK-SD-NEXT: str x8, [x29]
+; CHECK-SD-NEXT: add x8, x29, #16
+; CHECK-SD-NEXT: stur x8, [x29, #-8]
; CHECK-SD-NEXT: mov w8, #1 // =0x1
; CHECK-SD-NEXT: mov w0, w8
; CHECK-SD-NEXT: .cfi_def_cfa wsp, 192
diff --git a/llvm/test/CodeGen/AArch64/wide-scalar-shift-by-byte-multiple-legalization.ll b/llvm/test/CodeGen/AArch64/wide-scalar-shift-by-byte-multiple-legalization.ll
index b02788ab1b34c..8fb7c49740444 100644
--- a/llvm/test/CodeGen/AArch64/wide-scalar-shift-by-byte-multiple-legalization.ll
+++ b/llvm/test/CodeGen/AArch64/wide-scalar-shift-by-byte-multiple-legalization.ll
@@ -257,16 +257,15 @@ define void @shl_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; ALL-NEXT: ldr x10, [x1]
; ALL-NEXT: ldr q1, [x0]
; ALL-NEXT: stp x9, x8, [sp, #48]
-; ALL-NEXT: mov x8, sp
+; ALL-NEXT: add x8, sp, #32
; ALL-NEXT: and x9, x10, #0x18
-; ALL-NEXT: add x8, x8, #32
-; ALL-NEXT: stp q0, q0, [sp]
-; ALL-NEXT: str q1, [sp, #32]
+; ALL-NEXT: stp q0, q1, [sp, #16]
; ALL-NEXT: sub x8, x8, x9
; ALL-NEXT: lsl x9, x10, #3
+; ALL-NEXT: str q0, [sp]
; ALL-NEXT: ldp x10, x11, [x8]
-; ALL-NEXT: ldp x12, x8, [x8, #16]
; ALL-NEXT: mvn w13, w9
+; ALL-NEXT: ldp x12, x8, [x8, #16]
; ALL-NEXT: and x9, x9, #0x38
; ALL-NEXT: lsr x14, x10, #1
; ALL-NEXT: lsr x15, x11, #1
@@ -302,12 +301,11 @@ define void @shl_32bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) nou
; ALL-NEXT: ldr x10, [x1]
; ALL-NEXT: ldr q1, [x0]
; ALL-NEXT: stp x9, x8, [sp, #48]
-; ALL-NEXT: mov x8, sp
-; ALL-NEXT: ubfiz x9, x10, #3, #2
-; ALL-NEXT: add x8, x8, #32
+; ALL-NEXT: ubfiz x8, x10, #3, #2
+; ALL-NEXT: add x9, sp, #32
; ALL-NEXT: stp q0, q1, [sp, #16]
; ALL-NEXT: str q0, [sp]
-; ALL-NEXT: sub x8, x8, x9
+; ALL-NEXT: sub x8, x9, x8
; ALL-NEXT: ldp x9, x10, [x8, #16]
; ALL-NEXT: ldr q0, [x8]
; ALL-NEXT: str q0, [x2]
diff --git a/llvm/test/CodeGen/AArch64/wide-scalar-shift-legalization.ll b/llvm/test/CodeGen/AArch64/wide-scalar-shift-legalization.ll
index 92fd4fe30980c..059ba54edae42 100644
--- a/llvm/test/CodeGen/AArch64/wide-scalar-shift-legalization.ll
+++ b/llvm/test/CodeGen/AArch64/wide-scalar-shift-legalization.ll
@@ -204,8 +204,7 @@ define void @shl_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
; ALL-NEXT: ldr q1, [x0]
; ALL-NEXT: stp x9, x8, [sp, #48]
; ALL-NEXT: lsr x8, x10, #3
-; ALL-NEXT: mov x9, sp
-; ALL-NEXT: add x9, x9, #32
+; ALL-NEXT: add x9, sp, #32
; ALL-NEXT: stp q0, q1, [sp, #16]
; ALL-NEXT: and x12, x10, #0x3f
; ALL-NEXT: and x8, x8, #0x18
>From 9ec0d397db6e1d8a0a6c4df4914f170afe96efdc Mon Sep 17 00:00:00 2001
From: Andrew Gaul <andrew at gaul.org>
Date: Sat, 5 Sep 2026 11:09:48 -0700
Subject: [PATCH 3/3] [AArch64] Select the frame-index offset fold via a
TableGen pattern
Replace the manual ISel code with a pattern built on the generic
to_tframeindex SDNodeXForm, and drop -verify-machineinstrs from the
early-ifcvt test. The pattern uses an ImmLeaf rather than
addsub_shifted_imm64 so that an LSL-12 immediate, whose shift the frame
index rewrite ignores, cannot match, and it carries an AddedComplexity
above the generic add-immediate instruction pattern so the frame index
is not materialised separately. Codegen is unchanged.
Co-Authored-By: Claude Fable 5 <noreply at anthropic.com>
Claude-Session: https://claude.ai/code/session_01RN4gpjgX8QWXRuQoBzYace
---
.../Target/AArch64/AArch64ISelDAGToDAG.cpp | 50 -------------------
llvm/lib/Target/AArch64/AArch64InstrInfo.td | 17 +++++++
.../AArch64/early-ifcvt-frame-index.mir | 2 +-
3 files changed, 18 insertions(+), 51 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index 30eaca99b4a52..48c0e37151b9c 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -457,8 +457,6 @@ class AArch64DAGToDAGISel : public SelectionDAGISel {
bool tryFoldCselToFMaxMin(SDNode *N);
- bool tryFoldFrameIndexOffset(SDNode *N);
-
// Include the pieces autogenerated from the target description.
#include "AArch64GenDAGISel.inc"
@@ -5193,49 +5191,6 @@ static SDValue getZeroRegister(SelectionDAG &DAG, SDLoc DL, EVT VT) {
VT == MVT::i32 ? AArch64::WZR : AArch64::XZR, VT);
}
-/// Fold a constant offset into the ADDXri that materialises a frame index:
-/// (add (FrameIndex FI), C) -> ADDXri <FI>, C
-///
-/// A bare frame index selects to "ADDXri <FI>, 0" (see the ISD::FrameIndex
-/// case below), and PrologEpilogInserter adds whatever immediate that
-/// instruction already carries to the resolved frame offset, so an interior
-/// pointer costs no more than the base pointer does. Selecting the add
-/// separately instead costs a second ADDXri, and frequently an extra live
-/// register too, since several interior pointers off one frame index all keep
-/// the materialised base alive.
-bool AArch64DAGToDAGISel::tryFoldFrameIndexOffset(SDNode *N) {
- if (N->getValueType(0) != MVT::i64)
- return false;
-
- SDValue Base = N->getOperand(0);
- if (Base.getOpcode() != ISD::FrameIndex)
- return false;
-
- auto *Off = dyn_cast<ConstantSDNode>(N->getOperand(1));
- if (!Off)
- return false;
-
- // rewriteAArch64FrameIndex() reads the immediate operand of the ADDXri it
- // resolves but ignores the shift operand, so only fold offsets that the
- // unshifted immediate can hold. A larger offset keeps its own add, which
- // is what would have been emitted anyway.
- uint64_t Offset = Off->getZExtValue();
- if (Offset > 0xFFF)
- return false;
-
- SDLoc DL(N);
- const TargetLowering *TLI = getTargetLowering();
- SDValue TFI =
- CurDAG->getTargetFrameIndex(cast<FrameIndexSDNode>(Base)->getIndex(),
- TLI->getPointerTy(CurDAG->getDataLayout()));
- SDValue Ops[] = {
- TFI, CurDAG->getTargetConstant(Offset, DL, MVT::i32),
- CurDAG->getTargetConstant(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0),
- DL, MVT::i32)};
- CurDAG->SelectNodeTo(N, AArch64::ADDXri, MVT::i64, Ops);
- return true;
-}
-
void AArch64DAGToDAGISel::Select(SDNode *Node) {
// If we have a custom node, we already have selected!
if (Node->isMachineOpcode()) {
@@ -5303,11 +5258,6 @@ void AArch64DAGToDAGISel::Select(SDNode *Node) {
return;
break;
- case ISD::ADD:
- if (tryFoldFrameIndexOffset(Node))
- return;
- break;
-
case ISD::EXTRACT_SUBVECTOR: {
if (trySelectCastScalableToFixedLengthVector(Node))
return;
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index 3c2fd32f83d75..ad2f4d1e76c9b 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -2891,6 +2891,23 @@ def : Pat<(AArch64sub_flag GPR64:$Rn, neg_addsub_shifted_imm64:$imm),
(ADDSXri GPR64:$Rn, neg_addsub_shifted_imm64:$imm)>;
}
+def imm0_4095 : ImmLeaf<i64, [{
+ return ((uint64_t)Imm) < 4096;
+}]>;
+
+// The ADDXri that materialises a frame index can absorb a constant offset,
+// because rewriteAArch64FrameIndex() adds the resolved frame offset to the
+// immediate the instruction already carries. The rewrite ignores the shift
+// operand, so only offsets that fit the unshifted immediate may be folded;
+// a larger offset keeps its own add off the materialised base, which is what
+// would have been emitted anyway. The complexity bump raises this above the
+// generic add-immediate instruction pattern (AddedComplexity 6 plus the
+// ComplexPattern priority bonus), which would leave the frame index to be
+// materialised as a separate ADDXri.
+let AddedComplexity = 20 in
+def : Pat<(add (i64 frameindex:$Rn), (i64 imm0_4095:$offset)),
+ (ADDXri (i64 (to_tframeindex $Rn)), (trunc_imm $offset), 0)>;
+
def negate_imm : SDNodeXForm<imm, [{
return CurDAG->getTargetConstant(-N->getAPIntValue(), SDLoc(N), N->getValueType(0));
}]>;
diff --git a/llvm/test/CodeGen/AArch64/early-ifcvt-frame-index.mir b/llvm/test/CodeGen/AArch64/early-ifcvt-frame-index.mir
index e1a4fe87534e7..cb142401022f1 100644
--- a/llvm/test/CodeGen/AArch64/early-ifcvt-frame-index.mir
+++ b/llvm/test/CodeGen/AArch64/early-ifcvt-frame-index.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -mtriple=aarch64-- -run-pass=early-ifcvt -stress-early-ifcvt -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=aarch64-- -run-pass=early-ifcvt -stress-early-ifcvt %s -o - | FileCheck %s
# RUN: llc -mtriple=aarch64-- -passes=early-ifcvt -stress-early-ifcvt %s -o - | FileCheck %s
# An ADDXri that materialises a frame index holds a frame index, not a
More information about the llvm-commits
mailing list