[llvm] [LFI][AArch64] Add rewrites for memory accesses (PR #195167)
Zachary Yedidia via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 30 12:46:39 PDT 2026
https://github.com/zyedidia updated https://github.com/llvm/llvm-project/pull/195167
>From 9a48268636e1f1f784926c933bb57be16bace86b Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia at gmail.com>
Date: Wed, 29 Apr 2026 21:40:01 -0400
Subject: [PATCH 1/2] [LFI] Add rewrites for memory accesses
---
llvm/docs/LFI.rst | 6 -
llvm/lib/Target/AArch64/AArch64Features.td | 5 +
llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 32 +-
.../MCTargetDesc/AArch64MCLFIRewriter.cpp | 963 +++++++++++++++++-
.../MCTargetDesc/AArch64MCLFIRewriter.h | 31 +
llvm/test/MC/AArch64/LFI/exclusive.s | 140 +++
llvm/test/MC/AArch64/LFI/fp.s | 204 ++++
llvm/test/MC/AArch64/LFI/jumps-only.s | 41 +
llvm/test/MC/AArch64/LFI/literal.s | 37 +
llvm/test/MC/AArch64/LFI/lse.s | 166 +++
llvm/test/MC/AArch64/LFI/mem.s | 507 +++++++++
llvm/test/MC/AArch64/LFI/no-lfi-loads.s | 33 +
llvm/test/MC/AArch64/LFI/no-lfi-stores.s | 35 +
llvm/test/MC/AArch64/LFI/prefetch.s | 81 ++
llvm/test/MC/AArch64/LFI/rcpc.s | 19 +
llvm/test/MC/AArch64/LFI/simd.s | 650 ++++++++++++
llvm/test/MC/AArch64/LFI/stack.s | 41 +
llvm/test/MC/AArch64/LFI/sys.s | 24 +
18 files changed, 3005 insertions(+), 10 deletions(-)
create mode 100644 llvm/test/MC/AArch64/LFI/exclusive.s
create mode 100644 llvm/test/MC/AArch64/LFI/fp.s
create mode 100644 llvm/test/MC/AArch64/LFI/jumps-only.s
create mode 100644 llvm/test/MC/AArch64/LFI/literal.s
create mode 100644 llvm/test/MC/AArch64/LFI/lse.s
create mode 100644 llvm/test/MC/AArch64/LFI/mem.s
create mode 100644 llvm/test/MC/AArch64/LFI/no-lfi-loads.s
create mode 100644 llvm/test/MC/AArch64/LFI/no-lfi-stores.s
create mode 100644 llvm/test/MC/AArch64/LFI/prefetch.s
create mode 100644 llvm/test/MC/AArch64/LFI/rcpc.s
create mode 100644 llvm/test/MC/AArch64/LFI/simd.s
create mode 100644 llvm/test/MC/AArch64/LFI/stack.s
diff --git a/llvm/docs/LFI.rst b/llvm/docs/LFI.rst
index a173cd57e0ee0..fb28195fa23d4 100644
--- a/llvm/docs/LFI.rst
+++ b/llvm/docs/LFI.rst
@@ -115,8 +115,6 @@ Example:
Compiler Options
++++++++++++++++
-**Note**: these options are not yet implemented.
-
The LFI target has several configuration options, specified via ``-mattr=``:
* ``+no-lfi-loads``: Disable sandboxing for load instructions (stores-only mode).
@@ -197,8 +195,6 @@ require any rewrite.
Memory accesses
~~~~~~~~~~~~~~~
-**Note**: not yet implemented.
-
Memory accesses are rewritten to use the ``[x27, wM, uxtw]`` addressing mode if
it is available, which is automatically safe. Otherwise, rewrites fall back to
using ``x28`` along with an instruction to safely load it with the target
@@ -279,8 +275,6 @@ address.
Stack pointer modification
~~~~~~~~~~~~~~~~~~~~~~~~~~
-**Note**: not yet implemented.
-
When the stack pointer is modified, we write the modified value to a temporary,
before moving it back into ``sp`` with a safe ``add``.
diff --git a/llvm/lib/Target/AArch64/AArch64Features.td b/llvm/lib/Target/AArch64/AArch64Features.td
index 8cd2278a05747..ab77106eaa1bc 100644
--- a/llvm/lib/Target/AArch64/AArch64Features.td
+++ b/llvm/lib/Target/AArch64/AArch64Features.td
@@ -1075,6 +1075,11 @@ def FeatureHardenSlsNoComdat : SubtargetFeature<"harden-sls-nocomdat",
"Generate thunk code for SLS mitigation in the normal text section">;
+def FeatureNoLFILoads : SubtargetFeature<"no-lfi-loads", "NoLFILoads", "true",
+ "Disable LFI sandboxing for load instructions">;
+def FeatureNoLFIStores : SubtargetFeature<"no-lfi-stores", "NoLFIStores", "true",
+ "Disable LFI sandboxing for store instructions">;
+
// Only intended to be used by disassemblers.
def FeatureAll
: SubtargetFeature<"all", "IsAll", "true", "Enable all instructions">;
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index 844adc3eb54c4..decb3572c6638 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -16,6 +16,7 @@
#include "AArch64PointerAuth.h"
#include "AArch64Subtarget.h"
#include "MCTargetDesc/AArch64AddressingModes.h"
+#include "MCTargetDesc/AArch64MCLFIRewriter.h"
#include "MCTargetDesc/AArch64MCTargetDesc.h"
#include "Utils/AArch64BaseInfo.h"
#include "llvm/ADT/ArrayRef.h"
@@ -129,14 +130,39 @@ static std::optional<unsigned> getLFIInstSizeInBytes(const MachineInstr &MI) {
if (MI.getOperand(0).getReg() != AArch64::LR)
return 8;
return 4;
+ case AArch64::SYSxt:
+ // VA-based DC/IC ops (op1=3, Cn=7, op2=1) expand to 2 instructions.
+ if (MI.getOperand(0).getImm() == 3 && MI.getOperand(1).getImm() == 7 &&
+ MI.getOperand(3).getImm() == 1)
+ return 8;
+ return std::nullopt;
default:
break;
}
- // Instructions that explicitly modify LR expand to 2 instructions.
+ // Instructions that explicitly modify LR get an extra LR mask.
+ bool ModifiesLR = false;
for (const MachineOperand &MO : MI.explicit_operands())
- if (MO.isReg() && MO.isDef() && MO.getReg() == AArch64::LR)
- return 8;
+ if (MO.isReg() && MO.isDef() && MO.getReg() == AArch64::LR) {
+ ModifiesLR = true;
+ break;
+ }
+
+ // Memory accesses expand to a base-register guard plus the rewritten access
+ // (8 bytes), with an extra base-register update for pre/post-index forms (12
+ // bytes total). If the access also defines LR, an LR mask is appended (+4
+ // bytes). Depending on additional optimizations that the rewriter performs,
+ // this may be an overestimate.
+ if (MI.mayLoadOrStore()) {
+ unsigned Size = isLFIPrePostMemAccess(MI.getOpcode()) ? 12 : 8;
+ if (ModifiesLR)
+ Size += 4;
+ return Size;
+ }
+
+ // Non-memory instructions that modify LR expand to 2 instructions.
+ if (ModifiesLR)
+ return 8;
// Default case: instructions that don't cause expansion.
// - TP accesses in LFI are a single load/store, so no expansion.
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIRewriter.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIRewriter.cpp
index 6066ddb7e59a9..6a81a59341c0a 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIRewriter.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIRewriter.cpp
@@ -17,6 +17,8 @@
#include "Utils/AArch64BaseInfo.h"
#include "llvm/MC/MCInst.h"
+#include "llvm/MC/MCInstrDesc.h"
+#include "llvm/MC/MCInstrInfo.h"
#include "llvm/MC/MCStreamer.h"
#include "llvm/MC/MCSubtargetInfo.h"
@@ -64,6 +66,97 @@ static bool isPrivilegedTPAccess(const MCInst &Inst) {
return false;
}
+// Instructions that have mayLoad/mayStore set in TableGen but don't actually
+// perform memory accesses.
+static bool isNotMemAccess(const MCInst &Inst) {
+ switch (Inst.getOpcode()) {
+ case AArch64::DMB:
+ case AArch64::DSB:
+ case AArch64::ISB:
+ case AArch64::HINT:
+ return true;
+ default:
+ return false;
+ }
+}
+
+static bool mayPrefetch(const MCInst &Inst) {
+ switch (Inst.getOpcode()) {
+ case AArch64::PRFMl:
+ case AArch64::PRFMroW:
+ case AArch64::PRFMroX:
+ case AArch64::PRFMui:
+ case AArch64::PRFUMi:
+ return true;
+ default:
+ return false;
+ }
+}
+
+// User-mode DC/IC instructions that take a virtual address operand. Encoded as
+// SYSxt with op1=3, Cn=7, op2=1 where the Cm field selects the operation.
+static bool isVASysOp(const MCInst &Inst) {
+ if (Inst.getOpcode() != AArch64::SYSxt)
+ return false;
+ if (Inst.getOperand(0).getImm() != 3 || Inst.getOperand(1).getImm() != 7 ||
+ Inst.getOperand(3).getImm() != 1)
+ return false;
+ switch (Inst.getOperand(2).getImm()) {
+ case 4: // DC ZVA
+ case 5: // IC IVAU
+ case 10: // DC CVAC
+ case 11: // DC CVAU
+ case 12: // DC CVAP
+ case 13: // DC CVADP
+ case 14: // DC CIVAC
+ return true;
+ default:
+ return false;
+ }
+}
+
+static MCInst replaceRegAt(const MCInst &Inst, unsigned Idx,
+ MCRegister NewReg) {
+ MCInst New;
+ New.setOpcode(Inst.getOpcode());
+ New.setLoc(Inst.getLoc());
+ for (unsigned I = 0, E = Inst.getNumOperands(); I != E; ++I) {
+ if (I == Idx)
+ New.addOperand(MCOperand::createReg(NewReg));
+ else
+ New.addOperand(Inst.getOperand(I));
+ }
+ return New;
+}
+
+// Memory instruction information for the basic load/store rewriting path.
+// Provides operand indices for the base register and offset, and whether
+// the instruction uses pre/post-indexed addressing.
+struct MemInstInfo {
+ int BaseRegIdx;
+ int OffsetIdx; // -1 if no offset operand.
+ bool IsPrePost;
+ bool IsLiteral;
+};
+
+// Returns memory instruction info for a given opcode, or std::nullopt if
+// the opcode is not a recognized memory instruction.
+static std::optional<MemInstInfo> getMemInstInfo(unsigned Op);
+
+static unsigned convertUiToRoW(unsigned Op);
+static unsigned convertPreToRoW(unsigned Op);
+static unsigned convertPostToRoW(unsigned Op);
+static unsigned convertRoXToRoW(unsigned Op, unsigned &Shift);
+static bool getRoWShift(unsigned Op, unsigned &Shift);
+static unsigned getPrePostScale(unsigned Op);
+static unsigned convertPrePostToBase(unsigned Op, bool &IsPre,
+ bool &IsNoOffset);
+static int getSIMDNaturalOffset(unsigned Op);
+
+bool AArch64MCLFIRewriter::mayModifyStack(const MCInst &Inst) const {
+ return mayModifyRegister(Inst, AArch64::SP);
+}
+
bool AArch64MCLFIRewriter::mayModifyReserved(const MCInst &Inst) const {
return mayModifyRegister(Inst, LFIAddrReg) ||
mayModifyRegister(Inst, LFIBaseReg) ||
@@ -111,6 +204,75 @@ void AArch64MCLFIRewriter::emitMov(MCRegister Dest, MCRegister Src,
emitInst(Inst, Out, STI);
}
+void AArch64MCLFIRewriter::emitAddImm(MCRegister Dest, MCRegister Src,
+ int64_t Imm, MCStreamer &Out,
+ const MCSubtargetInfo &STI) {
+ assert(std::abs(Imm) <= 4095);
+ MCInst Inst;
+ if (Imm >= 0) {
+ Inst.setOpcode(AArch64::ADDXri);
+ Inst.addOperand(MCOperand::createReg(Dest));
+ Inst.addOperand(MCOperand::createReg(Src));
+ Inst.addOperand(MCOperand::createImm(Imm));
+ Inst.addOperand(MCOperand::createImm(0)); // shift
+ } else {
+ Inst.setOpcode(AArch64::SUBXri);
+ Inst.addOperand(MCOperand::createReg(Dest));
+ Inst.addOperand(MCOperand::createReg(Src));
+ Inst.addOperand(MCOperand::createImm(-Imm));
+ Inst.addOperand(MCOperand::createImm(0)); // shift
+ }
+ emitInst(Inst, Out, STI);
+}
+
+void AArch64MCLFIRewriter::emitAddReg(MCRegister Dest, MCRegister Src1,
+ MCRegister Src2, unsigned Shift,
+ MCStreamer &Out,
+ const MCSubtargetInfo &STI) {
+ // add Dest, Src1, Src2, lsl #Shift
+ MCInst Inst;
+ Inst.setOpcode(AArch64::ADDXrs);
+ Inst.addOperand(MCOperand::createReg(Dest));
+ Inst.addOperand(MCOperand::createReg(Src1));
+ Inst.addOperand(MCOperand::createReg(Src2));
+ Inst.addOperand(
+ MCOperand::createImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, Shift)));
+ emitInst(Inst, Out, STI);
+}
+
+void AArch64MCLFIRewriter::emitAddRegExtend(MCRegister Dest, MCRegister Src1,
+ MCRegister Src2,
+ AArch64_AM::ShiftExtendType ExtType,
+ unsigned Shift, MCStreamer &Out,
+ const MCSubtargetInfo &STI) {
+ // add Dest, Src1, Src2, ExtType #Shift
+ MCInst Inst;
+ if (ExtType == AArch64_AM::SXTX || ExtType == AArch64_AM::UXTX)
+ Inst.setOpcode(AArch64::ADDXrx64);
+ else
+ Inst.setOpcode(AArch64::ADDXrx);
+ Inst.addOperand(MCOperand::createReg(Dest));
+ Inst.addOperand(MCOperand::createReg(Src1));
+ Inst.addOperand(MCOperand::createReg(Src2));
+ Inst.addOperand(
+ MCOperand::createImm(AArch64_AM::getArithExtendImm(ExtType, Shift)));
+ emitInst(Inst, Out, STI);
+}
+
+void AArch64MCLFIRewriter::emitMemRoW(unsigned Opcode, const MCOperand &DataOp,
+ MCRegister BaseReg, MCStreamer &Out,
+ const MCSubtargetInfo &STI) {
+ // Op DataOp, [LFIBaseReg, W(BaseReg), uxtw]
+ MCInst Inst;
+ Inst.setOpcode(Opcode);
+ Inst.addOperand(DataOp);
+ Inst.addOperand(MCOperand::createReg(LFIBaseReg));
+ Inst.addOperand(MCOperand::createReg(getWRegFromXReg(BaseReg)));
+ Inst.addOperand(MCOperand::createImm(0)); // S bit = 0 (UXTW).
+ Inst.addOperand(MCOperand::createImm(0)); // Shift amount = 0 (unscaled).
+ emitInst(Inst, Out, STI);
+}
+
// {br,blr} xN
// ->
// add x28, x27, wN, uxtw
@@ -151,7 +313,11 @@ void AArch64MCLFIRewriter::rewriteReturn(const MCInst &Inst, MCStreamer &Out,
void AArch64MCLFIRewriter::rewriteLRModification(const MCInst &Inst,
MCStreamer &Out,
const MCSubtargetInfo &STI) {
- emitInst(Inst, Out, STI);
+ if (!isNotMemAccess(Inst) &&
+ (mayLoad(Inst) || mayStore(Inst) || mayPrefetch(Inst)))
+ rewriteLoadStore(Inst, Out, STI);
+ else
+ emitInst(Inst, Out, STI);
emitAddMask(AArch64::LR, AArch64::LR, Out, STI);
}
@@ -211,6 +377,254 @@ void AArch64MCLFIRewriter::rewriteTPWrite(const MCInst &Inst, MCStreamer &Out,
emitInst(Store, Out, STI);
}
+bool AArch64MCLFIRewriter::rewriteLoadStoreRoW(const MCInst &Inst,
+ MCStreamer &Out,
+ const MCSubtargetInfo &STI) {
+ unsigned Op = Inst.getOpcode();
+ unsigned MemOp;
+
+ // Case 1: Indexed load/store with zero immediate offset.
+ // ldr xN, [xM, #0] -> ldr xN, [x27, wM, uxtw]
+ if ((MemOp = convertUiToRoW(Op)) != AArch64::INSTRUCTION_LIST_END) {
+ MCRegister BaseReg = Inst.getOperand(1).getReg();
+ if (BaseReg == AArch64::SP)
+ return false;
+ const MCOperand &OffsetOp = Inst.getOperand(2);
+ if (OffsetOp.isImm() && OffsetOp.getImm() == 0) {
+ emitMemRoW(MemOp, Inst.getOperand(0), BaseReg, Out, STI);
+ return true;
+ }
+ return false;
+ }
+
+ // Case 2: Pre-index load/store.
+ // ldr xN, [xM, #imm]! -> add xM, xM, #imm; ldr xN, [x27, wM, uxtw]
+ if ((MemOp = convertPreToRoW(Op)) != AArch64::INSTRUCTION_LIST_END) {
+ MCRegister BaseReg = Inst.getOperand(2).getReg();
+ if (BaseReg == AArch64::SP)
+ return false;
+ int64_t Imm = Inst.getOperand(3).getImm();
+ emitAddImm(BaseReg, BaseReg, Imm, Out, STI);
+ emitMemRoW(MemOp, Inst.getOperand(1), BaseReg, Out, STI);
+ return true;
+ }
+
+ // Case 3: Post-index load/store.
+ // ldr xN, [xM], #imm -> ldr xN, [x27, wM, uxtw]; add xM, xM, #imm
+ if ((MemOp = convertPostToRoW(Op)) != AArch64::INSTRUCTION_LIST_END) {
+ MCRegister BaseReg = Inst.getOperand(2).getReg();
+ if (BaseReg == AArch64::SP)
+ return false;
+ int64_t Imm = Inst.getOperand(3).getImm();
+ emitMemRoW(MemOp, Inst.getOperand(1), BaseReg, Out, STI);
+ emitAddImm(BaseReg, BaseReg, Imm, Out, STI);
+ return true;
+ }
+
+ // Case 4: Register-offset-X load/store.
+ // ldr xN, [xM1, xM2] -> add x26, xM1, xM2; ldr xN, [x27, w26, uxtw]
+ unsigned Shift;
+ if ((MemOp = convertRoXToRoW(Op, Shift)) != AArch64::INSTRUCTION_LIST_END) {
+ MCRegister Reg1 = Inst.getOperand(1).getReg();
+ MCRegister Reg2 = Inst.getOperand(2).getReg();
+ int64_t Extend = Inst.getOperand(3).getImm();
+ int64_t IsShift = Inst.getOperand(4).getImm();
+
+ if (!IsShift)
+ Shift = 0;
+
+ if (Extend)
+ emitAddRegExtend(LFIScratchReg, Reg1, Reg2, AArch64_AM::SXTX, Shift, Out,
+ STI);
+ else
+ emitAddReg(LFIScratchReg, Reg1, Reg2, Shift, Out, STI);
+ emitMemRoW(MemOp, Inst.getOperand(0), LFIScratchReg, Out, STI);
+ return true;
+ }
+
+ // Case 5: Register-offset-W load/store.
+ // ldr xN, [xM1, wM2, uxtw] -> add x26, xM1, wM2, uxtw;
+ // ldr xN, [x27, w26, uxtw]
+ if (getRoWShift(Op, Shift)) {
+ MCRegister Reg1 = Inst.getOperand(1).getReg();
+ MCRegister Reg2 = Inst.getOperand(2).getReg();
+ int64_t S = Inst.getOperand(3).getImm();
+ int64_t IsShift = Inst.getOperand(4).getImm();
+
+ if (!IsShift)
+ Shift = 0;
+
+ if (S)
+ emitAddRegExtend(LFIScratchReg, Reg1, Reg2, AArch64_AM::SXTW, Shift, Out,
+ STI);
+ else
+ emitAddRegExtend(LFIScratchReg, Reg1, Reg2, AArch64_AM::UXTW, Shift, Out,
+ STI);
+ emitMemRoW(Op, Inst.getOperand(0), LFIScratchReg, Out, STI);
+ return true;
+ }
+
+ return false;
+}
+
+void AArch64MCLFIRewriter::rewriteLoadStoreBasic(const MCInst &Inst,
+ MCStreamer &Out,
+ const MCSubtargetInfo &STI) {
+ unsigned Opcode = Inst.getOpcode();
+ auto Info = getMemInstInfo(Opcode);
+
+ if (!Info)
+ return error(Inst, "unknown addressing mode for memory instruction in LFI");
+
+ if (Info->IsLiteral)
+ return error(Inst, "PC-relative literal loads are not supported in LFI");
+
+ MCRegister BaseReg = Inst.getOperand(Info->BaseRegIdx).getReg();
+
+ // Stack accesses don't need address sandboxing, except when sp is modified
+ // with a register post-index operand.
+ bool BaseIsSP = BaseReg == AArch64::SP;
+ if (BaseIsSP) {
+ if (Info->OffsetIdx < 0 || !Inst.getOperand(Info->OffsetIdx).isReg())
+ return emitInst(Inst, Out, STI);
+ MCRegister OffReg = Inst.getOperand(Info->OffsetIdx).getReg();
+ if (OffReg == AArch64::XZR || OffReg == AArch64::WZR)
+ return emitInst(Inst, Out, STI);
+ }
+
+ // Guard the base register, unless it is SP.
+ MCRegister AccessBase = BaseIsSP ? AArch64::SP : LFIAddrReg;
+ if (!BaseIsSP)
+ emitAddMask(LFIAddrReg, BaseReg, Out, STI);
+
+ if (Info->IsPrePost) {
+ bool IsPre = false;
+ bool IsNoOffset = false;
+ unsigned BaseOpcode = convertPrePostToBase(Opcode, IsPre, IsNoOffset);
+
+ if (BaseOpcode != AArch64::INSTRUCTION_LIST_END) {
+ // Demote pre/post-index to base indexed form.
+ MCInst NewInst;
+ NewInst.setOpcode(BaseOpcode);
+ NewInst.setLoc(Inst.getLoc());
+
+ // Skip writeback operand (operand 0) and copy data operands up to base.
+ for (int I = 1; I < Info->BaseRegIdx; ++I)
+ NewInst.addOperand(Inst.getOperand(I));
+
+ // Add the access base register (LFIAddrReg or SP).
+ NewInst.addOperand(MCOperand::createReg(AccessBase));
+
+ // For pre-index, include the offset; for post-index, use zero.
+ if (IsPre && Info->OffsetIdx >= 0)
+ NewInst.addOperand(Inst.getOperand(Info->OffsetIdx));
+ else if (!IsNoOffset)
+ NewInst.addOperand(MCOperand::createImm(0));
+
+ emitInst(NewInst, Out, STI);
+
+ // Update the base register with the offset. If the base is SP, a
+ // register offset must be sandboxed (the result is otherwise
+ // unbounded), and ADDXrs cannot take SP, so the extended-register form
+ // via the scratch register is used.
+ if (Info->OffsetIdx >= 0) {
+ const MCOperand &OffsetOp = Inst.getOperand(Info->OffsetIdx);
+ if (OffsetOp.isImm()) {
+ int64_t Scale = getPrePostScale(Opcode);
+ int64_t Offset = OffsetOp.getImm() * Scale;
+ emitAddImm(BaseReg, BaseReg, Offset, Out, STI);
+ } else if (OffsetOp.isReg()) {
+ // SIMD post-index uses a register offset (XZR for natural offset).
+ MCRegister OffReg = OffsetOp.getReg();
+ if (OffReg == AArch64::XZR) {
+ int NaturalOffset = getSIMDNaturalOffset(Opcode);
+ if (NaturalOffset > 0)
+ emitAddImm(BaseReg, BaseReg, NaturalOffset, Out, STI);
+ } else if (OffReg != AArch64::WZR) {
+ if (BaseIsSP) {
+ emitAddRegExtend(LFIScratchReg, AArch64::SP, OffReg,
+ AArch64_AM::UXTX, 0, Out, STI);
+ emitAddMask(AArch64::SP, LFIScratchReg, Out, STI);
+ } else {
+ emitAddReg(BaseReg, BaseReg, OffReg, 0, Out, STI);
+ }
+ }
+ }
+ }
+ } else {
+ error(Inst, "unhandled pre/post-index instruction in LFI rewriter");
+ }
+ } else {
+ // Non-pre/post instruction: replace the base register operand.
+ MCInst NewInst = replaceRegAt(Inst, Info->BaseRegIdx, LFIAddrReg);
+ emitInst(NewInst, Out, STI);
+ }
+}
+
+void AArch64MCLFIRewriter::rewriteLoadStore(const MCInst &Inst, MCStreamer &Out,
+ const MCSubtargetInfo &STI) {
+ bool IsStore = mayStore(Inst);
+ bool IsLoad = mayLoad(Inst) || mayPrefetch(Inst);
+
+ bool SkipLoads = STI.hasFeature(AArch64::FeatureNoLFILoads);
+ bool SkipStores = STI.hasFeature(AArch64::FeatureNoLFIStores);
+
+ if ((!IsLoad || SkipLoads) && (!IsStore || SkipStores))
+ return emitInst(Inst, Out, STI);
+
+ if (rewriteLoadStoreRoW(Inst, Out, STI))
+ return;
+
+ rewriteLoadStoreBasic(Inst, Out, STI);
+}
+
+// modify sp
+// ->
+// modify x26
+// add sp, x27, w26, uxtw
+void AArch64MCLFIRewriter::rewriteStackModification(
+ const MCInst &Inst, MCStreamer &Out, const MCSubtargetInfo &STI) {
+ // Route through rewriteLRModification or rewriteLoadStore for memory
+ // accesses. Those helpers automatically handle dangerous stack modifications
+ // that can happen via register post-index.
+ if (mayLoad(Inst) || mayStore(Inst)) {
+ if (mayModifyRegister(Inst, AArch64::LR))
+ return rewriteLRModification(Inst, Out, STI);
+ return rewriteLoadStore(Inst, Out, STI);
+ }
+
+ // No stack sandboxing if sandboxing is disabled for both loads and stores.
+ bool SkipLoads = STI.hasFeature(AArch64::FeatureNoLFILoads);
+ bool SkipStores = STI.hasFeature(AArch64::FeatureNoLFIStores);
+ if (SkipLoads && SkipStores)
+ return emitInst(Inst, Out, STI);
+
+ // Redirect SP modification destination to scratch, then sandbox.
+ MCInst ModInst = replaceRegAt(Inst, 0, LFIScratchReg);
+ emitInst(ModInst, Out, STI);
+ emitAddMask(AArch64::SP, LFIScratchReg, Out, STI);
+}
+
+// {dc,ic} <op>, xN
+// ->
+// add x28, x27, wN, uxtw
+// {dc,ic} <op>, x28
+void AArch64MCLFIRewriter::rewriteVASysOp(const MCInst &Inst, MCStreamer &Out,
+ const MCSubtargetInfo &STI) {
+ MCRegister AddrReg = Inst.getOperand(4).getReg();
+
+ emitAddMask(LFIAddrReg, AddrReg, Out, STI);
+
+ MCInst NewInst;
+ NewInst.setOpcode(AArch64::SYSxt);
+ NewInst.addOperand(Inst.getOperand(0));
+ NewInst.addOperand(Inst.getOperand(1));
+ NewInst.addOperand(Inst.getOperand(2));
+ NewInst.addOperand(Inst.getOperand(3));
+ NewInst.addOperand(MCOperand::createReg(LFIAddrReg));
+ emitInst(NewInst, Out, STI);
+}
+
// NOTE: when adding new rewrites, the size estimates in
// AArch64InstrInfo::getLFIInstSizeInBytes must be updated to match.
void AArch64MCLFIRewriter::doRewriteInst(const MCInst &Inst, MCStreamer &Out,
@@ -236,6 +650,9 @@ void AArch64MCLFIRewriter::doRewriteInst(const MCInst &Inst, MCStreamer &Out,
return;
}
+ if (isVASysOp(Inst))
+ return rewriteVASysOp(Inst, Out, STI);
+
// Control flow.
switch (Inst.getOpcode()) {
case AArch64::RET:
@@ -245,13 +662,36 @@ void AArch64MCLFIRewriter::doRewriteInst(const MCInst &Inst, MCStreamer &Out,
return rewriteIndirectBranch(Inst, Out, STI);
}
+ // Register modifications that require sandboxing.
+ if (mayModifyStack(Inst))
+ return rewriteStackModification(Inst, Out, STI);
+
// Link register modification.
if (explicitlyModifiesRegister(Inst, AArch64::LR))
return rewriteLRModification(Inst, Out, STI);
+ // Memory access.
+ if (!isNotMemAccess(Inst) &&
+ (mayLoad(Inst) || mayStore(Inst) || mayPrefetch(Inst)))
+ return rewriteLoadStore(Inst, Out, STI);
+
emitInst(Inst, Out, STI);
}
+// This function is made available to the size estimator so that it can
+// classify Pre/Post-index instructions.
+bool llvm::isLFIPrePostMemAccess(unsigned Opcode) {
+ if (convertPreToRoW(Opcode) != AArch64::INSTRUCTION_LIST_END)
+ return true;
+ if (convertPostToRoW(Opcode) != AArch64::INSTRUCTION_LIST_END)
+ return true;
+ bool IsPre, IsNoOffset;
+ if (convertPrePostToBase(Opcode, IsPre, IsNoOffset) !=
+ AArch64::INSTRUCTION_LIST_END)
+ return true;
+ return false;
+}
+
bool AArch64MCLFIRewriter::rewriteInst(const MCInst &Inst, MCStreamer &Out,
const MCSubtargetInfo &STI) {
// The guard prevents rewrite-recursion when we emit instructions from inside
@@ -265,3 +705,524 @@ bool AArch64MCLFIRewriter::rewriteInst(const MCInst &Inst, MCStreamer &Out,
Guard = false;
return true;
}
+
+// Opcode X-macro Tables
+//
+// These macros define groups of related opcodes and are used to generate
+// multiple switch tables without repetition. Each macro takes a callback X and
+// invokes X(NAME, VALUE) for each entry.
+
+// Scalar memory ops that have ui, pre, post, roX, and roW variants.
+// PRFM is excluded because it has no pre/post forms.
+// The second column is the log2 element size (shift for scaled addressing).
+#define SCALAR_MEM_OPS(X) \
+ X(LDRBB, 0) \
+ X(LDRB, 0) \
+ X(LDRSBW, 0) \
+ X(LDRSBX, 0) \
+ X(STRBB, 0) \
+ X(STRB, 0) \
+ X(LDRHH, 1) \
+ X(LDRH, 1) \
+ X(LDRSHW, 1) \
+ X(LDRSHX, 1) \
+ X(STRHH, 1) \
+ X(STRH, 1) \
+ X(LDRSW, 2) \
+ X(LDRS, 2) \
+ X(LDRW, 2) \
+ X(STRS, 2) \
+ X(STRW, 2) \
+ X(LDRD, 3) \
+ X(LDRX, 3) \
+ X(STRD, 3) \
+ X(STRX, 3) \
+ X(LDRQ, 4) \
+ X(STRQ, 4)
+
+// LDP/STP pair ops. The second column is the scale factor for pre/post
+// immediates.
+#define PAIR_OPS(X) \
+ X(LDPD, 8) \
+ X(LDPQ, 16) \
+ X(LDPSW, 4) \
+ X(LDPS, 4) \
+ X(LDPW, 4) \
+ X(LDPX, 8) \
+ X(STPD, 8) \
+ X(STPQ, 16) \
+ X(STPS, 4) \
+ X(STPW, 4) \
+ X(STPX, 8)
+
+// SIMD post-index ops, split into three groups by operand layout.
+// The second column is the natural byte offset for post-index.
+
+// Lane stores: Vt, idx, [Rn] (base=2) / wback, Vt, idx, [Rn], Xm (base=3).
+
+// clang-format off
+#define SIMD_LANE_STORE_OPS(X) \
+ X(ST1i8, 1) X(ST1i16, 2) X(ST1i32, 4) X(ST1i64, 8) \
+ X(ST2i8, 2) X(ST2i16, 4) X(ST2i32, 8) X(ST2i64, 16) \
+ X(ST3i8, 3) X(ST3i16, 6) X(ST3i32, 12) X(ST3i64, 24) \
+ X(ST4i8, 4) X(ST4i16, 8) X(ST4i32, 16) X(ST4i64, 32)
+
+// Lane loads: Vt(out), Vt(tied), idx, [Rn] (base=3) /
+// wback, Vt(out), Vt(tied), idx, [Rn], Xm (base=4).
+#define SIMD_LANE_LOAD_OPS(X) \
+ X(LD1i8, 1) X(LD1i16, 2) X(LD1i32, 4) X(LD1i64, 8) \
+ X(LD2i8, 2) X(LD2i16, 4) X(LD2i32, 8) X(LD2i64, 16) \
+ X(LD3i8, 3) X(LD3i16, 6) X(LD3i32, 12) X(LD3i64, 24) \
+ X(LD4i8, 4) X(LD4i16, 8) X(LD4i32, 16) X(LD4i64, 32)
+
+// Multiple structure and replicate: Vt, [Rn] (base=1) /
+// wback, Vt, [Rn], Xm (base=2).
+// Each line pairs the LD and ST variants (replicates are LD-only).
+#define SIMD_MULTI_OPS(X) \
+ /* Replicate loads (LD only). */ \
+ X(LD1Rv8b, 1) X(LD1Rv16b, 1) X(LD1Rv4h, 2) X(LD1Rv8h, 2) \
+ X(LD1Rv2s, 4) X(LD1Rv4s, 4) X(LD1Rv1d, 8) X(LD1Rv2d, 8) \
+ X(LD2Rv8b, 2) X(LD2Rv16b, 2) X(LD2Rv4h, 4) X(LD2Rv8h, 4) \
+ X(LD2Rv2s, 8) X(LD2Rv4s, 8) X(LD2Rv1d, 16) X(LD2Rv2d, 16) \
+ X(LD3Rv8b, 3) X(LD3Rv16b, 3) X(LD3Rv4h, 6) X(LD3Rv8h, 6) \
+ X(LD3Rv2s, 12) X(LD3Rv4s, 12) X(LD3Rv1d, 24) X(LD3Rv2d, 24) \
+ X(LD4Rv8b, 4) X(LD4Rv16b, 4) X(LD4Rv4h, 8) X(LD4Rv8h, 8) \
+ X(LD4Rv2s, 16) X(LD4Rv4s, 16) X(LD4Rv1d, 32) X(LD4Rv2d, 32) \
+ /* LD1/ST1 One register. */ \
+ X(LD1Onev8b, 8) X(LD1Onev16b, 16) \
+ X(LD1Onev4h, 8) X(LD1Onev8h, 16) \
+ X(LD1Onev2s, 8) X(LD1Onev4s, 16) \
+ X(LD1Onev1d, 8) X(LD1Onev2d, 16) \
+ X(ST1Onev8b, 8) X(ST1Onev16b, 16) \
+ X(ST1Onev4h, 8) X(ST1Onev8h, 16) \
+ X(ST1Onev2s, 8) X(ST1Onev4s, 16) \
+ X(ST1Onev1d, 8) X(ST1Onev2d, 16) \
+ /* LD1/ST1 Two registers. */ \
+ X(LD1Twov8b, 16) X(LD1Twov16b, 32) \
+ X(LD1Twov4h, 16) X(LD1Twov8h, 32) \
+ X(LD1Twov2s, 16) X(LD1Twov4s, 32) \
+ X(LD1Twov1d, 16) X(LD1Twov2d, 32) \
+ X(ST1Twov8b, 16) X(ST1Twov16b, 32) \
+ X(ST1Twov4h, 16) X(ST1Twov8h, 32) \
+ X(ST1Twov2s, 16) X(ST1Twov4s, 32) \
+ X(ST1Twov1d, 16) X(ST1Twov2d, 32) \
+ /* LD1/ST1 Three registers. */ \
+ X(LD1Threev8b, 24) X(LD1Threev16b, 48) \
+ X(LD1Threev4h, 24) X(LD1Threev8h, 48) \
+ X(LD1Threev2s, 24) X(LD1Threev4s, 48) \
+ X(LD1Threev1d, 24) X(LD1Threev2d, 48) \
+ X(ST1Threev8b, 24) X(ST1Threev16b, 48) \
+ X(ST1Threev4h, 24) X(ST1Threev8h, 48) \
+ X(ST1Threev2s, 24) X(ST1Threev4s, 48) \
+ X(ST1Threev1d, 24) X(ST1Threev2d, 48) \
+ /* LD1/ST1 Four registers. */ \
+ X(LD1Fourv8b, 32) X(LD1Fourv16b, 64) \
+ X(LD1Fourv4h, 32) X(LD1Fourv8h, 64) \
+ X(LD1Fourv2s, 32) X(LD1Fourv4s, 64) \
+ X(LD1Fourv1d, 32) X(LD1Fourv2d, 64) \
+ X(ST1Fourv8b, 32) X(ST1Fourv16b, 64) \
+ X(ST1Fourv4h, 32) X(ST1Fourv8h, 64) \
+ X(ST1Fourv2s, 32) X(ST1Fourv4s, 64) \
+ X(ST1Fourv1d, 32) X(ST1Fourv2d, 64) \
+ /* LD2/ST2 Two registers. */ \
+ X(LD2Twov8b, 16) X(LD2Twov16b, 32) \
+ X(LD2Twov4h, 16) X(LD2Twov8h, 32) \
+ X(LD2Twov2s, 16) X(LD2Twov4s, 32) X(LD2Twov2d, 32) \
+ X(ST2Twov8b, 16) X(ST2Twov16b, 32) \
+ X(ST2Twov4h, 16) X(ST2Twov8h, 32) \
+ X(ST2Twov2s, 16) X(ST2Twov4s, 32) X(ST2Twov2d, 32) \
+ /* LD3/ST3 Three registers. */ \
+ X(LD3Threev8b, 24) X(LD3Threev16b, 48) \
+ X(LD3Threev4h, 24) X(LD3Threev8h, 48) \
+ X(LD3Threev2s, 24) X(LD3Threev4s, 48) X(LD3Threev2d, 48) \
+ X(ST3Threev8b, 24) X(ST3Threev16b, 48) \
+ X(ST3Threev4h, 24) X(ST3Threev8h, 48) \
+ X(ST3Threev2s, 24) X(ST3Threev4s, 48) X(ST3Threev2d, 48) \
+ /* LD4/ST4 Four registers. */ \
+ X(LD4Fourv8b, 32) X(LD4Fourv16b, 64) \
+ X(LD4Fourv4h, 32) X(LD4Fourv8h, 64) \
+ X(LD4Fourv2s, 32) X(LD4Fourv4s, 64) X(LD4Fourv2d, 64) \
+ X(ST4Fourv8b, 32) X(ST4Fourv16b, 64) \
+ X(ST4Fourv4h, 32) X(ST4Fourv8h, 64) \
+ X(ST4Fourv2s, 32) X(ST4Fourv4s, 64) X(ST4Fourv2d, 64)
+// clang-format on
+
+// Union of all SIMD post-index ops.
+#define SIMD_POST_OPS(X) \
+ SIMD_LANE_STORE_OPS(X) \
+ SIMD_LANE_LOAD_OPS(X) \
+ SIMD_MULTI_OPS(X)
+
+// Memory Instruction Info Table
+//
+// Returns information about how to find the base register and offset operands
+// in a memory instruction, and whether it uses pre/post-indexed addressing.
+// Used by rewriteLoadStoreBasic as a fallback when RoW conversion isn't
+// possible.
+static std::optional<MemInstInfo> getMemInstInfo(unsigned Op) {
+ switch (Op) {
+ default:
+ return std::nullopt;
+
+ // PC-relative literal loads: Rt, label
+ case AArch64::LDRWl:
+ case AArch64::LDRXl:
+ case AArch64::LDRSWl:
+ case AArch64::LDRSl:
+ case AArch64::LDRDl:
+ case AArch64::LDRQl:
+ case AArch64::PRFMl:
+ return MemInstInfo{0, -1, false, true};
+
+ // Scalar indexed/unscaled/register-offset: Rt, [Rn, ...]
+#define X(NAME, S) \
+ case AArch64::NAME##ui: \
+ case AArch64::NAME##roW: \
+ case AArch64::NAME##roX:
+ SCALAR_MEM_OPS(X)
+#undef X
+ case AArch64::PRFMui:
+ case AArch64::PRFMroW:
+ case AArch64::PRFMroX:
+ // Unscaled variants.
+ case AArch64::LDURBBi:
+ case AArch64::LDURBi:
+ case AArch64::LDURDi:
+ case AArch64::LDURHHi:
+ case AArch64::LDURHi:
+ case AArch64::LDURQi:
+ case AArch64::LDURSBWi:
+ case AArch64::LDURSBXi:
+ case AArch64::LDURSHWi:
+ case AArch64::LDURSHXi:
+ case AArch64::LDURSWi:
+ case AArch64::LDURSi:
+ case AArch64::LDURWi:
+ case AArch64::LDURXi:
+ case AArch64::STURBBi:
+ case AArch64::STURBi:
+ case AArch64::STURDi:
+ case AArch64::STURHHi:
+ case AArch64::STURHi:
+ case AArch64::STURQi:
+ case AArch64::STURSi:
+ case AArch64::STURWi:
+ case AArch64::STURXi:
+ case AArch64::PRFUMi:
+ return MemInstInfo{1, 2, false, false};
+
+ // Scalar pre/post-index: wback, Rt, [Rn], #imm
+#define X(NAME, S) \
+ case AArch64::NAME##pre: \
+ case AArch64::NAME##post:
+ SCALAR_MEM_OPS(X)
+#undef X
+ return MemInstInfo{2, 3, true, false};
+
+ // Exclusive/acquire loads: Rt, [Rn]
+ case AArch64::LDXRB:
+ case AArch64::LDXRH:
+ case AArch64::LDXRW:
+ case AArch64::LDXRX:
+ case AArch64::LDAXRB:
+ case AArch64::LDAXRH:
+ case AArch64::LDAXRW:
+ case AArch64::LDAXRX:
+ case AArch64::LDARB:
+ case AArch64::LDARH:
+ case AArch64::LDARW:
+ case AArch64::LDARX:
+ case AArch64::LDLARB:
+ case AArch64::LDLARH:
+ case AArch64::LDLARW:
+ case AArch64::LDLARX:
+ // RCPC loads: Rt, [Rn]
+ case AArch64::LDAPRB:
+ case AArch64::LDAPRH:
+ case AArch64::LDAPRW:
+ case AArch64::LDAPRX:
+ // Store-release: Rt, [Rn]
+ case AArch64::STLRB:
+ case AArch64::STLRH:
+ case AArch64::STLRW:
+ case AArch64::STLRX:
+ case AArch64::STLLRB:
+ case AArch64::STLLRH:
+ case AArch64::STLLRW:
+ case AArch64::STLLRX:
+ // SIMD multiple/replicate (base form): Vt, [Rn]
+#define X(NAME, OFF) case AArch64::NAME:
+ SIMD_MULTI_OPS(X)
+#undef X
+ return MemInstInfo{1, -1, false, false};
+
+ // Exclusive stores: Ws, Rt, [Rn]
+ case AArch64::STXRB:
+ case AArch64::STXRH:
+ case AArch64::STXRW:
+ case AArch64::STXRX:
+ case AArch64::STLXRB:
+ case AArch64::STLXRH:
+ case AArch64::STLXRW:
+ case AArch64::STLXRX:
+ return MemInstInfo{2, -1, false, false};
+
+ // Exclusive pair loads: Rt, Rt2, [Rn]
+ case AArch64::LDXPW:
+ case AArch64::LDXPX:
+ case AArch64::LDAXPW:
+ case AArch64::LDAXPX:
+ return MemInstInfo{2, -1, false, false};
+
+ // Pair indexed loads/stores: Rt, Rt2, [Rn, #imm]
+#define X(NAME, SCALE) case AArch64::NAME##i:
+ PAIR_OPS(X)
+#undef X
+ case AArch64::LDNPDi:
+ case AArch64::LDNPQi:
+ case AArch64::LDNPSi:
+ case AArch64::LDNPWi:
+ case AArch64::LDNPXi:
+ case AArch64::STNPDi:
+ case AArch64::STNPQi:
+ case AArch64::STNPSi:
+ case AArch64::STNPWi:
+ case AArch64::STNPXi:
+ return MemInstInfo{2, 3, false, false};
+
+ // CAS: Rs(out), Rs(in, tied), Rt, [Rn]
+#define CAS_VARIANTS(NAME) \
+ case AArch64::NAME##B: \
+ case AArch64::NAME##H: \
+ case AArch64::NAME##W: \
+ case AArch64::NAME##X:
+ CAS_VARIANTS(CAS)
+ CAS_VARIANTS(CASA)
+ CAS_VARIANTS(CASL)
+ CAS_VARIANTS(CASAL)
+#undef CAS_VARIANTS
+ return MemInstInfo{3, -1, false, false};
+
+ // LSE atomics: Rs, Rt, [Rn] - all 16 size/ordering variants per operation.
+#define LSE_VARIANTS(NAME) \
+ case AArch64::NAME##B: \
+ case AArch64::NAME##H: \
+ case AArch64::NAME##W: \
+ case AArch64::NAME##X: \
+ case AArch64::NAME##AB: \
+ case AArch64::NAME##AH: \
+ case AArch64::NAME##AW: \
+ case AArch64::NAME##AX: \
+ case AArch64::NAME##LB: \
+ case AArch64::NAME##LH: \
+ case AArch64::NAME##LW: \
+ case AArch64::NAME##LX: \
+ case AArch64::NAME##ALB: \
+ case AArch64::NAME##ALH: \
+ case AArch64::NAME##ALW: \
+ case AArch64::NAME##ALX:
+ LSE_VARIANTS(LDADD)
+ LSE_VARIANTS(LDCLR)
+ LSE_VARIANTS(LDEOR)
+ LSE_VARIANTS(LDSET)
+ LSE_VARIANTS(LDSMAX)
+ LSE_VARIANTS(LDSMIN)
+ LSE_VARIANTS(LDUMAX)
+ LSE_VARIANTS(LDUMIN)
+ LSE_VARIANTS(SWP)
+#undef LSE_VARIANTS
+ // SIMD lane stores (base form): Vt, idx, [Rn]
+#define X(NAME, OFF) case AArch64::NAME:
+ SIMD_LANE_STORE_OPS(X)
+#undef X
+ return MemInstInfo{2, -1, false, false};
+
+ // SIMD lane loads (base form): Vt(out), Vt(tied), idx, [Rn]
+#define X(NAME, OFF) case AArch64::NAME:
+ SIMD_LANE_LOAD_OPS(X)
+#undef X
+ return MemInstInfo{3, -1, false, false};
+
+ // Exclusive store pairs: Ws, Rt, Rt2, [Rn]
+ case AArch64::STXPW:
+ case AArch64::STXPX:
+ case AArch64::STLXPW:
+ case AArch64::STLXPX:
+ return MemInstInfo{3, -1, false, false};
+
+ // Pair pre/post-index: wback, Rt, Rt2, [Rn, #imm]! / [Rn], #imm
+#define X(NAME, SCALE) \
+ case AArch64::NAME##pre: \
+ case AArch64::NAME##post:
+ PAIR_OPS(X)
+#undef X
+ return MemInstInfo{3, 4, true, false};
+
+ // CASP: Ws_Ws2(out), Ws_Ws2(in, tied), Wt_Wt2, [Rn]
+ case AArch64::CASPW:
+ case AArch64::CASPX:
+ case AArch64::CASPAW:
+ case AArch64::CASPAX:
+ case AArch64::CASPLW:
+ case AArch64::CASPLX:
+ case AArch64::CASPALW:
+ case AArch64::CASPALX:
+ return MemInstInfo{3, -1, false, false};
+
+ // SIMD multiple/replicate post-index: wback, Vt, [Rn], Xm
+#define X(NAME, OFF) case AArch64::NAME##_POST:
+ SIMD_MULTI_OPS(X)
+#undef X
+ return MemInstInfo{2, 3, true, false};
+
+ // SIMD lane store post-index: wback, Vt, idx, [Rn], Xm
+#define X(NAME, OFF) case AArch64::NAME##_POST:
+ SIMD_LANE_STORE_OPS(X)
+#undef X
+ return MemInstInfo{3, 4, true, false};
+
+ // SIMD lane load post-index: wback, Vt(out), Vt(tied), idx, [Rn], Xm
+#define X(NAME, OFF) case AArch64::NAME##_POST:
+ SIMD_LANE_LOAD_OPS(X)
+#undef X
+ return MemInstInfo{4, 5, true, false};
+ }
+}
+
+// RoW (Register-offset-W) Opcode Conversion Tables
+//
+// These tables convert various load/store addressing modes to the
+// register-offset-W form ([X27, Wn, uxtw]) which provides sandboxing in a
+// single instruction by zero-extending the 32-bit offset register.
+//
+// All scalar load/store instructions that support RoW addressing are listed
+// here. Each entry is (NAME, SHIFT) where NAME is the opcode base (e.g. LDRX)
+// and SHIFT is the log2 element size used for scaled addressing.
+static unsigned convertUiToRoW(unsigned Op) {
+ switch (Op) {
+#define X(NAME, S) \
+ case AArch64::NAME##ui: \
+ return AArch64::NAME##roW;
+ SCALAR_MEM_OPS(X)
+#undef X
+ case AArch64::PRFMui:
+ return AArch64::PRFMroW;
+ default:
+ return AArch64::INSTRUCTION_LIST_END;
+ }
+}
+
+static unsigned convertPreToRoW(unsigned Op) {
+ switch (Op) {
+#define X(NAME, S) \
+ case AArch64::NAME##pre: \
+ return AArch64::NAME##roW;
+ SCALAR_MEM_OPS(X)
+#undef X
+ default:
+ return AArch64::INSTRUCTION_LIST_END;
+ }
+}
+
+static unsigned convertPostToRoW(unsigned Op) {
+ switch (Op) {
+#define X(NAME, S) \
+ case AArch64::NAME##post: \
+ return AArch64::NAME##roW;
+ SCALAR_MEM_OPS(X)
+#undef X
+ default:
+ return AArch64::INSTRUCTION_LIST_END;
+ }
+}
+
+static unsigned convertRoXToRoW(unsigned Op, unsigned &Shift) {
+ Shift = 0;
+ switch (Op) {
+#define X(NAME, S) \
+ case AArch64::NAME##roX: \
+ Shift = S; \
+ return AArch64::NAME##roW;
+ SCALAR_MEM_OPS(X)
+#undef X
+ case AArch64::PRFMroX:
+ Shift = 3;
+ return AArch64::PRFMroW;
+ default:
+ return AArch64::INSTRUCTION_LIST_END;
+ }
+}
+
+static bool getRoWShift(unsigned Op, unsigned &Shift) {
+ Shift = 0;
+ switch (Op) {
+#define X(NAME, S) \
+ case AArch64::NAME##roW: \
+ Shift = S; \
+ return true;
+ SCALAR_MEM_OPS(X)
+#undef X
+ case AArch64::PRFMroW:
+ Shift = 3;
+ return true;
+ default:
+ return false;
+ }
+}
+
+// Pre/Post-Index to Base Conversion and Natural Offset Tables
+//
+// SIMD post-index instructions are listed once in SIMD_POST_OPS and used to
+// generate both convertPrePostToBase and getSIMDNaturalOffset.
+// Each entry is (NAME, OFFSET) where the post opcode is NAME##_POST, the base
+// opcode is NAME, and OFFSET is the natural byte increment.
+static unsigned convertPrePostToBase(unsigned Op, bool &IsPre,
+ bool &IsNoOffset) {
+ IsPre = false;
+ IsNoOffset = false;
+ switch (Op) {
+ // LDP/STP pairs.
+#define X(NAME, SCALE) \
+ case AArch64::NAME##post: \
+ return AArch64::NAME##i; \
+ case AArch64::NAME##pre: \
+ IsPre = true; \
+ return AArch64::NAME##i;
+ PAIR_OPS(X)
+#undef X
+ // SIMD post-index.
+#define X(NAME, OFF) \
+ case AArch64::NAME##_POST: \
+ IsNoOffset = true; \
+ return AArch64::NAME;
+ SIMD_POST_OPS(X)
+#undef X
+ default:
+ return AArch64::INSTRUCTION_LIST_END;
+ }
+}
+
+static unsigned getPrePostScale(unsigned Op) {
+ switch (Op) {
+#define X(NAME, SCALE) \
+ case AArch64::NAME##post: \
+ case AArch64::NAME##pre: \
+ return SCALE;
+ PAIR_OPS(X)
+#undef X
+ default:
+ return 1;
+ }
+}
+
+static int getSIMDNaturalOffset(unsigned Op) {
+ switch (Op) {
+#define X(NAME, OFF) \
+ case AArch64::NAME##_POST: \
+ return OFF;
+ SIMD_POST_OPS(X)
+#undef X
+ default:
+ return -1;
+ }
+}
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIRewriter.h b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIRewriter.h
index 439e2fe4b7ef4..1355df47bee82 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIRewriter.h
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIRewriter.h
@@ -13,6 +13,7 @@
#ifndef LLVM_LIB_TARGET_AARCH64_MCTARGETDESC_AARCH64MCLFIREWRITER_H
#define LLVM_LIB_TARGET_AARCH64_MCTARGETDESC_AARCH64MCLFIREWRITER_H
+#include "AArch64AddressingModes.h"
#include "llvm/MC/MCInstrInfo.h"
#include "llvm/MC/MCLFIRewriter.h"
#include "llvm/MC/MCRegister.h"
@@ -21,6 +22,7 @@
namespace llvm {
class MCContext;
class MCInst;
+class MCOperand;
class MCStreamer;
class MCSubtargetInfo;
@@ -51,6 +53,7 @@ class AArch64MCLFIRewriter : public MCLFIRewriter {
bool Guard = false;
// Instruction classification.
+ bool mayModifyStack(const MCInst &Inst) const;
bool mayModifyReserved(const MCInst &Inst) const;
// Instruction emission.
@@ -62,6 +65,15 @@ class AArch64MCLFIRewriter : public MCLFIRewriter {
const MCSubtargetInfo &STI);
void emitMov(MCRegister Dest, MCRegister Src, MCStreamer &Out,
const MCSubtargetInfo &STI);
+ void emitAddImm(MCRegister Dest, MCRegister Src, int64_t Imm, MCStreamer &Out,
+ const MCSubtargetInfo &STI);
+ void emitAddReg(MCRegister Dest, MCRegister Src1, MCRegister Src2,
+ unsigned Shift, MCStreamer &Out, const MCSubtargetInfo &STI);
+ void emitAddRegExtend(MCRegister Dest, MCRegister Src1, MCRegister Src2,
+ AArch64_AM::ShiftExtendType ExtType, unsigned Shift,
+ MCStreamer &Out, const MCSubtargetInfo &STI);
+ void emitMemRoW(unsigned Opcode, const MCOperand &DataOp, MCRegister BaseReg,
+ MCStreamer &Out, const MCSubtargetInfo &STI);
// Rewriting logic.
void doRewriteInst(const MCInst &Inst, MCStreamer &Out,
@@ -73,6 +85,18 @@ class AArch64MCLFIRewriter : public MCLFIRewriter {
void rewriteReturn(const MCInst &Inst, MCStreamer &Out,
const MCSubtargetInfo &STI);
+ // Memory access.
+ void rewriteLoadStore(const MCInst &Inst, MCStreamer &Out,
+ const MCSubtargetInfo &STI);
+ void rewriteLoadStoreBasic(const MCInst &Inst, MCStreamer &Out,
+ const MCSubtargetInfo &STI);
+ bool rewriteLoadStoreRoW(const MCInst &Inst, MCStreamer &Out,
+ const MCSubtargetInfo &STI);
+
+ // Register modification.
+ void rewriteStackModification(const MCInst &Inst, MCStreamer &Out,
+ const MCSubtargetInfo &STI);
+
// Link register modification.
void rewriteLRModification(const MCInst &Inst, MCStreamer &Out,
const MCSubtargetInfo &STI);
@@ -84,8 +108,15 @@ class AArch64MCLFIRewriter : public MCLFIRewriter {
const MCSubtargetInfo &STI);
void rewriteTPWrite(const MCInst &Inst, MCStreamer &Out,
const MCSubtargetInfo &STI);
+ void rewriteVASysOp(const MCInst &Inst, MCStreamer &Out,
+ const MCSubtargetInfo &STI);
};
+/// Returns true if \p Opcode is a pre- or post-indexed memory access that the
+/// LFI rewriter expands with a base-register update (i.e. an extra
+/// instruction beyond the guard + access pair).
+bool isLFIPrePostMemAccess(unsigned Opcode);
+
} // namespace llvm
#endif // LLVM_LIB_TARGET_AARCH64_MCTARGETDESC_AARCH64MCLFIREWRITER_H
diff --git a/llvm/test/MC/AArch64/LFI/exclusive.s b/llvm/test/MC/AArch64/LFI/exclusive.s
new file mode 100644
index 0000000000000..eb42bd234848c
--- /dev/null
+++ b/llvm/test/MC/AArch64/LFI/exclusive.s
@@ -0,0 +1,140 @@
+// RUN: llvm-mc -triple aarch64_lfi %s | FileCheck %s
+
+// Load exclusive
+ldxr x0, [x1]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: ldxr x0, [x28]
+
+ldxr w0, [x1]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: ldxr w0, [x28]
+
+ldxrb w0, [x1]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: ldxrb w0, [x28]
+
+ldxrh w0, [x1]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: ldxrh w0, [x28]
+
+// Store exclusive
+stxr w0, x1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: stxr w0, x1, [x28]
+
+stxr w0, w1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: stxr w0, w1, [x28]
+
+stxrb w0, w1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: stxrb w0, w1, [x28]
+
+stxrh w0, w1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: stxrh w0, w1, [x28]
+
+// Load-acquire exclusive
+ldaxr x0, [x1]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: ldaxr x0, [x28]
+
+ldaxr w0, [x1]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: ldaxr w0, [x28]
+
+ldaxrb w0, [x1]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: ldaxrb w0, [x28]
+
+ldaxrh w0, [x1]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: ldaxrh w0, [x28]
+
+// Store-release exclusive
+stlxr w0, x1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: stlxr w0, x1, [x28]
+
+stlxr w0, w1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: stlxr w0, w1, [x28]
+
+stlxrb w0, w1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: stlxrb w0, w1, [x28]
+
+stlxrh w0, w1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: stlxrh w0, w1, [x28]
+
+// Exclusive pairs
+ldxp x0, x1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: ldxp x0, x1, [x28]
+
+ldxp w0, w1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: ldxp w0, w1, [x28]
+
+stxp w0, x1, x2, [x3]
+// CHECK: add x28, x27, w3, uxtw
+// CHECK-NEXT: stxp w0, x1, x2, [x28]
+
+stxp w0, w1, w2, [x3]
+// CHECK: add x28, x27, w3, uxtw
+// CHECK-NEXT: stxp w0, w1, w2, [x28]
+
+ldaxp x0, x1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: ldaxp x0, x1, [x28]
+
+stlxp w0, x1, x2, [x3]
+// CHECK: add x28, x27, w3, uxtw
+// CHECK-NEXT: stlxp w0, x1, x2, [x28]
+
+// Load-acquire / Store-release (non-exclusive)
+ldar x0, [x1]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: ldar x0, [x28]
+
+ldar w0, [x1]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: ldar w0, [x28]
+
+ldarb w0, [x1]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: ldarb w0, [x28]
+
+ldarh w0, [x1]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: ldarh w0, [x28]
+
+stlr x0, [x1]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: stlr x0, [x28]
+
+stlr w0, [x1]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: stlr w0, [x28]
+
+stlrb w0, [x1]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: stlrb w0, [x28]
+
+stlrh w0, [x1]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: stlrh w0, [x28]
+
+// SP-relative exclusive (no sandboxing needed)
+ldxr x0, [sp]
+// CHECK: ldxr x0, [sp]
+
+stxr w0, x1, [sp]
+// CHECK: stxr w0, x1, [sp]
+
+ldar x0, [sp]
+// CHECK: ldar x0, [sp]
+
+stlr x0, [sp]
+// CHECK: stlr x0, [sp]
diff --git a/llvm/test/MC/AArch64/LFI/fp.s b/llvm/test/MC/AArch64/LFI/fp.s
new file mode 100644
index 0000000000000..d1fc4aac25a23
--- /dev/null
+++ b/llvm/test/MC/AArch64/LFI/fp.s
@@ -0,0 +1,204 @@
+// RUN: llvm-mc -triple aarch64_lfi %s | FileCheck %s
+
+// FP/SIMD scalar loads (zero offset -> RoW)
+ldr b0, [x1]
+// CHECK: ldr b0, [x27, w1, uxtw]
+
+ldr h0, [x1]
+// CHECK: ldr h0, [x27, w1, uxtw]
+
+ldr s0, [x1]
+// CHECK: ldr s0, [x27, w1, uxtw]
+
+ldr d0, [x1]
+// CHECK: ldr d0, [x27, w1, uxtw]
+
+ldr q0, [x1]
+// CHECK: ldr q0, [x27, w1, uxtw]
+
+// FP/SIMD scalar stores (zero offset -> RoW)
+str b0, [x1]
+// CHECK: str b0, [x27, w1, uxtw]
+
+str h0, [x1]
+// CHECK: str h0, [x27, w1, uxtw]
+
+str s0, [x1]
+// CHECK: str s0, [x27, w1, uxtw]
+
+str d0, [x1]
+// CHECK: str d0, [x27, w1, uxtw]
+
+str q0, [x1]
+// CHECK: str q0, [x27, w1, uxtw]
+
+// FP loads with non-zero offset (demoted)
+ldr s0, [x1, #4]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: ldr s0, [x28, #4]
+
+ldr d0, [x1, #8]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: ldr d0, [x28, #8]
+
+ldr q0, [x1, #16]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: ldr q0, [x28, #16]
+
+// FP stores with non-zero offset (demoted)
+str s0, [x1, #4]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: str s0, [x28, #4]
+
+str d0, [x1, #8]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: str d0, [x28, #8]
+
+str q0, [x1, #16]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: str q0, [x28, #16]
+
+// FP pre-index
+ldr s0, [x1, #4]!
+// CHECK: add x1, x1, #4
+// CHECK-NEXT: ldr s0, [x27, w1, uxtw]
+
+ldr d0, [x1, #8]!
+// CHECK: add x1, x1, #8
+// CHECK-NEXT: ldr d0, [x27, w1, uxtw]
+
+ldr q0, [x1, #16]!
+// CHECK: add x1, x1, #16
+// CHECK-NEXT: ldr q0, [x27, w1, uxtw]
+
+str s0, [x1, #4]!
+// CHECK: add x1, x1, #4
+// CHECK-NEXT: str s0, [x27, w1, uxtw]
+
+// FP post-index
+ldr s0, [x1], #4
+// CHECK: ldr s0, [x27, w1, uxtw]
+// CHECK-NEXT: add x1, x1, #4
+
+ldr d0, [x1], #8
+// CHECK: ldr d0, [x27, w1, uxtw]
+// CHECK-NEXT: add x1, x1, #8
+
+ldr q0, [x1], #16
+// CHECK: ldr q0, [x27, w1, uxtw]
+// CHECK-NEXT: add x1, x1, #16
+
+str d0, [x1], #8
+// CHECK: str d0, [x27, w1, uxtw]
+// CHECK-NEXT: add x1, x1, #8
+
+// FP register offset
+ldr s0, [x1, x2]
+// CHECK: add x26, x1, x2
+// CHECK-NEXT: ldr s0, [x27, w26, uxtw]
+
+ldr d0, [x1, x2, lsl #3]
+// CHECK: add x26, x1, x2, lsl #3
+// CHECK-NEXT: ldr d0, [x27, w26, uxtw]
+
+ldr q0, [x1, x2, lsl #4]
+// CHECK: add x26, x1, x2, lsl #4
+// CHECK-NEXT: ldr q0, [x27, w26, uxtw]
+
+str s0, [x1, x2, lsl #2]
+// CHECK: add x26, x1, x2, lsl #2
+// CHECK-NEXT: str s0, [x27, w26, uxtw]
+
+str d0, [x1, w2, sxtw]
+// CHECK: add x26, x1, w2, sxtw
+// CHECK-NEXT: str d0, [x27, w26, uxtw]
+
+str q0, [x1, w2, uxtw #4]
+// CHECK: add x26, x1, w2, uxtw #4
+// CHECK-NEXT: str q0, [x27, w26, uxtw]
+
+// FP unscaled offset
+ldur s0, [x1]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: ldur s0, [x28]
+
+ldur d0, [x1, #8]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: ldur d0, [x28, #8]
+
+ldur q0, [x1, #-16]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: ldur q0, [x28, #-16]
+
+stur s0, [x1]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: stur s0, [x28]
+
+stur d0, [x1, #8]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: stur d0, [x28, #8]
+
+// FP pair loads/stores
+ldp s0, s1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: ldp s0, s1, [x28]
+
+ldp d0, d1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: ldp d0, d1, [x28]
+
+ldp q0, q1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: ldp q0, q1, [x28]
+
+stp s0, s1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: stp s0, s1, [x28]
+
+stp d0, d1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: stp d0, d1, [x28]
+
+stp q0, q1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: stp q0, q1, [x28]
+
+// FP pair with offset
+ldp s0, s1, [x2, #8]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: ldp s0, s1, [x28, #8]
+
+ldp d0, d1, [x2, #16]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: ldp d0, d1, [x28, #16]
+
+// FP pair pre/post-index
+ldp s0, s1, [x2], #8
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: ldp s0, s1, [x28]
+// CHECK-NEXT: add x2, x2, #8
+
+ldp d0, d1, [x2, #16]!
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: ldp d0, d1, [x28, #16]
+// CHECK-NEXT: add x2, x2, #16
+
+stp q0, q1, [x2], #32
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: stp q0, q1, [x28]
+// CHECK-NEXT: add x2, x2, #32
+
+stp d0, d1, [x2, #-16]!
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: stp d0, d1, [x28, #-16]
+// CHECK-NEXT: sub x2, x2, #16
+
+// SP-relative FP loads (no sandboxing needed)
+ldr s0, [sp]
+// CHECK: ldr s0, [sp]
+
+ldr d0, [sp, #8]
+// CHECK: ldr d0, [sp, #8]
+
+ldp q0, q1, [sp, #32]
+// CHECK: ldp q0, q1, [sp, #32]
diff --git a/llvm/test/MC/AArch64/LFI/jumps-only.s b/llvm/test/MC/AArch64/LFI/jumps-only.s
new file mode 100644
index 0000000000000..162a203158559
--- /dev/null
+++ b/llvm/test/MC/AArch64/LFI/jumps-only.s
@@ -0,0 +1,41 @@
+// RUN: llvm-mc -triple aarch64_lfi -mattr=+no-lfi-loads,+no-lfi-stores %s | FileCheck %s
+
+// Jumps-only mode: only branches are sandboxed.
+
+ldr x0, [x1]
+// CHECK: ldr x0, [x1]
+
+ldr x0, [x1, #8]
+// CHECK: ldr x0, [x1, #8]
+
+str x0, [x1]
+// CHECK: str x0, [x1]
+
+stp x0, x1, [x2, #16]
+// CHECK: stp x0, x1, [x2, #16]
+
+add sp, sp, #8
+// CHECK: add sp, sp, #8
+
+sub sp, sp, #8
+// CHECK: sub sp, sp, #8
+
+mov sp, x0
+// CHECK: mov sp, x0
+
+br x0
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: br x28
+
+ret
+// CHECK: ret
+
+blr x1
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: blr x28
+
+bl some_func
+// CHECK: bl some_func
+
+b some_func
+// CHECK: b some_func
diff --git a/llvm/test/MC/AArch64/LFI/literal.s b/llvm/test/MC/AArch64/LFI/literal.s
new file mode 100644
index 0000000000000..89a0230aace8d
--- /dev/null
+++ b/llvm/test/MC/AArch64/LFI/literal.s
@@ -0,0 +1,37 @@
+// RUN: not llvm-mc -triple aarch64_lfi %s 2>&1 | FileCheck %s
+
+ldr x0, foo
+// CHECK: error: PC-relative literal loads are not supported in LFI
+
+ldr w0, bar
+// CHECK: error: PC-relative literal loads are not supported in LFI
+
+ldr s0, baz
+// CHECK: error: PC-relative literal loads are not supported in LFI
+
+ldr d0, qux
+// CHECK: error: PC-relative literal loads are not supported in LFI
+
+ldr q0, quux
+// CHECK: error: PC-relative literal loads are not supported in LFI
+
+ldrsw x0, signed_word
+// CHECK: error: PC-relative literal loads are not supported in LFI
+
+prfm pldl1keep, prefetch_target
+// CHECK: error: PC-relative literal loads are not supported in LFI
+
+foo:
+ .quad 0
+bar:
+ .word 0
+baz:
+ .single 0.0
+qux:
+ .double 0.0
+quux:
+ .zero 16
+signed_word:
+ .word -1
+prefetch_target:
+ .quad 0
diff --git a/llvm/test/MC/AArch64/LFI/lse.s b/llvm/test/MC/AArch64/LFI/lse.s
new file mode 100644
index 0000000000000..482778b936f6d
--- /dev/null
+++ b/llvm/test/MC/AArch64/LFI/lse.s
@@ -0,0 +1,166 @@
+// RUN: llvm-mc -triple aarch64_lfi %s | FileCheck %s
+// RUN: llvm-mc -triple aarch64_lfi -mattr=+no-lfi-loads %s | FileCheck %s --check-prefix=NOLOADS
+
+.arch_extension lse
+
+// LDADD variants
+// Atomics are both loads and stores, so +no-lfi-loads must still sandbox them.
+ldadd x0, x1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: ldadd x0, x1, [x28]
+// NOLOADS: add x28, x27, w2, uxtw
+// NOLOADS-NEXT: ldadd x0, x1, [x28]
+
+ldadd w0, w1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: ldadd w0, w1, [x28]
+
+ldadda x0, x1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: ldadda x0, x1, [x28]
+
+ldaddal x0, x1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: ldaddal x0, x1, [x28]
+
+ldaddl x0, x1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: ldaddl x0, x1, [x28]
+
+ldaddab w0, w1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: ldaddab w0, w1, [x28]
+
+ldaddah w0, w1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: ldaddah w0, w1, [x28]
+
+// LDCLR variants
+ldclr x0, x1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: ldclr x0, x1, [x28]
+
+ldclra x0, x1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: ldclra x0, x1, [x28]
+
+ldclral x0, x1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: ldclral x0, x1, [x28]
+
+ldclrl x0, x1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: ldclrl x0, x1, [x28]
+
+// LDEOR variants
+ldeor x0, x1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: ldeor x0, x1, [x28]
+
+ldeora x0, x1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: ldeora x0, x1, [x28]
+
+// LDSET variants
+ldset x0, x1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: ldset x0, x1, [x28]
+
+ldseta x0, x1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: ldseta x0, x1, [x28]
+
+// SWP variants
+swp x0, x1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: swp x0, x1, [x28]
+// NOLOADS: swp x0, x1, [x28]
+
+swpa x0, x1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: swpa x0, x1, [x28]
+
+swpal x0, x1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: swpal x0, x1, [x28]
+
+swpl x0, x1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: swpl x0, x1, [x28]
+
+swpab w0, w1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: swpab w0, w1, [x28]
+
+swpah w0, w1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: swpah w0, w1, [x28]
+
+swpalb w0, w1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: swpalb w0, w1, [x28]
+
+swpalh w0, w1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: swpalh w0, w1, [x28]
+
+swpal w0, w0, [x1]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: swpal w0, w0, [x28]
+
+// CAS variants
+cas x0, x1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: cas x0, x1, [x28]
+// NOLOADS: cas x0, x1, [x28]
+
+casa x0, x1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: casa x0, x1, [x28]
+
+casal x0, x1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: casal x0, x1, [x28]
+
+casl x0, x1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: casl x0, x1, [x28]
+
+casab w0, w1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: casab w0, w1, [x28]
+
+casah w0, w1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: casah w0, w1, [x28]
+
+// CASP variants (pair)
+casp x0, x1, x2, x3, [x4]
+// CHECK: add x28, x27, w4, uxtw
+// CHECK-NEXT: casp x0, x1, x2, x3, [x28]
+
+caspa x0, x1, x2, x3, [x4]
+// CHECK: add x28, x27, w4, uxtw
+// CHECK-NEXT: caspa x0, x1, x2, x3, [x28]
+
+caspal x0, x1, x2, x3, [x4]
+// CHECK: add x28, x27, w4, uxtw
+// CHECK-NEXT: caspal x0, x1, x2, x3, [x28]
+
+caspl x0, x1, x2, x3, [x4]
+// CHECK: add x28, x27, w4, uxtw
+// CHECK-NEXT: caspl x0, x1, x2, x3, [x28]
+
+caspal w0, w1, w2, w3, [x4]
+// CHECK: add x28, x27, w4, uxtw
+// CHECK-NEXT: caspal w0, w1, w2, w3, [x28]
+
+// SP-relative atomics (no sandboxing needed)
+ldadd x0, x1, [sp]
+// CHECK: ldadd x0, x1, [sp]
+
+swp x0, x1, [sp]
+// CHECK: swp x0, x1, [sp]
+
+cas x0, x1, [sp]
+// CHECK: cas x0, x1, [sp]
diff --git a/llvm/test/MC/AArch64/LFI/mem.s b/llvm/test/MC/AArch64/LFI/mem.s
new file mode 100644
index 0000000000000..ccf1efa31517b
--- /dev/null
+++ b/llvm/test/MC/AArch64/LFI/mem.s
@@ -0,0 +1,507 @@
+// RUN: llvm-mc -triple aarch64_lfi %s | FileCheck %s
+
+ldr x0, [sp]
+// CHECK: ldr x0, [sp]
+
+ldr x0, [sp, #8]
+// CHECK: ldr x0, [sp, #8]
+
+ldp x0, x1, [sp, #8]
+// CHECK: ldp x0, x1, [sp, #8]
+
+str x0, [sp]
+// CHECK: str x0, [sp]
+
+str x0, [sp, #8]
+// CHECK: str x0, [sp, #8]
+
+stp x0, x1, [sp, #8]
+// CHECK: stp x0, x1, [sp, #8]
+
+ldur x0, [x1]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: ldur x0, [x28]
+
+stur x0, [x1]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: stur x0, [x28]
+
+ldp x0, x1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: ldp x0, x1, [x28]
+
+stp x0, x1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: stp x0, x1, [x28]
+
+ldr x0, [x1]
+// CHECK: ldr x0, [x27, w1, uxtw]
+
+ldr x0, [x1, #8]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: ldr x0, [x28, #8]
+
+ldr x0, [x1, #8]!
+// CHECK: add x1, x1, #8
+// CHECK-NEXT: ldr x0, [x27, w1, uxtw]
+
+str x0, [x1, #8]!
+// CHECK: add x1, x1, #8
+// CHECK-NEXT: str x0, [x27, w1, uxtw]
+
+ldr x0, [x1, #-8]!
+// CHECK: sub x1, x1, #8
+// CHECK-NEXT: ldr x0, [x27, w1, uxtw]
+
+str x0, [x1, #-8]!
+// CHECK: sub x1, x1, #8
+// CHECK-NEXT: str x0, [x27, w1, uxtw]
+
+ldr x0, [x1], #8
+// CHECK: ldr x0, [x27, w1, uxtw]
+// CHECK-NEXT: add x1, x1, #8
+
+str x0, [x1], #8
+// CHECK: str x0, [x27, w1, uxtw]
+// CHECK-NEXT: add x1, x1, #8
+
+ldr x0, [x1], #-8
+// CHECK: ldr x0, [x27, w1, uxtw]
+// CHECK-NEXT: sub x1, x1, #8
+
+str x0, [x1], #-8
+// CHECK: str x0, [x27, w1, uxtw]
+// CHECK-NEXT: sub x1, x1, #8
+
+ldr x0, [x1, x2]
+// CHECK: add x26, x1, x2
+// CHECK-NEXT: ldr x0, [x27, w26, uxtw]
+
+ldr x0, [x1, x2, lsl #3]
+// CHECK: add x26, x1, x2, lsl #3
+// CHECK-NEXT: ldr x0, [x27, w26, uxtw]
+
+ldr x0, [x1, x2, sxtx #0]
+// CHECK: add x26, x1, x2, sxtx
+// CHECK-NEXT: ldr x0, [x27, w26, uxtw]
+
+ldr x0, [x1, x2, sxtx #3]
+// CHECK: add x26, x1, x2, sxtx #3
+// CHECK-NEXT: ldr x0, [x27, w26, uxtw]
+
+ldr x0, [x1, w2, uxtw]
+// CHECK: add x26, x1, w2, uxtw
+// CHECK-NEXT: ldr x0, [x27, w26, uxtw]
+
+ldr x0, [x1, w2, uxtw #3]
+// CHECK: add x26, x1, w2, uxtw #3
+// CHECK-NEXT: ldr x0, [x27, w26, uxtw]
+
+ldr x0, [x1, w2, sxtw]
+// CHECK: add x26, x1, w2, sxtw
+// CHECK-NEXT: ldr x0, [x27, w26, uxtw]
+
+ldr x0, [x1, w2, sxtw #3]
+// CHECK: add x26, x1, w2, sxtw #3
+// CHECK-NEXT: ldr x0, [x27, w26, uxtw]
+
+ldp x0, x1, [sp], #8
+// CHECK: ldp x0, x1, [sp], #8
+
+ldp x0, x1, [x2], #8
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: ldp x0, x1, [x28]
+// CHECK-NEXT: add x2, x2, #8
+
+ldp x0, x1, [x2, #8]!
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: ldp x0, x1, [x28, #8]
+// CHECK-NEXT: add x2, x2, #8
+
+ldp x0, x1, [x2], #-8
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: ldp x0, x1, [x28]
+// CHECK-NEXT: sub x2, x2, #8
+
+ldp x0, x1, [x2, #-8]!
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: ldp x0, x1, [x28, #-8]
+// CHECK-NEXT: sub x2, x2, #8
+
+stp x0, x1, [x2, #-8]!
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: stp x0, x1, [x28, #-8]
+// CHECK-NEXT: sub x2, x2, #8
+
+ld3 { v0.4s, v1.4s, v2.4s }, [x0], #48
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld3 { v0.4s, v1.4s, v2.4s }, [x28]
+// CHECK-NEXT: add x0, x0, #48
+
+st2 { v1.8b, v2.8b }, [x14], #16
+// CHECK: add x28, x27, w14, uxtw
+// CHECK-NEXT: st2 { v1.8b, v2.8b }, [x28]
+// CHECK-NEXT: add x14, x14, #16
+
+st2 { v1.8b, v2.8b }, [x14]
+// CHECK: add x28, x27, w14, uxtw
+// CHECK-NEXT: st2 { v1.8b, v2.8b }, [x28]
+
+ld1 { v0.s }[1], [x8]
+// CHECK: add x28, x27, w8, uxtw
+// CHECK-NEXT: ld1 { v0.s }[1], [x28]
+
+ld1r { v3.2d }, [x9]
+// CHECK: add x28, x27, w9, uxtw
+// CHECK-NEXT: ld1r { v3.2d }, [x28]
+
+ld1 { v0.s }[1], [x8], x10
+// CHECK: add x28, x27, w8, uxtw
+// CHECK-NEXT: ld1 { v0.s }[1], [x28]
+// CHECK-NEXT: add x8, x8, x10
+
+ldaxr x0, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: ldaxr x0, [x28]
+
+stlxr w15, w17, [x1]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: stlxr w15, w17, [x28]
+
+ldr w4, [sp, w3, uxtw #2]
+// CHECK: add x26, sp, w3, uxtw #2
+// CHECK-NEXT: ldr w4, [x27, w26, uxtw]
+
+stxrb w11, w10, [x8]
+// CHECK: add x28, x27, w8, uxtw
+// CHECK-NEXT: stxrb w11, w10, [x28]
+
+ldr x0, [x0, :got_lo12:x]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ldr x0, [x28, :got_lo12:x]
+
+prfm pstl1strm, [x10]
+// CHECK: prfm pstl1strm, [x27, w10, uxtw]
+
+prfm pstl1strm, [x10, x11]
+// CHECK: add x26, x10, x11
+// CHECK-NEXT: prfm pstl1strm, [x27, w26, uxtw]
+
+// Byte loads/stores
+ldrb w0, [x1]
+// CHECK: ldrb w0, [x27, w1, uxtw]
+
+ldrb w0, [x1, #1]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: ldrb w0, [x28, #1]
+
+strb w0, [x1]
+// CHECK: strb w0, [x27, w1, uxtw]
+
+ldrsb w0, [x1]
+// CHECK: ldrsb w0, [x27, w1, uxtw]
+
+ldrsb x0, [x1]
+// CHECK: ldrsb x0, [x27, w1, uxtw]
+
+// Halfword loads/stores
+ldrh w0, [x1]
+// CHECK: ldrh w0, [x27, w1, uxtw]
+
+ldrh w0, [x1, #2]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: ldrh w0, [x28, #2]
+
+strh w0, [x1]
+// CHECK: strh w0, [x27, w1, uxtw]
+
+ldrsh w0, [x1]
+// CHECK: ldrsh w0, [x27, w1, uxtw]
+
+ldrsh x0, [x1]
+// CHECK: ldrsh x0, [x27, w1, uxtw]
+
+// Word loads/stores
+ldr w0, [x1]
+// CHECK: ldr w0, [x27, w1, uxtw]
+
+ldr w0, [x1, #4]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: ldr w0, [x28, #4]
+
+str w0, [x1]
+// CHECK: str w0, [x27, w1, uxtw]
+
+ldrsw x0, [x1]
+// CHECK: ldrsw x0, [x27, w1, uxtw]
+
+// 32-bit pairs
+ldp w0, w1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: ldp w0, w1, [x28]
+
+stp w0, w1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: stp w0, w1, [x28]
+
+// Unscaled loads/stores
+ldurb w0, [x1]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: ldurb w0, [x28]
+
+ldurb w0, [x1, #1]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: ldurb w0, [x28, #1]
+
+ldursb w0, [x1]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: ldursb w0, [x28]
+
+ldurh w0, [x1]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: ldurh w0, [x28]
+
+ldursh w0, [x1]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: ldursh w0, [x28]
+
+ldur w0, [x1]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: ldur w0, [x28]
+
+ldursw x0, [x1]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: ldursw x0, [x28]
+
+sturb w0, [x1]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: sturb w0, [x28]
+
+sturh w0, [x1]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: sturh w0, [x28]
+
+stur w0, [x1]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: stur w0, [x28]
+
+// Byte pre/post-index
+ldrb w0, [x1, #1]!
+// CHECK: add x1, x1, #1
+// CHECK-NEXT: ldrb w0, [x27, w1, uxtw]
+
+ldrb w0, [x1], #1
+// CHECK: ldrb w0, [x27, w1, uxtw]
+// CHECK-NEXT: add x1, x1, #1
+
+strb w0, [x1, #1]!
+// CHECK: add x1, x1, #1
+// CHECK-NEXT: strb w0, [x27, w1, uxtw]
+
+strb w0, [x1], #1
+// CHECK: strb w0, [x27, w1, uxtw]
+// CHECK-NEXT: add x1, x1, #1
+
+// Halfword pre/post-index
+ldrh w0, [x1, #2]!
+// CHECK: add x1, x1, #2
+// CHECK-NEXT: ldrh w0, [x27, w1, uxtw]
+
+ldrh w0, [x1], #2
+// CHECK: ldrh w0, [x27, w1, uxtw]
+// CHECK-NEXT: add x1, x1, #2
+
+// Word pre/post-index
+ldr w0, [x1, #4]!
+// CHECK: add x1, x1, #4
+// CHECK-NEXT: ldr w0, [x27, w1, uxtw]
+
+ldr w0, [x1], #4
+// CHECK: ldr w0, [x27, w1, uxtw]
+// CHECK-NEXT: add x1, x1, #4
+
+// Register offset with different sizes
+ldrb w0, [x1, x2]
+// CHECK: add x26, x1, x2
+// CHECK-NEXT: ldrb w0, [x27, w26, uxtw]
+
+ldrh w0, [x1, x2]
+// CHECK: add x26, x1, x2
+// CHECK-NEXT: ldrh w0, [x27, w26, uxtw]
+
+ldrh w0, [x1, x2, lsl #1]
+// CHECK: add x26, x1, x2, lsl #1
+// CHECK-NEXT: ldrh w0, [x27, w26, uxtw]
+
+ldr w0, [x1, x2]
+// CHECK: add x26, x1, x2
+// CHECK-NEXT: ldr w0, [x27, w26, uxtw]
+
+ldr w0, [x1, x2, lsl #2]
+// CHECK: add x26, x1, x2, lsl #2
+// CHECK-NEXT: ldr w0, [x27, w26, uxtw]
+
+strb w0, [x1, x2]
+// CHECK: add x26, x1, x2
+// CHECK-NEXT: strb w0, [x27, w26, uxtw]
+
+strh w0, [x1, x2]
+// CHECK: add x26, x1, x2
+// CHECK-NEXT: strh w0, [x27, w26, uxtw]
+
+str w0, [x1, x2]
+// CHECK: add x26, x1, x2
+// CHECK-NEXT: str w0, [x27, w26, uxtw]
+
+str x0, [x1, x2]
+// CHECK: add x26, x1, x2
+// CHECK-NEXT: str x0, [x27, w26, uxtw]
+
+// Sign/zero extension variants
+ldrb w0, [x1, w2, uxtw]
+// CHECK: add x26, x1, w2, uxtw
+// CHECK-NEXT: ldrb w0, [x27, w26, uxtw]
+
+ldrb w0, [x1, w2, sxtw]
+// CHECK: add x26, x1, w2, sxtw
+// CHECK-NEXT: ldrb w0, [x27, w26, uxtw]
+
+ldrh w0, [x1, w2, uxtw]
+// CHECK: add x26, x1, w2, uxtw
+// CHECK-NEXT: ldrh w0, [x27, w26, uxtw]
+
+ldrh w0, [x1, w2, uxtw #1]
+// CHECK: add x26, x1, w2, uxtw #1
+// CHECK-NEXT: ldrh w0, [x27, w26, uxtw]
+
+ldr w0, [x1, w2, sxtw #2]
+// CHECK: add x26, x1, w2, sxtw #2
+// CHECK-NEXT: ldr w0, [x27, w26, uxtw]
+
+// Byte loads with #0 shift (shift amount omitted in output).
+ldrsb x0, [x1, x2, sxtx #0]
+// CHECK: add x26, x1, x2, sxtx{{$}}
+// CHECK-NEXT: ldrsb x0, [x27, w26, uxtw]
+
+ldrsb w0, [x1, x2, sxtx #0]
+// CHECK: add x26, x1, x2, sxtx{{$}}
+// CHECK-NEXT: ldrsb w0, [x27, w26, uxtw]
+
+ldrsb w0, [x1, w2, sxtw #0]
+// CHECK: add x26, x1, w2, sxtw{{$}}
+// CHECK-NEXT: ldrsb w0, [x27, w26, uxtw]
+
+ldrsb x0, [x1, w2, uxtw #0]
+// CHECK: add x26, x1, w2, uxtw{{$}}
+// CHECK-NEXT: ldrsb x0, [x27, w26, uxtw]
+
+ldrsh x0, [x1, x2, sxtx #1]
+// CHECK: add x26, x1, x2, sxtx #1
+// CHECK-NEXT: ldrsh x0, [x27, w26, uxtw]
+
+ldrsh w0, [x1, x2, sxtx #1]
+// CHECK: add x26, x1, x2, sxtx #1
+// CHECK-NEXT: ldrsh w0, [x27, w26, uxtw]
+
+ldrsh w0, [x1, w2, sxtw #1]
+// CHECK: add x26, x1, w2, sxtw #1
+// CHECK-NEXT: ldrsh w0, [x27, w26, uxtw]
+
+ldrsw x0, [x1, x2, sxtx #2]
+// CHECK: add x26, x1, x2, sxtx #2
+// CHECK-NEXT: ldrsw x0, [x27, w26, uxtw]
+
+ldrsw x0, [x1, w2, sxtw #2]
+// CHECK: add x26, x1, w2, sxtw #2
+// CHECK-NEXT: ldrsw x0, [x27, w26, uxtw]
+
+// 32-bit pair pre/post-index
+ldp w0, w1, [x2], #8
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: ldp w0, w1, [x28]
+// CHECK-NEXT: add x2, x2, #8
+
+ldp w0, w1, [x2, #8]!
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: ldp w0, w1, [x28, #8]
+// CHECK-NEXT: add x2, x2, #8
+
+stp w0, w1, [x2], #8
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: stp w0, w1, [x28]
+// CHECK-NEXT: add x2, x2, #8
+
+stp w0, w1, [x2, #8]!
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: stp w0, w1, [x28, #8]
+// CHECK-NEXT: add x2, x2, #8
+
+// Memory accesses that define LR (x30) must sandbox the base register
+// in addition to masking LR after the access.
+
+ldr x30, [x0, #0x100]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ldr x30, [x28, #256]
+// CHECK-NEXT: add x30, x27, w30, uxtw
+
+ldr x30, [x0]
+// CHECK: ldr x30, [x27, w0, uxtw]
+// CHECK-NEXT: add x30, x27, w30, uxtw
+
+ldr x30, [x0, #8]!
+// CHECK: add x0, x0, #8
+// CHECK-NEXT: ldr x30, [x27, w0, uxtw]
+// CHECK-NEXT: add x30, x27, w30, uxtw
+
+ldr x30, [x0, #-8]!
+// CHECK: sub x0, x0, #8
+// CHECK-NEXT: ldr x30, [x27, w0, uxtw]
+// CHECK-NEXT: add x30, x27, w30, uxtw
+
+ldr x30, [x0], #8
+// CHECK: ldr x30, [x27, w0, uxtw]
+// CHECK-NEXT: add x0, x0, #8
+// CHECK-NEXT: add x30, x27, w30, uxtw
+
+ldr x30, [x0, x1]
+// CHECK: add x26, x0, x1
+// CHECK-NEXT: ldr x30, [x27, w26, uxtw]
+// CHECK-NEXT: add x30, x27, w30, uxtw
+
+ldr x30, [x0, x1, lsl #3]
+// CHECK: add x26, x0, x1, lsl #3
+// CHECK-NEXT: ldr x30, [x27, w26, uxtw]
+// CHECK-NEXT: add x30, x27, w30, uxtw
+
+ldur x30, [x0, #4]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ldur x30, [x28, #4]
+// CHECK-NEXT: add x30, x27, w30, uxtw
+
+ldp x29, x30, [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ldp x29, x30, [x28]
+// CHECK-NEXT: add x30, x27, w30, uxtw
+
+ldp x29, x30, [x0, #16]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ldp x29, x30, [x28, #16]
+// CHECK-NEXT: add x30, x27, w30, uxtw
+
+ldp x29, x30, [x0, #16]!
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ldp x29, x30, [x28, #16]
+// CHECK-NEXT: add x0, x0, #16
+// CHECK-NEXT: add x30, x27, w30, uxtw
+
+ldp x29, x30, [x0], #16
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ldp x29, x30, [x28]
+// CHECK-NEXT: add x0, x0, #16
+// CHECK-NEXT: add x30, x27, w30, uxtw
+
+// SP-based LR loads are safe without base sandboxing.
+
+ldr x30, [sp, #16]
+// CHECK: ldr x30, [sp, #16]
+// CHECK-NEXT: add x30, x27, w30, uxtw
diff --git a/llvm/test/MC/AArch64/LFI/no-lfi-loads.s b/llvm/test/MC/AArch64/LFI/no-lfi-loads.s
new file mode 100644
index 0000000000000..1a606452275f2
--- /dev/null
+++ b/llvm/test/MC/AArch64/LFI/no-lfi-loads.s
@@ -0,0 +1,33 @@
+// RUN: llvm-mc -triple aarch64_lfi -mattr=+no-lfi-loads %s | FileCheck %s
+
+// Stores-only mode: loads pass through, stores are sandboxed.
+
+ldr x0, [x1]
+// CHECK: ldr x0, [x1]
+
+ldr x0, [x1, #8]
+// CHECK: ldr x0, [x1, #8]
+
+ldp x0, x1, [x2]
+// CHECK: ldp x0, x1, [x2]
+
+str x0, [x1]
+// CHECK: str x0, [x27, w1, uxtw]
+
+stp x0, x1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: stp x0, x1, [x28]
+
+ldr x0, [sp, #8]
+// CHECK: ldr x0, [sp, #8]
+
+str x0, [sp, #8]
+// CHECK: str x0, [sp, #8]
+
+add sp, sp, #8
+// CHECK: add x26, sp, #8
+// CHECK-NEXT: add sp, x27, w26, uxtw
+
+br x0
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: br x28
diff --git a/llvm/test/MC/AArch64/LFI/no-lfi-stores.s b/llvm/test/MC/AArch64/LFI/no-lfi-stores.s
new file mode 100644
index 0000000000000..b1ee8e16b745e
--- /dev/null
+++ b/llvm/test/MC/AArch64/LFI/no-lfi-stores.s
@@ -0,0 +1,35 @@
+// RUN: llvm-mc -triple aarch64_lfi -mattr=+no-lfi-stores %s | FileCheck %s
+
+// Loads-only mode: stores pass through, loads are sandboxed. This
+// configuration is not very useful in practice.
+
+ldr x0, [x1]
+// CHECK: ldr x0, [x27, w1, uxtw]
+
+ldr x0, [x1, #8]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: ldr x0, [x28, #8]
+
+ldp x0, x1, [x2]
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: ldp x0, x1, [x28]
+
+str x0, [x1]
+// CHECK: str x0, [x1]
+
+stp x0, x1, [x2]
+// CHECK: stp x0, x1, [x2]
+
+ldr x0, [sp, #8]
+// CHECK: ldr x0, [sp, #8]
+
+str x0, [sp, #8]
+// CHECK: str x0, [sp, #8]
+
+add sp, sp, #8
+// CHECK: add x26, sp, #8
+// CHECK-NEXT: add sp, x27, w26, uxtw
+
+br x0
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: br x28
diff --git a/llvm/test/MC/AArch64/LFI/prefetch.s b/llvm/test/MC/AArch64/LFI/prefetch.s
new file mode 100644
index 0000000000000..55b6bb78fa85c
--- /dev/null
+++ b/llvm/test/MC/AArch64/LFI/prefetch.s
@@ -0,0 +1,81 @@
+// RUN: llvm-mc -triple aarch64_lfi %s | FileCheck %s
+
+prfm pldl1keep, [x0]
+// CHECK: prfm pldl1keep, [x27, w0, uxtw]
+
+prfm pldl1strm, [x0]
+// CHECK: prfm pldl1strm, [x27, w0, uxtw]
+
+prfm pldl2keep, [x0]
+// CHECK: prfm pldl2keep, [x27, w0, uxtw]
+
+prfm pldl2strm, [x0]
+// CHECK: prfm pldl2strm, [x27, w0, uxtw]
+
+prfm pldl3keep, [x0]
+// CHECK: prfm pldl3keep, [x27, w0, uxtw]
+
+prfm pldl3strm, [x0]
+// CHECK: prfm pldl3strm, [x27, w0, uxtw]
+
+prfm pstl1keep, [x0]
+// CHECK: prfm pstl1keep, [x27, w0, uxtw]
+
+prfm pstl1strm, [x0]
+// CHECK: prfm pstl1strm, [x27, w0, uxtw]
+
+prfm pstl2keep, [x0]
+// CHECK: prfm pstl2keep, [x27, w0, uxtw]
+
+prfm pstl2strm, [x0]
+// CHECK: prfm pstl2strm, [x27, w0, uxtw]
+
+prfm pldl1keep, [x0, #8]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: prfm pldl1keep, [x28, #8]
+
+prfm pstl1strm, [x0, #16]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: prfm pstl1strm, [x28, #16]
+
+prfm pldl1keep, [x0, x1]
+// CHECK: add x26, x0, x1
+// CHECK-NEXT: prfm pldl1keep, [x27, w26, uxtw]
+
+prfm pldl1keep, [x0, x1, lsl #3]
+// CHECK: add x26, x0, x1, lsl #3
+// CHECK-NEXT: prfm pldl1keep, [x27, w26, uxtw]
+
+prfm pldl1keep, [x0, w1, uxtw]
+// CHECK: add x26, x0, w1, uxtw
+// CHECK-NEXT: prfm pldl1keep, [x27, w26, uxtw]
+
+prfm pldl1keep, [x0, w1, sxtw]
+// CHECK: add x26, x0, w1, sxtw
+// CHECK-NEXT: prfm pldl1keep, [x27, w26, uxtw]
+
+prfm pldl1keep, [x0, w1, uxtw #3]
+// CHECK: add x26, x0, w1, uxtw #3
+// CHECK-NEXT: prfm pldl1keep, [x27, w26, uxtw]
+
+prfm pldl1keep, [x0, w1, sxtw #3]
+// CHECK: add x26, x0, w1, sxtw #3
+// CHECK-NEXT: prfm pldl1keep, [x27, w26, uxtw]
+
+prfum pldl1keep, [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: prfum pldl1keep, [x28]
+
+prfum pldl1keep, [x0, #1]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: prfum pldl1keep, [x28, #1]
+
+prfum pstl1strm, [x0, #-8]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: prfum pstl1strm, [x28, #-8]
+
+prfm pldl1keep, [sp]
+// CHECK: prfm pldl1keep, [sp]
+
+prfm pldl1keep, [sp, #8]
+// CHECK: prfm pldl1keep, [sp, #8]
diff --git a/llvm/test/MC/AArch64/LFI/rcpc.s b/llvm/test/MC/AArch64/LFI/rcpc.s
new file mode 100644
index 0000000000000..eab3d8ed263d3
--- /dev/null
+++ b/llvm/test/MC/AArch64/LFI/rcpc.s
@@ -0,0 +1,19 @@
+// RUN: llvm-mc -triple aarch64_lfi %s | FileCheck %s
+
+.arch_extension rcpc
+
+ldapr x0, [x8]
+// CHECK: add x28, x27, w8, uxtw
+// CHECK-NEXT: ldapr x0, [x28]
+
+ldapr w0, [x8]
+// CHECK: add x28, x27, w8, uxtw
+// CHECK-NEXT: ldapr w0, [x28]
+
+ldaprh w0, [x8]
+// CHECK: add x28, x27, w8, uxtw
+// CHECK-NEXT: ldaprh w0, [x28]
+
+ldaprb w0, [x8]
+// CHECK: add x28, x27, w8, uxtw
+// CHECK-NEXT: ldaprb w0, [x28]
diff --git a/llvm/test/MC/AArch64/LFI/simd.s b/llvm/test/MC/AArch64/LFI/simd.s
new file mode 100644
index 0000000000000..8adb203a8df44
--- /dev/null
+++ b/llvm/test/MC/AArch64/LFI/simd.s
@@ -0,0 +1,650 @@
+// RUN: llvm-mc -triple aarch64_lfi %s | FileCheck %s
+
+// LD1/ST1 single structure (no post-index)
+
+ld1 { v0.b }[0], [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld1 { v0.b }[0], [x28]
+
+ld1 { v0.h }[1], [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld1 { v0.h }[1], [x28]
+
+ld1 { v0.s }[2], [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld1 { v0.s }[2], [x28]
+
+ld1 { v0.d }[1], [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld1 { v0.d }[1], [x28]
+
+st1 { v0.b }[0], [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st1 { v0.b }[0], [x28]
+
+st1 { v0.h }[1], [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st1 { v0.h }[1], [x28]
+
+st1 { v0.s }[2], [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st1 { v0.s }[2], [x28]
+
+st1 { v0.d }[1], [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st1 { v0.d }[1], [x28]
+
+// LD1/ST1 single structure with post-index (natural offset)
+
+ld1 { v0.b }[0], [x0], #1
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld1 { v0.b }[0], [x28]
+// CHECK-NEXT: add x0, x0, #1
+
+ld1 { v0.h }[1], [x0], #2
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld1 { v0.h }[1], [x28]
+// CHECK-NEXT: add x0, x0, #2
+
+ld1 { v0.s }[2], [x0], #4
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld1 { v0.s }[2], [x28]
+// CHECK-NEXT: add x0, x0, #4
+
+ld1 { v0.d }[1], [x0], #8
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld1 { v0.d }[1], [x28]
+// CHECK-NEXT: add x0, x0, #8
+
+st1 { v0.b }[0], [x0], #1
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st1 { v0.b }[0], [x28]
+// CHECK-NEXT: add x0, x0, #1
+
+st1 { v0.h }[1], [x0], #2
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st1 { v0.h }[1], [x28]
+// CHECK-NEXT: add x0, x0, #2
+
+st1 { v0.s }[2], [x0], #4
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st1 { v0.s }[2], [x28]
+// CHECK-NEXT: add x0, x0, #4
+
+st1 { v0.d }[1], [x0], #8
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st1 { v0.d }[1], [x28]
+// CHECK-NEXT: add x0, x0, #8
+
+// LD1/ST1 single structure with post-index (register offset)
+
+ld1 { v0.b }[0], [x0], x1
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld1 { v0.b }[0], [x28]
+// CHECK-NEXT: add x0, x0, x1
+
+ld1 { v0.s }[2], [x0], x1
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld1 { v0.s }[2], [x28]
+// CHECK-NEXT: add x0, x0, x1
+
+st1 { v0.d }[1], [x0], x1
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st1 { v0.d }[1], [x28]
+// CHECK-NEXT: add x0, x0, x1
+
+// LD1R (replicate single element to all lanes)
+
+ld1r { v0.8b }, [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld1r { v0.8b }, [x28]
+
+ld1r { v0.16b }, [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld1r { v0.16b }, [x28]
+
+ld1r { v0.4h }, [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld1r { v0.4h }, [x28]
+
+ld1r { v0.8h }, [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld1r { v0.8h }, [x28]
+
+ld1r { v0.2s }, [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld1r { v0.2s }, [x28]
+
+ld1r { v0.4s }, [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld1r { v0.4s }, [x28]
+
+ld1r { v0.1d }, [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld1r { v0.1d }, [x28]
+
+ld1r { v0.2d }, [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld1r { v0.2d }, [x28]
+
+// LD1R with post-index
+
+ld1r { v0.8b }, [x0], #1
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld1r { v0.8b }, [x28]
+// CHECK-NEXT: add x0, x0, #1
+
+ld1r { v0.4h }, [x0], #2
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld1r { v0.4h }, [x28]
+// CHECK-NEXT: add x0, x0, #2
+
+ld1r { v0.2s }, [x0], #4
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld1r { v0.2s }, [x28]
+// CHECK-NEXT: add x0, x0, #4
+
+ld1r { v0.1d }, [x0], #8
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld1r { v0.1d }, [x28]
+// CHECK-NEXT: add x0, x0, #8
+
+ld1r { v0.2d }, [x0], x1
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld1r { v0.2d }, [x28]
+// CHECK-NEXT: add x0, x0, x1
+
+// LD1/ST1 multiple structures (1-4 registers)
+
+ld1 { v0.16b }, [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld1 { v0.16b }, [x28]
+
+ld1 { v0.16b, v1.16b }, [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld1 { v0.16b, v1.16b }, [x28]
+
+ld1 { v0.16b, v1.16b, v2.16b }, [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld1 { v0.16b, v1.16b, v2.16b }, [x28]
+
+ld1 { v0.16b, v1.16b, v2.16b, v3.16b }, [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld1 { v0.16b, v1.16b, v2.16b, v3.16b }, [x28]
+
+st1 { v0.16b }, [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st1 { v0.16b }, [x28]
+
+st1 { v0.16b, v1.16b }, [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st1 { v0.16b, v1.16b }, [x28]
+
+st1 { v0.16b, v1.16b, v2.16b }, [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st1 { v0.16b, v1.16b, v2.16b }, [x28]
+
+st1 { v0.16b, v1.16b, v2.16b, v3.16b }, [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st1 { v0.16b, v1.16b, v2.16b, v3.16b }, [x28]
+
+// LD1/ST1 multiple structures with post-index
+
+ld1 { v0.16b }, [x0], #16
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld1 { v0.16b }, [x28]
+// CHECK-NEXT: add x0, x0, #16
+
+ld1 { v0.16b, v1.16b }, [x0], #32
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld1 { v0.16b, v1.16b }, [x28]
+// CHECK-NEXT: add x0, x0, #32
+
+ld1 { v0.16b, v1.16b, v2.16b }, [x0], #48
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld1 { v0.16b, v1.16b, v2.16b }, [x28]
+// CHECK-NEXT: add x0, x0, #48
+
+ld1 { v0.16b, v1.16b, v2.16b, v3.16b }, [x0], #64
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld1 { v0.16b, v1.16b, v2.16b, v3.16b }, [x28]
+// CHECK-NEXT: add x0, x0, #64
+
+ld1 { v0.16b }, [x0], x1
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld1 { v0.16b }, [x28]
+// CHECK-NEXT: add x0, x0, x1
+
+// LD2/ST2 multiple structures
+
+ld2 { v0.8b, v1.8b }, [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld2 { v0.8b, v1.8b }, [x28]
+
+ld2 { v0.16b, v1.16b }, [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld2 { v0.16b, v1.16b }, [x28]
+
+ld2 { v0.4h, v1.4h }, [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld2 { v0.4h, v1.4h }, [x28]
+
+ld2 { v0.8h, v1.8h }, [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld2 { v0.8h, v1.8h }, [x28]
+
+ld2 { v0.2s, v1.2s }, [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld2 { v0.2s, v1.2s }, [x28]
+
+ld2 { v0.4s, v1.4s }, [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld2 { v0.4s, v1.4s }, [x28]
+
+ld2 { v0.2d, v1.2d }, [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld2 { v0.2d, v1.2d }, [x28]
+
+st2 { v0.16b, v1.16b }, [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st2 { v0.16b, v1.16b }, [x28]
+
+ld2 { v0.16b, v1.16b }, [x0], #32
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld2 { v0.16b, v1.16b }, [x28]
+// CHECK-NEXT: add x0, x0, #32
+
+st2 { v0.16b, v1.16b }, [x0], #32
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st2 { v0.16b, v1.16b }, [x28]
+// CHECK-NEXT: add x0, x0, #32
+
+// LD3/ST3 multiple structures
+
+ld3 { v0.8b, v1.8b, v2.8b }, [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld3 { v0.8b, v1.8b, v2.8b }, [x28]
+
+ld3 { v0.16b, v1.16b, v2.16b }, [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld3 { v0.16b, v1.16b, v2.16b }, [x28]
+
+ld3 { v0.4s, v1.4s, v2.4s }, [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld3 { v0.4s, v1.4s, v2.4s }, [x28]
+
+st3 { v0.4s, v1.4s, v2.4s }, [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st3 { v0.4s, v1.4s, v2.4s }, [x28]
+
+ld3 { v0.4s, v1.4s, v2.4s }, [x0], #48
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld3 { v0.4s, v1.4s, v2.4s }, [x28]
+// CHECK-NEXT: add x0, x0, #48
+
+ld3 { v0.4s, v1.4s, v2.4s }, [x0], x1
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld3 { v0.4s, v1.4s, v2.4s }, [x28]
+// CHECK-NEXT: add x0, x0, x1
+
+// LD4/ST4 multiple structures
+
+ld4 { v0.8b, v1.8b, v2.8b, v3.8b }, [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld4 { v0.8b, v1.8b, v2.8b, v3.8b }, [x28]
+
+ld4 { v0.16b, v1.16b, v2.16b, v3.16b }, [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld4 { v0.16b, v1.16b, v2.16b, v3.16b }, [x28]
+
+ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x28]
+
+ld4 { v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld4 { v0.2d, v1.2d, v2.2d, v3.2d }, [x28]
+
+st4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x28]
+
+ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0], #64
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x28]
+// CHECK-NEXT: add x0, x0, #64
+
+ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0], x1
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x28]
+// CHECK-NEXT: add x0, x0, x1
+
+// LD2R/LD3R/LD4R (replicate)
+
+ld2r { v0.8b, v1.8b }, [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld2r { v0.8b, v1.8b }, [x28]
+
+ld3r { v0.4s, v1.4s, v2.4s }, [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld3r { v0.4s, v1.4s, v2.4s }, [x28]
+
+ld4r { v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld4r { v0.2d, v1.2d, v2.2d, v3.2d }, [x28]
+
+ld2r { v0.8b, v1.8b }, [x0], #2
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld2r { v0.8b, v1.8b }, [x28]
+// CHECK-NEXT: add x0, x0, #2
+
+ld3r { v0.4s, v1.4s, v2.4s }, [x0], #12
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld3r { v0.4s, v1.4s, v2.4s }, [x28]
+// CHECK-NEXT: add x0, x0, #12
+
+ld4r { v0.2d, v1.2d, v2.2d, v3.2d }, [x0], #32
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld4r { v0.2d, v1.2d, v2.2d, v3.2d }, [x28]
+// CHECK-NEXT: add x0, x0, #32
+
+// LD2/LD3/LD4 single structure (lane loads)
+
+ld2 { v0.b, v1.b }[0], [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld2 { v0.b, v1.b }[0], [x28]
+
+ld3 { v0.s, v1.s, v2.s }[1], [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld3 { v0.s, v1.s, v2.s }[1], [x28]
+
+ld4 { v0.d, v1.d, v2.d, v3.d }[0], [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld4 { v0.d, v1.d, v2.d, v3.d }[0], [x28]
+
+st2 { v0.h, v1.h }[3], [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st2 { v0.h, v1.h }[3], [x28]
+
+st3 { v0.s, v1.s, v2.s }[2], [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st3 { v0.s, v1.s, v2.s }[2], [x28]
+
+st4 { v0.d, v1.d, v2.d, v3.d }[1], [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st4 { v0.d, v1.d, v2.d, v3.d }[1], [x28]
+
+ld2 { v0.b, v1.b }[0], [x0], #2
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld2 { v0.b, v1.b }[0], [x28]
+// CHECK-NEXT: add x0, x0, #2
+
+ld3 { v0.s, v1.s, v2.s }[1], [x0], #12
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld3 { v0.s, v1.s, v2.s }[1], [x28]
+// CHECK-NEXT: add x0, x0, #12
+
+ld4 { v0.d, v1.d, v2.d, v3.d }[0], [x0], #32
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld4 { v0.d, v1.d, v2.d, v3.d }[0], [x28]
+// CHECK-NEXT: add x0, x0, #32
+
+ld2 { v0.s, v1.s }[1], [x0], x1
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld2 { v0.s, v1.s }[1], [x28]
+// CHECK-NEXT: add x0, x0, x1
+
+// ST2/ST3/ST4 lane stores with immediate post-index
+
+st2 { v0.b, v1.b }[0], [x0], #2
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st2 { v0.b, v1.b }[0], [x28]
+// CHECK-NEXT: add x0, x0, #2
+
+st2 { v0.h, v1.h }[1], [x0], #4
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st2 { v0.h, v1.h }[1], [x28]
+// CHECK-NEXT: add x0, x0, #4
+
+st2 { v0.s, v1.s }[2], [x0], #8
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st2 { v0.s, v1.s }[2], [x28]
+// CHECK-NEXT: add x0, x0, #8
+
+st2 { v0.d, v1.d }[1], [x0], #16
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st2 { v0.d, v1.d }[1], [x28]
+// CHECK-NEXT: add x0, x0, #16
+
+st3 { v0.b, v1.b, v2.b }[0], [x0], #3
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st3 { v0.b, v1.b, v2.b }[0], [x28]
+// CHECK-NEXT: add x0, x0, #3
+
+st3 { v0.h, v1.h, v2.h }[1], [x0], #6
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st3 { v0.h, v1.h, v2.h }[1], [x28]
+// CHECK-NEXT: add x0, x0, #6
+
+st3 { v0.s, v1.s, v2.s }[2], [x0], #12
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st3 { v0.s, v1.s, v2.s }[2], [x28]
+// CHECK-NEXT: add x0, x0, #12
+
+st3 { v0.d, v1.d, v2.d }[0], [x0], #24
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st3 { v0.d, v1.d, v2.d }[0], [x28]
+// CHECK-NEXT: add x0, x0, #24
+
+st4 { v0.b, v1.b, v2.b, v3.b }[0], [x0], #4
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st4 { v0.b, v1.b, v2.b, v3.b }[0], [x28]
+// CHECK-NEXT: add x0, x0, #4
+
+st4 { v0.h, v1.h, v2.h, v3.h }[1], [x0], #8
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st4 { v0.h, v1.h, v2.h, v3.h }[1], [x28]
+// CHECK-NEXT: add x0, x0, #8
+
+st4 { v0.s, v1.s, v2.s, v3.s }[2], [x0], #16
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st4 { v0.s, v1.s, v2.s, v3.s }[2], [x28]
+// CHECK-NEXT: add x0, x0, #16
+
+st4 { v0.d, v1.d, v2.d, v3.d }[0], [x0], #32
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st4 { v0.d, v1.d, v2.d, v3.d }[0], [x28]
+// CHECK-NEXT: add x0, x0, #32
+
+// ST1/ST2/ST3/ST4 lane stores with register post-index
+
+st1 { v0.b }[0], [x0], x1
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st1 { v0.b }[0], [x28]
+// CHECK-NEXT: add x0, x0, x1
+
+st1 { v0.h }[1], [x0], x1
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st1 { v0.h }[1], [x28]
+// CHECK-NEXT: add x0, x0, x1
+
+st1 { v0.s }[2], [x0], x1
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st1 { v0.s }[2], [x28]
+// CHECK-NEXT: add x0, x0, x1
+
+st2 { v0.b, v1.b }[0], [x0], x1
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st2 { v0.b, v1.b }[0], [x28]
+// CHECK-NEXT: add x0, x0, x1
+
+st2 { v0.s, v1.s }[1], [x0], x1
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st2 { v0.s, v1.s }[1], [x28]
+// CHECK-NEXT: add x0, x0, x1
+
+st2 { v0.d, v1.d }[0], [x0], x1
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st2 { v0.d, v1.d }[0], [x28]
+// CHECK-NEXT: add x0, x0, x1
+
+st3 { v0.b, v1.b, v2.b }[0], [x0], x1
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st3 { v0.b, v1.b, v2.b }[0], [x28]
+// CHECK-NEXT: add x0, x0, x1
+
+st3 { v0.s, v1.s, v2.s }[1], [x0], x1
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st3 { v0.s, v1.s, v2.s }[1], [x28]
+// CHECK-NEXT: add x0, x0, x1
+
+st3 { v0.d, v1.d, v2.d }[0], [x0], x1
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st3 { v0.d, v1.d, v2.d }[0], [x28]
+// CHECK-NEXT: add x0, x0, x1
+
+st4 { v0.b, v1.b, v2.b, v3.b }[0], [x0], x1
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st4 { v0.b, v1.b, v2.b, v3.b }[0], [x28]
+// CHECK-NEXT: add x0, x0, x1
+
+st4 { v0.s, v1.s, v2.s, v3.s }[1], [x0], x1
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st4 { v0.s, v1.s, v2.s, v3.s }[1], [x28]
+// CHECK-NEXT: add x0, x0, x1
+
+st4 { v0.d, v1.d, v2.d, v3.d }[0], [x0], x1
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st4 { v0.d, v1.d, v2.d, v3.d }[0], [x28]
+// CHECK-NEXT: add x0, x0, x1
+
+ld3 { v0.b, v1.b, v2.b }[0], [x0], x1
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld3 { v0.b, v1.b, v2.b }[0], [x28]
+// CHECK-NEXT: add x0, x0, x1
+
+ld3 { v0.d, v1.d, v2.d }[0], [x0], x1
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld3 { v0.d, v1.d, v2.d }[0], [x28]
+// CHECK-NEXT: add x0, x0, x1
+
+ld4 { v0.b, v1.b, v2.b, v3.b }[0], [x0], x1
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld4 { v0.b, v1.b, v2.b, v3.b }[0], [x28]
+// CHECK-NEXT: add x0, x0, x1
+
+ld4 { v0.s, v1.s, v2.s, v3.s }[1], [x0], x1
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld4 { v0.s, v1.s, v2.s, v3.s }[1], [x28]
+// CHECK-NEXT: add x0, x0, x1
+
+// ST1/ST2/ST3/ST4 multi-register stores with register post-index
+
+st1 { v0.16b }, [x0], x1
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st1 { v0.16b }, [x28]
+// CHECK-NEXT: add x0, x0, x1
+
+st1 { v0.16b, v1.16b }, [x0], x1
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st1 { v0.16b, v1.16b }, [x28]
+// CHECK-NEXT: add x0, x0, x1
+
+st1 { v0.16b, v1.16b, v2.16b }, [x0], x1
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st1 { v0.16b, v1.16b, v2.16b }, [x28]
+// CHECK-NEXT: add x0, x0, x1
+
+st1 { v0.16b, v1.16b, v2.16b, v3.16b }, [x0], x1
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st1 { v0.16b, v1.16b, v2.16b, v3.16b }, [x28]
+// CHECK-NEXT: add x0, x0, x1
+
+st2 { v0.16b, v1.16b }, [x0], x1
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st2 { v0.16b, v1.16b }, [x28]
+// CHECK-NEXT: add x0, x0, x1
+
+st2 { v0.4s, v1.4s }, [x0], x1
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st2 { v0.4s, v1.4s }, [x28]
+// CHECK-NEXT: add x0, x0, x1
+
+st3 { v0.16b, v1.16b, v2.16b }, [x0], x1
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st3 { v0.16b, v1.16b, v2.16b }, [x28]
+// CHECK-NEXT: add x0, x0, x1
+
+st3 { v0.4s, v1.4s, v2.4s }, [x0], x1
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st3 { v0.4s, v1.4s, v2.4s }, [x28]
+// CHECK-NEXT: add x0, x0, x1
+
+st4 { v0.16b, v1.16b, v2.16b, v3.16b }, [x0], x1
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st4 { v0.16b, v1.16b, v2.16b, v3.16b }, [x28]
+// CHECK-NEXT: add x0, x0, x1
+
+st4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0], x1
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x28]
+// CHECK-NEXT: add x0, x0, x1
+
+st4 { v0.2d, v1.2d, v2.2d, v3.2d }, [x0], x1
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: st4 { v0.2d, v1.2d, v2.2d, v3.2d }, [x28]
+// CHECK-NEXT: add x0, x0, x1
+
+ld2 { v0.16b, v1.16b }, [x0], x1
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld2 { v0.16b, v1.16b }, [x28]
+// CHECK-NEXT: add x0, x0, x1
+
+ld2 { v0.4s, v1.4s }, [x0], x1
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: ld2 { v0.4s, v1.4s }, [x28]
+// CHECK-NEXT: add x0, x0, x1
+
+// SIMD post-index with SP base.
+
+ld1 { v0.16b }, [sp], #16
+// CHECK: ld1 { v0.16b }, [sp], #16
+
+ld1 { v0.16b }, [sp], x1
+// CHECK: ld1 { v0.16b }, [sp]
+// CHECK-NEXT: add x26, sp, x1
+// CHECK-NEXT: add sp, x27, w26, uxtw
+
+st1 { v0.16b }, [sp], #16
+// CHECK: st1 { v0.16b }, [sp], #16
+
+st1 { v0.16b }, [sp], x1
+// CHECK: st1 { v0.16b }, [sp]
+// CHECK-NEXT: add x26, sp, x1
+// CHECK-NEXT: add sp, x27, w26, uxtw
+
+ld2 { v0.16b, v1.16b }, [sp], #32
+// CHECK: ld2 { v0.16b, v1.16b }, [sp], #32
+
+ld2 { v0.16b, v1.16b }, [sp], x2
+// CHECK: ld2 { v0.16b, v1.16b }, [sp]
+// CHECK-NEXT: add x26, sp, x2
+// CHECK-NEXT: add sp, x27, w26, uxtw
+
+st4 { v0.4s, v1.4s, v2.4s, v3.4s }, [sp], x1
+// CHECK: st4 { v0.4s, v1.4s, v2.4s, v3.4s }, [sp]
+// CHECK-NEXT: add x26, sp, x1
+// CHECK-NEXT: add sp, x27, w26, uxtw
+
+ld1 { v0.b }[0], [sp], #1
+// CHECK: ld1 { v0.b }[0], [sp], #1
+
+ld1 { v0.b }[0], [sp], x1
+// CHECK: ld1 { v0.b }[0], [sp]
+// CHECK-NEXT: add x26, sp, x1
+// CHECK-NEXT: add sp, x27, w26, uxtw
+
+st1 { v0.s }[2], [sp], x1
+// CHECK: st1 { v0.s }[2], [sp]
+// CHECK-NEXT: add x26, sp, x1
+// CHECK-NEXT: add sp, x27, w26, uxtw
diff --git a/llvm/test/MC/AArch64/LFI/stack.s b/llvm/test/MC/AArch64/LFI/stack.s
new file mode 100644
index 0000000000000..c0b8c37082d9f
--- /dev/null
+++ b/llvm/test/MC/AArch64/LFI/stack.s
@@ -0,0 +1,41 @@
+// RUN: llvm-mc -triple aarch64_lfi %s | FileCheck %s
+
+ldr x0, [sp, #16]!
+// CHECK: ldr x0, [sp, #16]!
+
+ldr x0, [sp], #16
+// CHECK: ldr x0, [sp], #16
+
+str x0, [sp, #16]!
+// CHECK: str x0, [sp, #16]!
+
+str x0, [sp], #16
+// CHECK: str x0, [sp], #16
+
+mov sp, x0
+// CHECK: add x26, x0, #0
+// CHECK-NEXT: add sp, x27, w26, uxtw
+
+add sp, sp, #8
+// CHECK: add x26, sp, #8
+// CHECK-NEXT: add sp, x27, w26, uxtw
+
+sub sp, sp, #8
+// CHECK: sub x26, sp, #8
+// CHECK-NEXT: add sp, x27, w26, uxtw
+
+add sp, sp, x0
+// CHECK: add x26, sp, x0
+// CHECK-NEXT: add sp, x27, w26, uxtw
+
+sub sp, sp, x0
+// CHECK: sub x26, sp, x0
+// CHECK-NEXT: add sp, x27, w26, uxtw
+
+sub sp, sp, #1, lsl #12
+// CHECK: sub x26, sp, #1, lsl #12
+// CHECK-NEXT: add sp, x27, w26, uxtw
+
+ldp x29, x30, [sp], #16
+// CHECK: ldp x29, x30, [sp], #16
+// CHECK-NEXT: add x30, x27, w30, uxtw
diff --git a/llvm/test/MC/AArch64/LFI/sys.s b/llvm/test/MC/AArch64/LFI/sys.s
index df4b694600851..045c333962537 100644
--- a/llvm/test/MC/AArch64/LFI/sys.s
+++ b/llvm/test/MC/AArch64/LFI/sys.s
@@ -5,3 +5,27 @@ svc #0
// CHECK-NEXT: ldur x30, [x27, #-8]
// CHECK-NEXT: blr x30
// CHECK-NEXT: add x30, x27, w26, uxtw
+
+dc zva, x0
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: dc zva, x28
+
+dc zva, x5
+// CHECK: add x28, x27, w5, uxtw
+// CHECK-NEXT: dc zva, x28
+
+dc cvac, x0
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: dc cvac, x28
+
+dc cvau, x1
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: dc cvau, x28
+
+dc civac, x2
+// CHECK: add x28, x27, w2, uxtw
+// CHECK-NEXT: dc civac, x28
+
+ic ivau, x4
+// CHECK: add x28, x27, w4, uxtw
+// CHECK-NEXT: ic ivau, x28
>From a561be134e012a5b424de35bc8bab11e04f1abfc Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia at gmail.com>
Date: Thu, 30 Apr 2026 15:46:26 -0400
Subject: [PATCH 2/2] Add size estimation for sp modification
---
llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 18 +++++++++++++-----
1 file changed, 13 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index decb3572c6638..2cdbf24c52e8c 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -140,13 +140,17 @@ static std::optional<unsigned> getLFIInstSizeInBytes(const MachineInstr &MI) {
break;
}
- // Instructions that explicitly modify LR get an extra LR mask.
+ // Detect instructions that explicitly define SP or LR.
bool ModifiesLR = false;
- for (const MachineOperand &MO : MI.explicit_operands())
- if (MO.isReg() && MO.isDef() && MO.getReg() == AArch64::LR) {
+ bool ModifiesSP = false;
+ for (const MachineOperand &MO : MI.defs()) {
+ if (!MO.isReg())
+ continue;
+ if (MO.getReg() == AArch64::LR)
ModifiesLR = true;
- break;
- }
+ else if (MO.getReg() == AArch64::SP)
+ ModifiesSP = true;
+ }
// Memory accesses expand to a base-register guard plus the rewritten access
// (8 bytes), with an extra base-register update for pre/post-index forms (12
@@ -160,6 +164,10 @@ static std::optional<unsigned> getLFIInstSizeInBytes(const MachineInstr &MI) {
return Size;
}
+ // SP modification expands to 2 instructions.
+ if (ModifiesSP)
+ return 8;
+
// Non-memory instructions that modify LR expand to 2 instructions.
if (ModifiesLR)
return 8;
More information about the llvm-commits
mailing list