[llvm] [PAC][AArch64] Lower ptrauth constants in code (PR #94241)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Jun 24 10:41:12 PDT 2024
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-globalisel
@llvm/pr-subscribers-llvm-selectiondag
Author: Daniil Kovalev (kovdan01)
<details>
<summary>Changes</summary>
Depends on #<!-- -->94240.
Define the following pseudos for lowering ptrauth constants in code:
- non-`extern_weak`:
- no GOT load needed: `MOVaddrPAC` - similar to `MOVaddr`, with added PAC;
- GOT load needed: `LOADgotPAC` - similar to `LOADgot`, with added PAC;
- `extern_weak`: `LOADauthptrstatic` - similar to `LOADgot`, but use a special stub slot named `sym$auth_ptr$key$disc` filled by dynamic linker during relocation resolving instead of a GOT slot.
Co-authored-by: Ahmed Bougacha <ahmed@<!-- -->bougacha.org>
---
Patch is 73.19 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/94241.diff
25 Files Affected:
- (modified) llvm/docs/GlobalISel/GenericOpcode.rst (+10)
- (modified) llvm/include/llvm/CodeGen/ISDOpcodes.h (+6)
- (modified) llvm/include/llvm/CodeGen/MachineModuleInfoImpls.h (+20)
- (modified) llvm/include/llvm/Support/TargetOpcodes.def (+3)
- (modified) llvm/include/llvm/Target/GenericOpcodes.td (+6)
- (modified) llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp (+10-1)
- (modified) llvm/lib/CodeGen/MachineModuleInfoImpls.cpp (+23)
- (modified) llvm/lib/CodeGen/MachineVerifier.cpp (+6)
- (modified) llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp (+7)
- (modified) llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp (+2-2)
- (modified) llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp (+252)
- (modified) llvm/lib/Target/AArch64/AArch64ISelLowering.cpp (+132)
- (modified) llvm/lib/Target/AArch64/AArch64ISelLowering.h (+6)
- (modified) llvm/lib/Target/AArch64/AArch64InstrInfo.td (+32)
- (modified) llvm/lib/Target/AArch64/AArch64TargetObjectFile.cpp (+37)
- (modified) llvm/lib/Target/AArch64/AArch64TargetObjectFile.h (+6)
- (modified) llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp (+144)
- (modified) llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp (+3)
- (modified) llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir (+4)
- (added) llvm/test/CodeGen/AArch64/GlobalISel/ptrauth-constant-in-code.ll (+235)
- (added) llvm/test/CodeGen/AArch64/ptrauth-constant-in-code.ll (+230)
- (modified) llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-cxx.td (+1-1)
- (modified) llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-variadics.td (+1-1)
- (modified) llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td (+31-31)
- (modified) llvm/test/TableGen/GlobalISelEmitter.td (+1-1)
``````````diff
diff --git a/llvm/docs/GlobalISel/GenericOpcode.rst b/llvm/docs/GlobalISel/GenericOpcode.rst
index 5c28c6fcd30fb..19adb20f70030 100644
--- a/llvm/docs/GlobalISel/GenericOpcode.rst
+++ b/llvm/docs/GlobalISel/GenericOpcode.rst
@@ -60,6 +60,16 @@ The address of a global value.
%0(p0) = G_GLOBAL_VALUE @var_local
+G_PTRAUTH_GLOBAL_VALUE
+^^^^^^^^^^^^^^^^^^^^^^
+
+The signed address of a global value. Operands: address to be signed (pointer),
+key (32-bit imm), address for address discrimination (zero if not needed) and
+an extra discriminator (64-bit imm).
+
+.. code-block:: none
+ %0:_(p0) = G_PTRAUTH_GLOBAL_VALUE %1:_(p0), s32, %2:_(p0), s64
+
G_BLOCK_ADDR
^^^^^^^^^^^^
diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h
index c8c86ed5eef29..a6f4a03ec5de2 100644
--- a/llvm/include/llvm/CodeGen/ISDOpcodes.h
+++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h
@@ -83,6 +83,12 @@ enum NodeType {
ExternalSymbol,
BlockAddress,
+ /// A ptrauth constant.
+ /// ptr, key, addr-disc, disc
+ /// Note that the addr-disc can be a non-constant value, to allow representing
+ /// a constant global address signed using address-diversification, in code.
+ PtrAuthGlobalAddress,
+
/// The address of the GOT
GLOBAL_OFFSET_TABLE,
diff --git a/llvm/include/llvm/CodeGen/MachineModuleInfoImpls.h b/llvm/include/llvm/CodeGen/MachineModuleInfoImpls.h
index f8a328f13eded..64d841d86c7c4 100644
--- a/llvm/include/llvm/CodeGen/MachineModuleInfoImpls.h
+++ b/llvm/include/llvm/CodeGen/MachineModuleInfoImpls.h
@@ -61,10 +61,20 @@ class MachineModuleInfoMachO : public MachineModuleInfoImpl {
/// MachineModuleInfoELF - This is a MachineModuleInfoImpl implementation
/// for ELF targets.
class MachineModuleInfoELF : public MachineModuleInfoImpl {
+public:
+ struct AuthStubInfo {
+ const MCExpr *AuthPtrRef;
+ };
+
+private:
/// GVStubs - These stubs are used to materialize global addresses in PIC
/// mode.
DenseMap<MCSymbol *, StubValueTy> GVStubs;
+ /// AuthPtrStubs - These stubs are used to materialize signed addresses for
+ /// extern_weak symbols.
+ DenseMap<MCSymbol *, AuthStubInfo> AuthPtrStubs;
+
virtual void anchor(); // Out of line virtual method.
public:
@@ -75,9 +85,19 @@ class MachineModuleInfoELF : public MachineModuleInfoImpl {
return GVStubs[Sym];
}
+ AuthStubInfo &getAuthPtrStubEntry(MCSymbol *Sym) {
+ assert(Sym && "Key cannot be null");
+ return AuthPtrStubs[Sym];
+ }
+
/// Accessor methods to return the set of stubs in sorted order.
SymbolListTy GetGVStubList() { return getSortedStubs(GVStubs); }
+
+ using AuthStubPairTy = std::pair<MCSymbol *, AuthStubInfo>;
+ typedef std::vector<AuthStubPairTy> AuthStubListTy;
+
+ AuthStubListTy getAuthGVStubList();
};
/// MachineModuleInfoCOFF - This is a MachineModuleInfoImpl implementation
diff --git a/llvm/include/llvm/Support/TargetOpcodes.def b/llvm/include/llvm/Support/TargetOpcodes.def
index 559a588c25148..2bb4a0ffd43fe 100644
--- a/llvm/include/llvm/Support/TargetOpcodes.def
+++ b/llvm/include/llvm/Support/TargetOpcodes.def
@@ -294,6 +294,9 @@ HANDLE_TARGET_OPCODE(G_FRAME_INDEX)
/// Generic reference to global value.
HANDLE_TARGET_OPCODE(G_GLOBAL_VALUE)
+/// Generic ptrauth-signed reference to global value.
+HANDLE_TARGET_OPCODE(G_PTRAUTH_GLOBAL_VALUE)
+
/// Generic instruction to materialize the address of an object in the constant
/// pool.
HANDLE_TARGET_OPCODE(G_CONSTANT_POOL)
diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td
index c40498e554215..6846a7e008f3a 100644
--- a/llvm/include/llvm/Target/GenericOpcodes.td
+++ b/llvm/include/llvm/Target/GenericOpcodes.td
@@ -110,6 +110,12 @@ def G_GLOBAL_VALUE : GenericInstruction {
let hasSideEffects = false;
}
+def G_PTRAUTH_GLOBAL_VALUE : GenericInstruction {
+ let OutOperandList = (outs type0:$dst);
+ let InOperandList = (ins unknown:$addr, i32imm:$key, type1:$addrdisc, i64imm:$disc);
+ let hasSideEffects = 0;
+}
+
def G_CONSTANT_POOL : GenericInstruction {
let OutOperandList = (outs type0:$dst);
let InOperandList = (ins unknown:$src);
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 7efcf21460260..0b59cd836fcc9 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -3486,7 +3486,16 @@ bool IRTranslator::translate(const Constant &C, Register Reg) {
EntryBuilder->buildConstant(Reg, 0);
else if (auto GV = dyn_cast<GlobalValue>(&C))
EntryBuilder->buildGlobalValue(Reg, GV);
- else if (auto CAZ = dyn_cast<ConstantAggregateZero>(&C)) {
+ else if (auto CPA = dyn_cast<ConstantPtrAuth>(&C)) {
+ Register Addr = getOrCreateVReg(*CPA->getPointer());
+ Register AddrDisc = getOrCreateVReg(*CPA->getAddrDiscriminator());
+ EntryBuilder->buildInstr(TargetOpcode::G_PTRAUTH_GLOBAL_VALUE)
+ .addDef(Reg)
+ .addUse(Addr)
+ .addImm(CPA->getKey()->getZExtValue())
+ .addUse(AddrDisc)
+ .addImm(CPA->getDiscriminator()->getZExtValue());
+ } else if (auto CAZ = dyn_cast<ConstantAggregateZero>(&C)) {
if (!isa<FixedVectorType>(CAZ->getType()))
return false;
// Return the scalar if it is a <1 x Ty> vector.
diff --git a/llvm/lib/CodeGen/MachineModuleInfoImpls.cpp b/llvm/lib/CodeGen/MachineModuleInfoImpls.cpp
index 9c3b31935f6d6..f114f1ecc0bae 100644
--- a/llvm/lib/CodeGen/MachineModuleInfoImpls.cpp
+++ b/llvm/lib/CodeGen/MachineModuleInfoImpls.cpp
@@ -13,6 +13,7 @@
#include "llvm/CodeGen/MachineModuleInfoImpls.h"
#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/STLExtras.h"
#include "llvm/MC/MCSymbol.h"
using namespace llvm;
@@ -41,3 +42,25 @@ MachineModuleInfoImpl::SymbolListTy MachineModuleInfoImpl::getSortedStubs(
Map.clear();
return List;
}
+
+template <typename MachineModuleInfoTarget>
+static typename MachineModuleInfoTarget::AuthStubListTy getAuthGVStubListHelper(
+ DenseMap<MCSymbol *, typename MachineModuleInfoTarget::AuthStubInfo>
+ &AuthPtrStubs) {
+ typename MachineModuleInfoTarget::AuthStubListTy List(AuthPtrStubs.begin(),
+ AuthPtrStubs.end());
+
+ if (!List.empty())
+ llvm::sort(List.begin(), List.end(),
+ [](const typename MachineModuleInfoTarget::AuthStubPairTy &LHS,
+ const typename MachineModuleInfoTarget::AuthStubPairTy &RHS) {
+ return LHS.first->getName() < RHS.first->getName();
+ });
+
+ AuthPtrStubs.clear();
+ return List;
+}
+
+MachineModuleInfoELF::AuthStubListTy MachineModuleInfoELF::getAuthGVStubList() {
+ return getAuthGVStubListHelper<MachineModuleInfoELF>(AuthPtrStubs);
+}
diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp
index 9ea238c61ed91..0c8a0f2b24a1e 100644
--- a/llvm/lib/CodeGen/MachineVerifier.cpp
+++ b/llvm/lib/CodeGen/MachineVerifier.cpp
@@ -2066,6 +2066,12 @@ void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) {
report("Dst operand 0 must be a pointer", MI);
break;
}
+ case TargetOpcode::G_PTRAUTH_GLOBAL_VALUE: {
+ const MachineOperand &AddrOp = MI->getOperand(1);
+ if (!AddrOp.isReg() || !MRI->getType(AddrOp.getReg()).isPointer())
+ report("addr operand must be a pointer", &AddrOp, 1);
+ break;
+ }
default:
break;
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 8838cce9810f8..3adaa6e7f5564 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -1802,6 +1802,13 @@ SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
+ if (const ConstantPtrAuth *CPA = dyn_cast<ConstantPtrAuth>(C)) {
+ return DAG.getNode(ISD::PtrAuthGlobalAddress, getCurSDLoc(), VT,
+ getValue(CPA->getPointer()), getValue(CPA->getKey()),
+ getValue(CPA->getAddrDiscriminator()),
+ getValue(CPA->getDiscriminator()));
+ }
+
if (isa<ConstantPointerNull>(C)) {
unsigned AS = V->getType()->getPointerAddressSpace();
return DAG.getConstant(0, getCurSDLoc(),
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
index 12a7b7f11778d..1459c2bf00ed3 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
@@ -75,6 +75,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
}
return "<<Unknown Node #" + utostr(getOpcode()) + ">>";
+ // clang-format off
#ifndef NDEBUG
case ISD::DELETED_NODE: return "<<Deleted Node!>>";
#endif
@@ -126,6 +127,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
case ISD::ConstantFP: return "ConstantFP";
case ISD::GlobalAddress: return "GlobalAddress";
case ISD::GlobalTLSAddress: return "GlobalTLSAddress";
+ case ISD::PtrAuthGlobalAddress: return "PtrAuthGlobalAddress";
case ISD::FrameIndex: return "FrameIndex";
case ISD::JumpTable: return "JumpTable";
case ISD::JUMP_TABLE_DEBUG_INFO:
@@ -168,8 +170,6 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
return "OpaqueTargetConstant";
return "TargetConstant";
- // clang-format off
-
case ISD::TargetConstantFP: return "TargetConstantFP";
case ISD::TargetGlobalAddress: return "TargetGlobalAddress";
case ISD::TargetGlobalTLSAddress: return "TargetGlobalTLSAddress";
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index da11539eab348..611c3b579ff15 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -133,6 +133,13 @@ class AArch64AsmPrinter : public AsmPrinter {
unsigned emitPtrauthDiscriminator(uint16_t Disc, unsigned AddrDisc,
unsigned &InstsEmitted);
+ // Emit the sequence for LOADauthptrstatic
+ void LowerLOADauthptrstatic(const MachineInstr &MI);
+
+ // Emit the sequence for LOADgotPAC/MOVaddrPAC (either GOT adrp-ldr or
+ // adrp-add followed by PAC sign)
+ void LowerMOVaddrPAC(const MachineInstr &MI);
+
/// tblgen'erated driver function for lowering simple MI->MC
/// pseudo instructions.
bool emitPseudoExpansionLowering(MCStreamer &OutStreamer,
@@ -840,6 +847,15 @@ void AArch64AsmPrinter::emitHwasanMemaccessSymbols(Module &M) {
}
}
+template <typename MachineModuleInfoTarget>
+static void emitAuthenticatedPointer(
+ MCStreamer &OutStreamer, MCSymbol *StubLabel,
+ const typename MachineModuleInfoTarget::AuthStubInfo &StubInfo) {
+ // sym$auth_ptr$key$disc:
+ OutStreamer.emitLabel(StubLabel);
+ OutStreamer.emitValue(StubInfo.AuthPtrRef, /*size=*/8);
+}
+
void AArch64AsmPrinter::emitEndOfAsmFile(Module &M) {
emitHwasanMemaccessSymbols(M);
@@ -853,6 +869,25 @@ void AArch64AsmPrinter::emitEndOfAsmFile(Module &M) {
OutStreamer->emitAssemblerFlag(MCAF_SubsectionsViaSymbols);
}
+ if (TT.isOSBinFormatELF()) {
+ // Output authenticated pointers as indirect symbols, if we have any.
+ MachineModuleInfoELF &MMIELF = MMI->getObjFileInfo<MachineModuleInfoELF>();
+
+ auto Stubs = MMIELF.getAuthGVStubList();
+
+ if (!Stubs.empty()) {
+ const TargetLoweringObjectFile &TLOF = getObjFileLowering();
+ OutStreamer->switchSection(TLOF.getDataSection());
+ emitAlignment(Align(8));
+
+ for (const auto &Stub : Stubs)
+ emitAuthenticatedPointer<MachineModuleInfoELF>(*OutStreamer, Stub.first,
+ Stub.second);
+
+ OutStreamer->addBlankLine();
+ }
+ }
+
// Emit stack and fault map information.
FM.serializeToFaultMapSection();
@@ -1623,6 +1658,214 @@ AArch64AsmPrinter::lowerConstantPtrAuth(const ConstantPtrAuth &CPA) {
CPA.hasAddressDiscriminator(), Ctx);
}
+void AArch64AsmPrinter::LowerLOADauthptrstatic(const MachineInstr &MI) {
+ unsigned DstReg = MI.getOperand(0).getReg();
+ const MachineOperand &GAOp = MI.getOperand(1);
+ const uint64_t KeyC = MI.getOperand(2).getImm();
+ assert(KeyC <= AArch64PACKey::LAST &&
+ "key is out of range [0, AArch64PACKey::LAST]");
+ const auto Key = (AArch64PACKey::ID)KeyC;
+ const uint64_t Disc = MI.getOperand(3).getImm();
+ assert(isUInt<16>(Disc) &&
+ "constant discriminator is out of range [0, 0xffff]");
+
+ // Emit instruction sequence like the following:
+ // ADRP x16, symbol$auth_ptr$key$disc
+ // LDR x16, [x16, :lo12:symbol$auth_ptr$key$disc]
+ //
+ // Where the $auth_ptr$ symbol is the stub slot containing the signed pointer
+ // to symbol.
+ assert(TM.getTargetTriple().isOSBinFormatELF() &&
+ "LOADauthptrstatic is implemented only for ELF");
+ const auto &TLOF =
+ static_cast<const AArch64_ELFTargetObjectFile &>(getObjFileLowering());
+
+ assert(GAOp.getOffset() == 0 &&
+ "non-zero offset for $auth_ptr$ stub slots is not supported");
+ const MCSymbol *GASym = TM.getSymbol(GAOp.getGlobal());
+ MCSymbol *AuthPtrStubSym =
+ TLOF.getAuthPtrSlotSymbol(TM, &MF->getMMI(), GASym, Key, Disc);
+
+ MachineOperand StubMOHi =
+ MachineOperand::CreateMCSymbol(AuthPtrStubSym, AArch64II::MO_PAGE);
+ MachineOperand StubMOLo = MachineOperand::CreateMCSymbol(
+ AuthPtrStubSym, AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
+ MCOperand StubMCHi, StubMCLo;
+
+ MCInstLowering.lowerOperand(StubMOHi, StubMCHi);
+ MCInstLowering.lowerOperand(StubMOLo, StubMCLo);
+
+ EmitToStreamer(
+ *OutStreamer,
+ MCInstBuilder(AArch64::ADRP).addReg(DstReg).addOperand(StubMCHi));
+
+ EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::LDRXui)
+ .addReg(DstReg)
+ .addReg(DstReg)
+ .addOperand(StubMCLo));
+}
+
+void AArch64AsmPrinter::LowerMOVaddrPAC(const MachineInstr &MI) {
+ unsigned InstsEmitted = 0;
+ auto EmitAndIncrement = [this, &InstsEmitted](const MCInst &Inst) {
+ EmitToStreamer(*OutStreamer, Inst);
+ ++InstsEmitted;
+ };
+
+ const bool IsGOTLoad = MI.getOpcode() == AArch64::LOADgotPAC;
+ MachineOperand GAOp = MI.getOperand(0);
+ const uint64_t KeyC = MI.getOperand(1).getImm();
+ assert(KeyC <= AArch64PACKey::LAST &&
+ "key is out of range [0, AArch64PACKey::LAST]");
+ const auto Key = (AArch64PACKey::ID)KeyC;
+ const unsigned AddrDisc = MI.getOperand(2).getReg();
+ const uint64_t Disc = MI.getOperand(3).getImm();
+ assert(isUInt<16>(Disc) &&
+ "constant discriminator is out of range [0, 0xffff]");
+
+ const int64_t Offset = GAOp.getOffset();
+ GAOp.setOffset(0);
+
+ // Emit:
+ // target materialization:
+ // - via GOT:
+ // adrp x16, :got:target
+ // ldr x16, [x16, :got_lo12:target]
+ // add offset to x16 if offset != 0
+ //
+ // - direct:
+ // adrp x16, target
+ // add x16, x16, :lo12:target
+ // add offset to x16 if offset != 0
+ //
+ // add offset to x16:
+ // - abs(offset) fits 24 bits:
+ // add/sub x16, x16, #<offset>[, #lsl 12] (up to 2 instructions)
+ // - abs(offset) does not fit 24 bits:
+ // - offset < 0:
+ // movn+movk sequence filling x17 register with the offset (up to 4
+ // instructions)
+ // add x16, x16, x17
+ // - offset > 0:
+ // movz+movk sequence filling x17 register with the offset (up to 4
+ // instructions)
+ // add x16, x16, x17
+ //
+ // signing:
+ // - 0 discriminator:
+ // paciza x16
+ // - Non-0 discriminator, no address discriminator:
+ // mov x17, #Disc
+ // pacia x16, x17
+ // - address discriminator (with potentially folded immediate discriminator):
+ // pacia x16, xAddrDisc
+
+ MachineOperand GAMOHi(GAOp), GAMOLo(GAOp);
+ MCOperand GAMCHi, GAMCLo;
+
+ GAMOHi.setTargetFlags(AArch64II::MO_PAGE);
+ GAMOLo.setTargetFlags(AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
+ if (IsGOTLoad) {
+ GAMOHi.addTargetFlag(AArch64II::MO_GOT);
+ GAMOLo.addTargetFlag(AArch64II::MO_GOT);
+ }
+
+ MCInstLowering.lowerOperand(GAMOHi, GAMCHi);
+ MCInstLowering.lowerOperand(GAMOLo, GAMCLo);
+
+ EmitAndIncrement(
+ MCInstBuilder(AArch64::ADRP).addReg(AArch64::X16).addOperand(GAMCHi));
+
+ if (IsGOTLoad) {
+ EmitAndIncrement(MCInstBuilder(AArch64::LDRXui)
+ .addReg(AArch64::X16)
+ .addReg(AArch64::X16)
+ .addOperand(GAMCLo));
+ } else {
+ EmitAndIncrement(MCInstBuilder(AArch64::ADDXri)
+ .addReg(AArch64::X16)
+ .addReg(AArch64::X16)
+ .addOperand(GAMCLo)
+ .addImm(0));
+ }
+
+ if (Offset != 0) {
+ const uint64_t AbsOffset = (Offset > 0 ? Offset : -((uint64_t)Offset));
+ const bool IsNeg = Offset < 0;
+ if (isUInt<24>(AbsOffset)) {
+ for (int BitPos = 0; BitPos != 24 && (AbsOffset >> BitPos);
+ BitPos += 12) {
+ EmitAndIncrement(
+ MCInstBuilder(IsNeg ? AArch64::SUBXri : AArch64::ADDXri)
+ .addReg(AArch64::X16)
+ .addReg(AArch64::X16)
+ .addImm((AbsOffset >> BitPos) & 0xfff)
+ .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, BitPos)));
+ }
+ } else {
+ constexpr uint64_t Mask16 = 0xffff;
+ const uint64_t UOffset = Offset;
+ EmitAndIncrement(MCInstBuilder(IsNeg ? AArch64::MOVNXi : AArch64::MOVZXi)
+ .addReg(AArch64::X17)
+ .addImm((IsNeg ? ~UOffset : UOffset) & Mask16)
+ .addImm(/*shift=*/0));
+ auto NeedMovk = [Mask16, IsNeg, UOffset](int BitPos) -> bool {
+ assert(BitPos == 16 || BitPos == 32 || BitPos == 48);
+ uint64_t Shifted = UOffset >> BitPos;
+ if (!IsNeg)
+ return Shifted != 0;
+ for (int I = 0; I != 64 - BitPos; I += 16)
+ if (((Shifted >> I) & Mask16) != Mask16)
+ return true;
+ return false;
+ };
+ for (int BitPos = 16; BitPos != 64 && NeedMovk(BitPos); BitPos += 16) {
+ EmitAndIncrement(MCInstBuilder(AArch64::MOVKXi)
+ .addReg(AArch64::X17)
+ .addReg(AArch64::X17)
+ .addImm((UOffset >> BitPos) & Mask16)
+ .addImm(/*shift=*/BitPos));
+ }
+ EmitAndIncrement(MCInstBuilder(AArch64::ADDXrs)
+ .addReg(AArch64::X16)
+ .addReg(AArch64::X16)
+ .addReg(AArch64::X17)
+ .addImm(/*shift=*/0));
+ }
+ }
+
+ unsigned DiscReg = AddrDisc;
+ if (Disc != 0) {
+ if (AddrDisc != AArch64::XZR) {
+ EmitAndIncrement(MCInstBuilder(AArch64::ORRXrs)
+ .addReg(AArch64::X17)
+ .addReg(AArch64::XZR)
+ .addReg(AddrDisc)
+ .addImm(0));
+ EmitAndIncrement(MCInstBuilder(AArch64::MOVKXi)
+ .addReg(AArch64::X17)
+ .addReg(AArch64::X17)
+ .addImm(Disc)
+ .addImm(/*shift=*/48));
+ } else {
+ EmitAndIncrement(MCInstBuilder(AArch64::MOVZXi)
+ .addReg(AArch64::X17)
+ .addImm(Disc)
+ .addImm(/*shift=*/0));
+ }
+ DiscReg = AArch64::X17;
+ }
+
+ auto MIB = MCInstBuilder(getPACOpcodeForKey(Key, DiscReg == AArch64::XZR))
+ .addReg(AArch64::X16)
+ .addReg(AArch64::X16);
+ if (DiscReg != AArch64::XZR)
+ MIB.addReg(DiscReg);
+ EmitAndIncrement(MIB);
+
+ assert(STI->getInstrInfo()->getInstSizeInBytes(MI) >= InstsEmitted * 4);
+}
+
// Simple pseudo-instructions have their lowering (with expansion to real
// in...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/94241
More information about the llvm-commits
mailing list