[llvm] 1488fb4 - [PAC][AArch64] Lower ptrauth constants in code (#96879)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Jun 27 21:29:43 PDT 2024
Author: Daniil Kovalev
Date: 2024-06-28T07:29:38+03:00
New Revision: 1488fb4153367b9c20e8c0ca7aa40bc8437ea3d3
URL: https://github.com/llvm/llvm-project/commit/1488fb4153367b9c20e8c0ca7aa40bc8437ea3d3
DIFF: https://github.com/llvm/llvm-project/commit/1488fb4153367b9c20e8c0ca7aa40bc8437ea3d3.diff
LOG: [PAC][AArch64] Lower ptrauth constants in code (#96879)
This re-applies #94241 after fixing buildbot failure, see
https://lab.llvm.org/buildbot/#/builders/51/builds/570
According to standard, `constexpr` variables and `const` variables
initialized with constant expressions can be used in lambdas w/o
capturing - see https://en.cppreference.com/w/cpp/language/lambda.
However, MSVC used on buildkite seems to ignore that rule and does not
allow using such uncaptured variables in lambdas: we have "error C3493:
'Mask16' cannot be implicitly captured because no default capture mode
has been specified" - see
https://buildkite.com/llvm-project/github-pull-requests/builds/73238
Explicitly capturing such a variable, however, makes buildbot fail with
"error: lambda capture 'Mask16' is not required to be captured for this
use [-Werror,-Wunused-lambda-capture]" - see
https://lab.llvm.org/buildbot/#/builders/51/builds/570.
Fix both cases by using `0xffff` value directly instead of giving a name
to it.
Original PR description below.
Depends on #94240.
Define the following pseudos for lowering ptrauth constants in code:
- non-`extern_weak`:
- no GOT load needed: `MOVaddrPAC` - similar to `MOVaddr`, with added
PAC;
- GOT load needed: `LOADgotPAC` - similar to `LOADgot`, with added PAC;
- `extern_weak`: `LOADauthptrstatic` - similar to `LOADgot`, but use a
special stub slot named `sym$auth_ptr$key$disc` filled by dynamic linker
during relocation resolving instead of a GOT slot.
---------
Co-authored-by: Ahmed Bougacha <ahmed at bougacha.org>
Added:
llvm/test/CodeGen/AArch64/GlobalISel/ptrauth-constant-in-code.ll
llvm/test/CodeGen/AArch64/ptrauth-constant-in-code.ll
Modified:
llvm/docs/GlobalISel/GenericOpcode.rst
llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
llvm/include/llvm/CodeGen/ISDOpcodes.h
llvm/include/llvm/CodeGen/MachineModuleInfoImpls.h
llvm/include/llvm/Support/TargetOpcodes.def
llvm/include/llvm/Target/GenericOpcodes.td
llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
llvm/lib/CodeGen/MachineModuleInfoImpls.cpp
llvm/lib/CodeGen/MachineVerifier.cpp
llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
llvm/lib/Target/AArch64/AArch64ISelLowering.h
llvm/lib/Target/AArch64/AArch64InstrInfo.td
llvm/lib/Target/AArch64/AArch64TargetObjectFile.cpp
llvm/lib/Target/AArch64/AArch64TargetObjectFile.h
llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-cxx.td
llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-variadics.td
llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td
llvm/test/TableGen/GlobalISelEmitter.td
Removed:
################################################################################
diff --git a/llvm/docs/GlobalISel/GenericOpcode.rst b/llvm/docs/GlobalISel/GenericOpcode.rst
index 42f56348885b4..b05394aeee003 100644
--- a/llvm/docs/GlobalISel/GenericOpcode.rst
+++ b/llvm/docs/GlobalISel/GenericOpcode.rst
@@ -60,6 +60,17 @@ The address of a global value.
%0(p0) = G_GLOBAL_VALUE @var_local
+G_PTRAUTH_GLOBAL_VALUE
+^^^^^^^^^^^^^^^^^^^^^^
+
+The signed address of a global value. Operands: address to be signed (pointer),
+key (32-bit imm), address for address discrimination (zero if not needed) and
+an extra discriminator (64-bit imm).
+
+.. code-block:: none
+
+ %0:_(p0) = G_PTRAUTH_GLOBAL_VALUE %1:_(p0), s32, %2:_(p0), s64
+
G_BLOCK_ADDR
^^^^^^^^^^^^
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
index a95ed3109feb8..954511eb731fa 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
@@ -886,6 +886,13 @@ class MachineIRBuilder {
MachineInstrBuilder buildFConstant(const DstOp &Res, double Val);
MachineInstrBuilder buildFConstant(const DstOp &Res, const APFloat &Val);
+ /// Build and insert G_PTRAUTH_GLOBAL_VALUE
+ ///
+ /// \return a MachineInstrBuilder for the newly created instruction.
+ MachineInstrBuilder buildConstantPtrAuth(const DstOp &Res,
+ const ConstantPtrAuth *CPA,
+ Register Addr, Register AddrDisc);
+
/// Build and insert \p Res = COPY Op
///
/// Register-to-register COPY sets \p Res to \p Op.
diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h
index 6bb89fb58a296..88e3339e2453f 100644
--- a/llvm/include/llvm/CodeGen/ISDOpcodes.h
+++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h
@@ -83,6 +83,12 @@ enum NodeType {
ExternalSymbol,
BlockAddress,
+ /// A ptrauth constant.
+ /// ptr, key, addr-disc, disc
+ /// Note that the addr-disc can be a non-constant value, to allow representing
+ /// a constant global address signed using address-diversification, in code.
+ PtrAuthGlobalAddress,
+
/// The address of the GOT
GLOBAL_OFFSET_TABLE,
diff --git a/llvm/include/llvm/CodeGen/MachineModuleInfoImpls.h b/llvm/include/llvm/CodeGen/MachineModuleInfoImpls.h
index f8a328f13eded..64d841d86c7c4 100644
--- a/llvm/include/llvm/CodeGen/MachineModuleInfoImpls.h
+++ b/llvm/include/llvm/CodeGen/MachineModuleInfoImpls.h
@@ -61,10 +61,20 @@ class MachineModuleInfoMachO : public MachineModuleInfoImpl {
/// MachineModuleInfoELF - This is a MachineModuleInfoImpl implementation
/// for ELF targets.
class MachineModuleInfoELF : public MachineModuleInfoImpl {
+public:
+ struct AuthStubInfo {
+ const MCExpr *AuthPtrRef;
+ };
+
+private:
/// GVStubs - These stubs are used to materialize global addresses in PIC
/// mode.
DenseMap<MCSymbol *, StubValueTy> GVStubs;
+ /// AuthPtrStubs - These stubs are used to materialize signed addresses for
+ /// extern_weak symbols.
+ DenseMap<MCSymbol *, AuthStubInfo> AuthPtrStubs;
+
virtual void anchor(); // Out of line virtual method.
public:
@@ -75,9 +85,19 @@ class MachineModuleInfoELF : public MachineModuleInfoImpl {
return GVStubs[Sym];
}
+ AuthStubInfo &getAuthPtrStubEntry(MCSymbol *Sym) {
+ assert(Sym && "Key cannot be null");
+ return AuthPtrStubs[Sym];
+ }
+
/// Accessor methods to return the set of stubs in sorted order.
SymbolListTy GetGVStubList() { return getSortedStubs(GVStubs); }
+
+ using AuthStubPairTy = std::pair<MCSymbol *, AuthStubInfo>;
+ typedef std::vector<AuthStubPairTy> AuthStubListTy;
+
+ AuthStubListTy getAuthGVStubList();
};
/// MachineModuleInfoCOFF - This is a MachineModuleInfoImpl implementation
diff --git a/llvm/include/llvm/Support/TargetOpcodes.def b/llvm/include/llvm/Support/TargetOpcodes.def
index df4b264af72a8..e7f40e87ed24a 100644
--- a/llvm/include/llvm/Support/TargetOpcodes.def
+++ b/llvm/include/llvm/Support/TargetOpcodes.def
@@ -294,6 +294,9 @@ HANDLE_TARGET_OPCODE(G_FRAME_INDEX)
/// Generic reference to global value.
HANDLE_TARGET_OPCODE(G_GLOBAL_VALUE)
+/// Generic ptrauth-signed reference to global value.
+HANDLE_TARGET_OPCODE(G_PTRAUTH_GLOBAL_VALUE)
+
/// Generic instruction to materialize the address of an object in the constant
/// pool.
HANDLE_TARGET_OPCODE(G_CONSTANT_POOL)
diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td
index 4abffe6476c85..e1710ff2d8abf 100644
--- a/llvm/include/llvm/Target/GenericOpcodes.td
+++ b/llvm/include/llvm/Target/GenericOpcodes.td
@@ -110,6 +110,12 @@ def G_GLOBAL_VALUE : GenericInstruction {
let hasSideEffects = false;
}
+def G_PTRAUTH_GLOBAL_VALUE : GenericInstruction {
+ let OutOperandList = (outs type0:$dst);
+ let InOperandList = (ins unknown:$addr, i32imm:$key, type1:$addrdisc, i64imm:$disc);
+ let hasSideEffects = 0;
+}
+
def G_CONSTANT_POOL : GenericInstruction {
let OutOperandList = (outs type0:$dst);
let InOperandList = (ins unknown:$src);
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index ddf6d3c20f022..c9986dfede1a1 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -3494,7 +3494,11 @@ bool IRTranslator::translate(const Constant &C, Register Reg) {
EntryBuilder->buildConstant(Reg, 0);
else if (auto GV = dyn_cast<GlobalValue>(&C))
EntryBuilder->buildGlobalValue(Reg, GV);
- else if (auto CAZ = dyn_cast<ConstantAggregateZero>(&C)) {
+ else if (auto CPA = dyn_cast<ConstantPtrAuth>(&C)) {
+ Register Addr = getOrCreateVReg(*CPA->getPointer());
+ Register AddrDisc = getOrCreateVReg(*CPA->getAddrDiscriminator());
+ EntryBuilder->buildConstantPtrAuth(Reg, CPA, Addr, AddrDisc);
+ } else if (auto CAZ = dyn_cast<ConstantAggregateZero>(&C)) {
if (!isa<FixedVectorType>(CAZ->getType()))
return false;
// Return the scalar if it is a <1 x Ty> vector.
diff --git a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
index 37aa4e0aed7e8..06a6c1f93ef1f 100644
--- a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
@@ -397,6 +397,19 @@ MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res,
return buildFConstant(Res, *CFP);
}
+MachineInstrBuilder
+MachineIRBuilder::buildConstantPtrAuth(const DstOp &Res,
+ const ConstantPtrAuth *CPA,
+ Register Addr, Register AddrDisc) {
+ auto MIB = buildInstr(TargetOpcode::G_PTRAUTH_GLOBAL_VALUE);
+ Res.addDefToMIB(*getMRI(), MIB);
+ MIB.addUse(Addr);
+ MIB.addImm(CPA->getKey()->getZExtValue());
+ MIB.addUse(AddrDisc);
+ MIB.addImm(CPA->getDiscriminator()->getZExtValue());
+ return MIB;
+}
+
MachineInstrBuilder MachineIRBuilder::buildBrCond(const SrcOp &Tst,
MachineBasicBlock &Dest) {
assert(Tst.getLLTTy(*getMRI()).isScalar() && "invalid operand type");
diff --git a/llvm/lib/CodeGen/MachineModuleInfoImpls.cpp b/llvm/lib/CodeGen/MachineModuleInfoImpls.cpp
index 9c3b31935f6d6..f114f1ecc0bae 100644
--- a/llvm/lib/CodeGen/MachineModuleInfoImpls.cpp
+++ b/llvm/lib/CodeGen/MachineModuleInfoImpls.cpp
@@ -13,6 +13,7 @@
#include "llvm/CodeGen/MachineModuleInfoImpls.h"
#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/STLExtras.h"
#include "llvm/MC/MCSymbol.h"
using namespace llvm;
@@ -41,3 +42,25 @@ MachineModuleInfoImpl::SymbolListTy MachineModuleInfoImpl::getSortedStubs(
Map.clear();
return List;
}
+
+template <typename MachineModuleInfoTarget>
+static typename MachineModuleInfoTarget::AuthStubListTy getAuthGVStubListHelper(
+ DenseMap<MCSymbol *, typename MachineModuleInfoTarget::AuthStubInfo>
+ &AuthPtrStubs) {
+ typename MachineModuleInfoTarget::AuthStubListTy List(AuthPtrStubs.begin(),
+ AuthPtrStubs.end());
+
+ if (!List.empty())
+ llvm::sort(List.begin(), List.end(),
+ [](const typename MachineModuleInfoTarget::AuthStubPairTy &LHS,
+ const typename MachineModuleInfoTarget::AuthStubPairTy &RHS) {
+ return LHS.first->getName() < RHS.first->getName();
+ });
+
+ AuthPtrStubs.clear();
+ return List;
+}
+
+MachineModuleInfoELF::AuthStubListTy MachineModuleInfoELF::getAuthGVStubList() {
+ return getAuthGVStubListHelper<MachineModuleInfoELF>(AuthPtrStubs);
+}
diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp
index 9ea238c61ed91..0c8a0f2b24a1e 100644
--- a/llvm/lib/CodeGen/MachineVerifier.cpp
+++ b/llvm/lib/CodeGen/MachineVerifier.cpp
@@ -2066,6 +2066,12 @@ void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) {
report("Dst operand 0 must be a pointer", MI);
break;
}
+ case TargetOpcode::G_PTRAUTH_GLOBAL_VALUE: {
+ const MachineOperand &AddrOp = MI->getOperand(1);
+ if (!AddrOp.isReg() || !MRI->getType(AddrOp.getReg()).isPointer())
+ report("addr operand must be a pointer", &AddrOp, 1);
+ break;
+ }
default:
break;
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index e45569bfa472e..465919d03d8ca 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -1802,6 +1802,13 @@ SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
+ if (const ConstantPtrAuth *CPA = dyn_cast<ConstantPtrAuth>(C)) {
+ return DAG.getNode(ISD::PtrAuthGlobalAddress, getCurSDLoc(), VT,
+ getValue(CPA->getPointer()), getValue(CPA->getKey()),
+ getValue(CPA->getAddrDiscriminator()),
+ getValue(CPA->getDiscriminator()));
+ }
+
if (isa<ConstantPointerNull>(C)) {
unsigned AS = V->getType()->getPointerAddressSpace();
return DAG.getConstant(0, getCurSDLoc(),
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
index a7555d6d31f26..c1d2c095b103c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
@@ -75,6 +75,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
}
return "<<Unknown Node #" + utostr(getOpcode()) + ">>";
+ // clang-format off
#ifndef NDEBUG
case ISD::DELETED_NODE: return "<<Deleted Node!>>";
#endif
@@ -126,6 +127,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
case ISD::ConstantFP: return "ConstantFP";
case ISD::GlobalAddress: return "GlobalAddress";
case ISD::GlobalTLSAddress: return "GlobalTLSAddress";
+ case ISD::PtrAuthGlobalAddress: return "PtrAuthGlobalAddress";
case ISD::FrameIndex: return "FrameIndex";
case ISD::JumpTable: return "JumpTable";
case ISD::JUMP_TABLE_DEBUG_INFO:
@@ -168,8 +170,6 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
return "OpaqueTargetConstant";
return "TargetConstant";
- // clang-format off
-
case ISD::TargetConstantFP: return "TargetConstantFP";
case ISD::TargetGlobalAddress: return "TargetGlobalAddress";
case ISD::TargetGlobalTLSAddress: return "TargetGlobalTLSAddress";
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index da11539eab348..00b8d7bd257e4 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -133,6 +133,13 @@ class AArch64AsmPrinter : public AsmPrinter {
unsigned emitPtrauthDiscriminator(uint16_t Disc, unsigned AddrDisc,
unsigned &InstsEmitted);
+ // Emit the sequence for LOADauthptrstatic
+ void LowerLOADauthptrstatic(const MachineInstr &MI);
+
+ // Emit the sequence for LOADgotPAC/MOVaddrPAC (either GOT adrp-ldr or
+ // adrp-add followed by PAC sign)
+ void LowerMOVaddrPAC(const MachineInstr &MI);
+
/// tblgen'erated driver function for lowering simple MI->MC
/// pseudo instructions.
bool emitPseudoExpansionLowering(MCStreamer &OutStreamer,
@@ -840,6 +847,15 @@ void AArch64AsmPrinter::emitHwasanMemaccessSymbols(Module &M) {
}
}
+template <typename MachineModuleInfoTarget>
+static void emitAuthenticatedPointer(
+ MCStreamer &OutStreamer, MCSymbol *StubLabel,
+ const typename MachineModuleInfoTarget::AuthStubInfo &StubInfo) {
+ // sym$auth_ptr$key$disc:
+ OutStreamer.emitLabel(StubLabel);
+ OutStreamer.emitValue(StubInfo.AuthPtrRef, /*size=*/8);
+}
+
void AArch64AsmPrinter::emitEndOfAsmFile(Module &M) {
emitHwasanMemaccessSymbols(M);
@@ -853,6 +869,25 @@ void AArch64AsmPrinter::emitEndOfAsmFile(Module &M) {
OutStreamer->emitAssemblerFlag(MCAF_SubsectionsViaSymbols);
}
+ if (TT.isOSBinFormatELF()) {
+ // Output authenticated pointers as indirect symbols, if we have any.
+ MachineModuleInfoELF &MMIELF = MMI->getObjFileInfo<MachineModuleInfoELF>();
+
+ auto Stubs = MMIELF.getAuthGVStubList();
+
+ if (!Stubs.empty()) {
+ const TargetLoweringObjectFile &TLOF = getObjFileLowering();
+ OutStreamer->switchSection(TLOF.getDataSection());
+ emitAlignment(Align(8));
+
+ for (const auto &Stub : Stubs)
+ emitAuthenticatedPointer<MachineModuleInfoELF>(*OutStreamer, Stub.first,
+ Stub.second);
+
+ OutStreamer->addBlankLine();
+ }
+ }
+
// Emit stack and fault map information.
FM.serializeToFaultMapSection();
@@ -1623,6 +1658,213 @@ AArch64AsmPrinter::lowerConstantPtrAuth(const ConstantPtrAuth &CPA) {
CPA.hasAddressDiscriminator(), Ctx);
}
+void AArch64AsmPrinter::LowerLOADauthptrstatic(const MachineInstr &MI) {
+ unsigned DstReg = MI.getOperand(0).getReg();
+ const MachineOperand &GAOp = MI.getOperand(1);
+ const uint64_t KeyC = MI.getOperand(2).getImm();
+ assert(KeyC <= AArch64PACKey::LAST &&
+ "key is out of range [0, AArch64PACKey::LAST]");
+ const auto Key = (AArch64PACKey::ID)KeyC;
+ const uint64_t Disc = MI.getOperand(3).getImm();
+ assert(isUInt<16>(Disc) &&
+ "constant discriminator is out of range [0, 0xffff]");
+
+ // Emit instruction sequence like the following:
+ // ADRP x16, symbol$auth_ptr$key$disc
+ // LDR x16, [x16, :lo12:symbol$auth_ptr$key$disc]
+ //
+ // Where the $auth_ptr$ symbol is the stub slot containing the signed pointer
+ // to symbol.
+ assert(TM.getTargetTriple().isOSBinFormatELF() &&
+ "LOADauthptrstatic is implemented only for ELF");
+ const auto &TLOF =
+ static_cast<const AArch64_ELFTargetObjectFile &>(getObjFileLowering());
+
+ assert(GAOp.getOffset() == 0 &&
+ "non-zero offset for $auth_ptr$ stub slots is not supported");
+ const MCSymbol *GASym = TM.getSymbol(GAOp.getGlobal());
+ MCSymbol *AuthPtrStubSym =
+ TLOF.getAuthPtrSlotSymbol(TM, &MF->getMMI(), GASym, Key, Disc);
+
+ MachineOperand StubMOHi =
+ MachineOperand::CreateMCSymbol(AuthPtrStubSym, AArch64II::MO_PAGE);
+ MachineOperand StubMOLo = MachineOperand::CreateMCSymbol(
+ AuthPtrStubSym, AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
+ MCOperand StubMCHi, StubMCLo;
+
+ MCInstLowering.lowerOperand(StubMOHi, StubMCHi);
+ MCInstLowering.lowerOperand(StubMOLo, StubMCLo);
+
+ EmitToStreamer(
+ *OutStreamer,
+ MCInstBuilder(AArch64::ADRP).addReg(DstReg).addOperand(StubMCHi));
+
+ EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::LDRXui)
+ .addReg(DstReg)
+ .addReg(DstReg)
+ .addOperand(StubMCLo));
+}
+
+void AArch64AsmPrinter::LowerMOVaddrPAC(const MachineInstr &MI) {
+ unsigned InstsEmitted = 0;
+ auto EmitAndIncrement = [this, &InstsEmitted](const MCInst &Inst) {
+ EmitToStreamer(*OutStreamer, Inst);
+ ++InstsEmitted;
+ };
+
+ const bool IsGOTLoad = MI.getOpcode() == AArch64::LOADgotPAC;
+ MachineOperand GAOp = MI.getOperand(0);
+ const uint64_t KeyC = MI.getOperand(1).getImm();
+ assert(KeyC <= AArch64PACKey::LAST &&
+ "key is out of range [0, AArch64PACKey::LAST]");
+ const auto Key = (AArch64PACKey::ID)KeyC;
+ const unsigned AddrDisc = MI.getOperand(2).getReg();
+ const uint64_t Disc = MI.getOperand(3).getImm();
+ assert(isUInt<16>(Disc) &&
+ "constant discriminator is out of range [0, 0xffff]");
+
+ const int64_t Offset = GAOp.getOffset();
+ GAOp.setOffset(0);
+
+ // Emit:
+ // target materialization:
+ // - via GOT:
+ // adrp x16, :got:target
+ // ldr x16, [x16, :got_lo12:target]
+ // add offset to x16 if offset != 0
+ //
+ // - direct:
+ // adrp x16, target
+ // add x16, x16, :lo12:target
+ // add offset to x16 if offset != 0
+ //
+ // add offset to x16:
+ // - abs(offset) fits 24 bits:
+ // add/sub x16, x16, #<offset>[, #lsl 12] (up to 2 instructions)
+ // - abs(offset) does not fit 24 bits:
+ // - offset < 0:
+ // movn+movk sequence filling x17 register with the offset (up to 4
+ // instructions)
+ // add x16, x16, x17
+ // - offset > 0:
+ // movz+movk sequence filling x17 register with the offset (up to 4
+ // instructions)
+ // add x16, x16, x17
+ //
+ // signing:
+ // - 0 discriminator:
+ // paciza x16
+ // - Non-0 discriminator, no address discriminator:
+ // mov x17, #Disc
+ // pacia x16, x17
+ // - address discriminator (with potentially folded immediate discriminator):
+ // pacia x16, xAddrDisc
+
+ MachineOperand GAMOHi(GAOp), GAMOLo(GAOp);
+ MCOperand GAMCHi, GAMCLo;
+
+ GAMOHi.setTargetFlags(AArch64II::MO_PAGE);
+ GAMOLo.setTargetFlags(AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
+ if (IsGOTLoad) {
+ GAMOHi.addTargetFlag(AArch64II::MO_GOT);
+ GAMOLo.addTargetFlag(AArch64II::MO_GOT);
+ }
+
+ MCInstLowering.lowerOperand(GAMOHi, GAMCHi);
+ MCInstLowering.lowerOperand(GAMOLo, GAMCLo);
+
+ EmitAndIncrement(
+ MCInstBuilder(AArch64::ADRP).addReg(AArch64::X16).addOperand(GAMCHi));
+
+ if (IsGOTLoad) {
+ EmitAndIncrement(MCInstBuilder(AArch64::LDRXui)
+ .addReg(AArch64::X16)
+ .addReg(AArch64::X16)
+ .addOperand(GAMCLo));
+ } else {
+ EmitAndIncrement(MCInstBuilder(AArch64::ADDXri)
+ .addReg(AArch64::X16)
+ .addReg(AArch64::X16)
+ .addOperand(GAMCLo)
+ .addImm(0));
+ }
+
+ if (Offset != 0) {
+ const uint64_t AbsOffset = (Offset > 0 ? Offset : -((uint64_t)Offset));
+ const bool IsNeg = Offset < 0;
+ if (isUInt<24>(AbsOffset)) {
+ for (int BitPos = 0; BitPos != 24 && (AbsOffset >> BitPos);
+ BitPos += 12) {
+ EmitAndIncrement(
+ MCInstBuilder(IsNeg ? AArch64::SUBXri : AArch64::ADDXri)
+ .addReg(AArch64::X16)
+ .addReg(AArch64::X16)
+ .addImm((AbsOffset >> BitPos) & 0xfff)
+ .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, BitPos)));
+ }
+ } else {
+ const uint64_t UOffset = Offset;
+ EmitAndIncrement(MCInstBuilder(IsNeg ? AArch64::MOVNXi : AArch64::MOVZXi)
+ .addReg(AArch64::X17)
+ .addImm((IsNeg ? ~UOffset : UOffset) & 0xffff)
+ .addImm(/*shift=*/0));
+ auto NeedMovk = [IsNeg, UOffset](int BitPos) -> bool {
+ assert(BitPos == 16 || BitPos == 32 || BitPos == 48);
+ uint64_t Shifted = UOffset >> BitPos;
+ if (!IsNeg)
+ return Shifted != 0;
+ for (int I = 0; I != 64 - BitPos; I += 16)
+ if (((Shifted >> I) & 0xffff) != 0xffff)
+ return true;
+ return false;
+ };
+ for (int BitPos = 16; BitPos != 64 && NeedMovk(BitPos); BitPos += 16) {
+ EmitAndIncrement(MCInstBuilder(AArch64::MOVKXi)
+ .addReg(AArch64::X17)
+ .addReg(AArch64::X17)
+ .addImm((UOffset >> BitPos) & 0xffff)
+ .addImm(/*shift=*/BitPos));
+ }
+ EmitAndIncrement(MCInstBuilder(AArch64::ADDXrs)
+ .addReg(AArch64::X16)
+ .addReg(AArch64::X16)
+ .addReg(AArch64::X17)
+ .addImm(/*shift=*/0));
+ }
+ }
+
+ unsigned DiscReg = AddrDisc;
+ if (Disc != 0) {
+ if (AddrDisc != AArch64::XZR) {
+ EmitAndIncrement(MCInstBuilder(AArch64::ORRXrs)
+ .addReg(AArch64::X17)
+ .addReg(AArch64::XZR)
+ .addReg(AddrDisc)
+ .addImm(0));
+ EmitAndIncrement(MCInstBuilder(AArch64::MOVKXi)
+ .addReg(AArch64::X17)
+ .addReg(AArch64::X17)
+ .addImm(Disc)
+ .addImm(/*shift=*/48));
+ } else {
+ EmitAndIncrement(MCInstBuilder(AArch64::MOVZXi)
+ .addReg(AArch64::X17)
+ .addImm(Disc)
+ .addImm(/*shift=*/0));
+ }
+ DiscReg = AArch64::X17;
+ }
+
+ auto MIB = MCInstBuilder(getPACOpcodeForKey(Key, DiscReg == AArch64::XZR))
+ .addReg(AArch64::X16)
+ .addReg(AArch64::X16);
+ if (DiscReg != AArch64::XZR)
+ MIB.addReg(DiscReg);
+ EmitAndIncrement(MIB);
+
+ assert(STI->getInstrInfo()->getInstSizeInBytes(MI) >= InstsEmitted * 4);
+}
+
// Simple pseudo-instructions have their lowering (with expansion to real
// instructions) auto-generated.
#include "AArch64GenMCPseudoLowering.inc"
@@ -1758,6 +2000,15 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
return;
}
+ case AArch64::LOADauthptrstatic:
+ LowerLOADauthptrstatic(*MI);
+ return;
+
+ case AArch64::LOADgotPAC:
+ case AArch64::MOVaddrPAC:
+ LowerMOVaddrPAC(*MI);
+ return;
+
case AArch64::BLRA:
emitPtrauthBranch(MI);
return;
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 61de30cdcaef6..fba6b6295db0b 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -511,6 +511,8 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::JumpTable, MVT::i64, Custom);
setOperationAction(ISD::SETCCCARRY, MVT::i64, Custom);
+ setOperationAction(ISD::PtrAuthGlobalAddress, MVT::i64, Custom);
+
setOperationAction(ISD::SHL_PARTS, MVT::i64, Custom);
setOperationAction(ISD::SRA_PARTS, MVT::i64, Custom);
setOperationAction(ISD::SRL_PARTS, MVT::i64, Custom);
@@ -6667,6 +6669,8 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
return LowerGlobalAddress(Op, DAG);
case ISD::GlobalTLSAddress:
return LowerGlobalTLSAddress(Op, DAG);
+ case ISD::PtrAuthGlobalAddress:
+ return LowerPtrAuthGlobalAddress(Op, DAG);
case ISD::SETCC:
case ISD::STRICT_FSETCC:
case ISD::STRICT_FSETCCS:
@@ -9517,6 +9521,133 @@ SDValue AArch64TargetLowering::LowerGlobalTLSAddress(SDValue Op,
llvm_unreachable("Unexpected platform trying to use TLS");
}
+//===----------------------------------------------------------------------===//
+// PtrAuthGlobalAddress lowering
+//
+// We have 3 lowering alternatives to choose from:
+// - MOVaddrPAC: similar to MOVaddr, with added PAC.
+// If the GV doesn't need a GOT load (i.e., is locally defined)
+// materialize the pointer using adrp+add+pac. See LowerMOVaddrPAC.
+//
+// - LOADgotPAC: similar to LOADgot, with added PAC.
+// If the GV needs a GOT load, materialize the pointer using the usual
+// GOT adrp+ldr, +pac. Pointers in GOT are assumed to be not signed, the GOT
+// section is assumed to be read-only (for example, via relro mechanism). See
+// LowerMOVaddrPAC.
+//
+// - LOADauthptrstatic: similar to LOADgot, but use a
+// special stub slot instead of a GOT slot.
+// Load a signed pointer for symbol 'sym' from a stub slot named
+// 'sym$auth_ptr$key$disc' filled by dynamic linker during relocation
+// resolving. This usually lowers to adrp+ldr, but also emits an entry into
+// .data with an
+// @AUTH relocation. See LowerLOADauthptrstatic.
+//
+// All 3 are pseudos that are expand late to longer sequences: this lets us
+// provide integrity guarantees on the to-be-signed intermediate values.
+//
+// LOADauthptrstatic is undesirable because it requires a large section filled
+// with often similarly-signed pointers, making it a good harvesting target.
+// Thus, it's only used for ptrauth references to extern_weak to avoid null
+// checks.
+
+SDValue AArch64TargetLowering::LowerPtrAuthGlobalAddressStatically(
+ SDValue TGA, SDLoc DL, EVT VT, AArch64PACKey::ID KeyC,
+ SDValue Discriminator, SDValue AddrDiscriminator, SelectionDAG &DAG) const {
+ const auto *TGN = cast<GlobalAddressSDNode>(TGA.getNode());
+ assert(TGN->getGlobal()->hasExternalWeakLinkage());
+
+ // Offsets and extern_weak don't mix well: ptrauth aside, you'd get the
+ // offset alone as a pointer if the symbol wasn't available, which would
+ // probably break null checks in users. Ptrauth complicates things further:
+ // error out.
+ if (TGN->getOffset() != 0)
+ report_fatal_error(
+ "unsupported non-zero offset in weak ptrauth global reference");
+
+ if (!isNullConstant(AddrDiscriminator))
+ report_fatal_error("unsupported weak addr-div ptrauth global");
+
+ SDValue Key = DAG.getTargetConstant(KeyC, DL, MVT::i32);
+ return SDValue(DAG.getMachineNode(AArch64::LOADauthptrstatic, DL, MVT::i64,
+ {TGA, Key, Discriminator}),
+ 0);
+}
+
+SDValue
+AArch64TargetLowering::LowerPtrAuthGlobalAddress(SDValue Op,
+ SelectionDAG &DAG) const {
+ SDValue Ptr = Op.getOperand(0);
+ uint64_t KeyC = Op.getConstantOperandVal(1);
+ SDValue AddrDiscriminator = Op.getOperand(2);
+ uint64_t DiscriminatorC = Op.getConstantOperandVal(3);
+ EVT VT = Op.getValueType();
+ SDLoc DL(Op);
+
+ if (KeyC > AArch64PACKey::LAST)
+ report_fatal_error("key in ptrauth global out of range [0, " +
+ Twine((int)AArch64PACKey::LAST) + "]");
+
+ // Blend only works if the integer discriminator is 16-bit wide.
+ if (!isUInt<16>(DiscriminatorC))
+ report_fatal_error(
+ "constant discriminator in ptrauth global out of range [0, 0xffff]");
+
+ // Choosing between 3 lowering alternatives is target-specific.
+ if (!Subtarget->isTargetELF())
+ report_fatal_error("ptrauth global lowering is only implemented for ELF");
+
+ int64_t PtrOffsetC = 0;
+ if (Ptr.getOpcode() == ISD::ADD) {
+ PtrOffsetC = Ptr.getConstantOperandVal(1);
+ Ptr = Ptr.getOperand(0);
+ }
+ const auto *PtrN = cast<GlobalAddressSDNode>(Ptr.getNode());
+ const GlobalValue *PtrGV = PtrN->getGlobal();
+
+ // Classify the reference to determine whether it needs a GOT load.
+ const unsigned OpFlags =
+ Subtarget->ClassifyGlobalReference(PtrGV, getTargetMachine());
+ const bool NeedsGOTLoad = ((OpFlags & AArch64II::MO_GOT) != 0);
+ assert(((OpFlags & (~AArch64II::MO_GOT)) == 0) &&
+ "unsupported non-GOT op flags on ptrauth global reference");
+
+ // Fold any offset into the GV; our pseudos expect it there.
+ PtrOffsetC += PtrN->getOffset();
+ SDValue TPtr = DAG.getTargetGlobalAddress(PtrGV, DL, VT, PtrOffsetC,
+ /*TargetFlags=*/0);
+ assert(PtrN->getTargetFlags() == 0 &&
+ "unsupported target flags on ptrauth global");
+
+ SDValue Key = DAG.getTargetConstant(KeyC, DL, MVT::i32);
+ SDValue Discriminator = DAG.getTargetConstant(DiscriminatorC, DL, MVT::i64);
+ SDValue TAddrDiscriminator = !isNullConstant(AddrDiscriminator)
+ ? AddrDiscriminator
+ : DAG.getRegister(AArch64::XZR, MVT::i64);
+
+ // No GOT load needed -> MOVaddrPAC
+ if (!NeedsGOTLoad) {
+ assert(!PtrGV->hasExternalWeakLinkage() && "extern_weak should use GOT");
+ return SDValue(
+ DAG.getMachineNode(AArch64::MOVaddrPAC, DL, MVT::i64,
+ {TPtr, Key, TAddrDiscriminator, Discriminator}),
+ 0);
+ }
+
+ // GOT load -> LOADgotPAC
+ // Note that we disallow extern_weak refs to avoid null checks later.
+ if (!PtrGV->hasExternalWeakLinkage())
+ return SDValue(
+ DAG.getMachineNode(AArch64::LOADgotPAC, DL, MVT::i64,
+ {TPtr, Key, TAddrDiscriminator, Discriminator}),
+ 0);
+
+ // extern_weak ref -> LOADauthptrstatic
+ return LowerPtrAuthGlobalAddressStatically(
+ TPtr, DL, VT, (AArch64PACKey::ID)KeyC, Discriminator, AddrDiscriminator,
+ DAG);
+}
+
// Looks through \param Val to determine the bit that can be used to
// check the sign of the value. It returns the unextended value and
// the sign bit position.
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 5200b24d1388a..047c852bb01d2 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -1127,6 +1127,12 @@ class AArch64TargetLowering : public TargetLowering {
SDValue LowerELFTLSDescCallSeq(SDValue SymAddr, const SDLoc &DL,
SelectionDAG &DAG) const;
SDValue LowerWindowsGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
+ SDValue LowerPtrAuthGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
+ SDValue LowerPtrAuthGlobalAddressStatically(SDValue TGA, SDLoc DL, EVT VT,
+ AArch64PACKey::ID Key,
+ SDValue Discriminator,
+ SDValue AddrDiscriminator,
+ SelectionDAG &DAG) const;
SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerBR_CC(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index f3aac3b46d173..1e06d5fdc7562 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -1765,6 +1765,38 @@ let Predicates = [HasPAuth] in {
defm LDRAA : AuthLoad<0, "ldraa", simm10Scaled>;
defm LDRAB : AuthLoad<1, "ldrab", simm10Scaled>;
+ // Materialize a signed global address, with adrp+add and PAC.
+ def MOVaddrPAC : Pseudo<(outs),
+ (ins i64imm:$Addr, i32imm:$Key,
+ GPR64noip:$AddrDisc, i64imm:$Disc), []>,
+ Sched<[WriteI, ReadI]> {
+ let isReMaterializable = 1;
+ let isCodeGenOnly = 1;
+ let Size = 40; // 12 fixed + 28 variable, for pointer offset, and discriminator
+ let Defs = [X16,X17];
+ }
+
+ // Materialize a signed global address, using a GOT load and PAC.
+ def LOADgotPAC : Pseudo<(outs),
+ (ins i64imm:$Addr, i32imm:$Key,
+ GPR64noip:$AddrDisc, i64imm:$Disc), []>,
+ Sched<[WriteI, ReadI]> {
+ let isReMaterializable = 1;
+ let isCodeGenOnly = 1;
+ let Size = 40; // 12 fixed + 28 variable, for pointer offset, and discriminator
+ let Defs = [X16,X17];
+ }
+
+ // Load a signed global address from a special $auth_ptr$ stub slot.
+ def LOADauthptrstatic : Pseudo<(outs GPR64:$dst),
+ (ins i64imm:$Addr, i32imm:$Key,
+ i64imm:$Disc), []>,
+ Sched<[WriteI, ReadI]> {
+ let isReMaterializable = 1;
+ let isCodeGenOnly = 1;
+ let Size = 8;
+ }
+
// Size 16: 4 fixed + 8 variable, to compute discriminator.
let isCall = 1, isTerminator = 1, isReturn = 1, isBarrier = 1, Size = 16,
Uses = [SP] in {
diff --git a/llvm/lib/Target/AArch64/AArch64TargetObjectFile.cpp b/llvm/lib/Target/AArch64/AArch64TargetObjectFile.cpp
index e5c6b6d60d068..1c1e87d723ac7 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetObjectFile.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetObjectFile.cpp
@@ -8,7 +8,10 @@
#include "AArch64TargetObjectFile.h"
#include "AArch64TargetMachine.h"
+#include "MCTargetDesc/AArch64MCExpr.h"
+#include "llvm/ADT/StringExtras.h"
#include "llvm/BinaryFormat/Dwarf.h"
+#include "llvm/CodeGen/MachineModuleInfoImpls.h"
#include "llvm/IR/Mangler.h"
#include "llvm/MC/MCContext.h"
#include "llvm/MC/MCExpr.h"
@@ -88,3 +91,37 @@ void AArch64_MachoTargetObjectFile::getNameWithPrefix(
// be accessed via at least a linker-private symbol.
getMangler().getNameWithPrefix(OutName, GV, /* CannotUsePrivateLabel */ true);
}
+
+template <typename MachineModuleInfoTarget>
+static MCSymbol *getAuthPtrSlotSymbolHelper(
+ MCContext &Ctx, const TargetMachine &TM, MachineModuleInfo *MMI,
+ MachineModuleInfoTarget &TargetMMI, const MCSymbol *RawSym,
+ AArch64PACKey::ID Key, uint16_t Discriminator) {
+ const DataLayout &DL = MMI->getModule()->getDataLayout();
+
+ MCSymbol *StubSym = Ctx.getOrCreateSymbol(
+ DL.getLinkerPrivateGlobalPrefix() + RawSym->getName() +
+ Twine("$auth_ptr$") + AArch64PACKeyIDToString(Key) + Twine('$') +
+ Twine(Discriminator));
+
+ typename MachineModuleInfoTarget::AuthStubInfo &StubInfo =
+ TargetMMI.getAuthPtrStubEntry(StubSym);
+
+ if (StubInfo.AuthPtrRef)
+ return StubSym;
+
+ const MCExpr *Sym = MCSymbolRefExpr::create(RawSym, Ctx);
+
+ StubInfo.AuthPtrRef =
+ AArch64AuthMCExpr::create(Sym, Discriminator, Key,
+ /*HasAddressDiversity=*/false, Ctx);
+ return StubSym;
+}
+
+MCSymbol *AArch64_ELFTargetObjectFile::getAuthPtrSlotSymbol(
+ const TargetMachine &TM, MachineModuleInfo *MMI, const MCSymbol *RawSym,
+ AArch64PACKey::ID Key, uint16_t Discriminator) const {
+ auto &ELFMMI = MMI->getObjFileInfo<MachineModuleInfoELF>();
+ return getAuthPtrSlotSymbolHelper(getContext(), TM, MMI, ELFMMI, RawSym, Key,
+ Discriminator);
+}
diff --git a/llvm/lib/Target/AArch64/AArch64TargetObjectFile.h b/llvm/lib/Target/AArch64/AArch64TargetObjectFile.h
index 7cd56fc9a189d..c5ebf03c39c77 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetObjectFile.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetObjectFile.h
@@ -9,6 +9,7 @@
#ifndef LLVM_LIB_TARGET_AARCH64_AARCH64TARGETOBJECTFILE_H
#define LLVM_LIB_TARGET_AARCH64_AARCH64TARGETOBJECTFILE_H
+#include "Utils/AArch64BaseInfo.h"
#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
#include "llvm/Target/TargetLoweringObjectFile.h"
@@ -29,6 +30,11 @@ class AArch64_ELFTargetObjectFile : public TargetLoweringObjectFileELF {
const MCValue &MV, int64_t Offset,
MachineModuleInfo *MMI,
MCStreamer &Streamer) const override;
+
+ MCSymbol *getAuthPtrSlotSymbol(const TargetMachine &TM,
+ MachineModuleInfo *MMI, const MCSymbol *RawSym,
+ AArch64PACKey::ID Key,
+ uint16_t Discriminator) const;
};
/// AArch64_MachoTargetObjectFile - This TLOF implementation is used for Darwin.
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index 0357a7206c478..9e0860934f777 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -224,6 +224,8 @@ class AArch64InstructionSelector : public InstructionSelector {
bool selectJumpTable(MachineInstr &I, MachineRegisterInfo &MRI);
bool selectBrJT(MachineInstr &I, MachineRegisterInfo &MRI);
bool selectTLSGlobalValue(MachineInstr &I, MachineRegisterInfo &MRI);
+ bool selectPtrAuthGlobalValue(MachineInstr &I,
+ MachineRegisterInfo &MRI) const;
bool selectReduction(MachineInstr &I, MachineRegisterInfo &MRI);
bool selectMOPS(MachineInstr &I, MachineRegisterInfo &MRI);
bool selectUSMovFromExtend(MachineInstr &I, MachineRegisterInfo &MRI);
@@ -2848,6 +2850,9 @@ bool AArch64InstructionSelector::select(MachineInstr &I) {
return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
}
+ case TargetOpcode::G_PTRAUTH_GLOBAL_VALUE:
+ return selectPtrAuthGlobalValue(I, MRI);
+
case TargetOpcode::G_ZEXTLOAD:
case TargetOpcode::G_LOAD:
case TargetOpcode::G_STORE: {
@@ -6583,6 +6588,145 @@ bool AArch64InstructionSelector::selectIntrinsic(MachineInstr &I,
return false;
}
+// G_PTRAUTH_GLOBAL_VALUE lowering
+//
+// We have 3 lowering alternatives to choose from:
+// - MOVaddrPAC: similar to MOVaddr, with added PAC.
+// If the GV doesn't need a GOT load (i.e., is locally defined)
+// materialize the pointer using adrp+add+pac. See LowerMOVaddrPAC.
+//
+// - LOADgotPAC: similar to LOADgot, with added PAC.
+// If the GV needs a GOT load, materialize the pointer using the usual
+// GOT adrp+ldr, +pac. Pointers in GOT are assumed to be not signed, the GOT
+// section is assumed to be read-only (for example, via relro mechanism). See
+// LowerMOVaddrPAC.
+//
+// - LOADauthptrstatic: similar to LOADgot, but use a
+// special stub slot instead of a GOT slot.
+// Load a signed pointer for symbol 'sym' from a stub slot named
+// 'sym$auth_ptr$key$disc' filled by dynamic linker during relocation
+// resolving. This usually lowers to adrp+ldr, but also emits an entry into
+// .data with an
+// @AUTH relocation. See LowerLOADauthptrstatic.
+//
+// All 3 are pseudos that are expand late to longer sequences: this lets us
+// provide integrity guarantees on the to-be-signed intermediate values.
+//
+// LOADauthptrstatic is undesirable because it requires a large section filled
+// with often similarly-signed pointers, making it a good harvesting target.
+// Thus, it's only used for ptrauth references to extern_weak to avoid null
+// checks.
+
+bool AArch64InstructionSelector::selectPtrAuthGlobalValue(
+ MachineInstr &I, MachineRegisterInfo &MRI) const {
+ Register DefReg = I.getOperand(0).getReg();
+ Register Addr = I.getOperand(1).getReg();
+ uint64_t Key = I.getOperand(2).getImm();
+ Register AddrDisc = I.getOperand(3).getReg();
+ uint64_t Disc = I.getOperand(4).getImm();
+ int64_t Offset = 0;
+
+ if (Key > AArch64PACKey::LAST)
+ report_fatal_error("key in ptrauth global out of range [0, " +
+ Twine((int)AArch64PACKey::LAST) + "]");
+
+ // Blend only works if the integer discriminator is 16-bit wide.
+ if (!isUInt<16>(Disc))
+ report_fatal_error(
+ "constant discriminator in ptrauth global out of range [0, 0xffff]");
+
+ // Choosing between 3 lowering alternatives is target-specific.
+ if (!STI.isTargetELF())
+ report_fatal_error("ptrauth global lowering is only implemented for ELF");
+
+ if (!MRI.hasOneDef(Addr))
+ return false;
+
+ // First match any offset we take from the real global.
+ const MachineInstr *DefMI = &*MRI.def_instr_begin(Addr);
+ if (DefMI->getOpcode() == TargetOpcode::G_PTR_ADD) {
+ Register OffsetReg = DefMI->getOperand(2).getReg();
+ if (!MRI.hasOneDef(OffsetReg))
+ return false;
+ const MachineInstr &OffsetMI = *MRI.def_instr_begin(OffsetReg);
+ if (OffsetMI.getOpcode() != TargetOpcode::G_CONSTANT)
+ return false;
+
+ Addr = DefMI->getOperand(1).getReg();
+ if (!MRI.hasOneDef(Addr))
+ return false;
+
+ DefMI = &*MRI.def_instr_begin(Addr);
+ Offset = OffsetMI.getOperand(1).getCImm()->getSExtValue();
+ }
+
+ // We should be left with a genuine unauthenticated GlobalValue.
+ const GlobalValue *GV;
+ if (DefMI->getOpcode() == TargetOpcode::G_GLOBAL_VALUE) {
+ GV = DefMI->getOperand(1).getGlobal();
+ Offset += DefMI->getOperand(1).getOffset();
+ } else if (DefMI->getOpcode() == AArch64::G_ADD_LOW) {
+ GV = DefMI->getOperand(2).getGlobal();
+ Offset += DefMI->getOperand(2).getOffset();
+ } else {
+ return false;
+ }
+
+ MachineIRBuilder MIB(I);
+
+ // Classify the reference to determine whether it needs a GOT load.
+ unsigned OpFlags = STI.ClassifyGlobalReference(GV, TM);
+ const bool NeedsGOTLoad = ((OpFlags & AArch64II::MO_GOT) != 0);
+ assert(((OpFlags & (~AArch64II::MO_GOT)) == 0) &&
+ "unsupported non-GOT op flags on ptrauth global reference");
+ assert((!GV->hasExternalWeakLinkage() || NeedsGOTLoad) &&
+ "unsupported non-GOT reference to weak ptrauth global");
+
+ std::optional<APInt> AddrDiscVal = getIConstantVRegVal(AddrDisc, MRI);
+ bool HasAddrDisc = !AddrDiscVal || *AddrDiscVal != 0;
+
+ // Non-extern_weak:
+ // - No GOT load needed -> MOVaddrPAC
+ // - GOT load for non-extern_weak -> LOADgotPAC
+ // Note that we disallow extern_weak refs to avoid null checks later.
+ if (!GV->hasExternalWeakLinkage()) {
+ MIB.buildInstr(TargetOpcode::IMPLICIT_DEF, {AArch64::X16}, {});
+ MIB.buildInstr(TargetOpcode::IMPLICIT_DEF, {AArch64::X17}, {});
+ MIB.buildInstr(NeedsGOTLoad ? AArch64::LOADgotPAC : AArch64::MOVaddrPAC)
+ .addGlobalAddress(GV, Offset)
+ .addImm(Key)
+ .addReg(HasAddrDisc ? AddrDisc : AArch64::XZR)
+ .addImm(Disc)
+ .constrainAllUses(TII, TRI, RBI);
+ MIB.buildCopy(DefReg, Register(AArch64::X16));
+ RBI.constrainGenericRegister(DefReg, AArch64::GPR64RegClass, MRI);
+ I.eraseFromParent();
+ return true;
+ }
+
+ // extern_weak -> LOADauthptrstatic
+
+ // Offsets and extern_weak don't mix well: ptrauth aside, you'd get the
+ // offset alone as a pointer if the symbol wasn't available, which would
+ // probably break null checks in users. Ptrauth complicates things further:
+ // error out.
+ if (Offset != 0)
+ report_fatal_error(
+ "unsupported non-zero offset in weak ptrauth global reference");
+
+ if (HasAddrDisc)
+ report_fatal_error("unsupported weak addr-div ptrauth global");
+
+ MIB.buildInstr(AArch64::LOADauthptrstatic, {DefReg}, {})
+ .addGlobalAddress(GV, Offset)
+ .addImm(Key)
+ .addImm(Disc);
+ RBI.constrainGenericRegister(DefReg, AArch64::GPR64RegClass, MRI);
+
+ I.eraseFromParent();
+ return true;
+}
+
void AArch64InstructionSelector::SelectTable(MachineInstr &I,
MachineRegisterInfo &MRI,
unsigned NumVec, unsigned Opc1,
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index fef0b722efe45..50ba3057928ee 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -760,6 +760,9 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
else
getActionDefinitionsBuilder(G_GLOBAL_VALUE).legalFor({p0});
+ getActionDefinitionsBuilder(G_PTRAUTH_GLOBAL_VALUE)
+ .legalIf(all(typeIs(0, p0), typeIs(1, p0)));
+
getActionDefinitionsBuilder(G_PTRTOINT)
.legalFor({{s64, p0}, {v2s64, v2p0}})
.widenScalarToNextPow2(0, 64)
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
index f03491924f7f4..1f048528ea153 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
@@ -86,6 +86,10 @@
# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
#
+# DEBUG-NEXT: G_PTRAUTH_GLOBAL_VALUE (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
+#
# DEBUG-NEXT: G_CONSTANT_POOL (opcode {{[0-9]+}}): 1 type index, 0 imm indices
# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/ptrauth-constant-in-code.ll b/llvm/test/CodeGen/AArch64/GlobalISel/ptrauth-constant-in-code.ll
new file mode 100644
index 0000000000000..7b85b12bb8952
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/ptrauth-constant-in-code.ll
@@ -0,0 +1,235 @@
+; RUN: rm -rf %t && split-file %s %t && cd %t
+
+;--- err1.ll
+
+; RUN: not --crash llc < err1.ll -mtriple aarch64-elf -mattr=+pauth \
+; RUN: -global-isel=1 -verify-machineinstrs -global-isel-abort=1 2>&1 | \
+; RUN: FileCheck --check-prefix=ERR1 %s
+
+ at g = external global i32
+
+define ptr @foo() {
+; ERR1: LLVM ERROR: key in ptrauth global out of range [0, 3]
+ ret ptr ptrauth (ptr @g, i32 4)
+}
+
+;--- err2.ll
+
+; RUN: not --crash llc < err2.ll -mtriple aarch64-elf -mattr=+pauth \
+; RUN: -global-isel=1 -verify-machineinstrs -global-isel-abort=1 2>&1 | \
+; RUN: FileCheck --check-prefix=ERR2 %s
+
+ at g = external global i32
+
+define ptr @foo() {
+; ERR2: LLVM ERROR: constant discriminator in ptrauth global out of range [0, 0xffff]
+ ret ptr ptrauth (ptr @g, i32 2, i64 65536)
+}
+
+;--- err3.ll
+
+; RUN: not --crash llc < err3.ll -mtriple aarch64-elf -mattr=+pauth \
+; RUN: -global-isel=1 -verify-machineinstrs -global-isel-abort=1 2>&1 | \
+; RUN: FileCheck --check-prefix=ERR3 %s
+
+ at g_weak = extern_weak global i32
+
+define ptr @foo() {
+; ERR3: LLVM ERROR: unsupported non-zero offset in weak ptrauth global reference
+ ret ptr ptrauth (ptr getelementptr (i8, ptr @g_weak, i64 16), i32 2, i64 42)
+}
+
+;--- err4.ll
+
+; RUN: not --crash llc < err4.ll -mtriple aarch64-elf -mattr=+pauth \
+; RUN: -global-isel=1 -verify-machineinstrs -global-isel-abort=1 2>&1 | \
+; RUN: FileCheck --check-prefix=ERR4 %s
+
+ at g_weak = extern_weak global i32
+ at g_weak.ref.da.42.addr = dso_local constant ptr ptrauth (ptr @g_weak, i32 2, i64 42, ptr @g_weak.ref.da.42.addr)
+
+define ptr @foo() {
+; ERR4: LLVM ERROR: unsupported weak addr-div ptrauth global
+ ret ptr ptrauth (ptr @g_weak, i32 0, i64 42, ptr @g_weak.ref.da.42.addr)
+}
+
+;--- err5.ll
+
+; RUN: not --crash llc < err5.ll -mtriple arm64-apple-darwin -mattr=+pauth \
+; RUN: -global-isel=1 -verify-machineinstrs -global-isel-abort=1 2>&1 | \
+; RUN: FileCheck --check-prefix=ERR5 %s
+
+ at g = external global i32
+
+define ptr @foo() {
+; ERR5: LLVM ERROR: ptrauth global lowering is only implemented for ELF
+ ret ptr ptrauth (ptr @g, i32 0)
+}
+
+;--- ok.ll
+
+; RUN: llc < ok.ll -mtriple aarch64-elf -mattr=+pauth -global-isel=1 \
+; RUN: -verify-machineinstrs -global-isel-abort=1 | FileCheck %s
+; RUN: llc < ok.ll -mtriple aarch64-elf -mattr=+pauth -global-isel=1 \
+; RUN: -verify-machineinstrs -global-isel-abort=1 -filetype=obj
+
+ at g = external global i32
+ at g_weak = extern_weak global i32
+ at g_strong_def = dso_local constant i32 42
+
+define ptr @test_global_zero_disc() {
+; CHECK-LABEL: test_global_zero_disc:
+; CHECK: // %bb.0:
+; CHECK-NEXT: adrp x16, :got:g
+; CHECK-NEXT: ldr x16, [x16, :got_lo12:g]
+; CHECK-NEXT: paciza x16
+; CHECK-NEXT: mov x0, x16
+; CHECK-NEXT: ret
+
+ ret ptr ptrauth (ptr @g, i32 0)
+}
+
+define ptr @test_global_offset_zero_disc() {
+; CHECK-LABEL: test_global_offset_zero_disc:
+; CHECK: // %bb.0:
+; CHECK-NEXT: adrp x16, :got:g
+; CHECK-NEXT: ldr x16, [x16, :got_lo12:g]
+; CHECK-NEXT: add x16, x16, #16
+; CHECK-NEXT: pacdza x16
+; CHECK-NEXT: mov x0, x16
+; CHECK-NEXT: ret
+
+ ret ptr ptrauth (ptr getelementptr (i8, ptr @g, i64 16), i32 2)
+}
+
+define ptr @test_global_neg_offset_zero_disc() {
+; CHECK-LABEL: test_global_neg_offset_zero_disc:
+; CHECK: // %bb.0:
+; CHECK-NEXT: adrp x16, :got:g
+; CHECK-NEXT: ldr x16, [x16, :got_lo12:g]
+; CHECK-NEXT: sub x16, x16, #576
+; CHECK-NEXT: sub x16, x16, #30, lsl #12
+; CHECK-NEXT: pacdza x16
+; CHECK-NEXT: mov x0, x16
+; CHECK-NEXT: ret
+
+ ret ptr ptrauth (ptr getelementptr (i8, ptr @g, i64 -123456), i32 2)
+}
+
+define ptr @test_global_big_offset_zero_disc() {
+; CHECK-LABEL: test_global_big_offset_zero_disc:
+; CHECK: // %bb.0:
+; CHECK-NEXT: adrp x16, :got:g
+; CHECK-NEXT: ldr x16, [x16, :got_lo12:g]
+; CHECK-NEXT: mov x17, #1
+; CHECK-NEXT: movk x17, #32769, lsl #16
+; CHECK-NEXT: add x16, x16, x17
+; CHECK-NEXT: pacdza x16
+; CHECK-NEXT: mov x0, x16
+; CHECK-NEXT: ret
+
+ ret ptr ptrauth (ptr getelementptr (i8, ptr @g, i64 add (i64 2147483648, i64 65537)), i32 2)
+}
+
+define ptr @test_global_big_neg_offset_zero_disc() {
+; CHECK-LABEL: test_global_big_neg_offset_zero_disc:
+; CHECK: // %bb.0:
+; CHECK-NEXT: adrp x16, :got:g
+; CHECK-NEXT: ldr x16, [x16, :got_lo12:g]
+; CHECK-NEXT: mov x17, #-52501
+; CHECK-NEXT: movk x17, #63652, lsl #16
+; CHECK-NEXT: add x16, x16, x17
+; CHECK-NEXT: pacdza x16
+; CHECK-NEXT: mov x0, x16
+; CHECK-NEXT: ret
+
+ ret ptr ptrauth (ptr getelementptr (i8, ptr @g, i64 -123456789), i32 2)
+}
+
+define ptr @test_global_huge_neg_offset_zero_disc() {
+; CHECK-LABEL: test_global_huge_neg_offset_zero_disc:
+; CHECK: // %bb.0:
+; CHECK-NEXT: adrp x16, :got:g
+; CHECK-NEXT: ldr x16, [x16, :got_lo12:g]
+; CHECK-NEXT: mov x17, #-65536
+; CHECK-NEXT: movk x17, #0, lsl #16
+; CHECK-NEXT: movk x17, #0, lsl #32
+; CHECK-NEXT: movk x17, #32768, lsl #48
+; CHECK-NEXT: add x16, x16, x17
+; CHECK-NEXT: pacdza x16
+; CHECK-NEXT: mov x0, x16
+; CHECK-NEXT: ret
+
+ ret ptr ptrauth (ptr getelementptr (i8, ptr @g, i64 -9223372036854775808), i32 2)
+}
+
+define ptr @test_global_disc() {
+; CHECK-LABEL: test_global_disc:
+; CHECK: // %bb.0:
+; CHECK-NEXT: adrp x16, :got:g
+; CHECK-NEXT: ldr x16, [x16, :got_lo12:g]
+; CHECK-NEXT: mov x17, #42 // =0x2a
+; CHECK-NEXT: pacia x16, x17
+; CHECK-NEXT: mov x0, x16
+; CHECK-NEXT: ret
+
+ ret ptr ptrauth (ptr @g, i32 0, i64 42)
+}
+
+ at g.ref.da.42.addr = dso_local constant ptr ptrauth (ptr @g, i32 2, i64 42, ptr @g.ref.da.42.addr)
+
+define ptr @test_global_addr_disc() {
+; CHECK-LABEL: test_global_addr_disc:
+; CHECK: // %bb.0:
+; CHECK-NEXT: adrp x8, g.ref.da.42.addr
+; CHECK-NEXT: add x8, x8, :lo12:g.ref.da.42.addr
+; CHECK-NEXT: adrp x16, :got:g
+; CHECK-NEXT: ldr x16, [x16, :got_lo12:g]
+; CHECK-NEXT: mov x17, x8
+; CHECK-NEXT: movk x17, #42, lsl #48
+; CHECK-NEXT: pacda x16, x17
+; CHECK-NEXT: mov x0, x16
+; CHECK-NEXT: ret
+
+ ret ptr ptrauth (ptr @g, i32 2, i64 42, ptr @g.ref.da.42.addr)
+}
+
+define ptr @test_global_process_specific() {
+; CHECK-LABEL: test_global_process_specific:
+; CHECK: // %bb.0:
+; CHECK-NEXT: adrp x16, :got:g
+; CHECK-NEXT: ldr x16, [x16, :got_lo12:g]
+; CHECK-NEXT: pacizb x16
+; CHECK-NEXT: mov x0, x16
+; CHECK-NEXT: ret
+ ret ptr ptrauth (ptr @g, i32 1)
+}
+
+; weak symbols can't be assumed to be non-nil. Use $auth_ptr$ stub slot always.
+; The alternative is to emit a null-check here, but that'd be redundant with
+; whatever null-check follows in user code.
+
+define ptr @test_global_weak() {
+; CHECK-LABEL: test_global_weak:
+; CHECK: // %bb.0:
+; CHECK-NEXT: adrp x0, g_weak$auth_ptr$ia$42
+; CHECK-NEXT: ldr x0, [x0, :lo12:g_weak$auth_ptr$ia$42]
+; CHECK-NEXT: ret
+ ret ptr ptrauth (ptr @g_weak, i32 0, i64 42)
+}
+
+; Non-external symbols don't need to be accessed through the GOT.
+
+define ptr @test_global_strong_def() {
+; CHECK-LABEL: test_global_strong_def:
+; CHECK: // %bb.0:
+; CHECK-NEXT: adrp x16, g_strong_def
+; CHECK-NEXT: add x16, x16, :lo12:g_strong_def
+; CHECK-NEXT: pacdza x16
+; CHECK-NEXT: mov x0, x16
+; CHECK-NEXT: ret
+ ret ptr ptrauth (ptr @g_strong_def, i32 2)
+}
+
+; CHECK-LABEL: g_weak$auth_ptr$ia$42:
+; CHECK-NEXT: .xword g_weak at AUTH(ia,42)
diff --git a/llvm/test/CodeGen/AArch64/ptrauth-constant-in-code.ll b/llvm/test/CodeGen/AArch64/ptrauth-constant-in-code.ll
new file mode 100644
index 0000000000000..73993fa87bf27
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/ptrauth-constant-in-code.ll
@@ -0,0 +1,230 @@
+; RUN: rm -rf %t && split-file %s %t && cd %t
+
+;--- err1.ll
+
+; RUN: not --crash llc < err1.ll -mtriple aarch64-elf -mattr=+pauth \
+; RUN: -global-isel=0 -verify-machineinstrs 2>&1 | FileCheck --check-prefix=ERR1 %s
+
+ at g = external global i32
+
+define ptr @foo() {
+; ERR1: LLVM ERROR: key in ptrauth global out of range [0, 3]
+ ret ptr ptrauth (ptr @g, i32 4)
+}
+
+;--- err2.ll
+
+; RUN: not --crash llc < err2.ll -mtriple aarch64-elf -mattr=+pauth \
+; RUN: -global-isel=0 -verify-machineinstrs 2>&1 | FileCheck --check-prefix=ERR2 %s
+
+ at g = external global i32
+
+define ptr @foo() {
+; ERR2: LLVM ERROR: constant discriminator in ptrauth global out of range [0, 0xffff]
+ ret ptr ptrauth (ptr @g, i32 2, i64 65536)
+}
+
+;--- err3.ll
+
+; RUN: not --crash llc < err3.ll -mtriple aarch64-elf -mattr=+pauth \
+; RUN: -global-isel=0 -verify-machineinstrs 2>&1 | FileCheck --check-prefix=ERR3 %s
+
+ at g_weak = extern_weak global i32
+
+define ptr @foo() {
+; ERR3: LLVM ERROR: unsupported non-zero offset in weak ptrauth global reference
+ ret ptr ptrauth (ptr getelementptr (i8, ptr @g_weak, i64 16), i32 2, i64 42)
+}
+
+;--- err4.ll
+
+; RUN: not --crash llc < err4.ll -mtriple aarch64-elf -mattr=+pauth \
+; RUN: -global-isel=0 -verify-machineinstrs 2>&1 | FileCheck --check-prefix=ERR4 %s
+
+ at g_weak = extern_weak global i32
+ at g_weak.ref.da.42.addr = dso_local constant ptr ptrauth (ptr @g_weak, i32 2, i64 42, ptr @g_weak.ref.da.42.addr)
+
+define ptr @foo() {
+; ERR4: LLVM ERROR: unsupported weak addr-div ptrauth global
+ ret ptr ptrauth (ptr @g_weak, i32 0, i64 42, ptr @g_weak.ref.da.42.addr)
+}
+
+;--- err5.ll
+
+; RUN: not --crash llc < err5.ll -mtriple arm64-apple-darwin -mattr=+pauth \
+; RUN: -global-isel=0 -verify-machineinstrs 2>&1 | FileCheck --check-prefix=ERR5 %s
+
+ at g = external global i32
+
+define ptr @foo() {
+; ERR5: LLVM ERROR: ptrauth global lowering is only implemented for ELF
+ ret ptr ptrauth (ptr @g, i32 0)
+}
+
+;--- ok.ll
+
+; RUN: llc < ok.ll -mtriple aarch64-elf -mattr=+pauth -global-isel=0 \
+; RUN: -verify-machineinstrs | FileCheck %s
+; RUN: llc < ok.ll -mtriple aarch64-elf -mattr=+pauth -global-isel=0 \
+; RUN: -verify-machineinstrs -filetype=obj
+
+ at g = external global i32
+ at g_weak = extern_weak global i32
+ at g_strong_def = dso_local constant i32 42
+
+define ptr @test_global_zero_disc() {
+; CHECK-LABEL: test_global_zero_disc:
+; CHECK: // %bb.0:
+; CHECK-NEXT: adrp x16, :got:g
+; CHECK-NEXT: ldr x16, [x16, :got_lo12:g]
+; CHECK-NEXT: paciza x16
+; CHECK-NEXT: mov x0, x16
+; CHECK-NEXT: ret
+
+ ret ptr ptrauth (ptr @g, i32 0)
+}
+
+define ptr @test_global_offset_zero_disc() {
+; CHECK-LABEL: test_global_offset_zero_disc:
+; CHECK: // %bb.0:
+; CHECK-NEXT: adrp x16, :got:g
+; CHECK-NEXT: ldr x16, [x16, :got_lo12:g]
+; CHECK-NEXT: add x16, x16, #16
+; CHECK-NEXT: pacdza x16
+; CHECK-NEXT: mov x0, x16
+; CHECK-NEXT: ret
+
+ ret ptr ptrauth (ptr getelementptr (i8, ptr @g, i64 16), i32 2)
+}
+
+define ptr @test_global_neg_offset_zero_disc() {
+; CHECK-LABEL: test_global_neg_offset_zero_disc:
+; CHECK: // %bb.0:
+; CHECK-NEXT: adrp x16, :got:g
+; CHECK-NEXT: ldr x16, [x16, :got_lo12:g]
+; CHECK-NEXT: sub x16, x16, #576
+; CHECK-NEXT: sub x16, x16, #30, lsl #12
+; CHECK-NEXT: pacdza x16
+; CHECK-NEXT: mov x0, x16
+; CHECK-NEXT: ret
+
+ ret ptr ptrauth (ptr getelementptr (i8, ptr @g, i64 -123456), i32 2)
+}
+
+define ptr @test_global_big_offset_zero_disc() {
+; CHECK-LABEL: test_global_big_offset_zero_disc:
+; CHECK: // %bb.0:
+; CHECK-NEXT: adrp x16, :got:g
+; CHECK-NEXT: ldr x16, [x16, :got_lo12:g]
+; CHECK-NEXT: mov x17, #1
+; CHECK-NEXT: movk x17, #32769, lsl #16
+; CHECK-NEXT: add x16, x16, x17
+; CHECK-NEXT: pacdza x16
+; CHECK-NEXT: mov x0, x16
+; CHECK-NEXT: ret
+
+ ret ptr ptrauth (ptr getelementptr (i8, ptr @g, i64 add (i64 2147483648, i64 65537)), i32 2)
+}
+
+define ptr @test_global_big_neg_offset_zero_disc() {
+; CHECK-LABEL: test_global_big_neg_offset_zero_disc:
+; CHECK: // %bb.0:
+; CHECK-NEXT: adrp x16, :got:g
+; CHECK-NEXT: ldr x16, [x16, :got_lo12:g]
+; CHECK-NEXT: mov x17, #-52501
+; CHECK-NEXT: movk x17, #63652, lsl #16
+; CHECK-NEXT: add x16, x16, x17
+; CHECK-NEXT: pacdza x16
+; CHECK-NEXT: mov x0, x16
+; CHECK-NEXT: ret
+
+ ret ptr ptrauth (ptr getelementptr (i8, ptr @g, i64 -123456789), i32 2)
+}
+
+define ptr @test_global_huge_neg_offset_zero_disc() {
+; CHECK-LABEL: test_global_huge_neg_offset_zero_disc:
+; CHECK: // %bb.0:
+; CHECK-NEXT: adrp x16, :got:g
+; CHECK-NEXT: ldr x16, [x16, :got_lo12:g]
+; CHECK-NEXT: mov x17, #-65536
+; CHECK-NEXT: movk x17, #0, lsl #16
+; CHECK-NEXT: movk x17, #0, lsl #32
+; CHECK-NEXT: movk x17, #32768, lsl #48
+; CHECK-NEXT: add x16, x16, x17
+; CHECK-NEXT: pacdza x16
+; CHECK-NEXT: mov x0, x16
+; CHECK-NEXT: ret
+
+ ret ptr ptrauth (ptr getelementptr (i8, ptr @g, i64 -9223372036854775808), i32 2)
+}
+
+define ptr @test_global_disc() {
+; CHECK-LABEL: test_global_disc:
+; CHECK: // %bb.0:
+; CHECK-NEXT: adrp x16, :got:g
+; CHECK-NEXT: ldr x16, [x16, :got_lo12:g]
+; CHECK-NEXT: mov x17, #42 // =0x2a
+; CHECK-NEXT: pacia x16, x17
+; CHECK-NEXT: mov x0, x16
+; CHECK-NEXT: ret
+
+ ret ptr ptrauth (ptr @g, i32 0, i64 42)
+}
+
+ at g.ref.da.42.addr = dso_local constant ptr ptrauth (ptr @g, i32 2, i64 42, ptr @g.ref.da.42.addr)
+
+define ptr @test_global_addr_disc() {
+; CHECK-LABEL: test_global_addr_disc:
+; CHECK: // %bb.0:
+; CHECK-NEXT: adrp x8, g.ref.da.42.addr
+; CHECK-NEXT: add x8, x8, :lo12:g.ref.da.42.addr
+; CHECK-NEXT: adrp x16, :got:g
+; CHECK-NEXT: ldr x16, [x16, :got_lo12:g]
+; CHECK-NEXT: mov x17, x8
+; CHECK-NEXT: movk x17, #42, lsl #48
+; CHECK-NEXT: pacda x16, x17
+; CHECK-NEXT: mov x0, x16
+; CHECK-NEXT: ret
+
+ ret ptr ptrauth (ptr @g, i32 2, i64 42, ptr @g.ref.da.42.addr)
+}
+
+define ptr @test_global_process_specific() {
+; CHECK-LABEL: test_global_process_specific:
+; CHECK: // %bb.0:
+; CHECK-NEXT: adrp x16, :got:g
+; CHECK-NEXT: ldr x16, [x16, :got_lo12:g]
+; CHECK-NEXT: pacizb x16
+; CHECK-NEXT: mov x0, x16
+; CHECK-NEXT: ret
+ ret ptr ptrauth (ptr @g, i32 1)
+}
+
+; weak symbols can't be assumed to be non-nil. Use $auth_ptr$ stub slot always.
+; The alternative is to emit a null-check here, but that'd be redundant with
+; whatever null-check follows in user code.
+
+define ptr @test_global_weak() {
+; CHECK-LABEL: test_global_weak:
+; CHECK: // %bb.0:
+; CHECK-NEXT: adrp x0, g_weak$auth_ptr$ia$42
+; CHECK-NEXT: ldr x0, [x0, :lo12:g_weak$auth_ptr$ia$42]
+; CHECK-NEXT: ret
+ ret ptr ptrauth (ptr @g_weak, i32 0, i64 42)
+}
+
+; Non-external symbols don't need to be accessed through the GOT.
+
+define ptr @test_global_strong_def() {
+; CHECK-LABEL: test_global_strong_def:
+; CHECK: // %bb.0:
+; CHECK-NEXT: adrp x16, g_strong_def
+; CHECK-NEXT: add x16, x16, :lo12:g_strong_def
+; CHECK-NEXT: pacdza x16
+; CHECK-NEXT: mov x0, x16
+; CHECK-NEXT: ret
+ ret ptr ptrauth (ptr @g_strong_def, i32 2)
+}
+
+; CHECK-LABEL: g_weak$auth_ptr$ia$42:
+; CHECK-NEXT: .xword g_weak at AUTH(ia,42)
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-cxx.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-cxx.td
index bb1acde296429..2825ade2d2134 100644
--- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-cxx.td
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-cxx.td
@@ -85,7 +85,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
// CHECK: const uint8_t *GenMyCombiner::getMatchTable() const {
// CHECK-NEXT: constexpr static uint8_t MatchTable0[] = {
-// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(94), GIMT_Encode2(194), /*)*//*default:*//*Label 4*/ GIMT_Encode4(464),
+// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(95), GIMT_Encode2(195), /*)*//*default:*//*Label 4*/ GIMT_Encode4(464),
// CHECK-NEXT: /*TargetOpcode::G_STORE*//*Label 0*/ GIMT_Encode4(410), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
// CHECK-NEXT: /*TargetOpcode::G_SEXT*//*Label 1*/ GIMT_Encode4(428), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
// CHECK-NEXT: /*TargetOpcode::G_FNEG*//*Label 2*/ GIMT_Encode4(440), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-variadics.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-variadics.td
index 35bddf912a541..43cdd4f5546bb 100644
--- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-variadics.td
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-variadics.td
@@ -37,7 +37,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
// CHECK: const uint8_t *GenMyCombiner::getMatchTable() const {
// CHECK-NEXT: constexpr static uint8_t MatchTable0[] = {
-// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(69), GIMT_Encode2(73), /*)*//*default:*//*Label 2*/ GIMT_Encode4(84),
+// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(70), GIMT_Encode2(74), /*)*//*default:*//*Label 2*/ GIMT_Encode4(84),
// CHECK-NEXT: /*TargetOpcode::G_UNMERGE_VALUES*//*Label 0*/ GIMT_Encode4(26), GIMT_Encode4(0), GIMT_Encode4(0),
// CHECK-NEXT: /*TargetOpcode::G_BUILD_VECTOR*//*Label 1*/ GIMT_Encode4(55),
// CHECK-NEXT: // Label 0: @26
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td
index 513a86754d6d9..7ff637fa1e0e3 100644
--- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td
@@ -135,15 +135,15 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
// Verify match table.
// CHECK: const uint8_t *GenMyCombiner::getMatchTable() const {
// CHECK-NEXT: constexpr static uint8_t MatchTable0[] = {
-// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(19), GIMT_Encode2(133), /*)*//*default:*//*Label 6*/ GIMT_Encode4(653),
-// CHECK-NEXT: /*TargetOpcode::COPY*//*Label 0*/ GIMT_Encode4(466), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
-// CHECK-NEXT: /*TargetOpcode::G_AND*//*Label 1*/ GIMT_Encode4(502), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
-// CHECK-NEXT: /*TargetOpcode::G_STORE*//*Label 2*/ GIMT_Encode4(549), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
-// CHECK-NEXT: /*TargetOpcode::G_TRUNC*//*Label 3*/ GIMT_Encode4(583), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
-// CHECK-NEXT: /*TargetOpcode::G_SEXT*//*Label 4*/ GIMT_Encode4(606), GIMT_Encode4(0),
-// CHECK-NEXT: /*TargetOpcode::G_ZEXT*//*Label 5*/ GIMT_Encode4(618),
-// CHECK-NEXT: // Label 0: @466
-// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 7*/ GIMT_Encode4(490), // Rule ID 4 //
+// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(19), GIMT_Encode2(134), /*)*//*default:*//*Label 6*/ GIMT_Encode4(657),
+// CHECK-NEXT: /*TargetOpcode::COPY*//*Label 0*/ GIMT_Encode4(470), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
+// CHECK-NEXT: /*TargetOpcode::G_AND*//*Label 1*/ GIMT_Encode4(506), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
+// CHECK-NEXT: /*TargetOpcode::G_STORE*//*Label 2*/ GIMT_Encode4(553), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
+// CHECK-NEXT: /*TargetOpcode::G_TRUNC*//*Label 3*/ GIMT_Encode4(587), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
+// CHECK-NEXT: /*TargetOpcode::G_SEXT*//*Label 4*/ GIMT_Encode4(610), GIMT_Encode4(0),
+// CHECK-NEXT: /*TargetOpcode::G_ZEXT*//*Label 5*/ GIMT_Encode4(622),
+// CHECK-NEXT: // Label 0: @470
+// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 7*/ GIMT_Encode4(494), // Rule ID 4 //
// CHECK-NEXT: GIM_CheckFeatures, GIMT_Encode2(GIFBS_HasAnswerToEverything),
// CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule3Enabled),
// CHECK-NEXT: // MIs[0] a
@@ -156,8 +156,8 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
// CHECK-NEXT: GIM_CheckIsSafeToFold, /*NumInsns*/1,
// CHECK-NEXT: // Combiner Rule #3: InstTest1
// CHECK-NEXT: GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner2),
-// CHECK-NEXT: // Label 7: @490
-// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 8*/ GIMT_Encode4(501), // Rule ID 3 //
+// CHECK-NEXT: // Label 7: @494
+// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 8*/ GIMT_Encode4(505), // Rule ID 3 //
// CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule2Enabled),
// CHECK-NEXT: // MIs[0] a
// CHECK-NEXT: // No operand predicates
@@ -165,10 +165,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
// CHECK-NEXT: // No operand predicates
// CHECK-NEXT: // Combiner Rule #2: InstTest0
// CHECK-NEXT: GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner1),
-// CHECK-NEXT: // Label 8: @501
+// CHECK-NEXT: // Label 8: @505
// CHECK-NEXT: GIM_Reject,
-// CHECK-NEXT: // Label 1: @502
-// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 9*/ GIMT_Encode4(548), // Rule ID 6 //
+// CHECK-NEXT: // Label 1: @506
+// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 9*/ GIMT_Encode4(552), // Rule ID 6 //
// CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule5Enabled),
// CHECK-NEXT: GIM_RootCheckType, /*Op*/2, /*Type*/GILLT_s32,
// CHECK-NEXT: // MIs[0] dst
@@ -185,10 +185,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
// CHECK-NEXT: GIR_RootToRootCopy, /*OpIdx*/0, // dst
// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/1, // z
// CHECK-NEXT: GIR_EraseRootFromParent_Done,
-// CHECK-NEXT: // Label 9: @548
+// CHECK-NEXT: // Label 9: @552
// CHECK-NEXT: GIM_Reject,
-// CHECK-NEXT: // Label 2: @549
-// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 10*/ GIMT_Encode4(582), // Rule ID 5 //
+// CHECK-NEXT: // Label 2: @553
+// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 10*/ GIMT_Encode4(586), // Rule ID 5 //
// CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule4Enabled),
// CHECK-NEXT: // MIs[0] tmp
// CHECK-NEXT: GIM_RecordInsnIgnoreCopies, /*DefineMI*/1, /*MI*/0, /*OpIdx*/0, // MIs[1]
@@ -204,29 +204,29 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
// CHECK-NEXT: GIR_RootToRootCopy, /*OpIdx*/1, // ptr
// CHECK-NEXT: GIR_MergeMemOperands, /*InsnID*/0, /*NumInsns*/2, /*MergeInsnID's*/0, 1,
// CHECK-NEXT: GIR_EraseRootFromParent_Done,
-// CHECK-NEXT: // Label 10: @582
+// CHECK-NEXT: // Label 10: @586
// CHECK-NEXT: GIM_Reject,
-// CHECK-NEXT: // Label 3: @583
-// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 11*/ GIMT_Encode4(594), // Rule ID 0 //
+// CHECK-NEXT: // Label 3: @587
+// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 11*/ GIMT_Encode4(598), // Rule ID 0 //
// CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled),
// CHECK-NEXT: // Combiner Rule #0: WipOpcodeTest0; wip_match_opcode 'G_TRUNC'
// CHECK-NEXT: GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner0),
-// CHECK-NEXT: // Label 11: @594
-// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 12*/ GIMT_Encode4(605), // Rule ID 1 //
+// CHECK-NEXT: // Label 11: @598
+// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 12*/ GIMT_Encode4(609), // Rule ID 1 //
// CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule1Enabled),
// CHECK-NEXT: // Combiner Rule #1: WipOpcodeTest1; wip_match_opcode 'G_TRUNC'
// CHECK-NEXT: GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner0),
-// CHECK-NEXT: // Label 12: @605
+// CHECK-NEXT: // Label 12: @609
// CHECK-NEXT: GIM_Reject,
-// CHECK-NEXT: // Label 4: @606
-// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 13*/ GIMT_Encode4(617), // Rule ID 2 //
+// CHECK-NEXT: // Label 4: @610
+// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 13*/ GIMT_Encode4(621), // Rule ID 2 //
// CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule1Enabled),
// CHECK-NEXT: // Combiner Rule #1: WipOpcodeTest1; wip_match_opcode 'G_SEXT'
// CHECK-NEXT: GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner0),
-// CHECK-NEXT: // Label 13: @617
+// CHECK-NEXT: // Label 13: @621
// CHECK-NEXT: GIM_Reject,
-// CHECK-NEXT: // Label 5: @618
-// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 14*/ GIMT_Encode4(652), // Rule ID 7 //
+// CHECK-NEXT: // Label 5: @622
+// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 14*/ GIMT_Encode4(656), // Rule ID 7 //
// CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule6Enabled),
// CHECK-NEXT: // MIs[0] dst
// CHECK-NEXT: // No operand predicates
@@ -240,10 +240,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
// CHECK-NEXT: GIR_RootToRootCopy, /*OpIdx*/0, // dst
// CHECK-NEXT: GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0,
// CHECK-NEXT: GIR_EraseRootFromParent_Done,
-// CHECK-NEXT: // Label 14: @652
+// CHECK-NEXT: // Label 14: @656
// CHECK-NEXT: GIM_Reject,
-// CHECK-NEXT: // Label 6: @653
+// CHECK-NEXT: // Label 6: @657
// CHECK-NEXT: GIM_Reject,
-// CHECK-NEXT: }; // Size: 654 bytes
+// CHECK-NEXT: }; // Size: 658 bytes
// CHECK-NEXT: return MatchTable0;
// CHECK-NEXT: }
diff --git a/llvm/test/TableGen/GlobalISelEmitter.td b/llvm/test/TableGen/GlobalISelEmitter.td
index 5d5bf92664a79..796f595930319 100644
--- a/llvm/test/TableGen/GlobalISelEmitter.td
+++ b/llvm/test/TableGen/GlobalISelEmitter.td
@@ -513,7 +513,7 @@ def : Pat<(frag GPR32:$src1, complex:$src2, complex:$src3),
// R00O-NEXT: GIM_Reject,
// R00O: // Label [[DEFAULT_NUM]]: @[[DEFAULT]]
// R00O-NEXT: GIM_Reject,
-// R00O-NEXT: }; // Size: 1804 bytes
+// R00O-NEXT: }; // Size: 1808 bytes
def INSNBOB : I<(outs GPR32:$dst), (ins GPR32:$src1, GPR32:$src2, GPR32:$src3, GPR32:$src4),
[(set GPR32:$dst,
More information about the llvm-commits
mailing list