[llvm] [PAC][AArch64] Lower ptrauth constants in code (PR #94241)

Daniil Kovalev via llvm-commits llvm-commits at lists.llvm.org
Mon Jun 3 09:17:20 PDT 2024


https://github.com/kovdan01 created https://github.com/llvm/llvm-project/pull/94241

Define the following pseudos for lowering ptrauth constants in code:

- non-`extern_weak`:
  - no GOT load needed: `MOVaddrPAC` - similar to `MOVaddr`, with added PAC;
  - GOT load needed: `LOADgotPAC` - similar to `LOADgot`, with added PAC;
- `extern_weak`: `LOADauthptrstatic` - similar to `LOADgot`, but use a special stub slot named `sym$auth_ptr$key$disc` filled by dynamic linker during relocation resolving instead of a GOT slot.

>From 9207e1252f824adc2c0d186b91ae8c49c97c19b1 Mon Sep 17 00:00:00 2001
From: Daniil Kovalev <dkovalev at accesssoftek.com>
Date: Mon, 3 Jun 2024 19:15:08 +0300
Subject: [PATCH] [PAC][AArch64] Lower ptrauth constants in code

Define the following pseudos for lowering ptrauth constants in code:

- non-`extern_weak`:
  - no GOT load needed: `MOVaddrPAC` - similar to `MOVaddr`, with added PAC;
  - GOT load needed: `LOADgotPAC` - similar to `LOADgot`, with added PAC;
- `extern_weak`: `LOADauthptrstatic` - similar to `LOADgot`, but use a special
  stub slot named `sym$auth_ptr$key$disc` filled by dynamic linker during
  relocation resolving instead of a GOT slot.

Co-authored-by: Ahmed Bougacha <ahmed at bougacha.org>
---
 llvm/include/llvm/CodeGen/ISDOpcodes.h        |   6 +
 .../llvm/CodeGen/MachineModuleInfoImpls.h     |  13 ++
 llvm/include/llvm/Support/TargetOpcodes.def   |   3 +
 llvm/include/llvm/Target/GenericOpcodes.td    |   6 +
 llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp  |  11 +-
 .../SelectionDAG/SelectionDAGBuilder.cpp      |   7 +
 .../SelectionDAG/SelectionDAGDumper.cpp       |   4 +-
 llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 183 ++++++++++++++++++
 .../Target/AArch64/AArch64ISelLowering.cpp    | 129 ++++++++++++
 llvm/lib/Target/AArch64/AArch64ISelLowering.h |   6 +
 llvm/lib/Target/AArch64/AArch64InstrInfo.td   |  32 +++
 .../AArch64/AArch64TargetObjectFile.cpp       |  37 ++++
 .../Target/AArch64/AArch64TargetObjectFile.h  |   6 +
 .../GISel/AArch64InstructionSelector.cpp      | 138 +++++++++++++
 .../AArch64/GISel/AArch64LegalizerInfo.cpp    |   3 +
 .../GlobalISel/legalizer-info-validation.mir  |   4 +
 .../GlobalISel/ptrauth-constant-in-code.ll    | 118 +++++++++++
 .../AArch64/ptrauth-constant-in-code.ll       | 117 +++++++++++
 .../match-table-cxx.td                        |   2 +-
 .../match-table-variadics.td                  |   2 +-
 .../GlobalISelCombinerEmitter/match-table.td  |  62 +++---
 llvm/test/TableGen/GlobalISelEmitter.td       |   2 +-
 22 files changed, 854 insertions(+), 37 deletions(-)
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/ptrauth-constant-in-code.ll
 create mode 100644 llvm/test/CodeGen/AArch64/ptrauth-constant-in-code.ll

diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h
index 0f87e062e2da6..20d3180e5a805 100644
--- a/llvm/include/llvm/CodeGen/ISDOpcodes.h
+++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h
@@ -83,6 +83,12 @@ enum NodeType {
   ExternalSymbol,
   BlockAddress,
 
+  /// A ptrauth constant.
+  /// ptr, key, addr-disc, disc
+  /// Note that the addr-disc can be a non-constant value, to allow representing
+  /// a constant global address signed using address-diversification, in code.
+  PtrAuthGlobalAddress,
+
   /// The address of the GOT
   GLOBAL_OFFSET_TABLE,
 
diff --git a/llvm/include/llvm/CodeGen/MachineModuleInfoImpls.h b/llvm/include/llvm/CodeGen/MachineModuleInfoImpls.h
index f8a328f13eded..76f8ce270e805 100644
--- a/llvm/include/llvm/CodeGen/MachineModuleInfoImpls.h
+++ b/llvm/include/llvm/CodeGen/MachineModuleInfoImpls.h
@@ -61,10 +61,18 @@ class MachineModuleInfoMachO : public MachineModuleInfoImpl {
 /// MachineModuleInfoELF - This is a MachineModuleInfoImpl implementation
 /// for ELF targets.
 class MachineModuleInfoELF : public MachineModuleInfoImpl {
+public:
+  struct AuthStubInfo {
+    const MCExpr *AuthPtrRef;
+  };
+
+private:
   /// GVStubs - These stubs are used to materialize global addresses in PIC
   /// mode.
   DenseMap<MCSymbol *, StubValueTy> GVStubs;
 
+  DenseMap<MCSymbol *, AuthStubInfo> AuthPtrStubs;
+
   virtual void anchor(); // Out of line virtual method.
 
 public:
@@ -75,6 +83,11 @@ class MachineModuleInfoELF : public MachineModuleInfoImpl {
     return GVStubs[Sym];
   }
 
+  AuthStubInfo &getAuthPtrStubEntry(MCSymbol *Sym) {
+    assert(Sym && "Key cannot be null");
+    return AuthPtrStubs[Sym];
+  }
+
   /// Accessor methods to return the set of stubs in sorted order.
 
   SymbolListTy GetGVStubList() { return getSortedStubs(GVStubs); }
diff --git a/llvm/include/llvm/Support/TargetOpcodes.def b/llvm/include/llvm/Support/TargetOpcodes.def
index 559a588c25148..2bb4a0ffd43fe 100644
--- a/llvm/include/llvm/Support/TargetOpcodes.def
+++ b/llvm/include/llvm/Support/TargetOpcodes.def
@@ -294,6 +294,9 @@ HANDLE_TARGET_OPCODE(G_FRAME_INDEX)
 /// Generic reference to global value.
 HANDLE_TARGET_OPCODE(G_GLOBAL_VALUE)
 
+/// Generic ptrauth-signed reference to global value.
+HANDLE_TARGET_OPCODE(G_PTRAUTH_GLOBAL_VALUE)
+
 /// Generic instruction to materialize the address of an object in the constant
 /// pool.
 HANDLE_TARGET_OPCODE(G_CONSTANT_POOL)
diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td
index c40498e554215..6846a7e008f3a 100644
--- a/llvm/include/llvm/Target/GenericOpcodes.td
+++ b/llvm/include/llvm/Target/GenericOpcodes.td
@@ -110,6 +110,12 @@ def G_GLOBAL_VALUE : GenericInstruction {
   let hasSideEffects = false;
 }
 
+def G_PTRAUTH_GLOBAL_VALUE : GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins unknown:$addr, i32imm:$key, type1:$addrdisc, i64imm:$disc);
+  let hasSideEffects = 0;
+}
+
 def G_CONSTANT_POOL : GenericInstruction {
   let OutOperandList = (outs type0:$dst);
   let InOperandList = (ins unknown:$src);
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 67c51f018a807..e1bfe7d41b070 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -3490,7 +3490,16 @@ bool IRTranslator::translate(const Constant &C, Register Reg) {
     EntryBuilder->buildConstant(Reg, 0);
   else if (auto GV = dyn_cast<GlobalValue>(&C))
     EntryBuilder->buildGlobalValue(Reg, GV);
-  else if (auto CAZ = dyn_cast<ConstantAggregateZero>(&C)) {
+  else if (auto CPA = dyn_cast<ConstantPtrAuth>(&C)) {
+    Register Addr = getOrCreateVReg(*CPA->getPointer());
+    Register AddrDisc = getOrCreateVReg(*CPA->getAddrDiscriminator());
+    EntryBuilder->buildInstr(TargetOpcode::G_PTRAUTH_GLOBAL_VALUE)
+        .addDef(Reg)
+        .addUse(Addr)
+        .addImm(CPA->getKey()->getZExtValue())
+        .addUse(AddrDisc)
+        .addImm(CPA->getDiscriminator()->getZExtValue());
+  } else if (auto CAZ = dyn_cast<ConstantAggregateZero>(&C)) {
     if (!isa<FixedVectorType>(CAZ->getType()))
       return false;
     // Return the scalar if it is a <1 x Ty> vector.
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index c7e0c62ec1263..b6731dcaf40b9 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -1794,6 +1794,13 @@ SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
     if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
       return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
 
+    if (const ConstantPtrAuth *CPA = dyn_cast<ConstantPtrAuth>(C)) {
+      return DAG.getNode(ISD::PtrAuthGlobalAddress, getCurSDLoc(), VT,
+                         getValue(CPA->getPointer()), getValue(CPA->getKey()),
+                         getValue(CPA->getAddrDiscriminator()),
+                         getValue(CPA->getDiscriminator()));
+    }
+
     if (isa<ConstantPointerNull>(C)) {
       unsigned AS = V->getType()->getPointerAddressSpace();
       return DAG.getConstant(0, getCurSDLoc(),
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
index 2198c2354483c..ae1e77385526c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
@@ -75,6 +75,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
     }
     return "<<Unknown Node #" + utostr(getOpcode()) + ">>";
 
+    // clang-format off
 #ifndef NDEBUG
   case ISD::DELETED_NODE:               return "<<Deleted Node!>>";
 #endif
@@ -124,6 +125,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
   case ISD::ConstantFP:                 return "ConstantFP";
   case ISD::GlobalAddress:              return "GlobalAddress";
   case ISD::GlobalTLSAddress:           return "GlobalTLSAddress";
+  case ISD::PtrAuthGlobalAddress:       return "PtrAuthGlobalAddress";
   case ISD::FrameIndex:                 return "FrameIndex";
   case ISD::JumpTable:                  return "JumpTable";
   case ISD::JUMP_TABLE_DEBUG_INFO:
@@ -166,8 +168,6 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
       return "OpaqueTargetConstant";
     return "TargetConstant";
 
-    // clang-format off
-
   case ISD::TargetConstantFP:           return "TargetConstantFP";
   case ISD::TargetGlobalAddress:        return "TargetGlobalAddress";
   case ISD::TargetGlobalTLSAddress:     return "TargetGlobalTLSAddress";
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index 7da540f8ef8e5..14c18330c185e 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -131,6 +131,13 @@ class AArch64AsmPrinter : public AsmPrinter {
   unsigned emitPtrauthDiscriminator(uint16_t Disc, unsigned AddrDisc,
                                     unsigned &InstsEmitted);
 
+  // Emit the sequence for LOADauthptrstatic
+  void LowerLOADauthptrstatic(const MachineInstr &MI);
+
+  // Emit the sequence for LOADgotPAC/MOVaddrPAC (either GOT adrp-ldr or
+  // adrp-add followed by PAC sign)
+  void LowerMOVaddrPAC(const MachineInstr &MI);
+
   /// tblgen'erated driver function for lowering simple MI->MC
   /// pseudo instructions.
   bool emitPseudoExpansionLowering(MCStreamer &OutStreamer,
@@ -1575,6 +1582,173 @@ void AArch64AsmPrinter::emitPtrauthBranch(const MachineInstr *MI) {
   assert(STI->getInstrInfo()->getInstSizeInBytes(*MI) >= InstsEmitted * 4);
 }
 
+void AArch64AsmPrinter::LowerLOADauthptrstatic(const MachineInstr &MI) {
+  unsigned DstReg = MI.getOperand(0).getReg();
+  MachineOperand GAOp = MI.getOperand(1);
+  uint64_t KeyC = MI.getOperand(2).getImm();
+  assert(KeyC <= AArch64PACKey::LAST && "Key is out of range");
+  auto Key = (AArch64PACKey::ID)KeyC;
+  uint64_t Disc = MI.getOperand(3).getImm();
+  assert(isUInt<16>(Disc) && "Constant discriminator is too wide");
+
+  const MCSymbol *GASym = TM.getSymbol(GAOp.getGlobal());
+  uint64_t Offset = GAOp.getOffset();
+
+  // Emit instruction sequence like the following:
+  //   ADRP x16, symbol$auth_ptr$key$disc
+  //   LDR x16, [x16, :lo12:symbol$auth_ptr$key$disc]
+  //
+  // Where the $auth_ptr$ symbol is the stub slot containing the signed pointer
+  // to symbol.
+  assert(TM.getTargetTriple().isOSBinFormatELF() &&
+         "LOADauthptrstatic only implemented on ELF");
+  assert(Offset == 0 &&
+         "Non-zero offset for $auth_ptr$ stub slots is not supported");
+
+  const auto &TLOF =
+      static_cast<const AArch64_ELFTargetObjectFile &>(getObjFileLowering());
+  MCSymbol *AuthPtrStubSym =
+      TLOF.getAuthPtrSlotSymbol(TM, &MF->getMMI(), GASym, Key, Disc);
+
+  MachineOperand StubMOHi =
+      MachineOperand::CreateMCSymbol(AuthPtrStubSym, AArch64II::MO_PAGE);
+  MachineOperand StubMOLo = MachineOperand::CreateMCSymbol(
+      AuthPtrStubSym, AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
+  MCOperand StubMCHi, StubMCLo;
+
+  MCInstLowering.lowerOperand(StubMOHi, StubMCHi);
+  MCInstLowering.lowerOperand(StubMOLo, StubMCLo);
+
+  EmitToStreamer(
+      *OutStreamer,
+      MCInstBuilder(AArch64::ADRP).addReg(DstReg).addOperand(StubMCHi));
+
+  EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::LDRXui)
+                                   .addReg(DstReg)
+                                   .addReg(DstReg)
+                                   .addOperand(StubMCLo));
+}
+
+void AArch64AsmPrinter::LowerMOVaddrPAC(const MachineInstr &MI) {
+  unsigned InstsEmitted = 0;
+
+  const bool IsGOTLoad = MI.getOpcode() == AArch64::LOADgotPAC;
+  MachineOperand GAOp = MI.getOperand(0);
+  uint64_t KeyC = MI.getOperand(1).getImm();
+  assert(KeyC <= AArch64PACKey::LAST && "Key is out of range");
+  auto Key = (AArch64PACKey::ID)KeyC;
+  unsigned AddrDisc = MI.getOperand(2).getReg();
+  uint64_t Disc = MI.getOperand(3).getImm();
+  assert(isUInt<16>(Disc) && "Constant discriminator is too wide");
+
+  uint64_t Offset = GAOp.getOffset();
+  GAOp.setOffset(0);
+
+  // Emit:
+  // target materialization:
+  //   via GOT:
+  //     adrp x16, :got:target
+  //     ldr x16, [x16, :got_lo12:target]
+  //     add x16, x16, #<offset> ; if offset != 0; up to 3 depending on width
+  //
+  //   direct:
+  //     adrp x16, target
+  //     add x16, x16, :lo12:target
+  //     add x16, x16, #<offset> ; if offset != 0; up to 3 depending on width
+  //
+  // signing:
+  // - 0 discriminator:
+  //     paciza x16
+  // - Non-0 discriminator, no address discriminator:
+  //     mov x17, #Disc
+  //     pacia x16, x17
+  // - address discriminator (with potentially folded immediate discriminator):
+  //     pacia x16, xAddrDisc
+
+  MachineOperand GAMOHi(GAOp), GAMOLo(GAOp);
+  MCOperand GAMCHi, GAMCLo;
+
+  GAMOHi.setTargetFlags(AArch64II::MO_PAGE);
+  GAMOLo.setTargetFlags(AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
+  if (IsGOTLoad) {
+    GAMOHi.addTargetFlag(AArch64II::MO_GOT);
+    GAMOLo.addTargetFlag(AArch64II::MO_GOT);
+  }
+
+  MCInstLowering.lowerOperand(GAMOHi, GAMCHi);
+  MCInstLowering.lowerOperand(GAMOLo, GAMCLo);
+
+  EmitToStreamer(
+      *OutStreamer,
+      MCInstBuilder(AArch64::ADRP).addReg(AArch64::X16).addOperand(GAMCHi));
+  ++InstsEmitted;
+
+  if (IsGOTLoad) {
+    EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::LDRXui)
+                                     .addReg(AArch64::X16)
+                                     .addReg(AArch64::X16)
+                                     .addOperand(GAMCLo));
+    ++InstsEmitted;
+  } else {
+    EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ADDXri)
+                                     .addReg(AArch64::X16)
+                                     .addReg(AArch64::X16)
+                                     .addOperand(GAMCLo)
+                                     .addImm(0));
+    ++InstsEmitted;
+  }
+
+  if (Offset) {
+    if (!isUInt<32>(Offset))
+      report_fatal_error("ptrauth global offset too large, 32bit max encoding");
+
+    for (int BitPos = 0; BitPos < 32 && (Offset >> BitPos); BitPos += 12) {
+      EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ADDXri)
+                                       .addReg(AArch64::X16)
+                                       .addReg(AArch64::X16)
+                                       .addImm((Offset >> BitPos) & 0xfff)
+                                       .addImm(AArch64_AM::getShifterImm(
+                                           AArch64_AM::LSL, BitPos)));
+      ++InstsEmitted;
+    }
+  }
+
+  unsigned DiscReg = AddrDisc;
+  if (Disc) {
+    if (AddrDisc != AArch64::XZR) {
+      EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ORRXrs)
+                                       .addReg(AArch64::X17)
+                                       .addReg(AArch64::XZR)
+                                       .addReg(AddrDisc)
+                                       .addImm(0));
+      ++InstsEmitted;
+      EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::MOVKXi)
+                                       .addReg(AArch64::X17)
+                                       .addReg(AArch64::X17)
+                                       .addImm(Disc)
+                                       .addImm(/*shift=*/48));
+      ++InstsEmitted;
+    } else {
+      EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::MOVZXi)
+                                       .addReg(AArch64::X17)
+                                       .addImm(Disc)
+                                       .addImm(/*shift=*/0));
+      ++InstsEmitted;
+    }
+    DiscReg = AArch64::X17;
+  }
+
+  auto MIB = MCInstBuilder(getPACOpcodeForKey(Key, DiscReg == AArch64::XZR))
+                 .addReg(AArch64::X16)
+                 .addReg(AArch64::X16);
+  if (DiscReg != AArch64::XZR)
+    MIB.addReg(DiscReg);
+  EmitToStreamer(*OutStreamer, MIB);
+  ++InstsEmitted;
+
+  assert(STI->getInstrInfo()->getInstSizeInBytes(MI) >= InstsEmitted * 4);
+}
+
 // Simple pseudo-instructions have their lowering (with expansion to real
 // instructions) auto-generated.
 #include "AArch64GenMCPseudoLowering.inc"
@@ -1710,6 +1884,15 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
     return;
   }
 
+  case AArch64::LOADauthptrstatic:
+    LowerLOADauthptrstatic(*MI);
+    return;
+
+  case AArch64::LOADgotPAC:
+  case AArch64::MOVaddrPAC:
+    LowerMOVaddrPAC(*MI);
+    return;
+
   case AArch64::BLRA:
     emitPtrauthBranch(MI);
     return;
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index ca519c4c1788a..040081c6a122c 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -511,6 +511,8 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
   setOperationAction(ISD::JumpTable, MVT::i64, Custom);
   setOperationAction(ISD::SETCCCARRY, MVT::i64, Custom);
 
+  setOperationAction(ISD::PtrAuthGlobalAddress, MVT::i64, Custom);
+
   setOperationAction(ISD::SHL_PARTS, MVT::i64, Custom);
   setOperationAction(ISD::SRA_PARTS, MVT::i64, Custom);
   setOperationAction(ISD::SRL_PARTS, MVT::i64, Custom);
@@ -6546,6 +6548,8 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
     return LowerGlobalAddress(Op, DAG);
   case ISD::GlobalTLSAddress:
     return LowerGlobalTLSAddress(Op, DAG);
+  case ISD::PtrAuthGlobalAddress:
+    return LowerPtrAuthGlobalAddress(Op, DAG);
   case ISD::SETCC:
   case ISD::STRICT_FSETCC:
   case ISD::STRICT_FSETCCS:
@@ -9374,6 +9378,131 @@ SDValue AArch64TargetLowering::LowerGlobalTLSAddress(SDValue Op,
   llvm_unreachable("Unexpected platform trying to use TLS");
 }
 
+//===----------------------------------------------------------------------===//
+//                      PtrAuthGlobalAddress lowering
+//
+// We have 3 lowering alternatives to choose from:
+// - MOVaddrPAC: similar to MOVaddr, with added PAC.
+//   If the GV doesn't need a GOT load (i.e., is locally defined)
+//   materialize the pointer using adrp+add+pac.
+//   This is the preferred lowering.
+//
+// - LOADgotPAC: similar to LOADgot, with added PAC.
+//   If the GV needs a GOT load, materialize the pointer using the usual
+//   GOT adrp+ldr, +pac. Pointers in GOT should not be signed, the GOT section
+//   should be read-only (for example, via relro mechanism).
+//
+// - LOADauthptrstatic: similar to LOADgot, but use a
+//   special stub slot instead of a GOT slot.
+//   Load a signed pointer for symbol sym from a stub slot named
+//   sym$auth_ptr$key$disc filled by dynamic linker during relocation resolving.
+//   This usually lowers to adrp+ldr, but also emits an entry into .data with an
+//   @AUTH relocation. This is done in LowerPtrAuthGlobalAddressStatically.
+//
+// All 3 are pseudos that are expand late to longer sequences: this lets us
+// provide integrity guarantees on the to-be-signed intermediate values.
+//
+// LOADauthptrstatic is undesirable because it requires a large section filled
+// with often similarly-signed pointers, making it a good harvesting target.
+// Thus, it's only used for ptrauth references to extern_weak to avoid null
+// checks.
+
+SDValue AArch64TargetLowering::LowerPtrAuthGlobalAddressStatically(
+    SDValue TGA, SDLoc DL, EVT VT, AArch64PACKey::ID KeyC,
+    SDValue Discriminator, SDValue AddrDiscriminator, SelectionDAG &DAG) const {
+  const auto *TGN = cast<GlobalAddressSDNode>(TGA.getNode());
+  const GlobalValue *GV = TGN->getGlobal();
+  assert(GV->hasExternalWeakLinkage());
+
+  // Offsets and extern_weak don't mix well: ptrauth aside, you'd get the
+  // offset alone as a pointer if the symbol wasn't available, which probably
+  // breaking null checks in users.
+  // Ptrauth complicates things further: error out.
+  if (TGN->getOffset() != 0)
+    report_fatal_error("Unsupported offset in weak ptrauth global reference");
+
+  if (!isNullConstant(AddrDiscriminator))
+    report_fatal_error("Unsupported weak addr-div ptrauth global");
+
+  SDValue Key = DAG.getTargetConstant(KeyC, DL, MVT::i32);
+  return SDValue(DAG.getMachineNode(AArch64::LOADauthptrstatic, DL, MVT::i64,
+                                    {TGA, Key, Discriminator}),
+                 0);
+}
+
+SDValue
+AArch64TargetLowering::LowerPtrAuthGlobalAddress(SDValue Op,
+                                                 SelectionDAG &DAG) const {
+  SDValue Ptr = Op.getOperand(0);
+  uint64_t KeyC = Op.getConstantOperandVal(1);
+  SDValue AddrDiscriminator = Op.getOperand(2);
+  uint64_t DiscriminatorC = Op.getConstantOperandVal(3);
+  EVT VT = Op.getValueType();
+  SDLoc DL(Op);
+
+  // Choosing between 3 lowering alternatives is target-specific.
+  if (!Subtarget->isTargetELF())
+    llvm_unreachable("Unimplemented ptrauth global lowering");
+
+  uint64_t PtrOffsetC = 0;
+  if (Ptr.getOpcode() == ISD::ADD) {
+    PtrOffsetC = Ptr.getConstantOperandVal(1);
+    Ptr = Ptr.getOperand(0);
+  }
+  const GlobalAddressSDNode *PtrN = cast<GlobalAddressSDNode>(Ptr.getNode());
+  const GlobalValue *PtrGV = PtrN->getGlobal();
+
+  // Classify the reference to determine whether it needs a GOT load.
+  const unsigned OpFlags =
+      Subtarget->ClassifyGlobalReference(PtrGV, getTargetMachine());
+  const bool NeedsGOTLoad = ((OpFlags & AArch64II::MO_GOT) != 0);
+  assert(((OpFlags & (~AArch64II::MO_GOT)) == 0) &&
+         "Unsupported non-GOT op flags on ptrauth global reference");
+
+  // Fold any offset into the GV; our pseudos expect it there.
+  PtrOffsetC += PtrN->getOffset();
+  SDValue TPtr = DAG.getTargetGlobalAddress(PtrGV, DL, VT, PtrOffsetC,
+                                            /*TargetFlags=*/0);
+  assert(PtrN->getTargetFlags() == 0 && "Unsupported tflags on ptrauth global");
+
+  // None of our lowerings support an offset larger than 32-bit.
+  if (!isUInt<32>(PtrOffsetC))
+    report_fatal_error("Unsupported >32-bit-wide offset in ptrauth global");
+
+  // Blend only works if the integer discriminator is 16-bit wide.
+  if (!isUInt<16>(DiscriminatorC))
+    report_fatal_error(
+        "Unsupported >16-bit-wide constant discriminator in ptrauth global");
+
+  SDValue Key = DAG.getTargetConstant(KeyC, DL, MVT::i32);
+  SDValue Discriminator = DAG.getTargetConstant(DiscriminatorC, DL, MVT::i64);
+  SDValue TAddrDiscriminator = !isNullConstant(AddrDiscriminator)
+                                   ? AddrDiscriminator
+                                   : DAG.getRegister(AArch64::XZR, MVT::i64);
+
+  // No GOT load needed -> MOVaddrPAC
+  if (!NeedsGOTLoad) {
+    assert(!PtrGV->hasExternalWeakLinkage() && "extern_weak should use GOT");
+    return SDValue(
+        DAG.getMachineNode(AArch64::MOVaddrPAC, DL, MVT::i64,
+                           {TPtr, Key, TAddrDiscriminator, Discriminator}),
+        0);
+  }
+
+  // GOT load -> LOADgotPAC
+  // Note that we disallow extern_weak refs to avoid null checks later.
+  if (!PtrGV->hasExternalWeakLinkage())
+    return SDValue(
+        DAG.getMachineNode(AArch64::LOADgotPAC, DL, MVT::i64,
+                           {TPtr, Key, TAddrDiscriminator, Discriminator}),
+        0);
+
+  // extern_weak ref -> LOADauthptrstatic
+  return LowerPtrAuthGlobalAddressStatically(
+      TPtr, DL, VT, (AArch64PACKey::ID)KeyC, Discriminator, AddrDiscriminator,
+      DAG);
+}
+
 // Looks through \param Val to determine the bit that can be used to
 // check the sign of the value. It returns the unextended value and
 // the sign bit position.
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 48a4ea91c2782..ef3da9f0356bf 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -1122,6 +1122,12 @@ class AArch64TargetLowering : public TargetLowering {
   SDValue LowerELFTLSDescCallSeq(SDValue SymAddr, const SDLoc &DL,
                                  SelectionDAG &DAG) const;
   SDValue LowerWindowsGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
+  SDValue LowerPtrAuthGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
+  SDValue LowerPtrAuthGlobalAddressStatically(SDValue TGA, SDLoc DL, EVT VT,
+                                              AArch64PACKey::ID Key,
+                                              SDValue Discriminator,
+                                              SDValue AddrDiscriminator,
+                                              SelectionDAG &DAG) const;
   SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerBR_CC(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index 081d64921ee1c..3ec2a451444db 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -1762,6 +1762,38 @@ let Predicates = [HasPAuth] in {
   defm LDRAA  : AuthLoad<0, "ldraa", simm10Scaled>;
   defm LDRAB  : AuthLoad<1, "ldrab", simm10Scaled>;
 
+  // Materialize a signed global address, with adrp+add and PAC.
+  def MOVaddrPAC : Pseudo<(outs),
+                          (ins i64imm:$Addr, i32imm:$Key,
+                               GPR64noip:$AddrDisc, i64imm:$Disc), []>,
+               Sched<[WriteI, ReadI]> {
+    let isReMaterializable = 1;
+    let isCodeGenOnly = 1;
+    let Size = 32; // 12 fixed + 20 variable, for pointer offset, and discriminator
+    let Defs = [X16,X17];
+  }
+
+  // Materialize a signed global address, using a GOT load and PAC.
+  def LOADgotPAC : Pseudo<(outs),
+                          (ins i64imm:$Addr, i32imm:$Key,
+                               GPR64noip:$AddrDisc, i64imm:$Disc), []>,
+               Sched<[WriteI, ReadI]> {
+    let isReMaterializable = 1;
+    let isCodeGenOnly = 1;
+    let Size = 32; // 12 fixed + 20 variable, for pointer offset, and discriminator
+    let Defs = [X16,X17];
+  }
+
+  // Load a signed global address from a special $auth_ptr$ stub slot.
+  def LOADauthptrstatic : Pseudo<(outs GPR64:$dst),
+                              (ins i64imm:$Addr, i32imm:$Key,
+                                   i64imm:$Disc), []>,
+               Sched<[WriteI, ReadI]> {
+    let isReMaterializable = 1;
+    let isCodeGenOnly = 1;
+    let Size = 8;
+  }
+
   // Size 16: 4 fixed + 8 variable, to compute discriminator.
   let isCall = 1, isTerminator = 1, isReturn = 1, isBarrier = 1, Size = 16,
       Uses = [SP] in {
diff --git a/llvm/lib/Target/AArch64/AArch64TargetObjectFile.cpp b/llvm/lib/Target/AArch64/AArch64TargetObjectFile.cpp
index e5c6b6d60d068..1c1e87d723ac7 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetObjectFile.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetObjectFile.cpp
@@ -8,7 +8,10 @@
 
 #include "AArch64TargetObjectFile.h"
 #include "AArch64TargetMachine.h"
+#include "MCTargetDesc/AArch64MCExpr.h"
+#include "llvm/ADT/StringExtras.h"
 #include "llvm/BinaryFormat/Dwarf.h"
+#include "llvm/CodeGen/MachineModuleInfoImpls.h"
 #include "llvm/IR/Mangler.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
@@ -88,3 +91,37 @@ void AArch64_MachoTargetObjectFile::getNameWithPrefix(
   // be accessed via at least a linker-private symbol.
   getMangler().getNameWithPrefix(OutName, GV, /* CannotUsePrivateLabel */ true);
 }
+
+template <typename MachineModuleInfoTarget>
+static MCSymbol *getAuthPtrSlotSymbolHelper(
+    MCContext &Ctx, const TargetMachine &TM, MachineModuleInfo *MMI,
+    MachineModuleInfoTarget &TargetMMI, const MCSymbol *RawSym,
+    AArch64PACKey::ID Key, uint16_t Discriminator) {
+  const DataLayout &DL = MMI->getModule()->getDataLayout();
+
+  MCSymbol *StubSym = Ctx.getOrCreateSymbol(
+      DL.getLinkerPrivateGlobalPrefix() + RawSym->getName() +
+      Twine("$auth_ptr$") + AArch64PACKeyIDToString(Key) + Twine('$') +
+      Twine(Discriminator));
+
+  typename MachineModuleInfoTarget::AuthStubInfo &StubInfo =
+      TargetMMI.getAuthPtrStubEntry(StubSym);
+
+  if (StubInfo.AuthPtrRef)
+    return StubSym;
+
+  const MCExpr *Sym = MCSymbolRefExpr::create(RawSym, Ctx);
+
+  StubInfo.AuthPtrRef =
+      AArch64AuthMCExpr::create(Sym, Discriminator, Key,
+                                /*HasAddressDiversity=*/false, Ctx);
+  return StubSym;
+}
+
+MCSymbol *AArch64_ELFTargetObjectFile::getAuthPtrSlotSymbol(
+    const TargetMachine &TM, MachineModuleInfo *MMI, const MCSymbol *RawSym,
+    AArch64PACKey::ID Key, uint16_t Discriminator) const {
+  auto &ELFMMI = MMI->getObjFileInfo<MachineModuleInfoELF>();
+  return getAuthPtrSlotSymbolHelper(getContext(), TM, MMI, ELFMMI, RawSym, Key,
+                                    Discriminator);
+}
diff --git a/llvm/lib/Target/AArch64/AArch64TargetObjectFile.h b/llvm/lib/Target/AArch64/AArch64TargetObjectFile.h
index 7cd56fc9a189d..c5ebf03c39c77 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetObjectFile.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetObjectFile.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_LIB_TARGET_AARCH64_AARCH64TARGETOBJECTFILE_H
 #define LLVM_LIB_TARGET_AARCH64_AARCH64TARGETOBJECTFILE_H
 
+#include "Utils/AArch64BaseInfo.h"
 #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
 #include "llvm/Target/TargetLoweringObjectFile.h"
 
@@ -29,6 +30,11 @@ class AArch64_ELFTargetObjectFile : public TargetLoweringObjectFileELF {
                                           const MCValue &MV, int64_t Offset,
                                           MachineModuleInfo *MMI,
                                           MCStreamer &Streamer) const override;
+
+  MCSymbol *getAuthPtrSlotSymbol(const TargetMachine &TM,
+                                 MachineModuleInfo *MMI, const MCSymbol *RawSym,
+                                 AArch64PACKey::ID Key,
+                                 uint16_t Discriminator) const;
 };
 
 /// AArch64_MachoTargetObjectFile - This TLOF implementation is used for Darwin.
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index 4a7c82b393c10..8afd750cfcd9b 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -224,6 +224,8 @@ class AArch64InstructionSelector : public InstructionSelector {
   bool selectJumpTable(MachineInstr &I, MachineRegisterInfo &MRI);
   bool selectBrJT(MachineInstr &I, MachineRegisterInfo &MRI);
   bool selectTLSGlobalValue(MachineInstr &I, MachineRegisterInfo &MRI);
+  bool selectPtrAuthGlobalValue(MachineInstr &I,
+                                MachineRegisterInfo &MRI) const;
   bool selectReduction(MachineInstr &I, MachineRegisterInfo &MRI);
   bool selectMOPS(MachineInstr &I, MachineRegisterInfo &MRI);
   bool selectUSMovFromExtend(MachineInstr &I, MachineRegisterInfo &MRI);
@@ -2841,6 +2843,9 @@ bool AArch64InstructionSelector::select(MachineInstr &I) {
     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
   }
 
+  case TargetOpcode::G_PTRAUTH_GLOBAL_VALUE:
+    return selectPtrAuthGlobalValue(I, MRI);
+
   case TargetOpcode::G_ZEXTLOAD:
   case TargetOpcode::G_LOAD:
   case TargetOpcode::G_STORE: {
@@ -6573,6 +6578,139 @@ bool AArch64InstructionSelector::selectIntrinsic(MachineInstr &I,
   return false;
 }
 
+// G_PTRAUTH_GLOBAL_VALUE lowering
+
+// We have 3 lowering alternatives to choose from:
+// - MOVaddrPAC: similar to MOVaddr, with added PAC.
+//   If the GV doesn't need a GOT load (i.e., is locally defined)
+//   materialize the pointer using adrp+add+pac.
+//   This is the preferred lowering.
+//
+// - LOADgotPAC: similar to LOADgot, with added PAC.
+//   If the GV needs a GOT load, materialize the pointer using the usual
+//   GOT adrp+ldr, +pac. Pointers in GOT should not be signed, the GOT section
+//   should be read-only (for example, via relro mechanism).
+//
+// - LOADauthptrstatic: similar to LOADgot, but use a
+//   special stub slot instead of a GOT slot.
+//   Load a signed pointer for symbol sym from a stub slot named
+//   sym$auth_ptr$key$disc filled by dynamic linker during relocation resolving.
+//   This usually lowers to adrp+ldr, but also emits an entry into .data with an
+//   @AUTH relocation. This is done in LowerPtrAuthGlobalAddressStatically.
+//
+// All 3 are pseudos that are expand late to longer sequences: this lets us
+// provide integrity guarantees on the to-be-signed intermediate values.
+//
+// LOADauthptrstatic is undesirable because it requires a large section filled
+// with often similarly-signed pointers, making it a good harvesting target.
+// Thus, it's only used for ptrauth references to extern_weak to avoid null
+// checks.
+
+bool AArch64InstructionSelector::selectPtrAuthGlobalValue(
+    MachineInstr &I, MachineRegisterInfo &MRI) const {
+  Register DefReg = I.getOperand(0).getReg();
+  Register Addr = I.getOperand(1).getReg();
+  auto Key = static_cast<AArch64PACKey::ID>(I.getOperand(2).getImm());
+  Register AddrDisc = I.getOperand(3).getReg();
+  uint64_t Disc = I.getOperand(4).getImm();
+  uint64_t Offset = 0;
+
+  if (!MRI.hasOneDef(Addr))
+    return false;
+
+  // First match any offset we take from the real global.
+  const MachineInstr *DefMI = &*MRI.def_instr_begin(Addr);
+  if (DefMI->getOpcode() == TargetOpcode::G_PTR_ADD) {
+    Register OffsetReg = DefMI->getOperand(2).getReg();
+    if (!MRI.hasOneDef(OffsetReg))
+      return false;
+    const MachineInstr &OffsetMI = *MRI.def_instr_begin(OffsetReg);
+    if (OffsetMI.getOpcode() != TargetOpcode::G_CONSTANT)
+      return false;
+
+    Addr = DefMI->getOperand(1).getReg();
+    if (!MRI.hasOneDef(Addr))
+      return false;
+
+    DefMI = &*MRI.def_instr_begin(Addr);
+    Offset = OffsetMI.getOperand(1).getCImm()->getZExtValue();
+  }
+
+  // We should be left with a genuine unauthenticated GlobalValue.
+  const GlobalValue *GV = nullptr;
+  if (DefMI->getOpcode() == TargetOpcode::G_GLOBAL_VALUE) {
+    GV = DefMI->getOperand(1).getGlobal();
+    Offset += DefMI->getOperand(1).getOffset();
+  } else if (DefMI->getOpcode() == AArch64::G_ADD_LOW) {
+    GV = DefMI->getOperand(2).getGlobal();
+    Offset += DefMI->getOperand(2).getOffset();
+  } else {
+    return false;
+  }
+
+  MachineIRBuilder MIB(I);
+
+  // Classify the reference to determine whether it needs a GOT load.
+  unsigned OpFlags = STI.ClassifyGlobalReference(GV, TM);
+  const bool NeedsGOTLoad = ((OpFlags & AArch64II::MO_GOT) != 0);
+  assert(((OpFlags & (~AArch64II::MO_GOT)) == 0) &&
+         "Unsupported non-GOT op flags on ptrauth global reference");
+  assert((!GV->hasExternalWeakLinkage() || NeedsGOTLoad) &&
+         "Unsupported non-GOT reference to weak ptrauth global");
+
+  // None of our lowerings support an offset larger than 32-bit.
+  if (!isUInt<32>(Offset))
+    report_fatal_error("Unsupported >32-bit-wide offset in ptrauth global");
+
+  // Blend only works if the integer discriminator is 16-bit wide.
+  if (!isUInt<16>(Disc))
+    report_fatal_error(
+        "Unsupported >16-bit-wide constant discriminator in ptrauth global");
+
+  std::optional<APInt> AddrDiscVal = getIConstantVRegVal(AddrDisc, MRI);
+  bool HasAddrDisc = !AddrDiscVal || *AddrDiscVal != 0;
+
+  // Non-extern_weak:
+  // - No GOT load needed -> MOVaddrPAC
+  // - GOT load for non-extern_weak -> LOADgotPAC
+  //   Note that we disallow extern_weak refs to avoid null checks later.
+  if (!GV->hasExternalWeakLinkage()) {
+    MIB.buildInstr(TargetOpcode::IMPLICIT_DEF, {AArch64::X16}, {});
+    MIB.buildInstr(TargetOpcode::IMPLICIT_DEF, {AArch64::X17}, {});
+    MIB.buildInstr(NeedsGOTLoad ? AArch64::LOADgotPAC : AArch64::MOVaddrPAC)
+        .addGlobalAddress(GV, Offset)
+        .addImm(Key)
+        .addReg(HasAddrDisc ? AddrDisc : AArch64::XZR)
+        .addImm(Disc)
+        .constrainAllUses(TII, TRI, RBI);
+    MIB.buildCopy(DefReg, Register(AArch64::X16));
+    RBI.constrainGenericRegister(DefReg, AArch64::GPR64RegClass, MRI);
+    I.eraseFromParent();
+    return true;
+  }
+
+  // extern_weak -> LOADauthptrstatic
+
+  // Offsets and extern_weak don't mix well: ptrauth aside, you'd get the
+  // offset alone as a pointer if the symbol wasn't available, which probably
+  // breaking null checks in users.
+  // ptrauth complicates things further: error out.
+  if (Offset)
+    report_fatal_error("Unsupported offset in weak ptrauth global reference");
+
+  if (HasAddrDisc)
+    report_fatal_error("Unsupported weak addr-div ptrauth global");
+
+  MIB.buildInstr(AArch64::LOADauthptrstatic, {DefReg}, {})
+      .addGlobalAddress(GV, Offset)
+      .addImm(Key)
+      .addImm(Disc);
+  RBI.constrainGenericRegister(DefReg, AArch64::GPR64RegClass, MRI);
+
+  I.eraseFromParent();
+  return true;
+}
+
 void AArch64InstructionSelector::SelectTable(MachineInstr &I,
                                              MachineRegisterInfo &MRI,
                                              unsigned NumVec, unsigned Opc1,
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index 07a0473888ee5..c84e531fbed4d 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -743,6 +743,9 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
   else
     getActionDefinitionsBuilder(G_GLOBAL_VALUE).legalFor({p0});
 
+  getActionDefinitionsBuilder(G_PTRAUTH_GLOBAL_VALUE)
+      .legalIf(all(typeIs(0, p0), typeIs(1, p0)));
+
   getActionDefinitionsBuilder(G_PTRTOINT)
       .legalFor({{s64, p0}, {v2s64, v2p0}})
       .widenScalarToNextPow2(0, 64)
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
index d71111b57efe5..ec2ada500edea 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
@@ -86,6 +86,10 @@
 # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
 #
+# DEBUG-NEXT: G_PTRAUTH_GLOBAL_VALUE (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
+#
 # DEBUG-NEXT: G_CONSTANT_POOL (opcode {{[0-9]+}}): 1 type index, 0 imm indices
 # DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/ptrauth-constant-in-code.ll b/llvm/test/CodeGen/AArch64/GlobalISel/ptrauth-constant-in-code.ll
new file mode 100644
index 0000000000000..6861db15d1dd3
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/ptrauth-constant-in-code.ll
@@ -0,0 +1,118 @@
+; RUN: llc < %s -mtriple aarch64-elf -mattr=+pauth -global-isel \
+; RUN:   -verify-machineinstrs -global-isel-abort=1 | FileCheck %s
+
+ at g = external global i32
+ at g_weak = extern_weak global i32
+ at g_strong_def = dso_local constant i32 42
+
+define i8* @test_global_zero_disc() {
+; CHECK-LABEL: test_global_zero_disc:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp    x16, :got:g
+; CHECK-NEXT:    ldr     x16, [x16, :got_lo12:g]
+; CHECK-NEXT:    paciza  x16
+; CHECK-NEXT:    mov     x0, x16
+; CHECK-NEXT:    ret
+
+  ret ptr ptrauth (ptr @g, i32 0)
+}
+
+define i8* @test_global_offset_zero_disc() {
+; CHECK-LABEL: test_global_offset_zero_disc:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp    x16, :got:g
+; CHECK-NEXT:    ldr     x16, [x16, :got_lo12:g]
+; CHECK-NEXT:    add     x16, x16, #16
+; CHECK-NEXT:    pacdza  x16
+; CHECK-NEXT:    mov     x0, x16
+; CHECK-NEXT:    ret
+
+  ret ptr ptrauth (i8* getelementptr (i8, ptr @g, i64 16), i32 2)
+}
+
+; For large offsets, materializing it can take up to 3 add instructions.
+; We limit the offset to 32-bits.  We theoretically could support up to
+; 64 bit offsets, but 32 bits Ought To Be Enough For Anybody.
+
+define i8* @test_global_big_offset_zero_disc() {
+; CHECK-LABEL: test_global_big_offset_zero_disc:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp    x16, :got:g
+; CHECK-NEXT:    ldr     x16, [x16, :got_lo12:g]
+; CHECK-NEXT:    add     x16, x16, #1
+; CHECK-NEXT:    add     x16, x16, #16, lsl #12          // =65536
+; CHECK-NEXT:    add     x16, x16, #128, lsl #24         // =2147483648
+; CHECK-NEXT:    pacdza  x16
+; CHECK-NEXT:    mov     x0, x16
+; CHECK-NEXT:    ret
+
+  ret ptr ptrauth (i8* getelementptr (i8, ptr @g, i64 add (i64 2147483648, i64 65537)), i32 2)
+}
+
+define i8* @test_global_disc() {
+; CHECK-LABEL: test_global_disc:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp    x16, :got:g
+; CHECK-NEXT:    ldr     x16, [x16, :got_lo12:g]
+; CHECK-NEXT:    mov     x17, #42                        // =0x2a
+; CHECK-NEXT:    pacia   x16, x17
+; CHECK-NEXT:    mov     x0, x16
+; CHECK-NEXT:    ret
+
+  ret ptr ptrauth (ptr @g, i32 0, i64 42)
+}
+
+ at g.ref.da.42.addr = dso_local constant ptr ptrauth (ptr @g, i32 2, i64 42, ptr @g.ref.da.42.addr)
+
+define i8* @test_global_addr_disc() {
+; CHECK-LABEL: test_global_addr_disc:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, g.ref.da.42.addr
+; CHECK-NEXT:    add x8, x8, :lo12:g.ref.da.42.addr
+; CHECK-NEXT:    adrp x16, :got:g
+; CHECK-NEXT:    ldr x16, [x16, :got_lo12:g]
+; CHECK-NEXT:    mov x17, x8
+; CHECK-NEXT:    movk x17, #42, lsl #48
+; CHECK-NEXT:    pacda x16, x17
+; CHECK-NEXT:    mov x0, x16
+; CHECK-NEXT:    ret
+
+  ret ptr ptrauth (ptr @g, i32 2, i64 42, ptr @g.ref.da.42.addr)
+}
+
+define i8* @test_global_process_specific() {
+; CHECK-LABEL: test_global_process_specific:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp    x16, :got:g
+; CHECK-NEXT:    ldr     x16, [x16, :got_lo12:g]
+; CHECK-NEXT:    pacizb  x16
+; CHECK-NEXT:    mov     x0, x16
+; CHECK-NEXT:    ret
+  ret ptr ptrauth (ptr @g, i32 1)
+}
+
+; weak symbols can't be assumed to be non-nil. Use $auth_ptr$ stub slot always.
+; The alternative is to emit a null-check here, but that'd be redundant with
+; whatever null-check follows in user code.
+
+define i8* @test_global_weak() {
+; CHECK-LABEL: test_global_weak:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp    x0, g_weak$auth_ptr$ia$42
+; CHECK-NEXT:    ldr     x0, [x0, :lo12:g_weak$auth_ptr$ia$42]
+; CHECK-NEXT:    ret
+  ret ptr ptrauth (ptr @g_weak, i32 0, i64 42)
+}
+
+; Non-external symbols don't need to be accessed through the GOT.
+
+define i8* @test_global_strong_def() {
+; CHECK-LABEL: test_global_strong_def:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp    x16, g_strong_def
+; CHECK-NEXT:    add     x16, x16, :lo12:g_strong_def
+; CHECK-NEXT:    pacdza  x16
+; CHECK-NEXT:    mov     x0, x16
+; CHECK-NEXT:    ret
+  ret ptr ptrauth (ptr @g_strong_def, i32 2)
+}
diff --git a/llvm/test/CodeGen/AArch64/ptrauth-constant-in-code.ll b/llvm/test/CodeGen/AArch64/ptrauth-constant-in-code.ll
new file mode 100644
index 0000000000000..320dac757458a
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/ptrauth-constant-in-code.ll
@@ -0,0 +1,117 @@
+; RUN: llc < %s -mtriple aarch64-elf -mattr=+pauth | FileCheck %s
+
+ at g = external global i32
+ at g_weak = extern_weak global i32
+ at g_strong_def = dso_local constant i32 42
+
+define i8* @test_global_zero_disc() {
+; CHECK-LABEL: test_global_zero_disc:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp    x16, :got:g
+; CHECK-NEXT:    ldr     x16, [x16, :got_lo12:g]
+; CHECK-NEXT:    paciza  x16
+; CHECK-NEXT:    mov     x0, x16
+; CHECK-NEXT:    ret
+
+  ret ptr ptrauth (ptr @g, i32 0)
+}
+
+define i8* @test_global_offset_zero_disc() {
+; CHECK-LABEL: test_global_offset_zero_disc:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp    x16, :got:g
+; CHECK-NEXT:    ldr     x16, [x16, :got_lo12:g]
+; CHECK-NEXT:    add     x16, x16, #16
+; CHECK-NEXT:    pacdza  x16
+; CHECK-NEXT:    mov     x0, x16
+; CHECK-NEXT:    ret
+
+  ret ptr ptrauth (i8* getelementptr (i8, ptr @g, i64 16), i32 2)
+}
+
+; For large offsets, materializing it can take up to 3 add instructions.
+; We limit the offset to 32-bits.  We theoretically could support up to
+; 64 bit offsets, but 32 bits Ought To Be Enough For Anybody.
+
+define i8* @test_global_big_offset_zero_disc() {
+; CHECK-LABEL: test_global_big_offset_zero_disc:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp    x16, :got:g
+; CHECK-NEXT:    ldr     x16, [x16, :got_lo12:g]
+; CHECK-NEXT:    add     x16, x16, #1
+; CHECK-NEXT:    add     x16, x16, #16, lsl #12          // =65536
+; CHECK-NEXT:    add     x16, x16, #128, lsl #24         // =2147483648
+; CHECK-NEXT:    pacdza  x16
+; CHECK-NEXT:    mov     x0, x16
+; CHECK-NEXT:    ret
+
+  ret ptr ptrauth (i8* getelementptr (i8, ptr @g, i64 add (i64 2147483648, i64 65537)), i32 2)
+}
+
+define i8* @test_global_disc() {
+; CHECK-LABEL: test_global_disc:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp    x16, :got:g
+; CHECK-NEXT:    ldr     x16, [x16, :got_lo12:g]
+; CHECK-NEXT:    mov     x17, #42                        // =0x2a
+; CHECK-NEXT:    pacia   x16, x17
+; CHECK-NEXT:    mov     x0, x16
+; CHECK-NEXT:    ret
+
+  ret ptr ptrauth (ptr @g, i32 0, i64 42)
+}
+
+ at g.ref.da.42.addr = dso_local constant ptr ptrauth (ptr @g, i32 2, i64 42, ptr @g.ref.da.42.addr)
+
+define i8* @test_global_addr_disc() {
+; CHECK-LABEL: test_global_addr_disc:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, g.ref.da.42.addr
+; CHECK-NEXT:    add x8, x8, :lo12:g.ref.da.42.addr
+; CHECK-NEXT:    adrp x16, :got:g
+; CHECK-NEXT:    ldr x16, [x16, :got_lo12:g]
+; CHECK-NEXT:    mov x17, x8
+; CHECK-NEXT:    movk x17, #42, lsl #48
+; CHECK-NEXT:    pacda x16, x17
+; CHECK-NEXT:    mov x0, x16
+; CHECK-NEXT:    ret
+
+  ret ptr ptrauth (ptr @g, i32 2, i64 42, ptr @g.ref.da.42.addr)
+}
+
+define i8* @test_global_process_specific() {
+; CHECK-LABEL: test_global_process_specific:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp    x16, :got:g
+; CHECK-NEXT:    ldr     x16, [x16, :got_lo12:g]
+; CHECK-NEXT:    pacizb  x16
+; CHECK-NEXT:    mov     x0, x16
+; CHECK-NEXT:    ret
+  ret ptr ptrauth (ptr @g, i32 1)
+}
+
+; weak symbols can't be assumed to be non-nil. Use $auth_ptr$ stub slot always.
+; The alternative is to emit a null-check here, but that'd be redundant with
+; whatever null-check follows in user code.
+
+define i8* @test_global_weak() {
+; CHECK-LABEL: test_global_weak:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp    x0, g_weak$auth_ptr$ia$42
+; CHECK-NEXT:    ldr     x0, [x0, :lo12:g_weak$auth_ptr$ia$42]
+; CHECK-NEXT:    ret
+  ret ptr ptrauth (ptr @g_weak, i32 0, i64 42)
+}
+
+; Non-external symbols don't need to be accessed through the GOT.
+
+define i8* @test_global_strong_def() {
+; CHECK-LABEL: test_global_strong_def:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp    x16, g_strong_def
+; CHECK-NEXT:    add     x16, x16, :lo12:g_strong_def
+; CHECK-NEXT:    pacdza  x16
+; CHECK-NEXT:    mov     x0, x16
+; CHECK-NEXT:    ret
+  ret ptr ptrauth (ptr @g_strong_def, i32 2)
+}
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-cxx.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-cxx.td
index bb1acde296429..2825ade2d2134 100644
--- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-cxx.td
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-cxx.td
@@ -85,7 +85,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 
 // CHECK:      const uint8_t *GenMyCombiner::getMatchTable() const {
 // CHECK-NEXT:   constexpr static uint8_t MatchTable0[] = {
-// CHECK-NEXT:     GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(94), GIMT_Encode2(194), /*)*//*default:*//*Label 4*/ GIMT_Encode4(464),
+// CHECK-NEXT:     GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(95), GIMT_Encode2(195), /*)*//*default:*//*Label 4*/ GIMT_Encode4(464),
 // CHECK-NEXT:     /*TargetOpcode::G_STORE*//*Label 0*/ GIMT_Encode4(410), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
 // CHECK-NEXT:     /*TargetOpcode::G_SEXT*//*Label 1*/ GIMT_Encode4(428), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
 // CHECK-NEXT:     /*TargetOpcode::G_FNEG*//*Label 2*/ GIMT_Encode4(440), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-variadics.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-variadics.td
index 35bddf912a541..43cdd4f5546bb 100644
--- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-variadics.td
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-variadics.td
@@ -37,7 +37,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 
 // CHECK:      const uint8_t *GenMyCombiner::getMatchTable() const {
 // CHECK-NEXT:   constexpr static uint8_t MatchTable0[] = {
-// CHECK-NEXT:     GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(69), GIMT_Encode2(73), /*)*//*default:*//*Label 2*/ GIMT_Encode4(84),
+// CHECK-NEXT:     GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(70), GIMT_Encode2(74), /*)*//*default:*//*Label 2*/ GIMT_Encode4(84),
 // CHECK-NEXT:     /*TargetOpcode::G_UNMERGE_VALUES*//*Label 0*/ GIMT_Encode4(26), GIMT_Encode4(0), GIMT_Encode4(0),
 // CHECK-NEXT:     /*TargetOpcode::G_BUILD_VECTOR*//*Label 1*/ GIMT_Encode4(55),
 // CHECK-NEXT:     // Label 0: @26
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td
index 513a86754d6d9..7ff637fa1e0e3 100644
--- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td
@@ -135,15 +135,15 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // Verify match table.
 // CHECK:      const uint8_t *GenMyCombiner::getMatchTable() const {
 // CHECK-NEXT:   constexpr static uint8_t MatchTable0[] = {
-// CHECK-NEXT:     GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(19), GIMT_Encode2(133), /*)*//*default:*//*Label 6*/ GIMT_Encode4(653),
-// CHECK-NEXT:     /*TargetOpcode::COPY*//*Label 0*/ GIMT_Encode4(466), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
-// CHECK-NEXT:     /*TargetOpcode::G_AND*//*Label 1*/ GIMT_Encode4(502), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
-// CHECK-NEXT:     /*TargetOpcode::G_STORE*//*Label 2*/ GIMT_Encode4(549), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
-// CHECK-NEXT:     /*TargetOpcode::G_TRUNC*//*Label 3*/ GIMT_Encode4(583), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
-// CHECK-NEXT:     /*TargetOpcode::G_SEXT*//*Label 4*/ GIMT_Encode4(606), GIMT_Encode4(0),
-// CHECK-NEXT:     /*TargetOpcode::G_ZEXT*//*Label 5*/ GIMT_Encode4(618),
-// CHECK-NEXT:     // Label 0: @466
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 7*/ GIMT_Encode4(490), // Rule ID 4 //
+// CHECK-NEXT:     GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(19), GIMT_Encode2(134), /*)*//*default:*//*Label 6*/ GIMT_Encode4(657),
+// CHECK-NEXT:     /*TargetOpcode::COPY*//*Label 0*/ GIMT_Encode4(470), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
+// CHECK-NEXT:     /*TargetOpcode::G_AND*//*Label 1*/ GIMT_Encode4(506), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
+// CHECK-NEXT:     /*TargetOpcode::G_STORE*//*Label 2*/ GIMT_Encode4(553), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
+// CHECK-NEXT:     /*TargetOpcode::G_TRUNC*//*Label 3*/ GIMT_Encode4(587), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
+// CHECK-NEXT:     /*TargetOpcode::G_SEXT*//*Label 4*/ GIMT_Encode4(610), GIMT_Encode4(0),
+// CHECK-NEXT:     /*TargetOpcode::G_ZEXT*//*Label 5*/ GIMT_Encode4(622),
+// CHECK-NEXT:     // Label 0: @470
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 7*/ GIMT_Encode4(494), // Rule ID 4 //
 // CHECK-NEXT:       GIM_CheckFeatures, GIMT_Encode2(GIFBS_HasAnswerToEverything),
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule3Enabled),
 // CHECK-NEXT:       // MIs[0] a
@@ -156,8 +156,8 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       GIM_CheckIsSafeToFold, /*NumInsns*/1,
 // CHECK-NEXT:       // Combiner Rule #3: InstTest1
 // CHECK-NEXT:       GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner2),
-// CHECK-NEXT:     // Label 7: @490
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 8*/ GIMT_Encode4(501), // Rule ID 3 //
+// CHECK-NEXT:     // Label 7: @494
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 8*/ GIMT_Encode4(505), // Rule ID 3 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule2Enabled),
 // CHECK-NEXT:       // MIs[0] a
 // CHECK-NEXT:       // No operand predicates
@@ -165,10 +165,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       // Combiner Rule #2: InstTest0
 // CHECK-NEXT:       GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner1),
-// CHECK-NEXT:     // Label 8: @501
+// CHECK-NEXT:     // Label 8: @505
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 1: @502
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 9*/ GIMT_Encode4(548), // Rule ID 6 //
+// CHECK-NEXT:     // Label 1: @506
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 9*/ GIMT_Encode4(552), // Rule ID 6 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule5Enabled),
 // CHECK-NEXT:       GIM_RootCheckType, /*Op*/2, /*Type*/GILLT_s32,
 // CHECK-NEXT:       // MIs[0] dst
@@ -185,10 +185,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       GIR_RootToRootCopy, /*OpIdx*/0, // dst
 // CHECK-NEXT:       GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/1, // z
 // CHECK-NEXT:       GIR_EraseRootFromParent_Done,
-// CHECK-NEXT:     // Label 9: @548
+// CHECK-NEXT:     // Label 9: @552
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 2: @549
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 10*/ GIMT_Encode4(582), // Rule ID 5 //
+// CHECK-NEXT:     // Label 2: @553
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 10*/ GIMT_Encode4(586), // Rule ID 5 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule4Enabled),
 // CHECK-NEXT:       // MIs[0] tmp
 // CHECK-NEXT:       GIM_RecordInsnIgnoreCopies, /*DefineMI*/1, /*MI*/0, /*OpIdx*/0, // MIs[1]
@@ -204,29 +204,29 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       GIR_RootToRootCopy, /*OpIdx*/1, // ptr
 // CHECK-NEXT:       GIR_MergeMemOperands, /*InsnID*/0, /*NumInsns*/2, /*MergeInsnID's*/0, 1,
 // CHECK-NEXT:       GIR_EraseRootFromParent_Done,
-// CHECK-NEXT:     // Label 10: @582
+// CHECK-NEXT:     // Label 10: @586
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 3: @583
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 11*/ GIMT_Encode4(594), // Rule ID 0 //
+// CHECK-NEXT:     // Label 3: @587
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 11*/ GIMT_Encode4(598), // Rule ID 0 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled),
 // CHECK-NEXT:       // Combiner Rule #0: WipOpcodeTest0; wip_match_opcode 'G_TRUNC'
 // CHECK-NEXT:       GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner0),
-// CHECK-NEXT:     // Label 11: @594
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 12*/ GIMT_Encode4(605), // Rule ID 1 //
+// CHECK-NEXT:     // Label 11: @598
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 12*/ GIMT_Encode4(609), // Rule ID 1 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule1Enabled),
 // CHECK-NEXT:       // Combiner Rule #1: WipOpcodeTest1; wip_match_opcode 'G_TRUNC'
 // CHECK-NEXT:       GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner0),
-// CHECK-NEXT:     // Label 12: @605
+// CHECK-NEXT:     // Label 12: @609
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 4: @606
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 13*/ GIMT_Encode4(617), // Rule ID 2 //
+// CHECK-NEXT:     // Label 4: @610
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 13*/ GIMT_Encode4(621), // Rule ID 2 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule1Enabled),
 // CHECK-NEXT:       // Combiner Rule #1: WipOpcodeTest1; wip_match_opcode 'G_SEXT'
 // CHECK-NEXT:       GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner0),
-// CHECK-NEXT:     // Label 13: @617
+// CHECK-NEXT:     // Label 13: @621
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 5: @618
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 14*/ GIMT_Encode4(652), // Rule ID 7 //
+// CHECK-NEXT:     // Label 5: @622
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 14*/ GIMT_Encode4(656), // Rule ID 7 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule6Enabled),
 // CHECK-NEXT:       // MIs[0] dst
 // CHECK-NEXT:       // No operand predicates
@@ -240,10 +240,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       GIR_RootToRootCopy, /*OpIdx*/0, // dst
 // CHECK-NEXT:       GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0,
 // CHECK-NEXT:       GIR_EraseRootFromParent_Done,
-// CHECK-NEXT:     // Label 14: @652
+// CHECK-NEXT:     // Label 14: @656
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 6: @653
+// CHECK-NEXT:     // Label 6: @657
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     }; // Size: 654 bytes
+// CHECK-NEXT:     }; // Size: 658 bytes
 // CHECK-NEXT:   return MatchTable0;
 // CHECK-NEXT: }
diff --git a/llvm/test/TableGen/GlobalISelEmitter.td b/llvm/test/TableGen/GlobalISelEmitter.td
index 5d5bf92664a79..796f595930319 100644
--- a/llvm/test/TableGen/GlobalISelEmitter.td
+++ b/llvm/test/TableGen/GlobalISelEmitter.td
@@ -513,7 +513,7 @@ def : Pat<(frag GPR32:$src1, complex:$src2, complex:$src3),
 // R00O-NEXT:  GIM_Reject,
 // R00O:       // Label [[DEFAULT_NUM]]: @[[DEFAULT]]
 // R00O-NEXT:  GIM_Reject,
-// R00O-NEXT:  }; // Size: 1804 bytes
+// R00O-NEXT:  }; // Size: 1808 bytes
 
 def INSNBOB : I<(outs GPR32:$dst), (ins GPR32:$src1, GPR32:$src2, GPR32:$src3, GPR32:$src4),
                  [(set GPR32:$dst,



More information about the llvm-commits mailing list