[llvm] cc548ec - [AArch64][PAC] Lower authenticated calls with ptrauth bundles. (#85736)
via llvm-commits
llvm-commits at lists.llvm.org
Fri May 31 14:08:14 PDT 2024
Author: Ahmed Bougacha
Date: 2024-05-31T14:08:10-07:00
New Revision: cc548ec47c05971a67ed9ec7086d414aabcf8b05
URL: https://github.com/llvm/llvm-project/commit/cc548ec47c05971a67ed9ec7086d414aabcf8b05
DIFF: https://github.com/llvm/llvm-project/commit/cc548ec47c05971a67ed9ec7086d414aabcf8b05.diff
LOG: [AArch64][PAC] Lower authenticated calls with ptrauth bundles. (#85736)
This adds codegen support for the "ptrauth" operand bundles, which can
be used to augment indirect calls with the equivalent of an
`@llvm.ptrauth.auth` intrinsic call on the call target (possibly
preceded by an `@llvm.ptrauth.blend` on the auth discriminator if
applicable.)
This allows the generation of combined authenticating calls
on AArch64 (in the BLRA* PAuth instructions), while avoiding
the raw just-authenticated function pointer from being
exposed to attackers.
This is done by threading a PtrAuthInfo descriptor through
the call lowering infrastructure, eventually selecting a BLRA
pseudo. The pseudo encapsulates the safe discriminator
computation, which together with the real BLRA* call get emitted
in late pseudo expansion in AsmPrinter.
Note that this also applies to the other forms of indirect calls,
notably invokes, rvmarker, and tail calls. Tail-calls in particular
bring some additional complexity, with the intersecting register
constraints of BTI and PAC discriminator computation.
However this doesn't currently support PAuth_LR tail-call variants.
This also adopts an x8+ allocation order for GPR64noip, matching
GPR64.
Added:
llvm/test/CodeGen/AArch64/ptrauth-bti-call.ll
llvm/test/CodeGen/AArch64/ptrauth-call-rv-marker.ll
llvm/test/CodeGen/AArch64/ptrauth-call.ll
llvm/test/CodeGen/AArch64/ptrauth-invoke.ll
Modified:
llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h
llvm/include/llvm/CodeGen/TargetLowering.h
llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
llvm/lib/Target/AArch64/AArch64ISelLowering.h
llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
llvm/lib/Target/AArch64/AArch64InstrInfo.td
llvm/lib/Target/AArch64/AArch64RegisterInfo.td
llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
llvm/lib/Target/AArch64/GISel/AArch64GlobalISelUtils.cpp
llvm/lib/Target/AArch64/GISel/AArch64GlobalISelUtils.h
llvm/test/CodeGen/AArch64/branch-target-enforcement-indirect-calls.ll
Removed:
################################################################################
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h b/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h
index 4c187a3068d82..bb3be3e2b4f98 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h
@@ -99,6 +99,11 @@ class CallLowering {
ArgInfo() = default;
};
+ struct PtrAuthInfo {
+ uint64_t Key;
+ Register Discriminator;
+ };
+
struct CallLoweringInfo {
/// Calling convention to be used for the call.
CallingConv::ID CallConv = CallingConv::C;
@@ -125,6 +130,9 @@ class CallLowering {
MDNode *KnownCallees = nullptr;
+ /// The auth-call information in the "ptrauth" bundle, if present.
+ std::optional<PtrAuthInfo> PAI;
+
/// True if the call must be tail call optimized.
bool IsMustTailCall = false;
@@ -587,7 +595,7 @@ class CallLowering {
bool lowerCall(MachineIRBuilder &MIRBuilder, const CallBase &Call,
ArrayRef<Register> ResRegs,
ArrayRef<ArrayRef<Register>> ArgRegs, Register SwiftErrorVReg,
- Register ConvergenceCtrlToken,
+ std::optional<PtrAuthInfo> PAI, Register ConvergenceCtrlToken,
std::function<unsigned()> GetCalleeReg) const;
/// For targets which want to use big-endian can enable it with
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 2765caec960c9..d1912b1c4c0f6 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -4367,6 +4367,9 @@ class TargetLowering : public TargetLoweringBase {
/// Return true if the target supports kcfi operand bundles.
virtual bool supportKCFIBundles() const { return false; }
+ /// Return true if the target supports ptrauth operand bundles.
+ virtual bool supportPtrAuthBundles() const { return false; }
+
/// Perform necessary initialization to handle a subset of CSRs explicitly
/// via copies. This function is called at the beginning of instruction
/// selection.
@@ -4478,6 +4481,14 @@ class TargetLowering : public TargetLoweringBase {
llvm_unreachable("Not Implemented");
}
+ /// This structure contains the information necessary for lowering
+ /// pointer-authenticating indirect calls. It is equivalent to the "ptrauth"
+ /// operand bundle found on the call instruction, if any.
+ struct PtrAuthInfo {
+ uint64_t Key;
+ SDValue Discriminator;
+ };
+
/// This structure contains all information that is necessary for lowering
/// calls. It is passed to TLI::LowerCallTo when the SelectionDAG builder
/// needs to lower a call, and targets will see this struct in their LowerCall
@@ -4517,6 +4528,8 @@ class TargetLowering : public TargetLoweringBase {
const ConstantInt *CFIType = nullptr;
SDValue ConvergenceControlToken;
+ std::optional<PtrAuthInfo> PAI;
+
CallLoweringInfo(SelectionDAG &DAG)
: RetSExt(false), RetZExt(false), IsVarArg(false), IsInReg(false),
DoesNotReturn(false), IsReturnValueUsed(true), IsConvergent(false),
@@ -4639,6 +4652,11 @@ class TargetLowering : public TargetLoweringBase {
return *this;
}
+ CallLoweringInfo &setPtrAuth(PtrAuthInfo Value) {
+ PAI = Value;
+ return *this;
+ }
+
CallLoweringInfo &setIsPostTypeLegalization(bool Value=true) {
IsPostTypeLegalization = Value;
return *this;
diff --git a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
index 363fad53b76c3..412cd0a21ad41 100644
--- a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
@@ -92,6 +92,7 @@ bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, const CallBase &CB,
ArrayRef<Register> ResRegs,
ArrayRef<ArrayRef<Register>> ArgRegs,
Register SwiftErrorVReg,
+ std::optional<PtrAuthInfo> PAI,
Register ConvergenceCtrlToken,
std::function<unsigned()> GetCalleeReg) const {
CallLoweringInfo Info;
@@ -188,6 +189,7 @@ bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, const CallBase &CB,
Info.KnownCallees = CB.getMetadata(LLVMContext::MD_callees);
Info.CallConv = CallConv;
Info.SwiftErrorVReg = SwiftErrorVReg;
+ Info.PAI = PAI;
Info.ConvergenceCtrlToken = ConvergenceCtrlToken;
Info.IsMustTailCall = CB.isMustTailCall();
Info.IsTailCall = CanBeTailCalled;
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 9fc9aa354b0db..67c51f018a807 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -2644,6 +2644,20 @@ bool IRTranslator::translateCallBase(const CallBase &CB,
}
}
+ std::optional<CallLowering::PtrAuthInfo> PAI;
+ if (CB.countOperandBundlesOfType(LLVMContext::OB_ptrauth)) {
+ // Functions should never be ptrauth-called directly.
+ assert(!CB.getCalledFunction() && "invalid direct ptrauth call");
+
+ auto PAB = CB.getOperandBundle("ptrauth");
+ const Value *Key = PAB->Inputs[0];
+ const Value *Discriminator = PAB->Inputs[1];
+
+ Register DiscReg = getOrCreateVReg(*Discriminator);
+ PAI = CallLowering::PtrAuthInfo{cast<ConstantInt>(Key)->getZExtValue(),
+ DiscReg};
+ }
+
Register ConvergenceCtrlToken = 0;
if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_convergencectrl)) {
const auto &Token = *Bundle->Inputs[0].get();
@@ -2654,7 +2668,7 @@ bool IRTranslator::translateCallBase(const CallBase &CB,
// optimize into tail calls. Instead, we defer that to selection where a final
// scan is done to check if any instructions are calls.
bool Success = CLI->lowerCall(
- MIRBuilder, CB, Res, Args, SwiftErrorVReg, ConvergenceCtrlToken,
+ MIRBuilder, CB, Res, Args, SwiftErrorVReg, PAI, ConvergenceCtrlToken,
[&]() { return getOrCreateVReg(*CB.getCalledOperand()); });
// Check if we just inserted a tail call.
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 85e4cc3b82e6e..c7e0c62ec1263 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -3307,12 +3307,12 @@ void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
const BasicBlock *EHPadBB = I.getSuccessor(1);
MachineBasicBlock *EHPadMBB = FuncInfo.MBBMap[EHPadBB];
- // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
+ // Deopt and ptrauth bundles are lowered in helper functions, and we don't
// have to do anything here to lower funclet bundles.
assert(!I.hasOperandBundlesOtherThan(
{LLVMContext::OB_deopt, LLVMContext::OB_gc_transition,
LLVMContext::OB_gc_live, LLVMContext::OB_funclet,
- LLVMContext::OB_cfguardtarget,
+ LLVMContext::OB_cfguardtarget, LLVMContext::OB_ptrauth,
LLVMContext::OB_clang_arc_attachedcall}) &&
"Cannot lower invokes with arbitrary operand bundles yet!");
@@ -3363,6 +3363,8 @@ void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
// intrinsic, and right now there are no plans to support other intrinsics
// with deopt state.
LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);
+ } else if (I.countOperandBundlesOfType(LLVMContext::OB_ptrauth)) {
+ LowerCallSiteWithPtrAuthBundle(cast<CallBase>(I), EHPadBB);
} else {
LowerCallTo(I, getValue(Callee), false, false, EHPadBB);
}
@@ -8598,9 +8600,9 @@ SelectionDAGBuilder::lowerInvokable(TargetLowering::CallLoweringInfo &CLI,
}
void SelectionDAGBuilder::LowerCallTo(const CallBase &CB, SDValue Callee,
- bool isTailCall,
- bool isMustTailCall,
- const BasicBlock *EHPadBB) {
+ bool isTailCall, bool isMustTailCall,
+ const BasicBlock *EHPadBB,
+ const TargetLowering::PtrAuthInfo *PAI) {
auto &DL = DAG.getDataLayout();
FunctionType *FTy = CB.getFunctionType();
Type *RetTy = CB.getType();
@@ -8707,6 +8709,15 @@ void SelectionDAGBuilder::LowerCallTo(const CallBase &CB, SDValue Callee,
CB.countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0)
.setCFIType(CFIType)
.setConvergenceControlToken(ConvControlToken);
+
+ // Set the pointer authentication info if we have it.
+ if (PAI) {
+ if (!TLI.supportPtrAuthBundles())
+ report_fatal_error(
+ "This target doesn't support calls with ptrauth operand bundles.");
+ CLI.setPtrAuth(*PAI);
+ }
+
std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
if (Result.first.getNode()) {
@@ -9252,6 +9263,11 @@ void SelectionDAGBuilder::visitCall(const CallInst &I) {
}
}
+ if (I.countOperandBundlesOfType(LLVMContext::OB_ptrauth)) {
+ LowerCallSiteWithPtrAuthBundle(cast<CallBase>(I), /*EHPadBB=*/nullptr);
+ return;
+ }
+
// Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
// have to do anything here to lower funclet bundles.
// CFGuardTarget bundles are lowered in LowerCallTo.
@@ -9273,6 +9289,31 @@ void SelectionDAGBuilder::visitCall(const CallInst &I) {
LowerCallTo(I, Callee, I.isTailCall(), I.isMustTailCall());
}
+void SelectionDAGBuilder::LowerCallSiteWithPtrAuthBundle(
+ const CallBase &CB, const BasicBlock *EHPadBB) {
+ auto PAB = CB.getOperandBundle("ptrauth");
+ const Value *CalleeV = CB.getCalledOperand();
+
+ // Gather the call ptrauth data from the operand bundle:
+ // [ i32 <key>, i64 <discriminator> ]
+ const auto *Key = cast<ConstantInt>(PAB->Inputs[0]);
+ const Value *Discriminator = PAB->Inputs[1];
+
+ assert(Key->getType()->isIntegerTy(32) && "Invalid ptrauth key");
+ assert(Discriminator->getType()->isIntegerTy(64) &&
+ "Invalid ptrauth discriminator");
+
+ // Functions should never be ptrauth-called directly.
+ assert(!isa<Function>(CalleeV) && "invalid direct ptrauth call");
+
+ // Otherwise, do an authenticated indirect call.
+ TargetLowering::PtrAuthInfo PAI = {Key->getZExtValue(),
+ getValue(Discriminator)};
+
+ LowerCallTo(CB, getValue(CalleeV), CB.isTailCall(), CB.isMustTailCall(),
+ EHPadBB, &PAI);
+}
+
namespace {
/// AsmOperandInfo - This contains information for each constraint that we are
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
index ae361f8c500a0..6cb2c78a547ac 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
@@ -406,7 +406,8 @@ class SelectionDAGBuilder {
void CopyToExportRegsIfNeeded(const Value *V);
void ExportFromCurrentBlock(const Value *V);
void LowerCallTo(const CallBase &CB, SDValue Callee, bool IsTailCall,
- bool IsMustTailCall, const BasicBlock *EHPadBB = nullptr);
+ bool IsMustTailCall, const BasicBlock *EHPadBB = nullptr,
+ const TargetLowering::PtrAuthInfo *PAI = nullptr);
// Lower range metadata from 0 to N to assert zext to an integer of nearest
// floor power of two.
@@ -490,6 +491,9 @@ class SelectionDAGBuilder {
bool VarArgDisallowed,
bool ForceVoidReturnTy);
+ void LowerCallSiteWithPtrAuthBundle(const CallBase &CB,
+ const BasicBlock *EHPadBB);
+
/// Returns the type of FrameIndex and TargetFrameIndex nodes.
MVT getFrameIndexTy() {
return DAG.getTargetLoweringInfo().getFrameIndexTy(DAG.getDataLayout());
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index bdc3fc630a4e3..7da540f8ef8e5 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -125,6 +125,12 @@ class AArch64AsmPrinter : public AsmPrinter {
void emitSled(const MachineInstr &MI, SledKind Kind);
+ // Emit the sequence for BLRA (authenticate + branch).
+ void emitPtrauthBranch(const MachineInstr *MI);
+ // Emit the sequence to compute a discriminator into x17, or reuse AddrDisc.
+ unsigned emitPtrauthDiscriminator(uint16_t Disc, unsigned AddrDisc,
+ unsigned &InstsEmitted);
+
/// tblgen'erated driver function for lowering simple MI->MC
/// pseudo instructions.
bool emitPseudoExpansionLowering(MCStreamer &OutStreamer,
@@ -1497,6 +1503,78 @@ void AArch64AsmPrinter::emitFMov0(const MachineInstr &MI) {
}
}
+unsigned AArch64AsmPrinter::emitPtrauthDiscriminator(uint16_t Disc,
+ unsigned AddrDisc,
+ unsigned &InstsEmitted) {
+ // So far we've used NoRegister in pseudos. Now we need real encodings.
+ if (AddrDisc == AArch64::NoRegister)
+ AddrDisc = AArch64::XZR;
+
+ // If there is no constant discriminator, there's no blend involved:
+ // just use the address discriminator register as-is (XZR or not).
+ if (!Disc)
+ return AddrDisc;
+
+ // If there's only a constant discriminator, MOV it into x17.
+ if (AddrDisc == AArch64::XZR) {
+ EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::MOVZXi)
+ .addReg(AArch64::X17)
+ .addImm(Disc)
+ .addImm(/*shift=*/0));
+ ++InstsEmitted;
+ return AArch64::X17;
+ }
+
+ // If there are both, emit a blend into x17.
+ EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ORRXrs)
+ .addReg(AArch64::X17)
+ .addReg(AArch64::XZR)
+ .addReg(AddrDisc)
+ .addImm(0));
+ ++InstsEmitted;
+ EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::MOVKXi)
+ .addReg(AArch64::X17)
+ .addReg(AArch64::X17)
+ .addImm(Disc)
+ .addImm(/*shift=*/48));
+ ++InstsEmitted;
+ return AArch64::X17;
+}
+
+void AArch64AsmPrinter::emitPtrauthBranch(const MachineInstr *MI) {
+ unsigned InstsEmitted = 0;
+ unsigned BrTarget = MI->getOperand(0).getReg();
+
+ auto Key = (AArch64PACKey::ID)MI->getOperand(1).getImm();
+ assert((Key == AArch64PACKey::IA || Key == AArch64PACKey::IB) &&
+ "Invalid auth call key");
+
+ uint64_t Disc = MI->getOperand(2).getImm();
+ assert(isUInt<16>(Disc));
+
+ unsigned AddrDisc = MI->getOperand(3).getReg();
+
+ // Compute discriminator into x17
+ unsigned DiscReg = emitPtrauthDiscriminator(Disc, AddrDisc, InstsEmitted);
+ bool IsZeroDisc = DiscReg == AArch64::XZR;
+
+ unsigned Opc;
+ if (Key == AArch64PACKey::IA)
+ Opc = IsZeroDisc ? AArch64::BLRAAZ : AArch64::BLRAA;
+ else
+ Opc = IsZeroDisc ? AArch64::BLRABZ : AArch64::BLRAB;
+
+ MCInst BRInst;
+ BRInst.setOpcode(Opc);
+ BRInst.addOperand(MCOperand::createReg(BrTarget));
+ if (!IsZeroDisc)
+ BRInst.addOperand(MCOperand::createReg(DiscReg));
+ EmitToStreamer(*OutStreamer, BRInst);
+ ++InstsEmitted;
+
+ assert(STI->getInstrInfo()->getInstSizeInBytes(*MI) >= InstsEmitted * 4);
+}
+
// Simple pseudo-instructions have their lowering (with expansion to real
// instructions) auto-generated.
#include "AArch64GenMCPseudoLowering.inc"
@@ -1632,9 +1710,63 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
return;
}
+ case AArch64::BLRA:
+ emitPtrauthBranch(MI);
+ return;
+
// Tail calls use pseudo instructions so they have the proper code-gen
// attributes (isCall, isReturn, etc.). We lower them to the real
// instruction here.
+ case AArch64::AUTH_TCRETURN:
+ case AArch64::AUTH_TCRETURN_BTI: {
+ const uint64_t Key = MI->getOperand(2).getImm();
+ assert((Key == AArch64PACKey::IA || Key == AArch64PACKey::IB) &&
+ "Invalid auth key for tail-call return");
+
+ const uint64_t Disc = MI->getOperand(3).getImm();
+ assert(isUInt<16>(Disc) && "Integer discriminator is too wide");
+
+ Register AddrDisc = MI->getOperand(4).getReg();
+
+ Register ScratchReg = MI->getOperand(0).getReg() == AArch64::X16
+ ? AArch64::X17
+ : AArch64::X16;
+
+ unsigned DiscReg = AddrDisc;
+ if (Disc) {
+ if (AddrDisc != AArch64::NoRegister) {
+ EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ORRXrs)
+ .addReg(ScratchReg)
+ .addReg(AArch64::XZR)
+ .addReg(AddrDisc)
+ .addImm(0));
+ EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::MOVKXi)
+ .addReg(ScratchReg)
+ .addReg(ScratchReg)
+ .addImm(Disc)
+ .addImm(/*shift=*/48));
+ } else {
+ EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::MOVZXi)
+ .addReg(ScratchReg)
+ .addImm(Disc)
+ .addImm(/*shift=*/0));
+ }
+ DiscReg = ScratchReg;
+ }
+
+ const bool IsZero = DiscReg == AArch64::NoRegister;
+ const unsigned Opcodes[2][2] = {{AArch64::BRAA, AArch64::BRAAZ},
+ {AArch64::BRAB, AArch64::BRABZ}};
+
+ MCInst TmpInst;
+ TmpInst.setOpcode(Opcodes[Key][IsZero]);
+ TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
+ if (!IsZero)
+ TmpInst.addOperand(MCOperand::createReg(DiscReg));
+ EmitToStreamer(*OutStreamer, TmpInst);
+ return;
+ }
+
case AArch64::TCRETURNri:
case AArch64::TCRETURNrix16x17:
case AArch64::TCRETURNrix17:
diff --git a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
index 36957bb0f5a05..9b7fc228d5de8 100644
--- a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
@@ -787,26 +787,24 @@ bool AArch64ExpandPseudo::expandSVESpillFill(MachineBasicBlock &MBB,
return true;
}
-// Create a call to CallTarget, copying over all the operands from *MBBI,
-// starting at the regmask.
-static MachineInstr *createCall(MachineBasicBlock &MBB,
- MachineBasicBlock::iterator MBBI,
- const AArch64InstrInfo *TII,
- MachineOperand &CallTarget,
- unsigned RegMaskStartIdx) {
- unsigned Opc = CallTarget.isGlobal() ? AArch64::BL : AArch64::BLR;
- MachineInstr *Call =
- BuildMI(MBB, MBBI, MBBI->getDebugLoc(), TII->get(Opc)).getInstr();
-
- assert((CallTarget.isGlobal() || CallTarget.isReg()) &&
- "invalid operand for regular call");
- Call->addOperand(CallTarget);
+// Create a call with the passed opcode and explicit operands, copying over all
+// the implicit operands from *MBBI, starting at the regmask.
+static MachineInstr *createCallWithOps(MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator MBBI,
+ const AArch64InstrInfo *TII,
+ unsigned Opcode,
+ ArrayRef<MachineOperand> ExplicitOps,
+ unsigned RegMaskStartIdx) {
+ // Build the MI, with explicit operands first (including the call target).
+ MachineInstr *Call = BuildMI(MBB, MBBI, MBBI->getDebugLoc(), TII->get(Opcode))
+ .add(ExplicitOps)
+ .getInstr();
// Register arguments are added during ISel, but cannot be added as explicit
// operands of the branch as it expects to be B <target> which is only one
// operand. Instead they are implicit operands used by the branch.
while (!MBBI->getOperand(RegMaskStartIdx).isRegMask()) {
- auto MOP = MBBI->getOperand(RegMaskStartIdx);
+ const MachineOperand &MOP = MBBI->getOperand(RegMaskStartIdx);
assert(MOP.isReg() && "can only add register operands");
Call->addOperand(MachineOperand::CreateReg(
MOP.getReg(), /*Def=*/false, /*Implicit=*/true, /*isKill=*/false,
@@ -820,6 +818,20 @@ static MachineInstr *createCall(MachineBasicBlock &MBB,
return Call;
}
+// Create a call to CallTarget, copying over all the operands from *MBBI,
+// starting at the regmask.
+static MachineInstr *createCall(MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator MBBI,
+ const AArch64InstrInfo *TII,
+ MachineOperand &CallTarget,
+ unsigned RegMaskStartIdx) {
+ unsigned Opc = CallTarget.isGlobal() ? AArch64::BL : AArch64::BLR;
+
+ assert((CallTarget.isGlobal() || CallTarget.isReg()) &&
+ "invalid operand for regular call");
+ return createCallWithOps(MBB, MBBI, TII, Opc, CallTarget, RegMaskStartIdx);
+}
+
bool AArch64ExpandPseudo::expandCALL_RVMARKER(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) {
// Expand CALL_RVMARKER pseudo to:
@@ -830,10 +842,30 @@ bool AArch64ExpandPseudo::expandCALL_RVMARKER(
MachineInstr &MI = *MBBI;
MachineOperand &RVTarget = MI.getOperand(0);
assert(RVTarget.isGlobal() && "invalid operand for attached call");
- MachineInstr *OriginalCall =
- createCall(MBB, MBBI, TII, MI.getOperand(1),
- // Regmask starts after the RV and call targets.
- /*RegMaskStartIdx=*/2);
+
+ MachineInstr *OriginalCall = nullptr;
+
+ if (MI.getOpcode() == AArch64::BLRA_RVMARKER) {
+ // ptrauth call.
+ const MachineOperand &CallTarget = MI.getOperand(1);
+ const MachineOperand &Key = MI.getOperand(2);
+ const MachineOperand &IntDisc = MI.getOperand(3);
+ const MachineOperand &AddrDisc = MI.getOperand(4);
+
+ assert((Key.getImm() == AArch64PACKey::IA ||
+ Key.getImm() == AArch64PACKey::IB) &&
+ "Invalid auth call key");
+
+ MachineOperand Ops[] = {CallTarget, Key, IntDisc, AddrDisc};
+
+ OriginalCall = createCallWithOps(MBB, MBBI, TII, AArch64::BLRA, Ops,
+ /*RegMaskStartIdx=*/5);
+ } else {
+ assert(MI.getOpcode() == AArch64::BLR_RVMARKER && "unknown rvmarker MI");
+ OriginalCall = createCall(MBB, MBBI, TII, MI.getOperand(1),
+ // Regmask starts after the RV and call targets.
+ /*RegMaskStartIdx=*/2);
+ }
BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ORRXrs))
.addReg(AArch64::FP, RegState::Define)
@@ -1542,6 +1574,7 @@ bool AArch64ExpandPseudo::expandMI(MachineBasicBlock &MBB,
case AArch64::LDR_PPXI:
return expandSVESpillFill(MBB, MBBI, AArch64::LDR_PXI, 2);
case AArch64::BLR_RVMARKER:
+ case AArch64::BLRA_RVMARKER:
return expandCALL_RVMARKER(MBB, MBBI);
case AArch64::BLR_BTI:
return expandCALL_BTI(MBB, MBBI);
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index ac6f1e07c4184..ca519c4c1788a 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -338,6 +338,40 @@ static bool isZeroingInactiveLanes(SDValue Op) {
}
}
+static std::tuple<SDValue, SDValue>
+extractPtrauthBlendDiscriminators(SDValue Disc, SelectionDAG *DAG) {
+ SDLoc DL(Disc);
+ SDValue AddrDisc;
+ SDValue ConstDisc;
+
+ // If this is a blend, remember the constant and address discriminators.
+ // Otherwise, it's either a constant discriminator, or a non-blended
+ // address discriminator.
+ if (Disc->getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
+ Disc->getConstantOperandVal(0) == Intrinsic::ptrauth_blend) {
+ AddrDisc = Disc->getOperand(1);
+ ConstDisc = Disc->getOperand(2);
+ } else {
+ ConstDisc = Disc;
+ }
+
+ // If the constant discriminator (either the blend RHS, or the entire
+ // discriminator value) isn't a 16-bit constant, bail out, and let the
+ // discriminator be computed separately.
+ const auto *ConstDiscN = dyn_cast<ConstantSDNode>(ConstDisc);
+ if (!ConstDiscN || !isUInt<16>(ConstDiscN->getZExtValue()))
+ return std::make_tuple(DAG->getTargetConstant(0, DL, MVT::i64), Disc);
+
+ // If there's no address discriminator, use NoRegister, which we'll later
+ // replace with XZR, or directly use a Z variant of the inst. when available.
+ if (!AddrDisc)
+ AddrDisc = DAG->getRegister(AArch64::NoRegister, MVT::i64);
+
+ return std::make_tuple(
+ DAG->getTargetConstant(ConstDiscN->getZExtValue(), DL, MVT::i64),
+ AddrDisc);
+}
+
AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
const AArch64Subtarget &STI)
: TargetLowering(TM), Subtarget(&STI) {
@@ -2464,6 +2498,9 @@ const char *AArch64TargetLowering::getTargetNodeName(unsigned Opcode) const {
MAKE_CASE(AArch64ISD::ADRP)
MAKE_CASE(AArch64ISD::ADR)
MAKE_CASE(AArch64ISD::ADDlow)
+ MAKE_CASE(AArch64ISD::AUTH_CALL)
+ MAKE_CASE(AArch64ISD::AUTH_TC_RETURN)
+ MAKE_CASE(AArch64ISD::AUTH_CALL_RVMARKER)
MAKE_CASE(AArch64ISD::LOADgot)
MAKE_CASE(AArch64ISD::RET_GLUE)
MAKE_CASE(AArch64ISD::BRCOND)
@@ -8507,10 +8544,31 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
InGlue = Chain.getValue(1);
}
+ unsigned Opc = IsTailCall ? AArch64ISD::TC_RETURN : AArch64ISD::CALL;
+
std::vector<SDValue> Ops;
Ops.push_back(Chain);
Ops.push_back(Callee);
+ // Calls with operand bundle "clang.arc.attachedcall" are special. They should
+ // be expanded to the call, directly followed by a special marker sequence and
+ // a call to an ObjC library function. Use CALL_RVMARKER to do that.
+ if (CLI.CB && objcarc::hasAttachedCallOpBundle(CLI.CB)) {
+ assert(!IsTailCall &&
+ "tail calls cannot be marked with clang.arc.attachedcall");
+ Opc = AArch64ISD::CALL_RVMARKER;
+
+ // Add a target global address for the retainRV/claimRV runtime function
+ // just before the call target.
+ Function *ARCFn = *objcarc::getAttachedARCFunction(CLI.CB);
+ auto GA = DAG.getTargetGlobalAddress(ARCFn, DL, PtrVT);
+ Ops.insert(Ops.begin() + 1, GA);
+ } else if (CallConv == CallingConv::ARM64EC_Thunk_X64) {
+ Opc = AArch64ISD::CALL_ARM64EC_TO_X64;
+ } else if (GuardWithBTI) {
+ Opc = AArch64ISD::CALL_BTI;
+ }
+
if (IsTailCall) {
// Each tail call may have to adjust the stack by a
diff erent amount, so
// this information must travel along with the operation for eventual
@@ -8518,6 +8576,25 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
Ops.push_back(DAG.getTargetConstant(FPDiff, DL, MVT::i32));
}
+ if (CLI.PAI) {
+ const uint64_t Key = CLI.PAI->Key;
+ assert((Key == AArch64PACKey::IA || Key == AArch64PACKey::IB) &&
+ "Invalid auth call key");
+
+ // Split the discriminator into address/integer components.
+ SDValue AddrDisc, IntDisc;
+ std::tie(IntDisc, AddrDisc) =
+ extractPtrauthBlendDiscriminators(CLI.PAI->Discriminator, &DAG);
+
+ if (Opc == AArch64ISD::CALL_RVMARKER)
+ Opc = AArch64ISD::AUTH_CALL_RVMARKER;
+ else
+ Opc = IsTailCall ? AArch64ISD::AUTH_TC_RETURN : AArch64ISD::AUTH_CALL;
+ Ops.push_back(DAG.getTargetConstant(Key, DL, MVT::i32));
+ Ops.push_back(IntDisc);
+ Ops.push_back(AddrDisc);
+ }
+
// Add argument registers to the end of the list so that they are known live
// into the call.
for (auto &RegToPass : RegsToPass)
@@ -8555,8 +8632,7 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
// actual call instruction.
if (IsTailCall) {
MF.getFrameInfo().setHasTailCall();
- SDValue Ret = DAG.getNode(AArch64ISD::TC_RETURN, DL, NodeTys, Ops);
-
+ SDValue Ret = DAG.getNode(Opc, DL, NodeTys, Ops);
if (IsCFICall)
Ret.getNode()->setCFIType(CLI.CFIType->getZExtValue());
@@ -8565,29 +8641,8 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
return Ret;
}
- unsigned CallOpc = AArch64ISD::CALL;
- // Calls with operand bundle "clang.arc.attachedcall" are special. They should
- // be expanded to the call, directly followed by a special marker sequence and
- // a call to an ObjC library function. Use CALL_RVMARKER to do that.
- if (CLI.CB && objcarc::hasAttachedCallOpBundle(CLI.CB)) {
- assert(!IsTailCall &&
- "tail calls cannot be marked with clang.arc.attachedcall");
- CallOpc = AArch64ISD::CALL_RVMARKER;
-
- // Add a target global address for the retainRV/claimRV runtime function
- // just before the call target.
- Function *ARCFn = *objcarc::getAttachedARCFunction(CLI.CB);
- auto GA = DAG.getTargetGlobalAddress(ARCFn, DL, PtrVT);
- Ops.insert(Ops.begin() + 1, GA);
- } else if (CallConv == CallingConv::ARM64EC_Thunk_X64) {
- CallOpc = AArch64ISD::CALL_ARM64EC_TO_X64;
- } else if (GuardWithBTI) {
- CallOpc = AArch64ISD::CALL_BTI;
- }
-
// Returns a chain and a flag for retval copy to use.
- Chain = DAG.getNode(CallOpc, DL, NodeTys, Ops);
-
+ Chain = DAG.getNode(Opc, DL, NodeTys, Ops);
if (IsCFICall)
Chain.getNode()->setCFIType(CLI.CFIType->getZExtValue());
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 73bc9ad53bb8a..48a4ea91c2782 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -58,6 +58,16 @@ enum NodeType : unsigned {
CALL_BTI, // Function call followed by a BTI instruction.
+ // Function call, authenticating the callee value first:
+ // AUTH_CALL chain, callee, auth key #, int disc, addr disc, operands.
+ AUTH_CALL,
+ // AUTH_TC_RETURN chain, callee, fp
diff , auth key #, int disc, addr disc,
+ // operands.
+ AUTH_TC_RETURN,
+
+ // Authenticated variant of CALL_RVMARKER.
+ AUTH_CALL_RVMARKER,
+
COALESCER_BARRIER,
SMSTART,
@@ -911,6 +921,8 @@ class AArch64TargetLowering : public TargetLowering {
return true;
}
+ bool supportPtrAuthBundles() const override { return true; }
+
bool supportKCFIBundles() const override { return true; }
MachineInstr *EmitKCFICheck(MachineBasicBlock &MBB,
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index aa0b7c93f8661..96afb10ac3ff4 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -2511,6 +2511,8 @@ bool AArch64InstrInfo::isTailCallReturnInst(const MachineInstr &MI) {
case AArch64::TCRETURNrix17:
case AArch64::TCRETURNrinotx16:
case AArch64::TCRETURNriALL:
+ case AArch64::AUTH_TCRETURN:
+ case AArch64::AUTH_TCRETURN_BTI:
return true;
}
}
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index dd54520c8ddad..081d64921ee1c 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -660,6 +660,30 @@ def AArch64call_arm64ec_to_x64 : SDNode<"AArch64ISD::CALL_ARM64EC_TO_X64",
[SDNPHasChain, SDNPOptInGlue, SDNPOutGlue,
SDNPVariadic]>;
+def AArch64authcall : SDNode<"AArch64ISD::AUTH_CALL",
+ SDTypeProfile<0, -1, [SDTCisPtrTy<0>,
+ SDTCisVT<1, i32>,
+ SDTCisVT<2, i64>,
+ SDTCisVT<3, i64>]>,
+ [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue,
+ SDNPVariadic]>;
+
+def AArch64authtcret: SDNode<"AArch64ISD::AUTH_TC_RETURN",
+ SDTypeProfile<0, 5, [SDTCisPtrTy<0>,
+ SDTCisVT<2, i32>,
+ SDTCisVT<3, i64>,
+ SDTCisVT<4, i64>]>,
+ [SDNPHasChain, SDNPOptInGlue, SDNPVariadic]>;
+
+def AArch64authcall_rvmarker : SDNode<"AArch64ISD::AUTH_CALL_RVMARKER",
+ SDTypeProfile<0, -1, [SDTCisPtrTy<0>,
+ SDTCisPtrTy<1>,
+ SDTCisVT<2, i32>,
+ SDTCisVT<3, i64>,
+ SDTCisVT<4, i64>]>,
+ [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue,
+ SDNPVariadic]>;
+
def AArch64brcond : SDNode<"AArch64ISD::BRCOND", SDT_AArch64Brcond,
[SDNPHasChain]>;
def AArch64cbz : SDNode<"AArch64ISD::CBZ", SDT_AArch64cbz,
@@ -1696,6 +1720,38 @@ let Predicates = [HasPAuth] in {
def BLRABZ : AuthOneOperand<0b001, 1, "blrabz">;
}
+ // BLRA pseudo, a generalized version of BLRAA/BLRAB/Z.
+ // This directly manipulates x16/x17 to materialize the discriminator.
+ // x16/x17 are generally used as the safe registers for sensitive ptrauth
+ // operations (such as raw address manipulation or discriminator
+ // materialization here), in part because they're handled in a safer way by
+ // the kernel, notably on Darwin.
+ def BLRA : Pseudo<(outs), (ins GPR64noip:$Rn, i32imm:$Key, i64imm:$Disc,
+ GPR64noip:$AddrDisc),
+ [(AArch64authcall GPR64noip:$Rn, timm:$Key, timm:$Disc,
+ GPR64noip:$AddrDisc)]>, Sched<[]> {
+ let isCodeGenOnly = 1;
+ let hasSideEffects = 1;
+ let mayStore = 0;
+ let mayLoad = 0;
+ let isCall = 1;
+ let Size = 12; // 4 fixed + 8 variable, to compute discriminator.
+ let Defs = [X17,LR];
+ let Uses = [SP];
+ }
+
+ def BLRA_RVMARKER : Pseudo<
+ (outs), (ins i64imm:$rvfunc, GPR64noip:$Rn, i32imm:$Key, i64imm:$Disc,
+ GPR64noip:$AddrDisc),
+ [(AArch64authcall_rvmarker tglobaladdr:$rvfunc,
+ GPR64noip:$Rn, timm:$Key, timm:$Disc,
+ GPR64noip:$AddrDisc)]>, Sched<[]> {
+ let isCodeGenOnly = 1;
+ let isCall = 1;
+ let Defs = [X17,LR];
+ let Uses = [SP];
+ }
+
let isReturn = 1, isTerminator = 1, isBarrier = 1 in {
def RETAA : AuthReturn<0b010, 0, "retaa">;
def RETAB : AuthReturn<0b010, 1, "retab">;
@@ -1706,6 +1762,32 @@ let Predicates = [HasPAuth] in {
defm LDRAA : AuthLoad<0, "ldraa", simm10Scaled>;
defm LDRAB : AuthLoad<1, "ldrab", simm10Scaled>;
+ // Size 16: 4 fixed + 8 variable, to compute discriminator.
+ let isCall = 1, isTerminator = 1, isReturn = 1, isBarrier = 1, Size = 16,
+ Uses = [SP] in {
+ def AUTH_TCRETURN
+ : Pseudo<(outs), (ins tcGPR64:$dst, i32imm:$FPDiff, i32imm:$Key,
+ i64imm:$Disc, tcGPR64:$AddrDisc),
+ []>, Sched<[WriteBrReg]>;
+ def AUTH_TCRETURN_BTI
+ : Pseudo<(outs), (ins tcGPRx16x17:$dst, i32imm:$FPDiff, i32imm:$Key,
+ i64imm:$Disc, tcGPR64:$AddrDisc),
+ []>, Sched<[WriteBrReg]>;
+ }
+
+ let Predicates = [TailCallAny] in
+ def : Pat<(AArch64authtcret tcGPR64:$dst, (i32 timm:$FPDiff), (i32 timm:$Key),
+ (i64 timm:$Disc), tcGPR64:$AddrDisc),
+ (AUTH_TCRETURN tcGPR64:$dst, imm:$FPDiff, imm:$Key, imm:$Disc,
+ tcGPR64:$AddrDisc)>;
+
+ let Predicates = [TailCallX16X17] in
+ def : Pat<(AArch64authtcret tcGPRx16x17:$dst, (i32 timm:$FPDiff),
+ (i32 timm:$Key), (i64 timm:$Disc),
+ tcGPR64:$AddrDisc),
+ (AUTH_TCRETURN_BTI tcGPRx16x17:$dst, imm:$FPDiff, imm:$Key,
+ imm:$Disc, tcGPR64:$AddrDisc)>;
+
}
// v9.5-A pointer authentication extensions
diff --git a/llvm/lib/Target/AArch64/AArch64RegisterInfo.td b/llvm/lib/Target/AArch64/AArch64RegisterInfo.td
index 80d0f9c57f4b3..dfaa67dd1959d 100644
--- a/llvm/lib/Target/AArch64/AArch64RegisterInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64RegisterInfo.td
@@ -234,7 +234,10 @@ def tcGPRnotx16 : RegisterClass<"AArch64", [i64], 64, (sub tcGPR64, X16)>;
// Register set that excludes registers that are reserved for procedure calls.
// This is used for pseudo-instructions that are actually implemented using a
// procedure call.
-def GPR64noip : RegisterClass<"AArch64", [i64], 64, (sub GPR64, X16, X17, LR)>;
+def GPR64noip : RegisterClass<"AArch64", [i64], 64, (sub GPR64, X16, X17, LR)> {
+ let AltOrders = [(rotl GPR64noip, 8)];
+ let AltOrderSelect = [{ return 1; }];
+}
// GPR register classes for post increment amount of vector load/store that
// has alternate printing when Rm=31 and prints a constant immediate value
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
index c4197ff73187a..7068d25ed4aaa 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
@@ -13,6 +13,7 @@
//===----------------------------------------------------------------------===//
#include "AArch64CallLowering.h"
+#include "AArch64GlobalISelUtils.h"
#include "AArch64ISelLowering.h"
#include "AArch64MachineFunctionInfo.h"
#include "AArch64RegisterInfo.h"
@@ -50,6 +51,7 @@
#define DEBUG_TYPE "aarch64-call-lowering"
using namespace llvm;
+using namespace AArch64GISelUtils;
AArch64CallLowering::AArch64CallLowering(const AArch64TargetLowering &TLI)
: CallLowering(&TLI) {}
@@ -1012,11 +1014,20 @@ bool AArch64CallLowering::isEligibleForTailCallOptimization(
}
static unsigned getCallOpcode(const MachineFunction &CallerF, bool IsIndirect,
- bool IsTailCall) {
+ bool IsTailCall,
+ std::optional<CallLowering::PtrAuthInfo> &PAI,
+ MachineRegisterInfo &MRI) {
const AArch64FunctionInfo *FuncInfo = CallerF.getInfo<AArch64FunctionInfo>();
- if (!IsTailCall)
- return IsIndirect ? getBLRCallOpcode(CallerF) : (unsigned)AArch64::BL;
+ if (!IsTailCall) {
+ if (!PAI)
+ return IsIndirect ? getBLRCallOpcode(CallerF) : AArch64::BL;
+
+ assert(IsIndirect && "Direct call should not be authenticated");
+ assert((PAI->Key == AArch64PACKey::IA || PAI->Key == AArch64PACKey::IB) &&
+ "Invalid auth call key");
+ return AArch64::BLRA;
+ }
if (!IsIndirect)
return AArch64::TCRETURNdi;
@@ -1024,13 +1035,22 @@ static unsigned getCallOpcode(const MachineFunction &CallerF, bool IsIndirect,
// When BTI or PAuthLR are enabled, there are restrictions on using x16 and
// x17 to hold the function pointer.
if (FuncInfo->branchTargetEnforcement()) {
- if (FuncInfo->branchProtectionPAuthLR())
+ if (FuncInfo->branchProtectionPAuthLR()) {
+ assert(!PAI && "ptrauth tail-calls not yet supported with PAuthLR");
return AArch64::TCRETURNrix17;
- else
- return AArch64::TCRETURNrix16x17;
- } else if (FuncInfo->branchProtectionPAuthLR())
+ }
+ if (PAI)
+ return AArch64::AUTH_TCRETURN_BTI;
+ return AArch64::TCRETURNrix16x17;
+ }
+
+ if (FuncInfo->branchProtectionPAuthLR()) {
+ assert(!PAI && "ptrauth tail-calls not yet supported with PAuthLR");
return AArch64::TCRETURNrinotx16;
+ }
+ if (PAI)
+ return AArch64::AUTH_TCRETURN;
return AArch64::TCRETURNri;
}
@@ -1066,14 +1086,6 @@ bool AArch64CallLowering::lowerTailCall(
Info.CallConv != CallingConv::Tail &&
Info.CallConv != CallingConv::SwiftTail;
- // TODO: Right now, regbankselect doesn't know how to handle the rtcGPR64
- // register class. Until we can do that, we should fall back here.
- if (MF.getInfo<AArch64FunctionInfo>()->branchTargetEnforcement()) {
- LLVM_DEBUG(
- dbgs() << "Cannot lower indirect tail calls with BTI enabled yet.\n");
- return false;
- }
-
// Find out which ABI gets to decide where things go.
CallingConv::ID CalleeCC = Info.CallConv;
CCAssignFn *AssignFnFixed;
@@ -1084,17 +1096,41 @@ bool AArch64CallLowering::lowerTailCall(
if (!IsSibCall)
CallSeqStart = MIRBuilder.buildInstr(AArch64::ADJCALLSTACKDOWN);
- unsigned Opc = getCallOpcode(MF, Info.Callee.isReg(), true);
+ unsigned Opc = getCallOpcode(MF, Info.Callee.isReg(), true, Info.PAI, MRI);
auto MIB = MIRBuilder.buildInstrNoInsert(Opc);
MIB.add(Info.Callee);
+ // Tell the call which registers are clobbered.
+ const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
+ auto TRI = Subtarget.getRegisterInfo();
+
// Byte offset for the tail call. When we are sibcalling, this will always
// be 0.
MIB.addImm(0);
+ // Authenticated tail calls always take key/discriminator arguments.
+ if (Opc == AArch64::AUTH_TCRETURN || Opc == AArch64::AUTH_TCRETURN_BTI) {
+ assert((Info.PAI->Key == AArch64PACKey::IA ||
+ Info.PAI->Key == AArch64PACKey::IB) &&
+ "Invalid auth call key");
+ MIB.addImm(Info.PAI->Key);
+
+ Register AddrDisc = 0;
+ uint16_t IntDisc = 0;
+ std::tie(IntDisc, AddrDisc) =
+ extractPtrauthBlendDiscriminators(Info.PAI->Discriminator, MRI);
+
+ MIB.addImm(IntDisc);
+ MIB.addUse(AddrDisc);
+ if (AddrDisc != AArch64::NoRegister) {
+ MIB->getOperand(4).setReg(constrainOperandRegClass(
+ MF, *TRI, MRI, *MF.getSubtarget().getInstrInfo(),
+ *MF.getSubtarget().getRegBankInfo(), *MIB, MIB->getDesc(),
+ MIB->getOperand(4), 4));
+ }
+ }
+
// Tell the call which registers are clobbered.
- const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
- auto TRI = Subtarget.getRegisterInfo();
const uint32_t *Mask = TRI->getCallPreservedMask(MF, CalleeCC);
if (Subtarget.hasCustomCallingConv())
TRI->UpdateCustomCallPreservedMask(MF, &Mask);
@@ -1294,7 +1330,7 @@ bool AArch64CallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
// be expanded to the call, directly followed by a special marker sequence and
// a call to an ObjC library function.
if (Info.CB && objcarc::hasAttachedCallOpBundle(Info.CB))
- Opc = AArch64::BLR_RVMARKER;
+ Opc = Info.PAI ? AArch64::BLRA_RVMARKER : AArch64::BLR_RVMARKER;
// A call to a returns twice function like setjmp must be followed by a bti
// instruction.
else if (Info.CB && Info.CB->hasFnAttr(Attribute::ReturnsTwice) &&
@@ -1310,13 +1346,13 @@ bool AArch64CallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
MIB.addExternalSymbol(Info.Callee.getSymbolName(), AArch64II::MO_GOT);
Info.Callee = MachineOperand::CreateReg(MIB.getReg(0), false);
}
- Opc = getCallOpcode(MF, Info.Callee.isReg(), false);
+ Opc = getCallOpcode(MF, Info.Callee.isReg(), false, Info.PAI, MRI);
}
auto MIB = MIRBuilder.buildInstrNoInsert(Opc);
unsigned CalleeOpNo = 0;
- if (Opc == AArch64::BLR_RVMARKER) {
+ if (Opc == AArch64::BLR_RVMARKER || Opc == AArch64::BLRA_RVMARKER) {
// Add a target global address for the retainRV/claimRV runtime function
// just before the call target.
Function *ARCFn = *objcarc::getAttachedARCFunction(Info.CB);
@@ -1342,6 +1378,28 @@ bool AArch64CallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
Mask = getMaskForArgs(OutArgs, Info, *TRI, MF);
+ if (Opc == AArch64::BLRA || Opc == AArch64::BLRA_RVMARKER) {
+ assert((Info.PAI->Key == AArch64PACKey::IA ||
+ Info.PAI->Key == AArch64PACKey::IB) &&
+ "Invalid auth call key");
+ MIB.addImm(Info.PAI->Key);
+
+ Register AddrDisc = 0;
+ uint16_t IntDisc = 0;
+ std::tie(IntDisc, AddrDisc) =
+ extractPtrauthBlendDiscriminators(Info.PAI->Discriminator, MRI);
+
+ MIB.addImm(IntDisc);
+ MIB.addUse(AddrDisc);
+ if (AddrDisc != AArch64::NoRegister) {
+ constrainOperandRegClass(MF, *TRI, MRI, *MF.getSubtarget().getInstrInfo(),
+ *MF.getSubtarget().getRegBankInfo(), *MIB,
+ MIB->getDesc(), MIB->getOperand(CalleeOpNo + 3),
+ CalleeOpNo + 3);
+ }
+ }
+
+ // Tell the call which registers are clobbered.
if (MF.getSubtarget<AArch64Subtarget>().hasCustomCallingConv())
TRI->UpdateCustomCallPreservedMask(MF, &Mask);
MIB.addRegMask(Mask);
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64GlobalISelUtils.cpp b/llvm/lib/Target/AArch64/GISel/AArch64GlobalISelUtils.cpp
index 80fe4bcb8b58f..a2b909205ea84 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64GlobalISelUtils.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64GlobalISelUtils.cpp
@@ -96,6 +96,35 @@ bool AArch64GISelUtils::tryEmitBZero(MachineInstr &MI,
return true;
}
+std::tuple<uint16_t, Register>
+AArch64GISelUtils::extractPtrauthBlendDiscriminators(Register Disc,
+ MachineRegisterInfo &MRI) {
+ Register AddrDisc = Disc;
+ uint16_t ConstDisc = 0;
+
+ if (auto ConstDiscVal = getIConstantVRegVal(Disc, MRI)) {
+ if (isUInt<16>(ConstDiscVal->getZExtValue())) {
+ ConstDisc = ConstDiscVal->getZExtValue();
+ AddrDisc = AArch64::NoRegister;
+ }
+ return std::make_tuple(ConstDisc, AddrDisc);
+ }
+
+ const MachineInstr *DiscMI = MRI.getVRegDef(Disc);
+ if (!DiscMI || DiscMI->getOpcode() != TargetOpcode::G_INTRINSIC ||
+ DiscMI->getOperand(1).getIntrinsicID() != Intrinsic::ptrauth_blend)
+ return std::make_tuple(ConstDisc, AddrDisc);
+
+ if (auto ConstDiscVal =
+ getIConstantVRegVal(DiscMI->getOperand(3).getReg(), MRI)) {
+ if (isUInt<16>(ConstDiscVal->getZExtValue())) {
+ ConstDisc = ConstDiscVal->getZExtValue();
+ AddrDisc = DiscMI->getOperand(2).getReg();
+ }
+ }
+ return std::make_tuple(ConstDisc, AddrDisc);
+}
+
void AArch64GISelUtils::changeFCMPPredToAArch64CC(
const CmpInst::Predicate P, AArch64CC::CondCode &CondCode,
AArch64CC::CondCode &CondCode2) {
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64GlobalISelUtils.h b/llvm/lib/Target/AArch64/GISel/AArch64GlobalISelUtils.h
index 791db7efaf0be..9ef833f0fc0ca 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64GlobalISelUtils.h
+++ b/llvm/lib/Target/AArch64/GISel/AArch64GlobalISelUtils.h
@@ -53,6 +53,12 @@ bool isCMN(const MachineInstr *MaybeSub, const CmpInst::Predicate &Pred,
/// \returns true if \p MI was replaced with a G_BZERO.
bool tryEmitBZero(MachineInstr &MI, MachineIRBuilder &MIRBuilder, bool MinSize);
+/// Analyze a ptrauth discriminator value to try to find the constant integer
+/// and address parts, cracking a ptrauth_blend intrinsic if there is one.
+/// \returns integer/address disc. parts, with NoRegister if no address disc.
+std::tuple<uint16_t, Register>
+extractPtrauthBlendDiscriminators(Register Disc, MachineRegisterInfo &MRI);
+
/// Find the AArch64 condition codes necessary to represent \p P for a scalar
/// floating point comparison.
///
diff --git a/llvm/test/CodeGen/AArch64/branch-target-enforcement-indirect-calls.ll b/llvm/test/CodeGen/AArch64/branch-target-enforcement-indirect-calls.ll
index 833a6d5b1d1da..3d8a5976559ba 100644
--- a/llvm/test/CodeGen/AArch64/branch-target-enforcement-indirect-calls.ll
+++ b/llvm/test/CodeGen/AArch64/branch-target-enforcement-indirect-calls.ll
@@ -1,7 +1,5 @@
; RUN: llc -mtriple aarch64 -mattr=+bti < %s | FileCheck %s
-; RUN: llc -mtriple aarch64 -global-isel -global-isel-abort=2 -pass-remarks-missed=gisel* -mattr=+bti %s -verify-machineinstrs -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,FALLBACK
-
-; FALLBACK: remark: <unknown>:0:0: unable to translate instruction: call: ' tail call void %p()' (in function: bti_enabled)
+; RUN: llc -mtriple aarch64 -global-isel -mattr=+bti < %s | FileCheck %s
target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64"
diff --git a/llvm/test/CodeGen/AArch64/ptrauth-bti-call.ll b/llvm/test/CodeGen/AArch64/ptrauth-bti-call.ll
new file mode 100644
index 0000000000000..3a992929a8003
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/ptrauth-bti-call.ll
@@ -0,0 +1,105 @@
+; RUN: llc -mtriple arm64e-apple-darwin -mattr=+bti -asm-verbose=false -o - %s | FileCheck %s
+; RUN: llc -mtriple arm64e-apple-darwin -mattr=+bti -global-isel -asm-verbose=false -o - %s | FileCheck %s
+; RUN: llc -mtriple arm64e-apple-darwin -mattr=+bti -fast-isel -asm-verbose=false -o - %s | FileCheck %s
+
+; ptrauth tail-calls can only use x16/x17 with BTI.
+
+; CHECK-LABEL: _test_tailcall_ia_0:
+; CHECK-NEXT: bti c
+; CHECK-NEXT: mov x16, x0
+; CHECK-NEXT: braaz x16
+define i32 @test_tailcall_ia_0(i32 ()* %arg0) #0 {
+ %tmp0 = tail call i32 %arg0() [ "ptrauth"(i32 0, i64 0) ]
+ ret i32 %tmp0
+}
+
+; CHECK-LABEL: _test_tailcall_ib_0:
+; CHECK-NEXT: bti c
+; CHECK-NEXT: mov x16, x0
+; CHECK-NEXT: brabz x16
+define i32 @test_tailcall_ib_0(i32 ()* %arg0) #0 {
+ %tmp0 = tail call i32 %arg0() [ "ptrauth"(i32 1, i64 0) ]
+ ret i32 %tmp0
+}
+
+; CHECK-LABEL: _test_tailcall_ia_imm:
+; CHECK-NEXT: bti c
+; CHECK-NEXT: mov x16, x0
+; CHECK-NEXT: mov x17, #42
+; CHECK-NEXT: braa x16, x17
+define i32 @test_tailcall_ia_imm(i32 ()* %arg0) #0 {
+ %tmp0 = tail call i32 %arg0() [ "ptrauth"(i32 0, i64 42) ]
+ ret i32 %tmp0
+}
+
+; CHECK-LABEL: _test_tailcall_ib_imm:
+; CHECK-NEXT: bti c
+; CHECK-NEXT: mov x16, x0
+; CHECK-NEXT: mov x17, #42
+; CHECK-NEXT: brab x16, x17
+define i32 @test_tailcall_ib_imm(i32 ()* %arg0) #0 {
+ %tmp0 = tail call i32 %arg0() [ "ptrauth"(i32 1, i64 42) ]
+ ret i32 %tmp0
+}
+
+; CHECK-LABEL: _test_tailcall_ia_var:
+; CHECK-NEXT: bti c
+; CHECK-NEXT: mov x16, x0
+; CHECK-NEXT: ldr x0, [x1]
+; CHECK-NEXT: braa x16, x0
+define i32 @test_tailcall_ia_var(i32 ()* %arg0, i64* %arg1) #0 {
+ %tmp0 = load i64, i64* %arg1
+ %tmp1 = tail call i32 %arg0() [ "ptrauth"(i32 0, i64 %tmp0) ]
+ ret i32 %tmp1
+}
+
+; CHECK-LABEL: _test_tailcall_ib_var:
+; CHECK-NEXT: bti c
+; CHECK-NEXT: mov x16, x0
+; CHECK-NEXT: ldr x0, [x1]
+; CHECK-NEXT: brab x16, x0
+define i32 @test_tailcall_ib_var(i32 ()* %arg0, i64* %arg1) #0 {
+ %tmp0 = load i64, i64* %arg1
+ %tmp1 = tail call i32 %arg0() [ "ptrauth"(i32 1, i64 %tmp0) ]
+ ret i32 %tmp1
+}
+
+; CHECK-LABEL: _test_tailcall_ia_arg:
+; CHECK-NEXT: bti c
+; CHECK-NEXT: mov x16, x0
+; CHECK-NEXT: braa x16, x1
+define i32 @test_tailcall_ia_arg(i32 ()* %arg0, i64 %arg1) #0 {
+ %tmp0 = tail call i32 %arg0() [ "ptrauth"(i32 0, i64 %arg1) ]
+ ret i32 %tmp0
+}
+
+; CHECK-LABEL: _test_tailcall_ib_arg:
+; CHECK-NEXT: bti c
+; CHECK-NEXT: mov x16, x0
+; CHECK-NEXT: brab x16, x1
+define i32 @test_tailcall_ib_arg(i32 ()* %arg0, i64 %arg1) #0 {
+ %tmp0 = tail call i32 %arg0() [ "ptrauth"(i32 1, i64 %arg1) ]
+ ret i32 %tmp0
+}
+
+; CHECK-LABEL: _test_tailcall_ia_arg_ind:
+; CHECK-NEXT: bti c
+; CHECK-NEXT: ldr x16, [x0]
+; CHECK-NEXT: braa x16, x1
+define i32 @test_tailcall_ia_arg_ind(i32 ()** %arg0, i64 %arg1) #0 {
+ %tmp0 = load i32 ()*, i32 ()** %arg0
+ %tmp1 = tail call i32 %tmp0() [ "ptrauth"(i32 0, i64 %arg1) ]
+ ret i32 %tmp1
+}
+
+; CHECK-LABEL: _test_tailcall_ib_arg_ind:
+; CHECK-NEXT: bti c
+; CHECK-NEXT: ldr x16, [x0]
+; CHECK-NEXT: brab x16, x1
+define i32 @test_tailcall_ib_arg_ind(i32 ()** %arg0, i64 %arg1) #0 {
+ %tmp0 = load i32 ()*, i32 ()** %arg0
+ %tmp1 = tail call i32 %tmp0() [ "ptrauth"(i32 1, i64 %arg1) ]
+ ret i32 %tmp1
+}
+
+attributes #0 = { nounwind "branch-target-enforcement"="true" }
diff --git a/llvm/test/CodeGen/AArch64/ptrauth-call-rv-marker.ll b/llvm/test/CodeGen/AArch64/ptrauth-call-rv-marker.ll
new file mode 100644
index 0000000000000..9cf77b125e107
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/ptrauth-call-rv-marker.ll
@@ -0,0 +1,154 @@
+; RUN: llc -verify-machineinstrs -o - %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -global-isel -o - %s | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "arm64e-apple-iphoneos"
+
+declare i8* @foo0(i32)
+declare i8* @foo1()
+
+declare void @llvm.objc.release(i8*)
+declare i8* @llvm.objc.retainAutoreleasedReturnValue(i8*)
+declare i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8*)
+
+declare void @foo2(i8*)
+
+declare void @foo(i64, i64, i64)
+
+define void @rv_marker_ptrauth_blraa(i8* ()** %arg0, i64 %arg1) {
+; CHECK-LABEL: rv_marker_ptrauth_blraa
+; CHECK: ldr [[ADDR:x[0-9]+]], [
+; CHECK-NEXT: blraa [[ADDR]], x1
+; CHECK-NEXT: mov x29, x29
+; CHECK-NEXT: bl objc_retainAutoreleasedReturnValue
+;
+entry:
+ %tmp0 = load i8* ()*, i8* ()** %arg0
+ %call0 = call i8* %tmp0() [ "ptrauth"(i32 0, i64 %arg1), "clang.arc.attachedcall"(i8* (i8*)* @llvm.objc.retainAutoreleasedReturnValue) ]
+ tail call void @foo2(i8* %call0)
+ tail call void @llvm.objc.release(i8* %call0)
+ ret void
+}
+
+define void @rv_marker_ptrauth_blraa_unsafeClaim(i8* ()** %arg0, i64 %arg1) {
+; CHECK-LABEL: rv_marker_ptrauth_blraa_unsafeClaim
+; CHECK: ldr [[ADDR:x[0-9]+]], [
+; CHECK-NEXT: blraa [[ADDR]], x1
+; CHECK-NEXT: mov x29, x29
+; CHECK-NEXT: bl objc_unsafeClaimAutoreleasedReturnValue
+;
+entry:
+ %tmp0 = load i8* ()*, i8* ()** %arg0
+ %call0 = call i8* %tmp0() [ "ptrauth"(i32 0, i64 %arg1), "clang.arc.attachedcall"(i8* (i8*)* @llvm.objc.unsafeClaimAutoreleasedReturnValue) ]
+ tail call void @foo2(i8* %call0)
+ tail call void @llvm.objc.release(i8* %call0)
+ ret void
+}
+
+define void @rv_marker_ptrauth_blraa_disc_imm16(i8* ()** %arg0) {
+; CHECK-LABEL: rv_marker_ptrauth_blraa_disc_imm16
+; CHECK: ldr [[ADDR:x[0-9]+]], [
+; CHECK-NEXT: mov x17, #45431
+; CHECK-NEXT: blrab [[ADDR]], x17
+; CHECK-NEXT: mov x29, x29
+; CHECK-NEXT: bl objc_retainAutoreleasedReturnValue
+;
+ %tmp0 = load i8* ()*, i8* ()** %arg0
+ %call0 = call i8* %tmp0() [ "ptrauth"(i32 1, i64 45431), "clang.arc.attachedcall"(i8* (i8*)* @llvm.objc.retainAutoreleasedReturnValue) ]
+ tail call void @foo2(i8* %call0)
+ tail call void @llvm.objc.release(i8* %call0)
+ ret void
+}
+
+define void @rv_marker_ptrauth_blraa_multiarg(i8* (i64, i64, i64)** %arg0, i64 %arg1, i64 %a, i64 %b, i64 %c) {
+; CHECK-LABEL: rv_marker_ptrauth_blraa_multiarg
+; CHECK: mov [[TMP:x[0-9]+]], x1
+; CHECK-DAG: ldr [[ADDR:x[0-9]+]]
+; CHECK-DAG: mov x0, x4
+; CHECK-DAG: mov x1, x3
+; CHECK-NEXT: blraa [[ADDR]], [[TMP]]
+; CHECK-NEXT: mov x29, x29
+; CHECK-NEXT: bl objc_retainAutoreleasedReturnValue
+;
+entry:
+ %tmp0 = load i8* (i64, i64, i64)*, i8* (i64, i64, i64)** %arg0
+ %call0 = call i8* %tmp0(i64 %c, i64 %b, i64 %a) [ "ptrauth"(i32 0, i64 %arg1), "clang.arc.attachedcall"(i8* (i8*)* @llvm.objc.retainAutoreleasedReturnValue) ]
+ tail call void @foo2(i8* %call0)
+ tail call void @llvm.objc.release(i8* %call0)
+ ret void
+}
+
+define void @rv_marker_ptrauth_blrab(i8* ()** %arg0, i64 %arg1) {
+; CHECK-LABEL: rv_marker_ptrauth_blrab
+; CHECK: ldr [[ADDR:x[0-9]+]], [
+; CHECK-NEXT: blrab [[ADDR]], x1
+; CHECK-NEXT: mov x29, x29
+; CHECK-NEXT: bl objc_retainAutoreleasedReturnValue
+;
+ %tmp0 = load i8* ()*, i8* ()** %arg0
+ %call0 = call i8* %tmp0() [ "ptrauth"(i32 1, i64 %arg1), "clang.arc.attachedcall"(i8* (i8*)* @llvm.objc.retainAutoreleasedReturnValue) ]
+ tail call void @foo2(i8* %call0)
+ tail call void @llvm.objc.release(i8* %call0)
+ ret void
+}
+
+define void @rv_marker_ptrauth_blrab_disc_imm16(i8* ()** %arg0) {
+; CHECK-LABEL: rv_marker_ptrauth_blrab_disc_imm16
+; CHECK: ldr [[ADDR:x[0-9]+]], [
+; CHECK-NEXT: mov x17, #256
+; CHECK-NEXT: blrab [[ADDR]], x17
+; CHECK-NEXT: mov x29, x29
+; CHECK-NEXT: bl objc_retainAutoreleasedReturnValue
+;
+ %tmp0 = load i8* ()*, i8* ()** %arg0
+ %call0 = call i8* %tmp0() [ "ptrauth"(i32 1, i64 256), "clang.arc.attachedcall"(i8* (i8*)* @llvm.objc.retainAutoreleasedReturnValue) ]
+ tail call void @foo2(i8* %call0)
+ tail call void @llvm.objc.release(i8* %call0)
+ ret void
+}
+
+define void @rv_marker_ptrauth_blraaz(i8* ()** %arg0) {
+; CHECK-LABEL: rv_marker_ptrauth_blraaz
+; CHECK: ldr [[ADDR:x[0-9]+]], [
+; CHECK-NEXT: blraaz [[ADDR]]
+; CHECK-NEXT: mov x29, x29
+; CHECK-NEXT: bl objc_retainAutoreleasedReturnValue
+;
+ %tmp0 = load i8* ()*, i8* ()** %arg0
+ %call0 = call i8* %tmp0() [ "ptrauth"(i32 0, i64 0), "clang.arc.attachedcall"(i8* (i8*)* @llvm.objc.retainAutoreleasedReturnValue) ]
+ tail call void @foo2(i8* %call0)
+ tail call void @llvm.objc.release(i8* %call0)
+ ret void
+}
+
+define void @rv_marker_ptrauth_blrabz(i8* ()** %arg0) {
+; CHECK-LABEL: rv_marker_ptrauth_blrabz
+; CHECK: ldr [[ADDR:x[0-9]+]], [
+; CHECK-NEXT: blrabz [[ADDR]]
+; CHECK-NEXT: mov x29, x29
+; CHECK-NEXT: bl objc_retainAutoreleasedReturnValue
+;
+ %tmp0 = load i8* ()*, i8* ()** %arg0
+ %call0 = call i8* %tmp0() [ "ptrauth"(i32 1, i64 0), "clang.arc.attachedcall"(i8* (i8*)* @llvm.objc.retainAutoreleasedReturnValue) ]
+ tail call void @foo2(i8* %call0)
+ tail call void @llvm.objc.release(i8* %call0)
+ ret void
+}
+
+define void @rv_marker_ptrauth_blrabz_multiarg(i8* (i64, i64, i64)** %arg0, i64 %a, i64 %b, i64 %c) {
+; CHECK-LABEL: rv_marker_ptrauth_blrabz_multiarg
+; CHECK: mov [[TMP:x[0-9]+]], x1
+; CHECK-DAG: ldr [[ADDR:x[0-9]+]], [
+; CHECK-DAG: mov x0, x3
+; CHECK-DAG: mov x1, x2
+; CHECK-DAG: mov x2, [[TMP]]
+; CHECK-NEXT: blrabz [[ADDR]]
+; CHECK-NEXT: mov x29, x29
+; CHECK-NEXT: bl objc_retainAutoreleasedReturnValue
+;
+ %tmp0 = load i8* (i64, i64, i64)*, i8* (i64, i64, i64)** %arg0
+ %call0 = call i8* %tmp0(i64 %c, i64 %b, i64 %a) [ "ptrauth"(i32 1, i64 0), "clang.arc.attachedcall"(i8* (i8*)* @llvm.objc.retainAutoreleasedReturnValue) ]
+ tail call void @foo2(i8* %call0)
+ tail call void @llvm.objc.release(i8* %call0)
+ ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/ptrauth-call.ll b/llvm/test/CodeGen/AArch64/ptrauth-call.ll
new file mode 100644
index 0000000000000..b1249891b04b4
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/ptrauth-call.ll
@@ -0,0 +1,272 @@
+; RUN: llc -mtriple arm64e-apple-darwin -o - %s -asm-verbose=0 \
+; RUN: | FileCheck %s --check-prefixes=CHECK,DARWIN
+
+; RUN: llc -mtriple aarch64 -mattr=+pauth -o - %s -asm-verbose=0 \
+; RUN: | FileCheck %s --check-prefixes=CHECK,ELF
+
+; RUN: llc -mtriple arm64e-apple-darwin -o - %s -asm-verbose=0 \
+; RUN: -global-isel -global-isel-abort=1 -verify-machineinstrs \
+; RUN: | FileCheck %s --check-prefixes=CHECK,DARWIN
+
+; RUN: llc -mtriple aarch64 -mattr=+pauth -o - %s -asm-verbose=0 \
+; RUN: -global-isel -global-isel-abort=1 -verify-machineinstrs \
+; RUN: | FileCheck %s --check-prefixes=CHECK,ELF
+
+
+define i32 @test_call_ia_0(i32 ()* %arg0) #0 {
+; DARWIN-LABEL: test_call_ia_0:
+; DARWIN-NEXT: stp x29, x30, [sp, #-16]!
+; DARWIN-NEXT: blraaz x0
+; DARWIN-NEXT: ldp x29, x30, [sp], #16
+; DARWIN-NEXT: ret
+;
+; ELF-LABEL: test_call_ia_0:
+; ELF-NEXT: str x30, [sp, #-16]!
+; ELF-NEXT: blraaz x0
+; ELF-NEXT: ldr x30, [sp], #16
+; ELF-NEXT: ret
+ %tmp0 = call i32 %arg0() [ "ptrauth"(i32 0, i64 0) ]
+ ret i32 %tmp0
+}
+
+define i32 @test_call_ib_0(i32 ()* %arg0) #0 {
+; DARWIN-LABEL: test_call_ib_0:
+; DARWIN-NEXT: stp x29, x30, [sp, #-16]!
+; DARWIN-NEXT: blrabz x0
+; DARWIN-NEXT: ldp x29, x30, [sp], #16
+; DARWIN-NEXT: ret
+;
+; ELF-LABEL: test_call_ib_0:
+; ELF-NEXT: str x30, [sp, #-16]!
+; ELF-NEXT: blrabz x0
+; ELF-NEXT: ldr x30, [sp], #16
+; ELF-NEXT: ret
+ %tmp0 = call i32 %arg0() [ "ptrauth"(i32 1, i64 0) ]
+ ret i32 %tmp0
+}
+
+define i32 @test_tailcall_ia_0(i32 ()* %arg0) #0 {
+; CHECK-LABEL: test_tailcall_ia_0:
+; CHECK: braaz x0
+ %tmp0 = tail call i32 %arg0() [ "ptrauth"(i32 0, i64 0) ]
+ ret i32 %tmp0
+}
+
+define i32 @test_tailcall_ib_0(i32 ()* %arg0) #0 {
+; CHECK-LABEL: test_tailcall_ib_0:
+; CHECK: brabz x0
+ %tmp0 = tail call i32 %arg0() [ "ptrauth"(i32 1, i64 0) ]
+ ret i32 %tmp0
+}
+
+define i32 @test_call_ia_imm(i32 ()* %arg0) #0 {
+; DARWIN-LABEL: test_call_ia_imm:
+; DARWIN-NEXT: stp x29, x30, [sp, #-16]!
+; DARWIN-NEXT: mov x17, #42
+; DARWIN-NEXT: blraa x0, x17
+; DARWIN-NEXT: ldp x29, x30, [sp], #16
+; DARWIN-NEXT: ret
+;
+; ELF-LABEL: test_call_ia_imm:
+; ELF-NEXT: str x30, [sp, #-16]!
+; ELF-NEXT: mov x17, #42
+; ELF-NEXT: blraa x0, x17
+; ELF-NEXT: ldr x30, [sp], #16
+; ELF-NEXT: ret
+ %tmp0 = call i32 %arg0() [ "ptrauth"(i32 0, i64 42) ]
+ ret i32 %tmp0
+}
+
+define i32 @test_call_ib_imm(i32 ()* %arg0) #0 {
+; DARWIN-LABEL: test_call_ib_imm:
+; DARWIN-NEXT: stp x29, x30, [sp, #-16]!
+; DARWIN-NEXT: mov x17, #42
+; DARWIN-NEXT: blrab x0, x17
+; DARWIN-NEXT: ldp x29, x30, [sp], #16
+; DARWIN-NEXT: ret
+;
+; ELF-LABEL: test_call_ib_imm:
+; ELF-NEXT: str x30, [sp, #-16]!
+; ELF-NEXT: mov x17, #42
+; ELF-NEXT: blrab x0, x17
+; ELF-NEXT: ldr x30, [sp], #16
+; ELF-NEXT: ret
+ %tmp0 = call i32 %arg0() [ "ptrauth"(i32 1, i64 42) ]
+ ret i32 %tmp0
+}
+
+define i32 @test_tailcall_ia_imm(i32 ()* %arg0) #0 {
+; CHECK-LABEL: test_tailcall_ia_imm:
+; CHECK-NEXT: mov x16, #42
+; CHECK-NEXT: braa x0, x16
+ %tmp0 = tail call i32 %arg0() [ "ptrauth"(i32 0, i64 42) ]
+ ret i32 %tmp0
+}
+
+define i32 @test_tailcall_ib_imm(i32 ()* %arg0) #0 {
+; CHECK-LABEL: test_tailcall_ib_imm:
+; CHECK-NEXT: mov x16, #42
+; CHECK-NEXT: brab x0, x16
+ %tmp0 = tail call i32 %arg0() [ "ptrauth"(i32 1, i64 42) ]
+ ret i32 %tmp0
+}
+
+define i32 @test_call_ia_var(i32 ()* %arg0, i64* %arg1) #0 {
+; DARWIN-LABEL: test_call_ia_var:
+; DARWIN-NEXT: stp x29, x30, [sp, #-16]!
+; DARWIN-NEXT: ldr x8, [x1]
+; DARWIN-NEXT: blraa x0, x8
+; DARWIN-NEXT: ldp x29, x30, [sp], #16
+; DARWIN-NEXT: ret
+;
+; ELF-LABEL: test_call_ia_var:
+; ELF-NEXT: str x30, [sp, #-16]!
+; ELF-NEXT: ldr x8, [x1]
+; ELF-NEXT: blraa x0, x8
+; ELF-NEXT: ldr x30, [sp], #16
+; ELF-NEXT: ret
+ %tmp0 = load i64, i64* %arg1
+ %tmp1 = call i32 %arg0() [ "ptrauth"(i32 0, i64 %tmp0) ]
+ ret i32 %tmp1
+}
+
+define i32 @test_call_ib_var(i32 ()* %arg0, i64* %arg1) #0 {
+; DARWIN-LABEL: test_call_ib_var:
+; DARWIN-NEXT: stp x29, x30, [sp, #-16]!
+; DARWIN-NEXT: ldr x8, [x1]
+; DARWIN-NEXT: blrab x0, x8
+; DARWIN-NEXT: ldp x29, x30, [sp], #16
+; DARWIN-NEXT: ret
+;
+; ELF-LABEL: test_call_ib_var:
+; ELF-NEXT: str x30, [sp, #-16]!
+; ELF-NEXT: ldr x8, [x1]
+; ELF-NEXT: blrab x0, x8
+; ELF-NEXT: ldr x30, [sp], #16
+; ELF-NEXT: ret
+ %tmp0 = load i64, i64* %arg1
+ %tmp1 = call i32 %arg0() [ "ptrauth"(i32 1, i64 %tmp0) ]
+ ret i32 %tmp1
+}
+
+define i32 @test_tailcall_ia_var(i32 ()* %arg0, i64* %arg1) #0 {
+; CHECK-LABEL: test_tailcall_ia_var:
+; CHECK: ldr x1, [x1]
+; CHECK: braa x0, x1
+ %tmp0 = load i64, i64* %arg1
+ %tmp1 = tail call i32 %arg0() [ "ptrauth"(i32 0, i64 %tmp0) ]
+ ret i32 %tmp1
+}
+
+define i32 @test_tailcall_ib_var(i32 ()* %arg0, i64* %arg1) #0 {
+; CHECK-LABEL: test_tailcall_ib_var:
+; CHECK: ldr x1, [x1]
+; CHECK: brab x0, x1
+ %tmp0 = load i64, i64* %arg1
+ %tmp1 = tail call i32 %arg0() [ "ptrauth"(i32 1, i64 %tmp0) ]
+ ret i32 %tmp1
+}
+
+define i32 @test_call_ia_arg(i32 ()* %arg0, i64 %arg1) #0 {
+; DARWIN-LABEL: test_call_ia_arg:
+; DARWIN-NEXT: stp x29, x30, [sp, #-16]!
+; DARWIN-NEXT: blraa x0, x1
+; DARWIN-NEXT: ldp x29, x30, [sp], #16
+; DARWIN-NEXT: ret
+;
+; ELF-LABEL: test_call_ia_arg:
+; ELF-NEXT: str x30, [sp, #-16]!
+; ELF-NEXT: blraa x0, x1
+; ELF-NEXT: ldr x30, [sp], #16
+; ELF-NEXT: ret
+ %tmp0 = call i32 %arg0() [ "ptrauth"(i32 0, i64 %arg1) ]
+ ret i32 %tmp0
+}
+
+define i32 @test_call_ib_arg(i32 ()* %arg0, i64 %arg1) #0 {
+; DARWIN-LABEL: test_call_ib_arg:
+; DARWIN-NEXT: stp x29, x30, [sp, #-16]!
+; DARWIN-NEXT: blrab x0, x1
+; DARWIN-NEXT: ldp x29, x30, [sp], #16
+; DARWIN-NEXT: ret
+;
+; ELF-LABEL: test_call_ib_arg:
+; ELF-NEXT: str x30, [sp, #-16]!
+; ELF-NEXT: blrab x0, x1
+; ELF-NEXT: ldr x30, [sp], #16
+; ELF-NEXT: ret
+ %tmp0 = call i32 %arg0() [ "ptrauth"(i32 1, i64 %arg1) ]
+ ret i32 %tmp0
+}
+
+define i32 @test_tailcall_ia_arg(i32 ()* %arg0, i64 %arg1) #0 {
+; CHECK-LABEL: test_tailcall_ia_arg:
+; CHECK: braa x0, x1
+ %tmp0 = tail call i32 %arg0() [ "ptrauth"(i32 0, i64 %arg1) ]
+ ret i32 %tmp0
+}
+
+define i32 @test_tailcall_ib_arg(i32 ()* %arg0, i64 %arg1) #0 {
+; CHECK-LABEL: test_tailcall_ib_arg:
+; CHECK: brab x0, x1
+ %tmp0 = tail call i32 %arg0() [ "ptrauth"(i32 1, i64 %arg1) ]
+ ret i32 %tmp0
+}
+
+define i32 @test_call_ia_arg_ind(i32 ()** %arg0, i64 %arg1) #0 {
+; DARWIN-LABEL: test_call_ia_arg_ind:
+; DARWIN-NEXT: stp x29, x30, [sp, #-16]!
+; DARWIN-NEXT: ldr x8, [x0]
+; DARWIN-NEXT: blraa x8, x1
+; DARWIN-NEXT: ldp x29, x30, [sp], #16
+; DARWIN-NEXT: ret
+;
+; ELF-LABEL: test_call_ia_arg_ind:
+; ELF-NEXT: str x30, [sp, #-16]!
+; ELF-NEXT: ldr x8, [x0]
+; ELF-NEXT: blraa x8, x1
+; ELF-NEXT: ldr x30, [sp], #16
+; ELF-NEXT: ret
+ %tmp0 = load i32 ()*, i32 ()** %arg0
+ %tmp1 = call i32 %tmp0() [ "ptrauth"(i32 0, i64 %arg1) ]
+ ret i32 %tmp1
+}
+
+define i32 @test_call_ib_arg_ind(i32 ()** %arg0, i64 %arg1) #0 {
+; DARWIN-LABEL: test_call_ib_arg_ind:
+; DARWIN-NEXT: stp x29, x30, [sp, #-16]!
+; DARWIN-NEXT: ldr x8, [x0]
+; DARWIN-NEXT: blrab x8, x1
+; DARWIN-NEXT: ldp x29, x30, [sp], #16
+; DARWIN-NEXT: ret
+;
+; ELF-LABEL: test_call_ib_arg_ind:
+; ELF-NEXT: str x30, [sp, #-16]!
+; ELF-NEXT: ldr x8, [x0]
+; ELF-NEXT: blrab x8, x1
+; ELF-NEXT: ldr x30, [sp], #16
+; ELF-NEXT: ret
+ %tmp0 = load i32 ()*, i32 ()** %arg0
+ %tmp1 = call i32 %tmp0() [ "ptrauth"(i32 1, i64 %arg1) ]
+ ret i32 %tmp1
+}
+
+define i32 @test_tailcall_ia_arg_ind(i32 ()** %arg0, i64 %arg1) #0 {
+; CHECK-LABEL: test_tailcall_ia_arg_ind:
+; CHECK: ldr x0, [x0]
+; CHECK: braa x0, x1
+ %tmp0 = load i32 ()*, i32 ()** %arg0
+ %tmp1 = tail call i32 %tmp0() [ "ptrauth"(i32 0, i64 %arg1) ]
+ ret i32 %tmp1
+}
+
+define i32 @test_tailcall_ib_arg_ind(i32 ()** %arg0, i64 %arg1) #0 {
+; CHECK-LABEL: test_tailcall_ib_arg_ind:
+; CHECK: ldr x0, [x0]
+; CHECK: brab x0, x1
+ %tmp0 = load i32 ()*, i32 ()** %arg0
+ %tmp1 = tail call i32 %tmp0() [ "ptrauth"(i32 1, i64 %arg1) ]
+ ret i32 %tmp1
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/AArch64/ptrauth-invoke.ll b/llvm/test/CodeGen/AArch64/ptrauth-invoke.ll
new file mode 100644
index 0000000000000..fcd0ddb788336
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/ptrauth-invoke.ll
@@ -0,0 +1,276 @@
+; RUN: llc -mtriple arm64e-apple-darwin -o - %s \
+; RUN: | FileCheck %s --check-prefixes=CHECK,DARWIN,DARWIN-SDAG
+
+; RUN: llc -mtriple aarch64 -mattr=+pauth -o - %s \
+; RUN: | FileCheck %s --check-prefixes=CHECK,ELF,ELF-SDAG
+
+; RUN: llc -mtriple arm64e-apple-darwin -o - %s \
+; RUN: -global-isel -global-isel-abort=1 -verify-machineinstrs \
+; RUN: | FileCheck %s --check-prefixes=CHECK,DARWIN,DARWIN-GISEL
+
+; RUN: llc -mtriple aarch64 -mattr=+pauth -o - %s \
+; RUN: -global-isel -global-isel-abort=1 -verify-machineinstrs \
+; RUN: | FileCheck %s --check-prefixes=CHECK,ELF,ELF-GISEL
+
+; DARWIN-LABEL: _test_invoke_ia_0:
+; DARWIN-NEXT: [[FNBEGIN:L.*]]:
+; DARWIN-NEXT: .cfi_startproc
+; DARWIN-NEXT: .cfi_personality 155, ___gxx_personality_v0
+; DARWIN-NEXT: .cfi_lsda 16, [[EXCEPT:Lexception[0-9]+]]
+; DARWIN-NEXT: ; %bb.0:
+; DARWIN-NEXT: stp x20, x19, [sp, #-32]!
+; DARWIN-NEXT: stp x29, x30, [sp, #16]
+; DARWIN-NEXT: .cfi_def_cfa_offset 32
+; DARWIN-NEXT: .cfi_offset w30, -8
+; DARWIN-NEXT: .cfi_offset w29, -16
+; DARWIN-NEXT: .cfi_offset w19, -24
+; DARWIN-NEXT: .cfi_offset w20, -32
+; DARWIN-NEXT: [[PRECALL:L.*]]:
+; DARWIN-NEXT: blraaz x0
+
+; DARWIN-SDAG-NEXT: [[POSTCALL:L.*]]:
+; DARWIN-SDAG-NEXT: ; %bb.1:
+; DARWIN-SDAG-NEXT: mov x19, x0
+
+; DARWIN-GISEL-NEXT: mov x19, x0
+; DARWIN-GISEL-NEXT: [[POSTCALL:L.*]]:
+
+; DARWIN-NEXT: [[CALLBB:L.*]]:
+; DARWIN-NEXT: bl _foo
+; DARWIN-NEXT: mov x0, x19
+; DARWIN-NEXT: ldp x29, x30, [sp, #16]
+; DARWIN-NEXT: ldp x20, x19, [sp], #32
+; DARWIN-NEXT: ret
+; DARWIN-NEXT: [[LPADBB:LBB[0-9_]+]]:
+; DARWIN-NEXT: [[LPAD:L.*]]:
+; DARWIN-NEXT: mov w19, #-1
+; DARWIN-NEXT: b [[CALLBB]]
+
+; ELF-LABEL: test_invoke_ia_0:
+; ELF-NEXT: [[FNBEGIN:.L.*]]:
+; ELF-NEXT: .cfi_startproc
+; ELF-NEXT: .cfi_personality 156, DW.ref.__gxx_personality_v0
+; ELF-NEXT: .cfi_lsda 28, [[EXCEPT:.Lexception[0-9]+]]
+; ELF-NEXT: // %bb.0:
+; ELF-NEXT: stp x30, x19, [sp, #-16]!
+; ELF-NEXT: .cfi_def_cfa_offset 16
+; ELF-NEXT: .cfi_offset w19, -8
+; ELF-NEXT: .cfi_offset w30, -16
+; ELF-NEXT: [[PRECALL:.L.*]]:
+; ELF-NEXT: blraaz x0
+
+; ELF-SDAG-NEXT: [[POSTCALL:.L.*]]:
+; ELF-SDAG-NEXT: // %bb.1:
+; ELF-SDAG-NEXT: mov w19, w0
+
+; ELF-GISEL-NEXT: mov w19, w0
+; ELF-GISEL-NEXT: [[POSTCALL:.L.*]]:
+
+; ELF-NEXT: [[CALLBB:.L.*]]:
+; ELF-NEXT: bl foo
+; ELF-NEXT: mov w0, w19
+; ELF-NEXT: ldp x30, x19, [sp], #16
+; ELF-NEXT: ret
+; ELF-NEXT: [[LPADBB:.LBB[0-9_]+]]:
+; ELF-NEXT: [[LPAD:.L.*]]:
+; ELF-NEXT: mov w19, #-1
+; ELF-NEXT: b [[CALLBB]]
+
+; CHECK-LABEL: GCC_except_table{{.*}}:
+; CHECK-NEXT: [[EXCEPT]]:
+; CHECK: .uleb128 [[POSTCALL]]-[[PRECALL]] {{.*}} Call between [[PRECALL]] and [[POSTCALL]]
+; CHECK-NEXT: .uleb128 [[LPAD]]-[[FNBEGIN]] {{.*}} jumps to [[LPAD]]
+; CHECK-NEXT: .byte 0 {{.*}} On action: cleanup
+
+define i32 @test_invoke_ia_0(ptr %arg0) #0 personality ptr @__gxx_personality_v0 {
+ %tmp0 = invoke i32 %arg0() [ "ptrauth"(i32 0, i64 0) ] to label %continuebb
+ unwind label %unwindbb
+
+unwindbb:
+ %tmp1 = landingpad { ptr, i32 } cleanup
+ call void @foo()
+ ret i32 -1
+
+continuebb:
+ call void @foo()
+ ret i32 %tmp0
+}
+
+ at _ZTIPKc = external constant ptr
+ at hello_str = private unnamed_addr constant [6 x i8] c"hello\00", align 1
+
+; DARWIN-LABEL: _test_invoke_ib_42_catch:
+; DARWIN-NEXT: [[FNBEGIN:L.*]]:
+; DARWIN-NEXT: .cfi_startproc
+; DARWIN-NEXT: .cfi_personality 155, ___gxx_personality_v0
+; DARWIN-NEXT: .cfi_lsda 16, [[EXCEPT:Lexception[0-9]+]]
+; DARWIN-NEXT: ; %bb.0:
+; DARWIN-NEXT: stp x20, x19, [sp, #-32]!
+; DARWIN-NEXT: stp x29, x30, [sp, #16]
+; DARWIN-NEXT: .cfi_def_cfa_offset 32
+; DARWIN-NEXT: .cfi_offset w30, -8
+; DARWIN-NEXT: .cfi_offset w29, -16
+; DARWIN-NEXT: .cfi_offset w19, -24
+; DARWIN-NEXT: .cfi_offset w20, -32
+; DARWIN-NEXT: mov x19, x0
+; DARWIN-NEXT: mov w0, #8
+; DARWIN-NEXT: bl ___cxa_allocate_exception
+; DARWIN-NEXT: Lloh{{.*}}:
+; DARWIN-NEXT: adrp x8, l_hello_str at PAGE
+; DARWIN-NEXT: Lloh{{.*}}:
+; DARWIN-NEXT: add x8, x8, l_hello_str at PAGEOFF
+; DARWIN-NEXT: str x8, [x0]
+; DARWIN-NEXT: [[PRECALL:L.*]]:
+; DARWIN-NEXT: Lloh{{.*}}:
+; DARWIN-NEXT: adrp x1, __ZTIPKc at GOTPAGE
+; DARWIN-NEXT: Lloh{{.*}}:
+; DARWIN-NEXT: ldr x1, [x1, __ZTIPKc at GOTPAGEOFF]
+; DARWIN-NEXT: mov x2, #0
+; DARWIN-NEXT: mov x17, #42
+; DARWIN-NEXT: blrab x19, x17
+; DARWIN-NEXT: [[POSTCALL:L.*]]:
+; DARWIN-NEXT: ; %bb.1:
+; DARWIN-NEXT: brk #0x1
+; DARWIN-NEXT: [[LPADBB:LBB[0-9_]+]]:
+; DARWIN-NEXT: [[LPAD:L.*]]:
+; DARWIN-NEXT: mov x19, x1
+; DARWIN-NEXT: bl ___cxa_begin_catch
+; DARWIN-NEXT: cmp w19, #2
+; DARWIN-NEXT: b.ne [[EXITBB:LBB[0-9_]+]]
+; DARWIN-NEXT: ; %bb.3:
+; DARWIN-NEXT: bl _bar
+; DARWIN-NEXT: [[EXITBB]]:
+; DARWIN-NEXT: bl _foo
+; DARWIN-NEXT: bl ___cxa_end_catch
+; DARWIN-NEXT: ldp x29, x30, [sp, #16]
+; DARWIN-NEXT: ldp x20, x19, [sp], #32
+; DARWIN-NEXT: ret
+; DARWIN-NEXT: .loh {{.*}}
+; DARWIN-NEXT: .loh {{.*}}
+; DARWIN-NEXT: [[FNEND:L.*]]:
+
+; ELF-LABEL: test_invoke_ib_42_catch:
+; ELF-NEXT: [[FNBEGIN:.L.*]]:
+; ELF-NEXT: .cfi_startproc
+; ELF-NEXT: .cfi_personality 156, DW.ref.__gxx_personality_v0
+; ELF-NEXT: .cfi_lsda 28, [[EXCEPT:.Lexception[0-9]+]]
+; ELF-NEXT: // %bb.0:
+; ELF-NEXT: stp x30, x19, [sp, #-16]!
+; ELF-NEXT: .cfi_def_cfa_offset 16
+; ELF-NEXT: .cfi_offset w19, -8
+; ELF-NEXT: .cfi_offset w30, -16
+; ELF-NEXT: mov x19, x0
+; ELF-NEXT: mov w0, #8
+; ELF-NEXT: bl __cxa_allocate_exception
+; ELF-NEXT: adrp x8, .Lhello_str
+; ELF-NEXT: add x8, x8, :lo12:.Lhello_str
+; ELF-NEXT: str x8, [x0]
+; ELF-NEXT: [[PRECALL:.L.*]]:
+; ELF-NEXT: adrp x1, :got:_ZTIPKc
+; ELF-NEXT: mov x2, xzr
+; ELF-NEXT: ldr x1, [x1, :got_lo12:_ZTIPKc]
+; ELF-NEXT: mov x17, #42
+; ELF-NEXT: blrab x19, x17
+; ELF-NEXT: [[POSTCALL:.L.*]]:
+; ELF-NEXT: // %bb.1:
+; ELF-NEXT: [[LPADBB:.LBB[0-9_]+]]:
+; ELF-NEXT: [[LPAD:.L.*]]:
+; ELF-NEXT: mov x19, x1
+; ELF-NEXT: bl __cxa_begin_catch
+; ELF-NEXT: cmp w19, #2
+; ELF-NEXT: b.ne [[EXITBB:.LBB[0-9_]+]]
+; ELF-NEXT: // %bb.3:
+; ELF-NEXT: bl bar
+; ELF-NEXT: [[EXITBB]]:
+; ELF-NEXT: bl foo
+; ELF-NEXT: bl __cxa_end_catch
+; ELF-NEXT: ldp x30, x19, [sp], #16
+; ELF-NEXT: ret
+; ELF-NEXT: [[FNEND:.L.*]]:
+
+; CHECK-LABEL: GCC_except_table{{.*}}:
+; CHECK-NEXT: [[EXCEPT]]:
+; CHECK-NEXT: .byte 255 {{.*}} @LPStart Encoding = omit
+; DARWIN-NEXT: .byte 155 {{.*}} @TType Encoding = indirect pcrel sdata4
+; ELF-NEXT: .byte 156 {{.*}} @TType Encoding = indirect pcrel sdata8
+; CHECK-NEXT: .uleb128 [[TT:.?L.*]]-[[TTREF:.?L.*]]
+; CHECK-NEXT: [[TTREF]]:
+; CHECK-NEXT: .byte 1 {{.*}} Call site Encoding = uleb128
+; CHECK-NEXT: .uleb128 [[CSEND:.?L.*]]-[[CSBEGIN:.?L.*]]
+; CHECK-NEXT: [[CSBEGIN]]:
+; CHECK-NEXT: .uleb128 [[FNBEGIN]]-[[FNBEGIN]] {{.*}} >> Call Site 1 <<
+; CHECK-NEXT: .uleb128 [[PRECALL]]-[[FNBEGIN]] {{.*}} Call between [[FNBEGIN]] and [[PRECALL]]
+; CHECK-NEXT: .byte 0 {{.*}} has no landing pad
+; CHECK-NEXT: .byte 0 {{.*}} On action: cleanup
+; CHECK-NEXT: .uleb128 [[PRECALL]]-[[FNBEGIN]] {{.*}} >> Call Site 2 <<
+; CHECK-NEXT: .uleb128 [[POSTCALL]]-[[PRECALL]] {{.*}} Call between [[PRECALL]] and [[POSTCALL]]
+; CHECK-NEXT: .uleb128 [[LPAD]]-[[FNBEGIN]] {{.*}} jumps to [[LPAD]]
+; CHECK-NEXT: .byte 3 {{.*}} On action: 2
+; CHECK-NEXT: .uleb128 [[POSTCALL]]-[[FNBEGIN]] {{.*}} >> Call Site 3 <<
+; CHECK-NEXT: .uleb128 [[FNEND]]-[[POSTCALL]] {{.*}} Call between [[POSTCALL]] and [[FNEND]]
+; CHECK-NEXT: .byte 0 {{.*}} has no landing pad
+; CHECK-NEXT: .byte 0 {{.*}} On action: cleanup
+; CHECK-NEXT: [[CSEND]]:
+
+; CHECK-NEXT: .byte 1 {{.*}} >> Action Record 1 <<
+; CHECK-NEXT: {{.*}} Catch TypeInfo 1
+; CHECK-NEXT: .byte 0 {{.*}} No further actions
+; CHECK-NEXT: .byte 2 {{.*}} >> Action Record 2 <<
+; CHECK-NEXT: {{.*}} Catch TypeInfo 2
+; CHECK-NEXT: .byte 125 {{.*}} Continue to action 1
+; CHECK-NEXT: .p2align 2
+; CHECK-NEXT: {{.*}} >> Catch TypeInfos <<
+
+; DARWIN-NEXT: [[TI:L.*]]: {{.*}} TypeInfo 2
+; DARWIN-NEXT: .long __ZTIPKc at GOT-[[TI]]
+; DARWIN-NEXT: .long 0 {{.*}} TypeInfo 1
+; ELF-NEXT: [[TI:.?L.*]]: {{.*}} TypeInfo 2
+; ELF-NEXT: .xword .L_ZTIPKc.DW.stub-[[TI]]
+; ELF-NEXT: .xword 0 {{.*}} TypeInfo 1
+
+; CHECK-NEXT: [[TT]]:
+
+; ELF-LABEL: .L_ZTIPKc.DW.stub:
+; ELF-NEXT: .xword _ZTIPKc
+
+define void @test_invoke_ib_42_catch(ptr %fptr) #0 personality ptr @__gxx_personality_v0 {
+ %tmp0 = call ptr @__cxa_allocate_exception(i64 8)
+ store ptr getelementptr inbounds ([6 x i8], ptr @hello_str, i64 0, i64 0), ptr %tmp0, align 8
+ invoke void %fptr(ptr %tmp0, ptr @_ZTIPKc, ptr null) [ "ptrauth"(i32 1, i64 42) ]
+ to label %continuebb unwind label %catchbb
+
+catchbb:
+ %tmp2 = landingpad { ptr, i32 }
+ catch ptr @_ZTIPKc
+ catch ptr null
+ %tmp3 = extractvalue { ptr, i32 } %tmp2, 0
+ %tmp4 = extractvalue { ptr, i32 } %tmp2, 1
+ %tmp5 = call i32 @llvm.eh.typeid.for(ptr @_ZTIPKc)
+ %tmp6 = icmp eq i32 %tmp4, %tmp5
+ %tmp7 = call ptr @__cxa_begin_catch(ptr %tmp3)
+ br i1 %tmp6, label %PKc_catchbb, label %any_catchbb
+
+PKc_catchbb:
+ call void @bar(ptr %tmp7)
+ br label %any_catchbb
+
+any_catchbb:
+ call void @foo()
+ call void @__cxa_end_catch()
+ ret void
+
+continuebb:
+ unreachable
+}
+
+declare void @foo()
+declare void @bar(ptr)
+
+declare i32 @__gxx_personality_v0(...)
+declare ptr @__cxa_allocate_exception(i64)
+declare void @__cxa_throw(ptr, ptr, ptr)
+declare i32 @llvm.eh.typeid.for(ptr)
+declare ptr @__cxa_begin_catch(ptr)
+declare void @__cxa_end_catch()
+
+attributes #0 = { nounwind }
More information about the llvm-commits
mailing list