[llvm] ea0eec6 - IR+AArch64: add a "swiftasync" argument attribute.
Tim Northover via llvm-commits
llvm-commits at lists.llvm.org
Fri May 14 03:44:04 PDT 2021
Author: Tim Northover
Date: 2021-05-14T11:43:58+01:00
New Revision: ea0eec69f16e0f1b00fec413986e4e44f6f627fa
URL: https://github.com/llvm/llvm-project/commit/ea0eec69f16e0f1b00fec413986e4e44f6f627fa
DIFF: https://github.com/llvm/llvm-project/commit/ea0eec69f16e0f1b00fec413986e4e44f6f627fa.diff
LOG: IR+AArch64: add a "swiftasync" argument attribute.
This extends any frame record created in the function to include that
parameter, passed in X22.
The new record looks like [X22, FP, LR] in memory, and FP is stored with 0b0001
in bits 63:60 (CodeGen assumes they are 0b0000 in normal operation). The effect
of this is that tools walking the stack should expect to see one of three
values there:
* 0b0000 => a normal, non-extended record with just [FP, LR]
* 0b0001 => the extended record [X22, FP, LR]
* 0b1111 => kernel space, and a non-extended record.
All other values are currently reserved.
If compiling for arm64e this context pointer is address-discriminated with the
discriminator 0xc31a and the DB (process-specific) key.
There is also an "i8** @llvm.swift.async.context.addr()" intrinsic providing
front-ends access to this slot (and forcing its creation initialized to nullptr
if necessary).
Added:
llvm/test/CodeGen/AArch64/swift-async-reg.ll
llvm/test/CodeGen/AArch64/swift-async-unwind.ll
llvm/test/CodeGen/AArch64/swift-async.ll
llvm/test/Verifier/swiftasync.ll
Modified:
llvm/docs/LangRef.rst
llvm/include/llvm/AsmParser/LLToken.h
llvm/include/llvm/Bitcode/LLVMBitCodes.h
llvm/include/llvm/CodeGen/TargetCallingConv.h
llvm/include/llvm/CodeGen/TargetFrameLowering.h
llvm/include/llvm/CodeGen/TargetLowering.h
llvm/include/llvm/IR/Attributes.td
llvm/include/llvm/IR/Intrinsics.td
llvm/include/llvm/Target/TargetCallingConv.td
llvm/lib/AsmParser/LLLexer.cpp
llvm/lib/AsmParser/LLParser.cpp
llvm/lib/Bitcode/Reader/BitcodeReader.cpp
llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
llvm/lib/CodeGen/PrologEpilogInserter.cpp
llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
llvm/lib/IR/Attributes.cpp
llvm/lib/IR/Verifier.cpp
llvm/lib/Target/AArch64/AArch64CallingConvention.td
llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
llvm/lib/Target/AArch64/AArch64FastISel.cpp
llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
llvm/lib/Target/AArch64/AArch64FrameLowering.h
llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
llvm/lib/Target/AArch64/AArch64InstrFormats.td
llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
llvm/lib/Target/AArch64/AArch64InstrInfo.td
llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h
llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp
llvm/lib/Transforms/Utils/CodeExtractor.cpp
llvm/test/Bitcode/attributes.ll
llvm/test/Bitcode/compatibility.ll
Removed:
################################################################################
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index d85a5c3df392a..dfdf0c191ada3 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -1278,6 +1278,12 @@ Currently, only the following parameter attributes are defined:
a valid attribute for return values and can only be applied to one
parameter.
+``swiftasync``
+ This indicates that the parameter is the asynchronous context parameter and
+ triggers the creation of a target-specific extended frame record to store
+ this pointer. This is not a valid attribute for return values and can only
+ be applied to one parameter.
+
``swifterror``
This attribute is motivated to model and optimize Swift error handling. It
can be applied to a parameter with pointer to pointer type or a
@@ -12360,6 +12366,29 @@ Note that calling this intrinsic does not prevent function inlining or
other aggressive transformations, so the value returned may not be that
of the obvious source-language caller.
+'``llvm.swift.async.context.addr``' Intrinsic
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+
+::
+
+ declare i8** @llvm.swift.async.context.addr()
+
+Overview:
+"""""""""
+
+The '``llvm.swift.async.context.addr``' intrinsic returns a pointer to
+the part of the extended frame record containing the asynchronous
+context of a Swift execution.
+
+Semantics:
+""""""""""
+
+If the function has a ``swiftasync`` parameter, that argument will initially
+be stored at the returned address. If not, it will be initialized to null.
+
'``llvm.localescape``' and '``llvm.localrecover``' Intrinsics
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/llvm/include/llvm/AsmParser/LLToken.h b/llvm/include/llvm/AsmParser/LLToken.h
index b4b10c919ec74..e068d9079f9e2 100644
--- a/llvm/include/llvm/AsmParser/LLToken.h
+++ b/llvm/include/llvm/AsmParser/LLToken.h
@@ -239,6 +239,7 @@ enum Kind {
kw_strictfp,
kw_swifterror,
kw_swiftself,
+ kw_swiftasync,
kw_uwtable,
kw_vscale_range,
kw_willreturn,
diff --git a/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
index 5589b777ad41d..9bc6863e851e6 100644
--- a/llvm/include/llvm/Bitcode/LLVMBitCodes.h
+++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
@@ -665,6 +665,7 @@ enum AttributeKindCodes {
ATTR_KIND_HOT = 72,
ATTR_KIND_NO_PROFILE = 73,
ATTR_KIND_VSCALE_RANGE = 74,
+ ATTR_KIND_SWIFT_ASYNC = 75,
};
enum ComdatSelectionKindCodes {
diff --git a/llvm/include/llvm/CodeGen/TargetCallingConv.h b/llvm/include/llvm/CodeGen/TargetCallingConv.h
index 88785a8f45d26..29e644898f6be 100644
--- a/llvm/include/llvm/CodeGen/TargetCallingConv.h
+++ b/llvm/include/llvm/CodeGen/TargetCallingConv.h
@@ -39,6 +39,7 @@ namespace ISD {
unsigned IsPreallocated : 1; ///< ByVal without the copy
unsigned IsSplitEnd : 1; ///< Last part of a split
unsigned IsSwiftSelf : 1; ///< Swift self parameter
+ unsigned IsSwiftAsync : 1; ///< Swift async context parameter
unsigned IsSwiftError : 1; ///< Swift error parameter
unsigned IsCFGuardTarget : 1; ///< Control Flow Guard target
unsigned IsHva : 1; ///< HVA field for
@@ -58,11 +59,12 @@ namespace ISD {
public:
ArgFlagsTy()
- : IsZExt(0), IsSExt(0), IsInReg(0), IsSRet(0), IsByVal(0), IsByRef(0),
- IsNest(0), IsReturned(0), IsSplit(0), IsInAlloca(0), IsPreallocated(0),
- IsSplitEnd(0), IsSwiftSelf(0), IsSwiftError(0), IsCFGuardTarget(0),
- IsHva(0), IsHvaStart(0), IsSecArgPass(0), MemAlign(0),
- OrigAlign(0), IsInConsecutiveRegsLast(0), IsInConsecutiveRegs(0),
+ : IsZExt(0), IsSExt(0), IsInReg(0), IsSRet(0), IsByVal(0), IsByRef(0),
+ IsNest(0), IsReturned(0), IsSplit(0), IsInAlloca(0),
+ IsPreallocated(0), IsSplitEnd(0), IsSwiftSelf(0), IsSwiftAsync(0),
+ IsSwiftError(0), IsCFGuardTarget(0), IsHva(0), IsHvaStart(0),
+ IsSecArgPass(0), MemAlign(0), OrigAlign(0),
+ IsInConsecutiveRegsLast(0), IsInConsecutiveRegs(0),
IsCopyElisionCandidate(0), IsPointer(0), ByValOrByRefSize(0),
PointerAddrSpace(0) {
static_assert(sizeof(*this) == 3 * sizeof(unsigned), "flags are too big");
@@ -95,6 +97,9 @@ namespace ISD {
bool isSwiftSelf() const { return IsSwiftSelf; }
void setSwiftSelf() { IsSwiftSelf = 1; }
+ bool isSwiftAsync() const { return IsSwiftAsync; }
+ void setSwiftAsync() { IsSwiftAsync = 1; }
+
bool isSwiftError() const { return IsSwiftError; }
void setSwiftError() { IsSwiftError = 1; }
diff --git a/llvm/include/llvm/CodeGen/TargetFrameLowering.h b/llvm/include/llvm/CodeGen/TargetFrameLowering.h
index ec49d541a24e3..59dfe98125123 100644
--- a/llvm/include/llvm/CodeGen/TargetFrameLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetFrameLowering.h
@@ -149,6 +149,14 @@ class TargetFrameLowering {
/// returns false, spill slots will be assigned using generic implementation.
/// assignCalleeSavedSpillSlots() may add, delete or rearrange elements of
/// CSI.
+ virtual bool assignCalleeSavedSpillSlots(MachineFunction &MF,
+ const TargetRegisterInfo *TRI,
+ std::vector<CalleeSavedInfo> &CSI,
+ unsigned &MinCSFrameIndex,
+ unsigned &MaxCSFrameIndex) const {
+ return assignCalleeSavedSpillSlots(MF, TRI, CSI);
+ }
+
virtual bool
assignCalleeSavedSpillSlots(MachineFunction &MF,
const TargetRegisterInfo *TRI,
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index e72bc6982cca7..ea88dac91cd59 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -284,6 +284,7 @@ class TargetLoweringBase {
bool IsPreallocated : 1;
bool IsReturned : 1;
bool IsSwiftSelf : 1;
+ bool IsSwiftAsync : 1;
bool IsSwiftError : 1;
bool IsCFGuardTarget : 1;
MaybeAlign Alignment = None;
@@ -294,7 +295,7 @@ class TargetLoweringBase {
: IsSExt(false), IsZExt(false), IsInReg(false), IsSRet(false),
IsNest(false), IsByVal(false), IsByRef(false), IsInAlloca(false),
IsPreallocated(false), IsReturned(false), IsSwiftSelf(false),
- IsSwiftError(false), IsCFGuardTarget(false) {}
+ IsSwiftAsync(false), IsSwiftError(false), IsCFGuardTarget(false) {}
void setAttributes(const CallBase *Call, unsigned ArgIdx);
};
diff --git a/llvm/include/llvm/IR/Attributes.td b/llvm/include/llvm/IR/Attributes.td
index 9684ffa0009b1..d6223eff63029 100644
--- a/llvm/include/llvm/IR/Attributes.td
+++ b/llvm/include/llvm/IR/Attributes.td
@@ -244,6 +244,9 @@ def SwiftError : EnumAttr<"swifterror">;
/// Argument is swift self/context.
def SwiftSelf : EnumAttr<"swiftself">;
+/// Argument is swift async context.
+def SwiftAsync : EnumAttr<"swiftasync">;
+
/// Function must be in a unwind table.
def UWTable : EnumAttr<"uwtable">;
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index 2f5890d7f0754..665bee8d93e7e 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -479,7 +479,12 @@ def int_objc_arc_annotation_bottomup_bbstart : Intrinsic<[],
def int_objc_arc_annotation_bottomup_bbend : Intrinsic<[],
[llvm_ptrptr_ty,
llvm_ptrptr_ty]>;
+//===--------------- Swift asynchronous context intrinsics ----------------===//
+// Returns the location of the Swift asynchronous context (usually stored just
+// before the frame pointer), and triggers the creation of a null context if it
+// would otherwise be unneeded.
+def int_swift_async_context_addr : Intrinsic<[llvm_ptrptr_ty], [], [IntrNoMem]>;
//===--------------------- Code Generator Intrinsics ----------------------===//
//
diff --git a/llvm/include/llvm/Target/TargetCallingConv.td b/llvm/include/llvm/Target/TargetCallingConv.td
index b3d4fe9d0dbb8..a719581e0ac3c 100644
--- a/llvm/include/llvm/Target/TargetCallingConv.td
+++ b/llvm/include/llvm/Target/TargetCallingConv.td
@@ -51,6 +51,11 @@ class CCIfPreallocated<CCAction A> : CCIf<"ArgFlags.isPreallocated()", A> {
class CCIfSwiftSelf<CCAction A> : CCIf<"ArgFlags.isSwiftSelf()", A> {
}
+/// CCIfSwiftAsync - If the current argument has swiftasync parameter attribute,
+/// apply Action A.
+class CCIfSwiftAsync<CCAction A> : CCIf<"ArgFlags.isSwiftAsync()", A> {
+}
+
/// CCIfSwiftError - If the current argument has swifterror parameter attribute,
/// apply Action A.
class CCIfSwiftError<CCAction A> : CCIf<"ArgFlags.isSwiftError()", A> {
diff --git a/llvm/lib/AsmParser/LLLexer.cpp b/llvm/lib/AsmParser/LLLexer.cpp
index 67227f05f3609..4374982bef8ce 100644
--- a/llvm/lib/AsmParser/LLLexer.cpp
+++ b/llvm/lib/AsmParser/LLLexer.cpp
@@ -696,6 +696,7 @@ lltok::Kind LLLexer::LexIdentifier() {
KEYWORD(speculative_load_hardening);
KEYWORD(swifterror);
KEYWORD(swiftself);
+ KEYWORD(swiftasync);
KEYWORD(uwtable);
KEYWORD(vscale_range);
KEYWORD(willreturn);
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index b8ea14a5efc9d..f4546b2662a1a 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -1460,6 +1460,7 @@ bool LLParser::parseFnAttributeValuePairs(AttrBuilder &B,
case lltok::kw_sret:
case lltok::kw_swifterror:
case lltok::kw_swiftself:
+ case lltok::kw_swiftasync:
case lltok::kw_immarg:
case lltok::kw_byref:
HaveError |=
@@ -1799,6 +1800,7 @@ bool LLParser::parseOptionalParamAttrs(AttrBuilder &B) {
case lltok::kw_signext: B.addAttribute(Attribute::SExt); break;
case lltok::kw_swifterror: B.addAttribute(Attribute::SwiftError); break;
case lltok::kw_swiftself: B.addAttribute(Attribute::SwiftSelf); break;
+ case lltok::kw_swiftasync: B.addAttribute(Attribute::SwiftAsync); break;
case lltok::kw_writeonly: B.addAttribute(Attribute::WriteOnly); break;
case lltok::kw_zeroext: B.addAttribute(Attribute::ZExt); break;
case lltok::kw_immarg: B.addAttribute(Attribute::ImmArg); break;
@@ -1905,6 +1907,7 @@ bool LLParser::parseOptionalReturnAttrs(AttrBuilder &B) {
case lltok::kw_sret:
case lltok::kw_swifterror:
case lltok::kw_swiftself:
+ case lltok::kw_swiftasync:
case lltok::kw_immarg:
case lltok::kw_byref:
HaveError |=
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index 3a56a67c4a886..25edf24037cb3 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -1524,6 +1524,8 @@ static Attribute::AttrKind getAttrFromCode(uint64_t Code) {
return Attribute::SwiftError;
case bitc::ATTR_KIND_SWIFT_SELF:
return Attribute::SwiftSelf;
+ case bitc::ATTR_KIND_SWIFT_ASYNC:
+ return Attribute::SwiftAsync;
case bitc::ATTR_KIND_UW_TABLE:
return Attribute::UWTable;
case bitc::ATTR_KIND_VSCALE_RANGE:
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index 50fb3d4afac55..c5ff5619cd909 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -736,6 +736,8 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) {
return bitc::ATTR_KIND_SWIFT_ERROR;
case Attribute::SwiftSelf:
return bitc::ATTR_KIND_SWIFT_SELF;
+ case Attribute::SwiftAsync:
+ return bitc::ATTR_KIND_SWIFT_ASYNC;
case Attribute::UWTable:
return bitc::ATTR_KIND_UW_TABLE;
case Attribute::VScaleRange:
diff --git a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
index 194d824e27d34..d0db5e2ee31fb 100644
--- a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
@@ -54,6 +54,8 @@ addFlagsUsingAttrFn(ISD::ArgFlagsTy &Flags,
Flags.setReturned();
if (AttrFn(Attribute::SwiftSelf))
Flags.setSwiftSelf();
+ if (AttrFn(Attribute::SwiftAsync))
+ Flags.setSwiftAsync();
if (AttrFn(Attribute::SwiftError))
Flags.setSwiftError();
}
diff --git a/llvm/lib/CodeGen/PrologEpilogInserter.cpp b/llvm/lib/CodeGen/PrologEpilogInserter.cpp
index cccab5fe03fcf..c12417a2cbf2c 100644
--- a/llvm/lib/CodeGen/PrologEpilogInserter.cpp
+++ b/llvm/lib/CodeGen/PrologEpilogInserter.cpp
@@ -399,7 +399,8 @@ static void assignCalleeSavedSpillSlots(MachineFunction &F,
const TargetFrameLowering *TFI = F.getSubtarget().getFrameLowering();
MachineFrameInfo &MFI = F.getFrameInfo();
- if (!TFI->assignCalleeSavedSpillSlots(F, RegInfo, CSI)) {
+ if (!TFI->assignCalleeSavedSpillSlots(F, RegInfo, CSI, MinCSFrameIndex,
+ MaxCSFrameIndex)) {
// If target doesn't implement this, use generic code.
if (CSI.empty())
@@ -677,10 +678,12 @@ computeFreeStackSlots(MachineFrameInfo &MFI, bool StackGrowsDown,
// StackSlot scavenging is only implemented for the default stack.
if (MFI.getStackID(i) == TargetStackID::Default)
AllocatedFrameSlots.push_back(i);
- // Add callee-save objects.
- for (int i = MinCSFrameIndex; i <= (int)MaxCSFrameIndex; ++i)
- if (MFI.getStackID(i) == TargetStackID::Default)
- AllocatedFrameSlots.push_back(i);
+ // Add callee-save objects if there are any.
+ if (MinCSFrameIndex <= MaxCSFrameIndex) {
+ for (int i = MinCSFrameIndex; i <= (int)MaxCSFrameIndex; ++i)
+ if (MFI.getStackID(i) == TargetStackID::Default)
+ AllocatedFrameSlots.push_back(i);
+ }
for (int i : AllocatedFrameSlots) {
// These are converted from int64_t, but they should always fit in int
@@ -833,7 +836,7 @@ void PEI::calculateFrameObjectOffsets(MachineFunction &MF) {
// First assign frame offsets to stack objects that are used to spill
// callee saved registers.
- if (StackGrowsDown) {
+ if (StackGrowsDown && MaxCSFrameIndex >= MinCSFrameIndex) {
for (unsigned i = MinCSFrameIndex; i <= MaxCSFrameIndex; ++i) {
if (MFI.getStackID(i) !=
TargetStackID::Default) // Only allocate objects on the default stack.
diff --git a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
index 796d4a6ed7bba..514cfc6632a34 100644
--- a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
@@ -1048,6 +1048,8 @@ bool FastISel::lowerCallTo(CallLoweringInfo &CLI) {
Flags.setSRet();
if (Arg.IsSwiftSelf)
Flags.setSwiftSelf();
+ if (Arg.IsSwiftAsync)
+ Flags.setSwiftAsync();
if (Arg.IsSwiftError)
Flags.setSwiftError();
if (Arg.IsCFGuardTarget)
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index e9eea5b2ebbf3..7d6d1ea950f2f 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -9385,6 +9385,7 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
Entry.IsByRef = false;
Entry.IsReturned = false;
Entry.IsSwiftSelf = false;
+ Entry.IsSwiftAsync = false;
Entry.IsSwiftError = false;
Entry.IsCFGuardTarget = false;
Entry.Alignment = Alignment;
@@ -9498,6 +9499,8 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
Flags.setSRet();
if (Args[i].IsSwiftSelf)
Flags.setSwiftSelf();
+ if (Args[i].IsSwiftAsync)
+ Flags.setSwiftAsync();
if (Args[i].IsSwiftError)
Flags.setSwiftError();
if (Args[i].IsCFGuardTarget)
@@ -10035,6 +10038,8 @@ void SelectionDAGISel::LowerArguments(const Function &F) {
Flags.setSRet();
if (Arg.hasAttribute(Attribute::SwiftSelf))
Flags.setSwiftSelf();
+ if (Arg.hasAttribute(Attribute::SwiftAsync))
+ Flags.setSwiftAsync();
if (Arg.hasAttribute(Attribute::SwiftError))
Flags.setSwiftError();
if (Arg.hasAttribute(Attribute::ByVal))
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 53105dd04aa66..5ff65af95565b 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -115,6 +115,7 @@ void TargetLoweringBase::ArgListEntry::setAttributes(const CallBase *Call,
IsNest = Attrs.hasParamAttribute(ArgIdx, Attribute::Nest);
IsReturned = Attrs.hasParamAttribute(ArgIdx, Attribute::Returned);
IsSwiftSelf = Attrs.hasParamAttribute(ArgIdx, Attribute::SwiftSelf);
+ IsSwiftAsync = Attrs.hasParamAttr(ArgIdx, Attribute::SwiftAsync);
IsSwiftError = Attrs.hasParamAttribute(ArgIdx, Attribute::SwiftError);
Alignment = Attrs.getParamStackAlignment(ArgIdx);
diff --git a/llvm/lib/IR/Attributes.cpp b/llvm/lib/IR/Attributes.cpp
index d4bf3b1aa44ce..adf8e6df6ce19 100644
--- a/llvm/lib/IR/Attributes.cpp
+++ b/llvm/lib/IR/Attributes.cpp
@@ -384,6 +384,8 @@ std::string Attribute::getAsString(bool InAttrGrp) const {
return "swifterror";
if (hasAttribute(Attribute::SwiftSelf))
return "swiftself";
+ if (hasAttribute(Attribute::SwiftAsync))
+ return "swiftasync";
if (hasAttribute(Attribute::InaccessibleMemOnly))
return "inaccessiblememonly";
if (hasAttribute(Attribute::InaccessibleMemOrArgMemOnly))
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 1fb2efb1b14c7..da197f3cd11e0 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -1910,6 +1910,7 @@ void Verifier::verifyFunctionAttrs(FunctionType *FT, AttributeList Attrs,
bool SawReturned = false;
bool SawSRet = false;
bool SawSwiftSelf = false;
+ bool SawSwiftAsync = false;
bool SawSwiftError = false;
// Verify return value attributes.
@@ -1924,11 +1925,12 @@ void Verifier::verifyFunctionAttrs(FunctionType *FT, AttributeList Attrs,
!RetAttrs.hasAttribute(Attribute::Preallocated) &&
!RetAttrs.hasAttribute(Attribute::ByRef) &&
!RetAttrs.hasAttribute(Attribute::SwiftSelf) &&
+ !RetAttrs.hasAttribute(Attribute::SwiftAsync) &&
!RetAttrs.hasAttribute(Attribute::SwiftError)),
"Attributes 'byval', 'inalloca', 'preallocated', 'byref', "
"'nest', 'sret', 'nocapture', 'nofree', "
- "'returned', 'swiftself', and 'swifterror' do not apply to return "
- "values!",
+ "'returned', 'swiftself', 'swiftasync', and 'swifterror'"
+ " do not apply to return values!",
V);
Assert((!RetAttrs.hasAttribute(Attribute::ReadOnly) &&
!RetAttrs.hasAttribute(Attribute::WriteOnly) &&
@@ -1976,6 +1978,11 @@ void Verifier::verifyFunctionAttrs(FunctionType *FT, AttributeList Attrs,
SawSwiftSelf = true;
}
+ if (ArgAttrs.hasAttribute(Attribute::SwiftAsync)) {
+ Assert(!SawSwiftAsync, "Cannot have multiple 'swiftasync' parameters!", V);
+ SawSwiftAsync = true;
+ }
+
if (ArgAttrs.hasAttribute(Attribute::SwiftError)) {
Assert(!SawSwiftError, "Cannot have multiple 'swifterror' parameters!",
V);
@@ -3370,9 +3377,10 @@ static bool isTypeCongruent(Type *L, Type *R) {
static AttrBuilder getParameterABIAttributes(int I, AttributeList Attrs) {
static const Attribute::AttrKind ABIAttrs[] = {
- Attribute::StructRet, Attribute::ByVal, Attribute::InAlloca,
- Attribute::InReg, Attribute::SwiftSelf, Attribute::SwiftError,
- Attribute::Preallocated, Attribute::ByRef, Attribute::StackAlignment};
+ Attribute::StructRet, Attribute::ByVal, Attribute::InAlloca,
+ Attribute::InReg, Attribute::SwiftSelf, Attribute::SwiftAsync,
+ Attribute::SwiftError, Attribute::SwiftAsync, Attribute::Preallocated,
+ Attribute::ByRef};
AttrBuilder Copy;
for (auto AK : ABIAttrs) {
if (Attrs.hasParamAttribute(I, AK))
diff --git a/llvm/lib/Target/AArch64/AArch64CallingConvention.td b/llvm/lib/Target/AArch64/AArch64CallingConvention.td
index 6277417401fe6..0cfddb55a06c9 100644
--- a/llvm/lib/Target/AArch64/AArch64CallingConvention.td
+++ b/llvm/lib/Target/AArch64/AArch64CallingConvention.td
@@ -69,6 +69,10 @@ def CC_AArch64_AAPCS : CallingConv<[
// A SwiftError is passed in X21.
CCIfSwiftError<CCIfType<[i64], CCAssignToRegWithShadow<[X21], [W21]>>>,
+ // Pass SwiftAsync in an otherwise callee saved register so that it will be
+ // preserved for normal function calls.
+ CCIfSwiftAsync<CCIfType<[i64], CCAssignToRegWithShadow<[X22], [W22]>>>,
+
CCIfConsecutiveRegs<CCCustom<"CC_AArch64_Custom_Block">>,
CCIfType<[nxv16i8, nxv8i16, nxv4i32, nxv2i64, nxv2f16, nxv4f16, nxv8f16,
@@ -203,6 +207,10 @@ def CC_AArch64_DarwinPCS : CallingConv<[
// A SwiftError is passed in X21.
CCIfSwiftError<CCIfType<[i64], CCAssignToRegWithShadow<[X21], [W21]>>>,
+ // Pass SwiftAsync in an otherwise callee saved register so that it will be
+ // preserved for normal function calls.
+ CCIfSwiftAsync<CCIfType<[i64], CCAssignToRegWithShadow<[X22], [W22]>>>,
+
CCIfConsecutiveRegs<CCCustom<"CC_AArch64_Custom_Block">>,
// Handle i1, i8, i16, i32, i64, f32, f64 and v2f64 by passing in registers,
diff --git a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
index 5f2c2b0900def..4d303876ace24 100644
--- a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
@@ -86,6 +86,8 @@ class AArch64ExpandPseudo : public MachineFunctionPass {
unsigned N);
bool expandCALL_RVMARKER(MachineBasicBlock &MBB,
MachineBasicBlock::iterator MBBI);
+ bool expandStoreSwiftAsyncContext(MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator MBBI);
};
} // end anonymous namespace
@@ -696,6 +698,63 @@ bool AArch64ExpandPseudo::expandCALL_RVMARKER(
return true;
}
+bool AArch64ExpandPseudo::expandStoreSwiftAsyncContext(
+ MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) {
+ Register CtxReg = MBBI->getOperand(0).getReg();
+ Register BaseReg = MBBI->getOperand(1).getReg();
+ int Offset = MBBI->getOperand(2).getImm();
+ DebugLoc DL(MBBI->getDebugLoc());
+ auto &STI = MBB.getParent()->getSubtarget<AArch64Subtarget>();
+
+ if (STI.getTargetTriple().getArchName() != "arm64e") {
+ BuildMI(MBB, MBBI, DL, TII->get(AArch64::STRXui))
+ .addUse(CtxReg)
+ .addUse(BaseReg)
+ .addImm(Offset / 8)
+ .setMIFlag(MachineInstr::FrameSetup);
+ MBBI->eraseFromParent();
+ return true;
+ }
+
+ // We need to sign the context in an address-discriminated way. 0xc31a is a
+ // fixed random value, chosen as part of the ABI.
+ // add x16, xBase, #Offset
+ // movk x16, #0xc31a, lsl #48
+ // mov x17, x22/xzr
+ // pacdb x17, x16
+ // str x17, [xBase, #Offset]
+ unsigned Opc = Offset >= 0 ? AArch64::ADDXri : AArch64::SUBXri;
+ BuildMI(MBB, MBBI, DL, TII->get(Opc), AArch64::X16)
+ .addUse(BaseReg)
+ .addImm(abs(Offset))
+ .addImm(0)
+ .setMIFlag(MachineInstr::FrameSetup);
+ BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVKXi), AArch64::X16)
+ .addUse(AArch64::X16)
+ .addImm(0xc31a)
+ .addImm(48)
+ .setMIFlag(MachineInstr::FrameSetup);
+ // We're not allowed to clobber X22 (and couldn't clobber XZR if we tried), so
+ // move it somewhere before signing.
+ BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORRXrs), AArch64::X17)
+ .addUse(AArch64::XZR)
+ .addUse(CtxReg)
+ .addImm(0)
+ .setMIFlag(MachineInstr::FrameSetup);
+ BuildMI(MBB, MBBI, DL, TII->get(AArch64::PACDB), AArch64::X17)
+ .addUse(AArch64::X17)
+ .addUse(AArch64::X16)
+ .setMIFlag(MachineInstr::FrameSetup);
+ BuildMI(MBB, MBBI, DL, TII->get(AArch64::STRXui))
+ .addUse(AArch64::X17)
+ .addUse(BaseReg)
+ .addImm(Offset / 8)
+ .setMIFlag(MachineInstr::FrameSetup);
+
+ MBBI->eraseFromParent();
+ return true;
+}
+
/// If MBBI references a pseudo instruction that should be expanded here,
/// do the expansion and return true. Otherwise return false.
bool AArch64ExpandPseudo::expandMI(MachineBasicBlock &MBB,
@@ -1110,6 +1169,8 @@ bool AArch64ExpandPseudo::expandMI(MachineBasicBlock &MBB,
return expandSVESpillFill(MBB, MBBI, AArch64::LDR_ZXI, 2);
case AArch64::BLR_RVMARKER:
return expandCALL_RVMARKER(MBB, MBBI);
+ case AArch64::StoreSwiftAsyncContext:
+ return expandStoreSwiftAsyncContext(MBB, MBBI);
}
return false;
}
diff --git a/llvm/lib/Target/AArch64/AArch64FastISel.cpp b/llvm/lib/Target/AArch64/AArch64FastISel.cpp
index ab1fafddbb25e..88f0b545c67cc 100644
--- a/llvm/lib/Target/AArch64/AArch64FastISel.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FastISel.cpp
@@ -2899,6 +2899,7 @@ bool AArch64FastISel::fastLowerArguments() {
Arg.hasAttribute(Attribute::InReg) ||
Arg.hasAttribute(Attribute::StructRet) ||
Arg.hasAttribute(Attribute::SwiftSelf) ||
+ Arg.hasAttribute(Attribute::SwiftAsync) ||
Arg.hasAttribute(Attribute::SwiftError) ||
Arg.hasAttribute(Attribute::Nest))
return false;
@@ -3157,7 +3158,7 @@ bool AArch64FastISel::fastLowerCall(CallLoweringInfo &CLI) {
for (auto Flag : CLI.OutFlags)
if (Flag.isInReg() || Flag.isSRet() || Flag.isNest() || Flag.isByVal() ||
- Flag.isSwiftSelf() || Flag.isSwiftError())
+ Flag.isSwiftSelf() || Flag.isSwiftAsync() || Flag.isSwiftError())
return false;
// Set up the argument vectors.
diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
index 06cc68155c374..d3b638b5a2b4b 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
@@ -47,8 +47,9 @@
// | callee-saved gpr registers | <--.
// | | | On Darwin platforms these
// |- - - - - - - - - - - - - - - - - -| | callee saves are swapped,
-// | | | (frame record first)
-// | prev_fp, prev_lr | <--'
+// | prev_lr | | (frame record first)
+// | prev_fp | <--'
+// | async context if needed |
// | (a.k.a. "frame record") |
// |-----------------------------------| <- fp(=x29)
// | |
@@ -940,6 +941,16 @@ static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec(
SEH->eraseFromParent();
}
+ // If the first store isn't right where we want SP then we can't fold the
+ // update in so create a normal arithmetic instruction instead.
+ if (MBBI->getOperand(MBBI->getNumOperands() - 1).getImm() != 0) {
+ emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP,
+ StackOffset::getFixed(CSStackSizeInc), TII,
+ InProlog ? MachineInstr::FrameSetup
+ : MachineInstr::FrameDestroy);
+ return std::prev(MBBI);
+ }
+
MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(NewOpc));
MIB.addReg(AArch64::SP, RegState::Define);
@@ -1128,6 +1139,18 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
.setMIFlags(MachineInstr::FrameSetup);
}
+ // We signal the presence of a Swift extended frame to external tools by
+ // storing FP with 0b0001 in bits 63:60. In normal userland operation a simple
+ // ORR is sufficient, it is assumed a Swift kernel would initialize the TBI
+ // bits so that is still true.
+ if (HasFP && AFI->hasSwiftAsyncContext()) {
+ // ORR x29, x29, #0x1000_0000_0000_0000
+ BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORRXri), AArch64::FP)
+ .addUse(AArch64::FP)
+ .addImm(0x1100)
+ .setMIFlag(MachineInstr::FrameSetup);
+ }
+
// All calls are tail calls in GHC calling conv, and functions have no
// prologue/epilogue.
if (MF.getFunction().getCallingConv() == CallingConv::GHC)
@@ -1234,6 +1257,20 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
if (CombineSPBump)
FPOffset += AFI->getLocalStackSize();
+ if (AFI->hasSwiftAsyncContext()) {
+ // Before we update the live FP we have to ensure there's a valid (or
+ // null) asynchronous context in its slot just before FP in the frame
+ // record, so store it now.
+ const auto &Attrs = MF.getFunction().getAttributes();
+ bool HaveInitialContext = Attrs.hasAttrSomewhere(Attribute::SwiftAsync);
+
+ BuildMI(MBB, MBBI, DL, TII->get(AArch64::StoreSwiftAsyncContext))
+ .addUse(HaveInitialContext ? AArch64::X22 : AArch64::XZR)
+ .addUse(AArch64::SP)
+ .addImm(FPOffset - 8)
+ .setMIFlags(MachineInstr::FrameSetup);
+ }
+
if (HomPrologEpilog) {
auto Prolog = MBBI;
--Prolog;
@@ -1745,6 +1782,18 @@ void AArch64FrameLowering::emitEpilogue(MachineFunction &MF,
.setMIFlag(MachineInstr::FrameDestroy);
}
+ if (hasFP(MF) && AFI->hasSwiftAsyncContext()) {
+ // We need to reset FP to its untagged state on return. Bit 60 is currently
+ // used to show the presence of an extended frame.
+
+ // BIC x29, x29, #0x1000_0000_0000_0000
+ BuildMI(MBB, MBB.getFirstTerminator(), DL, TII->get(AArch64::ANDXri),
+ AArch64::FP)
+ .addUse(AArch64::FP)
+ .addImm(0x10fe)
+ .setMIFlag(MachineInstr::FrameDestroy);
+ }
+
const StackOffset &SVEStackSize = getSVEStackSize(MF);
// If there is a single SP update, insert it before the ret and we're done.
@@ -2309,6 +2358,12 @@ static void computeCalleeSaveRegisterPairs(
else
ByteOffset += StackFillDir * (RPI.isPaired() ? 2 * Scale : Scale);
+ // Swift's async context is directly before FP, so allocate an extra
+ // 8 bytes for it.
+ if (NeedsFrameRecord && AFI->hasSwiftAsyncContext() &&
+ RPI.Reg2 == AArch64::FP)
+ ByteOffset += StackFillDir * 8;
+
assert(!(RPI.isScalable() && RPI.isPaired()) &&
"Paired spill/fill instructions don't exist for SVE vectors");
@@ -2332,6 +2387,12 @@ static void computeCalleeSaveRegisterPairs(
// If filling top down (default), we want the offset after incrementing it.
// If fillibg bootom up (WinCFI) we need the original offset.
int Offset = NeedsWinCFI ? OffsetPre : OffsetPost;
+
+ // The FP, LR pair goes 8 bytes into our expanded 24-byte slot so that the
+ // Swift context can directly precede FP.
+ if (NeedsFrameRecord && AFI->hasSwiftAsyncContext() &&
+ RPI.Reg2 == AArch64::FP)
+ Offset += 8;
RPI.Offset = Offset / Scale;
assert(((!RPI.isScalable() && RPI.Offset >= -64 && RPI.Offset <= 63) ||
@@ -2800,6 +2861,12 @@ void AArch64FrameLowering::determineCalleeSaves(MachineFunction &MF,
// Adding the size of additional 64bit GPR saves.
CSStackSize += 8 * (SavedRegs.count() - NumSavedRegs);
+
+ // A Swift asynchronous context extends the frame record with a pointer
+ // directly before FP.
+ if (hasFP(MF) && AFI->hasSwiftAsyncContext())
+ CSStackSize += 8;
+
uint64_t AlignedCSStackSize = alignTo(CSStackSize, 16);
LLVM_DEBUG(dbgs() << "Estimated stack frame size: "
<< EstimatedStackSize + AlignedCSStackSize
@@ -2817,8 +2884,9 @@ void AArch64FrameLowering::determineCalleeSaves(MachineFunction &MF,
}
bool AArch64FrameLowering::assignCalleeSavedSpillSlots(
- MachineFunction &MF, const TargetRegisterInfo *TRI,
- std::vector<CalleeSavedInfo> &CSI) const {
+ MachineFunction &MF, const TargetRegisterInfo *RegInfo,
+ std::vector<CalleeSavedInfo> &CSI, unsigned &MinCSFrameIndex,
+ unsigned &MaxCSFrameIndex) const {
bool NeedsWinCFI = needsWinCFI(MF);
// To match the canonical windows frame layout, reverse the list of
// callee saved registers to get them laid out by PrologEpilogInserter
@@ -2827,8 +2895,35 @@ bool AArch64FrameLowering::assignCalleeSavedSpillSlots(
// the top, thus have the CSI array start from the highest registers.)
if (NeedsWinCFI)
std::reverse(CSI.begin(), CSI.end());
- // Let the generic code do the rest of the setup.
- return false;
+
+ if (CSI.empty())
+ return true; // Early exit if no callee saved registers are modified!
+
+ // Now that we know which registers need to be saved and restored, allocate
+ // stack slots for them.
+ MachineFrameInfo &MFI = MF.getFrameInfo();
+ auto *AFI = MF.getInfo<AArch64FunctionInfo>();
+ for (auto &CS : CSI) {
+ Register Reg = CS.getReg();
+ const TargetRegisterClass *RC = RegInfo->getMinimalPhysRegClass(Reg);
+
+ unsigned Size = RegInfo->getSpillSize(*RC);
+ Align Alignment(RegInfo->getSpillAlign(*RC));
+ int FrameIdx = MFI.CreateStackObject(Size, Alignment, true);
+ CS.setFrameIdx(FrameIdx);
+
+ if ((unsigned)FrameIdx < MinCSFrameIndex) MinCSFrameIndex = FrameIdx;
+ if ((unsigned)FrameIdx > MaxCSFrameIndex) MaxCSFrameIndex = FrameIdx;
+
+ // Grab 8 bytes below FP for the extended asynchronous frame info.
+ if (hasFP(MF) && AFI->hasSwiftAsyncContext() && Reg == AArch64::FP) {
+ FrameIdx = MFI.CreateStackObject(8, Alignment, true);
+ AFI->setSwiftAsyncContextFrameIdx(FrameIdx);
+ if ((unsigned)FrameIdx < MinCSFrameIndex) MinCSFrameIndex = FrameIdx;
+ if ((unsigned)FrameIdx > MaxCSFrameIndex) MaxCSFrameIndex = FrameIdx;
+ }
+ }
+ return true;
}
bool AArch64FrameLowering::enableStackSlotScavenging(
diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.h b/llvm/lib/Target/AArch64/AArch64FrameLowering.h
index c5b4ad48ddd84..88acc96023012 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.h
@@ -67,10 +67,13 @@ class AArch64FrameLowering : public TargetFrameLowering {
bool hasFP(const MachineFunction &MF) const override;
bool hasReservedCallFrame(const MachineFunction &MF) const override;
- bool
- assignCalleeSavedSpillSlots(MachineFunction &MF,
- const TargetRegisterInfo *TRI,
- std::vector<CalleeSavedInfo> &CSI) const override;
+ bool hasSwiftExtendedFrame(const MachineFunction &MF) const;
+
+ bool assignCalleeSavedSpillSlots(MachineFunction &MF,
+ const TargetRegisterInfo *TRI,
+ std::vector<CalleeSavedInfo> &CSI,
+ unsigned &MinCSFrameIndex,
+ unsigned &MaxCSFrameIndex) const override;
void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs,
RegScavenger *RS) const override;
diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index c3d6d315cd365..2d529d5d74a33 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -3901,6 +3901,18 @@ void AArch64DAGToDAGISel::Select(SDNode *Node) {
if (tryMULLV64LaneV128(IntNo, Node))
return;
break;
+ case Intrinsic::swift_async_context_addr: {
+ SDLoc DL(Node);
+ CurDAG->SelectNodeTo(Node, AArch64::SUBXri, MVT::i64,
+ CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL,
+ AArch64::FP, MVT::i64),
+ CurDAG->getTargetConstant(8, DL, MVT::i32),
+ CurDAG->getTargetConstant(0, DL, MVT::i32));
+ auto &MF = CurDAG->getMachineFunction();
+ MF.getFrameInfo().setFrameAddressIsTaken(true);
+ MF.getInfo<AArch64FunctionInfo>()->setHasSwiftAsyncContext(true);
+ return;
+ }
}
break;
}
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index ed113d8072e15..735f9a3ee1496 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -4810,6 +4810,9 @@ SDValue AArch64TargetLowering::LowerFormalArguments(
continue;
}
+ if (Ins[i].Flags.isSwiftAsync())
+ MF.getInfo<AArch64FunctionInfo>()->setHasSwiftAsyncContext(true);
+
SDValue ArgValue;
if (VA.isRegLoc()) {
// Arguments stored in registers.
diff --git a/llvm/lib/Target/AArch64/AArch64InstrFormats.td b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
index 66d3963000ad0..9d3f467e74d89 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrFormats.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
@@ -1957,7 +1957,8 @@ class OneXRegData<bits<3> opc, string asm, SDPatternOperator node>
}
class SignAuthOneData<bits<3> opcode_prefix, bits<2> opcode, string asm>
- : I<(outs GPR64:$Rd), (ins GPR64sp:$Rn), asm, "\t$Rd, $Rn", "",
+ : I<(outs GPR64:$Rd), (ins GPR64:$src, GPR64sp:$Rn), asm, "\t$Rd, $Rn",
+ "$Rd = $src",
[]>,
Sched<[WriteI, ReadI]> {
bits<5> Rd;
@@ -1970,7 +1971,8 @@ class SignAuthOneData<bits<3> opcode_prefix, bits<2> opcode, string asm>
}
class SignAuthZero<bits<3> opcode_prefix, bits<2> opcode, string asm>
- : I<(outs GPR64:$Rd), (ins), asm, "\t$Rd", "", []>, Sched<[]> {
+ : I<(outs GPR64:$Rd), (ins GPR64:$src), asm, "\t$Rd", "$Rd = $src",
+ []>, Sched<[]> {
bits<5> Rd;
let Inst{31-15} = 0b11011010110000010;
let Inst{14-12} = opcode_prefix;
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index 6c904712d2eac..be01f1684eeea 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -135,6 +135,9 @@ unsigned AArch64InstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
case AArch64::SPACE:
NumBytes = MI.getOperand(1).getImm();
break;
+ case AArch64::StoreSwiftAsyncContext:
+ NumBytes = 20;
+ break;
case TargetOpcode::BUNDLE:
NumBytes = getInstBundleLength(MI);
break;
@@ -2640,6 +2643,13 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale,
MinOffset = 0;
MaxOffset = 4095;
break;
+ case AArch64::StoreSwiftAsyncContext:
+ // Store is an STRXui, but there might be an ADDXri in the expansion too.
+ Scale = TypeSize::Fixed(1);
+ Width = 8;
+ MinOffset = 0;
+ MaxOffset = 4095;
+ break;
case AArch64::LDPWi:
case AArch64::LDPSi:
case AArch64::LDNPWi:
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index beca38f3e598f..2fc7217c01a71 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -7979,6 +7979,11 @@ let Predicates = [HasLS64] in {
def : ST64BPattern<int_aarch64_st64bv0, ST64BV0>;
}
+let Defs = [X16, X17], mayStore = 1, isCodeGenOnly = 1 in
+def StoreSwiftAsyncContext
+ : Pseudo<(outs), (ins GPR64:$ctx, GPR64sp:$base, simm9:$offset),
+ []>, Sched<[]>;
+
include "AArch64InstrAtomics.td"
include "AArch64SVEInstrInfo.td"
diff --git a/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h b/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h
index f60e2b6c316e4..f3a85f2361767 100644
--- a/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h
@@ -159,6 +159,14 @@ class AArch64FunctionInfo final : public MachineFunctionInfo {
/// indirect branch destinations.
bool BranchTargetEnforcement = false;
+ /// Whether this function has an extended frame record [Ctx, FP, LR]. If so,
+ /// bit 60 of the in-memory FP will be 1 to enable other tools to detect the
+ /// extended record.
+ bool HasSwiftAsyncContext = false;
+
+ /// The stack slot where the Swift asynchronous context is stored.
+ int SwiftAsyncContextFrameIdx = std::numeric_limits<int>::max();
+
public:
explicit AArch64FunctionInfo(MachineFunction &MF);
@@ -239,6 +247,13 @@ class AArch64FunctionInfo final : public MachineFunctionInfo {
MaxOffset = std::max<int64_t>(Offset + ObjSize, MaxOffset);
}
+ if (SwiftAsyncContextFrameIdx != std::numeric_limits<int>::max()) {
+ int64_t Offset = MFI.getObjectOffset(getSwiftAsyncContextFrameIdx());
+ int64_t ObjSize = MFI.getObjectSize(getSwiftAsyncContextFrameIdx());
+ MinOffset = std::min<int64_t>(Offset, MinOffset);
+ MaxOffset = std::max<int64_t>(Offset + ObjSize, MaxOffset);
+ }
+
unsigned Size = alignTo(MaxOffset - MinOffset, 16);
assert((!HasCalleeSavedStackSize || getCalleeSavedStackSize() == Size) &&
"Invalid size calculated for callee saves");
@@ -372,6 +387,16 @@ class AArch64FunctionInfo final : public MachineFunctionInfo {
bool branchTargetEnforcement() const { return BranchTargetEnforcement; }
+ void setHasSwiftAsyncContext(bool HasContext) {
+ HasSwiftAsyncContext = HasContext;
+ }
+ bool hasSwiftAsyncContext() const { return HasSwiftAsyncContext; }
+
+ void setSwiftAsyncContextFrameIdx(int FI) {
+ SwiftAsyncContextFrameIdx = FI;
+ }
+ int getSwiftAsyncContextFrameIdx() const { return SwiftAsyncContextFrameIdx; }
+
private:
// Hold the lists of LOHs.
MILOHContainer LOHContainerSet;
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
index e3649feb709ca..85f810ff5ffed 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
@@ -537,6 +537,9 @@ bool AArch64CallLowering::lowerFormalArguments(
ArgInfo OrigArg{VRegs[i], Arg};
setArgFlags(OrigArg, i + AttributeList::FirstArgIndex, DL, F);
+ if (Arg.hasAttribute(Attribute::SwiftAsync))
+ MF.getInfo<AArch64FunctionInfo>()->setHasSwiftAsyncContext(true);
+
splitToValueTypes(OrigArg, SplitArgs, DL, F.getCallingConv());
++i;
}
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index 3fd1f04adb90e..d91495e325886 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -5042,6 +5042,17 @@ bool AArch64InstructionSelector::selectIntrinsic(MachineInstr &I,
I.eraseFromParent();
return true;
}
+ case Intrinsic::swift_async_context_addr:
+ auto Sub = MIB.buildInstr(AArch64::SUBXri, {I.getOperand(0).getReg()},
+ {Register(AArch64::FP)})
+ .addImm(8)
+ .addImm(0);
+ constrainSelectedInstRegOperands(*Sub, TII, TRI, RBI);
+
+ MF->getFrameInfo().setFrameAddressIsTaken(true);
+ MF->getInfo<AArch64FunctionInfo>()->setHasSwiftAsyncContext(true);
+ I.eraseFromParent();
+ return true;
}
return false;
}
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp
index 1f6f95b6a9e50..02570e864e40e 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp
@@ -574,6 +574,7 @@ class DarwinAArch64AsmBackend : public AArch64AsmBackend {
unsigned StackSize = 0;
uint32_t CompactUnwindEncoding = 0;
+ int CurOffset = 0;
for (size_t i = 0, e = Instrs.size(); i != e; ++i) {
const MCCFIInstruction &Inst = Instrs[i];
@@ -603,6 +604,9 @@ class DarwinAArch64AsmBackend : public AArch64AsmBackend {
assert(FPPush.getOperation() == MCCFIInstruction::OpOffset &&
"Frame pointer not pushed!");
+ assert(FPPush.getOffset() + 8 == LRPush.getOffset());
+ CurOffset = FPPush.getOffset();
+
unsigned LRReg = *MRI.getLLVMRegNum(LRPush.getRegister(), true);
unsigned FPReg = *MRI.getLLVMRegNum(FPPush.getRegister(), true);
@@ -629,11 +633,19 @@ class DarwinAArch64AsmBackend : public AArch64AsmBackend {
if (i + 1 == e)
return CU::UNWIND_ARM64_MODE_DWARF;
+ if (CurOffset != 0 && Inst.getOffset() != CurOffset - 8)
+ return CU::UNWIND_ARM64_MODE_DWARF;
+ CurOffset = Inst.getOffset();
+
const MCCFIInstruction &Inst2 = Instrs[++i];
if (Inst2.getOperation() != MCCFIInstruction::OpOffset)
return CU::UNWIND_ARM64_MODE_DWARF;
unsigned Reg2 = *MRI.getLLVMRegNum(Inst2.getRegister(), true);
+ if (Inst2.getOffset() != CurOffset - 8)
+ return CU::UNWIND_ARM64_MODE_DWARF;
+ CurOffset = Inst2.getOffset();
+
// N.B. The encodings must be in register number order, and the X
// registers before the D registers.
diff --git a/llvm/lib/Transforms/Utils/CodeExtractor.cpp b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
index ac47e952ac7f8..a39ede9aaff66 100644
--- a/llvm/lib/Transforms/Utils/CodeExtractor.cpp
+++ b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
@@ -929,6 +929,7 @@ Function *CodeExtractor::constructFunction(const ValueSet &inputs,
case Attribute::StructRet:
case Attribute::SwiftError:
case Attribute::SwiftSelf:
+ case Attribute::SwiftAsync:
case Attribute::WillReturn:
case Attribute::WriteOnly:
case Attribute::ZExt:
diff --git a/llvm/test/Bitcode/attributes.ll b/llvm/test/Bitcode/attributes.ll
index 936a2953901a9..0ee8a43cc2ca2 100644
--- a/llvm/test/Bitcode/attributes.ll
+++ b/llvm/test/Bitcode/attributes.ll
@@ -447,6 +447,12 @@ define void @f75() vscale_range(0,0)
ret void
}
+; CHECK: define void @f76(i8* swiftasync %0)
+define void @f76(i8* swiftasync %0)
+{
+ ret void;
+}
+
; CHECK: attributes #0 = { noreturn }
; CHECK: attributes #1 = { nounwind }
; CHECK: attributes #2 = { readnone }
diff --git a/llvm/test/Bitcode/compatibility.ll b/llvm/test/Bitcode/compatibility.ll
index 0a182202c1e7e..0d08d8d1c9dc6 100644
--- a/llvm/test/Bitcode/compatibility.ll
+++ b/llvm/test/Bitcode/compatibility.ll
@@ -552,6 +552,12 @@ declare void @f.param.dereferenceable_or_null(i8* dereferenceable_or_null(4))
; CHECK: declare void @f.param.dereferenceable_or_null(i8* dereferenceable_or_null(4))
declare void @f.param.stack_align([2 x double] alignstack(16))
; CHECK: declare void @f.param.stack_align([2 x double] alignstack(16))
+declare void @f.param.swiftself(i8* swiftself)
+; CHECK: declare void @f.param.swiftself(i8* swiftself)
+declare void @f.param.swiftasync(i8* swiftasync)
+; CHECK: declare void @f.param.swiftasync(i8* swiftasync)
+declare void @f.param.swifterror(i8** swifterror)
+; CHECK: declare void @f.param.swifterror(i8** swifterror)
; Functions -- unnamed_addr and local_unnamed_addr
declare void @f.unnamed_addr() unnamed_addr
diff --git a/llvm/test/CodeGen/AArch64/swift-async-reg.ll b/llvm/test/CodeGen/AArch64/swift-async-reg.ll
new file mode 100644
index 0000000000000..8b23c4b35b260
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/swift-async-reg.ll
@@ -0,0 +1,18 @@
+; RUN: llc -mtriple=arm64-apple-ios %s -o - | FileCheck %s
+; RUN: llc -mtriple=arm64-apple-ios %s -o - -global-isel | FileCheck %s
+; RUN: llc -mtriple=arm64-apple-ios %s -o - -fast-isel | FileCheck %s
+
+define i8* @argument(i8* swiftasync %in) {
+; CHECK-LABEL: argument:
+; CHECK: mov x0, x22
+
+ ret i8* %in
+}
+
+define void @call(i8* %in) {
+; CHECK-LABEL: call:
+; CHECK: mov x22, x0
+
+ call i8* @argument(i8* swiftasync %in)
+ ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/swift-async-unwind.ll b/llvm/test/CodeGen/AArch64/swift-async-unwind.ll
new file mode 100644
index 0000000000000..5c98705c85fa3
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/swift-async-unwind.ll
@@ -0,0 +1,13 @@
+; RUN: llc -mtriple=arm64-apple-ios %s -filetype=obj -o - | llvm-objdump --unwind-info - | FileCheck %s
+
+; Swift asynchronous context is incompatible with the compact unwind encoding
+; that currently exists and assumes callee-saved registers are right next to FP
+; in a particular order. This isn't a problem now because C++ exceptions aren't
+; allowed to unwind through Swift code, but at least make sure the compact info
+; says to use DWARF correctly.
+
+; CHECK: compact encoding: 0x03000000
+define void @foo(i8* swiftasync %in) "frame-pointer"="all" {
+ call void asm sideeffect "", "~{x23}"()
+ ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/swift-async.ll b/llvm/test/CodeGen/AArch64/swift-async.ll
new file mode 100644
index 0000000000000..0fd0140b84554
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/swift-async.ll
@@ -0,0 +1,157 @@
+; RUN: llc -mtriple=arm64-apple-ios %s -o - | FileCheck %s --check-prefixes=CHECK-NOAUTH,CHECK
+; RUN: llc -mtriple=arm64-apple-ios -mcpu=apple-a13 %s -o - | FileCheck %s --check-prefixes=CHECK-NOAUTH,CHECK
+; RUN: llc -mtriple=arm64e-apple-ios %s -o - | FileCheck %s --check-prefixes=CHECK-AUTH,CHECK
+
+; Important details in prologue:
+; * x22 is stored just below x29
+; * Enough stack space is allocated for everything
+define void @simple(i8* swiftasync %ctx) "frame-pointer"="all" {
+; CHECK-LABEL: simple:
+; CHECK: orr x29, x29, #0x100000000000000
+; CHECK: sub sp, sp, #32
+; CHECK: stp x29, x30, [sp, #16]
+
+; CHECK-NOAUTH: str x22, [sp, #8]
+; CHECK-AUTH: add x16, sp, #8
+; CHECK-AUTH: movk x16, #49946, lsl #48
+; CHECK-AUTH: mov x17, x22
+; CHECK-AUTH: pacdb x17, x16
+; CHECK-AUTH: str x17, [sp, #8]
+
+; CHECK: add x29, sp, #16
+; CHECK: .cfi_def_cfa w29, 16
+; CHECK: .cfi_offset w30, -8
+; CHECK: .cfi_offset w29, -16
+
+;[...]
+
+; CHECK: ldp x29, x30, [sp, #16]
+; CHECK: and x29, x29, #0xefffffffffffffff
+; CHECK: add sp, sp, #32
+
+ ret void
+}
+
+define void @more_csrs(i8* swiftasync %ctx) "frame-pointer"="all" {
+; CHECK-LABEL: more_csrs:
+; CHECK: orr x29, x29, #0x100000000000000
+; CHECK: sub sp, sp, #48
+; CHECK: stp x24, x23, [sp, #8]
+; CHECK: stp x29, x30, [sp, #32]
+
+; CHECK-NOAUTH: str x22, [sp, #24]
+; CHECK-AUTH: add x16, sp, #24
+; CHECK-AUTH: movk x16, #49946, lsl #48
+; CHECK-AUTH: mov x17, x22
+; CHECK-AUTH: pacdb x17, x16
+; CHECK-AUTH: str x17, [sp, #24]
+
+; CHECK: add x29, sp, #32
+; CHECK: .cfi_def_cfa w29, 16
+; CHECK: .cfi_offset w30, -8
+; CHECK: .cfi_offset w29, -16
+; CHECK: .cfi_offset w23, -32
+
+; [...]
+
+; CHECK: ldp x29, x30, [sp, #32]
+; CHECK: ldp x24, x23, [sp, #8]
+; CHECK: and x29, x29, #0xefffffffffffffff
+; CHECK: add sp, sp, #48
+ call void asm sideeffect "", "~{x23}"()
+ ret void
+}
+
+define void @locals(i8* swiftasync %ctx) "frame-pointer"="all" {
+; CHECK-LABEL: locals:
+; CHECK: orr x29, x29, #0x100000000000000
+; CHECK: sub sp, sp, #64
+; CHECK: stp x29, x30, [sp, #48]
+
+; CHECK-NOAUTH: str x22, [sp, #40]
+; CHECK-AUTH: add x16, sp, #40
+; CHECK-AUTH: movk x16, #49946, lsl #48
+; CHECK-AUTH: mov x17, x22
+; CHECK-AUTH: pacdb x17, x16
+; CHECK-AUTH: str x17, [sp, #40]
+
+; CHECK: add x29, sp, #48
+; CHECK: .cfi_def_cfa w29, 16
+; CHECK: .cfi_offset w30, -8
+; CHECK: .cfi_offset w29, -16
+
+; CHECK: mov x0, sp
+; CHECK: bl _bar
+
+; [...]
+
+; CHECK: ldp x29, x30, [sp, #48]
+; CHECK: and x29, x29, #0xefffffffffffffff
+; CHECK: add sp, sp, #64
+ %var = alloca i32, i32 10
+ call void @bar(i32* %var)
+ ret void
+}
+
+define void @use_input_context(i8* swiftasync %ctx, i8** %ptr) "frame-pointer"="all" {
+; CHECK-LABEL: use_input_context:
+
+; CHECK-NOAUTH: str x22, [sp
+; CHECK-AUTH: mov x17, x22
+
+; CHECK-NOT: x22
+; CHECK: str x22, [x0]
+
+ store i8* %ctx, i8** %ptr
+ ret void
+}
+
+define i8** @context_in_func() "frame-pointer"="non-leaf" {
+; CHECK-LABEL: context_in_func:
+
+; CHECK-NOAUTH: str xzr, [sp, #8]
+; CHECK-AUTH: add x16, sp, #8
+; CHECK-AUTH: movk x16, #49946, lsl #48
+; CHECK-AUTH: mov x17, xzr
+; CHECK-AUTH: pacdb x17, x16
+; CHECK-AUTH: str x17, [sp, #8]
+
+ %ptr = call i8** @llvm.swift.async.context.addr()
+ ret i8** %ptr
+}
+
+define void @write_frame_context(i8* swiftasync %ctx, i8* %newctx) "frame-pointer"="non-leaf" {
+; CHECK-LABEL: write_frame_context:
+; CHECK: sub x[[ADDR:[0-9]+]], x29, #8
+; CHECK: str x0, [x[[ADDR]]]
+ %ptr = call i8** @llvm.swift.async.context.addr()
+ store i8* %newctx, i8** %ptr
+ ret void
+}
+
+define void @simple_fp_elim(i8* swiftasync %ctx) "frame-pointer"="non-leaf" {
+; CHECK-LABEL: simple_fp_elim:
+; CHECK-NOT: orr x29, x29, #0x100000000000000
+
+ ret void
+}
+
+define void @large_frame(i8* swiftasync %ctx) "frame-pointer"="all" {
+; CHECK-LABEL: large_frame:
+; CHECK: sub sp, sp, #48
+; CHECK: stp x28, x27, [sp, #8]
+; CHECK: stp x29, x30, [sp, #32]
+; CHECK-NOAUTH: str x22, [sp, #24]
+; CHECK: add x29, sp, #32
+; CHECK: sub sp, sp, #1024
+; [...]
+; CHECK: add sp, sp, #1024
+; CHECK: ldp x29, x30, [sp, #32]
+; CHECK: ldp x28, x27, [sp, #8]
+; CHECK: ret
+ %var = alloca i8, i32 1024
+ ret void
+}
+
+declare void @bar(i32*)
+declare i8** @llvm.swift.async.context.addr()
diff --git a/llvm/test/Verifier/swiftasync.ll b/llvm/test/Verifier/swiftasync.ll
new file mode 100644
index 0000000000000..a22a67d46b72a
--- /dev/null
+++ b/llvm/test/Verifier/swiftasync.ll
@@ -0,0 +1,4 @@
+; RUN: not llvm-as %s -o /dev/null 2>&1 | FileCheck %s
+
+declare void @a(i32* swiftasync %a, i32* swiftasync %b)
+; CHECK: Cannot have multiple 'swiftasync' parameters!
More information about the llvm-commits
mailing list