[llvm] [llvm] Support IFuncs on Darwin platforms (PR #73686)
Jon Roelofs via llvm-commits
llvm-commits at lists.llvm.org
Fri Dec 1 13:33:01 PST 2023
https://github.com/jroelofs updated https://github.com/llvm/llvm-project/pull/73686
>From bc152095691b32d1ad8539dfd60f5089df5eed8d Mon Sep 17 00:00:00 2001
From: Jon Roelofs <jonathan_roelofs at apple.com>
Date: Tue, 28 Nov 2023 10:39:44 -0800
Subject: [PATCH 1/9] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20in?=
=?UTF-8?q?itial=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.4
---
llvm/docs/LangRef.rst | 7 +-
llvm/include/llvm/CodeGen/AsmPrinter.h | 6 +-
llvm/lib/CodeGen/GlobalISel/CallLowering.cpp | 7 +-
llvm/lib/IR/Verifier.cpp | 12 +-
llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 308 ++++++++++++++++++
llvm/lib/Target/X86/X86AsmPrinter.cpp | 28 ++
llvm/lib/Target/X86/X86AsmPrinter.h | 1 +
.../AArch64/GlobalISel/call-lowering-ifunc.ll | 37 +++
llvm/test/CodeGen/AArch64/addrsig-macho.ll | 4 +-
llvm/test/CodeGen/AArch64/ifunc-asm.ll | 82 +++++
llvm/test/CodeGen/X86/ifunc-asm.ll | 28 +-
llvm/test/Verifier/ifunc-macho.ll | 42 +++
12 files changed, 539 insertions(+), 23 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/call-lowering-ifunc.ll
create mode 100644 llvm/test/CodeGen/AArch64/ifunc-asm.ll
create mode 100644 llvm/test/Verifier/ifunc-macho.ll
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index e448c5ed5c5d947..cb222e979db29d4 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -934,10 +934,11 @@ IFuncs
-------
IFuncs, like as aliases, don't create any new data or func. They are just a new
-symbol that dynamic linker resolves at runtime by calling a resolver function.
+symbol that is resolved at runtime by calling a resolver function.
-IFuncs have a name and a resolver that is a function called by dynamic linker
-that returns address of another function associated with the name.
+On ELF platforms, IFuncs are resolved by the dynamic linker at load time. On
+MachO platforms, they are lowered in terms of ``.symbol_resolver``s, which
+lazily resolve the callee the first time they are called.
IFunc may have an optional :ref:`linkage type <linkage>` and an optional
:ref:`visibility style <visibility>`.
diff --git a/llvm/include/llvm/CodeGen/AsmPrinter.h b/llvm/include/llvm/CodeGen/AsmPrinter.h
index 2731ef452c79cbb..48fa6c478464c73 100644
--- a/llvm/include/llvm/CodeGen/AsmPrinter.h
+++ b/llvm/include/llvm/CodeGen/AsmPrinter.h
@@ -882,7 +882,11 @@ class AsmPrinter : public MachineFunctionPass {
GCMetadataPrinter *getOrCreateGCPrinter(GCStrategy &S);
void emitGlobalAlias(Module &M, const GlobalAlias &GA);
- void emitGlobalIFunc(Module &M, const GlobalIFunc &GI);
+
+protected:
+ virtual void emitGlobalIFunc(Module &M, const GlobalIFunc &GI);
+
+private:
/// This method decides whether the specified basic block requires a label.
bool shouldEmitLabelForBasicBlock(const MachineBasicBlock &MBB) const;
diff --git a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
index 2527b1431289677..e0080b145d4f995 100644
--- a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
@@ -144,7 +144,12 @@ bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, const CallBase &CB,
// Try looking through a bitcast from one function type to another.
// Commonly happens with calls to objc_msgSend().
const Value *CalleeV = CB.getCalledOperand()->stripPointerCasts();
- if (const Function *F = dyn_cast<Function>(CalleeV))
+ if (const GlobalIFunc *IF = dyn_cast<GlobalIFunc>(CalleeV);
+ IF && MF.getTarget().getTargetTriple().isOSBinFormatMachO()) {
+ // ld64 requires that .symbol_resolvers to be called via a stub, so these
+ // must always be a diret call.
+ Info.Callee = MachineOperand::CreateGA(IF, 0);
+ } else if (const Function *F = dyn_cast<Function>(CalleeV))
Info.Callee = MachineOperand::CreateGA(F, 0);
else
Info.Callee = MachineOperand::CreateReg(GetCalleeReg(), false);
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 5560c037aa3ee6b..94e76a43bf38d6d 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -959,6 +959,7 @@ void Verifier::visitGlobalIFunc(const GlobalIFunc &GI) {
GlobalIFunc::getResolverFunctionType(GI.getValueType());
Check(ResolverTy == ResolverFuncTy->getPointerTo(GI.getAddressSpace()),
"IFunc resolver has incorrect type", &GI);
+
}
void Verifier::visitNamedMDNode(const NamedMDNode &NMD) {
@@ -2239,13 +2240,10 @@ void Verifier::verifyFunctionAttrs(FunctionType *FT, AttributeList Attrs,
}
// Check EVEX512 feature.
- if (MaxParameterWidth >= 512 && Attrs.hasFnAttr("target-features")) {
- Triple T(M.getTargetTriple());
- if (T.isX86()) {
- StringRef TF = Attrs.getFnAttr("target-features").getValueAsString();
- Check(!TF.contains("+avx512f") || !TF.contains("-evex512"),
- "512-bit vector arguments require 'evex512' for AVX512", V);
- }
+ if (MaxParameterWidth >= 512 && Attrs.hasFnAttr("target-features") && TT.isX86()) {
+ StringRef TF = Attrs.getFnAttr("target-features").getValueAsString();
+ Check(!TF.contains("+avx512f") || !TF.contains("-evex512"),
+ "512-bit vector arguments require 'evex512' for AVX512", V);
}
checkUnsignedBaseTenFuncAttr(Attrs, "patchable-function-prefix", V);
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index f4d3a85f34c88da..2dab8e126c9abd0 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -30,6 +30,7 @@
#include "llvm/ADT/Twine.h"
#include "llvm/BinaryFormat/COFF.h"
#include "llvm/BinaryFormat/ELF.h"
+#include "llvm/BinaryFormat/MachO.h"
#include "llvm/CodeGen/AsmPrinter.h"
#include "llvm/CodeGen/FaultMaps.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
@@ -47,10 +48,12 @@
#include "llvm/MC/MCInst.h"
#include "llvm/MC/MCInstBuilder.h"
#include "llvm/MC/MCSectionELF.h"
+#include "llvm/MC/MCSectionMachO.h"
#include "llvm/MC/MCStreamer.h"
#include "llvm/MC/MCSymbol.h"
#include "llvm/MC/TargetRegistry.h"
#include "llvm/Support/Casting.h"
+#include "llvm/Support/CommandLine.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetMachine.h"
@@ -68,6 +71,22 @@ using namespace llvm;
namespace {
+enum class IFuncLowering { SymbolResolverIfSupported, SymbolResolverAlways, SymbolResolverNever };
+
+static cl::opt<IFuncLowering> PreferredIFuncLowering(
+ "arm64-darwin-ifunc-symbol_resolver", cl::init(IFuncLowering::SymbolResolverNever),
+ cl::desc("Pick the lowering for ifuncs on darwin platforms"), cl::Hidden,
+ cl::values(
+ clEnumValN(
+ IFuncLowering::SymbolResolverIfSupported, "if_supported",
+ "Use .symbol_resolver's when known to be supported by the linker."),
+ clEnumValN(IFuncLowering::SymbolResolverAlways, "always",
+ "Always use .symbol_resolvers. NOTE: this might not be "
+ "supported by the linker in all cases."),
+ clEnumValN(IFuncLowering::SymbolResolverNever, "never",
+ "Use a manual lowering, doing what the linker would have "
+ "done, but in the compiler.")));
+
class AArch64AsmPrinter : public AsmPrinter {
AArch64MCInstLower MCInstLowering;
FaultMaps FM;
@@ -198,6 +217,11 @@ class AArch64AsmPrinter : public AsmPrinter {
bool shouldEmitWeakSwiftAsyncExtendedFramePointerFlags() const override {
return ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags;
}
+
+ void emitGlobalIFunc(Module &M, const GlobalIFunc &GI) override;
+
+ void emitLinkerSymbolResolver(Module &M, const GlobalIFunc &GI);
+ void emitManualSymbolResolver(Module &M, const GlobalIFunc &GI);
};
} // end anonymous namespace
@@ -1809,6 +1833,290 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
EmitToStreamer(*OutStreamer, TmpInst);
}
+void AArch64AsmPrinter::emitLinkerSymbolResolver(Module &M,
+ const GlobalIFunc &GI) {
+ OutStreamer->switchSection(OutContext.getObjectFileInfo()->getTextSection());
+
+ MCSymbol *Name = getSymbol(&GI);
+
+ // NOTE: non-global .symbol_resolvers are not yet supported by Darwin linkers
+
+ if (GI.hasExternalLinkage() || !MAI->getWeakRefDirective())
+ OutStreamer->emitSymbolAttribute(Name, MCSA_Global);
+ else if (GI.hasWeakLinkage() || GI.hasLinkOnceLinkage())
+ OutStreamer->emitSymbolAttribute(Name, MCSA_WeakReference);
+ else
+ assert(GI.hasLocalLinkage() && "Invalid ifunc linkage");
+
+ OutStreamer->emitCodeAlignment(Align(4), STI);
+ OutStreamer->emitLabel(Name);
+ OutStreamer->emitSymbolAttribute(Name, MCSA_SymbolResolver);
+ emitVisibility(Name, GI.getVisibility());
+
+ // ld-prime does not seem to support aliases of symbol resolvers, so we have to
+ // tail call the resolver manually.
+ OutStreamer->emitInstruction(
+ MCInstBuilder(AArch64::B)
+ .addOperand(MCOperand::createExpr(lowerConstant(GI.getResolver()))),
+ *STI);
+}
+
+/// \brief Emit a manually-constructed .symbol_resolver that implements the
+/// symbol resolution duties of the IFunc.
+///
+/// Normally, this would be handled by linker magic, but unfortunately there are
+/// a few limitations in ld64 and ld-prime's implementation of .symbol_resolver
+/// that mean we can't always use them:
+///
+/// * resolvers cannot be the target of an alias
+/// * resolvers cannot have private linkage
+/// * resolvers cannot have linkonce linkage
+/// * resolvers cannot appear in executables
+/// * resolvers cannot appear in bundles
+///
+/// This works around that by emitting a close approximation of what the linker
+/// would have done.
+void AArch64AsmPrinter::emitManualSymbolResolver(Module &M,
+ const GlobalIFunc &GI) {
+ auto EmitLinkage = [&](MCSymbol *Sym) {
+ if (GI.hasExternalLinkage() || !MAI->getWeakRefDirective())
+ OutStreamer->emitSymbolAttribute(Sym, MCSA_Global);
+ else if (GI.hasWeakLinkage() || GI.hasLinkOnceLinkage())
+ OutStreamer->emitSymbolAttribute(Sym, MCSA_WeakReference);
+ else
+ assert(GI.hasLocalLinkage() && "Invalid ifunc linkage");
+ };
+
+ MCSymbol *LazyPointer = TM.getObjFileLowering()->getContext().getOrCreateSymbol(
+ "_" + GI.getName() + ".lazy_pointer");
+ MCSymbol *StubHelper =
+ TM.getObjFileLowering()->getContext().getOrCreateSymbol(
+ "_" + GI.getName() + ".stub_helper");
+
+ OutStreamer->switchSection(OutContext.getObjectFileInfo()->getDataSection());
+
+ EmitLinkage(LazyPointer);
+ OutStreamer->emitLabel(LazyPointer);
+ emitVisibility(LazyPointer, GI.getVisibility());
+ OutStreamer->emitValue(MCSymbolRefExpr::create(StubHelper, OutContext), 8);
+
+ OutStreamer->switchSection(OutContext.getObjectFileInfo()->getTextSection());
+
+ MCSymbol *Stub = getSymbol(&GI);
+
+ EmitLinkage(Stub);
+ OutStreamer->emitCodeAlignment(Align(4), STI);
+ OutStreamer->emitLabel(Stub);
+ emitVisibility(Stub, GI.getVisibility());
+
+ // adrp x16, lazy_pointer at GOTPAGE
+ // ldr x16, [x16, lazy_pointer at GOTPAGEOFF]
+ // ldr x16, [x16]
+ // br x16
+
+ {
+ MCInst Adrp;
+ Adrp.setOpcode(AArch64::ADRP);
+ Adrp.addOperand(MCOperand::createReg(AArch64::X16));
+ MCOperand SymPage;
+ MCInstLowering.lowerOperand(
+ MachineOperand::CreateES(LazyPointer->getName().data() + 1,
+ AArch64II::MO_GOT | AArch64II::MO_PAGE),
+ SymPage);
+ Adrp.addOperand(SymPage);
+ OutStreamer->emitInstruction(Adrp, *STI);
+ }
+
+ {
+ MCInst Ldr;
+ Ldr.setOpcode(AArch64::LDRXui);
+ Ldr.addOperand(MCOperand::createReg(AArch64::X16));
+ Ldr.addOperand(MCOperand::createReg(AArch64::X16));
+ MCOperand SymPageOff;
+ MCInstLowering.lowerOperand(
+ MachineOperand::CreateES(LazyPointer->getName().data() + 1,
+ AArch64II::MO_GOT | AArch64II::MO_PAGEOFF),
+ SymPageOff);
+ Ldr.addOperand(SymPageOff);
+ Ldr.addOperand(MCOperand::createImm(0));
+ OutStreamer->emitInstruction(Ldr, *STI);
+ }
+
+ OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDRXui)
+ .addReg(AArch64::X16)
+ .addReg(AArch64::X16)
+ .addImm(0), *STI);
+
+ OutStreamer->emitInstruction(MCInstBuilder(TM.getTargetTriple().isArm64e()
+ ? AArch64::BRAAZ
+ : AArch64::BR)
+ .addReg(AArch64::X16),
+ *STI);
+
+ EmitLinkage(StubHelper);
+ OutStreamer->emitCodeAlignment(Align(4), STI);
+ OutStreamer->emitLabel(StubHelper);
+ emitVisibility(StubHelper, GI.getVisibility());
+
+ // stp fp, lr, [sp, #-16]!
+ // mov fp, sp
+ // stp x1, x0, [sp, #-16]!
+ // stp x3, x2, [sp, #-16]!
+ // stp x5, x4, [sp, #-16]!
+ // stp x7, x6, [sp, #-16]!
+ // stp d1, d0, [sp, #-16]!
+ // stp d3, d2, [sp, #-16]!
+ // stp d5, d4, [sp, #-16]!
+ // stp d7, d6, [sp, #-16]!
+ // bl _resolver
+ // adrp x16, lazy_pointer at GOTPAGE
+ // ldr x16, [x16, lazy_pointer at GOTPAGEOFF]
+ // str x0, [x16]
+ // mov x16, x0
+ // ldp d7, d6, [sp], #16
+ // ldp d5, d4, [sp], #16
+ // ldp d3, d2, [sp], #16
+ // ldp d1, d0, [sp], #16
+ // ldp x7, x6, [sp], #16
+ // ldp x5, x4, [sp], #16
+ // ldp x3, x2, [sp], #16
+ // ldp x1, x0, [sp], #16
+ // ldp fp, lr, [sp], #16
+ // br x16
+
+ OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre)
+ .addReg(AArch64::SP)
+ .addReg(AArch64::FP)
+ .addReg(AArch64::LR)
+ .addReg(AArch64::SP)
+ .addImm(-2),
+ *STI);
+
+ OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri)
+ .addReg(AArch64::FP)
+ .addReg(AArch64::SP)
+ .addImm(0)
+ .addImm(0),
+ *STI);
+
+ for (int I = 0; I != 4; ++I)
+ OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre)
+ .addReg(AArch64::SP)
+ .addReg(AArch64::X1 + 2 * I)
+ .addReg(AArch64::X0 + 2 * I)
+ .addReg(AArch64::SP)
+ .addImm(-2),
+ *STI);
+
+ for (int I = 0; I != 4; ++I)
+ OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPDpre)
+ .addReg(AArch64::SP)
+ .addReg(AArch64::D1 + 2 * I)
+ .addReg(AArch64::D0 + 2 * I)
+ .addReg(AArch64::SP)
+ .addImm(-2),
+ *STI);
+
+ OutStreamer->emitInstruction(
+ MCInstBuilder(AArch64::BL)
+ .addOperand(MCOperand::createExpr(lowerConstant(GI.getResolver()))),
+ *STI);
+
+ {
+ MCInst Adrp;
+ Adrp.setOpcode(AArch64::ADRP);
+ Adrp.addOperand(MCOperand::createReg(AArch64::X16));
+ MCOperand SymPage;
+ MCInstLowering.lowerOperand(
+ MachineOperand::CreateES(LazyPointer->getName().data() + 1,
+ AArch64II::MO_GOT | AArch64II::MO_PAGE),
+ SymPage);
+ Adrp.addOperand(SymPage);
+ OutStreamer->emitInstruction(Adrp, *STI);
+ }
+
+ {
+ MCInst Ldr;
+ Ldr.setOpcode(AArch64::LDRXui);
+ Ldr.addOperand(MCOperand::createReg(AArch64::X16));
+ Ldr.addOperand(MCOperand::createReg(AArch64::X16));
+ MCOperand SymPageOff;
+ MCInstLowering.lowerOperand(
+ MachineOperand::CreateES(LazyPointer->getName().data() + 1,
+ AArch64II::MO_GOT | AArch64II::MO_PAGEOFF),
+ SymPageOff);
+ Ldr.addOperand(SymPageOff);
+ Ldr.addOperand(MCOperand::createImm(0));
+ OutStreamer->emitInstruction(Ldr, *STI);
+ }
+
+ OutStreamer->emitInstruction(MCInstBuilder(AArch64::STRXui)
+ .addReg(AArch64::X0)
+ .addReg(AArch64::X16)
+ .addImm(0),
+ *STI);
+
+ OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri)
+ .addReg(AArch64::X16)
+ .addReg(AArch64::X0)
+ .addImm(0)
+ .addImm(0),
+ *STI);
+
+ for (int I = 3; I != -1; --I)
+ OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPDpost)
+ .addReg(AArch64::SP)
+ .addReg(AArch64::D1 + 2 * I)
+ .addReg(AArch64::D0 + 2 * I)
+ .addReg(AArch64::SP)
+ .addImm(2),
+ *STI);
+
+ for (int I = 3; I != -1; --I)
+ OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPXpost)
+ .addReg(AArch64::SP)
+ .addReg(AArch64::X1 + 2 * I)
+ .addReg(AArch64::X0 + 2 * I)
+ .addReg(AArch64::SP)
+ .addImm(2),
+ *STI);
+
+ OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPXpost)
+ .addReg(AArch64::SP)
+ .addReg(AArch64::FP)
+ .addReg(AArch64::LR)
+ .addReg(AArch64::SP)
+ .addImm(2),
+ *STI);
+
+ OutStreamer->emitInstruction(MCInstBuilder(TM.getTargetTriple().isArm64e()
+ ? AArch64::BRAAZ
+ : AArch64::BR)
+ .addReg(AArch64::X16),
+ *STI);
+}
+
+void AArch64AsmPrinter::emitGlobalIFunc(Module &M, const GlobalIFunc &GI) {
+ if (!TM.getTargetTriple().isOSBinFormatMachO())
+ return AsmPrinter::emitGlobalIFunc(M, GI);
+
+ switch (PreferredIFuncLowering) {
+ case IFuncLowering::SymbolResolverAlways:
+ return emitLinkerSymbolResolver(M, GI);
+ case IFuncLowering::SymbolResolverNever:
+ return emitManualSymbolResolver(M, GI);
+ case IFuncLowering::SymbolResolverIfSupported:
+ if (GI.hasExternalLinkage() || !MAI->getWeakRefDirective())
+ return emitLinkerSymbolResolver(M, GI);
+ else if (GI.hasWeakLinkage() || GI.hasLinkOnceLinkage())
+ // NOTE: non-global .symbol_resolvers are not yet supported by Darwin
+ // linkers
+ return emitManualSymbolResolver(M, GI);
+ else
+ assert(GI.hasLocalLinkage() && "Invalid ifunc linkage");
+ }
+}
+
// Force static initialization.
extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAArch64AsmPrinter() {
RegisterAsmPrinter<AArch64AsmPrinter> X(getTheAArch64leTarget());
diff --git a/llvm/lib/Target/X86/X86AsmPrinter.cpp b/llvm/lib/Target/X86/X86AsmPrinter.cpp
index 73c7450620966cd..5241aa6e1c0eade 100644
--- a/llvm/lib/Target/X86/X86AsmPrinter.cpp
+++ b/llvm/lib/Target/X86/X86AsmPrinter.cpp
@@ -530,6 +530,34 @@ void X86AsmPrinter::PrintIntelMemReference(const MachineInstr *MI,
O << ']';
}
+void X86AsmPrinter::emitGlobalIFunc(Module &M, const GlobalIFunc &GI) {
+ if (!TM.getTargetTriple().isOSBinFormatMachO())
+ return AsmPrinter::emitGlobalIFunc(M, GI);
+
+ OutStreamer->switchSection(OutContext.getObjectFileInfo()->getTextSection());
+
+ MCSymbol *Name = getSymbol(&GI);
+
+ if (GI.hasExternalLinkage() || !MAI->getWeakRefDirective())
+ OutStreamer->emitSymbolAttribute(Name, MCSA_Global);
+ else if (GI.hasWeakLinkage() || GI.hasLinkOnceLinkage())
+ OutStreamer->emitSymbolAttribute(Name, MCSA_WeakReference);
+ else
+ assert(GI.hasLocalLinkage() && "Invalid ifunc linkage");
+
+ OutStreamer->emitCodeAlignment(Align(16), Subtarget);
+ OutStreamer->emitLabel(Name);
+ OutStreamer->emitSymbolAttribute(Name, MCSA_SymbolResolver);
+ emitVisibility(Name, GI.getVisibility());
+
+ // ld64 does not seem to support aliases of symbol resolvers, so we have to
+ // tail call the resolver manually.
+ MCInst JMP;
+ JMP.setOpcode(X86::JMP_4);
+ JMP.addOperand(MCOperand::createExpr(lowerConstant(GI.getResolver())));
+ OutStreamer->emitInstruction(JMP, *Subtarget);
+}
+
static bool printAsmMRegister(const X86AsmPrinter &P, const MachineOperand &MO,
char Mode, raw_ostream &O) {
Register Reg = MO.getReg();
diff --git a/llvm/lib/Target/X86/X86AsmPrinter.h b/llvm/lib/Target/X86/X86AsmPrinter.h
index c81651cf7f2f0e6..47f3b7c00c99ddb 100644
--- a/llvm/lib/Target/X86/X86AsmPrinter.h
+++ b/llvm/lib/Target/X86/X86AsmPrinter.h
@@ -120,6 +120,7 @@ class LLVM_LIBRARY_VISIBILITY X86AsmPrinter : public AsmPrinter {
const char *Modifier);
void PrintIntelMemReference(const MachineInstr *MI, unsigned OpNo,
raw_ostream &O, const char *Modifier);
+ void emitGlobalIFunc(Module &M, const GlobalIFunc &GI) override;
public:
X86AsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer);
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/call-lowering-ifunc.ll b/llvm/test/CodeGen/AArch64/GlobalISel/call-lowering-ifunc.ll
new file mode 100644
index 000000000000000..8e51845c2faa9c0
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/call-lowering-ifunc.ll
@@ -0,0 +1,37 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -mtriple=aarch64-macho -global-isel -stop-after=irtranslator -verify-machineinstrs -o - %s | FileCheck %s --check-prefixes=MACHO,CHECK
+; RUN: llc -mtriple=aarch64-elf -global-isel -stop-after=irtranslator -verify-machineinstrs -o - %s | FileCheck %s --check-prefixes=ELF,CHECK
+
+ at foo_ifunc = ifunc i32 (i32), ptr @foo_resolver
+
+define internal ptr @foo_resolver() {
+ ; CHECK-LABEL: name: foo_resolver
+ ; CHECK: bb.1.entry:
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(p0) = G_CONSTANT i64 0
+ ; CHECK-NEXT: $x0 = COPY [[C]](p0)
+ ; CHECK-NEXT: RET_ReallyLR implicit $x0
+entry:
+ ret ptr null
+}
+
+define void @caller() {
+ ; MACHO-LABEL: name: caller
+ ; MACHO: bb.1.entry:
+ ; MACHO-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+ ; MACHO-NEXT: BL @foo_ifunc, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit-def $w0
+ ; MACHO-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+ ; MACHO-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+ ; MACHO-NEXT: RET_ReallyLR
+ ;
+ ; ELF-LABEL: name: caller
+ ; ELF: bb.1.entry:
+ ; ELF-NEXT: [[GV:%[0-9]+]]:gpr64(p0) = G_GLOBAL_VALUE @foo_ifunc
+ ; ELF-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+ ; ELF-NEXT: BLR [[GV]](p0), csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit-def $w0
+ ; ELF-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+ ; ELF-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+ ; ELF-NEXT: RET_ReallyLR
+entry:
+ %0 = call i32 @foo_ifunc()
+ ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/addrsig-macho.ll b/llvm/test/CodeGen/AArch64/addrsig-macho.ll
index 360876fccaad34e..980b0e7bc446695 100644
--- a/llvm/test/CodeGen/AArch64/addrsig-macho.ll
+++ b/llvm/test/CodeGen/AArch64/addrsig-macho.ll
@@ -118,8 +118,8 @@ declare void @f3() unnamed_addr
@a1 = alias i32, i32* @g1
@a2 = internal local_unnamed_addr alias i32, i32* @g2
- at i1 = ifunc void(), void()* ()* @f1
- at i2 = internal local_unnamed_addr ifunc void(), void()* ()* @f2
+ at i1 = external ifunc void(), void()* ()* @f1
+ at i2 = external local_unnamed_addr ifunc void(), void()* ()* @f2
declare void @llvm.dbg.value(metadata, metadata, metadata)
diff --git a/llvm/test/CodeGen/AArch64/ifunc-asm.ll b/llvm/test/CodeGen/AArch64/ifunc-asm.ll
new file mode 100644
index 000000000000000..fbc0f74cee46baa
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/ifunc-asm.ll
@@ -0,0 +1,82 @@
+; RUN: llc -mtriple=arm64-unknown-linux-gnu %s -filetype=asm -o - | FileCheck %s --check-prefixes=ELF
+; RUN: llc -mtriple=arm64-apple-darwin %s -filetype=asm -o - -arm64-darwin-ifunc-symbol_resolver=always | FileCheck %s --check-prefixes=MACHO,MACHO-LINKER
+; RUN: llc -mtriple=arm64-apple-darwin %s -filetype=asm -o - -arm64-darwin-ifunc-symbol_resolver=if_supported | FileCheck %s --check-prefixes=MACHO,MACHO-DEFAULT
+; RUN: llc -mtriple=arm64-apple-darwin %s -filetype=asm -o - -arm64-darwin-ifunc-symbol_resolver=never | FileCheck %s --check-prefixes=MACHO,MACHO-MANUAL
+; RUN: llc -mtriple=arm64-apple-darwin %s -filetype=asm -o - | FileCheck %s --check-prefixes=MACHO,MACHO-MANUAL
+
+define internal ptr @foo_resolver() {
+entry:
+ ret ptr null
+}
+; ELF: .type foo_resolver, at function
+; ELF-NEXT: foo_resolver:
+
+; MACHO: .p2align 2
+; MACHO-NEXT: _foo_resolver
+
+
+ at foo_ifunc = ifunc i32 (i32), ptr @foo_resolver
+; ELF: .globl foo_ifunc
+; ELF-NEXT: .type foo_ifunc, at gnu_indirect_function
+; ELF-NEXT: .set foo_ifunc, foo_resolver
+
+; MACHO-LINKER: .globl _foo_ifunc
+; MACHO-LINKER-NEXT: .p2align 2
+; MACHO-LINKER-NEXT: _foo_ifunc:
+; MACHO-LINKER-NEXT: .symbol_resolver _foo_ifunc
+; MACHO-LINKER-NEXT: b _foo_resolver
+
+; MACHO-DEFAULT: .globl _foo_ifunc
+; MACHO-DEFAULT-NEXT: .p2align 2
+; MACHO-DEFAULT-NEXT: _foo_ifunc:
+; MACHO-DEFAULT-NEXT: .symbol_resolver _foo_ifunc
+; MACHO-DEFAULT-NEXT: b _foo_resolver
+
+; MACHO-MANUAL: .section __DATA,__data
+; MACHO-MANUAL-NEXT: .globl _foo_ifunc.lazy_pointer
+; MACHO-MANUAL-NEXT: _foo_ifunc.lazy_pointer:
+; MACHO-MANUAL-NEXT: .quad _foo_ifunc.stub_helper
+
+; MACHO-MANUAL: .section __TEXT,__text,regular,pure_instructions
+; MACHO-MANUAL-NEXT: .globl _foo_ifunc
+; MACHO-MANUAL-NEXT: .p2align 2
+; MACHO-MANUAL-NEXT: _foo_ifunc:
+; MACHO-MANUAL-NEXT: adrp x16, _foo_ifunc.lazy_pointer at GOTPAGE
+; MACHO-MANUAL-NEXT: ldr x16, [x16, _foo_ifunc.lazy_pointer at GOTPAGEOFF]
+; MACHO-MANUAL-NEXT: ldr x16, [x16]
+; MACHO-MANUAL-NEXT: br x16
+; MACHO-MANUAL-NEXT: .globl _foo_ifunc.stub_helper
+; MACHO-MANUAL-NEXT: .p2align 2
+; MACHO-MANUAL-NEXT: _foo_ifunc.stub_helper:
+; MACHO-MANUAL-NEXT: stp x29, x30, [sp, #-16]!
+; MACHO-MANUAL-NEXT: mov x29, sp
+; MACHO-MANUAL-NEXT: stp x1, x0, [sp, #-16]!
+; MACHO-MANUAL-NEXT: stp x3, x2, [sp, #-16]!
+; MACHO-MANUAL-NEXT: stp x5, x4, [sp, #-16]!
+; MACHO-MANUAL-NEXT: stp x7, x6, [sp, #-16]!
+; MACHO-MANUAL-NEXT: stp d1, d0, [sp, #-16]!
+; MACHO-MANUAL-NEXT: stp d3, d2, [sp, #-16]!
+; MACHO-MANUAL-NEXT: stp d5, d4, [sp, #-16]!
+; MACHO-MANUAL-NEXT: stp d7, d6, [sp, #-16]!
+; MACHO-MANUAL-NEXT: bl _foo_resolver
+; MACHO-MANUAL-NEXT: adrp x16, _foo_ifunc.lazy_pointer at GOTPAGE
+; MACHO-MANUAL-NEXT: ldr x16, [x16, _foo_ifunc.lazy_pointer at GOTPAGEOFF]
+; MACHO-MANUAL-NEXT: str x0, [x16]
+; MACHO-MANUAL-NEXT: add x16, x0, #0
+; MACHO-MANUAL-NEXT: ldp d7, d6, [sp], #16
+; MACHO-MANUAL-NEXT: ldp d5, d4, [sp], #16
+; MACHO-MANUAL-NEXT: ldp d3, d2, [sp], #16
+; MACHO-MANUAL-NEXT: ldp d1, d0, [sp], #16
+; MACHO-MANUAL-NEXT: ldp x7, x6, [sp], #16
+; MACHO-MANUAL-NEXT: ldp x5, x4, [sp], #16
+; MACHO-MANUAL-NEXT: ldp x3, x2, [sp], #16
+; MACHO-MANUAL-NEXT: ldp x1, x0, [sp], #16
+; MACHO-MANUAL-NEXT: ldp x29, x30, [sp], #16
+; MACHO-MANUAL-NEXT: br x16
+
+
+ at weak_ifunc = weak ifunc i32 (i32), ptr @foo_resolver
+; ELF: .type weak_ifunc, at gnu_indirect_function
+; MACHO-LINKER: .symbol_resolver _weak_ifunc
+; MACHO-MANUAL: _weak_ifunc.stub_helper:
+; MACHO-DEFEAULT: _weak_ifunc.stub_helper:
\ No newline at end of file
diff --git a/llvm/test/CodeGen/X86/ifunc-asm.ll b/llvm/test/CodeGen/X86/ifunc-asm.ll
index 4b380c8ae330135..76efda71153207d 100644
--- a/llvm/test/CodeGen/X86/ifunc-asm.ll
+++ b/llvm/test/CodeGen/X86/ifunc-asm.ll
@@ -1,14 +1,24 @@
-; RUN: llvm-as < %s -o - | llc -filetype=asm | FileCheck %s
+; RUN: llc -filetype=asm -mtriple=x86_64-unknown-linux-gnu %s -o - | FileCheck %s --check-prefixes=ELF
+; RUN: llc -filetype=asm -mtriple=x86_64-apple-darwin %s -o - | FileCheck %s --check-prefixes=MACHO
-target triple = "x86_64-unknown-linux-gnu"
-
-define internal ptr @foo_ifunc() {
+define internal ptr @foo_resolver() {
entry:
ret ptr null
}
-; CHECK: .type foo_ifunc, at function
-; CHECK-NEXT: foo_ifunc:
+; ELF: .type foo_resolver, at function
+; ELF-NEXT: foo_resolver:
+
+; MACHO: .p2align 4, 0x90
+; MACHO-NEXT: _foo_resolver
+
+
+ at foo_ifunc = ifunc i32 (i32), ptr @foo_resolver
+; ELF: .globl foo_ifunc
+; ELF-NEXT: .type foo_ifunc, at gnu_indirect_function
+; ELF-NEXT: .set foo_ifunc, foo_resolver
- at foo = ifunc i32 (i32), ptr @foo_ifunc
-; CHECK: .type foo, at gnu_indirect_function
-; CHECK-NEXT: .set foo, foo_ifunc
+; MACHO: .globl _foo_ifunc
+; MACHO-NEXT: .p2align 4, 0x90
+; MACHO-NEXT: _foo_ifunc:
+; MACHO-NEXT: .symbol_resolver _foo_ifunc
+; MACHO-NEXT: jmp _foo_resolver
diff --git a/llvm/test/Verifier/ifunc-macho.ll b/llvm/test/Verifier/ifunc-macho.ll
new file mode 100644
index 000000000000000..2e2166645983ac3
--- /dev/null
+++ b/llvm/test/Verifier/ifunc-macho.ll
@@ -0,0 +1,42 @@
+; RUN: not llvm-as %s -o /dev/null 2>&1 | FileCheck %s
+
+target triple = "arm64-apple-ios"
+
+define ptr @resolver() {
+ ret ptr null
+}
+
+ at g = external global i32
+ at inval_objtype = ifunc void (), ptr @g
+; CHECK: IFunc must have a Function resolver
+
+declare ptr @resolver_decl()
+ at inval_resolver_decl = ifunc void (), ptr @resolver_decl
+; CHECK: IFunc resolver must be a definition
+; CHECK-NEXT: @inval_resolver_decl
+
+define available_externally ptr @resolver_linker_decl() {
+ ret ptr null
+}
+ at inval_resolver_decl2 = ifunc void (), ptr @resolver_linker_decl
+; CHECK: IFunc resolver must be a definition
+; CHECK-NEXT: @inval_resolver_decl2
+
+ at ifunc_nonpointer_return_type = ifunc i32 (), ptr @resolver_returns_nonpointer
+; CHECK: IFunc resolver must return a pointer
+; CHECK-NEXT: ptr @ifunc_nonpointer_return_type
+
+define i32 @resolver_returns_nonpointer() {
+ ret i32 0
+}
+
+ at valid_external = ifunc void (), ptr @resolver
+; CHECK-NOT: valid_external
+
+ at inval_linkonce = linkonce ifunc void (), ptr @resolver
+
+ at inval_weak = weak ifunc void (), ptr @resolver
+
+ at inval_weak_extern = extern_weak ifunc void (), ptr @resolver
+
+ at inval_private = private ifunc void (), ptr @resolver
>From 8f6755e6b211c9b0206197f65304443e26e244eb Mon Sep 17 00:00:00 2001
From: Jon Roelofs <jonathan_roelofs at apple.com>
Date: Tue, 28 Nov 2023 11:04:43 -0800
Subject: [PATCH 2/9] git clang-format
Created using spr 1.3.4
---
llvm/include/llvm/CodeGen/AsmPrinter.h | 1 -
llvm/lib/IR/Verifier.cpp | 4 +--
llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 25 ++++++++++++-------
llvm/lib/Target/X86/X86AsmPrinter.cpp | 2 ++
4 files changed, 20 insertions(+), 12 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/AsmPrinter.h b/llvm/include/llvm/CodeGen/AsmPrinter.h
index 48fa6c478464c73..07b92871a0f0868 100644
--- a/llvm/include/llvm/CodeGen/AsmPrinter.h
+++ b/llvm/include/llvm/CodeGen/AsmPrinter.h
@@ -887,7 +887,6 @@ class AsmPrinter : public MachineFunctionPass {
virtual void emitGlobalIFunc(Module &M, const GlobalIFunc &GI);
private:
-
/// This method decides whether the specified basic block requires a label.
bool shouldEmitLabelForBasicBlock(const MachineBasicBlock &MBB) const;
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 94e76a43bf38d6d..bd90047a411a654 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -959,7 +959,6 @@ void Verifier::visitGlobalIFunc(const GlobalIFunc &GI) {
GlobalIFunc::getResolverFunctionType(GI.getValueType());
Check(ResolverTy == ResolverFuncTy->getPointerTo(GI.getAddressSpace()),
"IFunc resolver has incorrect type", &GI);
-
}
void Verifier::visitNamedMDNode(const NamedMDNode &NMD) {
@@ -2240,7 +2239,8 @@ void Verifier::verifyFunctionAttrs(FunctionType *FT, AttributeList Attrs,
}
// Check EVEX512 feature.
- if (MaxParameterWidth >= 512 && Attrs.hasFnAttr("target-features") && TT.isX86()) {
+ if (MaxParameterWidth >= 512 && Attrs.hasFnAttr("target-features") &&
+ TT.isX86()) {
StringRef TF = Attrs.getFnAttr("target-features").getValueAsString();
Check(!TF.contains("+avx512f") || !TF.contains("-evex512"),
"512-bit vector arguments require 'evex512' for AVX512", V);
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index 2dab8e126c9abd0..f4128332008fb83 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -71,10 +71,15 @@ using namespace llvm;
namespace {
-enum class IFuncLowering { SymbolResolverIfSupported, SymbolResolverAlways, SymbolResolverNever };
+enum class IFuncLowering {
+ SymbolResolverIfSupported,
+ SymbolResolverAlways,
+ SymbolResolverNever
+};
static cl::opt<IFuncLowering> PreferredIFuncLowering(
- "arm64-darwin-ifunc-symbol_resolver", cl::init(IFuncLowering::SymbolResolverNever),
+ "arm64-darwin-ifunc-symbol_resolver",
+ cl::init(IFuncLowering::SymbolResolverNever),
cl::desc("Pick the lowering for ifuncs on darwin platforms"), cl::Hidden,
cl::values(
clEnumValN(
@@ -1853,8 +1858,8 @@ void AArch64AsmPrinter::emitLinkerSymbolResolver(Module &M,
OutStreamer->emitSymbolAttribute(Name, MCSA_SymbolResolver);
emitVisibility(Name, GI.getVisibility());
- // ld-prime does not seem to support aliases of symbol resolvers, so we have to
- // tail call the resolver manually.
+ // ld-prime does not seem to support aliases of symbol resolvers, so we have
+ // to tail call the resolver manually.
OutStreamer->emitInstruction(
MCInstBuilder(AArch64::B)
.addOperand(MCOperand::createExpr(lowerConstant(GI.getResolver()))),
@@ -1887,8 +1892,9 @@ void AArch64AsmPrinter::emitManualSymbolResolver(Module &M,
assert(GI.hasLocalLinkage() && "Invalid ifunc linkage");
};
- MCSymbol *LazyPointer = TM.getObjFileLowering()->getContext().getOrCreateSymbol(
- "_" + GI.getName() + ".lazy_pointer");
+ MCSymbol *LazyPointer =
+ TM.getObjFileLowering()->getContext().getOrCreateSymbol(
+ "_" + GI.getName() + ".lazy_pointer");
MCSymbol *StubHelper =
TM.getObjFileLowering()->getContext().getOrCreateSymbol(
"_" + GI.getName() + ".stub_helper");
@@ -1943,9 +1949,10 @@ void AArch64AsmPrinter::emitManualSymbolResolver(Module &M,
}
OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDRXui)
- .addReg(AArch64::X16)
- .addReg(AArch64::X16)
- .addImm(0), *STI);
+ .addReg(AArch64::X16)
+ .addReg(AArch64::X16)
+ .addImm(0),
+ *STI);
OutStreamer->emitInstruction(MCInstBuilder(TM.getTargetTriple().isArm64e()
? AArch64::BRAAZ
diff --git a/llvm/lib/Target/X86/X86AsmPrinter.cpp b/llvm/lib/Target/X86/X86AsmPrinter.cpp
index 5241aa6e1c0eade..37158900d2404dd 100644
--- a/llvm/lib/Target/X86/X86AsmPrinter.cpp
+++ b/llvm/lib/Target/X86/X86AsmPrinter.cpp
@@ -556,6 +556,8 @@ void X86AsmPrinter::emitGlobalIFunc(Module &M, const GlobalIFunc &GI) {
JMP.setOpcode(X86::JMP_4);
JMP.addOperand(MCOperand::createExpr(lowerConstant(GI.getResolver())));
OutStreamer->emitInstruction(JMP, *Subtarget);
+
+ // FIXME: do the manual .symbol_resolver lowering that we did in AArch64AsmPrinter.
}
static bool printAsmMRegister(const X86AsmPrinter &P, const MachineOperand &MO,
>From dc1fcb464e3d9e8dd37a5bda1aa8894e127654e9 Mon Sep 17 00:00:00 2001
From: Jon Roelofs <jonathan_roelofs at apple.com>
Date: Tue, 28 Nov 2023 14:24:45 -0800
Subject: [PATCH 3/9] avoid writeback stp's
Created using spr 1.3.4
---
llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp | 48 ++++----
llvm/lib/CodeGen/GlobalISel/CallLowering.cpp | 2 +-
llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 110 ++++++++++--------
llvm/test/CodeGen/AArch64/addrsig-macho.ll | 17 ++-
llvm/test/CodeGen/AArch64/ifunc-asm.ll | 99 ++++++++--------
llvm/test/Verifier/ifunc-macho.ll | 42 -------
6 files changed, 153 insertions(+), 165 deletions(-)
delete mode 100644 llvm/test/Verifier/ifunc-macho.ll
diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index 15ff39883680369..b4ac0a70e7fde9c 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -2300,6 +2300,32 @@ bool AsmPrinter::doFinalization(Module &M) {
// through user plugins.
emitStackMaps();
+ // Print aliases in topological order, that is, for each alias a = b,
+ // b must be printed before a.
+ // This is because on some targets (e.g. PowerPC) linker expects aliases in
+ // such an order to generate correct TOC information.
+ SmallVector<const GlobalAlias *, 16> AliasStack;
+ SmallPtrSet<const GlobalAlias *, 16> AliasVisited;
+ for (const auto &Alias : M.aliases()) {
+ if (Alias.hasAvailableExternallyLinkage())
+ continue;
+ for (const GlobalAlias *Cur = &Alias; Cur;
+ Cur = dyn_cast<GlobalAlias>(Cur->getAliasee())) {
+ if (!AliasVisited.insert(Cur).second)
+ break;
+ AliasStack.push_back(Cur);
+ }
+ for (const GlobalAlias *AncestorAlias : llvm::reverse(AliasStack))
+ emitGlobalAlias(M, *AncestorAlias);
+ AliasStack.clear();
+ }
+
+ // IFuncs must come before deubginfo in case the backend decides to emit them
+ // as actual functions, since on MachO targets, we cannot create regular
+ // sections after DWARF.
+ for (const auto &IFunc : M.ifuncs())
+ emitGlobalIFunc(M, IFunc);
+
// Finalize debug and EH information.
for (const HandlerInfo &HI : Handlers) {
NamedRegionTimer T(HI.TimerName, HI.TimerDescription, HI.TimerGroupName,
@@ -2339,28 +2365,6 @@ bool AsmPrinter::doFinalization(Module &M) {
}
}
- // Print aliases in topological order, that is, for each alias a = b,
- // b must be printed before a.
- // This is because on some targets (e.g. PowerPC) linker expects aliases in
- // such an order to generate correct TOC information.
- SmallVector<const GlobalAlias *, 16> AliasStack;
- SmallPtrSet<const GlobalAlias *, 16> AliasVisited;
- for (const auto &Alias : M.aliases()) {
- if (Alias.hasAvailableExternallyLinkage())
- continue;
- for (const GlobalAlias *Cur = &Alias; Cur;
- Cur = dyn_cast<GlobalAlias>(Cur->getAliasee())) {
- if (!AliasVisited.insert(Cur).second)
- break;
- AliasStack.push_back(Cur);
- }
- for (const GlobalAlias *AncestorAlias : llvm::reverse(AliasStack))
- emitGlobalAlias(M, *AncestorAlias);
- AliasStack.clear();
- }
- for (const auto &IFunc : M.ifuncs())
- emitGlobalIFunc(M, IFunc);
-
GCModuleInfo *MI = getAnalysisIfAvailable<GCModuleInfo>();
assert(MI && "AsmPrinter didn't require GCModuleInfo?");
for (GCModuleInfo::iterator I = MI->end(), E = MI->begin(); I != E; )
diff --git a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
index e0080b145d4f995..ce736178afc8b5a 100644
--- a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
@@ -147,7 +147,7 @@ bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, const CallBase &CB,
if (const GlobalIFunc *IF = dyn_cast<GlobalIFunc>(CalleeV);
IF && MF.getTarget().getTargetTriple().isOSBinFormatMachO()) {
// ld64 requires that .symbol_resolvers to be called via a stub, so these
- // must always be a diret call.
+ // must always be a direct call.
Info.Callee = MachineOperand::CreateGA(IF, 0);
} else if (const Function *F = dyn_cast<Function>(CalleeV))
Info.Callee = MachineOperand::CreateGA(F, 0);
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index f4128332008fb83..26b3a14e22b2ad9 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -1965,65 +1965,71 @@ void AArch64AsmPrinter::emitManualSymbolResolver(Module &M,
OutStreamer->emitLabel(StubHelper);
emitVisibility(StubHelper, GI.getVisibility());
- // stp fp, lr, [sp, #-16]!
- // mov fp, sp
- // stp x1, x0, [sp, #-16]!
- // stp x3, x2, [sp, #-16]!
- // stp x5, x4, [sp, #-16]!
- // stp x7, x6, [sp, #-16]!
- // stp d1, d0, [sp, #-16]!
- // stp d3, d2, [sp, #-16]!
- // stp d5, d4, [sp, #-16]!
- // stp d7, d6, [sp, #-16]!
+ // stp fp, lr, [sp, #-16]
+ // sub fp, sp, 16
+ // stp x1, x0, [sp, #-32]
+ // stp x3, x2, [sp, #-48]
+ // stp x5, x4, [sp, #-64]
+ // stp x7, x6, [sp, #-80]
+ // stp d1, d0, [sp, #-96]
+ // stp d3, d2, [sp, #-112]
+ // stp d5, d4, [sp, #-128]
+ // stp d7, d6, [sp, #-144]
+ // sub sp, sp, 144
// bl _resolver
// adrp x16, lazy_pointer at GOTPAGE
// ldr x16, [x16, lazy_pointer at GOTPAGEOFF]
// str x0, [x16]
// mov x16, x0
- // ldp d7, d6, [sp], #16
- // ldp d5, d4, [sp], #16
- // ldp d3, d2, [sp], #16
- // ldp d1, d0, [sp], #16
- // ldp x7, x6, [sp], #16
- // ldp x5, x4, [sp], #16
- // ldp x3, x2, [sp], #16
- // ldp x1, x0, [sp], #16
- // ldp fp, lr, [sp], #16
+ // add sp, sp, 144
+ // ldp d7, d6, [sp, #-144]
+ // ldp d5, d4, [sp, #-128]
+ // ldp d3, d2, [sp, #-112]
+ // ldp d1, d0, [sp, #-96]
+ // ldp x7, x6, [sp, #-80]
+ // ldp x5, x4, [sp, #-64]
+ // ldp x3, x2, [sp, #-48]
+ // ldp x1, x0, [sp, #-32]
+ // ldp fp, lr, [sp, #-16]
// br x16
- OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre)
- .addReg(AArch64::SP)
+ OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXi)
.addReg(AArch64::FP)
.addReg(AArch64::LR)
.addReg(AArch64::SP)
.addImm(-2),
*STI);
- OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri)
+ OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBXri)
.addReg(AArch64::FP)
.addReg(AArch64::SP)
- .addImm(0)
+ .addImm(16)
.addImm(0),
*STI);
- for (int I = 0; I != 4; ++I)
- OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre)
- .addReg(AArch64::SP)
- .addReg(AArch64::X1 + 2 * I)
- .addReg(AArch64::X0 + 2 * I)
+ for (int I = 0; I != 8; I += 2)
+ OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXi)
+ .addReg(AArch64::X1 + I)
+ .addReg(AArch64::X0 + I)
.addReg(AArch64::SP)
- .addImm(-2),
+ .addImm(-4 - I),
*STI);
- for (int I = 0; I != 4; ++I)
- OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPDpre)
+ for (int I = 0; I != 8; I += 2)
+ OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPDi)
+ .addReg(AArch64::D1 + I)
+ .addReg(AArch64::D0 + I)
.addReg(AArch64::SP)
- .addReg(AArch64::D1 + 2 * I)
- .addReg(AArch64::D0 + 2 * I)
- .addReg(AArch64::SP)
- .addImm(-2),
+ .addImm(-12 - I),
*STI);
+ OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBXri)
+ .addReg(AArch64::SP)
+ .addReg(AArch64::SP)
+ .addImm(144)
+ .addImm(0),
+ *STI);
+
OutStreamer->emitInstruction(
MCInstBuilder(AArch64::BL)
.addOperand(MCOperand::createExpr(lowerConstant(GI.getResolver()))),
@@ -2070,30 +2076,34 @@ void AArch64AsmPrinter::emitManualSymbolResolver(Module &M,
.addImm(0),
*STI);
- for (int I = 3; I != -1; --I)
- OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPDpost)
- .addReg(AArch64::SP)
- .addReg(AArch64::D1 + 2 * I)
- .addReg(AArch64::D0 + 2 * I)
+ OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri)
+ .addReg(AArch64::SP)
+ .addReg(AArch64::SP)
+ .addImm(144)
+ .addImm(0),
+ *STI);
+
+ for (int I = 6; I != -2; I -= 2)
+ OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPDi)
+ .addReg(AArch64::D1 + I)
+ .addReg(AArch64::D0 + I)
.addReg(AArch64::SP)
- .addImm(2),
+ .addImm(-12 - I),
*STI);
- for (int I = 3; I != -1; --I)
- OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPXpost)
- .addReg(AArch64::SP)
- .addReg(AArch64::X1 + 2 * I)
- .addReg(AArch64::X0 + 2 * I)
+ for (int I = 6; I != -2; I -= 2)
+ OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPXi)
+ .addReg(AArch64::X1 + I)
+ .addReg(AArch64::X0 + I)
.addReg(AArch64::SP)
- .addImm(2),
+ .addImm(-4 - I),
*STI);
- OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPXpost)
- .addReg(AArch64::SP)
+ OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPXi)
.addReg(AArch64::FP)
.addReg(AArch64::LR)
.addReg(AArch64::SP)
- .addImm(2),
+ .addImm(-2),
*STI);
OutStreamer->emitInstruction(MCInstBuilder(TM.getTargetTriple().isArm64e()
diff --git a/llvm/test/CodeGen/AArch64/addrsig-macho.ll b/llvm/test/CodeGen/AArch64/addrsig-macho.ll
index 980b0e7bc446695..62bc764e0251b33 100644
--- a/llvm/test/CodeGen/AArch64/addrsig-macho.ll
+++ b/llvm/test/CodeGen/AArch64/addrsig-macho.ll
@@ -3,6 +3,19 @@
; RUN: llvm-objdump --macho --section-headers %t | FileCheck %s --check-prefix=SECTIONS
; RUN: llvm-objdump --macho --reloc %t | FileCheck %s --check-prefix=RELOCS
+; CHECK: .section __DATA,__data
+; CHECK: _i1.lazy_pointer:
+; CHECK: .section __TEXT,__text,regular,pure_instructions
+; CHECK: _i1:
+; CHECK: _i1.stub_helper:
+; CHECK: .section __DATA,__data
+; CHECK: _i2.lazy_pointer:
+; CHECK: .section __TEXT,__text,regular,pure_instructions
+; CHECK: _i2:
+; CHECK: _i2.stub_helper:
+
+; CHECK: .section __DWARF
+
; CHECK: .addrsig{{$}}
; CHECK-NEXT: .addrsig_sym _func03_takeaddr
; CHECK-NEXT: .addrsig_sym _f1
@@ -118,8 +131,8 @@ declare void @f3() unnamed_addr
@a1 = alias i32, i32* @g1
@a2 = internal local_unnamed_addr alias i32, i32* @g2
- at i1 = external ifunc void(), void()* ()* @f1
- at i2 = external local_unnamed_addr ifunc void(), void()* ()* @f2
+ at i1 = ifunc void(), void()* ()* @f1
+ at i2 = internal local_unnamed_addr ifunc void(), void()* ()* @f2
declare void @llvm.dbg.value(metadata, metadata, metadata)
diff --git a/llvm/test/CodeGen/AArch64/ifunc-asm.ll b/llvm/test/CodeGen/AArch64/ifunc-asm.ll
index fbc0f74cee46baa..ede669aa52703f7 100644
--- a/llvm/test/CodeGen/AArch64/ifunc-asm.ll
+++ b/llvm/test/CodeGen/AArch64/ifunc-asm.ll
@@ -3,79 +3,82 @@
; RUN: llc -mtriple=arm64-apple-darwin %s -filetype=asm -o - -arm64-darwin-ifunc-symbol_resolver=if_supported | FileCheck %s --check-prefixes=MACHO,MACHO-DEFAULT
; RUN: llc -mtriple=arm64-apple-darwin %s -filetype=asm -o - -arm64-darwin-ifunc-symbol_resolver=never | FileCheck %s --check-prefixes=MACHO,MACHO-MANUAL
; RUN: llc -mtriple=arm64-apple-darwin %s -filetype=asm -o - | FileCheck %s --check-prefixes=MACHO,MACHO-MANUAL
+; RUN: llc -mtriple=arm64-apple-darwin %s -filetype=asm -o - -global-isel | FileCheck %s --check-prefixes=MACHO,MACHO-MANUAL
-define internal ptr @foo_resolver() {
+define internal ptr @the_resolver() {
entry:
ret ptr null
}
-; ELF: .type foo_resolver, at function
-; ELF-NEXT: foo_resolver:
+; ELF: .type the_resolver, at function
+; ELF-NEXT: the_resolver:
; MACHO: .p2align 2
-; MACHO-NEXT: _foo_resolver
+; MACHO-NEXT: _the_resolver
- at foo_ifunc = ifunc i32 (i32), ptr @foo_resolver
-; ELF: .globl foo_ifunc
-; ELF-NEXT: .type foo_ifunc, at gnu_indirect_function
-; ELF-NEXT: .set foo_ifunc, foo_resolver
+ at global_ifunc = ifunc i32 (i32), ptr @the_resolver
+; ELF: .globl global_ifunc
+; ELF-NEXT: .type global_ifunc, at gnu_indirect_function
+; ELF-NEXT: .set global_ifunc, the_resolver
-; MACHO-LINKER: .globl _foo_ifunc
+; MACHO-LINKER: .globl _global_ifunc
; MACHO-LINKER-NEXT: .p2align 2
-; MACHO-LINKER-NEXT: _foo_ifunc:
-; MACHO-LINKER-NEXT: .symbol_resolver _foo_ifunc
-; MACHO-LINKER-NEXT: b _foo_resolver
+; MACHO-LINKER-NEXT: _global_ifunc:
+; MACHO-LINKER-NEXT: .symbol_resolver _global_ifunc
+; MACHO-LINKER-NEXT: b _the_resolver
-; MACHO-DEFAULT: .globl _foo_ifunc
+; MACHO-DEFAULT: .globl _global_ifunc
; MACHO-DEFAULT-NEXT: .p2align 2
-; MACHO-DEFAULT-NEXT: _foo_ifunc:
-; MACHO-DEFAULT-NEXT: .symbol_resolver _foo_ifunc
-; MACHO-DEFAULT-NEXT: b _foo_resolver
+; MACHO-DEFAULT-NEXT: _global_ifunc:
+; MACHO-DEFAULT-NEXT: .symbol_resolver _global_ifunc
+; MACHO-DEFAULT-NEXT: b _the_resolver
; MACHO-MANUAL: .section __DATA,__data
-; MACHO-MANUAL-NEXT: .globl _foo_ifunc.lazy_pointer
-; MACHO-MANUAL-NEXT: _foo_ifunc.lazy_pointer:
-; MACHO-MANUAL-NEXT: .quad _foo_ifunc.stub_helper
+; MACHO-MANUAL-NEXT: .globl _global_ifunc.lazy_pointer
+; MACHO-MANUAL-NEXT: _global_ifunc.lazy_pointer:
+; MACHO-MANUAL-NEXT: .quad _global_ifunc.stub_helper
; MACHO-MANUAL: .section __TEXT,__text,regular,pure_instructions
-; MACHO-MANUAL-NEXT: .globl _foo_ifunc
+; MACHO-MANUAL-NEXT: .globl _global_ifunc
; MACHO-MANUAL-NEXT: .p2align 2
-; MACHO-MANUAL-NEXT: _foo_ifunc:
-; MACHO-MANUAL-NEXT: adrp x16, _foo_ifunc.lazy_pointer at GOTPAGE
-; MACHO-MANUAL-NEXT: ldr x16, [x16, _foo_ifunc.lazy_pointer at GOTPAGEOFF]
+; MACHO-MANUAL-NEXT: _global_ifunc:
+; MACHO-MANUAL-NEXT: adrp x16, _global_ifunc.lazy_pointer at GOTPAGE
+; MACHO-MANUAL-NEXT: ldr x16, [x16, _global_ifunc.lazy_pointer at GOTPAGEOFF]
; MACHO-MANUAL-NEXT: ldr x16, [x16]
; MACHO-MANUAL-NEXT: br x16
-; MACHO-MANUAL-NEXT: .globl _foo_ifunc.stub_helper
+; MACHO-MANUAL-NEXT: .globl _global_ifunc.stub_helper
; MACHO-MANUAL-NEXT: .p2align 2
-; MACHO-MANUAL-NEXT: _foo_ifunc.stub_helper:
-; MACHO-MANUAL-NEXT: stp x29, x30, [sp, #-16]!
-; MACHO-MANUAL-NEXT: mov x29, sp
-; MACHO-MANUAL-NEXT: stp x1, x0, [sp, #-16]!
-; MACHO-MANUAL-NEXT: stp x3, x2, [sp, #-16]!
-; MACHO-MANUAL-NEXT: stp x5, x4, [sp, #-16]!
-; MACHO-MANUAL-NEXT: stp x7, x6, [sp, #-16]!
-; MACHO-MANUAL-NEXT: stp d1, d0, [sp, #-16]!
-; MACHO-MANUAL-NEXT: stp d3, d2, [sp, #-16]!
-; MACHO-MANUAL-NEXT: stp d5, d4, [sp, #-16]!
-; MACHO-MANUAL-NEXT: stp d7, d6, [sp, #-16]!
-; MACHO-MANUAL-NEXT: bl _foo_resolver
-; MACHO-MANUAL-NEXT: adrp x16, _foo_ifunc.lazy_pointer at GOTPAGE
-; MACHO-MANUAL-NEXT: ldr x16, [x16, _foo_ifunc.lazy_pointer at GOTPAGEOFF]
+; MACHO-MANUAL-NEXT: _global_ifunc.stub_helper:
+; MACHO-MANUAL-NEXT: stp x29, x30, [sp, #-16]
+; MACHO-MANUAL-NEXT: sub x29, sp, #16
+; MACHO-MANUAL-NEXT: stp x1, x0, [sp, #-32]
+; MACHO-MANUAL-NEXT: stp x3, x2, [sp, #-48]
+; MACHO-MANUAL-NEXT: stp x5, x4, [sp, #-64]
+; MACHO-MANUAL-NEXT: stp x7, x6, [sp, #-80]
+; MACHO-MANUAL-NEXT: stp d1, d0, [sp, #-96]
+; MACHO-MANUAL-NEXT: stp d3, d2, [sp, #-112]
+; MACHO-MANUAL-NEXT: stp d5, d4, [sp, #-128]
+; MACHO-MANUAL-NEXT: stp d7, d6, [sp, #-144]
+; MACHO-MANUAL-NEXT: sub sp, sp, #144
+; MACHO-MANUAL-NEXT: bl _the_resolver
+; MACHO-MANUAL-NEXT: adrp x16, _global_ifunc.lazy_pointer at GOTPAGE
+; MACHO-MANUAL-NEXT: ldr x16, [x16, _global_ifunc.lazy_pointer at GOTPAGEOFF]
; MACHO-MANUAL-NEXT: str x0, [x16]
; MACHO-MANUAL-NEXT: add x16, x0, #0
-; MACHO-MANUAL-NEXT: ldp d7, d6, [sp], #16
-; MACHO-MANUAL-NEXT: ldp d5, d4, [sp], #16
-; MACHO-MANUAL-NEXT: ldp d3, d2, [sp], #16
-; MACHO-MANUAL-NEXT: ldp d1, d0, [sp], #16
-; MACHO-MANUAL-NEXT: ldp x7, x6, [sp], #16
-; MACHO-MANUAL-NEXT: ldp x5, x4, [sp], #16
-; MACHO-MANUAL-NEXT: ldp x3, x2, [sp], #16
-; MACHO-MANUAL-NEXT: ldp x1, x0, [sp], #16
-; MACHO-MANUAL-NEXT: ldp x29, x30, [sp], #16
+; MACHO-MANUAL-NEXT: add sp, sp, #144
+; MACHO-MANUAL-NEXT: ldp d7, d6, [sp, #-144]
+; MACHO-MANUAL-NEXT: ldp d5, d4, [sp, #-128]
+; MACHO-MANUAL-NEXT: ldp d3, d2, [sp, #-112]
+; MACHO-MANUAL-NEXT: ldp d1, d0, [sp, #-96]
+; MACHO-MANUAL-NEXT: ldp x7, x6, [sp, #-80]
+; MACHO-MANUAL-NEXT: ldp x5, x4, [sp, #-64]
+; MACHO-MANUAL-NEXT: ldp x3, x2, [sp, #-48]
+; MACHO-MANUAL-NEXT: ldp x1, x0, [sp, #-32]
+; MACHO-MANUAL-NEXT: ldp x29, x30, [sp, #-16]
; MACHO-MANUAL-NEXT: br x16
- at weak_ifunc = weak ifunc i32 (i32), ptr @foo_resolver
+ at weak_ifunc = weak ifunc i32 (i32), ptr @the_resolver
; ELF: .type weak_ifunc, at gnu_indirect_function
; MACHO-LINKER: .symbol_resolver _weak_ifunc
; MACHO-MANUAL: _weak_ifunc.stub_helper:
diff --git a/llvm/test/Verifier/ifunc-macho.ll b/llvm/test/Verifier/ifunc-macho.ll
deleted file mode 100644
index 2e2166645983ac3..000000000000000
--- a/llvm/test/Verifier/ifunc-macho.ll
+++ /dev/null
@@ -1,42 +0,0 @@
-; RUN: not llvm-as %s -o /dev/null 2>&1 | FileCheck %s
-
-target triple = "arm64-apple-ios"
-
-define ptr @resolver() {
- ret ptr null
-}
-
- at g = external global i32
- at inval_objtype = ifunc void (), ptr @g
-; CHECK: IFunc must have a Function resolver
-
-declare ptr @resolver_decl()
- at inval_resolver_decl = ifunc void (), ptr @resolver_decl
-; CHECK: IFunc resolver must be a definition
-; CHECK-NEXT: @inval_resolver_decl
-
-define available_externally ptr @resolver_linker_decl() {
- ret ptr null
-}
- at inval_resolver_decl2 = ifunc void (), ptr @resolver_linker_decl
-; CHECK: IFunc resolver must be a definition
-; CHECK-NEXT: @inval_resolver_decl2
-
- at ifunc_nonpointer_return_type = ifunc i32 (), ptr @resolver_returns_nonpointer
-; CHECK: IFunc resolver must return a pointer
-; CHECK-NEXT: ptr @ifunc_nonpointer_return_type
-
-define i32 @resolver_returns_nonpointer() {
- ret i32 0
-}
-
- at valid_external = ifunc void (), ptr @resolver
-; CHECK-NOT: valid_external
-
- at inval_linkonce = linkonce ifunc void (), ptr @resolver
-
- at inval_weak = weak ifunc void (), ptr @resolver
-
- at inval_weak_extern = extern_weak ifunc void (), ptr @resolver
-
- at inval_private = private ifunc void (), ptr @resolver
>From bbeb3beaf37d91911a96a20bb1825fa16dcfc094 Mon Sep 17 00:00:00 2001
From: Jon Roelofs <jonathan_roelofs at apple.com>
Date: Wed, 29 Nov 2023 08:23:21 -0800
Subject: [PATCH 4/9] x86 support
Created using spr 1.3.4
---
llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 72 +++++-----
llvm/lib/Target/X86/X86AsmPrinter.cpp | 124 +++++++++++++++---
llvm/test/CodeGen/X86/ifunc-asm.ll | 48 +++++--
3 files changed, 178 insertions(+), 66 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index 26b3a14e22b2ad9..1b5b7c556c79f6b 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -1901,6 +1901,9 @@ void AArch64AsmPrinter::emitManualSymbolResolver(Module &M,
OutStreamer->switchSection(OutContext.getObjectFileInfo()->getDataSection());
+ // _ifunc.lazy_pointer:
+ // .quad _ifunc.stub_helper
+
EmitLinkage(LazyPointer);
OutStreamer->emitLabel(LazyPointer);
emitVisibility(LazyPointer, GI.getVisibility());
@@ -1908,18 +1911,18 @@ void AArch64AsmPrinter::emitManualSymbolResolver(Module &M,
OutStreamer->switchSection(OutContext.getObjectFileInfo()->getTextSection());
- MCSymbol *Stub = getSymbol(&GI);
+ // _ifunc:
+ // adrp x16, lazy_pointer at GOTPAGE
+ // ldr x16, [x16, lazy_pointer at GOTPAGEOFF]
+ // ldr x16, [x16]
+ // br x16
+ MCSymbol *Stub = getSymbol(&GI);
EmitLinkage(Stub);
OutStreamer->emitCodeAlignment(Align(4), STI);
OutStreamer->emitLabel(Stub);
emitVisibility(Stub, GI.getVisibility());
- // adrp x16, lazy_pointer at GOTPAGE
- // ldr x16, [x16, lazy_pointer at GOTPAGEOFF]
- // ldr x16, [x16]
- // br x16
-
{
MCInst Adrp;
Adrp.setOpcode(AArch64::ADRP);
@@ -1960,39 +1963,40 @@ void AArch64AsmPrinter::emitManualSymbolResolver(Module &M,
.addReg(AArch64::X16),
*STI);
+ // _ifunc.stub_helper:
+ // stp fp, lr, [sp, #-16]
+ // sub fp, sp, 16
+ // stp x1, x0, [sp, #-32]
+ // stp x3, x2, [sp, #-48]
+ // stp x5, x4, [sp, #-64]
+ // stp x7, x6, [sp, #-80]
+ // stp d1, d0, [sp, #-96]
+ // stp d3, d2, [sp, #-112]
+ // stp d5, d4, [sp, #-128]
+ // stp d7, d6, [sp, #-144]
+ // sub sp, sp, 144
+ // bl _resolver
+ // adrp x16, lazy_pointer at GOTPAGE
+ // ldr x16, [x16, lazy_pointer at GOTPAGEOFF]
+ // str x0, [x16]
+ // mov x16, x0
+ // add sp, sp, 144
+ // ldp d7, d6, [sp, #-144]
+ // ldp d5, d4, [sp, #-128]
+ // ldp d3, d2, [sp, #-112]
+ // ldp d1, d0, [sp, #-96]
+ // ldp x7, x6, [sp, #-80]
+ // ldp x5, x4, [sp, #-64]
+ // ldp x3, x2, [sp, #-48]
+ // ldp x1, x0, [sp, #-32]
+ // ldp fp, lr, [sp, #-16]
+ // br x16
+
EmitLinkage(StubHelper);
OutStreamer->emitCodeAlignment(Align(4), STI);
OutStreamer->emitLabel(StubHelper);
emitVisibility(StubHelper, GI.getVisibility());
- // stp fp, lr, [sp, #-16]
- // sub fp, sp, 16
- // stp x1, x0, [sp, #-32]
- // stp x3, x2, [sp, #-48]
- // stp x5, x4, [sp, #-64]
- // stp x7, x6, [sp, #-80]
- // stp d1, d0, [sp, #-96]
- // stp d3, d2, [sp, #-112]
- // stp d5, d4, [sp, #-128]
- // stp d7, d6, [sp, #-144]
- // sub sp, sp, 144
- // bl _resolver
- // adrp x16, lazy_pointer at GOTPAGE
- // ldr x16, [x16, lazy_pointer at GOTPAGEOFF]
- // str x0, [x16]
- // mov x16, x0
- // add sp, sp, 144
- // ldp d7, d6, [sp, #-144]
- // ldp d5, d4, [sp, #-128]
- // ldp d3, d2, [sp, #-112]
- // ldp d1, d0, [sp, #-96]
- // ldp x7, x6, [sp, #-80]
- // ldp x5, x4, [sp, #-64]
- // ldp x3, x2, [sp, #-48]
- // ldp x1, x0, [sp, #-32]
- // ldp fp, lr, [sp, #-16]
- // br x16
-
OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXi)
.addReg(AArch64::FP)
.addReg(AArch64::LR)
diff --git a/llvm/lib/Target/X86/X86AsmPrinter.cpp b/llvm/lib/Target/X86/X86AsmPrinter.cpp
index 37158900d2404dd..b0f4b9d98437284 100644
--- a/llvm/lib/Target/X86/X86AsmPrinter.cpp
+++ b/llvm/lib/Target/X86/X86AsmPrinter.cpp
@@ -14,6 +14,7 @@
#include "X86AsmPrinter.h"
#include "MCTargetDesc/X86ATTInstPrinter.h"
#include "MCTargetDesc/X86BaseInfo.h"
+#include "MCTargetDesc/X86MCTargetDesc.h"
#include "MCTargetDesc/X86TargetStreamer.h"
#include "TargetInfo/X86TargetInfo.h"
#include "X86InstrInfo.h"
@@ -34,6 +35,7 @@
#include "llvm/MC/MCCodeEmitter.h"
#include "llvm/MC/MCContext.h"
#include "llvm/MC/MCExpr.h"
+#include "llvm/MC/MCInst.h"
#include "llvm/MC/MCInstBuilder.h"
#include "llvm/MC/MCSectionCOFF.h"
#include "llvm/MC/MCSectionELF.h"
@@ -534,30 +536,112 @@ void X86AsmPrinter::emitGlobalIFunc(Module &M, const GlobalIFunc &GI) {
if (!TM.getTargetTriple().isOSBinFormatMachO())
return AsmPrinter::emitGlobalIFunc(M, GI);
- OutStreamer->switchSection(OutContext.getObjectFileInfo()->getTextSection());
+ auto EmitLinkage = [&](MCSymbol *Sym) {
+ if (GI.hasExternalLinkage() || !MAI->getWeakRefDirective())
+ OutStreamer->emitSymbolAttribute(Sym, MCSA_Global);
+ else if (GI.hasWeakLinkage() || GI.hasLinkOnceLinkage())
+ OutStreamer->emitSymbolAttribute(Sym, MCSA_WeakReference);
+ else
+ assert(GI.hasLocalLinkage() && "Invalid ifunc linkage");
+ };
- MCSymbol *Name = getSymbol(&GI);
+ MCSymbol *LazyPointer =
+ TM.getObjFileLowering()->getContext().getOrCreateSymbol(
+ "_" + GI.getName() + ".lazy_pointer");
+ MCSymbol *StubHelper =
+ TM.getObjFileLowering()->getContext().getOrCreateSymbol(
+ "_" + GI.getName() + ".stub_helper");
- if (GI.hasExternalLinkage() || !MAI->getWeakRefDirective())
- OutStreamer->emitSymbolAttribute(Name, MCSA_Global);
- else if (GI.hasWeakLinkage() || GI.hasLinkOnceLinkage())
- OutStreamer->emitSymbolAttribute(Name, MCSA_WeakReference);
- else
- assert(GI.hasLocalLinkage() && "Invalid ifunc linkage");
+ OutStreamer->switchSection(OutContext.getObjectFileInfo()->getDataSection());
+
+ // _ifunc.lazy_pointer:
+ // .quad _ifunc.stub_helper
+ EmitLinkage(LazyPointer);
+ OutStreamer->emitLabel(LazyPointer);
+ emitVisibility(LazyPointer, GI.getVisibility());
+ OutStreamer->emitValue(MCSymbolRefExpr::create(StubHelper, OutContext), 8);
+
+ OutStreamer->switchSection(OutContext.getObjectFileInfo()->getTextSection());
+
+ // _ifunc:
+ // jmpq *lazy_pointer(%rip)
+
+ MCSymbol *Stub = getSymbol(&GI);
+ EmitLinkage(Stub);
+ OutStreamer->emitCodeAlignment(Align(16), Subtarget);
+ OutStreamer->emitLabel(Stub);
+ emitVisibility(Stub, GI.getVisibility());
+
+ OutStreamer->emitInstruction(
+ MCInstBuilder(X86::JMP32m)
+ .addReg(X86::RIP)
+ .addImm(1)
+ .addReg(0)
+ .addOperand(MCOperand::createExpr(
+ MCSymbolRefExpr::create(LazyPointer, OutContext)))
+ .addReg(0),
+ *Subtarget);
+
+ // _ifunc.stub_helper:
+ // push %rax
+ // push %rdi
+ // push %rsi
+ // push %rdx
+ // push %rcx
+ // push %r8
+ // push %r9
+ // callq foo
+ // movq %rax,lazy_pointer(%rip)
+ // pop %r9
+ // pop %r8
+ // pop %rcx
+ // pop %rdx
+ // pop %rsi
+ // pop %rdi
+ // pop %rax
+ // jmpq *lazy_pointer(%rip)
+
+ EmitLinkage(StubHelper);
OutStreamer->emitCodeAlignment(Align(16), Subtarget);
- OutStreamer->emitLabel(Name);
- OutStreamer->emitSymbolAttribute(Name, MCSA_SymbolResolver);
- emitVisibility(Name, GI.getVisibility());
-
- // ld64 does not seem to support aliases of symbol resolvers, so we have to
- // tail call the resolver manually.
- MCInst JMP;
- JMP.setOpcode(X86::JMP_4);
- JMP.addOperand(MCOperand::createExpr(lowerConstant(GI.getResolver())));
- OutStreamer->emitInstruction(JMP, *Subtarget);
-
- // FIXME: do the manual .symbol_resolver lowering that we did in AArch64AsmPrinter.
+ OutStreamer->emitLabel(StubHelper);
+ emitVisibility(StubHelper, GI.getVisibility());
+
+ for (int Reg :
+ {X86::RAX, X86::RDI, X86::RSI, X86::RDX, X86::RCX, X86::R8, X86::R9})
+ OutStreamer->emitInstruction(MCInstBuilder(X86::PUSH64r).addReg(Reg),
+ *Subtarget);
+
+ OutStreamer->emitInstruction(
+ MCInstBuilder(X86::CALL64pcrel32)
+ .addOperand(MCOperand::createExpr(lowerConstant(GI.getResolver()))),
+ *Subtarget);
+
+ OutStreamer->emitInstruction(
+ MCInstBuilder(X86::MOV64mr)
+ .addReg(X86::RIP)
+ .addImm(1)
+ .addReg(0)
+ .addOperand(MCOperand::createExpr(
+ MCSymbolRefExpr::create(LazyPointer, OutContext)))
+ .addReg(0)
+ .addReg(X86::RAX),
+ *Subtarget);
+
+ for (int Reg :
+ {X86::R9, X86::R8, X86::RCX, X86::RDX, X86::RSI, X86::RDI, X86::RAX})
+ OutStreamer->emitInstruction(MCInstBuilder(X86::POP64r).addReg(Reg),
+ *Subtarget);
+
+ OutStreamer->emitInstruction(
+ MCInstBuilder(X86::JMP32m)
+ .addReg(X86::RIP)
+ .addImm(1)
+ .addReg(0)
+ .addOperand(MCOperand::createExpr(
+ MCSymbolRefExpr::create(LazyPointer, OutContext)))
+ .addReg(0),
+ *Subtarget);
}
static bool printAsmMRegister(const X86AsmPrinter &P, const MachineOperand &MO,
diff --git a/llvm/test/CodeGen/X86/ifunc-asm.ll b/llvm/test/CodeGen/X86/ifunc-asm.ll
index 76efda71153207d..0f66febbe95b2da 100644
--- a/llvm/test/CodeGen/X86/ifunc-asm.ll
+++ b/llvm/test/CodeGen/X86/ifunc-asm.ll
@@ -5,20 +5,44 @@ define internal ptr @foo_resolver() {
entry:
ret ptr null
}
-; ELF: .type foo_resolver, at function
-; ELF-NEXT: foo_resolver:
+; ELF: .type foo_resolver, at function
+; ELF-NEXT: foo_resolver:
-; MACHO: .p2align 4, 0x90
-; MACHO-NEXT: _foo_resolver
+; MACHO: .p2align 4, 0x90
+; MACHO-NEXT: _foo_resolver
@foo_ifunc = ifunc i32 (i32), ptr @foo_resolver
-; ELF: .globl foo_ifunc
-; ELF-NEXT: .type foo_ifunc, at gnu_indirect_function
-; ELF-NEXT: .set foo_ifunc, foo_resolver
+; ELF: .globl foo_ifunc
+; ELF-NEXT: .type foo_ifunc, at gnu_indirect_function
+; ELF-NEXT: .set foo_ifunc, foo_resolver
-; MACHO: .globl _foo_ifunc
-; MACHO-NEXT: .p2align 4, 0x90
-; MACHO-NEXT: _foo_ifunc:
-; MACHO-NEXT: .symbol_resolver _foo_ifunc
-; MACHO-NEXT: jmp _foo_resolver
+; MACHO: .section __DATA,__data
+; MACHO-NEXT: .globl _foo_ifunc.lazy_pointer
+; MACHO-NEXT: _foo_ifunc.lazy_pointer:
+; MACHO-NEXT: .quad _foo_ifunc.stub_helper
+; MACHO-NEXT: .section __TEXT,__text,regular,pure_instructions
+; MACHO-NEXT: .globl _foo_ifunc
+; MACHO-NEXT: .p2align 4, 0x90
+; MACHO-NEXT: _foo_ifunc:
+; MACHO-NEXT: jmpl *_foo_ifunc.lazy_pointer(%rip)
+; MACHO-NEXT: .globl _foo_ifunc.stub_helper
+; MACHO-NEXT: .p2align 4, 0x90
+; MACHO-NEXT: _foo_ifunc.stub_helper:
+; MACHO-NEXT: pushq %rax
+; MACHO-NEXT: pushq %rdi
+; MACHO-NEXT: pushq %rsi
+; MACHO-NEXT: pushq %rdx
+; MACHO-NEXT: pushq %rcx
+; MACHO-NEXT: pushq %r8
+; MACHO-NEXT: pushq %r9
+; MACHO-NEXT: callq _foo_resolver
+; MACHO-NEXT: movq %rax, _foo_ifunc.lazy_pointer(%rip)
+; MACHO-NEXT: popq %r9
+; MACHO-NEXT: popq %r8
+; MACHO-NEXT: popq %rcx
+; MACHO-NEXT: popq %rdx
+; MACHO-NEXT: popq %rsi
+; MACHO-NEXT: popq %rdi
+; MACHO-NEXT: popq %rax
+; MACHO-NEXT: jmpl *_foo_ifunc.lazy_pointer(%rip)
\ No newline at end of file
>From b2d46d62984b685cd94597c957708e7b5b34d671 Mon Sep 17 00:00:00 2001
From: Jon Roelofs <jonathan_roelofs at apple.com>
Date: Wed, 29 Nov 2023 13:56:55 -0800
Subject: [PATCH 5/9] review feedback
Created using spr 1.3.4
---
llvm/docs/LangRef.rst | 2 +-
llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index cb222e979db29d4..fece4ac7f127d6b 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -937,7 +937,7 @@ IFuncs, like as aliases, don't create any new data or func. They are just a new
symbol that is resolved at runtime by calling a resolver function.
On ELF platforms, IFuncs are resolved by the dynamic linker at load time. On
-MachO platforms, they are lowered in terms of ``.symbol_resolver``s, which
+Mach-O platforms, they are lowered in terms of ``.symbol_resolver``s, which
lazily resolve the callee the first time they are called.
IFunc may have an optional :ref:`linkage type <linkage>` and an optional
diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index b4ac0a70e7fde9c..aaa7693c61f0e85 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -2321,7 +2321,7 @@ bool AsmPrinter::doFinalization(Module &M) {
}
// IFuncs must come before deubginfo in case the backend decides to emit them
- // as actual functions, since on MachO targets, we cannot create regular
+ // as actual functions, since on Mach-O targets, we cannot create regular
// sections after DWARF.
for (const auto &IFunc : M.ifuncs())
emitGlobalIFunc(M, IFunc);
>From e6af9eaa469fa0f2cf3b1df7ec80bed3c2354cbe Mon Sep 17 00:00:00 2001
From: Jon Roelofs <jonathan_roelofs at apple.com>
Date: Wed, 29 Nov 2023 14:07:49 -0800
Subject: [PATCH 6/9] fix docs build
Created using spr 1.3.4
---
llvm/docs/LangRef.rst | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index fece4ac7f127d6b..51c60ecf5ac6586 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -937,8 +937,8 @@ IFuncs, like as aliases, don't create any new data or func. They are just a new
symbol that is resolved at runtime by calling a resolver function.
On ELF platforms, IFuncs are resolved by the dynamic linker at load time. On
-Mach-O platforms, they are lowered in terms of ``.symbol_resolver``s, which
-lazily resolve the callee the first time they are called.
+Mach-O platforms, they are lowered in terms of ``.symbol_resolver`` functions,
+which lazily resolve the callee the first time they are called.
IFunc may have an optional :ref:`linkage type <linkage>` and an optional
:ref:`visibility style <visibility>`.
>From a9cc93c37fba9e40df0d983f1990908cc31dfe76 Mon Sep 17 00:00:00 2001
From: Jon Roelofs <jonathan_roelofs at apple.com>
Date: Wed, 29 Nov 2023 15:19:14 -0800
Subject: [PATCH 7/9] align data pointer
Created using spr 1.3.4
---
llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 1 +
llvm/lib/Target/X86/X86AsmPrinter.cpp | 1 +
llvm/test/CodeGen/AArch64/ifunc-asm.ll | 1 +
llvm/test/CodeGen/X86/ifunc-asm.ll | 1 +
4 files changed, 4 insertions(+)
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index 1b5b7c556c79f6b..a3513ca439ac261 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -1905,6 +1905,7 @@ void AArch64AsmPrinter::emitManualSymbolResolver(Module &M,
// .quad _ifunc.stub_helper
EmitLinkage(LazyPointer);
+ OutStreamer->emitValueToAlignment(Align(8), /*Value=*/0);
OutStreamer->emitLabel(LazyPointer);
emitVisibility(LazyPointer, GI.getVisibility());
OutStreamer->emitValue(MCSymbolRefExpr::create(StubHelper, OutContext), 8);
diff --git a/llvm/lib/Target/X86/X86AsmPrinter.cpp b/llvm/lib/Target/X86/X86AsmPrinter.cpp
index b0f4b9d98437284..a490d775abd551e 100644
--- a/llvm/lib/Target/X86/X86AsmPrinter.cpp
+++ b/llvm/lib/Target/X86/X86AsmPrinter.cpp
@@ -558,6 +558,7 @@ void X86AsmPrinter::emitGlobalIFunc(Module &M, const GlobalIFunc &GI) {
// .quad _ifunc.stub_helper
EmitLinkage(LazyPointer);
+ OutStreamer->emitValueToAlignment(Align(8), /*Value=*/0);
OutStreamer->emitLabel(LazyPointer);
emitVisibility(LazyPointer, GI.getVisibility());
OutStreamer->emitValue(MCSymbolRefExpr::create(StubHelper, OutContext), 8);
diff --git a/llvm/test/CodeGen/AArch64/ifunc-asm.ll b/llvm/test/CodeGen/AArch64/ifunc-asm.ll
index ede669aa52703f7..7eff692da83ff90 100644
--- a/llvm/test/CodeGen/AArch64/ifunc-asm.ll
+++ b/llvm/test/CodeGen/AArch64/ifunc-asm.ll
@@ -35,6 +35,7 @@ entry:
; MACHO-MANUAL: .section __DATA,__data
; MACHO-MANUAL-NEXT: .globl _global_ifunc.lazy_pointer
+; MACHO-MANUAL-NEXT: .p2align 3, 0x0
; MACHO-MANUAL-NEXT: _global_ifunc.lazy_pointer:
; MACHO-MANUAL-NEXT: .quad _global_ifunc.stub_helper
diff --git a/llvm/test/CodeGen/X86/ifunc-asm.ll b/llvm/test/CodeGen/X86/ifunc-asm.ll
index 0f66febbe95b2da..39c326bc6b6c37c 100644
--- a/llvm/test/CodeGen/X86/ifunc-asm.ll
+++ b/llvm/test/CodeGen/X86/ifunc-asm.ll
@@ -19,6 +19,7 @@ entry:
; MACHO: .section __DATA,__data
; MACHO-NEXT: .globl _foo_ifunc.lazy_pointer
+; MACHO-NEXT: .p2align 3, 0x0
; MACHO-NEXT: _foo_ifunc.lazy_pointer:
; MACHO-NEXT: .quad _foo_ifunc.stub_helper
; MACHO-NEXT: .section __TEXT,__text,regular,pure_instructions
>From 6fe823c600230e92eb790dcca58a5f41e1e7900e Mon Sep 17 00:00:00 2001
From: Jon Roelofs <jonathan_roelofs at apple.com>
Date: Fri, 1 Dec 2023 13:08:03 -0800
Subject: [PATCH 8/9] drop the non-manual .symbol_resolver lowering
Created using spr 1.3.4
---
llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 220 ++++++------------
llvm/test/CodeGen/AArch64/ifunc-asm.ll | 126 +++++-----
2 files changed, 128 insertions(+), 218 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index a3513ca439ac261..f19dcb5be172271 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -71,27 +71,6 @@ using namespace llvm;
namespace {
-enum class IFuncLowering {
- SymbolResolverIfSupported,
- SymbolResolverAlways,
- SymbolResolverNever
-};
-
-static cl::opt<IFuncLowering> PreferredIFuncLowering(
- "arm64-darwin-ifunc-symbol_resolver",
- cl::init(IFuncLowering::SymbolResolverNever),
- cl::desc("Pick the lowering for ifuncs on darwin platforms"), cl::Hidden,
- cl::values(
- clEnumValN(
- IFuncLowering::SymbolResolverIfSupported, "if_supported",
- "Use .symbol_resolver's when known to be supported by the linker."),
- clEnumValN(IFuncLowering::SymbolResolverAlways, "always",
- "Always use .symbol_resolvers. NOTE: this might not be "
- "supported by the linker in all cases."),
- clEnumValN(IFuncLowering::SymbolResolverNever, "never",
- "Use a manual lowering, doing what the linker would have "
- "done, but in the compiler.")));
-
class AArch64AsmPrinter : public AsmPrinter {
AArch64MCInstLower MCInstLowering;
FaultMaps FM;
@@ -224,9 +203,6 @@ class AArch64AsmPrinter : public AsmPrinter {
}
void emitGlobalIFunc(Module &M, const GlobalIFunc &GI) override;
-
- void emitLinkerSymbolResolver(Module &M, const GlobalIFunc &GI);
- void emitManualSymbolResolver(Module &M, const GlobalIFunc &GI);
};
} // end anonymous namespace
@@ -1838,51 +1814,26 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
EmitToStreamer(*OutStreamer, TmpInst);
}
-void AArch64AsmPrinter::emitLinkerSymbolResolver(Module &M,
- const GlobalIFunc &GI) {
- OutStreamer->switchSection(OutContext.getObjectFileInfo()->getTextSection());
-
- MCSymbol *Name = getSymbol(&GI);
-
- // NOTE: non-global .symbol_resolvers are not yet supported by Darwin linkers
-
- if (GI.hasExternalLinkage() || !MAI->getWeakRefDirective())
- OutStreamer->emitSymbolAttribute(Name, MCSA_Global);
- else if (GI.hasWeakLinkage() || GI.hasLinkOnceLinkage())
- OutStreamer->emitSymbolAttribute(Name, MCSA_WeakReference);
- else
- assert(GI.hasLocalLinkage() && "Invalid ifunc linkage");
-
- OutStreamer->emitCodeAlignment(Align(4), STI);
- OutStreamer->emitLabel(Name);
- OutStreamer->emitSymbolAttribute(Name, MCSA_SymbolResolver);
- emitVisibility(Name, GI.getVisibility());
+void AArch64AsmPrinter::emitGlobalIFunc(Module &M, const GlobalIFunc &GI) {
+ if (!TM.getTargetTriple().isOSBinFormatMachO())
+ return AsmPrinter::emitGlobalIFunc(M, GI);
- // ld-prime does not seem to support aliases of symbol resolvers, so we have
- // to tail call the resolver manually.
- OutStreamer->emitInstruction(
- MCInstBuilder(AArch64::B)
- .addOperand(MCOperand::createExpr(lowerConstant(GI.getResolver()))),
- *STI);
-}
+ // On Darwin platforms, emit a manually-constructed .symbol_resolver that
+ // implements the symbol resolution duties of the IFunc.
+ //
+ // Normally, this would be handled by linker magic, but unfortunately there are
+ // a few limitations in ld64 and ld-prime's implementation of .symbol_resolver
+ // that mean we can't always use them:
+ //
+ // * resolvers cannot be the target of an alias
+ // * resolvers cannot have private linkage
+ // * resolvers cannot have linkonce linkage
+ // * resolvers cannot appear in executables
+ // * resolvers cannot appear in bundles
+ //
+ // This works around that by emitting a close approximation of what the linker
+ // would have done.
-/// \brief Emit a manually-constructed .symbol_resolver that implements the
-/// symbol resolution duties of the IFunc.
-///
-/// Normally, this would be handled by linker magic, but unfortunately there are
-/// a few limitations in ld64 and ld-prime's implementation of .symbol_resolver
-/// that mean we can't always use them:
-///
-/// * resolvers cannot be the target of an alias
-/// * resolvers cannot have private linkage
-/// * resolvers cannot have linkonce linkage
-/// * resolvers cannot appear in executables
-/// * resolvers cannot appear in bundles
-///
-/// This works around that by emitting a close approximation of what the linker
-/// would have done.
-void AArch64AsmPrinter::emitManualSymbolResolver(Module &M,
- const GlobalIFunc &GI) {
auto EmitLinkage = [&](MCSymbol *Sym) {
if (GI.hasExternalLinkage() || !MAI->getWeakRefDirective())
OutStreamer->emitSymbolAttribute(Sym, MCSA_Global);
@@ -1964,33 +1915,35 @@ void AArch64AsmPrinter::emitManualSymbolResolver(Module &M,
.addReg(AArch64::X16),
*STI);
+ // These stub helpers are only ever called once, so here we're optimizing for
+ // minimum size by using the pre-indexed store variants, which saves a few
+ // bytes of instructions to bump & restore sp.
+
// _ifunc.stub_helper:
- // stp fp, lr, [sp, #-16]
- // sub fp, sp, 16
- // stp x1, x0, [sp, #-32]
- // stp x3, x2, [sp, #-48]
- // stp x5, x4, [sp, #-64]
- // stp x7, x6, [sp, #-80]
- // stp d1, d0, [sp, #-96]
- // stp d3, d2, [sp, #-112]
- // stp d5, d4, [sp, #-128]
- // stp d7, d6, [sp, #-144]
- // sub sp, sp, 144
+ // stp fp, lr, [sp, #-16]!
+ // mov fp, sp
+ // stp x1, x0, [sp, #-16]!
+ // stp x3, x2, [sp, #-16]!
+ // stp x5, x4, [sp, #-16]!
+ // stp x7, x6, [sp, #-16]!
+ // stp d1, d0, [sp, #-16]!
+ // stp d3, d2, [sp, #-16]!
+ // stp d5, d4, [sp, #-16]!
+ // stp d7, d6, [sp, #-16]!
// bl _resolver
// adrp x16, lazy_pointer at GOTPAGE
// ldr x16, [x16, lazy_pointer at GOTPAGEOFF]
// str x0, [x16]
// mov x16, x0
- // add sp, sp, 144
- // ldp d7, d6, [sp, #-144]
- // ldp d5, d4, [sp, #-128]
- // ldp d3, d2, [sp, #-112]
- // ldp d1, d0, [sp, #-96]
- // ldp x7, x6, [sp, #-80]
- // ldp x5, x4, [sp, #-64]
- // ldp x3, x2, [sp, #-48]
- // ldp x1, x0, [sp, #-32]
- // ldp fp, lr, [sp, #-16]
+ // ldp d7, d6, [sp], #16
+ // ldp d5, d4, [sp], #16
+ // ldp d3, d2, [sp], #16
+ // ldp d1, d0, [sp], #16
+ // ldp x7, x6, [sp], #16
+ // ldp x5, x4, [sp], #16
+ // ldp x3, x2, [sp], #16
+ // ldp x1, x0, [sp], #16
+ // ldp fp, lr, [sp], #16
// br x16
EmitLinkage(StubHelper);
@@ -1998,43 +1951,39 @@ void AArch64AsmPrinter::emitManualSymbolResolver(Module &M,
OutStreamer->emitLabel(StubHelper);
emitVisibility(StubHelper, GI.getVisibility());
- OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXi)
+ OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre)
+ .addReg(AArch64::SP)
.addReg(AArch64::FP)
.addReg(AArch64::LR)
.addReg(AArch64::SP)
.addImm(-2),
*STI);
- OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBXri)
+ OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri)
.addReg(AArch64::FP)
.addReg(AArch64::SP)
- .addImm(16)
+ .addImm(0)
.addImm(0),
*STI);
- for (int I = 0; I != 8; I += 2)
- OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXi)
- .addReg(AArch64::X1 + I)
- .addReg(AArch64::X0 + I)
+ for (int I = 0; I != 4; ++I)
+ OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre)
.addReg(AArch64::SP)
- .addImm(-4 - I),
+ .addReg(AArch64::X1 + 2 * I)
+ .addReg(AArch64::X0 + 2 * I)
+ .addReg(AArch64::SP)
+ .addImm(-2),
*STI);
- for (int I = 0; I != 8; I += 2)
- OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPDi)
- .addReg(AArch64::D1 + I)
- .addReg(AArch64::D0 + I)
+ for (int I = 0; I != 4; ++I)
+ OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPDpre)
+ .addReg(AArch64::SP)
+ .addReg(AArch64::D1 + 2 * I)
+ .addReg(AArch64::D0 + 2 * I)
.addReg(AArch64::SP)
- .addImm(-12 - I),
+ .addImm(-2),
*STI);
- OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBXri)
- .addReg(AArch64::SP)
- .addReg(AArch64::SP)
- .addImm(144)
- .addImm(0),
- *STI);
-
OutStreamer->emitInstruction(
MCInstBuilder(AArch64::BL)
.addOperand(MCOperand::createExpr(lowerConstant(GI.getResolver()))),
@@ -2081,34 +2030,30 @@ void AArch64AsmPrinter::emitManualSymbolResolver(Module &M,
.addImm(0),
*STI);
- OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri)
- .addReg(AArch64::SP)
- .addReg(AArch64::SP)
- .addImm(144)
- .addImm(0),
- *STI);
-
- for (int I = 6; I != -2; I -= 2)
- OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPDi)
- .addReg(AArch64::D1 + I)
- .addReg(AArch64::D0 + I)
+ for (int I = 3; I != -1; --I)
+ OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPDpost)
+ .addReg(AArch64::SP)
+ .addReg(AArch64::D1 + 2 * I)
+ .addReg(AArch64::D0 + 2 * I)
.addReg(AArch64::SP)
- .addImm(-12 - I),
+ .addImm(2),
*STI);
- for (int I = 6; I != -2; I -= 2)
- OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPXi)
- .addReg(AArch64::X1 + I)
- .addReg(AArch64::X0 + I)
+ for (int I = 3; I != -1; --I)
+ OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPXpost)
.addReg(AArch64::SP)
- .addImm(-4 - I),
+ .addReg(AArch64::X1 + 2 * I)
+ .addReg(AArch64::X0 + 2 * I)
+ .addReg(AArch64::SP)
+ .addImm(2),
*STI);
- OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPXi)
+ OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPXpost)
+ .addReg(AArch64::SP)
.addReg(AArch64::FP)
.addReg(AArch64::LR)
.addReg(AArch64::SP)
- .addImm(-2),
+ .addImm(2),
*STI);
OutStreamer->emitInstruction(MCInstBuilder(TM.getTargetTriple().isArm64e()
@@ -2118,27 +2063,6 @@ void AArch64AsmPrinter::emitManualSymbolResolver(Module &M,
*STI);
}
-void AArch64AsmPrinter::emitGlobalIFunc(Module &M, const GlobalIFunc &GI) {
- if (!TM.getTargetTriple().isOSBinFormatMachO())
- return AsmPrinter::emitGlobalIFunc(M, GI);
-
- switch (PreferredIFuncLowering) {
- case IFuncLowering::SymbolResolverAlways:
- return emitLinkerSymbolResolver(M, GI);
- case IFuncLowering::SymbolResolverNever:
- return emitManualSymbolResolver(M, GI);
- case IFuncLowering::SymbolResolverIfSupported:
- if (GI.hasExternalLinkage() || !MAI->getWeakRefDirective())
- return emitLinkerSymbolResolver(M, GI);
- else if (GI.hasWeakLinkage() || GI.hasLinkOnceLinkage())
- // NOTE: non-global .symbol_resolvers are not yet supported by Darwin
- // linkers
- return emitManualSymbolResolver(M, GI);
- else
- assert(GI.hasLocalLinkage() && "Invalid ifunc linkage");
- }
-}
-
// Force static initialization.
extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAArch64AsmPrinter() {
RegisterAsmPrinter<AArch64AsmPrinter> X(getTheAArch64leTarget());
diff --git a/llvm/test/CodeGen/AArch64/ifunc-asm.ll b/llvm/test/CodeGen/AArch64/ifunc-asm.ll
index 7eff692da83ff90..18c57b577333e9b 100644
--- a/llvm/test/CodeGen/AArch64/ifunc-asm.ll
+++ b/llvm/test/CodeGen/AArch64/ifunc-asm.ll
@@ -1,86 +1,72 @@
; RUN: llc -mtriple=arm64-unknown-linux-gnu %s -filetype=asm -o - | FileCheck %s --check-prefixes=ELF
-; RUN: llc -mtriple=arm64-apple-darwin %s -filetype=asm -o - -arm64-darwin-ifunc-symbol_resolver=always | FileCheck %s --check-prefixes=MACHO,MACHO-LINKER
-; RUN: llc -mtriple=arm64-apple-darwin %s -filetype=asm -o - -arm64-darwin-ifunc-symbol_resolver=if_supported | FileCheck %s --check-prefixes=MACHO,MACHO-DEFAULT
-; RUN: llc -mtriple=arm64-apple-darwin %s -filetype=asm -o - -arm64-darwin-ifunc-symbol_resolver=never | FileCheck %s --check-prefixes=MACHO,MACHO-MANUAL
-; RUN: llc -mtriple=arm64-apple-darwin %s -filetype=asm -o - | FileCheck %s --check-prefixes=MACHO,MACHO-MANUAL
-; RUN: llc -mtriple=arm64-apple-darwin %s -filetype=asm -o - -global-isel | FileCheck %s --check-prefixes=MACHO,MACHO-MANUAL
+; RUN: llc -mtriple=arm64-apple-darwin %s -filetype=asm -o - | FileCheck %s --check-prefix=MACHO
+; RUN: llc -mtriple=arm64-apple-darwin %s -filetype=asm -o - -global-isel | FileCheck %s --check-prefix=MACHO
define internal ptr @the_resolver() {
entry:
ret ptr null
}
-; ELF: .type the_resolver, at function
-; ELF-NEXT: the_resolver:
+; ELF: .type the_resolver, at function
+; ELF-NEXT: the_resolver:
-; MACHO: .p2align 2
-; MACHO-NEXT: _the_resolver
+; MACHO: .p2align 2
+; MACHO-NEXT: _the_resolver:
@global_ifunc = ifunc i32 (i32), ptr @the_resolver
-; ELF: .globl global_ifunc
-; ELF-NEXT: .type global_ifunc, at gnu_indirect_function
-; ELF-NEXT: .set global_ifunc, the_resolver
+; ELF: .globl global_ifunc
+; ELF-NEXT: .type global_ifunc, at gnu_indirect_function
+; ELF-NEXT: .set global_ifunc, the_resolver
-; MACHO-LINKER: .globl _global_ifunc
-; MACHO-LINKER-NEXT: .p2align 2
-; MACHO-LINKER-NEXT: _global_ifunc:
-; MACHO-LINKER-NEXT: .symbol_resolver _global_ifunc
-; MACHO-LINKER-NEXT: b _the_resolver
+; MACHO: .section __DATA,__data
+; MACHO-NEXT: .globl _global_ifunc.lazy_pointer
+; MACHO-NEXT: .p2align 3, 0x0
+; MACHO-NEXT: _global_ifunc.lazy_pointer:
+; MACHO-NEXT: .quad _global_ifunc.stub_helper
-; MACHO-DEFAULT: .globl _global_ifunc
-; MACHO-DEFAULT-NEXT: .p2align 2
-; MACHO-DEFAULT-NEXT: _global_ifunc:
-; MACHO-DEFAULT-NEXT: .symbol_resolver _global_ifunc
-; MACHO-DEFAULT-NEXT: b _the_resolver
-
-; MACHO-MANUAL: .section __DATA,__data
-; MACHO-MANUAL-NEXT: .globl _global_ifunc.lazy_pointer
-; MACHO-MANUAL-NEXT: .p2align 3, 0x0
-; MACHO-MANUAL-NEXT: _global_ifunc.lazy_pointer:
-; MACHO-MANUAL-NEXT: .quad _global_ifunc.stub_helper
-
-; MACHO-MANUAL: .section __TEXT,__text,regular,pure_instructions
-; MACHO-MANUAL-NEXT: .globl _global_ifunc
-; MACHO-MANUAL-NEXT: .p2align 2
-; MACHO-MANUAL-NEXT: _global_ifunc:
-; MACHO-MANUAL-NEXT: adrp x16, _global_ifunc.lazy_pointer at GOTPAGE
-; MACHO-MANUAL-NEXT: ldr x16, [x16, _global_ifunc.lazy_pointer at GOTPAGEOFF]
-; MACHO-MANUAL-NEXT: ldr x16, [x16]
-; MACHO-MANUAL-NEXT: br x16
-; MACHO-MANUAL-NEXT: .globl _global_ifunc.stub_helper
-; MACHO-MANUAL-NEXT: .p2align 2
-; MACHO-MANUAL-NEXT: _global_ifunc.stub_helper:
-; MACHO-MANUAL-NEXT: stp x29, x30, [sp, #-16]
-; MACHO-MANUAL-NEXT: sub x29, sp, #16
-; MACHO-MANUAL-NEXT: stp x1, x0, [sp, #-32]
-; MACHO-MANUAL-NEXT: stp x3, x2, [sp, #-48]
-; MACHO-MANUAL-NEXT: stp x5, x4, [sp, #-64]
-; MACHO-MANUAL-NEXT: stp x7, x6, [sp, #-80]
-; MACHO-MANUAL-NEXT: stp d1, d0, [sp, #-96]
-; MACHO-MANUAL-NEXT: stp d3, d2, [sp, #-112]
-; MACHO-MANUAL-NEXT: stp d5, d4, [sp, #-128]
-; MACHO-MANUAL-NEXT: stp d7, d6, [sp, #-144]
-; MACHO-MANUAL-NEXT: sub sp, sp, #144
-; MACHO-MANUAL-NEXT: bl _the_resolver
-; MACHO-MANUAL-NEXT: adrp x16, _global_ifunc.lazy_pointer at GOTPAGE
-; MACHO-MANUAL-NEXT: ldr x16, [x16, _global_ifunc.lazy_pointer at GOTPAGEOFF]
-; MACHO-MANUAL-NEXT: str x0, [x16]
-; MACHO-MANUAL-NEXT: add x16, x0, #0
-; MACHO-MANUAL-NEXT: add sp, sp, #144
-; MACHO-MANUAL-NEXT: ldp d7, d6, [sp, #-144]
-; MACHO-MANUAL-NEXT: ldp d5, d4, [sp, #-128]
-; MACHO-MANUAL-NEXT: ldp d3, d2, [sp, #-112]
-; MACHO-MANUAL-NEXT: ldp d1, d0, [sp, #-96]
-; MACHO-MANUAL-NEXT: ldp x7, x6, [sp, #-80]
-; MACHO-MANUAL-NEXT: ldp x5, x4, [sp, #-64]
-; MACHO-MANUAL-NEXT: ldp x3, x2, [sp, #-48]
-; MACHO-MANUAL-NEXT: ldp x1, x0, [sp, #-32]
-; MACHO-MANUAL-NEXT: ldp x29, x30, [sp, #-16]
-; MACHO-MANUAL-NEXT: br x16
+; MACHO: .section __TEXT,__text,regular,pure_instructions
+; MACHO-NEXT: .globl _global_ifunc
+; MACHO-NEXT: .p2align 2
+; MACHO-NEXT: _global_ifunc:
+; MACHO-NEXT: adrp x16, _global_ifunc.lazy_pointer at GOTPAGE
+; MACHO-NEXT: ldr x16, [x16, _global_ifunc.lazy_pointer at GOTPAGEOFF]
+; MACHO-NEXT: ldr x16, [x16]
+; MACHO-NEXT: br x16
+; MACHO-NEXT: .globl _global_ifunc.stub_helper
+; MACHO-NEXT: .p2align 2
+; MACHO-NEXT: _global_ifunc.stub_helper:
+; MACHO-NEXT: stp x29, x30, [sp, #-16]!
+; MACHO-NEXT: mov x29, sp
+; MACHO-NEXT: stp x1, x0, [sp, #-16]!
+; MACHO-NEXT: stp x3, x2, [sp, #-16]!
+; MACHO-NEXT: stp x5, x4, [sp, #-16]!
+; MACHO-NEXT: stp x7, x6, [sp, #-16]!
+; MACHO-NEXT: stp d1, d0, [sp, #-16]!
+; MACHO-NEXT: stp d3, d2, [sp, #-16]!
+; MACHO-NEXT: stp d5, d4, [sp, #-16]!
+; MACHO-NEXT: stp d7, d6, [sp, #-16]!
+; MACHO-NEXT: bl _the_resolver
+; MACHO-NEXT: adrp x16, _global_ifunc.lazy_pointer at GOTPAGE
+; MACHO-NEXT: ldr x16, [x16, _global_ifunc.lazy_pointer at GOTPAGEOFF]
+; MACHO-NEXT: str x0, [x16]
+; MACHO-NEXT: add x16, x0, #0
+; MACHO-NEXT: ldp d7, d6, [sp], #16
+; MACHO-NEXT: ldp d5, d4, [sp], #16
+; MACHO-NEXT: ldp d3, d2, [sp], #16
+; MACHO-NEXT: ldp d1, d0, [sp], #16
+; MACHO-NEXT: ldp x7, x6, [sp], #16
+; MACHO-NEXT: ldp x5, x4, [sp], #16
+; MACHO-NEXT: ldp x3, x2, [sp], #16
+; MACHO-NEXT: ldp x1, x0, [sp], #16
+; MACHO-NEXT: ldp x29, x30, [sp], #16
+; MACHO-NEXT: br x16
@weak_ifunc = weak ifunc i32 (i32), ptr @the_resolver
; ELF: .type weak_ifunc, at gnu_indirect_function
-; MACHO-LINKER: .symbol_resolver _weak_ifunc
-; MACHO-MANUAL: _weak_ifunc.stub_helper:
-; MACHO-DEFEAULT: _weak_ifunc.stub_helper:
\ No newline at end of file
+; MACHO: .weak_reference _weak_ifunc.lazy_pointer
+; MACHO: _weak_ifunc.lazy_pointer:
+; MACHO: .weak_reference _weak_ifunc
+; MACHO: _weak_ifunc:
+; MACHO: .weak_reference _weak_ifunc.stub_helper
+; MACHO: _weak_ifunc.stub_helper:
\ No newline at end of file
>From 72853bb876a884f2c9af03a858cc3b86d09b09bc Mon Sep 17 00:00:00 2001
From: Jon Roelofs <jonathan_roelofs at apple.com>
Date: Fri, 1 Dec 2023 13:32:40 -0800
Subject: [PATCH 9/9] fix formatting
Created using spr 1.3.4
---
llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index f19dcb5be172271..3aca166786128ee 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -1821,9 +1821,9 @@ void AArch64AsmPrinter::emitGlobalIFunc(Module &M, const GlobalIFunc &GI) {
// On Darwin platforms, emit a manually-constructed .symbol_resolver that
// implements the symbol resolution duties of the IFunc.
//
- // Normally, this would be handled by linker magic, but unfortunately there are
- // a few limitations in ld64 and ld-prime's implementation of .symbol_resolver
- // that mean we can't always use them:
+ // Normally, this would be handled by linker magic, but unfortunately there
+ // are a few limitations in ld64 and ld-prime's implementation of
+ // .symbol_resolver that mean we can't always use them:
//
// * resolvers cannot be the target of an alias
// * resolvers cannot have private linkage
More information about the llvm-commits
mailing list