[lld] f2efb57 - [LLD][COFF] Cover usage of LLD-as-a-library in tests
Alexandre Ganea via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 24 12:09:09 PDT 2020
Author: Alexandre Ganea
Date: 2020-09-24T15:07:50-04:00
New Revision: f2efb5742cc9f74ad73987760651e3d23894a416
URL: https://github.com/llvm/llvm-project/commit/f2efb5742cc9f74ad73987760651e3d23894a416
DIFF: https://github.com/llvm/llvm-project/commit/f2efb5742cc9f74ad73987760651e3d23894a416.diff
LOG: [LLD][COFF] Cover usage of LLD-as-a-library in tests
In lit tests, we run each LLD invocation twice (LLD_IN_TEST=2), without shutting down the process in-between. This ensures a full cleanup is properly done between runs.
Only active for the COFF driver for now. Other drivers still use LLD_IN_TEST=1 which executes just one iteration with full cleanup, like before.
When the environment variable LLD_IN_TEST is unset, a shortcut is taken, only one iteration is executed, no cleanup for faster exit, like before.
A public API, lld::safeLldMain(), is also available when using LLD as a library.
Differential Revision: https://reviews.llvm.org/D70378
Added:
lld/test/COFF/lit.local.cfg
Modified:
lld/COFF/Driver.cpp
lld/COFF/Writer.cpp
lld/COFF/Writer.h
lld/Common/ErrorHandler.cpp
lld/ELF/Driver.cpp
lld/MachO/Driver.cpp
lld/include/lld/Common/Driver.h
lld/include/lld/Common/ErrorHandler.h
lld/lib/Driver/DarwinLdDriver.cpp
lld/tools/lld/lld.cpp
lld/wasm/Driver.cpp
Removed:
################################################################################
diff --git a/lld/COFF/Driver.cpp b/lld/COFF/Driver.cpp
index 55e97d50c226..354ca10582d5 100644
--- a/lld/COFF/Driver.cpp
+++ b/lld/COFF/Driver.cpp
@@ -68,6 +68,17 @@ bool link(ArrayRef<const char *> args, bool canExitEarly, raw_ostream &stdoutOS,
lld::stdoutOS = &stdoutOS;
lld::stderrOS = &stderrOS;
+ errorHandler().cleanupCallback = []() {
+ freeArena();
+ ObjFile::instances.clear();
+ PDBInputFile::instances.clear();
+ ImportFile::instances.clear();
+ BitcodeFile::instances.clear();
+ memset(MergeChunk::instances, 0, sizeof(MergeChunk::instances));
+ TpiSource::clear();
+ OutputSection::clear();
+ };
+
errorHandler().logName = args::getFilenameWithoutExe(args[0]);
errorHandler().errorLimitExceededMsg =
"too many errors emitted, stopping now"
@@ -85,13 +96,6 @@ bool link(ArrayRef<const char *> args, bool canExitEarly, raw_ostream &stdoutOS,
if (canExitEarly)
exitLld(errorCount() ? 1 : 0);
- freeArena();
- ObjFile::instances.clear();
- ImportFile::instances.clear();
- BitcodeFile::instances.clear();
- memset(MergeChunk::instances, 0, sizeof(MergeChunk::instances));
- TpiSource::clear();
-
return !errorCount();
}
@@ -1204,6 +1208,7 @@ void LinkerDriver::link(ArrayRef<const char *> argsArr) {
v.push_back("lld-link (LLVM option parsing)");
for (auto *arg : args.filtered(OPT_mllvm))
v.push_back(arg->getValue());
+ cl::ResetAllOptionOccurrences();
cl::ParseCommandLineOptions(v.size(), v.data());
// Handle /errorlimit early, because error() depends on it.
diff --git a/lld/COFF/Writer.cpp b/lld/COFF/Writer.cpp
index 82dff2e5bb62..d1081b008ea4 100644
--- a/lld/COFF/Writer.cpp
+++ b/lld/COFF/Writer.cpp
@@ -88,6 +88,8 @@ OutputSection *Chunk::getOutputSection() const {
return osidx == 0 ? nullptr : outputSections[osidx - 1];
}
+void OutputSection::clear() { outputSections.clear(); }
+
namespace {
class DebugDirectoryChunk : public NonSectionChunk {
@@ -601,9 +603,6 @@ void Writer::finalizeAddresses() {
void Writer::run() {
ScopedTimer t1(codeLayoutTimer);
- // First, clear the output sections from previous runs
- outputSections.clear();
-
createImportTables();
createSections();
createMiscChunks();
diff --git a/lld/COFF/Writer.h b/lld/COFF/Writer.h
index 96389df2ac0a..2bb26da7d428 100644
--- a/lld/COFF/Writer.h
+++ b/lld/COFF/Writer.h
@@ -50,6 +50,9 @@ class OutputSection {
void writeHeaderTo(uint8_t *buf);
void addContributingPartialSection(PartialSection *sec);
+ // Clear the output sections static container.
+ static void clear();
+
// Returns the size of this section in an executable memory image.
// This may be smaller than the raw size (the raw size is multiple
// of disk sector size, so there may be padding at end), or may be
diff --git a/lld/Common/ErrorHandler.cpp b/lld/Common/ErrorHandler.cpp
index 94ff23173d69..3c3609e00743 100644
--- a/lld/Common/ErrorHandler.cpp
+++ b/lld/Common/ErrorHandler.cpp
@@ -13,15 +13,13 @@
#include "llvm/ADT/Twine.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/DiagnosticPrinter.h"
+#include "llvm/Support/CrashRecoveryContext.h"
#include "llvm/Support/ManagedStatic.h"
+#include "llvm/Support/Process.h"
#include "llvm/Support/raw_ostream.h"
#include <mutex>
#include <regex>
-#if !defined(_MSC_VER) && !defined(__MINGW32__)
-#include <unistd.h>
-#endif
-
using namespace llvm;
using namespace lld;
@@ -43,14 +41,23 @@ static StringRef getSeparator(const Twine &msg) {
raw_ostream *lld::stdoutOS;
raw_ostream *lld::stderrOS;
-raw_ostream &lld::outs() { return stdoutOS ? *stdoutOS : llvm::outs(); }
-raw_ostream &lld::errs() { return stderrOS ? *stderrOS : llvm::errs(); }
-
ErrorHandler &lld::errorHandler() {
static ErrorHandler handler;
return handler;
}
+raw_ostream &lld::outs() {
+ if (errorHandler().disableOutput)
+ return llvm::nulls();
+ return stdoutOS ? *stdoutOS : llvm::outs();
+}
+
+raw_ostream &lld::errs() {
+ if (errorHandler().disableOutput)
+ return llvm::nulls();
+ return stderrOS ? *stderrOS : llvm::errs();
+}
+
void lld::exitLld(int val) {
// Delete any temporary file, while keeping the memory mapping open.
if (errorHandler().outputBuffer)
@@ -60,14 +67,15 @@ void lld::exitLld(int val) {
// In an LTO build, allows us to get the output of -time-passes.
// Ensures that the thread pool for the parallel algorithms is stopped to
// avoid intermittent crashes on Windows when exiting.
- llvm_shutdown();
+ if (!CrashRecoveryContext::GetCurrent())
+ llvm_shutdown();
{
std::lock_guard<std::mutex> lock(mu);
lld::outs().flush();
lld::errs().flush();
}
- _exit(val);
+ llvm::sys::Process::Exit(val);
}
void lld::diagnosticHandler(const DiagnosticInfo &di) {
@@ -153,13 +161,15 @@ std::string ErrorHandler::getLocation(const Twine &msg) {
}
void ErrorHandler::log(const Twine &msg) {
- if (!verbose)
+ if (!verbose || disableOutput)
return;
std::lock_guard<std::mutex> lock(mu);
lld::errs() << logName << ": " << msg << "\n";
}
void ErrorHandler::message(const Twine &msg) {
+ if (disableOutput)
+ return;
std::lock_guard<std::mutex> lock(mu);
lld::outs() << msg << "\n";
lld::outs().flush();
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index 0f2e80b65987..9205a1660a6a 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -80,6 +80,27 @@ bool elf::link(ArrayRef<const char *> args, bool canExitEarly,
lld::stdoutOS = &stdoutOS;
lld::stderrOS = &stderrOS;
+ errorHandler().cleanupCallback = []() {
+ freeArena();
+
+ inputSections.clear();
+ outputSections.clear();
+ archiveFiles.clear();
+ binaryFiles.clear();
+ bitcodeFiles.clear();
+ lazyObjFiles.clear();
+ objectFiles.clear();
+ sharedFiles.clear();
+ backwardReferences.clear();
+
+ tar = nullptr;
+ memset(&in, 0, sizeof(in));
+
+ partitions = {Partition()};
+
+ SharedFile::vernauxNum = 0;
+ };
+
errorHandler().logName = args::getFilenameWithoutExe(args[0]);
errorHandler().errorLimitExceededMsg =
"too many errors emitted, stopping now (use "
@@ -87,28 +108,13 @@ bool elf::link(ArrayRef<const char *> args, bool canExitEarly,
errorHandler().exitEarly = canExitEarly;
stderrOS.enable_colors(stderrOS.has_colors());
- inputSections.clear();
- outputSections.clear();
- archiveFiles.clear();
- binaryFiles.clear();
- bitcodeFiles.clear();
- lazyObjFiles.clear();
- objectFiles.clear();
- sharedFiles.clear();
- backwardReferences.clear();
-
config = make<Configuration>();
driver = make<LinkerDriver>();
script = make<LinkerScript>();
symtab = make<SymbolTable>();
- tar = nullptr;
- memset(&in, 0, sizeof(in));
-
partitions = {Partition()};
- SharedFile::vernauxNum = 0;
-
config->progName = args[0];
driver->main(args);
@@ -119,7 +125,6 @@ bool elf::link(ArrayRef<const char *> args, bool canExitEarly,
if (canExitEarly)
exitLld(errorCount() ? 1 : 0);
- freeArena();
return !errorCount();
}
@@ -887,6 +892,7 @@ static void parseClangOption(StringRef opt, const Twine &msg) {
raw_string_ostream os(err);
const char *argv[] = {config->progName.data(), opt.data()};
+ cl::ResetAllOptionOccurrences();
if (cl::ParseCommandLineOptions(2, argv, "", &os))
return;
os.flush();
diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index 5e1a498f759f..02a7ccca7fbb 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -523,6 +523,8 @@ bool macho::link(llvm::ArrayRef<const char *> argsArr, bool canExitEarly,
stderrOS.enable_colors(stderrOS.has_colors());
// TODO: Set up error handler properly, e.g. the errorLimitExceededMsg
+ errorHandler().cleanupCallback = []() { freeArena(); };
+
MachOOptTable parser;
opt::InputArgList args = parser.parse(argsArr.slice(1));
@@ -676,6 +678,5 @@ bool macho::link(llvm::ArrayRef<const char *> argsArr, bool canExitEarly,
if (canExitEarly)
exitLld(errorCount() ? 1 : 0);
- freeArena();
return !errorCount();
}
diff --git a/lld/include/lld/Common/Driver.h b/lld/include/lld/Common/Driver.h
index 6db3d234eb56..7a0ad4b364cc 100644
--- a/lld/include/lld/Common/Driver.h
+++ b/lld/include/lld/Common/Driver.h
@@ -13,6 +13,18 @@
#include "llvm/Support/raw_ostream.h"
namespace lld {
+struct SafeReturn {
+ int ret;
+ bool canRunAgain;
+};
+
+// Generic entry point when using LLD as a library, safe for re-entry, supports
+// crash recovery. Returns a general completion code and a boolean telling
+// whether it can be called again. In some cases, a crash could corrupt memory
+// and re-entry would not be possible anymore.
+SafeReturn safeLldMain(int argc, const char **argv, llvm::raw_ostream &stdoutOS,
+ llvm::raw_ostream &stderrOS);
+
namespace coff {
bool link(llvm::ArrayRef<const char *> args, bool canExitEarly,
llvm::raw_ostream &stdoutOS, llvm::raw_ostream &stderrOS);
diff --git a/lld/include/lld/Common/ErrorHandler.h b/lld/include/lld/Common/ErrorHandler.h
index 8eed38af8c88..4ffc564e67e2 100644
--- a/lld/include/lld/Common/ErrorHandler.h
+++ b/lld/include/lld/Common/ErrorHandler.h
@@ -99,6 +99,8 @@ class ErrorHandler {
bool fatalWarnings = false;
bool verbose = false;
bool vsDiagnostics = false;
+ bool disableOutput = false;
+ std::function<void()> cleanupCallback;
void error(const Twine &msg);
LLVM_ATTRIBUTE_NORETURN void fatal(const Twine &msg);
@@ -106,6 +108,12 @@ class ErrorHandler {
void message(const Twine &msg);
void warn(const Twine &msg);
+ void reset() {
+ if (cleanupCallback)
+ cleanupCallback();
+ *this = ErrorHandler();
+ }
+
std::unique_ptr<llvm::FileOutputBuffer> outputBuffer;
private:
diff --git a/lld/lib/Driver/DarwinLdDriver.cpp b/lld/lib/Driver/DarwinLdDriver.cpp
index 1a57def4ebbe..49f4bdb77a14 100644
--- a/lld/lib/Driver/DarwinLdDriver.cpp
+++ b/lld/lib/Driver/DarwinLdDriver.cpp
@@ -307,6 +307,7 @@ static void parseLLVMOptions(const LinkingContext &ctx) {
for (unsigned i = 0; i != numArgs; ++i)
args[i + 1] = ctx.llvmOptions()[i];
args[numArgs + 1] = nullptr;
+ llvm::cl::ResetAllOptionOccurrences();
llvm::cl::ParseCommandLineOptions(numArgs + 1, args);
}
}
diff --git a/lld/test/COFF/lit.local.cfg b/lld/test/COFF/lit.local.cfg
new file mode 100644
index 000000000000..5d0c856d2a10
--- /dev/null
+++ b/lld/test/COFF/lit.local.cfg
@@ -0,0 +1 @@
+config.environment['LLD_IN_TEST'] = '2'
diff --git a/lld/tools/lld/lld.cpp b/lld/tools/lld/lld.cpp
index d4e2fbb0309a..48d5ae1a0ea0 100644
--- a/lld/tools/lld/lld.cpp
+++ b/lld/tools/lld/lld.cpp
@@ -26,6 +26,7 @@
//===----------------------------------------------------------------------===//
#include "lld/Common/Driver.h"
+#include "lld/Common/ErrorHandler.h"
#include "lld/Common/Memory.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallVector.h"
@@ -33,12 +34,19 @@
#include "llvm/ADT/Triple.h"
#include "llvm/ADT/Twine.h"
#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/CrashRecoveryContext.h"
#include "llvm/Support/Host.h"
#include "llvm/Support/InitLLVM.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/PluginLoader.h"
+#include "llvm/Support/Signals.h"
#include <cstdlib>
+#if !defined(_MSC_VER) && !defined(__MINGW32__)
+#include <signal.h> // for raise
+#include <unistd.h> // for _exit
+#endif
+
using namespace lld;
using namespace llvm;
using namespace llvm::sys;
@@ -133,36 +141,103 @@ static Flavor parseFlavor(std::vector<const char *> &v) {
return parseProgname(arg0);
}
-// If this function returns true, lld calls _exit() so that it quickly
-// exits without invoking destructors of globally allocated objects.
-//
-// We don't want to do that if we are running tests though, because
-// doing that breaks leak sanitizer. So, lit sets this environment variable,
-// and we use it to detect whether we are running tests or not.
-static bool canExitEarly() { return StringRef(getenv("LLD_IN_TEST")) != "1"; }
-
/// Universal linker main(). This linker emulates the gnu, darwin, or
/// windows linker based on the argv[0] or -flavor option.
-int main(int argc, const char **argv) {
- InitLLVM x(argc, argv);
-
+static int lldMain(int argc, const char **argv, llvm::raw_ostream &stdoutOS,
+ llvm::raw_ostream &stderrOS, bool exitEarly = true) {
std::vector<const char *> args(argv, argv + argc);
switch (parseFlavor(args)) {
case Gnu:
if (isPETarget(args))
- return !mingw::link(args, canExitEarly(), llvm::outs(), llvm::errs());
- return !elf::link(args, canExitEarly(), llvm::outs(), llvm::errs());
+ return !mingw::link(args, exitEarly, stdoutOS, stderrOS);
+ return !elf::link(args, exitEarly, stdoutOS, stderrOS);
case WinLink:
- return !coff::link(args, canExitEarly(), llvm::outs(), llvm::errs());
+ return !coff::link(args, exitEarly, stdoutOS, stderrOS);
case Darwin:
- return !mach_o::link(args, canExitEarly(), llvm::outs(), llvm::errs());
+ return !mach_o::link(args, exitEarly, stdoutOS, stderrOS);
case DarwinNew:
- return !macho::link(args, canExitEarly(), llvm::outs(), llvm::errs());
+ return !macho::link(args, exitEarly, stdoutOS, stderrOS);
case Wasm:
- return !wasm::link(args, canExitEarly(), llvm::outs(), llvm::errs());
+ return !lld::wasm::link(args, exitEarly, stdoutOS, stderrOS);
default:
die("lld is a generic driver.\n"
"Invoke ld.lld (Unix), ld64.lld (macOS), lld-link (Windows), wasm-ld"
" (WebAssembly) instead");
}
}
+
+// Similar to lldMain except that exceptions are caught.
+SafeReturn lld::safeLldMain(int argc, const char **argv,
+ llvm::raw_ostream &stdoutOS,
+ llvm::raw_ostream &stderrOS) {
+ int r = 0;
+ {
+ // The crash recovery is here only to be able to recover from arbitrary
+ // control flow when fatal() is called (through setjmp/longjmp or
+ // __try/__except).
+ llvm::CrashRecoveryContext crc;
+ if (!crc.RunSafely([&]() {
+ r = lldMain(argc, argv, stdoutOS, stderrOS, /*exitEarly=*/false);
+ }))
+ r = crc.RetCode;
+ }
+
+ // Cleanup memory and reset everything back in pristine condition. This path
+ // is only taken when LLD is in test, or when it is used as a library.
+ llvm::CrashRecoveryContext crc;
+ if (!crc.RunSafely([&]() { errorHandler().reset(); })) {
+ // The memory is corrupted beyond any possible recovery.
+ return {r, /*canRunAgain=*/false};
+ }
+ return {r, /*canRunAgain=*/true};
+}
+
+// When in lit tests, tells how many times the LLD tool should re-execute the
+// main loop with the same inputs. When not in test, returns a value of 0 which
+// signifies that LLD shall not release any memory after execution, to speed up
+// process destruction.
+static unsigned inTestVerbosity() {
+ unsigned v = 0;
+ StringRef(getenv("LLD_IN_TEST")).getAsInteger(10, v);
+ return v;
+}
+
+int main(int argc, const char **argv) {
+ InitLLVM x(argc, argv);
+
+ // Not running in lit tests, just take the shortest codepath with global
+ // exception handling and no memory cleanup on exit.
+ if (!inTestVerbosity())
+ return lldMain(argc, argv, llvm::outs(), llvm::errs());
+
+ Optional<int> mainRet;
+ CrashRecoveryContext::Enable();
+
+ for (unsigned i = inTestVerbosity(); i > 0; --i) {
+ // Disable stdout/stderr for all iterations but the last one.
+ if (i != 1)
+ errorHandler().disableOutput = true;
+
+ // Execute one iteration.
+ auto r = safeLldMain(argc, argv, llvm::outs(), llvm::errs());
+ if (!r.canRunAgain)
+ _exit(r.ret); // Exit now, can't re-execute again.
+
+ if (!mainRet) {
+ mainRet = r.ret;
+ } else if (r.ret != *mainRet) {
+ // Exit now, to fail the tests if the result is
diff erent between runs.
+ return r.ret;
+ }
+ }
+#if LLVM_ON_UNIX
+ // Re-throw the signal so it can be caught by WIFSIGNALED in
+ // llvm/lib/Support/Unix/Program.inc. This is required to correctly handle
+ // usages of `not --crash`.
+ if (*mainRet > 128) {
+ llvm::sys::unregisterHandlers();
+ raise(*mainRet - 128);
+ }
+#endif
+ return *mainRet;
+}
diff --git a/lld/wasm/Driver.cpp b/lld/wasm/Driver.cpp
index 09318421574c..9b5f6690ebf0 100644
--- a/lld/wasm/Driver.cpp
+++ b/lld/wasm/Driver.cpp
@@ -85,6 +85,8 @@ bool link(ArrayRef<const char *> args, bool canExitEarly, raw_ostream &stdoutOS,
lld::stdoutOS = &stdoutOS;
lld::stderrOS = &stderrOS;
+ errorHandler().cleanupCallback = []() { freeArena(); };
+
errorHandler().logName = args::getFilenameWithoutExe(args[0]);
errorHandler().errorLimitExceededMsg =
"too many errors emitted, stopping now (use "
@@ -103,7 +105,6 @@ bool link(ArrayRef<const char *> args, bool canExitEarly, raw_ostream &stdoutOS,
if (canExitEarly)
exitLld(errorCount() ? 1 : 0);
- freeArena();
return !errorCount();
}
@@ -776,6 +777,7 @@ void LinkerDriver::link(ArrayRef<const char *> argsArr) {
v.push_back("wasm-ld (LLVM option parsing)");
for (auto *arg : args.filtered(OPT_mllvm))
v.push_back(arg->getValue());
+ cl::ResetAllOptionOccurrences();
cl::ParseCommandLineOptions(v.size(), v.data());
errorHandler().errorLimit = args::getInteger(args, OPT_error_limit, 20);
More information about the llvm-commits
mailing list