[lld] [lld][WebAssemlby] Implement --thinlto-object-suffix-replace/--thinlto-prefix-replace (PR #114625)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Nov 1 16:36:38 PDT 2024
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-lld-wasm
Author: Sam Clegg (sbc100)
<details>
<summary>Changes</summary>
Fixes: #<!-- -->79604
---
Full diff: https://github.com/llvm/llvm-project/pull/114625.diff
9 Files Affected:
- (added) lld/test/wasm/lto/thinlto-emit-index.ll (+108)
- (added) lld/test/wasm/lto/thinlto-object-suffix-replace.ll (+47)
- (added) lld/test/wasm/lto/thinlto-prefix-replace.ll (+22)
- (modified) lld/wasm/Config.h (+4)
- (modified) lld/wasm/Driver.cpp (+46-1)
- (modified) lld/wasm/InputFiles.cpp (+9)
- (modified) lld/wasm/InputFiles.h (+2)
- (modified) lld/wasm/LTO.cpp (+11-2)
- (modified) lld/wasm/Options.td (+2)
``````````diff
diff --git a/lld/test/wasm/lto/thinlto-emit-index.ll b/lld/test/wasm/lto/thinlto-emit-index.ll
new file mode 100644
index 00000000000000..1697d30710c468
--- /dev/null
+++ b/lld/test/wasm/lto/thinlto-emit-index.ll
@@ -0,0 +1,108 @@
+;; Mostly copied/updated from thinlto-index-only.ll
+;; First ensure that the ThinLTO handling in lld handles
+;; bitcode without summary sections gracefully and generates index file.
+; RUN: rm -rf %t && mkdir %t && cd %t
+; RUN: mkdir d
+; RUN: llvm-as %s -o 1.o
+; RUN: llvm-as %p/Inputs/thinlto.ll -o d/2.o
+; RUN: wasm-ld --thinlto-emit-index-files -shared 1.o d/2.o -o 3
+; RUN: ls d/2.o.thinlto.bc
+; RUN: ls 3
+; RUN: wasm-ld -shared 1.o d/2.o -o 3
+; RUN: llvm-nm 3 | FileCheck %s --check-prefix=NM
+
+;; Basic ThinLTO tests.
+; RUN: opt -module-summary %s -o 1.o
+; RUN: opt -module-summary %p/Inputs/thinlto.ll -o d/2.o
+; RUN: opt -module-summary %p/Inputs/thinlto_empty.ll -o 3.o
+; RUN: cp 3.o 4.o
+
+;; Ensure lld generates an index and also a binary if requested.
+; RUN: wasm-ld --thinlto-emit-index-files -shared 1.o --start-lib d/2.o 3.o --end-lib 4.o -o 4
+; RUN: ls 4
+; RUN: llvm-bcanalyzer -dump 1.o.thinlto.bc | FileCheck %s --check-prefix=BACKEND1
+; RUN: llvm-bcanalyzer -dump d/2.o.thinlto.bc | FileCheck %s --check-prefix=BACKEND2
+; RUN: llvm-dis < 3.o.thinlto.bc | FileCheck %s --check-prefix=BACKEND3
+; RUN: llvm-dis < 4.o.thinlto.bc | FileCheck %s --check-prefix=BACKEND4
+
+; IMPORTS1: d/2.o
+
+;; Ensure lld generates an index and not a binary if both emit-index and index-only are present.
+; RUN: wasm-ld --thinlto-emit-index-files --thinlto-index-only -shared 1.o d/2.o -o 5
+; RUN: not ls 5
+
+;; Test that LLD generates an empty index even for lazy object file that is not added to link.
+;; Test that LLD also generates empty imports file with the --thinlto-emit-imports-files option.
+; RUN: rm -f 1.o.thinlto.bc 1.o.imports
+; RUN: wasm-ld --thinlto-emit-index-files -shared d/2.o --start-lib 1.o --end-lib \
+; RUN: --thinlto-emit-imports-files -o 7
+; RUN: ls 7
+; RUN: ls 1.o.thinlto.bc
+; RUN: ls 1.o.imports
+
+;; Ensure LLD generates an empty index for each bitcode file even if all bitcode files are lazy.
+; RUN: rm -f 1.o.thinlto.bc
+; RUN: llvm-mc -filetype=obj -triple=wasm32-unknown-linux /dev/null -o dummy.o
+; RUN: wasm-ld --thinlto-emit-index-files -shared dummy.o --start-lib 1.o --end-lib -o 8
+; RUN: ls 8
+; RUN: ls 1.o.thinlto.bc
+
+;; Test that LLD errors out when run with suffix replacement, or prefix replacement
+; RUN: not wasm-ld --thinlto-emit-index-files -shared d/2.o --start-lib 1.o --end-lib \
+; RUN: --thinlto-prefix-replace="abc;xyz" 2>&1 | FileCheck %s --check-prefix=ERR1
+; ERR1: --thinlto-prefix-replace is not supported with --thinlto-emit-index-files
+
+; RUN: not wasm-ld --thinlto-emit-index-files -shared d/2.o --start-lib 1.o --end-lib \
+; RUN: --thinlto-object-suffix-replace="abc;xyz" 2>&1 | FileCheck %s --check-prefix=ERR2
+; ERR2: --thinlto-object-suffix-replace is not supported with --thinlto-emit-index-files
+
+;; But not when passed with index only as well
+; RUN: wasm-ld --thinlto-emit-index-files -shared d/2.o --start-lib 1.o --end-lib \
+; RUN: --thinlto-prefix-replace="abc;xyz" --thinlto-index-only
+
+; RUN: wasm-ld --thinlto-emit-index-files -shared d/2.o --start-lib 1.o --end-lib \
+; RUN: --thinlto-object-suffix-replace="abc;xyz" --thinlto-index-only
+
+; NM: T f
+
+;; The backend index for this module contains summaries from itself and
+;; Inputs/thinlto.ll, as it imports from the latter.
+; BACKEND1: <MODULE_STRTAB_BLOCK
+; BACKEND1-NEXT: <ENTRY {{.*}} record string = '1.o'
+; BACKEND1-NEXT: <ENTRY {{.*}} record string = 'd/2.o'
+; BACKEND1-NEXT: </MODULE_STRTAB_BLOCK
+; BACKEND1: <GLOBALVAL_SUMMARY_BLOCK
+; BACKEND1: <VERSION
+; BACKEND1: <FLAGS
+; BACKEND1: <VALUE_GUID {{.*}} op0={{1|2}} {{op1=3060885059 op2=1207956914|op1=3432075125 op2=3712786831}}
+; BACKEND1: <VALUE_GUID {{.*}} op0={{1|2}} {{op1=3060885059 op2=1207956914|op1=3432075125 op2=3712786831}}
+; BACKEND1: <COMBINED
+; BACKEND1: <COMBINED
+; BACKEND1: </GLOBALVAL_SUMMARY_BLOCK
+
+;; The backend index for Input/thinlto.ll contains summaries from itself only,
+;; as it does not import anything.
+; BACKEND2: <MODULE_STRTAB_BLOCK
+; BACKEND2-NEXT: <ENTRY {{.*}} record string = 'd/2.o'
+; BACKEND2-NEXT: </MODULE_STRTAB_BLOCK
+; BACKEND2-NEXT: <GLOBALVAL_SUMMARY_BLOCK
+; BACKEND2-NEXT: <VERSION
+; BACKEND2-NEXT: <FLAGS
+; BACKEND2-NEXT: <VALUE_GUID {{.*}} op0=1 op1=3060885059 op2=1207956914
+; BACKEND2-NEXT: <COMBINED
+; BACKEND2-NEXT: </GLOBALVAL_SUMMARY_BLOCK
+
+; BACKEND3: ^0 = flags:
+
+; BACKEND4: ^0 = module: (path: "4.o", hash: (0, 0, 0, 0, 0))
+
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
+target triple = "wasm32-unknown-unknown"
+
+declare void @g(...)
+
+define void @f() {
+entry:
+ call void (...) @g()
+ ret void
+}
diff --git a/lld/test/wasm/lto/thinlto-object-suffix-replace.ll b/lld/test/wasm/lto/thinlto-object-suffix-replace.ll
new file mode 100644
index 00000000000000..68566d2050c0dd
--- /dev/null
+++ b/lld/test/wasm/lto/thinlto-object-suffix-replace.ll
@@ -0,0 +1,47 @@
+; REQUIRES: x86
+;; Test to make sure the thinlto-object-suffix-replace option is handled
+;; correctly.
+; RUN: rm -rf %t && mkdir %t && cd %t
+
+;; Generate bitcode file with summary, as well as a minimized bitcode without
+; the debug metadata for the thin link.
+; RUN: opt --thinlto-bc %s -thin-link-bitcode-file=1.thinlink.bc -o 1.o
+
+;; First perform the thin link on the normal bitcode file, and save the
+;; resulting index.
+; RUN: wasm-ld --thinlto-index-only -shared 1.o -o 3
+; RUN: cp 1.o.thinlto.bc 1.o.thinlto.bc.orig
+
+;; Next perform the thin link on the minimized bitcode file, and compare dump
+;; of the resulting index to the above dump to ensure they are identical.
+; RUN: rm -f 1.o.thinlto.bc
+;; Make sure it isn't inadvertently using the regular bitcode file.
+; RUN: rm -f 1.o
+; RUN: wasm-ld --thinlto-index-only --thinlto-object-suffix-replace=".thinlink.bc;.o" \
+; RUN: -shared 1.thinlink.bc -o 3
+; RUN: cmp 1.o.thinlto.bc.orig 1.o.thinlto.bc
+
+;; Ensure lld generates error if object suffix replace option does not have 'old;new' format
+; RUN: rm -f 1.o.thinlto.bc
+; RUN: not wasm-ld --thinlto-index-only --thinlto-object-suffix-replace="abc:def" -shared 1.thinlink.bc \
+; RUN: -o 3 2>&1 | FileCheck %s --check-prefix=ERR1
+; ERR1: --thinlto-object-suffix-replace= expects 'old;new' format, but got abc:def
+
+;; If filename does not end with old suffix, no suffix change should occur,
+;; so ".thinlto.bc" will simply be appended to the input file name.
+; RUN: rm -f 1.thinlink.bc.thinlto.bc
+; RUN: wasm-ld --thinlto-index-only --thinlto-object-suffix-replace=".abc;.o" -shared 1.thinlink.bc -o /dev/null
+; RUN: ls 1.thinlink.bc.thinlto.bc
+
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
+target triple = "wasm32-unknown-unknown"
+
+define void @f() {
+entry:
+ ret void
+}
+
+!llvm.dbg.cu = !{}
+
+!1 = !{i32 2, !"Debug Info Version", i32 3}
+!llvm.module.flags = !{!1}
diff --git a/lld/test/wasm/lto/thinlto-prefix-replace.ll b/lld/test/wasm/lto/thinlto-prefix-replace.ll
new file mode 100644
index 00000000000000..23ab1235809150
--- /dev/null
+++ b/lld/test/wasm/lto/thinlto-prefix-replace.ll
@@ -0,0 +1,22 @@
+; Check that changing the output path via thinlto-prefix-replace works
+; RUN: mkdir -p %t/oldpath
+; RUN: opt -module-summary %s -o %t/oldpath/thinlto_prefix_replace.o
+
+; Ensure that there is no existing file at the new path, so we properly
+; test the creation of the new file there.
+; RUN: rm -f %t/newpath/thinlto_prefix_replace.o.thinlto.bc
+; RUN: wasm-ld --thinlto-index-only --thinlto-prefix-replace="%t/oldpath/;%t/newpath/" -shared %t/oldpath/thinlto_prefix_replace.o -o %t/thinlto_prefix_replace
+; RUN: ls %t/newpath/thinlto_prefix_replace.o.thinlto.bc
+
+; Ensure that lld generates error if prefix replace option does not have 'old;new' format.
+; RUN: rm -f %t/newpath/thinlto_prefix_replace.o.thinlto.bc
+; RUN: not wasm-ld --thinlto-index-only --thinlto-prefix-replace=abc:def -shared %t/oldpath/thinlto_prefix_replace.o -o /dev/null 2>&1 | FileCheck %s --check-prefix=ERR
+; ERR: --thinlto-prefix-replace= expects 'old;new' format, but got abc:def
+
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
+target triple = "wasm32-unknown-unknown"
+
+define void @f() {
+entry:
+ ret void
+}
diff --git a/lld/wasm/Config.h b/lld/wasm/Config.h
index 18966f630e3dc2..eb32ce80f4a3d9 100644
--- a/lld/wasm/Config.h
+++ b/lld/wasm/Config.h
@@ -110,6 +110,10 @@ struct Configuration {
llvm::StringRef thinLTOCacheDir;
llvm::StringRef thinLTOJobs;
llvm::StringRef thinLTOIndexOnlyArg;
+ std::pair<llvm::StringRef, llvm::StringRef> thinLTOObjectSuffixReplace;
+ llvm::StringRef thinLTOPrefixReplaceOld;
+ llvm::StringRef thinLTOPrefixReplaceNew;
+ llvm::StringRef thinLTOPrefixReplaceNativeObject;
llvm::StringRef whyExtract;
llvm::StringSet<> allowUndefinedSymbols;
diff --git a/lld/wasm/Driver.cpp b/lld/wasm/Driver.cpp
index 65d412aa3c9833..43e13c3a5ca22d 100644
--- a/lld/wasm/Driver.cpp
+++ b/lld/wasm/Driver.cpp
@@ -425,6 +425,33 @@ void LinkerDriver::createFiles(opt::InputArgList &args) {
error("no input files");
}
+static StringRef getAliasSpelling(opt::Arg *arg) {
+ if (const opt::Arg *alias = arg->getAlias())
+ return alias->getSpelling();
+ return arg->getSpelling();
+}
+
+static std::pair<StringRef, StringRef> getOldNewOptions(opt::InputArgList &args,
+ unsigned id) {
+ auto *arg = args.getLastArg(id);
+ if (!arg)
+ return {"", ""};
+
+ StringRef s = arg->getValue();
+ std::pair<StringRef, StringRef> ret = s.split(';');
+ if (ret.second.empty())
+ error(getAliasSpelling(arg) + " expects 'old;new' format, but got " + s);
+ return ret;
+}
+
+// Parse options of the form "old;new[;extra]".
+static std::tuple<StringRef, StringRef, StringRef>
+getOldNewOptionsExtra(opt::InputArgList &args, unsigned id) {
+ auto [oldDir, second] = getOldNewOptions(args, id);
+ auto [newDir, extraDir] = second.split(';');
+ return {oldDir, newDir, extraDir};
+}
+
static StringRef getEntry(opt::InputArgList &args) {
auto *arg = args.getLastArg(OPT_entry, OPT_no_entry);
if (!arg) {
@@ -577,6 +604,24 @@ static void readConfigs(opt::InputArgList &args) {
config->thinLTOIndexOnly = args.hasArg(OPT_thinlto_index_only) ||
args.hasArg(OPT_thinlto_index_only_eq);
config->thinLTOIndexOnlyArg = args.getLastArgValue(OPT_thinlto_index_only_eq);
+ config->thinLTOObjectSuffixReplace =
+ getOldNewOptions(args, OPT_thinlto_object_suffix_replace_eq);
+ std::tie(config->thinLTOPrefixReplaceOld, config->thinLTOPrefixReplaceNew,
+ config->thinLTOPrefixReplaceNativeObject) =
+ getOldNewOptionsExtra(args, OPT_thinlto_prefix_replace_eq);
+ if (config->thinLTOEmitIndexFiles && !config->thinLTOIndexOnly) {
+ if (args.hasArg(OPT_thinlto_object_suffix_replace_eq))
+ error("--thinlto-object-suffix-replace is not supported with "
+ "--thinlto-emit-index-files");
+ else if (args.hasArg(OPT_thinlto_prefix_replace_eq))
+ error("--thinlto-prefix-replace is not supported with "
+ "--thinlto-emit-index-files");
+ }
+ if (!config->thinLTOPrefixReplaceNativeObject.empty() &&
+ config->thinLTOIndexOnlyArg.empty()) {
+ error("--thinlto-prefix-replace=old_dir;new_dir;obj_dir must be used with "
+ "--thinlto-index-only=");
+ }
config->unresolvedSymbols = getUnresolvedSymbolPolicy(args);
config->whyExtract = args.getLastArgValue(OPT_why_extract);
errorHandler().verbose = args.hasArg(OPT_verbose);
@@ -721,7 +766,7 @@ static void checkOptions(opt::InputArgList &args) {
if (config->pie && config->shared)
error("-shared and -pie may not be used together");
- if (config->outputFile.empty())
+ if (config->outputFile.empty() && !config->thinLTOIndexOnly)
error("no output file specified");
if (config->importTable && config->exportTable)
diff --git a/lld/wasm/InputFiles.cpp b/lld/wasm/InputFiles.cpp
index 420865e2aea8e3..fd06788457966a 100644
--- a/lld/wasm/InputFiles.cpp
+++ b/lld/wasm/InputFiles.cpp
@@ -46,6 +46,13 @@ std::string toString(const wasm::InputFile *file) {
namespace wasm {
+std::string replaceThinLTOSuffix(StringRef path) {
+ auto [suffix, repl] = config->thinLTOObjectSuffixReplace;
+ if (path.consume_back(suffix))
+ return (path + repl).str();
+ return std::string(path);
+}
+
void InputFile::checkArch(Triple::ArchType arch) const {
bool is64 = arch == Triple::wasm64;
if (is64 && !config->is64) {
@@ -837,6 +844,8 @@ BitcodeFile::BitcodeFile(MemoryBufferRef m, StringRef archiveName,
this->archiveName = std::string(archiveName);
std::string path = mb.getBufferIdentifier().str();
+ if (config->thinLTOIndexOnly)
+ path = replaceThinLTOSuffix(mb.getBufferIdentifier());
// ThinLTO assumes that all MemoryBufferRefs given to it have a unique
// name. If two archives define two members with the same name, this
diff --git a/lld/wasm/InputFiles.h b/lld/wasm/InputFiles.h
index c3a667523ee021..1b1de98d2d17a2 100644
--- a/lld/wasm/InputFiles.h
+++ b/lld/wasm/InputFiles.h
@@ -195,6 +195,8 @@ InputFile *createObjectFile(MemoryBufferRef mb, StringRef archiveName = "",
// Opens a given file.
std::optional<MemoryBufferRef> readFile(StringRef path);
+std::string replaceThinLTOSuffix(StringRef path);
+
} // namespace wasm
std::string toString(const wasm::InputFile *file);
diff --git a/lld/wasm/LTO.cpp b/lld/wasm/LTO.cpp
index 94f50eae317014..d9fff748bdb657 100644
--- a/lld/wasm/LTO.cpp
+++ b/lld/wasm/LTO.cpp
@@ -43,6 +43,11 @@ using namespace llvm;
using namespace lld::wasm;
using namespace lld;
+static std::string getThinLTOOutputFile(StringRef modulePath) {
+ return lto::getThinLTOOutputFile(modulePath, config->thinLTOPrefixReplaceOld,
+ config->thinLTOPrefixReplaceNew);
+}
+
static lto::Config createConfig() {
lto::Config c;
c.Options = initTargetOptionsFromCodeGenFlags();
@@ -84,7 +89,10 @@ BitcodeCompiler::BitcodeCompiler() {
auto onIndexWrite = [&](StringRef s) { thinIndices.erase(s); };
if (config->thinLTOIndexOnly) {
backend = lto::createWriteIndexesThinBackend(
- llvm::hardware_concurrency(config->thinLTOJobs), "", "", "",
+ llvm::hardware_concurrency(config->thinLTOJobs),
+ std::string(config->thinLTOPrefixReplaceOld),
+ std::string(config->thinLTOPrefixReplaceNew),
+ std::string(config->thinLTOPrefixReplaceNativeObject),
config->thinLTOEmitImportsFiles, indexFile.get(), onIndexWrite);
} else {
backend = lto::createInProcessThinBackend(
@@ -158,7 +166,8 @@ static void thinLTOCreateEmptyIndexFiles() {
continue;
if (linkedBitCodeFiles.contains(f->getName()))
continue;
- std::string path(f->obj->getName());
+ std::string path =
+ replaceThinLTOSuffix(getThinLTOOutputFile(f->obj->getName()));
std::unique_ptr<raw_fd_ostream> os = openFile(path + ".thinlto.bc");
if (!os)
continue;
diff --git a/lld/wasm/Options.td b/lld/wasm/Options.td
index 1a17452fbe8a7b..1316dc5c70d936 100644
--- a/lld/wasm/Options.td
+++ b/lld/wasm/Options.td
@@ -309,6 +309,8 @@ def thinlto_index_only: FF<"thinlto-index-only">;
def thinlto_index_only_eq: JJ<"thinlto-index-only=">;
def thinlto_jobs: JJ<"thinlto-jobs=">,
HelpText<"Number of ThinLTO jobs. Default to --threads=">;
+def thinlto_object_suffix_replace_eq: JJ<"thinlto-object-suffix-replace=">;
+def thinlto_prefix_replace_eq: JJ<"thinlto-prefix-replace=">;
def lto_debug_pass_manager: FF<"lto-debug-pass-manager">,
HelpText<"Debug new pass manager">;
``````````
</details>
https://github.com/llvm/llvm-project/pull/114625
More information about the llvm-commits
mailing list