[lld] [lld][WebAssembly] Implement various thinlto flags (PR #114327)
Sam Clegg via llvm-commits
llvm-commits at lists.llvm.org
Fri Nov 1 16:33:44 PDT 2024
https://github.com/sbc100 updated https://github.com/llvm/llvm-project/pull/114327
>From 21a6c2ece91141be3b540cacfe08a56032d5d00b Mon Sep 17 00:00:00 2001
From: Sam Clegg <sbc at chromium.org>
Date: Wed, 30 Oct 2024 14:02:10 -0700
Subject: [PATCH] [lld][WebAssembly] Implement various thinlto flags
The changes in this PR (both in the code and the tests) are largely
copied directly from the ELF linker.
Parial fix for #79604.
---
lld/test/wasm/lto/Inputs/thinlto_empty.ll | 2 +
lld/test/wasm/lto/obj-path.ll | 94 ++++++++++++++
lld/test/wasm/lto/parallel.ll | 10 +-
lld/test/wasm/lto/thinlto-index-only.ll | 145 ++++++++++++++++++++++
lld/test/wasm/lto/thinlto.ll | 67 +++++-----
lld/wasm/CMakeLists.txt | 1 +
lld/wasm/Config.h | 8 +-
lld/wasm/Driver.cpp | 12 ++
lld/wasm/LTO.cpp | 138 +++++++++++++++++---
lld/wasm/LTO.h | 9 +-
lld/wasm/Options.td | 5 +
lld/wasm/SymbolTable.cpp | 4 +-
12 files changed, 437 insertions(+), 58 deletions(-)
create mode 100644 lld/test/wasm/lto/Inputs/thinlto_empty.ll
create mode 100644 lld/test/wasm/lto/obj-path.ll
create mode 100644 lld/test/wasm/lto/thinlto-index-only.ll
diff --git a/lld/test/wasm/lto/Inputs/thinlto_empty.ll b/lld/test/wasm/lto/Inputs/thinlto_empty.ll
new file mode 100644
index 00000000000000..3dfdc5fbb67115
--- /dev/null
+++ b/lld/test/wasm/lto/Inputs/thinlto_empty.ll
@@ -0,0 +1,2 @@
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
+target triple = "wasm32-unknown-unknown"
diff --git a/lld/test/wasm/lto/obj-path.ll b/lld/test/wasm/lto/obj-path.ll
new file mode 100644
index 00000000000000..3a8dba15e9c1d8
--- /dev/null
+++ b/lld/test/wasm/lto/obj-path.ll
@@ -0,0 +1,94 @@
+;; Copied from testr/ELF/lto/obj-path.ll
+;; Test --lto-obj-path= for regular LTO.
+
+; RUN: rm -rf %t && split-file %s %t && cd %t
+; RUN: mkdir d
+; RUN: opt 1.ll -o 1.bc
+; RUN: opt 2.ll -o d/2.bc
+
+; RUN: rm -f objpath.o
+; RUN: wasm-ld --lto-obj-path=objpath.o -shared 1.bc d/2.bc -o 3
+; RUN: llvm-nm 3 | FileCheck %s --check-prefix=NM
+; RUN: llvm-objdump -d objpath.o | FileCheck %s
+; RUN: ls 3* objpath* | count 2
+
+; RUN: rm -f 3 objpath.o
+; RUN: wasm-ld --thinlto-index-only=3.txt --lto-obj-path=objpath.o -shared 1.bc d/2.bc -o 3
+; RUN: llvm-objdump -d objpath.o | FileCheck %s
+; RUN: not ls 3
+
+; NM: T f
+; NM: T g
+
+; CHECK: file format wasm
+; CHECK: <f>:
+; CHECK: <g>:
+
+;; Test --lto-obj-path= for ThinLTO.
+; RUN: opt -module-summary 1.ll -o 1.bc
+; RUN: opt -module-summary 2.ll -o d/2.bc
+
+; RUN: wasm-ld --lto-obj-path=objpath.o -shared 1.bc d/2.bc -o 3
+; RUN: llvm-nm 3 | FileCheck %s --check-prefix=NM3
+; RUN: llvm-objdump -d objpath.o1 | FileCheck %s --check-prefix=CHECK1
+; RUN: llvm-objdump -d objpath.o2 | FileCheck %s --check-prefix=CHECK2
+
+; NM3: T f
+; NM3-NEXT: T g
+
+; CHECK1: file format wasm
+; CHECK1-EMPTY:
+; CHECK1-NEXT: Disassembly of section CODE:
+; CHECK1: <f>:
+; CHECK1-EMPTY:
+; CHECK1-NEXT: end
+; CHECK1-NOT: {{.}}
+
+; CHECK2: file format wasm
+; CHECK2-EMPTY:
+; CHECK2-NEXT: Disassembly of section CODE:
+; CHECK2: <g>:
+; CHECK2-EMPTY:
+; CHECK2-NEXT: end
+; CHECK2-NOT: {{.}}
+
+;; With --thinlto-index-only, --lto-obj-path= creates just one file.
+; RUN: rm -f objpath.o objpath.o1 objpath.o2
+; RUN: wasm-ld --thinlto-index-only --lto-obj-path=objpath.o -shared 1.bc d/2.bc -o /dev/null
+; RUN: llvm-objdump -d objpath.o | FileCheck %s --check-prefix=EMPTY
+; RUN: not ls objpath.o1
+; RUN: not ls objpath.o2
+
+;; Ensure lld emits empty combined module if specific obj-path.
+; RUN: mkdir obj
+; RUN: wasm-ld --lto-obj-path=objpath.o -shared 1.bc d/2.bc -o obj/out --save-temps
+; RUN: ls obj/out.lto.o out.lto.1.o d/out.lto.2.o
+
+;; Ensure lld does not emit empty combined module by default.
+; RUN: rm -fr obj && mkdir obj
+; RUN: wasm-ld -shared 1.bc d/2.bc -o obj/out --save-temps
+; RUN: not test -e obj/out.lto.o
+
+; EMPTY: file format wasm
+; EMPTY-NOT: {{.}}
+
+;--- 1.ll
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
+target triple = "wasm32-unknown-unknown"
+
+declare void @g(...)
+
+define void @f() {
+entry:
+ call void (...) @g()
+ ret void
+}
+
+;--- 2.ll
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
+target triple = "wasm32-unknown-unknown"
+
+define void @g() {
+entry:
+ ret void
+}
diff --git a/lld/test/wasm/lto/parallel.ll b/lld/test/wasm/lto/parallel.ll
index a9e8b653d11a35..507fea6ee8558b 100644
--- a/lld/test/wasm/lto/parallel.ll
+++ b/lld/test/wasm/lto/parallel.ll
@@ -1,8 +1,8 @@
-; RUN: llvm-as -o %t.bc %s
-; RUN: rm -f %t.lto.o %t1.lto.o
-; RUN: wasm-ld --lto-partitions=2 -save-temps -o %t %t.bc -r
-; RUN: llvm-nm %t.lto.o | FileCheck --check-prefix=CHECK0 %s
-; RUN: llvm-nm %t1.lto.o | FileCheck --check-prefix=CHECK1 %s
+; RUN: rm -rf %t && mkdir %t && cd %t
+; RUN: llvm-as -o a.bc %s
+; RUN: wasm-ld --lto-partitions=2 -save-temps -o out a.bc -r
+; RUN: llvm-nm out.lto.o | FileCheck --check-prefix=CHECK0 %s
+; RUN: llvm-nm out.lto.1.o | FileCheck --check-prefix=CHECK1 %s
target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
target triple = "wasm32-unknown-unknown-wasm"
diff --git a/lld/test/wasm/lto/thinlto-index-only.ll b/lld/test/wasm/lto/thinlto-index-only.ll
new file mode 100644
index 00000000000000..40b8b00c63f5f2
--- /dev/null
+++ b/lld/test/wasm/lto/thinlto-index-only.ll
@@ -0,0 +1,145 @@
+; RUN: rm -rf %t && split-file %s %t && cd %t
+; RUN: mkdir d
+
+;; First ensure that the ThinLTO handling in lld handles
+;; bitcode without summary sections gracefully and generates index file.
+; RUN: llvm-as 1.ll -o 1.o
+; RUN: llvm-as %p/Inputs/thinlto.ll -o d/2.o
+; RUN: wasm-ld --thinlto-index-only -shared 1.o d/2.o -o 3
+; RUN: ls d/2.o.thinlto.bc
+; RUN: not test -e 3
+; RUN: wasm-ld -shared 1.o d/2.o -o 3
+; RUN: llvm-nm 3 | FileCheck %s --check-prefix=NM
+
+;; Basic ThinLTO tests.
+; RUN: llvm-as 0.ll -o 0.o
+; RUN: opt -module-summary 1.ll -o 1.o
+; RUN: opt -module-summary %p/Inputs/thinlto.ll -o d/2.o
+; RUN: opt -module-summary %p/Inputs/thinlto_empty.ll -o 3.o
+; RUN: cp 3.o 4.o
+
+;; Ensure lld doesn't generates index files when --thinlto-index-only is not enabled.
+; RUN: rm -f 1.o.thinlto.bc d/2.o.thinlto.bc
+; RUN: wasm-ld -shared 1.o d/2.o -o /dev/null
+; RUN: not ls 1.o.thinlto.bc
+; RUN: not ls d/2.o.thinlto.bc
+
+;; Ensure lld generates an index and not a binary if requested.
+; RUN: wasm-ld --thinlto-index-only -shared 1.o --start-lib d/2.o 3.o --end-lib 4.o -o 4
+; RUN: not test -e 4
+; RUN: llvm-bcanalyzer -dump 1.o.thinlto.bc | FileCheck %s --check-prefix=BACKEND1
+; RUN: llvm-bcanalyzer -dump d/2.o.thinlto.bc | FileCheck %s --check-prefix=BACKEND2
+; RUN: llvm-dis < 3.o.thinlto.bc | FileCheck %s --check-prefix=BACKEND3
+; RUN: llvm-dis < 4.o.thinlto.bc | FileCheck %s --check-prefix=BACKEND4
+
+; RUN: rm -f 1.o.thinlto.bc d/2.o.thinlto.bc 3.o.thinlto.bc 4.o.thinlto.bc
+; RUN: wasm-ld --thinlto-index-only=4.txt --thinlto-emit-imports-files -shared 1.o --start-lib d/2.o 3.o --end-lib 4.o -o 4
+; RUN: not test -e 4
+; RUN: FileCheck %s --check-prefix=RSP --implicit-check-not={{.}} < 4.txt
+; RUN: llvm-bcanalyzer -dump 1.o.thinlto.bc | FileCheck %s --check-prefix=BACKEND1
+; RUN: llvm-bcanalyzer -dump d/2.o.thinlto.bc | FileCheck %s --check-prefix=BACKEND2
+; RUN: llvm-dis < 3.o.thinlto.bc | FileCheck %s --check-prefix=BACKEND3
+; RUN: llvm-dis < 4.o.thinlto.bc | FileCheck %s --check-prefix=BACKEND4
+; RUN: FileCheck %s --check-prefix=IMPORTS1 --implicit-check-not={{.}} < 1.o.imports
+; RUN: count 0 < d/2.o.imports
+;; Test that LLD generates an empty index even for lazy object file that is not added to link.
+; RUN: count 0 < 3.o.imports
+; RUN: count 0 < 4.o.imports
+
+;; Test interaction with --save-temps.
+; RUN: rm -f 4.txt 1.o.thinlto.bc d/2.o.thinlto.bc 3.o.thinlto.bc 4.o.thinlto.bc
+; RUN: wasm-ld --thinlto-index-only=4.txt --thinlto-emit-imports-files --save-temps -shared 0.o 1.o --start-lib d/2.o 3.o --end-lib 4.o -o t
+; RUN: not test -e 4
+; RUN: FileCheck %s --check-prefix=RSP --implicit-check-not={{.}} < 4.txt
+; RUN: llvm-bcanalyzer -dump 1.o.thinlto.bc | FileCheck %s --check-prefix=BACKEND1
+; RUN: FileCheck %s --check-prefix=IMPORTS1 --implicit-check-not={{.}} < 1.o.imports
+; RUN: FileCheck %s --check-prefix=RESOLUTION < t.resolution.txt
+; RUN: llvm-dis < t.index.bc | FileCheck %s --check-prefix=INDEX-BC
+
+; RSP: 1.o
+; RSP-NEXT: d/2.o
+; RSP-NEXT: 4.o
+
+; IMPORTS1: d/2.o
+
+; RESOLUTION: 0.o
+; RESOLUTION-NEXT: -r=0.o,foo,px
+; RESOLUTION-NEXT: 1.o
+
+; INDEX-BC: ^0 = module: (path: "1.o", hash: (0, 0, 0, 0, 0))
+; INDEX-BC-NEXT: ^1 = module: (path: "4.o", hash: (0, 0, 0, 0, 0))
+; INDEX-BC-NEXT: ^2 = module: (path: "d/2.o", hash: (0, 0, 0, 0, 0))
+
+;; Ensure LLD generates an empty index for each bitcode file even if all bitcode files are lazy.
+; RUN: rm -f 1.o.thinlto.bc
+; RUN: llvm-mc -filetype=obj -triple=wasm32-unknown-unknown /dev/null -o dummy.o
+; RUN: wasm-ld --thinlto-index-only -shared dummy.o --start-lib 1.o --end-lib -o /dev/null
+; RUN: ls 1.o.thinlto.bc
+
+;; Ensure when the same bitcode object is given as both lazy and non-lazy,
+;; LLD does not generate an empty index for the lazy object.
+; RUN: rm -f d/2.o.thinlto.bc
+; RUN: wasm-ld --thinlto-index-only -shared 1.o d/2.o --start-lib d/2.o --end-lib -o /dev/null
+; RUN: llvm-dis < d/2.o.thinlto.bc | grep -q '\^0 = module:'
+; RUN: rm -f d/2.o.thinlto.bc
+; RUN: wasm-ld --thinlto-index-only -shared --start-lib d/2.o --end-lib d/2.o 1.o -o /dev/null
+; RUN: llvm-dis < d/2.o.thinlto.bc | grep -q '\^0 = module:'
+
+;; Ensure when the same lazy bitcode object is given multiple times,
+;; no empty index file is generated if one of the copies is linked.
+; RUN: rm -f d/2.o.thinlto.bc
+; RUN: wasm-ld --thinlto-index-only -shared 1.o --start-lib d/2.o --end-lib --start-lib d/2.o --end-lib -o /dev/null
+; RUN: llvm-dis < d/2.o.thinlto.bc | grep -q '\^0 = module:'
+
+; NM: T f
+
+;; The backend index for this module contains summaries from itself and
+;; Inputs/thinlto.ll, as it imports from the latter.
+; BACKEND1: <MODULE_STRTAB_BLOCK
+; BACKEND1-NEXT: <ENTRY {{.*}} record string = '1.o'
+; BACKEND1-NEXT: <ENTRY {{.*}} record string = 'd/2.o'
+; BACKEND1-NEXT: </MODULE_STRTAB_BLOCK
+; BACKEND1: <GLOBALVAL_SUMMARY_BLOCK
+; BACKEND1: <VERSION
+; BACKEND1: <FLAGS
+; BACKEND1: <VALUE_GUID {{.*}} op0={{1|2}} {{op1=3060885059 op2=1207956914|op1=3432075125 op2=3712786831}}
+; BACKEND1: <VALUE_GUID {{.*}} op0={{1|2}} {{op1=3060885059 op2=1207956914|op1=3432075125 op2=3712786831}}
+; BACKEND1: <COMBINED
+; BACKEND1: <COMBINED
+; BACKEND1: </GLOBALVAL_SUMMARY_BLOCK
+
+;; The backend index for Input/thinlto.ll contains summaries from itself only,
+;; as it does not import anything.
+; BACKEND2: <MODULE_STRTAB_BLOCK
+; BACKEND2-NEXT: <ENTRY {{.*}} record string = 'd/2.o'
+; BACKEND2-NEXT: </MODULE_STRTAB_BLOCK
+; BACKEND2-NEXT: <GLOBALVAL_SUMMARY_BLOCK
+; BACKEND2-NEXT: <VERSION
+; BACKEND2-NEXT: <FLAGS
+; BACKEND2-NEXT: <VALUE_GUID {{.*}} op0=1 op1=3060885059 op2=1207956914
+; BACKEND2-NEXT: <COMBINED
+; BACKEND2-NEXT: </GLOBALVAL_SUMMARY_BLOCK
+
+; BACKEND3: ^0 = flags:
+
+; BACKEND4: ^0 = module: (path: "4.o", hash: (0, 0, 0, 0, 0))
+
+;--- 0.ll
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
+target triple = "wasm32-unknown-unknown"
+
+define void @foo() {
+ ret void
+}
+
+;--- 1.ll
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20"
+target triple = "wasm32-unknown-unknown"
+
+declare void @g(...)
+
+define void @f() {
+entry:
+ call void (...) @g()
+ ret void
+}
diff --git a/lld/test/wasm/lto/thinlto.ll b/lld/test/wasm/lto/thinlto.ll
index c7dd73adcd4b76..df539cc38c83a0 100644
--- a/lld/test/wasm/lto/thinlto.ll
+++ b/lld/test/wasm/lto/thinlto.ll
@@ -1,53 +1,56 @@
; Basic ThinLTO tests.
-; RUN: opt -module-summary %s -o %t1.o
-; RUN: opt -module-summary %p/Inputs/thinlto.ll -o %t2.o
+; RUN: rm -rf %t && mkdir %t && cd %t
+; RUN: mkdir d e
+
+; RUN: opt -module-summary %s -o a.o
+; RUN: opt -module-summary %p/Inputs/thinlto.ll -o d/b.o
; First force single-threaded mode
-; RUN: rm -f %t31.lto.o %t32.lto.o
-; RUN: wasm-ld -r -save-temps --thinlto-jobs=1 %t1.o %t2.o -o %t3
-; RUN: llvm-nm %t31.lto.o | FileCheck %s --check-prefix=NM1
-; RUN: llvm-nm %t32.lto.o | FileCheck %s --check-prefix=NM2
+; RUN: rm -f out.lto.a.o d/out.lto.b.o
+; RUN: wasm-ld -r -save-temps --thinlto-jobs=1 a.o d/b.o -o e/out
+; RUN: llvm-nm out.lto.a.o | FileCheck %s --check-prefix=NM1
+; RUN: llvm-nm d/out.lto.b.o | FileCheck %s --check-prefix=NM2
; Next force multi-threaded mode
-; RUN: rm -f %t31.lto.o %t32.lto.o
-; RUN: wasm-ld -r -save-temps --thinlto-jobs=2 %t1.o %t2.o -o %t3
-; RUN: llvm-nm %t31.lto.o | FileCheck %s --check-prefix=NM1
-; RUN: llvm-nm %t32.lto.o | FileCheck %s --check-prefix=NM2
+; RUN: rm -f out.lto.a.o d/out.lto.b.o
+; RUN: wasm-ld -r -save-temps --thinlto-jobs=2 a.o d/b.o -o e/out
+; RUN: llvm-nm out.lto.a.o | FileCheck %s --check-prefix=NM1
+; RUN: llvm-nm d/out.lto.b.o | FileCheck %s --check-prefix=NM2
;; --thinlto-jobs= defaults to --threads=.
-; RUN: rm -f %t31.lto.o %t32.lto.o
-; RUN: wasm-ld -r -save-temps --threads=2 %t1.o %t2.o -o %t3
-; RUN: llvm-nm %t31.lto.o | FileCheck %s --check-prefix=NM1
-; RUN: llvm-nm %t32.lto.o | FileCheck %s --check-prefix=NM2
+; RUN: rm -f out.lto.a.o d/out.lto.b.o
+; RUN: wasm-ld -r -save-temps --threads=2 a.o d/b.o -o e/out
+; RUN: llvm-nm out.lto.a.o | FileCheck %s --check-prefix=NM1
+; RUN: llvm-nm d/out.lto.b.o | FileCheck %s --check-prefix=NM2
;; --thinlto-jobs= overrides --threads=.
-; RUN: rm -f %t31.lto.o %t32.lto.o
-; RUN: wasm-ld -r -save-temps --threads=1 --thinlto-jobs=2 %t1.o %t2.o -o %t3
-; RUN: llvm-nm %t31.lto.o | FileCheck %s --check-prefix=NM1
-; RUN: llvm-nm %t32.lto.o | FileCheck %s --check-prefix=NM2
+; RUN: rm -f out.lto.a.o d/out.lto.b.o
+; RUN: wasm-ld -r -save-temps --threads=1 --thinlto-jobs=2 a.o d/b.o -o e/out
+; RUN: llvm-nm out.lto.a.o | FileCheck %s --check-prefix=NM1
+; RUN: llvm-nm d/out.lto.b.o | FileCheck %s --check-prefix=NM2
; Test with all threads, on all cores, on all CPU sockets
-; RUN: rm -f %t31.lto.o %t32.lto.o
-; RUN: wasm-ld -r -save-temps --thinlto-jobs=all %t1.o %t2.o -o %t3
-; RUN: llvm-nm %t31.lto.o | FileCheck %s --check-prefix=NM1
-; RUN: llvm-nm %t32.lto.o | FileCheck %s --check-prefix=NM2
+; RUN: rm -f out.lto.a.o d/out.lto.b.o
+; RUN: wasm-ld -r -save-temps --thinlto-jobs=all a.o d/b.o -o e/out
+; RUN: llvm-nm out.lto.a.o | FileCheck %s --check-prefix=NM1
+; RUN: llvm-nm d/out.lto.b.o | FileCheck %s --check-prefix=NM2
; Test with many more threads than the system has
-; RUN: rm -f %t31.lto.o %t32.lto.o
-; RUN: wasm-ld -r -save-temps --thinlto-jobs=100 %t1.o %t2.o -o %t3
-; RUN: llvm-nm %t31.lto.o | FileCheck %s --check-prefix=NM1
-; RUN: llvm-nm %t32.lto.o | FileCheck %s --check-prefix=NM2
+; RUN: rm -f out.lto.a.o d/out.lto.b.o
+; RUN: wasm-ld -r -save-temps --thinlto-jobs=100 a.o d/b.o -o e/out
+; RUN: llvm-nm out.lto.a.o | FileCheck %s --check-prefix=NM1
+; RUN: llvm-nm d/out.lto.b.o | FileCheck %s --check-prefix=NM2
; Test with a bad value
-; RUN: rm -f %t31.lto.o %t32.lto.o
-; RUN: not wasm-ld -r -save-temps --thinlto-jobs=foo %t1.o %t2.o -o %t3 2>&1 | FileCheck %s --check-prefix=BAD-JOBS
+; RUN: rm -f out.lto.a.o d/out.lto.b.o
+; RUN: not wasm-ld -r -save-temps --thinlto-jobs=foo a.o d/b.o -o e/out 2>&1 | FileCheck %s --check-prefix=BAD-JOBS
; BAD-JOBS: error: --thinlto-jobs: invalid job count: foo
; Check without --thinlto-jobs (which currently defaults to heavyweight_hardware_concurrency, meanning one thread per hardware core -- not SMT)
-; RUN: rm -f %t31.lto.o %t32.lto.o
-; RUN: wasm-ld -r -save-temps %t1.o %t2.o -o %t3
-; RUN: llvm-nm %t31.lto.o | FileCheck %s --check-prefix=NM1
-; RUN: llvm-nm %t32.lto.o | FileCheck %s --check-prefix=NM2
+; RUN: rm -f out.lto.a.o d/out.lto.b.o
+; RUN: wasm-ld -r -save-temps a.o d/b.o -o e/out
+; RUN: llvm-nm out.lto.a.o | FileCheck %s --check-prefix=NM1
+; RUN: llvm-nm d/out.lto.b.o | FileCheck %s --check-prefix=NM2
; NM1: T f
; NM2: T g
diff --git a/lld/wasm/CMakeLists.txt b/lld/wasm/CMakeLists.txt
index ba89a6fee3504e..05b12d76df50c5 100644
--- a/lld/wasm/CMakeLists.txt
+++ b/lld/wasm/CMakeLists.txt
@@ -21,6 +21,7 @@ add_lld_library(lldWasm
LINK_COMPONENTS
${LLVM_TARGETS_TO_BUILD}
BinaryFormat
+ BitWriter
Core
Demangle
LTO
diff --git a/lld/wasm/Config.h b/lld/wasm/Config.h
index 05a547ff9278a1..18966f630e3dc2 100644
--- a/lld/wasm/Config.h
+++ b/lld/wasm/Config.h
@@ -79,6 +79,9 @@ struct Configuration {
// Because dyamanic linking under Wasm is still experimental we default to
// static linking
bool isStatic = true;
+ bool thinLTOEmitImportsFiles;
+ bool thinLTOEmitIndexFiles;
+ bool thinLTOIndexOnly;
bool trace;
uint64_t globalBase;
uint64_t initialHeap;
@@ -95,16 +98,18 @@ struct Configuration {
unsigned ltoo;
llvm::CodeGenOptLevel ltoCgo;
unsigned optimize;
- llvm::StringRef thinLTOJobs;
bool ltoDebugPassManager;
UnresolvedPolicy unresolvedSymbols;
BuildIdKind buildId = BuildIdKind::None;
llvm::StringRef entry;
+ llvm::StringRef ltoObjPath;
llvm::StringRef mapFile;
llvm::StringRef outputFile;
llvm::StringRef soName;
llvm::StringRef thinLTOCacheDir;
+ llvm::StringRef thinLTOJobs;
+ llvm::StringRef thinLTOIndexOnlyArg;
llvm::StringRef whyExtract;
llvm::StringSet<> allowUndefinedSymbols;
@@ -126,6 +131,7 @@ struct Ctx {
llvm::SmallVector<StubFile *, 0> stubFiles;
llvm::SmallVector<SharedFile *, 0> sharedFiles;
llvm::SmallVector<BitcodeFile *, 0> bitcodeFiles;
+ llvm::SmallVector<BitcodeFile *, 0> lazyBitcodeFiles;
llvm::SmallVector<InputFunction *, 0> syntheticFunctions;
llvm::SmallVector<InputGlobal *, 0> syntheticGlobals;
llvm::SmallVector<InputTable *, 0> syntheticTables;
diff --git a/lld/wasm/Driver.cpp b/lld/wasm/Driver.cpp
index 9a27fc90457f08..65d412aa3c9833 100644
--- a/lld/wasm/Driver.cpp
+++ b/lld/wasm/Driver.cpp
@@ -542,6 +542,7 @@ static void readConfigs(opt::InputArgList &args) {
else
error("invalid codegen optimization level for LTO: " + Twine(ltoCgo));
config->ltoPartitions = args::getInteger(args, OPT_lto_partitions, 1);
+ config->ltoObjPath = args.getLastArgValue(OPT_lto_obj_path_eq);
config->ltoDebugPassManager = args.hasArg(OPT_lto_debug_pass_manager);
config->mapFile = args.getLastArgValue(OPT_Map);
config->optimize = args::getInteger(args, OPT_O, 1);
@@ -569,6 +570,13 @@ static void readConfigs(opt::InputArgList &args) {
config->thinLTOCachePolicy = CHECK(
parseCachePruningPolicy(args.getLastArgValue(OPT_thinlto_cache_policy)),
"--thinlto-cache-policy: invalid cache policy");
+ config->thinLTOEmitImportsFiles = args.hasArg(OPT_thinlto_emit_imports_files);
+ config->thinLTOEmitIndexFiles = args.hasArg(OPT_thinlto_emit_index_files) ||
+ args.hasArg(OPT_thinlto_index_only) ||
+ args.hasArg(OPT_thinlto_index_only_eq);
+ config->thinLTOIndexOnly = args.hasArg(OPT_thinlto_index_only) ||
+ args.hasArg(OPT_thinlto_index_only_eq);
+ config->thinLTOIndexOnlyArg = args.getLastArgValue(OPT_thinlto_index_only_eq);
config->unresolvedSymbols = getUnresolvedSymbolPolicy(args);
config->whyExtract = args.getLastArgValue(OPT_why_extract);
errorHandler().verbose = args.hasArg(OPT_verbose);
@@ -1379,6 +1387,10 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
writeWhyExtract();
+ // Bail out if normal linked output is skipped due to LTO.
+ if (config->thinLTOIndexOnly)
+ return;
+
createOptionalSymbols();
// Resolve any variant symbols that were created due to signature
diff --git a/lld/wasm/LTO.cpp b/lld/wasm/LTO.cpp
index e523f0f6171535..94f50eae317014 100644
--- a/lld/wasm/LTO.cpp
+++ b/lld/wasm/LTO.cpp
@@ -11,13 +11,16 @@
#include "InputFiles.h"
#include "Symbols.h"
#include "lld/Common/Args.h"
+#include "lld/Common/CommonLinkerContext.h"
#include "lld/Common/ErrorHandler.h"
+#include "lld/Common/Filesystem.h"
#include "lld/Common/Strings.h"
#include "lld/Common/TargetOptionsCommandFlags.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/Twine.h"
+#include "llvm/Bitcode/BitcodeWriter.h"
#include "llvm/IR/DiagnosticPrinter.h"
#include "llvm/LTO/Config.h"
#include "llvm/LTO/LTO.h"
@@ -27,6 +30,7 @@
#include "llvm/Support/Error.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/Path.h"
#include "llvm/Support/raw_ostream.h"
#include <algorithm>
#include <cstddef>
@@ -36,9 +40,10 @@
#include <vector>
using namespace llvm;
+using namespace lld::wasm;
+using namespace lld;
-namespace lld::wasm {
-static std::unique_ptr<lto::LTO> createLTO() {
+static lto::Config createConfig() {
lto::Config c;
c.Options = initTargetOptionsFromCodeGenFlags();
@@ -52,6 +57,7 @@ static std::unique_ptr<lto::LTO> createLTO() {
c.MAttrs = getMAttrs();
c.CGOptLevel = config->ltoCgo;
c.DebugPassManager = config->ltoDebugPassManager;
+ c.AlwaysEmitRegularLTOObj = !config->ltoObjPath.empty();
if (config->relocatable)
c.RelocModel = std::nullopt;
@@ -63,13 +69,32 @@ static std::unique_ptr<lto::LTO> createLTO() {
if (config->saveTemps)
checkError(c.addSaveTemps(config->outputFile.str() + ".",
/*UseInputModulePath*/ true));
- lto::ThinBackend backend = lto::createInProcessThinBackend(
- llvm::heavyweight_hardware_concurrency(config->thinLTOJobs));
- return std::make_unique<lto::LTO>(std::move(c), backend,
- config->ltoPartitions);
+ return c;
}
-BitcodeCompiler::BitcodeCompiler() : ltoObj(createLTO()) {}
+namespace lld::wasm {
+
+BitcodeCompiler::BitcodeCompiler() {
+ // Initialize indexFile.
+ if (!config->thinLTOIndexOnlyArg.empty())
+ indexFile = openFile(config->thinLTOIndexOnlyArg);
+
+ // Initialize ltoObj.
+ lto::ThinBackend backend;
+ auto onIndexWrite = [&](StringRef s) { thinIndices.erase(s); };
+ if (config->thinLTOIndexOnly) {
+ backend = lto::createWriteIndexesThinBackend(
+ llvm::hardware_concurrency(config->thinLTOJobs), "", "", "",
+ config->thinLTOEmitImportsFiles, indexFile.get(), onIndexWrite);
+ } else {
+ backend = lto::createInProcessThinBackend(
+ llvm::heavyweight_hardware_concurrency(config->thinLTOJobs),
+ onIndexWrite, config->thinLTOEmitIndexFiles,
+ config->thinLTOEmitImportsFiles);
+ }
+ ltoObj = std::make_unique<lto::LTO>(createConfig(), backend,
+ config->ltoPartitions);
+}
BitcodeCompiler::~BitcodeCompiler() = default;
@@ -90,6 +115,10 @@ void BitcodeCompiler::add(BitcodeFile &f) {
ArrayRef<Symbol *> syms = f.getSymbols();
std::vector<lto::SymbolResolution> resols(syms.size());
+ if (config->thinLTOEmitIndexFiles) {
+ thinIndices.insert(obj.getName());
+ }
+
// Provide a resolution to the LTO API for each symbol.
for (const lto::InputFile::Symbol &objSym : obj.symbols()) {
Symbol *sym = syms[symNum];
@@ -116,6 +145,32 @@ void BitcodeCompiler::add(BitcodeFile &f) {
checkError(ltoObj->add(std::move(f.obj), resols));
}
+// If LazyObjFile has not been added to link, emit empty index files.
+// This is needed because this is what GNU gold plugin does and we have a
+// distributed build system that depends on that behavior.
+static void thinLTOCreateEmptyIndexFiles() {
+ DenseSet<StringRef> linkedBitCodeFiles;
+ for (BitcodeFile *f : ctx.bitcodeFiles)
+ linkedBitCodeFiles.insert(f->getName());
+
+ for (BitcodeFile *f : ctx.lazyBitcodeFiles) {
+ if (!f->lazy)
+ continue;
+ if (linkedBitCodeFiles.contains(f->getName()))
+ continue;
+ std::string path(f->obj->getName());
+ std::unique_ptr<raw_fd_ostream> os = openFile(path + ".thinlto.bc");
+ if (!os)
+ continue;
+
+ ModuleSummaryIndex m(/*HaveGVs*/ false);
+ m.setSkipModuleByDistributedBackend();
+ writeIndexToFile(m, *os);
+ if (config->thinLTOEmitImportsFiles)
+ openFile(path + ".imports");
+ }
+}
+
// Merge all the bitcode files we have seen, codegen the result
// and return the resulting objects.
std::vector<StringRef> BitcodeCompiler::compile() {
@@ -136,25 +191,78 @@ std::vector<StringRef> BitcodeCompiler::compile() {
checkError(ltoObj->run(
[&](size_t task, const Twine &moduleName) {
+ buf[task].first = moduleName.str();
return std::make_unique<CachedFileStream>(
- std::make_unique<raw_svector_ostream>(buf[task]));
+ std::make_unique<raw_svector_ostream>(buf[task].second));
},
cache));
+ // Emit empty index files for non-indexed files but not in single-module mode.
+ for (StringRef s : thinIndices) {
+ std::string path(s);
+ openFile(path + ".thinlto.bc");
+ if (config->thinLTOEmitImportsFiles)
+ openFile(path + ".imports");
+ }
+
+ if (config->thinLTOEmitIndexFiles)
+ thinLTOCreateEmptyIndexFiles();
+
+ if (config->thinLTOIndexOnly) {
+ if (!config->ltoObjPath.empty())
+ saveBuffer(buf[0].second, config->ltoObjPath);
+
+ // ThinLTO with index only option is required to generate only the index
+ // files. After that, we exit from linker and ThinLTO backend runs in a
+ // distributed environment.
+ if (indexFile)
+ indexFile->close();
+ return {};
+ }
+
if (!config->thinLTOCacheDir.empty())
pruneCache(config->thinLTOCacheDir, config->thinLTOCachePolicy, files);
std::vector<StringRef> ret;
for (unsigned i = 0; i != maxTasks; ++i) {
- if (buf[i].empty())
+ StringRef objBuf = buf[i].second;
+ StringRef bitcodeFilePath = buf[i].first;
+ if (objBuf.empty())
+ continue;
+ ret.emplace_back(objBuf.data(), objBuf.size());
+ if (!config->saveTemps)
continue;
- if (config->saveTemps) {
- if (i == 0)
- saveBuffer(buf[i], config->outputFile + ".lto.o");
- else
- saveBuffer(buf[i], config->outputFile + Twine(i) + ".lto.o");
+
+ // If the input bitcode file is path/to/x.o and -o specifies a.out, the
+ // corresponding native relocatable file path will look like:
+ // path/to/a.out.lto.x.o.
+ StringRef ltoObjName;
+ if (bitcodeFilePath == "ld-temp.o") {
+ ltoObjName =
+ saver().save(Twine(config->outputFile) + ".lto" +
+ (i == 0 ? Twine("") : Twine('.') + Twine(i)) + ".o");
+ } else {
+ StringRef directory = sys::path::parent_path(bitcodeFilePath);
+ // For an archive member, which has an identifier like "d/a.a(coll.o at
+ // 8)" (see BitcodeFile::BitcodeFile), use the filename; otherwise, use
+ // the stem (d/a.o => a).
+ StringRef baseName = bitcodeFilePath.ends_with(")")
+ ? sys::path::filename(bitcodeFilePath)
+ : sys::path::stem(bitcodeFilePath);
+ StringRef outputFileBaseName = sys::path::filename(config->outputFile);
+ SmallString<256> path;
+ sys::path::append(path, directory,
+ outputFileBaseName + ".lto." + baseName + ".o");
+ sys::path::remove_dots(path, true);
+ ltoObjName = saver().save(path.str());
}
- ret.emplace_back(buf[i].data(), buf[i].size());
+ saveBuffer(objBuf, ltoObjName);
+ }
+
+ if (!config->ltoObjPath.empty()) {
+ saveBuffer(buf[0].second, config->ltoObjPath);
+ for (unsigned i = 1; i != maxTasks; ++i)
+ saveBuffer(buf[i].second, config->ltoObjPath + Twine(i));
}
for (std::unique_ptr<MemoryBuffer> &file : files)
diff --git a/lld/wasm/LTO.h b/lld/wasm/LTO.h
index bb57c6651394b7..43c7672fb5639b 100644
--- a/lld/wasm/LTO.h
+++ b/lld/wasm/LTO.h
@@ -20,9 +20,11 @@
#ifndef LLD_WASM_LTO_H
#define LLD_WASM_LTO_H
+#include "Writer.h"
#include "lld/Common/LLVM.h"
+#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/SmallString.h"
-#include "Writer.h"
+#include "llvm/Support/raw_ostream.h"
#include <memory>
#include <vector>
@@ -47,8 +49,11 @@ class BitcodeCompiler {
private:
std::unique_ptr<llvm::lto::LTO> ltoObj;
- std::vector<SmallString<0>> buf;
+ // An array of (module name, native relocatable file content) pairs.
+ SmallVector<std::pair<std::string, SmallString<0>>, 0> buf;
std::vector<std::unique_ptr<MemoryBuffer>> files;
+ std::unique_ptr<llvm::raw_fd_ostream> indexFile;
+ llvm::DenseSet<StringRef> thinIndices;
};
} // namespace lld::wasm
diff --git a/lld/wasm/Options.td b/lld/wasm/Options.td
index cff29e709a1a09..1a17452fbe8a7b 100644
--- a/lld/wasm/Options.td
+++ b/lld/wasm/Options.td
@@ -297,11 +297,16 @@ def lto_CGO: JJ<"lto-CGO">, MetaVarName<"<cgopt-level>">,
HelpText<"Codegen optimization level for LTO">;
def lto_partitions: JJ<"lto-partitions=">,
HelpText<"Number of LTO codegen partitions">;
+def lto_obj_path_eq: JJ<"lto-obj-path=">;
def disable_verify: F<"disable-verify">;
def save_temps: F<"save-temps">, HelpText<"Save intermediate LTO compilation results">;
def thinlto_cache_dir: JJ<"thinlto-cache-dir=">,
HelpText<"Path to ThinLTO cached object file directory">;
defm thinlto_cache_policy: EEq<"thinlto-cache-policy", "Pruning policy for the ThinLTO cache">;
+def thinlto_emit_index_files: FF<"thinlto-emit-index-files">;
+def thinlto_emit_imports_files: FF<"thinlto-emit-imports-files">;
+def thinlto_index_only: FF<"thinlto-index-only">;
+def thinlto_index_only_eq: JJ<"thinlto-index-only=">;
def thinlto_jobs: JJ<"thinlto-jobs=">,
HelpText<"Number of ThinLTO jobs. Default to --threads=">;
def lto_debug_pass_manager: FF<"lto-debug-pass-manager">,
diff --git a/lld/wasm/SymbolTable.cpp b/lld/wasm/SymbolTable.cpp
index d2216ff5a39a0e..4cbf44b4d0398a 100644
--- a/lld/wasm/SymbolTable.cpp
+++ b/lld/wasm/SymbolTable.cpp
@@ -29,6 +29,7 @@ void SymbolTable::addFile(InputFile *file, StringRef symName) {
// Lazy object file
if (file->lazy) {
if (auto *f = dyn_cast<BitcodeFile>(file)) {
+ ctx.lazyBitcodeFiles.push_back(f);
f->parseLazy();
} else {
cast<ObjFile>(file)->parseLazy();
@@ -81,9 +82,6 @@ void SymbolTable::compileBitcodeFiles() {
// Prevent further LTO objects being included
BitcodeFile::doneLTO = true;
- if (ctx.bitcodeFiles.empty())
- return;
-
// Compile bitcode files and replace bitcode symbols.
lto.reset(new BitcodeCompiler);
for (BitcodeFile *f : ctx.bitcodeFiles)
More information about the llvm-commits
mailing list