[lld] Reland [CGData] LLD for MachO #90166 (PR #108733)
Kyungwoo Lee via llvm-commits
llvm-commits at lists.llvm.org
Sat Sep 14 23:52:01 PDT 2024
https://github.com/kyulee-com updated https://github.com/llvm/llvm-project/pull/108733
>From 4a02c2a13692a8e978a090ed05defc3f01ece452 Mon Sep 17 00:00:00 2001
From: Kyungwoo Lee <kyulee at meta.com>
Date: Sat, 14 Sep 2024 20:38:14 -0700
Subject: [PATCH 1/2] Reland [CGData] LLD for MachO (#90166)
It reads raw CG data encoded in the custom section (__llvm_outline) in
object files and merges them into the indexed codegen data file
specified by `-codegen-data-generate-path={path}`.
This depends on https://github.com/llvm/llvm-project/pull/90074.
This is a patch for
https://discourse.llvm.org/t/rfc-enhanced-machine-outliner-part-2-thinlto-nolto/78753.
---
lld/MachO/Config.h | 1 +
lld/MachO/Driver.cpp | 38 ++++++++++++++
lld/MachO/InputSection.h | 1 +
lld/MachO/Options.td | 4 ++
lld/test/CMakeLists.txt | 1 +
lld/test/MachO/cgdata-generate.s | 89 ++++++++++++++++++++++++++++++++
lld/test/lit.cfg.py | 1 +
7 files changed, 135 insertions(+)
create mode 100644 lld/test/MachO/cgdata-generate.s
diff --git a/lld/MachO/Config.h b/lld/MachO/Config.h
index 5fca3f16a897ed..8f6da6330d7ad4 100644
--- a/lld/MachO/Config.h
+++ b/lld/MachO/Config.h
@@ -210,6 +210,7 @@ struct Configuration {
std::vector<SectionAlign> sectionAlignments;
std::vector<SegmentProtection> segmentProtections;
bool ltoDebugPassManager = false;
+ llvm::StringRef codegenDataGeneratePath;
bool csProfileGenerate = false;
llvm::StringRef csProfilePath;
bool pgoWarnMismatch;
diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index 73b83123f49b4d..6444e54fad252e 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -36,6 +36,7 @@
#include "llvm/ADT/StringRef.h"
#include "llvm/BinaryFormat/MachO.h"
#include "llvm/BinaryFormat/Magic.h"
+#include "llvm/CGData/CodeGenDataWriter.h"
#include "llvm/Config/llvm-config.h"
#include "llvm/LTO/LTO.h"
#include "llvm/Object/Archive.h"
@@ -1322,6 +1323,37 @@ static void gatherInputSections() {
}
}
+static void codegenDataGenerate() {
+ TimeTraceScope timeScope("Generating codegen data");
+
+ OutlinedHashTreeRecord globalOutlineRecord;
+ for (ConcatInputSection *isec : inputSections)
+ if (isec->getSegName() == segment_names::data &&
+ isec->getName() == section_names::outlinedHashTree) {
+ // Read outlined hash tree from each section.
+ OutlinedHashTreeRecord localOutlineRecord;
+ auto *data = isec->data.data();
+ localOutlineRecord.deserialize(data);
+
+ // Merge it to the global hash tree.
+ globalOutlineRecord.merge(localOutlineRecord);
+ }
+
+ CodeGenDataWriter Writer;
+ if (!globalOutlineRecord.empty())
+ Writer.addRecord(globalOutlineRecord);
+
+ std::error_code EC;
+ auto fileName = config->codegenDataGeneratePath;
+ assert(!fileName.empty());
+ raw_fd_ostream Output(fileName, EC, sys::fs::OF_None);
+ if (EC)
+ error("fail to create " + fileName + ": " + EC.message());
+
+ if (auto E = Writer.write(Output))
+ error("fail to write CGData: " + toString(std::move(E)));
+}
+
static void foldIdenticalLiterals() {
TimeTraceScope timeScope("Fold identical literals");
// We always create a cStringSection, regardless of whether dedupLiterals is
@@ -1759,6 +1791,8 @@ bool link(ArrayRef<const char *> argsArr, llvm::raw_ostream &stdoutOS,
config->ignoreAutoLinkOptions.insert(arg->getValue());
config->strictAutoLink = args.hasArg(OPT_strict_auto_link);
config->ltoDebugPassManager = args.hasArg(OPT_lto_debug_pass_manager);
+ config->codegenDataGeneratePath =
+ args.getLastArgValue(OPT_codegen_data_generate_path);
config->csProfileGenerate = args.hasArg(OPT_cs_profile_generate);
config->csProfilePath = args.getLastArgValue(OPT_cs_profile_path);
config->pgoWarnMismatch =
@@ -2103,6 +2137,10 @@ bool link(ArrayRef<const char *> argsArr, llvm::raw_ostream &stdoutOS,
}
gatherInputSections();
+
+ if (!config->codegenDataGeneratePath.empty())
+ codegenDataGenerate();
+
if (config->callGraphProfileSort)
priorityBuilder.extractCallGraphProfile();
diff --git a/lld/MachO/InputSection.h b/lld/MachO/InputSection.h
index 4e238d8ef77793..7ef0e31066f372 100644
--- a/lld/MachO/InputSection.h
+++ b/lld/MachO/InputSection.h
@@ -354,6 +354,7 @@ constexpr const char objcMethname[] = "__objc_methname";
constexpr const char objcNonLazyCatList[] = "__objc_nlcatlist";
constexpr const char objcNonLazyClassList[] = "__objc_nlclslist";
constexpr const char objcProtoList[] = "__objc_protolist";
+constexpr const char outlinedHashTree[] = "__llvm_outline";
constexpr const char pageZero[] = "__pagezero";
constexpr const char pointers[] = "__pointers";
constexpr const char rebase[] = "__rebase";
diff --git a/lld/MachO/Options.td b/lld/MachO/Options.td
index cbd28bbceef786..70eb7c8b9e466b 100644
--- a/lld/MachO/Options.td
+++ b/lld/MachO/Options.td
@@ -162,6 +162,10 @@ def no_objc_category_merging : Flag<["-"], "no_objc_category_merging">,
Group<grp_lld>;
def lto_debug_pass_manager: Flag<["--"], "lto-debug-pass-manager">,
HelpText<"Debug new pass manager">, Group<grp_lld>;
+def codegen_data_generate_path : Separate<["--"], "codegen-data-generate-path">, Group<grp_lld>;
+def codegen_data_generate_path_eq : Joined<["--"], "codegen-data-generate-path=">,
+ Alias<!cast<Separate>(codegen_data_generate_path)>, MetaVarName<"<cgdata>">,
+ HelpText<"Write the CG data to the specified path <cgdata>.">, Group<grp_lld>;
def cs_profile_generate: Flag<["--"], "cs-profile-generate">,
HelpText<"Perform context sensitive PGO instrumentation">, Group<grp_lld>;
def cs_profile_path: Joined<["--"], "cs-profile-path=">,
diff --git a/lld/test/CMakeLists.txt b/lld/test/CMakeLists.txt
index 5d4a2757c529b8..abc8ea75da180b 100644
--- a/lld/test/CMakeLists.txt
+++ b/lld/test/CMakeLists.txt
@@ -48,6 +48,7 @@ if (NOT LLD_BUILT_STANDALONE)
llvm-ar
llvm-as
llvm-bcanalyzer
+ llvm-cgdata
llvm-config
llvm-cvtres
llvm-dis
diff --git a/lld/test/MachO/cgdata-generate.s b/lld/test/MachO/cgdata-generate.s
new file mode 100644
index 00000000000000..6295e33a39bd3b
--- /dev/null
+++ b/lld/test/MachO/cgdata-generate.s
@@ -0,0 +1,89 @@
+# UNSUPPORTED: system-windows
+# REQUIRES: aarch64
+
+# RUN: rm -rf %t; split-file %s %t
+
+# Synthesize raw cgdata without the header (24 byte) from the indexed cgdata.
+# RUN: llvm-cgdata --convert --format binary %t/raw-1.cgtext -o %t/raw-1.cgdata
+# RUN: od -t x1 -j 24 -An %t/raw-1.cgdata | tr -d '\n\r\t' | sed 's/ \+/ /g; s/^ *//; s/ *$//; s/ /,0x/g; s/^/0x/' > %t/raw-1-bytes.txt
+# RUN: sed "s/<RAW_BYTES>/$(cat %t/raw-1-bytes.txt)/g" %t/merge-template.s > %t/merge-1.s
+# RUN: llvm-cgdata --convert --format binary %t/raw-2.cgtext -o %t/raw-2.cgdata
+# RUN: od -t x1 -j 24 -An %t/raw-2.cgdata | tr -d '\n\r\t' | sed 's/ \+/ /g; s/^ *//; s/ *$//; s/ /,0x/g; s/^/0x/' > %t/raw-2-bytes.txt
+# RUN: sed "s/<RAW_BYTES>/$(cat %t/raw-2-bytes.txt)/g" %t/merge-template.s > %t/merge-2.s
+
+# RUN: llvm-mc -filetype obj -triple arm64-apple-darwin %t/merge-1.s -o %t/merge-1.o
+# RUN: llvm-mc -filetype obj -triple arm64-apple-darwin %t/merge-2.s -o %t/merge-2.o
+# RUN: llvm-mc -filetype obj -triple arm64-apple-darwin %t/main.s -o %t/main.o
+
+# This checks if the codegen data from the linker is identical to the merged codegen data
+# from each object file, which is obtained using the llvm-cgdata tool.
+# RUN: %no-arg-lld -dylib -arch arm64 -platform_version ios 14.0 15.0 -o %t/out \
+# RUN: %t/merge-1.o %t/merge-2.o %t/main.o --codegen-data-generate-path=%t/out-cgdata
+# RUN: llvm-cgdata --merge %t/merge-1.o %t/merge-2.o %t/main.o -o %t/merge-cgdata
+# RUN: diff %t/out-cgdata %t/merge-cgdata
+
+# Merge order doesn't matter. `main.o` is dropped due to missing __llvm_outline.
+# RUN: llvm-cgdata --merge %t/merge-2.o %t/merge-1.o -o %t/merge-cgdata-shuffle
+# RUN: diff %t/out-cgdata %t/merge-cgdata-shuffle
+
+# We can also generate the merged codegen data from the executable that is not dead-stripped.
+# RUN: llvm-objdump -h %t/out| FileCheck %s
+# CHECK: __llvm_outline
+# RUN: llvm-cgdata --merge %t/out -o %t/merge-cgdata-exe
+# RUN: diff %t/merge-cgdata-exe %t/merge-cgdata
+
+# Dead-strip will remove __llvm_outline sections from the final executable.
+# But the codeden data is still correctly produced from the linker.
+# RUN: %no-arg-lld -dylib -arch arm64 -platform_version ios 14.0 15.0 -o %t/out-strip \
+# RUN: %t/merge-1.o %t/merge-2.o %t/main.o -dead_strip --codegen-data-generate-path=%t/out-cgdata-strip
+# RUN: llvm-cgdata --merge %t/merge-1.o %t/merge-2.o %t/main.o -o %t/merge-cgdata-strip
+# RUN: diff %t/out-cgdata-strip %t/merge-cgdata-strip
+# RUN: diff %t/out-cgdata-strip %t/merge-cgdata
+
+# Ensure no __llvm_outline section remains in the executable.
+# RUN: llvm-objdump -h %t/out-strip | FileCheck %s --check-prefix=STRIP
+# STRIP-NOT: __llvm_outline
+
+#--- raw-1.cgtext
+:outlined_hash_tree
+0:
+ Hash: 0x0
+ Terminals: 0
+ SuccessorIds: [ 1 ]
+1:
+ Hash: 0x1
+ Terminals: 0
+ SuccessorIds: [ 2 ]
+2:
+ Hash: 0x2
+ Terminals: 4
+ SuccessorIds: [ ]
+...
+
+#--- raw-2.cgtext
+:outlined_hash_tree
+0:
+ Hash: 0x0
+ Terminals: 0
+ SuccessorIds: [ 1 ]
+1:
+ Hash: 0x1
+ Terminals: 0
+ SuccessorIds: [ 2 ]
+2:
+ Hash: 0x3
+ Terminals: 5
+ SuccessorIds: [ ]
+...
+
+#--- merge-template.s
+.section __DATA,__llvm_outline
+_data:
+.byte <RAW_BYTES>
+
+#--- main.s
+.globl _main
+
+.text
+_main:
+ ret
diff --git a/lld/test/lit.cfg.py b/lld/test/lit.cfg.py
index d309c2ad4ee284..859094e2b57dbc 100644
--- a/lld/test/lit.cfg.py
+++ b/lld/test/lit.cfg.py
@@ -40,6 +40,7 @@
tool_patterns = [
"llc",
"llvm-as",
+ "llvm-cgdata",
"llvm-mc",
"llvm-nm",
"llvm-objdump",
>From 3741cac55f54185ceb3ecf206accf2d42a343c35 Mon Sep 17 00:00:00 2001
From: Kyungwoo Lee <kyulee at meta.com>
Date: Sat, 14 Sep 2024 23:34:39 -0700
Subject: [PATCH 2/2] Fix build break
---
lld/MachO/CMakeLists.txt | 1 +
lld/test/MachO/cgdata-generate.s | 4 ++--
2 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/lld/MachO/CMakeLists.txt b/lld/MachO/CMakeLists.txt
index 385a5f5d022ebf..ecf6ce609e59f2 100644
--- a/lld/MachO/CMakeLists.txt
+++ b/lld/MachO/CMakeLists.txt
@@ -40,6 +40,7 @@ add_lld_library(lldMachO
BinaryFormat
BitReader
BitWriter
+ CGData
Core
DebugInfoDWARF
Demangle
diff --git a/lld/test/MachO/cgdata-generate.s b/lld/test/MachO/cgdata-generate.s
index 6295e33a39bd3b..174df39d666c5d 100644
--- a/lld/test/MachO/cgdata-generate.s
+++ b/lld/test/MachO/cgdata-generate.s
@@ -5,10 +5,10 @@
# Synthesize raw cgdata without the header (24 byte) from the indexed cgdata.
# RUN: llvm-cgdata --convert --format binary %t/raw-1.cgtext -o %t/raw-1.cgdata
-# RUN: od -t x1 -j 24 -An %t/raw-1.cgdata | tr -d '\n\r\t' | sed 's/ \+/ /g; s/^ *//; s/ *$//; s/ /,0x/g; s/^/0x/' > %t/raw-1-bytes.txt
+# RUN: od -t x1 -j 24 -An %t/raw-1.cgdata | tr -d '\n\r\t' | sed 's/[ ][ ]*/ /g; s/^[ ]*//; s/[ ]*$//; s/[ ]/,0x/g; s/^/0x/' > %t/raw-1-bytes.txt
# RUN: sed "s/<RAW_BYTES>/$(cat %t/raw-1-bytes.txt)/g" %t/merge-template.s > %t/merge-1.s
# RUN: llvm-cgdata --convert --format binary %t/raw-2.cgtext -o %t/raw-2.cgdata
-# RUN: od -t x1 -j 24 -An %t/raw-2.cgdata | tr -d '\n\r\t' | sed 's/ \+/ /g; s/^ *//; s/ *$//; s/ /,0x/g; s/^/0x/' > %t/raw-2-bytes.txt
+# RUN: od -t x1 -j 24 -An %t/raw-2.cgdata | tr -d '\n\r\t' | sed 's/[ ][ ]*/ /g; s/^[ ]*//; s/[ ]*$//; s/[ ]/,0x/g; s/^/0x/' > %t/raw-2-bytes.txt
# RUN: sed "s/<RAW_BYTES>/$(cat %t/raw-2-bytes.txt)/g" %t/merge-template.s > %t/merge-2.s
# RUN: llvm-mc -filetype obj -triple arm64-apple-darwin %t/merge-1.s -o %t/merge-1.o
More information about the llvm-commits
mailing list