[clang] [llvm] [ThinLTO] Apply -vfsoverlay to distributed backend inputs (PR #236610)
Anton Yartsev via cfe-commits
cfe-commits at lists.llvm.org
Mon Nov 2 15:48:11 PST 2026
https://github.com/ayartsev7 updated https://github.com/llvm/llvm-project/pull/236610
>From ef51ca08203e2f3bcaaf8ee5874ee71ed9aa6c79 Mon Sep 17 00:00:00 2001
From: Anton Yartsev <anton.yartsev at gmail.com>
Date: Thu, 29 Oct 2026 02:55:09 +0300
Subject: [PATCH 1/7] ThinLTO: read the index and imported modules through the
VFS
---
clang/lib/CodeGen/BackendUtil.cpp | 8 ++++----
llvm/include/llvm/Bitcode/BitcodeReader.h | 10 ++++++++--
llvm/include/llvm/LTO/LTOBackend.h | 10 ++++++++--
llvm/lib/Bitcode/Reader/BitcodeReader.cpp | 10 ++++++++--
llvm/lib/LTO/LTOBackend.cpp | 8 ++++++--
5 files changed, 34 insertions(+), 12 deletions(-)
diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp
index fe1454026f569d..6eb1ca23890202 100644
--- a/clang/lib/CodeGen/BackendUtil.cpp
+++ b/clang/lib/CodeGen/BackendUtil.cpp
@@ -1335,7 +1335,7 @@ static void
runThinLTOBackend(CompilerInstance &CI, ModuleSummaryIndex *CombinedIndex,
llvm::Module *M, std::unique_ptr<raw_pwrite_stream> OS,
std::string SampleProfile, std::string ProfileRemapping,
- BackendAction Action) {
+ BackendAction Action, llvm::vfs::FileSystem &FS) {
DiagnosticsEngine &Diags = CI.getDiagnostics();
const auto &CGOpts = CI.getCodeGenOpts();
const auto &TOpts = CI.getTargetOpts();
@@ -1446,7 +1446,7 @@ runThinLTOBackend(CompilerInstance &CI, ModuleSummaryIndex *CombinedIndex,
Conf, -1, AddStream, *M, *CombinedIndex, ImportList,
ModuleToDefinedGVSummaries[M->getModuleIdentifier()],
/*ModuleMap=*/nullptr, Conf.CodeGenOnly, /*BitcodeLibFuncs=*/{},
- /*IRAddStream=*/nullptr, CGOpts.CmdArgs)) {
+ /*IRAddStream=*/nullptr, CGOpts.CmdArgs, &FS)) {
handleAllErrors(std::move(E), [&](ErrorInfoBase &EIB) {
errs() << "Error running ThinLTO backend: " << EIB.message() << '\n';
});
@@ -1556,7 +1556,7 @@ void clang::emitBackendOutput(CompilerInstance &CI, CodeGenOptions &CGOpts,
std::unique_ptr<ModuleSummaryIndex> CombinedIndex;
if (Error E = llvm::getModuleSummaryIndexForFile(
CGOpts.ThinLTOIndexFile,
- /*IgnoreEmptyThinLTOIndexFile*/ true)
+ /*IgnoreEmptyThinLTOIndexFile*/ true, VFS.get())
.moveInto(CombinedIndex)) {
logAllUnhandledErrors(std::move(E), errs(),
"Error loading index file '" +
@@ -1571,7 +1571,7 @@ void clang::emitBackendOutput(CompilerInstance &CI, CodeGenOptions &CGOpts,
if (!CombinedIndex->skipModuleByDistributedBackend()) {
runThinLTOBackend(CI, CombinedIndex.get(), M, std::move(OS),
CGOpts.SampleProfileFile, CGOpts.ProfileRemappingFile,
- Action);
+ Action, *VFS);
return;
}
// Distributed indexing detected that nothing from the module is needed
diff --git a/llvm/include/llvm/Bitcode/BitcodeReader.h b/llvm/include/llvm/Bitcode/BitcodeReader.h
index df30463f556197..32d00fb05ef653 100644
--- a/llvm/include/llvm/Bitcode/BitcodeReader.h
+++ b/llvm/include/llvm/Bitcode/BitcodeReader.h
@@ -39,6 +39,10 @@ class Type;
class Value;
struct ValueInfo;
+namespace vfs {
+class FileSystem;
+} // namespace vfs
+
// Callback to override the data layout string of an imported bitcode module.
// The first argument is the target triple, the second argument the data layout
// string from the input, or a default string. It will be used if the callback
@@ -249,10 +253,12 @@ struct ParserCallbacks {
/// Parse the module summary index out of an IR file and return the module
/// summary index object if found, or an empty summary if not. If Path refers
/// to an empty file and IgnoreEmptyThinLTOIndexFile is true, then
- /// this function will return nullptr.
+ /// this function will return nullptr. If FS is not nullptr, the file is read
+ /// through it instead of the real file system.
LLVM_ABI Expected<std::unique_ptr<ModuleSummaryIndex>>
getModuleSummaryIndexForFile(StringRef Path,
- bool IgnoreEmptyThinLTOIndexFile = false);
+ bool IgnoreEmptyThinLTOIndexFile = false,
+ vfs::FileSystem *FS = nullptr);
/// isBitcodeWrapper - Return true if the given bytes are the magic bytes
/// for an LLVM IR bitcode wrapper.
diff --git a/llvm/include/llvm/LTO/LTOBackend.h b/llvm/include/llvm/LTO/LTOBackend.h
index 4bb38529ec7540..b623cafb9e6f3b 100644
--- a/llvm/include/llvm/LTO/LTOBackend.h
+++ b/llvm/include/llvm/LTO/LTOBackend.h
@@ -32,6 +32,10 @@ class Error;
class Module;
class Target;
+namespace vfs {
+class FileSystem;
+} // namespace vfs
+
namespace lto {
/// Runs middle-end LTO optimizations on \p Mod.
@@ -57,7 +61,8 @@ LLVM_ABI Error backend(const Config &C, AddStreamFn AddStream,
/// one source module will be kept open at the most. If \p CodeGenOnly is true,
/// the backend will skip optimization and only perform code generation. If
/// \p IRAddStream is not nullptr, it will be called just before code generation
-/// to serialize the optimized IR.
+/// to serialize the optimized IR. If \p FS is not nullptr, module files to be
+/// imported on demand are read through it instead of the real file system.
LLVM_ABI Error thinBackend(
const Config &C, unsigned Task, AddStreamFn AddStream, Module &M,
const ModuleSummaryIndex &CombinedIndex,
@@ -65,7 +70,8 @@ LLVM_ABI Error thinBackend(
const GVSummaryMapTy &DefinedGlobals,
MapVector<StringRef, BitcodeModule> *ModuleMap, bool CodeGenOnly,
ArrayRef<StringRef> BitcodeLibFuncs, AddStreamFn IRAddStream = nullptr,
- const std::vector<uint8_t> &CmdArgs = std::vector<uint8_t>());
+ const std::vector<uint8_t> &CmdArgs = std::vector<uint8_t>(),
+ vfs::FileSystem *FS = nullptr);
LLVM_ABI Error finalizeOptimizationRemarks(LLVMRemarkFileHandle DiagOutputFile);
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index 0535d89057671c..b72209f9c29bd2 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -74,6 +74,7 @@
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/ModRef.h"
#include "llvm/Support/SwapByteOrder.h"
+#include "llvm/Support/VirtualFileSystem.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/TargetParser/Triple.h"
#include <algorithm>
@@ -9133,9 +9134,14 @@ Expected<BitcodeLTOInfo> llvm::getBitcodeLTOInfo(MemoryBufferRef Buffer) {
Expected<std::unique_ptr<ModuleSummaryIndex>>
llvm::getModuleSummaryIndexForFile(StringRef Path,
- bool IgnoreEmptyThinLTOIndexFile) {
+ bool IgnoreEmptyThinLTOIndexFile,
+ vfs::FileSystem *FS) {
ErrorOr<std::unique_ptr<MemoryBuffer>> FileOrErr =
- MemoryBuffer::getFileOrSTDIN(Path);
+ FS && Path != "-"
+ ? FS->getBufferForFile(Path, /*FileSize=*/-1,
+ /*RequiresNullTerminator=*/true,
+ /*IsVolatile=*/false, /*IsText=*/false)
+ : MemoryBuffer::getFileOrSTDIN(Path);
if (!FileOrErr)
return errorCodeToError(FileOrErr.getError());
if (IgnoreEmptyThinLTOIndexFile && !(*FileOrErr)->getBufferSize())
diff --git a/llvm/lib/LTO/LTOBackend.cpp b/llvm/lib/LTO/LTOBackend.cpp
index ed431be770f593..d06d10f6013e19 100644
--- a/llvm/lib/LTO/LTOBackend.cpp
+++ b/llvm/lib/LTO/LTOBackend.cpp
@@ -660,7 +660,8 @@ Error lto::thinBackend(const Config &Conf, unsigned Task, AddStreamFn AddStream,
MapVector<StringRef, BitcodeModule> *ModuleMap,
bool CodeGenOnly, ArrayRef<StringRef> BitcodeLibFuncs,
AddStreamFn IRAddStream,
- const std::vector<uint8_t> &CmdArgs) {
+ const std::vector<uint8_t> &CmdArgs,
+ vfs::FileSystem *FS) {
llvm::TimeTraceScope timeScope("Thin backend", Mod.getModuleIdentifier());
Expected<const Target *> TOrErr = initAndLookupTarget(Conf, Mod);
if (!TOrErr)
@@ -750,7 +751,10 @@ Error lto::thinBackend(const Config &Conf, unsigned Task, AddStreamFn AddStream,
}
ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> MBOrErr =
- llvm::MemoryBuffer::getFile(Identifier);
+ FS ? FS->getBufferForFile(Identifier, /*FileSize=*/-1,
+ /*RequiresNullTerminator=*/true,
+ /*IsVolatile=*/false, /*IsText=*/false)
+ : llvm::MemoryBuffer::getFile(Identifier);
if (!MBOrErr)
return Expected<std::unique_ptr<llvm::Module>>(make_error<StringError>(
Twine("Error loading imported file ") + Identifier + " : ",
>From 5223a36bc651c370f51c710279e77d3453769f3c Mon Sep 17 00:00:00 2001
From: Anton Yartsev <anton.yartsev at gmail.com>
Date: Thu, 29 Oct 2026 02:55:10 +0300
Subject: [PATCH 2/7] Apply VFS overlays so the driver sees the same files as
-cc1
---
clang/lib/Driver/Driver.cpp | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index e43bd6b1049b81..e72bceaabca249 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -1589,6 +1589,22 @@ Compilation *Driver::BuildCompilation(ArrayRef<const char *> ArgList) {
if (VFS->setCurrentWorkingDirectory(WD->getValue()))
Diag(diag::err_drv_unable_to_set_working_directory) << WD->getValue();
+ // Apply VFS overlays so the driver sees the same files as -cc1.
+ for (const Arg *A :
+ Args.filtered(options::OPT_ivfsoverlay, options::OPT_vfsoverlay)) {
+ StringRef File = A->getValue();
+ llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> Buffer =
+ VFS->getBufferForFile(File);
+ if (!Buffer)
+ continue;
+ if (IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS =
+ llvm::vfs::getVFSFromYAML(
+ std::move(*Buffer),
+ [](const llvm::SMDiagnostic &, void *) {}, File,
+ /*DiagContext=*/nullptr, VFS))
+ VFS = std::move(FS);
+ }
+
// Check for missing include directories.
if (!Diags.isIgnored(diag::warn_missing_include_dirs, SourceLocation())) {
for (auto IncludeDir : Args.getAllArgValues(options::OPT_I_Group)) {
>From 4a3e6c30f84de9e46305c6655b5124562b0e7e90 Mon Sep 17 00:00:00 2001
From: Anton Yartsev <anton.yartsev at gmail.com>
Date: Thu, 29 Oct 2026 02:55:12 +0300
Subject: [PATCH 3/7] Set module identifier to the command-line path for
ThinLTO
---
clang/lib/CodeGen/CodeGenAction.cpp | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/clang/lib/CodeGen/CodeGenAction.cpp b/clang/lib/CodeGen/CodeGenAction.cpp
index c54cc093218c91..aa12396a4d40b2 100644
--- a/clang/lib/CodeGen/CodeGenAction.cpp
+++ b/clang/lib/CodeGen/CodeGenAction.cpp
@@ -455,6 +455,12 @@ CodeGenAction::loadModule(MemoryBufferRef MBRef) {
Bm->parseModule(*VMContext);
if (!MOrErr)
return DiagErrors(MOrErr.takeError());
+ // Set module identifier to the command-line path.
+ // The ThinLTO index identifies modules by their command-line paths, a VFS
+ // overlay can change this module's identifier, and then it cannot be found
+ // in the index.
+ if (getCurrentInput().isFile() && getCurrentFile() != "-")
+ (*MOrErr)->setModuleIdentifier(getCurrentFile());
return std::move(*MOrErr);
}
>From fd69b4a763fb0180c8016693c424671afcbc954c Mon Sep 17 00:00:00 2001
From: Anton Yartsev <anton.yartsev at gmail.com>
Date: Thu, 29 Oct 2026 02:55:13 +0300
Subject: [PATCH 4/7] ThinLTO backend reads its inputs through -vfsoverlay
---
clang/test/CodeGen/thinlto-distributed-vfs.ll | 46 +++++++++++++++++++
1 file changed, 46 insertions(+)
create mode 100644 clang/test/CodeGen/thinlto-distributed-vfs.ll
diff --git a/clang/test/CodeGen/thinlto-distributed-vfs.ll b/clang/test/CodeGen/thinlto-distributed-vfs.ll
new file mode 100644
index 00000000000000..8d3923d08b3b95
--- /dev/null
+++ b/clang/test/CodeGen/thinlto-distributed-vfs.ll
@@ -0,0 +1,46 @@
+; REQUIRES: x86-registered-target
+
+; Test that the ThinLTO backend reads its files through -vfsoverlay if provided.
+
+; RUN: rm -rf %t && mkdir -p %t/orig %t/remote
+; RUN: opt -module-summary -o %t/orig/1.o %s
+; RUN: opt -module-summary -o %t/orig/2.o %S/Inputs/thinlto_backend.ll
+; RUN: llvm-lto -thinlto -o %t/orig/all %t/orig/1.o %t/orig/2.o
+; RUN: mv %t/orig/1.o %t/orig/2.o %t/orig/all.thinlto.bc %t/remote
+
+; No overlay provided, the index file is not found.
+; RUN: %clang -target x86_64-unknown-linux-gnu -O2 -c -x ir %t/remote/1.o \
+; RUN: -fthinlto-index=%t/orig/all.thinlto.bc -o %t/none.o 2>&1 \
+; RUN: | FileCheck %s --check-prefix=CHECK-NONE
+; CHECK-NONE: Error loading index file
+
+; Overlay of the index only, the import is still missing.
+; RUN: echo "{'version': 0, 'use-external-names': false, 'roots': [{'type': 'file', 'name': '%/t/orig/all.thinlto.bc', 'external-contents': '%/t/remote/all.thinlto.bc'}]}" > %t/one-file.yaml
+; RUN: %clang -target x86_64-unknown-linux-gnu -O2 -c -x ir %t/remote/1.o \
+; RUN: -fthinlto-index=%t/orig/all.thinlto.bc -vfsoverlay %t/one-file.yaml \
+; RUN: -o %t/one-file.o 2>&1 | FileCheck %s --check-prefix=CHECK-ONE
+; CHECK-ONE: Error loading imported file
+
+; Remap the directory so f2 is imported and f1 stays for either external names.
+; RUN: echo "{'version': 0, 'use-external-names': false, 'roots': [{'type': 'directory-remap', 'name': '%/t/orig', 'external-contents': '%/t/remote'}]}" > %t/vfs.yaml
+; RUN: %clang -target x86_64-unknown-linux-gnu -O2 -c -x ir %t/orig/1.o \
+; RUN: -fthinlto-index=%t/orig/all.thinlto.bc -vfsoverlay %t/vfs.yaml \
+; RUN: -o %t/remapped.o
+; RUN: llvm-nm %t/remapped.o | FileCheck %s --check-prefix=CHECK-OBJ
+; RUN: echo "{'version': 0, 'use-external-names': true, 'roots': [{'type': 'directory-remap', 'name': '%/t/orig', 'external-contents': '%/t/remote'}]}" > %t/vfs-external.yaml
+; RUN: %clang -target x86_64-unknown-linux-gnu -O2 -c -x ir %t/orig/1.o \
+; RUN: -fthinlto-index=%t/orig/all.thinlto.bc -vfsoverlay %t/vfs-external.yaml \
+; RUN: -o %t/remapped-external.o
+; RUN: llvm-nm %t/remapped-external.o | FileCheck %s --check-prefix=CHECK-OBJ
+; CHECK-OBJ: T f1
+; CHECK-OBJ-NOT: U f2
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare void @f2()
+
+define void @f1() {
+ call void @f2()
+ ret void
+}
>From 9078b235fb1edb1f69123aeed3cfd3b76f3b8c93 Mon Sep 17 00:00:00 2001
From: Anton Yartsev <anton.yartsev at gmail.com>
Date: Thu, 29 Oct 2026 02:55:14 +0300
Subject: [PATCH 5/7] Test that the driver resolves its inputs through
-vfsoverlay
---
clang/test/Driver/vfsoverlay.c | 15 +++++++++++++++
1 file changed, 15 insertions(+)
diff --git a/clang/test/Driver/vfsoverlay.c b/clang/test/Driver/vfsoverlay.c
index 694cf623c4476d..152c9c332a4ea6 100644
--- a/clang/test/Driver/vfsoverlay.c
+++ b/clang/test/Driver/vfsoverlay.c
@@ -6,3 +6,18 @@
// RUN: not %clang -ivfsoverlay foo.h %s 2>&1 | FileCheck -check-prefix=CHECK-MISSING %s
// CHECK-MISSING: virtual filesystem overlay file 'foo.h' not found
+
+// The driver applies overlays.
+// RUN: rm -rf %t && mkdir -p %t/real
+// RUN: echo "int a;" > %t/real/a.c
+// RUN: echo "{'version': 0, 'roots': [{'type': 'directory-remap', 'name': '%/t/virtual', 'external-contents': '%/t/real'}]}" > %t/vfs.yaml
+// RUN: not %clang -fsyntax-only %t/virtual/a.c 2>&1 | FileCheck %s --check-prefix=CHECK-NO-OVERLAY
+// CHECK-NO-OVERLAY: error: no such file or directory: '{{.*}}a.c'
+// RUN: %clang -fsyntax-only -ivfsoverlay %t/vfs.yaml %t/virtual/a.c
+// RUN: %clang -fsyntax-only -vfsoverlay %t/vfs.yaml %t/virtual/a.c
+
+// The driver leaves invalid overlays for -cc1 to diagnose.
+// RUN: echo "invalid" > %t/invalid.yaml
+// RUN: not %clang -fsyntax-only -ivfsoverlay %t/invalid.yaml %t/real/a.c 2>&1 | FileCheck -check-prefix=CHECK-INVALID %s
+// RUN: not %clang -fsyntax-only -vfsoverlay %t/invalid.yaml %t/real/a.c 2>&1 | FileCheck -check-prefix=CHECK-INVALID %s
+// CHECK-INVALID: invalid virtual filesystem overlay file '{{.*}}invalid.yaml'
>From 144c8787d5e67a740327d760b858be819a904aa3 Mon Sep 17 00:00:00 2001
From: Anton Yartsev <anton.yartsev at gmail.com>
Date: Thu, 29 Oct 2026 02:55:16 +0300
Subject: [PATCH 6/7] clang-format
---
clang/lib/Driver/Driver.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index e72bceaabca249..77486304501e29 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -1599,8 +1599,8 @@ Compilation *Driver::BuildCompilation(ArrayRef<const char *> ArgList) {
continue;
if (IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS =
llvm::vfs::getVFSFromYAML(
- std::move(*Buffer),
- [](const llvm::SMDiagnostic &, void *) {}, File,
+ std::move(*Buffer), [](const llvm::SMDiagnostic &, void *) {},
+ File,
/*DiagContext=*/nullptr, VFS))
VFS = std::move(FS);
}
>From f6df8261d49f5b0eac47e43ae51dda1fda2b5e71 Mon Sep 17 00:00:00 2001
From: Anton Yartsev <anton.yartsev at gmail.com>
Date: Tue, 3 Nov 2026 02:47:40 +0300
Subject: [PATCH 7/7] Use the requested path as the ThinLTO module identifier
---
clang/lib/CodeGen/CodeGenAction.cpp | 16 ++++++++++------
1 file changed, 10 insertions(+), 6 deletions(-)
diff --git a/clang/lib/CodeGen/CodeGenAction.cpp b/clang/lib/CodeGen/CodeGenAction.cpp
index aa12396a4d40b2..53732709f919a3 100644
--- a/clang/lib/CodeGen/CodeGenAction.cpp
+++ b/clang/lib/CodeGen/CodeGenAction.cpp
@@ -455,12 +455,16 @@ CodeGenAction::loadModule(MemoryBufferRef MBRef) {
Bm->parseModule(*VMContext);
if (!MOrErr)
return DiagErrors(MOrErr.takeError());
- // Set module identifier to the command-line path.
- // The ThinLTO index identifies modules by their command-line paths, a VFS
- // overlay can change this module's identifier, and then it cannot be found
- // in the index.
- if (getCurrentInput().isFile() && getCurrentFile() != "-")
- (*MOrErr)->setModuleIdentifier(getCurrentFile());
+ // The ThinLTO index identifies this module by the path given on the
+ // command line. With a VFS overlay that uses 'use-external-names', the
+ // FileManager renames the main file to its external path (see the FIXME in
+ // FileManager::getFileRefImpl), and BitcodeReader takes the module
+ // identifier from it. As a result, the module does not match its index
+ // entry, and the ThinLTO backend mishandles it.
+ // We restore the requested name so the module matches its index entry.
+ if (OptionalFileEntryRef FE = SM.getFileEntryRefForID(SM.getMainFileID()))
+ if (FE->getName() != FE->getNameAsRequested())
+ (*MOrErr)->setModuleIdentifier(FE->getNameAsRequested());
return std::move(*MOrErr);
}
More information about the cfe-commits
mailing list