[clang] Revert "[clang][Dependency Scanning] Fix the In-Memory Buffer Used for By-Name Scanning" (PR #185150)
via cfe-commits
cfe-commits at lists.llvm.org
Fri Mar 6 20:25:37 PST 2026
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-clang
Author: Thurston Dang (thurstond)
<details>
<summary>Changes</summary>
Reverts llvm/llvm-project#<!-- -->183396 due to buildbot breakage (reported earlier in https://github.com/llvm/llvm-project/pull/183396#issuecomment-4015167408)
---
Full diff: https://github.com/llvm/llvm-project/pull/185150.diff
6 Files Affected:
- (modified) clang/include/clang/DependencyScanning/DependencyScannerImpl.h (+1)
- (modified) clang/include/clang/DependencyScanning/DependencyScanningWorker.h (+14)
- (modified) clang/include/clang/Tooling/DependencyScanningTool.h (-10)
- (modified) clang/lib/DependencyScanning/DependencyScanningWorker.cpp (+55)
- (modified) clang/lib/Tooling/DependencyScanningTool.cpp (+2-64)
- (modified) clang/test/ClangScanDeps/modules-full-by-mult-mod-names-diagnostics.c (+3-3)
``````````diff
diff --git a/clang/include/clang/DependencyScanning/DependencyScannerImpl.h b/clang/include/clang/DependencyScanning/DependencyScannerImpl.h
index a54a6269dbdc4..24cf1afaaa450 100644
--- a/clang/include/clang/DependencyScanning/DependencyScannerImpl.h
+++ b/clang/include/clang/DependencyScanning/DependencyScannerImpl.h
@@ -16,6 +16,7 @@
#include "clang/Frontend/CompilerInstance.h"
#include "clang/Frontend/CompilerInvocation.h"
#include "clang/Frontend/TextDiagnosticPrinter.h"
+#include "clang/Serialization/ObjectFilePCHContainerReader.h"
#include "llvm/Support/VirtualFileSystem.h"
namespace clang {
diff --git a/clang/include/clang/DependencyScanning/DependencyScanningWorker.h b/clang/include/clang/DependencyScanning/DependencyScanningWorker.h
index 92da219d85d56..0a5c4936f7fe7 100644
--- a/clang/include/clang/DependencyScanning/DependencyScanningWorker.h
+++ b/clang/include/clang/DependencyScanning/DependencyScanningWorker.h
@@ -140,6 +140,20 @@ class DependencyScanningWorker {
friend tooling::CompilerInstanceWithContext;
};
+
+std::pair<IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem>,
+ std::vector<std::string>>
+initVFSForTUBufferScanning(IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS,
+ ArrayRef<std::string> CommandLine,
+ StringRef WorkingDirectory,
+ llvm::MemoryBufferRef TUBuffer);
+
+std::pair<IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem>,
+ std::vector<std::string>>
+initVFSForByNameScanning(IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS,
+ ArrayRef<std::string> CommandLine,
+ StringRef WorkingDirectory, StringRef ModuleName);
+
} // end namespace dependencies
} // end namespace clang
diff --git a/clang/include/clang/Tooling/DependencyScanningTool.h b/clang/include/clang/Tooling/DependencyScanningTool.h
index 30846ae0ebf3e..40e791fd5bff5 100644
--- a/clang/include/clang/Tooling/DependencyScanningTool.h
+++ b/clang/include/clang/Tooling/DependencyScanningTool.h
@@ -226,16 +226,6 @@ class CompilerInstanceWithContext {
StringRef ModuleName,
const llvm::DenseSet<dependencies::ModuleID> &AlreadySeen,
dependencies::LookupModuleOutputCallback LookupModuleOutput);
-
- // MaxNumOfQueries is the upper limit of the number of names the by-name
- // scanning API (computeDependencies) can support after a
- // CompilerInstanceWithContext is initialized. At the time of this commit, the
- // estimated number of total unique importable names is around 3000 from
- // Apple's SDKs. We usually import them in parallel, so it is unlikely that
- // all names are all scanned by the same dependency scanning worker. Therefore
- // the 64k (20x bigger than our estimate) size is sufficient to hold the
- // unique source locations to report diagnostics per worker.
- static const int32_t MaxNumOfQueries = 1 << 16;
};
} // end namespace tooling
diff --git a/clang/lib/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/DependencyScanning/DependencyScanningWorker.cpp
index 75eb821ac651a..7a7f34a43b338 100644
--- a/clang/lib/DependencyScanning/DependencyScanningWorker.cpp
+++ b/clang/lib/DependencyScanning/DependencyScanningWorker.cpp
@@ -108,3 +108,58 @@ bool DependencyScanningWorker::computeDependencies(
return Success && Action.hasScanned();
}
+
+std::pair<IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem>,
+ std::vector<std::string>>
+dependencies::initVFSForTUBufferScanning(
+ IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS,
+ ArrayRef<std::string> CommandLine, StringRef WorkingDirectory,
+ llvm::MemoryBufferRef TUBuffer) {
+ // Reset what might have been modified in the previous worker invocation.
+ BaseFS->setCurrentWorkingDirectory(WorkingDirectory);
+
+ auto OverlayFS =
+ llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(BaseFS);
+ auto InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
+ InMemoryFS->setCurrentWorkingDirectory(WorkingDirectory);
+ auto InputPath = TUBuffer.getBufferIdentifier();
+ InMemoryFS->addFile(
+ InputPath, 0, llvm::MemoryBuffer::getMemBufferCopy(TUBuffer.getBuffer()));
+ IntrusiveRefCntPtr<llvm::vfs::FileSystem> InMemoryOverlay = InMemoryFS;
+
+ OverlayFS->pushOverlay(InMemoryOverlay);
+ std::vector<std::string> ModifiedCommandLine(CommandLine);
+ ModifiedCommandLine.emplace_back(InputPath);
+
+ return std::make_pair(OverlayFS, ModifiedCommandLine);
+}
+
+std::pair<IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem>,
+ std::vector<std::string>>
+dependencies::initVFSForByNameScanning(
+ IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS,
+ ArrayRef<std::string> CommandLine, StringRef WorkingDirectory,
+ StringRef ModuleName) {
+ // Reset what might have been modified in the previous worker invocation.
+ BaseFS->setCurrentWorkingDirectory(WorkingDirectory);
+
+ // If we're scanning based on a module name alone, we don't expect the client
+ // to provide us with an input file. However, the driver really wants to have
+ // one. Let's just make it up to make the driver happy.
+ auto OverlayFS =
+ llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(BaseFS);
+ auto InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
+ InMemoryFS->setCurrentWorkingDirectory(WorkingDirectory);
+ SmallString<128> FakeInputPath;
+ // TODO: We should retry the creation if the path already exists.
+ llvm::sys::fs::createUniquePath(ModuleName + "-%%%%%%%%.input", FakeInputPath,
+ /*MakeAbsolute=*/false);
+ InMemoryFS->addFile(FakeInputPath, 0, llvm::MemoryBuffer::getMemBuffer(""));
+ IntrusiveRefCntPtr<llvm::vfs::FileSystem> InMemoryOverlay = InMemoryFS;
+ OverlayFS->pushOverlay(InMemoryOverlay);
+
+ std::vector<std::string> ModifiedCommandLine(CommandLine);
+ ModifiedCommandLine.emplace_back(FakeInputPath);
+
+ return std::make_pair(OverlayFS, ModifiedCommandLine);
+}
diff --git a/clang/lib/Tooling/DependencyScanningTool.cpp b/clang/lib/Tooling/DependencyScanningTool.cpp
index 147588eff59c7..b570bc8e5ae30 100644
--- a/clang/lib/Tooling/DependencyScanningTool.cpp
+++ b/clang/lib/Tooling/DependencyScanningTool.cpp
@@ -253,65 +253,6 @@ std::optional<P1689Rule> DependencyScanningTool::getP1689ModuleDependencyFile(
return Rule;
}
-static std::pair<IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem>,
- std::vector<std::string>>
-initVFSForTUBufferScanning(IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS,
- ArrayRef<std::string> CommandLine,
- StringRef WorkingDirectory,
- llvm::MemoryBufferRef TUBuffer) {
- // Reset what might have been modified in the previous worker invocation.
- BaseFS->setCurrentWorkingDirectory(WorkingDirectory);
-
- auto OverlayFS =
- llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(BaseFS);
- auto InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
- InMemoryFS->setCurrentWorkingDirectory(WorkingDirectory);
- auto InputPath = TUBuffer.getBufferIdentifier();
- InMemoryFS->addFile(
- InputPath, 0, llvm::MemoryBuffer::getMemBufferCopy(TUBuffer.getBuffer()));
- IntrusiveRefCntPtr<llvm::vfs::FileSystem> InMemoryOverlay = InMemoryFS;
-
- OverlayFS->pushOverlay(InMemoryOverlay);
- std::vector<std::string> ModifiedCommandLine(CommandLine);
- ModifiedCommandLine.emplace_back(InputPath);
-
- return std::make_pair(OverlayFS, ModifiedCommandLine);
-}
-
-// The fake input buffer is read-only, and it is used to produce
-// unique source locations for the diagnostics. Therefore sharing
-// this global buffer across threads is ok.
-static const std::string
- FakeInput(" ",
- clang::tooling::CompilerInstanceWithContext::MaxNumOfQueries);
-
-static std::pair<IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem>,
- std::vector<std::string>>
-initVFSForByNameScanning(IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS,
- ArrayRef<std::string> CommandLine,
- StringRef WorkingDirectory) {
- // Reset what might have been modified in the previous worker invocation.
- BaseFS->setCurrentWorkingDirectory(WorkingDirectory);
-
- // If we're scanning based on a module name alone, we don't expect the client
- // to provide us with an input file. However, the driver really wants to have
- // one. Let's just make it up to make the driver happy.
- auto OverlayFS =
- llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(BaseFS);
- auto InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
- InMemoryFS->setCurrentWorkingDirectory(WorkingDirectory);
- StringRef FakeInputPath("module-include.input");
- InMemoryFS->addFile(FakeInputPath, 0,
- llvm::MemoryBuffer::getMemBuffer(FakeInput));
- IntrusiveRefCntPtr<llvm::vfs::FileSystem> InMemoryOverlay = InMemoryFS;
- OverlayFS->pushOverlay(InMemoryOverlay);
-
- std::vector<std::string> ModifiedCommandLine(CommandLine);
- ModifiedCommandLine.emplace_back(FakeInputPath);
-
- return std::make_pair(OverlayFS, ModifiedCommandLine);
-}
-
std::optional<TranslationUnitDeps>
DependencyScanningTool::getTranslationUnitDependencies(
ArrayRef<std::string> CommandLine, StringRef CWD,
@@ -379,8 +320,8 @@ std::optional<CompilerInstanceWithContext>
CompilerInstanceWithContext::initializeFromCommandline(
DependencyScanningTool &Tool, StringRef CWD,
ArrayRef<std::string> CommandLine, DiagnosticConsumer &DC) {
- auto [OverlayFS, ModifiedCommandLine] =
- initVFSForByNameScanning(&Tool.Worker.getVFS(), CommandLine, CWD);
+ auto [OverlayFS, ModifiedCommandLine] = initVFSForByNameScanning(
+ &Tool.Worker.getVFS(), CommandLine, CWD, "ScanningByName");
auto DiagEngineWithCmdAndOpts =
std::make_unique<DiagnosticsEngineWithDiagOpts>(ModifiedCommandLine,
OverlayFS, DC);
@@ -505,9 +446,6 @@ bool CompilerInstanceWithContext::initialize(
bool CompilerInstanceWithContext::computeDependencies(
StringRef ModuleName, DependencyConsumer &Consumer,
DependencyActionController &Controller) {
- if (SrcLocOffset >= MaxNumOfQueries)
- llvm::report_fatal_error("exceeded maximum by-name scans for worker");
-
assert(CIPtr && "CIPtr must be initialized before calling this method");
auto &CI = *CIPtr;
diff --git a/clang/test/ClangScanDeps/modules-full-by-mult-mod-names-diagnostics.c b/clang/test/ClangScanDeps/modules-full-by-mult-mod-names-diagnostics.c
index f99690c3a7715..9889982354e90 100644
--- a/clang/test/ClangScanDeps/modules-full-by-mult-mod-names-diagnostics.c
+++ b/clang/test/ClangScanDeps/modules-full-by-mult-mod-names-diagnostics.c
@@ -27,11 +27,11 @@ module root2 { header "root2.h" }
// RUN: cat %t/result.json | sed 's:\\\\\?:/:g' | FileCheck -DPREFIX=%/t %s
// ERROR: Error while scanning dependencies for modA:
-// ERROR-NEXT: module-include.input:1:1: fatal error: module 'modA' not found
+// ERROR-NEXT: {{.*}}: fatal error: module 'modA' not found
// ERROR-NEXT: Error while scanning dependencies for modB:
-// ERROR-NEXT: module-include.input:1:3: fatal error: module 'modB' not found
+// ERROR-NEXT: {{.*}}: fatal error: module 'modB' not found
// ERROR-NEXT: Error while scanning dependencies for modC:
-// ERROR-NEXT: module-include.input:1:4: fatal error: module 'modC' not found
+// ERROR-NEXT: {{.*}}: fatal error: module 'modC' not found
// CHECK: {
// CHECK-NEXT: "modules": [
// CHECK-NEXT: {
``````````
</details>
https://github.com/llvm/llvm-project/pull/185150
More information about the cfe-commits
mailing list