[clang] fcab930 - [clang][deps] Handle response files in dep scanner
Ben Langmuir via cfe-commits
cfe-commits at lists.llvm.org
Mon Mar 13 15:48:58 PDT 2023
Author: Ben Langmuir
Date: 2023-03-13T15:47:35-07:00
New Revision: fcab930cd32e6d977816fa7094a0de0bc0d7dd1f
URL: https://github.com/llvm/llvm-project/commit/fcab930cd32e6d977816fa7094a0de0bc0d7dd1f
DIFF: https://github.com/llvm/llvm-project/commit/fcab930cd32e6d977816fa7094a0de0bc0d7dd1f.diff
LOG: [clang][deps] Handle response files in dep scanner
Extract the code the driver uses to expand response files and reuse it
in the dependency scanner.
rdar://106155880
Differential Revision: https://reviews.llvm.org/D145838
Added:
clang/test/ClangScanDeps/response-file.c
Modified:
clang/include/clang/Basic/DiagnosticDriverKinds.td
clang/include/clang/Driver/Driver.h
clang/lib/Driver/Driver.cpp
clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
clang/tools/driver/driver.cpp
Removed:
################################################################################
diff --git a/clang/include/clang/Basic/DiagnosticDriverKinds.td b/clang/include/clang/Basic/DiagnosticDriverKinds.td
index 3b32227d21acd..2c6bb0aa9331a 100644
--- a/clang/include/clang/Basic/DiagnosticDriverKinds.td
+++ b/clang/include/clang/Basic/DiagnosticDriverKinds.td
@@ -716,4 +716,7 @@ def err_drv_riscv_unsupported_with_linker_relaxation : Error<
def err_drv_loongarch_invalid_mfpu_EQ : Error<
"invalid argument '%0' to -mfpu=; must be one of: 64, 32, none, 0 (alias for none)">;
+
+def err_drv_expand_response_file : Error<
+ "failed to expand response file: %0">;
}
diff --git a/clang/include/clang/Driver/Driver.h b/clang/include/clang/Driver/Driver.h
index 691deb7a90eeb..c85658d7c56e3 100644
--- a/clang/include/clang/Driver/Driver.h
+++ b/clang/include/clang/Driver/Driver.h
@@ -808,6 +808,16 @@ llvm::StringRef getDriverMode(StringRef ProgName, ArrayRef<const char *> Args);
/// Checks whether the value produced by getDriverMode is for CL mode.
bool IsClangCL(StringRef DriverMode);
+/// Expand response files from a clang driver or cc1 invocation.
+///
+/// \param Args The arguments that will be expanded.
+/// \param ClangCLMode Whether clang is in CL mode.
+/// \param Alloc Allocator for new arguments.
+/// \param FS Filesystem to use when expanding files.
+llvm::Error expandResponseFiles(SmallVectorImpl<const char *> &Args,
+ bool ClangCLMode, llvm::BumpPtrAllocator &Alloc,
+ llvm::vfs::FileSystem *FS = nullptr);
+
} // end namespace driver
} // end namespace clang
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index b991a1822812c..6533d4e49cbbe 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -6434,3 +6434,58 @@ llvm::StringRef clang::driver::getDriverMode(StringRef ProgName,
}
bool driver::IsClangCL(StringRef DriverMode) { return DriverMode.equals("cl"); }
+
+llvm::Error driver::expandResponseFiles(SmallVectorImpl<const char *> &Args,
+ bool ClangCLMode,
+ llvm::BumpPtrAllocator &Alloc,
+ llvm::vfs::FileSystem *FS) {
+ // Parse response files using the GNU syntax, unless we're in CL mode. There
+ // are two ways to put clang in CL compatibility mode: ProgName is either
+ // clang-cl or cl, or --driver-mode=cl is on the command line. The normal
+ // command line parsing can't happen until after response file parsing, so we
+ // have to manually search for a --driver-mode=cl argument the hard way.
+ // Finally, our -cc1 tools don't care which tokenization mode we use because
+ // response files written by clang will tokenize the same way in either mode.
+ enum { Default, POSIX, Windows } RSPQuoting = Default;
+ for (const char *F : Args) {
+ if (strcmp(F, "--rsp-quoting=posix") == 0)
+ RSPQuoting = POSIX;
+ else if (strcmp(F, "--rsp-quoting=windows") == 0)
+ RSPQuoting = Windows;
+ }
+
+ // Determines whether we want nullptr markers in Args to indicate response
+ // files end-of-lines. We only use this for the /LINK driver argument with
+ // clang-cl.exe on Windows.
+ bool MarkEOLs = ClangCLMode;
+
+ llvm::cl::TokenizerCallback Tokenizer;
+ if (RSPQuoting == Windows || (RSPQuoting == Default && ClangCLMode))
+ Tokenizer = &llvm::cl::TokenizeWindowsCommandLine;
+ else
+ Tokenizer = &llvm::cl::TokenizeGNUCommandLine;
+
+ if (MarkEOLs && Args.size() > 1 && StringRef(Args[1]).startswith("-cc1"))
+ MarkEOLs = false;
+
+ llvm::cl::ExpansionContext ECtx(Alloc, Tokenizer);
+ ECtx.setMarkEOLs(MarkEOLs);
+ if (FS)
+ ECtx.setVFS(FS);
+
+ if (llvm::Error Err = ECtx.expandResponseFiles(Args))
+ return Err;
+
+ // If -cc1 came from a response file, remove the EOL sentinels.
+ auto FirstArg = llvm::find_if(llvm::drop_begin(Args),
+ [](const char *A) { return A != nullptr; });
+ if (FirstArg != Args.end() && StringRef(*FirstArg).startswith("-cc1")) {
+ // If -cc1 came from a response file, remove the EOL sentinels.
+ if (MarkEOLs) {
+ auto newEnd = std::remove(Args.begin(), Args.end(), nullptr);
+ Args.resize(newEnd - Args.begin());
+ }
+ }
+
+ return llvm::Error::success();
+}
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
index 10a26ab05ddf9..5869418b98dfe 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
@@ -7,6 +7,7 @@
//===----------------------------------------------------------------------===//
#include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h"
+#include "clang/Basic/DiagnosticDriver.h"
#include "clang/Basic/DiagnosticFrontend.h"
#include "clang/CodeGen/ObjectFilePCHContainerOperations.h"
#include "clang/Driver/Compilation.h"
@@ -22,6 +23,8 @@
#include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
#include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
#include "clang/Tooling/Tooling.h"
+#include "llvm/Support/Allocator.h"
+#include "llvm/Support/Error.h"
#include "llvm/TargetParser/Host.h"
#include <optional>
@@ -366,16 +369,29 @@ llvm::Error DependencyScanningWorker::computeDependencies(
}
static bool forEachDriverJob(
- ArrayRef<std::string> Args, DiagnosticsEngine &Diags, FileManager &FM,
+ ArrayRef<std::string> ArgStrs, DiagnosticsEngine &Diags, FileManager &FM,
llvm::function_ref<bool(const driver::Command &Cmd)> Callback) {
+ SmallVector<const char *, 256> Argv;
+ Argv.reserve(ArgStrs.size());
+ for (const std::string &Arg : ArgStrs)
+ Argv.push_back(Arg.c_str());
+
+ llvm::vfs::FileSystem *FS = &FM.getVirtualFileSystem();
+
std::unique_ptr<driver::Driver> Driver = std::make_unique<driver::Driver>(
- Args[0], llvm::sys::getDefaultTargetTriple(), Diags,
- "clang LLVM compiler", &FM.getVirtualFileSystem());
+ Argv[0], llvm::sys::getDefaultTargetTriple(), Diags,
+ "clang LLVM compiler", FS);
Driver->setTitle("clang_based_tool");
- std::vector<const char *> Argv;
- for (const std::string &Arg : Args)
- Argv.push_back(Arg.c_str());
+ llvm::BumpPtrAllocator Alloc;
+ bool CLMode = driver::IsClangCL(
+ driver::getDriverMode(Argv[0], ArrayRef(Argv).slice(1)));
+
+ if (llvm::Error E = driver::expandResponseFiles(Argv, CLMode, Alloc, FS)) {
+ Diags.Report(diag::err_drv_expand_response_file)
+ << llvm::toString(std::move(E));
+ return false;
+ }
const std::unique_ptr<driver::Compilation> Compilation(
Driver->BuildCompilation(llvm::ArrayRef(Argv)));
diff --git a/clang/test/ClangScanDeps/response-file.c b/clang/test/ClangScanDeps/response-file.c
new file mode 100644
index 0000000000000..c08105c127202
--- /dev/null
+++ b/clang/test/ClangScanDeps/response-file.c
@@ -0,0 +1,41 @@
+// Check that the scanner can handle a response file input.
+
+// RUN: rm -rf %t
+// RUN: split-file %s %t
+// RUN: sed -e "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json
+
+// RUN: clang-scan-deps -format experimental-full -compilation-database %t/cdb.json > %t/deps.json
+
+// RUN: cat %t/deps.json | sed 's:\\\\\?:/:g' | FileCheck -DPREFIX=%/t %s
+
+// CHECK: "command-line": [
+// CHECK: "-fsyntax-only"
+// CHECK: "-x"
+// CHECK-NEXT: "c"
+// CHECK: "tu.c"
+// CHECK: "-I"
+// CHECK-NEXT: "include"
+// CHECK: ],
+// CHECK: "file-deps": [
+// CHECK-NEXT: "[[PREFIX]]/tu.c"
+// CHECK-NEXT: "[[PREFIX]]/include/header.h"
+// CHECK-NEXT: ]
+
+//--- cdb.json.template
+[{
+ "file": "DIR/t.c",
+ "directory": "DIR",
+ "command": "clang @DIR/args.txt"
+}]
+
+//--- args.txt
+ at args_nested.txt
+-fsyntax-only tu.c
+
+//--- args_nested.txt
+-I include
+
+//--- include/header.h
+
+//--- tu.c
+#include "header.h"
diff --git a/clang/tools/driver/driver.cpp b/clang/tools/driver/driver.cpp
index 3ec2bcc0bd753..a05e0956a4796 100644
--- a/clang/tools/driver/driver.cpp
+++ b/clang/tools/driver/driver.cpp
@@ -391,55 +391,17 @@ int clang_main(int Argc, char **Argv, const llvm::ToolContext &ToolContext) {
const char *ProgName =
ToolContext.NeedsPrependArg ? ToolContext.PrependArg : ToolContext.Path;
- // Parse response files using the GNU syntax, unless we're in CL mode. There
- // are two ways to put clang in CL compatibility mode: ProgName is either
- // clang-cl or cl, or --driver-mode=cl is on the command line. The normal
- // command line parsing can't happen until after response file parsing, so we
- // have to manually search for a --driver-mode=cl argument the hard way.
- // Finally, our -cc1 tools don't care which tokenization mode we use because
- // response files written by clang will tokenize the same way in either mode.
bool ClangCLMode =
IsClangCL(getDriverMode(ProgName, llvm::ArrayRef(Args).slice(1)));
- enum { Default, POSIX, Windows } RSPQuoting = Default;
- for (const char *F : Args) {
- if (strcmp(F, "--rsp-quoting=posix") == 0)
- RSPQuoting = POSIX;
- else if (strcmp(F, "--rsp-quoting=windows") == 0)
- RSPQuoting = Windows;
- }
- // Determines whether we want nullptr markers in Args to indicate response
- // files end-of-lines. We only use this for the /LINK driver argument with
- // clang-cl.exe on Windows.
- bool MarkEOLs = ClangCLMode;
-
- llvm::cl::TokenizerCallback Tokenizer;
- if (RSPQuoting == Windows || (RSPQuoting == Default && ClangCLMode))
- Tokenizer = &llvm::cl::TokenizeWindowsCommandLine;
- else
- Tokenizer = &llvm::cl::TokenizeGNUCommandLine;
-
- if (MarkEOLs && Args.size() > 1 && StringRef(Args[1]).startswith("-cc1"))
- MarkEOLs = false;
- llvm::cl::ExpansionContext ECtx(A, Tokenizer);
- ECtx.setMarkEOLs(MarkEOLs);
- if (llvm::Error Err = ECtx.expandResponseFiles(Args)) {
+ if (llvm::Error Err = expandResponseFiles(Args, ClangCLMode, A)) {
llvm::errs() << toString(std::move(Err)) << '\n';
return 1;
}
- // Handle -cc1 integrated tools, even if -cc1 was expanded from a response
- // file.
- auto FirstArg = llvm::find_if(llvm::drop_begin(Args),
- [](const char *A) { return A != nullptr; });
- if (FirstArg != Args.end() && StringRef(*FirstArg).startswith("-cc1")) {
- // If -cc1 came from a response file, remove the EOL sentinels.
- if (MarkEOLs) {
- auto newEnd = std::remove(Args.begin(), Args.end(), nullptr);
- Args.resize(newEnd - Args.begin());
- }
+ // Handle -cc1 integrated tools.
+ if (Args.size() >= 2 && StringRef(Args[1]).startswith("-cc1"))
return ExecuteCC1Tool(Args, ToolContext);
- }
// Handle options that need handling before the real command line parsing in
// Driver::BuildCompilation()
More information about the cfe-commits
mailing list