[llvm] 9d37d0e - [Support] Expand `<CFGDIR>` as the base directory in configuration files.
Jack Andersen via llvm-commits
llvm-commits at lists.llvm.org
Thu Dec 30 10:45:12 PST 2021
Author: Jack Andersen
Date: 2021-12-30T13:43:47-05:00
New Revision: 9d37d0ea34858288faf6351b9bdc0a0b91107c82
URL: https://github.com/llvm/llvm-project/commit/9d37d0ea34858288faf6351b9bdc0a0b91107c82
DIFF: https://github.com/llvm/llvm-project/commit/9d37d0ea34858288faf6351b9bdc0a0b91107c82.diff
LOG: [Support] Expand `<CFGDIR>` as the base directory in configuration files.
Extends response file expansion to recognize `<CFGDIR>` and expand to the
current file's directory. This makes it much easier to author clang config
files rooted in portable, potentially not-installed SDK directories.
A typical use case may be something like the following:
```
# sample_sdk.cfg
--target=sample
-isystem <CFGDIR>/include
-L <CFGDIR>/lib
-T <CFGDIR>/ldscripts/link.ld
```
Reviewed By: sepavloff
Differential Revision: https://reviews.llvm.org/D115604
Added:
Modified:
clang/docs/ReleaseNotes.rst
clang/docs/UsersManual.rst
clang/lib/Tooling/ExpandResponseFilesCompilationDatabase.cpp
llvm/include/llvm/Support/CommandLine.h
llvm/lib/Support/CommandLine.cpp
llvm/unittests/Support/CommandLineTest.cpp
Removed:
################################################################################
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 44485fcd7e26f..ce9b3547155af 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -63,6 +63,9 @@ Non-comprehensive list of changes in this release
- Maximum _ExtInt size was decreased from 16,777,215 bits to 8,388,608 bits.
Motivation for this was discussed in PR51829.
+- Configuration file syntax extended with ``<CFGDIR>`` token. This expands to
+ the base path of the current config file. See :ref:`configuration-files` for
+ details.
New Compiler Flags
------------------
diff --git a/clang/docs/UsersManual.rst b/clang/docs/UsersManual.rst
index 26da5a0ff2554..1173fd337841c 100644
--- a/clang/docs/UsersManual.rst
+++ b/clang/docs/UsersManual.rst
@@ -843,6 +843,8 @@ a special character, which is the convention used by GNU Make. The -MV
option tells Clang to put double-quotes around the entire filename, which
is the convention used by NMake and Jom.
+.. _configuration-files:
+
Configuration files
-------------------
@@ -917,6 +919,22 @@ relative to the including file. For example, if a configuration file
`~/.llvm/target.cfg` contains the directive `@os/linux.opts`, the file
`linux.opts` is searched for in the directory `~/.llvm/os`.
+To generate paths relative to the configuration file, the `<CFGDIR>` token may
+be used. This will expand to the absolute path of the directory containing the
+configuration file.
+
+In cases where a configuration file is deployed alongside SDK contents, the
+SDK directory can remain fully portable by using `<CFGDIR>` prefixed paths.
+In this way, the user may only need to specify a root configuration file with
+`--config` to establish every aspect of the SDK with the compiler:
+
+::
+
+ --target=foo
+ -isystem <CFGDIR>/include
+ -L <CFGDIR>/lib
+ -T <CFGDIR>/ldscripts/link.ld
+
Language and Target-Independent Features
========================================
diff --git a/clang/lib/Tooling/ExpandResponseFilesCompilationDatabase.cpp b/clang/lib/Tooling/ExpandResponseFilesCompilationDatabase.cpp
index 29787b8a88942..75d0d50d851f9 100644
--- a/clang/lib/Tooling/ExpandResponseFilesCompilationDatabase.cpp
+++ b/clang/lib/Tooling/ExpandResponseFilesCompilationDatabase.cpp
@@ -61,7 +61,7 @@ class ExpandResponseFilesDatabase : public CompilationDatabase {
continue;
llvm::BumpPtrAllocator Alloc;
llvm::StringSaver Saver(Alloc);
- llvm::cl::ExpandResponseFiles(Saver, Tokenizer, Argv, false, false,
+ llvm::cl::ExpandResponseFiles(Saver, Tokenizer, Argv, false, false, false,
llvm::StringRef(Cmd.Directory), *FS);
// Don't assign directly, Argv aliases CommandLine.
std::vector<std::string> ExpandedArgv(Argv.begin(), Argv.end());
diff --git a/llvm/include/llvm/Support/CommandLine.h b/llvm/include/llvm/Support/CommandLine.h
index 2c3edd858a3fb..120ab18409158 100644
--- a/llvm/include/llvm/Support/CommandLine.h
+++ b/llvm/include/llvm/Support/CommandLine.h
@@ -2082,7 +2082,8 @@ void tokenizeConfigFile(StringRef Source, StringSaver &Saver,
///
/// It reads content of the specified file, tokenizes it and expands "@file"
/// commands resolving file names in them relative to the directory where
-/// CfgFilename resides.
+/// CfgFilename resides. It also expands "<CFGDIR>" to the base path of the
+/// current config file.
///
bool readConfigFile(StringRef CfgFileName, StringSaver &Saver,
SmallVectorImpl<const char *> &Argv);
@@ -2102,13 +2103,15 @@ bool readConfigFile(StringRef CfgFileName, StringSaver &Saver,
/// with nullptrs in the Argv vector.
/// \param [in] RelativeNames true if names of nested response files must be
/// resolved relative to including file.
+/// \param [in] ExpandBasePath If true, "<CFGDIR>" expands to the base path of
+/// the current response file.
/// \param [in] FS File system used for all file access when running the tool.
/// \param [in] CurrentDir Path used to resolve relative rsp files. If set to
/// None, process' cwd is used instead.
/// \return true if all @files were expanded successfully or there were none.
bool ExpandResponseFiles(StringSaver &Saver, TokenizerCallback Tokenizer,
SmallVectorImpl<const char *> &Argv, bool MarkEOLs,
- bool RelativeNames,
+ bool RelativeNames, bool ExpandBasePath,
llvm::Optional<llvm::StringRef> CurrentDir,
llvm::vfs::FileSystem &FS);
@@ -2117,7 +2120,7 @@ bool ExpandResponseFiles(StringSaver &Saver, TokenizerCallback Tokenizer,
bool ExpandResponseFiles(
StringSaver &Saver, TokenizerCallback Tokenizer,
SmallVectorImpl<const char *> &Argv, bool MarkEOLs = false,
- bool RelativeNames = false,
+ bool RelativeNames = false, bool ExpandBasePath = false,
llvm::Optional<llvm::StringRef> CurrentDir = llvm::None);
/// A convenience helper which concatenates the options specified by the
diff --git a/llvm/lib/Support/CommandLine.cpp b/llvm/lib/Support/CommandLine.cpp
index 4153a69abf5d5..481ba56c4077b 100644
--- a/llvm/lib/Support/CommandLine.cpp
+++ b/llvm/lib/Support/CommandLine.cpp
@@ -1078,11 +1078,45 @@ static bool hasUTF8ByteOrderMark(ArrayRef<char> S) {
return (S.size() >= 3 && S[0] == '\xef' && S[1] == '\xbb' && S[2] == '\xbf');
}
+// Substitute <CFGDIR> with the file's base path.
+static void ExpandBasePaths(StringRef BasePath, StringSaver &Saver,
+ const char *&Arg) {
+ assert(sys::path::is_absolute(BasePath));
+ constexpr StringLiteral Token("<CFGDIR>");
+ const StringRef ArgString(Arg);
+
+ SmallString<128> ResponseFile;
+ StringRef::size_type StartPos = 0;
+ for (StringRef::size_type TokenPos = ArgString.find(Token);
+ TokenPos != StringRef::npos;
+ TokenPos = ArgString.find(Token, StartPos)) {
+ // Token may appear more than once per arg (e.g. comma-separated linker
+ // args). Support by using path-append on any subsequent appearances.
+ const StringRef LHS = ArgString.substr(StartPos, TokenPos - StartPos);
+ if (ResponseFile.empty())
+ ResponseFile = LHS;
+ else
+ llvm::sys::path::append(ResponseFile, LHS);
+ ResponseFile.append(BasePath);
+ StartPos = TokenPos + Token.size();
+ }
+
+ if (!ResponseFile.empty()) {
+ // Path-append the remaining arg substring if at least one token appeared.
+ const StringRef Remaining = ArgString.substr(StartPos);
+ if (!Remaining.empty())
+ llvm::sys::path::append(ResponseFile, Remaining);
+ Arg = Saver.save(ResponseFile.str()).data();
+ }
+}
+
// FName must be an absolute path.
-static llvm::Error ExpandResponseFile(
- StringRef FName, StringSaver &Saver, TokenizerCallback Tokenizer,
- SmallVectorImpl<const char *> &NewArgv, bool MarkEOLs, bool RelativeNames,
- llvm::vfs::FileSystem &FS) {
+static llvm::Error ExpandResponseFile(StringRef FName, StringSaver &Saver,
+ TokenizerCallback Tokenizer,
+ SmallVectorImpl<const char *> &NewArgv,
+ bool MarkEOLs, bool RelativeNames,
+ bool ExpandBasePath,
+ llvm::vfs::FileSystem &FS) {
assert(sys::path::is_absolute(FName));
llvm::ErrorOr<std::unique_ptr<MemoryBuffer>> MemBufOrErr =
FS.getBufferForFile(FName);
@@ -1116,8 +1150,15 @@ static llvm::Error ExpandResponseFile(
// file, replace the included response file names with their full paths
// obtained by required resolution.
for (auto &Arg : NewArgv) {
+ if (!Arg)
+ continue;
+
+ // Substitute <CFGDIR> with the file's base path.
+ if (ExpandBasePath)
+ ExpandBasePaths(BasePath, Saver, Arg);
+
// Skip non-rsp file arguments.
- if (!Arg || Arg[0] != '@')
+ if (Arg[0] != '@')
continue;
StringRef FileName(Arg + 1);
@@ -1129,7 +1170,7 @@ static llvm::Error ExpandResponseFile(
ResponseFile.push_back('@');
ResponseFile.append(BasePath);
llvm::sys::path::append(ResponseFile, FileName);
- Arg = Saver.save(ResponseFile.c_str()).data();
+ Arg = Saver.save(ResponseFile.str()).data();
}
return Error::success();
}
@@ -1138,7 +1179,7 @@ static llvm::Error ExpandResponseFile(
/// StringSaver and tokenization strategy.
bool cl::ExpandResponseFiles(StringSaver &Saver, TokenizerCallback Tokenizer,
SmallVectorImpl<const char *> &Argv, bool MarkEOLs,
- bool RelativeNames,
+ bool RelativeNames, bool ExpandBasePath,
llvm::Optional<llvm::StringRef> CurrentDir,
llvm::vfs::FileSystem &FS) {
bool AllExpanded = true;
@@ -1218,7 +1259,7 @@ bool cl::ExpandResponseFiles(StringSaver &Saver, TokenizerCallback Tokenizer,
SmallVector<const char *, 0> ExpandedArgv;
if (llvm::Error Err =
ExpandResponseFile(FName, Saver, Tokenizer, ExpandedArgv, MarkEOLs,
- RelativeNames, FS)) {
+ RelativeNames, ExpandBasePath, FS)) {
// We couldn't read this file, so we leave it in the argument stream and
// move on.
// TODO: The error should be propagated up the stack.
@@ -1250,11 +1291,11 @@ bool cl::ExpandResponseFiles(StringSaver &Saver, TokenizerCallback Tokenizer,
bool cl::ExpandResponseFiles(StringSaver &Saver, TokenizerCallback Tokenizer,
SmallVectorImpl<const char *> &Argv, bool MarkEOLs,
- bool RelativeNames,
+ bool RelativeNames, bool ExpandBasePath,
llvm::Optional<StringRef> CurrentDir) {
return ExpandResponseFiles(Saver, std::move(Tokenizer), Argv, MarkEOLs,
- RelativeNames, std::move(CurrentDir),
- *vfs::getRealFileSystem());
+ RelativeNames, ExpandBasePath,
+ std::move(CurrentDir), *vfs::getRealFileSystem());
}
bool cl::expandResponseFiles(int Argc, const char *const *Argv,
@@ -1281,16 +1322,17 @@ bool cl::readConfigFile(StringRef CfgFile, StringSaver &Saver,
llvm::sys::path::append(AbsPath, CfgFile);
CfgFile = AbsPath.str();
}
- if (llvm::Error Err =
- ExpandResponseFile(CfgFile, Saver, cl::tokenizeConfigFile, Argv,
- /*MarkEOLs=*/false, /*RelativeNames=*/true,
- *llvm::vfs::getRealFileSystem())) {
+ if (llvm::Error Err = ExpandResponseFile(
+ CfgFile, Saver, cl::tokenizeConfigFile, Argv,
+ /*MarkEOLs=*/false, /*RelativeNames=*/true, /*ExpandBasePath=*/true,
+ *llvm::vfs::getRealFileSystem())) {
// TODO: The error should be propagated up the stack.
llvm::consumeError(std::move(Err));
return false;
}
return ExpandResponseFiles(Saver, cl::tokenizeConfigFile, Argv,
- /*MarkEOLs=*/false, /*RelativeNames=*/true);
+ /*MarkEOLs=*/false, /*RelativeNames=*/true,
+ /*ExpandBasePath=*/true, llvm::None);
}
static void initCommonOptions();
diff --git a/llvm/unittests/Support/CommandLineTest.cpp b/llvm/unittests/Support/CommandLineTest.cpp
index db7255e5569a4..4e1160fe2dbc5 100644
--- a/llvm/unittests/Support/CommandLineTest.cpp
+++ b/llvm/unittests/Support/CommandLineTest.cpp
@@ -827,7 +827,7 @@ TEST(CommandLineTest, ResponseFiles) {
llvm::BumpPtrAllocator A;
llvm::StringSaver Saver(A);
ASSERT_TRUE(llvm::cl::ExpandResponseFiles(
- Saver, llvm::cl::TokenizeGNUCommandLine, Argv, false, true,
+ Saver, llvm::cl::TokenizeGNUCommandLine, Argv, false, true, false,
/*CurrentDir=*/StringRef(TestRoot), FS));
EXPECT_THAT(Argv, testing::Pointwise(
StringEquality(),
@@ -889,9 +889,9 @@ TEST(CommandLineTest, RecursiveResponseFiles) {
#else
cl::TokenizerCallback Tokenizer = cl::TokenizeGNUCommandLine;
#endif
- ASSERT_FALSE(cl::ExpandResponseFiles(Saver, Tokenizer, Argv, false, false,
- /*CurrentDir=*/llvm::StringRef(TestRoot),
- FS));
+ ASSERT_FALSE(
+ cl::ExpandResponseFiles(Saver, Tokenizer, Argv, false, false, false,
+ /*CurrentDir=*/llvm::StringRef(TestRoot), FS));
EXPECT_THAT(Argv,
testing::Pointwise(StringEquality(),
@@ -929,7 +929,7 @@ TEST(CommandLineTest, ResponseFilesAtArguments) {
BumpPtrAllocator A;
StringSaver Saver(A);
ASSERT_FALSE(cl::ExpandResponseFiles(Saver, cl::TokenizeGNUCommandLine, Argv,
- false, false,
+ false, false, false,
/*CurrentDir=*/StringRef(TestRoot), FS));
// ASSERT instead of EXPECT to prevent potential out-of-bounds access.
@@ -964,7 +964,7 @@ TEST(CommandLineTest, ResponseFileRelativePath) {
BumpPtrAllocator A;
StringSaver Saver(A);
ASSERT_TRUE(cl::ExpandResponseFiles(Saver, cl::TokenizeGNUCommandLine, Argv,
- false, true,
+ false, true, false,
/*CurrentDir=*/StringRef(TestRoot), FS));
EXPECT_THAT(Argv,
testing::Pointwise(StringEquality(), {"test/test", "-flag"}));
@@ -984,7 +984,7 @@ TEST(CommandLineTest, ResponseFileEOLs) {
BumpPtrAllocator A;
StringSaver Saver(A);
ASSERT_TRUE(cl::ExpandResponseFiles(Saver, cl::TokenizeWindowsCommandLine,
- Argv, true, true,
+ Argv, true, true, false,
/*CurrentDir=*/StringRef(TestRoot), FS));
const char *Expected[] = {"clang", "-Xclang", "-Wno-whatever", nullptr,
"input.cpp"};
@@ -1038,25 +1038,39 @@ TEST(CommandLineTest, ReadConfigFile) {
llvm::SmallVector<const char *, 1> Argv;
TempDir TestDir("unittest", /*Unique*/ true);
+ TempDir TestSubDir(TestDir.path("subdir"), /*Unique*/ false);
- llvm::SmallString<128> TestCfg;
- llvm::sys::path::append(TestCfg, TestDir.path(), "foo");
-
+ llvm::SmallString<128> TestCfg = TestDir.path("foo");
TempFile ConfigFile(TestCfg, "",
"# Comment\n"
"-option_1\n"
+ "-option_2=<CFGDIR>/dir1\n"
+ "-option_3=<CFGDIR>\n"
+ "-option_4 <CFGDIR>\n"
+ "-option_5=<CFG\\\n"
+ "DIR>\n"
+ "-option_6=<CFGDIR>/dir1,<CFGDIR>/dir2\n"
"@subconfig\n"
- "-option_3=abcd\n"
- "-option_4=\\\n"
+ "-option_11=abcd\n"
+ "-option_12=\\\n"
"cdef\n");
- llvm::SmallString<128> TestCfg2;
- llvm::sys::path::append(TestCfg2, TestDir.path(), "subconfig");
+ llvm::SmallString<128> TestCfg2 = TestDir.path("subconfig");
TempFile ConfigFile2(TestCfg2, "",
- "-option_2\n"
+ "-option_7\n"
+ "-option_8=<CFGDIR>/dir2\n"
+ "@subdir/subfoo\n"
"\n"
" # comment\n");
+ llvm::SmallString<128> TestCfg3 = TestSubDir.path("subfoo");
+ TempFile ConfigFile3(TestCfg3, "",
+ "-option_9=<CFGDIR>/dir3\n"
+ "@<CFGDIR>/subfoo2\n");
+
+ llvm::SmallString<128> TestCfg4 = TestSubDir.path("subfoo2");
+ TempFile ConfigFile4(TestCfg4, "", "-option_10\n");
+
// Make sure the current directory is not the directory where config files
// resides. In this case the code that expands response files will not find
// 'subconfig' unless it resolves nested inclusions relative to the including
@@ -1071,11 +1085,26 @@ TEST(CommandLineTest, ReadConfigFile) {
bool Result = llvm::cl::readConfigFile(ConfigFile.path(), Saver, Argv);
EXPECT_TRUE(Result);
- EXPECT_EQ(Argv.size(), 4U);
+ EXPECT_EQ(Argv.size(), 13U);
EXPECT_STREQ(Argv[0], "-option_1");
- EXPECT_STREQ(Argv[1], "-option_2");
- EXPECT_STREQ(Argv[2], "-option_3=abcd");
- EXPECT_STREQ(Argv[3], "-option_4=cdef");
+ EXPECT_STREQ(Argv[1],
+ ("-option_2=" + TestDir.path() + "/dir1").str().c_str());
+ EXPECT_STREQ(Argv[2], ("-option_3=" + TestDir.path()).str().c_str());
+ EXPECT_STREQ(Argv[3], "-option_4");
+ EXPECT_STREQ(Argv[4], TestDir.path().str().c_str());
+ EXPECT_STREQ(Argv[5], ("-option_5=" + TestDir.path()).str().c_str());
+ EXPECT_STREQ(Argv[6], ("-option_6=" + TestDir.path() + "/dir1," +
+ TestDir.path() + "/dir2")
+ .str()
+ .c_str());
+ EXPECT_STREQ(Argv[7], "-option_7");
+ EXPECT_STREQ(Argv[8],
+ ("-option_8=" + TestDir.path() + "/dir2").str().c_str());
+ EXPECT_STREQ(Argv[9],
+ ("-option_9=" + TestSubDir.path() + "/dir3").str().c_str());
+ EXPECT_STREQ(Argv[10], "-option_10");
+ EXPECT_STREQ(Argv[11], "-option_11=abcd");
+ EXPECT_STREQ(Argv[12], "-option_12=cdef");
}
TEST(CommandLineTest, PositionalEatArgsError) {
More information about the llvm-commits
mailing list