[clang] [clang][Lex] Preserve leading-space flags when retaining whitespace (PR #236408)
via cfe-commits
cfe-commits at lists.llvm.org
Wed Oct 28 06:05:04 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-clang
Author: Chen Miao (ChenMiaoi)
<details>
<summary>Changes</summary>
When returning whitespace tokens, the lexer does not propagate leading-space information to the following token. Preserve this information in whitespace retention mode, accounting for whether the whitespace ends with a newline.
Avoid inserting an extra space in traditional preprocessor output when the previous token already supplies whitespace.
---
Full diff: https://github.com/llvm/llvm-project/pull/236408.diff
4 Files Affected:
- (modified) clang/lib/Frontend/PrintPreprocessedOutput.cpp (+13-2)
- (modified) clang/lib/Lex/Lexer.cpp (+2-2)
- (modified) clang/test/Preprocessor/traditional-cpp.c (+8)
- (modified) clang/unittests/Lex/LexerTest.cpp (+46)
``````````diff
diff --git a/clang/lib/Frontend/PrintPreprocessedOutput.cpp b/clang/lib/Frontend/PrintPreprocessedOutput.cpp
index ab5ac8a4a69c6d..00c122ed7a56c9 100644
--- a/clang/lib/Frontend/PrintPreprocessedOutput.cpp
+++ b/clang/lib/Frontend/PrintPreprocessedOutput.cpp
@@ -792,6 +792,17 @@ void PrintPPOutputPPCallbacks::HandleWhitespaceBeforeTok(const Token &Tok,
!Tok.is(tok::annot_embed) && !Tok.is(tok::annot_module_name)))
return;
+ bool HasLeadingSpace = Tok.hasLeadingSpace();
+ if (HasLeadingSpace && PP.getLangOpts().TraditionalCPP &&
+ PrevTok.is(tok::unknown) && PrevTok.getLength()) {
+ // Traditional preprocessing emits whitespace as tokens. Do not add another
+ // space if the previous token already supplied it.
+ SmallString<128> SpellingBuffer;
+ StringRef Spelling = PP.getSpelling(PrevTok, SpellingBuffer);
+ if (!Spelling.empty() && isWhitespace(Spelling.back()))
+ HasLeadingSpace = false;
+ }
+
// EmittedDirectiveOnThisLine takes priority over RequireSameLine.
if ((!RequireSameLine || EmittedDirectiveOnThisLine) &&
MoveToLine(Tok, /*RequireStartOfLine=*/EmittedDirectiveOnThisLine)) {
@@ -808,7 +819,7 @@ void PrintPPOutputPPCallbacks::HandleWhitespaceBeforeTok(const Token &Tok,
// expect leading white space, if a macro expansion in column 1 starts
// with an empty macro argument, or an empty nested macro expansion. In
// this case, move the token to column 2.
- if (ColNo == 1 && Tok.hasLeadingSpace())
+ if (ColNo == 1 && HasLeadingSpace)
ColNo = 2;
// This hack prevents stuff like:
@@ -831,7 +842,7 @@ void PrintPPOutputPPCallbacks::HandleWhitespaceBeforeTok(const Token &Tok,
// whitespace-minimization mode
// - The whitespace is necessary to keep the tokens apart and there is not
// already a newline between them
- if (RequireSpace || (!MinimizeWhitespace && Tok.hasLeadingSpace()) ||
+ if (RequireSpace || (!MinimizeWhitespace && HasLeadingSpace) ||
((EmittedTokensOnThisLine || EmittedDirectiveOnThisLine) &&
AvoidConcat(PrevPrevTok, PrevTok, Tok)))
*OS << ' ';
diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp
index b659780bac593a..8a3676d94503f7 100644
--- a/clang/lib/Lex/Lexer.cpp
+++ b/clang/lib/Lex/Lexer.cpp
@@ -2663,7 +2663,7 @@ bool Lexer::SkipWhitespace(Token &Result, const char *CurPtr) {
IsAtStartOfLine = true;
IsAtPhysicalStartOfLine = true;
}
- // FIXME: The next token will not have LeadingSpace set.
+ HasLeadingSpace = !isVerticalWhitespace(CurPtr[-1]);
return true;
}
@@ -3875,7 +3875,7 @@ bool Lexer::LexTokenInternal(Token &Result) {
// whitespace.
if (isKeepWhitespaceMode()) {
FormTokenWithChars(Result, CurPtr, tok::unknown);
- // FIXME: The next token will not have LeadingSpace set.
+ HasLeadingSpace = true;
return true;
}
diff --git a/clang/test/Preprocessor/traditional-cpp.c b/clang/test/Preprocessor/traditional-cpp.c
index f311be0bb59360..23fddb1cdc4147 100644
--- a/clang/test/Preprocessor/traditional-cpp.c
+++ b/clang/test/Preprocessor/traditional-cpp.c
@@ -29,6 +29,14 @@ foo // bar
indented!
tab separated values
+/* CHECK: {{^}}a b{{$}}
+ * CHECK: {{^}}a b{{$}}
+ * CHECK: {{^}} indented_with_spaces{{$}}
+ */
+a b
+a b
+ indented_with_spaces
+
#define bracket(x) >>>x<<<
bracket(| spaces |)
/* CHECK: {{^}}>>>| spaces |<<<{{$}}
diff --git a/clang/unittests/Lex/LexerTest.cpp b/clang/unittests/Lex/LexerTest.cpp
index 8f632ccc5b9315..8ca1abbe826c90 100644
--- a/clang/unittests/Lex/LexerTest.cpp
+++ b/clang/unittests/Lex/LexerTest.cpp
@@ -719,6 +719,52 @@ TEST_F(LexerTest, CreatedFIDCountForPredefinedBuffer) {
1U);
}
+TEST_F(LexerTest, LeadingSpaceWithWhitespaceRetention) {
+ const struct {
+ const char *Whitespace;
+ bool LeadingSpace;
+ bool StartOfLine;
+ } Cases[] = {
+ {"", false, false}, {" ", true, false},
+ {"\t", true, false}, {" \t\f\v", true, false},
+ {"\n", false, true}, {"\r", false, true},
+ {"\r\n", false, true}, {"\n ", true, true},
+ {"\r\n\t", true, true}, {" \n", false, true},
+ {" \n \n", false, true}, {" \n \n\t", true, true},
+ };
+
+ for (const auto &Case : Cases) {
+ for (bool KeepWhitespace : {false, true}) {
+ SCOPED_TRACE(Case.Whitespace);
+ SCOPED_TRACE(KeepWhitespace);
+ auto Buffer = llvm::MemoryBuffer::getMemBufferCopy(
+ std::string("a+") + Case.Whitespace + "b+c");
+ Lexer L(SourceLocation(), LangOpts, Buffer->getBufferStart(),
+ Buffer->getBufferStart(), Buffer->getBufferEnd());
+ L.SetKeepWhitespaceMode(KeepWhitespace);
+
+ for (unsigned I = 0; I != 5; ++I) {
+ Token T;
+ do {
+ L.LexFromRawLexer(T);
+ if (T.is(tok::unknown)) {
+ ASSERT_TRUE(KeepWhitespace);
+ ASSERT_GT(T.getLength(), 0u);
+ }
+ } while (T.is(tok::unknown));
+
+ ASSERT_EQ(T.getKind(),
+ I == 1 || I == 3 ? tok::plus : tok::raw_identifier);
+ EXPECT_EQ(T.hasLeadingSpace(), I == 2 && Case.LeadingSpace);
+ EXPECT_EQ(T.isAtStartOfLine(), I == 0 || (I == 2 && Case.StartOfLine));
+ }
+ Token T;
+ L.LexFromRawLexer(T);
+ EXPECT_TRUE(T.is(tok::eof));
+ }
+ }
+}
+
TEST_F(LexerTest, RawAndNormalLexSameForLineComments) {
const llvm::StringLiteral Source = R"cpp(
// First line comment.
``````````
</details>
https://github.com/llvm/llvm-project/pull/236408
More information about the cfe-commits
mailing list