[clang] [clang-tools-extra] [clangd] fix preprocessor caching-lexer state tracking (PR #203716)
Yihan Wang via cfe-commits
cfe-commits at lists.llvm.org
Tue Jun 23 06:59:34 PDT 2026
https://github.com/yronglin updated https://github.com/llvm/llvm-project/pull/203716
>From d2a6eaa165128c9cc32447dcbfd36413c758d35b Mon Sep 17 00:00:00 2001
From: yronglin <yronglin777 at gmail.com>
Date: Sat, 13 Jun 2026 09:56:56 -0700
Subject: [PATCH 1/6] [clangd] Fix clangd crash when code completion EOF is
reached while skipping a function body
Signed-off-by: yronglin <yronglin777 at gmail.com>
---
.../clangd/unittests/CodeCompleteTests.cpp | 17 +++++++++++++++++
clang/lib/Parse/ParseStmt.cpp | 14 +++++---------
2 files changed, 22 insertions(+), 9 deletions(-)
diff --git a/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp b/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp
index 5808b2145965f..dc94af4dddf8c 100644
--- a/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp
+++ b/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp
@@ -1918,6 +1918,23 @@ TEST(SignatureHelpTest, StalePreamble) {
EXPECT_EQ(0, Results.activeParameter);
}
+TEST(SignatureHelpTest, EOFInSkippedFunctionBody) {
+ Annotations Test(R"cpp(
+#ifdef IS_HEADER
+void frameSizeBlocksWarning() {
+ auto fn = []() {
+ };
+ fn();
+}
+#else
+#define IS_HEADER
+#include __FILE__
+#^endif
+)cpp");
+ auto Results = signatures(Test.code(), Test.point());
+ EXPECT_THAT(Results.signatures, IsEmpty());
+}
+
class IndexRequestCollector : public SymbolIndex {
public:
IndexRequestCollector(std::vector<Symbol> Syms = {}) : Symbols(Syms) {}
diff --git a/clang/lib/Parse/ParseStmt.cpp b/clang/lib/Parse/ParseStmt.cpp
index 37f142e059930..e77d6066847b2 100644
--- a/clang/lib/Parse/ParseStmt.cpp
+++ b/clang/lib/Parse/ParseStmt.cpp
@@ -2496,8 +2496,10 @@ bool Parser::trySkippingFunctionBody() {
// We're in code-completion mode. Skip parsing for all function bodies unless
// the body contains the code-completion point.
+ if (Tok.is(tok::kw_try))
+ return false;
+
TentativeParsingAction PA(*this);
- bool IsTryCatch = Tok.is(tok::kw_try);
CachedTokens Toks;
bool ErrorInPrologue = ConsumeAndStoreFunctionPrologue(Toks);
if (llvm::any_of(Toks, [](const Token &Tok) {
@@ -2511,18 +2513,12 @@ bool Parser::trySkippingFunctionBody() {
SkipMalformedDecl();
return true;
}
- if (!SkipUntil(tok::r_brace, StopAtCodeCompletion)) {
+ if (!SkipUntil(tok::r_brace, StopAtCodeCompletion | StopBeforeMatch)) {
PA.Revert();
return false;
}
- while (IsTryCatch && Tok.is(tok::kw_catch)) {
- if (!SkipUntil(tok::l_brace, StopAtCodeCompletion) ||
- !SkipUntil(tok::r_brace, StopAtCodeCompletion)) {
- PA.Revert();
- return false;
- }
- }
PA.Commit();
+ ConsumeBrace();
return true;
}
>From 3aa2ba5f83ec50223d41bc7bede88d4719e3809c Mon Sep 17 00:00:00 2001
From: yronglin <yronglin777 at gmail.com>
Date: Mon, 22 Jun 2026 16:57:31 +0800
Subject: [PATCH 2/6] Add explicit flag for caching lex mode
Signed-off-by: yronglin <yronglin777 at gmail.com>
---
clang/include/clang/Lex/Preprocessor.h | 16 +++++++++-------
clang/lib/Lex/PPCaching.cpp | 4 +++-
clang/lib/Lex/PPLexerChange.cpp | 6 +++++-
clang/lib/Lex/Preprocessor.cpp | 6 ++++--
clang/lib/Parse/ParseStmt.cpp | 13 +++++++++----
5 files changed, 30 insertions(+), 15 deletions(-)
diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h
index 8b684e85eb1c1..4d47976ec536f 100644
--- a/clang/include/clang/Lex/Preprocessor.h
+++ b/clang/include/clang/Lex/Preprocessor.h
@@ -1190,6 +1190,9 @@ class Preprocessor {
/// Lex() should be invoked.
CachedTokensTy::size_type CachedLexPos = 0;
+ /// True when the caching lexer is installed as the active lexer layer.
+ bool IsCachingLexMode = false;
+
/// Stack of backtrack positions, allowing nested backtracks.
///
/// The EnableBacktrackAtThisPos() method pushes a position to
@@ -2583,7 +2586,7 @@ class Preprocessor {
friend void TokenLexer::ExpandFunctionArguments();
void PushIncludeMacroStack() {
- assert(CurLexerCallback != CLK_CachingLexer &&
+ assert(!IsCachingLexMode && CurLexerCallback != CLK_CachingLexer &&
"cannot push a caching lexer");
IncludeMacroStack.emplace_back(CurLexerCallback, CurLexerSubmodule,
std::move(CurLexer), CurPPLexer,
@@ -2592,6 +2595,7 @@ class Preprocessor {
}
void PopIncludeMacroStack() {
+ IsCachingLexMode = false;
if (CurLexer)
PendingDestroyLexers.push_back(std::move(CurLexer));
CurLexer = std::move(IncludeMacroStack.back().TheLexer);
@@ -2819,18 +2823,16 @@ class Preprocessor {
// Caching stuff.
void CachingLex(Token &Result);
- bool InCachingLexMode() const {
- // If the Lexer pointers are 0 and IncludeMacroStack is empty, it means
- // that we are past EOF, not that we are in CachingLex mode.
- return !CurPPLexer && !CurTokenLexer && !IncludeMacroStack.empty();
- }
+ bool InCachingLexMode() const { return IsCachingLexMode; }
void EnterCachingLexMode();
void EnterCachingLexModeUnchecked();
void ExitCachingLexMode() {
- if (InCachingLexMode())
+ if (InCachingLexMode()) {
+ IsCachingLexMode = false;
RemoveTopOfLexerStack();
+ }
}
const Token &PeekAhead(unsigned N);
diff --git a/clang/lib/Lex/PPCaching.cpp b/clang/lib/Lex/PPCaching.cpp
index cbacda9d31ae2..d64183a35122a 100644
--- a/clang/lib/Lex/PPCaching.cpp
+++ b/clang/lib/Lex/PPCaching.cpp
@@ -126,8 +126,10 @@ void Preprocessor::EnterCachingLexMode() {
}
void Preprocessor::EnterCachingLexModeUnchecked() {
- assert(CurLexerCallback != CLK_CachingLexer && "already in caching lex mode");
+ assert(!InCachingLexMode() && CurLexerCallback != CLK_CachingLexer &&
+ "already in caching lex mode");
PushIncludeMacroStack();
+ IsCachingLexMode = true;
CurLexerCallback = CLK_CachingLexer;
}
diff --git a/clang/lib/Lex/PPLexerChange.cpp b/clang/lib/Lex/PPLexerChange.cpp
index 98ff9a9a04e7c..d901d02a44f7a 100644
--- a/clang/lib/Lex/PPLexerChange.cpp
+++ b/clang/lib/Lex/PPLexerChange.cpp
@@ -106,6 +106,9 @@ bool Preprocessor::EnterSourceFile(FileID FID, ConstSearchDirIterator CurDir,
/// and start lexing tokens from it instead of the current buffer.
void Preprocessor::EnterSourceFileWithLexer(std::unique_ptr<Lexer> TheLexer,
ConstSearchDirIterator CurDir) {
+ if (InCachingLexMode())
+ ExitCachingLexMode();
+
PreprocessorLexer *PrevPPLexer = CurPPLexer;
// Add the current lexer to the include stack.
@@ -172,7 +175,8 @@ void Preprocessor::EnterMacro(Token &Tok, SourceLocation ILEnd,
void Preprocessor::EnterTokenStream(const Token *Toks, unsigned NumToks,
bool DisableMacroExpansion, bool OwnsTokens,
bool IsReinject) {
- if (CurLexerCallback == CLK_CachingLexer) {
+ if (InCachingLexMode()) {
+ assert(CurLexerCallback == CLK_CachingLexer && "Unexpected lexer kind");
if (CachedLexPos < CachedTokens.size()) {
assert(IsReinject && "new tokens in the middle of cached stream");
// We're entering tokens into the middle of our cached token stream. We
diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp
index 1e21b4a94cea3..c69d084d6514f 100644
--- a/clang/lib/Lex/Preprocessor.cpp
+++ b/clang/lib/Lex/Preprocessor.cpp
@@ -417,14 +417,16 @@ StringRef Preprocessor::getLastMacroWithSpelling(
}
void Preprocessor::recomputeCurLexerKind() {
- if (CurLexer)
+ if (InCachingLexMode())
+ CurLexerCallback = CLK_CachingLexer;
+ else if (CurLexer)
CurLexerCallback = CurLexer->isDependencyDirectivesLexer()
? CLK_DependencyDirectivesLexer
: CLK_Lexer;
else if (CurTokenLexer)
CurLexerCallback = CLK_TokenLexer;
else
- CurLexerCallback = CLK_CachingLexer;
+ CurLexerCallback = CLK_Lexer;
}
bool Preprocessor::SetCodeCompletionPoint(FileEntryRef File,
diff --git a/clang/lib/Parse/ParseStmt.cpp b/clang/lib/Parse/ParseStmt.cpp
index e77d6066847b2..e8dd987414249 100644
--- a/clang/lib/Parse/ParseStmt.cpp
+++ b/clang/lib/Parse/ParseStmt.cpp
@@ -2496,10 +2496,9 @@ bool Parser::trySkippingFunctionBody() {
// We're in code-completion mode. Skip parsing for all function bodies unless
// the body contains the code-completion point.
- if (Tok.is(tok::kw_try))
- return false;
TentativeParsingAction PA(*this);
+ bool IsTryCatch = Tok.is(tok::kw_try);
CachedTokens Toks;
bool ErrorInPrologue = ConsumeAndStoreFunctionPrologue(Toks);
if (llvm::any_of(Toks, [](const Token &Tok) {
@@ -2513,12 +2512,18 @@ bool Parser::trySkippingFunctionBody() {
SkipMalformedDecl();
return true;
}
- if (!SkipUntil(tok::r_brace, StopAtCodeCompletion | StopBeforeMatch)) {
+ if (!SkipUntil(tok::r_brace, StopAtCodeCompletion)) {
PA.Revert();
return false;
}
+ while (IsTryCatch && Tok.is(tok::kw_catch)) {
+ if (!SkipUntil(tok::l_brace, StopAtCodeCompletion) ||
+ !SkipUntil(tok::r_brace, StopAtCodeCompletion)) {
+ PA.Revert();
+ return false;
+ }
+ }
PA.Commit();
- ConsumeBrace();
return true;
}
>From 234dac06b8aa8d413f0a7379a096a95e39211011 Mon Sep 17 00:00:00 2001
From: Yihan Wang <yronglin777 at gmail.com>
Date: Mon, 22 Jun 2026 22:51:12 +0800
Subject: [PATCH 3/6] Revert blank changes.
---
clang/lib/Parse/ParseStmt.cpp | 1 -
1 file changed, 1 deletion(-)
diff --git a/clang/lib/Parse/ParseStmt.cpp b/clang/lib/Parse/ParseStmt.cpp
index e8dd987414249..37f142e059930 100644
--- a/clang/lib/Parse/ParseStmt.cpp
+++ b/clang/lib/Parse/ParseStmt.cpp
@@ -2496,7 +2496,6 @@ bool Parser::trySkippingFunctionBody() {
// We're in code-completion mode. Skip parsing for all function bodies unless
// the body contains the code-completion point.
-
TentativeParsingAction PA(*this);
bool IsTryCatch = Tok.is(tok::kw_try);
CachedTokens Toks;
>From bc341ea416dfbe7bf45a0df6a833e2594c8f136b Mon Sep 17 00:00:00 2001
From: yronglin <yronglin777 at gmail.com>
Date: Tue, 23 Jun 2026 21:40:54 +0800
Subject: [PATCH 4/6] Remove IsCachingLexMode flag
Signed-off-by: yronglin <yronglin777 at gmail.com>
---
clang/include/clang/Lex/Preprocessor.h | 12 +++---------
clang/lib/Lex/PPCaching.cpp | 4 +---
clang/lib/Lex/PPLexerChange.cpp | 3 ---
3 files changed, 4 insertions(+), 15 deletions(-)
diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h
index 4d47976ec536f..fc66bd745a618 100644
--- a/clang/include/clang/Lex/Preprocessor.h
+++ b/clang/include/clang/Lex/Preprocessor.h
@@ -1190,9 +1190,6 @@ class Preprocessor {
/// Lex() should be invoked.
CachedTokensTy::size_type CachedLexPos = 0;
- /// True when the caching lexer is installed as the active lexer layer.
- bool IsCachingLexMode = false;
-
/// Stack of backtrack positions, allowing nested backtracks.
///
/// The EnableBacktrackAtThisPos() method pushes a position to
@@ -2586,7 +2583,7 @@ class Preprocessor {
friend void TokenLexer::ExpandFunctionArguments();
void PushIncludeMacroStack() {
- assert(!IsCachingLexMode && CurLexerCallback != CLK_CachingLexer &&
+ assert(CurLexerCallback != CLK_CachingLexer &&
"cannot push a caching lexer");
IncludeMacroStack.emplace_back(CurLexerCallback, CurLexerSubmodule,
std::move(CurLexer), CurPPLexer,
@@ -2595,7 +2592,6 @@ class Preprocessor {
}
void PopIncludeMacroStack() {
- IsCachingLexMode = false;
if (CurLexer)
PendingDestroyLexers.push_back(std::move(CurLexer));
CurLexer = std::move(IncludeMacroStack.back().TheLexer);
@@ -2823,16 +2819,14 @@ class Preprocessor {
// Caching stuff.
void CachingLex(Token &Result);
- bool InCachingLexMode() const { return IsCachingLexMode; }
+ bool InCachingLexMode() const { return CurLexerCallback == CLK_CachingLexer; }
void EnterCachingLexMode();
void EnterCachingLexModeUnchecked();
void ExitCachingLexMode() {
- if (InCachingLexMode()) {
- IsCachingLexMode = false;
+ if (InCachingLexMode())
RemoveTopOfLexerStack();
- }
}
const Token &PeekAhead(unsigned N);
diff --git a/clang/lib/Lex/PPCaching.cpp b/clang/lib/Lex/PPCaching.cpp
index d64183a35122a..844ac2d57599c 100644
--- a/clang/lib/Lex/PPCaching.cpp
+++ b/clang/lib/Lex/PPCaching.cpp
@@ -126,10 +126,8 @@ void Preprocessor::EnterCachingLexMode() {
}
void Preprocessor::EnterCachingLexModeUnchecked() {
- assert(!InCachingLexMode() && CurLexerCallback != CLK_CachingLexer &&
- "already in caching lex mode");
+ assert(!InCachingLexMode() && "already in caching lex mode");
PushIncludeMacroStack();
- IsCachingLexMode = true;
CurLexerCallback = CLK_CachingLexer;
}
diff --git a/clang/lib/Lex/PPLexerChange.cpp b/clang/lib/Lex/PPLexerChange.cpp
index d901d02a44f7a..a81a4758bcc24 100644
--- a/clang/lib/Lex/PPLexerChange.cpp
+++ b/clang/lib/Lex/PPLexerChange.cpp
@@ -106,9 +106,6 @@ bool Preprocessor::EnterSourceFile(FileID FID, ConstSearchDirIterator CurDir,
/// and start lexing tokens from it instead of the current buffer.
void Preprocessor::EnterSourceFileWithLexer(std::unique_ptr<Lexer> TheLexer,
ConstSearchDirIterator CurDir) {
- if (InCachingLexMode())
- ExitCachingLexMode();
-
PreprocessorLexer *PrevPPLexer = CurPPLexer;
// Add the current lexer to the include stack.
>From f376ea970ad8c532a4317cc82d7d853c7771b97c Mon Sep 17 00:00:00 2001
From: yronglin <yronglin777 at gmail.com>
Date: Tue, 23 Jun 2026 21:43:20 +0800
Subject: [PATCH 5/6] Remove unecessary assert
Signed-off-by: yronglin <yronglin777 at gmail.com>
---
clang/lib/Lex/PPCaching.cpp | 4 +---
clang/lib/Lex/PPLexerChange.cpp | 1 -
2 files changed, 1 insertion(+), 4 deletions(-)
diff --git a/clang/lib/Lex/PPCaching.cpp b/clang/lib/Lex/PPCaching.cpp
index 844ac2d57599c..3f0ebd8455685 100644
--- a/clang/lib/Lex/PPCaching.cpp
+++ b/clang/lib/Lex/PPCaching.cpp
@@ -117,10 +117,8 @@ void Preprocessor::EnterCachingLexMode() {
assert(LexLevel == 0 &&
"entered caching lex mode while lexing something else");
- if (InCachingLexMode()) {
- assert(CurLexerCallback == CLK_CachingLexer && "Unexpected lexer kind");
+ if (InCachingLexMode())
return;
- }
EnterCachingLexModeUnchecked();
}
diff --git a/clang/lib/Lex/PPLexerChange.cpp b/clang/lib/Lex/PPLexerChange.cpp
index a81a4758bcc24..b44bf2cd3c253 100644
--- a/clang/lib/Lex/PPLexerChange.cpp
+++ b/clang/lib/Lex/PPLexerChange.cpp
@@ -173,7 +173,6 @@ void Preprocessor::EnterTokenStream(const Token *Toks, unsigned NumToks,
bool DisableMacroExpansion, bool OwnsTokens,
bool IsReinject) {
if (InCachingLexMode()) {
- assert(CurLexerCallback == CLK_CachingLexer && "Unexpected lexer kind");
if (CachedLexPos < CachedTokens.size()) {
assert(IsReinject && "new tokens in the middle of cached stream");
// We're entering tokens into the middle of our cached token stream. We
>From 607da62462b5f3e4ee9b8d3a3d40a4e6cdc0f9a2 Mon Sep 17 00:00:00 2001
From: yronglin <yronglin777 at gmail.com>
Date: Tue, 23 Jun 2026 21:59:18 +0800
Subject: [PATCH 6/6] Add release notes
Signed-off-by: yronglin <yronglin777 at gmail.com>
---
clang/docs/ReleaseNotes.rst | 3 +++
1 file changed, 3 insertions(+)
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 37428df0974f4..1cc4ad3b18161 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -721,6 +721,9 @@ Bug Fixes in This Version
- Fixed a potential stack-use-after-return issue in Clang when copy-initializing
an array via an element-at-a-time copy loop (#GH192026)
- Fixed an issue where certain designated initializers would be rejected for constexpr variables. (#GH193373)
+- Fixed ``clang::Preprocessor::recomputeCurLexerKind`` to avoid default fallback to ``CurLexerCallback = CLK_CachingLexer;``. This prevents code-completion
+ EOF handling from accidentally restoring CLK_CachingLexer while a tentative parse is still active, which could trigger a caching lexer re-entry assertion
+ in clangd signature help. (#GH200677)
- Fixed a crash when ``#embed`` is used with C++ modules (#GH195350)
- Fixed a bug where ``-x cuda`` caused clang to immediately resolve templates that should not be. (#GH200545)
- Fixed an issue where ``__typeof_unqual`` and ``__typeof_unqual__`` were rejected as a declaration specifier in block scope in C++.
More information about the cfe-commits
mailing list