[clang] [Clang] Detects invalid unicode characters in token formed by `##` (PR #208757)
Corentin Jabot via cfe-commits
cfe-commits at lists.llvm.org
Fri Jul 10 09:32:04 PDT 2026
https://github.com/cor3ntin updated https://github.com/llvm/llvm-project/pull/208757
>From ee0debec1f54e761bdb23c57be1ff7a42a29cc54 Mon Sep 17 00:00:00 2001
From: Corentin Jabot <corentinjabot at gmail.com>
Date: Fri, 10 Jul 2026 17:25:19 +0200
Subject: [PATCH 1/2] [Clang] Detects invalid unicode characters in token
formed by `##`
When parsing identifiers with invalid (non-xid_continue),
we would emit a diagnostics and continue, in an effort to
improve recovery.
However, in raw mode we would not emit a diagnostic, but still continue.
So identifiers formed in raw mode could be gibberish.
We fixed that by not-continuing on error in raw mode.
Fixes #101342
---
clang/docs/ReleaseNotes.md | 1 +
clang/lib/Lex/Lexer.cpp | 55 +++++++++++++--------
clang/test/Preprocessor/ucn-pp-identifier.c | 15 ++++++
3 files changed, 50 insertions(+), 21 deletions(-)
diff --git a/clang/docs/ReleaseNotes.md b/clang/docs/ReleaseNotes.md
index 96317a3eece97..0792ea012b02f 100644
--- a/clang/docs/ReleaseNotes.md
+++ b/clang/docs/ReleaseNotes.md
@@ -800,6 +800,7 @@ latest release, please see the [Clang Web Site](https://clang.llvm.org) or the
- Fixed crash when checking for overflow for unary operator that can't overflow (#GH170072)
- Clang no longer handles a `" q-char-sequence "` header name as a string literal (#GH132643).
- Under `-fdollars-in-identifiers`, the `$` can now appear in user-defined-literals. (#GH173985)
+- Clang now detects invalid unicode characters in identifiers formed by token concatenation. (#GH101342)
- Fixed an assertion where we improperly handled implicit conversions to integral types from an atomic-type with a conversion function. (#GH201770)
- Fixed assertion failures involving code completion with delayed default arguments and exception specifications. (#GH200879)
- Fixed a regression where calling a function that takes a class-type parameter by value inside `decltype` of a concept could be incorrectly rejected when used as a non-type template argument. (#GH175831)
diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp
index 32ad310ca2672..d5085ca6d4c8a 100644
--- a/clang/lib/Lex/Lexer.cpp
+++ b/clang/lib/Lex/Lexer.cpp
@@ -1809,11 +1809,13 @@ static void maybeDiagnoseUTF8Homoglyph(DiagnosticsEngine &Diags, uint32_t C,
}
}
-static void diagnoseInvalidUnicodeCodepointInIdentifier(
- DiagnosticsEngine &Diags, const LangOptions &LangOpts, uint32_t CodePoint,
- CharSourceRange Range, bool IsFirst) {
+static bool CheckCodepointValidInIdentifier(const Preprocessor *PP,
+ const LangOptions &LangOpts,
+ uint32_t CodePoint,
+ CharSourceRange Range, bool IsFirst,
+ bool Diagnose) {
if (isASCII(CodePoint))
- return;
+ return true;
bool IsExtension;
bool IsIDStart = isAllowedInitiallyIDChar(CodePoint, LangOpts, IsExtension);
@@ -1821,19 +1823,23 @@ static void diagnoseInvalidUnicodeCodepointInIdentifier(
IsIDStart || isAllowedIDChar(CodePoint, LangOpts, IsExtension);
if ((IsFirst && IsIDStart) || (!IsFirst && IsIDContinue))
- return;
+ return true;
+
+ if (!Diagnose)
+ return false;
bool InvalidOnlyAtStart = IsFirst && !IsIDStart && IsIDContinue;
if (!IsFirst || InvalidOnlyAtStart) {
- Diags.Report(Range.getBegin(), diag::err_character_not_allowed_identifier)
+ PP->Diag(Range.getBegin(), diag::err_character_not_allowed_identifier)
<< Range << codepointAsHexString(CodePoint) << int(InvalidOnlyAtStart)
<< FixItHint::CreateRemoval(Range);
} else {
- Diags.Report(Range.getBegin(), diag::err_character_not_allowed)
+ PP->Diag(Range.getBegin(), diag::err_character_not_allowed)
<< Range << codepointAsHexString(CodePoint)
<< FixItHint::CreateRemoval(Range);
}
+ return false;
}
bool Lexer::tryConsumeIdentifierUCN(const char *&CurPtr, unsigned Size,
@@ -1847,13 +1853,15 @@ bool Lexer::tryConsumeIdentifierUCN(const char *&CurPtr, unsigned Size,
if (!isAllowedIDChar(CodePoint, LangOpts, IsExtension)) {
if (isASCII(CodePoint) || isUnicodeWhitespace(CodePoint))
return false;
- if (!isLexingRawMode() && !ParsingPreprocessorDirective &&
- !PP->isPreprocessedOutput())
- diagnoseInvalidUnicodeCodepointInIdentifier(
- PP->getDiagnostics(), LangOpts, CodePoint,
- makeCharRange(*this, CurPtr, UCNPtr),
- /*IsFirst=*/false);
+ bool DiagnoseAndContinue = !isLexingRawMode() &&
+ !ParsingPreprocessorDirective &&
+ !PP->isPreprocessedOutput();
+ if (!CheckCodepointValidInIdentifier(
+ PP, LangOpts, CodePoint, makeCharRange(*this, CurPtr, UCNPtr),
+ /*IsFirst=*/false, DiagnoseAndContinue) &&
+ !DiagnoseAndContinue)
+ return false;
// We got a unicode codepoint that is neither a space nor a
// a valid identifier part.
// Carry on as if the codepoint was valid for recovery purposes.
@@ -1900,11 +1908,16 @@ bool Lexer::tryConsumeIdentifierUTF8Char(const char *&CurPtr, Token &Result) {
if (isASCII(CodePoint) || isUnicodeWhitespace(CodePoint))
return false;
- if (!isLexingRawMode() && !ParsingPreprocessorDirective &&
- !PP->isPreprocessedOutput())
- diagnoseInvalidUnicodeCodepointInIdentifier(
- PP->getDiagnostics(), LangOpts, CodePoint,
- makeCharRange(*this, CharStart, UnicodePtr), /*IsFirst=*/false);
+ bool DiagnoseAndContinue = !isLexingRawMode() &&
+ !ParsingPreprocessorDirective &&
+ !PP->isPreprocessedOutput();
+
+ if (!CheckCodepointValidInIdentifier(
+ PP, LangOpts, CodePoint,
+ makeCharRange(*this, CharStart, UnicodePtr), /*IsFirst=*/false,
+ DiagnoseAndContinue) &&
+ !DiagnoseAndContinue)
+ return false;
// We got a unicode codepoint that is neither a space nor a
// a valid identifier part. Carry on as if the codepoint was
// valid for recovery purposes.
@@ -1960,9 +1973,9 @@ bool Lexer::LexUnicodeIdentifierStart(Token &Result, uint32_t C,
// loophole in the mapping of Unicode characters to basic character set
// characters that allows us to map these particular characters to, say,
// whitespace.
- diagnoseInvalidUnicodeCodepointInIdentifier(
- PP->getDiagnostics(), LangOpts, C,
- makeCharRange(*this, BufferPtr, CurPtr), /*IsStart*/ true);
+ CheckCodepointValidInIdentifier(PP, LangOpts, C,
+ makeCharRange(*this, BufferPtr, CurPtr),
+ /*IsStart=*/true, /*Diagnose=*/true);
BufferPtr = CurPtr;
return false;
}
diff --git a/clang/test/Preprocessor/ucn-pp-identifier.c b/clang/test/Preprocessor/ucn-pp-identifier.c
index ee008a73eb882..0cb9965822984 100644
--- a/clang/test/Preprocessor/ucn-pp-identifier.c
+++ b/clang/test/Preprocessor/ucn-pp-identifier.c
@@ -162,3 +162,18 @@ int a\N{LATIN CAPITAL LETTER A WITH GRAVE??>;
// GH64161
int A\N{LEFT-TO-RIGHT OVERRIDE}; // expected-error {{character <U+202D> not allowed in an identifier}}
+
+
+// GH101342
+// We should not form identifiers by concatenating invalid UCNs.
+
+#define DOT •
+#define CONCAT_IMPL(Left, Separator, Right) Left##Separator##Right
+#define CONCAT2(Left, Separator, Right) CONCAT_IMPL(Left, Separator, Right)
+#define MAKE_CLASS_NAME(A, B) CONCAT2(A, DOT, B)
+
+// struct foo•bar {} x;
+struct MAKE_CLASS_NAME(foo, bar);
+// expected-error at -1 {{pasting formed 'foo•', an invalid preprocessing token}} \
+// expected-error at -1 {{pasting formed '•bar', an invalid preprocessing token}} \
+// expected-error at -1 {{expected unqualified-id}}
>From 4bba8f5b7b68d9d9c95633f7d59ea0168ea21467 Mon Sep 17 00:00:00 2001
From: Corentin Jabot <corentinjabot at gmail.com>
Date: Fri, 10 Jul 2026 18:31:42 +0200
Subject: [PATCH 2/2] tests
---
clang/test/Preprocessor/GH101342.c | 20 ++++++++++++++++++++
clang/test/Preprocessor/ucn-pp-identifier.c | 14 +-------------
2 files changed, 21 insertions(+), 13 deletions(-)
create mode 100644 clang/test/Preprocessor/GH101342.c
diff --git a/clang/test/Preprocessor/GH101342.c b/clang/test/Preprocessor/GH101342.c
new file mode 100644
index 0000000000000..5ff401bde9c8e
--- /dev/null
+++ b/clang/test/Preprocessor/GH101342.c
@@ -0,0 +1,20 @@
+// RUN: %clang_cc1 %s -fsyntax-only -pedantic -verify
+// RUN: %clang_cc1 %s -fsyntax-only -x c++ -pedantic -verify
+
+// GH101342
+// We should not form identifiers by concatenating invalid UCNs.
+
+#define DOT •
+#define CONCAT_IMPL(Left, Separator, Right) Left##Separator##Right
+#define CONCAT(Left, Separator, Right) CONCAT_IMPL(Left, Separator, Right)
+#define MAKE_CLASS_NAME(A, B) CONCAT(A, DOT, B)
+
+// struct foo•bar {} x;
+struct MAKE_CLASS_NAME(foo, bar);
+// expected-error at -1 {{pasting formed 'foo•', an invalid preprocessing token}} \
+// expected-error at -1 {{pasting formed '•bar', an invalid preprocessing token}}
+#ifdef __cplusplus
+// expected-error at -4 {{expected unqualified-id}}
+#else
+// expected-error at -6 {{expected identifier or '('}}
+#endif
diff --git a/clang/test/Preprocessor/ucn-pp-identifier.c b/clang/test/Preprocessor/ucn-pp-identifier.c
index 0cb9965822984..e702743d47a3c 100644
--- a/clang/test/Preprocessor/ucn-pp-identifier.c
+++ b/clang/test/Preprocessor/ucn-pp-identifier.c
@@ -164,16 +164,4 @@ int a\N{LATIN CAPITAL LETTER A WITH GRAVE??>;
int A\N{LEFT-TO-RIGHT OVERRIDE}; // expected-error {{character <U+202D> not allowed in an identifier}}
-// GH101342
-// We should not form identifiers by concatenating invalid UCNs.
-
-#define DOT •
-#define CONCAT_IMPL(Left, Separator, Right) Left##Separator##Right
-#define CONCAT2(Left, Separator, Right) CONCAT_IMPL(Left, Separator, Right)
-#define MAKE_CLASS_NAME(A, B) CONCAT2(A, DOT, B)
-
-// struct foo•bar {} x;
-struct MAKE_CLASS_NAME(foo, bar);
-// expected-error at -1 {{pasting formed 'foo•', an invalid preprocessing token}} \
-// expected-error at -1 {{pasting formed '•bar', an invalid preprocessing token}} \
-// expected-error at -1 {{expected unqualified-id}}
+
More information about the cfe-commits
mailing list