[clang] [clang] Clear `NeedsCleaning` flag after `ExpandBuiltinMacro` (PR #133574)

[marius doerner via cfe-commits]

https://github.com/mariusdr created https://github.com/llvm/llvm-project/pull/133574

After builtin macro expansion in `Preprocessor::ExpandBuiltinMacro` the result token may have the `Token::NeedsCleaning` flag set which causes an assertion failure later on when the lexer retrieves the spelling of the token in `getSpellingSlow`.

This commit moves the `Tok.clearFlag(Token::NeedsCleaning)` call to the end of `ExpandBuiltinMacro`.

Closes #128384 

>From ae2279bb0fa44ff290a3305b90598caf60284066 Mon Sep 17 00:00:00 2001
From: marius doerner <mariusdr at users.noreply.github.com>
Date: Sat, 29 Mar 2025 09:21:20 +0100
Subject: [PATCH] [clang] Clear `NeedsCleaning` flag after `ExpandBuiltinMacro`

After builtin macro expansion in `Preprocessor::ExpandBuiltinMacro`
the result token may have the `Token::NeedsCleaning` flag set
which causes an assertion failure later on when the lexer retrives
the spelling of the token in `getSpellingSlow` in Lexer.cpp.
---
 clang/lib/Lex/PPMacroExpansion.cpp          |  2 +-
 clang/test/Preprocessor/embed___has_embed.c | 19 ++++++++
 clang/test/Preprocessor/has_attribute.c     | 20 +++++++++
 clang/test/Preprocessor/has_attribute.cpp   | 20 +++++++++
 clang/test/Preprocessor/has_c_attribute.c   | 20 +++++++++
 clang/test/Preprocessor/has_include.c       | 49 +++++++++++++++++++++
 6 files changed, 129 insertions(+), 1 deletion(-)

diff --git a/clang/lib/Lex/PPMacroExpansion.cpp b/clang/lib/Lex/PPMacroExpansion.cpp
index 8e35d56d3f1a6..107d5c31e039d 100644
--- a/clang/lib/Lex/PPMacroExpansion.cpp
+++ b/clang/lib/Lex/PPMacroExpansion.cpp
@@ -1646,7 +1646,6 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) {
 
   // Set up the return result.
   Tok.setIdentifierInfo(nullptr);
-  Tok.clearFlag(Token::NeedsCleaning);
   bool IsAtStartOfLine = Tok.isAtStartOfLine();
   bool HasLeadingSpace = Tok.hasLeadingSpace();
 
@@ -2089,6 +2088,7 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) {
   CreateString(OS.str(), Tok, Tok.getLocation(), Tok.getLocation());
   Tok.setFlagValue(Token::StartOfLine, IsAtStartOfLine);
   Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace);
+  Tok.clearFlag(Token::NeedsCleaning);
 }
 
 void Preprocessor::markMacroAsUsed(MacroInfo *MI) {
diff --git a/clang/test/Preprocessor/embed___has_embed.c b/clang/test/Preprocessor/embed___has_embed.c
index 43a3068b5f53a..2705b5ef6fd5b 100644
--- a/clang/test/Preprocessor/embed___has_embed.c
+++ b/clang/test/Preprocessor/embed___has_embed.c
@@ -58,3 +58,22 @@ unsigned char buffer[] = {
 #else
 #error 17
 #endif
+
+#if __has_embed(__FILE__\
+)
+#else
+#error 18
+#endif
+
+#define F __FI\
+LE__
+#if __has_embed(F)
+#else
+#error 19
+#endif
+
+#if __has_embed(F\
+)
+#else 
+#error 20
+#endif
diff --git a/clang/test/Preprocessor/has_attribute.c b/clang/test/Preprocessor/has_attribute.c
index 0ba664a53e649..6f6f519bd299d 100644
--- a/clang/test/Preprocessor/has_attribute.c
+++ b/clang/test/Preprocessor/has_attribute.c
@@ -68,3 +68,23 @@ int has_no_volatile_attribute();
 int has_fallthrough;
 #endif
 // CHECK: int has_fallthrough;
+
+#if __has_attribute(F\
+)
+int has_fallthrough_2;
+#endif
+// CHECK: int has_fallthrough_2;
+
+#define F_2 fall\
+through
+
+#if __has_attribute(F_2)
+int has_fallthrough_3;
+#endif
+// CHECK: int has_fallthrough_3;
+
+#if __has_attribute(F_2\
+)
+int has_fallthrough_4;
+#endif
+// CHECK: int has_fallthrough_4;
diff --git a/clang/test/Preprocessor/has_attribute.cpp b/clang/test/Preprocessor/has_attribute.cpp
index 00ec57615c84b..72af6de27e8bb 100644
--- a/clang/test/Preprocessor/has_attribute.cpp
+++ b/clang/test/Preprocessor/has_attribute.cpp
@@ -116,6 +116,26 @@ int funclike_1;
 int funclike_2;
 #endif
 // CHECK: int funclike_2;
+
+#if __has_cpp_attribute(CF\
+)
+int has_clang_falthrough_5;
+#endif
+// CHECK: int has_clang_falthrough_5;
+
+#define CF_2 clang::\
+fallthrough
+
+#if __has_cpp_attribute(CF_2)
+int has_clang_falthrough_6;
+#endif
+// CHECK: int has_clang_falthrough_6;
+
+#if __has_cpp_attribute(CF_2\
+)
+int has_clang_falthrough_7;
+#endif
+// CHECK: int has_clang_falthrough_7;
 }
 
 // Test for Microsoft __declspec attributes
diff --git a/clang/test/Preprocessor/has_c_attribute.c b/clang/test/Preprocessor/has_c_attribute.c
index 3332571d758c8..d8be13f5898a9 100644
--- a/clang/test/Preprocessor/has_c_attribute.c
+++ b/clang/test/Preprocessor/has_c_attribute.c
@@ -84,3 +84,23 @@ int funclike_1;
 int funclike_2;
 #endif
 // CHECK: int funclike_2;
+
+#if __has_c_attribute(CL\
+)
+int has_clang_likely_5;
+#endif
+// CHECK: int has_clang_likely_5;
+
+#define CL_2 clang::\
+likely
+
+#if __has_c_attribute(CL_2)
+int has_clang_likely_6;
+#endif
+// CHECK: int has_clang_likely_6;
+
+#if __has_c_attribute(CL_2\
+)
+int has_clang_likely_7;
+#endif
+// CHECK: int has_clang_likely_7;
diff --git a/clang/test/Preprocessor/has_include.c b/clang/test/Preprocessor/has_include.c
index c95025d83860a..ff199bf23063f 100644
--- a/clang/test/Preprocessor/has_include.c
+++ b/clang/test/Preprocessor/has_include.c
@@ -197,3 +197,52 @@ __has_include
 #ifdef FOO
 #elif __has_include(<foo>)
 #endif
+
+#if __has_include(<stdint.h>\
+)
+#else
+  #error "__has_include failed (10)."
+#endif
+
+#define MACRO6 <stdint.h>
+#if __has_include(MACRO6\
+)
+#else
+  #error "__has_include failed (11)."
+#endif
+
+#if __has_include_next(<stdint.h>/*expected-warning {{#include_next in primary source file}}*/\
+)
+#else
+  #error "__has_include_next failed (9)."
+#endif
+
+#if __has_include_next(MACRO6/*expected-warning {{#include_next in primary source file}}*/\
+) 
+#else
+  #error "__has_include_next failed (10)."
+#endif
+
+#define MACRO7 <std\
+int.h>
+#if __has_include(MACRO7)
+#else
+  #error "__has_include failed (12)."
+#endif
+
+#if __has_include(MACRO7\
+)
+#else
+  #error "__has_include failed (13)."
+#endif
+
+#if __has_include_next(MACRO7) //expected-warning {{#include_next in primary source file}}
+#else
+  #error "__has_include_next failed (11)."
+#endif
+
+#if __has_include_next(MACRO7/*expected-warning {{#include_next in primary source file}}*/\
+) 
+#else
+  #error "__has_include_next failed (12)."
+#endif