[Lldb-commits] [clang] [lldb] [lldb] Re-use clang's keyword enable/disable mechanism in CPlusPlusNameParser.cpp (PR #164284)
Daniel Sanders via lldb-commits
lldb-commits at lists.llvm.org
Tue Nov 4 18:54:07 PST 2025
https://github.com/dsandersllvm updated https://github.com/llvm/llvm-project/pull/164284
>From e1175e636933074c6c00f52f9c5287b4a4cf002c Mon Sep 17 00:00:00 2001
From: Daniel Sanders <daniel_l_sanders at apple.com>
Date: Fri, 17 Oct 2025 18:10:51 -0700
Subject: [PATCH 01/11] [lldb] Re-use clang's keyword enable/disable mechanism
in CPlusPlusNameParser.cpp
The problem is that CPlusPlusNameParser doesn't respect the language dialect
for `target variable Class::constant`. If `constant` is a keyword in some
(downstream in this case) dialects then it lexes `constant` as `kw_constant`
instead of `kw_raw_identifier` even if that dialect is not active. As a
result:
* `target variable constant` works
* `target variable Class::constant` fails to look up constant because it sees
`kw_raw_identifier, kw_coloncolon, kw_constant` instead of
`kw_raw_identifier, kw_coloncolon, kw_raw_identifier`
* `expr -l c++ -- Class::constant` works because clang is parsing it
Note: There's seemingly a separate bug where GetLangOptions() does not account
for the process/frame language and just uses a pre-defined set of language
options. I have not attempted to fix that since, for now, at least
hardcoding my dialect to be disabled by that function does the
right thing for existing tests.
Also fixed a trivial return-after-else that clangd pointed out in the same
area
---
clang/include/clang/Basic/IdentifierTable.h | 53 +++++++++++++++++++
clang/lib/Basic/IdentifierTable.cpp | 53 +------------------
.../CPlusPlus/CPlusPlusNameParser.cpp | 20 ++++---
3 files changed, 67 insertions(+), 59 deletions(-)
diff --git a/clang/include/clang/Basic/IdentifierTable.h b/clang/include/clang/Basic/IdentifierTable.h
index e4044bcdfcc60..8177029f13ca9 100644
--- a/clang/include/clang/Basic/IdentifierTable.h
+++ b/clang/include/clang/Basic/IdentifierTable.h
@@ -46,6 +46,59 @@ class LangOptions;
class MultiKeywordSelector;
class SourceLocation;
+// This is an inline namespace to allow both clang and lldb to use them without
+// namespace prefixes (via `using namespace` in lldb's case).
+inline namespace TokenKeyEnumerators {
+enum TokenKey : unsigned {
+ KEYC99 = 0x1,
+ KEYCXX = 0x2,
+ KEYCXX11 = 0x4,
+ KEYGNU = 0x8,
+ KEYMS = 0x10,
+ BOOLSUPPORT = 0x20,
+ KEYALTIVEC = 0x40,
+ KEYNOCXX = 0x80,
+ KEYBORLAND = 0x100,
+ KEYOPENCLC = 0x200,
+ KEYC23 = 0x400,
+ KEYNOMS18 = 0x800,
+ KEYNOOPENCL = 0x1000,
+ WCHARSUPPORT = 0x2000,
+ HALFSUPPORT = 0x4000,
+ CHAR8SUPPORT = 0x8000,
+ KEYOBJC = 0x10000,
+ KEYZVECTOR = 0x20000,
+ KEYCOROUTINES = 0x40000,
+ KEYMODULES = 0x80000,
+ KEYCXX20 = 0x100000,
+ KEYOPENCLCXX = 0x200000,
+ KEYMSCOMPAT = 0x400000,
+ KEYSYCL = 0x800000,
+ KEYCUDA = 0x1000000,
+ KEYZOS = 0x2000000,
+ KEYNOZOS = 0x4000000,
+ KEYHLSL = 0x8000000,
+ KEYFIXEDPOINT = 0x10000000,
+ KEYMAX = KEYFIXEDPOINT, // The maximum key
+ KEYALLCXX = KEYCXX | KEYCXX11 | KEYCXX20,
+ KEYALL = (KEYMAX | (KEYMAX - 1)) & ~KEYNOMS18 & ~KEYNOOPENCL &
+ ~KEYNOZOS // KEYNOMS18, KEYNOOPENCL, KEYNOZOS are excluded.
+};
+} // namespace TokenKeyEnumerators
+
+/// How a keyword is treated in the selected standard. This enum is ordered
+/// intentionally so that the value that 'wins' is the most 'permissive'.
+enum KeywordStatus {
+ KS_Unknown, // Not yet calculated. Used when figuring out the status.
+ KS_Disabled, // Disabled
+ KS_Future, // Is a keyword in future standard
+ KS_Extension, // Is an extension
+ KS_Enabled, // Enabled
+};
+
+KeywordStatus getKeywordStatus(const LangOptions &LangOpts,
+ unsigned Flags);
+
enum class ReservedIdentifierStatus {
NotReserved = 0,
StartsWithUnderscoreAtGlobalScope,
diff --git a/clang/lib/Basic/IdentifierTable.cpp b/clang/lib/Basic/IdentifierTable.cpp
index 4a2b77cd16bfc..65ed1273350c5 100644
--- a/clang/lib/Basic/IdentifierTable.cpp
+++ b/clang/lib/Basic/IdentifierTable.cpp
@@ -77,57 +77,6 @@ IdentifierTable::IdentifierTable(const LangOptions &LangOpts,
// Language Keyword Implementation
//===----------------------------------------------------------------------===//
-// Constants for TokenKinds.def
-namespace {
-
-enum TokenKey : unsigned {
- KEYC99 = 0x1,
- KEYCXX = 0x2,
- KEYCXX11 = 0x4,
- KEYGNU = 0x8,
- KEYMS = 0x10,
- BOOLSUPPORT = 0x20,
- KEYALTIVEC = 0x40,
- KEYNOCXX = 0x80,
- KEYBORLAND = 0x100,
- KEYOPENCLC = 0x200,
- KEYC23 = 0x400,
- KEYNOMS18 = 0x800,
- KEYNOOPENCL = 0x1000,
- WCHARSUPPORT = 0x2000,
- HALFSUPPORT = 0x4000,
- CHAR8SUPPORT = 0x8000,
- KEYOBJC = 0x10000,
- KEYZVECTOR = 0x20000,
- KEYCOROUTINES = 0x40000,
- KEYMODULES = 0x80000,
- KEYCXX20 = 0x100000,
- KEYOPENCLCXX = 0x200000,
- KEYMSCOMPAT = 0x400000,
- KEYSYCL = 0x800000,
- KEYCUDA = 0x1000000,
- KEYZOS = 0x2000000,
- KEYNOZOS = 0x4000000,
- KEYHLSL = 0x8000000,
- KEYFIXEDPOINT = 0x10000000,
- KEYMAX = KEYFIXEDPOINT, // The maximum key
- KEYALLCXX = KEYCXX | KEYCXX11 | KEYCXX20,
- KEYALL = (KEYMAX | (KEYMAX - 1)) & ~KEYNOMS18 & ~KEYNOOPENCL &
- ~KEYNOZOS // KEYNOMS18, KEYNOOPENCL, KEYNOZOS are excluded.
-};
-
-/// How a keyword is treated in the selected standard. This enum is ordered
-/// intentionally so that the value that 'wins' is the most 'permissive'.
-enum KeywordStatus {
- KS_Unknown, // Not yet calculated. Used when figuring out the status.
- KS_Disabled, // Disabled
- KS_Future, // Is a keyword in future standard
- KS_Extension, // Is an extension
- KS_Enabled, // Enabled
-};
-
-} // namespace
-
// This works on a single TokenKey flag and checks the LangOpts to get the
// KeywordStatus based exclusively on this flag, so that it can be merged in
// getKeywordStatus. Most should be enabled/disabled, but some might imply
@@ -222,7 +171,7 @@ static KeywordStatus getKeywordStatusHelper(const LangOptions &LangOpts,
/// Translates flags as specified in TokenKinds.def into keyword status
/// in the given language standard.
-static KeywordStatus getKeywordStatus(const LangOptions &LangOpts,
+KeywordStatus clang::getKeywordStatus(const LangOptions &LangOpts,
unsigned Flags) {
// KEYALL means always enabled, so special case this one.
if (Flags == KEYALL) return KS_Enabled;
diff --git a/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusNameParser.cpp b/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusNameParser.cpp
index d8c095d6edeb7..13cea5e513005 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusNameParser.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusNameParser.cpp
@@ -719,9 +719,8 @@ CPlusPlusNameParser::ParseFullNameImpl() {
}
start_position.Remove();
return result;
- } else {
- return std::nullopt;
}
+ return std::nullopt;
}
llvm::StringRef CPlusPlusNameParser::GetTextForRange(const Range &range) {
@@ -755,12 +754,19 @@ static const clang::LangOptions &GetLangOptions() {
return g_options;
}
-static const llvm::StringMap<tok::TokenKind> &GetKeywordsMap() {
- static llvm::StringMap<tok::TokenKind> g_map{
-#define KEYWORD(Name, Flags) {llvm::StringRef(#Name), tok::kw_##Name},
+static const llvm::StringMap<tok::TokenKind> GetKeywordsMap() {
+ using namespace clang::TokenKeyEnumerators;
+ llvm::StringMap<tok::TokenKind> g_map;
+
+ auto LangOpts = GetLangOptions();
+
+ clang::KeywordStatus AddResult;
+#define KEYWORD(NAME, FLAGS) \
+ AddResult = getKeywordStatus(LangOpts, FLAGS); \
+ if (AddResult != clang::KS_Disabled) \
+ g_map[llvm::StringRef(#NAME)] = tok::kw_##NAME;
#include "clang/Basic/TokenKinds.def"
#undef KEYWORD
- };
return g_map;
}
@@ -769,7 +775,7 @@ void CPlusPlusNameParser::ExtractTokens() {
return;
clang::Lexer lexer(clang::SourceLocation(), GetLangOptions(), m_text.data(),
m_text.data(), m_text.data() + m_text.size());
- const auto &kw_map = GetKeywordsMap();
+ const auto kw_map = GetKeywordsMap();
clang::Token token;
for (lexer.LexFromRawLexer(token); !token.is(clang::tok::eof);
lexer.LexFromRawLexer(token)) {
>From 0275d00b577c8f273b9d3949122367ddcc311b3c Mon Sep 17 00:00:00 2001
From: Daniel Sanders <daniel_l_sanders at apple.com>
Date: Mon, 27 Oct 2025 14:20:10 -0700
Subject: [PATCH 02/11] fixup: use `using namespace clang` instead
---
clang/include/clang/Basic/IdentifierTable.h | 4 ----
.../Plugins/Language/CPlusPlus/CPlusPlusNameParser.cpp | 6 +++---
2 files changed, 3 insertions(+), 7 deletions(-)
diff --git a/clang/include/clang/Basic/IdentifierTable.h b/clang/include/clang/Basic/IdentifierTable.h
index 8177029f13ca9..54500b95b2c06 100644
--- a/clang/include/clang/Basic/IdentifierTable.h
+++ b/clang/include/clang/Basic/IdentifierTable.h
@@ -46,9 +46,6 @@ class LangOptions;
class MultiKeywordSelector;
class SourceLocation;
-// This is an inline namespace to allow both clang and lldb to use them without
-// namespace prefixes (via `using namespace` in lldb's case).
-inline namespace TokenKeyEnumerators {
enum TokenKey : unsigned {
KEYC99 = 0x1,
KEYCXX = 0x2,
@@ -84,7 +81,6 @@ enum TokenKey : unsigned {
KEYALL = (KEYMAX | (KEYMAX - 1)) & ~KEYNOMS18 & ~KEYNOOPENCL &
~KEYNOZOS // KEYNOMS18, KEYNOOPENCL, KEYNOZOS are excluded.
};
-} // namespace TokenKeyEnumerators
/// How a keyword is treated in the selected standard. This enum is ordered
/// intentionally so that the value that 'wins' is the most 'permissive'.
diff --git a/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusNameParser.cpp b/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusNameParser.cpp
index 13cea5e513005..8560a44340508 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusNameParser.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusNameParser.cpp
@@ -755,15 +755,15 @@ static const clang::LangOptions &GetLangOptions() {
}
static const llvm::StringMap<tok::TokenKind> GetKeywordsMap() {
- using namespace clang::TokenKeyEnumerators;
+ using namespace clang;
llvm::StringMap<tok::TokenKind> g_map;
auto LangOpts = GetLangOptions();
- clang::KeywordStatus AddResult;
+ KeywordStatus AddResult;
#define KEYWORD(NAME, FLAGS) \
AddResult = getKeywordStatus(LangOpts, FLAGS); \
- if (AddResult != clang::KS_Disabled) \
+ if (AddResult != KS_Disabled) \
g_map[llvm::StringRef(#NAME)] = tok::kw_##NAME;
#include "clang/Basic/TokenKinds.def"
#undef KEYWORD
>From f221e49028db66859bb0c9ca00a0beee7be3d6a7 Mon Sep 17 00:00:00 2001
From: Daniel Sanders <daniel_l_sanders at apple.com>
Date: Mon, 27 Oct 2025 14:37:16 -0700
Subject: [PATCH 03/11] Revert "[lldb] Re-use clang's keyword enable/disable
mechanism in CPlusPlusNameParser.cpp"
This is just to allow me to separate the clang and lldb parts without breaking
the github PR's comment history
---
clang/include/clang/Basic/IdentifierTable.h | 49 -----------------
clang/lib/Basic/IdentifierTable.cpp | 53 ++++++++++++++++++-
.../CPlusPlus/CPlusPlusNameParser.cpp | 20 +++----
3 files changed, 59 insertions(+), 63 deletions(-)
diff --git a/clang/include/clang/Basic/IdentifierTable.h b/clang/include/clang/Basic/IdentifierTable.h
index 54500b95b2c06..e4044bcdfcc60 100644
--- a/clang/include/clang/Basic/IdentifierTable.h
+++ b/clang/include/clang/Basic/IdentifierTable.h
@@ -46,55 +46,6 @@ class LangOptions;
class MultiKeywordSelector;
class SourceLocation;
-enum TokenKey : unsigned {
- KEYC99 = 0x1,
- KEYCXX = 0x2,
- KEYCXX11 = 0x4,
- KEYGNU = 0x8,
- KEYMS = 0x10,
- BOOLSUPPORT = 0x20,
- KEYALTIVEC = 0x40,
- KEYNOCXX = 0x80,
- KEYBORLAND = 0x100,
- KEYOPENCLC = 0x200,
- KEYC23 = 0x400,
- KEYNOMS18 = 0x800,
- KEYNOOPENCL = 0x1000,
- WCHARSUPPORT = 0x2000,
- HALFSUPPORT = 0x4000,
- CHAR8SUPPORT = 0x8000,
- KEYOBJC = 0x10000,
- KEYZVECTOR = 0x20000,
- KEYCOROUTINES = 0x40000,
- KEYMODULES = 0x80000,
- KEYCXX20 = 0x100000,
- KEYOPENCLCXX = 0x200000,
- KEYMSCOMPAT = 0x400000,
- KEYSYCL = 0x800000,
- KEYCUDA = 0x1000000,
- KEYZOS = 0x2000000,
- KEYNOZOS = 0x4000000,
- KEYHLSL = 0x8000000,
- KEYFIXEDPOINT = 0x10000000,
- KEYMAX = KEYFIXEDPOINT, // The maximum key
- KEYALLCXX = KEYCXX | KEYCXX11 | KEYCXX20,
- KEYALL = (KEYMAX | (KEYMAX - 1)) & ~KEYNOMS18 & ~KEYNOOPENCL &
- ~KEYNOZOS // KEYNOMS18, KEYNOOPENCL, KEYNOZOS are excluded.
-};
-
-/// How a keyword is treated in the selected standard. This enum is ordered
-/// intentionally so that the value that 'wins' is the most 'permissive'.
-enum KeywordStatus {
- KS_Unknown, // Not yet calculated. Used when figuring out the status.
- KS_Disabled, // Disabled
- KS_Future, // Is a keyword in future standard
- KS_Extension, // Is an extension
- KS_Enabled, // Enabled
-};
-
-KeywordStatus getKeywordStatus(const LangOptions &LangOpts,
- unsigned Flags);
-
enum class ReservedIdentifierStatus {
NotReserved = 0,
StartsWithUnderscoreAtGlobalScope,
diff --git a/clang/lib/Basic/IdentifierTable.cpp b/clang/lib/Basic/IdentifierTable.cpp
index 65ed1273350c5..4a2b77cd16bfc 100644
--- a/clang/lib/Basic/IdentifierTable.cpp
+++ b/clang/lib/Basic/IdentifierTable.cpp
@@ -77,6 +77,57 @@ IdentifierTable::IdentifierTable(const LangOptions &LangOpts,
// Language Keyword Implementation
//===----------------------------------------------------------------------===//
+// Constants for TokenKinds.def
+namespace {
+
+enum TokenKey : unsigned {
+ KEYC99 = 0x1,
+ KEYCXX = 0x2,
+ KEYCXX11 = 0x4,
+ KEYGNU = 0x8,
+ KEYMS = 0x10,
+ BOOLSUPPORT = 0x20,
+ KEYALTIVEC = 0x40,
+ KEYNOCXX = 0x80,
+ KEYBORLAND = 0x100,
+ KEYOPENCLC = 0x200,
+ KEYC23 = 0x400,
+ KEYNOMS18 = 0x800,
+ KEYNOOPENCL = 0x1000,
+ WCHARSUPPORT = 0x2000,
+ HALFSUPPORT = 0x4000,
+ CHAR8SUPPORT = 0x8000,
+ KEYOBJC = 0x10000,
+ KEYZVECTOR = 0x20000,
+ KEYCOROUTINES = 0x40000,
+ KEYMODULES = 0x80000,
+ KEYCXX20 = 0x100000,
+ KEYOPENCLCXX = 0x200000,
+ KEYMSCOMPAT = 0x400000,
+ KEYSYCL = 0x800000,
+ KEYCUDA = 0x1000000,
+ KEYZOS = 0x2000000,
+ KEYNOZOS = 0x4000000,
+ KEYHLSL = 0x8000000,
+ KEYFIXEDPOINT = 0x10000000,
+ KEYMAX = KEYFIXEDPOINT, // The maximum key
+ KEYALLCXX = KEYCXX | KEYCXX11 | KEYCXX20,
+ KEYALL = (KEYMAX | (KEYMAX - 1)) & ~KEYNOMS18 & ~KEYNOOPENCL &
+ ~KEYNOZOS // KEYNOMS18, KEYNOOPENCL, KEYNOZOS are excluded.
+};
+
+/// How a keyword is treated in the selected standard. This enum is ordered
+/// intentionally so that the value that 'wins' is the most 'permissive'.
+enum KeywordStatus {
+ KS_Unknown, // Not yet calculated. Used when figuring out the status.
+ KS_Disabled, // Disabled
+ KS_Future, // Is a keyword in future standard
+ KS_Extension, // Is an extension
+ KS_Enabled, // Enabled
+};
+
+} // namespace
+
// This works on a single TokenKey flag and checks the LangOpts to get the
// KeywordStatus based exclusively on this flag, so that it can be merged in
// getKeywordStatus. Most should be enabled/disabled, but some might imply
@@ -171,7 +222,7 @@ static KeywordStatus getKeywordStatusHelper(const LangOptions &LangOpts,
/// Translates flags as specified in TokenKinds.def into keyword status
/// in the given language standard.
-KeywordStatus clang::getKeywordStatus(const LangOptions &LangOpts,
+static KeywordStatus getKeywordStatus(const LangOptions &LangOpts,
unsigned Flags) {
// KEYALL means always enabled, so special case this one.
if (Flags == KEYALL) return KS_Enabled;
diff --git a/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusNameParser.cpp b/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusNameParser.cpp
index 8560a44340508..d8c095d6edeb7 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusNameParser.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusNameParser.cpp
@@ -719,8 +719,9 @@ CPlusPlusNameParser::ParseFullNameImpl() {
}
start_position.Remove();
return result;
+ } else {
+ return std::nullopt;
}
- return std::nullopt;
}
llvm::StringRef CPlusPlusNameParser::GetTextForRange(const Range &range) {
@@ -754,19 +755,12 @@ static const clang::LangOptions &GetLangOptions() {
return g_options;
}
-static const llvm::StringMap<tok::TokenKind> GetKeywordsMap() {
- using namespace clang;
- llvm::StringMap<tok::TokenKind> g_map;
-
- auto LangOpts = GetLangOptions();
-
- KeywordStatus AddResult;
-#define KEYWORD(NAME, FLAGS) \
- AddResult = getKeywordStatus(LangOpts, FLAGS); \
- if (AddResult != KS_Disabled) \
- g_map[llvm::StringRef(#NAME)] = tok::kw_##NAME;
+static const llvm::StringMap<tok::TokenKind> &GetKeywordsMap() {
+ static llvm::StringMap<tok::TokenKind> g_map{
+#define KEYWORD(Name, Flags) {llvm::StringRef(#Name), tok::kw_##Name},
#include "clang/Basic/TokenKinds.def"
#undef KEYWORD
+ };
return g_map;
}
@@ -775,7 +769,7 @@ void CPlusPlusNameParser::ExtractTokens() {
return;
clang::Lexer lexer(clang::SourceLocation(), GetLangOptions(), m_text.data(),
m_text.data(), m_text.data() + m_text.size());
- const auto kw_map = GetKeywordsMap();
+ const auto &kw_map = GetKeywordsMap();
clang::Token token;
for (lexer.LexFromRawLexer(token); !token.is(clang::tok::eof);
lexer.LexFromRawLexer(token)) {
>From 7af1e873756be458855afc9488dd2f95a2f21568 Mon Sep 17 00:00:00 2001
From: Daniel Sanders <daniel_l_sanders at apple.com>
Date: Fri, 17 Oct 2025 18:10:51 -0700
Subject: [PATCH 04/11] [clang] Refactor clang's keyword enable/disable
mechanism to allow lldb to re-use it
lldb's CPlusPlusNameParser is currently identifying keywords using it's own
map implemented using clang/Basic/TokenKinds.def. However, it does not respect
the language options so identifiers can incorrectly determined to be keywords
when using languages in which they are not keywords.
Rather than implement the logic to enable/disable keywords in both projects it
makes sense for lldb to use clang's implementation.
See #164284 for more information
---
clang/include/clang/Basic/IdentifierTable.h | 49 +++++++++++++++++++
clang/lib/Basic/IdentifierTable.cpp | 53 +--------------------
2 files changed, 50 insertions(+), 52 deletions(-)
diff --git a/clang/include/clang/Basic/IdentifierTable.h b/clang/include/clang/Basic/IdentifierTable.h
index e4044bcdfcc60..54500b95b2c06 100644
--- a/clang/include/clang/Basic/IdentifierTable.h
+++ b/clang/include/clang/Basic/IdentifierTable.h
@@ -46,6 +46,55 @@ class LangOptions;
class MultiKeywordSelector;
class SourceLocation;
+enum TokenKey : unsigned {
+ KEYC99 = 0x1,
+ KEYCXX = 0x2,
+ KEYCXX11 = 0x4,
+ KEYGNU = 0x8,
+ KEYMS = 0x10,
+ BOOLSUPPORT = 0x20,
+ KEYALTIVEC = 0x40,
+ KEYNOCXX = 0x80,
+ KEYBORLAND = 0x100,
+ KEYOPENCLC = 0x200,
+ KEYC23 = 0x400,
+ KEYNOMS18 = 0x800,
+ KEYNOOPENCL = 0x1000,
+ WCHARSUPPORT = 0x2000,
+ HALFSUPPORT = 0x4000,
+ CHAR8SUPPORT = 0x8000,
+ KEYOBJC = 0x10000,
+ KEYZVECTOR = 0x20000,
+ KEYCOROUTINES = 0x40000,
+ KEYMODULES = 0x80000,
+ KEYCXX20 = 0x100000,
+ KEYOPENCLCXX = 0x200000,
+ KEYMSCOMPAT = 0x400000,
+ KEYSYCL = 0x800000,
+ KEYCUDA = 0x1000000,
+ KEYZOS = 0x2000000,
+ KEYNOZOS = 0x4000000,
+ KEYHLSL = 0x8000000,
+ KEYFIXEDPOINT = 0x10000000,
+ KEYMAX = KEYFIXEDPOINT, // The maximum key
+ KEYALLCXX = KEYCXX | KEYCXX11 | KEYCXX20,
+ KEYALL = (KEYMAX | (KEYMAX - 1)) & ~KEYNOMS18 & ~KEYNOOPENCL &
+ ~KEYNOZOS // KEYNOMS18, KEYNOOPENCL, KEYNOZOS are excluded.
+};
+
+/// How a keyword is treated in the selected standard. This enum is ordered
+/// intentionally so that the value that 'wins' is the most 'permissive'.
+enum KeywordStatus {
+ KS_Unknown, // Not yet calculated. Used when figuring out the status.
+ KS_Disabled, // Disabled
+ KS_Future, // Is a keyword in future standard
+ KS_Extension, // Is an extension
+ KS_Enabled, // Enabled
+};
+
+KeywordStatus getKeywordStatus(const LangOptions &LangOpts,
+ unsigned Flags);
+
enum class ReservedIdentifierStatus {
NotReserved = 0,
StartsWithUnderscoreAtGlobalScope,
diff --git a/clang/lib/Basic/IdentifierTable.cpp b/clang/lib/Basic/IdentifierTable.cpp
index 4a2b77cd16bfc..65ed1273350c5 100644
--- a/clang/lib/Basic/IdentifierTable.cpp
+++ b/clang/lib/Basic/IdentifierTable.cpp
@@ -77,57 +77,6 @@ IdentifierTable::IdentifierTable(const LangOptions &LangOpts,
// Language Keyword Implementation
//===----------------------------------------------------------------------===//
-// Constants for TokenKinds.def
-namespace {
-
-enum TokenKey : unsigned {
- KEYC99 = 0x1,
- KEYCXX = 0x2,
- KEYCXX11 = 0x4,
- KEYGNU = 0x8,
- KEYMS = 0x10,
- BOOLSUPPORT = 0x20,
- KEYALTIVEC = 0x40,
- KEYNOCXX = 0x80,
- KEYBORLAND = 0x100,
- KEYOPENCLC = 0x200,
- KEYC23 = 0x400,
- KEYNOMS18 = 0x800,
- KEYNOOPENCL = 0x1000,
- WCHARSUPPORT = 0x2000,
- HALFSUPPORT = 0x4000,
- CHAR8SUPPORT = 0x8000,
- KEYOBJC = 0x10000,
- KEYZVECTOR = 0x20000,
- KEYCOROUTINES = 0x40000,
- KEYMODULES = 0x80000,
- KEYCXX20 = 0x100000,
- KEYOPENCLCXX = 0x200000,
- KEYMSCOMPAT = 0x400000,
- KEYSYCL = 0x800000,
- KEYCUDA = 0x1000000,
- KEYZOS = 0x2000000,
- KEYNOZOS = 0x4000000,
- KEYHLSL = 0x8000000,
- KEYFIXEDPOINT = 0x10000000,
- KEYMAX = KEYFIXEDPOINT, // The maximum key
- KEYALLCXX = KEYCXX | KEYCXX11 | KEYCXX20,
- KEYALL = (KEYMAX | (KEYMAX - 1)) & ~KEYNOMS18 & ~KEYNOOPENCL &
- ~KEYNOZOS // KEYNOMS18, KEYNOOPENCL, KEYNOZOS are excluded.
-};
-
-/// How a keyword is treated in the selected standard. This enum is ordered
-/// intentionally so that the value that 'wins' is the most 'permissive'.
-enum KeywordStatus {
- KS_Unknown, // Not yet calculated. Used when figuring out the status.
- KS_Disabled, // Disabled
- KS_Future, // Is a keyword in future standard
- KS_Extension, // Is an extension
- KS_Enabled, // Enabled
-};
-
-} // namespace
-
// This works on a single TokenKey flag and checks the LangOpts to get the
// KeywordStatus based exclusively on this flag, so that it can be merged in
// getKeywordStatus. Most should be enabled/disabled, but some might imply
@@ -222,7 +171,7 @@ static KeywordStatus getKeywordStatusHelper(const LangOptions &LangOpts,
/// Translates flags as specified in TokenKinds.def into keyword status
/// in the given language standard.
-static KeywordStatus getKeywordStatus(const LangOptions &LangOpts,
+KeywordStatus clang::getKeywordStatus(const LangOptions &LangOpts,
unsigned Flags) {
// KEYALL means always enabled, so special case this one.
if (Flags == KEYALL) return KS_Enabled;
>From e03585e4f3700c90574065baabb25b093f634090 Mon Sep 17 00:00:00 2001
From: Daniel Sanders <daniel_l_sanders at apple.com>
Date: Mon, 27 Oct 2025 14:39:42 -0700
Subject: [PATCH 05/11] [lldb] Re-use clang's keyword enable/disable mechanism
in CPlusPlusNameParser.cpp
The problem is that CPlusPlusNameParser doesn't respect the language dialect
for `target variable Class::constant`. If `constant` is a keyword in some
(downstream in this case) dialects then it lexes `constant` as `kw_constant`
instead of `kw_raw_identifier` even if that dialect is not active. As a
result:
* `target variable constant` works
* `target variable Class::constant` fails to look up constant because it sees
`kw_raw_identifier, kw_coloncolon, kw_constant` instead of
`kw_raw_identifier, kw_coloncolon, kw_raw_identifier`
* `expr -l c++ -- Class::constant` works because clang is parsing it
Note: There's seemingly a separate bug where GetLangOptions() does not account
for the process/frame language and just uses a pre-defined set of language
options. I have not attempted to fix that since, for now, at least
hardcoding my dialect to be disabled by that function does the
right thing for existing tests.
Also fixed a trivial return-after-else that clangd pointed out in the same
area
---
.../CPlusPlus/CPlusPlusNameParser.cpp | 20 ++++++++++++-------
1 file changed, 13 insertions(+), 7 deletions(-)
diff --git a/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusNameParser.cpp b/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusNameParser.cpp
index d8c095d6edeb7..8560a44340508 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusNameParser.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusNameParser.cpp
@@ -719,9 +719,8 @@ CPlusPlusNameParser::ParseFullNameImpl() {
}
start_position.Remove();
return result;
- } else {
- return std::nullopt;
}
+ return std::nullopt;
}
llvm::StringRef CPlusPlusNameParser::GetTextForRange(const Range &range) {
@@ -755,12 +754,19 @@ static const clang::LangOptions &GetLangOptions() {
return g_options;
}
-static const llvm::StringMap<tok::TokenKind> &GetKeywordsMap() {
- static llvm::StringMap<tok::TokenKind> g_map{
-#define KEYWORD(Name, Flags) {llvm::StringRef(#Name), tok::kw_##Name},
+static const llvm::StringMap<tok::TokenKind> GetKeywordsMap() {
+ using namespace clang;
+ llvm::StringMap<tok::TokenKind> g_map;
+
+ auto LangOpts = GetLangOptions();
+
+ KeywordStatus AddResult;
+#define KEYWORD(NAME, FLAGS) \
+ AddResult = getKeywordStatus(LangOpts, FLAGS); \
+ if (AddResult != KS_Disabled) \
+ g_map[llvm::StringRef(#NAME)] = tok::kw_##NAME;
#include "clang/Basic/TokenKinds.def"
#undef KEYWORD
- };
return g_map;
}
@@ -769,7 +775,7 @@ void CPlusPlusNameParser::ExtractTokens() {
return;
clang::Lexer lexer(clang::SourceLocation(), GetLangOptions(), m_text.data(),
m_text.data(), m_text.data() + m_text.size());
- const auto &kw_map = GetKeywordsMap();
+ const auto kw_map = GetKeywordsMap();
clang::Token token;
for (lexer.LexFromRawLexer(token); !token.is(clang::tok::eof);
lexer.LexFromRawLexer(token)) {
>From 8e0c1a7a39447f7785e6301ad54307b3f3479a48 Mon Sep 17 00:00:00 2001
From: Daniel Sanders <daniel_l_sanders at apple.com>
Date: Fri, 17 Oct 2025 18:10:51 -0700
Subject: [PATCH 06/11] [clang] Refactor clang's keyword enable/disable
mechanism to allow lldb to re-use it
lldb's CPlusPlusNameParser is currently identifying keywords using it's own
map implemented using clang/Basic/TokenKinds.def. However, it does not respect
the language options so identifiers can incorrectly determined to be keywords
when using languages in which they are not keywords.
Rather than implement the logic to enable/disable keywords in both projects it
makes sense for lldb to use clang's implementation.
See #164284 for more information
---
clang/include/clang/Basic/IdentifierTable.h | 49 +++++++++++++++++++
clang/lib/Basic/IdentifierTable.cpp | 53 +--------------------
2 files changed, 50 insertions(+), 52 deletions(-)
diff --git a/clang/include/clang/Basic/IdentifierTable.h b/clang/include/clang/Basic/IdentifierTable.h
index e4044bcdfcc60..54500b95b2c06 100644
--- a/clang/include/clang/Basic/IdentifierTable.h
+++ b/clang/include/clang/Basic/IdentifierTable.h
@@ -46,6 +46,55 @@ class LangOptions;
class MultiKeywordSelector;
class SourceLocation;
+enum TokenKey : unsigned {
+ KEYC99 = 0x1,
+ KEYCXX = 0x2,
+ KEYCXX11 = 0x4,
+ KEYGNU = 0x8,
+ KEYMS = 0x10,
+ BOOLSUPPORT = 0x20,
+ KEYALTIVEC = 0x40,
+ KEYNOCXX = 0x80,
+ KEYBORLAND = 0x100,
+ KEYOPENCLC = 0x200,
+ KEYC23 = 0x400,
+ KEYNOMS18 = 0x800,
+ KEYNOOPENCL = 0x1000,
+ WCHARSUPPORT = 0x2000,
+ HALFSUPPORT = 0x4000,
+ CHAR8SUPPORT = 0x8000,
+ KEYOBJC = 0x10000,
+ KEYZVECTOR = 0x20000,
+ KEYCOROUTINES = 0x40000,
+ KEYMODULES = 0x80000,
+ KEYCXX20 = 0x100000,
+ KEYOPENCLCXX = 0x200000,
+ KEYMSCOMPAT = 0x400000,
+ KEYSYCL = 0x800000,
+ KEYCUDA = 0x1000000,
+ KEYZOS = 0x2000000,
+ KEYNOZOS = 0x4000000,
+ KEYHLSL = 0x8000000,
+ KEYFIXEDPOINT = 0x10000000,
+ KEYMAX = KEYFIXEDPOINT, // The maximum key
+ KEYALLCXX = KEYCXX | KEYCXX11 | KEYCXX20,
+ KEYALL = (KEYMAX | (KEYMAX - 1)) & ~KEYNOMS18 & ~KEYNOOPENCL &
+ ~KEYNOZOS // KEYNOMS18, KEYNOOPENCL, KEYNOZOS are excluded.
+};
+
+/// How a keyword is treated in the selected standard. This enum is ordered
+/// intentionally so that the value that 'wins' is the most 'permissive'.
+enum KeywordStatus {
+ KS_Unknown, // Not yet calculated. Used when figuring out the status.
+ KS_Disabled, // Disabled
+ KS_Future, // Is a keyword in future standard
+ KS_Extension, // Is an extension
+ KS_Enabled, // Enabled
+};
+
+KeywordStatus getKeywordStatus(const LangOptions &LangOpts,
+ unsigned Flags);
+
enum class ReservedIdentifierStatus {
NotReserved = 0,
StartsWithUnderscoreAtGlobalScope,
diff --git a/clang/lib/Basic/IdentifierTable.cpp b/clang/lib/Basic/IdentifierTable.cpp
index 4a2b77cd16bfc..65ed1273350c5 100644
--- a/clang/lib/Basic/IdentifierTable.cpp
+++ b/clang/lib/Basic/IdentifierTable.cpp
@@ -77,57 +77,6 @@ IdentifierTable::IdentifierTable(const LangOptions &LangOpts,
// Language Keyword Implementation
//===----------------------------------------------------------------------===//
-// Constants for TokenKinds.def
-namespace {
-
-enum TokenKey : unsigned {
- KEYC99 = 0x1,
- KEYCXX = 0x2,
- KEYCXX11 = 0x4,
- KEYGNU = 0x8,
- KEYMS = 0x10,
- BOOLSUPPORT = 0x20,
- KEYALTIVEC = 0x40,
- KEYNOCXX = 0x80,
- KEYBORLAND = 0x100,
- KEYOPENCLC = 0x200,
- KEYC23 = 0x400,
- KEYNOMS18 = 0x800,
- KEYNOOPENCL = 0x1000,
- WCHARSUPPORT = 0x2000,
- HALFSUPPORT = 0x4000,
- CHAR8SUPPORT = 0x8000,
- KEYOBJC = 0x10000,
- KEYZVECTOR = 0x20000,
- KEYCOROUTINES = 0x40000,
- KEYMODULES = 0x80000,
- KEYCXX20 = 0x100000,
- KEYOPENCLCXX = 0x200000,
- KEYMSCOMPAT = 0x400000,
- KEYSYCL = 0x800000,
- KEYCUDA = 0x1000000,
- KEYZOS = 0x2000000,
- KEYNOZOS = 0x4000000,
- KEYHLSL = 0x8000000,
- KEYFIXEDPOINT = 0x10000000,
- KEYMAX = KEYFIXEDPOINT, // The maximum key
- KEYALLCXX = KEYCXX | KEYCXX11 | KEYCXX20,
- KEYALL = (KEYMAX | (KEYMAX - 1)) & ~KEYNOMS18 & ~KEYNOOPENCL &
- ~KEYNOZOS // KEYNOMS18, KEYNOOPENCL, KEYNOZOS are excluded.
-};
-
-/// How a keyword is treated in the selected standard. This enum is ordered
-/// intentionally so that the value that 'wins' is the most 'permissive'.
-enum KeywordStatus {
- KS_Unknown, // Not yet calculated. Used when figuring out the status.
- KS_Disabled, // Disabled
- KS_Future, // Is a keyword in future standard
- KS_Extension, // Is an extension
- KS_Enabled, // Enabled
-};
-
-} // namespace
-
// This works on a single TokenKey flag and checks the LangOpts to get the
// KeywordStatus based exclusively on this flag, so that it can be merged in
// getKeywordStatus. Most should be enabled/disabled, but some might imply
@@ -222,7 +171,7 @@ static KeywordStatus getKeywordStatusHelper(const LangOptions &LangOpts,
/// Translates flags as specified in TokenKinds.def into keyword status
/// in the given language standard.
-static KeywordStatus getKeywordStatus(const LangOptions &LangOpts,
+KeywordStatus clang::getKeywordStatus(const LangOptions &LangOpts,
unsigned Flags) {
// KEYALL means always enabled, so special case this one.
if (Flags == KEYALL) return KS_Enabled;
>From 2c7391fadb211cbea19bf423bbf8064638cbfa78 Mon Sep 17 00:00:00 2001
From: Daniel Sanders <daniel_l_sanders at apple.com>
Date: Mon, 27 Oct 2025 14:59:59 -0700
Subject: [PATCH 07/11] fixup: formatting
---
clang/include/clang/Basic/IdentifierTable.h | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/clang/include/clang/Basic/IdentifierTable.h b/clang/include/clang/Basic/IdentifierTable.h
index 54500b95b2c06..7062aea3100f3 100644
--- a/clang/include/clang/Basic/IdentifierTable.h
+++ b/clang/include/clang/Basic/IdentifierTable.h
@@ -92,8 +92,7 @@ enum KeywordStatus {
KS_Enabled, // Enabled
};
-KeywordStatus getKeywordStatus(const LangOptions &LangOpts,
- unsigned Flags);
+KeywordStatus getKeywordStatus(const LangOptions &LangOpts, unsigned Flags);
enum class ReservedIdentifierStatus {
NotReserved = 0,
>From af4a46ff22c1e4f565aadca98b29e409d7cf8574 Mon Sep 17 00:00:00 2001
From: Daniel Sanders <daniel_l_sanders at apple.com>
Date: Tue, 4 Nov 2025 15:45:43 -0800
Subject: [PATCH 08/11] fixup: Move getKeywordStatus's comment to the header
---
clang/include/clang/Basic/IdentifierTable.h | 2 ++
clang/lib/Basic/IdentifierTable.cpp | 2 --
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/clang/include/clang/Basic/IdentifierTable.h b/clang/include/clang/Basic/IdentifierTable.h
index 7062aea3100f3..24d51b36df640 100644
--- a/clang/include/clang/Basic/IdentifierTable.h
+++ b/clang/include/clang/Basic/IdentifierTable.h
@@ -92,6 +92,8 @@ enum KeywordStatus {
KS_Enabled, // Enabled
};
+/// Translates flags as specified in TokenKinds.def into keyword status
+/// in the given language standard.
KeywordStatus getKeywordStatus(const LangOptions &LangOpts, unsigned Flags);
enum class ReservedIdentifierStatus {
diff --git a/clang/lib/Basic/IdentifierTable.cpp b/clang/lib/Basic/IdentifierTable.cpp
index 65ed1273350c5..d1c959b9687c4 100644
--- a/clang/lib/Basic/IdentifierTable.cpp
+++ b/clang/lib/Basic/IdentifierTable.cpp
@@ -169,8 +169,6 @@ static KeywordStatus getKeywordStatusHelper(const LangOptions &LangOpts,
}
}
-/// Translates flags as specified in TokenKinds.def into keyword status
-/// in the given language standard.
KeywordStatus clang::getKeywordStatus(const LangOptions &LangOpts,
unsigned Flags) {
// KEYALL means always enabled, so special case this one.
>From 6a80842c253cc8b425cc5fa03291993aa1708a25 Mon Sep 17 00:00:00 2001
From: Daniel Sanders <daniel_l_sanders at apple.com>
Date: Tue, 4 Nov 2025 16:04:02 -0800
Subject: [PATCH 09/11] fixup: Restore comment for TokenKey
This comment was originally on the namespace containing TokenKey and
KeywordStatus. I've only restored it to TokenKey because KeywordStatus
is used by the code that uses TokenKinds.def not TokenKinds.def itself.
---
clang/include/clang/Basic/IdentifierTable.h | 1 +
1 file changed, 1 insertion(+)
diff --git a/clang/include/clang/Basic/IdentifierTable.h b/clang/include/clang/Basic/IdentifierTable.h
index 24d51b36df640..b27492d19a65b 100644
--- a/clang/include/clang/Basic/IdentifierTable.h
+++ b/clang/include/clang/Basic/IdentifierTable.h
@@ -46,6 +46,7 @@ class LangOptions;
class MultiKeywordSelector;
class SourceLocation;
+/// Constants for TokenKinds.def
enum TokenKey : unsigned {
KEYC99 = 0x1,
KEYCXX = 0x2,
>From 3acac96844ee5aa8bfe0c851445cfcb75e7f9f40 Mon Sep 17 00:00:00 2001
From: Daniel Sanders <daniel_l_sanders at apple.com>
Date: Tue, 4 Nov 2025 16:11:24 -0800
Subject: [PATCH 10/11] fixup: Move using
---
lldb/source/Plugins/Language/CPlusPlus/CPlusPlusNameParser.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusNameParser.cpp b/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusNameParser.cpp
index 8560a44340508..b50fac678bbbe 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusNameParser.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusNameParser.cpp
@@ -755,11 +755,11 @@ static const clang::LangOptions &GetLangOptions() {
}
static const llvm::StringMap<tok::TokenKind> GetKeywordsMap() {
- using namespace clang;
llvm::StringMap<tok::TokenKind> g_map;
auto LangOpts = GetLangOptions();
+ using namespace clang;
KeywordStatus AddResult;
#define KEYWORD(NAME, FLAGS) \
AddResult = getKeywordStatus(LangOpts, FLAGS); \
>From 72e9d76787258eeee1b7ca10b0a50ce17d584e11 Mon Sep 17 00:00:00 2001
From: Daniel Sanders <daniel_l_sanders at apple.com>
Date: Tue, 4 Nov 2025 17:46:23 -0800
Subject: [PATCH 11/11] fixup: Test Class::constant in
CPlusPlusLanguageTest.cpp
---
lldb/unittests/Language/CPlusPlus/CPlusPlusLanguageTest.cpp | 3 +++
1 file changed, 3 insertions(+)
diff --git a/lldb/unittests/Language/CPlusPlus/CPlusPlusLanguageTest.cpp b/lldb/unittests/Language/CPlusPlus/CPlusPlusLanguageTest.cpp
index 8c36f540370f3..fd580584d98a2 100644
--- a/lldb/unittests/Language/CPlusPlus/CPlusPlusLanguageTest.cpp
+++ b/lldb/unittests/Language/CPlusPlus/CPlusPlusLanguageTest.cpp
@@ -336,6 +336,9 @@ TEST(CPlusPlusLanguage, ExtractContextAndIdentifier) {
{"operator<<A::B>", "", "operator<<A::B>"},
{"A::operator<<<A::B>", "A", "operator<<<A::B>"},
{"operator<<<A::B>", "", "operator<<<A::B>"},
+ // There are languages based on C++ in which `constant` is a keyword.
+ // Ensure we can still parse this in C++ even when that's the case.
+ {"Class::constant", "Class", "constant"},
};
llvm::StringRef context, basename;
More information about the lldb-commits
mailing list