[clang-tools-extra] 5f1adf0 - [clangd] Fix crash with null check for Token at Loc (#94528)
via cfe-commits
cfe-commits at lists.llvm.org
Fri Jun 7 02:08:28 PDT 2024
Author: Utkarsh Saxena
Date: 2024-06-07T11:08:25+02:00
New Revision: 5f1adf0433c6007f8be885b832c852da67e8524c
URL: https://github.com/llvm/llvm-project/commit/5f1adf0433c6007f8be885b832c852da67e8524c
DIFF: https://github.com/llvm/llvm-project/commit/5f1adf0433c6007f8be885b832c852da67e8524c.diff
LOG: [clangd] Fix crash with null check for Token at Loc (#94528)
Fixes https://github.com/llvm/llvm-project/issues/94599
Added:
Modified:
clang-tools-extra/clangd/FindSymbols.cpp
clang-tools-extra/clangd/IncludeCleaner.cpp
clang-tools-extra/clangd/SemanticHighlighting.cpp
clang-tools-extra/clangd/XRefs.cpp
clang-tools-extra/clangd/refactor/Rename.cpp
clang-tools-extra/clangd/unittests/PreambleTests.cpp
clang-tools-extra/clangd/unittests/XRefsTests.cpp
clang/include/clang/Tooling/Syntax/Tokens.h
clang/lib/Tooling/Syntax/Tokens.cpp
clang/unittests/Tooling/Syntax/TokensTest.cpp
Removed:
################################################################################
diff --git a/clang-tools-extra/clangd/FindSymbols.cpp b/clang-tools-extra/clangd/FindSymbols.cpp
index 5244a4e893769..55f16b7085a6f 100644
--- a/clang-tools-extra/clangd/FindSymbols.cpp
+++ b/clang-tools-extra/clangd/FindSymbols.cpp
@@ -454,7 +454,7 @@ class DocumentOutline {
if (!MacroName.isValid() || !MacroName.isFileID())
continue;
// All conditions satisfied, add the macro.
- if (auto *Tok = AST.getTokens().spelledTokenAt(MacroName))
+ if (auto *Tok = AST.getTokens().spelledTokenContaining(MacroName))
CurParent = &CurParent->inMacro(
*Tok, SM, AST.getTokens().expansionStartingAt(Tok));
}
diff --git a/clang-tools-extra/clangd/IncludeCleaner.cpp b/clang-tools-extra/clangd/IncludeCleaner.cpp
index 01b47679790f1..dc5b7ec95db5f 100644
--- a/clang-tools-extra/clangd/IncludeCleaner.cpp
+++ b/clang-tools-extra/clangd/IncludeCleaner.cpp
@@ -303,7 +303,7 @@ collectMacroReferences(ParsedAST &AST) {
for (const auto &[_, Refs] : AST.getMacros().MacroRefs) {
for (const auto &Ref : Refs) {
auto Loc = SM.getComposedLoc(SM.getMainFileID(), Ref.StartOffset);
- const auto *Tok = AST.getTokens().spelledTokenAt(Loc);
+ const auto *Tok = AST.getTokens().spelledTokenContaining(Loc);
if (!Tok)
continue;
auto Macro = locateMacroAt(*Tok, PP);
diff --git a/clang-tools-extra/clangd/SemanticHighlighting.cpp b/clang-tools-extra/clangd/SemanticHighlighting.cpp
index eb025f21f3616..a366f1331c2d3 100644
--- a/clang-tools-extra/clangd/SemanticHighlighting.cpp
+++ b/clang-tools-extra/clangd/SemanticHighlighting.cpp
@@ -447,11 +447,10 @@ class HighlightingsBuilder {
if (!RLoc.isValid())
return;
- const auto *RTok = TB.spelledTokenAt(RLoc);
- // Handle `>>`. RLoc is always pointing at the right location, just change
- // the end to be offset by 1.
- // We'll either point at the beginning of `>>`, hence get a proper spelled
- // or point in the middle of `>>` hence get no spelled tok.
+ const auto *RTok = TB.spelledTokenContaining(RLoc);
+ // Handle `>>`. RLoc is either part of `>>` or a spelled token on its own
+ // `>`. If it's the former, slice to have length of 1, if latter use the
+ // token as-is.
if (!RTok || RTok->kind() == tok::greatergreater) {
Position Begin = sourceLocToPosition(SourceMgr, RLoc);
Position End = sourceLocToPosition(SourceMgr, RLoc.getLocWithOffset(1));
@@ -577,7 +576,7 @@ class HighlightingsBuilder {
return std::nullopt;
// We might have offsets in the main file that don't correspond to any
// spelled tokens.
- const auto *Tok = TB.spelledTokenAt(Loc);
+ const auto *Tok = TB.spelledTokenContaining(Loc);
if (!Tok)
return std::nullopt;
return halfOpenToRange(SourceMgr,
diff --git a/clang-tools-extra/clangd/XRefs.cpp b/clang-tools-extra/clangd/XRefs.cpp
index cd909266489a8..f94cadeffaa29 100644
--- a/clang-tools-extra/clangd/XRefs.cpp
+++ b/clang-tools-extra/clangd/XRefs.cpp
@@ -844,7 +844,7 @@ std::vector<DocumentLink> getDocumentLinks(ParsedAST &AST) {
if (Inc.Resolved.empty())
continue;
auto HashLoc = SM.getComposedLoc(SM.getMainFileID(), Inc.HashOffset);
- const auto *HashTok = AST.getTokens().spelledTokenAt(HashLoc);
+ const auto *HashTok = AST.getTokens().spelledTokenContaining(HashLoc);
assert(HashTok && "got inclusion at wrong offset");
const auto *IncludeTok = std::next(HashTok);
const auto *FileTok = std::next(IncludeTok);
@@ -938,7 +938,7 @@ class ReferenceFinder : public index::IndexDataConsumer {
CollectorOpts.CollectMainFileSymbols = true;
for (SourceLocation L : Locs) {
L = SM.getFileLoc(L);
- if (const auto *Tok = TB.spelledTokenAt(L))
+ if (const auto *Tok = TB.spelledTokenContaining(L))
References.push_back(
{*Tok, Roles,
SymbolCollector::getRefContainer(ASTNode.Parent, CollectorOpts)});
@@ -1216,7 +1216,7 @@ DocumentHighlight toHighlight(const ReferenceFinder::Reference &Ref,
std::optional<DocumentHighlight> toHighlight(SourceLocation Loc,
const syntax::TokenBuffer &TB) {
Loc = TB.sourceManager().getFileLoc(Loc);
- if (const auto *Tok = TB.spelledTokenAt(Loc)) {
+ if (const auto *Tok = TB.spelledTokenContaining(Loc)) {
DocumentHighlight Result;
Result.range = halfOpenToRange(
TB.sourceManager(),
@@ -1353,7 +1353,8 @@ maybeFindIncludeReferences(ParsedAST &AST, Position Pos,
Loc = SM.getIncludeLoc(SM.getFileID(Loc));
ReferencesResult::Reference Result;
- const auto *Token = AST.getTokens().spelledTokenAt(Loc);
+ const auto *Token = AST.getTokens().spelledTokenContaining(Loc);
+ assert(Token && "references expected token here");
Result.Loc.range = Range{sourceLocToPosition(SM, Token->location()),
sourceLocToPosition(SM, Token->endLocation())};
Result.Loc.uri = URIMainFile;
diff --git a/clang-tools-extra/clangd/refactor/Rename.cpp b/clang-tools-extra/clangd/refactor/Rename.cpp
index c0fc4453a3fcc..c85e13dbdfe97 100644
--- a/clang-tools-extra/clangd/refactor/Rename.cpp
+++ b/clang-tools-extra/clangd/refactor/Rename.cpp
@@ -748,7 +748,7 @@ std::vector<SymbolRange> collectRenameIdentifierRanges(
clangd::Range tokenRangeForLoc(ParsedAST &AST, SourceLocation TokLoc,
const SourceManager &SM,
const LangOptions &LangOpts) {
- const auto *Token = AST.getTokens().spelledTokenAt(TokLoc);
+ const auto *Token = AST.getTokens().spelledTokenContaining(TokLoc);
assert(Token && "rename expects spelled tokens");
clangd::Range Result;
Result.start = sourceLocToPosition(SM, Token->location());
diff --git a/clang-tools-extra/clangd/unittests/PreambleTests.cpp b/clang-tools-extra/clangd/unittests/PreambleTests.cpp
index 6420516e78557..16a2f9448b1ec 100644
--- a/clang-tools-extra/clangd/unittests/PreambleTests.cpp
+++ b/clang-tools-extra/clangd/unittests/PreambleTests.cpp
@@ -417,7 +417,7 @@ TEST(PreamblePatchTest, LocateMacroAtWorks) {
ASSERT_TRUE(AST);
const auto &SM = AST->getSourceManager();
- auto *MacroTok = AST->getTokens().spelledTokenAt(
+ auto *MacroTok = AST->getTokens().spelledTokenContaining(
SM.getComposedLoc(SM.getMainFileID(), Modified.point("use")));
ASSERT_TRUE(MacroTok);
@@ -441,7 +441,7 @@ TEST(PreamblePatchTest, LocateMacroAtDeletion) {
ASSERT_TRUE(AST);
const auto &SM = AST->getSourceManager();
- auto *MacroTok = AST->getTokens().spelledTokenAt(
+ auto *MacroTok = AST->getTokens().spelledTokenContaining(
SM.getComposedLoc(SM.getMainFileID(), Modified.point()));
ASSERT_TRUE(MacroTok);
@@ -512,9 +512,10 @@ TEST(PreamblePatchTest, RefsToMacros) {
ExpectedLocations.push_back(referenceRangeIs(R));
for (const auto &P : Modified.points()) {
- auto *MacroTok = AST->getTokens().spelledTokenAt(SM.getComposedLoc(
- SM.getMainFileID(),
- llvm::cantFail(positionToOffset(Modified.code(), P))));
+ auto *MacroTok =
+ AST->getTokens().spelledTokenContaining(SM.getComposedLoc(
+ SM.getMainFileID(),
+ llvm::cantFail(positionToOffset(Modified.code(), P))));
ASSERT_TRUE(MacroTok);
EXPECT_THAT(findReferences(*AST, P, 0).References,
testing::ElementsAreArray(ExpectedLocations));
diff --git a/clang-tools-extra/clangd/unittests/XRefsTests.cpp b/clang-tools-extra/clangd/unittests/XRefsTests.cpp
index f53cbf01b7992..cbceb9a343f87 100644
--- a/clang-tools-extra/clangd/unittests/XRefsTests.cpp
+++ b/clang-tools-extra/clangd/unittests/XRefsTests.cpp
@@ -2173,6 +2173,11 @@ TEST(FindReferences, WithinAST) {
using $def[[MyTypeD^ef]] = int;
enum MyEnum : $(MyEnum)[[MyTy^peDef]] { };
)cpp",
+ // UDL
+ R"cpp(
+ bool $decl[[operator]]"" _u^dl(unsigned long long value);
+ bool x = $(x)[[1_udl]];
+ )cpp",
};
for (const char *Test : Tests)
checkFindRefs(Test);
@@ -2358,7 +2363,13 @@ TEST(FindReferences, UsedSymbolsFromInclude) {
R"cpp([[#in^clude <vector>]]
std::[[vector]]<int> vec;
- )cpp"};
+ )cpp",
+
+ R"cpp(
+ [[#include ^"udl_header.h"]]
+ auto x = [[1_b]];
+ )cpp",
+ };
for (const char *Test : Tests) {
Annotations T(Test);
auto TU = TestTU::withCode(T.code());
@@ -2375,6 +2386,9 @@ TEST(FindReferences, UsedSymbolsFromInclude) {
class vector{};
}
)cpp");
+ TU.AdditionalFiles["udl_header.h"] = guard(R"cpp(
+ bool operator"" _b(unsigned long long value);
+ )cpp");
TU.ExtraArgs.push_back("-isystem" + testPath("system"));
auto AST = TU.build();
diff --git a/clang/include/clang/Tooling/Syntax/Tokens.h b/clang/include/clang/Tooling/Syntax/Tokens.h
index b1bdefed7c97f..f71b8d67bfea4 100644
--- a/clang/include/clang/Tooling/Syntax/Tokens.h
+++ b/clang/include/clang/Tooling/Syntax/Tokens.h
@@ -292,9 +292,9 @@ class TokenBuffer {
/// "DECL", "(", "a", ")", ";"}
llvm::ArrayRef<syntax::Token> spelledTokens(FileID FID) const;
- /// Returns the spelled Token starting at Loc, if there are no such tokens
+ /// Returns the spelled Token containing the Loc, if there are no such tokens
/// returns nullptr.
- const syntax::Token *spelledTokenAt(SourceLocation Loc) const;
+ const syntax::Token *spelledTokenContaining(SourceLocation Loc) const;
/// Get all tokens that expand a macro in \p FID. For the following input
/// #define FOO B
diff --git a/clang/lib/Tooling/Syntax/Tokens.cpp b/clang/lib/Tooling/Syntax/Tokens.cpp
index 8d32c45a4a70c..0a656dff38421 100644
--- a/clang/lib/Tooling/Syntax/Tokens.cpp
+++ b/clang/lib/Tooling/Syntax/Tokens.cpp
@@ -383,12 +383,13 @@ llvm::ArrayRef<syntax::Token> TokenBuffer::spelledTokens(FileID FID) const {
return It->second.SpelledTokens;
}
-const syntax::Token *TokenBuffer::spelledTokenAt(SourceLocation Loc) const {
+const syntax::Token *
+TokenBuffer::spelledTokenContaining(SourceLocation Loc) const {
assert(Loc.isFileID());
const auto *Tok = llvm::partition_point(
spelledTokens(SourceMgr->getFileID(Loc)),
- [&](const syntax::Token &Tok) { return Tok.location() < Loc; });
- if (!Tok || Tok->location() != Loc)
+ [&](const syntax::Token &Tok) { return Tok.endLocation() <= Loc; });
+ if (!Tok || Loc < Tok->location())
return nullptr;
return Tok;
}
diff --git a/clang/unittests/Tooling/Syntax/TokensTest.cpp b/clang/unittests/Tooling/Syntax/TokensTest.cpp
index 42f5169713965..dc8a11dbc345c 100644
--- a/clang/unittests/Tooling/Syntax/TokensTest.cpp
+++ b/clang/unittests/Tooling/Syntax/TokensTest.cpp
@@ -374,11 +374,24 @@ TEST_F(TokenCollectorTest, Locations) {
auto StartLoc = SourceMgr->getLocForStartOfFile(SourceMgr->getMainFileID());
for (auto &R : Code.ranges()) {
- EXPECT_THAT(Buffer.spelledTokenAt(StartLoc.getLocWithOffset(R.Begin)),
- Pointee(RangeIs(R)));
+ EXPECT_THAT(
+ Buffer.spelledTokenContaining(StartLoc.getLocWithOffset(R.Begin)),
+ Pointee(RangeIs(R)));
}
}
+TEST_F(TokenCollectorTest, LocationInMiddleOfSpelledToken) {
+ llvm::Annotations Code(R"cpp(
+ int foo = [[baa^aar]];
+ )cpp");
+ recordTokens(Code.code());
+ // Check spelled tokens.
+ auto StartLoc = SourceMgr->getLocForStartOfFile(SourceMgr->getMainFileID());
+ EXPECT_THAT(
+ Buffer.spelledTokenContaining(StartLoc.getLocWithOffset(Code.point())),
+ Pointee(RangeIs(Code.range())));
+}
+
TEST_F(TokenCollectorTest, MacroDirectives) {
// Macro directives are not stored anywhere at the moment.
std::string Code = R"cpp(
More information about the cfe-commits
mailing list