[cfe-commits] r117001 - in /cfe/trunk: test/Index/annotate-tokens-include.c test/Index/annotate-tokens-pp.c test/Index/annotate-tokens.m tools/libclang/CIndex.cpp
Douglas Gregor
dgregor at apple.com
Wed Oct 20 23:10:04 PDT 2010
Author: dgregor
Date: Thu Oct 21 01:10:04 2010
New Revision: 117001
URL: http://llvm.org/viewvc/llvm-project?rev=117001&view=rev
Log:
Improve the implementation of libclang's token-annotation logic for
entities in the preprocessing record. Previously, we would only end up
getting the first token of a preprocessing record annotated
correctly. For example, given
#include "foo.h"
we would only get the '#' annotated as an inclusion directive; the
'include' and '"foo.h"' tokens would be given the general 'processing
directive' annotation.
Now, we get proper annotations for entities in the preprocessing
record.
Modified:
cfe/trunk/test/Index/annotate-tokens-include.c
cfe/trunk/test/Index/annotate-tokens-pp.c
cfe/trunk/test/Index/annotate-tokens.m
cfe/trunk/tools/libclang/CIndex.cpp
Modified: cfe/trunk/test/Index/annotate-tokens-include.c
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/Index/annotate-tokens-include.c?rev=117001&r1=117000&r2=117001&view=diff
==============================================================================
--- cfe/trunk/test/Index/annotate-tokens-include.c (original)
+++ cfe/trunk/test/Index/annotate-tokens-include.c Thu Oct 21 01:10:04 2010
@@ -2,6 +2,6 @@
// RUN: c-index-test -test-annotate-tokens=%s:1:1:2:1 %s | FileCheck %s
// CHECK: Punctuation: "#" [1:1 - 1:2] inclusion directive=annotate-tokens-include.h
-// CHECK: Identifier: "include" [1:2 - 1:9] preprocessing directive=
-// CHECK: Literal: ""annotate-tokens-include.h"" [1:10 - 1:37] preprocessing directive=
+// CHECK: Identifier: "include" [1:2 - 1:9] inclusion directive=annotate-tokens-include.h
+// CHECK: Literal: ""annotate-tokens-include.h"" [1:10 - 1:37] inclusion directive=annotate-tokens-include.h
Modified: cfe/trunk/test/Index/annotate-tokens-pp.c
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/Index/annotate-tokens-pp.c?rev=117001&r1=117000&r2=117001&view=diff
==============================================================================
--- cfe/trunk/test/Index/annotate-tokens-pp.c (original)
+++ cfe/trunk/test/Index/annotate-tokens-pp.c Thu Oct 21 01:10:04 2010
@@ -29,27 +29,27 @@
// CHECK: Punctuation: "#" [2:1 - 2:2] preprocessing directive=
// CHECK: Identifier: "define" [2:2 - 2:8] preprocessing directive=
// CHECK: Identifier: "STILL_NOTHING" [2:9 - 2:22] macro definition=STILL_NOTHING
-// CHECK: Identifier: "NOTHING" [2:23 - 2:30] preprocessing directive=
-// CHECK: Punctuation: "(" [2:30 - 2:31] preprocessing directive=
-// CHECK: Identifier: "honk" [2:31 - 2:35] preprocessing directive=
-// CHECK: Punctuation: "," [2:35 - 2:36] preprocessing directive=
-// CHECK: Identifier: "warble" [2:36 - 2:42] preprocessing directive=
-// CHECK: Punctuation: ")" [2:42 - 2:43] preprocessing directive=
+// CHECK: Identifier: "NOTHING" [2:23 - 2:30] macro definition=STILL_NOTHING
+// CHECK: Punctuation: "(" [2:30 - 2:31] macro definition=STILL_NOTHING
+// CHECK: Identifier: "honk" [2:31 - 2:35] macro definition=STILL_NOTHING
+// CHECK: Punctuation: "," [2:35 - 2:36] macro definition=STILL_NOTHING
+// CHECK: Identifier: "warble" [2:36 - 2:42] macro definition=STILL_NOTHING
+// CHECK: Punctuation: ")" [2:42 - 2:43] macro definition=STILL_NOTHING
// CHECK: Punctuation: "#" [3:1 - 3:2] preprocessing directive=
// CHECK: Identifier: "define" [3:2 - 3:8] preprocessing directive=
// CHECK: Identifier: "BAR" [3:9 - 3:12] macro definition=BAR
-// CHECK: Identifier: "baz" [3:13 - 3:16] preprocessing directive=
+// CHECK: Identifier: "baz" [3:13 - 3:16] macro definition=BAR
// CHECK: Punctuation: "#" [4:1 - 4:2] preprocessing directive=
// CHECK: Identifier: "define" [4:2 - 4:8] preprocessing directive=
// CHECK: Identifier: "WIBBLE" [4:9 - 4:15] macro definition=WIBBLE
-// CHECK: Punctuation: "(" [4:15 - 4:16] preprocessing directive=
-// CHECK: Identifier: "X" [4:16 - 4:17] preprocessing directive=
-// CHECK: Punctuation: "," [4:17 - 4:18] preprocessing directive=
-// CHECK: Identifier: "Y" [4:19 - 4:20] preprocessing directive=
-// CHECK: Punctuation: ")" [4:20 - 4:21] preprocessing directive=
-// CHECK: Identifier: "X" [4:22 - 4:23] preprocessing directive=
-// CHECK: Punctuation: "##" [4:23 - 4:25] preprocessing directive=
-// CHECK: Identifier: "Y" [4:25 - 4:26] preprocessing directive=
+// CHECK: Punctuation: "(" [4:15 - 4:16] macro definition=WIBBLE
+// CHECK: Identifier: "X" [4:16 - 4:17] macro definition=WIBBLE
+// CHECK: Punctuation: "," [4:17 - 4:18] macro definition=WIBBLE
+// CHECK: Identifier: "Y" [4:19 - 4:20] macro definition=WIBBLE
+// CHECK: Punctuation: ")" [4:20 - 4:21] macro definition=WIBBLE
+// CHECK: Identifier: "X" [4:22 - 4:23] macro definition=WIBBLE
+// CHECK: Punctuation: "##" [4:23 - 4:25] macro definition=WIBBLE
+// CHECK: Identifier: "Y" [4:25 - 4:26] macro definition=WIBBLE
// CHECK: Identifier: "NOTHING" [5:1 - 5:8] macro instantiation=NOTHING:1:9
// CHECK: Punctuation: "(" [5:8 - 5:9]
// CHECK: Identifier: "more" [5:9 - 5:13]
@@ -69,31 +69,31 @@
// CHECK: Identifier: "STILL_NOTHING" [6:9 - 6:22] macro instantiation=STILL_NOTHING:2:9
// CHECK: Punctuation: ";" [6:22 - 6:23]
// CHECK: Punctuation: "#" [7:1 - 7:2] inclusion directive=foo.h
-// CHECK: Identifier: "include" [7:2 - 7:9] preprocessing directive=
-// CHECK: Literal: ""foo.h"" [7:10 - 7:17] preprocessing directive=
+// CHECK: Identifier: "include" [7:2 - 7:9] inclusion directive=foo.h
+// CHECK: Literal: ""foo.h"" [7:10 - 7:17] inclusion directive=foo.h
// CHECK: Punctuation: "#" [8:1 - 8:2] preprocessing directive=
// CHECK: Identifier: "undef" [8:2 - 8:7] preprocessing directive=
// CHECK: Identifier: "BAR" [8:8 - 8:11] preprocessing directive=
// CHECK: Punctuation: "#" [10:1 - 10:2] preprocessing directive=
// CHECK: Identifier: "define" [10:2 - 10:8] preprocessing directive=
// CHECK: Identifier: "REVERSE_MACRO" [10:9 - 10:22] macro definition=REVERSE_MACRO
-// CHECK: Punctuation: "(" [10:22 - 10:23] preprocessing directive=
-// CHECK: Identifier: "x" [10:23 - 10:24] preprocessing directive=
-// CHECK: Punctuation: "," [10:24 - 10:25] preprocessing directive=
-// CHECK: Identifier: "y" [10:25 - 10:26] preprocessing directive=
-// CHECK: Punctuation: ")" [10:26 - 10:27] preprocessing directive=
-// CHECK: Identifier: "y" [10:28 - 10:29] preprocessing directive=
-// CHECK: Punctuation: "+" [10:30 - 10:31] preprocessing directive=
-// CHECK: Identifier: "x" [10:32 - 10:33] preprocessing directive=
+// CHECK: Punctuation: "(" [10:22 - 10:23] macro definition=REVERSE_MACRO
+// CHECK: Identifier: "x" [10:23 - 10:24] macro definition=REVERSE_MACRO
+// CHECK: Punctuation: "," [10:24 - 10:25] macro definition=REVERSE_MACRO
+// CHECK: Identifier: "y" [10:25 - 10:26] macro definition=REVERSE_MACRO
+// CHECK: Punctuation: ")" [10:26 - 10:27] macro definition=REVERSE_MACRO
+// CHECK: Identifier: "y" [10:28 - 10:29] macro definition=REVERSE_MACRO
+// CHECK: Punctuation: "+" [10:30 - 10:31] macro definition=REVERSE_MACRO
+// CHECK: Identifier: "x" [10:32 - 10:33] macro definition=REVERSE_MACRO
// CHECK: Punctuation: "#" [11:1 - 11:2] preprocessing directive=
// CHECK: Identifier: "define" [11:2 - 11:8] preprocessing directive=
// CHECK: Identifier: "TWICE_MACRO" [11:9 - 11:20] macro definition=TWICE_MACRO
-// CHECK: Punctuation: "(" [11:20 - 11:21] preprocessing directive=
-// CHECK: Identifier: "y" [11:21 - 11:22] preprocessing directive=
-// CHECK: Punctuation: ")" [11:22 - 11:23] preprocessing directive=
-// CHECK: Identifier: "y" [11:24 - 11:25] preprocessing directive=
-// CHECK: Punctuation: "+" [11:26 - 11:27] preprocessing directive=
-// CHECK: Identifier: "y" [11:28 - 11:29] preprocessing directive=
+// CHECK: Punctuation: "(" [11:20 - 11:21] macro definition=TWICE_MACRO
+// CHECK: Identifier: "y" [11:21 - 11:22] macro definition=TWICE_MACRO
+// CHECK: Punctuation: ")" [11:22 - 11:23] macro definition=TWICE_MACRO
+// CHECK: Identifier: "y" [11:24 - 11:25] macro definition=TWICE_MACRO
+// CHECK: Punctuation: "+" [11:26 - 11:27] macro definition=TWICE_MACRO
+// CHECK: Identifier: "y" [11:28 - 11:29] macro definition=TWICE_MACRO
// CHECK: Keyword: "void" [13:1 - 13:5] FunctionDecl=test_macro_args:13:6 (Definition)
// CHECK: Identifier: "test_macro_args" [13:6 - 13:21] FunctionDecl=test_macro_args:13:6 (Definition)
// CHECK: Punctuation: "(" [13:21 - 13:22] FunctionDecl=test_macro_args:13:6 (Definition)
@@ -140,23 +140,23 @@
// CHECK: Punctuation: "#" [21:1 - 21:2] preprocessing directive=
// CHECK: Identifier: "define" [21:2 - 21:8] preprocessing directive=
// CHECK: Identifier: "fun_with_macro_bodies" [21:9 - 21:30] macro definition=fun_with_macro_bodies
-// CHECK: Punctuation: "(" [21:30 - 21:31] preprocessing directive=
-// CHECK: Identifier: "x" [21:31 - 21:32] preprocessing directive=
-// CHECK: Punctuation: "," [21:32 - 21:33] preprocessing directive=
-// CHECK: Identifier: "y" [21:34 - 21:35] preprocessing directive=
-// CHECK: Punctuation: ")" [21:35 - 21:36] preprocessing directive=
-// CHECK: Keyword: "do" [21:37 - 21:39] preprocessing directive=
-// CHECK: Punctuation: "{" [21:40 - 21:41] preprocessing directive=
-// CHECK: Keyword: "if" [21:42 - 21:44] preprocessing directive=
-// CHECK: Punctuation: "(" [21:45 - 21:46] preprocessing directive=
-// CHECK: Identifier: "x" [21:46 - 21:47] preprocessing directive=
-// CHECK: Punctuation: ")" [21:47 - 21:48] preprocessing directive=
-// CHECK: Identifier: "y" [21:49 - 21:50] preprocessing directive=
-// CHECK: Punctuation: "}" [21:51 - 21:52] preprocessing directive=
-// CHECK: Keyword: "while" [21:53 - 21:58] preprocessing directive=
-// CHECK: Punctuation: "(" [21:59 - 21:60] preprocessing directive=
-// CHECK: Literal: "0" [21:60 - 21:61] preprocessing directive=
-// CHECK: Punctuation: ")" [21:61 - 21:62] preprocessing directive=
+// CHECK: Punctuation: "(" [21:30 - 21:31] macro definition=fun_with_macro_bodies
+// CHECK: Identifier: "x" [21:31 - 21:32] macro definition=fun_with_macro_bodies
+// CHECK: Punctuation: "," [21:32 - 21:33] macro definition=fun_with_macro_bodies
+// CHECK: Identifier: "y" [21:34 - 21:35] macro definition=fun_with_macro_bodies
+// CHECK: Punctuation: ")" [21:35 - 21:36] macro definition=fun_with_macro_bodies
+// CHECK: Keyword: "do" [21:37 - 21:39] macro definition=fun_with_macro_bodies
+// CHECK: Punctuation: "{" [21:40 - 21:41] macro definition=fun_with_macro_bodies
+// CHECK: Keyword: "if" [21:42 - 21:44] macro definition=fun_with_macro_bodies
+// CHECK: Punctuation: "(" [21:45 - 21:46] macro definition=fun_with_macro_bodies
+// CHECK: Identifier: "x" [21:46 - 21:47] macro definition=fun_with_macro_bodies
+// CHECK: Punctuation: ")" [21:47 - 21:48] macro definition=fun_with_macro_bodies
+// CHECK: Identifier: "y" [21:49 - 21:50] macro definition=fun_with_macro_bodies
+// CHECK: Punctuation: "}" [21:51 - 21:52] macro definition=fun_with_macro_bodies
+// CHECK: Keyword: "while" [21:53 - 21:58] macro definition=fun_with_macro_bodies
+// CHECK: Punctuation: "(" [21:59 - 21:60] macro definition=fun_with_macro_bodies
+// CHECK: Literal: "0" [21:60 - 21:61] macro definition=fun_with_macro_bodies
+// CHECK: Punctuation: ")" [21:61 - 21:62] macro definition=fun_with_macro_bodies
// CHECK: Keyword: "void" [23:1 - 23:5] FunctionDecl=test:23:6 (Definition)
// CHECK: Identifier: "test" [23:6 - 23:10] FunctionDecl=test:23:6 (Definition)
// CHECK: Punctuation: "(" [23:10 - 23:11] FunctionDecl=test:23:6 (Definition)
Modified: cfe/trunk/test/Index/annotate-tokens.m
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/Index/annotate-tokens.m?rev=117001&r1=117000&r2=117001&view=diff
==============================================================================
--- cfe/trunk/test/Index/annotate-tokens.m (original)
+++ cfe/trunk/test/Index/annotate-tokens.m Thu Oct 21 01:10:04 2010
@@ -285,7 +285,7 @@
// CHECK: Punctuation: "#" [63:1 - 63:2] preprocessing directive=
// CHECK: Identifier: "define" [63:2 - 63:8] preprocessing directive=
// CHECK: Identifier: "VAL" [63:9 - 63:12] macro definition=VAL
-// CHECK: Literal: "0" [63:13 - 63:14] preprocessing directive=
+// CHECK: Literal: "0" [63:13 - 63:14] macro definition=VAL
// CHECK: Punctuation: "@" [65:1 - 65:2] ObjCInterfaceDecl=R7974151:65:12
// CHECK: Keyword: "interface" [65:2 - 65:11] ObjCInterfaceDecl=R7974151:65:12
// CHECK: Identifier: "R7974151" [65:12 - 65:20] ObjCInterfaceDecl=R7974151:65:12
Modified: cfe/trunk/tools/libclang/CIndex.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/tools/libclang/CIndex.cpp?rev=117001&r1=117000&r2=117001&view=diff
==============================================================================
--- cfe/trunk/tools/libclang/CIndex.cpp (original)
+++ cfe/trunk/tools/libclang/CIndex.cpp Thu Oct 21 01:10:04 2010
@@ -3657,6 +3657,7 @@
CXCursor *Cursors;
unsigned NumTokens;
unsigned TokIdx;
+ unsigned PreprocessingTokIdx;
CursorVisitor AnnotateVis;
SourceManager &SrcMgr;
@@ -3672,7 +3673,7 @@
CXToken *tokens, CXCursor *cursors, unsigned numTokens,
ASTUnit *CXXUnit, SourceRange RegionOfInterest)
: Annotated(annotated), Tokens(tokens), Cursors(cursors),
- NumTokens(numTokens), TokIdx(0),
+ NumTokens(numTokens), TokIdx(0), PreprocessingTokIdx(0),
AnnotateVis(CXXUnit, AnnotateTokensVisitor, this,
Decl::MaxPCHLevel, RegionOfInterest),
SrcMgr(CXXUnit->getSourceManager()) {}
@@ -3690,7 +3691,9 @@
for (unsigned I = 0 ; I < TokIdx ; ++I) {
AnnotateTokensData::iterator Pos = Annotated.find(Tokens[I].int_data[1]);
- if (Pos != Annotated.end())
+ if (Pos != Annotated.end() &&
+ (clang_isInvalid(Cursors[I].kind) ||
+ Pos->second.kind != CXCursor_PreprocessingDirective))
Cursors[I] = Pos->second;
}
@@ -3706,16 +3709,66 @@
}
enum CXChildVisitResult
-AnnotateTokensWorker::Visit(CXCursor cursor, CXCursor parent) {
+AnnotateTokensWorker::Visit(CXCursor cursor, CXCursor parent) {
CXSourceLocation Loc = clang_getCursorLocation(cursor);
- // We can always annotate a preprocessing directive/macro instantiation.
- if (clang_isPreprocessing(cursor.kind)) {
- Annotated[Loc.int_data] = cursor;
+ SourceRange cursorRange = getRawCursorExtent(cursor);
+
+ if (clang_isPreprocessing(cursor.kind)) {
+ // For macro instantiations, just note where the beginning of the macro
+ // instantiation occurs.
+ if (cursor.kind == CXCursor_MacroInstantiation) {
+ Annotated[Loc.int_data] = cursor;
+ return CXChildVisit_Recurse;
+ }
+
+ if (cursorRange.isInvalid())
+ return CXChildVisit_Continue;
+
+ // Items in the preprocessing record are kept separate from items in
+ // declarations, so we keep a separate token index.
+ unsigned SavedTokIdx = TokIdx;
+ TokIdx = PreprocessingTokIdx;
+
+ // Skip tokens up until we catch up to the beginning of the preprocessing
+ // entry.
+ while (MoreTokens()) {
+ const unsigned I = NextToken();
+ SourceLocation TokLoc = GetTokenLoc(I);
+ switch (LocationCompare(SrcMgr, TokLoc, cursorRange)) {
+ case RangeBefore:
+ AdvanceToken();
+ continue;
+ case RangeAfter:
+ case RangeOverlap:
+ break;
+ }
+ break;
+ }
+
+ // Look at all of the tokens within this range.
+ while (MoreTokens()) {
+ const unsigned I = NextToken();
+ SourceLocation TokLoc = GetTokenLoc(I);
+ switch (LocationCompare(SrcMgr, TokLoc, cursorRange)) {
+ case RangeBefore:
+ assert(0 && "Infeasible");
+ case RangeAfter:
+ break;
+ case RangeOverlap:
+ Cursors[I] = cursor;
+ AdvanceToken();
+ continue;
+ }
+ break;
+ }
+
+ // Save the preprocessing token index; restore the non-preprocessing
+ // token index.
+ PreprocessingTokIdx = TokIdx;
+ TokIdx = SavedTokIdx;
return CXChildVisit_Recurse;
}
- SourceRange cursorRange = getRawCursorExtent(cursor);
-
if (cursorRange.isInvalid())
return CXChildVisit_Continue;
@@ -3753,7 +3806,7 @@
// directive. Here we assume that the default construction of CXCursor
// results in CXCursor.kind being an initialized value (i.e., 0). If
// this isn't the case, we can fix by doing lookup + insertion.
-
+
CXCursor &oldC = Annotated[rawEncoding];
if (!clang_isPreprocessing(oldC.kind))
oldC = cursor;
@@ -3815,6 +3868,7 @@
for (unsigned I = BeforeChildren; I != AfterChildren; ++I) {
if (!clang_isInvalid(clang_getCursorKind(Cursors[I])))
break;
+
Cursors[I] = cursor;
}
// Scan the tokens that are at the end of the cursor, but are not captured
@@ -3841,14 +3895,14 @@
if (NumTokens == 0 || !Tokens || !Cursors)
return;
+ // Any token we don't specifically annotate will have a NULL cursor.
+ CXCursor C = clang_getNullCursor();
+ for (unsigned I = 0; I != NumTokens; ++I)
+ Cursors[I] = C;
+
ASTUnit *CXXUnit = static_cast<ASTUnit *>(TU);
- if (!CXXUnit) {
- // Any token we don't specifically annotate will have a NULL cursor.
- const CXCursor &C = clang_getNullCursor();
- for (unsigned I = 0; I != NumTokens; ++I)
- Cursors[I] = C;
+ if (!CXXUnit)
return;
- }
ASTUnit::ConcurrencyCheck Check(*CXXUnit);
More information about the cfe-commits
mailing list