[cfe-commits] r98786 - in /cfe/trunk: include/clang-c/Index.h test/Index/annotate-tokens-pp.c tools/CIndex/CIndex.cpp tools/CIndex/CIndex.exports tools/CIndex/CXCursor.cpp tools/CIndex/CXCursor.h
Douglas Gregor
dgregor at apple.com
Wed Mar 17 17:42:48 PDT 2010
Author: dgregor
Date: Wed Mar 17 19:42:48 2010
New Revision: 98786
URL: http://llvm.org/viewvc/llvm-project?rev=98786&view=rev
Log:
Experimental stab at using relexing to identify preprocessor
directives while annotating tokens in CIndex. This functionality
should probably be factored out of this routine, but we're not there
yet.
Added:
cfe/trunk/test/Index/annotate-tokens-pp.c
Modified:
cfe/trunk/include/clang-c/Index.h
cfe/trunk/tools/CIndex/CIndex.cpp
cfe/trunk/tools/CIndex/CIndex.exports
cfe/trunk/tools/CIndex/CXCursor.cpp
cfe/trunk/tools/CIndex/CXCursor.h
Modified: cfe/trunk/include/clang-c/Index.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang-c/Index.h?rev=98786&r1=98785&r2=98786&view=diff
==============================================================================
--- cfe/trunk/include/clang-c/Index.h (original)
+++ cfe/trunk/include/clang-c/Index.h Wed Mar 17 19:42:48 2010
@@ -796,7 +796,13 @@
CXCursor_IBActionAttr = 401,
CXCursor_IBOutletAttr = 402,
- CXCursor_LastAttr = CXCursor_IBOutletAttr
+ CXCursor_LastAttr = CXCursor_IBOutletAttr,
+
+ /* Preprocessing */
+ CXCursor_PreprocessingDirective = 500,
+
+ CXCursor_FirstPreprocessing = CXCursor_PreprocessingDirective,
+ CXCursor_LastPreprocessing = CXCursor_PreprocessingDirective
};
/**
@@ -889,6 +895,12 @@
CINDEX_LINKAGE unsigned clang_isTranslationUnit(enum CXCursorKind);
/***
+ * \brief Determine whether the given cursor represents a preprocessing
+ * element, such as a preprocessor directive or macro instantiation.
+ */
+CINDEX_LINKAGE unsigned clang_isPreprocessing(enum CXCursorKind);
+
+/***
* \brief Determine whether the given cursor represents a currently
* unexposed piece of the AST (e.g., CXCursor_UnexposedStmt).
*/
Added: cfe/trunk/test/Index/annotate-tokens-pp.c
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/Index/annotate-tokens-pp.c?rev=98786&view=auto
==============================================================================
--- cfe/trunk/test/Index/annotate-tokens-pp.c (added)
+++ cfe/trunk/test/Index/annotate-tokens-pp.c Wed Mar 17 19:42:48 2010
@@ -0,0 +1,31 @@
+#define BAR baz
+#define WIBBLE(X, Y)
+WIBBLE(int, float)
+int BAR;
+#include "foo.h"
+
+// RUN: c-index-test -test-annotate-tokens=%s:1:1:6:1 -I%S/Inputs %s | FileCheck %s
+// CHECK: Punctuation: "#" [1:1 - 1:2] preprocessing directive=
+// CHECK: Identifier: "define" [1:2 - 1:8] preprocessing directive=
+// CHECK: Identifier: "BAR" [1:9 - 1:12] preprocessing directive=
+// CHECK: Identifier: "baz" [1:13 - 1:16] preprocessing directive=
+// CHECK: Punctuation: "#" [2:1 - 2:2] preprocessing directive=
+// CHECK: Identifier: "define" [2:2 - 2:8] preprocessing directive=
+// CHECK: Identifier: "WIBBLE" [2:9 - 2:15] preprocessing directive=
+// CHECK: Punctuation: "(" [2:15 - 2:16] preprocessing directive=
+// CHECK: Identifier: "X" [2:16 - 2:17] preprocessing directive=
+// CHECK: Punctuation: "," [2:17 - 2:18] preprocessing directive=
+// CHECK: Identifier: "Y" [2:19 - 2:20] preprocessing directive=
+// CHECK: Punctuation: ")" [2:20 - 2:21] preprocessing directive=
+// CHECK: Identifier: "WIBBLE" [3:1 - 3:7]
+// CHECK: Punctuation: "(" [3:7 - 3:8]
+// CHECK: Keyword: "int" [3:8 - 3:11]
+// CHECK: Punctuation: "," [3:11 - 3:12]
+// CHECK: Keyword: "float" [3:13 - 3:18]
+// CHECK: Punctuation: ")" [3:18 - 3:19]
+// CHECK: Keyword: "int" [4:1 - 4:4]
+// CHECK: Identifier: "BAR" [4:5 - 4:8]
+// CHECK: Punctuation: ";" [4:8 - 4:9]
+// CHECK: Punctuation: "#" [5:1 - 5:2] preprocessing directive=
+// CHECK: Identifier: "include" [5:2 - 5:9] preprocessing directive=
+// CHECK: Literal: ""foo.h"" [5:10 - 5:17] preprocessing directive=
Modified: cfe/trunk/tools/CIndex/CIndex.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/tools/CIndex/CIndex.cpp?rev=98786&r1=98785&r2=98786&view=diff
==============================================================================
--- cfe/trunk/tools/CIndex/CIndex.cpp (original)
+++ cfe/trunk/tools/CIndex/CIndex.cpp Wed Mar 17 19:42:48 2010
@@ -1518,8 +1518,10 @@
return createCXString("UnexposedAttr");
case CXCursor_IBActionAttr:
return createCXString("attribute(ibaction)");
- case CXCursor_IBOutletAttr:
- return createCXString("attribute(iboutlet)");
+ case CXCursor_IBOutletAttr:
+ return createCXString("attribute(iboutlet)");
+ case CXCursor_PreprocessingDirective:
+ return createCXString("preprocessing directive");
}
llvm_unreachable("Unhandled CXCursorKind");
@@ -1590,6 +1592,10 @@
return K == CXCursor_TranslationUnit;
}
+unsigned clang_isPreprocessing(enum CXCursorKind K) {
+ return K >= CXCursor_FirstPreprocessing && K <= CXCursor_LastPreprocessing;
+}
+
unsigned clang_isUnexposed(enum CXCursorKind K) {
switch (K) {
case CXCursor_UnexposedDecl:
@@ -1642,6 +1648,11 @@
return cxloc::translateSourceLocation(getCursorContext(C),
getLocationFromExpr(getCursorExpr(C)));
+ if (C.kind == CXCursor_PreprocessingDirective) {
+ SourceLocation L = cxcursor::getCursorPreprocessingDirective(C).getBegin();
+ return cxloc::translateSourceLocation(getCursorContext(C), L);
+ }
+
if (!getCursorDecl(C))
return clang_getNullLocation();
@@ -1693,6 +1704,11 @@
return cxloc::translateSourceRange(getCursorContext(C),
getCursorStmt(C)->getSourceRange());
+ if (C.kind == CXCursor_PreprocessingDirective) {
+ SourceRange R = cxcursor::getCursorPreprocessingDirective(C);
+ return cxloc::translateSourceRange(getCursorContext(C), R);
+ }
+
if (!getCursorDecl(C))
return clang_getNullRange();
@@ -2216,7 +2232,8 @@
ASTUnit::ConcurrencyCheck Check(*CXXUnit);
- // Annotate all of the source locations in the region of interest that map
+ // Annotate all of the source locations in the region of interest that map to
+ // a specific cursor.
SourceRange RegionOfInterest;
RegionOfInterest.setBegin(
cxloc::translateSourceLocation(clang_getTokenLocation(TU, Tokens[0])));
@@ -2224,23 +2241,114 @@
= cxloc::translateSourceLocation(clang_getTokenLocation(TU,
Tokens[NumTokens - 1]));
RegionOfInterest.setEnd(CXXUnit->getPreprocessor().getLocForEndOfToken(End));
- // FIXME: Would be great to have a "hint" cursor, then walk from that
- // hint cursor upward until we find a cursor whose source range encloses
- // the region of interest, rather than starting from the translation unit.
+
AnnotateTokensData Annotated;
CXCursor Parent = clang_getTranslationUnitCursor(CXXUnit);
CursorVisitor AnnotateVis(CXXUnit, AnnotateTokensVisitor, &Annotated,
Decl::MaxPCHLevel, RegionOfInterest);
AnnotateVis.VisitChildren(Parent);
+ // Look for macro instantiations and preprocessing directives in the
+ // source range containing the annotated tokens. We do this by re-lexing the
+ // tokens in the source range.
+ SourceManager &SourceMgr = CXXUnit->getSourceManager();
+ std::pair<FileID, unsigned> BeginLocInfo
+ = SourceMgr.getDecomposedLoc(RegionOfInterest.getBegin());
+ std::pair<FileID, unsigned> EndLocInfo
+ = SourceMgr.getDecomposedLoc(RegionOfInterest.getEnd());
+
+ bool RelexOkay = true;
+
+ // Cannot re-tokenize across files.
+ if (BeginLocInfo.first != EndLocInfo.first)
+ RelexOkay = false;
+
+ llvm::StringRef Buffer;
+ if (RelexOkay) {
+ // Create a lexer
+ bool Invalid = false;
+ Buffer = SourceMgr.getBufferData(BeginLocInfo.first, &Invalid);
+ if (Invalid)
+ RelexOkay = false;
+ }
+
+ if (RelexOkay) {
+ Lexer Lex(SourceMgr.getLocForStartOfFile(BeginLocInfo.first),
+ CXXUnit->getASTContext().getLangOptions(),
+ Buffer.begin(), Buffer.data() + BeginLocInfo.second, Buffer.end());
+ Lex.SetCommentRetentionState(true);
+
+ // Lex tokens in raw mode until we hit the end of the range, to avoid
+ // entering #includes or expanding macros.
+ std::vector<Token> TokenStream;
+ const char *EffectiveBufferEnd = Buffer.data() + EndLocInfo.second;
+ Preprocessor &PP = CXXUnit->getPreprocessor();
+ while (Lex.getBufferLocation() <= EffectiveBufferEnd) {
+ Token Tok;
+ Lex.LexFromRawLexer(Tok);
+
+ reprocess:
+ if (Tok.is(tok::hash) && Tok.isAtStartOfLine()) {
+ // We have found a preprocessing directive. Gobble it up so that we
+ // don't see it while preprocessing these tokens later, but keep track of
+ // all of the token locations inside this preprocessing directive so that
+ // we can annotate them appropriately.
+ //
+ // FIXME: Some simple tests here could identify macro definitions and
+ // #undefs, to provide specific cursor kinds for those.
+ std::vector<SourceLocation> Locations;
+ do {
+ Locations.push_back(Tok.getLocation());
+ Lex.LexFromRawLexer(Tok);
+ } while (!Tok.isAtStartOfLine() && !Tok.is(tok::eof));
+
+ using namespace cxcursor;
+ CXCursor Cursor
+ = MakePreprocessingDirectiveCursor(SourceRange(Locations.front(),
+ Locations.back()),
+ CXXUnit);
+ for (unsigned I = 0, N = Locations.size(); I != N; ++I) {
+ Annotated[Locations[I].getRawEncoding()] = Cursor;
+ }
+
+ if (Tok.is(tok::eof))
+ break;
+
+ if (Tok.isAtStartOfLine())
+ goto reprocess;
+
+ continue;
+ }
+
+ // If this is a ## token, change its kind to unknown so that repreprocessing
+ // it will not produce an error.
+ if (Tok.is(tok::hashhash))
+ Tok.setKind(tok::unknown);
+
+ // If this raw token is an identifier, the raw lexer won't have looked up
+ // the corresponding identifier info for it. Do this now so that it will be
+ // macro expanded when we re-preprocess it.
+ if (Tok.is(tok::identifier)) {
+ // Change the kind of this identifier to the appropriate token kind, e.g.
+ // turning "for" into a keyword.
+ Tok.setKind(PP.LookUpIdentifierInfo(Tok)->getTokenID());
+ }
+
+ TokenStream.push_back(Tok);
+
+ if (Tok.is(tok::eof))
+ break;
+ }
+ }
+
for (unsigned I = 0; I != NumTokens; ++I) {
// Determine whether we saw a cursor at this token's location.
AnnotateTokensData::iterator Pos = Annotated.find(Tokens[I].int_data[1]);
if (Pos == Annotated.end())
continue;
-
+
Cursors[I] = Pos->second;
- }
+ }
}
void clang_disposeTokens(CXTranslationUnit TU,
Modified: cfe/trunk/tools/CIndex/CIndex.exports
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/tools/CIndex/CIndex.exports?rev=98786&r1=98785&r2=98786&view=diff
==============================================================================
--- cfe/trunk/tools/CIndex/CIndex.exports (original)
+++ cfe/trunk/tools/CIndex/CIndex.exports Wed Mar 17 19:42:48 2010
@@ -70,6 +70,7 @@
_clang_isDeclaration
_clang_isExpression
_clang_isInvalid
+_clang_isPreprocessing
_clang_isReference
_clang_isStatement
_clang_isTranslationUnit
Modified: cfe/trunk/tools/CIndex/CXCursor.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/tools/CIndex/CXCursor.cpp?rev=98786&r1=98785&r2=98786&view=diff
==============================================================================
--- cfe/trunk/tools/CIndex/CXCursor.cpp (original)
+++ cfe/trunk/tools/CIndex/CXCursor.cpp Wed Mar 17 19:42:48 2010
@@ -296,6 +296,24 @@
reinterpret_cast<uintptr_t>(C.data[1])));
}
+CXCursor cxcursor::MakePreprocessingDirectiveCursor(SourceRange Range,
+ ASTUnit *TU) {
+ CXCursor C = { CXCursor_PreprocessingDirective,
+ { reinterpret_cast<void *>(Range.getBegin().getRawEncoding()),
+ reinterpret_cast<void *>(Range.getEnd().getRawEncoding()),
+ TU }
+ };
+ return C;
+}
+
+SourceRange cxcursor::getCursorPreprocessingDirective(CXCursor C) {
+ assert(C.kind == CXCursor_PreprocessingDirective);
+ return SourceRange(SourceLocation::getFromRawEncoding(
+ reinterpret_cast<uintptr_t> (C.data[0])),
+ SourceLocation::getFromRawEncoding(
+ reinterpret_cast<uintptr_t> (C.data[1])));
+}
+
Decl *cxcursor::getCursorDecl(CXCursor Cursor) {
return (Decl *)Cursor.data[0];
}
Modified: cfe/trunk/tools/CIndex/CXCursor.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/tools/CIndex/CXCursor.h?rev=98786&r1=98785&r2=98786&view=diff
==============================================================================
--- cfe/trunk/tools/CIndex/CXCursor.h (original)
+++ cfe/trunk/tools/CIndex/CXCursor.h Wed Mar 17 19:42:48 2010
@@ -73,6 +73,12 @@
/// and optionally the location where the reference occurred.
std::pair<TypeDecl *, SourceLocation> getCursorTypeRef(CXCursor C);
+/// \brief Create a preprocessing directive cursor.
+CXCursor MakePreprocessingDirectiveCursor(SourceRange Range, ASTUnit *TU);
+
+/// \brief Unpack a given preprocessing directive to retrieve its source range.
+SourceRange getCursorPreprocessingDirective(CXCursor C);
+
Decl *getCursorDecl(CXCursor Cursor);
Expr *getCursorExpr(CXCursor Cursor);
Stmt *getCursorStmt(CXCursor Cursor);
More information about the cfe-commits
mailing list