[cfe-commits] r94551 - in /cfe/trunk: include/clang-c/Index.h include/clang/Lex/Lexer.h test/Index/annotate-tokens.c tools/CIndex/CIndex.cpp tools/CIndex/CIndex.exports tools/CIndex/CIndexer.h tools/CIndex/CXSourceLocation.h tools/c-index-test/c-index-test.c
Douglas Gregor
dgregor at apple.com
Tue Jan 26 09:06:04 PST 2010
Author: dgregor
Date: Tue Jan 26 11:06:03 2010
New Revision: 94551
URL: http://llvm.org/viewvc/llvm-project?rev=94551&view=rev
Log:
Introduce a CIndex API for lexing the raw tokens within a given source
range. The token-annotation function does nothing, yet.
Added:
cfe/trunk/test/Index/annotate-tokens.c (with props)
Modified:
cfe/trunk/include/clang-c/Index.h
cfe/trunk/include/clang/Lex/Lexer.h
cfe/trunk/tools/CIndex/CIndex.cpp
cfe/trunk/tools/CIndex/CIndex.exports
cfe/trunk/tools/CIndex/CIndexer.h
cfe/trunk/tools/CIndex/CXSourceLocation.h
cfe/trunk/tools/c-index-test/c-index-test.c
Modified: cfe/trunk/include/clang-c/Index.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang-c/Index.h?rev=94551&r1=94550&r2=94551&view=diff
==============================================================================
--- cfe/trunk/include/clang-c/Index.h (original)
+++ cfe/trunk/include/clang-c/Index.h Tue Jan 26 11:06:03 2010
@@ -861,6 +861,125 @@
*/
/**
+ * \defgroup CINDEX_LEX Lexing and syntactic analysis
+ *
+ * @{
+ */
+
+/**
+ * \brief Describes a kind of token.
+ */
+typedef enum CXTokenKind {
+ /**
+ * \brief A token that contains some kind of punctuation.
+ */
+ CXToken_Punctuation,
+
+ /**
+ * \brief A a language keyword.
+ */
+ CXToken_Keyword,
+
+ /**
+ * \brief An identifier (that is not a keyword).
+ */
+ CXToken_Identifier,
+
+ /**
+ * \brief A numeric, string, or character literal.
+ */
+ CXToken_Literal,
+
+ /**
+ * \brief A comment.
+ */
+ CXToken_Comment
+} CXTokenKind;
+
+/**
+ * \brief Describes a single preprocessing token.
+ */
+typedef struct {
+ unsigned int_data[4];
+ void *ptr_data;
+} CXToken;
+
+/**
+ * \brief Determine the kind of the given token.
+ */
+CINDEX_LINKAGE CXTokenKind clang_getTokenKind(CXToken);
+
+/**
+ * \brief Determine the spelling of the given token.
+ *
+ * The spelling of a token is the textual representation of that token, e.g.,
+ * the text of an identifier or keyword.
+ */
+CINDEX_LINKAGE CXString clang_getTokenSpelling(CXTranslationUnit, CXToken);
+
+/**
+ * \brief Retrieve the source location of the given token.
+ */
+CINDEX_LINKAGE CXSourceLocation clang_getTokenLocation(CXTranslationUnit,
+ CXToken);
+
+/**
+ * \brief Retrieve a source range that covers the given token.
+ */
+CINDEX_LINKAGE CXSourceRange clang_getTokenExtent(CXTranslationUnit, CXToken);
+
+/**
+ * \brief Tokenize the source code described by the given range into raw
+ * lexical tokens.
+ *
+ * \param TU the translation unit whose text is being tokenized.
+ *
+ * \param Range the source range in which text should be tokenized. All of the
+ * tokens produced by tokenization will fall within this source range,
+ *
+ * \param Tokens this pointer will be set to point to the array of tokens
+ * that occur within the given source range. The returned pointer must be
+ * freed with clang_disposeTokens() before the translation unit is destroyed.
+ *
+ * \param NumTokens will be set to the number of tokens in the \c *Tokens
+ * array.
+ *
+ */
+CINDEX_LINKAGE void clang_tokenize(CXTranslationUnit TU, CXSourceRange Range,
+ CXToken **Tokens, unsigned *NumTokens);
+
+/**
+ * \brief Annotate the given set of tokens by providing cursors for each token
+ * that can be mapped to a specific entity within the abstract syntax tree.
+ *
+ * This token-annotation routine is equivalent to invoking clang_getCursor()
+ * for the source locations of each of the tokens, then accepting only those
+ * cursors that refer to a specific token.
+ *
+ * \param TU the translation unit that owns the given tokens.
+ *
+ * \param Tokens the set of tokens to annotate.
+ *
+ * \param NumTokens the number of tokens in \p Tokens.
+ *
+ * \param Cursors an array of \p NumTokens cursors, whose contents will be
+ * replaced with the cursors corresponding to each token.
+ */
+CINDEX_LINKAGE void clang_annotateTokens(CXTranslationUnit TU,
+ CXToken *Tokens, unsigned NumTokens,
+ CXCursor *Cursors);
+
+/**
+ * \brief Free the given set of tokens.
+ */
+CINDEX_LINKAGE void clang_disposeTokens(CXTranslationUnit TU,
+ CXToken *Tokens, unsigned NumTokens);
+
+/**
+ * @}
+ */
+
+/**
* \defgroup CINDEX_DEBUG Debugging facilities
*
* These routines are used for testing and debugging, only, and should not
Modified: cfe/trunk/include/clang/Lex/Lexer.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Lex/Lexer.h?rev=94551&r1=94550&r2=94551&view=diff
==============================================================================
--- cfe/trunk/include/clang/Lex/Lexer.h (original)
+++ cfe/trunk/include/clang/Lex/Lexer.h Tue Jan 26 11:06:03 2010
@@ -199,6 +199,9 @@
/// the current file.
SourceLocation getSourceLocation() { return getSourceLocation(BufferPtr); }
+ /// \brief Return the current location in the buffer.
+ const char *getBufferLocation() const { return BufferPtr; }
+
/// Stringify - Convert the specified string into a C string by escaping '\'
/// and " characters. This does not add surrounding ""'s to the string.
/// If Charify is true, this escapes the ' character instead of ".
Added: cfe/trunk/test/Index/annotate-tokens.c
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/Index/annotate-tokens.c?rev=94551&view=auto
==============================================================================
--- cfe/trunk/test/Index/annotate-tokens.c (added)
+++ cfe/trunk/test/Index/annotate-tokens.c Tue Jan 26 11:06:03 2010
@@ -0,0 +1,63 @@
+typedef int T;
+struct X { int a, b; };
+void f(void *ptr) {
+ T* t_ptr = (T *)ptr;
+ (void)sizeof(T);
+ /* A comment */
+ struct X x = (struct X){1, 2};
+ void *xx = ptr ? : &x;
+ const char * hello = "Hello";
+}
+
+// RUN: c-index-test -test-annotate-tokens=%s:4:1:9:32 %s | FileCheck %s
+// CHECK: Identifier: "T" [4:3 - 4:3]
+// CHECK: Punctuation: "*" [4:4 - 4:4]
+// CHECK: Identifier: "t_ptr" [4:6 - 4:10]
+// CHECK: Punctuation: "=" [4:12 - 4:12]
+// CHECK: Punctuation: "(" [4:14 - 4:14]
+// CHECK: Identifier: "T" [4:15 - 4:15]
+// CHECK: Punctuation: "*" [4:17 - 4:17]
+// CHECK: Punctuation: ")" [4:18 - 4:18]
+// CHECK: Identifier: "ptr" [4:19 - 4:21]
+// CHECK: Punctuation: ";" [4:22 - 4:22]
+// CHECK: Punctuation: "(" [5:3 - 5:3]
+// CHECK: Keyword: "void" [5:4 - 5:7]
+// CHECK: Punctuation: ")" [5:8 - 5:8]
+// CHECK: Keyword: "sizeof" [5:9 - 5:14]
+// CHECK: Punctuation: "(" [5:15 - 5:15]
+// CHECK: Identifier: "T" [5:16 - 5:16]
+// CHECK: Punctuation: ")" [5:17 - 5:17]
+// CHECK: Punctuation: ";" [5:18 - 5:18]
+// CHECK: Comment: "/* A comment */" [6:3 - 6:17]
+// CHECK: Keyword: "struct" [7:3 - 7:8]
+// CHECK: Identifier: "X" [7:10 - 7:10]
+// CHECK: Identifier: "x" [7:12 - 7:12]
+// CHECK: Punctuation: "=" [7:14 - 7:14]
+// CHECK: Punctuation: "(" [7:16 - 7:16]
+// CHECK: Keyword: "struct" [7:17 - 7:22]
+// CHECK: Identifier: "X" [7:24 - 7:24]
+// CHECK: Punctuation: ")" [7:25 - 7:25]
+// CHECK: Punctuation: "{" [7:26 - 7:26]
+// CHECK: Literal: "1" [7:27 - 7:27]
+// CHECK: Punctuation: "," [7:28 - 7:28]
+// CHECK: Literal: "2" [7:30 - 7:30]
+// CHECK: Punctuation: "}" [7:31 - 7:31]
+// CHECK: Punctuation: ";" [7:32 - 7:32]
+// CHECK: Keyword: "void" [8:3 - 8:6]
+// CHECK: Punctuation: "*" [8:8 - 8:8]
+// CHECK: Identifier: "xx" [8:9 - 8:10]
+// CHECK: Punctuation: "=" [8:12 - 8:12]
+// CHECK: Identifier: "ptr" [8:14 - 8:16]
+// CHECK: Punctuation: "?" [8:18 - 8:18]
+// CHECK: Punctuation: ":" [8:20 - 8:20]
+// CHECK: Punctuation: "&" [8:22 - 8:22]
+// CHECK: Identifier: "x" [8:23 - 8:23]
+// CHECK: Punctuation: ";" [8:24 - 8:24]
+// CHECK: Keyword: "const" [9:3 - 9:7]
+// CHECK: Keyword: "char" [9:9 - 9:12]
+// CHECK: Punctuation: "*" [9:14 - 9:14]
+// CHECK: Identifier: "hello" [9:16 - 9:20]
+// CHECK: Punctuation: "=" [9:22 - 9:22]
+// CHECK: Literal: ""Hello"" [9:24 - 9:30]
+// CHECK: Punctuation: ";" [9:31 - 9:31]
+// CHECK: Punctuation: "}" [10:1 - 10:1]
Propchange: cfe/trunk/test/Index/annotate-tokens.c
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: cfe/trunk/test/Index/annotate-tokens.c
------------------------------------------------------------------------------
svn:keywords = Id
Propchange: cfe/trunk/test/Index/annotate-tokens.c
------------------------------------------------------------------------------
svn:mime-type = text/plain
Modified: cfe/trunk/tools/CIndex/CIndex.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/tools/CIndex/CIndex.cpp?rev=94551&r1=94550&r2=94551&view=diff
==============================================================================
--- cfe/trunk/tools/CIndex/CIndex.cpp (original)
+++ cfe/trunk/tools/CIndex/CIndex.cpp Tue Jan 26 11:06:03 2010
@@ -876,6 +876,21 @@
return Str;
}
+CXString CIndexer::createCXString(llvm::StringRef String, bool DupString) {
+ CXString Result;
+ if (DupString || (!String.empty() && String.data()[String.size()] != 0)) {
+ char *Spelling = (char *)malloc(String.size() + 1);
+ memmove(Spelling, String.data(), String.size());
+ Spelling[String.size()] = 0;
+ Result.Spelling = Spelling;
+ Result.MustFreeString = 1;
+ } else {
+ Result.Spelling = String.data();
+ Result.MustFreeString = 0;
+ }
+ return Result;
+}
+
extern "C" {
CXIndex clang_createIndex(int excludeDeclarationsFromPCH,
int displayDiagnostics) {
@@ -1882,6 +1897,183 @@
} // end: extern "C"
//===----------------------------------------------------------------------===//
+// Token-based Operations.
+//===----------------------------------------------------------------------===//
+
+/* CXToken layout:
+ * int_data[0]: a CXTokenKind
+ * int_data[1]: starting token location
+ * int_data[2]: token length
+ * int_data[3]: reserved
+ * ptr_data: for identifiers and keywords, an IdentifierInfo*.
+ * otherwise unused.
+ */
+extern "C" {
+
+CXTokenKind clang_getTokenKind(CXToken CXTok) {
+ return static_cast<CXTokenKind>(CXTok.int_data[0]);
+}
+
+CXString clang_getTokenSpelling(CXTranslationUnit TU, CXToken CXTok) {
+ switch (clang_getTokenKind(CXTok)) {
+ case CXToken_Identifier:
+ case CXToken_Keyword:
+ // We know we have an IdentifierInfo*, so use that.
+ return CIndexer::createCXString(
+ static_cast<IdentifierInfo *>(CXTok.ptr_data)->getNameStart());
+
+ case CXToken_Literal: {
+ // We have stashed the starting pointer in the ptr_data field. Use it.
+ const char *Text = static_cast<const char *>(CXTok.ptr_data);
+ return CIndexer::createCXString(llvm::StringRef(Text, CXTok.int_data[2]),
+ true);
+ }
+
+ case CXToken_Punctuation:
+ case CXToken_Comment:
+ break;
+ }
+
+ // We have to find the starting buffer pointer the hard way, by
+ // deconstructing the source location.
+ ASTUnit *CXXUnit = static_cast<ASTUnit *>(TU);
+ if (!CXXUnit)
+ return CIndexer::createCXString("");
+
+ SourceLocation Loc = SourceLocation::getFromRawEncoding(CXTok.int_data[1]);
+ std::pair<FileID, unsigned> LocInfo
+ = CXXUnit->getSourceManager().getDecomposedLoc(Loc);
+ std::pair<const char *,const char *> Buffer
+ = CXXUnit->getSourceManager().getBufferData(LocInfo.first);
+
+ return CIndexer::createCXString(llvm::StringRef(Buffer.first+LocInfo.second,
+ CXTok.int_data[2]),
+ true);
+}
+
+CXSourceLocation clang_getTokenLocation(CXTranslationUnit TU, CXToken CXTok) {
+ ASTUnit *CXXUnit = static_cast<ASTUnit *>(TU);
+ if (!CXXUnit)
+ return clang_getNullLocation();
+
+ return cxloc::translateSourceLocation(CXXUnit->getASTContext(),
+ SourceLocation::getFromRawEncoding(CXTok.int_data[1]));
+}
+
+CXSourceRange clang_getTokenExtent(CXTranslationUnit TU, CXToken CXTok) {
+ ASTUnit *CXXUnit = static_cast<ASTUnit *>(TU);
+ if (!CXXUnit) {
+ CXSourceRange Result = { 0, 0, 0 };
+ return Result;
+ }
+
+ return cxloc::translateSourceRange(CXXUnit->getASTContext(),
+ SourceLocation::getFromRawEncoding(CXTok.int_data[1]));
+}
+
+void clang_tokenize(CXTranslationUnit TU, CXSourceRange Range,
+ CXToken **Tokens, unsigned *NumTokens) {
+ if (Tokens)
+ *Tokens = 0;
+ if (NumTokens)
+ *NumTokens = 0;
+
+ ASTUnit *CXXUnit = static_cast<ASTUnit *>(TU);
+ if (!CXXUnit || !Tokens || !NumTokens)
+ return;
+
+ SourceRange R = cxloc::translateSourceRange(Range);
+ if (R.isInvalid())
+ return;
+
+ SourceManager &SourceMgr = CXXUnit->getSourceManager();
+ std::pair<FileID, unsigned> BeginLocInfo
+ = SourceMgr.getDecomposedLoc(R.getBegin());
+ std::pair<FileID, unsigned> EndLocInfo
+ = SourceMgr.getDecomposedLoc(R.getEnd());
+
+ // Cannot tokenize across files.
+ if (BeginLocInfo.first != EndLocInfo.first)
+ return;
+
+ // Create a lexer
+ std::pair<const char *,const char *> Buffer
+ = SourceMgr.getBufferData(BeginLocInfo.first);
+ Lexer Lex(SourceMgr.getLocForStartOfFile(BeginLocInfo.first),
+ CXXUnit->getASTContext().getLangOptions(),
+ Buffer.first, Buffer.first + BeginLocInfo.second, Buffer.second);
+ Lex.SetCommentRetentionState(true);
+
+ // Lex tokens until we hit the end of the range.
+ const char *EffectiveBufferEnd = Buffer.first + EndLocInfo.second;
+ llvm::SmallVector<CXToken, 32> CXTokens;
+ Token Tok;
+ do {
+ // Lex the next token
+ Lex.LexFromRawLexer(Tok);
+ if (Tok.is(tok::eof))
+ break;
+
+ // Initialize the CXToken.
+ CXToken CXTok;
+
+ // - Common fields
+ CXTok.int_data[1] = Tok.getLocation().getRawEncoding();
+ CXTok.int_data[2] = Tok.getLength();
+ CXTok.int_data[3] = 0;
+
+ // - Kind-specific fields
+ if (Tok.isLiteral()) {
+ CXTok.int_data[0] = CXToken_Literal;
+ CXTok.ptr_data = (void *)Tok.getLiteralData();
+ } else if (Tok.is(tok::identifier)) {
+ // Lookup the identifier to determine whether we have a
+ std::pair<FileID, unsigned> LocInfo
+ = SourceMgr.getDecomposedLoc(Tok.getLocation());
+ const char *StartPos
+ = CXXUnit->getSourceManager().getBufferData(LocInfo.first).first +
+ LocInfo.second;
+ IdentifierInfo *II
+ = CXXUnit->getPreprocessor().LookUpIdentifierInfo(Tok, StartPos);
+ CXTok.int_data[0] = II->getTokenID() == tok::identifier?
+ CXToken_Identifier
+ : CXToken_Keyword;
+ CXTok.ptr_data = II;
+ } else if (Tok.is(tok::comment)) {
+ CXTok.int_data[0] = CXToken_Comment;
+ CXTok.ptr_data = 0;
+ } else {
+ CXTok.int_data[0] = CXToken_Punctuation;
+ CXTok.ptr_data = 0;
+ }
+ CXTokens.push_back(CXTok);
+ } while (Lex.getBufferLocation() <= EffectiveBufferEnd);
+
+ if (CXTokens.empty())
+ return;
+
+ *Tokens = (CXToken *)malloc(sizeof(CXToken) * CXTokens.size());
+ memmove(*Tokens, CXTokens.data(), sizeof(CXToken) * CXTokens.size());
+ *NumTokens = CXTokens.size();
+}
+
+void clang_annotateTokens(CXTranslationUnit TU,
+ CXToken *Tokens, unsigned NumTokens,
+ CXCursor *Cursors) {
+ // FIXME: Actually perform some meaningful lookup here.
+ for (unsigned I = 0; I != NumTokens; ++I)
+ Cursors[I] = clang_getNullCursor();
+}
+
+void clang_disposeTokens(CXTranslationUnit TU,
+ CXToken *Tokens, unsigned NumTokens) {
+ if (Tokens)
+ free(Tokens);
+}
+
+} // end: extern "C"
+
+//===----------------------------------------------------------------------===//
// CXString Operations.
//===----------------------------------------------------------------------===//
Modified: cfe/trunk/tools/CIndex/CIndex.exports
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/tools/CIndex/CIndex.exports?rev=94551&r1=94550&r2=94551&view=diff
==============================================================================
--- cfe/trunk/tools/CIndex/CIndex.exports (original)
+++ cfe/trunk/tools/CIndex/CIndex.exports Tue Jan 26 11:06:03 2010
@@ -1,3 +1,4 @@
+_clang_annotateTokens
_clang_codeComplete
_clang_createIndex
_clang_createTranslationUnit
@@ -5,6 +6,7 @@
_clang_disposeCodeCompleteResults
_clang_disposeIndex
_clang_disposeString
+_clang_disposeTokens
_clang_disposeTranslationUnit
_clang_equalCursors
_clang_equalLocations
@@ -35,6 +37,10 @@
_clang_getRange
_clang_getRangeEnd
_clang_getRangeStart
+_clang_getTokenExtent
+_clang_getTokenKind
+_clang_getTokenLocation
+_clang_getTokenSpelling
_clang_getTranslationUnitCursor
_clang_getTranslationUnitSpelling
_clang_isCursorDefinition
@@ -45,4 +51,5 @@
_clang_isStatement
_clang_isTranslationUnit
_clang_setUseExternalASTGeneration
+_clang_tokenize
_clang_visitChildren
Modified: cfe/trunk/tools/CIndex/CIndexer.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/tools/CIndex/CIndexer.h?rev=94551&r1=94550&r2=94551&view=diff
==============================================================================
--- cfe/trunk/tools/CIndex/CIndexer.h (original)
+++ cfe/trunk/tools/CIndex/CIndexer.h Tue Jan 26 11:06:03 2010
@@ -18,6 +18,7 @@
#include "clang-c/Index.h"
#include "clang/Frontend/CompilerInstance.h"
#include "clang/Frontend/ASTUnit.h"
+#include "llvm/ADT/StringRef.h"
#include "llvm/System/Path.h"
#include <vector>
@@ -76,6 +77,8 @@
std::string getClangResourcesPath();
static CXString createCXString(const char *String, bool DupString = false);
+ static CXString createCXString(llvm::StringRef String,
+ bool DupString = false);
};
namespace clang {
Modified: cfe/trunk/tools/CIndex/CXSourceLocation.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/tools/CIndex/CXSourceLocation.h?rev=94551&r1=94550&r2=94551&view=diff
==============================================================================
--- cfe/trunk/tools/CIndex/CXSourceLocation.h (original)
+++ cfe/trunk/tools/CIndex/CXSourceLocation.h Tue Jan 26 11:06:03 2010
@@ -38,8 +38,8 @@
static inline CXSourceRange translateSourceRange(ASTContext &Context,
SourceRange R) {
CXSourceRange Result = { &Context,
- R.getBegin().getRawEncoding(),
- R.getEnd().getRawEncoding() };
+ R.getBegin().getRawEncoding(),
+ R.getEnd().getRawEncoding() };
return Result;
}
Modified: cfe/trunk/tools/c-index-test/c-index-test.c
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/tools/c-index-test/c-index-test.c?rev=94551&r1=94550&r2=94551&view=diff
==============================================================================
--- cfe/trunk/tools/c-index-test/c-index-test.c (original)
+++ cfe/trunk/tools/c-index-test/c-index-test.c Tue Jan 26 11:06:03 2010
@@ -481,42 +481,62 @@
on failure. If successful, the pointer *filename will contain newly-allocated
memory (that will be owned by the caller) to store the file name. */
int parse_file_line_column(const char *input, char **filename, unsigned *line,
- unsigned *column) {
+ unsigned *column, unsigned *second_line,
+ unsigned *second_column) {
/* Find the second colon. */
- const char *second_colon = strrchr(input, ':'), *first_colon;
+ const char *last_colon = strrchr(input, ':');
+ unsigned values[4], i;
+ unsigned num_values = (second_line && second_column)? 4 : 2;
+
char *endptr = 0;
- if (!second_colon || second_colon == input) {
- fprintf(stderr, "could not parse filename:line:column in '%s'\n", input);
+ if (!last_colon || last_colon == input) {
+ if (num_values == 4)
+ fprintf(stderr, "could not parse filename:line:column:line:column in "
+ "'%s'\n", input);
+ else
+ fprintf(stderr, "could not parse filename:line:column in '%s'\n", input);
return 1;
}
- /* Parse the column number. */
- *column = strtol(second_colon + 1, &endptr, 10);
- if (*endptr != 0) {
- fprintf(stderr, "could not parse column in '%s'\n", input);
- return 1;
- }
+ for (i = 0; i != num_values; ++i) {
+ const char *prev_colon;
- /* Find the first colon. */
- first_colon = second_colon - 1;
- while (first_colon != input && *first_colon != ':')
- --first_colon;
- if (first_colon == input) {
- fprintf(stderr, "could not parse line in '%s'\n", input);
- return 1;
- }
+ /* Parse the next line or column. */
+ values[num_values - i - 1] = strtol(last_colon + 1, &endptr, 10);
+ if (*endptr != 0 && *endptr != ':') {
+ fprintf(stderr, "could not parse %s in '%s'\n",
+ (i % 2 ? "column" : "line"), input);
+ return 1;
+ }
+
+ if (i + 1 == num_values)
+ break;
- /* Parse the line number. */
- *line = strtol(first_colon + 1, &endptr, 10);
- if (*endptr != ':') {
- fprintf(stderr, "could not parse line in '%s'\n", input);
- return 1;
+ /* Find the previous colon. */
+ prev_colon = last_colon - 1;
+ while (prev_colon != input && *prev_colon != ':')
+ --prev_colon;
+ if (prev_colon == input) {
+ fprintf(stderr, "could not parse %s in '%s'\n",
+ (i % 2 == 0? "column" : "line"), input);
+ return 1;
+ }
+
+ last_colon = prev_colon;
}
+
+ *line = values[0];
+ *column = values[1];
+ if (second_line && second_column) {
+ *second_line = values[2];
+ *second_column = values[3];
+ }
+
/* Copy the file name. */
- *filename = (char*)malloc(first_colon - input + 1);
- memcpy(*filename, input, first_colon - input);
- (*filename)[first_colon - input] = 0;
+ *filename = (char*)malloc(last_colon - input + 1);
+ memcpy(*filename, input, last_colon - input);
+ (*filename)[last_colon - input] = 0;
return 0;
}
@@ -595,7 +615,8 @@
CXCodeCompleteResults *results = 0;
input += strlen("-code-completion-at=");
- if ((errorCode = parse_file_line_column(input, &filename, &line, &column)))
+ if ((errorCode = parse_file_line_column(input, &filename, &line, &column,
+ 0, 0)))
return errorCode;
if (parse_remapped_files(argc, argv, 2, &unsaved_files, &num_unsaved_files))
@@ -650,7 +671,7 @@
const char *input = argv[Loc + 1] + strlen("-cursor-at=");
if ((errorCode = parse_file_line_column(input, &Locations[Loc].filename,
&Locations[Loc].line,
- &Locations[Loc].column)))
+ &Locations[Loc].column, 0, 0)))
return errorCode;
}
@@ -689,6 +710,104 @@
return 0;
}
+int perform_token_annotation(int argc, const char **argv) {
+ const char *input = argv[1];
+ char *filename = 0;
+ unsigned line, second_line;
+ unsigned column, second_column;
+ CXIndex CIdx;
+ CXTranslationUnit TU = 0;
+ int errorCode;
+ struct CXUnsavedFile *unsaved_files = 0;
+ int num_unsaved_files = 0;
+ CXToken *tokens;
+ unsigned num_tokens;
+ CXSourceRange range;
+ CXSourceLocation startLoc, endLoc;
+ CXFile file = 0;
+ CXCursor *cursors = 0;
+ unsigned i;
+
+ input += strlen("-test-annotate-tokens=");
+ if ((errorCode = parse_file_line_column(input, &filename, &line, &column,
+ &second_line, &second_column)))
+ return errorCode;
+
+ if (parse_remapped_files(argc, argv, 2, &unsaved_files, &num_unsaved_files))
+ return -1;
+
+ CIdx = clang_createIndex(0, 0);
+ TU = clang_createTranslationUnitFromSourceFile(CIdx, argv[argc - 1],
+ argc - num_unsaved_files - 3,
+ argv + num_unsaved_files + 2,
+ num_unsaved_files,
+ unsaved_files);
+ if (!TU) {
+ fprintf(stderr, "unable to parse input\n");
+ clang_disposeIndex(CIdx);
+ free(filename);
+ free_remapped_files(unsaved_files, num_unsaved_files);
+ return -1;
+ }
+ errorCode = 0;
+
+ file = clang_getFile(TU, filename);
+ if (!file) {
+ fprintf(stderr, "file %s is not in this translation unit\n", filename);
+ errorCode = -1;
+ goto teardown;
+ }
+
+ startLoc = clang_getLocation(TU, file, line, column);
+ if (clang_equalLocations(clang_getNullLocation(), startLoc)) {
+ fprintf(stderr, "invalid source location %s:%d:%d\n", filename, line,
+ column);
+ errorCode = -1;
+ goto teardown;
+ }
+
+ endLoc = clang_getLocation(TU, file, second_line, second_column);
+ if (clang_equalLocations(clang_getNullLocation(), endLoc)) {
+ fprintf(stderr, "invalid source location %s:%d:%d\n", filename,
+ second_line, second_column);
+ errorCode = -1;
+ goto teardown;
+ }
+
+ range = clang_getRange(startLoc, endLoc);
+ clang_tokenize(TU, range, &tokens, &num_tokens);
+ cursors = (CXCursor *)malloc(num_tokens * sizeof(CXCursor));
+ clang_annotateTokens(TU, tokens, num_tokens, cursors);
+ for (i = 0; i != num_tokens; ++i) {
+ const char *kind = "<unknown>";
+ CXString spelling = clang_getTokenSpelling(TU, tokens[i]);
+ CXSourceRange extent = clang_getTokenExtent(TU, tokens[i]);
+ unsigned start_line, start_column, end_line, end_column;
+
+ switch (clang_getTokenKind(tokens[i])) {
+ case CXToken_Punctuation: kind = "Punctuation"; break;
+ case CXToken_Keyword: kind = "Keyword"; break;
+ case CXToken_Identifier: kind = "Identifier"; break;
+ case CXToken_Literal: kind = "Literal"; break;
+ case CXToken_Comment: kind = "Comment"; break;
+ }
+ clang_getInstantiationLocation(clang_getRangeStart(extent),
+ 0, &start_line, &start_column);
+ clang_getInstantiationLocation(clang_getRangeEnd(extent),
+ 0, &end_line, &end_column);
+ printf("%s: \"%s\" [%d:%d - %d:%d]\n", kind, clang_getCString(spelling),
+ start_line, start_column, end_line, end_column);
+ }
+ free(cursors);
+
+ teardown:
+ clang_disposeTranslationUnit(TU);
+ clang_disposeIndex(CIdx);
+ free(filename);
+ free_remapped_files(unsaved_files, num_unsaved_files);
+ return errorCode;
+}
+
/******************************************************************************/
/* Command line processing. */
/******************************************************************************/
@@ -712,8 +831,9 @@
" c-index-test -test-load-tu-usrs <AST file> <symbol filter> "
"[FileCheck prefix]\n"
" c-index-test -test-load-source <symbol filter> {<args>}*\n"
- " c-index-test -test-load-source-usrs <symbol filter> {<args>}*\n\n");
+ " c-index-test -test-load-source-usrs <symbol filter> {<args>}*\n");
fprintf(stderr,
+ " c-index-test -test-annotate-tokens=<range> {<args>}* \n\n"
" <symbol filter> values:\n%s",
" all - load all symbols, including those from PCH\n"
" local - load all symbols except those in PCH\n"
@@ -743,7 +863,8 @@
else if (argc >= 4 && strcmp(argv[1], "-test-file-scan") == 0)
return perform_file_scan(argv[2], argv[3],
argc >= 5 ? argv[4] : 0);
-
+ else if (argc > 2 && strstr(argv[1], "-test-annotate-tokens=") == argv[1])
+ return perform_token_annotation(argc, argv);
print_usage();
return 1;
}
More information about the cfe-commits
mailing list