[cfe-commits] r58911 - in /cfe/trunk: include/clang/Basic/TokenKinds.def include/clang/Lex/Preprocessor.h include/clang/Lex/Token.h lib/Lex/PPCaching.cpp
Argiris Kirtzidis
akyrtzi at gmail.com
Sat Nov 8 08:17:05 PST 2008
Author: akirtzidis
Date: Sat Nov 8 10:17:04 2008
New Revision: 58911
URL: http://llvm.org/viewvc/llvm-project?rev=58911&view=rev
Log:
Introduce annotation tokens, a special kind of token, created and used only by the parser to replace a group of tokens with a single token encoding semantic information.
Will be fully utilized later for C++ nested-name-specifiers.
Modified:
cfe/trunk/include/clang/Basic/TokenKinds.def
cfe/trunk/include/clang/Lex/Preprocessor.h
cfe/trunk/include/clang/Lex/Token.h
cfe/trunk/lib/Lex/PPCaching.cpp
Modified: cfe/trunk/include/clang/Basic/TokenKinds.def
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Basic/TokenKinds.def?rev=58911&r1=58910&r2=58911&view=diff
==============================================================================
--- cfe/trunk/include/clang/Basic/TokenKinds.def (original)
+++ cfe/trunk/include/clang/Basic/TokenKinds.def Sat Nov 8 10:17:04 2008
@@ -35,6 +35,9 @@
#ifndef OBJC2_AT_KEYWORD
#define OBJC2_AT_KEYWORD(X)
#endif
+#ifndef ANNOTATION
+#define ANNOTATION(X) TOK(annot_ ## X)
+#endif
//===----------------------------------------------------------------------===//
// Preprocessor keywords.
@@ -365,6 +368,11 @@
// TODO: What to do about context-sensitive keywords like:
// bycopy/byref/in/inout/oneway/out?
+ANNOTATION(cxxscope) // annotation for a C++ scope spec, e.g. "::foo::bar::"
+ANNOTATION(qualtypename) // annotation for a C typedef name, or a C++ (possibly
+ // qualified) typename, e.g. "foo::MyClass"
+
+#undef ANNOTATION
#undef OBJC2_AT_KEYWORD
#undef OBJC1_AT_KEYWORD
#undef CXX_KEYWORD_OPERATOR
Modified: cfe/trunk/include/clang/Lex/Preprocessor.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Lex/Preprocessor.h?rev=58911&r1=58910&r2=58911&view=diff
==============================================================================
--- cfe/trunk/include/clang/Lex/Preprocessor.h (original)
+++ cfe/trunk/include/clang/Lex/Preprocessor.h Sat Nov 8 10:17:04 2008
@@ -352,6 +352,28 @@
else
return PeekAhead(N+1);
}
+
+ /// EnterToken - Enters a token in the token stream to be lexed next. If
+ /// BackTrack() is called afterwards, the token will remain at the insertion
+ /// point.
+ void EnterToken(const Token &Tok) {
+ EnterCachingLexMode();
+ CachedTokens.insert(CachedTokens.begin()+CachedLexPos, Tok);
+ }
+
+ /// AnnotateCachedTokens - We notify the Preprocessor that if it is caching
+ /// tokens (because backtrack is enabled) it should replace the most recent
+ /// cached tokens with the given annotation token. This function has no effect
+ /// if backtracking is not enabled.
+ ///
+ /// Note that the use of this function is just for optimization; so that the
+ /// cached tokens doesn't get re-parsed and re-resolved after a backtrack is
+ /// invoked.
+ void AnnotateCachedTokens(const Token &Tok) {
+ assert(Tok.isAnnotationToken() && "Expected annotation token");
+ if (CachedLexPos != 0 && InCachingLexMode())
+ AnnotatePreviousCachedTokens(Tok);
+ }
/// Diag - Forwarding function for diagnostics. This emits a diagnostic at
/// the specified Token's location, translating the token's start
@@ -559,6 +581,7 @@
RemoveTopOfLexerStack();
}
const Token &PeekAhead(unsigned N);
+ void AnnotatePreviousCachedTokens(const Token &Tok);
//===--------------------------------------------------------------------===//
/// Handle*Directive - implement the various preprocessor directives. These
Modified: cfe/trunk/include/clang/Lex/Token.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Lex/Token.h?rev=58911&r1=58910&r2=58911&view=diff
==============================================================================
--- cfe/trunk/include/clang/Lex/Token.h (original)
+++ cfe/trunk/include/clang/Lex/Token.h Sat Nov 8 10:17:04 2008
@@ -25,14 +25,31 @@
/// It is not intended to be space efficient, it is intended to return as much
/// information as possible about each returned token. This is expected to be
/// compressed into a smaller form if memory footprint is important.
+///
+/// The parser can create a special "annotation token" representing a stream of
+/// tokens that were parsed and semantically resolved, e.g.: "foo::MyClass<int>"
+/// can be represented by a single typename annotation token that carries
+/// information about the SourceRange of the tokens and the type object.
class Token {
- /// The location and length of the token text itself.
+ /// The location of the token.
SourceLocation Loc;
- unsigned Length;
+
+ union {
+ /// The end of the SourceRange of an annotation token.
+ unsigned AnnotEndLocID;
+
+ /// The length of the token text itself.
+ unsigned Length;
+ };
- /// IdentifierInfo - If this was an identifier, this points to the uniqued
- /// information about this identifier.
- IdentifierInfo *IdentInfo;
+ union {
+ /// IdentifierInfo - If this was an identifier, this points to the uniqued
+ /// information about this identifier.
+ IdentifierInfo *IdentInfo;
+
+ /// AnnotVal - Information specific to an annotation token.
+ void *AnnotVal;
+ };
/// Kind - The actual flavor of token this is.
///
@@ -60,13 +77,39 @@
bool is(tok::TokenKind K) const { return Kind == (unsigned) K; }
bool isNot(tok::TokenKind K) const { return Kind != (unsigned) K; }
+ bool isAnnotationToken() const {
+ return is(tok::annot_qualtypename) || is(tok::annot_cxxscope);
+ }
+
/// getLocation - Return a source location identifier for the specified
/// offset in the current file.
SourceLocation getLocation() const { return Loc; }
- unsigned getLength() const { return Length; }
+ unsigned getLength() const {
+ assert(!isAnnotationToken() && "Used Length on annotation token");
+ return Length;
+ }
void setLocation(SourceLocation L) { Loc = L; }
void setLength(unsigned Len) { Length = Len; }
+
+ SourceLocation getAnnotationEndLoc() const {
+ assert(isAnnotationToken() && "Used AnnotEndLocID on non-annotation token");
+ return SourceLocation::getFromRawEncoding(AnnotEndLocID);
+ }
+ void setAnnotationEndLoc(SourceLocation L) {
+ assert(isAnnotationToken() && "Used AnnotEndLocID on non-annotation token");
+ AnnotEndLocID = L.getRawEncoding();
+ }
+
+ /// getAnnotationRange - SourceRange of the group of tokens that this
+ /// annotation token represents.
+ SourceRange getAnnotationRange() const {
+ return SourceRange(getLocation(), getAnnotationEndLoc());
+ }
+ void setAnnotationRange(SourceRange R) {
+ setLocation(R.getBegin());
+ setAnnotationEndLoc(R.getEnd());
+ }
const char *getName() const {
return tok::getTokenName( (tok::TokenKind) Kind);
@@ -80,10 +123,22 @@
Loc = SourceLocation();
}
- IdentifierInfo *getIdentifierInfo() const { return IdentInfo; }
+ IdentifierInfo *getIdentifierInfo() const {
+ assert(!isAnnotationToken() && "Used IdentInfo on annotation token");
+ return IdentInfo;
+ }
void setIdentifierInfo(IdentifierInfo *II) {
IdentInfo = II;
}
+
+ void *getAnnotationValue() const {
+ assert(isAnnotationToken() && "Used AnnotVal on non-annotation token");
+ return AnnotVal;
+ }
+ void setAnnotationValue(void *val) {
+ assert(isAnnotationToken() && "Used AnnotVal on non-annotation token");
+ AnnotVal = val;
+ }
/// setFlag - Set the specified flag.
void setFlag(TokenFlags Flag) {
Modified: cfe/trunk/lib/Lex/PPCaching.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/PPCaching.cpp?rev=58911&r1=58910&r2=58911&view=diff
==============================================================================
--- cfe/trunk/lib/Lex/PPCaching.cpp (original)
+++ cfe/trunk/lib/Lex/PPCaching.cpp Sat Nov 8 10:17:04 2008
@@ -93,3 +93,27 @@
EnterCachingLexMode();
return CachedTokens.back();
}
+
+void Preprocessor::AnnotatePreviousCachedTokens(const Token &Tok) {
+ assert(Tok.isAnnotationToken() && "Expected annotation token");
+ assert(CachedLexPos != 0 && "Expected to have some cached tokens");
+ assert(CachedTokens[CachedLexPos-1].getLocation() == Tok.getAnnotationEndLoc()
+ && "The annotation should be until the most recent cached token");
+
+ // Start from the end of the cached tokens list and look for the token
+ // that is the beginning of the annotation token.
+ for (CachedTokensTy::size_type i = CachedLexPos; i != 0; --i) {
+ CachedTokensTy::iterator AnnotBegin = CachedTokens.begin() + i-1;
+ if (AnnotBegin->getLocation() == Tok.getLocation()) {
+ assert((BacktrackPositions.empty() || BacktrackPositions.back() < i) &&
+ "The backtrack pos points inside the annotated tokens!");
+ // Replace the cached tokens with the single annotation token.
+ CachedTokens.erase(AnnotBegin + 1, CachedTokens.begin() + CachedLexPos);
+ *AnnotBegin = Tok;
+ CachedLexPos = i;
+ return;
+ }
+ }
+
+ assert(0&&"Didn't find the first token represented by the annotation token!");
+}
More information about the cfe-commits
mailing list