r182768 - A first step towards giving format tokens pointer identity.
Manuel Klimek
klimek at google.com
Tue May 28 04:55:07 PDT 2013
Author: klimek
Date: Tue May 28 06:55:06 2013
New Revision: 182768
URL: http://llvm.org/viewvc/llvm-project?rev=182768&view=rev
Log:
A first step towards giving format tokens pointer identity.
With this patch, we create all tokens in one go before parsing and pass
an ArrayRef<FormatToken*> to the UnwrappedLineParser. The
UnwrappedLineParser is switched to use pointer-to-token internally.
The UnwrappedLineParser still copies the tokens into the UnwrappedLines.
This will be fixed in an upcoming patch.
Modified:
cfe/trunk/lib/Format/Format.cpp
cfe/trunk/lib/Format/UnwrappedLineParser.cpp
cfe/trunk/lib/Format/UnwrappedLineParser.h
Modified: cfe/trunk/lib/Format/Format.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Format/Format.cpp?rev=182768&r1=182767&r2=182768&view=diff
==============================================================================
--- cfe/trunk/lib/Format/Format.cpp (original)
+++ cfe/trunk/lib/Format/Format.cpp Tue May 28 06:55:06 2013
@@ -1087,60 +1087,73 @@ private:
unsigned Count;
};
-class LexerBasedFormatTokenSource : public FormatTokenSource {
+class FormatTokenLexer {
public:
- LexerBasedFormatTokenSource(Lexer &Lex, SourceManager &SourceMgr)
- : GreaterStashed(false), TrailingWhitespace(0), Lex(Lex),
+ FormatTokenLexer(Lexer &Lex, SourceManager &SourceMgr)
+ : FormatTok(NULL), GreaterStashed(false), TrailingWhitespace(0), Lex(Lex),
SourceMgr(SourceMgr), IdentTable(Lex.getLangOpts()) {
Lex.SetKeepWhitespaceMode(true);
}
- virtual FormatToken getNextToken() {
+ ArrayRef<FormatToken *> lex() {
+ assert(Tokens.empty());
+ do {
+ Tokens.push_back(getNextToken());
+ } while (Tokens.back()->Tok.isNot(tok::eof));
+ return Tokens;
+ }
+
+ IdentifierTable &getIdentTable() { return IdentTable; }
+
+private:
+ FormatToken *getNextToken() {
if (GreaterStashed) {
- FormatTok.NewlinesBefore = 0;
+ FormatTok = new (Allocator.Allocate()) FormatToken(*FormatTok);
+ FormatTok->NewlinesBefore = 0;
SourceLocation GreaterLocation =
- FormatTok.Tok.getLocation().getLocWithOffset(1);
- FormatTok.WhitespaceRange = SourceRange(GreaterLocation, GreaterLocation);
+ FormatTok->Tok.getLocation().getLocWithOffset(1);
+ FormatTok->WhitespaceRange =
+ SourceRange(GreaterLocation, GreaterLocation);
GreaterStashed = false;
return FormatTok;
}
- FormatTok = FormatToken();
- Lex.LexFromRawLexer(FormatTok.Tok);
- StringRef Text = rawTokenText(FormatTok.Tok);
+ FormatTok = new (Allocator.Allocate()) FormatToken;
+ Lex.LexFromRawLexer(FormatTok->Tok);
+ StringRef Text = rawTokenText(FormatTok->Tok);
SourceLocation WhitespaceStart =
- FormatTok.Tok.getLocation().getLocWithOffset(-TrailingWhitespace);
+ FormatTok->Tok.getLocation().getLocWithOffset(-TrailingWhitespace);
if (SourceMgr.getFileOffset(WhitespaceStart) == 0)
- FormatTok.IsFirst = true;
+ FormatTok->IsFirst = true;
// Consume and record whitespace until we find a significant token.
unsigned WhitespaceLength = TrailingWhitespace;
- while (FormatTok.Tok.is(tok::unknown)) {
+ while (FormatTok->Tok.is(tok::unknown)) {
unsigned Newlines = Text.count('\n');
if (Newlines > 0)
- FormatTok.LastNewlineOffset = WhitespaceLength + Text.rfind('\n') + 1;
+ FormatTok->LastNewlineOffset = WhitespaceLength + Text.rfind('\n') + 1;
unsigned EscapedNewlines = Text.count("\\\n");
- FormatTok.NewlinesBefore += Newlines;
- FormatTok.HasUnescapedNewline |= EscapedNewlines != Newlines;
- WhitespaceLength += FormatTok.Tok.getLength();
+ FormatTok->NewlinesBefore += Newlines;
+ FormatTok->HasUnescapedNewline |= EscapedNewlines != Newlines;
+ WhitespaceLength += FormatTok->Tok.getLength();
- if (FormatTok.Tok.is(tok::eof)) {
- FormatTok.WhitespaceRange =
+ if (FormatTok->Tok.is(tok::eof)) {
+ FormatTok->WhitespaceRange =
SourceRange(WhitespaceStart,
WhitespaceStart.getLocWithOffset(WhitespaceLength));
return FormatTok;
}
- Lex.LexFromRawLexer(FormatTok.Tok);
- Text = rawTokenText(FormatTok.Tok);
+ Lex.LexFromRawLexer(FormatTok->Tok);
+ Text = rawTokenText(FormatTok->Tok);
}
// Now FormatTok is the next non-whitespace token.
- FormatTok.TokenLength = Text.size();
+ FormatTok->TokenLength = Text.size();
TrailingWhitespace = 0;
- if (FormatTok.Tok.is(tok::comment)) {
+ if (FormatTok->Tok.is(tok::comment)) {
TrailingWhitespace = Text.size() - Text.rtrim().size();
- FormatTok.TokenLength -= TrailingWhitespace;
+ FormatTok->TokenLength -= TrailingWhitespace;
}
// In case the token starts with escaped newlines, we want to
@@ -1151,41 +1164,40 @@ public:
// FIXME: Add a more explicit test.
unsigned i = 0;
while (i + 1 < Text.size() && Text[i] == '\\' && Text[i + 1] == '\n') {
- // FIXME: ++FormatTok.NewlinesBefore is missing...
+ // FIXME: ++FormatTok->NewlinesBefore is missing...
WhitespaceLength += 2;
- FormatTok.TokenLength -= 2;
+ FormatTok->TokenLength -= 2;
i += 2;
}
- if (FormatTok.Tok.is(tok::raw_identifier)) {
+ if (FormatTok->Tok.is(tok::raw_identifier)) {
IdentifierInfo &Info = IdentTable.get(Text);
- FormatTok.Tok.setIdentifierInfo(&Info);
- FormatTok.Tok.setKind(Info.getTokenID());
+ FormatTok->Tok.setIdentifierInfo(&Info);
+ FormatTok->Tok.setKind(Info.getTokenID());
}
- if (FormatTok.Tok.is(tok::greatergreater)) {
- FormatTok.Tok.setKind(tok::greater);
- FormatTok.TokenLength = 1;
+ if (FormatTok->Tok.is(tok::greatergreater)) {
+ FormatTok->Tok.setKind(tok::greater);
+ FormatTok->TokenLength = 1;
GreaterStashed = true;
}
- FormatTok.WhitespaceRange = SourceRange(
+ FormatTok->WhitespaceRange = SourceRange(
WhitespaceStart, WhitespaceStart.getLocWithOffset(WhitespaceLength));
- FormatTok.TokenText = StringRef(
- SourceMgr.getCharacterData(FormatTok.getStartOfNonWhitespace()),
- FormatTok.TokenLength);
+ FormatTok->TokenText = StringRef(
+ SourceMgr.getCharacterData(FormatTok->getStartOfNonWhitespace()),
+ FormatTok->TokenLength);
return FormatTok;
}
- IdentifierTable &getIdentTable() { return IdentTable; }
-
-private:
- FormatToken FormatTok;
+ FormatToken *FormatTok;
bool GreaterStashed;
unsigned TrailingWhitespace;
Lexer &Lex;
SourceManager &SourceMgr;
IdentifierTable IdentTable;
+ llvm::SpecificBumpPtrAllocator<FormatToken> Allocator;
+ SmallVector<FormatToken *, 16> Tokens;
/// Returns the text of \c FormatTok.
StringRef rawTokenText(Token &Tok) {
@@ -1204,8 +1216,9 @@ public:
virtual ~Formatter() {}
tooling::Replacements format() {
- LexerBasedFormatTokenSource Tokens(Lex, SourceMgr);
- UnwrappedLineParser Parser(Style, Tokens, *this);
+ FormatTokenLexer Tokens(Lex, SourceMgr);
+
+ UnwrappedLineParser Parser(Style, Tokens.lex(), *this);
bool StructuralError = Parser.parse();
TokenAnnotator Annotator(Style, SourceMgr, Lex,
Tokens.getIdentTable().get("in"));
Modified: cfe/trunk/lib/Format/UnwrappedLineParser.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Format/UnwrappedLineParser.cpp?rev=182768&r1=182767&r2=182768&view=diff
==============================================================================
--- cfe/trunk/lib/Format/UnwrappedLineParser.cpp (original)
+++ cfe/trunk/lib/Format/UnwrappedLineParser.cpp Tue May 28 06:55:06 2013
@@ -21,6 +21,15 @@
namespace clang {
namespace format {
+class FormatTokenSource {
+public:
+ virtual ~FormatTokenSource() {}
+ virtual FormatToken *getNextToken() = 0;
+
+ virtual unsigned getPosition() = 0;
+ virtual FormatToken *setPosition(unsigned Position) = 0;
+};
+
class ScopedDeclarationState {
public:
ScopedDeclarationState(UnwrappedLine &Line, std::vector<bool> &Stack,
@@ -44,11 +53,11 @@ private:
class ScopedMacroState : public FormatTokenSource {
public:
ScopedMacroState(UnwrappedLine &Line, FormatTokenSource *&TokenSource,
- FormatToken &ResetToken, bool &StructuralError)
+ FormatToken *&ResetToken, bool &StructuralError)
: Line(Line), TokenSource(TokenSource), ResetToken(ResetToken),
PreviousLineLevel(Line.Level), PreviousTokenSource(TokenSource),
StructuralError(StructuralError),
- PreviousStructuralError(StructuralError) {
+ PreviousStructuralError(StructuralError), Token(NULL) {
TokenSource = this;
Line.Level = 0;
Line.InPPDirective = true;
@@ -62,13 +71,13 @@ public:
StructuralError = PreviousStructuralError;
}
- virtual FormatToken getNextToken() {
+ virtual FormatToken *getNextToken() {
// The \c UnwrappedLineParser guards against this by never calling
// \c getNextToken() after it has encountered the first eof token.
assert(!eof());
Token = PreviousTokenSource->getNextToken();
if (eof())
- return createEOF();
+ return getFakeEOF();
return Token;
}
@@ -76,30 +85,34 @@ public:
return PreviousTokenSource->getPosition();
}
- virtual FormatToken setPosition(unsigned Position) {
+ virtual FormatToken *setPosition(unsigned Position) {
Token = PreviousTokenSource->setPosition(Position);
return Token;
}
private:
- bool eof() { return Token.HasUnescapedNewline; }
+ bool eof() { return Token && Token->HasUnescapedNewline; }
- FormatToken createEOF() {
- FormatToken FormatTok;
- FormatTok.Tok.startToken();
- FormatTok.Tok.setKind(tok::eof);
- return FormatTok;
+ FormatToken *getFakeEOF() {
+ static bool EOFInitialized = false;
+ static FormatToken FormatTok;
+ if (!EOFInitialized) {
+ FormatTok.Tok.startToken();
+ FormatTok.Tok.setKind(tok::eof);
+ EOFInitialized = true;
+ }
+ return &FormatTok;
}
UnwrappedLine &Line;
FormatTokenSource *&TokenSource;
- FormatToken &ResetToken;
+ FormatToken *&ResetToken;
unsigned PreviousLineLevel;
FormatTokenSource *PreviousTokenSource;
bool &StructuralError;
bool PreviousStructuralError;
- FormatToken Token;
+ FormatToken *Token;
};
class ScopedLineState {
@@ -135,10 +148,10 @@ private:
class IndexedTokenSource : public FormatTokenSource {
public:
- IndexedTokenSource(ArrayRef<FormatToken> Tokens)
+ IndexedTokenSource(ArrayRef<FormatToken *> Tokens)
: Tokens(Tokens), Position(-1) {}
- virtual FormatToken getNextToken() {
+ virtual FormatToken *getNextToken() {
++Position;
return Tokens[Position];
}
@@ -148,28 +161,23 @@ public:
return Position;
}
- virtual FormatToken setPosition(unsigned P) {
+ virtual FormatToken *setPosition(unsigned P) {
Position = P;
return Tokens[Position];
}
private:
- ArrayRef<FormatToken> Tokens;
+ ArrayRef<FormatToken *> Tokens;
int Position;
};
UnwrappedLineParser::UnwrappedLineParser(const FormatStyle &Style,
- FormatTokenSource &Tokens,
+ ArrayRef<FormatToken *> Tokens,
UnwrappedLineConsumer &Callback)
: Line(new UnwrappedLine), MustBreakBeforeNextToken(false),
- CurrentLines(&Lines), StructuralError(false), Style(Style),
- Tokens(NULL), Callback(Callback) {
- FormatToken Tok;
- do {
- Tok = Tokens.getNextToken();
- AllTokens.push_back(Tok);
- } while (Tok.Tok.isNot(tok::eof));
- LBraces.resize(AllTokens.size(), BS_Unknown);
+ CurrentLines(&Lines), StructuralError(false), Style(Style), Tokens(NULL),
+ Callback(Callback), AllTokens(Tokens) {
+ LBraces.resize(Tokens.size(), BS_Unknown);
}
bool UnwrappedLineParser::parse() {
@@ -201,7 +209,7 @@ void UnwrappedLineParser::parseFile() {
void UnwrappedLineParser::parseLevel(bool HasOpeningBrace) {
do {
- switch (FormatTok.Tok.getKind()) {
+ switch (FormatTok->Tok.getKind()) {
case tok::comment:
nextToken();
addUnwrappedLine();
@@ -233,15 +241,15 @@ void UnwrappedLineParser::calculateBrace
// definitions, too.
unsigned StoredPosition = Tokens->getPosition();
unsigned Position = StoredPosition;
- FormatToken Tok = FormatTok;
+ FormatToken *Tok = FormatTok;
// Keep a stack of positions of lbrace tokens. We will
// update information about whether an lbrace starts a
// braced init list or a different block during the loop.
SmallVector<unsigned, 8> LBraceStack;
- assert(Tok.Tok.is(tok::l_brace));
+ assert(Tok->Tok.is(tok::l_brace));
do {
- FormatToken NextTok = Tokens->getNextToken();
- switch (Tok.Tok.getKind()) {
+ FormatToken *NextTok = Tokens->getNextToken();
+ switch (Tok->Tok.getKind()) {
case tok::l_brace:
LBraceStack.push_back(Position);
break;
@@ -256,8 +264,8 @@ void UnwrappedLineParser::calculateBrace
// Thus, if the parent is a braced init list, we consider all
// brace blocks inside it braced init list. That works good enough
// for now, but we will need to fix it to correctly handle lambdas.
- if (NextTok.Tok.is(tok::comma) || NextTok.Tok.is(tok::semi) ||
- NextTok.Tok.is(tok::r_paren) || NextTok.Tok.is(tok::l_brace))
+ if (NextTok->Tok.is(tok::comma) || NextTok->Tok.is(tok::semi) ||
+ NextTok->Tok.is(tok::r_paren) || NextTok->Tok.is(tok::l_brace))
LBraces[LBraceStack.back()] = BS_BracedInit;
else
LBraces[LBraceStack.back()] = BS_Block;
@@ -279,7 +287,7 @@ void UnwrappedLineParser::calculateBrace
}
Tok = NextTok;
++Position;
- } while (Tok.Tok.isNot(tok::eof));
+ } while (Tok->Tok.isNot(tok::eof));
// Assume other blocks for all unclosed opening braces.
for (unsigned i = 0, e = LBraceStack.size(); i != e; ++i) {
if (LBraces[LBraceStack[i]] == BS_Unknown)
@@ -290,7 +298,7 @@ void UnwrappedLineParser::calculateBrace
void UnwrappedLineParser::parseBlock(bool MustBeDeclaration,
unsigned AddLevels) {
- assert(FormatTok.Tok.is(tok::l_brace) && "'{' expected");
+ assert(FormatTok->Tok.is(tok::l_brace) && "'{' expected");
nextToken();
addUnwrappedLine();
@@ -300,7 +308,7 @@ void UnwrappedLineParser::parseBlock(boo
Line->Level += AddLevels;
parseLevel(/*HasOpeningBrace=*/ true);
- if (!FormatTok.Tok.is(tok::r_brace)) {
+ if (!FormatTok->Tok.is(tok::r_brace)) {
Line->Level -= AddLevels;
StructuralError = true;
return;
@@ -311,16 +319,16 @@ void UnwrappedLineParser::parseBlock(boo
}
void UnwrappedLineParser::parsePPDirective() {
- assert(FormatTok.Tok.is(tok::hash) && "'#' expected");
+ assert(FormatTok->Tok.is(tok::hash) && "'#' expected");
ScopedMacroState MacroState(*Line, Tokens, FormatTok, StructuralError);
nextToken();
- if (FormatTok.Tok.getIdentifierInfo() == NULL) {
+ if (FormatTok->Tok.getIdentifierInfo() == NULL) {
parsePPUnknown();
return;
}
- switch (FormatTok.Tok.getIdentifierInfo()->getPPKeywordID()) {
+ switch (FormatTok->Tok.getIdentifierInfo()->getPPKeywordID()) {
case tok::pp_define:
parsePPDefine();
return;
@@ -355,10 +363,10 @@ void UnwrappedLineParser::pushPPConditio
void UnwrappedLineParser::parsePPIf() {
nextToken();
- if ((FormatTok.Tok.isLiteral() &&
- StringRef(FormatTok.Tok.getLiteralData(), FormatTok.Tok.getLength()) ==
+ if ((FormatTok->Tok.isLiteral() &&
+ StringRef(FormatTok->Tok.getLiteralData(), FormatTok->Tok.getLength()) ==
"0") ||
- FormatTok.Tok.is(tok::kw_false)) {
+ FormatTok->Tok.is(tok::kw_false)) {
PPStack.push_back(PP_Unreachable);
} else {
pushPPConditional();
@@ -391,14 +399,14 @@ void UnwrappedLineParser::parsePPEndIf()
void UnwrappedLineParser::parsePPDefine() {
nextToken();
- if (FormatTok.Tok.getKind() != tok::identifier) {
+ if (FormatTok->Tok.getKind() != tok::identifier) {
parsePPUnknown();
return;
}
nextToken();
- if (FormatTok.Tok.getKind() == tok::l_paren &&
- FormatTok.WhitespaceRange.getBegin() ==
- FormatTok.WhitespaceRange.getEnd()) {
+ if (FormatTok->Tok.getKind() == tok::l_paren &&
+ FormatTok->WhitespaceRange.getBegin() ==
+ FormatTok->WhitespaceRange.getEnd()) {
parseParens();
}
addUnwrappedLine();
@@ -447,15 +455,15 @@ bool tokenCanStartNewLine(clang::Token T
}
void UnwrappedLineParser::parseStructuralElement() {
- assert(!FormatTok.Tok.is(tok::l_brace));
- switch (FormatTok.Tok.getKind()) {
+ assert(!FormatTok->Tok.is(tok::l_brace));
+ switch (FormatTok->Tok.getKind()) {
case tok::at:
nextToken();
- if (FormatTok.Tok.is(tok::l_brace)) {
+ if (FormatTok->Tok.is(tok::l_brace)) {
parseBracedList();
break;
}
- switch (FormatTok.Tok.getObjCKeywordID()) {
+ switch (FormatTok->Tok.getObjCKeywordID()) {
case tok::objc_public:
case tok::objc_protected:
case tok::objc_package:
@@ -482,7 +490,7 @@ void UnwrappedLineParser::parseStructura
return;
case tok::kw_inline:
nextToken();
- if (FormatTok.Tok.is(tok::kw_namespace)) {
+ if (FormatTok->Tok.is(tok::kw_namespace)) {
parseNamespace();
return;
}
@@ -517,9 +525,9 @@ void UnwrappedLineParser::parseStructura
return;
case tok::kw_extern:
nextToken();
- if (FormatTok.Tok.is(tok::string_literal)) {
+ if (FormatTok->Tok.is(tok::string_literal)) {
nextToken();
- if (FormatTok.Tok.is(tok::l_brace)) {
+ if (FormatTok->Tok.is(tok::l_brace)) {
parseBlock(/*MustBeDeclaration=*/ true, 0);
addUnwrappedLine();
return;
@@ -531,10 +539,10 @@ void UnwrappedLineParser::parseStructura
break;
}
do {
- switch (FormatTok.Tok.getKind()) {
+ switch (FormatTok->Tok.getKind()) {
case tok::at:
nextToken();
- if (FormatTok.Tok.is(tok::l_brace))
+ if (FormatTok->Tok.is(tok::l_brace))
parseBracedList();
break;
case tok::kw_enum:
@@ -576,15 +584,15 @@ void UnwrappedLineParser::parseStructura
case tok::identifier:
nextToken();
if (Line->Tokens.size() == 1) {
- if (FormatTok.Tok.is(tok::colon)) {
+ if (FormatTok->Tok.is(tok::colon)) {
parseLabel();
return;
}
// Recognize function-like macro usages without trailing semicolon.
- if (FormatTok.Tok.is(tok::l_paren)) {
+ if (FormatTok->Tok.is(tok::l_paren)) {
parseParens();
- if (FormatTok.HasUnescapedNewline &&
- tokenCanStartNewLine(FormatTok.Tok)) {
+ if (FormatTok->HasUnescapedNewline &&
+ tokenCanStartNewLine(FormatTok->Tok)) {
addUnwrappedLine();
return;
}
@@ -593,7 +601,7 @@ void UnwrappedLineParser::parseStructura
break;
case tok::equal:
nextToken();
- if (FormatTok.Tok.is(tok::l_brace)) {
+ if (FormatTok->Tok.is(tok::l_brace)) {
parseBracedList();
}
break;
@@ -624,7 +632,7 @@ void UnwrappedLineParser::parseBracedLis
// FIXME: When we start to support lambdas, we'll want to parse them away
// here, otherwise our bail-out scenarios below break. The better solution
// might be to just implement a more or less complete expression parser.
- switch (FormatTok.Tok.getKind()) {
+ switch (FormatTok->Tok.getKind()) {
case tok::l_brace:
if (!StartOfExpression) {
// Probably a missing closing brace. Bail out.
@@ -656,10 +664,10 @@ void UnwrappedLineParser::parseReturn()
nextToken();
do {
- switch (FormatTok.Tok.getKind()) {
+ switch (FormatTok->Tok.getKind()) {
case tok::l_brace:
parseBracedList();
- if (FormatTok.Tok.isNot(tok::semi)) {
+ if (FormatTok->Tok.isNot(tok::semi)) {
// Assume missing ';'.
addUnwrappedLine();
return;
@@ -684,10 +692,10 @@ void UnwrappedLineParser::parseReturn()
}
void UnwrappedLineParser::parseParens() {
- assert(FormatTok.Tok.is(tok::l_paren) && "'(' expected.");
+ assert(FormatTok->Tok.is(tok::l_paren) && "'(' expected.");
nextToken();
do {
- switch (FormatTok.Tok.getKind()) {
+ switch (FormatTok->Tok.getKind()) {
case tok::l_paren:
parseParens();
break;
@@ -708,7 +716,7 @@ void UnwrappedLineParser::parseParens()
}
case tok::at:
nextToken();
- if (FormatTok.Tok.is(tok::l_brace))
+ if (FormatTok->Tok.is(tok::l_brace))
parseBracedList();
break;
default:
@@ -719,12 +727,12 @@ void UnwrappedLineParser::parseParens()
}
void UnwrappedLineParser::parseIfThenElse() {
- assert(FormatTok.Tok.is(tok::kw_if) && "'if' expected");
+ assert(FormatTok->Tok.is(tok::kw_if) && "'if' expected");
nextToken();
- if (FormatTok.Tok.is(tok::l_paren))
+ if (FormatTok->Tok.is(tok::l_paren))
parseParens();
bool NeedsUnwrappedLine = false;
- if (FormatTok.Tok.is(tok::l_brace)) {
+ if (FormatTok->Tok.is(tok::l_brace)) {
parseBlock(/*MustBeDeclaration=*/ false);
NeedsUnwrappedLine = true;
} else {
@@ -733,12 +741,12 @@ void UnwrappedLineParser::parseIfThenEls
parseStructuralElement();
--Line->Level;
}
- if (FormatTok.Tok.is(tok::kw_else)) {
+ if (FormatTok->Tok.is(tok::kw_else)) {
nextToken();
- if (FormatTok.Tok.is(tok::l_brace)) {
+ if (FormatTok->Tok.is(tok::l_brace)) {
parseBlock(/*MustBeDeclaration=*/ false);
addUnwrappedLine();
- } else if (FormatTok.Tok.is(tok::kw_if)) {
+ } else if (FormatTok->Tok.is(tok::kw_if)) {
parseIfThenElse();
} else {
addUnwrappedLine();
@@ -752,18 +760,18 @@ void UnwrappedLineParser::parseIfThenEls
}
void UnwrappedLineParser::parseNamespace() {
- assert(FormatTok.Tok.is(tok::kw_namespace) && "'namespace' expected");
+ assert(FormatTok->Tok.is(tok::kw_namespace) && "'namespace' expected");
nextToken();
- if (FormatTok.Tok.is(tok::identifier))
+ if (FormatTok->Tok.is(tok::identifier))
nextToken();
- if (FormatTok.Tok.is(tok::l_brace)) {
+ if (FormatTok->Tok.is(tok::l_brace)) {
if (Style.BreakBeforeBraces == FormatStyle::BS_Linux)
addUnwrappedLine();
parseBlock(/*MustBeDeclaration=*/ true, 0);
// Munch the semicolon after a namespace. This is more common than one would
// think. Puttin the semicolon into its own line is very ugly.
- if (FormatTok.Tok.is(tok::semi))
+ if (FormatTok->Tok.is(tok::semi))
nextToken();
addUnwrappedLine();
}
@@ -771,12 +779,12 @@ void UnwrappedLineParser::parseNamespace
}
void UnwrappedLineParser::parseForOrWhileLoop() {
- assert((FormatTok.Tok.is(tok::kw_for) || FormatTok.Tok.is(tok::kw_while)) &&
+ assert((FormatTok->Tok.is(tok::kw_for) || FormatTok->Tok.is(tok::kw_while)) &&
"'for' or 'while' expected");
nextToken();
- if (FormatTok.Tok.is(tok::l_paren))
+ if (FormatTok->Tok.is(tok::l_paren))
parseParens();
- if (FormatTok.Tok.is(tok::l_brace)) {
+ if (FormatTok->Tok.is(tok::l_brace)) {
parseBlock(/*MustBeDeclaration=*/ false);
addUnwrappedLine();
} else {
@@ -788,9 +796,9 @@ void UnwrappedLineParser::parseForOrWhil
}
void UnwrappedLineParser::parseDoWhile() {
- assert(FormatTok.Tok.is(tok::kw_do) && "'do' expected");
+ assert(FormatTok->Tok.is(tok::kw_do) && "'do' expected");
nextToken();
- if (FormatTok.Tok.is(tok::l_brace)) {
+ if (FormatTok->Tok.is(tok::l_brace)) {
parseBlock(/*MustBeDeclaration=*/ false);
} else {
addUnwrappedLine();
@@ -800,7 +808,7 @@ void UnwrappedLineParser::parseDoWhile()
}
// FIXME: Add error handling.
- if (!FormatTok.Tok.is(tok::kw_while)) {
+ if (!FormatTok->Tok.is(tok::kw_while)) {
addUnwrappedLine();
return;
}
@@ -810,15 +818,15 @@ void UnwrappedLineParser::parseDoWhile()
}
void UnwrappedLineParser::parseLabel() {
- if (FormatTok.Tok.isNot(tok::colon))
+ if (FormatTok->Tok.isNot(tok::colon))
return;
nextToken();
unsigned OldLineLevel = Line->Level;
if (Line->Level > 1 || (!Line->InPPDirective && Line->Level > 0))
--Line->Level;
- if (CommentsBeforeNextToken.empty() && FormatTok.Tok.is(tok::l_brace)) {
+ if (CommentsBeforeNextToken.empty() && FormatTok->Tok.is(tok::l_brace)) {
parseBlock(/*MustBeDeclaration=*/ false);
- if (FormatTok.Tok.is(tok::kw_break))
+ if (FormatTok->Tok.is(tok::kw_break))
parseStructuralElement(); // "break;" after "}" goes on the same line.
}
addUnwrappedLine();
@@ -826,20 +834,20 @@ void UnwrappedLineParser::parseLabel() {
}
void UnwrappedLineParser::parseCaseLabel() {
- assert(FormatTok.Tok.is(tok::kw_case) && "'case' expected");
+ assert(FormatTok->Tok.is(tok::kw_case) && "'case' expected");
// FIXME: fix handling of complex expressions here.
do {
nextToken();
- } while (!eof() && !FormatTok.Tok.is(tok::colon));
+ } while (!eof() && !FormatTok->Tok.is(tok::colon));
parseLabel();
}
void UnwrappedLineParser::parseSwitch() {
- assert(FormatTok.Tok.is(tok::kw_switch) && "'switch' expected");
+ assert(FormatTok->Tok.is(tok::kw_switch) && "'switch' expected");
nextToken();
- if (FormatTok.Tok.is(tok::l_paren))
+ if (FormatTok->Tok.is(tok::l_paren))
parseParens();
- if (FormatTok.Tok.is(tok::l_brace)) {
+ if (FormatTok->Tok.is(tok::l_brace)) {
parseBlock(/*MustBeDeclaration=*/ false, Style.IndentCaseLabels ? 2 : 1);
addUnwrappedLine();
} else {
@@ -853,30 +861,30 @@ void UnwrappedLineParser::parseSwitch()
void UnwrappedLineParser::parseAccessSpecifier() {
nextToken();
// Otherwise, we don't know what it is, and we'd better keep the next token.
- if (FormatTok.Tok.is(tok::colon))
+ if (FormatTok->Tok.is(tok::colon))
nextToken();
addUnwrappedLine();
}
void UnwrappedLineParser::parseEnum() {
nextToken();
- if (FormatTok.Tok.is(tok::identifier) ||
- FormatTok.Tok.is(tok::kw___attribute) ||
- FormatTok.Tok.is(tok::kw___declspec)) {
+ if (FormatTok->Tok.is(tok::identifier) ||
+ FormatTok->Tok.is(tok::kw___attribute) ||
+ FormatTok->Tok.is(tok::kw___declspec)) {
nextToken();
// We can have macros or attributes in between 'enum' and the enum name.
- if (FormatTok.Tok.is(tok::l_paren)) {
+ if (FormatTok->Tok.is(tok::l_paren)) {
parseParens();
}
- if (FormatTok.Tok.is(tok::identifier))
+ if (FormatTok->Tok.is(tok::identifier))
nextToken();
}
- if (FormatTok.Tok.is(tok::l_brace)) {
+ if (FormatTok->Tok.is(tok::l_brace)) {
nextToken();
addUnwrappedLine();
++Line->Level;
do {
- switch (FormatTok.Tok.getKind()) {
+ switch (FormatTok->Tok.getKind()) {
case tok::l_paren:
parseParens();
break;
@@ -902,18 +910,19 @@ void UnwrappedLineParser::parseEnum() {
void UnwrappedLineParser::parseRecord() {
nextToken();
- if (FormatTok.Tok.is(tok::identifier) ||
- FormatTok.Tok.is(tok::kw___attribute) ||
- FormatTok.Tok.is(tok::kw___declspec)) {
+ if (FormatTok->Tok.is(tok::identifier) ||
+ FormatTok->Tok.is(tok::kw___attribute) ||
+ FormatTok->Tok.is(tok::kw___declspec)) {
nextToken();
// We can have macros or attributes in between 'class' and the class name.
- if (FormatTok.Tok.is(tok::l_paren)) {
+ if (FormatTok->Tok.is(tok::l_paren)) {
parseParens();
}
// The actual identifier can be a nested name specifier, and in macros
// it is often token-pasted.
- while (FormatTok.Tok.is(tok::identifier) ||
- FormatTok.Tok.is(tok::coloncolon) || FormatTok.Tok.is(tok::hashhash))
+ while (FormatTok->Tok.is(tok::identifier) ||
+ FormatTok->Tok.is(tok::coloncolon) ||
+ FormatTok->Tok.is(tok::hashhash))
nextToken();
// Note that parsing away template declarations here leads to incorrectly
@@ -926,15 +935,15 @@ void UnwrappedLineParser::parseRecord()
// and thus rule out the record production in case there is no template
// (this would still leave us with an ambiguity between template function
// and class declarations).
- if (FormatTok.Tok.is(tok::colon) || FormatTok.Tok.is(tok::less)) {
- while (!eof() && FormatTok.Tok.isNot(tok::l_brace)) {
- if (FormatTok.Tok.is(tok::semi))
+ if (FormatTok->Tok.is(tok::colon) || FormatTok->Tok.is(tok::less)) {
+ while (!eof() && FormatTok->Tok.isNot(tok::l_brace)) {
+ if (FormatTok->Tok.is(tok::semi))
return;
nextToken();
}
}
}
- if (FormatTok.Tok.is(tok::l_brace)) {
+ if (FormatTok->Tok.is(tok::l_brace)) {
if (Style.BreakBeforeBraces == FormatStyle::BS_Linux)
addUnwrappedLine();
@@ -946,16 +955,16 @@ void UnwrappedLineParser::parseRecord()
}
void UnwrappedLineParser::parseObjCProtocolList() {
- assert(FormatTok.Tok.is(tok::less) && "'<' expected.");
+ assert(FormatTok->Tok.is(tok::less) && "'<' expected.");
do
nextToken();
- while (!eof() && FormatTok.Tok.isNot(tok::greater));
+ while (!eof() && FormatTok->Tok.isNot(tok::greater));
nextToken(); // Skip '>'.
}
void UnwrappedLineParser::parseObjCUntilAtEnd() {
do {
- if (FormatTok.Tok.isObjCAtKeyword(tok::objc_end)) {
+ if (FormatTok->Tok.isObjCAtKeyword(tok::objc_end)) {
nextToken();
addUnwrappedLine();
break;
@@ -969,18 +978,18 @@ void UnwrappedLineParser::parseObjCInter
nextToken(); // interface name
// @interface can be followed by either a base class, or a category.
- if (FormatTok.Tok.is(tok::colon)) {
+ if (FormatTok->Tok.is(tok::colon)) {
nextToken();
nextToken(); // base class name
- } else if (FormatTok.Tok.is(tok::l_paren))
+ } else if (FormatTok->Tok.is(tok::l_paren))
// Skip category, if present.
parseParens();
- if (FormatTok.Tok.is(tok::less))
+ if (FormatTok->Tok.is(tok::less))
parseObjCProtocolList();
// If instance variables are present, keep the '{' on the first line too.
- if (FormatTok.Tok.is(tok::l_brace))
+ if (FormatTok->Tok.is(tok::l_brace))
parseBlock(/*MustBeDeclaration=*/ true);
// With instance variables, this puts '}' on its own line. Without instance
@@ -994,11 +1003,11 @@ void UnwrappedLineParser::parseObjCProto
nextToken();
nextToken(); // protocol name
- if (FormatTok.Tok.is(tok::less))
+ if (FormatTok->Tok.is(tok::less))
parseObjCProtocolList();
// Check for protocol declaration.
- if (FormatTok.Tok.is(tok::semi)) {
+ if (FormatTok->Tok.is(tok::semi)) {
nextToken();
return addUnwrappedLine();
}
@@ -1034,15 +1043,15 @@ void UnwrappedLineParser::addUnwrappedLi
}
}
-bool UnwrappedLineParser::eof() const { return FormatTok.Tok.is(tok::eof); }
+bool UnwrappedLineParser::eof() const { return FormatTok->Tok.is(tok::eof); }
void UnwrappedLineParser::flushComments(bool NewlineBeforeNext) {
bool JustComments = Line->Tokens.empty();
- for (SmallVectorImpl<FormatToken>::const_iterator
+ for (SmallVectorImpl<FormatToken *>::const_iterator
I = CommentsBeforeNextToken.begin(),
E = CommentsBeforeNextToken.end();
I != E; ++I) {
- if (I->NewlinesBefore && JustComments) {
+ if ((*I)->NewlinesBefore && JustComments) {
addUnwrappedLine();
}
pushToken(*I);
@@ -1056,7 +1065,7 @@ void UnwrappedLineParser::flushComments(
void UnwrappedLineParser::nextToken() {
if (eof())
return;
- flushComments(FormatTok.NewlinesBefore > 0);
+ flushComments(FormatTok->NewlinesBefore > 0);
pushToken(FormatTok);
readToken();
}
@@ -1065,8 +1074,8 @@ void UnwrappedLineParser::readToken() {
bool CommentsInCurrentLine = true;
do {
FormatTok = Tokens->getNextToken();
- while (!Line->InPPDirective && FormatTok.Tok.is(tok::hash) &&
- (FormatTok.HasUnescapedNewline || FormatTok.IsFirst)) {
+ while (!Line->InPPDirective && FormatTok->Tok.is(tok::hash) &&
+ (FormatTok->HasUnescapedNewline || FormatTok->IsFirst)) {
// If there is an unfinished unwrapped line, we flush the preprocessor
// directives only after that unwrapped line was finished later.
bool SwitchToPreprocessorLines =
@@ -1075,7 +1084,7 @@ void UnwrappedLineParser::readToken() {
// Comments stored before the preprocessor directive need to be output
// before the preprocessor directive, at the same level as the
// preprocessor directive, as we consider them to apply to the directive.
- flushComments(FormatTok.NewlinesBefore > 0);
+ flushComments(FormatTok->NewlinesBefore > 0);
parsePPDirective();
}
@@ -1084,9 +1093,9 @@ void UnwrappedLineParser::readToken() {
continue;
}
- if (!FormatTok.Tok.is(tok::comment))
+ if (!FormatTok->Tok.is(tok::comment))
return;
- if (FormatTok.NewlinesBefore > 0 || FormatTok.IsFirst) {
+ if (FormatTok->NewlinesBefore > 0 || FormatTok->IsFirst) {
CommentsInCurrentLine = false;
}
if (CommentsInCurrentLine) {
@@ -1097,8 +1106,8 @@ void UnwrappedLineParser::readToken() {
} while (!eof());
}
-void UnwrappedLineParser::pushToken(const FormatToken &Tok) {
- Line->Tokens.push_back(Tok);
+void UnwrappedLineParser::pushToken(FormatToken *Tok) {
+ Line->Tokens.push_back(*Tok);
if (MustBreakBeforeNextToken) {
Line->Tokens.back().MustBreakBefore = true;
MustBreakBeforeNextToken = false;
Modified: cfe/trunk/lib/Format/UnwrappedLineParser.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Format/UnwrappedLineParser.h?rev=182768&r1=182767&r2=182768&view=diff
==============================================================================
--- cfe/trunk/lib/Format/UnwrappedLineParser.h (original)
+++ cfe/trunk/lib/Format/UnwrappedLineParser.h Tue May 28 06:55:06 2013
@@ -111,24 +111,11 @@ public:
virtual void consumeUnwrappedLine(const UnwrappedLine &Line) = 0;
};
-class FormatTokenSource {
-public:
- virtual ~FormatTokenSource() {
- }
- virtual FormatToken getNextToken() = 0;
-
- // FIXME: This interface will become an implementation detail of
- // the UnwrappedLineParser once we switch to generate all tokens
- // up-front.
- virtual unsigned getPosition() { return 0; }
- virtual FormatToken setPosition(unsigned Position) {
- llvm_unreachable("Interface in transition; do not call!");
- }
-};
+class FormatTokenSource;
class UnwrappedLineParser {
public:
- UnwrappedLineParser(const FormatStyle &Style, FormatTokenSource &Tokens,
+ UnwrappedLineParser(const FormatStyle &Style, ArrayRef<FormatToken *> Tokens,
UnwrappedLineConsumer &Callback);
/// Returns true in case of a structural error.
@@ -170,7 +157,7 @@ private:
void nextToken();
void readToken();
void flushComments(bool NewlineBeforeNext);
- void pushToken(const FormatToken &Tok);
+ void pushToken(FormatToken *Tok);
void calculateBraceTypes();
void pushPPConditional();
@@ -190,8 +177,8 @@ private:
// line as the previous token, or not. If not, they belong to the next token.
// Since the next token might already be in a new unwrapped line, we need to
// store the comments belonging to that token.
- SmallVector<FormatToken, 1> CommentsBeforeNextToken;
- FormatToken FormatTok;
+ SmallVector<FormatToken *, 1> CommentsBeforeNextToken;
+ FormatToken *FormatTok;
bool MustBreakBeforeNextToken;
// The parsed lines. Only added to through \c CurrentLines.
@@ -223,7 +210,7 @@ private:
// FIXME: This is a temporary measure until we have reworked the ownership
// of the format tokens. The goal is to have the actual tokens created and
// owned outside of and handed into the UnwrappedLineParser.
- SmallVector<FormatToken, 16> AllTokens;
+ ArrayRef<FormatToken *> AllTokens;
// FIXME: Currently we cannot store attributes with tokens, as we treat
// them as read-only; thus, we now store the brace state indexed by the
More information about the cfe-commits
mailing list