[cfe-commits] r169605 - in /cfe/trunk/lib/Format: Format.cpp UnwrappedLineParser.cpp UnwrappedLineParser.h
Alexander Kornienko
alexfh at google.com
Fri Dec 7 08:15:44 PST 2012
Author: alexfh
Date: Fri Dec 7 10:15:44 2012
New Revision: 169605
URL: http://llvm.org/viewvc/llvm-project?rev=169605&view=rev
Log:
Clang-format: extracted FormatTokenSource from UnwrappedLineParser.
Summary: FormatTokenLexer is here, FormatTokenBuffer is on the way. This will allow to re-parse unwrapped lines when needed.
Reviewers: djasper, klimek
Reviewed By: klimek
CC: cfe-commits
Differential Revision: http://llvm-reviews.chandlerc.com/D186
Modified:
cfe/trunk/lib/Format/Format.cpp
cfe/trunk/lib/Format/UnwrappedLineParser.cpp
cfe/trunk/lib/Format/UnwrappedLineParser.h
Modified: cfe/trunk/lib/Format/Format.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Format/Format.cpp?rev=169605&r1=169604&r2=169605&view=diff
==============================================================================
--- cfe/trunk/lib/Format/Format.cpp (original)
+++ cfe/trunk/lib/Format/Format.cpp Fri Dec 7 10:15:44 2012
@@ -751,6 +751,67 @@
std::vector<TokenAnnotation> Annotations;
};
+class LexerBasedFormatTokenSource : public FormatTokenSource {
+public:
+ LexerBasedFormatTokenSource(Lexer &Lex, SourceManager &SourceMgr)
+ : GreaterStashed(false),
+ Lex(Lex),
+ SourceMgr(SourceMgr),
+ IdentTable(Lex.getLangOpts()) {
+ Lex.SetKeepWhitespaceMode(true);
+ }
+
+ virtual FormatToken getNextToken() {
+ if (GreaterStashed) {
+ FormatTok.NewlinesBefore = 0;
+ FormatTok.WhiteSpaceStart =
+ FormatTok.Tok.getLocation().getLocWithOffset(1);
+ FormatTok.WhiteSpaceLength = 0;
+ GreaterStashed = false;
+ return FormatTok;
+ }
+
+ FormatTok = FormatToken();
+ Lex.LexFromRawLexer(FormatTok.Tok);
+ FormatTok.WhiteSpaceStart = FormatTok.Tok.getLocation();
+
+ // Consume and record whitespace until we find a significant token.
+ while (FormatTok.Tok.is(tok::unknown)) {
+ FormatTok.NewlinesBefore += tokenText(FormatTok.Tok).count('\n');
+ FormatTok.WhiteSpaceLength += FormatTok.Tok.getLength();
+
+ if (FormatTok.Tok.is(tok::eof))
+ return FormatTok;
+ Lex.LexFromRawLexer(FormatTok.Tok);
+ }
+
+ if (FormatTok.Tok.is(tok::raw_identifier)) {
+ const IdentifierInfo &Info = IdentTable.get(tokenText(FormatTok.Tok));
+ FormatTok.Tok.setKind(Info.getTokenID());
+ }
+
+ if (FormatTok.Tok.is(tok::greatergreater)) {
+ FormatTok.Tok.setKind(tok::greater);
+ GreaterStashed = true;
+ }
+
+ return FormatTok;
+ }
+
+private:
+ FormatToken FormatTok;
+ bool GreaterStashed;
+ Lexer &Lex;
+ SourceManager &SourceMgr;
+ IdentifierTable IdentTable;
+
+ /// Returns the text of \c FormatTok.
+ StringRef tokenText(Token &Tok) {
+ return StringRef(SourceMgr.getCharacterData(Tok.getLocation()),
+ Tok.getLength());
+ }
+};
+
class Formatter : public UnwrappedLineConsumer {
public:
Formatter(const FormatStyle &Style, Lexer &Lex, SourceManager &SourceMgr,
@@ -766,7 +827,8 @@
}
tooling::Replacements format() {
- UnwrappedLineParser Parser(Style, Lex, SourceMgr, *this);
+ LexerBasedFormatTokenSource Tokens(Lex, SourceMgr);
+ UnwrappedLineParser Parser(Style, Tokens, *this);
StructuralError = Parser.parse();
for (std::vector<UnwrappedLine>::iterator I = UnwrappedLines.begin(),
E = UnwrappedLines.end();
Modified: cfe/trunk/lib/Format/UnwrappedLineParser.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Format/UnwrappedLineParser.cpp?rev=169605&r1=169604&r2=169605&view=diff
==============================================================================
--- cfe/trunk/lib/Format/UnwrappedLineParser.cpp (original)
+++ cfe/trunk/lib/Format/UnwrappedLineParser.cpp Fri Dec 7 10:15:44 2012
@@ -22,20 +22,16 @@
namespace clang {
namespace format {
-UnwrappedLineParser::UnwrappedLineParser(const FormatStyle &Style, Lexer &Lex,
- SourceManager &SourceMgr,
+UnwrappedLineParser::UnwrappedLineParser(const FormatStyle &Style,
+ FormatTokenSource &Tokens,
UnwrappedLineConsumer &Callback)
- : GreaterStashed(false),
- Style(Style),
- Lex(Lex),
- SourceMgr(SourceMgr),
- IdentTable(Lex.getLangOpts()),
+ : Style(Style),
+ Tokens(Tokens),
Callback(Callback) {
- Lex.SetKeepWhitespaceMode(true);
}
bool UnwrappedLineParser::parse() {
- parseToken();
+ FormatTok = Tokens.getNextToken();
return parseLevel();
}
@@ -371,47 +367,7 @@
if (eof())
return;
Line.Tokens.push_back(FormatTok);
- parseToken();
-}
-
-void UnwrappedLineParser::parseToken() {
- if (GreaterStashed) {
- FormatTok.NewlinesBefore = 0;
- FormatTok.WhiteSpaceStart = FormatTok.Tok.getLocation().getLocWithOffset(1);
- FormatTok.WhiteSpaceLength = 0;
- GreaterStashed = false;
- return;
- }
-
- FormatTok = FormatToken();
- Lex.LexFromRawLexer(FormatTok.Tok);
- FormatTok.WhiteSpaceStart = FormatTok.Tok.getLocation();
-
- // Consume and record whitespace until we find a significant token.
- while (FormatTok.Tok.is(tok::unknown)) {
- FormatTok.NewlinesBefore += tokenText().count('\n');
- FormatTok.WhiteSpaceLength += FormatTok.Tok.getLength();
-
- if (eof())
- return;
- Lex.LexFromRawLexer(FormatTok.Tok);
- }
-
- if (FormatTok.Tok.is(tok::raw_identifier)) {
- const IdentifierInfo &Info = IdentTable.get(tokenText());
- FormatTok.Tok.setKind(Info.getTokenID());
- }
-
- if (FormatTok.Tok.is(tok::greatergreater)) {
- FormatTok.Tok.setKind(tok::greater);
- GreaterStashed = true;
- }
-}
-
-StringRef UnwrappedLineParser::tokenText() {
- StringRef Data(SourceMgr.getCharacterData(FormatTok.Tok.getLocation()),
- FormatTok.Tok.getLength());
- return Data;
+ FormatTok = Tokens.getNextToken();
}
} // end namespace format
Modified: cfe/trunk/lib/Format/UnwrappedLineParser.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Format/UnwrappedLineParser.h?rev=169605&r1=169604&r2=169605&view=diff
==============================================================================
--- cfe/trunk/lib/Format/UnwrappedLineParser.h (original)
+++ cfe/trunk/lib/Format/UnwrappedLineParser.h Fri Dec 7 10:15:44 2012
@@ -77,10 +77,14 @@
virtual void consumeUnwrappedLine(const UnwrappedLine &Line) = 0;
};
+class FormatTokenSource {
+public:
+ virtual FormatToken getNextToken() = 0;
+};
+
class UnwrappedLineParser {
public:
- UnwrappedLineParser(const FormatStyle &Style, Lexer &Lex,
- SourceManager &SourceMgr,
+ UnwrappedLineParser(const FormatStyle &Style, FormatTokenSource &Tokens,
UnwrappedLineConsumer &Callback);
/// Returns true in case of a structural error.
@@ -105,19 +109,12 @@
void addUnwrappedLine();
bool eof() const;
void nextToken();
- void parseToken();
-
- /// Returns the text of \c FormatTok.
- StringRef tokenText();
UnwrappedLine Line;
FormatToken FormatTok;
- bool GreaterStashed;
const FormatStyle &Style;
- Lexer &Lex;
- SourceManager &SourceMgr;
- IdentifierTable IdentTable;
+ FormatTokenSource &Tokens;
UnwrappedLineConsumer &Callback;
};
More information about the cfe-commits
mailing list