[cfe-commits] r159270 - in /cfe/trunk: include/clang/AST/CommentLexer.h lib/AST/CommentLexer.cpp unittests/AST/CommentLexer.cpp
Dmitri Gribenko
gribozavr at gmail.com
Wed Jun 27 09:53:58 PDT 2012
Author: gribozavr
Date: Wed Jun 27 11:53:58 2012
New Revision: 159270
URL: http://llvm.org/viewvc/llvm-project?rev=159270&view=rev
Log:
Remove unsigned and a pointer from a comment token (so that each token can have only one semantic string value attached to it), at a cost of adding an additional token.
Modified:
cfe/trunk/include/clang/AST/CommentLexer.h
cfe/trunk/lib/AST/CommentLexer.cpp
cfe/trunk/unittests/AST/CommentLexer.cpp
Modified: cfe/trunk/include/clang/AST/CommentLexer.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/AST/CommentLexer.h?rev=159270&r1=159269&r2=159270&view=diff
==============================================================================
--- cfe/trunk/include/clang/AST/CommentLexer.h (original)
+++ cfe/trunk/include/clang/AST/CommentLexer.h Wed Jun 27 11:53:58 2012
@@ -34,7 +34,8 @@
verbatim_block_begin,
verbatim_block_line,
verbatim_block_end,
- verbatim_line,
+ verbatim_line_name,
+ verbatim_line_text,
html_tag_open, // <tag
html_ident, // attr
html_equals, // =
@@ -75,10 +76,6 @@
const char *TextPtr1;
unsigned TextLen1;
- /// Contains text value associated with a token.
- const char *TextPtr2;
- unsigned TextLen2;
-
public:
SourceLocation getLocation() const LLVM_READONLY { return Loc; }
void setLocation(SourceLocation SL) { Loc = SL; }
@@ -138,25 +135,25 @@
/// Returns the name of verbatim line command.
StringRef getVerbatimLineName() const LLVM_READONLY {
- assert(is(tok::verbatim_line));
+ assert(is(tok::verbatim_line_name));
return StringRef(TextPtr1, TextLen1);
}
void setVerbatimLineName(StringRef Name) {
- assert(is(tok::verbatim_line));
+ assert(is(tok::verbatim_line_name));
TextPtr1 = Name.data();
TextLen1 = Name.size();
}
StringRef getVerbatimLineText() const LLVM_READONLY {
- assert(is(tok::verbatim_line));
- return StringRef(TextPtr2, TextLen2);
+ assert(is(tok::verbatim_line_text));
+ return StringRef(TextPtr1, TextLen1);
}
void setVerbatimLineText(StringRef Text) {
- assert(is(tok::verbatim_line));
- TextPtr2 = Text.data();
- TextLen2 = Text.size();
+ assert(is(tok::verbatim_line_text));
+ TextPtr1 = Text.data();
+ TextLen1 = Text.size();
}
StringRef getHTMLTagOpenName() const LLVM_READONLY {
@@ -245,6 +242,10 @@
/// decorations.
LS_VerbatimBlockBody,
+ /// Finished lexing verbatim line beginning command, will lex text (one
+ /// line).
+ LS_VerbatimLineText,
+
/// Finished lexing \verbatim <TAG \endverbatim part, lexing tag attributes.
LS_HTMLOpenTag
};
@@ -292,8 +293,6 @@
#ifndef NDEBUG
Result.TextPtr1 = "<UNSET>";
Result.TextLen1 = 7;
- Result.TextPtr2 = "<UNSET>";
- Result.TextLen2 = 7;
#endif
BufferPtr = TokEnd;
}
@@ -320,7 +319,9 @@
void lexVerbatimBlockBody(Token &T);
- void lexVerbatimLine(Token &T, const char *TextBegin);
+ void setupAndLexVerbatimLine(Token &T, const char *TextBegin);
+
+ void lexVerbatimLineText(Token &T);
void setupAndLexHTMLOpenTag(Token &T);
Modified: cfe/trunk/lib/AST/CommentLexer.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/AST/CommentLexer.cpp?rev=159270&r1=159269&r2=159270&view=diff
==============================================================================
--- cfe/trunk/lib/AST/CommentLexer.cpp (original)
+++ cfe/trunk/lib/AST/CommentLexer.cpp Wed Jun 27 11:53:58 2012
@@ -264,6 +264,9 @@
case LS_VerbatimBlockBody:
lexVerbatimBlockBody(T);
return;
+ case LS_VerbatimLineText:
+ lexVerbatimLineText(T);
+ return;
case LS_HTMLOpenTag:
lexHTMLOpenTag(T);
return;
@@ -333,7 +336,7 @@
return;
}
if (isVerbatimLineCommand(CommandName)) {
- lexVerbatimLine(T, TokenPtr);
+ setupAndLexVerbatimLine(T, TokenPtr);
return;
}
formTokenWithChars(T, TokenPtr, tok::command);
@@ -444,16 +447,24 @@
lexVerbatimBlockFirstLine(T);
}
-void Lexer::lexVerbatimLine(Token &T, const char *TextBegin) {
- // Extract current line.
- const char *Newline = findNewline(BufferPtr, CommentEnd);
-
+void Lexer::setupAndLexVerbatimLine(Token &T, const char *TextBegin) {
const StringRef Name(BufferPtr + 1, TextBegin - BufferPtr - 1);
- const StringRef Text(TextBegin, Newline - TextBegin);
-
- formTokenWithChars(T, Newline, tok::verbatim_line);
+ formTokenWithChars(T, TextBegin, tok::verbatim_line_name);
T.setVerbatimLineName(Name);
+
+ State = LS_VerbatimLineText;
+}
+
+void Lexer::lexVerbatimLineText(Token &T) {
+ assert(State == LS_VerbatimLineText);
+
+ // Extract current line.
+ const char *Newline = findNewline(BufferPtr, CommentEnd);
+ const StringRef Text(BufferPtr, Newline - BufferPtr);
+ formTokenWithChars(T, Newline, tok::verbatim_line_text);
T.setVerbatimLineText(Text);
+
+ State = LS_Normal;
}
void Lexer::setupAndLexHTMLOpenTag(Token &T) {
Modified: cfe/trunk/unittests/AST/CommentLexer.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/unittests/AST/CommentLexer.cpp?rev=159270&r1=159269&r2=159270&view=diff
==============================================================================
--- cfe/trunk/unittests/AST/CommentLexer.cpp (original)
+++ cfe/trunk/unittests/AST/CommentLexer.cpp Wed Jun 27 11:53:58 2012
@@ -588,15 +588,14 @@
ASSERT_EQ(4U, Toks.size());
- ASSERT_EQ(tok::text, Toks[0].getKind());
- ASSERT_EQ(StringRef(" "), Toks[0].getText());
+ ASSERT_EQ(tok::text, Toks[0].getKind());
+ ASSERT_EQ(StringRef(" "), Toks[0].getText());
- ASSERT_EQ(tok::verbatim_line, Toks[1].getKind());
- ASSERT_EQ(StringRef("fn"), Toks[1].getVerbatimLineName());
- ASSERT_EQ(StringRef(""), Toks[1].getVerbatimLineText());
+ ASSERT_EQ(tok::verbatim_line_name, Toks[1].getKind());
+ ASSERT_EQ(StringRef("fn"), Toks[1].getVerbatimLineName());
- ASSERT_EQ(tok::newline, Toks[2].getKind());
- ASSERT_EQ(tok::newline, Toks[3].getKind());
+ ASSERT_EQ(tok::newline, Toks[2].getKind());
+ ASSERT_EQ(tok::newline, Toks[3].getKind());
}
}
@@ -612,18 +611,20 @@
lexString(Sources[i], Toks);
- ASSERT_EQ(4U, Toks.size());
+ ASSERT_EQ(5U, Toks.size());
- ASSERT_EQ(tok::text, Toks[0].getKind());
- ASSERT_EQ(StringRef(" "), Toks[0].getText());
+ ASSERT_EQ(tok::text, Toks[0].getKind());
+ ASSERT_EQ(StringRef(" "), Toks[0].getText());
- ASSERT_EQ(tok::verbatim_line, Toks[1].getKind());
- ASSERT_EQ(StringRef("fn"), Toks[1].getVerbatimLineName());
+ ASSERT_EQ(tok::verbatim_line_name, Toks[1].getKind());
+ ASSERT_EQ(StringRef("fn"), Toks[1].getVerbatimLineName());
+
+ ASSERT_EQ(tok::verbatim_line_text, Toks[2].getKind());
ASSERT_EQ(StringRef(" void *foo(const char *zzz = \"\\$\");"),
- Toks[1].getVerbatimLineText());
+ Toks[2].getVerbatimLineText());
- ASSERT_EQ(tok::newline, Toks[2].getKind());
- ASSERT_EQ(tok::newline, Toks[3].getKind());
+ ASSERT_EQ(tok::newline, Toks[3].getKind());
+ ASSERT_EQ(tok::newline, Toks[4].getKind());
}
}
@@ -638,26 +639,28 @@
lexString(Source, Toks);
- ASSERT_EQ(8U, Toks.size());
+ ASSERT_EQ(9U, Toks.size());
- ASSERT_EQ(tok::text, Toks[0].getKind());
- ASSERT_EQ(StringRef(" "), Toks[0].getText());
+ ASSERT_EQ(tok::text, Toks[0].getKind());
+ ASSERT_EQ(StringRef(" "), Toks[0].getText());
+
+ ASSERT_EQ(tok::verbatim_line_name, Toks[1].getKind());
+ ASSERT_EQ(StringRef("fn"), Toks[1].getVerbatimLineName());
- ASSERT_EQ(tok::verbatim_line, Toks[1].getKind());
- ASSERT_EQ(StringRef("fn"), Toks[1].getVerbatimLineName());
+ ASSERT_EQ(tok::verbatim_line_text, Toks[2].getKind());
ASSERT_EQ(StringRef(" void *foo(const char *zzz = \"\\$\");"),
- Toks[1].getVerbatimLineText());
- ASSERT_EQ(tok::newline, Toks[2].getKind());
+ Toks[2].getVerbatimLineText());
+ ASSERT_EQ(tok::newline, Toks[3].getKind());
- ASSERT_EQ(tok::text, Toks[3].getKind());
- ASSERT_EQ(StringRef(" Meow"), Toks[3].getText());
- ASSERT_EQ(tok::newline, Toks[4].getKind());
+ ASSERT_EQ(tok::text, Toks[4].getKind());
+ ASSERT_EQ(StringRef(" Meow"), Toks[4].getText());
+ ASSERT_EQ(tok::newline, Toks[5].getKind());
- ASSERT_EQ(tok::text, Toks[5].getKind());
- ASSERT_EQ(StringRef(" "), Toks[5].getText());
+ ASSERT_EQ(tok::text, Toks[6].getKind());
+ ASSERT_EQ(StringRef(" "), Toks[6].getText());
- ASSERT_EQ(tok::newline, Toks[6].getKind());
- ASSERT_EQ(tok::newline, Toks[7].getKind());
+ ASSERT_EQ(tok::newline, Toks[7].getKind());
+ ASSERT_EQ(tok::newline, Toks[8].getKind());
}
TEST_F(CommentLexerTest, HTML1) {
More information about the cfe-commits
mailing list