[cfe-commits] r160682 - in /cfe/trunk: lib/AST/CommentParser.cpp unittests/AST/CommentParser.cpp

Dmitri Gribenko gribozavr at gmail.com
Tue Jul 24 11:23:32 PDT 2012


Author: gribozavr
Date: Tue Jul 24 13:23:31 2012
New Revision: 160682

URL: http://llvm.org/viewvc/llvm-project?rev=160682&view=rev
Log:
Comment parsing: allow newlines between \param, direction specification (e.g.,
[in]), parameter name and description paragraph.

Modified:
    cfe/trunk/lib/AST/CommentParser.cpp
    cfe/trunk/unittests/AST/CommentParser.cpp

Modified: cfe/trunk/lib/AST/CommentParser.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/AST/CommentParser.cpp?rev=160682&r1=160681&r2=160682&view=diff
==============================================================================
--- cfe/trunk/lib/AST/CommentParser.cpp (original)
+++ cfe/trunk/lib/AST/CommentParser.cpp Tue Jul 24 13:23:31 2012
@@ -20,8 +20,14 @@
 class TextTokenRetokenizer {
   llvm::BumpPtrAllocator &Allocator;
   Parser &P;
+
+  /// This flag is set when there are no more tokens we can fetch from lexer.
+  bool NoMoreInterestingTokens;
+
+  /// Token buffer: tokens we have processed and lookahead.
   SmallVector<Token, 16> Toks;
 
+  /// A position in \c Toks.
   struct Position {
     unsigned CurToken;
     const char *BufferStart;
@@ -65,10 +71,11 @@
     Pos.BufferPtr++;
     if (Pos.BufferPtr == Pos.BufferEnd) {
       Pos.CurToken++;
-      if (isEnd() && addToken()) {
-        assert(!isEnd());
-        setupBuffer();
-      }
+      if (isEnd() && !addToken())
+        return;
+
+      assert(!isEnd());
+      setupBuffer();
     }
   }
 
@@ -76,9 +83,24 @@
   /// Returns true on success, false if there are no interesting tokens to
   /// fetch from lexer.
   bool addToken() {
-    if (P.Tok.isNot(tok::text))
+    if (NoMoreInterestingTokens)
       return false;
 
+    if (P.Tok.is(tok::newline)) {
+      // If we see a single newline token between text tokens, skip it.
+      Token Newline = P.Tok;
+      P.consumeToken();
+      if (P.Tok.isNot(tok::text)) {
+        P.putBack(Newline);
+        NoMoreInterestingTokens = true;
+        return false;
+      }
+    }
+    if (P.Tok.isNot(tok::text)) {
+      NoMoreInterestingTokens = true;
+      return false;
+    }
+
     Toks.push_back(P.Tok);
     P.consumeToken();
     if (Toks.size() == 1)
@@ -117,7 +139,7 @@
 
 public:
   TextTokenRetokenizer(llvm::BumpPtrAllocator &Allocator, Parser &P):
-      Allocator(Allocator), P(P) {
+      Allocator(Allocator), P(P), NoMoreInterestingTokens(false) {
     Pos.CurToken = 0;
     addToken();
   }

Modified: cfe/trunk/unittests/AST/CommentParser.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/unittests/AST/CommentParser.cpp?rev=160682&r1=160681&r2=160682&view=diff
==============================================================================
--- cfe/trunk/unittests/AST/CommentParser.cpp (original)
+++ cfe/trunk/unittests/AST/CommentParser.cpp Tue Jul 24 13:23:31 2012
@@ -205,10 +205,15 @@
         << " direction, "
            "expected " << (IsDirectionExplicit ? "explicit" : "implicit");
 
+  if (!PCC->hasParamName())
+    return ::testing::AssertionFailure()
+        << "ParamCommandComment has no parameter name";
+
   StringRef ActualParamName = PCC->getParamName();
   if (ActualParamName != ParamName)
     return ::testing::AssertionFailure()
-        << "ParamCommandComment has name \"" << ActualParamName.str() << "\", "
+        << "ParamCommandComment has parameter name \"" << ActualParamName.str()
+        << "\", "
            "expected \"" << ParamName.str() << "\"";
 
   Paragraph = PCC->getParagraph();
@@ -672,69 +677,102 @@
 }
 
 TEST_F(CommentParserTest, ParamCommand1) {
-  const char *Source =
-    "// \\param aaa Bbb\n";
+  const char *Sources[] = {
+    "// \\param aaa Bbb\n",
+    "// \\param\n"
+    "//     aaa Bbb\n",
+    "// \\param \n"
+    "//     aaa Bbb\n",
+    "// \\param aaa\n"
+    "// Bbb\n"
+  };
 
-  FullComment *FC = parseString(Source);
-  ASSERT_TRUE(HasChildCount(FC, 2));
+  for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
+    FullComment *FC = parseString(Sources[i]);
+    ASSERT_TRUE(HasChildCount(FC, 2));
 
-  ASSERT_TRUE(HasParagraphCommentAt(FC, 0, " "));
-  {
-    ParamCommandComment *PCC;
-    ParagraphComment *PC;
-    ASSERT_TRUE(HasParamCommandAt(FC, 1, PCC, "param",
-                                  ParamCommandComment::In,
-                                  /* IsDirectionExplicit = */ false,
-                                  "aaa", PC));
-    ASSERT_TRUE(HasChildCount(PCC, 1));
-    ASSERT_TRUE(HasParagraphCommentAt(PCC, 0, " Bbb"));
+    ASSERT_TRUE(HasParagraphCommentAt(FC, 0, " "));
+    {
+      ParamCommandComment *PCC;
+      ParagraphComment *PC;
+      ASSERT_TRUE(HasParamCommandAt(FC, 1, PCC, "param",
+                                    ParamCommandComment::In,
+                                    /* IsDirectionExplicit = */ false,
+                                    "aaa", PC));
+      ASSERT_TRUE(HasChildCount(PCC, 1));
+      ASSERT_TRUE(HasParagraphCommentAt(PCC, 0, " Bbb"));
+    }
   }
 }
 
 TEST_F(CommentParserTest, ParamCommand2) {
-  const char *Source =
-    "// \\param [in] aaa Bbb\n";
+  const char *Sources[] = {
+    "// \\param [in] aaa Bbb\n",
+    "// \\param\n"
+    "//     [in] aaa Bbb\n",
+    "// \\param [in]\n"
+    "//     aaa Bbb\n",
+    "// \\param [in] aaa\n"
+    "// Bbb\n",
+  };
 
-  FullComment *FC = parseString(Source);
-  ASSERT_TRUE(HasChildCount(FC, 2));
+  for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
+    FullComment *FC = parseString(Sources[i]);
+    ASSERT_TRUE(HasChildCount(FC, 2));
 
-  ASSERT_TRUE(HasParagraphCommentAt(FC, 0, " "));
-  {
-    ParamCommandComment *PCC;
-    ParagraphComment *PC;
-    ASSERT_TRUE(HasParamCommandAt(FC, 1, PCC, "param",
-                                  ParamCommandComment::In,
-                                  /* IsDirectionExplicit = */ true,
-                                  "aaa", PC));
-    ASSERT_TRUE(HasChildCount(PCC, 1));
-    ASSERT_TRUE(HasParagraphCommentAt(PCC, 0, " Bbb"));
+    ASSERT_TRUE(HasParagraphCommentAt(FC, 0, " "));
+    {
+      ParamCommandComment *PCC;
+      ParagraphComment *PC;
+      ASSERT_TRUE(HasParamCommandAt(FC, 1, PCC, "param",
+                                    ParamCommandComment::In,
+                                    /* IsDirectionExplicit = */ true,
+                                    "aaa", PC));
+      ASSERT_TRUE(HasChildCount(PCC, 1));
+      ASSERT_TRUE(HasParagraphCommentAt(PCC, 0, " Bbb"));
+    }
   }
 }
 
 TEST_F(CommentParserTest, ParamCommand3) {
-  const char *Source =
-    "// \\param [out] aaa Bbb\n";
+  const char *Sources[] = {
+    "// \\param [out] aaa Bbb\n",
+    "// \\param\n"
+    "//     [out] aaa Bbb\n",
+    "// \\param [out]\n"
+    "//     aaa Bbb\n",
+    "// \\param [out] aaa\n"
+    "// Bbb\n",
+  };
 
-  FullComment *FC = parseString(Source);
-  ASSERT_TRUE(HasChildCount(FC, 2));
+  for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
+    FullComment *FC = parseString(Sources[i]);
+    ASSERT_TRUE(HasChildCount(FC, 2));
 
-  ASSERT_TRUE(HasParagraphCommentAt(FC, 0, " "));
-  {
-    ParamCommandComment *PCC;
-    ParagraphComment *PC;
-    ASSERT_TRUE(HasParamCommandAt(FC, 1, PCC, "param",
-                                  ParamCommandComment::Out,
-                                  /* IsDirectionExplicit = */ true,
-                                  "aaa", PC));
-    ASSERT_TRUE(HasChildCount(PCC, 1));
-    ASSERT_TRUE(HasParagraphCommentAt(PCC, 0, " Bbb"));
+    ASSERT_TRUE(HasParagraphCommentAt(FC, 0, " "));
+    {
+      ParamCommandComment *PCC;
+      ParagraphComment *PC;
+      ASSERT_TRUE(HasParamCommandAt(FC, 1, PCC, "param",
+                                    ParamCommandComment::Out,
+                                    /* IsDirectionExplicit = */ true,
+                                    "aaa", PC));
+      ASSERT_TRUE(HasChildCount(PCC, 1));
+      ASSERT_TRUE(HasParagraphCommentAt(PCC, 0, " Bbb"));
+    }
   }
 }
 
 TEST_F(CommentParserTest, ParamCommand4) {
   const char *Sources[] = {
     "// \\param [in,out] aaa Bbb\n",
-    "// \\param [in, out] aaa Bbb\n"
+    "// \\param [in, out] aaa Bbb\n",
+    "// \\param [in,\n"
+    "//     out] aaa Bbb\n",
+    "// \\param [in,out]\n"
+    "//     aaa Bbb\n",
+    "// \\param [in,out] aaa\n"
+    "// Bbb\n"
   };
 
   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {





More information about the cfe-commits mailing list