[clang] 04ed86f - [clang-format][NFC] Bring FormatTokenSource under test.

[Manuel Klimek via cfe-commits]

Author: Manuel Klimek
Date: 2023-01-31T16:06:46Z
New Revision: 04ed86ff1b7272faf8e62fa32da4acaa2d3c6add

URL: https://github.com/llvm/llvm-project/commit/04ed86ff1b7272faf8e62fa32da4acaa2d3c6add
DIFF: https://github.com/llvm/llvm-project/commit/04ed86ff1b7272faf8e62fa32da4acaa2d3c6add.diff

LOG: [clang-format][NFC] Bring FormatTokenSource under test.

Add tests for FormatTokenSource and harden it against edge cases.

Added: 
    clang/unittests/Format/FormatTokenSourceTest.cpp

Modified: 
    clang/lib/Format/FormatTokenSource.h
    clang/unittests/Format/CMakeLists.txt

Removed: 
    


################################################################################
diff  --git a/clang/lib/Format/FormatTokenSource.h b/clang/lib/Format/FormatTokenSource.h
index e14f72b7d1f44..359e0e52878bb 100644
--- a/clang/lib/Format/FormatTokenSource.h
+++ b/clang/lib/Format/FormatTokenSource.h
@@ -77,7 +77,9 @@ class IndexedTokenSource : public FormatTokenSource {
     return Position > 0 ? Tokens[Position - 1] : nullptr;
   }
 
-  FormatToken *peekNextToken(bool SkipComment) override {
+  FormatToken *peekNextToken(bool SkipComment = false) override {
+    if (isEOF())
+      return Tokens[Position];
     int Next = Position + 1;
     if (SkipComment)
       while (Tokens[Next]->is(tok::comment))
@@ -89,7 +91,9 @@ class IndexedTokenSource : public FormatTokenSource {
     return Tokens[Next];
   }
 
-  bool isEOF() override { return Tokens[Position]->is(tok::eof); }
+  bool isEOF() override {
+    return Position == -1 ? false : Tokens[Position]->is(tok::eof);
+  }
 
   unsigned getPosition() override {
     LLVM_DEBUG(llvm::dbgs() << "Getting Position: " << Position << "\n");

diff  --git a/clang/unittests/Format/CMakeLists.txt b/clang/unittests/Format/CMakeLists.txt
index 09457c105ca30..1db1f602415bb 100644
--- a/clang/unittests/Format/CMakeLists.txt
+++ b/clang/unittests/Format/CMakeLists.txt
@@ -21,6 +21,7 @@ add_clang_unittest(FormatTests
   FormatTestTableGen.cpp
   FormatTestTextProto.cpp
   FormatTestVerilog.cpp
+  FormatTokenSourceTest.cpp
   IntegerLiteralSeparatorTest.cpp
   MacroCallReconstructorTest.cpp
   MacroExpanderTest.cpp

diff  --git a/clang/unittests/Format/FormatTokenSourceTest.cpp b/clang/unittests/Format/FormatTokenSourceTest.cpp
new file mode 100644
index 0000000000000..3c2f317adf5c3
--- /dev/null
+++ b/clang/unittests/Format/FormatTokenSourceTest.cpp
@@ -0,0 +1,75 @@
+//===- unittest/Format/FormatTokenSourceTest.cpp --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "../../lib/Format/FormatTokenSource.h"
+#include "TestLexer.h"
+#include "clang/Basic/TokenKinds.h"
+#include "gtest/gtest.h"
+
+namespace clang {
+namespace format {
+namespace {
+
+class IndexedTokenSourceTest : public ::testing::Test {
+protected:
+  TokenList lex(llvm::StringRef Code,
+                const FormatStyle &Style = getLLVMStyle()) {
+    return TestLexer(Allocator, Buffers, Style).lex(Code);
+  }
+  llvm::SpecificBumpPtrAllocator<FormatToken> Allocator;
+  std::vector<std::unique_ptr<llvm::MemoryBuffer>> Buffers;
+};
+
+#define EXPECT_TOKEN_KIND(FormatTok, Kind)                                     \
+  do {                                                                         \
+    FormatToken *Tok = FormatTok;                                              \
+    EXPECT_EQ((Tok)->Tok.getKind(), Kind) << *(Tok);                           \
+  } while (false);
+
+TEST_F(IndexedTokenSourceTest, EmptyInput) {
+  IndexedTokenSource Source(lex(""));
+  EXPECT_FALSE(Source.isEOF());
+  EXPECT_TOKEN_KIND(Source.getNextToken(), tok::eof);
+  EXPECT_TRUE(Source.isEOF());
+  EXPECT_TOKEN_KIND(Source.getNextToken(), tok::eof);
+  EXPECT_TRUE(Source.isEOF());
+  EXPECT_TOKEN_KIND(Source.peekNextToken(/*SkipComment=*/false), tok::eof);
+  EXPECT_TOKEN_KIND(Source.peekNextToken(/*SkipComment=*/true), tok::eof);
+  EXPECT_EQ(Source.getPreviousToken(), nullptr);
+  EXPECT_TRUE(Source.isEOF());
+}
+
+TEST_F(IndexedTokenSourceTest, NavigateTokenStream) {
+  IndexedTokenSource Source(lex("int a;"));
+  EXPECT_TOKEN_KIND(Source.peekNextToken(), tok::kw_int);
+  EXPECT_TOKEN_KIND(Source.getNextToken(), tok::kw_int);
+  EXPECT_EQ(Source.getPreviousToken(), nullptr);
+  EXPECT_TOKEN_KIND(Source.peekNextToken(), tok::identifier);
+  EXPECT_TOKEN_KIND(Source.getNextToken(), tok::identifier);
+  EXPECT_TOKEN_KIND(Source.getPreviousToken(), tok::kw_int);
+  EXPECT_TOKEN_KIND(Source.peekNextToken(), tok::semi);
+  EXPECT_TOKEN_KIND(Source.getNextToken(), tok::semi);
+  EXPECT_TOKEN_KIND(Source.getPreviousToken(), tok::identifier);
+  EXPECT_TOKEN_KIND(Source.peekNextToken(), tok::eof);
+  EXPECT_TOKEN_KIND(Source.getNextToken(), tok::eof);
+  EXPECT_TOKEN_KIND(Source.getPreviousToken(), tok::semi);
+}
+
+TEST_F(IndexedTokenSourceTest, ResetPosition) {
+  IndexedTokenSource Source(lex("int a;"));
+  Source.getNextToken();
+  unsigned Position = Source.getPosition();
+  Source.getNextToken();
+  Source.getNextToken();
+  EXPECT_TOKEN_KIND(Source.getNextToken(), tok::eof);
+  EXPECT_TOKEN_KIND(Source.setPosition(Position), tok::kw_int);
+}
+
+} // namespace
+} // namespace format
+} // namespace clang
\ No newline at end of file