[clang] c3bc61d - [clang-format][NFC] Pull FormatTokenSource into its own header.
Manuel Klimek via cfe-commits
cfe-commits at lists.llvm.org
Tue Jan 31 06:33:44 PST 2023
Author: Manuel Klimek
Date: 2023-01-31T14:32:31Z
New Revision: c3bc61d72f8da9a2b45e610ee3c2ccfc5f884f69
URL: https://github.com/llvm/llvm-project/commit/c3bc61d72f8da9a2b45e610ee3c2ccfc5f884f69
DIFF: https://github.com/llvm/llvm-project/commit/c3bc61d72f8da9a2b45e610ee3c2ccfc5f884f69.diff
LOG: [clang-format][NFC] Pull FormatTokenSource into its own header.
Prepare getting FormatTokenSource under unit testing.
Added:
clang/lib/Format/FormatTokenSource.h
Modified:
clang/lib/Format/FormatToken.h
clang/lib/Format/UnwrappedLineParser.cpp
Removed:
################################################################################
diff --git a/clang/lib/Format/FormatToken.h b/clang/lib/Format/FormatToken.h
index 9d055efd80075..f3349a4dc2dfa 100644
--- a/clang/lib/Format/FormatToken.h
+++ b/clang/lib/Format/FormatToken.h
@@ -1803,6 +1803,25 @@ struct AdditionalKeywords {
std::unordered_set<IdentifierInfo *> VerilogExtraKeywords;
};
+inline bool isLineComment(const FormatToken &FormatTok) {
+ return FormatTok.is(tok::comment) && !FormatTok.TokenText.startswith("/*");
+}
+
+// Checks if \p FormatTok is a line comment that continues the line comment
+// \p Previous. The original column of \p MinColumnToken is used to determine
+// whether \p FormatTok is indented enough to the right to continue \p Previous.
+inline bool continuesLineComment(const FormatToken &FormatTok,
+ const FormatToken *Previous,
+ const FormatToken *MinColumnToken) {
+ if (!Previous || !MinColumnToken)
+ return false;
+ unsigned MinContinueColumn =
+ MinColumnToken->OriginalColumn + (isLineComment(*MinColumnToken) ? 0 : 1);
+ return isLineComment(FormatTok) && FormatTok.NewlinesBefore == 1 &&
+ isLineComment(*Previous) &&
+ FormatTok.OriginalColumn >= MinContinueColumn;
+}
+
} // namespace format
} // namespace clang
diff --git a/clang/lib/Format/FormatTokenSource.h b/clang/lib/Format/FormatTokenSource.h
new file mode 100644
index 0000000000000..11c735d368647
--- /dev/null
+++ b/clang/lib/Format/FormatTokenSource.h
@@ -0,0 +1,132 @@
+
+//===--- FormatTokenSource.h - Format C++ code ------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file defines the \c TokenSource interface, which provides a token
+/// stream as well as the ability to manipulate the token stream.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_LIB_FORMAT_FORMATTOKENSOURCE_H
+#define LLVM_CLANG_LIB_FORMAT_FORMATTOKENSOURCE_H
+
+#include "FormatToken.h"
+#include "UnwrappedLineParser.h"
+
+#define DEBUG_TYPE "format-token-source"
+
+namespace clang {
+namespace format {
+
+class FormatTokenSource {
+public:
+ virtual ~FormatTokenSource() {}
+
+ // Returns the next token in the token stream.
+ virtual FormatToken *getNextToken() = 0;
+
+ // Returns the token preceding the token returned by the last call to
+ // getNextToken() in the token stream, or nullptr if no such token exists.
+ virtual FormatToken *getPreviousToken() = 0;
+
+ // Returns the token that would be returned by the next call to
+ // getNextToken().
+ virtual FormatToken *peekNextToken(bool SkipComment = false) = 0;
+
+ // Returns whether we are at the end of the file.
+ // This can be
diff erent from whether getNextToken() returned an eof token
+ // when the FormatTokenSource is a view on a part of the token stream.
+ virtual bool isEOF() = 0;
+
+ // Gets the current position in the token stream, to be used by setPosition().
+ virtual unsigned getPosition() = 0;
+
+ // Resets the token stream to the state it was in when getPosition() returned
+ // Position, and return the token at that position in the stream.
+ virtual FormatToken *setPosition(unsigned Position) = 0;
+};
+
+class ScopedMacroState : public FormatTokenSource {
+public:
+ ScopedMacroState(UnwrappedLine &Line, FormatTokenSource *&TokenSource,
+ FormatToken *&ResetToken)
+ : Line(Line), TokenSource(TokenSource), ResetToken(ResetToken),
+ PreviousLineLevel(Line.Level), PreviousTokenSource(TokenSource),
+ Token(nullptr), PreviousToken(nullptr) {
+ FakeEOF.Tok.startToken();
+ FakeEOF.Tok.setKind(tok::eof);
+ TokenSource = this;
+ Line.Level = 0;
+ Line.InPPDirective = true;
+ // InMacroBody gets set after the `#define x` part.
+ }
+
+ ~ScopedMacroState() override {
+ TokenSource = PreviousTokenSource;
+ ResetToken = Token;
+ Line.InPPDirective = false;
+ Line.InMacroBody = false;
+ Line.Level = PreviousLineLevel;
+ }
+
+ FormatToken *getNextToken() override {
+ // The \c UnwrappedLineParser guards against this by never calling
+ // \c getNextToken() after it has encountered the first eof token.
+ assert(!eof());
+ PreviousToken = Token;
+ Token = PreviousTokenSource->getNextToken();
+ if (eof())
+ return &FakeEOF;
+ return Token;
+ }
+
+ FormatToken *getPreviousToken() override {
+ return PreviousTokenSource->getPreviousToken();
+ }
+
+ FormatToken *peekNextToken(bool SkipComment) override {
+ if (eof())
+ return &FakeEOF;
+ return PreviousTokenSource->peekNextToken(SkipComment);
+ }
+
+ bool isEOF() override { return PreviousTokenSource->isEOF(); }
+
+ unsigned getPosition() override { return PreviousTokenSource->getPosition(); }
+
+ FormatToken *setPosition(unsigned Position) override {
+ PreviousToken = nullptr;
+ Token = PreviousTokenSource->setPosition(Position);
+ return Token;
+ }
+
+private:
+ bool eof() {
+ return Token && Token->HasUnescapedNewline &&
+ !continuesLineComment(*Token, PreviousToken,
+ /*MinColumnToken=*/PreviousToken);
+ }
+
+ FormatToken FakeEOF;
+ UnwrappedLine &Line;
+ FormatTokenSource *&TokenSource;
+ FormatToken *&ResetToken;
+ unsigned PreviousLineLevel;
+ FormatTokenSource *PreviousTokenSource;
+
+ FormatToken *Token;
+ FormatToken *PreviousToken;
+};
+
+} // namespace format
+} // namespace clang
+
+#undef DEBUG_TYPE
+
+#endif
\ No newline at end of file
diff --git a/clang/lib/Format/UnwrappedLineParser.cpp b/clang/lib/Format/UnwrappedLineParser.cpp
index 3e58b6f90559b..9d90e5390bb81 100644
--- a/clang/lib/Format/UnwrappedLineParser.cpp
+++ b/clang/lib/Format/UnwrappedLineParser.cpp
@@ -14,6 +14,7 @@
#include "UnwrappedLineParser.h"
#include "FormatToken.h"
+#include "FormatTokenSource.h"
#include "TokenAnnotator.h"
#include "clang/Basic/TokenKinds.h"
#include "llvm/ADT/STLExtras.h"
@@ -28,34 +29,6 @@
namespace clang {
namespace format {
-class FormatTokenSource {
-public:
- virtual ~FormatTokenSource() {}
-
- // Returns the next token in the token stream.
- virtual FormatToken *getNextToken() = 0;
-
- // Returns the token preceding the token returned by the last call to
- // getNextToken() in the token stream, or nullptr if no such token exists.
- virtual FormatToken *getPreviousToken() = 0;
-
- // Returns the token that would be returned by the next call to
- // getNextToken().
- virtual FormatToken *peekNextToken(bool SkipComment = false) = 0;
-
- // Returns whether we are at the end of the file.
- // This can be
diff erent from whether getNextToken() returned an eof token
- // when the FormatTokenSource is a view on a part of the token stream.
- virtual bool isEOF() = 0;
-
- // Gets the current position in the token stream, to be used by setPosition().
- virtual unsigned getPosition() = 0;
-
- // Resets the token stream to the state it was in when getPosition() returned
- // Position, and return the token at that position in the stream.
- virtual FormatToken *setPosition(unsigned Position) = 0;
-};
-
namespace {
void printLine(llvm::raw_ostream &OS, const UnwrappedLine &Line,
@@ -112,97 +85,6 @@ class ScopedDeclarationState {
llvm::BitVector &Stack;
};
-static bool isLineComment(const FormatToken &FormatTok) {
- return FormatTok.is(tok::comment) && !FormatTok.TokenText.startswith("/*");
-}
-
-// Checks if \p FormatTok is a line comment that continues the line comment
-// \p Previous. The original column of \p MinColumnToken is used to determine
-// whether \p FormatTok is indented enough to the right to continue \p Previous.
-static bool continuesLineComment(const FormatToken &FormatTok,
- const FormatToken *Previous,
- const FormatToken *MinColumnToken) {
- if (!Previous || !MinColumnToken)
- return false;
- unsigned MinContinueColumn =
- MinColumnToken->OriginalColumn + (isLineComment(*MinColumnToken) ? 0 : 1);
- return isLineComment(FormatTok) && FormatTok.NewlinesBefore == 1 &&
- isLineComment(*Previous) &&
- FormatTok.OriginalColumn >= MinContinueColumn;
-}
-
-class ScopedMacroState : public FormatTokenSource {
-public:
- ScopedMacroState(UnwrappedLine &Line, FormatTokenSource *&TokenSource,
- FormatToken *&ResetToken)
- : Line(Line), TokenSource(TokenSource), ResetToken(ResetToken),
- PreviousLineLevel(Line.Level), PreviousTokenSource(TokenSource),
- Token(nullptr), PreviousToken(nullptr) {
- FakeEOF.Tok.startToken();
- FakeEOF.Tok.setKind(tok::eof);
- TokenSource = this;
- Line.Level = 0;
- Line.InPPDirective = true;
- // InMacroBody gets set after the `#define x` part.
- }
-
- ~ScopedMacroState() override {
- TokenSource = PreviousTokenSource;
- ResetToken = Token;
- Line.InPPDirective = false;
- Line.InMacroBody = false;
- Line.Level = PreviousLineLevel;
- }
-
- FormatToken *getNextToken() override {
- // The \c UnwrappedLineParser guards against this by never calling
- // \c getNextToken() after it has encountered the first eof token.
- assert(!eof());
- PreviousToken = Token;
- Token = PreviousTokenSource->getNextToken();
- if (eof())
- return &FakeEOF;
- return Token;
- }
-
- FormatToken *getPreviousToken() override {
- return PreviousTokenSource->getPreviousToken();
- }
-
- FormatToken *peekNextToken(bool SkipComment) override {
- if (eof())
- return &FakeEOF;
- return PreviousTokenSource->peekNextToken(SkipComment);
- }
-
- bool isEOF() override { return PreviousTokenSource->isEOF(); }
-
- unsigned getPosition() override { return PreviousTokenSource->getPosition(); }
-
- FormatToken *setPosition(unsigned Position) override {
- PreviousToken = nullptr;
- Token = PreviousTokenSource->setPosition(Position);
- return Token;
- }
-
-private:
- bool eof() {
- return Token && Token->HasUnescapedNewline &&
- !continuesLineComment(*Token, PreviousToken,
- /*MinColumnToken=*/PreviousToken);
- }
-
- FormatToken FakeEOF;
- UnwrappedLine &Line;
- FormatTokenSource *&TokenSource;
- FormatToken *&ResetToken;
- unsigned PreviousLineLevel;
- FormatTokenSource *PreviousTokenSource;
-
- FormatToken *Token;
- FormatToken *PreviousToken;
-};
-
} // end anonymous namespace
class ScopedLineState {
More information about the cfe-commits
mailing list