r217444 - clang-format: [JS] Support regex literals with trailing escaped slash.
Daniel Jasper
djasper at google.com
Tue Sep 9 07:37:39 PDT 2014
Author: djasper
Date: Tue Sep 9 09:37:39 2014
New Revision: 217444
URL: http://llvm.org/viewvc/llvm-project?rev=217444&view=rev
Log:
clang-format: [JS] Support regex literals with trailing escaped slash.
Before:
var regex = / a\//; int i;
After:
var regex = /a\//;
int i;
This required pushing the Lexer into its wrapper class and generating a
new one in this specific case. Otherwise, the sequence get lexed as a
//-comment. This is hacky, but I don't know a better way (short of
supporting regex literals in the Lexer).
Pushing the Lexer down seems to make all the call sites simpler.
Modified:
cfe/trunk/include/clang/Format/Format.h
cfe/trunk/lib/Format/Format.cpp
cfe/trunk/lib/Index/CommentToXML.cpp
cfe/trunk/tools/clang-format/ClangFormat.cpp
cfe/trunk/unittests/Format/FormatTestJS.cpp
Modified: cfe/trunk/include/clang/Format/Format.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Format/Format.h?rev=217444&r1=217443&r2=217444&view=diff
==============================================================================
--- cfe/trunk/include/clang/Format/Format.h (original)
+++ cfe/trunk/include/clang/Format/Format.h Tue Sep 9 09:37:39 2014
@@ -487,14 +487,21 @@ std::string configurationAsText(const Fo
/// \brief Reformats the given \p Ranges in the token stream coming out of
/// \c Lex.
///
+/// DEPRECATED: Do not use.
+tooling::Replacements reformat(const FormatStyle &Style, Lexer &Lex,
+ SourceManager &SourceMgr,
+ std::vector<CharSourceRange> Ranges);
+
+/// \brief Reformats the given \p Ranges in the file \p ID.
+///
/// Each range is extended on either end to its next bigger logic unit, i.e.
/// everything that might influence its formatting or might be influenced by its
/// formatting.
///
/// Returns the \c Replacements necessary to make all \p Ranges comply with
/// \p Style.
-tooling::Replacements reformat(const FormatStyle &Style, Lexer &Lex,
- SourceManager &SourceMgr,
+tooling::Replacements reformat(const FormatStyle &Style,
+ SourceManager &SourceMgr, FileID ID,
std::vector<CharSourceRange> Ranges);
/// \brief Reformats the given \p Ranges in \p Code.
Modified: cfe/trunk/lib/Format/Format.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Format/Format.cpp?rev=217444&r1=217443&r2=217444&view=diff
==============================================================================
--- cfe/trunk/lib/Format/Format.cpp (original)
+++ cfe/trunk/lib/Format/Format.cpp Tue Sep 9 09:37:39 2014
@@ -1273,13 +1273,16 @@ private:
class FormatTokenLexer {
public:
- FormatTokenLexer(Lexer &Lex, SourceManager &SourceMgr, FormatStyle &Style,
+ FormatTokenLexer(SourceManager &SourceMgr, FileID ID, FormatStyle &Style,
encoding::Encoding Encoding)
: FormatTok(nullptr), IsFirstToken(true), GreaterStashed(false),
- Column(0), TrailingWhitespace(0), Lex(Lex), SourceMgr(SourceMgr),
- Style(Style), IdentTable(getFormattingLangOpts(Style)),
- Encoding(Encoding), FirstInLineIndex(0), FormattingDisabled(false) {
- Lex.SetKeepWhitespaceMode(true);
+ Column(0), TrailingWhitespace(0),
+ SourceMgr(SourceMgr), ID(ID), Style(Style),
+ IdentTable(getFormattingLangOpts(Style)), Encoding(Encoding),
+ FirstInLineIndex(0), FormattingDisabled(false) {
+ Lex.reset(new Lexer(ID, SourceMgr.getBuffer(ID), SourceMgr,
+ getFormattingLangOpts(Style)));
+ Lex->SetKeepWhitespaceMode(true);
for (const std::string &ForEachMacro : Style.ForEachMacros)
ForEachMacros.push_back(&IdentTable.get(ForEachMacro));
@@ -1308,10 +1311,10 @@ private:
return;
if (Style.Language == FormatStyle::LK_JavaScript) {
- if (tryMergeEscapeSequence())
- return;
if (tryMergeJSRegexLiteral())
return;
+ if (tryMergeEscapeSequence())
+ return;
static tok::TokenKind JSIdentity[] = { tok::equalequal, tok::equal };
static tok::TokenKind JSNotIdentity[] = { tok::exclaimequal, tok::equal };
@@ -1376,9 +1379,18 @@ private:
// "(;,{}![:?", a binary operator or 'return', as those cannot be followed by
// a division.
bool tryMergeJSRegexLiteral() {
- if (Tokens.size() < 2 || Tokens.back()->isNot(tok::slash) ||
- (Tokens[Tokens.size() - 2]->is(tok::unknown) &&
- Tokens[Tokens.size() - 2]->TokenText == "\\"))
+ if (Tokens.size() < 2)
+ return false;
+ // If a regex literal ends in "\//", this gets represented by an unknown
+ // token "\" and a comment.
+ bool MightEndWithEscapedSlash =
+ Tokens.back()->is(tok::comment) &&
+ Tokens.back()->TokenText.startswith("//") &&
+ Tokens[Tokens.size() - 2]->TokenText == "\\";
+ if (!MightEndWithEscapedSlash &&
+ (Tokens.back()->isNot(tok::slash) ||
+ (Tokens[Tokens.size() - 2]->is(tok::unknown) &&
+ Tokens[Tokens.size() - 2]->TokenText == "\\")))
return false;
unsigned TokenCount = 0;
unsigned LastColumn = Tokens.back()->OriginalColumn;
@@ -1389,6 +1401,17 @@ private:
tok::exclaim, tok::l_square, tok::colon, tok::comma,
tok::question, tok::kw_return) ||
I[1]->isBinaryOperator())) {
+ if (MightEndWithEscapedSlash) {
+ StringRef Buffer = SourceMgr.getBufferData(ID);
+ // This regex literal ends in '\//'. Skip past the '//' of the last
+ // token and re-start lexing from there.
+ int offset =
+ SourceMgr.getFileOffset(Tokens.back()->Tok.getLocation()) + 2;
+ Lex.reset(new Lexer(SourceMgr.getLocForStartOfFile(ID),
+ getFormattingLangOpts(Style), Buffer.begin(),
+ Buffer.begin() + offset, Buffer.end()));
+ Lex->SetKeepWhitespaceMode(true);
+ }
Tokens.resize(Tokens.size() - TokenCount);
Tokens.back()->Tok.setKind(tok::unknown);
Tokens.back()->Type = TT_RegexLiteral;
@@ -1641,8 +1664,9 @@ private:
bool GreaterStashed;
unsigned Column;
unsigned TrailingWhitespace;
- Lexer &Lex;
+ std::unique_ptr<Lexer> Lex;
SourceManager &SourceMgr;
+ FileID ID;
FormatStyle &Style;
IdentifierTable IdentTable;
encoding::Encoding Encoding;
@@ -1655,7 +1679,7 @@ private:
bool FormattingDisabled;
void readRawToken(FormatToken &Tok) {
- Lex.LexFromRawLexer(Tok.Tok);
+ Lex->LexFromRawLexer(Tok.Tok);
Tok.TokenText = StringRef(SourceMgr.getCharacterData(Tok.Tok.getLocation()),
Tok.Tok.getLength());
// For formatting, treat unterminated string literals like normal string
@@ -1692,12 +1716,13 @@ static StringRef getLanguageName(FormatS
class Formatter : public UnwrappedLineConsumer {
public:
- Formatter(const FormatStyle &Style, Lexer &Lex, SourceManager &SourceMgr,
+ Formatter(const FormatStyle &Style, SourceManager &SourceMgr, FileID ID,
const std::vector<CharSourceRange> &Ranges)
- : Style(Style), Lex(Lex), SourceMgr(SourceMgr),
- Whitespaces(SourceMgr, Style, inputUsesCRLF(Lex.getBuffer())),
+ : Style(Style), ID(ID), SourceMgr(SourceMgr),
+ Whitespaces(SourceMgr, Style,
+ inputUsesCRLF(SourceMgr.getBufferData(ID))),
Ranges(Ranges.begin(), Ranges.end()), UnwrappedLines(1),
- Encoding(encoding::detectEncoding(Lex.getBuffer())) {
+ Encoding(encoding::detectEncoding(SourceMgr.getBufferData(ID))) {
DEBUG(llvm::dbgs() << "File encoding: "
<< (Encoding == encoding::Encoding_UTF8 ? "UTF8"
: "unknown")
@@ -1708,7 +1733,7 @@ public:
tooling::Replacements format() {
tooling::Replacements Result;
- FormatTokenLexer Tokens(Lex, SourceMgr, Style, Encoding);
+ FormatTokenLexer Tokens(SourceMgr, ID, Style, Encoding);
UnwrappedLineParser Parser(Style, Tokens.lex(), *this);
bool StructuralError = Parser.parse();
@@ -1962,7 +1987,7 @@ private:
}
FormatStyle Style;
- Lexer &Lex;
+ FileID ID;
SourceManager &SourceMgr;
WhitespaceManager Whitespaces;
SmallVector<CharSourceRange, 8> Ranges;
@@ -1977,18 +2002,27 @@ private:
tooling::Replacements reformat(const FormatStyle &Style, Lexer &Lex,
SourceManager &SourceMgr,
std::vector<CharSourceRange> Ranges) {
- if (Style.DisableFormat) {
- tooling::Replacements EmptyResult;
- return EmptyResult;
- }
+ if (Style.DisableFormat)
+ return tooling::Replacements();
+ return reformat(Style, SourceMgr,
+ SourceMgr.getFileID(Lex.getSourceLocation()), Ranges);
+}
- Formatter formatter(Style, Lex, SourceMgr, Ranges);
+tooling::Replacements reformat(const FormatStyle &Style,
+ SourceManager &SourceMgr, FileID ID,
+ std::vector<CharSourceRange> Ranges) {
+ if (Style.DisableFormat)
+ return tooling::Replacements();
+ Formatter formatter(Style, SourceMgr, ID, Ranges);
return formatter.format();
}
tooling::Replacements reformat(const FormatStyle &Style, StringRef Code,
std::vector<tooling::Range> Ranges,
StringRef FileName) {
+ if (Style.DisableFormat)
+ return tooling::Replacements();
+
FileManager Files((FileSystemOptions()));
DiagnosticsEngine Diagnostics(
IntrusiveRefCntPtr<DiagnosticIDs>(new DiagnosticIDs),
@@ -2001,8 +2035,6 @@ tooling::Replacements reformat(const For
SourceMgr.overrideFileContents(Entry, std::move(Buf));
FileID ID =
SourceMgr.createFileID(Entry, SourceLocation(), clang::SrcMgr::C_User);
- Lexer Lex(ID, SourceMgr.getBuffer(ID), SourceMgr,
- getFormattingLangOpts(Style));
SourceLocation StartOfFile = SourceMgr.getLocForStartOfFile(ID);
std::vector<CharSourceRange> CharRanges;
for (unsigned i = 0, e = Ranges.size(); i != e; ++i) {
@@ -2010,7 +2042,7 @@ tooling::Replacements reformat(const For
SourceLocation End = Start.getLocWithOffset(Ranges[i].getLength());
CharRanges.push_back(CharSourceRange::getCharRange(Start, End));
}
- return reformat(Style, Lex, SourceMgr, CharRanges);
+ return reformat(Style, SourceMgr, ID, CharRanges);
}
LangOptions getFormattingLangOpts(const FormatStyle &Style) {
Modified: cfe/trunk/lib/Index/CommentToXML.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Index/CommentToXML.cpp?rev=217444&r1=217443&r2=217444&view=diff
==============================================================================
--- cfe/trunk/lib/Index/CommentToXML.cpp (original)
+++ cfe/trunk/lib/Index/CommentToXML.cpp Tue Sep 9 09:37:39 2014
@@ -15,7 +15,6 @@
#include "clang/AST/CommentVisitor.h"
#include "clang/Format/Format.h"
#include "clang/Index/USRGeneration.h"
-#include "clang/Lex/Lexer.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/TinyPtrVector.h"
#include "llvm/Support/raw_ostream.h"
@@ -611,12 +610,8 @@ void CommentASTToXMLConverter::formatTex
std::vector<CharSourceRange> Ranges(
1, CharSourceRange::getCharRange(Start, Start.getLocWithOffset(Length)));
- ASTContext &Context = DI->CurrentDecl->getASTContext();
- const LangOptions &LangOpts = Context.getLangOpts();
- Lexer Lex(ID, FormatRewriterContext.Sources.getBuffer(ID),
- FormatRewriterContext.Sources, LangOpts);
tooling::Replacements Replace = reformat(
- format::getLLVMStyle(), Lex, FormatRewriterContext.Sources, Ranges);
+ format::getLLVMStyle(), FormatRewriterContext.Sources, ID, Ranges);
applyAllReplacements(Replace, FormatRewriterContext.Rewrite);
Declaration = FormatRewriterContext.getRewrittenText(ID);
}
Modified: cfe/trunk/tools/clang-format/ClangFormat.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/tools/clang-format/ClangFormat.cpp?rev=217444&r1=217443&r2=217444&view=diff
==============================================================================
--- cfe/trunk/tools/clang-format/ClangFormat.cpp (original)
+++ cfe/trunk/tools/clang-format/ClangFormat.cpp Tue Sep 9 09:37:39 2014
@@ -19,7 +19,6 @@
#include "clang/Basic/SourceManager.h"
#include "clang/Basic/Version.h"
#include "clang/Format/Format.h"
-#include "clang/Lex/Lexer.h"
#include "clang/Rewrite/Core/Rewriter.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/Support/Debug.h"
@@ -225,9 +224,7 @@ static bool format(StringRef FileName) {
FormatStyle FormatStyle = getStyle(
Style, (FileName == "-") ? AssumeFilename : FileName, FallbackStyle);
- Lexer Lex(ID, Sources.getBuffer(ID), Sources,
- getFormattingLangOpts(FormatStyle));
- tooling::Replacements Replaces = reformat(FormatStyle, Lex, Sources, Ranges);
+ tooling::Replacements Replaces = reformat(FormatStyle, Sources, ID, Ranges);
if (OutputXML) {
llvm::outs()
<< "<?xml version='1.0'?>\n<replacements xml:space='preserve'>\n";
Modified: cfe/trunk/unittests/Format/FormatTestJS.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/unittests/Format/FormatTestJS.cpp?rev=217444&r1=217443&r2=217444&view=diff
==============================================================================
--- cfe/trunk/unittests/Format/FormatTestJS.cpp (original)
+++ cfe/trunk/unittests/Format/FormatTestJS.cpp Tue Sep 9 09:37:39 2014
@@ -330,6 +330,8 @@ TEST_F(FormatTestJS, RegexLiteralSpecial
verifyFormat("var regex = /\\\\/g;");
verifyFormat("var regex = /\\a\\\\/g;");
verifyFormat("var regex = /\a\\//g;");
+ verifyFormat("var regex = /a\\//;\n"
+ "var x = 0;");
}
TEST_F(FormatTestJS, RegexLiteralModifiers) {
More information about the cfe-commits
mailing list