r192490 - clang-format: Don't remove 'unknown' tokens.
Daniel Jasper
djasper at google.com
Fri Oct 11 12:45:02 PDT 2013
Author: djasper
Date: Fri Oct 11 14:45:02 2013
New Revision: 192490
URL: http://llvm.org/viewvc/llvm-project?rev=192490&view=rev
Log:
clang-format: Don't remove 'unknown' tokens.
In certain macros or incorrect string literals, the token stream can
contain 'unknown' tokens, e.g. a single backslash or a set of empty
ticks. clang-format simply treated them as whitespace and removed them
prior to this patch.
This fixes llvm.org/PR17215
Modified:
cfe/trunk/lib/Format/Format.cpp
cfe/trunk/lib/Format/TokenAnnotator.cpp
cfe/trunk/unittests/Format/FormatTest.cpp
Modified: cfe/trunk/lib/Format/Format.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Format/Format.cpp?rev=192490&r1=192489&r2=192490&view=diff
==============================================================================
--- cfe/trunk/lib/Format/Format.cpp (original)
+++ cfe/trunk/lib/Format/Format.cpp Fri Oct 11 14:45:02 2013
@@ -691,18 +691,32 @@ private:
FormatTok->LastNewlineOffset = WhitespaceLength + i + 1;
Column = 0;
break;
+ case '\r':
+ case '\f':
+ case '\v':
+ Column = 0;
+ break;
case ' ':
++Column;
break;
case '\t':
Column += Style.TabWidth - Column % Style.TabWidth;
break;
+ case '\\':
+ ++Column;
+ if (i + 1 == e || (FormatTok->TokenText[i + 1] != '\r' &&
+ FormatTok->TokenText[i + 1] != '\n'))
+ FormatTok->Type = TT_ImplicitStringLiteral;
+ break;
default:
+ FormatTok->Type = TT_ImplicitStringLiteral;
++Column;
break;
}
}
+ if (FormatTok->Type == TT_ImplicitStringLiteral)
+ break;
WhitespaceLength += FormatTok->Tok.getLength();
readRawToken(*FormatTok);
Modified: cfe/trunk/lib/Format/TokenAnnotator.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Format/TokenAnnotator.cpp?rev=192490&r1=192489&r2=192490&view=diff
==============================================================================
--- cfe/trunk/lib/Format/TokenAnnotator.cpp (original)
+++ cfe/trunk/lib/Format/TokenAnnotator.cpp Fri Oct 11 14:45:02 2013
@@ -525,7 +525,8 @@ private:
// Reset token type in case we have already looked at it and then recovered
// from an error (e.g. failure to find the matching >).
- if (CurrentToken != NULL && CurrentToken->Type != TT_LambdaLSquare)
+ if (CurrentToken != NULL && CurrentToken->Type != TT_LambdaLSquare &&
+ CurrentToken->Type != TT_ImplicitStringLiteral)
CurrentToken->Type = TT_Unknown;
}
@@ -1292,6 +1293,8 @@ bool TokenAnnotator::spaceRequiredBefore
const FormatToken &Tok) {
if (Tok.Tok.getIdentifierInfo() && Tok.Previous->Tok.getIdentifierInfo())
return true; // Never ever merge two identifiers.
+ if (Tok.Previous->Type == TT_ImplicitStringLiteral)
+ return Tok.WhitespaceRange.getBegin() != Tok.WhitespaceRange.getEnd();
if (Line.Type == LT_ObjCMethodDecl) {
if (Tok.Previous->Type == TT_ObjCMethodSpecifier)
return true;
Modified: cfe/trunk/unittests/Format/FormatTest.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/unittests/Format/FormatTest.cpp?rev=192490&r1=192489&r2=192490&view=diff
==============================================================================
--- cfe/trunk/unittests/Format/FormatTest.cpp (original)
+++ cfe/trunk/unittests/Format/FormatTest.cpp Fri Oct 11 14:45:02 2013
@@ -1890,6 +1890,16 @@ TEST_F(FormatTest, EndOfFileEndsPPDirect
EXPECT_EQ("#define A B", format("# \\\n define \\\n A \\\n B"));
}
+TEST_F(FormatTest, DoesntRemoveUnknownTokens) {
+ verifyFormat("#define A \\x20");
+ verifyFormat("#define A \\ x20");
+ EXPECT_EQ("#define A \\ x20", format("#define A \\ x20"));
+ verifyFormat("#define A ''");
+ verifyFormat("#define A ''qqq");
+ verifyFormat("#define A `qqq");
+ verifyFormat("f(\"aaaa, bbbb, \"\\\"ccccc\\\"\");");
+}
+
TEST_F(FormatTest, IndentsPPDirectiveInReducedSpace) {
verifyFormat("#define A(BB)", getLLVMStyleWithColumns(13));
verifyFormat("#define A( \\\n BB)", getLLVMStyleWithColumns(12));
More information about the cfe-commits
mailing list