[PATCH] Use lexing mode based on FormatStyle.Standard.

[Alexander Kornienko]

Hi klimek,

Some valid pre-C++11 constructs change meaning when lexed in C++11
mode, e.g.
#define x(_a) printf("foo"_a);
(example from http://llvm.org/bugs/show_bug.cgi?id=16342). "foo"_a is treated as
a user-defined string literal when parsed in C++11 mode.
In order to deal with this correctly, we need to set lexing mode according to
which standard the code conforms to. We already have a configuration value for
this (FormatStyle.Standard), which seems to be appropriate to use in this case
as well.

http://llvm-reviews.chandlerc.com/D1028

Files:
  include/clang/Format/Format.h
  lib/Format/Format.cpp
  tools/clang-format/ClangFormat.cpp
  unittests/Format/FormatTest.cpp

Index: include/clang/Format/Format.h
===================================================================
--- include/clang/Format/Format.h
+++ include/clang/Format/Format.h
@@ -218,7 +218,11 @@
                                StringRef FileName = "<stdin>");
 
 /// \brief Returns the \c LangOpts that the formatter expects you to set.
-LangOptions getFormattingLangOpts();
+///
+/// \p Standard determines lexing mode: LC_Cpp11 and LS_Auto turn on C++11
+/// lexing mode, LS_Cpp03 - C++03 mode.
+LangOptions getFormattingLangOpts(FormatStyle::LanguageStandard Standard =
+                                      FormatStyle::LS_Cpp11);
 
 } // end namespace format
 } // end namespace clang
Index: lib/Format/Format.cpp
===================================================================
--- lib/Format/Format.cpp
+++ lib/Format/Format.cpp
@@ -1660,7 +1660,8 @@
   SourceMgr.overrideFileContents(Entry, Buf);
   FileID ID =
       SourceMgr.createFileID(Entry, SourceLocation(), clang::SrcMgr::C_User);
-  Lexer Lex(ID, SourceMgr.getBuffer(ID), SourceMgr, getFormattingLangOpts());
+  Lexer Lex(ID, SourceMgr.getBuffer(ID), SourceMgr,
+            getFormattingLangOpts(Style.Standard));
   SourceLocation StartOfFile = SourceMgr.getLocForStartOfFile(ID);
   std::vector<CharSourceRange> CharRanges;
   for (unsigned i = 0, e = Ranges.size(); i != e; ++i) {
@@ -1671,10 +1672,10 @@
   return reformat(Style, Lex, SourceMgr, CharRanges);
 }
 
-LangOptions getFormattingLangOpts() {
+LangOptions getFormattingLangOpts(FormatStyle::LanguageStandard Standard) {
   LangOptions LangOpts;
   LangOpts.CPlusPlus = 1;
-  LangOpts.CPlusPlus11 = 1;
+  LangOpts.CPlusPlus11 = Standard == FormatStyle::LS_Cpp03 ? 0 : 1;
   LangOpts.LineComment = 1;
   LangOpts.Bool = 1;
   LangOpts.ObjC1 = 1;
Index: tools/clang-format/ClangFormat.cpp
===================================================================
--- tools/clang-format/ClangFormat.cpp
+++ tools/clang-format/ClangFormat.cpp
@@ -163,7 +163,6 @@
     return true;
   }
   FileID ID = createInMemoryFile(FileName, Code.get(), Sources, Files);
-  Lexer Lex(ID, Sources.getBuffer(ID), Sources, getFormattingLangOpts());
   if (Offsets.empty())
     Offsets.push_back(0);
   if (Offsets.size() != Lengths.size() &&
@@ -195,8 +194,10 @@
     }
     Ranges.push_back(CharSourceRange::getCharRange(Start, End));
   }
-  tooling::Replacements Replaces =
-      reformat(getStyle(Style, FileName), Lex, Sources, Ranges);
+  FormatStyle FormatStyle = getStyle(Style, FileName);
+  Lexer Lex(ID, Sources.getBuffer(ID), Sources,
+            getFormattingLangOpts(FormatStyle.Standard));
+  tooling::Replacements Replaces = reformat(FormatStyle, Lex, Sources, Ranges);
   if (OutputXML) {
     llvm::outs()
         << "<?xml version='1.0'?>\n<replacements xml:space='preserve'>\n";
Index: unittests/Format/FormatTest.cpp
===================================================================
--- unittests/Format/FormatTest.cpp
+++ unittests/Format/FormatTest.cpp
@@ -3097,9 +3097,9 @@
   verifyFormat("a-- > b;");
   verifyFormat("b ? -a : c;");
   verifyFormat("n * sizeof char16;");
-  verifyFormat("n * alignof char16;");
+  verifyFormat("n * alignof char16;", getGoogleStyle());
   verifyFormat("sizeof(char);");
-  verifyFormat("alignof(char);");
+  verifyFormat("alignof(char);", getGoogleStyle());
 
   verifyFormat("return -1;");
   verifyFormat("switch (a) {\n"
@@ -3374,16 +3374,16 @@
   verifyFormat("void f(int i = (kValue) * kMask) {}");
   verifyFormat("void f(int i = (kA * kB) & kMask) {}");
   verifyFormat("int a = sizeof(int) * b;");
-  verifyFormat("int a = alignof(int) * b;");
+  verifyFormat("int a = alignof(int) * b;", getGoogleStyle());
   verifyFormat("template <> void f<int>(int i) SOME_ANNOTATION;");
   verifyFormat("f(\"%\" SOME_MACRO(ll) \"d\");");
 
   // These are not casts, but at some point were confused with casts.
   verifyFormat("virtual void foo(int *) override;");
   verifyFormat("virtual void foo(char &) const;");
   verifyFormat("virtual void foo(int *a, char *) const;");
   verifyFormat("int a = sizeof(int *) + b;");
-  verifyFormat("int a = alignof(int *) + b;");
+  verifyFormat("int a = alignof(int *) + b;", getGoogleStyle());
 }
 
 TEST_F(FormatTest, FormatsFunctionTypes) {
@@ -4250,7 +4250,7 @@
   verifyFormat("int a = ++[foo bar:baz];");
   verifyFormat("int a = --[foo bar:baz];");
   verifyFormat("int a = sizeof [foo bar:baz];");
-  verifyFormat("int a = alignof [foo bar:baz];");
+  verifyFormat("int a = alignof [foo bar:baz];", getGoogleStyle());
   verifyFormat("int a = &[foo bar:baz];");
   verifyFormat("int a = *[foo bar:baz];");
   // FIXME: Make casts work, without breaking f()[4].
@@ -4682,16 +4682,22 @@
 }
 
 TEST_F(FormatTest, SkipsUnknownStringLiterals) {
-  EXPECT_EQ("u8\"unsupported literal\";",
-            format("u8\"unsupported literal\";", getLLVMStyleWithColumns(15)));
+  EXPECT_EQ(
+      "u8\"unsupported literal\";",
+      format("u8\"unsupported literal\";", getGoogleStyleWithColumns(15)));
   EXPECT_EQ("u\"unsupported literal\";",
-            format("u\"unsupported literal\";", getLLVMStyleWithColumns(15)));
+            format("u\"unsupported literal\";", getGoogleStyleWithColumns(15)));
   EXPECT_EQ("U\"unsupported literal\";",
-            format("U\"unsupported literal\";", getLLVMStyleWithColumns(15)));
+            format("U\"unsupported literal\";", getGoogleStyleWithColumns(15)));
   EXPECT_EQ("L\"unsupported literal\";",
-            format("L\"unsupported literal\";", getLLVMStyleWithColumns(15)));
+            format("L\"unsupported literal\";", getGoogleStyleWithColumns(15)));
   EXPECT_EQ("R\"x(raw literal)x\";",
-            format("R\"x(raw literal)x\";", getLLVMStyleWithColumns(15)));
+            format("R\"x(raw literal)x\";", getGoogleStyleWithColumns(15)));
+}
+
+TEST_F(FormatTest, DoesNotTryToParseUDLiteralsInPreCpp11Code) {
+  EXPECT_EQ("#define x(_a) printf(\"foo\" _a);",
+            format("#define x(_a) printf(\"foo\"_a);", getLLVMStyle()));
 }
 
 TEST_F(FormatTest, BreakStringLiteralsBeforeUnbreakableTokenSequence) {
@@ -4773,7 +4779,7 @@
             "\"1\"",
             format("\"test\\000000000001\"", getLLVMStyleWithColumns(10)));
   EXPECT_EQ("R\"(\\x\\x00)\"\n",
-            format("R\"(\\x\\x00)\"\n", getLLVMStyleWithColumns(7)));
+            format("R\"(\\x\\x00)\"\n", getGoogleStyleWithColumns(7)));
 }
 
 TEST_F(FormatTest, DoNotCreateUnreasonableUnwrappedLines) {
-------------- next part --------------
A non-text attachment was scrubbed...
Name: D1028.1.patch
Type: text/x-patch
Size: 6458 bytes
Desc: not available
URL: <http://lists.llvm.org/pipermail/cfe-commits/attachments/20130622/59036942/attachment.bin>