[PATCH] D118471: [clang][Lexer] Make raw and normal lexer behave the same for line comments

[Kadir Cetinkaya via Phabricator via cfe-commits]

This revision was landed with ongoing or failed builds.
This revision was automatically updated to reflect the committed changes.
kadircet marked 2 inline comments as done.
Closed by commit rGff77071a4d67: [clang][Lexer] Make raw and normal lexer behave the same for line comments (authored by kadircet).

Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D118471/new/

https://reviews.llvm.org/D118471

Files:
  clang/lib/Lex/Lexer.cpp
  clang/unittests/Lex/LexerTest.cpp


Index: clang/unittests/Lex/LexerTest.cpp
===================================================================
--- clang/unittests/Lex/LexerTest.cpp
+++ clang/unittests/Lex/LexerTest.cpp
@@ -23,6 +23,8 @@
 #include "clang/Lex/ModuleLoader.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Lex/PreprocessorOptions.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/StringRef.h"
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
 #include <memory>
@@ -632,4 +634,27 @@
   EXPECT_EQ(SourceMgr.getNumCreatedFIDsForFileID(PP->getPredefinesFileID()),
             1U);
 }
+
+TEST_F(LexerTest, RawAndNormalLexSameForLineComments) {
+  const llvm::StringLiteral Source = R"cpp(
+  // First line comment.
+  //* Second line comment which is ambigious.
+  )cpp";
+  LangOpts.LineComment = false;
+  auto Toks = Lex(Source);
+  auto &SM = PP->getSourceManager();
+  auto SrcBuffer = SM.getBufferData(SM.getMainFileID());
+  Lexer L(SM.getLocForStartOfFile(SM.getMainFileID()), PP->getLangOpts(),
+          SrcBuffer.data(), SrcBuffer.data(),
+          SrcBuffer.data() + SrcBuffer.size());
+
+  auto ToksView = llvm::makeArrayRef(Toks);
+  clang::Token T;
+  while (!L.LexFromRawLexer(T)) {
+    ASSERT_TRUE(!ToksView.empty());
+    EXPECT_EQ(T.getKind(), ToksView.front().getKind());
+    ToksView = ToksView.drop_front();
+  }
+  EXPECT_TRUE(ToksView.empty());
+}
 } // anonymous namespace
Index: clang/lib/Lex/Lexer.cpp
===================================================================
--- clang/lib/Lex/Lexer.cpp
+++ clang/lib/Lex/Lexer.cpp
@@ -2378,8 +2378,9 @@
                             bool &TokAtPhysicalStartOfLine) {
   // If Line comments aren't explicitly enabled for this language, emit an
   // extension warning.
-  if (!LangOpts.LineComment && !isLexingRawMode()) {
-    Diag(BufferPtr, diag::ext_line_comment);
+  if (!LangOpts.LineComment) {
+    if (!isLexingRawMode()) // There's no PP in raw mode, so can't emit diags.
+      Diag(BufferPtr, diag::ext_line_comment);
 
     // Mark them enabled so we only emit one warning for this translation
     // unit.


-------------- next part --------------
A non-text attachment was scrubbed...
Name: D118471.404514.patch
Type: text/x-patch
Size: 2086 bytes
Desc: not available
URL: <http://lists.llvm.org/pipermail/cfe-commits/attachments/20220131/f083bf15/attachment.bin>