[PATCH] D74842: [clangd] Make use of syntax tokens in ReplayPreamble

[Kadir Cetinkaya via Phabricator via cfe-commits]

kadircet created this revision.
kadircet added a reviewer: sammccall.
Herald added subscribers: cfe-commits, usaxena95, arphaman, jkorous, MaskRay, ilya-biryukov.
Herald added a project: clang.

Replace usage of RawLexer with syntax tokens inside ReplayPreamble.


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D74842

Files:
  clang-tools-extra/clangd/ParsedAST.cpp


Index: clang-tools-extra/clangd/ParsedAST.cpp
===================================================================
--- clang-tools-extra/clangd/ParsedAST.cpp
+++ clang-tools-extra/clangd/ParsedAST.cpp
@@ -134,8 +134,9 @@
   ReplayPreamble(const IncludeStructure &Includes, PPCallbacks *Delegate,
                  const SourceManager &SM, Preprocessor &PP,
                  const LangOptions &LangOpts)
-      : Includes(Includes), Delegate(Delegate), SM(SM), PP(PP),
-        LangOpts(LangOpts) {}
+      : Includes(Includes), Delegate(Delegate), SM(SM), PP(PP) {
+    MainFileTokens = syntax::tokenize(SM.getMainFileID(), SM, LangOpts);
+  }
 
   // In a normal compile, the preamble traverses the following structure:
   //
@@ -167,29 +168,50 @@
         if (auto FE = SM.getFileManager().getFile(Inc.Resolved))
           File = *FE;
 
+      // Re-lex the #include directive to find its interesting parts.
+      auto HashLoc = SM.getComposedLoc(SM.getMainFileID(), Inc.HashOffset)
+                         .getRawEncoding();
+      auto HashTok =
+          llvm::find_if(MainFileTokens, [&HashLoc](const syntax::Token &T) {
+            return T.location().getRawEncoding() == HashLoc;
+          });
+      assert(HashTok != MainFileTokens.end() && HashTok->kind() == tok::hash);
+
+      auto IncTok = std::next(HashTok);
+      assert(IncTok != MainFileTokens.end());
+
+      auto FileTok = std::next(IncTok);
+      assert(FileTok != MainFileTokens.end());
+
+      // We imitate the PP logic here, except clang::Token::Flags, none of the
+      // callers seem to care about it (yet).
+      Token IncludeTok;
+      IncludeTok.startToken();
+      IncludeTok.setLocation(IncTok->location());
+      IncludeTok.setLength(IncTok->length());
+      IncludeTok.setKind(tok::raw_identifier);
+      IncludeTok.setRawIdentifierData(IncTok->text(SM).data());
+      PP.LookUpIdentifierInfo(IncludeTok);
+
+      // Again we imitate PP logic here, except clang::Token::Flags.
+      Token FilenameTok;
+      FilenameTok.startToken();
+      FilenameTok.setLocation(FileTok->location());
+      // Note that we can't make use of FileTok->length/text in here as in the
+      // case of angled includes this will contain tok::less instead of
+      // filename. Whereas Inc.Written contains the full header name including
+      // quotes/angles.
+      FilenameTok.setLength(Inc.Written.length());
+      FilenameTok.setKind(tok::header_name);
+      FilenameTok.setLiteralData(Inc.Written.data());
+
       llvm::StringRef WrittenFilename =
           llvm::StringRef(Inc.Written).drop_front().drop_back();
-      bool Angled = llvm::StringRef(Inc.Written).startswith("<");
-
-      // Re-lex the #include directive to find its interesting parts.
-      llvm::StringRef Src = SM.getBufferData(SM.getMainFileID());
-      Lexer RawLexer(SM.getLocForStartOfFile(SM.getMainFileID()), LangOpts,
-                     Src.begin(), Src.begin() + Inc.HashOffset, Src.end());
-      Token HashTok, IncludeTok, FilenameTok;
-      RawLexer.LexFromRawLexer(HashTok);
-      assert(HashTok.getKind() == tok::hash);
-      RawLexer.setParsingPreprocessorDirective(true);
-      RawLexer.LexFromRawLexer(IncludeTok);
-      IdentifierInfo *II = PP.getIdentifierInfo(IncludeTok.getRawIdentifier());
-      IncludeTok.setIdentifierInfo(II);
-      IncludeTok.setKind(II->getTokenID());
-      RawLexer.LexIncludeFilename(FilenameTok);
-
-      Delegate->InclusionDirective(
-          HashTok.getLocation(), IncludeTok, WrittenFilename, Angled,
-          CharSourceRange::getCharRange(FilenameTok.getLocation(),
-                                        FilenameTok.getEndLoc()),
-          File, "SearchPath", "RelPath", /*Imported=*/nullptr, Inc.FileKind);
+      Delegate->InclusionDirective(HashTok->location(), IncludeTok,
+                                   WrittenFilename, Inc.Written.front() == '<',
+                                   FileTok->range(SM).toCharRange(SM), File,
+                                   "SearchPath", "RelPath",
+                                   /*Imported=*/nullptr, Inc.FileKind);
       if (File)
         // FIXME: Use correctly named FileEntryRef.
         Delegate->FileSkipped(FileEntryRef(File->getName(), *File), FilenameTok,
@@ -205,7 +227,7 @@
   PPCallbacks *Delegate;
   const SourceManager &SM;
   Preprocessor &PP;
-  const LangOptions &LangOpts;
+  std::vector<syntax::Token> MainFileTokens;
 };
 
 } // namespace


-------------- next part --------------
A non-text attachment was scrubbed...
Name: D74842.245421.patch
Type: text/x-patch
Size: 4461 bytes
Desc: not available
URL: <http://lists.llvm.org/pipermail/cfe-commits/attachments/20200219/0a48c879/attachment-0001.bin>