[PATCH] D136539: [Lex] Bring back the magic number 50 in updateConsecutiveMacroArgTokens.

[Haojian Wu via Phabricator via cfe-commits]

hokein updated this revision to Diff 470599.
hokein marked an inline comment as not done.
hokein added a comment.

refine the test.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D136539/new/

https://reviews.llvm.org/D136539

Files:
  clang/lib/Lex/TokenLexer.cpp
  clang/test/Lexer/update_consecutive_macro_address_space.c


Index: clang/test/Lexer/update_consecutive_macro_address_space.c
===================================================================
--- /dev/null
+++ clang/test/Lexer/update_consecutive_macro_address_space.c
@@ -0,0 +1,36 @@
+// RUN: %clang -cc1 -print-stats %s 2>&1 | FileCheck %s
+// CHECK: 6 local SLocEntry's allocated
+//
+// Verify that the macro arg expansion is split to two file ids, we have 6 file
+// ids rather than 5:
+//   0: invalid file id
+//   1: main file
+//   2: builtin file
+//   3: macro expansion for X
+//   4: macro arg expansions for 1
+//   5: macro arg expansions for == 2
+#define X(x) (int)(x);
+void func() {
+  X(1
+/*************************************************************************************************/
+/*************************************************************************************************/
+/*************************************************************************************************/
+/*************************************************************************************************/
+/*************************************************************************************************/
+/*************************************************************************************************/
+/*************************************************************************************************/
+/*************************************************************************************************/
+/*************************************************************************************************/
+/*************************************************************************************************/
+/*************************************************************************************************/
+/*************************************************************************************************/
+/*************************************************************************************************/
+/*************************************************************************************************/
+/*************************************************************************************************/
+/*************************************************************************************************/
+/*************************************************************************************************/
+/*************************************************************************************************/
+/*************************************************************************************************/
+/*************************************************************************************************/
+== 2);
+}
\ No newline at end of file
Index: clang/lib/Lex/TokenLexer.cpp
===================================================================
--- clang/lib/Lex/TokenLexer.cpp
+++ clang/lib/Lex/TokenLexer.cpp
@@ -993,6 +993,16 @@
   llvm::MutableArrayRef<Token> All(begin_tokens, end_tokens);
   llvm::MutableArrayRef<Token> Partition;
 
+  auto NearLast = [&, Last = BeginLoc](SourceLocation Loc) mutable {
+    // The maximum distance between two consecutive tokens in a partition.
+    // This is an important trick to avoid using too much SourceLocation address
+    // space!
+    static constexpr SourceLocation::IntTy MaxDistance = 50;
+    auto Distance = Loc.getRawEncoding() - Last.getRawEncoding();
+    Last = Loc;
+    return Distance <= MaxDistance;
+  };
+
   // Partition the tokens by their FileID.
   // This is a hot function, and calling getFileID can be expensive, the
   // implementation is optimized by reducing the number of getFileID.
@@ -1000,7 +1010,7 @@
     // Consecutive tokens not written in macros must be from the same file.
     // (Neither #include nor eof can occur inside a macro argument.)
     Partition = All.take_while([&](const Token &T) {
-      return T.getLocation().isFileID();
+      return T.getLocation().isFileID() && NearLast(T.getLocation());
     });
   } else {
     // Call getFileID once to calculate the bounds, and use the cheaper
@@ -1009,7 +1019,8 @@
     SourceLocation Limit =
         SM.getComposedLoc(BeginFID, SM.getFileIDSize(BeginFID));
     Partition = All.take_while([&](const Token &T) {
-      return T.getLocation() >= BeginLoc && T.getLocation() < Limit;
+      return T.getLocation() >= BeginLoc && T.getLocation() < Limit &&
+             NearLast(T.getLocation());
     });
   }
   assert(!Partition.empty());


-------------- next part --------------
A non-text attachment was scrubbed...
Name: D136539.470599.patch
Type: text/x-patch
Size: 4459 bytes
Desc: not available
URL: <http://lists.llvm.org/pipermail/cfe-commits/attachments/20221025/6ebf3288/attachment-0001.bin>