[lld] [ELF] ScriptLexer: generate tokens lazily (PR #100493)

[Fangrui Song via llvm-commits]

================
@@ -10,18 +10,36 @@
 #define LLD_ELF_SCRIPT_LEXER_H
 
 #include "lld/Common/LLVM.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/MemoryBufferRef.h"
 #include <vector>
 
 namespace lld::elf {
 
 class ScriptLexer {
+protected:
+  struct Buffer {
+    // The unparsed buffer and the filename.
+    StringRef s, filename;
+  };
+  // The current buffer and parent buffers due to INCLUDE.
+  Buffer cur;
+  SmallVector<Buffer, 0> buffers;
+
+  // The token before the last next().
+  StringRef prevTok;
+  // The cache value of peek(). This is valid if curTokState and inExpr match.
+  StringRef curTok;
+  // The inExpr state when curTok is cached.
+  bool curTokState = false;
----------------
MaskRay wrote:

The idea is that I believe we will soon introduce new states, so I avoided `inExpr` in the name here.

https://github.com/llvm/llvm-project/pull/100493