[lld] [ELF] Added `struct Token` and changed `next()` and `peek()` to return Token (PR #100180)

[via llvm-commits]

================
@@ -155,15 +170,199 @@ void ScriptLexer::tokenize(MemoryBufferRef mb) {
 
     // A character that cannot start a word (which is usually a
     // punctuation) forms a single character token.
-    if (pos == 0)
+    if (pos == 0) {
       pos = 1;
-    vec.push_back(s.substr(0, pos));
+      vec.push_back(getOperatorToken(s));
+    } else {
+      vec.push_back(getKeywordorIdentifier(s.substr(0, pos)));
+    }
     s = s.substr(pos);
   }
 
   tokens.insert(tokens.begin() + pos, vec.begin(), vec.end());
 }
 
+ScriptLexer::Token ScriptLexer::getOperatorToken(StringRef s) {
+  auto createToken = [&](Kind kind, size_t pos) -> Token {
+    return {kind, s.substr(0, pos)};
+  };
+
+  switch (s.front()) {
+  case EOF:
+    return createToken(Kind::Eof, 0);
+  case '(':
+    return createToken(Kind::BracektBegin, 1);
+  case ')':
+    return createToken(Kind::BracektEnd, 1);
+  case '{':
+    return createToken(Kind::CurlyBegin, 1);
+  case '}':
+    return createToken(Kind::CurlyEnd, 1);
+  case ';':
+    return createToken(Kind::Semicolon, 1);
+  case ',':
+    return createToken(Kind::Comma, 1);
+  case ':':
+    return createToken(Kind::Colon, 1);
+  case '?':
+    return createToken(Kind::Question, 1);
+  case '%':
+    return createToken(Kind::Percent, 1);
+  case '!':
+    if (s.size() > 1 && s[1] == '=')
+      return createToken(Kind::NotEqual, 2);
+    return createToken(Kind::Excalamation, 1);
+  case '*':
+    if (s.size() > 1 && s[1] == '=')
+      return createToken(Kind::MulAssign, 2);
+    return createToken(Kind::Asterisk, 1);
+  case '/':
+    if (s.size() > 1 && s[1] == '=')
+      return createToken(Kind::DivAssign, 2);
+    return createToken(Kind::Slash, 1);
+  case '=':
+    if (s.size() > 1 && s[1] == '=')
+      return createToken(Kind::Equal, 2);
+    return createToken(Kind::Assign, 1);
+  case '+':
+    if (s.size() > 1 && s[1] == '=')
+      return createToken(Kind::PlusAssign, 2);
+    return createToken(Kind::Plus, 1);
+  case '-':
+    if (s.size() > 1 && s[1] == '=')
+      return createToken(Kind::MinusAssign, 2);
+    return createToken(Kind::Minus, 1);
+  case '<':
+    if (s.size() > 2 && s[1] == s[0] && s[2] == '=')
+      return createToken(Kind::LeftShiftAssign, 3);
+    if (s.size() > 1) {
+      if (s[1] == '=')
+        return createToken(Kind::LessEqual, 2);
+      if (s[1] == '<')
+        return createToken(Kind::LeftShift, 2);
+    }
+    return createToken(Kind::Less, 1);
+  case '>':
+    if (s.size() > 2 && s[1] == s[0] && s[2] == '=')
+      return createToken(Kind::RightShiftAssign, 3);
+    if (s.size() > 1) {
+      if (s[1] == '=')
+        return createToken(Kind::GreaterEqual, 2);
+      if (s[1] == '>')
+        return createToken(Kind::RightShift, 2);
+    }
+    return createToken(Kind::Greater, 1);
+  case '&':
+    if (s.size() > 1) {
+      if (s[1] == '=')
+        return createToken(Kind::AndAssign, 2);
+      if (s[1] == '&')
+        return createToken(Kind::AndGate, 2);
+    }
+    return createToken(Kind::Bitwise, 1);
+  case '^':
+    if (s.size() > 1 && s[1] == '=')
+      return createToken(Kind::XorAssign, 2);
+    return createToken(Kind::Xor, 1);
+  case '|':
+    if (s.size() > 1) {
+      if (s[1] == '=')
+        return createToken(Kind::OrAssign, 2);
+      if (s[1] == '|')
+        return createToken(Kind::OrGate, 2);
+    }
+    return createToken(Kind::Or, 1);
+  case '.':
+    return createToken(Kind::Dot, 1);
+  case '_':
+    return createToken(Kind::Underscore, 1);
+  case '0':
+  case '1':
+  case '2':
+  case '3':
+  case '4':
+  case '5':
+  case '6':
+  case '7':
+  case '8':
+  case '9':
+    return createToken(Kind::Decimal, 1);
+  default:
+    return {Kind::Identifier, s};
+  }
+}
+
+ScriptLexer::Token ScriptLexer::getKeywordorIdentifier(StringRef s) {
+  static const std::unordered_map<std::string, Kind> keywords = {
----------------
PiJoules wrote:

Could be made a static constexpr global instead?

https://github.com/llvm/llvm-project/pull/100180