[PATCH] D14252: [AsmParser] Allow tokens to be put back in to the token stream.

[Colin LeMahieu via llvm-commits]

colinl created this revision.
colinl added reviewers: mcrosier, sidneym.
colinl added a subscriber: llvm-commits.
colinl set the repository for this revision to rL LLVM.

This patch modifies the AsmParser to allow tokens to be injected back in to the token stream.  This is useful for parsing some grammars that otherwise confuse the lexer.

Examples:
"v31:30.h" The sequence seen by the lexer is Token "v31" Colon ":" Double "30." Token ".h"  It's easier to change the tokens to Token "v31:30" Token ".h"

"memw(r0 << #2 + #60)" This sequence confuses the expression parser when parsing "2 + #", The plus looks like an infix addition but it's actually part of the instruction text.

Repository:
  rL LLVM

http://reviews.llvm.org/D14252

Files:
  include/llvm/MC/MCParser/MCAsmLexer.h
  lib/MC/MCParser/MCAsmLexer.cpp

Index: lib/MC/MCParser/MCAsmLexer.cpp
===================================================================
--- lib/MC/MCParser/MCAsmLexer.cpp
+++ lib/MC/MCParser/MCAsmLexer.cpp
@@ -12,8 +12,8 @@
 
 using namespace llvm;
 
-MCAsmLexer::MCAsmLexer() : CurTok(AsmToken::Error, StringRef()),
-                           TokStart(nullptr), SkipSpace(true) {
+MCAsmLexer::MCAsmLexer() : TokStart(nullptr), SkipSpace(true) {
+  CurTok.emplace_back(AsmToken::Error, StringRef());
 }
 
 MCAsmLexer::~MCAsmLexer() {
Index: include/llvm/MC/MCParser/MCAsmLexer.h
===================================================================
--- include/llvm/MC/MCParser/MCAsmLexer.h
+++ include/llvm/MC/MCParser/MCAsmLexer.h
@@ -118,7 +118,7 @@
 /// lexers.
 class MCAsmLexer {
   /// The current token, stored in the base class for faster access.
-  AsmToken CurTok;
+  SmallVector<AsmToken, 1> CurTok;
 
   /// The location and description of the current error
   SMLoc ErrLoc;
@@ -148,7 +148,15 @@
   /// The lexer will continuosly return the end-of-file token once the end of
   /// the main input file has been reached.
   const AsmToken &Lex() {
-    return CurTok = LexToken();
+    assert(!CurTok.empty());
+    CurTok.erase(CurTok.begin());
+    if (CurTok.empty())
+      CurTok.emplace_back(LexToken());
+    return CurTok.front();
+  }
+
+  void UnLex(AsmToken const &Token) {
+    CurTok.insert(CurTok.begin(), Token);
   }
 
   virtual StringRef LexUntilEndOfStatement() = 0;
@@ -158,7 +166,7 @@
 
   /// Get the current (last) lexed token.
   const AsmToken &getTok() const {
-    return CurTok;
+    return CurTok[0];
   }
 
   /// Look ahead at the next token to be lexed.


-------------- next part --------------
A non-text attachment was scrubbed...
Name: D14252.38959.patch
Type: text/x-patch
Size: 1667 bytes
Desc: not available
URL: <http://lists.llvm.org/pipermail/llvm-commits/attachments/20151102/e092b1bc/attachment.bin>