[cfe-commits] r59169 - in /cfe/trunk: include/clang/Lex/PTHLexer.h lib/Lex/PTHLexer.cpp

[Ted Kremenek]

Author: kremenek
Date: Wed Nov 12 15:37:15 2008
New Revision: 59169

URL: http://llvm.org/viewvc/llvm-project?rev=59169&view=rev
Log:
Add skeleton for PTH lexer.

Added:
    cfe/trunk/include/clang/Lex/PTHLexer.h
    cfe/trunk/lib/Lex/PTHLexer.cpp

Added: cfe/trunk/include/clang/Lex/PTHLexer.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Lex/PTHLexer.h?rev=59169&view=auto

==============================================================================
--- cfe/trunk/include/clang/Lex/PTHLexer.h (added)
+++ cfe/trunk/include/clang/Lex/PTHLexer.h Wed Nov 12 15:37:15 2008
@@ -0,0 +1,67 @@
+//===--- PTHLexer.h - Lexer based on Pre-tokenized input --------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the PTHLexer interface.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_PTHLexer_H
+#define LLVM_CLANG_PTHLexer_H
+
+#include "clang/Lex/PreprocessorLexer.h"
+
+namespace clang {
+  
+class PTHLexer : public PreprocessorLexer {
+
+  /// PP - Preprocessor.
+  Preprocessor& PP;
+  
+  /// FileLoc - Location for the start of the file.
+  ///
+  SourceLocation FileLoc;
+  
+  /// Tokens - This is the pointer to an array of tokens that the macro is
+  /// defined to, with arguments expanded for function-like macros.  If this is
+  /// a token stream, these are the tokens we are returning.
+  const Token *Tokens;
+  
+  /// NumTokens - This is the length of the Tokens array.
+  ///
+  unsigned NumTokens;
+  
+  /// CurToken - This is the next token that Lex will return.
+  ///
+  unsigned CurToken;
+        
+  PTHLexer(const PTHLexer&);  // DO NOT IMPLEMENT
+  void operator=(const PTHLexer&); // DO NOT IMPLEMENT
+
+public:
+
+  /// Create a PTHLexer for the specified token stream.
+  PTHLexer(Preprocessor& pp, SourceLocation fileloc,
+           const Token *TokArray, unsigned NumToks);
+  ~PTHLexer() {}
+    
+  /// Lex - Return the next token.
+  void Lex(Token &Tok);
+  
+  void setEOF(Token &Tok);
+  
+  /// getFileLoc - Return the File Location for the file we are lexing out of.
+  /// The physical location encodes the location where the characters come from,
+  /// the virtual location encodes where we should *claim* the characters came
+  /// from.  Currently this is only used by _Pragma handling.
+  SourceLocation getFileLoc() const { return FileLoc; }
+};
+
+}  // end namespace clang
+
+#endif

Added: cfe/trunk/lib/Lex/PTHLexer.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/PTHLexer.cpp?rev=59169&view=auto

==============================================================================
--- cfe/trunk/lib/Lex/PTHLexer.cpp (added)
+++ cfe/trunk/lib/Lex/PTHLexer.cpp Wed Nov 12 15:37:15 2008
@@ -0,0 +1,79 @@
+//===--- PTHLexer.cpp - Lex from a token stream ---------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the PTHLexer interface.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Lex/PTHLexer.h"
+#include "clang/Lex/Preprocessor.h"
+#include "clang/Lex/Token.h"
+#include "clang/Basic/TokenKinds.h"
+
+using namespace clang;
+
+PTHLexer::PTHLexer(Preprocessor& pp, SourceLocation fileloc,
+                   const Token *TokArray, unsigned NumToks)
+  : PP(pp), FileLoc(fileloc), Tokens(TokArray), NumTokens(NumToks), CurToken(0){
+
+  assert (Tokens[NumTokens-1].is(tok::eof));
+  --NumTokens;
+    
+  LexingRawMode = false;
+  ParsingPreprocessorDirective = false;
+  ParsingFilename = false;
+}
+
+void PTHLexer::Lex(Token& Tok) {
+
+  if (CurToken == NumTokens) {    
+    // If we hit the end of the file while parsing a preprocessor directive,
+    // end the preprocessor directive first.  The next token returned will
+    // then be the end of file.
+    //   OR
+    // If we are in raw mode, return this event as an EOF token.  Let the caller
+    // that put us in raw mode handle the event.
+    if (ParsingPreprocessorDirective || LexingRawMode) {
+      // Done parsing the "line".
+      ParsingPreprocessorDirective = false;
+      Tok = Tokens[CurToken]; // not an out-of-bound access
+      // FIXME: eom handling?
+    }
+    else
+      PP.HandleEndOfFile(Tok, false);
+    
+    return;
+  }
+
+  Tok = Tokens[CurToken];
+  
+  if (ParsingPreprocessorDirective && Tok.isAtStartOfLine()) {
+    ParsingPreprocessorDirective = false; // Done parsing the "line".
+    MIOpt.ReadToken();
+    // FIXME:  Need to replicate:
+    // FormTokenWithChars(Tok, CurPtr, tok::eom);
+    Tok.setKind(tok::eom);    
+    return;
+  }
+  else // Otherwise, advance to the next token.
+    ++CurToken;
+
+  if (Tok.isAtStartOfLine() && Tok.is(tok::hash) && !LexingRawMode) {
+    PP.HandleDirective(Tok);
+    PP.Lex(Tok);
+    return;
+  }
+    
+  MIOpt.ReadToken();
+}
+
+void PTHLexer::setEOF(Token& Tok) {
+  Tok = Tokens[NumTokens]; // NumTokens is already adjusted, so this isn't
+                           // an overflow.
+}