[cfe-commits] r57409 - in /cfe/trunk: Driver/RewriteTest.cpp include/clang/Rewrite/TokenRewriter.h lib/Rewrite/TokenRewriter.cpp
Chris Lattner
sabre at nondot.org
Sat Oct 11 23:09:53 PDT 2008
Author: lattner
Date: Sun Oct 12 01:09:52 2008
New Revision: 57409
URL: http://llvm.org/viewvc/llvm-project?rev=57409&view=rev
Log:
make the -rewrite-test a bit more interesting: it now
wraps comments in <i> tags. Extend rewrite tokens to support
this minimal functionality.
Modified:
cfe/trunk/Driver/RewriteTest.cpp
cfe/trunk/include/clang/Rewrite/TokenRewriter.h
cfe/trunk/lib/Rewrite/TokenRewriter.cpp
Modified: cfe/trunk/Driver/RewriteTest.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/Driver/RewriteTest.cpp?rev=57409&r1=57408&r2=57409&view=diff
==============================================================================
--- cfe/trunk/Driver/RewriteTest.cpp (original)
+++ cfe/trunk/Driver/RewriteTest.cpp Sun Oct 12 01:09:52 2008
@@ -11,9 +11,9 @@
//
//===----------------------------------------------------------------------===//
-#include "clang/Rewrite/TokenRewriter.h"
#include "clang.h"
#include "clang/Lex/Preprocessor.h"
+#include "clang/Rewrite/TokenRewriter.h"
#include <iostream>
void clang::DoRewriteTest(Preprocessor &PP, const std::string &InFileName,
@@ -22,27 +22,18 @@
const LangOptions &LangOpts = PP.getLangOptions();
TokenRewriter Rewriter(SM.getMainFileID(), SM, LangOpts);
-
-
-
-
-
- std::pair<const char*,const char*> File =SM.getBufferData(SM.getMainFileID());
-
- // Create a lexer to lex all the tokens of the main file in raw mode. Even
- // though it is in raw mode, it will not return comments.
- Lexer RawLex(SourceLocation::getFileLoc(SM.getMainFileID(), 0),
- LangOpts, File.first, File.second);
-
- RawLex.SetKeepWhitespaceMode(true);
-
- Token RawTok;
- RawLex.LexFromRawLexer(RawTok);
- while (RawTok.isNot(tok::eof)) {
- std::cout << PP.getSpelling(RawTok);
- RawLex.LexFromRawLexer(RawTok);
+
+ // Throw <i> </i> tags around comments.
+ for (TokenRewriter::token_iterator I = Rewriter.token_begin(),
+ E = Rewriter.token_end(); I != E; ++I) {
+ if (I->isNot(tok::comment)) continue;
+
+ Rewriter.AddTokenBefore(I, "<i>");
+ Rewriter.AddTokenAfter(I, "</i>");
}
+
+ // Print out the output.
for (TokenRewriter::token_iterator I = Rewriter.token_begin(),
E = Rewriter.token_end(); I != E; ++I)
std::cout << PP.getSpelling(*I);
Modified: cfe/trunk/include/clang/Rewrite/TokenRewriter.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Rewrite/TokenRewriter.h?rev=57409&r1=57408&r2=57409&view=diff
==============================================================================
--- cfe/trunk/include/clang/Rewrite/TokenRewriter.h (original)
+++ cfe/trunk/include/clang/Rewrite/TokenRewriter.h Sun Oct 12 01:09:52 2008
@@ -16,12 +16,14 @@
#define LLVM_CLANG_TOKENREWRITER_H
#include "clang/Basic/SourceLocation.h"
+#include "llvm/ADT/OwningPtr.h"
#include <list>
#include <map>
namespace clang {
class Token;
class LangOptions;
+ class ScratchBuffer;
class TokenRewriter {
/// TokenList - This is the list of raw tokens that make up this file. Each
@@ -37,20 +39,37 @@
/// backwards.
std::map<SourceLocation, TokenRefTy> TokenAtLoc;
+ /// ScratchBuf - This is the buffer that we create scratch tokens from.
+ ///
+ llvm::OwningPtr<ScratchBuffer> ScratchBuf;
+
+ TokenRewriter(const TokenRewriter&); // DO NOT IMPLEMENT
+ void operator=(const TokenRewriter&); // DO NOT IMPLEMENT.
public:
/// TokenRewriter - This creates a TokenRewriter for the file with the
/// specified FileID.
TokenRewriter(unsigned FileID, SourceManager &SM, const LangOptions &LO);
-
+ ~TokenRewriter();
typedef std::list<Token>::const_iterator token_iterator;
token_iterator token_begin() const { return TokenList.begin(); }
token_iterator token_end() const { return TokenList.end(); }
+
+ token_iterator AddTokenBefore(token_iterator I, const char *Val);
+ token_iterator AddTokenAfter(token_iterator I, const char *Val) {
+ assert(I != token_end() && "Cannot insert after token_end()!");
+ return AddTokenBefore(++I, Val);
+ }
+
private:
+ /// RemapIterator - Convert from token_iterator (a const iterator) to
+ /// TokenRefTy (a non-const iterator).
+ TokenRefTy RemapIterator(token_iterator I);
+
/// AddToken - Add the specified token into the Rewriter before the other
/// position.
- void AddToken(const Token &T, TokenRefTy Where);
+ TokenRefTy AddToken(const Token &T, TokenRefTy Where);
};
Modified: cfe/trunk/lib/Rewrite/TokenRewriter.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Rewrite/TokenRewriter.cpp?rev=57409&r1=57408&r2=57409&view=diff
==============================================================================
--- cfe/trunk/lib/Rewrite/TokenRewriter.cpp (original)
+++ cfe/trunk/lib/Rewrite/TokenRewriter.cpp Sun Oct 12 01:09:52 2008
@@ -14,11 +14,13 @@
#include "clang/Rewrite/TokenRewriter.h"
#include "clang/Lex/Lexer.h"
+#include "clang/Lex/ScratchBuffer.h"
#include "clang/Basic/SourceManager.h"
using namespace clang;
TokenRewriter::TokenRewriter(unsigned FileID, SourceManager &SM,
const LangOptions &LangOpts) {
+ ScratchBuf.reset(new ScratchBuffer(SM));
std::pair<const char*,const char*> File = SM.getBufferData(FileID);
@@ -33,21 +35,66 @@
Token RawTok;
RawLex.LexFromRawLexer(RawTok);
while (RawTok.isNot(tok::eof)) {
+#if 0
+ if (Tok.is(tok::identifier)) {
+ // Look up the identifier info for the token. This should use
+ // IdentifierTable directly instead of PP.
+ Tok.setIdentifierInfo(PP.LookUpIdentifierInfo(Tok));
+ }
+#endif
+
AddToken(RawTok, TokenList.end());
RawLex.LexFromRawLexer(RawTok);
}
-
-
}
+TokenRewriter::~TokenRewriter() {
+}
+
+
+/// RemapIterator - Convert from token_iterator (a const iterator) to
+/// TokenRefTy (a non-const iterator).
+TokenRewriter::TokenRefTy TokenRewriter::RemapIterator(token_iterator I) {
+ if (I == token_end()) return TokenList.end();
+
+ // FIXME: This is horrible, we should use our own list or something to avoid
+ // this.
+ std::map<SourceLocation, TokenRefTy>::iterator MapIt =
+ TokenAtLoc.find(I->getLocation());
+ assert(MapIt != TokenAtLoc.end() && "iterator not in rewriter?");
+ return MapIt->second;
+}
+
+
/// AddToken - Add the specified token into the Rewriter before the other
/// position.
-void TokenRewriter::AddToken(const Token &T, TokenRefTy Where) {
+TokenRewriter::TokenRefTy
+TokenRewriter::AddToken(const Token &T, TokenRefTy Where) {
Where = TokenList.insert(Where, T);
bool InsertSuccess = TokenAtLoc.insert(std::make_pair(T.getLocation(),
Where)).second;
assert(InsertSuccess && "Token location already in rewriter!");
InsertSuccess = InsertSuccess;
+ return Where;
+}
+
+
+TokenRewriter::token_iterator
+TokenRewriter::AddTokenBefore(token_iterator I, const char *Val){
+ unsigned Len = strlen(Val);
+
+ // Plop the string into the scratch buffer, then create a token for this
+ // string.
+ Token Tok;
+ Tok.startToken();
+ Tok.setLocation(ScratchBuf->getToken(Val, Len));
+ Tok.setLength(Len);
+
+ // TODO: Form a whole lexer around this and relex the token! For now, just
+ // set kind to tok::unknown.
+ Tok.setKind(tok::unknown);
+
+ return AddToken(Tok, RemapIterator(I));
}
-
+
More information about the cfe-commits
mailing list