[cfe-commits] r119479 - in /cfe/trunk: include/clang/Lex/Lexer.h include/clang/Lex/Preprocessor.h lib/Lex/Lexer.cpp lib/Lex/LiteralSupport.cpp lib/Lex/Preprocessor.cpp
Chris Lattner
sabre at nondot.org
Tue Nov 16 23:26:21 PST 2010
Author: lattner
Date: Wed Nov 17 01:26:20 2010
New Revision: 119479
URL: http://llvm.org/viewvc/llvm-project?rev=119479&view=rev
Log:
move getSpelling from Preprocessor to Lexer, which it is more conceptually related to.
Modified:
cfe/trunk/include/clang/Lex/Lexer.h
cfe/trunk/include/clang/Lex/Preprocessor.h
cfe/trunk/lib/Lex/Lexer.cpp
cfe/trunk/lib/Lex/LiteralSupport.cpp
cfe/trunk/lib/Lex/Preprocessor.cpp
Modified: cfe/trunk/include/clang/Lex/Lexer.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Lex/Lexer.h?rev=119479&r1=119478&r2=119479&view=diff
==============================================================================
--- cfe/trunk/include/clang/Lex/Lexer.h (original)
+++ cfe/trunk/include/clang/Lex/Lexer.h Wed Nov 17 01:26:20 2010
@@ -211,6 +211,32 @@
/// and " characters. This does not add surrounding ""'s to the string.
static void Stringify(llvm::SmallVectorImpl<char> &Str);
+
+ /// getSpelling - This method is used to get the spelling of a token into a
+ /// preallocated buffer, instead of as an std::string. The caller is required
+ /// to allocate enough space for the token, which is guaranteed to be at least
+ /// Tok.getLength() bytes long. The length of the actual result is returned.
+ ///
+ /// Note that this method may do two possible things: it may either fill in
+ /// the buffer specified with characters, or it may *change the input pointer*
+ /// to point to a constant buffer with the data already in it (avoiding a
+ /// copy). The caller is not allowed to modify the returned buffer pointer
+ /// if an internal buffer is returned.
+ static unsigned getSpelling(const Token &Tok, const char *&Buffer,
+ const SourceManager &SourceMgr,
+ const LangOptions &Features,
+ bool *Invalid = 0);
+
+ /// getSpelling() - Return the 'spelling' of the Tok token. The spelling of a
+ /// token is the characters used to represent the token in the source file
+ /// after trigraph expansion and escaped-newline folding. In particular, this
+ /// wants to get the true, uncanonicalized, spelling of things like digraphs
+ /// UCNs, etc.
+ static std::string getSpelling(const Token &Tok,
+ const SourceManager &SourceMgr,
+ const LangOptions &Features,
+ bool *Invalid = 0);
+
/// MeasureTokenLength - Relex the token at the specified location and return
/// its length in bytes in the input file. If the token needs cleaning (e.g.
/// includes a trigraph or an escaped newline) then this count includes bytes
Modified: cfe/trunk/include/clang/Lex/Preprocessor.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Lex/Preprocessor.h?rev=119479&r1=119478&r2=119479&view=diff
==============================================================================
--- cfe/trunk/include/clang/Lex/Preprocessor.h (original)
+++ cfe/trunk/include/clang/Lex/Preprocessor.h Wed Nov 17 01:26:20 2010
@@ -641,17 +641,9 @@
/// UCNs, etc.
///
/// \param Invalid If non-NULL, will be set \c true if an error occurs.
- std::string getSpelling(const Token &Tok, bool *Invalid = 0) const;
-
- /// getSpelling() - Return the 'spelling' of the Tok token. The spelling of a
- /// token is the characters used to represent the token in the source file
- /// after trigraph expansion and escaped-newline folding. In particular, this
- /// wants to get the true, uncanonicalized, spelling of things like digraphs
- /// UCNs, etc.
- static std::string getSpelling(const Token &Tok,
- const SourceManager &SourceMgr,
- const LangOptions &Features,
- bool *Invalid = 0);
+ std::string getSpelling(const Token &Tok, bool *Invalid = 0) const {
+ return Lexer::getSpelling(Tok, SourceMgr, Features, Invalid);
+ }
/// getSpelling - This method is used to get the spelling of a token into a
/// preallocated buffer, instead of as an std::string. The caller is required
@@ -665,12 +657,8 @@
/// if an internal buffer is returned.
unsigned getSpelling(const Token &Tok, const char *&Buffer,
bool *Invalid = 0) const {
- return getSpelling(Tok, Buffer, SourceMgr, Features, Invalid);
+ return Lexer::getSpelling(Tok, Buffer, SourceMgr, Features, Invalid);
}
- static unsigned getSpelling(const Token &Tok, const char *&Buffer,
- const SourceManager &SourceMgr,
- const LangOptions &Features,
- bool *Invalid = 0);
/// getSpelling - This method is used to get the spelling of a token into a
/// SmallVector. Note that the returned StringRef may not point to the
Modified: cfe/trunk/lib/Lex/Lexer.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/Lexer.cpp?rev=119479&r1=119478&r2=119479&view=diff
==============================================================================
--- cfe/trunk/lib/Lex/Lexer.cpp (original)
+++ cfe/trunk/lib/Lex/Lexer.cpp Wed Nov 17 01:26:20 2010
@@ -212,6 +212,107 @@
}
}
+//===----------------------------------------------------------------------===//
+// Token Spelling
+//===----------------------------------------------------------------------===//
+
+/// getSpelling() - Return the 'spelling' of this token. The spelling of a
+/// token are the characters used to represent the token in the source file
+/// after trigraph expansion and escaped-newline folding. In particular, this
+/// wants to get the true, uncanonicalized, spelling of things like digraphs
+/// UCNs, etc.
+std::string Lexer::getSpelling(const Token &Tok, const SourceManager &SourceMgr,
+ const LangOptions &Features, bool *Invalid) {
+ assert((int)Tok.getLength() >= 0 && "Token character range is bogus!");
+
+ // If this token contains nothing interesting, return it directly.
+ bool CharDataInvalid = false;
+ const char* TokStart = SourceMgr.getCharacterData(Tok.getLocation(),
+ &CharDataInvalid);
+ if (Invalid)
+ *Invalid = CharDataInvalid;
+ if (CharDataInvalid)
+ return std::string();
+
+ if (!Tok.needsCleaning())
+ return std::string(TokStart, TokStart+Tok.getLength());
+
+ std::string Result;
+ Result.reserve(Tok.getLength());
+
+ // Otherwise, hard case, relex the characters into the string.
+ for (const char *Ptr = TokStart, *End = TokStart+Tok.getLength();
+ Ptr != End; ) {
+ unsigned CharSize;
+ Result.push_back(Lexer::getCharAndSizeNoWarn(Ptr, CharSize, Features));
+ Ptr += CharSize;
+ }
+ assert(Result.size() != unsigned(Tok.getLength()) &&
+ "NeedsCleaning flag set on something that didn't need cleaning!");
+ return Result;
+}
+
+/// getSpelling - This method is used to get the spelling of a token into a
+/// preallocated buffer, instead of as an std::string. The caller is required
+/// to allocate enough space for the token, which is guaranteed to be at least
+/// Tok.getLength() bytes long. The actual length of the token is returned.
+///
+/// Note that this method may do two possible things: it may either fill in
+/// the buffer specified with characters, or it may *change the input pointer*
+/// to point to a constant buffer with the data already in it (avoiding a
+/// copy). The caller is not allowed to modify the returned buffer pointer
+/// if an internal buffer is returned.
+unsigned Lexer::getSpelling(const Token &Tok, const char *&Buffer,
+ const SourceManager &SourceMgr,
+ const LangOptions &Features, bool *Invalid) {
+ assert((int)Tok.getLength() >= 0 && "Token character range is bogus!");
+
+ // If this token is an identifier, just return the string from the identifier
+ // table, which is very quick.
+ if (const IdentifierInfo *II = Tok.getIdentifierInfo()) {
+ Buffer = II->getNameStart();
+ return II->getLength();
+ }
+
+ // Otherwise, compute the start of the token in the input lexer buffer.
+ const char *TokStart = 0;
+
+ if (Tok.isLiteral())
+ TokStart = Tok.getLiteralData();
+
+ if (TokStart == 0) {
+ bool CharDataInvalid = false;
+ TokStart = SourceMgr.getCharacterData(Tok.getLocation(), &CharDataInvalid);
+ if (Invalid)
+ *Invalid = CharDataInvalid;
+ if (CharDataInvalid) {
+ Buffer = "";
+ return 0;
+ }
+ }
+
+ // If this token contains nothing interesting, return it directly.
+ if (!Tok.needsCleaning()) {
+ Buffer = TokStart;
+ return Tok.getLength();
+ }
+
+ // Otherwise, hard case, relex the characters into the string.
+ char *OutBuf = const_cast<char*>(Buffer);
+ for (const char *Ptr = TokStart, *End = TokStart+Tok.getLength();
+ Ptr != End; ) {
+ unsigned CharSize;
+ *OutBuf++ = Lexer::getCharAndSizeNoWarn(Ptr, CharSize, Features);
+ Ptr += CharSize;
+ }
+ assert(unsigned(OutBuf-Buffer) != Tok.getLength() &&
+ "NeedsCleaning flag set on something that didn't need cleaning!");
+
+ return OutBuf-Buffer;
+}
+
+
+
static bool isWhitespace(unsigned char c);
/// MeasureTokenLength - Relex the token at the specified location and return
Modified: cfe/trunk/lib/Lex/LiteralSupport.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/LiteralSupport.cpp?rev=119479&r1=119478&r2=119479&view=diff
==============================================================================
--- cfe/trunk/lib/Lex/LiteralSupport.cpp (original)
+++ cfe/trunk/lib/Lex/LiteralSupport.cpp Wed Nov 17 01:26:20 2010
@@ -898,8 +898,8 @@
// and 'spelled' tokens can only shrink.
bool StringInvalid = false;
unsigned ThisTokLen =
- Preprocessor::getSpelling(StringToks[i], ThisTokBuf, SM, Features,
- &StringInvalid);
+ Lexer::getSpelling(StringToks[i], ThisTokBuf, SM, Features,
+ &StringInvalid);
if (StringInvalid) {
hadError = 1;
continue;
@@ -1019,8 +1019,8 @@
bool StringInvalid = false;
const char *SpellingPtr = &SpellingBuffer[0];
- unsigned TokLen = Preprocessor::getSpelling(Tok, SpellingPtr, SM, Features,
- &StringInvalid);
+ unsigned TokLen = Lexer::getSpelling(Tok, SpellingPtr, SM, Features,
+ &StringInvalid);
if (StringInvalid)
return 0;
Modified: cfe/trunk/lib/Lex/Preprocessor.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/Preprocessor.cpp?rev=119479&r1=119478&r2=119479&view=diff
==============================================================================
--- cfe/trunk/lib/Lex/Preprocessor.cpp (original)
+++ cfe/trunk/lib/Lex/Preprocessor.cpp Wed Nov 17 01:26:20 2010
@@ -278,116 +278,6 @@
CodeComplete->CodeCompleteNaturalLanguage();
}
-//===----------------------------------------------------------------------===//
-// Token Spelling
-//===----------------------------------------------------------------------===//
-
-/// getSpelling() - Return the 'spelling' of this token. The spelling of a
-/// token are the characters used to represent the token in the source file
-/// after trigraph expansion and escaped-newline folding. In particular, this
-/// wants to get the true, uncanonicalized, spelling of things like digraphs
-/// UCNs, etc.
-std::string Preprocessor::getSpelling(const Token &Tok,
- const SourceManager &SourceMgr,
- const LangOptions &Features,
- bool *Invalid) {
- assert((int)Tok.getLength() >= 0 && "Token character range is bogus!");
-
- // If this token contains nothing interesting, return it directly.
- bool CharDataInvalid = false;
- const char* TokStart = SourceMgr.getCharacterData(Tok.getLocation(),
- &CharDataInvalid);
- if (Invalid)
- *Invalid = CharDataInvalid;
- if (CharDataInvalid)
- return std::string();
-
- if (!Tok.needsCleaning())
- return std::string(TokStart, TokStart+Tok.getLength());
-
- std::string Result;
- Result.reserve(Tok.getLength());
-
- // Otherwise, hard case, relex the characters into the string.
- for (const char *Ptr = TokStart, *End = TokStart+Tok.getLength();
- Ptr != End; ) {
- unsigned CharSize;
- Result.push_back(Lexer::getCharAndSizeNoWarn(Ptr, CharSize, Features));
- Ptr += CharSize;
- }
- assert(Result.size() != unsigned(Tok.getLength()) &&
- "NeedsCleaning flag set on something that didn't need cleaning!");
- return Result;
-}
-
-/// getSpelling() - Return the 'spelling' of this token. The spelling of a
-/// token are the characters used to represent the token in the source file
-/// after trigraph expansion and escaped-newline folding. In particular, this
-/// wants to get the true, uncanonicalized, spelling of things like digraphs
-/// UCNs, etc.
-std::string Preprocessor::getSpelling(const Token &Tok, bool *Invalid) const {
- return getSpelling(Tok, SourceMgr, Features, Invalid);
-}
-
-/// getSpelling - This method is used to get the spelling of a token into a
-/// preallocated buffer, instead of as an std::string. The caller is required
-/// to allocate enough space for the token, which is guaranteed to be at least
-/// Tok.getLength() bytes long. The actual length of the token is returned.
-///
-/// Note that this method may do two possible things: it may either fill in
-/// the buffer specified with characters, or it may *change the input pointer*
-/// to point to a constant buffer with the data already in it (avoiding a
-/// copy). The caller is not allowed to modify the returned buffer pointer
-/// if an internal buffer is returned.
-unsigned Preprocessor::getSpelling(const Token &Tok, const char *&Buffer,
- const SourceManager &SourceMgr,
- const LangOptions &Features,
- bool *Invalid) {
- assert((int)Tok.getLength() >= 0 && "Token character range is bogus!");
-
- // If this token is an identifier, just return the string from the identifier
- // table, which is very quick.
- if (const IdentifierInfo *II = Tok.getIdentifierInfo()) {
- Buffer = II->getNameStart();
- return II->getLength();
- }
-
- // Otherwise, compute the start of the token in the input lexer buffer.
- const char *TokStart = 0;
-
- if (Tok.isLiteral())
- TokStart = Tok.getLiteralData();
-
- if (TokStart == 0) {
- bool CharDataInvalid = false;
- TokStart = SourceMgr.getCharacterData(Tok.getLocation(), &CharDataInvalid);
- if (Invalid)
- *Invalid = CharDataInvalid;
- if (CharDataInvalid) {
- Buffer = "";
- return 0;
- }
- }
-
- // If this token contains nothing interesting, return it directly.
- if (!Tok.needsCleaning()) {
- Buffer = TokStart;
- return Tok.getLength();
- }
-
- // Otherwise, hard case, relex the characters into the string.
- char *OutBuf = const_cast<char*>(Buffer);
- for (const char *Ptr = TokStart, *End = TokStart+Tok.getLength();
- Ptr != End; ) {
- unsigned CharSize;
- *OutBuf++ = Lexer::getCharAndSizeNoWarn(Ptr, CharSize, Features);
- Ptr += CharSize;
- }
- assert(unsigned(OutBuf-Buffer) != Tok.getLength() &&
- "NeedsCleaning flag set on something that didn't need cleaning!");
-
- return OutBuf-Buffer;
-}
/// getSpelling - This method is used to get the spelling of a token into a
/// SmallVector. Note that the returned StringRef may not point to the
More information about the cfe-commits
mailing list