[cfe-commits] r66373 - in /cfe/trunk: include/clang/Lex/Lexer.h lib/Lex/Lexer.cpp lib/Lex/PPMacroExpansion.cpp lib/Lex/Pragma.cpp lib/Lex/ScratchBuffer.cpp lib/Lex/TokenLexer.cpp
Chris Lattner
sabre at nondot.org
Sun Mar 8 00:08:47 PST 2009
Author: lattner
Date: Sun Mar 8 03:08:45 2009
New Revision: 66373
URL: http://llvm.org/viewvc/llvm-project?rev=66373&view=rev
Log:
simplify some logic by making ScratchBuffer handle the application of trailing
\0's to created tokens instead of making all clients do it. No functionality
change.
Modified:
cfe/trunk/include/clang/Lex/Lexer.h
cfe/trunk/lib/Lex/Lexer.cpp
cfe/trunk/lib/Lex/PPMacroExpansion.cpp
cfe/trunk/lib/Lex/Pragma.cpp
cfe/trunk/lib/Lex/ScratchBuffer.cpp
cfe/trunk/lib/Lex/TokenLexer.cpp
Modified: cfe/trunk/include/clang/Lex/Lexer.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Lex/Lexer.h?rev=66373&r1=66372&r2=66373&view=diff
==============================================================================
--- cfe/trunk/include/clang/Lex/Lexer.h (original)
+++ cfe/trunk/include/clang/Lex/Lexer.h Sun Mar 8 03:08:45 2009
@@ -138,7 +138,7 @@
/// LexFromRawLexer - Lex a token from a designated raw lexer (one with no
/// associated preprocessor object. Return true if the 'next character to
- /// read' pointer points and the end of the lexer buffer, false otherwise.
+ /// read' pointer points at the end of the lexer buffer, false otherwise.
bool LexFromRawLexer(Token &Result) {
assert(LexingRawMode && "Not already in raw mode!");
Lex(Result);
Modified: cfe/trunk/lib/Lex/Lexer.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/Lexer.cpp?rev=66373&r1=66372&r2=66373&view=diff
==============================================================================
--- cfe/trunk/lib/Lex/Lexer.cpp (original)
+++ cfe/trunk/lib/Lex/Lexer.cpp Sun Mar 8 03:08:45 2009
@@ -167,6 +167,7 @@
L->BufferPtr = StrData;
L->BufferEnd = StrData+TokLen;
+ assert(L->BufferEnd[0] == 0 && "Buffer is not nul terminated!");
// Set the SourceLocation with the remapping information. This ensures that
// GetMappedTokenLoc will remap the tokens as they are lexed.
Modified: cfe/trunk/lib/Lex/PPMacroExpansion.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/PPMacroExpansion.cpp?rev=66373&r1=66372&r2=66373&view=diff
==============================================================================
--- cfe/trunk/lib/Lex/PPMacroExpansion.cpp (original)
+++ cfe/trunk/lib/Lex/PPMacroExpansion.cpp Sun Mar 8 03:08:45 2009
@@ -475,14 +475,10 @@
Loc = SourceMgr.getInstantiationRange(Loc).second;
PresumedLoc PLoc = SourceMgr.getPresumedLoc(Loc);
- // __LINE__ expands to a simple numeric value. Add a space after it so that
- // it will tokenize as a number (and not run into stuff after it in the temp
- // buffer).
- sprintf(TmpBuffer, "%u ", PLoc.getLine());
- unsigned Length = strlen(TmpBuffer)-1;
+ // __LINE__ expands to a simple numeric value.
+ sprintf(TmpBuffer, "%u", PLoc.getLine());
Tok.setKind(tok::numeric_constant);
- CreateString(TmpBuffer, Length+1, Tok, Tok.getLocation());
- Tok.setLength(Length); // Trim off space.
+ CreateString(TmpBuffer, strlen(TmpBuffer), Tok, Tok.getLocation());
} else if (II == Ident__FILE__ || II == Ident__BASE_FILE__) {
// C99 6.10.8: "__FILE__: The presumed name of the current source file (a
// character string literal)". This can be affected by #line.
@@ -532,14 +528,10 @@
for (; PLoc.isValid(); ++Depth)
PLoc = SourceMgr.getPresumedLoc(PLoc.getIncludeLoc());
- // __INCLUDE_LEVEL__ expands to a simple numeric value. Add a space after
- // it so that it will tokenize as a number (and not run into stuff after it
- // in the temp buffer).
- sprintf(TmpBuffer, "%u ", Depth);
- unsigned Length = strlen(TmpBuffer)-1;
+ // __INCLUDE_LEVEL__ expands to a simple numeric value.
+ sprintf(TmpBuffer, "%u", Depth);
Tok.setKind(tok::numeric_constant);
- CreateString(TmpBuffer, Length, Tok, Tok.getLocation());
- Tok.setLength(Length); // Trim off space.
+ CreateString(TmpBuffer, strlen(TmpBuffer), Tok, Tok.getLocation());
} else if (II == Ident__TIMESTAMP__) {
// MSVC, ICC, GCC, VisualAge C++ extension. The generated string should be
// of the form "Ddd Mmm dd hh::mm::ss yyyy", which is returned by asctime.
@@ -565,10 +557,9 @@
TmpBuffer[0] = '"';
strcpy(TmpBuffer+1, Result);
unsigned Len = strlen(TmpBuffer);
- TmpBuffer[Len-1] = '"'; // Replace the newline with a quote.
+ TmpBuffer[Len] = '"'; // Replace the newline with a quote.
Tok.setKind(tok::string_literal);
CreateString(TmpBuffer, Len+1, Tok, Tok.getLocation());
- Tok.setLength(Len); // Trim off space.
} else {
assert(0 && "Unknown identifier!");
}
Modified: cfe/trunk/lib/Lex/Pragma.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/Pragma.cpp?rev=66373&r1=66372&r2=66373&view=diff
==============================================================================
--- cfe/trunk/lib/Lex/Pragma.cpp (original)
+++ cfe/trunk/lib/Lex/Pragma.cpp Sun Mar 8 03:08:45 2009
@@ -141,9 +141,8 @@
// contents appear to have a space before them.
StrVal[0] = ' ';
- // Replace the terminating quote with a \n\0.
+ // Replace the terminating quote with a \n.
StrVal[StrVal.size()-1] = '\n';
- StrVal += '\0';
// Remove escaped quotes and escapes.
for (unsigned i = 0, e = StrVal.size(); i != e-1; ++i) {
@@ -165,8 +164,7 @@
// Make and enter a lexer object so that we lex and expand the tokens just
// like any others.
Lexer *TL = Lexer::Create_PragmaLexer(TokLoc, PragmaLoc, RParenLoc,
- // do not include the null in the count.
- StrVal.size()-1, *this);
+ StrVal.size(), *this);
EnterSourceFileWithLexer(TL, 0);
Modified: cfe/trunk/lib/Lex/ScratchBuffer.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/ScratchBuffer.cpp?rev=66373&r1=66372&r2=66373&view=diff
==============================================================================
--- cfe/trunk/lib/Lex/ScratchBuffer.cpp (original)
+++ cfe/trunk/lib/Lex/ScratchBuffer.cpp Sun Mar 8 03:08:45 2009
@@ -32,7 +32,7 @@
/// token.
SourceLocation ScratchBuffer::getToken(const char *Buf, unsigned Len,
const char *&DestPtr) {
- if (BytesUsed+Len > ScratchBufSize)
+ if (BytesUsed+Len+1 > ScratchBufSize)
AllocScratchBuffer(Len);
// Return a pointer to the character data.
@@ -42,16 +42,21 @@
memcpy(CurBuffer+BytesUsed, Buf, Len);
// Remember that we used these bytes.
- BytesUsed += Len;
+ BytesUsed += Len+1;
+
+ // Add a NUL terminator to the token. This keeps the tokens separated, in
+ // case they get relexed, and puts them on their own virtual lines in case a
+ // diagnostic points to one.
+ CurBuffer[BytesUsed-1] = '\0';
- return BufferStartLoc.getFileLocWithOffset(BytesUsed-Len);
+ return BufferStartLoc.getFileLocWithOffset(BytesUsed-Len-1);
}
void ScratchBuffer::AllocScratchBuffer(unsigned RequestLen) {
// Only pay attention to the requested length if it is larger than our default
// page size. If it is, we allocate an entire chunk for it. This is to
// support gigantic tokens, which almost certainly won't happen. :)
- if (RequestLen < ScratchBufSize)
+ if (RequestLen+1 < ScratchBufSize)
RequestLen = ScratchBufSize;
llvm::MemoryBuffer *Buf =
@@ -59,5 +64,6 @@
FileID FID = SourceMgr.createFileIDForMemBuffer(Buf);
BufferStartLoc = SourceMgr.getLocForStartOfFile(FID);
CurBuffer = const_cast<char*>(Buf->getBufferStart());
- BytesUsed = 0;
+ BytesUsed = 1;
+ CurBuffer[0] = '0'; // Start out with a \0 for cleanliness.
}
Modified: cfe/trunk/lib/Lex/TokenLexer.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/TokenLexer.cpp?rev=66373&r1=66372&r2=66373&view=diff
==============================================================================
--- cfe/trunk/lib/Lex/TokenLexer.cpp (original)
+++ cfe/trunk/lib/Lex/TokenLexer.cpp Sun Mar 8 03:08:45 2009
@@ -365,11 +365,9 @@
// Get the RHS token.
const Token &RHS = Tokens[CurToken];
- bool isInvalid = false;
-
// Allocate space for the result token. This is guaranteed to be enough for
- // the two tokens and a null terminator.
- Buffer.resize(Tok.getLength() + RHS.getLength() + 1);
+ // the two tokens.
+ Buffer.resize(Tok.getLength() + RHS.getLength());
// Get the spelling of the LHS token in Buffer.
const char *BufPtr = &Buffer[0];
@@ -382,11 +380,8 @@
if (BufPtr != &Buffer[LHSLen]) // Really, we want the chars in Buffer!
memcpy(&Buffer[LHSLen], BufPtr, RHSLen);
- // Add null terminator.
- Buffer[LHSLen+RHSLen] = '\0';
-
// Trim excess space.
- Buffer.resize(LHSLen+RHSLen+1);
+ Buffer.resize(LHSLen+RHSLen);
// Plop the pasted result (including the trailing newline and null) into a
// scratch buffer where we can lex it.
@@ -425,45 +420,43 @@
// Make a lexer object so that we lex and expand the paste result.
Lexer TL(SourceMgr.getLocForStartOfFile(LocFileID),
PP.getLangOptions(), ScratchBufStart,
- ResultTokStrPtr,
- ResultTokStrPtr+LHSLen+RHSLen /*don't include null*/);
+ ResultTokStrPtr, ResultTokStrPtr+LHSLen+RHSLen);
// Lex a token in raw mode. This way it won't look up identifiers
// automatically, lexing off the end will return an eof token, and
// warnings are disabled. This returns true if the result token is the
// entire buffer.
- bool IsComplete = TL.LexFromRawLexer(Result);
+ bool isInvalid = !TL.LexFromRawLexer(Result);
// If we got an EOF token, we didn't form even ONE token. For example, we
// did "/ ## /" to get "//".
- IsComplete &= Result.isNot(tok::eof);
- isInvalid = !IsComplete;
- }
+ isInvalid |= Result.is(tok::eof);
- // If pasting the two tokens didn't form a full new token, this is an error.
- // This occurs with "x ## +" and other stuff. Return with Tok unmodified
- // and with RHS as the next token to lex.
- if (isInvalid) {
- // Test for the Microsoft extension of /##/ turning into // here on the
- // error path.
- if (PP.getLangOptions().Microsoft && Tok.is(tok::slash) &&
- RHS.is(tok::slash)) {
- HandleMicrosoftCommentPaste(Tok);
- return true;
- }
+ // If pasting the two tokens didn't form a full new token, this is an
+ // error. This occurs with "x ## +" and other stuff. Return with Tok
+ // unmodified and with RHS as the next token to lex.
+ if (isInvalid) {
+ // Test for the Microsoft extension of /##/ turning into // here on the
+ // error path.
+ if (PP.getLangOptions().Microsoft && Tok.is(tok::slash) &&
+ RHS.is(tok::slash)) {
+ HandleMicrosoftCommentPaste(Tok);
+ return true;
+ }
- // TODO: If not in assembler language mode.
- PP.Diag(PasteOpLoc, diag::err_pp_bad_paste)
+ // TODO: If not in assembler language mode.
+ PP.Diag(PasteOpLoc, diag::err_pp_bad_paste)
<< std::string(Buffer.begin(), Buffer.end()-1);
- return false;
- }
-
- // Turn ## into 'unknown' to avoid # ## # from looking like a paste
- // operator.
- if (Result.is(tok::hashhash))
- Result.setKind(tok::unknown);
- // FIXME: Turn __VA_ARGS__ into "not a token"?
+ return false;
+ }
+ // Turn ## into 'unknown' to avoid # ## # from looking like a paste
+ // operator.
+ if (Result.is(tok::hashhash))
+ Result.setKind(tok::unknown);
+ // FIXME: Turn __VA_ARGS__ into "not a token"?
+ }
+
// Transfer properties of the LHS over the the Result.
Result.setFlagValue(Token::StartOfLine , Tok.isAtStartOfLine());
Result.setFlagValue(Token::LeadingSpace, Tok.hasLeadingSpace());
More information about the cfe-commits
mailing list