r356530 - Replace tok::angle_string_literal with new tok::header_name.
Richard Smith via cfe-commits
cfe-commits at lists.llvm.org
Tue Mar 19 15:09:55 PDT 2019
Author: rsmith
Date: Tue Mar 19 15:09:55 2019
New Revision: 356530
URL: http://llvm.org/viewvc/llvm-project?rev=356530&view=rev
Log:
Replace tok::angle_string_literal with new tok::header_name.
Use the new kind for both angled header-name tokens and for
double-quoted header-name tokens.
This is in preparation for C++20's context-sensitive header-name token
formation rules.
Modified:
cfe/trunk/include/clang/Basic/TokenKinds.def
cfe/trunk/include/clang/Basic/TokenKinds.h
cfe/trunk/include/clang/Lex/Preprocessor.h
cfe/trunk/include/clang/Lex/PreprocessorLexer.h
cfe/trunk/lib/Lex/Lexer.cpp
cfe/trunk/lib/Lex/PPDirectives.cpp
cfe/trunk/lib/Lex/PPMacroExpansion.cpp
cfe/trunk/lib/Lex/Pragma.cpp
cfe/trunk/lib/Lex/Preprocessor.cpp
cfe/trunk/test/Preprocessor/_Pragma-dependency.c
Modified: cfe/trunk/include/clang/Basic/TokenKinds.def
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Basic/TokenKinds.def?rev=356530&r1=356529&r2=356530&view=diff
==============================================================================
--- cfe/trunk/include/clang/Basic/TokenKinds.def (original)
+++ cfe/trunk/include/clang/Basic/TokenKinds.def Tue Mar 19 15:09:55 2019
@@ -154,7 +154,9 @@ TOK(utf32_char_constant) // U'a'
// C99 6.4.5: String Literals.
TOK(string_literal) // "foo"
TOK(wide_string_literal) // L"foo"
-TOK(angle_string_literal)// <foo>
+
+// C11 6.4.7: Header Names
+TOK(header_name) // <foo>, or "foo" lexed as a header-name
// C++11 String Literals.
TOK(utf8_string_literal) // u8"foo"
Modified: cfe/trunk/include/clang/Basic/TokenKinds.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Basic/TokenKinds.h?rev=356530&r1=356529&r2=356530&view=diff
==============================================================================
--- cfe/trunk/include/clang/Basic/TokenKinds.h (original)
+++ cfe/trunk/include/clang/Basic/TokenKinds.h Tue Mar 19 15:09:55 2019
@@ -86,7 +86,7 @@ inline bool isLiteral(TokenKind K) {
return K == tok::numeric_constant || K == tok::char_constant ||
K == tok::wide_char_constant || K == tok::utf8_char_constant ||
K == tok::utf16_char_constant || K == tok::utf32_char_constant ||
- isStringLiteral(K) || K == tok::angle_string_literal;
+ isStringLiteral(K) || K == tok::header_name;
}
/// Return true if this is any of tok::annot_* kinds.
Modified: cfe/trunk/include/clang/Lex/Preprocessor.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Lex/Preprocessor.h?rev=356530&r1=356529&r2=356530&view=diff
==============================================================================
--- cfe/trunk/include/clang/Lex/Preprocessor.h (original)
+++ cfe/trunk/include/clang/Lex/Preprocessor.h Tue Mar 19 15:09:55 2019
@@ -1274,7 +1274,7 @@ public:
void Lex(Token &Result);
/// Lex a token, forming a header-name token if possible.
- bool LexHeaderName(Token &Result, bool AllowConcatenation = true);
+ bool LexHeaderName(Token &Result, bool AllowMacroExpansion = true);
void LexAfterModuleImport(Token &Result);
Modified: cfe/trunk/include/clang/Lex/PreprocessorLexer.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Lex/PreprocessorLexer.h?rev=356530&r1=356529&r2=356530&view=diff
==============================================================================
--- cfe/trunk/include/clang/Lex/PreprocessorLexer.h (original)
+++ cfe/trunk/include/clang/Lex/PreprocessorLexer.h Tue Mar 19 15:09:55 2019
@@ -48,8 +48,7 @@ protected:
/// True when parsing \#XXX; turns '\\n' into a tok::eod token.
bool ParsingPreprocessorDirective = false;
- /// True after \#include; turns \<xx> into a tok::angle_string_literal
- /// token.
+ /// True after \#include; turns \<xx> or "xxx" into a tok::header_name token.
bool ParsingFilename = false;
/// True if in raw mode.
Modified: cfe/trunk/lib/Lex/Lexer.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/Lexer.cpp?rev=356530&r1=356529&r2=356530&view=diff
==============================================================================
--- cfe/trunk/lib/Lex/Lexer.cpp (original)
+++ cfe/trunk/lib/Lex/Lexer.cpp Tue Mar 19 15:09:55 2019
@@ -2072,7 +2072,7 @@ bool Lexer::LexAngledStringLiteral(Token
// Update the location of token as well as BufferPtr.
const char *TokStart = BufferPtr;
- FormTokenWithChars(Result, CurPtr, tok::angle_string_literal);
+ FormTokenWithChars(Result, CurPtr, tok::header_name);
Result.setLiteralData(TokStart);
return true;
}
@@ -3465,7 +3465,9 @@ LexNextToken:
case '"':
// Notify MIOpt that we read a non-whitespace/non-comment token.
MIOpt.ReadToken();
- return LexStringLiteral(Result, CurPtr, tok::string_literal);
+ return LexStringLiteral(Result, CurPtr,
+ ParsingFilename ? tok::header_name
+ : tok::string_literal);
// C99 6.4.6: Punctuators.
case '?':
Modified: cfe/trunk/lib/Lex/PPDirectives.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/PPDirectives.cpp?rev=356530&r1=356529&r2=356530&view=diff
==============================================================================
--- cfe/trunk/lib/Lex/PPDirectives.cpp (original)
+++ cfe/trunk/lib/Lex/PPDirectives.cpp Tue Mar 19 15:09:55 2019
@@ -1446,6 +1446,14 @@ bool Preprocessor::GetIncludeFilenameSpe
// Get the text form of the filename.
assert(!Buffer.empty() && "Can't have tokens with empty spellings!");
+ // FIXME: Consider warning on some of the cases described in C11 6.4.7/3 and
+ // C++20 [lex.header]/2:
+ //
+ // If `"`, `'`, `\`, `/*`, or `//` appears in a header-name, then
+ // in C: behavior is undefined
+ // in C++: program is conditionally-supported with implementation-defined
+ // semantics
+
// Make sure the filename is <x> or "x".
bool isAngled;
if (Buffer[0] == '<') {
@@ -1613,7 +1621,7 @@ void Preprocessor::HandleIncludeDirectiv
if (LexHeaderName(FilenameTok))
return;
- if (!FilenameTok.isOneOf(tok::angle_string_literal, tok::string_literal)) {
+ if (FilenameTok.isNot(tok::header_name)) {
Diag(FilenameTok.getLocation(), diag::err_pp_expects_filename);
if (FilenameTok.isNot(tok::eod))
DiscardUntilEndOfDirective();
Modified: cfe/trunk/lib/Lex/PPMacroExpansion.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/PPMacroExpansion.cpp?rev=356530&r1=356529&r2=356530&view=diff
==============================================================================
--- cfe/trunk/lib/Lex/PPMacroExpansion.cpp (original)
+++ cfe/trunk/lib/Lex/PPMacroExpansion.cpp Tue Mar 19 15:09:55 2019
@@ -1166,7 +1166,7 @@ static bool EvaluateHasIncludeCommon(Tok
PP.Diag(LParenLoc, diag::err_pp_expected_after) << II << tok::l_paren;
// If the next token looks like a filename or the start of one,
// assume it is and process it as such.
- if (!Tok.is(tok::angle_string_literal) && !Tok.is(tok::string_literal))
+ if (Tok.isNot(tok::header_name))
return false;
} else {
// Save '(' location for possible missing ')' message.
@@ -1175,7 +1175,7 @@ static bool EvaluateHasIncludeCommon(Tok
return false;
}
- if (!Tok.isOneOf(tok::angle_string_literal, tok::string_literal)) {
+ if (Tok.isNot(tok::header_name)) {
PP.Diag(Tok.getLocation(), diag::err_pp_expects_filename);
return false;
}
Modified: cfe/trunk/lib/Lex/Pragma.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/Pragma.cpp?rev=356530&r1=356529&r2=356530&view=diff
==============================================================================
--- cfe/trunk/lib/Lex/Pragma.cpp (original)
+++ cfe/trunk/lib/Lex/Pragma.cpp Tue Mar 19 15:09:55 2019
@@ -486,7 +486,7 @@ void Preprocessor::HandlePragmaDependenc
return;
// If the next token wasn't a header-name, diagnose the error.
- if (!FilenameTok.isOneOf(tok::angle_string_literal, tok::string_literal)) {
+ if (FilenameTok.isNot(tok::header_name)) {
Diag(FilenameTok.getLocation(), diag::err_pp_expects_filename);
return;
}
@@ -670,8 +670,7 @@ void Preprocessor::HandlePragmaIncludeAl
StringRef SourceFileName;
SmallString<128> FileNameBuffer;
- if (SourceFilenameTok.is(tok::string_literal) ||
- SourceFilenameTok.is(tok::angle_string_literal)) {
+ if (SourceFilenameTok.is(tok::header_name)) {
SourceFileName = getSpelling(SourceFilenameTok, FileNameBuffer);
} else {
Diag(Tok, diag::warn_pragma_include_alias_expected_filename);
@@ -691,8 +690,7 @@ void Preprocessor::HandlePragmaIncludeAl
return;
StringRef ReplaceFileName;
- if (ReplaceFilenameTok.is(tok::string_literal) ||
- ReplaceFilenameTok.is(tok::angle_string_literal)) {
+ if (ReplaceFilenameTok.is(tok::header_name)) {
ReplaceFileName = getSpelling(ReplaceFilenameTok, FileNameBuffer);
} else {
Diag(Tok, diag::warn_pragma_include_alias_expected_filename);
Modified: cfe/trunk/lib/Lex/Preprocessor.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/Preprocessor.cpp?rev=356530&r1=356529&r2=356530&view=diff
==============================================================================
--- cfe/trunk/lib/Lex/Preprocessor.cpp (original)
+++ cfe/trunk/lib/Lex/Preprocessor.cpp Tue Mar 19 15:09:55 2019
@@ -901,11 +901,12 @@ void Preprocessor::Lex(Token &Result) {
/// \param FilenameTok Filled in with the next token. On success, this will
/// be either an angle_header_name or a string_literal token. On
/// failure, it will be whatever other token was found instead.
-/// \param AllowConcatenation If \c true, allow a < token, followed by other
-/// tokens and finally a > token, to form a single header-name token.
+/// \param AllowMacroExpansion If \c true, allow the header name to be formed
+/// by macro expansion (concatenating tokens as necessary if the first
+/// token is a '<').
/// \return \c true if we reached EOD or EOF while looking for a > token in
/// a concatenated header name and diagnosed it. \c false otherwise.
-bool Preprocessor::LexHeaderName(Token &FilenameTok, bool AllowConcatenation) {
+bool Preprocessor::LexHeaderName(Token &FilenameTok, bool AllowMacroExpansion) {
// Lex using header-name tokenization rules if tokens are being lexed from
// a file. Just grab a token normally if we're in a macro expansion.
if (CurPPLexer)
@@ -915,13 +916,16 @@ bool Preprocessor::LexHeaderName(Token &
// This could be a <foo/bar.h> file coming from a macro expansion. In this
// case, glue the tokens together into an angle_string_literal token.
- if (FilenameTok.is(tok::less) && AllowConcatenation) {
- SmallString<128> FilenameBuffer;
+ SmallString<128> FilenameBuffer;
+ if (FilenameTok.is(tok::less) && AllowMacroExpansion) {
SourceLocation Start = FilenameTok.getLocation();
SourceLocation End;
FilenameBuffer.push_back('<');
// Consume tokens until we find a '>'.
+ // FIXME: A header-name could be formed starting or ending with an
+ // alternative token. It's not clear whether that's ill-formed in all
+ // cases.
while (FilenameTok.isNot(tok::greater)) {
Lex(FilenameTok);
if (FilenameTok.isOneOf(tok::eod, tok::eof)) {
@@ -962,8 +966,22 @@ bool Preprocessor::LexHeaderName(Token &
}
FilenameTok.startToken();
- FilenameTok.setKind(tok::angle_string_literal);
+ FilenameTok.setKind(tok::header_name);
CreateString(FilenameBuffer, FilenameTok, Start, End);
+ } else if (FilenameTok.is(tok::string_literal) && AllowMacroExpansion) {
+ // Convert a string-literal token of the form " h-char-sequence "
+ // (produced by macro expansion) into a header-name token.
+ //
+ // The rules for header-names don't quite match the rules for
+ // string-literals, but all the places where they differ result in
+ // undefined behavior, so we can and do treat them the same.
+ //
+ // A string-literal with a prefix or suffix is not translated into a
+ // header-name. This could theoretically be observable via the C++20
+ // context-sensitive header-name formation rules.
+ StringRef Str = getSpelling(FilenameTok, FilenameBuffer);
+ if (Str.size() >= 2 && Str.front() == '"' && Str.back() == '"')
+ FilenameTok.setKind(tok::header_name);
}
return false;
Modified: cfe/trunk/test/Preprocessor/_Pragma-dependency.c
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/Preprocessor/_Pragma-dependency.c?rev=356530&r1=356529&r2=356530&view=diff
==============================================================================
--- cfe/trunk/test/Preprocessor/_Pragma-dependency.c (original)
+++ cfe/trunk/test/Preprocessor/_Pragma-dependency.c Tue Mar 19 15:09:55 2019
@@ -1,5 +1,11 @@
// RUN: %clang_cc1 -E -verify %s
+#pragma GCC dependency "./_Pragma-dependency.c"
+
+#define self "./_Pragma-dependency.c"
+// expected-error at +1 {{expected "FILENAME" or <FILENAME>}}
+#pragma GCC dependency self
+
#define DO_PRAGMA _Pragma
#define STR "GCC dependency \"parse.y\"")
// expected-error at +1 {{'parse.y' file not found}}
More information about the cfe-commits
mailing list