r356530 - Replace tok::angle_string_literal with new tok::header_name.

Richard Smith via cfe-commits cfe-commits at lists.llvm.org
Tue Mar 19 15:09:55 PDT 2019


Author: rsmith
Date: Tue Mar 19 15:09:55 2019
New Revision: 356530

URL: http://llvm.org/viewvc/llvm-project?rev=356530&view=rev
Log:
Replace tok::angle_string_literal with new tok::header_name.

Use the new kind for both angled header-name tokens and for
double-quoted header-name tokens.

This is in preparation for C++20's context-sensitive header-name token
formation rules.

Modified:
    cfe/trunk/include/clang/Basic/TokenKinds.def
    cfe/trunk/include/clang/Basic/TokenKinds.h
    cfe/trunk/include/clang/Lex/Preprocessor.h
    cfe/trunk/include/clang/Lex/PreprocessorLexer.h
    cfe/trunk/lib/Lex/Lexer.cpp
    cfe/trunk/lib/Lex/PPDirectives.cpp
    cfe/trunk/lib/Lex/PPMacroExpansion.cpp
    cfe/trunk/lib/Lex/Pragma.cpp
    cfe/trunk/lib/Lex/Preprocessor.cpp
    cfe/trunk/test/Preprocessor/_Pragma-dependency.c

Modified: cfe/trunk/include/clang/Basic/TokenKinds.def
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Basic/TokenKinds.def?rev=356530&r1=356529&r2=356530&view=diff
==============================================================================
--- cfe/trunk/include/clang/Basic/TokenKinds.def (original)
+++ cfe/trunk/include/clang/Basic/TokenKinds.def Tue Mar 19 15:09:55 2019
@@ -154,7 +154,9 @@ TOK(utf32_char_constant) // U'a'
 // C99 6.4.5: String Literals.
 TOK(string_literal)      // "foo"
 TOK(wide_string_literal) // L"foo"
-TOK(angle_string_literal)// <foo>
+
+// C11 6.4.7: Header Names
+TOK(header_name)         // <foo>, or "foo" lexed as a header-name
 
 // C++11 String Literals.
 TOK(utf8_string_literal) // u8"foo"

Modified: cfe/trunk/include/clang/Basic/TokenKinds.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Basic/TokenKinds.h?rev=356530&r1=356529&r2=356530&view=diff
==============================================================================
--- cfe/trunk/include/clang/Basic/TokenKinds.h (original)
+++ cfe/trunk/include/clang/Basic/TokenKinds.h Tue Mar 19 15:09:55 2019
@@ -86,7 +86,7 @@ inline bool isLiteral(TokenKind K) {
   return K == tok::numeric_constant || K == tok::char_constant ||
          K == tok::wide_char_constant || K == tok::utf8_char_constant ||
          K == tok::utf16_char_constant || K == tok::utf32_char_constant ||
-         isStringLiteral(K) || K == tok::angle_string_literal;
+         isStringLiteral(K) || K == tok::header_name;
 }
 
 /// Return true if this is any of tok::annot_* kinds.

Modified: cfe/trunk/include/clang/Lex/Preprocessor.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Lex/Preprocessor.h?rev=356530&r1=356529&r2=356530&view=diff
==============================================================================
--- cfe/trunk/include/clang/Lex/Preprocessor.h (original)
+++ cfe/trunk/include/clang/Lex/Preprocessor.h Tue Mar 19 15:09:55 2019
@@ -1274,7 +1274,7 @@ public:
   void Lex(Token &Result);
 
   /// Lex a token, forming a header-name token if possible.
-  bool LexHeaderName(Token &Result, bool AllowConcatenation = true);
+  bool LexHeaderName(Token &Result, bool AllowMacroExpansion = true);
 
   void LexAfterModuleImport(Token &Result);
 

Modified: cfe/trunk/include/clang/Lex/PreprocessorLexer.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Lex/PreprocessorLexer.h?rev=356530&r1=356529&r2=356530&view=diff
==============================================================================
--- cfe/trunk/include/clang/Lex/PreprocessorLexer.h (original)
+++ cfe/trunk/include/clang/Lex/PreprocessorLexer.h Tue Mar 19 15:09:55 2019
@@ -48,8 +48,7 @@ protected:
   /// True when parsing \#XXX; turns '\\n' into a tok::eod token.
   bool ParsingPreprocessorDirective = false;
 
-  /// True after \#include; turns \<xx> into a tok::angle_string_literal
-  /// token.
+  /// True after \#include; turns \<xx> or "xxx" into a tok::header_name token.
   bool ParsingFilename = false;
 
   /// True if in raw mode.

Modified: cfe/trunk/lib/Lex/Lexer.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/Lexer.cpp?rev=356530&r1=356529&r2=356530&view=diff
==============================================================================
--- cfe/trunk/lib/Lex/Lexer.cpp (original)
+++ cfe/trunk/lib/Lex/Lexer.cpp Tue Mar 19 15:09:55 2019
@@ -2072,7 +2072,7 @@ bool Lexer::LexAngledStringLiteral(Token
 
   // Update the location of token as well as BufferPtr.
   const char *TokStart = BufferPtr;
-  FormTokenWithChars(Result, CurPtr, tok::angle_string_literal);
+  FormTokenWithChars(Result, CurPtr, tok::header_name);
   Result.setLiteralData(TokStart);
   return true;
 }
@@ -3465,7 +3465,9 @@ LexNextToken:
   case '"':
     // Notify MIOpt that we read a non-whitespace/non-comment token.
     MIOpt.ReadToken();
-    return LexStringLiteral(Result, CurPtr, tok::string_literal);
+    return LexStringLiteral(Result, CurPtr,
+                            ParsingFilename ? tok::header_name
+                                            : tok::string_literal);
 
   // C99 6.4.6: Punctuators.
   case '?':

Modified: cfe/trunk/lib/Lex/PPDirectives.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/PPDirectives.cpp?rev=356530&r1=356529&r2=356530&view=diff
==============================================================================
--- cfe/trunk/lib/Lex/PPDirectives.cpp (original)
+++ cfe/trunk/lib/Lex/PPDirectives.cpp Tue Mar 19 15:09:55 2019
@@ -1446,6 +1446,14 @@ bool Preprocessor::GetIncludeFilenameSpe
   // Get the text form of the filename.
   assert(!Buffer.empty() && "Can't have tokens with empty spellings!");
 
+  // FIXME: Consider warning on some of the cases described in C11 6.4.7/3 and
+  // C++20 [lex.header]/2:
+  //
+  // If `"`, `'`, `\`, `/*`, or `//` appears in a header-name, then
+  //   in C: behavior is undefined
+  //   in C++: program is conditionally-supported with implementation-defined
+  //           semantics
+
   // Make sure the filename is <x> or "x".
   bool isAngled;
   if (Buffer[0] == '<') {
@@ -1613,7 +1621,7 @@ void Preprocessor::HandleIncludeDirectiv
   if (LexHeaderName(FilenameTok))
     return;
 
-  if (!FilenameTok.isOneOf(tok::angle_string_literal, tok::string_literal)) {
+  if (FilenameTok.isNot(tok::header_name)) {
     Diag(FilenameTok.getLocation(), diag::err_pp_expects_filename);
     if (FilenameTok.isNot(tok::eod))
       DiscardUntilEndOfDirective();

Modified: cfe/trunk/lib/Lex/PPMacroExpansion.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/PPMacroExpansion.cpp?rev=356530&r1=356529&r2=356530&view=diff
==============================================================================
--- cfe/trunk/lib/Lex/PPMacroExpansion.cpp (original)
+++ cfe/trunk/lib/Lex/PPMacroExpansion.cpp Tue Mar 19 15:09:55 2019
@@ -1166,7 +1166,7 @@ static bool EvaluateHasIncludeCommon(Tok
     PP.Diag(LParenLoc, diag::err_pp_expected_after) << II << tok::l_paren;
     // If the next token looks like a filename or the start of one,
     // assume it is and process it as such.
-    if (!Tok.is(tok::angle_string_literal) && !Tok.is(tok::string_literal))
+    if (Tok.isNot(tok::header_name))
       return false;
   } else {
     // Save '(' location for possible missing ')' message.
@@ -1175,7 +1175,7 @@ static bool EvaluateHasIncludeCommon(Tok
       return false;
   }
 
-  if (!Tok.isOneOf(tok::angle_string_literal, tok::string_literal)) {
+  if (Tok.isNot(tok::header_name)) {
     PP.Diag(Tok.getLocation(), diag::err_pp_expects_filename);
     return false;
   }

Modified: cfe/trunk/lib/Lex/Pragma.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/Pragma.cpp?rev=356530&r1=356529&r2=356530&view=diff
==============================================================================
--- cfe/trunk/lib/Lex/Pragma.cpp (original)
+++ cfe/trunk/lib/Lex/Pragma.cpp Tue Mar 19 15:09:55 2019
@@ -486,7 +486,7 @@ void Preprocessor::HandlePragmaDependenc
     return;
 
   // If the next token wasn't a header-name, diagnose the error.
-  if (!FilenameTok.isOneOf(tok::angle_string_literal, tok::string_literal)) {
+  if (FilenameTok.isNot(tok::header_name)) {
     Diag(FilenameTok.getLocation(), diag::err_pp_expects_filename);
     return;
   }
@@ -670,8 +670,7 @@ void Preprocessor::HandlePragmaIncludeAl
 
   StringRef SourceFileName;
   SmallString<128> FileNameBuffer;
-  if (SourceFilenameTok.is(tok::string_literal) ||
-      SourceFilenameTok.is(tok::angle_string_literal)) {
+  if (SourceFilenameTok.is(tok::header_name)) {
     SourceFileName = getSpelling(SourceFilenameTok, FileNameBuffer);
   } else {
     Diag(Tok, diag::warn_pragma_include_alias_expected_filename);
@@ -691,8 +690,7 @@ void Preprocessor::HandlePragmaIncludeAl
     return;
 
   StringRef ReplaceFileName;
-  if (ReplaceFilenameTok.is(tok::string_literal) ||
-      ReplaceFilenameTok.is(tok::angle_string_literal)) {
+  if (ReplaceFilenameTok.is(tok::header_name)) {
     ReplaceFileName = getSpelling(ReplaceFilenameTok, FileNameBuffer);
   } else {
     Diag(Tok, diag::warn_pragma_include_alias_expected_filename);

Modified: cfe/trunk/lib/Lex/Preprocessor.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/Preprocessor.cpp?rev=356530&r1=356529&r2=356530&view=diff
==============================================================================
--- cfe/trunk/lib/Lex/Preprocessor.cpp (original)
+++ cfe/trunk/lib/Lex/Preprocessor.cpp Tue Mar 19 15:09:55 2019
@@ -901,11 +901,12 @@ void Preprocessor::Lex(Token &Result) {
 /// \param FilenameTok Filled in with the next token. On success, this will
 ///        be either an angle_header_name or a string_literal token. On
 ///        failure, it will be whatever other token was found instead.
-/// \param AllowConcatenation If \c true, allow a < token, followed by other
-///        tokens and finally a > token, to form a single header-name token.
+/// \param AllowMacroExpansion If \c true, allow the header name to be formed
+///        by macro expansion (concatenating tokens as necessary if the first
+///        token is a '<').
 /// \return \c true if we reached EOD or EOF while looking for a > token in
 ///         a concatenated header name and diagnosed it. \c false otherwise.
-bool Preprocessor::LexHeaderName(Token &FilenameTok, bool AllowConcatenation) {
+bool Preprocessor::LexHeaderName(Token &FilenameTok, bool AllowMacroExpansion) {
   // Lex using header-name tokenization rules if tokens are being lexed from
   // a file. Just grab a token normally if we're in a macro expansion.
   if (CurPPLexer)
@@ -915,13 +916,16 @@ bool Preprocessor::LexHeaderName(Token &
 
   // This could be a <foo/bar.h> file coming from a macro expansion.  In this
   // case, glue the tokens together into an angle_string_literal token.
-  if (FilenameTok.is(tok::less) && AllowConcatenation) {
-    SmallString<128> FilenameBuffer;
+  SmallString<128> FilenameBuffer;
+  if (FilenameTok.is(tok::less) && AllowMacroExpansion) {
     SourceLocation Start = FilenameTok.getLocation();
     SourceLocation End;
     FilenameBuffer.push_back('<');
 
     // Consume tokens until we find a '>'.
+    // FIXME: A header-name could be formed starting or ending with an
+    // alternative token. It's not clear whether that's ill-formed in all
+    // cases.
     while (FilenameTok.isNot(tok::greater)) {
       Lex(FilenameTok);
       if (FilenameTok.isOneOf(tok::eod, tok::eof)) {
@@ -962,8 +966,22 @@ bool Preprocessor::LexHeaderName(Token &
     }
 
     FilenameTok.startToken();
-    FilenameTok.setKind(tok::angle_string_literal);
+    FilenameTok.setKind(tok::header_name);
     CreateString(FilenameBuffer, FilenameTok, Start, End);
+  } else if (FilenameTok.is(tok::string_literal) && AllowMacroExpansion) {
+    // Convert a string-literal token of the form " h-char-sequence "
+    // (produced by macro expansion) into a header-name token.
+    //
+    // The rules for header-names don't quite match the rules for
+    // string-literals, but all the places where they differ result in
+    // undefined behavior, so we can and do treat them the same.
+    //
+    // A string-literal with a prefix or suffix is not translated into a
+    // header-name. This could theoretically be observable via the C++20
+    // context-sensitive header-name formation rules.
+    StringRef Str = getSpelling(FilenameTok, FilenameBuffer);
+    if (Str.size() >= 2 && Str.front() == '"' && Str.back() == '"')
+      FilenameTok.setKind(tok::header_name);
   }
 
   return false;

Modified: cfe/trunk/test/Preprocessor/_Pragma-dependency.c
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/Preprocessor/_Pragma-dependency.c?rev=356530&r1=356529&r2=356530&view=diff
==============================================================================
--- cfe/trunk/test/Preprocessor/_Pragma-dependency.c (original)
+++ cfe/trunk/test/Preprocessor/_Pragma-dependency.c Tue Mar 19 15:09:55 2019
@@ -1,5 +1,11 @@
 // RUN: %clang_cc1 -E -verify %s
 
+#pragma GCC dependency "./_Pragma-dependency.c"
+
+#define self "./_Pragma-dependency.c"
+// expected-error at +1 {{expected "FILENAME" or <FILENAME>}}
+#pragma GCC dependency self
+
 #define DO_PRAGMA _Pragma 
 #define STR "GCC dependency \"parse.y\"")
 // expected-error at +1 {{'parse.y' file not found}}




More information about the cfe-commits mailing list