[libcxx-commits] [clang] [libcxx] F literals (PR #221273)
Bengt Gustafsson via libcxx-commits
libcxx-commits at lists.llvm.org
Fri Sep 4 09:16:55 PDT 2026
https://github.com/BengtGustafsson created https://github.com/llvm/llvm-project/pull/221273
None
>From 9b3c16e73e6dce0c5d43af04eaff95344e8814de Mon Sep 17 00:00:00 2001
From: "bengt.gustafsson" <bengt.gustafsson at beamways.com>
Date: Sun, 9 Jun 2024 17:15:52 +0200
Subject: [PATCH 01/17] Implemented this in the lexer directly., This required
making a few methods in Preprocessor public
This works except with nested f-literals in f-literals. This does not come out right as the token order gets jumbled because the inner MacroInfo object gets pushed first, instead of being pushed as part of evaluation of the outer macro object.
Another problem with this simple approach as that it doesn't handle concatenation of literals correctly, i.e. it doesn't collect all argument expressions after the last of a sequence of string literals, a subset of which may be of (the same) prefix kind.
---
clang/include/clang/Lex/Lexer.h | 8 +
clang/include/clang/Lex/Preprocessor.h | 5 +
clang/lib/Lex/Lexer.cpp | 318 ++++++++++++++++++++++++-
3 files changed, 330 insertions(+), 1 deletion(-)
diff --git a/clang/include/clang/Lex/Lexer.h b/clang/include/clang/Lex/Lexer.h
index b6ecc7e5ded9e..f62ff4bbe8b95 100644
--- a/clang/include/clang/Lex/Lexer.h
+++ b/clang/include/clang/Lex/Lexer.h
@@ -745,6 +745,14 @@ class Lexer : public PreprocessorLexer {
tok::TokenKind Kind);
bool LexRawStringLiteral (Token &Result, const char *CurPtr,
tok::TokenKind Kind);
+ void processExtractionField(std::vector<Token> &tokens, const char *litStart,
+ std::string &lit);
+ Token processExpression (std::vector<Token> &tokens);
+ Token processNested (std::vector<Token> &tokens);
+ void processNestedParenthesis(std::vector<Token> &tokens, const Token &token,
+ tok::TokenKind closer);
+ bool LexFLiteral(Token &Result, const char *CurPtr,
+ tok::TokenKind Kind);
bool LexAngledStringLiteral(Token &Result, const char *CurPtr);
bool LexCharConstant (Token &Result, const char *CurPtr,
tok::TokenKind Kind);
diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h
index c0850a8fa9f7f..099fae9666f08 100644
--- a/clang/include/clang/Lex/Preprocessor.h
+++ b/clang/include/clang/Lex/Preprocessor.h
@@ -2464,8 +2464,11 @@ class Preprocessor {
/// macro name.
void updateModuleMacroInfo(const IdentifierInfo *II, ModuleMacroInfo &Info);
+ public:
DefMacroDirective *AllocateDefMacroDirective(MacroInfo *MI,
SourceLocation Loc);
+ private:
+
UndefMacroDirective *AllocateUndefMacroDirective(SourceLocation UndefLoc);
VisibilityMacroDirective *AllocateVisibilityMacroDirective(SourceLocation Loc,
bool isPublic);
@@ -2564,8 +2567,10 @@ class Preprocessor {
/// If an identifier token is read that is to be expanded as a macro, handle
/// it and return the next token as 'Tok'. If we lexed a token, return true;
/// otherwise the caller should lex again.
+public:
bool HandleMacroExpandedIdentifier(Token &Identifier, const MacroDefinition &MD);
+private:
/// Cache macro expanded tokens for TokenLexers.
//
/// Works like a stack; a TokenLexer adds the macro expanded tokens that is
diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp
index c7543a48c0b50..c534d3c7e592b 100644
--- a/clang/lib/Lex/Lexer.cpp
+++ b/clang/lib/Lex/Lexer.cpp
@@ -2338,6 +2338,307 @@ bool Lexer::LexRawStringLiteral(Token &Result, const char *CurPtr,
return true;
}
+
+// Process and extraction field in a f-literal. Add the tokens of the expresion(s) to tokens, and add the literal fragments to lit,
+// if any. Start at BufferPtr, and leave it after the field, i.e. after the } of the extraction field. litstart is the position after the
+// starting " of the literal or the } of the previous extraction field.
+void clang::Lexer::processExtractionField(std::vector<Token>& tokens, const char* litStart, std::string& lit) {
+ // Add the comma that should go before each extracted expression.
+ Token cTok;
+ cTok.startToken();
+ cTok.setKind(tok::comma);
+ cTok.setLocation(getSourceLocation());
+ tokens.push_back(cTok);
+
+ const char* exprStart = BufferPtr;
+ Token terminator = processExpression(tokens); // Parse the part before the colon or }, returning the : or } token itself.
+
+ // Handle the extra feature that an expression ending with = is its own label (as in Python). No expression ending with =
+ // is valid anyway.
+ if (tokens.back().getKind() == tok::equal) {
+ lit.append(litStart, BufferPtr); // Add entire expression including the = and any spaces after that to the literal.
+ tokens.pop_back(); // Remove the =
+ }
+ else {
+ lit.append(litStart, exprStart); // Add entire expression including the = and any spaces after that to the literal.
+ }
+
+ if (terminator.getKind() != tok::colon) {
+ assert(terminator.getKind() == tok::r_brace);
+ lit.push_back('}');
+ return;
+ }
+
+ lit.push_back(':');
+
+ // If : check for nested fields in the format-spec. This works character by character.
+ auto toLit = [&] {
+ lit += *BufferPtr++;
+ };
+
+ while (*BufferPtr != '}') {
+ if (*BufferPtr == '{') { // Nested field starts
+ toLit();
+
+ // Add a comma in the token stream before the expression tokens.
+ cTok.setLocation(getSourceLocation());
+ tokens.push_back(cTok);
+ Token terminator = processExpression(tokens);
+ if (terminator.getKind() != tok::r_brace) // Colon not allowed inside nested expression-field.
+ Diag(BufferPtr, diag::ext_unterminated_char_or_string) << 1; // TODO: new diagnostic: formatting spec for dynamic format spec not allowed.
+
+ lit.push_back('}'); // The } of the dynamic format
+ }
+ else
+ toLit(); // Transfer other formatting argument char to the resulting string.
+ }
+ toLit(); // Transfer the final } and advance PufferPtr.
+}
+
+Token clang::Lexer::processExpression(std::vector<Token>& tokens)
+{
+ while (true) {
+ Token token = processNested(tokens);
+ switch (token.getKind()) {
+ case tok::r_paren:
+ Diag(BufferPtr, diag::ext_unterminated_char_or_string) << 1; // TODO: new diagnostic: "Extraneous ) in expression-field"
+ break;
+
+ case tok::r_square:
+ Diag(BufferPtr, diag::ext_unterminated_char_or_string) << 1; // TODO: new diagnostic: "Extraneous ] in expression-field"
+ break;
+
+ case tok::question:
+ tokens.push_back(token);
+ Token terminator = processExpression(tokens);
+ if (terminator.getKind() != tok::colon)
+ Diag(BufferPtr, diag::ext_unterminated_char_or_string) << 1; // TODO: new diagnostic: "Mismatched ? in expression-field"
+
+ tokens.push_back(terminator);
+ return processExpression(tokens);
+
+ case tok::r_brace:
+ case tok::colon:
+ return token;
+
+ case tok::coloncolon: {
+ // If a coloncolon token is not followed by an identifier it is to
+ // be interpreted as the colon starting the format spec anyway.
+ const char *save =
+ BufferPtr - 1; // Points to the second of colons.
+ Token ident;
+ while (!Lex(ident))
+ ;
+
+ if (ident.getKind() != tok::identifier) {
+ BufferPtr = save;
+
+ Token cTok;
+ cTok.startToken();
+ cTok.setKind(tok::colon);
+ return cTok;
+ }
+ tokens.push_back(token);
+ tokens.push_back(ident);
+ break;
+ }
+
+ default:
+ tokens.push_back(token);
+ }
+ }
+}
+
+// Pass over any number of matched parentheses, pushing all the tokens thus passed over.
+// Return the first token outside any matched parentheses.
+Token clang::Lexer::processNested(std::vector<Token>& tokens)
+{
+ while (true) {
+ Token token;
+ while(!Lex(token))
+ ;
+ switch (token.getKind()) {
+ case tok::l_paren:
+ processNestedParenthesis(tokens, token, tok::r_paren);
+ break;
+ case tok::l_square:
+ processNestedParenthesis(tokens, token, tok::r_square);
+ break;
+ case tok::l_brace:
+ processNestedParenthesis(tokens, token, tok::r_brace);
+ break;
+
+ default:
+ return token;
+ }
+ }
+}
+
+// Pass over a certain type of parenthesis, indicated by the expected closer token kind.
+void clang::Lexer::processNestedParenthesis(std::vector<Token>& tokens, const Token &token, tok::TokenKind closer) {
+ tokens.push_back(token);
+ while (true) {
+ Token next = processNested(tokens);
+ tokens.push_back(next);
+ if (next.getKind() == closer)
+ return;
+
+ if (next.getKind() == tok::r_paren || next.getKind() == tok::r_square || next.getKind() == tok::r_brace)
+ Diag(BufferPtr, diag::ext_unterminated_char_or_string) << 1; // TODO: new diagnostic: "Mismatched {}. A {} was found where a {} was expected.", introducers[intIx], c, terminators[intIx]));
+ }
+}
+
+
+bool clang::Lexer::LexFLiteral(Token &Result, const char *CurPtr, tok::TokenKind Kind) {
+ const char *AfterQuote = CurPtr;
+ // Does this string contain the \0 character?
+ const char *NulCharacter = nullptr;
+
+ if (PP == nullptr)
+ return LexStringLiteral(Result, CurPtr, Kind);
+
+ if (!isLexingRawMode() &&
+ (Kind == tok::utf8_string_literal || Kind == tok::utf16_string_literal ||
+ Kind == tok::utf32_string_literal))
+ Diag(BufferPtr, LangOpts.CPlusPlus ? diag::warn_cxx98_compat_unicode_literal
+ : diag::warn_c99_compat_unicode_literal);
+
+ // As the string literal is no longer the same as in the source code we must save it into this variable
+ // until the right " is found. Then we create a token around it, with its source location set to the start of the original literal
+ // and push this into the MacroInfo after the initial std::format( sequence. To handle the life time of the string contents
+ // we use the BP allocator function.
+ std::string resultingLiteral = "\"";
+
+ // Collect all tokens in this vector until we know how many we have, i.e. at the end.
+ std::vector<Token> tokens;
+
+ // Form the prefix tokens.
+
+ Token ccTok;
+ ccTok.startToken();
+ ccTok.setKind(tok::coloncolon);
+ ccTok.setLocation(getSourceLocation());
+ tokens.push_back(ccTok);
+
+ Token stdTok;
+ stdTok.startToken();
+ stdTok.setLocation(getSourceLocation());
+ stdTok.setIdentifierInfo(PP->getIdentifierInfo("std"));
+ stdTok.setKind(tok::identifier);
+ tokens.push_back(stdTok);
+
+ tokens.push_back(ccTok);
+
+ Token fmtTok;
+ fmtTok.startToken();
+ fmtTok.setLocation(getSourceLocation());
+ fmtTok.setIdentifierInfo(PP->getIdentifierInfo("format"));
+ fmtTok.setKind(tok::identifier);
+ tokens.push_back(fmtTok);
+
+ Token lpTok;
+ lpTok.startToken();
+ lpTok.setLocation(getSourceLocation());
+ lpTok.setKind(tok::l_paren);
+ tokens.push_back(lpTok);
+
+ const size_t literalPos = tokens.size(); // 4
+
+ tokens.push_back(Token()); // Placeholder for the literal.
+
+ const char* litStart = CurPtr;
+ const char *litPartStart = CurPtr;
+ char C = getAndAdvanceChar(CurPtr, Result);
+
+ while (C != '"') {
+ // Skip escaped characters. Escaped newlines will already be processed by
+ // getAndAdvanceChar.
+ if (C == '\\')
+ C = getAndAdvanceChar(CurPtr, Result);
+
+ if (C == '\n' || C == '\r' || // Newline.
+ (C == 0 && CurPtr - 1 == BufferEnd)) { // End of file.
+ if (!isLexingRawMode() && !LangOpts.AsmPreprocessor)
+ Diag(BufferPtr, diag::ext_unterminated_char_or_string) << 1;
+ FormTokenWithChars(Result, CurPtr - 1, tok::unknown);
+ return true;
+ }
+
+ // Embyrotic implementation which just removes everything between { and }
+ // Note: {{ a quoted brace, We need to keep both as the resulting literal is still input to std::format!
+ if (C == '{' && *CurPtr != '{') {
+ BufferPtr = CurPtr;
+ processExtractionField(tokens, litPartStart, resultingLiteral);
+ CurPtr = BufferPtr;
+ litPartStart = CurPtr;
+ }
+
+ if (C == 0) {
+ if (isCodeCompletionPoint(CurPtr - 1)) {
+ if (ParsingFilename)
+ codeCompleteIncludedFile(AfterQuote, CurPtr - 1, /*IsAngled=*/false);
+ else
+ PP->CodeCompleteNaturalLanguage();
+ FormTokenWithChars(Result, CurPtr - 1, tok::unknown);
+ cutOffLexing();
+ return true;
+ }
+
+ NulCharacter = CurPtr - 1;
+ }
+ C = getAndAdvanceChar(CurPtr, Result);
+ }
+
+ // f literals can't be combined with ud suffixes, so don't parse such.
+
+ // If a nul character existed in the string, warn about it.
+ if (NulCharacter && !isLexingRawMode())
+ Diag(NulCharacter, diag::null_in_char_or_string) << 1;
+
+ // If we are in C++11, lex the optional ud-suffix. This is probably not useful for f literals as stD::format would not accept the result of the customer defined operator, which is
+ // no longer a string literal.
+ if (LangOpts.CPlusPlus)
+ CurPtr = LexUDSuffix(Result, CurPtr, true);
+
+ // COmplete the resulting literal by adding the bit after the last }
+ resultingLiteral.append(litPartStart, CurPtr);
+
+ // Fill in the literalPos token with the remaining string. But to keep its contents first allocate
+ // using the bumpAllocator from PP.
+ char *lit = new (PP->getPreprocessorAllocator()) char[resultingLiteral.size() + 1];
+ memcpy(lit, resultingLiteral.c_str(), resultingLiteral.size() + 1);
+
+ Token rpTok;
+ rpTok.startToken();
+ rpTok.setKind(tok::r_paren);
+ rpTok.setLocation(getSourceLocation(CurPtr));
+ rpTok.setLength(1);
+ tokens.push_back(rpTok);
+
+ SourceLocation litLoc = getSourceLocation(litStart, CurPtr - BufferPtr);
+ Token &litTok = tokens[literalPos];
+ litTok.startToken();
+ litTok.setKind(Kind);
+ litTok.setLocation(getSourceLocation(litStart));
+ litTok.setLiteralData(lit);
+ litTok.setKind(Kind);
+ litTok.setLength(resultingLiteral.size());
+
+ // The completed lexing is represented by a MacroInfo object which is pushed
+ // onto the macro expansion stack This expansion consists of std::format("...
+ // the remaining string ...", ...the extracted argument tokens...)
+
+ MacroInfo *mi = PP->AllocateMacroInfo(litLoc);
+ mi->setTokens(tokens, PP->getPreprocessorAllocator());
+
+ DefMacroDirective *defMD = PP->AllocateDefMacroDirective(mi, litLoc);
+
+ MacroDefinition md(defMD, {}, false);
+ BufferPtr = CurPtr;
+ return PP->HandleMacroExpandedIdentifier(tokens[3], md); // Token here must be an identifier. tokens[3] is "format".
+}
+
+
/// LexAngledStringLiteral - Lex the remainder of an angled string literal,
/// after having lexed the '<' character. This is used for #include filenames.
bool Lexer::LexAngledStringLiteral(Token &Result, const char *CurPtr) {
@@ -3961,6 +4262,21 @@ bool Lexer::LexTokenInternal(Token &Result, bool TokAtPhysicalStartOfLine) {
// treat R like the start of an identifier.
return LexIdentifierContinue(Result, CurPtr);
+ case 'f':
+ // Notify MIOpt that we read a non-whitespace/non-comment token.
+ MIOpt.ReadToken();
+
+ if (LangOpts.CPlusPlus11) {
+ Char = getCharAndSize(CurPtr, SizeTmp);
+
+ if (Char == '"')
+ // LexFLiteral pushes a MacroInfo object containing everything except the std of the std::format( prefix which is set in Result instead.
+ return LexFLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result), tok::string_literal);
+ }
+
+ // treat R like the start of an identifier.
+ return LexIdentifierContinue(Result, CurPtr);
+
case 'L': // Identifier (Loony) or wide literal (L'x' or L"xyz").
// Notify MIOpt that we read a non-whitespace/non-comment token.
MIOpt.ReadToken();
@@ -3991,7 +4307,7 @@ bool Lexer::LexTokenInternal(Token &Result, bool TokAtPhysicalStartOfLine) {
case 'H': case 'I': case 'J': case 'K': /*'L'*/case 'M': case 'N':
case 'O': case 'P': case 'Q': /*'R'*/case 'S': case 'T': /*'U'*/
case 'V': case 'W': case 'X': case 'Y': case 'Z':
- case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g':
+ case 'a': case 'b': case 'c': case 'd': case 'e': case 'g':
case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n':
case 'o': case 'p': case 'q': case 'r': case 's': case 't': /*'u'*/
case 'v': case 'w': case 'x': case 'y': case 'z':
>From c3f9d3d73f9d77982d6210830ffbdc783f37b237 Mon Sep 17 00:00:00 2001
From: "bengt.gustafsson" <bengt.gustafsson at beamways.com>
Date: Sun, 9 Jun 2024 23:28:26 +0200
Subject: [PATCH 02/17] Handle comma between ? and : on top level to match
assignment-expression.
Debug the debug feature.
---
clang/include/clang/Lex/Lexer.h | 2 +-
clang/lib/Lex/Lexer.cpp | 55 ++++++++++++++++++---------------
2 files changed, 31 insertions(+), 26 deletions(-)
diff --git a/clang/include/clang/Lex/Lexer.h b/clang/include/clang/Lex/Lexer.h
index f62ff4bbe8b95..c5b21e77f85c2 100644
--- a/clang/include/clang/Lex/Lexer.h
+++ b/clang/include/clang/Lex/Lexer.h
@@ -747,7 +747,7 @@ class Lexer : public PreprocessorLexer {
tok::TokenKind Kind);
void processExtractionField(std::vector<Token> &tokens, const char *litStart,
std::string &lit);
- Token processExpression (std::vector<Token> &tokens);
+ Token processExpression (std::vector<Token> &tokens, bool allowComma);
Token processNested (std::vector<Token> &tokens);
void processNestedParenthesis(std::vector<Token> &tokens, const Token &token,
tok::TokenKind closer);
diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp
index c534d3c7e592b..4e75c150b2f50 100644
--- a/clang/lib/Lex/Lexer.cpp
+++ b/clang/lib/Lex/Lexer.cpp
@@ -2351,12 +2351,14 @@ void clang::Lexer::processExtractionField(std::vector<Token>& tokens, const char
tokens.push_back(cTok);
const char* exprStart = BufferPtr;
- Token terminator = processExpression(tokens); // Parse the part before the colon or }, returning the : or } token itself.
+ Token terminator = processExpression(tokens, false); // Parse the part before the colon or }, returning the : or } token itself.
// Handle the extra feature that an expression ending with = is its own label (as in Python). No expression ending with =
// is valid anyway.
if (tokens.back().getKind() == tok::equal) {
- lit.append(litStart, BufferPtr); // Add entire expression including the = and any spaces after that to the literal.
+ lit.append(litStart, exprStart - 1); // Exclude the { which is to come after the label
+ lit.append(exprStart, BufferPtr - 1); // Add entire expression including the = and any spaces after that to the literal.
+ lit += "{"; // Add the { after the = and any trailing spaces.
tokens.pop_back(); // Remove the =
}
else {
@@ -2383,7 +2385,7 @@ void clang::Lexer::processExtractionField(std::vector<Token>& tokens, const char
// Add a comma in the token stream before the expression tokens.
cTok.setLocation(getSourceLocation());
tokens.push_back(cTok);
- Token terminator = processExpression(tokens);
+ Token terminator = processExpression(tokens, false);
if (terminator.getKind() != tok::r_brace) // Colon not allowed inside nested expression-field.
Diag(BufferPtr, diag::ext_unterminated_char_or_string) << 1; // TODO: new diagnostic: formatting spec for dynamic format spec not allowed.
@@ -2395,7 +2397,7 @@ void clang::Lexer::processExtractionField(std::vector<Token>& tokens, const char
toLit(); // Transfer the final } and advance PufferPtr.
}
-Token clang::Lexer::processExpression(std::vector<Token>& tokens)
+Token clang::Lexer::processExpression(std::vector<Token>& tokens, bool allowComma)
{
while (true) {
Token token = processNested(tokens);
@@ -2410,38 +2412,41 @@ Token clang::Lexer::processExpression(std::vector<Token>& tokens)
case tok::question:
tokens.push_back(token);
- Token terminator = processExpression(tokens);
+ Token terminator = processExpression(tokens, true); // Note: Comma allowed before : by the C++ grammar
if (terminator.getKind() != tok::colon)
Diag(BufferPtr, diag::ext_unterminated_char_or_string) << 1; // TODO: new diagnostic: "Mismatched ? in expression-field"
tokens.push_back(terminator);
- return processExpression(tokens);
+ return processExpression(tokens, false);
case tok::r_brace:
case tok::colon:
return token;
case tok::coloncolon: {
- // If a coloncolon token is not followed by an identifier it is to
- // be interpreted as the colon starting the format spec anyway.
- const char *save =
- BufferPtr - 1; // Points to the second of colons.
- Token ident;
- while (!Lex(ident))
- ;
-
- if (ident.getKind() != tok::identifier) {
- BufferPtr = save;
-
- Token cTok;
- cTok.startToken();
- cTok.setKind(tok::colon);
- return cTok;
- }
- tokens.push_back(token);
- tokens.push_back(ident);
- break;
+ // If a coloncolon token is not followed by an identifier it is to
+ // be interpreted as the colon starting the format spec anyway.
+ const char *save = BufferPtr - 1; // Points to the second of colons.
+ Token ident;
+ while (!Lex(ident))
+ ;
+
+ if (ident.getKind() != tok::identifier) {
+ BufferPtr = save;
+
+ Token cTok;
+ cTok.startToken();
+ cTok.setKind(tok::colon);
+ return cTok;
+ }
+ tokens.push_back(token);
+ tokens.push_back(ident);
+ break;
}
+ case tok::comma:
+ if (!allowComma)
+ // Important to diagnose erroneous comma here as std::format allows extra expressions, which is ridiculous for f-literals as they can't be translated.
+ Diag(BufferPtr, diag::ext_unterminated_char_or_string) << 1; // TODO: new diagnostic: "comma in extraction-field. Comman is not allowed in assignment-expression."
default:
tokens.push_back(token);
>From 9fe85034a15165be49b238e4368e194c326caa4e Mon Sep 17 00:00:00 2001
From: "bengt.gustafsson" <bengt.gustafsson at beamways.com>
Date: Thu, 13 Jun 2024 23:02:13 +0200
Subject: [PATCH 03/17] Use "EnterTokenStream" instead of a fake macro to
inject the resulting tokens after processing a f-literal.
Use PP->Lex instead of local Lex to make nested f-literals work. This could maybe spell trouble when trying to implement literal concatenation, but I don't really think so.
---
clang/lib/Lex/Lexer.cpp | 20 +++++++++-----------
1 file changed, 9 insertions(+), 11 deletions(-)
diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp
index 4e75c150b2f50..988e736e3d647 100644
--- a/clang/lib/Lex/Lexer.cpp
+++ b/clang/lib/Lex/Lexer.cpp
@@ -2428,8 +2428,7 @@ Token clang::Lexer::processExpression(std::vector<Token>& tokens, bool allowComm
// be interpreted as the colon starting the format spec anyway.
const char *save = BufferPtr - 1; // Points to the second of colons.
Token ident;
- while (!Lex(ident))
- ;
+ PP->Lex(ident);
if (ident.getKind() != tok::identifier) {
BufferPtr = save;
@@ -2460,8 +2459,8 @@ Token clang::Lexer::processNested(std::vector<Token>& tokens)
{
while (true) {
Token token;
- while(!Lex(token))
- ;
+ PP->Lex(token);
+
switch (token.getKind()) {
case tok::l_paren:
processNestedParenthesis(tokens, token, tok::r_paren);
@@ -2632,15 +2631,14 @@ bool clang::Lexer::LexFLiteral(Token &Result, const char *CurPtr, tok::TokenKind
// The completed lexing is represented by a MacroInfo object which is pushed
// onto the macro expansion stack This expansion consists of std::format("...
// the remaining string ...", ...the extracted argument tokens...)
-
- MacroInfo *mi = PP->AllocateMacroInfo(litLoc);
- mi->setTokens(tokens, PP->getPreprocessorAllocator());
-
- DefMacroDirective *defMD = PP->AllocateDefMacroDirective(mi, litLoc);
- MacroDefinition md(defMD, {}, false);
BufferPtr = CurPtr;
- return PP->HandleMacroExpandedIdentifier(tokens[3], md); // Token here must be an identifier. tokens[3] is "format".
+
+ auto tokStore = std::make_unique<Token[]>(tokens.size());
+ std::copy(tokens.begin(), tokens.end(), tokStore.get());
+
+ PP->EnterTokenStream(std::move(tokStore), tokens.size(), true, false);
+ return false; // Result has not been filled in.
}
>From f2b236a2695a6e5d70b0a748b4662cb463618d00 Mon Sep 17 00:00:00 2001
From: "bengt.gustafsson" <bengt.gustafsson at beamways.com>
Date: Wed, 4 Sep 2024 00:11:55 +0200
Subject: [PATCH 04/17] Error handling which almoar works, except that an empty
string should be forwarded to parsing at errors. Some errors also allow
continued parsing of the f-literal for more fields, which is dubious if the
previous field is not correct.
Added a flag to the lexer to not emit error messages in (nested) string literals if unterminated, allowing interpreting this as a missing } before the trailing " of the outer string literal.
---
.../include/clang/Basic/DiagnosticLexKinds.td | 7 +
clang/include/clang/Lex/Lexer.h | 11 +-
clang/lib/Lex/Lexer.cpp | 295 +++++++++++-------
3 files changed, 204 insertions(+), 109 deletions(-)
diff --git a/clang/include/clang/Basic/DiagnosticLexKinds.td b/clang/include/clang/Basic/DiagnosticLexKinds.td
index 25fbfe83fa2bc..85d7c54ebf5df 100644
--- a/clang/include/clang/Basic/DiagnosticLexKinds.td
+++ b/clang/include/clang/Basic/DiagnosticLexKinds.td
@@ -87,6 +87,13 @@ def warn_c23_keyword : Warning<"'%0' is a keyword in C23">,
def ext_unterminated_char_or_string : ExtWarn<
"missing terminating %select{'|'\"'}0 character">, InGroup<InvalidPPToken>;
+
+def err_nested_format_spec : Error<"format specifier in nested expression field">;
+def err_unmatched_paren : Error<"unmatched %0 in expression field">;
+def err_incomplete_ternary : Error<"ternary ? operator without matching : in expression field">;
+def err_comma_in_expression_field : Error<"comma not allowed in expression field">;
+def err_unescaped_r_brace : Error<"unquoted } in format string">;
+
def ext_empty_character : ExtWarn<"empty character constant">,
InGroup<InvalidPPToken>;
def err_unterminated_block_comment : Error<"unterminated /* comment">;
diff --git a/clang/include/clang/Lex/Lexer.h b/clang/include/clang/Lex/Lexer.h
index c5b21e77f85c2..0fa8ce89306fb 100644
--- a/clang/include/clang/Lex/Lexer.h
+++ b/clang/include/clang/Lex/Lexer.h
@@ -143,6 +143,11 @@ class Lexer : public PreprocessorLexer {
/// True if this is the first time we're lexing the input file.
bool IsFirstTimeLexingFile;
+ /// Don't print errors finding unterminated (non-raw) literal. Instead restore BufferPtr to
+ /// the first " found. This is set when lexing a nested expression as the unmatched " means that there was a syntax error
+ /// in the expression which is the error to be reported. In this case Lex must return an unknown Token.
+ bool IgnoreUnterminatedLiterals = false;
+
// NewLinePtr - A pointer to new line character '\n' being lexed. For '\r\n',
// it also points to '\n.'
const char *NewLinePtr;
@@ -745,11 +750,11 @@ class Lexer : public PreprocessorLexer {
tok::TokenKind Kind);
bool LexRawStringLiteral (Token &Result, const char *CurPtr,
tok::TokenKind Kind);
- void processExtractionField(std::vector<Token> &tokens, const char *litStart,
+ bool processExtractionField(std::vector<Token> &tokens, const char *litStart,
std::string &lit);
Token processExpression (std::vector<Token> &tokens, bool allowComma);
- Token processNested (std::vector<Token> &tokens);
- void processNestedParenthesis(std::vector<Token> &tokens, const Token &token,
+ int processNested (std::vector<Token> &tokens, Token& token);
+ int processNestedParenthesis(std::vector<Token> &tokens, const Token &token,
tok::TokenKind closer);
bool LexFLiteral(Token &Result, const char *CurPtr,
tok::TokenKind Kind);
diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp
index 988e736e3d647..5e1778991bf7f 100644
--- a/clang/lib/Lex/Lexer.cpp
+++ b/clang/lib/Lex/Lexer.cpp
@@ -2210,6 +2210,9 @@ bool Lexer::LexStringLiteral(Token &Result, const char *CurPtr,
if (C == '\n' || C == '\r' || // Newline.
(C == 0 && CurPtr-1 == BufferEnd)) { // End of file.
+ if (IgnoreUnterminatedLiterals)
+ return true; // Result still unknown. This is detected by extraction field processing.
+
if (!isLexingRawMode() && !LangOpts.AsmPreprocessor)
Diag(BufferPtr, diag::ext_unterminated_char_or_string) << 1;
FormTokenWithChars(Result, CurPtr-1, tok::unknown);
@@ -2342,7 +2345,7 @@ bool Lexer::LexRawStringLiteral(Token &Result, const char *CurPtr,
// Process and extraction field in a f-literal. Add the tokens of the expresion(s) to tokens, and add the literal fragments to lit,
// if any. Start at BufferPtr, and leave it after the field, i.e. after the } of the extraction field. litstart is the position after the
// starting " of the literal or the } of the previous extraction field.
-void clang::Lexer::processExtractionField(std::vector<Token>& tokens, const char* litStart, std::string& lit) {
+bool clang::Lexer::processExtractionField(std::vector<Token>& tokens, const char* litStart, std::string& lit) {
// Add the comma that should go before each extracted expression.
Token cTok;
cTok.startToken();
@@ -2354,8 +2357,10 @@ void clang::Lexer::processExtractionField(std::vector<Token>& tokens, const char
Token terminator = processExpression(tokens, false); // Parse the part before the colon or }, returning the : or } token itself.
// Handle the extra feature that an expression ending with = is its own label (as in Python). No expression ending with =
- // is valid anyway.
- if (tokens.back().getKind() == tok::equal) {
+ // is valid anyway. Well, this is not strictly true: &MyClass::operator= is a valid expression but for the greater good we ignore this.
+ // If anyone wants to emit this operator's address they can use a parenthesis. (Or we could specify that 'opewrator=' does not c�use
+ // label output.
+ if (tokens.back().getKind() == tok::equal && (tokens.size() < 2 || tokens[tokens.size() - 2].getKind() != tok::kw_operator)) {
lit.append(litStart, exprStart - 1); // Exclude the { which is to come after the label
lit.append(exprStart, BufferPtr - 1); // Add entire expression including the = and any spaces after that to the literal.
lit += "{"; // Add the { after the = and any trailing spaces.
@@ -2365,10 +2370,10 @@ void clang::Lexer::processExtractionField(std::vector<Token>& tokens, const char
lit.append(litStart, exprStart); // Add entire expression including the = and any spaces after that to the literal.
}
- if (terminator.getKind() != tok::colon) {
- assert(terminator.getKind() == tok::r_brace);
+ if (terminator.getKind() != tok::colon) { // This includes unknown, i.e. a " before }
+ assert(terminator.getKind() == tok::r_brace || terminator.getKind() == tok::unknown);
lit.push_back('}');
- return;
+ return terminator.getKind() != tok::unknown;
}
lit.push_back(':');
@@ -2379,121 +2384,176 @@ void clang::Lexer::processExtractionField(std::vector<Token>& tokens, const char
};
while (*BufferPtr != '}') {
- if (*BufferPtr == '{') { // Nested field starts
- toLit();
-
- // Add a comma in the token stream before the expression tokens.
- cTok.setLocation(getSourceLocation());
- tokens.push_back(cTok);
- Token terminator = processExpression(tokens, false);
- if (terminator.getKind() != tok::r_brace) // Colon not allowed inside nested expression-field.
- Diag(BufferPtr, diag::ext_unterminated_char_or_string) << 1; // TODO: new diagnostic: formatting spec for dynamic format spec not allowed.
-
- lit.push_back('}'); // The } of the dynamic format
- }
- else
- toLit(); // Transfer other formatting argument char to the resulting string.
+ if (*BufferPtr == '{') { // Nested field starts
+ toLit();
+ if (*BufferPtr != '{') {
+ // Add a comma in the token stream before the expression tokens.
+ cTok.setLocation(getSourceLocation());
+ tokens.push_back(cTok);
+ Token terminator = processExpression(tokens, false);
+ if (terminator.getKind() == tok::unknown)
+ return false;
+
+ if (terminator.getKind() != tok::r_brace) {
+ // Colon not allowed inside nested expression-field.
+ Diag(BufferPtr, diag::err_nested_format_spec);
+ return false;
+ }
+
+ lit.push_back('}'); // The } after the nested expression field
+ }
+ }
+ else if (*BufferPtr == '}') {
+ toLit();
+ if (*BufferPtr != '}') {
+ Diag(BufferPtr, diag::err_unescaped_r_brace);
+ return false;
+ }
+ }
+ else
+ toLit(); // Transfer other formatting argument char to the resulting
+ // string.
}
- toLit(); // Transfer the final } and advance PufferPtr.
+ toLit(); // Transfer the final } and advance PufferPtr.
+ return true;
}
Token clang::Lexer::processExpression(std::vector<Token>& tokens, bool allowComma)
{
- while (true) {
- Token token = processNested(tokens);
- switch (token.getKind()) {
- case tok::r_paren:
- Diag(BufferPtr, diag::ext_unterminated_char_or_string) << 1; // TODO: new diagnostic: "Extraneous ) in expression-field"
- break;
-
- case tok::r_square:
- Diag(BufferPtr, diag::ext_unterminated_char_or_string) << 1; // TODO: new diagnostic: "Extraneous ] in expression-field"
- break;
-
- case tok::question:
- tokens.push_back(token);
- Token terminator = processExpression(tokens, true); // Note: Comma allowed before : by the C++ grammar
- if (terminator.getKind() != tok::colon)
- Diag(BufferPtr, diag::ext_unterminated_char_or_string) << 1; // TODO: new diagnostic: "Mismatched ? in expression-field"
-
- tokens.push_back(terminator);
- return processExpression(tokens, false);
-
- case tok::r_brace:
- case tok::colon:
- return token;
-
- case tok::coloncolon: {
- // If a coloncolon token is not followed by an identifier it is to
- // be interpreted as the colon starting the format spec anyway.
- const char *save = BufferPtr - 1; // Points to the second of colons.
- Token ident;
- PP->Lex(ident);
-
- if (ident.getKind() != tok::identifier) {
- BufferPtr = save;
-
- Token cTok;
- cTok.startToken();
- cTok.setKind(tok::colon);
- return cTok;
- }
- tokens.push_back(token);
- tokens.push_back(ident);
- break;
- }
- case tok::comma:
- if (!allowComma)
- // Important to diagnose erroneous comma here as std::format allows extra expressions, which is ridiculous for f-literals as they can't be translated.
- Diag(BufferPtr, diag::ext_unterminated_char_or_string) << 1; // TODO: new diagnostic: "comma in extraction-field. Comman is not allowed in assignment-expression."
+ const char *lpos = BufferPtr - 1;
+ while (true) {
+ Token token;
+ int res = processNested(tokens, token);
+ if (res != 0)
+ return Token();
+
+ switch (token.getKind()) {
+ case tok::unknown:
+ Diag(lpos, diag::err_unmatched_paren) << "{";
+ return token;
+ break;
- default:
- tokens.push_back(token);
- }
+ case tok::r_paren:
+ Diag(BufferPtr - 1, diag::err_unmatched_paren) << ")";
+ break;
+
+ case tok::r_square:
+ Diag(BufferPtr - 1, diag::err_unmatched_paren) << "]";
+ break;
+
+ case tok::question: {
+ tokens.push_back(token);
+ const char *qpos = BufferPtr - 1;
+ Token terminator = processExpression(
+ tokens,
+ true); // Note: Comma allowed between ? and : by the C++ grammar
+ if (terminator.getKind() == tok::unknown)
+ return terminator;
+
+ if (terminator.getKind() != tok::colon) {
+ Diag(qpos, diag::err_incomplete_ternary);
+ return terminator; // This must be }
+ }
+
+ tokens.push_back(terminator);
+ return processExpression(tokens, false);
+ }
+ case tok::r_brace:
+ case tok::colon:
+ return token;
+
+ case tok::coloncolon: {
+ // If a coloncolon token is not followed by an identifier it is to
+ // be interpreted as the colon starting the format spec anyway.
+ const char *save = BufferPtr - 1; // Points to the second of colons.
+ Token ident;
+ PP->Lex(ident);
+
+ if (ident.getKind() != tok::identifier) {
+ BufferPtr = save;
+
+ Token cTok;
+ cTok.startToken();
+ cTok.setKind(tok::colon);
+ return cTok;
+ }
+ tokens.push_back(token);
+ tokens.push_back(ident);
+ break;
+ }
+ case tok::comma:
+ if (!allowComma) {
+ // Important to diagnose erroneous comma here as std::format allows
+ // extra expressions, which is ridiculous for f-literals as they
+ // can't be translated.
+ Diag(BufferPtr - 1, diag::err_comma_in_expression_field);
+ return Token();
+ }
+ [[fallthrough]];
+ default:
+ tokens.push_back(token);
}
+ }
}
// Pass over any number of matched parentheses, pushing all the tokens thus passed over.
// Return the first token outside any matched parentheses.
-Token clang::Lexer::processNested(std::vector<Token>& tokens)
+int clang::Lexer::processNested(std::vector<Token>& tokens, Token& token)
{
while (true) {
- Token token;
PP->Lex(token);
- switch (token.getKind()) {
- case tok::l_paren:
- processNestedParenthesis(tokens, token, tok::r_paren);
- break;
- case tok::l_square:
- processNestedParenthesis(tokens, token, tok::r_square);
- break;
- case tok::l_brace:
- processNestedParenthesis(tokens, token, tok::r_brace);
- break;
-
- default:
- return token;
- }
+ int res = 0;
+ switch (token.getKind()) {
+ case tok::l_paren:
+ res = processNestedParenthesis(tokens, token, tok::r_paren);
+ break;
+ case tok::l_square:
+ res = processNestedParenthesis(tokens, token, tok::r_square);
+ break;
+ case tok::l_brace:
+ res = processNestedParenthesis(tokens, token, tok::r_brace);
+ break;
+
+ default:
+ return 0;
+ }
+
+ if (res != 0)
+ return res;
}
}
// Pass over a certain type of parenthesis, indicated by the expected closer token kind.
-void clang::Lexer::processNestedParenthesis(std::vector<Token>& tokens, const Token &token, tok::TokenKind closer) {
+int clang::Lexer::processNestedParenthesis(std::vector<Token>& tokens, const Token &token, tok::TokenKind closer) {
tokens.push_back(token);
+ const char *pos = BufferPtr - 1;
while (true) {
- Token next = processNested(tokens);
- tokens.push_back(next);
- if (next.getKind() == closer)
- return;
+ Token next;
+ int res = processNested(tokens, next);
+ if (res != 0)
+ return res;
+
+ if (next.getKind() == tok::unknown) {
+ Diag(pos, diag::err_unmatched_paren) << StringRef(pos, 1);
+ return 1; // Premature " before any matching r-parens seen.
+ }
+
+ tokens.push_back(next);
+ if (next.getKind() == closer)
+ return 0;
- if (next.getKind() == tok::r_paren || next.getKind() == tok::r_square || next.getKind() == tok::r_brace)
- Diag(BufferPtr, diag::ext_unterminated_char_or_string) << 1; // TODO: new diagnostic: "Mismatched {}. A {} was found where a {} was expected.", introducers[intIx], c, terminators[intIx]));
+ if (next.getKind() == tok::r_paren || next.getKind() == tok::r_square ||
+ next.getKind() == tok::r_brace) {
+ Diag(pos, diag::err_unmatched_paren) << StringRef(pos, 1);
+ return 2;
+ }
}
}
bool clang::Lexer::LexFLiteral(Token &Result, const char *CurPtr, tok::TokenKind Kind) {
+ const char *litStart = BufferPtr; // Before all prefixes
const char *AfterQuote = CurPtr;
// Does this string contain the \0 character?
const char *NulCharacter = nullptr;
@@ -2550,7 +2610,6 @@ bool clang::Lexer::LexFLiteral(Token &Result, const char *CurPtr, tok::TokenKind
tokens.push_back(Token()); // Placeholder for the literal.
- const char* litStart = CurPtr;
const char *litPartStart = CurPtr;
char C = getAndAdvanceChar(CurPtr, Result);
@@ -2562,22 +2621,46 @@ bool clang::Lexer::LexFLiteral(Token &Result, const char *CurPtr, tok::TokenKind
if (C == '\n' || C == '\r' || // Newline.
(C == 0 && CurPtr - 1 == BufferEnd)) { // End of file.
+ if (IgnoreUnterminatedLiterals)
+ return true; // Result still unknown. This is detected by extraction
+ // field processing in enclosing extraction field.
+
if (!isLexingRawMode() && !LangOpts.AsmPreprocessor)
Diag(BufferPtr, diag::ext_unterminated_char_or_string) << 1;
FormTokenWithChars(Result, CurPtr - 1, tok::unknown);
return true;
}
- // Embyrotic implementation which just removes everything between { and }
// Note: {{ a quoted brace, We need to keep both as the resulting literal is still input to std::format!
- if (C == '{' && *CurPtr != '{') {
- BufferPtr = CurPtr;
- processExtractionField(tokens, litPartStart, resultingLiteral);
- CurPtr = BufferPtr;
- litPartStart = CurPtr;
+ if (C == '{') {
+ if (*CurPtr != '{') {
+ BufferPtr = CurPtr;
+ // Note: If we get a
+ bool save = IgnoreUnterminatedLiterals;
+ IgnoreUnterminatedLiterals = true;
+ bool ok =
+ processExtractionField(tokens, litPartStart, resultingLiteral);
+ IgnoreUnterminatedLiterals = save;
+ if (!ok) {
+ BufferPtr = litStart + 1; // Pass the f: TODO: somehow pass any L, u, U or u8 but not the f to LexStringLiteral. Or just create a fake token, we're already at the trailing " in the input anyway.
+ return LexStringLiteral(
+ Result, AfterQuote,
+ Kind); // After reporting errors here just pass a regular string
+ // literal along. This is to avoid errors later.
+ }
+ CurPtr = BufferPtr;
+ litPartStart = CurPtr;
+ }
+ else
+ CurPtr++;
}
-
- if (C == 0) {
+ else if (C == '}') {
+ if (*CurPtr != '}')
+ Diag(CurPtr - 1, diag::err_unescaped_r_brace);
+ else
+ CurPtr++; // Pass the second }.
+ }
+ else if (C == 0) {
if (isCodeCompletionPoint(CurPtr - 1)) {
if (ParsingFilename)
codeCompleteIncludedFile(AfterQuote, CurPtr - 1, /*IsAngled=*/false);
@@ -2619,11 +2702,11 @@ bool clang::Lexer::LexFLiteral(Token &Result, const char *CurPtr, tok::TokenKind
rpTok.setLength(1);
tokens.push_back(rpTok);
- SourceLocation litLoc = getSourceLocation(litStart, CurPtr - BufferPtr);
+ SourceLocation litLoc = getSourceLocation(AfterQuote, CurPtr - BufferPtr);
Token &litTok = tokens[literalPos];
litTok.startToken();
litTok.setKind(Kind);
- litTok.setLocation(getSourceLocation(litStart));
+ litTok.setLocation(getSourceLocation(AfterQuote));
litTok.setLiteralData(lit);
litTok.setKind(Kind);
litTok.setLength(resultingLiteral.size());
@@ -3015,7 +3098,7 @@ bool Lexer::SkipLineComment(Token &Result, const char *CurPtr,
// Otherwise, eat the \n character. We don't care if this is a \n\r or
// \r\n sequence. This is an efficiency hack (because we know the \n can't
// contribute to another token), it isn't needed for correctness. Note that
- // this is ok even in KeepWhitespaceMode, because we would have returned the
+ // this is res even in KeepWhitespaceMode, because we would have returned the
// comment above in that mode.
NewLinePtr = CurPtr++;
>From 0dbf6bae7a8b282405a2812e06644fb782726e7d Mon Sep 17 00:00:00 2001
From: "bengt.gustafsson" <bengt.gustafsson at beamways.com>
Date: Mon, 16 Sep 2024 09:28:48 +0200
Subject: [PATCH 05/17] Added new token kinds for f-literals for all encoding
prefixes. Would probably be better off with a token flag but this way we will
find all uses easier.
Refactored f-literal discovery to a new function handling all encoding prefixes.
---
clang/include/clang/Basic/TokenKinds.def | 7 +
clang/include/clang/Basic/TokenKinds.h | 6 +
clang/include/clang/Lex/Lexer.h | 6 +
clang/include/clang/Lex/Token.h | 23 +++-
clang/lib/Lex/Lexer.cpp | 167 +++++++++++------------
5 files changed, 119 insertions(+), 90 deletions(-)
diff --git a/clang/include/clang/Basic/TokenKinds.def b/clang/include/clang/Basic/TokenKinds.def
index b5a0e9df9f7ae..4cccd9c2ed873 100644
--- a/clang/include/clang/Basic/TokenKinds.def
+++ b/clang/include/clang/Basic/TokenKinds.def
@@ -185,6 +185,13 @@ TOK(utf8_string_literal) // u8"foo"
TOK(utf16_string_literal)// u"foo"
TOK(utf32_string_literal)// U"foo"
+// C++26 f-literals. These are distinct as this conveys from lexer to preprocessor/parser that Token::PtrData points to a Token::FLiteralInfo struct (in the arena)
+TOK(f_string_literal) // f"foo"
+TOK(wide_f_string_literal) // Lf"foo"
+TOK(utf8_f_string_literal) // u8f"foo"
+TOK(utf16_f_string_literal) // uf"foo"
+TOK(utf32_f_string_literal) // Uf"foo"
+
// C99 6.4.6: Punctuators.
PUNCTUATOR(l_square, "[")
PUNCTUATOR(r_square, "]")
diff --git a/clang/include/clang/Basic/TokenKinds.h b/clang/include/clang/Basic/TokenKinds.h
index e5183a27d2bc5..67e87198de419 100644
--- a/clang/include/clang/Basic/TokenKinds.h
+++ b/clang/include/clang/Basic/TokenKinds.h
@@ -92,6 +92,12 @@ inline bool isStringLiteral(TokenKind K) {
K == tok::utf32_string_literal;
}
+inline bool isFLiteral(TokenKind K) {
+ return K == tok::f_string_literal || K == tok::wide_f_string_literal ||
+ K == tok::utf8_f_string_literal || K == tok::utf16_f_string_literal ||
+ K == tok::utf32_f_string_literal;
+}
+
/// Return true if this is a "literal" kind, like a numeric
/// constant, string, etc.
inline bool isLiteral(TokenKind K) {
diff --git a/clang/include/clang/Lex/Lexer.h b/clang/include/clang/Lex/Lexer.h
index 0fa8ce89306fb..a364281f81e7d 100644
--- a/clang/include/clang/Lex/Lexer.h
+++ b/clang/include/clang/Lex/Lexer.h
@@ -756,8 +756,14 @@ class Lexer : public PreprocessorLexer {
int processNested (std::vector<Token> &tokens, Token& token);
int processNestedParenthesis(std::vector<Token> &tokens, const Token &token,
tok::TokenKind closer);
+ bool LexStringOrCharLiteral(Token &Result, const char *CurPtr,
+ tok::TokenKind StringLiteralKind,
+ tok::TokenKind CharLiteralKind,
+ tok::TokenKind FLiteralKind);
bool LexFLiteral(Token &Result, const char *CurPtr,
tok::TokenKind Kind);
+ bool LexRawFLiteral(Token &Result, const char *CurPtr,
+ tok::TokenKind Kind);
bool LexAngledStringLiteral(Token &Result, const char *CurPtr);
bool LexCharConstant (Token &Result, const char *CurPtr,
tok::TokenKind Kind);
diff --git a/clang/include/clang/Lex/Token.h b/clang/include/clang/Lex/Token.h
index 4f29fb7d11415..b50a091249de1 100644
--- a/clang/include/clang/Lex/Token.h
+++ b/clang/include/clang/Lex/Token.h
@@ -44,7 +44,7 @@ class Token {
/// UintData - This holds either the length of the token text, when
/// a normal token, or the end of the SourceRange when an annotation
- /// token.
+ /// token. (For f-literals the text pointer is in the FLiteralInfo)
SourceLocation::UIntTy UintData;
/// PtrData - This is a union of four different pointer types, which depends
@@ -55,6 +55,8 @@ class Token {
/// Literals: isLiteral() returns true.
/// This is a pointer to the start of the token in a text buffer, which
/// may be dirty (have trigraphs / escaped newlines).
+ /// f-literals: isFLiteral() returns true.
+ /// This is a pointer to a Token::FLiteralInfo struct.
/// Annotations (resolved type names, C++ scopes, etc): isAnnotation().
/// This is a pointer to sema-specific data for the annotation token.
/// Eof:
@@ -91,6 +93,12 @@ class Token {
// tokens are *not* reinjected.
};
+ struct FLiteralInfo {
+ const char *Text; // Pointer to the text after removing the expression-fields.
+ SourceLocation::UIntTy TokenCount; // Number of tokens in TokenPtr array.
+ Token *TokenPtr; // Pointer to an array of tokens constituting the comma separated list of all expression-fields.
+ };
+
tok::TokenKind getKind() const { return Kind; }
void setKind(tok::TokenKind K) { Kind = K; }
@@ -117,6 +125,19 @@ class Token {
return tok::isLiteral(getKind());
}
+ /// Return true if this is a "f-iteral", i.e. a string literal with expression-fields extracted to a token sequence.
+ bool isFLiteral() const { return tok::isFLiteral(getKind()); }
+
+ const FLiteralInfo& getFLiteralInfo() const {
+ assert(isFLiteral() && "Cannot get FliteralInfo data of non-f-literal");
+ return *reinterpret_cast<const FLiteralInfo*>(PtrData);
+ }
+
+ void setFLiteralData(FLiteralInfo &data) {
+ assert(isFLiteral() && "Cannot set FliteralInfo data of non-f-literal");
+ PtrData = &data;
+ }
+
/// Return true if this is any of tok::annot_* kind tokens.
bool isAnnotation() const { return tok::isAnnotation(getKind()); }
diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp
index 5e1778991bf7f..3c9888e24dcf6 100644
--- a/clang/lib/Lex/Lexer.cpp
+++ b/clang/lib/Lex/Lexer.cpp
@@ -1256,6 +1256,7 @@ static char GetTrigraphCharForLetter(char Letter) {
/// DecodeTrigraphChar - If the specified character is a legal trigraph when
/// prefixed with ??, emit a trigraph warning. If trigraphs are enabled,
+///
/// return the result character. Finally, emit a warning about trigraph use
/// whether trigraphs are enabled or not.
static char DecodeTrigraphChar(const char *CP, Lexer *L, bool Trigraphs) {
@@ -2552,6 +2553,59 @@ int clang::Lexer::processNestedParenthesis(std::vector<Token>& tokens, const Tok
}
+// Lex any string or char literal. When we get here we have seen u,u8, U or L. This is evident from the first tokenkind. The second tokenkind is provided as the corresponding char literal token in case we only saw
+// a unicode or wide prefix. If a literal actually followed returns true. If the letter was just the start of an identifier return false.
+bool clang::Lexer::LexStringOrCharLiteral(Token &Result, const char *CurPtr,
+ tok::TokenKind StringLiteralKind,
+ tok::TokenKind CharLiteralKind,
+ tok::TokenKind FLiteralKind) {
+ if (!LangOpts.CPlusPlus11 && !LangOpts.C11) {
+
+ unsigned SizeTmp;
+ char Char = getCharAndSize(CurPtr, SizeTmp);
+
+ // string literal
+ if (Char == '"')
+ return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result),
+ StringLiteralKind);
+
+ // character constant
+ if (Char == '\'')
+ return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result),
+ CharLiteralKind);
+
+ // UTF-16 raw string literal
+ unsigned SizeTmp2;
+ if (Char == 'R' && LangOpts.CPlusPlus11 &&
+ getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"')
+ return LexRawStringLiteral(
+ Result,
+ ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), SizeTmp2, Result),
+ StringLiteralKind);
+
+ if (Char == 'f' && LangOpts.CPlusPlus26) {
+ Char = getCharAndSize(CurPtr + SizeTmp, SizeTmp2);
+ if (Char == '"')
+ return LexFLiteral(
+ Result,
+ ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), SizeTmp2, Result),
+ FLiteralKind);
+ unsigned SizeTmp3;
+ if (Char == 'R' && LangOpts.CPlusPlus11 &&
+ getCharAndSize(CurPtr + SizeTmp + SizeTmp2, SizeTmp3) == '"')
+ return LexRawFLiteral(
+ Result,
+ ConsumeChar(ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
+ SizeTmp2, Result),
+ SizeTmp3, Result),
+ FLiteralKind);
+ }
+ }
+ return false;
+}
+
+
+
bool clang::Lexer::LexFLiteral(Token &Result, const char *CurPtr, tok::TokenKind Kind) {
const char *litStart = BufferPtr; // Before all prefixes
const char *AfterQuote = CurPtr;
@@ -4251,53 +4305,16 @@ bool Lexer::LexTokenInternal(Token &Result, bool TokAtPhysicalStartOfLine) {
if (LangOpts.CPlusPlus11 || LangOpts.C11) {
Char = getCharAndSize(CurPtr, SizeTmp);
+ bool ok;
+ if (Char == '8')
+ ok = LexStringOrCharLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result),
+ tok::utf8_string_literal, tok::utf8_char_constant, tok::utf8_f_string_literal);
+ else
+ ok = LexStringOrCharLiteral(Result, CurPtr,
+ tok::utf16_string_literal, tok::utf16_char_constant, tok::utf16_f_string_literal);
- // UTF-16 string literal
- if (Char == '"')
- return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result),
- tok::utf16_string_literal);
-
- // UTF-16 character constant
- if (Char == '\'')
- return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result),
- tok::utf16_char_constant);
-
- // UTF-16 raw string literal
- if (Char == 'R' && LangOpts.CPlusPlus11 &&
- getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"')
- return LexRawStringLiteral(Result,
- ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
- SizeTmp2, Result),
- tok::utf16_string_literal);
-
- if (Char == '8') {
- char Char2 = getCharAndSize(CurPtr + SizeTmp, SizeTmp2);
-
- // UTF-8 string literal
- if (Char2 == '"')
- return LexStringLiteral(Result,
- ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
- SizeTmp2, Result),
- tok::utf8_string_literal);
- if (Char2 == '\'' && (LangOpts.CPlusPlus17 || LangOpts.C23))
- return LexCharConstant(
- Result, ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
- SizeTmp2, Result),
- tok::utf8_char_constant);
-
- if (Char2 == 'R' && LangOpts.CPlusPlus11) {
- unsigned SizeTmp3;
- char Char3 = getCharAndSize(CurPtr + SizeTmp + SizeTmp2, SizeTmp3);
- // UTF-8 raw string literal
- if (Char3 == '"') {
- return LexRawStringLiteral(Result,
- ConsumeChar(ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
- SizeTmp2, Result),
- SizeTmp3, Result),
- tok::utf8_string_literal);
- }
- }
- }
+ if (ok)
+ return true;
}
// treat u like the start of an identifier.
@@ -4307,27 +4324,9 @@ bool Lexer::LexTokenInternal(Token &Result, bool TokAtPhysicalStartOfLine) {
// Notify MIOpt that we read a non-whitespace/non-comment token.
MIOpt.ReadToken();
- if (LangOpts.CPlusPlus11 || LangOpts.C11) {
- Char = getCharAndSize(CurPtr, SizeTmp);
-
- // UTF-32 string literal
- if (Char == '"')
- return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result),
- tok::utf32_string_literal);
-
- // UTF-32 character constant
- if (Char == '\'')
- return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result),
- tok::utf32_char_constant);
-
- // UTF-32 raw string literal
- if (Char == 'R' && LangOpts.CPlusPlus11 &&
- getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"')
- return LexRawStringLiteral(Result,
- ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
- SizeTmp2, Result),
- tok::utf32_string_literal);
- }
+ if (LexStringOrCharLiteral(Result, CurPtr,
+ tok::utf32_string_literal, tok::utf32_char_constant, tok::utf32_f_string_literal))
+ return true;
// treat U like the start of an identifier.
return LexIdentifierContinue(Result, CurPtr);
@@ -4353,11 +4352,15 @@ bool Lexer::LexTokenInternal(Token &Result, bool TokAtPhysicalStartOfLine) {
MIOpt.ReadToken();
if (LangOpts.CPlusPlus11) {
- Char = getCharAndSize(CurPtr, SizeTmp);
-
- if (Char == '"')
- // LexFLiteral pushes a MacroInfo object containing everything except the std of the std::format( prefix which is set in Result instead.
- return LexFLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result), tok::string_literal);
+ Char = getCharAndSize(CurPtr, SizeTmp);
+ if (Char == '"')
+ return LexFLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result), tok::f_string_literal);
+ unsigned SizeTmp2;
+ if (Char == 'R' && LangOpts.CPlusPlus11 &&
+ getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"')
+ return LexRawFLiteral(Result,
+ ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), SizeTmp2, Result),
+ tok::f_string_literal);
}
// treat R like the start of an identifier.
@@ -4366,25 +4369,11 @@ bool Lexer::LexTokenInternal(Token &Result, bool TokAtPhysicalStartOfLine) {
case 'L': // Identifier (Loony) or wide literal (L'x' or L"xyz").
// Notify MIOpt that we read a non-whitespace/non-comment token.
MIOpt.ReadToken();
- Char = getCharAndSize(CurPtr, SizeTmp);
-
- // Wide string literal.
- if (Char == '"')
- return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result),
- tok::wide_string_literal);
- // Wide raw string literal.
- if (LangOpts.CPlusPlus11 && Char == 'R' &&
- getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"')
- return LexRawStringLiteral(Result,
- ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
- SizeTmp2, Result),
- tok::wide_string_literal);
+ if (LexStringOrCharLiteral(Result, CurPtr,
+ tok::wide_string_literal, tok::wide_char_constant, tok::wide_f_string_literal))
+ return true;
- // Wide character constant.
- if (Char == '\'')
- return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result),
- tok::wide_char_constant);
// FALL THROUGH, treating L like the start of an identifier.
[[fallthrough]];
>From 118e75198bb9857585c0cd018da2ad33da978971 Mon Sep 17 00:00:00 2001
From: "bengt.gustafsson" <bengt.gustafsson at beamways.com>
Date: Wed, 25 Sep 2024 21:34:13 +0200
Subject: [PATCH 06/17] Entire test program runs.
Replaced the five new token kinds with a token flag IsFLiteral.
Handle :> when ] not expected as : >
Pass the f of f-literals in StringLiteralParser::init
Implemented the formation of a std::format call from all the collected expressions and literals with all literals before all expressions in the Parser::ParseStringLiteralExpression function. Then it recurses to ParseCastExpression.
Removed comma error as we can't do this in the lexer, or template argument lists won't work.
---
.../include/clang/Basic/DiagnosticLexKinds.td | 1 -
clang/include/clang/Basic/TokenKinds.def | 7 -
clang/include/clang/Basic/TokenKinds.h | 6 -
clang/include/clang/Lex/Lexer.h | 6 +-
clang/include/clang/Lex/Token.h | 20 +-
clang/lib/Lex/Lexer.cpp | 329 ++++++++++++------
clang/lib/Lex/LiteralSupport.cpp | 2 +
clang/lib/Parse/ParseExpr.cpp | 92 ++++-
8 files changed, 327 insertions(+), 136 deletions(-)
diff --git a/clang/include/clang/Basic/DiagnosticLexKinds.td b/clang/include/clang/Basic/DiagnosticLexKinds.td
index 85d7c54ebf5df..faa55e68b27ce 100644
--- a/clang/include/clang/Basic/DiagnosticLexKinds.td
+++ b/clang/include/clang/Basic/DiagnosticLexKinds.td
@@ -91,7 +91,6 @@ def ext_unterminated_char_or_string : ExtWarn<
def err_nested_format_spec : Error<"format specifier in nested expression field">;
def err_unmatched_paren : Error<"unmatched %0 in expression field">;
def err_incomplete_ternary : Error<"ternary ? operator without matching : in expression field">;
-def err_comma_in_expression_field : Error<"comma not allowed in expression field">;
def err_unescaped_r_brace : Error<"unquoted } in format string">;
def ext_empty_character : ExtWarn<"empty character constant">,
diff --git a/clang/include/clang/Basic/TokenKinds.def b/clang/include/clang/Basic/TokenKinds.def
index 4cccd9c2ed873..b5a0e9df9f7ae 100644
--- a/clang/include/clang/Basic/TokenKinds.def
+++ b/clang/include/clang/Basic/TokenKinds.def
@@ -185,13 +185,6 @@ TOK(utf8_string_literal) // u8"foo"
TOK(utf16_string_literal)// u"foo"
TOK(utf32_string_literal)// U"foo"
-// C++26 f-literals. These are distinct as this conveys from lexer to preprocessor/parser that Token::PtrData points to a Token::FLiteralInfo struct (in the arena)
-TOK(f_string_literal) // f"foo"
-TOK(wide_f_string_literal) // Lf"foo"
-TOK(utf8_f_string_literal) // u8f"foo"
-TOK(utf16_f_string_literal) // uf"foo"
-TOK(utf32_f_string_literal) // Uf"foo"
-
// C99 6.4.6: Punctuators.
PUNCTUATOR(l_square, "[")
PUNCTUATOR(r_square, "]")
diff --git a/clang/include/clang/Basic/TokenKinds.h b/clang/include/clang/Basic/TokenKinds.h
index 67e87198de419..e5183a27d2bc5 100644
--- a/clang/include/clang/Basic/TokenKinds.h
+++ b/clang/include/clang/Basic/TokenKinds.h
@@ -92,12 +92,6 @@ inline bool isStringLiteral(TokenKind K) {
K == tok::utf32_string_literal;
}
-inline bool isFLiteral(TokenKind K) {
- return K == tok::f_string_literal || K == tok::wide_f_string_literal ||
- K == tok::utf8_f_string_literal || K == tok::utf16_f_string_literal ||
- K == tok::utf32_f_string_literal;
-}
-
/// Return true if this is a "literal" kind, like a numeric
/// constant, string, etc.
inline bool isLiteral(TokenKind K) {
diff --git a/clang/include/clang/Lex/Lexer.h b/clang/include/clang/Lex/Lexer.h
index a364281f81e7d..51f277d40d0be 100644
--- a/clang/include/clang/Lex/Lexer.h
+++ b/clang/include/clang/Lex/Lexer.h
@@ -753,13 +753,13 @@ class Lexer : public PreprocessorLexer {
bool processExtractionField(std::vector<Token> &tokens, const char *litStart,
std::string &lit);
Token processExpression (std::vector<Token> &tokens, bool allowComma);
- int processNested (std::vector<Token> &tokens, Token& token);
+ int processNested(std::vector<Token> &tokens, Token &token,
+ bool expectRBracket);
int processNestedParenthesis(std::vector<Token> &tokens, const Token &token,
tok::TokenKind closer);
bool LexStringOrCharLiteral(Token &Result, const char *CurPtr,
tok::TokenKind StringLiteralKind,
- tok::TokenKind CharLiteralKind,
- tok::TokenKind FLiteralKind);
+ tok::TokenKind CharLiteralKind);
bool LexFLiteral(Token &Result, const char *CurPtr,
tok::TokenKind Kind);
bool LexRawFLiteral(Token &Result, const char *CurPtr,
diff --git a/clang/include/clang/Lex/Token.h b/clang/include/clang/Lex/Token.h
index b50a091249de1..e4dc3f04ed570 100644
--- a/clang/include/clang/Lex/Token.h
+++ b/clang/include/clang/Lex/Token.h
@@ -91,6 +91,7 @@ class Token {
IsReinjected = 0x800, // A phase 4 token that was produced before and
// re-added, e.g. via EnterTokenStream. Annotation
// tokens are *not* reinjected.
+ IsFLiteral = 0x1000, // Token is a f-literal, PtrData is FLiteralInfo*
};
struct FLiteralInfo {
@@ -124,17 +125,23 @@ class Token {
bool isLiteral() const {
return tok::isLiteral(getKind());
}
+ bool isStringLiteral() const {
+ return tok::isStringLiteral(getKind());
+ }
/// Return true if this is a "f-iteral", i.e. a string literal with expression-fields extracted to a token sequence.
- bool isFLiteral() const { return tok::isFLiteral(getKind()); }
+ bool isFLiteral() const {
+ return getFlag(IsFLiteral);
+ }
const FLiteralInfo& getFLiteralInfo() const {
assert(isFLiteral() && "Cannot get FliteralInfo data of non-f-literal");
return *reinterpret_cast<const FLiteralInfo*>(PtrData);
}
- void setFLiteralData(FLiteralInfo &data) {
- assert(isFLiteral() && "Cannot set FliteralInfo data of non-f-literal");
+ void setFLiteralInfo(FLiteralInfo &data) {
+ assert(isStringLiteral() && "Cannot set FliteralInfo data of non-string-literal");
+ setFlag(IsFLiteral);
PtrData = &data;
}
@@ -245,10 +252,13 @@ class Token {
/// otherwise.
const char *getLiteralData() const {
assert(isLiteral() && "Cannot get literal data of non-literal");
- return reinterpret_cast<const char*>(PtrData);
+ if (isFLiteral())
+ return getFLiteralInfo().Text;
+ else
+ return reinterpret_cast<const char*>(PtrData);
}
void setLiteralData(const char *Ptr) {
- assert(isLiteral() && "Cannot set literal data of non-literal");
+ assert(isLiteral() && !isFLiteral() && "Cannot set literal data of non-literal");
PtrData = const_cast<char*>(Ptr);
}
diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp
index 3c9888e24dcf6..4d9ddc54cd5ce 100644
--- a/clang/lib/Lex/Lexer.cpp
+++ b/clang/lib/Lex/Lexer.cpp
@@ -2424,7 +2424,7 @@ Token clang::Lexer::processExpression(std::vector<Token>& tokens, bool allowComm
const char *lpos = BufferPtr - 1;
while (true) {
Token token;
- int res = processNested(tokens, token);
+ int res = processNested(tokens, token, false);
if (res != 0)
return Token();
@@ -2482,15 +2482,7 @@ Token clang::Lexer::processExpression(std::vector<Token>& tokens, bool allowComm
tokens.push_back(ident);
break;
}
- case tok::comma:
- if (!allowComma) {
- // Important to diagnose erroneous comma here as std::format allows
- // extra expressions, which is ridiculous for f-literals as they
- // can't be translated.
- Diag(BufferPtr - 1, diag::err_comma_in_expression_field);
- return Token();
- }
- [[fallthrough]];
+
default:
tokens.push_back(token);
}
@@ -2499,7 +2491,7 @@ Token clang::Lexer::processExpression(std::vector<Token>& tokens, bool allowComm
// Pass over any number of matched parentheses, pushing all the tokens thus passed over.
// Return the first token outside any matched parentheses.
-int clang::Lexer::processNested(std::vector<Token>& tokens, Token& token)
+int clang::Lexer::processNested(std::vector<Token>& tokens, Token& token, bool expectRBracket)
{
while (true) {
PP->Lex(token);
@@ -2516,12 +2508,23 @@ int clang::Lexer::processNested(std::vector<Token>& tokens, Token& token)
res = processNestedParenthesis(tokens, token, tok::r_brace);
break;
+ case tok::r_square:
+ // Kludge to handle the :> digraph converting to ] unless this is not
+ // expected in which case it reverts to : followed by >
+ if (!expectRBracket && BufferPtr[-1] == '>') {
+ // Create a fake : token to return, and back BufferPtr one char so
+ // that the next char will be >
+ token.setKind(tok::colon);
+ BufferPtr--;
+ }
+ [[fallthrough]];
+
default:
return 0;
}
- if (res != 0)
- return res;
+ if (res != 0)
+ return res;
}
}
@@ -2531,7 +2534,7 @@ int clang::Lexer::processNestedParenthesis(std::vector<Token>& tokens, const Tok
const char *pos = BufferPtr - 1;
while (true) {
Token next;
- int res = processNested(tokens, next);
+ int res = processNested(tokens, next, closer == tok::r_square);
if (res != 0)
return res;
@@ -2553,13 +2556,13 @@ int clang::Lexer::processNestedParenthesis(std::vector<Token>& tokens, const Tok
}
-// Lex any string or char literal. When we get here we have seen u,u8, U or L. This is evident from the first tokenkind. The second tokenkind is provided as the corresponding char literal token in case we only saw
-// a unicode or wide prefix. If a literal actually followed returns true. If the letter was just the start of an identifier return false.
+// Lex any string or char literal. When we get here we have seen u,u8, U or L. Which is evident from the first tokenkind. The second
+// tokenkind is provided as the corresponding char literal token in case we only saw a unicode or wide prefix. If a literal actually
+// followed returns true. If the letter was just the start of an identifier return false.
bool clang::Lexer::LexStringOrCharLiteral(Token &Result, const char *CurPtr,
tok::TokenKind StringLiteralKind,
- tok::TokenKind CharLiteralKind,
- tok::TokenKind FLiteralKind) {
- if (!LangOpts.CPlusPlus11 && !LangOpts.C11) {
+ tok::TokenKind CharLiteralKind) {
+ if (LangOpts.CPlusPlus11 || LangOpts.C11) {
unsigned SizeTmp;
char Char = getCharAndSize(CurPtr, SizeTmp);
@@ -2589,7 +2592,7 @@ bool clang::Lexer::LexStringOrCharLiteral(Token &Result, const char *CurPtr,
return LexFLiteral(
Result,
ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), SizeTmp2, Result),
- FLiteralKind);
+ StringLiteralKind);
unsigned SizeTmp3;
if (Char == 'R' && LangOpts.CPlusPlus11 &&
getCharAndSize(CurPtr + SizeTmp + SizeTmp2, SizeTmp3) == '"')
@@ -2598,7 +2601,7 @@ bool clang::Lexer::LexStringOrCharLiteral(Token &Result, const char *CurPtr,
ConsumeChar(ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
SizeTmp2, Result),
SizeTmp3, Result),
- FLiteralKind);
+ StringLiteralKind);
}
}
return false;
@@ -2615,55 +2618,15 @@ bool clang::Lexer::LexFLiteral(Token &Result, const char *CurPtr, tok::TokenKind
if (PP == nullptr)
return LexStringLiteral(Result, CurPtr, Kind);
- if (!isLexingRawMode() &&
- (Kind == tok::utf8_string_literal || Kind == tok::utf16_string_literal ||
- Kind == tok::utf32_string_literal))
- Diag(BufferPtr, LangOpts.CPlusPlus ? diag::warn_cxx98_compat_unicode_literal
- : diag::warn_c99_compat_unicode_literal);
-
// As the string literal is no longer the same as in the source code we must save it into this variable
// until the right " is found. Then we create a token around it, with its source location set to the start of the original literal
- // and push this into the MacroInfo after the initial std::format( sequence. To handle the life time of the string contents
- // we use the BP allocator function.
- std::string resultingLiteral = "\"";
+ // and push this into the Text member of the Token::FLiteralInfo created using an arena allocation to avoid dangling.
+ std::string resultingLiteral = "\""; // Usually string literals have the entire prefix included, but only the leading " is actually checked by StringLiteralParser::init when concatenating.
// Collect all tokens in this vector until we know how many we have, i.e. at the end.
+ // Maybe there is a better way of doing this as we are moving the tokens to an arena allocated array later.
std::vector<Token> tokens;
- // Form the prefix tokens.
-
- Token ccTok;
- ccTok.startToken();
- ccTok.setKind(tok::coloncolon);
- ccTok.setLocation(getSourceLocation());
- tokens.push_back(ccTok);
-
- Token stdTok;
- stdTok.startToken();
- stdTok.setLocation(getSourceLocation());
- stdTok.setIdentifierInfo(PP->getIdentifierInfo("std"));
- stdTok.setKind(tok::identifier);
- tokens.push_back(stdTok);
-
- tokens.push_back(ccTok);
-
- Token fmtTok;
- fmtTok.startToken();
- fmtTok.setLocation(getSourceLocation());
- fmtTok.setIdentifierInfo(PP->getIdentifierInfo("format"));
- fmtTok.setKind(tok::identifier);
- tokens.push_back(fmtTok);
-
- Token lpTok;
- lpTok.startToken();
- lpTok.setLocation(getSourceLocation());
- lpTok.setKind(tok::l_paren);
- tokens.push_back(lpTok);
-
- const size_t literalPos = tokens.size(); // 4
-
- tokens.push_back(Token()); // Placeholder for the literal.
-
const char *litPartStart = CurPtr;
char C = getAndAdvanceChar(CurPtr, Result);
@@ -2677,7 +2640,7 @@ bool clang::Lexer::LexFLiteral(Token &Result, const char *CurPtr, tok::TokenKind
(C == 0 && CurPtr - 1 == BufferEnd)) { // End of file.
if (IgnoreUnterminatedLiterals)
return true; // Result still unknown. This is detected by extraction
- // field processing in enclosing extraction field.
+ // field processing in enclosing extraction field.
if (!isLexingRawMode() && !LangOpts.AsmPreprocessor)
Diag(BufferPtr, diag::ext_unterminated_char_or_string) << 1;
@@ -2692,15 +2655,13 @@ bool clang::Lexer::LexFLiteral(Token &Result, const char *CurPtr, tok::TokenKind
// Note: If we get a
bool save = IgnoreUnterminatedLiterals;
IgnoreUnterminatedLiterals = true;
- bool ok =
- processExtractionField(tokens, litPartStart, resultingLiteral);
+ bool ok = processExtractionField(tokens, litPartStart, resultingLiteral);
IgnoreUnterminatedLiterals = save;
if (!ok) {
- BufferPtr = litStart + 1; // Pass the f: TODO: somehow pass any L, u, U or u8 but not the f to LexStringLiteral. Or just create a fake token, we're already at the trailing " in the input anyway.
- return LexStringLiteral(
- Result, AfterQuote,
- Kind); // After reporting errors here just pass a regular string
- // literal along. This is to avoid errors later.
+ BufferPtr = litStart + 1;
+ // reporting errors here just parse as a regular string literal. TODO: Instead don't reparse but forward an empty string literal of the appropriate kind to the preprocessor.
+ return LexStringLiteral(Result, AfterQuote, Kind);
+ // literal along. This is to avoid errors later.
}
CurPtr = BufferPtr;
litPartStart = CurPtr;
@@ -2735,13 +2696,8 @@ bool clang::Lexer::LexFLiteral(Token &Result, const char *CurPtr, tok::TokenKind
// If a nul character existed in the string, warn about it.
if (NulCharacter && !isLexingRawMode())
Diag(NulCharacter, diag::null_in_char_or_string) << 1;
-
- // If we are in C++11, lex the optional ud-suffix. This is probably not useful for f literals as stD::format would not accept the result of the customer defined operator, which is
- // no longer a string literal.
- if (LangOpts.CPlusPlus)
- CurPtr = LexUDSuffix(Result, CurPtr, true);
- // COmplete the resulting literal by adding the bit after the last }
+ // Complete the resulting literal by adding the bit after the last }
resultingLiteral.append(litPartStart, CurPtr);
// Fill in the literalPos token with the remaining string. But to keep its contents first allocate
@@ -2749,21 +2705,162 @@ bool clang::Lexer::LexFLiteral(Token &Result, const char *CurPtr, tok::TokenKind
char *lit = new (PP->getPreprocessorAllocator()) char[resultingLiteral.size() + 1];
memcpy(lit, resultingLiteral.c_str(), resultingLiteral.size() + 1);
- Token rpTok;
- rpTok.startToken();
- rpTok.setKind(tok::r_paren);
- rpTok.setLocation(getSourceLocation(CurPtr));
- rpTok.setLength(1);
- tokens.push_back(rpTok);
-
- SourceLocation litLoc = getSourceLocation(AfterQuote, CurPtr - BufferPtr);
- Token &litTok = tokens[literalPos];
- litTok.startToken();
- litTok.setKind(Kind);
- litTok.setLocation(getSourceLocation(AfterQuote));
- litTok.setLiteralData(lit);
- litTok.setKind(Kind);
- litTok.setLength(resultingLiteral.size());
+ // The completed lexing is represented by a MacroInfo object which is pushed
+ // onto the macro expansion stack This expansion consists of std::format("...
+ // the remaining string ...", ...the extracted argument tokens...)
+
+ BufferPtr = CurPtr;
+
+ Token* tokStore = new (PP->getPreprocessorAllocator()) Token[tokens.size()];
+ std::copy(tokens.begin(), tokens.end(), tokStore);
+
+ Token::FLiteralInfo* fli = new (PP->getPreprocessorAllocator()) Token::FLiteralInfo;
+ fli->Text = lit;
+ fli->TokenCount = SourceLocation::UIntTy(tokens.size());
+ fli->TokenPtr = tokStore;
+
+ Result.setKind(Kind);
+ Result.setLocation(getSourceLocation(AfterQuote)); // Probably not needed
+ Result.setFLiteralInfo(*fli);
+ Result.setLength(resultingLiteral.size());
+
+ return true; // Result has been filled in.
+}
+
+
+bool clang::Lexer::LexRawFLiteral(Token &Result, const char *CurPtr, tok::TokenKind Kind) {
+ const char *litStart = BufferPtr; // Before all prefixes
+ const char *AfterQuote = CurPtr;
+
+ if (PP == nullptr)
+ return LexRawStringLiteral(Result, CurPtr, Kind);
+
+ // This function doesn't use getAndAdvanceChar because C++0x [lex.pptoken]p3:
+ // Between the initial and final double quote characters of the raw string,
+ // any transformations performed in phases 1 and 2 (trigraphs,
+ // universal-character-names, and line splicing) are reverted.
+
+ if (!isLexingRawMode())
+ Diag(BufferPtr, diag::warn_cxx98_compat_raw_string_literal);
+
+ unsigned PrefixLen = 0;
+
+ while (PrefixLen != 16 && isRawStringDelimBody(CurPtr[PrefixLen])) {
+ ++PrefixLen;
+ if (!isLexingRawMode() &&
+ llvm::is_contained({'$', '@', '`'}, CurPtr[PrefixLen])) {
+ const char *Pos = &CurPtr[PrefixLen];
+ Diag(Pos, LangOpts.CPlusPlus26
+ ? diag::warn_cxx26_compat_raw_string_literal_character_set
+ : diag::ext_cxx26_raw_string_literal_character_set)
+ << StringRef(Pos, 1);
+ }
+ }
+
+ // If the last character was not a '(', then we didn't lex a valid delimiter.
+ if (CurPtr[PrefixLen] != '(') {
+ if (!isLexingRawMode()) {
+ const char *PrefixEnd = &CurPtr[PrefixLen];
+ if (PrefixLen == 16) {
+ Diag(PrefixEnd, diag::err_raw_delim_too_long);
+ } else if (*PrefixEnd == '\n') {
+ Diag(PrefixEnd, diag::err_invalid_newline_raw_delim);
+ } else {
+ Diag(PrefixEnd, diag::err_invalid_char_raw_delim)
+ << StringRef(PrefixEnd, 1);
+ }
+ }
+
+ // Search for the next '"' in hopes of salvaging the lexer. Unfortunately,
+ // it's possible the '"' was intended to be part of the raw string, but
+ // there's not much we can do about that.
+ while (true) {
+ char C = *CurPtr++;
+
+ if (C == '"')
+ break;
+ if (C == 0 && CurPtr-1 == BufferEnd) {
+ --CurPtr;
+ break;
+ }
+ }
+
+ FormTokenWithChars(Result, CurPtr, tok::unknown);
+ return true;
+ }
+
+ // Save prefix and move CurPtr past it
+ const char *Prefix = CurPtr;
+ CurPtr += PrefixLen + 1; // skip over prefix and '('
+
+ // As the string literal is no longer the same as in the source code we must save it into this variable
+ // until the right " is found. Then we create a token around it, with its source location set to the start of the original literal
+ // and push this into the MacroInfo after the initial std::format( sequence. To handle the life time of the string contents
+ // we use the BP allocator function.
+ std::string resultingLiteral(BufferPtr, CurPtr); // Pick up all the prefixes
+
+ // Collect all tokens in this vector until we know how many we have, i.e. at the end.
+ // Maybe there is a better way of doing this as we are moving the tokens to an arena allocated array later.
+ std::vector<Token> tokens;
+
+ const char *litPartStart = CurPtr;
+
+ while (true) {
+ char C = *CurPtr++;
+
+ if (C == ')') {
+ // Check for prefix match and closing quote.
+ if (strncmp(CurPtr, Prefix, PrefixLen) == 0 && CurPtr[PrefixLen] == '"') {
+ CurPtr += PrefixLen + 1; // skip over prefix and '"'
+ break;
+ }
+ }
+ else if (C == 0 && CurPtr-1 == BufferEnd) { // End of file.
+ if (!isLexingRawMode())
+ Diag(BufferPtr, diag::err_unterminated_raw_string)
+ << StringRef(Prefix, PrefixLen);
+ FormTokenWithChars(Result, CurPtr-1, tok::unknown);
+ return true;
+ }
+
+ // Note: {{ a quoted brace, We need to keep both as the resulting literal is still input to std::format!
+ if (C == '{') {
+ if (*CurPtr != '{') {
+ BufferPtr = CurPtr;
+ // Note: If we get a
+ bool save = IgnoreUnterminatedLiterals;
+ IgnoreUnterminatedLiterals = true;
+ bool ok = processExtractionField(tokens, litPartStart, resultingLiteral);
+ IgnoreUnterminatedLiterals = save;
+ if (!ok) {
+ BufferPtr = litStart + 1; // Pass the f: TODO: somehow pass any L, u, U or u8 but not the f to LexStringLiteral. Or just create a fake token, we're already at the trailing " in the input anyway. After
+ // reporting errors here just pass a regular string
+ // literal along. This is to avoid errors later.
+ return LexRawStringLiteral(Result, AfterQuote, Kind);
+ }
+ CurPtr = BufferPtr;
+ litPartStart = CurPtr;
+ }
+ else
+ CurPtr++;
+ }
+ else if (C == '}') {
+ if (*CurPtr != '}')
+ Diag(CurPtr - 1, diag::err_unescaped_r_brace);
+ else
+ CurPtr++; // Pass the second }.
+ }
+ }
+
+ // f literals can't be combined with ud suffixes, so don't parse such.
+
+ // Complete the resulting literal by adding the bit after the last }
+ resultingLiteral.append(litPartStart, CurPtr);
+
+ // Fill in the literalPos token with the remaining string. But to keep its contents first allocate
+ // using the bumpAllocator from PP.
+ char *lit = new (PP->getPreprocessorAllocator()) char[resultingLiteral.size() + 1];
+ memcpy(lit, resultingLiteral.c_str(), resultingLiteral.size() + 1);
// The completed lexing is represented by a MacroInfo object which is pushed
// onto the macro expansion stack This expansion consists of std::format("...
@@ -2771,11 +2868,21 @@ bool clang::Lexer::LexFLiteral(Token &Result, const char *CurPtr, tok::TokenKind
BufferPtr = CurPtr;
- auto tokStore = std::make_unique<Token[]>(tokens.size());
- std::copy(tokens.begin(), tokens.end(), tokStore.get());
+ Token* tokStore = new (PP->getPreprocessorAllocator()) Token[tokens.size()];
+ std::copy(tokens.begin(), tokens.end(), tokStore);
+
+ Token::FLiteralInfo* fli = new (PP->getPreprocessorAllocator()) Token::FLiteralInfo;
+ fli->Text = lit;
+ fli->TokenCount = SourceLocation::UIntTy(tokens.size());
+ fli->TokenPtr = tokStore;
- PP->EnterTokenStream(std::move(tokStore), tokens.size(), true, false);
- return false; // Result has not been filled in.
+ Result.startToken();
+ Result.setKind(Kind);
+ Result.setLocation(getSourceLocation(AfterQuote)); // Probably not needed
+ Result.setFLiteralInfo(*fli);
+ Result.setLength(resultingLiteral.size());
+
+ return true; // Result has been filled in.
}
@@ -4308,10 +4415,10 @@ bool Lexer::LexTokenInternal(Token &Result, bool TokAtPhysicalStartOfLine) {
bool ok;
if (Char == '8')
ok = LexStringOrCharLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result),
- tok::utf8_string_literal, tok::utf8_char_constant, tok::utf8_f_string_literal);
+ tok::utf8_string_literal, tok::utf8_char_constant);
else
ok = LexStringOrCharLiteral(Result, CurPtr,
- tok::utf16_string_literal, tok::utf16_char_constant, tok::utf16_f_string_literal);
+ tok::utf16_string_literal, tok::utf16_char_constant);
if (ok)
return true;
@@ -4325,7 +4432,7 @@ bool Lexer::LexTokenInternal(Token &Result, bool TokAtPhysicalStartOfLine) {
MIOpt.ReadToken();
if (LexStringOrCharLiteral(Result, CurPtr,
- tok::utf32_string_literal, tok::utf32_char_constant, tok::utf32_f_string_literal))
+ tok::utf32_string_literal, tok::utf32_char_constant))
return true;
// treat U like the start of an identifier.
@@ -4352,15 +4459,17 @@ bool Lexer::LexTokenInternal(Token &Result, bool TokAtPhysicalStartOfLine) {
MIOpt.ReadToken();
if (LangOpts.CPlusPlus11) {
- Char = getCharAndSize(CurPtr, SizeTmp);
- if (Char == '"')
- return LexFLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result), tok::f_string_literal);
- unsigned SizeTmp2;
- if (Char == 'R' && LangOpts.CPlusPlus11 &&
- getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"')
- return LexRawFLiteral(Result,
- ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), SizeTmp2, Result),
- tok::f_string_literal);
+ Char = getCharAndSize(CurPtr, SizeTmp);
+ if (Char == '"')
+ return LexFLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result),
+ tok::string_literal);
+ unsigned SizeTmp2;
+ if (Char == 'R' && LangOpts.CPlusPlus11 &&
+ getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"')
+ return LexRawFLiteral(
+ Result,
+ ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), SizeTmp2, Result),
+ tok::string_literal);
}
// treat R like the start of an identifier.
@@ -4371,7 +4480,7 @@ bool Lexer::LexTokenInternal(Token &Result, bool TokAtPhysicalStartOfLine) {
MIOpt.ReadToken();
if (LexStringOrCharLiteral(Result, CurPtr,
- tok::wide_string_literal, tok::wide_char_constant, tok::wide_f_string_literal))
+ tok::wide_string_literal, tok::wide_char_constant))
return true;
// FALL THROUGH, treating L like the start of an identifier.
diff --git a/clang/lib/Lex/LiteralSupport.cpp b/clang/lib/Lex/LiteralSupport.cpp
index 3df0391bdda77..8b9c65c96dcb1 100644
--- a/clang/lib/Lex/LiteralSupport.cpp
+++ b/clang/lib/Lex/LiteralSupport.cpp
@@ -2122,6 +2122,8 @@ void StringLiteralParser::init(ArrayRef<Token> StringToks){
if (ThisTokBuf[0] == '8')
++ThisTokBuf;
}
+ if (ThisTokBuf[0] == 'f') // Skip f prefix, if any.
+ ++ThisTokBuf;
// Check for raw string
if (ThisTokBuf[0] == 'R') {
diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp
index eb7447fa038e4..1d474f8ed8113 100644
--- a/clang/lib/Parse/ParseExpr.cpp
+++ b/clang/lib/Parse/ParseExpr.cpp
@@ -3387,10 +3387,94 @@ ExprResult Parser::ParseStringLiteralExpression(bool AllowUserDefinedLiteral,
return Actions.ActOnUnevaluatedStringLiteral(StringToks);
}
- // Pass the set of string tokens, ready for concatenation, to the actions.
- return Actions.ActOnStringLiteral(StringToks,
- AllowUserDefinedLiteral ? getCurScope()
- : nullptr);
+ // All the tokens of all f-literals are collected here. If not empty after the
+ // loop the tokens are injected in the token stream.
+
+
+ if (!std::any_of(StringToks.begin(), StringToks.end(), [](Token& tok) { return tok.isFLiteral(); })) {
+ // Pass the set of string tokens, ready for concatenation, to the actions.
+ return Actions.ActOnStringLiteral(
+ StringToks, AllowUserDefinedLiteral ? getCurScope() : nullptr);
+ }
+
+ // Probably unevaluated must not be set here, i.e. we disallow f-literals in such contents (which does not include consteval functions).
+ // Shuffle the tokens to produce a ::std::formatted_string(<all literals> <all tokens>) sequence.
+ // Inject this into the PP and then recurse. back to cast_expression.
+ std::vector<Token> tokens;
+
+ // Inject the preamble consisting of `::std::format(`
+ // TODO: Change this to std::formatted_string ctor call.
+
+ // Form the prefix tokens.
+
+ auto loc = StringToks.front().getLocation();
+ Token ccTok;
+ ccTok.startToken();
+ ccTok.setKind(tok::coloncolon);
+ ccTok.setLocation(loc);
+ tokens.push_back(ccTok);
+
+ Token stdTok;
+ stdTok.startToken();
+ stdTok.setLocation(loc);
+ stdTok.setIdentifierInfo(PP.getIdentifierInfo("std"));
+ stdTok.setKind(tok::identifier);
+ tokens.push_back(stdTok);
+
+ tokens.push_back(ccTok);
+
+ Token fmtTok;
+ fmtTok.startToken();
+ fmtTok.setLocation(loc);
+ fmtTok.setIdentifierInfo(PP.getIdentifierInfo("format"));
+ fmtTok.setKind(tok::identifier);
+ tokens.push_back(fmtTok);
+
+ Token lpTok;
+ lpTok.startToken();
+ lpTok.setLocation(loc);
+ lpTok.setKind(tok::l_paren);
+ tokens.push_back(lpTok);
+
+ // First loop emits the literals. For f-literals it creates a new string-literal token to prepare for the regular concatenation
+ // in the recursive call.
+ for (const Token &t : StringToks) {
+ if (t.isFLiteral()) {
+ Token litTok = t;
+ litTok.clearFlag(Token::IsFLiteral);
+ litTok.setLiteralData(t.getLiteralData()); // Note: In t, which is a f-literal, the actual chars are in FLiteralData::Text.
+ tokens.push_back(litTok);
+ }
+ else
+ tokens.push_back(t); // Regular literals are already as they should
+ }
+
+ // Second loop appends all the extracted expressions. These already have leading commas and source location set up.
+ for (const Token &Tok : StringToks) {
+ if (Tok.isFLiteral())
+ tokens.insert(
+ tokens.end(), Tok.getFLiteralInfo().TokenPtr,
+ Tok.getFLiteralInfo().TokenPtr + Tok.getFLiteralInfo().TokenCount);
+ }
+
+ // Trailing rparen.
+ Token rpTok;
+ rpTok.startToken();
+ rpTok.setLocation(loc);
+ rpTok.setKind(tok::r_paren);
+ tokens.push_back(rpTok);
+
+ // Inject the upcoming token last to not change the token order.
+ tokens.push_back(Tok);
+
+ // Convert vector to unique_ptr and push the token sequence back into the preprocessor
+
+ auto tokStore = std::make_unique<Token[]>(tokens.size());
+ std::copy(tokens.begin(), tokens.end(), tokStore.get());
+
+ PP.EnterTokenStream(std::move(tokStore), tokens.size(), true, false);
+ ConsumeAnyToken(); // Consume the token we moved last, and make the leadiung :: we injected the current token.
+ return ParseCastExpression(AnyCastExpr, false);
}
/// ParseGenericSelectionExpression - Parse a C11 generic-selection
>From 944f236e0b6d42396805ce4bea7788912897d17a Mon Sep 17 00:00:00 2001
From: "bengt.gustafsson" <bengt.gustafsson at beamways.com>
Date: Fri, 4 Oct 2024 18:26:36 +0200
Subject: [PATCH 07/17] Fixed a bug in the handling of }} and {{ in nested
expression-fields.
Create call to make_formatted_string in parsing.
First try at formulating the library support.
---
clang/lib/Lex/Lexer.cpp | 13 ++--
clang/lib/Parse/ParseExpr.cpp | 2 +-
libcxx/include/__format/formatted_string.h | 84 ++++++++++++++++++++++
libcxx/include/format | 1 +
4 files changed, 91 insertions(+), 9 deletions(-)
create mode 100644 libcxx/include/__format/formatted_string.h
diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp
index 4d9ddc54cd5ce..28e781b2f951e 100644
--- a/clang/lib/Lex/Lexer.cpp
+++ b/clang/lib/Lex/Lexer.cpp
@@ -2384,8 +2384,8 @@ bool clang::Lexer::processExtractionField(std::vector<Token>& tokens, const char
lit += *BufferPtr++;
};
- while (*BufferPtr != '}') {
- if (*BufferPtr == '{') { // Nested field starts
+ while (true) {
+ if (*BufferPtr == '{') { // Nested expression-field starts
toLit();
if (*BufferPtr != '{') {
// Add a comma in the token stream before the expression tokens.
@@ -2406,19 +2406,16 @@ bool clang::Lexer::processExtractionField(std::vector<Token>& tokens, const char
}
else if (*BufferPtr == '}') {
toLit();
- if (*BufferPtr != '}') {
- Diag(BufferPtr, diag::err_unescaped_r_brace);
- return false;
- }
+ if (*BufferPtr != '}')
+ return true;
}
else
toLit(); // Transfer other formatting argument char to the resulting
// string.
}
- toLit(); // Transfer the final } and advance PufferPtr.
- return true;
}
+
Token clang::Lexer::processExpression(std::vector<Token>& tokens, bool allowComma)
{
const char *lpos = BufferPtr - 1;
diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp
index 1d474f8ed8113..9070bfb0e72cd 100644
--- a/clang/lib/Parse/ParseExpr.cpp
+++ b/clang/lib/Parse/ParseExpr.cpp
@@ -3426,7 +3426,7 @@ ExprResult Parser::ParseStringLiteralExpression(bool AllowUserDefinedLiteral,
Token fmtTok;
fmtTok.startToken();
fmtTok.setLocation(loc);
- fmtTok.setIdentifierInfo(PP.getIdentifierInfo("format"));
+ fmtTok.setIdentifierInfo(PP.getIdentifierInfo("make_formatted_string"));
fmtTok.setKind(tok::identifier);
tokens.push_back(fmtTok);
diff --git a/libcxx/include/__format/formatted_string.h b/libcxx/include/__format/formatted_string.h
new file mode 100644
index 0000000000000..d3906262ee25e
--- /dev/null
+++ b/libcxx/include/__format/formatted_string.h
@@ -0,0 +1,84 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FORMAT_FORMATTED_STRING_H
+#define _LIBCPP___FORMAT_FORMATTED_STRING_H
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <typename CharT> struct __select_format_context {};
+template <> struct __select_format_context<char> {
+ using type = std::format_context;
+};
+template <> struct __select_format_context<wchar_t> {
+ using type = std::wformat_context;
+};
+
+template <typename CharT> using __select_format_context_t = __select_format_context<CharT>::type;
+
+template <typename CharT, typename... Args>
+struct basic_formatted_string {
+ basic_formatted_string(std::basic_format_string<CharT, Args...> fmt, const Args&... as)
+ : args(std::make_format_args<__select_format_context_t<CharT>>(as...)), literal(fmt.get()) {}
+
+ operator std::string() const { return std::vformat(literal, args); }
+
+ decltype(std::make_format_args<__select_format_context_t<CharT>>(std::declval<const Args&>()...)) args;
+ std::basic_string_view<CharT> literal;
+};
+
+template <typename... Args>
+auto make_formatted_string(std::format_string<Args...> fmt, Args&&... as) {
+ return basic_formatted_string<char, Args...>(fmt, as...);
+}
+template <typename... Args>
+auto make_formatted_string(std::wformat_string<Args...> fmt, Args&&... as) {
+ return basic_formatted_string<wchar_t, Args...>(fmt, as...);
+}
+
+template <typename CharT, typename... Args>
+std::basic_ostream<CharT>& operator<<(std::basic_ostream<CharT>& os, const basic_formatted_string<CharT, Args...>& fs) {
+ std::vformat_to(std::ostream_iterator<CharT, CharT>(os), fs.literal, fs.args);
+ return os;
+}
+
+template <typename CharT, typename... Args>
+void print(std::basic_ostream<CharT>& os, const basic_formatted_string<CharT, Args...>& fs) {
+ os << fs;
+}
+template <typename CharT, typename... Args>
+void println(std::basic_ostream<CharT>& os, const basic_formatted_string<CharT, Args...>& fs) {
+ os << fs << std::endl; // Or just \n?
+}
+
+template <typename... Args>
+void print(const basic_formatted_string<char, Args...>& fs) {
+ print(std::cout, fs);
+}
+
+template <typename... Args>
+void print(const basic_formatted_string<wchar_t, Args...>& fs) {
+ print(std::wcout, fs);
+}
+
+template <typename... Args>
+void println(const basic_formatted_string<char, Args...>& fs) {
+ println(std::cout, fs);
+}
+
+template <typename... Args>
+void println(const basic_formatted_string<wchar_t, Args...>& fs) {
+ println(std::wcout, fs);
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#error here
+
+#endif
diff --git a/libcxx/include/format b/libcxx/include/format
index 07c2ba083199e..a7710d3d953c5 100644
--- a/libcxx/include/format
+++ b/libcxx/include/format
@@ -203,6 +203,7 @@ namespace std {
#include <__format/format_functions.h>
#include <__format/format_parse_context.h>
#include <__format/format_string.h>
+#include <__format/formatted_string.h>
#include <__format/format_to_n_result.h>
#include <__format/formatter.h>
#include <__format/formatter_bool.h>
>From de0243ac349bc5affc49b54ba15f7ddf5c7d56e0 Mon Sep 17 00:00:00 2001
From: "bengt.gustafsson" <bengt.gustafsson at beamways.com>
Date: Fri, 4 Oct 2024 18:36:52 +0200
Subject: [PATCH 08/17] Add a non-template operator new to tease gcc to compile
lexer.cpp.
---
clang/lib/Lex/Lexer.cpp | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp
index 28e781b2f951e..65ed8389c26c4 100644
--- a/clang/lib/Lex/Lexer.cpp
+++ b/clang/lib/Lex/Lexer.cpp
@@ -2604,7 +2604,13 @@ bool clang::Lexer::LexStringOrCharLiteral(Token &Result, const char *CurPtr,
return false;
}
-
+// gcc does not seem to be able to find the generic operator new in Allocator.h
+void *operator new(
+ size_t Size,
+ llvm::BumpPtrAllocator &Allocator) {
+ return Allocator.Allocate(Size, std::min((size_t)llvm::NextPowerOf2(Size),
+ alignof(std::max_align_t)));
+}
bool clang::Lexer::LexFLiteral(Token &Result, const char *CurPtr, tok::TokenKind Kind) {
const char *litStart = BufferPtr; // Before all prefixes
>From 099772ed517fd801c267a02c80dfde84f87ea952 Mon Sep 17 00:00:00 2001
From: "bengt.gustafsson" <bengt.gustafsson at beamways.com>
Date: Fri, 4 Oct 2024 18:44:32 +0200
Subject: [PATCH 09/17] Tried with an include instead.
---
clang/lib/Lex/Lexer.cpp | 15 ++++++++-------
1 file changed, 8 insertions(+), 7 deletions(-)
diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp
index 65ed8389c26c4..874b705fdcb9f 100644
--- a/clang/lib/Lex/Lexer.cpp
+++ b/clang/lib/Lex/Lexer.cpp
@@ -37,6 +37,7 @@
#include "llvm/Support/NativeFormatting.h"
#include "llvm/Support/Unicode.h"
#include "llvm/Support/UnicodeCharRanges.h"
+#include "llvm/Support/Allocator.h"
#include <algorithm>
#include <cassert>
#include <cstddef>
@@ -2604,13 +2605,13 @@ bool clang::Lexer::LexStringOrCharLiteral(Token &Result, const char *CurPtr,
return false;
}
-// gcc does not seem to be able to find the generic operator new in Allocator.h
-void *operator new(
- size_t Size,
- llvm::BumpPtrAllocator &Allocator) {
- return Allocator.Allocate(Size, std::min((size_t)llvm::NextPowerOf2(Size),
- alignof(std::max_align_t)));
-}
+//// gcc does not seem to be able to find the generic operator new in Allocator.h
+//void *operator new(
+// size_t Size,
+// llvm::BumpPtrAllocator &Allocator) {
+// return Allocator.Allocate(Size, std::min((size_t)llvm::NextPowerOf2(Size),
+// alignof(std::max_align_t)));
+//}
bool clang::Lexer::LexFLiteral(Token &Result, const char *CurPtr, tok::TokenKind Kind) {
const char *litStart = BufferPtr; // Before all prefixes
>From 51b94c083067243d75a415722ff178bf3344518a Mon Sep 17 00:00:00 2001
From: "bengt.gustafsson" <bengt.gustafsson at beamways.com>
Date: Fri, 4 Oct 2024 19:10:52 +0200
Subject: [PATCH 10/17] Don't use array new for the pp allocator, does not work
on gcc.
---
clang/lib/Lex/Lexer.cpp | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp
index 874b705fdcb9f..fdbdaee4fc5dc 100644
--- a/clang/lib/Lex/Lexer.cpp
+++ b/clang/lib/Lex/Lexer.cpp
@@ -2706,7 +2706,7 @@ bool clang::Lexer::LexFLiteral(Token &Result, const char *CurPtr, tok::TokenKind
// Fill in the literalPos token with the remaining string. But to keep its contents first allocate
// using the bumpAllocator from PP.
- char *lit = new (PP->getPreprocessorAllocator()) char[resultingLiteral.size() + 1];
+ char *lit = PP->getPreprocessorAllocator().Allocate<char>(resultingLiteral.size() + 1);
memcpy(lit, resultingLiteral.c_str(), resultingLiteral.size() + 1);
// The completed lexing is represented by a MacroInfo object which is pushed
@@ -2715,7 +2715,7 @@ bool clang::Lexer::LexFLiteral(Token &Result, const char *CurPtr, tok::TokenKind
BufferPtr = CurPtr;
- Token* tokStore = new (PP->getPreprocessorAllocator()) Token[tokens.size()];
+ Token* tokStore = PP->getPreprocessorAllocator().Allocate<Token>(tokens.size());
std::copy(tokens.begin(), tokens.end(), tokStore);
Token::FLiteralInfo* fli = new (PP->getPreprocessorAllocator()) Token::FLiteralInfo;
@@ -2863,7 +2863,8 @@ bool clang::Lexer::LexRawFLiteral(Token &Result, const char *CurPtr, tok::TokenK
// Fill in the literalPos token with the remaining string. But to keep its contents first allocate
// using the bumpAllocator from PP.
- char *lit = new (PP->getPreprocessorAllocator()) char[resultingLiteral.size() + 1];
+ char *lit = PP->getPreprocessorAllocator().Allocate<char>(
+ resultingLiteral.size() + 1);
memcpy(lit, resultingLiteral.c_str(), resultingLiteral.size() + 1);
// The completed lexing is represented by a MacroInfo object which is pushed
@@ -2872,7 +2873,8 @@ bool clang::Lexer::LexRawFLiteral(Token &Result, const char *CurPtr, tok::TokenK
BufferPtr = CurPtr;
- Token* tokStore = new (PP->getPreprocessorAllocator()) Token[tokens.size()];
+ Token *tokStore =
+ PP->getPreprocessorAllocator().Allocate<Token>(tokens.size());
std::copy(tokens.begin(), tokens.end(), tokStore);
Token::FLiteralInfo* fli = new (PP->getPreprocessorAllocator()) Token::FLiteralInfo;
>From d478e7f8b7df59403fb1aa8162f8440b22f62db1 Mon Sep 17 00:00:00 2001
From: "bengt.gustafsson" <bengt.gustafsson at beamways.com>
Date: Sat, 5 Oct 2024 16:16:53 +0200
Subject: [PATCH 11/17] Adjusted {{and }} handling in format-specifiers.
Implemented doubling of braces in non-f literals when concatenated with f-literals.
---
clang/lib/Lex/Lexer.cpp | 18 +++++++++---------
clang/lib/Parse/ParseExpr.cpp | 32 ++++++++++++++++++++++++++++----
2 files changed, 37 insertions(+), 13 deletions(-)
diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp
index 513ea11840114..6d1ddc0b264f1 100644
--- a/clang/lib/Lex/Lexer.cpp
+++ b/clang/lib/Lex/Lexer.cpp
@@ -2380,14 +2380,10 @@ bool clang::Lexer::processExtractionField(std::vector<Token>& tokens, const char
lit.push_back(':');
- // If : check for nested fields in the format-spec. This works character by character.
- auto toLit = [&] {
- lit += *BufferPtr++;
- };
-
+ // If : check for nested fields in the format-spec. This works character by character. Also replace {{ with { and }} with }.
while (true) {
if (*BufferPtr == '{') { // Nested expression-field starts
- toLit();
+ lit.push_back(*BufferPtr++);
if (*BufferPtr != '{') {
// Add a comma in the token stream before the expression tokens.
cTok.setLocation(getSourceLocation());
@@ -2404,15 +2400,19 @@ bool clang::Lexer::processExtractionField(std::vector<Token>& tokens, const char
lit.push_back('}'); // The } after the nested expression field
}
+ else
+ BufferPtr++; // Pass second {, thus getting rid of the doubling.
}
else if (*BufferPtr == '}') {
- toLit();
+ lit.push_back(*BufferPtr++);
if (*BufferPtr != '}')
return true;
+ else
+ BufferPtr++; // Pass second }, thus getting rid of the doubling.
}
else
- toLit(); // Transfer other formatting argument char to the resulting
- // string.
+ // Transfer other formatting argument char to the resulting string.
+ lit.push_back(*BufferPtr++);
}
}
diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp
index 0017870773856..585ad51b53f4d 100644
--- a/clang/lib/Parse/ParseExpr.cpp
+++ b/clang/lib/Parse/ParseExpr.cpp
@@ -3453,7 +3453,7 @@ ExprResult Parser::ParseStringLiteralExpression(bool AllowUserDefinedLiteral,
// Inject the preamble consisting of `::std::format(`
// TODO: Change this to std::formatted_string ctor call.
- // Form the prefix tokens.
+ // Form the prefix tokens.
auto loc = StringToks.front().getLocation();
Token ccTok;
@@ -3486,15 +3486,39 @@ ExprResult Parser::ParseStringLiteralExpression(bool AllowUserDefinedLiteral,
// First loop emits the literals. For f-literals it creates a new string-literal token to prepare for the regular concatenation
// in the recursive call.
- for (const Token &t : StringToks) {
+ for (Token &t : StringToks) {
if (t.isFLiteral()) {
Token litTok = t;
litTok.clearFlag(Token::IsFLiteral);
litTok.setLiteralData(t.getLiteralData()); // Note: In t, which is a f-literal, the actual chars are in FLiteralData::Text.
tokens.push_back(litTok);
+ } else {
+ // Check for doubling of { and } characters. If there aren't any no
+ // action is needed and no allocation. Count the number of { and } chars
+ // to see how many extra bytes are needed.
+ unsigned braces = 0;
+ const char *str = t.getLiteralData();
+ for (unsigned i = 0; i < t.getLength(); i++) {
+ if (str[i] == '{' || str[i] == '}')
+ braces++;
+ }
+ if (braces > 0) { // Rearranging needed to inject doubled braces.
+ char *newStr = PP.getPreprocessorAllocator().Allocate<char>(
+ t.getLength() + braces);
+ char *dest = newStr;
+ for (unsigned i = 0; i < t.getLength(); i++) {
+ *dest++ = str[i];
+ if (str[i] == '{')
+ *dest++ = '{';
+ if (str[i] == '}')
+ *dest++ = '}';
+ }
+
+ t.setLiteralData(newStr);
+ t.setLength(t.getLength() + braces);
+ }
+ tokens.push_back(t); // Add the literal
}
- else
- tokens.push_back(t); // Regular literals are already as they should
}
// Second loop appends all the extracted expressions. These already have leading commas and source location set up.
>From 3124ca25831dfc3c5ec4b8056ccbb7d7910a68a6 Mon Sep 17 00:00:00 2001
From: "bengt.gustafsson" <bengt.gustafsson at beamways.com>
Date: Sun, 6 Oct 2024 21:56:32 +0200
Subject: [PATCH 12/17] Handle also pre-11 C++ compile modes.
Copy formatted_string.h when installing.
---
clang/lib/Lex/Lexer.cpp | 14 ++++++++++++++
libcxx/include/CMakeLists.txt | 1 +
2 files changed, 15 insertions(+)
diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp
index 6d1ddc0b264f1..668d4e2db130e 100644
--- a/clang/lib/Lex/Lexer.cpp
+++ b/clang/lib/Lex/Lexer.cpp
@@ -2602,6 +2602,20 @@ bool clang::Lexer::LexStringOrCharLiteral(Token &Result, const char *CurPtr,
StringLiteralKind);
}
}
+ else if (StringLiteralKind == tok::string_literal || StringLiteralKind == tok::wide_string_literal) {
+ unsigned SizeTmp;
+ char Char = getCharAndSize(CurPtr, SizeTmp);
+
+ if (Char == '"')
+ return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result),
+ StringLiteralKind);
+
+ // character constant
+ if (Char == '\'')
+ return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result),
+ CharLiteralKind);
+ }
+
return false;
}
diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index 9bd1b41b8bfac..f0230747d13f7 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -382,6 +382,7 @@ set(files
__format/formatter_pointer.h
__format/formatter_string.h
__format/formatter_tuple.h
+ __format/formatted_string.h
__format/indic_conjunct_break_table.h
__format/parser_std_format_spec.h
__format/range_default_formatter.h
>From a04585ed0c7d5a740631995be5fb5cdc35dd3bcd Mon Sep 17 00:00:00 2001
From: "bengt.gustafsson" <bengt.gustafsson at beamways.com>
Date: Sun, 6 Oct 2024 23:02:30 +0200
Subject: [PATCH 13/17] Added include of iostream and iterator to
formatted_String.h
This is problably not kosher.
---
libcxx/include/__format/formatted_string.h | 3 +++
1 file changed, 3 insertions(+)
diff --git a/libcxx/include/__format/formatted_string.h b/libcxx/include/__format/formatted_string.h
index d3906262ee25e..49f8e8d4a1875 100644
--- a/libcxx/include/__format/formatted_string.h
+++ b/libcxx/include/__format/formatted_string.h
@@ -10,6 +10,9 @@
#ifndef _LIBCPP___FORMAT_FORMATTED_STRING_H
#define _LIBCPP___FORMAT_FORMATTED_STRING_H
+#include <iostream>
+#include <iterator>
+
_LIBCPP_BEGIN_NAMESPACE_STD
template <typename CharT> struct __select_format_context {};
>From 76f8aa374eef518d8d24671565a0a331b6376a8b Mon Sep 17 00:00:00 2001
From: "bengt.gustafsson" <bengt.gustafsson at beamways.com>
Date: Sun, 6 Oct 2024 23:18:28 +0200
Subject: [PATCH 14/17] removed #error
---
libcxx/include/__format/formatted_string.h | 2 --
1 file changed, 2 deletions(-)
diff --git a/libcxx/include/__format/formatted_string.h b/libcxx/include/__format/formatted_string.h
index 49f8e8d4a1875..68e5242202bd7 100644
--- a/libcxx/include/__format/formatted_string.h
+++ b/libcxx/include/__format/formatted_string.h
@@ -82,6 +82,4 @@ void println(const basic_formatted_string<wchar_t, Args...>& fs) {
_LIBCPP_END_NAMESPACE_STD
-#error here
-
#endif
>From 291453089e03faf013f474a3a0846532e5367ccf Mon Sep 17 00:00:00 2001
From: Bengt Gustafsson <bengt.gustafsson at beamways.com>
Date: Tue, 8 Oct 2024 21:45:44 +0200
Subject: [PATCH 15/17] Debugged the library additions. Note that these are
visible in C++23 mode as it doesn't seem that Clang sets the _LIBCPP_STD_VER
higher even with -std=c++2b
---
libcxx/include/__format/formatted_string.h | 60 ++++++----------------
libcxx/include/__ostream/print.h | 40 +++++++++++++++
libcxx/include/ostream | 7 +++
libcxx/include/print | 43 ++++++++++++++++
4 files changed, 106 insertions(+), 44 deletions(-)
diff --git a/libcxx/include/__format/formatted_string.h b/libcxx/include/__format/formatted_string.h
index 68e5242202bd7..79711af733441 100644
--- a/libcxx/include/__format/formatted_string.h
+++ b/libcxx/include/__format/formatted_string.h
@@ -10,8 +10,7 @@
#ifndef _LIBCPP___FORMAT_FORMATTED_STRING_H
#define _LIBCPP___FORMAT_FORMATTED_STRING_H
-#include <iostream>
-#include <iterator>
+#if _LIBCPP_STD_VER >= 23 // Should be 26
_LIBCPP_BEGIN_NAMESPACE_STD
@@ -25,61 +24,34 @@ template <> struct __select_format_context<wchar_t> {
template <typename CharT> using __select_format_context_t = __select_format_context<CharT>::type;
-template <typename CharT, typename... Args>
+template <typename CharT, typename... _Args>
struct basic_formatted_string {
- basic_formatted_string(std::basic_format_string<CharT, Args...> fmt, const Args&... as)
+ basic_formatted_string(std::basic_format_string<CharT, _Args...> fmt, const _Args&... as)
: args(std::make_format_args<__select_format_context_t<CharT>>(as...)), literal(fmt.get()) {}
operator std::string() const { return std::vformat(literal, args); }
- decltype(std::make_format_args<__select_format_context_t<CharT>>(std::declval<const Args&>()...)) args;
+ decltype(std::make_format_args<__select_format_context_t<CharT>>(std::declval<const _Args&>()...)) args;
std::basic_string_view<CharT> literal;
};
-template <typename... Args>
-auto make_formatted_string(std::format_string<Args...> fmt, Args&&... as) {
- return basic_formatted_string<char, Args...>(fmt, as...);
-}
-template <typename... Args>
-auto make_formatted_string(std::wformat_string<Args...> fmt, Args&&... as) {
- return basic_formatted_string<wchar_t, Args...>(fmt, as...);
-}
+template<typename... _Args>
+using formatted_string = basic_formatted_string<char, _Args...>;
-template <typename CharT, typename... Args>
-std::basic_ostream<CharT>& operator<<(std::basic_ostream<CharT>& os, const basic_formatted_string<CharT, Args...>& fs) {
- std::vformat_to(std::ostream_iterator<CharT, CharT>(os), fs.literal, fs.args);
- return os;
-}
+template<typename... _Args>
+using wformatted_string = basic_formatted_string<wchar_t, _Args...>;
-template <typename CharT, typename... Args>
-void print(std::basic_ostream<CharT>& os, const basic_formatted_string<CharT, Args...>& fs) {
- os << fs;
-}
-template <typename CharT, typename... Args>
-void println(std::basic_ostream<CharT>& os, const basic_formatted_string<CharT, Args...>& fs) {
- os << fs << std::endl; // Or just \n?
+template <typename... _Args>
+auto make_formatted_string(std::format_string<_Args...> fmt, _Args&&... as) {
+ return basic_formatted_string<char, _Args...>(fmt, as...);
}
-
-template <typename... Args>
-void print(const basic_formatted_string<char, Args...>& fs) {
- print(std::cout, fs);
-}
-
-template <typename... Args>
-void print(const basic_formatted_string<wchar_t, Args...>& fs) {
- print(std::wcout, fs);
-}
-
-template <typename... Args>
-void println(const basic_formatted_string<char, Args...>& fs) {
- println(std::cout, fs);
-}
-
-template <typename... Args>
-void println(const basic_formatted_string<wchar_t, Args...>& fs) {
- println(std::wcout, fs);
+template <typename... _Args>
+auto make_formatted_string(std::wformat_string<_Args...> fmt, _Args&&... as) {
+ return basic_formatted_string<wchar_t, _Args...>(fmt, as...);
}
_LIBCPP_END_NAMESPACE_STD
#endif
+
+#endif
diff --git a/libcxx/include/__ostream/print.h b/libcxx/include/__ostream/print.h
index 6c82b11e1de96..99f5ada3c86e7 100644
--- a/libcxx/include/__ostream/print.h
+++ b/libcxx/include/__ostream/print.h
@@ -177,6 +177,46 @@ _LIBCPP_HIDE_FROM_ABI inline void println(ostream& __os) {
# endif // _LIBCPP_STD_VER >= 23
+#if _LIBCPP_STD_VER >= 23 // SHould be 26
+
+template <typename... _Args>
+_LIBCPP_HIDE_FROM_ABI void print(ostream& __stream, const formatted_string<_Args...>& fs) {
+# ifndef _LIBCPP_HAS_NO_UNICODE
+ // Note the wording in the Standard is inefficient. The output of
+ // std::format is a std::string which is then copied. This solution
+ // just appends a newline at the end of the output.
+ if constexpr (__print::__use_unicode_execution_charset)
+ std::__vprint_unicode(__stream, fs.literal, fs.args, false);
+ else
+ std::__vprint_nonunicode(__stream, fs.literal, fs.args, false);
+# else // _LIBCPP_HAS_NO_UNICODE
+ std::::__vprint_nonunicode(__stream, fs.literal, fs.args, false);
+# endif // _LIBCPP_HAS_NO_UNICODE
+}
+
+template <typename... _Args>
+_LIBCPP_HIDE_FROM_ABI void println(ostream& __stream, const formatted_string<_Args...>& fs) {
+# ifndef _LIBCPP_HAS_NO_UNICODE
+ // Note the wording in the Standard is inefficient. The output of
+ // std::format is a std::string which is then copied. This solution
+ // just appends a newline at the end of the output.
+ if constexpr (__print::__use_unicode_execution_charset)
+ std::__vprint_unicode(__stream, fs.literal, fs.args, true);
+ else
+ std::__vprint_nonunicode(__stream, fs.literal, fs.args, true);
+# else // _LIBCPP_HAS_NO_UNICODE
+ std::__vprint_nonunicode(__stream, fs.literal, fs.args, true);
+# endif // _LIBCPP_HAS_NO_UNICODE
+}
+
+template <typename... _Args>
+ostream& operator<<(ostream& os, const formatted_string<_Args...>& fs) {
+ print(os, fs);
+ return os;
+}
+
+# endif // _LIBCPP_STD_VER >= 26
+
_LIBCPP_END_NAMESPACE_STD
#endif // !defined(_LIBCPP_HAS_NO_LOCALIZATION)
diff --git a/libcxx/include/ostream b/libcxx/include/ostream
index 83742882f7bb6..b67e74331234e 100644
--- a/libcxx/include/ostream
+++ b/libcxx/include/ostream
@@ -166,6 +166,13 @@ template<class... Args>
void println(ostream& os, format_string<Args...> fmt, Args&&... args);
void println(ostream& os); // since C++26
+ostream& operator<<(ostream& os, const formated_string<Args...>& fs); // since C++26
+
+template<class... Args> // since C++26
+void print(ostream& os, const formated_string<Args...>& fs);
+template<class... Args> // since C++26
+void println(ostream& os, const formated_string<Args...>& fs);
+
void vprint_unicode(ostream& os, string_view fmt, format_args args); // since C++23
void vprint_nonunicode(ostream& os, string_view fmt, format_args args); // since C++23
} // std
diff --git a/libcxx/include/print b/libcxx/include/print
index 2798a6bda2626..d41068efbd640 100644
--- a/libcxx/include/print
+++ b/libcxx/include/print
@@ -397,6 +397,49 @@ _LIBCPP_HIDE_FROM_ABI inline void vprint_nonunicode(string_view __fmt, format_ar
#endif // _LIBCPP_STD_VER >= 23
+#if _LIBCPP_STD_VER >= 23 // Should be 26
+
+template <typename... _Args>
+_LIBCPP_HIDE_FROM_ABI void print(FILE* __stream, const formatted_string<_Args...>& fs) {
+# ifndef _LIBCPP_HAS_NO_UNICODE
+ // Note the wording in the Standard is inefficient. The output of
+ // std::format is a std::string which is then copied. This solution
+ // just appends a newline at the end of the output.
+ if constexpr (__print::__use_unicode_execution_charset)
+ __print::__vprint_unicode(__stream, fs.literal, fs.args, false);
+ else
+ __print::__vprint_nonunicode(__stream, fs.literal, fs.args, false);
+# else // _LIBCPP_HAS_NO_UNICODE
+ __print::__vprint_nonunicode(__stream, fs.literal, fs.args, false);
+# endif // _LIBCPP_HAS_NO_UNICODE
+}
+template <typename... _Args>
+_LIBCPP_HIDE_FROM_ABI void println(FILE* __stream, const formatted_string<_Args...>& fs) {
+# ifndef _LIBCPP_HAS_NO_UNICODE
+ // Note the wording in the Standard is inefficient. The output of
+ // std::format is a std::string which is then copied. This solution
+ // just appends a newline at the end of the output.
+ if constexpr (__print::__use_unicode_execution_charset)
+ __print::__vprint_unicode(__stream, fs.literal, fs.args, true);
+ else
+ __print::__vprint_nonunicode(__stream, fs.literal, fs.args, true);
+# else // _LIBCPP_HAS_NO_UNICODE
+ __print::__vprint_nonunicode(__stream, fs.literal, fs.args, true);
+# endif // _LIBCPP_HAS_NO_UNICODE
+}
+
+template <typename... _Args>
+_LIBCPP_HIDE_FROM_ABI void print(const basic_formatted_string<char, _Args...>& fs) {
+ print(stdout, fs);
+}
+
+template <typename... _Args>
+_LIBCPP_HIDE_FROM_ABI void println(const basic_formatted_string<char, _Args...>& fs) {
+ println(stdout, fs);
+}
+
+#endif // _LIBCPP_STD_VER >= 26
+
_LIBCPP_END_NAMESPACE_STD
#endif // _LIBCPP_PRINT
>From dd7fff57a1e05d3d5153490aecf0abacad368bb7 Mon Sep 17 00:00:00 2001
From: Bengt Gustafsson <bengt.gustafsson at beamways.com>
Date: Wed, 9 Oct 2024 11:32:44 +0200
Subject: [PATCH 16/17] Tried to show an error if operator= ends an expression.
---
clang/include/clang/Basic/DiagnosticIDs.h | 2 +-
clang/include/clang/Basic/DiagnosticLexKinds.td | 1 +
clang/lib/Lex/Lexer.cpp | 12 ++++++++++--
3 files changed, 12 insertions(+), 3 deletions(-)
diff --git a/clang/include/clang/Basic/DiagnosticIDs.h b/clang/include/clang/Basic/DiagnosticIDs.h
index a051af327de28..28d0a54763855 100644
--- a/clang/include/clang/Basic/DiagnosticIDs.h
+++ b/clang/include/clang/Basic/DiagnosticIDs.h
@@ -36,7 +36,7 @@ namespace clang {
DIAG_SIZE_DRIVER = 400,
DIAG_SIZE_FRONTEND = 200,
DIAG_SIZE_SERIALIZATION = 120,
- DIAG_SIZE_LEX = 400,
+ DIAG_SIZE_LEX = 500,
DIAG_SIZE_PARSE = 700,
DIAG_SIZE_AST = 300,
DIAG_SIZE_COMMENT = 100,
diff --git a/clang/include/clang/Basic/DiagnosticLexKinds.td b/clang/include/clang/Basic/DiagnosticLexKinds.td
index 6c1ef4235bf1c..3db692f5b2245 100644
--- a/clang/include/clang/Basic/DiagnosticLexKinds.td
+++ b/clang/include/clang/Basic/DiagnosticLexKinds.td
@@ -88,6 +88,7 @@ def warn_c23_keyword : Warning<"'%0' is a keyword in C23">,
def ext_unterminated_char_or_string : ExtWarn<
"missing terminating %select{'|'\"'}0 character">, InGroup<InvalidPPToken>;
+def err_operator_equal_end : Error<"expression field ends in 'operator='. This is not allowed as expressions ending in = indicate that the expression is its own label">;
def err_nested_format_spec : Error<"format specifier in nested expression field">;
def err_unmatched_paren : Error<"unmatched %0 in expression field">;
def err_incomplete_ternary : Error<"ternary ? operator without matching : in expression field">;
diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp
index 668d4e2db130e..7edf591461885 100644
--- a/clang/lib/Lex/Lexer.cpp
+++ b/clang/lib/Lex/Lexer.cpp
@@ -2360,9 +2360,17 @@ bool clang::Lexer::processExtractionField(std::vector<Token>& tokens, const char
// Handle the extra feature that an expression ending with = is its own label (as in Python). No expression ending with =
// is valid anyway. Well, this is not strictly true: &MyClass::operator= is a valid expression but for the greater good we ignore this.
- // If anyone wants to emit this operator's address they can use a parenthesis. (Or we could specify that 'opewrator=' does not c�use
+ // If anyone wants to emit this operator's address they can use a parenthesis. (Or we could specify that 'operator=' does not cause
// label output.
- if (tokens.back().getKind() == tok::equal && (tokens.size() < 2 || tokens[tokens.size() - 2].getKind() != tok::kw_operator)) {
+ // As you can't format a member function pointer address anyway it is mostly a matter of which error message you want. By not
+ // checking for an operator token we get an error message about an expression ending with the token operator, which is probably
+ // more understandable as users will soon realize that the = was treated as a label indicator.
+ if (tokens.back().getKind() == tok::equal) {
+ if (tokens.size() >= 2 && tokens[tokens.size() - 2].getKind() == tok::kw_operator) {
+ Diag(BufferPtr - 1, diag::err_operator_equal_end);
+ return false;
+ }
+
lit.append(litStart, exprStart - 1); // Exclude the { which is to come after the label
lit.append(exprStart, BufferPtr - 1); // Add entire expression including the = and any spaces after that to the literal.
lit += "{"; // Add the { after the = and any trailing spaces.
>From c6a8500e2545ea30e9fd983d0d668dae30f3a31f Mon Sep 17 00:00:00 2001
From: "bengt.gustafsson" <bengt.gustafsson at beamways.com>
Date: Wed, 9 Oct 2024 22:58:22 +0200
Subject: [PATCH 17/17] Add parens around expression-fields when extracted.
Handle the keyword operator aferr :: in addtion to identfiier.
---
clang/lib/Lex/Lexer.cpp | 165 +++++++++++++++++++++++-----------------
1 file changed, 96 insertions(+), 69 deletions(-)
diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp
index 7edf591461885..ec9875167398f 100644
--- a/clang/lib/Lex/Lexer.cpp
+++ b/clang/lib/Lex/Lexer.cpp
@@ -2347,81 +2347,108 @@ bool Lexer::LexRawStringLiteral(Token &Result, const char *CurPtr,
// Process and extraction field in a f-literal. Add the tokens of the expresion(s) to tokens, and add the literal fragments to lit,
// if any. Start at BufferPtr, and leave it after the field, i.e. after the } of the extraction field. litstart is the position after the
// starting " of the literal or the } of the previous extraction field.
-bool clang::Lexer::processExtractionField(std::vector<Token>& tokens, const char* litStart, std::string& lit) {
- // Add the comma that should go before each extracted expression.
- Token cTok;
- cTok.startToken();
- cTok.setKind(tok::comma);
- cTok.setLocation(getSourceLocation());
- tokens.push_back(cTok);
-
- const char* exprStart = BufferPtr;
- Token terminator = processExpression(tokens, false); // Parse the part before the colon or }, returning the : or } token itself.
-
- // Handle the extra feature that an expression ending with = is its own label (as in Python). No expression ending with =
- // is valid anyway. Well, this is not strictly true: &MyClass::operator= is a valid expression but for the greater good we ignore this.
- // If anyone wants to emit this operator's address they can use a parenthesis. (Or we could specify that 'operator=' does not cause
- // label output.
- // As you can't format a member function pointer address anyway it is mostly a matter of which error message you want. By not
- // checking for an operator token we get an error message about an expression ending with the token operator, which is probably
- // more understandable as users will soon realize that the = was treated as a label indicator.
- if (tokens.back().getKind() == tok::equal) {
- if (tokens.size() >= 2 && tokens[tokens.size() - 2].getKind() == tok::kw_operator) {
- Diag(BufferPtr - 1, diag::err_operator_equal_end);
- return false;
- }
-
- lit.append(litStart, exprStart - 1); // Exclude the { which is to come after the label
- lit.append(exprStart, BufferPtr - 1); // Add entire expression including the = and any spaces after that to the literal.
- lit += "{"; // Add the { after the = and any trailing spaces.
- tokens.pop_back(); // Remove the =
- }
- else {
- lit.append(litStart, exprStart); // Add entire expression including the = and any spaces after that to the literal.
+bool clang::Lexer::processExtractionField(std::vector<Token> &tokens,
+ const char *litStart,
+ std::string &lit) {
+ // Add the comma that should go before each extracted expression.
+ Token cTok;
+ cTok.startToken();
+ cTok.setKind(tok::comma);
+ cTok.setLocation(getSourceLocation());
+ tokens.push_back(cTok);
+
+ Token lpTok;
+ lpTok.startToken();
+ lpTok.setLocation(getSourceLocation());
+ lpTok.setKind(tok::l_paren);
+ tokens.push_back(lpTok);
+
+ Token rpTok;
+ rpTok.startToken();
+ rpTok.setLocation(getSourceLocation());
+ rpTok.setKind(tok::r_paren);
+
+ const char *exprStart = BufferPtr;
+ Token terminator =
+ processExpression(tokens, false); // Parse the part before the colon or },
+ // returning the : or } token itself.
+
+ // Handle the extra feature that an expression ending with = is its own
+ // label (as in Python). No expression ending with = is valid anyway. Well,
+ // this is not strictly true: &MyClass::operator= is a valid expression but
+ // for the greater good we ignore this. If anyone wants to emit this
+ // operator's address they can use a parenthesis. (Or we could specify that
+ // 'operator=' does not cause label output. As you can't format a member
+ // function pointer address anyway it is mostly a matter of which error
+ // message you want. By not checking for an operator token we get an error
+ // message about an expression ending with the token operator, which is
+ // probably more understandable as users will soon realize that the = was
+ // treated as a label indicator.
+ if (tokens.back().getKind() == tok::equal) {
+ if (tokens.size() >= 2 &&
+ tokens[tokens.size() - 2].getKind() == tok::kw_operator) {
+ Diag(BufferPtr - 1, diag::err_operator_equal_end);
+ return false;
}
- if (terminator.getKind() != tok::colon) { // This includes unknown, i.e. a " before }
- assert(terminator.getKind() == tok::r_brace || terminator.getKind() == tok::unknown);
- lit.push_back('}');
- return terminator.getKind() != tok::unknown;
- }
+ lit.append(litStart,
+ exprStart - 1); // Exclude the { which is to come after the label
+ lit.append(exprStart,
+ BufferPtr - 1); // Add entire expression including the = and
+ // any spaces after that to the literal.
+ lit += "{"; // Add the { after the = and any trailing spaces.
+ tokens.pop_back(); // Remove the =
+ } else {
+ lit.append(litStart,
+ exprStart); // Add entire expression including the = and any
+ // spaces after that to the literal.
+ }
- lit.push_back(':');
+ tokens.push_back(rpTok);
+ if (terminator.getKind() !=
+ tok::colon) { // This includes unknown, i.e. a " before }
+ assert(terminator.getKind() == tok::r_brace ||
+ terminator.getKind() == tok::unknown);
+ lit.push_back('}');
+ return terminator.getKind() != tok::unknown;
+ }
- // If : check for nested fields in the format-spec. This works character by character. Also replace {{ with { and }} with }.
- while (true) {
- if (*BufferPtr == '{') { // Nested expression-field starts
- lit.push_back(*BufferPtr++);
- if (*BufferPtr != '{') {
- // Add a comma in the token stream before the expression tokens.
- cTok.setLocation(getSourceLocation());
- tokens.push_back(cTok);
- Token terminator = processExpression(tokens, false);
- if (terminator.getKind() == tok::unknown)
- return false;
-
- if (terminator.getKind() != tok::r_brace) {
- // Colon not allowed inside nested expression-field.
- Diag(BufferPtr, diag::err_nested_format_spec);
- return false;
- }
+ lit.push_back(':');
- lit.push_back('}'); // The } after the nested expression field
+ // If : check for nested fields in the format-spec. This works character by
+ // character. Also replace {{ with { and }} with }.
+ while (true) {
+ if (*BufferPtr == '{') { // Nested expression-field starts
+ lit.push_back(*BufferPtr++);
+ if (*BufferPtr != '{') {
+ // Add a comma in the token stream before the expression tokens.
+ cTok.setLocation(getSourceLocation());
+ tokens.push_back(cTok);
+ tokens.push_back(lpTok);
+ Token terminator = processExpression(tokens, false);
+ if (terminator.getKind() == tok::unknown)
+ return false;
+
+ if (terminator.getKind() != tok::r_brace) {
+ // Colon not allowed inside nested expression-field.
+ Diag(BufferPtr, diag::err_nested_format_spec);
+ return false;
}
- else
- BufferPtr++; // Pass second {, thus getting rid of the doubling.
- }
- else if (*BufferPtr == '}') {
- lit.push_back(*BufferPtr++);
- if (*BufferPtr != '}')
- return true;
- else
- BufferPtr++; // Pass second }, thus getting rid of the doubling.
- }
+
+ tokens.push_back(rpTok);
+ lit.push_back('}'); // The } after the nested expression field
+ } else
+ BufferPtr++; // Pass second {, thus getting rid of the doubling.
+ } else if (*BufferPtr == '}') {
+ lit.push_back(*BufferPtr++);
+ if (*BufferPtr != '}')
+ return true;
else
- // Transfer other formatting argument char to the resulting string.
- lit.push_back(*BufferPtr++);
- }
+ BufferPtr++; // Pass second }, thus getting rid of the doubling.
+ } else
+ // Transfer other formatting argument char to the resulting string.
+ lit.push_back(*BufferPtr++);
+ }
}
@@ -2476,7 +2503,7 @@ Token clang::Lexer::processExpression(std::vector<Token>& tokens, bool allowComm
Token ident;
PP->Lex(ident);
- if (ident.getKind() != tok::identifier) {
+ if (ident.getKind() != tok::identifier && ident.getKind() != tok::kw_operator) {
BufferPtr = save;
Token cTok;
More information about the libcxx-commits
mailing list