[flang-commits] [flang] aac53ad - [flang] Don't perform macro replacement on exponents (#136176)
via flang-commits
flang-commits at lists.llvm.org
Fri Apr 18 12:52:07 PDT 2025
Author: Peter Klausler
Date: 2025-04-18T12:52:04-07:00
New Revision: aac53ad4d6eafda84ca92196a62e75e96e5fad25
URL: https://github.com/llvm/llvm-project/commit/aac53ad4d6eafda84ca92196a62e75e96e5fad25
DIFF: https://github.com/llvm/llvm-project/commit/aac53ad4d6eafda84ca92196a62e75e96e5fad25.diff
LOG: [flang] Don't perform macro replacement on exponents (#136176)
See new test. I inadvertently broke this behavior with a recent fix for
another problem, because the effects of the overloaded
TokenSequence::Put() member function on token merging were confusing.
Rename and document the various overloads.
Added:
flang/test/Preprocessing/exponent-bug.F90
Modified:
flang/include/flang/Parser/token-sequence.h
flang/lib/Parser/preprocessor.cpp
flang/lib/Parser/prescan.cpp
flang/lib/Parser/token-sequence.cpp
Removed:
################################################################################
diff --git a/flang/include/flang/Parser/token-sequence.h b/flang/include/flang/Parser/token-sequence.h
index 047c0bed00762..69291e69526e2 100644
--- a/flang/include/flang/Parser/token-sequence.h
+++ b/flang/include/flang/Parser/token-sequence.h
@@ -35,10 +35,10 @@ class Prescanner;
class TokenSequence {
public:
TokenSequence() {}
- TokenSequence(const TokenSequence &that) { Put(that); }
+ TokenSequence(const TokenSequence &that) { CopyAll(that); }
TokenSequence(
const TokenSequence &that, std::size_t at, std::size_t count = 1) {
- Put(that, at, count);
+ AppendRange(that, at, count);
}
TokenSequence(TokenSequence &&that)
: start_{std::move(that.start_)}, nextStart_{that.nextStart_},
@@ -48,7 +48,7 @@ class TokenSequence {
TokenSequence &operator=(const TokenSequence &that) {
clear();
- Put(that);
+ CopyAll(that);
return *this;
}
TokenSequence &operator=(TokenSequence &&that);
@@ -100,14 +100,26 @@ class TokenSequence {
start_.pop_back();
}
- void Put(const TokenSequence &);
- void Put(const TokenSequence &, ProvenanceRange);
- void Put(const TokenSequence &, std::size_t at, std::size_t tokens = 1);
+ // These append characters with provenance and then close the token.
+ // When the last token of this sequence remains open beforehand,
+ // the new characters are appended to it.
void Put(const char *, std::size_t, Provenance);
void Put(const CharBlock &, Provenance);
void Put(const std::string &, Provenance);
void Put(llvm::raw_string_ostream &, Provenance);
+ // Appends a full copy of another sequence. When the last token of this
+ // sequence remains open beforehand, it is closed before the new text
+ // is appended.
+ void CopyAll(const TokenSequence &);
+ // Copies a range of tokens from another sequence. If the last token of
+ // this sequence remains open, the first token of the copied range will be
+ // appended to it.
+ void AppendRange(
+ const TokenSequence &, std::size_t at, std::size_t tokens = 1);
+ // Copies tokens (via Put above) with new provenance.
+ void CopyWithProvenance(const TokenSequence &, ProvenanceRange);
+
Provenance GetCharProvenance(std::size_t) const;
Provenance GetTokenProvenance(
std::size_t token, std::size_t offset = 0) const;
diff --git a/flang/lib/Parser/preprocessor.cpp b/flang/lib/Parser/preprocessor.cpp
index 7e6a1c2ca6977..a47f9c32ad27c 100644
--- a/flang/lib/Parser/preprocessor.cpp
+++ b/flang/lib/Parser/preprocessor.cpp
@@ -101,7 +101,7 @@ TokenSequence Definition::Tokenize(const std::vector<std::string> &argNames,
continue;
}
}
- result.Put(token, firstToken + j, 1);
+ result.AppendRange(token, firstToken + j, 1);
}
return result;
}
@@ -170,7 +170,7 @@ static TokenSequence TokenPasting(TokenSequence &&text) {
}
} else if (pasting && text.TokenAt(j).IsBlank()) {
} else {
- result.Put(text, j, 1);
+ result.AppendRange(text, j, 1);
pasting = false;
}
}
@@ -223,7 +223,7 @@ TokenSequence Definition::Apply(const std::vector<TokenSequence> &args,
CHECK(resultSize > 0 &&
result.TokenAt(resultSize - 1) == replacement_.TokenAt(prev - 1));
result.pop_back();
- result.Put(Stringify(args[index], prescanner.allSources()));
+ result.CopyAll(Stringify(args[index], prescanner.allSources()));
} else {
const TokenSequence *arg{&args[index]};
std::optional<TokenSequence> replaced;
@@ -243,7 +243,7 @@ TokenSequence Definition::Apply(const std::vector<TokenSequence> &args,
}
}
}
- result.Put(DEREF(arg));
+ result.CopyAll(DEREF(arg));
}
} else if (bytes == 11 && isVariadic_ &&
token.ToString() == "__VA_ARGS__") {
@@ -254,7 +254,7 @@ TokenSequence Definition::Apply(const std::vector<TokenSequence> &args,
if (k > argumentCount()) {
result.Put(","s, commaProvenance);
}
- result.Put(args[k]);
+ result.CopyAll(args[k]);
}
} else if (bytes == 10 && isVariadic_ && token.ToString() == "__VA_OPT__" &&
j + 2 < tokens && replacement_.TokenAt(j + 1).OnlyNonBlank() == '(' &&
@@ -274,7 +274,7 @@ TokenSequence Definition::Apply(const std::vector<TokenSequence> &args,
}
}
}
- result.Put(replacement_, j);
+ result.AppendRange(replacement_, j);
}
}
return TokenPasting(std::move(result));
@@ -338,18 +338,18 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
inIfExpression](std::size_t after, const TokenSequence &replacement,
std::size_t pFLMOffset) {
if (after < input.SizeInTokens()) {
- result.Put(replacement, 0, pFLMOffset);
+ result.AppendRange(replacement, 0, pFLMOffset);
TokenSequence suffix;
- suffix.Put(
+ suffix.AppendRange(
replacement, pFLMOffset, replacement.SizeInTokens() - pFLMOffset);
- suffix.Put(input, after, input.SizeInTokens() - after);
+ suffix.AppendRange(input, after, input.SizeInTokens() - after);
auto further{ReplaceMacros(
suffix, prescanner, partialFunctionLikeMacro, inIfExpression)};
if (partialFunctionLikeMacro && *partialFunctionLikeMacro) {
// still not closed
**partialFunctionLikeMacro += result.SizeInTokens();
}
- result.Put(further);
+ result.CopyAll(further);
return true;
} else {
if (partialFunctionLikeMacro) {
@@ -362,7 +362,7 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
for (; j < tokens; ++j) {
CharBlock token{input.TokenAt(j)};
if (token.IsBlank() || !IsLegalIdentifierStart(token[0])) {
- result.Put(input, j);
+ result.AppendRange(input, j);
continue;
}
// Process identifier in replacement text.
@@ -388,12 +388,12 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
}
}
if (it == definitions_.end()) {
- result.Put(input, j);
+ result.AppendRange(input, j);
continue;
}
Definition *def{&it->second};
if (def->isDisabled()) {
- result.Put(input, j);
+ result.AppendRange(input, j);
continue;
}
if (!def->isFunctionLike()) {
@@ -444,7 +444,7 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
ProvenanceRange use{input.GetTokenProvenanceRange(j)};
ProvenanceRange newRange{
allSources_.AddMacroCall(from, use, replaced.ToString())};
- result.Put(replaced, newRange);
+ result.CopyWithProvenance(replaced, newRange);
}
} else {
// Possible function-like macro call. Skip spaces and newlines to see
@@ -461,10 +461,10 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
if (!leftParen) {
if (partialFunctionLikeMacro) {
*partialFunctionLikeMacro = result.SizeInTokens();
- result.Put(input, j, tokens - j);
+ result.AppendRange(input, j, tokens - j);
return result;
} else {
- result.Put(input, j);
+ result.AppendRange(input, j);
continue;
}
}
@@ -491,11 +491,11 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
}
if (k >= tokens && partialFunctionLikeMacro) {
*partialFunctionLikeMacro = result.SizeInTokens();
- result.Put(input, j, tokens - j);
+ result.AppendRange(input, j, tokens - j);
return result;
} else if (k >= tokens || argStart.size() < def->argumentCount() ||
(argStart.size() > def->argumentCount() && !def->isVariadic())) {
- result.Put(input, j);
+ result.AppendRange(input, j);
continue;
}
std::vector<TokenSequence> args;
@@ -520,7 +520,7 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
ProvenanceRange use{input.GetIntervalProvenanceRange(j, k - j)};
ProvenanceRange newRange{
allSources_.AddMacroCall(from, use, replaced.ToString())};
- result.Put(replaced, newRange);
+ result.CopyWithProvenance(replaced, newRange);
}
j = k; // advance to the terminal ')'
}
diff --git a/flang/lib/Parser/prescan.cpp b/flang/lib/Parser/prescan.cpp
index 2db9fa0a937f3..46e04c15ade01 100644
--- a/flang/lib/Parser/prescan.cpp
+++ b/flang/lib/Parser/prescan.cpp
@@ -903,8 +903,7 @@ bool Prescanner::HandleExponent(TokenSequence &tokens) {
EmitCharAndAdvance(possible, *at_);
}
possible.CloseToken();
- tokens.CloseToken();
- tokens.Put(possible);
+ tokens.AppendRange(possible, 0); // appends to current token
return true;
}
// Not an exponent; backtrack
@@ -937,9 +936,9 @@ bool Prescanner::HandleKindSuffix(TokenSequence &tokens) {
preprocessor_.IsNameDefined(separate.TokenAt(1)) &&
!preprocessor_.IsNameDefined(withUnderscore.ToCharBlock())) {
// "_foo" is not defined, but "foo" is
- tokens.Put(separate); // '_' "foo"
+ tokens.CopyAll(separate); // '_' "foo"
} else {
- tokens.Put(withUnderscore); // "_foo"
+ tokens.CopyAll(withUnderscore); // "_foo"
}
return true;
}
@@ -1016,7 +1015,7 @@ void Prescanner::QuotedCharacterLiteral(
ppTokens.Put(id, GetProvenance(idStart));
if (auto replaced{
preprocessor_.MacroReplacement(ppTokens, *this)}) {
- tokens.Put(*replaced);
+ tokens.CopyAll(*replaced);
at_ = &idStart[idLen - 1];
NextLine();
continue; // try again on the next line
@@ -1836,7 +1835,7 @@ bool Prescanner::CompilerDirectiveContinuation(
}
if (ok) {
tokens.pop_back(); // delete original '&'
- tokens.Put(followingTokens, startAt, following - startAt);
+ tokens.AppendRange(followingTokens, startAt, following - startAt);
tokens.RemoveRedundantBlanks();
} else {
nextLine_ = origNextLine;
@@ -1866,7 +1865,7 @@ bool Prescanner::SourceLineContinuation(TokenSequence &tokens) {
}
followingTokens.RemoveRedundantBlanks();
tokens.pop_back(); // delete original '&'
- tokens.Put(followingTokens);
+ tokens.CopyAll(followingTokens);
return true;
}
}
diff --git a/flang/lib/Parser/token-sequence.cpp b/flang/lib/Parser/token-sequence.cpp
index c0655f608b97d..aee76938550f5 100644
--- a/flang/lib/Parser/token-sequence.cpp
+++ b/flang/lib/Parser/token-sequence.cpp
@@ -96,7 +96,7 @@ bool TokenSequence::IsAnythingLeft(std::size_t at) const {
return false;
}
-void TokenSequence::Put(const TokenSequence &that) {
+void TokenSequence::CopyAll(const TokenSequence &that) {
if (nextStart_ < char_.size()) {
start_.push_back(nextStart_);
}
@@ -109,7 +109,8 @@ void TokenSequence::Put(const TokenSequence &that) {
provenances_.Put(that.provenances_);
}
-void TokenSequence::Put(const TokenSequence &that, ProvenanceRange range) {
+void TokenSequence::CopyWithProvenance(
+ const TokenSequence &that, ProvenanceRange range) {
std::size_t offset{0};
std::size_t tokens{that.SizeInTokens()};
for (std::size_t j{0}; j < tokens; ++j) {
@@ -120,7 +121,7 @@ void TokenSequence::Put(const TokenSequence &that, ProvenanceRange range) {
CHECK(offset == range.size());
}
-void TokenSequence::Put(
+void TokenSequence::AppendRange(
const TokenSequence &that, std::size_t at, std::size_t tokens) {
ProvenanceRange provenance;
std::size_t offset{0};
@@ -246,7 +247,7 @@ TokenSequence &TokenSequence::RemoveBlanks(std::size_t firstChar) {
TokenSequence result;
for (std::size_t j{0}; j < tokens; ++j) {
if (!TokenAt(j).IsBlank() || start_[j] < firstChar) {
- result.Put(*this, j);
+ result.AppendRange(*this, j);
}
}
swap(result);
@@ -260,7 +261,7 @@ TokenSequence &TokenSequence::RemoveRedundantBlanks(std::size_t firstChar) {
for (std::size_t j{0}; j < tokens; ++j) {
bool isBlank{TokenAt(j).IsBlank()};
if (!isBlank || !lastWasBlank || start_[j] < firstChar) {
- result.Put(*this, j);
+ result.AppendRange(*this, j);
}
lastWasBlank = isBlank;
}
@@ -294,7 +295,7 @@ TokenSequence &TokenSequence::ClipComment(
} else {
TokenSequence result;
if (j > 0) {
- result.Put(*this, 0, j - 1);
+ result.AppendRange(*this, 0, j - 1);
}
swap(result);
return *this;
diff --git a/flang/test/Preprocessing/exponent-bug.F90 b/flang/test/Preprocessing/exponent-bug.F90
new file mode 100644
index 0000000000000..903b41bb846a4
--- /dev/null
+++ b/flang/test/Preprocessing/exponent-bug.F90
@@ -0,0 +1,5 @@
+! RUN: %flang -E %s 2>&1 | FileCheck %s
+! CHECK: 3.14159e00
+#define e00 e666
+print *, 3.14159e00
+end
More information about the flang-commits
mailing list