[flang-commits] [flang] Revert "[flang] Recognize compiler directives after expansion in comm… (PR #180982)
via flang-commits
flang-commits at lists.llvm.org
Wed Feb 11 10:12:04 PST 2026
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-flang-parser
Author: Peter Klausler (klausler)
<details>
<summary>Changes</summary>
…ent (#<!-- -->180062)"
This reverts commit 0d64801bc3b99a73d20032f74df3b87e0a7ed04e.
Lines like "!MACRO ... &" in which MACRO expands to a compiler directive are now failing because they are not being recognized as having line continuations. Will fix and try again.
---
Full diff: https://github.com/llvm/llvm-project/pull/180982.diff
5 Files Affected:
- (modified) flang/include/flang/Parser/preprocessor.h (+4-4)
- (modified) flang/lib/Parser/preprocessor.cpp (+8-12)
- (modified) flang/lib/Parser/prescan.cpp (+67-117)
- (modified) flang/lib/Parser/prescan.h (+3-9)
- (removed) flang/test/Preprocessing/bug178481.F90 (-7)
``````````diff
diff --git a/flang/include/flang/Parser/preprocessor.h b/flang/include/flang/Parser/preprocessor.h
index 5db2b020f879a..0405d42e64f7b 100644
--- a/flang/include/flang/Parser/preprocessor.h
+++ b/flang/include/flang/Parser/preprocessor.h
@@ -49,8 +49,8 @@ class Definition {
bool set_isDisabled(bool disable);
- TokenSequence Apply(const std::vector<TokenSequence> &args,
- const Prescanner &, bool inIfExpression = false) const;
+ TokenSequence Apply(const std::vector<TokenSequence> &args, Prescanner &,
+ bool inIfExpression = false);
void Print(llvm::raw_ostream &out, const char *macroName = "") const;
@@ -95,7 +95,7 @@ class Preprocessor {
// that result and try again. All other Fortran preprocessors share this
// behavior.
std::optional<TokenSequence> MacroReplacement(const TokenSequence &,
- const Prescanner &,
+ Prescanner &,
std::optional<std::size_t> *partialFunctionLikeMacro = nullptr,
bool inIfExpression = false);
@@ -109,7 +109,7 @@ class Preprocessor {
enum class CanDeadElseAppear { No, Yes };
CharBlock SaveTokenAsName(const CharBlock &);
- TokenSequence ReplaceMacros(const TokenSequence &, const Prescanner &,
+ TokenSequence ReplaceMacros(const TokenSequence &, Prescanner &,
std::optional<std::size_t> *partialFunctionLikeMacro = nullptr,
bool inIfExpression = false);
void SkipDisabledConditionalCode(
diff --git a/flang/lib/Parser/preprocessor.cpp b/flang/lib/Parser/preprocessor.cpp
index 3f9abf05d796b..529d2c345c112 100644
--- a/flang/lib/Parser/preprocessor.cpp
+++ b/flang/lib/Parser/preprocessor.cpp
@@ -214,7 +214,7 @@ constexpr bool IsDefinedKeyword(CharBlock token) {
}
TokenSequence Definition::Apply(const std::vector<TokenSequence> &args,
- const Prescanner &prescanner, bool inIfExpression) const {
+ Prescanner &prescanner, bool inIfExpression) {
TokenSequence result;
bool skipping{false};
int parenthesesNesting{0};
@@ -254,9 +254,7 @@ TokenSequence Definition::Apply(const std::vector<TokenSequence> &args,
CHECK(resultSize > 0 &&
result.TokenAt(resultSize - 1) == replacement_.TokenAt(prev - 1));
result.pop_back();
- AllSources &allSources{
- *const_cast<AllSources *>(&prescanner.allSources())};
- result.CopyAll(Stringify(args[index], allSources));
+ result.CopyAll(Stringify(args[index], prescanner.allSources()));
} else {
const TokenSequence *arg{&args[index]};
std::optional<TokenSequence> replaced;
@@ -269,9 +267,7 @@ TokenSequence Definition::Apply(const std::vector<TokenSequence> &args,
auto next{replacement_.SkipBlanks(j + 1)};
if (next >= tokens || !IsTokenPasting(replacement_.TokenAt(next))) {
// Apply macro replacement to the actual argument
- Preprocessor &preprocessor{
- *const_cast<Preprocessor *>(&prescanner.preprocessor())};
- replaced = preprocessor.MacroReplacement(
+ replaced = prescanner.preprocessor().MacroReplacement(
*arg, prescanner, nullptr, inIfExpression);
if (replaced) {
arg = &*replaced;
@@ -282,9 +278,9 @@ TokenSequence Definition::Apply(const std::vector<TokenSequence> &args,
}
} else if (bytes == 11 && isVariadic_ &&
token.ToString() == "__VA_ARGS__") {
- AllSources &allSources{
- *const_cast<AllSources *>(&prescanner.allSources())};
- Provenance commaProvenance{allSources.CompilerInsertionProvenance(',')};
+ Provenance commaProvenance{
+ prescanner.preprocessor().allSources().CompilerInsertionProvenance(
+ ',')};
for (std::size_t k{argumentCount()}; k < args.size(); ++k) {
if (k > argumentCount()) {
result.Put(","s, commaProvenance);
@@ -444,7 +440,7 @@ void Preprocessor::Define(const std::string ¯o, const std::string &value) {
void Preprocessor::Undefine(std::string macro) { definitions_.erase(macro); }
std::optional<TokenSequence> Preprocessor::MacroReplacement(
- const TokenSequence &input, const Prescanner &prescanner,
+ const TokenSequence &input, Prescanner &prescanner,
std::optional<std::size_t> *partialFunctionLikeMacro, bool inIfExpression) {
// Do quick scan for any use of a defined name.
if (!inIfExpression && definitions_.empty()) {
@@ -667,7 +663,7 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
}
TokenSequence Preprocessor::ReplaceMacros(const TokenSequence &tokens,
- const Prescanner &prescanner,
+ Prescanner &prescanner,
std::optional<std::size_t> *partialFunctionLikeMacro, bool inIfExpression) {
if (std::optional<TokenSequence> repl{MacroReplacement(
tokens, prescanner, partialFunctionLikeMacro, inIfExpression)}) {
diff --git a/flang/lib/Parser/prescan.cpp b/flang/lib/Parser/prescan.cpp
index 0d57c35ebc3c1..036b2f3fca3e1 100644
--- a/flang/lib/Parser/prescan.cpp
+++ b/flang/lib/Parser/prescan.cpp
@@ -212,14 +212,6 @@ void Prescanner::Statement() {
}
break;
}
- case LineClassification::Kind::CompilerDirectiveAfterMacroExpansion:
- BeginStatementAndAdvance();
- SkipSpaces();
- if (*at_ == '!') {
- tokens.Put(at_++, 1, GetCurrentProvenance());
- ++column_;
- }
- break;
case LineClassification::Kind::Source: {
BeginStatementAndAdvance();
bool checkLabelField{false};
@@ -246,17 +238,30 @@ void Prescanner::Statement() {
// a comment marker or directive sentinel. If so, disable line
// continuation, so that NextToken() won't consume anything from
// following lines.
- if (auto kwName{GetKeywordMacroName(at_)}) {
- checkLabelField = false;
- Provenance here{GetCurrentProvenance()};
- TokenSequence replacement{ExpandKeywordMacro(*kwName, here)};
- auto newLineClass{ClassifyLine(replacement, here)};
- if (newLineClass.kind == LineClassification::Kind::CompilerDirective) {
- directiveSentinel_ = newLineClass.sentinel;
- disableSourceContinuation_ = false;
- } else {
- disableSourceContinuation_ = !replacement.empty() &&
- newLineClass.kind != LineClassification::Kind::Source;
+ if (IsLegalIdentifierStart(*at_)) {
+ // TODO: Only bother with these cases when any keyword macro has
+ // been defined with replacement text that could begin a comment
+ // or directive sentinel.
+ const char *p{at_};
+ while (IsLegalInIdentifier(*++p)) {
+ }
+ CharBlock id{at_, static_cast<std::size_t>(p - at_)};
+ if (preprocessor_.IsNameDefined(id) &&
+ !preprocessor_.IsFunctionLikeDefinition(id)) {
+ checkLabelField = false;
+ TokenSequence toks;
+ toks.Put(id, GetProvenance(at_));
+ if (auto replaced{preprocessor_.MacroReplacement(toks, *this)}) {
+ auto newLineClass{ClassifyLine(*replaced, GetCurrentProvenance())};
+ if (newLineClass.kind ==
+ LineClassification::Kind::CompilerDirective) {
+ directiveSentinel_ = newLineClass.sentinel;
+ disableSourceContinuation_ = false;
+ } else {
+ disableSourceContinuation_ = !replaced->empty() &&
+ newLineClass.kind != LineClassification::Kind::Source;
+ }
+ }
}
}
if (checkLabelField) {
@@ -299,7 +304,6 @@ void Prescanner::Statement() {
CheckAndEmitLine(preprocessed->ToLowerCase(), newlineProvenance);
break;
case LineClassification::Kind::CompilerDirective:
- case LineClassification::Kind::CompilerDirectiveAfterMacroExpansion:
if (preprocessed->HasRedundantBlanks()) {
preprocessed->RemoveRedundantBlanks();
}
@@ -331,39 +335,40 @@ void Prescanner::Statement() {
preprocessed->ToLowerCase().ClipComment(*this), newlineProvenance);
break;
}
- } else if (line.kind == LineClassification::Kind::CompilerDirective) {
- while (CompilerDirectiveContinuation(tokens, line.sentinel)) {
- newlineProvenance = GetCurrentProvenance();
- }
- if (preprocessingOnly_ && inFixedForm_ && InConditionalLine() &&
- nextLine_ < limit_) {
- // In -E mode, when the line after !$ conditional compilation is a
- // regular fixed form continuation line, append a '&' to the line.
- const char *p{nextLine_};
- int col{1};
- while (int n{IsSpace(p)}) {
- if (*p == '\t') {
- break;
+ } else { // no macro replacement
+ if (line.kind == LineClassification::Kind::CompilerDirective) {
+ while (CompilerDirectiveContinuation(tokens, line.sentinel)) {
+ newlineProvenance = GetCurrentProvenance();
+ }
+ if (preprocessingOnly_ && inFixedForm_ && InConditionalLine() &&
+ nextLine_ < limit_) {
+ // In -E mode, when the line after !$ conditional compilation is a
+ // regular fixed form continuation line, append a '&' to the line.
+ const char *p{nextLine_};
+ int col{1};
+ while (int n{IsSpace(p)}) {
+ if (*p == '\t') {
+ break;
+ }
+ p += n;
+ ++col;
+ }
+ if (col == 6 && *p != '0' && *p != '\t' && *p != '\n') {
+ EmitChar(tokens, '&');
+ tokens.CloseToken();
}
- p += n;
- ++col;
}
- if (col == 6 && *p != '0' && *p != '\t' && *p != '\n') {
- EmitChar(tokens, '&');
- tokens.CloseToken();
+ tokens.ToLowerCase();
+ if (!SourceFormChange(tokens.ToString())) {
+ CheckAndEmitLine(tokens, newlineProvenance);
+ }
+ } else { // Kind::Source
+ tokens.ToLowerCase();
+ if (inFixedForm_) {
+ EnforceStupidEndStatementRules(tokens);
}
- }
- tokens.ToLowerCase();
- if (!SourceFormChange(tokens.ToString())) {
CheckAndEmitLine(tokens, newlineProvenance);
}
- } else {
- CHECK(line.kind == LineClassification::Kind::Source);
- tokens.ToLowerCase();
- if (inFixedForm_) {
- EnforceStupidEndStatementRules(tokens);
- }
- CheckAndEmitLine(tokens, newlineProvenance);
}
directiveSentinel_ = nullptr;
}
@@ -562,8 +567,7 @@ bool Prescanner::MustSkipToEndOfLine() const {
return true; // skip over ignored columns in right margin (73:80)
} else if (*at_ == '!' && !inCharLiteral_ &&
(!inFixedForm_ || tabInCurrentLine_ || column_ != 6)) {
- return InCompilerDirective() ||
- !IsCompilerDirectiveSentinelAfterKeywordMacro(at_ + 1);
+ return InCompilerDirective() || !IsCompilerDirectiveSentinel(at_ + 1);
} else {
return false;
}
@@ -1612,9 +1616,6 @@ Prescanner::IsFixedFormCompilerDirectiveLine(const char *start) const {
if (!IsFixedFormCommentChar(col1)) {
return std::nullopt;
}
- // TODO: Handle keyword macros that expand to directives in fixed form.
- // The comment character can't be 'c' or 'C'. Need to figure out whether
- // fixed form continuation should apply to the expansions.
char sentinel[5], *sp{sentinel};
int column{2};
for (; column < 6; ++column) {
@@ -1677,28 +1678,20 @@ Prescanner::IsFixedFormCompilerDirectiveLine(const char *start) const {
std::optional<Prescanner::LineClassification>
Prescanner::IsFreeFormCompilerDirectiveLine(const char *start) const {
if (const char *p{SkipWhiteSpaceIncludingEmptyMacros(start)};
- p && *p == '!') {
- if (auto lnClass{IsCompilerDirectiveSentinelAfterKeywordMacro(p + 1)}) {
- if (lnClass->kind == LineClassification::Kind::CompilerDirective) {
- const char *sentinel{lnClass->sentinel};
- CHECK(sentinel != nullptr);
- const char *payload{nullptr};
- if (sentinel[0] == '$' && sentinel[1] == '\0') {
- payload = p + 2; // !$
- } else if (sentinel[1] == '@') {
- payload = p + 5; // !@acc or !@cuf
- }
- if (payload) {
- if (const char *comment{IsFreeFormComment(payload)}) {
- if (*comment == '!') { // !$ !blah or !@acc !blah
- // Conditional line comment - treat as comment
- return std::nullopt;
- }
+ p && *p++ == '!') {
+ if (auto maybePair{IsCompilerDirectiveSentinel(p)}) {
+ auto offset{static_cast<std::size_t>(p - start - 1)};
+ const char *sentinel{maybePair->first};
+ if ((sentinel[0] == '$' && sentinel[1] == '\0') || sentinel[1] == '@') {
+ if (const char *comment{IsFreeFormComment(maybePair->second)}) {
+ if (*comment == '!') {
+ // Conditional line comment - treat as comment
+ return std::nullopt;
}
}
- lnClass->payloadOffset = static_cast<std::size_t>(p - start);
}
- return lnClass;
+ return {LineClassification{
+ LineClassification::Kind::CompilerDirective, offset, sentinel}};
}
}
return std::nullopt;
@@ -1715,31 +1708,6 @@ Prescanner &Prescanner::AddCompilerDirectiveSentinel(const std::string &dir) {
return *this;
}
-std::optional<CharBlock> Prescanner::GetKeywordMacroName(
- const char *start) const {
- if (IsLegalIdentifierStart(*start)) {
- // TODO: Only bother with these cases when any keyword macro has
- // been defined with replacement text that could begin a comment
- // or directive sentinel.
- const char *p{start};
- while (IsLegalInIdentifier(*++p)) {
- }
- CharBlock name{start, static_cast<std::size_t>(p - start)};
- if (preprocessor_.IsNameDefined(name) &&
- !preprocessor_.IsFunctionLikeDefinition(name)) {
- return name;
- }
- }
- return std::nullopt;
-}
-
-TokenSequence Prescanner::ExpandKeywordMacro(
- CharBlock name, Provenance provenance) const {
- TokenSequence toks;
- toks.Put(name, provenance);
- return preprocessor_.MacroReplacement(toks, *this).value();
-}
-
const char *Prescanner::IsCompilerDirectiveSentinel(
const char *sentinel, std::size_t len) const {
std::uint64_t packed{0};
@@ -1797,26 +1765,8 @@ Prescanner::IsCompilerDirectiveSentinel(const char *p) const {
return std::nullopt;
}
-auto Prescanner::IsCompilerDirectiveSentinelAfterKeywordMacro(
- const char *p) const -> std::optional<LineClassification> {
- if (auto name{GetKeywordMacroName(p)}) {
- Provenance provenance{GetProvenance(p)};
- TokenSequence expansion{ExpandKeywordMacro(*name, provenance)};
- expansion.Put("\n", 1, provenance); // termination
- CharBlock block{expansion.ToLowerCase().ToCharBlock()};
- if (auto maybePair{IsCompilerDirectiveSentinel(block.begin())}) {
- return LineClassification{
- LineClassification::Kind::CompilerDirectiveAfterMacroExpansion,
- name->size(), maybePair->first};
- }
- } else if (auto maybePair{IsCompilerDirectiveSentinel(p)}) {
- return LineClassification{LineClassification::Kind::CompilerDirective,
- static_cast<std::size_t>(maybePair->second - p), maybePair->first};
- }
- return std::nullopt;
-}
-
-auto Prescanner::ClassifyLine(const char *start) const -> LineClassification {
+Prescanner::LineClassification Prescanner::ClassifyLine(
+ const char *start) const {
if (inFixedForm_) {
if (std::optional<LineClassification> lc{
IsFixedFormCompilerDirectiveLine(start)}) {
diff --git a/flang/lib/Parser/prescan.h b/flang/lib/Parser/prescan.h
index 2748bcb98cbf8..efbeb3709a94c 100644
--- a/flang/lib/Parser/prescan.h
+++ b/flang/lib/Parser/prescan.h
@@ -81,9 +81,6 @@ class Prescanner {
TokenSequence TokenizePreprocessorDirective();
Provenance GetCurrentProvenance() const { return GetProvenance(at_); }
- std::optional<CharBlock> GetKeywordMacroName(const char *) const;
- TokenSequence ExpandKeywordMacro(CharBlock, Provenance) const;
-
const char *IsCompilerDirectiveSentinel(const char *, std::size_t) const;
const char *IsCompilerDirectiveSentinel(CharBlock) const;
// 'first' is the sentinel, 'second' is beginning of payload
@@ -112,7 +109,6 @@ class Prescanner {
PreprocessorDirective,
IncludeLine, // Fortran INCLUDE
CompilerDirective,
- CompilerDirectiveAfterMacroExpansion, // !MACRO -> !$OMP ...
Source
};
LineClassification(Kind k, std::size_t po = 0, const char *s = nullptr)
@@ -160,7 +156,7 @@ class Prescanner {
}
void EmitInsertedChar(TokenSequence &tokens, char ch) {
- Provenance provenance{allSources().CompilerInsertionProvenance(ch)};
+ Provenance provenance{allSources_.CompilerInsertionProvenance(ch)};
tokens.PutNextTokenChar(ch, provenance);
}
@@ -244,8 +240,6 @@ class Prescanner {
bool SourceFormChange(std::string &&);
bool CompilerDirectiveContinuation(TokenSequence &, const char *sentinel);
bool SourceLineContinuation(TokenSequence &);
- std::optional<LineClassification>
- IsCompilerDirectiveSentinelAfterKeywordMacro(const char *p) const;
Messages &messages_;
CookedSource &cooked_;
@@ -304,9 +298,9 @@ class Prescanner {
const std::size_t firstCookedCharacterOffset_{cooked_.BufferedBytes()};
const Provenance spaceProvenance_{
- allSources().CompilerInsertionProvenance(' ')};
+ allSources_.CompilerInsertionProvenance(' ')};
const Provenance backslashProvenance_{
- allSources().CompilerInsertionProvenance('\\')};
+ allSources_.CompilerInsertionProvenance('\\')};
// To avoid probing the set of active compiler directive sentinel strings
// on every comment line, they're checked first with a cheap Bloom filter.
diff --git a/flang/test/Preprocessing/bug178481.F90 b/flang/test/Preprocessing/bug178481.F90
deleted file mode 100644
index 39b71a3e8e2e1..0000000000000
--- a/flang/test/Preprocessing/bug178481.F90
+++ /dev/null
@@ -1,7 +0,0 @@
-!RUN: %flang_fc1 -fdebug-unparse -fopenmp %s 2>&1 | FileCheck %s
-!CHECK: !$OMP DECLARE TARGET
-#define OMP_DECLARE_TARGET $OMP declare target
-subroutine s
- !OMP_DECLARE_TARGET
-end
-
``````````
</details>
https://github.com/llvm/llvm-project/pull/180982
More information about the flang-commits
mailing list