[flang-commits] [flang] [flang][preprocessor] Handle compiler directives with continuations a… (PR #70128)
Peter Klausler via flang-commits
flang-commits at lists.llvm.org
Mon Oct 30 12:09:01 PDT 2023
https://github.com/klausler updated https://github.com/llvm/llvm-project/pull/70128
>From 1ecc0001512838c1c6e4148901aa3fbe6d742bc2 Mon Sep 17 00:00:00 2001
From: Peter Klausler <pklausler at nvidia.com>
Date: Mon, 23 Oct 2023 17:16:09 -0700
Subject: [PATCH] [flang][preprocessor] Handle line continuations after macro
expansion
There are cases in which free source form line continuation is not
detectable before macro expansion. The '&' continuation marker
can be the result of a macro expansion, and (for compiler directives)
the continuation line may not be recognizable as such before its
continuation sentinel (e.g., "!$omp &") is produced by macro expansion.
Check for preprocessed compiler directive and source lines that end
in '&' after macro expansion, and in both cases attempt to locate and
preprocess their continuation lines. Also check for compiler directive
continuation lines that might result from macro replacement after an
initial compiler directive line that had no macro replacement.
Last, also allow a continuation marker (&) to appear in a macro actual
argument.
---
flang/docs/Preprocessing.md | 2 +-
flang/lib/Parser/prescan.cpp | 128 +++++++++++++++++-
flang/lib/Parser/prescan.h | 2 +
.../directive-contin-with-pp.F90 | 41 ++++++
flang/test/Preprocessing/pp130.F90 | 5 +-
5 files changed, 168 insertions(+), 10 deletions(-)
create mode 100644 flang/test/Preprocessing/directive-contin-with-pp.F90
diff --git a/flang/docs/Preprocessing.md b/flang/docs/Preprocessing.md
index 7465ff538e42e2c..3c523472f39bd01 100644
--- a/flang/docs/Preprocessing.md
+++ b/flang/docs/Preprocessing.md
@@ -228,5 +228,5 @@ E . . E E . pp125.F90 #DEFINE works in free form
. . E . E E pp127.F90 FLM call with closing ')' on next line (not a continuation)
E . E . E E pp128.F90 FLM call with '(' on next line (not a continuation)
. . N . . N pp129.F90 #define KWM !, then KWM works as comment line initiator
-E . E . . E pp130.F90 #define KWM &, use for continuation w/o pasting (ifort and nag seem to continue #define)
+. . E . . E pp130.F90 #define KWM &, use for continuation w/o pasting (ifort and nag seem to continue #define)
```
diff --git a/flang/lib/Parser/prescan.cpp b/flang/lib/Parser/prescan.cpp
index f61eff5b0dd6aa9..5b600822282a443 100644
--- a/flang/lib/Parser/prescan.cpp
+++ b/flang/lib/Parser/prescan.cpp
@@ -213,6 +213,9 @@ void Prescanner::Statement() {
if (preprocessed->HasRedundantBlanks()) {
preprocessed->RemoveRedundantBlanks();
}
+ while (CompilerDirectiveContinuation(*preprocessed, ppl.sentinel)) {
+ newlineProvenance = GetCurrentProvenance();
+ }
NormalizeCompilerDirectiveCommentMarker(*preprocessed);
preprocessed->ToLowerCase();
SourceFormChange(preprocessed->ToString());
@@ -227,6 +230,9 @@ void Prescanner::Statement() {
preprocessed->RemoveBlanks(/*after column*/ 6);
}
} else {
+ while (SourceLineContinuation(*preprocessed)) {
+ newlineProvenance = GetCurrentProvenance();
+ }
if (preprocessed->HasRedundantBlanks()) {
preprocessed->RemoveRedundantBlanks();
}
@@ -239,12 +245,17 @@ void Prescanner::Statement() {
break;
}
} else {
- tokens.ToLowerCase();
if (line.kind == LineClassification::Kind::CompilerDirective) {
+ while (CompilerDirectiveContinuation(tokens, line.sentinel)) {
+ newlineProvenance = GetCurrentProvenance();
+ }
+ tokens.ToLowerCase();
SourceFormChange(tokens.ToString());
- }
- if (inFixedForm_ && line.kind == LineClassification::Kind::Source) {
- EnforceStupidEndStatementRules(tokens);
+ } else { // Kind::Source
+ tokens.ToLowerCase();
+ if (inFixedForm_) {
+ EnforceStupidEndStatementRules(tokens);
+ }
}
tokens.CheckBadFortranCharacters(messages_)
.CheckBadParentheses(messages_)
@@ -1132,8 +1143,10 @@ bool Prescanner::FreeFormContinuation() {
if (*p != '\n') {
if (inCharLiteral_) {
return false;
- } else if (*p != '!' &&
- features_.ShouldWarn(LanguageFeature::CruftAfterAmpersand)) {
+ } else if (*p == '!') { // & ! comment - ok
+ } else if (ampersand && isPossibleMacroCall_ && (*p == ',' || *p == ')')) {
+ return false; // allow & at end of a macro argument
+ } else if (features_.ShouldWarn(LanguageFeature::CruftAfterAmpersand)) {
Say(GetProvenance(p), "missing ! before comment after &"_warn_en_US);
}
}
@@ -1318,4 +1331,107 @@ void Prescanner::SourceFormChange(std::string &&dir) {
inFixedForm_ = true;
}
}
+
+// Acquire and append compiler directive continuation lines to
+// the tokens that constitute a compiler directive, even when those
+// directive continuation lines are the result of macro expansion.
+// (Not used when neither the original compiler directive line nor
+// the directive continuation line result from preprocessing; regular
+// line continuation during tokenization handles that normal case.)
+bool Prescanner::CompilerDirectiveContinuation(
+ TokenSequence &tokens, const char *origSentinel) {
+ if (inFixedForm_ || tokens.empty() ||
+ tokens.TokenAt(tokens.SizeInTokens() - 1) != "&") {
+ return false;
+ }
+ LineClassification followingLine{ClassifyLine(nextLine_)};
+ if (followingLine.kind == LineClassification::Kind::Comment) {
+ nextLine_ += followingLine.payloadOffset; // advance to '!' or newline
+ NextLine();
+ return true;
+ }
+ CHECK(origSentinel != nullptr);
+ directiveSentinel_ = origSentinel; // so IsDirective() is true
+ const char *nextContinuation{
+ followingLine.kind == LineClassification::Kind::CompilerDirective
+ ? FreeFormContinuationLine(true)
+ : nullptr};
+ if (!nextContinuation &&
+ followingLine.kind != LineClassification::Kind::Source) {
+ return false;
+ }
+ auto origNextLine{nextLine_};
+ BeginSourceLine(nextLine_);
+ NextLine();
+ TokenSequence followingTokens;
+ if (nextContinuation) {
+ // What follows is !DIR$ & xxx; skip over the & so that it
+ // doesn't cause a spurious continuation.
+ at_ = nextContinuation;
+ } else {
+ // What follows looks like a source line before macro expansion,
+ // but might become a directive continuation afterwards.
+ SkipSpaces();
+ }
+ while (NextToken(followingTokens)) {
+ }
+ if (auto followingPrepro{
+ preprocessor_.MacroReplacement(followingTokens, *this)}) {
+ followingTokens = std::move(*followingPrepro);
+ }
+ followingTokens.RemoveRedundantBlanks();
+ std::size_t startAt{0};
+ std::size_t keep{followingTokens.SizeInTokens()};
+ bool ok{false};
+ if (nextContinuation) {
+ ok = true;
+ } else {
+ if (keep >= 3 && followingTokens.TokenAt(0) == "!" &&
+ followingTokens.TokenAt(2) == "&") {
+ CharBlock sentinel{followingTokens.TokenAt(1)};
+ if (!sentinel.empty() &&
+ std::memcmp(sentinel.begin(), origSentinel, sentinel.size()) == 0) {
+ startAt = 3;
+ keep -= 3;
+ ok = true;
+ }
+ }
+ }
+ if (ok) {
+ tokens.pop_back(); // delete original '&'
+ tokens.Put(followingTokens, startAt, keep);
+ } else {
+ nextLine_ = origNextLine;
+ }
+ return ok;
+}
+
+// Similar, but for source line continuation after macro replacement.
+bool Prescanner::SourceLineContinuation(TokenSequence &tokens) {
+ if (!inFixedForm_ && !tokens.empty() &&
+ tokens.TokenAt(tokens.SizeInTokens() - 1) == "&") {
+ LineClassification followingLine{ClassifyLine(nextLine_)};
+ if (followingLine.kind == LineClassification::Kind::Comment) {
+ nextLine_ += followingLine.payloadOffset; // advance to '!' or newline
+ NextLine();
+ return true;
+ } else if (const char *nextContinuation{FreeFormContinuationLine(true)}) {
+ BeginSourceLine(nextLine_);
+ NextLine();
+ TokenSequence followingTokens;
+ at_ = nextContinuation;
+ while (NextToken(followingTokens)) {
+ }
+ if (auto followingPrepro{
+ preprocessor_.MacroReplacement(followingTokens, *this)}) {
+ followingTokens = std::move(*followingPrepro);
+ }
+ followingTokens.RemoveRedundantBlanks();
+ tokens.pop_back(); // delete original '&'
+ tokens.Put(followingTokens);
+ return true;
+ }
+ }
+ return false;
+}
} // namespace Fortran::parser
diff --git a/flang/lib/Parser/prescan.h b/flang/lib/Parser/prescan.h
index 021632657a98c13..5f80d26636dc719 100644
--- a/flang/lib/Parser/prescan.h
+++ b/flang/lib/Parser/prescan.h
@@ -186,6 +186,8 @@ class Prescanner {
const char *) const;
LineClassification ClassifyLine(const char *) const;
void SourceFormChange(std::string &&);
+ bool CompilerDirectiveContinuation(TokenSequence &, const char *sentinel);
+ bool SourceLineContinuation(TokenSequence &);
Messages &messages_;
CookedSource &cooked_;
diff --git a/flang/test/Preprocessing/directive-contin-with-pp.F90 b/flang/test/Preprocessing/directive-contin-with-pp.F90
new file mode 100644
index 000000000000000..9a06ae84382104a
--- /dev/null
+++ b/flang/test/Preprocessing/directive-contin-with-pp.F90
@@ -0,0 +1,41 @@
+! RUN: %flang -E %s 2>&1 | FileCheck %s
+
+#define DIR_START !dir$
+#define DIR_CONT !dir$&
+#define FIRST(x) DIR_START x
+#define NEXT(x) DIR_CONT x
+#define AMPER &
+
+subroutine s(x1, x2, x3, x4, x5, x6, x7)
+
+!dir$ ignore_tkr x1
+
+!dir$ ignore_tkr &
+!dir$& x2
+
+DIR_START ignore_tkr x3
+
+!dir$ ignore_tkr AMPER
+DIR_CONT x4
+
+FIRST(ignore_tkr &)
+!dir$& x5
+
+FIRST(ignore_tkr &)
+NEXT(x6)
+
+FIRST(ignore_tkr &)
+NEXT(x7 &)
+NEXT(x8)
+
+end
+
+!CHECK: subroutine s(x1, x2, x3, x4, x5, x6, x7)
+!CHECK: !dir$ ignore_tkr x1
+!CHECK: !dir$ ignore_tkr x2
+!CHECK: !dir$ ignore_tkr x3
+!CHECK: !dir$ ignore_tkr x4
+!CHECK: !dir$ ignore_tkr x5
+!CHECK: !dir$ ignore_tkr x6
+!CHECK: !dir$ ignore_tkr x7 x8
+!CHECK: end
diff --git a/flang/test/Preprocessing/pp130.F90 b/flang/test/Preprocessing/pp130.F90
index fcc3b3958f7f9ec..6116cf1c20e5f98 100644
--- a/flang/test/Preprocessing/pp130.F90
+++ b/flang/test/Preprocessing/pp130.F90
@@ -1,8 +1,7 @@
-! RUN: not %flang -E %s 2>&1 | FileCheck %s
-! CHECK: error: bad character ('&') in Fortran token
+! RUN: %flang -E %s 2>&1 | FileCheck %s
+! CHECK: j = j + 111
! #define KWM &, use for continuation w/o pasting (ifort and nag seem to continue #define)
#define KWM &
-
integer :: j
j = 666
j = j + KWM
More information about the flang-commits
mailing list