[flang-commits] [flang] [flang] Implement !DIR$ VECTOR ALWAYS (PR #93830)
via flang-commits
flang-commits at lists.llvm.org
Thu May 30 07:37:04 PDT 2024
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-flang-parser
@llvm/pr-subscribers-flang-semantics
Author: David Truby (DavidTruby)
<details>
<summary>Changes</summary>
This patch implements support for the VECTOR ALWAYS directive, which forces
vectorization to occurr when possible regardless of a decision by the cost
model. This is done by adding an attribute to the branch into the loop in LLVM
to indicate that the loop should always be vectorized.
---
Full diff: https://github.com/llvm/llvm-project/pull/93830.diff
12 Files Affected:
- (modified) flang/include/flang/Lower/PFTBuilder.h (+1)
- (modified) flang/include/flang/Optimizer/Dialect/FIROps.h (+1)
- (modified) flang/include/flang/Optimizer/Dialect/FIROps.td (+2-1)
- (modified) flang/include/flang/Parser/dump-parse-tree.h (+1)
- (modified) flang/include/flang/Parser/parse-tree.h (+2-1)
- (modified) flang/lib/Lower/Bridge.cpp (+48-4)
- (modified) flang/lib/Optimizer/Transforms/ControlFlowConverter.cpp (+5-1)
- (modified) flang/lib/Parser/Fortran-parsers.cpp (+3)
- (modified) flang/lib/Parser/unparse.cpp (+3)
- (modified) flang/lib/Semantics/resolve-names.cpp (+3)
- (added) flang/test/Fir/vector-always.fir (+42)
- (added) flang/test/Lower/vector-always.f90 (+29)
``````````diff
diff --git a/flang/include/flang/Lower/PFTBuilder.h b/flang/include/flang/Lower/PFTBuilder.h
index 9913f584133fa..aa83f1603c2a8 100644
--- a/flang/include/flang/Lower/PFTBuilder.h
+++ b/flang/include/flang/Lower/PFTBuilder.h
@@ -347,6 +347,7 @@ struct Evaluation : EvaluationVariant {
parser::CharBlock position{};
std::optional<parser::Label> label{};
std::unique_ptr<EvaluationList> evaluationList; // nested evaluations
+ llvm::SmallVector<const parser::CompilerDirective *> dirs;
Evaluation *parentConstruct{nullptr}; // set for nodes below the top level
Evaluation *lexicalSuccessor{nullptr}; // set for leaf nodes, some directives
Evaluation *controlSuccessor{nullptr}; // set for some leaf nodes
diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.h b/flang/include/flang/Optimizer/Dialect/FIROps.h
index 9f07364ddb627..a21f8bbe17685 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.h
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.h
@@ -16,6 +16,7 @@
#include "flang/Optimizer/Dialect/FortranVariableInterface.h"
#include "mlir/Dialect/Arith/IR/Arith.h"
#include "mlir/Dialect/Func/IR/FuncOps.h"
+#include "mlir/Dialect/LLVMIR/LLVMAttrs.h"
#include "mlir/Interfaces/LoopLikeInterface.h"
#include "mlir/Interfaces/SideEffectInterfaces.h"
diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index d9c1149040066..7bf68908e37dd 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -2096,7 +2096,8 @@ def fir_DoLoopOp : region_Op<"do_loop",
Index:$step,
Variadic<AnyType>:$initArgs,
OptionalAttr<UnitAttr>:$unordered,
- OptionalAttr<UnitAttr>:$finalValue
+ OptionalAttr<UnitAttr>:$finalValue,
+ OptionalAttr<LoopAnnotationAttr>:$loop_annotation
);
let results = (outs Variadic<AnyType>:$results);
let regions = (region SizedRegion<1>:$region);
diff --git a/flang/include/flang/Parser/dump-parse-tree.h b/flang/include/flang/Parser/dump-parse-tree.h
index 68ae50c312cde..8cf790650fb49 100644
--- a/flang/include/flang/Parser/dump-parse-tree.h
+++ b/flang/include/flang/Parser/dump-parse-tree.h
@@ -206,6 +206,7 @@ class ParseTreeDumper {
NODE(CompilerDirective, IgnoreTKR)
NODE(CompilerDirective, LoopCount)
NODE(CompilerDirective, AssumeAligned)
+ NODE(CompilerDirective, VectorAlways)
NODE(CompilerDirective, NameValue)
NODE(CompilerDirective, Unrecognized)
NODE(parser, ComplexLiteralConstant)
diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index 0a40aa8b8f616..116e5f02ff32b 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -3316,6 +3316,7 @@ struct CompilerDirective {
TUPLE_CLASS_BOILERPLATE(AssumeAligned);
std::tuple<common::Indirection<Designator>, uint64_t> t;
};
+ EMPTY_CLASS(VectorAlways);
struct NameValue {
TUPLE_CLASS_BOILERPLATE(NameValue);
std::tuple<Name, std::optional<std::uint64_t>> t;
@@ -3323,7 +3324,7 @@ struct CompilerDirective {
EMPTY_CLASS(Unrecognized);
CharBlock source;
std::variant<std::list<IgnoreTKR>, LoopCount, std::list<AssumeAligned>,
- std::list<NameValue>, Unrecognized>
+ VectorAlways, std::list<NameValue>, Unrecognized>
u;
};
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 4e50de3e7ee9c..8ce68f9b8caab 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -1881,7 +1881,7 @@ class FirConverter : public Fortran::lower::AbstractConverter {
// Increment loop begin code. (Infinite/while code was already generated.)
if (!infiniteLoop && !whileCondition)
- genFIRIncrementLoopBegin(incrementLoopNestInfo);
+ genFIRIncrementLoopBegin(incrementLoopNestInfo, doStmtEval.dirs);
// Loop body code.
auto iter = eval.getNestedEvaluations().begin();
@@ -1926,8 +1926,22 @@ class FirConverter : public Fortran::lower::AbstractConverter {
return builder->createIntegerConstant(loc, controlType, 1); // step
}
+ void addLoopAnnotationAttr(IncrementLoopInfo &info) {
+ mlir::BoolAttr f = mlir::BoolAttr::get(builder->getContext(), false);
+ mlir::LLVM::LoopVectorizeAttr va = mlir::LLVM::LoopVectorizeAttr::get(
+ builder->getContext(), f, {}, {}, {}, {}, {}, {});
+ mlir::LLVM::AccessGroupAttr ag =
+ mlir::LLVM::AccessGroupAttr::get(builder->getContext());
+ mlir::LLVM::LoopAnnotationAttr la = mlir::LLVM::LoopAnnotationAttr::get(
+ builder->getContext(), {}, va, {}, {}, {}, {}, {}, {}, {}, {}, {}, {},
+ {}, {}, {ag});
+ info.doLoop.setLoopAnnotationAttr(la);
+ }
+
/// Generate FIR to begin a structured or unstructured increment loop nest.
- void genFIRIncrementLoopBegin(IncrementLoopNestInfo &incrementLoopNestInfo) {
+ void genFIRIncrementLoopBegin(
+ IncrementLoopNestInfo &incrementLoopNestInfo,
+ llvm::SmallVectorImpl<const Fortran::parser::CompilerDirective *> &dirs) {
assert(!incrementLoopNestInfo.empty() && "empty loop nest");
mlir::Location loc = toLocation();
for (IncrementLoopInfo &info : incrementLoopNestInfo) {
@@ -1978,6 +1992,15 @@ class FirConverter : public Fortran::lower::AbstractConverter {
}
if (info.hasLocalitySpecs())
handleLocalitySpecs(info);
+
+ for (const auto *dir : dirs) {
+ std::visit(
+ Fortran::common::visitors{
+ [&](const Fortran::parser::CompilerDirective::VectorAlways
+ &d) { addLoopAnnotationAttr(info); },
+ [&](const auto &) {}},
+ dir->u);
+ }
continue;
}
@@ -2508,8 +2531,29 @@ class FirConverter : public Fortran::lower::AbstractConverter {
}
}
- void genFIR(const Fortran::parser::CompilerDirective &) {
- // TODO
+ void attachLoopDirective(const Fortran::parser::CompilerDirective &dir,
+ Fortran::lower::pft::Evaluation *e) {
+ while (e->isDirective()) {
+ e = e->lexicalSuccessor;
+ }
+
+ if (e->isA<Fortran::parser::NonLabelDoStmt>()) {
+ e->dirs.push_back(&dir);
+ } else {
+ fir::emitFatalError(toLocation(), "loop directive must appear before a loop");
+ }
+ }
+
+ void genFIR(const Fortran::parser::CompilerDirective &dir) {
+ Fortran::lower::pft::Evaluation &eval = getEval();
+
+ std::visit(
+ Fortran::common::visitors{
+ [&](const Fortran::parser::CompilerDirective::VectorAlways &) {
+ attachLoopDirective(dir, &eval);
+ },
+ [&](const auto &) {}},
+ dir.u);
}
void genFIR(const Fortran::parser::OpenACCConstruct &acc) {
diff --git a/flang/lib/Optimizer/Transforms/ControlFlowConverter.cpp b/flang/lib/Optimizer/Transforms/ControlFlowConverter.cpp
index a233e7fbdcd1e..b40c06de8787b 100644
--- a/flang/lib/Optimizer/Transforms/ControlFlowConverter.cpp
+++ b/flang/lib/Optimizer/Transforms/ControlFlowConverter.cpp
@@ -132,10 +132,14 @@ class CfgLoopConv : public mlir::OpRewritePattern<fir::DoLoopOp> {
auto comparison = rewriter.create<mlir::arith::CmpIOp>(
loc, arith::CmpIPredicate::sgt, itersLeft, zero);
- rewriter.create<mlir::cf::CondBranchOp>(
+ auto cond = rewriter.create<mlir::cf::CondBranchOp>(
loc, comparison, firstBlock, llvm::ArrayRef<mlir::Value>(), endBlock,
llvm::ArrayRef<mlir::Value>());
+ if (auto ann = loop.getLoopAnnotation()) {
+ cond->setAttr("loop_annotation", *ann);
+ }
+
// The result of the loop operation is the values of the condition block
// arguments except the induction variable on the last iteration.
auto args = loop.getFinalValue()
diff --git a/flang/lib/Parser/Fortran-parsers.cpp b/flang/lib/Parser/Fortran-parsers.cpp
index ff01974b549a1..d2241fb66a013 100644
--- a/flang/lib/Parser/Fortran-parsers.cpp
+++ b/flang/lib/Parser/Fortran-parsers.cpp
@@ -1276,10 +1276,13 @@ constexpr auto loopCount{
constexpr auto assumeAligned{"ASSUME_ALIGNED" >>
optionalList(construct<CompilerDirective::AssumeAligned>(
indirect(designator), ":"_tok >> digitString64))};
+constexpr auto vectorAlways{
+ "VECTOR ALWAYS" >> construct<CompilerDirective::VectorAlways>()};
TYPE_PARSER(beginDirective >> "DIR$ "_tok >>
sourced((construct<CompilerDirective>(ignore_tkr) ||
construct<CompilerDirective>(loopCount) ||
construct<CompilerDirective>(assumeAligned) ||
+ construct<CompilerDirective>(vectorAlways) ||
construct<CompilerDirective>(
many(construct<CompilerDirective::NameValue>(
name, maybe(("="_tok || ":"_tok) >> digitString64))))) /
diff --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index bdd968b19a43f..5a4c7550d64dd 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -1824,6 +1824,9 @@ class UnparseVisitor {
Word("!DIR$ ASSUME_ALIGNED ");
Walk(" ", assumeAligned, ", ");
},
+ [&](const CompilerDirective::VectorAlways &valways) {
+ Word("!DIR$ VECTOR ALWAYS");
+ },
[&](const std::list<CompilerDirective::NameValue> &names) {
Walk("!DIR$ ", names, " ");
},
diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index a46c0f378d5d0..e65788a02b725 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -8854,6 +8854,9 @@ void ResolveNamesVisitor::Post(const parser::AssignedGotoStmt &x) {
}
void ResolveNamesVisitor::Post(const parser::CompilerDirective &x) {
+ if (const auto *dir{
+ std::get_if<parser::CompilerDirective::VectorAlways>(&x.u)})
+ return;
if (const auto *tkr{
std::get_if<std::list<parser::CompilerDirective::IgnoreTKR>>(&x.u)}) {
if (currScope().IsTopLevel() ||
diff --git a/flang/test/Fir/vector-always.fir b/flang/test/Fir/vector-always.fir
new file mode 100644
index 0000000000000..b6dcf237ed59a
--- /dev/null
+++ b/flang/test/Fir/vector-always.fir
@@ -0,0 +1,42 @@
+// RUN: %flang_fc1 -emit-llvm -o - %s | FileCheck %s
+
+#access_group = #llvm.access_group<id = distinct[0]<>>
+#loop_vectorize = #llvm.loop_vectorize<disable = false>
+#loop_annotation = #llvm.loop_annotation<vectorize = #loop_vectorize, parallelAccesses = #access_group>
+
+// CHECK-LABEL: @vector_always_
+// CHECK: br i1 {{.*}}, label {{.*}}, label {{.*}}, !llvm.loop ![[ANNOTATION:.*]]
+func.func @_QPvector_always() {
+ %c1 = arith.constant 1 : index
+ %c10_i32 = arith.constant 10 : i32
+ %c1_i32 = arith.constant 1 : i32
+ %c10 = arith.constant 10 : index
+ %0 = fir.alloca !fir.array<10xi32> {bindc_name = "a", uniq_name = "_QFvector_alwaysEa"}
+ %1 = fir.shape %c10 : (index) -> !fir.shape<1>
+ %2 = fir.declare %0(%1) {uniq_name = "_QFvector_alwaysEa"} : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>) -> !fir.ref<!fir.array<10xi32>>
+ %3 = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFvector_alwaysEi"}
+ %4 = fir.declare %3 {uniq_name = "_QFvector_alwaysEi"} : (!fir.ref<i32>) -> !fir.ref<i32>
+ %5 = fir.convert %c1_i32 : (i32) -> index
+ %6 = fir.convert %c10_i32 : (i32) -> index
+ %7 = fir.convert %5 : (index) -> i32
+ %8:2 = fir.do_loop %arg0 = %5 to %6 step %c1 iter_args(%arg1 = %7) -> (index, i32) attributes {loop_annotation = #loop_annotation} {
+ fir.store %arg1 to %4 : !fir.ref<i32>
+ %9 = fir.load %4 : !fir.ref<i32>
+ %10 = fir.load %4 : !fir.ref<i32>
+ %11 = fir.convert %10 : (i32) -> i64
+ %12 = fir.array_coor %2(%1) %11 : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>, i64) -> !fir.ref<i32>
+ fir.store %9 to %12 : !fir.ref<i32>
+ %13 = arith.addi %arg0, %c1 : index
+ %14 = fir.convert %c1 : (index) -> i32
+ %15 = fir.load %4 : !fir.ref<i32>
+ %16 = arith.addi %15, %14 : i32
+ fir.result %13, %16 : index, i32
+ }
+ fir.store %8#1 to %4 : !fir.ref<i32>
+ return
+ }
+
+// CHECK: ![[ANNOTATION]] = distinct !{![[ANNOTATION]], ![[VECTORIZE:.*]], ![[PAR_ACCESS:.*]]}
+// CHECK: ![[VECTORIZE]] = !{!"llvm.loop.vectorize.enable", i1 true}
+// CHECK: ![[PAR_ACCESS]] = !{!"llvm.loop.parallel_accesses", ![[DISTINCT:.*]]}
+// CHECK: ![[DISTINCT]] = distinct !{}
diff --git a/flang/test/Lower/vector-always.f90 b/flang/test/Lower/vector-always.f90
new file mode 100644
index 0000000000000..1994163626f16
--- /dev/null
+++ b/flang/test/Lower/vector-always.f90
@@ -0,0 +1,29 @@
+! RUN: %flang_fc1 -emit-fir -o - %s | FileCheck %s
+
+! CHECK: #access_group = #llvm.access_group<id = distinct[0]<>>
+! CHECK: #access_group1 = #llvm.access_group<id = distinct[1]<>>
+! CHECK: #loop_vectorize = #llvm.loop_vectorize<disable = false>
+! CHECK: #loop_annotation = #llvm.loop_annotation<vectorize = #loop_vectorize, parallelAccesses = #access_group>
+! CHECK: #loop_annotation1 = #llvm.loop_annotation<vectorize = #loop_vectorize, parallelAccesses = #access_group1>
+
+! CHECK-LABEL: vector_always
+subroutine vector_always
+ integer :: a(10)
+ !dir$ vector always
+ !CHECK: fir.do_loop {{.*}} attributes {loop_annotation = #loop_annotation}
+ do i=1,10
+ a(i)=i
+ end do
+end subroutine vector_always
+
+
+! CHECK-LABEL: intermediate_directive
+subroutine intermediate_directive
+ integer :: a(10)
+ !dir$ vector always
+ !dir$ unknown
+ !CHECK: fir.do_loop {{.*}} attributes {loop_annotation = #loop_annotation1}
+ do i=1,10
+ a(i)=i
+ end do
+end subroutine intermediate_directive
``````````
</details>
https://github.com/llvm/llvm-project/pull/93830
More information about the flang-commits
mailing list