[Mlir-commits] [flang] [mlir] [flang][OpenMP] Support ALLOCATABLE list items in LINEAR (PR #228827)
Aditya Trivedi
llvmlistbot at llvm.org
Mon Oct 19 00:35:19 PDT 2026
https://github.com/adit4443ya updated https://github.com/llvm/llvm-project/pull/228827
>From d0d55c0be723e804b31e507cc871a3c390f99e66 Mon Sep 17 00:00:00 2001
From: Aditya Trivedi <adit4443ya at gmail.com>
Date: Mon, 19 Oct 2026 00:06:06 +0530
Subject: [PATCH 1/2] [mlir][OpenMP] Privatize linear variables with linear
privatizers
The linear clause now names an omp.private of type linear for each
variable and defines an entry block argument for its private copy. An
optional data_address region on the privatizer gives the address of the
value for types that wrap it, such as Fortran descriptors. Translation
uses the privatizer for every linear variable instead of rewriting uses
of the original in place.
---
.../mlir/Dialect/OpenMP/OpenMPClauses.td | 13 +-
.../mlir/Dialect/OpenMP/OpenMPEnums.td | 4 +-
mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td | 65 ++-
.../Dialect/OpenMP/OpenMPOpsInterfaces.td | 7 +-
mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp | 308 ++++++++++++--
.../OpenMP/OpenMPToLLVMIRTranslation.cpp | 389 +++++++++++-------
mlir/test/Dialect/OpenMP/invalid.mlir | 99 ++++-
mlir/test/Dialect/OpenMP/ops.mlir | 73 +++-
.../LLVMIR/openmp-linear-privatizer.mlir | 148 +++++++
.../Target/LLVMIR/openmp-llvm-invalid.mlir | 20 +-
mlir/test/Target/LLVMIR/openmp-llvm.mlir | 47 ++-
.../Target/LLVMIR/openmp-simd-guided.mlir | 4 +-
.../Target/LLVMIR/openmp-simd-linear.mlir | 39 +-
.../Target/LLVMIR/openmp-simd-ordered.mlir | 22 +-
.../Target/LLVMIR/openmp-wsloop-linear.mlir | 22 +-
.../LLVMIR/openmp-wsloop-simd-ordered.mlir | 22 +-
16 files changed, 975 insertions(+), 307 deletions(-)
create mode 100644 mlir/test/Target/LLVMIR/openmp-linear-privatizer.mlir
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPClauses.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPClauses.td
index bd8bc6ed12770..81419ac4b9920 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPClauses.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPClauses.td
@@ -821,16 +821,21 @@ class OpenMP_LinearClauseSkip<
bit description = false, bit extraClassDeclaration = false
> : OpenMP_Clause<traits, arguments, assemblyFormat, description,
extraClassDeclaration> {
+ let traits = [
+ BlockArgOpenMPOpInterface
+ ];
+
let arguments = (ins Variadic<AnyType>:$linear_vars,
Variadic<AnyInteger>:$linear_step_vars,
OptionalAttr<ArrayAttr>:$linear_var_types,
- OptionalAttr<ArrayAttr>:$linear_modifiers);
+ OptionalAttr<ArrayAttr>:$linear_modifiers,
+ OptionalAttr<SymbolRefArrayAttr>:$linear_syms);
let optAssemblyFormat = [{
`linear` `(`
custom<LinearClause>($linear_vars, type($linear_vars),
$linear_step_vars, type($linear_step_vars),
- $linear_modifiers) `)`
+ $linear_modifiers, $linear_syms) `)`
( `linear_var_types` `(` $linear_var_types^ `)` )?
}];
@@ -843,6 +848,10 @@ class OpenMP_LinearClauseSkip<
The `linear_modifiers` attribute optionally specifies a per-variable
linear-modifier: `val`, `ref`, or `uval`. When omitted, the default
modifier is determined by the language semantics.
+
+ On operations with a region, `linear_syms` names the `omp.private`
+ declaration of `linear` type that privatizes each linear variable, and each
+ linear variable defines an entry block argument for its private copy.
}];
}
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPEnums.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPEnums.td
index 50b02c0d9e61c..a44d15fa408fd 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPEnums.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPEnums.td
@@ -200,12 +200,14 @@ def ClauseTaskDependAttr : OpenMP_EnumAttr<ClauseTaskDepend,
def DataSharingTypePrivate : I32EnumAttrCase<"Private", 0, "private">;
def DataSharingTypeFirstPrivate
: I32EnumAttrCase<"FirstPrivate", 1, "firstprivate">;
+def DataSharingTypeLinear : I32EnumAttrCase<"Linear", 2, "linear">;
def DataSharingClauseType : OpenMP_I32EnumAttr<
"DataSharingClauseType",
"Type of a data-sharing clause", [
DataSharingTypePrivate,
- DataSharingTypeFirstPrivate
+ DataSharingTypeFirstPrivate,
+ DataSharingTypeLinear
]>;
def DataSharingClauseTypeAttr : OpenMP_EnumAttr<DataSharingClauseType,
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index 6bc99cf765699..5592a2cf980f0 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -82,10 +82,30 @@ def PrivateClauseOp
}
```
+ * `linear(x)` for an i32 would be emitted as:
+ ```mlir
+ omp.private {type = linear} @x.privatizer : i32
+ ```
+
+ * `linear(x)` for types wrapping the underlying linear variable (e.g.
+ Fortran's ALLOCATABLE) would be emitted as:
+ ```mlir
+ omp.private {type = linear} @x.privatizer : !some.type init {
+ ...
+ } data_address {
+ ^bb0(%arg0: !some.pointer<!some.type>):
+ // return the address of the data that %arg0 describes.
+ omp.yield(%addr : !llvm.ptr)
+ } dealloc {
+ ...
+ }
+ ```
+
There are no restrictions on the body except for:
- - The `dealloc` regions has a single argument.
+ - The `dealloc` & `data_address` regions have a single argument.
- The `init` & `copy` regions have 2 arguments.
- - All three regions are terminated by `omp.yield` ops.
+ - All regions are terminated by `omp.yield` ops.
+ - The `data_address` region yields a single value.
The above restrictions and other obvious restrictions (e.g. verifying the
type of yielded values) are verified by the custom op verifier. The actual
contents of the blocks inside all regions are not verified.
@@ -102,7 +122,10 @@ def PrivateClauseOp
the regions should be a type which represents a pointer to `type`.
The `data_sharing_type` attribute specifies whether privatizer corresponds
- to a `private` or a `firstprivate` clause.
+ to a `private`, `firstprivate` or `linear` clause. A `linear` privatizer is
+ only referenced by `linear` clauses; its optional `data_address` region
+ gives the address of the value the linear clause updates. Without it, the
+ variable itself holds that value.
}];
let arguments = (ins SymbolNameAttr:$sym_name,
@@ -112,13 +135,15 @@ def PrivateClauseOp
let regions = (region AnyRegion:$init_region,
AnyRegion:$copy_region,
- AnyRegion:$dealloc_region);
+ AnyRegion:$dealloc_region,
+ AnyRegion:$data_address_region);
let assemblyFormat = [{
` ` `{` `type` `=` enum($data_sharing_type) `}`
($sym_visibility^)? $sym_name `:` $type
(`init` $init_region^)?
(`copy` $copy_region^)?
+ (`data_address` $data_address_region^)?
(`dealloc` $dealloc_region^)? attr-dict
}];
@@ -145,6 +170,10 @@ def PrivateClauseOp
auto ®ion = getCopyRegion();
return region.empty() ? nullptr : region.getArgument(1);
}
+ BlockArgument getDataAddressArg() {
+ auto ®ion = getDataAddressRegion();
+ return region.empty() ? nullptr : region.getArgument(0);
+ }
BlockArgument getDeallocMoldArg() {
auto ®ion = getDeallocRegion();
return region.empty() ? nullptr : region.getArgument(0);
@@ -848,7 +877,8 @@ def WsloopOp : OpenMP_Op<"wsloop", traits = [
DeclareOpInterfaceMethods<LoopWrapperInterface>, NoTerminator,
RecursiveMemoryEffects, SingleBlock
], clauses = [
- OpenMP_AllocateClause, OpenMP_LinearClause, OpenMP_NowaitClause,
+ OpenMP_AllocateClause, OpenMP_LinearClauseSkip<assemblyFormat = true>,
+ OpenMP_NowaitClause,
OpenMP_OrderClause, OpenMP_OrderedClause, OpenMP_PrivateClause,
OpenMP_ReductionClause, OpenMP_ScheduleClause
], singleRegion = true> {
@@ -882,9 +912,12 @@ def WsloopOp : OpenMP_Op<"wsloop", traits = [
];
let assemblyFormat = clausesAssemblyFormat # [{
- custom<PrivateReductionRegion>($region, $private_vars, type($private_vars),
- $private_syms, $private_needs_barrier, $reduction_mod, $reduction_vars,
- type($reduction_vars), $reduction_byref, $reduction_syms) attr-dict
+ custom<LinearPrivateReductionRegion>($region, $linear_vars,
+ type($linear_vars), $linear_step_vars, type($linear_step_vars),
+ $linear_var_types, $linear_modifiers, $linear_syms, $private_vars,
+ type($private_vars), $private_syms, $private_needs_barrier,
+ $reduction_mod, $reduction_vars, type($reduction_vars),
+ $reduction_byref, $reduction_syms) attr-dict
}];
let hasVerifier = 1;
@@ -900,8 +933,8 @@ def SimdOp : OpenMP_Op<"simd", traits = [
DeclareOpInterfaceMethods<LoopWrapperInterface>, NoTerminator,
RecursiveMemoryEffects, SingleBlock
], clauses = [
- OpenMP_AlignedClause, OpenMP_IfClause, OpenMP_LinearClause,
- OpenMP_NontemporalClause, OpenMP_OrderClause, OpenMP_PrivateClause,
+ OpenMP_AlignedClause, OpenMP_IfClause,
+ OpenMP_LinearClauseSkip<assemblyFormat = true>, OpenMP_NontemporalClause, OpenMP_OrderClause, OpenMP_PrivateClause,
OpenMP_ReductionClause, OpenMP_SafelenClause, OpenMP_SimdlenClause
], singleRegion = true> {
let summary = "simd construct";
@@ -936,9 +969,12 @@ def SimdOp : OpenMP_Op<"simd", traits = [
];
let assemblyFormat = clausesAssemblyFormat # [{
- custom<PrivateReductionRegion>($region, $private_vars, type($private_vars),
- $private_syms, $private_needs_barrier, $reduction_mod, $reduction_vars,
- type($reduction_vars), $reduction_byref, $reduction_syms) attr-dict
+ custom<LinearPrivateReductionRegion>($region, $linear_vars,
+ type($linear_vars), $linear_step_vars, type($linear_step_vars),
+ $linear_var_types, $linear_modifiers, $linear_syms, $private_vars,
+ type($private_vars), $private_syms, $private_needs_barrier,
+ $reduction_mod, $reduction_vars, type($reduction_vars),
+ $reduction_byref, $reduction_syms) attr-dict
}];
let hasVerifier = 1;
@@ -2681,7 +2717,8 @@ def WorkdistributeOp : OpenMP_Op<"workdistribute", traits = [
def DeclareSimdOp
: OpenMP_Op<"declare_simd", traits = [AttrSizedOperandSegments],
clauses = [OpenMP_AlignedClause, OpenMP_InbranchClause,
- OpenMP_LinearClause, OpenMP_NotinbranchClause,
+ OpenMP_LinearClauseSkip<traits = true>,
+ OpenMP_NotinbranchClause,
OpenMP_SimdlenClause, OpenMP_UniformClause]> {
let summary = "declare simd directive";
let description = [{
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsInterfaces.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsInterfaces.td
index 35f879e224179..4d0b440665520 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsInterfaces.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsInterfaces.td
@@ -123,8 +123,10 @@ def BlockArgHostEvalClause : BlockArgOpenMPClause<
"host_eval", "HostEval", BlockArgHasDeviceAddrClause>;
def BlockArgInReductionClause : BlockArgOpenMPClause<
"in_reduction", "InReduction", BlockArgHostEvalClause>;
+def BlockArgLinearClause : BlockArgOpenMPClause<
+ "linear", "Linear", BlockArgInReductionClause>;
def BlockArgMapClause : BlockArgOpenMPClause<
- "map", "Map", BlockArgInReductionClause>;
+ "map", "Map", BlockArgLinearClause>;
def BlockArgPrivateClause : BlockArgOpenMPClause<
"private", "Private", BlockArgMapClause>;
def BlockArgReductionClause : BlockArgOpenMPClause<
@@ -145,7 +147,8 @@ def BlockArgOpenMPOpInterface : OpInterface<"BlockArgOpenMPOpInterface"> {
let cppNamespace = "::mlir::omp";
defvar clauses = [ BlockArgHasDeviceAddrClause, BlockArgHostEvalClause,
- BlockArgInReductionClause, BlockArgMapClause, BlockArgPrivateClause,
+ BlockArgInReductionClause, BlockArgLinearClause, BlockArgMapClause,
+ BlockArgPrivateClause,
BlockArgReductionClause, BlockArgTaskReductionClause,
BlockArgUseDeviceAddrClause, BlockArgUseDevicePtrClause ];
diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
index c6a424bd93214..d2ee8fd3aaf79 100644
--- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
+++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
@@ -471,7 +471,8 @@ static ParseResult parseLinearClause(
SmallVectorImpl<OpAsmParser::UnresolvedOperand> &linearVars,
SmallVectorImpl<Type> &linearTypes,
SmallVectorImpl<OpAsmParser::UnresolvedOperand> &linearStepVars,
- SmallVectorImpl<Type> &linearStepTypes, ArrayAttr &linearModifiers) {
+ SmallVectorImpl<Type> &linearStepTypes, ArrayAttr &linearModifiers,
+ ArrayAttr &linearSyms) {
SmallVector<Attribute> modifiers;
auto result = parser.parseCommaSeparatedList([&]() {
OpAsmParser::UnresolvedOperand var;
@@ -517,11 +518,12 @@ static ParseResult parseLinearClause(
return success();
}
-/// Print Linear Clause
+/// Print Linear Clause. `linearSyms` is only used by ops with a region, which
+/// print the linear clause with their entry block arguments.
static void printLinearClause(OpAsmPrinter &p, Operation *op,
ValueRange linearVars, TypeRange linearTypes,
ValueRange linearStepVars, TypeRange stepVarTypes,
- ArrayAttr linearModifiers) {
+ ArrayAttr linearModifiers, ArrayAttr linearSyms) {
size_t linearVarsSize = linearVars.size();
for (unsigned i = 0; i < linearVarsSize; ++i) {
if (i != 0)
@@ -1272,6 +1274,23 @@ struct PrivateParseArgs {
mapIndices(mapIndices) {}
};
+struct LinearParseArgs {
+ SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars;
+ SmallVectorImpl<Type> &types;
+ SmallVectorImpl<OpAsmParser::UnresolvedOperand> &stepVars;
+ SmallVectorImpl<Type> &stepTypes;
+ ArrayAttr &varTypes;
+ ArrayAttr &modifiers;
+ ArrayAttr &syms;
+ LinearParseArgs(SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars,
+ SmallVectorImpl<Type> &types,
+ SmallVectorImpl<OpAsmParser::UnresolvedOperand> &stepVars,
+ SmallVectorImpl<Type> &stepTypes, ArrayAttr &varTypes,
+ ArrayAttr &modifiers, ArrayAttr &syms)
+ : vars(vars), types(types), stepVars(stepVars), stepTypes(stepTypes),
+ varTypes(varTypes), modifiers(modifiers), syms(syms) {}
+};
+
struct ReductionParseArgs {
SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars;
SmallVectorImpl<Type> &types;
@@ -1288,6 +1307,7 @@ struct AllRegionParseArgs {
std::optional<MapParseArgs> hasDeviceAddrArgs;
std::optional<MapParseArgs> hostEvalArgs;
std::optional<ReductionParseArgs> inReductionArgs;
+ std::optional<LinearParseArgs> linearArgs;
std::optional<MapParseArgs> mapArgs;
std::optional<PrivateParseArgs> privateArgs;
std::optional<ReductionParseArgs> reductionArgs;
@@ -1453,6 +1473,63 @@ static ParseResult parseBlockArgClause(
return success();
}
+/// linear-entry := [modifier `(`] symbol ssa-id `->` ssa-id `:` type `=`
+/// ssa-id `:` type [`)`]
+static ParseResult parseBlockArgClause(
+ OpAsmParser &parser,
+ llvm::SmallVectorImpl<OpAsmParser::Argument> &entryBlockArgs,
+ StringRef keyword, std::optional<LinearParseArgs> linearArgs) {
+ if (!linearArgs || failed(parser.parseOptionalKeyword(keyword)))
+ return success();
+
+ MLIRContext *ctx = parser.getContext();
+ SmallVector<Attribute> syms, modifiers;
+ if (parser.parseLParen() ||
+ parser.parseCommaSeparatedList([&]() -> ParseResult {
+ std::optional<LinearModifier> modifier;
+ if (succeeded(parser.parseOptionalKeyword("val")))
+ modifier = LinearModifier::val;
+ else if (succeeded(parser.parseOptionalKeyword("ref")))
+ modifier = LinearModifier::ref;
+ else if (succeeded(parser.parseOptionalKeyword("uval")))
+ modifier = LinearModifier::uval;
+ if (modifier && parser.parseLParen())
+ return failure();
+
+ SymbolRefAttr sym;
+ OpAsmParser::Argument &arg = entryBlockArgs.emplace_back();
+ if (parser.parseAttribute(sym) ||
+ parser.parseOperand(linearArgs->vars.emplace_back()) ||
+ parser.parseArrow() || parser.parseArgument(arg) ||
+ parser.parseColonType(linearArgs->types.emplace_back()) ||
+ parser.parseEqual() ||
+ parser.parseOperand(linearArgs->stepVars.emplace_back()) ||
+ parser.parseColonType(linearArgs->stepTypes.emplace_back()))
+ return failure();
+ arg.type = linearArgs->types.back();
+
+ if (modifier && parser.parseRParen())
+ return failure();
+
+ syms.push_back(sym);
+ modifiers.push_back(
+ modifier ? Attribute(LinearModifierAttr::get(ctx, *modifier))
+ : Attribute(UnitAttr::get(ctx)));
+ return success();
+ }) ||
+ parser.parseRParen())
+ return failure();
+
+ linearArgs->syms = ArrayAttr::get(ctx, syms);
+ linearArgs->modifiers = ArrayAttr::get(ctx, modifiers);
+
+ if (succeeded(parser.parseOptionalKeyword("linear_var_types")) &&
+ (parser.parseLParen() || parser.parseAttribute(linearArgs->varTypes) ||
+ parser.parseRParen()))
+ return failure();
+ return success();
+}
+
static ParseResult parseBlockArgRegion(OpAsmParser &parser, Region ®ion,
AllRegionParseArgs args) {
llvm::SmallVector<OpAsmParser::Argument> entryBlockArgs;
@@ -1472,6 +1549,11 @@ static ParseResult parseBlockArgRegion(OpAsmParser &parser, Region ®ion,
return parser.emitError(parser.getCurrentLocation())
<< "invalid `in_reduction` format";
+ if (failed(parseBlockArgClause(parser, entryBlockArgs, "linear",
+ args.linearArgs)))
+ return parser.emitError(parser.getCurrentLocation())
+ << "invalid `linear` format";
+
if (failed(parseBlockArgClause(parser, entryBlockArgs, "map_entries",
args.mapArgs)))
return parser.emitError(parser.getCurrentLocation())
@@ -1591,6 +1673,30 @@ static ParseResult parsePrivateReductionRegion(
return parseBlockArgRegion(parser, region, args);
}
+static ParseResult parseLinearPrivateReductionRegion(
+ OpAsmParser &parser, Region ®ion,
+ SmallVectorImpl<OpAsmParser::UnresolvedOperand> &linearVars,
+ SmallVectorImpl<Type> &linearTypes,
+ SmallVectorImpl<OpAsmParser::UnresolvedOperand> &linearStepVars,
+ SmallVectorImpl<Type> &linearStepTypes, ArrayAttr &linearVarTypes,
+ ArrayAttr &linearModifiers, ArrayAttr &linearSyms,
+ llvm::SmallVectorImpl<OpAsmParser::UnresolvedOperand> &privateVars,
+ llvm::SmallVectorImpl<Type> &privateTypes, ArrayAttr &privateSyms,
+ UnitAttr &privateNeedsBarrier, ReductionModifierAttr &reductionMod,
+ SmallVectorImpl<OpAsmParser::UnresolvedOperand> &reductionVars,
+ SmallVectorImpl<Type> &reductionTypes, DenseBoolArrayAttr &reductionByref,
+ ArrayAttr &reductionSyms) {
+ AllRegionParseArgs args;
+ args.linearArgs.emplace(linearVars, linearTypes, linearStepVars,
+ linearStepTypes, linearVarTypes, linearModifiers,
+ linearSyms);
+ args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
+ privateNeedsBarrier);
+ args.reductionArgs.emplace(reductionVars, reductionTypes, reductionByref,
+ reductionSyms, &reductionMod);
+ return parseBlockArgRegion(parser, region, args);
+}
+
static ParseResult parseTaskReductionRegion(
OpAsmParser &parser, Region ®ion,
SmallVectorImpl<OpAsmParser::UnresolvedOperand> &taskReductionVars,
@@ -1635,6 +1741,20 @@ struct PrivatePrintArgs {
: vars(vars), types(types), syms(syms), needsBarrier(needsBarrier),
mapIndices(mapIndices) {}
};
+struct LinearPrintArgs {
+ ValueRange vars;
+ TypeRange types;
+ ValueRange stepVars;
+ TypeRange stepTypes;
+ ArrayAttr varTypes;
+ ArrayAttr modifiers;
+ ArrayAttr syms;
+ LinearPrintArgs(ValueRange vars, TypeRange types, ValueRange stepVars,
+ TypeRange stepTypes, ArrayAttr varTypes, ArrayAttr modifiers,
+ ArrayAttr syms)
+ : vars(vars), types(types), stepVars(stepVars), stepTypes(stepTypes),
+ varTypes(varTypes), modifiers(modifiers), syms(syms) {}
+};
struct ReductionPrintArgs {
ValueRange vars;
TypeRange types;
@@ -1649,6 +1769,7 @@ struct AllRegionPrintArgs {
std::optional<MapPrintArgs> hasDeviceAddrArgs;
std::optional<MapPrintArgs> hostEvalArgs;
std::optional<ReductionPrintArgs> inReductionArgs;
+ std::optional<LinearPrintArgs> linearArgs;
std::optional<MapPrintArgs> mapArgs;
std::optional<PrivatePrintArgs> privateArgs;
std::optional<ReductionPrintArgs> reductionArgs;
@@ -1739,6 +1860,35 @@ printBlockArgClause(OpAsmPrinter &p, MLIRContext *ctx, StringRef clauseName,
reductionArgs->byref, reductionArgs->modifier);
}
+static void printBlockArgClause(OpAsmPrinter &p, MLIRContext *ctx,
+ StringRef clauseName, ValueRange argsSubrange,
+ std::optional<LinearPrintArgs> linearArgs) {
+ if (!linearArgs || argsSubrange.empty())
+ return;
+
+ p << clauseName << "(";
+ for (auto [i, var] : llvm::enumerate(linearArgs->vars)) {
+ if (i != 0)
+ p << ", ";
+ auto modifier =
+ linearArgs->modifiers
+ ? dyn_cast_if_present<LinearModifierAttr>(linearArgs->modifiers[i])
+ : nullptr;
+ if (modifier)
+ p << stringifyLinearModifier(modifier.getValue()) << "(";
+ if (linearArgs->syms)
+ p << linearArgs->syms[i] << " ";
+ p << var << " -> " << argsSubrange[i] << " : " << linearArgs->types[i]
+ << " = " << linearArgs->stepVars[i] << " : " << linearArgs->stepTypes[i];
+ if (modifier)
+ p << ")";
+ }
+ p << ") ";
+
+ if (linearArgs->varTypes)
+ p << "linear_var_types(" << linearArgs->varTypes << ") ";
+}
+
static void printBlockArgRegion(OpAsmPrinter &p, Operation *op, Region ®ion,
const AllRegionPrintArgs &args) {
auto iface = llvm::cast<mlir::omp::BlockArgOpenMPOpInterface>(op);
@@ -1751,6 +1901,8 @@ static void printBlockArgRegion(OpAsmPrinter &p, Operation *op, Region ®ion,
args.hostEvalArgs);
printBlockArgClause(p, ctx, "in_reduction", iface.getInReductionBlockArgs(),
args.inReductionArgs);
+ printBlockArgClause(p, ctx, "linear", iface.getLinearBlockArgs(),
+ args.linearArgs);
printBlockArgClause(p, ctx, "map_entries", iface.getMapBlockArgs(),
args.mapArgs);
printBlockArgClause(p, ctx, "private", iface.getPrivateBlockArgs(),
@@ -1848,6 +2000,26 @@ static void printPrivateReductionRegion(
printBlockArgRegion(p, op, region, args);
}
+static void printLinearPrivateReductionRegion(
+ OpAsmPrinter &p, Operation *op, Region ®ion, ValueRange linearVars,
+ TypeRange linearTypes, ValueRange linearStepVars, TypeRange linearStepTypes,
+ ArrayAttr linearVarTypes, ArrayAttr linearModifiers, ArrayAttr linearSyms,
+ ValueRange privateVars, TypeRange privateTypes, ArrayAttr privateSyms,
+ UnitAttr privateNeedsBarrier, ReductionModifierAttr reductionMod,
+ ValueRange reductionVars, TypeRange reductionTypes,
+ DenseBoolArrayAttr reductionByref, ArrayAttr reductionSyms) {
+ AllRegionPrintArgs args;
+ args.linearArgs.emplace(linearVars, linearTypes, linearStepVars,
+ linearStepTypes, linearVarTypes, linearModifiers,
+ linearSyms);
+ args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
+ privateNeedsBarrier,
+ /*mapIndices=*/nullptr);
+ args.reductionArgs.emplace(reductionVars, reductionTypes, reductionByref,
+ reductionSyms, reductionMod);
+ printBlockArgRegion(p, op, region, args);
+}
+
static void printTaskReductionRegion(OpAsmPrinter &p, Operation *op,
Region ®ion,
ValueRange taskReductionVars,
@@ -3062,17 +3234,51 @@ static LogicalResult verifyPrivateVarList(OpType &op) {
if (privatizerType && (varType != privatizerType))
return op.emitError()
<< "type mismatch between a "
- << (privatizerOp.getDataSharingType() ==
- DataSharingClauseType::Private
- ? "private"
- : "firstprivate")
+ << stringifyDataSharingClauseType(
+ privatizerOp.getDataSharingType())
<< " variable and its privatizer op, var type: " << varType
<< " vs. privatizer op type: " << privatizerType;
+
+ if (privatizerOp.getDataSharingType() == DataSharingClauseType::Linear)
+ return op.emitError() << "`linear` privatizer '" << privateSym
+ << "' can only be used in a `linear` clause";
}
return success();
}
+/// Verifies that each linear variable has a `linear` privatizer.
+template <typename OpType>
+static LogicalResult verifyLinearSyms(OpType op) {
+ OperandRange linearVars = op.getLinearVars();
+ std::optional<ArrayAttr> linearSyms = op.getLinearSyms();
+ if (linearVars.empty() && (!linearSyms || linearSyms->empty()))
+ return success();
+
+ if (!linearSyms || linearSyms->size() != linearVars.size())
+ return op.emitOpError()
+ << "expected as many linear privatizers as linear variables";
+
+ for (auto [var, sym] : llvm::zip_equal(linearVars, *linearSyms)) {
+ auto symRef = cast<SymbolRefAttr>(sym);
+ auto privatizer =
+ SymbolTable::lookupNearestSymbolFrom<PrivateClauseOp>(op, symRef);
+ if (!privatizer)
+ return op.emitOpError() << "failed to lookup privatizer op with symbol: '"
+ << symRef << "'";
+ if (privatizer.getDataSharingType() != DataSharingClauseType::Linear)
+ return op.emitOpError() << "privatizer '" << symRef
+ << "' of a linear variable must be `linear`";
+ Type argType = privatizer.getArgType();
+ if (argType && argType != var.getType())
+ return op.emitOpError()
+ << "type mismatch between a linear variable and its privatizer "
+ "op, var type: "
+ << var.getType() << " vs. privatizer op type: " << argType;
+ }
+ return success();
+}
+
LogicalResult ParallelOp::verify() {
if (failed(verifyPrivateVarList(*this)))
return failure();
@@ -3509,7 +3715,8 @@ void WsloopOp::build(OpBuilder &builder, OperationState &state,
/*allocate_private_indices=*/nullptr,
/*linear_vars=*/ValueRange(), /*linear_step_vars=*/ValueRange(),
/*linear_var_types*/ nullptr, /*linear_modifiers=*/nullptr,
- /*nowait=*/false, /*order=*/nullptr, /*order_mod=*/nullptr,
+ /*linear_syms=*/nullptr, /*nowait=*/false, /*order=*/nullptr,
+ /*order_mod=*/nullptr,
/*ordered=*/nullptr, /*private_vars=*/{}, /*private_syms=*/nullptr,
/*private_needs_barrier=*/false,
/*reduction_mod=*/nullptr, /*reduction_vars=*/ValueRange(),
@@ -3528,10 +3735,10 @@ void WsloopOp::build(OpBuilder &builder, OperationState &state,
makeDenseI64ArrayAttr(ctx, clauses.allocateAlignments),
makeDenseI64ArrayAttr(ctx, clauses.allocatePrivateIndices),
clauses.linearVars, clauses.linearStepVars, clauses.linearVarTypes,
- clauses.linearModifiers, clauses.nowait, clauses.order, clauses.orderMod,
- clauses.ordered, clauses.privateVars,
- makeArrayAttr(ctx, clauses.privateSyms), clauses.privateNeedsBarrier,
- clauses.reductionMod, clauses.reductionVars,
+ clauses.linearModifiers, makeArrayAttr(ctx, clauses.linearSyms),
+ clauses.nowait, clauses.order, clauses.orderMod, clauses.ordered,
+ clauses.privateVars, makeArrayAttr(ctx, clauses.privateSyms),
+ clauses.privateNeedsBarrier, clauses.reductionMod, clauses.reductionVars,
makeDenseBoolArrayAttr(ctx, clauses.reductionByref),
makeArrayAttr(ctx, clauses.reductionSyms), clauses.scheduleKind,
clauses.scheduleChunk, clauses.scheduleMod, clauses.scheduleSimd);
@@ -3551,7 +3758,7 @@ LogicalResult WsloopOp::verify() {
getLinearVarTypes().value().size() != getLinearVars().size())
return emitError() << "Ill-formed type attributes for linear variables";
- if (failed(verifyPrivateVarList(*this)))
+ if (failed(verifyLinearSyms(*this)) || failed(verifyPrivateVarList(*this)))
return failure();
return verifyReductionVarList(*this, getReductionSyms(), getReductionVars(),
@@ -3590,17 +3797,17 @@ LogicalResult WsloopOp::verifyRegions() {
void SimdOp::build(OpBuilder &builder, OperationState &state,
const SimdOperands &clauses) {
MLIRContext *ctx = builder.getContext();
- SimdOp::build(builder, state, clauses.alignedVars,
- makeArrayAttr(ctx, clauses.alignments), clauses.ifExpr,
- clauses.linearVars, clauses.linearStepVars,
- clauses.linearVarTypes, clauses.linearModifiers,
- clauses.nontemporalVars, clauses.order, clauses.orderMod,
- clauses.privateVars, makeArrayAttr(ctx, clauses.privateSyms),
- clauses.privateNeedsBarrier, clauses.reductionMod,
- clauses.reductionVars,
- makeDenseBoolArrayAttr(ctx, clauses.reductionByref),
- makeArrayAttr(ctx, clauses.reductionSyms), clauses.safelen,
- clauses.simdlen);
+ SimdOp::build(
+ builder, state, clauses.alignedVars,
+ makeArrayAttr(ctx, clauses.alignments), clauses.ifExpr,
+ clauses.linearVars, clauses.linearStepVars, clauses.linearVarTypes,
+ clauses.linearModifiers, makeArrayAttr(ctx, clauses.linearSyms),
+ clauses.nontemporalVars, clauses.order, clauses.orderMod,
+ clauses.privateVars, makeArrayAttr(ctx, clauses.privateSyms),
+ clauses.privateNeedsBarrier, clauses.reductionMod, clauses.reductionVars,
+ makeDenseBoolArrayAttr(ctx, clauses.reductionByref),
+ makeArrayAttr(ctx, clauses.reductionSyms), clauses.safelen,
+ clauses.simdlen);
}
LogicalResult SimdOp::verify() {
@@ -3655,6 +3862,9 @@ LogicalResult SimdOp::verify() {
getLinearVarTypes().value().size() != getLinearVars().size())
return emitError() << "Ill-formed type attributes for linear variables";
+ if (failed(verifyLinearSyms(*this)))
+ return failure();
+
llvm::DenseSet<Value> privateVars(llvm::from_range, getPrivateVars());
llvm::DenseSet<Value> reductionVars(llvm::from_range, getReductionVars());
// TODO Check lastprivate vars when their support is added to SimdOp.
@@ -5332,8 +5542,8 @@ void PrivateClauseOp::build(OpBuilder &odsBuilder, OperationState &odsState,
LogicalResult PrivateClauseOp::verifyRegions() {
Type argType = getArgType();
- auto verifyTerminator = [&](Operation *terminator,
- bool yieldsValue) -> LogicalResult {
+ auto verifyTerminator = [&](Operation *terminator, bool yieldsValue,
+ bool yieldsAnyType) -> LogicalResult {
if (!terminator->getBlock()->getSuccessors().empty())
return success();
@@ -5352,12 +5562,17 @@ LogicalResult PrivateClauseOp::verifyRegions() {
<< "Did not expect any values to be yielded.";
}
- if (yieldedTypes.size() == 1 && yieldedTypes.front() == argType)
+ if (yieldedTypes.size() == 1 &&
+ (yieldsAnyType || yieldedTypes.front() == argType))
return success();
auto error = mlir::emitError(yieldOp.getLoc())
- << "Invalid yielded value. Expected type: " << argType
- << ", got: ";
+ << "Invalid yielded value. Expected ";
+ if (yieldsAnyType)
+ error << "a single value";
+ else
+ error << "type: " << argType;
+ error << ", got: ";
if (yieldedTypes.empty())
error << "None";
@@ -5368,8 +5583,8 @@ LogicalResult PrivateClauseOp::verifyRegions() {
};
auto verifyRegion = [&](Region ®ion, unsigned expectedNumArgs,
- StringRef regionName,
- bool yieldsValue) -> LogicalResult {
+ StringRef regionName, bool yieldsValue,
+ bool yieldsAnyType = false) -> LogicalResult {
assert(!region.empty());
if (region.getNumArguments() != expectedNumArgs)
@@ -5382,7 +5597,8 @@ LogicalResult PrivateClauseOp::verifyRegions() {
if (!block.mightHaveTerminator())
continue;
- if (failed(verifyTerminator(block.getTerminator(), yieldsValue)))
+ if (failed(verifyTerminator(block.getTerminator(), yieldsValue,
+ yieldsAnyType)))
return failure();
}
@@ -5416,6 +5632,19 @@ LogicalResult PrivateClauseOp::verifyRegions() {
/*yieldsValue=*/true)))
return failure();
+ if (dsType == DataSharingClauseType::Linear && !getCopyRegion().empty())
+ return emitError("`linear` clauses do not require a `copy` region.");
+
+ if (!getDataAddressRegion().empty()) {
+ if (dsType != DataSharingClauseType::Linear)
+ return emitError(
+ "only `linear` clauses can have a `data_address` region.");
+ if (failed(verifyRegion(getDataAddressRegion(), /*expectedNumArgs=*/1,
+ "data_address", /*yieldsValue=*/true,
+ /*yieldsAnyType=*/true)))
+ return failure();
+ }
+
if (!getDeallocRegion().empty() &&
failed(verifyRegion(getDeallocRegion(), /*expectedNumArgs=*/1, "dealloc",
/*yieldsValue=*/false)))
@@ -5577,18 +5806,21 @@ LogicalResult DeclareSimdOp::verify() {
/*isDeclareSimd=*/true)))
return failure();
+ if (getLinearSyms())
+ return emitOpError("linear privatizers are not allowed");
+
return verifyAlignedClause(*this, getAlignments(), getAlignedVars());
}
void DeclareSimdOp::build(OpBuilder &odsBuilder, OperationState &odsState,
const DeclareSimdOperands &clauses) {
MLIRContext *ctx = odsBuilder.getContext();
- DeclareSimdOp::build(odsBuilder, odsState, clauses.alignedVars,
- makeArrayAttr(ctx, clauses.alignments), clauses.inbranch,
- clauses.linearVars, clauses.linearStepVars,
- clauses.linearVarTypes, clauses.linearModifiers,
- clauses.notinbranch, clauses.simdlen,
- clauses.uniformVars);
+ DeclareSimdOp::build(
+ odsBuilder, odsState, clauses.alignedVars,
+ makeArrayAttr(ctx, clauses.alignments), clauses.inbranch,
+ clauses.linearVars, clauses.linearStepVars, clauses.linearVarTypes,
+ clauses.linearModifiers, makeArrayAttr(ctx, clauses.linearSyms),
+ clauses.notinbranch, clauses.simdlen, clauses.uniformVars);
}
//===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index ba9b30e69aace..6cb179cc10729 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -51,6 +51,11 @@
using namespace mlir;
+static llvm::Expected<llvm::Value *>
+getPrivateDataAddress(llvm::IRBuilderBase &builder,
+ LLVM::ModuleTranslation &moduleTranslation,
+ omp::PrivateClauseOp privatizer, llvm::Value *var);
+
namespace {
static llvm::omp::ScheduleKind
convertToScheduleKind(std::optional<omp::ClauseScheduleKind> schedKind) {
@@ -144,15 +149,46 @@ char PreviouslyReportedError::ID = 0;
class LinearClauseProcessor {
private:
- SmallVector<llvm::Value *> linearPreconditionVars;
- SmallVector<llvm::Value *> linearLoopBodyTemps;
- SmallVector<llvm::Value *> linearOrigVal;
- SmallVector<llvm::Value *> linearSteps;
SmallVector<llvm::Type *> linearVarTypes;
+ SmallVector<llvm::Value *> linearSteps;
+ // The `linear` privatizer, original and private copy of each linear var.
+ SmallVector<omp::PrivateClauseOp> linearPrivatizers;
+ SmallVector<llvm::Value *> linearOrigVars;
+ SmallVector<llvm::Value *> linearPrivateVars;
+ // The value of each linear var on entry to the loop.
+ SmallVector<llvm::Value *> linearStartVals;
llvm::BasicBlock *linearFinalizationBB;
llvm::BasicBlock *linearExitBB;
llvm::BasicBlock *linearLastIterExitBB;
- Value linearLoopIV;
+ std::optional<size_t> linearLoopIVIndex;
+
+ // Address of the value of linear var `index` held by `var`.
+ llvm::Expected<llvm::Value *>
+ getAddress(llvm::IRBuilderBase &builder,
+ LLVM::ModuleTranslation &moduleTranslation, size_t index,
+ llvm::Value *var) {
+ return getPrivateDataAddress(builder, moduleTranslation,
+ linearPrivatizers[index], var);
+ }
+
+ // Copy the current value of each linear var back to the original.
+ llvm::Error storeLinearVars(llvm::IRBuilderBase &builder,
+ LLVM::ModuleTranslation &moduleTranslation) {
+ for (size_t index = 0; index < linearOrigVars.size(); index++) {
+ llvm::Expected<llvm::Value *> from = getAddress(
+ builder, moduleTranslation, index, linearPrivateVars[index]);
+ if (!from)
+ return from.takeError();
+ llvm::LoadInst *linearVarTemp =
+ builder.CreateLoad(linearVarTypes[index], *from);
+ llvm::Expected<llvm::Value *> to =
+ getAddress(builder, moduleTranslation, index, linearOrigVars[index]);
+ if (!to)
+ return to.takeError();
+ builder.CreateStore(linearVarTemp, *to);
+ }
+ return llvm::Error::success();
+ }
public:
// Register type for the linear variables
@@ -162,16 +198,18 @@ class LinearClauseProcessor {
mlir::cast<mlir::TypeAttr>(ty).getValue()));
}
- // Allocate space for linear variabes
+ // Register a linear var, allocating space for its start value at `allocaIP`
void createLinearVar(llvm::IRBuilderBase &builder,
- LLVM::ModuleTranslation &moduleTranslation,
- llvm::Value *linearVar, int idx) {
- linearPreconditionVars.push_back(
+ const llvm::OpenMPIRBuilder::InsertPointTy &allocaIP,
+ omp::PrivateClauseOp privatizer, llvm::Value *origVar,
+ llvm::Value *privateVar, int idx) {
+ llvm::IRBuilderBase::InsertPointGuard guard(builder);
+ builder.SetInsertPoint(allocaIP.getNodeParent()->getTerminator());
+ linearStartVals.push_back(
builder.CreateAlloca(linearVarTypes[idx], nullptr, ".linear_var"));
- llvm::Value *linearLoopBodyTemp =
- builder.CreateAlloca(linearVarTypes[idx], nullptr, ".linear_result");
- linearOrigVal.push_back(linearVar);
- linearLoopBodyTemps.push_back(linearLoopBodyTemp);
+ linearPrivatizers.push_back(privatizer);
+ linearOrigVars.push_back(origVar);
+ linearPrivateVars.push_back(privateVar);
}
// Initialize linear step
@@ -181,15 +219,20 @@ class LinearClauseProcessor {
}
// Emit IR for initialization of linear variables
- void initLinearVar(llvm::IRBuilderBase &builder,
- LLVM::ModuleTranslation &moduleTranslation,
- llvm::BasicBlock *loopPreHeader) {
+ llvm::Error initLinearVar(llvm::IRBuilderBase &builder,
+ LLVM::ModuleTranslation &moduleTranslation,
+ llvm::BasicBlock *loopPreHeader) {
builder.SetInsertPoint(loopPreHeader->getTerminator());
- for (size_t index = 0; index < linearOrigVal.size(); index++) {
+ for (size_t index = 0; index < linearOrigVars.size(); index++) {
+ llvm::Expected<llvm::Value *> orig =
+ getAddress(builder, moduleTranslation, index, linearOrigVars[index]);
+ if (!orig)
+ return orig.takeError();
llvm::LoadInst *linearVarLoad =
- builder.CreateLoad(linearVarTypes[index], linearOrigVal[index]);
- builder.CreateStore(linearVarLoad, linearPreconditionVars[index]);
+ builder.CreateLoad(linearVarTypes[index], *orig);
+ builder.CreateStore(linearVarLoad, linearStartVals[index]);
}
+ return llvm::Error::success();
}
// Find linear iteration variable and save it for later updates
@@ -199,36 +242,71 @@ class LinearClauseProcessor {
if (loopOp.getIVs().size() != 1)
return success();
// Currently, frontends using `omp.simd` always generate a store from the
- // `omp.loop_nest`'s IV to the corresponding iteration variable.
- // We leverage this to find the linear iteration variable.
+ // `omp.loop_nest`'s IV to the corresponding iteration variable, possibly
+ // through the data address of its private copy.
//
// TODO Add an attribute to `omp.loop_nest` that explicitly lists the
// variables that correspond to the loop induction variables.
- BlockArgument arg = loopOp.getIVs().front();
- for (const Operation *user : arg.getUsers()) {
- if (auto storeOp = dyn_cast<LLVM::StoreOp>(user)) {
- for (Value linearVar : simdOp.getLinearVars()) {
- if (linearVar == storeOp.getAddr()) {
- if (linearLoopIV && linearLoopIV != linearVar)
- return simdOp.emitError(
- "Could not determine the linear variable associated with the "
- "loop nest induction variable");
- linearLoopIV = linearVar;
+ MutableArrayRef<BlockArgument> linearArgs =
+ cast<omp::BlockArgOpenMPOpInterface>(*simdOp).getLinearBlockArgs();
+ SmallVector<Value> ivValues{loopOp.getIVs().front()};
+ for (Operation *user : loopOp.getIVs().front().getUsers())
+ if (isa<LLVM::SExtOp, LLVM::ZExtOp, LLVM::TruncOp>(user))
+ ivValues.push_back(user->getResult(0));
+
+ for (Value ivValue : ivValues) {
+ for (Operation *user : ivValue.getUsers()) {
+ auto storeOp = dyn_cast<LLVM::StoreOp>(user);
+ if (!storeOp || storeOp.getValue() != ivValue)
+ continue;
+ // Look through loads, GEPs and copies of a private copy, e.g. of a
+ // descriptor, to find the linear block argument the IV is stored to.
+ Value addr = storeOp.getAddr();
+ while (true) {
+ Operation *def = addr.getDefiningOp();
+ if (auto load = dyn_cast_if_present<LLVM::LoadOp>(def)) {
+ addr = load.getAddr();
+ } else if (auto gep = dyn_cast_if_present<LLVM::GEPOp>(def)) {
+ addr = gep.getBase();
+ } else {
+ auto copy = llvm::find_if(addr.getUsers(), [&](Operation *user) {
+ auto memcpy = dyn_cast<LLVM::MemcpyOp>(user);
+ return memcpy && memcpy.getDst() == addr &&
+ llvm::is_contained(linearArgs, memcpy.getSrc());
+ });
+ if (copy == addr.getUsers().end())
+ break;
+ addr = cast<LLVM::MemcpyOp>(*copy).getSrc();
}
}
+ auto *it = llvm::find(linearArgs, addr);
+ if (it == linearArgs.end())
+ continue;
+ size_t index = std::distance(linearArgs.begin(), it);
+ if (linearLoopIVIndex && *linearLoopIVIndex != index)
+ return simdOp.emitError(
+ "Could not determine the linear variable associated with the "
+ "loop nest induction variable");
+ linearLoopIVIndex = index;
}
}
return success();
}
// Emit IR for updating Linear variables
- void updateLinearVar(llvm::IRBuilderBase &builder, llvm::BasicBlock *loopBody,
- llvm::Value *loopInductionVar) {
+ llvm::Error updateLinearVar(llvm::IRBuilderBase &builder,
+ LLVM::ModuleTranslation &moduleTranslation,
+ llvm::BasicBlock *loopBody,
+ llvm::Value *loopInductionVar) {
builder.SetInsertPoint(loopBody->getTerminator());
- for (size_t index = 0; index < linearPreconditionVars.size(); index++) {
+ for (size_t index = 0; index < linearStartVals.size(); index++) {
llvm::Type *linearVarType = linearVarTypes[index];
llvm::Value *iv = loopInductionVar;
llvm::Value *step = linearSteps[index];
+ llvm::Expected<llvm::Value *> result = getAddress(
+ builder, moduleTranslation, index, linearPrivateVars[index]);
+ if (!result)
+ return result.takeError();
if (!iv->getType()->isIntegerTy())
llvm_unreachable("OpenMP loop induction variable must be an integer "
@@ -240,53 +318,48 @@ class LinearClauseProcessor {
step = builder.CreateSExtOrTrunc(step, linearVarType);
llvm::LoadInst *linearVarStart =
- builder.CreateLoad(linearVarType, linearPreconditionVars[index]);
+ builder.CreateLoad(linearVarType, linearStartVals[index]);
llvm::Value *mulInst = builder.CreateMul(iv, step);
llvm::Value *addInst = builder.CreateAdd(linearVarStart, mulInst);
- builder.CreateStore(addInst, linearLoopBodyTemps[index]);
+ builder.CreateStore(addInst, *result);
} else if (linearVarType->isFloatingPointTy()) {
// Float path: perform multiply in integer, then convert to float
step = builder.CreateSExtOrTrunc(step, iv->getType());
llvm::Value *mulInst = builder.CreateMul(iv, step);
llvm::LoadInst *linearVarStart =
- builder.CreateLoad(linearVarType, linearPreconditionVars[index]);
+ builder.CreateLoad(linearVarType, linearStartVals[index]);
llvm::Value *mulFp = builder.CreateSIToFP(mulInst, linearVarType);
llvm::Value *addInst = builder.CreateFAdd(linearVarStart, mulFp);
- builder.CreateStore(addInst, linearLoopBodyTemps[index]);
+ builder.CreateStore(addInst, *result);
} else {
llvm_unreachable(
"Linear variable must be of integer or floating-point type");
}
}
+ return llvm::Error::success();
}
// Emit IR for updating linear iteration variables on loop exit
- void updateLinearIV(llvm::IRBuilderBase &builder,
- LLVM::ModuleTranslation &moduleTranslation) {
- if (!linearLoopIV)
- return;
- llvm::Value *linearIV = moduleTranslation.lookupValue(linearLoopIV);
-
- // Find linearIV's index
- size_t index;
- for (index = 0; index < linearOrigVal.size(); index++)
- if (linearIV == linearOrigVal[index])
- break;
- if (index == linearOrigVal.size())
- return;
+ llvm::Error updateLinearIV(llvm::IRBuilderBase &builder,
+ LLVM::ModuleTranslation &moduleTranslation) {
+ if (!linearLoopIVIndex)
+ return llvm::Error::success();
+ size_t index = *linearLoopIVIndex;
// Add one more step to the linear iteration variable
llvm::Type *varType = linearVarTypes[index];
- llvm::Value *var = linearLoopBodyTemps[index];
- llvm::Value *step = linearSteps[index];
if (!varType->isIntegerTy())
llvm_unreachable("Linear iteration variable must be of integer type");
-
- step = builder.CreateSExtOrTrunc(step, varType);
- llvm::Value *val = builder.CreateLoad(varType, var);
+ llvm::Expected<llvm::Value *> var =
+ getAddress(builder, moduleTranslation, index, linearPrivateVars[index]);
+ if (!var)
+ return var.takeError();
+ llvm::Value *step = builder.CreateSExtOrTrunc(linearSteps[index], varType);
+ llvm::Value *val = builder.CreateLoad(varType, *var);
llvm::Value *addInst = builder.CreateAdd(val, step);
- builder.CreateStore(addInst, var);
+ builder.CreateStore(addInst, *var);
+ return llvm::Error::success();
}
// Linear variable finalization is conditional on the last logical iteration.
@@ -316,11 +389,8 @@ class LinearClauseProcessor {
llvm::Type::getInt32Ty(builder.getContext()), 0));
// Store the linear variable values to original variables.
builder.SetInsertPoint(linearLastIterExitBB->getTerminator());
- for (size_t index = 0; index < linearOrigVal.size(); index++) {
- llvm::LoadInst *linearVarTemp =
- builder.CreateLoad(linearVarTypes[index], linearLoopBodyTemps[index]);
- builder.CreateStore(linearVarTemp, linearOrigVal[index]);
- }
+ if (llvm::Error err = storeLinearVars(builder, moduleTranslation))
+ return std::move(err);
// Create conditional branch such that the linear variable
// values are stored to original variables only at the
@@ -336,48 +406,10 @@ class LinearClauseProcessor {
// Emit stores for linear variables. Useful in case of SIMD
// construct.
- void emitStoresForLinearVar(llvm::IRBuilderBase &builder) {
- for (size_t index = 0; index < linearOrigVal.size(); index++) {
- llvm::LoadInst *linearVarTemp =
- builder.CreateLoad(linearVarTypes[index], linearLoopBodyTemps[index]);
- builder.CreateStore(linearVarTemp, linearOrigVal[index]);
- }
- }
-
- // Rewrite all uses of the original variable, in the basic blocks in the
- // [startBB, endBB] interval, with the linear variable in-place.
- void rewriteInPlace(llvm::IRBuilderBase &builder, llvm::BasicBlock *startBB,
- llvm::BasicBlock *endBB, size_t varIndex) {
- llvm::SmallVector<llvm::BasicBlock *, 32> worklist;
- llvm::SmallPtrSet<llvm::BasicBlock *, 32> collectedBBs;
-
- assert(startBB && endBB && "Invalid startBB/endBB");
-
- // Collect basic blocks from startBB to endBB.
- worklist.push_back(startBB);
- collectedBBs.insert(startBB);
-
- while (!worklist.empty()) {
- llvm::BasicBlock *bb = worklist.pop_back_val();
-
- if (bb == endBB)
- continue;
-
- for (llvm::BasicBlock *succ : llvm::successors(bb)) {
- if (collectedBBs.insert(succ).second)
- worklist.push_back(succ);
- }
- }
-
- // Rewrite all uses in the collected BBs.
- llvm::SmallVector<llvm::User *> users(linearOrigVal[varIndex]->users());
- for (auto *user : users) {
- if (auto *userInst = dyn_cast<llvm::Instruction>(user)) {
- if (collectedBBs.contains(userInst->getParent()))
- user->replaceUsesOfWith(linearOrigVal[varIndex],
- linearLoopBodyTemps[varIndex]);
- }
- }
+ llvm::Error
+ emitStoresForLinearVar(llvm::IRBuilderBase &builder,
+ LLVM::ModuleTranslation &moduleTranslation) {
+ return storeLinearVars(builder, moduleTranslation);
}
};
@@ -1036,8 +1068,8 @@ struct PrivateVarsInfo {
template <typename OP>
PrivateVarsInfo(OP op)
- : blockArgs(
- cast<omp::BlockArgOpenMPOpInterface>(*op).getPrivateBlockArgs()) {
+ : blockArgs(llvm::to_vector(
+ cast<omp::BlockArgOpenMPOpInterface>(*op).getPrivateBlockArgs())) {
mlirVars.reserve(blockArgs.size());
llvmVars.reserve(blockArgs.size());
collectPrivatizationDecls<OP>(op);
@@ -1046,7 +1078,23 @@ struct PrivateVarsInfo {
mlirVars.push_back(privateVar);
}
- MutableArrayRef<BlockArgument> blockArgs;
+ /// Adds the linear variables of `op`, which are privatized by their
+ /// `linear` privatizers, after the private variables.
+ template <typename OP>
+ void addLinearVars(OP op) {
+ numPrivateVars = blockArgs.size();
+ llvm::append_range(
+ blockArgs,
+ cast<omp::BlockArgOpenMPOpInterface>(*op).getLinearBlockArgs());
+ llvm::append_range(mlirVars, op.getLinearVars());
+ if (std::optional<ArrayAttr> attr = op.getLinearSyms())
+ for (auto symbolRef : attr->getAsRange<SymbolRefAttr>())
+ privatizers.push_back(findPrivatizer(op, symbolRef));
+ }
+
+ /// Number of private variables before the linear ones.
+ size_t numPrivateVars = 0;
+ SmallVector<BlockArgument> blockArgs;
SmallVector<mlir::Value> mlirVars;
SmallVector<llvm::Value *> llvmVars;
SmallVector<omp::PrivateClauseOp> privatizers;
@@ -1872,6 +1920,26 @@ materializeRegionArgValue(llvm::IRBuilderBase &builder,
return builder.CreateLoad(regionArgType, value);
}
+/// Returns the address of the data in `var`, via the `data_address` region.
+static llvm::Expected<llvm::Value *>
+getPrivateDataAddress(llvm::IRBuilderBase &builder,
+ LLVM::ModuleTranslation &moduleTranslation,
+ omp::PrivateClauseOp privatizer, llvm::Value *var) {
+ Region ®ion = privatizer.getDataAddressRegion();
+ if (region.empty())
+ return var;
+
+ BlockArgument arg = privatizer.getDataAddressArg();
+ moduleTranslation.mapValue(
+ arg, materializeRegionArgValue(builder, moduleTranslation, arg, var));
+ SmallVector<llvm::Value *, 1> address;
+ if (failed(inlineConvertOmpRegions(region, "omp.private.data_address",
+ builder, moduleTranslation, &address)))
+ return llvm::make_error<PreviouslyReportedError>();
+ setInsertPointForPossiblyEmptyBlock(builder);
+ return address.front();
+}
+
/// Initialize a single (first)private variable. You probably want to use
/// allocateAndInitPrivateVars instead of this.
/// This returns the private variable which has been initialized. This
@@ -4761,6 +4829,7 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
}
PrivateVarsInfo privateVarsInfo(wsloopOp);
+ privateVarsInfo.addLinearVars(wsloopOp);
SmallVector<omp::DeclareReductionOp> reductionDecls;
collectReductionDecls(wsloopOp, reductionDecls);
@@ -4850,10 +4919,13 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
for (mlir::Attribute linearVarType : linearVarTypes)
linearClauseProcessor.registerType(moduleTranslation, linearVarType);
- for (auto [idx, linearVar] : llvm::enumerate(wsloopOp.getLinearVars()))
+ for (auto [idx, linearVar] : llvm::enumerate(wsloopOp.getLinearVars())) {
+ size_t privIdx = privateVarsInfo.numPrivateVars + idx;
linearClauseProcessor.createLinearVar(
- builder, moduleTranslation, moduleTranslation.lookupValue(linearVar),
- idx);
+ builder, allocaIP, privateVarsInfo.privatizers[privIdx],
+ moduleTranslation.lookupValue(linearVar),
+ privateVarsInfo.llvmVars[privIdx], idx);
+ }
for (mlir::Value linearStep : wsloopOp.getLinearStepVars())
linearClauseProcessor.initLinearStep(moduleTranslation, linearStep);
}
@@ -4868,16 +4940,22 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
// Emit Initialization and Update IR for linear variables
if (!wsloopOp.getLinearVars().empty()) {
- linearClauseProcessor.initLinearVar(builder, moduleTranslation,
- loopInfo->getPreheader());
+ if (failed(handleError(
+ linearClauseProcessor.initLinearVar(builder, moduleTranslation,
+ loopInfo->getPreheader()),
+ opInst)))
+ return failure();
llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterBarrierIP =
moduleTranslation.getOpenMPBuilder()->createBarrier(
builder, llvm::omp::OMPD_barrier);
if (failed(handleError(afterBarrierIP, *loopOp)))
return failure();
builder.restoreIP(*afterBarrierIP);
- linearClauseProcessor.updateLinearVar(builder, loopInfo->getBody(),
- loopInfo->getIndVar());
+ if (failed(handleError(linearClauseProcessor.updateLinearVar(
+ builder, moduleTranslation, loopInfo->getBody(),
+ loopInfo->getIndVar()),
+ opInst)))
+ return failure();
linearClauseProcessor.splitLinearFiniBB(builder, loopInfo->getExit());
}
@@ -4897,10 +4975,6 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
noLoopMode = true;
}
- for (size_t index = 0; index < wsloopOp.getLinearVars().size(); index++)
- linearClauseProcessor.rewriteInPlace(builder, loopInfo->getBody(),
- loopInfo->getLatch(), index);
-
llvm::OpenMPIRBuilder::InsertPointOrErrorTy wsloopIP =
ompBuilder->applyWorkshareLoop(
ompLoc.DL, loopInfo, allocaIP, loopNeedsBarrier,
@@ -5188,6 +5262,7 @@ convertOmpSimd(Operation &opInst, llvm::IRBuilderBase &builder,
return failure();
PrivateVarsInfo privateVarsInfo(simdOp);
+ privateVarsInfo.addLinearVars(simdOp);
MutableArrayRef<BlockArgument> reductionArgs =
cast<omp::BlockArgOpenMPOpInterface>(opInst).getReductionBlockArgs();
@@ -5208,6 +5283,20 @@ convertOmpSimd(Operation &opInst, llvm::IRBuilderBase &builder,
if (handleError(afterAllocas, opInst).failed())
return failure();
+ if (failed(allocReductionVars(simdOp, reductionArgs, builder,
+ moduleTranslation, allocaIP, reductionDecls,
+ privateReductionVariables, reductionVariableMap,
+ deferredStores, isByRef)))
+ return failure();
+
+ if (handleError(initPrivateVars(builder, moduleTranslation, privateVarsInfo),
+ opInst)
+ .failed())
+ return failure();
+
+ // No call to copyFirstPrivateVars because FIRSTPRIVATE is not allowed for
+ // SIMD.
+
// Initialize linear variables and linear step
LinearClauseProcessor linearClauseProcessor;
if (linearClauseProcessor.initLinearIV(simdOp).failed())
@@ -5218,42 +5307,16 @@ convertOmpSimd(Operation &opInst, llvm::IRBuilderBase &builder,
for (mlir::Attribute linearVarType : linearVarTypes)
linearClauseProcessor.registerType(moduleTranslation, linearVarType);
for (auto [idx, linearVar] : llvm::enumerate(simdOp.getLinearVars())) {
- bool isImplicit = false;
- for (auto [mlirPrivVar, llvmPrivateVar] : llvm::zip_equal(
- privateVarsInfo.mlirVars, privateVarsInfo.llvmVars)) {
- // If the linear variable is implicit, reuse the already
- // existing llvm::Value
- if (linearVar == mlirPrivVar) {
- isImplicit = true;
- linearClauseProcessor.createLinearVar(builder, moduleTranslation,
- llvmPrivateVar, idx);
- break;
- }
- }
-
- if (!isImplicit)
- linearClauseProcessor.createLinearVar(
- builder, moduleTranslation,
- moduleTranslation.lookupValue(linearVar), idx);
+ size_t privIdx = privateVarsInfo.numPrivateVars + idx;
+ linearClauseProcessor.createLinearVar(
+ builder, allocaIP, privateVarsInfo.privatizers[privIdx],
+ moduleTranslation.lookupValue(linearVar),
+ privateVarsInfo.llvmVars[privIdx], idx);
}
for (mlir::Value linearStep : simdOp.getLinearStepVars())
linearClauseProcessor.initLinearStep(moduleTranslation, linearStep);
}
- if (failed(allocReductionVars(simdOp, reductionArgs, builder,
- moduleTranslation, allocaIP, reductionDecls,
- privateReductionVariables, reductionVariableMap,
- deferredStores, isByRef)))
- return failure();
-
- if (handleError(initPrivateVars(builder, moduleTranslation, privateVarsInfo),
- opInst)
- .failed())
- return failure();
-
- // No call to copyFirstPrivateVars because FIRSTPRIVATE is not allowed for
- // SIMD.
-
assert(afterAllocas.get()->getSinglePredecessor());
if (failed(initReductionVars(simdOp, reductionArgs, builder,
moduleTranslation,
@@ -5307,26 +5370,34 @@ convertOmpSimd(Operation &opInst, llvm::IRBuilderBase &builder,
llvm::CanonicalLoopInfo *loopInfo = findCurrentLoopInfo(moduleTranslation);
// Emit Initialization for linear variables
if (simdOp.getLinearVars().size()) {
- linearClauseProcessor.initLinearVar(builder, moduleTranslation,
- loopInfo->getPreheader());
+ if (failed(handleError(
+ linearClauseProcessor.initLinearVar(builder, moduleTranslation,
+ loopInfo->getPreheader()),
+ opInst)))
+ return failure();
- linearClauseProcessor.updateLinearVar(builder, loopInfo->getBody(),
- loopInfo->getIndVar());
+ if (failed(handleError(linearClauseProcessor.updateLinearVar(
+ builder, moduleTranslation, loopInfo->getBody(),
+ loopInfo->getIndVar()),
+ opInst)))
+ return failure();
}
builder.SetInsertPoint((*regionBlock)->begin());
- for (size_t index = 0; index < simdOp.getLinearVars().size(); index++)
- linearClauseProcessor.rewriteInPlace(builder, loopInfo->getBody(),
- loopInfo->getLatch(), index);
-
ompBuilder->applySimd(loopInfo, alignedVars,
simdOp.getIfExpr()
? moduleTranslation.lookupValue(simdOp.getIfExpr())
: nullptr,
order, simdlen, safelen);
- linearClauseProcessor.updateLinearIV(builder, moduleTranslation);
- linearClauseProcessor.emitStoresForLinearVar(builder);
+ if (failed(handleError(
+ linearClauseProcessor.updateLinearIV(builder, moduleTranslation),
+ opInst)))
+ return failure();
+ if (failed(handleError(linearClauseProcessor.emitStoresForLinearVar(
+ builder, moduleTranslation),
+ opInst)))
+ return failure();
// We now need to reduce the per-simd-lane reduction variable into the
// original variable. This works a bit differently to other reductions (e.g.
diff --git a/mlir/test/Dialect/OpenMP/invalid.mlir b/mlir/test/Dialect/OpenMP/invalid.mlir
index c3fe4f60e4166..b8c34ee93676a 100644
--- a/mlir/test/Dialect/OpenMP/invalid.mlir
+++ b/mlir/test/Dialect/OpenMP/invalid.mlir
@@ -3756,6 +3756,91 @@ omp.private {type = private} @x.privatizer : f32 copy {
// -----
+// expected-error @below {{`linear` clauses do not require a `copy` region.}}
+omp.private {type = linear} @x.privatizer : f32 copy {
+^bb0(%arg0: f32, %arg1 : f32):
+ omp.yield(%arg0 : f32)
+}
+
+// -----
+
+// expected-error @below {{only `linear` clauses can have a `data_address` region.}}
+omp.private {type = private} @x.privatizer : !llvm.ptr data_address {
+^bb0(%arg0: !llvm.ptr):
+ omp.yield(%arg0 : !llvm.ptr)
+}
+
+// -----
+
+// expected-error @below {{`data_address`: expected 1 region arguments, got: 2}}
+omp.private {type = linear} @x.privatizer : !llvm.ptr data_address {
+^bb0(%arg0: !llvm.ptr, %arg1: !llvm.ptr):
+ omp.yield(%arg0 : !llvm.ptr)
+}
+
+// -----
+
+omp.private {type = linear} @x.privatizer : !llvm.ptr data_address {
+^bb0(%arg0: !llvm.ptr):
+ // expected-error @below {{Invalid yielded value. Expected a single value, got: None}}
+ omp.yield
+}
+
+// -----
+
+omp.private {type = linear} @x.privatizer : !llvm.ptr
+
+func.func @linear_privatizer_in_private(%x : !llvm.ptr, %lb : i32) {
+ // expected-error @below {{`linear` privatizer '@x.privatizer' can only be used in a `linear` clause}}
+ omp.simd private(@x.privatizer %x -> %arg0 : !llvm.ptr) {
+ omp.loop_nest (%iv) : i32 = (%lb) to (%lb) step (%lb) {
+ omp.yield
+ }
+ }
+ return
+}
+
+// -----
+
+omp.private {type = private} @x.privatizer : i32
+
+func.func @linear_with_private_privatizer(%x : !llvm.ptr, %lb : i32) {
+ // expected-error @below {{privatizer '@x.privatizer' of a linear variable must be `linear`}}
+ omp.simd linear(@x.privatizer %x -> %arg0 : !llvm.ptr = %lb : i32) linear_var_types([i32]) {
+ omp.loop_nest (%iv) : i32 = (%lb) to (%lb) step (%lb) {
+ omp.yield
+ }
+ }
+ return
+}
+
+// -----
+
+func.func @linear_without_privatizer(%x : !llvm.ptr, %lb : i32) {
+ // expected-error @below {{expected as many linear privatizers as linear variables}}
+ "omp.wsloop"(%x, %lb) ({
+ ^bb0(%arg0: !llvm.ptr):
+ omp.loop_nest (%iv) : i32 = (%lb) to (%lb) step (%lb) {
+ omp.yield
+ }
+ }) {linear_var_types = [i32],
+ operandSegmentSizes = array<i32: 0, 0, 1, 1, 0, 0, 0>} : (!llvm.ptr, i32) -> ()
+ return
+}
+
+// -----
+
+omp.private {type = linear} @x.privatizer : i32
+
+func.func @declare_simd_linear_privatizer(%x : i32, %step : i32) {
+ // expected-error @below {{'omp.declare_simd' op linear privatizers are not allowed}}
+ "omp.declare_simd"(%x, %step) <{linear_syms = [@x.privatizer],
+ operandSegmentSizes = array<i32: 0, 1, 1, 0>}> : (i32, i32) -> ()
+ return
+}
+
+// -----
+
// expected-error @below {{`firstprivate` clauses require at least a `copy` region.}}
omp.private {type = firstprivate} @x.privatizer : f32 init {
^bb0(%arg0: f32, %arg1: f32):
@@ -4801,10 +4886,12 @@ func.func @omp_declare_simd_branch() -> () {
// -----
+omp.private {type = linear} @lin.i32 : i32
+
func.func @omp_wsloop_linear_ref(%lb : index, %ub : index, %step : index,
%data_var : memref<i32>, %linear_var : i32) {
// expected-error @+1 {{'omp.wsloop' op linear modifier 'ref' may only be specified on a declare simd directive}}
- omp.wsloop linear(ref(%data_var : memref<i32> = %linear_var : i32)) linear_var_types([i32]) {
+ omp.wsloop linear(ref(@lin.i32 %data_var -> %l0 : memref<i32> = %linear_var : i32)) linear_var_types([i32]) {
omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
omp.yield
}
@@ -4814,10 +4901,12 @@ func.func @omp_wsloop_linear_ref(%lb : index, %ub : index, %step : index,
// -----
+omp.private {type = linear} @lin.i32 : i32
+
func.func @omp_wsloop_linear_uval(%lb : index, %ub : index, %step : index,
%data_var : memref<i32>, %linear_var : i32) {
// expected-error @+1 {{'omp.wsloop' op linear modifier 'uval' may only be specified on a declare simd directive}}
- omp.wsloop linear(uval(%data_var : memref<i32> = %linear_var : i32)) linear_var_types([i32]) {
+ omp.wsloop linear(uval(@lin.i32 %data_var -> %l0 : memref<i32> = %linear_var : i32)) linear_var_types([i32]) {
omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
omp.yield
}
@@ -4827,10 +4916,12 @@ func.func @omp_wsloop_linear_uval(%lb : index, %ub : index, %step : index,
// -----
+omp.private {type = linear} @lin.i32 : i32
+
func.func @omp_simd_linear_ref(%lb : index, %ub : index, %step : index,
%data_var : memref<i32>, %linear_var : i32) {
// expected-error @+1 {{'omp.simd' op linear modifier 'ref' may only be specified on a declare simd directive}}
- omp.simd linear(ref(%data_var : memref<i32> = %linear_var : i32)) linear_var_types([i32]) {
+ omp.simd linear(ref(@lin.i32 %data_var -> %l0 : memref<i32> = %linear_var : i32)) linear_var_types([i32]) {
omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
omp.yield
}
@@ -4844,6 +4935,7 @@ func.func @omp_wsloop_linear_modifiers_mismatch(%lb : index, %ub : index, %step
%data_var : memref<i32>, %linear_var : i32) {
// expected-error @below {{'omp.wsloop' op expected as many linear modifiers as linear variables}}
"omp.wsloop"(%data_var, %linear_var) ({
+ ^bb0(%l0: memref<i32>):
omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
omp.yield
}
@@ -4858,6 +4950,7 @@ func.func @omp_simd_linear_modifiers_mismatch(%lb : index, %ub : index, %step :
%data_var : memref<i32>, %linear_var : i32) {
// expected-error @below {{'omp.simd' op expected as many linear modifiers as linear variables}}
"omp.simd"(%data_var, %linear_var) ({
+ ^bb0(%l0: memref<i32>):
omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
omp.yield
}
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index 9371d3599b72a..0276244bdc7b0 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -503,6 +503,8 @@ func.func @omp_loop_nest_pretty_multiple_collapse_tiles(%lb1 : i32, %ub1 : i32,
return
}
+omp.private {type = linear} @lin.i32 : i32
+
// CHECK-LABEL: omp_wsloop
func.func @omp_wsloop(%lb : index, %ub : index, %step : index, %data_var : memref<i32>, %linear_var : i32, %chunk_var : i32) -> () {
@@ -515,33 +517,36 @@ func.func @omp_wsloop(%lb : index, %ub : index, %step : index, %data_var : memre
}) {operandSegmentSizes = array<i32: 0,0,0,0,0,0,0>, ordered = 1} :
() -> ()
- // CHECK: omp.wsloop linear(%{{.*}} : memref<i32> = %{{.*}} : i32) linear_var_types([i32]) schedule(static) {
+ // CHECK: omp.wsloop schedule(static) linear(@lin.i32 %{{.*}} -> %{{.*}} : memref<i32> = %{{.*}} : i32) linear_var_types([i32]) {
// CHECK-NEXT: omp.loop_nest
"omp.wsloop" (%data_var, %linear_var) ({
+ ^bb0(%l0: memref<i32>):
omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
omp.yield
}
}) {operandSegmentSizes = array<i32: 0,0,1,1,0,0,0>, schedule_kind = #omp.schedulekind<static>,
- linear_var_types = [i32]} : (memref<i32>, i32) -> ()
+ linear_var_types = [i32], linear_syms = [@lin.i32]} : (memref<i32>, i32) -> ()
- // CHECK: omp.wsloop linear(%{{.*}} : memref<i32> = %{{.*}} : i32, %{{.*}} : memref<i32> = %{{.*}} : i32) linear_var_types([i32, i32]) schedule(static) {
+ // CHECK: omp.wsloop schedule(static) linear(@lin.i32 %{{.*}} -> %{{.*}} : memref<i32> = %{{.*}} : i32, @lin.i32 %{{.*}} -> %{{.*}} : memref<i32> = %{{.*}} : i32) linear_var_types([i32, i32]) {
// CHECK-NEXT: omp.loop_nest
"omp.wsloop" (%data_var, %data_var, %linear_var, %linear_var) ({
+ ^bb0(%l0: memref<i32>, %l1: memref<i32>):
omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
omp.yield
}
}) {operandSegmentSizes = array<i32: 0,0,2,2,0,0,0>, schedule_kind = #omp.schedulekind<static>,
- linear_var_types = [i32,i32]} :
+ linear_var_types = [i32,i32], linear_syms = [@lin.i32, @lin.i32]} :
(memref<i32>, memref<i32>, i32, i32) -> ()
- // CHECK: omp.wsloop linear(%{{.*}} : memref<i32> = %{{.*}} : i32) linear_var_types([i32]) ordered(2) schedule(dynamic = %{{.*}}) {
+ // CHECK: omp.wsloop ordered(2) schedule(dynamic = %{{.*}}) linear(@lin.i32 %{{.*}} -> %{{.*}} : memref<i32> = %{{.*}} : i32) linear_var_types([i32]) {
// CHECK-NEXT: omp.loop_nest
"omp.wsloop" (%data_var, %linear_var, %chunk_var) ({
+ ^bb0(%l0: memref<i32>):
omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
omp.yield
}
}) {operandSegmentSizes = array<i32: 0,0,1,1,0,0,1>, schedule_kind = #omp.schedulekind<dynamic>, ordered = 2,
- linear_var_types = [i32]} : (memref<i32>, i32, i32) -> ()
+ linear_var_types = [i32], linear_syms = [@lin.i32]} : (memref<i32>, i32, i32) -> ()
// CHECK: omp.wsloop nowait schedule(auto) {
// CHECK-NEXT: omp.loop_nest
@@ -577,42 +582,42 @@ func.func @omp_wsloop_pretty(%lb : index, %ub : index, %step : index, %data_var
}
}
- // CHECK: omp.wsloop linear(%{{.*}} : memref<i32> = %{{.*}} : i32) linear_var_types([i32]) schedule(static) {
+ // CHECK: omp.wsloop schedule(static) linear(@lin.i32 %{{.*}} -> %{{.*}} : memref<i32> = %{{.*}} : i32) linear_var_types([i32]) {
// CHECK-NEXT: omp.loop_nest
- omp.wsloop schedule(static) linear(%data_var : memref<i32> = %linear_var : i32) linear_var_types([i32]) {
+ omp.wsloop schedule(static) linear(@lin.i32 %data_var -> %l0 : memref<i32> = %linear_var : i32) linear_var_types([i32]) {
omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
omp.yield
}
}
- // CHECK: omp.wsloop linear(val(%{{.*}} : memref<i32> = %{{.*}} : i32), val(%{{.*}} : memref<i32> = %{{.*}} : i32)) linear_var_types([i32, i32]) schedule(static) {
+ // CHECK: omp.wsloop schedule(static) linear(val(@lin.i32 %{{.*}} -> %{{.*}} : memref<i32> = %{{.*}} : i32), val(@lin.i32 %{{.*}} -> %{{.*}} : memref<i32> = %{{.*}} : i32)) linear_var_types([i32, i32]) {
// CHECK-NEXT: omp.loop_nest
- omp.wsloop schedule(static) linear(val(%data_var : memref<i32> = %linear_var : i32),
- val(%data_var : memref<i32> = %linear_var : i32)) linear_var_types([i32, i32]) {
+ omp.wsloop schedule(static) linear(val(@lin.i32 %data_var -> %l0 : memref<i32> = %linear_var : i32),
+ val(@lin.i32 %data_var -> %l1 : memref<i32> = %linear_var : i32)) linear_var_types([i32, i32]) {
omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
omp.yield
}
}
- // CHECK: omp.wsloop linear(%{{.*}} : memref<i32> = %{{.*}} : i32) linear_var_types([i32]) ordered(2) schedule(static = %{{.*}} : i32) {
+ // CHECK: omp.wsloop ordered(2) schedule(static = %{{.*}} : i32) linear(@lin.i32 %{{.*}} -> %{{.*}} : memref<i32> = %{{.*}} : i32) linear_var_types([i32]) {
// CHECK-NEXT: omp.loop_nest
- omp.wsloop ordered(2) linear(%data_var : memref<i32> = %linear_var : i32) linear_var_types([i32]) schedule(static = %chunk_var : i32) {
+ omp.wsloop ordered(2) schedule(static = %chunk_var : i32) linear(@lin.i32 %data_var -> %l0 : memref<i32> = %linear_var : i32) linear_var_types([i32]) {
omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
omp.yield
}
}
- // CHECK: omp.wsloop linear(%{{.*}} : memref<i32> = %{{.*}} : i32) linear_var_types([i32]) ordered(2) schedule(dynamic = %{{.*}} : i32, nonmonotonic) {
+ // CHECK: omp.wsloop ordered(2) schedule(dynamic = %{{.*}} : i32, nonmonotonic) linear(@lin.i32 %{{.*}} -> %{{.*}} : memref<i32> = %{{.*}} : i32) linear_var_types([i32]) {
// CHECK-NEXT: omp.loop_nest
- omp.wsloop ordered(2) linear(%data_var : memref<i32> = %linear_var : i32) linear_var_types([i32]) schedule(dynamic = %chunk_var : i32, nonmonotonic) {
+ omp.wsloop ordered(2) schedule(dynamic = %chunk_var : i32, nonmonotonic) linear(@lin.i32 %data_var -> %l0 : memref<i32> = %linear_var : i32) linear_var_types([i32]) {
omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
omp.yield
}
}
- // CHECK: omp.wsloop linear(%{{.*}} : memref<i32> = %{{.*}} : i32) linear_var_types([i32]) ordered(2) schedule(dynamic = %{{.*}} : i16, monotonic) {
+ // CHECK: omp.wsloop ordered(2) schedule(dynamic = %{{.*}} : i16, monotonic) linear(@lin.i32 %{{.*}} -> %{{.*}} : memref<i32> = %{{.*}} : i32) linear_var_types([i32]) {
// CHECK-NEXT: omp.loop_nest
- omp.wsloop ordered(2) linear(%data_var : memref<i32> = %linear_var : i32) linear_var_types([i32]) schedule(dynamic = %chunk_var2 : i16, monotonic) {
+ omp.wsloop ordered(2) schedule(dynamic = %chunk_var2 : i16, monotonic) linear(@lin.i32 %data_var -> %l0 : memref<i32> = %linear_var : i32) linear_var_types([i32]) {
omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
omp.yield
}
@@ -817,8 +822,8 @@ func.func @omp_simd_pretty_order(%lb : index, %ub : index, %step : index) -> ()
// CHECK-LABEL: omp_simd_linear_val
func.func @omp_simd_linear_val(%lb : index, %ub : index, %step : index,
%data_var : memref<i32>, %linear_var : i32) -> () {
- // CHECK: omp.simd linear(val(%{{.*}} : memref<i32> = %{{.*}} : i32)) linear_var_types([i32])
- omp.simd linear(val(%data_var : memref<i32> = %linear_var : i32)) linear_var_types([i32]) {
+ // CHECK: omp.simd linear(val(@lin.i32 %{{.*}} -> %{{.*}} : memref<i32> = %{{.*}} : i32)) linear_var_types([i32])
+ omp.simd linear(val(@lin.i32 %data_var -> %l0 : memref<i32> = %linear_var : i32)) linear_var_types([i32]) {
omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
omp.yield
}
@@ -830,9 +835,9 @@ func.func @omp_simd_linear_val(%lb : index, %ub : index, %step : index,
func.func @omp_simd_linear_val_multiple(%lb : index, %ub : index, %step : index,
%d1 : memref<i32>, %d2 : memref<i32>,
%s1 : i32, %s2 : i32) -> () {
- // CHECK: omp.simd linear(val(%{{.*}} : memref<i32> = %{{.*}} : i32), val(%{{.*}} : memref<i32> = %{{.*}} : i32)) linear_var_types([i32, i32])
- omp.simd linear(val(%d1 : memref<i32> = %s1 : i32),
- val(%d2 : memref<i32> = %s2 : i32)) linear_var_types([i32, i32]) {
+ // CHECK: omp.simd linear(val(@lin.i32 %{{.*}} -> %{{.*}} : memref<i32> = %{{.*}} : i32), val(@lin.i32 %{{.*}} -> %{{.*}} : memref<i32> = %{{.*}} : i32)) linear_var_types([i32, i32])
+ omp.simd linear(val(@lin.i32 %d1 -> %l0 : memref<i32> = %s1 : i32),
+ val(@lin.i32 %d2 -> %l1 : memref<i32> = %s2 : i32)) linear_var_types([i32, i32]) {
omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
omp.yield
}
@@ -4670,3 +4675,27 @@ func.func @omp_dispatch_nocontext_novariants(%cond : i1, %x : memref<i32>) -> ()
// CHECK-LABEL: func.func private @variant(memref<i32>)
func.func private @variant(memref<i32>) -> ()
+
+// CHECK-LABEL: omp.private {type = linear} @linear.privatizer : !llvm.ptr init {
+omp.private {type = linear} @linear.privatizer : !llvm.ptr init {
+^bb0(%arg0: !llvm.ptr, %arg1: !llvm.ptr):
+ omp.yield(%arg1 : !llvm.ptr)
+// CHECK: } data_address {
+} data_address {
+// CHECK: ^bb0(%{{.*}}: !llvm.ptr):
+^bb0(%arg0: !llvm.ptr):
+ %0 = llvm.load %arg0 : !llvm.ptr -> !llvm.ptr
+ // CHECK: omp.yield(%{{.*}} : !llvm.ptr)
+ omp.yield(%0 : !llvm.ptr)
+}
+
+// CHECK-LABEL: func @simd_linear_privatizer
+func.func @simd_linear_privatizer(%x : !llvm.ptr, %y : !llvm.ptr, %lb : i32, %ub : i32, %step : i32) {
+ // CHECK: omp.simd linear(@linear.privatizer %{{.*}} -> %{{.*}} : !llvm.ptr = %{{.*}} : i32) linear_var_types([i32]) private(@x.privatizer %{{.*}} -> %{{.*}} : !llvm.ptr)
+ omp.simd linear(@linear.privatizer %x -> %arg0 : !llvm.ptr = %step : i32) linear_var_types([i32]) private(@x.privatizer %y -> %arg1 : !llvm.ptr) {
+ omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
+ omp.yield
+ }
+ }
+ return
+}
diff --git a/mlir/test/Target/LLVMIR/openmp-linear-privatizer.mlir b/mlir/test/Target/LLVMIR/openmp-linear-privatizer.mlir
new file mode 100644
index 0000000000000..3f3b78f9c4e57
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/openmp-linear-privatizer.mlir
@@ -0,0 +1,148 @@
+// Linear vars whose `linear` privatizer has a `data_address` region.
+// RUN: mlir-translate -mlir-to-llvmir -split-input-file %s | FileCheck %s
+
+llvm.func @malloc(i64) -> !llvm.ptr
+llvm.func @free(!llvm.ptr)
+
+// A "box" holding a pointer to an i32.
+omp.private {type = linear} @box.linear : !llvm.ptr init {
+^bb0(%mold: !llvm.ptr, %priv: !llvm.ptr):
+ %c4 = llvm.mlir.constant(4 : i64) : i64
+ %mem = llvm.call @malloc(%c4) : (i64) -> !llvm.ptr
+ llvm.store %mem, %priv : !llvm.ptr, !llvm.ptr
+ omp.yield(%priv : !llvm.ptr)
+} data_address {
+^bb0(%box: !llvm.ptr):
+ %data = llvm.load %box : !llvm.ptr -> !llvm.ptr
+ omp.yield(%data : !llvm.ptr)
+} dealloc {
+^bb0(%box: !llvm.ptr):
+ %data = llvm.load %box : !llvm.ptr -> !llvm.ptr
+ llvm.call @free(%data) : (!llvm.ptr) -> ()
+ omp.yield
+}
+
+// CHECK-LABEL: define void @simd_linear_privatizer(
+// CHECK-SAME: ptr %[[BOX:.*]], ptr %{{.*}})
+// CHECK: %[[PRIV:.*]] = alloca ptr
+// CHECK: %[[START:.linear_var]] = alloca i32
+// CHECK: omp_loop.preheader:
+// CHECK: %[[ORIG_DATA:.*]] = load ptr, ptr %[[BOX]]
+// CHECK: %[[V:.*]] = load i32, ptr %[[ORIG_DATA]]
+// CHECK: store i32 %[[V]], ptr %[[START]]
+// CHECK: omp_loop.body:
+// CHECK: %[[PRIV_DATA:.*]] = load ptr, ptr %[[PRIV]]
+// CHECK: %[[S:.*]] = load i32, ptr %[[START]]
+// CHECK: %[[MUL:.*]] = mul i32 %{{.*}}, 1
+// CHECK: %[[ADD:.*]] = add i32 %[[S]], %[[MUL]]
+// CHECK: store i32 %[[ADD]], ptr %[[PRIV_DATA]]
+// CHECK: omp.region.cont:
+// CHECK: %[[PRIV_DATA2:.*]] = load ptr, ptr %[[PRIV]]
+// CHECK: %[[LAST:.*]] = load i32, ptr %[[PRIV_DATA2]]
+// CHECK: %[[ORIG_DATA2:.*]] = load ptr, ptr %[[BOX]]
+// CHECK: store i32 %[[LAST]], ptr %[[ORIG_DATA2]]
+// CHECK: call void @free(
+llvm.func @simd_linear_privatizer(%box : !llvm.ptr, %out : !llvm.ptr) {
+ %c1 = llvm.mlir.constant(1 : i32) : i32
+ %c10 = llvm.mlir.constant(10 : i32) : i32
+ omp.simd linear(@box.linear %box -> %arg0 : !llvm.ptr = %c1 : i32) linear_var_types([i32]) {
+ omp.loop_nest (%iv) : i32 = (%c1) to (%c10) inclusive step (%c1) {
+ %data = llvm.load %arg0 : !llvm.ptr -> !llvm.ptr
+ %v = llvm.load %data : !llvm.ptr -> i32
+ llvm.store %v, %out : i32, !llvm.ptr
+ omp.yield
+ }
+ }
+ llvm.return
+}
+
+// CHECK-LABEL: define void @wsloop_linear_privatizer(
+// CHECK-SAME: ptr %[[BOX:.*]], ptr %{{.*}})
+// CHECK: %[[PRIV:.*]] = alloca ptr
+// CHECK: %[[START:.linear_var]] = alloca i32
+// CHECK: omp_loop.preheader:
+// CHECK: %[[ORIG_DATA:.*]] = load ptr, ptr %[[BOX]]
+// CHECK: %[[V:.*]] = load i32, ptr %[[ORIG_DATA]]
+// CHECK: store i32 %[[V]], ptr %[[START]]
+// CHECK: omp_loop.body:
+// CHECK: %[[PRIV_DATA:.*]] = load ptr, ptr %[[PRIV]]
+// CHECK: %[[S:.*]] = load i32, ptr %[[START]]
+// CHECK: %[[MUL:.*]] = mul i32 %{{.*}}, 2
+// CHECK: %[[ADD:.*]] = add i32 %[[S]], %[[MUL]]
+// CHECK: store i32 %[[ADD]], ptr %[[PRIV_DATA]]
+// CHECK: omp_loop.linear_lastiter_exit:
+// CHECK: %[[PRIV_DATA2:.*]] = load ptr, ptr %[[PRIV]]
+// CHECK: %[[LAST:.*]] = load i32, ptr %[[PRIV_DATA2]]
+// CHECK: %[[ORIG_DATA2:.*]] = load ptr, ptr %[[BOX]]
+// CHECK: store i32 %[[LAST]], ptr %[[ORIG_DATA2]]
+// CHECK: omp.region.cont:
+// CHECK: call void @free(
+llvm.func @wsloop_linear_privatizer(%box : !llvm.ptr, %out : !llvm.ptr) {
+ %c1 = llvm.mlir.constant(1 : i32) : i32
+ %c2 = llvm.mlir.constant(2 : i32) : i32
+ %c10 = llvm.mlir.constant(10 : i32) : i32
+ omp.wsloop linear(@box.linear %box -> %arg0 : !llvm.ptr = %c2 : i32) linear_var_types([i32]) {
+ omp.loop_nest (%iv) : i32 = (%c1) to (%c10) inclusive step (%c1) {
+ %data = llvm.load %arg0 : !llvm.ptr -> !llvm.ptr
+ %v = llvm.load %data : !llvm.ptr -> i32
+ llvm.store %v, %out : i32, !llvm.ptr
+ omp.yield
+ }
+ }
+ llvm.return
+}
+
+// The loop IV is stored through the data address of the private copy; it gets
+// one more step after the loop.
+// CHECK-LABEL: define void @simd_linear_privatizer_iv(
+// CHECK-SAME: ptr %[[BOX:.*]])
+// CHECK: %[[PRIV:.*]] = alloca ptr
+// CHECK: omp.region.cont:
+// CHECK: %[[DATA:.*]] = load ptr, ptr %[[PRIV]]
+// CHECK: %[[LAST:.*]] = load i32, ptr %[[DATA]]
+// CHECK: %[[NEXT:.*]] = add i32 %[[LAST]], 2
+// CHECK: store i32 %[[NEXT]], ptr %[[DATA]]
+// CHECK: %[[DATA2:.*]] = load ptr, ptr %[[PRIV]]
+// CHECK: %[[FINAL:.*]] = load i32, ptr %[[DATA2]]
+// CHECK: %[[ORIG:.*]] = load ptr, ptr %[[BOX]]
+// CHECK: store i32 %[[FINAL]], ptr %[[ORIG]]
+// CHECK: call void @free(
+llvm.func @simd_linear_privatizer_iv(%box : !llvm.ptr) {
+ %c1 = llvm.mlir.constant(1 : i32) : i32
+ %c2 = llvm.mlir.constant(2 : i32) : i32
+ %c5 = llvm.mlir.constant(5 : i32) : i32
+ omp.simd linear(@box.linear %box -> %arg0 : !llvm.ptr = %c2 : i32) linear_var_types([i32]) {
+ omp.loop_nest (%iv) : i32 = (%c1) to (%c5) inclusive step (%c2) {
+ %data = llvm.load %arg0 : !llvm.ptr -> !llvm.ptr
+ llvm.store %iv, %data : i32, !llvm.ptr
+ omp.yield
+ }
+ }
+ llvm.return
+}
+
+// Same, with the private copy copied before its data address is loaded.
+// CHECK-LABEL: define void @simd_linear_privatizer_iv_copy(
+// CHECK: %[[PRIV:omp.private.alloc]] = alloca ptr
+// CHECK: omp.region.cont:
+// CHECK: %[[DATA:.*]] = load ptr, ptr %[[PRIV]]
+// CHECK: %[[LAST:.*]] = load i32, ptr %[[DATA]]
+// CHECK: %[[NEXT:.*]] = add i32 %[[LAST]], 2
+// CHECK: store i32 %[[NEXT]], ptr %[[DATA]]
+llvm.func @simd_linear_privatizer_iv_copy(%box : !llvm.ptr) {
+ %c1 = llvm.mlir.constant(1 : i32) : i32
+ %c2 = llvm.mlir.constant(2 : i32) : i32
+ %c5 = llvm.mlir.constant(5 : i32) : i32
+ %c8 = llvm.mlir.constant(8 : i32) : i32
+ %one = llvm.mlir.constant(1 : i64) : i64
+ %tmp = llvm.alloca %one x !llvm.ptr : (i64) -> !llvm.ptr
+ omp.simd linear(@box.linear %box -> %arg0 : !llvm.ptr = %c2 : i32) linear_var_types([i32]) {
+ omp.loop_nest (%iv) : i32 = (%c1) to (%c5) inclusive step (%c2) {
+ "llvm.intr.memcpy"(%tmp, %arg0, %c8) <{isVolatile = false}> : (!llvm.ptr, !llvm.ptr, i32) -> ()
+ %data = llvm.load %tmp : !llvm.ptr -> !llvm.ptr
+ llvm.store %iv, %data : i32, !llvm.ptr
+ omp.yield
+ }
+ }
+ llvm.return
+}
diff --git a/mlir/test/Target/LLVMIR/openmp-llvm-invalid.mlir b/mlir/test/Target/LLVMIR/openmp-llvm-invalid.mlir
index f2a45c7ed7e32..89b0a3cc069e8 100644
--- a/mlir/test/Target/LLVMIR/openmp-llvm-invalid.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-llvm-invalid.mlir
@@ -92,9 +92,11 @@ llvm.func @omp_threadprivate() {
// -----
+omp.private {type = linear} @lin.i32 : i32
+
llvm.func @wsloop_linear(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
// expected-error @below {{Ill-formed type attributes for linear variables}}
- omp.wsloop linear(%x : !llvm.ptr = %step : i32) linear_var_types([]) {
+ omp.wsloop linear(@lin.i32 %x -> %lin0 : !llvm.ptr = %step : i32) linear_var_types([]) {
omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
omp.yield
}
@@ -104,9 +106,11 @@ llvm.func @wsloop_linear(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
// -----
+omp.private {type = linear} @lin.i32 : i32
+
llvm.func @simd_linear(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
// expected-error @below {{Ill-formed type attributes for linear variables}}
- omp.simd linear(%x : !llvm.ptr = %step : i32) linear_var_types([]) {
+ omp.simd linear(@lin.i32 %x -> %lin0 : !llvm.ptr = %step : i32) linear_var_types([]) {
omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
omp.yield
}
@@ -117,9 +121,11 @@ llvm.func @simd_linear(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
// -----
omp.private {type = private} @i_private_i32 : i32
+omp.private {type = linear} @lin.i32 : i32
+
llvm.func @simd_linear_private(%lb : i32, %ub : i32, %step : i32, %i : !llvm.ptr) {
// expected-error @below {{linear variables cannot appear in other data-sharing clauses}}
- omp.simd linear(%i : !llvm.ptr = %step : i32) linear_var_types([i32])
+ omp.simd linear(@lin.i32 %i -> %lin0 : !llvm.ptr = %step : i32) linear_var_types([i32])
private(@i_private_i32 %i -> %priv_i : !llvm.ptr) {
omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
omp.yield
@@ -130,13 +136,15 @@ llvm.func @simd_linear_private(%lb : i32, %ub : i32, %step : i32, %i : !llvm.ptr
// -----
+omp.private {type = linear} @lin.i32 : i32
+
llvm.func @simd_linear_ambiguous_iv(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr, %y : !llvm.ptr) {
// expected-error @below {{Could not determine the linear variable associated with the loop nest induction variable}}
// expected-error @below {{LLVM Translation failed for operation: omp.simd}}
- omp.simd linear(%x : !llvm.ptr = %step : i32, %y : !llvm.ptr = %step : i32) linear_var_types([i32, i32]) {
+ omp.simd linear(@lin.i32 %x -> %lin0 : !llvm.ptr = %step : i32, @lin.i32 %y -> %lin1 : !llvm.ptr = %step : i32) linear_var_types([i32, i32]) {
omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
- llvm.store %iv, %x : i32, !llvm.ptr
- llvm.store %iv, %y : i32, !llvm.ptr
+ llvm.store %iv, %lin0 : i32, !llvm.ptr
+ llvm.store %iv, %lin1 : i32, !llvm.ptr
omp.yield
}
}
diff --git a/mlir/test/Target/LLVMIR/openmp-llvm.mlir b/mlir/test/Target/LLVMIR/openmp-llvm.mlir
index 96b60d586baed..ce6d3858ea4dd 100644
--- a/mlir/test/Target/LLVMIR/openmp-llvm.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-llvm.mlir
@@ -351,15 +351,17 @@ llvm.func @test_omp_masked(%arg0: i32)-> () {
// -----
+omp.private {type = linear} @lin.i32 : i32
+
llvm.func @wsloop_linear(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
// CHECK-LABEL: @wsloop_linear
+// CHECK: %[[LINEAR_RESULT:omp.private.alloc]] = alloca i32, align 4
+// CHECK: %[[LINEAR_VAR:.linear_var]] = alloca i32, align 4
// CHECK: %p.lastiter = alloca i32, align 4
// CHECK: %p.lowerbound = alloca i32, align 4
// CHECK: %p.upperbound = alloca i32, align 4
// CHECK: %p.stride = alloca i32, align 4
-// CHECK: %[[LINEAR_VAR:.*]] = alloca i32, align 4
-// CHECK: %[[LINEAR_RESULT:.*]] = alloca i32, align 4
// CHECK: omp_loop.preheader:
// CHECK: %[[LOAD:.*]] = load i32, ptr %{{.*}}, align 4
@@ -387,7 +389,7 @@ llvm.func @wsloop_linear(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
// CHECK: call void @__kmpc_barrier(ptr {{.*}}, i32 %[[THREAD_ID]])
// CHECK: br label %omp_loop.after
- omp.wsloop linear(%x : !llvm.ptr = %step : i32) linear_var_types([i32]) {
+ omp.wsloop linear(@lin.i32 %x -> %lin0 : !llvm.ptr = %step : i32) linear_var_types([i32]) {
omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
omp.yield
}
@@ -764,13 +766,15 @@ llvm.func @simd_simple(%lb : i64, %ub : i64, %step : i64, %arg0: !llvm.ptr) {
// -----
+omp.private {type = linear} @lin.i32 : i32
+
llvm.func @simd_linear(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
// CHECK-LABEL: @simd_linear
// CHECK-SAME: (i32 %{{.*}}, i32 %{{.*}}, i32 %[[STEP:.*]], ptr %[[X:.*]])
-// CHECK: %[[LINEAR_VAR:.*]] = alloca i32, align 4
-// CHECK: %[[LINEAR_RESULT:.*]] = alloca i32, align 4
+// CHECK: %[[LINEAR_RESULT:omp.private.alloc]] = alloca i32, align 4
+// CHECK: %[[LINEAR_VAR:.linear_var]] = alloca i32, align 4
// CHECK: omp_loop.preheader:
// CHECK: %[[LOAD:.*]] = load i32, ptr {{.*}}, align 4
@@ -790,10 +794,10 @@ llvm.func @simd_linear(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
// CHECK-NEXT: store i32 %[[ADD]], ptr %[[LINEAR_RESULT]]
// CHECK-NEXT: %[[LOAD:.*]] = load i32, ptr %[[LINEAR_RESULT]]
// CHECK-NEXT: store i32 %[[LOAD]], ptr %[[X]], align 4
- omp.simd linear(%x : !llvm.ptr = %step : i32) linear_var_types([i32]) {
+ omp.simd linear(@lin.i32 %x -> %lin0 : !llvm.ptr = %step : i32) linear_var_types([i32]) {
omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
- llvm.store %iv, %x : i32, !llvm.ptr
- llvm.store %iv, %x : i32, !llvm.ptr
+ llvm.store %iv, %lin0 : i32, !llvm.ptr
+ llvm.store %iv, %lin0 : i32, !llvm.ptr
omp.yield
}
}
@@ -802,6 +806,8 @@ llvm.func @simd_linear(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
// -----
+omp.private {type = linear} @lin.i64 : i64
+
// Test linear clause with mismatched types (i32 step, i64 variable)
// This is a regression test for issue #173332
llvm.func @simd_linear_i64_var_i32_step(%lb : i32, %ub : i32, %x : !llvm.ptr) {
@@ -809,8 +815,8 @@ llvm.func @simd_linear_i64_var_i32_step(%lb : i32, %ub : i32, %x : !llvm.ptr) {
// CHECK-LABEL: @simd_linear_i64_var_i32_step
-// CHECK: %[[LINEAR_VAR:.*]] = alloca i64
-// CHECK: %[[LINEAR_RESULT:.*]] = alloca i64
+// CHECK: %[[LINEAR_RESULT:omp.private.alloc]] = alloca i64
+// CHECK: %[[LINEAR_VAR:.linear_var]] = alloca i64
// CHECK: omp_loop.preheader:
// CHECK: %[[LOAD:.*]] = load i64, ptr {{.*}}
@@ -824,7 +830,7 @@ llvm.func @simd_linear_i64_var_i32_step(%lb : i32, %ub : i32, %x : !llvm.ptr) {
// CHECK: %[[MUL:.*]] = mul i64 %[[IV_I64]], {{.*}}
// CHECK: %[[ADD:.*]] = add i64 %[[LOAD]], %[[MUL]]
// CHECK: store i64 %[[ADD]], ptr %[[LINEAR_RESULT]], {{.*}}!llvm.access.group
- omp.simd linear(%x : !llvm.ptr = %step : i32) linear_var_types([i64]) {
+ omp.simd linear(@lin.i64 %x -> %lin0 : !llvm.ptr = %step : i32) linear_var_types([i64]) {
omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
omp.yield
}
@@ -834,6 +840,8 @@ llvm.func @simd_linear_i64_var_i32_step(%lb : i32, %ub : i32, %x : !llvm.ptr) {
// -----
+omp.private {type = linear} @lin.f64 : f64
+
// Test linear clause with floating-point variable and i32 step
// This tests the floating-point path in updateLinearVar
llvm.func @simd_linear_f64_var_i32_step(%lb : i32, %ub : i32, %x : !llvm.ptr) {
@@ -841,8 +849,8 @@ llvm.func @simd_linear_f64_var_i32_step(%lb : i32, %ub : i32, %x : !llvm.ptr) {
// CHECK-LABEL: @simd_linear_f64_var_i32_step
-// CHECK: %[[LINEAR_VAR:.*]] = alloca double
-// CHECK: %[[LINEAR_RESULT:.*]] = alloca double
+// CHECK: %[[LINEAR_RESULT:omp.private.alloc]] = alloca double
+// CHECK: %[[LINEAR_VAR:.linear_var]] = alloca double
// CHECK: omp_loop.preheader:
// CHECK: %[[LOAD:.*]] = load double, ptr {{.*}}
@@ -856,7 +864,7 @@ llvm.func @simd_linear_f64_var_i32_step(%lb : i32, %ub : i32, %x : !llvm.ptr) {
// CHECK-NEXT: %[[MUL_FP:.*]] = sitofp i32 %[[MUL_INT]] to double
// CHECK-NEXT: %[[ADD:.*]] = fadd double %[[LOAD]], %[[MUL_FP]]
// CHECK-NEXT: store double %[[ADD]], ptr %[[LINEAR_RESULT]], {{.*}}!llvm.access.group
- omp.simd linear(%x : !llvm.ptr = %step : i32) linear_var_types([f64]) {
+ omp.simd linear(@lin.f64 %x -> %lin0 : !llvm.ptr = %step : i32) linear_var_types([f64]) {
omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
omp.yield
}
@@ -866,15 +874,16 @@ llvm.func @simd_linear_f64_var_i32_step(%lb : i32, %ub : i32, %x : !llvm.ptr) {
// -----
+omp.private {type = linear} @lin.i32 : i32
+
// Test the update of omp.simd linear iteration variables, when nested inside
// omp.wsloop.
llvm.func @wsloop_simd_linear(%x : !llvm.ptr) {
// CHECK-LABEL: @wsloop_simd_linear
-// CHECK: omp.wsloop.region:
-// CHECK: %[[LINEAR_VAR:.*]] = alloca i32
-// CHECK: %[[LINEAR_RESULT:.*]] = alloca i32
+// CHECK: %[[LINEAR_RESULT:omp.private.alloc]] = alloca i32
+// CHECK: %[[LINEAR_VAR:.linear_var]] = alloca i32
// CHECK: omp.region.cont2:
// CHECK: %[[VAL:.*]] = load i32, ptr %[[LINEAR_RESULT]]
@@ -884,9 +893,9 @@ llvm.func @wsloop_simd_linear(%x : !llvm.ptr) {
%ub = llvm.mlir.constant(100 : i32) : i32
%step = llvm.mlir.constant(25 : i32) : i32
omp.wsloop {
- omp.simd linear(%x : !llvm.ptr = %step : i32) linear_var_types([i32]) {
+ omp.simd linear(@lin.i32 %x -> %lin0 : !llvm.ptr = %step : i32) linear_var_types([i32]) {
omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
- llvm.store %iv, %x : i32, !llvm.ptr
+ llvm.store %iv, %lin0 : i32, !llvm.ptr
omp.yield
}
} {omp.composite}
diff --git a/mlir/test/Target/LLVMIR/openmp-simd-guided.mlir b/mlir/test/Target/LLVMIR/openmp-simd-guided.mlir
index e274918828af1..7bbb4a3b8f8f5 100644
--- a/mlir/test/Target/LLVMIR/openmp-simd-guided.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-simd-guided.mlir
@@ -1,6 +1,8 @@
// Ensure that schedule can be used with the guided kind-type and simd construct
// RUN: mlir-translate -mlir-to-llvmir -split-input-file %s | FileCheck %s
+omp.private {type = linear} @lin.i32 : i32
+
llvm.func @test_simd_guided() {
%0 = llvm.mlir.constant (1 : i64) : i64
%c1_i32 = llvm.mlir.constant (1 : i32) : i32
@@ -9,7 +11,7 @@ llvm.func @test_simd_guided() {
%c4_i32 = llvm.mlir.constant (4 : i32) : i32
%1 = llvm.alloca %0 x i32 {bindc_name = "i"} : (i64) -> !llvm.ptr
omp.wsloop schedule(guided = %c4_i32 : i32, simd) {
- omp.simd linear(%1 : !llvm.ptr = %c1_i32 : i32) linear_var_types([i32]) {
+ omp.simd linear(@lin.i32 %1 -> %lin0 : !llvm.ptr = %c1_i32 : i32) linear_var_types([i32]) {
omp.loop_nest (%arg1) : i32 = (%c0_i32) to (%c64_i32) inclusive step (%c1_i32) {
omp.yield
}
diff --git a/mlir/test/Target/LLVMIR/openmp-simd-linear.mlir b/mlir/test/Target/LLVMIR/openmp-simd-linear.mlir
index af31abb1de90f..ce52dd288810e 100644
--- a/mlir/test/Target/LLVMIR/openmp-simd-linear.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-simd-linear.mlir
@@ -4,11 +4,19 @@
// -----
-// CHECK-LABEL: void @test_simd_linear()
-// CHECK-NOT: %.linear_result
-// CHECK: %.linear_result = alloca i32
+// CHECK-LABEL: void @test_simd_linear..omp_par
+// CHECK: omp.par.entry:
+// CHECK: %[[PRIV:omp.private.alloc[0-9]+]] = alloca i32
+// CHECK: %[[START:.linear_var]] = alloca i32
+// CHECK: omp_loop.preheader{{[0-9]+}}:
+// CHECK: store i32 %{{.*}}, ptr %[[START]]
+// CHECK: omp_loop.body{{[0-9]+}}:
+// CHECK: load i32, ptr %[[START]]
+// CHECK: store i32 %{{.*}}, ptr %[[PRIV]]
omp.private {type = private} @test_simd_linear_private_i32 : i32
+omp.private {type = linear} @lin.i32 : i32
+
llvm.func @test_simd_linear() {
%0 = llvm.mlir.constant(1 : i64) : i64
%1 = llvm.mlir.constant(1 : i32) : i32
@@ -21,9 +29,9 @@ llvm.func @test_simd_linear() {
omp.loop_nest (%arg1) : i32 = (%1) to (%3) inclusive step (%1) {
llvm.store %arg1, %arg0 : i32, !llvm.ptr
llvm.store %2, %4 : i32, !llvm.ptr
- omp.simd linear(%4 : !llvm.ptr = %1 : i32) linear_var_types([i32]) {
+ omp.simd linear(@lin.i32 %4 -> %lin0 : !llvm.ptr = %1 : i32) linear_var_types([i32]) {
omp.loop_nest (%arg2) : i32 = (%1) to (%3) inclusive step (%1) {
- llvm.store %arg2, %4 : i32, !llvm.ptr
+ llvm.store %arg2, %lin0 : i32, !llvm.ptr
omp.yield
}
}
@@ -39,10 +47,19 @@ llvm.func @test_simd_linear() {
// -----
// CHECK-LABEL: @test_simd_linear2({{.*}})
+// CHECK: %[[I:.*]] = alloca i32, i64 0
+// CHECK: %[[PRIV:omp.private.alloc]] = alloca i32
// CHECK: omp.loop_nest.region:
-// CHECK: store i32 %{{.*}}, ptr %.linear_result
-// CHECK: %{{.*}} = load i32, ptr %.linear_result
-// CHECK: omp.region.cont1:
+// CHECK: store i32 %{{.*}}, ptr %[[PRIV]]
+// CHECK: %{{.*}} = load i32, ptr %[[PRIV]]
+// CHECK: omp.region.cont:
+// CHECK: %[[LAST:.*]] = load i32, ptr %[[PRIV]]
+// CHECK: %[[NEXT:.*]] = add i32 %[[LAST]], 1
+// CHECK: store i32 %[[NEXT]], ptr %[[PRIV]]
+// CHECK: %[[FINAL:.*]] = load i32, ptr %[[PRIV]]
+// CHECK: store i32 %[[FINAL]], ptr %[[I]]
+
+omp.private {type = linear} @lin.i32 : i32
llvm.func @test_simd_linear2(%a : !llvm.ptr) {
%c0_i64 = llvm.mlir.constant(0 : i64) : i64
@@ -64,10 +81,10 @@ llvm.func @test_simd_linear2(%a : !llvm.ptr) {
llvm.br ^bb1(%i_next, %iv_next : i32, i64)
^bb3: // pred: ^bb1
llvm.store %i, %i_ptr : i32, !llvm.ptr
- omp.simd linear(%i_ptr : !llvm.ptr = %c1_i32 : i32) linear_var_types([i32]) {
+ omp.simd linear(@lin.i32 %i_ptr -> %lin0 : !llvm.ptr = %c1_i32 : i32) linear_var_types([i32]) {
omp.loop_nest (%arg0) : i32 = (%c1_i32) to (%c100_i32) inclusive step (%c1_i32) {
- llvm.store %arg0, %i_ptr : i32, !llvm.ptr
- %i2 = llvm.load %i_ptr : !llvm.ptr -> i32
+ llvm.store %arg0, %lin0 : i32, !llvm.ptr
+ %i2 = llvm.load %lin0 : !llvm.ptr -> i32
%3 = llvm.sext %i2 : i32 to i64
%4 = llvm.sub %3, %c1_i64 overflow<nsw, nuw> : i64
%5 = llvm.mul %4, %c1_i64 overflow<nsw, nuw> : i64
diff --git a/mlir/test/Target/LLVMIR/openmp-simd-ordered.mlir b/mlir/test/Target/LLVMIR/openmp-simd-ordered.mlir
index e7cd456c77119..fcf562a936ecc 100644
--- a/mlir/test/Target/LLVMIR/openmp-simd-ordered.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-simd-ordered.mlir
@@ -1,7 +1,7 @@
// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
// Test that linear variables in SIMD loops with ordered regions
-// are correctly rewritten to use .linear_result in:
+// use the private copy of the linear variable in:
// 1. The ordered region (omp.ordered.region)
// 2. Code after the ordered region (omp_region.finalize)
//
@@ -17,6 +17,8 @@
// !$omp end simd
module {
+ omp.private {type = linear} @lin.i32 : i32
+
// CHECK-LABEL: define void @simd_ordered_linear
llvm.func @simd_ordered_linear() {
%c0_i64 = llvm.mlir.constant(0 : i64) : i64
@@ -33,17 +35,17 @@ module {
%c = llvm.alloca %c100_i64 x i32 : (i64) -> !llvm.ptr
%i = llvm.alloca %c1_i64 x i32 {bindc_name = "i"} : (i64) -> !llvm.ptr
+ // CHECK: %[[PRIV:omp.private.alloc]] = alloca i32
// CHECK: %.linear_var = alloca i32
- // CHECK: %.linear_result = alloca i32
- omp.simd linear(%i : !llvm.ptr = %c1_i32 : i32) linear_var_types([i32]) {
+ omp.simd linear(@lin.i32 %i -> %lin0 : !llvm.ptr = %c1_i32 : i32) linear_var_types([i32]) {
omp.loop_nest (%iv) : i32 = (%c1_i32) to (%c10_i32) inclusive step (%c1_i32) {
// CHECK: omp.loop_nest.region:
- // CHECK: load i32, ptr %.linear_result
- llvm.store %iv, %i : i32, !llvm.ptr
+ // CHECK: load i32, ptr %[[PRIV]]
+ llvm.store %iv, %lin0 : i32, !llvm.ptr
// Compute a[i] = b[i] * 10
- %i_val = llvm.load %i : !llvm.ptr -> i32
+ %i_val = llvm.load %lin0 : !llvm.ptr -> i32
%i_idx = llvm.sext %i_val : i32 to i64
%i_off = llvm.sub %i_idx, %c1_i64 : i64
%b_ptr = llvm.getelementptr %b[%i_off] : (!llvm.ptr, i64) -> !llvm.ptr, i32
@@ -55,8 +57,8 @@ module {
// Ordered region
omp.ordered.region par_level_simd {
// CHECK: omp.ordered.region:
- // CHECK: load i32, ptr %.linear_result
- %i_ord = llvm.load %i : !llvm.ptr -> i32
+ // CHECK: load i32, ptr %[[PRIV]]
+ %i_ord = llvm.load %lin0 : !llvm.ptr -> i32
%i_ord_idx = llvm.sext %i_ord : i32 to i64
%i_ord_off = llvm.sub %i_ord_idx, %c1_i64 : i64
%a_ord_ptr = llvm.getelementptr %a[%i_ord_off] : (!llvm.ptr, i64) -> !llvm.ptr, i32
@@ -66,8 +68,8 @@ module {
// Compute c[i] = a[i] * 2 (code after ordered region)
// CHECK: omp_region.finalize:
- // CHECK: load i32, ptr %.linear_result
- %i_post = llvm.load %i : !llvm.ptr -> i32
+ // CHECK: load i32, ptr %[[PRIV]]
+ %i_post = llvm.load %lin0 : !llvm.ptr -> i32
%i_post_idx = llvm.sext %i_post : i32 to i64
%i_post_off = llvm.sub %i_post_idx, %c1_i64 : i64
%a_post_ptr = llvm.getelementptr %a[%i_post_off] : (!llvm.ptr, i64) -> !llvm.ptr, i32
diff --git a/mlir/test/Target/LLVMIR/openmp-wsloop-linear.mlir b/mlir/test/Target/LLVMIR/openmp-wsloop-linear.mlir
index 503800c9d1a8f..bc1b1f5d628f9 100644
--- a/mlir/test/Target/LLVMIR/openmp-wsloop-linear.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-wsloop-linear.mlir
@@ -3,25 +3,29 @@
// -----
-// %.linear_result must appear only in the loop body and in the
+// The private copy must appear only in the loop body and in the
// linear_lastiter_exit block, where it's used to update the original
// variable.
// CHECK-LABEL: void @wsloop_linear_post_use({{.*}})
-// CHECK: %.linear_result = alloca i32
-// CHECK-NOT: %.linear_result
+// CHECK: %[[PRIV:omp.private.alloc]] = alloca i32
+// CHECK-NOT: %[[PRIV]]
// CHECK: omp_loop.body:
-// CHECK: %.linear_result
+// CHECK: store i32 %{{.*}}, ptr %[[PRIV]]
+// CHECK: omp.loop_nest.region:
+// CHECK: load i32, ptr %[[PRIV]]
// CHECK: omp_loop.inc:
-// CHECK-NOT: %.linear_result
+// CHECK-NOT: %[[PRIV]]
// CHECK: omp_loop.linear_lastiter_exit:
-// CHECK: load i32, ptr %.linear_result
-// CHECK-NOT: %.linear_result
+// CHECK: load i32, ptr %[[PRIV]]
+// CHECK-NOT: %[[PRIV]]
+
+omp.private {type = linear} @lin.i32 : i32
llvm.func @wsloop_linear_post_use(%lb : i32, %ub : i32, %step : i32,
%x : !llvm.ptr, %out : !llvm.ptr) {
- omp.wsloop linear(%x : !llvm.ptr = %step : i32) linear_var_types([i32]) {
+ omp.wsloop linear(@lin.i32 %x -> %lin0 : !llvm.ptr = %step : i32) linear_var_types([i32]) {
omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
- %cur = llvm.load %x : !llvm.ptr -> i32
+ %cur = llvm.load %lin0 : !llvm.ptr -> i32
llvm.store %cur, %out : i32, !llvm.ptr
omp.yield
}
diff --git a/mlir/test/Target/LLVMIR/openmp-wsloop-simd-ordered.mlir b/mlir/test/Target/LLVMIR/openmp-wsloop-simd-ordered.mlir
index a31b05840a7df..ca2ee3ee917bf 100644
--- a/mlir/test/Target/LLVMIR/openmp-wsloop-simd-ordered.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-wsloop-simd-ordered.mlir
@@ -1,7 +1,7 @@
// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
// Test that linear variables in worksharing+SIMD loops with ordered regions
-// are correctly rewritten to use .linear_result in:
+// use the private copy of the linear variable in:
// 1. The ordered region (omp.ordered.region)
// 2. Code after the ordered region (omp_region.finalize)
//
@@ -17,6 +17,8 @@
// !$omp end do simd
module {
+ omp.private {type = linear} @lin.i32 : i32
+
// CHECK-LABEL: define void @wsloop_simd_ordered_linear
llvm.func @wsloop_simd_ordered_linear() {
%c0_i64 = llvm.mlir.constant(0 : i64) : i64
@@ -33,18 +35,18 @@ module {
%c = llvm.alloca %c100_i64 x i32 : (i64) -> !llvm.ptr
%i = llvm.alloca %c1_i64 x i32 {bindc_name = "i"} : (i64) -> !llvm.ptr
+ // CHECK: %[[PRIV:omp.private.alloc]] = alloca i32
// CHECK: %.linear_var = alloca i32
- // CHECK: %.linear_result = alloca i32
omp.wsloop ordered(0) {
- omp.simd linear(%i : !llvm.ptr = %c1_i32 : i32) linear_var_types([i32]) {
+ omp.simd linear(@lin.i32 %i -> %lin0 : !llvm.ptr = %c1_i32 : i32) linear_var_types([i32]) {
omp.loop_nest (%iv) : i32 = (%c1_i32) to (%c100_i32) inclusive step (%c1_i32) {
// CHECK: omp.loop_nest.region:
- // CHECK: load i32, ptr %.linear_result
- llvm.store %iv, %i : i32, !llvm.ptr
+ // CHECK: load i32, ptr %[[PRIV]]
+ llvm.store %iv, %lin0 : i32, !llvm.ptr
// Compute a[i] = b[i] * 10
- %i_val = llvm.load %i : !llvm.ptr -> i32
+ %i_val = llvm.load %lin0 : !llvm.ptr -> i32
%i_idx = llvm.sext %i_val : i32 to i64
%i_off = llvm.sub %i_idx, %c1_i64 : i64
%b_ptr = llvm.getelementptr %b[%i_off] : (!llvm.ptr, i64) -> !llvm.ptr, i32
@@ -56,8 +58,8 @@ module {
// Ordered region
omp.ordered.region par_level_simd {
// CHECK: omp.ordered.region:
- // CHECK: load i32, ptr %.linear_result
- %i_ord = llvm.load %i : !llvm.ptr -> i32
+ // CHECK: load i32, ptr %[[PRIV]]
+ %i_ord = llvm.load %lin0 : !llvm.ptr -> i32
%i_ord_idx = llvm.sext %i_ord : i32 to i64
%i_ord_off = llvm.sub %i_ord_idx, %c1_i64 : i64
%a_ord_ptr = llvm.getelementptr %a[%i_ord_off] : (!llvm.ptr, i64) -> !llvm.ptr, i32
@@ -68,8 +70,8 @@ module {
// Compute c[i] = a[i] * 2 (code after ordered region)
// CHECK: omp_region.finalize:
- // CHECK: load i32, ptr %.linear_result
- %i_post = llvm.load %i : !llvm.ptr -> i32
+ // CHECK: load i32, ptr %[[PRIV]]
+ %i_post = llvm.load %lin0 : !llvm.ptr -> i32
%i_post_idx = llvm.sext %i_post : i32 to i64
%i_post_off = llvm.sub %i_post_idx, %c1_i64 : i64
%a_post_ptr = llvm.getelementptr %a[%i_post_off] : (!llvm.ptr, i64) -> !llvm.ptr, i32
>From 8e814d50e309408c6797c583d7951c99c345cc73 Mon Sep 17 00:00:00 2001
From: Aditya Trivedi <adit4443ya at gmail.com>
Date: Mon, 19 Oct 2026 00:06:07 +0530
Subject: [PATCH 2/2] [flang][OpenMP] Support ALLOCATABLE list items in LINEAR
Create a linear privatizer for every LINEAR item and bind the symbol to
the linear block argument. For an allocatable, the privatizer's
data_address region returns the data address of the descriptor.
Fixes #182401
---
flang/include/flang/Lower/Support/Utils.h | 3 +-
flang/include/flang/Support/OpenMP-utils.h | 5 +-
flang/lib/Lower/OpenMP/ClauseProcessor.cpp | 7 +-
flang/lib/Lower/OpenMP/ClauseProcessor.h | 6 +-
.../lib/Lower/OpenMP/DataSharingProcessor.cpp | 31 +----
flang/lib/Lower/OpenMP/DataSharingProcessor.h | 5 +-
flang/lib/Lower/OpenMP/OpenMP.cpp | 112 +++++++++++++----
flang/lib/Lower/Support/Utils.cpp | 26 +++-
flang/lib/Support/OpenMP-utils.cpp | 8 +-
.../Todo/linear-no-delayed-privatization.f90 | 10 ++
.../Lower/OpenMP/composite_simd_linear.f90 | 36 +++---
.../OpenMP/compound-version-override.f90 | 7 +-
.../OpenMP/distribute-parallel-do-simd.f90 | 8 +-
flang/test/Lower/OpenMP/distribute-simd.f90 | 4 +-
.../indirect-reference-privatization.f90 | 4 +-
.../test/Lower/OpenMP/linear-allocatable.f90 | 113 ++++++++++++++++++
flang/test/Lower/OpenMP/linear_modifier.f90 | 16 +--
.../test/Lower/OpenMP/metadirective-loop.f90 | 41 ++++---
flang/test/Lower/OpenMP/simd-linear.f90 | 16 +--
flang/test/Lower/OpenMP/wsloop-linear.f90 | 18 +--
flang/test/Lower/OpenMP/wsloop-simd.f90 | 6 +-
21 files changed, 345 insertions(+), 137 deletions(-)
create mode 100644 flang/test/Lower/OpenMP/Todo/linear-no-delayed-privatization.f90
create mode 100644 flang/test/Lower/OpenMP/linear-allocatable.f90
diff --git a/flang/include/flang/Lower/Support/Utils.h b/flang/include/flang/Lower/Support/Utils.h
index 15d30905aadcd..afd4f220eaf86 100644
--- a/flang/include/flang/Lower/Support/Utils.h
+++ b/flang/include/flang/Lower/Support/Utils.h
@@ -100,7 +100,8 @@ void privatizeSymbol(
llvm::SmallPtrSet<const semantics::Symbol *, 16> &mightHaveReadHostSym,
const semantics::Symbol *symToPrivatize, OperandsStructType *clauseOps,
std::optional<llvm::omp::Directive> dir = std::nullopt,
- bool forceHeapAllocationForPrivateDynamicArrays = false);
+ bool forceHeapAllocationForPrivateDynamicArrays = false,
+ bool isLinear = false);
} // end namespace Fortran::lower
diff --git a/flang/include/flang/Support/OpenMP-utils.h b/flang/include/flang/Support/OpenMP-utils.h
index 0a648e9612ec4..3512a72ab01e8 100644
--- a/flang/include/flang/Support/OpenMP-utils.h
+++ b/flang/include/flang/Support/OpenMP-utils.h
@@ -22,6 +22,7 @@ struct EntryBlockArgs {
llvm::ArrayRef<mlir::Value> hasDeviceAddrVars;
llvm::ArrayRef<mlir::Value> hostEvalVars;
llvm::ArrayRef<mlir::Value> inReductionVars;
+ llvm::ArrayRef<mlir::Value> linearVars;
llvm::ArrayRef<mlir::Value> mapVars;
llvm::ArrayRef<mlir::Value> privVars;
llvm::ArrayRef<mlir::Value> reductionVars;
@@ -31,8 +32,8 @@ struct EntryBlockArgs {
auto getVars() const {
return llvm::concat<const mlir::Value>(hasDeviceAddrVars, hostEvalVars,
- inReductionVars, mapVars, privVars, reductionVars, taskReductionVars,
- useDeviceAddrVars, useDevicePtrVars);
+ inReductionVars, linearVars, mapVars, privVars, reductionVars,
+ taskReductionVars, useDeviceAddrVars, useDevicePtrVars);
}
};
diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
index e41223b5b16c0..5bbe77f36ec95 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
@@ -1775,8 +1775,9 @@ bool ClauseProcessor::processIsDevicePtr(
return clauseFound;
}
-bool ClauseProcessor::processLinear(mlir::omp::LinearClauseOps &result,
- bool isDeclareSimd) const {
+bool ClauseProcessor::processLinear(
+ mlir::omp::LinearClauseOps &result, bool isDeclareSimd,
+ llvm::SmallVectorImpl<const semantics::Symbol *> *linearSyms) const {
lower::StatementContext stmtCtx;
std::vector<mlir::Attribute> typeAttrs;
std::vector<mlir::Attribute> linearModAttrs;
@@ -1806,6 +1807,8 @@ bool ClauseProcessor::processLinear(mlir::omp::LinearClauseOps &result,
semantics::Symbol *sym = object.sym();
const mlir::Value variable = converter.getSymbolAddress(*sym);
result.linearVars.push_back(variable);
+ if (linearSyms)
+ linearSyms->push_back(sym);
mlir::Type ty = converter.genType(*sym);
typeAttrs.push_back(mlir::TypeAttr::get(ty));
diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.h b/flang/lib/Lower/OpenMP/ClauseProcessor.h
index 6b0e0cf606f48..33f44754bb84b 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.h
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.h
@@ -154,8 +154,10 @@ class ClauseProcessor {
bool processIsDevicePtr(lower::StatementContext &stmtCtx,
mlir::omp::IsDevicePtrClauseOps &result,
llvm::SmallVectorImpl<Object> &isDeviceObjects) const;
- bool processLinear(mlir::omp::LinearClauseOps &result,
- bool isDeclareSimd = false) const;
+ bool
+ processLinear(mlir::omp::LinearClauseOps &result, bool isDeclareSimd = false,
+ llvm::SmallVectorImpl<const semantics::Symbol *> *linearSyms =
+ nullptr) const;
bool
processLink(llvm::SmallVectorImpl<DeclareTargetCaptureInfo> &result) const;
diff --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
index 8d0d191058cfb..243d07b440f8e 100644
--- a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
@@ -284,27 +284,8 @@ void DataSharingProcessor::collectSymbolsForPrivatization() {
// Such cases are suggested to be clearly documented and explained
// instead of being silently skipped
auto isException = [&](const Fortran::semantics::Symbol *sym) -> bool {
- // The handling of linear clause is deferred to the OpenMP
- // IRBuilder which is responsible for all its aspects,
- // including privatization. Privatizing linear variables at this point would
- // cause the following structure:
- //
- // omp.op linear(%linear = %step : !fir.ref<type>) {
- // Use %linear in this BB
- // }
- //
- // to be changed to the following:
- //
- // omp. op linear(%linear = %step : !fir.ref<type>)
- // private(%linear -> %arg0 : !fir.ref<i32>) {
- // Declare and use %arg0 in this BB
- // }
- //
- // The OpenMP IRBuilder needs to map the linear MLIR value
- // (i.e. %linear) to its `uses` in the BB to correctly
- // implement the functionalities of linear clause. However,
- // privatizing here disallows the IRBuilder to
- // draw a relation between %linear and %arg0. Hence skip.
+ // Linear variables are privatized by the `linear` privatizers of the
+ // linear clause, which genLinearPrivatizers creates.
if (sym->test(Fortran::semantics::Symbol::Flag::OmpLinear))
return true;
return false;
@@ -572,8 +553,8 @@ void DataSharingProcessor::collectPrivatizedSymbols(
}
auto shouldCollectSymbol = [&](const semantics::Symbol *sym) {
- // Linear symbols are privatized by OpenMP IRBuilder, except when they are
- // enclosed within TARGET.
+ // Linear symbols are privatized by the linear clause, except when they are
+ // enclosed within TARGET, where implicit linearization is skipped.
mlir::Operation *currentOp =
firOpBuilder.getInsertionBlock()->getParentOp();
bool inTarget =
@@ -801,7 +782,7 @@ void DataSharingProcessor::copyLastPrivatize(mlir::Operation *op) {
void DataSharingProcessor::privatizeSymbol(
const semantics::Symbol *symToPrivatize,
mlir::omp::PrivateClauseOps *clauseOps,
- std::optional<llvm::omp::Directive> dir) {
+ std::optional<llvm::omp::Directive> dir, bool isLinear) {
if (!useDelayedPrivatization) {
cloneSymbol(symToPrivatize);
copyFirstPrivateSymbol(symToPrivatize);
@@ -812,7 +793,7 @@ void DataSharingProcessor::privatizeSymbol(
mlir::omp::PrivateClauseOps>(
converter, firOpBuilder, symTable, allPrivatizedSymbols,
mightHaveReadHostSym, symToPrivatize, clauseOps, dir,
- forceHeapAllocationForPrivateDynamicArrays);
+ forceHeapAllocationForPrivateDynamicArrays, isLinear);
}
} // namespace omp
} // namespace lower
diff --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.h b/flang/lib/Lower/OpenMP/DataSharingProcessor.h
index 25d848c8baffc..7854f84723876 100644
--- a/flang/lib/Lower/OpenMP/DataSharingProcessor.h
+++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.h
@@ -219,7 +219,10 @@ class DataSharingProcessor {
void privatizeSymbol(const semantics::Symbol *symToPrivatize,
mlir::omp::PrivateClauseOps *clauseOps,
- std::optional<llvm::omp::Directive> dir = std::nullopt);
+ std::optional<llvm::omp::Directive> dir = std::nullopt,
+ bool isLinear = false);
+
+ bool usesDelayedPrivatization() const { return useDelayedPrivatization; }
const llvm::SetVector<const semantics::Symbol *> &
getConditionalLastprivateSymbols() const {
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 0b20eb1c16dc1..aa299296340c6 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -207,6 +207,7 @@ struct ObjectEntryBlockArgs {
ObjectEntryBlockArgsEntry hasDeviceAddr;
llvm::ArrayRef<mlir::Value> hostEvalVars;
ObjectEntryBlockArgsEntry inReduction;
+ ObjectEntryBlockArgsEntry linear;
ObjectEntryBlockArgsEntry map;
ObjectEntryBlockArgsEntry priv;
ObjectEntryBlockArgsEntry reduction;
@@ -216,8 +217,9 @@ struct ObjectEntryBlockArgs {
std::size_t sourceUseDeviceAddrCount{0};
bool isValid() const {
- return hasDeviceAddr.isValid() && inReduction.isValid() && map.isValid() &&
- priv.isValid() && reduction.isValid() && taskReduction.isValid() &&
+ return hasDeviceAddr.isValid() && inReduction.isValid() &&
+ linear.isValid() && map.isValid() && priv.isValid() &&
+ reduction.isValid() && taskReduction.isValid() &&
useDeviceAddr.isValid() && useDevicePtr.isValid() &&
sourceUseDeviceAddrCount <= useDeviceAddr.objects.size();
}
@@ -231,6 +233,7 @@ struct ObjectEntryBlockArgs {
};
appendSyms(hasDeviceAddr);
appendSyms(inReduction);
+ appendSyms(linear);
appendSyms(map);
appendSyms(priv);
appendSyms(reduction);
@@ -242,9 +245,9 @@ struct ObjectEntryBlockArgs {
auto getVars() const {
return llvm::concat<const mlir::Value>(
- hasDeviceAddr.vars, hostEvalVars, inReduction.vars, map.vars, priv.vars,
- reduction.vars, taskReduction.vars, useDeviceAddr.vars,
- useDevicePtr.vars);
+ hasDeviceAddr.vars, hostEvalVars, inReduction.vars, linear.vars,
+ map.vars, priv.vars, reduction.vars, taskReduction.vars,
+ useDeviceAddr.vars, useDevicePtr.vars);
}
Fortran::common::openmp::EntryBlockArgs asEntryBlockArgs() const {
@@ -252,6 +255,7 @@ struct ObjectEntryBlockArgs {
args.hasDeviceAddrVars = hasDeviceAddr.vars;
args.hostEvalVars = hostEvalVars;
args.inReductionVars = inReduction.vars;
+ args.linearVars = linear.vars;
args.mapVars = map.vars;
args.privVars = priv.vars;
args.reductionVars = reduction.vars;
@@ -965,6 +969,8 @@ static void bindEntryBlockArgs(lower::AbstractConverter &converter,
bindMapLike(args.hasDeviceAddr.objects, op.getHasDeviceAddrBlockArgs());
bindPrivateLike(args.inReduction.objects, args.inReduction.vars,
op.getInReductionBlockArgs());
+ bindPrivateLike(args.linear.objects, args.linear.vars,
+ op.getLinearBlockArgs());
bindMapLike(args.map.objects, op.getMapBlockArgs());
bindPrivateLike(args.priv.objects, args.priv.vars, op.getPrivateBlockArgs());
bindPrivateLike(args.reduction.objects, args.reduction.vars,
@@ -2402,6 +2408,7 @@ static void genSimdClauses(
const List<Clause> &clauses, mlir::Location loc,
mlir::omp::SimdOperands &clauseOps,
llvm::SmallVectorImpl<Object> &reductionObjects,
+ llvm::SmallVectorImpl<const semantics::Symbol *> &linearSyms,
llvm::DenseMap<const semantics::Symbol *, mlir::Value> *reductionVarCache =
nullptr) {
ClauseProcessor cp(converter, semaCtx, clauses);
@@ -2412,17 +2419,17 @@ static void genSimdClauses(
cp.processReduction(loc, clauseOps, reductionObjects, reductionVarCache);
cp.processSafelen(clauseOps);
cp.processSimdlen(clauseOps);
- cp.processLinear(clauseOps);
+ cp.processLinear(clauseOps, /*isDeclareSimd=*/false, &linearSyms);
}
// SIMD construct may have implicit
// linear semantics on IV. Process the same here.
-static void
-genSimdImplicitLinear(lower::AbstractConverter &converter,
- semantics::SemanticsContext &semaCtx,
- mlir::omp::SimdOperands &clauseOps,
- mlir::omp::LoopNestOperands loopNestClauseOps,
- llvm::SmallVector<const semantics::Symbol *> iv) {
+static void genSimdImplicitLinear(
+ lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
+ mlir::omp::SimdOperands &clauseOps,
+ mlir::omp::LoopNestOperands loopNestClauseOps,
+ llvm::SmallVector<const semantics::Symbol *> iv,
+ llvm::SmallVectorImpl<const semantics::Symbol *> &linearSyms) {
// If the (standalone/composite) SIMD is enclosed within TARGET,
// implicit linearization will cause invalid FIR due to
@@ -2475,6 +2482,7 @@ genSimdImplicitLinear(lower::AbstractConverter &converter,
mlir::UnitAttr::get(&converter.getMLIRContext()));
clauseOps.linearVars.push_back(variable);
clauseOps.linearStepVars.push_back(loopStep);
+ linearSyms.push_back(loopVar);
}
}
if (!typeAttrs.empty()) {
@@ -2485,6 +2493,29 @@ genSimdImplicitLinear(lower::AbstractConverter &converter,
}
}
+/// Create the `linear` privatizers of the LINEAR items `linearSyms`. Their
+/// private copies are entry block arguments of the op with the linear clause.
+static void
+genLinearPrivatizers(DataSharingProcessor &dsp, mlir::Location loc,
+ llvm::ArrayRef<const semantics::Symbol *> linearSyms,
+ mlir::omp::LinearClauseOps &clauseOps,
+ ObjectEntryBlockArgsEntry &linearArgs) {
+ if (linearSyms.empty())
+ return;
+ if (!dsp.usesDelayedPrivatization())
+ TODO(loc, "LINEAR clause without delayed privatization");
+
+ for (auto [sym, var] : llvm::zip_equal(linearSyms, clauseOps.linearVars)) {
+ mlir::omp::PrivateClauseOps privateOps;
+ dsp.privatizeSymbol(sym, &privateOps, /*dir=*/std::nullopt,
+ /*isLinear=*/true);
+ var = privateOps.privateVars.front();
+ clauseOps.linearSyms.push_back(privateOps.privateSyms.front());
+ }
+ linearArgs.objects = makeObjects(linearSyms);
+ linearArgs.vars = clauseOps.linearVars;
+}
+
static void genScopeClauses(lower::AbstractConverter &converter,
semantics::SemanticsContext &semaCtx,
const List<Clause> &clauses, mlir::Location loc,
@@ -2717,6 +2748,7 @@ static void genWsloopClauses(
lower::StatementContext &stmtCtx, const List<Clause> &clauses,
mlir::Location loc, mlir::omp::WsloopOperands &clauseOps,
llvm::SmallVectorImpl<Object> &reductionObjects,
+ llvm::SmallVectorImpl<const semantics::Symbol *> &linearSyms,
llvm::DenseMap<const semantics::Symbol *, mlir::Value> *reductionVarCache =
nullptr) {
ClauseProcessor cp(converter, semaCtx, clauses);
@@ -2726,7 +2758,7 @@ static void genWsloopClauses(
cp.processOrdered(clauseOps);
cp.processReduction(loc, clauseOps, reductionObjects, reductionVarCache);
cp.processSchedule(stmtCtx, clauseOps);
- cp.processLinear(clauseOps);
+ cp.processLinear(clauseOps, /*isDeclareSimd=*/false, &linearSyms);
}
//===----------------------------------------------------------------------===//
@@ -4908,8 +4940,9 @@ static mlir::omp::WsloopOp genStandaloneDo(
llvm::ArrayRef<const semantics::Symbol *> metadirectiveLoopIVs) {
mlir::omp::WsloopOperands wsloopClauseOps;
llvm::SmallVector<Object> wsloopReductionObjects;
+ llvm::SmallVector<const semantics::Symbol *> wsloopLinearSyms;
genWsloopClauses(converter, semaCtx, stmtCtx, item->clauses, loc,
- wsloopClauseOps, wsloopReductionObjects);
+ wsloopClauseOps, wsloopReductionObjects, wsloopLinearSyms);
DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval,
/*shouldCollectPreDeterminedSymbols=*/true,
@@ -4988,6 +5021,8 @@ static mlir::omp::WsloopOp genStandaloneDo(
loopNestClauseOps, iv);
ObjectEntryBlockArgs wsloopArgs;
+ genLinearPrivatizers(dsp, loc, wsloopLinearSyms, wsloopClauseOps,
+ wsloopArgs.linear);
wsloopArgs.priv.objects = makeObjects(dsp.getDelayedPrivSymbols());
wsloopArgs.priv.vars = wsloopClauseOps.privateVars;
wsloopArgs.reduction.objects = wsloopReductionObjects;
@@ -5132,8 +5167,9 @@ static mlir::omp::SimdOp genStandaloneSimd(
llvm::ArrayRef<const semantics::Symbol *> metadirectiveLoopIVs) {
mlir::omp::SimdOperands simdClauseOps;
llvm::SmallVector<Object> simdReductionObjects;
+ llvm::SmallVector<const semantics::Symbol *> simdLinearSyms;
genSimdClauses(converter, semaCtx, item->clauses, loc, simdClauseOps,
- simdReductionObjects);
+ simdReductionObjects, simdLinearSyms);
DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval,
/*shouldCollectPreDeterminedSymbols=*/true,
@@ -5149,9 +5185,11 @@ static mlir::omp::SimdOp genStandaloneSimd(
genLoopNestClauses(converter, semaCtx, eval, item->clauses, loc,
loopNestClauseOps, iv);
genSimdImplicitLinear(converter, semaCtx, simdClauseOps, loopNestClauseOps,
- iv);
+ iv, simdLinearSyms);
ObjectEntryBlockArgs simdArgs;
+ genLinearPrivatizers(dsp, loc, simdLinearSyms, simdClauseOps,
+ simdArgs.linear);
simdArgs.priv.objects = makeObjects(dsp.getDelayedPrivSymbols());
simdArgs.priv.vars = simdClauseOps.privateVars;
simdArgs.reduction.objects = simdReductionObjects;
@@ -5266,8 +5304,9 @@ static mlir::omp::DistributeOp genCompositeDistributeParallelDo(
mlir::omp::WsloopOperands wsloopClauseOps;
llvm::SmallVector<Object> wsloopReductionObjects;
+ llvm::SmallVector<const semantics::Symbol *> wsloopLinearSyms;
genWsloopClauses(converter, semaCtx, stmtCtx, doItem->clauses, loc,
- wsloopClauseOps, wsloopReductionObjects);
+ wsloopClauseOps, wsloopReductionObjects, wsloopLinearSyms);
mlir::omp::LoopNestOperands loopNestClauseOps;
llvm::SmallVector<const semantics::Symbol *> iv;
@@ -5283,6 +5322,8 @@ static mlir::omp::DistributeOp genCompositeDistributeParallelDo(
ObjectEntryBlockArgs wsloopArgs;
// TODO: Add private syms and vars.
+ genLinearPrivatizers(dsp, loc, wsloopLinearSyms, wsloopClauseOps,
+ wsloopArgs.linear);
wsloopArgs.reduction.objects = wsloopReductionObjects;
wsloopArgs.reduction.vars = wsloopClauseOps.reductionVars;
auto wsloopOp = genWrapperOp<mlir::omp::WsloopOp>(
@@ -5340,13 +5381,16 @@ static mlir::omp::DistributeOp genCompositeDistributeParallelDoSimd(
mlir::omp::WsloopOperands wsloopClauseOps;
llvm::SmallVector<Object> wsloopReductionObjects;
+ llvm::SmallVector<const semantics::Symbol *> wsloopLinearSyms;
genWsloopClauses(converter, semaCtx, stmtCtx, doItem->clauses, loc,
- wsloopClauseOps, wsloopReductionObjects, &reductionVarCache);
+ wsloopClauseOps, wsloopReductionObjects, wsloopLinearSyms,
+ &reductionVarCache);
mlir::omp::SimdOperands simdClauseOps;
llvm::SmallVector<Object> simdReductionObjects;
+ llvm::SmallVector<const semantics::Symbol *> simdLinearSyms;
genSimdClauses(converter, semaCtx, simdItem->clauses, loc, simdClauseOps,
- simdReductionObjects, &reductionVarCache);
+ simdReductionObjects, simdLinearSyms, &reductionVarCache);
// Same as genCompositeDoSimd.
if (!simdClauseOps.linearVars.empty()) {
@@ -5358,6 +5402,8 @@ static mlir::omp::DistributeOp genCompositeDistributeParallelDoSimd(
simdClauseOps.linearStepVars.clear();
simdClauseOps.linearVarTypes = nullptr;
simdClauseOps.linearModifiers = nullptr;
+ wsloopLinearSyms = std::move(simdLinearSyms);
+ simdLinearSyms.clear();
}
DataSharingProcessor simdItemDSP(converter, semaCtx, simdItem->clauses, eval,
/*shouldCollectPreDeterminedSymbols=*/true,
@@ -5369,7 +5415,7 @@ static mlir::omp::DistributeOp genCompositeDistributeParallelDoSimd(
genLoopNestClauses(converter, semaCtx, eval, simdItem->clauses, loc,
loopNestClauseOps, iv);
genSimdImplicitLinear(converter, semaCtx, simdClauseOps, loopNestClauseOps,
- iv);
+ iv, simdLinearSyms);
// Operation creation.
ObjectEntryBlockArgs distributeArgs;
@@ -5380,6 +5426,8 @@ static mlir::omp::DistributeOp genCompositeDistributeParallelDoSimd(
ObjectEntryBlockArgs wsloopArgs;
// TODO: Add private syms and vars.
+ genLinearPrivatizers(simdItemDSP, loc, wsloopLinearSyms, wsloopClauseOps,
+ wsloopArgs.linear);
wsloopArgs.reduction.objects = wsloopReductionObjects;
wsloopArgs.reduction.vars = wsloopClauseOps.reductionVars;
auto wsloopOp = genWrapperOp<mlir::omp::WsloopOp>(
@@ -5387,6 +5435,8 @@ static mlir::omp::DistributeOp genCompositeDistributeParallelDoSimd(
wsloopOp.setComposite(/*val=*/true);
ObjectEntryBlockArgs simdArgs;
+ genLinearPrivatizers(simdItemDSP, loc, simdLinearSyms, simdClauseOps,
+ simdArgs.linear);
simdArgs.priv.objects = makeObjects(simdItemDSP.getDelayedPrivSymbols());
simdArgs.priv.vars = simdClauseOps.privateVars;
simdArgs.reduction.objects = simdReductionObjects;
@@ -5421,8 +5471,9 @@ static mlir::omp::DistributeOp genCompositeDistributeSimd(
mlir::omp::SimdOperands simdClauseOps;
llvm::SmallVector<Object> simdReductionObjects;
+ llvm::SmallVector<const semantics::Symbol *> simdLinearSyms;
genSimdClauses(converter, semaCtx, simdItem->clauses, loc, simdClauseOps,
- simdReductionObjects);
+ simdReductionObjects, simdLinearSyms);
DataSharingProcessor distributeItemDSP(
converter, semaCtx, distributeItem->clauses, eval,
@@ -5442,7 +5493,7 @@ static mlir::omp::DistributeOp genCompositeDistributeSimd(
genLoopNestClauses(converter, semaCtx, eval, simdItem->clauses, loc,
loopNestClauseOps, iv);
genSimdImplicitLinear(converter, semaCtx, simdClauseOps, loopNestClauseOps,
- iv);
+ iv, simdLinearSyms);
// Operation creation.
ObjectEntryBlockArgs distributeArgs;
@@ -5454,6 +5505,8 @@ static mlir::omp::DistributeOp genCompositeDistributeSimd(
distributeOp.setComposite(/*val=*/true);
ObjectEntryBlockArgs simdArgs;
+ genLinearPrivatizers(simdItemDSP, loc, simdLinearSyms, simdClauseOps,
+ simdArgs.linear);
simdArgs.priv.objects = makeObjects(simdItemDSP.getDelayedPrivSymbols());
simdArgs.priv.vars = simdClauseOps.privateVars;
simdArgs.reduction.objects = simdReductionObjects;
@@ -5488,13 +5541,16 @@ static mlir::omp::WsloopOp genCompositeDoSimd(
mlir::omp::WsloopOperands wsloopClauseOps;
llvm::SmallVector<Object> wsloopReductionObjects;
+ llvm::SmallVector<const semantics::Symbol *> wsloopLinearSyms;
genWsloopClauses(converter, semaCtx, stmtCtx, doItem->clauses, loc,
- wsloopClauseOps, wsloopReductionObjects, &reductionVarCache);
+ wsloopClauseOps, wsloopReductionObjects, wsloopLinearSyms,
+ &reductionVarCache);
mlir::omp::SimdOperands simdClauseOps;
llvm::SmallVector<Object> simdReductionObjects;
+ llvm::SmallVector<const semantics::Symbol *> simdLinearSyms;
genSimdClauses(converter, semaCtx, simdItem->clauses, loc, simdClauseOps,
- simdReductionObjects, &reductionVarCache);
+ simdReductionObjects, simdLinearSyms, &reductionVarCache);
// omp.simd writes back linear vars unconditionally, causing a race when
// inside a parallel region. Move them to wsloop which has proper last-iter
@@ -5508,6 +5564,8 @@ static mlir::omp::WsloopOp genCompositeDoSimd(
simdClauseOps.linearStepVars.clear();
simdClauseOps.linearVarTypes = nullptr;
simdClauseOps.linearModifiers = nullptr;
+ wsloopLinearSyms = std::move(simdLinearSyms);
+ simdLinearSyms.clear();
}
DataSharingProcessor wsloopItemDSP(
converter, semaCtx, doItem->clauses, eval,
@@ -5531,10 +5589,12 @@ static mlir::omp::WsloopOp genCompositeDoSimd(
genLoopNestClauses(converter, semaCtx, eval, simdItem->clauses, loc,
loopNestClauseOps, iv);
genSimdImplicitLinear(converter, semaCtx, simdClauseOps, loopNestClauseOps,
- iv);
+ iv, simdLinearSyms);
// Operation creation.
ObjectEntryBlockArgs wsloopArgs;
+ genLinearPrivatizers(simdItemDSP, loc, wsloopLinearSyms, wsloopClauseOps,
+ wsloopArgs.linear);
wsloopArgs.priv.objects = makeObjects(wsloopItemDSP.getDelayedPrivSymbols());
wsloopArgs.priv.vars = wsloopClauseOps.privateVars;
wsloopArgs.reduction.objects = wsloopReductionObjects;
@@ -5544,6 +5604,8 @@ static mlir::omp::WsloopOp genCompositeDoSimd(
wsloopOp.setComposite(/*val=*/true);
ObjectEntryBlockArgs simdArgs;
+ genLinearPrivatizers(simdItemDSP, loc, simdLinearSyms, simdClauseOps,
+ simdArgs.linear);
simdArgs.priv.objects = makeObjects(simdItemDSP.getDelayedPrivSymbols());
simdArgs.priv.vars = simdClauseOps.privateVars;
simdArgs.reduction.objects = simdReductionObjects;
diff --git a/flang/lib/Lower/Support/Utils.cpp b/flang/lib/Lower/Support/Utils.cpp
index e6b6e923308af..579840c05b703 100644
--- a/flang/lib/Lower/Support/Utils.cpp
+++ b/flang/lib/Lower/Support/Utils.cpp
@@ -672,7 +672,7 @@ void privatizeSymbol(
llvm::SmallPtrSet<const semantics::Symbol *, 16> &mightHaveReadHostSym,
const semantics::Symbol *symToPrivatize, OperandsStructType *clauseOps,
std::optional<llvm::omp::Directive> dir,
- bool forceHeapAllocationForPrivateDynamicArrays) {
+ bool forceHeapAllocationForPrivateDynamicArrays, bool isLinear) {
constexpr bool isDoConcurrent =
std::is_same_v<OpType, fir::LocalitySpecifierOp>;
mlir::OpBuilder::InsertPoint dcIP;
@@ -710,6 +710,8 @@ void privatizeSymbol(
// not emit the copy region when dealing with the SIMD directive.
if (dir && dir == llvm::omp::Directive::OMPD_simd)
emitCopyRegion = false;
+ assert((!isLinear || std::is_same_v<OpType, mlir::omp::PrivateClauseOp>) &&
+ "only omp.private has a `linear` type");
mlir::Value privVal = hsb.getAddr();
mlir::Type allocType = privVal.getType();
@@ -765,7 +767,8 @@ void privatizeSymbol(
auto moduleOp = firOpBuilder.getModule();
std::string privatizerPrefix =
converter.mangleName(*sym) +
- (emitCopyRegion ? "_firstprivate" : "_private");
+ (emitCopyRegion ? "_firstprivate"
+ : (isLinear ? "_linear" : "_private"));
if (forceHeapAllocation)
privatizerPrefix += "_heap";
auto uniquePrivatizerName = fir::getTypeAsString(
@@ -790,6 +793,7 @@ void privatizeSymbol(
firOpBuilder, symLoc, uniquePrivatizerName,
/*sym_visibility=*/nullptr, allocType,
emitCopyRegion ? mlir::omp::DataSharingClauseType::FirstPrivate
+ : isLinear ? mlir::omp::DataSharingClauseType::Linear
: mlir::omp::DataSharingClauseType::Private);
} else {
result =
@@ -888,6 +892,20 @@ void privatizeSymbol(
}
}
+ // A `linear` privatizer of a descriptor gives the address of its data.
+ if constexpr (std::is_same_v<OpType, mlir::omp::PrivateClauseOp>) {
+ if (isLinear && mlir::isa<fir::BaseBoxType>(allocType)) {
+ mlir::Block *block =
+ firOpBuilder.createBlock(&result.getDataAddressRegion(),
+ /*insertPt=*/{}, {argType}, {symLoc});
+ firOpBuilder.setInsertionPointToEnd(block);
+ mlir::Value box =
+ fir::LoadOp::create(firOpBuilder, symLoc, block->getArgument(0));
+ mlir::Value addr = fir::BoxAddrOp::create(firOpBuilder, symLoc, box);
+ mlir::omp::YieldOp::create(firOpBuilder, symLoc, addr);
+ }
+ }
+
return result;
}();
@@ -912,7 +930,7 @@ privatizeSymbol<mlir::omp::PrivateClauseOp, mlir::omp::PrivateClauseOps>(
const semantics::Symbol *symToPrivatize,
mlir::omp::PrivateClauseOps *clauseOps,
std::optional<llvm::omp::Directive> dir,
- bool forceHeapAllocationForPrivateDynamicArrays);
+ bool forceHeapAllocationForPrivateDynamicArrays, bool isLinear);
template void
privatizeSymbol<fir::LocalitySpecifierOp, fir::LocalitySpecifierOperands>(
@@ -923,6 +941,6 @@ privatizeSymbol<fir::LocalitySpecifierOp, fir::LocalitySpecifierOperands>(
const semantics::Symbol *symToPrivatize,
fir::LocalitySpecifierOperands *clauseOps,
std::optional<llvm::omp::Directive> dir,
- bool forceHeapAllocationForPrivateDynamicArrays);
+ bool forceHeapAllocationForPrivateDynamicArrays, bool isLinear);
} // end namespace Fortran::lower
diff --git a/flang/lib/Support/OpenMP-utils.cpp b/flang/lib/Support/OpenMP-utils.cpp
index b8db9e9bd72eb..dc7e69ba1bba7 100644
--- a/flang/lib/Support/OpenMP-utils.cpp
+++ b/flang/lib/Support/OpenMP-utils.cpp
@@ -18,9 +18,10 @@ mlir::Block *genEntryBlock(mlir::OpBuilder &builder, const EntryBlockArgs &args,
llvm::SmallVector<mlir::Type> types;
llvm::SmallVector<mlir::Location> locs;
unsigned numVars = args.hasDeviceAddrVars.size() + args.hostEvalVars.size() +
- args.inReductionVars.size() + args.mapVars.size() + args.privVars.size() +
- args.reductionVars.size() + args.taskReductionVars.size() +
- args.useDeviceAddrVars.size() + args.useDevicePtrVars.size();
+ args.inReductionVars.size() + args.linearVars.size() +
+ args.mapVars.size() + args.privVars.size() + args.reductionVars.size() +
+ args.taskReductionVars.size() + args.useDeviceAddrVars.size() +
+ args.useDevicePtrVars.size();
types.reserve(numVars);
locs.reserve(numVars);
@@ -36,6 +37,7 @@ mlir::Block *genEntryBlock(mlir::OpBuilder &builder, const EntryBlockArgs &args,
extractTypeLoc(args.hasDeviceAddrVars);
extractTypeLoc(args.hostEvalVars);
extractTypeLoc(args.inReductionVars);
+ extractTypeLoc(args.linearVars);
extractTypeLoc(args.mapVars);
extractTypeLoc(args.privVars);
extractTypeLoc(args.reductionVars);
diff --git a/flang/test/Lower/OpenMP/Todo/linear-no-delayed-privatization.f90 b/flang/test/Lower/OpenMP/Todo/linear-no-delayed-privatization.f90
new file mode 100644
index 0000000000000..a550529c2f22f
--- /dev/null
+++ b/flang/test/Lower/OpenMP/Todo/linear-no-delayed-privatization.f90
@@ -0,0 +1,10 @@
+! RUN: %not_todo_cmd %flang_fc1 -emit-hlfir -fopenmp -mmlir --enable-delayed-privatization=false -o - %s 2>&1 | FileCheck %s
+
+! CHECK: not yet implemented: LINEAR clause without delayed privatization
+subroutine linear_no_delayed_privatization()
+ integer :: i, x
+ !$omp simd linear(x)
+ do i = 1, 10
+ x = x + 1
+ end do
+end subroutine
diff --git a/flang/test/Lower/OpenMP/composite_simd_linear.f90 b/flang/test/Lower/OpenMP/composite_simd_linear.f90
index 35d4080f2041d..8a4f11c08daa8 100644
--- a/flang/test/Lower/OpenMP/composite_simd_linear.f90
+++ b/flang/test/Lower/OpenMP/composite_simd_linear.f90
@@ -8,10 +8,10 @@ subroutine do_simd
!CHECK: %[[CONST:.*]] = arith.constant 1 : i32
!CHECK: %{{.*}} = arith.constant 1 : i32
!CHECK: %[[IV_STEP:.*]] = arith.constant 1 : i32
-!DEFAULT: omp.wsloop linear(%[[X]]#0 : !fir.ref<i32> = {{.*}}) linear_var_types([i32]) {
-!OPENMP52: omp.wsloop linear(val(%[[X]]#0 : !fir.ref<i32> = {{.*}})) linear_var_types([i32]) {
-!DEFAULT: omp.simd linear(%[[I]]#0 : !fir.ref<i32> = %[[IV_STEP]] : i32) linear_var_types([i32]) {
-!OPENMP52: omp.simd linear(val(%[[I]]#0 : !fir.ref<i32> = %[[IV_STEP]] : i32)) linear_var_types([i32]) {
+!DEFAULT: omp.wsloop linear(@{{[^ ]+}} %[[X]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = {{.*}}) linear_var_types([i32]) {
+!OPENMP52: omp.wsloop linear(val(@{{[^ ]+}} %[[X]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = {{.*}})) linear_var_types([i32]) {
+!DEFAULT: omp.simd linear(@{{[^ ]+}} %[[I]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[IV_STEP]] : i32) linear_var_types([i32]) {
+!OPENMP52: omp.simd linear(val(@{{[^ ]+}} %[[I]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[IV_STEP]] : i32)) linear_var_types([i32]) {
!CHECK: }
!CHECK: } {omp.composite}
!CHECK: } {omp.composite}
@@ -50,10 +50,10 @@ subroutine distribute_parallel_do
!CHECK: %{{.*}} = arith.constant 1 : i32
!CHECK: %[[IV_STEP:.*]] = arith.constant 1 : i32
!CHECK: omp.distribute {
-!DEFAULT: omp.wsloop linear(%[[I]]#0 : !fir.ref<i32> = %[[CONST]] : i32) linear_var_types([i32]) {
-!OPENMP52: omp.wsloop linear(val(%[[I]]#0 : !fir.ref<i32> = %[[CONST]] : i32)) linear_var_types([i32]) {
-!DEFAULT: omp.simd linear(%[[I]]#0 : !fir.ref<i32> = %[[IV_STEP]] : i32) linear_var_types([i32]) {
-!OPENMP52: omp.simd linear(val(%[[I]]#0 : !fir.ref<i32> = %[[IV_STEP]] : i32)) linear_var_types([i32]) {
+!DEFAULT: omp.wsloop linear(@{{[^ ]+}} %[[I]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[CONST]] : i32) linear_var_types([i32]) {
+!OPENMP52: omp.wsloop linear(val(@{{[^ ]+}} %[[I]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[CONST]] : i32)) linear_var_types([i32]) {
+!DEFAULT: omp.simd linear(@{{[^ ]+}} %{{[^ ]+}} -> %{{[^ ]+}} : !fir.ref<i32> = %[[IV_STEP]] : i32) linear_var_types([i32]) {
+!OPENMP52: omp.simd linear(val(@{{[^ ]+}} %{{[^ ]+}} -> %{{[^ ]+}} : !fir.ref<i32> = %[[IV_STEP]] : i32)) linear_var_types([i32]) {
!$omp teams
!$omp distribute parallel do simd linear(i:1)
do i = 1, N
@@ -71,10 +71,10 @@ subroutine parallel_do
!CHECK: %[[LINEAR_STEP:.*]] = arith.constant 2 : i32
!CHECK: %{{.*}} = arith.constant 1 : i32
!CHECK: %[[IV_STEP:.*]] = arith.constant 1 : i32
-!DEFAULT: omp.wsloop linear(%[[X]]#0 : !fir.ref<i32> = %[[LINEAR_STEP]] : i32) linear_var_types([i32]) {
-!OPENMP52: omp.wsloop linear(val(%[[X]]#0 : !fir.ref<i32> = %[[LINEAR_STEP]] : i32)) linear_var_types([i32]) {
-!DEFAULT: omp.simd linear(%[[I]]#0 : !fir.ref<i32> = %[[IV_STEP]] : i32) linear_var_types([i32]) {
-!OPENMP52: omp.simd linear(val(%[[I]]#0 : !fir.ref<i32> = %[[IV_STEP]] : i32)) linear_var_types([i32]) {
+!DEFAULT: omp.wsloop linear(@{{[^ ]+}} %[[X]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[LINEAR_STEP]] : i32) linear_var_types([i32]) {
+!OPENMP52: omp.wsloop linear(val(@{{[^ ]+}} %[[X]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[LINEAR_STEP]] : i32)) linear_var_types([i32]) {
+!DEFAULT: omp.simd linear(@{{[^ ]+}} %[[I]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[IV_STEP]] : i32) linear_var_types([i32]) {
+!OPENMP52: omp.simd linear(val(@{{[^ ]+}} %[[I]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[IV_STEP]] : i32)) linear_var_types([i32]) {
integer :: x
!$omp parallel do simd linear(x:2)
do i = 1, N
@@ -92,8 +92,8 @@ subroutine teams_distribute
!CHECK: {{.*}} = arith.constant 1 : i32
!CHECK: %[[IV_STEP:.*]] = arith.constant 1 : i32
!CHECK: omp.distribute {
-!DEFAULT: omp.simd linear(%[[X]]#0 : !fir.ref<i32> = %[[LINEAR_STEP]] : i32, %[[I]]#0 : !fir.ref<i32> = %[[IV_STEP]] : i32) linear_var_types([i32, i32]) {
-!OPENMP52: omp.simd linear(val(%[[X]]#0 : !fir.ref<i32> = %[[LINEAR_STEP]] : i32), val(%[[I]]#0 : !fir.ref<i32> = %[[IV_STEP]] : i32)) linear_var_types([i32, i32]) {
+!DEFAULT: omp.simd linear(@{{[^ ]+}} %[[X]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[LINEAR_STEP]] : i32, @{{[^ ]+}} %[[I]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[IV_STEP]] : i32) linear_var_types([i32, i32]) {
+!OPENMP52: omp.simd linear(val(@{{[^ ]+}} %[[X]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[LINEAR_STEP]] : i32), val(@{{[^ ]+}} %[[I]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[IV_STEP]] : i32)) linear_var_types([i32, i32]) {
integer :: x
!$omp teams distribute simd linear(x)
do i = 1, N
@@ -111,10 +111,10 @@ subroutine teams_distribute_parallel_do
!CHECK: %{{.*}} = arith.constant 1 : i32
!CHECK: %[[IV_STEP:.*]] = arith.constant 1 : i32
!CHECK: omp.distribute {
-!DEFAULT: omp.wsloop linear(%[[X]]#0 : !fir.ref<i32> = %[[LINEAR_STEP]] : i32) linear_var_types([i32]) {
-!OPENMP52: omp.wsloop linear(val(%[[X]]#0 : !fir.ref<i32> = %[[LINEAR_STEP]] : i32)) linear_var_types([i32]) {
-!DEFAULT: omp.simd linear(%[[I]]#0 : !fir.ref<i32> = %[[IV_STEP]] : i32) linear_var_types([i32]) {
-!OPENMP52: omp.simd linear(val(%[[I]]#0 : !fir.ref<i32> = %[[IV_STEP]] : i32)) linear_var_types([i32]) {
+!DEFAULT: omp.wsloop linear(@{{[^ ]+}} %[[X]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[LINEAR_STEP]] : i32) linear_var_types([i32]) {
+!OPENMP52: omp.wsloop linear(val(@{{[^ ]+}} %[[X]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[LINEAR_STEP]] : i32)) linear_var_types([i32]) {
+!DEFAULT: omp.simd linear(@{{[^ ]+}} %[[I]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[IV_STEP]] : i32) linear_var_types([i32]) {
+!OPENMP52: omp.simd linear(val(@{{[^ ]+}} %[[I]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[IV_STEP]] : i32)) linear_var_types([i32]) {
integer :: x
!$omp teams distribute parallel do simd linear(x)
do i = 1, N
diff --git a/flang/test/Lower/OpenMP/compound-version-override.f90 b/flang/test/Lower/OpenMP/compound-version-override.f90
index 112ab98e4cd27..01a89c8ef9c32 100644
--- a/flang/test/Lower/OpenMP/compound-version-override.f90
+++ b/flang/test/Lower/OpenMP/compound-version-override.f90
@@ -4,9 +4,10 @@
!CHECK: warning: IF clause is not allowed on DO SIMD directive in OpenMP v4.5, try -fopenmp-version=50
!CHECK: omp.wsloop {
-!CHECK: omp.simd if(%true) linear(%[[V0:[0-9]+]]#0 : !fir.ref<i32> = %c1_i32_0 : i32) linear_var_types([i32]) {
-!CHECK: omp.loop_nest (%arg2) : i32 = (%c1_i32) to (%{{[0-9]+}}) inclusive step (%c1_i32_0) {
-!CHECK: hlfir.assign %arg2 to %[[V0]]#0 : i32, !fir.ref<i32>
+!CHECK: omp.simd if(%true) linear(@{{[^ ]+}} %{{[0-9]+}}#0 -> %[[I_ARG:arg[0-9]+]] : !fir.ref<i32> = %c1_i32_0 : i32) linear_var_types([i32]) {
+!CHECK: omp.loop_nest (%[[IV:arg[0-9]+]]) : i32 = (%c1_i32) to (%{{[0-9]+}}) inclusive step (%c1_i32_0) {
+!CHECK: %[[V0:[0-9]+]]:2 = hlfir.declare %[[I_ARG]] uniq_name("_QFcompoundEi")
+!CHECK: hlfir.assign %[[IV]] to %[[V0]]#0 : i32, !fir.ref<i32>
!CHECK: %[[V1:[0-9]+]] = fir.load %[[V0]]#0 : !fir.ref<i32>
!CHECK: %[[V2:[0-9]+]] = fir.convert %[[V1]] : (i32) -> i64
!CHECK: %[[V3:[0-9]+]] = hlfir.designate %10#0 (%[[V2]]) : (!fir.box<!fir.array<?xf32>>, i64) -> !fir.ref<f32>
diff --git a/flang/test/Lower/OpenMP/distribute-parallel-do-simd.f90 b/flang/test/Lower/OpenMP/distribute-parallel-do-simd.f90
index 266e5482e9756..5f6366bab960a 100644
--- a/flang/test/Lower/OpenMP/distribute-parallel-do-simd.f90
+++ b/flang/test/Lower/OpenMP/distribute-parallel-do-simd.f90
@@ -67,8 +67,8 @@ subroutine distribute_parallel_do_simd_simdlen()
! CHECK: omp.parallel {
! CHECK: omp.distribute {
! CHECK-NEXT: omp.wsloop {
- ! DEFAULT-NEXT: omp.simd linear({{.*}}) linear_var_types([i32]) simdlen(4) {
- ! OPENMP52-NEXT: omp.simd linear(val({{.*}})) linear_var_types([i32]) simdlen(4) {
+ ! DEFAULT-NEXT: omp.simd simdlen(4) linear({{.*}}) linear_var_types([i32]) {
+ ! OPENMP52-NEXT: omp.simd simdlen(4) linear(val({{.*}})) linear_var_types([i32]) {
! CHECK-NEXT: omp.loop_nest
!$omp distribute parallel do simd simdlen(4)
do index_ = 1, 10
@@ -92,9 +92,9 @@ subroutine distribute_parallel_do_simd_private()
! CHECK: omp.parallel {
! CHECK: omp.distribute {
! CHECK-NEXT: omp.wsloop {
- ! DEFAULT-NEXT: omp.simd linear(%{{.*}}) linear_var_types([i32]) private(@{{.*}} %[[X]]#0 -> %[[X_ARG:[^:]+]]
+ ! DEFAULT-NEXT: omp.simd linear({{.*}}) linear_var_types([i32]) private(@{{.*}} %[[X]]#0 -> %[[X_ARG:[^:]+]]
! DEFAULT-SAME: : !fir.ref<i64>) {
- ! OPENMP52-NEXT: omp.simd linear(val(%{{.*}})) linear_var_types([i32]) private(@{{.*}} %[[X]]#0 -> %[[X_ARG:[^:]+]]
+ ! OPENMP52-NEXT: omp.simd linear(val({{.*}})) linear_var_types([i32]) private(@{{.*}} %[[X]]#0 -> %[[X_ARG:[^:]+]]
! OPENMP52-SAME: : !fir.ref<i64>) {
! CHECK-NEXT: omp.loop_nest
! CHECK: %[[X_PRIV:.*]]:2 = hlfir.declare %[[X_ARG]]
diff --git a/flang/test/Lower/OpenMP/distribute-simd.f90 b/flang/test/Lower/OpenMP/distribute-simd.f90
index d01cda1083d81..69b1a507ead78 100644
--- a/flang/test/Lower/OpenMP/distribute-simd.f90
+++ b/flang/test/Lower/OpenMP/distribute-simd.f90
@@ -66,12 +66,14 @@ subroutine distribute_simd_private()
!$omp teams
! CHECK: omp.distribute
! CHECK: omp.simd
+ ! CHECK-SAME: linear(@{{.*}} %[[INDEX]]#0 -> %[[INDEX_ARG:[^ ]+]] :
! CHECK-SAME: private(@[[PRIV_BOX_SYM:.*]] %{{.*}} -> %[[PRIV_BOX:.*]] : !fir.ref<!fir.box<!fir.heap<i32>>>)
! CHECK-NEXT: omp.loop_nest (%[[IVAR:.*]]) : i32
!$omp distribute simd private(tmp)
do index_ = 1, 10
+ ! CHECK: %[[INDEX_PRIV:.*]]:2 = hlfir.declare %[[INDEX_ARG]]
! CHECK: %[[PRIV_BOX_DECL:.*]]:2 = hlfir.declare %[[PRIV_BOX]]
- ! CHECK: hlfir.assign %[[IVAR]] to %[[INDEX]]#0
+ ! CHECK: hlfir.assign %[[IVAR]] to %[[INDEX_PRIV]]#0
end do
!$omp end distribute simd
!$omp end teams
diff --git a/flang/test/Lower/OpenMP/indirect-reference-privatization.f90 b/flang/test/Lower/OpenMP/indirect-reference-privatization.f90
index 5f80f25152b53..9ff831bd4484c 100644
--- a/flang/test/Lower/OpenMP/indirect-reference-privatization.f90
+++ b/flang/test/Lower/OpenMP/indirect-reference-privatization.f90
@@ -3,7 +3,7 @@
!CHECK-LABEL: func @_QPparallel_simd
!CHECK: omp.parallel private(@_QFparallel_simdEk2_private_i32 {{.*}} -> %[[ARG:.*]] : !fir.ref<i32>)
!CHECK: %[[PRIV_K2:.*]]:2 = hlfir.declare %[[ARG]] uniq_name("_QFparallel_simdEk2")
-!CHECK: omp.simd linear(%[[PRIV_K2]]#0 {{.*}})
+!CHECK: omp.simd linear(@{{[^ ]+}} %[[PRIV_K2]]#0 -> %{{[^ ]+}} {{.*}})
subroutine parallel_simd
integer :: k1, k2
@@ -25,7 +25,7 @@ end subroutine parallel_simd
!CHECK-LABEL: func @_QPtask_simd
!CHECK: omp.task private(@_QFtask_simdEk_firstprivate_i32 %{{.*}})
!CHECK: %[[PRIV_K:.*]]:2 = hlfir.declare %{{.*}} uniq_name("_QFtask_simdEk")
-!CHECK: omp.simd linear(%[[PRIV_K]]#0 : !fir.ref<i32> {{.*}})
+!CHECK: omp.simd linear(@{{[^ ]+}} %[[PRIV_K]]#0 -> %{{[^ ]+}} : !fir.ref<i32> {{.*}})
subroutine task_simd
integer :: k
diff --git a/flang/test/Lower/OpenMP/linear-allocatable.f90 b/flang/test/Lower/OpenMP/linear-allocatable.f90
new file mode 100644
index 0000000000000..b6b62f5cf6872
--- /dev/null
+++ b/flang/test/Lower/OpenMP/linear-allocatable.f90
@@ -0,0 +1,113 @@
+! LINEAR on an allocatable uses a `linear` omp.private with a data_address.
+
+! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
+! RUN: bbc -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
+
+! CHECK-LABEL: omp.private {type = linear} @_QFsimd_linear_allocatableEa_linear_box_heap_i32 : !fir.box<!fir.heap<i32>> init {
+! CHECK: fir.allocmem i32
+! CHECK: } data_address {
+! CHECK: ^bb0(%[[ARG:.*]]: !fir.ref<!fir.box<!fir.heap<i32>>>):
+! CHECK: %[[BOX:.*]] = fir.load %[[ARG]] : !fir.ref<!fir.box<!fir.heap<i32>>>
+! CHECK: %[[ADDR:.*]] = fir.box_addr %[[BOX]] : (!fir.box<!fir.heap<i32>>) -> !fir.heap<i32>
+! CHECK: omp.yield(%[[ADDR]] : !fir.heap<i32>)
+! CHECK: } dealloc {
+! CHECK: fir.freemem
+
+! CHECK-LABEL: func.func @_QPsimd_linear_allocatable
+! CHECK: %[[A:.*]]:2 = hlfir.declare %{{.*}} uniq_name("_QFsimd_linear_allocatableEa") fortran_attrs<allocatable>
+! CHECK: omp.simd linear(@_QFsimd_linear_allocatableEa_linear_box_heap_i32 %[[A]]#0 -> %[[ARG:.*]] : !fir.ref<!fir.box<!fir.heap<i32>>> = %{{.*}} : i32,
+! CHECK-SAME: linear_var_types([i32
+! CHECK: omp.loop_nest
+! CHECK: %[[PRIV_A:.*]]:2 = hlfir.declare %[[ARG]] uniq_name("_QFsimd_linear_allocatableEa") fortran_attrs<allocatable>
+! CHECK: hlfir.assign %{{.*}} to %[[PRIV_A]]#0 realloc : i32, !fir.ref<!fir.box<!fir.heap<i32>>>
+! CHECK-NEXT: omp.yield
+subroutine simd_linear_allocatable()
+ integer, allocatable :: a
+ integer :: i
+ allocate(a)
+ a = 0
+ !$omp simd linear(a)
+ do i = 1, 2
+ a = 2
+ end do
+ !$omp end simd
+end subroutine
+
+! CHECK-LABEL: func.func @_QPdo_linear_allocatable
+! CHECK: %[[A:.*]]:2 = hlfir.declare %{{.*}} uniq_name("_QFdo_linear_allocatableEa") fortran_attrs<allocatable>
+! CHECK: omp.parallel
+! CHECK: omp.wsloop linear(@_QFdo_linear_allocatableEa_linear_box_heap_i32 %[[A]]#0 -> %[[ARG:.*]] : !fir.ref<!fir.box<!fir.heap<i32>>> = %{{.*}} : i32)
+! CHECK: omp.loop_nest
+! CHECK: hlfir.declare %[[ARG]] uniq_name("_QFdo_linear_allocatableEa") fortran_attrs<allocatable>
+subroutine do_linear_allocatable()
+ integer, allocatable :: a
+ integer :: i
+ allocate(a)
+ a = 0
+ !$omp parallel do linear(a:2)
+ do i = 1, 10
+ a = a + 2
+ end do
+ !$omp end parallel do
+end subroutine
+
+! Composite: the linear clause is on omp.wsloop.
+! CHECK-LABEL: func.func @_QPdo_simd_linear_allocatable
+! CHECK: %[[A:.*]]:2 = hlfir.declare %{{.*}} uniq_name("_QFdo_simd_linear_allocatableEa") fortran_attrs<allocatable>
+! CHECK: omp.wsloop linear(@_QFdo_simd_linear_allocatableEa_linear_box_heap_i64 %[[A]]#0 -> %[[ARG:.*]] : !fir.ref<!fir.box<!fir.heap<i64>>> = %{{.*}} : i64)
+! CHECK-NEXT: omp.simd linear(@_QFdo_simd_linear_allocatableEi_linear_i32
+! CHECK: omp.loop_nest
+! CHECK: hlfir.declare %[[ARG]] uniq_name("_QFdo_simd_linear_allocatableEa") fortran_attrs<allocatable>
+subroutine do_simd_linear_allocatable()
+ integer(8), allocatable :: a
+ integer :: i
+ allocate(a)
+ a = 0
+ !$omp do simd linear(a)
+ do i = 1, 10
+ a = a + 1
+ end do
+ !$omp end do simd
+end subroutine
+
+! An allocatable loop iteration variable.
+! CHECK-LABEL: func.func @_QPallocatable_iv
+! CHECK: %[[I:.*]]:2 = hlfir.declare %{{.*}} uniq_name("_QFallocatable_ivEi") fortran_attrs<allocatable>
+! CHECK: omp.simd linear(@_QFallocatable_ivEi_linear_box_heap_i32 %[[I]]#0 -> %[[ARG:.*]] : !fir.ref<!fir.box<!fir.heap<i32>>> = %{{.*}} : i32,
+! CHECK: omp.loop_nest (%[[IV:.*]]) :
+! CHECK: %[[PRIV_I:.*]]:2 = hlfir.declare %[[ARG]] uniq_name("_QFallocatable_ivEi") fortran_attrs<allocatable>
+! CHECK: %[[BOX:.*]] = fir.load %[[PRIV_I]]#0
+! CHECK: %[[ADDR:.*]] = fir.box_addr %[[BOX]]
+! CHECK: hlfir.assign %{{.*}} to %[[ADDR]] : i32, !fir.heap<i32>
+subroutine allocatable_iv()
+ integer, allocatable :: i
+ integer :: seen(3), k
+ allocate(i)
+ k = 1
+ !$omp simd linear(i:2) linear(k)
+ do i = 1, 5, 2
+ seen(k) = i
+ k = k + 1
+ end do
+ !$omp end simd
+end subroutine
+
+! The original is used after the loop.
+! CHECK-LABEL: func.func @_QPoriginal_used_after_loop
+! CHECK: %[[A:.*]]:2 = hlfir.declare %{{.*}} uniq_name("_QForiginal_used_after_loopEa") fortran_attrs<allocatable>
+! CHECK: omp.simd linear
+! CHECK: omp.yield
+! CHECK: }
+! CHECK: hlfir.assign %{{.*}} to %[[A]]#0 realloc : i32, !fir.ref<!fir.box<!fir.heap<i32>>>
+subroutine original_used_after_loop()
+ integer, allocatable :: a
+ integer :: i
+ allocate(a)
+ a = 0
+ !$omp simd linear(a)
+ do i = 1, 2
+ a = 2
+ end do
+ !$omp end simd
+ a = 5
+end subroutine
diff --git a/flang/test/Lower/OpenMP/linear_modifier.f90 b/flang/test/Lower/OpenMP/linear_modifier.f90
index 47dfe92aca952..1d3c507c617a2 100644
--- a/flang/test/Lower/OpenMP/linear_modifier.f90
+++ b/flang/test/Lower/OpenMP/linear_modifier.f90
@@ -9,8 +9,8 @@
subroutine simple_linear
implicit none
integer :: x, y, i
- !OPENMP52: omp.wsloop linear(val(%[[X]]#0 : !fir.ref<i32> = %[[const]] : i32)) {{.*}}
- !OPENMP45: omp.wsloop linear(%[[X]]#0 : !fir.ref<i32> = %[[const]] : i32) {{.*}}
+ !OPENMP52: omp.wsloop linear(val(@{{[^ ]+}} %[[X]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[const]] : i32)) {{.*}}
+ !OPENMP45: omp.wsloop linear(@{{[^ ]+}} %[[X]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[const]] : i32) {{.*}}
!$omp do linear(x)
do i = 1, 10
y = x + 2
@@ -25,8 +25,8 @@ subroutine linear_step
implicit none
integer :: x, y, i
!CHECK: %[[const:.*]] = arith.constant 4 : i32
- !OPENMP52: omp.wsloop linear(val(%[[X]]#0 : !fir.ref<i32> = %[[const]] : i32)) {{.*}}
- !OPENMP45: omp.wsloop linear(%[[X]]#0 : !fir.ref<i32> = %[[const]] : i32) {{.*}}
+ !OPENMP52: omp.wsloop linear(val(@{{[^ ]+}} %[[X]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[const]] : i32)) {{.*}}
+ !OPENMP45: omp.wsloop linear(@{{[^ ]+}} %[[X]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[const]] : i32) {{.*}}
!$omp do linear(x:4)
do i = 1, 10
y = x + 2
@@ -41,10 +41,10 @@ subroutine do_simd_linear
!CHECK: %[[CONST:.*]] = arith.constant 1 : i32
!CHECK: %{{.*}} = arith.constant 1 : i32
!CHECK: %[[IV_STEP:.*]] = arith.constant 1 : i32
-!OPENMP52: omp.wsloop linear(val(%[[X]]#0 : !fir.ref<i32> = %[[CONST]] : i32)) linear_var_types([i32]) {
-!OPENMP45: omp.wsloop linear(%[[X]]#0 : !fir.ref<i32> = %[[CONST]] : i32) linear_var_types([i32]) {
-!OPENMP52: omp.simd linear(val(%[[I]]#0 : !fir.ref<i32> = %[[IV_STEP]] : i32)) linear_var_types([i32]) {
-!OPENMP45: omp.simd linear(%[[I]]#0 : !fir.ref<i32> = %[[IV_STEP]] : i32) linear_var_types([i32]) {
+!OPENMP52: omp.wsloop linear(val(@{{[^ ]+}} %[[X]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[CONST]] : i32)) linear_var_types([i32]) {
+!OPENMP45: omp.wsloop linear(@{{[^ ]+}} %[[X]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[CONST]] : i32) linear_var_types([i32]) {
+!OPENMP52: omp.simd linear(val(@{{[^ ]+}} %[[I]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[IV_STEP]] : i32)) linear_var_types([i32]) {
+!OPENMP45: omp.simd linear(@{{[^ ]+}} %[[I]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[IV_STEP]] : i32) linear_var_types([i32]) {
integer :: x
!$omp do simd linear(x:1)
do i = 1, 10
diff --git a/flang/test/Lower/OpenMP/metadirective-loop.f90 b/flang/test/Lower/OpenMP/metadirective-loop.f90
index e96c3fe65b0f8..81dfac22c9ae7 100644
--- a/flang/test/Lower/OpenMP/metadirective-loop.f90
+++ b/flang/test/Lower/OpenMP/metadirective-loop.f90
@@ -64,15 +64,16 @@ subroutine test_do(n, a)
! CHECK: %[[SIMD_I:.*]]:2 = hlfir.declare {{.*}}_QFtest_simdEi
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK: omp.simd linear(
-! CHECK-SAME: val(%[[SIMD_I]]#0 : !fir.ref<i32> =
+! CHECK-SAME: val(@{{[^ ]+}} %[[SIMD_I]]#0 -> %[[SIMD_I_ARG:[^ ]+]] : !fir.ref<i32> =
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK: omp.loop_nest (%[[SIMD_IV:.*]]) :
+! CHECK: %[[SIMD_I_PRIV:.*]]:2 = hlfir.declare %[[SIMD_I_ARG]]
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
-! CHECK: hlfir.assign %[[SIMD_IV]] to %[[SIMD_I]]#0
+! CHECK: hlfir.assign %[[SIMD_IV]] to %[[SIMD_I_PRIV]]#0
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
-! CHECK: %[[SIMD_VALUE:.*]] = fir.load %[[SIMD_I]]#0
+! CHECK: %[[SIMD_VALUE:.*]] = fir.load %[[SIMD_I_PRIV]]#0
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
-! CHECK: %[[SIMD_INDEX:.*]] = fir.load %[[SIMD_I]]#0
+! CHECK: %[[SIMD_INDEX:.*]] = fir.load %[[SIMD_I_PRIV]]#0
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK: %[[SIMD_INDEX_I64:.*]] =
! CHECK-SAME: fir.convert %[[SIMD_INDEX]]
@@ -109,18 +110,19 @@ subroutine test_simd(n, a, after)
! CHECK: omp.wsloop
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK: omp.simd linear(
-! CHECK-SAME: val(%[[DO_SIMD_I]]#0 : !fir.ref<i32> =
+! CHECK-SAME: val(@{{[^ ]+}} %[[DO_SIMD_I]]#0 -> %[[DO_SIMD_I_ARG:[^ ]+]] : !fir.ref<i32> =
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK: omp.loop_nest (%[[DO_SIMD_IV:.*]]) :
+! CHECK: %[[DO_SIMD_I_PRIV:.*]]:2 = hlfir.declare %[[DO_SIMD_I_ARG]]
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK: hlfir.assign %[[DO_SIMD_IV]] to
-! CHECK-SAME: %[[DO_SIMD_I]]#0
+! CHECK-SAME: %[[DO_SIMD_I_PRIV]]#0
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK: %[[DO_SIMD_VALUE:.*]] =
-! CHECK-SAME: fir.load %[[DO_SIMD_I]]#0
+! CHECK-SAME: fir.load %[[DO_SIMD_I_PRIV]]#0
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK: %[[DO_SIMD_INDEX:.*]] =
-! CHECK-SAME: fir.load %[[DO_SIMD_I]]#0
+! CHECK-SAME: fir.load %[[DO_SIMD_I_PRIV]]#0
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK: %[[DO_SIMD_INDEX_I64:.*]] =
! CHECK-SAME: fir.convert %[[DO_SIMD_INDEX]]
@@ -728,11 +730,12 @@ pure logical function metadirective_identity(value)
! CHECK: fir.if {{.*}} {
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK: omp.simd linear(
-! CHECK-SAME: val(%[[PURE_I]]#0 : !fir.ref<i32> =
+! CHECK-SAME: val(@{{[^ ]+}} %[[PURE_I]]#0 -> %[[PURE_I_ARG:[^ ]+]] : !fir.ref<i32> =
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK: omp.loop_nest (%[[PURE_SIMD_IV:.*]]) :
+! CHECK: %[[PURE_I_PRIV:.*]]:2 = hlfir.declare %[[PURE_I_ARG]]
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
-! CHECK: hlfir.assign %[[PURE_SIMD_IV]] to %[[PURE_I]]#0
+! CHECK: hlfir.assign %[[PURE_SIMD_IV]] to %[[PURE_I_PRIV]]#0
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK-NOT: {{^ *}}}
! CHECK: %[[PURE_SIMD_ELEMENT:.*]] = hlfir.designate
@@ -829,18 +832,19 @@ subroutine test_pure_runtime_conditions(flag, n, a)
! CHECK: } else {
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK: omp.simd linear(
-! CHECK-SAME: val(%[[DYNAMIC_I]]#0 : !fir.ref<i32> =
+! CHECK-SAME: val(@{{[^ ]+}} %[[DYNAMIC_I]]#0 -> %[[DYNAMIC_I_ARG:[^ ]+]] : !fir.ref<i32> =
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK: omp.loop_nest (%[[DYNAMIC_SIMD_IV:.*]]) :
+! CHECK: %[[DYNAMIC_I_PRIV:.*]]:2 = hlfir.declare %[[DYNAMIC_I_ARG]]
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK: hlfir.assign %[[DYNAMIC_SIMD_IV]] to
-! CHECK-SAME: %[[DYNAMIC_I]]#0
+! CHECK-SAME: %[[DYNAMIC_I_PRIV]]#0
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK: %[[DYNAMIC_SIMD_VALUE:.*]] =
-! CHECK-SAME: fir.load %[[DYNAMIC_I]]#0
+! CHECK-SAME: fir.load %[[DYNAMIC_I_PRIV]]#0
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK: %[[DYNAMIC_SIMD_INDEX:.*]] =
-! CHECK-SAME: fir.load %[[DYNAMIC_I]]#0
+! CHECK-SAME: fir.load %[[DYNAMIC_I_PRIV]]#0
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK: %[[DYNAMIC_SIMD_INDEX_I64:.*]] =
! CHECK-SAME: fir.convert %[[DYNAMIC_SIMD_INDEX]]
@@ -1283,16 +1287,17 @@ subroutine test_dynamic_intervening_compiler_directives(flag, n, a)
! CHECK: } else {
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK: omp.simd linear(
-! CHECK-SAME: val(%[[DC_I]]#0 : !fir.ref<i32> =
+! CHECK-SAME: val(@{{[^ ]+}} %[[DC_I]]#0 -> %[[DC_I_ARG:[^ ]+]] : !fir.ref<i32> =
! CHECK-SAME: %[[DC_FB_STEP:[^ ]+]] : i32))
! CHECK-SAME: linear_var_types([i32]) {
! CHECK-NOT: Ei_private
! CHECK-NOT: Ej_private
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK: omp.loop_nest (%[[DC_FB_IV:.*]]) : i32
+! CHECK: %[[DC_I_PRIV:.*]]:2 = hlfir.declare %[[DC_I_ARG]]
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK-NOT: {{^ *}}}
-! CHECK: hlfir.assign %[[DC_FB_IV]] to %[[DC_I]]#0
+! CHECK: hlfir.assign %[[DC_FB_IV]] to %[[DC_I_PRIV]]#0
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK: fir.do_loop %[[DC_INNER_IV:.*]] =
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
@@ -1300,12 +1305,12 @@ subroutine test_dynamic_intervening_compiler_directives(flag, n, a)
! CHECK: fir.store %[[DC_INNER_IV]] to %[[DC_J]]#0
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK-NOT: {{^ *}}}
-! CHECK: %[[DC_FB_I_VALUE:.*]] = fir.load %[[DC_I]]#0
+! CHECK: %[[DC_FB_I_VALUE:.*]] = fir.load %[[DC_I_PRIV]]#0
! CHECK: %[[DC_FB_J_VALUE:.*]] = fir.load %[[DC_J]]#0
! CHECK: %[[DC_FB_SUM:.*]] = arith.addi
! CHECK-SAME: %[[DC_FB_I_VALUE]], %[[DC_FB_J_VALUE]]
! CHECK: %[[DC_FB_J_INDEX:.*]] = fir.load %[[DC_J]]#0
-! CHECK: %[[DC_FB_I_INDEX:.*]] = fir.load %[[DC_I]]#0
+! CHECK: %[[DC_FB_I_INDEX:.*]] = fir.load %[[DC_I_PRIV]]#0
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK-NOT: {{^ *}}}
! CHECK: %[[DC_FB_ELEMENT:.*]] = hlfir.designate
diff --git a/flang/test/Lower/OpenMP/simd-linear.f90 b/flang/test/Lower/OpenMP/simd-linear.f90
index 1149bdb300233..ef2fcafff39bf 100644
--- a/flang/test/Lower/OpenMP/simd-linear.f90
+++ b/flang/test/Lower/OpenMP/simd-linear.f90
@@ -16,9 +16,9 @@
subroutine simple_linear
implicit none
integer :: x, y, i
- !CHECK: omp.simd linear(val(%[[X]]#0 : !fir.ref<i32> = %[[const]] : i32)) {{.*}}
+ !CHECK: omp.simd linear(val(@{{[^ ]+}} %[[X]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[const]] : i32)) {{.*}}
- !IMPLICIT: omp.simd linear(%[[X]]#0 : !fir.ref<i32> = %[[const]] : i32, %[[I]]#0 : !fir.ref<i32> = %{{.*}} : i32) {{.*}}
+ !IMPLICIT: omp.simd linear(@{{[^ ]+}} %[[X]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[const]] : i32, @{{[^ ]+}} %[[I]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %{{.*}} : i32) {{.*}}
!$omp simd linear(x)
do i = 1, 10
end do
@@ -39,9 +39,9 @@ subroutine linear_step
implicit none
integer :: x, y, i
!CHECK: %[[const:.*]] = arith.constant 4 : i32
- !CHECK: omp.simd linear(val(%[[X]]#0 : !fir.ref<i32> = %[[const]] : i32)) {{.*}}
+ !CHECK: omp.simd linear(val(@{{[^ ]+}} %[[X]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[const]] : i32)) {{.*}}
- !IMPLICIT: omp.simd linear(%[[X]]#0 : !fir.ref<i32> = %[[const]] : i32, %[[I]]#0 : !fir.ref<i32> = %{{.*}} : i32) {{.*}}
+ !IMPLICIT: omp.simd linear(@{{[^ ]+}} %[[X]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[const]] : i32, @{{[^ ]+}} %[[I]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %{{.*}} : i32) {{.*}}
!$omp simd linear(x:4)
do i = 1, 10
end do
@@ -71,9 +71,9 @@ subroutine linear_expr
!IMPLICIT: %[[const:.*]] = arith.constant 4 : i32
!IMPLICIT: %[[LINEAR_EXPR:.*]] = arith.addi %[[LOAD_A]], %[[const]] : i32
- !CHECK: omp.simd linear(val(%[[X]]#0 : !fir.ref<i32> = %[[LINEAR_EXPR]] : i32)) {{.*}}
+ !CHECK: omp.simd linear(val(@{{[^ ]+}} %[[X]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[LINEAR_EXPR]] : i32)) {{.*}}
- !IMPLICIT: omp.simd linear(%[[X]]#0 : !fir.ref<i32> = %[[LINEAR_EXPR]] : i32, %[[I]]#0 : !fir.ref<i32> = {{.*}} : i32) {{.*}}
+ !IMPLICIT: omp.simd linear(@{{[^ ]+}} %[[X]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[LINEAR_EXPR]] : i32, @{{[^ ]+}} %[[I]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = {{.*}} : i32) {{.*}}
!$omp simd linear(x:a+4)
do i = 1, 10
end do
@@ -96,8 +96,8 @@ subroutine non_i32_type
!IMPLICIT: %[[STEP_VAR_CONST:.*]] = arith.constant 1 : i32
integer(kind=8)::j=0
- !CHECK: omp.simd linear(val(%[[J_DECLARE]]#0 : !fir.ref<i64> = %[[CONST]] : i64)) {{.*}}
- !IMPLICIT: omp.simd linear(%[[J_DECLARE]]#0 : !fir.ref<i64> = %[[CONST]] : i64, %[[I_DECLARE]]#0 : !fir.ref<i32> = %[[STEP_VAR_CONST]] : i32)
+ !CHECK: omp.simd linear(val(@{{[^ ]+}} %[[J_DECLARE]]#0 -> %{{[^ ]+}} : !fir.ref<i64> = %[[CONST]] : i64)) {{.*}}
+ !IMPLICIT: omp.simd linear(@{{[^ ]+}} %[[J_DECLARE]]#0 -> %{{[^ ]+}} : !fir.ref<i64> = %[[CONST]] : i64, @{{[^ ]+}} %[[I_DECLARE]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[STEP_VAR_CONST]] : i32)
!$omp simd linear(j:1_8)
do i = 1,10
end do
diff --git a/flang/test/Lower/OpenMP/wsloop-linear.f90 b/flang/test/Lower/OpenMP/wsloop-linear.f90
index a9fcf56175027..886ec1426c593 100644
--- a/flang/test/Lower/OpenMP/wsloop-linear.f90
+++ b/flang/test/Lower/OpenMP/wsloop-linear.f90
@@ -10,10 +10,11 @@
subroutine simple_linear
implicit none
integer :: x, y, i
- !DEFAULT: omp.wsloop linear(%[[X]]#0 : !fir.ref<i32> = %[[const]] : i32) {{.*}}
- !OPENMP52: omp.wsloop linear(val(%[[X]]#0 : !fir.ref<i32> = %[[const]] : i32)) {{.*}}
+ !DEFAULT: omp.wsloop linear(@{{[^ ]+}} %[[X]]#0 -> %[[X_ARG:[^ ]+]] : !fir.ref<i32> = %[[const]] : i32) {{.*}}
+ !OPENMP52: omp.wsloop linear(val(@{{[^ ]+}} %[[X]]#0 -> %[[X_ARG:[^ ]+]] : !fir.ref<i32> = %[[const]] : i32)) {{.*}}
!$omp do linear(x)
- !CHECK: %[[LOAD:.*]] = fir.load %[[X]]#0 : !fir.ref<i32>
+ !CHECK: %[[X_PRIV:.*]]:2 = hlfir.declare %[[X_ARG]]
+ !CHECK: %[[LOAD:.*]] = fir.load %[[X_PRIV]]#0 : !fir.ref<i32>
!CHECK: %[[const:.*]] = arith.constant 2 : i32
!CHECK: %[[RESULT:.*]] = arith.addi %[[LOAD]], %[[const]] : i32
do i = 1, 10
@@ -30,10 +31,11 @@ subroutine linear_step
implicit none
integer :: x, y, i
!CHECK: %[[const:.*]] = arith.constant 4 : i32
- !DEFAULT: omp.wsloop linear(%[[X]]#0 : !fir.ref<i32> = %[[const]] : i32) {{.*}}
- !OPENMP52: omp.wsloop linear(val(%[[X]]#0 : !fir.ref<i32> = %[[const]] : i32)) {{.*}}
+ !DEFAULT: omp.wsloop linear(@{{[^ ]+}} %[[X]]#0 -> %[[X_ARG:[^ ]+]] : !fir.ref<i32> = %[[const]] : i32) {{.*}}
+ !OPENMP52: omp.wsloop linear(val(@{{[^ ]+}} %[[X]]#0 -> %[[X_ARG:[^ ]+]] : !fir.ref<i32> = %[[const]] : i32)) {{.*}}
!$omp do linear(x:4)
- !CHECK: %[[LOAD:.*]] = fir.load %[[X]]#0 : !fir.ref<i32>
+ !CHECK: %[[X_PRIV:.*]]:2 = hlfir.declare %[[X_ARG]]
+ !CHECK: %[[LOAD:.*]] = fir.load %[[X_PRIV]]#0 : !fir.ref<i32>
!CHECK: %[[const:.*]] = arith.constant 2 : i32
!CHECK: %[[RESULT:.*]] = arith.addi %[[LOAD]], %[[const]] : i32
do i = 1, 10
@@ -53,8 +55,8 @@ subroutine linear_expr
!CHECK: %[[LOAD_A:.*]] = fir.load %[[A]]#0 : !fir.ref<i32>
!CHECK: %[[const:.*]] = arith.constant 4 : i32
!CHECK: %[[LINEAR_EXPR:.*]] = arith.addi %[[LOAD_A]], %[[const]] : i32
- !DEFAULT: omp.wsloop linear(%[[X]]#0 : !fir.ref<i32> = %[[LINEAR_EXPR]] : i32) {{.*}}
- !OPENMP52: omp.wsloop linear(val(%[[X]]#0 : !fir.ref<i32> = %[[LINEAR_EXPR]] : i32)) {{.*}}
+ !DEFAULT: omp.wsloop linear(@{{[^ ]+}} %[[X]]#0 -> %[[X_ARG:[^ ]+]] : !fir.ref<i32> = %[[LINEAR_EXPR]] : i32) {{.*}}
+ !OPENMP52: omp.wsloop linear(val(@{{[^ ]+}} %[[X]]#0 -> %[[X_ARG:[^ ]+]] : !fir.ref<i32> = %[[LINEAR_EXPR]] : i32)) {{.*}}
!$omp do linear(x:a+4)
do i = 1, 10
y = x + 2
diff --git a/flang/test/Lower/OpenMP/wsloop-simd.f90 b/flang/test/Lower/OpenMP/wsloop-simd.f90
index 61b4d9e0dd7dd..639e6e4b6ef41 100644
--- a/flang/test/Lower/OpenMP/wsloop-simd.f90
+++ b/flang/test/Lower/OpenMP/wsloop-simd.f90
@@ -92,12 +92,14 @@ subroutine do_simd_private()
! CHECK: %[[I_DECL:.*]]:2 = hlfir.declare %{{.*}} uniq_name("_QFdo_simd_privateEi")
! CHECK: omp.wsloop
! CHECK-NEXT: omp.simd
+ ! CHECK-SAME: linear(@{{.*}} %[[I_DECL]]#0 -> %[[I_ARG:[^ ]+]] :
! CHECK-SAME: private(@[[PRIV_BOX_SYM:.*]] %{{.*}} -> %[[PRIV_BOX:.*]] : !fir.ref<!fir.box<!fir.heap<i32>>>)
! CHECK-NEXT: omp.loop_nest (%[[IVAR:.*]]) : i32
!$omp do simd private(tmp)
do i=1, 10
+ ! CHECK: %[[I_PRIV:.*]]:2 = hlfir.declare %[[I_ARG]]
! CHECK: %[[PRIV_BOX_DECL:.*]]:2 = hlfir.declare %[[PRIV_BOX]]
- ! CHECK: hlfir.assign %[[IVAR]] to %[[I_DECL]]#0
+ ! CHECK: hlfir.assign %[[IVAR]] to %[[I_PRIV]]#0
! CHECK: %[[PRIV_BOX_LOAD:.*]] = fir.load %[[PRIV_BOX_DECL]]
! CHECK: hlfir.assign %{{.*}} to %[[PRIV_BOX_DECL]]#0
! CHECK: omp.yield
@@ -111,7 +113,7 @@ subroutine do_simd_lastprivate_firstprivate()
! CHECK: omp.wsloop
! CHECK-SAME: private(@[[FIRSTPRIVATE_A_SYM:.*]] %{{.*}} -> %[[FIRSTPRIVATE_A:.*]] : !fir.ref<i32>)
! CHECK-NEXT: omp.simd
- ! CHECK-SAME: linear({{.*}}#0 : !fir.ref<i32> = %{{[^:]*}} : i32)
+ ! CHECK-SAME: linear(@{{.*}}#0 -> %{{[^ ]+}} : !fir.ref<i32> = %{{[^:]*}} : i32)
! CHECK-SAME: private(@[[PRIVATE_A_SYM:.*]] %{{.*}} -> %[[PRIVATE_A:.*]] : !fir.ref<i32>)
!$omp do simd lastprivate(a) firstprivate(a)
do i = 1, 10
More information about the Mlir-commits
mailing list