[Mlir-commits] [mlir] ac5d395 - [mlir][flang][OpenMP] Privatize LINEAR list items with linear privatizers; support ALLOCATABLE (#228827)
llvmlistbot at llvm.org
llvmlistbot at llvm.org
Tue Oct 20 02:05:39 PDT 2026
Author: Aditya Trivedi
Date: 2026-10-20T10:05:32+01:00
New Revision: ac5d3955cd1931f71109caa8c026319ca69ca252
URL: https://github.com/llvm/llvm-project/commit/ac5d3955cd1931f71109caa8c026319ca69ca252
DIFF: https://github.com/llvm/llvm-project/commit/ac5d3955cd1931f71109caa8c026319ca69ca252.diff
LOG: [mlir][flang][OpenMP] Privatize LINEAR list items with linear privatizers; support ALLOCATABLE (#228827)
Flang miscompiled `LINEAR` on an `ALLOCATABLE` list item (#182401). The
linear
operand was the descriptor reference, but `linear_var_types` held the
element
type, so translation created an undersized private copy and the program
crashed.
This patch makes linear list items use privatizer recipes, the same way
`private` and `reduction` do.
**MLIR**
- The `linear` clause on `omp.wsloop` and `omp.simd` now carries a
privatizer
symbol and an entry block argument for each list item:
```mlir
omp.private {type = linear} @x.linear : i32
omp.simd linear(@x.linear %x -> %arg0 : !llvm.ptr = %step : i32)
linear_var_types([i32]) {
```
- New `linear` `DataSharingType` for `omp.private`. Also an optional
`data_address` region that yields the address of the underlying linear
value when the privatized type wraps it (e.g. a Fortran descriptor).
- Verifier checks:
- a linear clause must use a `linear` privatizer whose type matches its
list item;
- `linear` privatizers can't be used in other clauses;
- `data_address` is only allowed on `linear` privatizers, which can't
have a copy region;
- `omp.declare_simd` doesn't accept linear privatizers.
**Translation**
- Linear variables are allocated, initialized and deallocated through
the
existing private-variable path.
- The start value is read from the original variable, the private copy
is
set to `start + iv * step`, and the value from the last iteration is
copied back. All accesses go through the `data_address` region when the
privatizer has one.
- The old `.linear_result` temporary and the in-place rewrite of the
original variable are removed.
**Flang**
- `processLinear` and the implicit linear IV of `simd` now create
`linear`
privatizers through `DataSharingProcessor`. For an allocatable, the
recipe has a `data_address` region (`fir.load` + `fir.box_addr`).
- For composite `do simd` and `distribute parallel do simd`, linear
privatizers go on `omp.wsloop`.
- Lowering linear without delayed privatization now emits a TODO.
Fixes #182401
Added:
flang/test/Lower/OpenMP/Todo/linear-no-delayed-privatization.f90
flang/test/Lower/OpenMP/linear-allocatable.f90
mlir/test/Target/LLVMIR/openmp-linear-privatizer.mlir
Modified:
flang/include/flang/Lower/Support/Utils.h
flang/include/flang/Support/OpenMP-utils.h
flang/lib/Lower/OpenMP/ClauseProcessor.cpp
flang/lib/Lower/OpenMP/ClauseProcessor.h
flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
flang/lib/Lower/OpenMP/DataSharingProcessor.h
flang/lib/Lower/OpenMP/OpenMP.cpp
flang/lib/Lower/Support/Utils.cpp
flang/lib/Support/OpenMP-utils.cpp
flang/test/Lower/OpenMP/composite_simd_linear.f90
flang/test/Lower/OpenMP/compound-version-override.f90
flang/test/Lower/OpenMP/distribute-parallel-do-simd.f90
flang/test/Lower/OpenMP/distribute-simd.f90
flang/test/Lower/OpenMP/indirect-reference-privatization.f90
flang/test/Lower/OpenMP/linear_modifier.f90
flang/test/Lower/OpenMP/metadirective-loop.f90
flang/test/Lower/OpenMP/simd-linear.f90
flang/test/Lower/OpenMP/wsloop-linear.f90
flang/test/Lower/OpenMP/wsloop-simd.f90
mlir/include/mlir/Dialect/OpenMP/OpenMPClauses.td
mlir/include/mlir/Dialect/OpenMP/OpenMPEnums.td
mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
mlir/include/mlir/Dialect/OpenMP/OpenMPOpsInterfaces.td
mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
mlir/test/Dialect/OpenMP/invalid.mlir
mlir/test/Dialect/OpenMP/ops.mlir
mlir/test/Target/LLVMIR/omptarget-wsloop.mlir
mlir/test/Target/LLVMIR/openmp-llvm-invalid.mlir
mlir/test/Target/LLVMIR/openmp-llvm.mlir
mlir/test/Target/LLVMIR/openmp-simd-guided.mlir
mlir/test/Target/LLVMIR/openmp-simd-linear.mlir
mlir/test/Target/LLVMIR/openmp-simd-ordered.mlir
mlir/test/Target/LLVMIR/openmp-wsloop-linear.mlir
mlir/test/Target/LLVMIR/openmp-wsloop-simd-ordered.mlir
Removed:
################################################################################
diff --git a/flang/include/flang/Lower/Support/Utils.h b/flang/include/flang/Lower/Support/Utils.h
index 15d30905aadcda..afd4f220eaf864 100644
--- a/flang/include/flang/Lower/Support/Utils.h
+++ b/flang/include/flang/Lower/Support/Utils.h
@@ -100,7 +100,8 @@ void privatizeSymbol(
llvm::SmallPtrSet<const semantics::Symbol *, 16> &mightHaveReadHostSym,
const semantics::Symbol *symToPrivatize, OperandsStructType *clauseOps,
std::optional<llvm::omp::Directive> dir = std::nullopt,
- bool forceHeapAllocationForPrivateDynamicArrays = false);
+ bool forceHeapAllocationForPrivateDynamicArrays = false,
+ bool isLinear = false);
} // end namespace Fortran::lower
diff --git a/flang/include/flang/Support/OpenMP-utils.h b/flang/include/flang/Support/OpenMP-utils.h
index 0a648e9612ec42..3512a72ab01e89 100644
--- a/flang/include/flang/Support/OpenMP-utils.h
+++ b/flang/include/flang/Support/OpenMP-utils.h
@@ -22,6 +22,7 @@ struct EntryBlockArgs {
llvm::ArrayRef<mlir::Value> hasDeviceAddrVars;
llvm::ArrayRef<mlir::Value> hostEvalVars;
llvm::ArrayRef<mlir::Value> inReductionVars;
+ llvm::ArrayRef<mlir::Value> linearVars;
llvm::ArrayRef<mlir::Value> mapVars;
llvm::ArrayRef<mlir::Value> privVars;
llvm::ArrayRef<mlir::Value> reductionVars;
@@ -31,8 +32,8 @@ struct EntryBlockArgs {
auto getVars() const {
return llvm::concat<const mlir::Value>(hasDeviceAddrVars, hostEvalVars,
- inReductionVars, mapVars, privVars, reductionVars, taskReductionVars,
- useDeviceAddrVars, useDevicePtrVars);
+ inReductionVars, linearVars, mapVars, privVars, reductionVars,
+ taskReductionVars, useDeviceAddrVars, useDevicePtrVars);
}
};
diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
index b1f29bb23361f0..34961d3d44ba2f 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
@@ -1745,8 +1745,9 @@ bool ClauseProcessor::processIsDevicePtr(
return clauseFound;
}
-bool ClauseProcessor::processLinear(mlir::omp::LinearClauseOps &result,
- bool isDeclareSimd) const {
+bool ClauseProcessor::processLinear(
+ mlir::omp::LinearClauseOps &result, bool isDeclareSimd,
+ llvm::SmallVectorImpl<const semantics::Symbol *> *linearSyms) const {
lower::StatementContext stmtCtx;
std::vector<mlir::Attribute> typeAttrs;
std::vector<mlir::Attribute> linearModAttrs;
@@ -1776,6 +1777,8 @@ bool ClauseProcessor::processLinear(mlir::omp::LinearClauseOps &result,
semantics::Symbol *sym = object.sym();
const mlir::Value variable = converter.getSymbolAddress(*sym);
result.linearVars.push_back(variable);
+ if (linearSyms)
+ linearSyms->push_back(sym);
mlir::Type ty = converter.genType(*sym);
typeAttrs.push_back(mlir::TypeAttr::get(ty));
diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.h b/flang/lib/Lower/OpenMP/ClauseProcessor.h
index 6b0e0cf606f48f..33f44754bb84ba 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.h
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.h
@@ -154,8 +154,10 @@ class ClauseProcessor {
bool processIsDevicePtr(lower::StatementContext &stmtCtx,
mlir::omp::IsDevicePtrClauseOps &result,
llvm::SmallVectorImpl<Object> &isDeviceObjects) const;
- bool processLinear(mlir::omp::LinearClauseOps &result,
- bool isDeclareSimd = false) const;
+ bool
+ processLinear(mlir::omp::LinearClauseOps &result, bool isDeclareSimd = false,
+ llvm::SmallVectorImpl<const semantics::Symbol *> *linearSyms =
+ nullptr) const;
bool
processLink(llvm::SmallVectorImpl<DeclareTargetCaptureInfo> &result) const;
diff --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
index 98561803ea8bbe..e18aaab1a38127 100644
--- a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
@@ -264,27 +264,8 @@ void DataSharingProcessor::collectSymbolsForPrivatization() {
// Such cases are suggested to be clearly documented and explained
// instead of being silently skipped
auto isException = [&](const Fortran::semantics::Symbol *sym) -> bool {
- // The handling of linear clause is deferred to the OpenMP
- // IRBuilder which is responsible for all its aspects,
- // including privatization. Privatizing linear variables at this point would
- // cause the following structure:
- //
- // omp.op linear(%linear = %step : !fir.ref<type>) {
- // Use %linear in this BB
- // }
- //
- // to be changed to the following:
- //
- // omp. op linear(%linear = %step : !fir.ref<type>)
- // private(%linear -> %arg0 : !fir.ref<i32>) {
- // Declare and use %arg0 in this BB
- // }
- //
- // The OpenMP IRBuilder needs to map the linear MLIR value
- // (i.e. %linear) to its `uses` in the BB to correctly
- // implement the functionalities of linear clause. However,
- // privatizing here disallows the IRBuilder to
- // draw a relation between %linear and %arg0. Hence skip.
+ // Linear variables are privatized by the `linear` privatizers of the
+ // linear clause, which genLinearPrivatizers creates.
if (sym->test(Fortran::semantics::Symbol::Flag::OmpLinear))
return true;
return false;
@@ -560,8 +541,8 @@ void DataSharingProcessor::collectPrivatizedSymbols(
}
auto shouldCollectSymbol = [&](const semantics::Symbol *sym) {
- // Linear symbols are privatized by OpenMP IRBuilder, except when they are
- // enclosed within TARGET.
+ // Linear symbols are privatized by the linear clause, except when they are
+ // enclosed within TARGET, where implicit linearization is skipped.
mlir::Operation *currentOp =
firOpBuilder.getInsertionBlock()->getParentOp();
bool inTarget =
@@ -748,7 +729,7 @@ void DataSharingProcessor::copyLastPrivatize(mlir::Operation *op) {
void DataSharingProcessor::privatizeSymbol(
const semantics::Symbol *symToPrivatize,
mlir::omp::PrivateClauseOps *clauseOps,
- std::optional<llvm::omp::Directive> dir) {
+ std::optional<llvm::omp::Directive> dir, bool isLinear) {
if (!useDelayedPrivatization) {
cloneSymbol(symToPrivatize);
copyFirstPrivateSymbol(symToPrivatize);
@@ -759,7 +740,7 @@ void DataSharingProcessor::privatizeSymbol(
mlir::omp::PrivateClauseOps>(
converter, firOpBuilder, symTable, allPrivatizedSymbols,
mightHaveReadHostSym, symToPrivatize, clauseOps, dir,
- forceHeapAllocationForPrivateDynamicArrays);
+ forceHeapAllocationForPrivateDynamicArrays, isLinear);
}
} // namespace omp
} // namespace lower
diff --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.h b/flang/lib/Lower/OpenMP/DataSharingProcessor.h
index 73044c304800ba..0c0c2ab04a0b10 100644
--- a/flang/lib/Lower/OpenMP/DataSharingProcessor.h
+++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.h
@@ -146,7 +146,10 @@ class DataSharingProcessor {
void privatizeSymbol(const semantics::Symbol *symToPrivatize,
mlir::omp::PrivateClauseOps *clauseOps,
- std::optional<llvm::omp::Directive> dir = std::nullopt);
+ std::optional<llvm::omp::Directive> dir = std::nullopt,
+ bool isLinear = false);
+
+ bool usesDelayedPrivatization() const { return useDelayedPrivatization; }
const llvm::SetVector<const semantics::Symbol *> &
getConditionalLastprivateSymbols() const {
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 57978826c5d014..abe1df79f6aef2 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -207,6 +207,7 @@ struct ObjectEntryBlockArgs {
ObjectEntryBlockArgsEntry hasDeviceAddr;
llvm::ArrayRef<mlir::Value> hostEvalVars;
ObjectEntryBlockArgsEntry inReduction;
+ ObjectEntryBlockArgsEntry linear;
ObjectEntryBlockArgsEntry map;
ObjectEntryBlockArgsEntry priv;
ObjectEntryBlockArgsEntry reduction;
@@ -216,8 +217,9 @@ struct ObjectEntryBlockArgs {
std::size_t sourceUseDeviceAddrCount{0};
bool isValid() const {
- return hasDeviceAddr.isValid() && inReduction.isValid() && map.isValid() &&
- priv.isValid() && reduction.isValid() && taskReduction.isValid() &&
+ return hasDeviceAddr.isValid() && inReduction.isValid() &&
+ linear.isValid() && map.isValid() && priv.isValid() &&
+ reduction.isValid() && taskReduction.isValid() &&
useDeviceAddr.isValid() && useDevicePtr.isValid() &&
sourceUseDeviceAddrCount <= useDeviceAddr.objects.size();
}
@@ -231,6 +233,7 @@ struct ObjectEntryBlockArgs {
};
appendSyms(hasDeviceAddr);
appendSyms(inReduction);
+ appendSyms(linear);
appendSyms(map);
appendSyms(priv);
appendSyms(reduction);
@@ -242,9 +245,9 @@ struct ObjectEntryBlockArgs {
auto getVars() const {
return llvm::concat<const mlir::Value>(
- hasDeviceAddr.vars, hostEvalVars, inReduction.vars, map.vars, priv.vars,
- reduction.vars, taskReduction.vars, useDeviceAddr.vars,
- useDevicePtr.vars);
+ hasDeviceAddr.vars, hostEvalVars, inReduction.vars, linear.vars,
+ map.vars, priv.vars, reduction.vars, taskReduction.vars,
+ useDeviceAddr.vars, useDevicePtr.vars);
}
Fortran::common::openmp::EntryBlockArgs asEntryBlockArgs() const {
@@ -252,6 +255,7 @@ struct ObjectEntryBlockArgs {
args.hasDeviceAddrVars = hasDeviceAddr.vars;
args.hostEvalVars = hostEvalVars;
args.inReductionVars = inReduction.vars;
+ args.linearVars = linear.vars;
args.mapVars = map.vars;
args.privVars = priv.vars;
args.reductionVars = reduction.vars;
@@ -965,6 +969,8 @@ static void bindEntryBlockArgs(lower::AbstractConverter &converter,
bindMapLike(args.hasDeviceAddr.objects, op.getHasDeviceAddrBlockArgs());
bindPrivateLike(args.inReduction.objects, args.inReduction.vars,
op.getInReductionBlockArgs());
+ bindPrivateLike(args.linear.objects, args.linear.vars,
+ op.getLinearBlockArgs());
bindMapLike(args.map.objects, op.getMapBlockArgs());
bindPrivateLike(args.priv.objects, args.priv.vars, op.getPrivateBlockArgs());
bindPrivateLike(args.reduction.objects, args.reduction.vars,
@@ -2402,6 +2408,7 @@ static void genSimdClauses(
const List<Clause> &clauses, mlir::Location loc,
mlir::omp::SimdOperands &clauseOps,
llvm::SmallVectorImpl<Object> &reductionObjects,
+ llvm::SmallVectorImpl<const semantics::Symbol *> &linearSyms,
llvm::DenseMap<const semantics::Symbol *, mlir::Value> *reductionVarCache =
nullptr) {
ClauseProcessor cp(converter, semaCtx, clauses);
@@ -2412,17 +2419,17 @@ static void genSimdClauses(
cp.processReduction(loc, clauseOps, reductionObjects, reductionVarCache);
cp.processSafelen(clauseOps);
cp.processSimdlen(clauseOps);
- cp.processLinear(clauseOps);
+ cp.processLinear(clauseOps, /*isDeclareSimd=*/false, &linearSyms);
}
// SIMD construct may have implicit
// linear semantics on IV. Process the same here.
-static void
-genSimdImplicitLinear(lower::AbstractConverter &converter,
- semantics::SemanticsContext &semaCtx,
- mlir::omp::SimdOperands &clauseOps,
- mlir::omp::LoopNestOperands loopNestClauseOps,
- llvm::SmallVector<const semantics::Symbol *> iv) {
+static void genSimdImplicitLinear(
+ lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
+ mlir::omp::SimdOperands &clauseOps,
+ mlir::omp::LoopNestOperands loopNestClauseOps,
+ llvm::SmallVector<const semantics::Symbol *> iv,
+ llvm::SmallVectorImpl<const semantics::Symbol *> &linearSyms) {
// If the (standalone/composite) SIMD is enclosed within TARGET,
// implicit linearization will cause invalid FIR due to
@@ -2475,6 +2482,7 @@ genSimdImplicitLinear(lower::AbstractConverter &converter,
mlir::UnitAttr::get(&converter.getMLIRContext()));
clauseOps.linearVars.push_back(variable);
clauseOps.linearStepVars.push_back(loopStep);
+ linearSyms.push_back(loopVar);
}
}
if (!typeAttrs.empty()) {
@@ -2485,6 +2493,29 @@ genSimdImplicitLinear(lower::AbstractConverter &converter,
}
}
+/// Create the `linear` privatizers of the LINEAR items `linearSyms`. Their
+/// private copies are entry block arguments of the op with the linear clause.
+static void
+genLinearPrivatizers(DataSharingProcessor &dsp, mlir::Location loc,
+ llvm::ArrayRef<const semantics::Symbol *> linearSyms,
+ mlir::omp::LinearClauseOps &clauseOps,
+ ObjectEntryBlockArgsEntry &linearArgs) {
+ if (linearSyms.empty())
+ return;
+ if (!dsp.usesDelayedPrivatization())
+ TODO(loc, "LINEAR clause without delayed privatization");
+
+ for (auto [sym, var] : llvm::zip_equal(linearSyms, clauseOps.linearVars)) {
+ mlir::omp::PrivateClauseOps privateOps;
+ dsp.privatizeSymbol(sym, &privateOps, /*dir=*/std::nullopt,
+ /*isLinear=*/true);
+ var = privateOps.privateVars.front();
+ clauseOps.linearSyms.push_back(privateOps.privateSyms.front());
+ }
+ linearArgs.objects = makeObjects(linearSyms);
+ linearArgs.vars = clauseOps.linearVars;
+}
+
static void genScopeClauses(lower::AbstractConverter &converter,
semantics::SemanticsContext &semaCtx,
const List<Clause> &clauses, mlir::Location loc,
@@ -2717,6 +2748,7 @@ static void genWsloopClauses(
lower::StatementContext &stmtCtx, const List<Clause> &clauses,
mlir::Location loc, mlir::omp::WsloopOperands &clauseOps,
llvm::SmallVectorImpl<Object> &reductionObjects,
+ llvm::SmallVectorImpl<const semantics::Symbol *> &linearSyms,
llvm::DenseMap<const semantics::Symbol *, mlir::Value> *reductionVarCache =
nullptr) {
ClauseProcessor cp(converter, semaCtx, clauses);
@@ -2726,7 +2758,7 @@ static void genWsloopClauses(
cp.processOrdered(clauseOps);
cp.processReduction(loc, clauseOps, reductionObjects, reductionVarCache);
cp.processSchedule(stmtCtx, clauseOps);
- cp.processLinear(clauseOps);
+ cp.processLinear(clauseOps, /*isDeclareSimd=*/false, &linearSyms);
}
//===----------------------------------------------------------------------===//
@@ -4911,8 +4943,9 @@ static mlir::omp::WsloopOp genStandaloneDo(
llvm::ArrayRef<const semantics::Symbol *> metadirectiveLoopIVs) {
mlir::omp::WsloopOperands wsloopClauseOps;
llvm::SmallVector<Object> wsloopReductionObjects;
+ llvm::SmallVector<const semantics::Symbol *> wsloopLinearSyms;
genWsloopClauses(converter, semaCtx, stmtCtx, item->clauses, loc,
- wsloopClauseOps, wsloopReductionObjects);
+ wsloopClauseOps, wsloopReductionObjects, wsloopLinearSyms);
DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval,
/*shouldCollectPreDeterminedSymbols=*/true,
@@ -4991,6 +5024,8 @@ static mlir::omp::WsloopOp genStandaloneDo(
loopNestClauseOps, iv);
ObjectEntryBlockArgs wsloopArgs;
+ genLinearPrivatizers(dsp, loc, wsloopLinearSyms, wsloopClauseOps,
+ wsloopArgs.linear);
wsloopArgs.priv.objects = makeObjects(dsp.getDelayedPrivSymbols());
wsloopArgs.priv.vars = wsloopClauseOps.privateVars;
wsloopArgs.reduction.objects = wsloopReductionObjects;
@@ -5135,8 +5170,9 @@ static mlir::omp::SimdOp genStandaloneSimd(
llvm::ArrayRef<const semantics::Symbol *> metadirectiveLoopIVs) {
mlir::omp::SimdOperands simdClauseOps;
llvm::SmallVector<Object> simdReductionObjects;
+ llvm::SmallVector<const semantics::Symbol *> simdLinearSyms;
genSimdClauses(converter, semaCtx, item->clauses, loc, simdClauseOps,
- simdReductionObjects);
+ simdReductionObjects, simdLinearSyms);
DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval,
/*shouldCollectPreDeterminedSymbols=*/true,
@@ -5152,9 +5188,11 @@ static mlir::omp::SimdOp genStandaloneSimd(
genLoopNestClauses(converter, semaCtx, eval, item->clauses, loc,
loopNestClauseOps, iv);
genSimdImplicitLinear(converter, semaCtx, simdClauseOps, loopNestClauseOps,
- iv);
+ iv, simdLinearSyms);
ObjectEntryBlockArgs simdArgs;
+ genLinearPrivatizers(dsp, loc, simdLinearSyms, simdClauseOps,
+ simdArgs.linear);
simdArgs.priv.objects = makeObjects(dsp.getDelayedPrivSymbols());
simdArgs.priv.vars = simdClauseOps.privateVars;
simdArgs.reduction.objects = simdReductionObjects;
@@ -5274,8 +5312,9 @@ static mlir::omp::DistributeOp genCompositeDistributeParallelDo(
mlir::omp::WsloopOperands wsloopClauseOps;
llvm::SmallVector<Object> wsloopReductionObjects;
+ llvm::SmallVector<const semantics::Symbol *> wsloopLinearSyms;
genWsloopClauses(converter, semaCtx, stmtCtx, doItem->clauses, loc,
- wsloopClauseOps, wsloopReductionObjects);
+ wsloopClauseOps, wsloopReductionObjects, wsloopLinearSyms);
mlir::omp::LoopNestOperands loopNestClauseOps;
llvm::SmallVector<const semantics::Symbol *> iv;
@@ -5291,6 +5330,8 @@ static mlir::omp::DistributeOp genCompositeDistributeParallelDo(
ObjectEntryBlockArgs wsloopArgs;
// TODO: Add private syms and vars.
+ genLinearPrivatizers(dsp, loc, wsloopLinearSyms, wsloopClauseOps,
+ wsloopArgs.linear);
wsloopArgs.reduction.objects = wsloopReductionObjects;
wsloopArgs.reduction.vars = wsloopClauseOps.reductionVars;
auto wsloopOp = genWrapperOp<mlir::omp::WsloopOp>(
@@ -5356,13 +5397,16 @@ static mlir::omp::DistributeOp genCompositeDistributeParallelDoSimd(
mlir::omp::WsloopOperands wsloopClauseOps;
llvm::SmallVector<Object> wsloopReductionObjects;
+ llvm::SmallVector<const semantics::Symbol *> wsloopLinearSyms;
genWsloopClauses(converter, semaCtx, stmtCtx, doItem->clauses, loc,
- wsloopClauseOps, wsloopReductionObjects, &reductionVarCache);
+ wsloopClauseOps, wsloopReductionObjects, wsloopLinearSyms,
+ &reductionVarCache);
mlir::omp::SimdOperands simdClauseOps;
llvm::SmallVector<Object> simdReductionObjects;
+ llvm::SmallVector<const semantics::Symbol *> simdLinearSyms;
genSimdClauses(converter, semaCtx, simdItem->clauses, loc, simdClauseOps,
- simdReductionObjects, &reductionVarCache);
+ simdReductionObjects, simdLinearSyms, &reductionVarCache);
// Same as genCompositeDoSimd.
if (!simdClauseOps.linearVars.empty()) {
@@ -5374,6 +5418,8 @@ static mlir::omp::DistributeOp genCompositeDistributeParallelDoSimd(
simdClauseOps.linearStepVars.clear();
simdClauseOps.linearVarTypes = nullptr;
simdClauseOps.linearModifiers = nullptr;
+ wsloopLinearSyms = std::move(simdLinearSyms);
+ simdLinearSyms.clear();
}
DataSharingProcessor simdItemDSP(converter, semaCtx, simdItem->clauses, eval,
/*shouldCollectPreDeterminedSymbols=*/true,
@@ -5385,7 +5431,7 @@ static mlir::omp::DistributeOp genCompositeDistributeParallelDoSimd(
genLoopNestClauses(converter, semaCtx, eval, simdItem->clauses, loc,
loopNestClauseOps, iv);
genSimdImplicitLinear(converter, semaCtx, simdClauseOps, loopNestClauseOps,
- iv);
+ iv, simdLinearSyms);
// Operation creation.
ObjectEntryBlockArgs distributeArgs;
@@ -5396,6 +5442,8 @@ static mlir::omp::DistributeOp genCompositeDistributeParallelDoSimd(
ObjectEntryBlockArgs wsloopArgs;
// TODO: Add private syms and vars.
+ genLinearPrivatizers(simdItemDSP, loc, wsloopLinearSyms, wsloopClauseOps,
+ wsloopArgs.linear);
wsloopArgs.reduction.objects = wsloopReductionObjects;
wsloopArgs.reduction.vars = wsloopClauseOps.reductionVars;
auto wsloopOp = genWrapperOp<mlir::omp::WsloopOp>(
@@ -5403,6 +5451,8 @@ static mlir::omp::DistributeOp genCompositeDistributeParallelDoSimd(
wsloopOp.setComposite(/*val=*/true);
ObjectEntryBlockArgs simdArgs;
+ genLinearPrivatizers(simdItemDSP, loc, simdLinearSyms, simdClauseOps,
+ simdArgs.linear);
simdArgs.priv.objects = makeObjects(simdItemDSP.getDelayedPrivSymbols());
simdArgs.priv.vars = simdClauseOps.privateVars;
simdArgs.reduction.objects = simdReductionObjects;
@@ -5442,8 +5492,9 @@ static mlir::omp::DistributeOp genCompositeDistributeSimd(
mlir::omp::SimdOperands simdClauseOps;
llvm::SmallVector<Object> simdReductionObjects;
+ llvm::SmallVector<const semantics::Symbol *> simdLinearSyms;
genSimdClauses(converter, semaCtx, simdItem->clauses, loc, simdClauseOps,
- simdReductionObjects);
+ simdReductionObjects, simdLinearSyms);
DataSharingProcessor distributeItemDSP(
converter, semaCtx, distributeItem->clauses, eval,
@@ -5463,7 +5514,7 @@ static mlir::omp::DistributeOp genCompositeDistributeSimd(
genLoopNestClauses(converter, semaCtx, eval, simdItem->clauses, loc,
loopNestClauseOps, iv);
genSimdImplicitLinear(converter, semaCtx, simdClauseOps, loopNestClauseOps,
- iv);
+ iv, simdLinearSyms);
// Operation creation.
ObjectEntryBlockArgs distributeArgs;
@@ -5475,6 +5526,8 @@ static mlir::omp::DistributeOp genCompositeDistributeSimd(
distributeOp.setComposite(/*val=*/true);
ObjectEntryBlockArgs simdArgs;
+ genLinearPrivatizers(simdItemDSP, loc, simdLinearSyms, simdClauseOps,
+ simdArgs.linear);
simdArgs.priv.objects = makeObjects(simdItemDSP.getDelayedPrivSymbols());
simdArgs.priv.vars = simdClauseOps.privateVars;
simdArgs.reduction.objects = simdReductionObjects;
@@ -5509,13 +5562,16 @@ static mlir::omp::WsloopOp genCompositeDoSimd(
mlir::omp::WsloopOperands wsloopClauseOps;
llvm::SmallVector<Object> wsloopReductionObjects;
+ llvm::SmallVector<const semantics::Symbol *> wsloopLinearSyms;
genWsloopClauses(converter, semaCtx, stmtCtx, doItem->clauses, loc,
- wsloopClauseOps, wsloopReductionObjects, &reductionVarCache);
+ wsloopClauseOps, wsloopReductionObjects, wsloopLinearSyms,
+ &reductionVarCache);
mlir::omp::SimdOperands simdClauseOps;
llvm::SmallVector<Object> simdReductionObjects;
+ llvm::SmallVector<const semantics::Symbol *> simdLinearSyms;
genSimdClauses(converter, semaCtx, simdItem->clauses, loc, simdClauseOps,
- simdReductionObjects, &reductionVarCache);
+ simdReductionObjects, simdLinearSyms, &reductionVarCache);
// omp.simd writes back linear vars unconditionally, causing a race when
// inside a parallel region. Move them to wsloop which has proper last-iter
@@ -5529,6 +5585,8 @@ static mlir::omp::WsloopOp genCompositeDoSimd(
simdClauseOps.linearStepVars.clear();
simdClauseOps.linearVarTypes = nullptr;
simdClauseOps.linearModifiers = nullptr;
+ wsloopLinearSyms = std::move(simdLinearSyms);
+ simdLinearSyms.clear();
}
DataSharingProcessor wsloopItemDSP(
converter, semaCtx, doItem->clauses, eval,
@@ -5552,10 +5610,12 @@ static mlir::omp::WsloopOp genCompositeDoSimd(
genLoopNestClauses(converter, semaCtx, eval, simdItem->clauses, loc,
loopNestClauseOps, iv);
genSimdImplicitLinear(converter, semaCtx, simdClauseOps, loopNestClauseOps,
- iv);
+ iv, simdLinearSyms);
// Operation creation.
ObjectEntryBlockArgs wsloopArgs;
+ genLinearPrivatizers(simdItemDSP, loc, wsloopLinearSyms, wsloopClauseOps,
+ wsloopArgs.linear);
wsloopArgs.priv.objects = makeObjects(wsloopItemDSP.getDelayedPrivSymbols());
wsloopArgs.priv.vars = wsloopClauseOps.privateVars;
wsloopArgs.reduction.objects = wsloopReductionObjects;
@@ -5565,6 +5625,8 @@ static mlir::omp::WsloopOp genCompositeDoSimd(
wsloopOp.setComposite(/*val=*/true);
ObjectEntryBlockArgs simdArgs;
+ genLinearPrivatizers(simdItemDSP, loc, simdLinearSyms, simdClauseOps,
+ simdArgs.linear);
simdArgs.priv.objects = makeObjects(simdItemDSP.getDelayedPrivSymbols());
simdArgs.priv.vars = simdClauseOps.privateVars;
simdArgs.reduction.objects = simdReductionObjects;
diff --git a/flang/lib/Lower/Support/Utils.cpp b/flang/lib/Lower/Support/Utils.cpp
index 056e7a5af6431a..c5440820d7eb09 100644
--- a/flang/lib/Lower/Support/Utils.cpp
+++ b/flang/lib/Lower/Support/Utils.cpp
@@ -672,7 +672,7 @@ void privatizeSymbol(
llvm::SmallPtrSet<const semantics::Symbol *, 16> &mightHaveReadHostSym,
const semantics::Symbol *symToPrivatize, OperandsStructType *clauseOps,
std::optional<llvm::omp::Directive> dir,
- bool forceHeapAllocationForPrivateDynamicArrays) {
+ bool forceHeapAllocationForPrivateDynamicArrays, bool isLinear) {
constexpr bool isDoConcurrent =
std::is_same_v<OpType, fir::LocalitySpecifierOp>;
mlir::OpBuilder::InsertPoint dcIP;
@@ -710,6 +710,8 @@ void privatizeSymbol(
// not emit the copy region when dealing with the SIMD directive.
if (dir && dir == llvm::omp::Directive::OMPD_simd)
emitCopyRegion = false;
+ assert((!isLinear || std::is_same_v<OpType, mlir::omp::PrivateClauseOp>) &&
+ "only omp.private has a `linear` type");
mlir::Value privVal = hsb.getAddr();
mlir::Type allocType = privVal.getType();
@@ -769,7 +771,8 @@ void privatizeSymbol(
auto moduleOp = firOpBuilder.getModule();
std::string privatizerPrefix =
converter.mangleName(*sym) +
- (emitCopyRegion ? "_firstprivate" : "_private");
+ (emitCopyRegion ? "_firstprivate"
+ : (isLinear ? "_linear" : "_private"));
if (forceHeapAllocation)
privatizerPrefix += "_heap";
auto uniquePrivatizerName = fir::getTypeAsString(
@@ -794,6 +797,7 @@ void privatizeSymbol(
firOpBuilder, symLoc, uniquePrivatizerName,
/*sym_visibility=*/nullptr, allocType,
emitCopyRegion ? mlir::omp::DataSharingClauseType::FirstPrivate
+ : isLinear ? mlir::omp::DataSharingClauseType::Linear
: mlir::omp::DataSharingClauseType::Private);
} else {
result =
@@ -892,6 +896,20 @@ void privatizeSymbol(
}
}
+ // A `linear` privatizer of a descriptor gives the address of its data.
+ if constexpr (std::is_same_v<OpType, mlir::omp::PrivateClauseOp>) {
+ if (isLinear && mlir::isa<fir::BaseBoxType>(allocType)) {
+ mlir::Block *block =
+ firOpBuilder.createBlock(&result.getDataAddressRegion(),
+ /*insertPt=*/{}, {argType}, {symLoc});
+ firOpBuilder.setInsertionPointToEnd(block);
+ mlir::Value box =
+ fir::LoadOp::create(firOpBuilder, symLoc, block->getArgument(0));
+ mlir::Value addr = fir::BoxAddrOp::create(firOpBuilder, symLoc, box);
+ mlir::omp::YieldOp::create(firOpBuilder, symLoc, addr);
+ }
+ }
+
return result;
}();
@@ -916,7 +934,7 @@ privatizeSymbol<mlir::omp::PrivateClauseOp, mlir::omp::PrivateClauseOps>(
const semantics::Symbol *symToPrivatize,
mlir::omp::PrivateClauseOps *clauseOps,
std::optional<llvm::omp::Directive> dir,
- bool forceHeapAllocationForPrivateDynamicArrays);
+ bool forceHeapAllocationForPrivateDynamicArrays, bool isLinear);
template void
privatizeSymbol<fir::LocalitySpecifierOp, fir::LocalitySpecifierOperands>(
@@ -927,6 +945,6 @@ privatizeSymbol<fir::LocalitySpecifierOp, fir::LocalitySpecifierOperands>(
const semantics::Symbol *symToPrivatize,
fir::LocalitySpecifierOperands *clauseOps,
std::optional<llvm::omp::Directive> dir,
- bool forceHeapAllocationForPrivateDynamicArrays);
+ bool forceHeapAllocationForPrivateDynamicArrays, bool isLinear);
} // end namespace Fortran::lower
diff --git a/flang/lib/Support/OpenMP-utils.cpp b/flang/lib/Support/OpenMP-utils.cpp
index b8db9e9bd72eb0..dc7e69ba1bba71 100644
--- a/flang/lib/Support/OpenMP-utils.cpp
+++ b/flang/lib/Support/OpenMP-utils.cpp
@@ -18,9 +18,10 @@ mlir::Block *genEntryBlock(mlir::OpBuilder &builder, const EntryBlockArgs &args,
llvm::SmallVector<mlir::Type> types;
llvm::SmallVector<mlir::Location> locs;
unsigned numVars = args.hasDeviceAddrVars.size() + args.hostEvalVars.size() +
- args.inReductionVars.size() + args.mapVars.size() + args.privVars.size() +
- args.reductionVars.size() + args.taskReductionVars.size() +
- args.useDeviceAddrVars.size() + args.useDevicePtrVars.size();
+ args.inReductionVars.size() + args.linearVars.size() +
+ args.mapVars.size() + args.privVars.size() + args.reductionVars.size() +
+ args.taskReductionVars.size() + args.useDeviceAddrVars.size() +
+ args.useDevicePtrVars.size();
types.reserve(numVars);
locs.reserve(numVars);
@@ -36,6 +37,7 @@ mlir::Block *genEntryBlock(mlir::OpBuilder &builder, const EntryBlockArgs &args,
extractTypeLoc(args.hasDeviceAddrVars);
extractTypeLoc(args.hostEvalVars);
extractTypeLoc(args.inReductionVars);
+ extractTypeLoc(args.linearVars);
extractTypeLoc(args.mapVars);
extractTypeLoc(args.privVars);
extractTypeLoc(args.reductionVars);
diff --git a/flang/test/Lower/OpenMP/Todo/linear-no-delayed-privatization.f90 b/flang/test/Lower/OpenMP/Todo/linear-no-delayed-privatization.f90
new file mode 100644
index 00000000000000..a550529c2f22f5
--- /dev/null
+++ b/flang/test/Lower/OpenMP/Todo/linear-no-delayed-privatization.f90
@@ -0,0 +1,10 @@
+! RUN: %not_todo_cmd %flang_fc1 -emit-hlfir -fopenmp -mmlir --enable-delayed-privatization=false -o - %s 2>&1 | FileCheck %s
+
+! CHECK: not yet implemented: LINEAR clause without delayed privatization
+subroutine linear_no_delayed_privatization()
+ integer :: i, x
+ !$omp simd linear(x)
+ do i = 1, 10
+ x = x + 1
+ end do
+end subroutine
diff --git a/flang/test/Lower/OpenMP/composite_simd_linear.f90 b/flang/test/Lower/OpenMP/composite_simd_linear.f90
index 35d4080f2041d9..8a4f11c08daa8a 100644
--- a/flang/test/Lower/OpenMP/composite_simd_linear.f90
+++ b/flang/test/Lower/OpenMP/composite_simd_linear.f90
@@ -8,10 +8,10 @@ subroutine do_simd
!CHECK: %[[CONST:.*]] = arith.constant 1 : i32
!CHECK: %{{.*}} = arith.constant 1 : i32
!CHECK: %[[IV_STEP:.*]] = arith.constant 1 : i32
-!DEFAULT: omp.wsloop linear(%[[X]]#0 : !fir.ref<i32> = {{.*}}) linear_var_types([i32]) {
-!OPENMP52: omp.wsloop linear(val(%[[X]]#0 : !fir.ref<i32> = {{.*}})) linear_var_types([i32]) {
-!DEFAULT: omp.simd linear(%[[I]]#0 : !fir.ref<i32> = %[[IV_STEP]] : i32) linear_var_types([i32]) {
-!OPENMP52: omp.simd linear(val(%[[I]]#0 : !fir.ref<i32> = %[[IV_STEP]] : i32)) linear_var_types([i32]) {
+!DEFAULT: omp.wsloop linear(@{{[^ ]+}} %[[X]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = {{.*}}) linear_var_types([i32]) {
+!OPENMP52: omp.wsloop linear(val(@{{[^ ]+}} %[[X]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = {{.*}})) linear_var_types([i32]) {
+!DEFAULT: omp.simd linear(@{{[^ ]+}} %[[I]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[IV_STEP]] : i32) linear_var_types([i32]) {
+!OPENMP52: omp.simd linear(val(@{{[^ ]+}} %[[I]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[IV_STEP]] : i32)) linear_var_types([i32]) {
!CHECK: }
!CHECK: } {omp.composite}
!CHECK: } {omp.composite}
@@ -50,10 +50,10 @@ subroutine distribute_parallel_do
!CHECK: %{{.*}} = arith.constant 1 : i32
!CHECK: %[[IV_STEP:.*]] = arith.constant 1 : i32
!CHECK: omp.distribute {
-!DEFAULT: omp.wsloop linear(%[[I]]#0 : !fir.ref<i32> = %[[CONST]] : i32) linear_var_types([i32]) {
-!OPENMP52: omp.wsloop linear(val(%[[I]]#0 : !fir.ref<i32> = %[[CONST]] : i32)) linear_var_types([i32]) {
-!DEFAULT: omp.simd linear(%[[I]]#0 : !fir.ref<i32> = %[[IV_STEP]] : i32) linear_var_types([i32]) {
-!OPENMP52: omp.simd linear(val(%[[I]]#0 : !fir.ref<i32> = %[[IV_STEP]] : i32)) linear_var_types([i32]) {
+!DEFAULT: omp.wsloop linear(@{{[^ ]+}} %[[I]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[CONST]] : i32) linear_var_types([i32]) {
+!OPENMP52: omp.wsloop linear(val(@{{[^ ]+}} %[[I]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[CONST]] : i32)) linear_var_types([i32]) {
+!DEFAULT: omp.simd linear(@{{[^ ]+}} %{{[^ ]+}} -> %{{[^ ]+}} : !fir.ref<i32> = %[[IV_STEP]] : i32) linear_var_types([i32]) {
+!OPENMP52: omp.simd linear(val(@{{[^ ]+}} %{{[^ ]+}} -> %{{[^ ]+}} : !fir.ref<i32> = %[[IV_STEP]] : i32)) linear_var_types([i32]) {
!$omp teams
!$omp distribute parallel do simd linear(i:1)
do i = 1, N
@@ -71,10 +71,10 @@ subroutine parallel_do
!CHECK: %[[LINEAR_STEP:.*]] = arith.constant 2 : i32
!CHECK: %{{.*}} = arith.constant 1 : i32
!CHECK: %[[IV_STEP:.*]] = arith.constant 1 : i32
-!DEFAULT: omp.wsloop linear(%[[X]]#0 : !fir.ref<i32> = %[[LINEAR_STEP]] : i32) linear_var_types([i32]) {
-!OPENMP52: omp.wsloop linear(val(%[[X]]#0 : !fir.ref<i32> = %[[LINEAR_STEP]] : i32)) linear_var_types([i32]) {
-!DEFAULT: omp.simd linear(%[[I]]#0 : !fir.ref<i32> = %[[IV_STEP]] : i32) linear_var_types([i32]) {
-!OPENMP52: omp.simd linear(val(%[[I]]#0 : !fir.ref<i32> = %[[IV_STEP]] : i32)) linear_var_types([i32]) {
+!DEFAULT: omp.wsloop linear(@{{[^ ]+}} %[[X]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[LINEAR_STEP]] : i32) linear_var_types([i32]) {
+!OPENMP52: omp.wsloop linear(val(@{{[^ ]+}} %[[X]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[LINEAR_STEP]] : i32)) linear_var_types([i32]) {
+!DEFAULT: omp.simd linear(@{{[^ ]+}} %[[I]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[IV_STEP]] : i32) linear_var_types([i32]) {
+!OPENMP52: omp.simd linear(val(@{{[^ ]+}} %[[I]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[IV_STEP]] : i32)) linear_var_types([i32]) {
integer :: x
!$omp parallel do simd linear(x:2)
do i = 1, N
@@ -92,8 +92,8 @@ subroutine teams_distribute
!CHECK: {{.*}} = arith.constant 1 : i32
!CHECK: %[[IV_STEP:.*]] = arith.constant 1 : i32
!CHECK: omp.distribute {
-!DEFAULT: omp.simd linear(%[[X]]#0 : !fir.ref<i32> = %[[LINEAR_STEP]] : i32, %[[I]]#0 : !fir.ref<i32> = %[[IV_STEP]] : i32) linear_var_types([i32, i32]) {
-!OPENMP52: omp.simd linear(val(%[[X]]#0 : !fir.ref<i32> = %[[LINEAR_STEP]] : i32), val(%[[I]]#0 : !fir.ref<i32> = %[[IV_STEP]] : i32)) linear_var_types([i32, i32]) {
+!DEFAULT: omp.simd linear(@{{[^ ]+}} %[[X]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[LINEAR_STEP]] : i32, @{{[^ ]+}} %[[I]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[IV_STEP]] : i32) linear_var_types([i32, i32]) {
+!OPENMP52: omp.simd linear(val(@{{[^ ]+}} %[[X]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[LINEAR_STEP]] : i32), val(@{{[^ ]+}} %[[I]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[IV_STEP]] : i32)) linear_var_types([i32, i32]) {
integer :: x
!$omp teams distribute simd linear(x)
do i = 1, N
@@ -111,10 +111,10 @@ subroutine teams_distribute_parallel_do
!CHECK: %{{.*}} = arith.constant 1 : i32
!CHECK: %[[IV_STEP:.*]] = arith.constant 1 : i32
!CHECK: omp.distribute {
-!DEFAULT: omp.wsloop linear(%[[X]]#0 : !fir.ref<i32> = %[[LINEAR_STEP]] : i32) linear_var_types([i32]) {
-!OPENMP52: omp.wsloop linear(val(%[[X]]#0 : !fir.ref<i32> = %[[LINEAR_STEP]] : i32)) linear_var_types([i32]) {
-!DEFAULT: omp.simd linear(%[[I]]#0 : !fir.ref<i32> = %[[IV_STEP]] : i32) linear_var_types([i32]) {
-!OPENMP52: omp.simd linear(val(%[[I]]#0 : !fir.ref<i32> = %[[IV_STEP]] : i32)) linear_var_types([i32]) {
+!DEFAULT: omp.wsloop linear(@{{[^ ]+}} %[[X]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[LINEAR_STEP]] : i32) linear_var_types([i32]) {
+!OPENMP52: omp.wsloop linear(val(@{{[^ ]+}} %[[X]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[LINEAR_STEP]] : i32)) linear_var_types([i32]) {
+!DEFAULT: omp.simd linear(@{{[^ ]+}} %[[I]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[IV_STEP]] : i32) linear_var_types([i32]) {
+!OPENMP52: omp.simd linear(val(@{{[^ ]+}} %[[I]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[IV_STEP]] : i32)) linear_var_types([i32]) {
integer :: x
!$omp teams distribute parallel do simd linear(x)
do i = 1, N
diff --git a/flang/test/Lower/OpenMP/compound-version-override.f90 b/flang/test/Lower/OpenMP/compound-version-override.f90
index 112ab98e4cd274..01a89c8ef9c32f 100644
--- a/flang/test/Lower/OpenMP/compound-version-override.f90
+++ b/flang/test/Lower/OpenMP/compound-version-override.f90
@@ -4,9 +4,10 @@
!CHECK: warning: IF clause is not allowed on DO SIMD directive in OpenMP v4.5, try -fopenmp-version=50
!CHECK: omp.wsloop {
-!CHECK: omp.simd if(%true) linear(%[[V0:[0-9]+]]#0 : !fir.ref<i32> = %c1_i32_0 : i32) linear_var_types([i32]) {
-!CHECK: omp.loop_nest (%arg2) : i32 = (%c1_i32) to (%{{[0-9]+}}) inclusive step (%c1_i32_0) {
-!CHECK: hlfir.assign %arg2 to %[[V0]]#0 : i32, !fir.ref<i32>
+!CHECK: omp.simd if(%true) linear(@{{[^ ]+}} %{{[0-9]+}}#0 -> %[[I_ARG:arg[0-9]+]] : !fir.ref<i32> = %c1_i32_0 : i32) linear_var_types([i32]) {
+!CHECK: omp.loop_nest (%[[IV:arg[0-9]+]]) : i32 = (%c1_i32) to (%{{[0-9]+}}) inclusive step (%c1_i32_0) {
+!CHECK: %[[V0:[0-9]+]]:2 = hlfir.declare %[[I_ARG]] uniq_name("_QFcompoundEi")
+!CHECK: hlfir.assign %[[IV]] to %[[V0]]#0 : i32, !fir.ref<i32>
!CHECK: %[[V1:[0-9]+]] = fir.load %[[V0]]#0 : !fir.ref<i32>
!CHECK: %[[V2:[0-9]+]] = fir.convert %[[V1]] : (i32) -> i64
!CHECK: %[[V3:[0-9]+]] = hlfir.designate %10#0 (%[[V2]]) : (!fir.box<!fir.array<?xf32>>, i64) -> !fir.ref<f32>
diff --git a/flang/test/Lower/OpenMP/distribute-parallel-do-simd.f90 b/flang/test/Lower/OpenMP/distribute-parallel-do-simd.f90
index 44b6e342b99f03..067c6f700ee901 100644
--- a/flang/test/Lower/OpenMP/distribute-parallel-do-simd.f90
+++ b/flang/test/Lower/OpenMP/distribute-parallel-do-simd.f90
@@ -99,8 +99,8 @@ subroutine distribute_parallel_do_simd_simdlen()
! CHECK: omp.parallel {
! CHECK: omp.distribute {
! CHECK-NEXT: omp.wsloop {
- ! DEFAULT-NEXT: omp.simd linear({{.*}}) linear_var_types([i32]) simdlen(4) {
- ! OPENMP52-NEXT: omp.simd linear(val({{.*}})) linear_var_types([i32]) simdlen(4) {
+ ! DEFAULT-NEXT: omp.simd simdlen(4) linear({{.*}}) linear_var_types([i32]) {
+ ! OPENMP52-NEXT: omp.simd simdlen(4) linear(val({{.*}})) linear_var_types([i32]) {
! CHECK-NEXT: omp.loop_nest
!$omp distribute parallel do simd simdlen(4)
do index_ = 1, 10
@@ -124,9 +124,9 @@ subroutine distribute_parallel_do_simd_private()
! CHECK: omp.parallel {
! CHECK: omp.distribute {
! CHECK-NEXT: omp.wsloop {
- ! DEFAULT-NEXT: omp.simd linear(%{{.*}}) linear_var_types([i32]) private(@{{.*}} %[[X]]#0 -> %[[X_ARG:[^:]+]]
+ ! DEFAULT-NEXT: omp.simd linear({{.*}}) linear_var_types([i32]) private(@{{.*}} %[[X]]#0 -> %[[X_ARG:[^:]+]]
! DEFAULT-SAME: : !fir.ref<i64>) {
- ! OPENMP52-NEXT: omp.simd linear(val(%{{.*}})) linear_var_types([i32]) private(@{{.*}} %[[X]]#0 -> %[[X_ARG:[^:]+]]
+ ! OPENMP52-NEXT: omp.simd linear(val({{.*}})) linear_var_types([i32]) private(@{{.*}} %[[X]]#0 -> %[[X_ARG:[^:]+]]
! OPENMP52-SAME: : !fir.ref<i64>) {
! CHECK-NEXT: omp.loop_nest
! CHECK: %[[X_PRIV:.*]]:2 = hlfir.declare %[[X_ARG]]
diff --git a/flang/test/Lower/OpenMP/distribute-simd.f90 b/flang/test/Lower/OpenMP/distribute-simd.f90
index d01cda1083d81d..69b1a507ead788 100644
--- a/flang/test/Lower/OpenMP/distribute-simd.f90
+++ b/flang/test/Lower/OpenMP/distribute-simd.f90
@@ -66,12 +66,14 @@ subroutine distribute_simd_private()
!$omp teams
! CHECK: omp.distribute
! CHECK: omp.simd
+ ! CHECK-SAME: linear(@{{.*}} %[[INDEX]]#0 -> %[[INDEX_ARG:[^ ]+]] :
! CHECK-SAME: private(@[[PRIV_BOX_SYM:.*]] %{{.*}} -> %[[PRIV_BOX:.*]] : !fir.ref<!fir.box<!fir.heap<i32>>>)
! CHECK-NEXT: omp.loop_nest (%[[IVAR:.*]]) : i32
!$omp distribute simd private(tmp)
do index_ = 1, 10
+ ! CHECK: %[[INDEX_PRIV:.*]]:2 = hlfir.declare %[[INDEX_ARG]]
! CHECK: %[[PRIV_BOX_DECL:.*]]:2 = hlfir.declare %[[PRIV_BOX]]
- ! CHECK: hlfir.assign %[[IVAR]] to %[[INDEX]]#0
+ ! CHECK: hlfir.assign %[[IVAR]] to %[[INDEX_PRIV]]#0
end do
!$omp end distribute simd
!$omp end teams
diff --git a/flang/test/Lower/OpenMP/indirect-reference-privatization.f90 b/flang/test/Lower/OpenMP/indirect-reference-privatization.f90
index 5f80f25152b53b..9ff831bd4484c9 100644
--- a/flang/test/Lower/OpenMP/indirect-reference-privatization.f90
+++ b/flang/test/Lower/OpenMP/indirect-reference-privatization.f90
@@ -3,7 +3,7 @@
!CHECK-LABEL: func @_QPparallel_simd
!CHECK: omp.parallel private(@_QFparallel_simdEk2_private_i32 {{.*}} -> %[[ARG:.*]] : !fir.ref<i32>)
!CHECK: %[[PRIV_K2:.*]]:2 = hlfir.declare %[[ARG]] uniq_name("_QFparallel_simdEk2")
-!CHECK: omp.simd linear(%[[PRIV_K2]]#0 {{.*}})
+!CHECK: omp.simd linear(@{{[^ ]+}} %[[PRIV_K2]]#0 -> %{{[^ ]+}} {{.*}})
subroutine parallel_simd
integer :: k1, k2
@@ -25,7 +25,7 @@ end subroutine parallel_simd
!CHECK-LABEL: func @_QPtask_simd
!CHECK: omp.task private(@_QFtask_simdEk_firstprivate_i32 %{{.*}})
!CHECK: %[[PRIV_K:.*]]:2 = hlfir.declare %{{.*}} uniq_name("_QFtask_simdEk")
-!CHECK: omp.simd linear(%[[PRIV_K]]#0 : !fir.ref<i32> {{.*}})
+!CHECK: omp.simd linear(@{{[^ ]+}} %[[PRIV_K]]#0 -> %{{[^ ]+}} : !fir.ref<i32> {{.*}})
subroutine task_simd
integer :: k
diff --git a/flang/test/Lower/OpenMP/linear-allocatable.f90 b/flang/test/Lower/OpenMP/linear-allocatable.f90
new file mode 100644
index 00000000000000..b6b62f5cf6872e
--- /dev/null
+++ b/flang/test/Lower/OpenMP/linear-allocatable.f90
@@ -0,0 +1,113 @@
+! LINEAR on an allocatable uses a `linear` omp.private with a data_address.
+
+! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
+! RUN: bbc -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
+
+! CHECK-LABEL: omp.private {type = linear} @_QFsimd_linear_allocatableEa_linear_box_heap_i32 : !fir.box<!fir.heap<i32>> init {
+! CHECK: fir.allocmem i32
+! CHECK: } data_address {
+! CHECK: ^bb0(%[[ARG:.*]]: !fir.ref<!fir.box<!fir.heap<i32>>>):
+! CHECK: %[[BOX:.*]] = fir.load %[[ARG]] : !fir.ref<!fir.box<!fir.heap<i32>>>
+! CHECK: %[[ADDR:.*]] = fir.box_addr %[[BOX]] : (!fir.box<!fir.heap<i32>>) -> !fir.heap<i32>
+! CHECK: omp.yield(%[[ADDR]] : !fir.heap<i32>)
+! CHECK: } dealloc {
+! CHECK: fir.freemem
+
+! CHECK-LABEL: func.func @_QPsimd_linear_allocatable
+! CHECK: %[[A:.*]]:2 = hlfir.declare %{{.*}} uniq_name("_QFsimd_linear_allocatableEa") fortran_attrs<allocatable>
+! CHECK: omp.simd linear(@_QFsimd_linear_allocatableEa_linear_box_heap_i32 %[[A]]#0 -> %[[ARG:.*]] : !fir.ref<!fir.box<!fir.heap<i32>>> = %{{.*}} : i32,
+! CHECK-SAME: linear_var_types([i32
+! CHECK: omp.loop_nest
+! CHECK: %[[PRIV_A:.*]]:2 = hlfir.declare %[[ARG]] uniq_name("_QFsimd_linear_allocatableEa") fortran_attrs<allocatable>
+! CHECK: hlfir.assign %{{.*}} to %[[PRIV_A]]#0 realloc : i32, !fir.ref<!fir.box<!fir.heap<i32>>>
+! CHECK-NEXT: omp.yield
+subroutine simd_linear_allocatable()
+ integer, allocatable :: a
+ integer :: i
+ allocate(a)
+ a = 0
+ !$omp simd linear(a)
+ do i = 1, 2
+ a = 2
+ end do
+ !$omp end simd
+end subroutine
+
+! CHECK-LABEL: func.func @_QPdo_linear_allocatable
+! CHECK: %[[A:.*]]:2 = hlfir.declare %{{.*}} uniq_name("_QFdo_linear_allocatableEa") fortran_attrs<allocatable>
+! CHECK: omp.parallel
+! CHECK: omp.wsloop linear(@_QFdo_linear_allocatableEa_linear_box_heap_i32 %[[A]]#0 -> %[[ARG:.*]] : !fir.ref<!fir.box<!fir.heap<i32>>> = %{{.*}} : i32)
+! CHECK: omp.loop_nest
+! CHECK: hlfir.declare %[[ARG]] uniq_name("_QFdo_linear_allocatableEa") fortran_attrs<allocatable>
+subroutine do_linear_allocatable()
+ integer, allocatable :: a
+ integer :: i
+ allocate(a)
+ a = 0
+ !$omp parallel do linear(a:2)
+ do i = 1, 10
+ a = a + 2
+ end do
+ !$omp end parallel do
+end subroutine
+
+! Composite: the linear clause is on omp.wsloop.
+! CHECK-LABEL: func.func @_QPdo_simd_linear_allocatable
+! CHECK: %[[A:.*]]:2 = hlfir.declare %{{.*}} uniq_name("_QFdo_simd_linear_allocatableEa") fortran_attrs<allocatable>
+! CHECK: omp.wsloop linear(@_QFdo_simd_linear_allocatableEa_linear_box_heap_i64 %[[A]]#0 -> %[[ARG:.*]] : !fir.ref<!fir.box<!fir.heap<i64>>> = %{{.*}} : i64)
+! CHECK-NEXT: omp.simd linear(@_QFdo_simd_linear_allocatableEi_linear_i32
+! CHECK: omp.loop_nest
+! CHECK: hlfir.declare %[[ARG]] uniq_name("_QFdo_simd_linear_allocatableEa") fortran_attrs<allocatable>
+subroutine do_simd_linear_allocatable()
+ integer(8), allocatable :: a
+ integer :: i
+ allocate(a)
+ a = 0
+ !$omp do simd linear(a)
+ do i = 1, 10
+ a = a + 1
+ end do
+ !$omp end do simd
+end subroutine
+
+! An allocatable loop iteration variable.
+! CHECK-LABEL: func.func @_QPallocatable_iv
+! CHECK: %[[I:.*]]:2 = hlfir.declare %{{.*}} uniq_name("_QFallocatable_ivEi") fortran_attrs<allocatable>
+! CHECK: omp.simd linear(@_QFallocatable_ivEi_linear_box_heap_i32 %[[I]]#0 -> %[[ARG:.*]] : !fir.ref<!fir.box<!fir.heap<i32>>> = %{{.*}} : i32,
+! CHECK: omp.loop_nest (%[[IV:.*]]) :
+! CHECK: %[[PRIV_I:.*]]:2 = hlfir.declare %[[ARG]] uniq_name("_QFallocatable_ivEi") fortran_attrs<allocatable>
+! CHECK: %[[BOX:.*]] = fir.load %[[PRIV_I]]#0
+! CHECK: %[[ADDR:.*]] = fir.box_addr %[[BOX]]
+! CHECK: hlfir.assign %{{.*}} to %[[ADDR]] : i32, !fir.heap<i32>
+subroutine allocatable_iv()
+ integer, allocatable :: i
+ integer :: seen(3), k
+ allocate(i)
+ k = 1
+ !$omp simd linear(i:2) linear(k)
+ do i = 1, 5, 2
+ seen(k) = i
+ k = k + 1
+ end do
+ !$omp end simd
+end subroutine
+
+! The original is used after the loop.
+! CHECK-LABEL: func.func @_QPoriginal_used_after_loop
+! CHECK: %[[A:.*]]:2 = hlfir.declare %{{.*}} uniq_name("_QForiginal_used_after_loopEa") fortran_attrs<allocatable>
+! CHECK: omp.simd linear
+! CHECK: omp.yield
+! CHECK: }
+! CHECK: hlfir.assign %{{.*}} to %[[A]]#0 realloc : i32, !fir.ref<!fir.box<!fir.heap<i32>>>
+subroutine original_used_after_loop()
+ integer, allocatable :: a
+ integer :: i
+ allocate(a)
+ a = 0
+ !$omp simd linear(a)
+ do i = 1, 2
+ a = 2
+ end do
+ !$omp end simd
+ a = 5
+end subroutine
diff --git a/flang/test/Lower/OpenMP/linear_modifier.f90 b/flang/test/Lower/OpenMP/linear_modifier.f90
index 47dfe92aca952b..1d3c507c617a27 100644
--- a/flang/test/Lower/OpenMP/linear_modifier.f90
+++ b/flang/test/Lower/OpenMP/linear_modifier.f90
@@ -9,8 +9,8 @@
subroutine simple_linear
implicit none
integer :: x, y, i
- !OPENMP52: omp.wsloop linear(val(%[[X]]#0 : !fir.ref<i32> = %[[const]] : i32)) {{.*}}
- !OPENMP45: omp.wsloop linear(%[[X]]#0 : !fir.ref<i32> = %[[const]] : i32) {{.*}}
+ !OPENMP52: omp.wsloop linear(val(@{{[^ ]+}} %[[X]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[const]] : i32)) {{.*}}
+ !OPENMP45: omp.wsloop linear(@{{[^ ]+}} %[[X]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[const]] : i32) {{.*}}
!$omp do linear(x)
do i = 1, 10
y = x + 2
@@ -25,8 +25,8 @@ subroutine linear_step
implicit none
integer :: x, y, i
!CHECK: %[[const:.*]] = arith.constant 4 : i32
- !OPENMP52: omp.wsloop linear(val(%[[X]]#0 : !fir.ref<i32> = %[[const]] : i32)) {{.*}}
- !OPENMP45: omp.wsloop linear(%[[X]]#0 : !fir.ref<i32> = %[[const]] : i32) {{.*}}
+ !OPENMP52: omp.wsloop linear(val(@{{[^ ]+}} %[[X]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[const]] : i32)) {{.*}}
+ !OPENMP45: omp.wsloop linear(@{{[^ ]+}} %[[X]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[const]] : i32) {{.*}}
!$omp do linear(x:4)
do i = 1, 10
y = x + 2
@@ -41,10 +41,10 @@ subroutine do_simd_linear
!CHECK: %[[CONST:.*]] = arith.constant 1 : i32
!CHECK: %{{.*}} = arith.constant 1 : i32
!CHECK: %[[IV_STEP:.*]] = arith.constant 1 : i32
-!OPENMP52: omp.wsloop linear(val(%[[X]]#0 : !fir.ref<i32> = %[[CONST]] : i32)) linear_var_types([i32]) {
-!OPENMP45: omp.wsloop linear(%[[X]]#0 : !fir.ref<i32> = %[[CONST]] : i32) linear_var_types([i32]) {
-!OPENMP52: omp.simd linear(val(%[[I]]#0 : !fir.ref<i32> = %[[IV_STEP]] : i32)) linear_var_types([i32]) {
-!OPENMP45: omp.simd linear(%[[I]]#0 : !fir.ref<i32> = %[[IV_STEP]] : i32) linear_var_types([i32]) {
+!OPENMP52: omp.wsloop linear(val(@{{[^ ]+}} %[[X]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[CONST]] : i32)) linear_var_types([i32]) {
+!OPENMP45: omp.wsloop linear(@{{[^ ]+}} %[[X]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[CONST]] : i32) linear_var_types([i32]) {
+!OPENMP52: omp.simd linear(val(@{{[^ ]+}} %[[I]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[IV_STEP]] : i32)) linear_var_types([i32]) {
+!OPENMP45: omp.simd linear(@{{[^ ]+}} %[[I]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[IV_STEP]] : i32) linear_var_types([i32]) {
integer :: x
!$omp do simd linear(x:1)
do i = 1, 10
diff --git a/flang/test/Lower/OpenMP/metadirective-loop.f90 b/flang/test/Lower/OpenMP/metadirective-loop.f90
index e96c3fe65b0f86..81dfac22c9ae77 100644
--- a/flang/test/Lower/OpenMP/metadirective-loop.f90
+++ b/flang/test/Lower/OpenMP/metadirective-loop.f90
@@ -64,15 +64,16 @@ subroutine test_do(n, a)
! CHECK: %[[SIMD_I:.*]]:2 = hlfir.declare {{.*}}_QFtest_simdEi
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK: omp.simd linear(
-! CHECK-SAME: val(%[[SIMD_I]]#0 : !fir.ref<i32> =
+! CHECK-SAME: val(@{{[^ ]+}} %[[SIMD_I]]#0 -> %[[SIMD_I_ARG:[^ ]+]] : !fir.ref<i32> =
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK: omp.loop_nest (%[[SIMD_IV:.*]]) :
+! CHECK: %[[SIMD_I_PRIV:.*]]:2 = hlfir.declare %[[SIMD_I_ARG]]
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
-! CHECK: hlfir.assign %[[SIMD_IV]] to %[[SIMD_I]]#0
+! CHECK: hlfir.assign %[[SIMD_IV]] to %[[SIMD_I_PRIV]]#0
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
-! CHECK: %[[SIMD_VALUE:.*]] = fir.load %[[SIMD_I]]#0
+! CHECK: %[[SIMD_VALUE:.*]] = fir.load %[[SIMD_I_PRIV]]#0
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
-! CHECK: %[[SIMD_INDEX:.*]] = fir.load %[[SIMD_I]]#0
+! CHECK: %[[SIMD_INDEX:.*]] = fir.load %[[SIMD_I_PRIV]]#0
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK: %[[SIMD_INDEX_I64:.*]] =
! CHECK-SAME: fir.convert %[[SIMD_INDEX]]
@@ -109,18 +110,19 @@ subroutine test_simd(n, a, after)
! CHECK: omp.wsloop
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK: omp.simd linear(
-! CHECK-SAME: val(%[[DO_SIMD_I]]#0 : !fir.ref<i32> =
+! CHECK-SAME: val(@{{[^ ]+}} %[[DO_SIMD_I]]#0 -> %[[DO_SIMD_I_ARG:[^ ]+]] : !fir.ref<i32> =
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK: omp.loop_nest (%[[DO_SIMD_IV:.*]]) :
+! CHECK: %[[DO_SIMD_I_PRIV:.*]]:2 = hlfir.declare %[[DO_SIMD_I_ARG]]
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK: hlfir.assign %[[DO_SIMD_IV]] to
-! CHECK-SAME: %[[DO_SIMD_I]]#0
+! CHECK-SAME: %[[DO_SIMD_I_PRIV]]#0
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK: %[[DO_SIMD_VALUE:.*]] =
-! CHECK-SAME: fir.load %[[DO_SIMD_I]]#0
+! CHECK-SAME: fir.load %[[DO_SIMD_I_PRIV]]#0
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK: %[[DO_SIMD_INDEX:.*]] =
-! CHECK-SAME: fir.load %[[DO_SIMD_I]]#0
+! CHECK-SAME: fir.load %[[DO_SIMD_I_PRIV]]#0
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK: %[[DO_SIMD_INDEX_I64:.*]] =
! CHECK-SAME: fir.convert %[[DO_SIMD_INDEX]]
@@ -728,11 +730,12 @@ pure logical function metadirective_identity(value)
! CHECK: fir.if {{.*}} {
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK: omp.simd linear(
-! CHECK-SAME: val(%[[PURE_I]]#0 : !fir.ref<i32> =
+! CHECK-SAME: val(@{{[^ ]+}} %[[PURE_I]]#0 -> %[[PURE_I_ARG:[^ ]+]] : !fir.ref<i32> =
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK: omp.loop_nest (%[[PURE_SIMD_IV:.*]]) :
+! CHECK: %[[PURE_I_PRIV:.*]]:2 = hlfir.declare %[[PURE_I_ARG]]
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
-! CHECK: hlfir.assign %[[PURE_SIMD_IV]] to %[[PURE_I]]#0
+! CHECK: hlfir.assign %[[PURE_SIMD_IV]] to %[[PURE_I_PRIV]]#0
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK-NOT: {{^ *}}}
! CHECK: %[[PURE_SIMD_ELEMENT:.*]] = hlfir.designate
@@ -829,18 +832,19 @@ subroutine test_pure_runtime_conditions(flag, n, a)
! CHECK: } else {
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK: omp.simd linear(
-! CHECK-SAME: val(%[[DYNAMIC_I]]#0 : !fir.ref<i32> =
+! CHECK-SAME: val(@{{[^ ]+}} %[[DYNAMIC_I]]#0 -> %[[DYNAMIC_I_ARG:[^ ]+]] : !fir.ref<i32> =
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK: omp.loop_nest (%[[DYNAMIC_SIMD_IV:.*]]) :
+! CHECK: %[[DYNAMIC_I_PRIV:.*]]:2 = hlfir.declare %[[DYNAMIC_I_ARG]]
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK: hlfir.assign %[[DYNAMIC_SIMD_IV]] to
-! CHECK-SAME: %[[DYNAMIC_I]]#0
+! CHECK-SAME: %[[DYNAMIC_I_PRIV]]#0
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK: %[[DYNAMIC_SIMD_VALUE:.*]] =
-! CHECK-SAME: fir.load %[[DYNAMIC_I]]#0
+! CHECK-SAME: fir.load %[[DYNAMIC_I_PRIV]]#0
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK: %[[DYNAMIC_SIMD_INDEX:.*]] =
-! CHECK-SAME: fir.load %[[DYNAMIC_I]]#0
+! CHECK-SAME: fir.load %[[DYNAMIC_I_PRIV]]#0
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK: %[[DYNAMIC_SIMD_INDEX_I64:.*]] =
! CHECK-SAME: fir.convert %[[DYNAMIC_SIMD_INDEX]]
@@ -1283,16 +1287,17 @@ subroutine test_dynamic_intervening_compiler_directives(flag, n, a)
! CHECK: } else {
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK: omp.simd linear(
-! CHECK-SAME: val(%[[DC_I]]#0 : !fir.ref<i32> =
+! CHECK-SAME: val(@{{[^ ]+}} %[[DC_I]]#0 -> %[[DC_I_ARG:[^ ]+]] : !fir.ref<i32> =
! CHECK-SAME: %[[DC_FB_STEP:[^ ]+]] : i32))
! CHECK-SAME: linear_var_types([i32]) {
! CHECK-NOT: Ei_private
! CHECK-NOT: Ej_private
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK: omp.loop_nest (%[[DC_FB_IV:.*]]) : i32
+! CHECK: %[[DC_I_PRIV:.*]]:2 = hlfir.declare %[[DC_I_ARG]]
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK-NOT: {{^ *}}}
-! CHECK: hlfir.assign %[[DC_FB_IV]] to %[[DC_I]]#0
+! CHECK: hlfir.assign %[[DC_FB_IV]] to %[[DC_I_PRIV]]#0
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK: fir.do_loop %[[DC_INNER_IV:.*]] =
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
@@ -1300,12 +1305,12 @@ subroutine test_dynamic_intervening_compiler_directives(flag, n, a)
! CHECK: fir.store %[[DC_INNER_IV]] to %[[DC_J]]#0
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK-NOT: {{^ *}}}
-! CHECK: %[[DC_FB_I_VALUE:.*]] = fir.load %[[DC_I]]#0
+! CHECK: %[[DC_FB_I_VALUE:.*]] = fir.load %[[DC_I_PRIV]]#0
! CHECK: %[[DC_FB_J_VALUE:.*]] = fir.load %[[DC_J]]#0
! CHECK: %[[DC_FB_SUM:.*]] = arith.addi
! CHECK-SAME: %[[DC_FB_I_VALUE]], %[[DC_FB_J_VALUE]]
! CHECK: %[[DC_FB_J_INDEX:.*]] = fir.load %[[DC_J]]#0
-! CHECK: %[[DC_FB_I_INDEX:.*]] = fir.load %[[DC_I]]#0
+! CHECK: %[[DC_FB_I_INDEX:.*]] = fir.load %[[DC_I_PRIV]]#0
! CHECK-NOT: {{omp\.(wsloop|simd|loop_nest)|fir\.do_loop}}
! CHECK-NOT: {{^ *}}}
! CHECK: %[[DC_FB_ELEMENT:.*]] = hlfir.designate
diff --git a/flang/test/Lower/OpenMP/simd-linear.f90 b/flang/test/Lower/OpenMP/simd-linear.f90
index 1149bdb3002331..ef2fcafff39bf3 100644
--- a/flang/test/Lower/OpenMP/simd-linear.f90
+++ b/flang/test/Lower/OpenMP/simd-linear.f90
@@ -16,9 +16,9 @@
subroutine simple_linear
implicit none
integer :: x, y, i
- !CHECK: omp.simd linear(val(%[[X]]#0 : !fir.ref<i32> = %[[const]] : i32)) {{.*}}
+ !CHECK: omp.simd linear(val(@{{[^ ]+}} %[[X]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[const]] : i32)) {{.*}}
- !IMPLICIT: omp.simd linear(%[[X]]#0 : !fir.ref<i32> = %[[const]] : i32, %[[I]]#0 : !fir.ref<i32> = %{{.*}} : i32) {{.*}}
+ !IMPLICIT: omp.simd linear(@{{[^ ]+}} %[[X]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[const]] : i32, @{{[^ ]+}} %[[I]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %{{.*}} : i32) {{.*}}
!$omp simd linear(x)
do i = 1, 10
end do
@@ -39,9 +39,9 @@ subroutine linear_step
implicit none
integer :: x, y, i
!CHECK: %[[const:.*]] = arith.constant 4 : i32
- !CHECK: omp.simd linear(val(%[[X]]#0 : !fir.ref<i32> = %[[const]] : i32)) {{.*}}
+ !CHECK: omp.simd linear(val(@{{[^ ]+}} %[[X]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[const]] : i32)) {{.*}}
- !IMPLICIT: omp.simd linear(%[[X]]#0 : !fir.ref<i32> = %[[const]] : i32, %[[I]]#0 : !fir.ref<i32> = %{{.*}} : i32) {{.*}}
+ !IMPLICIT: omp.simd linear(@{{[^ ]+}} %[[X]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[const]] : i32, @{{[^ ]+}} %[[I]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %{{.*}} : i32) {{.*}}
!$omp simd linear(x:4)
do i = 1, 10
end do
@@ -71,9 +71,9 @@ subroutine linear_expr
!IMPLICIT: %[[const:.*]] = arith.constant 4 : i32
!IMPLICIT: %[[LINEAR_EXPR:.*]] = arith.addi %[[LOAD_A]], %[[const]] : i32
- !CHECK: omp.simd linear(val(%[[X]]#0 : !fir.ref<i32> = %[[LINEAR_EXPR]] : i32)) {{.*}}
+ !CHECK: omp.simd linear(val(@{{[^ ]+}} %[[X]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[LINEAR_EXPR]] : i32)) {{.*}}
- !IMPLICIT: omp.simd linear(%[[X]]#0 : !fir.ref<i32> = %[[LINEAR_EXPR]] : i32, %[[I]]#0 : !fir.ref<i32> = {{.*}} : i32) {{.*}}
+ !IMPLICIT: omp.simd linear(@{{[^ ]+}} %[[X]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[LINEAR_EXPR]] : i32, @{{[^ ]+}} %[[I]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = {{.*}} : i32) {{.*}}
!$omp simd linear(x:a+4)
do i = 1, 10
end do
@@ -96,8 +96,8 @@ subroutine non_i32_type
!IMPLICIT: %[[STEP_VAR_CONST:.*]] = arith.constant 1 : i32
integer(kind=8)::j=0
- !CHECK: omp.simd linear(val(%[[J_DECLARE]]#0 : !fir.ref<i64> = %[[CONST]] : i64)) {{.*}}
- !IMPLICIT: omp.simd linear(%[[J_DECLARE]]#0 : !fir.ref<i64> = %[[CONST]] : i64, %[[I_DECLARE]]#0 : !fir.ref<i32> = %[[STEP_VAR_CONST]] : i32)
+ !CHECK: omp.simd linear(val(@{{[^ ]+}} %[[J_DECLARE]]#0 -> %{{[^ ]+}} : !fir.ref<i64> = %[[CONST]] : i64)) {{.*}}
+ !IMPLICIT: omp.simd linear(@{{[^ ]+}} %[[J_DECLARE]]#0 -> %{{[^ ]+}} : !fir.ref<i64> = %[[CONST]] : i64, @{{[^ ]+}} %[[I_DECLARE]]#0 -> %{{[^ ]+}} : !fir.ref<i32> = %[[STEP_VAR_CONST]] : i32)
!$omp simd linear(j:1_8)
do i = 1,10
end do
diff --git a/flang/test/Lower/OpenMP/wsloop-linear.f90 b/flang/test/Lower/OpenMP/wsloop-linear.f90
index a9fcf56175027f..886ec1426c5933 100644
--- a/flang/test/Lower/OpenMP/wsloop-linear.f90
+++ b/flang/test/Lower/OpenMP/wsloop-linear.f90
@@ -10,10 +10,11 @@
subroutine simple_linear
implicit none
integer :: x, y, i
- !DEFAULT: omp.wsloop linear(%[[X]]#0 : !fir.ref<i32> = %[[const]] : i32) {{.*}}
- !OPENMP52: omp.wsloop linear(val(%[[X]]#0 : !fir.ref<i32> = %[[const]] : i32)) {{.*}}
+ !DEFAULT: omp.wsloop linear(@{{[^ ]+}} %[[X]]#0 -> %[[X_ARG:[^ ]+]] : !fir.ref<i32> = %[[const]] : i32) {{.*}}
+ !OPENMP52: omp.wsloop linear(val(@{{[^ ]+}} %[[X]]#0 -> %[[X_ARG:[^ ]+]] : !fir.ref<i32> = %[[const]] : i32)) {{.*}}
!$omp do linear(x)
- !CHECK: %[[LOAD:.*]] = fir.load %[[X]]#0 : !fir.ref<i32>
+ !CHECK: %[[X_PRIV:.*]]:2 = hlfir.declare %[[X_ARG]]
+ !CHECK: %[[LOAD:.*]] = fir.load %[[X_PRIV]]#0 : !fir.ref<i32>
!CHECK: %[[const:.*]] = arith.constant 2 : i32
!CHECK: %[[RESULT:.*]] = arith.addi %[[LOAD]], %[[const]] : i32
do i = 1, 10
@@ -30,10 +31,11 @@ subroutine linear_step
implicit none
integer :: x, y, i
!CHECK: %[[const:.*]] = arith.constant 4 : i32
- !DEFAULT: omp.wsloop linear(%[[X]]#0 : !fir.ref<i32> = %[[const]] : i32) {{.*}}
- !OPENMP52: omp.wsloop linear(val(%[[X]]#0 : !fir.ref<i32> = %[[const]] : i32)) {{.*}}
+ !DEFAULT: omp.wsloop linear(@{{[^ ]+}} %[[X]]#0 -> %[[X_ARG:[^ ]+]] : !fir.ref<i32> = %[[const]] : i32) {{.*}}
+ !OPENMP52: omp.wsloop linear(val(@{{[^ ]+}} %[[X]]#0 -> %[[X_ARG:[^ ]+]] : !fir.ref<i32> = %[[const]] : i32)) {{.*}}
!$omp do linear(x:4)
- !CHECK: %[[LOAD:.*]] = fir.load %[[X]]#0 : !fir.ref<i32>
+ !CHECK: %[[X_PRIV:.*]]:2 = hlfir.declare %[[X_ARG]]
+ !CHECK: %[[LOAD:.*]] = fir.load %[[X_PRIV]]#0 : !fir.ref<i32>
!CHECK: %[[const:.*]] = arith.constant 2 : i32
!CHECK: %[[RESULT:.*]] = arith.addi %[[LOAD]], %[[const]] : i32
do i = 1, 10
@@ -53,8 +55,8 @@ subroutine linear_expr
!CHECK: %[[LOAD_A:.*]] = fir.load %[[A]]#0 : !fir.ref<i32>
!CHECK: %[[const:.*]] = arith.constant 4 : i32
!CHECK: %[[LINEAR_EXPR:.*]] = arith.addi %[[LOAD_A]], %[[const]] : i32
- !DEFAULT: omp.wsloop linear(%[[X]]#0 : !fir.ref<i32> = %[[LINEAR_EXPR]] : i32) {{.*}}
- !OPENMP52: omp.wsloop linear(val(%[[X]]#0 : !fir.ref<i32> = %[[LINEAR_EXPR]] : i32)) {{.*}}
+ !DEFAULT: omp.wsloop linear(@{{[^ ]+}} %[[X]]#0 -> %[[X_ARG:[^ ]+]] : !fir.ref<i32> = %[[LINEAR_EXPR]] : i32) {{.*}}
+ !OPENMP52: omp.wsloop linear(val(@{{[^ ]+}} %[[X]]#0 -> %[[X_ARG:[^ ]+]] : !fir.ref<i32> = %[[LINEAR_EXPR]] : i32)) {{.*}}
!$omp do linear(x:a+4)
do i = 1, 10
y = x + 2
diff --git a/flang/test/Lower/OpenMP/wsloop-simd.f90 b/flang/test/Lower/OpenMP/wsloop-simd.f90
index 61b4d9e0dd7ddc..639e6e4b6ef41b 100644
--- a/flang/test/Lower/OpenMP/wsloop-simd.f90
+++ b/flang/test/Lower/OpenMP/wsloop-simd.f90
@@ -92,12 +92,14 @@ subroutine do_simd_private()
! CHECK: %[[I_DECL:.*]]:2 = hlfir.declare %{{.*}} uniq_name("_QFdo_simd_privateEi")
! CHECK: omp.wsloop
! CHECK-NEXT: omp.simd
+ ! CHECK-SAME: linear(@{{.*}} %[[I_DECL]]#0 -> %[[I_ARG:[^ ]+]] :
! CHECK-SAME: private(@[[PRIV_BOX_SYM:.*]] %{{.*}} -> %[[PRIV_BOX:.*]] : !fir.ref<!fir.box<!fir.heap<i32>>>)
! CHECK-NEXT: omp.loop_nest (%[[IVAR:.*]]) : i32
!$omp do simd private(tmp)
do i=1, 10
+ ! CHECK: %[[I_PRIV:.*]]:2 = hlfir.declare %[[I_ARG]]
! CHECK: %[[PRIV_BOX_DECL:.*]]:2 = hlfir.declare %[[PRIV_BOX]]
- ! CHECK: hlfir.assign %[[IVAR]] to %[[I_DECL]]#0
+ ! CHECK: hlfir.assign %[[IVAR]] to %[[I_PRIV]]#0
! CHECK: %[[PRIV_BOX_LOAD:.*]] = fir.load %[[PRIV_BOX_DECL]]
! CHECK: hlfir.assign %{{.*}} to %[[PRIV_BOX_DECL]]#0
! CHECK: omp.yield
@@ -111,7 +113,7 @@ subroutine do_simd_lastprivate_firstprivate()
! CHECK: omp.wsloop
! CHECK-SAME: private(@[[FIRSTPRIVATE_A_SYM:.*]] %{{.*}} -> %[[FIRSTPRIVATE_A:.*]] : !fir.ref<i32>)
! CHECK-NEXT: omp.simd
- ! CHECK-SAME: linear({{.*}}#0 : !fir.ref<i32> = %{{[^:]*}} : i32)
+ ! CHECK-SAME: linear(@{{.*}}#0 -> %{{[^ ]+}} : !fir.ref<i32> = %{{[^:]*}} : i32)
! CHECK-SAME: private(@[[PRIVATE_A_SYM:.*]] %{{.*}} -> %[[PRIVATE_A:.*]] : !fir.ref<i32>)
!$omp do simd lastprivate(a) firstprivate(a)
do i = 1, 10
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPClauses.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPClauses.td
index bd8bc6ed12770c..81419ac4b9920b 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPClauses.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPClauses.td
@@ -821,16 +821,21 @@ class OpenMP_LinearClauseSkip<
bit description = false, bit extraClassDeclaration = false
> : OpenMP_Clause<traits, arguments, assemblyFormat, description,
extraClassDeclaration> {
+ let traits = [
+ BlockArgOpenMPOpInterface
+ ];
+
let arguments = (ins Variadic<AnyType>:$linear_vars,
Variadic<AnyInteger>:$linear_step_vars,
OptionalAttr<ArrayAttr>:$linear_var_types,
- OptionalAttr<ArrayAttr>:$linear_modifiers);
+ OptionalAttr<ArrayAttr>:$linear_modifiers,
+ OptionalAttr<SymbolRefArrayAttr>:$linear_syms);
let optAssemblyFormat = [{
`linear` `(`
custom<LinearClause>($linear_vars, type($linear_vars),
$linear_step_vars, type($linear_step_vars),
- $linear_modifiers) `)`
+ $linear_modifiers, $linear_syms) `)`
( `linear_var_types` `(` $linear_var_types^ `)` )?
}];
@@ -843,6 +848,10 @@ class OpenMP_LinearClauseSkip<
The `linear_modifiers` attribute optionally specifies a per-variable
linear-modifier: `val`, `ref`, or `uval`. When omitted, the default
modifier is determined by the language semantics.
+
+ On operations with a region, `linear_syms` names the `omp.private`
+ declaration of `linear` type that privatizes each linear variable, and each
+ linear variable defines an entry block argument for its private copy.
}];
}
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPEnums.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPEnums.td
index 50b02c0d9e61c9..a44d15fa408fd6 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPEnums.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPEnums.td
@@ -200,12 +200,14 @@ def ClauseTaskDependAttr : OpenMP_EnumAttr<ClauseTaskDepend,
def DataSharingTypePrivate : I32EnumAttrCase<"Private", 0, "private">;
def DataSharingTypeFirstPrivate
: I32EnumAttrCase<"FirstPrivate", 1, "firstprivate">;
+def DataSharingTypeLinear : I32EnumAttrCase<"Linear", 2, "linear">;
def DataSharingClauseType : OpenMP_I32EnumAttr<
"DataSharingClauseType",
"Type of a data-sharing clause", [
DataSharingTypePrivate,
- DataSharingTypeFirstPrivate
+ DataSharingTypeFirstPrivate,
+ DataSharingTypeLinear
]>;
def DataSharingClauseTypeAttr : OpenMP_EnumAttr<DataSharingClauseType,
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index 4cfafef8a32c95..307b91ebe99af2 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -83,10 +83,30 @@ def PrivateClauseOp
}
```
+ * `linear(x)` for an i32 would be emitted as:
+ ```mlir
+ omp.private {type = linear} @x.privatizer : i32
+ ```
+
+ * `linear(x)` for types wrapping the underlying linear variable (e.g.
+ Fortran's ALLOCATABLE) would be emitted as:
+ ```mlir
+ omp.private {type = linear} @x.privatizer : !some.type init {
+ ...
+ } data_address {
+ ^bb0(%arg0: !some.pointer<!some.type>):
+ // return the address of the data that %arg0 describes.
+ omp.yield(%addr : !llvm.ptr)
+ } dealloc {
+ ...
+ }
+ ```
+
There are no restrictions on the body except for:
- - The `dealloc` regions has a single argument.
+ - The `dealloc` & `data_address` regions have a single argument.
- The `init` & `copy` regions have 2 arguments.
- - All three regions are terminated by `omp.yield` ops.
+ - All regions are terminated by `omp.yield` ops.
+ - The `data_address` region yields a single value.
The above restrictions and other obvious restrictions (e.g. verifying the
type of yielded values) are verified by the custom op verifier. The actual
contents of the blocks inside all regions are not verified.
@@ -103,7 +123,10 @@ def PrivateClauseOp
the regions should be a type which represents a pointer to `type`.
The `data_sharing_type` attribute specifies whether privatizer corresponds
- to a `private` or a `firstprivate` clause.
+ to a `private`, `firstprivate` or `linear` clause. A `linear` privatizer is
+ only referenced by `linear` clauses; its optional `data_address` region
+ gives the address of the value the linear clause updates. Without it, the
+ variable itself holds that value.
}];
let arguments = (ins SymbolNameAttr:$sym_name,
@@ -113,13 +136,15 @@ def PrivateClauseOp
let regions = (region AnyRegion:$init_region,
AnyRegion:$copy_region,
- AnyRegion:$dealloc_region);
+ AnyRegion:$dealloc_region,
+ AnyRegion:$data_address_region);
let assemblyFormat = [{
` ` `{` `type` `=` enum($data_sharing_type) `}`
($sym_visibility^)? $sym_name `:` $type
(`init` $init_region^)?
(`copy` $copy_region^)?
+ (`data_address` $data_address_region^)?
(`dealloc` $dealloc_region^)? attr-dict
}];
@@ -146,6 +171,10 @@ def PrivateClauseOp
auto ®ion = getCopyRegion();
return region.empty() ? nullptr : region.getArgument(1);
}
+ BlockArgument getDataAddressArg() {
+ auto ®ion = getDataAddressRegion();
+ return region.empty() ? nullptr : region.getArgument(0);
+ }
BlockArgument getDeallocMoldArg() {
auto ®ion = getDeallocRegion();
return region.empty() ? nullptr : region.getArgument(0);
@@ -851,7 +880,8 @@ def WsloopOp : OpenMP_Op<"wsloop", traits = [
DeclareOpInterfaceMethods<LoopWrapperInterface>, HasParallelRegion,
NoTerminator, RecursiveMemoryEffects, SingleBlock
], clauses = [
- OpenMP_AllocateClause, OpenMP_LinearClause, OpenMP_NowaitClause,
+ OpenMP_AllocateClause, OpenMP_LinearClauseSkip<assemblyFormat = true>,
+ OpenMP_NowaitClause,
OpenMP_OrderClause, OpenMP_OrderedClause, OpenMP_PrivateClause,
OpenMP_ReductionClause, OpenMP_ScheduleClause
], singleRegion = true> {
@@ -885,9 +915,12 @@ def WsloopOp : OpenMP_Op<"wsloop", traits = [
];
let assemblyFormat = clausesAssemblyFormat # [{
- custom<PrivateReductionRegion>($region, $private_vars, type($private_vars),
- $private_syms, $private_needs_barrier, $reduction_mod, $reduction_vars,
- type($reduction_vars), $reduction_byref, $reduction_syms) attr-dict
+ custom<LinearPrivateReductionRegion>($region, $linear_vars,
+ type($linear_vars), $linear_step_vars, type($linear_step_vars),
+ $linear_var_types, $linear_modifiers, $linear_syms, $private_vars,
+ type($private_vars), $private_syms, $private_needs_barrier,
+ $reduction_mod, $reduction_vars, type($reduction_vars),
+ $reduction_byref, $reduction_syms) attr-dict
}];
let hasVerifier = 1;
@@ -903,8 +936,8 @@ def SimdOp : OpenMP_Op<"simd", traits = [
DeclareOpInterfaceMethods<LoopWrapperInterface>, HasParallelRegion,
NoTerminator, RecursiveMemoryEffects, SingleBlock
], clauses = [
- OpenMP_AlignedClause, OpenMP_IfClause, OpenMP_LinearClause,
- OpenMP_NontemporalClause, OpenMP_OrderClause, OpenMP_PrivateClause,
+ OpenMP_AlignedClause, OpenMP_IfClause,
+ OpenMP_LinearClauseSkip<assemblyFormat = true>, OpenMP_NontemporalClause, OpenMP_OrderClause, OpenMP_PrivateClause,
OpenMP_ReductionClause, OpenMP_SafelenClause, OpenMP_SimdlenClause
], singleRegion = true> {
let summary = "simd construct";
@@ -939,9 +972,12 @@ def SimdOp : OpenMP_Op<"simd", traits = [
];
let assemblyFormat = clausesAssemblyFormat # [{
- custom<PrivateReductionRegion>($region, $private_vars, type($private_vars),
- $private_syms, $private_needs_barrier, $reduction_mod, $reduction_vars,
- type($reduction_vars), $reduction_byref, $reduction_syms) attr-dict
+ custom<LinearPrivateReductionRegion>($region, $linear_vars,
+ type($linear_vars), $linear_step_vars, type($linear_step_vars),
+ $linear_var_types, $linear_modifiers, $linear_syms, $private_vars,
+ type($private_vars), $private_syms, $private_needs_barrier,
+ $reduction_mod, $reduction_vars, type($reduction_vars),
+ $reduction_byref, $reduction_syms) attr-dict
}];
let hasVerifier = 1;
@@ -2686,7 +2722,8 @@ def WorkdistributeOp : OpenMP_Op<"workdistribute", traits = [
def DeclareSimdOp
: OpenMP_Op<"declare_simd", traits = [AttrSizedOperandSegments],
clauses = [OpenMP_AlignedClause, OpenMP_InbranchClause,
- OpenMP_LinearClause, OpenMP_NotinbranchClause,
+ OpenMP_LinearClauseSkip<traits = true>,
+ OpenMP_NotinbranchClause,
OpenMP_SimdlenClause, OpenMP_UniformClause]> {
let summary = "declare simd directive";
let description = [{
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsInterfaces.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsInterfaces.td
index 35f879e2241793..4d0b440665520e 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsInterfaces.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsInterfaces.td
@@ -123,8 +123,10 @@ def BlockArgHostEvalClause : BlockArgOpenMPClause<
"host_eval", "HostEval", BlockArgHasDeviceAddrClause>;
def BlockArgInReductionClause : BlockArgOpenMPClause<
"in_reduction", "InReduction", BlockArgHostEvalClause>;
+def BlockArgLinearClause : BlockArgOpenMPClause<
+ "linear", "Linear", BlockArgInReductionClause>;
def BlockArgMapClause : BlockArgOpenMPClause<
- "map", "Map", BlockArgInReductionClause>;
+ "map", "Map", BlockArgLinearClause>;
def BlockArgPrivateClause : BlockArgOpenMPClause<
"private", "Private", BlockArgMapClause>;
def BlockArgReductionClause : BlockArgOpenMPClause<
@@ -145,7 +147,8 @@ def BlockArgOpenMPOpInterface : OpInterface<"BlockArgOpenMPOpInterface"> {
let cppNamespace = "::mlir::omp";
defvar clauses = [ BlockArgHasDeviceAddrClause, BlockArgHostEvalClause,
- BlockArgInReductionClause, BlockArgMapClause, BlockArgPrivateClause,
+ BlockArgInReductionClause, BlockArgLinearClause, BlockArgMapClause,
+ BlockArgPrivateClause,
BlockArgReductionClause, BlockArgTaskReductionClause,
BlockArgUseDeviceAddrClause, BlockArgUseDevicePtrClause ];
diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
index c7dd250e066612..d9b6e40dbcb065 100644
--- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
+++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
@@ -471,7 +471,8 @@ static ParseResult parseLinearClause(
SmallVectorImpl<OpAsmParser::UnresolvedOperand> &linearVars,
SmallVectorImpl<Type> &linearTypes,
SmallVectorImpl<OpAsmParser::UnresolvedOperand> &linearStepVars,
- SmallVectorImpl<Type> &linearStepTypes, ArrayAttr &linearModifiers) {
+ SmallVectorImpl<Type> &linearStepTypes, ArrayAttr &linearModifiers,
+ ArrayAttr &linearSyms) {
SmallVector<Attribute> modifiers;
auto result = parser.parseCommaSeparatedList([&]() {
OpAsmParser::UnresolvedOperand var;
@@ -517,11 +518,12 @@ static ParseResult parseLinearClause(
return success();
}
-/// Print Linear Clause
+/// Print Linear Clause. `linearSyms` is only used by ops with a region, which
+/// print the linear clause with their entry block arguments.
static void printLinearClause(OpAsmPrinter &p, Operation *op,
ValueRange linearVars, TypeRange linearTypes,
ValueRange linearStepVars, TypeRange stepVarTypes,
- ArrayAttr linearModifiers) {
+ ArrayAttr linearModifiers, ArrayAttr linearSyms) {
size_t linearVarsSize = linearVars.size();
for (unsigned i = 0; i < linearVarsSize; ++i) {
if (i != 0)
@@ -1272,6 +1274,23 @@ struct PrivateParseArgs {
mapIndices(mapIndices) {}
};
+struct LinearParseArgs {
+ SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars;
+ SmallVectorImpl<Type> &types;
+ SmallVectorImpl<OpAsmParser::UnresolvedOperand> &stepVars;
+ SmallVectorImpl<Type> &stepTypes;
+ ArrayAttr &varTypes;
+ ArrayAttr &modifiers;
+ ArrayAttr &syms;
+ LinearParseArgs(SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars,
+ SmallVectorImpl<Type> &types,
+ SmallVectorImpl<OpAsmParser::UnresolvedOperand> &stepVars,
+ SmallVectorImpl<Type> &stepTypes, ArrayAttr &varTypes,
+ ArrayAttr &modifiers, ArrayAttr &syms)
+ : vars(vars), types(types), stepVars(stepVars), stepTypes(stepTypes),
+ varTypes(varTypes), modifiers(modifiers), syms(syms) {}
+};
+
struct ReductionParseArgs {
SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars;
SmallVectorImpl<Type> &types;
@@ -1288,6 +1307,7 @@ struct AllRegionParseArgs {
std::optional<MapParseArgs> hasDeviceAddrArgs;
std::optional<MapParseArgs> hostEvalArgs;
std::optional<ReductionParseArgs> inReductionArgs;
+ std::optional<LinearParseArgs> linearArgs;
std::optional<MapParseArgs> mapArgs;
std::optional<PrivateParseArgs> privateArgs;
std::optional<ReductionParseArgs> reductionArgs;
@@ -1453,6 +1473,63 @@ static ParseResult parseBlockArgClause(
return success();
}
+/// linear-entry := [modifier `(`] symbol ssa-id `->` ssa-id `:` type `=`
+/// ssa-id `:` type [`)`]
+static ParseResult parseBlockArgClause(
+ OpAsmParser &parser,
+ llvm::SmallVectorImpl<OpAsmParser::Argument> &entryBlockArgs,
+ StringRef keyword, std::optional<LinearParseArgs> linearArgs) {
+ if (!linearArgs || failed(parser.parseOptionalKeyword(keyword)))
+ return success();
+
+ MLIRContext *ctx = parser.getContext();
+ SmallVector<Attribute> syms, modifiers;
+ if (parser.parseLParen() ||
+ parser.parseCommaSeparatedList([&]() -> ParseResult {
+ std::optional<LinearModifier> modifier;
+ if (succeeded(parser.parseOptionalKeyword("val")))
+ modifier = LinearModifier::val;
+ else if (succeeded(parser.parseOptionalKeyword("ref")))
+ modifier = LinearModifier::ref;
+ else if (succeeded(parser.parseOptionalKeyword("uval")))
+ modifier = LinearModifier::uval;
+ if (modifier && parser.parseLParen())
+ return failure();
+
+ SymbolRefAttr sym;
+ OpAsmParser::Argument &arg = entryBlockArgs.emplace_back();
+ if (parser.parseAttribute(sym) ||
+ parser.parseOperand(linearArgs->vars.emplace_back()) ||
+ parser.parseArrow() || parser.parseArgument(arg) ||
+ parser.parseColonType(linearArgs->types.emplace_back()) ||
+ parser.parseEqual() ||
+ parser.parseOperand(linearArgs->stepVars.emplace_back()) ||
+ parser.parseColonType(linearArgs->stepTypes.emplace_back()))
+ return failure();
+ arg.type = linearArgs->types.back();
+
+ if (modifier && parser.parseRParen())
+ return failure();
+
+ syms.push_back(sym);
+ modifiers.push_back(
+ modifier ? Attribute(LinearModifierAttr::get(ctx, *modifier))
+ : Attribute(UnitAttr::get(ctx)));
+ return success();
+ }) ||
+ parser.parseRParen())
+ return failure();
+
+ linearArgs->syms = ArrayAttr::get(ctx, syms);
+ linearArgs->modifiers = ArrayAttr::get(ctx, modifiers);
+
+ if (succeeded(parser.parseOptionalKeyword("linear_var_types")) &&
+ (parser.parseLParen() || parser.parseAttribute(linearArgs->varTypes) ||
+ parser.parseRParen()))
+ return failure();
+ return success();
+}
+
static ParseResult parseBlockArgRegion(OpAsmParser &parser, Region ®ion,
AllRegionParseArgs args) {
llvm::SmallVector<OpAsmParser::Argument> entryBlockArgs;
@@ -1472,6 +1549,11 @@ static ParseResult parseBlockArgRegion(OpAsmParser &parser, Region ®ion,
return parser.emitError(parser.getCurrentLocation())
<< "invalid `in_reduction` format";
+ if (failed(parseBlockArgClause(parser, entryBlockArgs, "linear",
+ args.linearArgs)))
+ return parser.emitError(parser.getCurrentLocation())
+ << "invalid `linear` format";
+
if (failed(parseBlockArgClause(parser, entryBlockArgs, "map_entries",
args.mapArgs)))
return parser.emitError(parser.getCurrentLocation())
@@ -1591,6 +1673,30 @@ static ParseResult parsePrivateReductionRegion(
return parseBlockArgRegion(parser, region, args);
}
+static ParseResult parseLinearPrivateReductionRegion(
+ OpAsmParser &parser, Region ®ion,
+ SmallVectorImpl<OpAsmParser::UnresolvedOperand> &linearVars,
+ SmallVectorImpl<Type> &linearTypes,
+ SmallVectorImpl<OpAsmParser::UnresolvedOperand> &linearStepVars,
+ SmallVectorImpl<Type> &linearStepTypes, ArrayAttr &linearVarTypes,
+ ArrayAttr &linearModifiers, ArrayAttr &linearSyms,
+ llvm::SmallVectorImpl<OpAsmParser::UnresolvedOperand> &privateVars,
+ llvm::SmallVectorImpl<Type> &privateTypes, ArrayAttr &privateSyms,
+ UnitAttr &privateNeedsBarrier, ReductionModifierAttr &reductionMod,
+ SmallVectorImpl<OpAsmParser::UnresolvedOperand> &reductionVars,
+ SmallVectorImpl<Type> &reductionTypes, DenseBoolArrayAttr &reductionByref,
+ ArrayAttr &reductionSyms) {
+ AllRegionParseArgs args;
+ args.linearArgs.emplace(linearVars, linearTypes, linearStepVars,
+ linearStepTypes, linearVarTypes, linearModifiers,
+ linearSyms);
+ args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
+ privateNeedsBarrier);
+ args.reductionArgs.emplace(reductionVars, reductionTypes, reductionByref,
+ reductionSyms, &reductionMod);
+ return parseBlockArgRegion(parser, region, args);
+}
+
static ParseResult parseTaskReductionRegion(
OpAsmParser &parser, Region ®ion,
SmallVectorImpl<OpAsmParser::UnresolvedOperand> &taskReductionVars,
@@ -1635,6 +1741,20 @@ struct PrivatePrintArgs {
: vars(vars), types(types), syms(syms), needsBarrier(needsBarrier),
mapIndices(mapIndices) {}
};
+struct LinearPrintArgs {
+ ValueRange vars;
+ TypeRange types;
+ ValueRange stepVars;
+ TypeRange stepTypes;
+ ArrayAttr varTypes;
+ ArrayAttr modifiers;
+ ArrayAttr syms;
+ LinearPrintArgs(ValueRange vars, TypeRange types, ValueRange stepVars,
+ TypeRange stepTypes, ArrayAttr varTypes, ArrayAttr modifiers,
+ ArrayAttr syms)
+ : vars(vars), types(types), stepVars(stepVars), stepTypes(stepTypes),
+ varTypes(varTypes), modifiers(modifiers), syms(syms) {}
+};
struct ReductionPrintArgs {
ValueRange vars;
TypeRange types;
@@ -1649,6 +1769,7 @@ struct AllRegionPrintArgs {
std::optional<MapPrintArgs> hasDeviceAddrArgs;
std::optional<MapPrintArgs> hostEvalArgs;
std::optional<ReductionPrintArgs> inReductionArgs;
+ std::optional<LinearPrintArgs> linearArgs;
std::optional<MapPrintArgs> mapArgs;
std::optional<PrivatePrintArgs> privateArgs;
std::optional<ReductionPrintArgs> reductionArgs;
@@ -1739,6 +1860,35 @@ printBlockArgClause(OpAsmPrinter &p, MLIRContext *ctx, StringRef clauseName,
reductionArgs->byref, reductionArgs->modifier);
}
+static void printBlockArgClause(OpAsmPrinter &p, MLIRContext *ctx,
+ StringRef clauseName, ValueRange argsSubrange,
+ std::optional<LinearPrintArgs> linearArgs) {
+ if (!linearArgs || argsSubrange.empty())
+ return;
+
+ p << clauseName << "(";
+ for (auto [i, var] : llvm::enumerate(linearArgs->vars)) {
+ if (i != 0)
+ p << ", ";
+ auto modifier =
+ linearArgs->modifiers
+ ? dyn_cast_if_present<LinearModifierAttr>(linearArgs->modifiers[i])
+ : nullptr;
+ if (modifier)
+ p << stringifyLinearModifier(modifier.getValue()) << "(";
+ if (linearArgs->syms)
+ p << linearArgs->syms[i] << " ";
+ p << var << " -> " << argsSubrange[i] << " : " << linearArgs->types[i]
+ << " = " << linearArgs->stepVars[i] << " : " << linearArgs->stepTypes[i];
+ if (modifier)
+ p << ")";
+ }
+ p << ") ";
+
+ if (linearArgs->varTypes)
+ p << "linear_var_types(" << linearArgs->varTypes << ") ";
+}
+
static void printBlockArgRegion(OpAsmPrinter &p, Operation *op, Region ®ion,
const AllRegionPrintArgs &args) {
auto iface = llvm::cast<mlir::omp::BlockArgOpenMPOpInterface>(op);
@@ -1751,6 +1901,8 @@ static void printBlockArgRegion(OpAsmPrinter &p, Operation *op, Region ®ion,
args.hostEvalArgs);
printBlockArgClause(p, ctx, "in_reduction", iface.getInReductionBlockArgs(),
args.inReductionArgs);
+ printBlockArgClause(p, ctx, "linear", iface.getLinearBlockArgs(),
+ args.linearArgs);
printBlockArgClause(p, ctx, "map_entries", iface.getMapBlockArgs(),
args.mapArgs);
printBlockArgClause(p, ctx, "private", iface.getPrivateBlockArgs(),
@@ -1848,6 +2000,26 @@ static void printPrivateReductionRegion(
printBlockArgRegion(p, op, region, args);
}
+static void printLinearPrivateReductionRegion(
+ OpAsmPrinter &p, Operation *op, Region ®ion, ValueRange linearVars,
+ TypeRange linearTypes, ValueRange linearStepVars, TypeRange linearStepTypes,
+ ArrayAttr linearVarTypes, ArrayAttr linearModifiers, ArrayAttr linearSyms,
+ ValueRange privateVars, TypeRange privateTypes, ArrayAttr privateSyms,
+ UnitAttr privateNeedsBarrier, ReductionModifierAttr reductionMod,
+ ValueRange reductionVars, TypeRange reductionTypes,
+ DenseBoolArrayAttr reductionByref, ArrayAttr reductionSyms) {
+ AllRegionPrintArgs args;
+ args.linearArgs.emplace(linearVars, linearTypes, linearStepVars,
+ linearStepTypes, linearVarTypes, linearModifiers,
+ linearSyms);
+ args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
+ privateNeedsBarrier,
+ /*mapIndices=*/nullptr);
+ args.reductionArgs.emplace(reductionVars, reductionTypes, reductionByref,
+ reductionSyms, reductionMod);
+ printBlockArgRegion(p, op, region, args);
+}
+
static void printTaskReductionRegion(OpAsmPrinter &p, Operation *op,
Region ®ion,
ValueRange taskReductionVars,
@@ -3063,17 +3235,51 @@ static LogicalResult verifyPrivateVarList(OpType &op) {
if (privatizerType && (varType != privatizerType))
return op.emitError()
<< "type mismatch between a "
- << (privatizerOp.getDataSharingType() ==
- DataSharingClauseType::Private
- ? "private"
- : "firstprivate")
+ << stringifyDataSharingClauseType(
+ privatizerOp.getDataSharingType())
<< " variable and its privatizer op, var type: " << varType
<< " vs. privatizer op type: " << privatizerType;
+
+ if (privatizerOp.getDataSharingType() == DataSharingClauseType::Linear)
+ return op.emitError() << "`linear` privatizer '" << privateSym
+ << "' can only be used in a `linear` clause";
}
return success();
}
+/// Verifies that each linear variable has a `linear` privatizer.
+template <typename OpType>
+static LogicalResult verifyLinearSyms(OpType op) {
+ OperandRange linearVars = op.getLinearVars();
+ std::optional<ArrayAttr> linearSyms = op.getLinearSyms();
+ if (linearVars.empty() && (!linearSyms || linearSyms->empty()))
+ return success();
+
+ if (!linearSyms || linearSyms->size() != linearVars.size())
+ return op.emitOpError()
+ << "expected as many linear privatizers as linear variables";
+
+ for (auto [var, sym] : llvm::zip_equal(linearVars, *linearSyms)) {
+ auto symRef = cast<SymbolRefAttr>(sym);
+ auto privatizer =
+ SymbolTable::lookupNearestSymbolFrom<PrivateClauseOp>(op, symRef);
+ if (!privatizer)
+ return op.emitOpError() << "failed to lookup privatizer op with symbol: '"
+ << symRef << "'";
+ if (privatizer.getDataSharingType() != DataSharingClauseType::Linear)
+ return op.emitOpError() << "privatizer '" << symRef
+ << "' of a linear variable must be `linear`";
+ Type argType = privatizer.getArgType();
+ if (argType && argType != var.getType())
+ return op.emitOpError()
+ << "type mismatch between a linear variable and its privatizer "
+ "op, var type: "
+ << var.getType() << " vs. privatizer op type: " << argType;
+ }
+ return success();
+}
+
LogicalResult ParallelOp::verify() {
if (failed(verifyPrivateVarList(*this)))
return failure();
@@ -3510,7 +3716,8 @@ void WsloopOp::build(OpBuilder &builder, OperationState &state,
/*allocate_private_indices=*/nullptr,
/*linear_vars=*/ValueRange(), /*linear_step_vars=*/ValueRange(),
/*linear_var_types*/ nullptr, /*linear_modifiers=*/nullptr,
- /*nowait=*/false, /*order=*/nullptr, /*order_mod=*/nullptr,
+ /*linear_syms=*/nullptr, /*nowait=*/false, /*order=*/nullptr,
+ /*order_mod=*/nullptr,
/*ordered=*/nullptr, /*private_vars=*/{}, /*private_syms=*/nullptr,
/*private_needs_barrier=*/false,
/*reduction_mod=*/nullptr, /*reduction_vars=*/ValueRange(),
@@ -3529,10 +3736,10 @@ void WsloopOp::build(OpBuilder &builder, OperationState &state,
makeDenseI64ArrayAttr(ctx, clauses.allocateAlignments),
makeDenseI64ArrayAttr(ctx, clauses.allocatePrivateIndices),
clauses.linearVars, clauses.linearStepVars, clauses.linearVarTypes,
- clauses.linearModifiers, clauses.nowait, clauses.order, clauses.orderMod,
- clauses.ordered, clauses.privateVars,
- makeArrayAttr(ctx, clauses.privateSyms), clauses.privateNeedsBarrier,
- clauses.reductionMod, clauses.reductionVars,
+ clauses.linearModifiers, makeArrayAttr(ctx, clauses.linearSyms),
+ clauses.nowait, clauses.order, clauses.orderMod, clauses.ordered,
+ clauses.privateVars, makeArrayAttr(ctx, clauses.privateSyms),
+ clauses.privateNeedsBarrier, clauses.reductionMod, clauses.reductionVars,
makeDenseBoolArrayAttr(ctx, clauses.reductionByref),
makeArrayAttr(ctx, clauses.reductionSyms), clauses.scheduleKind,
clauses.scheduleChunk, clauses.scheduleMod, clauses.scheduleSimd);
@@ -3552,7 +3759,7 @@ LogicalResult WsloopOp::verify() {
getLinearVarTypes().value().size() != getLinearVars().size())
return emitError() << "Ill-formed type attributes for linear variables";
- if (failed(verifyPrivateVarList(*this)))
+ if (failed(verifyLinearSyms(*this)) || failed(verifyPrivateVarList(*this)))
return failure();
return verifyReductionVarList(*this, getReductionSyms(), getReductionVars(),
@@ -3591,17 +3798,17 @@ LogicalResult WsloopOp::verifyRegions() {
void SimdOp::build(OpBuilder &builder, OperationState &state,
const SimdOperands &clauses) {
MLIRContext *ctx = builder.getContext();
- SimdOp::build(builder, state, clauses.alignedVars,
- makeArrayAttr(ctx, clauses.alignments), clauses.ifExpr,
- clauses.linearVars, clauses.linearStepVars,
- clauses.linearVarTypes, clauses.linearModifiers,
- clauses.nontemporalVars, clauses.order, clauses.orderMod,
- clauses.privateVars, makeArrayAttr(ctx, clauses.privateSyms),
- clauses.privateNeedsBarrier, clauses.reductionMod,
- clauses.reductionVars,
- makeDenseBoolArrayAttr(ctx, clauses.reductionByref),
- makeArrayAttr(ctx, clauses.reductionSyms), clauses.safelen,
- clauses.simdlen);
+ SimdOp::build(
+ builder, state, clauses.alignedVars,
+ makeArrayAttr(ctx, clauses.alignments), clauses.ifExpr,
+ clauses.linearVars, clauses.linearStepVars, clauses.linearVarTypes,
+ clauses.linearModifiers, makeArrayAttr(ctx, clauses.linearSyms),
+ clauses.nontemporalVars, clauses.order, clauses.orderMod,
+ clauses.privateVars, makeArrayAttr(ctx, clauses.privateSyms),
+ clauses.privateNeedsBarrier, clauses.reductionMod, clauses.reductionVars,
+ makeDenseBoolArrayAttr(ctx, clauses.reductionByref),
+ makeArrayAttr(ctx, clauses.reductionSyms), clauses.safelen,
+ clauses.simdlen);
}
LogicalResult SimdOp::verify() {
@@ -3656,6 +3863,9 @@ LogicalResult SimdOp::verify() {
getLinearVarTypes().value().size() != getLinearVars().size())
return emitError() << "Ill-formed type attributes for linear variables";
+ if (failed(verifyLinearSyms(*this)))
+ return failure();
+
llvm::DenseSet<Value> privateVars(llvm::from_range, getPrivateVars());
llvm::DenseSet<Value> reductionVars(llvm::from_range, getReductionVars());
// TODO Check lastprivate vars when their support is added to SimdOp.
@@ -5333,8 +5543,8 @@ void PrivateClauseOp::build(OpBuilder &odsBuilder, OperationState &odsState,
LogicalResult PrivateClauseOp::verifyRegions() {
Type argType = getArgType();
- auto verifyTerminator = [&](Operation *terminator,
- bool yieldsValue) -> LogicalResult {
+ auto verifyTerminator = [&](Operation *terminator, bool yieldsValue,
+ bool yieldsAnyType) -> LogicalResult {
if (!terminator->getBlock()->getSuccessors().empty())
return success();
@@ -5353,12 +5563,17 @@ LogicalResult PrivateClauseOp::verifyRegions() {
<< "Did not expect any values to be yielded.";
}
- if (yieldedTypes.size() == 1 && yieldedTypes.front() == argType)
+ if (yieldedTypes.size() == 1 &&
+ (yieldsAnyType || yieldedTypes.front() == argType))
return success();
auto error = mlir::emitError(yieldOp.getLoc())
- << "Invalid yielded value. Expected type: " << argType
- << ", got: ";
+ << "Invalid yielded value. Expected ";
+ if (yieldsAnyType)
+ error << "a single value";
+ else
+ error << "type: " << argType;
+ error << ", got: ";
if (yieldedTypes.empty())
error << "None";
@@ -5369,8 +5584,8 @@ LogicalResult PrivateClauseOp::verifyRegions() {
};
auto verifyRegion = [&](Region ®ion, unsigned expectedNumArgs,
- StringRef regionName,
- bool yieldsValue) -> LogicalResult {
+ StringRef regionName, bool yieldsValue,
+ bool yieldsAnyType = false) -> LogicalResult {
assert(!region.empty());
if (region.getNumArguments() != expectedNumArgs)
@@ -5383,7 +5598,8 @@ LogicalResult PrivateClauseOp::verifyRegions() {
if (!block.mightHaveTerminator())
continue;
- if (failed(verifyTerminator(block.getTerminator(), yieldsValue)))
+ if (failed(verifyTerminator(block.getTerminator(), yieldsValue,
+ yieldsAnyType)))
return failure();
}
@@ -5417,6 +5633,19 @@ LogicalResult PrivateClauseOp::verifyRegions() {
/*yieldsValue=*/true)))
return failure();
+ if (dsType == DataSharingClauseType::Linear && !getCopyRegion().empty())
+ return emitError("`linear` clauses do not require a `copy` region.");
+
+ if (!getDataAddressRegion().empty()) {
+ if (dsType != DataSharingClauseType::Linear)
+ return emitError(
+ "only `linear` clauses can have a `data_address` region.");
+ if (failed(verifyRegion(getDataAddressRegion(), /*expectedNumArgs=*/1,
+ "data_address", /*yieldsValue=*/true,
+ /*yieldsAnyType=*/true)))
+ return failure();
+ }
+
if (!getDeallocRegion().empty() &&
failed(verifyRegion(getDeallocRegion(), /*expectedNumArgs=*/1, "dealloc",
/*yieldsValue=*/false)))
@@ -5578,18 +5807,21 @@ LogicalResult DeclareSimdOp::verify() {
/*isDeclareSimd=*/true)))
return failure();
+ if (getLinearSyms())
+ return emitOpError("linear privatizers are not allowed");
+
return verifyAlignedClause(*this, getAlignments(), getAlignedVars());
}
void DeclareSimdOp::build(OpBuilder &odsBuilder, OperationState &odsState,
const DeclareSimdOperands &clauses) {
MLIRContext *ctx = odsBuilder.getContext();
- DeclareSimdOp::build(odsBuilder, odsState, clauses.alignedVars,
- makeArrayAttr(ctx, clauses.alignments), clauses.inbranch,
- clauses.linearVars, clauses.linearStepVars,
- clauses.linearVarTypes, clauses.linearModifiers,
- clauses.notinbranch, clauses.simdlen,
- clauses.uniformVars);
+ DeclareSimdOp::build(
+ odsBuilder, odsState, clauses.alignedVars,
+ makeArrayAttr(ctx, clauses.alignments), clauses.inbranch,
+ clauses.linearVars, clauses.linearStepVars, clauses.linearVarTypes,
+ clauses.linearModifiers, makeArrayAttr(ctx, clauses.linearSyms),
+ clauses.notinbranch, clauses.simdlen, clauses.uniformVars);
}
//===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 1f3d32fd0b9933..b10cf42717945b 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -51,6 +51,11 @@
using namespace mlir;
+static llvm::Expected<llvm::Value *>
+getPrivateDataAddress(llvm::IRBuilderBase &builder,
+ LLVM::ModuleTranslation &moduleTranslation,
+ omp::PrivateClauseOp privatizer, llvm::Value *var);
+
namespace {
static llvm::omp::ScheduleKind
convertToScheduleKind(std::optional<omp::ClauseScheduleKind> schedKind) {
@@ -144,15 +149,46 @@ char PreviouslyReportedError::ID = 0;
class LinearClauseProcessor {
private:
- SmallVector<llvm::Value *> linearPreconditionVars;
- SmallVector<llvm::Value *> linearLoopBodyTemps;
- SmallVector<llvm::Value *> linearOrigVal;
- SmallVector<llvm::Value *> linearSteps;
SmallVector<llvm::Type *> linearVarTypes;
+ SmallVector<llvm::Value *> linearSteps;
+ // The `linear` privatizer, original and private copy of each linear var.
+ SmallVector<omp::PrivateClauseOp> linearPrivatizers;
+ SmallVector<llvm::Value *> linearOrigVars;
+ SmallVector<llvm::Value *> linearPrivateVars;
+ // The value of each linear var on entry to the loop.
+ SmallVector<llvm::Value *> linearStartVals;
llvm::BasicBlock *linearFinalizationBB;
llvm::BasicBlock *linearExitBB;
llvm::BasicBlock *linearLastIterExitBB;
- Value linearLoopIV;
+ std::optional<size_t> linearLoopIVIndex;
+
+ // Address of the value of linear var `index` held by `var`.
+ llvm::Expected<llvm::Value *>
+ getAddress(llvm::IRBuilderBase &builder,
+ LLVM::ModuleTranslation &moduleTranslation, size_t index,
+ llvm::Value *var) {
+ return getPrivateDataAddress(builder, moduleTranslation,
+ linearPrivatizers[index], var);
+ }
+
+ // Copy the current value of each linear var back to the original.
+ llvm::Error storeLinearVars(llvm::IRBuilderBase &builder,
+ LLVM::ModuleTranslation &moduleTranslation) {
+ for (size_t index = 0; index < linearOrigVars.size(); index++) {
+ llvm::Expected<llvm::Value *> from = getAddress(
+ builder, moduleTranslation, index, linearPrivateVars[index]);
+ if (!from)
+ return from.takeError();
+ llvm::LoadInst *linearVarTemp =
+ builder.CreateLoad(linearVarTypes[index], *from);
+ llvm::Expected<llvm::Value *> to =
+ getAddress(builder, moduleTranslation, index, linearOrigVars[index]);
+ if (!to)
+ return to.takeError();
+ builder.CreateStore(linearVarTemp, *to);
+ }
+ return llvm::Error::success();
+ }
public:
// Register type for the linear variables
@@ -162,16 +198,18 @@ class LinearClauseProcessor {
mlir::cast<mlir::TypeAttr>(ty).getValue()));
}
- // Allocate space for linear variabes
+ // Register a linear var, allocating space for its start value at `allocaIP`
void createLinearVar(llvm::IRBuilderBase &builder,
- LLVM::ModuleTranslation &moduleTranslation,
- llvm::Value *linearVar, int idx) {
- linearPreconditionVars.push_back(
+ const llvm::OpenMPIRBuilder::InsertPointTy &allocaIP,
+ omp::PrivateClauseOp privatizer, llvm::Value *origVar,
+ llvm::Value *privateVar, int idx) {
+ llvm::IRBuilderBase::InsertPointGuard guard(builder);
+ builder.SetInsertPoint(allocaIP.getNodeParent()->getTerminator());
+ linearStartVals.push_back(
builder.CreateAlloca(linearVarTypes[idx], nullptr, ".linear_var"));
- llvm::Value *linearLoopBodyTemp =
- builder.CreateAlloca(linearVarTypes[idx], nullptr, ".linear_result");
- linearOrigVal.push_back(linearVar);
- linearLoopBodyTemps.push_back(linearLoopBodyTemp);
+ linearPrivatizers.push_back(privatizer);
+ linearOrigVars.push_back(origVar);
+ linearPrivateVars.push_back(privateVar);
}
// Initialize linear step
@@ -181,15 +219,20 @@ class LinearClauseProcessor {
}
// Emit IR for initialization of linear variables
- void initLinearVar(llvm::IRBuilderBase &builder,
- LLVM::ModuleTranslation &moduleTranslation,
- llvm::BasicBlock *loopPreHeader) {
+ llvm::Error initLinearVar(llvm::IRBuilderBase &builder,
+ LLVM::ModuleTranslation &moduleTranslation,
+ llvm::BasicBlock *loopPreHeader) {
builder.SetInsertPoint(loopPreHeader->getTerminator());
- for (size_t index = 0; index < linearOrigVal.size(); index++) {
+ for (size_t index = 0; index < linearOrigVars.size(); index++) {
+ llvm::Expected<llvm::Value *> orig =
+ getAddress(builder, moduleTranslation, index, linearOrigVars[index]);
+ if (!orig)
+ return orig.takeError();
llvm::LoadInst *linearVarLoad =
- builder.CreateLoad(linearVarTypes[index], linearOrigVal[index]);
- builder.CreateStore(linearVarLoad, linearPreconditionVars[index]);
+ builder.CreateLoad(linearVarTypes[index], *orig);
+ builder.CreateStore(linearVarLoad, linearStartVals[index]);
}
+ return llvm::Error::success();
}
// Find linear iteration variable and save it for later updates
@@ -199,36 +242,71 @@ class LinearClauseProcessor {
if (loopOp.getIVs().size() != 1)
return success();
// Currently, frontends using `omp.simd` always generate a store from the
- // `omp.loop_nest`'s IV to the corresponding iteration variable.
- // We leverage this to find the linear iteration variable.
+ // `omp.loop_nest`'s IV to the corresponding iteration variable, possibly
+ // through the data address of its private copy.
//
// TODO Add an attribute to `omp.loop_nest` that explicitly lists the
// variables that correspond to the loop induction variables.
- BlockArgument arg = loopOp.getIVs().front();
- for (const Operation *user : arg.getUsers()) {
- if (auto storeOp = dyn_cast<LLVM::StoreOp>(user)) {
- for (Value linearVar : simdOp.getLinearVars()) {
- if (linearVar == storeOp.getAddr()) {
- if (linearLoopIV && linearLoopIV != linearVar)
- return simdOp.emitError(
- "Could not determine the linear variable associated with the "
- "loop nest induction variable");
- linearLoopIV = linearVar;
+ MutableArrayRef<BlockArgument> linearArgs =
+ cast<omp::BlockArgOpenMPOpInterface>(*simdOp).getLinearBlockArgs();
+ SmallVector<Value> ivValues{loopOp.getIVs().front()};
+ for (Operation *user : loopOp.getIVs().front().getUsers())
+ if (isa<LLVM::SExtOp, LLVM::ZExtOp, LLVM::TruncOp>(user))
+ ivValues.push_back(user->getResult(0));
+
+ for (Value ivValue : ivValues) {
+ for (Operation *user : ivValue.getUsers()) {
+ auto storeOp = dyn_cast<LLVM::StoreOp>(user);
+ if (!storeOp || storeOp.getValue() != ivValue)
+ continue;
+ // Look through loads, GEPs and copies of a private copy, e.g. of a
+ // descriptor, to find the linear block argument the IV is stored to.
+ Value addr = storeOp.getAddr();
+ while (true) {
+ Operation *def = addr.getDefiningOp();
+ if (auto load = dyn_cast_if_present<LLVM::LoadOp>(def)) {
+ addr = load.getAddr();
+ } else if (auto gep = dyn_cast_if_present<LLVM::GEPOp>(def)) {
+ addr = gep.getBase();
+ } else {
+ auto copy = llvm::find_if(addr.getUsers(), [&](Operation *user) {
+ auto memcpy = dyn_cast<LLVM::MemcpyOp>(user);
+ return memcpy && memcpy.getDst() == addr &&
+ llvm::is_contained(linearArgs, memcpy.getSrc());
+ });
+ if (copy == addr.getUsers().end())
+ break;
+ addr = cast<LLVM::MemcpyOp>(*copy).getSrc();
}
}
+ auto *it = llvm::find(linearArgs, addr);
+ if (it == linearArgs.end())
+ continue;
+ size_t index = std::distance(linearArgs.begin(), it);
+ if (linearLoopIVIndex && *linearLoopIVIndex != index)
+ return simdOp.emitError(
+ "Could not determine the linear variable associated with the "
+ "loop nest induction variable");
+ linearLoopIVIndex = index;
}
}
return success();
}
// Emit IR for updating Linear variables
- void updateLinearVar(llvm::IRBuilderBase &builder, llvm::BasicBlock *loopBody,
- llvm::Value *loopInductionVar) {
+ llvm::Error updateLinearVar(llvm::IRBuilderBase &builder,
+ LLVM::ModuleTranslation &moduleTranslation,
+ llvm::BasicBlock *loopBody,
+ llvm::Value *loopInductionVar) {
builder.SetInsertPoint(loopBody->getTerminator());
- for (size_t index = 0; index < linearPreconditionVars.size(); index++) {
+ for (size_t index = 0; index < linearStartVals.size(); index++) {
llvm::Type *linearVarType = linearVarTypes[index];
llvm::Value *iv = loopInductionVar;
llvm::Value *step = linearSteps[index];
+ llvm::Expected<llvm::Value *> result = getAddress(
+ builder, moduleTranslation, index, linearPrivateVars[index]);
+ if (!result)
+ return result.takeError();
if (!iv->getType()->isIntegerTy())
llvm_unreachable("OpenMP loop induction variable must be an integer "
@@ -240,53 +318,48 @@ class LinearClauseProcessor {
step = builder.CreateSExtOrTrunc(step, linearVarType);
llvm::LoadInst *linearVarStart =
- builder.CreateLoad(linearVarType, linearPreconditionVars[index]);
+ builder.CreateLoad(linearVarType, linearStartVals[index]);
llvm::Value *mulInst = builder.CreateMul(iv, step);
llvm::Value *addInst = builder.CreateAdd(linearVarStart, mulInst);
- builder.CreateStore(addInst, linearLoopBodyTemps[index]);
+ builder.CreateStore(addInst, *result);
} else if (linearVarType->isFloatingPointTy()) {
// Float path: perform multiply in integer, then convert to float
step = builder.CreateSExtOrTrunc(step, iv->getType());
llvm::Value *mulInst = builder.CreateMul(iv, step);
llvm::LoadInst *linearVarStart =
- builder.CreateLoad(linearVarType, linearPreconditionVars[index]);
+ builder.CreateLoad(linearVarType, linearStartVals[index]);
llvm::Value *mulFp = builder.CreateSIToFP(mulInst, linearVarType);
llvm::Value *addInst = builder.CreateFAdd(linearVarStart, mulFp);
- builder.CreateStore(addInst, linearLoopBodyTemps[index]);
+ builder.CreateStore(addInst, *result);
} else {
llvm_unreachable(
"Linear variable must be of integer or floating-point type");
}
}
+ return llvm::Error::success();
}
// Emit IR for updating linear iteration variables on loop exit
- void updateLinearIV(llvm::IRBuilderBase &builder,
- LLVM::ModuleTranslation &moduleTranslation) {
- if (!linearLoopIV)
- return;
- llvm::Value *linearIV = moduleTranslation.lookupValue(linearLoopIV);
-
- // Find linearIV's index
- size_t index;
- for (index = 0; index < linearOrigVal.size(); index++)
- if (linearIV == linearOrigVal[index])
- break;
- if (index == linearOrigVal.size())
- return;
+ llvm::Error updateLinearIV(llvm::IRBuilderBase &builder,
+ LLVM::ModuleTranslation &moduleTranslation) {
+ if (!linearLoopIVIndex)
+ return llvm::Error::success();
+ size_t index = *linearLoopIVIndex;
// Add one more step to the linear iteration variable
llvm::Type *varType = linearVarTypes[index];
- llvm::Value *var = linearLoopBodyTemps[index];
- llvm::Value *step = linearSteps[index];
if (!varType->isIntegerTy())
llvm_unreachable("Linear iteration variable must be of integer type");
-
- step = builder.CreateSExtOrTrunc(step, varType);
- llvm::Value *val = builder.CreateLoad(varType, var);
+ llvm::Expected<llvm::Value *> var =
+ getAddress(builder, moduleTranslation, index, linearPrivateVars[index]);
+ if (!var)
+ return var.takeError();
+ llvm::Value *step = builder.CreateSExtOrTrunc(linearSteps[index], varType);
+ llvm::Value *val = builder.CreateLoad(varType, *var);
llvm::Value *addInst = builder.CreateAdd(val, step);
- builder.CreateStore(addInst, var);
+ builder.CreateStore(addInst, *var);
+ return llvm::Error::success();
}
// Linear variable finalization is conditional on the last logical iteration.
@@ -316,11 +389,8 @@ class LinearClauseProcessor {
llvm::Type::getInt32Ty(builder.getContext()), 0));
// Store the linear variable values to original variables.
builder.SetInsertPoint(linearLastIterExitBB->getTerminator());
- for (size_t index = 0; index < linearOrigVal.size(); index++) {
- llvm::LoadInst *linearVarTemp =
- builder.CreateLoad(linearVarTypes[index], linearLoopBodyTemps[index]);
- builder.CreateStore(linearVarTemp, linearOrigVal[index]);
- }
+ if (llvm::Error err = storeLinearVars(builder, moduleTranslation))
+ return std::move(err);
// Create conditional branch such that the linear variable
// values are stored to original variables only at the
@@ -336,48 +406,10 @@ class LinearClauseProcessor {
// Emit stores for linear variables. Useful in case of SIMD
// construct.
- void emitStoresForLinearVar(llvm::IRBuilderBase &builder) {
- for (size_t index = 0; index < linearOrigVal.size(); index++) {
- llvm::LoadInst *linearVarTemp =
- builder.CreateLoad(linearVarTypes[index], linearLoopBodyTemps[index]);
- builder.CreateStore(linearVarTemp, linearOrigVal[index]);
- }
- }
-
- // Rewrite all uses of the original variable, in the basic blocks in the
- // [startBB, endBB] interval, with the linear variable in-place.
- void rewriteInPlace(llvm::IRBuilderBase &builder, llvm::BasicBlock *startBB,
- llvm::BasicBlock *endBB, size_t varIndex) {
- llvm::SmallVector<llvm::BasicBlock *, 32> worklist;
- llvm::SmallPtrSet<llvm::BasicBlock *, 32> collectedBBs;
-
- assert(startBB && endBB && "Invalid startBB/endBB");
-
- // Collect basic blocks from startBB to endBB.
- worklist.push_back(startBB);
- collectedBBs.insert(startBB);
-
- while (!worklist.empty()) {
- llvm::BasicBlock *bb = worklist.pop_back_val();
-
- if (bb == endBB)
- continue;
-
- for (llvm::BasicBlock *succ : llvm::successors(bb)) {
- if (collectedBBs.insert(succ).second)
- worklist.push_back(succ);
- }
- }
-
- // Rewrite all uses in the collected BBs.
- llvm::SmallVector<llvm::User *> users(linearOrigVal[varIndex]->users());
- for (auto *user : users) {
- if (auto *userInst = dyn_cast<llvm::Instruction>(user)) {
- if (collectedBBs.contains(userInst->getParent()))
- user->replaceUsesOfWith(linearOrigVal[varIndex],
- linearLoopBodyTemps[varIndex]);
- }
- }
+ llvm::Error
+ emitStoresForLinearVar(llvm::IRBuilderBase &builder,
+ LLVM::ModuleTranslation &moduleTranslation) {
+ return storeLinearVars(builder, moduleTranslation);
}
};
@@ -1037,8 +1069,8 @@ struct PrivateVarsInfo {
template <typename OP>
PrivateVarsInfo(OP op)
- : blockArgs(
- cast<omp::BlockArgOpenMPOpInterface>(*op).getPrivateBlockArgs()) {
+ : blockArgs(llvm::to_vector(
+ cast<omp::BlockArgOpenMPOpInterface>(*op).getPrivateBlockArgs())) {
mlirVars.reserve(blockArgs.size());
llvmVars.reserve(blockArgs.size());
collectPrivatizationDecls<OP>(op);
@@ -1047,7 +1079,23 @@ struct PrivateVarsInfo {
mlirVars.push_back(privateVar);
}
- MutableArrayRef<BlockArgument> blockArgs;
+ /// Adds the linear variables of `op`, which are privatized by their
+ /// `linear` privatizers, after the private variables.
+ template <typename OP>
+ void addLinearVars(OP op) {
+ numPrivateVars = blockArgs.size();
+ llvm::append_range(
+ blockArgs,
+ cast<omp::BlockArgOpenMPOpInterface>(*op).getLinearBlockArgs());
+ llvm::append_range(mlirVars, op.getLinearVars());
+ if (std::optional<ArrayAttr> attr = op.getLinearSyms())
+ for (auto symbolRef : attr->getAsRange<SymbolRefAttr>())
+ privatizers.push_back(findPrivatizer(op, symbolRef));
+ }
+
+ /// Number of private variables before the linear ones.
+ size_t numPrivateVars = 0;
+ SmallVector<BlockArgument> blockArgs;
SmallVector<mlir::Value> mlirVars;
SmallVector<llvm::Value *> llvmVars;
SmallVector<omp::PrivateClauseOp> privatizers;
@@ -1873,6 +1921,26 @@ materializeRegionArgValue(llvm::IRBuilderBase &builder,
return builder.CreateLoad(regionArgType, value);
}
+/// Returns the address of the data in `var`, via the `data_address` region.
+static llvm::Expected<llvm::Value *>
+getPrivateDataAddress(llvm::IRBuilderBase &builder,
+ LLVM::ModuleTranslation &moduleTranslation,
+ omp::PrivateClauseOp privatizer, llvm::Value *var) {
+ Region ®ion = privatizer.getDataAddressRegion();
+ if (region.empty())
+ return var;
+
+ BlockArgument arg = privatizer.getDataAddressArg();
+ moduleTranslation.mapValue(
+ arg, materializeRegionArgValue(builder, moduleTranslation, arg, var));
+ SmallVector<llvm::Value *, 1> address;
+ if (failed(inlineConvertOmpRegions(region, "omp.private.data_address",
+ builder, moduleTranslation, &address)))
+ return llvm::make_error<PreviouslyReportedError>();
+ setInsertPointForPossiblyEmptyBlock(builder);
+ return address.front();
+}
+
/// Initialize a single (first)private variable. You probably want to use
/// allocateAndInitPrivateVars instead of this.
/// This returns the private variable which has been initialized. This
@@ -4788,6 +4856,7 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
}
PrivateVarsInfo privateVarsInfo(wsloopOp);
+ privateVarsInfo.addLinearVars(wsloopOp);
SmallVector<omp::DeclareReductionOp> reductionDecls;
collectReductionDecls(wsloopOp, reductionDecls);
@@ -4878,10 +4947,13 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
for (mlir::Attribute linearVarType : linearVarTypes)
linearClauseProcessor.registerType(moduleTranslation, linearVarType);
- for (auto [idx, linearVar] : llvm::enumerate(wsloopOp.getLinearVars()))
+ for (auto [idx, linearVar] : llvm::enumerate(wsloopOp.getLinearVars())) {
+ size_t privIdx = privateVarsInfo.numPrivateVars + idx;
linearClauseProcessor.createLinearVar(
- builder, moduleTranslation, moduleTranslation.lookupValue(linearVar),
- idx);
+ builder, allocaIP, privateVarsInfo.privatizers[privIdx],
+ moduleTranslation.lookupValue(linearVar),
+ privateVarsInfo.llvmVars[privIdx], idx);
+ }
for (mlir::Value linearStep : wsloopOp.getLinearStepVars())
linearClauseProcessor.initLinearStep(moduleTranslation, linearStep);
}
@@ -4896,16 +4968,22 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
// Emit Initialization and Update IR for linear variables
if (!wsloopOp.getLinearVars().empty()) {
- linearClauseProcessor.initLinearVar(builder, moduleTranslation,
- loopInfo->getPreheader());
+ if (failed(handleError(
+ linearClauseProcessor.initLinearVar(builder, moduleTranslation,
+ loopInfo->getPreheader()),
+ opInst)))
+ return failure();
llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterBarrierIP =
moduleTranslation.getOpenMPBuilder()->createBarrier(
builder, llvm::omp::OMPD_barrier);
if (failed(handleError(afterBarrierIP, *loopOp)))
return failure();
builder.restoreIP(*afterBarrierIP);
- linearClauseProcessor.updateLinearVar(builder, loopInfo->getBody(),
- loopInfo->getIndVar());
+ if (failed(handleError(linearClauseProcessor.updateLinearVar(
+ builder, moduleTranslation, loopInfo->getBody(),
+ loopInfo->getIndVar()),
+ opInst)))
+ return failure();
linearClauseProcessor.splitLinearFiniBB(builder, loopInfo->getExit());
}
@@ -4925,10 +5003,6 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
noLoopMode = true;
}
- for (size_t index = 0; index < wsloopOp.getLinearVars().size(); index++)
- linearClauseProcessor.rewriteInPlace(builder, loopInfo->getBody(),
- loopInfo->getLatch(), index);
-
// Nested regions can move the instruction referenced by allocaIP.
allocaIP = allocaBlock->getFirstInsertionPt();
llvm::OpenMPIRBuilder::InsertPointOrErrorTy wsloopIP =
@@ -5219,6 +5293,7 @@ convertOmpSimd(Operation &opInst, llvm::IRBuilderBase &builder,
return failure();
PrivateVarsInfo privateVarsInfo(simdOp);
+ privateVarsInfo.addLinearVars(simdOp);
MutableArrayRef<BlockArgument> reductionArgs =
cast<omp::BlockArgOpenMPOpInterface>(opInst).getReductionBlockArgs();
@@ -5239,38 +5314,6 @@ convertOmpSimd(Operation &opInst, llvm::IRBuilderBase &builder,
if (handleError(afterAllocas, opInst).failed())
return failure();
- // Initialize linear variables and linear step
- LinearClauseProcessor linearClauseProcessor;
- if (linearClauseProcessor.initLinearIV(simdOp).failed())
- return failure();
-
- if (!simdOp.getLinearVars().empty()) {
- auto linearVarTypes = simdOp.getLinearVarTypes().value();
- for (mlir::Attribute linearVarType : linearVarTypes)
- linearClauseProcessor.registerType(moduleTranslation, linearVarType);
- for (auto [idx, linearVar] : llvm::enumerate(simdOp.getLinearVars())) {
- bool isImplicit = false;
- for (auto [mlirPrivVar, llvmPrivateVar] : llvm::zip_equal(
- privateVarsInfo.mlirVars, privateVarsInfo.llvmVars)) {
- // If the linear variable is implicit, reuse the already
- // existing llvm::Value
- if (linearVar == mlirPrivVar) {
- isImplicit = true;
- linearClauseProcessor.createLinearVar(builder, moduleTranslation,
- llvmPrivateVar, idx);
- break;
- }
- }
-
- if (!isImplicit)
- linearClauseProcessor.createLinearVar(
- builder, moduleTranslation,
- moduleTranslation.lookupValue(linearVar), idx);
- }
- for (mlir::Value linearStep : simdOp.getLinearStepVars())
- linearClauseProcessor.initLinearStep(moduleTranslation, linearStep);
- }
-
if (failed(allocReductionVars(simdOp, reductionArgs, builder,
moduleTranslation, allocaIP, reductionDecls,
privateReductionVariables, reductionVariableMap,
@@ -5286,6 +5329,26 @@ convertOmpSimd(Operation &opInst, llvm::IRBuilderBase &builder,
// is not a worksharing construct, so a privatization barrier here could
// deadlock.
+ // Initialize linear variables and linear step
+ LinearClauseProcessor linearClauseProcessor;
+ if (linearClauseProcessor.initLinearIV(simdOp).failed())
+ return failure();
+
+ if (!simdOp.getLinearVars().empty()) {
+ auto linearVarTypes = simdOp.getLinearVarTypes().value();
+ for (mlir::Attribute linearVarType : linearVarTypes)
+ linearClauseProcessor.registerType(moduleTranslation, linearVarType);
+ for (auto [idx, linearVar] : llvm::enumerate(simdOp.getLinearVars())) {
+ size_t privIdx = privateVarsInfo.numPrivateVars + idx;
+ linearClauseProcessor.createLinearVar(
+ builder, allocaIP, privateVarsInfo.privatizers[privIdx],
+ moduleTranslation.lookupValue(linearVar),
+ privateVarsInfo.llvmVars[privIdx], idx);
+ }
+ for (mlir::Value linearStep : simdOp.getLinearStepVars())
+ linearClauseProcessor.initLinearStep(moduleTranslation, linearStep);
+ }
+
assert(afterAllocas.get()->getSinglePredecessor());
if (failed(initReductionVars(simdOp, reductionArgs, builder,
moduleTranslation,
@@ -5339,26 +5402,34 @@ convertOmpSimd(Operation &opInst, llvm::IRBuilderBase &builder,
llvm::CanonicalLoopInfo *loopInfo = findCurrentLoopInfo(moduleTranslation);
// Emit Initialization for linear variables
if (simdOp.getLinearVars().size()) {
- linearClauseProcessor.initLinearVar(builder, moduleTranslation,
- loopInfo->getPreheader());
+ if (failed(handleError(
+ linearClauseProcessor.initLinearVar(builder, moduleTranslation,
+ loopInfo->getPreheader()),
+ opInst)))
+ return failure();
- linearClauseProcessor.updateLinearVar(builder, loopInfo->getBody(),
- loopInfo->getIndVar());
+ if (failed(handleError(linearClauseProcessor.updateLinearVar(
+ builder, moduleTranslation, loopInfo->getBody(),
+ loopInfo->getIndVar()),
+ opInst)))
+ return failure();
}
builder.SetInsertPoint((*regionBlock)->begin());
- for (size_t index = 0; index < simdOp.getLinearVars().size(); index++)
- linearClauseProcessor.rewriteInPlace(builder, loopInfo->getBody(),
- loopInfo->getLatch(), index);
-
ompBuilder->applySimd(loopInfo, alignedVars,
simdOp.getIfExpr()
? moduleTranslation.lookupValue(simdOp.getIfExpr())
: nullptr,
order, simdlen, safelen);
- linearClauseProcessor.updateLinearIV(builder, moduleTranslation);
- linearClauseProcessor.emitStoresForLinearVar(builder);
+ if (failed(handleError(
+ linearClauseProcessor.updateLinearIV(builder, moduleTranslation),
+ opInst)))
+ return failure();
+ if (failed(handleError(linearClauseProcessor.emitStoresForLinearVar(
+ builder, moduleTranslation),
+ opInst)))
+ return failure();
// We now need to reduce the per-simd-lane reduction variable into the
// original variable. This works a bit
diff erently to other reductions (e.g.
diff --git a/mlir/test/Dialect/OpenMP/invalid.mlir b/mlir/test/Dialect/OpenMP/invalid.mlir
index bf586abf6222f5..56884868927da4 100644
--- a/mlir/test/Dialect/OpenMP/invalid.mlir
+++ b/mlir/test/Dialect/OpenMP/invalid.mlir
@@ -3764,6 +3764,91 @@ omp.private {type = private} @x.privatizer : f32 copy {
// -----
+// expected-error @below {{`linear` clauses do not require a `copy` region.}}
+omp.private {type = linear} @x.privatizer : f32 copy {
+^bb0(%arg0: f32, %arg1 : f32):
+ omp.yield(%arg0 : f32)
+}
+
+// -----
+
+// expected-error @below {{only `linear` clauses can have a `data_address` region.}}
+omp.private {type = private} @x.privatizer : !llvm.ptr data_address {
+^bb0(%arg0: !llvm.ptr):
+ omp.yield(%arg0 : !llvm.ptr)
+}
+
+// -----
+
+// expected-error @below {{`data_address`: expected 1 region arguments, got: 2}}
+omp.private {type = linear} @x.privatizer : !llvm.ptr data_address {
+^bb0(%arg0: !llvm.ptr, %arg1: !llvm.ptr):
+ omp.yield(%arg0 : !llvm.ptr)
+}
+
+// -----
+
+omp.private {type = linear} @x.privatizer : !llvm.ptr data_address {
+^bb0(%arg0: !llvm.ptr):
+ // expected-error @below {{Invalid yielded value. Expected a single value, got: None}}
+ omp.yield
+}
+
+// -----
+
+omp.private {type = linear} @x.privatizer : !llvm.ptr
+
+func.func @linear_privatizer_in_private(%x : !llvm.ptr, %lb : i32) {
+ // expected-error @below {{`linear` privatizer '@x.privatizer' can only be used in a `linear` clause}}
+ omp.simd private(@x.privatizer %x -> %arg0 : !llvm.ptr) {
+ omp.loop_nest (%iv) : i32 = (%lb) to (%lb) step (%lb) {
+ omp.yield
+ }
+ }
+ return
+}
+
+// -----
+
+omp.private {type = private} @x.privatizer : i32
+
+func.func @linear_with_private_privatizer(%x : !llvm.ptr, %lb : i32) {
+ // expected-error @below {{privatizer '@x.privatizer' of a linear variable must be `linear`}}
+ omp.simd linear(@x.privatizer %x -> %arg0 : !llvm.ptr = %lb : i32) linear_var_types([i32]) {
+ omp.loop_nest (%iv) : i32 = (%lb) to (%lb) step (%lb) {
+ omp.yield
+ }
+ }
+ return
+}
+
+// -----
+
+func.func @linear_without_privatizer(%x : !llvm.ptr, %lb : i32) {
+ // expected-error @below {{expected as many linear privatizers as linear variables}}
+ "omp.wsloop"(%x, %lb) ({
+ ^bb0(%arg0: !llvm.ptr):
+ omp.loop_nest (%iv) : i32 = (%lb) to (%lb) step (%lb) {
+ omp.yield
+ }
+ }) {linear_var_types = [i32],
+ operandSegmentSizes = array<i32: 0, 0, 1, 1, 0, 0, 0>} : (!llvm.ptr, i32) -> ()
+ return
+}
+
+// -----
+
+omp.private {type = linear} @x.privatizer : i32
+
+func.func @declare_simd_linear_privatizer(%x : i32, %step : i32) {
+ // expected-error @below {{'omp.declare_simd' op linear privatizers are not allowed}}
+ "omp.declare_simd"(%x, %step) <{linear_syms = [@x.privatizer],
+ operandSegmentSizes = array<i32: 0, 1, 1, 0>}> : (i32, i32) -> ()
+ return
+}
+
+// -----
+
// expected-error @below {{`firstprivate` clauses require at least a `copy` region.}}
omp.private {type = firstprivate} @x.privatizer : f32 init {
^bb0(%arg0: f32, %arg1: f32):
@@ -4809,10 +4894,12 @@ func.func @omp_declare_simd_branch() -> () {
// -----
+omp.private {type = linear} @lin.i32 : i32
+
func.func @omp_wsloop_linear_ref(%lb : index, %ub : index, %step : index,
%data_var : memref<i32>, %linear_var : i32) {
// expected-error @+1 {{'omp.wsloop' op linear modifier 'ref' may only be specified on a declare simd directive}}
- omp.wsloop linear(ref(%data_var : memref<i32> = %linear_var : i32)) linear_var_types([i32]) {
+ omp.wsloop linear(ref(@lin.i32 %data_var -> %l0 : memref<i32> = %linear_var : i32)) linear_var_types([i32]) {
omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
omp.yield
}
@@ -4822,10 +4909,12 @@ func.func @omp_wsloop_linear_ref(%lb : index, %ub : index, %step : index,
// -----
+omp.private {type = linear} @lin.i32 : i32
+
func.func @omp_wsloop_linear_uval(%lb : index, %ub : index, %step : index,
%data_var : memref<i32>, %linear_var : i32) {
// expected-error @+1 {{'omp.wsloop' op linear modifier 'uval' may only be specified on a declare simd directive}}
- omp.wsloop linear(uval(%data_var : memref<i32> = %linear_var : i32)) linear_var_types([i32]) {
+ omp.wsloop linear(uval(@lin.i32 %data_var -> %l0 : memref<i32> = %linear_var : i32)) linear_var_types([i32]) {
omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
omp.yield
}
@@ -4835,10 +4924,12 @@ func.func @omp_wsloop_linear_uval(%lb : index, %ub : index, %step : index,
// -----
+omp.private {type = linear} @lin.i32 : i32
+
func.func @omp_simd_linear_ref(%lb : index, %ub : index, %step : index,
%data_var : memref<i32>, %linear_var : i32) {
// expected-error @+1 {{'omp.simd' op linear modifier 'ref' may only be specified on a declare simd directive}}
- omp.simd linear(ref(%data_var : memref<i32> = %linear_var : i32)) linear_var_types([i32]) {
+ omp.simd linear(ref(@lin.i32 %data_var -> %l0 : memref<i32> = %linear_var : i32)) linear_var_types([i32]) {
omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
omp.yield
}
@@ -4852,6 +4943,7 @@ func.func @omp_wsloop_linear_modifiers_mismatch(%lb : index, %ub : index, %step
%data_var : memref<i32>, %linear_var : i32) {
// expected-error @below {{'omp.wsloop' op expected as many linear modifiers as linear variables}}
"omp.wsloop"(%data_var, %linear_var) ({
+ ^bb0(%l0: memref<i32>):
omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
omp.yield
}
@@ -4866,6 +4958,7 @@ func.func @omp_simd_linear_modifiers_mismatch(%lb : index, %ub : index, %step :
%data_var : memref<i32>, %linear_var : i32) {
// expected-error @below {{'omp.simd' op expected as many linear modifiers as linear variables}}
"omp.simd"(%data_var, %linear_var) ({
+ ^bb0(%l0: memref<i32>):
omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
omp.yield
}
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index 7416c3da3b3951..a9e3efdf843daa 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -503,6 +503,8 @@ func.func @omp_loop_nest_pretty_multiple_collapse_tiles(%lb1 : i32, %ub1 : i32,
return
}
+omp.private {type = linear} @lin.i32 : i32
+
// CHECK-LABEL: omp_wsloop
func.func @omp_wsloop(%lb : index, %ub : index, %step : index, %data_var : memref<i32>, %linear_var : i32, %chunk_var : i32) -> () {
@@ -515,33 +517,36 @@ func.func @omp_wsloop(%lb : index, %ub : index, %step : index, %data_var : memre
}) {operandSegmentSizes = array<i32: 0,0,0,0,0,0,0>, ordered = 1} :
() -> ()
- // CHECK: omp.wsloop linear(%{{.*}} : memref<i32> = %{{.*}} : i32) linear_var_types([i32]) schedule(static) {
+ // CHECK: omp.wsloop schedule(static) linear(@lin.i32 %{{.*}} -> %{{.*}} : memref<i32> = %{{.*}} : i32) linear_var_types([i32]) {
// CHECK-NEXT: omp.loop_nest
"omp.wsloop" (%data_var, %linear_var) ({
+ ^bb0(%l0: memref<i32>):
omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
omp.yield
}
}) {operandSegmentSizes = array<i32: 0,0,1,1,0,0,0>, schedule_kind = #omp.schedulekind<static>,
- linear_var_types = [i32]} : (memref<i32>, i32) -> ()
+ linear_var_types = [i32], linear_syms = [@lin.i32]} : (memref<i32>, i32) -> ()
- // CHECK: omp.wsloop linear(%{{.*}} : memref<i32> = %{{.*}} : i32, %{{.*}} : memref<i32> = %{{.*}} : i32) linear_var_types([i32, i32]) schedule(static) {
+ // CHECK: omp.wsloop schedule(static) linear(@lin.i32 %{{.*}} -> %{{.*}} : memref<i32> = %{{.*}} : i32, @lin.i32 %{{.*}} -> %{{.*}} : memref<i32> = %{{.*}} : i32) linear_var_types([i32, i32]) {
// CHECK-NEXT: omp.loop_nest
"omp.wsloop" (%data_var, %data_var, %linear_var, %linear_var) ({
+ ^bb0(%l0: memref<i32>, %l1: memref<i32>):
omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
omp.yield
}
}) {operandSegmentSizes = array<i32: 0,0,2,2,0,0,0>, schedule_kind = #omp.schedulekind<static>,
- linear_var_types = [i32,i32]} :
+ linear_var_types = [i32,i32], linear_syms = [@lin.i32, @lin.i32]} :
(memref<i32>, memref<i32>, i32, i32) -> ()
- // CHECK: omp.wsloop linear(%{{.*}} : memref<i32> = %{{.*}} : i32) linear_var_types([i32]) ordered(2) schedule(dynamic = %{{.*}}) {
+ // CHECK: omp.wsloop ordered(2) schedule(dynamic = %{{.*}}) linear(@lin.i32 %{{.*}} -> %{{.*}} : memref<i32> = %{{.*}} : i32) linear_var_types([i32]) {
// CHECK-NEXT: omp.loop_nest
"omp.wsloop" (%data_var, %linear_var, %chunk_var) ({
+ ^bb0(%l0: memref<i32>):
omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
omp.yield
}
}) {operandSegmentSizes = array<i32: 0,0,1,1,0,0,1>, schedule_kind = #omp.schedulekind<dynamic>, ordered = 2,
- linear_var_types = [i32]} : (memref<i32>, i32, i32) -> ()
+ linear_var_types = [i32], linear_syms = [@lin.i32]} : (memref<i32>, i32, i32) -> ()
// CHECK: omp.wsloop nowait schedule(auto) {
// CHECK-NEXT: omp.loop_nest
@@ -577,42 +582,42 @@ func.func @omp_wsloop_pretty(%lb : index, %ub : index, %step : index, %data_var
}
}
- // CHECK: omp.wsloop linear(%{{.*}} : memref<i32> = %{{.*}} : i32) linear_var_types([i32]) schedule(static) {
+ // CHECK: omp.wsloop schedule(static) linear(@lin.i32 %{{.*}} -> %{{.*}} : memref<i32> = %{{.*}} : i32) linear_var_types([i32]) {
// CHECK-NEXT: omp.loop_nest
- omp.wsloop schedule(static) linear(%data_var : memref<i32> = %linear_var : i32) linear_var_types([i32]) {
+ omp.wsloop schedule(static) linear(@lin.i32 %data_var -> %l0 : memref<i32> = %linear_var : i32) linear_var_types([i32]) {
omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
omp.yield
}
}
- // CHECK: omp.wsloop linear(val(%{{.*}} : memref<i32> = %{{.*}} : i32), val(%{{.*}} : memref<i32> = %{{.*}} : i32)) linear_var_types([i32, i32]) schedule(static) {
+ // CHECK: omp.wsloop schedule(static) linear(val(@lin.i32 %{{.*}} -> %{{.*}} : memref<i32> = %{{.*}} : i32), val(@lin.i32 %{{.*}} -> %{{.*}} : memref<i32> = %{{.*}} : i32)) linear_var_types([i32, i32]) {
// CHECK-NEXT: omp.loop_nest
- omp.wsloop schedule(static) linear(val(%data_var : memref<i32> = %linear_var : i32),
- val(%data_var : memref<i32> = %linear_var : i32)) linear_var_types([i32, i32]) {
+ omp.wsloop schedule(static) linear(val(@lin.i32 %data_var -> %l0 : memref<i32> = %linear_var : i32),
+ val(@lin.i32 %data_var -> %l1 : memref<i32> = %linear_var : i32)) linear_var_types([i32, i32]) {
omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
omp.yield
}
}
- // CHECK: omp.wsloop linear(%{{.*}} : memref<i32> = %{{.*}} : i32) linear_var_types([i32]) ordered(2) schedule(static = %{{.*}} : i32) {
+ // CHECK: omp.wsloop ordered(2) schedule(static = %{{.*}} : i32) linear(@lin.i32 %{{.*}} -> %{{.*}} : memref<i32> = %{{.*}} : i32) linear_var_types([i32]) {
// CHECK-NEXT: omp.loop_nest
- omp.wsloop ordered(2) linear(%data_var : memref<i32> = %linear_var : i32) linear_var_types([i32]) schedule(static = %chunk_var : i32) {
+ omp.wsloop ordered(2) schedule(static = %chunk_var : i32) linear(@lin.i32 %data_var -> %l0 : memref<i32> = %linear_var : i32) linear_var_types([i32]) {
omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
omp.yield
}
}
- // CHECK: omp.wsloop linear(%{{.*}} : memref<i32> = %{{.*}} : i32) linear_var_types([i32]) ordered(2) schedule(dynamic = %{{.*}} : i32, nonmonotonic) {
+ // CHECK: omp.wsloop ordered(2) schedule(dynamic = %{{.*}} : i32, nonmonotonic) linear(@lin.i32 %{{.*}} -> %{{.*}} : memref<i32> = %{{.*}} : i32) linear_var_types([i32]) {
// CHECK-NEXT: omp.loop_nest
- omp.wsloop ordered(2) linear(%data_var : memref<i32> = %linear_var : i32) linear_var_types([i32]) schedule(dynamic = %chunk_var : i32, nonmonotonic) {
+ omp.wsloop ordered(2) schedule(dynamic = %chunk_var : i32, nonmonotonic) linear(@lin.i32 %data_var -> %l0 : memref<i32> = %linear_var : i32) linear_var_types([i32]) {
omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
omp.yield
}
}
- // CHECK: omp.wsloop linear(%{{.*}} : memref<i32> = %{{.*}} : i32) linear_var_types([i32]) ordered(2) schedule(dynamic = %{{.*}} : i16, monotonic) {
+ // CHECK: omp.wsloop ordered(2) schedule(dynamic = %{{.*}} : i16, monotonic) linear(@lin.i32 %{{.*}} -> %{{.*}} : memref<i32> = %{{.*}} : i32) linear_var_types([i32]) {
// CHECK-NEXT: omp.loop_nest
- omp.wsloop ordered(2) linear(%data_var : memref<i32> = %linear_var : i32) linear_var_types([i32]) schedule(dynamic = %chunk_var2 : i16, monotonic) {
+ omp.wsloop ordered(2) schedule(dynamic = %chunk_var2 : i16, monotonic) linear(@lin.i32 %data_var -> %l0 : memref<i32> = %linear_var : i32) linear_var_types([i32]) {
omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
omp.yield
}
@@ -817,8 +822,8 @@ func.func @omp_simd_pretty_order(%lb : index, %ub : index, %step : index) -> ()
// CHECK-LABEL: omp_simd_linear_val
func.func @omp_simd_linear_val(%lb : index, %ub : index, %step : index,
%data_var : memref<i32>, %linear_var : i32) -> () {
- // CHECK: omp.simd linear(val(%{{.*}} : memref<i32> = %{{.*}} : i32)) linear_var_types([i32])
- omp.simd linear(val(%data_var : memref<i32> = %linear_var : i32)) linear_var_types([i32]) {
+ // CHECK: omp.simd linear(val(@lin.i32 %{{.*}} -> %{{.*}} : memref<i32> = %{{.*}} : i32)) linear_var_types([i32])
+ omp.simd linear(val(@lin.i32 %data_var -> %l0 : memref<i32> = %linear_var : i32)) linear_var_types([i32]) {
omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
omp.yield
}
@@ -830,9 +835,9 @@ func.func @omp_simd_linear_val(%lb : index, %ub : index, %step : index,
func.func @omp_simd_linear_val_multiple(%lb : index, %ub : index, %step : index,
%d1 : memref<i32>, %d2 : memref<i32>,
%s1 : i32, %s2 : i32) -> () {
- // CHECK: omp.simd linear(val(%{{.*}} : memref<i32> = %{{.*}} : i32), val(%{{.*}} : memref<i32> = %{{.*}} : i32)) linear_var_types([i32, i32])
- omp.simd linear(val(%d1 : memref<i32> = %s1 : i32),
- val(%d2 : memref<i32> = %s2 : i32)) linear_var_types([i32, i32]) {
+ // CHECK: omp.simd linear(val(@lin.i32 %{{.*}} -> %{{.*}} : memref<i32> = %{{.*}} : i32), val(@lin.i32 %{{.*}} -> %{{.*}} : memref<i32> = %{{.*}} : i32)) linear_var_types([i32, i32])
+ omp.simd linear(val(@lin.i32 %d1 -> %l0 : memref<i32> = %s1 : i32),
+ val(@lin.i32 %d2 -> %l1 : memref<i32> = %s2 : i32)) linear_var_types([i32, i32]) {
omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
omp.yield
}
@@ -4691,3 +4696,27 @@ func.func @omp_dispatch_nocontext_novariants(%cond : i1, %x : memref<i32>) -> ()
// CHECK-LABEL: func.func private @variant(memref<i32>)
func.func private @variant(memref<i32>) -> ()
+
+// CHECK-LABEL: omp.private {type = linear} @linear.privatizer : !llvm.ptr init {
+omp.private {type = linear} @linear.privatizer : !llvm.ptr init {
+^bb0(%arg0: !llvm.ptr, %arg1: !llvm.ptr):
+ omp.yield(%arg1 : !llvm.ptr)
+// CHECK: } data_address {
+} data_address {
+// CHECK: ^bb0(%{{.*}}: !llvm.ptr):
+^bb0(%arg0: !llvm.ptr):
+ %0 = llvm.load %arg0 : !llvm.ptr -> !llvm.ptr
+ // CHECK: omp.yield(%{{.*}} : !llvm.ptr)
+ omp.yield(%0 : !llvm.ptr)
+}
+
+// CHECK-LABEL: func @simd_linear_privatizer
+func.func @simd_linear_privatizer(%x : !llvm.ptr, %y : !llvm.ptr, %lb : i32, %ub : i32, %step : i32) {
+ // CHECK: omp.simd linear(@linear.privatizer %{{.*}} -> %{{.*}} : !llvm.ptr = %{{.*}} : i32) linear_var_types([i32]) private(@x.privatizer %{{.*}} -> %{{.*}} : !llvm.ptr)
+ omp.simd linear(@linear.privatizer %x -> %arg0 : !llvm.ptr = %step : i32) linear_var_types([i32]) private(@x.privatizer %y -> %arg1 : !llvm.ptr) {
+ omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
+ omp.yield
+ }
+ }
+ return
+}
diff --git a/mlir/test/Target/LLVMIR/omptarget-wsloop.mlir b/mlir/test/Target/LLVMIR/omptarget-wsloop.mlir
index 86dc03e1818848..7819dc04aec588 100644
--- a/mlir/test/Target/LLVMIR/omptarget-wsloop.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-wsloop.mlir
@@ -54,11 +54,13 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
llvm.return
}
+ omp.private {type = linear} @lin.i32 : i32
+
llvm.func @target_wsloop_linear(%arg0: !llvm.ptr) attributes {omp.declare_target = #omp.declaretarget<device_type = any, capture_clause = to>} {
%loop_ub = llvm.mlir.constant(9 : i32) : i32
%loop_lb = llvm.mlir.constant(0 : i32) : i32
%loop_step = llvm.mlir.constant(1 : i32) : i32
- omp.wsloop linear(%arg0 : !llvm.ptr = %loop_step : i32) linear_var_types([i32]) {
+ omp.wsloop linear(@lin.i32 %arg0 -> %arg1 : !llvm.ptr = %loop_step : i32) linear_var_types([i32]) {
omp.loop_nest (%loop_cnt) : i32 = (%loop_lb) to (%loop_ub) inclusive step (%loop_step) {
omp.yield
}
diff --git a/mlir/test/Target/LLVMIR/openmp-linear-privatizer.mlir b/mlir/test/Target/LLVMIR/openmp-linear-privatizer.mlir
new file mode 100644
index 00000000000000..3f3b78f9c4e570
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/openmp-linear-privatizer.mlir
@@ -0,0 +1,148 @@
+// Linear vars whose `linear` privatizer has a `data_address` region.
+// RUN: mlir-translate -mlir-to-llvmir -split-input-file %s | FileCheck %s
+
+llvm.func @malloc(i64) -> !llvm.ptr
+llvm.func @free(!llvm.ptr)
+
+// A "box" holding a pointer to an i32.
+omp.private {type = linear} @box.linear : !llvm.ptr init {
+^bb0(%mold: !llvm.ptr, %priv: !llvm.ptr):
+ %c4 = llvm.mlir.constant(4 : i64) : i64
+ %mem = llvm.call @malloc(%c4) : (i64) -> !llvm.ptr
+ llvm.store %mem, %priv : !llvm.ptr, !llvm.ptr
+ omp.yield(%priv : !llvm.ptr)
+} data_address {
+^bb0(%box: !llvm.ptr):
+ %data = llvm.load %box : !llvm.ptr -> !llvm.ptr
+ omp.yield(%data : !llvm.ptr)
+} dealloc {
+^bb0(%box: !llvm.ptr):
+ %data = llvm.load %box : !llvm.ptr -> !llvm.ptr
+ llvm.call @free(%data) : (!llvm.ptr) -> ()
+ omp.yield
+}
+
+// CHECK-LABEL: define void @simd_linear_privatizer(
+// CHECK-SAME: ptr %[[BOX:.*]], ptr %{{.*}})
+// CHECK: %[[PRIV:.*]] = alloca ptr
+// CHECK: %[[START:.linear_var]] = alloca i32
+// CHECK: omp_loop.preheader:
+// CHECK: %[[ORIG_DATA:.*]] = load ptr, ptr %[[BOX]]
+// CHECK: %[[V:.*]] = load i32, ptr %[[ORIG_DATA]]
+// CHECK: store i32 %[[V]], ptr %[[START]]
+// CHECK: omp_loop.body:
+// CHECK: %[[PRIV_DATA:.*]] = load ptr, ptr %[[PRIV]]
+// CHECK: %[[S:.*]] = load i32, ptr %[[START]]
+// CHECK: %[[MUL:.*]] = mul i32 %{{.*}}, 1
+// CHECK: %[[ADD:.*]] = add i32 %[[S]], %[[MUL]]
+// CHECK: store i32 %[[ADD]], ptr %[[PRIV_DATA]]
+// CHECK: omp.region.cont:
+// CHECK: %[[PRIV_DATA2:.*]] = load ptr, ptr %[[PRIV]]
+// CHECK: %[[LAST:.*]] = load i32, ptr %[[PRIV_DATA2]]
+// CHECK: %[[ORIG_DATA2:.*]] = load ptr, ptr %[[BOX]]
+// CHECK: store i32 %[[LAST]], ptr %[[ORIG_DATA2]]
+// CHECK: call void @free(
+llvm.func @simd_linear_privatizer(%box : !llvm.ptr, %out : !llvm.ptr) {
+ %c1 = llvm.mlir.constant(1 : i32) : i32
+ %c10 = llvm.mlir.constant(10 : i32) : i32
+ omp.simd linear(@box.linear %box -> %arg0 : !llvm.ptr = %c1 : i32) linear_var_types([i32]) {
+ omp.loop_nest (%iv) : i32 = (%c1) to (%c10) inclusive step (%c1) {
+ %data = llvm.load %arg0 : !llvm.ptr -> !llvm.ptr
+ %v = llvm.load %data : !llvm.ptr -> i32
+ llvm.store %v, %out : i32, !llvm.ptr
+ omp.yield
+ }
+ }
+ llvm.return
+}
+
+// CHECK-LABEL: define void @wsloop_linear_privatizer(
+// CHECK-SAME: ptr %[[BOX:.*]], ptr %{{.*}})
+// CHECK: %[[PRIV:.*]] = alloca ptr
+// CHECK: %[[START:.linear_var]] = alloca i32
+// CHECK: omp_loop.preheader:
+// CHECK: %[[ORIG_DATA:.*]] = load ptr, ptr %[[BOX]]
+// CHECK: %[[V:.*]] = load i32, ptr %[[ORIG_DATA]]
+// CHECK: store i32 %[[V]], ptr %[[START]]
+// CHECK: omp_loop.body:
+// CHECK: %[[PRIV_DATA:.*]] = load ptr, ptr %[[PRIV]]
+// CHECK: %[[S:.*]] = load i32, ptr %[[START]]
+// CHECK: %[[MUL:.*]] = mul i32 %{{.*}}, 2
+// CHECK: %[[ADD:.*]] = add i32 %[[S]], %[[MUL]]
+// CHECK: store i32 %[[ADD]], ptr %[[PRIV_DATA]]
+// CHECK: omp_loop.linear_lastiter_exit:
+// CHECK: %[[PRIV_DATA2:.*]] = load ptr, ptr %[[PRIV]]
+// CHECK: %[[LAST:.*]] = load i32, ptr %[[PRIV_DATA2]]
+// CHECK: %[[ORIG_DATA2:.*]] = load ptr, ptr %[[BOX]]
+// CHECK: store i32 %[[LAST]], ptr %[[ORIG_DATA2]]
+// CHECK: omp.region.cont:
+// CHECK: call void @free(
+llvm.func @wsloop_linear_privatizer(%box : !llvm.ptr, %out : !llvm.ptr) {
+ %c1 = llvm.mlir.constant(1 : i32) : i32
+ %c2 = llvm.mlir.constant(2 : i32) : i32
+ %c10 = llvm.mlir.constant(10 : i32) : i32
+ omp.wsloop linear(@box.linear %box -> %arg0 : !llvm.ptr = %c2 : i32) linear_var_types([i32]) {
+ omp.loop_nest (%iv) : i32 = (%c1) to (%c10) inclusive step (%c1) {
+ %data = llvm.load %arg0 : !llvm.ptr -> !llvm.ptr
+ %v = llvm.load %data : !llvm.ptr -> i32
+ llvm.store %v, %out : i32, !llvm.ptr
+ omp.yield
+ }
+ }
+ llvm.return
+}
+
+// The loop IV is stored through the data address of the private copy; it gets
+// one more step after the loop.
+// CHECK-LABEL: define void @simd_linear_privatizer_iv(
+// CHECK-SAME: ptr %[[BOX:.*]])
+// CHECK: %[[PRIV:.*]] = alloca ptr
+// CHECK: omp.region.cont:
+// CHECK: %[[DATA:.*]] = load ptr, ptr %[[PRIV]]
+// CHECK: %[[LAST:.*]] = load i32, ptr %[[DATA]]
+// CHECK: %[[NEXT:.*]] = add i32 %[[LAST]], 2
+// CHECK: store i32 %[[NEXT]], ptr %[[DATA]]
+// CHECK: %[[DATA2:.*]] = load ptr, ptr %[[PRIV]]
+// CHECK: %[[FINAL:.*]] = load i32, ptr %[[DATA2]]
+// CHECK: %[[ORIG:.*]] = load ptr, ptr %[[BOX]]
+// CHECK: store i32 %[[FINAL]], ptr %[[ORIG]]
+// CHECK: call void @free(
+llvm.func @simd_linear_privatizer_iv(%box : !llvm.ptr) {
+ %c1 = llvm.mlir.constant(1 : i32) : i32
+ %c2 = llvm.mlir.constant(2 : i32) : i32
+ %c5 = llvm.mlir.constant(5 : i32) : i32
+ omp.simd linear(@box.linear %box -> %arg0 : !llvm.ptr = %c2 : i32) linear_var_types([i32]) {
+ omp.loop_nest (%iv) : i32 = (%c1) to (%c5) inclusive step (%c2) {
+ %data = llvm.load %arg0 : !llvm.ptr -> !llvm.ptr
+ llvm.store %iv, %data : i32, !llvm.ptr
+ omp.yield
+ }
+ }
+ llvm.return
+}
+
+// Same, with the private copy copied before its data address is loaded.
+// CHECK-LABEL: define void @simd_linear_privatizer_iv_copy(
+// CHECK: %[[PRIV:omp.private.alloc]] = alloca ptr
+// CHECK: omp.region.cont:
+// CHECK: %[[DATA:.*]] = load ptr, ptr %[[PRIV]]
+// CHECK: %[[LAST:.*]] = load i32, ptr %[[DATA]]
+// CHECK: %[[NEXT:.*]] = add i32 %[[LAST]], 2
+// CHECK: store i32 %[[NEXT]], ptr %[[DATA]]
+llvm.func @simd_linear_privatizer_iv_copy(%box : !llvm.ptr) {
+ %c1 = llvm.mlir.constant(1 : i32) : i32
+ %c2 = llvm.mlir.constant(2 : i32) : i32
+ %c5 = llvm.mlir.constant(5 : i32) : i32
+ %c8 = llvm.mlir.constant(8 : i32) : i32
+ %one = llvm.mlir.constant(1 : i64) : i64
+ %tmp = llvm.alloca %one x !llvm.ptr : (i64) -> !llvm.ptr
+ omp.simd linear(@box.linear %box -> %arg0 : !llvm.ptr = %c2 : i32) linear_var_types([i32]) {
+ omp.loop_nest (%iv) : i32 = (%c1) to (%c5) inclusive step (%c2) {
+ "llvm.intr.memcpy"(%tmp, %arg0, %c8) <{isVolatile = false}> : (!llvm.ptr, !llvm.ptr, i32) -> ()
+ %data = llvm.load %tmp : !llvm.ptr -> !llvm.ptr
+ llvm.store %iv, %data : i32, !llvm.ptr
+ omp.yield
+ }
+ }
+ llvm.return
+}
diff --git a/mlir/test/Target/LLVMIR/openmp-llvm-invalid.mlir b/mlir/test/Target/LLVMIR/openmp-llvm-invalid.mlir
index f2a45c7ed7e32a..89b0a3cc069e8b 100644
--- a/mlir/test/Target/LLVMIR/openmp-llvm-invalid.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-llvm-invalid.mlir
@@ -92,9 +92,11 @@ llvm.func @omp_threadprivate() {
// -----
+omp.private {type = linear} @lin.i32 : i32
+
llvm.func @wsloop_linear(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
// expected-error @below {{Ill-formed type attributes for linear variables}}
- omp.wsloop linear(%x : !llvm.ptr = %step : i32) linear_var_types([]) {
+ omp.wsloop linear(@lin.i32 %x -> %lin0 : !llvm.ptr = %step : i32) linear_var_types([]) {
omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
omp.yield
}
@@ -104,9 +106,11 @@ llvm.func @wsloop_linear(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
// -----
+omp.private {type = linear} @lin.i32 : i32
+
llvm.func @simd_linear(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
// expected-error @below {{Ill-formed type attributes for linear variables}}
- omp.simd linear(%x : !llvm.ptr = %step : i32) linear_var_types([]) {
+ omp.simd linear(@lin.i32 %x -> %lin0 : !llvm.ptr = %step : i32) linear_var_types([]) {
omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
omp.yield
}
@@ -117,9 +121,11 @@ llvm.func @simd_linear(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
// -----
omp.private {type = private} @i_private_i32 : i32
+omp.private {type = linear} @lin.i32 : i32
+
llvm.func @simd_linear_private(%lb : i32, %ub : i32, %step : i32, %i : !llvm.ptr) {
// expected-error @below {{linear variables cannot appear in other data-sharing clauses}}
- omp.simd linear(%i : !llvm.ptr = %step : i32) linear_var_types([i32])
+ omp.simd linear(@lin.i32 %i -> %lin0 : !llvm.ptr = %step : i32) linear_var_types([i32])
private(@i_private_i32 %i -> %priv_i : !llvm.ptr) {
omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
omp.yield
@@ -130,13 +136,15 @@ llvm.func @simd_linear_private(%lb : i32, %ub : i32, %step : i32, %i : !llvm.ptr
// -----
+omp.private {type = linear} @lin.i32 : i32
+
llvm.func @simd_linear_ambiguous_iv(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr, %y : !llvm.ptr) {
// expected-error @below {{Could not determine the linear variable associated with the loop nest induction variable}}
// expected-error @below {{LLVM Translation failed for operation: omp.simd}}
- omp.simd linear(%x : !llvm.ptr = %step : i32, %y : !llvm.ptr = %step : i32) linear_var_types([i32, i32]) {
+ omp.simd linear(@lin.i32 %x -> %lin0 : !llvm.ptr = %step : i32, @lin.i32 %y -> %lin1 : !llvm.ptr = %step : i32) linear_var_types([i32, i32]) {
omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
- llvm.store %iv, %x : i32, !llvm.ptr
- llvm.store %iv, %y : i32, !llvm.ptr
+ llvm.store %iv, %lin0 : i32, !llvm.ptr
+ llvm.store %iv, %lin1 : i32, !llvm.ptr
omp.yield
}
}
diff --git a/mlir/test/Target/LLVMIR/openmp-llvm.mlir b/mlir/test/Target/LLVMIR/openmp-llvm.mlir
index 96b60d586baed8..ce6d3858ea4ddd 100644
--- a/mlir/test/Target/LLVMIR/openmp-llvm.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-llvm.mlir
@@ -351,15 +351,17 @@ llvm.func @test_omp_masked(%arg0: i32)-> () {
// -----
+omp.private {type = linear} @lin.i32 : i32
+
llvm.func @wsloop_linear(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
// CHECK-LABEL: @wsloop_linear
+// CHECK: %[[LINEAR_RESULT:omp.private.alloc]] = alloca i32, align 4
+// CHECK: %[[LINEAR_VAR:.linear_var]] = alloca i32, align 4
// CHECK: %p.lastiter = alloca i32, align 4
// CHECK: %p.lowerbound = alloca i32, align 4
// CHECK: %p.upperbound = alloca i32, align 4
// CHECK: %p.stride = alloca i32, align 4
-// CHECK: %[[LINEAR_VAR:.*]] = alloca i32, align 4
-// CHECK: %[[LINEAR_RESULT:.*]] = alloca i32, align 4
// CHECK: omp_loop.preheader:
// CHECK: %[[LOAD:.*]] = load i32, ptr %{{.*}}, align 4
@@ -387,7 +389,7 @@ llvm.func @wsloop_linear(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
// CHECK: call void @__kmpc_barrier(ptr {{.*}}, i32 %[[THREAD_ID]])
// CHECK: br label %omp_loop.after
- omp.wsloop linear(%x : !llvm.ptr = %step : i32) linear_var_types([i32]) {
+ omp.wsloop linear(@lin.i32 %x -> %lin0 : !llvm.ptr = %step : i32) linear_var_types([i32]) {
omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
omp.yield
}
@@ -764,13 +766,15 @@ llvm.func @simd_simple(%lb : i64, %ub : i64, %step : i64, %arg0: !llvm.ptr) {
// -----
+omp.private {type = linear} @lin.i32 : i32
+
llvm.func @simd_linear(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
// CHECK-LABEL: @simd_linear
// CHECK-SAME: (i32 %{{.*}}, i32 %{{.*}}, i32 %[[STEP:.*]], ptr %[[X:.*]])
-// CHECK: %[[LINEAR_VAR:.*]] = alloca i32, align 4
-// CHECK: %[[LINEAR_RESULT:.*]] = alloca i32, align 4
+// CHECK: %[[LINEAR_RESULT:omp.private.alloc]] = alloca i32, align 4
+// CHECK: %[[LINEAR_VAR:.linear_var]] = alloca i32, align 4
// CHECK: omp_loop.preheader:
// CHECK: %[[LOAD:.*]] = load i32, ptr {{.*}}, align 4
@@ -790,10 +794,10 @@ llvm.func @simd_linear(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
// CHECK-NEXT: store i32 %[[ADD]], ptr %[[LINEAR_RESULT]]
// CHECK-NEXT: %[[LOAD:.*]] = load i32, ptr %[[LINEAR_RESULT]]
// CHECK-NEXT: store i32 %[[LOAD]], ptr %[[X]], align 4
- omp.simd linear(%x : !llvm.ptr = %step : i32) linear_var_types([i32]) {
+ omp.simd linear(@lin.i32 %x -> %lin0 : !llvm.ptr = %step : i32) linear_var_types([i32]) {
omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
- llvm.store %iv, %x : i32, !llvm.ptr
- llvm.store %iv, %x : i32, !llvm.ptr
+ llvm.store %iv, %lin0 : i32, !llvm.ptr
+ llvm.store %iv, %lin0 : i32, !llvm.ptr
omp.yield
}
}
@@ -802,6 +806,8 @@ llvm.func @simd_linear(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
// -----
+omp.private {type = linear} @lin.i64 : i64
+
// Test linear clause with mismatched types (i32 step, i64 variable)
// This is a regression test for issue #173332
llvm.func @simd_linear_i64_var_i32_step(%lb : i32, %ub : i32, %x : !llvm.ptr) {
@@ -809,8 +815,8 @@ llvm.func @simd_linear_i64_var_i32_step(%lb : i32, %ub : i32, %x : !llvm.ptr) {
// CHECK-LABEL: @simd_linear_i64_var_i32_step
-// CHECK: %[[LINEAR_VAR:.*]] = alloca i64
-// CHECK: %[[LINEAR_RESULT:.*]] = alloca i64
+// CHECK: %[[LINEAR_RESULT:omp.private.alloc]] = alloca i64
+// CHECK: %[[LINEAR_VAR:.linear_var]] = alloca i64
// CHECK: omp_loop.preheader:
// CHECK: %[[LOAD:.*]] = load i64, ptr {{.*}}
@@ -824,7 +830,7 @@ llvm.func @simd_linear_i64_var_i32_step(%lb : i32, %ub : i32, %x : !llvm.ptr) {
// CHECK: %[[MUL:.*]] = mul i64 %[[IV_I64]], {{.*}}
// CHECK: %[[ADD:.*]] = add i64 %[[LOAD]], %[[MUL]]
// CHECK: store i64 %[[ADD]], ptr %[[LINEAR_RESULT]], {{.*}}!llvm.access.group
- omp.simd linear(%x : !llvm.ptr = %step : i32) linear_var_types([i64]) {
+ omp.simd linear(@lin.i64 %x -> %lin0 : !llvm.ptr = %step : i32) linear_var_types([i64]) {
omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
omp.yield
}
@@ -834,6 +840,8 @@ llvm.func @simd_linear_i64_var_i32_step(%lb : i32, %ub : i32, %x : !llvm.ptr) {
// -----
+omp.private {type = linear} @lin.f64 : f64
+
// Test linear clause with floating-point variable and i32 step
// This tests the floating-point path in updateLinearVar
llvm.func @simd_linear_f64_var_i32_step(%lb : i32, %ub : i32, %x : !llvm.ptr) {
@@ -841,8 +849,8 @@ llvm.func @simd_linear_f64_var_i32_step(%lb : i32, %ub : i32, %x : !llvm.ptr) {
// CHECK-LABEL: @simd_linear_f64_var_i32_step
-// CHECK: %[[LINEAR_VAR:.*]] = alloca double
-// CHECK: %[[LINEAR_RESULT:.*]] = alloca double
+// CHECK: %[[LINEAR_RESULT:omp.private.alloc]] = alloca double
+// CHECK: %[[LINEAR_VAR:.linear_var]] = alloca double
// CHECK: omp_loop.preheader:
// CHECK: %[[LOAD:.*]] = load double, ptr {{.*}}
@@ -856,7 +864,7 @@ llvm.func @simd_linear_f64_var_i32_step(%lb : i32, %ub : i32, %x : !llvm.ptr) {
// CHECK-NEXT: %[[MUL_FP:.*]] = sitofp i32 %[[MUL_INT]] to double
// CHECK-NEXT: %[[ADD:.*]] = fadd double %[[LOAD]], %[[MUL_FP]]
// CHECK-NEXT: store double %[[ADD]], ptr %[[LINEAR_RESULT]], {{.*}}!llvm.access.group
- omp.simd linear(%x : !llvm.ptr = %step : i32) linear_var_types([f64]) {
+ omp.simd linear(@lin.f64 %x -> %lin0 : !llvm.ptr = %step : i32) linear_var_types([f64]) {
omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
omp.yield
}
@@ -866,15 +874,16 @@ llvm.func @simd_linear_f64_var_i32_step(%lb : i32, %ub : i32, %x : !llvm.ptr) {
// -----
+omp.private {type = linear} @lin.i32 : i32
+
// Test the update of omp.simd linear iteration variables, when nested inside
// omp.wsloop.
llvm.func @wsloop_simd_linear(%x : !llvm.ptr) {
// CHECK-LABEL: @wsloop_simd_linear
-// CHECK: omp.wsloop.region:
-// CHECK: %[[LINEAR_VAR:.*]] = alloca i32
-// CHECK: %[[LINEAR_RESULT:.*]] = alloca i32
+// CHECK: %[[LINEAR_RESULT:omp.private.alloc]] = alloca i32
+// CHECK: %[[LINEAR_VAR:.linear_var]] = alloca i32
// CHECK: omp.region.cont2:
// CHECK: %[[VAL:.*]] = load i32, ptr %[[LINEAR_RESULT]]
@@ -884,9 +893,9 @@ llvm.func @wsloop_simd_linear(%x : !llvm.ptr) {
%ub = llvm.mlir.constant(100 : i32) : i32
%step = llvm.mlir.constant(25 : i32) : i32
omp.wsloop {
- omp.simd linear(%x : !llvm.ptr = %step : i32) linear_var_types([i32]) {
+ omp.simd linear(@lin.i32 %x -> %lin0 : !llvm.ptr = %step : i32) linear_var_types([i32]) {
omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
- llvm.store %iv, %x : i32, !llvm.ptr
+ llvm.store %iv, %lin0 : i32, !llvm.ptr
omp.yield
}
} {omp.composite}
diff --git a/mlir/test/Target/LLVMIR/openmp-simd-guided.mlir b/mlir/test/Target/LLVMIR/openmp-simd-guided.mlir
index e274918828af12..7bbb4a3b8f8f5a 100644
--- a/mlir/test/Target/LLVMIR/openmp-simd-guided.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-simd-guided.mlir
@@ -1,6 +1,8 @@
// Ensure that schedule can be used with the guided kind-type and simd construct
// RUN: mlir-translate -mlir-to-llvmir -split-input-file %s | FileCheck %s
+omp.private {type = linear} @lin.i32 : i32
+
llvm.func @test_simd_guided() {
%0 = llvm.mlir.constant (1 : i64) : i64
%c1_i32 = llvm.mlir.constant (1 : i32) : i32
@@ -9,7 +11,7 @@ llvm.func @test_simd_guided() {
%c4_i32 = llvm.mlir.constant (4 : i32) : i32
%1 = llvm.alloca %0 x i32 {bindc_name = "i"} : (i64) -> !llvm.ptr
omp.wsloop schedule(guided = %c4_i32 : i32, simd) {
- omp.simd linear(%1 : !llvm.ptr = %c1_i32 : i32) linear_var_types([i32]) {
+ omp.simd linear(@lin.i32 %1 -> %lin0 : !llvm.ptr = %c1_i32 : i32) linear_var_types([i32]) {
omp.loop_nest (%arg1) : i32 = (%c0_i32) to (%c64_i32) inclusive step (%c1_i32) {
omp.yield
}
diff --git a/mlir/test/Target/LLVMIR/openmp-simd-linear.mlir b/mlir/test/Target/LLVMIR/openmp-simd-linear.mlir
index af31abb1de90f1..ce52dd288810ed 100644
--- a/mlir/test/Target/LLVMIR/openmp-simd-linear.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-simd-linear.mlir
@@ -4,11 +4,19 @@
// -----
-// CHECK-LABEL: void @test_simd_linear()
-// CHECK-NOT: %.linear_result
-// CHECK: %.linear_result = alloca i32
+// CHECK-LABEL: void @test_simd_linear..omp_par
+// CHECK: omp.par.entry:
+// CHECK: %[[PRIV:omp.private.alloc[0-9]+]] = alloca i32
+// CHECK: %[[START:.linear_var]] = alloca i32
+// CHECK: omp_loop.preheader{{[0-9]+}}:
+// CHECK: store i32 %{{.*}}, ptr %[[START]]
+// CHECK: omp_loop.body{{[0-9]+}}:
+// CHECK: load i32, ptr %[[START]]
+// CHECK: store i32 %{{.*}}, ptr %[[PRIV]]
omp.private {type = private} @test_simd_linear_private_i32 : i32
+omp.private {type = linear} @lin.i32 : i32
+
llvm.func @test_simd_linear() {
%0 = llvm.mlir.constant(1 : i64) : i64
%1 = llvm.mlir.constant(1 : i32) : i32
@@ -21,9 +29,9 @@ llvm.func @test_simd_linear() {
omp.loop_nest (%arg1) : i32 = (%1) to (%3) inclusive step (%1) {
llvm.store %arg1, %arg0 : i32, !llvm.ptr
llvm.store %2, %4 : i32, !llvm.ptr
- omp.simd linear(%4 : !llvm.ptr = %1 : i32) linear_var_types([i32]) {
+ omp.simd linear(@lin.i32 %4 -> %lin0 : !llvm.ptr = %1 : i32) linear_var_types([i32]) {
omp.loop_nest (%arg2) : i32 = (%1) to (%3) inclusive step (%1) {
- llvm.store %arg2, %4 : i32, !llvm.ptr
+ llvm.store %arg2, %lin0 : i32, !llvm.ptr
omp.yield
}
}
@@ -39,10 +47,19 @@ llvm.func @test_simd_linear() {
// -----
// CHECK-LABEL: @test_simd_linear2({{.*}})
+// CHECK: %[[I:.*]] = alloca i32, i64 0
+// CHECK: %[[PRIV:omp.private.alloc]] = alloca i32
// CHECK: omp.loop_nest.region:
-// CHECK: store i32 %{{.*}}, ptr %.linear_result
-// CHECK: %{{.*}} = load i32, ptr %.linear_result
-// CHECK: omp.region.cont1:
+// CHECK: store i32 %{{.*}}, ptr %[[PRIV]]
+// CHECK: %{{.*}} = load i32, ptr %[[PRIV]]
+// CHECK: omp.region.cont:
+// CHECK: %[[LAST:.*]] = load i32, ptr %[[PRIV]]
+// CHECK: %[[NEXT:.*]] = add i32 %[[LAST]], 1
+// CHECK: store i32 %[[NEXT]], ptr %[[PRIV]]
+// CHECK: %[[FINAL:.*]] = load i32, ptr %[[PRIV]]
+// CHECK: store i32 %[[FINAL]], ptr %[[I]]
+
+omp.private {type = linear} @lin.i32 : i32
llvm.func @test_simd_linear2(%a : !llvm.ptr) {
%c0_i64 = llvm.mlir.constant(0 : i64) : i64
@@ -64,10 +81,10 @@ llvm.func @test_simd_linear2(%a : !llvm.ptr) {
llvm.br ^bb1(%i_next, %iv_next : i32, i64)
^bb3: // pred: ^bb1
llvm.store %i, %i_ptr : i32, !llvm.ptr
- omp.simd linear(%i_ptr : !llvm.ptr = %c1_i32 : i32) linear_var_types([i32]) {
+ omp.simd linear(@lin.i32 %i_ptr -> %lin0 : !llvm.ptr = %c1_i32 : i32) linear_var_types([i32]) {
omp.loop_nest (%arg0) : i32 = (%c1_i32) to (%c100_i32) inclusive step (%c1_i32) {
- llvm.store %arg0, %i_ptr : i32, !llvm.ptr
- %i2 = llvm.load %i_ptr : !llvm.ptr -> i32
+ llvm.store %arg0, %lin0 : i32, !llvm.ptr
+ %i2 = llvm.load %lin0 : !llvm.ptr -> i32
%3 = llvm.sext %i2 : i32 to i64
%4 = llvm.sub %3, %c1_i64 overflow<nsw, nuw> : i64
%5 = llvm.mul %4, %c1_i64 overflow<nsw, nuw> : i64
diff --git a/mlir/test/Target/LLVMIR/openmp-simd-ordered.mlir b/mlir/test/Target/LLVMIR/openmp-simd-ordered.mlir
index e7cd456c77119c..fcf562a936eccd 100644
--- a/mlir/test/Target/LLVMIR/openmp-simd-ordered.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-simd-ordered.mlir
@@ -1,7 +1,7 @@
// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
// Test that linear variables in SIMD loops with ordered regions
-// are correctly rewritten to use .linear_result in:
+// use the private copy of the linear variable in:
// 1. The ordered region (omp.ordered.region)
// 2. Code after the ordered region (omp_region.finalize)
//
@@ -17,6 +17,8 @@
// !$omp end simd
module {
+ omp.private {type = linear} @lin.i32 : i32
+
// CHECK-LABEL: define void @simd_ordered_linear
llvm.func @simd_ordered_linear() {
%c0_i64 = llvm.mlir.constant(0 : i64) : i64
@@ -33,17 +35,17 @@ module {
%c = llvm.alloca %c100_i64 x i32 : (i64) -> !llvm.ptr
%i = llvm.alloca %c1_i64 x i32 {bindc_name = "i"} : (i64) -> !llvm.ptr
+ // CHECK: %[[PRIV:omp.private.alloc]] = alloca i32
// CHECK: %.linear_var = alloca i32
- // CHECK: %.linear_result = alloca i32
- omp.simd linear(%i : !llvm.ptr = %c1_i32 : i32) linear_var_types([i32]) {
+ omp.simd linear(@lin.i32 %i -> %lin0 : !llvm.ptr = %c1_i32 : i32) linear_var_types([i32]) {
omp.loop_nest (%iv) : i32 = (%c1_i32) to (%c10_i32) inclusive step (%c1_i32) {
// CHECK: omp.loop_nest.region:
- // CHECK: load i32, ptr %.linear_result
- llvm.store %iv, %i : i32, !llvm.ptr
+ // CHECK: load i32, ptr %[[PRIV]]
+ llvm.store %iv, %lin0 : i32, !llvm.ptr
// Compute a[i] = b[i] * 10
- %i_val = llvm.load %i : !llvm.ptr -> i32
+ %i_val = llvm.load %lin0 : !llvm.ptr -> i32
%i_idx = llvm.sext %i_val : i32 to i64
%i_off = llvm.sub %i_idx, %c1_i64 : i64
%b_ptr = llvm.getelementptr %b[%i_off] : (!llvm.ptr, i64) -> !llvm.ptr, i32
@@ -55,8 +57,8 @@ module {
// Ordered region
omp.ordered.region par_level_simd {
// CHECK: omp.ordered.region:
- // CHECK: load i32, ptr %.linear_result
- %i_ord = llvm.load %i : !llvm.ptr -> i32
+ // CHECK: load i32, ptr %[[PRIV]]
+ %i_ord = llvm.load %lin0 : !llvm.ptr -> i32
%i_ord_idx = llvm.sext %i_ord : i32 to i64
%i_ord_off = llvm.sub %i_ord_idx, %c1_i64 : i64
%a_ord_ptr = llvm.getelementptr %a[%i_ord_off] : (!llvm.ptr, i64) -> !llvm.ptr, i32
@@ -66,8 +68,8 @@ module {
// Compute c[i] = a[i] * 2 (code after ordered region)
// CHECK: omp_region.finalize:
- // CHECK: load i32, ptr %.linear_result
- %i_post = llvm.load %i : !llvm.ptr -> i32
+ // CHECK: load i32, ptr %[[PRIV]]
+ %i_post = llvm.load %lin0 : !llvm.ptr -> i32
%i_post_idx = llvm.sext %i_post : i32 to i64
%i_post_off = llvm.sub %i_post_idx, %c1_i64 : i64
%a_post_ptr = llvm.getelementptr %a[%i_post_off] : (!llvm.ptr, i64) -> !llvm.ptr, i32
diff --git a/mlir/test/Target/LLVMIR/openmp-wsloop-linear.mlir b/mlir/test/Target/LLVMIR/openmp-wsloop-linear.mlir
index 503800c9d1a8fa..bc1b1f5d628f98 100644
--- a/mlir/test/Target/LLVMIR/openmp-wsloop-linear.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-wsloop-linear.mlir
@@ -3,25 +3,29 @@
// -----
-// %.linear_result must appear only in the loop body and in the
+// The private copy must appear only in the loop body and in the
// linear_lastiter_exit block, where it's used to update the original
// variable.
// CHECK-LABEL: void @wsloop_linear_post_use({{.*}})
-// CHECK: %.linear_result = alloca i32
-// CHECK-NOT: %.linear_result
+// CHECK: %[[PRIV:omp.private.alloc]] = alloca i32
+// CHECK-NOT: %[[PRIV]]
// CHECK: omp_loop.body:
-// CHECK: %.linear_result
+// CHECK: store i32 %{{.*}}, ptr %[[PRIV]]
+// CHECK: omp.loop_nest.region:
+// CHECK: load i32, ptr %[[PRIV]]
// CHECK: omp_loop.inc:
-// CHECK-NOT: %.linear_result
+// CHECK-NOT: %[[PRIV]]
// CHECK: omp_loop.linear_lastiter_exit:
-// CHECK: load i32, ptr %.linear_result
-// CHECK-NOT: %.linear_result
+// CHECK: load i32, ptr %[[PRIV]]
+// CHECK-NOT: %[[PRIV]]
+
+omp.private {type = linear} @lin.i32 : i32
llvm.func @wsloop_linear_post_use(%lb : i32, %ub : i32, %step : i32,
%x : !llvm.ptr, %out : !llvm.ptr) {
- omp.wsloop linear(%x : !llvm.ptr = %step : i32) linear_var_types([i32]) {
+ omp.wsloop linear(@lin.i32 %x -> %lin0 : !llvm.ptr = %step : i32) linear_var_types([i32]) {
omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
- %cur = llvm.load %x : !llvm.ptr -> i32
+ %cur = llvm.load %lin0 : !llvm.ptr -> i32
llvm.store %cur, %out : i32, !llvm.ptr
omp.yield
}
diff --git a/mlir/test/Target/LLVMIR/openmp-wsloop-simd-ordered.mlir b/mlir/test/Target/LLVMIR/openmp-wsloop-simd-ordered.mlir
index a31b05840a7df2..ca2ee3ee917bf0 100644
--- a/mlir/test/Target/LLVMIR/openmp-wsloop-simd-ordered.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-wsloop-simd-ordered.mlir
@@ -1,7 +1,7 @@
// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
// Test that linear variables in worksharing+SIMD loops with ordered regions
-// are correctly rewritten to use .linear_result in:
+// use the private copy of the linear variable in:
// 1. The ordered region (omp.ordered.region)
// 2. Code after the ordered region (omp_region.finalize)
//
@@ -17,6 +17,8 @@
// !$omp end do simd
module {
+ omp.private {type = linear} @lin.i32 : i32
+
// CHECK-LABEL: define void @wsloop_simd_ordered_linear
llvm.func @wsloop_simd_ordered_linear() {
%c0_i64 = llvm.mlir.constant(0 : i64) : i64
@@ -33,18 +35,18 @@ module {
%c = llvm.alloca %c100_i64 x i32 : (i64) -> !llvm.ptr
%i = llvm.alloca %c1_i64 x i32 {bindc_name = "i"} : (i64) -> !llvm.ptr
+ // CHECK: %[[PRIV:omp.private.alloc]] = alloca i32
// CHECK: %.linear_var = alloca i32
- // CHECK: %.linear_result = alloca i32
omp.wsloop ordered(0) {
- omp.simd linear(%i : !llvm.ptr = %c1_i32 : i32) linear_var_types([i32]) {
+ omp.simd linear(@lin.i32 %i -> %lin0 : !llvm.ptr = %c1_i32 : i32) linear_var_types([i32]) {
omp.loop_nest (%iv) : i32 = (%c1_i32) to (%c100_i32) inclusive step (%c1_i32) {
// CHECK: omp.loop_nest.region:
- // CHECK: load i32, ptr %.linear_result
- llvm.store %iv, %i : i32, !llvm.ptr
+ // CHECK: load i32, ptr %[[PRIV]]
+ llvm.store %iv, %lin0 : i32, !llvm.ptr
// Compute a[i] = b[i] * 10
- %i_val = llvm.load %i : !llvm.ptr -> i32
+ %i_val = llvm.load %lin0 : !llvm.ptr -> i32
%i_idx = llvm.sext %i_val : i32 to i64
%i_off = llvm.sub %i_idx, %c1_i64 : i64
%b_ptr = llvm.getelementptr %b[%i_off] : (!llvm.ptr, i64) -> !llvm.ptr, i32
@@ -56,8 +58,8 @@ module {
// Ordered region
omp.ordered.region par_level_simd {
// CHECK: omp.ordered.region:
- // CHECK: load i32, ptr %.linear_result
- %i_ord = llvm.load %i : !llvm.ptr -> i32
+ // CHECK: load i32, ptr %[[PRIV]]
+ %i_ord = llvm.load %lin0 : !llvm.ptr -> i32
%i_ord_idx = llvm.sext %i_ord : i32 to i64
%i_ord_off = llvm.sub %i_ord_idx, %c1_i64 : i64
%a_ord_ptr = llvm.getelementptr %a[%i_ord_off] : (!llvm.ptr, i64) -> !llvm.ptr, i32
@@ -68,8 +70,8 @@ module {
// Compute c[i] = a[i] * 2 (code after ordered region)
// CHECK: omp_region.finalize:
- // CHECK: load i32, ptr %.linear_result
- %i_post = llvm.load %i : !llvm.ptr -> i32
+ // CHECK: load i32, ptr %[[PRIV]]
+ %i_post = llvm.load %lin0 : !llvm.ptr -> i32
%i_post_idx = llvm.sext %i_post : i32 to i64
%i_post_off = llvm.sub %i_post_idx, %c1_i64 : i64
%a_post_ptr = llvm.getelementptr %a[%i_post_off] : (!llvm.ptr, i64) -> !llvm.ptr, i32
More information about the Mlir-commits
mailing list