[flang-commits] [flang] [llvm] [mlir] [openmp] [Flang][OpenMP] Lower scan directive and inscan reduction modifier (PR #206747)
CHANDRA GHALE via flang-commits
flang-commits at lists.llvm.org
Mon Aug 3 02:38:19 PDT 2026
https://github.com/chandraghale updated https://github.com/llvm/llvm-project/pull/206747
>From 82416bc775c4befdbcce1f999a5f5b97ef954cd8 Mon Sep 17 00:00:00 2001
From: Chandra Ghale <ghale at pe34genoa.hpc.amslabs.hpecorp.net>
Date: Tue, 30 Jun 2026 10:00:56 -0500
Subject: [PATCH 1/9] Lower scan directive and inscan reduction modifier
---
flang/lib/Lower/OpenMP/OpenMP.cpp | 57 ++-
.../Lower/OpenMP/Todo/nested-wsloop-scan.f90 | 34 ++
.../OpenMP/Todo/wsloop-scan-collapse.f90 | 29 ++
.../OpenMP/OpenMPToLLVMIRTranslation.cpp | 476 +++++++++++++++---
.../Target/LLVMIR/openmp-reduction-scan.mlir | 123 +++++
mlir/test/Target/LLVMIR/openmp-todo.mlir | 41 +-
openmp/runtime/test/scan/scan.f90 | 38 ++
7 files changed, 726 insertions(+), 72 deletions(-)
create mode 100644 flang/test/Lower/OpenMP/Todo/nested-wsloop-scan.f90
create mode 100644 flang/test/Lower/OpenMP/Todo/wsloop-scan-collapse.f90
create mode 100644 mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
create mode 100644 openmp/runtime/test/scan/scan.f90
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index ea8c279962508..e77584a142eea 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -2972,12 +2972,59 @@ genParallelOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
static mlir::omp::ScanOp
genScanOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
- semantics::SemanticsContext &semaCtx, mlir::Location loc,
- const ConstructQueue &queue, ConstructQueue::const_iterator item) {
+ semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
+ mlir::Location loc, const ConstructQueue &queue,
+ ConstructQueue::const_iterator item) {
mlir::omp::ScanOperands clauseOps;
genScanClauses(converter, semaCtx, item->clauses, loc, clauseOps);
- return mlir::omp::ScanOp::create(converter.getFirOpBuilder(),
- converter.getCurrentLocation(), clauseOps);
+ mlir::omp::ScanOp scanOp = mlir::omp::ScanOp::create(
+ converter.getFirOpBuilder(), converter.getCurrentLocation(), clauseOps);
+
+ /// Scan reduction is not implemented with nested workshare loops, linear
+ /// clause, tiling
+ mlir::omp::LoopNestOp loopNestOp =
+ scanOp->getParentOfType<mlir::omp::LoopNestOp>();
+ llvm::SmallVector<mlir::omp::LoopWrapperInterface> loopWrappers;
+ loopNestOp.gatherWrappers(loopWrappers);
+ mlir::Operation *loopWrapperOp = loopWrappers.front().getOperation();
+ if (llvm::isa<mlir::omp::SimdOp>(loopWrapperOp))
+ TODO(loc, "unsupported simd");
+ if (loopWrappers.size() > 1)
+ TODO(loc, "unsupported composite");
+ mlir::omp::WsloopOp wsLoopOp = llvm::cast<mlir::omp::WsloopOp>(loopWrapperOp);
+ bool isNested =
+ (loopNestOp.getNumLoops() > 1) ||
+ (wsLoopOp && (wsLoopOp->getParentOfType<mlir::omp::WsloopOp>()));
+ if (isNested)
+ TODO(loc, "Scan directive inside nested workshare loops");
+ if (wsLoopOp && !wsLoopOp.getLinearVars().empty())
+ TODO(loc, "Scan directive with linear clause");
+ if (loopNestOp.getTileSizes())
+ TODO(loc, "Scan directive with loop tiling");
+
+ // All loop indices should be loaded after the scan construct as otherwise,
+ // it would result in using the index variable across scan directive.
+ // (`Intra-iteration dependences from a statement in the structured
+ // block sequence that precede a scan directive to a statement in the
+ // structured block sequence that follows a scan directive must not exist,
+ // except for dependences for the list items specified in an inclusive or
+ // exclusive clause.`).
+ // TODO: Nested loops are not handled.
+ mlir::Region ®ion = loopNestOp->getRegion(0);
+ mlir::Value indexVal = fir::getBase(region.getArgument(0));
+ lower::pft::Evaluation *doConstructEval = eval.parentConstruct;
+ fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
+ lower::pft::Evaluation *doLoop = &doConstructEval->getFirstNestedEvaluation();
+ auto *doStmt = doLoop->getIf<parser::NonLabelDoStmt>();
+ assert(doStmt && "Expected do loop to be in the nested evaluation");
+ const auto &loopControl =
+ std::get<std::optional<parser::LoopControl>>(doStmt->t);
+ const parser::LoopControl::Bounds *bounds =
+ std::get_if<parser::LoopControl::Bounds>(&loopControl->u);
+ mlir::Operation *storeOp =
+ setLoopVar(converter, loc, indexVal, bounds->Name().thing.symbol);
+ firOpBuilder.setInsertionPointAfter(storeOp);
+ return scanOp;
}
static mlir::omp::SectionsOp
@@ -4305,7 +4352,7 @@ static void genOMPDispatch(lower::AbstractConverter &converter,
loc, queue, item);
break;
case llvm::omp::Directive::OMPD_scan:
- newOp = genScanOp(converter, symTable, semaCtx, loc, queue, item);
+ newOp = genScanOp(converter, symTable, semaCtx, eval, loc, queue, item);
break;
case llvm::omp::Directive::OMPD_section:
llvm_unreachable("genOMPDispatch: OMPD_section");
diff --git a/flang/test/Lower/OpenMP/Todo/nested-wsloop-scan.f90 b/flang/test/Lower/OpenMP/Todo/nested-wsloop-scan.f90
new file mode 100644
index 0000000000000..414e2ef2d5aed
--- /dev/null
+++ b/flang/test/Lower/OpenMP/Todo/nested-wsloop-scan.f90
@@ -0,0 +1,34 @@
+! Tests scan reduction behavior when used in nested workshare loops
+
+! RUN: %not_todo_cmd %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
+
+program nested_scan_example
+ implicit none
+ integer, parameter :: n = 4, m = 5
+ integer :: a(n, m), b(n, m)
+ integer :: i, j
+ integer :: row_sum, col_sum
+
+ do i = 1, n
+ do j = 1, m
+ a(i, j) = i + j
+ end do
+ end do
+
+ !$omp parallel do reduction(inscan, +: row_sum) private(col_sum, j)
+ do i = 1, n
+ row_sum = row_sum + i
+ !$omp scan inclusive(row_sum)
+
+ col_sum = 0
+ !$omp parallel do reduction(inscan, +: col_sum)
+ do j = 1, m
+ col_sum = col_sum + a(i, j)
+ !CHECK: not yet implemented: Scan directive inside nested workshare loops
+ !$omp scan inclusive(col_sum)
+ b(i, j) = col_sum + row_sum
+ end do
+ !$omp end parallel do
+ end do
+ !$omp end parallel do
+end program nested_scan_example
diff --git a/flang/test/Lower/OpenMP/Todo/wsloop-scan-collapse.f90 b/flang/test/Lower/OpenMP/Todo/wsloop-scan-collapse.f90
new file mode 100644
index 0000000000000..b8e6e831884ab
--- /dev/null
+++ b/flang/test/Lower/OpenMP/Todo/wsloop-scan-collapse.f90
@@ -0,0 +1,29 @@
+! Tests scan reduction behavior when used in nested workshare loops
+
+! RUN: %not_todo_cmd %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
+
+program nested_loop_example
+ implicit none
+ integer :: i, j, x
+ integer, parameter :: N = 100, M = 200
+ real :: A(N, M), B(N, M)
+ x = 0
+
+ do i = 1, N
+ do j = 1, M
+ A(i, j) = i * j
+ end do
+ end do
+
+ !$omp parallel do collapse(2) reduction(inscan, +:x)
+ do i = 1, N
+ do j = 1, M
+ x = x + A(i,j)
+ !CHECK: not yet implemented: Scan directive inside nested workshare loops
+ !$omp scan inclusive(x)
+ B(i, j) = x
+ end do
+ end do
+ !$omp end parallel do
+
+end program nested_loop_example
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index d860c408e0fd2..297ebac51de88 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -41,6 +41,8 @@
#include "llvm/TargetParser/Triple.h"
#include "llvm/Transforms/Utils/ModuleUtils.h"
+#include <cassert>
+#include <cstddef>
#include <cstdint>
#include <iterator>
#include <numeric>
@@ -84,6 +86,10 @@ class OpenMPAllocStackFrame
: allocInsertPoint(allocaIP), deallocBlocks(deallocBlocks) {}
llvm::OpenMPIRBuilder::InsertPointTy allocInsertPoint;
llvm::SmallVector<llvm::BasicBlock *> deallocBlocks;
+ /// Set to true when this alloca frame encloses an omp.parallel operation.
+ /// The alloca insertion point of a function in which a parallel op is
+ /// defined may be used to allocate the temporary buffer for scan reductions.
+ bool containsParallelOp = false;
};
/// Stack frame to hold a \see llvm::CanonicalLoopInfo representing the
@@ -93,7 +99,17 @@ class OpenMPLoopInfoStackFrame
: public StateStackFrameBase<OpenMPLoopInfoStackFrame> {
public:
MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(OpenMPLoopInfoStackFrame)
+ /// For constructs like `scan`, a single `omp.loop_nest` is split into an
+ /// input loop and a scan loop. In that case `loopInfo` holds the input loop
+ /// info and `scanloopInfo` holds the scan loop info.
llvm::CanonicalLoopInfo *loopInfo = nullptr;
+ llvm::CanonicalLoopInfo *scanloopInfo = nullptr;
+ llvm::ScanInfo *scanInfo = nullptr;
+ /// Map reduction variables to their LLVM types. Populated when the reduction
+ /// clause is processed and used when a `scan` directive is encountered in the
+ /// loop body.
+ std::unique_ptr<llvm::DenseMap<llvm::Value *, llvm::Type *>>
+ reductionVarToType = nullptr;
};
/// Custom error class to signal translation errors that don't need reporting,
@@ -359,6 +375,10 @@ static LogicalResult checkImplementationStatus(Operation &op) {
if (!op.getDependVars().empty() || op.getDependKinds())
result = todo("depend");
};
+ auto checkExclusive = [&todo](auto op, LogicalResult &result) {
+ if (!op.getExclusiveVars().empty())
+ result = todo("exclusive");
+ };
auto checkHint = [](auto op, LogicalResult &) {
if (op.getHint())
op.emitWarning("hint clause discarded");
@@ -389,21 +409,27 @@ static LogicalResult checkImplementationStatus(Operation &op) {
op.getReductionMod().value() != omp::ReductionModifier::defaultmod) {
omp::ReductionModifier mod = op.getReductionMod().value();
// The `task` reduction modifier is supported on the parallel and
- // worksharing (do/for and sections) constructs. Other modifiers, and the
- // `task` modifier on other constructs, are not yet implemented.
+ // worksharing (do/for and sections) constructs. The `inscan` modifier is
+ // supported on the worksharing-loop construct (it is translated as a scan
+ // reduction). Other modifiers, and these modifiers on other constructs,
+ // are not yet implemented.
bool taskModifierSupported =
mod == omp::ReductionModifier::task &&
isa<omp::ParallelOp, omp::WsloopOp, omp::SectionsOp>(op);
- if (!taskModifierSupported) {
+ bool inscanModifierSupported =
+ mod == omp::ReductionModifier::inscan && isa<omp::WsloopOp>(op);
+ if (!taskModifierSupported && !inscanModifierSupported) {
result = todo("reduction with modifier");
- } else if (auto byref = op.getReductionByref()) {
- // The task reduction modifier lowering only handles non-byref
- // reductions for now.
- for (bool isByRef : *byref)
- if (isByRef) {
- result = todo("task reduction modifier with by-ref reduction");
- break;
- }
+ } else if (taskModifierSupported) {
+ if (auto byref = op.getReductionByref()) {
+ // The task reduction modifier lowering only handles non-byref
+ // reductions for now.
+ for (bool isByRef : *byref)
+ if (isByRef) {
+ result = todo("task reduction modifier with by-ref reduction");
+ break;
+ }
+ }
}
}
};
@@ -460,6 +486,7 @@ static LogicalResult checkImplementationStatus(Operation &op) {
checkAllocate(op, result);
checkOrder(op, result);
})
+ .Case([&](omp::ScanOp op) { checkExclusive(op, result); })
.Case([&](omp::SectionsOp op) {
checkAllocate(op, result);
checkPrivate(op, result);
@@ -650,6 +677,97 @@ findCurrentLoopInfo(LLVM::ModuleTranslation &moduleTranslation) {
return loopInfo;
}
+/// Find the scan loop information structure for the scan loop nest being
+/// translated. It will return a `null` value unless called from the
+/// translation function for a loop wrapper operation after successfully
+/// translating its body.
+static llvm::CanonicalLoopInfo *
+findCurrentScanLoopInfo(LLVM::ModuleTranslation &moduleTranslation) {
+ llvm::CanonicalLoopInfo *scanLoopInfo = nullptr;
+ moduleTranslation.stackWalk<OpenMPLoopInfoStackFrame>(
+ [&](OpenMPLoopInfoStackFrame &frame) {
+ scanLoopInfo = frame.scanloopInfo;
+ return WalkResult::interrupt();
+ });
+ return scanLoopInfo;
+}
+
+/// Find the `ScanInfo` stored on the loop stack frame. Upon encountering an
+/// `inscan` reduction modifier, `scanInfoInitialize` initializes the
+/// `ScanInfo`, which is then used when a `scan` directive is encountered in the
+/// body of the loop nest.
+static llvm::ScanInfo *
+findScanInfo(LLVM::ModuleTranslation &moduleTranslation) {
+ llvm::ScanInfo *scanInfo = nullptr;
+ moduleTranslation.stackWalk<OpenMPLoopInfoStackFrame>(
+ [&](OpenMPLoopInfoStackFrame &frame) {
+ scanInfo = frame.scanInfo;
+ return WalkResult::interrupt();
+ });
+ return scanInfo;
+}
+
+/// The types of reduction variables are used for lowering a `scan` directive
+/// that appears in the body of the loop. The types are stored in the loop frame
+/// when the reduction clause is encountered and used when the `scan` directive
+/// is encountered.
+static llvm::DenseMap<llvm::Value *, llvm::Type *> *
+findReductionVarTypes(LLVM::ModuleTranslation &moduleTranslation) {
+ llvm::DenseMap<llvm::Value *, llvm::Type *> *reductionVarToType = nullptr;
+ moduleTranslation.stackWalk<OpenMPLoopInfoStackFrame>(
+ [&](OpenMPLoopInfoStackFrame &frame) {
+ if (!frame.reductionVarToType)
+ frame.reductionVarToType =
+ std::make_unique<llvm::DenseMap<llvm::Value *, llvm::Type *>>();
+ reductionVarToType = frame.reductionVarToType.get();
+ return WalkResult::interrupt();
+ });
+ return reductionVarToType;
+}
+
+/// Scan reduction requires a shared buffer to be allocated to perform the
+/// reduction. The allocation needs to be done outside the parallel region in
+/// which the scan operation is used.
+static llvm::OpenMPIRBuilder::InsertPointTy
+findParallelAllocaIP(llvm::IRBuilderBase &builder,
+ LLVM::ModuleTranslation &moduleTranslation) {
+ // If there is an alloca insertion point on the stack belonging to a frame
+ // that encloses a parallel op, use it.
+ llvm::OpenMPIRBuilder::InsertPointTy allocaInsertPoint;
+ WalkResult walkResult = moduleTranslation.stackWalk<OpenMPAllocStackFrame>(
+ [&](OpenMPAllocStackFrame &frame) {
+ if (frame.containsParallelOp) {
+ allocaInsertPoint = frame.allocInsertPoint;
+ return WalkResult::interrupt();
+ }
+ return WalkResult::skip();
+ });
+ if (walkResult.wasInterrupted())
+ return allocaInsertPoint;
+ // Otherwise, insert into the entry block of the surrounding function.
+ // If the current IRBuilder InsertPoint is the function's entry, it cannot
+ // also be used for alloca insertion which would result in insertion order
+ // confusion. Create a new BasicBlock for the Builder and use the entry block
+ // for the allocs.
+ // TODO: Create a dedicated alloca BasicBlock at function creation such that
+ // we do not need to move the current InsertPoint here.
+ if (builder.GetInsertBlock() ==
+ &builder.GetInsertBlock()->getParent()->getEntryBlock()) {
+ assert(builder.GetInsertPoint() == builder.GetInsertBlock()->end() &&
+ "Assuming end of basic block");
+ llvm::BasicBlock *entryBB = llvm::BasicBlock::Create(
+ builder.getContext(), "entry", builder.GetInsertBlock()->getParent(),
+ builder.GetInsertBlock()->getNextNode());
+ builder.CreateBr(entryBB);
+ builder.SetInsertPoint(entryBB);
+ }
+
+ llvm::BasicBlock &funcEntryBlock =
+ builder.GetInsertBlock()->getParent()->getEntryBlock();
+ return llvm::OpenMPIRBuilder::InsertPointTy(
+ &funcEntryBlock, funcEntryBlock.getFirstInsertionPt());
+}
+
/// Converts the given region that appears within an OpenMP dialect operation to
/// LLVM IR, creating a branch from the `sourceBlock` to the entry block of the
/// region, and a branch from any block with an successor-less OpenMP terminator
@@ -1414,7 +1532,8 @@ initReductionVars(OP op, ArrayRef<BlockArgument> reductionArgs,
SmallVectorImpl<llvm::Value *> &privateReductionVariables,
DenseMap<Value, llvm::Value *> &reductionVariableMap,
llvm::ArrayRef<bool> isByRef,
- SmallVectorImpl<DeferredStore> &deferredStores) {
+ SmallVectorImpl<DeferredStore> &deferredStores,
+ bool isInScanRegion = false) {
if (op.getNumReductionVars() == 0)
return success();
@@ -1451,11 +1570,17 @@ initReductionVars(OP op, ArrayRef<BlockArgument> reductionArgs,
for (auto [data, addr] : deferredStores)
builder.CreateStore(data, addr);
+ llvm::DenseMap<llvm::Value *, llvm::Type *> *reductionVarToType =
+ findReductionVarTypes(moduleTranslation);
// Before the loop, store the initial values of reductions into reduction
// variables. Although this could be done after allocas, we don't want to mess
// up with the alloca insertion point.
for (unsigned i = 0; i < op.getNumReductionVars(); ++i) {
SmallVector<llvm::Value *, 1> phis;
+ llvm::Type *reductionType =
+ moduleTranslation.convertType(reductionDecls[i].getType());
+ if (isInScanRegion && reductionVarToType != nullptr)
+ (*reductionVarToType)[privateReductionVariables[i]] = reductionType;
// map block argument to initializer region
mapInitializationArgs(op, moduleTranslation, builder, reductionDecls,
@@ -4406,11 +4531,15 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
return failure();
assert(afterAllocas.get()->getSinglePredecessor());
+ bool isInScanRegion =
+ wsloopOp.getReductionMod() && (wsloopOp.getReductionMod().value() ==
+ mlir::omp::ReductionModifier::inscan);
if (failed(initReductionVars(wsloopOp, reductionArgs, builder,
moduleTranslation,
afterAllocas.get()->getSinglePredecessor(),
reductionDecls, privateReductionVariables,
- reductionVariableMap, isByRef, deferredStores)))
+ reductionVariableMap, isByRef, deferredStores,
+ isInScanRegion)))
return failure();
// For `reduction(task, ...)` open a task-reduction scope for the worksharing
@@ -4432,6 +4561,10 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
std::optional<omp::ScheduleModifier> scheduleMod = wsloopOp.getScheduleMod();
bool isSimd = wsloopOp.getScheduleSimd();
bool loopNeedsBarrier = !wsloopOp.getNowait();
+ // TODO: Linear clause support needs to be enabled for scan reduction.
+ if (isInScanRegion)
+ assert(wsloopOp.getLinearVars().empty() &&
+ "Linear clause support is not enabled with scan reduction");
// The only legal way for the direct parent to be omp.distribute is that this
// represents 'distribute parallel do'. Otherwise, this is a regular
@@ -4469,21 +4602,93 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
if (failed(handleError(regionBlock, opInst)))
return failure();
+ // Generates the loop body for a worksharing loop, including linear-variable
+ // handling and the call into the OpenMPIRBuilder's worksharing-loop helper.
+ // For scan reductions this lambda is invoked twice: once for the input loop
+ // and once for the scan loop.
+ const auto &&wsloopCodeGen = [&](llvm::CanonicalLoopInfo *loopInfo,
+ bool noLoopMode,
+ bool inputScanLoop) -> LogicalResult {
+ // Emit Initialization and Update IR for linear variables
+ if (!wsloopOp.getLinearVars().empty()) {
+ linearClauseProcessor.initLinearVar(builder, moduleTranslation,
+ loopInfo->getPreheader());
+ llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterBarrierIP =
+ moduleTranslation.getOpenMPBuilder()->createBarrier(
+ builder.saveIP(), llvm::omp::OMPD_barrier);
+ if (failed(handleError(afterBarrierIP, *loopOp)))
+ return failure();
+ builder.restoreIP(*afterBarrierIP);
+ linearClauseProcessor.updateLinearVar(builder, loopInfo->getBody(),
+ loopInfo->getIndVar());
+ linearClauseProcessor.splitLinearFiniBB(builder, loopInfo->getExit());
+ }
+
+ builder.SetInsertPoint(*regionBlock, (*regionBlock)->begin());
+
+ for (size_t index = 0; index < wsloopOp.getLinearVars().size(); index++)
+ linearClauseProcessor.rewriteInPlace(builder, loopInfo->getBody(),
+ loopInfo->getLatch(), index);
+
+ llvm::OpenMPIRBuilder::InsertPointOrErrorTy wsloopIP =
+ ompBuilder->applyWorkshareLoop(
+ ompLoc.DL, loopInfo, allocaIP, loopNeedsBarrier,
+ convertToScheduleKind(schedule), chunk, isSimd,
+ scheduleMod == omp::ScheduleModifier::monotonic,
+ scheduleMod == omp::ScheduleModifier::nonmonotonic, isOrdered,
+ workshareLoopType, noLoopMode, hasDistSchedule, distScheduleChunk);
+
+ if (failed(handleError(wsloopIP, opInst)))
+ return failure();
+
+ // Emit finalization for linear vars.
+ if (!wsloopOp.getLinearVars().empty()) {
+ llvm::OpenMPIRBuilder::InsertPointTy oldIP = builder.saveIP();
+ assert(loopInfo->getLastIter() &&
+ "`lastiter` in CanonicalLoopInfo is nullptr");
+ llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterBarrierIP =
+ linearClauseProcessor.finalizeLinearVar(builder, moduleTranslation,
+ loopInfo->getLastIter());
+ if (failed(handleError(afterBarrierIP, *loopOp)))
+ return failure();
+
+ builder.restoreIP(oldIP);
+ }
+
+ // For scan reductions, the cancellation finalization callback is popped
+ // only after the scan (second) loop has been generated.
+ if (!inputScanLoop || !isInScanRegion)
+ popCancelFinalizationCB(cancelTerminators, *ompBuilder, wsloopIP.get());
+
+ return success();
+ };
+
llvm::CanonicalLoopInfo *loopInfo = findCurrentLoopInfo(moduleTranslation);
- // Emit Initialization and Update IR for linear variables
- if (!wsloopOp.getLinearVars().empty()) {
- linearClauseProcessor.initLinearVar(builder, moduleTranslation,
- loopInfo->getPreheader());
- llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterBarrierIP =
- moduleTranslation.getOpenMPBuilder()->createBarrier(
- builder.saveIP(), llvm::omp::OMPD_barrier);
- if (failed(handleError(afterBarrierIP, *loopOp)))
+ if (isInScanRegion) {
+ // Emit the scan reduction combiner between the input loop and the scan
+ // loop, operating on the per-iteration partial values stored in the shared
+ // buffer.
+ auto inputLoopFinishIp = loopInfo->getAfterIP();
+ builder.restoreIP(inputLoopFinishIp);
+ SmallVector<OwningReductionGen> owningReductionGens;
+ SmallVector<OwningAtomicReductionGen> owningAtomicReductionGens;
+ SmallVector<llvm::OpenMPIRBuilder::ReductionInfo, 2> reductionInfos;
+ SmallVector<OwningDataPtrPtrReductionGen> owningReductionGenRefDataPtrGens;
+ collectReductionInfo(wsloopOp, builder, moduleTranslation, reductionDecls,
+ owningReductionGens, owningAtomicReductionGens,
+ owningReductionGenRefDataPtrGens,
+ privateReductionVariables, reductionInfos, isByRef);
+ llvm::BasicBlock *cont = splitBB(builder, false, "omp.scan.loop.cont");
+ llvm::ScanInfo *scanInfo = findScanInfo(moduleTranslation);
+ llvm::OpenMPIRBuilder::InsertPointOrErrorTy redIP =
+ ompBuilder->emitScanReduction(builder.saveIP(), reductionInfos,
+ scanInfo);
+ if (failed(handleError(redIP, opInst)))
return failure();
- builder.restoreIP(*afterBarrierIP);
- linearClauseProcessor.updateLinearVar(builder, loopInfo->getBody(),
- loopInfo->getIndVar());
- linearClauseProcessor.splitLinearFiniBB(builder, loopInfo->getExit());
+
+ builder.restoreIP(*redIP);
+ builder.CreateBr(cont);
}
builder.SetInsertPoint(*regionBlock, (*regionBlock)->begin());
@@ -4502,50 +4707,41 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
noLoopMode = true;
}
- for (size_t index = 0; index < wsloopOp.getLinearVars().size(); index++)
- linearClauseProcessor.rewriteInPlace(builder, loopInfo->getBody(),
- loopInfo->getLatch(), index);
-
- llvm::OpenMPIRBuilder::InsertPointOrErrorTy wsloopIP =
- ompBuilder->applyWorkshareLoop(
- ompLoc.DL, loopInfo, allocaIP, loopNeedsBarrier,
- convertToScheduleKind(schedule), chunk, isSimd,
- scheduleMod == omp::ScheduleModifier::monotonic,
- scheduleMod == omp::ScheduleModifier::nonmonotonic, isOrdered,
- workshareLoopType, noLoopMode, hasDistSchedule, distScheduleChunk);
-
- if (failed(handleError(wsloopIP, opInst)))
+ // For scan loops, the input loop does not pop the cancellation finalization
+ // callback; that happens after the scan loop is generated below.
+ bool inputScanLoop = isInScanRegion;
+ if (failed(wsloopCodeGen(loopInfo, noLoopMode, inputScanLoop)))
return failure();
+ inputScanLoop = false;
- // Emit finalization and in-place rewrites for linear vars.
- if (!wsloopOp.getLinearVars().empty()) {
- llvm::OpenMPIRBuilder::InsertPointTy oldIP = builder.saveIP();
- assert(loopInfo->getLastIter() &&
- "`lastiter` in CanonicalLoopInfo is nullptr");
- llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterBarrierIP =
- linearClauseProcessor.finalizeLinearVar(builder, moduleTranslation,
- loopInfo->getLastIter());
- if (failed(handleError(afterBarrierIP, *loopOp)))
+ if (isInScanRegion) {
+ llvm::CanonicalLoopInfo *scanLoopInfo =
+ findCurrentScanLoopInfo(moduleTranslation);
+ if (failed(wsloopCodeGen(scanLoopInfo, noLoopMode, inputScanLoop)))
+ return failure();
+ SmallVector<Region *> reductionRegions;
+ llvm::transform(reductionDecls, std::back_inserter(reductionRegions),
+ [](omp::DeclareReductionOp reductionDecl) {
+ return &reductionDecl.getCleanupRegion();
+ });
+ if (failed(inlineOmpRegionCleanup(
+ reductionRegions, privateReductionVariables, moduleTranslation,
+ builder, "omp.reduction.cleanup")))
return failure();
+ } else {
+ // Close the task-reduction scope before the worksharing reduction combine.
+ if (isTaskReductionMod)
+ emitTaskReductionModifierFini(/*isWorksharing=*/true, builder,
+ moduleTranslation);
- builder.restoreIP(oldIP);
+ // Process the reductions if required.
+ if (failed(createReductionsAndCleanup(
+ wsloopOp, builder, moduleTranslation, allocaIP, reductionDecls,
+ privateReductionVariables, isByRef, wsloopOp.getNowait(),
+ /*isTeamsReduction=*/false)))
+ return failure();
}
- // Set the correct branch target for task cancellation
- popCancelFinalizationCB(cancelTerminators, *ompBuilder, wsloopIP.get());
-
- // Close the task-reduction scope before the worksharing reduction combine.
- if (isTaskReductionMod)
- emitTaskReductionModifierFini(/*isWorksharing=*/true, builder,
- moduleTranslation);
-
- // Process the reductions if required.
- if (failed(createReductionsAndCleanup(
- wsloopOp, builder, moduleTranslation, allocaIP, reductionDecls,
- privateReductionVariables, isByRef, wsloopOp.getNowait(),
- /*isTeamsReduction=*/false)))
- return failure();
-
return cleanupPrivateVars(wsloopOp, builder, moduleTranslation,
wsloopOp.getLoc(), privateVarsInfo);
}
@@ -4579,6 +4775,20 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,
opInst.getReductionMod() == omp::ReductionModifier::task &&
opInst.getNumReductionVars() > 0;
+ // Mark the enclosing alloca stack frame as containing a parallel op. Scan
+ // reductions use the alloca insertion point of the function enclosing the
+ // parallel region to allocate their shared temporary buffer.
+ bool foundParallelOp = false;
+ moduleTranslation.stackWalk<OpenMPAllocStackFrame>(
+ [&](OpenMPAllocStackFrame &frame) {
+ if (foundParallelOp) {
+ frame.containsParallelOp = true;
+ return WalkResult::interrupt();
+ }
+ foundParallelOp = true;
+ return WalkResult::skip();
+ });
+
auto bodyGenCB =
[&](InsertPointTy allocaIP, InsertPointTy codeGenIP,
llvm::ArrayRef<llvm::BasicBlock *> deallocBlocks) -> llvm::Error {
@@ -4972,6 +5182,99 @@ convertOmpSimd(Operation &opInst, llvm::IRBuilderBase &builder,
privateVarsInfo);
}
+static LogicalResult
+convertOmpScan(Operation &opInst, llvm::IRBuilderBase &builder,
+ LLVM::ModuleTranslation &moduleTranslation) {
+ if (failed(checkImplementationStatus(opInst)))
+ return failure();
+ auto scanOp = cast<omp::ScanOp>(opInst);
+ bool isInclusive = scanOp.hasInclusiveVars();
+ SmallVector<llvm::Value *> llvmScanVars;
+ SmallVector<llvm::Type *> llvmScanVarsType;
+ mlir::OperandRange mlirScanVars = scanOp.getInclusiveVars();
+ if (!isInclusive)
+ mlirScanVars = scanOp.getExclusiveVars();
+
+ llvm::DenseMap<llvm::Value *, llvm::Type *> *reductionVarToType =
+ findReductionVarTypes(moduleTranslation);
+ for (auto val : mlirScanVars) {
+ llvm::Value *llvmVal = moduleTranslation.lookupValue(val);
+ llvmScanVars.push_back(llvmVal);
+ llvmScanVarsType.push_back((*reductionVarToType)[llvmVal]);
+ }
+ llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
+ findParallelAllocaIP(builder, moduleTranslation);
+ llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
+ llvm::ScanInfo *scanInfo = findScanInfo(moduleTranslation);
+ llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
+ moduleTranslation.getOpenMPBuilder()->createScan(
+ ompLoc, allocaIP, llvmScanVars, llvmScanVarsType, isInclusive,
+ scanInfo);
+ if (failed(handleError(afterIP, opInst)))
+ return failure();
+ builder.restoreIP(*afterIP);
+ return success();
+}
+
+/// Re-initialize the scan reduction variables to their identity at the start of
+/// each input-loop iteration of a scan reduction. This is required because the
+/// OpenMPIRBuilder stores the per-iteration reduction value into the scan
+/// buffer (`buffer[i] = red`) and `emitScanReduction` then computes the prefix
+/// sum across the buffer. Without resetting `red` to the reduction identity
+/// each iteration, the buffer would hold a running sum and the prefix sum would
+/// be computed twice. This reset only applies to the input loop; in the scan
+/// loop the reduction variable is loaded back from the buffer.
+static LogicalResult
+initScanReductionVars(omp::LoopNestOp loopOp, llvm::IRBuilderBase &builder,
+ LLVM::ModuleTranslation &moduleTranslation) {
+ auto wsloopOp = loopOp->getParentOfType<omp::WsloopOp>();
+ if (!wsloopOp || !wsloopOp.getReductionMod() ||
+ wsloopOp.getReductionMod().value() != omp::ReductionModifier::inscan)
+ return success();
+
+ // Only reset in the input loop; in the scan loop the reduction variable holds
+ // the value loaded from the buffer and must not be overwritten.
+ llvm::ScanInfo *scanInfo = findScanInfo(moduleTranslation);
+ if (!scanInfo || !scanInfo->OMPFirstScanLoop)
+ return success();
+
+ unsigned numReductions = wsloopOp.getNumReductionVars();
+ if (numReductions == 0)
+ return success();
+
+ SmallVector<omp::DeclareReductionOp> reductionDecls;
+ collectReductionDecls(wsloopOp, reductionDecls);
+ ArrayRef<bool> isByRef = getIsByRef(wsloopOp.getReductionByref());
+ MutableArrayRef<BlockArgument> reductionArgs =
+ cast<omp::BlockArgOpenMPOpInterface>(wsloopOp.getOperation())
+ .getReductionBlockArgs();
+
+ DenseMap<Value, llvm::Value *> reductionVariableMap;
+ for (unsigned i = 0; i < numReductions; ++i)
+ reductionVariableMap.try_emplace(
+ wsloopOp.getReductionVars()[i],
+ moduleTranslation.lookupValue(reductionArgs[i]));
+
+ for (unsigned i = 0; i < numReductions; ++i) {
+ // Scan reductions only support by-value (scalar) reductions.
+ if (isByRef[i])
+ continue;
+ SmallVector<llvm::Value *, 1> phis;
+ mapInitializationArgs(wsloopOp, moduleTranslation, builder, reductionDecls,
+ reductionVariableMap, i);
+ if (failed(inlineConvertOmpRegions(
+ reductionDecls[i].getInitializerRegion(),
+ "omp.scan.reduction.init", builder, moduleTranslation, &phis)))
+ return failure();
+ assert(phis.size() == 1 &&
+ "expected one value to be yielded from the reduction init region");
+ setInsertPointForPossiblyEmptyBlock(builder);
+ builder.CreateStore(phis[0],
+ moduleTranslation.lookupValue(reductionArgs[i]));
+ }
+ return success();
+}
+
/// Converts an OpenMP loop nest into LLVM IR using OpenMPIRBuilder.
static LogicalResult
convertOmpLoopNest(Operation &opInst, llvm::IRBuilderBase &builder,
@@ -5004,6 +5307,10 @@ convertOmpLoopNest(Operation &opInst, llvm::IRBuilderBase &builder,
// Convert the body of the loop.
builder.restoreIP(ip);
+ // For scan reductions, reset the reduction variables to their identity at
+ // the start of each input-loop iteration, before the input phase runs.
+ if (failed(initScanReductionVars(loopOp, builder, moduleTranslation)))
+ return llvm::make_error<PreviouslyReportedError>();
llvm::Expected<llvm::BasicBlock *> regionBlock = convertOmpOpRegions(
loopOp.getRegion(), "omp.loop_nest.region", builder, moduleTranslation);
if (!regionBlock)
@@ -5036,6 +5343,46 @@ convertOmpLoopNest(Operation &opInst, llvm::IRBuilderBase &builder,
computeIP = loopInfos.front()->getPreheaderIP();
}
+ // If this loop is the worksharing loop of a scan reduction, generate a
+ // pair of canonical loops (an input loop and a scan loop) instead of a
+ // single loop. The `scan` directive in the body is translated against the
+ // `ScanInfo` recorded here.
+ bool isInScanRegion = false;
+ if (auto wsloopOp = loopOp->getParentOfType<omp::WsloopOp>())
+ isInScanRegion =
+ wsloopOp.getReductionMod() && (wsloopOp.getReductionMod().value() ==
+ mlir::omp::ReductionModifier::inscan);
+ if (isInScanRegion) {
+ llvm::Expected<llvm::ScanInfo *> res = ompBuilder->scanInfoInitialize();
+ if (failed(handleError(res, *loopOp)))
+ return failure();
+ llvm::ScanInfo *scanInfo = res.get();
+ moduleTranslation.stackWalk<OpenMPLoopInfoStackFrame>(
+ [&](OpenMPLoopInfoStackFrame &frame) {
+ frame.scanInfo = scanInfo;
+ return WalkResult::interrupt();
+ });
+ llvm::Expected<llvm::SmallVector<llvm::CanonicalLoopInfo *>> loopResults =
+ ompBuilder->createCanonicalScanLoops(
+ loc, bodyGen, lowerBound, upperBound, step,
+ /*IsSigned=*/true, loopOp.getLoopInclusive(), computeIP, "loop",
+ scanInfo);
+
+ if (failed(handleError(loopResults, *loopOp)))
+ return failure();
+ llvm::CanonicalLoopInfo *inputLoop = loopResults.get().front();
+ llvm::CanonicalLoopInfo *scanLoop = loopResults.get().back();
+ moduleTranslation.stackWalk<OpenMPLoopInfoStackFrame>(
+ [&](OpenMPLoopInfoStackFrame &frame) {
+ frame.loopInfo = inputLoop;
+ frame.scanloopInfo = scanLoop;
+ return WalkResult::interrupt();
+ });
+ builder.restoreIP(scanLoop->getAfterIP());
+ // TODO: tiling and collapse are not yet implemented for scan reduction.
+ return success();
+ }
+
llvm::Expected<llvm::CanonicalLoopInfo *> loopResult =
ompBuilder->createCanonicalLoop(
loc, bodyGen, lowerBound, upperBound, step,
@@ -9454,6 +9801,9 @@ LogicalResult OpenMPDialectLLVMIRTranslationInterface::convertOperation(
.Case([&](omp::WsloopOp) {
return convertOmpWsloop(*op, builder, moduleTranslation);
})
+ .Case([&](omp::ScanOp) {
+ return convertOmpScan(*op, builder, moduleTranslation);
+ })
.Case([&](omp::SimdOp) {
return convertOmpSimd(*op, builder, moduleTranslation);
})
diff --git a/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir b/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
new file mode 100644
index 0000000000000..c6b8daccad313
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
@@ -0,0 +1,123 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+omp.declare_reduction @add_reduction_i32 : i32 init {
+^bb0(%arg0: i32):
+ %0 = llvm.mlir.constant(0 : i32) : i32
+ omp.yield(%0 : i32)
+} combiner {
+^bb0(%arg0: i32, %arg1: i32):
+ %0 = llvm.add %arg0, %arg1 : i32
+ omp.yield(%0 : i32)
+}
+// CHECK-LABEL: @scan_reduction
+llvm.func @scan_reduction() {
+ %0 = llvm.mlir.constant(1 : i64) : i64
+ %1 = llvm.alloca %0 x i32 {bindc_name = "z"} : (i64) -> !llvm.ptr
+ %3 = llvm.alloca %0 x i32 {bindc_name = "y"} : (i64) -> !llvm.ptr
+ %5 = llvm.alloca %0 x i32 {bindc_name = "x"} : (i64) -> !llvm.ptr
+ %7 = llvm.alloca %0 x i32 {bindc_name = "k"} : (i64) -> !llvm.ptr
+ %10 = llvm.mlir.constant(100 : i32) : i32
+ %11 = llvm.mlir.constant(1 : i32) : i32
+ %12 = llvm.mlir.constant(0 : i32) : i32
+ %13 = llvm.mlir.constant(100 : index) : i64
+ %14 = llvm.mlir.addressof @_QFEa : !llvm.ptr
+ %15 = llvm.mlir.addressof @_QFEb : !llvm.ptr
+ omp.parallel {
+ %37 = llvm.mlir.constant(1 : i64) : i64
+ %38 = llvm.alloca %37 x i32 {bindc_name = "k", pinned} : (i64) -> !llvm.ptr
+ %39 = llvm.mlir.constant(1 : i64) : i64
+ omp.wsloop reduction(mod: inscan, @add_reduction_i32 %5 -> %arg0 : !llvm.ptr) {
+ omp.loop_nest (%arg1) : i32 = (%11) to (%10) inclusive step (%11) {
+ llvm.store %arg1, %38 : i32, !llvm.ptr
+ %40 = llvm.load %arg0 : !llvm.ptr -> i32
+ %41 = llvm.load %38 : !llvm.ptr -> i32
+ %42 = llvm.sext %41 : i32 to i64
+ %50 = llvm.getelementptr %14[%42] : (!llvm.ptr, i64) -> !llvm.ptr, i32
+ %51 = llvm.load %50 : !llvm.ptr -> i32
+ %52 = llvm.add %40, %51 : i32
+ llvm.store %52, %arg0 : i32, !llvm.ptr
+ omp.scan inclusive(%arg0 : !llvm.ptr)
+ llvm.store %arg1, %38 : i32, !llvm.ptr
+ %53 = llvm.load %arg0 : !llvm.ptr -> i32
+ %54 = llvm.load %38 : !llvm.ptr -> i32
+ %55 = llvm.sext %54 : i32 to i64
+ %63 = llvm.getelementptr %15[%55] : (!llvm.ptr, i64) -> !llvm.ptr, i32
+ llvm.store %53, %63 : i32, !llvm.ptr
+ omp.yield
+ }
+ }
+ omp.terminator
+ }
+ llvm.return
+}
+llvm.mlir.global internal @_QFEa() {addr_space = 0 : i32} : !llvm.array<100 x i32> {
+ %0 = llvm.mlir.zero : !llvm.array<100 x i32>
+ llvm.return %0 : !llvm.array<100 x i32>
+}
+llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i32> {
+ %0 = llvm.mlir.zero : !llvm.array<100 x i32>
+ llvm.return %0 : !llvm.array<100 x i32>
+}
+//CHECK: %vla = alloca ptr, align 8
+//CHECK: omp_parallel
+//CHECK: store ptr %vla, ptr %gep_vla, align 8
+//CHECK: @__kmpc_fork_call
+//CHECK: void @scan_reduction..omp_par
+//CHECK: %[[BUFF_PTR:.+]] = load ptr, ptr %gep_vla
+//CHECK: @__kmpc_masked
+//CHECK: @__kmpc_barrier
+//CHECK: @__kmpc_masked
+//CHECK: @__kmpc_barrier
+//CHECK: omp.scan.loop.cont:
+//CHECK: @__kmpc_masked
+//CHECK: @__kmpc_barrier
+//CHECK: %[[FREE_VAR:.+]] = load ptr, ptr %[[BUFF_PTR]], align 8
+//CHECK: %[[ARRLAST:.+]] = getelementptr inbounds i32, ptr %[[FREE_VAR]], i32 100
+//CHECK: %[[RES:.+]] = load i32, ptr %[[ARRLAST]], align 4
+//CHECK: store i32 %[[RES]], ptr %loadgep{{.*}}, align 4
+//CHECK: tail call void @free(ptr %[[FREE_VAR]])
+//CHECK: @__kmpc_end_masked
+//CHECK: omp.inscan.dispatch{{.*}}: ; preds = %omp_loop.body{{.*}}
+//CHECK: %[[BUFFVAR:.+]] = load ptr, ptr %[[BUFF_PTR]], align 8
+//CHECK: %[[arrayOffset1:.+]] = getelementptr inbounds i32, ptr %[[BUFFVAR]], i32 %{{.*}}
+//CHECK: %[[BUFFVAL1:.+]] = load i32, ptr %[[arrayOffset1]], align 4
+//CHECK: store i32 %[[BUFFVAL1]], ptr %{{.*}}, align 4
+//CHECK: %[[LOG:.+]] = call double @llvm.log2.f64(double 1.000000e+02) #0
+//CHECK: %[[CEIL:.+]] = call double @llvm.ceil.f64(double %[[LOG]]) #0
+//CHECK: %[[UB:.+]] = fptoui double %[[CEIL]] to i32
+//CHECK: br label %omp.outer.log.scan.body
+//CHECK: omp.outer.log.scan.body:
+//CHECK: %[[K:.+]] = phi i32 [ 0, %{{.*}} ], [ %[[NEXTK:.+]], %omp.inner.log.scan.exit ]
+//CHECK: %[[I:.+]] = phi i32 [ 1, %{{.*}} ], [ %[[NEXTI:.+]], %omp.inner.log.scan.exit ]
+//CHECK: %[[CMP1:.+]] = icmp uge i32 99, %[[I]]
+//CHECK: br i1 %[[CMP1]], label %omp.inner.log.scan.body, label %omp.inner.log.scan.exit
+//CHECK: omp.inner.log.scan.exit: ; preds = %omp.inner.log.scan.body, %omp.outer.log.scan.body
+//CHECK: %[[NEXTK]] = add nuw i32 %[[K]], 1
+//CHECK: %[[NEXTI]] = shl nuw i32 %[[I]], 1
+//CHECK: %[[CMP2:.+]] = icmp ne i32 %[[NEXTK]], %[[UB]]
+//CHECK: br i1 %[[CMP2]], label %omp.outer.log.scan.body, label %omp.outer.log.scan.exit
+//CHECK: omp.outer.log.scan.exit: ; preds = %omp.inner.log.scan.exit
+//CHECK: @__kmpc_end_masked
+//CHECK: omp.inner.log.scan.body: ; preds = %omp.inner.log.scan.body, %omp.outer.log.scan.body
+//CHECK: %[[CNT:.+]] = phi i32 [ 99, %omp.outer.log.scan.body ], [ %[[CNTNXT:.+]], %omp.inner.log.scan.body ]
+//CHECK: %[[BUFF:.+]] = load ptr, ptr %[[BUFF_PTR]]
+//CHECK: %[[IND1:.+]] = add i32 %[[CNT]], 1
+//CHECK: %[[IND1PTR:.+]] = getelementptr inbounds i32, ptr %[[BUFF]], i32 %[[IND1]]
+//CHECK: %[[IND2:.+]] = sub nuw i32 %[[IND1]], %[[I]]
+//CHECK: %[[IND2PTR:.+]] = getelementptr inbounds i32, ptr %[[BUFF]], i32 %[[IND2]]
+//CHECK: %[[IND1VAL:.+]] = load i32, ptr %[[IND1PTR]], align 4
+//CHECK: %[[IND2VAL:.+]] = load i32, ptr %[[IND2PTR]], align 4
+//CHECK: %[[REDVAL:.+]] = add i32 %[[IND1VAL]], %[[IND2VAL]]
+//CHECK: store i32 %[[REDVAL]], ptr %[[IND1PTR]], align 4
+//CHECK: %[[CNTNXT]] = sub nuw i32 %[[CNT]], 1
+//CHECK: %[[CMP3:.+]] = icmp uge i32 %[[CNTNXT]], %[[I]]
+//CHECK: br i1 %[[CMP3]], label %omp.inner.log.scan.body, label %omp.inner.log.scan.exit
+//CHECK: omp.inscan.dispatch: ; preds = %omp_loop.body
+//CHECK: br i1 true, label %omp.before.scan.bb, label %omp.after.scan.bb
+//CHECK: omp.before.scan.bb:
+//CHECK: store i32 0, ptr %[[REDPRIV:.+]], align 4
+//CHECK: omp.loop_nest.region: ; preds = %omp.before.scan.bb
+//CHECK: %[[BUFFER:.+]] = load ptr, ptr %loadgep_vla, align 8
+//CHECK: %[[ARRAYOFFSET2:.+]] = getelementptr inbounds i32, ptr %[[BUFFER]], i32 %{{.*}}
+//CHECK-NEXT: %[[REDPRIVVAL:.+]] = load i32, ptr %{{.*}}, align 4
+//CHECK: store i32 %[[REDPRIVVAL]], ptr %[[ARRAYOFFSET2]], align 4
+//CHECK: br label %omp.scan.loop.exit
diff --git a/mlir/test/Target/LLVMIR/openmp-todo.mlir b/mlir/test/Target/LLVMIR/openmp-todo.mlir
index 71e8628595aab..7edf88b5ea559 100644
--- a/mlir/test/Target/LLVMIR/openmp-todo.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-todo.mlir
@@ -120,10 +120,10 @@ atomic {
llvm.atomicrmw fadd %arg2, %2 monotonic : !llvm.ptr, f32
omp.yield
}
-llvm.func @scan_reduction(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
- // expected-error at below {{not yet implemented: Unhandled clause reduction with modifier in omp.wsloop operation}}
- // expected-error at below {{LLVM Translation failed for operation: omp.wsloop}}
- omp.wsloop reduction(mod:inscan, @add_f32 %x -> %prv : !llvm.ptr) {
+llvm.func @simd_reduction(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
+ // expected-error at below {{not yet implemented: Unhandled clause reduction with modifier in omp.simd operation}}
+ // expected-error at below {{LLVM Translation failed for operation: omp.simd}}
+ omp.simd reduction(mod:inscan, @add_f32 %x -> %prv : !llvm.ptr) {
omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
omp.scan inclusive(%prv : !llvm.ptr)
omp.yield
@@ -162,6 +162,39 @@ llvm.func @parallel_task_reduction_modifier_byref(%x : !llvm.ptr) {
// -----
+omp.declare_reduction @add_f32 : f32
+init {
+^bb0(%arg: f32):
+ %0 = llvm.mlir.constant(0.0 : f32) : f32
+ omp.yield (%0 : f32)
+}
+combiner {
+^bb1(%arg0: f32, %arg1: f32):
+ %1 = llvm.fadd %arg0, %arg1 : f32
+ omp.yield (%1 : f32)
+}
+atomic {
+^bb2(%arg2: !llvm.ptr, %arg3: !llvm.ptr):
+ %2 = llvm.load %arg3 : !llvm.ptr -> f32
+ llvm.atomicrmw fadd %arg2, %2 monotonic : !llvm.ptr, f32
+ omp.yield
+}
+llvm.func @scan_reduction(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
+ // expected-error at below {{LLVM Translation failed for operation: omp.wsloop}}
+ omp.wsloop reduction(mod:inscan, @add_f32 %x -> %prv : !llvm.ptr) {
+ // expected-error at below {{LLVM Translation failed for operation: omp.loop_nest}}
+ omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
+ // expected-error at below {{not yet implemented: Unhandled clause exclusive in omp.scan operation}}
+ // expected-error at below {{LLVM Translation failed for operation: omp.scan}}
+ omp.scan exclusive(%prv : !llvm.ptr)
+ omp.yield
+ }
+ }
+ llvm.return
+}
+
+// -----
+
llvm.func @single_allocate(%x : !llvm.ptr) {
// expected-error at below {{not yet implemented: Unhandled clause allocate in omp.single operation}}
// expected-error at below {{LLVM Translation failed for operation: omp.single}}
diff --git a/openmp/runtime/test/scan/scan.f90 b/openmp/runtime/test/scan/scan.f90
new file mode 100644
index 0000000000000..76263508105d0
--- /dev/null
+++ b/openmp/runtime/test/scan/scan.f90
@@ -0,0 +1,38 @@
+! RUN: %flang %flags %openmp_flags -fopenmp-version=51 %s -o %t.exe
+! RUN: %t.exe | FileCheck %s --match-full-lines
+program inclusive_scan
+ implicit none
+ integer, parameter :: n = 100
+ integer a(n), b(n)
+ integer x, k, y, z
+
+ ! initialization
+ x = 0
+ do k = 1, n
+ a(k) = k
+ end do
+
+ ! a(k) is included in the computation of producing results in b(k)
+ !$omp parallel do reduction(inscan, +: x)
+ do k = 1, n
+ x = x + a(k)
+ !$omp scan inclusive(x)
+ b(k) = x
+ end do
+
+ print *,'x =', x
+ do k = 1, 10
+ print *, 'b(', k, ') =', b(k)
+ end do
+end program
+!CHECK: x = 5050
+!CHECK: b( 1 ) = 1
+!CHECK: b( 2 ) = 3
+!CHECK: b( 3 ) = 6
+!CHECK: b( 4 ) = 10
+!CHECK: b( 5 ) = 15
+!CHECK: b( 6 ) = 21
+!CHECK: b( 7 ) = 28
+!CHECK: b( 8 ) = 36
+!CHECK: b( 9 ) = 45
+!CHECK: b( 10 ) = 55
>From 4011dd537d9830ee4a0379cba80413d1e4cdf9c3 Mon Sep 17 00:00:00 2001
From: Chandra Ghale <ghale at pe34genoa.hpc.amslabs.hpecorp.net>
Date: Wed, 1 Jul 2026 02:44:02 -0500
Subject: [PATCH 2/9] formatting correction
---
.../OpenMP/OpenMPToLLVMIRTranslation.cpp | 17 ++++++++---------
1 file changed, 8 insertions(+), 9 deletions(-)
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 297ebac51de88..e18cc0df79860 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -4534,12 +4534,11 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
bool isInScanRegion =
wsloopOp.getReductionMod() && (wsloopOp.getReductionMod().value() ==
mlir::omp::ReductionModifier::inscan);
- if (failed(initReductionVars(wsloopOp, reductionArgs, builder,
- moduleTranslation,
- afterAllocas.get()->getSinglePredecessor(),
- reductionDecls, privateReductionVariables,
- reductionVariableMap, isByRef, deferredStores,
- isInScanRegion)))
+ if (failed(initReductionVars(
+ wsloopOp, reductionArgs, builder, moduleTranslation,
+ afterAllocas.get()->getSinglePredecessor(), reductionDecls,
+ privateReductionVariables, reductionVariableMap, isByRef,
+ deferredStores, isInScanRegion)))
return failure();
// For `reduction(task, ...)` open a task-reduction scope for the worksharing
@@ -5262,9 +5261,9 @@ initScanReductionVars(omp::LoopNestOp loopOp, llvm::IRBuilderBase &builder,
SmallVector<llvm::Value *, 1> phis;
mapInitializationArgs(wsloopOp, moduleTranslation, builder, reductionDecls,
reductionVariableMap, i);
- if (failed(inlineConvertOmpRegions(
- reductionDecls[i].getInitializerRegion(),
- "omp.scan.reduction.init", builder, moduleTranslation, &phis)))
+ if (failed(inlineConvertOmpRegions(reductionDecls[i].getInitializerRegion(),
+ "omp.scan.reduction.init", builder,
+ moduleTranslation, &phis)))
return failure();
assert(phis.size() == 1 &&
"expected one value to be yielded from the reduction init region");
>From 64b530d3f37e0cd08ca622b538d9bec1eb348b50 Mon Sep 17 00:00:00 2001
From: Chandra Ghale <ghale at pe34genoa.hpc.amslabs.hpecorp.net>
Date: Wed, 22 Jul 2026 05:09:34 -0500
Subject: [PATCH 3/9] Address review comments
---
llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp | 46 +++++++++++---
.../OpenMP/OpenMPToLLVMIRTranslation.cpp | 26 +++++++-
.../Target/LLVMIR/openmp-reduction-scan.mlir | 14 +++--
mlir/test/Target/LLVMIR/openmp-todo.mlir | 31 ++++++++++
openmp/runtime/test/scan/scan-bounds.f90 | 61 +++++++++++++++++++
openmp/runtime/test/scan/scan-edge.f90 | 61 +++++++++++++++++++
openmp/runtime/test/scan/scan-nonidentity.f90 | 28 +++++++++
7 files changed, 251 insertions(+), 16 deletions(-)
create mode 100644 openmp/runtime/test/scan/scan-bounds.f90
create mode 100644 openmp/runtime/test/scan/scan-edge.f90
create mode 100644 openmp/runtime/test/scan/scan-nonidentity.f90
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index f8c1999fe1b89..f62ea4cf2f9da 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -5307,7 +5307,17 @@ Error OpenMPIRBuilder::emitScanBasedDirectiveFinalsIR(
"arrayOffset");
Value *Src = Builder.CreateLoad(SrcTy, Val);
- Builder.CreateStore(Src, OrigVar);
+ // For a zero-trip loop the buffer slot is uninitialized; keep the
+ // original variable's incoming value instead of storing garbage. The
+ // load above stays in bounds because the buffer was allocated with
+ // `Span + 1` elements, so index `Span` is always valid.
+ Value *Cur = Builder.CreateLoad(SrcTy, OrigVar);
+ Value *HasIters = Builder.CreateICmpUGT(
+ ScanRedInfo->Span,
+ llvm::ConstantInt::get(ScanRedInfo->Span->getType(), 0));
+ Value *Final = Builder.CreateSelect(HasIters, Src, Cur);
+
+ Builder.CreateStore(Final, OrigVar);
Builder.CreateFree(Buff);
}
return Error::success();
@@ -5370,7 +5380,15 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
ScanRedInfo->Span,
llvm::ConstantInt::get(ScanRedInfo->Span->getType(), 1));
Builder.SetInsertPoint(InputBB);
- Builder.CreateBr(LoopBB);
+ // The log-scan prefix computation is only well-defined when there are at
+ // least two elements: `log2(Span)` is invalid for Span == 0 and the loop
+ // trip computation underflows/loops forever for Span <= 1. For Span <= 1
+ // the buffer already holds the final value(s), so skip straight to the
+ // exit.
+ llvm::Value *SpanGuard = Builder.CreateICmpUGT(
+ ScanRedInfo->Span,
+ llvm::ConstantInt::get(ScanRedInfo->Span->getType(), 1));
+ Builder.CreateCondBr(SpanGuard, LoopBB, ExitBB);
emitBlock(LoopBB, CurFn);
Builder.SetInsertPoint(LoopBB);
@@ -5617,7 +5635,6 @@ OpenMPIRBuilder::createCanonicalScanLoops(
auto BodyGen = [=](InsertPointTy CodeGenIP, Value *IV) {
Builder.restoreIP(CodeGenIP);
- ScanRedInfo->IV = IV;
createScanBBs(ScanRedInfo);
BasicBlock *InputBlock = Builder.GetInsertBlock();
Instruction *Terminator = InputBlock->getTerminator();
@@ -5636,9 +5653,13 @@ OpenMPIRBuilder::createCanonicalScanLoops(
};
const auto &&InputLoopGen = [&]() -> Error {
+ // Pass an empty ComputeIP: the trip count must be (re)computed at the
+ // loop's own location. Reusing the outer ComputeIP is invalid here because
+ // it was invalidated by the `scan.init` split above and would emit the
+ // recomputed trip count after a block terminator for runtime bounds.
Expected<CanonicalLoopInfo *> LoopInfo = createCanonicalLoop(
Builder.saveIP(), BodyGen, Start, Stop, Step, IsSigned, InclusiveStop,
- ComputeIP, Name, true, ScanRedInfo);
+ /*ComputeIP=*/InsertPointTy(), Name, true, ScanRedInfo);
if (!LoopInfo)
return LoopInfo.takeError();
Result.push_back(*LoopInfo);
@@ -5646,9 +5667,9 @@ OpenMPIRBuilder::createCanonicalScanLoops(
return Error::success();
};
const auto &&ScanLoopGen = [&](LocationDescription Loc) -> Error {
- Expected<CanonicalLoopInfo *> LoopInfo =
- createCanonicalLoop(Loc, BodyGen, Start, Stop, Step, IsSigned,
- InclusiveStop, ComputeIP, Name, true, ScanRedInfo);
+ Expected<CanonicalLoopInfo *> LoopInfo = createCanonicalLoop(
+ Loc, BodyGen, Start, Stop, Step, IsSigned, InclusiveStop,
+ /*ComputeIP=*/InsertPointTy(), Name, true, ScanRedInfo);
if (!LoopInfo)
return LoopInfo.takeError();
Result.push_back(*LoopInfo);
@@ -5737,8 +5758,15 @@ Expected<CanonicalLoopInfo *> OpenMPIRBuilder::createCanonicalLoop(
Builder.restoreIP(CodeGenIP);
Value *Span = Builder.CreateMul(IV, Step);
Value *IndVar = Builder.CreateAdd(Span, Start);
- if (InScan)
- ScanRedInfo->IV = IndVar;
+ if (InScan) {
+ // The scan buffer is 1-indexed by iteration number (buffer[1..TripCount])
+ // rather than by the source induction variable. Using the source value
+ // would index out of bounds for loops that do not start at 1 or that use
+ // a non-unit stride. `IV` here is the 0-based canonical loop counter, so
+ // the 1-based buffer index is `IV + 1`.
+ ScanRedInfo->IV =
+ Builder.CreateAdd(IV, ConstantInt::get(IV->getType(), 1));
+ }
return BodyGenCB(Builder.saveIP(), IndVar);
};
LocationDescription LoopLoc =
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index e18cc0df79860..5af1a384d3edb 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -430,6 +430,16 @@ static LogicalResult checkImplementationStatus(Operation &op) {
break;
}
}
+ } else if (inscanModifierSupported) {
+ if (auto byref = op.getReductionByref()) {
+ // Scan reductions (inscan modifier) only support by-value (scalar)
+ // reductions for now.
+ for (bool isByRef : *byref)
+ if (isByRef) {
+ result = todo("inscan reduction modifier with by-ref reduction");
+ break;
+ }
+ }
}
}
};
@@ -5268,8 +5278,20 @@ initScanReductionVars(omp::LoopNestOp loopOp, llvm::IRBuilderBase &builder,
assert(phis.size() == 1 &&
"expected one value to be yielded from the reduction init region");
setInsertPointForPossiblyEmptyBlock(builder);
- builder.CreateStore(phis[0],
- moduleTranslation.lookupValue(reductionArgs[i]));
+
+ // On the first iteration, seed the reduction variable with the original
+ // variable's incoming value so that a non-identity initial value
+ // participates in the scan (e.g. `x = 10` before an inclusive-scan loop of
+ // `+1` must yield 11, 12, ...). On subsequent iterations, reset to the
+ // reduction identity. `scanInfo->IV` is the 1-based iteration index.
+ llvm::Value *identity = phis[0];
+ llvm::Value *origVar =
+ moduleTranslation.lookupValue(wsloopOp.getReductionVars()[i]);
+ llvm::Value *origVal = builder.CreateLoad(identity->getType(), origVar);
+ llvm::Value *isFirstIter = builder.CreateICmpEQ(
+ scanInfo->IV, llvm::ConstantInt::get(scanInfo->IV->getType(), 1));
+ llvm::Value *seed = builder.CreateSelect(isFirstIter, origVal, identity);
+ builder.CreateStore(seed, moduleTranslation.lookupValue(reductionArgs[i]));
}
return success();
}
diff --git a/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir b/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
index c6b8daccad313..6ef58b5013e08 100644
--- a/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
@@ -73,7 +73,9 @@ llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i3
//CHECK: %[[FREE_VAR:.+]] = load ptr, ptr %[[BUFF_PTR]], align 8
//CHECK: %[[ARRLAST:.+]] = getelementptr inbounds i32, ptr %[[FREE_VAR]], i32 100
//CHECK: %[[RES:.+]] = load i32, ptr %[[ARRLAST]], align 4
-//CHECK: store i32 %[[RES]], ptr %loadgep{{.*}}, align 4
+//CHECK: %[[CUR:.+]] = load i32, ptr %loadgep{{.*}}, align 4
+//CHECK: %[[FIN:.+]] = select i1 {{.*}}, i32 %[[RES]], i32 %[[CUR]]
+//CHECK: store i32 %[[FIN]], ptr %loadgep{{.*}}, align 4
//CHECK: tail call void @free(ptr %[[FREE_VAR]])
//CHECK: @__kmpc_end_masked
//CHECK: omp.inscan.dispatch{{.*}}: ; preds = %omp_loop.body{{.*}}
@@ -84,7 +86,9 @@ llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i3
//CHECK: %[[LOG:.+]] = call double @llvm.log2.f64(double 1.000000e+02) #0
//CHECK: %[[CEIL:.+]] = call double @llvm.ceil.f64(double %[[LOG]]) #0
//CHECK: %[[UB:.+]] = fptoui double %[[CEIL]] to i32
-//CHECK: br label %omp.outer.log.scan.body
+//CHECK: br i1 {{.*}}, label %omp.outer.log.scan.body, label %omp.outer.log.scan.exit
+//CHECK: omp.outer.log.scan.exit: ; preds = %omp.inner.log.scan.exit, %omp_region.body{{.*}}
+//CHECK: @__kmpc_end_masked
//CHECK: omp.outer.log.scan.body:
//CHECK: %[[K:.+]] = phi i32 [ 0, %{{.*}} ], [ %[[NEXTK:.+]], %omp.inner.log.scan.exit ]
//CHECK: %[[I:.+]] = phi i32 [ 1, %{{.*}} ], [ %[[NEXTI:.+]], %omp.inner.log.scan.exit ]
@@ -95,8 +99,6 @@ llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i3
//CHECK: %[[NEXTI]] = shl nuw i32 %[[I]], 1
//CHECK: %[[CMP2:.+]] = icmp ne i32 %[[NEXTK]], %[[UB]]
//CHECK: br i1 %[[CMP2]], label %omp.outer.log.scan.body, label %omp.outer.log.scan.exit
-//CHECK: omp.outer.log.scan.exit: ; preds = %omp.inner.log.scan.exit
-//CHECK: @__kmpc_end_masked
//CHECK: omp.inner.log.scan.body: ; preds = %omp.inner.log.scan.body, %omp.outer.log.scan.body
//CHECK: %[[CNT:.+]] = phi i32 [ 99, %omp.outer.log.scan.body ], [ %[[CNTNXT:.+]], %omp.inner.log.scan.body ]
//CHECK: %[[BUFF:.+]] = load ptr, ptr %[[BUFF_PTR]]
@@ -114,7 +116,9 @@ llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i3
//CHECK: omp.inscan.dispatch: ; preds = %omp_loop.body
//CHECK: br i1 true, label %omp.before.scan.bb, label %omp.after.scan.bb
//CHECK: omp.before.scan.bb:
-//CHECK: store i32 0, ptr %[[REDPRIV:.+]], align 4
+//CHECK: %[[ISFIRST:.+]] = icmp eq i32 %{{.*}}, 1
+//CHECK: %[[SEED:.+]] = select i1 %[[ISFIRST]], i32 %{{.*}}, i32 0
+//CHECK: store i32 %[[SEED]], ptr %[[REDPRIV:.+]], align 4
//CHECK: omp.loop_nest.region: ; preds = %omp.before.scan.bb
//CHECK: %[[BUFFER:.+]] = load ptr, ptr %loadgep_vla, align 8
//CHECK: %[[ARRAYOFFSET2:.+]] = getelementptr inbounds i32, ptr %[[BUFFER]], i32 %{{.*}}
diff --git a/mlir/test/Target/LLVMIR/openmp-todo.mlir b/mlir/test/Target/LLVMIR/openmp-todo.mlir
index 7edf88b5ea559..fe3c4259e75e0 100644
--- a/mlir/test/Target/LLVMIR/openmp-todo.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-todo.mlir
@@ -195,6 +195,37 @@ llvm.func @scan_reduction(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
// -----
+omp.declare_reduction @add_f32 : f32
+init {
+^bb0(%arg: f32):
+ %0 = llvm.mlir.constant(0.0 : f32) : f32
+ omp.yield (%0 : f32)
+}
+combiner {
+^bb1(%arg0: f32, %arg1: f32):
+ %1 = llvm.fadd %arg0, %arg1 : f32
+ omp.yield (%1 : f32)
+}
+atomic {
+^bb2(%arg2: !llvm.ptr, %arg3: !llvm.ptr):
+ %2 = llvm.load %arg3 : !llvm.ptr -> f32
+ llvm.atomicrmw fadd %arg2, %2 monotonic : !llvm.ptr, f32
+ omp.yield
+}
+llvm.func @scan_reduction_byref(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
+ // expected-error at below {{not yet implemented: Unhandled clause inscan reduction modifier with by-ref reduction in omp.wsloop operation}}
+ // expected-error at below {{LLVM Translation failed for operation: omp.wsloop}}
+ omp.wsloop reduction(mod:inscan, byref @add_f32 %x -> %prv : !llvm.ptr) {
+ omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
+ omp.scan inclusive(%prv : !llvm.ptr)
+ omp.yield
+ }
+ }
+ llvm.return
+}
+
+// -----
+
llvm.func @single_allocate(%x : !llvm.ptr) {
// expected-error at below {{not yet implemented: Unhandled clause allocate in omp.single operation}}
// expected-error at below {{LLVM Translation failed for operation: omp.single}}
diff --git a/openmp/runtime/test/scan/scan-bounds.f90 b/openmp/runtime/test/scan/scan-bounds.f90
new file mode 100644
index 0000000000000..9d624954568ff
--- /dev/null
+++ b/openmp/runtime/test/scan/scan-bounds.f90
@@ -0,0 +1,61 @@
+! RUN: %flang %flags %openmp_flags -fopenmp-version=51 %s -o %t.exe
+! RUN: %t.exe | FileCheck %s --match-full-lines
+
+! Arbitrary loop bounds and a non-unit stride must index the scan buffer by the
+! logical iteration number, not by the induction-variable value.
+program scan_bounds
+ implicit none
+ integer, parameter :: n = 30
+ integer :: a(n), b(n)
+ integer :: x, k
+
+ do k = 1, n
+ a(k) = k
+ end do
+
+ ! Arbitrary bounds: iterate from 10 to 20.
+ b = -1
+ x = 0
+ !$omp parallel do reduction(inscan, +: x)
+ do k = 10, 20
+ x = x + a(k)
+ !$omp scan inclusive(x)
+ b(k) = x
+ end do
+ print *, 'bounds x =', x
+ do k = 10, 20
+ print *, 'bb(', k, ') =', b(k)
+ end do
+
+ ! Non-unit stride: iterate from 1 to 10 step 2.
+ b = -1
+ x = 0
+ !$omp parallel do reduction(inscan, +: x)
+ do k = 1, 10, 2
+ x = x + a(k)
+ !$omp scan inclusive(x)
+ b(k) = x
+ end do
+ print *, 'stride x =', x
+ do k = 1, 10, 2
+ print *, 'sb(', k, ') =', b(k)
+ end do
+end program
+!CHECK: bounds x = 165
+!CHECK: bb( 10 ) = 10
+!CHECK: bb( 11 ) = 21
+!CHECK: bb( 12 ) = 33
+!CHECK: bb( 13 ) = 46
+!CHECK: bb( 14 ) = 60
+!CHECK: bb( 15 ) = 75
+!CHECK: bb( 16 ) = 91
+!CHECK: bb( 17 ) = 108
+!CHECK: bb( 18 ) = 126
+!CHECK: bb( 19 ) = 145
+!CHECK: bb( 20 ) = 165
+!CHECK: stride x = 25
+!CHECK: sb( 1 ) = 1
+!CHECK: sb( 3 ) = 4
+!CHECK: sb( 5 ) = 9
+!CHECK: sb( 7 ) = 16
+!CHECK: sb( 9 ) = 25
diff --git a/openmp/runtime/test/scan/scan-edge.f90 b/openmp/runtime/test/scan/scan-edge.f90
new file mode 100644
index 0000000000000..7cc00da958e6e
--- /dev/null
+++ b/openmp/runtime/test/scan/scan-edge.f90
@@ -0,0 +1,61 @@
+! RUN: %flang %flags %openmp_flags -fopenmp-version=51 %s -o %t.exe
+! RUN: %t.exe | FileCheck %s --match-full-lines
+
+! Zero-trip loops must not crash and must leave the reduction variable
+! unchanged, and runtime (non-constant) trip counts must work correctly.
+module scan_edge_mod
+contains
+ subroutine run_scan(n, a, b)
+ implicit none
+ integer :: n, k
+ integer :: a(n), b(n)
+ integer :: x
+ x = 0
+ !$omp parallel do reduction(inscan, +: x)
+ do k = 1, n
+ x = x + a(k)
+ !$omp scan inclusive(x)
+ b(k) = x
+ end do
+ end subroutine
+end module
+
+program scan_edge
+ use scan_edge_mod
+ implicit none
+ integer :: a(6), b(6), k
+ integer :: z, zk
+ integer :: zb(1)
+
+ ! Runtime bounds: the trip count of run_scan is not a compile-time constant.
+ do k = 1, 6
+ a(k) = k
+ end do
+ b = -1
+ call run_scan(6, a, b)
+ do k = 1, 6
+ print *, 'rb(', k, ') =', b(k)
+ end do
+
+ ! Runtime zero-trip: n = 0 must not crash.
+ call run_scan(0, a, b)
+ print *, 'runtime zero-trip ok'
+
+ ! Constant zero-trip with a non-identity start: z must be unchanged.
+ z = 42
+ !$omp parallel do reduction(inscan, +: z)
+ do zk = 1, 0
+ z = z + 1
+ !$omp scan inclusive(z)
+ zb(zk) = z
+ end do
+ print *, 'z =', z
+end program
+!CHECK: rb( 1 ) = 1
+!CHECK: rb( 2 ) = 3
+!CHECK: rb( 3 ) = 6
+!CHECK: rb( 4 ) = 10
+!CHECK: rb( 5 ) = 15
+!CHECK: rb( 6 ) = 21
+!CHECK: runtime zero-trip ok
+!CHECK: z = 42
diff --git a/openmp/runtime/test/scan/scan-nonidentity.f90 b/openmp/runtime/test/scan/scan-nonidentity.f90
new file mode 100644
index 0000000000000..064b0d757af6e
--- /dev/null
+++ b/openmp/runtime/test/scan/scan-nonidentity.f90
@@ -0,0 +1,28 @@
+! RUN: %flang %flags %openmp_flags -fopenmp-version=51 %s -o %t.exe
+! RUN: %t.exe | FileCheck %s --match-full-lines
+
+! The reduction variable's non-identity incoming value must be included in the
+! first partial result of an inclusive scan.
+program scan_nonidentity
+ implicit none
+ integer :: x, k
+ integer :: b(4)
+
+ x = 10
+ !$omp parallel do reduction(inscan, +: x)
+ do k = 1, 4
+ x = x + 1
+ !$omp scan inclusive(x)
+ b(k) = x
+ end do
+
+ print *, 'x =', x
+ do k = 1, 4
+ print *, 'b(', k, ') =', b(k)
+ end do
+end program
+!CHECK: x = 14
+!CHECK: b( 1 ) = 11
+!CHECK: b( 2 ) = 12
+!CHECK: b( 3 ) = 13
+!CHECK: b( 4 ) = 14
>From 5506275c5cec52b0991c9f85bddeb1c8ebabf451 Mon Sep 17 00:00:00 2001
From: Chandra Ghale <ghale at pe34genoa.hpc.amslabs.hpecorp.net>
Date: Mon, 27 Jul 2026 10:07:57 -0500
Subject: [PATCH 4/9] updated with review comments fixes
---
llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp | 23 +++--
.../OpenMP/OpenMPToLLVMIRTranslation.cpp | 25 ++++-
mlir/test/Target/LLVMIR/openmp-todo.mlir | 94 +++++++++++++++++--
openmp/runtime/test/scan/scan-i64.f90 | 37 ++++++++
openmp/runtime/test/scan/scan-nowait.f90 | 38 ++++++++
5 files changed, 198 insertions(+), 19 deletions(-)
create mode 100644 openmp/runtime/test/scan/scan-i64.f90
create mode 100644 openmp/runtime/test/scan/scan-nowait.f90
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 90db03da7380c..0ed215b3e2ea6 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -5350,10 +5350,13 @@ Error OpenMPIRBuilder::emitScanBasedDirectiveDeclsIR(
auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
Builder.restoreIP(CodeGenIP);
- Value *AllocSpan =
- Builder.CreateAdd(ScanRedInfo->Span, Builder.getInt32(1));
+ // Use the loop's own index type (matching `Span`) so the buffer allocation
+ // works for both i32 and i64 (e.g. `integer(kind=8)`) induction variables.
+ Type *IndexTy = ScanRedInfo->Span->getType();
+ Value *AllocSpan = Builder.CreateAdd(ScanRedInfo->Span,
+ ConstantInt::get(IndexTy, 1));
for (size_t i = 0; i < ScanVars.size(); i++) {
- Type *IntPtrTy = Builder.getInt32Ty();
+ Type *IntPtrTy = IndexTy;
Constant *Allocsize = ConstantExpr::getSizeOf(ScanVarsType[i]);
Allocsize = ConstantExpr::getTruncOrBitCast(Allocsize, IntPtrTy);
Value *Buff = Builder.CreateMalloc(IntPtrTy, ScanVarsType[i], Allocsize,
@@ -5453,6 +5456,10 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
Builder.restoreIP(CodeGenIP);
Function *CurFn = Builder.GetInsertBlock()->getParent();
+ // Index/counter values that participate in buffer-offset arithmetic must
+ // use the loop's own index type (`Span`'s type) so scan works for both i32
+ // and i64 (e.g. `integer(kind=8)`) induction variables.
+ Type *IndexTy = ScanRedInfo->Span->getType();
// for (int k = 0; k <= ceil(log2(n)); ++k)
llvm::BasicBlock *LoopBB =
BasicBlock::Create(CurFn->getContext(), "omp.outer.log.scan.body");
@@ -5488,10 +5495,10 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
PHINode *Counter = Builder.CreatePHI(Builder.getInt32Ty(), 2);
// size pow2k = 1;
- PHINode *Pow2K = Builder.CreatePHI(Builder.getInt32Ty(), 2);
+ PHINode *Pow2K = Builder.CreatePHI(IndexTy, 2);
Counter->addIncoming(llvm::ConstantInt::get(Builder.getInt32Ty(), 0),
InputBB);
- Pow2K->addIncoming(llvm::ConstantInt::get(Builder.getInt32Ty(), 1),
+ Pow2K->addIncoming(llvm::ConstantInt::get(IndexTy, 1),
InputBB);
// for (size i = n - 1; i >= 2 ^ k; --i)
// tmp[i] op= tmp[i-pow2k];
@@ -5503,14 +5510,14 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
emitBlock(InnerLoopBB, CurFn);
Builder.SetInsertPoint(InnerLoopBB);
- PHINode *IVal = Builder.CreatePHI(Builder.getInt32Ty(), 2);
+ PHINode *IVal = Builder.CreatePHI(IndexTy, 2);
IVal->addIncoming(NMin1, LoopBB);
for (ReductionInfo RedInfo : ReductionInfos) {
Value *ReductionVal = RedInfo.PrivateVariable;
Value *BuffPtr = (*(ScanRedInfo->ScanBuffPtrs))[ReductionVal];
Value *Buff = Builder.CreateLoad(Builder.getPtrTy(), BuffPtr);
Type *DestTy = RedInfo.ElementType;
- Value *IV = Builder.CreateAdd(IVal, Builder.getInt32(1));
+ Value *IV = Builder.CreateAdd(IVal, ConstantInt::get(IndexTy, 1));
Value *LHSPtr =
Builder.CreateInBoundsGEP(DestTy, Buff, IV, "arrayOffset");
Value *OffsetIval = Builder.CreateNUWSub(IV, Pow2K);
@@ -5526,7 +5533,7 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
Builder.CreateStore(Result, LHSPtr);
}
llvm::Value *NextIVal = Builder.CreateNUWSub(
- IVal, llvm::ConstantInt::get(Builder.getInt32Ty(), 1));
+ IVal, llvm::ConstantInt::get(IndexTy, 1));
IVal->addIncoming(NextIVal, Builder.GetInsertBlock());
CmpI = Builder.CreateICmpUGE(NextIVal, Pow2K);
Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 7b8be030da4dc..b7381244a50b0 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -471,6 +471,22 @@ static LogicalResult checkImplementationStatus(Operation &op) {
break;
}
}
+ // Scan reductions allocate a shared temporary buffer in the alloca
+ // frame of the enclosing parallel region. An orphaned worksharing loop
+ // (e.g. an `omp do` in a subroutine that is called from within a
+ // parallel region) has no such frame in its own function, so the buffer
+ // would be allocated per-thread and produce wrong results.
+ if (!op->template getParentOfType<omp::ParallelOp>())
+ result = todo("inscan reduction on an orphaned worksharing loop "
+ "(no enclosing parallel region)");
+ // Scan reductions are only implemented for a single loop; a collapsed
+ // (multi-dimensional) loop nest is not yet supported.
+ if (auto wsloopOp = dyn_cast<omp::WsloopOp>(op.getOperation()))
+ if (auto loopNest = dyn_cast_or_null<omp::LoopNestOp>(
+ wsloopOp.getWrappedLoop()))
+ if (loopNest.getNumLoops() > 1)
+ result = todo("inscan reduction on a collapsed "
+ "(multi-dimensional) loop nest");
}
}
};
@@ -4699,9 +4715,16 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
linearClauseProcessor.rewriteInPlace(builder, loopInfo->getBody(),
loopInfo->getLatch(), index);
+ // Scan reductions need a barrier at the end of the input (first) loop so
+ // that every thread has finished writing the temporary buffer before the
+ // masked prefix-sum reads it. A source-level `nowait` only elides the final
+ // barrier after the scan (second) loop, so force the barrier for the input
+ // loop regardless of `nowait` to avoid a data race.
+ bool needsBarrier =
+ loopNeedsBarrier || (isInScanRegion && inputScanLoop);
llvm::OpenMPIRBuilder::InsertPointOrErrorTy wsloopIP =
ompBuilder->applyWorkshareLoop(
- ompLoc.DL, loopInfo, allocaIP, loopNeedsBarrier,
+ ompLoc.DL, loopInfo, allocaIP, needsBarrier,
convertToScheduleKind(schedule), chunk, isSimd,
scheduleMod == omp::ScheduleModifier::monotonic,
scheduleMod == omp::ScheduleModifier::nonmonotonic, isOrdered,
diff --git a/mlir/test/Target/LLVMIR/openmp-todo.mlir b/mlir/test/Target/LLVMIR/openmp-todo.mlir
index 44817110d8b20..befa9bba2680e 100644
--- a/mlir/test/Target/LLVMIR/openmp-todo.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-todo.mlir
@@ -180,15 +180,19 @@ atomic {
omp.yield
}
llvm.func @scan_reduction(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
- // expected-error at below {{LLVM Translation failed for operation: omp.wsloop}}
- omp.wsloop reduction(mod:inscan, @add_f32 %x -> %prv : !llvm.ptr) {
- // expected-error at below {{LLVM Translation failed for operation: omp.loop_nest}}
- omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
- // expected-error at below {{not yet implemented: Unhandled clause exclusive in omp.scan operation}}
- // expected-error at below {{LLVM Translation failed for operation: omp.scan}}
- omp.scan exclusive(%prv : !llvm.ptr)
- omp.yield
+ // expected-error at below {{LLVM Translation failed for operation: omp.parallel}}
+ omp.parallel {
+ // expected-error at below {{LLVM Translation failed for operation: omp.wsloop}}
+ omp.wsloop reduction(mod:inscan, @add_f32 %x -> %prv : !llvm.ptr) {
+ // expected-error at below {{LLVM Translation failed for operation: omp.loop_nest}}
+ omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
+ // expected-error at below {{not yet implemented: Unhandled clause exclusive in omp.scan operation}}
+ // expected-error at below {{LLVM Translation failed for operation: omp.scan}}
+ omp.scan exclusive(%prv : !llvm.ptr)
+ omp.yield
+ }
}
+ omp.terminator
}
llvm.return
}
@@ -213,9 +217,44 @@ atomic {
omp.yield
}
llvm.func @scan_reduction_byref(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
- // expected-error at below {{not yet implemented: Unhandled clause inscan reduction modifier with by-ref reduction in omp.wsloop operation}}
+ // expected-error at below {{LLVM Translation failed for operation: omp.parallel}}
+ omp.parallel {
+ // expected-error at below {{not yet implemented: Unhandled clause inscan reduction modifier with by-ref reduction in omp.wsloop operation}}
+ // expected-error at below {{LLVM Translation failed for operation: omp.wsloop}}
+ omp.wsloop reduction(mod:inscan, byref @add_f32 %x -> %prv : !llvm.ptr) {
+ omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
+ omp.scan inclusive(%prv : !llvm.ptr)
+ omp.yield
+ }
+ }
+ omp.terminator
+ }
+ llvm.return
+}
+
+// -----
+
+omp.declare_reduction @add_f32 : f32
+init {
+^bb0(%arg: f32):
+ %0 = llvm.mlir.constant(0.0 : f32) : f32
+ omp.yield (%0 : f32)
+}
+combiner {
+^bb1(%arg0: f32, %arg1: f32):
+ %1 = llvm.fadd %arg0, %arg1 : f32
+ omp.yield (%1 : f32)
+}
+atomic {
+^bb2(%arg2: !llvm.ptr, %arg3: !llvm.ptr):
+ %2 = llvm.load %arg3 : !llvm.ptr -> f32
+ llvm.atomicrmw fadd %arg2, %2 monotonic : !llvm.ptr, f32
+ omp.yield
+}
+llvm.func @scan_reduction_orphaned(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
+ // expected-error at below {{not yet implemented: Unhandled clause inscan reduction on an orphaned worksharing loop (no enclosing parallel region) in omp.wsloop operation}}
// expected-error at below {{LLVM Translation failed for operation: omp.wsloop}}
- omp.wsloop reduction(mod:inscan, byref @add_f32 %x -> %prv : !llvm.ptr) {
+ omp.wsloop reduction(mod:inscan, @add_f32 %x -> %prv : !llvm.ptr) {
omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
omp.scan inclusive(%prv : !llvm.ptr)
omp.yield
@@ -226,6 +265,41 @@ llvm.func @scan_reduction_byref(%lb : i32, %ub : i32, %step : i32, %x : !llvm.pt
// -----
+omp.declare_reduction @add_f32 : f32
+init {
+^bb0(%arg: f32):
+ %0 = llvm.mlir.constant(0.0 : f32) : f32
+ omp.yield (%0 : f32)
+}
+combiner {
+^bb1(%arg0: f32, %arg1: f32):
+ %1 = llvm.fadd %arg0, %arg1 : f32
+ omp.yield (%1 : f32)
+}
+atomic {
+^bb2(%arg2: !llvm.ptr, %arg3: !llvm.ptr):
+ %2 = llvm.load %arg3 : !llvm.ptr -> f32
+ llvm.atomicrmw fadd %arg2, %2 monotonic : !llvm.ptr, f32
+ omp.yield
+}
+llvm.func @scan_reduction_multidim(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
+ // expected-error at below {{LLVM Translation failed for operation: omp.parallel}}
+ omp.parallel {
+ // expected-error at below {{not yet implemented: Unhandled clause inscan reduction on a collapsed (multi-dimensional) loop nest in omp.wsloop operation}}
+ // expected-error at below {{LLVM Translation failed for operation: omp.wsloop}}
+ omp.wsloop reduction(mod:inscan, @add_f32 %x -> %prv : !llvm.ptr) {
+ omp.loop_nest (%iv, %iv2) : i32 = (%lb, %lb) to (%ub, %ub) step (%step, %step) {
+ omp.scan inclusive(%prv : !llvm.ptr)
+ omp.yield
+ }
+ }
+ omp.terminator
+ }
+ llvm.return
+}
+
+// -----
+
llvm.func @single_allocate(%x : !llvm.ptr) {
// expected-error at below {{not yet implemented: Unhandled clause allocate in omp.single operation}}
// expected-error at below {{LLVM Translation failed for operation: omp.single}}
diff --git a/openmp/runtime/test/scan/scan-i64.f90 b/openmp/runtime/test/scan/scan-i64.f90
new file mode 100644
index 0000000000000..c2ef3e6e0ac76
--- /dev/null
+++ b/openmp/runtime/test/scan/scan-i64.f90
@@ -0,0 +1,37 @@
+! RUN: %flang %flags %openmp_flags -fopenmp-version=51 %s -o %t.exe
+! RUN: %t.exe | FileCheck %s --match-full-lines
+
+! An `integer(kind=8)` (i64) loop induction variable must work with an inscan
+! reduction. The scan temporary buffer is indexed using the loop's own index
+! type, so an i64 induction variable must not trip the i32/i64 type mismatch in
+! the scan buffer allocation or the prefix-sum computation.
+program scan_i64
+ implicit none
+ integer(kind=8) :: i, n
+ integer :: k
+ integer :: x
+ integer :: b(8)
+
+ n = 8
+ x = 0
+ !$omp parallel do reduction(inscan, +: x)
+ do i = 1_8, n
+ x = x + 1
+ !$omp scan inclusive(x)
+ b(i) = x
+ end do
+
+ print *, 'x =', x
+ do k = 1, 8
+ print *, 'b(', k, ') =', b(k)
+ end do
+end program
+!CHECK: x = 8
+!CHECK: b( 1 ) = 1
+!CHECK: b( 2 ) = 2
+!CHECK: b( 3 ) = 3
+!CHECK: b( 4 ) = 4
+!CHECK: b( 5 ) = 5
+!CHECK: b( 6 ) = 6
+!CHECK: b( 7 ) = 7
+!CHECK: b( 8 ) = 8
diff --git a/openmp/runtime/test/scan/scan-nowait.f90 b/openmp/runtime/test/scan/scan-nowait.f90
new file mode 100644
index 0000000000000..bb17aeeedf9d9
--- /dev/null
+++ b/openmp/runtime/test/scan/scan-nowait.f90
@@ -0,0 +1,38 @@
+! RUN: %flang %flags %openmp_flags -fopenmp-version=51 %s -o %t.exe
+! RUN: env OMP_NUM_THREADS=4 %t.exe | FileCheck %s --match-full-lines
+
+! A source-level `nowait` on a scan worksharing loop must only remove the final
+! barrier after the scan (second) loop. The barrier at the end of the input
+! (first) loop is required so that every thread finishes writing the temporary
+! buffer before the masked prefix-sum reads it. Forcing multiple threads
+! exercises the data race that occurred when `nowait` incorrectly removed that
+! internal barrier.
+program scan_nowait
+ implicit none
+ integer :: x, k
+ integer :: b(8)
+
+ b = -1
+ x = 0
+ !$omp parallel
+ !$omp do reduction(inscan, +: x)
+ do k = 1, 8
+ x = x + 1
+ !$omp scan inclusive(x)
+ b(k) = x
+ end do
+ !$omp end do nowait
+ !$omp end parallel
+
+ do k = 1, 8
+ print *, 'b(', k, ') =', b(k)
+ end do
+end program
+!CHECK: b( 1 ) = 1
+!CHECK: b( 2 ) = 2
+!CHECK: b( 3 ) = 3
+!CHECK: b( 4 ) = 4
+!CHECK: b( 5 ) = 5
+!CHECK: b( 6 ) = 6
+!CHECK: b( 7 ) = 7
+!CHECK: b( 8 ) = 8
>From c5283fcf901ea9d20f64a59ac15a2795c19abac0 Mon Sep 17 00:00:00 2001
From: Chandra Ghale <ghale at pe34genoa.hpc.amslabs.hpecorp.net>
Date: Mon, 27 Jul 2026 11:37:51 -0500
Subject: [PATCH 5/9] git formatting
---
llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp | 11 +++++------
.../Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp | 7 +++----
2 files changed, 8 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 9f82ba18ded81..ff5d325075b93 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -5378,8 +5378,8 @@ Error OpenMPIRBuilder::emitScanBasedDirectiveDeclsIR(
// Use the loop's own index type (matching `Span`) so the buffer allocation
// works for both i32 and i64 (e.g. `integer(kind=8)`) induction variables.
Type *IndexTy = ScanRedInfo->Span->getType();
- Value *AllocSpan = Builder.CreateAdd(ScanRedInfo->Span,
- ConstantInt::get(IndexTy, 1));
+ Value *AllocSpan =
+ Builder.CreateAdd(ScanRedInfo->Span, ConstantInt::get(IndexTy, 1));
for (size_t i = 0; i < ScanVars.size(); i++) {
Type *IntPtrTy = IndexTy;
Constant *Allocsize = ConstantExpr::getSizeOf(ScanVarsType[i]);
@@ -5523,8 +5523,7 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
PHINode *Pow2K = Builder.CreatePHI(IndexTy, 2);
Counter->addIncoming(llvm::ConstantInt::get(Builder.getInt32Ty(), 0),
InputBB);
- Pow2K->addIncoming(llvm::ConstantInt::get(IndexTy, 1),
- InputBB);
+ Pow2K->addIncoming(llvm::ConstantInt::get(IndexTy, 1), InputBB);
// for (size i = n - 1; i >= 2 ^ k; --i)
// tmp[i] op= tmp[i-pow2k];
llvm::BasicBlock *InnerLoopBB =
@@ -5557,8 +5556,8 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
return AfterIP.takeError();
Builder.CreateStore(Result, LHSPtr);
}
- llvm::Value *NextIVal = Builder.CreateNUWSub(
- IVal, llvm::ConstantInt::get(IndexTy, 1));
+ llvm::Value *NextIVal =
+ Builder.CreateNUWSub(IVal, llvm::ConstantInt::get(IndexTy, 1));
IVal->addIncoming(NextIVal, Builder.GetInsertBlock());
CmpI = Builder.CreateICmpUGE(NextIVal, Pow2K);
Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index d1a31a6870ced..7c27a18d285fb 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -540,8 +540,8 @@ static LogicalResult checkImplementationStatus(Operation &op) {
// Scan reductions are only implemented for a single loop; a collapsed
// (multi-dimensional) loop nest is not yet supported.
if (auto wsloopOp = dyn_cast<omp::WsloopOp>(op.getOperation()))
- if (auto loopNest = dyn_cast_or_null<omp::LoopNestOp>(
- wsloopOp.getWrappedLoop()))
+ if (auto loopNest =
+ dyn_cast_or_null<omp::LoopNestOp>(wsloopOp.getWrappedLoop()))
if (loopNest.getNumLoops() > 1)
result = todo("inscan reduction on a collapsed "
"(multi-dimensional) loop nest");
@@ -4876,8 +4876,7 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
// masked prefix-sum reads it. A source-level `nowait` only elides the final
// barrier after the scan (second) loop, so force the barrier for the input
// loop regardless of `nowait` to avoid a data race.
- bool needsBarrier =
- loopNeedsBarrier || (isInScanRegion && inputScanLoop);
+ bool needsBarrier = loopNeedsBarrier || (isInScanRegion && inputScanLoop);
llvm::OpenMPIRBuilder::InsertPointOrErrorTy wsloopIP =
ompBuilder->applyWorkshareLoop(
ompLoc.DL, loopInfo, allocaIP, needsBarrier,
>From 42d781704befcc65923ff419c6ba461d4796afbb Mon Sep 17 00:00:00 2001
From: Chandra Ghale <ghale at pe34genoa.hpc.amslabs.hpecorp.net>
Date: Tue, 28 Jul 2026 11:01:16 -0500
Subject: [PATCH 6/9] rework on second race cond
---
.../llvm/Frontend/OpenMP/OMPIRBuilder.h | 14 +++-
llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp | 40 +++++++---
.../OpenMP/OpenMPToLLVMIRTranslation.cpp | 19 +++--
.../LLVMIR/openmp-reduction-scan-nowait.mlir | 73 +++++++++++++++++++
4 files changed, 127 insertions(+), 19 deletions(-)
create mode 100644 mlir/test/Target/LLVMIR/openmp-reduction-scan-nowait.mlir
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
index 05307414c94d0..87dc1f067463e 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
@@ -1940,11 +1940,17 @@ class OpenMPIRBuilder {
/// \param ReductionInfos Array type containing the ReductionOps.
/// \param ScanRedInfo Pointer to the ScanInfo objected created using
/// `ScanInfoInitialize`.
+ /// \param NoWait Whether the enclosing worksharing construct has a
+ /// `nowait` clause. The barrier that keeps the shared
+ /// scan buffer alive until every thread has finished
+ /// reading it is always emitted; only the
+ /// end-of-construct barrier after the masked region is
+ /// skipped when \p NoWait is set.
///
/// \return error if any produced, else return success.
Error emitScanBasedDirectiveFinalsIR(
ArrayRef<llvm::OpenMPIRBuilder::ReductionInfo> ReductionInfos,
- ScanInfo *ScanInfo);
+ ScanInfo *ScanInfo, bool NoWait = false);
/// This function emits a helper that gathers Reduce lists from the first
/// lane of every active warp to lanes in the first warp.
@@ -3173,12 +3179,16 @@ class OpenMPIRBuilder {
/// \param ReductionInfos Array type containing the ReductionOps.
/// \param ScanRedInfo Pointer to the ScanInfo objected created using
/// `ScanInfoInitialize`.
+ /// \param NoWait Whether the enclosing worksharing construct has a
+ /// `nowait` clause. Only the end-of-construct barrier
+ /// emitted after freeing the shared scan buffer is
+ /// elided; barriers required for correctness are kept.
///
/// \returns The insertion position *after* the masked.
LLVM_ABI InsertPointOrErrorTy emitScanReduction(
const LocationDescription &Loc,
ArrayRef<llvm::OpenMPIRBuilder::ReductionInfo> ReductionInfos,
- ScanInfo *ScanRedInfo);
+ ScanInfo *ScanRedInfo, bool NoWait = false);
/// This directive split and directs the control flow to input phase
/// blocks or scan phase blocks based on 1. whether input loop or scan loop
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index ff5d325075b93..e81768e884922 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -5414,7 +5414,8 @@ Error OpenMPIRBuilder::emitScanBasedDirectiveDeclsIR(
}
Error OpenMPIRBuilder::emitScanBasedDirectiveFinalsIR(
- ArrayRef<ReductionInfo> ReductionInfos, ScanInfo *ScanRedInfo) {
+ ArrayRef<ReductionInfo> ReductionInfos, ScanInfo *ScanRedInfo,
+ bool NoWait) {
auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
Builder.restoreIP(CodeGenIP);
@@ -5453,27 +5454,45 @@ Error OpenMPIRBuilder::emitScanBasedDirectiveFinalsIR(
else
Builder.SetInsertPoint(ScanRedInfo->OMPScanFinish);
- llvm::Value *FilterVal = Builder.getInt32(0);
+ // Synchronize before the masked region frees the shared scan buffer. The
+ // scan (second) loop loads each thread's prefix value from the buffer, so
+ // every thread must have finished that loop before the single masked thread
+ // frees it; otherwise a thread still executing the loop could load from
+ // freed memory. A `masked` region is not a barrier, so this synchronization
+ // is required for correctness and is emitted even under `nowait`.
llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
- createMasked(Builder.saveIP(), BodyGenCB, FiniCB, FilterVal);
-
+ createBarrier(Builder.saveIP(), llvm::omp::OMPD_barrier);
if (!AfterIP)
return AfterIP.takeError();
Builder.restoreIP(*AfterIP);
- BasicBlock *InputBB = Builder.GetInsertBlock();
- if (InputBB->hasTerminator())
- Builder.SetInsertPoint(Builder.GetInsertBlock()->getTerminator());
- AfterIP = createBarrier(Builder.saveIP(), llvm::omp::OMPD_barrier);
+
+ llvm::Value *FilterVal = Builder.getInt32(0);
+ AfterIP = createMasked(Builder.saveIP(), BodyGenCB, FiniCB, FilterVal);
+
if (!AfterIP)
return AfterIP.takeError();
Builder.restoreIP(*AfterIP);
+
+ // The barrier after the masked region provides the construct's
+ // end-of-construct synchronization and publishes the final reduction value
+ // stored into the original variable inside the masked region. Skip it when
+ // the construct has `nowait`, matching a normal worksharing loop.
+ if (!NoWait) {
+ BasicBlock *InputBB = Builder.GetInsertBlock();
+ if (InputBB->hasTerminator())
+ Builder.SetInsertPoint(Builder.GetInsertBlock()->getTerminator());
+ AfterIP = createBarrier(Builder.saveIP(), llvm::omp::OMPD_barrier);
+ if (!AfterIP)
+ return AfterIP.takeError();
+ Builder.restoreIP(*AfterIP);
+ }
return Error::success();
}
OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
const LocationDescription &Loc,
ArrayRef<llvm::OpenMPIRBuilder::ReductionInfo> ReductionInfos,
- ScanInfo *ScanRedInfo) {
+ ScanInfo *ScanRedInfo, bool NoWait) {
if (!updateToLocation(Loc))
return Loc.IP;
@@ -5590,7 +5609,8 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
if (!AfterIP)
return AfterIP.takeError();
Builder.restoreIP(*AfterIP);
- Error Err = emitScanBasedDirectiveFinalsIR(ReductionInfos, ScanRedInfo);
+ Error Err =
+ emitScanBasedDirectiveFinalsIR(ReductionInfos, ScanRedInfo, NoWait);
if (Err)
return Err;
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 7c27a18d285fb..e5e0acf008116 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -4871,12 +4871,17 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
linearClauseProcessor.rewriteInPlace(builder, loopInfo->getBody(),
loopInfo->getLatch(), index);
- // Scan reductions need a barrier at the end of the input (first) loop so
- // that every thread has finished writing the temporary buffer before the
- // masked prefix-sum reads it. A source-level `nowait` only elides the final
- // barrier after the scan (second) loop, so force the barrier for the input
- // loop regardless of `nowait` to avoid a data race.
- bool needsBarrier = loopNeedsBarrier || (isInScanRegion && inputScanLoop);
+ // Scan reductions manage the barriers around the shared temporary buffer
+ // explicitly:
+ // * The input (first) loop always needs its trailing barrier so that every
+ // thread has finished writing the buffer before the masked prefix-sum
+ // reads it; this is required even under `nowait`.
+ // * The scan (second) loop must not emit its own trailing barrier here.
+ // `emitScanReduction` emits a barrier *before* the masked region frees
+ // the buffer (so no thread frees it while another is still reading it)
+ // and, unless `nowait` is present, a barrier *after* it for
+ // end-of-construct synchronization.
+ bool needsBarrier = isInScanRegion ? inputScanLoop : loopNeedsBarrier;
llvm::OpenMPIRBuilder::InsertPointOrErrorTy wsloopIP =
ompBuilder->applyWorkshareLoop(
ompLoc.DL, loopInfo, allocaIP, needsBarrier,
@@ -4930,7 +4935,7 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
llvm::ScanInfo *scanInfo = findScanInfo(moduleTranslation);
llvm::OpenMPIRBuilder::InsertPointOrErrorTy redIP =
ompBuilder->emitScanReduction(builder.saveIP(), reductionInfos,
- scanInfo);
+ scanInfo, wsloopOp.getNowait());
if (failed(handleError(redIP, opInst)))
return failure();
diff --git a/mlir/test/Target/LLVMIR/openmp-reduction-scan-nowait.mlir b/mlir/test/Target/LLVMIR/openmp-reduction-scan-nowait.mlir
new file mode 100644
index 0000000000000..95306535c92cd
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/openmp-reduction-scan-nowait.mlir
@@ -0,0 +1,73 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+// Verify buffer-lifetime handling for an inclusive scan reduction on a
+// worksharing loop with `nowait`. The scan (second) loop reads each thread's
+// prefix value from the shared scan buffer, and a masked region then frees
+// that buffer. A barrier must be emitted *before* the masked region so the
+// buffer cannot be freed while another thread is still reading it (a masked
+// region is not a barrier). Because of `nowait`, no end-of-construct barrier
+// is emitted *after* the masked region.
+
+omp.declare_reduction @add_reduction_i32 : i32 init {
+^bb0(%arg0: i32):
+ %0 = llvm.mlir.constant(0 : i32) : i32
+ omp.yield(%0 : i32)
+} combiner {
+^bb0(%arg0: i32, %arg1: i32):
+ %0 = llvm.add %arg0, %arg1 : i32
+ omp.yield(%0 : i32)
+}
+llvm.func @scan_reduction_nowait() {
+ %0 = llvm.mlir.constant(1 : i64) : i64
+ %5 = llvm.alloca %0 x i32 {bindc_name = "x"} : (i64) -> !llvm.ptr
+ %10 = llvm.mlir.constant(100 : i32) : i32
+ %11 = llvm.mlir.constant(1 : i32) : i32
+ %14 = llvm.mlir.addressof @_QFEa : !llvm.ptr
+ %15 = llvm.mlir.addressof @_QFEb : !llvm.ptr
+ omp.parallel {
+ %37 = llvm.mlir.constant(1 : i64) : i64
+ %38 = llvm.alloca %37 x i32 {bindc_name = "k", pinned} : (i64) -> !llvm.ptr
+ omp.wsloop nowait reduction(mod: inscan, @add_reduction_i32 %5 -> %arg0 : !llvm.ptr) {
+ omp.loop_nest (%arg1) : i32 = (%11) to (%10) inclusive step (%11) {
+ llvm.store %arg1, %38 : i32, !llvm.ptr
+ %40 = llvm.load %arg0 : !llvm.ptr -> i32
+ %41 = llvm.load %38 : !llvm.ptr -> i32
+ %42 = llvm.sext %41 : i32 to i64
+ %50 = llvm.getelementptr %14[%42] : (!llvm.ptr, i64) -> !llvm.ptr, i32
+ %51 = llvm.load %50 : !llvm.ptr -> i32
+ %52 = llvm.add %40, %51 : i32
+ llvm.store %52, %arg0 : i32, !llvm.ptr
+ omp.scan inclusive(%arg0 : !llvm.ptr)
+ llvm.store %arg1, %38 : i32, !llvm.ptr
+ %53 = llvm.load %arg0 : !llvm.ptr -> i32
+ %54 = llvm.load %38 : !llvm.ptr -> i32
+ %55 = llvm.sext %54 : i32 to i64
+ %63 = llvm.getelementptr %15[%55] : (!llvm.ptr, i64) -> !llvm.ptr, i32
+ llvm.store %53, %63 : i32, !llvm.ptr
+ omp.yield
+ }
+ }
+ omp.terminator
+ }
+ llvm.return
+}
+llvm.mlir.global internal @_QFEa() {addr_space = 0 : i32} : !llvm.array<100 x i32> {
+ %0 = llvm.mlir.zero : !llvm.array<100 x i32>
+ llvm.return %0 : !llvm.array<100 x i32>
+}
+llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i32> {
+ %0 = llvm.mlir.zero : !llvm.array<100 x i32>
+ llvm.return %0 : !llvm.array<100 x i32>
+}
+
+// CHECK-LABEL: define internal void @scan_reduction_nowait..omp_par
+// After the scan (second) loop, the mandatory buffer-lifetime barrier is
+// emitted before the masked region that frees the shared scan buffer.
+// CHECK: omp.scan.loop.cont:
+// CHECK: call void @__kmpc_barrier
+// CHECK: call i32 @__kmpc_masked
+// CHECK: call void @free(ptr
+// CHECK: call void @__kmpc_end_masked
+// With `nowait`, there is no end-of-construct barrier after the buffer is freed.
+// CHECK-NOT: call void @__kmpc_barrier
+// CHECK: ret void
>From 46cbc9122d8fa036d02c61ea3d740030701eb824 Mon Sep 17 00:00:00 2001
From: Chandra Ghale <ghale at pe34genoa.hpc.amslabs.hpecorp.net>
Date: Tue, 28 Jul 2026 12:18:10 -0500
Subject: [PATCH 7/9] fixex iteration input now starts from reduction identity
---
llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp | 32 +++++++++++++++++++
.../OpenMP/OpenMPToLLVMIRTranslation.cpp | 21 ++++++------
.../Target/LLVMIR/openmp-reduction-scan.mlir | 16 ++++++++--
3 files changed, 54 insertions(+), 15 deletions(-)
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index e81768e884922..a75c39be196c1 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -5525,6 +5525,38 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
ScanRedInfo->Span,
llvm::ConstantInt::get(ScanRedInfo->Span->getType(), 1));
Builder.SetInsertPoint(InputBB);
+ // Combine the original variable's incoming value (orig-val) into the first
+ // buffer element before computing the prefix sum. Per the OpenMP scan
+ // semantics orig-val is a single prefix element that must be reflected in
+ // every inclusive/exclusive scan result and in the final reduction value.
+ // Folding it in here lets the log-scan prefix computation propagate it into
+ // every element (and into the finals' read of `buffer[Span]`), regardless
+ // of whether an iteration's input phase accumulates into or overwrites the
+ // reduction variable. `select(Span > 0, 1, 0)` keeps the access in bounds
+ // when Span == 0: buffer[0] is otherwise unused and the finals' zero-trip
+ // guard discards it.
+ llvm::Value *HasElem = Builder.CreateICmpUGT(
+ ScanRedInfo->Span, llvm::ConstantInt::get(IndexTy, 0));
+ llvm::Value *FirstIdx =
+ Builder.CreateSelect(HasElem, llvm::ConstantInt::get(IndexTy, 1),
+ llvm::ConstantInt::get(IndexTy, 0));
+ for (ReductionInfo RedInfo : ReductionInfos) {
+ Value *ReductionVal = RedInfo.PrivateVariable;
+ Value *BuffPtr = (*(ScanRedInfo->ScanBuffPtrs))[ReductionVal];
+ Value *Buff = Builder.CreateLoad(Builder.getPtrTy(), BuffPtr);
+ Type *DestTy = RedInfo.ElementType;
+ Value *ElemPtr =
+ Builder.CreateInBoundsGEP(DestTy, Buff, FirstIdx, "arrayOffset");
+ Value *OrigVal = Builder.CreateLoad(DestTy, RedInfo.Variable);
+ Value *Elem = Builder.CreateLoad(DestTy, ElemPtr);
+ llvm::Value *Combined;
+ InsertPointOrErrorTy AfterIP =
+ RedInfo.ReductionGen(Builder.saveIP(), Elem, OrigVal, Combined);
+ if (!AfterIP)
+ return AfterIP.takeError();
+ Builder.restoreIP(*AfterIP);
+ Builder.CreateStore(Combined, ElemPtr);
+ }
// The log-scan prefix computation is only well-defined when there are at
// least two elements: `log2(Span)` is invalid for Span == 0 and the loop
// trip computation underflows/loops forever for Span <= 1. For Span <= 1
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index e5e0acf008116..23f2d06fdabe2 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -5526,19 +5526,16 @@ initScanReductionVars(omp::LoopNestOp loopOp, llvm::IRBuilderBase &builder,
"expected one value to be yielded from the reduction init region");
setInsertPointForPossiblyEmptyBlock(builder);
- // On the first iteration, seed the reduction variable with the original
- // variable's incoming value so that a non-identity initial value
- // participates in the scan (e.g. `x = 10` before an inclusive-scan loop of
- // `+1` must yield 11, 12, ...). On subsequent iterations, reset to the
- // reduction identity. `scanInfo->IV` is the 1-based iteration index.
+ // Reset the private reduction variable to the reduction identity at the
+ // start of every input-loop iteration. The original variable's incoming
+ // value (orig-val) must not be folded into any single iteration here: per
+ // the OpenMP scan semantics it is a separate prefix element that is
+ // combined once into the scan buffer (see `emitScanReduction`). Seeding a
+ // particular iteration would be lost whenever the input phase overwrites
+ // the reduction variable (e.g. `x = i`), producing incorrect results.
llvm::Value *identity = phis[0];
- llvm::Value *origVar =
- moduleTranslation.lookupValue(wsloopOp.getReductionVars()[i]);
- llvm::Value *origVal = builder.CreateLoad(identity->getType(), origVar);
- llvm::Value *isFirstIter = builder.CreateICmpEQ(
- scanInfo->IV, llvm::ConstantInt::get(scanInfo->IV->getType(), 1));
- llvm::Value *seed = builder.CreateSelect(isFirstIter, origVal, identity);
- builder.CreateStore(seed, moduleTranslation.lookupValue(reductionArgs[i]));
+ builder.CreateStore(identity,
+ moduleTranslation.lookupValue(reductionArgs[i]));
}
return success();
}
diff --git a/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir b/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
index 6ef58b5013e08..2aa100d686bc1 100644
--- a/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
@@ -86,6 +86,15 @@ llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i3
//CHECK: %[[LOG:.+]] = call double @llvm.log2.f64(double 1.000000e+02) #0
//CHECK: %[[CEIL:.+]] = call double @llvm.ceil.f64(double %[[LOG]]) #0
//CHECK: %[[UB:.+]] = fptoui double %[[CEIL]] to i32
+// Combine the original variable's incoming value (orig-val) into the first
+// buffer element once, before the prefix sum, so it is reflected in every
+// scan result and in the final reduction value.
+//CHECK: %[[OBUFF:.+]] = load ptr, ptr %{{.*}}, align 8
+//CHECK: %[[OELEMPTR:.+]] = getelementptr inbounds i32, ptr %[[OBUFF]], i32 1
+//CHECK: %[[OORIG:.+]] = load i32, ptr %{{.*}}, align 4
+//CHECK: %[[OELEM:.+]] = load i32, ptr %[[OELEMPTR]], align 4
+//CHECK: %[[OCOMB:.+]] = add i32 %[[OELEM]], %[[OORIG]]
+//CHECK: store i32 %[[OCOMB]], ptr %[[OELEMPTR]], align 4
//CHECK: br i1 {{.*}}, label %omp.outer.log.scan.body, label %omp.outer.log.scan.exit
//CHECK: omp.outer.log.scan.exit: ; preds = %omp.inner.log.scan.exit, %omp_region.body{{.*}}
//CHECK: @__kmpc_end_masked
@@ -116,9 +125,10 @@ llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i3
//CHECK: omp.inscan.dispatch: ; preds = %omp_loop.body
//CHECK: br i1 true, label %omp.before.scan.bb, label %omp.after.scan.bb
//CHECK: omp.before.scan.bb:
-//CHECK: %[[ISFIRST:.+]] = icmp eq i32 %{{.*}}, 1
-//CHECK: %[[SEED:.+]] = select i1 %[[ISFIRST]], i32 %{{.*}}, i32 0
-//CHECK: store i32 %[[SEED]], ptr %[[REDPRIV:.+]], align 4
+// Each input-loop iteration resets the private reduction variable to the
+// reduction identity (0 for `+`). The original variable's incoming value
+// (orig-val) is NOT seeded here; it is combined once in the masked region.
+//CHECK: store i32 0, ptr %{{.*}}, align 4
//CHECK: omp.loop_nest.region: ; preds = %omp.before.scan.bb
//CHECK: %[[BUFFER:.+]] = load ptr, ptr %loadgep_vla, align 8
//CHECK: %[[ARRAYOFFSET2:.+]] = getelementptr inbounds i32, ptr %[[BUFFER]], i32 %{{.*}}
>From cbfd4440aa06d37f99c1bcd8e0e8649199b3d9f0 Mon Sep 17 00:00:00 2001
From: Chandra Ghale <ghale at pe34genoa.hpc.amslabs.hpecorp.net>
Date: Wed, 29 Jul 2026 04:52:51 -0500
Subject: [PATCH 8/9] inscan reduction orig-val handling and guard the scan
seed for zero-trip loops
---
llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp | 37 +++++++++++--------
.../Target/LLVMIR/openmp-reduction-scan.mlir | 12 ++++--
2 files changed, 30 insertions(+), 19 deletions(-)
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index a75c39be196c1..fd9b857d88b9b 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -5525,6 +5525,18 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
ScanRedInfo->Span,
llvm::ConstantInt::get(ScanRedInfo->Span->getType(), 1));
Builder.SetInsertPoint(InputBB);
+ // Branch around the orig-val seed load and combine when the loop has zero
+ // logical iterations. When Span == 0 no scan-buffer entry is populated, so
+ // loading buffer[0] and feeding it to a (possibly input-sensitive
+ // user-defined) reduction combiner would be undefined behavior. In that
+ // case skip straight to the exit; the finals' zero-trip guard preserves
+ // orig-val as the result.
+ llvm::Value *HasElem = Builder.CreateICmpUGT(
+ ScanRedInfo->Span, llvm::ConstantInt::get(IndexTy, 0));
+ llvm::BasicBlock *SeedBB =
+ BasicBlock::Create(CurFn->getContext(), "omp.scan.seed", CurFn, ExitBB);
+ Builder.CreateCondBr(HasElem, SeedBB, ExitBB);
+ Builder.SetInsertPoint(SeedBB);
// Combine the original variable's incoming value (orig-val) into the first
// buffer element before computing the prefix sum. Per the OpenMP scan
// semantics orig-val is a single prefix element that must be reflected in
@@ -5532,21 +5544,14 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
// Folding it in here lets the log-scan prefix computation propagate it into
// every element (and into the finals' read of `buffer[Span]`), regardless
// of whether an iteration's input phase accumulates into or overwrites the
- // reduction variable. `select(Span > 0, 1, 0)` keeps the access in bounds
- // when Span == 0: buffer[0] is otherwise unused and the finals' zero-trip
- // guard discards it.
- llvm::Value *HasElem = Builder.CreateICmpUGT(
- ScanRedInfo->Span, llvm::ConstantInt::get(IndexTy, 0));
- llvm::Value *FirstIdx =
- Builder.CreateSelect(HasElem, llvm::ConstantInt::get(IndexTy, 1),
- llvm::ConstantInt::get(IndexTy, 0));
+ // reduction variable. Span > 0 is guaranteed here, so buffer[1] is valid.
for (ReductionInfo RedInfo : ReductionInfos) {
Value *ReductionVal = RedInfo.PrivateVariable;
Value *BuffPtr = (*(ScanRedInfo->ScanBuffPtrs))[ReductionVal];
Value *Buff = Builder.CreateLoad(Builder.getPtrTy(), BuffPtr);
Type *DestTy = RedInfo.ElementType;
- Value *ElemPtr =
- Builder.CreateInBoundsGEP(DestTy, Buff, FirstIdx, "arrayOffset");
+ Value *ElemPtr = Builder.CreateInBoundsGEP(
+ DestTy, Buff, llvm::ConstantInt::get(IndexTy, 1), "arrayOffset");
Value *OrigVal = Builder.CreateLoad(DestTy, RedInfo.Variable);
Value *Elem = Builder.CreateLoad(DestTy, ElemPtr);
llvm::Value *Combined;
@@ -5559,9 +5564,9 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
}
// The log-scan prefix computation is only well-defined when there are at
// least two elements: `log2(Span)` is invalid for Span == 0 and the loop
- // trip computation underflows/loops forever for Span <= 1. For Span <= 1
- // the buffer already holds the final value(s), so skip straight to the
- // exit.
+ // trip computation underflows/loops forever for Span <= 1. For Span == 1
+ // the buffer already holds the final (seeded) value, so skip straight to
+ // the exit.
llvm::Value *SpanGuard = Builder.CreateICmpUGT(
ScanRedInfo->Span,
llvm::ConstantInt::get(ScanRedInfo->Span->getType(), 1));
@@ -5572,9 +5577,11 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
PHINode *Counter = Builder.CreatePHI(Builder.getInt32Ty(), 2);
// size pow2k = 1;
PHINode *Pow2K = Builder.CreatePHI(IndexTy, 2);
+ // The loop is now entered from the seed block (guarded by Span > 0), not
+ // directly from InputBB.
Counter->addIncoming(llvm::ConstantInt::get(Builder.getInt32Ty(), 0),
- InputBB);
- Pow2K->addIncoming(llvm::ConstantInt::get(IndexTy, 1), InputBB);
+ SeedBB);
+ Pow2K->addIncoming(llvm::ConstantInt::get(IndexTy, 1), SeedBB);
// for (size i = n - 1; i >= 2 ^ k; --i)
// tmp[i] op= tmp[i-pow2k];
llvm::BasicBlock *InnerLoopBB =
diff --git a/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir b/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
index 2aa100d686bc1..38e413a504315 100644
--- a/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
@@ -86,9 +86,15 @@ llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i3
//CHECK: %[[LOG:.+]] = call double @llvm.log2.f64(double 1.000000e+02) #0
//CHECK: %[[CEIL:.+]] = call double @llvm.ceil.f64(double %[[LOG]]) #0
//CHECK: %[[UB:.+]] = fptoui double %[[CEIL]] to i32
+// Zero-trip guard: skip the orig-val seed load/combine when Span == 0 so
+// buffer[0] is never loaded or passed to the reduction combiner.
+//CHECK: br i1 {{.*}}, label %omp.scan.seed, label %omp.outer.log.scan.exit
+//CHECK: omp.outer.log.scan.exit:
+//CHECK: @__kmpc_end_masked
+//CHECK: omp.scan.seed:
// Combine the original variable's incoming value (orig-val) into the first
-// buffer element once, before the prefix sum, so it is reflected in every
-// scan result and in the final reduction value.
+// buffer element once (Span > 0 is guaranteed here), before the prefix sum, so
+// it is reflected in every scan result and in the final reduction value.
//CHECK: %[[OBUFF:.+]] = load ptr, ptr %{{.*}}, align 8
//CHECK: %[[OELEMPTR:.+]] = getelementptr inbounds i32, ptr %[[OBUFF]], i32 1
//CHECK: %[[OORIG:.+]] = load i32, ptr %{{.*}}, align 4
@@ -96,8 +102,6 @@ llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i3
//CHECK: %[[OCOMB:.+]] = add i32 %[[OELEM]], %[[OORIG]]
//CHECK: store i32 %[[OCOMB]], ptr %[[OELEMPTR]], align 4
//CHECK: br i1 {{.*}}, label %omp.outer.log.scan.body, label %omp.outer.log.scan.exit
-//CHECK: omp.outer.log.scan.exit: ; preds = %omp.inner.log.scan.exit, %omp_region.body{{.*}}
-//CHECK: @__kmpc_end_masked
//CHECK: omp.outer.log.scan.body:
//CHECK: %[[K:.+]] = phi i32 [ 0, %{{.*}} ], [ %[[NEXTK:.+]], %omp.inner.log.scan.exit ]
//CHECK: %[[I:.+]] = phi i32 [ 1, %{{.*}} ], [ %[[NEXTI:.+]], %omp.inner.log.scan.exit ]
>From a51ea62ab9586e3219388f07e8df525af46733d9 Mon Sep 17 00:00:00 2001
From: Chandra Ghale <ghale at pe34genoa.hpc.amslabs.hpecorp.net>
Date: Mon, 3 Aug 2026 04:37:41 -0500
Subject: [PATCH 9/9] fix operand order in reduction seeding
---
llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp | 14 ++-
.../openmp-reduction-scan-noncommutative.mlir | 95 +++++++++++++++++++
.../Target/LLVMIR/openmp-reduction-scan.mlir | 4 +-
3 files changed, 109 insertions(+), 4 deletions(-)
create mode 100644 mlir/test/Target/LLVMIR/openmp-reduction-scan-noncommutative.mlir
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index fd9b857d88b9b..961ec39af4efc 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -5555,8 +5555,12 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
Value *OrigVal = Builder.CreateLoad(DestTy, RedInfo.Variable);
Value *Elem = Builder.CreateLoad(DestTy, ElemPtr);
llvm::Value *Combined;
+ // orig-val is the leftmost/earliest element of the prefix, so it must be
+ // the accumulator (`omp_out`, the combiner's first/lhs operand) while the
+ // first buffer element is the incoming value (`omp_in`). The combiner is
+ // associative but not necessarily commutative, so this order matters.
InsertPointOrErrorTy AfterIP =
- RedInfo.ReductionGen(Builder.saveIP(), Elem, OrigVal, Combined);
+ RedInfo.ReductionGen(Builder.saveIP(), OrigVal, Elem, Combined);
if (!AfterIP)
return AfterIP.takeError();
Builder.restoreIP(*AfterIP);
@@ -5608,8 +5612,14 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
Value *LHS = Builder.CreateLoad(DestTy, LHSPtr);
Value *RHS = Builder.CreateLoad(DestTy, RHSPtr);
llvm::Value *Result;
+ // buffer[IV-pow2k] holds the earlier partial prefix and buffer[IV] the
+ // later one. The earlier element must be the accumulator (`omp_out`, the
+ // combiner's first/lhs operand) and the later element the incoming value
+ // (`omp_in`); the combiner is associative but not necessarily
+ // commutative, so passing them in prefix order is required. The result
+ // is written back into the later slot (LHSPtr = buffer[IV]).
InsertPointOrErrorTy AfterIP =
- RedInfo.ReductionGen(Builder.saveIP(), LHS, RHS, Result);
+ RedInfo.ReductionGen(Builder.saveIP(), RHS, LHS, Result);
if (!AfterIP)
return AfterIP.takeError();
Builder.CreateStore(Result, LHSPtr);
diff --git a/mlir/test/Target/LLVMIR/openmp-reduction-scan-noncommutative.mlir b/mlir/test/Target/LLVMIR/openmp-reduction-scan-noncommutative.mlir
new file mode 100644
index 0000000000000..0c595de9e80c1
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/openmp-reduction-scan-noncommutative.mlir
@@ -0,0 +1,95 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+// Regression test for inscan reduction combiner operand ordering. The combiner
+// here is intentionally NONcommutative (subtraction) so that reversing the two
+// operands passed to the reduction combiner changes the emitted IR. This pins
+// the operand order at the two combiner call sites of the log-scan lowering:
+// the orig-val seed and the prefix computation. A commutative combiner such as
+// `+` produces identical IR regardless of operand order and would not catch a
+// reversal, so this complements openmp-reduction-scan.mlir. The scan combines
+// elements left-to-right, so the earlier prefix element must be the
+// accumulator (`omp_out`, the combiner's first operand) and the later element
+// the incoming value (`omp_in`, the second operand).
+
+omp.declare_reduction @sub_reduction_i32 : i32 init {
+^bb0(%arg0: i32):
+ %0 = llvm.mlir.constant(0 : i32) : i32
+ omp.yield(%0 : i32)
+} combiner {
+^bb0(%arg0: i32, %arg1: i32):
+ %0 = llvm.sub %arg0, %arg1 : i32
+ omp.yield(%0 : i32)
+}
+// CHECK-LABEL: @scan_reduction_noncommutative
+llvm.func @scan_reduction_noncommutative() {
+ %0 = llvm.mlir.constant(1 : i64) : i64
+ %1 = llvm.alloca %0 x i32 {bindc_name = "z"} : (i64) -> !llvm.ptr
+ %3 = llvm.alloca %0 x i32 {bindc_name = "y"} : (i64) -> !llvm.ptr
+ %5 = llvm.alloca %0 x i32 {bindc_name = "x"} : (i64) -> !llvm.ptr
+ %7 = llvm.alloca %0 x i32 {bindc_name = "k"} : (i64) -> !llvm.ptr
+ %10 = llvm.mlir.constant(100 : i32) : i32
+ %11 = llvm.mlir.constant(1 : i32) : i32
+ %12 = llvm.mlir.constant(0 : i32) : i32
+ %13 = llvm.mlir.constant(100 : index) : i64
+ %14 = llvm.mlir.addressof @_QFEa : !llvm.ptr
+ %15 = llvm.mlir.addressof @_QFEb : !llvm.ptr
+ omp.parallel {
+ %37 = llvm.mlir.constant(1 : i64) : i64
+ %38 = llvm.alloca %37 x i32 {bindc_name = "k", pinned} : (i64) -> !llvm.ptr
+ %39 = llvm.mlir.constant(1 : i64) : i64
+ omp.wsloop reduction(mod: inscan, @sub_reduction_i32 %5 -> %arg0 : !llvm.ptr) {
+ omp.loop_nest (%arg1) : i32 = (%11) to (%10) inclusive step (%11) {
+ llvm.store %arg1, %38 : i32, !llvm.ptr
+ %40 = llvm.load %arg0 : !llvm.ptr -> i32
+ %41 = llvm.load %38 : !llvm.ptr -> i32
+ %42 = llvm.sext %41 : i32 to i64
+ %50 = llvm.getelementptr %14[%42] : (!llvm.ptr, i64) -> !llvm.ptr, i32
+ %51 = llvm.load %50 : !llvm.ptr -> i32
+ %52 = llvm.add %40, %51 : i32
+ llvm.store %52, %arg0 : i32, !llvm.ptr
+ omp.scan inclusive(%arg0 : !llvm.ptr)
+ llvm.store %arg1, %38 : i32, !llvm.ptr
+ %53 = llvm.load %arg0 : !llvm.ptr -> i32
+ %54 = llvm.load %38 : !llvm.ptr -> i32
+ %55 = llvm.sext %54 : i32 to i64
+ %63 = llvm.getelementptr %15[%55] : (!llvm.ptr, i64) -> !llvm.ptr, i32
+ llvm.store %53, %63 : i32, !llvm.ptr
+ omp.yield
+ }
+ }
+ omp.terminator
+ }
+ llvm.return
+}
+llvm.mlir.global internal @_QFEa() {addr_space = 0 : i32} : !llvm.array<100 x i32> {
+ %0 = llvm.mlir.zero : !llvm.array<100 x i32>
+ llvm.return %0 : !llvm.array<100 x i32>
+}
+llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i32> {
+ %0 = llvm.mlir.zero : !llvm.array<100 x i32>
+ llvm.return %0 : !llvm.array<100 x i32>
+}
+
+// The orig-val seed must combine as sub(orig-val, buffer[1]): the original
+// variable's incoming value (the leftmost prefix element) is the accumulator
+// (first operand) and buffer[1] is the incoming value (second operand).
+//CHECK: omp.scan.seed:
+//CHECK: %[[OBUFF:.+]] = load ptr, ptr %{{.*}}, align 8
+//CHECK: %[[OELEMPTR:.+]] = getelementptr inbounds i32, ptr %[[OBUFF]], i32 1
+//CHECK: %[[OORIG:.+]] = load i32, ptr %{{.*}}, align 4
+//CHECK: %[[OELEM:.+]] = load i32, ptr %[[OELEMPTR]], align 4
+//CHECK: %[[OCOMB:.+]] = sub i32 %[[OORIG]], %[[OELEM]]
+//CHECK: store i32 %[[OCOMB]], ptr %[[OELEMPTR]], align 4
+
+// The prefix computation must combine as sub(buffer[i-pow2k], buffer[i]): the
+// earlier partial prefix is the accumulator (first operand) and the later one
+// the incoming value (second operand). buffer[i] (the later slot) is updated.
+//CHECK: omp.inner.log.scan.body:
+//CHECK: %[[IND1:.+]] = add i32 %{{.*}}, 1
+//CHECK: %[[IND1PTR:.+]] = getelementptr inbounds i32, ptr %{{.*}}, i32 %[[IND1]]
+//CHECK: %[[IND2:.+]] = sub nuw i32 %[[IND1]], %{{.*}}
+//CHECK: %[[IND2PTR:.+]] = getelementptr inbounds i32, ptr %{{.*}}, i32 %[[IND2]]
+//CHECK: %[[IND1VAL:.+]] = load i32, ptr %[[IND1PTR]], align 4
+//CHECK: %[[IND2VAL:.+]] = load i32, ptr %[[IND2PTR]], align 4
+//CHECK: %[[REDVAL:.+]] = sub i32 %[[IND2VAL]], %[[IND1VAL]]
+//CHECK: store i32 %[[REDVAL]], ptr %[[IND1PTR]], align 4
diff --git a/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir b/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
index 38e413a504315..0006434167c7c 100644
--- a/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
@@ -99,7 +99,7 @@ llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i3
//CHECK: %[[OELEMPTR:.+]] = getelementptr inbounds i32, ptr %[[OBUFF]], i32 1
//CHECK: %[[OORIG:.+]] = load i32, ptr %{{.*}}, align 4
//CHECK: %[[OELEM:.+]] = load i32, ptr %[[OELEMPTR]], align 4
-//CHECK: %[[OCOMB:.+]] = add i32 %[[OELEM]], %[[OORIG]]
+//CHECK: %[[OCOMB:.+]] = add i32 %[[OORIG]], %[[OELEM]]
//CHECK: store i32 %[[OCOMB]], ptr %[[OELEMPTR]], align 4
//CHECK: br i1 {{.*}}, label %omp.outer.log.scan.body, label %omp.outer.log.scan.exit
//CHECK: omp.outer.log.scan.body:
@@ -121,7 +121,7 @@ llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i3
//CHECK: %[[IND2PTR:.+]] = getelementptr inbounds i32, ptr %[[BUFF]], i32 %[[IND2]]
//CHECK: %[[IND1VAL:.+]] = load i32, ptr %[[IND1PTR]], align 4
//CHECK: %[[IND2VAL:.+]] = load i32, ptr %[[IND2PTR]], align 4
-//CHECK: %[[REDVAL:.+]] = add i32 %[[IND1VAL]], %[[IND2VAL]]
+//CHECK: %[[REDVAL:.+]] = add i32 %[[IND2VAL]], %[[IND1VAL]]
//CHECK: store i32 %[[REDVAL]], ptr %[[IND1PTR]], align 4
//CHECK: %[[CNTNXT]] = sub nuw i32 %[[CNT]], 1
//CHECK: %[[CMP3:.+]] = icmp uge i32 %[[CNTNXT]], %[[I]]
More information about the flang-commits
mailing list