[flang-commits] [flang] 57e9c03 - [flang] Decouple HLFIR array expansion from optimization level (#233478)
via flang-commits
flang-commits at lists.llvm.org
Tue Oct 20 01:33:16 PDT 2026
Author: jeanPerier
Date: 2026-10-20T10:33:10+02:00
New Revision: 57e9c031bb2151184e60132ae438e9ea1bb48143
URL: https://github.com/llvm/llvm-project/commit/57e9c031bb2151184e60132ae438e9ea1bb48143
DIFF: https://github.com/llvm/llvm-project/commit/57e9c031bb2151184e60132ae438e9ea1bb48143.diff
LOG: [flang] Decouple HLFIR array expansion from optimization level (#233478)
Add optimizeArrayExpression option to the pass pipeline configuration,
set to false by default for now.
It will allow preserving runtime calls for array expressions when set to
false once array temporary
optimizations are default at O0.
Always attempt to inline scalar-to-array assignments, including host
code at O0.
Update pipeline expectations and add coverage for broadcasts, array
expressions, and reductions.
The goal is to make this option always true by default once we start
emitting some fir.assert operation.
Part of
https://discourse.llvm.org/t/rfc-flang-expanding-array-expressions-and-assignments-at-o0/91971
Added:
flang/test/Driver/array-runtime.f90
flang/test/Integration/scalar-to-array-assign-host-O0.f90
Modified:
flang/include/flang/Optimizer/HLFIR/Passes.td
flang/include/flang/Tools/CrossToolHelpers.h
flang/lib/Frontend/FrontendActions.cpp
flang/lib/Optimizer/HLFIR/Transforms/InlineHLFIRAssign.cpp
flang/lib/Optimizer/Passes/Pipelines.cpp
flang/test/CMakeLists.txt
flang/test/Driver/mlir-debug-pass-pipeline.f90
flang/test/Driver/mlir-omp-pass-pipeline.f90
flang/test/Driver/mlir-pass-pipeline.f90
flang/test/Integration/OpenMP/parallel-private-reduction-worstcase.f90
flang/test/Integration/OpenMP/private-global.f90
flang/test/Integration/prefetch.f90
flang/test/Lower/CUDA/scalar-to-array-assign-device-O0.cuf
flang/test/Lower/OpenMP/workdistribute-saxpy-and-scalar-assign.f90
flang/test/Lower/OpenMP/workdistribute-scalar-assign.f90
flang/tools/bbc/bbc.cpp
Removed:
flang/test/Lower/scalar-to-array-assign-host-O0.f90
################################################################################
diff --git a/flang/include/flang/Optimizer/HLFIR/Passes.td b/flang/include/flang/Optimizer/HLFIR/Passes.td
index 2835b3119fa2f..e0221c655834f 100644
--- a/flang/include/flang/Optimizer/HLFIR/Passes.td
+++ b/flang/include/flang/Optimizer/HLFIR/Passes.td
@@ -103,13 +103,8 @@ def InlineHLFIRAssign : Pass<"inline-hlfir-assign"> {
/*default=*/"false",
"Only inline assignments where RHS is a numeric or "
"logical scalar broadcast "
- "to an array LHS. Used at O0 to avoid runtime calls in "
- "device code without inlining all array assignments.">,
- Option<"onlyCUDADeviceContext", "only-cuda-device-context",
- "bool", /*default=*/"false",
- "Only inline assignments in CUDA Fortran device "
- "code. Used at O0, where a module holds both host "
- "and device subprograms, to leave host code alone.">];
+ "to an array LHS, even when other array operations "
+ "are left to the runtime.">];
}
def InlineHLFIRCopy : Pass<"inline-hlfir-copy"> {
diff --git a/flang/include/flang/Tools/CrossToolHelpers.h b/flang/include/flang/Tools/CrossToolHelpers.h
index b9add141663b9..7324bc19f18a1 100644
--- a/flang/include/flang/Tools/CrossToolHelpers.h
+++ b/flang/include/flang/Tools/CrossToolHelpers.h
@@ -112,13 +112,13 @@ class FlangEPCallBacks {
/// Configuriation for the MLIR to LLVM pass pipeline.
struct MLIRToLLVMPassPipelineConfig : public FlangEPCallBacks {
- explicit MLIRToLLVMPassPipelineConfig(llvm::OptimizationLevel level) {
- OptLevel = level;
- }
+ explicit MLIRToLLVMPassPipelineConfig(llvm::OptimizationLevel level)
+ : OptLevel(level),
+ optimizeArrayExpression(level != llvm::OptimizationLevel::O0) {}
explicit MLIRToLLVMPassPipelineConfig(llvm::OptimizationLevel level,
const Fortran::frontend::CodeGenOptions &opts,
- const Fortran::common::MathOptionsBase &mathOpts) {
- OptLevel = level;
+ const Fortran::common::MathOptionsBase &mathOpts)
+ : MLIRToLLVMPassPipelineConfig(level) {
StackArrays = opts.StackArrays;
EnableSafeTrampoline = opts.EnableSafeTrampoline;
Underscoring = opts.Underscoring;
@@ -151,6 +151,8 @@ struct MLIRToLLVMPassPipelineConfig : public FlangEPCallBacks {
}
llvm::OptimizationLevel OptLevel; ///< optimisation level
+ /// Enable HLFIR optimizations; expensive ones still require O3.
+ bool optimizeArrayExpression;
bool StackArrays = false; ///< convert memory allocations to alloca.
bool EnableSafeTrampoline{false}; ///< Use runtime trampoline pool (W^X).
bool Underscoring = true; ///< add underscores to function names.
@@ -174,10 +176,7 @@ struct MLIRToLLVMPassPipelineConfig : public FlangEPCallBacks {
///< functions.
bool NSWOnLoopVarInc = true; ///< Add nsw flag to loop variable increments.
bool EnableOpenACC = false; ///< Enable OpenACC lowering.
- bool EnableCUDA = false; ///< Enable CUDA Fortran lowering.
bool EnableOpenMP = false; ///< Enable OpenMP lowering.
- bool EnableOpenMPIsTargetDevice =
- false; ///< Compiling for an OpenMP target device.
bool UseSampleProfile = false; ///< Enable sample based profiling
bool UniqueInternalLinkageNames = false; ///< Append MD5 hash suffix to
///< internal linkage symbol names.
diff --git a/flang/lib/Frontend/FrontendActions.cpp b/flang/lib/Frontend/FrontendActions.cpp
index 62dd3fb054863..95cd8ac0eff53 100644
--- a/flang/lib/Frontend/FrontendActions.cpp
+++ b/flang/lib/Frontend/FrontendActions.cpp
@@ -647,13 +647,6 @@ void CodeGenAction::lowerHLFIRToFIR() {
MLIRToLLVMPassPipelineConfig config(level);
config.fpMaxminBehavior =
ci.getInvocation().getLoweringOpts().getFPMaxminBehavior();
- if (ci.getInvocation().getLangOpts().OpenMPIsTargetDevice)
- config.EnableOpenMPIsTargetDevice = true;
- // Parser options, not frontend options: they are seeded from the frontend
- // ones and additionally get CUDA enabled for a .cuf/.CUF input.
- if (ci.getInvocation().getFortranOpts().features.IsEnabled(
- Fortran::common::LanguageFeature::CUDA))
- config.EnableCUDA = true;
// Give plugins a chance to register passes at the extension points.
fir::invokePassPipelineConfigCallbacks(config);
// Create the pass pipeline
@@ -790,9 +783,6 @@ void CodeGenAction::generateLLVMIR() {
Fortran::common::LanguageFeature::OpenMP))
config.EnableOpenMP = true;
- if (ci.getInvocation().getLangOpts().OpenMPIsTargetDevice)
- config.EnableOpenMPIsTargetDevice = true;
-
if (ci.getInvocation().getLangOpts().OpenMPSimd)
config.EnableOpenMPSimd = true;
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/InlineHLFIRAssign.cpp b/flang/lib/Optimizer/HLFIR/Transforms/InlineHLFIRAssign.cpp
index 59464cb06d4e7..44affd514f613 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/InlineHLFIRAssign.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/InlineHLFIRAssign.cpp
@@ -13,11 +13,9 @@
#include "flang/Optimizer/Analysis/AliasAnalysis.h"
#include "flang/Optimizer/Analysis/ArraySectionAnalyzer.h"
#include "flang/Optimizer/Builder/BoxValue.h"
-#include "flang/Optimizer/Builder/CUFCommon.h"
#include "flang/Optimizer/Builder/FIRBuilder.h"
#include "flang/Optimizer/Builder/HLFIRTools.h"
#include "flang/Optimizer/Builder/MutableBox.h"
-#include "flang/Optimizer/Dialect/CUF/Attributes/CUFAttr.h"
#include "flang/Optimizer/HLFIR/HLFIROps.h"
#include "flang/Optimizer/HLFIR/Passes.h"
#include "flang/Optimizer/OpenMP/Passes.h"
@@ -41,19 +39,6 @@ static llvm::cl::opt<bool> inlineAllocatableExprAssignFlag(
"hlfir.expr (e.g., from hlfir.elemental)"),
llvm::cl::init(false));
-/// Returns true if \p op is CUDA Fortran device code. attributes(host,device)
-/// counts: this pass runs before the device copy is outlined, so the one
-/// func.func here becomes both copies.
-static bool isCUDADeviceCode(mlir::Operation *op) {
- if (cuf::isCUDADeviceContext(op))
- return true;
- if (auto func = op->getParentOfType<mlir::func::FuncOp>())
- if (auto procAttr =
- func->getAttrOfType<cuf::ProcAttributeAttr>(cuf::getProcAttrName()))
- return procAttr.getValue() == cuf::ProcAttribute::HostDevice;
- return false;
-}
-
namespace {
/// Expand hlfir.assign of array RHS to array LHS into a loop nest
/// of element-by-element assignments. Also handles scalar RHS broadcast
@@ -80,13 +65,10 @@ namespace {
class InlineHLFIRAssignConversion
: public mlir::OpRewritePattern<hlfir::AssignOp> {
bool onlyScalarRHS;
- bool onlyCUDADeviceContext;
public:
- InlineHLFIRAssignConversion(mlir::MLIRContext *context, bool onlyScalarRHS,
- bool onlyCUDADeviceContext)
- : OpRewritePattern(context), onlyScalarRHS(onlyScalarRHS),
- onlyCUDADeviceContext(onlyCUDADeviceContext) {}
+ InlineHLFIRAssignConversion(mlir::MLIRContext *context, bool onlyScalarRHS)
+ : OpRewritePattern(context), onlyScalarRHS(onlyScalarRHS) {}
llvm::LogicalResult
matchAndRewrite(hlfir::AssignOp assign,
@@ -106,11 +88,6 @@ class InlineHLFIRAssignConversion
return rewriter.notifyMatchFailure(
assign, "onlyScalarRHS: skipping array-to-array assignment");
- // Host code keeps the runtime call so that a breakpoint on the assignment
- // fires once, not once per element.
- if (onlyCUDADeviceContext && !isCUDADeviceCode(assign))
- return rewriter.notifyMatchFailure(assign, "skipping host code");
-
mlir::Type rhsEleTy = rhs.getFortranElementType();
if (!fir::isa_trivial(rhsEleTy))
return rewriter.notifyMatchFailure(
@@ -379,29 +356,13 @@ class InlineHLFIRAssignPass
void runOnOperation() override {
mlir::MLIRContext *context = &getContext();
- // Bail out before the greedy driver runs: it folds and removes dead ops
- // as it walks, which would perturb host code at O0 even though the
- // pattern rewrites nothing there.
- if (onlyCUDADeviceContext) {
- bool anyDeviceAssign = false;
- getOperation()->walk([&](hlfir::AssignOp assign) {
- if (!isCUDADeviceCode(assign))
- return mlir::WalkResult::advance();
- anyDeviceAssign = true;
- return mlir::WalkResult::interrupt();
- });
- if (!anyDeviceAssign)
- return;
- }
-
mlir::GreedyRewriteConfig config;
// Prevent the pattern driver from merging blocks.
config.setRegionSimplificationLevel(
mlir::GreedySimplifyRegionLevel::Disabled);
mlir::RewritePatternSet patterns(context);
- patterns.insert<InlineHLFIRAssignConversion>(context, onlyScalarRHS,
- onlyCUDADeviceContext);
+ patterns.insert<InlineHLFIRAssignConversion>(context, onlyScalarRHS);
// Optionally add the allocatable expr assignment pattern
if (inlineAllocatableExprAssignFlag) {
diff --git a/flang/lib/Optimizer/Passes/Pipelines.cpp b/flang/lib/Optimizer/Passes/Pipelines.cpp
index 953104d0625b7..bb217c89f86ea 100644
--- a/flang/lib/Optimizer/Passes/Pipelines.cpp
+++ b/flang/lib/Optimizer/Passes/Pipelines.cpp
@@ -292,7 +292,7 @@ void createHLFIRToFIRPassPipeline(mlir::PassManager &pm,
config.invokeHLFIROptEarlyEPCallbacks(pm, optLevel);
- if (optLevel != llvm::OptimizationLevel::O0) {
+ if (config.optimizeArrayExpression) {
addNestedPassToAllTopLevelOperations<PassConstructor>(
pm, hlfir::createExpressionSimplification);
addCanonicalizerPassWithoutRegionSimplification(pm);
@@ -305,7 +305,7 @@ void createHLFIRToFIRPassPipeline(mlir::PassManager &pm,
pm, hlfir::createInlineElementals);
addNestedPassToAllTopLevelOperations<PassConstructor>(
pm, hlfir::createSeparateAllocatableAssign);
- if (optLevel != llvm::OptimizationLevel::O0) {
+ if (config.optimizeArrayExpression) {
addCanonicalizerPassWithoutRegionSimplification(pm);
pm.addPass(mlir::createCSEPass());
// Run SimplifyHLFIRIntrinsics pass late after CSE,
@@ -325,40 +325,26 @@ void createHLFIRToFIRPassPipeline(mlir::PassManager &pm,
addNestedPassToAllTopLevelOperations<PassConstructor>(
pm, hlfir::createInlineHLFIRCopy);
}
- } else if (config.EnableOpenMPIsTargetDevice &&
- enableOpenMP == EnableOpenMP::Full) {
- // At O0, only inline scalar-to-array broadcasts when compiling for an
- // OpenMP target device. This avoids emitting Fortran runtime calls
- // (e.g. _FortranAAssign) that use malloc/free in device code generated
- // by OpenMP target offloading. Restricting this to target-device
- // compilation preserves the runtime call on the host at -O0 so that a
- // line breakpoint on a scalar-to-array assignment hits once instead of
- // once per element.
+ } else {
+ // Always attempt to inline scalar-to-array assignments, even when other
+ // array operations are left to the runtime.
addNestedPassToAllTopLevelOperations(pm, [&]() {
return hlfir::createInlineHLFIRAssign({/*onlyScalarRHS=*/true});
});
- } else if (config.EnableCUDA) {
- // Same at O0 for CUDA Fortran device code, where the runtime call also
- // inflates the stack frame the device linker reserves for the kernel.
- // The module holds host code too, hence onlyCUDADeviceContext.
- addNestedPassToAllTopLevelOperations(pm, [&]() {
- return hlfir::createInlineHLFIRAssign(
- {/*onlyScalarRHS=*/true, /*onlyCUDADeviceContext=*/true});
- });
}
pm.addPass(hlfir::createLowerHLFIROrderedAssignments(
- {/*tryFusingAssignments=*/optLevel != llvm::OptimizationLevel::O0}));
+ {/*tryFusingAssignments=*/config.optimizeArrayExpression}));
config.invokeHLFIROptLastEPCallbacks(pm, optLevel);
pm.addPass(hlfir::createLowerHLFIRIntrinsics());
hlfir::BufferizeHLFIROptions bufferizeOptions;
- // For opt-for-speed, avoid running any of the loops resulting
+ // When expanding array operations, avoid running any of the loops resulting
// from hlfir.elemental lowering, if the result is an empty array.
// This helps to avoid long running loops for elementals with
// shapes like (0, HUGE).
- if (optLevel != llvm::OptimizationLevel::O0)
+ if (config.optimizeArrayExpression)
bufferizeOptions.optimizeEmptyElementals = true;
pm.addPass(hlfir::createBufferizeHLFIR(bufferizeOptions));
// Run hlfir.assign inlining again after BufferizeHLFIR,
@@ -368,7 +354,7 @@ void createHLFIRToFIRPassPipeline(mlir::PassManager &pm,
// TODO: we can remove the previous InlineHLFIRAssign, when
// FIR AliasAnalysis is good enough to say that a temporary
// array does not alias with any user object.
- if (optLevel != llvm::OptimizationLevel::O0)
+ if (config.optimizeArrayExpression)
addNestedPassToAllTopLevelOperations<PassConstructor>(
pm, hlfir::createInlineHLFIRAssign);
pm.addPass(hlfir::createConvertHLFIRtoFIR());
diff --git a/flang/test/CMakeLists.txt b/flang/test/CMakeLists.txt
index 50980a521b241..34cc7c72016ad 100644
--- a/flang/test/CMakeLists.txt
+++ b/flang/test/CMakeLists.txt
@@ -114,6 +114,7 @@ if (NOT FLANG_STANDALONE_BUILD)
not
llvm-bcanalyzer
llvm-dis
+ llvm-dwarfdump
llvm-objcopy
llvm-objdump
llvm-profdata
diff --git a/flang/test/Driver/array-runtime.f90 b/flang/test/Driver/array-runtime.f90
new file mode 100644
index 0000000000000..9acec878fb3da
--- /dev/null
+++ b/flang/test/Driver/array-runtime.f90
@@ -0,0 +1,65 @@
+! Numeric broadcasts to non-allocatable arrays are expanded even at -O0.
+! Other array operations retain the optimization-level-dependent default.
+! RUN: %flang_fc1 -emit-fir -O0 %s -o - | FileCheck %s --check-prefixes=ALL,RUNTIME
+! RUN: %flang_fc1 -emit-fir -O1 %s -o - | FileCheck %s --check-prefixes=ALL,EXPAND
+! RUN: %flang_fc1 -emit-fir -O2 %s -o - | FileCheck %s --check-prefixes=ALL,EXPAND
+
+subroutine broadcast(x)
+ real :: x(:)
+ x = 1.
+end subroutine
+
+! ALL-LABEL: func{{(\.func)?}} @_QPbroadcast(
+! ALL-NOT: fir.call @_FortranAAssign
+! ALL: fir.do_loop
+! ALL-NOT: fir.call @_FortranAAssign
+! ALL: return
+
+subroutine allocatable_broadcast(x)
+ real, allocatable :: x(:)
+ x = 1.
+end subroutine
+
+! ALL-LABEL: func{{(\.func)?}} @_QPallocatable_broadcast(
+! RUNTIME: fir.call @_FortranAAssign
+! EXPAND-NOT: fir.call @_FortranAAssign
+! EXPAND: fir.do_loop
+! EXPAND-NOT: fir.call @_FortranAAssign
+! ALL: return
+
+subroutine array_copy(x, y)
+ real :: x(:), y(:)
+ x = y
+end subroutine
+
+! ALL-LABEL: func{{(\.func)?}} @_QParray_copy(
+! RUNTIME: fir.call @_FortranAAssign
+! EXPAND-NOT: fir.call @_FortranAAssign
+! EXPAND: fir.do_loop
+! EXPAND-NOT: fir.call @_FortranAAssign
+! ALL: return
+
+subroutine array_expression(x, y, z)
+ real :: x(:), y(:), z(:)
+ x = y + z
+end subroutine
+
+! ALL-LABEL: func{{(\.func)?}} @_QParray_expression(
+! RUNTIME: fir.call @_FortranAAssign
+! EXPAND-NOT: fir.call @_FortranAAssign
+! EXPAND: fir.do_loop
+! EXPAND: arith.addf
+! EXPAND-NOT: fir.call @_FortranAAssign
+! ALL: return
+
+function reduction(x)
+ real :: x(:), reduction
+ reduction = sum(x)
+end function
+
+! ALL-LABEL: func{{(\.func)?}} @_QPreduction(
+! RUNTIME: fir.call @_FortranASumReal4
+! EXPAND-NOT: fir.call @_FortranASum
+! EXPAND: fir.do_loop
+! EXPAND-NOT: fir.call @_FortranASum
+! ALL: return
diff --git a/flang/test/Driver/mlir-debug-pass-pipeline.f90 b/flang/test/Driver/mlir-debug-pass-pipeline.f90
index 1cc4fad4f76a4..25fdc4ee55780 100644
--- a/flang/test/Driver/mlir-debug-pass-pipeline.f90
+++ b/flang/test/Driver/mlir-debug-pass-pipeline.f90
@@ -31,18 +31,23 @@
! ALL-NEXT: 'fir.global' Pipeline
! ALL-NEXT: InlineElementals
! ALL-NEXT: SeparateAllocatableAssign
+! ALL-NEXT: InlineHLFIRAssign
! ALL-NEXT: 'func.func' Pipeline
! ALL-NEXT: InlineElementals
! ALL-NEXT: SeparateAllocatableAssign
+! ALL-NEXT: InlineHLFIRAssign
! ALL-NEXT: 'omp.declare_mapper' Pipeline
! ALL-NEXT: InlineElementals
! ALL-NEXT: SeparateAllocatableAssign
+! ALL-NEXT: InlineHLFIRAssign
! ALL-NEXT: 'omp.declare_reduction' Pipeline
! ALL-NEXT: InlineElementals
! ALL-NEXT: SeparateAllocatableAssign
+! ALL-NEXT: InlineHLFIRAssign
! ALL-NEXT: 'omp.private' Pipeline
! ALL-NEXT: InlineElementals
! ALL-NEXT: SeparateAllocatableAssign
+! ALL-NEXT: InlineHLFIRAssign
! ALL-NEXT: LowerHLFIROrderedAssignments
! ALL-NEXT: LowerHLFIRIntrinsics
! ALL-NEXT: BufferizeHLFIR
diff --git a/flang/test/Driver/mlir-omp-pass-pipeline.f90 b/flang/test/Driver/mlir-omp-pass-pipeline.f90
index 40e2c52b993a4..01eb1ee739126 100644
--- a/flang/test/Driver/mlir-omp-pass-pipeline.f90
+++ b/flang/test/Driver/mlir-omp-pass-pipeline.f90
@@ -14,7 +14,7 @@
! NO_OMP-NOT: AutomapToTargetDataPass
! NO_OMP-NOT: MapInfoFinalizationPass
! NO_OMP-NOT: GenericLoopConversionPass
-! NO_OMP-NOT: InlineHLFIRAssign
+! NO_OMP: InlineHLFIRAssign
! NO_OMP-NOT: LowerWorkshare
! NO_OMP-NOT: LowerWorkdistribute
! NO_OMP-NOT: SimdOnlyPass
@@ -30,7 +30,7 @@
! FULL: AutomapToTargetDataPass
! FULL: MapInfoFinalizationPass
! FULL: GenericLoopConversionPass
-! FULL-NOT: InlineHLFIRAssign
+! FULL: InlineHLFIRAssign
! FULL: LowerWorkshare
! FULL: LowerWorkdistribute
! FULL-NOT: SimdOnlyPass
@@ -47,7 +47,7 @@
! SIMD-NOT: AutomapToTargetDataPass
! SIMD-NOT: MapInfoFinalizationPass
! SIMD-NOT: GenericLoopConversionPass
-! SIMD-NOT: InlineHLFIRAssign
+! SIMD: InlineHLFIRAssign
! SIMD-NOT: LowerWorkshare
! SIMD-NOT: LowerWorkdistribute
! SIMD: SimdOnlyPass
diff --git a/flang/test/Driver/mlir-pass-pipeline.f90 b/flang/test/Driver/mlir-pass-pipeline.f90
index f7f4a29ab2de9..d91d0e0177bf3 100644
--- a/flang/test/Driver/mlir-pass-pipeline.f90
+++ b/flang/test/Driver/mlir-pass-pipeline.f90
@@ -9,11 +9,6 @@
end program
-! At -O0 on the host (no OpenMP target-device compilation), InlineHLFIRAssign
-! is no longer scheduled. See PR #197092 follow-up restricting the -O0 pass
-! to OpenMP target-device compilation.
-! O0-NOT: InlineHLFIRAssign
-
! ALL: Pass statistics report
! ALL: Fortran::lower::VerifierPass
@@ -37,22 +32,27 @@
! O2-NEXT: SimplifyHLFIRIntrinsics
! ALL: InlineElementals
! ALL-NEXT: SeparateAllocatableAssign
+! O0-NEXT: InlineHLFIRAssign
! ALL-NEXT:'func.func' Pipeline
! O2-NEXT: SimplifyHLFIRIntrinsics
! ALL: InlineElementals
! ALL-NEXT: SeparateAllocatableAssign
+! O0-NEXT: InlineHLFIRAssign
! ALL-NEXT:'omp.declare_mapper' Pipeline
! O2-NEXT: SimplifyHLFIRIntrinsics
! ALL: InlineElementals
! ALL-NEXT: SeparateAllocatableAssign
+! O0-NEXT: InlineHLFIRAssign
! ALL-NEXT:'omp.declare_reduction' Pipeline
! O2-NEXT: SimplifyHLFIRIntrinsics
! ALL: InlineElementals
! ALL-NEXT: SeparateAllocatableAssign
+! O0-NEXT: InlineHLFIRAssign
! ALL-NEXT:'omp.private' Pipeline
! O2-NEXT: SimplifyHLFIRIntrinsics
! ALL: InlineElementals
! ALL-NEXT: SeparateAllocatableAssign
+! O0-NEXT: InlineHLFIRAssign
! O2-NEXT: Canonicalizer
! O2-NEXT: CSE
! O2-NEXT: (S) {{.*}} num-cse'd
diff --git a/flang/test/Integration/OpenMP/parallel-private-reduction-worstcase.f90 b/flang/test/Integration/OpenMP/parallel-private-reduction-worstcase.f90
index ea52c2a2b5bb8..4f86c590b7e95 100644
--- a/flang/test/Integration/OpenMP/parallel-private-reduction-worstcase.f90
+++ b/flang/test/Integration/OpenMP/parallel-private-reduction-worstcase.f90
@@ -117,44 +117,62 @@ subroutine worst_case(a, b, c, d)
! [malloc and assign the default value to the reduction variable]
! CHECK: br label %omp.reduction.neutral46
-! CHECK: omp.reduction.neutral46: ; preds = %omp.reduction.neutral44, %omp.reduction.neutral45
+! CHECK: omp.reduction.neutral46: ; preds = %omp.reduction.neutral47, %omp.reduction.neutral45
+! [loop assigning the neutral value to each element]
+! CHECK-NEXT: %{{.*}} = phi i64
+! CHECK-NEXT: %{{.*}} = phi i64
+! CHECK: br i1 %{{.*}}, label %omp.reduction.neutral47, label %omp.reduction.neutral48
+
+! CHECK: omp.reduction.neutral48: ; preds = %omp.reduction.neutral46
+! CHECK: br label %omp.reduction.neutral49
+
+! CHECK: omp.reduction.neutral49: ; preds = %omp.reduction.neutral44, %omp.reduction.neutral48
! CHECK-NEXT: br label %omp.region.cont43
-! CHECK: omp.region.cont43: ; preds = %omp.reduction.neutral46
+! CHECK: omp.region.cont43: ; preds = %omp.reduction.neutral49
! CHECK-NEXT: %{{.*}} = phi ptr
-! CHECK-NEXT: br label %omp.reduction.neutral48
+! CHECK-NEXT: br label %omp.reduction.neutral51
-! CHECK: omp.reduction.neutral48: ; preds = %omp.region.cont43
+! CHECK: omp.reduction.neutral51: ; preds = %omp.region.cont43
! [start of reduction initialization region]
! [null check:]
-! CHECK: br i1 %{{.*}}, label %omp.reduction.neutral49, label %omp.reduction.neutral50
+! CHECK: br i1 %{{.*}}, label %omp.reduction.neutral52, label %omp.reduction.neutral53
-! CHECK: omp.reduction.neutral50: ; preds = %omp.reduction.neutral48
+! CHECK: omp.reduction.neutral53: ; preds = %omp.reduction.neutral51
! [malloc and assign the default value to the reduction variable]
-! CHECK: br label %omp.reduction.neutral51
+! CHECK: br label %omp.reduction.neutral54
+
+! CHECK: omp.reduction.neutral54: ; preds = %omp.reduction.neutral55, %omp.reduction.neutral53
+! [loop assigning the neutral value to each element]
+! CHECK-NEXT: %{{.*}} = phi i64
+! CHECK-NEXT: %{{.*}} = phi i64
+! CHECK: br i1 %{{.*}}, label %omp.reduction.neutral55, label %omp.reduction.neutral56
+
+! CHECK: omp.reduction.neutral56: ; preds = %omp.reduction.neutral54
+! CHECK: br label %omp.reduction.neutral57
-! CHECK: omp.reduction.neutral51: ; preds = %omp.reduction.neutral49, %omp.reduction.neutral50
-! CHECK-NEXT: br label %omp.region.cont47
+! CHECK: omp.reduction.neutral57: ; preds = %omp.reduction.neutral52, %omp.reduction.neutral56
+! CHECK-NEXT: br label %omp.region.cont50
-! CHECK: omp.region.cont47: ; preds = %omp.reduction.neutral51
+! CHECK: omp.region.cont50: ; preds = %omp.reduction.neutral57
! CHECK-NEXT: %{{.*}} = phi ptr
-! CHECK-NEXT: br label %omp.par.region53
+! CHECK-NEXT: br label %omp.par.region59
-! CHECK: omp.par.region53: ; preds = %omp.region.cont47
-! CHECK-NEXT: br label %omp.par.region54
+! CHECK: omp.par.region59: ; preds = %omp.region.cont50
+! CHECK-NEXT: br label %omp.par.region60
-! CHECK: omp.par.region54: ; preds = %omp.par.region53
+! CHECK: omp.par.region60: ; preds = %omp.par.region59
! [call SUM runtime function]
! [if (sum(a) == 1)]
-! CHECK: br i1 %{{.*}}, label %omp.par.region55, label %omp.par.region56
+! CHECK: br i1 %{{.*}}, label %omp.par.region61, label %omp.par.region62
-! CHECK: omp.par.region56: ; preds = %omp.par.region54
-! CHECK-NEXT: br label %omp.par.region57
+! CHECK: omp.par.region62: ; preds = %omp.par.region60
+! CHECK-NEXT: br label %omp.par.region63
-! CHECK: omp.par.region57: ; preds = %omp.par.region56
-! CHECK-NEXT: br label %omp.region.cont52
+! CHECK: omp.par.region63: ; preds = %omp.par.region62
+! CHECK-NEXT: br label %omp.region.cont58
-! CHECK: omp.region.cont52: ; preds = %omp.par.region57
+! CHECK: omp.region.cont58: ; preds = %omp.par.region63
! [omp parallel region done, call into the runtime to complete reduction]
! CHECK: %[[VAL_233:.*]] = call i32 @__kmpc_reduce(
! CHECK: switch i32 %[[VAL_233]], label %reduce.finalize [
@@ -162,16 +180,16 @@ subroutine worst_case(a, b, c, d)
! CHECK-NEXT: i32 2, label %reduce.switch.atomic
! CHECK-NEXT: ]
-! CHECK: reduce.switch.atomic: ; preds = %omp.region.cont52
+! CHECK: reduce.switch.atomic: ; preds = %omp.region.cont58
! CHECK-NEXT: unreachable
-! CHECK: reduce.switch.nonatomic: ; preds = %omp.region.cont52
+! CHECK: reduce.switch.nonatomic: ; preds = %omp.region.cont58
! CHECK-NEXT: %[[red_private_value_0:.*]] = load ptr, ptr %{{.*}}, align 8
! CHECK-NEXT: br label %omp.reduction.nonatomic.body
! [various blocks implementing the reduction]
-! CHECK: omp.region.cont62: ; preds =
+! CHECK: omp.region.cont68: ; preds =
! CHECK-NEXT: %{{.*}} = phi ptr
! CHECK-NEXT: call void @__kmpc_end_reduce(
! CHECK-NEXT: br label %reduce.finalize
@@ -188,30 +206,38 @@ subroutine worst_case(a, b, c, d)
! CHECK: omp.reduction.cleanup: ; preds = %.fini
! [null check]
-! CHECK: br i1 %{{.*}}, label %omp.reduction.cleanup68, label %omp.reduction.cleanup69
+! CHECK: br i1 %{{.*}}, label %omp.reduction.cleanup74, label %omp.reduction.cleanup75
-! CHECK: omp.reduction.cleanup69: ; preds = %omp.reduction.cleanup68, %omp.reduction.cleanup
-! CHECK-NEXT: br label %omp.region.cont67
+! CHECK: omp.reduction.cleanup75: ; preds = %omp.reduction.cleanup74, %omp.reduction.cleanup
+! CHECK-NEXT: br label %omp.region.cont73
-! CHECK: omp.region.cont67: ; preds = %omp.reduction.cleanup69
+! CHECK: omp.region.cont73: ; preds = %omp.reduction.cleanup75
! CHECK-NEXT: %{{.*}} = load ptr, ptr
-! CHECK-NEXT: br label %omp.reduction.cleanup71
+! CHECK-NEXT: br label %omp.reduction.cleanup77
-! CHECK: omp.reduction.cleanup71: ; preds = %omp.region.cont67
+! CHECK: omp.reduction.cleanup77: ; preds = %omp.region.cont73
! [null check]
-! CHECK: br i1 %{{.*}}, label %omp.reduction.cleanup72, label %omp.reduction.cleanup73
+! CHECK: br i1 %{{.*}}, label %omp.reduction.cleanup78, label %omp.reduction.cleanup79
-! CHECK: omp.par.region55: ; preds = %omp.par.region54
+! CHECK: omp.par.region61: ; preds = %omp.par.region60
! CHECK-NEXT: call void @_FortranAStopStatement
! CHECK-NEXT: unreachable
-! CHECK: omp.reduction.neutral49: ; preds = %omp.reduction.neutral48
+! CHECK: omp.reduction.neutral55: ; preds = %omp.reduction.neutral54
+! CHECK: store float 0.000000e+00, ptr %{{.*}}, align 4
+! CHECK: br label %omp.reduction.neutral54
+
+! CHECK: omp.reduction.neutral52: ; preds = %omp.reduction.neutral51
! [source length was zero: finish initializing array]
-! CHECK: br label %omp.reduction.neutral51
+! CHECK: br label %omp.reduction.neutral57
+
+! CHECK: omp.reduction.neutral47: ; preds = %omp.reduction.neutral46
+! CHECK: store float 0.000000e+00, ptr %{{.*}}, align 4
+! CHECK: br label %omp.reduction.neutral46
! CHECK: omp.reduction.neutral44: ; preds = %omp.reduction.neutral
! [source length was zero: finish initializing array]
-! CHECK: br label %omp.reduction.neutral46
+! CHECK: br label %omp.reduction.neutral49
! CHECK: omp.private.copy41: ; preds = %omp.private.copy40, %omp.private.copy37
! [source length was non-zero: call assign runtime]
@@ -247,5 +273,5 @@ subroutine worst_case(a, b, c, d)
! [var extent was non-zero: malloc a private array]
! CHECK: br label %omp.private.init5
-! CHECK: omp.par.exit.exitStub: ; preds = %omp.region.cont77
+! CHECK: omp.par.exit.exitStub: ; preds = %omp.region.cont83
! CHECK-NEXT: ret void
diff --git a/flang/test/Integration/OpenMP/private-global.f90 b/flang/test/Integration/OpenMP/private-global.f90
index 56310a3ca21a4..2d1e63eefd8ca 100644
--- a/flang/test/Integration/OpenMP/private-global.f90
+++ b/flang/test/Integration/OpenMP/private-global.f90
@@ -22,17 +22,13 @@ program bug
! and no heap allocation for the private itself.
! CHECK: %[[PRIV_TABLE:.*]] = alloca [10 x i32], align 4
-! check that we use the private copy of table for the assignment: its data
-! pointer is placed in a descriptor which -- threaded through the store and
-! memcpy below -- is the destination handed to the runtime assign.
-! CHECK: omp.par.region1:
-! CHECK: %[[TABLE_DESC:.*]] = insertvalue {{.*}}, ptr %[[PRIV_TABLE]], 0
-! CHECK: store {{.*}} %[[TABLE_DESC]], ptr %[[DESC_MEM:.*]], align
-! CHECK: call void @llvm.memcpy.p0.p0.i32(ptr %[[ASSIGN_DST:.*]], ptr %[[DESC_MEM]], i32 {{.*}}, i1 false)
-! CHECK: call void @_FortranAAssign(ptr %[[ASSIGN_DST]],
-
! check that we use the private copy of table for table/=50
-! CHECK: omp.par.region3:
+! CHECK: omp.par.region7:
! CHECK: %[[ELT_PTR:.*]] = getelementptr {{.*}} i32, ptr %[[PRIV_TABLE]], i64 %{{.*}}
! CHECK: %[[ELT:.*]] = load i32, ptr %[[ELT_PTR]]
! CHECK: icmp ne i32 %[[ELT]], 50
+
+! check that the inline assignment loop stores into the private copy of table
+! CHECK: omp.par.region3:
+! CHECK: %[[ASSIGN_ELT_PTR:.*]] = getelementptr {{.*}} i32, ptr %[[PRIV_TABLE]], i64 %{{.*}}
+! CHECK-NEXT: store i32 50, ptr %[[ASSIGN_ELT_PTR]], align 4
diff --git a/flang/test/Integration/prefetch.f90 b/flang/test/Integration/prefetch.f90
index c015b6736972a..76227caf02b43 100644
--- a/flang/test/Integration/prefetch.f90
+++ b/flang/test/Integration/prefetch.f90
@@ -13,7 +13,6 @@
!===============================================================================
subroutine test_prefetch_01()
- ! LLVM: {{.*}} = alloca i32, i64 1, align 4
! LLVM: %[[VAR_J:.*]] = alloca i32, i64 1, align 4
! LLVM: %[[VAR_I:.*]] = alloca i32, i64 1, align 4
! LLVM: %[[VAR_A:.*]] = alloca [256 x i32], i64 1, align 4
diff --git a/flang/test/Integration/scalar-to-array-assign-host-O0.f90 b/flang/test/Integration/scalar-to-array-assign-host-O0.f90
new file mode 100644
index 0000000000000..1a4e6ae26b67f
--- /dev/null
+++ b/flang/test/Integration/scalar-to-array-assign-host-O0.f90
@@ -0,0 +1,12 @@
+! A breakpoint on a scalar broadcast must stop once at -O0, even though
+! the assignment is expanded into a loop. Require exactly one statement row.
+! REQUIRES: x86-registered-target
+
+! RUN: %flang_fc1 -triple x86_64-unknown-linux-gnu -emit-obj -debug-info-kind=line-tables-only -O0 %s -o %t.o && llvm-dwarfdump --debug-line %t.o | FileCheck %s
+
+subroutine host_scalar_broadcast(arr)
+ integer :: arr(4)
+ ! CHECK: 0x{{[0-9a-f]+}} [[#@LINE+2]] {{.*}} is_stmt
+ ! CHECK-NOT: 0x{{[0-9a-f]+}} [[#@LINE+1]] {{.*}} is_stmt
+ arr = 11
+end subroutine
diff --git a/flang/test/Lower/CUDA/scalar-to-array-assign-device-O0.cuf b/flang/test/Lower/CUDA/scalar-to-array-assign-device-O0.cuf
index 7fa6291b93f33..ad9ded3ff81b7 100644
--- a/flang/test/Lower/CUDA/scalar-to-array-assign-device-O0.cuf
+++ b/flang/test/Lower/CUDA/scalar-to-array-assign-device-O0.cuf
@@ -1,6 +1,6 @@
! At -O0, a scalar-to-array broadcast must be inlined in device code to keep
-! _FortranAAssign out of the kernel's call graph. Host code keeps the call so
-! that a breakpoint on the assignment fires once, not once per element.
+! _FortranAAssign out of the kernel's call graph. Host code also inlines
+! scalar-to-array broadcasts.
! CUDA Fortran is enabled here by the .cuf suffix rather than -x cuda.
! RUN: %flang_fc1 -emit-fir -O0 %s -o - | FileCheck %s
@@ -41,4 +41,6 @@ end module
! CHECK-NOT: fir.call @_FortranAAssign
! CHECK-LABEL: func.func @_QMmPhost_broadcast
-! CHECK: fir.call @_FortranAAssign
+! CHECK-NOT: fir.call @_FortranAAssign
+! CHECK: fir.do_loop
+! CHECK-NOT: fir.call @_FortranAAssign
diff --git a/flang/test/Lower/OpenMP/workdistribute-saxpy-and-scalar-assign.f90 b/flang/test/Lower/OpenMP/workdistribute-saxpy-and-scalar-assign.f90
index 09e3375efa487..8e005ddd9dea1 100644
--- a/flang/test/Lower/OpenMP/workdistribute-saxpy-and-scalar-assign.f90
+++ b/flang/test/Lower/OpenMP/workdistribute-saxpy-and-scalar-assign.f90
@@ -60,20 +60,15 @@ subroutine teams_workdistribute()
!$omp end teams workdistribute
end subroutine teams_workdistribute
-! At -O0 host (no -fopenmp-is-target-device):
-! - target teams workdistribute inlines both the saxpy and the scalar
-! broadcast via workdistributeRuntimeCallLower, so no _FortranAAssign
-! runtime call appears.
-! - teams workdistribute (no target) keeps the saxpy result write-back
-! and the scalar broadcast as _FortranAAssign runtime calls. The
-! two calls correspond to source lines 50 (saxpy) and 58 (scalar
-! broadcast) respectively; the ordered CHECK-O0 lines below match
-! each region in source order so neither expectation can accidentally
-! match the other assignment.
+! At -O0, target teams workdistribute expands both assignments. On the host,
+! the saxpy result write-back still uses the runtime, but the scalar broadcast
+! is inlined and workshared.
! CHECK-O0-LABEL: func @_QPtarget_teams_workdistribute
! CHECK-O0-NOT: fir.call @_FortranAAssign
! CHECK-O0-LABEL: func @_QPteams_workdistribute
! CHECK-O0: omp.wsloop
! CHECK-O0: fir.call @_FortranAAssignSimple({{.*}}%c50_i32)
-! CHECK-O0: fir.call @_FortranAAssign({{.*}}%c58_i32)
+! CHECK-O0-NOT: fir.call @_FortranAAssign
+! CHECK-O0: omp.wsloop
+! CHECK-O0-NOT: fir.call @_FortranAAssign
diff --git a/flang/test/Lower/OpenMP/workdistribute-scalar-assign.f90 b/flang/test/Lower/OpenMP/workdistribute-scalar-assign.f90
index 68c9060ee9bd6..ccc5f13bbe67b 100644
--- a/flang/test/Lower/OpenMP/workdistribute-scalar-assign.f90
+++ b/flang/test/Lower/OpenMP/workdistribute-scalar-assign.f90
@@ -35,16 +35,13 @@ subroutine teams_workdistribute_scalar_assign()
end subroutine teams_workdistribute_scalar_assign
-! At -O0 host (no -fopenmp-is-target-device), target teams workdistribute
-! still goes through workdistributeRuntimeCallLower, so the scalar
-! broadcast inside the target region is inlined and workshared, with no
-! _FortranAAssign runtime call.
+! At -O0, scalar broadcasts are inlined and workshared on both host and device.
! CHECK-O0-LABEL: func @_QPtarget_teams_workdistribute_scalar_assign
+! CHECK-O0-NOT: fir.call @_FortranAAssign
! CHECK-O0: omp.wsloop
! CHECK-O0-NOT: fir.call @_FortranAAssign
-! At -O0 host, teams workdistribute (no target) does not inline the
-! scalar broadcast after PR #201774, so the assignment remains a
-! _FortranAAssign runtime call. This pins the plain host -O0 behavior.
! CHECK-O0-LABEL: func @_QPteams_workdistribute_scalar_assign
-! CHECK-O0: fir.call @_FortranAAssign
+! CHECK-O0-NOT: fir.call @_FortranAAssign
+! CHECK-O0: omp.wsloop
+! CHECK-O0-NOT: fir.call @_FortranAAssign
diff --git a/flang/test/Lower/scalar-to-array-assign-host-O0.f90 b/flang/test/Lower/scalar-to-array-assign-host-O0.f90
deleted file mode 100644
index 88d4344da6f2b..0000000000000
--- a/flang/test/Lower/scalar-to-array-assign-host-O0.f90
+++ /dev/null
@@ -1,17 +0,0 @@
-! Regression test for the follow-up to PR llvm/llvm-project#197092.
-!
-! At -O0 on the host (no OpenMP target-device compilation), a scalar-to-array
-! broadcast assignment must lower to a Fortran runtime call
-! (_FortranAAssign), not to an inline assignment loop. Lowering it inline
-! at -O0 caused -g line breakpoints to hit once per array element instead
-! of once.
-
-! RUN: %flang_fc1 -emit-fir -O0 %s -o - | FileCheck %s
-
-! CHECK-LABEL: func @_QPhost_scalar_broadcast
-subroutine host_scalar_broadcast(arr)
- integer :: arr(4)
- ! CHECK: fir.call @_FortranAAssign
- ! CHECK-NOT: fir.do_loop
- arr = 11
-end subroutine
diff --git a/flang/tools/bbc/bbc.cpp b/flang/tools/bbc/bbc.cpp
index 8fc902369f637..6ab05058b05a1 100644
--- a/flang/tools/bbc/bbc.cpp
+++ b/flang/tools/bbc/bbc.cpp
@@ -664,8 +664,6 @@ static llvm::LogicalResult convertFortranSourceToMLIR(
config.SkipConvertComplexPow = targetMachine.getTargetTriple().isAMDGCN();
if (enableOpenMP)
config.EnableOpenMP = true;
- if (enableOpenMPDevice)
- config.EnableOpenMPIsTargetDevice = true;
config.NSWOnLoopVarInc = !integerWrapAround;
fir::registerDefaultInlinerPass(config);
fir::createDefaultFIROptimizerPassPipeline(pm, config);
More information about the flang-commits
mailing list