[Mlir-commits] [clang] [llvm] [mlir] [OMPIRBuilder] Add support for explicit deallocation points (PR #154752)
Sergio Afonso
llvmlistbot at llvm.org
Mon Apr 27 05:12:04 PDT 2026
https://github.com/skatrak updated https://github.com/llvm/llvm-project/pull/154752
>From 47de534edee8bea4827551aa3549d49f04574fa5 Mon Sep 17 00:00:00 2001
From: Sergio Afonso <safonsof at amd.com>
Date: Thu, 21 Aug 2025 13:18:49 +0100
Subject: [PATCH] [OMPIRBuilder] Add support for explicit deallocation points
In this patch, some OMPIRBuilder codegen functions and callbacks are updated to
work with arrays of deallocation insertion points. The purpose of this is to
enable the replacement of `alloca`s with other types of allocations that
require explicit deallocations in a way that makes it possible for
`CodeExtractor` instances created during OMPIRBuilder finalization to also use
them.
The OpenMP to LLVM IR MLIR translation pass is updated to properly store and
forward deallocation points together with their matching allocation point to
the OMPIRBuilder.
Currently, only the `DeviceSharedMemCodeExtractor` uses this feature to get the
`CodeExtractor` to use device shared memory for intermediate allocations when
outlining a parallel region inside of a Generic kernel (code path that is only
used by Flang via MLIR, currently). However, long term this might also be
useful to refactor finalization of variables with destructors, potentially
reducing the use of callbacks and simplifying privatization and reductions.
Instead of a single deallocation point, lists of those are used. This is to
cover cases where there are multiple exit blocks originating from a single
entry. If an allocation needing explicit deallocation is placed in the entry
block of such cases, it would need to be deallocated before each of the exits.
---
clang/lib/CodeGen/CGOpenMPRuntime.cpp | 4 +-
clang/lib/CodeGen/CGStmtOpenMP.cpp | 77 ++--
.../llvm/Frontend/OpenMP/OMPIRBuilder.h | 86 +++--
.../llvm/Transforms/Utils/CodeExtractor.h | 26 +-
llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp | 350 ++++++++----------
llvm/lib/Transforms/IPO/HotColdSplitting.cpp | 2 +-
llvm/lib/Transforms/IPO/IROutliner.cpp | 4 +-
llvm/lib/Transforms/IPO/OpenMPOpt.cpp | 11 +-
llvm/lib/Transforms/IPO/PartialInlining.cpp | 4 +-
llvm/lib/Transforms/Utils/CodeExtractor.cpp | 60 +--
.../Frontend/OpenMPIRBuilderTest.cpp | 321 ++++++++++------
.../Transforms/Utils/CodeExtractorTest.cpp | 4 +-
.../OpenMP/OpenMPToLLVMIRTranslation.cpp | 262 +++++++------
.../LLVMIR/omptarget-parallel-llvm.mlir | 18 +-
.../LLVMIR/omptarget-parallel-wsloop.mlir | 7 +-
.../LLVMIR/omptarget-region-device-llvm.mlir | 4 +-
.../openmp-target-private-allocatable.mlir | 2 +
17 files changed, 689 insertions(+), 553 deletions(-)
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index 332b439c87472..d5371e43f5930 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -11680,8 +11680,8 @@ void CGOpenMPRuntime::emitTargetDataCalls(
llvm::OpenMPIRBuilder::LocationDescription OmpLoc(CodeGenIP);
llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
cantFail(OMPBuilder.createTargetData(
- OmpLoc, AllocaIP, CodeGenIP, DeviceID, IfCondVal, Info, GenMapInfoCB,
- CustomMapperCB,
+ OmpLoc, AllocaIP, CodeGenIP, /*DeallocBlocks=*/{}, DeviceID,
+ IfCondVal, Info, GenMapInfoCB, CustomMapperCB,
/*MapperFunc=*/nullptr, BodyCB, DeviceAddrCB, RTLoc));
CGF.Builder.restoreIP(AfterIP);
}
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index 01de7a410c38f..82307d3a064c6 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -2122,10 +2122,10 @@ void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) {
const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
const Stmt *ParallelRegionBodyStmt = CS->getCapturedStmt();
- auto BodyGenCB = [&, this](InsertPointTy AllocaIP,
- InsertPointTy CodeGenIP) {
+ auto BodyGenCB = [&, this](InsertPointTy AllocIP, InsertPointTy CodeGenIP,
+ ArrayRef<llvm::BasicBlock *> DeallocBlocks) {
OMPBuilderCBHelpers::EmitOMPOutlinedRegionBody(
- *this, ParallelRegionBodyStmt, AllocaIP, CodeGenIP, "parallel");
+ *this, ParallelRegionBodyStmt, AllocIP, CodeGenIP, "parallel");
return llvm::Error::success();
};
@@ -2133,9 +2133,10 @@ void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) {
CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(*this, &CGSI);
llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
AllocaInsertPt->getParent(), AllocaInsertPt->getIterator());
- llvm::OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(
- OMPBuilder.createParallel(Builder, AllocaIP, BodyGenCB, PrivCB, FiniCB,
- IfCond, NumThreads, ProcBind, S.hasCancel()));
+ llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
+ cantFail(OMPBuilder.createParallel(
+ Builder, AllocaIP, /*DeallocBlocks=*/{}, BodyGenCB, PrivCB, FiniCB,
+ IfCond, NumThreads, ProcBind, S.hasCancel()));
Builder.restoreIP(AfterIP);
return;
}
@@ -4669,21 +4670,23 @@ void CodeGenFunction::EmitOMPSectionsDirective(const OMPSectionsDirective &S) {
llvm::SmallVector<BodyGenCallbackTy, 4> SectionCBVector;
if (CS) {
for (const Stmt *SubStmt : CS->children()) {
- auto SectionCB = [this, SubStmt](InsertPointTy AllocaIP,
- InsertPointTy CodeGenIP) {
- OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
- *this, SubStmt, AllocaIP, CodeGenIP, "section");
+ auto SectionCB = [this, SubStmt](
+ InsertPointTy AllocIP, InsertPointTy CodeGenIP,
+ ArrayRef<llvm::BasicBlock *> DeallocBlocks) {
+ OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(*this, SubStmt, AllocIP,
+ CodeGenIP, "section");
return llvm::Error::success();
};
SectionCBVector.push_back(SectionCB);
}
} else {
- auto SectionCB = [this, CapturedStmt](InsertPointTy AllocaIP,
- InsertPointTy CodeGenIP) {
- OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
- *this, CapturedStmt, AllocaIP, CodeGenIP, "section");
- return llvm::Error::success();
- };
+ auto SectionCB =
+ [this, CapturedStmt](InsertPointTy AllocIP, InsertPointTy CodeGenIP,
+ ArrayRef<llvm::BasicBlock *> DeallocBlocks) {
+ OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
+ *this, CapturedStmt, AllocIP, CodeGenIP, "section");
+ return llvm::Error::success();
+ };
SectionCBVector.push_back(SectionCB);
}
@@ -4737,10 +4740,11 @@ void CodeGenFunction::EmitOMPSectionDirective(const OMPSectionDirective &S) {
return llvm::Error::success();
};
- auto BodyGenCB = [SectionRegionBodyStmt, this](InsertPointTy AllocaIP,
- InsertPointTy CodeGenIP) {
+ auto BodyGenCB = [SectionRegionBodyStmt,
+ this](InsertPointTy AllocIP, InsertPointTy CodeGenIP,
+ ArrayRef<llvm::BasicBlock *> DeallocBlocks) {
OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
- *this, SectionRegionBodyStmt, AllocaIP, CodeGenIP, "section");
+ *this, SectionRegionBodyStmt, AllocIP, CodeGenIP, "section");
return llvm::Error::success();
};
@@ -4822,10 +4826,11 @@ void CodeGenFunction::EmitOMPMasterDirective(const OMPMasterDirective &S) {
return llvm::Error::success();
};
- auto BodyGenCB = [MasterRegionBodyStmt, this](InsertPointTy AllocaIP,
- InsertPointTy CodeGenIP) {
+ auto BodyGenCB = [MasterRegionBodyStmt,
+ this](InsertPointTy AllocIP, InsertPointTy CodeGenIP,
+ ArrayRef<llvm::BasicBlock *> DeallocBlocks) {
OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
- *this, MasterRegionBodyStmt, AllocaIP, CodeGenIP, "master");
+ *this, MasterRegionBodyStmt, AllocIP, CodeGenIP, "master");
return llvm::Error::success();
};
@@ -4872,10 +4877,11 @@ void CodeGenFunction::EmitOMPMaskedDirective(const OMPMaskedDirective &S) {
return llvm::Error::success();
};
- auto BodyGenCB = [MaskedRegionBodyStmt, this](InsertPointTy AllocaIP,
- InsertPointTy CodeGenIP) {
+ auto BodyGenCB = [MaskedRegionBodyStmt,
+ this](InsertPointTy AllocIP, InsertPointTy CodeGenIP,
+ ArrayRef<llvm::BasicBlock *> DeallocBlocks) {
OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
- *this, MaskedRegionBodyStmt, AllocaIP, CodeGenIP, "masked");
+ *this, MaskedRegionBodyStmt, AllocIP, CodeGenIP, "masked");
return llvm::Error::success();
};
@@ -4915,10 +4921,11 @@ void CodeGenFunction::EmitOMPCriticalDirective(const OMPCriticalDirective &S) {
return llvm::Error::success();
};
- auto BodyGenCB = [CriticalRegionBodyStmt, this](InsertPointTy AllocaIP,
- InsertPointTy CodeGenIP) {
+ auto BodyGenCB = [CriticalRegionBodyStmt,
+ this](InsertPointTy AllocIP, InsertPointTy CodeGenIP,
+ ArrayRef<llvm::BasicBlock *> DeallocBlocks) {
OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
- *this, CriticalRegionBodyStmt, AllocaIP, CodeGenIP, "critical");
+ *this, CriticalRegionBodyStmt, AllocIP, CodeGenIP, "critical");
return llvm::Error::success();
};
@@ -5885,8 +5892,8 @@ void CodeGenFunction::EmitOMPTaskgroupDirective(
InsertPointTy AllocaIP(AllocaInsertPt->getParent(),
AllocaInsertPt->getIterator());
- auto BodyGenCB = [&, this](InsertPointTy AllocaIP,
- InsertPointTy CodeGenIP) {
+ auto BodyGenCB = [&, this](InsertPointTy AllocIP, InsertPointTy CodeGenIP,
+ ArrayRef<llvm::BasicBlock *> DeallocBlocks) {
Builder.restoreIP(CodeGenIP);
EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
return llvm::Error::success();
@@ -5895,7 +5902,8 @@ void CodeGenFunction::EmitOMPTaskgroupDirective(
if (!CapturedStmtInfo)
CapturedStmtInfo = &CapStmtInfo;
llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
- cantFail(OMPBuilder.createTaskgroup(Builder, AllocaIP, BodyGenCB));
+ cantFail(OMPBuilder.createTaskgroup(Builder, AllocaIP,
+ /*DeallocBlocks=*/{}, BodyGenCB));
Builder.restoreIP(AfterIP);
return;
}
@@ -6475,8 +6483,9 @@ void CodeGenFunction::EmitOMPOrderedDirective(const OMPOrderedDirective &S) {
return llvm::Error::success();
};
- auto BodyGenCB = [&S, C, this](InsertPointTy AllocaIP,
- InsertPointTy CodeGenIP) {
+ auto BodyGenCB = [&S, C,
+ this](InsertPointTy AllocIP, InsertPointTy CodeGenIP,
+ ArrayRef<llvm::BasicBlock *> DeallocBlocks) {
Builder.restoreIP(CodeGenIP);
const CapturedStmt *CS = S.getInnermostCapturedStmt();
@@ -6493,7 +6502,7 @@ void CodeGenFunction::EmitOMPOrderedDirective(const OMPOrderedDirective &S) {
OutlinedFn, CapturedVars);
} else {
OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
- *this, CS->getCapturedStmt(), AllocaIP, CodeGenIP, "ordered");
+ *this, CS->getCapturedStmt(), AllocIP, CodeGenIP, "ordered");
}
return llvm::Error::success();
};
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
index 15223915a74d2..e0093656758cd 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
@@ -641,15 +641,17 @@ class OpenMPIRBuilder {
///
/// AllocaIP and CodeGenIP must not point to the same position.
///
- /// \param AllocaIP is the insertion point at which new alloca instructions
- /// should be placed. The BasicBlock it is pointing to must
- /// not be split.
- /// \param CodeGenIP is the insertion point at which the body code should be
- /// placed.
- ///
+ /// \param AllocaIP is the insertion point at which new allocations should
+ /// be placed. The BasicBlock it is pointing to must not be
+ /// split.
+ /// \param CodeGenIP is the insertion point at which the body code should be
+ /// placed.
+ /// \param DeallocBlocks is the list of insertion blocks where explicit
+ /// deallocations, if needed, should be placed.
/// \return an error, if any were triggered during execution.
using BodyGenCallbackTy =
- function_ref<Error(InsertPointTy AllocaIP, InsertPointTy CodeGenIP)>;
+ function_ref<Error(InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks)>;
/// Callback type for task duplication function code generation. This is the
/// task duplication function passed to __kmpc_taskloop. It is expected that
@@ -690,7 +692,8 @@ class OpenMPIRBuilder {
///
/// \return an error, if any were triggered during execution.
using StorableBodyGenCallbackTy =
- std::function<Error(InsertPointTy AllocaIP, InsertPointTy CodeGenIP)>;
+ std::function<Error(InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks)>;
/// Callback type for loop body code generation.
///
@@ -784,7 +787,9 @@ class OpenMPIRBuilder {
/// Generator for '#omp parallel'
///
/// \param Loc The insert and source location description.
- /// \param AllocaIP The insertion points to be used for alloca instructions.
+ /// \param AllocaIP The insertion point to be used for allocations.
+ /// \param DeallocBlocks The insertion blocks to be used for explicit
+ /// deallocations, if needed.
/// \param BodyGenCB Callback that will generate the region code.
/// \param PrivCB Callback to copy a given variable (think copy constructor).
/// \param FiniCB Callback to finalize variable copies.
@@ -796,9 +801,9 @@ class OpenMPIRBuilder {
/// \returns The insertion position *after* the parallel.
LLVM_ABI InsertPointOrErrorTy createParallel(
const LocationDescription &Loc, InsertPointTy AllocaIP,
- BodyGenCallbackTy BodyGenCB, PrivatizeCallbackTy PrivCB,
- FinalizeCallbackTy FiniCB, Value *IfCondition, Value *NumThreads,
- omp::ProcBindKind ProcBind, bool IsCancellable);
+ ArrayRef<BasicBlock *> DeallocBlocks, BodyGenCallbackTy BodyGenCB,
+ PrivatizeCallbackTy PrivCB, FinalizeCallbackTy FiniCB, Value *IfCondition,
+ Value *NumThreads, omp::ProcBindKind ProcBind, bool IsCancellable);
/// Generator for the control flow structure of an OpenMP canonical loop.
///
@@ -1548,6 +1553,8 @@ class OpenMPIRBuilder {
///
/// \param Loc The location where the taskloop construct was encountered.
/// \param AllocaIP The insertion point to be used for alloca instructions.
+ /// \param DeallocBlocks The list of insertion blocks where explicit
+ /// deallocations, if needed, should be placed.
/// \param BodyGenCB Callback that will generate the region code.
/// \param LoopInfo Callback that return the CLI
/// \param LBVal Lowerbound value of loop
@@ -1579,7 +1586,7 @@ class OpenMPIRBuilder {
/// bound, step} values in the task data.
LLVM_ABI InsertPointOrErrorTy createTaskloop(
const LocationDescription &Loc, InsertPointTy AllocaIP,
- BodyGenCallbackTy BodyGenCB,
+ ArrayRef<BasicBlock *> DeallocBlocks, BodyGenCallbackTy BodyGenCB,
llvm::function_ref<llvm::Expected<llvm::CanonicalLoopInfo *>()> LoopInfo,
Value *LBVal, Value *UBVal, Value *StepVal, bool Untied = false,
Value *IfCond = nullptr, Value *GrainSize = nullptr, bool NoGroup = false,
@@ -1591,7 +1598,9 @@ class OpenMPIRBuilder {
/// Generator for `#omp task`
///
/// \param Loc The location where the task construct was encountered.
- /// \param AllocaIP The insertion point to be used for alloca instructions.
+ /// \param AllocaIP The insertion point to be used for allocations.
+ /// \param DeallocBlocks The insertion blocks to be used for explicit
+ /// deallocations, if needed.
/// \param BodyGenCB Callback that will generate the region code.
/// \param Tied True if the task is tied, false if the task is untied.
/// \param Final i1 value which is `true` if the task is final, `false` if the
@@ -1613,19 +1622,22 @@ class OpenMPIRBuilder {
/// tasks that is generated by the construct
LLVM_ABI InsertPointOrErrorTy createTask(
const LocationDescription &Loc, InsertPointTy AllocaIP,
- BodyGenCallbackTy BodyGenCB, bool Tied = true, Value *Final = nullptr,
- Value *IfCondition = nullptr, const DependenciesInfo &Dependencies = {},
+ ArrayRef<BasicBlock *> DeallocBlocks, BodyGenCallbackTy BodyGenCB,
+ bool Tied = true, Value *Final = nullptr, Value *IfCondition = nullptr,
+ const DependenciesInfo &Dependencies = {},
const AffinityData &Affinities = {}, bool Mergeable = false,
Value *EventHandle = nullptr, Value *Priority = nullptr);
/// Generator for the taskgroup construct
///
/// \param Loc The location where the taskgroup construct was encountered.
- /// \param AllocaIP The insertion point to be used for alloca instructions.
+ /// \param AllocaIP The insertion point to be used for allocations.
+ /// \param DeallocBlocks The insertion blocks to be used for explicit
+ /// deallocation instructions, if needed.
/// \param BodyGenCB Callback that will generate the region code.
- LLVM_ABI InsertPointOrErrorTy createTaskgroup(const LocationDescription &Loc,
- InsertPointTy AllocaIP,
- BodyGenCallbackTy BodyGenCB);
+ LLVM_ABI InsertPointOrErrorTy createTaskgroup(
+ const LocationDescription &Loc, InsertPointTy AllocaIP,
+ ArrayRef<BasicBlock *> DeallocBlocks, BodyGenCallbackTy BodyGenCB);
using FileIdentifierInfoCallbackTy =
std::function<std::tuple<std::string, uint64_t>()>;
@@ -2551,7 +2563,8 @@ class OpenMPIRBuilder {
struct OutlineInfo {
using PostOutlineCBTy = std::function<void(Function &)>;
PostOutlineCBTy PostOutlineCB;
- BasicBlock *EntryBB, *ExitBB, *OuterAllocaBB;
+ BasicBlock *EntryBB, *ExitBB, *OuterAllocBB;
+ SmallVector<BasicBlock *> OuterDeallocBBs;
SmallVector<Value *, 2> ExcludeArgsFromAggregate;
SetVector<Value *> Inputs, Outputs;
// TODO: this should be safe to enable by default
@@ -2627,7 +2640,8 @@ class OpenMPIRBuilder {
/// \return an error, if any were triggered during execution.
LLVM_ABI Error emitIfClause(Value *Cond, BodyGenCallbackTy ThenGen,
BodyGenCallbackTy ElseGen,
- InsertPointTy AllocaIP = {});
+ InsertPointTy AllocaIP = {},
+ ArrayRef<BasicBlock *> DeallocBlocks = {});
/// Create the global variable holding the offload mappings information.
LLVM_ABI GlobalVariable *
@@ -3190,11 +3204,13 @@ class OpenMPIRBuilder {
/// Generator for `#omp distribute`
///
/// \param Loc The location where the distribute construct was encountered.
- /// \param AllocaIP The insertion points to be used for alloca instructions.
+ /// \param AllocaIP The insertion point to be used for allocations.
+ /// \param DeallocBlocks The insertion blocks to be used for explicit
+ /// deallocations, if needed.
/// \param BodyGenCB Callback that will generate the region code.
- LLVM_ABI InsertPointOrErrorTy createDistribute(const LocationDescription &Loc,
- InsertPointTy AllocaIP,
- BodyGenCallbackTy BodyGenCB);
+ LLVM_ABI InsertPointOrErrorTy createDistribute(
+ const LocationDescription &Loc, InsertPointTy AllocaIP,
+ ArrayRef<BasicBlock *> DeallocBlocks, BodyGenCallbackTy BodyGenCB);
/// Generate conditional branch and relevant BasicBlocks through which private
/// threads copy the 'copyin' variables from Master copy to threadprivate
@@ -3536,9 +3552,11 @@ class OpenMPIRBuilder {
/// Generator for '#omp target data'
///
/// \param Loc The location where the target data construct was encountered.
- /// \param AllocaIP The insertion points to be used for alloca instructions.
+ /// \param AllocaIP The insertion points to be used for allocations.
/// \param CodeGenIP The insertion point at which the target directive code
/// should be placed.
+ /// \param DeallocBlocks The insertion blocks at which explicit deallocations
+ /// should be placed, if needed.
/// \param IsBegin If true then emits begin mapper call otherwise emits
/// end mapper call.
/// \param DeviceID Stores the DeviceID from the device clause.
@@ -3552,9 +3570,9 @@ class OpenMPIRBuilder {
/// use_device_ptr and use_device_addr.
LLVM_ABI InsertPointOrErrorTy createTargetData(
const LocationDescription &Loc, InsertPointTy AllocaIP,
- InsertPointTy CodeGenIP, Value *DeviceID, Value *IfCond,
- TargetDataInfo &Info, GenMapInfoCallbackTy GenMapInfoCB,
- CustomMapperCallbackTy CustomMapperCB,
+ InsertPointTy CodeGenIP, ArrayRef<BasicBlock *> DeallocBlocks,
+ Value *DeviceID, Value *IfCond, TargetDataInfo &Info,
+ GenMapInfoCallbackTy GenMapInfoCB, CustomMapperCallbackTy CustomMapperCB,
omp::RuntimeFunction *MapperFunc = nullptr,
function_ref<InsertPointOrErrorTy(InsertPointTy CodeGenIP,
BodyGenTy BodyGenType)>
@@ -3563,7 +3581,8 @@ class OpenMPIRBuilder {
Value *SrcLocInfo = nullptr);
using TargetBodyGenCallbackTy = function_ref<InsertPointOrErrorTy(
- InsertPointTy AllocaIP, InsertPointTy CodeGenIP)>;
+ InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks)>;
using TargetGenArgAccessorsCallbackTy = function_ref<InsertPointOrErrorTy(
Argument &Arg, Value *Input, Value *&RetVal, InsertPointTy AllocaIP,
@@ -3575,6 +3594,8 @@ class OpenMPIRBuilder {
/// \param IsOffloadEntry whether it is an offload entry.
/// \param CodeGenIP The insertion point where the call to the outlined
/// function should be emitted.
+ /// \param DeallocBlocks The insertion points at which explicit deallocations
+ /// should be placed, if needed.
/// \param Info Stores all information realted to the Target directive.
/// \param EntryInfo The entry information about the function.
/// \param DefaultAttrs Structure containing the default attributes, including
@@ -3600,7 +3621,8 @@ class OpenMPIRBuilder {
LLVM_ABI InsertPointOrErrorTy createTarget(
const LocationDescription &Loc, bool IsOffloadEntry,
OpenMPIRBuilder::InsertPointTy AllocaIP,
- OpenMPIRBuilder::InsertPointTy CodeGenIP, TargetDataInfo &Info,
+ OpenMPIRBuilder::InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks, TargetDataInfo &Info,
TargetRegionEntryInfo &EntryInfo,
const TargetKernelDefaultAttrs &DefaultAttrs,
const TargetKernelRuntimeAttrs &RuntimeAttrs, Value *IfCond,
diff --git a/llvm/include/llvm/Transforms/Utils/CodeExtractor.h b/llvm/include/llvm/Transforms/Utils/CodeExtractor.h
index fdc9f9c82b0ad..05f8287aebf6f 100644
--- a/llvm/include/llvm/Transforms/Utils/CodeExtractor.h
+++ b/llvm/include/llvm/Transforms/Utils/CodeExtractor.h
@@ -18,6 +18,7 @@
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/IR/BasicBlock.h"
+#include "llvm/IR/IRBuilder.h"
#include "llvm/Support/Compiler.h"
#include <limits>
@@ -100,13 +101,13 @@ class LLVM_ABI CodeExtractor {
/// entry block of the function.
BasicBlock *AllocationBlock;
- /// A block outside of the extraction set where deallocations for intermediate
- /// allocations can be placed inside. Not used for automatically deallocated
- /// memory (e.g. `alloca`), which is the default.
+ /// A set of blocks outside of the extraction set where deallocations for
+ /// intermediate allocations should be placed. Not used for automatically
+ /// deallocated memory (e.g. `alloca`), which is the default.
///
- /// If it is null and needed, the end of the replacement basic block will be
+ /// If it is empty and needed, the end of the replacement basic block will be
/// used to place deallocations.
- BasicBlock *DeallocationBlock;
+ SmallVector<BasicBlock *> DeallocationBlocks;
/// If true, varargs functions can be extracted.
bool AllowVarArgs;
@@ -162,9 +163,9 @@ class LLVM_ABI CodeExtractor {
/// allocations will be placed in the AllocationBlock, unless it is null, in
/// which case it will be placed in the entry block of the function from which
/// the code is being extracted. Explicit deallocations for the aforementioned
- /// allocations will be placed in the DeallocationBlock or the end of the
- /// replacement block, if needed. If ArgsInZeroAddressSpace param is set to
- /// true, then the aggregate param pointer of the outlined function is
+ /// allocations will be placed, if needed, in all blocks in DeallocationBlocks
+ /// or the end of the replacement block. If ArgsInZeroAddressSpace param is
+ /// set to true, then the aggregate param pointer of the outlined function is
/// declared in zero address space. If VoidReturnWithSingleOutput is set to
/// true, then the return type of the outlined function is set void even if
/// there is only one output.
@@ -173,7 +174,7 @@ class LLVM_ABI CodeExtractor {
BranchProbabilityInfo *BPI = nullptr,
AssumptionCache *AC = nullptr, bool AllowVarArgs = false,
bool AllowAlloca = false, BasicBlock *AllocationBlock = nullptr,
- BasicBlock *DeallocationBlock = nullptr,
+ ArrayRef<BasicBlock *> DeallocationBlocks = {},
std::string Suffix = "", bool ArgsInZeroAddressSpace = false,
bool VoidReturnWithSingleOutput = true);
@@ -260,15 +261,14 @@ class LLVM_ABI CodeExtractor {
protected:
/// Allocate an intermediate variable at the specified point.
- virtual Instruction *allocateVar(BasicBlock *BB, BasicBlock::iterator AllocIP,
+ virtual Instruction *allocateVar(IRBuilder<>::InsertPoint AllocaIP,
Type *VarType, const Twine &Name = Twine(""),
AddrSpaceCastInst **CastedAlloc = nullptr);
/// Deallocate a previously-allocated intermediate variable at the specified
/// point.
- virtual Instruction *deallocateVar(BasicBlock *BB,
- BasicBlock::iterator DeallocIP, Value *Var,
- Type *VarType);
+ virtual Instruction *deallocateVar(IRBuilder<>::InsertPoint DeallocIP,
+ Value *Var, Type *VarType);
private:
struct LifetimeMarkerInfo {
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 5dbb8aba2403e..767fd91a27d4f 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -326,6 +326,12 @@ getTargetKernelExecMode(Function &Kernel) {
return static_cast<OMPTgtExecModeFlags>(KernelMode->getZExtValue());
}
+static bool isGenericKernel(Function &Fn) {
+ std::optional<omp::OMPTgtExecModeFlags> ExecMode =
+ getTargetKernelExecMode(Fn);
+ return !ExecMode || (*ExecMode & OMP_TGT_EXEC_MODE_GENERIC);
+}
+
/// Make \p Source branch to \p Target.
///
/// Handles two situations:
@@ -500,10 +506,10 @@ class OMPCodeExtractor : public CodeExtractor {
AssumptionCache *AC = nullptr, bool AllowVarArgs = false,
bool AllowAlloca = false,
BasicBlock *AllocationBlock = nullptr,
- BasicBlock *DeallocationBlock = nullptr,
+ ArrayRef<BasicBlock *> DeallocationBlocks = {},
std::string Suffix = "", bool ArgsInZeroAddressSpace = false)
: CodeExtractor(BBs, DT, AggregateArgs, BFI, BPI, AC, AllowVarArgs,
- AllowAlloca, AllocationBlock, DeallocationBlock, Suffix,
+ AllowAlloca, AllocationBlock, DeallocationBlocks, Suffix,
ArgsInZeroAddressSpace),
OMPBuilder(OMPBuilder) {}
@@ -515,53 +521,27 @@ class OMPCodeExtractor : public CodeExtractor {
class DeviceSharedMemCodeExtractor : public OMPCodeExtractor {
public:
- DeviceSharedMemCodeExtractor(
- OpenMPIRBuilder &OMPBuilder, BasicBlock *AllocBlockOverride,
- ArrayRef<BasicBlock *> BBs, DominatorTree *DT = nullptr,
- bool AggregateArgs = false, BlockFrequencyInfo *BFI = nullptr,
- BranchProbabilityInfo *BPI = nullptr, AssumptionCache *AC = nullptr,
- bool AllowVarArgs = false, bool AllowAlloca = false,
- BasicBlock *AllocationBlock = nullptr,
- BasicBlock *DeallocationBlock = nullptr, std::string Suffix = "",
- bool ArgsInZeroAddressSpace = false)
- : OMPCodeExtractor(OMPBuilder, BBs, DT, AggregateArgs, BFI, BPI, AC,
- AllowVarArgs, AllowAlloca, AllocationBlock,
- DeallocationBlock, Suffix, ArgsInZeroAddressSpace),
- AllocBlockOverride(AllocBlockOverride) {}
+ using OMPCodeExtractor::OMPCodeExtractor;
virtual ~DeviceSharedMemCodeExtractor() = default;
protected:
virtual Instruction *
- allocateVar(BasicBlock *, BasicBlock::iterator, Type *VarType,
+ allocateVar(IRBuilder<>::InsertPoint AllocaIP, Type *VarType,
const Twine &Name = Twine(""),
AddrSpaceCastInst **CastedAlloc = nullptr) override {
- // Ignore the CastedAlloc pointer, if requested, because shared memory
- // should not be casted to address space 0 to be passed around.
- return OMPBuilder.createOMPAllocShared(
- OpenMPIRBuilder::InsertPointTy(
- AllocBlockOverride, AllocBlockOverride->getFirstInsertionPt()),
- VarType, Name);
- }
-
- virtual Instruction *deallocateVar(BasicBlock *BB,
- BasicBlock::iterator DeallocIP, Value *Var,
- Type *VarType) override {
- return OMPBuilder.createOMPFreeShared(
- OpenMPIRBuilder::InsertPointTy(BB, DeallocIP), Var, VarType);
- }
-
-private:
- // TODO: Remove the need for this override and instead get the CodeExtractor
- // to provide a valid insert point for explicit deallocations by correctly
- // populating its DeallocationBlock.
- BasicBlock *AllocBlockOverride;
+ return OMPBuilder.createOMPAllocShared(AllocaIP, VarType, Name);
+ }
+
+ virtual Instruction *deallocateVar(IRBuilder<>::InsertPoint DeallocIP,
+ Value *Var, Type *VarType) override {
+ return OMPBuilder.createOMPFreeShared(DeallocIP, Var, VarType);
+ }
};
/// Helper storing information about regions to outline using device shared
/// memory for intermediate allocations.
struct DeviceSharedMemOutlineInfo : public OpenMPIRBuilder::OutlineInfo {
OpenMPIRBuilder &OMPBuilder;
- BasicBlock *AllocBlockOverride = nullptr;
DeviceSharedMemOutlineInfo(OpenMPIRBuilder &OMPBuilder)
: OMPBuilder(OMPBuilder) {}
@@ -1644,10 +1624,8 @@ static void targetParallelCallback(
: Builder.getInt32(1);
// If this is not a Generic kernel, we can skip generating the wrapper.
- std::optional<omp::OMPTgtExecModeFlags> ExecMode =
- getTargetKernelExecMode(*OuterFn);
Value *WrapperFn;
- if (ExecMode && (*ExecMode & OMP_TGT_EXEC_MODE_GENERIC))
+ if (isGenericKernel(*OuterFn))
WrapperFn = createTargetParallelWrapper(OMPIRBuilder, OutlinedFn);
else
WrapperFn = Constant::getNullValue(PtrTy);
@@ -1773,11 +1751,11 @@ hostParallelCallback(OpenMPIRBuilder *OMPIRBuilder, Function &OutlinedFn,
}
OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createParallel(
- const LocationDescription &Loc, InsertPointTy OuterAllocaIP,
- BodyGenCallbackTy BodyGenCB, PrivatizeCallbackTy PrivCB,
- FinalizeCallbackTy FiniCB, Value *IfCondition, Value *NumThreads,
- omp::ProcBindKind ProcBind, bool IsCancellable) {
- assert(!isConflictIP(Loc.IP, OuterAllocaIP) && "IPs must not be ambiguous");
+ const LocationDescription &Loc, InsertPointTy OuterAllocIP,
+ ArrayRef<BasicBlock *> OuterDeallocBlocks, BodyGenCallbackTy BodyGenCB,
+ PrivatizeCallbackTy PrivCB, FinalizeCallbackTy FiniCB, Value *IfCondition,
+ Value *NumThreads, omp::ProcBindKind ProcBind, bool IsCancellable) {
+ assert(!isConflictIP(Loc.IP, OuterAllocIP) && "IPs must not be ambiguous");
if (!updateToLocation(Loc))
return Loc.IP;
@@ -1819,7 +1797,7 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createParallel(
// Save the outer alloca block because the insertion iterator may get
// invalidated and we still need this later.
- BasicBlock *OuterAllocaBlock = OuterAllocaIP.getBlock();
+ BasicBlock *OuterAllocaBlock = OuterAllocIP.getBlock();
// Vector to remember instructions we used only during the modeling but which
// we want to delete at the end.
@@ -1915,51 +1893,19 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createParallel(
// Let the caller create the body.
assert(BodyGenCB && "Expected body generation callback!");
InsertPointTy CodeGenIP(PRegBodyBB, PRegBodyBB->begin());
- if (Error Err = BodyGenCB(InnerAllocaIP, CodeGenIP))
+ if (Error Err = BodyGenCB(InnerAllocaIP, CodeGenIP, PRegExitBB))
return Err;
LLVM_DEBUG(dbgs() << "After body codegen: " << *OuterFn << "\n");
- auto OI = [&]() -> std::unique_ptr<OutlineInfo> {
- if (Config.isTargetDevice()) {
- std::optional<omp::OMPTgtExecModeFlags> ExecMode =
- getTargetKernelExecMode(*OuterFn);
-
- // If OuterFn is not a Generic kernel, skip custom allocation. This causes
- // the CodeExtractor to follow its default behavior. Otherwise, we need to
- // use device shared memory to allocate argument structures.
- if (ExecMode && *ExecMode & OMP_TGT_EXEC_MODE_GENERIC) {
- auto Info = std::make_unique<DeviceSharedMemOutlineInfo>(*this);
-
- // Instead of using the insertion point provided by the CodeExtractor,
- // here we need to use the block that eventually calls the outlined
- // function for the `parallel` construct.
- //
- // The reason is that the explicit deallocation call will be inserted
- // within the outlined function, whereas the alloca insertion point
- // might actually be located somewhere else in the caller. This becomes
- // a problem when e.g. `parallel` is inside of a `distribute` construct,
- // because the deallocation would be executed multiple times and the
- // allocation just once (outside of the loop).
- //
- // TODO: Ideally, we'd want to do the allocation and deallocation
- // outside of the `parallel` outlined function, hence using here the
- // insertion point provided by the CodeExtractor. We can't do this at
- // the moment because there is currently no way of passing an eligible
- // insertion point for the explicit deallocation to the CodeExtractor,
- // as that block is created (at least when nested inside of
- // `distribute`) sometime after createParallel() completed, so it can't
- // be stored in the OutlineInfo structure here.
- //
- // The current approach results in an explicit allocation and
- // deallocation pair for each `distribute` loop iteration in that case,
- // which is suboptimal.
- Info->AllocBlockOverride = EntryBB;
- return Info;
- }
- }
- return std::make_unique<OutlineInfo>();
- }();
+ // If OuterFn is a Generic kernel, we need to use device shared memory to
+ // allocate argument structures. Otherwise, we use stack allocations as usual.
+ bool UsesDeviceSharedMemory =
+ Config.isTargetDevice() && isGenericKernel(*OuterFn);
+ std::unique_ptr<OutlineInfo> OI =
+ UsesDeviceSharedMemory
+ ? std::make_unique<DeviceSharedMemOutlineInfo>(*this)
+ : std::make_unique<OutlineInfo>();
if (Config.isTargetDevice()) {
// Generate OpenMP target specific runtime call
@@ -1979,9 +1925,11 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createParallel(
}
OI->FixUpNonEntryAllocas = true;
- OI->OuterAllocaBB = OuterAllocaBlock;
+ OI->OuterAllocBB = OuterAllocaBlock;
OI->EntryBB = PRegEntryBB;
OI->ExitBB = PRegExitBB;
+ OI->OuterDeallocBBs.reserve(OuterDeallocBlocks.size());
+ copy(OuterDeallocBlocks, OI->OuterDeallocBBs.end());
SmallPtrSet<BasicBlock *, 32> ParallelRegionBlockSet;
SmallVector<BasicBlock *, 32> Blocks;
@@ -1996,7 +1944,7 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createParallel(
/* AllowVarArgs */ true,
/* AllowAlloca */ true,
/* AllocationBlock */ OuterAllocaBlock,
- /* DeallocationBlock */ nullptr,
+ /* DeallocationBlocks */ {},
/* Suffix */ ".omp_par", ArgsInZeroAddressSpace);
// Find inputs to, outputs from the code region.
@@ -2042,9 +1990,20 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createParallel(
IRBuilder<>::InsertPointGuard Guard(Builder);
LLVM_DEBUG(llvm::dbgs() << "Forwarding input as pointer: " << V << "\n");
- Builder.restoreIP(OuterAllocaIP);
- Value *Ptr =
- Builder.CreateAlloca(V.getType(), nullptr, V.getName() + ".reloaded");
+ Builder.restoreIP(OuterAllocIP);
+ Value *Ptr;
+ if (UsesDeviceSharedMemory) {
+ // Use device shared memory instead, if needed.
+ Ptr = createOMPAllocShared(OuterAllocIP, V.getType(),
+ V.getName() + ".reloaded");
+ for (BasicBlock *DeallocBlock : OuterDeallocBlocks)
+ createOMPFreeShared(
+ InsertPointTy(DeallocBlock, DeallocBlock->getFirstInsertionPt()),
+ Ptr, V.getType());
+ } else {
+ Ptr = Builder.CreateAlloca(V.getType(), nullptr,
+ V.getName() + ".reloaded");
+ }
// Store to stack at end of the block that currently branches to the entry
// block of the to-be-outlined region.
@@ -2094,7 +2053,7 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createParallel(
// Reset the outer alloca insertion point to the entry of the relevant block
// in case it was invalidated.
- OuterAllocaIP = IRBuilder<>::InsertPoint(
+ OuterAllocIP = IRBuilder<>::InsertPoint(
OuterAllocaBlock, OuterAllocaBlock->getFirstInsertionPt());
for (Value *Input : Inputs) {
@@ -2340,7 +2299,7 @@ Expected<Value *> OpenMPIRBuilder::createTaskDuplicationFunction(
OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTaskloop(
const LocationDescription &Loc, InsertPointTy AllocaIP,
- BodyGenCallbackTy BodyGenCB,
+ ArrayRef<BasicBlock *> DeallocBlocks, BodyGenCallbackTy BodyGenCB,
llvm::function_ref<llvm::Expected<llvm::CanonicalLoopInfo *>()> LoopInfo,
Value *LBVal, Value *UBVal, Value *StepVal, bool Untied, Value *IfCond,
Value *GrainSize, bool NoGroup, int Sched, Value *Final, bool Mergeable,
@@ -2366,7 +2325,7 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTaskloop(
InsertPointTy TaskloopBodyIP =
InsertPointTy(TaskloopBodyBB, TaskloopBodyBB->begin());
- if (Error Err = BodyGenCB(TaskloopAllocaIP, TaskloopBodyIP))
+ if (Error Err = BodyGenCB(TaskloopAllocaIP, TaskloopBodyIP, TaskloopExitBB))
return Err;
llvm::Expected<llvm::CanonicalLoopInfo *> result = LoopInfo();
@@ -2377,8 +2336,10 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTaskloop(
llvm::CanonicalLoopInfo *CLI = result.get();
auto OI = std::make_unique<OutlineInfo>();
OI->EntryBB = TaskloopAllocaBB;
- OI->OuterAllocaBB = AllocaIP.getBlock();
+ OI->OuterAllocBB = AllocaIP.getBlock();
OI->ExitBB = TaskloopExitBB;
+ OI->OuterDeallocBBs.reserve(DeallocBlocks.size());
+ copy(DeallocBlocks, OI->OuterDeallocBBs.end());
// Add the thread ID argument.
SmallVector<Instruction *> ToBeDeleted;
@@ -2688,7 +2649,8 @@ llvm::StructType *OpenMPIRBuilder::getKmpTaskAffinityInfoTy() {
OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTask(
const LocationDescription &Loc, InsertPointTy AllocaIP,
- BodyGenCallbackTy BodyGenCB, bool Tied, Value *Final, Value *IfCondition,
+ ArrayRef<BasicBlock *> DeallocBlocks, BodyGenCallbackTy BodyGenCB,
+ bool Tied, Value *Final, Value *IfCondition,
const DependenciesInfo &Dependencies, const AffinityData &Affinities,
bool Mergeable, Value *EventHandle, Value *Priority) {
@@ -2722,13 +2684,15 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTask(
InsertPointTy TaskAllocaIP =
InsertPointTy(TaskAllocaBB, TaskAllocaBB->begin());
InsertPointTy TaskBodyIP = InsertPointTy(TaskBodyBB, TaskBodyBB->begin());
- if (Error Err = BodyGenCB(TaskAllocaIP, TaskBodyIP))
+ if (Error Err = BodyGenCB(TaskAllocaIP, TaskBodyIP, TaskExitBB))
return Err;
auto OI = std::make_unique<OutlineInfo>();
OI->EntryBB = TaskAllocaBB;
- OI->OuterAllocaBB = AllocaIP.getBlock();
+ OI->OuterAllocBB = AllocaIP.getBlock();
OI->ExitBB = TaskExitBB;
+ OI->OuterDeallocBBs.reserve(DeallocBlocks.size());
+ copy(DeallocBlocks, OI->OuterDeallocBBs.end());
// Add the thread ID argument.
SmallVector<Instruction *, 4> ToBeDeleted;
@@ -2966,10 +2930,9 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTask(
return Builder.saveIP();
}
-OpenMPIRBuilder::InsertPointOrErrorTy
-OpenMPIRBuilder::createTaskgroup(const LocationDescription &Loc,
- InsertPointTy AllocaIP,
- BodyGenCallbackTy BodyGenCB) {
+OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTaskgroup(
+ const LocationDescription &Loc, InsertPointTy AllocaIP,
+ ArrayRef<BasicBlock *> DeallocBlocks, BodyGenCallbackTy BodyGenCB) {
if (!updateToLocation(Loc))
return InsertPointTy();
@@ -2984,7 +2947,7 @@ OpenMPIRBuilder::createTaskgroup(const LocationDescription &Loc,
createRuntimeFunctionCall(TaskgroupFn, {Ident, ThreadID});
BasicBlock *TaskgroupExitBB = splitBB(Builder, true, "taskgroup.exit");
- if (Error Err = BodyGenCB(AllocaIP, Builder.saveIP()))
+ if (Error Err = BodyGenCB(AllocaIP, Builder.saveIP(), DeallocBlocks))
return Err;
Builder.SetInsertPoint(TaskgroupExitBB);
@@ -3037,8 +3000,9 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createSections(
SwitchStmt->addCase(Builder.getInt32(CaseNumber), CaseBB);
Builder.SetInsertPoint(CaseBB);
UncondBrInst *CaseEndBr = Builder.CreateBr(Continue);
- if (Error Err = SectionCB(InsertPointTy(), {CaseEndBr->getParent(),
- CaseEndBr->getIterator()}))
+ if (Error Err =
+ SectionCB(InsertPointTy(),
+ {CaseEndBr->getParent(), CaseEndBr->getIterator()}, {}))
return Err;
CaseNumber++;
}
@@ -5175,8 +5139,8 @@ Error OpenMPIRBuilder::emitScanBasedDirectiveDeclsIR(
}
// Allocate temporary buffer by master thread
- auto BodyGenCB = [&](InsertPointTy AllocaIP,
- InsertPointTy CodeGenIP) -> Error {
+ auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
Builder.restoreIP(CodeGenIP);
Value *AllocSpan =
Builder.CreateAdd(ScanRedInfo->Span, Builder.getInt32(1));
@@ -5215,8 +5179,8 @@ Error OpenMPIRBuilder::emitScanBasedDirectiveDeclsIR(
Error OpenMPIRBuilder::emitScanBasedDirectiveFinalsIR(
ArrayRef<ReductionInfo> ReductionInfos, ScanInfo *ScanRedInfo) {
- auto BodyGenCB = [&](InsertPointTy AllocaIP,
- InsertPointTy CodeGenIP) -> Error {
+ auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
Builder.restoreIP(CodeGenIP);
for (ReductionInfo RedInfo : ReductionInfos) {
Value *PrivateVar = RedInfo.PrivateVariable;
@@ -5267,8 +5231,8 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
if (!updateToLocation(Loc))
return Loc.IP;
- auto BodyGenCB = [&](InsertPointTy AllocaIP,
- InsertPointTy CodeGenIP) -> Error {
+ auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
Builder.restoreIP(CodeGenIP);
Function *CurFn = Builder.GetInsertBlock()->getParent();
// for (int k = 0; k <= ceil(log2(n)); ++k)
@@ -6231,13 +6195,13 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::applyWorkshareLoopTarget(
Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize, Flag);
auto OI = std::make_unique<OutlineInfo>();
- OI->OuterAllocaBB = CLI->getPreheader();
+ OI->OuterAllocBB = CLI->getPreheader();
Function *OuterFn = CLI->getPreheader()->getParent();
// Instructions which need to be deleted at the end of code generation
SmallVector<Instruction *, 4> ToBeDeleted;
- OI->OuterAllocaBB = AllocaIP.getBlock();
+ OI->OuterAllocBB = AllocaIP.getBlock();
// Mark the body loop as region which needs to be extracted
OI->EntryBB = CLI->getBody();
@@ -6274,7 +6238,7 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::applyWorkshareLoopTarget(
/* AllowVarArgs */ true,
/* AllowAlloca */ true,
/* AllocationBlock */ CLI->getPreheader(),
- /* DeallocationBlock */ nullptr,
+ /* DeallocationBlocks */ {},
/* Suffix */ ".omp_wsloop",
/* AggrArgsIn0AddrSpace */ true);
@@ -7770,8 +7734,9 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::EmitOMPInlinedRegion(
emitCommonDirectiveEntry(OMPD, EntryCall, ExitBB, Conditional);
// generate body
- if (Error Err = BodyGenCB(/* AllocaIP */ InsertPointTy(),
- /* CodeGenIP */ Builder.saveIP()))
+ if (Error Err =
+ BodyGenCB(/* AllocaIP */ InsertPointTy(),
+ /* CodeGenIP */ Builder.saveIP(), /* DeallocBlocks */ {}))
return Err;
// emit exit call and do any needed finalization.
@@ -8437,9 +8402,10 @@ Constant *OpenMPIRBuilder::registerTargetRegionFunction(
OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTargetData(
const LocationDescription &Loc, InsertPointTy AllocaIP,
- InsertPointTy CodeGenIP, Value *DeviceID, Value *IfCond,
- TargetDataInfo &Info, GenMapInfoCallbackTy GenMapInfoCB,
- CustomMapperCallbackTy CustomMapperCB, omp::RuntimeFunction *MapperFunc,
+ InsertPointTy CodeGenIP, ArrayRef<BasicBlock *> DeallocBlocks,
+ Value *DeviceID, Value *IfCond, TargetDataInfo &Info,
+ GenMapInfoCallbackTy GenMapInfoCB, CustomMapperCallbackTy CustomMapperCB,
+ omp::RuntimeFunction *MapperFunc,
function_ref<InsertPointOrErrorTy(InsertPointTy CodeGenIP,
BodyGenTy BodyGenType)>
BodyGenCB,
@@ -8454,8 +8420,8 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTargetData(
// Generate the code for the opening of the data environment. Capture all the
// arguments of the runtime call by reference because they are used in the
// closing of the region.
- auto BeginThenGen = [&](InsertPointTy AllocaIP,
- InsertPointTy CodeGenIP) -> Error {
+ auto BeginThenGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
MapInfo = &GenMapInfoCB(Builder.saveIP());
if (Error Err = emitOffloadingArrays(
AllocaIP, Builder.saveIP(), *MapInfo, Info, CustomMapperCB,
@@ -8543,8 +8509,8 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTargetData(
// If we need device pointer privatization, we need to emit the body of the
// region with no privatization in the 'else' branch of the conditional.
// Otherwise, we don't have to do anything.
- auto BeginElseGen = [&](InsertPointTy AllocaIP,
- InsertPointTy CodeGenIP) -> Error {
+ auto BeginElseGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
InsertPointOrErrorTy AfterIP =
BodyGenCB(Builder.saveIP(), BodyGenTy::DupNoPriv);
if (!AfterIP)
@@ -8554,7 +8520,8 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTargetData(
};
// Generate code for the closing of the data region.
- auto EndThenGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+ auto EndThenGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) {
TargetDataRTArgs RTArgs;
Info.EmitDebug = !MapInfo->Names.empty();
emitOffloadingArraysArgument(Builder, RTArgs, Info, /*ForEndCall=*/true);
@@ -8583,7 +8550,8 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTargetData(
// We don't have to do anything to close the region if the if clause evaluates
// to false.
- auto EndElseGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+ auto EndElseGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) {
return Error::success();
};
@@ -8592,7 +8560,7 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTargetData(
Error Err = [&]() {
if (IfCond)
return emitIfClause(IfCond, BeginThenGen, BeginElseGen, AllocaIP);
- return BeginThenGen(AllocaIP, Builder.saveIP());
+ return BeginThenGen(AllocaIP, Builder.saveIP(), DeallocBlocks);
}();
if (Err)
@@ -8608,11 +8576,11 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTargetData(
if (IfCond)
return emitIfClause(IfCond, EndThenGen, EndElseGen, AllocaIP);
- return EndThenGen(AllocaIP, Builder.saveIP());
+ return EndThenGen(AllocaIP, Builder.saveIP(), DeallocBlocks);
}
if (IfCond)
return emitIfClause(IfCond, BeginThenGen, EndElseGen, AllocaIP);
- return BeginThenGen(AllocaIP, Builder.saveIP());
+ return BeginThenGen(AllocaIP, Builder.saveIP(), DeallocBlocks);
}();
if (Err)
@@ -8853,15 +8821,18 @@ static Expected<Function *> createOutlinedFunction(
if (OMPBuilder.Config.isTargetDevice())
OMPBuilder.ConstantAllocaRaiseCandidates.emplace_back(Func);
- // Insert target deinit call in the device compilation pass.
+ BasicBlock *ExitBB = splitBB(Builder, /*CreateBranch=*/true, "target.exit");
BasicBlock *OutlinedBodyBB =
splitBB(Builder, /*CreateBranch=*/true, "outlined.body");
llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = CBFunc(
Builder.saveIP(),
- OpenMPIRBuilder::InsertPointTy(OutlinedBodyBB, OutlinedBodyBB->begin()));
+ OpenMPIRBuilder::InsertPointTy(OutlinedBodyBB, OutlinedBodyBB->begin()),
+ ExitBB);
if (!AfterIP)
return AfterIP.takeError();
- Builder.restoreIP(*AfterIP);
+ Builder.SetInsertPoint(ExitBB);
+
+ // Insert target deinit call in the device compilation pass.
if (OMPBuilder.Config.isTargetDevice())
OMPBuilder.createTargetDeinit(Builder);
@@ -9311,7 +9282,7 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitTargetTask(
auto OI = std::make_unique<OutlineInfo>();
OI->EntryBB = TargetTaskAllocaBB;
- OI->OuterAllocaBB = AllocaIP.getBlock();
+ OI->OuterAllocBB = AllocaIP.getBlock();
// Add the thread ID argument.
SmallVector<Instruction *, 4> ToBeDeleted;
@@ -9578,19 +9549,18 @@ Error OpenMPIRBuilder::emitOffloadingArraysAndArgs(
return Error::success();
}
-static void
-emitTargetCall(OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder,
- OpenMPIRBuilder::InsertPointTy AllocaIP,
- OpenMPIRBuilder::TargetDataInfo &Info,
- const OpenMPIRBuilder::TargetKernelDefaultAttrs &DefaultAttrs,
- const OpenMPIRBuilder::TargetKernelRuntimeAttrs &RuntimeAttrs,
- Value *IfCond, Function *OutlinedFn, Constant *OutlinedFnID,
- SmallVectorImpl<Value *> &Args,
- OpenMPIRBuilder::GenMapInfoCallbackTy GenMapInfoCB,
- OpenMPIRBuilder::CustomMapperCallbackTy CustomMapperCB,
- const OpenMPIRBuilder::DependenciesInfo &Dependencies,
- bool HasNoWait, Value *DynCGroupMem,
- OMPDynGroupprivateFallbackType DynCGroupMemFallback) {
+static void emitTargetCall(
+ OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder,
+ OpenMPIRBuilder::InsertPointTy AllocaIP,
+ ArrayRef<BasicBlock *> DeallocBlocks, OpenMPIRBuilder::TargetDataInfo &Info,
+ const OpenMPIRBuilder::TargetKernelDefaultAttrs &DefaultAttrs,
+ const OpenMPIRBuilder::TargetKernelRuntimeAttrs &RuntimeAttrs,
+ Value *IfCond, Function *OutlinedFn, Constant *OutlinedFnID,
+ SmallVectorImpl<Value *> &Args,
+ OpenMPIRBuilder::GenMapInfoCallbackTy GenMapInfoCB,
+ OpenMPIRBuilder::CustomMapperCallbackTy CustomMapperCB,
+ const OpenMPIRBuilder::DependenciesInfo &Dependencies, bool HasNoWait,
+ Value *DynCGroupMem, OMPDynGroupprivateFallbackType DynCGroupMemFallback) {
// Generate a function call to the host fallback implementation of the target
// region. This is called by the host when no offload entry was generated for
// the target region and when the offloading call fails at runtime.
@@ -9643,7 +9613,8 @@ emitTargetCall(OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder,
auto &&EmitTargetCallElse =
[&](OpenMPIRBuilder::InsertPointTy AllocaIP,
- OpenMPIRBuilder::InsertPointTy CodeGenIP) -> Error {
+ OpenMPIRBuilder::InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
// Assume no error was returned because EmitTargetCallFallbackCB doesn't
// produce any.
OpenMPIRBuilder::InsertPointTy AfterIP = cantFail([&]() {
@@ -9665,7 +9636,8 @@ emitTargetCall(OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder,
auto &&EmitTargetCallThen =
[&](OpenMPIRBuilder::InsertPointTy AllocaIP,
- OpenMPIRBuilder::InsertPointTy CodeGenIP) -> Error {
+ OpenMPIRBuilder::InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
Info.HasNoWait = HasNoWait;
OpenMPIRBuilder::MapInfosTy &MapInfo = GenMapInfoCB(Builder.saveIP());
@@ -9760,13 +9732,13 @@ emitTargetCall(OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder,
// wasn't created. In this case we just run the host fallback directly and
// ignore any potential 'if' clauses.
if (!OutlinedFnID) {
- cantFail(EmitTargetCallElse(AllocaIP, Builder.saveIP()));
+ cantFail(EmitTargetCallElse(AllocaIP, Builder.saveIP(), DeallocBlocks));
return;
}
// If there's no 'if' clause, only generate the kernel launch code path.
if (!IfCond) {
- cantFail(EmitTargetCallThen(AllocaIP, Builder.saveIP()));
+ cantFail(EmitTargetCallThen(AllocaIP, Builder.saveIP(), DeallocBlocks));
return;
}
@@ -9776,8 +9748,8 @@ emitTargetCall(OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder,
OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTarget(
const LocationDescription &Loc, bool IsOffloadEntry, InsertPointTy AllocaIP,
- InsertPointTy CodeGenIP, TargetDataInfo &Info,
- TargetRegionEntryInfo &EntryInfo,
+ InsertPointTy CodeGenIP, ArrayRef<BasicBlock *> DeallocBlocks,
+ TargetDataInfo &Info, TargetRegionEntryInfo &EntryInfo,
const TargetKernelDefaultAttrs &DefaultAttrs,
const TargetKernelRuntimeAttrs &RuntimeAttrs, Value *IfCond,
SmallVectorImpl<Value *> &Inputs, GenMapInfoCallbackTy GenMapInfoCB,
@@ -9806,10 +9778,10 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTarget(
// to make a remote call (offload) to the previously outlined function
// that represents the target region. Do that now.
if (!Config.isTargetDevice())
- emitTargetCall(*this, Builder, AllocaIP, Info, DefaultAttrs, RuntimeAttrs,
- IfCond, OutlinedFn, OutlinedFnID, Inputs, GenMapInfoCB,
- CustomMapperCB, Dependencies, HasNowait, DynCGroupMem,
- DynCGroupMemFallback);
+ emitTargetCall(*this, Builder, AllocaIP, DeallocBlocks, Info, DefaultAttrs,
+ RuntimeAttrs, IfCond, OutlinedFn, OutlinedFnID, Inputs,
+ GenMapInfoCB, CustomMapperCB, Dependencies, HasNowait,
+ DynCGroupMem, DynCGroupMemFallback);
return Builder.saveIP();
}
@@ -10596,15 +10568,16 @@ void OpenMPIRBuilder::emitBlock(BasicBlock *BB, Function *CurFn,
Error OpenMPIRBuilder::emitIfClause(Value *Cond, BodyGenCallbackTy ThenGen,
BodyGenCallbackTy ElseGen,
- InsertPointTy AllocaIP) {
+ InsertPointTy AllocaIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) {
// If the condition constant folds and can be elided, try to avoid emitting
// the condition and the dead arm of the if/else.
if (auto *CI = dyn_cast<ConstantInt>(Cond)) {
auto CondConstant = CI->getSExtValue();
if (CondConstant)
- return ThenGen(AllocaIP, Builder.saveIP());
+ return ThenGen(AllocaIP, Builder.saveIP(), DeallocBlocks);
- return ElseGen(AllocaIP, Builder.saveIP());
+ return ElseGen(AllocaIP, Builder.saveIP(), DeallocBlocks);
}
Function *CurFn = Builder.GetInsertBlock()->getParent();
@@ -10617,13 +10590,13 @@ Error OpenMPIRBuilder::emitIfClause(Value *Cond, BodyGenCallbackTy ThenGen,
Builder.CreateCondBr(Cond, ThenBlock, ElseBlock);
// Emit the 'then' code.
emitBlock(ThenBlock, CurFn);
- if (Error Err = ThenGen(AllocaIP, Builder.saveIP()))
+ if (Error Err = ThenGen(AllocaIP, Builder.saveIP(), DeallocBlocks))
return Err;
emitBranch(ContBlock);
// Emit the 'else' code if present.
// There is no need to emit line number for unconditional branch.
emitBlock(ElseBlock, CurFn);
- if (Error Err = ElseGen(AllocaIP, Builder.saveIP()))
+ if (Error Err = ElseGen(AllocaIP, Builder.saveIP(), DeallocBlocks))
return Err;
// There is no need to emit line number for unconditional branch.
emitBranch(ContBlock);
@@ -11339,13 +11312,13 @@ OpenMPIRBuilder::createTeams(const LocationDescription &Loc,
// Generate the body of teams.
InsertPointTy AllocaIP(AllocaBB, AllocaBB->begin());
InsertPointTy CodeGenIP(BodyBB, BodyBB->begin());
- if (Error Err = BodyGenCB(AllocaIP, CodeGenIP))
+ if (Error Err = BodyGenCB(AllocaIP, CodeGenIP, ExitBB))
return Err;
auto OI = std::make_unique<OutlineInfo>();
OI->EntryBB = AllocaBB;
OI->ExitBB = ExitBB;
- OI->OuterAllocaBB = &OuterAllocaBB;
+ OI->OuterAllocBB = &OuterAllocaBB;
// Insert fake values for global tid and bound tid.
SmallVector<Instruction *, 8> ToBeDeleted;
@@ -11397,19 +11370,18 @@ OpenMPIRBuilder::createTeams(const LocationDescription &Loc,
addOutlineInfo(std::move(OI));
- Builder.SetInsertPoint(ExitBB, ExitBB->begin());
+ Builder.SetInsertPoint(ExitBB);
return Builder.saveIP();
}
-OpenMPIRBuilder::InsertPointOrErrorTy
-OpenMPIRBuilder::createDistribute(const LocationDescription &Loc,
- InsertPointTy OuterAllocaIP,
- BodyGenCallbackTy BodyGenCB) {
+OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createDistribute(
+ const LocationDescription &Loc, InsertPointTy OuterAllocIP,
+ ArrayRef<BasicBlock *> OuterDeallocBlocks, BodyGenCallbackTy BodyGenCB) {
if (!updateToLocation(Loc))
return InsertPointTy();
- BasicBlock *OuterAllocaBB = OuterAllocaIP.getBlock();
+ BasicBlock *OuterAllocaBB = OuterAllocIP.getBlock();
if (OuterAllocaBB == Builder.GetInsertBlock()) {
BasicBlock *BodyBB =
@@ -11426,19 +11398,22 @@ OpenMPIRBuilder::createDistribute(const LocationDescription &Loc,
// Generate the body of distribute clause
InsertPointTy AllocaIP(AllocaBB, AllocaBB->begin());
InsertPointTy CodeGenIP(BodyBB, BodyBB->begin());
- if (Error Err = BodyGenCB(AllocaIP, CodeGenIP))
+ if (Error Err = BodyGenCB(AllocaIP, CodeGenIP, ExitBB))
return Err;
// When using target we use different runtime functions which require a
// callback.
if (Config.isTargetDevice()) {
auto OI = std::make_unique<OutlineInfo>();
- OI->OuterAllocaBB = OuterAllocaIP.getBlock();
+ OI->OuterAllocBB = OuterAllocIP.getBlock();
OI->EntryBB = AllocaBB;
OI->ExitBB = ExitBB;
+ OI->OuterDeallocBBs.reserve(OuterDeallocBlocks.size());
+ copy(OuterDeallocBlocks, OI->OuterDeallocBBs.end());
+
addOutlineInfo(std::move(OI));
}
- Builder.SetInsertPoint(ExitBB, ExitBB->begin());
+ Builder.SetInsertPoint(ExitBB);
return Builder.saveIP();
}
@@ -11501,32 +11476,33 @@ std::unique_ptr<CodeExtractor>
OpenMPIRBuilder::OutlineInfo::createCodeExtractor(ArrayRef<BasicBlock *> Blocks,
bool ArgsInZeroAddressSpace,
Twine Suffix) {
- return std::make_unique<CodeExtractor>(Blocks, /* DominatorTree */ nullptr,
- /* AggregateArgs */ true,
- /* BlockFrequencyInfo */ nullptr,
- /* BranchProbabilityInfo */ nullptr,
- /* AssumptionCache */ nullptr,
- /* AllowVarArgs */ true,
- /* AllowAlloca */ true,
- /* AllocationBlock*/ OuterAllocaBB,
- /* DeallocationBlock */ nullptr,
- /* Suffix */ Suffix.str(),
- ArgsInZeroAddressSpace);
+ return std::make_unique<CodeExtractor>(
+ Blocks, /* DominatorTree */ nullptr,
+ /* AggregateArgs */ true,
+ /* BlockFrequencyInfo */ nullptr,
+ /* BranchProbabilityInfo */ nullptr,
+ /* AssumptionCache */ nullptr,
+ /* AllowVarArgs */ true,
+ /* AllowAlloca */ true,
+ /* AllocationBlock*/ OuterAllocBB,
+ /* DeallocationBlocks */ ArrayRef<BasicBlock *>(),
+ /* Suffix */ Suffix.str(), ArgsInZeroAddressSpace);
}
std::unique_ptr<CodeExtractor> DeviceSharedMemOutlineInfo::createCodeExtractor(
ArrayRef<BasicBlock *> Blocks, bool ArgsInZeroAddressSpace, Twine Suffix) {
- // TODO: Initialize the DeallocationBlock with a proper pair to OuterAllocaBB.
return std::make_unique<DeviceSharedMemCodeExtractor>(
- OMPBuilder, AllocBlockOverride, Blocks, /* DominatorTree */ nullptr,
+ OMPBuilder, Blocks, /* DominatorTree */ nullptr,
/* AggregateArgs */ true,
/* BlockFrequencyInfo */ nullptr,
/* BranchProbabilityInfo */ nullptr,
/* AssumptionCache */ nullptr,
/* AllowVarArgs */ true,
/* AllowAlloca */ true,
- /* AllocationBlock*/ OuterAllocaBB,
- /* DeallocationBlock */ ExitBB,
+ /* AllocationBlock*/ OuterAllocBB,
+ /* DeallocationBlocks */ OuterDeallocBBs.empty()
+ ? SmallVector<BasicBlock *>{ExitBB}
+ : OuterDeallocBBs,
/* Suffix */ Suffix.str(), ArgsInZeroAddressSpace);
}
diff --git a/llvm/lib/Transforms/IPO/HotColdSplitting.cpp b/llvm/lib/Transforms/IPO/HotColdSplitting.cpp
index 6b2ecf2277cdf..bfd0b6ba4e058 100644
--- a/llvm/lib/Transforms/IPO/HotColdSplitting.cpp
+++ b/llvm/lib/Transforms/IPO/HotColdSplitting.cpp
@@ -721,7 +721,7 @@ bool HotColdSplitting::outlineColdRegions(Function &F, bool HasProfileSummary) {
SubRegion, &*DT, /* AggregateArgs */ false, /* BFI */ nullptr,
/* BPI */ nullptr, AC, /* AllowVarArgs */ false,
/* AllowAlloca */ false, /* AllocaBlock */ nullptr,
- /* DeallocationBlock */ nullptr,
+ /* DeallocationBlocks */ {},
/* Suffix */ "cold." + std::to_string(OutlinedFunctionID),
/* ArgsInZeroAddressSpace */ false,
/* VoidReturnWithSingleOutput */ false);
diff --git a/llvm/lib/Transforms/IPO/IROutliner.cpp b/llvm/lib/Transforms/IPO/IROutliner.cpp
index 2bdfe55592c70..c1640f3d0e2a7 100644
--- a/llvm/lib/Transforms/IPO/IROutliner.cpp
+++ b/llvm/lib/Transforms/IPO/IROutliner.cpp
@@ -2789,7 +2789,7 @@ unsigned IROutliner::doOutline(Module &M) {
OS->Candidate->getBasicBlocks(BlocksInRegion, BE);
OS->CE = new (ExtractorAllocator.Allocate())
CodeExtractor(BE, nullptr, false, nullptr, nullptr, nullptr, false,
- false, nullptr, nullptr, "outlined");
+ false, nullptr, {}, "outlined");
findAddInputsOutputs(M, *OS, NotSame);
if (!OS->IgnoreRegion)
OutlinedRegions.push_back(OS);
@@ -2900,7 +2900,7 @@ unsigned IROutliner::doOutline(Module &M) {
OS->Candidate->getBasicBlocks(BlocksInRegion, BE);
OS->CE = new (ExtractorAllocator.Allocate())
CodeExtractor(BE, nullptr, false, nullptr, nullptr, nullptr, false,
- false, nullptr, nullptr, "outlined");
+ false, nullptr, {}, "outlined");
bool FunctionOutlined = extractSection(*OS);
if (FunctionOutlined) {
unsigned StartIdx = OS->Candidate->getStartIdx();
diff --git a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
index 7da65ff4dbd51..31e9d41ca3410 100644
--- a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
+++ b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
@@ -1088,7 +1088,8 @@ struct OpenMPOpt {
SmallDenseMap<BasicBlock *, SmallPtrSet<Instruction *, 4>> BB2PRMap;
BasicBlock *StartBB = nullptr, *EndBB = nullptr;
- auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+ auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) {
BasicBlock *CGStartBB = CodeGenIP.getBlock();
BasicBlock *CGEndBB =
SplitBlock(CGStartBB, &*CodeGenIP.getPoint(), DT, LI);
@@ -1128,7 +1129,8 @@ struct OpenMPOpt {
const DebugLoc DL = ParentBB->getTerminator()->getDebugLoc();
ParentBB->getTerminator()->eraseFromParent();
- auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+ auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) {
BasicBlock *CGStartBB = CodeGenIP.getBlock();
BasicBlock *CGEndBB =
SplitBlock(CGStartBB, &*CodeGenIP.getPoint(), DT, LI);
@@ -1258,8 +1260,9 @@ struct OpenMPOpt {
// avoid overriding binding settings, and without explicit cancellation.
OpenMPIRBuilder::InsertPointTy AfterIP =
cantFail(OMPInfoCache.OMPBuilder.createParallel(
- Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB, nullptr, nullptr,
- OMP_PROC_BIND_default, /* IsCancellable */ false));
+ Loc, AllocaIP, /* DeallocBlocks */ {}, BodyGenCB, PrivCB, FiniCB,
+ nullptr, nullptr, OMP_PROC_BIND_default,
+ /* IsCancellable */ false));
UncondBrInst::Create(AfterBB, AfterIP.getBlock());
// Perform the actual outlining.
diff --git a/llvm/lib/Transforms/IPO/PartialInlining.cpp b/llvm/lib/Transforms/IPO/PartialInlining.cpp
index 5ad98d04a5725..31eac4064aaa2 100644
--- a/llvm/lib/Transforms/IPO/PartialInlining.cpp
+++ b/llvm/lib/Transforms/IPO/PartialInlining.cpp
@@ -1105,7 +1105,7 @@ bool PartialInlinerImpl::FunctionCloner::doMultiRegionFunctionOutlining() {
ClonedFuncBFI.get(), &BPI,
LookupAC(*RegionInfo.EntryBlock->getParent()),
/* AllowVarargs */ false, /* AllowAlloca */ false,
- /* AllocaBlock */ nullptr, /* DeallocationBlock */ nullptr,
+ /* AllocaBlock */ nullptr, /* DeallocationBlocks */ {},
/* Suffix */ "", /* ArgsInZeroAddressSpace */ false,
/* VoidReturnWithSingleOutput */ false);
@@ -1189,7 +1189,7 @@ PartialInlinerImpl::FunctionCloner::doSingleRegionFunctionOutlining() {
CodeExtractor(ToExtract, &DT, /*AggregateArgs*/ false,
ClonedFuncBFI.get(), &BPI, LookupAC(*ClonedFunc),
/* AllowVarargs */ true, /* AllowAlloca */ false,
- /* AllocaBlock */ nullptr, /* DeallocationBlock */ nullptr,
+ /* AllocaBlock */ nullptr, /* DeallocationBlocks */ {},
/* Suffix */ "", /* ArgsInZeroAddressSpace */ false,
/* VoidReturnWithSingleOutput */ false)
.extractCodeRegion(CEAC);
diff --git a/llvm/lib/Transforms/Utils/CodeExtractor.cpp b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
index ed3d4039f2e4a..d2215542ff785 100644
--- a/llvm/lib/Transforms/Utils/CodeExtractor.cpp
+++ b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
@@ -263,12 +263,12 @@ CodeExtractor::CodeExtractor(ArrayRef<BasicBlock *> BBs, DominatorTree *DT,
BranchProbabilityInfo *BPI, AssumptionCache *AC,
bool AllowVarArgs, bool AllowAlloca,
BasicBlock *AllocationBlock,
- BasicBlock *DeallocationBlock, std::string Suffix,
- bool ArgsInZeroAddressSpace,
+ ArrayRef<BasicBlock *> DeallocationBlocks,
+ std::string Suffix, bool ArgsInZeroAddressSpace,
bool VoidReturnWithSingleOutput)
: DT(DT), AggregateArgs(AggregateArgs || AggregateArgsOpt), BFI(BFI),
BPI(BPI), AC(AC), AllocationBlock(AllocationBlock),
- DeallocationBlock(DeallocationBlock), AllowVarArgs(AllowVarArgs),
+ DeallocationBlocks(DeallocationBlocks), AllowVarArgs(AllowVarArgs),
Blocks(buildExtractionBlockSet(BBs, DT, AllowVarArgs, AllowAlloca)),
Suffix(Suffix), ArgsInZeroAddressSpace(ArgsInZeroAddressSpace),
VoidReturnWithSingleOutput(VoidReturnWithSingleOutput) {}
@@ -445,24 +445,24 @@ CodeExtractor::findOrCreateBlockForHoisting(BasicBlock *CommonExitBlock) {
return CommonExitBlock;
}
-Instruction *CodeExtractor::allocateVar(BasicBlock *BB,
- BasicBlock::iterator AllocIP,
+Instruction *CodeExtractor::allocateVar(IRBuilder<>::InsertPoint AllocaIP,
Type *VarType, const Twine &Name,
AddrSpaceCastInst **CastedAlloc) {
- const DataLayout &DL = BB->getModule()->getDataLayout();
- Instruction *Alloca =
- new AllocaInst(VarType, DL.getAllocaAddrSpace(), nullptr, Name, AllocIP);
+ const DataLayout &DL = AllocaIP.getBlock()->getModule()->getDataLayout();
+ Instruction *Alloca = new AllocaInst(VarType, DL.getAllocaAddrSpace(),
+ nullptr, Name, AllocaIP.getPoint());
if (CastedAlloc && ArgsInZeroAddressSpace && DL.getAllocaAddrSpace() != 0) {
*CastedAlloc = new AddrSpaceCastInst(
- Alloca, PointerType::get(BB->getContext(), 0), Name + ".ascast");
+ Alloca, PointerType::get(AllocaIP.getBlock()->getContext(), 0),
+ Name + ".ascast");
(*CastedAlloc)->insertAfter(Alloca->getIterator());
}
return Alloca;
}
-Instruction *CodeExtractor::deallocateVar(BasicBlock *, BasicBlock::iterator,
- Value *, Type *) {
+Instruction *CodeExtractor::deallocateVar(IRBuilder<>::InsertPoint, Value *,
+ Type *) {
// Default alloca instructions created by allocateVar are released implicitly.
return nullptr;
}
@@ -1887,7 +1887,8 @@ CallInst *CodeExtractor::emitReplacerCall(
continue;
Value *OutAlloc =
- allocateVar(AllocaBlock, AllocaBlock->getFirstInsertionPt(),
+ allocateVar(IRBuilder<>::InsertPoint(
+ AllocaBlock, AllocaBlock->getFirstInsertionPt()),
output->getType(), output->getName() + ".loc");
params.push_back(OutAlloc);
ReloadOutputs.push_back(OutAlloc);
@@ -1896,7 +1897,8 @@ CallInst *CodeExtractor::emitReplacerCall(
Instruction *Struct = nullptr;
if (!StructValues.empty()) {
AddrSpaceCastInst *StructSpaceCast = nullptr;
- Struct = allocateVar(AllocaBlock, AllocaBlock->getFirstInsertionPt(),
+ Struct = allocateVar(IRBuilder<>::InsertPoint(
+ AllocaBlock, AllocaBlock->getFirstInsertionPt()),
StructArgTy, "structArg", &StructSpaceCast);
if (StructSpaceCast)
params.push_back(StructSpaceCast);
@@ -2045,22 +2047,26 @@ CallInst *CodeExtractor::emitReplacerCall(
{}, call);
// Deallocate intermediate variables if they need explicit deallocation.
- BasicBlock *DeallocBlock = codeReplacer;
- BasicBlock::iterator DeallocIP = codeReplacer->end();
- if (DeallocationBlock) {
- DeallocBlock = DeallocationBlock;
- DeallocIP = DeallocationBlock->getFirstInsertionPt();
- }
+ auto deallocVars = [&](BasicBlock *DeallocBlock,
+ BasicBlock::iterator DeallocIP) {
+ int Index = 0;
+ for (Value *Output : outputs) {
+ if (!StructValues.contains(Output))
+ deallocateVar(IRBuilder<>::InsertPoint(DeallocBlock, DeallocIP),
+ ReloadOutputs[Index++], Output->getType());
+ }
- int Index = 0;
- for (Value *Output : outputs) {
- if (!StructValues.contains(Output))
- deallocateVar(DeallocBlock, DeallocIP, ReloadOutputs[Index++],
- Output->getType());
- }
+ if (Struct)
+ deallocateVar(IRBuilder<>::InsertPoint(DeallocBlock, DeallocIP), Struct,
+ StructArgTy);
+ };
- if (Struct)
- deallocateVar(DeallocBlock, DeallocIP, Struct, StructArgTy);
+ if (DeallocationBlocks.empty()) {
+ deallocVars(codeReplacer, codeReplacer->end());
+ } else {
+ for (BasicBlock *DeallocationBlock : DeallocationBlocks)
+ deallocVars(DeallocationBlock, DeallocationBlock->getFirstInsertionPt());
+ }
return call;
}
diff --git a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
index 5b9b280ec7671..eb71d0949c854 100644
--- a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
+++ b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
@@ -58,8 +58,9 @@ using namespace omp;
}
#define BODYGENCB_WRAPPER(cb) \
- [&cb](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) -> Error { \
- cb(AllocaIP, CodeGenIP); \
+ [&cb](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, \
+ ArrayRef<BasicBlock *> DeallocBlocks) -> Error { \
+ cb(AllocaIP, CodeGenIP, DeallocBlocks); \
return Error::success(); \
}
@@ -651,7 +652,8 @@ TEST_F(OpenMPIRBuilderTest, ParallelSimpleGPU) {
unsigned NumPrivatizedVars = 0;
unsigned NumFinalizationPoints = 0;
- auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+ auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) {
++NumBodiesGenerated;
Builder.restoreIP(AllocaIP);
@@ -702,8 +704,8 @@ TEST_F(OpenMPIRBuilderTest, ParallelSimpleGPU) {
F->getEntryBlock().getFirstInsertionPt());
ASSERT_EXPECTED_INIT(OpenMPIRBuilder::InsertPointTy, AfterIP,
OMPBuilder.createParallel(
- Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB, nullptr,
- nullptr, OMP_PROC_BIND_default, false));
+ Loc, AllocaIP, {}, BodyGenCB, PrivCB, FiniCB,
+ nullptr, nullptr, OMP_PROC_BIND_default, false));
EXPECT_EQ(NumBodiesGenerated, 1U);
EXPECT_EQ(NumPrivatizedVars, 1U);
@@ -730,8 +732,9 @@ TEST_F(OpenMPIRBuilderTest, ParallelSimpleGPU) {
EXPECT_EQ(OutlinedFn->getArg(2)->getType(),
PointerType::get(M->getContext(), 0));
EXPECT_EQ(&OutlinedFn->getEntryBlock(), PrivAI->getParent());
- EXPECT_TRUE(OutlinedFn->hasOneUse());
- User *Usr = OutlinedFn->user_back();
+ EXPECT_TRUE(OutlinedFn->hasNUses(2));
+ User *Usr = *OutlinedFn->users().begin();
+ User *WrapperUsr = *++OutlinedFn->users().begin();
ASSERT_TRUE(isa<CallInst>(Usr));
CallInst *Parallel60CI = dyn_cast<CallInst>(Usr);
ASSERT_NE(Parallel60CI, nullptr);
@@ -742,6 +745,20 @@ TEST_F(OpenMPIRBuilderTest, ParallelSimpleGPU) {
EXPECT_TRUE(
isa<GlobalVariable>(Parallel60CI->getArgOperand(0)->stripPointerCasts()));
EXPECT_EQ(Parallel60CI, Usr);
+
+ ASSERT_TRUE(isa<CallInst>(WrapperUsr));
+ CallInst *OutlinedCI = dyn_cast<CallInst>(WrapperUsr);
+ ASSERT_NE(OutlinedCI, nullptr);
+ EXPECT_EQ(OutlinedCI->getCalledFunction(), OutlinedFn);
+
+ Function *WrapperFn = OutlinedCI->getFunction();
+ EXPECT_TRUE(WrapperFn->hasInternalLinkage());
+ EXPECT_EQ(WrapperFn->arg_size(), 2U);
+ EXPECT_EQ(WrapperFn->getArg(0)->getType(),
+ IntegerType::getInt16Ty(M->getContext()));
+ EXPECT_EQ(WrapperFn->getArg(1)->getType(),
+ IntegerType::getInt32Ty(M->getContext()));
+
M->setDataLayout(oldDLStr);
}
@@ -764,7 +781,8 @@ TEST_F(OpenMPIRBuilderTest, ParallelSimple) {
unsigned NumPrivatizedVars = 0;
unsigned NumFinalizationPoints = 0;
- auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+ auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) {
++NumBodiesGenerated;
Builder.restoreIP(AllocaIP);
@@ -815,8 +833,8 @@ TEST_F(OpenMPIRBuilderTest, ParallelSimple) {
F->getEntryBlock().getFirstInsertionPt());
ASSERT_EXPECTED_INIT(OpenMPIRBuilder::InsertPointTy, AfterIP,
OMPBuilder.createParallel(
- Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB, nullptr,
- nullptr, OMP_PROC_BIND_default, false));
+ Loc, AllocaIP, {}, BodyGenCB, PrivCB, FiniCB,
+ nullptr, nullptr, OMP_PROC_BIND_default, false));
EXPECT_EQ(NumBodiesGenerated, 1U);
EXPECT_EQ(NumPrivatizedVars, 1U);
EXPECT_EQ(NumFinalizationPoints, 1U);
@@ -872,7 +890,8 @@ TEST_F(OpenMPIRBuilderTest, ParallelNested) {
unsigned NumOuterBodiesGenerated = 0;
unsigned NumFinalizationPoints = 0;
- auto InnerBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+ auto InnerBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) {
++NumInnerBodiesGenerated;
return Error::success();
};
@@ -895,7 +914,8 @@ TEST_F(OpenMPIRBuilderTest, ParallelNested) {
return Error::success();
};
- auto OuterBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+ auto OuterBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) {
++NumOuterBodiesGenerated;
Builder.restoreIP(CodeGenIP);
BasicBlock *CGBB = CodeGenIP.getBlock();
@@ -905,7 +925,7 @@ TEST_F(OpenMPIRBuilderTest, ParallelNested) {
ASSERT_EXPECTED_INIT(
OpenMPIRBuilder::InsertPointTy, AfterIP,
OMPBuilder.createParallel(InsertPointTy(CGBB, CGBB->end()), AllocaIP,
- InnerBodyGenCB, PrivCB, FiniCB, nullptr,
+ {}, InnerBodyGenCB, PrivCB, FiniCB, nullptr,
nullptr, OMP_PROC_BIND_default, false));
Builder.restoreIP(AfterIP);
@@ -916,7 +936,7 @@ TEST_F(OpenMPIRBuilderTest, ParallelNested) {
F->getEntryBlock().getFirstInsertionPt());
ASSERT_EXPECTED_INIT(OpenMPIRBuilder::InsertPointTy, AfterIP,
OMPBuilder.createParallel(
- Loc, AllocaIP, BODYGENCB_WRAPPER(OuterBodyGenCB),
+ Loc, AllocaIP, {}, BODYGENCB_WRAPPER(OuterBodyGenCB),
PrivCB, FiniCB, nullptr, nullptr,
OMP_PROC_BIND_default, false));
@@ -973,7 +993,8 @@ TEST_F(OpenMPIRBuilderTest, ParallelNested2Inner) {
unsigned NumOuterBodiesGenerated = 0;
unsigned NumFinalizationPoints = 0;
- auto InnerBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+ auto InnerBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) {
++NumInnerBodiesGenerated;
return Error::success();
};
@@ -996,7 +1017,8 @@ TEST_F(OpenMPIRBuilderTest, ParallelNested2Inner) {
return Error::success();
};
- auto OuterBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+ auto OuterBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) {
++NumOuterBodiesGenerated;
Builder.restoreIP(CodeGenIP);
BasicBlock *CGBB = CodeGenIP.getBlock();
@@ -1010,7 +1032,7 @@ TEST_F(OpenMPIRBuilderTest, ParallelNested2Inner) {
ASSERT_EXPECTED_INIT(
OpenMPIRBuilder::InsertPointTy, AfterIP1,
OMPBuilder.createParallel(InsertPointTy(CGBB, CGBB->end()), AllocaIP,
- InnerBodyGenCB, PrivCB, FiniCB, nullptr,
+ {}, InnerBodyGenCB, PrivCB, FiniCB, nullptr,
nullptr, OMP_PROC_BIND_default, false));
Builder.restoreIP(AfterIP1);
@@ -1018,7 +1040,7 @@ TEST_F(OpenMPIRBuilderTest, ParallelNested2Inner) {
ASSERT_EXPECTED_INIT(OpenMPIRBuilder::InsertPointTy, AfterIP2,
OMPBuilder.createParallel(
- InsertPointTy(NewBB1, NewBB1->end()), AllocaIP,
+ InsertPointTy(NewBB1, NewBB1->end()), AllocaIP, {},
InnerBodyGenCB, PrivCB, FiniCB, nullptr, nullptr,
OMP_PROC_BIND_default, false));
@@ -1030,7 +1052,7 @@ TEST_F(OpenMPIRBuilderTest, ParallelNested2Inner) {
F->getEntryBlock().getFirstInsertionPt());
ASSERT_EXPECTED_INIT(OpenMPIRBuilder::InsertPointTy, AfterIP,
OMPBuilder.createParallel(
- Loc, AllocaIP, BODYGENCB_WRAPPER(OuterBodyGenCB),
+ Loc, AllocaIP, {}, BODYGENCB_WRAPPER(OuterBodyGenCB),
PrivCB, FiniCB, nullptr, nullptr,
OMP_PROC_BIND_default, false));
@@ -1094,7 +1116,8 @@ TEST_F(OpenMPIRBuilderTest, ParallelIfCond) {
unsigned NumPrivatizedVars = 0;
unsigned NumFinalizationPoints = 0;
- auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+ auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) {
++NumBodiesGenerated;
Builder.restoreIP(AllocaIP);
@@ -1146,7 +1169,7 @@ TEST_F(OpenMPIRBuilderTest, ParallelIfCond) {
F->getEntryBlock().getFirstInsertionPt());
ASSERT_EXPECTED_INIT(
OpenMPIRBuilder::InsertPointTy, AfterIP,
- OMPBuilder.createParallel(Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB,
+ OMPBuilder.createParallel(Loc, AllocaIP, {}, BodyGenCB, PrivCB, FiniCB,
Builder.CreateIsNotNull(F->arg_begin()),
nullptr, OMP_PROC_BIND_default, false));
@@ -1201,7 +1224,8 @@ TEST_F(OpenMPIRBuilderTest, ParallelCancelBarrier) {
unsigned NumFinalizationPoints = 0;
CallInst *CheckedBarrier = nullptr;
- auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+ auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) {
++NumBodiesGenerated;
Builder.restoreIP(CodeGenIP);
@@ -1269,11 +1293,12 @@ TEST_F(OpenMPIRBuilderTest, ParallelCancelBarrier) {
IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
F->getEntryBlock().getFirstInsertionPt());
- ASSERT_EXPECTED_INIT(OpenMPIRBuilder::InsertPointTy, AfterIP,
- OMPBuilder.createParallel(
- Loc, AllocaIP, BODYGENCB_WRAPPER(BodyGenCB), PrivCB,
- FiniCB, Builder.CreateIsNotNull(F->arg_begin()),
- nullptr, OMP_PROC_BIND_default, true));
+ ASSERT_EXPECTED_INIT(
+ OpenMPIRBuilder::InsertPointTy, AfterIP,
+ OMPBuilder.createParallel(Loc, AllocaIP, {}, BODYGENCB_WRAPPER(BodyGenCB),
+ PrivCB, FiniCB,
+ Builder.CreateIsNotNull(F->arg_begin()),
+ nullptr, OMP_PROC_BIND_default, true));
EXPECT_EQ(NumBodiesGenerated, 1U);
EXPECT_EQ(NumPrivatizedVars, 0U);
@@ -1336,7 +1361,8 @@ TEST_F(OpenMPIRBuilderTest, ParallelForwardAsPointers) {
Value *StructPtrVal = Builder.CreateCall(RetStructPtrFunc);
Instruction *Internal;
- auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+ auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) {
IRBuilder<>::InsertPointGuard Guard(Builder);
Builder.restoreIP(CodeGenIP);
Internal = Builder.CreateCall(TakeI32Func, I32Val);
@@ -1356,8 +1382,8 @@ TEST_F(OpenMPIRBuilderTest, ParallelForwardAsPointers) {
F->getEntryBlock().getFirstInsertionPt());
ASSERT_EXPECTED_INIT(OpenMPIRBuilder::InsertPointTy, AfterIP,
OMPBuilder.createParallel(
- Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB, nullptr,
- nullptr, OMP_PROC_BIND_default, false));
+ Loc, AllocaIP, {}, BodyGenCB, PrivCB, FiniCB,
+ nullptr, nullptr, OMP_PROC_BIND_default, false));
Builder.restoreIP(AfterIP);
Builder.CreateRetVoid();
@@ -2860,7 +2886,8 @@ TEST_F(OpenMPIRBuilderTest, MasterDirective) {
BasicBlock *EntryBB = nullptr;
BasicBlock *ThenBB = nullptr;
- auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+ auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) {
if (AllocaIP.isSet())
Builder.restoreIP(AllocaIP);
else
@@ -2941,7 +2968,8 @@ TEST_F(OpenMPIRBuilderTest, MaskedDirective) {
BasicBlock *EntryBB = nullptr;
BasicBlock *ThenBB = nullptr;
- auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+ auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) {
if (AllocaIP.isSet())
Builder.restoreIP(AllocaIP);
else
@@ -3021,7 +3049,8 @@ TEST_F(OpenMPIRBuilderTest, CriticalDirective) {
Type *PrivType = F->arg_begin()->getType();
AllocaInst *PrivAI = Builder.CreateAlloca(PrivType);
- auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+ auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) {
// actual start for bodyCB
llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();
@@ -3274,7 +3303,8 @@ TEST_F(OpenMPIRBuilderTest, OrderedDirectiveThreads) {
Type *PrivType = F->arg_begin()->getType();
AllocaInst *PrivAI = Builder.CreateAlloca(PrivType, nullptr, "priv.inst");
- auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+ auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) {
llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();
EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);
@@ -3350,7 +3380,8 @@ TEST_F(OpenMPIRBuilderTest, OrderedDirectiveSimd) {
Type *PrivType = F->arg_begin()->getType();
AllocaInst *PrivAI = Builder.CreateAlloca(PrivType, nullptr, "priv.inst");
- auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+ auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) {
llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();
EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);
@@ -3454,7 +3485,8 @@ TEST_F(OpenMPIRBuilderTest, SingleDirective) {
BasicBlock *EntryBB = nullptr;
BasicBlock *ThenBB = nullptr;
- auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+ auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) {
if (AllocaIP.isSet())
Builder.restoreIP(AllocaIP);
else
@@ -3547,7 +3579,8 @@ TEST_F(OpenMPIRBuilderTest, SingleDirectiveNowait) {
BasicBlock *EntryBB = nullptr;
BasicBlock *ThenBB = nullptr;
- auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+ auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) {
if (AllocaIP.isSet())
Builder.restoreIP(AllocaIP);
else
@@ -3668,7 +3701,8 @@ TEST_F(OpenMPIRBuilderTest, SingleDirectiveCopyPrivate) {
Function *CopyFunc =
Function::Create(CopyFuncTy, Function::PrivateLinkage, "copy_var", *M);
- auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+ auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) {
if (AllocaIP.isSet())
Builder.restoreIP(AllocaIP);
else
@@ -4606,7 +4640,8 @@ TEST_F(OpenMPIRBuilderTest, CreateTeams) {
AllocaInst *ValPtr128 = Builder.CreateAlloca(Builder.getInt128Ty());
Value *Val128 = Builder.CreateLoad(Builder.getInt128Ty(), ValPtr128, "load");
- auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+ auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) {
Builder.restoreIP(AllocaIP);
AllocaInst *Local128 = Builder.CreateAlloca(Builder.getInt128Ty(), nullptr,
"bodygen.alloca128");
@@ -4687,7 +4722,8 @@ TEST_F(OpenMPIRBuilderTest, CreateTeamsWithThreadLimit) {
Function::Create(FunctionType::get(Builder.getVoidTy(), false),
GlobalValue::ExternalLinkage, "fakeFunction", M.get());
- auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+ auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) {
Builder.restoreIP(CodeGenIP);
Builder.CreateCall(FakeFunction, {});
return Error::success();
@@ -4742,7 +4778,8 @@ TEST_F(OpenMPIRBuilderTest, CreateTeamsWithNumTeamsUpper) {
Function::Create(FunctionType::get(Builder.getVoidTy(), false),
GlobalValue::ExternalLinkage, "fakeFunction", M.get());
- auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+ auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) {
Builder.restoreIP(CodeGenIP);
Builder.CreateCall(FakeFunction, {});
return Error::success();
@@ -4803,7 +4840,8 @@ TEST_F(OpenMPIRBuilderTest, CreateTeamsWithNumTeamsBoth) {
Value *NumTeamsUpper =
Builder.CreateAdd(F->arg_begin(), Builder.getInt32(10), "numTeamsUpper");
- auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+ auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) {
Builder.restoreIP(CodeGenIP);
Builder.CreateCall(FakeFunction, {});
return Error::success();
@@ -4869,7 +4907,8 @@ TEST_F(OpenMPIRBuilderTest, CreateTeamsWithNumTeamsAndThreadLimit) {
Function::Create(FunctionType::get(Builder.getVoidTy(), false),
GlobalValue::ExternalLinkage, "fakeFunction", M.get());
- auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+ auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) {
Builder.restoreIP(CodeGenIP);
Builder.CreateCall(FakeFunction, {});
return Error::success();
@@ -4925,7 +4964,8 @@ TEST_F(OpenMPIRBuilderTest, CreateTeamsWithIfCondition) {
Function::Create(FunctionType::get(Builder.getVoidTy(), false),
GlobalValue::ExternalLinkage, "fakeFunction", M.get());
- auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+ auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) {
Builder.restoreIP(CodeGenIP);
Builder.CreateCall(FakeFunction, {});
return Error::success();
@@ -4992,7 +5032,8 @@ TEST_F(OpenMPIRBuilderTest, CreateTeamsWithIfConditionAndNumTeams) {
Function::Create(FunctionType::get(Builder.getVoidTy(), false),
GlobalValue::ExternalLinkage, "fakeFunction", M.get());
- auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+ auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) {
Builder.restoreIP(CodeGenIP);
Builder.CreateCall(FakeFunction, {});
return Error::success();
@@ -5210,7 +5251,8 @@ TEST_F(OpenMPIRBuilderTest, CreateReductions) {
// xor of thread-id;
// and store the result in global variables.
InsertPointTy BodyIP, BodyAllocaIP;
- auto BodyGenCB = [&](InsertPointTy InnerAllocaIP, InsertPointTy CodeGenIP) {
+ auto BodyGenCB = [&](InsertPointTy InnerAllocIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) {
IRBuilderBase::InsertPointGuard Guard(Builder);
Builder.restoreIP(CodeGenIP);
@@ -5228,7 +5270,7 @@ TEST_F(OpenMPIRBuilderTest, CreateReductions) {
Builder.CreateStore(Xor, XorReduced);
BodyIP = Builder.saveIP();
- BodyAllocaIP = InnerAllocaIP;
+ BodyAllocaIP = InnerAllocIP;
return Error::success();
};
@@ -5264,12 +5306,12 @@ TEST_F(OpenMPIRBuilderTest, CreateReductions) {
// Do nothing in finalization.
auto FiniCB = [&](InsertPointTy CodeGenIP) { return Error::success(); };
- ASSERT_EXPECTED_INIT(
- OpenMPIRBuilder::InsertPointTy, AfterIP,
- OMPBuilder.createParallel(Loc, OuterAllocaIP, BodyGenCB, PrivCB, FiniCB,
- /* IfCondition */ nullptr,
- /* NumThreads */ nullptr, OMP_PROC_BIND_default,
- /* IsCancellable */ false));
+ ASSERT_EXPECTED_INIT(OpenMPIRBuilder::InsertPointTy, AfterIP,
+ OMPBuilder.createParallel(
+ Loc, OuterAllocaIP, {}, BodyGenCB, PrivCB, FiniCB,
+ /* IfCondition */ nullptr,
+ /* NumThreads */ nullptr, OMP_PROC_BIND_default,
+ /* IsCancellable */ false));
Builder.restoreIP(AfterIP);
OpenMPIRBuilder::ReductionInfo ReductionInfos[] = {
@@ -5591,8 +5633,8 @@ TEST_F(OpenMPIRBuilderTest, CreateTwoReductions) {
Builder.CreateStore(Builder.getInt32(1), XorReduced);
InsertPointTy FirstBodyIP, FirstBodyAllocaIP;
- auto FirstBodyGenCB = [&](InsertPointTy InnerAllocaIP,
- InsertPointTy CodeGenIP) {
+ auto FirstBodyGenCB = [&](InsertPointTy InnerAllocIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) {
IRBuilderBase::InsertPointGuard Guard(Builder);
Builder.restoreIP(CodeGenIP);
@@ -5607,13 +5649,14 @@ TEST_F(OpenMPIRBuilderTest, CreateTwoReductions) {
Builder.CreateStore(Sum, SumReduced);
FirstBodyIP = Builder.saveIP();
- FirstBodyAllocaIP = InnerAllocaIP;
+ FirstBodyAllocaIP = InnerAllocIP;
return Error::success();
};
InsertPointTy SecondBodyIP, SecondBodyAllocaIP;
- auto SecondBodyGenCB = [&](InsertPointTy InnerAllocaIP,
- InsertPointTy CodeGenIP) {
+ auto SecondBodyGenCB = [&](InsertPointTy InnerAllocIP,
+ InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) {
IRBuilderBase::InsertPointGuard Guard(Builder);
Builder.restoreIP(CodeGenIP);
@@ -5626,7 +5669,7 @@ TEST_F(OpenMPIRBuilderTest, CreateTwoReductions) {
Builder.CreateStore(Xor, XorReduced);
SecondBodyIP = Builder.saveIP();
- SecondBodyAllocaIP = InnerAllocaIP;
+ SecondBodyAllocaIP = InnerAllocIP;
return Error::success();
};
@@ -5666,14 +5709,14 @@ TEST_F(OpenMPIRBuilderTest, CreateTwoReductions) {
ASSERT_EXPECTED_INIT(
OpenMPIRBuilder::InsertPointTy, AfterIP1,
- OMPBuilder.createParallel(Loc, OuterAllocaIP, FirstBodyGenCB, PrivCB,
+ OMPBuilder.createParallel(Loc, OuterAllocaIP, {}, FirstBodyGenCB, PrivCB,
FiniCB, /* IfCondition */ nullptr,
/* NumThreads */ nullptr, OMP_PROC_BIND_default,
/* IsCancellable */ false));
Builder.restoreIP(AfterIP1);
ASSERT_EXPECTED_INIT(
OpenMPIRBuilder::InsertPointTy, AfterIP2,
- OMPBuilder.createParallel({Builder.saveIP(), DL}, OuterAllocaIP,
+ OMPBuilder.createParallel({Builder.saveIP(), DL}, OuterAllocaIP, {},
SecondBodyGenCB, PrivCB, FiniCB,
/* IfCondition */ nullptr,
/* NumThreads */ nullptr, OMP_PROC_BIND_default,
@@ -5769,7 +5812,8 @@ TEST_F(OpenMPIRBuilderTest, CreateSectionsSimple) {
llvm::SmallVector<BodyGenCallbackTy, 4> SectionCBVector;
auto FiniCB = [&](InsertPointTy IP) { return Error::success(); };
- auto SectionCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+ auto SectionCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) {
return Error::success();
};
SectionCBVector.push_back(SectionCB);
@@ -5814,7 +5858,8 @@ TEST_F(OpenMPIRBuilderTest, CreateSections) {
EXPECT_NE(IPBB->end(), IP.getPoint());
};
- auto SectionCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+ auto SectionCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) {
++NumBodiesGenerated;
CaseBBs.push_back(CodeGenIP.getBlock());
SwitchBB = CodeGenIP.getBlock()->getSinglePredecessor();
@@ -6154,7 +6199,7 @@ TEST_F(OpenMPIRBuilderTest, TargetEnterData) {
ASSERT_EXPECTED_INIT(
OpenMPIRBuilder::InsertPointTy, AfterIP,
OMPBuilder.createTargetData(
- Loc, AllocaIP, Builder.saveIP(), Builder.getInt64(DeviceID),
+ Loc, AllocaIP, Builder.saveIP(), {}, Builder.getInt64(DeviceID),
/* IfCond= */ nullptr, Info, GenMapInfoCB, CustomMapperCB, &RTLFunc));
Builder.restoreIP(AfterIP);
@@ -6217,7 +6262,7 @@ TEST_F(OpenMPIRBuilderTest, TargetExitData) {
ASSERT_EXPECTED_INIT(
OpenMPIRBuilder::InsertPointTy, AfterIP,
OMPBuilder.createTargetData(
- Loc, AllocaIP, Builder.saveIP(), Builder.getInt64(DeviceID),
+ Loc, AllocaIP, Builder.saveIP(), {}, Builder.getInt64(DeviceID),
/* IfCond= */ nullptr, Info, GenMapInfoCB, CustomMapperCB, &RTLFunc));
Builder.restoreIP(AfterIP);
@@ -6328,7 +6373,7 @@ TEST_F(OpenMPIRBuilderTest, TargetDataRegion) {
ASSERT_EXPECTED_INIT(
OpenMPIRBuilder::InsertPointTy, TargetDataIP1,
- OMPBuilder.createTargetData(Loc, AllocaIP, Builder.saveIP(),
+ OMPBuilder.createTargetData(Loc, AllocaIP, Builder.saveIP(), {},
Builder.getInt64(DeviceID),
/* IfCond= */ nullptr, Info, GenMapInfoCB,
CustomMapperCB, nullptr, BodyCB));
@@ -6387,8 +6432,8 @@ TEST_F(OpenMPIRBuilderTest, TargetRegion) {
Builder.CreateStore(Builder.getInt32(10), APtr);
Builder.CreateStore(Builder.getInt32(20), BPtr);
- auto BodyGenCB = [&](InsertPointTy AllocaIP,
- InsertPointTy CodeGenIP) -> InsertPointTy {
+ auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) -> InsertPointTy {
IRBuilderBase::InsertPointGuard guard(Builder);
Builder.SetCurrentDebugLocation(llvm::DebugLoc());
Builder.restoreIP(CodeGenIP);
@@ -6459,10 +6504,10 @@ TEST_F(OpenMPIRBuilderTest, TargetRegion) {
ASSERT_EXPECTED_INIT(
OpenMPIRBuilder::InsertPointTy, AfterIP,
OMPBuilder.createTarget(OmpLoc, /*IsOffloadEntry=*/true, Builder.saveIP(),
- Builder.saveIP(), Info, EntryInfo, DefaultAttrs,
- RuntimeAttrs, /*IfCond=*/nullptr, Inputs,
- GenMapInfoCB, BodyGenCB, SimpleArgAccessorCB,
- CustomMapperCB, {}, false));
+ Builder.saveIP(), {}, Info, EntryInfo,
+ DefaultAttrs, RuntimeAttrs, /*IfCond=*/nullptr,
+ Inputs, GenMapInfoCB, BodyGenCB,
+ SimpleArgAccessorCB, CustomMapperCB, {}, false));
EXPECT_EQ(DL, Builder.getCurrentDebugLocation());
Builder.restoreIP(AfterIP);
@@ -6608,7 +6653,8 @@ TEST_F(OpenMPIRBuilderTest, TargetRegionDevice) {
auto CustomMapperCB = [&](unsigned int I) { return nullptr; };
auto BodyGenCB = [&](OpenMPIRBuilder::InsertPointTy AllocaIP,
- OpenMPIRBuilder::InsertPointTy CodeGenIP)
+ OpenMPIRBuilder::InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks)
-> OpenMPIRBuilder::InsertPointTy {
IRBuilderBase::InsertPointGuard guard(Builder);
Builder.SetCurrentDebugLocation(llvm::DebugLoc());
@@ -6633,7 +6679,7 @@ TEST_F(OpenMPIRBuilderTest, TargetRegionDevice) {
ASSERT_EXPECTED_INIT(
OpenMPIRBuilder::InsertPointTy, AfterIP,
OMPBuilder.createTarget(Loc, /*IsOffloadEntry=*/true, EntryIP, EntryIP,
- Info, EntryInfo, DefaultAttrs, RuntimeAttrs,
+ {}, Info, EntryInfo, DefaultAttrs, RuntimeAttrs,
/*IfCond=*/nullptr, CapturedArgs, GenMapInfoCB,
BodyGenCB, SimpleArgAccessorCB, CustomMapperCB,
{}, false));
@@ -6714,7 +6760,14 @@ TEST_F(OpenMPIRBuilderTest, TargetRegionDevice) {
Instruction *Value1 = &*OutlinedBlock->getFirstNonPHIIt();
EXPECT_EQ(Value1, Value);
EXPECT_EQ(Value1->getNextNode(), TargetStore);
- auto *Deinit = TargetStore->getNextNode();
+
+ auto *TargetExitBlockBr = TargetStore->getNextNode();
+ EXPECT_TRUE(isa<UncondBrInst>(TargetExitBlockBr));
+
+ auto *TargetExitBlock = TargetExitBlockBr->getSuccessor(0);
+ EXPECT_EQ(TargetExitBlock->getName(), "target.exit");
+
+ Instruction *Deinit = &*TargetExitBlock->getFirstNonPHIIt();
EXPECT_NE(Deinit, nullptr);
auto *DeinitCall = dyn_cast<CallInst>(Deinit);
@@ -6761,8 +6814,8 @@ TEST_F(OpenMPIRBuilderTest, TargetRegionSPMD) {
IRBuilder<> Builder(BB);
auto CustomMapperCB = [&](unsigned int I) { return nullptr; };
- auto BodyGenCB = [&](InsertPointTy,
- InsertPointTy CodeGenIP) -> InsertPointTy {
+ auto BodyGenCB = [&](InsertPointTy, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *>) -> InsertPointTy {
Builder.restoreIP(CodeGenIP);
return Builder.saveIP();
};
@@ -6796,10 +6849,10 @@ TEST_F(OpenMPIRBuilderTest, TargetRegionSPMD) {
ASSERT_EXPECTED_INIT(
OpenMPIRBuilder::InsertPointTy, AfterIP,
OMPBuilder.createTarget(OmpLoc, /*IsOffloadEntry=*/true, Builder.saveIP(),
- Builder.saveIP(), Info, EntryInfo, DefaultAttrs,
- RuntimeAttrs, /*IfCond=*/nullptr, Inputs,
- GenMapInfoCB, BodyGenCB, SimpleArgAccessorCB,
- CustomMapperCB, {}));
+ Builder.saveIP(), {}, Info, EntryInfo,
+ DefaultAttrs, RuntimeAttrs, /*IfCond=*/nullptr,
+ Inputs, GenMapInfoCB, BodyGenCB,
+ SimpleArgAccessorCB, CustomMapperCB, {}));
Builder.restoreIP(AfterIP);
OMPBuilder.finalize();
@@ -6881,9 +6934,10 @@ TEST_F(OpenMPIRBuilderTest, TargetRegionDeviceSPMD) {
};
auto CustomMapperCB = [&](unsigned int I) { return nullptr; };
- auto BodyGenCB = [&](OpenMPIRBuilder::InsertPointTy,
- OpenMPIRBuilder::InsertPointTy CodeGenIP)
- -> OpenMPIRBuilder::InsertPointTy {
+ auto BodyGenCB =
+ [&](OpenMPIRBuilder::InsertPointTy,
+ OpenMPIRBuilder::InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *>) -> OpenMPIRBuilder::InsertPointTy {
Builder.restoreIP(CodeGenIP);
OutlinedFn = CodeGenIP.getBlock()->getParent();
return Builder.saveIP();
@@ -6903,8 +6957,8 @@ TEST_F(OpenMPIRBuilderTest, TargetRegionDeviceSPMD) {
ASSERT_EXPECTED_INIT(OpenMPIRBuilder::InsertPointTy, AfterIP,
OMPBuilder.createTarget(
- Loc, /*IsOffloadEntry=*/true, EntryIP, EntryIP, Info,
- EntryInfo, DefaultAttrs, RuntimeAttrs,
+ Loc, /*IsOffloadEntry=*/true, EntryIP, EntryIP, {},
+ Info, EntryInfo, DefaultAttrs, RuntimeAttrs,
/*IfCond=*/nullptr, CapturedArgs, GenMapInfoCB,
BodyGenCB, SimpleArgAccessorCB, CustomMapperCB, {}));
Builder.restoreIP(AfterIP);
@@ -7002,7 +7056,8 @@ TEST_F(OpenMPIRBuilderTest, ConstantAllocaRaise) {
auto CustomMapperCB = [&](unsigned int I) { return nullptr; };
auto BodyGenCB = [&](OpenMPIRBuilder::InsertPointTy AllocaIP,
- OpenMPIRBuilder::InsertPointTy CodeGenIP)
+ OpenMPIRBuilder::InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks)
-> OpenMPIRBuilder::InsertPointTy {
IRBuilderBase::InsertPointGuard guard(Builder);
Builder.SetCurrentDebugLocation(llvm::DebugLoc());
@@ -7028,7 +7083,7 @@ TEST_F(OpenMPIRBuilderTest, ConstantAllocaRaise) {
ASSERT_EXPECTED_INIT(
OpenMPIRBuilder::InsertPointTy, AfterIP,
OMPBuilder.createTarget(Loc, /*IsOffloadEntry=*/true, EntryIP, EntryIP,
- Info, EntryInfo, DefaultAttrs, RuntimeAttrs,
+ {}, Info, EntryInfo, DefaultAttrs, RuntimeAttrs,
/*IfCond=*/nullptr, CapturedArgs, GenMapInfoCB,
BodyGenCB, SimpleArgAccessorCB, CustomMapperCB,
{}, false));
@@ -7105,7 +7160,14 @@ TEST_F(OpenMPIRBuilderTest, ConstantAllocaRaise) {
EXPECT_TRUE(isa<LoadInst>(Load2));
EXPECT_EQ(Load2, Value);
EXPECT_EQ(Load2->getNextNode(), TargetStore);
- auto *Deinit = TargetStore->getNextNode();
+
+ auto *TargetExitBlockBr = TargetStore->getNextNode();
+ EXPECT_TRUE(isa<UncondBrInst>(TargetExitBlockBr));
+
+ auto *TargetExitBlock = TargetExitBlockBr->getSuccessor(0);
+ EXPECT_EQ(TargetExitBlock->getName(), "target.exit");
+
+ Instruction *Deinit = &*TargetExitBlock->getFirstNonPHIIt();
EXPECT_NE(Deinit, nullptr);
auto *DeinitCall = dyn_cast<CallInst>(Deinit);
@@ -7170,7 +7232,8 @@ TEST_F(OpenMPIRBuilderTest, DebugRecordLoc) {
auto CustomMapperCB = [&](unsigned int I) { return nullptr; };
auto BodyGenCB = [&](OpenMPIRBuilder::InsertPointTy AllocaIP,
- OpenMPIRBuilder::InsertPointTy CodeGenIP)
+ OpenMPIRBuilder::InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks)
-> OpenMPIRBuilder::InsertPointTy {
IRBuilderBase::InsertPointGuard guard(Builder);
Builder.SetCurrentDebugLocation(llvm::DebugLoc());
@@ -7236,11 +7299,11 @@ TEST_F(OpenMPIRBuilderTest, DebugRecordLoc) {
ASSERT_EXPECTED_INIT(
OpenMPIRBuilder::InsertPointTy, AfterIP,
- OMPBuilder.createTarget(Loc, /*IsOffloadEntry=*/true, EntryIP, EntryIP,
- Info, EntryInfo, DefaultAttrs, RuntimeAttrs,
- /*IfCond=*/nullptr, CapturedArgs, GenMapInfoCB,
- BodyGenCB, SimpleArgAccessorCB, CustomMapperCB,
- {}, false));
+ OMPBuilder.createTarget(
+ Loc, /*IsOffloadEntry=*/true, EntryIP, EntryIP,
+ /*DeallocBlocks=*/{}, Info, EntryInfo, DefaultAttrs, RuntimeAttrs,
+ /*IfCond=*/nullptr, CapturedArgs, GenMapInfoCB, BodyGenCB,
+ SimpleArgAccessorCB, CustomMapperCB, {}, false));
EXPECT_EQ(DL, Builder.getCurrentDebugLocation());
Builder.restoreIP(AfterIP);
@@ -7283,7 +7346,8 @@ TEST_F(OpenMPIRBuilderTest, CreateTask) {
Value *Val128 =
Builder.CreateLoad(Builder.getInt128Ty(), ValPtr128, "bodygen.load");
- auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+ auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) {
Builder.restoreIP(AllocaIP);
AllocaInst *Local128 = Builder.CreateAlloca(Builder.getInt128Ty(), nullptr,
"bodygen.alloca128");
@@ -7312,7 +7376,7 @@ TEST_F(OpenMPIRBuilderTest, CreateTask) {
OpenMPIRBuilder::InsertPointTy, AfterIP,
OMPBuilder.createTask(
Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()),
- BodyGenCB));
+ /*DeallocBlocks=*/{}, BodyGenCB));
Builder.restoreIP(AfterIP);
OMPBuilder.finalize();
Builder.CreateRetVoid();
@@ -7411,7 +7475,8 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskNoArgs) {
F->setName("func");
IRBuilder<> Builder(BB);
- auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+ auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) {
return Error::success();
};
@@ -7423,7 +7488,7 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskNoArgs) {
OpenMPIRBuilder::InsertPointTy, AfterIP,
OMPBuilder.createTask(
Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()),
- BodyGenCB));
+ /*DeallocBlocks=*/{}, BodyGenCB));
Builder.restoreIP(AfterIP);
OMPBuilder.finalize();
Builder.CreateRetVoid();
@@ -7446,7 +7511,8 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskUntied) {
OMPBuilder.initialize();
F->setName("func");
IRBuilder<> Builder(BB);
- auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+ auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) {
return Error::success();
};
BasicBlock *AllocaBB = Builder.GetInsertBlock();
@@ -7457,7 +7523,7 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskUntied) {
OpenMPIRBuilder::InsertPointTy, AfterIP,
OMPBuilder.createTask(
Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()),
- BodyGenCB,
+ /*DeallocBlocks=*/{}, BodyGenCB,
/*Tied=*/false));
Builder.restoreIP(AfterIP);
OMPBuilder.finalize();
@@ -7482,7 +7548,8 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskDepend) {
OMPBuilder.initialize();
F->setName("func");
IRBuilder<> Builder(BB);
- auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+ auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) {
return Error::success();
};
BasicBlock *AllocaBB = Builder.GetInsertBlock();
@@ -7500,7 +7567,7 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskDepend) {
OpenMPIRBuilder::InsertPointTy, AfterIP,
OMPBuilder.createTask(
Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()),
- BodyGenCB,
+ /*DeallocBlocks=*/{}, BodyGenCB,
/*Tied=*/false, /*Final*/ nullptr, /*IfCondition*/ nullptr,
OpenMPIRBuilder::DependenciesInfo{std::move(DDS)}));
Builder.restoreIP(AfterIP);
@@ -7563,7 +7630,8 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskFinal) {
OMPBuilder.initialize();
F->setName("func");
IRBuilder<> Builder(BB);
- auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+ auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) {
return Error::success();
};
BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split");
@@ -7574,7 +7642,8 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskFinal) {
ConstantInt::get(Type::getInt32Ty(M->getContext()), 0U));
OpenMPIRBuilder::LocationDescription Loc(Builder.saveIP(), DL);
ASSERT_EXPECTED_INIT(OpenMPIRBuilder::InsertPointTy, AfterIP,
- OMPBuilder.createTask(Loc, AllocaIP, BodyGenCB,
+ OMPBuilder.createTask(Loc, AllocaIP,
+ /*DeallocBlocks=*/{}, BodyGenCB,
/*Tied=*/false, Final));
Builder.restoreIP(AfterIP);
OMPBuilder.finalize();
@@ -7621,7 +7690,8 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskIfCondition) {
OMPBuilder.initialize();
F->setName("func");
IRBuilder<> Builder(BB);
- auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+ auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) {
return Error::success();
};
BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split");
@@ -7631,10 +7701,10 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskIfCondition) {
CmpInst::Predicate::ICMP_EQ, F->getArg(0),
ConstantInt::get(Type::getInt32Ty(M->getContext()), 0U));
OpenMPIRBuilder::LocationDescription Loc(Builder.saveIP(), DL);
- ASSERT_EXPECTED_INIT(OpenMPIRBuilder::InsertPointTy, AfterIP,
- OMPBuilder.createTask(Loc, AllocaIP, BodyGenCB,
- /*Tied=*/false, /*Final=*/nullptr,
- IfCondition));
+ ASSERT_EXPECTED_INIT(
+ OpenMPIRBuilder::InsertPointTy, AfterIP,
+ OMPBuilder.createTask(Loc, AllocaIP, /*DeallocBlocks=*/{}, BodyGenCB,
+ /*Tied=*/false, /*Final=*/nullptr, IfCondition));
Builder.restoreIP(AfterIP);
OMPBuilder.finalize();
Builder.CreateRetVoid();
@@ -7692,7 +7762,8 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskAffinity) {
F->setName("func");
IRBuilder<> Builder(BB);
- auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+ auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) {
return Error::success();
};
@@ -7726,7 +7797,7 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskAffinity) {
OpenMPIRBuilder::InsertPointTy, AfterIP,
OMPBuilder.createTask(
Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()),
- BodyGenCB,
+ /*DeallocBlocks=*/{}, BodyGenCB,
/*Tied=*/true,
/*Final=*/nullptr,
/*IfCondition=*/nullptr,
@@ -7878,7 +7949,8 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskgroup) {
Value *InternalStoreInst, *InternalLoad32, *InternalLoad128, *InternalIfCmp;
- auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+ auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) {
Builder.restoreIP(AllocaIP);
AllocaInst *Local128 = Builder.CreateAlloca(Builder.getInt128Ty(), nullptr,
"bodygen.alloca128");
@@ -7907,7 +7979,7 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskgroup) {
ASSERT_EXPECTED_INIT(
OpenMPIRBuilder::InsertPointTy, AfterIP,
OMPBuilder.createTaskgroup(
- Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()),
+ Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()), {},
BodyGenCB));
Builder.restoreIP(AfterIP);
OMPBuilder.finalize();
@@ -7968,14 +8040,16 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskgroupWithTasks) {
F->setName("func");
IRBuilder<> Builder(BB);
- auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+ auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) {
Builder.restoreIP(AllocaIP);
AllocaInst *Alloca32 =
Builder.CreateAlloca(Builder.getInt32Ty(), nullptr, "bodygen.alloca32");
AllocaInst *Alloca64 =
Builder.CreateAlloca(Builder.getInt64Ty(), nullptr, "bodygen.alloca64");
Builder.restoreIP(CodeGenIP);
- auto TaskBodyGenCB1 = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+ auto TaskBodyGenCB1 = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) {
Builder.restoreIP(CodeGenIP);
LoadInst *LoadValue = Builder.CreateLoad(Builder.getInt64Ty(), Alloca64);
Value *AddInst = Builder.CreateAdd(LoadValue, Builder.getInt64(64));
@@ -7983,11 +8057,13 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskgroupWithTasks) {
return Error::success();
};
OpenMPIRBuilder::LocationDescription Loc(Builder.saveIP(), DL);
- ASSERT_EXPECTED_INIT(OpenMPIRBuilder::InsertPointTy, TaskIP1,
- OMPBuilder.createTask(Loc, AllocaIP, TaskBodyGenCB1));
+ ASSERT_EXPECTED_INIT(
+ OpenMPIRBuilder::InsertPointTy, TaskIP1,
+ OMPBuilder.createTask(Loc, AllocaIP, DeallocBlocks, TaskBodyGenCB1));
Builder.restoreIP(TaskIP1);
- auto TaskBodyGenCB2 = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+ auto TaskBodyGenCB2 = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+ ArrayRef<BasicBlock *> DeallocBlocks) {
Builder.restoreIP(CodeGenIP);
LoadInst *LoadValue = Builder.CreateLoad(Builder.getInt32Ty(), Alloca32);
Value *AddInst = Builder.CreateAdd(LoadValue, Builder.getInt32(32));
@@ -7995,8 +8071,9 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskgroupWithTasks) {
return Error::success();
};
OpenMPIRBuilder::LocationDescription Loc2(Builder.saveIP(), DL);
- ASSERT_EXPECTED_INIT(OpenMPIRBuilder::InsertPointTy, TaskIP2,
- OMPBuilder.createTask(Loc2, AllocaIP, TaskBodyGenCB2));
+ ASSERT_EXPECTED_INIT(
+ OpenMPIRBuilder::InsertPointTy, TaskIP2,
+ OMPBuilder.createTask(Loc2, AllocaIP, DeallocBlocks, TaskBodyGenCB2));
Builder.restoreIP(TaskIP2);
};
@@ -8007,7 +8084,7 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskgroupWithTasks) {
ASSERT_EXPECTED_INIT(
OpenMPIRBuilder::InsertPointTy, AfterIP,
OMPBuilder.createTaskgroup(
- Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()),
+ Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()), {},
BODYGENCB_WRAPPER(BodyGenCB)));
Builder.restoreIP(AfterIP);
OMPBuilder.finalize();
diff --git a/llvm/unittests/Transforms/Utils/CodeExtractorTest.cpp b/llvm/unittests/Transforms/Utils/CodeExtractorTest.cpp
index 93b1a070a880a..c299f2ca86745 100644
--- a/llvm/unittests/Transforms/Utils/CodeExtractorTest.cpp
+++ b/llvm/unittests/Transforms/Utils/CodeExtractorTest.cpp
@@ -178,7 +178,7 @@ TEST(CodeExtractor, InputOutputReturnMonitoring) {
CodeExtractor CE(Candidates, /* DT */ nullptr, /* AggregateArgs */ false,
/* BFI */ nullptr, /* BPI */ nullptr, /* AC */ nullptr,
/* AllowVarargs */ false, /* AllowAlloca */ false,
- /* AllocaBlock */ nullptr, /* DeallocationBlock */ nullptr,
+ /* AllocaBlock */ nullptr, /* DeallocationBlocks */ {},
/* Suffix */ "", /* ArgsInZeroAddressSpace */ false,
/* VoidReturnWithSingleOutput */ false);
EXPECT_TRUE(CE.isEligible());
@@ -780,7 +780,7 @@ TEST(CodeExtractor, OpenMPAggregateArgs) {
/* AllowVarArgs */ true,
/* AllowAlloca */ true,
/* AllocationBlock*/ &Func->getEntryBlock(),
- /* DeallocationBlock */ nullptr,
+ /* DeallocationBlocks */ {},
/* Suffix */ ".outlined",
/* ArgsInZeroAddressSpace */ true);
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 88c370252d2bb..06354aa17e323 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -70,14 +70,17 @@ convertToScheduleKind(std::optional<omp::ClauseScheduleKind> schedKind) {
/// ModuleTranslation stack frame for OpenMP operations. This keeps track of the
/// insertion points for allocas.
-class OpenMPAllocaStackFrame
- : public StateStackFrameBase<OpenMPAllocaStackFrame> {
+class OpenMPAllocStackFrame
+ : public StateStackFrameBase<OpenMPAllocStackFrame> {
public:
- MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(OpenMPAllocaStackFrame)
-
- explicit OpenMPAllocaStackFrame(llvm::OpenMPIRBuilder::InsertPointTy allocaIP)
- : allocaInsertPoint(allocaIP) {}
- llvm::OpenMPIRBuilder::InsertPointTy allocaInsertPoint;
+ MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(OpenMPAllocStackFrame)
+
+ explicit OpenMPAllocStackFrame(
+ llvm::OpenMPIRBuilder::InsertPointTy allocaIP,
+ llvm::ArrayRef<llvm::BasicBlock *> deallocBlocks)
+ : allocInsertPoint(allocaIP), deallocBlocks(deallocBlocks) {}
+ llvm::OpenMPIRBuilder::InsertPointTy allocInsertPoint;
+ llvm::SmallVector<llvm::BasicBlock *> deallocBlocks;
};
/// Stack frame to hold a \see llvm::CanonicalLoopInfo representing the
@@ -473,26 +476,32 @@ static LogicalResult handleError(llvm::Expected<T> &result, Operation &op) {
/// Find the insertion point for allocas given the current insertion point for
/// normal operations in the builder.
-static llvm::OpenMPIRBuilder::InsertPointTy
-findAllocaInsertPoint(llvm::IRBuilderBase &builder,
- LLVM::ModuleTranslation &moduleTranslation) {
- // If there is an alloca insertion point on stack, i.e. we are in a nested
+static llvm::OpenMPIRBuilder::InsertPointTy findAllocInsertPoints(
+ llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation,
+ llvm::SmallVectorImpl<llvm::BasicBlock *> *deallocBlocks = nullptr) {
+ // If there is an allocation insertion point on stack, i.e. we are in a nested
// operation and a specific point was provided by some surrounding operation,
// use it.
- llvm::OpenMPIRBuilder::InsertPointTy allocaInsertPoint;
- WalkResult walkResult = moduleTranslation.stackWalk<OpenMPAllocaStackFrame>(
- [&](OpenMPAllocaStackFrame &frame) {
- allocaInsertPoint = frame.allocaInsertPoint;
+ llvm::OpenMPIRBuilder::InsertPointTy allocInsertPoint;
+ llvm::ArrayRef<llvm::BasicBlock *> deallocInsertPoints;
+ WalkResult walkResult = moduleTranslation.stackWalk<OpenMPAllocStackFrame>(
+ [&](OpenMPAllocStackFrame &frame) {
+ allocInsertPoint = frame.allocInsertPoint;
+ deallocInsertPoints = frame.deallocBlocks;
return WalkResult::interrupt();
});
// In cases with multiple levels of outlining, the tree walk might find an
- // alloca insertion point that is inside the original function while the
- // builder insertion point is inside the outlined function. We need to make
- // sure that we do not use it in those cases.
+ // insertion point that is inside the original function while the builder
+ // insertion point is inside the outlined function. We need to make sure that
+ // we do not use it in those cases.
if (walkResult.wasInterrupted() &&
- allocaInsertPoint.getBlock()->getParent() ==
- builder.GetInsertBlock()->getParent())
- return allocaInsertPoint;
+ allocInsertPoint.getBlock()->getParent() ==
+ builder.GetInsertBlock()->getParent()) {
+ if (deallocBlocks)
+ deallocBlocks->insert(deallocBlocks->end(), deallocInsertPoints.begin(),
+ deallocInsertPoints.end());
+ return allocInsertPoint;
+ }
// Otherwise, insert to the entry block of the surrounding function.
// If the current IRBuilder InsertPoint is the function's entry, it cannot
@@ -500,7 +509,7 @@ findAllocaInsertPoint(llvm::IRBuilderBase &builder,
// confusion. Create a new BasicBlock for the Builder and use the entry block
// for the allocs.
// TODO: Create a dedicated alloca BasicBlock at function creation such that
- // we do not need to move the current InertPoint here.
+ // we do not need to move the current InsertPoint here.
if (builder.GetInsertBlock() ==
&builder.GetInsertBlock()->getParent()->getEntryBlock()) {
assert(builder.GetInsertPoint() == builder.GetInsertBlock()->end() &&
@@ -512,6 +521,19 @@ findAllocaInsertPoint(llvm::IRBuilderBase &builder,
builder.SetInsertPoint(entryBB);
}
+ // Collect exit blocks, which is where explicit deallocations should happen in
+ // this case.
+ if (deallocBlocks) {
+ for (llvm::BasicBlock &block : *builder.GetInsertBlock()->getParent()) {
+ // TODO: This currently results in no blocks being added to the list when
+ // all exit blocks of the enclosing function have not been lowered before
+ // this is reached.
+ llvm::Instruction *terminator = block.getTerminatorOrNull();
+ if (isa_and_present<llvm::ReturnInst>(terminator))
+ deallocBlocks->emplace_back(&block);
+ }
+ }
+
llvm::BasicBlock &funcEntryBlock =
builder.GetInsertBlock()->getParent()->getEntryBlock();
return llvm::OpenMPIRBuilder::InsertPointTy(
@@ -685,7 +707,8 @@ convertOmpMasked(Operation &opInst, llvm::IRBuilderBase &builder,
if (failed(checkImplementationStatus(opInst)))
return failure();
- auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) {
+ auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP,
+ llvm::ArrayRef<llvm::BasicBlock *> deallocBlocks) {
// MaskedOp has only one region associated with it.
auto ®ion = maskedOp.getRegion();
builder.restoreIP(codeGenIP);
@@ -729,7 +752,8 @@ convertOmpMaster(Operation &opInst, llvm::IRBuilderBase &builder,
if (failed(checkImplementationStatus(opInst)))
return failure();
- auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) {
+ auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP,
+ llvm::ArrayRef<llvm::BasicBlock *> deallocBlocks) {
// MasterOp has only one region associated with it.
auto ®ion = masterOp.getRegion();
builder.restoreIP(codeGenIP);
@@ -764,7 +788,8 @@ convertOmpCritical(Operation &opInst, llvm::IRBuilderBase &builder,
if (failed(checkImplementationStatus(opInst)))
return failure();
- auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) {
+ auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP,
+ llvm::ArrayRef<llvm::BasicBlock *> deallocBlocks) {
// CriticalOp has only one region associated with it.
auto ®ion = cast<omp::CriticalOp>(opInst).getRegion();
builder.restoreIP(codeGenIP);
@@ -1056,7 +1081,7 @@ convertOmpOrdered(Operation &opInst, llvm::IRBuilderBase &builder,
indexVecValues++;
}
llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
- findAllocaInsertPoint(builder, moduleTranslation);
+ findAllocInsertPoints(builder, moduleTranslation);
llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
builder.restoreIP(moduleTranslation.getOpenMPBuilder()->createOrderedDepend(
ompLoc, allocaIP, numLoops, storeValues, ".cnt.addr", isDependSource));
@@ -1075,7 +1100,8 @@ convertOmpOrderedRegion(Operation &opInst, llvm::IRBuilderBase &builder,
if (failed(checkImplementationStatus(opInst)))
return failure();
- auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) {
+ auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP,
+ llvm::ArrayRef<llvm::BasicBlock *> deallocBlocks) {
// OrderedOp has only one region associated with it.
auto ®ion = cast<omp::OrderedRegionOp>(opInst).getRegion();
builder.restoreIP(codeGenIP);
@@ -1955,7 +1981,7 @@ convertOmpSections(Operation &opInst, llvm::IRBuilderBase &builder,
SmallVector<omp::DeclareReductionOp> reductionDecls;
collectReductionDecls(sectionsOp, reductionDecls);
llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
- findAllocaInsertPoint(builder, moduleTranslation);
+ findAllocInsertPoints(builder, moduleTranslation);
SmallVector<llvm::Value *> privateReductionVariables(
sectionsOp.getNumReductionVars());
@@ -1979,7 +2005,8 @@ convertOmpSections(Operation &opInst, llvm::IRBuilderBase &builder,
Region ®ion = sectionOp.getRegion();
auto sectionCB = [§ionsOp, ®ion, &builder, &moduleTranslation](
- InsertPointTy allocaIP, InsertPointTy codeGenIP) {
+ InsertPointTy allocaIP, InsertPointTy codeGenIP,
+ ArrayRef<llvm::BasicBlock *> deallocBlocks) {
builder.restoreIP(codeGenIP);
// map the omp.section reduction block argument to the omp.sections block
@@ -2024,7 +2051,7 @@ convertOmpSections(Operation &opInst, llvm::IRBuilderBase &builder,
// called for variables which have destructors/finalizers.
auto finiCB = [&](InsertPointTy codeGenIP) { return llvm::Error::success(); };
- allocaIP = findAllocaInsertPoint(builder, moduleTranslation);
+ allocaIP = findAllocInsertPoints(builder, moduleTranslation);
bool isCancellable = constructIsCancellable(sectionsOp);
llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
@@ -2053,7 +2080,8 @@ convertOmpSingle(omp::SingleOp &singleOp, llvm::IRBuilderBase &builder,
if (failed(checkImplementationStatus(*singleOp)))
return failure();
- auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codegenIP) {
+ auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codegenIP,
+ llvm::ArrayRef<llvm::BasicBlock *> deallocBlocks) {
builder.restoreIP(codegenIP);
return convertOmpOpRegions(singleOp.getRegion(), "omp.single.region",
builder, moduleTranslation)
@@ -2139,7 +2167,7 @@ convertOmpTeams(omp::TeamsOp op, llvm::IRBuilderBase &builder,
SmallVector<llvm::Value *> privateReductionVariables(numReductionVars);
llvm::ArrayRef<bool> isByRef;
llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
- findAllocaInsertPoint(builder, moduleTranslation);
+ findAllocInsertPoints(builder, moduleTranslation);
// Only do teams reduction if there is no distribute op that captures the
// reduction instead.
@@ -2161,9 +2189,10 @@ convertOmpTeams(omp::TeamsOp op, llvm::IRBuilderBase &builder,
return failure();
}
- auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codegenIP) {
- LLVM::ModuleTranslation::SaveStack<OpenMPAllocaStackFrame> frame(
- moduleTranslation, allocaIP);
+ auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codegenIP,
+ llvm::ArrayRef<llvm::BasicBlock *> deallocBlocks) {
+ LLVM::ModuleTranslation::SaveStack<OpenMPAllocStackFrame> frame(
+ moduleTranslation, allocaIP, deallocBlocks);
builder.restoreIP(codegenIP);
return convertOmpOpRegions(op.getRegion(), "omp.teams.region", builder,
moduleTranslation)
@@ -2626,7 +2655,7 @@ buildAffinityData(mlir::omp::TaskOp &taskOp, llvm::IRBuilderBase &builder,
auto allocateAffinityList = [&](llvm::Value *count) -> llvm::Value * {
llvm::IRBuilderBase::InsertPointGuard guard(builder);
if (llvm::isa<llvm::Constant>(count) || llvm::isa<llvm::Argument>(count))
- builder.restoreIP(findAllocaInsertPoint(builder, moduleTranslation));
+ builder.restoreIP(findAllocInsertPoints(builder, moduleTranslation));
return builder.CreateAlloca(kmpTaskAffinityInfoTy, count,
"omp.affinity_list");
};
@@ -2832,9 +2861,9 @@ convertOmpTaskOp(omp::TaskOp taskOp, llvm::IRBuilderBase &builder,
// code outside of the outlined task region, which is what we want because
// this way the initialization and copy regions are executed immediately while
// the host variable data are still live.
-
- llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
- findAllocaInsertPoint(builder, moduleTranslation);
+ llvm::SmallVector<llvm::BasicBlock *> deallocBlocks;
+ InsertPointTy allocaIP =
+ findAllocInsertPoints(builder, moduleTranslation, &deallocBlocks);
// Not using splitBB() because that requires the current block to have a
// terminator.
@@ -2864,8 +2893,8 @@ convertOmpTaskOp(omp::TaskOp taskOp, llvm::IRBuilderBase &builder,
// Save the alloca insertion point on ModuleTranslation stack for use in
// nested regions.
- LLVM::ModuleTranslation::SaveStack<OpenMPAllocaStackFrame> frame(
- moduleTranslation, allocaIP);
+ LLVM::ModuleTranslation::SaveStack<OpenMPAllocStackFrame> frame(
+ moduleTranslation, allocaIP, deallocBlocks);
// Allocate and initialize private variables
builder.SetInsertPoint(initBlock->getTerminator());
@@ -2933,12 +2962,13 @@ convertOmpTaskOp(omp::TaskOp taskOp, llvm::IRBuilderBase &builder,
// Set up for call to createTask()
builder.SetInsertPoint(taskStartBlock);
- auto bodyCB = [&](InsertPointTy allocaIP,
- InsertPointTy codegenIP) -> llvm::Error {
+ auto bodyCB =
+ [&](InsertPointTy allocaIP, InsertPointTy codegenIP,
+ llvm::ArrayRef<llvm::BasicBlock *> deallocBlocks) -> llvm::Error {
// Save the alloca insertion point on ModuleTranslation stack for use in
// nested regions.
- LLVM::ModuleTranslation::SaveStack<OpenMPAllocaStackFrame> frame(
- moduleTranslation, allocaIP);
+ LLVM::ModuleTranslation::SaveStack<OpenMPAllocStackFrame> frame(
+ moduleTranslation, allocaIP, deallocBlocks);
// translate the body of the task:
builder.restoreIP(codegenIP);
@@ -3033,7 +3063,7 @@ convertOmpTaskOp(omp::TaskOp taskOp, llvm::IRBuilderBase &builder,
llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
moduleTranslation.getOpenMPBuilder()->createTask(
- ompLoc, allocaIP, bodyCB, !taskOp.getUntied(),
+ ompLoc, allocaIP, deallocBlocks, bodyCB, !taskOp.getUntied(),
moduleTranslation.lookupValue(taskOp.getFinal()),
moduleTranslation.lookupValue(taskOp.getIfExpr()), dependencies, ad,
taskOp.getMergeable(),
@@ -3259,8 +3289,9 @@ convertOmpTaskloopContextOp(omp::TaskloopContextOp contextOp,
TaskContextStructManager taskStructMgr{builder, moduleTranslation,
privateVarsInfo.privatizers};
+ llvm::SmallVector<llvm::BasicBlock *> deallocBlocks;
llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
- findAllocaInsertPoint(builder, moduleTranslation);
+ findAllocInsertPoints(builder, moduleTranslation, &deallocBlocks);
assert(builder.GetInsertPoint() == builder.GetInsertBlock()->end());
llvm::BasicBlock *taskloopStartBlock = llvm::BasicBlock::Create(
@@ -3275,8 +3306,8 @@ convertOmpTaskloopContextOp(omp::TaskloopContextOp contextOp,
llvm::BasicBlock *initBlock =
splitBB(builder, /*CreateBranch=*/true, "omp.private.init");
- LLVM::ModuleTranslation::SaveStack<OpenMPAllocaStackFrame> frame(
- moduleTranslation, allocaIP);
+ LLVM::ModuleTranslation::SaveStack<OpenMPAllocStackFrame> frame(
+ moduleTranslation, allocaIP, deallocBlocks);
// Allocate and initialize private variables
builder.SetInsertPoint(initBlock->getTerminator());
@@ -3338,12 +3369,13 @@ convertOmpTaskloopContextOp(omp::TaskloopContextOp contextOp,
loopOp, builder, moduleTranslation, lbVal, ubVal, stepVal))
return handleError(std::move(err), opInst);
- auto bodyCB = [&](InsertPointTy allocaIP,
- InsertPointTy codegenIP) -> llvm::Error {
+ auto bodyCB =
+ [&](InsertPointTy allocaIP, InsertPointTy codegenIP,
+ llvm::ArrayRef<llvm::BasicBlock *> deallocBlocks) -> llvm::Error {
// Save the alloca insertion point on ModuleTranslation stack for use in
// nested regions.
- LLVM::ModuleTranslation::SaveStack<OpenMPAllocaStackFrame> frame(
- moduleTranslation, allocaIP);
+ LLVM::ModuleTranslation::SaveStack<OpenMPAllocStackFrame> frame(
+ moduleTranslation, allocaIP, deallocBlocks);
// translate the body of the taskloop:
builder.restoreIP(codegenIP);
@@ -3540,9 +3572,10 @@ convertOmpTaskloopContextOp(omp::TaskloopContextOp contextOp,
llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
moduleTranslation.getOpenMPBuilder()->createTaskloop(
- ompLoc, allocaIP, bodyCB, loopInfo, lbVal, ubVal, stepVal,
- contextOp.getUntied(), ifCond, grainsize, contextOp.getNogroup(),
- sched, moduleTranslation.lookupValue(contextOp.getFinal()),
+ ompLoc, allocaIP, deallocBlocks, bodyCB, loopInfo, lbVal, ubVal,
+ stepVal, contextOp.getUntied(), ifCond, grainsize,
+ contextOp.getNogroup(), sched,
+ moduleTranslation.lookupValue(contextOp.getFinal()),
contextOp.getMergeable(),
moduleTranslation.lookupValue(contextOp.getPriority()),
loopOp.getCollapseNumLoops(), taskDupOrNull,
@@ -3565,18 +3598,21 @@ convertOmpTaskgroupOp(omp::TaskgroupOp tgOp, llvm::IRBuilderBase &builder,
if (failed(checkImplementationStatus(*tgOp)))
return failure();
- auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codegenIP) {
+ auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codegenIP,
+ llvm::ArrayRef<llvm::BasicBlock *> deallocBlocks) {
builder.restoreIP(codegenIP);
return convertOmpOpRegions(tgOp.getRegion(), "omp.taskgroup.region",
builder, moduleTranslation)
.takeError();
};
- InsertPointTy allocaIP = findAllocaInsertPoint(builder, moduleTranslation);
+ llvm::SmallVector<llvm::BasicBlock *> deallocBlocks;
+ InsertPointTy allocaIP =
+ findAllocInsertPoints(builder, moduleTranslation, &deallocBlocks);
llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
- moduleTranslation.getOpenMPBuilder()->createTaskgroup(ompLoc, allocaIP,
- bodyCB);
+ moduleTranslation.getOpenMPBuilder()->createTaskgroup(
+ ompLoc, allocaIP, deallocBlocks, bodyCB);
if (failed(handleError(afterIP, *tgOp)))
return failure();
@@ -3639,8 +3675,9 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
SmallVector<omp::DeclareReductionOp> reductionDecls;
collectReductionDecls(wsloopOp, reductionDecls);
+
llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
- findAllocaInsertPoint(builder, moduleTranslation);
+ findAllocInsertPoints(builder, moduleTranslation);
SmallVector<llvm::Value *> privateReductionVariables(
wsloopOp.getNumReductionVars());
@@ -3821,8 +3858,9 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,
opInst.getNumReductionVars());
SmallVector<DeferredStore> deferredStores;
- auto bodyGenCB = [&](InsertPointTy allocaIP,
- InsertPointTy codeGenIP) -> llvm::Error {
+ auto bodyGenCB =
+ [&](InsertPointTy allocaIP, InsertPointTy codeGenIP,
+ llvm::ArrayRef<llvm::BasicBlock *> deallocBlocks) -> llvm::Error {
llvm::Expected<llvm::BasicBlock *> afterAllocas = allocatePrivateVars(
opInst, builder, moduleTranslation, privateVarsInfo, allocaIP);
if (handleError(afterAllocas, *opInst).failed())
@@ -3868,8 +3906,8 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,
// Save the alloca insertion point on ModuleTranslation stack for use in
// nested regions.
- LLVM::ModuleTranslation::SaveStack<OpenMPAllocaStackFrame> frame(
- moduleTranslation, allocaIP);
+ LLVM::ModuleTranslation::SaveStack<OpenMPAllocStackFrame> frame(
+ moduleTranslation, allocaIP, deallocBlocks);
// ParallelOp has only one region associated with it.
llvm::Expected<llvm::BasicBlock *> regionBlock = convertOmpOpRegions(
@@ -3971,13 +4009,15 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,
if (auto bind = opInst.getProcBindKind())
pbKind = getProcBindKind(*bind);
+ llvm::SmallVector<llvm::BasicBlock *> deallocBlocks;
llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
- findAllocaInsertPoint(builder, moduleTranslation);
+ findAllocInsertPoints(builder, moduleTranslation, &deallocBlocks);
llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
- ompBuilder->createParallel(ompLoc, allocaIP, bodyGenCB, privCB, finiCB,
- ifCond, numThreads, pbKind, isCancellable);
+ ompBuilder->createParallel(ompLoc, allocaIP, deallocBlocks, bodyGenCB,
+ privCB, finiCB, ifCond, numThreads, pbKind,
+ isCancellable);
if (failed(handleError(afterIP, *opInst)))
return failure();
@@ -4022,7 +4062,7 @@ convertOmpSimd(Operation &opInst, llvm::IRBuilderBase &builder,
assert(isByRef.size() == simdOp.getNumReductionVars());
llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
- findAllocaInsertPoint(builder, moduleTranslation);
+ findAllocInsertPoints(builder, moduleTranslation);
llvm::Expected<llvm::BasicBlock *> afterAllocas = allocatePrivateVars(
simdOp, builder, moduleTranslation, privateVarsInfo, allocaIP);
@@ -4512,7 +4552,7 @@ convertOmpAtomicRead(Operation &opInst, llvm::IRBuilderBase &builder,
llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();
llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
- findAllocaInsertPoint(builder, moduleTranslation);
+ findAllocInsertPoints(builder, moduleTranslation);
llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
@@ -4539,7 +4579,7 @@ convertOmpAtomicWrite(Operation &opInst, llvm::IRBuilderBase &builder,
llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();
llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
- findAllocaInsertPoint(builder, moduleTranslation);
+ findAllocInsertPoints(builder, moduleTranslation);
llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
llvm::AtomicOrdering ao = convertAtomicOrdering(writeOp.getMemoryOrder());
@@ -4656,7 +4696,7 @@ convertOmpAtomicUpdate(omp::AtomicUpdateOp &opInst,
extractAtomicControlFlags(opInst, isIgnoreDenormalMode, isFineGrainedMemory,
isRemoteMemory);
// Handle ambiguous alloca, if any.
- auto allocaIP = findAllocaInsertPoint(builder, moduleTranslation);
+ auto allocaIP = findAllocInsertPoints(builder, moduleTranslation);
llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
ompBuilder->createAtomicUpdate(ompLoc, allocaIP, llvmAtomicX, llvmExpr,
@@ -4757,7 +4797,7 @@ convertOmpAtomicCapture(omp::AtomicCaptureOp atomicCaptureOp,
extractAtomicControlFlags(atomicUpdateOp, isIgnoreDenormalMode,
isFineGrainedMemory, isRemoteMemory);
// Handle ambiguous alloca, if any.
- auto allocaIP = findAllocaInsertPoint(builder, moduleTranslation);
+ auto allocaIP = findAllocInsertPoints(builder, moduleTranslation);
llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
ompBuilder->createAtomicCapture(
@@ -6012,7 +6052,7 @@ createAlteredByCaptureMap(MapInfoData &mapData,
if (!isPtrTy) {
auto curInsert = builder.saveIP();
llvm::DebugLoc DbgLoc = builder.getCurrentDebugLocation();
- builder.restoreIP(findAllocaInsertPoint(builder, moduleTranslation));
+ builder.restoreIP(findAllocInsertPoints(builder, moduleTranslation));
auto *memTempAlloc =
builder.CreateAlloca(builder.getPtrTy(), nullptr, ".casted");
builder.SetCurrentDebugLocation(DbgLoc);
@@ -6396,18 +6436,19 @@ convertOmpTargetData(Operation *op, llvm::IRBuilderBase &builder,
};
llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
+ llvm::SmallVector<llvm::BasicBlock *> deallocBlocks;
llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
- findAllocaInsertPoint(builder, moduleTranslation);
+ findAllocInsertPoints(builder, moduleTranslation, &deallocBlocks);
llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP = [&]() {
if (isa<omp::TargetDataOp>(op))
return ompBuilder->createTargetData(ompLoc, allocaIP, builder.saveIP(),
- deviceID, ifCond, info, genMapInfoCB,
- customMapperCB,
+ deallocBlocks, deviceID, ifCond, info,
+ genMapInfoCB, customMapperCB,
/*MapperFunc=*/nullptr, bodyGenCB,
/*DeviceAddrCB=*/nullptr);
return ompBuilder->createTargetData(ompLoc, allocaIP, builder.saveIP(),
- deviceID, ifCond, info, genMapInfoCB,
- customMapperCB, &RTLFn);
+ deallocBlocks, deviceID, ifCond, info,
+ genMapInfoCB, customMapperCB, &RTLFn);
}();
if (failed(handleError(afterIP, *op)))
@@ -6443,7 +6484,7 @@ convertOmpDistribute(Operation &opInst, llvm::IRBuilderBase &builder,
collectReductionDecls(teamsOp, reductionDecls);
llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
- findAllocaInsertPoint(builder, moduleTranslation);
+ findAllocInsertPoints(builder, moduleTranslation);
MutableArrayRef<BlockArgument> reductionArgs =
llvm::cast<omp::BlockArgOpenMPOpInterface>(*teamsOp)
@@ -6457,12 +6498,13 @@ convertOmpDistribute(Operation &opInst, llvm::IRBuilderBase &builder,
}
using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
- auto bodyGenCB = [&](InsertPointTy allocaIP,
- InsertPointTy codeGenIP) -> llvm::Error {
+ auto bodyGenCB =
+ [&](InsertPointTy allocaIP, InsertPointTy codeGenIP,
+ llvm::ArrayRef<llvm::BasicBlock *> deallocBlocks) -> llvm::Error {
// Save the alloca insertion point on ModuleTranslation stack for use in
// nested regions.
- LLVM::ModuleTranslation::SaveStack<OpenMPAllocaStackFrame> frame(
- moduleTranslation, allocaIP);
+ LLVM::ModuleTranslation::SaveStack<OpenMPAllocStackFrame> frame(
+ moduleTranslation, allocaIP, deallocBlocks);
// DistributeOp has only one region associated with it.
builder.restoreIP(codeGenIP);
@@ -6531,11 +6573,12 @@ convertOmpDistribute(Operation &opInst, llvm::IRBuilderBase &builder,
return llvm::Error::success();
};
+ llvm::SmallVector<llvm::BasicBlock *> deallocBlocks;
llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
- findAllocaInsertPoint(builder, moduleTranslation);
+ findAllocInsertPoints(builder, moduleTranslation, &deallocBlocks);
llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
- ompBuilder->createDistribute(ompLoc, allocaIP, bodyGenCB);
+ ompBuilder->createDistribute(ompLoc, allocaIP, deallocBlocks, bodyGenCB);
if (failed(handleError(afterIP, opInst)))
return failure();
@@ -7260,7 +7303,8 @@ convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder,
}
using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
- auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP)
+ auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP,
+ ArrayRef<llvm::BasicBlock *> deallocBlocks)
-> llvm::OpenMPIRBuilder::InsertPointOrErrorTy {
llvm::IRBuilderBase::InsertPointGuard guard(builder);
builder.SetCurrentDebugLocation(llvm::DebugLoc());
@@ -7320,33 +7364,21 @@ convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder,
targetOp.getPrivateNeedsBarrier(), &mappedPrivateVars)))
return llvm::make_error<PreviouslyReportedError>();
- SmallVector<Region *> privateCleanupRegions;
- llvm::transform(privateVarsInfo.privatizers,
- std::back_inserter(privateCleanupRegions),
- [](omp::PrivateClauseOp privatizer) {
- return &privatizer.getDeallocRegion();
- });
-
+ LLVM::ModuleTranslation::SaveStack<OpenMPAllocStackFrame> frame(
+ moduleTranslation, allocaIP, deallocBlocks);
llvm::Expected<llvm::BasicBlock *> exitBlock = convertOmpOpRegions(
targetRegion, "omp.target", builder, moduleTranslation);
- if (!exitBlock)
- return exitBlock.takeError();
-
- builder.SetInsertPoint(*exitBlock);
- if (!privateCleanupRegions.empty()) {
- if (failed(inlineOmpRegionCleanup(
- privateCleanupRegions, privateVarsInfo.llvmVars,
- moduleTranslation, builder, "omp.targetop.private.cleanup",
- /*shouldLoadCleanupRegionArg=*/false))) {
- return llvm::createStringError(
- "failed to inline `dealloc` region of `omp.private` "
- "op in the target region");
- }
- return builder.saveIP();
- }
+ if (failed(handleError(exitBlock, *targetOp)))
+ return llvm::make_error<PreviouslyReportedError>();
- return InsertPointTy(exitBlock.get(), exitBlock.get()->end());
+ builder.SetInsertPoint(exitBlock.get()->getTerminator());
+
+ if (failed(cleanupPrivateVars(targetOp, builder, moduleTranslation,
+ targetOp.getLoc(), privateVarsInfo)))
+ return llvm::make_error<PreviouslyReportedError>();
+
+ return builder.saveIP();
};
StringRef parentName = parentFn.getName();
@@ -7443,8 +7475,9 @@ convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder,
kernelInput.push_back(mapData.OriginalValue[i]);
}
+ llvm::SmallVector<llvm::BasicBlock *> deallocBlocks;
llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
- findAllocaInsertPoint(builder, moduleTranslation);
+ findAllocInsertPoints(builder, moduleTranslation, &deallocBlocks);
llvm::OpenMPIRBuilder::DependenciesInfo dds;
if (failed(buildDependData(
@@ -7474,9 +7507,10 @@ convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder,
llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
moduleTranslation.getOpenMPBuilder()->createTarget(
- ompLoc, isOffloadEntry, allocaIP, builder.saveIP(), info, entryInfo,
- defaultAttrs, runtimeAttrs, ifCond, kernelInput, genMapInfoCB, bodyCB,
- argAccessorCB, customMapperCB, dds, targetOp.getNowait());
+ ompLoc, isOffloadEntry, allocaIP, builder.saveIP(), deallocBlocks,
+ info, entryInfo, defaultAttrs, runtimeAttrs, ifCond, kernelInput,
+ genMapInfoCB, bodyCB, argAccessorCB, customMapperCB, dds,
+ targetOp.getNowait());
if (failed(handleError(afterIP, opInst)))
return failure();
diff --git a/mlir/test/Target/LLVMIR/omptarget-parallel-llvm.mlir b/mlir/test/Target/LLVMIR/omptarget-parallel-llvm.mlir
index 8b4b2120006e3..c24f5cf796468 100644
--- a/mlir/test/Target/LLVMIR/omptarget-parallel-llvm.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-parallel-llvm.mlir
@@ -55,21 +55,21 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
// CHECK: define weak_odr protected amdgpu_kernel void @[[FUNC0:.*]](
// CHECK-SAME: ptr %[[TMP0:.*]], ptr %[[TMP:.*]]) #{{[0-9]+}} {
// CHECK: %[[TMP1:.*]] = alloca [1 x ptr], align 8, addrspace(5)
-// CHECK: %[[TMP2:.*]] = addrspacecast ptr addrspace(5) %[[TMP1]] to ptr
-// CHECK: %[[TMP3:.*]] = alloca ptr, align 8, addrspace(5)
-// CHECK: %[[TMP4:.*]] = addrspacecast ptr addrspace(5) %[[TMP3]] to ptr
-// CHECK: store ptr %[[TMP0]], ptr %[[TMP4]], align 8
-// CHECK: %[[TMP5:.*]] = call i32 @__kmpc_target_init(ptr addrspacecast (ptr addrspace(1) @{{.*}} to ptr), ptr %[[TMP]])
-// CHECK: %[[EXEC_USER_CODE:.*]] = icmp eq i32 %[[TMP5]], -1
+// CHECK: %[[TMP2:.*]] = alloca ptr, align 8, addrspace(5)
+// CHECK: %[[TMP3:.*]] = addrspacecast ptr addrspace(5) %[[TMP2]] to ptr
+// CHECK: store ptr %[[TMP0]], ptr %[[TMP3]], align 8
+// CHECK: %[[TMP4:.*]] = call i32 @__kmpc_target_init(ptr addrspacecast (ptr addrspace(1) @{{.*}} to ptr), ptr %[[TMP]])
+// CHECK: %[[EXEC_USER_CODE:.*]] = icmp eq i32 %[[TMP4]], -1
// CHECK: br i1 %[[EXEC_USER_CODE]], label %[[USER_CODE_ENTRY:.*]], label %[[WORKER_EXIT:.*]]
-// CHECK: %[[TMP6:.*]] = load ptr, ptr %[[TMP4]], align 8
+// CHECK: %[[TMP5:.*]] = addrspacecast ptr addrspace(5) %[[TMP1]] to ptr
// CHECK: %[[STRUCTARG:.*]] = call align 8 ptr @__kmpc_alloc_shared(i64 8)
+// CHECK: %[[TMP6:.*]] = load ptr, ptr %[[TMP3]], align 8
// CHECK: %[[OMP_GLOBAL_THREAD_NUM:.*]] = call i32 @__kmpc_global_thread_num(ptr addrspacecast (ptr addrspace(1) @[[GLOB1:[0-9]+]] to ptr))
// CHECK: %[[GEP_:.*]] = getelementptr { ptr }, ptr %[[STRUCTARG]], i32 0, i32 0
// CHECK: store ptr %[[TMP6]], ptr %[[GEP_]], align 8
-// CHECK: %[[TMP7:.*]] = getelementptr inbounds [1 x ptr], ptr %[[TMP2]], i64 0, i64 0
+// CHECK: %[[TMP7:.*]] = getelementptr inbounds [1 x ptr], ptr %[[TMP5]], i64 0, i64 0
// CHECK: store ptr %[[STRUCTARG]], ptr %[[TMP7]], align 8
-// CHECK: call void @__kmpc_parallel_60(ptr addrspacecast (ptr addrspace(1) @[[GLOB1]] to ptr), i32 %[[OMP_GLOBAL_THREAD_NUM]], i32 1, i32 -1, i32 -1, ptr @[[FUNC1:.*]], ptr @[[FUNC1_WRAPPER:.*]], ptr %[[TMP2]], i64 1, i32 0)
+// CHECK: call void @__kmpc_parallel_60(ptr addrspacecast (ptr addrspace(1) @[[GLOB1]] to ptr), i32 %[[OMP_GLOBAL_THREAD_NUM]], i32 1, i32 -1, i32 -1, ptr @[[FUNC1:.*]], ptr @[[FUNC1_WRAPPER:.*]], ptr %[[TMP5]], i64 1, i32 0)
// CHECK: call void @__kmpc_free_shared(ptr %[[STRUCTARG]], i64 8)
// CHECK: call void @__kmpc_target_deinit()
diff --git a/mlir/test/Target/LLVMIR/omptarget-parallel-wsloop.mlir b/mlir/test/Target/LLVMIR/omptarget-parallel-wsloop.mlir
index fbf8ed7f9d6b0..838223b337fd1 100644
--- a/mlir/test/Target/LLVMIR/omptarget-parallel-wsloop.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-parallel-wsloop.mlir
@@ -29,7 +29,7 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
// CHECK: call void @__kmpc_parallel_60(ptr addrspacecast
// CHECK-SAME: (ptr addrspace(1) @[[GLOB:[0-9]+]] to ptr),
// CHECK-SAME: i32 %[[THREAD_NUM:.*]], i32 1, i32 -1, i32 -1,
-// CHECK-SAME: ptr @[[PARALLEL_FUNC:.*]], ptr null, ptr %[[PARALLEL_ARGS:.*]], i64 1, i32 0)
+// CHECK-SAME: ptr @[[PARALLEL_FUNC:.*]], ptr @[[PARALLEL_WRAPPER:.*]], ptr %[[PARALLEL_ARGS:.*]], i64 1, i32 0)
// CHECK: define internal void @[[PARALLEL_FUNC]]
// CHECK-SAME: (ptr noalias noundef %[[TID_ADDR:.*]], ptr noalias noundef %[[ZERO_ADDR:.*]],
@@ -41,6 +41,11 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
// CHECK: define internal void @[[LOOP_BODY_FUNC]](i32 %[[CNT:.*]], ptr %[[LOOP_BODY_ARG_PTR:.*]]) #[[ATTRS2:[0-9]+]] {
+// CHECK: define internal void @[[PARALLEL_WRAPPER]](i16 {{.*}}, i32 {{.*}}) {
+// CHECK-NOT: ret {{.*}}
+// CHECK: call void @[[PARALLEL_FUNC]]({{.*}})
+// CHECK-NEXT: ret void
+
// CHECK: attributes #[[ATTRS2]] = {
// CHECK-SAME: "target-cpu"="gfx90a"
// CHECK-SAME: "target-features"="+gfx9-insts,+wavefrontsize64"
diff --git a/mlir/test/Target/LLVMIR/omptarget-region-device-llvm.mlir b/mlir/test/Target/LLVMIR/omptarget-region-device-llvm.mlir
index 1dad4bcbd7203..76a842e7138f2 100644
--- a/mlir/test/Target/LLVMIR/omptarget-region-device-llvm.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-region-device-llvm.mlir
@@ -56,7 +56,9 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
// CHECK: %[[B:.*]] = load i32, ptr %[[PTR_B]], align 4
// CHECK: %[[C:.*]] = add i32 %[[A]], %[[B]]
// CHECK: store i32 %[[C]], ptr %[[PTR_C]], align 4
-// CHECK: br label %[[LABEL_DEINIT:.*]]
+// CHECK: br label %[[LABEL_TARGET_EXIT:.*]]
+// CHECK: [[LABEL_TARGET_EXIT]]:
+// CHECK-NEXT: br label %[[LABEL_DEINIT:.*]]
// CHECK: [[LABEL_DEINIT]]:
// CHECK-NEXT: call void @__kmpc_target_deinit()
// CHECK-NEXT: ret void
diff --git a/mlir/test/Target/LLVMIR/openmp-target-private-allocatable.mlir b/mlir/test/Target/LLVMIR/openmp-target-private-allocatable.mlir
index a214e4a1e7eea..6a89455a149c3 100644
--- a/mlir/test/Target/LLVMIR/openmp-target-private-allocatable.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-target-private-allocatable.mlir
@@ -70,4 +70,6 @@ llvm.func @_FortranAAssign(!llvm.ptr, !llvm.ptr, !llvm.ptr, i32) -> !llvm.struct
// CHECK: call void @dealloc_foo_1(ptr %[[DESC_TO_DEALLOC]])
// CHECK-NEXT: br label %[[CONT_BLOCK:.*]]
// CHECK: [[CONT_BLOCK]]:
+// CHECK-NEXT: br label %[[EXIT_BLOCK:.*]]
+// CHECK: [[EXIT_BLOCK]]:
// CHECK-NEXT: ret void
More information about the Mlir-commits
mailing list