[clang] [llvm] [mlir] [OpenMP] Migrate GPU Reductions CodeGen from Clang to OMPIRBuilder (PR #80343)
Akash Banerjee via cfe-commits
cfe-commits at lists.llvm.org
Wed Mar 20 10:06:32 PDT 2024
https://github.com/TIFitis updated https://github.com/llvm/llvm-project/pull/80343
>From 72ec9a48e223831501bb4d0183080814d8846154 Mon Sep 17 00:00:00 2001
From: Jan Leyonberg <jan_sjodin at yahoo.com>
Date: Mon, 15 Jan 2024 11:34:09 +0000
Subject: [PATCH 01/15] Jan migration work
---
.../llvm/Frontend/OpenMP/OMPIRBuilder.h | 109 +-
llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp | 1216 ++++++++++++++++-
2 files changed, 1304 insertions(+), 21 deletions(-)
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
index c9ee0c25194c23..e1dba2339338a3 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
@@ -16,6 +16,7 @@
#include "llvm/Analysis/MemorySSAUpdater.h"
#include "llvm/Frontend/OpenMP/OMPConstants.h"
+#include "llvm/Frontend/OpenMP/OMPGridValues.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/Support/Allocator.h"
@@ -99,14 +100,22 @@ class OpenMPIRBuilderConfig {
/// expanded.
std::optional<bool> IsGPU;
- // Flag for specifying if offloading is mandatory.
+ /// Flag for specifying if offloading is mandatory.
std::optional<bool> OpenMPOffloadMandatory;
+ /// Name of the target processor.
+ StringRef TargetCPU;
+ /// String representation of the target processor's features.
+ StringRef TargetFeatures;
+
/// First separator used between the initial two parts of a name.
std::optional<StringRef> FirstSeparator;
/// Separator used between all of the rest consecutive parts of s name
std::optional<StringRef> Separator;
+ // Grid Value for the GPU target
+ std::optional<omp::GV> GridValue;
+
OpenMPIRBuilderConfig();
OpenMPIRBuilderConfig(bool IsTargetDevice, bool IsGPU,
bool OpenMPOffloadMandatory,
@@ -132,6 +141,11 @@ class OpenMPIRBuilderConfig {
return *OpenMPOffloadMandatory;
}
+ omp::GV getGridValue() const {
+ assert(GridValue.has_value() && "GridValue is not set");
+ return *GridValue;
+ }
+
bool hasRequiresFlags() const { return RequiresFlags; }
bool hasRequiresReverseOffload() const;
bool hasRequiresUnifiedAddress() const;
@@ -167,6 +181,7 @@ class OpenMPIRBuilderConfig {
void setOpenMPOffloadMandatory(bool Value) { OpenMPOffloadMandatory = Value; }
void setFirstSeparator(StringRef FS) { FirstSeparator = FS; }
void setSeparator(StringRef S) { Separator = S; }
+ void setGridValue(omp::GV G) { GridValue = G; }
void setHasRequiresReverseOffload(bool Value);
void setHasRequiresUnifiedAddress(bool Value);
@@ -1235,18 +1250,26 @@ class OpenMPIRBuilder {
getTargetEntryUniqueInfo(FileIdentifierInfoCallbackTy CallBack,
StringRef ParentName = "");
+ // using ReductionGenTy =
+ // function_ref<InsertPointTy(InsertPointTy, Value *, Value *, Value *&)>;
+
+ // using AtomicReductionGenTy =
+ // function_ref<InsertPointTy(InsertPointTy, Type *, Value *, Value *)>;
+
+ /// Owning equivalents of OpenMPIRBuilder::(Atomic)ReductionGen that are used
+ /// to
+ /// store lambdas with capture.
/// Functions used to generate reductions. Such functions take two Values
/// representing LHS and RHS of the reduction, respectively, and a reference
/// to the value that is updated to refer to the reduction result.
- using ReductionGenTy =
- function_ref<InsertPointTy(InsertPointTy, Value *, Value *, Value *&)>;
-
+ using ReductionGenTy = std::function<OpenMPIRBuilder::InsertPointTy(
+ OpenMPIRBuilder::InsertPointTy, Value *, Value *, Value *&)>;
/// Functions used to generate atomic reductions. Such functions take two
/// Values representing pointers to LHS and RHS of the reduction, as well as
/// the element type of these pointers. They are expected to atomically
/// update the LHS to the reduced value.
- using AtomicReductionGenTy =
- function_ref<InsertPointTy(InsertPointTy, Type *, Value *, Value *)>;
+ using AtomicReductionGenTy = std::function<OpenMPIRBuilder::InsertPointTy(
+ OpenMPIRBuilder::InsertPointTy, Type *, Value *, Value *)>;
/// Information about an OpenMP reduction.
struct ReductionInfo {
@@ -1256,6 +1279,10 @@ class OpenMPIRBuilder {
: ElementType(ElementType), Variable(Variable),
PrivateVariable(PrivateVariable), ReductionGen(ReductionGen),
AtomicReductionGen(AtomicReductionGen) {}
+ ReductionInfo(Value *PrivateVariable)
+ : ElementType(nullptr), Variable(nullptr),
+ PrivateVariable(PrivateVariable), ReductionGen(),
+ AtomicReductionGen() {}
/// Reduction element type, must match pointee type of variable.
Type *ElementType;
@@ -1278,6 +1305,69 @@ class OpenMPIRBuilder {
AtomicReductionGenTy AtomicReductionGen;
};
+ /// A class that manages the reduction info to facilitate lowering of
+ /// reductions at multiple levels of parallelism. For example handling teams
+ /// and parallel reductions on GPUs
+
+ class ReductionInfoManager {
+ private:
+ SmallVector<ReductionInfo> ReductionInfos;
+ std::optional<InsertPointTy> PrivateVarAllocaIP;
+
+ public:
+ ReductionInfoManager(){};
+ void clear() {
+ ReductionInfos.clear();
+ PrivateVarAllocaIP.reset();
+ }
+
+ Value *
+ allocatePrivateReductionVar(IRBuilderBase &builder,
+ llvm::OpenMPIRBuilder::InsertPointTy &allocaIP,
+ Type *VarType) {
+ llvm::Type *ptrTy = llvm::PointerType::getUnqual(builder.getContext());
+ llvm::Value *var = builder.CreateAlloca(VarType);
+ var->setName("private_redvar");
+ llvm::Value *castVar =
+ builder.CreatePointerBitCastOrAddrSpaceCast(var, ptrTy);
+ ReductionInfos.push_back(ReductionInfo(castVar));
+ return castVar;
+ }
+
+ ReductionInfo getReductionInfo(unsigned Index) {
+ return ReductionInfos[Index];
+ }
+ ReductionInfo setReductionInfo(unsigned Index, ReductionInfo &RI) {
+ return ReductionInfos[Index] = RI;
+ }
+ Value *getPrivateReductionVariable(unsigned Index) {
+ return ReductionInfos[Index].PrivateVariable;
+ }
+ SmallVector<ReductionInfo> &getReductionInfos() { return ReductionInfos; }
+
+ bool hasPrivateVarAllocaIP() { return PrivateVarAllocaIP.has_value(); }
+ InsertPointTy getPrivateVarAllocaIP() {
+ assert(PrivateVarAllocaIP.has_value() && "AllocaIP not set");
+ return *PrivateVarAllocaIP;
+ }
+ void setPrivateVarAllocaIP(InsertPointTy IP) { PrivateVarAllocaIP = IP; }
+ };
+
+ /// \param Loc The location where the reduction was
+ /// encountered. Must be within the associate
+ /// directive and after the last local access to the
+ /// reduction variables.
+ /// \param AllocaIP An insertion point suitable for allocas usable
+ /// in reductions.
+ /// \param ReductionInfos A list of info on each reduction variable.
+ /// \param IsNoWait A flag set if the reduction is marked as nowait.
+ InsertPointTy createReductionsGPU(const LocationDescription &Loc,
+ InsertPointTy AllocaIP,
+ ArrayRef<ReductionInfo> ReductionInfos,
+ bool IsNoWait = false,
+ bool IsTeamsReduction = false,
+ bool HasDistribute = false);
+
// TODO: provide atomic and non-atomic reduction generators for reduction
// operators defined by the OpenMP specification.
@@ -1344,7 +1434,9 @@ class OpenMPIRBuilder {
InsertPointTy createReductions(const LocationDescription &Loc,
InsertPointTy AllocaIP,
ArrayRef<ReductionInfo> ReductionInfos,
- bool IsNoWait = false, bool IsByRef = false);
+ bool IsNoWait = false, bool IsByRef = false,
+ bool IsTeamsReduction = false,
+ bool HasDistribute = false);
///}
@@ -1485,6 +1577,9 @@ class OpenMPIRBuilder {
/// Info manager to keep track of target regions.
OffloadEntriesInfoManager OffloadInfoManager;
+ /// Info manager to keep track of reduction information;
+ ReductionInfoManager RIManager;
+
/// The target triple of the underlying module.
const Triple T;
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 16507a69ea8502..2b48df24e7e734 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -145,10 +145,19 @@ static bool isValidWorkshareLoopScheduleType(OMPScheduleType SchedType) {
}
#endif
-static const omp::GV &getGridValue(const Triple &T, Function *Kernel) {
+Function *GLOBAL_ReductionFunc = nullptr;
+
+static uint64_t getTypeSizeInBytes(Module &M, Type *Type) {
+ return divideCeil(M.getDataLayout().getTypeSizeInBits(Type), 8);
+}
+
+static Value *getTypeSizeInBytesValue(IRBuilder<> &Builder, Module &M,
+ Type *Type) {
+ return Builder.getInt64(getTypeSizeInBytes(M, Type));
+}
+
+static const omp::GV &getGridValue(const Triple &T, StringRef Features) {
if (T.isAMDGPU()) {
- StringRef Features =
- Kernel->getFnAttribute("target-features").getValueAsString();
if (Features.count("+wavefrontsize64"))
return omp::getAMDGPUGridValues<64>();
return omp::getAMDGPUGridValues<32>();
@@ -158,6 +167,17 @@ static const omp::GV &getGridValue(const Triple &T, Function *Kernel) {
llvm_unreachable("No grid value available for this architecture!");
}
+static const omp::GV &getGridValue(const Triple &T, Function *Kernel) {
+ if (T.isAMDGPU()) {
+ StringRef Features =
+ Kernel->getFnAttribute("target-features").getValueAsString();
+ return getGridValue(T, Features);
+ }
+ if (T.isNVPTX())
+ return omp::NVPTXGridValues;
+ llvm_unreachable("No grid value available for this architecture!");
+}
+
/// Determine which scheduling algorithm to use, determined from schedule clause
/// arguments.
static OMPScheduleType
@@ -2096,36 +2116,1182 @@ OpenMPIRBuilder::createSection(const LocationDescription &Loc,
/*IsCancellable*/ true);
}
+static Value *getGPUWarpSize(Module &M, OpenMPIRBuilder &OMPBuilder) {
+ return OMPBuilder.Builder.CreateCall(
+ OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_get_warp_size),
+ {});
+}
+
+static Value *getGPUThreadID(Module &M, OpenMPIRBuilder &OMPBuilder) {
+ return OMPBuilder.Builder.CreateCall(
+ OMPBuilder.getOrCreateRuntimeFunction(
+ M, OMPRTL___kmpc_get_hardware_thread_id_in_block),
+ {});
+}
+
+static Value *getGPUNumThreads(Module &M, OpenMPIRBuilder &OMPBuilder) {
+ const char *LocSize = "__kmpc_get_hardware_num_threads_in_block";
+ llvm::Function *F = M.getFunction(LocSize);
+ if (!F) {
+ LLVMContext &Ctx = M.getContext();
+ Type *I32Type = Type::getInt32Ty(Ctx);
+
+ F = Function::Create(FunctionType::get(I32Type, std::nullopt, false),
+ GlobalVariable::ExternalLinkage, LocSize, M);
+ }
+ return OMPBuilder.Builder.CreateCall(F, std::nullopt, "nvptx_num_threads");
+}
+
+static Value *getNVPTXWarpID(Module &M, OpenMPIRBuilder &OMPIRBuilder) {
+ unsigned LaneIDBits =
+ llvm::Log2_32(OMPIRBuilder.Config.getGridValue().GV_Warp_Size);
+ return OMPIRBuilder.Builder.CreateAShr(getGPUThreadID(M, OMPIRBuilder),
+ LaneIDBits, "nvptx_warp_id");
+}
+
+static Value *getNVPTXLaneID(Module &M, OpenMPIRBuilder &OMPIRBuilder) {
+ unsigned LaneIDBits =
+ llvm::Log2_32(OMPIRBuilder.Config.getGridValue().GV_Warp_Size);
+ assert(LaneIDBits < 32 && "Invalid LaneIDBits size in NVPTX device.");
+ unsigned LaneIDMask = ~0u >> (32u - LaneIDBits);
+ return OMPIRBuilder.Builder.CreateAnd(
+ getGPUThreadID(M, OMPIRBuilder),
+ OMPIRBuilder.Builder.getInt32(LaneIDMask), "nvptx_lane_id");
+}
+
+namespace {
+enum CopyAction : unsigned {
+ // RemoteLaneToThread: Copy over a Reduce list from a remote lane in
+ // the warp using shuffle instructions.
+ RemoteLaneToThread,
+ // ThreadCopy: Make a copy of a Reduce list on the thread's stack.
+ ThreadCopy,
+};
+} // namespace
+
+struct CopyOptionsTy {
+ llvm::Value *RemoteLaneOffset;
+ llvm::Value *ScratchpadIndex;
+ llvm::Value *ScratchpadWidth;
+};
+
+static Value *castValueToType(Module &M, OpenMPIRBuilder &OMPBuilder,
+ Value *From, Type *ToType,
+ OpenMPIRBuilder::InsertPointTy AllocaIP,
+ const OpenMPIRBuilder::LocationDescription &Loc) {
+ IRBuilder<> &Builder = OMPBuilder.Builder;
+ Type *FromType = From->getType();
+ uint64_t FromSize =
+ divideCeil(M.getDataLayout().getTypeSizeInBits(FromType), 8);
+ uint64_t ToSize = divideCeil(M.getDataLayout().getTypeSizeInBits(ToType), 8);
+ assert(FromSize > 0 && "From size must be greater than zero");
+ assert(ToSize > 0 && "From size must be greater than zero");
+ if (FromType == ToType)
+ return From;
+ if (FromSize == ToSize)
+ return Builder.CreateBitCast(From, ToType);
+ if (ToType->isIntegerTy() && FromType->isIntegerTy())
+ // FIXME(JAN): Assuming signed integer here, not sure how to find out
+ // if unsigned
+ return Builder.CreateIntCast(From, ToType, /*isSigned*/ true);
+ OpenMPIRBuilder::InsertPointTy CurIP = Builder.saveIP();
+ Builder.restoreIP(AllocaIP);
+ Value *CastItem = Builder.CreateAlloca(ToType, nullptr, "cast_tmp");
+ Builder.restoreIP(CurIP);
+
+ Value *ValCastItem = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ CastItem, FromType->getPointerTo(), "valcastitem");
+ Builder.CreateStore(From, ValCastItem);
+ return Builder.CreateLoad(ToType, CastItem, "castitemload");
+}
+
+static Value *
+createRuntimeShuffleFunction(Module &M, OpenMPIRBuilder &OMPBuilder,
+ const OpenMPIRBuilder::LocationDescription &Loc,
+ OpenMPIRBuilder::InsertPointTy AllocaIP,
+ Value *Element, Type *ElementType, Value *Offset) {
+ LLVMContext &Ctx = M.getContext();
+ IRBuilder<> &Builder = OMPBuilder.Builder;
+ uint64_t Size =
+ divideCeil(M.getDataLayout().getTypeSizeInBits(ElementType), 8);
+ assert(Size <= 8 && "Unsupported bitwidth in shuffle instruction");
+ Function *ShuffleFunc = OMPBuilder.getOrCreateRuntimeFunctionPtr(
+ Size <= 4 ? RuntimeFunction::OMPRTL___kmpc_shuffle_int32
+ : RuntimeFunction::OMPRTL___kmpc_shuffle_int64);
+ Type *IntType = Builder.getIntNTy(Size <= 4 ? 32 : 64);
+ Value *ElemCast = Builder.CreateCast(Instruction::SExt, Element, IntType);
+ Value *WarpSize = getGPUWarpSize(M, OMPBuilder);
+ Value *WarpSizeCast =
+ Builder.CreateIntCast(WarpSize, Type::getInt16Ty(Ctx), /*isSigned=*/true);
+ Value *ShuffleCall =
+ Builder.CreateCall(ShuffleFunc, {ElemCast, Offset, WarpSizeCast});
+ return castValueToType(M, OMPBuilder, ShuffleCall, IntType, AllocaIP, Loc);
+}
+
+static void shuffleAndStore(Value *SrcAddr, Value *DstAddr, Type *ElementType,
+ llvm::Value *Offset, Type *ReductionArrayTy,
+ const OpenMPIRBuilder::LocationDescription &Loc,
+ Module &M, OpenMPIRBuilder &OMPBuilder,
+ OpenMPIRBuilder::InsertPointTy AllocaIP) {
+ LLVMContext &Ctx = M.getContext();
+ IRBuilder<> &Builder = OMPBuilder.Builder;
+ uint64_t Size =
+ divideCeil(M.getDataLayout().getTypeSizeInBits(ElementType), 8);
+ Type *PtrTy = PointerType::getUnqual(Ctx);
+ Value *ElemPtr = DstAddr;
+ Value *Ptr = SrcAddr;
+ // Value *PtrEnd = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ // Builder.CreateConstGEP1_64(ReductionArrayTy, SrcAddr, 1), PtrTy);
+ for (int IntSize = 8; IntSize >= 1; IntSize /= 2) {
+ if (Size < IntSize)
+ continue;
+ // FIXME(JAN): Check if there is a function to convert from bytes to bits
+ Type *IntTy = Builder.getIntNTy(IntSize * 8);
+ Ptr = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ Ptr, IntTy->getPointerTo(), "ptrcast");
+ ElemPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ ElemPtr, IntTy->getPointerTo(), "elemptrcast");
+
+ // FIXME(JAN): Implement loop to handle larger size
+ assert(((Size / IntSize) <= 1) && "Unsupported IntSize");
+ Value *Val = Builder.CreateLoad(IntTy, Ptr);
+ Value *Res = createRuntimeShuffleFunction(M, OMPBuilder, Loc, AllocaIP, Val,
+ IntTy, Offset);
+ Builder.CreateStore(Res, ElemPtr);
+ Ptr = Builder.CreateConstGEP1_64(ReductionArrayTy, Ptr, 1, "ptrgep");
+ ElemPtr =
+ Builder.CreateConstGEP1_64(ReductionArrayTy, ElemPtr, 1, "elemptrgep");
+ Size = Size % IntSize;
+ }
+}
+
+static void
+emitReductionListCopy(CopyAction Action, Type *ReductionArrayTy,
+ ArrayRef<OpenMPIRBuilder::ReductionInfo> ReductionInfos,
+ Value *SrcBase, Value *DestBase, Module &M,
+ OpenMPIRBuilder &OMPBuilder,
+ const OpenMPIRBuilder::LocationDescription &Loc,
+ OpenMPIRBuilder::InsertPointTy AllocaIP,
+ CopyOptionsTy CopyOptions = {nullptr, nullptr, nullptr}) {
+ LLVMContext &Ctx = M.getContext();
+ IRBuilder<> &Builder = OMPBuilder.Builder;
+ Type *PtrTy = PointerType::getUnqual(Ctx);
+
+ Value *RemoteLaneOffset = CopyOptions.RemoteLaneOffset;
+
+ for (auto En : enumerate(ReductionInfos)) {
+ const OpenMPIRBuilder::ReductionInfo &RI = En.value();
+ Value *SrcElementAddr = nullptr;
+ Value *DestElementAddr = nullptr;
+ Value *DestElementPtrAddr = nullptr;
+ bool ShuffleInElement = false;
+ bool UpdateDestListPtr = false;
+
+ // Step 1.1: Get the address for the src element in the Reduce list.
+ Value *SrcElementPtrAddr = Builder.CreateConstGEP2_64(
+ ReductionArrayTy, SrcBase, 0, En.index(), "srcelementptraddr");
+ SrcElementAddr =
+ Builder.CreateLoad(PtrTy, SrcElementPtrAddr, "srcelementaddr");
+
+ // Step 1.2: Create a temporary to store the element in the destination
+ // Reduce list.
+ DestElementPtrAddr = Builder.CreateInBoundsGEP(
+ ReductionArrayTy, DestBase,
+ {Builder.getInt64(0), Builder.getInt64(En.index())},
+ "destelementptraddr");
+ switch (Action) {
+ case RemoteLaneToThread: {
+ OpenMPIRBuilder::InsertPointTy CurIP = Builder.saveIP();
+ Builder.restoreIP(AllocaIP);
+ DestElementAddr = Builder.CreateAlloca(RI.ElementType, nullptr,
+ ".omp.reduction.element");
+ Builder.restoreIP(CurIP);
+ ShuffleInElement = true;
+ UpdateDestListPtr = true;
+ break;
+ }
+ case ThreadCopy: {
+ DestElementAddr =
+ Builder.CreateLoad(PtrTy, DestElementPtrAddr, "destelementaddr");
+ break;
+ }
+ }
+
+ // FIXME(JAN): Original code in clanguses <Addr>.withElementType(...)
+ // check if this generates any code
+
+ if (ShuffleInElement) {
+ shuffleAndStore(SrcElementAddr, DestElementAddr, RI.ElementType,
+ RemoteLaneOffset, ReductionArrayTy, Loc, M, OMPBuilder,
+ AllocaIP);
+ } else {
+ // FIXME(JAN): Assume Scalar here (TEK_Scalar in Clang)
+ Value *Elem = Builder.CreateLoad(RI.ElementType, SrcElementAddr);
+ Builder.CreateStore(Elem, DestElementAddr);
+ }
+ // Step 3.1: Modify reference in dest Reduce list as needed.
+ // Modifying the reference in Reduce list to point to the newly
+ // created element. The element is live in the current function
+ // scope and that of functions it invokes (i.e., reduce_function).
+ // RemoteReduceData[i] = (void*)&RemoteElem
+ if (UpdateDestListPtr) {
+ Value *CastDestAddr = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ DestElementAddr, PtrTy, "castdestaddr");
+ Builder.CreateStore(CastDestAddr, DestElementPtrAddr);
+ }
+ }
+}
+
+static OpenMPIRBuilder::InsertPointTy getIPAfterInstr(Instruction *I) {
+ BasicBlock::iterator it(I);
+ it++;
+ return OpenMPIRBuilder::InsertPointTy(I->getParent(), it);
+}
+
+static Function *emitShuffleAndReduceFunction(
+ Module &M, const OpenMPIRBuilder::LocationDescription &Loc,
+ ArrayRef<OpenMPIRBuilder::ReductionInfo> ReductionInfos, Function *ReduceFn,
+ OpenMPIRBuilder &OMPBuilder) {
+ IRBuilder<> &Builder = OMPBuilder.Builder;
+
+ LLVMContext &Ctx = M.getContext();
+ Type *VoidTy = Type::getVoidTy(Ctx);
+ Type *PtrTy = PointerType::getUnqual(Ctx);
+ Type *I16Type = Type::getInt16Ty(Ctx);
+ auto FuncTy = FunctionType::get(VoidTy, {PtrTy, I16Type, I16Type, I16Type},
+ /* IsVarArg */ false);
+ Function *SarFunc =
+ Function::Create(FuncTy, GlobalVariable::InternalLinkage,
+ "_omp_reduction_shuffle_and_reduce_func", &M);
+ SarFunc->setDoesNotRecurse();
+
+ // Set arg names
+ Argument *Arg0 = SarFunc->getArg(0);
+ Argument *Arg1 = SarFunc->getArg(1);
+ Argument *Arg2 = SarFunc->getArg(2);
+ Argument *Arg3 = SarFunc->getArg(3);
+ Arg0->setName("reduce_list_arg");
+ Arg1->setName("lane_id_arg");
+ Arg2->setName("remote_lane_offset_arg");
+ Arg3->setName("algo_ver_arg");
+
+ BasicBlock *EntryBlock = BasicBlock::Create(Ctx, "", SarFunc);
+ Builder.SetInsertPoint(EntryBlock);
+
+ Type *Arg0Type = Arg0->getType();
+ Type *ArgNType = Arg1->getType();
+ Type *ArgNPtrType = Arg1->getType()->getPointerTo();
+ Value *ReduceListAlloca =
+ Builder.CreateAlloca(Arg0Type, nullptr, Arg0->getName() + ".addr");
+ Value *LaneIdAlloca =
+ Builder.CreateAlloca(ArgNType, nullptr, Arg1->getName() + ".addr");
+ Value *RemoteLaneOffsetAlloca =
+ Builder.CreateAlloca(ArgNType, nullptr, Arg2->getName() + ".addr");
+ Value *AlgoVerAlloca =
+ Builder.CreateAlloca(ArgNType, nullptr, Arg3->getName() + ".addr");
+ // FIXME(Jan): Compute reduction list array type
+ auto *RedListArrayTy = ArrayType::get(PtrTy, 1);
+ Instruction *RemoteReductionListAlloca = Builder.CreateAlloca(
+ RedListArrayTy, nullptr, ".omp.reduction.remote_reduce_list");
+
+ Value *ReduceListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ ReduceListAlloca, Arg0Type, ReduceListAlloca->getName() + ".acast");
+ Value *LaneIdAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ LaneIdAlloca, ArgNPtrType, LaneIdAlloca->getName() + ".acast");
+ Value *RemoteLaneOffsetAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ RemoteLaneOffsetAlloca, ArgNPtrType,
+ RemoteLaneOffsetAlloca->getName() + ".acast");
+ Value *AlgoVerAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ AlgoVerAlloca, ArgNPtrType, AlgoVerAlloca->getName() + ".acast");
+ Value *RemoteListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ RemoteReductionListAlloca, PtrTy,
+ RemoteReductionListAlloca->getName() + ".acast");
+
+ Builder.CreateStore(Arg0, ReduceListAddrCast);
+ Builder.CreateStore(Arg1, LaneIdAddrCast);
+ Builder.CreateStore(Arg2, RemoteLaneOffsetAddrCast);
+ Builder.CreateStore(Arg3, AlgoVerAddrCast);
+
+ Value *ReduceList =
+ Builder.CreateLoad(Arg0Type, ReduceListAddrCast, "reduce_list");
+ Value *LaneId = Builder.CreateLoad(ArgNType, LaneIdAddrCast, "lane_id");
+ Value *RemoteLaneOffset = Builder.CreateLoad(
+ ArgNType, RemoteLaneOffsetAddrCast, "remote_lane_offset");
+ Value *AlgoVer = Builder.CreateLoad(ArgNType, AlgoVerAddrCast, "algo_ver");
+
+ OpenMPIRBuilder::InsertPointTy AllocaIP =
+ getIPAfterInstr(RemoteReductionListAlloca);
+ emitReductionListCopy(RemoteLaneToThread, RedListArrayTy, ReductionInfos,
+ ReduceList, RemoteListAddrCast, M, OMPBuilder, Loc,
+ AllocaIP, {RemoteLaneOffset, nullptr, nullptr});
+
+ // The actions to be performed on the Remote Reduce list is dependent
+ // on the algorithm version.
+ //
+ // if (AlgoVer==0) || (AlgoVer==1 && (LaneId < Offset)) || (AlgoVer==2 &&
+ // LaneId % 2 == 0 && Offset > 0):
+ // do the reduction value aggregation
+ //
+ // The thread local variable Reduce list is mutated in place to host the
+ // reduced data, which is the aggregated value produced from local and
+ // remote lanes.
+ //
+ // Note that AlgoVer is expected to be a constant integer known at compile
+ // time.
+ // When AlgoVer==0, the first conjunction evaluates to true, making
+ // the entire predicate true during compile time.
+ // When AlgoVer==1, the second conjunction has only the second part to be
+ // evaluated during runtime. Other conjunctions evaluates to false
+ // during compile time.
+ // When AlgoVer==2, the third conjunction has only the second part to be
+ // evaluated during runtime. Other conjunctions evaluates to false
+ // during compile time.
+ Value *CondAlgo0 = Builder.CreateIsNull(AlgoVer);
+ Value *Algo1 = Builder.CreateICmpEQ(AlgoVer, Builder.getInt16(1));
+ Value *LaneComp = Builder.CreateICmpULT(LaneId, RemoteLaneOffset);
+ Value *CondAlgo1 = Builder.CreateAnd(Algo1, LaneComp);
+ Value *Algo2 = Builder.CreateICmpEQ(AlgoVer, Builder.getInt16(2));
+ Value *LaneIdAnd1 = Builder.CreateAnd(LaneId, Builder.getInt16(1));
+ Value *LaneIdComp = Builder.CreateIsNull(LaneIdAnd1);
+ Value *Algo2AndLaneIdComp = Builder.CreateAnd(Algo2, LaneIdComp);
+ Value *RemoteOffsetComp =
+ Builder.CreateICmpSGT(RemoteLaneOffset, Builder.getInt16(0));
+ Value *CondAlgo2 = Builder.CreateAnd(Algo2AndLaneIdComp, RemoteOffsetComp);
+ Value *CA0OrCA1 = Builder.CreateOr(CondAlgo0, CondAlgo1);
+ Value *CondReduce = Builder.CreateOr(CA0OrCA1, CondAlgo2);
+
+ BasicBlock *ThenBB = BasicBlock::Create(Ctx, "then", SarFunc);
+ BasicBlock *ElseBB = BasicBlock::Create(Ctx, "else", SarFunc);
+ BasicBlock *MergeBB = BasicBlock::Create(Ctx, "ifcont", SarFunc);
+
+ Builder.CreateCondBr(CondReduce, ThenBB, ElseBB);
+ Builder.SetInsertPoint(ThenBB);
+ // reduce_function(LocalReduceList, RemoteReduceList)
+ Value *LocalReduceListPtr =
+ Builder.CreatePointerBitCastOrAddrSpaceCast(ReduceList, PtrTy);
+ Value *RemoteReduceListPtr =
+ Builder.CreatePointerBitCastOrAddrSpaceCast(RemoteListAddrCast, PtrTy);
+ Builder.CreateCall(ReduceFn, {LocalReduceListPtr, RemoteReduceListPtr});
+ Builder.CreateBr(MergeBB);
+ Builder.SetInsertPoint(ElseBB);
+ Builder.CreateBr(MergeBB);
+ Builder.SetInsertPoint(MergeBB);
+
+ Value *Algo1_2 = Builder.CreateICmpEQ(AlgoVer, Builder.getInt16(1));
+ Value *LaneIdGtOffset = Builder.CreateICmpUGE(LaneId, RemoteLaneOffset);
+ Value *CondCopy = Builder.CreateAnd(Algo1_2, LaneIdGtOffset);
+
+ BasicBlock *CpyThenBB = BasicBlock::Create(Ctx, "cpy_then", SarFunc);
+ BasicBlock *CpyElseBB = BasicBlock::Create(Ctx, "cpy_else", SarFunc);
+ BasicBlock *CpyMergeBB = BasicBlock::Create(Ctx, "cpy_ifcont", SarFunc);
+
+ Builder.CreateCondBr(CondCopy, CpyThenBB, CpyElseBB);
+
+ Builder.SetInsertPoint(CpyThenBB);
+ emitReductionListCopy(ThreadCopy, RedListArrayTy, ReductionInfos,
+ RemoteListAddrCast, ReduceList, M, OMPBuilder, Loc,
+ AllocaIP);
+ Builder.CreateBr(CpyMergeBB);
+ Builder.SetInsertPoint(CpyElseBB);
+ Builder.CreateBr(CpyMergeBB);
+ Builder.SetInsertPoint(CpyMergeBB);
+ Builder.CreateRetVoid();
+
+ return SarFunc;
+}
+
+static Function *emitInterWarpCopyFunction(
+ Module &M, const OpenMPIRBuilder::LocationDescription &Loc,
+ ArrayRef<OpenMPIRBuilder::ReductionInfo> ReductionInfos,
+ OpenMPIRBuilder &OMPBuilder) {
+ IRBuilder<> &Builder = OMPBuilder.Builder;
+ OpenMPIRBuilder::InsertPointTy OldIP = Builder.saveIP();
+ LLVMContext &Ctx = M.getContext();
+ Type *VoidTy = Type::getVoidTy(Ctx);
+ Type *PtrTy = PointerType::getUnqual(Ctx);
+ Type *I32Type = Type::getInt32Ty(Ctx);
+ auto FuncTy =
+ FunctionType::get(VoidTy, {PtrTy, I32Type}, /* IsVarArg */ false);
+ Function *WcFunc =
+ Function::Create(FuncTy, GlobalVariable::InternalLinkage,
+ "_omp_reduction_inter_warp_copy_func", &M);
+ WcFunc->setDoesNotRecurse();
+
+ // Set arg names
+ Argument *Arg0 = WcFunc->getArg(0);
+ Argument *Arg1 = WcFunc->getArg(1);
+ Arg0->setName("reduce_list");
+ Arg1->setName("num_warps");
+
+ // Ensure data transfer storage
+ unsigned WarpSize = OMPBuilder.Config.getGridValue().GV_Warp_Size;
+ // FIXME(Jan): Not sure about the array type here, but it is I32 in Clang
+ auto *ArrayTy = ArrayType::get(I32Type, WarpSize);
+ StringRef TransferMediumName =
+ "__openmp_nvptx_data_transfer_temporary_storage";
+ GlobalVariable *TransferMedium = M.getGlobalVariable(TransferMediumName);
+ if (!TransferMedium) {
+ unsigned SharedAddressSpace =
+ 3; /* FIXME(Jan): C.getTargetAddressSpace(LangAS::cuda_shared); */
+ TransferMedium = new GlobalVariable(
+ M, ArrayTy, /*isConstant=*/false, GlobalVariable::WeakAnyLinkage,
+ UndefValue::get(ArrayTy), TransferMediumName,
+ /*InsertBefore=*/nullptr, GlobalVariable::NotThreadLocal,
+ SharedAddressSpace);
+ }
+
+ BasicBlock *EntryBlock = BasicBlock::Create(Ctx, "", WcFunc);
+ Builder.SetInsertPoint(EntryBlock);
+
+ Type *Arg0Type = Arg0->getType();
+ Type *Arg1Type = Arg1->getType();
+ Value *ReduceListAlloca =
+ Builder.CreateAlloca(Arg0Type, nullptr, Arg0->getName() + ".addr");
+ Instruction *NumWarpsAlloca =
+ Builder.CreateAlloca(Arg1Type, nullptr, Arg1->getName() + ".addr");
+ Value *ReduceListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ ReduceListAlloca, Arg0Type, ReduceListAlloca->getName() + ".acast");
+ Value *NumWarpsAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ NumWarpsAlloca, Arg1Type->getPointerTo(),
+ NumWarpsAlloca->getName() + ".acast");
+ Builder.CreateStore(Arg0, ReduceListAddrCast);
+ Builder.CreateStore(Arg1, NumWarpsAddrCast);
+
+ // Get GPU Info
+ Value *ThreadID = getGPUThreadID(M, OMPBuilder);
+ Value *LaneID = getNVPTXLaneID(M, OMPBuilder);
+ Value *WarpID = getNVPTXWarpID(M, OMPBuilder);
+
+ Value *ReduceListArg =
+ Builder.CreateLoad(PtrTy, ReduceListAddrCast, "reduce_list_arg");
+
+ for (auto En : enumerate(ReductionInfos)) {
+ const OpenMPIRBuilder::ReductionInfo &RI = En.value();
+ Type *ElementTy = RI.ElementType;
+ unsigned NumTypeBits = M.getDataLayout().getTypeSizeInBits(ElementTy);
+ unsigned RealTySize = divideCeil(NumTypeBits, 8);
+ for (unsigned TySize = 4; TySize > 0 && RealTySize > 0; TySize /= 2) {
+ unsigned NumIters = RealTySize / TySize;
+ if (NumIters == 0)
+ continue;
+ // Type *CopyTy = Builder.getIntNTy(TySize);
+ Type *Int32Ty = Builder.getInt32Ty();
+ Value *Cnt = nullptr;
+ Value *CntAddrAcast = nullptr;
+ BasicBlock *PrecondBB = nullptr;
+ BasicBlock *ExitBB = nullptr;
+
+ if (NumIters > 1) {
+ OpenMPIRBuilder::InsertPointTy CurrIP = Builder.saveIP();
+ Builder.SetInsertPoint(NumWarpsAlloca);
+ Value *CntAddr = Builder.CreateAlloca(Int32Ty, nullptr, ".cnt.addr");
+ Builder.restoreIP(CurrIP);
+ CntAddrAcast = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ CntAddr, PtrTy, CntAddr->getName() + ".acast");
+ Builder.CreateStore(Constant::getNullValue(Int32Ty), CntAddrAcast);
+ PrecondBB = BasicBlock::Create(Ctx, "precond", WcFunc);
+ ExitBB = BasicBlock::Create(Ctx, "exit", WcFunc);
+ BasicBlock *BodyBB = BasicBlock::Create(Ctx, "body", WcFunc);
+ Builder.CreateBr(PrecondBB);
+ Builder.SetInsertPoint(PrecondBB);
+ Cnt = Builder.CreateLoad(Int32Ty, CntAddrAcast, "cnt");
+ Value *Cmp = Builder.CreateICmpULT(Cnt, Builder.getInt32(NumIters));
+ Builder.CreateCondBr(Cmp, BodyBB, ExitBB);
+ Builder.SetInsertPoint(BodyBB);
+ }
+
+ OMPBuilder.createBarrier(
+ OpenMPIRBuilder::LocationDescription(Builder.saveIP(), Loc.DL),
+ omp::Directive::OMPD_unknown,
+ /* ForceSimpleCall */ false,
+ /* CheckCancelFlag */ true);
+ BasicBlock *ThenBB = BasicBlock::Create(Ctx, "then", WcFunc);
+ BasicBlock *ElseBB = BasicBlock::Create(Ctx, "else", WcFunc);
+ BasicBlock *MergeBB = BasicBlock::Create(Ctx, "ifcont", WcFunc);
+
+ // if (lane_id == 0)
+ Value *IsWarpMaster = Builder.CreateIsNull(LaneID, "warp_master");
+ Builder.CreateCondBr(IsWarpMaster, ThenBB, ElseBB);
+
+ // then
+ // Reduce element = LocalReduceList[i]
+ Builder.SetInsertPoint(ThenBB);
+ // FIXME(JAN): Should array type be passed in?
+ auto *RedListArrayTy = ArrayType::get(PtrTy, 1);
+ // FIXME(JAN): maybe it should be 0,0 and not use En.index()
+ Value *ReduceListElementPtrPtr = Builder.CreateConstInBoundsGEP2_64(
+ RedListArrayTy, ReduceListArg, 0, En.index());
+ Value *ReduceListElementPtr = Builder.CreateLoad(
+ PtrTy, ReduceListElementPtrPtr, "reduce_list_element_ptr");
+ if (NumIters > 1)
+ ReduceListElementPtr =
+ Builder.CreateGEP(Int32Ty, ReduceListElementPtr, Cnt);
+
+ Value *TransferElemAddr = Builder.CreateInBoundsGEP(
+ ArrayTy, TransferMedium, {Builder.getInt64(0), WarpID});
+ Value *ReduceListElement = Builder.CreateLoad(
+ I32Type, ReduceListElementPtr, "reduce_list_element");
+ Builder.CreateStore(ReduceListElement, TransferElemAddr,
+ /*IsVolatile*/ true);
+ Builder.CreateBr(MergeBB);
+
+ // else
+ Builder.SetInsertPoint(ElseBB);
+ Builder.CreateBr(MergeBB);
+
+ // endif
+ Builder.SetInsertPoint(MergeBB);
+ OMPBuilder.createBarrier(
+ OpenMPIRBuilder::LocationDescription(Builder.saveIP(), Loc.DL),
+ omp::Directive::OMPD_unknown,
+ /* ForceSimpleCall */ false,
+ /* CheckCancelFlag */ true);
+
+ // Warp 0 copies reduce element from transfer medium
+ BasicBlock *W0ThenBB = BasicBlock::Create(Ctx, "w0then", WcFunc);
+ BasicBlock *W0ElseBB = BasicBlock::Create(Ctx, "w0else", WcFunc);
+ BasicBlock *W0MergeBB = BasicBlock::Create(Ctx, "w0ifcont", WcFunc);
+
+ Value *NumWarpsVal =
+ Builder.CreateLoad(I32Type, NumWarpsAddrCast, "num_warps");
+ Value *IsActiveThread =
+ Builder.CreateICmpULT(ThreadID, NumWarpsVal, "is_active_thread");
+ Builder.CreateCondBr(IsActiveThread, W0ThenBB, W0ElseBB);
+
+ // W0then
+ // SecMEdiumPtr = &medium[tid]
+ Builder.SetInsertPoint(W0ThenBB);
+ Value *SrcMediumPtrVal = Builder.CreateInBoundsGEP(
+ ArrayTy, TransferMedium, {Builder.getInt64(0), ThreadID});
+ // SrcMediumVal = *SrcMediumPtr
+ // TODO(JAN): Bitcast here, but no load? skipping for now
+ Value *TargetElementPtrPtr = Builder.CreateConstInBoundsGEP2_64(
+ RedListArrayTy, ReduceListArg, 0, En.index());
+ Value *TargetElementPtr = Builder.CreateLoad(PtrTy, TargetElementPtrPtr);
+ if (NumIters > 1)
+ TargetElementPtr = Builder.CreateGEP(Int32Ty, TargetElementPtr, Cnt);
+
+ Value *SrcMediumValue =
+ Builder.CreateLoad(I32Type, SrcMediumPtrVal, /*IsVolatile*/ true);
+ Builder.CreateStore(SrcMediumValue, TargetElementPtr);
+ Builder.CreateBr(W0MergeBB);
+
+ // W0else
+ Builder.SetInsertPoint(W0ElseBB);
+ Builder.CreateBr(W0MergeBB);
+
+ // W0endif
+ Builder.SetInsertPoint(W0MergeBB);
+ if (NumIters > 1) {
+ Cnt = Builder.CreateNSWAdd(Cnt, Builder.getInt32(1));
+ Builder.CreateStore(Cnt, CntAddrAcast);
+ Builder.CreateBr(PrecondBB);
+ Builder.SetInsertPoint(ExitBB);
+ }
+ }
+ }
+
+ Builder.CreateRetVoid();
+ Builder.restoreIP(OldIP);
+ return WcFunc;
+}
+
+/// This function emits a helper that copies all the reduction variables from
+/// the team into the provided global buffer for the reduction variables.
+///
+/// void list_to_global_copy_func(void *buffer, int Idx, void *reduce_data)
+/// For all data entries D in reduce_data:
+/// Copy local D to buffer.D[Idx]
+static Function *emitListToGlobalCopyFunction(
+ Module &M, const OpenMPIRBuilder::LocationDescription &Loc,
+ ArrayRef<OpenMPIRBuilder::ReductionInfo> ReductionInfos,
+ OpenMPIRBuilder &OMPBuilder) {
+ IRBuilder<> &Builder = OMPBuilder.Builder;
+ OpenMPIRBuilder::InsertPointTy OldIP = Builder.saveIP();
+ LLVMContext &Ctx = M.getContext();
+ Type *VoidTy = Type::getVoidTy(Ctx);
+ Type *Int32Ty = Builder.getInt32Ty();
+ Type *PtrTy = PointerType::getUnqual(Ctx);
+ auto FuncTy =
+ FunctionType::get(VoidTy, {PtrTy, Int32Ty, PtrTy}, /* IsVarArg */ false);
+ Function *LtGCFunc =
+ Function::Create(FuncTy, GlobalVariable::InternalLinkage,
+ "_omp_reduction_list_to_global_copy_func", &M);
+ LtGCFunc->setDoesNotRecurse();
+
+ BasicBlock *EntryBlock = BasicBlock::Create(Ctx, "", LtGCFunc);
+ Builder.SetInsertPoint(EntryBlock);
+
+ // Set arg names
+ Argument *Arg0 = LtGCFunc->getArg(0);
+ Argument *Arg1 = LtGCFunc->getArg(1);
+ Argument *Arg2 = LtGCFunc->getArg(2);
+ Arg0->setName("buffer_arg");
+ Arg1->setName("idx_arg");
+ Arg2->setName("reduce_list_arg");
+
+ Value *BufferArgAlloca =
+ Builder.CreateAlloca(PtrTy, nullptr, Arg0->getName() + ".addr");
+ Value *IdxArgAlloca =
+ Builder.CreateAlloca(Int32Ty, nullptr, Arg1->getName() + ".addr");
+ Value *ReduceListArgAlloca =
+ Builder.CreateAlloca(PtrTy, nullptr, Arg2->getName() + ".addr");
+ Value *BufferArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ BufferArgAlloca, PtrTy, BufferArgAlloca->getName() + ".acast");
+ Value *IdxArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ IdxArgAlloca, PtrTy, IdxArgAlloca->getName() + ".acast");
+ Value *ReduceListArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ ReduceListArgAlloca, PtrTy, ReduceListArgAlloca->getName() + ".acast");
+ // FIXME(JAN): Assume a single globalized variable for now, this should be
+ // passed in
+ Type *SingleReductionTy = ReductionInfos.begin()->ElementType;
+ Type *TypeArgs[] = {SingleReductionTy};
+ StructType *ReductionsBufferTy =
+ StructType::create(Ctx, TypeArgs, "_globalized_locals_ty");
+
+ Builder.CreateStore(Arg0, BufferArgAddrCast);
+ Builder.CreateStore(Arg1, IdxArgAddrCast);
+ Builder.CreateStore(Arg2, ReduceListArgAddrCast);
+
+ Value *BufferArg = Builder.CreateLoad(PtrTy, BufferArgAddrCast, "buffer");
+ Value *Idxs[] = {
+ Builder.CreateLoad(Builder.getInt32Ty(), IdxArgAddrCast, "idxs")};
+ Value *ReduceListArg =
+ Builder.CreateLoad(PtrTy, ReduceListArgAddrCast, "reduce_list");
+ // FIXME(Jan): Assume TEK_SCALAR
+ for (auto En : enumerate(ReductionInfos)) {
+ const OpenMPIRBuilder::ReductionInfo &RI = En.value();
+ // FIXME(Jan): Compute array type
+ auto *RedListArrayTy = ArrayType::get(PtrTy, 1);
+ Value *TargetElementPtrPtr = Builder.CreateConstInBoundsGEP2_64(
+ RedListArrayTy, ReduceListArg, 0, En.index());
+ Value *TargetElementPtr = Builder.CreateLoad(PtrTy, TargetElementPtrPtr);
+
+ Value *BufferVD =
+ Builder.CreateInBoundsGEP(ReductionsBufferTy, BufferArg, Idxs);
+ Value *GlobValPtr = Builder.CreateConstInBoundsGEP2_32(
+ ReductionsBufferTy, BufferVD, 0, En.index());
+ Value *TargetElement = Builder.CreateLoad(RI.ElementType, TargetElementPtr);
+ Builder.CreateStore(TargetElement, GlobValPtr);
+ }
+
+ Builder.CreateRetVoid();
+ Builder.restoreIP(OldIP);
+ return LtGCFunc;
+}
+
+/// This function emits a helper that copies all the reduction variables from
+/// the team into the provided global buffer for the reduction variables.
+///
+/// void list_to_global_copy_func(void *buffer, int Idx, void *reduce_data)
+/// For all data entries D in reduce_data:
+/// Copy local D to buffer.D[Idx]
+static Function *emitGlobalToListCopyFunction(
+ Module &M, const OpenMPIRBuilder::LocationDescription &Loc,
+ ArrayRef<OpenMPIRBuilder::ReductionInfo> ReductionInfos,
+ OpenMPIRBuilder &OMPBuilder) {
+ IRBuilder<> &Builder = OMPBuilder.Builder;
+ OpenMPIRBuilder::InsertPointTy OldIP = Builder.saveIP();
+ LLVMContext &Ctx = M.getContext();
+ Type *VoidTy = Type::getVoidTy(Ctx);
+ Type *Int32Ty = Builder.getInt32Ty();
+ Type *PtrTy = PointerType::getUnqual(Ctx);
+ auto FuncTy =
+ FunctionType::get(VoidTy, {PtrTy, Int32Ty, PtrTy}, /* IsVarArg */ false);
+ Function *LtGCFunc =
+ Function::Create(FuncTy, GlobalVariable::InternalLinkage,
+ "_omp_reduction_global_to_list_copy_func", &M);
+ LtGCFunc->setDoesNotRecurse();
+
+ BasicBlock *EntryBlock = BasicBlock::Create(Ctx, "", LtGCFunc);
+ Builder.SetInsertPoint(EntryBlock);
+
+ // Set arg names
+ Argument *Arg0 = LtGCFunc->getArg(0);
+ Argument *Arg1 = LtGCFunc->getArg(1);
+ Argument *Arg2 = LtGCFunc->getArg(2);
+ Arg0->setName("buffer_arg");
+ Arg1->setName("idx_arg");
+ Arg2->setName("reduce_list_arg");
+
+ Value *BufferArgAlloca =
+ Builder.CreateAlloca(PtrTy, nullptr, Arg0->getName() + ".addr");
+ Value *IdxArgAlloca =
+ Builder.CreateAlloca(Int32Ty, nullptr, Arg1->getName() + ".addr");
+ Value *ReduceListArgAlloca =
+ Builder.CreateAlloca(PtrTy, nullptr, Arg2->getName() + ".addr");
+ Value *BufferArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ BufferArgAlloca, PtrTy, BufferArgAlloca->getName() + ".acast");
+ Value *IdxArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ IdxArgAlloca, PtrTy, IdxArgAlloca->getName() + ".acast");
+ Value *ReduceListArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ ReduceListArgAlloca, PtrTy, ReduceListArgAlloca->getName() + ".acast");
+ // FIXME(JAN): Assume a single globalized variable for now, this should be
+ // passed in
+ Type *SingleReductionTy = ReductionInfos.begin()->ElementType;
+ Type *TypeArgs[] = {SingleReductionTy};
+ StructType *ReductionsBufferTy =
+ StructType::create(Ctx, TypeArgs, "_globalized_locals_ty");
+
+ Builder.CreateStore(Arg0, BufferArgAddrCast);
+ Builder.CreateStore(Arg1, IdxArgAddrCast);
+ Builder.CreateStore(Arg2, ReduceListArgAddrCast);
+
+ Value *BufferArg = Builder.CreateLoad(PtrTy, BufferArgAddrCast, "buffer");
+ Value *Idxs[] = {
+ Builder.CreateLoad(Builder.getInt32Ty(), IdxArgAddrCast, "idxs")};
+ Value *ReduceListArg =
+ Builder.CreateLoad(PtrTy, ReduceListArgAddrCast, "reduce_list");
+ // FIXME(Jan): Assume TEK_SCALAR
+ for (auto En : enumerate(ReductionInfos)) {
+ const OpenMPIRBuilder::ReductionInfo &RI = En.value();
+ // FIXME(Jan): Compute array type
+ auto *RedListArrayTy = ArrayType::get(PtrTy, 1);
+ Value *TargetElementPtrPtr = Builder.CreateConstInBoundsGEP2_64(
+ RedListArrayTy, ReduceListArg, 0, En.index());
+ Value *TargetElementPtr = Builder.CreateLoad(PtrTy, TargetElementPtrPtr);
+
+ Value *BufferVD =
+ Builder.CreateInBoundsGEP(ReductionsBufferTy, BufferArg, Idxs);
+ Value *GlobValPtr = Builder.CreateConstInBoundsGEP2_32(
+ ReductionsBufferTy, BufferVD, 0, En.index());
+ Value *TargetElement = Builder.CreateLoad(RI.ElementType, GlobValPtr);
+ Builder.CreateStore(TargetElement, TargetElementPtr);
+ }
+
+ Builder.CreateRetVoid();
+ Builder.restoreIP(OldIP);
+ return LtGCFunc;
+}
+
+/// This function emits a helper that reduces all the reduction variables from
+/// the team into the provided global buffer for the reduction variables.
+///
+/// void list_to_global_reduce_func(void *buffer, int Idx, void *reduce_data)
+/// void *GlobPtrs[];
+/// GlobPtrs[0] = (void*)&buffer.D0[Idx];
+/// ...
+/// GlobPtrs[N] = (void*)&buffer.DN[Idx];
+/// reduce_function(GlobPtrs, reduce_data);
+/// Create a function with a unique name and a "void (i8*, i8*)" signature in
+/// the given module and return it.
+static Function *emitListToGlobalReduceFunction(
+ Module &M, const OpenMPIRBuilder::LocationDescription &Loc,
+ ArrayRef<OpenMPIRBuilder::ReductionInfo> ReductionInfos, Function *ReduceFn,
+ OpenMPIRBuilder &OMPBuilder) {
+ IRBuilder<> &Builder = OMPBuilder.Builder;
+ OpenMPIRBuilder::InsertPointTy OldIP = Builder.saveIP();
+ LLVMContext &Ctx = M.getContext();
+ Type *VoidTy = Type::getVoidTy(Ctx);
+ Type *Int32Ty = Builder.getInt32Ty();
+ Type *PtrTy = PointerType::getUnqual(Ctx);
+ auto FuncTy =
+ FunctionType::get(VoidTy, {PtrTy, Int32Ty, PtrTy}, /* IsVarArg */ false);
+ Function *LtGRFunc =
+ Function::Create(FuncTy, GlobalVariable::InternalLinkage,
+ "_omp_reduction_list_to_global_reduce_func", &M);
+ LtGRFunc->setDoesNotRecurse();
+
+ BasicBlock *EntryBlock = BasicBlock::Create(Ctx, "", LtGRFunc);
+ Builder.SetInsertPoint(EntryBlock);
+
+ // Set arg names
+ Argument *Arg0 = LtGRFunc->getArg(0);
+ Argument *Arg1 = LtGRFunc->getArg(1);
+ Argument *Arg2 = LtGRFunc->getArg(2);
+ Arg0->setName("buffer_arg");
+ Arg1->setName("idx_arg");
+ Arg2->setName("reduce_list_arg");
+
+ Value *BufferArgAlloca =
+ Builder.CreateAlloca(PtrTy, nullptr, Arg0->getName() + ".addr");
+ Value *IdxArgAlloca =
+ Builder.CreateAlloca(Int32Ty, nullptr, Arg1->getName() + ".addr");
+ Value *ReduceListArgAlloca =
+ Builder.CreateAlloca(PtrTy, nullptr, Arg2->getName() + ".addr");
+ // FIXME(Jan): Compute array type
+ auto *RedListArrayTy = ArrayType::get(PtrTy, 1);
+ Value *LocalReduceList =
+ Builder.CreateAlloca(RedListArrayTy, nullptr, ".omp.reduction.red_list");
+
+ Value *BufferArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ BufferArgAlloca, PtrTy, BufferArgAlloca->getName() + ".acast");
+ Value *IdxArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ IdxArgAlloca, PtrTy, IdxArgAlloca->getName() + ".acast");
+ Value *ReduceListArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ ReduceListArgAlloca, PtrTy, ReduceListArgAlloca->getName() + ".acast");
+ Value *LocalReduceListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ LocalReduceList, PtrTy, LocalReduceList->getName() + ".acast");
+ // FIXME(JAN): Assume a single globalized variable for now, this should be
+ // passed in
+ Type *SingleReductionTy = ReductionInfos.begin()->ElementType;
+ Type *TypeArgs[] = {SingleReductionTy};
+ StructType *ReductionsBufferTy =
+ StructType::create(Ctx, TypeArgs, "_globalized_locals_ty");
+
+ Builder.CreateStore(Arg0, BufferArgAddrCast);
+ Builder.CreateStore(Arg1, IdxArgAddrCast);
+ Builder.CreateStore(Arg2, ReduceListArgAddrCast);
+
+ Value *BufferArg = Builder.CreateLoad(PtrTy, BufferArgAddrCast, "buffer");
+ Value *Idxs[] = {Builder.CreateLoad(Int32Ty, IdxArgAddrCast, "idxs")};
+ // FIXME(Jan): Assume TEK_SCALAR
+ for (auto En : enumerate(ReductionInfos)) {
+ const OpenMPIRBuilder::ReductionInfo &RI = En.value();
+ Value *TargetElementPtrPtr = Builder.CreateConstInBoundsGEP2_64(
+ RedListArrayTy, LocalReduceListAddrCast, 0, En.index());
+ Value *BufferVD =
+ Builder.CreateInBoundsGEP(ReductionsBufferTy, BufferArg, Idxs);
+ Value *GlobValPtr = Builder.CreateConstInBoundsGEP2_32(
+ ReductionsBufferTy, BufferVD, 0, En.index());
+ Builder.CreateStore(GlobValPtr, TargetElementPtrPtr);
+ }
+
+ Value *ReduceList = Builder.CreateLoad(PtrTy, ReduceListArgAddrCast);
+ Builder.CreateCall(ReduceFn, {LocalReduceListAddrCast, ReduceList});
+ Builder.CreateRetVoid();
+ Builder.restoreIP(OldIP);
+ return LtGRFunc;
+}
+
+/// This function emits a helper that reduces all the reduction variables from
+/// the team into the provided global buffer for the reduction variables.
+///
+/// void list_to_global_reduce_func(void *buffer, int Idx, void *reduce_data)
+/// void *GlobPtrs[];
+/// GlobPtrs[0] = (void*)&buffer.D0[Idx];
+/// ...
+/// GlobPtrs[N] = (void*)&buffer.DN[Idx];
+/// reduce_function(GlobPtrs, reduce_data);
/// Create a function with a unique name and a "void (i8*, i8*)" signature in
/// the given module and return it.
-Function *getFreshReductionFunc(Module &M) {
+static Function *emitGlobalToListReduceFunction(
+ Module &M, const OpenMPIRBuilder::LocationDescription &Loc,
+ ArrayRef<OpenMPIRBuilder::ReductionInfo> ReductionInfos, Function *ReduceFn,
+ OpenMPIRBuilder &OMPBuilder) {
+ IRBuilder<> &Builder = OMPBuilder.Builder;
+ OpenMPIRBuilder::InsertPointTy OldIP = Builder.saveIP();
+ LLVMContext &Ctx = M.getContext();
+ Type *VoidTy = Type::getVoidTy(Ctx);
+ Type *Int32Ty = Builder.getInt32Ty();
+ Type *PtrTy = PointerType::getUnqual(Ctx);
+ auto FuncTy =
+ FunctionType::get(VoidTy, {PtrTy, Int32Ty, PtrTy}, /* IsVarArg */ false);
+ Function *LtGRFunc =
+ Function::Create(FuncTy, GlobalVariable::InternalLinkage,
+ "_omp_reduction_global_to_list_reduce_func", &M);
+ LtGRFunc->setDoesNotRecurse();
+
+ BasicBlock *EntryBlock = BasicBlock::Create(Ctx, "", LtGRFunc);
+ Builder.SetInsertPoint(EntryBlock);
+
+ // Set arg names
+ Argument *Arg0 = LtGRFunc->getArg(0);
+ Argument *Arg1 = LtGRFunc->getArg(1);
+ Argument *Arg2 = LtGRFunc->getArg(2);
+ Arg0->setName("buffer_arg");
+ Arg1->setName("idx_arg");
+ Arg2->setName("reduce_list_arg");
+
+ Value *BufferArgAlloca =
+ Builder.CreateAlloca(PtrTy, nullptr, Arg0->getName() + ".addr");
+ Value *IdxArgAlloca =
+ Builder.CreateAlloca(Int32Ty, nullptr, Arg1->getName() + ".addr");
+ Value *ReduceListArgAlloca =
+ Builder.CreateAlloca(PtrTy, nullptr, Arg2->getName() + ".addr");
+ // FIXME(Jan): Compute array type
+ auto *RedListArrayTy = ArrayType::get(PtrTy, 1);
+ Value *LocalReduceList =
+ Builder.CreateAlloca(RedListArrayTy, nullptr, ".omp.reduction.red_list");
+
+ Value *BufferArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ BufferArgAlloca, PtrTy, BufferArgAlloca->getName() + ".acast");
+ Value *IdxArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ IdxArgAlloca, PtrTy, IdxArgAlloca->getName() + ".acast");
+ Value *ReduceListArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ ReduceListArgAlloca, PtrTy, ReduceListArgAlloca->getName() + ".acast");
+ Value *LocalReduceListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ LocalReduceList, PtrTy, LocalReduceList->getName() + ".acast");
+ // FIXME(JAN): Assume a single globalized variable for now, this should be
+ // passed in
+ Type *SingleReductionTy = ReductionInfos.begin()->ElementType;
+ Type *TypeArgs[] = {SingleReductionTy};
+ StructType *ReductionsBufferTy =
+ StructType::create(Ctx, TypeArgs, "_globalized_locals_ty");
+
+ Builder.CreateStore(Arg0, BufferArgAddrCast);
+ Builder.CreateStore(Arg1, IdxArgAddrCast);
+ Builder.CreateStore(Arg2, ReduceListArgAddrCast);
+
+ Value *BufferArg = Builder.CreateLoad(PtrTy, BufferArgAddrCast, "buffer");
+ Value *Idxs[] = {Builder.CreateLoad(Int32Ty, IdxArgAddrCast, "idxs")};
+ // FIXME(Jan): Assume TEK_SCALAR
+ for (auto En : enumerate(ReductionInfos)) {
+ const OpenMPIRBuilder::ReductionInfo &RI = En.value();
+ Value *TargetElementPtrPtr = Builder.CreateConstInBoundsGEP2_64(
+ RedListArrayTy, LocalReduceListAddrCast, 0, En.index());
+ Value *BufferVD =
+ Builder.CreateInBoundsGEP(ReductionsBufferTy, BufferArg, Idxs);
+ Value *GlobValPtr = Builder.CreateConstInBoundsGEP2_32(
+ ReductionsBufferTy, BufferVD, 0, En.index());
+ Builder.CreateStore(GlobValPtr, TargetElementPtrPtr);
+ }
+
+ Value *ReduceList = Builder.CreateLoad(PtrTy, ReduceListArgAddrCast);
+ Builder.CreateCall(ReduceFn, {ReduceList, LocalReduceListAddrCast});
+ Builder.CreateRetVoid();
+ Builder.restoreIP(OldIP);
+ return LtGRFunc;
+}
+
+static Function *getFreshReductionFunc(Module &M) {
Type *VoidTy = Type::getVoidTy(M.getContext());
Type *Int8PtrTy = PointerType::getUnqual(M.getContext());
auto *FuncTy =
FunctionType::get(VoidTy, {Int8PtrTy, Int8PtrTy}, /* IsVarArg */ false);
return Function::Create(FuncTy, GlobalVariable::InternalLinkage,
- M.getDataLayout().getDefaultGlobalsAddressSpace(),
".omp.reduction.func", &M);
}
-OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductions(
- const LocationDescription &Loc, InsertPointTy AllocaIP,
- ArrayRef<ReductionInfo> ReductionInfos, bool IsNoWait, bool IsByRef) {
- for (const ReductionInfo &RI : ReductionInfos) {
+static void populateReductionFunction(
+ Function *ReductionFunc,
+ ArrayRef<OpenMPIRBuilder::ReductionInfo> ReductionInfos,
+ IRBuilder<> &Builder, bool IsGPU) {
+ Module *Module = ReductionFunc->getParent();
+ BasicBlock *ReductionFuncBlock =
+ BasicBlock::Create(Module->getContext(), "", ReductionFunc);
+ Builder.SetInsertPoint(ReductionFuncBlock);
+ Value *LHSArrayPtr = nullptr;
+ Value *RHSArrayPtr = nullptr;
+ if (IsGPU) {
+ // Need to alloca memory here and deal with the pointers before getting
+ // LHS/RHS pointers out
+ //
+ Argument *Arg0 = ReductionFunc->getArg(0);
+ Argument *Arg1 = ReductionFunc->getArg(1);
+ Type *Arg0Type = Arg0->getType();
+ Type *Arg1Type = Arg1->getType();
+
+ Value *LHSAlloca =
+ Builder.CreateAlloca(Arg0Type, nullptr, Arg0->getName() + ".addr");
+ Value *RHSAlloca =
+ Builder.CreateAlloca(Arg1Type, nullptr, Arg1->getName() + ".addr");
+ Value *LHSAddrCast =
+ Builder.CreatePointerBitCastOrAddrSpaceCast(LHSAlloca, Arg0Type);
+ Value *RHSAddrCast =
+ Builder.CreatePointerBitCastOrAddrSpaceCast(RHSAlloca, Arg1Type);
+ Builder.CreateStore(Arg0, LHSAddrCast);
+ Builder.CreateStore(Arg1, RHSAddrCast);
+ LHSArrayPtr = Builder.CreateLoad(Arg0Type, LHSAddrCast);
+ RHSArrayPtr = Builder.CreateLoad(Arg1Type, RHSAddrCast);
+ } else {
+ LHSArrayPtr = ReductionFunc->getArg(0);
+ RHSArrayPtr = ReductionFunc->getArg(1);
+ }
+
+ unsigned NumReductions = ReductionInfos.size();
+ Type *RedArrayTy = ArrayType::get(Builder.getPtrTy(), NumReductions);
+
+ for (auto En : enumerate(ReductionInfos)) {
+ const OpenMPIRBuilder::ReductionInfo &RI = En.value();
+ Value *LHSI8PtrPtr = Builder.CreateConstInBoundsGEP2_64(
+ RedArrayTy, LHSArrayPtr, 0, En.index());
+ Value *LHSI8Ptr = Builder.CreateLoad(Builder.getPtrTy(), LHSI8PtrPtr);
+ Value *LHSPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ LHSI8Ptr, RI.Variable->getType());
+ Value *LHS = Builder.CreateLoad(RI.ElementType, LHSPtr);
+ Value *RHSI8PtrPtr = Builder.CreateConstInBoundsGEP2_64(
+ RedArrayTy, RHSArrayPtr, 0, En.index());
+ Value *RHSI8Ptr = Builder.CreateLoad(Builder.getPtrTy(), RHSI8PtrPtr);
+ Value *RHSPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ RHSI8Ptr, RI.PrivateVariable->getType());
+ Value *RHS = Builder.CreateLoad(RI.ElementType, RHSPtr);
+ Value *Reduced;
+ Builder.restoreIP(RI.ReductionGen(Builder.saveIP(), LHS, RHS, Reduced));
+ if (!Builder.GetInsertBlock())
+ return;
+ Builder.CreateStore(Reduced, LHSPtr);
+ }
+ Builder.CreateRetVoid();
+}
+
+static void
+checkReductionInfos(ArrayRef<OpenMPIRBuilder::ReductionInfo> ReductionInfos,
+ bool IsGPU) {
+ for (const OpenMPIRBuilder::ReductionInfo &RI : ReductionInfos) {
(void)RI;
assert(RI.Variable && "expected non-null variable");
assert(RI.PrivateVariable && "expected non-null private variable");
assert(RI.ReductionGen && "expected non-null reduction generator callback");
- assert(RI.Variable->getType() == RI.PrivateVariable->getType() &&
- "expected variables and their private equivalents to have the same "
- "type");
+ // JAN: Skip this assertion for GPU, address spaces are present
+ if (!IsGPU) {
+ assert(
+ RI.Variable->getType() == RI.PrivateVariable->getType() &&
+ "expected variables and their private equivalents to have the same "
+ "type");
+ }
assert(RI.Variable->getType()->isPointerTy() &&
"expected variables to be pointers");
}
+}
+
+OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductionsGPU(
+ const LocationDescription &Loc, InsertPointTy AllocaIP,
+ ArrayRef<ReductionInfo> ReductionInfos, bool IsNoWait,
+ bool IsTeamsReduction, bool HasDistribute) {
+ checkReductionInfos(ReductionInfos, /*IsGPU*/ true);
+ LLVMContext &Ctx = M.getContext();
+ if (!updateToLocation(Loc))
+ return InsertPointTy();
+
+ if (ReductionInfos.size() == 0)
+ return Builder.saveIP();
+
+ assert(ReductionInfos.size() == 1 && "More than one reduction variable");
+
+ // Copied code from createReductions
+ BasicBlock *InsertBlock = Loc.IP.getBlock();
+ BasicBlock *ContinuationBlock =
+ InsertBlock->splitBasicBlock(Loc.IP.getPoint(), "reduce.finalize");
+ InsertBlock->getTerminator()->eraseFromParent();
+ Builder.SetInsertPoint(InsertBlock, InsertBlock->end());
+
+ Function *ReductionFunc = nullptr;
+ if (GLOBAL_ReductionFunc) {
+ ReductionFunc = GLOBAL_ReductionFunc;
+ } else {
+ ReductionFunc = getFreshReductionFunc(M);
+ GLOBAL_ReductionFunc = ReductionFunc;
+ InsertPointTy CurIP = Builder.saveIP();
+ populateReductionFunction(ReductionFunc, ReductionInfos, Builder, true);
+ Builder.restoreIP(CurIP);
+ }
+
+ uint32_t SrcLocStrSize;
+ Constant *SrcLocStr = getOrCreateDefaultSrcLocStr(SrcLocStrSize);
+ Value *RTLoc =
+ getOrCreateIdent(SrcLocStr, SrcLocStrSize, llvm::omp::IdentFlag(0), 0);
+
+ // 1. Build a list of reduction variables
+ auto Size = ReductionInfos.size();
+ // FIXME(JAN): skipping variably modified type storage for array size
+ Type *PtrTy = PointerType::getUnqual(Ctx);
+ Type *RedArrayTy = ArrayType::get(PtrTy, Size);
+ InsertPointTy CurIP = Builder.saveIP();
+ Builder.restoreIP(AllocaIP);
+ Value *ReductionListAlloca =
+ Builder.CreateAlloca(RedArrayTy, nullptr, ".omp.reduction.red_list");
+ Value *ReductionList =
+ Builder.CreatePointerBitCastOrAddrSpaceCast(ReductionListAlloca, PtrTy);
+ Builder.restoreIP(CurIP);
+ for (auto En : enumerate(ReductionInfos)) {
+ const ReductionInfo &RI = En.value();
+ Value *ElemPtr = Builder.CreateConstGEP2_64(RedArrayTy, ReductionList, 0,
+ En.index(), "elem_ptr");
+ Value *CastElem =
+ Builder.CreatePointerBitCastOrAddrSpaceCast(RI.PrivateVariable, PtrTy);
+ Builder.CreateStore(CastElem, ElemPtr);
+ }
+ CurIP = Builder.saveIP();
+ Function *SarFunc = emitShuffleAndReduceFunction(M, Loc, ReductionInfos,
+ ReductionFunc, *this);
+ Function *WcFunc = emitInterWarpCopyFunction(M, Loc, ReductionInfos, *this);
+ Builder.restoreIP(CurIP);
+
+ Value *RL =
+ Builder.CreatePointerBitCastOrAddrSpaceCast(ReductionList, PtrTy);
+ Value *ReductionDataSize =
+ getTypeSizeInBytesValue(Builder, M, ReductionInfos.begin()->ElementType);
+
+ Value *Res;
+ if (!IsTeamsReduction) {
+ Value *SarFuncCast =
+ Builder.CreatePointerBitCastOrAddrSpaceCast(SarFunc, PtrTy);
+ Value *WcFuncCast =
+ Builder.CreatePointerBitCastOrAddrSpaceCast(WcFunc, PtrTy);
+ Value *Args[] = {RTLoc, ReductionDataSize, RL, SarFuncCast, WcFuncCast};
+ Function *Pv2Ptr = getOrCreateRuntimeFunctionPtr(
+ RuntimeFunction::OMPRTL___kmpc_nvptx_parallel_reduce_nowait_v2);
+ Res = Builder.CreateCall(Pv2Ptr, Args);
+ } else {
+ CurIP = Builder.saveIP();
+ Function *LtGCFunc =
+ emitListToGlobalCopyFunction(M, Loc, ReductionInfos, *this);
+ Function *LtGRFunc = emitListToGlobalReduceFunction(M, Loc, ReductionInfos,
+ ReductionFunc, *this);
+ Function *GtLCFunc =
+ emitGlobalToListCopyFunction(M, Loc, ReductionInfos, *this);
+ Function *GtLRFunc = emitGlobalToListReduceFunction(M, Loc, ReductionInfos,
+ ReductionFunc, *this);
+ Builder.restoreIP(CurIP);
+
+ Function *RedFixedBuferFn = getOrCreateRuntimeFunctionPtr(
+ RuntimeFunction::OMPRTL___kmpc_reduction_get_fixed_buffer);
+
+ Value *KernelTeamsReductionPtr = Builder.CreateCall(RedFixedBuferFn, {});
+
+ Value *Args3[] = {RTLoc,
+ KernelTeamsReductionPtr,
+ Builder.getInt32(1024),
+ ReductionDataSize,
+ RL,
+ SarFunc,
+ WcFunc,
+ LtGCFunc,
+ LtGRFunc,
+ GtLCFunc,
+ GtLRFunc};
+
+ Function *TeamsReduceFn = getOrCreateRuntimeFunctionPtr(
+ RuntimeFunction::OMPRTL___kmpc_nvptx_teams_reduce_nowait_v2);
+ Res = Builder.CreateCall(TeamsReduceFn, Args3);
+ }
+
+ if (IsTeamsReduction || !HasDistribute) {
+ Function *CurFunc = Builder.GetInsertBlock()->getParent();
+ BasicBlock *ExitBB =
+ BasicBlock::Create(Ctx, ".omp.reduction.done", CurFunc);
+ BasicBlock *ThenBB =
+ BasicBlock::Create(Ctx, ".omp.reduction.then", CurFunc);
+ Value *Cond = Builder.CreateICmpEQ(Res, Builder.getInt32(1));
+ Builder.CreateCondBr(Cond, ThenBB, ExitBB);
+
+ Builder.SetInsertPoint(ThenBB);
+ for (auto En : enumerate(ReductionInfos)) {
+ const ReductionInfo &RI = En.value();
+ Value *InputVal = Builder.CreateLoad(RI.ElementType, RI.Variable);
+ Value *RedVal = Builder.CreateLoad(
+ RI.ElementType, Builder.CreatePointerBitCastOrAddrSpaceCast(
+ RI.PrivateVariable, PtrTy));
+ Value *sum;
+ Builder.restoreIP(
+ RI.ReductionGen(Builder.saveIP(), InputVal, RedVal, sum));
+ Builder.CreateStore(sum, RI.Variable);
+ Builder.CreateBr(ExitBB);
+ }
+ Builder.SetInsertPoint(ExitBB);
+ }
+ Builder.CreateBr(ContinuationBlock);
+ Builder.SetInsertPoint(ContinuationBlock);
+ return Builder.saveIP();
+}
+
+OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductions(
+ const LocationDescription &Loc, InsertPointTy AllocaIP,
+ ArrayRef<ReductionInfo> ReductionInfos, bool IsNoWait, bool IsByRef,
+ bool IsTeamsReduction, bool HasDistribute) {
+ if (Config.isGPU())
+ return createReductionsGPU(Loc, AllocaIP, ReductionInfos, IsNoWait,
+ IsTeamsReduction, HasDistribute);
+
+ checkReductionInfos(ReductionInfos, /*IsGPU*/ false);
if (!updateToLocation(Loc))
return InsertPointTy();
+ if (ReductionInfos.size() == 0)
+ return Builder.saveIP();
+
BasicBlock *InsertBlock = Loc.IP.getBlock();
BasicBlock *ContinuationBlock =
InsertBlock->splitBasicBlock(Loc.IP.getPoint(), "reduce.finalize");
@@ -2275,6 +3441,7 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductions(
}
Builder.CreateRetVoid();
+ populateReductionFunction(ReductionFunc, ReductionInfos, Builder, false);
Builder.SetInsertPoint(ContinuationBlock);
return Builder.saveIP();
}
@@ -2787,10 +3954,24 @@ getKmpcForStaticLoopForType(Type *Ty, OpenMPIRBuilder *OMPBuilder,
static void createTargetLoopWorkshareCall(
OpenMPIRBuilder *OMPBuilder, WorksharingLoopType LoopType,
BasicBlock *InsertBlock, Value *Ident, Value *LoopBodyArg,
- Type *ParallelTaskPtr, Value *TripCount, Function &LoopBodyFn) {
- Type *TripCountTy = TripCount->getType();
+ Type *ParallelTaskPtr, Value *TripCountOrig, Function &LoopBodyFn) {
Module &M = OMPBuilder->M;
IRBuilder<> &Builder = OMPBuilder->Builder;
+ Value *TripCount = TripCountOrig;
+ // FIXME(JAN): The trip count is 1 larger than it should be for GPU, this may
+ // not be the right way to fix it, but this works for now.
+ if (OMPBuilder->Config.isGPU()) {
+ if (LoopType != WorksharingLoopType::DistributeStaticLoop)
+ Builder.restoreIP({InsertBlock, std::prev(InsertBlock->end())});
+ LLVMContext &Ctx = M.getContext();
+ Type *IVTy = TripCountOrig->getType();
+ Type *InternalIVTy = IVTy->getIntegerBitWidth() <= 32
+ ? Type::getInt32Ty(Ctx)
+ : Type::getInt64Ty(Ctx);
+ Constant *One = ConstantInt::get(InternalIVTy, 1);
+ TripCount = Builder.CreateSub(TripCountOrig, One, "modified_trip_count");
+ }
+ Type *TripCountTy = TripCount->getType();
FunctionCallee RTLFn =
getKmpcForStaticLoopForType(TripCountTy, OMPBuilder, LoopType);
SmallVector<Value *, 8> RealArgs;
@@ -4551,6 +5732,9 @@ OpenMPIRBuilder::createTargetInit(const LocationDescription &Loc, bool IsSPMD,
Function *Kernel = Builder.GetInsertBlock()->getParent();
+ // Set the grid value in the config needed for lowering later on
+ Config.setGridValue(getGridValue(T, Config.TargetFeatures));
+
// Manifest the launch configuration in the metadata matching the kernel
// environment.
if (MinTeamsVal > 1 || MaxTeamsVal > 0)
@@ -4803,6 +5987,10 @@ void OpenMPIRBuilder::setOutlinedTargetRegionFunctionAttributes(
OutlinedFn->setVisibility(GlobalValue::ProtectedVisibility);
if (T.isAMDGCN())
OutlinedFn->setCallingConv(CallingConv::AMDGPU_KERNEL);
+ if (!Config.TargetCPU.empty())
+ OutlinedFn->addFnAttr("target-cpu", Config.TargetCPU);
+ if (!Config.TargetFeatures.empty())
+ OutlinedFn->addFnAttr("target-features", Config.TargetFeatures);
}
}
>From 8b44ec2a88011f95355eedc4b17627a7c062b693 Mon Sep 17 00:00:00 2001
From: Akash Banerjee <Akash.Banerjee at amd.com>
Date: Wed, 31 Jan 2024 20:05:04 +0000
Subject: [PATCH 02/15] Clang migrate to using OMPBuilder.
---
clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp | 1374 +-----------
clang/lib/CodeGen/CGOpenMPRuntimeGPU.h | 3 -
...arallel_reduction_codegen_tbaa_PR46146.cpp | 546 ++---
.../OpenMP/nvptx_teams_reduction_codegen.cpp | 24 +-
.../target_teams_generic_loop_codegen.cpp | 12 +-
.../llvm/Frontend/OpenMP/OMPIRBuilder.h | 623 +++++-
llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp | 1880 +++++++++--------
.../OpenMP/OpenMPToLLVMIRTranslation.cpp | 11 +-
8 files changed, 1959 insertions(+), 2514 deletions(-)
diff --git a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
index 299ee1460b3db0..7cddf73306f2d9 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
@@ -501,31 +501,6 @@ class CheckVarsEscapingDeclContext final
};
} // anonymous namespace
-/// Get the id of the warp in the block.
-/// We assume that the warp size is 32, which is always the case
-/// on the NVPTX device, to generate more efficient code.
-static llvm::Value *getNVPTXWarpID(CodeGenFunction &CGF) {
- CGBuilderTy &Bld = CGF.Builder;
- unsigned LaneIDBits =
- llvm::Log2_32(CGF.getTarget().getGridValue().GV_Warp_Size);
- auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime());
- return Bld.CreateAShr(RT.getGPUThreadID(CGF), LaneIDBits, "nvptx_warp_id");
-}
-
-/// Get the id of the current lane in the Warp.
-/// We assume that the warp size is 32, which is always the case
-/// on the NVPTX device, to generate more efficient code.
-static llvm::Value *getNVPTXLaneID(CodeGenFunction &CGF) {
- CGBuilderTy &Bld = CGF.Builder;
- unsigned LaneIDBits =
- llvm::Log2_32(CGF.getTarget().getGridValue().GV_Warp_Size);
- assert(LaneIDBits < 32 && "Invalid LaneIDBits size in NVPTX device.");
- unsigned LaneIDMask = ~0u >> (32u - LaneIDBits);
- auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime());
- return Bld.CreateAnd(RT.getGPUThreadID(CGF), Bld.getInt32(LaneIDMask),
- "nvptx_lane_id");
-}
-
CGOpenMPRuntimeGPU::ExecutionMode
CGOpenMPRuntimeGPU::getExecutionMode() const {
return CurrentExecutionMode;
@@ -1429,1132 +1404,6 @@ static llvm::Value *castValueToType(CodeGenFunction &CGF, llvm::Value *Val,
TBAAAccessInfo());
}
-/// This function creates calls to one of two shuffle functions to copy
-/// variables between lanes in a warp.
-static llvm::Value *createRuntimeShuffleFunction(CodeGenFunction &CGF,
- llvm::Value *Elem,
- QualType ElemType,
- llvm::Value *Offset,
- SourceLocation Loc) {
- CodeGenModule &CGM = CGF.CGM;
- CGBuilderTy &Bld = CGF.Builder;
- CGOpenMPRuntimeGPU &RT =
- *(static_cast<CGOpenMPRuntimeGPU *>(&CGM.getOpenMPRuntime()));
- llvm::OpenMPIRBuilder &OMPBuilder = RT.getOMPBuilder();
-
- CharUnits Size = CGF.getContext().getTypeSizeInChars(ElemType);
- assert(Size.getQuantity() <= 8 &&
- "Unsupported bitwidth in shuffle instruction.");
-
- RuntimeFunction ShuffleFn = Size.getQuantity() <= 4
- ? OMPRTL___kmpc_shuffle_int32
- : OMPRTL___kmpc_shuffle_int64;
-
- // Cast all types to 32- or 64-bit values before calling shuffle routines.
- QualType CastTy = CGF.getContext().getIntTypeForBitwidth(
- Size.getQuantity() <= 4 ? 32 : 64, /*Signed=*/1);
- llvm::Value *ElemCast = castValueToType(CGF, Elem, ElemType, CastTy, Loc);
- llvm::Value *WarpSize =
- Bld.CreateIntCast(RT.getGPUWarpSize(CGF), CGM.Int16Ty, /*isSigned=*/true);
-
- llvm::Value *ShuffledVal = CGF.EmitRuntimeCall(
- OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(), ShuffleFn),
- {ElemCast, Offset, WarpSize});
-
- return castValueToType(CGF, ShuffledVal, CastTy, ElemType, Loc);
-}
-
-static void shuffleAndStore(CodeGenFunction &CGF, Address SrcAddr,
- Address DestAddr, QualType ElemType,
- llvm::Value *Offset, SourceLocation Loc) {
- CGBuilderTy &Bld = CGF.Builder;
-
- CharUnits Size = CGF.getContext().getTypeSizeInChars(ElemType);
- // Create the loop over the big sized data.
- // ptr = (void*)Elem;
- // ptrEnd = (void*) Elem + 1;
- // Step = 8;
- // while (ptr + Step < ptrEnd)
- // shuffle((int64_t)*ptr);
- // Step = 4;
- // while (ptr + Step < ptrEnd)
- // shuffle((int32_t)*ptr);
- // ...
- Address ElemPtr = DestAddr;
- Address Ptr = SrcAddr;
- Address PtrEnd = Bld.CreatePointerBitCastOrAddrSpaceCast(
- Bld.CreateConstGEP(SrcAddr, 1), CGF.VoidPtrTy, CGF.Int8Ty);
- for (int IntSize = 8; IntSize >= 1; IntSize /= 2) {
- if (Size < CharUnits::fromQuantity(IntSize))
- continue;
- QualType IntType = CGF.getContext().getIntTypeForBitwidth(
- CGF.getContext().toBits(CharUnits::fromQuantity(IntSize)),
- /*Signed=*/1);
- llvm::Type *IntTy = CGF.ConvertTypeForMem(IntType);
- Ptr = Bld.CreatePointerBitCastOrAddrSpaceCast(Ptr, IntTy->getPointerTo(),
- IntTy);
- ElemPtr = Bld.CreatePointerBitCastOrAddrSpaceCast(
- ElemPtr, IntTy->getPointerTo(), IntTy);
- if (Size.getQuantity() / IntSize > 1) {
- llvm::BasicBlock *PreCondBB = CGF.createBasicBlock(".shuffle.pre_cond");
- llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".shuffle.then");
- llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".shuffle.exit");
- llvm::BasicBlock *CurrentBB = Bld.GetInsertBlock();
- CGF.EmitBlock(PreCondBB);
- llvm::PHINode *PhiSrc =
- Bld.CreatePHI(Ptr.getType(), /*NumReservedValues=*/2);
- PhiSrc->addIncoming(Ptr.getPointer(), CurrentBB);
- llvm::PHINode *PhiDest =
- Bld.CreatePHI(ElemPtr.getType(), /*NumReservedValues=*/2);
- PhiDest->addIncoming(ElemPtr.getPointer(), CurrentBB);
- Ptr = Address(PhiSrc, Ptr.getElementType(), Ptr.getAlignment());
- ElemPtr =
- Address(PhiDest, ElemPtr.getElementType(), ElemPtr.getAlignment());
- llvm::Value *PtrDiff = Bld.CreatePtrDiff(
- CGF.Int8Ty, PtrEnd.getPointer(),
- Bld.CreatePointerBitCastOrAddrSpaceCast(Ptr.getPointer(),
- CGF.VoidPtrTy));
- Bld.CreateCondBr(Bld.CreateICmpSGT(PtrDiff, Bld.getInt64(IntSize - 1)),
- ThenBB, ExitBB);
- CGF.EmitBlock(ThenBB);
- llvm::Value *Res = createRuntimeShuffleFunction(
- CGF,
- CGF.EmitLoadOfScalar(Ptr, /*Volatile=*/false, IntType, Loc,
- LValueBaseInfo(AlignmentSource::Type),
- TBAAAccessInfo()),
- IntType, Offset, Loc);
- CGF.EmitStoreOfScalar(Res, ElemPtr, /*Volatile=*/false, IntType,
- LValueBaseInfo(AlignmentSource::Type),
- TBAAAccessInfo());
- Address LocalPtr = Bld.CreateConstGEP(Ptr, 1);
- Address LocalElemPtr = Bld.CreateConstGEP(ElemPtr, 1);
- PhiSrc->addIncoming(LocalPtr.getPointer(), ThenBB);
- PhiDest->addIncoming(LocalElemPtr.getPointer(), ThenBB);
- CGF.EmitBranch(PreCondBB);
- CGF.EmitBlock(ExitBB);
- } else {
- llvm::Value *Res = createRuntimeShuffleFunction(
- CGF,
- CGF.EmitLoadOfScalar(Ptr, /*Volatile=*/false, IntType, Loc,
- LValueBaseInfo(AlignmentSource::Type),
- TBAAAccessInfo()),
- IntType, Offset, Loc);
- CGF.EmitStoreOfScalar(Res, ElemPtr, /*Volatile=*/false, IntType,
- LValueBaseInfo(AlignmentSource::Type),
- TBAAAccessInfo());
- Ptr = Bld.CreateConstGEP(Ptr, 1);
- ElemPtr = Bld.CreateConstGEP(ElemPtr, 1);
- }
- Size = Size % IntSize;
- }
-}
-
-namespace {
-enum CopyAction : unsigned {
- // RemoteLaneToThread: Copy over a Reduce list from a remote lane in
- // the warp using shuffle instructions.
- RemoteLaneToThread,
- // ThreadCopy: Make a copy of a Reduce list on the thread's stack.
- ThreadCopy,
-};
-} // namespace
-
-struct CopyOptionsTy {
- llvm::Value *RemoteLaneOffset;
- llvm::Value *ScratchpadIndex;
- llvm::Value *ScratchpadWidth;
-};
-
-/// Emit instructions to copy a Reduce list, which contains partially
-/// aggregated values, in the specified direction.
-static void emitReductionListCopy(
- CopyAction Action, CodeGenFunction &CGF, QualType ReductionArrayTy,
- ArrayRef<const Expr *> Privates, Address SrcBase, Address DestBase,
- CopyOptionsTy CopyOptions = {nullptr, nullptr, nullptr}) {
-
- CodeGenModule &CGM = CGF.CGM;
- ASTContext &C = CGM.getContext();
- CGBuilderTy &Bld = CGF.Builder;
-
- llvm::Value *RemoteLaneOffset = CopyOptions.RemoteLaneOffset;
-
- // Iterates, element-by-element, through the source Reduce list and
- // make a copy.
- unsigned Idx = 0;
- for (const Expr *Private : Privates) {
- Address SrcElementAddr = Address::invalid();
- Address DestElementAddr = Address::invalid();
- Address DestElementPtrAddr = Address::invalid();
- // Should we shuffle in an element from a remote lane?
- bool ShuffleInElement = false;
- // Set to true to update the pointer in the dest Reduce list to a
- // newly created element.
- bool UpdateDestListPtr = false;
- QualType PrivatePtrType = C.getPointerType(Private->getType());
- llvm::Type *PrivateLlvmPtrType = CGF.ConvertType(PrivatePtrType);
-
- switch (Action) {
- case RemoteLaneToThread: {
- // Step 1.1: Get the address for the src element in the Reduce list.
- Address SrcElementPtrAddr = Bld.CreateConstArrayGEP(SrcBase, Idx);
- SrcElementAddr = CGF.EmitLoadOfPointer(
- SrcElementPtrAddr.withElementType(PrivateLlvmPtrType),
- PrivatePtrType->castAs<PointerType>());
-
- // Step 1.2: Create a temporary to store the element in the destination
- // Reduce list.
- DestElementPtrAddr = Bld.CreateConstArrayGEP(DestBase, Idx);
- DestElementAddr =
- CGF.CreateMemTemp(Private->getType(), ".omp.reduction.element");
- ShuffleInElement = true;
- UpdateDestListPtr = true;
- break;
- }
- case ThreadCopy: {
- // Step 1.1: Get the address for the src element in the Reduce list.
- Address SrcElementPtrAddr = Bld.CreateConstArrayGEP(SrcBase, Idx);
- SrcElementAddr = CGF.EmitLoadOfPointer(
- SrcElementPtrAddr.withElementType(PrivateLlvmPtrType),
- PrivatePtrType->castAs<PointerType>());
-
- // Step 1.2: Get the address for dest element. The destination
- // element has already been created on the thread's stack.
- DestElementPtrAddr = Bld.CreateConstArrayGEP(DestBase, Idx);
- DestElementAddr = CGF.EmitLoadOfPointer(
- DestElementPtrAddr.withElementType(PrivateLlvmPtrType),
- PrivatePtrType->castAs<PointerType>());
- break;
- }
- }
-
- // Regardless of src and dest of copy, we emit the load of src
- // element as this is required in all directions
- SrcElementAddr = SrcElementAddr.withElementType(
- CGF.ConvertTypeForMem(Private->getType()));
- DestElementAddr =
- DestElementAddr.withElementType(SrcElementAddr.getElementType());
-
- // Now that all active lanes have read the element in the
- // Reduce list, shuffle over the value from the remote lane.
- if (ShuffleInElement) {
- shuffleAndStore(CGF, SrcElementAddr, DestElementAddr, Private->getType(),
- RemoteLaneOffset, Private->getExprLoc());
- } else {
- switch (CGF.getEvaluationKind(Private->getType())) {
- case TEK_Scalar: {
- llvm::Value *Elem = CGF.EmitLoadOfScalar(
- SrcElementAddr, /*Volatile=*/false, Private->getType(),
- Private->getExprLoc(), LValueBaseInfo(AlignmentSource::Type),
- TBAAAccessInfo());
- // Store the source element value to the dest element address.
- CGF.EmitStoreOfScalar(
- Elem, DestElementAddr, /*Volatile=*/false, Private->getType(),
- LValueBaseInfo(AlignmentSource::Type), TBAAAccessInfo());
- break;
- }
- case TEK_Complex: {
- CodeGenFunction::ComplexPairTy Elem = CGF.EmitLoadOfComplex(
- CGF.MakeAddrLValue(SrcElementAddr, Private->getType()),
- Private->getExprLoc());
- CGF.EmitStoreOfComplex(
- Elem, CGF.MakeAddrLValue(DestElementAddr, Private->getType()),
- /*isInit=*/false);
- break;
- }
- case TEK_Aggregate:
- CGF.EmitAggregateCopy(
- CGF.MakeAddrLValue(DestElementAddr, Private->getType()),
- CGF.MakeAddrLValue(SrcElementAddr, Private->getType()),
- Private->getType(), AggValueSlot::DoesNotOverlap);
- break;
- }
- }
-
- // Step 3.1: Modify reference in dest Reduce list as needed.
- // Modifying the reference in Reduce list to point to the newly
- // created element. The element is live in the current function
- // scope and that of functions it invokes (i.e., reduce_function).
- // RemoteReduceData[i] = (void*)&RemoteElem
- if (UpdateDestListPtr) {
- CGF.EmitStoreOfScalar(Bld.CreatePointerBitCastOrAddrSpaceCast(
- DestElementAddr.getPointer(), CGF.VoidPtrTy),
- DestElementPtrAddr, /*Volatile=*/false,
- C.VoidPtrTy);
- }
-
- ++Idx;
- }
-}
-
-/// This function emits a helper that gathers Reduce lists from the first
-/// lane of every active warp to lanes in the first warp.
-///
-/// void inter_warp_copy_func(void* reduce_data, num_warps)
-/// shared smem[warp_size];
-/// For all data entries D in reduce_data:
-/// sync
-/// If (I am the first lane in each warp)
-/// Copy my local D to smem[warp_id]
-/// sync
-/// if (I am the first warp)
-/// Copy smem[thread_id] to my local D
-static llvm::Value *emitInterWarpCopyFunction(CodeGenModule &CGM,
- ArrayRef<const Expr *> Privates,
- QualType ReductionArrayTy,
- SourceLocation Loc) {
- ASTContext &C = CGM.getContext();
- llvm::Module &M = CGM.getModule();
-
- // ReduceList: thread local Reduce list.
- // At the stage of the computation when this function is called, partially
- // aggregated values reside in the first lane of every active warp.
- ImplicitParamDecl ReduceListArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
- C.VoidPtrTy, ImplicitParamKind::Other);
- // NumWarps: number of warps active in the parallel region. This could
- // be smaller than 32 (max warps in a CTA) for partial block reduction.
- ImplicitParamDecl NumWarpsArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
- C.getIntTypeForBitwidth(32, /* Signed */ true),
- ImplicitParamKind::Other);
- FunctionArgList Args;
- Args.push_back(&ReduceListArg);
- Args.push_back(&NumWarpsArg);
-
- const CGFunctionInfo &CGFI =
- CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
- auto *Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(CGFI),
- llvm::GlobalValue::InternalLinkage,
- "_omp_reduction_inter_warp_copy_func", &M);
- CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI);
- Fn->setDoesNotRecurse();
- CodeGenFunction CGF(CGM);
- CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
-
- CGBuilderTy &Bld = CGF.Builder;
-
- // This array is used as a medium to transfer, one reduce element at a time,
- // the data from the first lane of every warp to lanes in the first warp
- // in order to perform the final step of a reduction in a parallel region
- // (reduction across warps). The array is placed in NVPTX __shared__ memory
- // for reduced latency, as well as to have a distinct copy for concurrently
- // executing target regions. The array is declared with common linkage so
- // as to be shared across compilation units.
- StringRef TransferMediumName =
- "__openmp_nvptx_data_transfer_temporary_storage";
- llvm::GlobalVariable *TransferMedium =
- M.getGlobalVariable(TransferMediumName);
- unsigned WarpSize = CGF.getTarget().getGridValue().GV_Warp_Size;
- if (!TransferMedium) {
- auto *Ty = llvm::ArrayType::get(CGM.Int32Ty, WarpSize);
- unsigned SharedAddressSpace = C.getTargetAddressSpace(LangAS::cuda_shared);
- TransferMedium = new llvm::GlobalVariable(
- M, Ty, /*isConstant=*/false, llvm::GlobalVariable::WeakAnyLinkage,
- llvm::UndefValue::get(Ty), TransferMediumName,
- /*InsertBefore=*/nullptr, llvm::GlobalVariable::NotThreadLocal,
- SharedAddressSpace);
- CGM.addCompilerUsedGlobal(TransferMedium);
- }
-
- auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime());
- // Get the CUDA thread id of the current OpenMP thread on the GPU.
- llvm::Value *ThreadID = RT.getGPUThreadID(CGF);
- // nvptx_lane_id = nvptx_id % warpsize
- llvm::Value *LaneID = getNVPTXLaneID(CGF);
- // nvptx_warp_id = nvptx_id / warpsize
- llvm::Value *WarpID = getNVPTXWarpID(CGF);
-
- Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg);
- llvm::Type *ElemTy = CGF.ConvertTypeForMem(ReductionArrayTy);
- Address LocalReduceList(
- Bld.CreatePointerBitCastOrAddrSpaceCast(
- CGF.EmitLoadOfScalar(
- AddrReduceListArg, /*Volatile=*/false, C.VoidPtrTy, Loc,
- LValueBaseInfo(AlignmentSource::Type), TBAAAccessInfo()),
- ElemTy->getPointerTo()),
- ElemTy, CGF.getPointerAlign());
-
- unsigned Idx = 0;
- for (const Expr *Private : Privates) {
- //
- // Warp master copies reduce element to transfer medium in __shared__
- // memory.
- //
- unsigned RealTySize =
- C.getTypeSizeInChars(Private->getType())
- .alignTo(C.getTypeAlignInChars(Private->getType()))
- .getQuantity();
- for (unsigned TySize = 4; TySize > 0 && RealTySize > 0; TySize /=2) {
- unsigned NumIters = RealTySize / TySize;
- if (NumIters == 0)
- continue;
- QualType CType = C.getIntTypeForBitwidth(
- C.toBits(CharUnits::fromQuantity(TySize)), /*Signed=*/1);
- llvm::Type *CopyType = CGF.ConvertTypeForMem(CType);
- CharUnits Align = CharUnits::fromQuantity(TySize);
- llvm::Value *Cnt = nullptr;
- Address CntAddr = Address::invalid();
- llvm::BasicBlock *PrecondBB = nullptr;
- llvm::BasicBlock *ExitBB = nullptr;
- if (NumIters > 1) {
- CntAddr = CGF.CreateMemTemp(C.IntTy, ".cnt.addr");
- CGF.EmitStoreOfScalar(llvm::Constant::getNullValue(CGM.IntTy), CntAddr,
- /*Volatile=*/false, C.IntTy);
- PrecondBB = CGF.createBasicBlock("precond");
- ExitBB = CGF.createBasicBlock("exit");
- llvm::BasicBlock *BodyBB = CGF.createBasicBlock("body");
- // There is no need to emit line number for unconditional branch.
- (void)ApplyDebugLocation::CreateEmpty(CGF);
- CGF.EmitBlock(PrecondBB);
- Cnt = CGF.EmitLoadOfScalar(CntAddr, /*Volatile=*/false, C.IntTy, Loc);
- llvm::Value *Cmp =
- Bld.CreateICmpULT(Cnt, llvm::ConstantInt::get(CGM.IntTy, NumIters));
- Bld.CreateCondBr(Cmp, BodyBB, ExitBB);
- CGF.EmitBlock(BodyBB);
- }
- // kmpc_barrier.
- CGM.getOpenMPRuntime().emitBarrierCall(CGF, Loc, OMPD_unknown,
- /*EmitChecks=*/false,
- /*ForceSimpleCall=*/true);
- llvm::BasicBlock *ThenBB = CGF.createBasicBlock("then");
- llvm::BasicBlock *ElseBB = CGF.createBasicBlock("else");
- llvm::BasicBlock *MergeBB = CGF.createBasicBlock("ifcont");
-
- // if (lane_id == 0)
- llvm::Value *IsWarpMaster = Bld.CreateIsNull(LaneID, "warp_master");
- Bld.CreateCondBr(IsWarpMaster, ThenBB, ElseBB);
- CGF.EmitBlock(ThenBB);
-
- // Reduce element = LocalReduceList[i]
- Address ElemPtrPtrAddr = Bld.CreateConstArrayGEP(LocalReduceList, Idx);
- llvm::Value *ElemPtrPtr = CGF.EmitLoadOfScalar(
- ElemPtrPtrAddr, /*Volatile=*/false, C.VoidPtrTy, SourceLocation());
- // elemptr = ((CopyType*)(elemptrptr)) + I
- Address ElemPtr(ElemPtrPtr, CopyType, Align);
- if (NumIters > 1)
- ElemPtr = Bld.CreateGEP(ElemPtr, Cnt);
-
- // Get pointer to location in transfer medium.
- // MediumPtr = &medium[warp_id]
- llvm::Value *MediumPtrVal = Bld.CreateInBoundsGEP(
- TransferMedium->getValueType(), TransferMedium,
- {llvm::Constant::getNullValue(CGM.Int64Ty), WarpID});
- // Casting to actual data type.
- // MediumPtr = (CopyType*)MediumPtrAddr;
- Address MediumPtr(MediumPtrVal, CopyType, Align);
-
- // elem = *elemptr
- //*MediumPtr = elem
- llvm::Value *Elem = CGF.EmitLoadOfScalar(
- ElemPtr, /*Volatile=*/false, CType, Loc,
- LValueBaseInfo(AlignmentSource::Type), TBAAAccessInfo());
- // Store the source element value to the dest element address.
- CGF.EmitStoreOfScalar(Elem, MediumPtr, /*Volatile=*/true, CType,
- LValueBaseInfo(AlignmentSource::Type),
- TBAAAccessInfo());
-
- Bld.CreateBr(MergeBB);
-
- CGF.EmitBlock(ElseBB);
- Bld.CreateBr(MergeBB);
-
- CGF.EmitBlock(MergeBB);
-
- // kmpc_barrier.
- CGM.getOpenMPRuntime().emitBarrierCall(CGF, Loc, OMPD_unknown,
- /*EmitChecks=*/false,
- /*ForceSimpleCall=*/true);
-
- //
- // Warp 0 copies reduce element from transfer medium.
- //
- llvm::BasicBlock *W0ThenBB = CGF.createBasicBlock("then");
- llvm::BasicBlock *W0ElseBB = CGF.createBasicBlock("else");
- llvm::BasicBlock *W0MergeBB = CGF.createBasicBlock("ifcont");
-
- Address AddrNumWarpsArg = CGF.GetAddrOfLocalVar(&NumWarpsArg);
- llvm::Value *NumWarpsVal = CGF.EmitLoadOfScalar(
- AddrNumWarpsArg, /*Volatile=*/false, C.IntTy, Loc);
-
- // Up to 32 threads in warp 0 are active.
- llvm::Value *IsActiveThread =
- Bld.CreateICmpULT(ThreadID, NumWarpsVal, "is_active_thread");
- Bld.CreateCondBr(IsActiveThread, W0ThenBB, W0ElseBB);
-
- CGF.EmitBlock(W0ThenBB);
-
- // SrcMediumPtr = &medium[tid]
- llvm::Value *SrcMediumPtrVal = Bld.CreateInBoundsGEP(
- TransferMedium->getValueType(), TransferMedium,
- {llvm::Constant::getNullValue(CGM.Int64Ty), ThreadID});
- // SrcMediumVal = *SrcMediumPtr;
- Address SrcMediumPtr(SrcMediumPtrVal, CopyType, Align);
-
- // TargetElemPtr = (CopyType*)(SrcDataAddr[i]) + I
- Address TargetElemPtrPtr = Bld.CreateConstArrayGEP(LocalReduceList, Idx);
- llvm::Value *TargetElemPtrVal = CGF.EmitLoadOfScalar(
- TargetElemPtrPtr, /*Volatile=*/false, C.VoidPtrTy, Loc);
- Address TargetElemPtr(TargetElemPtrVal, CopyType, Align);
- if (NumIters > 1)
- TargetElemPtr = Bld.CreateGEP(TargetElemPtr, Cnt);
-
- // *TargetElemPtr = SrcMediumVal;
- llvm::Value *SrcMediumValue =
- CGF.EmitLoadOfScalar(SrcMediumPtr, /*Volatile=*/true, CType, Loc);
- CGF.EmitStoreOfScalar(SrcMediumValue, TargetElemPtr, /*Volatile=*/false,
- CType);
- Bld.CreateBr(W0MergeBB);
-
- CGF.EmitBlock(W0ElseBB);
- Bld.CreateBr(W0MergeBB);
-
- CGF.EmitBlock(W0MergeBB);
-
- if (NumIters > 1) {
- Cnt = Bld.CreateNSWAdd(Cnt, llvm::ConstantInt::get(CGM.IntTy, /*V=*/1));
- CGF.EmitStoreOfScalar(Cnt, CntAddr, /*Volatile=*/false, C.IntTy);
- CGF.EmitBranch(PrecondBB);
- (void)ApplyDebugLocation::CreateEmpty(CGF);
- CGF.EmitBlock(ExitBB);
- }
- RealTySize %= TySize;
- }
- ++Idx;
- }
-
- CGF.FinishFunction();
- return Fn;
-}
-
-/// Emit a helper that reduces data across two OpenMP threads (lanes)
-/// in the same warp. It uses shuffle instructions to copy over data from
-/// a remote lane's stack. The reduction algorithm performed is specified
-/// by the fourth parameter.
-///
-/// Algorithm Versions.
-/// Full Warp Reduce (argument value 0):
-/// This algorithm assumes that all 32 lanes are active and gathers
-/// data from these 32 lanes, producing a single resultant value.
-/// Contiguous Partial Warp Reduce (argument value 1):
-/// This algorithm assumes that only a *contiguous* subset of lanes
-/// are active. This happens for the last warp in a parallel region
-/// when the user specified num_threads is not an integer multiple of
-/// 32. This contiguous subset always starts with the zeroth lane.
-/// Partial Warp Reduce (argument value 2):
-/// This algorithm gathers data from any number of lanes at any position.
-/// All reduced values are stored in the lowest possible lane. The set
-/// of problems every algorithm addresses is a super set of those
-/// addressable by algorithms with a lower version number. Overhead
-/// increases as algorithm version increases.
-///
-/// Terminology
-/// Reduce element:
-/// Reduce element refers to the individual data field with primitive
-/// data types to be combined and reduced across threads.
-/// Reduce list:
-/// Reduce list refers to a collection of local, thread-private
-/// reduce elements.
-/// Remote Reduce list:
-/// Remote Reduce list refers to a collection of remote (relative to
-/// the current thread) reduce elements.
-///
-/// We distinguish between three states of threads that are important to
-/// the implementation of this function.
-/// Alive threads:
-/// Threads in a warp executing the SIMT instruction, as distinguished from
-/// threads that are inactive due to divergent control flow.
-/// Active threads:
-/// The minimal set of threads that has to be alive upon entry to this
-/// function. The computation is correct iff active threads are alive.
-/// Some threads are alive but they are not active because they do not
-/// contribute to the computation in any useful manner. Turning them off
-/// may introduce control flow overheads without any tangible benefits.
-/// Effective threads:
-/// In order to comply with the argument requirements of the shuffle
-/// function, we must keep all lanes holding data alive. But at most
-/// half of them perform value aggregation; we refer to this half of
-/// threads as effective. The other half is simply handing off their
-/// data.
-///
-/// Procedure
-/// Value shuffle:
-/// In this step active threads transfer data from higher lane positions
-/// in the warp to lower lane positions, creating Remote Reduce list.
-/// Value aggregation:
-/// In this step, effective threads combine their thread local Reduce list
-/// with Remote Reduce list and store the result in the thread local
-/// Reduce list.
-/// Value copy:
-/// In this step, we deal with the assumption made by algorithm 2
-/// (i.e. contiguity assumption). When we have an odd number of lanes
-/// active, say 2k+1, only k threads will be effective and therefore k
-/// new values will be produced. However, the Reduce list owned by the
-/// (2k+1)th thread is ignored in the value aggregation. Therefore
-/// we copy the Reduce list from the (2k+1)th lane to (k+1)th lane so
-/// that the contiguity assumption still holds.
-static llvm::Function *emitShuffleAndReduceFunction(
- CodeGenModule &CGM, ArrayRef<const Expr *> Privates,
- QualType ReductionArrayTy, llvm::Function *ReduceFn, SourceLocation Loc) {
- ASTContext &C = CGM.getContext();
-
- // Thread local Reduce list used to host the values of data to be reduced.
- ImplicitParamDecl ReduceListArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
- C.VoidPtrTy, ImplicitParamKind::Other);
- // Current lane id; could be logical.
- ImplicitParamDecl LaneIDArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.ShortTy,
- ImplicitParamKind::Other);
- // Offset of the remote source lane relative to the current lane.
- ImplicitParamDecl RemoteLaneOffsetArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
- C.ShortTy, ImplicitParamKind::Other);
- // Algorithm version. This is expected to be known at compile time.
- ImplicitParamDecl AlgoVerArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
- C.ShortTy, ImplicitParamKind::Other);
- FunctionArgList Args;
- Args.push_back(&ReduceListArg);
- Args.push_back(&LaneIDArg);
- Args.push_back(&RemoteLaneOffsetArg);
- Args.push_back(&AlgoVerArg);
-
- const CGFunctionInfo &CGFI =
- CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
- auto *Fn = llvm::Function::Create(
- CGM.getTypes().GetFunctionType(CGFI), llvm::GlobalValue::InternalLinkage,
- "_omp_reduction_shuffle_and_reduce_func", &CGM.getModule());
- CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI);
- Fn->setDoesNotRecurse();
-
- CodeGenFunction CGF(CGM);
- CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
-
- CGBuilderTy &Bld = CGF.Builder;
-
- Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg);
- llvm::Type *ElemTy = CGF.ConvertTypeForMem(ReductionArrayTy);
- Address LocalReduceList(
- Bld.CreatePointerBitCastOrAddrSpaceCast(
- CGF.EmitLoadOfScalar(AddrReduceListArg, /*Volatile=*/false,
- C.VoidPtrTy, SourceLocation()),
- ElemTy->getPointerTo()),
- ElemTy, CGF.getPointerAlign());
-
- Address AddrLaneIDArg = CGF.GetAddrOfLocalVar(&LaneIDArg);
- llvm::Value *LaneIDArgVal = CGF.EmitLoadOfScalar(
- AddrLaneIDArg, /*Volatile=*/false, C.ShortTy, SourceLocation());
-
- Address AddrRemoteLaneOffsetArg = CGF.GetAddrOfLocalVar(&RemoteLaneOffsetArg);
- llvm::Value *RemoteLaneOffsetArgVal = CGF.EmitLoadOfScalar(
- AddrRemoteLaneOffsetArg, /*Volatile=*/false, C.ShortTy, SourceLocation());
-
- Address AddrAlgoVerArg = CGF.GetAddrOfLocalVar(&AlgoVerArg);
- llvm::Value *AlgoVerArgVal = CGF.EmitLoadOfScalar(
- AddrAlgoVerArg, /*Volatile=*/false, C.ShortTy, SourceLocation());
-
- // Create a local thread-private variable to host the Reduce list
- // from a remote lane.
- Address RemoteReduceList =
- CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.remote_reduce_list");
-
- // This loop iterates through the list of reduce elements and copies,
- // element by element, from a remote lane in the warp to RemoteReduceList,
- // hosted on the thread's stack.
- emitReductionListCopy(RemoteLaneToThread, CGF, ReductionArrayTy, Privates,
- LocalReduceList, RemoteReduceList,
- {/*RemoteLaneOffset=*/RemoteLaneOffsetArgVal,
- /*ScratchpadIndex=*/nullptr,
- /*ScratchpadWidth=*/nullptr});
-
- // The actions to be performed on the Remote Reduce list is dependent
- // on the algorithm version.
- //
- // if (AlgoVer==0) || (AlgoVer==1 && (LaneId < Offset)) || (AlgoVer==2 &&
- // LaneId % 2 == 0 && Offset > 0):
- // do the reduction value aggregation
- //
- // The thread local variable Reduce list is mutated in place to host the
- // reduced data, which is the aggregated value produced from local and
- // remote lanes.
- //
- // Note that AlgoVer is expected to be a constant integer known at compile
- // time.
- // When AlgoVer==0, the first conjunction evaluates to true, making
- // the entire predicate true during compile time.
- // When AlgoVer==1, the second conjunction has only the second part to be
- // evaluated during runtime. Other conjunctions evaluates to false
- // during compile time.
- // When AlgoVer==2, the third conjunction has only the second part to be
- // evaluated during runtime. Other conjunctions evaluates to false
- // during compile time.
- llvm::Value *CondAlgo0 = Bld.CreateIsNull(AlgoVerArgVal);
-
- llvm::Value *Algo1 = Bld.CreateICmpEQ(AlgoVerArgVal, Bld.getInt16(1));
- llvm::Value *CondAlgo1 = Bld.CreateAnd(
- Algo1, Bld.CreateICmpULT(LaneIDArgVal, RemoteLaneOffsetArgVal));
-
- llvm::Value *Algo2 = Bld.CreateICmpEQ(AlgoVerArgVal, Bld.getInt16(2));
- llvm::Value *CondAlgo2 = Bld.CreateAnd(
- Algo2, Bld.CreateIsNull(Bld.CreateAnd(LaneIDArgVal, Bld.getInt16(1))));
- CondAlgo2 = Bld.CreateAnd(
- CondAlgo2, Bld.CreateICmpSGT(RemoteLaneOffsetArgVal, Bld.getInt16(0)));
-
- llvm::Value *CondReduce = Bld.CreateOr(CondAlgo0, CondAlgo1);
- CondReduce = Bld.CreateOr(CondReduce, CondAlgo2);
-
- llvm::BasicBlock *ThenBB = CGF.createBasicBlock("then");
- llvm::BasicBlock *ElseBB = CGF.createBasicBlock("else");
- llvm::BasicBlock *MergeBB = CGF.createBasicBlock("ifcont");
- Bld.CreateCondBr(CondReduce, ThenBB, ElseBB);
-
- CGF.EmitBlock(ThenBB);
- // reduce_function(LocalReduceList, RemoteReduceList)
- llvm::Value *LocalReduceListPtr = Bld.CreatePointerBitCastOrAddrSpaceCast(
- LocalReduceList.getPointer(), CGF.VoidPtrTy);
- llvm::Value *RemoteReduceListPtr = Bld.CreatePointerBitCastOrAddrSpaceCast(
- RemoteReduceList.getPointer(), CGF.VoidPtrTy);
- CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
- CGF, Loc, ReduceFn, {LocalReduceListPtr, RemoteReduceListPtr});
- Bld.CreateBr(MergeBB);
-
- CGF.EmitBlock(ElseBB);
- Bld.CreateBr(MergeBB);
-
- CGF.EmitBlock(MergeBB);
-
- // if (AlgoVer==1 && (LaneId >= Offset)) copy Remote Reduce list to local
- // Reduce list.
- Algo1 = Bld.CreateICmpEQ(AlgoVerArgVal, Bld.getInt16(1));
- llvm::Value *CondCopy = Bld.CreateAnd(
- Algo1, Bld.CreateICmpUGE(LaneIDArgVal, RemoteLaneOffsetArgVal));
-
- llvm::BasicBlock *CpyThenBB = CGF.createBasicBlock("then");
- llvm::BasicBlock *CpyElseBB = CGF.createBasicBlock("else");
- llvm::BasicBlock *CpyMergeBB = CGF.createBasicBlock("ifcont");
- Bld.CreateCondBr(CondCopy, CpyThenBB, CpyElseBB);
-
- CGF.EmitBlock(CpyThenBB);
- emitReductionListCopy(ThreadCopy, CGF, ReductionArrayTy, Privates,
- RemoteReduceList, LocalReduceList);
- Bld.CreateBr(CpyMergeBB);
-
- CGF.EmitBlock(CpyElseBB);
- Bld.CreateBr(CpyMergeBB);
-
- CGF.EmitBlock(CpyMergeBB);
-
- CGF.FinishFunction();
- return Fn;
-}
-
-/// This function emits a helper that copies all the reduction variables from
-/// the team into the provided global buffer for the reduction variables.
-///
-/// void list_to_global_copy_func(void *buffer, int Idx, void *reduce_data)
-/// For all data entries D in reduce_data:
-/// Copy local D to buffer.D[Idx]
-static llvm::Value *emitListToGlobalCopyFunction(
- CodeGenModule &CGM, ArrayRef<const Expr *> Privates,
- QualType ReductionArrayTy, SourceLocation Loc,
- const RecordDecl *TeamReductionRec,
- const llvm::SmallDenseMap<const ValueDecl *, const FieldDecl *>
- &VarFieldMap) {
- ASTContext &C = CGM.getContext();
-
- // Buffer: global reduction buffer.
- ImplicitParamDecl BufferArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
- C.VoidPtrTy, ImplicitParamKind::Other);
- // Idx: index of the buffer.
- ImplicitParamDecl IdxArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.IntTy,
- ImplicitParamKind::Other);
- // ReduceList: thread local Reduce list.
- ImplicitParamDecl ReduceListArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
- C.VoidPtrTy, ImplicitParamKind::Other);
- FunctionArgList Args;
- Args.push_back(&BufferArg);
- Args.push_back(&IdxArg);
- Args.push_back(&ReduceListArg);
-
- const CGFunctionInfo &CGFI =
- CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
- auto *Fn = llvm::Function::Create(
- CGM.getTypes().GetFunctionType(CGFI), llvm::GlobalValue::InternalLinkage,
- "_omp_reduction_list_to_global_copy_func", &CGM.getModule());
- CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI);
- Fn->setDoesNotRecurse();
- CodeGenFunction CGF(CGM);
- CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
-
- CGBuilderTy &Bld = CGF.Builder;
-
- Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg);
- Address AddrBufferArg = CGF.GetAddrOfLocalVar(&BufferArg);
- llvm::Type *ElemTy = CGF.ConvertTypeForMem(ReductionArrayTy);
- Address LocalReduceList(
- Bld.CreatePointerBitCastOrAddrSpaceCast(
- CGF.EmitLoadOfScalar(AddrReduceListArg, /*Volatile=*/false,
- C.VoidPtrTy, Loc),
- ElemTy->getPointerTo()),
- ElemTy, CGF.getPointerAlign());
- QualType StaticTy = C.getRecordType(TeamReductionRec);
- llvm::Type *LLVMReductionsBufferTy =
- CGM.getTypes().ConvertTypeForMem(StaticTy);
- llvm::Value *BufferArrPtr = Bld.CreatePointerBitCastOrAddrSpaceCast(
- CGF.EmitLoadOfScalar(AddrBufferArg, /*Volatile=*/false, C.VoidPtrTy, Loc),
- LLVMReductionsBufferTy->getPointerTo());
- llvm::Value *Idxs[] = {CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(&IdxArg),
- /*Volatile=*/false, C.IntTy,
- Loc)};
- unsigned Idx = 0;
- for (const Expr *Private : Privates) {
- // Reduce element = LocalReduceList[i]
- Address ElemPtrPtrAddr = Bld.CreateConstArrayGEP(LocalReduceList, Idx);
- llvm::Value *ElemPtrPtr = CGF.EmitLoadOfScalar(
- ElemPtrPtrAddr, /*Volatile=*/false, C.VoidPtrTy, SourceLocation());
- // elemptr = ((CopyType*)(elemptrptr)) + I
- ElemTy = CGF.ConvertTypeForMem(Private->getType());
- ElemPtrPtr = Bld.CreatePointerBitCastOrAddrSpaceCast(
- ElemPtrPtr, ElemTy->getPointerTo());
- Address ElemPtr =
- Address(ElemPtrPtr, ElemTy, C.getTypeAlignInChars(Private->getType()));
- const ValueDecl *VD = cast<DeclRefExpr>(Private)->getDecl();
- // Global = Buffer.VD[Idx];
- const FieldDecl *FD = VarFieldMap.lookup(VD);
- llvm::Value *BufferPtr =
- Bld.CreateInBoundsGEP(LLVMReductionsBufferTy, BufferArrPtr, Idxs);
- LValue GlobLVal = CGF.EmitLValueForField(
- CGF.MakeNaturalAlignAddrLValue(BufferPtr, StaticTy), FD);
- Address GlobAddr = GlobLVal.getAddress(CGF);
- GlobLVal.setAddress(Address(GlobAddr.getPointer(),
- CGF.ConvertTypeForMem(Private->getType()),
- GlobAddr.getAlignment()));
- switch (CGF.getEvaluationKind(Private->getType())) {
- case TEK_Scalar: {
- llvm::Value *V = CGF.EmitLoadOfScalar(
- ElemPtr, /*Volatile=*/false, Private->getType(), Loc,
- LValueBaseInfo(AlignmentSource::Type), TBAAAccessInfo());
- CGF.EmitStoreOfScalar(V, GlobLVal);
- break;
- }
- case TEK_Complex: {
- CodeGenFunction::ComplexPairTy V = CGF.EmitLoadOfComplex(
- CGF.MakeAddrLValue(ElemPtr, Private->getType()), Loc);
- CGF.EmitStoreOfComplex(V, GlobLVal, /*isInit=*/false);
- break;
- }
- case TEK_Aggregate:
- CGF.EmitAggregateCopy(GlobLVal,
- CGF.MakeAddrLValue(ElemPtr, Private->getType()),
- Private->getType(), AggValueSlot::DoesNotOverlap);
- break;
- }
- ++Idx;
- }
-
- CGF.FinishFunction();
- return Fn;
-}
-
-/// This function emits a helper that reduces all the reduction variables from
-/// the team into the provided global buffer for the reduction variables.
-///
-/// void list_to_global_reduce_func(void *buffer, int Idx, void *reduce_data)
-/// void *GlobPtrs[];
-/// GlobPtrs[0] = (void*)&buffer.D0[Idx];
-/// ...
-/// GlobPtrs[N] = (void*)&buffer.DN[Idx];
-/// reduce_function(GlobPtrs, reduce_data);
-static llvm::Value *emitListToGlobalReduceFunction(
- CodeGenModule &CGM, ArrayRef<const Expr *> Privates,
- QualType ReductionArrayTy, SourceLocation Loc,
- const RecordDecl *TeamReductionRec,
- const llvm::SmallDenseMap<const ValueDecl *, const FieldDecl *>
- &VarFieldMap,
- llvm::Function *ReduceFn) {
- ASTContext &C = CGM.getContext();
-
- // Buffer: global reduction buffer.
- ImplicitParamDecl BufferArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
- C.VoidPtrTy, ImplicitParamKind::Other);
- // Idx: index of the buffer.
- ImplicitParamDecl IdxArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.IntTy,
- ImplicitParamKind::Other);
- // ReduceList: thread local Reduce list.
- ImplicitParamDecl ReduceListArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
- C.VoidPtrTy, ImplicitParamKind::Other);
- FunctionArgList Args;
- Args.push_back(&BufferArg);
- Args.push_back(&IdxArg);
- Args.push_back(&ReduceListArg);
-
- const CGFunctionInfo &CGFI =
- CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
- auto *Fn = llvm::Function::Create(
- CGM.getTypes().GetFunctionType(CGFI), llvm::GlobalValue::InternalLinkage,
- "_omp_reduction_list_to_global_reduce_func", &CGM.getModule());
- CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI);
- Fn->setDoesNotRecurse();
- CodeGenFunction CGF(CGM);
- CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
-
- CGBuilderTy &Bld = CGF.Builder;
-
- Address AddrBufferArg = CGF.GetAddrOfLocalVar(&BufferArg);
- QualType StaticTy = C.getRecordType(TeamReductionRec);
- llvm::Type *LLVMReductionsBufferTy =
- CGM.getTypes().ConvertTypeForMem(StaticTy);
- llvm::Value *BufferArrPtr = Bld.CreatePointerBitCastOrAddrSpaceCast(
- CGF.EmitLoadOfScalar(AddrBufferArg, /*Volatile=*/false, C.VoidPtrTy, Loc),
- LLVMReductionsBufferTy->getPointerTo());
-
- // 1. Build a list of reduction variables.
- // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
- Address ReductionList =
- CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list");
- auto IPriv = Privates.begin();
- llvm::Value *Idxs[] = {CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(&IdxArg),
- /*Volatile=*/false, C.IntTy,
- Loc)};
- unsigned Idx = 0;
- for (unsigned I = 0, E = Privates.size(); I < E; ++I, ++IPriv, ++Idx) {
- Address Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx);
- // Global = Buffer.VD[Idx];
- const ValueDecl *VD = cast<DeclRefExpr>(*IPriv)->getDecl();
- const FieldDecl *FD = VarFieldMap.lookup(VD);
- llvm::Value *BufferPtr =
- Bld.CreateInBoundsGEP(LLVMReductionsBufferTy, BufferArrPtr, Idxs);
- LValue GlobLVal = CGF.EmitLValueForField(
- CGF.MakeNaturalAlignAddrLValue(BufferPtr, StaticTy), FD);
- Address GlobAddr = GlobLVal.getAddress(CGF);
- CGF.EmitStoreOfScalar(GlobAddr.getPointer(), Elem, /*Volatile=*/false,
- C.VoidPtrTy);
- if ((*IPriv)->getType()->isVariablyModifiedType()) {
- // Store array size.
- ++Idx;
- Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx);
- llvm::Value *Size = CGF.Builder.CreateIntCast(
- CGF.getVLASize(
- CGF.getContext().getAsVariableArrayType((*IPriv)->getType()))
- .NumElts,
- CGF.SizeTy, /*isSigned=*/false);
- CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy),
- Elem);
- }
- }
-
- // Call reduce_function(GlobalReduceList, ReduceList)
- llvm::Value *GlobalReduceList = ReductionList.getPointer();
- Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg);
- llvm::Value *ReducedPtr = CGF.EmitLoadOfScalar(
- AddrReduceListArg, /*Volatile=*/false, C.VoidPtrTy, Loc);
- CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
- CGF, Loc, ReduceFn, {GlobalReduceList, ReducedPtr});
- CGF.FinishFunction();
- return Fn;
-}
-
-/// This function emits a helper that copies all the reduction variables from
-/// the team into the provided global buffer for the reduction variables.
-///
-/// void list_to_global_copy_func(void *buffer, int Idx, void *reduce_data)
-/// For all data entries D in reduce_data:
-/// Copy buffer.D[Idx] to local D;
-static llvm::Value *emitGlobalToListCopyFunction(
- CodeGenModule &CGM, ArrayRef<const Expr *> Privates,
- QualType ReductionArrayTy, SourceLocation Loc,
- const RecordDecl *TeamReductionRec,
- const llvm::SmallDenseMap<const ValueDecl *, const FieldDecl *>
- &VarFieldMap) {
- ASTContext &C = CGM.getContext();
-
- // Buffer: global reduction buffer.
- ImplicitParamDecl BufferArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
- C.VoidPtrTy, ImplicitParamKind::Other);
- // Idx: index of the buffer.
- ImplicitParamDecl IdxArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.IntTy,
- ImplicitParamKind::Other);
- // ReduceList: thread local Reduce list.
- ImplicitParamDecl ReduceListArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
- C.VoidPtrTy, ImplicitParamKind::Other);
- FunctionArgList Args;
- Args.push_back(&BufferArg);
- Args.push_back(&IdxArg);
- Args.push_back(&ReduceListArg);
-
- const CGFunctionInfo &CGFI =
- CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
- auto *Fn = llvm::Function::Create(
- CGM.getTypes().GetFunctionType(CGFI), llvm::GlobalValue::InternalLinkage,
- "_omp_reduction_global_to_list_copy_func", &CGM.getModule());
- CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI);
- Fn->setDoesNotRecurse();
- CodeGenFunction CGF(CGM);
- CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
-
- CGBuilderTy &Bld = CGF.Builder;
-
- Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg);
- Address AddrBufferArg = CGF.GetAddrOfLocalVar(&BufferArg);
- llvm::Type *ElemTy = CGF.ConvertTypeForMem(ReductionArrayTy);
- Address LocalReduceList(
- Bld.CreatePointerBitCastOrAddrSpaceCast(
- CGF.EmitLoadOfScalar(AddrReduceListArg, /*Volatile=*/false,
- C.VoidPtrTy, Loc),
- ElemTy->getPointerTo()),
- ElemTy, CGF.getPointerAlign());
- QualType StaticTy = C.getRecordType(TeamReductionRec);
- llvm::Type *LLVMReductionsBufferTy =
- CGM.getTypes().ConvertTypeForMem(StaticTy);
- llvm::Value *BufferArrPtr = Bld.CreatePointerBitCastOrAddrSpaceCast(
- CGF.EmitLoadOfScalar(AddrBufferArg, /*Volatile=*/false, C.VoidPtrTy, Loc),
- LLVMReductionsBufferTy->getPointerTo());
-
- llvm::Value *Idxs[] = {CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(&IdxArg),
- /*Volatile=*/false, C.IntTy,
- Loc)};
- unsigned Idx = 0;
- for (const Expr *Private : Privates) {
- // Reduce element = LocalReduceList[i]
- Address ElemPtrPtrAddr = Bld.CreateConstArrayGEP(LocalReduceList, Idx);
- llvm::Value *ElemPtrPtr = CGF.EmitLoadOfScalar(
- ElemPtrPtrAddr, /*Volatile=*/false, C.VoidPtrTy, SourceLocation());
- // elemptr = ((CopyType*)(elemptrptr)) + I
- ElemTy = CGF.ConvertTypeForMem(Private->getType());
- ElemPtrPtr = Bld.CreatePointerBitCastOrAddrSpaceCast(
- ElemPtrPtr, ElemTy->getPointerTo());
- Address ElemPtr =
- Address(ElemPtrPtr, ElemTy, C.getTypeAlignInChars(Private->getType()));
- const ValueDecl *VD = cast<DeclRefExpr>(Private)->getDecl();
- // Global = Buffer.VD[Idx];
- const FieldDecl *FD = VarFieldMap.lookup(VD);
- llvm::Value *BufferPtr =
- Bld.CreateInBoundsGEP(LLVMReductionsBufferTy, BufferArrPtr, Idxs);
- LValue GlobLVal = CGF.EmitLValueForField(
- CGF.MakeNaturalAlignAddrLValue(BufferPtr, StaticTy), FD);
- Address GlobAddr = GlobLVal.getAddress(CGF);
- GlobLVal.setAddress(Address(GlobAddr.getPointer(),
- CGF.ConvertTypeForMem(Private->getType()),
- GlobAddr.getAlignment()));
- switch (CGF.getEvaluationKind(Private->getType())) {
- case TEK_Scalar: {
- llvm::Value *V = CGF.EmitLoadOfScalar(GlobLVal, Loc);
- CGF.EmitStoreOfScalar(V, ElemPtr, /*Volatile=*/false, Private->getType(),
- LValueBaseInfo(AlignmentSource::Type),
- TBAAAccessInfo());
- break;
- }
- case TEK_Complex: {
- CodeGenFunction::ComplexPairTy V = CGF.EmitLoadOfComplex(GlobLVal, Loc);
- CGF.EmitStoreOfComplex(V, CGF.MakeAddrLValue(ElemPtr, Private->getType()),
- /*isInit=*/false);
- break;
- }
- case TEK_Aggregate:
- CGF.EmitAggregateCopy(CGF.MakeAddrLValue(ElemPtr, Private->getType()),
- GlobLVal, Private->getType(),
- AggValueSlot::DoesNotOverlap);
- break;
- }
- ++Idx;
- }
-
- CGF.FinishFunction();
- return Fn;
-}
-
-/// This function emits a helper that reduces all the reduction variables from
-/// the team into the provided global buffer for the reduction variables.
-///
-/// void global_to_list_reduce_func(void *buffer, int Idx, void *reduce_data)
-/// void *GlobPtrs[];
-/// GlobPtrs[0] = (void*)&buffer.D0[Idx];
-/// ...
-/// GlobPtrs[N] = (void*)&buffer.DN[Idx];
-/// reduce_function(reduce_data, GlobPtrs);
-static llvm::Value *emitGlobalToListReduceFunction(
- CodeGenModule &CGM, ArrayRef<const Expr *> Privates,
- QualType ReductionArrayTy, SourceLocation Loc,
- const RecordDecl *TeamReductionRec,
- const llvm::SmallDenseMap<const ValueDecl *, const FieldDecl *>
- &VarFieldMap,
- llvm::Function *ReduceFn) {
- ASTContext &C = CGM.getContext();
-
- // Buffer: global reduction buffer.
- ImplicitParamDecl BufferArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
- C.VoidPtrTy, ImplicitParamKind::Other);
- // Idx: index of the buffer.
- ImplicitParamDecl IdxArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.IntTy,
- ImplicitParamKind::Other);
- // ReduceList: thread local Reduce list.
- ImplicitParamDecl ReduceListArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
- C.VoidPtrTy, ImplicitParamKind::Other);
- FunctionArgList Args;
- Args.push_back(&BufferArg);
- Args.push_back(&IdxArg);
- Args.push_back(&ReduceListArg);
-
- const CGFunctionInfo &CGFI =
- CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
- auto *Fn = llvm::Function::Create(
- CGM.getTypes().GetFunctionType(CGFI), llvm::GlobalValue::InternalLinkage,
- "_omp_reduction_global_to_list_reduce_func", &CGM.getModule());
- CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI);
- Fn->setDoesNotRecurse();
- CodeGenFunction CGF(CGM);
- CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
-
- CGBuilderTy &Bld = CGF.Builder;
-
- Address AddrBufferArg = CGF.GetAddrOfLocalVar(&BufferArg);
- QualType StaticTy = C.getRecordType(TeamReductionRec);
- llvm::Type *LLVMReductionsBufferTy =
- CGM.getTypes().ConvertTypeForMem(StaticTy);
- llvm::Value *BufferArrPtr = Bld.CreatePointerBitCastOrAddrSpaceCast(
- CGF.EmitLoadOfScalar(AddrBufferArg, /*Volatile=*/false, C.VoidPtrTy, Loc),
- LLVMReductionsBufferTy->getPointerTo());
-
- // 1. Build a list of reduction variables.
- // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
- Address ReductionList =
- CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list");
- auto IPriv = Privates.begin();
- llvm::Value *Idxs[] = {CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(&IdxArg),
- /*Volatile=*/false, C.IntTy,
- Loc)};
- unsigned Idx = 0;
- for (unsigned I = 0, E = Privates.size(); I < E; ++I, ++IPriv, ++Idx) {
- Address Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx);
- // Global = Buffer.VD[Idx];
- const ValueDecl *VD = cast<DeclRefExpr>(*IPriv)->getDecl();
- const FieldDecl *FD = VarFieldMap.lookup(VD);
- llvm::Value *BufferPtr =
- Bld.CreateInBoundsGEP(LLVMReductionsBufferTy, BufferArrPtr, Idxs);
- LValue GlobLVal = CGF.EmitLValueForField(
- CGF.MakeNaturalAlignAddrLValue(BufferPtr, StaticTy), FD);
- Address GlobAddr = GlobLVal.getAddress(CGF);
- CGF.EmitStoreOfScalar(GlobAddr.getPointer(), Elem, /*Volatile=*/false,
- C.VoidPtrTy);
- if ((*IPriv)->getType()->isVariablyModifiedType()) {
- // Store array size.
- ++Idx;
- Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx);
- llvm::Value *Size = CGF.Builder.CreateIntCast(
- CGF.getVLASize(
- CGF.getContext().getAsVariableArrayType((*IPriv)->getType()))
- .NumElts,
- CGF.SizeTy, /*isSigned=*/false);
- CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy),
- Elem);
- }
- }
-
- // Call reduce_function(ReduceList, GlobalReduceList)
- llvm::Value *GlobalReduceList = ReductionList.getPointer();
- Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg);
- llvm::Value *ReducedPtr = CGF.EmitLoadOfScalar(
- AddrReduceListArg, /*Volatile=*/false, C.VoidPtrTy, Loc);
- CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
- CGF, Loc, ReduceFn, {ReducedPtr, GlobalReduceList});
- CGF.FinishFunction();
- return Fn;
-}
-
///
/// Design of OpenMP reductions on the GPU
///
@@ -2805,10 +1654,13 @@ void CGOpenMPRuntimeGPU::emitReduction(
return;
bool ParallelReduction = isOpenMPParallelDirective(Options.ReductionKind);
+ bool DistributeReduction = isOpenMPDistributeDirective(Options.ReductionKind);
#ifndef NDEBUG
bool TeamsReduction = isOpenMPTeamsDirective(Options.ReductionKind);
#endif
+ ASTContext &C = CGM.getContext();
+
if (Options.SimpleReduction) {
assert(!TeamsReduction && !ParallelReduction &&
"Invalid reduction selection in emitReduction.");
@@ -2817,155 +1669,78 @@ void CGOpenMPRuntimeGPU::emitReduction(
return;
}
- assert((TeamsReduction || ParallelReduction) &&
- "Invalid reduction selection in emitReduction.");
-
- llvm::SmallDenseMap<const ValueDecl *, const FieldDecl *> VarFieldMap;
- llvm::SmallVector<const ValueDecl *, 4> PrivatesReductions(Privates.size());
- int Cnt = 0;
- for (const Expr *DRE : Privates) {
- PrivatesReductions[Cnt] = cast<DeclRefExpr>(DRE)->getDecl();
- ++Cnt;
- }
-
- ASTContext &C = CGM.getContext();
- const RecordDecl *ReductionRec = ::buildRecordForGlobalizedVars(
- CGM.getContext(), PrivatesReductions, std::nullopt, VarFieldMap, 1);
-
- // Build res = __kmpc_reduce{_nowait}(<gtid>, <n>, sizeof(RedList),
- // RedList, shuffle_reduce_func, interwarp_copy_func);
- // or
- // Build res = __kmpc_reduce_teams_nowait_simple(<loc>, <gtid>, <lck>);
+ // Source location for theident struct
llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
- llvm::Value *Res;
- // 1. Build a list of reduction variables.
- // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
- auto Size = RHSExprs.size();
- for (const Expr *E : Privates) {
- if (E->getType()->isVariablyModifiedType())
- // Reserve place for array size.
- ++Size;
- }
- llvm::APInt ArraySize(/*unsigned int numBits=*/32, Size);
- QualType ReductionArrayTy = C.getConstantArrayType(
- C.VoidPtrTy, ArraySize, nullptr, ArraySizeModifier::Normal,
- /*IndexTypeQuals=*/0);
- Address ReductionList =
- CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list");
- auto IPriv = Privates.begin();
- unsigned Idx = 0;
- for (unsigned I = 0, E = RHSExprs.size(); I < E; ++I, ++IPriv, ++Idx) {
- Address Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx);
- CGF.Builder.CreateStore(
- CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
- CGF.EmitLValue(RHSExprs[I]).getPointer(CGF), CGF.VoidPtrTy),
- Elem);
- if ((*IPriv)->getType()->isVariablyModifiedType()) {
- // Store array size.
- ++Idx;
- Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx);
- llvm::Value *Size = CGF.Builder.CreateIntCast(
- CGF.getVLASize(
- CGF.getContext().getAsVariableArrayType((*IPriv)->getType()))
- .NumElts,
- CGF.SizeTy, /*isSigned=*/false);
- CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy),
- Elem);
- }
- }
-
- llvm::Value *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
- ReductionList.getPointer(), CGF.VoidPtrTy);
- llvm::Function *ReductionFn = emitReductionFunction(
- CGF.CurFn->getName(), Loc, CGF.ConvertTypeForMem(ReductionArrayTy),
- Privates, LHSExprs, RHSExprs, ReductionOps);
- llvm::Value *ReductionDataSize =
- CGF.getTypeSize(C.getRecordType(ReductionRec));
- ReductionDataSize =
- CGF.Builder.CreateSExtOrTrunc(ReductionDataSize, CGF.Int64Ty);
- llvm::Function *ShuffleAndReduceFn = emitShuffleAndReduceFunction(
- CGM, Privates, ReductionArrayTy, ReductionFn, Loc);
- llvm::Value *InterWarpCopyFn =
- emitInterWarpCopyFunction(CGM, Privates, ReductionArrayTy, Loc);
-
- if (ParallelReduction) {
- llvm::Value *Args[] = {RTLoc, ReductionDataSize, RL, ShuffleAndReduceFn,
- InterWarpCopyFn};
-
- Res = CGF.EmitRuntimeCall(
- OMPBuilder.getOrCreateRuntimeFunction(
- CGM.getModule(), OMPRTL___kmpc_nvptx_parallel_reduce_nowait_v2),
- Args);
- } else {
- assert(TeamsReduction && "expected teams reduction.");
- TeamsReductions.push_back(ReductionRec);
- auto *KernelTeamsReductionPtr = CGF.EmitRuntimeCall(
- OMPBuilder.getOrCreateRuntimeFunction(
- CGM.getModule(), OMPRTL___kmpc_reduction_get_fixed_buffer),
- {}, "_openmp_teams_reductions_buffer_$_$ptr");
- llvm::Value *GlobalToBufferCpyFn = ::emitListToGlobalCopyFunction(
- CGM, Privates, ReductionArrayTy, Loc, ReductionRec, VarFieldMap);
- llvm::Value *GlobalToBufferRedFn = ::emitListToGlobalReduceFunction(
- CGM, Privates, ReductionArrayTy, Loc, ReductionRec, VarFieldMap,
- ReductionFn);
- llvm::Value *BufferToGlobalCpyFn = ::emitGlobalToListCopyFunction(
- CGM, Privates, ReductionArrayTy, Loc, ReductionRec, VarFieldMap);
- llvm::Value *BufferToGlobalRedFn = ::emitGlobalToListReduceFunction(
- CGM, Privates, ReductionArrayTy, Loc, ReductionRec, VarFieldMap,
- ReductionFn);
-
- llvm::Value *Args[] = {
- RTLoc,
- KernelTeamsReductionPtr,
- CGF.Builder.getInt32(C.getLangOpts().OpenMPCUDAReductionBufNum),
- ReductionDataSize,
- RL,
- ShuffleAndReduceFn,
- InterWarpCopyFn,
- GlobalToBufferCpyFn,
- GlobalToBufferRedFn,
- BufferToGlobalCpyFn,
- BufferToGlobalRedFn};
-
- Res = CGF.EmitRuntimeCall(
- OMPBuilder.getOrCreateRuntimeFunction(
- CGM.getModule(), OMPRTL___kmpc_nvptx_teams_reduce_nowait_v2),
- Args);
- }
+ using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
+ InsertPointTy AllocaIP(CGF.AllocaInsertPt->getParent(),
+ CGF.AllocaInsertPt->getIterator());
+ InsertPointTy CodeGenIP(CGF.Builder.GetInsertBlock(),
+ CGF.Builder.GetInsertPoint());
+ llvm::OpenMPIRBuilder::LocationDescription OmpLoc(CodeGenIP);
+ llvm::SmallVector<llvm::OpenMPIRBuilder::ReductionInfo> ReductionInfos;
- // 5. Build if (res == 1)
- llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.reduction.done");
- llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.then");
- llvm::Value *Cond = CGF.Builder.CreateICmpEQ(
- Res, llvm::ConstantInt::get(CGM.Int32Ty, /*V=*/1));
- CGF.Builder.CreateCondBr(Cond, ThenBB, ExitBB);
-
- // 6. Build then branch: where we have reduced values in the master
- // thread in each team.
- // __kmpc_end_reduce{_nowait}(<gtid>);
- // break;
- CGF.EmitBlock(ThenBB);
-
- // Add emission of __kmpc_end_reduce{_nowait}(<gtid>);
- auto &&CodeGen = [Privates, LHSExprs, RHSExprs, ReductionOps,
- this](CodeGenFunction &CGF, PrePostActionTy &Action) {
- auto IPriv = Privates.begin();
- auto ILHS = LHSExprs.begin();
- auto IRHS = RHSExprs.begin();
- for (const Expr *E : ReductionOps) {
- emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS),
- cast<DeclRefExpr>(*IRHS));
- ++IPriv;
- ++ILHS;
- ++IRHS;
+ CodeGenFunction::OMPPrivateScope Scope(CGF);
+ unsigned Idx = 0;
+ for (const Expr *Private : Privates) {
+ llvm::Type *ElementType;
+ llvm::Value *Variable;
+ llvm::Value *PrivateVariable;
+ llvm::OpenMPIRBuilder::AtomicReductionGenCB AtomicReductionGen = nullptr;
+ ElementType = CGF.ConvertTypeForMem(Private->getType());
+ const auto *RHSVar =
+ cast<VarDecl>(cast<DeclRefExpr>(RHSExprs[Idx])->getDecl());
+ PrivateVariable = CGF.GetAddrOfLocalVar(RHSVar).getPointer();
+ const auto *LHSVar =
+ cast<VarDecl>(cast<DeclRefExpr>(LHSExprs[Idx])->getDecl());
+ Variable = CGF.GetAddrOfLocalVar(LHSVar).getPointer();
+ llvm::OpenMPIRBuilder::EvaluationKindTy EvalKind;
+ switch (CGF.getEvaluationKind(Private->getType())) {
+ case TEK_Scalar:
+ EvalKind = llvm::OpenMPIRBuilder::EvaluationKindTy::Scalar;
+ break;
+ case TEK_Complex:
+ EvalKind = llvm::OpenMPIRBuilder::EvaluationKindTy::Complex;
+ break;
+ case TEK_Aggregate:
+ EvalKind = llvm::OpenMPIRBuilder::EvaluationKindTy::Aggregate;
+ break;
}
- };
- RegionCodeGenTy RCG(CodeGen);
- RCG(CGF);
- // There is no need to emit line number for unconditional branch.
- (void)ApplyDebugLocation::CreateEmpty(CGF);
- CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
+ auto ReductionGen = [&](InsertPointTy CodeGenIP, unsigned I,
+ llvm::Value **LHSPtr, llvm::Value **RHSPtr,
+ llvm::Function *NewFunc) {
+ CGF.Builder.restoreIP(CodeGenIP);
+ auto *CurFn = CGF.CurFn;
+ CGF.CurFn = NewFunc;
+
+ *LHSPtr = CGF.GetAddrOfLocalVar(
+ cast<VarDecl>(cast<DeclRefExpr>(LHSExprs[I])->getDecl()))
+ .getPointer();
+ *RHSPtr = CGF.GetAddrOfLocalVar(
+ cast<VarDecl>(cast<DeclRefExpr>(RHSExprs[I])->getDecl()))
+ .getPointer();
+
+ emitSingleReductionCombiner(CGF, ReductionOps[I], Privates[I],
+ cast<DeclRefExpr>(LHSExprs[I]),
+ cast<DeclRefExpr>(RHSExprs[I]));
+
+ CGF.CurFn = CurFn;
+
+ return InsertPointTy(CGF.Builder.GetInsertBlock(),
+ CGF.Builder.GetInsertPoint());
+ };
+ ReductionInfos.emplace_back(llvm::OpenMPIRBuilder::ReductionInfo(
+ ElementType, Variable, PrivateVariable, EvalKind,
+ /*ReductionGen=*/nullptr, ReductionGen, AtomicReductionGen));
+ Idx++;
+ }
+
+ CGF.Builder.restoreIP(OMPBuilder.createReductionsGPU(
+ OmpLoc, AllocaIP, CodeGenIP, ReductionInfos, false, TeamsReduction,
+ DistributeReduction, llvm::OpenMPIRBuilder::ReductionGenCBTy::Clang,
+ CGF.getTarget().getGridValue(), C.getLangOpts().OpenMPCUDAReductionBufNum,
+ RTLoc));
+ return;
}
const VarDecl *
@@ -3560,10 +2335,3 @@ llvm::Value *CGOpenMPRuntimeGPU::getGPUThreadID(CodeGenFunction &CGF) {
CGM.getModule(), OMPRTL___kmpc_get_hardware_thread_id_in_block),
Args);
}
-
-llvm::Value *CGOpenMPRuntimeGPU::getGPUWarpSize(CodeGenFunction &CGF) {
- ArrayRef<llvm::Value *> Args{};
- return CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
- CGM.getModule(), OMPRTL___kmpc_get_warp_size),
- Args);
-}
diff --git a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.h b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.h
index 141436f26230dd..4d586ec972f8d6 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.h
+++ b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.h
@@ -150,9 +150,6 @@ class CGOpenMPRuntimeGPU : public CGOpenMPRuntime {
CodeGenFunction &CGF,
const std::pair<llvm::Value *, llvm::Value *> &AddrSizePair) override;
- /// Get the GPU warp size.
- llvm::Value *getGPUWarpSize(CodeGenFunction &CGF);
-
/// Get the id of the current thread on the GPU.
llvm::Value *getGPUThreadID(CodeGenFunction &CGF);
diff --git a/clang/test/OpenMP/nvptx_target_parallel_reduction_codegen_tbaa_PR46146.cpp b/clang/test/OpenMP/nvptx_target_parallel_reduction_codegen_tbaa_PR46146.cpp
index fac42732022ad8..567598250301fd 100644
--- a/clang/test/OpenMP/nvptx_target_parallel_reduction_codegen_tbaa_PR46146.cpp
+++ b/clang/test/OpenMP/nvptx_target_parallel_reduction_codegen_tbaa_PR46146.cpp
@@ -36,14 +36,14 @@ void test() {
// CHECK1-NEXT: [[DYN_PTR_ADDR:%.*]] = alloca ptr, align 8
// CHECK1-NEXT: [[DOTZERO_ADDR:%.*]] = alloca i32, align 4
// CHECK1-NEXT: [[DOTTHREADID_TEMP_:%.*]] = alloca i32, align 4
-// CHECK1-NEXT: store ptr [[DYN_PTR]], ptr [[DYN_PTR_ADDR]], align 8, !tbaa [[TBAA10:![0-9]+]]
+// CHECK1-NEXT: store ptr [[DYN_PTR]], ptr [[DYN_PTR_ADDR]], align 8
// CHECK1-NEXT: [[TMP0:%.*]] = call i32 @__kmpc_target_init(ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z17complex_reductionIfEvv_l16_kernel_environment, ptr [[DYN_PTR]])
// CHECK1-NEXT: [[EXEC_USER_CODE:%.*]] = icmp eq i32 [[TMP0]], -1
// CHECK1-NEXT: br i1 [[EXEC_USER_CODE]], label [[USER_CODE_ENTRY:%.*]], label [[WORKER_EXIT:%.*]]
// CHECK1: user_code.entry:
// CHECK1-NEXT: [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1:[0-9]+]])
// CHECK1-NEXT: store i32 0, ptr [[DOTZERO_ADDR]], align 4
-// CHECK1-NEXT: store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA14:![0-9]+]]
+// CHECK1-NEXT: store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4
// CHECK1-NEXT: call void @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z17complex_reductionIfEvv_l16_omp_outlined(ptr [[DOTTHREADID_TEMP_]], ptr [[DOTZERO_ADDR]]) #[[ATTR4:[0-9]+]]
// CHECK1-NEXT: call void @__kmpc_target_deinit()
// CHECK1-NEXT: ret void
@@ -66,78 +66,78 @@ void test() {
// CHECK1-NEXT: [[REF_TMP:%.*]] = alloca float, align 4
// CHECK1-NEXT: [[REF_TMP2:%.*]] = alloca float, align 4
// CHECK1-NEXT: [[CAPTURED_VARS_ADDRS:%.*]] = alloca [3 x ptr], align 8
-// CHECK1-NEXT: store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT: store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT: store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8
+// CHECK1-NEXT: store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8
// CHECK1-NEXT: [[ISTART:%.*]] = call align 16 ptr @__kmpc_alloc_shared(i64 4)
// CHECK1-NEXT: [[IEND:%.*]] = call align 16 ptr @__kmpc_alloc_shared(i64 4)
// CHECK1-NEXT: [[PARTIAL_SUM:%.*]] = call align 16 ptr @__kmpc_alloc_shared(i64 8)
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_IV]]) #[[ATTR4]]
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_LB]]) #[[ATTR4]]
-// CHECK1-NEXT: store i32 0, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: store i32 0, ptr [[DOTOMP_LB]], align 4
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_UB]]) #[[ATTR4]]
-// CHECK1-NEXT: store i32 99, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: store i32 99, ptr [[DOTOMP_UB]], align 4
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_STRIDE]]) #[[ATTR4]]
-// CHECK1-NEXT: store i32 1, ptr [[DOTOMP_STRIDE]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: store i32 1, ptr [[DOTOMP_STRIDE]], align 4
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_IS_LAST]]) #[[ATTR4]]
-// CHECK1-NEXT: store i32 0, ptr [[DOTOMP_IS_LAST]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[IB]]) #[[ATTR4]]
// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
-// CHECK1-NEXT: [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4
// CHECK1-NEXT: call void @__kmpc_distribute_static_init_4(ptr @[[GLOB2:[0-9]+]], i32 [[TMP1]], i32 92, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
-// CHECK1-NEXT: [[TMP2:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP2:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
// CHECK1-NEXT: [[CMP:%.*]] = icmp sgt i32 [[TMP2]], 99
// CHECK1-NEXT: br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
// CHECK1: cond.true:
// CHECK1-NEXT: br label [[COND_END:%.*]]
// CHECK1: cond.false:
-// CHECK1-NEXT: [[TMP3:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP3:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
// CHECK1-NEXT: br label [[COND_END]]
// CHECK1: cond.end:
// CHECK1-NEXT: [[COND:%.*]] = phi i32 [ 99, [[COND_TRUE]] ], [ [[TMP3]], [[COND_FALSE]] ]
-// CHECK1-NEXT: store i32 [[COND]], ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT: [[TMP4:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT: store i32 [[TMP4]], ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: store i32 [[COND]], ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT: [[TMP4:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
+// CHECK1-NEXT: store i32 [[TMP4]], ptr [[DOTOMP_IV]], align 4
// CHECK1-NEXT: br label [[OMP_INNER_FOR_COND:%.*]]
// CHECK1: omp.inner.for.cond:
-// CHECK1-NEXT: [[TMP5:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT: [[TMP6:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP5:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT: [[TMP6:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
// CHECK1-NEXT: [[CMP1:%.*]] = icmp sle i32 [[TMP5]], [[TMP6]]
// CHECK1-NEXT: br i1 [[CMP1]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_COND_CLEANUP:%.*]]
// CHECK1: omp.inner.for.cond.cleanup:
// CHECK1-NEXT: br label [[OMP_INNER_FOR_END:%.*]]
// CHECK1: omp.inner.for.body:
-// CHECK1-NEXT: [[TMP7:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP7:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
// CHECK1-NEXT: [[MUL:%.*]] = mul nsw i32 [[TMP7]], 1
// CHECK1-NEXT: [[ADD:%.*]] = add nsw i32 0, [[MUL]]
-// CHECK1-NEXT: store i32 [[ADD]], ptr [[IB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: store i32 [[ADD]], ptr [[IB]], align 4
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[REF_TMP]]) #[[ATTR4]]
-// CHECK1-NEXT: store float 0.000000e+00, ptr [[REF_TMP]], align 4, !tbaa [[TBAA16:![0-9]+]]
+// CHECK1-NEXT: store float 0.000000e+00, ptr [[REF_TMP]], align 4
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[REF_TMP2]]) #[[ATTR4]]
-// CHECK1-NEXT: store float 0.000000e+00, ptr [[REF_TMP2]], align 4, !tbaa [[TBAA16]]
+// CHECK1-NEXT: store float 0.000000e+00, ptr [[REF_TMP2]], align 4
// CHECK1-NEXT: call void @_ZNSt7complexIfEC1ERKfS2_(ptr nonnull align 4 dereferenceable(8) [[PARTIAL_SUM]], ptr nonnull align 4 dereferenceable(4) [[REF_TMP]], ptr nonnull align 4 dereferenceable(4) [[REF_TMP2]]) #[[ATTR12:[0-9]+]]
// CHECK1-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr [[REF_TMP2]]) #[[ATTR4]]
// CHECK1-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr [[REF_TMP]]) #[[ATTR4]]
-// CHECK1-NEXT: [[TMP8:%.*]] = load i32, ptr [[IB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP8:%.*]] = load i32, ptr [[IB]], align 4
// CHECK1-NEXT: [[MUL3:%.*]] = mul nsw i32 [[TMP8]], 4
-// CHECK1-NEXT: store i32 [[MUL3]], ptr [[ISTART]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT: [[TMP9:%.*]] = load i32, ptr [[IB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: store i32 [[MUL3]], ptr [[ISTART]], align 4
+// CHECK1-NEXT: [[TMP9:%.*]] = load i32, ptr [[IB]], align 4
// CHECK1-NEXT: [[ADD4:%.*]] = add nsw i32 [[TMP9]], 1
// CHECK1-NEXT: [[MUL5:%.*]] = mul nsw i32 [[ADD4]], 4
-// CHECK1-NEXT: store i32 [[MUL5]], ptr [[IEND]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: store i32 [[MUL5]], ptr [[IEND]], align 4
// CHECK1-NEXT: [[TMP10:%.*]] = getelementptr inbounds [3 x ptr], ptr [[CAPTURED_VARS_ADDRS]], i64 0, i64 0
-// CHECK1-NEXT: store ptr [[ISTART]], ptr [[TMP10]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT: store ptr [[ISTART]], ptr [[TMP10]], align 8
// CHECK1-NEXT: [[TMP11:%.*]] = getelementptr inbounds [3 x ptr], ptr [[CAPTURED_VARS_ADDRS]], i64 0, i64 1
-// CHECK1-NEXT: store ptr [[IEND]], ptr [[TMP11]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT: store ptr [[IEND]], ptr [[TMP11]], align 8
// CHECK1-NEXT: [[TMP12:%.*]] = getelementptr inbounds [3 x ptr], ptr [[CAPTURED_VARS_ADDRS]], i64 0, i64 2
-// CHECK1-NEXT: store ptr [[PARTIAL_SUM]], ptr [[TMP12]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT: store ptr [[PARTIAL_SUM]], ptr [[TMP12]], align 8
// CHECK1-NEXT: call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP1]], i32 1, i32 -1, i32 -1, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z17complex_reductionIfEvv_l16_omp_outlined_omp_outlined, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z17complex_reductionIfEvv_l16_omp_outlined_omp_outlined_wrapper, ptr [[CAPTURED_VARS_ADDRS]], i64 3)
// CHECK1-NEXT: br label [[OMP_BODY_CONTINUE:%.*]]
// CHECK1: omp.body.continue:
// CHECK1-NEXT: br label [[OMP_INNER_FOR_INC:%.*]]
// CHECK1: omp.inner.for.inc:
-// CHECK1-NEXT: [[TMP13:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP13:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
// CHECK1-NEXT: [[ADD6:%.*]] = add nsw i32 [[TMP13]], 1
-// CHECK1-NEXT: store i32 [[ADD6]], ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: store i32 [[ADD6]], ptr [[DOTOMP_IV]], align 4
// CHECK1-NEXT: br label [[OMP_INNER_FOR_COND]]
// CHECK1: omp.inner.for.end:
// CHECK1-NEXT: br label [[OMP_LOOP_EXIT:%.*]]
@@ -161,9 +161,9 @@ void test() {
// CHECK1-NEXT: [[THIS_ADDR:%.*]] = alloca ptr, align 8
// CHECK1-NEXT: [[__RE_ADDR:%.*]] = alloca ptr, align 8
// CHECK1-NEXT: [[__IM_ADDR:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT: store ptr [[THIS]], ptr [[THIS_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT: store ptr [[__RE]], ptr [[__RE_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT: store ptr [[__IM]], ptr [[__IM_ADDR]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT: store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
+// CHECK1-NEXT: store ptr [[__RE]], ptr [[__RE_ADDR]], align 8
+// CHECK1-NEXT: store ptr [[__IM]], ptr [[__IM_ADDR]], align 8
// CHECK1-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[__RE_ADDR]], align 8
// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[__IM_ADDR]], align 8
@@ -197,79 +197,79 @@ void test() {
// CHECK1-NEXT: [[REF_TMP15:%.*]] = alloca float, align 4
// CHECK1-NEXT: [[REF_TMP16:%.*]] = alloca float, align 4
// CHECK1-NEXT: [[DOTOMP_REDUCTION_RED_LIST:%.*]] = alloca [1 x ptr], align 8
-// CHECK1-NEXT: store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT: store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT: store ptr [[ISTART]], ptr [[ISTART_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT: store ptr [[IEND]], ptr [[IEND_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT: store ptr [[PARTIAL_SUM]], ptr [[PARTIAL_SUM_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[ISTART_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[IEND_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[PARTIAL_SUM_ADDR]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT: store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8
+// CHECK1-NEXT: store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8
+// CHECK1-NEXT: store ptr [[ISTART]], ptr [[ISTART_ADDR]], align 8
+// CHECK1-NEXT: store ptr [[IEND]], ptr [[IEND_ADDR]], align 8
+// CHECK1-NEXT: store ptr [[PARTIAL_SUM]], ptr [[PARTIAL_SUM_ADDR]], align 8
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[ISTART_ADDR]], align 8
+// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[IEND_ADDR]], align 8
+// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[PARTIAL_SUM_ADDR]], align 8
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_IV]]) #[[ATTR4]]
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTCAPTURE_EXPR_]]) #[[ATTR4]]
-// CHECK1-NEXT: [[TMP3:%.*]] = load i32, ptr [[TMP0]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT: store i32 [[TMP3]], ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP3:%.*]] = load i32, ptr [[TMP0]], align 4
+// CHECK1-NEXT: store i32 [[TMP3]], ptr [[DOTCAPTURE_EXPR_]], align 4
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTCAPTURE_EXPR_1]]) #[[ATTR4]]
-// CHECK1-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT: store i32 [[TMP4]], ptr [[DOTCAPTURE_EXPR_1]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 4
+// CHECK1-NEXT: store i32 [[TMP4]], ptr [[DOTCAPTURE_EXPR_1]], align 4
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTCAPTURE_EXPR_2]]) #[[ATTR4]]
-// CHECK1-NEXT: [[TMP5:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT: [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP5:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK1-NEXT: [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
// CHECK1-NEXT: [[SUB:%.*]] = sub i32 [[TMP5]], [[TMP6]]
// CHECK1-NEXT: [[SUB3:%.*]] = sub i32 [[SUB]], 1
// CHECK1-NEXT: [[ADD:%.*]] = add i32 [[SUB3]], 1
// CHECK1-NEXT: [[DIV:%.*]] = udiv i32 [[ADD]], 1
// CHECK1-NEXT: [[SUB4:%.*]] = sub i32 [[DIV]], 1
-// CHECK1-NEXT: store i32 [[SUB4]], ptr [[DOTCAPTURE_EXPR_2]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: store i32 [[SUB4]], ptr [[DOTCAPTURE_EXPR_2]], align 4
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[I]]) #[[ATTR4]]
-// CHECK1-NEXT: [[TMP7:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT: store i32 [[TMP7]], ptr [[I]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP7:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT: store i32 [[TMP7]], ptr [[I]], align 4
// CHECK1-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr [[I]]) #[[ATTR4]]
-// CHECK1-NEXT: [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT: [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT: [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
// CHECK1-NEXT: [[CMP:%.*]] = icmp slt i32 [[TMP8]], [[TMP9]]
// CHECK1-NEXT: br i1 [[CMP]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END:%.*]]
// CHECK1: omp.precond.then:
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_LB]]) #[[ATTR4]]
-// CHECK1-NEXT: store i32 0, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: store i32 0, ptr [[DOTOMP_LB]], align 4
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_UB]]) #[[ATTR4]]
-// CHECK1-NEXT: [[TMP10:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT: store i32 [[TMP10]], ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP10:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT: store i32 [[TMP10]], ptr [[DOTOMP_UB]], align 4
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_STRIDE]]) #[[ATTR4]]
-// CHECK1-NEXT: store i32 1, ptr [[DOTOMP_STRIDE]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: store i32 1, ptr [[DOTOMP_STRIDE]], align 4
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_IS_LAST]]) #[[ATTR4]]
-// CHECK1-NEXT: store i32 0, ptr [[DOTOMP_IS_LAST]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[PARTIAL_SUM5]]) #[[ATTR4]]
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[REF_TMP]]) #[[ATTR4]]
-// CHECK1-NEXT: store float 0.000000e+00, ptr [[REF_TMP]], align 4, !tbaa [[TBAA16]]
+// CHECK1-NEXT: store float 0.000000e+00, ptr [[REF_TMP]], align 4
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[REF_TMP6]]) #[[ATTR4]]
-// CHECK1-NEXT: store float 0.000000e+00, ptr [[REF_TMP6]], align 4, !tbaa [[TBAA16]]
+// CHECK1-NEXT: store float 0.000000e+00, ptr [[REF_TMP6]], align 4
// CHECK1-NEXT: call void @_ZNSt7complexIfEC1ERKfS2_(ptr nonnull align 4 dereferenceable(8) [[PARTIAL_SUM5]], ptr nonnull align 4 dereferenceable(4) [[REF_TMP]], ptr nonnull align 4 dereferenceable(4) [[REF_TMP6]]) #[[ATTR12]]
// CHECK1-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr [[REF_TMP6]]) #[[ATTR4]]
// CHECK1-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr [[REF_TMP]]) #[[ATTR4]]
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[I7]]) #[[ATTR4]]
// CHECK1-NEXT: [[TMP11:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
-// CHECK1-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4
// CHECK1-NEXT: call void @__kmpc_for_static_init_4u(ptr @[[GLOB3:[0-9]+]], i32 [[TMP12]], i32 33, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
// CHECK1-NEXT: br label [[OMP_DISPATCH_COND:%.*]]
// CHECK1: omp.dispatch.cond:
-// CHECK1-NEXT: [[TMP13:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT: [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP13:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT: [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
// CHECK1-NEXT: [[CMP8:%.*]] = icmp ugt i32 [[TMP13]], [[TMP14]]
// CHECK1-NEXT: br i1 [[CMP8]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
// CHECK1: cond.true:
-// CHECK1-NEXT: [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
// CHECK1-NEXT: br label [[COND_END:%.*]]
// CHECK1: cond.false:
-// CHECK1-NEXT: [[TMP16:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP16:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
// CHECK1-NEXT: br label [[COND_END]]
// CHECK1: cond.end:
// CHECK1-NEXT: [[COND:%.*]] = phi i32 [ [[TMP15]], [[COND_TRUE]] ], [ [[TMP16]], [[COND_FALSE]] ]
-// CHECK1-NEXT: store i32 [[COND]], ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT: [[TMP17:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT: store i32 [[TMP17]], ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT: [[TMP18:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT: [[TMP19:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: store i32 [[COND]], ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT: [[TMP17:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
+// CHECK1-NEXT: store i32 [[TMP17]], ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT: [[TMP18:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT: [[TMP19:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
// CHECK1-NEXT: [[ADD9:%.*]] = add i32 [[TMP19]], 1
// CHECK1-NEXT: [[CMP10:%.*]] = icmp ult i32 [[TMP18]], [[ADD9]]
// CHECK1-NEXT: br i1 [[CMP10]], label [[OMP_DISPATCH_BODY:%.*]], label [[OMP_DISPATCH_CLEANUP:%.*]]
@@ -278,28 +278,28 @@ void test() {
// CHECK1: omp.dispatch.body:
// CHECK1-NEXT: br label [[OMP_INNER_FOR_COND:%.*]]
// CHECK1: omp.inner.for.cond:
-// CHECK1-NEXT: [[TMP20:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT: [[TMP21:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP20:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT: [[TMP21:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
// CHECK1-NEXT: [[ADD11:%.*]] = add i32 [[TMP21]], 1
// CHECK1-NEXT: [[CMP12:%.*]] = icmp ult i32 [[TMP20]], [[ADD11]]
// CHECK1-NEXT: br i1 [[CMP12]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_COND_CLEANUP:%.*]]
// CHECK1: omp.inner.for.cond.cleanup:
// CHECK1-NEXT: br label [[OMP_INNER_FOR_END:%.*]]
// CHECK1: omp.inner.for.body:
-// CHECK1-NEXT: [[TMP22:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT: [[TMP23:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP22:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT: [[TMP23:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
// CHECK1-NEXT: [[MUL:%.*]] = mul i32 [[TMP23]], 1
// CHECK1-NEXT: [[ADD13:%.*]] = add i32 [[TMP22]], [[MUL]]
-// CHECK1-NEXT: store i32 [[ADD13]], ptr [[I7]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: store i32 [[ADD13]], ptr [[I7]], align 4
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[REF_TMP14]]) #[[ATTR4]]
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[REF_TMP15]]) #[[ATTR4]]
-// CHECK1-NEXT: [[TMP24:%.*]] = load i32, ptr [[I7]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP24:%.*]] = load i32, ptr [[I7]], align 4
// CHECK1-NEXT: [[CONV:%.*]] = sitofp i32 [[TMP24]] to float
-// CHECK1-NEXT: store float [[CONV]], ptr [[REF_TMP15]], align 4, !tbaa [[TBAA16]]
+// CHECK1-NEXT: store float [[CONV]], ptr [[REF_TMP15]], align 4
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[REF_TMP16]]) #[[ATTR4]]
-// CHECK1-NEXT: [[TMP25:%.*]] = load i32, ptr [[I7]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP25:%.*]] = load i32, ptr [[I7]], align 4
// CHECK1-NEXT: [[CONV17:%.*]] = sitofp i32 [[TMP25]] to float
-// CHECK1-NEXT: store float [[CONV17]], ptr [[REF_TMP16]], align 4, !tbaa [[TBAA16]]
+// CHECK1-NEXT: store float [[CONV17]], ptr [[REF_TMP16]], align 4
// CHECK1-NEXT: call void @_ZNSt7complexIfEC1ERKfS2_(ptr nonnull align 4 dereferenceable(8) [[REF_TMP14]], ptr nonnull align 4 dereferenceable(4) [[REF_TMP15]], ptr nonnull align 4 dereferenceable(4) [[REF_TMP16]]) #[[ATTR12]]
// CHECK1-NEXT: [[CALL:%.*]] = call nonnull align 4 dereferenceable(8) ptr @_ZNSt7complexIfEpLIfEERS0_RKS_IT_E(ptr nonnull align 4 dereferenceable(8) [[PARTIAL_SUM5]], ptr nonnull align 4 dereferenceable(8) [[REF_TMP14]]) #[[ATTR12]]
// CHECK1-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr [[REF_TMP16]]) #[[ATTR4]]
@@ -309,25 +309,25 @@ void test() {
// CHECK1: omp.body.continue:
// CHECK1-NEXT: br label [[OMP_INNER_FOR_INC:%.*]]
// CHECK1: omp.inner.for.inc:
-// CHECK1-NEXT: [[TMP26:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP26:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
// CHECK1-NEXT: [[ADD18:%.*]] = add i32 [[TMP26]], 1
-// CHECK1-NEXT: store i32 [[ADD18]], ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: store i32 [[ADD18]], ptr [[DOTOMP_IV]], align 4
// CHECK1-NEXT: br label [[OMP_INNER_FOR_COND]]
// CHECK1: omp.inner.for.end:
// CHECK1-NEXT: br label [[OMP_DISPATCH_INC:%.*]]
// CHECK1: omp.dispatch.inc:
-// CHECK1-NEXT: [[TMP27:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT: [[TMP28:%.*]] = load i32, ptr [[DOTOMP_STRIDE]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP27:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
+// CHECK1-NEXT: [[TMP28:%.*]] = load i32, ptr [[DOTOMP_STRIDE]], align 4
// CHECK1-NEXT: [[ADD19:%.*]] = add i32 [[TMP27]], [[TMP28]]
-// CHECK1-NEXT: store i32 [[ADD19]], ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT: [[TMP29:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT: [[TMP30:%.*]] = load i32, ptr [[DOTOMP_STRIDE]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: store i32 [[ADD19]], ptr [[DOTOMP_LB]], align 4
+// CHECK1-NEXT: [[TMP29:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT: [[TMP30:%.*]] = load i32, ptr [[DOTOMP_STRIDE]], align 4
// CHECK1-NEXT: [[ADD20:%.*]] = add i32 [[TMP29]], [[TMP30]]
-// CHECK1-NEXT: store i32 [[ADD20]], ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: store i32 [[ADD20]], ptr [[DOTOMP_UB]], align 4
// CHECK1-NEXT: br label [[OMP_DISPATCH_COND]]
// CHECK1: omp.dispatch.end:
// CHECK1-NEXT: [[TMP31:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
-// CHECK1-NEXT: [[TMP32:%.*]] = load i32, ptr [[TMP31]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP32:%.*]] = load i32, ptr [[TMP31]], align 4
// CHECK1-NEXT: call void @__kmpc_for_static_fini(ptr @[[GLOB3]], i32 [[TMP32]])
// CHECK1-NEXT: [[TMP33:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOMP_REDUCTION_RED_LIST]], i64 0, i64 0
// CHECK1-NEXT: store ptr [[PARTIAL_SUM5]], ptr [[TMP33]], align 8
@@ -358,53 +358,53 @@ void test() {
// CHECK1-NEXT: entry:
// CHECK1-NEXT: [[THIS_ADDR:%.*]] = alloca ptr, align 8
// CHECK1-NEXT: [[__C_ADDR:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT: store ptr [[THIS]], ptr [[THIS_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT: store ptr [[__C]], ptr [[__C_ADDR]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT: store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
+// CHECK1-NEXT: store ptr [[__C]], ptr [[__C_ADDR]], align 8
// CHECK1-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[__C_ADDR]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[__C_ADDR]], align 8
// CHECK1-NEXT: [[CALL:%.*]] = call float @_ZNKSt7complexIfE4realEv(ptr nonnull align 4 dereferenceable(8) [[TMP0]]) #[[ATTR12]]
// CHECK1-NEXT: [[__RE_:%.*]] = getelementptr inbounds %"class.std::complex", ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT: [[TMP1:%.*]] = load float, ptr [[__RE_]], align 4, !tbaa [[TBAA18:![0-9]+]]
+// CHECK1-NEXT: [[TMP1:%.*]] = load float, ptr [[__RE_]], align 4
// CHECK1-NEXT: [[ADD:%.*]] = fadd float [[TMP1]], [[CALL]]
-// CHECK1-NEXT: store float [[ADD]], ptr [[__RE_]], align 4, !tbaa [[TBAA18]]
-// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[__C_ADDR]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT: store float [[ADD]], ptr [[__RE_]], align 4
+// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[__C_ADDR]], align 8
// CHECK1-NEXT: [[CALL2:%.*]] = call float @_ZNKSt7complexIfE4imagEv(ptr nonnull align 4 dereferenceable(8) [[TMP2]]) #[[ATTR12]]
// CHECK1-NEXT: [[__IM_:%.*]] = getelementptr inbounds %"class.std::complex", ptr [[THIS1]], i32 0, i32 1
-// CHECK1-NEXT: [[TMP3:%.*]] = load float, ptr [[__IM_]], align 4, !tbaa [[TBAA20:![0-9]+]]
+// CHECK1-NEXT: [[TMP3:%.*]] = load float, ptr [[__IM_]], align 4
// CHECK1-NEXT: [[ADD3:%.*]] = fadd float [[TMP3]], [[CALL2]]
-// CHECK1-NEXT: store float [[ADD3]], ptr [[__IM_]], align 4, !tbaa [[TBAA20]]
+// CHECK1-NEXT: store float [[ADD3]], ptr [[__IM_]], align 4
// CHECK1-NEXT: ret ptr [[THIS1]]
//
//
// CHECK1-LABEL: define {{[^@]+}}@_omp_reduction_shuffle_and_reduce_func
-// CHECK1-SAME: (ptr [[TMP0:%.*]], i16 signext [[TMP1:%.*]], i16 signext [[TMP2:%.*]], i16 signext [[TMP3:%.*]]) #[[ATTR7:[0-9]+]] {
+// CHECK1-SAME: (ptr noundef [[TMP0:%.*]], i16 noundef signext [[TMP1:%.*]], i16 noundef signext [[TMP2:%.*]], i16 noundef signext [[TMP3:%.*]]) #[[ATTR7:[0-9]+]] {
// CHECK1-NEXT: entry:
// CHECK1-NEXT: [[DOTADDR:%.*]] = alloca ptr, align 8
// CHECK1-NEXT: [[DOTADDR1:%.*]] = alloca i16, align 2
// CHECK1-NEXT: [[DOTADDR2:%.*]] = alloca i16, align 2
// CHECK1-NEXT: [[DOTADDR3:%.*]] = alloca i16, align 2
// CHECK1-NEXT: [[DOTOMP_REDUCTION_REMOTE_REDUCE_LIST:%.*]] = alloca [1 x ptr], align 8
-// CHECK1-NEXT: [[DOTOMP_REDUCTION_ELEMENT:%.*]] = alloca %"class.std::complex", align 4
-// CHECK1-NEXT: store ptr [[TMP0]], ptr [[DOTADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT: store i16 [[TMP1]], ptr [[DOTADDR1]], align 2, !tbaa [[TBAA21:![0-9]+]]
-// CHECK1-NEXT: store i16 [[TMP2]], ptr [[DOTADDR2]], align 2, !tbaa [[TBAA21]]
-// CHECK1-NEXT: store i16 [[TMP3]], ptr [[DOTADDR3]], align 2, !tbaa [[TBAA21]]
-// CHECK1-NEXT: [[TMP4:%.*]] = load ptr, ptr [[DOTADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT: [[TMP5:%.*]] = load i16, ptr [[DOTADDR1]], align 2, !tbaa [[TBAA21]]
-// CHECK1-NEXT: [[TMP6:%.*]] = load i16, ptr [[DOTADDR2]], align 2, !tbaa [[TBAA21]]
-// CHECK1-NEXT: [[TMP7:%.*]] = load i16, ptr [[DOTADDR3]], align 2, !tbaa [[TBAA21]]
+// CHECK1-NEXT: [[DOTOMP_REDUCTION_ELEMENT:%.*]] = alloca %"class.std::complex", align 8
+// CHECK1-NEXT: store ptr [[TMP0]], ptr [[DOTADDR]], align 8
+// CHECK1-NEXT: store i16 [[TMP1]], ptr [[DOTADDR1]], align 2
+// CHECK1-NEXT: store i16 [[TMP2]], ptr [[DOTADDR2]], align 2
+// CHECK1-NEXT: store i16 [[TMP3]], ptr [[DOTADDR3]], align 2
+// CHECK1-NEXT: [[TMP4:%.*]] = load ptr, ptr [[DOTADDR]], align 8
+// CHECK1-NEXT: [[TMP5:%.*]] = load i16, ptr [[DOTADDR1]], align 2
+// CHECK1-NEXT: [[TMP6:%.*]] = load i16, ptr [[DOTADDR2]], align 2
+// CHECK1-NEXT: [[TMP7:%.*]] = load i16, ptr [[DOTADDR3]], align 2
// CHECK1-NEXT: [[TMP8:%.*]] = getelementptr inbounds [1 x ptr], ptr [[TMP4]], i64 0, i64 0
// CHECK1-NEXT: [[TMP9:%.*]] = load ptr, ptr [[TMP8]], align 8
// CHECK1-NEXT: [[TMP10:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOMP_REDUCTION_REMOTE_REDUCE_LIST]], i64 0, i64 0
// CHECK1-NEXT: [[TMP11:%.*]] = getelementptr %"class.std::complex", ptr [[TMP9]], i64 1
-// CHECK1-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP9]], align 4
+// CHECK1-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP9]], align 8
// CHECK1-NEXT: [[TMP13:%.*]] = call i32 @__kmpc_get_warp_size()
// CHECK1-NEXT: [[TMP14:%.*]] = trunc i32 [[TMP13]] to i16
// CHECK1-NEXT: [[TMP15:%.*]] = call i64 @__kmpc_shuffle_int64(i64 [[TMP12]], i16 [[TMP6]], i16 [[TMP14]])
-// CHECK1-NEXT: store i64 [[TMP15]], ptr [[DOTOMP_REDUCTION_ELEMENT]], align 4
+// CHECK1-NEXT: store i64 [[TMP15]], ptr [[DOTOMP_REDUCTION_ELEMENT]], align 8
// CHECK1-NEXT: [[TMP16:%.*]] = getelementptr i64, ptr [[TMP9]], i64 1
// CHECK1-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[DOTOMP_REDUCTION_ELEMENT]], i64 1
-// CHECK1-NEXT: store ptr [[DOTOMP_REDUCTION_ELEMENT]], ptr [[TMP10]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT: store ptr [[DOTOMP_REDUCTION_ELEMENT]], ptr [[TMP10]], align 8
// CHECK1-NEXT: [[TMP18:%.*]] = icmp eq i16 [[TMP7]], 0
// CHECK1-NEXT: [[TMP19:%.*]] = icmp eq i16 [[TMP7]], 1
// CHECK1-NEXT: [[TMP20:%.*]] = icmp ult i16 [[TMP5]], [[TMP6]]
@@ -433,7 +433,7 @@ void test() {
// CHECK1-NEXT: [[TMP34:%.*]] = load ptr, ptr [[TMP33]], align 8
// CHECK1-NEXT: [[TMP35:%.*]] = getelementptr inbounds [1 x ptr], ptr [[TMP4]], i64 0, i64 0
// CHECK1-NEXT: [[TMP36:%.*]] = load ptr, ptr [[TMP35]], align 8
-// CHECK1-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[TMP36]], ptr align 4 [[TMP34]], i64 8, i1 false), !tbaa.struct [[TBAA_STRUCT23:![0-9]+]]
+// CHECK1-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP36]], ptr align 8 [[TMP34]], i64 8, i1 false)
// CHECK1-NEXT: br label [[IFCONT6:%.*]]
// CHECK1: else5:
// CHECK1-NEXT: br label [[IFCONT6]]
@@ -442,24 +442,24 @@ void test() {
//
//
// CHECK1-LABEL: define {{[^@]+}}@_omp_reduction_inter_warp_copy_func
-// CHECK1-SAME: (ptr [[TMP0:%.*]], i32 [[TMP1:%.*]]) #[[ATTR7]] {
+// CHECK1-SAME: (ptr noundef [[TMP0:%.*]], i32 noundef [[TMP1:%.*]]) #[[ATTR7]] {
// CHECK1-NEXT: entry:
// CHECK1-NEXT: [[DOTADDR:%.*]] = alloca ptr, align 8
// CHECK1-NEXT: [[DOTADDR1:%.*]] = alloca i32, align 4
// CHECK1-NEXT: [[DOTCNT_ADDR:%.*]] = alloca i32, align 4
// CHECK1-NEXT: [[TMP2:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]])
-// CHECK1-NEXT: store ptr [[TMP0]], ptr [[DOTADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT: store i32 [[TMP1]], ptr [[DOTADDR1]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: store ptr [[TMP0]], ptr [[DOTADDR]], align 8
+// CHECK1-NEXT: store i32 [[TMP1]], ptr [[DOTADDR1]], align 4
// CHECK1-NEXT: [[TMP3:%.*]] = call i32 @__kmpc_get_hardware_thread_id_in_block()
// CHECK1-NEXT: [[TMP4:%.*]] = call i32 @__kmpc_get_hardware_thread_id_in_block()
// CHECK1-NEXT: [[NVPTX_LANE_ID:%.*]] = and i32 [[TMP4]], 31
// CHECK1-NEXT: [[TMP5:%.*]] = call i32 @__kmpc_get_hardware_thread_id_in_block()
// CHECK1-NEXT: [[NVPTX_WARP_ID:%.*]] = ashr i32 [[TMP5]], 5
// CHECK1-NEXT: [[TMP6:%.*]] = load ptr, ptr [[DOTADDR]], align 8
-// CHECK1-NEXT: store i32 0, ptr [[DOTCNT_ADDR]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: store i32 0, ptr [[DOTCNT_ADDR]], align 4
// CHECK1-NEXT: br label [[PRECOND:%.*]]
// CHECK1: precond:
-// CHECK1-NEXT: [[TMP7:%.*]] = load i32, ptr [[DOTCNT_ADDR]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP7:%.*]] = load i32, ptr [[DOTCNT_ADDR]], align 4
// CHECK1-NEXT: [[TMP8:%.*]] = icmp ult i32 [[TMP7]], 2
// CHECK1-NEXT: br i1 [[TMP8]], label [[BODY:%.*]], label [[EXIT:%.*]]
// CHECK1: body:
@@ -468,7 +468,7 @@ void test() {
// CHECK1-NEXT: br i1 [[WARP_MASTER]], label [[THEN:%.*]], label [[ELSE:%.*]]
// CHECK1: then:
// CHECK1-NEXT: [[TMP9:%.*]] = getelementptr inbounds [1 x ptr], ptr [[TMP6]], i64 0, i64 0
-// CHECK1-NEXT: [[TMP10:%.*]] = load ptr, ptr [[TMP9]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT: [[TMP10:%.*]] = load ptr, ptr [[TMP9]], align 8
// CHECK1-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[TMP10]], i32 [[TMP7]]
// CHECK1-NEXT: [[TMP12:%.*]] = getelementptr inbounds [32 x i32], ptr addrspace(3) @__openmp_nvptx_data_transfer_temporary_storage, i64 0, i32 [[NVPTX_WARP_ID]]
// CHECK1-NEXT: [[TMP13:%.*]] = load i32, ptr [[TMP11]], align 4
@@ -478,45 +478,45 @@ void test() {
// CHECK1-NEXT: br label [[IFCONT]]
// CHECK1: ifcont:
// CHECK1-NEXT: call void @__kmpc_barrier(ptr @[[GLOB4]], i32 [[TMP2]])
-// CHECK1-NEXT: [[TMP14:%.*]] = load i32, ptr [[DOTADDR1]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP14:%.*]] = load i32, ptr [[DOTADDR1]], align 4
// CHECK1-NEXT: [[IS_ACTIVE_THREAD:%.*]] = icmp ult i32 [[TMP3]], [[TMP14]]
// CHECK1-NEXT: br i1 [[IS_ACTIVE_THREAD]], label [[THEN2:%.*]], label [[ELSE3:%.*]]
// CHECK1: then2:
// CHECK1-NEXT: [[TMP15:%.*]] = getelementptr inbounds [32 x i32], ptr addrspace(3) @__openmp_nvptx_data_transfer_temporary_storage, i64 0, i32 [[TMP3]]
// CHECK1-NEXT: [[TMP16:%.*]] = getelementptr inbounds [1 x ptr], ptr [[TMP6]], i64 0, i64 0
-// CHECK1-NEXT: [[TMP17:%.*]] = load ptr, ptr [[TMP16]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT: [[TMP17:%.*]] = load ptr, ptr [[TMP16]], align 8
// CHECK1-NEXT: [[TMP18:%.*]] = getelementptr i32, ptr [[TMP17]], i32 [[TMP7]]
-// CHECK1-NEXT: [[TMP19:%.*]] = load volatile i32, ptr addrspace(3) [[TMP15]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT: store i32 [[TMP19]], ptr [[TMP18]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP19:%.*]] = load volatile i32, ptr addrspace(3) [[TMP15]], align 4
+// CHECK1-NEXT: store i32 [[TMP19]], ptr [[TMP18]], align 4
// CHECK1-NEXT: br label [[IFCONT4:%.*]]
// CHECK1: else3:
// CHECK1-NEXT: br label [[IFCONT4]]
// CHECK1: ifcont4:
// CHECK1-NEXT: [[TMP20:%.*]] = add nsw i32 [[TMP7]], 1
-// CHECK1-NEXT: store i32 [[TMP20]], ptr [[DOTCNT_ADDR]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: store i32 [[TMP20]], ptr [[DOTCNT_ADDR]], align 4
// CHECK1-NEXT: br label [[PRECOND]]
// CHECK1: exit:
// CHECK1-NEXT: ret void
//
//
// CHECK1-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z17complex_reductionIfEvv_l16_omp_outlined_omp_outlined_wrapper
-// CHECK1-SAME: (i16 zeroext [[TMP0:%.*]], i32 [[TMP1:%.*]]) #[[ATTR7]] {
+// CHECK1-SAME: (i16 zeroext [[TMP0:%.*]], i32 [[TMP1:%.*]]) #[[ATTR8:[0-9]+]] {
// CHECK1-NEXT: entry:
// CHECK1-NEXT: [[DOTADDR:%.*]] = alloca i16, align 2
// CHECK1-NEXT: [[DOTADDR1:%.*]] = alloca i32, align 4
// CHECK1-NEXT: [[DOTZERO_ADDR:%.*]] = alloca i32, align 4
// CHECK1-NEXT: [[GLOBAL_ARGS:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT: store i16 [[TMP0]], ptr [[DOTADDR]], align 2, !tbaa [[TBAA21]]
-// CHECK1-NEXT: store i32 [[TMP1]], ptr [[DOTADDR1]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: store i16 [[TMP0]], ptr [[DOTADDR]], align 2
+// CHECK1-NEXT: store i32 [[TMP1]], ptr [[DOTADDR1]], align 4
// CHECK1-NEXT: store i32 0, ptr [[DOTZERO_ADDR]], align 4
// CHECK1-NEXT: call void @__kmpc_get_shared_variables(ptr [[GLOBAL_ARGS]])
// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[GLOBAL_ARGS]], align 8
// CHECK1-NEXT: [[TMP3:%.*]] = getelementptr inbounds ptr, ptr [[TMP2]], i64 0
-// CHECK1-NEXT: [[TMP4:%.*]] = load ptr, ptr [[TMP3]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT: [[TMP4:%.*]] = load ptr, ptr [[TMP3]], align 8
// CHECK1-NEXT: [[TMP5:%.*]] = getelementptr inbounds ptr, ptr [[TMP2]], i64 1
-// CHECK1-NEXT: [[TMP6:%.*]] = load ptr, ptr [[TMP5]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT: [[TMP6:%.*]] = load ptr, ptr [[TMP5]], align 8
// CHECK1-NEXT: [[TMP7:%.*]] = getelementptr inbounds ptr, ptr [[TMP2]], i64 2
-// CHECK1-NEXT: [[TMP8:%.*]] = load ptr, ptr [[TMP7]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT: [[TMP8:%.*]] = load ptr, ptr [[TMP7]], align 8
// CHECK1-NEXT: call void @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z17complex_reductionIfEvv_l16_omp_outlined_omp_outlined(ptr [[DOTADDR1]], ptr [[DOTZERO_ADDR]], ptr [[TMP4]], ptr [[TMP6]], ptr [[TMP8]]) #[[ATTR4]]
// CHECK1-NEXT: ret void
//
@@ -527,14 +527,14 @@ void test() {
// CHECK1-NEXT: [[DYN_PTR_ADDR:%.*]] = alloca ptr, align 8
// CHECK1-NEXT: [[DOTZERO_ADDR:%.*]] = alloca i32, align 4
// CHECK1-NEXT: [[DOTTHREADID_TEMP_:%.*]] = alloca i32, align 4
-// CHECK1-NEXT: store ptr [[DYN_PTR]], ptr [[DYN_PTR_ADDR]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT: store ptr [[DYN_PTR]], ptr [[DYN_PTR_ADDR]], align 8
// CHECK1-NEXT: [[TMP0:%.*]] = call i32 @__kmpc_target_init(ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z17complex_reductionIdEvv_l16_kernel_environment, ptr [[DYN_PTR]])
// CHECK1-NEXT: [[EXEC_USER_CODE:%.*]] = icmp eq i32 [[TMP0]], -1
// CHECK1-NEXT: br i1 [[EXEC_USER_CODE]], label [[USER_CODE_ENTRY:%.*]], label [[WORKER_EXIT:%.*]]
// CHECK1: user_code.entry:
// CHECK1-NEXT: [[TMP1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]])
// CHECK1-NEXT: store i32 0, ptr [[DOTZERO_ADDR]], align 4
-// CHECK1-NEXT: store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: store i32 [[TMP1]], ptr [[DOTTHREADID_TEMP_]], align 4
// CHECK1-NEXT: call void @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z17complex_reductionIdEvv_l16_omp_outlined(ptr [[DOTTHREADID_TEMP_]], ptr [[DOTZERO_ADDR]]) #[[ATTR4]]
// CHECK1-NEXT: call void @__kmpc_target_deinit()
// CHECK1-NEXT: ret void
@@ -557,78 +557,78 @@ void test() {
// CHECK1-NEXT: [[REF_TMP:%.*]] = alloca double, align 8
// CHECK1-NEXT: [[REF_TMP2:%.*]] = alloca double, align 8
// CHECK1-NEXT: [[CAPTURED_VARS_ADDRS:%.*]] = alloca [3 x ptr], align 8
-// CHECK1-NEXT: store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT: store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT: store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8
+// CHECK1-NEXT: store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8
// CHECK1-NEXT: [[ISTART:%.*]] = call align 16 ptr @__kmpc_alloc_shared(i64 4)
// CHECK1-NEXT: [[IEND:%.*]] = call align 16 ptr @__kmpc_alloc_shared(i64 4)
// CHECK1-NEXT: [[PARTIAL_SUM:%.*]] = call align 16 ptr @__kmpc_alloc_shared(i64 16)
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_IV]]) #[[ATTR4]]
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_LB]]) #[[ATTR4]]
-// CHECK1-NEXT: store i32 0, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: store i32 0, ptr [[DOTOMP_LB]], align 4
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_UB]]) #[[ATTR4]]
-// CHECK1-NEXT: store i32 99, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: store i32 99, ptr [[DOTOMP_UB]], align 4
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_STRIDE]]) #[[ATTR4]]
-// CHECK1-NEXT: store i32 1, ptr [[DOTOMP_STRIDE]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: store i32 1, ptr [[DOTOMP_STRIDE]], align 4
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_IS_LAST]]) #[[ATTR4]]
-// CHECK1-NEXT: store i32 0, ptr [[DOTOMP_IS_LAST]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[IB]]) #[[ATTR4]]
// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
-// CHECK1-NEXT: [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4
// CHECK1-NEXT: call void @__kmpc_distribute_static_init_4(ptr @[[GLOB2]], i32 [[TMP1]], i32 92, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
-// CHECK1-NEXT: [[TMP2:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP2:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
// CHECK1-NEXT: [[CMP:%.*]] = icmp sgt i32 [[TMP2]], 99
// CHECK1-NEXT: br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
// CHECK1: cond.true:
// CHECK1-NEXT: br label [[COND_END:%.*]]
// CHECK1: cond.false:
-// CHECK1-NEXT: [[TMP3:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP3:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
// CHECK1-NEXT: br label [[COND_END]]
// CHECK1: cond.end:
// CHECK1-NEXT: [[COND:%.*]] = phi i32 [ 99, [[COND_TRUE]] ], [ [[TMP3]], [[COND_FALSE]] ]
-// CHECK1-NEXT: store i32 [[COND]], ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT: [[TMP4:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT: store i32 [[TMP4]], ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: store i32 [[COND]], ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT: [[TMP4:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
+// CHECK1-NEXT: store i32 [[TMP4]], ptr [[DOTOMP_IV]], align 4
// CHECK1-NEXT: br label [[OMP_INNER_FOR_COND:%.*]]
// CHECK1: omp.inner.for.cond:
-// CHECK1-NEXT: [[TMP5:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT: [[TMP6:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP5:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT: [[TMP6:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
// CHECK1-NEXT: [[CMP1:%.*]] = icmp sle i32 [[TMP5]], [[TMP6]]
// CHECK1-NEXT: br i1 [[CMP1]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_COND_CLEANUP:%.*]]
// CHECK1: omp.inner.for.cond.cleanup:
// CHECK1-NEXT: br label [[OMP_INNER_FOR_END:%.*]]
// CHECK1: omp.inner.for.body:
-// CHECK1-NEXT: [[TMP7:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP7:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
// CHECK1-NEXT: [[MUL:%.*]] = mul nsw i32 [[TMP7]], 1
// CHECK1-NEXT: [[ADD:%.*]] = add nsw i32 0, [[MUL]]
-// CHECK1-NEXT: store i32 [[ADD]], ptr [[IB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: store i32 [[ADD]], ptr [[IB]], align 4
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[REF_TMP]]) #[[ATTR4]]
-// CHECK1-NEXT: store double 0.000000e+00, ptr [[REF_TMP]], align 8, !tbaa [[TBAA24:![0-9]+]]
+// CHECK1-NEXT: store double 0.000000e+00, ptr [[REF_TMP]], align 8
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[REF_TMP2]]) #[[ATTR4]]
-// CHECK1-NEXT: store double 0.000000e+00, ptr [[REF_TMP2]], align 8, !tbaa [[TBAA24]]
+// CHECK1-NEXT: store double 0.000000e+00, ptr [[REF_TMP2]], align 8
// CHECK1-NEXT: call void @_ZNSt7complexIdEC1ERKdS2_(ptr nonnull align 8 dereferenceable(16) [[PARTIAL_SUM]], ptr nonnull align 8 dereferenceable(8) [[REF_TMP]], ptr nonnull align 8 dereferenceable(8) [[REF_TMP2]]) #[[ATTR12]]
// CHECK1-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[REF_TMP2]]) #[[ATTR4]]
// CHECK1-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[REF_TMP]]) #[[ATTR4]]
-// CHECK1-NEXT: [[TMP8:%.*]] = load i32, ptr [[IB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP8:%.*]] = load i32, ptr [[IB]], align 4
// CHECK1-NEXT: [[MUL3:%.*]] = mul nsw i32 [[TMP8]], 4
-// CHECK1-NEXT: store i32 [[MUL3]], ptr [[ISTART]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT: [[TMP9:%.*]] = load i32, ptr [[IB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: store i32 [[MUL3]], ptr [[ISTART]], align 4
+// CHECK1-NEXT: [[TMP9:%.*]] = load i32, ptr [[IB]], align 4
// CHECK1-NEXT: [[ADD4:%.*]] = add nsw i32 [[TMP9]], 1
// CHECK1-NEXT: [[MUL5:%.*]] = mul nsw i32 [[ADD4]], 4
-// CHECK1-NEXT: store i32 [[MUL5]], ptr [[IEND]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: store i32 [[MUL5]], ptr [[IEND]], align 4
// CHECK1-NEXT: [[TMP10:%.*]] = getelementptr inbounds [3 x ptr], ptr [[CAPTURED_VARS_ADDRS]], i64 0, i64 0
-// CHECK1-NEXT: store ptr [[ISTART]], ptr [[TMP10]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT: store ptr [[ISTART]], ptr [[TMP10]], align 8
// CHECK1-NEXT: [[TMP11:%.*]] = getelementptr inbounds [3 x ptr], ptr [[CAPTURED_VARS_ADDRS]], i64 0, i64 1
-// CHECK1-NEXT: store ptr [[IEND]], ptr [[TMP11]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT: store ptr [[IEND]], ptr [[TMP11]], align 8
// CHECK1-NEXT: [[TMP12:%.*]] = getelementptr inbounds [3 x ptr], ptr [[CAPTURED_VARS_ADDRS]], i64 0, i64 2
-// CHECK1-NEXT: store ptr [[PARTIAL_SUM]], ptr [[TMP12]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT: store ptr [[PARTIAL_SUM]], ptr [[TMP12]], align 8
// CHECK1-NEXT: call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP1]], i32 1, i32 -1, i32 -1, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z17complex_reductionIdEvv_l16_omp_outlined_omp_outlined, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z17complex_reductionIdEvv_l16_omp_outlined_omp_outlined_wrapper, ptr [[CAPTURED_VARS_ADDRS]], i64 3)
// CHECK1-NEXT: br label [[OMP_BODY_CONTINUE:%.*]]
// CHECK1: omp.body.continue:
// CHECK1-NEXT: br label [[OMP_INNER_FOR_INC:%.*]]
// CHECK1: omp.inner.for.inc:
-// CHECK1-NEXT: [[TMP13:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP13:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
// CHECK1-NEXT: [[ADD6:%.*]] = add nsw i32 [[TMP13]], 1
-// CHECK1-NEXT: store i32 [[ADD6]], ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: store i32 [[ADD6]], ptr [[DOTOMP_IV]], align 4
// CHECK1-NEXT: br label [[OMP_INNER_FOR_COND]]
// CHECK1: omp.inner.for.end:
// CHECK1-NEXT: br label [[OMP_LOOP_EXIT:%.*]]
@@ -652,9 +652,9 @@ void test() {
// CHECK1-NEXT: [[THIS_ADDR:%.*]] = alloca ptr, align 8
// CHECK1-NEXT: [[__RE_ADDR:%.*]] = alloca ptr, align 8
// CHECK1-NEXT: [[__IM_ADDR:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT: store ptr [[THIS]], ptr [[THIS_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT: store ptr [[__RE]], ptr [[__RE_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT: store ptr [[__IM]], ptr [[__IM_ADDR]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT: store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
+// CHECK1-NEXT: store ptr [[__RE]], ptr [[__RE_ADDR]], align 8
+// CHECK1-NEXT: store ptr [[__IM]], ptr [[__IM_ADDR]], align 8
// CHECK1-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[__RE_ADDR]], align 8
// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[__IM_ADDR]], align 8
@@ -688,79 +688,79 @@ void test() {
// CHECK1-NEXT: [[REF_TMP15:%.*]] = alloca double, align 8
// CHECK1-NEXT: [[REF_TMP16:%.*]] = alloca double, align 8
// CHECK1-NEXT: [[DOTOMP_REDUCTION_RED_LIST:%.*]] = alloca [1 x ptr], align 8
-// CHECK1-NEXT: store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT: store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT: store ptr [[ISTART]], ptr [[ISTART_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT: store ptr [[IEND]], ptr [[IEND_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT: store ptr [[PARTIAL_SUM]], ptr [[PARTIAL_SUM_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[ISTART_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[IEND_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[PARTIAL_SUM_ADDR]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT: store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8
+// CHECK1-NEXT: store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8
+// CHECK1-NEXT: store ptr [[ISTART]], ptr [[ISTART_ADDR]], align 8
+// CHECK1-NEXT: store ptr [[IEND]], ptr [[IEND_ADDR]], align 8
+// CHECK1-NEXT: store ptr [[PARTIAL_SUM]], ptr [[PARTIAL_SUM_ADDR]], align 8
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[ISTART_ADDR]], align 8
+// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[IEND_ADDR]], align 8
+// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[PARTIAL_SUM_ADDR]], align 8
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_IV]]) #[[ATTR4]]
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTCAPTURE_EXPR_]]) #[[ATTR4]]
-// CHECK1-NEXT: [[TMP3:%.*]] = load i32, ptr [[TMP0]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT: store i32 [[TMP3]], ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP3:%.*]] = load i32, ptr [[TMP0]], align 4
+// CHECK1-NEXT: store i32 [[TMP3]], ptr [[DOTCAPTURE_EXPR_]], align 4
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTCAPTURE_EXPR_1]]) #[[ATTR4]]
-// CHECK1-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT: store i32 [[TMP4]], ptr [[DOTCAPTURE_EXPR_1]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 4
+// CHECK1-NEXT: store i32 [[TMP4]], ptr [[DOTCAPTURE_EXPR_1]], align 4
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTCAPTURE_EXPR_2]]) #[[ATTR4]]
-// CHECK1-NEXT: [[TMP5:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT: [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP5:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
+// CHECK1-NEXT: [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
// CHECK1-NEXT: [[SUB:%.*]] = sub i32 [[TMP5]], [[TMP6]]
// CHECK1-NEXT: [[SUB3:%.*]] = sub i32 [[SUB]], 1
// CHECK1-NEXT: [[ADD:%.*]] = add i32 [[SUB3]], 1
// CHECK1-NEXT: [[DIV:%.*]] = udiv i32 [[ADD]], 1
// CHECK1-NEXT: [[SUB4:%.*]] = sub i32 [[DIV]], 1
-// CHECK1-NEXT: store i32 [[SUB4]], ptr [[DOTCAPTURE_EXPR_2]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: store i32 [[SUB4]], ptr [[DOTCAPTURE_EXPR_2]], align 4
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[I]]) #[[ATTR4]]
-// CHECK1-NEXT: [[TMP7:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT: store i32 [[TMP7]], ptr [[I]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP7:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT: store i32 [[TMP7]], ptr [[I]], align 4
// CHECK1-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr [[I]]) #[[ATTR4]]
-// CHECK1-NEXT: [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT: [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT: [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
// CHECK1-NEXT: [[CMP:%.*]] = icmp slt i32 [[TMP8]], [[TMP9]]
// CHECK1-NEXT: br i1 [[CMP]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END:%.*]]
// CHECK1: omp.precond.then:
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_LB]]) #[[ATTR4]]
-// CHECK1-NEXT: store i32 0, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: store i32 0, ptr [[DOTOMP_LB]], align 4
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_UB]]) #[[ATTR4]]
-// CHECK1-NEXT: [[TMP10:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT: store i32 [[TMP10]], ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP10:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT: store i32 [[TMP10]], ptr [[DOTOMP_UB]], align 4
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_STRIDE]]) #[[ATTR4]]
-// CHECK1-NEXT: store i32 1, ptr [[DOTOMP_STRIDE]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: store i32 1, ptr [[DOTOMP_STRIDE]], align 4
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_IS_LAST]]) #[[ATTR4]]
-// CHECK1-NEXT: store i32 0, ptr [[DOTOMP_IS_LAST]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 16, ptr [[PARTIAL_SUM5]]) #[[ATTR4]]
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[REF_TMP]]) #[[ATTR4]]
-// CHECK1-NEXT: store double 0.000000e+00, ptr [[REF_TMP]], align 8, !tbaa [[TBAA24]]
+// CHECK1-NEXT: store double 0.000000e+00, ptr [[REF_TMP]], align 8
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[REF_TMP6]]) #[[ATTR4]]
-// CHECK1-NEXT: store double 0.000000e+00, ptr [[REF_TMP6]], align 8, !tbaa [[TBAA24]]
+// CHECK1-NEXT: store double 0.000000e+00, ptr [[REF_TMP6]], align 8
// CHECK1-NEXT: call void @_ZNSt7complexIdEC1ERKdS2_(ptr nonnull align 8 dereferenceable(16) [[PARTIAL_SUM5]], ptr nonnull align 8 dereferenceable(8) [[REF_TMP]], ptr nonnull align 8 dereferenceable(8) [[REF_TMP6]]) #[[ATTR12]]
// CHECK1-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[REF_TMP6]]) #[[ATTR4]]
// CHECK1-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[REF_TMP]]) #[[ATTR4]]
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[I7]]) #[[ATTR4]]
// CHECK1-NEXT: [[TMP11:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
-// CHECK1-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4
// CHECK1-NEXT: call void @__kmpc_for_static_init_4u(ptr @[[GLOB3]], i32 [[TMP12]], i32 33, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
// CHECK1-NEXT: br label [[OMP_DISPATCH_COND:%.*]]
// CHECK1: omp.dispatch.cond:
-// CHECK1-NEXT: [[TMP13:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT: [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP13:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT: [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
// CHECK1-NEXT: [[CMP8:%.*]] = icmp ugt i32 [[TMP13]], [[TMP14]]
// CHECK1-NEXT: br i1 [[CMP8]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
// CHECK1: cond.true:
-// CHECK1-NEXT: [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
// CHECK1-NEXT: br label [[COND_END:%.*]]
// CHECK1: cond.false:
-// CHECK1-NEXT: [[TMP16:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP16:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
// CHECK1-NEXT: br label [[COND_END]]
// CHECK1: cond.end:
// CHECK1-NEXT: [[COND:%.*]] = phi i32 [ [[TMP15]], [[COND_TRUE]] ], [ [[TMP16]], [[COND_FALSE]] ]
-// CHECK1-NEXT: store i32 [[COND]], ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT: [[TMP17:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT: store i32 [[TMP17]], ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT: [[TMP18:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT: [[TMP19:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: store i32 [[COND]], ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT: [[TMP17:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
+// CHECK1-NEXT: store i32 [[TMP17]], ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT: [[TMP18:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT: [[TMP19:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
// CHECK1-NEXT: [[ADD9:%.*]] = add i32 [[TMP19]], 1
// CHECK1-NEXT: [[CMP10:%.*]] = icmp ult i32 [[TMP18]], [[ADD9]]
// CHECK1-NEXT: br i1 [[CMP10]], label [[OMP_DISPATCH_BODY:%.*]], label [[OMP_DISPATCH_CLEANUP:%.*]]
@@ -769,28 +769,28 @@ void test() {
// CHECK1: omp.dispatch.body:
// CHECK1-NEXT: br label [[OMP_INNER_FOR_COND:%.*]]
// CHECK1: omp.inner.for.cond:
-// CHECK1-NEXT: [[TMP20:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT: [[TMP21:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP20:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT: [[TMP21:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
// CHECK1-NEXT: [[ADD11:%.*]] = add i32 [[TMP21]], 1
// CHECK1-NEXT: [[CMP12:%.*]] = icmp ult i32 [[TMP20]], [[ADD11]]
// CHECK1-NEXT: br i1 [[CMP12]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_COND_CLEANUP:%.*]]
// CHECK1: omp.inner.for.cond.cleanup:
// CHECK1-NEXT: br label [[OMP_INNER_FOR_END:%.*]]
// CHECK1: omp.inner.for.body:
-// CHECK1-NEXT: [[TMP22:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT: [[TMP23:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP22:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT: [[TMP23:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
// CHECK1-NEXT: [[MUL:%.*]] = mul i32 [[TMP23]], 1
// CHECK1-NEXT: [[ADD13:%.*]] = add i32 [[TMP22]], [[MUL]]
-// CHECK1-NEXT: store i32 [[ADD13]], ptr [[I7]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: store i32 [[ADD13]], ptr [[I7]], align 4
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 16, ptr [[REF_TMP14]]) #[[ATTR4]]
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[REF_TMP15]]) #[[ATTR4]]
-// CHECK1-NEXT: [[TMP24:%.*]] = load i32, ptr [[I7]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP24:%.*]] = load i32, ptr [[I7]], align 4
// CHECK1-NEXT: [[CONV:%.*]] = sitofp i32 [[TMP24]] to double
-// CHECK1-NEXT: store double [[CONV]], ptr [[REF_TMP15]], align 8, !tbaa [[TBAA24]]
+// CHECK1-NEXT: store double [[CONV]], ptr [[REF_TMP15]], align 8
// CHECK1-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[REF_TMP16]]) #[[ATTR4]]
-// CHECK1-NEXT: [[TMP25:%.*]] = load i32, ptr [[I7]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP25:%.*]] = load i32, ptr [[I7]], align 4
// CHECK1-NEXT: [[CONV17:%.*]] = sitofp i32 [[TMP25]] to double
-// CHECK1-NEXT: store double [[CONV17]], ptr [[REF_TMP16]], align 8, !tbaa [[TBAA24]]
+// CHECK1-NEXT: store double [[CONV17]], ptr [[REF_TMP16]], align 8
// CHECK1-NEXT: call void @_ZNSt7complexIdEC1ERKdS2_(ptr nonnull align 8 dereferenceable(16) [[REF_TMP14]], ptr nonnull align 8 dereferenceable(8) [[REF_TMP15]], ptr nonnull align 8 dereferenceable(8) [[REF_TMP16]]) #[[ATTR12]]
// CHECK1-NEXT: [[CALL:%.*]] = call nonnull align 8 dereferenceable(16) ptr @_ZNSt7complexIdEpLIdEERS0_RKS_IT_E(ptr nonnull align 8 dereferenceable(16) [[PARTIAL_SUM5]], ptr nonnull align 8 dereferenceable(16) [[REF_TMP14]]) #[[ATTR12]]
// CHECK1-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[REF_TMP16]]) #[[ATTR4]]
@@ -800,25 +800,25 @@ void test() {
// CHECK1: omp.body.continue:
// CHECK1-NEXT: br label [[OMP_INNER_FOR_INC:%.*]]
// CHECK1: omp.inner.for.inc:
-// CHECK1-NEXT: [[TMP26:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP26:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
// CHECK1-NEXT: [[ADD18:%.*]] = add i32 [[TMP26]], 1
-// CHECK1-NEXT: store i32 [[ADD18]], ptr [[DOTOMP_IV]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: store i32 [[ADD18]], ptr [[DOTOMP_IV]], align 4
// CHECK1-NEXT: br label [[OMP_INNER_FOR_COND]]
// CHECK1: omp.inner.for.end:
// CHECK1-NEXT: br label [[OMP_DISPATCH_INC:%.*]]
// CHECK1: omp.dispatch.inc:
-// CHECK1-NEXT: [[TMP27:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT: [[TMP28:%.*]] = load i32, ptr [[DOTOMP_STRIDE]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP27:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
+// CHECK1-NEXT: [[TMP28:%.*]] = load i32, ptr [[DOTOMP_STRIDE]], align 4
// CHECK1-NEXT: [[ADD19:%.*]] = add i32 [[TMP27]], [[TMP28]]
-// CHECK1-NEXT: store i32 [[ADD19]], ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT: [[TMP29:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT: [[TMP30:%.*]] = load i32, ptr [[DOTOMP_STRIDE]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: store i32 [[ADD19]], ptr [[DOTOMP_LB]], align 4
+// CHECK1-NEXT: [[TMP29:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT: [[TMP30:%.*]] = load i32, ptr [[DOTOMP_STRIDE]], align 4
// CHECK1-NEXT: [[ADD20:%.*]] = add i32 [[TMP29]], [[TMP30]]
-// CHECK1-NEXT: store i32 [[ADD20]], ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: store i32 [[ADD20]], ptr [[DOTOMP_UB]], align 4
// CHECK1-NEXT: br label [[OMP_DISPATCH_COND]]
// CHECK1: omp.dispatch.end:
// CHECK1-NEXT: [[TMP31:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
-// CHECK1-NEXT: [[TMP32:%.*]] = load i32, ptr [[TMP31]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP32:%.*]] = load i32, ptr [[TMP31]], align 4
// CHECK1-NEXT: call void @__kmpc_for_static_fini(ptr @[[GLOB3]], i32 [[TMP32]])
// CHECK1-NEXT: [[TMP33:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOMP_REDUCTION_RED_LIST]], i64 0, i64 0
// CHECK1-NEXT: store ptr [[PARTIAL_SUM5]], ptr [[TMP33]], align 8
@@ -849,26 +849,26 @@ void test() {
// CHECK1-NEXT: entry:
// CHECK1-NEXT: [[THIS_ADDR:%.*]] = alloca ptr, align 8
// CHECK1-NEXT: [[__C_ADDR:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT: store ptr [[THIS]], ptr [[THIS_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT: store ptr [[__C]], ptr [[__C_ADDR]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT: store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
+// CHECK1-NEXT: store ptr [[__C]], ptr [[__C_ADDR]], align 8
// CHECK1-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[__C_ADDR]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[__C_ADDR]], align 8
// CHECK1-NEXT: [[CALL:%.*]] = call double @_ZNKSt7complexIdE4realEv(ptr nonnull align 8 dereferenceable(16) [[TMP0]]) #[[ATTR12]]
// CHECK1-NEXT: [[__RE_:%.*]] = getelementptr inbounds %"class.std::complex.0", ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT: [[TMP1:%.*]] = load double, ptr [[__RE_]], align 8, !tbaa [[TBAA26:![0-9]+]]
+// CHECK1-NEXT: [[TMP1:%.*]] = load double, ptr [[__RE_]], align 8
// CHECK1-NEXT: [[ADD:%.*]] = fadd double [[TMP1]], [[CALL]]
-// CHECK1-NEXT: store double [[ADD]], ptr [[__RE_]], align 8, !tbaa [[TBAA26]]
-// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[__C_ADDR]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT: store double [[ADD]], ptr [[__RE_]], align 8
+// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[__C_ADDR]], align 8
// CHECK1-NEXT: [[CALL2:%.*]] = call double @_ZNKSt7complexIdE4imagEv(ptr nonnull align 8 dereferenceable(16) [[TMP2]]) #[[ATTR12]]
// CHECK1-NEXT: [[__IM_:%.*]] = getelementptr inbounds %"class.std::complex.0", ptr [[THIS1]], i32 0, i32 1
-// CHECK1-NEXT: [[TMP3:%.*]] = load double, ptr [[__IM_]], align 8, !tbaa [[TBAA28:![0-9]+]]
+// CHECK1-NEXT: [[TMP3:%.*]] = load double, ptr [[__IM_]], align 8
// CHECK1-NEXT: [[ADD3:%.*]] = fadd double [[TMP3]], [[CALL2]]
-// CHECK1-NEXT: store double [[ADD3]], ptr [[__IM_]], align 8, !tbaa [[TBAA28]]
+// CHECK1-NEXT: store double [[ADD3]], ptr [[__IM_]], align 8
// CHECK1-NEXT: ret ptr [[THIS1]]
//
//
// CHECK1-LABEL: define {{[^@]+}}@_omp_reduction_shuffle_and_reduce_func1
-// CHECK1-SAME: (ptr [[TMP0:%.*]], i16 signext [[TMP1:%.*]], i16 signext [[TMP2:%.*]], i16 signext [[TMP3:%.*]]) #[[ATTR7]] {
+// CHECK1-SAME: (ptr noundef [[TMP0:%.*]], i16 noundef signext [[TMP1:%.*]], i16 noundef signext [[TMP2:%.*]], i16 noundef signext [[TMP3:%.*]]) #[[ATTR7]] {
// CHECK1-NEXT: entry:
// CHECK1-NEXT: [[DOTADDR:%.*]] = alloca ptr, align 8
// CHECK1-NEXT: [[DOTADDR1:%.*]] = alloca i16, align 2
@@ -876,14 +876,14 @@ void test() {
// CHECK1-NEXT: [[DOTADDR3:%.*]] = alloca i16, align 2
// CHECK1-NEXT: [[DOTOMP_REDUCTION_REMOTE_REDUCE_LIST:%.*]] = alloca [1 x ptr], align 8
// CHECK1-NEXT: [[DOTOMP_REDUCTION_ELEMENT:%.*]] = alloca %"class.std::complex.0", align 8
-// CHECK1-NEXT: store ptr [[TMP0]], ptr [[DOTADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT: store i16 [[TMP1]], ptr [[DOTADDR1]], align 2, !tbaa [[TBAA21]]
-// CHECK1-NEXT: store i16 [[TMP2]], ptr [[DOTADDR2]], align 2, !tbaa [[TBAA21]]
-// CHECK1-NEXT: store i16 [[TMP3]], ptr [[DOTADDR3]], align 2, !tbaa [[TBAA21]]
-// CHECK1-NEXT: [[TMP4:%.*]] = load ptr, ptr [[DOTADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT: [[TMP5:%.*]] = load i16, ptr [[DOTADDR1]], align 2, !tbaa [[TBAA21]]
-// CHECK1-NEXT: [[TMP6:%.*]] = load i16, ptr [[DOTADDR2]], align 2, !tbaa [[TBAA21]]
-// CHECK1-NEXT: [[TMP7:%.*]] = load i16, ptr [[DOTADDR3]], align 2, !tbaa [[TBAA21]]
+// CHECK1-NEXT: store ptr [[TMP0]], ptr [[DOTADDR]], align 8
+// CHECK1-NEXT: store i16 [[TMP1]], ptr [[DOTADDR1]], align 2
+// CHECK1-NEXT: store i16 [[TMP2]], ptr [[DOTADDR2]], align 2
+// CHECK1-NEXT: store i16 [[TMP3]], ptr [[DOTADDR3]], align 2
+// CHECK1-NEXT: [[TMP4:%.*]] = load ptr, ptr [[DOTADDR]], align 8
+// CHECK1-NEXT: [[TMP5:%.*]] = load i16, ptr [[DOTADDR1]], align 2
+// CHECK1-NEXT: [[TMP6:%.*]] = load i16, ptr [[DOTADDR2]], align 2
+// CHECK1-NEXT: [[TMP7:%.*]] = load i16, ptr [[DOTADDR3]], align 2
// CHECK1-NEXT: [[TMP8:%.*]] = getelementptr inbounds [1 x ptr], ptr [[TMP4]], i64 0, i64 0
// CHECK1-NEXT: [[TMP9:%.*]] = load ptr, ptr [[TMP8]], align 8
// CHECK1-NEXT: [[TMP10:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOMP_REDUCTION_REMOTE_REDUCE_LIST]], i64 0, i64 0
@@ -908,7 +908,7 @@ void test() {
// CHECK1-NEXT: [[TMP24]] = getelementptr i64, ptr [[TMP13]], i64 1
// CHECK1-NEXT: br label [[DOTSHUFFLE_PRE_COND]]
// CHECK1: .shuffle.exit:
-// CHECK1-NEXT: store ptr [[DOTOMP_REDUCTION_ELEMENT]], ptr [[TMP10]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT: store ptr [[DOTOMP_REDUCTION_ELEMENT]], ptr [[TMP10]], align 8
// CHECK1-NEXT: [[TMP25:%.*]] = icmp eq i16 [[TMP7]], 0
// CHECK1-NEXT: [[TMP26:%.*]] = icmp eq i16 [[TMP7]], 1
// CHECK1-NEXT: [[TMP27:%.*]] = icmp ult i16 [[TMP5]], [[TMP6]]
@@ -937,7 +937,7 @@ void test() {
// CHECK1-NEXT: [[TMP41:%.*]] = load ptr, ptr [[TMP40]], align 8
// CHECK1-NEXT: [[TMP42:%.*]] = getelementptr inbounds [1 x ptr], ptr [[TMP4]], i64 0, i64 0
// CHECK1-NEXT: [[TMP43:%.*]] = load ptr, ptr [[TMP42]], align 8
-// CHECK1-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP43]], ptr align 8 [[TMP41]], i64 16, i1 false), !tbaa.struct [[TBAA_STRUCT29:![0-9]+]]
+// CHECK1-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP43]], ptr align 8 [[TMP41]], i64 16, i1 false)
// CHECK1-NEXT: br label [[IFCONT6:%.*]]
// CHECK1: else5:
// CHECK1-NEXT: br label [[IFCONT6]]
@@ -946,24 +946,24 @@ void test() {
//
//
// CHECK1-LABEL: define {{[^@]+}}@_omp_reduction_inter_warp_copy_func2
-// CHECK1-SAME: (ptr [[TMP0:%.*]], i32 [[TMP1:%.*]]) #[[ATTR7]] {
+// CHECK1-SAME: (ptr noundef [[TMP0:%.*]], i32 noundef [[TMP1:%.*]]) #[[ATTR7]] {
// CHECK1-NEXT: entry:
// CHECK1-NEXT: [[DOTADDR:%.*]] = alloca ptr, align 8
// CHECK1-NEXT: [[DOTADDR1:%.*]] = alloca i32, align 4
// CHECK1-NEXT: [[DOTCNT_ADDR:%.*]] = alloca i32, align 4
// CHECK1-NEXT: [[TMP2:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]])
-// CHECK1-NEXT: store ptr [[TMP0]], ptr [[DOTADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT: store i32 [[TMP1]], ptr [[DOTADDR1]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: store ptr [[TMP0]], ptr [[DOTADDR]], align 8
+// CHECK1-NEXT: store i32 [[TMP1]], ptr [[DOTADDR1]], align 4
// CHECK1-NEXT: [[TMP3:%.*]] = call i32 @__kmpc_get_hardware_thread_id_in_block()
// CHECK1-NEXT: [[TMP4:%.*]] = call i32 @__kmpc_get_hardware_thread_id_in_block()
// CHECK1-NEXT: [[NVPTX_LANE_ID:%.*]] = and i32 [[TMP4]], 31
// CHECK1-NEXT: [[TMP5:%.*]] = call i32 @__kmpc_get_hardware_thread_id_in_block()
// CHECK1-NEXT: [[NVPTX_WARP_ID:%.*]] = ashr i32 [[TMP5]], 5
// CHECK1-NEXT: [[TMP6:%.*]] = load ptr, ptr [[DOTADDR]], align 8
-// CHECK1-NEXT: store i32 0, ptr [[DOTCNT_ADDR]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: store i32 0, ptr [[DOTCNT_ADDR]], align 4
// CHECK1-NEXT: br label [[PRECOND:%.*]]
// CHECK1: precond:
-// CHECK1-NEXT: [[TMP7:%.*]] = load i32, ptr [[DOTCNT_ADDR]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP7:%.*]] = load i32, ptr [[DOTCNT_ADDR]], align 4
// CHECK1-NEXT: [[TMP8:%.*]] = icmp ult i32 [[TMP7]], 4
// CHECK1-NEXT: br i1 [[TMP8]], label [[BODY:%.*]], label [[EXIT:%.*]]
// CHECK1: body:
@@ -972,7 +972,7 @@ void test() {
// CHECK1-NEXT: br i1 [[WARP_MASTER]], label [[THEN:%.*]], label [[ELSE:%.*]]
// CHECK1: then:
// CHECK1-NEXT: [[TMP9:%.*]] = getelementptr inbounds [1 x ptr], ptr [[TMP6]], i64 0, i64 0
-// CHECK1-NEXT: [[TMP10:%.*]] = load ptr, ptr [[TMP9]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT: [[TMP10:%.*]] = load ptr, ptr [[TMP9]], align 8
// CHECK1-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[TMP10]], i32 [[TMP7]]
// CHECK1-NEXT: [[TMP12:%.*]] = getelementptr inbounds [32 x i32], ptr addrspace(3) @__openmp_nvptx_data_transfer_temporary_storage, i64 0, i32 [[NVPTX_WARP_ID]]
// CHECK1-NEXT: [[TMP13:%.*]] = load i32, ptr [[TMP11]], align 4
@@ -982,45 +982,45 @@ void test() {
// CHECK1-NEXT: br label [[IFCONT]]
// CHECK1: ifcont:
// CHECK1-NEXT: call void @__kmpc_barrier(ptr @[[GLOB4]], i32 [[TMP2]])
-// CHECK1-NEXT: [[TMP14:%.*]] = load i32, ptr [[DOTADDR1]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP14:%.*]] = load i32, ptr [[DOTADDR1]], align 4
// CHECK1-NEXT: [[IS_ACTIVE_THREAD:%.*]] = icmp ult i32 [[TMP3]], [[TMP14]]
// CHECK1-NEXT: br i1 [[IS_ACTIVE_THREAD]], label [[THEN2:%.*]], label [[ELSE3:%.*]]
// CHECK1: then2:
// CHECK1-NEXT: [[TMP15:%.*]] = getelementptr inbounds [32 x i32], ptr addrspace(3) @__openmp_nvptx_data_transfer_temporary_storage, i64 0, i32 [[TMP3]]
// CHECK1-NEXT: [[TMP16:%.*]] = getelementptr inbounds [1 x ptr], ptr [[TMP6]], i64 0, i64 0
-// CHECK1-NEXT: [[TMP17:%.*]] = load ptr, ptr [[TMP16]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT: [[TMP17:%.*]] = load ptr, ptr [[TMP16]], align 8
// CHECK1-NEXT: [[TMP18:%.*]] = getelementptr i32, ptr [[TMP17]], i32 [[TMP7]]
-// CHECK1-NEXT: [[TMP19:%.*]] = load volatile i32, ptr addrspace(3) [[TMP15]], align 4, !tbaa [[TBAA14]]
-// CHECK1-NEXT: store i32 [[TMP19]], ptr [[TMP18]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: [[TMP19:%.*]] = load volatile i32, ptr addrspace(3) [[TMP15]], align 4
+// CHECK1-NEXT: store i32 [[TMP19]], ptr [[TMP18]], align 4
// CHECK1-NEXT: br label [[IFCONT4:%.*]]
// CHECK1: else3:
// CHECK1-NEXT: br label [[IFCONT4]]
// CHECK1: ifcont4:
// CHECK1-NEXT: [[TMP20:%.*]] = add nsw i32 [[TMP7]], 1
-// CHECK1-NEXT: store i32 [[TMP20]], ptr [[DOTCNT_ADDR]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: store i32 [[TMP20]], ptr [[DOTCNT_ADDR]], align 4
// CHECK1-NEXT: br label [[PRECOND]]
// CHECK1: exit:
// CHECK1-NEXT: ret void
//
//
// CHECK1-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z17complex_reductionIdEvv_l16_omp_outlined_omp_outlined_wrapper
-// CHECK1-SAME: (i16 zeroext [[TMP0:%.*]], i32 [[TMP1:%.*]]) #[[ATTR7]] {
+// CHECK1-SAME: (i16 zeroext [[TMP0:%.*]], i32 [[TMP1:%.*]]) #[[ATTR8]] {
// CHECK1-NEXT: entry:
// CHECK1-NEXT: [[DOTADDR:%.*]] = alloca i16, align 2
// CHECK1-NEXT: [[DOTADDR1:%.*]] = alloca i32, align 4
// CHECK1-NEXT: [[DOTZERO_ADDR:%.*]] = alloca i32, align 4
// CHECK1-NEXT: [[GLOBAL_ARGS:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT: store i16 [[TMP0]], ptr [[DOTADDR]], align 2, !tbaa [[TBAA21]]
-// CHECK1-NEXT: store i32 [[TMP1]], ptr [[DOTADDR1]], align 4, !tbaa [[TBAA14]]
+// CHECK1-NEXT: store i16 [[TMP0]], ptr [[DOTADDR]], align 2
+// CHECK1-NEXT: store i32 [[TMP1]], ptr [[DOTADDR1]], align 4
// CHECK1-NEXT: store i32 0, ptr [[DOTZERO_ADDR]], align 4
// CHECK1-NEXT: call void @__kmpc_get_shared_variables(ptr [[GLOBAL_ARGS]])
// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[GLOBAL_ARGS]], align 8
// CHECK1-NEXT: [[TMP3:%.*]] = getelementptr inbounds ptr, ptr [[TMP2]], i64 0
-// CHECK1-NEXT: [[TMP4:%.*]] = load ptr, ptr [[TMP3]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT: [[TMP4:%.*]] = load ptr, ptr [[TMP3]], align 8
// CHECK1-NEXT: [[TMP5:%.*]] = getelementptr inbounds ptr, ptr [[TMP2]], i64 1
-// CHECK1-NEXT: [[TMP6:%.*]] = load ptr, ptr [[TMP5]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT: [[TMP6:%.*]] = load ptr, ptr [[TMP5]], align 8
// CHECK1-NEXT: [[TMP7:%.*]] = getelementptr inbounds ptr, ptr [[TMP2]], i64 2
-// CHECK1-NEXT: [[TMP8:%.*]] = load ptr, ptr [[TMP7]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT: [[TMP8:%.*]] = load ptr, ptr [[TMP7]], align 8
// CHECK1-NEXT: call void @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z17complex_reductionIdEvv_l16_omp_outlined_omp_outlined(ptr [[DOTADDR1]], ptr [[DOTZERO_ADDR]], ptr [[TMP4]], ptr [[TMP6]], ptr [[TMP8]]) #[[ATTR4]]
// CHECK1-NEXT: ret void
//
@@ -1031,18 +1031,18 @@ void test() {
// CHECK1-NEXT: [[THIS_ADDR:%.*]] = alloca ptr, align 8
// CHECK1-NEXT: [[__RE_ADDR:%.*]] = alloca ptr, align 8
// CHECK1-NEXT: [[__IM_ADDR:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT: store ptr [[THIS]], ptr [[THIS_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT: store ptr [[__RE]], ptr [[__RE_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT: store ptr [[__IM]], ptr [[__IM_ADDR]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT: store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
+// CHECK1-NEXT: store ptr [[__RE]], ptr [[__RE_ADDR]], align 8
+// CHECK1-NEXT: store ptr [[__IM]], ptr [[__IM_ADDR]], align 8
// CHECK1-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
// CHECK1-NEXT: [[__RE_:%.*]] = getelementptr inbounds %"class.std::complex", ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[__RE_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT: [[TMP1:%.*]] = load float, ptr [[TMP0]], align 4, !tbaa [[TBAA16]]
-// CHECK1-NEXT: store float [[TMP1]], ptr [[__RE_]], align 4, !tbaa [[TBAA18]]
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[__RE_ADDR]], align 8
+// CHECK1-NEXT: [[TMP1:%.*]] = load float, ptr [[TMP0]], align 4
+// CHECK1-NEXT: store float [[TMP1]], ptr [[__RE_]], align 4
// CHECK1-NEXT: [[__IM_:%.*]] = getelementptr inbounds %"class.std::complex", ptr [[THIS1]], i32 0, i32 1
-// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[__IM_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT: [[TMP3:%.*]] = load float, ptr [[TMP2]], align 4, !tbaa [[TBAA16]]
-// CHECK1-NEXT: store float [[TMP3]], ptr [[__IM_]], align 4, !tbaa [[TBAA20]]
+// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[__IM_ADDR]], align 8
+// CHECK1-NEXT: [[TMP3:%.*]] = load float, ptr [[TMP2]], align 4
+// CHECK1-NEXT: store float [[TMP3]], ptr [[__IM_]], align 4
// CHECK1-NEXT: ret void
//
//
@@ -1050,10 +1050,10 @@ void test() {
// CHECK1-SAME: (ptr nonnull align 4 dereferenceable(8) [[THIS:%.*]]) #[[ATTR6]] comdat align 2 {
// CHECK1-NEXT: entry:
// CHECK1-NEXT: [[THIS_ADDR:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT: store ptr [[THIS]], ptr [[THIS_ADDR]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT: store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
// CHECK1-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
// CHECK1-NEXT: [[__RE_:%.*]] = getelementptr inbounds %"class.std::complex", ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT: [[TMP0:%.*]] = load float, ptr [[__RE_]], align 4, !tbaa [[TBAA18]]
+// CHECK1-NEXT: [[TMP0:%.*]] = load float, ptr [[__RE_]], align 4
// CHECK1-NEXT: ret float [[TMP0]]
//
//
@@ -1061,10 +1061,10 @@ void test() {
// CHECK1-SAME: (ptr nonnull align 4 dereferenceable(8) [[THIS:%.*]]) #[[ATTR6]] comdat align 2 {
// CHECK1-NEXT: entry:
// CHECK1-NEXT: [[THIS_ADDR:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT: store ptr [[THIS]], ptr [[THIS_ADDR]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT: store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
// CHECK1-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
// CHECK1-NEXT: [[__IM_:%.*]] = getelementptr inbounds %"class.std::complex", ptr [[THIS1]], i32 0, i32 1
-// CHECK1-NEXT: [[TMP0:%.*]] = load float, ptr [[__IM_]], align 4, !tbaa [[TBAA20]]
+// CHECK1-NEXT: [[TMP0:%.*]] = load float, ptr [[__IM_]], align 4
// CHECK1-NEXT: ret float [[TMP0]]
//
//
@@ -1074,18 +1074,18 @@ void test() {
// CHECK1-NEXT: [[THIS_ADDR:%.*]] = alloca ptr, align 8
// CHECK1-NEXT: [[__RE_ADDR:%.*]] = alloca ptr, align 8
// CHECK1-NEXT: [[__IM_ADDR:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT: store ptr [[THIS]], ptr [[THIS_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT: store ptr [[__RE]], ptr [[__RE_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT: store ptr [[__IM]], ptr [[__IM_ADDR]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT: store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
+// CHECK1-NEXT: store ptr [[__RE]], ptr [[__RE_ADDR]], align 8
+// CHECK1-NEXT: store ptr [[__IM]], ptr [[__IM_ADDR]], align 8
// CHECK1-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
// CHECK1-NEXT: [[__RE_:%.*]] = getelementptr inbounds %"class.std::complex.0", ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[__RE_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT: [[TMP1:%.*]] = load double, ptr [[TMP0]], align 8, !tbaa [[TBAA24]]
-// CHECK1-NEXT: store double [[TMP1]], ptr [[__RE_]], align 8, !tbaa [[TBAA26]]
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[__RE_ADDR]], align 8
+// CHECK1-NEXT: [[TMP1:%.*]] = load double, ptr [[TMP0]], align 8
+// CHECK1-NEXT: store double [[TMP1]], ptr [[__RE_]], align 8
// CHECK1-NEXT: [[__IM_:%.*]] = getelementptr inbounds %"class.std::complex.0", ptr [[THIS1]], i32 0, i32 1
-// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[__IM_ADDR]], align 8, !tbaa [[TBAA10]]
-// CHECK1-NEXT: [[TMP3:%.*]] = load double, ptr [[TMP2]], align 8, !tbaa [[TBAA24]]
-// CHECK1-NEXT: store double [[TMP3]], ptr [[__IM_]], align 8, !tbaa [[TBAA28]]
+// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[__IM_ADDR]], align 8
+// CHECK1-NEXT: [[TMP3:%.*]] = load double, ptr [[TMP2]], align 8
+// CHECK1-NEXT: store double [[TMP3]], ptr [[__IM_]], align 8
// CHECK1-NEXT: ret void
//
//
@@ -1093,10 +1093,10 @@ void test() {
// CHECK1-SAME: (ptr nonnull align 8 dereferenceable(16) [[THIS:%.*]]) #[[ATTR6]] comdat align 2 {
// CHECK1-NEXT: entry:
// CHECK1-NEXT: [[THIS_ADDR:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT: store ptr [[THIS]], ptr [[THIS_ADDR]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT: store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
// CHECK1-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
// CHECK1-NEXT: [[__RE_:%.*]] = getelementptr inbounds %"class.std::complex.0", ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT: [[TMP0:%.*]] = load double, ptr [[__RE_]], align 8, !tbaa [[TBAA26]]
+// CHECK1-NEXT: [[TMP0:%.*]] = load double, ptr [[__RE_]], align 8
// CHECK1-NEXT: ret double [[TMP0]]
//
//
@@ -1104,9 +1104,9 @@ void test() {
// CHECK1-SAME: (ptr nonnull align 8 dereferenceable(16) [[THIS:%.*]]) #[[ATTR6]] comdat align 2 {
// CHECK1-NEXT: entry:
// CHECK1-NEXT: [[THIS_ADDR:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT: store ptr [[THIS]], ptr [[THIS_ADDR]], align 8, !tbaa [[TBAA10]]
+// CHECK1-NEXT: store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
// CHECK1-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
// CHECK1-NEXT: [[__IM_:%.*]] = getelementptr inbounds %"class.std::complex.0", ptr [[THIS1]], i32 0, i32 1
-// CHECK1-NEXT: [[TMP0:%.*]] = load double, ptr [[__IM_]], align 8, !tbaa [[TBAA28]]
+// CHECK1-NEXT: [[TMP0:%.*]] = load double, ptr [[__IM_]], align 8
// CHECK1-NEXT: ret double [[TMP0]]
//
diff --git a/clang/test/OpenMP/nvptx_teams_reduction_codegen.cpp b/clang/test/OpenMP/nvptx_teams_reduction_codegen.cpp
index 360a780c75383c..e7beb174872321 100644
--- a/clang/test/OpenMP/nvptx_teams_reduction_codegen.cpp
+++ b/clang/test/OpenMP/nvptx_teams_reduction_codegen.cpp
@@ -579,7 +579,7 @@ int bar(int n){
// CHECK1-NEXT: [[TMP8:%.*]] = getelementptr inbounds [[STRUCT__GLOBALIZED_LOCALS_TY_0:%.*]], ptr [[TMP4]], i32 [[TMP5]]
// CHECK1-NEXT: [[C:%.*]] = getelementptr inbounds [[STRUCT__GLOBALIZED_LOCALS_TY_0]], ptr [[TMP8]], i32 0, i32 0
// CHECK1-NEXT: [[TMP9:%.*]] = load i8, ptr [[TMP7]], align 1
-// CHECK1-NEXT: store i8 [[TMP9]], ptr [[C]], align 4
+// CHECK1-NEXT: store i8 [[TMP9]], ptr [[C]], align 1
// CHECK1-NEXT: [[TMP10:%.*]] = getelementptr inbounds [2 x ptr], ptr [[TMP3]], i64 0, i64 1
// CHECK1-NEXT: [[TMP11:%.*]] = load ptr, ptr [[TMP10]], align 8
// CHECK1-NEXT: [[TMP12:%.*]] = getelementptr inbounds [[STRUCT__GLOBALIZED_LOCALS_TY_0]], ptr [[TMP4]], i32 [[TMP5]]
@@ -630,7 +630,7 @@ int bar(int n){
// CHECK1-NEXT: [[TMP7:%.*]] = load ptr, ptr [[TMP6]], align 8
// CHECK1-NEXT: [[TMP8:%.*]] = getelementptr inbounds [[STRUCT__GLOBALIZED_LOCALS_TY_0:%.*]], ptr [[TMP4]], i32 [[TMP5]]
// CHECK1-NEXT: [[C:%.*]] = getelementptr inbounds [[STRUCT__GLOBALIZED_LOCALS_TY_0]], ptr [[TMP8]], i32 0, i32 0
-// CHECK1-NEXT: [[TMP9:%.*]] = load i8, ptr [[C]], align 4
+// CHECK1-NEXT: [[TMP9:%.*]] = load i8, ptr [[C]], align 1
// CHECK1-NEXT: store i8 [[TMP9]], ptr [[TMP7]], align 1
// CHECK1-NEXT: [[TMP10:%.*]] = getelementptr inbounds [2 x ptr], ptr [[TMP3]], i64 0, i64 1
// CHECK1-NEXT: [[TMP11:%.*]] = load ptr, ptr [[TMP10]], align 8
@@ -1156,7 +1156,7 @@ int bar(int n){
// CHECK1-NEXT: [[TMP12:%.*]] = getelementptr inbounds [[STRUCT__GLOBALIZED_LOCALS_TY_2]], ptr [[TMP4]], i32 [[TMP5]]
// CHECK1-NEXT: [[B:%.*]] = getelementptr inbounds [[STRUCT__GLOBALIZED_LOCALS_TY_2]], ptr [[TMP12]], i32 0, i32 1
// CHECK1-NEXT: [[TMP13:%.*]] = load i16, ptr [[TMP11]], align 2
-// CHECK1-NEXT: store i16 [[TMP13]], ptr [[B]], align 4
+// CHECK1-NEXT: store i16 [[TMP13]], ptr [[B]], align 2
// CHECK1-NEXT: ret void
//
//
@@ -1207,7 +1207,7 @@ int bar(int n){
// CHECK1-NEXT: [[TMP11:%.*]] = load ptr, ptr [[TMP10]], align 8
// CHECK1-NEXT: [[TMP12:%.*]] = getelementptr inbounds [[STRUCT__GLOBALIZED_LOCALS_TY_2]], ptr [[TMP4]], i32 [[TMP5]]
// CHECK1-NEXT: [[B:%.*]] = getelementptr inbounds [[STRUCT__GLOBALIZED_LOCALS_TY_2]], ptr [[TMP12]], i32 0, i32 1
-// CHECK1-NEXT: [[TMP13:%.*]] = load i16, ptr [[B]], align 4
+// CHECK1-NEXT: [[TMP13:%.*]] = load i16, ptr [[B]], align 2
// CHECK1-NEXT: store i16 [[TMP13]], ptr [[TMP11]], align 2
// CHECK1-NEXT: ret void
//
@@ -1766,7 +1766,7 @@ int bar(int n){
// CHECK2-NEXT: [[TMP8:%.*]] = getelementptr inbounds [[STRUCT__GLOBALIZED_LOCALS_TY_0:%.*]], ptr [[TMP4]], i32 [[TMP5]]
// CHECK2-NEXT: [[C:%.*]] = getelementptr inbounds [[STRUCT__GLOBALIZED_LOCALS_TY_0]], ptr [[TMP8]], i32 0, i32 0
// CHECK2-NEXT: [[TMP9:%.*]] = load i8, ptr [[TMP7]], align 1
-// CHECK2-NEXT: store i8 [[TMP9]], ptr [[C]], align 4
+// CHECK2-NEXT: store i8 [[TMP9]], ptr [[C]], align 1
// CHECK2-NEXT: [[TMP10:%.*]] = getelementptr inbounds [2 x ptr], ptr [[TMP3]], i32 0, i32 1
// CHECK2-NEXT: [[TMP11:%.*]] = load ptr, ptr [[TMP10]], align 4
// CHECK2-NEXT: [[TMP12:%.*]] = getelementptr inbounds [[STRUCT__GLOBALIZED_LOCALS_TY_0]], ptr [[TMP4]], i32 [[TMP5]]
@@ -1817,7 +1817,7 @@ int bar(int n){
// CHECK2-NEXT: [[TMP7:%.*]] = load ptr, ptr [[TMP6]], align 4
// CHECK2-NEXT: [[TMP8:%.*]] = getelementptr inbounds [[STRUCT__GLOBALIZED_LOCALS_TY_0:%.*]], ptr [[TMP4]], i32 [[TMP5]]
// CHECK2-NEXT: [[C:%.*]] = getelementptr inbounds [[STRUCT__GLOBALIZED_LOCALS_TY_0]], ptr [[TMP8]], i32 0, i32 0
-// CHECK2-NEXT: [[TMP9:%.*]] = load i8, ptr [[C]], align 4
+// CHECK2-NEXT: [[TMP9:%.*]] = load i8, ptr [[C]], align 1
// CHECK2-NEXT: store i8 [[TMP9]], ptr [[TMP7]], align 1
// CHECK2-NEXT: [[TMP10:%.*]] = getelementptr inbounds [2 x ptr], ptr [[TMP3]], i32 0, i32 1
// CHECK2-NEXT: [[TMP11:%.*]] = load ptr, ptr [[TMP10]], align 4
@@ -2343,7 +2343,7 @@ int bar(int n){
// CHECK2-NEXT: [[TMP12:%.*]] = getelementptr inbounds [[STRUCT__GLOBALIZED_LOCALS_TY_2]], ptr [[TMP4]], i32 [[TMP5]]
// CHECK2-NEXT: [[B:%.*]] = getelementptr inbounds [[STRUCT__GLOBALIZED_LOCALS_TY_2]], ptr [[TMP12]], i32 0, i32 1
// CHECK2-NEXT: [[TMP13:%.*]] = load i16, ptr [[TMP11]], align 2
-// CHECK2-NEXT: store i16 [[TMP13]], ptr [[B]], align 4
+// CHECK2-NEXT: store i16 [[TMP13]], ptr [[B]], align 2
// CHECK2-NEXT: ret void
//
//
@@ -2394,7 +2394,7 @@ int bar(int n){
// CHECK2-NEXT: [[TMP11:%.*]] = load ptr, ptr [[TMP10]], align 4
// CHECK2-NEXT: [[TMP12:%.*]] = getelementptr inbounds [[STRUCT__GLOBALIZED_LOCALS_TY_2]], ptr [[TMP4]], i32 [[TMP5]]
// CHECK2-NEXT: [[B:%.*]] = getelementptr inbounds [[STRUCT__GLOBALIZED_LOCALS_TY_2]], ptr [[TMP12]], i32 0, i32 1
-// CHECK2-NEXT: [[TMP13:%.*]] = load i16, ptr [[B]], align 4
+// CHECK2-NEXT: [[TMP13:%.*]] = load i16, ptr [[B]], align 2
// CHECK2-NEXT: store i16 [[TMP13]], ptr [[TMP11]], align 2
// CHECK2-NEXT: ret void
//
@@ -2953,7 +2953,7 @@ int bar(int n){
// CHECK3-NEXT: [[TMP8:%.*]] = getelementptr inbounds [[STRUCT__GLOBALIZED_LOCALS_TY_0:%.*]], ptr [[TMP4]], i32 [[TMP5]]
// CHECK3-NEXT: [[C:%.*]] = getelementptr inbounds [[STRUCT__GLOBALIZED_LOCALS_TY_0]], ptr [[TMP8]], i32 0, i32 0
// CHECK3-NEXT: [[TMP9:%.*]] = load i8, ptr [[TMP7]], align 1
-// CHECK3-NEXT: store i8 [[TMP9]], ptr [[C]], align 4
+// CHECK3-NEXT: store i8 [[TMP9]], ptr [[C]], align 1
// CHECK3-NEXT: [[TMP10:%.*]] = getelementptr inbounds [2 x ptr], ptr [[TMP3]], i32 0, i32 1
// CHECK3-NEXT: [[TMP11:%.*]] = load ptr, ptr [[TMP10]], align 4
// CHECK3-NEXT: [[TMP12:%.*]] = getelementptr inbounds [[STRUCT__GLOBALIZED_LOCALS_TY_0]], ptr [[TMP4]], i32 [[TMP5]]
@@ -3004,7 +3004,7 @@ int bar(int n){
// CHECK3-NEXT: [[TMP7:%.*]] = load ptr, ptr [[TMP6]], align 4
// CHECK3-NEXT: [[TMP8:%.*]] = getelementptr inbounds [[STRUCT__GLOBALIZED_LOCALS_TY_0:%.*]], ptr [[TMP4]], i32 [[TMP5]]
// CHECK3-NEXT: [[C:%.*]] = getelementptr inbounds [[STRUCT__GLOBALIZED_LOCALS_TY_0]], ptr [[TMP8]], i32 0, i32 0
-// CHECK3-NEXT: [[TMP9:%.*]] = load i8, ptr [[C]], align 4
+// CHECK3-NEXT: [[TMP9:%.*]] = load i8, ptr [[C]], align 1
// CHECK3-NEXT: store i8 [[TMP9]], ptr [[TMP7]], align 1
// CHECK3-NEXT: [[TMP10:%.*]] = getelementptr inbounds [2 x ptr], ptr [[TMP3]], i32 0, i32 1
// CHECK3-NEXT: [[TMP11:%.*]] = load ptr, ptr [[TMP10]], align 4
@@ -3530,7 +3530,7 @@ int bar(int n){
// CHECK3-NEXT: [[TMP12:%.*]] = getelementptr inbounds [[STRUCT__GLOBALIZED_LOCALS_TY_2]], ptr [[TMP4]], i32 [[TMP5]]
// CHECK3-NEXT: [[B:%.*]] = getelementptr inbounds [[STRUCT__GLOBALIZED_LOCALS_TY_2]], ptr [[TMP12]], i32 0, i32 1
// CHECK3-NEXT: [[TMP13:%.*]] = load i16, ptr [[TMP11]], align 2
-// CHECK3-NEXT: store i16 [[TMP13]], ptr [[B]], align 4
+// CHECK3-NEXT: store i16 [[TMP13]], ptr [[B]], align 2
// CHECK3-NEXT: ret void
//
//
@@ -3581,7 +3581,7 @@ int bar(int n){
// CHECK3-NEXT: [[TMP11:%.*]] = load ptr, ptr [[TMP10]], align 4
// CHECK3-NEXT: [[TMP12:%.*]] = getelementptr inbounds [[STRUCT__GLOBALIZED_LOCALS_TY_2]], ptr [[TMP4]], i32 [[TMP5]]
// CHECK3-NEXT: [[B:%.*]] = getelementptr inbounds [[STRUCT__GLOBALIZED_LOCALS_TY_2]], ptr [[TMP12]], i32 0, i32 1
-// CHECK3-NEXT: [[TMP13:%.*]] = load i16, ptr [[B]], align 4
+// CHECK3-NEXT: [[TMP13:%.*]] = load i16, ptr [[B]], align 2
// CHECK3-NEXT: store i16 [[TMP13]], ptr [[TMP11]], align 2
// CHECK3-NEXT: ret void
//
diff --git a/clang/test/OpenMP/target_teams_generic_loop_codegen.cpp b/clang/test/OpenMP/target_teams_generic_loop_codegen.cpp
index 22cf534bf0ba27..95f69ef430eaa0 100644
--- a/clang/test/OpenMP/target_teams_generic_loop_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_generic_loop_codegen.cpp
@@ -1197,6 +1197,7 @@ int foo() {
// IR-GPU-NEXT: [[J_CASTED:%.*]] = alloca i64, align 8, addrspace(5)
// IR-GPU-NEXT: [[CAPTURED_VARS_ADDRS:%.*]] = alloca [4 x ptr], align 8, addrspace(5)
// IR-GPU-NEXT: [[DOTOMP_REDUCTION_RED_LIST:%.*]] = alloca [1 x ptr], align 8, addrspace(5)
+// IR-GPU-NEXT: [[DOTOMP_REDUCTION_RED_LIST_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[DOTOMP_REDUCTION_RED_LIST]] to ptr
// IR-GPU-NEXT: [[DOTGLOBAL_TID__ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[DOTGLOBAL_TID__ADDR]] to ptr
// IR-GPU-NEXT: [[DOTBOUND_TID__ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[DOTBOUND_TID__ADDR]] to ptr
// IR-GPU-NEXT: [[J_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[J_ADDR]] to ptr
@@ -1214,7 +1215,6 @@ int foo() {
// IR-GPU-NEXT: [[J4_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[J4]] to ptr
// IR-GPU-NEXT: [[J_CASTED_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[J_CASTED]] to ptr
// IR-GPU-NEXT: [[CAPTURED_VARS_ADDRS_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[CAPTURED_VARS_ADDRS]] to ptr
-// IR-GPU-NEXT: [[DOTOMP_REDUCTION_RED_LIST_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[DOTOMP_REDUCTION_RED_LIST]] to ptr
// IR-GPU-NEXT: store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR_ASCAST]], align 8
// IR-GPU-NEXT: store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR_ASCAST]], align 8
// IR-GPU-NEXT: store i64 [[J]], ptr [[J_ADDR_ASCAST]], align 8
@@ -1370,6 +1370,7 @@ int foo() {
// IR-GPU-NEXT: [[I:%.*]] = alloca i32, align 4, addrspace(5)
// IR-GPU-NEXT: [[J5:%.*]] = alloca i32, align 4, addrspace(5)
// IR-GPU-NEXT: [[DOTOMP_REDUCTION_RED_LIST:%.*]] = alloca [1 x ptr], align 8, addrspace(5)
+// IR-GPU-NEXT: [[DOTOMP_REDUCTION_RED_LIST_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[DOTOMP_REDUCTION_RED_LIST]] to ptr
// IR-GPU-NEXT: [[DOTGLOBAL_TID__ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[DOTGLOBAL_TID__ADDR]] to ptr
// IR-GPU-NEXT: [[DOTBOUND_TID__ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[DOTBOUND_TID__ADDR]] to ptr
// IR-GPU-NEXT: [[DOTPREVIOUS_LB__ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[DOTPREVIOUS_LB__ADDR]] to ptr
@@ -1387,7 +1388,6 @@ int foo() {
// IR-GPU-NEXT: [[SUM4_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[SUM4]] to ptr
// IR-GPU-NEXT: [[I_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[I]] to ptr
// IR-GPU-NEXT: [[J5_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[J5]] to ptr
-// IR-GPU-NEXT: [[DOTOMP_REDUCTION_RED_LIST_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[DOTOMP_REDUCTION_RED_LIST]] to ptr
// IR-GPU-NEXT: store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR_ASCAST]], align 8
// IR-GPU-NEXT: store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR_ASCAST]], align 8
// IR-GPU-NEXT: store i64 [[DOTPREVIOUS_LB_]], ptr [[DOTPREVIOUS_LB__ADDR_ASCAST]], align 8
@@ -1511,12 +1511,12 @@ int foo() {
// IR-GPU-NEXT: [[DOTADDR3:%.*]] = alloca i16, align 2, addrspace(5)
// IR-GPU-NEXT: [[DOTOMP_REDUCTION_REMOTE_REDUCE_LIST:%.*]] = alloca [1 x ptr], align 8, addrspace(5)
// IR-GPU-NEXT: [[DOTOMP_REDUCTION_ELEMENT:%.*]] = alloca [10 x [10 x i32]], align 4, addrspace(5)
+// IR-GPU-NEXT: [[DOTOMP_REDUCTION_ELEMENT_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[DOTOMP_REDUCTION_ELEMENT]] to ptr
// IR-GPU-NEXT: [[DOTADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[DOTADDR]] to ptr
// IR-GPU-NEXT: [[DOTADDR1_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[DOTADDR1]] to ptr
// IR-GPU-NEXT: [[DOTADDR2_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[DOTADDR2]] to ptr
// IR-GPU-NEXT: [[DOTADDR3_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[DOTADDR3]] to ptr
// IR-GPU-NEXT: [[DOTOMP_REDUCTION_REMOTE_REDUCE_LIST_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[DOTOMP_REDUCTION_REMOTE_REDUCE_LIST]] to ptr
-// IR-GPU-NEXT: [[DOTOMP_REDUCTION_ELEMENT_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[DOTOMP_REDUCTION_ELEMENT]] to ptr
// IR-GPU-NEXT: store ptr [[TMP0]], ptr [[DOTADDR_ASCAST]], align 8
// IR-GPU-NEXT: store i16 [[TMP1]], ptr [[DOTADDR1_ASCAST]], align 2
// IR-GPU-NEXT: store i16 [[TMP2]], ptr [[DOTADDR2_ASCAST]], align 2
@@ -1592,10 +1592,10 @@ int foo() {
// IR-GPU-NEXT: [[DOTADDR:%.*]] = alloca ptr, align 8, addrspace(5)
// IR-GPU-NEXT: [[DOTADDR1:%.*]] = alloca i32, align 4, addrspace(5)
// IR-GPU-NEXT: [[DOTCNT_ADDR:%.*]] = alloca i32, align 4, addrspace(5)
+// IR-GPU-NEXT: [[DOTCNT_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[DOTCNT_ADDR]] to ptr
// IR-GPU-NEXT: [[TMP2:%.*]] = call i32 @__kmpc_global_thread_num(ptr addrspacecast (ptr addrspace(1) @[[GLOB1]] to ptr))
// IR-GPU-NEXT: [[DOTADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[DOTADDR]] to ptr
// IR-GPU-NEXT: [[DOTADDR1_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[DOTADDR1]] to ptr
-// IR-GPU-NEXT: [[DOTCNT_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[DOTCNT_ADDR]] to ptr
// IR-GPU-NEXT: store ptr [[TMP0]], ptr [[DOTADDR_ASCAST]], align 8
// IR-GPU-NEXT: store i32 [[TMP1]], ptr [[DOTADDR1_ASCAST]], align 4
// IR-GPU-NEXT: [[TMP3:%.*]] = call i32 @__kmpc_get_hardware_thread_id_in_block()
@@ -1656,12 +1656,12 @@ int foo() {
// IR-GPU-NEXT: [[DOTADDR3:%.*]] = alloca i16, align 2, addrspace(5)
// IR-GPU-NEXT: [[DOTOMP_REDUCTION_REMOTE_REDUCE_LIST:%.*]] = alloca [1 x ptr], align 8, addrspace(5)
// IR-GPU-NEXT: [[DOTOMP_REDUCTION_ELEMENT:%.*]] = alloca [10 x [10 x i32]], align 4, addrspace(5)
+// IR-GPU-NEXT: [[DOTOMP_REDUCTION_ELEMENT_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[DOTOMP_REDUCTION_ELEMENT]] to ptr
// IR-GPU-NEXT: [[DOTADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[DOTADDR]] to ptr
// IR-GPU-NEXT: [[DOTADDR1_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[DOTADDR1]] to ptr
// IR-GPU-NEXT: [[DOTADDR2_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[DOTADDR2]] to ptr
// IR-GPU-NEXT: [[DOTADDR3_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[DOTADDR3]] to ptr
// IR-GPU-NEXT: [[DOTOMP_REDUCTION_REMOTE_REDUCE_LIST_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[DOTOMP_REDUCTION_REMOTE_REDUCE_LIST]] to ptr
-// IR-GPU-NEXT: [[DOTOMP_REDUCTION_ELEMENT_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[DOTOMP_REDUCTION_ELEMENT]] to ptr
// IR-GPU-NEXT: store ptr [[TMP0]], ptr [[DOTADDR_ASCAST]], align 8
// IR-GPU-NEXT: store i16 [[TMP1]], ptr [[DOTADDR1_ASCAST]], align 2
// IR-GPU-NEXT: store i16 [[TMP2]], ptr [[DOTADDR2_ASCAST]], align 2
@@ -1737,10 +1737,10 @@ int foo() {
// IR-GPU-NEXT: [[DOTADDR:%.*]] = alloca ptr, align 8, addrspace(5)
// IR-GPU-NEXT: [[DOTADDR1:%.*]] = alloca i32, align 4, addrspace(5)
// IR-GPU-NEXT: [[DOTCNT_ADDR:%.*]] = alloca i32, align 4, addrspace(5)
+// IR-GPU-NEXT: [[DOTCNT_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[DOTCNT_ADDR]] to ptr
// IR-GPU-NEXT: [[TMP2:%.*]] = call i32 @__kmpc_global_thread_num(ptr addrspacecast (ptr addrspace(1) @[[GLOB1]] to ptr))
// IR-GPU-NEXT: [[DOTADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[DOTADDR]] to ptr
// IR-GPU-NEXT: [[DOTADDR1_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[DOTADDR1]] to ptr
-// IR-GPU-NEXT: [[DOTCNT_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[DOTCNT_ADDR]] to ptr
// IR-GPU-NEXT: store ptr [[TMP0]], ptr [[DOTADDR_ASCAST]], align 8
// IR-GPU-NEXT: store i32 [[TMP1]], ptr [[DOTADDR1_ASCAST]], align 4
// IR-GPU-NEXT: [[TMP3:%.*]] = call i32 @__kmpc_get_hardware_thread_id_in_block()
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
index e1dba2339338a3..4f2207cc700fbf 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
@@ -100,6 +100,9 @@ class OpenMPIRBuilderConfig {
/// expanded.
std::optional<bool> IsGPU;
+ /// Flag for specifying if LLVMUsed information should be emitted.
+ std::optional<bool> EmitLLVMUsed;
+
/// Flag for specifying if offloading is mandatory.
std::optional<bool> OpenMPOffloadMandatory;
@@ -178,6 +181,7 @@ class OpenMPIRBuilderConfig {
void setIsTargetDevice(bool Value) { IsTargetDevice = Value; }
void setIsGPU(bool Value) { IsGPU = Value; }
+ void setEmitLLVMUsed(bool Value = true) { EmitLLVMUsed = Value; }
void setOpenMPOffloadMandatory(bool Value) { OpenMPOffloadMandatory = Value; }
void setFirstSeparator(StringRef FS) { FirstSeparator = FS; }
void setSeparator(StringRef S) { Separator = S; }
@@ -622,15 +626,17 @@ class OpenMPIRBuilder {
/// Generator for '#omp barrier'
///
/// \param Loc The location where the barrier directive was encountered.
- /// \param DK The kind of directive that caused the barrier.
+ /// \param Kind The kind of directive that caused the barrier.
/// \param ForceSimpleCall Flag to force a simple (=non-cancellation) barrier.
/// \param CheckCancelFlag Flag to indicate a cancel barrier return value
/// should be checked and acted upon.
+ /// \param ThreadID Optional parameter to pass in any existing ThreadID value.
///
/// \returns The insertion point after the barrier.
InsertPointTy createBarrier(const LocationDescription &Loc, omp::Directive DK,
bool ForceSimpleCall = false,
- bool CheckCancelFlag = true);
+ bool CheckCancelFlag = true,
+ Value *ThreadID = nullptr);
/// Generator for '#omp cancel'
///
@@ -1250,39 +1256,56 @@ class OpenMPIRBuilder {
getTargetEntryUniqueInfo(FileIdentifierInfoCallbackTy CallBack,
StringRef ParentName = "");
- // using ReductionGenTy =
- // function_ref<InsertPointTy(InsertPointTy, Value *, Value *, Value *&)>;
+ /// Enum class for the RedctionGen CallBack type to be used.
+ enum class ReductionGenCBTy { Clang, MLIR };
- // using AtomicReductionGenTy =
- // function_ref<InsertPointTy(InsertPointTy, Type *, Value *, Value *)>;
+ /// ReductionGen CallBack for Clang
+ ///
+ /// \param CodeGenIP InsertPoint for CodeGen.
+ /// \param Index Index of the ReductionInfo to generate code for.
+ /// \param LHSPtr Optionally used by Clang to return the LHSPtr it used for
+ /// codegen, used for fixup later.
+ /// \param RHSPtr Optionally used by Clang to
+ /// return the RHSPtr it used for codegen, used for fixup later.
+ /// \param CurFn Optionally used by Clang to pass in the Current Function as
+ /// Clang context may be old.
+ using ReductionGenCBClang =
+ std::function<InsertPointTy(InsertPointTy CodeGenIP, unsigned Index,
+ Value **LHS, Value **RHS, Function *CurFn)>;
+
+ /// ReductionGen CallBack for MLIR
+ ///
+ /// \param CodeGenIP InsertPoint for CodeGen.
+ /// \param LHS Pass in the LHS Value to be used for CodeGen.
+ /// \param RHS Pass in the RHS Value to be used for CodeGen.
+ using ReductionGenCB = std::function<InsertPointTy(
+ InsertPointTy CodeGenIP, Value *LHS, Value *RHS, Value *&Res)>;
- /// Owning equivalents of OpenMPIRBuilder::(Atomic)ReductionGen that are used
- /// to
- /// store lambdas with capture.
- /// Functions used to generate reductions. Such functions take two Values
- /// representing LHS and RHS of the reduction, respectively, and a reference
- /// to the value that is updated to refer to the reduction result.
- using ReductionGenTy = std::function<OpenMPIRBuilder::InsertPointTy(
- OpenMPIRBuilder::InsertPointTy, Value *, Value *, Value *&)>;
/// Functions used to generate atomic reductions. Such functions take two
/// Values representing pointers to LHS and RHS of the reduction, as well as
/// the element type of these pointers. They are expected to atomically
/// update the LHS to the reduced value.
- using AtomicReductionGenTy = std::function<OpenMPIRBuilder::InsertPointTy(
- OpenMPIRBuilder::InsertPointTy, Type *, Value *, Value *)>;
+ using AtomicReductionGenCB =
+ std::function<InsertPointTy(InsertPointTy, Type *, Value *, Value *)>;
+
+ /// Enum class for reduction evaluation types scalar, complex and aggregate.
+ enum class EvaluationKindTy { Scalar, Complex, Aggregate };
/// Information about an OpenMP reduction.
struct ReductionInfo {
ReductionInfo(Type *ElementType, Value *Variable, Value *PrivateVariable,
- ReductionGenTy ReductionGen,
- AtomicReductionGenTy AtomicReductionGen)
+ EvaluationKindTy EvaluationKind, ReductionGenCB ReductionGen,
+ ReductionGenCBClang ReductionGenClang,
+ AtomicReductionGenCB AtomicReductionGen)
: ElementType(ElementType), Variable(Variable),
- PrivateVariable(PrivateVariable), ReductionGen(ReductionGen),
+ PrivateVariable(PrivateVariable), EvaluationKind(EvaluationKind),
+ ReductionGen(ReductionGen), ReductionGenClang(ReductionGenClang),
AtomicReductionGen(AtomicReductionGen) {}
ReductionInfo(Value *PrivateVariable)
: ElementType(nullptr), Variable(nullptr),
- PrivateVariable(PrivateVariable), ReductionGen(),
- AtomicReductionGen() {}
+ PrivateVariable(PrivateVariable),
+ EvaluationKind(EvaluationKindTy::Scalar), ReductionGen(),
+ ReductionGenClang(), AtomicReductionGen() {}
/// Reduction element type, must match pointee type of variable.
Type *ElementType;
@@ -1293,22 +1316,43 @@ class OpenMPIRBuilder {
/// Thread-private partial reduction variable.
Value *PrivateVariable;
+ /// Reduction evaluation type - scalar, complex or aggregate.
+ EvaluationKindTy EvaluationKind;
+
/// Callback for generating the reduction body. The IR produced by this will
/// be used to combine two values in a thread-safe context, e.g., under
/// lock or within the same thread, and therefore need not be atomic.
- ReductionGenTy ReductionGen;
+ ReductionGenCB ReductionGen;
+
+ /// Clang callback for generating the reduction body. The IR produced by
+ /// this will be used to combine two values in a thread-safe context, e.g.,
+ /// under lock or within the same thread, and therefore need not be atomic.
+ ReductionGenCBClang ReductionGenClang;
/// Callback for generating the atomic reduction body, may be null. The IR
/// produced by this will be used to atomically combine two values during
/// reduction. If null, the implementation will use the non-atomic version
/// along with the appropriate synchronization mechanisms.
- AtomicReductionGenTy AtomicReductionGen;
+ AtomicReductionGenCB AtomicReductionGen;
+ };
+
+ enum class CopyAction : unsigned {
+ // RemoteLaneToThread: Copy over a Reduce list from a remote lane in
+ // the warp using shuffle instructions.
+ RemoteLaneToThread,
+ // ThreadCopy: Make a copy of a Reduce list on the thread's stack.
+ ThreadCopy,
+ };
+
+ struct CopyOptionsTy {
+ Value *RemoteLaneOffset = nullptr;
+ Value *ScratchpadIndex = nullptr;
+ Value *ScratchpadWidth = nullptr;
};
/// A class that manages the reduction info to facilitate lowering of
/// reductions at multiple levels of parallelism. For example handling teams
/// and parallel reductions on GPUs
-
class ReductionInfoManager {
private:
SmallVector<ReductionInfo> ReductionInfos;
@@ -1321,15 +1365,12 @@ class OpenMPIRBuilder {
PrivateVarAllocaIP.reset();
}
- Value *
- allocatePrivateReductionVar(IRBuilderBase &builder,
- llvm::OpenMPIRBuilder::InsertPointTy &allocaIP,
- Type *VarType) {
- llvm::Type *ptrTy = llvm::PointerType::getUnqual(builder.getContext());
- llvm::Value *var = builder.CreateAlloca(VarType);
+ Value *allocatePrivateReductionVar(IRBuilderBase &builder,
+ InsertPointTy &allocaIP, Type *VarType) {
+ Type *ptrTy = PointerType::getUnqual(builder.getContext());
+ Value *var = builder.CreateAlloca(VarType);
var->setName("private_redvar");
- llvm::Value *castVar =
- builder.CreatePointerBitCastOrAddrSpaceCast(var, ptrTy);
+ Value *castVar = builder.CreatePointerBitCastOrAddrSpaceCast(var, ptrTy);
ReductionInfos.push_back(ReductionInfo(castVar));
return castVar;
}
@@ -1353,20 +1394,508 @@ class OpenMPIRBuilder {
void setPrivateVarAllocaIP(InsertPointTy IP) { PrivateVarAllocaIP = IP; }
};
+ /// Supporting functions for Reductions CodeGen.
+private:
+ /// Emit the llvm.used metadata.
+ void emitUsed(StringRef Name, std::vector<llvm::WeakTrackingVH> &List);
+
+ /// Get the id of the current thread on the GPU.
+ Value *getGPUThreadID();
+
+ /// Get the GPU warp size.
+ Value *getGPUWarpSize();
+
+ /// Get the id of the warp in the block.
+ /// We assume that the warp size is 32, which is always the case
+ /// on the NVPTX device, to generate more efficient code.
+ Value *getNVPTXWarpID();
+
+ /// Get the id of the current lane in the Warp.
+ /// We assume that the warp size is 32, which is always the case
+ /// on the NVPTX device, to generate more efficient code.
+ Value *getNVPTXLaneID();
+
+ /// Cast value to the specified type.
+ Value *castValueToType(InsertPointTy AllocaIP, Value *From, Type *ToType);
+
+ /// This function creates calls to one of two shuffle functions to copy
+ /// variables between lanes in a warp.
+ Value *createRuntimeShuffleFunction(InsertPointTy AllocaIP, Value *Element,
+ Type *ElementType, Value *Offset);
+
+ void shuffleAndStore(InsertPointTy AllocaIP, Value *SrcAddr, Value *DstAddr,
+ Type *ElementType, Value *Offset,
+ Type *ReductionArrayTy);
+
+ /// Emit instructions to copy a Reduce list, which contains partially
+ /// aggregated values, in the specified direction.
+ void emitReductionListCopy(
+ InsertPointTy AllocaIP, CopyAction Action, Type *ReductionArrayTy,
+ ArrayRef<ReductionInfo> ReductionInfos, Value *SrcBase, Value *DestBase,
+ CopyOptionsTy CopyOptions = {nullptr, nullptr, nullptr});
+
+ /// Emit a helper that reduces data across two OpenMP threads (lanes)
+ /// in the same warp. It uses shuffle instructions to copy over data from
+ /// a remote lane's stack. The reduction algorithm performed is specified
+ /// by the fourth parameter.
+ ///
+ /// Algorithm Versions.
+ /// Full Warp Reduce (argument value 0):
+ /// This algorithm assumes that all 32 lanes are active and gathers
+ /// data from these 32 lanes, producing a single resultant value.
+ /// Contiguous Partial Warp Reduce (argument value 1):
+ /// This algorithm assumes that only a *contiguous* subset of lanes
+ /// are active. This happens for the last warp in a parallel region
+ /// when the user specified num_threads is not an integer multiple of
+ /// 32. This contiguous subset always starts with the zeroth lane.
+ /// Partial Warp Reduce (argument value 2):
+ /// This algorithm gathers data from any number of lanes at any position.
+ /// All reduced values are stored in the lowest possible lane. The set
+ /// of problems every algorithm addresses is a super set of those
+ /// addressable by algorithms with a lower version number. Overhead
+ /// increases as algorithm version increases.
+ ///
+ /// Terminology
+ /// Reduce element:
+ /// Reduce element refers to the individual data field with primitive
+ /// data types to be combined and reduced across threads.
+ /// Reduce list:
+ /// Reduce list refers to a collection of local, thread-private
+ /// reduce elements.
+ /// Remote Reduce list:
+ /// Remote Reduce list refers to a collection of remote (relative to
+ /// the current thread) reduce elements.
+ ///
+ /// We distinguish between three states of threads that are important to
+ /// the implementation of this function.
+ /// Alive threads:
+ /// Threads in a warp executing the SIMT instruction, as distinguished from
+ /// threads that are inactive due to divergent control flow.
+ /// Active threads:
+ /// The minimal set of threads that has to be alive upon entry to this
+ /// function. The computation is correct iff active threads are alive.
+ /// Some threads are alive but they are not active because they do not
+ /// contribute to the computation in any useful manner. Turning them off
+ /// may introduce control flow overheads without any tangible benefits.
+ /// Effective threads:
+ /// In order to comply with the argument requirements of the shuffle
+ /// function, we must keep all lanes holding data alive. But at most
+ /// half of them perform value aggregation; we refer to this half of
+ /// threads as effective. The other half is simply handing off their
+ /// data.
+ ///
+ /// Procedure
+ /// Value shuffle:
+ /// In this step active threads transfer data from higher lane positions
+ /// in the warp to lower lane positions, creating Remote Reduce list.
+ /// Value aggregation:
+ /// In this step, effective threads combine their thread local Reduce list
+ /// with Remote Reduce list and store the result in the thread local
+ /// Reduce list.
+ /// Value copy:
+ /// In this step, we deal with the assumption made by algorithm 2
+ /// (i.e. contiguity assumption). When we have an odd number of lanes
+ /// active, say 2k+1, only k threads will be effective and therefore k
+ /// new values will be produced. However, the Reduce list owned by the
+ /// (2k+1)th thread is ignored in the value aggregation. Therefore
+ /// we copy the Reduce list from the (2k+1)th lane to (k+1)th lane so
+ /// that the contiguity assumption still holds.
+ ///
+ /// \param ReductionInfos Array type containing the ReductionOps.
+ /// \param ReduceFn The reduction function.
+ /// \param FuncAttrs Optional param to specify any function attributes that
+ /// need to be copied to the new function.
+ ///
+ /// \return The ShuffleAndReduce function.
+ Function *emitShuffleAndReduceFunction(
+ ArrayRef<OpenMPIRBuilder::ReductionInfo> ReductionInfos,
+ Function *ReduceFn, AttributeList FuncAttrs);
+
+ /// This function emits a helper that gathers Reduce lists from the first
+ /// lane of every active warp to lanes in the first warp.
+ ///
+ /// void inter_warp_copy_func(void* reduce_data, num_warps)
+ /// shared smem[warp_size];
+ /// For all data entries D in reduce_data:
+ /// sync
+ /// If (I am the first lane in each warp)
+ /// Copy my local D to smem[warp_id]
+ /// sync
+ /// if (I am the first warp)
+ /// Copy smem[thread_id] to my local D
+ ///
+ /// \param Loc The insert and source location description.
+ /// \param ReductionInfos Array type containing the ReductionOps.
+ /// \param FuncAttrs Optional param to specify any function attributes that
+ /// need to be copied to the new function.
+ ///
+ /// \return The InterWarpCopy function.
+ Function *emitInterWarpCopyFunction(const LocationDescription &Loc,
+ ArrayRef<ReductionInfo> ReductionInfos,
+ AttributeList FuncAttrs);
+
+ /// This function emits a helper that copies all the reduction variables from
+ /// the team into the provided global buffer for the reduction variables.
+ ///
+ /// void list_to_global_copy_func(void *buffer, int Idx, void *reduce_data)
+ /// For all data entries D in reduce_data:
+ /// Copy local D to buffer.D[Idx]
+ ///
+ /// \param Loc The insert and source location description.
+ /// \param ReductionsBufferTy The StructTy for the reductions buffer.
+ /// \param FuncAttrs Optional param to specify any function attributes that
+ /// need to be copied to the new function.
+ ///
+ /// \return The ListToGlobalCopy function.
+ Function *emitListToGlobalCopyFunction(ArrayRef<ReductionInfo> ReductionInfos,
+ Type *ReductionsBufferTy,
+ AttributeList FuncAttrs);
+
+ /// This function emits a helper that copies all the reduction variables from
+ /// the team into the provided global buffer for the reduction variables.
+ ///
+ /// void list_to_global_copy_func(void *buffer, int Idx, void *reduce_data)
+ /// For all data entries D in reduce_data:
+ /// Copy buffer.D[Idx] to local D;
+ ///
+ /// \param Loc The insert and source location description.
+ /// \param ReductionsBufferTy The StructTy for the reductions buffer.
+ /// \param FuncAttrs Optional param to specify any function attributes that
+ /// need to be copied to the new function.
+ ///
+ /// \return The GlobalToList function.
+ Function *emitGlobalToListCopyFunction(ArrayRef<ReductionInfo> ReductionInfos,
+ Type *ReductionsBufferTy,
+ AttributeList FuncAttrs);
+
+ /// This function emits a helper that reduces all the reduction variables from
+ /// the team into the provided global buffer for the reduction variables.
+ ///
+ /// void list_to_global_reduce_func(void *buffer, int Idx, void *reduce_data)
+ /// void *GlobPtrs[];
+ /// GlobPtrs[0] = (void*)&buffer.D0[Idx];
+ /// ...
+ /// GlobPtrs[N] = (void*)&buffer.DN[Idx];
+ /// reduce_function(GlobPtrs, reduce_data);
+ ///
+ /// \param Loc The insert and source location description.
+ /// \param ReductionsBufferTy The StructTy for the reductions buffer.
+ /// \param FuncAttrs Optional param to specify any function attributes that
+ /// need to be copied to the new function.
+ ///
+ /// \return The ListToGlobalReduce function.
+ Function *
+ emitListToGlobalReduceFunction(ArrayRef<ReductionInfo> ReductionInfos,
+ Function *ReduceFn, Type *ReductionsBufferTy,
+ AttributeList FuncAttrs);
+
+ /// This function emits a helper that reduces all the reduction variables from
+ /// the team into the provided global buffer for the reduction variables.
+ ///
+ /// void global_to_list_reduce_func(void *buffer, int Idx, void *reduce_data)
+ /// void *GlobPtrs[];
+ /// GlobPtrs[0] = (void*)&buffer.D0[Idx];
+ /// ...
+ /// GlobPtrs[N] = (void*)&buffer.DN[Idx];
+ /// reduce_function(reduce_data, GlobPtrs);
+ ///
+ /// \param Loc The insert and source location description.
+ /// \param ReductionsBufferTy The StructTy for the reductions buffer.
+ /// \param FuncAttrs Optional param to specify any function attributes that
+ /// need to be copied to the new function.
+ ///
+ /// \return The GlobalToListReduce function.
+ Function *
+ emitGlobalToListReduceFunction(ArrayRef<ReductionInfo> ReductionInfos,
+ Function *ReduceFn, Type *ReductionsBufferTy,
+ AttributeList FuncAttrs);
+
+ /// Get the function name of a reduction function.
+ std::string getReductionFuncName(StringRef Name) const;
+
+ /// Emits reduction function.
+ /// \param ReducerName Name of the function calling the reduction.
+ /// \param ReductionInfos Array type containing the ReductionOps.
+ /// \param IsGpu Optional param to specify CodeGen for GPU Offloading.
+ /// \param FuncAttrs Optional param to specify any function attributes that
+ /// need to be copied to the new function.
+ ///
+ /// \return The reduction function.
+ Function *createReductionFunction(
+ StringRef ReducerName, ArrayRef<ReductionInfo> ReductionInfos,
+ bool IsGpu = false,
+ ReductionGenCBTy ReductionGenCBTy = ReductionGenCBTy::MLIR,
+ AttributeList FuncAttrs = {});
+
+public:
+ ///
+ /// Design of OpenMP reductions on the GPU
+ ///
+ /// Consider a typical OpenMP program with one or more reduction
+ /// clauses:
+ ///
+ /// float foo;
+ /// double bar;
+ /// #pragma omp target teams distribute parallel for \
+ /// reduction(+:foo) reduction(*:bar)
+ /// for (int i = 0; i < N; i++) {
+ /// foo += A[i]; bar *= B[i];
+ /// }
+ ///
+ /// where 'foo' and 'bar' are reduced across all OpenMP threads in
+ /// all teams. In our OpenMP implementation on the NVPTX device an
+ /// OpenMP team is mapped to a CUDA threadblock and OpenMP threads
+ /// within a team are mapped to CUDA threads within a threadblock.
+ /// Our goal is to efficiently aggregate values across all OpenMP
+ /// threads such that:
+ ///
+ /// - the compiler and runtime are logically concise, and
+ /// - the reduction is performed efficiently in a hierarchical
+ /// manner as follows: within OpenMP threads in the same warp,
+ /// across warps in a threadblock, and finally across teams on
+ /// the NVPTX device.
+ ///
+ /// Introduction to Decoupling
+ ///
+ /// We would like to decouple the compiler and the runtime so that the
+ /// latter is ignorant of the reduction variables (number, data types)
+ /// and the reduction operators. This allows a simpler interface
+ /// and implementation while still attaining good performance.
+ ///
+ /// Pseudocode for the aforementioned OpenMP program generated by the
+ /// compiler is as follows:
+ ///
+ /// 1. Create private copies of reduction variables on each OpenMP
+ /// thread: 'foo_private', 'bar_private'
+ /// 2. Each OpenMP thread reduces the chunk of 'A' and 'B' assigned
+ /// to it and writes the result in 'foo_private' and 'bar_private'
+ /// respectively.
+ /// 3. Call the OpenMP runtime on the GPU to reduce within a team
+ /// and store the result on the team master:
+ ///
+ /// __kmpc_nvptx_parallel_reduce_nowait_v2(...,
+ /// reduceData, shuffleReduceFn, interWarpCpyFn)
+ ///
+ /// where:
+ /// struct ReduceData {
+ /// double *foo;
+ /// double *bar;
+ /// } reduceData
+ /// reduceData.foo = &foo_private
+ /// reduceData.bar = &bar_private
+ ///
+ /// 'shuffleReduceFn' and 'interWarpCpyFn' are pointers to two
+ /// auxiliary functions generated by the compiler that operate on
+ /// variables of type 'ReduceData'. They aid the runtime perform
+ /// algorithmic steps in a data agnostic manner.
+ ///
+ /// 'shuffleReduceFn' is a pointer to a function that reduces data
+ /// of type 'ReduceData' across two OpenMP threads (lanes) in the
+ /// same warp. It takes the following arguments as input:
+ ///
+ /// a. variable of type 'ReduceData' on the calling lane,
+ /// b. its lane_id,
+ /// c. an offset relative to the current lane_id to generate a
+ /// remote_lane_id. The remote lane contains the second
+ /// variable of type 'ReduceData' that is to be reduced.
+ /// d. an algorithm version parameter determining which reduction
+ /// algorithm to use.
+ ///
+ /// 'shuffleReduceFn' retrieves data from the remote lane using
+ /// efficient GPU shuffle intrinsics and reduces, using the
+ /// algorithm specified by the 4th parameter, the two operands
+ /// element-wise. The result is written to the first operand.
+ ///
+ /// Different reduction algorithms are implemented in different
+ /// runtime functions, all calling 'shuffleReduceFn' to perform
+ /// the essential reduction step. Therefore, based on the 4th
+ /// parameter, this function behaves slightly differently to
+ /// cooperate with the runtime to ensure correctness under
+ /// different circumstances.
+ ///
+ /// 'InterWarpCpyFn' is a pointer to a function that transfers
+ /// reduced variables across warps. It tunnels, through CUDA
+ /// shared memory, the thread-private data of type 'ReduceData'
+ /// from lane 0 of each warp to a lane in the first warp.
+ /// 4. Call the OpenMP runtime on the GPU to reduce across teams.
+ /// The last team writes the global reduced value to memory.
+ ///
+ /// ret = __kmpc_nvptx_teams_reduce_nowait(...,
+ /// reduceData, shuffleReduceFn, interWarpCpyFn,
+ /// scratchpadCopyFn, loadAndReduceFn)
+ ///
+ /// 'scratchpadCopyFn' is a helper that stores reduced
+ /// data from the team master to a scratchpad array in
+ /// global memory.
+ ///
+ /// 'loadAndReduceFn' is a helper that loads data from
+ /// the scratchpad array and reduces it with the input
+ /// operand.
+ ///
+ /// These compiler generated functions hide address
+ /// calculation and alignment information from the runtime.
+ /// 5. if ret == 1:
+ /// The team master of the last team stores the reduced
+ /// result to the globals in memory.
+ /// foo += reduceData.foo; bar *= reduceData.bar
+ ///
+ ///
+ /// Warp Reduction Algorithms
+ ///
+ /// On the warp level, we have three algorithms implemented in the
+ /// OpenMP runtime depending on the number of active lanes:
+ ///
+ /// Full Warp Reduction
+ ///
+ /// The reduce algorithm within a warp where all lanes are active
+ /// is implemented in the runtime as follows:
+ ///
+ /// full_warp_reduce(void *reduce_data,
+ /// kmp_ShuffleReductFctPtr ShuffleReduceFn) {
+ /// for (int offset = WARPSIZE/2; offset > 0; offset /= 2)
+ /// ShuffleReduceFn(reduce_data, 0, offset, 0);
+ /// }
+ ///
+ /// The algorithm completes in log(2, WARPSIZE) steps.
+ ///
+ /// 'ShuffleReduceFn' is used here with lane_id set to 0 because it is
+ /// not used therefore we save instructions by not retrieving lane_id
+ /// from the corresponding special registers. The 4th parameter, which
+ /// represents the version of the algorithm being used, is set to 0 to
+ /// signify full warp reduction.
+ ///
+ /// In this version, 'ShuffleReduceFn' behaves, per element, as follows:
+ ///
+ /// #reduce_elem refers to an element in the local lane's data structure
+ /// #remote_elem is retrieved from a remote lane
+ /// remote_elem = shuffle_down(reduce_elem, offset, WARPSIZE);
+ /// reduce_elem = reduce_elem REDUCE_OP remote_elem;
+ ///
+ /// Contiguous Partial Warp Reduction
+ ///
+ /// This reduce algorithm is used within a warp where only the first
+ /// 'n' (n <= WARPSIZE) lanes are active. It is typically used when the
+ /// number of OpenMP threads in a parallel region is not a multiple of
+ /// WARPSIZE. The algorithm is implemented in the runtime as follows:
+ ///
+ /// void
+ /// contiguous_partial_reduce(void *reduce_data,
+ /// kmp_ShuffleReductFctPtr ShuffleReduceFn,
+ /// int size, int lane_id) {
+ /// int curr_size;
+ /// int offset;
+ /// curr_size = size;
+ /// mask = curr_size/2;
+ /// while (offset>0) {
+ /// ShuffleReduceFn(reduce_data, lane_id, offset, 1);
+ /// curr_size = (curr_size+1)/2;
+ /// offset = curr_size/2;
+ /// }
+ /// }
+ ///
+ /// In this version, 'ShuffleReduceFn' behaves, per element, as follows:
+ ///
+ /// remote_elem = shuffle_down(reduce_elem, offset, WARPSIZE);
+ /// if (lane_id < offset)
+ /// reduce_elem = reduce_elem REDUCE_OP remote_elem
+ /// else
+ /// reduce_elem = remote_elem
+ ///
+ /// This algorithm assumes that the data to be reduced are located in a
+ /// contiguous subset of lanes starting from the first. When there is
+ /// an odd number of active lanes, the data in the last lane is not
+ /// aggregated with any other lane's dat but is instead copied over.
+ ///
+ /// Dispersed Partial Warp Reduction
+ ///
+ /// This algorithm is used within a warp when any discontiguous subset of
+ /// lanes are active. It is used to implement the reduction operation
+ /// across lanes in an OpenMP simd region or in a nested parallel region.
+ ///
+ /// void
+ /// dispersed_partial_reduce(void *reduce_data,
+ /// kmp_ShuffleReductFctPtr ShuffleReduceFn) {
+ /// int size, remote_id;
+ /// int logical_lane_id = number_of_active_lanes_before_me() * 2;
+ /// do {
+ /// remote_id = next_active_lane_id_right_after_me();
+ /// # the above function returns 0 of no active lane
+ /// # is present right after the current lane.
+ /// size = number_of_active_lanes_in_this_warp();
+ /// logical_lane_id /= 2;
+ /// ShuffleReduceFn(reduce_data, logical_lane_id,
+ /// remote_id-1-threadIdx.x, 2);
+ /// } while (logical_lane_id % 2 == 0 && size > 1);
+ /// }
+ ///
+ /// There is no assumption made about the initial state of the reduction.
+ /// Any number of lanes (>=1) could be active at any position. The reduction
+ /// result is returned in the first active lane.
+ ///
+ /// In this version, 'ShuffleReduceFn' behaves, per element, as follows:
+ ///
+ /// remote_elem = shuffle_down(reduce_elem, offset, WARPSIZE);
+ /// if (lane_id % 2 == 0 && offset > 0)
+ /// reduce_elem = reduce_elem REDUCE_OP remote_elem
+ /// else
+ /// reduce_elem = remote_elem
+ ///
+ ///
+ /// Intra-Team Reduction
+ ///
+ /// This function, as implemented in the runtime call
+ /// '__kmpc_nvptx_parallel_reduce_nowait_v2', aggregates data across OpenMP
+ /// threads in a team. It first reduces within a warp using the
+ /// aforementioned algorithms. We then proceed to gather all such
+ /// reduced values at the first warp.
+ ///
+ /// The runtime makes use of the function 'InterWarpCpyFn', which copies
+ /// data from each of the "warp master" (zeroth lane of each warp, where
+ /// warp-reduced data is held) to the zeroth warp. This step reduces (in
+ /// a mathematical sense) the problem of reduction across warp masters in
+ /// a block to the problem of warp reduction.
+ ///
+ ///
+ /// Inter-Team Reduction
+ ///
+ /// Once a team has reduced its data to a single value, it is stored in
+ /// a global scratchpad array. Since each team has a distinct slot, this
+ /// can be done without locking.
+ ///
+ /// The last team to write to the scratchpad array proceeds to reduce the
+ /// scratchpad array. One or more workers in the last team use the helper
+ /// 'loadAndReduceDataFn' to load and reduce values from the array, i.e.,
+ /// the k'th worker reduces every k'th element.
+ ///
+ /// Finally, a call is made to '__kmpc_nvptx_parallel_reduce_nowait_v2' to
+ /// reduce across workers and compute a globally reduced value.
+ ///
/// \param Loc The location where the reduction was
/// encountered. Must be within the associate
/// directive and after the last local access to the
/// reduction variables.
/// \param AllocaIP An insertion point suitable for allocas usable
/// in reductions.
+ /// \param AllocaIP An insertion point suitable for code generation.
/// \param ReductionInfos A list of info on each reduction variable.
- /// \param IsNoWait A flag set if the reduction is marked as nowait.
- InsertPointTy createReductionsGPU(const LocationDescription &Loc,
- InsertPointTy AllocaIP,
- ArrayRef<ReductionInfo> ReductionInfos,
- bool IsNoWait = false,
- bool IsTeamsReduction = false,
- bool HasDistribute = false);
+ /// \param IsNoWait Optional flag set if the reduction is marked as
+ /// nowait.
+ /// \param IsTeamsReduction Optional flag set if it is a teams
+ /// reduction.
+ /// \param HasDistribute Optional flag set if it is a
+ /// distribute reduction.
+ /// \param GridValue Optional GPU grid value.
+ /// \param GridValue Optional GPU grid value.
+ /// \param ReductionBufNum Optional OpenMPCUDAReductionBufNumValue to be
+ /// used for teams reduction.
+ InsertPointTy createReductionsGPU(
+ const LocationDescription &Loc, InsertPointTy AllocaIP,
+ InsertPointTy CodeGenIP, ArrayRef<ReductionInfo> ReductionInfos,
+ bool IsNoWait = false, bool IsTeamsReduction = false,
+ bool HasDistribute = false,
+ ReductionGenCBTy ReductionGenCBTy = ReductionGenCBTy::MLIR,
+ std::optional<omp::GV> GridValue = {}, unsigned ReductionBufNum = 1024,
+ Value *SrcLocInfo = nullptr);
// TODO: provide atomic and non-atomic reduction generators for reduction
// operators defined by the OpenMP specification.
@@ -1437,9 +1966,6 @@ class OpenMPIRBuilder {
bool IsNoWait = false, bool IsByRef = false,
bool IsTeamsReduction = false,
bool HasDistribute = false);
-
- ///}
-
/// Return the insertion point used by the underlying IRBuilder.
InsertPointTy getInsertionPoint() { return Builder.saveIP(); }
@@ -1513,19 +2039,6 @@ class OpenMPIRBuilder {
Value *NumThreads, Value *HostPtr,
ArrayRef<Value *> KernelArgs);
- /// Generate a barrier runtime call.
- ///
- /// \param Loc The location at which the request originated and is fulfilled.
- /// \param DK The directive which caused the barrier
- /// \param ForceSimpleCall Flag to force a simple (=non-cancellation) barrier.
- /// \param CheckCancelFlag Flag to indicate a cancel barrier return value
- /// should be checked and acted upon.
- ///
- /// \returns The insertion point after the barrier.
- InsertPointTy emitBarrierImpl(const LocationDescription &Loc,
- omp::Directive DK, bool ForceSimpleCall,
- bool CheckCancelFlag);
-
/// Generate a flush runtime call.
///
/// \param Loc The location at which the request originated and is fulfilled.
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 2b48df24e7e734..435af68e344a8a 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -147,15 +147,6 @@ static bool isValidWorkshareLoopScheduleType(OMPScheduleType SchedType) {
Function *GLOBAL_ReductionFunc = nullptr;
-static uint64_t getTypeSizeInBytes(Module &M, Type *Type) {
- return divideCeil(M.getDataLayout().getTypeSizeInBits(Type), 8);
-}
-
-static Value *getTypeSizeInBytesValue(IRBuilder<> &Builder, Module &M,
- Type *Type) {
- return Builder.getInt64(getTypeSizeInBytes(M, Type));
-}
-
static const omp::GV &getGridValue(const Triple &T, StringRef Features) {
if (T.isAMDGPU()) {
if (Features.count("+wavefrontsize64"))
@@ -812,6 +803,12 @@ void OpenMPIRBuilder::finalize(Function *Fn) {
if (!OffloadInfoManager.empty())
createOffloadEntriesAndInfoMetadata(ErrorReportFn);
+
+ if (Config.EmitLLVMUsed) {
+ std::vector<WeakTrackingVH> LLVMCompilerUsed = {
+ M.getGlobalVariable("__openmp_nvptx_data_transfer_temporary_storage")};
+ emitUsed("llvm.compiler.used", LLVMCompilerUsed);
+ }
}
OpenMPIRBuilder::~OpenMPIRBuilder() {
@@ -944,16 +941,12 @@ Value *OpenMPIRBuilder::getOrCreateThreadID(Value *Ident) {
}
OpenMPIRBuilder::InsertPointTy
-OpenMPIRBuilder::createBarrier(const LocationDescription &Loc, Directive DK,
- bool ForceSimpleCall, bool CheckCancelFlag) {
+OpenMPIRBuilder::createBarrier(const LocationDescription &Loc, Directive Kind,
+ bool ForceSimpleCall, bool CheckCancelFlag,
+ Value *ThreadID) {
if (!updateToLocation(Loc))
return Loc.IP;
- return emitBarrierImpl(Loc, DK, ForceSimpleCall, CheckCancelFlag);
-}
-OpenMPIRBuilder::InsertPointTy
-OpenMPIRBuilder::emitBarrierImpl(const LocationDescription &Loc, Directive Kind,
- bool ForceSimpleCall, bool CheckCancelFlag) {
// Build call __kmpc_cancel_barrier(loc, thread_id) or
// __kmpc_barrier(loc, thread_id);
@@ -978,9 +971,11 @@ OpenMPIRBuilder::emitBarrierImpl(const LocationDescription &Loc, Directive Kind,
uint32_t SrcLocStrSize;
Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
- Value *Args[] = {
- getOrCreateIdent(SrcLocStr, SrcLocStrSize, BarrierLocFlags),
- getOrCreateThreadID(getOrCreateIdent(SrcLocStr, SrcLocStrSize))};
+ if (!ThreadID)
+ ThreadID = getOrCreateThreadID(getOrCreateIdent(SrcLocStr, SrcLocStrSize));
+
+ Value *Args[] = {getOrCreateIdent(SrcLocStr, SrcLocStrSize, BarrierLocFlags),
+ ThreadID};
// If we are in a cancellable parallel region, barriers are cancellation
// points.
@@ -2116,219 +2111,273 @@ OpenMPIRBuilder::createSection(const LocationDescription &Loc,
/*IsCancellable*/ true);
}
-static Value *getGPUWarpSize(Module &M, OpenMPIRBuilder &OMPBuilder) {
- return OMPBuilder.Builder.CreateCall(
- OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_get_warp_size),
- {});
+static OpenMPIRBuilder::InsertPointTy getInsertPointAfterInstr(Instruction *I) {
+ BasicBlock::iterator IT(I);
+ IT++;
+ return OpenMPIRBuilder::InsertPointTy(I->getParent(), IT);
}
-static Value *getGPUThreadID(Module &M, OpenMPIRBuilder &OMPBuilder) {
- return OMPBuilder.Builder.CreateCall(
- OMPBuilder.getOrCreateRuntimeFunction(
- M, OMPRTL___kmpc_get_hardware_thread_id_in_block),
- {});
+void OpenMPIRBuilder::emitUsed(StringRef Name,
+ std::vector<WeakTrackingVH> &List) {
+ if (List.empty())
+ return;
+
+ // Convert List to what ConstantArray needs.
+ SmallVector<Constant *, 8> UsedArray;
+ UsedArray.resize(List.size());
+ for (unsigned I = 0, E = List.size(); I != E; ++I)
+ UsedArray[I] = ConstantExpr::getPointerBitCastOrAddrSpaceCast(
+ cast<Constant>(&*List[I]), Builder.getPtrTy());
+
+ if (UsedArray.empty())
+ return;
+ ArrayType *ATy = ArrayType::get(Builder.getPtrTy(), UsedArray.size());
+
+ auto *GV = new GlobalVariable(M, ATy, false, GlobalValue::AppendingLinkage,
+ ConstantArray::get(ATy, UsedArray), Name);
+
+ GV->setSection("llvm.metadata");
}
-static Value *getGPUNumThreads(Module &M, OpenMPIRBuilder &OMPBuilder) {
- const char *LocSize = "__kmpc_get_hardware_num_threads_in_block";
- llvm::Function *F = M.getFunction(LocSize);
- if (!F) {
- LLVMContext &Ctx = M.getContext();
- Type *I32Type = Type::getInt32Ty(Ctx);
+Value *OpenMPIRBuilder::getGPUThreadID() {
+ return Builder.CreateCall(
+ getOrCreateRuntimeFunction(M,
+ OMPRTL___kmpc_get_hardware_thread_id_in_block),
+ {});
+}
- F = Function::Create(FunctionType::get(I32Type, std::nullopt, false),
- GlobalVariable::ExternalLinkage, LocSize, M);
- }
- return OMPBuilder.Builder.CreateCall(F, std::nullopt, "nvptx_num_threads");
+Value *OpenMPIRBuilder::getGPUWarpSize() {
+ return Builder.CreateCall(
+ getOrCreateRuntimeFunction(M, OMPRTL___kmpc_get_warp_size), {});
}
-static Value *getNVPTXWarpID(Module &M, OpenMPIRBuilder &OMPIRBuilder) {
- unsigned LaneIDBits =
- llvm::Log2_32(OMPIRBuilder.Config.getGridValue().GV_Warp_Size);
- return OMPIRBuilder.Builder.CreateAShr(getGPUThreadID(M, OMPIRBuilder),
- LaneIDBits, "nvptx_warp_id");
+Value *OpenMPIRBuilder::getNVPTXWarpID() {
+ unsigned LaneIDBits = Log2_32(Config.getGridValue().GV_Warp_Size);
+ return Builder.CreateAShr(getGPUThreadID(), LaneIDBits, "nvptx_warp_id");
}
-static Value *getNVPTXLaneID(Module &M, OpenMPIRBuilder &OMPIRBuilder) {
- unsigned LaneIDBits =
- llvm::Log2_32(OMPIRBuilder.Config.getGridValue().GV_Warp_Size);
+Value *OpenMPIRBuilder::getNVPTXLaneID() {
+ unsigned LaneIDBits = Log2_32(Config.getGridValue().GV_Warp_Size);
assert(LaneIDBits < 32 && "Invalid LaneIDBits size in NVPTX device.");
unsigned LaneIDMask = ~0u >> (32u - LaneIDBits);
- return OMPIRBuilder.Builder.CreateAnd(
- getGPUThreadID(M, OMPIRBuilder),
- OMPIRBuilder.Builder.getInt32(LaneIDMask), "nvptx_lane_id");
+ return Builder.CreateAnd(getGPUThreadID(), Builder.getInt32(LaneIDMask),
+ "nvptx_lane_id");
}
-namespace {
-enum CopyAction : unsigned {
- // RemoteLaneToThread: Copy over a Reduce list from a remote lane in
- // the warp using shuffle instructions.
- RemoteLaneToThread,
- // ThreadCopy: Make a copy of a Reduce list on the thread's stack.
- ThreadCopy,
-};
-} // namespace
-
-struct CopyOptionsTy {
- llvm::Value *RemoteLaneOffset;
- llvm::Value *ScratchpadIndex;
- llvm::Value *ScratchpadWidth;
-};
-
-static Value *castValueToType(Module &M, OpenMPIRBuilder &OMPBuilder,
- Value *From, Type *ToType,
- OpenMPIRBuilder::InsertPointTy AllocaIP,
- const OpenMPIRBuilder::LocationDescription &Loc) {
- IRBuilder<> &Builder = OMPBuilder.Builder;
+Value *OpenMPIRBuilder::castValueToType(InsertPointTy AllocaIP, Value *From,
+ Type *ToType) {
Type *FromType = From->getType();
- uint64_t FromSize =
- divideCeil(M.getDataLayout().getTypeSizeInBits(FromType), 8);
- uint64_t ToSize = divideCeil(M.getDataLayout().getTypeSizeInBits(ToType), 8);
+ uint64_t FromSize = M.getDataLayout().getTypeStoreSize(FromType);
+ uint64_t ToSize = M.getDataLayout().getTypeStoreSize(ToType);
assert(FromSize > 0 && "From size must be greater than zero");
- assert(ToSize > 0 && "From size must be greater than zero");
+ assert(ToSize > 0 && "To size must be greater than zero");
if (FromType == ToType)
return From;
if (FromSize == ToSize)
return Builder.CreateBitCast(From, ToType);
if (ToType->isIntegerTy() && FromType->isIntegerTy())
- // FIXME(JAN): Assuming signed integer here, not sure how to find out
- // if unsigned
return Builder.CreateIntCast(From, ToType, /*isSigned*/ true);
- OpenMPIRBuilder::InsertPointTy CurIP = Builder.saveIP();
+ InsertPointTy SaveIP = Builder.saveIP();
Builder.restoreIP(AllocaIP);
- Value *CastItem = Builder.CreateAlloca(ToType, nullptr, "cast_tmp");
- Builder.restoreIP(CurIP);
+ Value *CastItem = Builder.CreateAlloca(ToType);
+ Builder.restoreIP(SaveIP);
Value *ValCastItem = Builder.CreatePointerBitCastOrAddrSpaceCast(
- CastItem, FromType->getPointerTo(), "valcastitem");
+ CastItem, FromType->getPointerTo());
Builder.CreateStore(From, ValCastItem);
- return Builder.CreateLoad(ToType, CastItem, "castitemload");
+ return Builder.CreateLoad(ToType, CastItem);
}
-static Value *
-createRuntimeShuffleFunction(Module &M, OpenMPIRBuilder &OMPBuilder,
- const OpenMPIRBuilder::LocationDescription &Loc,
- OpenMPIRBuilder::InsertPointTy AllocaIP,
- Value *Element, Type *ElementType, Value *Offset) {
- LLVMContext &Ctx = M.getContext();
- IRBuilder<> &Builder = OMPBuilder.Builder;
- uint64_t Size =
- divideCeil(M.getDataLayout().getTypeSizeInBits(ElementType), 8);
+Value *OpenMPIRBuilder::createRuntimeShuffleFunction(InsertPointTy AllocaIP,
+ Value *Element,
+ Type *ElementType,
+ Value *Offset) {
+ uint64_t Size = M.getDataLayout().getTypeStoreSize(ElementType);
assert(Size <= 8 && "Unsupported bitwidth in shuffle instruction");
- Function *ShuffleFunc = OMPBuilder.getOrCreateRuntimeFunctionPtr(
+
+ // Cast all types to 32- or 64-bit values before calling shuffle routines.
+ Type *CastTy = Builder.getIntNTy(Size <= 4 ? 32 : 64);
+ Value *ElemCast = castValueToType(AllocaIP, Element, CastTy);
+ Value *WarpSize =
+ Builder.CreateIntCast(getGPUWarpSize(), Builder.getInt16Ty(), true);
+ Function *ShuffleFunc = getOrCreateRuntimeFunctionPtr(
Size <= 4 ? RuntimeFunction::OMPRTL___kmpc_shuffle_int32
: RuntimeFunction::OMPRTL___kmpc_shuffle_int64);
- Type *IntType = Builder.getIntNTy(Size <= 4 ? 32 : 64);
- Value *ElemCast = Builder.CreateCast(Instruction::SExt, Element, IntType);
- Value *WarpSize = getGPUWarpSize(M, OMPBuilder);
Value *WarpSizeCast =
- Builder.CreateIntCast(WarpSize, Type::getInt16Ty(Ctx), /*isSigned=*/true);
+ Builder.CreateIntCast(WarpSize, Builder.getInt16Ty(), /*isSigned=*/true);
Value *ShuffleCall =
Builder.CreateCall(ShuffleFunc, {ElemCast, Offset, WarpSizeCast});
- return castValueToType(M, OMPBuilder, ShuffleCall, IntType, AllocaIP, Loc);
-}
-
-static void shuffleAndStore(Value *SrcAddr, Value *DstAddr, Type *ElementType,
- llvm::Value *Offset, Type *ReductionArrayTy,
- const OpenMPIRBuilder::LocationDescription &Loc,
- Module &M, OpenMPIRBuilder &OMPBuilder,
- OpenMPIRBuilder::InsertPointTy AllocaIP) {
- LLVMContext &Ctx = M.getContext();
- IRBuilder<> &Builder = OMPBuilder.Builder;
- uint64_t Size =
- divideCeil(M.getDataLayout().getTypeSizeInBits(ElementType), 8);
- Type *PtrTy = PointerType::getUnqual(Ctx);
+ return castValueToType(AllocaIP, ShuffleCall, CastTy);
+}
+
+void OpenMPIRBuilder::shuffleAndStore(InsertPointTy AllocaIP, Value *SrcAddr,
+ Value *DstAddr, Type *ElemType,
+ Value *Offset, Type *ReductionArrayTy) {
+ uint64_t Size = M.getDataLayout().getTypeStoreSize(ElemType);
+ // Create the loop over the big sized data.
+ // ptr = (void*)Elem;
+ // ptrEnd = (void*) Elem + 1;
+ // Step = 8;
+ // while (ptr + Step < ptrEnd)
+ // shuffle((int64_t)*ptr);
+ // Step = 4;
+ // while (ptr + Step < ptrEnd)
+ // shuffle((int32_t)*ptr);
+ // ...
+ Type *IndexTy = Builder.getIndexTy(
+ M.getDataLayout(), M.getDataLayout().getDefaultGlobalsAddressSpace());
Value *ElemPtr = DstAddr;
Value *Ptr = SrcAddr;
- // Value *PtrEnd = Builder.CreatePointerBitCastOrAddrSpaceCast(
- // Builder.CreateConstGEP1_64(ReductionArrayTy, SrcAddr, 1), PtrTy);
- for (int IntSize = 8; IntSize >= 1; IntSize /= 2) {
+ for (unsigned IntSize = 8; IntSize >= 1; IntSize /= 2) {
if (Size < IntSize)
continue;
- // FIXME(JAN): Check if there is a function to convert from bytes to bits
- Type *IntTy = Builder.getIntNTy(IntSize * 8);
+ Type *IntType = Builder.getIntNTy(IntSize * 8);
Ptr = Builder.CreatePointerBitCastOrAddrSpaceCast(
- Ptr, IntTy->getPointerTo(), "ptrcast");
+ Ptr, IntType->getPointerTo(), Ptr->getName() + ".ascast");
+ Value *SrcAddrGEP =
+ Builder.CreateGEP(ElemType, SrcAddr, {ConstantInt::get(IndexTy, 1)});
ElemPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
- ElemPtr, IntTy->getPointerTo(), "elemptrcast");
-
- // FIXME(JAN): Implement loop to handle larger size
- assert(((Size / IntSize) <= 1) && "Unsupported IntSize");
- Value *Val = Builder.CreateLoad(IntTy, Ptr);
- Value *Res = createRuntimeShuffleFunction(M, OMPBuilder, Loc, AllocaIP, Val,
- IntTy, Offset);
- Builder.CreateStore(Res, ElemPtr);
- Ptr = Builder.CreateConstGEP1_64(ReductionArrayTy, Ptr, 1, "ptrgep");
- ElemPtr =
- Builder.CreateConstGEP1_64(ReductionArrayTy, ElemPtr, 1, "elemptrgep");
+ ElemPtr, IntType->getPointerTo(), ElemPtr->getName() + ".ascast");
+
+ Function *CurFunc = Builder.GetInsertBlock()->getParent();
+ if ((Size / IntSize) > 1) {
+ Value *PtrEnd = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ SrcAddrGEP, Builder.getPtrTy());
+ BasicBlock *PreCondBB =
+ BasicBlock::Create(M.getContext(), ".shuffle.pre_cond");
+ BasicBlock *ThenBB = BasicBlock::Create(M.getContext(), ".shuffle.then");
+ BasicBlock *ExitBB = BasicBlock::Create(M.getContext(), ".shuffle.exit");
+ BasicBlock *CurrentBB = Builder.GetInsertBlock();
+ emitBlock(PreCondBB, CurFunc);
+ PHINode *PhiSrc =
+ Builder.CreatePHI(Ptr->getType(), /*NumReservedValues=*/2);
+ PhiSrc->addIncoming(Ptr, CurrentBB);
+ PHINode *PhiDest =
+ Builder.CreatePHI(ElemPtr->getType(), /*NumReservedValues=*/2);
+ PhiDest->addIncoming(ElemPtr, CurrentBB);
+ Ptr = PhiSrc;
+ ElemPtr = PhiDest;
+ Value *PtrDiff = Builder.CreatePtrDiff(
+ Builder.getInt8Ty(), PtrEnd,
+ Builder.CreatePointerBitCastOrAddrSpaceCast(Ptr, Builder.getPtrTy()));
+ Builder.CreateCondBr(
+ Builder.CreateICmpSGT(PtrDiff, Builder.getInt64(IntSize - 1)), ThenBB,
+ ExitBB);
+ emitBlock(ThenBB, CurFunc);
+ Value *Res = createRuntimeShuffleFunction(
+ AllocaIP,
+ Builder.CreateAlignedLoad(
+ IntType, Ptr, M.getDataLayout().getPrefTypeAlign(ElemType)),
+ IntType, Offset);
+ Builder.CreateAlignedStore(Res, ElemPtr,
+ M.getDataLayout().getPrefTypeAlign(ElemType));
+ Value *LocalPtr =
+ Builder.CreateGEP(IntType, Ptr, {ConstantInt::get(IndexTy, 1)});
+ Value *LocalElemPtr =
+ Builder.CreateGEP(IntType, ElemPtr, {ConstantInt::get(IndexTy, 1)});
+ PhiSrc->addIncoming(LocalPtr, ThenBB);
+ PhiDest->addIncoming(LocalElemPtr, ThenBB);
+ emitBranch(PreCondBB);
+ emitBlock(ExitBB, CurFunc);
+ } else {
+ Value *Res = createRuntimeShuffleFunction(
+ AllocaIP, Builder.CreateLoad(IntType, Ptr), IntType, Offset);
+ if (ElemType->isIntegerTy() && ElemType->getScalarSizeInBits() <
+ Res->getType()->getScalarSizeInBits())
+ Res = Builder.CreateTrunc(Res, ElemType);
+ Builder.CreateStore(Res, ElemPtr);
+ Ptr = Builder.CreateGEP(IntType, Ptr, {ConstantInt::get(IndexTy, 1)});
+ ElemPtr =
+ Builder.CreateGEP(IntType, ElemPtr, {ConstantInt::get(IndexTy, 1)});
+ }
Size = Size % IntSize;
}
}
-static void
-emitReductionListCopy(CopyAction Action, Type *ReductionArrayTy,
- ArrayRef<OpenMPIRBuilder::ReductionInfo> ReductionInfos,
- Value *SrcBase, Value *DestBase, Module &M,
- OpenMPIRBuilder &OMPBuilder,
- const OpenMPIRBuilder::LocationDescription &Loc,
- OpenMPIRBuilder::InsertPointTy AllocaIP,
- CopyOptionsTy CopyOptions = {nullptr, nullptr, nullptr}) {
- LLVMContext &Ctx = M.getContext();
- IRBuilder<> &Builder = OMPBuilder.Builder;
- Type *PtrTy = PointerType::getUnqual(Ctx);
-
+void OpenMPIRBuilder::emitReductionListCopy(
+ InsertPointTy AllocaIP, CopyAction Action, Type *ReductionArrayTy,
+ ArrayRef<ReductionInfo> ReductionInfos, Value *SrcBase, Value *DestBase,
+ CopyOptionsTy CopyOptions) {
+ Type *IndexTy = Builder.getIndexTy(
+ M.getDataLayout(), M.getDataLayout().getDefaultGlobalsAddressSpace());
Value *RemoteLaneOffset = CopyOptions.RemoteLaneOffset;
+ // Iterates, element-by-element, through the source Reduce list and
+ // make a copy.
for (auto En : enumerate(ReductionInfos)) {
- const OpenMPIRBuilder::ReductionInfo &RI = En.value();
+ const ReductionInfo &RI = En.value();
Value *SrcElementAddr = nullptr;
Value *DestElementAddr = nullptr;
Value *DestElementPtrAddr = nullptr;
+ // Should we shuffle in an element from a remote lane?
bool ShuffleInElement = false;
+ // Set to true to update the pointer in the dest Reduce list to a
+ // newly created element.
bool UpdateDestListPtr = false;
// Step 1.1: Get the address for the src element in the Reduce list.
- Value *SrcElementPtrAddr = Builder.CreateConstGEP2_64(
- ReductionArrayTy, SrcBase, 0, En.index(), "srcelementptraddr");
- SrcElementAddr =
- Builder.CreateLoad(PtrTy, SrcElementPtrAddr, "srcelementaddr");
+ Value *SrcElementPtrAddr = Builder.CreateInBoundsGEP(
+ ReductionArrayTy, SrcBase,
+ {ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});
+ SrcElementAddr = Builder.CreateLoad(Builder.getPtrTy(), SrcElementPtrAddr);
// Step 1.2: Create a temporary to store the element in the destination
// Reduce list.
DestElementPtrAddr = Builder.CreateInBoundsGEP(
ReductionArrayTy, DestBase,
- {Builder.getInt64(0), Builder.getInt64(En.index())},
- "destelementptraddr");
+ {ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});
switch (Action) {
- case RemoteLaneToThread: {
- OpenMPIRBuilder::InsertPointTy CurIP = Builder.saveIP();
+ case CopyAction::RemoteLaneToThread: {
+ InsertPointTy CurIP = Builder.saveIP();
Builder.restoreIP(AllocaIP);
- DestElementAddr = Builder.CreateAlloca(RI.ElementType, nullptr,
- ".omp.reduction.element");
+ AllocaInst *DestAlloca = Builder.CreateAlloca(RI.ElementType, nullptr,
+ ".omp.reduction.element");
+ DestAlloca->setAlignment(
+ M.getDataLayout().getPrefTypeAlign(RI.ElementType));
+ DestElementAddr = DestAlloca;
+ DestElementAddr =
+ Builder.CreateAddrSpaceCast(DestElementAddr, Builder.getPtrTy(),
+ DestElementAddr->getName() + ".ascast");
Builder.restoreIP(CurIP);
ShuffleInElement = true;
UpdateDestListPtr = true;
break;
}
- case ThreadCopy: {
+ case CopyAction::ThreadCopy: {
DestElementAddr =
- Builder.CreateLoad(PtrTy, DestElementPtrAddr, "destelementaddr");
+ Builder.CreateLoad(Builder.getPtrTy(), DestElementPtrAddr);
break;
}
}
- // FIXME(JAN): Original code in clanguses <Addr>.withElementType(...)
- // check if this generates any code
-
+ // Now that all active lanes have read the element in the
+ // Reduce list, shuffle over the value from the remote lane.
if (ShuffleInElement) {
- shuffleAndStore(SrcElementAddr, DestElementAddr, RI.ElementType,
- RemoteLaneOffset, ReductionArrayTy, Loc, M, OMPBuilder,
- AllocaIP);
+ shuffleAndStore(AllocaIP, SrcElementAddr, DestElementAddr, RI.ElementType,
+ RemoteLaneOffset, ReductionArrayTy);
} else {
- // FIXME(JAN): Assume Scalar here (TEK_Scalar in Clang)
- Value *Elem = Builder.CreateLoad(RI.ElementType, SrcElementAddr);
- Builder.CreateStore(Elem, DestElementAddr);
+ switch (RI.EvaluationKind) {
+ case EvaluationKindTy::Scalar: {
+ Value *Elem = Builder.CreateLoad(RI.ElementType, SrcElementAddr);
+ // Store the source element value to the dest element address.
+ Builder.CreateStore(Elem, DestElementAddr);
+ break;
+ }
+ case EvaluationKindTy::Complex: {
+ break;
+ }
+ case EvaluationKindTy::Aggregate: {
+ Value *SizeVal = Builder.getInt64(
+ M.getDataLayout().getTypeStoreSize(RI.ElementType));
+ Builder.CreateMemCpy(
+ DestElementAddr, M.getDataLayout().getPrefTypeAlign(RI.ElementType),
+ SrcElementAddr, M.getDataLayout().getPrefTypeAlign(RI.ElementType),
+ SizeVal, false);
+ break;
+ }
+ };
}
+
// Step 3.1: Modify reference in dest Reduce list as needed.
// Modifying the reference in Reduce list to point to the newly
// created element. The element is live in the current function
@@ -2336,443 +2385,500 @@ emitReductionListCopy(CopyAction Action, Type *ReductionArrayTy,
// RemoteReduceData[i] = (void*)&RemoteElem
if (UpdateDestListPtr) {
Value *CastDestAddr = Builder.CreatePointerBitCastOrAddrSpaceCast(
- DestElementAddr, PtrTy, "castdestaddr");
+ DestElementAddr, Builder.getPtrTy(),
+ DestElementAddr->getName() + ".ascast");
Builder.CreateStore(CastDestAddr, DestElementPtrAddr);
}
}
}
-static OpenMPIRBuilder::InsertPointTy getIPAfterInstr(Instruction *I) {
- BasicBlock::iterator it(I);
- it++;
- return OpenMPIRBuilder::InsertPointTy(I->getParent(), it);
-}
-
-static Function *emitShuffleAndReduceFunction(
- Module &M, const OpenMPIRBuilder::LocationDescription &Loc,
- ArrayRef<OpenMPIRBuilder::ReductionInfo> ReductionInfos, Function *ReduceFn,
- OpenMPIRBuilder &OMPBuilder) {
- IRBuilder<> &Builder = OMPBuilder.Builder;
-
- LLVMContext &Ctx = M.getContext();
- Type *VoidTy = Type::getVoidTy(Ctx);
- Type *PtrTy = PointerType::getUnqual(Ctx);
- Type *I16Type = Type::getInt16Ty(Ctx);
- auto FuncTy = FunctionType::get(VoidTy, {PtrTy, I16Type, I16Type, I16Type},
- /* IsVarArg */ false);
- Function *SarFunc =
- Function::Create(FuncTy, GlobalVariable::InternalLinkage,
- "_omp_reduction_shuffle_and_reduce_func", &M);
- SarFunc->setDoesNotRecurse();
-
- // Set arg names
- Argument *Arg0 = SarFunc->getArg(0);
- Argument *Arg1 = SarFunc->getArg(1);
- Argument *Arg2 = SarFunc->getArg(2);
- Argument *Arg3 = SarFunc->getArg(3);
- Arg0->setName("reduce_list_arg");
- Arg1->setName("lane_id_arg");
- Arg2->setName("remote_lane_offset_arg");
- Arg3->setName("algo_ver_arg");
-
- BasicBlock *EntryBlock = BasicBlock::Create(Ctx, "", SarFunc);
- Builder.SetInsertPoint(EntryBlock);
-
- Type *Arg0Type = Arg0->getType();
- Type *ArgNType = Arg1->getType();
- Type *ArgNPtrType = Arg1->getType()->getPointerTo();
- Value *ReduceListAlloca =
- Builder.CreateAlloca(Arg0Type, nullptr, Arg0->getName() + ".addr");
- Value *LaneIdAlloca =
- Builder.CreateAlloca(ArgNType, nullptr, Arg1->getName() + ".addr");
- Value *RemoteLaneOffsetAlloca =
- Builder.CreateAlloca(ArgNType, nullptr, Arg2->getName() + ".addr");
- Value *AlgoVerAlloca =
- Builder.CreateAlloca(ArgNType, nullptr, Arg3->getName() + ".addr");
- // FIXME(Jan): Compute reduction list array type
- auto *RedListArrayTy = ArrayType::get(PtrTy, 1);
- Instruction *RemoteReductionListAlloca = Builder.CreateAlloca(
- RedListArrayTy, nullptr, ".omp.reduction.remote_reduce_list");
-
- Value *ReduceListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
- ReduceListAlloca, Arg0Type, ReduceListAlloca->getName() + ".acast");
- Value *LaneIdAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
- LaneIdAlloca, ArgNPtrType, LaneIdAlloca->getName() + ".acast");
- Value *RemoteLaneOffsetAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
- RemoteLaneOffsetAlloca, ArgNPtrType,
- RemoteLaneOffsetAlloca->getName() + ".acast");
- Value *AlgoVerAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
- AlgoVerAlloca, ArgNPtrType, AlgoVerAlloca->getName() + ".acast");
- Value *RemoteListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
- RemoteReductionListAlloca, PtrTy,
- RemoteReductionListAlloca->getName() + ".acast");
-
- Builder.CreateStore(Arg0, ReduceListAddrCast);
- Builder.CreateStore(Arg1, LaneIdAddrCast);
- Builder.CreateStore(Arg2, RemoteLaneOffsetAddrCast);
- Builder.CreateStore(Arg3, AlgoVerAddrCast);
-
- Value *ReduceList =
- Builder.CreateLoad(Arg0Type, ReduceListAddrCast, "reduce_list");
- Value *LaneId = Builder.CreateLoad(ArgNType, LaneIdAddrCast, "lane_id");
- Value *RemoteLaneOffset = Builder.CreateLoad(
- ArgNType, RemoteLaneOffsetAddrCast, "remote_lane_offset");
- Value *AlgoVer = Builder.CreateLoad(ArgNType, AlgoVerAddrCast, "algo_ver");
-
- OpenMPIRBuilder::InsertPointTy AllocaIP =
- getIPAfterInstr(RemoteReductionListAlloca);
- emitReductionListCopy(RemoteLaneToThread, RedListArrayTy, ReductionInfos,
- ReduceList, RemoteListAddrCast, M, OMPBuilder, Loc,
- AllocaIP, {RemoteLaneOffset, nullptr, nullptr});
-
- // The actions to be performed on the Remote Reduce list is dependent
- // on the algorithm version.
- //
- // if (AlgoVer==0) || (AlgoVer==1 && (LaneId < Offset)) || (AlgoVer==2 &&
- // LaneId % 2 == 0 && Offset > 0):
- // do the reduction value aggregation
- //
- // The thread local variable Reduce list is mutated in place to host the
- // reduced data, which is the aggregated value produced from local and
- // remote lanes.
- //
- // Note that AlgoVer is expected to be a constant integer known at compile
- // time.
- // When AlgoVer==0, the first conjunction evaluates to true, making
- // the entire predicate true during compile time.
- // When AlgoVer==1, the second conjunction has only the second part to be
- // evaluated during runtime. Other conjunctions evaluates to false
- // during compile time.
- // When AlgoVer==2, the third conjunction has only the second part to be
- // evaluated during runtime. Other conjunctions evaluates to false
- // during compile time.
- Value *CondAlgo0 = Builder.CreateIsNull(AlgoVer);
- Value *Algo1 = Builder.CreateICmpEQ(AlgoVer, Builder.getInt16(1));
- Value *LaneComp = Builder.CreateICmpULT(LaneId, RemoteLaneOffset);
- Value *CondAlgo1 = Builder.CreateAnd(Algo1, LaneComp);
- Value *Algo2 = Builder.CreateICmpEQ(AlgoVer, Builder.getInt16(2));
- Value *LaneIdAnd1 = Builder.CreateAnd(LaneId, Builder.getInt16(1));
- Value *LaneIdComp = Builder.CreateIsNull(LaneIdAnd1);
- Value *Algo2AndLaneIdComp = Builder.CreateAnd(Algo2, LaneIdComp);
- Value *RemoteOffsetComp =
- Builder.CreateICmpSGT(RemoteLaneOffset, Builder.getInt16(0));
- Value *CondAlgo2 = Builder.CreateAnd(Algo2AndLaneIdComp, RemoteOffsetComp);
- Value *CA0OrCA1 = Builder.CreateOr(CondAlgo0, CondAlgo1);
- Value *CondReduce = Builder.CreateOr(CA0OrCA1, CondAlgo2);
-
- BasicBlock *ThenBB = BasicBlock::Create(Ctx, "then", SarFunc);
- BasicBlock *ElseBB = BasicBlock::Create(Ctx, "else", SarFunc);
- BasicBlock *MergeBB = BasicBlock::Create(Ctx, "ifcont", SarFunc);
-
- Builder.CreateCondBr(CondReduce, ThenBB, ElseBB);
- Builder.SetInsertPoint(ThenBB);
- // reduce_function(LocalReduceList, RemoteReduceList)
- Value *LocalReduceListPtr =
- Builder.CreatePointerBitCastOrAddrSpaceCast(ReduceList, PtrTy);
- Value *RemoteReduceListPtr =
- Builder.CreatePointerBitCastOrAddrSpaceCast(RemoteListAddrCast, PtrTy);
- Builder.CreateCall(ReduceFn, {LocalReduceListPtr, RemoteReduceListPtr});
- Builder.CreateBr(MergeBB);
- Builder.SetInsertPoint(ElseBB);
- Builder.CreateBr(MergeBB);
- Builder.SetInsertPoint(MergeBB);
-
- Value *Algo1_2 = Builder.CreateICmpEQ(AlgoVer, Builder.getInt16(1));
- Value *LaneIdGtOffset = Builder.CreateICmpUGE(LaneId, RemoteLaneOffset);
- Value *CondCopy = Builder.CreateAnd(Algo1_2, LaneIdGtOffset);
-
- BasicBlock *CpyThenBB = BasicBlock::Create(Ctx, "cpy_then", SarFunc);
- BasicBlock *CpyElseBB = BasicBlock::Create(Ctx, "cpy_else", SarFunc);
- BasicBlock *CpyMergeBB = BasicBlock::Create(Ctx, "cpy_ifcont", SarFunc);
-
- Builder.CreateCondBr(CondCopy, CpyThenBB, CpyElseBB);
-
- Builder.SetInsertPoint(CpyThenBB);
- emitReductionListCopy(ThreadCopy, RedListArrayTy, ReductionInfos,
- RemoteListAddrCast, ReduceList, M, OMPBuilder, Loc,
- AllocaIP);
- Builder.CreateBr(CpyMergeBB);
- Builder.SetInsertPoint(CpyElseBB);
- Builder.CreateBr(CpyMergeBB);
- Builder.SetInsertPoint(CpyMergeBB);
- Builder.CreateRetVoid();
-
- return SarFunc;
-}
-
-static Function *emitInterWarpCopyFunction(
- Module &M, const OpenMPIRBuilder::LocationDescription &Loc,
- ArrayRef<OpenMPIRBuilder::ReductionInfo> ReductionInfos,
- OpenMPIRBuilder &OMPBuilder) {
- IRBuilder<> &Builder = OMPBuilder.Builder;
- OpenMPIRBuilder::InsertPointTy OldIP = Builder.saveIP();
+Function *OpenMPIRBuilder::emitInterWarpCopyFunction(
+ const LocationDescription &Loc, ArrayRef<ReductionInfo> ReductionInfos,
+ AttributeList FuncAttrs) {
+ InsertPointTy SavedIP = Builder.saveIP();
LLVMContext &Ctx = M.getContext();
- Type *VoidTy = Type::getVoidTy(Ctx);
- Type *PtrTy = PointerType::getUnqual(Ctx);
- Type *I32Type = Type::getInt32Ty(Ctx);
- auto FuncTy =
- FunctionType::get(VoidTy, {PtrTy, I32Type}, /* IsVarArg */ false);
+ FunctionType *FuncTy = FunctionType::get(
+ Builder.getVoidTy(), {Builder.getPtrTy(), Builder.getInt32Ty()},
+ /* IsVarArg */ false);
Function *WcFunc =
Function::Create(FuncTy, GlobalVariable::InternalLinkage,
"_omp_reduction_inter_warp_copy_func", &M);
- WcFunc->setDoesNotRecurse();
-
- // Set arg names
- Argument *Arg0 = WcFunc->getArg(0);
- Argument *Arg1 = WcFunc->getArg(1);
- Arg0->setName("reduce_list");
- Arg1->setName("num_warps");
-
- // Ensure data transfer storage
- unsigned WarpSize = OMPBuilder.Config.getGridValue().GV_Warp_Size;
- // FIXME(Jan): Not sure about the array type here, but it is I32 in Clang
- auto *ArrayTy = ArrayType::get(I32Type, WarpSize);
+ WcFunc->setAttributes(FuncAttrs);
+ WcFunc->addParamAttr(0, Attribute::NoUndef);
+ WcFunc->addParamAttr(1, Attribute::NoUndef);
+ BasicBlock *EntryBB = BasicBlock::Create(M.getContext(), "entry", WcFunc);
+ Builder.SetInsertPoint(EntryBB);
+
+ // ReduceList: thread local Reduce list.
+ // At the stage of the computation when this function is called, partially
+ // aggregated values reside in the first lane of every active warp.
+ Argument *ReduceListArg = WcFunc->getArg(0);
+ // NumWarps: number of warps active in the parallel region. This could
+ // be smaller than 32 (max warps in a CTA) for partial block reduction.
+ Argument *NumWarpsArg = WcFunc->getArg(1);
+
+ // This array is used as a medium to transfer, one reduce element at a time,
+ // the data from the first lane of every warp to lanes in the first warp
+ // in order to perform the final step of a reduction in a parallel region
+ // (reduction across warps). The array is placed in NVPTX __shared__ memory
+ // for reduced latency, as well as to have a distinct copy for concurrently
+ // executing target regions. The array is declared with common linkage so
+ // as to be shared across compilation units.
StringRef TransferMediumName =
"__openmp_nvptx_data_transfer_temporary_storage";
GlobalVariable *TransferMedium = M.getGlobalVariable(TransferMediumName);
+ unsigned WarpSize = Config.getGridValue().GV_Warp_Size;
+ ArrayType *ArrayTy = ArrayType::get(Builder.getInt32Ty(), WarpSize);
if (!TransferMedium) {
- unsigned SharedAddressSpace =
- 3; /* FIXME(Jan): C.getTargetAddressSpace(LangAS::cuda_shared); */
TransferMedium = new GlobalVariable(
M, ArrayTy, /*isConstant=*/false, GlobalVariable::WeakAnyLinkage,
UndefValue::get(ArrayTy), TransferMediumName,
/*InsertBefore=*/nullptr, GlobalVariable::NotThreadLocal,
- SharedAddressSpace);
+ /*AddressSpace=*/3);
}
- BasicBlock *EntryBlock = BasicBlock::Create(Ctx, "", WcFunc);
- Builder.SetInsertPoint(EntryBlock);
+ uint32_t SrcLocStrSize;
+ Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
- Type *Arg0Type = Arg0->getType();
- Type *Arg1Type = Arg1->getType();
- Value *ReduceListAlloca =
- Builder.CreateAlloca(Arg0Type, nullptr, Arg0->getName() + ".addr");
- Instruction *NumWarpsAlloca =
- Builder.CreateAlloca(Arg1Type, nullptr, Arg1->getName() + ".addr");
+ // Get the CUDA thread id of the current OpenMP thread on the GPU.
+ Value *GPUThreadID = getGPUThreadID();
+ // nvptx_lane_id = nvptx_id % warpsize
+ Value *LaneID = getNVPTXLaneID();
+ // nvptx_warp_id = nvptx_id / warpsize
+ Value *WarpID = getNVPTXWarpID();
+
+ InsertPointTy AllocaIP =
+ InsertPointTy(Builder.GetInsertBlock(),
+ Builder.GetInsertBlock()->getFirstInsertionPt());
+ Type *Arg0Type = ReduceListArg->getType();
+ Type *Arg1Type = NumWarpsArg->getType();
+ Builder.restoreIP(AllocaIP);
+ AllocaInst *ReduceListAlloca = Builder.CreateAlloca(
+ Arg0Type, nullptr, ReduceListArg->getName() + ".addr");
+ AllocaInst *NumWarpsAlloca =
+ Builder.CreateAlloca(Arg1Type, nullptr, NumWarpsArg->getName() + ".addr");
+ Value *ThreadID =
+ getOrCreateThreadID(getOrCreateIdent(SrcLocStr, SrcLocStrSize));
Value *ReduceListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
- ReduceListAlloca, Arg0Type, ReduceListAlloca->getName() + ".acast");
+ ReduceListAlloca, Arg0Type, ReduceListAlloca->getName() + ".ascast");
Value *NumWarpsAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
NumWarpsAlloca, Arg1Type->getPointerTo(),
- NumWarpsAlloca->getName() + ".acast");
- Builder.CreateStore(Arg0, ReduceListAddrCast);
- Builder.CreateStore(Arg1, NumWarpsAddrCast);
-
- // Get GPU Info
- Value *ThreadID = getGPUThreadID(M, OMPBuilder);
- Value *LaneID = getNVPTXLaneID(M, OMPBuilder);
- Value *WarpID = getNVPTXWarpID(M, OMPBuilder);
+ NumWarpsAlloca->getName() + ".ascast");
+ Builder.CreateStore(ReduceListArg, ReduceListAddrCast);
+ Builder.CreateStore(NumWarpsArg, NumWarpsAddrCast);
+ AllocaIP = getInsertPointAfterInstr(NumWarpsAlloca);
+ InsertPointTy CodeGenIP =
+ getInsertPointAfterInstr(&Builder.GetInsertBlock()->back());
+ Builder.restoreIP(CodeGenIP);
- Value *ReduceListArg =
- Builder.CreateLoad(PtrTy, ReduceListAddrCast, "reduce_list_arg");
+ Value *ReduceList =
+ Builder.CreateLoad(Builder.getPtrTy(), ReduceListAddrCast);
for (auto En : enumerate(ReductionInfos)) {
- const OpenMPIRBuilder::ReductionInfo &RI = En.value();
- Type *ElementTy = RI.ElementType;
- unsigned NumTypeBits = M.getDataLayout().getTypeSizeInBits(ElementTy);
- unsigned RealTySize = divideCeil(NumTypeBits, 8);
+ //
+ // Warp master copies reduce element to transfer medium in __shared__
+ // memory.
+ //
+ const ReductionInfo &RI = En.value();
+ unsigned RealTySize = M.getDataLayout().getTypeAllocSize(RI.ElementType);
for (unsigned TySize = 4; TySize > 0 && RealTySize > 0; TySize /= 2) {
+ Type *CType = Builder.getIntNTy(TySize * 8);
+
unsigned NumIters = RealTySize / TySize;
if (NumIters == 0)
continue;
- // Type *CopyTy = Builder.getIntNTy(TySize);
- Type *Int32Ty = Builder.getInt32Ty();
Value *Cnt = nullptr;
- Value *CntAddrAcast = nullptr;
+ Value *CntAddr = nullptr;
BasicBlock *PrecondBB = nullptr;
BasicBlock *ExitBB = nullptr;
-
if (NumIters > 1) {
- OpenMPIRBuilder::InsertPointTy CurrIP = Builder.saveIP();
- Builder.SetInsertPoint(NumWarpsAlloca);
- Value *CntAddr = Builder.CreateAlloca(Int32Ty, nullptr, ".cnt.addr");
- Builder.restoreIP(CurrIP);
- CntAddrAcast = Builder.CreatePointerBitCastOrAddrSpaceCast(
- CntAddr, PtrTy, CntAddr->getName() + ".acast");
- Builder.CreateStore(Constant::getNullValue(Int32Ty), CntAddrAcast);
- PrecondBB = BasicBlock::Create(Ctx, "precond", WcFunc);
- ExitBB = BasicBlock::Create(Ctx, "exit", WcFunc);
- BasicBlock *BodyBB = BasicBlock::Create(Ctx, "body", WcFunc);
- Builder.CreateBr(PrecondBB);
- Builder.SetInsertPoint(PrecondBB);
- Cnt = Builder.CreateLoad(Int32Ty, CntAddrAcast, "cnt");
- Value *Cmp = Builder.CreateICmpULT(Cnt, Builder.getInt32(NumIters));
+ CodeGenIP = Builder.saveIP();
+ Builder.restoreIP(AllocaIP);
+ CntAddr =
+ Builder.CreateAlloca(Builder.getInt32Ty(), nullptr, ".cnt.addr");
+
+ CntAddr = Builder.CreateAddrSpaceCast(CntAddr, Builder.getPtrTy(),
+ CntAddr->getName() + ".ascast");
+ Builder.restoreIP(CodeGenIP);
+ Builder.CreateStore(Constant::getNullValue(Builder.getInt32Ty()),
+ CntAddr,
+ /*Volatile=*/false);
+ PrecondBB = BasicBlock::Create(Ctx, "precond");
+ ExitBB = BasicBlock::Create(Ctx, "exit");
+ BasicBlock *BodyBB = BasicBlock::Create(Ctx, "body");
+ emitBlock(PrecondBB, Builder.GetInsertBlock()->getParent());
+ Cnt = Builder.CreateLoad(Builder.getInt32Ty(), CntAddr,
+ /*Volatile=*/false);
+ Value *Cmp = Builder.CreateICmpULT(
+ Cnt, ConstantInt::get(Builder.getInt32Ty(), NumIters));
Builder.CreateCondBr(Cmp, BodyBB, ExitBB);
- Builder.SetInsertPoint(BodyBB);
+ emitBlock(BodyBB, Builder.GetInsertBlock()->getParent());
}
- OMPBuilder.createBarrier(
- OpenMPIRBuilder::LocationDescription(Builder.saveIP(), Loc.DL),
- omp::Directive::OMPD_unknown,
- /* ForceSimpleCall */ false,
- /* CheckCancelFlag */ true);
- BasicBlock *ThenBB = BasicBlock::Create(Ctx, "then", WcFunc);
- BasicBlock *ElseBB = BasicBlock::Create(Ctx, "else", WcFunc);
- BasicBlock *MergeBB = BasicBlock::Create(Ctx, "ifcont", WcFunc);
+ // kmpc_barrier.
+ createBarrier(LocationDescription(Builder.saveIP(), Loc.DL),
+ omp::Directive::OMPD_unknown,
+ /* ForceSimpleCall */ false,
+ /* CheckCancelFlag */ true, ThreadID);
+ BasicBlock *ThenBB = BasicBlock::Create(Ctx, "then");
+ BasicBlock *ElseBB = BasicBlock::Create(Ctx, "else");
+ BasicBlock *MergeBB = BasicBlock::Create(Ctx, "ifcont");
// if (lane_id == 0)
Value *IsWarpMaster = Builder.CreateIsNull(LaneID, "warp_master");
Builder.CreateCondBr(IsWarpMaster, ThenBB, ElseBB);
+ emitBlock(ThenBB, Builder.GetInsertBlock()->getParent());
- // then
// Reduce element = LocalReduceList[i]
- Builder.SetInsertPoint(ThenBB);
- // FIXME(JAN): Should array type be passed in?
- auto *RedListArrayTy = ArrayType::get(PtrTy, 1);
- // FIXME(JAN): maybe it should be 0,0 and not use En.index()
- Value *ReduceListElementPtrPtr = Builder.CreateConstInBoundsGEP2_64(
- RedListArrayTy, ReduceListArg, 0, En.index());
- Value *ReduceListElementPtr = Builder.CreateLoad(
- PtrTy, ReduceListElementPtrPtr, "reduce_list_element_ptr");
+ auto *RedListArrayTy =
+ ArrayType::get(Builder.getPtrTy(), ReductionInfos.size());
+ Type *IndexTy = Builder.getIndexTy(
+ M.getDataLayout(), M.getDataLayout().getDefaultGlobalsAddressSpace());
+ Value *ElemPtrPtr =
+ Builder.CreateInBoundsGEP(RedListArrayTy, ReduceList,
+ {ConstantInt::get(IndexTy, 0),
+ ConstantInt::get(IndexTy, En.index())});
+ // elemptr = ((CopyType*)(elemptrptr)) + I
+ Value *ElemPtr = Builder.CreateLoad(Builder.getPtrTy(), ElemPtrPtr);
if (NumIters > 1)
- ReduceListElementPtr =
- Builder.CreateGEP(Int32Ty, ReduceListElementPtr, Cnt);
+ ElemPtr = Builder.CreateGEP(Builder.getInt32Ty(), ElemPtr, Cnt);
- Value *TransferElemAddr = Builder.CreateInBoundsGEP(
+ // Get pointer to location in transfer medium.
+ // MediumPtr = &medium[warp_id]
+ Value *MediumPtr = Builder.CreateInBoundsGEP(
ArrayTy, TransferMedium, {Builder.getInt64(0), WarpID});
- Value *ReduceListElement = Builder.CreateLoad(
- I32Type, ReduceListElementPtr, "reduce_list_element");
- Builder.CreateStore(ReduceListElement, TransferElemAddr,
+ // elem = *elemptr
+ //*MediumPtr = elem
+ Value *Elem = Builder.CreateLoad(CType, ElemPtr);
+ // Store the source element value to the dest element address.
+ Builder.CreateStore(Elem, MediumPtr,
/*IsVolatile*/ true);
Builder.CreateBr(MergeBB);
// else
- Builder.SetInsertPoint(ElseBB);
+ emitBlock(ElseBB, Builder.GetInsertBlock()->getParent());
Builder.CreateBr(MergeBB);
// endif
- Builder.SetInsertPoint(MergeBB);
- OMPBuilder.createBarrier(
- OpenMPIRBuilder::LocationDescription(Builder.saveIP(), Loc.DL),
- omp::Directive::OMPD_unknown,
- /* ForceSimpleCall */ false,
- /* CheckCancelFlag */ true);
+ emitBlock(MergeBB, Builder.GetInsertBlock()->getParent());
+ createBarrier(LocationDescription(Builder.saveIP(), Loc.DL),
+ omp::Directive::OMPD_unknown,
+ /* ForceSimpleCall */ false,
+ /* CheckCancelFlag */ true, ThreadID);
// Warp 0 copies reduce element from transfer medium
- BasicBlock *W0ThenBB = BasicBlock::Create(Ctx, "w0then", WcFunc);
- BasicBlock *W0ElseBB = BasicBlock::Create(Ctx, "w0else", WcFunc);
- BasicBlock *W0MergeBB = BasicBlock::Create(Ctx, "w0ifcont", WcFunc);
+ BasicBlock *W0ThenBB = BasicBlock::Create(Ctx, "then");
+ BasicBlock *W0ElseBB = BasicBlock::Create(Ctx, "else");
+ BasicBlock *W0MergeBB = BasicBlock::Create(Ctx, "ifcont");
Value *NumWarpsVal =
- Builder.CreateLoad(I32Type, NumWarpsAddrCast, "num_warps");
+ Builder.CreateLoad(Builder.getInt32Ty(), NumWarpsAddrCast);
+ // Up to 32 threads in warp 0 are active.
Value *IsActiveThread =
- Builder.CreateICmpULT(ThreadID, NumWarpsVal, "is_active_thread");
+ Builder.CreateICmpULT(GPUThreadID, NumWarpsVal, "is_active_thread");
Builder.CreateCondBr(IsActiveThread, W0ThenBB, W0ElseBB);
- // W0then
- // SecMEdiumPtr = &medium[tid]
- Builder.SetInsertPoint(W0ThenBB);
- Value *SrcMediumPtrVal = Builder.CreateInBoundsGEP(
- ArrayTy, TransferMedium, {Builder.getInt64(0), ThreadID});
+ emitBlock(W0ThenBB, Builder.GetInsertBlock()->getParent());
+
+ // SecMediumPtr = &medium[tid]
// SrcMediumVal = *SrcMediumPtr
- // TODO(JAN): Bitcast here, but no load? skipping for now
- Value *TargetElementPtrPtr = Builder.CreateConstInBoundsGEP2_64(
- RedListArrayTy, ReduceListArg, 0, En.index());
- Value *TargetElementPtr = Builder.CreateLoad(PtrTy, TargetElementPtrPtr);
+ Value *SrcMediumPtrVal = Builder.CreateInBoundsGEP(
+ ArrayTy, TransferMedium, {Builder.getInt64(0), GPUThreadID});
+ // TargetElemPtr = (CopyType*)(SrcDataAddr[i]) + I
+ Value *TargetElemPtrPtr =
+ Builder.CreateInBoundsGEP(RedListArrayTy, ReduceList,
+ {ConstantInt::get(IndexTy, 0),
+ ConstantInt::get(IndexTy, En.index())});
+ Value *TargetElemPtrVal =
+ Builder.CreateLoad(Builder.getPtrTy(), TargetElemPtrPtr);
+ Value *TargetElemPtr = TargetElemPtrVal;
if (NumIters > 1)
- TargetElementPtr = Builder.CreateGEP(Int32Ty, TargetElementPtr, Cnt);
+ TargetElemPtr =
+ Builder.CreateGEP(Builder.getInt32Ty(), TargetElemPtr, Cnt);
+ // *TargetElemPtr = SrcMediumVal;
Value *SrcMediumValue =
- Builder.CreateLoad(I32Type, SrcMediumPtrVal, /*IsVolatile*/ true);
- Builder.CreateStore(SrcMediumValue, TargetElementPtr);
+ Builder.CreateLoad(CType, SrcMediumPtrVal, /*IsVolatile*/ true);
+ Builder.CreateStore(SrcMediumValue, TargetElemPtr);
Builder.CreateBr(W0MergeBB);
- // W0else
- Builder.SetInsertPoint(W0ElseBB);
+ emitBlock(W0ElseBB, Builder.GetInsertBlock()->getParent());
Builder.CreateBr(W0MergeBB);
- // W0endif
- Builder.SetInsertPoint(W0MergeBB);
+ emitBlock(W0MergeBB, Builder.GetInsertBlock()->getParent());
+
if (NumIters > 1) {
- Cnt = Builder.CreateNSWAdd(Cnt, Builder.getInt32(1));
- Builder.CreateStore(Cnt, CntAddrAcast);
- Builder.CreateBr(PrecondBB);
- Builder.SetInsertPoint(ExitBB);
+ Cnt = Builder.CreateNSWAdd(
+ Cnt, ConstantInt::get(Builder.getInt32Ty(), /*V=*/1));
+ Builder.CreateStore(Cnt, CntAddr, /*Volatile=*/false);
+
+ auto *CurFn = Builder.GetInsertBlock()->getParent();
+ emitBranch(PrecondBB);
+ emitBlock(ExitBB, CurFn);
}
+ RealTySize %= TySize;
}
}
Builder.CreateRetVoid();
- Builder.restoreIP(OldIP);
+ Builder.restoreIP(SavedIP);
+
return WcFunc;
}
-/// This function emits a helper that copies all the reduction variables from
-/// the team into the provided global buffer for the reduction variables.
-///
-/// void list_to_global_copy_func(void *buffer, int Idx, void *reduce_data)
-/// For all data entries D in reduce_data:
-/// Copy local D to buffer.D[Idx]
-static Function *emitListToGlobalCopyFunction(
- Module &M, const OpenMPIRBuilder::LocationDescription &Loc,
- ArrayRef<OpenMPIRBuilder::ReductionInfo> ReductionInfos,
- OpenMPIRBuilder &OMPBuilder) {
- IRBuilder<> &Builder = OMPBuilder.Builder;
+Function *OpenMPIRBuilder::emitShuffleAndReduceFunction(
+ ArrayRef<ReductionInfo> ReductionInfos, Function *ReduceFn,
+ AttributeList FuncAttrs) {
+ LLVMContext &Ctx = M.getContext();
+ FunctionType *FuncTy =
+ FunctionType::get(Builder.getVoidTy(),
+ {Builder.getPtrTy(), Builder.getInt16Ty(),
+ Builder.getInt16Ty(), Builder.getInt16Ty()},
+ /* IsVarArg */ false);
+ Function *SarFunc =
+ Function::Create(FuncTy, GlobalVariable::InternalLinkage,
+ "_omp_reduction_shuffle_and_reduce_func", &M);
+ SarFunc->setAttributes(FuncAttrs);
+ SarFunc->addParamAttr(0, Attribute::NoUndef);
+ SarFunc->addParamAttr(1, Attribute::NoUndef);
+ SarFunc->addParamAttr(2, Attribute::NoUndef);
+ SarFunc->addParamAttr(3, Attribute::NoUndef);
+ SarFunc->addParamAttr(1, Attribute::SExt);
+ SarFunc->addParamAttr(2, Attribute::SExt);
+ SarFunc->addParamAttr(3, Attribute::SExt);
+ BasicBlock *EntryBB = BasicBlock::Create(M.getContext(), "entry", SarFunc);
+ Builder.SetInsertPoint(EntryBB);
+
+ // Thread local Reduce list used to host the values of data to be reduced.
+ Argument *ReduceListArg = SarFunc->getArg(0);
+ // Current lane id; could be logical.
+ Argument *LaneIDArg = SarFunc->getArg(1);
+ // Offset of the remote source lane relative to the current lane.
+ Argument *RemoteLaneOffsetArg = SarFunc->getArg(2);
+ // Algorithm version. This is expected to be known at compile time.
+ Argument *AlgoVerArg = SarFunc->getArg(3);
+
+ Type *ReduceListArgType = ReduceListArg->getType();
+ Type *LaneIDArgType = LaneIDArg->getType();
+ Type *LaneIDArgPtrType = LaneIDArg->getType()->getPointerTo();
+ Value *ReduceListAlloca = Builder.CreateAlloca(
+ ReduceListArgType, nullptr, ReduceListArg->getName() + ".addr");
+ Value *LaneIdAlloca = Builder.CreateAlloca(LaneIDArgType, nullptr,
+ LaneIDArg->getName() + ".addr");
+ Value *RemoteLaneOffsetAlloca = Builder.CreateAlloca(
+ LaneIDArgType, nullptr, RemoteLaneOffsetArg->getName() + ".addr");
+ Value *AlgoVerAlloca = Builder.CreateAlloca(LaneIDArgType, nullptr,
+ AlgoVerArg->getName() + ".addr");
+ ArrayType *RedListArrayTy =
+ ArrayType::get(Builder.getPtrTy(), ReductionInfos.size());
+
+ // Create a local thread-private variable to host the Reduce list
+ // from a remote lane.
+ Instruction *RemoteReductionListAlloca = Builder.CreateAlloca(
+ RedListArrayTy, nullptr, ".omp.reduction.remote_reduce_list");
+
+ Value *ReduceListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ ReduceListAlloca, ReduceListArgType,
+ ReduceListAlloca->getName() + ".ascast");
+ Value *LaneIdAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ LaneIdAlloca, LaneIDArgPtrType, LaneIdAlloca->getName() + ".ascast");
+ Value *RemoteLaneOffsetAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ RemoteLaneOffsetAlloca, LaneIDArgPtrType,
+ RemoteLaneOffsetAlloca->getName() + ".ascast");
+ Value *AlgoVerAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ AlgoVerAlloca, LaneIDArgPtrType, AlgoVerAlloca->getName() + ".ascast");
+ Value *RemoteListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ RemoteReductionListAlloca, Builder.getPtrTy(),
+ RemoteReductionListAlloca->getName() + ".ascast");
+
+ Builder.CreateStore(ReduceListArg, ReduceListAddrCast);
+ Builder.CreateStore(LaneIDArg, LaneIdAddrCast);
+ Builder.CreateStore(RemoteLaneOffsetArg, RemoteLaneOffsetAddrCast);
+ Builder.CreateStore(AlgoVerArg, AlgoVerAddrCast);
+
+ Value *ReduceList = Builder.CreateLoad(ReduceListArgType, ReduceListAddrCast);
+ Value *LaneId = Builder.CreateLoad(LaneIDArgType, LaneIdAddrCast);
+ Value *RemoteLaneOffset =
+ Builder.CreateLoad(LaneIDArgType, RemoteLaneOffsetAddrCast);
+ Value *AlgoVer = Builder.CreateLoad(LaneIDArgType, AlgoVerAddrCast);
+
+ InsertPointTy AllocaIP = getInsertPointAfterInstr(RemoteReductionListAlloca);
+
+ // This loop iterates through the list of reduce elements and copies,
+ // element by element, from a remote lane in the warp to RemoteReduceList,
+ // hosted on the thread's stack.
+ emitReductionListCopy(
+ AllocaIP, CopyAction::RemoteLaneToThread, RedListArrayTy, ReductionInfos,
+ ReduceList, RemoteListAddrCast, {RemoteLaneOffset, nullptr, nullptr});
+
+ // The actions to be performed on the Remote Reduce list is dependent
+ // on the algorithm version.
+ //
+ // if (AlgoVer==0) || (AlgoVer==1 && (LaneId < Offset)) || (AlgoVer==2 &&
+ // LaneId % 2 == 0 && Offset > 0):
+ // do the reduction value aggregation
+ //
+ // The thread local variable Reduce list is mutated in place to host the
+ // reduced data, which is the aggregated value produced from local and
+ // remote lanes.
+ //
+ // Note that AlgoVer is expected to be a constant integer known at compile
+ // time.
+ // When AlgoVer==0, the first conjunction evaluates to true, making
+ // the entire predicate true during compile time.
+ // When AlgoVer==1, the second conjunction has only the second part to be
+ // evaluated during runtime. Other conjunctions evaluates to false
+ // during compile time.
+ // When AlgoVer==2, the third conjunction has only the second part to be
+ // evaluated during runtime. Other conjunctions evaluates to false
+ // during compile time.
+ Value *CondAlgo0 = Builder.CreateIsNull(AlgoVer);
+ Value *Algo1 = Builder.CreateICmpEQ(AlgoVer, Builder.getInt16(1));
+ Value *LaneComp = Builder.CreateICmpULT(LaneId, RemoteLaneOffset);
+ Value *CondAlgo1 = Builder.CreateAnd(Algo1, LaneComp);
+ Value *Algo2 = Builder.CreateICmpEQ(AlgoVer, Builder.getInt16(2));
+ Value *LaneIdAnd1 = Builder.CreateAnd(LaneId, Builder.getInt16(1));
+ Value *LaneIdComp = Builder.CreateIsNull(LaneIdAnd1);
+ Value *Algo2AndLaneIdComp = Builder.CreateAnd(Algo2, LaneIdComp);
+ Value *RemoteOffsetComp =
+ Builder.CreateICmpSGT(RemoteLaneOffset, Builder.getInt16(0));
+ Value *CondAlgo2 = Builder.CreateAnd(Algo2AndLaneIdComp, RemoteOffsetComp);
+ Value *CA0OrCA1 = Builder.CreateOr(CondAlgo0, CondAlgo1);
+ Value *CondReduce = Builder.CreateOr(CA0OrCA1, CondAlgo2);
+
+ BasicBlock *ThenBB = BasicBlock::Create(Ctx, "then");
+ BasicBlock *ElseBB = BasicBlock::Create(Ctx, "else");
+ BasicBlock *MergeBB = BasicBlock::Create(Ctx, "ifcont");
+
+ Builder.CreateCondBr(CondReduce, ThenBB, ElseBB);
+ emitBlock(ThenBB, Builder.GetInsertBlock()->getParent());
+ // reduce_function(LocalReduceList, RemoteReduceList)
+ Value *LocalReduceListPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ ReduceList, Builder.getPtrTy());
+ Value *RemoteReduceListPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ RemoteListAddrCast, Builder.getPtrTy());
+ Builder.CreateCall(ReduceFn, {LocalReduceListPtr, RemoteReduceListPtr})
+ ->addFnAttr(Attribute::NoUnwind);
+ Builder.CreateBr(MergeBB);
+
+ emitBlock(ElseBB, Builder.GetInsertBlock()->getParent());
+ Builder.CreateBr(MergeBB);
+
+ emitBlock(MergeBB, Builder.GetInsertBlock()->getParent());
+
+ // if (AlgoVer==1 && (LaneId >= Offset)) copy Remote Reduce list to local
+ // Reduce list.
+ Algo1 = Builder.CreateICmpEQ(AlgoVer, Builder.getInt16(1));
+ Value *LaneIdGtOffset = Builder.CreateICmpUGE(LaneId, RemoteLaneOffset);
+ Value *CondCopy = Builder.CreateAnd(Algo1, LaneIdGtOffset);
+
+ BasicBlock *CpyThenBB = BasicBlock::Create(Ctx, "then");
+ BasicBlock *CpyElseBB = BasicBlock::Create(Ctx, "else");
+ BasicBlock *CpyMergeBB = BasicBlock::Create(Ctx, "ifcont");
+ Builder.CreateCondBr(CondCopy, CpyThenBB, CpyElseBB);
+
+ emitBlock(CpyThenBB, Builder.GetInsertBlock()->getParent());
+ emitReductionListCopy(AllocaIP, CopyAction::ThreadCopy, RedListArrayTy,
+ ReductionInfos, RemoteListAddrCast, ReduceList);
+ Builder.CreateBr(CpyMergeBB);
+
+ emitBlock(CpyElseBB, Builder.GetInsertBlock()->getParent());
+ Builder.CreateBr(CpyMergeBB);
+
+ emitBlock(CpyMergeBB, Builder.GetInsertBlock()->getParent());
+
+ Builder.CreateRetVoid();
+
+ return SarFunc;
+}
+
+Function *OpenMPIRBuilder::emitListToGlobalCopyFunction(
+ ArrayRef<ReductionInfo> ReductionInfos, Type *ReductionsBufferTy,
+ AttributeList FuncAttrs) {
OpenMPIRBuilder::InsertPointTy OldIP = Builder.saveIP();
LLVMContext &Ctx = M.getContext();
- Type *VoidTy = Type::getVoidTy(Ctx);
- Type *Int32Ty = Builder.getInt32Ty();
- Type *PtrTy = PointerType::getUnqual(Ctx);
- auto FuncTy =
- FunctionType::get(VoidTy, {PtrTy, Int32Ty, PtrTy}, /* IsVarArg */ false);
+ FunctionType *FuncTy = FunctionType::get(
+ Builder.getVoidTy(),
+ {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()},
+ /* IsVarArg */ false);
Function *LtGCFunc =
Function::Create(FuncTy, GlobalVariable::InternalLinkage,
"_omp_reduction_list_to_global_copy_func", &M);
- LtGCFunc->setDoesNotRecurse();
+ LtGCFunc->setAttributes(FuncAttrs);
+ LtGCFunc->addParamAttr(0, Attribute::NoUndef);
+ LtGCFunc->addParamAttr(1, Attribute::NoUndef);
+ LtGCFunc->addParamAttr(2, Attribute::NoUndef);
- BasicBlock *EntryBlock = BasicBlock::Create(Ctx, "", LtGCFunc);
+ BasicBlock *EntryBlock = BasicBlock::Create(Ctx, "entry", LtGCFunc);
Builder.SetInsertPoint(EntryBlock);
- // Set arg names
- Argument *Arg0 = LtGCFunc->getArg(0);
- Argument *Arg1 = LtGCFunc->getArg(1);
- Argument *Arg2 = LtGCFunc->getArg(2);
- Arg0->setName("buffer_arg");
- Arg1->setName("idx_arg");
- Arg2->setName("reduce_list_arg");
-
- Value *BufferArgAlloca =
- Builder.CreateAlloca(PtrTy, nullptr, Arg0->getName() + ".addr");
- Value *IdxArgAlloca =
- Builder.CreateAlloca(Int32Ty, nullptr, Arg1->getName() + ".addr");
- Value *ReduceListArgAlloca =
- Builder.CreateAlloca(PtrTy, nullptr, Arg2->getName() + ".addr");
+ // Buffer: global reduction buffer.
+ Argument *BufferArg = LtGCFunc->getArg(0);
+ // Idx: index of the buffer.
+ Argument *IdxArg = LtGCFunc->getArg(1);
+ // ReduceList: thread local Reduce list.
+ Argument *ReduceListArg = LtGCFunc->getArg(2);
+
+ Value *BufferArgAlloca = Builder.CreateAlloca(Builder.getPtrTy(), nullptr,
+ BufferArg->getName() + ".addr");
+ Value *IdxArgAlloca = Builder.CreateAlloca(Builder.getInt32Ty(), nullptr,
+ IdxArg->getName() + ".addr");
+ Value *ReduceListArgAlloca = Builder.CreateAlloca(
+ Builder.getPtrTy(), nullptr, ReduceListArg->getName() + ".addr");
Value *BufferArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
- BufferArgAlloca, PtrTy, BufferArgAlloca->getName() + ".acast");
+ BufferArgAlloca, Builder.getPtrTy(),
+ BufferArgAlloca->getName() + ".ascast");
Value *IdxArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
- IdxArgAlloca, PtrTy, IdxArgAlloca->getName() + ".acast");
+ IdxArgAlloca, Builder.getPtrTy(), IdxArgAlloca->getName() + ".ascast");
Value *ReduceListArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
- ReduceListArgAlloca, PtrTy, ReduceListArgAlloca->getName() + ".acast");
- // FIXME(JAN): Assume a single globalized variable for now, this should be
- // passed in
- Type *SingleReductionTy = ReductionInfos.begin()->ElementType;
- Type *TypeArgs[] = {SingleReductionTy};
- StructType *ReductionsBufferTy =
- StructType::create(Ctx, TypeArgs, "_globalized_locals_ty");
-
- Builder.CreateStore(Arg0, BufferArgAddrCast);
- Builder.CreateStore(Arg1, IdxArgAddrCast);
- Builder.CreateStore(Arg2, ReduceListArgAddrCast);
-
- Value *BufferArg = Builder.CreateLoad(PtrTy, BufferArgAddrCast, "buffer");
- Value *Idxs[] = {
- Builder.CreateLoad(Builder.getInt32Ty(), IdxArgAddrCast, "idxs")};
- Value *ReduceListArg =
- Builder.CreateLoad(PtrTy, ReduceListArgAddrCast, "reduce_list");
- // FIXME(Jan): Assume TEK_SCALAR
- for (auto En : enumerate(ReductionInfos)) {
- const OpenMPIRBuilder::ReductionInfo &RI = En.value();
- // FIXME(Jan): Compute array type
- auto *RedListArrayTy = ArrayType::get(PtrTy, 1);
- Value *TargetElementPtrPtr = Builder.CreateConstInBoundsGEP2_64(
- RedListArrayTy, ReduceListArg, 0, En.index());
- Value *TargetElementPtr = Builder.CreateLoad(PtrTy, TargetElementPtrPtr);
+ ReduceListArgAlloca, Builder.getPtrTy(),
+ ReduceListArgAlloca->getName() + ".ascast");
+
+ Builder.CreateStore(BufferArg, BufferArgAddrCast);
+ Builder.CreateStore(IdxArg, IdxArgAddrCast);
+ Builder.CreateStore(ReduceListArg, ReduceListArgAddrCast);
+ Value *LocalReduceList =
+ Builder.CreateLoad(Builder.getPtrTy(), ReduceListArgAddrCast);
+ Value *BufferArgVal =
+ Builder.CreateLoad(Builder.getPtrTy(), BufferArgAddrCast);
+ Value *Idxs[] = {Builder.CreateLoad(Builder.getInt32Ty(), IdxArgAddrCast)};
+ Type *IndexTy = Builder.getIndexTy(
+ M.getDataLayout(), M.getDataLayout().getDefaultGlobalsAddressSpace());
+ for (auto En : enumerate(ReductionInfos)) {
+ const ReductionInfo &RI = En.value();
+ auto *RedListArrayTy =
+ ArrayType::get(Builder.getPtrTy(), ReductionInfos.size());
+ // Reduce element = LocalReduceList[i]
+ Value *ElemPtrPtr = Builder.CreateInBoundsGEP(
+ RedListArrayTy, LocalReduceList,
+ {ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});
+ // elemptr = ((CopyType*)(elemptrptr)) + I
+ Value *ElemPtr = Builder.CreateLoad(Builder.getPtrTy(), ElemPtrPtr);
+
+ // Global = Buffer.VD[Idx];
Value *BufferVD =
- Builder.CreateInBoundsGEP(ReductionsBufferTy, BufferArg, Idxs);
- Value *GlobValPtr = Builder.CreateConstInBoundsGEP2_32(
- ReductionsBufferTy, BufferVD, 0, En.index());
- Value *TargetElement = Builder.CreateLoad(RI.ElementType, TargetElementPtr);
- Builder.CreateStore(TargetElement, GlobValPtr);
+ Builder.CreateInBoundsGEP(ReductionsBufferTy, BufferArgVal, Idxs);
+ Value *GlobVal = Builder.CreateConstInBoundsGEP2_32(
+ ReductionsBufferTy, BufferVD, 0, En.index(), "sum");
+
+ switch (RI.EvaluationKind) {
+ case EvaluationKindTy::Scalar: {
+ Value *TargetElement = Builder.CreateLoad(RI.ElementType, ElemPtr);
+ Builder.CreateStore(TargetElement, GlobVal);
+ break;
+ }
+ case EvaluationKindTy::Complex: {
+ break;
+ }
+ case EvaluationKindTy::Aggregate:
+ Value *SizeVal =
+ Builder.getInt64(M.getDataLayout().getTypeStoreSize(RI.ElementType));
+ Builder.CreateMemCpy(
+ GlobVal, M.getDataLayout().getPrefTypeAlign(RI.ElementType), ElemPtr,
+ M.getDataLayout().getPrefTypeAlign(RI.ElementType), SizeVal, false);
+ break;
+ }
}
Builder.CreateRetVoid();
@@ -2780,287 +2886,287 @@ static Function *emitListToGlobalCopyFunction(
return LtGCFunc;
}
-/// This function emits a helper that copies all the reduction variables from
-/// the team into the provided global buffer for the reduction variables.
-///
-/// void list_to_global_copy_func(void *buffer, int Idx, void *reduce_data)
-/// For all data entries D in reduce_data:
-/// Copy local D to buffer.D[Idx]
-static Function *emitGlobalToListCopyFunction(
- Module &M, const OpenMPIRBuilder::LocationDescription &Loc,
- ArrayRef<OpenMPIRBuilder::ReductionInfo> ReductionInfos,
- OpenMPIRBuilder &OMPBuilder) {
- IRBuilder<> &Builder = OMPBuilder.Builder;
+Function *OpenMPIRBuilder::emitListToGlobalReduceFunction(
+ ArrayRef<ReductionInfo> ReductionInfos, Function *ReduceFn,
+ Type *ReductionsBufferTy, AttributeList FuncAttrs) {
OpenMPIRBuilder::InsertPointTy OldIP = Builder.saveIP();
LLVMContext &Ctx = M.getContext();
- Type *VoidTy = Type::getVoidTy(Ctx);
- Type *Int32Ty = Builder.getInt32Ty();
- Type *PtrTy = PointerType::getUnqual(Ctx);
- auto FuncTy =
- FunctionType::get(VoidTy, {PtrTy, Int32Ty, PtrTy}, /* IsVarArg */ false);
- Function *LtGCFunc =
+ FunctionType *FuncTy = FunctionType::get(
+ Builder.getVoidTy(),
+ {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()},
+ /* IsVarArg */ false);
+ Function *LtGRFunc =
Function::Create(FuncTy, GlobalVariable::InternalLinkage,
- "_omp_reduction_global_to_list_copy_func", &M);
- LtGCFunc->setDoesNotRecurse();
+ "_omp_reduction_list_to_global_reduce_func", &M);
+ LtGRFunc->setAttributes(FuncAttrs);
+ LtGRFunc->addParamAttr(0, Attribute::NoUndef);
+ LtGRFunc->addParamAttr(1, Attribute::NoUndef);
+ LtGRFunc->addParamAttr(2, Attribute::NoUndef);
- BasicBlock *EntryBlock = BasicBlock::Create(Ctx, "", LtGCFunc);
+ BasicBlock *EntryBlock = BasicBlock::Create(Ctx, "entry", LtGRFunc);
Builder.SetInsertPoint(EntryBlock);
- // Set arg names
- Argument *Arg0 = LtGCFunc->getArg(0);
- Argument *Arg1 = LtGCFunc->getArg(1);
- Argument *Arg2 = LtGCFunc->getArg(2);
- Arg0->setName("buffer_arg");
- Arg1->setName("idx_arg");
- Arg2->setName("reduce_list_arg");
-
- Value *BufferArgAlloca =
- Builder.CreateAlloca(PtrTy, nullptr, Arg0->getName() + ".addr");
- Value *IdxArgAlloca =
- Builder.CreateAlloca(Int32Ty, nullptr, Arg1->getName() + ".addr");
- Value *ReduceListArgAlloca =
- Builder.CreateAlloca(PtrTy, nullptr, Arg2->getName() + ".addr");
+ // Buffer: global reduction buffer.
+ Argument *BufferArg = LtGRFunc->getArg(0);
+ // Idx: index of the buffer.
+ Argument *IdxArg = LtGRFunc->getArg(1);
+ // ReduceList: thread local Reduce list.
+ Argument *ReduceListArg = LtGRFunc->getArg(2);
+
+ Value *BufferArgAlloca = Builder.CreateAlloca(Builder.getPtrTy(), nullptr,
+ BufferArg->getName() + ".addr");
+ Value *IdxArgAlloca = Builder.CreateAlloca(Builder.getInt32Ty(), nullptr,
+ IdxArg->getName() + ".addr");
+ Value *ReduceListArgAlloca = Builder.CreateAlloca(
+ Builder.getPtrTy(), nullptr, ReduceListArg->getName() + ".addr");
+ auto *RedListArrayTy =
+ ArrayType::get(Builder.getPtrTy(), ReductionInfos.size());
+
+ // 1. Build a list of reduction variables.
+ // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
+ Value *LocalReduceList =
+ Builder.CreateAlloca(RedListArrayTy, nullptr, ".omp.reduction.red_list");
+
Value *BufferArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
- BufferArgAlloca, PtrTy, BufferArgAlloca->getName() + ".acast");
+ BufferArgAlloca, Builder.getPtrTy(),
+ BufferArgAlloca->getName() + ".ascast");
Value *IdxArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
- IdxArgAlloca, PtrTy, IdxArgAlloca->getName() + ".acast");
+ IdxArgAlloca, Builder.getPtrTy(), IdxArgAlloca->getName() + ".ascast");
Value *ReduceListArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
- ReduceListArgAlloca, PtrTy, ReduceListArgAlloca->getName() + ".acast");
- // FIXME(JAN): Assume a single globalized variable for now, this should be
- // passed in
- Type *SingleReductionTy = ReductionInfos.begin()->ElementType;
- Type *TypeArgs[] = {SingleReductionTy};
- StructType *ReductionsBufferTy =
- StructType::create(Ctx, TypeArgs, "_globalized_locals_ty");
-
- Builder.CreateStore(Arg0, BufferArgAddrCast);
- Builder.CreateStore(Arg1, IdxArgAddrCast);
- Builder.CreateStore(Arg2, ReduceListArgAddrCast);
-
- Value *BufferArg = Builder.CreateLoad(PtrTy, BufferArgAddrCast, "buffer");
- Value *Idxs[] = {
- Builder.CreateLoad(Builder.getInt32Ty(), IdxArgAddrCast, "idxs")};
- Value *ReduceListArg =
- Builder.CreateLoad(PtrTy, ReduceListArgAddrCast, "reduce_list");
- // FIXME(Jan): Assume TEK_SCALAR
- for (auto En : enumerate(ReductionInfos)) {
- const OpenMPIRBuilder::ReductionInfo &RI = En.value();
- // FIXME(Jan): Compute array type
- auto *RedListArrayTy = ArrayType::get(PtrTy, 1);
- Value *TargetElementPtrPtr = Builder.CreateConstInBoundsGEP2_64(
- RedListArrayTy, ReduceListArg, 0, En.index());
- Value *TargetElementPtr = Builder.CreateLoad(PtrTy, TargetElementPtrPtr);
+ ReduceListArgAlloca, Builder.getPtrTy(),
+ ReduceListArgAlloca->getName() + ".ascast");
+ Value *LocalReduceListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ LocalReduceList, Builder.getPtrTy(),
+ LocalReduceList->getName() + ".ascast");
+
+ Builder.CreateStore(BufferArg, BufferArgAddrCast);
+ Builder.CreateStore(IdxArg, IdxArgAddrCast);
+ Builder.CreateStore(ReduceListArg, ReduceListArgAddrCast);
+ Value *BufferVal = Builder.CreateLoad(Builder.getPtrTy(), BufferArgAddrCast);
+ Value *Idxs[] = {Builder.CreateLoad(Builder.getInt32Ty(), IdxArgAddrCast)};
+ Type *IndexTy = Builder.getIndexTy(
+ M.getDataLayout(), M.getDataLayout().getDefaultGlobalsAddressSpace());
+ for (auto En : enumerate(ReductionInfos)) {
+ Value *TargetElementPtrPtr = Builder.CreateInBoundsGEP(
+ RedListArrayTy, LocalReduceListAddrCast,
+ {ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});
Value *BufferVD =
- Builder.CreateInBoundsGEP(ReductionsBufferTy, BufferArg, Idxs);
+ Builder.CreateInBoundsGEP(ReductionsBufferTy, BufferVal, Idxs);
+ // Global = Buffer.VD[Idx];
Value *GlobValPtr = Builder.CreateConstInBoundsGEP2_32(
- ReductionsBufferTy, BufferVD, 0, En.index());
- Value *TargetElement = Builder.CreateLoad(RI.ElementType, GlobValPtr);
- Builder.CreateStore(TargetElement, TargetElementPtr);
+ ReductionsBufferTy, BufferVD, 0, En.index(), "sum");
+ Builder.CreateStore(GlobValPtr, TargetElementPtrPtr);
}
+ // Call reduce_function(GlobalReduceList, ReduceList)
+ Value *ReduceList =
+ Builder.CreateLoad(Builder.getPtrTy(), ReduceListArgAddrCast);
+ Builder.CreateCall(ReduceFn, {LocalReduceListAddrCast, ReduceList})
+ ->addFnAttr(Attribute::NoUnwind);
Builder.CreateRetVoid();
Builder.restoreIP(OldIP);
- return LtGCFunc;
+ return LtGRFunc;
}
-/// This function emits a helper that reduces all the reduction variables from
-/// the team into the provided global buffer for the reduction variables.
-///
-/// void list_to_global_reduce_func(void *buffer, int Idx, void *reduce_data)
-/// void *GlobPtrs[];
-/// GlobPtrs[0] = (void*)&buffer.D0[Idx];
-/// ...
-/// GlobPtrs[N] = (void*)&buffer.DN[Idx];
-/// reduce_function(GlobPtrs, reduce_data);
-/// Create a function with a unique name and a "void (i8*, i8*)" signature in
-/// the given module and return it.
-static Function *emitListToGlobalReduceFunction(
- Module &M, const OpenMPIRBuilder::LocationDescription &Loc,
- ArrayRef<OpenMPIRBuilder::ReductionInfo> ReductionInfos, Function *ReduceFn,
- OpenMPIRBuilder &OMPBuilder) {
- IRBuilder<> &Builder = OMPBuilder.Builder;
+Function *OpenMPIRBuilder::emitGlobalToListCopyFunction(
+ ArrayRef<ReductionInfo> ReductionInfos, Type *ReductionsBufferTy,
+ AttributeList FuncAttrs) {
OpenMPIRBuilder::InsertPointTy OldIP = Builder.saveIP();
LLVMContext &Ctx = M.getContext();
- Type *VoidTy = Type::getVoidTy(Ctx);
- Type *Int32Ty = Builder.getInt32Ty();
- Type *PtrTy = PointerType::getUnqual(Ctx);
- auto FuncTy =
- FunctionType::get(VoidTy, {PtrTy, Int32Ty, PtrTy}, /* IsVarArg */ false);
- Function *LtGRFunc =
+ FunctionType *FuncTy = FunctionType::get(
+ Builder.getVoidTy(),
+ {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()},
+ /* IsVarArg */ false);
+ Function *LtGCFunc =
Function::Create(FuncTy, GlobalVariable::InternalLinkage,
- "_omp_reduction_list_to_global_reduce_func", &M);
- LtGRFunc->setDoesNotRecurse();
+ "_omp_reduction_global_to_list_copy_func", &M);
+ LtGCFunc->setAttributes(FuncAttrs);
+ LtGCFunc->addParamAttr(0, Attribute::NoUndef);
+ LtGCFunc->addParamAttr(1, Attribute::NoUndef);
+ LtGCFunc->addParamAttr(2, Attribute::NoUndef);
- BasicBlock *EntryBlock = BasicBlock::Create(Ctx, "", LtGRFunc);
+ BasicBlock *EntryBlock = BasicBlock::Create(Ctx, "entry", LtGCFunc);
Builder.SetInsertPoint(EntryBlock);
- // Set arg names
- Argument *Arg0 = LtGRFunc->getArg(0);
- Argument *Arg1 = LtGRFunc->getArg(1);
- Argument *Arg2 = LtGRFunc->getArg(2);
- Arg0->setName("buffer_arg");
- Arg1->setName("idx_arg");
- Arg2->setName("reduce_list_arg");
-
- Value *BufferArgAlloca =
- Builder.CreateAlloca(PtrTy, nullptr, Arg0->getName() + ".addr");
- Value *IdxArgAlloca =
- Builder.CreateAlloca(Int32Ty, nullptr, Arg1->getName() + ".addr");
- Value *ReduceListArgAlloca =
- Builder.CreateAlloca(PtrTy, nullptr, Arg2->getName() + ".addr");
- // FIXME(Jan): Compute array type
- auto *RedListArrayTy = ArrayType::get(PtrTy, 1);
- Value *LocalReduceList =
- Builder.CreateAlloca(RedListArrayTy, nullptr, ".omp.reduction.red_list");
-
+ // Buffer: global reduction buffer.
+ Argument *BufferArg = LtGCFunc->getArg(0);
+ // Idx: index of the buffer.
+ Argument *IdxArg = LtGCFunc->getArg(1);
+ // ReduceList: thread local Reduce list.
+ Argument *ReduceListArg = LtGCFunc->getArg(2);
+
+ Value *BufferArgAlloca = Builder.CreateAlloca(Builder.getPtrTy(), nullptr,
+ BufferArg->getName() + ".addr");
+ Value *IdxArgAlloca = Builder.CreateAlloca(Builder.getInt32Ty(), nullptr,
+ IdxArg->getName() + ".addr");
+ Value *ReduceListArgAlloca = Builder.CreateAlloca(
+ Builder.getPtrTy(), nullptr, ReduceListArg->getName() + ".addr");
Value *BufferArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
- BufferArgAlloca, PtrTy, BufferArgAlloca->getName() + ".acast");
+ BufferArgAlloca, Builder.getPtrTy(),
+ BufferArgAlloca->getName() + ".ascast");
Value *IdxArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
- IdxArgAlloca, PtrTy, IdxArgAlloca->getName() + ".acast");
+ IdxArgAlloca, Builder.getPtrTy(), IdxArgAlloca->getName() + ".ascast");
Value *ReduceListArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
- ReduceListArgAlloca, PtrTy, ReduceListArgAlloca->getName() + ".acast");
- Value *LocalReduceListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
- LocalReduceList, PtrTy, LocalReduceList->getName() + ".acast");
- // FIXME(JAN): Assume a single globalized variable for now, this should be
- // passed in
- Type *SingleReductionTy = ReductionInfos.begin()->ElementType;
- Type *TypeArgs[] = {SingleReductionTy};
- StructType *ReductionsBufferTy =
- StructType::create(Ctx, TypeArgs, "_globalized_locals_ty");
-
- Builder.CreateStore(Arg0, BufferArgAddrCast);
- Builder.CreateStore(Arg1, IdxArgAddrCast);
- Builder.CreateStore(Arg2, ReduceListArgAddrCast);
-
- Value *BufferArg = Builder.CreateLoad(PtrTy, BufferArgAddrCast, "buffer");
- Value *Idxs[] = {Builder.CreateLoad(Int32Ty, IdxArgAddrCast, "idxs")};
- // FIXME(Jan): Assume TEK_SCALAR
+ ReduceListArgAlloca, Builder.getPtrTy(),
+ ReduceListArgAlloca->getName() + ".ascast");
+ Builder.CreateStore(BufferArg, BufferArgAddrCast);
+ Builder.CreateStore(IdxArg, IdxArgAddrCast);
+ Builder.CreateStore(ReduceListArg, ReduceListArgAddrCast);
+
+ Value *LocalReduceList =
+ Builder.CreateLoad(Builder.getPtrTy(), ReduceListArgAddrCast);
+ Value *BufferVal = Builder.CreateLoad(Builder.getPtrTy(), BufferArgAddrCast);
+ Value *Idxs[] = {Builder.CreateLoad(Builder.getInt32Ty(), IdxArgAddrCast)};
+ Type *IndexTy = Builder.getIndexTy(
+ M.getDataLayout(), M.getDataLayout().getDefaultGlobalsAddressSpace());
for (auto En : enumerate(ReductionInfos)) {
const OpenMPIRBuilder::ReductionInfo &RI = En.value();
- Value *TargetElementPtrPtr = Builder.CreateConstInBoundsGEP2_64(
- RedListArrayTy, LocalReduceListAddrCast, 0, En.index());
+ auto *RedListArrayTy =
+ ArrayType::get(Builder.getPtrTy(), ReductionInfos.size());
+ // Reduce element = LocalReduceList[i]
+ Value *ElemPtrPtr = Builder.CreateInBoundsGEP(
+ RedListArrayTy, LocalReduceList,
+ {ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});
+ // elemptr = ((CopyType*)(elemptrptr)) + I
+ Value *ElemPtr = Builder.CreateLoad(Builder.getPtrTy(), ElemPtrPtr);
+ // Global = Buffer.VD[Idx];
Value *BufferVD =
- Builder.CreateInBoundsGEP(ReductionsBufferTy, BufferArg, Idxs);
+ Builder.CreateInBoundsGEP(ReductionsBufferTy, BufferVal, Idxs);
Value *GlobValPtr = Builder.CreateConstInBoundsGEP2_32(
- ReductionsBufferTy, BufferVD, 0, En.index());
- Builder.CreateStore(GlobValPtr, TargetElementPtrPtr);
+ ReductionsBufferTy, BufferVD, 0, En.index(), "sum");
+
+ switch (RI.EvaluationKind) {
+ case EvaluationKindTy::Scalar: {
+ Value *TargetElement = Builder.CreateLoad(RI.ElementType, GlobValPtr);
+ Builder.CreateStore(TargetElement, ElemPtr);
+ break;
+ }
+ case EvaluationKindTy::Complex: {
+ // FIXME(Jan): Complex type
+ break;
+ }
+ case EvaluationKindTy::Aggregate:
+ Value *SizeVal =
+ Builder.getInt64(M.getDataLayout().getTypeStoreSize(RI.ElementType));
+ Builder.CreateMemCpy(
+ ElemPtr, M.getDataLayout().getPrefTypeAlign(RI.ElementType),
+ GlobValPtr, M.getDataLayout().getPrefTypeAlign(RI.ElementType),
+ SizeVal, false);
+ break;
+ }
}
- Value *ReduceList = Builder.CreateLoad(PtrTy, ReduceListArgAddrCast);
- Builder.CreateCall(ReduceFn, {LocalReduceListAddrCast, ReduceList});
Builder.CreateRetVoid();
Builder.restoreIP(OldIP);
- return LtGRFunc;
+ return LtGCFunc;
}
-/// This function emits a helper that reduces all the reduction variables from
-/// the team into the provided global buffer for the reduction variables.
-///
-/// void list_to_global_reduce_func(void *buffer, int Idx, void *reduce_data)
-/// void *GlobPtrs[];
-/// GlobPtrs[0] = (void*)&buffer.D0[Idx];
-/// ...
-/// GlobPtrs[N] = (void*)&buffer.DN[Idx];
-/// reduce_function(GlobPtrs, reduce_data);
-/// Create a function with a unique name and a "void (i8*, i8*)" signature in
-/// the given module and return it.
-static Function *emitGlobalToListReduceFunction(
- Module &M, const OpenMPIRBuilder::LocationDescription &Loc,
- ArrayRef<OpenMPIRBuilder::ReductionInfo> ReductionInfos, Function *ReduceFn,
- OpenMPIRBuilder &OMPBuilder) {
- IRBuilder<> &Builder = OMPBuilder.Builder;
+Function *OpenMPIRBuilder::emitGlobalToListReduceFunction(
+ ArrayRef<ReductionInfo> ReductionInfos, Function *ReduceFn,
+ Type *ReductionsBufferTy, AttributeList FuncAttrs) {
OpenMPIRBuilder::InsertPointTy OldIP = Builder.saveIP();
LLVMContext &Ctx = M.getContext();
- Type *VoidTy = Type::getVoidTy(Ctx);
- Type *Int32Ty = Builder.getInt32Ty();
- Type *PtrTy = PointerType::getUnqual(Ctx);
- auto FuncTy =
- FunctionType::get(VoidTy, {PtrTy, Int32Ty, PtrTy}, /* IsVarArg */ false);
+ auto *FuncTy = FunctionType::get(
+ Builder.getVoidTy(),
+ {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()},
+ /* IsVarArg */ false);
Function *LtGRFunc =
Function::Create(FuncTy, GlobalVariable::InternalLinkage,
"_omp_reduction_global_to_list_reduce_func", &M);
- LtGRFunc->setDoesNotRecurse();
+ LtGRFunc->setAttributes(FuncAttrs);
+ LtGRFunc->addParamAttr(0, Attribute::NoUndef);
+ LtGRFunc->addParamAttr(1, Attribute::NoUndef);
+ LtGRFunc->addParamAttr(2, Attribute::NoUndef);
- BasicBlock *EntryBlock = BasicBlock::Create(Ctx, "", LtGRFunc);
+ BasicBlock *EntryBlock = BasicBlock::Create(Ctx, "entry", LtGRFunc);
Builder.SetInsertPoint(EntryBlock);
- // Set arg names
- Argument *Arg0 = LtGRFunc->getArg(0);
- Argument *Arg1 = LtGRFunc->getArg(1);
- Argument *Arg2 = LtGRFunc->getArg(2);
- Arg0->setName("buffer_arg");
- Arg1->setName("idx_arg");
- Arg2->setName("reduce_list_arg");
-
- Value *BufferArgAlloca =
- Builder.CreateAlloca(PtrTy, nullptr, Arg0->getName() + ".addr");
- Value *IdxArgAlloca =
- Builder.CreateAlloca(Int32Ty, nullptr, Arg1->getName() + ".addr");
- Value *ReduceListArgAlloca =
- Builder.CreateAlloca(PtrTy, nullptr, Arg2->getName() + ".addr");
- // FIXME(Jan): Compute array type
- auto *RedListArrayTy = ArrayType::get(PtrTy, 1);
+ // Buffer: global reduction buffer.
+ Argument *BufferArg = LtGRFunc->getArg(0);
+ // Idx: index of the buffer.
+ Argument *IdxArg = LtGRFunc->getArg(1);
+ // ReduceList: thread local Reduce list.
+ Argument *ReduceListArg = LtGRFunc->getArg(2);
+
+ Value *BufferArgAlloca = Builder.CreateAlloca(Builder.getPtrTy(), nullptr,
+ BufferArg->getName() + ".addr");
+ Value *IdxArgAlloca = Builder.CreateAlloca(Builder.getInt32Ty(), nullptr,
+ IdxArg->getName() + ".addr");
+ Value *ReduceListArgAlloca = Builder.CreateAlloca(
+ Builder.getPtrTy(), nullptr, ReduceListArg->getName() + ".addr");
+ ArrayType *RedListArrayTy =
+ ArrayType::get(Builder.getPtrTy(), ReductionInfos.size());
+
+ // 1. Build a list of reduction variables.
+ // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
Value *LocalReduceList =
Builder.CreateAlloca(RedListArrayTy, nullptr, ".omp.reduction.red_list");
Value *BufferArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
- BufferArgAlloca, PtrTy, BufferArgAlloca->getName() + ".acast");
+ BufferArgAlloca, Builder.getPtrTy(),
+ BufferArgAlloca->getName() + ".ascast");
Value *IdxArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
- IdxArgAlloca, PtrTy, IdxArgAlloca->getName() + ".acast");
+ IdxArgAlloca, Builder.getPtrTy(), IdxArgAlloca->getName() + ".ascast");
Value *ReduceListArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
- ReduceListArgAlloca, PtrTy, ReduceListArgAlloca->getName() + ".acast");
- Value *LocalReduceListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
- LocalReduceList, PtrTy, LocalReduceList->getName() + ".acast");
- // FIXME(JAN): Assume a single globalized variable for now, this should be
- // passed in
- Type *SingleReductionTy = ReductionInfos.begin()->ElementType;
- Type *TypeArgs[] = {SingleReductionTy};
- StructType *ReductionsBufferTy =
- StructType::create(Ctx, TypeArgs, "_globalized_locals_ty");
-
- Builder.CreateStore(Arg0, BufferArgAddrCast);
- Builder.CreateStore(Arg1, IdxArgAddrCast);
- Builder.CreateStore(Arg2, ReduceListArgAddrCast);
-
- Value *BufferArg = Builder.CreateLoad(PtrTy, BufferArgAddrCast, "buffer");
- Value *Idxs[] = {Builder.CreateLoad(Int32Ty, IdxArgAddrCast, "idxs")};
- // FIXME(Jan): Assume TEK_SCALAR
+ ReduceListArgAlloca, Builder.getPtrTy(),
+ ReduceListArgAlloca->getName() + ".ascast");
+ Value *ReductionList = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ LocalReduceList, Builder.getPtrTy(),
+ LocalReduceList->getName() + ".ascast");
+
+ Builder.CreateStore(BufferArg, BufferArgAddrCast);
+ Builder.CreateStore(IdxArg, IdxArgAddrCast);
+ Builder.CreateStore(ReduceListArg, ReduceListArgAddrCast);
+
+ Value *BufferVal = Builder.CreateLoad(Builder.getPtrTy(), BufferArgAddrCast);
+ Value *Idxs[] = {Builder.CreateLoad(Builder.getInt32Ty(), IdxArgAddrCast)};
+ Type *IndexTy = Builder.getIndexTy(
+ M.getDataLayout(), M.getDataLayout().getDefaultGlobalsAddressSpace());
for (auto En : enumerate(ReductionInfos)) {
- const OpenMPIRBuilder::ReductionInfo &RI = En.value();
- Value *TargetElementPtrPtr = Builder.CreateConstInBoundsGEP2_64(
- RedListArrayTy, LocalReduceListAddrCast, 0, En.index());
+ Value *TargetElementPtrPtr = Builder.CreateInBoundsGEP(
+ RedListArrayTy, ReductionList,
+ {ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});
+ // Global = Buffer.VD[Idx];
Value *BufferVD =
- Builder.CreateInBoundsGEP(ReductionsBufferTy, BufferArg, Idxs);
+ Builder.CreateInBoundsGEP(ReductionsBufferTy, BufferVal, Idxs);
Value *GlobValPtr = Builder.CreateConstInBoundsGEP2_32(
- ReductionsBufferTy, BufferVD, 0, En.index());
+ ReductionsBufferTy, BufferVD, 0, En.index(), "sum");
Builder.CreateStore(GlobValPtr, TargetElementPtrPtr);
}
- Value *ReduceList = Builder.CreateLoad(PtrTy, ReduceListArgAddrCast);
- Builder.CreateCall(ReduceFn, {ReduceList, LocalReduceListAddrCast});
+ // Call reduce_function(ReduceList, GlobalReduceList)
+ Value *ReduceList =
+ Builder.CreateLoad(Builder.getPtrTy(), ReduceListArgAddrCast);
+ Builder.CreateCall(ReduceFn, {ReduceList, ReductionList})
+ ->addFnAttr(Attribute::NoUnwind);
Builder.CreateRetVoid();
Builder.restoreIP(OldIP);
return LtGRFunc;
}
-static Function *getFreshReductionFunc(Module &M) {
- Type *VoidTy = Type::getVoidTy(M.getContext());
- Type *Int8PtrTy = PointerType::getUnqual(M.getContext());
- auto *FuncTy =
- FunctionType::get(VoidTy, {Int8PtrTy, Int8PtrTy}, /* IsVarArg */ false);
- return Function::Create(FuncTy, GlobalVariable::InternalLinkage,
- ".omp.reduction.func", &M);
-}
+std::string OpenMPIRBuilder::getReductionFuncName(StringRef Name) const {
+ std::string Suffix =
+ createPlatformSpecificName({"omp", "reduction", "reduction_func"});
+ return (Name + Suffix).str();
+}
+
+Function *OpenMPIRBuilder::createReductionFunction(
+ StringRef ReducerName, ArrayRef<ReductionInfo> ReductionInfos, bool IsGPU,
+ ReductionGenCBTy ReductionGenCBTy, AttributeList FuncAttrs) {
+ auto *FuncTy = FunctionType::get(Builder.getVoidTy(),
+ {Builder.getPtrTy(), Builder.getPtrTy()},
+ /* IsVarArg */ false);
+ std::string Name = getReductionFuncName(ReducerName);
+ auto *ReductionFunc =
+ Function::Create(FuncTy, GlobalVariable::InternalLinkage, Name, &M);
+ ReductionFunc->setAttributes(FuncAttrs);
+ ReductionFunc->addParamAttr(0, Attribute::NoUndef);
+ ReductionFunc->addParamAttr(1, Attribute::NoUndef);
+ BasicBlock *EntryBB =
+ BasicBlock::Create(M.getContext(), "entry", ReductionFunc);
+ Builder.SetInsertPoint(EntryBB);
-static void populateReductionFunction(
- Function *ReductionFunc,
- ArrayRef<OpenMPIRBuilder::ReductionInfo> ReductionInfos,
- IRBuilder<> &Builder, bool IsGPU) {
- Module *Module = ReductionFunc->getParent();
- BasicBlock *ReductionFuncBlock =
- BasicBlock::Create(Module->getContext(), "", ReductionFunc);
- Builder.SetInsertPoint(ReductionFuncBlock);
Value *LHSArrayPtr = nullptr;
Value *RHSArrayPtr = nullptr;
if (IsGPU) {
@@ -3076,10 +3182,10 @@ static void populateReductionFunction(
Builder.CreateAlloca(Arg0Type, nullptr, Arg0->getName() + ".addr");
Value *RHSAlloca =
Builder.CreateAlloca(Arg1Type, nullptr, Arg1->getName() + ".addr");
- Value *LHSAddrCast =
- Builder.CreatePointerBitCastOrAddrSpaceCast(LHSAlloca, Arg0Type);
- Value *RHSAddrCast =
- Builder.CreatePointerBitCastOrAddrSpaceCast(RHSAlloca, Arg1Type);
+ Value *LHSAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ LHSAlloca, Arg0Type, LHSAlloca->getName() + ".ascast");
+ Value *RHSAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ RHSAlloca, Arg1Type, RHSAlloca->getName() + ".ascast");
Builder.CreateStore(Arg0, LHSAddrCast);
Builder.CreateStore(Arg1, RHSAddrCast);
LHSArrayPtr = Builder.CreateLoad(Arg0Type, LHSAddrCast);
@@ -3089,30 +3195,65 @@ static void populateReductionFunction(
RHSArrayPtr = ReductionFunc->getArg(1);
}
- unsigned NumReductions = ReductionInfos.size();
- Type *RedArrayTy = ArrayType::get(Builder.getPtrTy(), NumReductions);
-
+ Type *RedArrayTy = ArrayType::get(Builder.getPtrTy(), ReductionInfos.size());
+ Type *IndexTy = Builder.getIndexTy(
+ M.getDataLayout(), M.getDataLayout().getDefaultGlobalsAddressSpace());
+ SmallVector<Value *> LHSPtrs, RHSPtrs;
for (auto En : enumerate(ReductionInfos)) {
- const OpenMPIRBuilder::ReductionInfo &RI = En.value();
- Value *LHSI8PtrPtr = Builder.CreateConstInBoundsGEP2_64(
- RedArrayTy, LHSArrayPtr, 0, En.index());
- Value *LHSI8Ptr = Builder.CreateLoad(Builder.getPtrTy(), LHSI8PtrPtr);
- Value *LHSPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
- LHSI8Ptr, RI.Variable->getType());
- Value *LHS = Builder.CreateLoad(RI.ElementType, LHSPtr);
- Value *RHSI8PtrPtr = Builder.CreateConstInBoundsGEP2_64(
- RedArrayTy, RHSArrayPtr, 0, En.index());
+ const ReductionInfo &RI = En.value();
+ Value *RHSI8PtrPtr = Builder.CreateInBoundsGEP(
+ RedArrayTy, RHSArrayPtr,
+ {ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});
Value *RHSI8Ptr = Builder.CreateLoad(Builder.getPtrTy(), RHSI8PtrPtr);
- Value *RHSPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
- RHSI8Ptr, RI.PrivateVariable->getType());
- Value *RHS = Builder.CreateLoad(RI.ElementType, RHSPtr);
- Value *Reduced;
- Builder.restoreIP(RI.ReductionGen(Builder.saveIP(), LHS, RHS, Reduced));
- if (!Builder.GetInsertBlock())
- return;
- Builder.CreateStore(Reduced, LHSPtr);
+ Value *RHS = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ RHSI8Ptr, RI.PrivateVariable->getType(),
+ RHSI8Ptr->getName() + ".ascast");
+
+ Value *LHSI8PtrPtr = Builder.CreateInBoundsGEP(
+ RedArrayTy, LHSArrayPtr,
+ {ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});
+ Value *LHSI8Ptr = Builder.CreateLoad(Builder.getPtrTy(), LHSI8PtrPtr);
+ Value *LHS = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ LHSI8Ptr, RI.Variable->getType(), LHSI8Ptr->getName() + ".ascast");
+
+ if (ReductionGenCBTy == ReductionGenCBTy::Clang) {
+ LHSPtrs.emplace_back(LHS);
+ RHSPtrs.emplace_back(RHS);
+ } else {
+ LHS = Builder.CreateLoad(RI.ElementType, LHS);
+ RHS = Builder.CreateLoad(RI.ElementType, RHS);
+ Value *Reduced;
+ RI.ReductionGen(Builder.saveIP(), LHS, RHS, Reduced);
+ if (!Builder.GetInsertBlock())
+ return ReductionFunc;
+ Builder.CreateStore(Reduced, LHS);
+ }
}
+
+ if (ReductionGenCBTy == ReductionGenCBTy::Clang)
+ for (auto En : enumerate(ReductionInfos)) {
+ unsigned Index = En.index();
+ const ReductionInfo &RI = En.value();
+ Value *LHSFixupPtr, *RHSFixupPtr;
+ Builder.restoreIP(RI.ReductionGenClang(
+ Builder.saveIP(), Index, &LHSFixupPtr, &RHSFixupPtr, ReductionFunc));
+
+ // Fix the CallBack code genereated to use the correct Values for the LHS
+ // and RHS
+ LHSFixupPtr->replaceUsesWithIf(
+ LHSPtrs[Index], [ReductionFunc](const Use &U) {
+ return cast<Instruction>(U.getUser())->getParent()->getParent() ==
+ ReductionFunc;
+ });
+ RHSFixupPtr->replaceUsesWithIf(
+ RHSPtrs[Index], [ReductionFunc](const Use &U) {
+ return cast<Instruction>(U.getUser())->getParent()->getParent() ==
+ ReductionFunc;
+ });
+ }
+
Builder.CreateRetVoid();
+ return ReductionFunc;
}
static void
@@ -3122,7 +3263,8 @@ checkReductionInfos(ArrayRef<OpenMPIRBuilder::ReductionInfo> ReductionInfos,
(void)RI;
assert(RI.Variable && "expected non-null variable");
assert(RI.PrivateVariable && "expected non-null private variable");
- assert(RI.ReductionGen && "expected non-null reduction generator callback");
+ assert((RI.ReductionGen || RI.ReductionGenClang) &&
+ "expected non-null reduction generator callback");
// JAN: Skip this assertion for GPU, address spaces are present
if (!IsGPU) {
assert(
@@ -3137,73 +3279,103 @@ checkReductionInfos(ArrayRef<OpenMPIRBuilder::ReductionInfo> ReductionInfos,
OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductionsGPU(
const LocationDescription &Loc, InsertPointTy AllocaIP,
- ArrayRef<ReductionInfo> ReductionInfos, bool IsNoWait,
- bool IsTeamsReduction, bool HasDistribute) {
- checkReductionInfos(ReductionInfos, /*IsGPU*/ true);
- LLVMContext &Ctx = M.getContext();
+ InsertPointTy CodeGenIP, ArrayRef<ReductionInfo> ReductionInfos,
+ bool IsNoWait, bool IsTeamsReduction, bool HasDistribute,
+ ReductionGenCBTy ReductionGenCBTy, std::optional<omp::GV> GridValue,
+ unsigned ReductionBufNum, Value *SrcLocInfo) {
if (!updateToLocation(Loc))
return InsertPointTy();
+ Builder.restoreIP(CodeGenIP);
+ checkReductionInfos(ReductionInfos, /*IsGPU*/ true);
+ LLVMContext &Ctx = M.getContext();
+
+ // Source location for the ident struct
+ if (!SrcLocInfo) {
+ uint32_t SrcLocStrSize;
+ Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
+ SrcLocInfo = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
+ }
if (ReductionInfos.size() == 0)
return Builder.saveIP();
- assert(ReductionInfos.size() == 1 && "More than one reduction variable");
+ Function *CurFunc = Builder.GetInsertBlock()->getParent();
+ AttributeList FuncAttrs;
+ AttrBuilder AttrBldr(Ctx);
+ for (auto Attr : CurFunc->getAttributes().getFnAttrs())
+ AttrBldr.addAttribute(Attr);
+ AttrBldr.removeAttribute(Attribute::OptimizeNone);
+ FuncAttrs = FuncAttrs.addFnAttributes(Ctx, AttrBldr);
- // Copied code from createReductions
- BasicBlock *InsertBlock = Loc.IP.getBlock();
- BasicBlock *ContinuationBlock =
- InsertBlock->splitBasicBlock(Loc.IP.getPoint(), "reduce.finalize");
- InsertBlock->getTerminator()->eraseFromParent();
- Builder.SetInsertPoint(InsertBlock, InsertBlock->end());
+ // Set the grid value in the config needed for lowering later on
+ if (GridValue.has_value())
+ Config.setGridValue(GridValue.value());
+ else
+ Config.setGridValue(getGridValue(T, Config.TargetFeatures));
Function *ReductionFunc = nullptr;
if (GLOBAL_ReductionFunc) {
ReductionFunc = GLOBAL_ReductionFunc;
} else {
- ReductionFunc = getFreshReductionFunc(M);
- GLOBAL_ReductionFunc = ReductionFunc;
- InsertPointTy CurIP = Builder.saveIP();
- populateReductionFunction(ReductionFunc, ReductionInfos, Builder, true);
- Builder.restoreIP(CurIP);
+ CodeGenIP = Builder.saveIP();
+ ReductionFunc = createReductionFunction(
+ Builder.GetInsertBlock()->getParent()->getName(), ReductionInfos, true,
+ ReductionGenCBTy, FuncAttrs);
+ Builder.restoreIP(CodeGenIP);
}
uint32_t SrcLocStrSize;
Constant *SrcLocStr = getOrCreateDefaultSrcLocStr(SrcLocStrSize);
Value *RTLoc =
- getOrCreateIdent(SrcLocStr, SrcLocStrSize, llvm::omp::IdentFlag(0), 0);
+ getOrCreateIdent(SrcLocStr, SrcLocStrSize, omp::IdentFlag(0), 0);
- // 1. Build a list of reduction variables
+ // Build res = __kmpc_reduce{_nowait}(<gtid>, <n>, sizeof(RedList),
+ // RedList, shuffle_reduce_func, interwarp_copy_func);
+ // or
+ // Build res = __kmpc_reduce_teams_nowait_simple(<loc>, <gtid>, <lck>);
+ Value *Res;
+
+ // 1. Build a list of reduction variables.
+ // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
auto Size = ReductionInfos.size();
- // FIXME(JAN): skipping variably modified type storage for array size
Type *PtrTy = PointerType::getUnqual(Ctx);
Type *RedArrayTy = ArrayType::get(PtrTy, Size);
- InsertPointTy CurIP = Builder.saveIP();
+ CodeGenIP = Builder.saveIP();
Builder.restoreIP(AllocaIP);
Value *ReductionListAlloca =
Builder.CreateAlloca(RedArrayTy, nullptr, ".omp.reduction.red_list");
- Value *ReductionList =
- Builder.CreatePointerBitCastOrAddrSpaceCast(ReductionListAlloca, PtrTy);
- Builder.restoreIP(CurIP);
+ Value *ReductionList = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ ReductionListAlloca, PtrTy, ReductionListAlloca->getName() + ".ascast");
+ Builder.restoreIP(CodeGenIP);
+ Type *IndexTy = Builder.getIndexTy(
+ M.getDataLayout(), M.getDataLayout().getDefaultGlobalsAddressSpace());
for (auto En : enumerate(ReductionInfos)) {
const ReductionInfo &RI = En.value();
- Value *ElemPtr = Builder.CreateConstGEP2_64(RedArrayTy, ReductionList, 0,
- En.index(), "elem_ptr");
+ Value *ElemPtr = Builder.CreateInBoundsGEP(
+ RedArrayTy, ReductionList,
+ {ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});
Value *CastElem =
Builder.CreatePointerBitCastOrAddrSpaceCast(RI.PrivateVariable, PtrTy);
Builder.CreateStore(CastElem, ElemPtr);
}
- CurIP = Builder.saveIP();
- Function *SarFunc = emitShuffleAndReduceFunction(M, Loc, ReductionInfos,
- ReductionFunc, *this);
- Function *WcFunc = emitInterWarpCopyFunction(M, Loc, ReductionInfos, *this);
- Builder.restoreIP(CurIP);
+ CodeGenIP = Builder.saveIP();
+ Function *SarFunc =
+ emitShuffleAndReduceFunction(ReductionInfos, ReductionFunc, FuncAttrs);
+ Function *WcFunc = emitInterWarpCopyFunction(Loc, ReductionInfos, FuncAttrs);
+ Builder.restoreIP(CodeGenIP);
- Value *RL =
- Builder.CreatePointerBitCastOrAddrSpaceCast(ReductionList, PtrTy);
- Value *ReductionDataSize =
- getTypeSizeInBytesValue(Builder, M, ReductionInfos.begin()->ElementType);
+ Value *RL = Builder.CreatePointerBitCastOrAddrSpaceCast(ReductionList, PtrTy);
- Value *Res;
+ unsigned MaxDataSize = 0;
+ SmallVector<Type *> ReductionTypeArgs;
+ for (auto En : enumerate(ReductionInfos)) {
+ auto Size = M.getDataLayout().getTypeStoreSize(En.value().ElementType);
+ if (Size > MaxDataSize)
+ MaxDataSize = Size;
+ ReductionTypeArgs.emplace_back(En.value().ElementType);
+ }
+ Value *ReductionDataSize =
+ Builder.getInt64(MaxDataSize * ReductionInfos.size());
if (!IsTeamsReduction) {
Value *SarFuncCast =
Builder.CreatePointerBitCastOrAddrSpaceCast(SarFunc, PtrTy);
@@ -3214,25 +3386,27 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductionsGPU(
RuntimeFunction::OMPRTL___kmpc_nvptx_parallel_reduce_nowait_v2);
Res = Builder.CreateCall(Pv2Ptr, Args);
} else {
- CurIP = Builder.saveIP();
- Function *LtGCFunc =
- emitListToGlobalCopyFunction(M, Loc, ReductionInfos, *this);
- Function *LtGRFunc = emitListToGlobalReduceFunction(M, Loc, ReductionInfos,
- ReductionFunc, *this);
- Function *GtLCFunc =
- emitGlobalToListCopyFunction(M, Loc, ReductionInfos, *this);
- Function *GtLRFunc = emitGlobalToListReduceFunction(M, Loc, ReductionInfos,
- ReductionFunc, *this);
- Builder.restoreIP(CurIP);
-
+ CodeGenIP = Builder.saveIP();
+ StructType *ReductionsBufferTy = StructType::create(
+ Ctx, ReductionTypeArgs, "struct._globalized_locals_ty");
Function *RedFixedBuferFn = getOrCreateRuntimeFunctionPtr(
RuntimeFunction::OMPRTL___kmpc_reduction_get_fixed_buffer);
+ Function *LtGCFunc = emitListToGlobalCopyFunction(
+ ReductionInfos, ReductionsBufferTy, FuncAttrs);
+ Function *LtGRFunc = emitListToGlobalReduceFunction(
+ ReductionInfos, ReductionFunc, ReductionsBufferTy, FuncAttrs);
+ Function *GtLCFunc = emitGlobalToListCopyFunction(
+ ReductionInfos, ReductionsBufferTy, FuncAttrs);
+ Function *GtLRFunc = emitGlobalToListReduceFunction(
+ ReductionInfos, ReductionFunc, ReductionsBufferTy, FuncAttrs);
+ Builder.restoreIP(CodeGenIP);
- Value *KernelTeamsReductionPtr = Builder.CreateCall(RedFixedBuferFn, {});
+ Value *KernelTeamsReductionPtr = Builder.CreateCall(
+ RedFixedBuferFn, {}, "_openmp_teams_reductions_buffer_$_$ptr");
Value *Args3[] = {RTLoc,
KernelTeamsReductionPtr,
- Builder.getInt32(1024),
+ Builder.getInt32(ReductionBufNum),
ReductionDataSize,
RL,
SarFunc,
@@ -3247,32 +3421,50 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductionsGPU(
Res = Builder.CreateCall(TeamsReduceFn, Args3);
}
- if (IsTeamsReduction || !HasDistribute) {
- Function *CurFunc = Builder.GetInsertBlock()->getParent();
- BasicBlock *ExitBB =
- BasicBlock::Create(Ctx, ".omp.reduction.done", CurFunc);
- BasicBlock *ThenBB =
- BasicBlock::Create(Ctx, ".omp.reduction.then", CurFunc);
- Value *Cond = Builder.CreateICmpEQ(Res, Builder.getInt32(1));
- Builder.CreateCondBr(Cond, ThenBB, ExitBB);
+ // 5. Build if (res == 1)
+ BasicBlock *ExitBB = BasicBlock::Create(Ctx, ".omp.reduction.done");
+ BasicBlock *ThenBB = BasicBlock::Create(Ctx, ".omp.reduction.then");
+ Value *Cond = Builder.CreateICmpEQ(Res, Builder.getInt32(1));
+ Builder.CreateCondBr(Cond, ThenBB, ExitBB);
- Builder.SetInsertPoint(ThenBB);
- for (auto En : enumerate(ReductionInfos)) {
- const ReductionInfo &RI = En.value();
- Value *InputVal = Builder.CreateLoad(RI.ElementType, RI.Variable);
- Value *RedVal = Builder.CreateLoad(
- RI.ElementType, Builder.CreatePointerBitCastOrAddrSpaceCast(
- RI.PrivateVariable, PtrTy));
- Value *sum;
- Builder.restoreIP(
- RI.ReductionGen(Builder.saveIP(), InputVal, RedVal, sum));
- Builder.CreateStore(sum, RI.Variable);
- Builder.CreateBr(ExitBB);
+ // 6. Build then branch: where we have reduced values in the master
+ // thread in each team.
+ // __kmpc_end_reduce{_nowait}(<gtid>);
+ // break;
+ emitBlock(ThenBB, CurFunc);
+
+ // Add emission of __kmpc_end_reduce{_nowait}(<gtid>);
+ for (auto En : enumerate(ReductionInfos)) {
+ const ReductionInfo &RI = En.value();
+ Value *LHS = RI.Variable;
+ Value *RHS =
+ Builder.CreatePointerBitCastOrAddrSpaceCast(RI.PrivateVariable, PtrTy);
+
+ if (ReductionGenCBTy == ReductionGenCBTy::Clang) {
+ Value *LHSPtr, *RHSPtr;
+ Builder.restoreIP(RI.ReductionGenClang(Builder.saveIP(), En.index(),
+ &LHSPtr, &RHSPtr, CurFunc));
+
+ // Fix the CallBack code genereated to use the correct Values for the LHS
+ // and RHS
+ LHSPtr->replaceUsesWithIf(LHS, [ReductionFunc](const Use &U) {
+ return cast<Instruction>(U.getUser())->getParent()->getParent() ==
+ ReductionFunc;
+ });
+ RHSPtr->replaceUsesWithIf(RHS, [ReductionFunc](const Use &U) {
+ return cast<Instruction>(U.getUser())->getParent()->getParent() ==
+ ReductionFunc;
+ });
+ } else {
+ // LHS = Builder.CreateLoad(LHS);
+ // LHS = Builder.CreateLoad(LHS);
+ // Builder.restoreIP(RI.ReductionGen(Builder.saveIP(), LHS, RHS));
}
- Builder.SetInsertPoint(ExitBB);
}
- Builder.CreateBr(ContinuationBlock);
- Builder.SetInsertPoint(ContinuationBlock);
+ emitBlock(ExitBB, CurFunc);
+
+ Config.setEmitLLVMUsed();
+
return Builder.saveIP();
}
@@ -3281,8 +3473,8 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductions(
ArrayRef<ReductionInfo> ReductionInfos, bool IsNoWait, bool IsByRef,
bool IsTeamsReduction, bool HasDistribute) {
if (Config.isGPU())
- return createReductionsGPU(Loc, AllocaIP, ReductionInfos, IsNoWait,
- IsTeamsReduction, HasDistribute);
+ return createReductionsGPU(Loc, AllocaIP, Builder.saveIP(), ReductionInfos,
+ IsNoWait, IsTeamsReduction, HasDistribute);
checkReductionInfos(ReductionInfos, /*IsGPU*/ false);
@@ -3320,10 +3512,9 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductions(
Module *Module = Func->getParent();
uint32_t SrcLocStrSize;
Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
- bool CanGenerateAtomic =
- llvm::all_of(ReductionInfos, [](const ReductionInfo &RI) {
- return RI.AtomicReductionGen;
- });
+ bool CanGenerateAtomic = all_of(ReductionInfos, [](const ReductionInfo &RI) {
+ return RI.AtomicReductionGen;
+ });
Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize,
CanGenerateAtomic
? IdentFlag::OMP_IDENT_FLAG_ATOMIC_REDUCE
@@ -3333,7 +3524,8 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductions(
const DataLayout &DL = Module->getDataLayout();
unsigned RedArrayByteSize = DL.getTypeStoreSize(RedArrayTy);
Constant *RedArraySize = Builder.getInt64(RedArrayByteSize);
- Function *ReductionFunc = getFreshReductionFunc(*Module);
+ Function *ReductionFunc = createReductionFunction(
+ Builder.GetInsertBlock()->getParent()->getName(), ReductionInfos);
Value *Lock = getOMPCriticalRegionLock(".reduction");
Function *ReduceFunc = getOrCreateRuntimeFunctionPtr(
IsNoWait ? RuntimeFunction::OMPRTL___kmpc_reduce_nowait
@@ -3412,36 +3604,7 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductions(
// Populate the outlined reduction function using the elementwise reduction
// function. Partial values are extracted from the type-erased array of
// pointers to private variables.
- BasicBlock *ReductionFuncBlock =
- BasicBlock::Create(Module->getContext(), "", ReductionFunc);
- Builder.SetInsertPoint(ReductionFuncBlock);
- Value *LHSArrayPtr = ReductionFunc->getArg(0);
- Value *RHSArrayPtr = ReductionFunc->getArg(1);
-
- for (auto En : enumerate(ReductionInfos)) {
- const ReductionInfo &RI = En.value();
- Value *LHSI8PtrPtr = Builder.CreateConstInBoundsGEP2_64(
- RedArrayTy, LHSArrayPtr, 0, En.index());
- Value *LHSI8Ptr = Builder.CreateLoad(Builder.getPtrTy(), LHSI8PtrPtr);
- Value *LHSPtr = Builder.CreateBitCast(LHSI8Ptr, RI.Variable->getType());
- Value *LHS = Builder.CreateLoad(RI.ElementType, LHSPtr);
- Value *RHSI8PtrPtr = Builder.CreateConstInBoundsGEP2_64(
- RedArrayTy, RHSArrayPtr, 0, En.index());
- Value *RHSI8Ptr = Builder.CreateLoad(Builder.getPtrTy(), RHSI8PtrPtr);
- Value *RHSPtr =
- Builder.CreateBitCast(RHSI8Ptr, RI.PrivateVariable->getType());
- Value *RHS = Builder.CreateLoad(RI.ElementType, RHSPtr);
- Value *Reduced;
- Builder.restoreIP(RI.ReductionGen(Builder.saveIP(), LHS, RHS, Reduced));
- if (!Builder.GetInsertBlock())
- return InsertPointTy();
- // store is inside of the reduction region when using by-ref
- if (!IsByRef)
- Builder.CreateStore(Reduced, LHSPtr);
- }
- Builder.CreateRetVoid();
-
- populateReductionFunction(ReductionFunc, ReductionInfos, Builder, false);
+ // populateReductionFunction(ReductionFunc, ReductionInfos, Builder, false);
Builder.SetInsertPoint(ContinuationBlock);
return Builder.saveIP();
}
@@ -5799,7 +5962,8 @@ OpenMPIRBuilder::createTargetInit(const LocationDescription &Loc, bool IsSPMD,
Ident,
DynamicEnvironment,
});
- Twine KernelEnvironmentName = KernelName + "_kernel_environment";
+ std::string KernelEnvironmentName =
+ (KernelName + "_kernel_environment").str();
GlobalVariable *KernelEnvironmentGV = new GlobalVariable(
M, KernelEnvironment, /*IsConstant=*/true, GlobalValue::WeakODRLinkage,
KernelEnvironmentInitializer, KernelEnvironmentName,
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 7df33470ea066a..0bb4d4d889168a 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -847,14 +847,17 @@ static void collectReductionInfo(
// Collect the reduction information.
reductionInfos.reserve(numReductions);
for (unsigned i = 0; i < numReductions; ++i) {
- llvm::OpenMPIRBuilder::AtomicReductionGenTy atomicGen = nullptr;
+ llvm::OpenMPIRBuilder::AtomicReductionGenCB atomicGen = nullptr;
if (owningAtomicReductionGens[i])
atomicGen = owningAtomicReductionGens[i];
llvm::Value *variable =
moduleTranslation.lookupValue(loop.getReductionVars()[i]);
- reductionInfos.push_back(
- {moduleTranslation.convertType(reductionDecls[i].getType()), variable,
- privateReductionVariables[i], owningReductionGens[i], atomicGen});
+ reductionInfos.push_back(llvm::OpenMPIRBuilder::ReductionInfo(
+ moduleTranslation.convertType(reductionDecls[i].getType()), variable,
+ privateReductionVariables[i],
+ /*EvaluationKind=*/llvm::OpenMPIRBuilder::EvaluationKindTy::Scalar,
+ owningReductionGens[i],
+ /*ReductionGenClang=*/nullptr, atomicGen));
}
}
>From 90b4595d2c16eb54ada298e8c7cd232fbbdfaa73 Mon Sep 17 00:00:00 2001
From: Akash Banerjee <Akash.Banerjee at amd.com>
Date: Wed, 7 Feb 2024 14:23:41 +0000
Subject: [PATCH 03/15] Remove ReductionInfoManager. Add assert failures for
complex data types.
---
.../llvm/Frontend/OpenMP/OMPIRBuilder.h | 47 -------------------
llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp | 12 +++--
2 files changed, 8 insertions(+), 51 deletions(-)
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
index 4f2207cc700fbf..0de05ffd6dac36 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
@@ -1350,50 +1350,6 @@ class OpenMPIRBuilder {
Value *ScratchpadWidth = nullptr;
};
- /// A class that manages the reduction info to facilitate lowering of
- /// reductions at multiple levels of parallelism. For example handling teams
- /// and parallel reductions on GPUs
- class ReductionInfoManager {
- private:
- SmallVector<ReductionInfo> ReductionInfos;
- std::optional<InsertPointTy> PrivateVarAllocaIP;
-
- public:
- ReductionInfoManager(){};
- void clear() {
- ReductionInfos.clear();
- PrivateVarAllocaIP.reset();
- }
-
- Value *allocatePrivateReductionVar(IRBuilderBase &builder,
- InsertPointTy &allocaIP, Type *VarType) {
- Type *ptrTy = PointerType::getUnqual(builder.getContext());
- Value *var = builder.CreateAlloca(VarType);
- var->setName("private_redvar");
- Value *castVar = builder.CreatePointerBitCastOrAddrSpaceCast(var, ptrTy);
- ReductionInfos.push_back(ReductionInfo(castVar));
- return castVar;
- }
-
- ReductionInfo getReductionInfo(unsigned Index) {
- return ReductionInfos[Index];
- }
- ReductionInfo setReductionInfo(unsigned Index, ReductionInfo &RI) {
- return ReductionInfos[Index] = RI;
- }
- Value *getPrivateReductionVariable(unsigned Index) {
- return ReductionInfos[Index].PrivateVariable;
- }
- SmallVector<ReductionInfo> &getReductionInfos() { return ReductionInfos; }
-
- bool hasPrivateVarAllocaIP() { return PrivateVarAllocaIP.has_value(); }
- InsertPointTy getPrivateVarAllocaIP() {
- assert(PrivateVarAllocaIP.has_value() && "AllocaIP not set");
- return *PrivateVarAllocaIP;
- }
- void setPrivateVarAllocaIP(InsertPointTy IP) { PrivateVarAllocaIP = IP; }
- };
-
/// Supporting functions for Reductions CodeGen.
private:
/// Emit the llvm.used metadata.
@@ -2090,9 +2046,6 @@ class OpenMPIRBuilder {
/// Info manager to keep track of target regions.
OffloadEntriesInfoManager OffloadInfoManager;
- /// Info manager to keep track of reduction information;
- ReductionInfoManager RIManager;
-
/// The target triple of the underlying module.
const Triple T;
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 435af68e344a8a..5b79baba1b3e1a 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -2364,6 +2364,7 @@ void OpenMPIRBuilder::emitReductionListCopy(
break;
}
case EvaluationKindTy::Complex: {
+ assert(false && "Complex data type not handled");
break;
}
case EvaluationKindTy::Aggregate: {
@@ -2869,9 +2870,10 @@ Function *OpenMPIRBuilder::emitListToGlobalCopyFunction(
break;
}
case EvaluationKindTy::Complex: {
+ assert(false && "Complex data type not handled");
break;
}
- case EvaluationKindTy::Aggregate:
+ case EvaluationKindTy::Aggregate: {
Value *SizeVal =
Builder.getInt64(M.getDataLayout().getTypeStoreSize(RI.ElementType));
Builder.CreateMemCpy(
@@ -2879,6 +2881,7 @@ Function *OpenMPIRBuilder::emitListToGlobalCopyFunction(
M.getDataLayout().getPrefTypeAlign(RI.ElementType), SizeVal, false);
break;
}
+ }
}
Builder.CreateRetVoid();
@@ -3043,10 +3046,10 @@ Function *OpenMPIRBuilder::emitGlobalToListCopyFunction(
break;
}
case EvaluationKindTy::Complex: {
- // FIXME(Jan): Complex type
+ assert(false && "Complex data type not handled");
break;
}
- case EvaluationKindTy::Aggregate:
+ case EvaluationKindTy::Aggregate: {
Value *SizeVal =
Builder.getInt64(M.getDataLayout().getTypeStoreSize(RI.ElementType));
Builder.CreateMemCpy(
@@ -3055,6 +3058,7 @@ Function *OpenMPIRBuilder::emitGlobalToListCopyFunction(
SizeVal, false);
break;
}
+ }
}
Builder.CreateRetVoid();
@@ -3265,7 +3269,6 @@ checkReductionInfos(ArrayRef<OpenMPIRBuilder::ReductionInfo> ReductionInfos,
assert(RI.PrivateVariable && "expected non-null private variable");
assert((RI.ReductionGen || RI.ReductionGenClang) &&
"expected non-null reduction generator callback");
- // JAN: Skip this assertion for GPU, address spaces are present
if (!IsGPU) {
assert(
RI.Variable->getType() == RI.PrivateVariable->getType() &&
@@ -4124,6 +4127,7 @@ static void createTargetLoopWorkshareCall(
// FIXME(JAN): The trip count is 1 larger than it should be for GPU, this may
// not be the right way to fix it, but this works for now.
if (OMPBuilder->Config.isGPU()) {
+ assert(false && "Akash");
if (LoopType != WorksharingLoopType::DistributeStaticLoop)
Builder.restoreIP({InsertBlock, std::prev(InsertBlock->end())});
LLVMContext &Ctx = M.getContext();
>From 97ea606dc962a467bbdd8ee348648fea009729f3 Mon Sep 17 00:00:00 2001
From: Akash Banerjee <Akash.Banerjee at amd.com>
Date: Wed, 7 Feb 2024 14:27:15 +0000
Subject: [PATCH 04/15] Remove unintended debug assert added in previous
commit.
---
llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp | 1 -
1 file changed, 1 deletion(-)
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 5b79baba1b3e1a..c48b7a3392bd06 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -4127,7 +4127,6 @@ static void createTargetLoopWorkshareCall(
// FIXME(JAN): The trip count is 1 larger than it should be for GPU, this may
// not be the right way to fix it, but this works for now.
if (OMPBuilder->Config.isGPU()) {
- assert(false && "Akash");
if (LoopType != WorksharingLoopType::DistributeStaticLoop)
Builder.restoreIP({InsertBlock, std::prev(InsertBlock->end())});
LLVMContext &Ctx = M.getContext();
>From ccfb057a717730277feefa285b49970de1f05025 Mon Sep 17 00:00:00 2001
From: Akash Banerjee <Akash.Banerjee at amd.com>
Date: Tue, 13 Feb 2024 12:48:38 +0000
Subject: [PATCH 05/15] Remove unintended code from MLIRTranslation.
---
.../OpenMP/OpenMPToLLVMIRTranslation.cpp | 52 +++++++++----------
1 file changed, 26 insertions(+), 26 deletions(-)
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 0bb4d4d889168a..c99c5c72f578e7 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -852,12 +852,12 @@ static void collectReductionInfo(
atomicGen = owningAtomicReductionGens[i];
llvm::Value *variable =
moduleTranslation.lookupValue(loop.getReductionVars()[i]);
- reductionInfos.push_back(llvm::OpenMPIRBuilder::ReductionInfo(
- moduleTranslation.convertType(reductionDecls[i].getType()), variable,
- privateReductionVariables[i],
- /*EvaluationKind=*/llvm::OpenMPIRBuilder::EvaluationKindTy::Scalar,
- owningReductionGens[i],
- /*ReductionGenClang=*/nullptr, atomicGen));
+ reductionInfos.push_back(
+ {moduleTranslation.convertType(reductionDecls[i].getType()), variable,
+ privateReductionVariables[i],
+ /*EvaluationKind=*/llvm::OpenMPIRBuilder::EvaluationKindTy::Scalar,
+ owningReductionGens[i],
+ /*ReductionGenClang=*/nullptr, atomicGen});
}
}
@@ -3057,26 +3057,26 @@ LogicalResult OpenMPDialectLLVMIRTranslationInterface::amendOperation(
moduleTranslation);
return failure();
})
- .Case("omp.requires",
- [&](Attribute attr) {
- if (auto requiresAttr =
- attr.dyn_cast<omp::ClauseRequiresAttr>()) {
- using Requires = omp::ClauseRequires;
- Requires flags = requiresAttr.getValue();
- llvm::OpenMPIRBuilderConfig &config =
- moduleTranslation.getOpenMPBuilder()->Config;
- config.setHasRequiresReverseOffload(
- bitEnumContainsAll(flags, Requires::reverse_offload));
- config.setHasRequiresUnifiedAddress(
- bitEnumContainsAll(flags, Requires::unified_address));
- config.setHasRequiresUnifiedSharedMemory(
- bitEnumContainsAll(flags, Requires::unified_shared_memory));
- config.setHasRequiresDynamicAllocators(
- bitEnumContainsAll(flags, Requires::dynamic_allocators));
- return success();
- }
- return failure();
- })
+ .Case(
+ "omp.requires",
+ [&](Attribute attr) {
+ if (auto requiresAttr = attr.dyn_cast<omp::ClauseRequiresAttr>()) {
+ using Requires = omp::ClauseRequires;
+ Requires flags = requiresAttr.getValue();
+ llvm::OpenMPIRBuilderConfig &config =
+ moduleTranslation.getOpenMPBuilder()->Config;
+ config.setHasRequiresReverseOffload(
+ bitEnumContainsAll(flags, Requires::reverse_offload));
+ config.setHasRequiresUnifiedAddress(
+ bitEnumContainsAll(flags, Requires::unified_address));
+ config.setHasRequiresUnifiedSharedMemory(
+ bitEnumContainsAll(flags, Requires::unified_shared_memory));
+ config.setHasRequiresDynamicAllocators(
+ bitEnumContainsAll(flags, Requires::dynamic_allocators));
+ return convertRequiresAttr(*op, requiresAttr, moduleTranslation);
+ }
+ return failure();
+ })
.Default([](Attribute) {
// Fall through for omp attributes that do not require lowering.
return success();
>From b57733f842dcef03347d1fca7b8f0f1f5ec92d11 Mon Sep 17 00:00:00 2001
From: Akash Banerjee <Akash.Banerjee at amd.com>
Date: Tue, 13 Feb 2024 13:48:43 +0000
Subject: [PATCH 06/15] Remove unintended code for WSLoop
---
llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp | 18 ++----------------
1 file changed, 2 insertions(+), 16 deletions(-)
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index c48b7a3392bd06..712e5f7e00ff98 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -4120,24 +4120,10 @@ getKmpcForStaticLoopForType(Type *Ty, OpenMPIRBuilder *OMPBuilder,
static void createTargetLoopWorkshareCall(
OpenMPIRBuilder *OMPBuilder, WorksharingLoopType LoopType,
BasicBlock *InsertBlock, Value *Ident, Value *LoopBodyArg,
- Type *ParallelTaskPtr, Value *TripCountOrig, Function &LoopBodyFn) {
+ Type *ParallelTaskPtr, Value *TripCount, Function &LoopBodyFn) {
+ Type *TripCountTy = TripCount->getType();
Module &M = OMPBuilder->M;
IRBuilder<> &Builder = OMPBuilder->Builder;
- Value *TripCount = TripCountOrig;
- // FIXME(JAN): The trip count is 1 larger than it should be for GPU, this may
- // not be the right way to fix it, but this works for now.
- if (OMPBuilder->Config.isGPU()) {
- if (LoopType != WorksharingLoopType::DistributeStaticLoop)
- Builder.restoreIP({InsertBlock, std::prev(InsertBlock->end())});
- LLVMContext &Ctx = M.getContext();
- Type *IVTy = TripCountOrig->getType();
- Type *InternalIVTy = IVTy->getIntegerBitWidth() <= 32
- ? Type::getInt32Ty(Ctx)
- : Type::getInt64Ty(Ctx);
- Constant *One = ConstantInt::get(InternalIVTy, 1);
- TripCount = Builder.CreateSub(TripCountOrig, One, "modified_trip_count");
- }
- Type *TripCountTy = TripCount->getType();
FunctionCallee RTLFn =
getKmpcForStaticLoopForType(TripCountTy, OMPBuilder, LoopType);
SmallVector<Value *, 8> RealArgs;
>From 5cc91c2c7ad160520537de585bf1eaac88061c68 Mon Sep 17 00:00:00 2001
From: Akash Banerjee <Akash.Banerjee at amd.com>
Date: Tue, 13 Feb 2024 15:02:45 +0000
Subject: [PATCH 07/15] Remove more non-essential code to this patch.
---
.../llvm/Frontend/OpenMP/OMPIRBuilder.h | 5 ---
llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp | 34 +++++--------------
2 files changed, 9 insertions(+), 30 deletions(-)
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
index 0de05ffd6dac36..4065ddfbe230ff 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
@@ -106,11 +106,6 @@ class OpenMPIRBuilderConfig {
/// Flag for specifying if offloading is mandatory.
std::optional<bool> OpenMPOffloadMandatory;
- /// Name of the target processor.
- StringRef TargetCPU;
- /// String representation of the target processor's features.
- StringRef TargetFeatures;
-
/// First separator used between the initial two parts of a name.
std::optional<StringRef> FirstSeparator;
/// Separator used between all of the rest consecutive parts of s name
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 712e5f7e00ff98..6c0e1070e705ef 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -147,22 +147,13 @@ static bool isValidWorkshareLoopScheduleType(OMPScheduleType SchedType) {
Function *GLOBAL_ReductionFunc = nullptr;
-static const omp::GV &getGridValue(const Triple &T, StringRef Features) {
- if (T.isAMDGPU()) {
- if (Features.count("+wavefrontsize64"))
- return omp::getAMDGPUGridValues<64>();
- return omp::getAMDGPUGridValues<32>();
- }
- if (T.isNVPTX())
- return omp::NVPTXGridValues;
- llvm_unreachable("No grid value available for this architecture!");
-}
-
static const omp::GV &getGridValue(const Triple &T, Function *Kernel) {
if (T.isAMDGPU()) {
StringRef Features =
Kernel->getFnAttribute("target-features").getValueAsString();
- return getGridValue(T, Features);
+ if (Features.count("+wavefrontsize64"))
+ return omp::getAMDGPUGridValues<64>();
+ return omp::getAMDGPUGridValues<32>();
}
if (T.isNVPTX())
return omp::NVPTXGridValues;
@@ -3310,12 +3301,6 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductionsGPU(
AttrBldr.removeAttribute(Attribute::OptimizeNone);
FuncAttrs = FuncAttrs.addFnAttributes(Ctx, AttrBldr);
- // Set the grid value in the config needed for lowering later on
- if (GridValue.has_value())
- Config.setGridValue(GridValue.value());
- else
- Config.setGridValue(getGridValue(T, Config.TargetFeatures));
-
Function *ReductionFunc = nullptr;
if (GLOBAL_ReductionFunc) {
ReductionFunc = GLOBAL_ReductionFunc;
@@ -3327,6 +3312,12 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductionsGPU(
Builder.restoreIP(CodeGenIP);
}
+ // Set the grid value in the config needed for lowering later on
+ if (GridValue.has_value())
+ Config.setGridValue(GridValue.value());
+ else
+ Config.setGridValue(getGridValue(T, ReductionFunc));
+
uint32_t SrcLocStrSize;
Constant *SrcLocStr = getOrCreateDefaultSrcLocStr(SrcLocStrSize);
Value *RTLoc =
@@ -5884,9 +5875,6 @@ OpenMPIRBuilder::createTargetInit(const LocationDescription &Loc, bool IsSPMD,
Function *Kernel = Builder.GetInsertBlock()->getParent();
- // Set the grid value in the config needed for lowering later on
- Config.setGridValue(getGridValue(T, Config.TargetFeatures));
-
// Manifest the launch configuration in the metadata matching the kernel
// environment.
if (MinTeamsVal > 1 || MaxTeamsVal > 0)
@@ -6140,10 +6128,6 @@ void OpenMPIRBuilder::setOutlinedTargetRegionFunctionAttributes(
OutlinedFn->setVisibility(GlobalValue::ProtectedVisibility);
if (T.isAMDGCN())
OutlinedFn->setCallingConv(CallingConv::AMDGPU_KERNEL);
- if (!Config.TargetCPU.empty())
- OutlinedFn->addFnAttr("target-cpu", Config.TargetCPU);
- if (!Config.TargetFeatures.empty())
- OutlinedFn->addFnAttr("target-features", Config.TargetFeatures);
}
}
>From 942a5a475c2495ead4f4f8c8cecafa94b7f83258 Mon Sep 17 00:00:00 2001
From: Akash Banerjee <Akash.Banerjee at amd.com>
Date: Wed, 14 Feb 2024 13:38:53 +0000
Subject: [PATCH 08/15] Update OMPIRBuilderTest.cpp.
---
llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp | 18 +++++++-----------
.../unittests/Frontend/OpenMPIRBuilderTest.cpp | 16 ++++++++++++----
2 files changed, 19 insertions(+), 15 deletions(-)
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 6c0e1070e705ef..41a5e904bf4394 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -3200,7 +3200,7 @@ Function *OpenMPIRBuilder::createReductionFunction(
RedArrayTy, RHSArrayPtr,
{ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});
Value *RHSI8Ptr = Builder.CreateLoad(Builder.getPtrTy(), RHSI8PtrPtr);
- Value *RHS = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ Value *RHSPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
RHSI8Ptr, RI.PrivateVariable->getType(),
RHSI8Ptr->getName() + ".ascast");
@@ -3208,20 +3208,20 @@ Function *OpenMPIRBuilder::createReductionFunction(
RedArrayTy, LHSArrayPtr,
{ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});
Value *LHSI8Ptr = Builder.CreateLoad(Builder.getPtrTy(), LHSI8PtrPtr);
- Value *LHS = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ Value *LHSPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
LHSI8Ptr, RI.Variable->getType(), LHSI8Ptr->getName() + ".ascast");
if (ReductionGenCBTy == ReductionGenCBTy::Clang) {
- LHSPtrs.emplace_back(LHS);
- RHSPtrs.emplace_back(RHS);
+ LHSPtrs.emplace_back(LHSPtr);
+ RHSPtrs.emplace_back(RHSPtr);
} else {
- LHS = Builder.CreateLoad(RI.ElementType, LHS);
- RHS = Builder.CreateLoad(RI.ElementType, RHS);
+ Value *LHS = Builder.CreateLoad(RI.ElementType, LHSPtr);
+ Value *RHS = Builder.CreateLoad(RI.ElementType, RHSPtr);
Value *Reduced;
RI.ReductionGen(Builder.saveIP(), LHS, RHS, Reduced);
if (!Builder.GetInsertBlock())
return ReductionFunc;
- Builder.CreateStore(Reduced, LHS);
+ Builder.CreateStore(Reduced, LHSPtr);
}
}
@@ -3595,10 +3595,6 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductions(
Builder.CreateUnreachable();
}
- // Populate the outlined reduction function using the elementwise reduction
- // function. Partial values are extracted from the type-erased array of
- // pointers to private variables.
- // populateReductionFunction(ReductionFunc, ReductionInfos, Builder, false);
Builder.SetInsertPoint(ContinuationBlock);
return Builder.saveIP();
}
diff --git a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
index 5c415cadcd686c..5cf1dbd0620f87 100644
--- a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
+++ b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
@@ -4980,8 +4980,12 @@ TEST_F(OpenMPIRBuilderTest, CreateReductions) {
Builder.restoreIP(AfterIP);
OpenMPIRBuilder::ReductionInfo ReductionInfos[] = {
- {SumType, SumReduced, SumPrivatized, sumReduction, sumAtomicReduction},
- {XorType, XorReduced, XorPrivatized, xorReduction, xorAtomicReduction}};
+ {SumType, SumReduced, SumPrivatized,
+ /*EvaluationKind=*/OpenMPIRBuilder::EvaluationKindTy::Scalar,
+ sumReduction, /*ReductionGenClang=*/nullptr, sumAtomicReduction},
+ {XorType, XorReduced, XorPrivatized,
+ /*EvaluationKind=*/OpenMPIRBuilder::EvaluationKindTy::Scalar,
+ xorReduction, /*ReductionGenClang=*/nullptr, xorAtomicReduction}};
OMPBuilder.createReductions(BodyIP, BodyAllocaIP, ReductionInfos);
@@ -5232,10 +5236,14 @@ TEST_F(OpenMPIRBuilderTest, CreateTwoReductions) {
OMPBuilder.createReductions(
FirstBodyIP, FirstBodyAllocaIP,
- {{SumType, SumReduced, SumPrivatized, sumReduction, sumAtomicReduction}});
+ {{SumType, SumReduced, SumPrivatized,
+ /*EvaluationKind=*/OpenMPIRBuilder::EvaluationKindTy::Scalar,
+ sumReduction, /*ReductionGenClang=*/nullptr, sumAtomicReduction}});
OMPBuilder.createReductions(
SecondBodyIP, SecondBodyAllocaIP,
- {{XorType, XorReduced, XorPrivatized, xorReduction, xorAtomicReduction}});
+ {{XorType, XorReduced, XorPrivatized,
+ /*EvaluationKind=*/OpenMPIRBuilder::EvaluationKindTy::Scalar,
+ xorReduction, /*ReductionGenClang=*/nullptr, xorAtomicReduction}});
Builder.restoreIP(AfterIP);
Builder.CreateRetVoid();
>From e36db3c07ced90ced4a28bb0fc606fbc03aa6344 Mon Sep 17 00:00:00 2001
From: Akash Banerjee <Akash.Banerjee at amd.com>
Date: Wed, 14 Feb 2024 15:48:55 +0000
Subject: [PATCH 09/15] Fix cpu reductions.
---
.../llvm/Frontend/OpenMP/OMPIRBuilder.h | 1 -
llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp | 103 +++++++++++++-----
2 files changed, 74 insertions(+), 30 deletions(-)
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
index 4065ddfbe230ff..6383bc1775a51e 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
@@ -1574,7 +1574,6 @@ class OpenMPIRBuilder {
/// \return The reduction function.
Function *createReductionFunction(
StringRef ReducerName, ArrayRef<ReductionInfo> ReductionInfos,
- bool IsGpu = false,
ReductionGenCBTy ReductionGenCBTy = ReductionGenCBTy::MLIR,
AttributeList FuncAttrs = {});
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 41a5e904bf4394..f8a8e70e4c3f4a 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -3147,13 +3147,13 @@ std::string OpenMPIRBuilder::getReductionFuncName(StringRef Name) const {
}
Function *OpenMPIRBuilder::createReductionFunction(
- StringRef ReducerName, ArrayRef<ReductionInfo> ReductionInfos, bool IsGPU,
+ StringRef ReducerName, ArrayRef<ReductionInfo> ReductionInfos,
ReductionGenCBTy ReductionGenCBTy, AttributeList FuncAttrs) {
auto *FuncTy = FunctionType::get(Builder.getVoidTy(),
{Builder.getPtrTy(), Builder.getPtrTy()},
/* IsVarArg */ false);
std::string Name = getReductionFuncName(ReducerName);
- auto *ReductionFunc =
+ Function *ReductionFunc =
Function::Create(FuncTy, GlobalVariable::InternalLinkage, Name, &M);
ReductionFunc->setAttributes(FuncAttrs);
ReductionFunc->addParamAttr(0, Attribute::NoUndef);
@@ -3162,33 +3162,27 @@ Function *OpenMPIRBuilder::createReductionFunction(
BasicBlock::Create(M.getContext(), "entry", ReductionFunc);
Builder.SetInsertPoint(EntryBB);
+ // Need to alloca memory here and deal with the pointers before getting
+ // LHS/RHS pointers out
Value *LHSArrayPtr = nullptr;
Value *RHSArrayPtr = nullptr;
- if (IsGPU) {
- // Need to alloca memory here and deal with the pointers before getting
- // LHS/RHS pointers out
- //
- Argument *Arg0 = ReductionFunc->getArg(0);
- Argument *Arg1 = ReductionFunc->getArg(1);
- Type *Arg0Type = Arg0->getType();
- Type *Arg1Type = Arg1->getType();
-
- Value *LHSAlloca =
- Builder.CreateAlloca(Arg0Type, nullptr, Arg0->getName() + ".addr");
- Value *RHSAlloca =
- Builder.CreateAlloca(Arg1Type, nullptr, Arg1->getName() + ".addr");
- Value *LHSAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
- LHSAlloca, Arg0Type, LHSAlloca->getName() + ".ascast");
- Value *RHSAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
- RHSAlloca, Arg1Type, RHSAlloca->getName() + ".ascast");
- Builder.CreateStore(Arg0, LHSAddrCast);
- Builder.CreateStore(Arg1, RHSAddrCast);
- LHSArrayPtr = Builder.CreateLoad(Arg0Type, LHSAddrCast);
- RHSArrayPtr = Builder.CreateLoad(Arg1Type, RHSAddrCast);
- } else {
- LHSArrayPtr = ReductionFunc->getArg(0);
- RHSArrayPtr = ReductionFunc->getArg(1);
- }
+ Argument *Arg0 = ReductionFunc->getArg(0);
+ Argument *Arg1 = ReductionFunc->getArg(1);
+ Type *Arg0Type = Arg0->getType();
+ Type *Arg1Type = Arg1->getType();
+
+ Value *LHSAlloca =
+ Builder.CreateAlloca(Arg0Type, nullptr, Arg0->getName() + ".addr");
+ Value *RHSAlloca =
+ Builder.CreateAlloca(Arg1Type, nullptr, Arg1->getName() + ".addr");
+ Value *LHSAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ LHSAlloca, Arg0Type, LHSAlloca->getName() + ".ascast");
+ Value *RHSAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ RHSAlloca, Arg1Type, RHSAlloca->getName() + ".ascast");
+ Builder.CreateStore(Arg0, LHSAddrCast);
+ Builder.CreateStore(Arg1, RHSAddrCast);
+ LHSArrayPtr = Builder.CreateLoad(Arg0Type, LHSAddrCast);
+ RHSArrayPtr = Builder.CreateLoad(Arg1Type, RHSAddrCast);
Type *RedArrayTy = ArrayType::get(Builder.getPtrTy(), ReductionInfos.size());
Type *IndexTy = Builder.getIndexTy(
@@ -3462,6 +3456,54 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductionsGPU(
return Builder.saveIP();
}
+static Function *getFreshReductionFunc(Module &M) {
+ Type *VoidTy = Type::getVoidTy(M.getContext());
+ Type *Int8PtrTy = PointerType::getUnqual(M.getContext());
+ auto *FuncTy =
+ FunctionType::get(VoidTy, {Int8PtrTy, Int8PtrTy}, /* IsVarArg */ false);
+ return Function::Create(FuncTy, GlobalVariable::InternalLinkage,
+ ".omp.reduction.func", &M);
+}
+
+static void populateReductionFunction(
+ Function *ReductionFunc,
+ ArrayRef<OpenMPIRBuilder::ReductionInfo> ReductionInfos,
+ IRBuilder<> &Builder) {
+ Module *Module = ReductionFunc->getParent();
+ BasicBlock *ReductionFuncBlock =
+ BasicBlock::Create(Module->getContext(), "", ReductionFunc);
+ Builder.SetInsertPoint(ReductionFuncBlock);
+ Value *LHSArrayPtr = nullptr;
+ Value *RHSArrayPtr = nullptr;
+ LHSArrayPtr = ReductionFunc->getArg(0);
+ RHSArrayPtr = ReductionFunc->getArg(1);
+
+ unsigned NumReductions = ReductionInfos.size();
+ Type *RedArrayTy = ArrayType::get(Builder.getPtrTy(), NumReductions);
+
+ for (auto En : enumerate(ReductionInfos)) {
+ const OpenMPIRBuilder::ReductionInfo &RI = En.value();
+ Value *LHSI8PtrPtr = Builder.CreateConstInBoundsGEP2_64(
+ RedArrayTy, LHSArrayPtr, 0, En.index());
+ Value *LHSI8Ptr = Builder.CreateLoad(Builder.getPtrTy(), LHSI8PtrPtr);
+ Value *LHSPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ LHSI8Ptr, RI.Variable->getType());
+ Value *LHS = Builder.CreateLoad(RI.ElementType, LHSPtr);
+ Value *RHSI8PtrPtr = Builder.CreateConstInBoundsGEP2_64(
+ RedArrayTy, RHSArrayPtr, 0, En.index());
+ Value *RHSI8Ptr = Builder.CreateLoad(Builder.getPtrTy(), RHSI8PtrPtr);
+ Value *RHSPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ RHSI8Ptr, RI.PrivateVariable->getType());
+ Value *RHS = Builder.CreateLoad(RI.ElementType, RHSPtr);
+ Value *Reduced;
+ Builder.restoreIP(RI.ReductionGen(Builder.saveIP(), LHS, RHS, Reduced));
+ if (!Builder.GetInsertBlock())
+ return;
+ Builder.CreateStore(Reduced, LHSPtr);
+ }
+ Builder.CreateRetVoid();
+}
+
OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductions(
const LocationDescription &Loc, InsertPointTy AllocaIP,
ArrayRef<ReductionInfo> ReductionInfos, bool IsNoWait, bool IsByRef,
@@ -3518,8 +3560,7 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductions(
const DataLayout &DL = Module->getDataLayout();
unsigned RedArrayByteSize = DL.getTypeStoreSize(RedArrayTy);
Constant *RedArraySize = Builder.getInt64(RedArrayByteSize);
- Function *ReductionFunc = createReductionFunction(
- Builder.GetInsertBlock()->getParent()->getName(), ReductionInfos);
+ Function *ReductionFunc = getFreshReductionFunc(M);
Value *Lock = getOMPCriticalRegionLock(".reduction");
Function *ReduceFunc = getOrCreateRuntimeFunctionPtr(
IsNoWait ? RuntimeFunction::OMPRTL___kmpc_reduce_nowait
@@ -3595,6 +3636,10 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductions(
Builder.CreateUnreachable();
}
+ // Populate the outlined reduction function using the elementwise reduction
+ // function. Partial values are extracted from the type-erased array of
+ // pointers to private variables.
+ populateReductionFunction(ReductionFunc, ReductionInfos, Builder);
Builder.SetInsertPoint(ContinuationBlock);
return Builder.saveIP();
}
>From 43fe699e223216f4c89fb12b83a68f10f49a7808 Mon Sep 17 00:00:00 2001
From: Akash Banerjee <Akash.Banerjee at amd.com>
Date: Wed, 14 Feb 2024 16:11:26 +0000
Subject: [PATCH 10/15] Fix nit issue,
---
llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index f8a8e70e4c3f4a..0ab34c30ba1c1d 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -3301,7 +3301,7 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductionsGPU(
} else {
CodeGenIP = Builder.saveIP();
ReductionFunc = createReductionFunction(
- Builder.GetInsertBlock()->getParent()->getName(), ReductionInfos, true,
+ Builder.GetInsertBlock()->getParent()->getName(), ReductionInfos,
ReductionGenCBTy, FuncAttrs);
Builder.restoreIP(CodeGenIP);
}
>From 966e14629a4c4db87b873734849c6aeff36b8be4 Mon Sep 17 00:00:00 2001
From: Akash Banerjee <Akash.Banerjee at amd.com>
Date: Wed, 14 Feb 2024 17:45:07 +0000
Subject: [PATCH 11/15] Fix OpenMPIRBuilderTest build failure.
---
llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
index 5cf1dbd0620f87..de4b6d6c226a81 100644
--- a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
+++ b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
@@ -4986,7 +4986,7 @@ TEST_F(OpenMPIRBuilderTest, CreateReductions) {
{XorType, XorReduced, XorPrivatized,
/*EvaluationKind=*/OpenMPIRBuilder::EvaluationKindTy::Scalar,
xorReduction, /*ReductionGenClang=*/nullptr, xorAtomicReduction}};
-
+ OMPBuilder.Config.setIsGPU(false);
OMPBuilder.createReductions(BodyIP, BodyAllocaIP, ReductionInfos);
Builder.restoreIP(AfterIP);
@@ -5234,6 +5234,7 @@ TEST_F(OpenMPIRBuilderTest, CreateTwoReductions) {
/* NumThreads */ nullptr, OMP_PROC_BIND_default,
/* IsCancellable */ false);
+ OMPBuilder.Config.setIsGPU(false);
OMPBuilder.createReductions(
FirstBodyIP, FirstBodyAllocaIP,
{{SumType, SumReduced, SumPrivatized,
>From ddf2ad352604610da25cf9f164a47c939993c03a Mon Sep 17 00:00:00 2001
From: Akash Banerjee <Akash.Banerjee at amd.com>
Date: Thu, 15 Feb 2024 14:17:27 +0000
Subject: [PATCH 12/15] Fix Clang CI build error.
---
clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp | 2 --
1 file changed, 2 deletions(-)
diff --git a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
index 7cddf73306f2d9..3fb15df5005d7f 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
@@ -1655,9 +1655,7 @@ void CGOpenMPRuntimeGPU::emitReduction(
bool ParallelReduction = isOpenMPParallelDirective(Options.ReductionKind);
bool DistributeReduction = isOpenMPDistributeDirective(Options.ReductionKind);
-#ifndef NDEBUG
bool TeamsReduction = isOpenMPTeamsDirective(Options.ReductionKind);
-#endif
ASTContext &C = CGM.getContext();
>From c1e6b2fd2e626232126a4d3a261636e27a94df5a Mon Sep 17 00:00:00 2001
From: Akash Banerjee <Akash.Banerjee at amd.com>
Date: Tue, 12 Mar 2024 17:21:55 +0000
Subject: [PATCH 13/15] Add support for Complex variable types.
---
llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp | 48 +++++++++++++++++++++--
1 file changed, 45 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 0ab34c30ba1c1d..f538296804b9e9 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -2355,7 +2355,21 @@ void OpenMPIRBuilder::emitReductionListCopy(
break;
}
case EvaluationKindTy::Complex: {
- assert(false && "Complex data type not handled");
+ Value *SrcRealPtr = Builder.CreateConstInBoundsGEP2_32(
+ RI.ElementType, SrcElementAddr, 0, 0, ".realp");
+ Value *SrcReal = Builder.CreateLoad(
+ RI.ElementType->getStructElementType(0), SrcRealPtr, ".real");
+ Value *SrcImgPtr = Builder.CreateConstInBoundsGEP2_32(
+ RI.ElementType, SrcElementAddr, 0, 1, ".imagp");
+ Value *SrcImg = Builder.CreateLoad(
+ RI.ElementType->getStructElementType(1), SrcImgPtr, ".imag");
+
+ Value *DestRealPtr = Builder.CreateConstInBoundsGEP2_32(
+ RI.ElementType, DestElementAddr, 0, 0, ".realp");
+ Value *DestImgPtr = Builder.CreateConstInBoundsGEP2_32(
+ RI.ElementType, DestElementAddr, 0, 1, ".imagp");
+ Builder.CreateStore(SrcReal, DestRealPtr);
+ Builder.CreateStore(SrcImg, DestImgPtr);
break;
}
case EvaluationKindTy::Aggregate: {
@@ -2861,7 +2875,21 @@ Function *OpenMPIRBuilder::emitListToGlobalCopyFunction(
break;
}
case EvaluationKindTy::Complex: {
- assert(false && "Complex data type not handled");
+ Value *SrcRealPtr = Builder.CreateConstInBoundsGEP2_32(
+ RI.ElementType, ElemPtr, 0, 0, ".realp");
+ Value *SrcReal = Builder.CreateLoad(
+ RI.ElementType->getStructElementType(0), SrcRealPtr, ".real");
+ Value *SrcImgPtr = Builder.CreateConstInBoundsGEP2_32(
+ RI.ElementType, ElemPtr, 0, 1, ".imagp");
+ Value *SrcImg = Builder.CreateLoad(
+ RI.ElementType->getStructElementType(1), SrcImgPtr, ".imag");
+
+ Value *DestRealPtr = Builder.CreateConstInBoundsGEP2_32(
+ RI.ElementType, GlobVal, 0, 0, ".realp");
+ Value *DestImgPtr = Builder.CreateConstInBoundsGEP2_32(
+ RI.ElementType, GlobVal, 0, 1, ".imagp");
+ Builder.CreateStore(SrcReal, DestRealPtr);
+ Builder.CreateStore(SrcImg, DestImgPtr);
break;
}
case EvaluationKindTy::Aggregate: {
@@ -3037,7 +3065,21 @@ Function *OpenMPIRBuilder::emitGlobalToListCopyFunction(
break;
}
case EvaluationKindTy::Complex: {
- assert(false && "Complex data type not handled");
+ Value *SrcRealPtr = Builder.CreateConstInBoundsGEP2_32(
+ RI.ElementType, GlobValPtr, 0, 0, ".realp");
+ Value *SrcReal = Builder.CreateLoad(
+ RI.ElementType->getStructElementType(0), SrcRealPtr, ".real");
+ Value *SrcImgPtr = Builder.CreateConstInBoundsGEP2_32(
+ RI.ElementType, GlobValPtr, 0, 1, ".imagp");
+ Value *SrcImg = Builder.CreateLoad(
+ RI.ElementType->getStructElementType(1), SrcImgPtr, ".imag");
+
+ Value *DestRealPtr = Builder.CreateConstInBoundsGEP2_32(
+ RI.ElementType, ElemPtr, 0, 0, ".realp");
+ Value *DestImgPtr = Builder.CreateConstInBoundsGEP2_32(
+ RI.ElementType, ElemPtr, 0, 1, ".imagp");
+ Builder.CreateStore(SrcReal, DestRealPtr);
+ Builder.CreateStore(SrcImg, DestImgPtr);
break;
}
case EvaluationKindTy::Aggregate: {
>From bf3e6b9f934cf265586a644a41ce8788aa3a055f Mon Sep 17 00:00:00 2001
From: Akash Banerjee <Akash.Banerjee at amd.com>
Date: Tue, 19 Mar 2024 19:33:58 +0000
Subject: [PATCH 14/15] Add complex reduction test.
---
clang/test/OpenMP/reduction_complex.c | 1072 +++++++++++++++++++++++++
1 file changed, 1072 insertions(+)
create mode 100644 clang/test/OpenMP/reduction_complex.c
diff --git a/clang/test/OpenMP/reduction_complex.c b/clang/test/OpenMP/reduction_complex.c
new file mode 100644
index 00000000000000..cbc5b14087239d
--- /dev/null
+++ b/clang/test/OpenMP/reduction_complex.c
@@ -0,0 +1,1072 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --include-generated-funcs --replace-value-regex "__omp_offloading_[0-9a-z]+_[0-9a-z]+" "reduction_size[.].+[.]" "pl_cond[.].+[.|,]" --prefix-filecheck-ir-name _
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-cuda-mode -x c++ \
+// RUN: -triple powerpc64le-unknown-unknown \
+// RUN: -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm-bc %s -o \
+// RUN: %t-ppc-host.bc
+
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-cuda-mode -x c++ \
+// RUN: -triple nvptx64-unknown-unknown -DCUA \
+// RUN: -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm %s \
+// RUN: -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc \
+// RUN: -o - | FileCheck %s --check-prefix CHECK
+
+// RUN: %clang_cc1 -verify -fopenmp -x c++ \
+// RUN: -triple powerpc64le-unknown-unknown -DDIAG\
+// RUN: -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm \
+// RUN: %s -o - | FileCheck %s \
+// RUN: --check-prefix=CHECK1
+
+// RUN: %clang_cc1 -verify -fopenmp -x c++ \
+// RUN: -triple i386-unknown-unknown \
+// RUN: -fopenmp-targets=i386-pc-linux-gnu -emit-llvm \
+// RUN: %s -o - | FileCheck %s \
+// RUN: --check-prefix=CHECK2
+
+// expected-no-diagnostics
+int foo() {
+ int i;
+ int j;
+ _Complex float sum = 0;
+
+#pragma omp target teams loop reduction(+:sum) collapse(2) bind(parallel) order(concurrent) lastprivate(j) map(tofrom:sum)
+
+ for(i=0; i<10; i++)
+ for(j=0; j<10; j++)
+ sum += i;
+
+ return 0;
+}
+// CHECK-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3foov_l31
+// CHECK-SAME: (ptr noalias noundef [[DYN_PTR:%.*]], i64 noundef [[J:%.*]], ptr noundef nonnull align 4 dereferenceable(8) [[SUM:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT: entry:
+// CHECK-NEXT: %[[VAL_0:.*]] = alloca ptr, align 8
+// CHECK-NEXT: %[[VAL_1:.*]] = alloca i64, align 8
+// CHECK-NEXT: %[[VAL_2:.*]] = alloca ptr, align 8
+// CHECK-NEXT: %[[VAL_3:.*]] = alloca i64, align 8
+// CHECK-NEXT: %[[VAL_4:.*]] = alloca i32, align 4
+// CHECK-NEXT: %[[VAL_5:.*]] = alloca i32, align 4
+// CHECK-NEXT: store ptr %[[VAL_6:.*]], ptr %[[VAL_0]], align 8
+// CHECK-NEXT: store i64 %[[VAL_7:.*]], ptr %[[VAL_1]], align 8
+// CHECK-NEXT: store ptr %[[VAL_8:.*]], ptr %[[VAL_2]], align 8
+// CHECK-NEXT: %[[VAL_9:.*]] = load ptr, ptr %[[VAL_2]], align 8
+// CHECK-NEXT: %[[VAL_10:.*]] = call i32 @__kmpc_target_init(ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3foov_l31_kernel_environment, ptr %[[VAL_6]])
+// CHECK-NEXT: %[[VAL_11:.*]] = icmp eq i32 %[[VAL_10]], -1
+// CHECK-NEXT: br i1 %[[VAL_11]], label %[[VAL_12:.*]], label %[[VAL_13:.*]]
+// CHECK: user_code.entry: ; preds = %[[VAL_14:.*]]
+// CHECK-NEXT: %[[VAL_15:.*]] = call i32 @__kmpc_global_thread_num(ptr @1)
+// CHECK-NEXT: %[[VAL_16:.*]] = load i32, ptr %[[VAL_1]], align 4
+// CHECK-NEXT: store i32 %[[VAL_16]], ptr %[[VAL_3]], align 4
+// CHECK-NEXT: %[[VAL_17:.*]] = load i64, ptr %[[VAL_3]], align 8
+// CHECK-NEXT: store i32 0, ptr %[[VAL_4]], align 4
+// CHECK-NEXT: store i32 %[[VAL_15]], ptr %[[VAL_5]], align 4
+// CHECK-NEXT: call void @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3foov_l31_omp_outlined(ptr %[[VAL_5]], ptr %[[VAL_4]], i64 %[[VAL_17]], ptr %[[VAL_9]]) #2
+// CHECK-NEXT: call void @__kmpc_target_deinit()
+// CHECK-NEXT: ret void
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3foov_l31_omp_outlined
+// CHECK-SAME: (ptr noalias noundef [[DOTGLOBAL_TID_:%.*]], ptr noalias noundef [[DOTBOUND_TID_:%.*]], i64 noundef [[J:%.*]], ptr noundef nonnull align 4 dereferenceable(8) [[SUM:%.*]]) #[[ATTR1:[0-9]+]] {
+// CHECK-NEXT: entry:
+// CHECK-NEXT: %[[VAL_18:.*]] = alloca ptr, align 8
+// CHECK-NEXT: %[[VAL_19:.*]] = alloca ptr, align 8
+// CHECK-NEXT: %[[VAL_20:.*]] = alloca i64, align 8
+// CHECK-NEXT: %[[VAL_21:.*]] = alloca ptr, align 8
+// CHECK-NEXT: %[[VAL_22:.*]] = alloca { float, float }, align 4
+// CHECK-NEXT: %[[VAL_23:.*]] = alloca i32, align 4
+// CHECK-NEXT: %[[VAL_24:.*]] = alloca i32, align 4
+// CHECK-NEXT: %[[VAL_25:.*]] = alloca i32, align 4
+// CHECK-NEXT: %[[VAL_26:.*]] = alloca i32, align 4
+// CHECK-NEXT: %[[VAL_27:.*]] = alloca i32, align 4
+// CHECK-NEXT: %[[VAL_28:.*]] = alloca i32, align 4
+// CHECK-NEXT: %[[VAL_29:.*]] = alloca i32, align 4
+// CHECK-NEXT: %[[VAL_30:.*]] = alloca i32, align 4
+// CHECK-NEXT: %[[VAL_31:.*]] = alloca i32, align 4
+// CHECK-NEXT: %[[VAL_32:.*]] = alloca i32, align 4
+// CHECK-NEXT: %[[VAL_33:.*]] = alloca i64, align 8
+// CHECK-NEXT: %[[VAL_34:.*]] = alloca [4 x ptr], align 8
+// CHECK-NEXT: %[[VAL_35:.*]] = alloca [1 x ptr], align 8
+// CHECK-NEXT: store ptr %[[VAL_36:.*]], ptr %[[VAL_18]], align 8
+// CHECK-NEXT: store ptr %[[VAL_37:.*]], ptr %[[VAL_19]], align 8
+// CHECK-NEXT: store i64 %[[VAL_38:.*]], ptr %[[VAL_20]], align 8
+// CHECK-NEXT: store ptr %[[VAL_39:.*]], ptr %[[VAL_21]], align 8
+// CHECK-NEXT: %[[VAL_40:.*]] = load ptr, ptr %[[VAL_21]], align 8
+// CHECK-NEXT: %[[VAL_41:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_22]], i32 0, i32 0
+// CHECK-NEXT: %[[VAL_42:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_22]], i32 0, i32 1
+// CHECK-NEXT: store float 0.000000e+00, ptr %[[VAL_41]], align 4
+// CHECK-NEXT: store float 0.000000e+00, ptr %[[VAL_42]], align 4
+// CHECK-NEXT: store i32 0, ptr %[[VAL_26]], align 4
+// CHECK-NEXT: store i32 99, ptr %[[VAL_27]], align 4
+// CHECK-NEXT: store i32 1, ptr %[[VAL_28]], align 4
+// CHECK-NEXT: store i32 0, ptr %[[VAL_29]], align 4
+// CHECK-NEXT: %[[VAL_43:.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
+// CHECK-NEXT: %[[VAL_44:.*]] = load ptr, ptr %[[VAL_18]], align 8
+// CHECK-NEXT: %[[VAL_45:.*]] = load i32, ptr %[[VAL_44]], align 4
+// CHECK-NEXT: call void @__kmpc_distribute_static_init_4(ptr @2, i32 %[[VAL_45]], i32 91, ptr %[[VAL_29]], ptr %[[VAL_26]], ptr %[[VAL_27]], ptr %[[VAL_28]], i32 1, i32 %[[VAL_43]])
+// CHECK-NEXT: %[[VAL_46:.*]] = load i32, ptr %[[VAL_27]], align 4
+// CHECK-NEXT: %[[VAL_47:.*]] = icmp sgt i32 %[[VAL_46]], 99
+// CHECK-NEXT: br i1 %[[VAL_47]], label %[[VAL_48:.*]], label %[[VAL_49:.*]]
+// CHECK: cond.true: ; preds = %[[VAL_50:.*]]
+// CHECK-NEXT: br label %[[VAL_51:.*]]
+// CHECK: cond.false: ; preds = %[[VAL_50]]
+// CHECK-NEXT: %[[VAL_52:.*]] = load i32, ptr %[[VAL_27]], align 4
+// CHECK-NEXT: br label %[[VAL_51]]
+// CHECK: cond.end: ; preds = %[[VAL_49]], %[[VAL_48]]
+// CHECK-NEXT: %[[VAL_53:.*]] = phi i32 [ 99, %[[VAL_48]] ], [ %[[VAL_52]], %[[VAL_49]] ]
+// CHECK-NEXT: store i32 %[[VAL_53]], ptr %[[VAL_27]], align 4
+// CHECK-NEXT: %[[VAL_54:.*]] = load i32, ptr %[[VAL_26]], align 4
+// CHECK-NEXT: store i32 %[[VAL_54]], ptr %[[VAL_23]], align 4
+// CHECK-NEXT: br label %[[VAL_55:.*]]
+// CHECK: omp.inner.for.cond: ; preds = %[[VAL_56:.*]], %[[VAL_51]]
+// CHECK-NEXT: %[[VAL_57:.*]] = load i32, ptr %[[VAL_23]], align 4
+// CHECK-NEXT: %[[VAL_58:.*]] = icmp slt i32 %[[VAL_57]], 100
+// CHECK-NEXT: br i1 %[[VAL_58]], label %[[VAL_59:.*]], label %[[VAL_60:.*]]
+// CHECK: omp.inner.for.body: ; preds = %[[VAL_55]]
+// CHECK-NEXT: %[[VAL_61:.*]] = load i32, ptr %[[VAL_26]], align 4
+// CHECK-NEXT: %[[VAL_62:.*]] = zext i32 %[[VAL_61]] to i64
+// CHECK-NEXT: %[[VAL_63:.*]] = load i32, ptr %[[VAL_27]], align 4
+// CHECK-NEXT: %[[VAL_64:.*]] = zext i32 %[[VAL_63]] to i64
+// CHECK-NEXT: %[[VAL_65:.*]] = load i32, ptr %[[VAL_30]], align 4
+// CHECK-NEXT: store i32 %[[VAL_65]], ptr %[[VAL_33]], align 4
+// CHECK-NEXT: %[[VAL_66:.*]] = load i64, ptr %[[VAL_33]], align 8
+// CHECK-NEXT: %[[VAL_67:.*]] = getelementptr inbounds [4 x ptr], ptr %[[VAL_34]], i64 0, i64 0
+// CHECK-NEXT: %[[VAL_68:.*]] = inttoptr i64 %[[VAL_62]] to ptr
+// CHECK-NEXT: store ptr %[[VAL_68]], ptr %[[VAL_67]], align 8
+// CHECK-NEXT: %[[VAL_69:.*]] = getelementptr inbounds [4 x ptr], ptr %[[VAL_34]], i64 0, i64 1
+// CHECK-NEXT: %[[VAL_70:.*]] = inttoptr i64 %[[VAL_64]] to ptr
+// CHECK-NEXT: store ptr %[[VAL_70]], ptr %[[VAL_69]], align 8
+// CHECK-NEXT: %[[VAL_71:.*]] = getelementptr inbounds [4 x ptr], ptr %[[VAL_34]], i64 0, i64 2
+// CHECK-NEXT: %[[VAL_72:.*]] = inttoptr i64 %[[VAL_66]] to ptr
+// CHECK-NEXT: store ptr %[[VAL_72]], ptr %[[VAL_71]], align 8
+// CHECK-NEXT: %[[VAL_73:.*]] = getelementptr inbounds [4 x ptr], ptr %[[VAL_34]], i64 0, i64 3
+// CHECK-NEXT: store ptr %[[VAL_22]], ptr %[[VAL_73]], align 8
+// CHECK-NEXT: call void @__kmpc_parallel_51(ptr @1, i32 %[[VAL_45]], i32 1, i32 -1, i32 -1, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3foov_l31_omp_outlined_omp_outlined, ptr null, ptr %[[VAL_34]], i64 4)
+// CHECK-NEXT: br label %[[VAL_74:.*]]
+// CHECK: omp.inner.for.inc: ; preds = %[[VAL_59]]
+// CHECK-NEXT: %[[VAL_75:.*]] = load i32, ptr %[[VAL_23]], align 4
+// CHECK-NEXT: %[[VAL_76:.*]] = load i32, ptr %[[VAL_28]], align 4
+// CHECK-NEXT: %[[VAL_77:.*]] = add nsw i32 %[[VAL_75]], %[[VAL_76]]
+// CHECK-NEXT: store i32 %[[VAL_77]], ptr %[[VAL_23]], align 4
+// CHECK-NEXT: %[[VAL_78:.*]] = load i32, ptr %[[VAL_26]], align 4
+// CHECK-NEXT: %[[VAL_79:.*]] = load i32, ptr %[[VAL_28]], align 4
+// CHECK-NEXT: %[[VAL_80:.*]] = add nsw i32 %[[VAL_78]], %[[VAL_79]]
+// CHECK-NEXT: store i32 %[[VAL_80]], ptr %[[VAL_26]], align 4
+// CHECK-NEXT: %[[VAL_81:.*]] = load i32, ptr %[[VAL_27]], align 4
+// CHECK-NEXT: %[[VAL_82:.*]] = load i32, ptr %[[VAL_28]], align 4
+// CHECK-NEXT: %[[VAL_83:.*]] = add nsw i32 %[[VAL_81]], %[[VAL_82]]
+// CHECK-NEXT: store i32 %[[VAL_83]], ptr %[[VAL_27]], align 4
+// CHECK-NEXT: %[[VAL_84:.*]] = load i32, ptr %[[VAL_27]], align 4
+// CHECK-NEXT: %[[VAL_85:.*]] = icmp sgt i32 %[[VAL_84]], 99
+// CHECK-NEXT: br i1 %[[VAL_85]], label %[[VAL_86:.*]], label %[[VAL_87:.*]]
+// CHECK: cond.true9: ; preds = %[[VAL_74]]
+// CHECK-NEXT: br label %[[VAL_56]]
+// CHECK: cond.false10: ; preds = %[[VAL_74]]
+// CHECK-NEXT: %[[VAL_88:.*]] = load i32, ptr %[[VAL_27]], align 4
+// CHECK-NEXT: br label %[[VAL_56]]
+// CHECK: cond.end11: ; preds = %[[VAL_87]], %[[VAL_86]]
+// CHECK-NEXT: %[[VAL_89:.*]] = phi i32 [ 99, %[[VAL_86]] ], [ %[[VAL_88]], %[[VAL_87]] ]
+// CHECK-NEXT: store i32 %[[VAL_89]], ptr %[[VAL_27]], align 4
+// CHECK-NEXT: %[[VAL_90:.*]] = load i32, ptr %[[VAL_26]], align 4
+// CHECK-NEXT: store i32 %[[VAL_90]], ptr %[[VAL_23]], align 4
+// CHECK-NEXT: br label %[[VAL_55]]
+// CHECK: omp.inner.for.end: ; preds = %[[VAL_55]]
+// CHECK-NEXT: br label %[[VAL_91:.*]]
+// CHECK: omp.loop.exit: ; preds = %[[VAL_60]]
+// CHECK-NEXT: call void @__kmpc_for_static_fini(ptr @3, i32 %[[VAL_45]])
+// CHECK-NEXT: %[[VAL_92:.*]] = load i32, ptr %[[VAL_29]], align 4
+// CHECK-NEXT: %[[VAL_93:.*]] = icmp ne i32 %[[VAL_92]], 0
+// CHECK-NEXT: br i1 %[[VAL_93]], label %[[VAL_94:.*]], label %[[VAL_95:.*]]
+// CHECK: .omp.lastprivate.then: ; preds = %[[VAL_91]]
+// CHECK-NEXT: store i32 10, ptr %[[VAL_30]], align 4
+// CHECK-NEXT: %[[VAL_96:.*]] = load i32, ptr %[[VAL_30]], align 4
+// CHECK-NEXT: store i32 %[[VAL_96]], ptr %[[VAL_20]], align 4
+// CHECK-NEXT: br label %[[VAL_95]]
+// CHECK: .omp.lastprivate.done: ; preds = %[[VAL_94]], %[[VAL_91]]
+// CHECK-NEXT: %[[VAL_97:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_35]], i64 0, i64 0
+// CHECK-NEXT: store ptr %[[VAL_22]], ptr %[[VAL_97]], align 8
+// CHECK-NEXT: %[[VAL_98:.*]]"_openmp_teams_reductions_buffer_$_$ptr" = call ptr @__kmpc_reduction_get_fixed_buffer()
+// CHECK-NEXT: %[[VAL_99:.*]] = call i32 @__kmpc_nvptx_teams_reduce_nowait_v2(ptr @1, ptr %[[VAL_98]]"_openmp_teams_reductions_buffer_$_$ptr", i32 1024, i64 8, ptr %[[VAL_35]], ptr @_omp_reduction_shuffle_and_reduce_func1, ptr @_omp_reduction_inter_warp_copy_func2, ptr @_omp_reduction_list_to_global_copy_func, ptr @_omp_reduction_list_to_global_reduce_func, ptr @_omp_reduction_global_to_list_copy_func, ptr @_omp_reduction_global_to_list_reduce_func)
+// CHECK-NEXT: %[[VAL_100:.*]] = icmp eq i32 %[[VAL_99]], 1
+// CHECK-NEXT: br i1 %[[VAL_100]], label %[[VAL_101:.*]], label %[[VAL_102:.*]]
+// CHECK: .omp.reduction.then: ; preds = %[[VAL_95]]
+// CHECK-NEXT: %[[VAL_103:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_40]], i32 0, i32 0
+// CHECK-NEXT: %[[VAL_104:.*]] = load float, ptr %[[VAL_103]], align 4
+// CHECK-NEXT: %[[VAL_105:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_40]], i32 0, i32 1
+// CHECK-NEXT: %[[VAL_106:.*]] = load float, ptr %[[VAL_105]], align 4
+// CHECK-NEXT: %[[VAL_107:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_22]], i32 0, i32 0
+// CHECK-NEXT: %[[VAL_108:.*]] = load float, ptr %[[VAL_107]], align 4
+// CHECK-NEXT: %[[VAL_109:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_22]], i32 0, i32 1
+// CHECK-NEXT: %[[VAL_110:.*]] = load float, ptr %[[VAL_109]], align 4
+// CHECK-NEXT: %[[VAL_111:.*]] = fadd float %[[VAL_104]], %[[VAL_108]]
+// CHECK-NEXT: %[[VAL_112:.*]] = fadd float %[[VAL_106]], %[[VAL_110]]
+// CHECK-NEXT: %[[VAL_113:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_40]], i32 0, i32 0
+// CHECK-NEXT: %[[VAL_114:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_40]], i32 0, i32 1
+// CHECK-NEXT: store float %[[VAL_111]], ptr %[[VAL_113]], align 4
+// CHECK-NEXT: store float %[[VAL_112]], ptr %[[VAL_114]], align 4
+// CHECK-NEXT: br label %[[VAL_102]]
+// CHECK: .omp.reduction.done: ; preds = %[[VAL_101]], %[[VAL_95]]
+// CHECK-NEXT: ret void
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@_omp_reduction_shuffle_and_reduce_func
+// CHECK-SAME: (ptr noundef [[TMP0:%.*]], i16 noundef signext [[TMP1:%.*]], i16 noundef signext [[TMP2:%.*]], i16 noundef signext [[TMP3:%.*]]) #[[ATTR2:[0-9]+]] {
+// CHECK-NEXT: entry:
+// CHECK-NEXT: %[[VAL_228:.*]] = alloca ptr, align 8
+// CHECK-NEXT: %[[VAL_229:.*]] = alloca i16, align 2
+// CHECK-NEXT: %[[VAL_230:.*]] = alloca i16, align 2
+// CHECK-NEXT: %[[VAL_231:.*]] = alloca i16, align 2
+// CHECK-NEXT: %[[VAL_232:.*]] = alloca [1 x ptr], align 8
+// CHECK-NEXT: %[[VAL_233:.*]] = alloca { float, float }, align 8
+// CHECK-NEXT: store ptr %[[VAL_234:.*]], ptr %[[VAL_228]], align 8
+// CHECK-NEXT: store i16 %[[VAL_235:.*]], ptr %[[VAL_229]], align 2
+// CHECK-NEXT: store i16 %[[VAL_236:.*]], ptr %[[VAL_230]], align 2
+// CHECK-NEXT: store i16 %[[VAL_237:.*]], ptr %[[VAL_231]], align 2
+// CHECK-NEXT: %[[VAL_238:.*]] = load ptr, ptr %[[VAL_228]], align 8
+// CHECK-NEXT: %[[VAL_239:.*]] = load i16, ptr %[[VAL_229]], align 2
+// CHECK-NEXT: %[[VAL_240:.*]] = load i16, ptr %[[VAL_230]], align 2
+// CHECK-NEXT: %[[VAL_241:.*]] = load i16, ptr %[[VAL_231]], align 2
+// CHECK-NEXT: %[[VAL_242:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_238]], i64 0, i64 0
+// CHECK-NEXT: %[[VAL_243:.*]] = load ptr, ptr %[[VAL_242]], align 8
+// CHECK-NEXT: %[[VAL_244:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_232]], i64 0, i64 0
+// CHECK-NEXT: %[[VAL_245:.*]] = getelementptr { float, float }, ptr %[[VAL_243]], i64 1
+// CHECK-NEXT: %[[VAL_246:.*]] = load i64, ptr %[[VAL_243]], align 8
+// CHECK-NEXT: %[[VAL_247:.*]] = call i32 @__kmpc_get_warp_size()
+// CHECK-NEXT: %[[VAL_248:.*]] = trunc i32 %[[VAL_247]] to i16
+// CHECK-NEXT: %[[VAL_249:.*]] = call i64 @__kmpc_shuffle_int64(i64 %[[VAL_246]], i16 %[[VAL_240]], i16 %[[VAL_248]])
+// CHECK-NEXT: store i64 %[[VAL_249]], ptr %[[VAL_233]], align 8
+// CHECK-NEXT: %[[VAL_250:.*]] = getelementptr i64, ptr %[[VAL_243]], i64 1
+// CHECK-NEXT: %[[VAL_251:.*]] = getelementptr i64, ptr %[[VAL_233]], i64 1
+// CHECK-NEXT: store ptr %[[VAL_233]], ptr %[[VAL_244]], align 8
+// CHECK-NEXT: %[[VAL_252:.*]] = icmp eq i16 %[[VAL_241]], 0
+// CHECK-NEXT: %[[VAL_253:.*]] = icmp eq i16 %[[VAL_241]], 1
+// CHECK-NEXT: %[[VAL_254:.*]] = icmp ult i16 %[[VAL_239]], %[[VAL_240]]
+// CHECK-NEXT: %[[VAL_255:.*]] = and i1 %[[VAL_253]], %[[VAL_254]]
+// CHECK-NEXT: %[[VAL_256:.*]] = icmp eq i16 %[[VAL_241]], 2
+// CHECK-NEXT: %[[VAL_257:.*]] = and i16 %[[VAL_239]], 1
+// CHECK-NEXT: %[[VAL_258:.*]] = icmp eq i16 %[[VAL_257]], 0
+// CHECK-NEXT: %[[VAL_259:.*]] = and i1 %[[VAL_256]], %[[VAL_258]]
+// CHECK-NEXT: %[[VAL_260:.*]] = icmp sgt i16 %[[VAL_240]], 0
+// CHECK-NEXT: %[[VAL_261:.*]] = and i1 %[[VAL_259]], %[[VAL_260]]
+// CHECK-NEXT: %[[VAL_262:.*]] = or i1 %[[VAL_252]], %[[VAL_255]]
+// CHECK-NEXT: %[[VAL_263:.*]] = or i1 %[[VAL_262]], %[[VAL_261]]
+// CHECK-NEXT: br i1 %[[VAL_263]], label %[[VAL_264:.*]], label %[[VAL_265:.*]]
+// CHECK: then: ; preds = %[[VAL_266:.*]]
+// CHECK-NEXT: call void @"{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3foov_l31_omp_outlined_omp_outlined_omp$reduction$reduction_func"(ptr %[[VAL_238]], ptr %[[VAL_232]]) #2
+// CHECK-NEXT: br label %[[VAL_267:.*]]
+// CHECK: else: ; preds = %[[VAL_266]]
+// CHECK-NEXT: br label %[[VAL_267]]
+// CHECK: ifcont: ; preds = %[[VAL_265]], %[[VAL_264]]
+// CHECK-NEXT: %[[VAL_268:.*]] = icmp eq i16 %[[VAL_241]], 1
+// CHECK-NEXT: %[[VAL_269:.*]] = icmp uge i16 %[[VAL_239]], %[[VAL_240]]
+// CHECK-NEXT: %[[VAL_270:.*]] = and i1 %[[VAL_268]], %[[VAL_269]]
+// CHECK-NEXT: br i1 %[[VAL_270]], label %[[VAL_271:.*]], label %[[VAL_272:.*]]
+// CHECK: then4: ; preds = %[[VAL_267]]
+// CHECK-NEXT: %[[VAL_273:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_232]], i64 0, i64 0
+// CHECK-NEXT: %[[VAL_274:.*]] = load ptr, ptr %[[VAL_273]], align 8
+// CHECK-NEXT: %[[VAL_275:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_238]], i64 0, i64 0
+// CHECK-NEXT: %[[VAL_276:.*]] = load ptr, ptr %[[VAL_275]], align 8
+// CHECK-NEXT: %[[VAL_277:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_274]], i32 0, i32 0
+// CHECK-NEXT: %[[VAL_278:.*]] = load float, ptr %[[VAL_277]], align 4
+// CHECK-NEXT: %[[VAL_279:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_274]], i32 0, i32 1
+// CHECK-NEXT: %[[VAL_280:.*]] = load float, ptr %[[VAL_279]], align 4
+// CHECK-NEXT: %[[VAL_281:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_276]], i32 0, i32 0
+// CHECK-NEXT: %[[VAL_282:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_276]], i32 0, i32 1
+// CHECK-NEXT: store float %[[VAL_278]], ptr %[[VAL_281]], align 4
+// CHECK-NEXT: store float %[[VAL_280]], ptr %[[VAL_282]], align 4
+// CHECK-NEXT: br label %[[VAL_283:.*]]
+// CHECK: else7: ; preds = %[[VAL_267]]
+// CHECK-NEXT: br label %[[VAL_283]]
+// CHECK: ifcont8: ; preds = %[[VAL_272]], %[[VAL_271]]
+// CHECK-NEXT: ret void
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@_omp_reduction_inter_warp_copy_func
+// CHECK-SAME: (ptr noundef [[TMP0:%.*]], i32 noundef [[TMP1:%.*]]) #[[ATTR2]] {
+// CHECK-NEXT: entry:
+// CHECK-NEXT: %[[VAL_400:.*]] = alloca ptr, align 8
+// CHECK-NEXT: %[[VAL_401:.*]] = alloca i32, align 4
+// CHECK-NEXT: %[[VAL_402:.*]] = alloca i32, align 4
+// CHECK-NEXT: %[[VAL_403:.*]] = call i32 @__kmpc_global_thread_num(ptr @1)
+// CHECK-NEXT: store ptr %[[VAL_404:.*]], ptr %[[VAL_400]], align 8
+// CHECK-NEXT: store i32 %[[VAL_405:.*]], ptr %[[VAL_401]], align 4
+// CHECK-NEXT: %[[VAL_406:.*]] = call i32 @__kmpc_get_hardware_thread_id_in_block()
+// CHECK-NEXT: %[[VAL_407:.*]] = call i32 @__kmpc_get_hardware_thread_id_in_block()
+// CHECK-NEXT: %[[VAL_408:.*]] = and i32 %[[VAL_407]], 31
+// CHECK-NEXT: %[[VAL_409:.*]] = call i32 @__kmpc_get_hardware_thread_id_in_block()
+// CHECK-NEXT: %[[VAL_410:.*]] = ashr i32 %[[VAL_409]], 5
+// CHECK-NEXT: %[[VAL_411:.*]] = load ptr, ptr %[[VAL_400]], align 8
+// CHECK-NEXT: store i32 0, ptr %[[VAL_402]], align 4
+// CHECK-NEXT: br label %[[VAL_412:.*]]
+// CHECK: precond: ; preds = %[[VAL_413:.*]], %[[VAL_414:.*]]
+// CHECK-NEXT: %[[VAL_415:.*]] = load i32, ptr %[[VAL_402]], align 4
+// CHECK-NEXT: %[[VAL_416:.*]] = icmp ult i32 %[[VAL_415]], 2
+// CHECK-NEXT: br i1 %[[VAL_416]], label %[[VAL_417:.*]], label %[[VAL_418:.*]]
+// CHECK: body: ; preds = %[[VAL_412]]
+// CHECK-NEXT: call void @__kmpc_barrier(ptr @4, i32 %[[VAL_403]])
+// CHECK-NEXT: %[[VAL_419:.*]] = icmp eq i32 %[[VAL_408]], 0
+// CHECK-NEXT: br i1 %[[VAL_419]], label %[[VAL_420:.*]], label %[[VAL_421:.*]]
+// CHECK: then: ; preds = %[[VAL_417]]
+// CHECK-NEXT: %[[VAL_422:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_411]], i64 0, i64 0
+// CHECK-NEXT: %[[VAL_423:.*]] = load ptr, ptr %[[VAL_422]], align 8
+// CHECK-NEXT: %[[VAL_424:.*]] = getelementptr i32, ptr %[[VAL_423]], i32 %[[VAL_415]]
+// CHECK-NEXT: %[[VAL_425:.*]] = getelementptr inbounds [32 x i32], ptr addrspace(3) @__openmp_nvptx_data_transfer_temporary_storage, i64 0, i32 %[[VAL_410]]
+// CHECK-NEXT: %[[VAL_426:.*]] = load i32, ptr %[[VAL_424]], align 4
+// CHECK-NEXT: store volatile i32 %[[VAL_426]], ptr addrspace(3) %[[VAL_425]], align 4
+// CHECK-NEXT: br label %[[VAL_427:.*]]
+// CHECK: else: ; preds = %[[VAL_417]]
+// CHECK-NEXT: br label %[[VAL_427]]
+// CHECK: ifcont: ; preds = %[[VAL_421]], %[[VAL_420]]
+// CHECK-NEXT: call void @__kmpc_barrier(ptr @4, i32 %[[VAL_403]])
+// CHECK-NEXT: %[[VAL_428:.*]] = load i32, ptr %[[VAL_401]], align 4
+// CHECK-NEXT: %[[VAL_429:.*]] = icmp ult i32 %[[VAL_406]], %[[VAL_428]]
+// CHECK-NEXT: br i1 %[[VAL_429]], label %[[VAL_430:.*]], label %[[VAL_431:.*]]
+// CHECK: then2: ; preds = %[[VAL_427]]
+// CHECK-NEXT: %[[VAL_432:.*]] = getelementptr inbounds [32 x i32], ptr addrspace(3) @__openmp_nvptx_data_transfer_temporary_storage, i64 0, i32 %[[VAL_406]]
+// CHECK-NEXT: %[[VAL_433:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_411]], i64 0, i64 0
+// CHECK-NEXT: %[[VAL_434:.*]] = load ptr, ptr %[[VAL_433]], align 8
+// CHECK-NEXT: %[[VAL_435:.*]] = getelementptr i32, ptr %[[VAL_434]], i32 %[[VAL_415]]
+// CHECK-NEXT: %[[VAL_436:.*]] = load volatile i32, ptr addrspace(3) %[[VAL_432]], align 4
+// CHECK-NEXT: store i32 %[[VAL_436]], ptr %[[VAL_435]], align 4
+// CHECK-NEXT: br label %[[VAL_413]]
+// CHECK: else3: ; preds = %[[VAL_427]]
+// CHECK-NEXT: br label %[[VAL_413]]
+// CHECK: ifcont4: ; preds = %[[VAL_431]], %[[VAL_430]]
+// CHECK-NEXT: %[[VAL_437:.*]] = add nsw i32 %[[VAL_415]], 1
+// CHECK-NEXT: store i32 %[[VAL_437]], ptr %[[VAL_402]], align 4
+// CHECK-NEXT: br label %[[VAL_412]]
+// CHECK: exit: ; preds = %[[VAL_412]]
+// CHECK-NEXT: ret void
+//
+//
+// CHECK1-LABEL: define {{[^@]+}}@_Z3foov
+// CHECK1-SAME: () #[[ATTR0:[0-9]+]] {
+// CHECK1-NEXT: entry:
+// CHECK1-NEXT: %[[VAL_0:.*]] = alloca i32, align 4
+// CHECK1-NEXT: %[[VAL_1:.*]] = alloca i32, align 4
+// CHECK1-NEXT: %[[VAL_2:.*]] = alloca { float, float }, align 4
+// CHECK1-NEXT: %[[VAL_3:.*]] = alloca i64, align 8
+// CHECK1-NEXT: %[[VAL_4:.*]] = alloca [2 x ptr], align 8
+// CHECK1-NEXT: %[[VAL_5:.*]] = alloca [2 x ptr], align 8
+// CHECK1-NEXT: %[[VAL_6:.*]] = alloca [2 x ptr], align 8
+// CHECK1-NEXT: %[[VAL_7:.*]] = alloca i32, align 4
+// CHECK1-NEXT: %[[VAL_8:.*]] = alloca i32, align 4
+// CHECK1-NEXT: %[[VAL_9:.*]] = alloca %[[VAL_10:.*]], align 8
+// CHECK1-NEXT: %[[VAL_11:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_2]], i32 0, i32 0
+// CHECK1-NEXT: %[[VAL_12:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_2]], i32 0, i32 1
+// CHECK1-NEXT: store float 0.000000e+00, ptr %[[VAL_11]], align 4
+// CHECK1-NEXT: store float 0.000000e+00, ptr %[[VAL_12]], align 4
+// CHECK1-NEXT: %[[VAL_13:.*]] = load i32, ptr %[[VAL_1]], align 4
+// CHECK1-NEXT: store i32 %[[VAL_13]], ptr %[[VAL_3]], align 4
+// CHECK1-NEXT: %[[VAL_14:.*]] = load i64, ptr %[[VAL_3]], align 8
+// CHECK1-NEXT: %[[VAL_15:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_4]], i32 0, i32 0
+// CHECK1-NEXT: store i64 %[[VAL_14]], ptr %[[VAL_15]], align 8
+// CHECK1-NEXT: %[[VAL_16:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_5]], i32 0, i32 0
+// CHECK1-NEXT: store i64 %[[VAL_14]], ptr %[[VAL_16]], align 8
+// CHECK1-NEXT: %[[VAL_17:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_6]], i64 0, i64 0
+// CHECK1-NEXT: store ptr null, ptr %[[VAL_17]], align 8
+// CHECK1-NEXT: %[[VAL_18:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_4]], i32 0, i32 1
+// CHECK1-NEXT: store ptr %[[VAL_2]], ptr %[[VAL_18]], align 8
+// CHECK1-NEXT: %[[VAL_19:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_5]], i32 0, i32 1
+// CHECK1-NEXT: store ptr %[[VAL_2]], ptr %[[VAL_19]], align 8
+// CHECK1-NEXT: %[[VAL_20:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_6]], i64 0, i64 1
+// CHECK1-NEXT: store ptr null, ptr %[[VAL_20]], align 8
+// CHECK1-NEXT: %[[VAL_21:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_4]], i32 0, i32 0
+// CHECK1-NEXT: %[[VAL_22:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_5]], i32 0, i32 0
+// CHECK1-NEXT: %[[VAL_23:.*]] = getelementptr inbounds %[[VAL_10]], ptr %[[VAL_9]], i32 0, i32 0
+// CHECK1-NEXT: store i32 2, ptr %[[VAL_23]], align 4
+// CHECK1-NEXT: %[[VAL_24:.*]] = getelementptr inbounds %[[VAL_10]], ptr %[[VAL_9]], i32 0, i32 1
+// CHECK1-NEXT: store i32 2, ptr %[[VAL_24]], align 4
+// CHECK1-NEXT: %[[VAL_25:.*]] = getelementptr inbounds %[[VAL_10]], ptr %[[VAL_9]], i32 0, i32 2
+// CHECK1-NEXT: store ptr %[[VAL_21]], ptr %[[VAL_25]], align 8
+// CHECK1-NEXT: %[[VAL_26:.*]] = getelementptr inbounds %[[VAL_10]], ptr %[[VAL_9]], i32 0, i32 3
+// CHECK1-NEXT: store ptr %[[VAL_22]], ptr %[[VAL_26]], align 8
+// CHECK1-NEXT: %[[VAL_27:.*]] = getelementptr inbounds %[[VAL_10]], ptr %[[VAL_9]], i32 0, i32 4
+// CHECK1-NEXT: store ptr @.offload_sizes, ptr %[[VAL_27]], align 8
+// CHECK1-NEXT: %[[VAL_28:.*]] = getelementptr inbounds %[[VAL_10]], ptr %[[VAL_9]], i32 0, i32 5
+// CHECK1-NEXT: store ptr @.offload_maptypes, ptr %[[VAL_28]], align 8
+// CHECK1-NEXT: %[[VAL_29:.*]] = getelementptr inbounds %[[VAL_10]], ptr %[[VAL_9]], i32 0, i32 6
+// CHECK1-NEXT: store ptr null, ptr %[[VAL_29]], align 8
+// CHECK1-NEXT: %[[VAL_30:.*]] = getelementptr inbounds %[[VAL_10]], ptr %[[VAL_9]], i32 0, i32 7
+// CHECK1-NEXT: store ptr null, ptr %[[VAL_30]], align 8
+// CHECK1-NEXT: %[[VAL_31:.*]] = getelementptr inbounds %[[VAL_10]], ptr %[[VAL_9]], i32 0, i32 8
+// CHECK1-NEXT: store i64 100, ptr %[[VAL_31]], align 8
+// CHECK1-NEXT: %[[VAL_32:.*]] = getelementptr inbounds %[[VAL_10]], ptr %[[VAL_9]], i32 0, i32 9
+// CHECK1-NEXT: store i64 0, ptr %[[VAL_32]], align 8
+// CHECK1-NEXT: %[[VAL_33:.*]] = getelementptr inbounds %[[VAL_10]], ptr %[[VAL_9]], i32 0, i32 10
+// CHECK1-NEXT: store [3 x i32] zeroinitializer, ptr %[[VAL_33]], align 4
+// CHECK1-NEXT: %[[VAL_34:.*]] = getelementptr inbounds %[[VAL_10]], ptr %[[VAL_9]], i32 0, i32 11
+// CHECK1-NEXT: store [3 x i32] zeroinitializer, ptr %[[VAL_34]], align 4
+// CHECK1-NEXT: %[[VAL_35:.*]] = getelementptr inbounds %[[VAL_10]], ptr %[[VAL_9]], i32 0, i32 12
+// CHECK1-NEXT: store i32 0, ptr %[[VAL_35]], align 4
+// CHECK1-NEXT: %[[VAL_36:.*]] = call i32 @__tgt_target_kernel(ptr @4, i64 -1, i32 0, i32 0, ptr @.{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3foov_l31.region_id, ptr %[[VAL_9]])
+// CHECK1-NEXT: %[[VAL_37:.*]] = icmp ne i32 %[[VAL_36]], 0
+// CHECK1-NEXT: br i1 %[[VAL_37]], label %[[VAL_38:.*]], label %[[VAL_39:.*]]
+// CHECK1: omp_offload.failed: ; preds = %[[VAL_40:.*]]
+// CHECK1-NEXT: call void @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3foov_l31(i64 %[[VAL_14]], ptr %[[VAL_2]]) #2
+// CHECK1-NEXT: br label %[[VAL_39]]
+// CHECK1: omp_offload.cont: ; preds = %[[VAL_38]], %[[VAL_40]]
+// CHECK1-NEXT: ret i32 0
+//
+//
+// CHECK1-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3foov_l31
+// CHECK1-SAME: (i64 noundef [[J:%.*]], ptr noundef nonnull align 4 dereferenceable(8) [[SUM:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK1-NEXT: entry:
+// CHECK1-NEXT: %[[VAL_41:.*]] = alloca i64, align 8
+// CHECK1-NEXT: %[[VAL_42:.*]] = alloca ptr, align 8
+// CHECK1-NEXT: %[[VAL_43:.*]] = alloca i64, align 8
+// CHECK1-NEXT: store i64 %[[VAL_44:.*]], ptr %[[VAL_41]], align 8
+// CHECK1-NEXT: store ptr %[[VAL_45:.*]], ptr %[[VAL_42]], align 8
+// CHECK1-NEXT: %[[VAL_46:.*]] = load ptr, ptr %[[VAL_42]], align 8
+// CHECK1-NEXT: %[[VAL_47:.*]] = load i32, ptr %[[VAL_41]], align 4
+// CHECK1-NEXT: store i32 %[[VAL_47]], ptr %[[VAL_43]], align 4
+// CHECK1-NEXT: %[[VAL_48:.*]] = load i64, ptr %[[VAL_43]], align 8
+// CHECK1-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @4, i32 2, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3foov_l31.omp_outlined, i64 %[[VAL_48]], ptr %[[VAL_46]])
+// CHECK1-NEXT: ret void
+//
+//
+// CHECK1-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3foov_l31.omp_outlined
+// CHECK1-SAME: (ptr noalias noundef [[DOTGLOBAL_TID_:%.*]], ptr noalias noundef [[DOTBOUND_TID_:%.*]], i64 noundef [[J:%.*]], ptr noundef nonnull align 4 dereferenceable(8) [[SUM:%.*]]) #[[ATTR1:[0-9]+]] {
+// CHECK1-NEXT: entry:
+// CHECK1-NEXT: %[[VAL_49:.*]] = alloca ptr, align 8
+// CHECK1-NEXT: %[[VAL_50:.*]] = alloca ptr, align 8
+// CHECK1-NEXT: %[[VAL_51:.*]] = alloca i64, align 8
+// CHECK1-NEXT: %[[VAL_52:.*]] = alloca ptr, align 8
+// CHECK1-NEXT: %[[VAL_53:.*]] = alloca { float, float }, align 4
+// CHECK1-NEXT: %[[VAL_54:.*]] = alloca i32, align 4
+// CHECK1-NEXT: %[[VAL_55:.*]] = alloca i32, align 4
+// CHECK1-NEXT: %[[VAL_56:.*]] = alloca i32, align 4
+// CHECK1-NEXT: %[[VAL_57:.*]] = alloca i32, align 4
+// CHECK1-NEXT: %[[VAL_58:.*]] = alloca i32, align 4
+// CHECK1-NEXT: %[[VAL_59:.*]] = alloca i32, align 4
+// CHECK1-NEXT: %[[VAL_60:.*]] = alloca i32, align 4
+// CHECK1-NEXT: %[[VAL_61:.*]] = alloca i32, align 4
+// CHECK1-NEXT: %[[VAL_62:.*]] = alloca i32, align 4
+// CHECK1-NEXT: %[[VAL_63:.*]] = alloca i32, align 4
+// CHECK1-NEXT: %[[VAL_64:.*]] = alloca i64, align 8
+// CHECK1-NEXT: %[[VAL_65:.*]] = alloca [1 x ptr], align 8
+// CHECK1-NEXT: %[[VAL_66:.*]] = alloca { float, float }, align 4
+// CHECK1-NEXT: %[[VAL_67:.*]] = alloca { float, float }, align 4
+// CHECK1-NEXT: %[[VAL_68:.*]] = alloca { float, float }, align 4
+// CHECK1-NEXT: store ptr %[[VAL_69:.*]], ptr %[[VAL_49]], align 8
+// CHECK1-NEXT: store ptr %[[VAL_70:.*]], ptr %[[VAL_50]], align 8
+// CHECK1-NEXT: store i64 %[[VAL_71:.*]], ptr %[[VAL_51]], align 8
+// CHECK1-NEXT: store ptr %[[VAL_72:.*]], ptr %[[VAL_52]], align 8
+// CHECK1-NEXT: %[[VAL_73:.*]] = load ptr, ptr %[[VAL_52]], align 8
+// CHECK1-NEXT: %[[VAL_74:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_53]], i32 0, i32 0
+// CHECK1-NEXT: %[[VAL_75:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_53]], i32 0, i32 1
+// CHECK1-NEXT: store float 0.000000e+00, ptr %[[VAL_74]], align 4
+// CHECK1-NEXT: store float 0.000000e+00, ptr %[[VAL_75]], align 4
+// CHECK1-NEXT: store i32 0, ptr %[[VAL_57]], align 4
+// CHECK1-NEXT: store i32 99, ptr %[[VAL_58]], align 4
+// CHECK1-NEXT: store i32 1, ptr %[[VAL_59]], align 4
+// CHECK1-NEXT: store i32 0, ptr %[[VAL_60]], align 4
+// CHECK1-NEXT: %[[VAL_76:.*]] = load ptr, ptr %[[VAL_49]], align 8
+// CHECK1-NEXT: %[[VAL_77:.*]] = load i32, ptr %[[VAL_76]], align 4
+// CHECK1-NEXT: call void @__kmpc_for_static_init_4(ptr @1, i32 %[[VAL_77]], i32 92, ptr %[[VAL_60]], ptr %[[VAL_57]], ptr %[[VAL_58]], ptr %[[VAL_59]], i32 1, i32 1)
+// CHECK1-NEXT: %[[VAL_78:.*]] = load i32, ptr %[[VAL_58]], align 4
+// CHECK1-NEXT: %[[VAL_79:.*]] = icmp sgt i32 %[[VAL_78]], 99
+// CHECK1-NEXT: br i1 %[[VAL_79]], label %[[VAL_80:.*]], label %[[VAL_81:.*]]
+// CHECK1: cond.true: ; preds = %[[VAL_82:.*]]
+// CHECK1-NEXT: br label %[[VAL_83:.*]]
+// CHECK1: cond.false: ; preds = %[[VAL_82]]
+// CHECK1-NEXT: %[[VAL_84:.*]] = load i32, ptr %[[VAL_58]], align 4
+// CHECK1-NEXT: br label %[[VAL_83]]
+// CHECK1: cond.end: ; preds = %[[VAL_81]], %[[VAL_80]]
+// CHECK1-NEXT: %[[VAL_85:.*]] = phi i32 [ 99, %[[VAL_80]] ], [ %[[VAL_84]], %[[VAL_81]] ]
+// CHECK1-NEXT: store i32 %[[VAL_85]], ptr %[[VAL_58]], align 4
+// CHECK1-NEXT: %[[VAL_86:.*]] = load i32, ptr %[[VAL_57]], align 4
+// CHECK1-NEXT: store i32 %[[VAL_86]], ptr %[[VAL_54]], align 4
+// CHECK1-NEXT: br label %[[VAL_87:.*]]
+// CHECK1: omp.inner.for.cond: ; preds = %[[VAL_88:.*]], %[[VAL_83]]
+// CHECK1-NEXT: %[[VAL_89:.*]] = load i32, ptr %[[VAL_54]], align 4
+// CHECK1-NEXT: %[[VAL_90:.*]] = load i32, ptr %[[VAL_58]], align 4
+// CHECK1-NEXT: %[[VAL_91:.*]] = icmp sle i32 %[[VAL_89]], %[[VAL_90]]
+// CHECK1-NEXT: br i1 %[[VAL_91]], label %[[VAL_92:.*]], label %[[VAL_93:.*]]
+// CHECK1: omp.inner.for.body: ; preds = %[[VAL_87]]
+// CHECK1-NEXT: %[[VAL_94:.*]] = load i32, ptr %[[VAL_57]], align 4
+// CHECK1-NEXT: %[[VAL_95:.*]] = zext i32 %[[VAL_94]] to i64
+// CHECK1-NEXT: %[[VAL_96:.*]] = load i32, ptr %[[VAL_58]], align 4
+// CHECK1-NEXT: %[[VAL_97:.*]] = zext i32 %[[VAL_96]] to i64
+// CHECK1-NEXT: %[[VAL_98:.*]] = load i32, ptr %[[VAL_61]], align 4
+// CHECK1-NEXT: store i32 %[[VAL_98]], ptr %[[VAL_64]], align 4
+// CHECK1-NEXT: %[[VAL_99:.*]] = load i64, ptr %[[VAL_64]], align 8
+// CHECK1-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @4, i32 4, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3foov_l31.omp_outlined.omp_outlined, i64 %[[VAL_95]], i64 %[[VAL_97]], i64 %[[VAL_99]], ptr %[[VAL_53]])
+// CHECK1-NEXT: br label %[[VAL_88]]
+// CHECK1: omp.inner.for.inc: ; preds = %[[VAL_92]]
+// CHECK1-NEXT: %[[VAL_100:.*]] = load i32, ptr %[[VAL_54]], align 4
+// CHECK1-NEXT: %[[VAL_101:.*]] = load i32, ptr %[[VAL_59]], align 4
+// CHECK1-NEXT: %[[VAL_102:.*]] = add nsw i32 %[[VAL_100]], %[[VAL_101]]
+// CHECK1-NEXT: store i32 %[[VAL_102]], ptr %[[VAL_54]], align 4
+// CHECK1-NEXT: br label %[[VAL_87]]
+// CHECK1: omp.inner.for.end: ; preds = %[[VAL_87]]
+// CHECK1-NEXT: br label %[[VAL_103:.*]]
+// CHECK1: omp.loop.exit: ; preds = %[[VAL_93]]
+// CHECK1-NEXT: call void @__kmpc_for_static_fini(ptr @2, i32 %[[VAL_77]])
+// CHECK1-NEXT: %[[VAL_104:.*]] = load i32, ptr %[[VAL_60]], align 4
+// CHECK1-NEXT: %[[VAL_105:.*]] = icmp ne i32 %[[VAL_104]], 0
+// CHECK1-NEXT: br i1 %[[VAL_105]], label %[[VAL_106:.*]], label %[[VAL_107:.*]]
+// CHECK1: .omp.lastprivate.then: ; preds = %[[VAL_103]]
+// CHECK1-NEXT: store i32 10, ptr %[[VAL_61]], align 4
+// CHECK1-NEXT: %[[VAL_108:.*]] = load i32, ptr %[[VAL_61]], align 4
+// CHECK1-NEXT: store i32 %[[VAL_108]], ptr %[[VAL_51]], align 4
+// CHECK1-NEXT: br label %[[VAL_107]]
+// CHECK1: .omp.lastprivate.done: ; preds = %[[VAL_106]], %[[VAL_103]]
+// CHECK1-NEXT: %[[VAL_109:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_65]], i64 0, i64 0
+// CHECK1-NEXT: store ptr %[[VAL_53]], ptr %[[VAL_109]], align 8
+// CHECK1-NEXT: %[[VAL_110:.*]] = call i32 @__kmpc_reduce(ptr @3, i32 %[[VAL_77]], i32 1, i64 8, ptr %[[VAL_65]], ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3foov_l31.omp_outlined.omp.reduction.reduction_func, ptr @.gomp_critical_user_.reduction.var)
+// CHECK1-NEXT: switch i32 %[[VAL_110]], label %[[VAL_111:.*]] [
+// CHECK1-NEXT: i32 1, label %[[VAL_112:.*]]
+// CHECK1-NEXT: i32 2, label %[[VAL_113:.*]]
+// CHECK1-NEXT: ]
+// CHECK1: .omp.reduction.case1: ; preds = %[[VAL_107]]
+// CHECK1-NEXT: %[[VAL_114:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_73]], i32 0, i32 0
+// CHECK1-NEXT: %[[VAL_115:.*]] = load float, ptr %[[VAL_114]], align 4
+// CHECK1-NEXT: %[[VAL_116:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_73]], i32 0, i32 1
+// CHECK1-NEXT: %[[VAL_117:.*]] = load float, ptr %[[VAL_116]], align 4
+// CHECK1-NEXT: %[[VAL_118:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_53]], i32 0, i32 0
+// CHECK1-NEXT: %[[VAL_119:.*]] = load float, ptr %[[VAL_118]], align 4
+// CHECK1-NEXT: %[[VAL_120:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_53]], i32 0, i32 1
+// CHECK1-NEXT: %[[VAL_121:.*]] = load float, ptr %[[VAL_120]], align 4
+// CHECK1-NEXT: %[[VAL_122:.*]] = fadd float %[[VAL_115]], %[[VAL_119]]
+// CHECK1-NEXT: %[[VAL_123:.*]] = fadd float %[[VAL_117]], %[[VAL_121]]
+// CHECK1-NEXT: %[[VAL_124:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_73]], i32 0, i32 0
+// CHECK1-NEXT: %[[VAL_125:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_73]], i32 0, i32 1
+// CHECK1-NEXT: store float %[[VAL_122]], ptr %[[VAL_124]], align 4
+// CHECK1-NEXT: store float %[[VAL_123]], ptr %[[VAL_125]], align 4
+// CHECK1-NEXT: call void @__kmpc_end_reduce(ptr @3, i32 %[[VAL_77]], ptr @.gomp_critical_user_.reduction.var)
+// CHECK1-NEXT: br label %[[VAL_111]]
+// CHECK1: .omp.reduction.case2: ; preds = %[[VAL_107]]
+// CHECK1-NEXT: %[[VAL_126:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_53]], i32 0, i32 0
+// CHECK1-NEXT: %[[VAL_127:.*]] = load float, ptr %[[VAL_126]], align 4
+// CHECK1-NEXT: %[[VAL_128:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_53]], i32 0, i32 1
+// CHECK1-NEXT: %[[VAL_129:.*]] = load float, ptr %[[VAL_128]], align 4
+// CHECK1-NEXT: call void @__atomic_load(i64 noundef 8, ptr noundef %[[VAL_73]], ptr noundef %[[VAL_66]], i32 noundef signext 0)
+// CHECK1-NEXT: br label %[[VAL_130:.*]]
+// CHECK1: atomic_cont: ; preds = %[[VAL_130]], %[[VAL_113]]
+// CHECK1-NEXT: %[[VAL_131:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_66]], i32 0, i32 0
+// CHECK1-NEXT: %[[VAL_132:.*]] = load float, ptr %[[VAL_131]], align 4
+// CHECK1-NEXT: %[[VAL_133:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_66]], i32 0, i32 1
+// CHECK1-NEXT: %[[VAL_134:.*]] = load float, ptr %[[VAL_133]], align 4
+// CHECK1-NEXT: %[[VAL_135:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_68]], i32 0, i32 0
+// CHECK1-NEXT: %[[VAL_136:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_68]], i32 0, i32 1
+// CHECK1-NEXT: store float %[[VAL_132]], ptr %[[VAL_135]], align 4
+// CHECK1-NEXT: store float %[[VAL_134]], ptr %[[VAL_136]], align 4
+// CHECK1-NEXT: %[[VAL_137:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_68]], i32 0, i32 0
+// CHECK1-NEXT: %[[VAL_138:.*]] = load float, ptr %[[VAL_137]], align 4
+// CHECK1-NEXT: %[[VAL_139:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_68]], i32 0, i32 1
+// CHECK1-NEXT: %[[VAL_140:.*]] = load float, ptr %[[VAL_139]], align 4
+// CHECK1-NEXT: %[[VAL_141:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_53]], i32 0, i32 0
+// CHECK1-NEXT: %[[VAL_142:.*]] = load float, ptr %[[VAL_141]], align 4
+// CHECK1-NEXT: %[[VAL_143:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_53]], i32 0, i32 1
+// CHECK1-NEXT: %[[VAL_144:.*]] = load float, ptr %[[VAL_143]], align 4
+// CHECK1-NEXT: %[[VAL_145:.*]] = fadd float %[[VAL_138]], %[[VAL_142]]
+// CHECK1-NEXT: %[[VAL_146:.*]] = fadd float %[[VAL_140]], %[[VAL_144]]
+// CHECK1-NEXT: %[[VAL_147:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_67]], i32 0, i32 0
+// CHECK1-NEXT: %[[VAL_148:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_67]], i32 0, i32 1
+// CHECK1-NEXT: store float %[[VAL_145]], ptr %[[VAL_147]], align 4
+// CHECK1-NEXT: store float %[[VAL_146]], ptr %[[VAL_148]], align 4
+// CHECK1-NEXT: %[[VAL_149:.*]] = call noundef zeroext i1 @__atomic_compare_exchange(i64 noundef 8, ptr noundef %[[VAL_73]], ptr noundef %[[VAL_66]], ptr noundef %[[VAL_67]], i32 noundef signext 0, i32 noundef signext 0)
+// CHECK1-NEXT: br i1 %[[VAL_149]], label %[[VAL_150:.*]], label %[[VAL_130]]
+// CHECK1: atomic_exit: ; preds = %[[VAL_130]]
+// CHECK1-NEXT: call void @__kmpc_end_reduce(ptr @3, i32 %[[VAL_77]], ptr @.gomp_critical_user_.reduction.var)
+// CHECK1-NEXT: br label %[[VAL_111]]
+// CHECK1: .omp.reduction.default: ; preds = %[[VAL_150]], %[[VAL_112]], %[[VAL_107]]
+// CHECK1-NEXT: ret void
+//
+//
+// CHECK1-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3foov_l31.omp_outlined.omp.reduction.reduction_func
+// CHECK1-SAME: (ptr noundef [[TMP0:%.*]], ptr noundef [[TMP1:%.*]]) #[[ATTR3:[0-9]+]] {
+// CHECK1-NEXT: entry:
+// CHECK1-NEXT: %[[VAL_297:.*]] = alloca ptr, align 8
+// CHECK1-NEXT: %[[VAL_298:.*]] = alloca ptr, align 8
+// CHECK1-NEXT: store ptr %[[VAL_299:.*]], ptr %[[VAL_297]], align 8
+// CHECK1-NEXT: store ptr %[[VAL_300:.*]], ptr %[[VAL_298]], align 8
+// CHECK1-NEXT: %[[VAL_301:.*]] = load ptr, ptr %[[VAL_297]], align 8
+// CHECK1-NEXT: %[[VAL_302:.*]] = load ptr, ptr %[[VAL_298]], align 8
+// CHECK1-NEXT: %[[VAL_303:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_302]], i64 0, i64 0
+// CHECK1-NEXT: %[[VAL_304:.*]] = load ptr, ptr %[[VAL_303]], align 8
+// CHECK1-NEXT: %[[VAL_305:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_301]], i64 0, i64 0
+// CHECK1-NEXT: %[[VAL_306:.*]] = load ptr, ptr %[[VAL_305]], align 8
+// CHECK1-NEXT: %[[VAL_307:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_306]], i32 0, i32 0
+// CHECK1-NEXT: %[[VAL_308:.*]] = load float, ptr %[[VAL_307]], align 4
+// CHECK1-NEXT: %[[VAL_309:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_306]], i32 0, i32 1
+// CHECK1-NEXT: %[[VAL_310:.*]] = load float, ptr %[[VAL_309]], align 4
+// CHECK1-NEXT: %[[VAL_311:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_304]], i32 0, i32 0
+// CHECK1-NEXT: %[[VAL_312:.*]] = load float, ptr %[[VAL_311]], align 4
+// CHECK1-NEXT: %[[VAL_313:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_304]], i32 0, i32 1
+// CHECK1-NEXT: %[[VAL_314:.*]] = load float, ptr %[[VAL_313]], align 4
+// CHECK1-NEXT: %[[VAL_315:.*]] = fadd float %[[VAL_308]], %[[VAL_312]]
+// CHECK1-NEXT: %[[VAL_316:.*]] = fadd float %[[VAL_310]], %[[VAL_314]]
+// CHECK1-NEXT: %[[VAL_317:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_306]], i32 0, i32 0
+// CHECK1-NEXT: %[[VAL_318:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_306]], i32 0, i32 1
+// CHECK1-NEXT: store float %[[VAL_315]], ptr %[[VAL_317]], align 4
+// CHECK1-NEXT: store float %[[VAL_316]], ptr %[[VAL_318]], align 4
+// CHECK1-NEXT: ret void
+//
+//
+// CHECK2-LABEL: define {{[^@]+}}@_Z3foov
+// CHECK2-SAME: () #[[ATTR0:[0-9]+]] {
+// CHECK2-NEXT: entry:
+// CHECK2-NEXT: %[[VAL_0:.*]] = alloca i32, align 4
+// CHECK2-NEXT: %[[VAL_1:.*]] = alloca i32, align 4
+// CHECK2-NEXT: %[[VAL_2:.*]] = alloca { float, float }, align 4
+// CHECK2-NEXT: %[[VAL_3:.*]] = alloca i32, align 4
+// CHECK2-NEXT: %[[VAL_4:.*]] = alloca [2 x ptr], align 4
+// CHECK2-NEXT: %[[VAL_5:.*]] = alloca [2 x ptr], align 4
+// CHECK2-NEXT: %[[VAL_6:.*]] = alloca [2 x ptr], align 4
+// CHECK2-NEXT: %[[VAL_7:.*]] = alloca i32, align 4
+// CHECK2-NEXT: %[[VAL_8:.*]] = alloca i32, align 4
+// CHECK2-NEXT: %[[VAL_9:.*]] = alloca %[[VAL_10:.*]], align 8
+// CHECK2-NEXT: %[[VAL_11:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_2]], i32 0, i32 0
+// CHECK2-NEXT: %[[VAL_12:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_2]], i32 0, i32 1
+// CHECK2-NEXT: store float 0.000000e+00, ptr %[[VAL_11]], align 4
+// CHECK2-NEXT: store float 0.000000e+00, ptr %[[VAL_12]], align 4
+// CHECK2-NEXT: %[[VAL_13:.*]] = load i32, ptr %[[VAL_1]], align 4
+// CHECK2-NEXT: store i32 %[[VAL_13]], ptr %[[VAL_3]], align 4
+// CHECK2-NEXT: %[[VAL_14:.*]] = load i32, ptr %[[VAL_3]], align 4
+// CHECK2-NEXT: %[[VAL_15:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_4]], i32 0, i32 0
+// CHECK2-NEXT: store i32 %[[VAL_14]], ptr %[[VAL_15]], align 4
+// CHECK2-NEXT: %[[VAL_16:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_5]], i32 0, i32 0
+// CHECK2-NEXT: store i32 %[[VAL_14]], ptr %[[VAL_16]], align 4
+// CHECK2-NEXT: %[[VAL_17:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_6]], i32 0, i32 0
+// CHECK2-NEXT: store ptr null, ptr %[[VAL_17]], align 4
+// CHECK2-NEXT: %[[VAL_18:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_4]], i32 0, i32 1
+// CHECK2-NEXT: store ptr %[[VAL_2]], ptr %[[VAL_18]], align 4
+// CHECK2-NEXT: %[[VAL_19:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_5]], i32 0, i32 1
+// CHECK2-NEXT: store ptr %[[VAL_2]], ptr %[[VAL_19]], align 4
+// CHECK2-NEXT: %[[VAL_20:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_6]], i32 0, i32 1
+// CHECK2-NEXT: store ptr null, ptr %[[VAL_20]], align 4
+// CHECK2-NEXT: %[[VAL_21:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_4]], i32 0, i32 0
+// CHECK2-NEXT: %[[VAL_22:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_5]], i32 0, i32 0
+// CHECK2-NEXT: %[[VAL_23:.*]] = getelementptr inbounds %[[VAL_10]], ptr %[[VAL_9]], i32 0, i32 0
+// CHECK2-NEXT: store i32 2, ptr %[[VAL_23]], align 4
+// CHECK2-NEXT: %[[VAL_24:.*]] = getelementptr inbounds %[[VAL_10]], ptr %[[VAL_9]], i32 0, i32 1
+// CHECK2-NEXT: store i32 2, ptr %[[VAL_24]], align 4
+// CHECK2-NEXT: %[[VAL_25:.*]] = getelementptr inbounds %[[VAL_10]], ptr %[[VAL_9]], i32 0, i32 2
+// CHECK2-NEXT: store ptr %[[VAL_21]], ptr %[[VAL_25]], align 4
+// CHECK2-NEXT: %[[VAL_26:.*]] = getelementptr inbounds %[[VAL_10]], ptr %[[VAL_9]], i32 0, i32 3
+// CHECK2-NEXT: store ptr %[[VAL_22]], ptr %[[VAL_26]], align 4
+// CHECK2-NEXT: %[[VAL_27:.*]] = getelementptr inbounds %[[VAL_10]], ptr %[[VAL_9]], i32 0, i32 4
+// CHECK2-NEXT: store ptr @.offload_sizes, ptr %[[VAL_27]], align 4
+// CHECK2-NEXT: %[[VAL_28:.*]] = getelementptr inbounds %[[VAL_10]], ptr %[[VAL_9]], i32 0, i32 5
+// CHECK2-NEXT: store ptr @.offload_maptypes, ptr %[[VAL_28]], align 4
+// CHECK2-NEXT: %[[VAL_29:.*]] = getelementptr inbounds %[[VAL_10]], ptr %[[VAL_9]], i32 0, i32 6
+// CHECK2-NEXT: store ptr null, ptr %[[VAL_29]], align 4
+// CHECK2-NEXT: %[[VAL_30:.*]] = getelementptr inbounds %[[VAL_10]], ptr %[[VAL_9]], i32 0, i32 7
+// CHECK2-NEXT: store ptr null, ptr %[[VAL_30]], align 4
+// CHECK2-NEXT: %[[VAL_31:.*]] = getelementptr inbounds %[[VAL_10]], ptr %[[VAL_9]], i32 0, i32 8
+// CHECK2-NEXT: store i64 100, ptr %[[VAL_31]], align 8
+// CHECK2-NEXT: %[[VAL_32:.*]] = getelementptr inbounds %[[VAL_10]], ptr %[[VAL_9]], i32 0, i32 9
+// CHECK2-NEXT: store i64 0, ptr %[[VAL_32]], align 8
+// CHECK2-NEXT: %[[VAL_33:.*]] = getelementptr inbounds %[[VAL_10]], ptr %[[VAL_9]], i32 0, i32 10
+// CHECK2-NEXT: store [3 x i32] zeroinitializer, ptr %[[VAL_33]], align 4
+// CHECK2-NEXT: %[[VAL_34:.*]] = getelementptr inbounds %[[VAL_10]], ptr %[[VAL_9]], i32 0, i32 11
+// CHECK2-NEXT: store [3 x i32] zeroinitializer, ptr %[[VAL_34]], align 4
+// CHECK2-NEXT: %[[VAL_35:.*]] = getelementptr inbounds %[[VAL_10]], ptr %[[VAL_9]], i32 0, i32 12
+// CHECK2-NEXT: store i32 0, ptr %[[VAL_35]], align 4
+// CHECK2-NEXT: %[[VAL_36:.*]] = call i32 @__tgt_target_kernel(ptr @4, i64 -1, i32 0, i32 0, ptr @.{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3foov_l31.region_id, ptr %[[VAL_9]])
+// CHECK2-NEXT: %[[VAL_37:.*]] = icmp ne i32 %[[VAL_36]], 0
+// CHECK2-NEXT: br i1 %[[VAL_37]], label %[[VAL_38:.*]], label %[[VAL_39:.*]]
+// CHECK2: omp_offload.failed: ; preds = %[[VAL_40:.*]]
+// CHECK2-NEXT: call void @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3foov_l31(i32 %[[VAL_14]], ptr %[[VAL_2]]) #2
+// CHECK2-NEXT: br label %[[VAL_39]]
+// CHECK2: omp_offload.cont: ; preds = %[[VAL_38]], %[[VAL_40]]
+// CHECK2-NEXT: ret i32 0
+//
+//
+// CHECK2-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3foov_l31
+// CHECK2-SAME: (i32 noundef [[J:%.*]], ptr noundef nonnull align 4 dereferenceable(8) [[SUM:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK2-NEXT: entry:
+// CHECK2-NEXT: %[[VAL_41:.*]] = alloca i32, align 4
+// CHECK2-NEXT: %[[VAL_42:.*]] = alloca ptr, align 4
+// CHECK2-NEXT: %[[VAL_43:.*]] = alloca i32, align 4
+// CHECK2-NEXT: store i32 %[[VAL_44:.*]], ptr %[[VAL_41]], align 4
+// CHECK2-NEXT: store ptr %[[VAL_45:.*]], ptr %[[VAL_42]], align 4
+// CHECK2-NEXT: %[[VAL_46:.*]] = load ptr, ptr %[[VAL_42]], align 4
+// CHECK2-NEXT: %[[VAL_47:.*]] = load i32, ptr %[[VAL_41]], align 4
+// CHECK2-NEXT: store i32 %[[VAL_47]], ptr %[[VAL_43]], align 4
+// CHECK2-NEXT: %[[VAL_48:.*]] = load i32, ptr %[[VAL_43]], align 4
+// CHECK2-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @4, i32 2, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3foov_l31.omp_outlined, i32 %[[VAL_48]], ptr %[[VAL_46]])
+// CHECK2-NEXT: ret void
+//
+//
+// CHECK2-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3foov_l31.omp_outlined
+// CHECK2-SAME: (ptr noalias noundef [[DOTGLOBAL_TID_:%.*]], ptr noalias noundef [[DOTBOUND_TID_:%.*]], i32 noundef [[J:%.*]], ptr noundef nonnull align 4 dereferenceable(8) [[SUM:%.*]]) #[[ATTR1:[0-9]+]] {
+// CHECK2-NEXT: entry:
+// CHECK2-NEXT: %[[VAL_49:.*]] = alloca ptr, align 4
+// CHECK2-NEXT: %[[VAL_50:.*]] = alloca ptr, align 4
+// CHECK2-NEXT: %[[VAL_51:.*]] = alloca i32, align 4
+// CHECK2-NEXT: %[[VAL_52:.*]] = alloca ptr, align 4
+// CHECK2-NEXT: %[[VAL_53:.*]] = alloca { float, float }, align 4
+// CHECK2-NEXT: %[[VAL_54:.*]] = alloca i32, align 4
+// CHECK2-NEXT: %[[VAL_55:.*]] = alloca i32, align 4
+// CHECK2-NEXT: %[[VAL_56:.*]] = alloca i32, align 4
+// CHECK2-NEXT: %[[VAL_57:.*]] = alloca i32, align 4
+// CHECK2-NEXT: %[[VAL_58:.*]] = alloca i32, align 4
+// CHECK2-NEXT: %[[VAL_59:.*]] = alloca i32, align 4
+// CHECK2-NEXT: %[[VAL_60:.*]] = alloca i32, align 4
+// CHECK2-NEXT: %[[VAL_61:.*]] = alloca i32, align 4
+// CHECK2-NEXT: %[[VAL_62:.*]] = alloca i32, align 4
+// CHECK2-NEXT: %[[VAL_63:.*]] = alloca i32, align 4
+// CHECK2-NEXT: %[[VAL_64:.*]] = alloca i32, align 4
+// CHECK2-NEXT: %[[VAL_65:.*]] = alloca [1 x ptr], align 4
+// CHECK2-NEXT: %[[VAL_66:.*]] = alloca { float, float }, align 4
+// CHECK2-NEXT: %[[VAL_67:.*]] = alloca { float, float }, align 4
+// CHECK2-NEXT: %[[VAL_68:.*]] = alloca { float, float }, align 4
+// CHECK2-NEXT: store ptr %[[VAL_69:.*]], ptr %[[VAL_49]], align 4
+// CHECK2-NEXT: store ptr %[[VAL_70:.*]], ptr %[[VAL_50]], align 4
+// CHECK2-NEXT: store i32 %[[VAL_71:.*]], ptr %[[VAL_51]], align 4
+// CHECK2-NEXT: store ptr %[[VAL_72:.*]], ptr %[[VAL_52]], align 4
+// CHECK2-NEXT: %[[VAL_73:.*]] = load ptr, ptr %[[VAL_52]], align 4
+// CHECK2-NEXT: %[[VAL_74:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_53]], i32 0, i32 0
+// CHECK2-NEXT: %[[VAL_75:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_53]], i32 0, i32 1
+// CHECK2-NEXT: store float 0.000000e+00, ptr %[[VAL_74]], align 4
+// CHECK2-NEXT: store float 0.000000e+00, ptr %[[VAL_75]], align 4
+// CHECK2-NEXT: store i32 0, ptr %[[VAL_57]], align 4
+// CHECK2-NEXT: store i32 99, ptr %[[VAL_58]], align 4
+// CHECK2-NEXT: store i32 1, ptr %[[VAL_59]], align 4
+// CHECK2-NEXT: store i32 0, ptr %[[VAL_60]], align 4
+// CHECK2-NEXT: %[[VAL_76:.*]] = load ptr, ptr %[[VAL_49]], align 4
+// CHECK2-NEXT: %[[VAL_77:.*]] = load i32, ptr %[[VAL_76]], align 4
+// CHECK2-NEXT: call void @__kmpc_for_static_init_4(ptr @1, i32 %[[VAL_77]], i32 92, ptr %[[VAL_60]], ptr %[[VAL_57]], ptr %[[VAL_58]], ptr %[[VAL_59]], i32 1, i32 1)
+// CHECK2-NEXT: %[[VAL_78:.*]] = load i32, ptr %[[VAL_58]], align 4
+// CHECK2-NEXT: %[[VAL_79:.*]] = icmp sgt i32 %[[VAL_78]], 99
+// CHECK2-NEXT: br i1 %[[VAL_79]], label %[[VAL_80:.*]], label %[[VAL_81:.*]]
+// CHECK2: cond.true: ; preds = %[[VAL_82:.*]]
+// CHECK2-NEXT: br label %[[VAL_83:.*]]
+// CHECK2: cond.false: ; preds = %[[VAL_82]]
+// CHECK2-NEXT: %[[VAL_84:.*]] = load i32, ptr %[[VAL_58]], align 4
+// CHECK2-NEXT: br label %[[VAL_83]]
+// CHECK2: cond.end: ; preds = %[[VAL_81]], %[[VAL_80]]
+// CHECK2-NEXT: %[[VAL_85:.*]] = phi i32 [ 99, %[[VAL_80]] ], [ %[[VAL_84]], %[[VAL_81]] ]
+// CHECK2-NEXT: store i32 %[[VAL_85]], ptr %[[VAL_58]], align 4
+// CHECK2-NEXT: %[[VAL_86:.*]] = load i32, ptr %[[VAL_57]], align 4
+// CHECK2-NEXT: store i32 %[[VAL_86]], ptr %[[VAL_54]], align 4
+// CHECK2-NEXT: br label %[[VAL_87:.*]]
+// CHECK2: omp.inner.for.cond: ; preds = %[[VAL_88:.*]], %[[VAL_83]]
+// CHECK2-NEXT: %[[VAL_89:.*]] = load i32, ptr %[[VAL_54]], align 4
+// CHECK2-NEXT: %[[VAL_90:.*]] = load i32, ptr %[[VAL_58]], align 4
+// CHECK2-NEXT: %[[VAL_91:.*]] = icmp sle i32 %[[VAL_89]], %[[VAL_90]]
+// CHECK2-NEXT: br i1 %[[VAL_91]], label %[[VAL_92:.*]], label %[[VAL_93:.*]]
+// CHECK2: omp.inner.for.body: ; preds = %[[VAL_87]]
+// CHECK2-NEXT: %[[VAL_94:.*]] = load i32, ptr %[[VAL_57]], align 4
+// CHECK2-NEXT: %[[VAL_95:.*]] = load i32, ptr %[[VAL_58]], align 4
+// CHECK2-NEXT: %[[VAL_96:.*]] = load i32, ptr %[[VAL_61]], align 4
+// CHECK2-NEXT: store i32 %[[VAL_96]], ptr %[[VAL_64]], align 4
+// CHECK2-NEXT: %[[VAL_97:.*]] = load i32, ptr %[[VAL_64]], align 4
+// CHECK2-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @4, i32 4, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3foov_l31.omp_outlined.omp_outlined, i32 %[[VAL_94]], i32 %[[VAL_95]], i32 %[[VAL_97]], ptr %[[VAL_53]])
+// CHECK2-NEXT: br label %[[VAL_88]]
+// CHECK2: omp.inner.for.inc: ; preds = %[[VAL_92]]
+// CHECK2-NEXT: %[[VAL_98:.*]] = load i32, ptr %[[VAL_54]], align 4
+// CHECK2-NEXT: %[[VAL_99:.*]] = load i32, ptr %[[VAL_59]], align 4
+// CHECK2-NEXT: %[[VAL_100:.*]] = add nsw i32 %[[VAL_98]], %[[VAL_99]]
+// CHECK2-NEXT: store i32 %[[VAL_100]], ptr %[[VAL_54]], align 4
+// CHECK2-NEXT: br label %[[VAL_87]]
+// CHECK2: omp.inner.for.end: ; preds = %[[VAL_87]]
+// CHECK2-NEXT: br label %[[VAL_101:.*]]
+// CHECK2: omp.loop.exit: ; preds = %[[VAL_93]]
+// CHECK2-NEXT: call void @__kmpc_for_static_fini(ptr @2, i32 %[[VAL_77]])
+// CHECK2-NEXT: %[[VAL_102:.*]] = load i32, ptr %[[VAL_60]], align 4
+// CHECK2-NEXT: %[[VAL_103:.*]] = icmp ne i32 %[[VAL_102]], 0
+// CHECK2-NEXT: br i1 %[[VAL_103]], label %[[VAL_104:.*]], label %[[VAL_105:.*]]
+// CHECK2: .omp.lastprivate.then: ; preds = %[[VAL_101]]
+// CHECK2-NEXT: store i32 10, ptr %[[VAL_61]], align 4
+// CHECK2-NEXT: %[[VAL_106:.*]] = load i32, ptr %[[VAL_61]], align 4
+// CHECK2-NEXT: store i32 %[[VAL_106]], ptr %[[VAL_51]], align 4
+// CHECK2-NEXT: br label %[[VAL_105]]
+// CHECK2: .omp.lastprivate.done: ; preds = %[[VAL_104]], %[[VAL_101]]
+// CHECK2-NEXT: %[[VAL_107:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_65]], i32 0, i32 0
+// CHECK2-NEXT: store ptr %[[VAL_53]], ptr %[[VAL_107]], align 4
+// CHECK2-NEXT: %[[VAL_108:.*]] = call i32 @__kmpc_reduce(ptr @3, i32 %[[VAL_77]], i32 1, i32 4, ptr %[[VAL_65]], ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3foov_l31.omp_outlined.omp.reduction.reduction_func, ptr @.gomp_critical_user_.reduction.var)
+// CHECK2-NEXT: switch i32 %[[VAL_108]], label %[[VAL_109:.*]] [
+// CHECK2-NEXT: i32 1, label %[[VAL_110:.*]]
+// CHECK2-NEXT: i32 2, label %[[VAL_111:.*]]
+// CHECK2-NEXT: ]
+// CHECK2: .omp.reduction.case1: ; preds = %[[VAL_105]]
+// CHECK2-NEXT: %[[VAL_112:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_73]], i32 0, i32 0
+// CHECK2-NEXT: %[[VAL_113:.*]] = load float, ptr %[[VAL_112]], align 4
+// CHECK2-NEXT: %[[VAL_114:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_73]], i32 0, i32 1
+// CHECK2-NEXT: %[[VAL_115:.*]] = load float, ptr %[[VAL_114]], align 4
+// CHECK2-NEXT: %[[VAL_116:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_53]], i32 0, i32 0
+// CHECK2-NEXT: %[[VAL_117:.*]] = load float, ptr %[[VAL_116]], align 4
+// CHECK2-NEXT: %[[VAL_118:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_53]], i32 0, i32 1
+// CHECK2-NEXT: %[[VAL_119:.*]] = load float, ptr %[[VAL_118]], align 4
+// CHECK2-NEXT: %[[VAL_120:.*]] = fadd float %[[VAL_113]], %[[VAL_117]]
+// CHECK2-NEXT: %[[VAL_121:.*]] = fadd float %[[VAL_115]], %[[VAL_119]]
+// CHECK2-NEXT: %[[VAL_122:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_73]], i32 0, i32 0
+// CHECK2-NEXT: %[[VAL_123:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_73]], i32 0, i32 1
+// CHECK2-NEXT: store float %[[VAL_120]], ptr %[[VAL_122]], align 4
+// CHECK2-NEXT: store float %[[VAL_121]], ptr %[[VAL_123]], align 4
+// CHECK2-NEXT: call void @__kmpc_end_reduce(ptr @3, i32 %[[VAL_77]], ptr @.gomp_critical_user_.reduction.var)
+// CHECK2-NEXT: br label %[[VAL_109]]
+// CHECK2: .omp.reduction.case2: ; preds = %[[VAL_105]]
+// CHECK2-NEXT: %[[VAL_124:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_53]], i32 0, i32 0
+// CHECK2-NEXT: %[[VAL_125:.*]] = load float, ptr %[[VAL_124]], align 4
+// CHECK2-NEXT: %[[VAL_126:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_53]], i32 0, i32 1
+// CHECK2-NEXT: %[[VAL_127:.*]] = load float, ptr %[[VAL_126]], align 4
+// CHECK2-NEXT: call void @__atomic_load(i32 noundef 8, ptr noundef %[[VAL_73]], ptr noundef %[[VAL_66]], i32 noundef 0)
+// CHECK2-NEXT: br label %[[VAL_128:.*]]
+// CHECK2: atomic_cont: ; preds = %[[VAL_128]], %[[VAL_111]]
+// CHECK2-NEXT: %[[VAL_129:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_66]], i32 0, i32 0
+// CHECK2-NEXT: %[[VAL_130:.*]] = load float, ptr %[[VAL_129]], align 4
+// CHECK2-NEXT: %[[VAL_131:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_66]], i32 0, i32 1
+// CHECK2-NEXT: %[[VAL_132:.*]] = load float, ptr %[[VAL_131]], align 4
+// CHECK2-NEXT: %[[VAL_133:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_68]], i32 0, i32 0
+// CHECK2-NEXT: %[[VAL_134:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_68]], i32 0, i32 1
+// CHECK2-NEXT: store float %[[VAL_130]], ptr %[[VAL_133]], align 4
+// CHECK2-NEXT: store float %[[VAL_132]], ptr %[[VAL_134]], align 4
+// CHECK2-NEXT: %[[VAL_135:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_68]], i32 0, i32 0
+// CHECK2-NEXT: %[[VAL_136:.*]] = load float, ptr %[[VAL_135]], align 4
+// CHECK2-NEXT: %[[VAL_137:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_68]], i32 0, i32 1
+// CHECK2-NEXT: %[[VAL_138:.*]] = load float, ptr %[[VAL_137]], align 4
+// CHECK2-NEXT: %[[VAL_139:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_53]], i32 0, i32 0
+// CHECK2-NEXT: %[[VAL_140:.*]] = load float, ptr %[[VAL_139]], align 4
+// CHECK2-NEXT: %[[VAL_141:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_53]], i32 0, i32 1
+// CHECK2-NEXT: %[[VAL_142:.*]] = load float, ptr %[[VAL_141]], align 4
+// CHECK2-NEXT: %[[VAL_143:.*]] = fadd float %[[VAL_136]], %[[VAL_140]]
+// CHECK2-NEXT: %[[VAL_144:.*]] = fadd float %[[VAL_138]], %[[VAL_142]]
+// CHECK2-NEXT: %[[VAL_145:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_67]], i32 0, i32 0
+// CHECK2-NEXT: %[[VAL_146:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_67]], i32 0, i32 1
+// CHECK2-NEXT: store float %[[VAL_143]], ptr %[[VAL_145]], align 4
+// CHECK2-NEXT: store float %[[VAL_144]], ptr %[[VAL_146]], align 4
+// CHECK2-NEXT: %[[VAL_147:.*]] = call noundef zeroext i1 @__atomic_compare_exchange(i32 noundef 8, ptr noundef %[[VAL_73]], ptr noundef %[[VAL_66]], ptr noundef %[[VAL_67]], i32 noundef 0, i32 noundef 0)
+// CHECK2-NEXT: br i1 %[[VAL_147]], label %[[VAL_148:.*]], label %[[VAL_128]]
+// CHECK2: atomic_exit: ; preds = %[[VAL_128]]
+// CHECK2-NEXT: call void @__kmpc_end_reduce(ptr @3, i32 %[[VAL_77]], ptr @.gomp_critical_user_.reduction.var)
+// CHECK2-NEXT: br label %[[VAL_109]]
+// CHECK2: .omp.reduction.default: ; preds = %[[VAL_148]], %[[VAL_110]], %[[VAL_105]]
+// CHECK2-NEXT: ret void
+//
+//
+// CHECK2-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3foov_l31.omp_outlined.omp_outlined
+// CHECK2-SAME: (ptr noalias noundef [[DOTGLOBAL_TID_:%.*]], ptr noalias noundef [[DOTBOUND_TID_:%.*]], i32 noundef [[J:%.*]], ptr noundef nonnull align 4 dereferenceable(8) [[SUM:%.*]]) #[[ATTR1:[0-9]+]] {
+// CHECK2-NEXT: entry:
+// CHECK2-NEXT: %[[VAL_149:.*]] = alloca ptr, align 4
+// CHECK2-NEXT: %[[VAL_150:.*]] = alloca ptr, align 4
+// CHECK2-NEXT: %[[VAL_151:.*]] = alloca i32, align 4
+// CHECK2-NEXT: %[[VAL_152:.*]] = alloca i32, align 4
+// CHECK2-NEXT: %[[VAL_153:.*]] = alloca i32, align 4
+// CHECK2-NEXT: %[[VAL_154:.*]] = alloca ptr, align 4
+// CHECK2-NEXT: %[[VAL_155:.*]] = alloca i32, align 4
+// CHECK2-NEXT: %[[VAL_156:.*]] = alloca i32, align 4
+// CHECK2-NEXT: %[[VAL_157:.*]] = alloca i32, align 4
+// CHECK2-NEXT: %[[VAL_158:.*]] = alloca i32, align 4
+// CHECK2-NEXT: %[[VAL_159:.*]] = alloca i32, align 4
+// CHECK2-NEXT: %[[VAL_160:.*]] = alloca i32, align 4
+// CHECK2-NEXT: %[[VAL_161:.*]] = alloca i32, align 4
+// CHECK2-NEXT: %[[VAL_162:.*]] = alloca i32, align 4
+// CHECK2-NEXT: %[[VAL_163:.*]] = alloca { float, float }, align 4
+// CHECK2-NEXT: %[[VAL_164:.*]] = alloca i32, align 4
+// CHECK2-NEXT: %[[VAL_165:.*]] = alloca i32, align 4
+// CHECK2-NEXT: %[[VAL_166:.*]] = alloca [1 x ptr], align 4
+// CHECK2-NEXT: %[[VAL_167:.*]] = alloca { float, float }, align 4
+// CHECK2-NEXT: %[[VAL_168:.*]] = alloca { float, float }, align 4
+// CHECK2-NEXT: %[[VAL_169:.*]] = alloca { float, float }, align 4
+// CHECK2-NEXT: store ptr %[[VAL_170:.*]], ptr %[[VAL_149]], align 4
+// CHECK2-NEXT: store ptr %[[VAL_171:.*]], ptr %[[VAL_150]], align 4
+// CHECK2-NEXT: store i32 %[[VAL_172:.*]], ptr %[[VAL_151]], align 4
+// CHECK2-NEXT: store i32 %[[VAL_173:.*]], ptr %[[VAL_152]], align 4
+// CHECK2-NEXT: store i32 %[[VAL_174:.*]], ptr %[[VAL_153]], align 4
+// CHECK2-NEXT: store ptr %[[VAL_175:.*]], ptr %[[VAL_154]], align 4
+// CHECK2-NEXT: %[[VAL_176:.*]] = load ptr, ptr %[[VAL_154]], align 4
+// CHECK2-NEXT: store i32 0, ptr %[[VAL_158]], align 4
+// CHECK2-NEXT: store i32 99, ptr %[[VAL_159]], align 4
+// CHECK2-NEXT: %[[VAL_177:.*]] = load i32, ptr %[[VAL_151]], align 4
+// CHECK2-NEXT: %[[VAL_178:.*]] = load i32, ptr %[[VAL_152]], align 4
+// CHECK2-NEXT: store i32 %[[VAL_177]], ptr %[[VAL_158]], align 4
+// CHECK2-NEXT: store i32 %[[VAL_178]], ptr %[[VAL_159]], align 4
+// CHECK2-NEXT: store i32 1, ptr %[[VAL_160]], align 4
+// CHECK2-NEXT: store i32 0, ptr %[[VAL_161]], align 4
+// CHECK2-NEXT: %[[VAL_179:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_163]], i32 0, i32 0
+// CHECK2-NEXT: %[[VAL_180:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_163]], i32 0, i32 1
+// CHECK2-NEXT: store float 0.000000e+00, ptr %[[VAL_179]], align 4
+// CHECK2-NEXT: store float 0.000000e+00, ptr %[[VAL_180]], align 4
+// CHECK2-NEXT: %[[VAL_181:.*]] = load ptr, ptr %[[VAL_149]], align 4
+// CHECK2-NEXT: %[[VAL_182:.*]] = load i32, ptr %[[VAL_181]], align 4
+// CHECK2-NEXT: call void @__kmpc_for_static_init_4(ptr @2, i32 %[[VAL_182]], i32 34, ptr %[[VAL_161]], ptr %[[VAL_158]], ptr %[[VAL_159]], ptr %[[VAL_160]], i32 1, i32 1)
+// CHECK2-NEXT: %[[VAL_183:.*]] = load i32, ptr %[[VAL_159]], align 4
+// CHECK2-NEXT: %[[VAL_184:.*]] = icmp sgt i32 %[[VAL_183]], 99
+// CHECK2-NEXT: br i1 %[[VAL_184]], label %[[VAL_185:.*]], label %[[VAL_186:.*]]
+// CHECK2: cond.true: ; preds = %[[VAL_187:.*]]
+// CHECK2-NEXT: br label %[[VAL_188:.*]]
+// CHECK2: cond.false: ; preds = %[[VAL_187]]
+// CHECK2-NEXT: %[[VAL_189:.*]] = load i32, ptr %[[VAL_159]], align 4
+// CHECK2-NEXT: br label %[[VAL_188]]
+// CHECK2: cond.end: ; preds = %[[VAL_186]], %[[VAL_185]]
+// CHECK2-NEXT: %[[VAL_190:.*]] = phi i32 [ 99, %[[VAL_185]] ], [ %[[VAL_189]], %[[VAL_186]] ]
+// CHECK2-NEXT: store i32 %[[VAL_190]], ptr %[[VAL_159]], align 4
+// CHECK2-NEXT: %[[VAL_191:.*]] = load i32, ptr %[[VAL_158]], align 4
+// CHECK2-NEXT: store i32 %[[VAL_191]], ptr %[[VAL_155]], align 4
+// CHECK2-NEXT: br label %[[VAL_192:.*]]
+// CHECK2: omp.inner.for.cond: ; preds = %[[VAL_193:.*]], %[[VAL_188]]
+// CHECK2-NEXT: %[[VAL_194:.*]] = load i32, ptr %[[VAL_155]], align 4, !llvm.access.group !5
+// CHECK2-NEXT: %[[VAL_195:.*]] = load i32, ptr %[[VAL_159]], align 4, !llvm.access.group !5
+// CHECK2-NEXT: %[[VAL_196:.*]] = icmp sle i32 %[[VAL_194]], %[[VAL_195]]
+// CHECK2-NEXT: br i1 %[[VAL_196]], label %[[VAL_197:.*]], label %[[VAL_198:.*]]
+// CHECK2: omp.inner.for.body: ; preds = %[[VAL_192]]
+// CHECK2-NEXT: %[[VAL_199:.*]] = load i32, ptr %[[VAL_155]], align 4, !llvm.access.group !5
+// CHECK2-NEXT: %[[VAL_200:.*]] = sdiv i32 %[[VAL_199]], 10
+// CHECK2-NEXT: %[[VAL_201:.*]] = mul nsw i32 %[[VAL_200]], 1
+// CHECK2-NEXT: %[[VAL_202:.*]] = add nsw i32 0, %[[VAL_201]]
+// CHECK2-NEXT: store i32 %[[VAL_202]], ptr %[[VAL_164]], align 4, !llvm.access.group !5
+// CHECK2-NEXT: %[[VAL_203:.*]] = load i32, ptr %[[VAL_155]], align 4, !llvm.access.group !5
+// CHECK2-NEXT: %[[VAL_204:.*]] = load i32, ptr %[[VAL_155]], align 4, !llvm.access.group !5
+// CHECK2-NEXT: %[[VAL_205:.*]] = sdiv i32 %[[VAL_204]], 10
+// CHECK2-NEXT: %[[VAL_206:.*]] = mul nsw i32 %[[VAL_205]], 10
+// CHECK2-NEXT: %[[VAL_207:.*]] = sub nsw i32 %[[VAL_203]], %[[VAL_206]]
+// CHECK2-NEXT: %[[VAL_208:.*]] = mul nsw i32 %[[VAL_207]], 1
+// CHECK2-NEXT: %[[VAL_209:.*]] = add nsw i32 0, %[[VAL_208]]
+// CHECK2-NEXT: store i32 %[[VAL_209]], ptr %[[VAL_162]], align 4, !llvm.access.group !5
+// CHECK2-NEXT: %[[VAL_210:.*]] = load i32, ptr %[[VAL_164]], align 4, !llvm.access.group !5
+// CHECK2-NEXT: %[[VAL_211:.*]] = sitofp i32 %[[VAL_210]] to float
+// CHECK2-NEXT: %[[VAL_212:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_163]], i32 0, i32 0
+// CHECK2-NEXT: %[[VAL_213:.*]] = load float, ptr %[[VAL_212]], align 4, !llvm.access.group !5
+// CHECK2-NEXT: %[[VAL_214:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_163]], i32 0, i32 1
+// CHECK2-NEXT: %[[VAL_215:.*]] = load float, ptr %[[VAL_214]], align 4, !llvm.access.group !5
+// CHECK2-NEXT: %[[VAL_216:.*]] = fadd float %[[VAL_213]], %[[VAL_211]]
+// CHECK2-NEXT: %[[VAL_217:.*]] = fadd float %[[VAL_215]], 0.000000e+00
+// CHECK2-NEXT: %[[VAL_218:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_163]], i32 0, i32 0
+// CHECK2-NEXT: %[[VAL_219:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_163]], i32 0, i32 1
+// CHECK2-NEXT: store float %[[VAL_216]], ptr %[[VAL_218]], align 4, !llvm.access.group !5
+// CHECK2-NEXT: store float %[[VAL_217]], ptr %[[VAL_219]], align 4, !llvm.access.group !5
+// CHECK2-NEXT: br label %[[VAL_220:.*]]
+// CHECK2: omp.body.continue: ; preds = %[[VAL_197]]
+// CHECK2-NEXT: br label %[[VAL_193]]
+// CHECK2: omp.inner.for.inc: ; preds = %[[VAL_220]]
+// CHECK2-NEXT: %[[VAL_221:.*]] = load i32, ptr %[[VAL_155]], align 4, !llvm.access.group !5
+// CHECK2-NEXT: %[[VAL_222:.*]] = add nsw i32 %[[VAL_221]], 1
+// CHECK2-NEXT: store i32 %[[VAL_222]], ptr %[[VAL_155]], align 4, !llvm.access.group !5
+// CHECK2-NEXT: br label %[[VAL_192]], !llvm.loop !6
+// CHECK2: omp.inner.for.end: ; preds = %[[VAL_192]]
+// CHECK2-NEXT: br label %[[VAL_223:.*]]
+// CHECK2: omp.loop.exit: ; preds = %[[VAL_198]]
+// CHECK2-NEXT: call void @__kmpc_for_static_fini(ptr @2, i32 %[[VAL_182]])
+// CHECK2-NEXT: %[[VAL_224:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_166]], i32 0, i32 0
+// CHECK2-NEXT: store ptr %[[VAL_163]], ptr %[[VAL_224]], align 4
+// CHECK2-NEXT: %[[VAL_225:.*]] = call i32 @__kmpc_reduce(ptr @3, i32 %[[VAL_182]], i32 1, i32 4, ptr %[[VAL_166]], ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3foov_l31.omp_outlined.omp_outlined.omp.reduction.reduction_func, ptr @.gomp_critical_user_.reduction.var)
+// CHECK2-NEXT: switch i32 %[[VAL_225]], label %[[VAL_226:.*]] [
+// CHECK2-NEXT: i32 1, label %[[VAL_227:.*]]
+// CHECK2-NEXT: i32 2, label %[[VAL_228:.*]]
+// CHECK2-NEXT: ]
+// CHECK2: .omp.reduction.case1: ; preds = %[[VAL_223]]
+// CHECK2-NEXT: %[[VAL_229:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_176]], i32 0, i32 0
+// CHECK2-NEXT: %[[VAL_230:.*]] = load float, ptr %[[VAL_229]], align 4
+// CHECK2-NEXT: %[[VAL_231:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_176]], i32 0, i32 1
+// CHECK2-NEXT: %[[VAL_232:.*]] = load float, ptr %[[VAL_231]], align 4
+// CHECK2-NEXT: %[[VAL_233:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_163]], i32 0, i32 0
+// CHECK2-NEXT: %[[VAL_234:.*]] = load float, ptr %[[VAL_233]], align 4
+// CHECK2-NEXT: %[[VAL_235:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_163]], i32 0, i32 1
+// CHECK2-NEXT: %[[VAL_236:.*]] = load float, ptr %[[VAL_235]], align 4
+// CHECK2-NEXT: %[[VAL_237:.*]] = fadd float %[[VAL_230]], %[[VAL_234]]
+// CHECK2-NEXT: %[[VAL_238:.*]] = fadd float %[[VAL_232]], %[[VAL_236]]
+// CHECK2-NEXT: %[[VAL_239:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_176]], i32 0, i32 0
+// CHECK2-NEXT: %[[VAL_240:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_176]], i32 0, i32 1
+// CHECK2-NEXT: store float %[[VAL_237]], ptr %[[VAL_239]], align 4
+// CHECK2-NEXT: store float %[[VAL_238]], ptr %[[VAL_240]], align 4
+// CHECK2-NEXT: call void @__kmpc_end_reduce(ptr @3, i32 %[[VAL_182]], ptr @.gomp_critical_user_.reduction.var)
+// CHECK2-NEXT: br label %[[VAL_226]]
+// CHECK2: .omp.reduction.case2: ; preds = %[[VAL_223]]
+// CHECK2-NEXT: %[[VAL_241:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_163]], i32 0, i32 0
+// CHECK2-NEXT: %[[VAL_242:.*]] = load float, ptr %[[VAL_241]], align 4
+// CHECK2-NEXT: %[[VAL_243:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_163]], i32 0, i32 1
+// CHECK2-NEXT: %[[VAL_244:.*]] = load float, ptr %[[VAL_243]], align 4
+// CHECK2-NEXT: call void @__atomic_load(i32 noundef 8, ptr noundef %[[VAL_176]], ptr noundef %[[VAL_167]], i32 noundef 0)
+// CHECK2-NEXT: br label %[[VAL_245:.*]]
+// CHECK2: atomic_cont: ; preds = %[[VAL_245]], %[[VAL_228]]
+// CHECK2-NEXT: %[[VAL_246:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_167]], i32 0, i32 0
+// CHECK2-NEXT: %[[VAL_247:.*]] = load float, ptr %[[VAL_246]], align 4
+// CHECK2-NEXT: %[[VAL_248:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_167]], i32 0, i32 1
+// CHECK2-NEXT: %[[VAL_249:.*]] = load float, ptr %[[VAL_248]], align 4
+// CHECK2-NEXT: %[[VAL_250:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_169]], i32 0, i32 0
+// CHECK2-NEXT: %[[VAL_251:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_169]], i32 0, i32 1
+// CHECK2-NEXT: store float %[[VAL_247]], ptr %[[VAL_250]], align 4
+// CHECK2-NEXT: store float %[[VAL_249]], ptr %[[VAL_251]], align 4
+// CHECK2-NEXT: %[[VAL_252:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_169]], i32 0, i32 0
+// CHECK2-NEXT: %[[VAL_253:.*]] = load float, ptr %[[VAL_252]], align 4
+// CHECK2-NEXT: %[[VAL_254:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_169]], i32 0, i32 1
+// CHECK2-NEXT: %[[VAL_255:.*]] = load float, ptr %[[VAL_254]], align 4
+// CHECK2-NEXT: %[[VAL_256:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_163]], i32 0, i32 0
+// CHECK2-NEXT: %[[VAL_257:.*]] = load float, ptr %[[VAL_256]], align 4
+// CHECK2-NEXT: %[[VAL_258:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_163]], i32 0, i32 1
+// CHECK2-NEXT: %[[VAL_259:.*]] = load float, ptr %[[VAL_258]], align 4
+// CHECK2-NEXT: %[[VAL_260:.*]] = fadd float %[[VAL_253]], %[[VAL_257]]
+// CHECK2-NEXT: %[[VAL_261:.*]] = fadd float %[[VAL_255]], %[[VAL_259]]
+// CHECK2-NEXT: %[[VAL_262:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_168]], i32 0, i32 0
+// CHECK2-NEXT: %[[VAL_263:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_168]], i32 0, i32 1
+// CHECK2-NEXT: store float %[[VAL_260]], ptr %[[VAL_262]], align 4
+// CHECK2-NEXT: store float %[[VAL_261]], ptr %[[VAL_263]], align 4
+// CHECK2-NEXT: %[[VAL_264:.*]] = call noundef zeroext i1 @__atomic_compare_exchange(i32 noundef 8, ptr noundef %[[VAL_176]], ptr noundef %[[VAL_167]], ptr noundef %[[VAL_168]], i32 noundef 0, i32 noundef 0)
+// CHECK2-NEXT: br i1 %[[VAL_264]], label %[[VAL_265:.*]], label %[[VAL_245]]
+// CHECK2: atomic_exit: ; preds = %[[VAL_245]]
+// CHECK2-NEXT: call void @__kmpc_end_reduce(ptr @3, i32 %[[VAL_182]], ptr @.gomp_critical_user_.reduction.var)
+// CHECK2-NEXT: br label %[[VAL_226]]
+// CHECK2: .omp.reduction.default: ; preds = %[[VAL_265]], %[[VAL_227]], %[[VAL_223]]
+// CHECK2-NEXT: %[[VAL_266:.*]] = load i32, ptr %[[VAL_161]], align 4
+// CHECK2-NEXT: %[[VAL_267:.*]] = icmp ne i32 %[[VAL_266]], 0
+// CHECK2-NEXT: br i1 %[[VAL_267]], label %[[VAL_268:.*]], label %[[VAL_269:.*]]
+// CHECK2: .omp.lastprivate.then: ; preds = %[[VAL_226]]
+// CHECK2-NEXT: store i32 10, ptr %[[VAL_162]], align 4
+// CHECK2-NEXT: %[[VAL_270:.*]] = load i32, ptr %[[VAL_162]], align 4
+// CHECK2-NEXT: store i32 %[[VAL_270]], ptr %[[VAL_153]], align 4
+// CHECK2-NEXT: br label %[[VAL_269]]
+// CHECK2: .omp.lastprivate.done: ; preds = %[[VAL_268]], %[[VAL_226]]
+// CHECK2-NEXT: ret void
+//
+//
+// CHECK2-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3foov_l31.omp_outlined.omp_outlined.omp.reduction.reduction_func
+// CHECK2-SAME: (ptr noundef [[TMP0:%.*]], ptr noundef [[TMP1:%.*]]) #[[ATTR3:[0-9]+]] {
+// CHECK2-NEXT: entry:
+// CHECK2-NEXT: %[[VAL_271:.*]] = alloca ptr, align 4
+// CHECK2-NEXT: %[[VAL_272:.*]] = alloca ptr, align 4
+// CHECK2-NEXT: store ptr %[[VAL_273:.*]], ptr %[[VAL_271]], align 4
+// CHECK2-NEXT: store ptr %[[VAL_274:.*]], ptr %[[VAL_272]], align 4
+// CHECK2-NEXT: %[[VAL_275:.*]] = load ptr, ptr %[[VAL_271]], align 4
+// CHECK2-NEXT: %[[VAL_276:.*]] = load ptr, ptr %[[VAL_272]], align 4
+// CHECK2-NEXT: %[[VAL_277:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_276]], i32 0, i32 0
+// CHECK2-NEXT: %[[VAL_278:.*]] = load ptr, ptr %[[VAL_277]], align 4
+// CHECK2-NEXT: %[[VAL_279:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_275]], i32 0, i32 0
+// CHECK2-NEXT: %[[VAL_280:.*]] = load ptr, ptr %[[VAL_279]], align 4
+// CHECK2-NEXT: %[[VAL_281:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_280]], i32 0, i32 0
+// CHECK2-NEXT: %[[VAL_282:.*]] = load float, ptr %[[VAL_281]], align 4
+// CHECK2-NEXT: %[[VAL_283:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_280]], i32 0, i32 1
+// CHECK2-NEXT: %[[VAL_284:.*]] = load float, ptr %[[VAL_283]], align 4
+// CHECK2-NEXT: %[[VAL_285:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_278]], i32 0, i32 0
+// CHECK2-NEXT: %[[VAL_286:.*]] = load float, ptr %[[VAL_285]], align 4
+// CHECK2-NEXT: %[[VAL_287:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_278]], i32 0, i32 1
+// CHECK2-NEXT: %[[VAL_288:.*]] = load float, ptr %[[VAL_287]], align 4
+// CHECK2-NEXT: %[[VAL_289:.*]] = fadd float %[[VAL_282]], %[[VAL_286]]
+// CHECK2-NEXT: %[[VAL_290:.*]] = fadd float %[[VAL_284]], %[[VAL_288]]
+// CHECK2-NEXT: %[[VAL_291:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_280]], i32 0, i32 0
+// CHECK2-NEXT: %[[VAL_292:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_280]], i32 0, i32 1
+// CHECK2-NEXT: store float %[[VAL_289]], ptr %[[VAL_291]], align 4
+// CHECK2-NEXT: store float %[[VAL_290]], ptr %[[VAL_292]], align 4
+// CHECK2-NEXT: ret void
+//
+//
+// CHECK2-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3foov_l31.omp_outlined.omp.reduction.reduction_func
+// CHECK2-SAME: (ptr noundef [[TMP0:%.*]], ptr noundef [[TMP1:%.*]]) #[[ATTR3:[0-9]+]] {
+// CHECK2-NEXT: entry:
+// CHECK2-NEXT: %[[VAL_293:.*]] = alloca ptr, align 4
+// CHECK2-NEXT: %[[VAL_294:.*]] = alloca ptr, align 4
+// CHECK2-NEXT: store ptr %[[VAL_295:.*]], ptr %[[VAL_293]], align 4
+// CHECK2-NEXT: store ptr %[[VAL_296:.*]], ptr %[[VAL_294]], align 4
+// CHECK2-NEXT: %[[VAL_297:.*]] = load ptr, ptr %[[VAL_293]], align 4
+// CHECK2-NEXT: %[[VAL_298:.*]] = load ptr, ptr %[[VAL_294]], align 4
+// CHECK2-NEXT: %[[VAL_299:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_298]], i32 0, i32 0
+// CHECK2-NEXT: %[[VAL_300:.*]] = load ptr, ptr %[[VAL_299]], align 4
+// CHECK2-NEXT: %[[VAL_301:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_297]], i32 0, i32 0
+// CHECK2-NEXT: %[[VAL_302:.*]] = load ptr, ptr %[[VAL_301]], align 4
+// CHECK2-NEXT: %[[VAL_303:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_302]], i32 0, i32 0
+// CHECK2-NEXT: %[[VAL_304:.*]] = load float, ptr %[[VAL_303]], align 4
+// CHECK2-NEXT: %[[VAL_305:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_302]], i32 0, i32 1
+// CHECK2-NEXT: %[[VAL_306:.*]] = load float, ptr %[[VAL_305]], align 4
+// CHECK2-NEXT: %[[VAL_307:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_300]], i32 0, i32 0
+// CHECK2-NEXT: %[[VAL_308:.*]] = load float, ptr %[[VAL_307]], align 4
+// CHECK2-NEXT: %[[VAL_309:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_300]], i32 0, i32 1
+// CHECK2-NEXT: %[[VAL_310:.*]] = load float, ptr %[[VAL_309]], align 4
+// CHECK2-NEXT: %[[VAL_311:.*]] = fadd float %[[VAL_304]], %[[VAL_308]]
+// CHECK2-NEXT: %[[VAL_312:.*]] = fadd float %[[VAL_306]], %[[VAL_310]]
+// CHECK2-NEXT: %[[VAL_313:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_302]], i32 0, i32 0
+// CHECK2-NEXT: %[[VAL_314:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_302]], i32 0, i32 1
+// CHECK2-NEXT: store float %[[VAL_311]], ptr %[[VAL_313]], align 4
+// CHECK2-NEXT: store float %[[VAL_312]], ptr %[[VAL_314]], align 4
+// CHECK2-NEXT: ret void
\ No newline at end of file
>From 4a82cae0d3235c3cca1d8fa86f87f186d57acc6f Mon Sep 17 00:00:00 2001
From: Akash Banerjee <Akash.Banerjee at amd.com>
Date: Wed, 20 Mar 2024 17:05:01 +0000
Subject: [PATCH 15/15] Rebased
---
clang/test/OpenMP/reduction_complex.c | 151 +++++++++---------
.../llvm/Frontend/OpenMP/OMPIRBuilder.h | 7 +-
llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp | 93 +++++------
.../OpenMP/OpenMPToLLVMIRTranslation.cpp | 40 ++---
4 files changed, 141 insertions(+), 150 deletions(-)
diff --git a/clang/test/OpenMP/reduction_complex.c b/clang/test/OpenMP/reduction_complex.c
index cbc5b14087239d..cb7c20926d4ea2 100644
--- a/clang/test/OpenMP/reduction_complex.c
+++ b/clang/test/OpenMP/reduction_complex.c
@@ -171,7 +171,7 @@ int foo() {
// CHECK: omp.inner.for.end: ; preds = %[[VAL_55]]
// CHECK-NEXT: br label %[[VAL_91:.*]]
// CHECK: omp.loop.exit: ; preds = %[[VAL_60]]
-// CHECK-NEXT: call void @__kmpc_for_static_fini(ptr @3, i32 %[[VAL_45]])
+// CHECK-NEXT: call void @__kmpc_distribute_static_fini(ptr @2, i32 %[[VAL_45]])
// CHECK-NEXT: %[[VAL_92:.*]] = load i32, ptr %[[VAL_29]], align 4
// CHECK-NEXT: %[[VAL_93:.*]] = icmp ne i32 %[[VAL_92]], 0
// CHECK-NEXT: br i1 %[[VAL_93]], label %[[VAL_94:.*]], label %[[VAL_95:.*]]
@@ -500,7 +500,7 @@ int foo() {
// CHECK1: omp.inner.for.end: ; preds = %[[VAL_87]]
// CHECK1-NEXT: br label %[[VAL_103:.*]]
// CHECK1: omp.loop.exit: ; preds = %[[VAL_93]]
-// CHECK1-NEXT: call void @__kmpc_for_static_fini(ptr @2, i32 %[[VAL_77]])
+// CHECK1-NEXT: call void @__kmpc_for_static_fini(ptr @1, i32 %[[VAL_77]])
// CHECK1-NEXT: %[[VAL_104:.*]] = load i32, ptr %[[VAL_60]], align 4
// CHECK1-NEXT: %[[VAL_105:.*]] = icmp ne i32 %[[VAL_104]], 0
// CHECK1-NEXT: br i1 %[[VAL_105]], label %[[VAL_106:.*]], label %[[VAL_107:.*]]
@@ -764,7 +764,7 @@ int foo() {
// CHECK2: omp.inner.for.end: ; preds = %[[VAL_87]]
// CHECK2-NEXT: br label %[[VAL_101:.*]]
// CHECK2: omp.loop.exit: ; preds = %[[VAL_93]]
-// CHECK2-NEXT: call void @__kmpc_for_static_fini(ptr @2, i32 %[[VAL_77]])
+// CHECK2-NEXT: call void @__kmpc_for_static_fini(ptr @1, i32 %[[VAL_77]])
// CHECK2-NEXT: %[[VAL_102:.*]] = load i32, ptr %[[VAL_60]], align 4
// CHECK2-NEXT: %[[VAL_103:.*]] = icmp ne i32 %[[VAL_102]], 0
// CHECK2-NEXT: br i1 %[[VAL_103]], label %[[VAL_104:.*]], label %[[VAL_105:.*]]
@@ -923,92 +923,91 @@ int foo() {
// CHECK2-NEXT: %[[VAL_214:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_163]], i32 0, i32 1
// CHECK2-NEXT: %[[VAL_215:.*]] = load float, ptr %[[VAL_214]], align 4, !llvm.access.group !5
// CHECK2-NEXT: %[[VAL_216:.*]] = fadd float %[[VAL_213]], %[[VAL_211]]
-// CHECK2-NEXT: %[[VAL_217:.*]] = fadd float %[[VAL_215]], 0.000000e+00
-// CHECK2-NEXT: %[[VAL_218:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_163]], i32 0, i32 0
-// CHECK2-NEXT: %[[VAL_219:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_163]], i32 0, i32 1
-// CHECK2-NEXT: store float %[[VAL_216]], ptr %[[VAL_218]], align 4, !llvm.access.group !5
-// CHECK2-NEXT: store float %[[VAL_217]], ptr %[[VAL_219]], align 4, !llvm.access.group !5
-// CHECK2-NEXT: br label %[[VAL_220:.*]]
+// CHECK2-NEXT: %[[VAL_217:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_163]], i32 0, i32 0
+// CHECK2-NEXT: %[[VAL_218:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_163]], i32 0, i32 1
+// CHECK2-NEXT: store float %[[VAL_216]], ptr %[[VAL_217]], align 4, !llvm.access.group !5
+// CHECK2-NEXT: store float %[[VAL_215]], ptr %[[VAL_218]], align 4, !llvm.access.group !5
+// CHECK2-NEXT: br label %[[VAL_219:.*]]
// CHECK2: omp.body.continue: ; preds = %[[VAL_197]]
// CHECK2-NEXT: br label %[[VAL_193]]
-// CHECK2: omp.inner.for.inc: ; preds = %[[VAL_220]]
-// CHECK2-NEXT: %[[VAL_221:.*]] = load i32, ptr %[[VAL_155]], align 4, !llvm.access.group !5
-// CHECK2-NEXT: %[[VAL_222:.*]] = add nsw i32 %[[VAL_221]], 1
-// CHECK2-NEXT: store i32 %[[VAL_222]], ptr %[[VAL_155]], align 4, !llvm.access.group !5
+// CHECK2: omp.inner.for.inc: ; preds = %[[VAL_219]]
+// CHECK2-NEXT: %[[VAL_220:.*]] = load i32, ptr %[[VAL_155]], align 4, !llvm.access.group !5
+// CHECK2-NEXT: %[[VAL_221:.*]] = add nsw i32 %[[VAL_220]], 1
+// CHECK2-NEXT: store i32 %[[VAL_221]], ptr %[[VAL_155]], align 4, !llvm.access.group !5
// CHECK2-NEXT: br label %[[VAL_192]], !llvm.loop !6
// CHECK2: omp.inner.for.end: ; preds = %[[VAL_192]]
-// CHECK2-NEXT: br label %[[VAL_223:.*]]
+// CHECK2-NEXT: br label %[[VAL_222:.*]]
// CHECK2: omp.loop.exit: ; preds = %[[VAL_198]]
// CHECK2-NEXT: call void @__kmpc_for_static_fini(ptr @2, i32 %[[VAL_182]])
-// CHECK2-NEXT: %[[VAL_224:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_166]], i32 0, i32 0
-// CHECK2-NEXT: store ptr %[[VAL_163]], ptr %[[VAL_224]], align 4
-// CHECK2-NEXT: %[[VAL_225:.*]] = call i32 @__kmpc_reduce(ptr @3, i32 %[[VAL_182]], i32 1, i32 4, ptr %[[VAL_166]], ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3foov_l31.omp_outlined.omp_outlined.omp.reduction.reduction_func, ptr @.gomp_critical_user_.reduction.var)
-// CHECK2-NEXT: switch i32 %[[VAL_225]], label %[[VAL_226:.*]] [
-// CHECK2-NEXT: i32 1, label %[[VAL_227:.*]]
-// CHECK2-NEXT: i32 2, label %[[VAL_228:.*]]
+// CHECK2-NEXT: %[[VAL_223:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_166]], i32 0, i32 0
+// CHECK2-NEXT: store ptr %[[VAL_163]], ptr %[[VAL_223]], align 4
+// CHECK2-NEXT: %[[VAL_224:.*]] = call i32 @__kmpc_reduce(ptr @3, i32 %[[VAL_182]], i32 1, i32 4, ptr %[[VAL_166]], ptr @__omp_offloading_1030b_4868a89__Z3foov_l31.omp_outlined.omp_outlined.omp.reduction.reduction_func, ptr @.gomp_critical_user_.reduction.var)
+// CHECK2-NEXT: switch i32 %[[VAL_224]], label %[[VAL_225:.*]] [
+// CHECK2-NEXT: i32 1, label %[[VAL_226:.*]]
+// CHECK2-NEXT: i32 2, label %[[VAL_227:.*]]
// CHECK2-NEXT: ]
-// CHECK2: .omp.reduction.case1: ; preds = %[[VAL_223]]
-// CHECK2-NEXT: %[[VAL_229:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_176]], i32 0, i32 0
-// CHECK2-NEXT: %[[VAL_230:.*]] = load float, ptr %[[VAL_229]], align 4
-// CHECK2-NEXT: %[[VAL_231:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_176]], i32 0, i32 1
-// CHECK2-NEXT: %[[VAL_232:.*]] = load float, ptr %[[VAL_231]], align 4
-// CHECK2-NEXT: %[[VAL_233:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_163]], i32 0, i32 0
-// CHECK2-NEXT: %[[VAL_234:.*]] = load float, ptr %[[VAL_233]], align 4
-// CHECK2-NEXT: %[[VAL_235:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_163]], i32 0, i32 1
-// CHECK2-NEXT: %[[VAL_236:.*]] = load float, ptr %[[VAL_235]], align 4
-// CHECK2-NEXT: %[[VAL_237:.*]] = fadd float %[[VAL_230]], %[[VAL_234]]
-// CHECK2-NEXT: %[[VAL_238:.*]] = fadd float %[[VAL_232]], %[[VAL_236]]
-// CHECK2-NEXT: %[[VAL_239:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_176]], i32 0, i32 0
-// CHECK2-NEXT: %[[VAL_240:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_176]], i32 0, i32 1
+// CHECK2: .omp.reduction.case1: ; preds = %[[VAL_222]]
+// CHECK2-NEXT: %[[VAL_228:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_176]], i32 0, i32 0
+// CHECK2-NEXT: %[[VAL_229:.*]] = load float, ptr %[[VAL_228]], align 4
+// CHECK2-NEXT: %[[VAL_230:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_176]], i32 0, i32 1
+// CHECK2-NEXT: %[[VAL_231:.*]] = load float, ptr %[[VAL_230]], align 4
+// CHECK2-NEXT: %[[VAL_232:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_163]], i32 0, i32 0
+// CHECK2-NEXT: %[[VAL_233:.*]] = load float, ptr %[[VAL_232]], align 4
+// CHECK2-NEXT: %[[VAL_234:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_163]], i32 0, i32 1
+// CHECK2-NEXT: %[[VAL_235:.*]] = load float, ptr %[[VAL_234]], align 4
+// CHECK2-NEXT: %[[VAL_236:.*]] = fadd float %[[VAL_229]], %[[VAL_233]]
+// CHECK2-NEXT: %[[VAL_237:.*]] = fadd float %[[VAL_231]], %[[VAL_235]]
+// CHECK2-NEXT: %[[VAL_238:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_176]], i32 0, i32 0
+// CHECK2-NEXT: %[[VAL_239:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_176]], i32 0, i32 1
+// CHECK2-NEXT: store float %[[VAL_236]], ptr %[[VAL_238]], align 4
// CHECK2-NEXT: store float %[[VAL_237]], ptr %[[VAL_239]], align 4
-// CHECK2-NEXT: store float %[[VAL_238]], ptr %[[VAL_240]], align 4
// CHECK2-NEXT: call void @__kmpc_end_reduce(ptr @3, i32 %[[VAL_182]], ptr @.gomp_critical_user_.reduction.var)
-// CHECK2-NEXT: br label %[[VAL_226]]
-// CHECK2: .omp.reduction.case2: ; preds = %[[VAL_223]]
-// CHECK2-NEXT: %[[VAL_241:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_163]], i32 0, i32 0
-// CHECK2-NEXT: %[[VAL_242:.*]] = load float, ptr %[[VAL_241]], align 4
-// CHECK2-NEXT: %[[VAL_243:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_163]], i32 0, i32 1
-// CHECK2-NEXT: %[[VAL_244:.*]] = load float, ptr %[[VAL_243]], align 4
+// CHECK2-NEXT: br label %[[VAL_225]]
+// CHECK2: .omp.reduction.case2: ; preds = %[[VAL_222]]
+// CHECK2-NEXT: %[[VAL_240:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_163]], i32 0, i32 0
+// CHECK2-NEXT: %[[VAL_241:.*]] = load float, ptr %[[VAL_240]], align 4
+// CHECK2-NEXT: %[[VAL_242:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_163]], i32 0, i32 1
+// CHECK2-NEXT: %[[VAL_243:.*]] = load float, ptr %[[VAL_242]], align 4
// CHECK2-NEXT: call void @__atomic_load(i32 noundef 8, ptr noundef %[[VAL_176]], ptr noundef %[[VAL_167]], i32 noundef 0)
-// CHECK2-NEXT: br label %[[VAL_245:.*]]
-// CHECK2: atomic_cont: ; preds = %[[VAL_245]], %[[VAL_228]]
-// CHECK2-NEXT: %[[VAL_246:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_167]], i32 0, i32 0
-// CHECK2-NEXT: %[[VAL_247:.*]] = load float, ptr %[[VAL_246]], align 4
-// CHECK2-NEXT: %[[VAL_248:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_167]], i32 0, i32 1
-// CHECK2-NEXT: %[[VAL_249:.*]] = load float, ptr %[[VAL_248]], align 4
-// CHECK2-NEXT: %[[VAL_250:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_169]], i32 0, i32 0
-// CHECK2-NEXT: %[[VAL_251:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_169]], i32 0, i32 1
-// CHECK2-NEXT: store float %[[VAL_247]], ptr %[[VAL_250]], align 4
-// CHECK2-NEXT: store float %[[VAL_249]], ptr %[[VAL_251]], align 4
-// CHECK2-NEXT: %[[VAL_252:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_169]], i32 0, i32 0
-// CHECK2-NEXT: %[[VAL_253:.*]] = load float, ptr %[[VAL_252]], align 4
-// CHECK2-NEXT: %[[VAL_254:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_169]], i32 0, i32 1
-// CHECK2-NEXT: %[[VAL_255:.*]] = load float, ptr %[[VAL_254]], align 4
-// CHECK2-NEXT: %[[VAL_256:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_163]], i32 0, i32 0
-// CHECK2-NEXT: %[[VAL_257:.*]] = load float, ptr %[[VAL_256]], align 4
-// CHECK2-NEXT: %[[VAL_258:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_163]], i32 0, i32 1
-// CHECK2-NEXT: %[[VAL_259:.*]] = load float, ptr %[[VAL_258]], align 4
-// CHECK2-NEXT: %[[VAL_260:.*]] = fadd float %[[VAL_253]], %[[VAL_257]]
-// CHECK2-NEXT: %[[VAL_261:.*]] = fadd float %[[VAL_255]], %[[VAL_259]]
-// CHECK2-NEXT: %[[VAL_262:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_168]], i32 0, i32 0
-// CHECK2-NEXT: %[[VAL_263:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_168]], i32 0, i32 1
+// CHECK2-NEXT: br label %[[VAL_244:.*]]
+// CHECK2: atomic_cont: ; preds = %[[VAL_244]], %[[VAL_227]]
+// CHECK2-NEXT: %[[VAL_245:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_167]], i32 0, i32 0
+// CHECK2-NEXT: %[[VAL_246:.*]] = load float, ptr %[[VAL_245]], align 4
+// CHECK2-NEXT: %[[VAL_247:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_167]], i32 0, i32 1
+// CHECK2-NEXT: %[[VAL_248:.*]] = load float, ptr %[[VAL_247]], align 4
+// CHECK2-NEXT: %[[VAL_249:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_169]], i32 0, i32 0
+// CHECK2-NEXT: %[[VAL_250:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_169]], i32 0, i32 1
+// CHECK2-NEXT: store float %[[VAL_246]], ptr %[[VAL_249]], align 4
+// CHECK2-NEXT: store float %[[VAL_248]], ptr %[[VAL_250]], align 4
+// CHECK2-NEXT: %[[VAL_251:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_169]], i32 0, i32 0
+// CHECK2-NEXT: %[[VAL_252:.*]] = load float, ptr %[[VAL_251]], align 4
+// CHECK2-NEXT: %[[VAL_253:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_169]], i32 0, i32 1
+// CHECK2-NEXT: %[[VAL_254:.*]] = load float, ptr %[[VAL_253]], align 4
+// CHECK2-NEXT: %[[VAL_255:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_163]], i32 0, i32 0
+// CHECK2-NEXT: %[[VAL_256:.*]] = load float, ptr %[[VAL_255]], align 4
+// CHECK2-NEXT: %[[VAL_257:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_163]], i32 0, i32 1
+// CHECK2-NEXT: %[[VAL_258:.*]] = load float, ptr %[[VAL_257]], align 4
+// CHECK2-NEXT: %[[VAL_259:.*]] = fadd float %[[VAL_252]], %[[VAL_256]]
+// CHECK2-NEXT: %[[VAL_260:.*]] = fadd float %[[VAL_254]], %[[VAL_258]]
+// CHECK2-NEXT: %[[VAL_261:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_168]], i32 0, i32 0
+// CHECK2-NEXT: %[[VAL_262:.*]] = getelementptr inbounds { float, float }, ptr %[[VAL_168]], i32 0, i32 1
+// CHECK2-NEXT: store float %[[VAL_259]], ptr %[[VAL_261]], align 4
// CHECK2-NEXT: store float %[[VAL_260]], ptr %[[VAL_262]], align 4
-// CHECK2-NEXT: store float %[[VAL_261]], ptr %[[VAL_263]], align 4
-// CHECK2-NEXT: %[[VAL_264:.*]] = call noundef zeroext i1 @__atomic_compare_exchange(i32 noundef 8, ptr noundef %[[VAL_176]], ptr noundef %[[VAL_167]], ptr noundef %[[VAL_168]], i32 noundef 0, i32 noundef 0)
-// CHECK2-NEXT: br i1 %[[VAL_264]], label %[[VAL_265:.*]], label %[[VAL_245]]
-// CHECK2: atomic_exit: ; preds = %[[VAL_245]]
+// CHECK2-NEXT: %[[VAL_263:.*]] = call noundef zeroext i1 @__atomic_compare_exchange(i32 noundef 8, ptr noundef %[[VAL_176]], ptr noundef %[[VAL_167]], ptr noundef %[[VAL_168]], i32 noundef 0, i32 noundef 0)
+// CHECK2-NEXT: br i1 %[[VAL_263]], label %[[VAL_264:.*]], label %[[VAL_244]]
+// CHECK2: atomic_exit: ; preds = %[[VAL_244]]
// CHECK2-NEXT: call void @__kmpc_end_reduce(ptr @3, i32 %[[VAL_182]], ptr @.gomp_critical_user_.reduction.var)
-// CHECK2-NEXT: br label %[[VAL_226]]
-// CHECK2: .omp.reduction.default: ; preds = %[[VAL_265]], %[[VAL_227]], %[[VAL_223]]
-// CHECK2-NEXT: %[[VAL_266:.*]] = load i32, ptr %[[VAL_161]], align 4
-// CHECK2-NEXT: %[[VAL_267:.*]] = icmp ne i32 %[[VAL_266]], 0
-// CHECK2-NEXT: br i1 %[[VAL_267]], label %[[VAL_268:.*]], label %[[VAL_269:.*]]
-// CHECK2: .omp.lastprivate.then: ; preds = %[[VAL_226]]
+// CHECK2-NEXT: br label %[[VAL_225]]
+// CHECK2: .omp.reduction.default: ; preds = %[[VAL_264]], %[[VAL_226]], %[[VAL_222]]
+// CHECK2-NEXT: %[[VAL_265:.*]] = load i32, ptr %[[VAL_161]], align 4
+// CHECK2-NEXT: %[[VAL_266:.*]] = icmp ne i32 %[[VAL_265]], 0
+// CHECK2-NEXT: br i1 %[[VAL_266]], label %[[VAL_267:.*]], label %[[VAL_268:.*]]
+// CHECK2: .omp.lastprivate.then: ; preds = %[[VAL_225]]
// CHECK2-NEXT: store i32 10, ptr %[[VAL_162]], align 4
-// CHECK2-NEXT: %[[VAL_270:.*]] = load i32, ptr %[[VAL_162]], align 4
-// CHECK2-NEXT: store i32 %[[VAL_270]], ptr %[[VAL_153]], align 4
-// CHECK2-NEXT: br label %[[VAL_269]]
-// CHECK2: .omp.lastprivate.done: ; preds = %[[VAL_268]], %[[VAL_226]]
+// CHECK2-NEXT: %[[VAL_269:.*]] = load i32, ptr %[[VAL_162]], align 4
+// CHECK2-NEXT: store i32 %[[VAL_269]], ptr %[[VAL_153]], align 4
+// CHECK2-NEXT: br label %[[VAL_268]]
+// CHECK2: .omp.lastprivate.done: ; preds = %[[VAL_267]], %[[VAL_225]]
// CHECK2-NEXT: ret void
//
//
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
index 6383bc1775a51e..ec5ddb117d5828 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
@@ -1913,9 +1913,10 @@ class OpenMPIRBuilder {
InsertPointTy createReductions(const LocationDescription &Loc,
InsertPointTy AllocaIP,
ArrayRef<ReductionInfo> ReductionInfos,
- bool IsNoWait = false, bool IsByRef = false,
- bool IsTeamsReduction = false,
- bool HasDistribute = false);
+ bool IsNoWait = false, bool IsByRef = false);
+
+ ///}
+
/// Return the insertion point used by the underlying IRBuilder.
InsertPointTy getInsertionPoint() { return Builder.saveIP(); }
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index f538296804b9e9..cf49d7039c0e96 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -3507,61 +3507,24 @@ static Function *getFreshReductionFunc(Module &M) {
".omp.reduction.func", &M);
}
-static void populateReductionFunction(
- Function *ReductionFunc,
- ArrayRef<OpenMPIRBuilder::ReductionInfo> ReductionInfos,
- IRBuilder<> &Builder) {
- Module *Module = ReductionFunc->getParent();
- BasicBlock *ReductionFuncBlock =
- BasicBlock::Create(Module->getContext(), "", ReductionFunc);
- Builder.SetInsertPoint(ReductionFuncBlock);
- Value *LHSArrayPtr = nullptr;
- Value *RHSArrayPtr = nullptr;
- LHSArrayPtr = ReductionFunc->getArg(0);
- RHSArrayPtr = ReductionFunc->getArg(1);
-
- unsigned NumReductions = ReductionInfos.size();
- Type *RedArrayTy = ArrayType::get(Builder.getPtrTy(), NumReductions);
-
- for (auto En : enumerate(ReductionInfos)) {
- const OpenMPIRBuilder::ReductionInfo &RI = En.value();
- Value *LHSI8PtrPtr = Builder.CreateConstInBoundsGEP2_64(
- RedArrayTy, LHSArrayPtr, 0, En.index());
- Value *LHSI8Ptr = Builder.CreateLoad(Builder.getPtrTy(), LHSI8PtrPtr);
- Value *LHSPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
- LHSI8Ptr, RI.Variable->getType());
- Value *LHS = Builder.CreateLoad(RI.ElementType, LHSPtr);
- Value *RHSI8PtrPtr = Builder.CreateConstInBoundsGEP2_64(
- RedArrayTy, RHSArrayPtr, 0, En.index());
- Value *RHSI8Ptr = Builder.CreateLoad(Builder.getPtrTy(), RHSI8PtrPtr);
- Value *RHSPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
- RHSI8Ptr, RI.PrivateVariable->getType());
- Value *RHS = Builder.CreateLoad(RI.ElementType, RHSPtr);
- Value *Reduced;
- Builder.restoreIP(RI.ReductionGen(Builder.saveIP(), LHS, RHS, Reduced));
- if (!Builder.GetInsertBlock())
- return;
- Builder.CreateStore(Reduced, LHSPtr);
- }
- Builder.CreateRetVoid();
-}
-
OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductions(
const LocationDescription &Loc, InsertPointTy AllocaIP,
- ArrayRef<ReductionInfo> ReductionInfos, bool IsNoWait, bool IsByRef,
- bool IsTeamsReduction, bool HasDistribute) {
- if (Config.isGPU())
- return createReductionsGPU(Loc, AllocaIP, Builder.saveIP(), ReductionInfos,
- IsNoWait, IsTeamsReduction, HasDistribute);
-
- checkReductionInfos(ReductionInfos, /*IsGPU*/ false);
+ ArrayRef<ReductionInfo> ReductionInfos, bool IsNoWait, bool IsByRef) {
+ for (const ReductionInfo &RI : ReductionInfos) {
+ (void)RI;
+ assert(RI.Variable && "expected non-null variable");
+ assert(RI.PrivateVariable && "expected non-null private variable");
+ assert(RI.ReductionGen && "expected non-null reduction generator callback");
+ assert(RI.Variable->getType() == RI.PrivateVariable->getType() &&
+ "expected variables and their private equivalents to have the same "
+ "type");
+ assert(RI.Variable->getType()->isPointerTy() &&
+ "expected variables to be pointers");
+ }
if (!updateToLocation(Loc))
return InsertPointTy();
- if (ReductionInfos.size() == 0)
- return Builder.saveIP();
-
BasicBlock *InsertBlock = Loc.IP.getBlock();
BasicBlock *ContinuationBlock =
InsertBlock->splitBasicBlock(Loc.IP.getPoint(), "reduce.finalize");
@@ -3602,7 +3565,7 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductions(
const DataLayout &DL = Module->getDataLayout();
unsigned RedArrayByteSize = DL.getTypeStoreSize(RedArrayTy);
Constant *RedArraySize = Builder.getInt64(RedArrayByteSize);
- Function *ReductionFunc = getFreshReductionFunc(M);
+ Function *ReductionFunc = getFreshReductionFunc(*Module);
Value *Lock = getOMPCriticalRegionLock(".reduction");
Function *ReduceFunc = getOrCreateRuntimeFunctionPtr(
IsNoWait ? RuntimeFunction::OMPRTL___kmpc_reduce_nowait
@@ -3681,7 +3644,35 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductions(
// Populate the outlined reduction function using the elementwise reduction
// function. Partial values are extracted from the type-erased array of
// pointers to private variables.
- populateReductionFunction(ReductionFunc, ReductionInfos, Builder);
+ BasicBlock *ReductionFuncBlock =
+ BasicBlock::Create(Module->getContext(), "", ReductionFunc);
+ Builder.SetInsertPoint(ReductionFuncBlock);
+ Value *LHSArrayPtr = ReductionFunc->getArg(0);
+ Value *RHSArrayPtr = ReductionFunc->getArg(1);
+
+ for (auto En : enumerate(ReductionInfos)) {
+ const ReductionInfo &RI = En.value();
+ Value *LHSI8PtrPtr = Builder.CreateConstInBoundsGEP2_64(
+ RedArrayTy, LHSArrayPtr, 0, En.index());
+ Value *LHSI8Ptr = Builder.CreateLoad(Builder.getPtrTy(), LHSI8PtrPtr);
+ Value *LHSPtr = Builder.CreateBitCast(LHSI8Ptr, RI.Variable->getType());
+ Value *LHS = Builder.CreateLoad(RI.ElementType, LHSPtr);
+ Value *RHSI8PtrPtr = Builder.CreateConstInBoundsGEP2_64(
+ RedArrayTy, RHSArrayPtr, 0, En.index());
+ Value *RHSI8Ptr = Builder.CreateLoad(Builder.getPtrTy(), RHSI8PtrPtr);
+ Value *RHSPtr =
+ Builder.CreateBitCast(RHSI8Ptr, RI.PrivateVariable->getType());
+ Value *RHS = Builder.CreateLoad(RI.ElementType, RHSPtr);
+ Value *Reduced;
+ Builder.restoreIP(RI.ReductionGen(Builder.saveIP(), LHS, RHS, Reduced));
+ if (!Builder.GetInsertBlock())
+ return InsertPointTy();
+ // store is inside of the reduction region when using by-ref
+ if (!IsByRef)
+ Builder.CreateStore(Reduced, LHSPtr);
+ }
+ Builder.CreateRetVoid();
+
Builder.SetInsertPoint(ContinuationBlock);
return Builder.saveIP();
}
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index c99c5c72f578e7..1e94da8defd723 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -3057,26 +3057,26 @@ LogicalResult OpenMPDialectLLVMIRTranslationInterface::amendOperation(
moduleTranslation);
return failure();
})
- .Case(
- "omp.requires",
- [&](Attribute attr) {
- if (auto requiresAttr = attr.dyn_cast<omp::ClauseRequiresAttr>()) {
- using Requires = omp::ClauseRequires;
- Requires flags = requiresAttr.getValue();
- llvm::OpenMPIRBuilderConfig &config =
- moduleTranslation.getOpenMPBuilder()->Config;
- config.setHasRequiresReverseOffload(
- bitEnumContainsAll(flags, Requires::reverse_offload));
- config.setHasRequiresUnifiedAddress(
- bitEnumContainsAll(flags, Requires::unified_address));
- config.setHasRequiresUnifiedSharedMemory(
- bitEnumContainsAll(flags, Requires::unified_shared_memory));
- config.setHasRequiresDynamicAllocators(
- bitEnumContainsAll(flags, Requires::dynamic_allocators));
- return convertRequiresAttr(*op, requiresAttr, moduleTranslation);
- }
- return failure();
- })
+ .Case("omp.requires",
+ [&](Attribute attr) {
+ if (auto requiresAttr =
+ attr.dyn_cast<omp::ClauseRequiresAttr>()) {
+ using Requires = omp::ClauseRequires;
+ Requires flags = requiresAttr.getValue();
+ llvm::OpenMPIRBuilderConfig &config =
+ moduleTranslation.getOpenMPBuilder()->Config;
+ config.setHasRequiresReverseOffload(
+ bitEnumContainsAll(flags, Requires::reverse_offload));
+ config.setHasRequiresUnifiedAddress(
+ bitEnumContainsAll(flags, Requires::unified_address));
+ config.setHasRequiresUnifiedSharedMemory(
+ bitEnumContainsAll(flags, Requires::unified_shared_memory));
+ config.setHasRequiresDynamicAllocators(
+ bitEnumContainsAll(flags, Requires::dynamic_allocators));
+ return success();
+ }
+ return failure();
+ })
.Default([](Attribute) {
// Fall through for omp attributes that do not require lowering.
return success();
More information about the cfe-commits
mailing list