[clang] 3401a5f - [OpenMP][OMPIRBuilder] Migrate emitOffloadingArrays and EmitNonContiguousDescriptor from Clang
Akash Banerjee via cfe-commits
cfe-commits at lists.llvm.org
Mon Jun 12 07:43:18 PDT 2023
Author: Akash Banerjee
Date: 2023-06-12T15:43:09+01:00
New Revision: 3401a5f7584a2f12a90a7538aee2ae37038c82a9
URL: https://github.com/llvm/llvm-project/commit/3401a5f7584a2f12a90a7538aee2ae37038c82a9
DIFF: https://github.com/llvm/llvm-project/commit/3401a5f7584a2f12a90a7538aee2ae37038c82a9.diff
LOG: [OpenMP][OMPIRBuilder] Migrate emitOffloadingArrays and EmitNonContiguousDescriptor from Clang
This patch migrates the emitOffloadingArrays and EmitNonContiguousDescriptor functions from Clang codegen to OpenMPIRBuilder.
Reviewed By: jdoerfert
Differential Revision: https://reviews.llvm.org/D149872
Added:
Modified:
clang/lib/CodeGen/CGOpenMPRuntime.cpp
llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
Removed:
################################################################################
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index 099c706e84ab2..994189277aff6 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -8969,74 +8969,6 @@ class MappableExprsHandler {
};
} // anonymous namespace
-static void emitNonContiguousDescriptor(
- CodeGenFunction &CGF, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo,
- CGOpenMPRuntime::TargetDataInfo &Info) {
- CodeGenModule &CGM = CGF.CGM;
- MappableExprsHandler::MapCombinedInfoTy::StructNonContiguousInfo
- &NonContigInfo = CombinedInfo.NonContigInfo;
-
- // Build an array of struct descriptor_dim and then assign it to
- // offload_args.
- //
- // struct descriptor_dim {
- // uint64_t offset;
- // uint64_t count;
- // uint64_t stride
- // };
- ASTContext &C = CGF.getContext();
- QualType Int64Ty = C.getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/0);
- RecordDecl *RD;
- RD = C.buildImplicitRecord("descriptor_dim");
- RD->startDefinition();
- addFieldToRecordDecl(C, RD, Int64Ty);
- addFieldToRecordDecl(C, RD, Int64Ty);
- addFieldToRecordDecl(C, RD, Int64Ty);
- RD->completeDefinition();
- QualType DimTy = C.getRecordType(RD);
-
- enum { OffsetFD = 0, CountFD, StrideFD };
- // We need two index variable here since the size of "Dims" is the same as the
- // size of Components, however, the size of offset, count, and stride is equal
- // to the size of base declaration that is non-contiguous.
- for (unsigned I = 0, L = 0, E = NonContigInfo.Dims.size(); I < E; ++I) {
- // Skip emitting ir if dimension size is 1 since it cannot be
- // non-contiguous.
- if (NonContigInfo.Dims[I] == 1)
- continue;
- llvm::APInt Size(/*numBits=*/32, NonContigInfo.Dims[I]);
- QualType ArrayTy =
- C.getConstantArrayType(DimTy, Size, nullptr, ArrayType::Normal, 0);
- Address DimsAddr = CGF.CreateMemTemp(ArrayTy, "dims");
- for (unsigned II = 0, EE = NonContigInfo.Dims[I]; II < EE; ++II) {
- unsigned RevIdx = EE - II - 1;
- LValue DimsLVal = CGF.MakeAddrLValue(
- CGF.Builder.CreateConstArrayGEP(DimsAddr, II), DimTy);
- // Offset
- LValue OffsetLVal = CGF.EmitLValueForField(
- DimsLVal, *std::next(RD->field_begin(), OffsetFD));
- CGF.EmitStoreOfScalar(NonContigInfo.Offsets[L][RevIdx], OffsetLVal);
- // Count
- LValue CountLVal = CGF.EmitLValueForField(
- DimsLVal, *std::next(RD->field_begin(), CountFD));
- CGF.EmitStoreOfScalar(NonContigInfo.Counts[L][RevIdx], CountLVal);
- // Stride
- LValue StrideLVal = CGF.EmitLValueForField(
- DimsLVal, *std::next(RD->field_begin(), StrideFD));
- CGF.EmitStoreOfScalar(NonContigInfo.Strides[L][RevIdx], StrideLVal);
- }
- // args[I] = &dims
- Address DAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
- DimsAddr, CGM.Int8PtrTy, CGM.Int8Ty);
- llvm::Value *P = CGF.Builder.CreateConstInBoundsGEP2_32(
- llvm::ArrayType::get(CGM.VoidPtrTy, Info.NumberOfPtrs),
- Info.RTArgs.PointersArray, 0, I);
- Address PAddr(P, CGM.VoidPtrTy, CGF.getPointerAlign());
- CGF.Builder.CreateStore(DAddr.getPointer(), PAddr);
- ++L;
- }
-}
-
// Try to extract the base declaration from a `this->x` expression if possible.
static ValueDecl *getDeclFromThisExpr(const Expr *E) {
if (!E)
@@ -9099,190 +9031,42 @@ static void emitOffloadingArrays(
Info.clearArrayInfo();
Info.NumberOfPtrs = CombinedInfo.BasePointers.size();
- if (Info.NumberOfPtrs) {
- // Detect if we have any capture size requiring runtime evaluation of the
- // size so that a constant array could be eventually used.
-
- llvm::APInt PointerNumAP(32, Info.NumberOfPtrs, /*isSigned=*/true);
- QualType PointerArrayType = Ctx.getConstantArrayType(
- Ctx.VoidPtrTy, PointerNumAP, nullptr, ArrayType::Normal,
- /*IndexTypeQuals=*/0);
-
- Info.RTArgs.BasePointersArray =
- CGF.CreateMemTemp(PointerArrayType, ".offload_baseptrs").getPointer();
- Info.RTArgs.PointersArray =
- CGF.CreateMemTemp(PointerArrayType, ".offload_ptrs").getPointer();
- Address MappersArray =
- CGF.CreateMemTemp(PointerArrayType, ".offload_mappers");
- Info.RTArgs.MappersArray = MappersArray.getPointer();
-
- // If we don't have any VLA types or other types that require runtime
- // evaluation, we can use a constant array for the map sizes, otherwise we
- // need to fill up the arrays as we do for the pointers.
- QualType Int64Ty =
- Ctx.getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1);
- SmallVector<llvm::Constant *> ConstSizes(
- CombinedInfo.Sizes.size(), llvm::ConstantInt::get(CGF.Int64Ty, 0));
- llvm::SmallBitVector RuntimeSizes(CombinedInfo.Sizes.size());
- for (unsigned I = 0, E = CombinedInfo.Sizes.size(); I < E; ++I) {
- if (auto *CI = dyn_cast<llvm::Constant>(CombinedInfo.Sizes[I])) {
- if (!isa<llvm::ConstantExpr>(CI) && !isa<llvm::GlobalValue>(CI)) {
- if (IsNonContiguous &&
- static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
- CombinedInfo.Types[I] &
- OpenMPOffloadMappingFlags::OMP_MAP_NON_CONTIG))
- ConstSizes[I] = llvm::ConstantInt::get(
- CGF.Int64Ty, CombinedInfo.NonContigInfo.Dims[I]);
- else
- ConstSizes[I] = CI;
- continue;
- }
- }
- RuntimeSizes.set(I);
- }
+ using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
+ InsertPointTy AllocaIP(CGF.AllocaInsertPt->getParent(),
+ CGF.AllocaInsertPt->getIterator());
+ InsertPointTy CodeGenIP(CGF.Builder.GetInsertBlock(),
+ CGF.Builder.GetInsertPoint());
- if (RuntimeSizes.all()) {
- QualType SizeArrayType = Ctx.getConstantArrayType(
- Int64Ty, PointerNumAP, nullptr, ArrayType::Normal,
- /*IndexTypeQuals=*/0);
- Info.RTArgs.SizesArray =
- CGF.CreateMemTemp(SizeArrayType, ".offload_sizes").getPointer();
- } else {
- auto *SizesArrayInit = llvm::ConstantArray::get(
- llvm::ArrayType::get(CGM.Int64Ty, ConstSizes.size()), ConstSizes);
- std::string Name = CGM.getOpenMPRuntime().getName({"offload_sizes"});
- auto *SizesArrayGbl = new llvm::GlobalVariable(
- CGM.getModule(), SizesArrayInit->getType(), /*isConstant=*/true,
- llvm::GlobalValue::PrivateLinkage, SizesArrayInit, Name);
- SizesArrayGbl->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
- if (RuntimeSizes.any()) {
- QualType SizeArrayType = Ctx.getConstantArrayType(
- Int64Ty, PointerNumAP, nullptr, ArrayType::Normal,
- /*IndexTypeQuals=*/0);
- Address Buffer = CGF.CreateMemTemp(SizeArrayType, ".offload_sizes");
- llvm::Value *GblConstPtr =
- CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
- SizesArrayGbl, CGM.Int64Ty->getPointerTo());
- CGF.Builder.CreateMemCpy(
- Buffer,
- Address(GblConstPtr, CGM.Int64Ty,
- CGM.getNaturalTypeAlignment(Ctx.getIntTypeForBitwidth(
- /*DestWidth=*/64, /*Signed=*/false))),
- CGF.getTypeSize(SizeArrayType));
- Info.RTArgs.SizesArray = Buffer.getPointer();
- } else {
- Info.RTArgs.SizesArray = SizesArrayGbl;
- }
- }
-
- // The map types are always constant so we don't need to generate code to
- // fill arrays. Instead, we create an array constant.
- SmallVector<uint64_t, 4> Mapping;
- for (auto mapFlag : CombinedInfo.Types)
- Mapping.push_back(
- static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
- mapFlag));
- std::string MaptypesName =
- CGM.getOpenMPRuntime().getName({"offload_maptypes"});
- auto *MapTypesArrayGbl =
- OMPBuilder.createOffloadMaptypes(Mapping, MaptypesName);
- Info.RTArgs.MapTypesArray = MapTypesArrayGbl;
-
- // The information types are only built if there is debug information
- // requested.
- if (CGM.getCodeGenOpts().getDebugInfo() ==
- llvm::codegenoptions::NoDebugInfo) {
- Info.RTArgs.MapNamesArray = llvm::Constant::getNullValue(
- llvm::Type::getInt8Ty(CGF.Builder.getContext())->getPointerTo());
- } else {
- auto fillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) {
- return emitMappingInformation(CGF, OMPBuilder, MapExpr);
- };
- SmallVector<llvm::Constant *, 4> InfoMap(CombinedInfo.Exprs.size());
- llvm::transform(CombinedInfo.Exprs, InfoMap.begin(), fillInfoMap);
- std::string MapnamesName =
- CGM.getOpenMPRuntime().getName({"offload_mapnames"});
- auto *MapNamesArrayGbl =
- OMPBuilder.createOffloadMapnames(InfoMap, MapnamesName);
- Info.RTArgs.MapNamesArray = MapNamesArrayGbl;
- }
-
- // If there's a present map type modifier, it must not be applied to the end
- // of a region, so generate a separate map type array in that case.
- if (Info.separateBeginEndCalls()) {
- bool EndMapTypesDiffer = false;
- for (uint64_t &Type : Mapping) {
- if (Type &
- static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
- OpenMPOffloadMappingFlags::OMP_MAP_PRESENT)) {
- Type &=
- ~static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
- OpenMPOffloadMappingFlags::OMP_MAP_PRESENT);
- EndMapTypesDiffer = true;
- }
- }
- if (EndMapTypesDiffer) {
- MapTypesArrayGbl =
- OMPBuilder.createOffloadMaptypes(Mapping, MaptypesName);
- Info.RTArgs.MapTypesArrayEnd = MapTypesArrayGbl;
- }
- }
+ auto fillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) {
+ return emitMappingInformation(CGF, OMPBuilder, MapExpr);
+ };
+ if (CGM.getCodeGenOpts().getDebugInfo() !=
+ llvm::codegenoptions::NoDebugInfo) {
+ CombinedInfo.Names.resize(CombinedInfo.Exprs.size());
+ llvm::transform(CombinedInfo.Exprs, CombinedInfo.Names.begin(),
+ fillInfoMap);
+ }
- for (unsigned I = 0; I < Info.NumberOfPtrs; ++I) {
- llvm::Value *BPVal = CombinedInfo.BasePointers[I];
- llvm::Value *BP = CGF.Builder.CreateConstInBoundsGEP2_32(
- llvm::ArrayType::get(CGM.VoidPtrTy, Info.NumberOfPtrs),
- Info.RTArgs.BasePointersArray, 0, I);
- BP = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
- BP, BPVal->getType()->getPointerTo(/*AddrSpace=*/0));
+ auto DeviceAddrCB = [&](unsigned int I, llvm::Value *BP, llvm::Value *BPVal) {
+ if (const ValueDecl *DevVD = CombinedInfo.DevicePtrDecls[I]) {
Address BPAddr(BP, BPVal->getType(),
Ctx.getTypeAlignInChars(Ctx.VoidPtrTy));
- CGF.Builder.CreateStore(BPVal, BPAddr);
-
- if (Info.requiresDevicePointerInfo())
- if (const ValueDecl *DevVD = CombinedInfo.DevicePtrDecls[I])
- Info.CaptureDeviceAddrMap.try_emplace(DevVD, BPAddr);
-
- llvm::Value *PVal = CombinedInfo.Pointers[I];
- llvm::Value *P = CGF.Builder.CreateConstInBoundsGEP2_32(
- llvm::ArrayType::get(CGM.VoidPtrTy, Info.NumberOfPtrs),
- Info.RTArgs.PointersArray, 0, I);
- P = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
- P, PVal->getType()->getPointerTo(/*AddrSpace=*/0));
- Address PAddr(P, PVal->getType(), Ctx.getTypeAlignInChars(Ctx.VoidPtrTy));
- CGF.Builder.CreateStore(PVal, PAddr);
-
- if (RuntimeSizes.test(I)) {
- llvm::Value *S = CGF.Builder.CreateConstInBoundsGEP2_32(
- llvm::ArrayType::get(CGM.Int64Ty, Info.NumberOfPtrs),
- Info.RTArgs.SizesArray,
- /*Idx0=*/0,
- /*Idx1=*/I);
- Address SAddr(S, CGM.Int64Ty, Ctx.getTypeAlignInChars(Int64Ty));
- CGF.Builder.CreateStore(CGF.Builder.CreateIntCast(CombinedInfo.Sizes[I],
- CGM.Int64Ty,
- /*isSigned=*/true),
- SAddr);
- }
-
- // Fill up the mapper array.
- llvm::Value *MFunc = llvm::ConstantPointerNull::get(CGM.VoidPtrTy);
- if (CombinedInfo.Mappers[I]) {
- MFunc = CGM.getOpenMPRuntime().getOrCreateUserDefinedMapperFunc(
- cast<OMPDeclareMapperDecl>(CombinedInfo.Mappers[I]));
- MFunc = CGF.Builder.CreatePointerCast(MFunc, CGM.VoidPtrTy);
- Info.HasMapper = true;
- }
- Address MAddr = CGF.Builder.CreateConstArrayGEP(MappersArray, I);
- CGF.Builder.CreateStore(MFunc, MAddr);
+ Info.CaptureDeviceAddrMap.try_emplace(DevVD, BPAddr);
}
- }
-
- if (!IsNonContiguous || CombinedInfo.NonContigInfo.Offsets.empty() ||
- Info.NumberOfPtrs == 0)
- return;
+ };
- emitNonContiguousDescriptor(CGF, CombinedInfo, Info);
+ auto CustomMapperCB = [&](unsigned int I) {
+ llvm::Value *MFunc = nullptr;
+ if (CombinedInfo.Mappers[I]) {
+ Info.HasMapper = true;
+ MFunc = CGF.CGM.getOpenMPRuntime().getOrCreateUserDefinedMapperFunc(
+ cast<OMPDeclareMapperDecl>(CombinedInfo.Mappers[I]));
+ }
+ return MFunc;
+ };
+ OMPBuilder.emitOffloadingArrays(AllocaIP, CodeGenIP, CombinedInfo, Info,
+ /*IsNonContiguous=*/true, DeviceAddrCB,
+ CustomMapperCB);
}
/// Check for inner distribute directive.
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
index 4ae9192b0b60d..ea459b2d8fe37 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
@@ -1627,6 +1627,21 @@ class OpenMPIRBuilder {
bool EmitDebug = false,
bool ForEndCall = false);
+ /// Emit an array of struct descriptors to be assigned to the offload args.
+ void emitNonContiguousDescriptor(InsertPointTy AllocaIP,
+ InsertPointTy CodeGenIP,
+ MapInfosTy &CombinedInfo,
+ TargetDataInfo &Info);
+
+ /// Emit the arrays used to pass the captures and map information to the
+ /// offloading runtime library. If there is no map or capture information,
+ /// return nullptr by reference.
+ void emitOffloadingArrays(
+ InsertPointTy AllocaIP, InsertPointTy CodeGenIP, MapInfosTy &CombinedInfo,
+ TargetDataInfo &Info, bool IsNonContiguous = false,
+ function_ref<void(unsigned int, Value *, Value *)> DeviceAddrCB = nullptr,
+ function_ref<Value *(unsigned int)> CustomMapperCB = nullptr);
+
/// Creates offloading entry for the provided entry ID \a ID, address \a
/// Addr, size \a Size, and flags \a Flags.
void createOffloadEntry(Constant *ID, Constant *Addr, uint64_t Size,
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 245f952906b7a..cc0cab480be43 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -4419,6 +4419,256 @@ void OpenMPIRBuilder::emitOffloadingArraysArgument(IRBuilderBase &Builder,
Builder.CreatePointerCast(Info.RTArgs.MappersArray, VoidPtrPtrTy);
}
+void OpenMPIRBuilder::emitNonContiguousDescriptor(InsertPointTy AllocaIP,
+ InsertPointTy CodeGenIP,
+ MapInfosTy &CombinedInfo,
+ TargetDataInfo &Info) {
+ MapInfosTy::StructNonContiguousInfo &NonContigInfo =
+ CombinedInfo.NonContigInfo;
+
+ // Build an array of struct descriptor_dim and then assign it to
+ // offload_args.
+ //
+ // struct descriptor_dim {
+ // uint64_t offset;
+ // uint64_t count;
+ // uint64_t stride
+ // };
+ Type *Int64Ty = Builder.getInt64Ty();
+ StructType *DimTy = StructType::create(
+ M.getContext(), ArrayRef<Type *>({Int64Ty, Int64Ty, Int64Ty}),
+ "struct.descriptor_dim");
+
+ enum { OffsetFD = 0, CountFD, StrideFD };
+ // We need two index variable here since the size of "Dims" is the same as
+ // the size of Components, however, the size of offset, count, and stride is
+ // equal to the size of base declaration that is non-contiguous.
+ for (unsigned I = 0, L = 0, E = NonContigInfo.Dims.size(); I < E; ++I) {
+ // Skip emitting ir if dimension size is 1 since it cannot be
+ // non-contiguous.
+ if (NonContigInfo.Dims[I] == 1)
+ continue;
+ Builder.restoreIP(AllocaIP);
+ ArrayType *ArrayTy = ArrayType::get(DimTy, NonContigInfo.Dims[I]);
+ AllocaInst *DimsAddr =
+ Builder.CreateAlloca(ArrayTy, /* ArraySize = */ nullptr, "dims");
+ Builder.restoreIP(CodeGenIP);
+ for (unsigned II = 0, EE = NonContigInfo.Dims[I]; II < EE; ++II) {
+ unsigned RevIdx = EE - II - 1;
+ Value *DimsLVal = Builder.CreateInBoundsGEP(
+ DimsAddr->getAllocatedType(), DimsAddr,
+ {Builder.getInt64(0), Builder.getInt64(II)});
+ // Offset
+ Value *OffsetLVal = Builder.CreateStructGEP(DimTy, DimsLVal, OffsetFD);
+ Builder.CreateAlignedStore(
+ NonContigInfo.Offsets[L][RevIdx], OffsetLVal,
+ M.getDataLayout().getPrefTypeAlign(OffsetLVal->getType()));
+ // Count
+ Value *CountLVal = Builder.CreateStructGEP(DimTy, DimsLVal, CountFD);
+ Builder.CreateAlignedStore(
+ NonContigInfo.Counts[L][RevIdx], CountLVal,
+ M.getDataLayout().getPrefTypeAlign(CountLVal->getType()));
+ // Stride
+ Value *StrideLVal = Builder.CreateStructGEP(DimTy, DimsLVal, StrideFD);
+ Builder.CreateAlignedStore(
+ NonContigInfo.Strides[L][RevIdx], StrideLVal,
+ M.getDataLayout().getPrefTypeAlign(CountLVal->getType()));
+ }
+ // args[I] = &dims
+ Builder.restoreIP(CodeGenIP);
+ Value *DAddr = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ DimsAddr, Builder.getInt8PtrTy());
+ Value *P = Builder.CreateConstInBoundsGEP2_32(
+ ArrayType::get(Builder.getInt8PtrTy(), Info.NumberOfPtrs),
+ Info.RTArgs.PointersArray, 0, I);
+ Builder.CreateAlignedStore(
+ DAddr, P, M.getDataLayout().getPrefTypeAlign(Builder.getInt8PtrTy()));
+ ++L;
+ }
+}
+
+void OpenMPIRBuilder::emitOffloadingArrays(
+ InsertPointTy AllocaIP, InsertPointTy CodeGenIP, MapInfosTy &CombinedInfo,
+ TargetDataInfo &Info, bool IsNonContiguous,
+ function_ref<void(unsigned int, Value *, Value *)> DeviceAddrCB,
+ function_ref<Value *(unsigned int)> CustomMapperCB) {
+
+ // Reset the array information.
+ Info.clearArrayInfo();
+ Info.NumberOfPtrs = CombinedInfo.BasePointers.size();
+
+ if (Info.NumberOfPtrs == 0)
+ return;
+
+ Builder.restoreIP(AllocaIP);
+ // Detect if we have any capture size requiring runtime evaluation of the
+ // size so that a constant array could be eventually used.
+ ArrayType *PointerArrayType =
+ ArrayType::get(Builder.getInt8PtrTy(), Info.NumberOfPtrs);
+
+ Info.RTArgs.BasePointersArray = Builder.CreateAlloca(
+ PointerArrayType, /* ArraySize = */ nullptr, ".offload_baseptrs");
+
+ Info.RTArgs.PointersArray = Builder.CreateAlloca(
+ PointerArrayType, /* ArraySize = */ nullptr, ".offload_ptrs");
+ AllocaInst *MappersArray = Builder.CreateAlloca(
+ PointerArrayType, /* ArraySize = */ nullptr, ".offload_mappers");
+ Info.RTArgs.MappersArray = MappersArray;
+
+ // If we don't have any VLA types or other types that require runtime
+ // evaluation, we can use a constant array for the map sizes, otherwise we
+ // need to fill up the arrays as we do for the pointers.
+ Type *Int64Ty = Builder.getInt64Ty();
+ SmallVector<Constant *> ConstSizes(CombinedInfo.Sizes.size(),
+ ConstantInt::get(Builder.getInt64Ty(), 0));
+ SmallBitVector RuntimeSizes(CombinedInfo.Sizes.size());
+ for (unsigned I = 0, E = CombinedInfo.Sizes.size(); I < E; ++I) {
+ if (auto *CI = dyn_cast<Constant>(CombinedInfo.Sizes[I])) {
+ if (!isa<ConstantExpr>(CI) && !isa<GlobalValue>(CI)) {
+ if (IsNonContiguous &&
+ static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
+ CombinedInfo.Types[I] &
+ OpenMPOffloadMappingFlags::OMP_MAP_NON_CONTIG))
+ ConstSizes[I] = ConstantInt::get(Builder.getInt64Ty(),
+ CombinedInfo.NonContigInfo.Dims[I]);
+ else
+ ConstSizes[I] = CI;
+ continue;
+ }
+ }
+ RuntimeSizes.set(I);
+ }
+
+ if (RuntimeSizes.all()) {
+ ArrayType *SizeArrayType = ArrayType::get(Int64Ty, Info.NumberOfPtrs);
+ Info.RTArgs.SizesArray = Builder.CreateAlloca(
+ SizeArrayType, /* ArraySize = */ nullptr, ".offload_sizes");
+ Builder.restoreIP(CodeGenIP);
+ } else {
+ auto *SizesArrayInit = ConstantArray::get(
+ ArrayType::get(Int64Ty, ConstSizes.size()), ConstSizes);
+ std::string Name = createPlatformSpecificName({"offload_sizes"});
+ auto *SizesArrayGbl =
+ new GlobalVariable(M, SizesArrayInit->getType(), /*isConstant=*/true,
+ GlobalValue::PrivateLinkage, SizesArrayInit, Name);
+ SizesArrayGbl->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
+
+ if (!RuntimeSizes.any()) {
+ Info.RTArgs.SizesArray = SizesArrayGbl;
+ } else {
+ unsigned IndexSize = M.getDataLayout().getIndexSizeInBits(0);
+ Align OffloadSizeAlign = M.getDataLayout().getABIIntegerTypeAlignment(64);
+ ArrayType *SizeArrayType = ArrayType::get(Int64Ty, Info.NumberOfPtrs);
+ AllocaInst *Buffer = Builder.CreateAlloca(
+ SizeArrayType, /* ArraySize = */ nullptr, ".offload_sizes");
+ Buffer->setAlignment(OffloadSizeAlign);
+ Builder.restoreIP(CodeGenIP);
+ Value *GblConstPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ SizesArrayGbl, Int64Ty->getPointerTo());
+ Builder.CreateMemCpy(
+ Buffer, M.getDataLayout().getPrefTypeAlign(Buffer->getType()),
+ GblConstPtr, OffloadSizeAlign,
+ Builder.getIntN(
+ IndexSize,
+ Buffer->getAllocationSize(M.getDataLayout())->getFixedValue()));
+
+ Info.RTArgs.SizesArray = Buffer;
+ }
+ Builder.restoreIP(CodeGenIP);
+ }
+
+ // The map types are always constant so we don't need to generate code to
+ // fill arrays. Instead, we create an array constant.
+ SmallVector<uint64_t, 4> Mapping;
+ for (auto mapFlag : CombinedInfo.Types)
+ Mapping.push_back(
+ static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
+ mapFlag));
+ std::string MaptypesName = createPlatformSpecificName({"offload_maptypes"});
+ auto *MapTypesArrayGbl = createOffloadMaptypes(Mapping, MaptypesName);
+ Info.RTArgs.MapTypesArray = MapTypesArrayGbl;
+
+ // The information types are only built if provided.
+ if (!CombinedInfo.Names.empty()) {
+ std::string MapnamesName = createPlatformSpecificName({"offload_mapnames"});
+ auto *MapNamesArrayGbl =
+ createOffloadMapnames(CombinedInfo.Names, MapnamesName);
+ Info.RTArgs.MapNamesArray = MapNamesArrayGbl;
+ }
+
+ // If there's a present map type modifier, it must not be applied to the end
+ // of a region, so generate a separate map type array in that case.
+ if (Info.separateBeginEndCalls()) {
+ bool EndMapTypesDiffer = false;
+ for (uint64_t &Type : Mapping) {
+ if (Type & static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
+ OpenMPOffloadMappingFlags::OMP_MAP_PRESENT)) {
+ Type &= ~static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
+ OpenMPOffloadMappingFlags::OMP_MAP_PRESENT);
+ EndMapTypesDiffer = true;
+ }
+ }
+ if (EndMapTypesDiffer) {
+ MapTypesArrayGbl = createOffloadMaptypes(Mapping, MaptypesName);
+ Info.RTArgs.MapTypesArrayEnd = MapTypesArrayGbl;
+ }
+ }
+
+ for (unsigned I = 0; I < Info.NumberOfPtrs; ++I) {
+ Value *BPVal = CombinedInfo.BasePointers[I];
+ Value *BP = Builder.CreateConstInBoundsGEP2_32(
+ ArrayType::get(Builder.getInt8PtrTy(), Info.NumberOfPtrs),
+ Info.RTArgs.BasePointersArray, 0, I);
+ BP = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ BP, BPVal->getType()->getPointerTo(/*AddrSpace=*/0));
+ Builder.CreateAlignedStore(
+ BPVal, BP, M.getDataLayout().getPrefTypeAlign(Builder.getInt8PtrTy()));
+
+ if (Info.requiresDevicePointerInfo()) {
+ assert(DeviceAddrCB &&
+ "DeviceAddrCB missing for DevicePtr code generation");
+ DeviceAddrCB(I, BP, BPVal);
+ }
+
+ Value *PVal = CombinedInfo.Pointers[I];
+ Value *P = Builder.CreateConstInBoundsGEP2_32(
+ ArrayType::get(Builder.getInt8PtrTy(), Info.NumberOfPtrs),
+ Info.RTArgs.PointersArray, 0, I);
+ P = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ P, PVal->getType()->getPointerTo(/*AddrSpace=*/0));
+ // TODO: Check alignment correct.
+ Builder.CreateAlignedStore(
+ PVal, P, M.getDataLayout().getPrefTypeAlign(Builder.getInt8PtrTy()));
+
+ if (RuntimeSizes.test(I)) {
+ Value *S = Builder.CreateConstInBoundsGEP2_32(
+ ArrayType::get(Int64Ty, Info.NumberOfPtrs), Info.RTArgs.SizesArray,
+ /*Idx0=*/0,
+ /*Idx1=*/I);
+ Builder.CreateAlignedStore(
+ Builder.CreateIntCast(CombinedInfo.Sizes[I], Int64Ty,
+ /*isSigned=*/true),
+ S, M.getDataLayout().getPrefTypeAlign(Builder.getInt8PtrTy()));
+ }
+ // Fill up the mapper array.
+ unsigned IndexSize = M.getDataLayout().getIndexSizeInBits(0);
+ Value *MFunc = ConstantPointerNull::get(Builder.getInt8PtrTy());
+ if (CustomMapperCB)
+ if (Value *CustomMFunc = CustomMapperCB(I))
+ MFunc = Builder.CreatePointerCast(CustomMFunc, Builder.getInt8PtrTy());
+ Value *MAddr = Builder.CreateInBoundsGEP(
+ MappersArray->getAllocatedType(), MappersArray,
+ {Builder.getIntN(IndexSize, 0), Builder.getIntN(IndexSize, I)});
+ Builder.CreateAlignedStore(
+ MFunc, MAddr, M.getDataLayout().getPrefTypeAlign(MAddr->getType()));
+ }
+
+ if (!IsNonContiguous || CombinedInfo.NonContigInfo.Offsets.empty() ||
+ Info.NumberOfPtrs == 0)
+ return;
+ emitNonContiguousDescriptor(AllocaIP, CodeGenIP, CombinedInfo, Info);
+}
+
bool OpenMPIRBuilder::checkAndEmitFlushAfterAtomic(
const LocationDescription &Loc, llvm::AtomicOrdering AO, AtomicKind AK) {
assert(!(AO == AtomicOrdering::NotAtomic ||
More information about the cfe-commits
mailing list