[clang] [flang] [llvm] [OpenMPIRBuilder] Emit __atomic_load and __atomic_compare_exchange libcalls for complex types in atomic update (PR #92364)
via cfe-commits
cfe-commits at lists.llvm.org
Sat Jul 6 02:45:40 PDT 2024
https://github.com/NimishMishra updated https://github.com/llvm/llvm-project/pull/92364
>From 6423811853ab17f7109a4de972b66ce37573e979 Mon Sep 17 00:00:00 2001
From: Nimish Mishra <neelam.nimish at gmail.com>
Date: Sat, 6 Jul 2024 15:14:32 +0530
Subject: [PATCH] [clang][flang][OpenMPIRBuilder] Emit __atomic_load and
__atomic_compare_exchange libcalls for complex types in atomic update
---
clang/lib/CodeGen/CGAtomic.cpp | 673 +++++++++---------
.../OpenMP/atomic-update-complex.f90 | 42 ++
llvm/include/llvm/Frontend/Atomic/Atomic.h | 235 ++++++
.../llvm/Frontend/OpenMP/OMPIRBuilder.h | 24 +
llvm/lib/Frontend/Atomic/Atomic.cpp | 18 +
llvm/lib/Frontend/Atomic/CMakeLists.txt | 15 +
llvm/lib/Frontend/CMakeLists.txt | 1 +
llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp | 47 ++
8 files changed, 701 insertions(+), 354 deletions(-)
create mode 100644 flang/test/Integration/OpenMP/atomic-update-complex.f90
create mode 100644 llvm/include/llvm/Frontend/Atomic/Atomic.h
create mode 100644 llvm/lib/Frontend/Atomic/Atomic.cpp
create mode 100644 llvm/lib/Frontend/Atomic/CMakeLists.txt
diff --git a/clang/lib/CodeGen/CGAtomic.cpp b/clang/lib/CodeGen/CGAtomic.cpp
index fbf942d06ca6e..a7bbbd20cc31b 100644
--- a/clang/lib/CodeGen/CGAtomic.cpp
+++ b/clang/lib/CodeGen/CGAtomic.cpp
@@ -19,6 +19,7 @@
#include "clang/CodeGen/CGFunctionInfo.h"
#include "clang/Frontend/FrontendDiagnostic.h"
#include "llvm/ADT/DenseMap.h"
+#include "llvm/Frontend/Atomic/Atomic.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/Operator.h"
@@ -27,275 +28,322 @@ using namespace clang;
using namespace CodeGen;
namespace {
- class AtomicInfo {
- CodeGenFunction &CGF;
- QualType AtomicTy;
+class AtomicInfo : public llvm::AtomicInfo<CGBuilderBaseTy> {
+ CodeGenFunction &CGF;
+ QualType AtomicTy;
+ QualType ValueTy;
+ TypeEvaluationKind EvaluationKind;
+ LValue LVal;
+ CGBitFieldInfo BFI;
+
+public:
+ llvm::AtomicInfo<CGBuilderBaseTy> &getBase() { return *this; }
+
+ static AtomicInfo fromValue(CodeGenFunction &CGF, LValue &lvalue) {
+ if (lvalue.isSimple())
+ return fromSimple(CGF, lvalue);
+ if (lvalue.isBitField())
+ return fromBitfield(CGF, lvalue);
+ if (lvalue.isVectorElt())
+ return fromVectorElt(CGF, lvalue);
+ if (lvalue.isExtVectorElt())
+ return fromExtVectorElt(CGF, lvalue);
+ llvm_unreachable("Type incompatible with atomics");
+ }
+
+ static AtomicInfo fromSimple(CodeGenFunction &CGF, LValue &lvalue) {
+ assert(lvalue.isSimple());
+ ASTContext &C = CGF.getContext();
+
+ QualType AtomicTy = lvalue.getType();
QualType ValueTy;
- uint64_t AtomicSizeInBits;
- uint64_t ValueSizeInBits;
- CharUnits AtomicAlign;
- CharUnits ValueAlign;
- TypeEvaluationKind EvaluationKind;
- bool UseLibcall;
- LValue LVal;
- CGBitFieldInfo BFI;
- public:
- AtomicInfo(CodeGenFunction &CGF, LValue &lvalue)
- : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),
- EvaluationKind(TEK_Scalar), UseLibcall(true) {
- assert(!lvalue.isGlobalReg());
- ASTContext &C = CGF.getContext();
- if (lvalue.isSimple()) {
- AtomicTy = lvalue.getType();
- if (auto *ATy = AtomicTy->getAs<AtomicType>())
- ValueTy = ATy->getValueType();
- else
- ValueTy = AtomicTy;
- EvaluationKind = CGF.getEvaluationKind(ValueTy);
-
- uint64_t ValueAlignInBits;
- uint64_t AtomicAlignInBits;
- TypeInfo ValueTI = C.getTypeInfo(ValueTy);
- ValueSizeInBits = ValueTI.Width;
- ValueAlignInBits = ValueTI.Align;
-
- TypeInfo AtomicTI = C.getTypeInfo(AtomicTy);
- AtomicSizeInBits = AtomicTI.Width;
- AtomicAlignInBits = AtomicTI.Align;
-
- assert(ValueSizeInBits <= AtomicSizeInBits);
- assert(ValueAlignInBits <= AtomicAlignInBits);
-
- AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits);
- ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits);
- if (lvalue.getAlignment().isZero())
- lvalue.setAlignment(AtomicAlign);
-
- LVal = lvalue;
- } else if (lvalue.isBitField()) {
- ValueTy = lvalue.getType();
- ValueSizeInBits = C.getTypeSize(ValueTy);
- auto &OrigBFI = lvalue.getBitFieldInfo();
- auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment());
- AtomicSizeInBits = C.toBits(
- C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1)
- .alignTo(lvalue.getAlignment()));
- llvm::Value *BitFieldPtr = lvalue.getRawBitFieldPointer(CGF);
- auto OffsetInChars =
- (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) *
- lvalue.getAlignment();
- llvm::Value *StoragePtr = CGF.Builder.CreateConstGEP1_64(
- CGF.Int8Ty, BitFieldPtr, OffsetInChars.getQuantity());
- StoragePtr = CGF.Builder.CreateAddrSpaceCast(
- StoragePtr, CGF.UnqualPtrTy, "atomic_bitfield_base");
- BFI = OrigBFI;
- BFI.Offset = Offset;
- BFI.StorageSize = AtomicSizeInBits;
- BFI.StorageOffset += OffsetInChars;
- llvm::Type *StorageTy = CGF.Builder.getIntNTy(AtomicSizeInBits);
- LVal = LValue::MakeBitfield(
- Address(StoragePtr, StorageTy, lvalue.getAlignment()), BFI,
- lvalue.getType(), lvalue.getBaseInfo(), lvalue.getTBAAInfo());
- AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned);
- if (AtomicTy.isNull()) {
- llvm::APInt Size(
- /*numBits=*/32,
- C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity());
- AtomicTy = C.getConstantArrayType(C.CharTy, Size, nullptr,
- ArraySizeModifier::Normal,
- /*IndexTypeQuals=*/0);
- }
- AtomicAlign = ValueAlign = lvalue.getAlignment();
- } else if (lvalue.isVectorElt()) {
- ValueTy = lvalue.getType()->castAs<VectorType>()->getElementType();
- ValueSizeInBits = C.getTypeSize(ValueTy);
- AtomicTy = lvalue.getType();
- AtomicSizeInBits = C.getTypeSize(AtomicTy);
- AtomicAlign = ValueAlign = lvalue.getAlignment();
- LVal = lvalue;
- } else {
- assert(lvalue.isExtVectorElt());
- ValueTy = lvalue.getType();
- ValueSizeInBits = C.getTypeSize(ValueTy);
- AtomicTy = ValueTy = CGF.getContext().getExtVectorType(
- lvalue.getType(), cast<llvm::FixedVectorType>(
- lvalue.getExtVectorAddress().getElementType())
- ->getNumElements());
- AtomicSizeInBits = C.getTypeSize(AtomicTy);
- AtomicAlign = ValueAlign = lvalue.getAlignment();
- LVal = lvalue;
- }
- UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(
- AtomicSizeInBits, C.toBits(lvalue.getAlignment()));
- }
+ if (auto *ATy = AtomicTy->getAs<AtomicType>())
+ ValueTy = ATy->getValueType();
+ else
+ ValueTy = AtomicTy;
+ TypeEvaluationKind EvaluationKind = CGF.getEvaluationKind(ValueTy);
- QualType getAtomicType() const { return AtomicTy; }
- QualType getValueType() const { return ValueTy; }
- CharUnits getAtomicAlignment() const { return AtomicAlign; }
- uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
- uint64_t getValueSizeInBits() const { return ValueSizeInBits; }
- TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
- bool shouldUseLibcall() const { return UseLibcall; }
- const LValue &getAtomicLValue() const { return LVal; }
- llvm::Value *getAtomicPointer() const {
- if (LVal.isSimple())
- return LVal.emitRawPointer(CGF);
- else if (LVal.isBitField())
- return LVal.getRawBitFieldPointer(CGF);
- else if (LVal.isVectorElt())
- return LVal.getRawVectorPointer(CGF);
- assert(LVal.isExtVectorElt());
- return LVal.getRawExtVectorPointer(CGF);
- }
- Address getAtomicAddress() const {
- llvm::Type *ElTy;
- if (LVal.isSimple())
- ElTy = LVal.getAddress().getElementType();
- else if (LVal.isBitField())
- ElTy = LVal.getBitFieldAddress().getElementType();
- else if (LVal.isVectorElt())
- ElTy = LVal.getVectorAddress().getElementType();
- else
- ElTy = LVal.getExtVectorAddress().getElementType();
- return Address(getAtomicPointer(), ElTy, getAtomicAlignment());
- }
+ TypeInfo ValueTI = C.getTypeInfo(ValueTy);
+ auto ValueSizeInBits = ValueTI.Width;
+ auto ValueAlignInBits = ValueTI.Align;
- Address getAtomicAddressAsAtomicIntPointer() const {
- return castToAtomicIntPointer(getAtomicAddress());
- }
+ TypeInfo AtomicTI = C.getTypeInfo(AtomicTy);
+ auto AtomicSizeInBits = AtomicTI.Width;
+ auto AtomicAlignInBits = AtomicTI.Align;
- /// Is the atomic size larger than the underlying value type?
- ///
- /// Note that the absence of padding does not mean that atomic
- /// objects are completely interchangeable with non-atomic
- /// objects: we might have promoted the alignment of a type
- /// without making it bigger.
- bool hasPadding() const {
- return (ValueSizeInBits != AtomicSizeInBits);
- }
+ assert(ValueSizeInBits <= AtomicSizeInBits);
+ assert(ValueAlignInBits <= AtomicAlignInBits);
+
+ auto AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits).getAsAlign();
+ auto ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits).getAsAlign();
+ if (lvalue.getAlignment().isZero())
+ lvalue.setAlignment(CharUnits::fromQuantity(AtomicAlign));
+
+ auto LVal = lvalue;
+ auto ElTy = LVal.getAddress().getElementType();
- bool emitMemSetZeroIfNecessary() const;
+ bool UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(
+ AtomicSizeInBits, C.toBits(lvalue.getAlignment()));
+ return AtomicInfo(CGF, ElTy, LVal, AtomicTy, ValueTy, EvaluationKind, {},
+ AtomicSizeInBits, ValueSizeInBits, AtomicAlign,
+ ValueAlign, UseLibcall);
+ }
- llvm::Value *getAtomicSizeValue() const {
- CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);
- return CGF.CGM.getSize(size);
+ static AtomicInfo fromBitfield(CodeGenFunction &CGF, LValue &lvalue) {
+ assert(lvalue.isBitField());
+ ASTContext &C = CGF.getContext();
+
+ auto ValueTy = lvalue.getType();
+ auto ValueSizeInBits = C.getTypeSize(ValueTy);
+ auto &OrigBFI = lvalue.getBitFieldInfo();
+ auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment());
+ auto AtomicSizeInBits = C.toBits(
+ C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1)
+ .alignTo(lvalue.getAlignment()));
+ llvm::Value *BitFieldPtr = lvalue.getRawBitFieldPointer(CGF);
+ auto OffsetInChars =
+ (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) *
+ lvalue.getAlignment();
+ llvm::Value *StoragePtr = CGF.Builder.CreateConstGEP1_64(
+ CGF.Int8Ty, BitFieldPtr, OffsetInChars.getQuantity());
+ StoragePtr = CGF.Builder.CreateAddrSpaceCast(StoragePtr, CGF.UnqualPtrTy,
+ "atomic_bitfield_base");
+ auto BFI = OrigBFI;
+ BFI.Offset = Offset;
+ BFI.StorageSize = AtomicSizeInBits;
+ BFI.StorageOffset += OffsetInChars;
+ llvm::Type *StorageTy = CGF.Builder.getIntNTy(AtomicSizeInBits);
+ auto LVal = LValue::MakeBitfield(
+ Address(StoragePtr, StorageTy, lvalue.getAlignment()), BFI,
+ lvalue.getType(), lvalue.getBaseInfo(), lvalue.getTBAAInfo());
+ auto AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned);
+ if (AtomicTy.isNull()) {
+ llvm::APInt Size(
+ /*numBits=*/32,
+ C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity());
+ AtomicTy = C.getConstantArrayType(C.CharTy, Size, nullptr,
+ ArraySizeModifier::Normal,
+ /*IndexTypeQuals=*/0);
}
+ auto ValueAlign = lvalue.getAlignment().getAsAlign();
+ auto AtomicAlign = ValueAlign;
- /// Cast the given pointer to an integer pointer suitable for atomic
- /// operations if the source.
- Address castToAtomicIntPointer(Address Addr) const;
+ auto ElTy = LVal.getBitFieldAddress().getElementType();
- /// If Addr is compatible with the iN that will be used for an atomic
- /// operation, bitcast it. Otherwise, create a temporary that is suitable
- /// and copy the value across.
- Address convertToAtomicIntPointer(Address Addr) const;
+ TypeEvaluationKind EvaluationKind = TEK_Scalar;
+ bool UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(
+ AtomicSizeInBits, C.toBits(lvalue.getAlignment()));
+ return AtomicInfo(CGF, ElTy, LVal, AtomicTy, ValueTy, EvaluationKind, {},
+ AtomicSizeInBits, ValueSizeInBits, AtomicAlign,
+ ValueAlign, UseLibcall);
+ }
- /// Turn an atomic-layout object into an r-value.
- RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot,
- SourceLocation loc, bool AsValue) const;
+ static AtomicInfo fromVectorElt(CodeGenFunction &CGF, LValue &lvalue) {
+ assert(lvalue.isVectorElt());
+ ASTContext &C = CGF.getContext();
+
+ auto ValueTy = lvalue.getType()->castAs<VectorType>()->getElementType();
+ auto ValueSizeInBits = C.getTypeSize(ValueTy);
+ auto AtomicTy = lvalue.getType();
+ auto AtomicSizeInBits = C.getTypeSize(AtomicTy);
+ auto ValueAlign = lvalue.getAlignment().getAsAlign();
+ auto AtomicAlign = ValueAlign;
+ auto LVal = lvalue;
+
+ auto ElTy = LVal.getVectorAddress().getElementType();
+
+ TypeEvaluationKind EvaluationKind = TEK_Scalar;
+ bool UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(
+ AtomicSizeInBits,
+ C.toBits(lvalue.getAlignment())); // TODO: Move to llvm::AtomicInfo
+ return AtomicInfo(CGF, ElTy, LVal, AtomicTy, ValueTy, EvaluationKind, {},
+ AtomicSizeInBits, ValueSizeInBits, AtomicAlign,
+ ValueAlign, UseLibcall);
+ }
- llvm::Value *getScalarRValValueOrNull(RValue RVal) const;
+ static AtomicInfo fromExtVectorElt(CodeGenFunction &CGF, LValue &lvalue) {
+ assert(lvalue.isExtVectorElt());
+ ASTContext &C = CGF.getContext();
+
+ auto ValueTy = lvalue.getType();
+ auto ValueSizeInBits = C.getTypeSize(ValueTy);
+ auto AtomicTy = ValueTy = CGF.getContext().getExtVectorType(
+ lvalue.getType(), cast<llvm::FixedVectorType>(
+ lvalue.getExtVectorAddress().getElementType())
+ ->getNumElements());
+ auto AtomicSizeInBits = C.getTypeSize(AtomicTy);
+ auto ValueAlign = lvalue.getAlignment().getAsAlign();
+ auto AtomicAlign = ValueAlign;
+ auto LVal = lvalue;
+
+ auto ElTy = LVal.getExtVectorAddress().getElementType();
+
+ TypeEvaluationKind EvaluationKind = TEK_Scalar;
+ bool UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(
+ AtomicSizeInBits,
+ C.toBits(lvalue.getAlignment())); // TODO: Move to llvm::AtomicInfo
+ return AtomicInfo(CGF, ElTy, LVal, AtomicTy, ValueTy, EvaluationKind, {},
+ AtomicSizeInBits, ValueSizeInBits, AtomicAlign,
+ ValueAlign, UseLibcall);
+ }
- /// Converts an rvalue to integer value if needed.
- llvm::Value *convertRValueToInt(RValue RVal, bool CmpXchg = false) const;
+ AtomicInfo(CodeGenFunction &CGF, llvm::Type *Ty, LValue &lvalue,
+ QualType AtomicTy, QualType ValueTy,
+ TypeEvaluationKind EvaluationKind, CGBitFieldInfo BFI,
+ uint64_t AtomicSizeInBits, uint64_t ValueSizeInBits,
+ llvm::Align AtomicAlign, llvm::Align ValueAlign, bool UseLibcall)
+ : llvm::AtomicInfo<CGBuilderBaseTy>(&CGF.Builder, Ty, AtomicSizeInBits,
+ ValueSizeInBits, AtomicAlign,
+ ValueAlign, UseLibcall),
+ CGF(CGF), AtomicTy(AtomicTy), ValueTy(ValueTy),
+ EvaluationKind(EvaluationKind), LVal(lvalue), BFI(BFI) {}
+
+ QualType getAtomicType() const { return AtomicTy; }
+ QualType getValueType() const { return ValueTy; }
+ CharUnits getAtomicAlignment() const {
+ return CharUnits::fromQuantity(AtomicAlign);
+ }
+ TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
+ const LValue &getAtomicLValue() const { return LVal; }
+ llvm::Value *getAtomicPointer() const override {
+ if (LVal.isSimple())
+ return LVal.emitRawPointer(CGF);
+ else if (LVal.isBitField())
+ return LVal.getRawBitFieldPointer(CGF);
+ else if (LVal.isVectorElt())
+ return LVal.getRawVectorPointer(CGF);
+ assert(LVal.isExtVectorElt());
+ return LVal.getRawExtVectorPointer(CGF);
+ }
+ Address getAtomicAddress() const {
+ llvm::Type *ElTy;
+ if (LVal.isSimple())
+ ElTy = LVal.getAddress().getElementType();
+ else if (LVal.isBitField())
+ ElTy = LVal.getBitFieldAddress().getElementType();
+ else if (LVal.isVectorElt())
+ ElTy = LVal.getVectorAddress().getElementType();
+ else
+ ElTy = LVal.getExtVectorAddress().getElementType();
+ return Address(getAtomicPointer(), ElTy, getAtomicAlignment());
+ }
- RValue ConvertToValueOrAtomic(llvm::Value *IntVal, AggValueSlot ResultSlot,
- SourceLocation Loc, bool AsValue,
- bool CmpXchg = false) const;
+ void decorateWithTBAA(llvm::Instruction *I) override {
+ CGF.CGM.DecorateInstructionWithTBAA(I, LVal.getTBAAInfo());
+ }
- /// Copy an atomic r-value into atomic-layout memory.
- void emitCopyIntoMemory(RValue rvalue) const;
+ llvm::AllocaInst *CreateAlloca(llvm::Type *Ty,
+ const llvm::Twine &Name) override {
+ return CGF.CreateTempAlloca(Ty, Name);
+ }
- /// Project an l-value down to the value field.
- LValue projectValue() const {
- assert(LVal.isSimple());
- Address addr = getAtomicAddress();
- if (hasPadding())
- addr = CGF.Builder.CreateStructGEP(addr, 0);
+ Address getAtomicAddressAsAtomicIntPointer() const {
+ return castToAtomicIntPointer(getAtomicAddress());
+ }
- return LValue::MakeAddr(addr, getValueType(), CGF.getContext(),
- LVal.getBaseInfo(), LVal.getTBAAInfo());
- }
+ bool emitMemSetZeroIfNecessary() const;
- /// Emits atomic load.
- /// \returns Loaded value.
- RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
- bool AsValue, llvm::AtomicOrdering AO,
- bool IsVolatile);
+ llvm::Value *getAtomicSizeValue() const {
+ CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);
+ return CGF.CGM.getSize(size);
+ }
+
+ /// Cast the given pointer to an integer pointer suitable for atomic
+ /// operations if the source.
+ Address castToAtomicIntPointer(Address Addr) const;
+
+ /// If Addr is compatible with the iN that will be used for an atomic
+ /// operation, bitcast it. Otherwise, create a temporary that is suitable
+ /// and copy the value across.
+ Address convertToAtomicIntPointer(Address Addr) const;
+
+ /// Turn an atomic-layout object into an r-value.
+ RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot,
+ SourceLocation loc, bool AsValue) const;
+
+ llvm::Value *getScalarRValValueOrNull(RValue RVal) const;
+
+ /// Converts an rvalue to integer value if needed.
+ llvm::Value *convertRValueToInt(RValue RVal, bool CmpXchg = false) const;
- /// Emits atomic compare-and-exchange sequence.
- /// \param Expected Expected value.
- /// \param Desired Desired value.
- /// \param Success Atomic ordering for success operation.
- /// \param Failure Atomic ordering for failed operation.
- /// \param IsWeak true if atomic operation is weak, false otherwise.
- /// \returns Pair of values: previous value from storage (value type) and
- /// boolean flag (i1 type) with true if success and false otherwise.
- std::pair<RValue, llvm::Value *>
- EmitAtomicCompareExchange(RValue Expected, RValue Desired,
- llvm::AtomicOrdering Success =
- llvm::AtomicOrdering::SequentiallyConsistent,
- llvm::AtomicOrdering Failure =
- llvm::AtomicOrdering::SequentiallyConsistent,
- bool IsWeak = false);
-
- /// Emits atomic update.
- /// \param AO Atomic ordering.
- /// \param UpdateOp Update operation for the current lvalue.
- void EmitAtomicUpdate(llvm::AtomicOrdering AO,
+ RValue ConvertToValueOrAtomic(llvm::Value *IntVal, AggValueSlot ResultSlot,
+ SourceLocation Loc, bool AsValue,
+ bool CmpXchg = false) const;
+
+ /// Copy an atomic r-value into atomic-layout memory.
+ void emitCopyIntoMemory(RValue rvalue) const;
+
+ /// Project an l-value down to the value field.
+ LValue projectValue() const {
+ assert(LVal.isSimple());
+ Address addr = getAtomicAddress();
+ if (hasPadding())
+ addr = CGF.Builder.CreateStructGEP(addr, 0);
+
+ return LValue::MakeAddr(addr, getValueType(), CGF.getContext(),
+ LVal.getBaseInfo(), LVal.getTBAAInfo());
+ }
+
+ /// Emits atomic load.
+ /// \returns Loaded value.
+ RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
+ bool AsValue, llvm::AtomicOrdering AO, bool IsVolatile);
+
+ /// Emits atomic compare-and-exchange sequence.
+ /// \param Expected Expected value.
+ /// \param Desired Desired value.
+ /// \param Success Atomic ordering for success operation.
+ /// \param Failure Atomic ordering for failed operation.
+ /// \param IsWeak true if atomic operation is weak, false otherwise.
+ /// \returns Pair of values: previous value from storage (value type) and
+ /// boolean flag (i1 type) with true if success and false otherwise.
+ std::pair<RValue, llvm::Value *>
+ EmitAtomicCompareExchange(RValue Expected, RValue Desired,
+ llvm::AtomicOrdering Success =
+ llvm::AtomicOrdering::SequentiallyConsistent,
+ llvm::AtomicOrdering Failure =
+ llvm::AtomicOrdering::SequentiallyConsistent,
+ bool IsWeak = false);
+
+ /// Emits atomic update.
+ /// \param AO Atomic ordering.
+ /// \param UpdateOp Update operation for the current lvalue.
+ void EmitAtomicUpdate(llvm::AtomicOrdering AO,
+ const llvm::function_ref<RValue(RValue)> &UpdateOp,
+ bool IsVolatile);
+ /// Emits atomic update.
+ /// \param AO Atomic ordering.
+ void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
+ bool IsVolatile);
+
+ /// Materialize an atomic r-value in atomic-layout memory.
+ Address materializeRValue(RValue rvalue) const;
+
+ /// Creates temp alloca for intermediate operations on atomic value.
+ Address CreateTempAlloca() const;
+
+private:
+ bool requiresMemSetZero(llvm::Type *type) const;
+
+ /// Emits atomic load as a libcall.
+ void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, llvm::AtomicOrdering AO,
+ bool IsVolatile);
+ /// Emit atomic update as libcalls.
+ void
+ EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
+ const llvm::function_ref<RValue(RValue)> &UpdateOp,
+ bool IsVolatile);
+ /// Emit atomic update as LLVM instructions.
+ void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
const llvm::function_ref<RValue(RValue)> &UpdateOp,
bool IsVolatile);
- /// Emits atomic update.
- /// \param AO Atomic ordering.
- void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
+ /// Emit atomic update as libcalls.
+ void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal,
+ bool IsVolatile);
+ /// Emit atomic update as LLVM instructions.
+ void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal,
bool IsVolatile);
-
- /// Materialize an atomic r-value in atomic-layout memory.
- Address materializeRValue(RValue rvalue) const;
-
- /// Creates temp alloca for intermediate operations on atomic value.
- Address CreateTempAlloca() const;
- private:
- bool requiresMemSetZero(llvm::Type *type) const;
-
-
- /// Emits atomic load as a libcall.
- void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
- llvm::AtomicOrdering AO, bool IsVolatile);
- /// Emits atomic load as LLVM instruction.
- llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile,
- bool CmpXchg = false);
- /// Emits atomic compare-and-exchange op as a libcall.
- llvm::Value *EmitAtomicCompareExchangeLibcall(
- llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,
- llvm::AtomicOrdering Success =
- llvm::AtomicOrdering::SequentiallyConsistent,
- llvm::AtomicOrdering Failure =
- llvm::AtomicOrdering::SequentiallyConsistent);
- /// Emits atomic compare-and-exchange op as LLVM instruction.
- std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp(
- llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
- llvm::AtomicOrdering Success =
- llvm::AtomicOrdering::SequentiallyConsistent,
- llvm::AtomicOrdering Failure =
- llvm::AtomicOrdering::SequentiallyConsistent,
- bool IsWeak = false);
- /// Emit atomic update as libcalls.
- void
- EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
- const llvm::function_ref<RValue(RValue)> &UpdateOp,
- bool IsVolatile);
- /// Emit atomic update as LLVM instructions.
- void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
- const llvm::function_ref<RValue(RValue)> &UpdateOp,
- bool IsVolatile);
- /// Emit atomic update as libcalls.
- void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal,
- bool IsVolatile);
- /// Emit atomic update as LLVM instructions.
- void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal,
- bool IsVolatile);
- };
+};
}
Address AtomicInfo::CreateTempAlloca() const {
@@ -1002,7 +1050,7 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
// need to make sure (via temporaries if necessary) that all incoming values
// are compatible.
LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy);
- AtomicInfo Atomics(*this, AtomicVal);
+ AtomicInfo Atomics = AtomicInfo::fromValue(*this, AtomicVal);
if (ShouldCastToIntPtrTy) {
Ptr = Atomics.castToAtomicIntPointer(Ptr);
@@ -1401,17 +1449,6 @@ RValue AtomicInfo::convertAtomicTempToRValue(Address addr,
LVal.getBaseInfo(), TBAAAccessInfo()));
}
-/// Return true if \param ValTy is a type that should be casted to integer
-/// around the atomic memory operation. If \param CmpXchg is true, then the
-/// cast of a floating point type is made as that instruction can not have
-/// floating point operands. TODO: Allow compare-and-exchange and FP - see
-/// comment in AtomicExpandPass.cpp.
-static bool shouldCastToInt(llvm::Type *ValTy, bool CmpXchg) {
- if (ValTy->isFloatingPointTy())
- return ValTy->isX86_FP80Ty() || CmpXchg;
- return !ValTy->isIntegerTy() && !ValTy->isPointerTy();
-}
-
RValue AtomicInfo::ConvertToValueOrAtomic(llvm::Value *Val,
AggValueSlot ResultSlot,
SourceLocation Loc, bool AsValue,
@@ -1470,28 +1507,12 @@ void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args);
}
-llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
- bool IsVolatile, bool CmpXchg) {
- // Okay, we're doing this natively.
- Address Addr = getAtomicAddress();
- if (shouldCastToInt(Addr.getElementType(), CmpXchg))
- Addr = castToAtomicIntPointer(Addr);
- llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load");
- Load->setAtomic(AO);
-
- // Other decoration.
- if (IsVolatile)
- Load->setVolatile(true);
- CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo());
- return Load;
-}
-
/// An LValue is a candidate for having its loads and stores be made atomic if
/// we are operating under /volatile:ms *and* the LValue itself is volatile and
/// performing such an operation can be performed without a libcall.
bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) {
if (!CGM.getLangOpts().MSVolatile) return false;
- AtomicInfo AI(*this, LV);
+ AtomicInfo AI = AtomicInfo::fromValue(*this, LV);
bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType());
// An atomic is inline if we don't need to use a libcall.
bool AtomicIsInline = !AI.shouldUseLibcall();
@@ -1551,9 +1572,8 @@ RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc,
llvm::AtomicOrdering AO, bool IsVolatile,
AggValueSlot resultSlot) {
- AtomicInfo Atomics(*this, src);
- return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO,
- IsVolatile);
+ return AtomicInfo::fromValue(*this, src)
+ .EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO, IsVolatile);
}
/// Copy an r-value into memory as part of storing to an atomic type.
@@ -1601,8 +1621,7 @@ Address AtomicInfo::materializeRValue(RValue rvalue) const {
// Otherwise, make a temporary and materialize into it.
LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType());
- AtomicInfo Atomics(CGF, TempLV);
- Atomics.emitCopyIntoMemory(rvalue);
+ AtomicInfo::fromValue(CGF, TempLV).emitCopyIntoMemory(rvalue);
return TempLV.getAddress();
}
@@ -1635,70 +1654,18 @@ llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal, bool CmpXchg) const {
return CGF.Builder.CreateLoad(Addr);
}
-std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
- llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
- llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) {
- // Do the atomic store.
- Address Addr = getAtomicAddressAsAtomicIntPointer();
- auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr, ExpectedVal, DesiredVal,
- Success, Failure);
- // Other decoration.
- Inst->setVolatile(LVal.isVolatileQualified());
- Inst->setWeak(IsWeak);
-
- // Okay, turn that back into the original value type.
- auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0);
- auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1);
- return std::make_pair(PreviousVal, SuccessFailureVal);
-}
-
-llvm::Value *
-AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr,
- llvm::Value *DesiredAddr,
- llvm::AtomicOrdering Success,
- llvm::AtomicOrdering Failure) {
- // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
- // void *desired, int success, int failure);
- CallArgList Args;
- Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
- Args.add(RValue::get(getAtomicPointer()), CGF.getContext().VoidPtrTy);
- Args.add(RValue::get(ExpectedAddr), CGF.getContext().VoidPtrTy);
- Args.add(RValue::get(DesiredAddr), CGF.getContext().VoidPtrTy);
- Args.add(RValue::get(
- llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))),
- CGF.getContext().IntTy);
- Args.add(RValue::get(
- llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))),
- CGF.getContext().IntTy);
- auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange",
- CGF.getContext().BoolTy, Args);
-
- return SuccessFailureRVal.getScalarVal();
-}
-
std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
RValue Expected, RValue Desired, llvm::AtomicOrdering Success,
llvm::AtomicOrdering Failure, bool IsWeak) {
- // Check whether we should use a library call.
- if (shouldUseLibcall()) {
- // Produce a source address.
- Address ExpectedAddr = materializeRValue(Expected);
- llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF);
- llvm::Value *DesiredPtr = materializeRValue(Desired).emitRawPointer(CGF);
- auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedPtr, DesiredPtr,
- Success, Failure);
- return std::make_pair(
- convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
- SourceLocation(), /*AsValue=*/false),
- Res);
- }
- // If we've got a scalar value of the right size, try to avoid going
- // through memory.
- auto *ExpectedVal = convertRValueToInt(Expected, /*CmpXchg=*/true);
+ Address ExpectedAddr = materializeRValue(Expected);
+ llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF);
auto *DesiredVal = convertRValueToInt(Desired, /*CmpXchg=*/true);
- auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
- Failure, IsWeak);
+
+ std::pair<llvm::Value *, llvm::Value *> Res =
+ getBase().EmitAtomicCompareExchange(ExpectedPtr, DesiredVal, Success,
+ Failure, LVal.isVolatileQualified(),
+ IsWeak);
return std::make_pair(
ConvertToValueOrAtomic(Res.first, AggValueSlot::ignored(),
SourceLocation(), /*AsValue=*/false,
@@ -1784,9 +1751,9 @@ void AtomicInfo::EmitAtomicUpdateLibcall(
EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr);
llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF);
llvm::Value *DesiredPtr = DesiredAddr.emitRawPointer(CGF);
- auto *Res =
+ auto Res =
EmitAtomicCompareExchangeLibcall(ExpectedPtr, DesiredPtr, AO, Failure);
- CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
+ CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
}
@@ -1872,9 +1839,9 @@ void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr);
llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF);
llvm::Value *DesiredPtr = DesiredAddr.emitRawPointer(CGF);
- auto *Res =
+ auto Res =
EmitAtomicCompareExchangeLibcall(ExpectedPtr, DesiredPtr, AO, Failure);
- CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
+ CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
}
@@ -1953,7 +1920,7 @@ void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest,
rvalue.getAggregateAddress().getElementType() ==
dest.getAddress().getElementType());
- AtomicInfo atomics(*this, dest);
+ AtomicInfo atomics = AtomicInfo ::fromValue(*this, dest);
LValue LVal = atomics.getAtomicLValue();
// If this is an initialization, just put the value there normally.
@@ -1988,7 +1955,8 @@ void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest,
// Do the atomic store.
Address Addr = atomics.getAtomicAddress();
if (llvm::Value *Value = atomics.getScalarRValValueOrNull(rvalue))
- if (shouldCastToInt(Value->getType(), /*CmpXchg=*/false)) {
+ if (llvm::AtomicInfo<CGBuilderTy>::shouldCastToInt(Value->getType(),
+ /*CmpXchg=*/false)) {
Addr = atomics.castToAtomicIntPointer(Addr);
ValToStore = Builder.CreateIntCast(ValToStore, Addr.getElementType(),
/*isSigned=*/false);
@@ -2028,21 +1996,18 @@ std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange(
assert(!Desired.isAggregate() ||
Desired.getAggregateAddress().getElementType() ==
Obj.getAddress().getElementType());
- AtomicInfo Atomics(*this, Obj);
-
- return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure,
- IsWeak);
+ return AtomicInfo::fromValue(*this, Obj)
+ .EmitAtomicCompareExchange(Expected, Desired, Success, Failure, IsWeak);
}
void CodeGenFunction::EmitAtomicUpdate(
LValue LVal, llvm::AtomicOrdering AO,
const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) {
- AtomicInfo Atomics(*this, LVal);
- Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile);
+ AtomicInfo::fromValue(*this, LVal).EmitAtomicUpdate(AO, UpdateOp, IsVolatile);
}
void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) {
- AtomicInfo atomics(*this, dest);
+ AtomicInfo atomics = AtomicInfo::fromValue(*this, dest);
switch (atomics.getEvaluationKind()) {
case TEK_Scalar: {
diff --git a/flang/test/Integration/OpenMP/atomic-update-complex.f90 b/flang/test/Integration/OpenMP/atomic-update-complex.f90
new file mode 100644
index 0000000000000..6e0b419d11f95
--- /dev/null
+++ b/flang/test/Integration/OpenMP/atomic-update-complex.f90
@@ -0,0 +1,42 @@
+!===----------------------------------------------------------------------===!
+! This directory can be used to add Integration tests involving multiple
+! stages of the compiler (for eg. from Fortran to LLVM IR). It should not
+! contain executable tests. We should only add tests here sparingly and only
+! if there is no other way to test. Repeat this message in each test that is
+! added to this directory and sub-directories.
+!===----------------------------------------------------------------------===!
+
+!RUN: %flang_fc1 -emit-llvm -fopenmp %s -o - | FileCheck %s
+
+!CHECK: define void @_QQmain() {
+!CHECK: %[[X_NEW_VAL:.*]] = alloca { float, float }, align 8
+!CHECK: {{.*}} = alloca { float, float }, i64 1, align 8
+!CHECK: %[[ORIG_VAL:.*]] = alloca { float, float }, i64 1, align 8
+!CHECK: store { float, float } { float 2.000000e+00, float 2.000000e+00 }, ptr %[[ORIG_VAL]], align 4
+!CHECK: br label %entry
+
+!CHECK: entry:
+!CHECK: %[[ATOMIC_TEMP_LOAD:.*]] = alloca { float, float }, align 16
+!CHECK: call void @__atomic_load(i64 8, ptr %[[ORIG_VAL]], ptr %[[ATOMIC_TEMP_LOAD]], i32 0)
+!CHECK: %[[PHI_NODE_ENTRY_1:.*]] = load { float, float }, ptr %[[ATOMIC_TEMP_LOAD]], align 16
+!CHECK: br label %.atomic.cont
+
+!CHECK: .atomic.cont
+!CHECK: %[[VAL_4:.*]] = phi { float, float } [ %[[PHI_NODE_ENTRY_1]], %entry ], [ %{{.*}}, %.atomic.cont ]
+!CHECK: %[[VAL_5:.*]] = extractvalue { float, float } %[[VAL_4]], 0
+!CHECK: %[[VAL_6:.*]] = extractvalue { float, float } %[[VAL_4]], 1
+!CHECK: %[[VAL_7:.*]] = fadd contract float %[[VAL_5]], 1.000000e+00
+!CHECK: %[[VAL_8:.*]] = fadd contract float %[[VAL_6]], 1.000000e+00
+!CHECK: %[[VAL_9:.*]] = insertvalue { float, float } undef, float %[[VAL_7]], 0
+!CHECK: %[[VAL_10:.*]] = insertvalue { float, float } %[[VAL_9]], float %[[VAL_8]], 1
+!CHECK: store { float, float } %[[VAL_10]], ptr %[[X_NEW_VAL]], align 4
+!CHECK: %[[VAL_11:.*]] = call i1 @__atomic_compare_exchange(i64 8, ptr %[[ORIG_VAL]], ptr %[[ATOMIC_TEMP_LOAD]], ptr %[[X_NEW_VAL]], i32 2, i32 2)
+!CHECK: %[[VAL_12:.*]] = load { float, float }, ptr %[[ATOMIC_TEMP_LOAD]], align 4
+!CHECK: br i1 %[[VAL_11]], label %.atomic.exit, label %.atomic.cont
+program main
+ complex*8 ia, ib
+ ia = (2, 2)
+ !$omp atomic update
+ ia = ia + (1, 1)
+ !$omp end atomic
+end program
diff --git a/llvm/include/llvm/Frontend/Atomic/Atomic.h b/llvm/include/llvm/Frontend/Atomic/Atomic.h
new file mode 100644
index 0000000000000..05296ef44c90f
--- /dev/null
+++ b/llvm/include/llvm/Frontend/Atomic/Atomic.h
@@ -0,0 +1,235 @@
+//===--- Atomic.h - Shared codegen of atomic operations -------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FRONTEND_ATOMIC_ATOMIC_H
+#define LLVM_FRONTEND_ATOMIC_ATOMIC_H
+
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/CodeGen/RuntimeLibcalls.h"
+#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Operator.h"
+
+namespace llvm {
+
+template <typename IRBuilderTy> struct AtomicInfo {
+ IRBuilderTy *Builder;
+
+ Type *Ty;
+ uint64_t AtomicSizeInBits;
+ uint64_t ValueSizeInBits;
+ llvm::Align AtomicAlign;
+ llvm::Align ValueAlign;
+ bool UseLibcall;
+
+public:
+ AtomicInfo(IRBuilderTy *Builder, Type *Ty, uint64_t AtomicSizeInBits,
+ uint64_t ValueSizeInBits, llvm::Align AtomicAlign,
+ llvm::Align ValueAlign, bool UseLibcall)
+ : Builder(Builder), Ty(Ty), AtomicSizeInBits(AtomicSizeInBits),
+ ValueSizeInBits(ValueSizeInBits), AtomicAlign(AtomicAlign),
+ ValueAlign(ValueAlign), UseLibcall(UseLibcall) {}
+
+ virtual ~AtomicInfo() = default;
+
+ llvm::Align getAtomicAlignment() const { return AtomicAlign; }
+ uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
+ uint64_t getValueSizeInBits() const { return ValueSizeInBits; }
+ bool shouldUseLibcall() const { return UseLibcall; }
+ llvm::Type *getAtomicTy() const { return Ty; }
+
+ virtual llvm::Value *getAtomicPointer() const = 0;
+ virtual void decorateWithTBAA(Instruction *I) {}
+ virtual llvm::AllocaInst *CreateAlloca(llvm::Type *Ty,
+ const llvm::Twine &Name) = 0;
+
+ /// Is the atomic size larger than the underlying value type?
+ ///
+ /// Note that the absence of padding does not mean that atomic
+ /// objects are completely interchangeable with non-atomic
+ /// objects: we might have promoted the alignment of a type
+ /// without making it bigger.
+ bool hasPadding() const { return (ValueSizeInBits != AtomicSizeInBits); }
+
+ LLVMContext &getLLVMContext() const { return Builder->getContext(); }
+
+ static bool shouldCastToInt(llvm::Type *ValTy, bool CmpXchg) {
+ if (ValTy->isFloatingPointTy())
+ return ValTy->isX86_FP80Ty() || CmpXchg;
+ return !ValTy->isIntegerTy() && !ValTy->isPointerTy();
+ }
+
+ llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile,
+ bool CmpXchg = false) {
+ Value *Ptr = getAtomicPointer();
+ Type *AtomicTy = Ty;
+ if (shouldCastToInt(Ty, CmpXchg))
+ AtomicTy = llvm::IntegerType::get(getLLVMContext(), AtomicSizeInBits);
+ LoadInst *Load =
+ Builder->CreateAlignedLoad(AtomicTy, Ptr, AtomicAlign, "atomic-load");
+ Load->setAtomic(AO);
+
+ // Other decoration.
+ if (IsVolatile)
+ Load->setVolatile(true);
+ decorateWithTBAA(Load);
+ return Load;
+ }
+
+ static CallInst *emitAtomicLibcalls2(IRBuilderTy *Builder, StringRef fnName,
+ Type *ResultType,
+ ArrayRef<Value *> Args) {
+ auto &C = Builder->getContext();
+
+ SmallVector<Type *, 6> ArgTys;
+ for (Value *Arg : Args)
+ ArgTys.push_back(Arg->getType());
+ FunctionType *FnType = FunctionType::get(ResultType, ArgTys, false);
+ Module *M = Builder->GetInsertBlock()->getModule();
+
+ // TODO: Use llvm::TargetLowering for Libcall ABI
+ llvm::AttrBuilder fnAttrB(C);
+ fnAttrB.addAttribute(llvm::Attribute::NoUnwind);
+ fnAttrB.addAttribute(llvm::Attribute::WillReturn);
+ llvm::AttributeList fnAttrs = llvm::AttributeList::get(
+ C, llvm::AttributeList::FunctionIndex, fnAttrB);
+ FunctionCallee LibcallFn = M->getOrInsertFunction(fnName, FnType, fnAttrs);
+ CallInst *Call = Builder->CreateCall(LibcallFn, Args);
+
+ return Call;
+ }
+
+ llvm::Value *getAtomicSizeValue() const {
+ auto &C = getLLVMContext();
+
+ // TODO: Get from llvm::TargetMachine / clang::TargetInfo
+ uint16_t SizeTBits = 64;
+ uint16_t BitsPerByte = 8;
+ return llvm::ConstantInt::get(llvm::IntegerType::get(C, SizeTBits),
+ AtomicSizeInBits / BitsPerByte);
+ }
+
+ std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeLibcall(
+ llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
+ llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure) {
+ auto &C = getLLVMContext();
+
+ // __atomic_compare_exchange's expected and desired are passed by pointers
+ // FIXME: types
+
+ // TODO: Get from llvm::TargetMachine / clang::TargetInfo
+ uint64_t IntBits = 32;
+
+ // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
+ // void *desired, int success, int failure);
+ llvm::Value *Args[6] = {
+ getAtomicSizeValue(),
+ getAtomicPointer(),
+ ExpectedVal,
+ DesiredVal,
+ llvm::Constant::getIntegerValue(
+ llvm::IntegerType::get(C, IntBits),
+ llvm::APInt(IntBits, (uint64_t)Success, /*signed=*/true)),
+ llvm::Constant::getIntegerValue(
+ llvm::IntegerType::get(C, IntBits),
+ llvm::APInt(IntBits, (uint64_t)Failure, /*signed=*/true)),
+ };
+ auto Res = emitAtomicLibcalls2(Builder, "__atomic_compare_exchange",
+ llvm::IntegerType::getInt1Ty(C), Args);
+ return std::make_pair(ExpectedVal, Res);
+ }
+
+ Value *castToAtomicIntPointer(Value *addr) const {
+ return addr; // opaque pointer
+ }
+
+ Value *getAtomicAddressAsAtomicIntPointer() const {
+ return castToAtomicIntPointer(getAtomicPointer());
+ }
+
+ std::pair<llvm::Value *, llvm::Value *>
+ EmitAtomicCompareExchangeOp(llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
+ llvm::AtomicOrdering Success,
+ llvm::AtomicOrdering Failure,
+ bool IsVolatile = false, bool IsWeak = false) {
+ // Do the atomic store.
+ Value *Addr = getAtomicAddressAsAtomicIntPointer();
+ auto *Inst = Builder->CreateAtomicCmpXchg(Addr, ExpectedVal, DesiredVal,
+ getAtomicAlignment(), Success,
+ Failure, llvm::SyncScope::System);
+ // Other decoration.
+ Inst->setVolatile(IsVolatile);
+ Inst->setWeak(IsWeak);
+
+ // Okay, turn that back into the original value type.
+ auto *PreviousVal = Builder->CreateExtractValue(Inst, /*Idxs=*/0);
+ auto *SuccessFailureVal = Builder->CreateExtractValue(Inst, /*Idxs=*/1);
+ return std::make_pair(PreviousVal, SuccessFailureVal);
+ }
+
+ std::pair<llvm::Value *, llvm::Value *>
+ EmitAtomicCompareExchange(llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
+ llvm::AtomicOrdering Success,
+ llvm::AtomicOrdering Failure, bool IsVolatile,
+ bool IsWeak) {
+
+ // Check whether we should use a library call.
+ if (shouldUseLibcall()) {
+ return EmitAtomicCompareExchangeLibcall(ExpectedVal, DesiredVal, Success,
+ Failure);
+ }
+
+ // If we've got a scalar value of the right size, try to avoid going
+ // through memory.
+ auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
+ Failure, IsVolatile, IsWeak);
+ return Res;
+ }
+
+ // void __atomic_load(size_t size, void *mem, void *return, int order);
+ std::pair<llvm::LoadInst *, llvm::AllocaInst *>
+ EmitAtomicLoadLibcall(llvm::AtomicOrdering AO) {
+ LLVMContext &Ctx = getLLVMContext();
+ Type *SizedIntTy = Type::getIntNTy(Ctx, getAtomicSizeInBits() * 8);
+ Type *ResultTy;
+ SmallVector<Value *, 6> Args;
+ AttributeList Attr;
+ Module *M = Builder->GetInsertBlock()->getModule();
+ const DataLayout &DL = M->getDataLayout();
+ Args.push_back(ConstantInt::get(DL.getIntPtrType(Ctx),
+ this->getAtomicSizeInBits() / 8));
+ Value *PtrVal = getAtomicPointer();
+ PtrVal = Builder->CreateAddrSpaceCast(PtrVal, PointerType::getUnqual(Ctx));
+ Args.push_back(PtrVal);
+ AllocaInst *AllocaResult =
+ CreateAlloca(Ty, getAtomicPointer()->getName() + "atomic.temp.load");
+ const Align AllocaAlignment = DL.getPrefTypeAlign(SizedIntTy);
+ AllocaResult->setAlignment(AllocaAlignment);
+ Args.push_back(AllocaResult);
+ Constant *OrderingVal =
+ ConstantInt::get(Type::getInt32Ty(Ctx), (int)toCABI(AO));
+ Args.push_back(OrderingVal);
+ ResultTy = Type::getVoidTy(Ctx);
+ SmallVector<Type *, 6> ArgTys;
+ for (Value *Arg : Args)
+ ArgTys.push_back(Arg->getType());
+ FunctionType *FnType = FunctionType::get(ResultTy, ArgTys, false);
+ FunctionCallee LibcallFn =
+ M->getOrInsertFunction("__atomic_load", FnType, Attr);
+ CallInst *Call = Builder->CreateCall(LibcallFn, Args);
+ Call->setAttributes(Attr);
+ return std::make_pair(
+ Builder->CreateAlignedLoad(Ty, AllocaResult, AllocaAlignment),
+ AllocaResult);
+ }
+};
+} // end namespace llvm
+
+#endif /* LLVM_FRONTEND_ATOMIC_ATOMIC_H */
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
index bff49dab4a313..badd6740acf9a 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
@@ -15,6 +15,7 @@
#define LLVM_FRONTEND_OPENMP_OMPIRBUILDER_H
#include "llvm/Analysis/MemorySSAUpdater.h"
+#include "llvm/Frontend/Atomic/Atomic.h"
#include "llvm/Frontend/OpenMP/OMPConstants.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/IRBuilder.h"
@@ -457,6 +458,29 @@ class OpenMPIRBuilder {
T(Triple(M.getTargetTriple())) {}
~OpenMPIRBuilder();
+ class AtomicInfo : public llvm::AtomicInfo<IRBuilder<>> {
+
+ llvm::Value *AtomicVar;
+
+ public:
+ AtomicInfo(IRBuilder<> *Builder, llvm::Type *Ty, uint64_t AtomicSizeInBits,
+ uint64_t ValueSizeInBits, llvm::Align AtomicAlign,
+ llvm::Align ValueAlign, bool UseLibcall, llvm::Value *AtomicVar)
+ : llvm::AtomicInfo<IRBuilder<>>(Builder, Ty, AtomicSizeInBits,
+ ValueSizeInBits, AtomicAlign,
+ ValueAlign, UseLibcall),
+ AtomicVar(AtomicVar) {}
+
+ llvm::Value *getAtomicPointer() const override { return AtomicVar; }
+ void decorateWithTBAA(llvm::Instruction *I) override {}
+ llvm::AllocaInst *CreateAlloca(llvm::Type *Ty,
+ const llvm::Twine &Name) override {
+ llvm::AllocaInst *allocaInst = Builder->CreateAlloca(Ty);
+ allocaInst->setName(Name);
+ return allocaInst;
+ }
+ };
+
/// Initialize the internal state, this will put structures types and
/// potentially other helpers into the underlying module. Must be called
/// before any other method and only once! This internal state includes types
diff --git a/llvm/lib/Frontend/Atomic/Atomic.cpp b/llvm/lib/Frontend/Atomic/Atomic.cpp
new file mode 100644
index 0000000000000..1b051b31a9234
--- /dev/null
+++ b/llvm/lib/Frontend/Atomic/Atomic.cpp
@@ -0,0 +1,18 @@
+//===--- Atomic.cpp - Shared codegen of atomic operations -----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Frontend/Atomic/Atomic.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/Frontend/Atomic/Atomic.h"
+#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/Operator.h"
+
+namespace {} // namespace
+
+namespace llvm {} // end namespace llvm
diff --git a/llvm/lib/Frontend/Atomic/CMakeLists.txt b/llvm/lib/Frontend/Atomic/CMakeLists.txt
new file mode 100644
index 0000000000000..06f950e4bee4b
--- /dev/null
+++ b/llvm/lib/Frontend/Atomic/CMakeLists.txt
@@ -0,0 +1,15 @@
+add_llvm_component_library(LLVMFrontendAtomic
+ Atomic.cpp
+
+ ADDITIONAL_HEADER_DIRS
+ ${LLVM_MAIN_INCLUDE_DIR}/llvm/Frontend/Atomic
+
+ DEPENDS
+ LLVMAnalysis
+ LLVMTargetParser
+
+ LINK_COMPONENTS
+ Core
+ Support
+ Analysis
+ )
diff --git a/llvm/lib/Frontend/CMakeLists.txt b/llvm/lib/Frontend/CMakeLists.txt
index 62dd0da1e6c2d..b305ce7d771ce 100644
--- a/llvm/lib/Frontend/CMakeLists.txt
+++ b/llvm/lib/Frontend/CMakeLists.txt
@@ -1,3 +1,4 @@
+add_subdirectory(Atomic)
add_subdirectory(Driver)
add_subdirectory(HLSL)
add_subdirectory(OpenACC)
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index ccec66fcb7bac..55018b89873c3 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -23,6 +23,7 @@
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Bitcode/BitcodeReader.h"
+#include "llvm/Frontend/Atomic/Atomic.h"
#include "llvm/Frontend/Offloading/Utility.h"
#include "llvm/Frontend/OpenMP/OMPGridValues.h"
#include "llvm/IR/Attributes.h"
@@ -6033,6 +6034,52 @@ std::pair<Value *, Value *> OpenMPIRBuilder::emitAtomicUpdate(
Res.second = Res.first;
else
Res.second = emitRMWOpAsInstruction(Res.first, Expr, RMWOp);
+ } else if (RMWOp == llvm::AtomicRMWInst::BinOp::BAD_BINOP) {
+ LoadInst *OldVal =
+ Builder.CreateLoad(XElemTy, X, X->getName() + ".atomic.load");
+ OldVal->setAtomic(AO);
+ const DataLayout &LoadDL = OldVal->getModule()->getDataLayout();
+ unsigned LoadSize =
+ LoadDL.getTypeStoreSize(OldVal->getPointerOperand()->getType());
+ OpenMPIRBuilder::AtomicInfo atomicInfo(&Builder, XElemTy, LoadSize * 8,
+ LoadSize * 8, OldVal->getAlign(),
+ OldVal->getAlign(), true, X);
+ auto AtomicLoadRes = atomicInfo.EmitAtomicLoadLibcall(AO);
+ BasicBlock *CurBB = Builder.GetInsertBlock();
+ Instruction *CurBBTI = CurBB->getTerminator();
+ CurBBTI = CurBBTI ? CurBBTI : Builder.CreateUnreachable();
+ BasicBlock *ExitBB =
+ CurBB->splitBasicBlock(CurBBTI, X->getName() + ".atomic.exit");
+ BasicBlock *ContBB = CurBB->splitBasicBlock(CurBB->getTerminator(),
+ X->getName() + ".atomic.cont");
+ ContBB->getTerminator()->eraseFromParent();
+ Builder.restoreIP(AllocaIP);
+ AllocaInst *NewAtomicAddr = Builder.CreateAlloca(XElemTy);
+ NewAtomicAddr->setName(X->getName() + "x.new.val");
+ Builder.SetInsertPoint(ContBB);
+ llvm::PHINode *PHI = Builder.CreatePHI(OldVal->getType(), 2);
+ PHI->addIncoming(AtomicLoadRes.first, CurBB);
+ Value *OldExprVal = PHI;
+ Value *Upd = UpdateOp(OldExprVal, Builder);
+ Builder.CreateStore(Upd, NewAtomicAddr);
+ AtomicOrdering Failure =
+ llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
+ auto Result = atomicInfo.EmitAtomicCompareExchangeLibcall(
+ AtomicLoadRes.second, NewAtomicAddr, AO, Failure);
+ LoadInst *PHILoad = Builder.CreateLoad(XElemTy, Result.first);
+ PHI->addIncoming(PHILoad, Builder.GetInsertBlock());
+ Builder.CreateCondBr(Result.second, ExitBB, ContBB);
+ OldVal->eraseFromParent();
+ Res.first = OldExprVal;
+ Res.second = Upd;
+
+ if (UnreachableInst *ExitTI =
+ dyn_cast<UnreachableInst>(ExitBB->getTerminator())) {
+ CurBBTI->eraseFromParent();
+ Builder.SetInsertPoint(ExitBB);
+ } else {
+ Builder.SetInsertPoint(ExitTI);
+ }
} else {
IntegerType *IntCastTy =
IntegerType::get(M.getContext(), XElemTy->getScalarSizeInBits());
More information about the cfe-commits
mailing list