[clang] 24539f1 - Add Alignment argument to IRBuilder CreateAtomicRMW and CreateAtomicCmpXchg.
James Y Knight via cfe-commits
cfe-commits at lists.llvm.org
Thu Feb 25 15:30:12 PST 2021
Author: James Y Knight
Date: 2021-02-25T18:29:42-05:00
New Revision: 24539f1ef2471d07bd87f833cb0288fc0f251f4b
URL: https://github.com/llvm/llvm-project/commit/24539f1ef2471d07bd87f833cb0288fc0f251f4b
DIFF: https://github.com/llvm/llvm-project/commit/24539f1ef2471d07bd87f833cb0288fc0f251f4b.diff
LOG: Add Alignment argument to IRBuilder CreateAtomicRMW and CreateAtomicCmpXchg.
And then push those change throughout LLVM.
Keep the old signature in Clang's CGBuilder for now -- that will be
updated in a follow-on patch (D97224).
The MLIR LLVM-IR dialect is not updated to support the new alignment
attribute, but preserves its existing behavior.
Differential Revision: https://reviews.llvm.org/D97223
Added:
Modified:
clang/lib/CodeGen/CGBuilder.h
llvm/include/llvm/CodeGen/AtomicExpandUtils.h
llvm/include/llvm/IR/IRBuilder.h
llvm/lib/CodeGen/AtomicExpandPass.cpp
llvm/lib/IR/AutoUpgrade.cpp
llvm/lib/IR/Core.cpp
llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp
llvm/lib/Transforms/Instrumentation/InstrOrderFile.cpp
llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
Removed:
################################################################################
diff --git a/clang/lib/CodeGen/CGBuilder.h b/clang/lib/CodeGen/CGBuilder.h
index 38e96c0f4ee6..4cdf9fce7a3e 100644
--- a/clang/lib/CodeGen/CGBuilder.h
+++ b/clang/lib/CodeGen/CGBuilder.h
@@ -132,6 +132,28 @@ class CGBuilderTy : public CGBuilderBaseTy {
return CreateAlignedStore(getInt1(Value), Addr, CharUnits::One());
}
+ // Temporarily use old signature; clang will be updated to an Address overload
+ // in a subsequent patch.
+ llvm::AtomicCmpXchgInst *
+ CreateAtomicCmpXchg(llvm::Value *Ptr, llvm::Value *Cmp, llvm::Value *New,
+ llvm::AtomicOrdering SuccessOrdering,
+ llvm::AtomicOrdering FailureOrdering,
+ llvm::SyncScope::ID SSID = llvm::SyncScope::System) {
+ return CGBuilderBaseTy::CreateAtomicCmpXchg(
+ Ptr, Cmp, New, llvm::MaybeAlign(), SuccessOrdering, FailureOrdering,
+ SSID);
+ }
+
+ // Temporarily use old signature; clang will be updated to an Address overload
+ // in a subsequent patch.
+ llvm::AtomicRMWInst *
+ CreateAtomicRMW(llvm::AtomicRMWInst::BinOp Op, llvm::Value *Ptr,
+ llvm::Value *Val, llvm::AtomicOrdering Ordering,
+ llvm::SyncScope::ID SSID = llvm::SyncScope::System) {
+ return CGBuilderBaseTy::CreateAtomicRMW(Op, Ptr, Val, llvm::MaybeAlign(),
+ Ordering, SSID);
+ }
+
using CGBuilderBaseTy::CreateBitCast;
Address CreateBitCast(Address Addr, llvm::Type *Ty,
const llvm::Twine &Name = "") {
diff --git a/llvm/include/llvm/CodeGen/AtomicExpandUtils.h b/llvm/include/llvm/CodeGen/AtomicExpandUtils.h
index 8a46c6e00d22..1471ec701894 100644
--- a/llvm/include/llvm/CodeGen/AtomicExpandUtils.h
+++ b/llvm/include/llvm/CodeGen/AtomicExpandUtils.h
@@ -22,8 +22,8 @@ class Value;
/// (the builder, %addr, %loaded, %new_val, ordering,
/// /* OUT */ %success, /* OUT */ %new_loaded)
using CreateCmpXchgInstFun =
- function_ref<void(IRBuilder<> &, Value *, Value *, Value *, AtomicOrdering,
- Value *&, Value *&)>;
+ function_ref<void(IRBuilder<> &, Value *, Value *, Value *, Align,
+ AtomicOrdering, Value *&, Value *&)>;
/// Expand an atomic RMW instruction into a loop utilizing
/// cmpxchg. You'll want to make sure your target machine likes cmpxchg
diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h
index c12159faf12c..8525e2e8ebc0 100644
--- a/llvm/include/llvm/IR/IRBuilder.h
+++ b/llvm/include/llvm/IR/IRBuilder.h
@@ -1704,21 +1704,30 @@ class IRBuilderBase {
return Insert(new FenceInst(Context, Ordering, SSID), Name);
}
- AtomicCmpXchgInst *CreateAtomicCmpXchg(
- Value *Ptr, Value *Cmp, Value *New, AtomicOrdering SuccessOrdering,
- AtomicOrdering FailureOrdering, SyncScope::ID SSID = SyncScope::System) {
- const DataLayout &DL = BB->getModule()->getDataLayout();
- Align Alignment(DL.getTypeStoreSize(New->getType()));
- return Insert(new AtomicCmpXchgInst(
- Ptr, Cmp, New, Alignment, SuccessOrdering, FailureOrdering, SSID));
+ AtomicCmpXchgInst *
+ CreateAtomicCmpXchg(Value *Ptr, Value *Cmp, Value *New, MaybeAlign Align,
+ AtomicOrdering SuccessOrdering,
+ AtomicOrdering FailureOrdering,
+ SyncScope::ID SSID = SyncScope::System) {
+ if (!Align) {
+ const DataLayout &DL = BB->getModule()->getDataLayout();
+ Align = llvm::Align(DL.getTypeStoreSize(New->getType()));
+ }
+
+ return Insert(new AtomicCmpXchgInst(Ptr, Cmp, New, *Align, SuccessOrdering,
+ FailureOrdering, SSID));
}
- AtomicRMWInst *CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr, Value *Val,
+ AtomicRMWInst *CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr,
+ Value *Val, MaybeAlign Align,
AtomicOrdering Ordering,
SyncScope::ID SSID = SyncScope::System) {
- const DataLayout &DL = BB->getModule()->getDataLayout();
- Align Alignment(DL.getTypeStoreSize(Val->getType()));
- return Insert(new AtomicRMWInst(Op, Ptr, Val, Alignment, Ordering, SSID));
+ if (!Align) {
+ const DataLayout &DL = BB->getModule()->getDataLayout();
+ Align = llvm::Align(DL.getTypeStoreSize(Val->getType()));
+ }
+
+ return Insert(new AtomicRMWInst(Op, Ptr, Val, *Align, Ordering, SSID));
}
Value *CreateGEP(Value *Ptr, ArrayRef<Value *> IdxList,
diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index 4026022caa07..f04a8d25ec1b 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -80,10 +80,11 @@ namespace {
bool tryExpandAtomicRMW(AtomicRMWInst *AI);
Value *
insertRMWLLSCLoop(IRBuilder<> &Builder, Type *ResultTy, Value *Addr,
- AtomicOrdering MemOpOrder,
+ Align AddrAlign, AtomicOrdering MemOpOrder,
function_ref<Value *(IRBuilder<> &, Value *)> PerformOp);
void expandAtomicOpToLLSC(
- Instruction *I, Type *ResultTy, Value *Addr, AtomicOrdering MemOpOrder,
+ Instruction *I, Type *ResultTy, Value *Addr, Align AddrAlign,
+ AtomicOrdering MemOpOrder,
function_ref<Value *(IRBuilder<> &, Value *)> PerformOp);
void expandPartwordAtomicRMW(
AtomicRMWInst *I,
@@ -95,7 +96,7 @@ namespace {
AtomicCmpXchgInst *convertCmpXchgToIntegerType(AtomicCmpXchgInst *CI);
static Value *insertRMWCmpXchgLoop(
- IRBuilder<> &Builder, Type *ResultType, Value *Addr,
+ IRBuilder<> &Builder, Type *ResultType, Value *Addr, Align AddrAlign,
AtomicOrdering MemOpOrder,
function_ref<Value *(IRBuilder<> &, Value *)> PerformOp,
CreateCmpXchgInstFun CreateCmpXchg);
@@ -368,7 +369,8 @@ bool AtomicExpand::tryExpandAtomicLoad(LoadInst *LI) {
return false;
case TargetLoweringBase::AtomicExpansionKind::LLSC:
expandAtomicOpToLLSC(
- LI, LI->getType(), LI->getPointerOperand(), LI->getOrdering(),
+ LI, LI->getType(), LI->getPointerOperand(), LI->getAlign(),
+ LI->getOrdering(),
[](IRBuilder<> &Builder, Value *Loaded) { return Loaded; });
return true;
case TargetLoweringBase::AtomicExpansionKind::LLOnly:
@@ -407,7 +409,7 @@ bool AtomicExpand::expandAtomicLoadToCmpXchg(LoadInst *LI) {
Constant *DummyVal = Constant::getNullValue(Ty);
Value *Pair = Builder.CreateAtomicCmpXchg(
- Addr, DummyVal, DummyVal, Order,
+ Addr, DummyVal, DummyVal, LI->getAlign(), Order,
AtomicCmpXchgInst::getStrongestFailureOrdering(Order));
Value *Loaded = Builder.CreateExtractValue(Pair, 0, "loaded");
@@ -454,9 +456,9 @@ bool AtomicExpand::expandAtomicStore(StoreInst *SI) {
// It is the responsibility of the target to only signal expansion via
// shouldExpandAtomicRMW in cases where this is required and possible.
IRBuilder<> Builder(SI);
- AtomicRMWInst *AI =
- Builder.CreateAtomicRMW(AtomicRMWInst::Xchg, SI->getPointerOperand(),
- SI->getValueOperand(), SI->getOrdering());
+ AtomicRMWInst *AI = Builder.CreateAtomicRMW(
+ AtomicRMWInst::Xchg, SI->getPointerOperand(), SI->getValueOperand(),
+ SI->getAlign(), SI->getOrdering());
SI->eraseFromParent();
// Now we have an appropriate swap instruction, lower it as usual.
@@ -464,9 +466,9 @@ bool AtomicExpand::expandAtomicStore(StoreInst *SI) {
}
static void createCmpXchgInstFun(IRBuilder<> &Builder, Value *Addr,
- Value *Loaded, Value *NewVal,
- AtomicOrdering MemOpOrder,
- Value *&Success, Value *&NewLoaded) {
+ Value *Loaded, Value *NewVal, Align AddrAlign,
+ AtomicOrdering MemOpOrder, Value *&Success,
+ Value *&NewLoaded) {
Type *OrigTy = NewVal->getType();
// This code can go away when cmpxchg supports FP types.
@@ -479,8 +481,8 @@ static void createCmpXchgInstFun(IRBuilder<> &Builder, Value *Addr,
Loaded = Builder.CreateBitCast(Loaded, IntTy);
}
- Value* Pair = Builder.CreateAtomicCmpXchg(
- Addr, Loaded, NewVal, MemOpOrder,
+ Value *Pair = Builder.CreateAtomicCmpXchg(
+ Addr, Loaded, NewVal, AddrAlign, MemOpOrder,
AtomicCmpXchgInst::getStrongestFailureOrdering(MemOpOrder));
Success = Builder.CreateExtractValue(Pair, 1, "success");
NewLoaded = Builder.CreateExtractValue(Pair, 0, "newloaded");
@@ -546,7 +548,7 @@ bool AtomicExpand::tryExpandAtomicRMW(AtomicRMWInst *AI) {
AI->getValOperand());
};
expandAtomicOpToLLSC(AI, AI->getType(), AI->getPointerOperand(),
- AI->getOrdering(), PerformOp);
+ AI->getAlign(), AI->getOrdering(), PerformOp);
}
return true;
}
@@ -581,6 +583,7 @@ struct PartwordMaskValues {
Type *WordType = nullptr;
Type *ValueType = nullptr;
Value *AlignedAddr = nullptr;
+ Align AlignedAddrAlignment;
// The remaining fields can be null.
Value *ShiftAmt = nullptr;
Value *Mask = nullptr;
@@ -603,6 +606,7 @@ raw_ostream &operator<<(raw_ostream &O, const PartwordMaskValues &PMV) {
PrintObj(PMV.ValueType);
O << " AlignedAddr: ";
PrintObj(PMV.AlignedAddr);
+ O << " AlignedAddrAlignment: " << PMV.AlignedAddrAlignment.value() << '\n';
O << " ShiftAmt: ";
PrintObj(PMV.ShiftAmt);
O << " Mask: ";
@@ -633,6 +637,7 @@ raw_ostream &operator<<(raw_ostream &O, const PartwordMaskValues &PMV) {
/// Inv_Mask: The inverse of Mask.
static PartwordMaskValues createMaskInstrs(IRBuilder<> &Builder, Instruction *I,
Type *ValueType, Value *Addr,
+ Align AddrAlign,
unsigned MinWordSize) {
PartwordMaskValues PMV;
@@ -646,6 +651,7 @@ static PartwordMaskValues createMaskInstrs(IRBuilder<> &Builder, Instruction *I,
: ValueType;
if (PMV.ValueType == PMV.WordType) {
PMV.AlignedAddr = Addr;
+ PMV.AlignedAddrAlignment = AddrAlign;
return PMV;
}
@@ -654,10 +660,12 @@ static PartwordMaskValues createMaskInstrs(IRBuilder<> &Builder, Instruction *I,
Type *WordPtrType =
PMV.WordType->getPointerTo(Addr->getType()->getPointerAddressSpace());
+ // TODO: we could skip some of this if AddrAlign >= MinWordSize.
Value *AddrInt = Builder.CreatePtrToInt(Addr, DL.getIntPtrType(Ctx));
PMV.AlignedAddr = Builder.CreateIntToPtr(
Builder.CreateAnd(AddrInt, ~(uint64_t)(MinWordSize - 1)), WordPtrType,
"AlignedAddr");
+ PMV.AlignedAddrAlignment = Align(MinWordSize);
Value *PtrLSB = Builder.CreateAnd(AddrInt, MinWordSize - 1, "PtrLSB");
if (DL.isLittleEndian()) {
@@ -765,7 +773,7 @@ void AtomicExpand::expandPartwordAtomicRMW(
PartwordMaskValues PMV =
createMaskInstrs(Builder, AI, AI->getType(), AI->getPointerOperand(),
- TLI->getMinCmpXchgSizeInBits() / 8);
+ AI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8);
Value *ValOperand_Shifted =
Builder.CreateShl(Builder.CreateZExt(AI->getValOperand(), PMV.WordType),
@@ -778,13 +786,14 @@ void AtomicExpand::expandPartwordAtomicRMW(
Value *OldResult;
if (ExpansionKind == TargetLoweringBase::AtomicExpansionKind::CmpXChg) {
- OldResult =
- insertRMWCmpXchgLoop(Builder, PMV.WordType, PMV.AlignedAddr, MemOpOrder,
- PerformPartwordOp, createCmpXchgInstFun);
+ OldResult = insertRMWCmpXchgLoop(Builder, PMV.WordType, PMV.AlignedAddr,
+ PMV.AlignedAddrAlignment, MemOpOrder,
+ PerformPartwordOp, createCmpXchgInstFun);
} else {
assert(ExpansionKind == TargetLoweringBase::AtomicExpansionKind::LLSC);
OldResult = insertRMWLLSCLoop(Builder, PMV.WordType, PMV.AlignedAddr,
- MemOpOrder, PerformPartwordOp);
+ PMV.AlignedAddrAlignment, MemOpOrder,
+ PerformPartwordOp);
}
Value *FinalOldResult = extractMaskedValue(Builder, OldResult, PMV);
@@ -803,7 +812,7 @@ AtomicRMWInst *AtomicExpand::widenPartwordAtomicRMW(AtomicRMWInst *AI) {
PartwordMaskValues PMV =
createMaskInstrs(Builder, AI, AI->getType(), AI->getPointerOperand(),
- TLI->getMinCmpXchgSizeInBits() / 8);
+ AI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8);
Value *ValOperand_Shifted =
Builder.CreateShl(Builder.CreateZExt(AI->getValOperand(), PMV.WordType),
@@ -817,8 +826,9 @@ AtomicRMWInst *AtomicExpand::widenPartwordAtomicRMW(AtomicRMWInst *AI) {
else
NewOperand = ValOperand_Shifted;
- AtomicRMWInst *NewAI = Builder.CreateAtomicRMW(Op, PMV.AlignedAddr,
- NewOperand, AI->getOrdering());
+ AtomicRMWInst *NewAI =
+ Builder.CreateAtomicRMW(Op, PMV.AlignedAddr, NewOperand,
+ PMV.AlignedAddrAlignment, AI->getOrdering());
Value *FinalOldResult = extractMaskedValue(Builder, NewAI, PMV);
AI->replaceAllUsesWith(FinalOldResult);
@@ -871,8 +881,6 @@ bool AtomicExpand::expandPartwordCmpXchg(AtomicCmpXchgInst *CI) {
IRBuilder<> Builder(CI);
LLVMContext &Ctx = Builder.getContext();
- const int WordSize = TLI->getMinCmpXchgSizeInBits() / 8;
-
BasicBlock *EndBB =
BB->splitBasicBlock(CI->getIterator(), "partword.cmpxchg.end");
auto FailureBB =
@@ -884,8 +892,9 @@ bool AtomicExpand::expandPartwordCmpXchg(AtomicCmpXchgInst *CI) {
std::prev(BB->end())->eraseFromParent();
Builder.SetInsertPoint(BB);
- PartwordMaskValues PMV = createMaskInstrs(
- Builder, CI, CI->getCompareOperand()->getType(), Addr, WordSize);
+ PartwordMaskValues PMV =
+ createMaskInstrs(Builder, CI, CI->getCompareOperand()->getType(), Addr,
+ CI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8);
// Shift the incoming values over, into the right location in the word.
Value *NewVal_Shifted =
@@ -909,8 +918,8 @@ bool AtomicExpand::expandPartwordCmpXchg(AtomicCmpXchgInst *CI) {
Value *FullWord_NewVal = Builder.CreateOr(Loaded_MaskOut, NewVal_Shifted);
Value *FullWord_Cmp = Builder.CreateOr(Loaded_MaskOut, Cmp_Shifted);
AtomicCmpXchgInst *NewCI = Builder.CreateAtomicCmpXchg(
- PMV.AlignedAddr, FullWord_Cmp, FullWord_NewVal, CI->getSuccessOrdering(),
- CI->getFailureOrdering(), CI->getSyncScopeID());
+ PMV.AlignedAddr, FullWord_Cmp, FullWord_NewVal, PMV.AlignedAddrAlignment,
+ CI->getSuccessOrdering(), CI->getFailureOrdering(), CI->getSyncScopeID());
NewCI->setVolatile(CI->isVolatile());
// When we're building a strong cmpxchg, we need a loop, so you
// might think we could use a weak cmpxchg inside. But, using strong
@@ -953,11 +962,12 @@ bool AtomicExpand::expandPartwordCmpXchg(AtomicCmpXchgInst *CI) {
}
void AtomicExpand::expandAtomicOpToLLSC(
- Instruction *I, Type *ResultType, Value *Addr, AtomicOrdering MemOpOrder,
+ Instruction *I, Type *ResultType, Value *Addr, Align AddrAlign,
+ AtomicOrdering MemOpOrder,
function_ref<Value *(IRBuilder<> &, Value *)> PerformOp) {
IRBuilder<> Builder(I);
- Value *Loaded =
- insertRMWLLSCLoop(Builder, ResultType, Addr, MemOpOrder, PerformOp);
+ Value *Loaded = insertRMWLLSCLoop(Builder, ResultType, Addr, AddrAlign,
+ MemOpOrder, PerformOp);
I->replaceAllUsesWith(Loaded);
I->eraseFromParent();
@@ -968,7 +978,7 @@ void AtomicExpand::expandAtomicRMWToMaskedIntrinsic(AtomicRMWInst *AI) {
PartwordMaskValues PMV =
createMaskInstrs(Builder, AI, AI->getType(), AI->getPointerOperand(),
- TLI->getMinCmpXchgSizeInBits() / 8);
+ AI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8);
// The value operand must be sign-extended for signed min/max so that the
// target's signed comparison instructions can be used. Otherwise, just
@@ -994,7 +1004,7 @@ void AtomicExpand::expandAtomicCmpXchgToMaskedIntrinsic(AtomicCmpXchgInst *CI) {
PartwordMaskValues PMV = createMaskInstrs(
Builder, CI, CI->getCompareOperand()->getType(), CI->getPointerOperand(),
- TLI->getMinCmpXchgSizeInBits() / 8);
+ CI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8);
Value *CmpVal_Shifted = Builder.CreateShl(
Builder.CreateZExt(CI->getCompareOperand(), PMV.WordType), PMV.ShiftAmt,
@@ -1017,13 +1027,16 @@ void AtomicExpand::expandAtomicCmpXchgToMaskedIntrinsic(AtomicCmpXchgInst *CI) {
}
Value *AtomicExpand::insertRMWLLSCLoop(
- IRBuilder<> &Builder, Type *ResultTy, Value *Addr,
+ IRBuilder<> &Builder, Type *ResultTy, Value *Addr, Align AddrAlign,
AtomicOrdering MemOpOrder,
function_ref<Value *(IRBuilder<> &, Value *)> PerformOp) {
LLVMContext &Ctx = Builder.getContext();
BasicBlock *BB = Builder.GetInsertBlock();
Function *F = BB->getParent();
+ assert(AddrAlign >= ResultTy->getPrimitiveSizeInBits() / 8 &&
+ "Expected at least natural alignment at this point.");
+
// Given: atomicrmw some_op iN* %addr, iN %incr ordering
//
// The standard expansion we produce is:
@@ -1082,11 +1095,9 @@ AtomicCmpXchgInst *AtomicExpand::convertCmpXchgToIntegerType(AtomicCmpXchgInst *
Value *NewCmp = Builder.CreatePtrToInt(CI->getCompareOperand(), NewTy);
Value *NewNewVal = Builder.CreatePtrToInt(CI->getNewValOperand(), NewTy);
-
- auto *NewCI = Builder.CreateAtomicCmpXchg(NewAddr, NewCmp, NewNewVal,
- CI->getSuccessOrdering(),
- CI->getFailureOrdering(),
- CI->getSyncScopeID());
+ auto *NewCI = Builder.CreateAtomicCmpXchg(
+ NewAddr, NewCmp, NewNewVal, CI->getAlign(), CI->getSuccessOrdering(),
+ CI->getFailureOrdering(), CI->getSyncScopeID());
NewCI->setVolatile(CI->isVolatile());
NewCI->setWeak(CI->isWeak());
LLVM_DEBUG(dbgs() << "Replaced " << *CI << " with " << *NewCI << "\n");
@@ -1211,7 +1222,7 @@ bool AtomicExpand::expandAtomicCmpXchg(AtomicCmpXchgInst *CI) {
PartwordMaskValues PMV =
createMaskInstrs(Builder, CI, CI->getCompareOperand()->getType(), Addr,
- TLI->getMinCmpXchgSizeInBits() / 8);
+ CI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8);
Builder.CreateBr(StartBB);
// Start the main loop block now that we've taken care of the preliminaries.
@@ -1379,7 +1390,7 @@ bool AtomicExpand::simplifyIdempotentRMW(AtomicRMWInst* RMWI) {
}
Value *AtomicExpand::insertRMWCmpXchgLoop(
- IRBuilder<> &Builder, Type *ResultTy, Value *Addr,
+ IRBuilder<> &Builder, Type *ResultTy, Value *Addr, Align AddrAlign,
AtomicOrdering MemOpOrder,
function_ref<Value *(IRBuilder<> &, Value *)> PerformOp,
CreateCmpXchgInstFun CreateCmpXchg) {
@@ -1411,9 +1422,7 @@ Value *AtomicExpand::insertRMWCmpXchgLoop(
// the branch entirely.
std::prev(BB->end())->eraseFromParent();
Builder.SetInsertPoint(BB);
- LoadInst *InitLoaded = Builder.CreateLoad(ResultTy, Addr);
- // Atomics require at least natural alignment.
- InitLoaded->setAlignment(Align(ResultTy->getPrimitiveSizeInBits() / 8));
+ LoadInst *InitLoaded = Builder.CreateAlignedLoad(ResultTy, Addr, AddrAlign);
Builder.CreateBr(LoopBB);
// Start the main loop block now that we've taken care of the preliminaries.
@@ -1426,7 +1435,7 @@ Value *AtomicExpand::insertRMWCmpXchgLoop(
Value *NewLoaded = nullptr;
Value *Success = nullptr;
- CreateCmpXchg(Builder, Addr, Loaded, NewVal,
+ CreateCmpXchg(Builder, Addr, Loaded, NewVal, AddrAlign,
MemOpOrder == AtomicOrdering::Unordered
? AtomicOrdering::Monotonic
: MemOpOrder,
@@ -1466,7 +1475,8 @@ bool llvm::expandAtomicRMWToCmpXchg(AtomicRMWInst *AI,
CreateCmpXchgInstFun CreateCmpXchg) {
IRBuilder<> Builder(AI);
Value *Loaded = AtomicExpand::insertRMWCmpXchgLoop(
- Builder, AI->getType(), AI->getPointerOperand(), AI->getOrdering(),
+ Builder, AI->getType(), AI->getPointerOperand(), AI->getAlign(),
+ AI->getOrdering(),
[&](IRBuilder<> &Builder, Value *Loaded) {
return performAtomicOp(AI->getOperation(), Builder, Loaded,
AI->getValOperand());
@@ -1614,20 +1624,20 @@ void AtomicExpand::expandAtomicRMWToLibcall(AtomicRMWInst *I) {
// libcalls (add/sub/etc) and we needed a generic. So, expand to a
// CAS libcall, via a CAS loop, instead.
if (!Success) {
- expandAtomicRMWToCmpXchg(I, [this](IRBuilder<> &Builder, Value *Addr,
- Value *Loaded, Value *NewVal,
- AtomicOrdering MemOpOrder,
- Value *&Success, Value *&NewLoaded) {
- // Create the CAS instruction normally...
- AtomicCmpXchgInst *Pair = Builder.CreateAtomicCmpXchg(
- Addr, Loaded, NewVal, MemOpOrder,
- AtomicCmpXchgInst::getStrongestFailureOrdering(MemOpOrder));
- Success = Builder.CreateExtractValue(Pair, 1, "success");
- NewLoaded = Builder.CreateExtractValue(Pair, 0, "newloaded");
-
- // ...and then expand the CAS into a libcall.
- expandAtomicCASToLibcall(Pair);
- });
+ expandAtomicRMWToCmpXchg(
+ I, [this](IRBuilder<> &Builder, Value *Addr, Value *Loaded,
+ Value *NewVal, Align Alignment, AtomicOrdering MemOpOrder,
+ Value *&Success, Value *&NewLoaded) {
+ // Create the CAS instruction normally...
+ AtomicCmpXchgInst *Pair = Builder.CreateAtomicCmpXchg(
+ Addr, Loaded, NewVal, Alignment, MemOpOrder,
+ AtomicCmpXchgInst::getStrongestFailureOrdering(MemOpOrder));
+ Success = Builder.CreateExtractValue(Pair, 1, "success");
+ NewLoaded = Builder.CreateExtractValue(Pair, 0, "newloaded");
+
+ // ...and then expand the CAS into a libcall.
+ expandAtomicCASToLibcall(Pair);
+ });
}
}
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index a4ec5388b0c7..e0d152b5ec21 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -3594,7 +3594,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
Name.startswith("atomic.load.add.f64.p"))) {
Value *Ptr = CI->getArgOperand(0);
Value *Val = CI->getArgOperand(1);
- Rep = Builder.CreateAtomicRMW(AtomicRMWInst::FAdd, Ptr, Val,
+ Rep = Builder.CreateAtomicRMW(AtomicRMWInst::FAdd, Ptr, Val, MaybeAlign(),
AtomicOrdering::SequentiallyConsistent);
} else if (IsNVVM && (Name == "max.i" || Name == "max.ll" ||
Name == "max.ui" || Name == "max.ull")) {
diff --git a/llvm/lib/IR/Core.cpp b/llvm/lib/IR/Core.cpp
index d38cd6fe19f3..fdcd64370c05 100644
--- a/llvm/lib/IR/Core.cpp
+++ b/llvm/lib/IR/Core.cpp
@@ -3977,9 +3977,10 @@ LLVMValueRef LLVMBuildAtomicRMW(LLVMBuilderRef B,LLVMAtomicRMWBinOp op,
LLVMAtomicOrdering ordering,
LLVMBool singleThread) {
AtomicRMWInst::BinOp intop = mapFromLLVMRMWBinOp(op);
- return wrap(unwrap(B)->CreateAtomicRMW(intop, unwrap(PTR), unwrap(Val),
- mapFromLLVMOrdering(ordering), singleThread ? SyncScope::SingleThread
- : SyncScope::System));
+ return wrap(unwrap(B)->CreateAtomicRMW(
+ intop, unwrap(PTR), unwrap(Val), MaybeAlign(),
+ mapFromLLVMOrdering(ordering),
+ singleThread ? SyncScope::SingleThread : SyncScope::System));
}
LLVMValueRef LLVMBuildAtomicCmpXchg(LLVMBuilderRef B, LLVMValueRef Ptr,
@@ -3988,10 +3989,11 @@ LLVMValueRef LLVMBuildAtomicCmpXchg(LLVMBuilderRef B, LLVMValueRef Ptr,
LLVMAtomicOrdering FailureOrdering,
LLVMBool singleThread) {
- return wrap(unwrap(B)->CreateAtomicCmpXchg(unwrap(Ptr), unwrap(Cmp),
- unwrap(New), mapFromLLVMOrdering(SuccessOrdering),
- mapFromLLVMOrdering(FailureOrdering),
- singleThread ? SyncScope::SingleThread : SyncScope::System));
+ return wrap(unwrap(B)->CreateAtomicCmpXchg(
+ unwrap(Ptr), unwrap(Cmp), unwrap(New), MaybeAlign(),
+ mapFromLLVMOrdering(SuccessOrdering),
+ mapFromLLVMOrdering(FailureOrdering),
+ singleThread ? SyncScope::SingleThread : SyncScope::System));
}
unsigned LLVMGetNumMaskElements(LLVMValueRef SVInst) {
diff --git a/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp b/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp
index bc1068f3a530..c2a3b91f2a5a 100644
--- a/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp
@@ -965,7 +965,7 @@ bool GCOVProfiler::emitProfileNotes(
Counters->getValueType(), Counters, 0, I);
if (Options.Atomic) {
Builder.CreateAtomicRMW(AtomicRMWInst::Add, V, Builder.getInt64(1),
- AtomicOrdering::Monotonic);
+ MaybeAlign(), AtomicOrdering::Monotonic);
} else {
Value *Count =
Builder.CreateLoad(Builder.getInt64Ty(), V, "gcov_ctr");
diff --git a/llvm/lib/Transforms/Instrumentation/InstrOrderFile.cpp b/llvm/lib/Transforms/Instrumentation/InstrOrderFile.cpp
index 853385fbf863..071feb876540 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrOrderFile.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrOrderFile.cpp
@@ -144,7 +144,7 @@ struct InstrOrderFile {
// Fill up UpdateOrderFileBB: grab the index, update the buffer!
Value *IdxVal = updateB.CreateAtomicRMW(
AtomicRMWInst::Add, BufferIdx, ConstantInt::get(Int32Ty, 1),
- AtomicOrdering::SequentiallyConsistent);
+ MaybeAlign(), AtomicOrdering::SequentiallyConsistent);
// We need to wrap around the index to fit it inside the buffer.
Value *WrappedIdx = updateB.CreateAnd(
IdxVal, ConstantInt::get(Int32Ty, INSTR_ORDER_FILE_BUFFER_MASK));
diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index 5aa8a1513817..70fef4678467 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -205,6 +205,7 @@ class PGOCounterPromoterHelper : public LoadAndStorePromoter {
// automic update currently can only be promoted across the current
// loop, not the whole loop nest.
Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, LiveInValue,
+ MaybeAlign(),
AtomicOrdering::SequentiallyConsistent);
else {
LoadInst *OldVal = Builder.CreateLoad(Ty, Addr, "pgocount.promoted");
@@ -702,7 +703,7 @@ void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) {
if (Options.Atomic || AtomicCounterUpdateAll ||
(Index == 0 && AtomicFirstCounter)) {
Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, Inc->getStep(),
- AtomicOrdering::Monotonic);
+ MaybeAlign(), AtomicOrdering::Monotonic);
} else {
Value *IncStep = Inc->getStep();
Value *Load = Builder.CreateLoad(IncStep->getType(), Addr, "pgocount");
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
index 935993769bd0..076b8ed96e4e 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
@@ -1526,6 +1526,7 @@ def AtomicOrdering : I64EnumAttr<
def LLVM_AtomicRMWType : AnyTypeOf<[LLVM_AnyFloat, LLVM_AnyInteger]>;
+// FIXME: Need to add alignment attribute to MLIR atomicrmw operation.
def LLVM_AtomicRMWOp : LLVM_Op<"atomicrmw"> {
let arguments = (ins AtomicBinOp:$bin_op,
LLVM_PointerTo<LLVM_AtomicRMWType>:$ptr,
@@ -1533,6 +1534,7 @@ def LLVM_AtomicRMWOp : LLVM_Op<"atomicrmw"> {
let results = (outs LLVM_AtomicRMWType:$res);
let llvmBuilder = [{
$res = builder.CreateAtomicRMW(getLLVMAtomicBinOp($bin_op), $ptr, $val,
+ llvm::MaybeAlign(),
getLLVMAtomicOrdering($ordering));
}];
let parser = [{ return parseAtomicRMWOp(parser, result); }];
@@ -1553,6 +1555,7 @@ def LLVM_AtomicCmpXchgResultType : Type<And<[
"an LLVM struct type with any integer or pointer followed by a single-bit "
"integer">;
+// FIXME: Need to add alignment attribute to MLIR cmpxchg operation.
def LLVM_AtomicCmpXchgOp : LLVM_Op<"cmpxchg"> {
let arguments = (ins LLVM_PointerTo<LLVM_AtomicCmpXchgType>:$ptr,
LLVM_AtomicCmpXchgType:$cmp, LLVM_AtomicCmpXchgType:$val,
@@ -1560,7 +1563,7 @@ def LLVM_AtomicCmpXchgOp : LLVM_Op<"cmpxchg"> {
AtomicOrdering:$failure_ordering);
let results = (outs LLVM_AtomicCmpXchgResultType:$res);
let llvmBuilder = [{
- $res = builder.CreateAtomicCmpXchg($ptr, $cmp, $val,
+ $res = builder.CreateAtomicCmpXchg($ptr, $cmp, $val, llvm::MaybeAlign(),
getLLVMAtomicOrdering($success_ordering),
getLLVMAtomicOrdering($failure_ordering));
}];
More information about the cfe-commits
mailing list