[llvm] 2d7a2c1 - AtomicExpand: Refactor atomic instruction handling (#102914)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 13 08:51:58 PDT 2024
Author: Matt Arsenault
Date: 2024-08-13T19:51:53+04:00
New Revision: 2d7a2c1212d3539a59afec7e998af8caaa7ea1c0
URL: https://github.com/llvm/llvm-project/commit/2d7a2c1212d3539a59afec7e998af8caaa7ea1c0
DIFF: https://github.com/llvm/llvm-project/commit/2d7a2c1212d3539a59afec7e998af8caaa7ea1c0.diff
LOG: AtomicExpand: Refactor atomic instruction handling (#102914)
Move the processing of an instruction into a helper function. Also
avoid redundant checking for all types of atomic instructions.
Including the assert, it was effectively performing the same check
3 times.
Added:
Modified:
llvm/lib/CodeGen/AtomicExpandPass.cpp
Removed:
################################################################################
diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index 49836b914784f..d8f33c42a8a14 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -119,6 +119,8 @@ class AtomicExpandImpl {
llvm::expandAtomicRMWToCmpXchg(AtomicRMWInst *AI,
CreateCmpXchgInstFun CreateCmpXchg);
+ bool processAtomicInstr(Instruction *I);
+
public:
bool run(Function &F, const TargetMachine *TM);
};
@@ -203,6 +205,143 @@ static bool atomicSizeSupported(const TargetLowering *TLI, Inst *I) {
Size <= TLI->getMaxAtomicSizeInBitsSupported() / 8;
}
+bool AtomicExpandImpl::processAtomicInstr(Instruction *I) {
+ auto *LI = dyn_cast<LoadInst>(I);
+ auto *SI = dyn_cast<StoreInst>(I);
+ auto *RMWI = dyn_cast<AtomicRMWInst>(I);
+ auto *CASI = dyn_cast<AtomicCmpXchgInst>(I);
+
+ bool MadeChange = false;
+
+ // If the Size/Alignment is not supported, replace with a libcall.
+ if (LI) {
+ if (!LI->isAtomic())
+ return false;
+
+ if (!atomicSizeSupported(TLI, LI)) {
+ expandAtomicLoadToLibcall(LI);
+ return true;
+ }
+
+ if (TLI->shouldCastAtomicLoadInIR(LI) ==
+ TargetLoweringBase::AtomicExpansionKind::CastToInteger) {
+ I = LI = convertAtomicLoadToIntegerType(LI);
+ MadeChange = true;
+ }
+ } else if (SI) {
+ if (!SI->isAtomic())
+ return false;
+
+ if (!atomicSizeSupported(TLI, SI)) {
+ expandAtomicStoreToLibcall(SI);
+ return true;
+ }
+
+ if (TLI->shouldCastAtomicStoreInIR(SI) ==
+ TargetLoweringBase::AtomicExpansionKind::CastToInteger) {
+ I = SI = convertAtomicStoreToIntegerType(SI);
+ MadeChange = true;
+ }
+ } else if (RMWI) {
+ if (!atomicSizeSupported(TLI, RMWI)) {
+ expandAtomicRMWToLibcall(RMWI);
+ return true;
+ }
+
+ if (TLI->shouldCastAtomicRMWIInIR(RMWI) ==
+ TargetLoweringBase::AtomicExpansionKind::CastToInteger) {
+ I = RMWI = convertAtomicXchgToIntegerType(RMWI);
+ MadeChange = true;
+ }
+ } else if (CASI) {
+ if (!atomicSizeSupported(TLI, CASI)) {
+ expandAtomicCASToLibcall(CASI);
+ return true;
+ }
+
+ // TODO: when we're ready to make the change at the IR level, we can
+ // extend convertCmpXchgToInteger for floating point too.
+ if (CASI->getCompareOperand()->getType()->isPointerTy()) {
+ // TODO: add a TLI hook to control this so that each target can
+ // convert to lowering the original type one at a time.
+ I = CASI = convertCmpXchgToIntegerType(CASI);
+ MadeChange = true;
+ }
+ } else
+ return false;
+
+ if (TLI->shouldInsertFencesForAtomic(I)) {
+ auto FenceOrdering = AtomicOrdering::Monotonic;
+ if (LI && isAcquireOrStronger(LI->getOrdering())) {
+ FenceOrdering = LI->getOrdering();
+ LI->setOrdering(AtomicOrdering::Monotonic);
+ } else if (SI && isReleaseOrStronger(SI->getOrdering())) {
+ FenceOrdering = SI->getOrdering();
+ SI->setOrdering(AtomicOrdering::Monotonic);
+ } else if (RMWI && (isReleaseOrStronger(RMWI->getOrdering()) ||
+ isAcquireOrStronger(RMWI->getOrdering()))) {
+ FenceOrdering = RMWI->getOrdering();
+ RMWI->setOrdering(AtomicOrdering::Monotonic);
+ } else if (CASI &&
+ TLI->shouldExpandAtomicCmpXchgInIR(CASI) ==
+ TargetLoweringBase::AtomicExpansionKind::None &&
+ (isReleaseOrStronger(CASI->getSuccessOrdering()) ||
+ isAcquireOrStronger(CASI->getSuccessOrdering()) ||
+ isAcquireOrStronger(CASI->getFailureOrdering()))) {
+ // If a compare and swap is lowered to LL/SC, we can do smarter fence
+ // insertion, with a stronger one on the success path than on the
+ // failure path. As a result, fence insertion is directly done by
+ // expandAtomicCmpXchg in that case.
+ FenceOrdering = CASI->getMergedOrdering();
+ CASI->setSuccessOrdering(AtomicOrdering::Monotonic);
+ CASI->setFailureOrdering(AtomicOrdering::Monotonic);
+ }
+
+ if (FenceOrdering != AtomicOrdering::Monotonic) {
+ MadeChange |= bracketInstWithFences(I, FenceOrdering);
+ }
+ } else if (I->hasAtomicStore() &&
+ TLI->shouldInsertTrailingFenceForAtomicStore(I)) {
+ auto FenceOrdering = AtomicOrdering::Monotonic;
+ if (SI)
+ FenceOrdering = SI->getOrdering();
+ else if (RMWI)
+ FenceOrdering = RMWI->getOrdering();
+ else if (CASI && TLI->shouldExpandAtomicCmpXchgInIR(CASI) !=
+ TargetLoweringBase::AtomicExpansionKind::LLSC)
+ // LLSC is handled in expandAtomicCmpXchg().
+ FenceOrdering = CASI->getSuccessOrdering();
+
+ IRBuilder Builder(I);
+ if (auto TrailingFence =
+ TLI->emitTrailingFence(Builder, I, FenceOrdering)) {
+ TrailingFence->moveAfter(I);
+ MadeChange = true;
+ }
+ }
+
+ if (LI)
+ MadeChange |= tryExpandAtomicLoad(LI);
+ else if (SI)
+ MadeChange |= tryExpandAtomicStore(SI);
+ else if (RMWI) {
+ // There are two
diff erent ways of expanding RMW instructions:
+ // - into a load if it is idempotent
+ // - into a Cmpxchg/LL-SC loop otherwise
+ // we try them in that order.
+
+ if (isIdempotentRMW(RMWI) && simplifyIdempotentRMW(RMWI)) {
+ MadeChange = true;
+
+ } else {
+ MadeChange |= tryExpandAtomicRMW(RMWI);
+ }
+ } else if (CASI)
+ MadeChange |= tryExpandAtomicCmpXchg(CASI);
+
+ return MadeChange;
+}
+
bool AtomicExpandImpl::run(Function &F, const TargetMachine *TM) {
const auto *Subtarget = TM->getSubtargetImpl(F);
if (!Subtarget->enableAtomicExpand())
@@ -210,6 +349,8 @@ bool AtomicExpandImpl::run(Function &F, const TargetMachine *TM) {
TLI = Subtarget->getTargetLowering();
DL = &F.getDataLayout();
+ bool MadeChange = false;
+
SmallVector<Instruction *, 1> AtomicInsts;
// Changing control-flow while iterating through it is a bad idea, so gather a
@@ -218,134 +359,11 @@ bool AtomicExpandImpl::run(Function &F, const TargetMachine *TM) {
if (I.isAtomic() && !isa<FenceInst>(&I))
AtomicInsts.push_back(&I);
- bool MadeChange = false;
for (auto *I : AtomicInsts) {
- auto LI = dyn_cast<LoadInst>(I);
- auto SI = dyn_cast<StoreInst>(I);
- auto RMWI = dyn_cast<AtomicRMWInst>(I);
- auto CASI = dyn_cast<AtomicCmpXchgInst>(I);
- assert((LI || SI || RMWI || CASI) && "Unknown atomic instruction");
-
- // If the Size/Alignment is not supported, replace with a libcall.
- if (LI) {
- if (!atomicSizeSupported(TLI, LI)) {
- expandAtomicLoadToLibcall(LI);
- MadeChange = true;
- continue;
- }
- } else if (SI) {
- if (!atomicSizeSupported(TLI, SI)) {
- expandAtomicStoreToLibcall(SI);
- MadeChange = true;
- continue;
- }
- } else if (RMWI) {
- if (!atomicSizeSupported(TLI, RMWI)) {
- expandAtomicRMWToLibcall(RMWI);
- MadeChange = true;
- continue;
- }
- } else if (CASI) {
- if (!atomicSizeSupported(TLI, CASI)) {
- expandAtomicCASToLibcall(CASI);
- MadeChange = true;
- continue;
- }
- }
-
- if (LI && TLI->shouldCastAtomicLoadInIR(LI) ==
- TargetLoweringBase::AtomicExpansionKind::CastToInteger) {
- I = LI = convertAtomicLoadToIntegerType(LI);
+ if (processAtomicInstr(I))
MadeChange = true;
- } else if (SI &&
- TLI->shouldCastAtomicStoreInIR(SI) ==
- TargetLoweringBase::AtomicExpansionKind::CastToInteger) {
- I = SI = convertAtomicStoreToIntegerType(SI);
- MadeChange = true;
- } else if (RMWI &&
- TLI->shouldCastAtomicRMWIInIR(RMWI) ==
- TargetLoweringBase::AtomicExpansionKind::CastToInteger) {
- I = RMWI = convertAtomicXchgToIntegerType(RMWI);
- MadeChange = true;
- } else if (CASI) {
- // TODO: when we're ready to make the change at the IR level, we can
- // extend convertCmpXchgToInteger for floating point too.
- if (CASI->getCompareOperand()->getType()->isPointerTy()) {
- // TODO: add a TLI hook to control this so that each target can
- // convert to lowering the original type one at a time.
- I = CASI = convertCmpXchgToIntegerType(CASI);
- MadeChange = true;
- }
- }
-
- if (TLI->shouldInsertFencesForAtomic(I)) {
- auto FenceOrdering = AtomicOrdering::Monotonic;
- if (LI && isAcquireOrStronger(LI->getOrdering())) {
- FenceOrdering = LI->getOrdering();
- LI->setOrdering(AtomicOrdering::Monotonic);
- } else if (SI && isReleaseOrStronger(SI->getOrdering())) {
- FenceOrdering = SI->getOrdering();
- SI->setOrdering(AtomicOrdering::Monotonic);
- } else if (RMWI && (isReleaseOrStronger(RMWI->getOrdering()) ||
- isAcquireOrStronger(RMWI->getOrdering()))) {
- FenceOrdering = RMWI->getOrdering();
- RMWI->setOrdering(AtomicOrdering::Monotonic);
- } else if (CASI &&
- TLI->shouldExpandAtomicCmpXchgInIR(CASI) ==
- TargetLoweringBase::AtomicExpansionKind::None &&
- (isReleaseOrStronger(CASI->getSuccessOrdering()) ||
- isAcquireOrStronger(CASI->getSuccessOrdering()) ||
- isAcquireOrStronger(CASI->getFailureOrdering()))) {
- // If a compare and swap is lowered to LL/SC, we can do smarter fence
- // insertion, with a stronger one on the success path than on the
- // failure path. As a result, fence insertion is directly done by
- // expandAtomicCmpXchg in that case.
- FenceOrdering = CASI->getMergedOrdering();
- CASI->setSuccessOrdering(AtomicOrdering::Monotonic);
- CASI->setFailureOrdering(AtomicOrdering::Monotonic);
- }
-
- if (FenceOrdering != AtomicOrdering::Monotonic) {
- MadeChange |= bracketInstWithFences(I, FenceOrdering);
- }
- } else if (I->hasAtomicStore() &&
- TLI->shouldInsertTrailingFenceForAtomicStore(I)) {
- auto FenceOrdering = AtomicOrdering::Monotonic;
- if (SI)
- FenceOrdering = SI->getOrdering();
- else if (RMWI)
- FenceOrdering = RMWI->getOrdering();
- else if (CASI && TLI->shouldExpandAtomicCmpXchgInIR(CASI) !=
- TargetLoweringBase::AtomicExpansionKind::LLSC)
- // LLSC is handled in expandAtomicCmpXchg().
- FenceOrdering = CASI->getSuccessOrdering();
-
- IRBuilder Builder(I);
- if (auto TrailingFence =
- TLI->emitTrailingFence(Builder, I, FenceOrdering)) {
- TrailingFence->moveAfter(I);
- MadeChange = true;
- }
- }
-
- if (LI)
- MadeChange |= tryExpandAtomicLoad(LI);
- else if (SI)
- MadeChange |= tryExpandAtomicStore(SI);
- else if (RMWI) {
- // There are two
diff erent ways of expanding RMW instructions:
- // - into a load if it is idempotent
- // - into a Cmpxchg/LL-SC loop otherwise
- // we try them in that order.
-
- if (isIdempotentRMW(RMWI) && simplifyIdempotentRMW(RMWI)) {
- MadeChange = true;
- } else {
- MadeChange |= tryExpandAtomicRMW(RMWI);
- }
- } else if (CASI)
- MadeChange |= tryExpandAtomicCmpXchg(CASI);
}
+
return MadeChange;
}
More information about the llvm-commits
mailing list