[llvm] r217080 - Refactor AtomicExpandPass and add a generic isAtomic() method to Instruction
Robin Morisset
morisset at google.com
Wed Sep 3 14:30:00 PDT 2014
Author: morisset
Date: Wed Sep 3 16:29:59 2014
New Revision: 217080
URL: http://llvm.org/viewvc/llvm-project?rev=217080&view=rev
Log:
Refactor AtomicExpandPass and add a generic isAtomic() method to Instruction
Summary:
Split shouldExpandAtomicInIR() into different versions for Stores/Loads/RMWs/CmpXchgs.
Makes runOnFunction cleaner (no more redundant checking/casting), and will help moving
the X86 backend to this pass.
This requires a way of easily detecting which instructions are atomic.
I followed the pattern of mayReadFromMemory, mayWriteOrReadMemory, etc.. in making
isAtomic() a method of Instruction implemented by a switch on the opcodes.
Test Plan: make check
Reviewers: jfb
Subscribers: mcrosier, llvm-commits
Differential Revision: http://reviews.llvm.org/D5035
Modified:
llvm/trunk/include/llvm/IR/Instruction.h
llvm/trunk/include/llvm/IR/Instructions.h
llvm/trunk/include/llvm/Target/TargetLowering.h
llvm/trunk/lib/CodeGen/AtomicExpandPass.cpp
llvm/trunk/lib/IR/Instruction.cpp
llvm/trunk/lib/Target/AArch64/AArch64ISelLowering.cpp
llvm/trunk/lib/Target/AArch64/AArch64ISelLowering.h
llvm/trunk/lib/Target/ARM/ARMISelLowering.cpp
llvm/trunk/lib/Target/ARM/ARMISelLowering.h
Modified: llvm/trunk/include/llvm/IR/Instruction.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/IR/Instruction.h?rev=217080&r1=217079&r2=217080&view=diff
==============================================================================
--- llvm/trunk/include/llvm/IR/Instruction.h (original)
+++ llvm/trunk/include/llvm/IR/Instruction.h Wed Sep 3 16:29:59 2014
@@ -338,6 +338,11 @@ public:
return mayReadFromMemory() || mayWriteToMemory();
}
+ /// isAtomic - Return true if this instruction has an
+ /// AtomicOrdering of unordered or higher.
+ ///
+ bool isAtomic() const;
+
/// mayThrow - Return true if this instruction may throw an exception.
///
bool mayThrow() const;
Modified: llvm/trunk/include/llvm/IR/Instructions.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/IR/Instructions.h?rev=217080&r1=217079&r2=217080&view=diff
==============================================================================
--- llvm/trunk/include/llvm/IR/Instructions.h (original)
+++ llvm/trunk/include/llvm/IR/Instructions.h Wed Sep 3 16:29:59 2014
@@ -241,7 +241,6 @@ public:
(xthread << 6));
}
- bool isAtomic() const { return getOrdering() != NotAtomic; }
void setAtomic(AtomicOrdering Ordering,
SynchronizationScope SynchScope = CrossThread) {
setOrdering(Ordering);
@@ -361,7 +360,6 @@ public:
(xthread << 6));
}
- bool isAtomic() const { return getOrdering() != NotAtomic; }
void setAtomic(AtomicOrdering Ordering,
SynchronizationScope SynchScope = CrossThread) {
setOrdering(Ordering);
Modified: llvm/trunk/include/llvm/Target/TargetLowering.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/Target/TargetLowering.h?rev=217080&r1=217079&r2=217080&view=diff
==============================================================================
--- llvm/trunk/include/llvm/Target/TargetLowering.h (original)
+++ llvm/trunk/include/llvm/Target/TargetLowering.h Wed Sep 3 16:29:59 2014
@@ -31,6 +31,7 @@
#include "llvm/IR/CallSite.h"
#include "llvm/IR/CallingConv.h"
#include "llvm/IR/InlineAsm.h"
+#include "llvm/IR/Instructions.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/MC/MCRegisterInfo.h"
#include "llvm/Target/TargetCallingConv.h"
@@ -955,7 +956,7 @@ public:
/// It is called by AtomicExpandPass before expanding an
/// AtomicRMW/AtomicCmpXchg/AtomicStore/AtomicLoad.
/// RMW and CmpXchg set both IsStore and IsLoad to true.
- /// Backends with !getInsertFencesForAtomic() should keep a no-op here
+ /// Backends with !getInsertFencesForAtomic() should keep a no-op here.
virtual void emitLeadingFence(IRBuilder<> &Builder, AtomicOrdering Ord,
bool IsStore, bool IsLoad) const {
assert(!getInsertFencesForAtomic());
@@ -965,20 +966,28 @@ public:
/// It is called by AtomicExpandPass after expanding an
/// AtomicRMW/AtomicCmpXchg/AtomicStore/AtomicLoad.
/// RMW and CmpXchg set both IsStore and IsLoad to true.
- /// Backends with !getInsertFencesForAtomic() should keep a no-op here
+ /// Backends with !getInsertFencesForAtomic() should keep a no-op here.
virtual void emitTrailingFence(IRBuilder<> &Builder, AtomicOrdering Ord,
bool IsStore, bool IsLoad) const {
assert(!getInsertFencesForAtomic());
}
- /// Return true if the given (atomic) instruction should be expanded by the
- /// IR-level AtomicExpand pass into a loop involving
- /// load-linked/store-conditional pairs. Atomic stores will be expanded in the
- /// same way as "atomic xchg" operations which ignore their output if needed.
- virtual bool shouldExpandAtomicInIR(Instruction *Inst) const {
+ /// Returns true if the given (atomic) store should be expanded by the
+ /// IR-level AtomicExpand pass into an "atomic xchg" which ignores its input.
+ virtual bool shouldExpandAtomicStoreInIR(StoreInst *SI) const {
return false;
}
+ /// Returns true if the given (atomic) load should be expanded by the
+ /// IR-level AtomicExpand pass into a load-linked instruction
+ /// (through emitLoadLinked()).
+ virtual bool shouldExpandAtomicLoadInIR(LoadInst *LI) const { return false; }
+
+ /// Returns true if the given AtomicRMW should be expanded by the
+ /// IR-level AtomicExpand pass into a loop using LoadLinked/StoreConditional.
+ virtual bool shouldExpandAtomicRMWInIR(AtomicRMWInst *RMWI) const {
+ return false;
+ }
//===--------------------------------------------------------------------===//
// TargetLowering Configuration Methods - These methods should be invoked by
Modified: llvm/trunk/lib/CodeGen/AtomicExpandPass.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/AtomicExpandPass.cpp?rev=217080&r1=217079&r2=217080&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/AtomicExpandPass.cpp (original)
+++ llvm/trunk/lib/CodeGen/AtomicExpandPass.cpp Wed Sep 3 16:29:59 2014
@@ -15,6 +15,7 @@
#include "llvm/CodeGen/Passes.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/InstIterator.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/Module.h"
@@ -38,10 +39,10 @@ namespace {
}
bool runOnFunction(Function &F) override;
- bool expandAtomicInsts(Function &F);
+ private:
bool expandAtomicLoad(LoadInst *LI);
- bool expandAtomicStore(StoreInst *LI);
+ bool expandAtomicStore(StoreInst *SI);
bool expandAtomicRMW(AtomicRMWInst *AI);
bool expandAtomicCmpXchg(AtomicCmpXchgInst *CI);
};
@@ -60,37 +61,37 @@ FunctionPass *llvm::createAtomicExpandPa
bool AtomicExpand::runOnFunction(Function &F) {
if (!TM || !TM->getSubtargetImpl()->enableAtomicExpand())
return false;
+ auto TargetLowering = TM->getSubtargetImpl()->getTargetLowering();
SmallVector<Instruction *, 1> AtomicInsts;
// Changing control-flow while iterating through it is a bad idea, so gather a
// list of all atomic instructions before we start.
- for (BasicBlock &BB : F)
- for (Instruction &Inst : BB) {
- if (isa<AtomicRMWInst>(&Inst) || isa<AtomicCmpXchgInst>(&Inst) ||
- (isa<LoadInst>(&Inst) && cast<LoadInst>(&Inst)->isAtomic()) ||
- (isa<StoreInst>(&Inst) && cast<StoreInst>(&Inst)->isAtomic()))
- AtomicInsts.push_back(&Inst);
- }
+ for (inst_iterator I = inst_begin(F), E = inst_end(F); I != E; ++I) {
+ if (I->isAtomic())
+ AtomicInsts.push_back(&*I);
+ }
bool MadeChange = false;
- for (Instruction *Inst : AtomicInsts) {
- if (!TM->getSubtargetImpl()->getTargetLowering()->shouldExpandAtomicInIR(
- Inst))
- continue;
-
- if (AtomicRMWInst *AI = dyn_cast<AtomicRMWInst>(Inst))
- MadeChange |= expandAtomicRMW(AI);
- else if (AtomicCmpXchgInst *CI = dyn_cast<AtomicCmpXchgInst>(Inst))
- MadeChange |= expandAtomicCmpXchg(CI);
- else if (LoadInst *LI = dyn_cast<LoadInst>(Inst))
+ for (auto I : AtomicInsts) {
+ auto LI = dyn_cast<LoadInst>(I);
+ auto SI = dyn_cast<StoreInst>(I);
+ auto RMWI = dyn_cast<AtomicRMWInst>(I);
+ auto CASI = dyn_cast<AtomicCmpXchgInst>(I);
+
+ assert((LI || SI || RMWI || CASI || isa<FenceInst>(I)) &&
+ "Unknown atomic instruction");
+
+ if (LI && TargetLowering->shouldExpandAtomicLoadInIR(LI)) {
MadeChange |= expandAtomicLoad(LI);
- else if (StoreInst *SI = dyn_cast<StoreInst>(Inst))
+ } else if (SI && TargetLowering->shouldExpandAtomicStoreInIR(SI)) {
MadeChange |= expandAtomicStore(SI);
- else
- llvm_unreachable("Unknown atomic instruction");
+ } else if (RMWI && TargetLowering->shouldExpandAtomicRMWInIR(RMWI)) {
+ MadeChange |= expandAtomicRMW(RMWI);
+ } else if (CASI) {
+ MadeChange |= expandAtomicCmpXchg(CASI);
+ }
}
-
return MadeChange;
}
@@ -146,10 +147,10 @@ bool AtomicExpand::expandAtomicRMW(Atomi
BasicBlock *BB = AI->getParent();
Function *F = BB->getParent();
LLVMContext &Ctx = F->getContext();
- // If getInsertFencesForAtomic() return true, then the target does not want to
- // deal with memory orders, and emitLeading/TrailingFence should take care of
- // everything. Otherwise, emitLeading/TrailingFence are no-op and we should
- // preserve the ordering.
+ // If getInsertFencesForAtomic() returns true, then the target does not want
+ // to deal with memory orders, and emitLeading/TrailingFence should take care
+ // of everything. Otherwise, emitLeading/TrailingFence are no-op and we
+ // should preserve the ordering.
AtomicOrdering MemOpOrder =
TLI->getInsertFencesForAtomic() ? Monotonic : Order;
@@ -252,10 +253,10 @@ bool AtomicExpand::expandAtomicCmpXchg(A
BasicBlock *BB = CI->getParent();
Function *F = BB->getParent();
LLVMContext &Ctx = F->getContext();
- // If getInsertFencesForAtomic() return true, then the target does not want to
- // deal with memory orders, and emitLeading/TrailingFence should take care of
- // everything. Otherwise, emitLeading/TrailingFence are no-op and we should
- // preserve the ordering.
+ // If getInsertFencesForAtomic() returns true, then the target does not want
+ // to deal with memory orders, and emitLeading/TrailingFence should take care
+ // of everything. Otherwise, emitLeading/TrailingFence are no-op and we
+ // should preserve the ordering.
AtomicOrdering MemOpOrder =
TLI->getInsertFencesForAtomic() ? Monotonic : SuccessOrder;
Modified: llvm/trunk/lib/IR/Instruction.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/IR/Instruction.cpp?rev=217080&r1=217079&r2=217080&view=diff
==============================================================================
--- llvm/trunk/lib/IR/Instruction.cpp (original)
+++ llvm/trunk/lib/IR/Instruction.cpp Wed Sep 3 16:29:59 2014
@@ -443,6 +443,21 @@ bool Instruction::mayWriteToMemory() con
}
}
+bool Instruction::isAtomic() const {
+ switch (getOpcode()) {
+ default:
+ return false;
+ case Instruction::AtomicCmpXchg:
+ case Instruction::AtomicRMW:
+ case Instruction::Fence:
+ return true;
+ case Instruction::Load:
+ return cast<LoadInst>(this)->getOrdering() != NotAtomic;
+ case Instruction::Store:
+ return cast<StoreInst>(this)->getOrdering() != NotAtomic;
+ }
+}
+
bool Instruction::mayThrow() const {
if (const CallInst *CI = dyn_cast<CallInst>(this))
return !CI->doesNotThrow();
Modified: llvm/trunk/lib/Target/AArch64/AArch64ISelLowering.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/AArch64/AArch64ISelLowering.cpp?rev=217080&r1=217079&r2=217080&view=diff
==============================================================================
--- llvm/trunk/lib/Target/AArch64/AArch64ISelLowering.cpp (original)
+++ llvm/trunk/lib/Target/AArch64/AArch64ISelLowering.cpp Wed Sep 3 16:29:59 2014
@@ -8513,19 +8513,6 @@ void AArch64TargetLowering::ReplaceNodeR
}
}
-bool AArch64TargetLowering::shouldExpandAtomicInIR(Instruction *Inst) const {
- // Loads and stores less than 128-bits are already atomic; ones above that
- // are doomed anyway, so defer to the default libcall and blame the OS when
- // things go wrong:
- if (StoreInst *SI = dyn_cast<StoreInst>(Inst))
- return SI->getValueOperand()->getType()->getPrimitiveSizeInBits() == 128;
- else if (LoadInst *LI = dyn_cast<LoadInst>(Inst))
- return LI->getType()->getPrimitiveSizeInBits() == 128;
-
- // For the real atomic operations, we have ldxr/stxr up to 128 bits.
- return Inst->getType()->getPrimitiveSizeInBits() <= 128;
-}
-
bool AArch64TargetLowering::useLoadStackGuardNode() const {
return true;
}
@@ -8542,6 +8529,28 @@ AArch64TargetLowering::getPreferredVecto
return TargetLoweringBase::getPreferredVectorAction(VT);
}
+// Loads and stores less than 128-bits are already atomic; ones above that
+// are doomed anyway, so defer to the default libcall and blame the OS when
+// things go wrong.
+bool AArch64TargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const {
+ unsigned Size = SI->getValueOperand()->getType()->getPrimitiveSizeInBits();
+ return Size == 128;
+}
+
+// Loads and stores less than 128-bits are already atomic; ones above that
+// are doomed anyway, so defer to the default libcall and blame the OS when
+// things go wrong.
+bool AArch64TargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
+ unsigned Size = LI->getType()->getPrimitiveSizeInBits();
+ return Size == 128;
+}
+
+// For the real atomic operations, we have ldxr/stxr up to 128 bits,
+bool AArch64TargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
+ unsigned Size = AI->getType()->getPrimitiveSizeInBits();
+ return Size <= 128;
+}
+
Value *AArch64TargetLowering::emitLoadLinked(IRBuilder<> &Builder, Value *Addr,
AtomicOrdering Ord) const {
Module *M = Builder.GetInsertBlock()->getParent()->getParent();
Modified: llvm/trunk/lib/Target/AArch64/AArch64ISelLowering.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/AArch64/AArch64ISelLowering.h?rev=217080&r1=217079&r2=217080&view=diff
==============================================================================
--- llvm/trunk/lib/Target/AArch64/AArch64ISelLowering.h (original)
+++ llvm/trunk/lib/Target/AArch64/AArch64ISelLowering.h Wed Sep 3 16:29:59 2014
@@ -322,7 +322,9 @@ public:
Value *emitStoreConditional(IRBuilder<> &Builder, Value *Val,
Value *Addr, AtomicOrdering Ord) const override;
- bool shouldExpandAtomicInIR(Instruction *Inst) const override;
+ bool shouldExpandAtomicLoadInIR(LoadInst *LI) const override;
+ bool shouldExpandAtomicStoreInIR(StoreInst *SI) const override;
+ bool shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override;
bool useLoadStackGuardNode() const override;
TargetLoweringBase::LegalizeTypeAction
Modified: llvm/trunk/lib/Target/ARM/ARMISelLowering.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/ARM/ARMISelLowering.cpp?rev=217080&r1=217079&r2=217080&view=diff
==============================================================================
--- llvm/trunk/lib/Target/ARM/ARMISelLowering.cpp (original)
+++ llvm/trunk/lib/Target/ARM/ARMISelLowering.cpp Wed Sep 3 16:29:59 2014
@@ -11039,23 +11039,29 @@ void ARMTargetLowering::emitTrailingFenc
}
}
-bool ARMTargetLowering::shouldExpandAtomicInIR(Instruction *Inst) const {
- // Loads and stores less than 64-bits are already atomic; ones above that
- // are doomed anyway, so defer to the default libcall and blame the OS when
- // things go wrong. Cortex M doesn't have ldrexd/strexd though, so don't emit
- // anything for those.
- bool IsMClass = Subtarget->isMClass();
- if (StoreInst *SI = dyn_cast<StoreInst>(Inst)) {
- unsigned Size = SI->getValueOperand()->getType()->getPrimitiveSizeInBits();
- return Size == 64 && !IsMClass;
- } else if (LoadInst *LI = dyn_cast<LoadInst>(Inst)) {
- return LI->getType()->getPrimitiveSizeInBits() == 64 && !IsMClass;
- }
+// Loads and stores less than 64-bits are already atomic; ones above that
+// are doomed anyway, so defer to the default libcall and blame the OS when
+// things go wrong. Cortex M doesn't have ldrexd/strexd though, so don't emit
+// anything for those.
+bool ARMTargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const {
+ unsigned Size = SI->getValueOperand()->getType()->getPrimitiveSizeInBits();
+ return (Size == 64) && !Subtarget->isMClass();
+}
+
+// Loads and stores less than 64-bits are already atomic; ones above that
+// are doomed anyway, so defer to the default libcall and blame the OS when
+// things go wrong. Cortex M doesn't have ldrexd/strexd though, so don't emit
+// anything for those.
+bool ARMTargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
+ unsigned Size = LI->getType()->getPrimitiveSizeInBits();
+ return (Size == 64) && !Subtarget->isMClass();
+}
- // For the real atomic operations, we have ldrex/strex up to 32 bits,
- // and up to 64 bits on the non-M profiles
- unsigned AtomicLimit = IsMClass ? 32 : 64;
- return Inst->getType()->getPrimitiveSizeInBits() <= AtomicLimit;
+// For the real atomic operations, we have ldrex/strex up to 32 bits,
+// and up to 64 bits on the non-M profiles
+bool ARMTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
+ unsigned Size = AI->getType()->getPrimitiveSizeInBits();
+ return Size <= (Subtarget->isMClass() ? 32 : 64);
}
// This has so far only been implemented for MachO.
Modified: llvm/trunk/lib/Target/ARM/ARMISelLowering.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/ARM/ARMISelLowering.h?rev=217080&r1=217079&r2=217080&view=diff
==============================================================================
--- llvm/trunk/lib/Target/ARM/ARMISelLowering.h (original)
+++ llvm/trunk/lib/Target/ARM/ARMISelLowering.h Wed Sep 3 16:29:59 2014
@@ -402,7 +402,9 @@ namespace llvm {
void emitTrailingFence(IRBuilder<> &Builder, AtomicOrdering Ord,
bool IsStore, bool IsLoad) const override;
- bool shouldExpandAtomicInIR(Instruction *Inst) const override;
+ bool shouldExpandAtomicLoadInIR(LoadInst *LI) const override;
+ bool shouldExpandAtomicStoreInIR(StoreInst *SI) const override;
+ bool shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override;
bool useLoadStackGuardNode() const override;
More information about the llvm-commits
mailing list