[llvm] [CodeGen] Port AtomicExpand to new Pass Manager (PR #71220)
via llvm-commits
llvm-commits at lists.llvm.org
Sat Jan 13 23:02:11 PST 2024
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-aarch64
@llvm/pr-subscribers-backend-risc-v
@llvm/pr-subscribers-backend-x86
@llvm/pr-subscribers-backend-loongarch
Author: Rishabh Bali (Ris-Bali)
<details>
<summary>Changes</summary>
Port the `atomicexpand` pass to the new Pass Manager.
Fixes #<!-- -->64559
---
Patch is 66.81 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/71220.diff
71 Files Affected:
- (added) llvm/include/llvm/CodeGen/ExpandAtomic.h (+30)
- (modified) llvm/include/llvm/CodeGen/MachinePassRegistry.def (+1-1)
- (modified) llvm/include/llvm/CodeGen/Passes.h (+3-3)
- (modified) llvm/include/llvm/CodeGen/TargetSubtargetInfo.h (+1-1)
- (modified) llvm/include/llvm/InitializePasses.h (+1-1)
- (modified) llvm/lib/CodeGen/CMakeLists.txt (+1-1)
- (modified) llvm/lib/CodeGen/CodeGen.cpp (+1-1)
- (renamed) llvm/lib/CodeGen/ExpandAtomicPass.cpp (+92-57)
- (modified) llvm/lib/CodeGen/TargetSubtargetInfo.cpp (+1-1)
- (modified) llvm/lib/Passes/PassBuilder.cpp (+2-1)
- (modified) llvm/lib/Passes/PassRegistry.def (+1)
- (modified) llvm/lib/Target/AArch64/AArch64TargetMachine.cpp (+1-1)
- (modified) llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp (+1-1)
- (modified) llvm/lib/Target/ARM/ARMTargetMachine.cpp (+1-1)
- (modified) llvm/lib/Target/CSKY/CSKYTargetMachine.cpp (+1-1)
- (modified) llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp (+1-1)
- (modified) llvm/lib/Target/LoongArch/LoongArchTargetMachine.cpp (+1-1)
- (modified) llvm/lib/Target/M68k/M68kTargetMachine.cpp (+1-1)
- (modified) llvm/lib/Target/Mips/MipsTargetMachine.cpp (+1-1)
- (modified) llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp (+1-1)
- (modified) llvm/lib/Target/PowerPC/PPCExpandAtomicPseudoInsts.cpp (+1-1)
- (modified) llvm/lib/Target/PowerPC/PPCTargetMachine.cpp (+1-1)
- (modified) llvm/lib/Target/RISCV/RISCVTargetMachine.cpp (+1-1)
- (modified) llvm/lib/Target/Sparc/SparcTargetMachine.cpp (+1-1)
- (modified) llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp (+1-1)
- (modified) llvm/lib/Target/VE/VETargetMachine.cpp (+1-1)
- (modified) llvm/lib/Target/WebAssembly/WebAssemblySubtarget.cpp (+1-1)
- (modified) llvm/lib/Target/WebAssembly/WebAssemblySubtarget.h (+1-1)
- (modified) llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp (+1-1)
- (modified) llvm/lib/Target/X86/X86TargetMachine.cpp (+1-1)
- (modified) llvm/lib/Target/XCore/XCoreTargetMachine.cpp (+1-1)
- (modified) llvm/test/CodeGen/AMDGPU/idemponent-atomics.ll (+1-1)
- (modified) llvm/test/CodeGen/AMDGPU/private-memory-atomics.ll (+1-1)
- (modified) llvm/test/Transforms/AtomicExpand/AArch64/atomicrmw-fp.ll (+1-1)
- (modified) llvm/test/Transforms/AtomicExpand/AArch64/expand-atomicrmw-xchg-fp.ll (+2-2)
- (modified) llvm/test/Transforms/AtomicExpand/AArch64/pcsections.ll (+1-1)
- (modified) llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i16-system.ll (+1-1)
- (modified) llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i16.ll (+2-2)
- (modified) llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i8-system.ll (+1-1)
- (modified) llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i8.ll (+2-2)
- (modified) llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fadd-flat-specialization.ll (+4-4)
- (modified) llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fadd.ll (+6-6)
- (modified) llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fmax.ll (+2-2)
- (modified) llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fmin.ll (+2-2)
- (modified) llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fsub.ll (+2-2)
- (modified) llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-nand.ll (+2-2)
- (modified) llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-simplify-cfg-CAS-block.ll (+1-1)
- (modified) llvm/test/Transforms/AtomicExpand/AMDGPU/unaligned-atomic.ll (+1-1)
- (modified) llvm/test/Transforms/AtomicExpand/ARM/atomic-expansion-v7.ll (+1-1)
- (modified) llvm/test/Transforms/AtomicExpand/ARM/atomic-expansion-v8.ll (+1-1)
- (modified) llvm/test/Transforms/AtomicExpand/ARM/atomicrmw-fp.ll (+1-1)
- (modified) llvm/test/Transforms/AtomicExpand/ARM/cmpxchg-weak.ll (+1-1)
- (modified) llvm/test/Transforms/AtomicExpand/Hexagon/atomicrmw-fp.ll (+1-1)
- (modified) llvm/test/Transforms/AtomicExpand/LoongArch/atomicrmw-fp.ll (+1-1)
- (modified) llvm/test/Transforms/AtomicExpand/LoongArch/load-store-atomic.ll (+2-2)
- (modified) llvm/test/Transforms/AtomicExpand/Mips/atomicrmw-fp.ll (+1-1)
- (modified) llvm/test/Transforms/AtomicExpand/PowerPC/atomicrmw-fp.ll (+1-1)
- (modified) llvm/test/Transforms/AtomicExpand/PowerPC/cfence-double.ll (+2-2)
- (modified) llvm/test/Transforms/AtomicExpand/PowerPC/cfence-float.ll (+2-2)
- (modified) llvm/test/Transforms/AtomicExpand/PowerPC/cmpxchg.ll (+2-2)
- (modified) llvm/test/Transforms/AtomicExpand/PowerPC/issue55983.ll (+2-2)
- (modified) llvm/test/Transforms/AtomicExpand/RISCV/atomicrmw-fp.ll (+1-1)
- (modified) llvm/test/Transforms/AtomicExpand/SPARC/libcalls.ll (+1-1)
- (modified) llvm/test/Transforms/AtomicExpand/SPARC/partword.ll (+1-1)
- (modified) llvm/test/Transforms/AtomicExpand/X86/expand-atomic-libcall.ll (+1-1)
- (modified) llvm/test/Transforms/AtomicExpand/X86/expand-atomic-non-integer.ll (+1-1)
- (modified) llvm/test/Transforms/AtomicExpand/X86/expand-atomic-rmw-fp.ll (+1-1)
- (modified) llvm/test/Transforms/AtomicExpand/X86/expand-atomic-rmw-initial-load.ll (+1-1)
- (modified) llvm/test/Transforms/AtomicExpand/X86/expand-atomic-xchg-fp.ll (+1-1)
- (modified) llvm/tools/opt/opt.cpp (-2)
- (modified) llvm/utils/gn/secondary/llvm/lib/CodeGen/BUILD.gn (+1-1)
``````````diff
diff --git a/llvm/include/llvm/CodeGen/ExpandAtomic.h b/llvm/include/llvm/CodeGen/ExpandAtomic.h
new file mode 100644
index 00000000000000..4ba49f8886ca94
--- /dev/null
+++ b/llvm/include/llvm/CodeGen/ExpandAtomic.h
@@ -0,0 +1,30 @@
+//===-- ExpandAtomic.h - Expand Atomic Instructions -------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_EXPANDATOMIC_H
+#define LLVM_CODEGEN_EXPANDATOMIC_H
+
+#include "llvm/IR/PassManager.h"
+
+namespace llvm {
+
+class Function;
+class TargetMachine;
+
+class ExpandAtomicPass : public PassInfoMixin<ExpandAtomicPass> {
+private:
+ const TargetMachine *TM;
+
+public:
+ ExpandAtomicPass(const TargetMachine *TM) : TM(TM) {}
+ PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+};
+
+} // end namespace llvm
+
+#endif // LLVM_CODEGEN_EXPANDATOMIC_H
diff --git a/llvm/include/llvm/CodeGen/MachinePassRegistry.def b/llvm/include/llvm/CodeGen/MachinePassRegistry.def
index a29269644ea1dc..a52e7d04912f31 100644
--- a/llvm/include/llvm/CodeGen/MachinePassRegistry.def
+++ b/llvm/include/llvm/CodeGen/MachinePassRegistry.def
@@ -116,7 +116,7 @@ DUMMY_FUNCTION_PASS("wasmehprepare", WasmEHPass, ())
DUMMY_FUNCTION_PASS("codegenprepare", CodeGenPreparePass, ())
DUMMY_FUNCTION_PASS("safe-stack", SafeStackPass, ())
DUMMY_FUNCTION_PASS("stack-protector", StackProtectorPass, ())
-DUMMY_FUNCTION_PASS("atomic-expand", AtomicExpandPass, ())
+DUMMY_FUNCTION_PASS("expand-atomic", ExpandAtomicPass, ())
DUMMY_FUNCTION_PASS("interleaved-access", InterleavedAccessPass, ())
DUMMY_FUNCTION_PASS("indirectbr-expand", IndirectBrExpandPass, ())
DUMMY_FUNCTION_PASS("cfguard-dispatch", CFGuardDispatchPass, ())
diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h
index 712048017bca1a..b9f9ce936dbb7f 100644
--- a/llvm/include/llvm/CodeGen/Passes.h
+++ b/llvm/include/llvm/CodeGen/Passes.h
@@ -41,10 +41,10 @@ class FileSystem;
// List of target independent CodeGen pass IDs.
namespace llvm {
- /// AtomicExpandPass - At IR level this pass replace atomic instructions with
+ /// ExpandAtomicPass - At IR level this pass replace atomic instructions with
/// __atomic_* library calls, or target specific instruction which implement the
/// same semantics in a way which better fits the target backend.
- FunctionPass *createAtomicExpandPass();
+ FunctionPass *createExpandAtomicLegacyPass();
/// createUnreachableBlockEliminationPass - The LLVM code generator does not
/// work well with unreachable basic blocks (what live ranges make sense for a
@@ -103,7 +103,7 @@ namespace llvm {
/// AtomicExpandID -- Lowers atomic operations in terms of either cmpxchg
/// load-linked/store-conditional loops.
- extern char &AtomicExpandID;
+ extern char &ExpandAtomicID;
/// MachineLoopInfo - This pass is a loop analysis pass.
extern char &MachineLoopInfoID;
diff --git a/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h b/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h
index 55ef95c2854319..da1fd6737b796f 100644
--- a/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h
@@ -215,7 +215,7 @@ class TargetSubtargetInfo : public MCSubtargetInfo {
virtual bool enablePostRAMachineScheduler() const;
/// True if the subtarget should run the atomic expansion pass.
- virtual bool enableAtomicExpand() const;
+ virtual bool enableExpandAtomic() const;
/// True if the subtarget should run the indirectbr expansion pass.
virtual bool enableIndirectBrExpand() const;
diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index fafae8b5ecd7a7..04365af3d1a74e 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -54,7 +54,7 @@ void initializeAlwaysInlinerLegacyPassPass(PassRegistry&);
void initializeAssignmentTrackingAnalysisPass(PassRegistry &);
void initializeAssumeBuilderPassLegacyPassPass(PassRegistry &);
void initializeAssumptionCacheTrackerPass(PassRegistry&);
-void initializeAtomicExpandPass(PassRegistry&);
+void initializeExpandAtomicLegacyPass(PassRegistry &);
void initializeBasicBlockPathCloningPass(PassRegistry &);
void initializeBasicBlockSectionsProfileReaderPass(PassRegistry &);
void initializeBasicBlockSectionsPass(PassRegistry &);
diff --git a/llvm/lib/CodeGen/CMakeLists.txt b/llvm/lib/CodeGen/CMakeLists.txt
index df2d1831ee5fdb..12bd12bc49caeb 100644
--- a/llvm/lib/CodeGen/CMakeLists.txt
+++ b/llvm/lib/CodeGen/CMakeLists.txt
@@ -40,7 +40,7 @@ add_llvm_component_library(LLVMCodeGen
AllocationOrder.cpp
Analysis.cpp
AssignmentTrackingAnalysis.cpp
- AtomicExpandPass.cpp
+ ExpandAtomicPass.cpp
BasicTargetTransformInfo.cpp
BranchFolding.cpp
BranchRelaxation.cpp
diff --git a/llvm/lib/CodeGen/CodeGen.cpp b/llvm/lib/CodeGen/CodeGen.cpp
index 79a95ee0d747a1..eacd8b36b610c8 100644
--- a/llvm/lib/CodeGen/CodeGen.cpp
+++ b/llvm/lib/CodeGen/CodeGen.cpp
@@ -19,7 +19,7 @@ using namespace llvm;
/// initializeCodeGen - Initialize all passes linked into the CodeGen library.
void llvm::initializeCodeGen(PassRegistry &Registry) {
initializeAssignmentTrackingAnalysisPass(Registry);
- initializeAtomicExpandPass(Registry);
+ initializeExpandAtomicLegacyPass(Registry);
initializeBasicBlockPathCloningPass(Registry);
initializeBasicBlockSectionsPass(Registry);
initializeBranchFolderPassPass(Registry);
diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/ExpandAtomicPass.cpp
similarity index 95%
rename from llvm/lib/CodeGen/AtomicExpandPass.cpp
rename to llvm/lib/CodeGen/ExpandAtomicPass.cpp
index ccf3e9ec649210..5f8e069bafc7a6 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/ExpandAtomicPass.cpp
@@ -1,4 +1,4 @@
-//===- AtomicExpandPass.cpp - Expand atomic instructions ------------------===//
+//===- ExpandAtomicPass.cpp - Expand atomic instructions ------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
@@ -19,6 +19,7 @@
#include "llvm/ADT/SmallVector.h"
#include "llvm/Analysis/InstSimplifyFolder.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
+#include "llvm/CodeGen/ExpandAtomic.h"
#include "llvm/CodeGen/AtomicExpandUtils.h"
#include "llvm/CodeGen/RuntimeLibcalls.h"
#include "llvm/CodeGen/TargetLowering.h"
@@ -55,23 +56,14 @@
using namespace llvm;
-#define DEBUG_TYPE "atomic-expand"
+#define DEBUG_TYPE "expand-atomic"
namespace {
-class AtomicExpand : public FunctionPass {
+class ExpandAtomicImpl {
const TargetLowering *TLI = nullptr;
const DataLayout *DL = nullptr;
-public:
- static char ID; // Pass identification, replacement for typeid
-
- AtomicExpand() : FunctionPass(ID) {
- initializeAtomicExpandPass(*PassRegistry::getPassRegistry());
- }
-
- bool runOnFunction(Function &F) override;
-
private:
bool bracketInstWithFences(Instruction *I, AtomicOrdering Order);
IntegerType *getCorrespondingIntegerType(Type *T, const DataLayout &DL);
@@ -124,6 +116,20 @@ class AtomicExpand : public FunctionPass {
friend bool
llvm::expandAtomicRMWToCmpXchg(AtomicRMWInst *AI,
CreateCmpXchgInstFun CreateCmpXchg);
+
+public:
+ bool run(Function &F, const TargetMachine *TM);
+};
+
+class ExpandAtomicLegacy : public FunctionPass {
+public:
+ static char ID; // Pass identification, replacement for typeid
+
+ ExpandAtomicLegacy() : FunctionPass(ID) {
+ initializeExpandAtomicLegacyPass(*PassRegistry::getPassRegistry());
+ }
+
+ bool runOnFunction(Function &F) override;
};
// IRBuilder to be used for replacement atomic instructions.
@@ -138,14 +144,15 @@ struct ReplacementIRBuilder : IRBuilder<InstSimplifyFolder> {
} // end anonymous namespace
-char AtomicExpand::ID = 0;
+char ExpandAtomicLegacy::ID = 0;
-char &llvm::AtomicExpandID = AtomicExpand::ID;
+char &llvm::ExpandAtomicID = ExpandAtomicLegacy::ID;
-INITIALIZE_PASS(AtomicExpand, DEBUG_TYPE, "Expand Atomic instructions", false,
- false)
-
-FunctionPass *llvm::createAtomicExpandPass() { return new AtomicExpand(); }
+INITIALIZE_PASS_BEGIN(ExpandAtomicLegacy, DEBUG_TYPE,
+ "Expand Atomic instructions", false, false)
+INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
+INITIALIZE_PASS_END(ExpandAtomicLegacy, DEBUG_TYPE,
+ "Expand Atomic instructions", false, false)
// Helper functions to retrieve the size of atomic instructions.
static unsigned getAtomicOpSize(LoadInst *LI) {
@@ -179,14 +186,9 @@ static bool atomicSizeSupported(const TargetLowering *TLI, Inst *I) {
Size <= TLI->getMaxAtomicSizeInBitsSupported() / 8;
}
-bool AtomicExpand::runOnFunction(Function &F) {
- auto *TPC = getAnalysisIfAvailable<TargetPassConfig>();
- if (!TPC)
- return false;
-
- auto &TM = TPC->getTM<TargetMachine>();
- const auto *Subtarget = TM.getSubtargetImpl(F);
- if (!Subtarget->enableAtomicExpand())
+bool ExpandAtomicImpl::run(Function &F, const TargetMachine *TM) {
+ const auto *Subtarget = TM->getSubtargetImpl(F);
+ if (!Subtarget->enableExpandAtomic())
return false;
TLI = Subtarget->getTargetLowering();
DL = &F.getParent()->getDataLayout();
@@ -340,7 +342,39 @@ bool AtomicExpand::runOnFunction(Function &F) {
return MadeChange;
}
-bool AtomicExpand::bracketInstWithFences(Instruction *I, AtomicOrdering Order) {
+bool ExpandAtomicLegacy::runOnFunction(Function &F) {
+ if (skipFunction(F))
+ return false;
+
+ auto *TPC = getAnalysisIfAvailable<TargetPassConfig>();
+ if (!TPC)
+ return false;
+
+ auto *TM = &TPC->getTM<TargetMachine>();
+
+ ExpandAtomicImpl AE;
+ return AE.run(F, TM);
+}
+
+FunctionPass *llvm::createExpandAtomicLegacyPass() {
+ return new ExpandAtomicLegacy();
+}
+
+PreservedAnalyses ExpandAtomicPass::run(Function &F,
+ FunctionAnalysisManager &AM) {
+ ExpandAtomicImpl AE;
+
+ bool Changed = AE.run(F, TM);
+ if (!Changed)
+ return PreservedAnalyses::all();
+
+ PreservedAnalyses PA;
+ PA.preserveSet<CFGAnalyses>();
+ return PA;
+}
+
+bool ExpandAtomicImpl::bracketInstWithFences(Instruction *I,
+ AtomicOrdering Order) {
ReplacementIRBuilder Builder(I, *DL);
auto LeadingFence = TLI->emitLeadingFence(Builder, I, Order);
@@ -355,8 +389,8 @@ bool AtomicExpand::bracketInstWithFences(Instruction *I, AtomicOrdering Order) {
}
/// Get the iX type with the same bitwidth as T.
-IntegerType *AtomicExpand::getCorrespondingIntegerType(Type *T,
- const DataLayout &DL) {
+IntegerType *
+ExpandAtomicImpl::getCorrespondingIntegerType(Type *T, const DataLayout &DL) {
EVT VT = TLI->getMemValueType(DL, T);
unsigned BitWidth = VT.getStoreSizeInBits();
assert(BitWidth == VT.getSizeInBits() && "must be a power of two");
@@ -366,7 +400,7 @@ IntegerType *AtomicExpand::getCorrespondingIntegerType(Type *T,
/// Convert an atomic load of a non-integral type to an integer load of the
/// equivalent bitwidth. See the function comment on
/// convertAtomicStoreToIntegerType for background.
-LoadInst *AtomicExpand::convertAtomicLoadToIntegerType(LoadInst *LI) {
+LoadInst *ExpandAtomicImpl::convertAtomicLoadToIntegerType(LoadInst *LI) {
auto *M = LI->getModule();
Type *NewTy = getCorrespondingIntegerType(LI->getType(), M->getDataLayout());
@@ -387,7 +421,7 @@ LoadInst *AtomicExpand::convertAtomicLoadToIntegerType(LoadInst *LI) {
}
AtomicRMWInst *
-AtomicExpand::convertAtomicXchgToIntegerType(AtomicRMWInst *RMWI) {
+ExpandAtomicImpl::convertAtomicXchgToIntegerType(AtomicRMWInst *RMWI) {
auto *M = RMWI->getModule();
Type *NewTy =
getCorrespondingIntegerType(RMWI->getType(), M->getDataLayout());
@@ -414,7 +448,7 @@ AtomicExpand::convertAtomicXchgToIntegerType(AtomicRMWInst *RMWI) {
return NewRMWI;
}
-bool AtomicExpand::tryExpandAtomicLoad(LoadInst *LI) {
+bool ExpandAtomicImpl::tryExpandAtomicLoad(LoadInst *LI) {
switch (TLI->shouldExpandAtomicLoadInIR(LI)) {
case TargetLoweringBase::AtomicExpansionKind::None:
return false;
@@ -436,7 +470,7 @@ bool AtomicExpand::tryExpandAtomicLoad(LoadInst *LI) {
}
}
-bool AtomicExpand::tryExpandAtomicStore(StoreInst *SI) {
+bool ExpandAtomicImpl::tryExpandAtomicStore(StoreInst *SI) {
switch (TLI->shouldExpandAtomicStoreInIR(SI)) {
case TargetLoweringBase::AtomicExpansionKind::None:
return false;
@@ -451,7 +485,7 @@ bool AtomicExpand::tryExpandAtomicStore(StoreInst *SI) {
}
}
-bool AtomicExpand::expandAtomicLoadToLL(LoadInst *LI) {
+bool ExpandAtomicImpl::expandAtomicLoadToLL(LoadInst *LI) {
ReplacementIRBuilder Builder(LI, *DL);
// On some architectures, load-linked instructions are atomic for larger
@@ -467,7 +501,7 @@ bool AtomicExpand::expandAtomicLoadToLL(LoadInst *LI) {
return true;
}
-bool AtomicExpand::expandAtomicLoadToCmpXchg(LoadInst *LI) {
+bool ExpandAtomicImpl::expandAtomicLoadToCmpXchg(LoadInst *LI) {
ReplacementIRBuilder Builder(LI, *DL);
AtomicOrdering Order = LI->getOrdering();
if (Order == AtomicOrdering::Unordered)
@@ -496,7 +530,7 @@ bool AtomicExpand::expandAtomicLoadToCmpXchg(LoadInst *LI) {
/// instruction select from the original atomic store, but as a migration
/// mechanism, we convert back to the old format which the backends understand.
/// Each backend will need individual work to recognize the new format.
-StoreInst *AtomicExpand::convertAtomicStoreToIntegerType(StoreInst *SI) {
+StoreInst *ExpandAtomicImpl::convertAtomicStoreToIntegerType(StoreInst *SI) {
ReplacementIRBuilder Builder(SI, *DL);
auto *M = SI->getModule();
Type *NewTy = getCorrespondingIntegerType(SI->getValueOperand()->getType(),
@@ -514,7 +548,7 @@ StoreInst *AtomicExpand::convertAtomicStoreToIntegerType(StoreInst *SI) {
return NewSI;
}
-void AtomicExpand::expandAtomicStore(StoreInst *SI) {
+void ExpandAtomicImpl::expandAtomicStore(StoreInst *SI) {
// This function is only called on atomic stores that are too large to be
// atomic if implemented as a native store. So we replace them by an
// atomic swap, that can be implemented for example as a ldrex/strex on ARM
@@ -561,7 +595,7 @@ static void createCmpXchgInstFun(IRBuilderBase &Builder, Value *Addr,
NewLoaded = Builder.CreateBitCast(NewLoaded, OrigTy);
}
-bool AtomicExpand::tryExpandAtomicRMW(AtomicRMWInst *AI) {
+bool ExpandAtomicImpl::tryExpandAtomicRMW(AtomicRMWInst *AI) {
LLVMContext &Ctx = AI->getModule()->getContext();
TargetLowering::AtomicExpansionKind Kind = TLI->shouldExpandAtomicRMWInIR(AI);
switch (Kind) {
@@ -843,7 +877,7 @@ static Value *performMaskedAtomicOp(AtomicRMWInst::BinOp Op,
/// way as a typical atomicrmw expansion. The only difference here is
/// that the operation inside of the loop may operate upon only a
/// part of the value.
-void AtomicExpand::expandPartwordAtomicRMW(
+void ExpandAtomicImpl::expandPartwordAtomicRMW(
AtomicRMWInst *AI, TargetLoweringBase::AtomicExpansionKind ExpansionKind) {
AtomicOrdering MemOpOrder = AI->getOrdering();
SyncScope::ID SSID = AI->getSyncScopeID();
@@ -887,7 +921,7 @@ void AtomicExpand::expandPartwordAtomicRMW(
}
// Widen the bitwise atomicrmw (or/xor/and) to the minimum supported width.
-AtomicRMWInst *AtomicExpand::widenPartwordAtomicRMW(AtomicRMWInst *AI) {
+AtomicRMWInst *ExpandAtomicImpl::widenPartwordAtomicRMW(AtomicRMWInst *AI) {
ReplacementIRBuilder Builder(AI, *DL);
AtomicRMWInst::BinOp Op = AI->getOperation();
@@ -922,7 +956,7 @@ AtomicRMWInst *AtomicExpand::widenPartwordAtomicRMW(AtomicRMWInst *AI) {
return NewAI;
}
-bool AtomicExpand::expandPartwordCmpXchg(AtomicCmpXchgInst *CI) {
+bool ExpandAtomicImpl::expandPartwordCmpXchg(AtomicCmpXchgInst *CI) {
// The basic idea here is that we're expanding a cmpxchg of a
// smaller memory size up to a word-sized cmpxchg. To do this, we
// need to add a retry-loop for strong cmpxchg, so that
@@ -1047,7 +1081,7 @@ bool AtomicExpand::expandPartwordCmpXchg(AtomicCmpXchgInst *CI) {
return true;
}
-void AtomicExpand::expandAtomicOpToLLSC(
+void ExpandAtomicImpl::expandAtomicOpToLLSC(
Instruction *I, Type *ResultType, Value *Addr, Align AddrAlign,
AtomicOrdering MemOpOrder,
function_ref<Value *(IRBuilderBase &, Value *)> PerformOp) {
@@ -1059,7 +1093,7 @@ void AtomicExpand::expandAtomicOpToLLSC(
I->eraseFromParent();
}
-void AtomicExpand::expandAtomicRMWToMaskedIntrinsic(AtomicRMWInst *AI) {
+void ExpandAtomicImpl::expandAtomicRMWToMaskedIntrinsic(AtomicRMWInst *AI) {
ReplacementIRBuilder Builder(AI, *DL);
PartwordMaskValues PMV =
@@ -1085,7 +1119,8 @@ void AtomicExpand::expandAtomicRMWToMaskedIntrinsic(AtomicRMWInst *AI) {
AI->eraseFromParent();
}
-void AtomicExpand::expandAtomicCmpXchgToMaskedIntrinsic(AtomicCmpXchgInst *CI) {
+void ExpandAtomicImpl::expandAtomicCmpXchgToMaskedIntrinsic(
+ AtomicCmpXchgInst *CI) {
ReplacementIRBuilder Builder(CI, *DL);
PartwordMaskValues PMV = createMaskInstrs(
@@ -1112,7 +1147,7 @@ void AtomicExpand::expandAtomicCmpXchgToMaskedIntrinsic(AtomicCmpXchgInst *CI) {
CI->eraseFromParent();
}
-Value *AtomicExpand::insertRMWLLSCLoop(
+Value *ExpandAtomicImpl::insertRMWLLSCLoop(
IRBuilderBase &Builder, Type *ResultTy, Value *Addr, Align AddrAlign,
AtomicOrdering MemOpOrder,
function_ref<Value *(IRBuilderBase &, Value *)> PerformOp) {
@@ -1168,7 +1203,7 @@ Value *AtomicExpand::insertRMWLLSCLoop(
/// way to represent a pointer cmpxchg so that we can update backends one by
/// one.
AtomicCmpXchgInst *
-AtomicExpand::convertCmpXchgToIntegerType(AtomicCmpXchgInst *CI) {
+ExpandAtomicImpl::convertCmpXchgToIntegerType(AtomicCmpXchgInst *CI) {
auto *M = CI->getModule();
Type *NewTy = getCorrespondingIntegerType(CI->getCompareOperand()->getType(),
M->getDataLayout());
@@ -1201,7 +1236,7 @@ AtomicExpand::convertCmpXchgToIntegerType(AtomicCmpXchgInst *CI) {
return NewCI;
}
-bool AtomicExpand::expandAtomicCmpXchg(AtomicCmpXchgInst *CI) {
+bool ExpandAtomicImpl::expandAtomicCmpXchg(AtomicCmpXchgInst *CI) {
AtomicOrdering SuccessOrder = CI->getSuccessOrdering();
AtomicOrdering FailureOrder = CI->getFailureOrdering();
Value *Addr = CI->getPointerOperand();
@@ -1447,7 +1482,7 @@ bool AtomicExpand::expandAtomicCmpXchg(AtomicCmpXchgInst *CI) {
return true;
}
-bool AtomicExpand::isIdempotentRMW(AtomicRMWInst *RMWI) {
+bool ExpandAtomicImpl::isIdempotentRMW(AtomicRMWInst *RMWI) {
auto C = dyn_cast<ConstantInt>(RMWI->getValOperand());
if (!C)
return false;
@@ -1467,7 +1502,7 @@ bool AtomicExpand::isIdempotentRMW(AtomicRMWInst *RMWI) {
}
}
-bool AtomicExpand::simplifyIdempotentRMW(AtomicRMWInst *RMWI) {
+bool ExpandAtomicImpl::simplifyIdempotentRMW(AtomicRMWInst *RMWI) {
if (auto ResultingLoad = TLI->lowerIdempotentRMWIntoFencedLoad(RMWI)) {
tryExpandAtomicLoad(ResultingLoad);
return true;
@@ -1475,7 +1510,7 @@ bool AtomicExpand::simplifyIdempotentRMW(AtomicRMWInst *RMWI) {
return false;
}
-Value *AtomicExpand::insertRMWCmpXchgLoop(
+Value *ExpandAtomicImpl::insertRMWCmpXchgLoop(
IRBuilderBase &Builder, Type *ResultTy, Value *Addr, Align AddrAlign,
AtomicOrdering MemOpOrder, SyncScope::ID SSID,
function_ref<Value *(IRBuilderBase &, Value *)> PerformOp,
@@ -1536,7 +1571,7 @@ Value *AtomicExpand::insertRMWCmpXchgLoop(
return NewLoaded;
}
-bool AtomicExpand::tryExpandAtomicCmpXchg(AtomicCmpXchgInst *CI) {
+bool ExpandAtomicImpl::tryExpandAtomicCmpXchg(AtomicCmpXchgInst *CI) {
unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;
unsigned ValueSize = getAtomicOpSize(CI);
@@ -1567,7 +1602,7 @@ bool llvm::expandAtomicRMWToCmpXchg(AtomicRMWInst *AI,
// FIXME: If FP exceptions are observable, we should force them off for the
// loop for the FP atomics.
- Value *Loaded = AtomicExpand::insertRMWCmpXchgLoop(
+ Value *Loaded = ExpandAtomicImpl::insertRMWCmpXchgLoop(
Builder, AI->getType(), AI->...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/71220
More information about the llvm-commits
mailing list