[llvm] c9a1fc8 - [LoongArch] Add memory barrier optimization pass (#218597)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 10 20:50:36 PDT 2026
Author: hev
Date: 2026-09-11T11:50:26+08:00
New Revision: c9a1fc8d1deee527a3ff43592ed4e0e5b81d7fca
URL: https://github.com/llvm/llvm-project/commit/c9a1fc8d1deee527a3ff43592ed4e0e5b81d7fca
DIFF: https://github.com/llvm/llvm-project/commit/c9a1fc8d1deee527a3ff43592ed4e0e5b81d7fca.diff
LOG: [LoongArch] Add memory barrier optimization pass (#218597)
Add a machine-level pass to remove redundant DBARs and merge them with
atomic memory operations. The pass uses machine dominator and
post-dominator trees to ensure the transformation is safe across basic
blocks.
Added:
llvm/lib/Target/LoongArch/LoongArchMemoryBarrierOpt.cpp
Modified:
llvm/lib/Target/LoongArch/CMakeLists.txt
llvm/lib/Target/LoongArch/LoongArch.h
llvm/lib/Target/LoongArch/LoongArchTargetMachine.cpp
llvm/test/CodeGen/LoongArch/memory-barrier-opt.ll
llvm/test/CodeGen/LoongArch/opt-pipeline.ll
llvm/utils/gn/secondary/llvm/lib/Target/LoongArch/BUILD.gn
Removed:
################################################################################
diff --git a/llvm/lib/Target/LoongArch/CMakeLists.txt b/llvm/lib/Target/LoongArch/CMakeLists.txt
index 084544ff4112d..26da74c926df8 100644
--- a/llvm/lib/Target/LoongArch/CMakeLists.txt
+++ b/llvm/lib/Target/LoongArch/CMakeLists.txt
@@ -26,6 +26,7 @@ add_llvm_target(LoongArchCodeGen
LoongArchISelLowering.cpp
LoongArchLateBranchOpt.cpp
LoongArchMCInstLower.cpp
+ LoongArchMemoryBarrierOpt.cpp
LoongArchMergeBaseOffset.cpp
LoongArchOptWInstrs.cpp
LoongArchRegisterInfo.cpp
diff --git a/llvm/lib/Target/LoongArch/LoongArch.h b/llvm/lib/Target/LoongArch/LoongArch.h
index 6762f91df72d9..1926420440a0c 100644
--- a/llvm/lib/Target/LoongArch/LoongArch.h
+++ b/llvm/lib/Target/LoongArch/LoongArch.h
@@ -37,6 +37,7 @@ FunctionPass *createLoongArchDeadRegisterDefinitionsPass();
FunctionPass *createLoongArchExpandAtomicPseudoPass();
FunctionPass *createLoongArchISelDag(LoongArchTargetMachine &TM,
CodeGenOptLevel OptLevel);
+FunctionPass *createLoongArchMemoryBarrierOptPass();
FunctionPass *createLoongArchMergeBaseOffsetOptPass();
FunctionPass *createLoongArchOptWInstrsPass();
FunctionPass *createLoongArchPreRAExpandPseudoPass();
@@ -46,6 +47,7 @@ void initializeLoongArchAsmPrinterPass(PassRegistry &);
void initializeLoongArchDAGToDAGISelLegacyPass(PassRegistry &);
void initializeLoongArchDeadRegisterDefinitionsPass(PassRegistry &);
void initializeLoongArchExpandAtomicPseudoPass(PassRegistry &);
+void initializeLoongArchMemoryBarrierOptPass(PassRegistry &);
void initializeLoongArchMergeBaseOffsetOptPass(PassRegistry &);
void initializeLoongArchOptWInstrsPass(PassRegistry &);
void initializeLoongArchPreRAExpandPseudoPass(PassRegistry &);
diff --git a/llvm/lib/Target/LoongArch/LoongArchMemoryBarrierOpt.cpp b/llvm/lib/Target/LoongArch/LoongArchMemoryBarrierOpt.cpp
new file mode 100644
index 0000000000000..d05f223da1a82
--- /dev/null
+++ b/llvm/lib/Target/LoongArch/LoongArchMemoryBarrierOpt.cpp
@@ -0,0 +1,559 @@
+//===---- LoongArchMemoryBarrierOpt.cpp - Memory barrier Optimization -----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// This pass removes or merges redundant memory barrier instructions.
+///
+/// - DBAR x + DBAR y -> DBAR (x & y)
+/// - DBAR x + AMO_DB -> AMO_DB
+/// - DBAR x + AMO -> AMO_DB
+/// - DBAR x + LL -> LL
+/// - AMO_DB + DBAR x -> AMO_DB
+/// - AMO + DBAR x -> AMO_DB
+/// - SC + DBAR x -> SC
+///
+//===----------------------------------------------------------------------===//
+
+#include "LoongArch.h"
+#include "LoongArchInstrInfo.h"
+#include "LoongArchSubtarget.h"
+#include "llvm/CodeGen/MachineDominators.h"
+#include "llvm/CodeGen/MachinePostDominators.h"
+#include "llvm/InitializePasses.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "loongarch-memory-barrier-opt"
+#define LOONGARCH_MEMORY_BARRIER_OPT_NAME \
+ "LoongArch Memory Barrier Optimisation pass"
+
+static cl::opt<bool> RequireNoPathBypass(
+ "loongarch-require-no-path-bypass",
+ cl::desc("Optimize only when no paths bypass either memory barrier"),
+ cl::init(true), cl::Hidden);
+
+static cl::opt<bool> MergeAMOWithMB(
+ "loongarch-merge-amo-with-dbar",
+ cl::desc("Merge AMOs with DBARs into AMO_DB during optimization"),
+ cl::init(true), cl::Hidden);
+
+static cl::opt<bool> DisableInlineAsm(
+ "loongarch-disable-inline-asm-barrier-opt",
+ cl::desc("Disable optimization of memory barriers in InlineAsm"),
+ cl::init(false), cl::Hidden);
+
+static cl::opt<bool> ReplaceEliminatedMBToNop(
+ "loongarch-replace-eliminated-dbar-to-nop",
+ cl::desc("Replace eliminated DBARs with NOPs to preserve code layout"),
+ cl::init(false), cl::Hidden);
+
+namespace {
+
+#define AMO_CASES \
+ CASE(AMSWAP, B) \
+ CASE(AMSWAP, H) \
+ CASE(AMSWAP, W) \
+ CASE(AMSWAP, D) \
+ CASE(AMADD, B) \
+ CASE(AMADD, H) \
+ CASE(AMADD, W) \
+ CASE(AMADD, D) \
+ CASE(AMAND, W) \
+ CASE(AMAND, D) \
+ CASE(AMOR, W) \
+ CASE(AMOR, D) \
+ CASE(AMXOR, W) \
+ CASE(AMXOR, D) \
+ CASE(AMMAX, W) \
+ CASE(AMMAX, D) \
+ CASE(AMMAX, WU) \
+ CASE(AMMAX, DU) \
+ CASE(AMMIN, W) \
+ CASE(AMMIN, D) \
+ CASE(AMMIN, WU) \
+ CASE(AMMIN, DU) \
+ CASE(AMCAS, B) \
+ CASE(AMCAS, H) \
+ CASE(AMCAS, W) \
+ CASE(AMCAS, D)
+
+static std::optional<std::pair<StringRef, StringRef>> parseMB(StringRef Asm) {
+ auto T1 = llvm::getToken(Asm);
+ if (!T1.first.equals_insensitive("dbar"))
+ return std::nullopt;
+ auto T2 = llvm::getToken(T1.second);
+ if (T2.first.empty())
+ return std::nullopt;
+ auto T3 = llvm::getToken(T2.second);
+ if (T3.first.trim().empty() || T3.first.starts_with('#'))
+ return std::pair(T1.first, T2.first);
+ return std::nullopt;
+}
+
+static std::optional<std::pair<StringRef, StringRef>>
+isAsmMB(const MachineInstr &MI) {
+ if (DisableInlineAsm)
+ return std::nullopt;
+ if (!MI.isInlineAsm())
+ return std::nullopt;
+ auto Asm = MI.getOperand(InlineAsm::MIOp_AsmString).getSymbolName();
+ return parseMB(Asm);
+}
+
+static StringRef getAMDB(StringRef Name) {
+#define CASE(Name, Suffix) \
+ .Case(#Name "." #Suffix, MergeAMOWithMB ? #Name "_DB." #Suffix : "") \
+ .Case(#Name "_DB." #Suffix, #Name "_DB." #Suffix)
+ return StringSwitch<StringRef>(Name.upper()) AMO_CASES.Default({});
+#undef CASE
+}
+
+static std::optional<std::pair<StringRef, StringRef>> parseAM(StringRef Asm) {
+ auto T1 = llvm::getToken(Asm);
+ auto OpName = getAMDB(T1.first);
+ if (OpName.empty())
+ return std::nullopt;
+ auto T2 = llvm::getToken(T1.second, ",");
+ if (T2.first.empty())
+ return std::nullopt;
+ auto T3 = llvm::getToken(T2.second, ",");
+ if (T3.first.empty())
+ return std::nullopt;
+ auto T4 = llvm::getToken(T3.second, ",");
+ if (T4.first.empty())
+ return std::nullopt;
+ auto T5 = llvm::getToken(T4.second);
+ if (T5.first.trim().empty() || T5.first.starts_with('#')) {
+ StringRef Operands(T2.first.data(),
+ T4.first.data() + T4.first.size() - T2.first.data());
+ return std::pair(OpName, Operands);
+ }
+ return std::nullopt;
+}
+
+static std::optional<std::pair<StringRef, StringRef>>
+isAsmAM(const MachineInstr &MI) {
+ if (DisableInlineAsm)
+ return std::nullopt;
+ if (!MI.isInlineAsm())
+ return std::nullopt;
+ auto Asm = MI.getOperand(InlineAsm::MIOp_AsmString).getSymbolName();
+ return parseAM(Asm);
+}
+
+static bool isMB(const MachineInstr &MI) {
+ return MI.getOpcode() == LoongArch::DBAR;
+}
+
+static bool isLL(const MachineInstr &MI) {
+ switch (MI.getOpcode()) {
+ case LoongArch::LL_W:
+ case LoongArch::LL_D:
+ return true;
+ default:
+ return false;
+ }
+}
+
+static bool isSC(const MachineInstr &MI) {
+ switch (MI.getOpcode()) {
+ case LoongArch::SC_W:
+ case LoongArch::SC_D:
+ case LoongArch::SC_Q:
+ return true;
+ default:
+ return false;
+ }
+}
+
+static std::optional<unsigned> isAM(const MachineInstr &MI) {
+#define CASE(Name, Suffix) \
+ case LoongArch::Name##_##Suffix: \
+ if (!MergeAMOWithMB) \
+ return std::nullopt; \
+ [[fallthrough]]; \
+ case LoongArch::Name##__DB_##Suffix: \
+ return LoongArch::Name##__DB_##Suffix;
+ switch (MI.getOpcode()) {
+ AMO_CASES
+ default:
+ return std::nullopt;
+ }
+#undef CASE
+}
+
+static bool isSafeToSkip(const MachineInstr &MI) {
+ if (MI.mayLoadOrStore())
+ return false;
+ if (MI.isCall() || MI.isReturn())
+ return false;
+ if (MI.isInlineAsm())
+ return isAsmMB(MI) != std::nullopt;
+ if (MI.hasUnmodeledSideEffects())
+ return isMB(MI);
+ return true;
+}
+
+struct BarrierHint {
+ BarrierHint(unsigned Hint) : Hint(Hint) {}
+
+ bool subsumes(const BarrierHint &O) const { return (Hint & O.Hint) == Hint; }
+
+ BarrierHint merge(const BarrierHint &O) const {
+ return BarrierHint(Hint & O.Hint);
+ }
+
+ static inline bool isValid(unsigned Hint) { return (Hint & ~0x1f) == 0; }
+
+ unsigned Hint;
+};
+
+struct InstBarrier {
+ InstBarrier(MachineInstr &MI)
+ : MI(&MI), Pre(0), Post(0), Data(0), IsMB(false), IsAM(false),
+ IsAsm(false) {
+ if (isMB(MI)) {
+ unsigned Hint = MI.getOperand(0).getImm();
+ if (!BarrierHint::isValid(Hint))
+ return;
+ IsMB = true;
+ Pre = Post = BarrierHint(Hint);
+ } else if (isLL(MI)) {
+ IsAM = true;
+ Pre = BarrierHint(0b10000);
+ Post = BarrierHint(0b11111);
+ } else if (isSC(MI)) {
+ IsAM = true;
+ Pre = BarrierHint(0b11111);
+ Post = BarrierHint(0b10000);
+ } else if (auto R = isAM(MI)) {
+ IsAM = true;
+ OpcAMDB = *R;
+ Pre = Post = BarrierHint(0b10000);
+ } else if (auto R = isAsmMB(MI)) {
+ OpName = (*R).first;
+ Operands = (*R).second;
+ auto B = parseAsmMB(Operands, MI);
+ if (!B || !BarrierHint::isValid((*B).first))
+ return;
+ IsMB = true;
+ IsAsm = true;
+ HintOff = (*B).second;
+ Pre = Post = BarrierHint((*B).first);
+ } else if (auto R = isAsmAM(MI)) {
+ OpName = (*R).first;
+ Operands = (*R).second;
+ IsAM = true;
+ IsAsm = true;
+ Pre = Post = BarrierHint(0b10000);
+ }
+ }
+
+ static std::optional<std::pair<unsigned, unsigned>>
+ parseAsmMB(StringRef Operand, MachineInstr &MI) {
+ unsigned Hint, HintOff = 0;
+ // DBAR N | 0xN
+ if (!Operand.starts_with('$')) {
+ if (Operand.getAsInteger(0, Hint))
+ return std::nullopt;
+ return std::pair(Hint, HintOff);
+ }
+ // DBAR $N
+ unsigned N = 0, Off, AsmDescOp;
+ if (Operand.drop_front().getAsInteger(0, Off))
+ return std::nullopt;
+ AsmDescOp = InlineAsm::MIOp_FirstOperand;
+ while (AsmDescOp != MI.getNumOperands()) {
+ const MachineOperand &MO = MI.getOperand(AsmDescOp);
+ assert(MO.isImm() && "Unexpected operand type!");
+ const InlineAsm::Flag F(MO.getImm());
+ if (N == Off) {
+ if (!F.isImmKind())
+ return std::nullopt;
+ HintOff = AsmDescOp + 1;
+ Hint = MI.getOperand(HintOff).getImm();
+ return std::pair(Hint, HintOff);
+ }
+ AsmDescOp += 1 + F.getNumOperandRegisters();
+ ++N;
+ }
+ return std::nullopt;
+ }
+
+ MachineInstr *MI;
+ BarrierHint Pre;
+ BarrierHint Post;
+ StringRef OpName;
+ StringRef Operands;
+ union {
+ unsigned OpcAMDB;
+ unsigned HintOff;
+ unsigned Data;
+ };
+ bool IsMB;
+ bool IsAM;
+ bool IsAsm;
+};
+
+class LoongArchMemoryBarrierOpt : public MachineFunctionPass {
+public:
+ static char ID;
+
+ LoongArchMemoryBarrierOpt() : MachineFunctionPass(ID) {}
+
+ StringRef getPassName() const override {
+ return LOONGARCH_MEMORY_BARRIER_OPT_NAME;
+ }
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.addRequired<MachineDominatorTreeWrapperPass>();
+ AU.addPreserved<MachineDominatorTreeWrapperPass>();
+ AU.addRequired<MachinePostDominatorTreeWrapperPass>();
+ AU.addPreserved<MachinePostDominatorTreeWrapperPass>();
+ MachineFunctionPass::getAnalysisUsage(AU);
+ }
+
+ bool runOnMachineFunction(MachineFunction &Fn) override;
+
+private:
+ enum : unsigned {
+ CandidateA = 1u << 0,
+ CandidateB = 1u << 1,
+ };
+
+ unsigned resolveBarrierRedundancy(const MachineInstr *A,
+ const MachineInstr *B) const;
+ bool eliminateRedundantBarrier(InstBarrier &IA, InstBarrier &IB) const;
+
+ MachineFunction *MF;
+ const MachineDominatorTree *MDT;
+ const MachinePostDominatorTree *MPDT;
+};
+
+static bool checkAllPathSafe(const MachineBasicBlock *MBBA,
+ const MachineBasicBlock *MBBB, bool IsAToB) {
+ const MachineBasicBlock *Start = IsAToB ? MBBA : MBBB;
+ const MachineBasicBlock *End = IsAToB ? MBBB : MBBA;
+
+ SmallVector<const MachineBasicBlock *, 16> Worklist;
+ DenseSet<const MachineBasicBlock *> Visited;
+
+ Worklist.push_back(Start);
+ Visited.insert(Start);
+
+ while (!Worklist.empty()) {
+ const MachineBasicBlock *BB = Worklist.pop_back_val();
+
+ if (BB == End)
+ continue;
+
+ if (BB != Start) {
+ for (const MachineInstr &MI : *BB) {
+ if (!isSafeToSkip(MI))
+ return false;
+ }
+ }
+
+ if (IsAToB) {
+ for (const MachineBasicBlock *Succ : BB->successors()) {
+ if (Visited.insert(Succ).second)
+ Worklist.push_back(Succ);
+ }
+ } else {
+ for (const MachineBasicBlock *Pred : BB->predecessors()) {
+ if (Visited.insert(Pred).second)
+ Worklist.push_back(Pred);
+ }
+ }
+ }
+
+ return true;
+}
+
+// Returns a bitmask indicating removal candidates: A (bit 1) and B (bit 2).
+unsigned LoongArchMemoryBarrierOpt::resolveBarrierRedundancy(
+ const MachineInstr *A, const MachineInstr *B) const {
+ const MachineBasicBlock *MBBA = A->getParent();
+ const MachineBasicBlock *MBBB = B->getParent();
+
+ if (MBBA == MBBB) {
+ /* A -> B */
+ for (auto It = std::next(A->getIterator()); It != MBBA->end(); ++It) {
+ if (It == B->getIterator())
+ return CandidateA | CandidateB;
+ if (!isSafeToSkip(*It))
+ return 0;
+ }
+ return 0;
+ }
+
+ // Cross-block walk
+ bool ADomB = MDT->dominates(MBBA, MBBB);
+ bool BPostDomA = MPDT->dominates(MBBB, MBBA);
+ unsigned Mask = 0;
+ if (!ADomB && !BPostDomA)
+ return 0;
+
+ /* A -> MBBA->end() */
+ for (auto It = std::next(A->getIterator()); It != MBBA->end(); ++It)
+ if (!isSafeToSkip(*It))
+ return 0;
+ /* B -> MBBB->begin() */
+ for (auto It = MBBB->begin(); It != B->getIterator(); ++It)
+ if (!isSafeToSkip(*It))
+ return 0;
+
+ /* MBBA -> MBBB */
+ if (BPostDomA)
+ if (checkAllPathSafe(MBBA, MBBB, true /*IsAToB*/))
+ Mask |= CandidateA;
+
+ /* MBBB -> MBBA */
+ if (ADomB)
+ if (checkAllPathSafe(MBBA, MBBB, false /*IsAToB*/))
+ Mask |= CandidateB;
+
+ return Mask;
+}
+
+// Update DBAR hint
+static void updateMB(InstBarrier &I, BarrierHint Hint, MachineFunction *MF) {
+ assert(I.IsMB && "Unexpected!");
+ I.Pre = I.Post = Hint;
+ if (!I.IsAsm) {
+ I.MI->getOperand(0).setImm(Hint.Hint);
+ return;
+ }
+ if (I.HintOff) {
+ I.MI->getOperand(I.HintOff).setImm(Hint.Hint);
+ return;
+ }
+ MachineOperand &MO = I.MI->getOperand(InlineAsm::MIOp_AsmString);
+ auto New = I.OpName.str() + " " + llvm::utostr(Hint.Hint);
+ MO.ChangeToES(MF->createExternalSymbolName(New), MO.getTargetFlags());
+}
+
+// Replace AMO to AMO_DB
+static void replaceAM(InstBarrier &I, MachineFunction *MF) {
+ if (!I.IsAM)
+ return;
+ if (I.OpcAMDB) {
+ auto &ST = MF->getSubtarget<LoongArchSubtarget>();
+ I.MI->setDesc(ST.getInstrInfo()->get(I.OpcAMDB));
+ return;
+ }
+ if (!I.IsAsm)
+ return;
+ MachineOperand &MO = I.MI->getOperand(InlineAsm::MIOp_AsmString);
+ auto New = I.OpName.str() + " " + I.Operands.str();
+ MO.ChangeToES(MF->createExternalSymbolName(New), MO.getTargetFlags());
+}
+
+bool LoongArchMemoryBarrierOpt::eliminateRedundantBarrier(
+ InstBarrier &IA, InstBarrier &IB) const {
+ MachineInstr *A = IA.MI;
+ MachineInstr *B = IB.MI;
+
+ if (!A || !B)
+ return false; // Already erased
+ if (A == B)
+ return false;
+
+ unsigned Mask = resolveBarrierRedundancy(A, B);
+ if (!Mask)
+ return false;
+
+ auto eraseOrReplaceWithNop = [&](MachineInstr *MI) {
+ if (ReplaceEliminatedMBToNop) {
+ auto &ST = MF->getSubtarget<LoongArchSubtarget>();
+ BuildMI(*MI->getParent(), MI->getIterator(), MI->getDebugLoc(),
+ ST.getInstrInfo()->get(LoongArch::ANDI), LoongArch::R0)
+ .addReg(LoongArch::R0)
+ .addImm(0);
+ }
+ MI->eraseFromParent();
+ };
+
+ // A B
+ // DBAR x + DBAR y -> DBAR (x & y)
+ // DBAR x + AMO_DB -> AMO_DB
+ // DBAR x + AMO -> AMO_DB
+ // DBAR x + LL -> LL
+ if ((Mask & CandidateA) && IA.IsMB) {
+ if (!IB.Pre.subsumes(IA.Post)) {
+ if (!IB.IsMB || (RequireNoPathBypass && !(Mask & CandidateB)))
+ return false;
+ updateMB(IB, IB.Pre.merge(IA.Post), MF);
+ }
+ replaceAM(IB, MF);
+ eraseOrReplaceWithNop(A);
+ IA.MI = nullptr;
+ return true;
+ }
+
+ // A B
+ // DBAR x + DBAR y -> DBAR (x & y)
+ // AMO_DB + DBAR x -> AMO_DB
+ // AMO + DBAR x -> AMO_DB
+ // SC + DBAR x -> SC
+ if ((Mask & CandidateB) && IB.IsMB) {
+ if (!IA.Post.subsumes(IB.Pre)) {
+ if (!IA.IsMB || (RequireNoPathBypass && !(Mask & CandidateA)))
+ return false;
+ updateMB(IA, IA.Post.merge(IB.Pre), MF);
+ }
+ replaceAM(IA, MF);
+ eraseOrReplaceWithNop(B);
+ IB.MI = nullptr;
+ return true;
+ }
+
+ return false;
+}
+
+bool LoongArchMemoryBarrierOpt::runOnMachineFunction(MachineFunction &Fn) {
+ if (skipFunction(Fn.getFunction()))
+ return false;
+
+ MF = &Fn;
+ MDT = &getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
+ MPDT = &getAnalysis<MachinePostDominatorTreeWrapperPass>().getPostDomTree();
+
+ SmallVector<InstBarrier, 32> Sites;
+ bool Changed = false;
+
+ for (MachineBasicBlock &MBB : Fn)
+ for (MachineInstr &MI : MBB) {
+ InstBarrier IB(MI);
+ if (IB.IsMB || IB.IsAM)
+ Sites.push_back(IB);
+ }
+
+ for (size_t a = 0; a < Sites.size(); ++a) {
+ for (size_t b = a + 1; b < Sites.size(); ++b) {
+ InstBarrier &IA = Sites[a];
+ InstBarrier &IB = Sites[b];
+ Changed |= eliminateRedundantBarrier(IA, IB);
+ Changed |= eliminateRedundantBarrier(IB, IA);
+ }
+ }
+
+ return Changed;
+}
+} // namespace
+
+char LoongArchMemoryBarrierOpt::ID = 0;
+INITIALIZE_PASS_BEGIN(LoongArchMemoryBarrierOpt, DEBUG_TYPE,
+ LOONGARCH_MEMORY_BARRIER_OPT_NAME, false, false)
+INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(MachinePostDominatorTreeWrapperPass)
+INITIALIZE_PASS_END(LoongArchMemoryBarrierOpt, DEBUG_TYPE,
+ LOONGARCH_MEMORY_BARRIER_OPT_NAME, false, false)
+
+FunctionPass *llvm::createLoongArchMemoryBarrierOptPass() {
+ return new LoongArchMemoryBarrierOpt();
+}
diff --git a/llvm/lib/Target/LoongArch/LoongArchTargetMachine.cpp b/llvm/lib/Target/LoongArch/LoongArchTargetMachine.cpp
index 1eab354ea191b..ce42a6d4b7773 100644
--- a/llvm/lib/Target/LoongArch/LoongArchTargetMachine.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchTargetMachine.cpp
@@ -37,6 +37,7 @@ LLVMInitializeLoongArchTarget() {
RegisterTargetMachine<LoongArchTargetMachine> Y(getTheLoongArch64Target());
auto *PR = PassRegistry::getPassRegistry();
initializeLoongArchDeadRegisterDefinitionsPass(*PR);
+ initializeLoongArchMemoryBarrierOptPass(*PR);
initializeLoongArchMergeBaseOffsetOptPass(*PR);
initializeLoongArchOptWInstrsPass(*PR);
initializeLoongArchPreRAExpandPseudoPass(*PR);
@@ -202,6 +203,8 @@ void LoongArchPassConfig::addPreEmitPass2() {
// avoiding the possibility for other passes to break the requirements for
// forward progress in the LL/SC block.
addPass(createLoongArchExpandAtomicPseudoPass());
+ if (TM->getOptLevel() != CodeGenOptLevel::None)
+ addPass(createLoongArchMemoryBarrierOptPass());
}
void LoongArchPassConfig::addMachineSSAOptimization() {
diff --git a/llvm/test/CodeGen/LoongArch/memory-barrier-opt.ll b/llvm/test/CodeGen/LoongArch/memory-barrier-opt.ll
index fa37dda01c7b1..cf3041b1094c0 100644
--- a/llvm/test/CodeGen/LoongArch/memory-barrier-opt.ll
+++ b/llvm/test/CodeGen/LoongArch/memory-barrier-opt.ll
@@ -1,14 +1,43 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s
+; RUN: llc --mtriple=loongarch64 -mattr=+d -verify-machineinstrs < %s \
+; RUN: | FileCheck %s --check-prefixes=CHECK,NORMAL
+; RUN: llc --mtriple=loongarch64 -mattr=+d --loongarch-require-no-path-bypass=false -verify-machineinstrs < %s \
+; RUN: | FileCheck %s --check-prefixes=CHECK,BYPASS
+; RUN: llc --mtriple=loongarch64 -mattr=+d --loongarch-merge-amo-with-dbar=false -verify-machineinstrs < %s \
+; RUN: | FileCheck %s --check-prefixes=CHECK,NOMAMO
+; RUN: llc --mtriple=loongarch64 -mattr=+d --loongarch-disable-inline-asm-barrier-opt=true -verify-machineinstrs < %s \
+; RUN: | FileCheck %s --check-prefixes=CHECK,NOIASM
+; RUN: llc --mtriple=loongarch64 -mattr=+d --loongarch-replace-eliminated-dbar-to-nop=true -verify-machineinstrs < %s \
+; RUN: | FileCheck %s --check-prefixes=CHECK,BTONOP
;; Positive
define void @dbar_acquire_acquire() nounwind {
-; CHECK-LABEL: dbar_acquire_acquire:
-; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: dbar 20
-; CHECK-NEXT: dbar 20
-; CHECK-NEXT: ret
+; NORMAL-LABEL: dbar_acquire_acquire:
+; NORMAL: # %bb.0: # %entry
+; NORMAL-NEXT: dbar 20
+; NORMAL-NEXT: ret
+;
+; BYPASS-LABEL: dbar_acquire_acquire:
+; BYPASS: # %bb.0: # %entry
+; BYPASS-NEXT: dbar 20
+; BYPASS-NEXT: ret
+;
+; NOMAMO-LABEL: dbar_acquire_acquire:
+; NOMAMO: # %bb.0: # %entry
+; NOMAMO-NEXT: dbar 20
+; NOMAMO-NEXT: ret
+;
+; NOIASM-LABEL: dbar_acquire_acquire:
+; NOIASM: # %bb.0: # %entry
+; NOIASM-NEXT: dbar 20
+; NOIASM-NEXT: ret
+;
+; BTONOP-LABEL: dbar_acquire_acquire:
+; BTONOP: # %bb.0: # %entry
+; BTONOP-NEXT: nop
+; BTONOP-NEXT: dbar 20
+; BTONOP-NEXT: ret
entry:
fence acquire
fence acquire
@@ -16,11 +45,31 @@ entry:
}
define void @dbar_acquire_release() nounwind {
-; CHECK-LABEL: dbar_acquire_release:
-; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: dbar 20
-; CHECK-NEXT: dbar 18
-; CHECK-NEXT: ret
+; NORMAL-LABEL: dbar_acquire_release:
+; NORMAL: # %bb.0: # %entry
+; NORMAL-NEXT: dbar 16
+; NORMAL-NEXT: ret
+;
+; BYPASS-LABEL: dbar_acquire_release:
+; BYPASS: # %bb.0: # %entry
+; BYPASS-NEXT: dbar 16
+; BYPASS-NEXT: ret
+;
+; NOMAMO-LABEL: dbar_acquire_release:
+; NOMAMO: # %bb.0: # %entry
+; NOMAMO-NEXT: dbar 16
+; NOMAMO-NEXT: ret
+;
+; NOIASM-LABEL: dbar_acquire_release:
+; NOIASM: # %bb.0: # %entry
+; NOIASM-NEXT: dbar 16
+; NOIASM-NEXT: ret
+;
+; BTONOP-LABEL: dbar_acquire_release:
+; BTONOP: # %bb.0: # %entry
+; BTONOP-NEXT: nop
+; BTONOP-NEXT: dbar 16
+; BTONOP-NEXT: ret
entry:
fence acquire
fence release
@@ -28,12 +77,36 @@ entry:
}
define i32 @dbar_acquire_add_acquire(i32 %a) nounwind {
-; CHECK-LABEL: dbar_acquire_add_acquire:
-; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: dbar 20
-; CHECK-NEXT: addi.w $a0, $a0, 1
-; CHECK-NEXT: dbar 20
-; CHECK-NEXT: ret
+; NORMAL-LABEL: dbar_acquire_add_acquire:
+; NORMAL: # %bb.0: # %entry
+; NORMAL-NEXT: addi.w $a0, $a0, 1
+; NORMAL-NEXT: dbar 20
+; NORMAL-NEXT: ret
+;
+; BYPASS-LABEL: dbar_acquire_add_acquire:
+; BYPASS: # %bb.0: # %entry
+; BYPASS-NEXT: addi.w $a0, $a0, 1
+; BYPASS-NEXT: dbar 20
+; BYPASS-NEXT: ret
+;
+; NOMAMO-LABEL: dbar_acquire_add_acquire:
+; NOMAMO: # %bb.0: # %entry
+; NOMAMO-NEXT: addi.w $a0, $a0, 1
+; NOMAMO-NEXT: dbar 20
+; NOMAMO-NEXT: ret
+;
+; NOIASM-LABEL: dbar_acquire_add_acquire:
+; NOIASM: # %bb.0: # %entry
+; NOIASM-NEXT: addi.w $a0, $a0, 1
+; NOIASM-NEXT: dbar 20
+; NOIASM-NEXT: ret
+;
+; BTONOP-LABEL: dbar_acquire_add_acquire:
+; BTONOP: # %bb.0: # %entry
+; BTONOP-NEXT: nop
+; BTONOP-NEXT: addi.w $a0, $a0, 1
+; BTONOP-NEXT: dbar 20
+; BTONOP-NEXT: ret
entry:
fence acquire
%0 = add i32 %a, 1
@@ -42,11 +115,31 @@ entry:
}
define void @dbar_release_acquire() nounwind {
-; CHECK-LABEL: dbar_release_acquire:
-; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: dbar 18
-; CHECK-NEXT: dbar 20
-; CHECK-NEXT: ret
+; NORMAL-LABEL: dbar_release_acquire:
+; NORMAL: # %bb.0: # %entry
+; NORMAL-NEXT: dbar 16
+; NORMAL-NEXT: ret
+;
+; BYPASS-LABEL: dbar_release_acquire:
+; BYPASS: # %bb.0: # %entry
+; BYPASS-NEXT: dbar 16
+; BYPASS-NEXT: ret
+;
+; NOMAMO-LABEL: dbar_release_acquire:
+; NOMAMO: # %bb.0: # %entry
+; NOMAMO-NEXT: dbar 16
+; NOMAMO-NEXT: ret
+;
+; NOIASM-LABEL: dbar_release_acquire:
+; NOIASM: # %bb.0: # %entry
+; NOIASM-NEXT: dbar 16
+; NOIASM-NEXT: ret
+;
+; BTONOP-LABEL: dbar_release_acquire:
+; BTONOP: # %bb.0: # %entry
+; BTONOP-NEXT: nop
+; BTONOP-NEXT: dbar 16
+; BTONOP-NEXT: ret
entry:
fence release
fence acquire
@@ -54,12 +147,32 @@ entry:
}
define void @dbar_seq_cst_acq_rel_seq_cst() nounwind {
-; CHECK-LABEL: dbar_seq_cst_acq_rel_seq_cst:
-; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: dbar 16
-; CHECK-NEXT: dbar 16
-; CHECK-NEXT: dbar 16
-; CHECK-NEXT: ret
+; NORMAL-LABEL: dbar_seq_cst_acq_rel_seq_cst:
+; NORMAL: # %bb.0: # %entry
+; NORMAL-NEXT: dbar 16
+; NORMAL-NEXT: ret
+;
+; BYPASS-LABEL: dbar_seq_cst_acq_rel_seq_cst:
+; BYPASS: # %bb.0: # %entry
+; BYPASS-NEXT: dbar 16
+; BYPASS-NEXT: ret
+;
+; NOMAMO-LABEL: dbar_seq_cst_acq_rel_seq_cst:
+; NOMAMO: # %bb.0: # %entry
+; NOMAMO-NEXT: dbar 16
+; NOMAMO-NEXT: ret
+;
+; NOIASM-LABEL: dbar_seq_cst_acq_rel_seq_cst:
+; NOIASM: # %bb.0: # %entry
+; NOIASM-NEXT: dbar 16
+; NOIASM-NEXT: ret
+;
+; BTONOP-LABEL: dbar_seq_cst_acq_rel_seq_cst:
+; BTONOP: # %bb.0: # %entry
+; BTONOP-NEXT: nop
+; BTONOP-NEXT: nop
+; BTONOP-NEXT: dbar 16
+; BTONOP-NEXT: ret
entry:
fence seq_cst
fence acq_rel
@@ -68,14 +181,36 @@ entry:
}
define void @dbar_acquire_asm_dbar_acquire() nounwind {
-; CHECK-LABEL: dbar_acquire_asm_dbar_acquire:
-; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: dbar 20
-; CHECK-NEXT: #APP
-; CHECK-NEXT: dbar 22
-; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: dbar 20
-; CHECK-NEXT: ret
+; NORMAL-LABEL: dbar_acquire_asm_dbar_acquire:
+; NORMAL: # %bb.0: # %entry
+; NORMAL-NEXT: dbar 20
+; NORMAL-NEXT: ret
+;
+; BYPASS-LABEL: dbar_acquire_asm_dbar_acquire:
+; BYPASS: # %bb.0: # %entry
+; BYPASS-NEXT: dbar 20
+; BYPASS-NEXT: ret
+;
+; NOMAMO-LABEL: dbar_acquire_asm_dbar_acquire:
+; NOMAMO: # %bb.0: # %entry
+; NOMAMO-NEXT: dbar 20
+; NOMAMO-NEXT: ret
+;
+; NOIASM-LABEL: dbar_acquire_asm_dbar_acquire:
+; NOIASM: # %bb.0: # %entry
+; NOIASM-NEXT: dbar 20
+; NOIASM-NEXT: #APP
+; NOIASM-NEXT: dbar 22
+; NOIASM-NEXT: #NO_APP
+; NOIASM-NEXT: dbar 20
+; NOIASM-NEXT: ret
+;
+; BTONOP-LABEL: dbar_acquire_asm_dbar_acquire:
+; BTONOP: # %bb.0: # %entry
+; BTONOP-NEXT: nop
+; BTONOP-NEXT: nop
+; BTONOP-NEXT: dbar 20
+; BTONOP-NEXT: ret
entry:
fence acquire
call void asm "dbar 22", ""()
@@ -84,14 +219,36 @@ entry:
}
define void @dbar_acquire_asm_space_dbar_acquire() nounwind {
-; CHECK-LABEL: dbar_acquire_asm_space_dbar_acquire:
-; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: dbar 20
-; CHECK-NEXT: #APP
-; CHECK-NEXT: dbar 22
-; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: dbar 20
-; CHECK-NEXT: ret
+; NORMAL-LABEL: dbar_acquire_asm_space_dbar_acquire:
+; NORMAL: # %bb.0: # %entry
+; NORMAL-NEXT: dbar 20
+; NORMAL-NEXT: ret
+;
+; BYPASS-LABEL: dbar_acquire_asm_space_dbar_acquire:
+; BYPASS: # %bb.0: # %entry
+; BYPASS-NEXT: dbar 20
+; BYPASS-NEXT: ret
+;
+; NOMAMO-LABEL: dbar_acquire_asm_space_dbar_acquire:
+; NOMAMO: # %bb.0: # %entry
+; NOMAMO-NEXT: dbar 20
+; NOMAMO-NEXT: ret
+;
+; NOIASM-LABEL: dbar_acquire_asm_space_dbar_acquire:
+; NOIASM: # %bb.0: # %entry
+; NOIASM-NEXT: dbar 20
+; NOIASM-NEXT: #APP
+; NOIASM-NEXT: dbar 22
+; NOIASM-NEXT: #NO_APP
+; NOIASM-NEXT: dbar 20
+; NOIASM-NEXT: ret
+;
+; BTONOP-LABEL: dbar_acquire_asm_space_dbar_acquire:
+; BTONOP: # %bb.0: # %entry
+; BTONOP-NEXT: nop
+; BTONOP-NEXT: nop
+; BTONOP-NEXT: dbar 20
+; BTONOP-NEXT: ret
entry:
fence acquire
call void asm " dbar 22", ""()
@@ -100,14 +257,36 @@ entry:
}
define void @dbar_acquire_asm_comment_dbar_acquire() nounwind {
-; CHECK-LABEL: dbar_acquire_asm_comment_dbar_acquire:
-; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: dbar 20
-; CHECK-NEXT: #APP
-; CHECK-NEXT: dbar 22 # barrier
-; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: dbar 20
-; CHECK-NEXT: ret
+; NORMAL-LABEL: dbar_acquire_asm_comment_dbar_acquire:
+; NORMAL: # %bb.0: # %entry
+; NORMAL-NEXT: dbar 20
+; NORMAL-NEXT: ret
+;
+; BYPASS-LABEL: dbar_acquire_asm_comment_dbar_acquire:
+; BYPASS: # %bb.0: # %entry
+; BYPASS-NEXT: dbar 20
+; BYPASS-NEXT: ret
+;
+; NOMAMO-LABEL: dbar_acquire_asm_comment_dbar_acquire:
+; NOMAMO: # %bb.0: # %entry
+; NOMAMO-NEXT: dbar 20
+; NOMAMO-NEXT: ret
+;
+; NOIASM-LABEL: dbar_acquire_asm_comment_dbar_acquire:
+; NOIASM: # %bb.0: # %entry
+; NOIASM-NEXT: dbar 20
+; NOIASM-NEXT: #APP
+; NOIASM-NEXT: dbar 22 # barrier
+; NOIASM-NEXT: #NO_APP
+; NOIASM-NEXT: dbar 20
+; NOIASM-NEXT: ret
+;
+; BTONOP-LABEL: dbar_acquire_asm_comment_dbar_acquire:
+; BTONOP: # %bb.0: # %entry
+; BTONOP-NEXT: nop
+; BTONOP-NEXT: nop
+; BTONOP-NEXT: dbar 20
+; BTONOP-NEXT: ret
entry:
fence acquire
call void asm "dbar 22 # barrier", ""()
@@ -116,14 +295,36 @@ entry:
}
define void @dbar_acquire_asm_arg0_dbar_acquire() nounwind {
-; CHECK-LABEL: dbar_acquire_asm_arg0_dbar_acquire:
-; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: dbar 20
-; CHECK-NEXT: #APP
-; CHECK-NEXT: dbar 16
-; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: dbar 20
-; CHECK-NEXT: ret
+; NORMAL-LABEL: dbar_acquire_asm_arg0_dbar_acquire:
+; NORMAL: # %bb.0: # %entry
+; NORMAL-NEXT: dbar 16
+; NORMAL-NEXT: ret
+;
+; BYPASS-LABEL: dbar_acquire_asm_arg0_dbar_acquire:
+; BYPASS: # %bb.0: # %entry
+; BYPASS-NEXT: dbar 16
+; BYPASS-NEXT: ret
+;
+; NOMAMO-LABEL: dbar_acquire_asm_arg0_dbar_acquire:
+; NOMAMO: # %bb.0: # %entry
+; NOMAMO-NEXT: dbar 16
+; NOMAMO-NEXT: ret
+;
+; NOIASM-LABEL: dbar_acquire_asm_arg0_dbar_acquire:
+; NOIASM: # %bb.0: # %entry
+; NOIASM-NEXT: dbar 20
+; NOIASM-NEXT: #APP
+; NOIASM-NEXT: dbar 16
+; NOIASM-NEXT: #NO_APP
+; NOIASM-NEXT: dbar 20
+; NOIASM-NEXT: ret
+;
+; BTONOP-LABEL: dbar_acquire_asm_arg0_dbar_acquire:
+; BTONOP: # %bb.0: # %entry
+; BTONOP-NEXT: nop
+; BTONOP-NEXT: nop
+; BTONOP-NEXT: dbar 16
+; BTONOP-NEXT: ret
entry:
fence acquire
call void asm sideeffect "dbar $0", "I,I,~{memory}"(i32 16, i32 22)
@@ -132,14 +333,36 @@ entry:
}
define void @dbar_acquire_asm_arg1_dbar_acquire() nounwind {
-; CHECK-LABEL: dbar_acquire_asm_arg1_dbar_acquire:
-; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: dbar 20
-; CHECK-NEXT: #APP
-; CHECK-NEXT: dbar 16
-; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: dbar 20
-; CHECK-NEXT: ret
+; NORMAL-LABEL: dbar_acquire_asm_arg1_dbar_acquire:
+; NORMAL: # %bb.0: # %entry
+; NORMAL-NEXT: dbar 16
+; NORMAL-NEXT: ret
+;
+; BYPASS-LABEL: dbar_acquire_asm_arg1_dbar_acquire:
+; BYPASS: # %bb.0: # %entry
+; BYPASS-NEXT: dbar 16
+; BYPASS-NEXT: ret
+;
+; NOMAMO-LABEL: dbar_acquire_asm_arg1_dbar_acquire:
+; NOMAMO: # %bb.0: # %entry
+; NOMAMO-NEXT: dbar 16
+; NOMAMO-NEXT: ret
+;
+; NOIASM-LABEL: dbar_acquire_asm_arg1_dbar_acquire:
+; NOIASM: # %bb.0: # %entry
+; NOIASM-NEXT: dbar 20
+; NOIASM-NEXT: #APP
+; NOIASM-NEXT: dbar 16
+; NOIASM-NEXT: #NO_APP
+; NOIASM-NEXT: dbar 20
+; NOIASM-NEXT: ret
+;
+; BTONOP-LABEL: dbar_acquire_asm_arg1_dbar_acquire:
+; BTONOP: # %bb.0: # %entry
+; BTONOP-NEXT: nop
+; BTONOP-NEXT: nop
+; BTONOP-NEXT: dbar 16
+; BTONOP-NEXT: ret
entry:
fence acquire
call void asm sideeffect "dbar $1", "I,I,~{memory}"(i32 22, i32 16)
@@ -148,11 +371,31 @@ entry:
}
define void @dbar_completion_acquire() nounwind {
-; CHECK-LABEL: dbar_completion_acquire:
-; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: dbar 0
-; CHECK-NEXT: dbar 20
-; CHECK-NEXT: ret
+; NORMAL-LABEL: dbar_completion_acquire:
+; NORMAL: # %bb.0: # %entry
+; NORMAL-NEXT: dbar 0
+; NORMAL-NEXT: ret
+;
+; BYPASS-LABEL: dbar_completion_acquire:
+; BYPASS: # %bb.0: # %entry
+; BYPASS-NEXT: dbar 0
+; BYPASS-NEXT: ret
+;
+; NOMAMO-LABEL: dbar_completion_acquire:
+; NOMAMO: # %bb.0: # %entry
+; NOMAMO-NEXT: dbar 0
+; NOMAMO-NEXT: ret
+;
+; NOIASM-LABEL: dbar_completion_acquire:
+; NOIASM: # %bb.0: # %entry
+; NOIASM-NEXT: dbar 0
+; NOIASM-NEXT: ret
+;
+; BTONOP-LABEL: dbar_completion_acquire:
+; BTONOP: # %bb.0: # %entry
+; BTONOP-NEXT: nop
+; BTONOP-NEXT: dbar 0
+; BTONOP-NEXT: ret
entry:
call void @llvm.loongarch.dbar(i32 0)
fence acquire
@@ -160,11 +403,31 @@ entry:
}
define void @dbar_release_completion() nounwind {
-; CHECK-LABEL: dbar_release_completion:
-; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: dbar 18
-; CHECK-NEXT: dbar 0
-; CHECK-NEXT: ret
+; NORMAL-LABEL: dbar_release_completion:
+; NORMAL: # %bb.0: # %entry
+; NORMAL-NEXT: dbar 0
+; NORMAL-NEXT: ret
+;
+; BYPASS-LABEL: dbar_release_completion:
+; BYPASS: # %bb.0: # %entry
+; BYPASS-NEXT: dbar 0
+; BYPASS-NEXT: ret
+;
+; NOMAMO-LABEL: dbar_release_completion:
+; NOMAMO: # %bb.0: # %entry
+; NOMAMO-NEXT: dbar 0
+; NOMAMO-NEXT: ret
+;
+; NOIASM-LABEL: dbar_release_completion:
+; NOIASM: # %bb.0: # %entry
+; NOIASM-NEXT: dbar 0
+; NOIASM-NEXT: ret
+;
+; BTONOP-LABEL: dbar_release_completion:
+; BTONOP: # %bb.0: # %entry
+; BTONOP-NEXT: nop
+; BTONOP-NEXT: dbar 0
+; BTONOP-NEXT: ret
entry:
fence release
call void @llvm.loongarch.dbar(i32 0)
@@ -172,12 +435,36 @@ entry:
}
define void @dbar_release_amswap_i8_acquire() nounwind {
-; CHECK-LABEL: dbar_release_amswap_i8_acquire:
-; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: dbar 18
-; CHECK-NEXT: addi.w $a0, $zero, -256
-; CHECK-NEXT: amand_db.w $a1, $a0, $zero
-; CHECK-NEXT: ret
+; NORMAL-LABEL: dbar_release_amswap_i8_acquire:
+; NORMAL: # %bb.0: # %entry
+; NORMAL-NEXT: addi.w $a0, $zero, -256
+; NORMAL-NEXT: amand_db.w $a1, $a0, $zero
+; NORMAL-NEXT: ret
+;
+; BYPASS-LABEL: dbar_release_amswap_i8_acquire:
+; BYPASS: # %bb.0: # %entry
+; BYPASS-NEXT: addi.w $a0, $zero, -256
+; BYPASS-NEXT: amand_db.w $a1, $a0, $zero
+; BYPASS-NEXT: ret
+;
+; NOMAMO-LABEL: dbar_release_amswap_i8_acquire:
+; NOMAMO: # %bb.0: # %entry
+; NOMAMO-NEXT: addi.w $a0, $zero, -256
+; NOMAMO-NEXT: amand_db.w $a1, $a0, $zero
+; NOMAMO-NEXT: ret
+;
+; NOIASM-LABEL: dbar_release_amswap_i8_acquire:
+; NOIASM: # %bb.0: # %entry
+; NOIASM-NEXT: addi.w $a0, $zero, -256
+; NOIASM-NEXT: amand_db.w $a1, $a0, $zero
+; NOIASM-NEXT: ret
+;
+; BTONOP-LABEL: dbar_release_amswap_i8_acquire:
+; BTONOP: # %bb.0: # %entry
+; BTONOP-NEXT: nop
+; BTONOP-NEXT: addi.w $a0, $zero, -256
+; BTONOP-NEXT: amand_db.w $a1, $a0, $zero
+; BTONOP-NEXT: ret
entry:
fence release
atomicrmw xchg ptr null, i8 0 acquire
@@ -185,11 +472,31 @@ entry:
}
define void @dbar_release_amswap_i32_acquire() nounwind {
-; CHECK-LABEL: dbar_release_amswap_i32_acquire:
-; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: dbar 18
-; CHECK-NEXT: amswap_db.w $a0, $zero, $zero
-; CHECK-NEXT: ret
+; NORMAL-LABEL: dbar_release_amswap_i32_acquire:
+; NORMAL: # %bb.0: # %entry
+; NORMAL-NEXT: amswap_db.w $a0, $zero, $zero
+; NORMAL-NEXT: ret
+;
+; BYPASS-LABEL: dbar_release_amswap_i32_acquire:
+; BYPASS: # %bb.0: # %entry
+; BYPASS-NEXT: amswap_db.w $a0, $zero, $zero
+; BYPASS-NEXT: ret
+;
+; NOMAMO-LABEL: dbar_release_amswap_i32_acquire:
+; NOMAMO: # %bb.0: # %entry
+; NOMAMO-NEXT: amswap_db.w $a0, $zero, $zero
+; NOMAMO-NEXT: ret
+;
+; NOIASM-LABEL: dbar_release_amswap_i32_acquire:
+; NOIASM: # %bb.0: # %entry
+; NOIASM-NEXT: amswap_db.w $a0, $zero, $zero
+; NOIASM-NEXT: ret
+;
+; BTONOP-LABEL: dbar_release_amswap_i32_acquire:
+; BTONOP: # %bb.0: # %entry
+; BTONOP-NEXT: nop
+; BTONOP-NEXT: amswap_db.w $a0, $zero, $zero
+; BTONOP-NEXT: ret
entry:
fence release
atomicrmw xchg ptr null, i32 0 acquire
@@ -197,11 +504,32 @@ entry:
}
define void @dbar_release_amswap_i32_monotonic() nounwind {
-; CHECK-LABEL: dbar_release_amswap_i32_monotonic:
-; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: dbar 18
-; CHECK-NEXT: amswap.w $a0, $zero, $zero
-; CHECK-NEXT: ret
+; NORMAL-LABEL: dbar_release_amswap_i32_monotonic:
+; NORMAL: # %bb.0: # %entry
+; NORMAL-NEXT: amswap_db.w $a0, $zero, $zero
+; NORMAL-NEXT: ret
+;
+; BYPASS-LABEL: dbar_release_amswap_i32_monotonic:
+; BYPASS: # %bb.0: # %entry
+; BYPASS-NEXT: amswap_db.w $a0, $zero, $zero
+; BYPASS-NEXT: ret
+;
+; NOMAMO-LABEL: dbar_release_amswap_i32_monotonic:
+; NOMAMO: # %bb.0: # %entry
+; NOMAMO-NEXT: dbar 18
+; NOMAMO-NEXT: amswap.w $a0, $zero, $zero
+; NOMAMO-NEXT: ret
+;
+; NOIASM-LABEL: dbar_release_amswap_i32_monotonic:
+; NOIASM: # %bb.0: # %entry
+; NOIASM-NEXT: amswap_db.w $a0, $zero, $zero
+; NOIASM-NEXT: ret
+;
+; BTONOP-LABEL: dbar_release_amswap_i32_monotonic:
+; BTONOP: # %bb.0: # %entry
+; BTONOP-NEXT: nop
+; BTONOP-NEXT: amswap_db.w $a0, $zero, $zero
+; BTONOP-NEXT: ret
entry:
fence release
atomicrmw xchg ptr null, i32 0 monotonic
@@ -221,12 +549,36 @@ entry:
}
define void @amswap_i8_release_dbar_acquire() nounwind {
-; CHECK-LABEL: amswap_i8_release_dbar_acquire:
-; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: addi.w $a0, $zero, -256
-; CHECK-NEXT: amand_db.w $a1, $a0, $zero
-; CHECK-NEXT: dbar 20
-; CHECK-NEXT: ret
+; NORMAL-LABEL: amswap_i8_release_dbar_acquire:
+; NORMAL: # %bb.0: # %entry
+; NORMAL-NEXT: addi.w $a0, $zero, -256
+; NORMAL-NEXT: amand_db.w $a1, $a0, $zero
+; NORMAL-NEXT: ret
+;
+; BYPASS-LABEL: amswap_i8_release_dbar_acquire:
+; BYPASS: # %bb.0: # %entry
+; BYPASS-NEXT: addi.w $a0, $zero, -256
+; BYPASS-NEXT: amand_db.w $a1, $a0, $zero
+; BYPASS-NEXT: ret
+;
+; NOMAMO-LABEL: amswap_i8_release_dbar_acquire:
+; NOMAMO: # %bb.0: # %entry
+; NOMAMO-NEXT: addi.w $a0, $zero, -256
+; NOMAMO-NEXT: amand_db.w $a1, $a0, $zero
+; NOMAMO-NEXT: ret
+;
+; NOIASM-LABEL: amswap_i8_release_dbar_acquire:
+; NOIASM: # %bb.0: # %entry
+; NOIASM-NEXT: addi.w $a0, $zero, -256
+; NOIASM-NEXT: amand_db.w $a1, $a0, $zero
+; NOIASM-NEXT: ret
+;
+; BTONOP-LABEL: amswap_i8_release_dbar_acquire:
+; BTONOP: # %bb.0: # %entry
+; BTONOP-NEXT: addi.w $a0, $zero, -256
+; BTONOP-NEXT: amand_db.w $a1, $a0, $zero
+; BTONOP-NEXT: nop
+; BTONOP-NEXT: ret
entry:
atomicrmw xchg ptr null, i8 0 release
fence acquire
@@ -234,11 +586,31 @@ entry:
}
define void @amswap_i32_release_dbar_acquire() nounwind {
-; CHECK-LABEL: amswap_i32_release_dbar_acquire:
-; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: amswap_db.w $a0, $zero, $zero
-; CHECK-NEXT: dbar 20
-; CHECK-NEXT: ret
+; NORMAL-LABEL: amswap_i32_release_dbar_acquire:
+; NORMAL: # %bb.0: # %entry
+; NORMAL-NEXT: amswap_db.w $a0, $zero, $zero
+; NORMAL-NEXT: ret
+;
+; BYPASS-LABEL: amswap_i32_release_dbar_acquire:
+; BYPASS: # %bb.0: # %entry
+; BYPASS-NEXT: amswap_db.w $a0, $zero, $zero
+; BYPASS-NEXT: ret
+;
+; NOMAMO-LABEL: amswap_i32_release_dbar_acquire:
+; NOMAMO: # %bb.0: # %entry
+; NOMAMO-NEXT: amswap_db.w $a0, $zero, $zero
+; NOMAMO-NEXT: ret
+;
+; NOIASM-LABEL: amswap_i32_release_dbar_acquire:
+; NOIASM: # %bb.0: # %entry
+; NOIASM-NEXT: amswap_db.w $a0, $zero, $zero
+; NOIASM-NEXT: ret
+;
+; BTONOP-LABEL: amswap_i32_release_dbar_acquire:
+; BTONOP: # %bb.0: # %entry
+; BTONOP-NEXT: amswap_db.w $a0, $zero, $zero
+; BTONOP-NEXT: nop
+; BTONOP-NEXT: ret
entry:
atomicrmw xchg ptr null, i32 0 release
fence acquire
@@ -258,13 +630,37 @@ entry:
}
define void @dbar_release_amswap_i8_release_dbar_acquire() nounwind {
-; CHECK-LABEL: dbar_release_amswap_i8_release_dbar_acquire:
-; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: dbar 18
-; CHECK-NEXT: addi.w $a0, $zero, -256
-; CHECK-NEXT: amand_db.w $a1, $a0, $zero
-; CHECK-NEXT: dbar 20
-; CHECK-NEXT: ret
+; NORMAL-LABEL: dbar_release_amswap_i8_release_dbar_acquire:
+; NORMAL: # %bb.0: # %entry
+; NORMAL-NEXT: addi.w $a0, $zero, -256
+; NORMAL-NEXT: amand_db.w $a1, $a0, $zero
+; NORMAL-NEXT: ret
+;
+; BYPASS-LABEL: dbar_release_amswap_i8_release_dbar_acquire:
+; BYPASS: # %bb.0: # %entry
+; BYPASS-NEXT: addi.w $a0, $zero, -256
+; BYPASS-NEXT: amand_db.w $a1, $a0, $zero
+; BYPASS-NEXT: ret
+;
+; NOMAMO-LABEL: dbar_release_amswap_i8_release_dbar_acquire:
+; NOMAMO: # %bb.0: # %entry
+; NOMAMO-NEXT: addi.w $a0, $zero, -256
+; NOMAMO-NEXT: amand_db.w $a1, $a0, $zero
+; NOMAMO-NEXT: ret
+;
+; NOIASM-LABEL: dbar_release_amswap_i8_release_dbar_acquire:
+; NOIASM: # %bb.0: # %entry
+; NOIASM-NEXT: addi.w $a0, $zero, -256
+; NOIASM-NEXT: amand_db.w $a1, $a0, $zero
+; NOIASM-NEXT: ret
+;
+; BTONOP-LABEL: dbar_release_amswap_i8_release_dbar_acquire:
+; BTONOP: # %bb.0: # %entry
+; BTONOP-NEXT: nop
+; BTONOP-NEXT: addi.w $a0, $zero, -256
+; BTONOP-NEXT: amand_db.w $a1, $a0, $zero
+; BTONOP-NEXT: nop
+; BTONOP-NEXT: ret
entry:
fence release
atomicrmw xchg ptr null, i8 0 release
@@ -273,12 +669,32 @@ entry:
}
define void @dbar_acquire_amswap_i32_release_dbar_acquire() nounwind {
-; CHECK-LABEL: dbar_acquire_amswap_i32_release_dbar_acquire:
-; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: dbar 20
-; CHECK-NEXT: amswap_db.w $a0, $zero, $zero
-; CHECK-NEXT: dbar 20
-; CHECK-NEXT: ret
+; NORMAL-LABEL: dbar_acquire_amswap_i32_release_dbar_acquire:
+; NORMAL: # %bb.0: # %entry
+; NORMAL-NEXT: amswap_db.w $a0, $zero, $zero
+; NORMAL-NEXT: ret
+;
+; BYPASS-LABEL: dbar_acquire_amswap_i32_release_dbar_acquire:
+; BYPASS: # %bb.0: # %entry
+; BYPASS-NEXT: amswap_db.w $a0, $zero, $zero
+; BYPASS-NEXT: ret
+;
+; NOMAMO-LABEL: dbar_acquire_amswap_i32_release_dbar_acquire:
+; NOMAMO: # %bb.0: # %entry
+; NOMAMO-NEXT: amswap_db.w $a0, $zero, $zero
+; NOMAMO-NEXT: ret
+;
+; NOIASM-LABEL: dbar_acquire_amswap_i32_release_dbar_acquire:
+; NOIASM: # %bb.0: # %entry
+; NOIASM-NEXT: amswap_db.w $a0, $zero, $zero
+; NOIASM-NEXT: ret
+;
+; BTONOP-LABEL: dbar_acquire_amswap_i32_release_dbar_acquire:
+; BTONOP: # %bb.0: # %entry
+; BTONOP-NEXT: nop
+; BTONOP-NEXT: amswap_db.w $a0, $zero, $zero
+; BTONOP-NEXT: nop
+; BTONOP-NEXT: ret
entry:
fence acquire
atomicrmw xchg ptr null, i32 0 release
@@ -301,13 +717,42 @@ entry:
}
define void @dbar_acquire_asm_amswap_i32_release() nounwind {
-; CHECK-LABEL: dbar_acquire_asm_amswap_i32_release:
-; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: dbar 20
-; CHECK-NEXT: #APP
-; CHECK-NEXT: amswap_db.w $a0, $a1, $a2
-; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: ret
+; NORMAL-LABEL: dbar_acquire_asm_amswap_i32_release:
+; NORMAL: # %bb.0: # %entry
+; NORMAL-NEXT: #APP
+; NORMAL-NEXT: amswap_db.w $a0, $a1, $a2
+; NORMAL-NEXT: #NO_APP
+; NORMAL-NEXT: ret
+;
+; BYPASS-LABEL: dbar_acquire_asm_amswap_i32_release:
+; BYPASS: # %bb.0: # %entry
+; BYPASS-NEXT: #APP
+; BYPASS-NEXT: amswap_db.w $a0, $a1, $a2
+; BYPASS-NEXT: #NO_APP
+; BYPASS-NEXT: ret
+;
+; NOMAMO-LABEL: dbar_acquire_asm_amswap_i32_release:
+; NOMAMO: # %bb.0: # %entry
+; NOMAMO-NEXT: #APP
+; NOMAMO-NEXT: amswap_db.w $a0, $a1, $a2
+; NOMAMO-NEXT: #NO_APP
+; NOMAMO-NEXT: ret
+;
+; NOIASM-LABEL: dbar_acquire_asm_amswap_i32_release:
+; NOIASM: # %bb.0: # %entry
+; NOIASM-NEXT: dbar 20
+; NOIASM-NEXT: #APP
+; NOIASM-NEXT: amswap_db.w $a0, $a1, $a2
+; NOIASM-NEXT: #NO_APP
+; NOIASM-NEXT: ret
+;
+; BTONOP-LABEL: dbar_acquire_asm_amswap_i32_release:
+; BTONOP: # %bb.0: # %entry
+; BTONOP-NEXT: nop
+; BTONOP-NEXT: #APP
+; BTONOP-NEXT: amswap_db.w $a0, $a1, $a2
+; BTONOP-NEXT: #NO_APP
+; BTONOP-NEXT: ret
entry:
fence acquire
call void asm "amswap_db.w $$r4, $$r5, $$r6", ""()
@@ -315,13 +760,43 @@ entry:
}
define void @dbar_acquire_asm_amswap_i32_monotonic() nounwind {
-; CHECK-LABEL: dbar_acquire_asm_amswap_i32_monotonic:
-; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: dbar 20
-; CHECK-NEXT: #APP
-; CHECK-NEXT: amswap.w $a0, $a1, $a2
-; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: ret
+; NORMAL-LABEL: dbar_acquire_asm_amswap_i32_monotonic:
+; NORMAL: # %bb.0: # %entry
+; NORMAL-NEXT: #APP
+; NORMAL-NEXT: amswap_db.w $a0, $a1, $a2
+; NORMAL-NEXT: #NO_APP
+; NORMAL-NEXT: ret
+;
+; BYPASS-LABEL: dbar_acquire_asm_amswap_i32_monotonic:
+; BYPASS: # %bb.0: # %entry
+; BYPASS-NEXT: #APP
+; BYPASS-NEXT: amswap_db.w $a0, $a1, $a2
+; BYPASS-NEXT: #NO_APP
+; BYPASS-NEXT: ret
+;
+; NOMAMO-LABEL: dbar_acquire_asm_amswap_i32_monotonic:
+; NOMAMO: # %bb.0: # %entry
+; NOMAMO-NEXT: dbar 20
+; NOMAMO-NEXT: #APP
+; NOMAMO-NEXT: amswap.w $a0, $a1, $a2
+; NOMAMO-NEXT: #NO_APP
+; NOMAMO-NEXT: ret
+;
+; NOIASM-LABEL: dbar_acquire_asm_amswap_i32_monotonic:
+; NOIASM: # %bb.0: # %entry
+; NOIASM-NEXT: dbar 20
+; NOIASM-NEXT: #APP
+; NOIASM-NEXT: amswap.w $a0, $a1, $a2
+; NOIASM-NEXT: #NO_APP
+; NOIASM-NEXT: ret
+;
+; BTONOP-LABEL: dbar_acquire_asm_amswap_i32_monotonic:
+; BTONOP: # %bb.0: # %entry
+; BTONOP-NEXT: nop
+; BTONOP-NEXT: #APP
+; BTONOP-NEXT: amswap_db.w $a0, $a1, $a2
+; BTONOP-NEXT: #NO_APP
+; BTONOP-NEXT: ret
entry:
fence acquire
call void asm sideeffect "amswap.w $$r4, $$r5, $$r6", ""()
@@ -329,13 +804,43 @@ entry:
}
define void @dbar_acquire_asm_space_amswap_i32_monotonic() nounwind {
-; CHECK-LABEL: dbar_acquire_asm_space_amswap_i32_monotonic:
-; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: dbar 20
-; CHECK-NEXT: #APP
-; CHECK-NEXT: amswap.w $a0, $a1, $a2
-; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: ret
+; NORMAL-LABEL: dbar_acquire_asm_space_amswap_i32_monotonic:
+; NORMAL: # %bb.0: # %entry
+; NORMAL-NEXT: #APP
+; NORMAL-NEXT: amswap_db.w $a0, $a1, $a2
+; NORMAL-NEXT: #NO_APP
+; NORMAL-NEXT: ret
+;
+; BYPASS-LABEL: dbar_acquire_asm_space_amswap_i32_monotonic:
+; BYPASS: # %bb.0: # %entry
+; BYPASS-NEXT: #APP
+; BYPASS-NEXT: amswap_db.w $a0, $a1, $a2
+; BYPASS-NEXT: #NO_APP
+; BYPASS-NEXT: ret
+;
+; NOMAMO-LABEL: dbar_acquire_asm_space_amswap_i32_monotonic:
+; NOMAMO: # %bb.0: # %entry
+; NOMAMO-NEXT: dbar 20
+; NOMAMO-NEXT: #APP
+; NOMAMO-NEXT: amswap.w $a0, $a1, $a2
+; NOMAMO-NEXT: #NO_APP
+; NOMAMO-NEXT: ret
+;
+; NOIASM-LABEL: dbar_acquire_asm_space_amswap_i32_monotonic:
+; NOIASM: # %bb.0: # %entry
+; NOIASM-NEXT: dbar 20
+; NOIASM-NEXT: #APP
+; NOIASM-NEXT: amswap.w $a0, $a1, $a2
+; NOIASM-NEXT: #NO_APP
+; NOIASM-NEXT: ret
+;
+; BTONOP-LABEL: dbar_acquire_asm_space_amswap_i32_monotonic:
+; BTONOP: # %bb.0: # %entry
+; BTONOP-NEXT: nop
+; BTONOP-NEXT: #APP
+; BTONOP-NEXT: amswap_db.w $a0, $a1, $a2
+; BTONOP-NEXT: #NO_APP
+; BTONOP-NEXT: ret
entry:
fence acquire
call void asm sideeffect " amswap.w $$r4, $$r5, $$r6", ""()
@@ -343,13 +848,43 @@ entry:
}
define void @dbar_acquire_asm_comment_amswap_i32_monotonic() nounwind {
-; CHECK-LABEL: dbar_acquire_asm_comment_amswap_i32_monotonic:
-; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: dbar 20
-; CHECK-NEXT: #APP
-; CHECK-NEXT: amswap.w $a0, $a1, $a2 # atomic
-; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: ret
+; NORMAL-LABEL: dbar_acquire_asm_comment_amswap_i32_monotonic:
+; NORMAL: # %bb.0: # %entry
+; NORMAL-NEXT: #APP
+; NORMAL-NEXT: amswap_db.w $a0, $a1, $a2 # atomic
+; NORMAL-NEXT: #NO_APP
+; NORMAL-NEXT: ret
+;
+; BYPASS-LABEL: dbar_acquire_asm_comment_amswap_i32_monotonic:
+; BYPASS: # %bb.0: # %entry
+; BYPASS-NEXT: #APP
+; BYPASS-NEXT: amswap_db.w $a0, $a1, $a2 # atomic
+; BYPASS-NEXT: #NO_APP
+; BYPASS-NEXT: ret
+;
+; NOMAMO-LABEL: dbar_acquire_asm_comment_amswap_i32_monotonic:
+; NOMAMO: # %bb.0: # %entry
+; NOMAMO-NEXT: dbar 20
+; NOMAMO-NEXT: #APP
+; NOMAMO-NEXT: amswap.w $a0, $a1, $a2 # atomic
+; NOMAMO-NEXT: #NO_APP
+; NOMAMO-NEXT: ret
+;
+; NOIASM-LABEL: dbar_acquire_asm_comment_amswap_i32_monotonic:
+; NOIASM: # %bb.0: # %entry
+; NOIASM-NEXT: dbar 20
+; NOIASM-NEXT: #APP
+; NOIASM-NEXT: amswap.w $a0, $a1, $a2 # atomic
+; NOIASM-NEXT: #NO_APP
+; NOIASM-NEXT: ret
+;
+; BTONOP-LABEL: dbar_acquire_asm_comment_amswap_i32_monotonic:
+; BTONOP: # %bb.0: # %entry
+; BTONOP-NEXT: nop
+; BTONOP-NEXT: #APP
+; BTONOP-NEXT: amswap_db.w $a0, $a1, $a2 # atomic
+; BTONOP-NEXT: #NO_APP
+; BTONOP-NEXT: ret
entry:
fence acquire
call void asm sideeffect "amswap.w $$r4, $$r5, $$r6 # atomic", ""()
@@ -357,13 +892,43 @@ entry:
}
define void @dbar_acquire_asm_args_amswap_i32_monotonic() nounwind {
-; CHECK-LABEL: dbar_acquire_asm_args_amswap_i32_monotonic:
-; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: dbar 20
-; CHECK-NEXT: #APP
-; CHECK-NEXT: amswap.w $a0, $zero, $zero
-; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: ret
+; NORMAL-LABEL: dbar_acquire_asm_args_amswap_i32_monotonic:
+; NORMAL: # %bb.0: # %entry
+; NORMAL-NEXT: #APP
+; NORMAL-NEXT: amswap_db.w $a0, $zero, $zero
+; NORMAL-NEXT: #NO_APP
+; NORMAL-NEXT: ret
+;
+; BYPASS-LABEL: dbar_acquire_asm_args_amswap_i32_monotonic:
+; BYPASS: # %bb.0: # %entry
+; BYPASS-NEXT: #APP
+; BYPASS-NEXT: amswap_db.w $a0, $zero, $zero
+; BYPASS-NEXT: #NO_APP
+; BYPASS-NEXT: ret
+;
+; NOMAMO-LABEL: dbar_acquire_asm_args_amswap_i32_monotonic:
+; NOMAMO: # %bb.0: # %entry
+; NOMAMO-NEXT: dbar 20
+; NOMAMO-NEXT: #APP
+; NOMAMO-NEXT: amswap.w $a0, $zero, $zero
+; NOMAMO-NEXT: #NO_APP
+; NOMAMO-NEXT: ret
+;
+; NOIASM-LABEL: dbar_acquire_asm_args_amswap_i32_monotonic:
+; NOIASM: # %bb.0: # %entry
+; NOIASM-NEXT: dbar 20
+; NOIASM-NEXT: #APP
+; NOIASM-NEXT: amswap.w $a0, $zero, $zero
+; NOIASM-NEXT: #NO_APP
+; NOIASM-NEXT: ret
+;
+; BTONOP-LABEL: dbar_acquire_asm_args_amswap_i32_monotonic:
+; BTONOP: # %bb.0: # %entry
+; BTONOP-NEXT: nop
+; BTONOP-NEXT: #APP
+; BTONOP-NEXT: amswap_db.w $a0, $zero, $zero
+; BTONOP-NEXT: #NO_APP
+; BTONOP-NEXT: ret
entry:
fence acquire
call i32 asm sideeffect "amswap.w $0, $1, $2", "=r,r,r,~{memory}"(i32 0, i32 0)
@@ -371,13 +936,42 @@ entry:
}
define void @dbar_acquire_asm_reg_args_amswap_i32_monotonic() nounwind {
-; CHECK-LABEL: dbar_acquire_asm_reg_args_amswap_i32_monotonic:
-; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: dbar 20
-; CHECK-NEXT: #APP
-; CHECK-NEXT: amswap_db.w $ra, $zero, $zero
-; CHECK-NEXT: #NO_APP
-; CHECK-NEXT: ret
+; NORMAL-LABEL: dbar_acquire_asm_reg_args_amswap_i32_monotonic:
+; NORMAL: # %bb.0: # %entry
+; NORMAL-NEXT: #APP
+; NORMAL-NEXT: amswap_db.w $ra, $zero, $zero
+; NORMAL-NEXT: #NO_APP
+; NORMAL-NEXT: ret
+;
+; BYPASS-LABEL: dbar_acquire_asm_reg_args_amswap_i32_monotonic:
+; BYPASS: # %bb.0: # %entry
+; BYPASS-NEXT: #APP
+; BYPASS-NEXT: amswap_db.w $ra, $zero, $zero
+; BYPASS-NEXT: #NO_APP
+; BYPASS-NEXT: ret
+;
+; NOMAMO-LABEL: dbar_acquire_asm_reg_args_amswap_i32_monotonic:
+; NOMAMO: # %bb.0: # %entry
+; NOMAMO-NEXT: #APP
+; NOMAMO-NEXT: amswap_db.w $ra, $zero, $zero
+; NOMAMO-NEXT: #NO_APP
+; NOMAMO-NEXT: ret
+;
+; NOIASM-LABEL: dbar_acquire_asm_reg_args_amswap_i32_monotonic:
+; NOIASM: # %bb.0: # %entry
+; NOIASM-NEXT: dbar 20
+; NOIASM-NEXT: #APP
+; NOIASM-NEXT: amswap_db.w $ra, $zero, $zero
+; NOIASM-NEXT: #NO_APP
+; NOIASM-NEXT: ret
+;
+; BTONOP-LABEL: dbar_acquire_asm_reg_args_amswap_i32_monotonic:
+; BTONOP: # %bb.0: # %entry
+; BTONOP-NEXT: nop
+; BTONOP-NEXT: #APP
+; BTONOP-NEXT: amswap_db.w $ra, $zero, $zero
+; BTONOP-NEXT: #NO_APP
+; BTONOP-NEXT: ret
entry:
fence acquire
call void asm sideeffect "amswap_db.w $$r1, $0, $1", "r,r,~{memory}"(i32 0, i32 0)
@@ -385,25 +979,100 @@ entry:
}
define void @dbar_acquire_cmpxchg_i32_dbar_release() nounwind {
-; CHECK-LABEL: dbar_acquire_cmpxchg_i32_dbar_release:
-; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: dbar 20
-; CHECK-NEXT: ori $a0, $zero, 1
-; CHECK-NEXT: .LBB28_1: # %entry
-; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ll.w $a1, $zero, 0
-; CHECK-NEXT: bne $a1, $zero, .LBB28_3
-; CHECK-NEXT: # %bb.2: # %entry
-; CHECK-NEXT: # in Loop: Header=BB28_1 Depth=1
-; CHECK-NEXT: move $a2, $a0
-; CHECK-NEXT: sc.w $a2, $zero, 0
-; CHECK-NEXT: beq $a2, $zero, .LBB28_1
-; CHECK-NEXT: b .LBB28_4
-; CHECK-NEXT: .LBB28_3: # %entry
-; CHECK-NEXT: dbar 20
-; CHECK-NEXT: .LBB28_4: # %entry
-; CHECK-NEXT: dbar 18
-; CHECK-NEXT: ret
+; NORMAL-LABEL: dbar_acquire_cmpxchg_i32_dbar_release:
+; NORMAL: # %bb.0: # %entry
+; NORMAL-NEXT: ori $a0, $zero, 1
+; NORMAL-NEXT: .LBB28_1: # %entry
+; NORMAL-NEXT: # =>This Inner Loop Header: Depth=1
+; NORMAL-NEXT: ll.w $a1, $zero, 0
+; NORMAL-NEXT: bne $a1, $zero, .LBB28_3
+; NORMAL-NEXT: # %bb.2: # %entry
+; NORMAL-NEXT: # in Loop: Header=BB28_1 Depth=1
+; NORMAL-NEXT: move $a2, $a0
+; NORMAL-NEXT: sc.w $a2, $zero, 0
+; NORMAL-NEXT: beq $a2, $zero, .LBB28_1
+; NORMAL-NEXT: b .LBB28_4
+; NORMAL-NEXT: .LBB28_3: # %entry
+; NORMAL-NEXT: dbar 20
+; NORMAL-NEXT: .LBB28_4: # %entry
+; NORMAL-NEXT: dbar 18
+; NORMAL-NEXT: ret
+;
+; BYPASS-LABEL: dbar_acquire_cmpxchg_i32_dbar_release:
+; BYPASS: # %bb.0: # %entry
+; BYPASS-NEXT: ori $a0, $zero, 1
+; BYPASS-NEXT: .LBB28_1: # %entry
+; BYPASS-NEXT: # =>This Inner Loop Header: Depth=1
+; BYPASS-NEXT: ll.w $a1, $zero, 0
+; BYPASS-NEXT: bne $a1, $zero, .LBB28_3
+; BYPASS-NEXT: # %bb.2: # %entry
+; BYPASS-NEXT: # in Loop: Header=BB28_1 Depth=1
+; BYPASS-NEXT: move $a2, $a0
+; BYPASS-NEXT: sc.w $a2, $zero, 0
+; BYPASS-NEXT: beq $a2, $zero, .LBB28_1
+; BYPASS-NEXT: b .LBB28_4
+; BYPASS-NEXT: .LBB28_3: # %entry
+; BYPASS-NEXT: .LBB28_4: # %entry
+; BYPASS-NEXT: dbar 16
+; BYPASS-NEXT: ret
+;
+; NOMAMO-LABEL: dbar_acquire_cmpxchg_i32_dbar_release:
+; NOMAMO: # %bb.0: # %entry
+; NOMAMO-NEXT: ori $a0, $zero, 1
+; NOMAMO-NEXT: .LBB28_1: # %entry
+; NOMAMO-NEXT: # =>This Inner Loop Header: Depth=1
+; NOMAMO-NEXT: ll.w $a1, $zero, 0
+; NOMAMO-NEXT: bne $a1, $zero, .LBB28_3
+; NOMAMO-NEXT: # %bb.2: # %entry
+; NOMAMO-NEXT: # in Loop: Header=BB28_1 Depth=1
+; NOMAMO-NEXT: move $a2, $a0
+; NOMAMO-NEXT: sc.w $a2, $zero, 0
+; NOMAMO-NEXT: beq $a2, $zero, .LBB28_1
+; NOMAMO-NEXT: b .LBB28_4
+; NOMAMO-NEXT: .LBB28_3: # %entry
+; NOMAMO-NEXT: dbar 20
+; NOMAMO-NEXT: .LBB28_4: # %entry
+; NOMAMO-NEXT: dbar 18
+; NOMAMO-NEXT: ret
+;
+; NOIASM-LABEL: dbar_acquire_cmpxchg_i32_dbar_release:
+; NOIASM: # %bb.0: # %entry
+; NOIASM-NEXT: ori $a0, $zero, 1
+; NOIASM-NEXT: .LBB28_1: # %entry
+; NOIASM-NEXT: # =>This Inner Loop Header: Depth=1
+; NOIASM-NEXT: ll.w $a1, $zero, 0
+; NOIASM-NEXT: bne $a1, $zero, .LBB28_3
+; NOIASM-NEXT: # %bb.2: # %entry
+; NOIASM-NEXT: # in Loop: Header=BB28_1 Depth=1
+; NOIASM-NEXT: move $a2, $a0
+; NOIASM-NEXT: sc.w $a2, $zero, 0
+; NOIASM-NEXT: beq $a2, $zero, .LBB28_1
+; NOIASM-NEXT: b .LBB28_4
+; NOIASM-NEXT: .LBB28_3: # %entry
+; NOIASM-NEXT: dbar 20
+; NOIASM-NEXT: .LBB28_4: # %entry
+; NOIASM-NEXT: dbar 18
+; NOIASM-NEXT: ret
+;
+; BTONOP-LABEL: dbar_acquire_cmpxchg_i32_dbar_release:
+; BTONOP: # %bb.0: # %entry
+; BTONOP-NEXT: nop
+; BTONOP-NEXT: ori $a0, $zero, 1
+; BTONOP-NEXT: .LBB28_1: # %entry
+; BTONOP-NEXT: # =>This Inner Loop Header: Depth=1
+; BTONOP-NEXT: ll.w $a1, $zero, 0
+; BTONOP-NEXT: bne $a1, $zero, .LBB28_3
+; BTONOP-NEXT: # %bb.2: # %entry
+; BTONOP-NEXT: # in Loop: Header=BB28_1 Depth=1
+; BTONOP-NEXT: move $a2, $a0
+; BTONOP-NEXT: sc.w $a2, $zero, 0
+; BTONOP-NEXT: beq $a2, $zero, .LBB28_1
+; BTONOP-NEXT: b .LBB28_4
+; BTONOP-NEXT: .LBB28_3: # %entry
+; BTONOP-NEXT: dbar 20
+; BTONOP-NEXT: .LBB28_4: # %entry
+; BTONOP-NEXT: dbar 18
+; BTONOP-NEXT: ret
entry:
fence acquire
cmpxchg ptr null, i32 0, i32 1 acquire acquire
@@ -412,17 +1081,60 @@ entry:
}
define void @cfg_dominate(i1 %a) {
-; CHECK-LABEL: cfg_dominate:
-; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: andi $a0, $a0, 1
-; CHECK-NEXT: dbar 20
-; CHECK-NEXT: beqz $a0, .LBB29_2
-; CHECK-NEXT: # %bb.1:
-; CHECK-NEXT: dbar 20
-; CHECK-NEXT: ret
-; CHECK-NEXT: .LBB29_2:
-; CHECK-NEXT: dbar 18
-; CHECK-NEXT: ret
+; NORMAL-LABEL: cfg_dominate:
+; NORMAL: # %bb.0: # %entry
+; NORMAL-NEXT: andi $a0, $a0, 1
+; NORMAL-NEXT: dbar 20
+; NORMAL-NEXT: beqz $a0, .LBB29_2
+; NORMAL-NEXT: # %bb.1:
+; NORMAL-NEXT: ret
+; NORMAL-NEXT: .LBB29_2:
+; NORMAL-NEXT: dbar 18
+; NORMAL-NEXT: ret
+;
+; BYPASS-LABEL: cfg_dominate:
+; BYPASS: # %bb.0: # %entry
+; BYPASS-NEXT: andi $a0, $a0, 1
+; BYPASS-NEXT: dbar 16
+; BYPASS-NEXT: beqz $a0, .LBB29_2
+; BYPASS-NEXT: # %bb.1:
+; BYPASS-NEXT: ret
+; BYPASS-NEXT: .LBB29_2:
+; BYPASS-NEXT: ret
+;
+; NOMAMO-LABEL: cfg_dominate:
+; NOMAMO: # %bb.0: # %entry
+; NOMAMO-NEXT: andi $a0, $a0, 1
+; NOMAMO-NEXT: dbar 20
+; NOMAMO-NEXT: beqz $a0, .LBB29_2
+; NOMAMO-NEXT: # %bb.1:
+; NOMAMO-NEXT: ret
+; NOMAMO-NEXT: .LBB29_2:
+; NOMAMO-NEXT: dbar 18
+; NOMAMO-NEXT: ret
+;
+; NOIASM-LABEL: cfg_dominate:
+; NOIASM: # %bb.0: # %entry
+; NOIASM-NEXT: andi $a0, $a0, 1
+; NOIASM-NEXT: dbar 20
+; NOIASM-NEXT: beqz $a0, .LBB29_2
+; NOIASM-NEXT: # %bb.1:
+; NOIASM-NEXT: ret
+; NOIASM-NEXT: .LBB29_2:
+; NOIASM-NEXT: dbar 18
+; NOIASM-NEXT: ret
+;
+; BTONOP-LABEL: cfg_dominate:
+; BTONOP: # %bb.0: # %entry
+; BTONOP-NEXT: andi $a0, $a0, 1
+; BTONOP-NEXT: dbar 20
+; BTONOP-NEXT: beqz $a0, .LBB29_2
+; BTONOP-NEXT: # %bb.1:
+; BTONOP-NEXT: nop
+; BTONOP-NEXT: ret
+; BTONOP-NEXT: .LBB29_2:
+; BTONOP-NEXT: dbar 18
+; BTONOP-NEXT: ret
entry:
fence acquire
br i1 %a, label %2, label %3
@@ -435,21 +1147,80 @@ entry:
}
define void @cfg_postdominate(i1 %a, i1 %b, i1 %c) {
-; CHECK-LABEL: cfg_postdominate:
-; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: andi $a0, $a0, 1
-; CHECK-NEXT: beqz $a0, .LBB30_2
-; CHECK-NEXT: .LBB30_1:
-; CHECK-NEXT: andi $a0, $a1, 1
-; CHECK-NEXT: dbar 20
-; CHECK-NEXT: beqz $a0, .LBB30_3
-; CHECK-NEXT: .LBB30_2:
-; CHECK-NEXT: andi $a0, $a2, 1
-; CHECK-NEXT: dbar 18
-; CHECK-NEXT: bnez $a0, .LBB30_1
-; CHECK-NEXT: .LBB30_3:
-; CHECK-NEXT: dbar 18
-; CHECK-NEXT: ret
+; NORMAL-LABEL: cfg_postdominate:
+; NORMAL: # %bb.0: # %entry
+; NORMAL-NEXT: andi $a0, $a0, 1
+; NORMAL-NEXT: beqz $a0, .LBB30_2
+; NORMAL-NEXT: .LBB30_1:
+; NORMAL-NEXT: andi $a0, $a1, 1
+; NORMAL-NEXT: dbar 20
+; NORMAL-NEXT: beqz $a0, .LBB30_3
+; NORMAL-NEXT: .LBB30_2:
+; NORMAL-NEXT: andi $a0, $a2, 1
+; NORMAL-NEXT: bnez $a0, .LBB30_1
+; NORMAL-NEXT: .LBB30_3:
+; NORMAL-NEXT: dbar 18
+; NORMAL-NEXT: ret
+;
+; BYPASS-LABEL: cfg_postdominate:
+; BYPASS: # %bb.0: # %entry
+; BYPASS-NEXT: andi $a0, $a0, 1
+; BYPASS-NEXT: beqz $a0, .LBB30_2
+; BYPASS-NEXT: .LBB30_1:
+; BYPASS-NEXT: andi $a0, $a1, 1
+; BYPASS-NEXT: beqz $a0, .LBB30_3
+; BYPASS-NEXT: .LBB30_2:
+; BYPASS-NEXT: andi $a0, $a2, 1
+; BYPASS-NEXT: bnez $a0, .LBB30_1
+; BYPASS-NEXT: .LBB30_3:
+; BYPASS-NEXT: dbar 16
+; BYPASS-NEXT: ret
+;
+; NOMAMO-LABEL: cfg_postdominate:
+; NOMAMO: # %bb.0: # %entry
+; NOMAMO-NEXT: andi $a0, $a0, 1
+; NOMAMO-NEXT: beqz $a0, .LBB30_2
+; NOMAMO-NEXT: .LBB30_1:
+; NOMAMO-NEXT: andi $a0, $a1, 1
+; NOMAMO-NEXT: dbar 20
+; NOMAMO-NEXT: beqz $a0, .LBB30_3
+; NOMAMO-NEXT: .LBB30_2:
+; NOMAMO-NEXT: andi $a0, $a2, 1
+; NOMAMO-NEXT: bnez $a0, .LBB30_1
+; NOMAMO-NEXT: .LBB30_3:
+; NOMAMO-NEXT: dbar 18
+; NOMAMO-NEXT: ret
+;
+; NOIASM-LABEL: cfg_postdominate:
+; NOIASM: # %bb.0: # %entry
+; NOIASM-NEXT: andi $a0, $a0, 1
+; NOIASM-NEXT: beqz $a0, .LBB30_2
+; NOIASM-NEXT: .LBB30_1:
+; NOIASM-NEXT: andi $a0, $a1, 1
+; NOIASM-NEXT: dbar 20
+; NOIASM-NEXT: beqz $a0, .LBB30_3
+; NOIASM-NEXT: .LBB30_2:
+; NOIASM-NEXT: andi $a0, $a2, 1
+; NOIASM-NEXT: bnez $a0, .LBB30_1
+; NOIASM-NEXT: .LBB30_3:
+; NOIASM-NEXT: dbar 18
+; NOIASM-NEXT: ret
+;
+; BTONOP-LABEL: cfg_postdominate:
+; BTONOP: # %bb.0: # %entry
+; BTONOP-NEXT: andi $a0, $a0, 1
+; BTONOP-NEXT: beqz $a0, .LBB30_2
+; BTONOP-NEXT: .LBB30_1:
+; BTONOP-NEXT: andi $a0, $a1, 1
+; BTONOP-NEXT: dbar 20
+; BTONOP-NEXT: beqz $a0, .LBB30_3
+; BTONOP-NEXT: .LBB30_2:
+; BTONOP-NEXT: andi $a0, $a2, 1
+; BTONOP-NEXT: nop
+; BTONOP-NEXT: bnez $a0, .LBB30_1
+; BTONOP-NEXT: .LBB30_3:
+; BTONOP-NEXT: dbar 18
+; BTONOP-NEXT: ret
entry:
br i1 %a, label %2, label %3
2:
diff --git a/llvm/test/CodeGen/LoongArch/opt-pipeline.ll b/llvm/test/CodeGen/LoongArch/opt-pipeline.ll
index 3bbd6676181d0..2da43df50041b 100644
--- a/llvm/test/CodeGen/LoongArch/opt-pipeline.ll
+++ b/llvm/test/CodeGen/LoongArch/opt-pipeline.ll
@@ -184,6 +184,9 @@
; LAXX-NEXT: Stack Frame Layout Analysis
; LAXX-NEXT: LoongArch pseudo instruction expansion pass
; LAXX-NEXT: LoongArch atomic pseudo instruction expansion pass
+; LAXX-NEXT: MachineDominator Tree Construction
+; LAXX-NEXT: MachinePostDominator Tree Construction
+; LAXX-NEXT: LoongArch Memory Barrier Optimisation pass
; LAXX-NEXT: Lazy Machine Block Frequency Analysis
; LAXX-NEXT: Machine Optimization Remark Emitter
; LAXX-NEXT: LoongArch Assembly Printer
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/LoongArch/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/LoongArch/BUILD.gn
index d68ccd8cbee73..ce2f84eea690b 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/LoongArch/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/LoongArch/BUILD.gn
@@ -50,6 +50,7 @@ static_library("LLVMLoongArchCodeGen") {
"LoongArchInstrInfo.cpp",
"LoongArchLateBranchOpt.cpp",
"LoongArchMCInstLower.cpp",
+ "LoongArchMemoryBarrierOpt.cpp",
"LoongArchMergeBaseOffset.cpp",
"LoongArchOptWInstrs.cpp",
"LoongArchRegisterInfo.cpp",
More information about the llvm-commits
mailing list