[llvm] 2cb0644 - [RISCV] Split the pseudo instruction splitting pass
Luís Marques via llvm-commits
llvm-commits at lists.llvm.org
Mon Jun 29 06:36:41 PDT 2020
Author: Luís Marques
Date: 2020-06-29T14:35:57+01:00
New Revision: 2cb0644f90b7deda78fd5a4df4b74e25bd1db701
URL: https://github.com/llvm/llvm-project/commit/2cb0644f90b7deda78fd5a4df4b74e25bd1db701
DIFF: https://github.com/llvm/llvm-project/commit/2cb0644f90b7deda78fd5a4df4b74e25bd1db701.diff
LOG: [RISCV] Split the pseudo instruction splitting pass
Extracts the atomic pseudo-instructions' splitting from `riscv-expand-pseudo`
/ `RISCVExpandPseudo` into its own pass, `riscv-expand-atomic-pseudo` /
`RISCVExpandAtomicPseudo`. This allows for the expansion of atomic operations
to continue to happen late (the new pass is added in `addPreEmitPass2`, so
those expansions continue to happen in the same place), while the remaining
pseudo-instructions can now be expanded earlier and benefit from more
optimization passes. The nonatomics pass is now added in `addPreSched2`.
Differential Revision: https://reviews.llvm.org/D79635
Added:
llvm/lib/Target/RISCV/RISCVExpandAtomicPseudoInsts.cpp
Modified:
llvm/lib/Target/RISCV/CMakeLists.txt
llvm/lib/Target/RISCV/RISCV.h
llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp
llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
llvm/test/CodeGen/RISCV/codemodel-lowering.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/RISCV/CMakeLists.txt b/llvm/lib/Target/RISCV/CMakeLists.txt
index 31a82be19812..9a0c2209c885 100644
--- a/llvm/lib/Target/RISCV/CMakeLists.txt
+++ b/llvm/lib/Target/RISCV/CMakeLists.txt
@@ -19,6 +19,7 @@ add_public_tablegen_target(RISCVCommonTableGen)
add_llvm_target(RISCVCodeGen
RISCVAsmPrinter.cpp
RISCVCallLowering.cpp
+ RISCVExpandAtomicPseudoInsts.cpp
RISCVExpandPseudoInsts.cpp
RISCVFrameLowering.cpp
RISCVInstrInfo.cpp
diff --git a/llvm/lib/Target/RISCV/RISCV.h b/llvm/lib/Target/RISCV/RISCV.h
index f23f742a4782..9baa2cc2741a 100644
--- a/llvm/lib/Target/RISCV/RISCV.h
+++ b/llvm/lib/Target/RISCV/RISCV.h
@@ -43,6 +43,9 @@ void initializeRISCVMergeBaseOffsetOptPass(PassRegistry &);
FunctionPass *createRISCVExpandPseudoPass();
void initializeRISCVExpandPseudoPass(PassRegistry &);
+FunctionPass *createRISCVExpandAtomicPseudoPass();
+void initializeRISCVExpandAtomicPseudoPass(PassRegistry &);
+
InstructionSelector *createRISCVInstructionSelector(const RISCVTargetMachine &,
RISCVSubtarget &,
RISCVRegisterBankInfo &);
diff --git a/llvm/lib/Target/RISCV/RISCVExpandAtomicPseudoInsts.cpp b/llvm/lib/Target/RISCV/RISCVExpandAtomicPseudoInsts.cpp
new file mode 100644
index 000000000000..b49c767ff2ca
--- /dev/null
+++ b/llvm/lib/Target/RISCV/RISCVExpandAtomicPseudoInsts.cpp
@@ -0,0 +1,615 @@
+//===-- RISCVExpandAtomicPseudoInsts.cpp - Expand atomic pseudo instrs. ---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains a pass that expands atomic pseudo instructions into
+// target instructions. This pass should be run at the last possible moment,
+// avoiding the possibility for other passes to break the requirements for
+// forward progress in the LR/SC block.
+//
+//===----------------------------------------------------------------------===//
+
+#include "RISCV.h"
+#include "RISCVInstrInfo.h"
+#include "RISCVTargetMachine.h"
+
+#include "llvm/CodeGen/LivePhysRegs.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+
+using namespace llvm;
+
+#define RISCV_EXPAND_ATOMIC_PSEUDO_NAME \
+ "RISCV atomic pseudo instruction expansion pass"
+
+namespace {
+
+class RISCVExpandAtomicPseudo : public MachineFunctionPass {
+public:
+ const RISCVInstrInfo *TII;
+ static char ID;
+
+ RISCVExpandAtomicPseudo() : MachineFunctionPass(ID) {
+ initializeRISCVExpandAtomicPseudoPass(*PassRegistry::getPassRegistry());
+ }
+
+ bool runOnMachineFunction(MachineFunction &MF) override;
+
+ StringRef getPassName() const override {
+ return RISCV_EXPAND_ATOMIC_PSEUDO_NAME;
+ }
+
+private:
+ bool expandMBB(MachineBasicBlock &MBB);
+ bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
+ MachineBasicBlock::iterator &NextMBBI);
+ bool expandAtomicBinOp(MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator MBBI, AtomicRMWInst::BinOp,
+ bool IsMasked, int Width,
+ MachineBasicBlock::iterator &NextMBBI);
+ bool expandAtomicMinMaxOp(MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator MBBI,
+ AtomicRMWInst::BinOp, bool IsMasked, int Width,
+ MachineBasicBlock::iterator &NextMBBI);
+ bool expandAtomicCmpXchg(MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator MBBI, bool IsMasked,
+ int Width, MachineBasicBlock::iterator &NextMBBI);
+};
+
+char RISCVExpandAtomicPseudo::ID = 0;
+
+bool RISCVExpandAtomicPseudo::runOnMachineFunction(MachineFunction &MF) {
+ TII = static_cast<const RISCVInstrInfo *>(MF.getSubtarget().getInstrInfo());
+ bool Modified = false;
+ for (auto &MBB : MF)
+ Modified |= expandMBB(MBB);
+ return Modified;
+}
+
+bool RISCVExpandAtomicPseudo::expandMBB(MachineBasicBlock &MBB) {
+ bool Modified = false;
+
+ MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
+ while (MBBI != E) {
+ MachineBasicBlock::iterator NMBBI = std::next(MBBI);
+ Modified |= expandMI(MBB, MBBI, NMBBI);
+ MBBI = NMBBI;
+ }
+
+ return Modified;
+}
+
+bool RISCVExpandAtomicPseudo::expandMI(MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator MBBI,
+ MachineBasicBlock::iterator &NextMBBI) {
+ switch (MBBI->getOpcode()) {
+ case RISCV::PseudoAtomicLoadNand32:
+ return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, false, 32,
+ NextMBBI);
+ case RISCV::PseudoAtomicLoadNand64:
+ return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, false, 64,
+ NextMBBI);
+ case RISCV::PseudoMaskedAtomicSwap32:
+ return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xchg, true, 32,
+ NextMBBI);
+ case RISCV::PseudoMaskedAtomicLoadAdd32:
+ return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Add, true, 32, NextMBBI);
+ case RISCV::PseudoMaskedAtomicLoadSub32:
+ return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Sub, true, 32, NextMBBI);
+ case RISCV::PseudoMaskedAtomicLoadNand32:
+ return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, true, 32,
+ NextMBBI);
+ case RISCV::PseudoMaskedAtomicLoadMax32:
+ return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Max, true, 32,
+ NextMBBI);
+ case RISCV::PseudoMaskedAtomicLoadMin32:
+ return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Min, true, 32,
+ NextMBBI);
+ case RISCV::PseudoMaskedAtomicLoadUMax32:
+ return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMax, true, 32,
+ NextMBBI);
+ case RISCV::PseudoMaskedAtomicLoadUMin32:
+ return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMin, true, 32,
+ NextMBBI);
+ case RISCV::PseudoCmpXchg32:
+ return expandAtomicCmpXchg(MBB, MBBI, false, 32, NextMBBI);
+ case RISCV::PseudoCmpXchg64:
+ return expandAtomicCmpXchg(MBB, MBBI, false, 64, NextMBBI);
+ case RISCV::PseudoMaskedCmpXchg32:
+ return expandAtomicCmpXchg(MBB, MBBI, true, 32, NextMBBI);
+ }
+
+ return false;
+}
+
+static unsigned getLRForRMW32(AtomicOrdering Ordering) {
+ switch (Ordering) {
+ default:
+ llvm_unreachable("Unexpected AtomicOrdering");
+ case AtomicOrdering::Monotonic:
+ return RISCV::LR_W;
+ case AtomicOrdering::Acquire:
+ return RISCV::LR_W_AQ;
+ case AtomicOrdering::Release:
+ return RISCV::LR_W;
+ case AtomicOrdering::AcquireRelease:
+ return RISCV::LR_W_AQ;
+ case AtomicOrdering::SequentiallyConsistent:
+ return RISCV::LR_W_AQ_RL;
+ }
+}
+
+static unsigned getSCForRMW32(AtomicOrdering Ordering) {
+ switch (Ordering) {
+ default:
+ llvm_unreachable("Unexpected AtomicOrdering");
+ case AtomicOrdering::Monotonic:
+ return RISCV::SC_W;
+ case AtomicOrdering::Acquire:
+ return RISCV::SC_W;
+ case AtomicOrdering::Release:
+ return RISCV::SC_W_RL;
+ case AtomicOrdering::AcquireRelease:
+ return RISCV::SC_W_RL;
+ case AtomicOrdering::SequentiallyConsistent:
+ return RISCV::SC_W_AQ_RL;
+ }
+}
+
+static unsigned getLRForRMW64(AtomicOrdering Ordering) {
+ switch (Ordering) {
+ default:
+ llvm_unreachable("Unexpected AtomicOrdering");
+ case AtomicOrdering::Monotonic:
+ return RISCV::LR_D;
+ case AtomicOrdering::Acquire:
+ return RISCV::LR_D_AQ;
+ case AtomicOrdering::Release:
+ return RISCV::LR_D;
+ case AtomicOrdering::AcquireRelease:
+ return RISCV::LR_D_AQ;
+ case AtomicOrdering::SequentiallyConsistent:
+ return RISCV::LR_D_AQ_RL;
+ }
+}
+
+static unsigned getSCForRMW64(AtomicOrdering Ordering) {
+ switch (Ordering) {
+ default:
+ llvm_unreachable("Unexpected AtomicOrdering");
+ case AtomicOrdering::Monotonic:
+ return RISCV::SC_D;
+ case AtomicOrdering::Acquire:
+ return RISCV::SC_D;
+ case AtomicOrdering::Release:
+ return RISCV::SC_D_RL;
+ case AtomicOrdering::AcquireRelease:
+ return RISCV::SC_D_RL;
+ case AtomicOrdering::SequentiallyConsistent:
+ return RISCV::SC_D_AQ_RL;
+ }
+}
+
+static unsigned getLRForRMW(AtomicOrdering Ordering, int Width) {
+ if (Width == 32)
+ return getLRForRMW32(Ordering);
+ if (Width == 64)
+ return getLRForRMW64(Ordering);
+ llvm_unreachable("Unexpected LR width\n");
+}
+
+static unsigned getSCForRMW(AtomicOrdering Ordering, int Width) {
+ if (Width == 32)
+ return getSCForRMW32(Ordering);
+ if (Width == 64)
+ return getSCForRMW64(Ordering);
+ llvm_unreachable("Unexpected SC width\n");
+}
+
+static void doAtomicBinOpExpansion(const RISCVInstrInfo *TII, MachineInstr &MI,
+ DebugLoc DL, MachineBasicBlock *ThisMBB,
+ MachineBasicBlock *LoopMBB,
+ MachineBasicBlock *DoneMBB,
+ AtomicRMWInst::BinOp BinOp, int Width) {
+ Register DestReg = MI.getOperand(0).getReg();
+ Register ScratchReg = MI.getOperand(1).getReg();
+ Register AddrReg = MI.getOperand(2).getReg();
+ Register IncrReg = MI.getOperand(3).getReg();
+ AtomicOrdering Ordering =
+ static_cast<AtomicOrdering>(MI.getOperand(4).getImm());
+
+ // .loop:
+ // lr.[w|d] dest, (addr)
+ // binop scratch, dest, val
+ // sc.[w|d] scratch, scratch, (addr)
+ // bnez scratch, loop
+ BuildMI(LoopMBB, DL, TII->get(getLRForRMW(Ordering, Width)), DestReg)
+ .addReg(AddrReg);
+ switch (BinOp) {
+ default:
+ llvm_unreachable("Unexpected AtomicRMW BinOp");
+ case AtomicRMWInst::Nand:
+ BuildMI(LoopMBB, DL, TII->get(RISCV::AND), ScratchReg)
+ .addReg(DestReg)
+ .addReg(IncrReg);
+ BuildMI(LoopMBB, DL, TII->get(RISCV::XORI), ScratchReg)
+ .addReg(ScratchReg)
+ .addImm(-1);
+ break;
+ }
+ BuildMI(LoopMBB, DL, TII->get(getSCForRMW(Ordering, Width)), ScratchReg)
+ .addReg(AddrReg)
+ .addReg(ScratchReg);
+ BuildMI(LoopMBB, DL, TII->get(RISCV::BNE))
+ .addReg(ScratchReg)
+ .addReg(RISCV::X0)
+ .addMBB(LoopMBB);
+}
+
+static void insertMaskedMerge(const RISCVInstrInfo *TII, DebugLoc DL,
+ MachineBasicBlock *MBB, Register DestReg,
+ Register OldValReg, Register NewValReg,
+ Register MaskReg, Register ScratchReg) {
+ assert(OldValReg != ScratchReg && "OldValReg and ScratchReg must be unique");
+ assert(OldValReg != MaskReg && "OldValReg and MaskReg must be unique");
+ assert(ScratchReg != MaskReg && "ScratchReg and MaskReg must be unique");
+
+ // We select bits from newval and oldval using:
+ // https://graphics.stanford.edu/~seander/bithacks.html#MaskedMerge
+ // r = oldval ^ ((oldval ^ newval) & masktargetdata);
+ BuildMI(MBB, DL, TII->get(RISCV::XOR), ScratchReg)
+ .addReg(OldValReg)
+ .addReg(NewValReg);
+ BuildMI(MBB, DL, TII->get(RISCV::AND), ScratchReg)
+ .addReg(ScratchReg)
+ .addReg(MaskReg);
+ BuildMI(MBB, DL, TII->get(RISCV::XOR), DestReg)
+ .addReg(OldValReg)
+ .addReg(ScratchReg);
+}
+
+static void doMaskedAtomicBinOpExpansion(
+ const RISCVInstrInfo *TII, MachineInstr &MI, DebugLoc DL,
+ MachineBasicBlock *ThisMBB, MachineBasicBlock *LoopMBB,
+ MachineBasicBlock *DoneMBB, AtomicRMWInst::BinOp BinOp, int Width) {
+ assert(Width == 32 && "Should never need to expand masked 64-bit operations");
+ Register DestReg = MI.getOperand(0).getReg();
+ Register ScratchReg = MI.getOperand(1).getReg();
+ Register AddrReg = MI.getOperand(2).getReg();
+ Register IncrReg = MI.getOperand(3).getReg();
+ Register MaskReg = MI.getOperand(4).getReg();
+ AtomicOrdering Ordering =
+ static_cast<AtomicOrdering>(MI.getOperand(5).getImm());
+
+ // .loop:
+ // lr.w destreg, (alignedaddr)
+ // binop scratch, destreg, incr
+ // xor scratch, destreg, scratch
+ // and scratch, scratch, masktargetdata
+ // xor scratch, destreg, scratch
+ // sc.w scratch, scratch, (alignedaddr)
+ // bnez scratch, loop
+ BuildMI(LoopMBB, DL, TII->get(getLRForRMW32(Ordering)), DestReg)
+ .addReg(AddrReg);
+ switch (BinOp) {
+ default:
+ llvm_unreachable("Unexpected AtomicRMW BinOp");
+ case AtomicRMWInst::Xchg:
+ BuildMI(LoopMBB, DL, TII->get(RISCV::ADDI), ScratchReg)
+ .addReg(IncrReg)
+ .addImm(0);
+ break;
+ case AtomicRMWInst::Add:
+ BuildMI(LoopMBB, DL, TII->get(RISCV::ADD), ScratchReg)
+ .addReg(DestReg)
+ .addReg(IncrReg);
+ break;
+ case AtomicRMWInst::Sub:
+ BuildMI(LoopMBB, DL, TII->get(RISCV::SUB), ScratchReg)
+ .addReg(DestReg)
+ .addReg(IncrReg);
+ break;
+ case AtomicRMWInst::Nand:
+ BuildMI(LoopMBB, DL, TII->get(RISCV::AND), ScratchReg)
+ .addReg(DestReg)
+ .addReg(IncrReg);
+ BuildMI(LoopMBB, DL, TII->get(RISCV::XORI), ScratchReg)
+ .addReg(ScratchReg)
+ .addImm(-1);
+ break;
+ }
+
+ insertMaskedMerge(TII, DL, LoopMBB, ScratchReg, DestReg, ScratchReg, MaskReg,
+ ScratchReg);
+
+ BuildMI(LoopMBB, DL, TII->get(getSCForRMW32(Ordering)), ScratchReg)
+ .addReg(AddrReg)
+ .addReg(ScratchReg);
+ BuildMI(LoopMBB, DL, TII->get(RISCV::BNE))
+ .addReg(ScratchReg)
+ .addReg(RISCV::X0)
+ .addMBB(LoopMBB);
+}
+
+bool RISCVExpandAtomicPseudo::expandAtomicBinOp(
+ MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
+ AtomicRMWInst::BinOp BinOp, bool IsMasked, int Width,
+ MachineBasicBlock::iterator &NextMBBI) {
+ MachineInstr &MI = *MBBI;
+ DebugLoc DL = MI.getDebugLoc();
+
+ MachineFunction *MF = MBB.getParent();
+ auto LoopMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
+ auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
+
+ // Insert new MBBs.
+ MF->insert(++MBB.getIterator(), LoopMBB);
+ MF->insert(++LoopMBB->getIterator(), DoneMBB);
+
+ // Set up successors and transfer remaining instructions to DoneMBB.
+ LoopMBB->addSuccessor(LoopMBB);
+ LoopMBB->addSuccessor(DoneMBB);
+ DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
+ DoneMBB->transferSuccessors(&MBB);
+ MBB.addSuccessor(LoopMBB);
+
+ if (!IsMasked)
+ doAtomicBinOpExpansion(TII, MI, DL, &MBB, LoopMBB, DoneMBB, BinOp, Width);
+ else
+ doMaskedAtomicBinOpExpansion(TII, MI, DL, &MBB, LoopMBB, DoneMBB, BinOp,
+ Width);
+
+ NextMBBI = MBB.end();
+ MI.eraseFromParent();
+
+ LivePhysRegs LiveRegs;
+ computeAndAddLiveIns(LiveRegs, *LoopMBB);
+ computeAndAddLiveIns(LiveRegs, *DoneMBB);
+
+ return true;
+}
+
+static void insertSext(const RISCVInstrInfo *TII, DebugLoc DL,
+ MachineBasicBlock *MBB, Register ValReg,
+ Register ShamtReg) {
+ BuildMI(MBB, DL, TII->get(RISCV::SLL), ValReg)
+ .addReg(ValReg)
+ .addReg(ShamtReg);
+ BuildMI(MBB, DL, TII->get(RISCV::SRA), ValReg)
+ .addReg(ValReg)
+ .addReg(ShamtReg);
+}
+
+bool RISCVExpandAtomicPseudo::expandAtomicMinMaxOp(
+ MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
+ AtomicRMWInst::BinOp BinOp, bool IsMasked, int Width,
+ MachineBasicBlock::iterator &NextMBBI) {
+ assert(IsMasked == true &&
+ "Should only need to expand masked atomic max/min");
+ assert(Width == 32 && "Should never need to expand masked 64-bit operations");
+
+ MachineInstr &MI = *MBBI;
+ DebugLoc DL = MI.getDebugLoc();
+ MachineFunction *MF = MBB.getParent();
+ auto LoopHeadMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
+ auto LoopIfBodyMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
+ auto LoopTailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
+ auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
+
+ // Insert new MBBs.
+ MF->insert(++MBB.getIterator(), LoopHeadMBB);
+ MF->insert(++LoopHeadMBB->getIterator(), LoopIfBodyMBB);
+ MF->insert(++LoopIfBodyMBB->getIterator(), LoopTailMBB);
+ MF->insert(++LoopTailMBB->getIterator(), DoneMBB);
+
+ // Set up successors and transfer remaining instructions to DoneMBB.
+ LoopHeadMBB->addSuccessor(LoopIfBodyMBB);
+ LoopHeadMBB->addSuccessor(LoopTailMBB);
+ LoopIfBodyMBB->addSuccessor(LoopTailMBB);
+ LoopTailMBB->addSuccessor(LoopHeadMBB);
+ LoopTailMBB->addSuccessor(DoneMBB);
+ DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
+ DoneMBB->transferSuccessors(&MBB);
+ MBB.addSuccessor(LoopHeadMBB);
+
+ Register DestReg = MI.getOperand(0).getReg();
+ Register Scratch1Reg = MI.getOperand(1).getReg();
+ Register Scratch2Reg = MI.getOperand(2).getReg();
+ Register AddrReg = MI.getOperand(3).getReg();
+ Register IncrReg = MI.getOperand(4).getReg();
+ Register MaskReg = MI.getOperand(5).getReg();
+ bool IsSigned = BinOp == AtomicRMWInst::Min || BinOp == AtomicRMWInst::Max;
+ AtomicOrdering Ordering =
+ static_cast<AtomicOrdering>(MI.getOperand(IsSigned ? 7 : 6).getImm());
+
+ //
+ // .loophead:
+ // lr.w destreg, (alignedaddr)
+ // and scratch2, destreg, mask
+ // mv scratch1, destreg
+ // [sext scratch2 if signed min/max]
+ // ifnochangeneeded scratch2, incr, .looptail
+ BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW32(Ordering)), DestReg)
+ .addReg(AddrReg);
+ BuildMI(LoopHeadMBB, DL, TII->get(RISCV::AND), Scratch2Reg)
+ .addReg(DestReg)
+ .addReg(MaskReg);
+ BuildMI(LoopHeadMBB, DL, TII->get(RISCV::ADDI), Scratch1Reg)
+ .addReg(DestReg)
+ .addImm(0);
+
+ switch (BinOp) {
+ default:
+ llvm_unreachable("Unexpected AtomicRMW BinOp");
+ case AtomicRMWInst::Max: {
+ insertSext(TII, DL, LoopHeadMBB, Scratch2Reg, MI.getOperand(6).getReg());
+ BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGE))
+ .addReg(Scratch2Reg)
+ .addReg(IncrReg)
+ .addMBB(LoopTailMBB);
+ break;
+ }
+ case AtomicRMWInst::Min: {
+ insertSext(TII, DL, LoopHeadMBB, Scratch2Reg, MI.getOperand(6).getReg());
+ BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGE))
+ .addReg(IncrReg)
+ .addReg(Scratch2Reg)
+ .addMBB(LoopTailMBB);
+ break;
+ }
+ case AtomicRMWInst::UMax:
+ BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGEU))
+ .addReg(Scratch2Reg)
+ .addReg(IncrReg)
+ .addMBB(LoopTailMBB);
+ break;
+ case AtomicRMWInst::UMin:
+ BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGEU))
+ .addReg(IncrReg)
+ .addReg(Scratch2Reg)
+ .addMBB(LoopTailMBB);
+ break;
+ }
+
+ // .loopifbody:
+ // xor scratch1, destreg, incr
+ // and scratch1, scratch1, mask
+ // xor scratch1, destreg, scratch1
+ insertMaskedMerge(TII, DL, LoopIfBodyMBB, Scratch1Reg, DestReg, IncrReg,
+ MaskReg, Scratch1Reg);
+
+ // .looptail:
+ // sc.w scratch1, scratch1, (addr)
+ // bnez scratch1, loop
+ BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW32(Ordering)), Scratch1Reg)
+ .addReg(AddrReg)
+ .addReg(Scratch1Reg);
+ BuildMI(LoopTailMBB, DL, TII->get(RISCV::BNE))
+ .addReg(Scratch1Reg)
+ .addReg(RISCV::X0)
+ .addMBB(LoopHeadMBB);
+
+ NextMBBI = MBB.end();
+ MI.eraseFromParent();
+
+ LivePhysRegs LiveRegs;
+ computeAndAddLiveIns(LiveRegs, *LoopHeadMBB);
+ computeAndAddLiveIns(LiveRegs, *LoopIfBodyMBB);
+ computeAndAddLiveIns(LiveRegs, *LoopTailMBB);
+ computeAndAddLiveIns(LiveRegs, *DoneMBB);
+
+ return true;
+}
+
+bool RISCVExpandAtomicPseudo::expandAtomicCmpXchg(
+ MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, bool IsMasked,
+ int Width, MachineBasicBlock::iterator &NextMBBI) {
+ MachineInstr &MI = *MBBI;
+ DebugLoc DL = MI.getDebugLoc();
+ MachineFunction *MF = MBB.getParent();
+ auto LoopHeadMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
+ auto LoopTailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
+ auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
+
+ // Insert new MBBs.
+ MF->insert(++MBB.getIterator(), LoopHeadMBB);
+ MF->insert(++LoopHeadMBB->getIterator(), LoopTailMBB);
+ MF->insert(++LoopTailMBB->getIterator(), DoneMBB);
+
+ // Set up successors and transfer remaining instructions to DoneMBB.
+ LoopHeadMBB->addSuccessor(LoopTailMBB);
+ LoopHeadMBB->addSuccessor(DoneMBB);
+ LoopTailMBB->addSuccessor(DoneMBB);
+ LoopTailMBB->addSuccessor(LoopHeadMBB);
+ DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
+ DoneMBB->transferSuccessors(&MBB);
+ MBB.addSuccessor(LoopHeadMBB);
+
+ Register DestReg = MI.getOperand(0).getReg();
+ Register ScratchReg = MI.getOperand(1).getReg();
+ Register AddrReg = MI.getOperand(2).getReg();
+ Register CmpValReg = MI.getOperand(3).getReg();
+ Register NewValReg = MI.getOperand(4).getReg();
+ AtomicOrdering Ordering =
+ static_cast<AtomicOrdering>(MI.getOperand(IsMasked ? 6 : 5).getImm());
+
+ if (!IsMasked) {
+ // .loophead:
+ // lr.[w|d] dest, (addr)
+ // bne dest, cmpval, done
+ BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW(Ordering, Width)), DestReg)
+ .addReg(AddrReg);
+ BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BNE))
+ .addReg(DestReg)
+ .addReg(CmpValReg)
+ .addMBB(DoneMBB);
+ // .looptail:
+ // sc.[w|d] scratch, newval, (addr)
+ // bnez scratch, loophead
+ BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW(Ordering, Width)), ScratchReg)
+ .addReg(AddrReg)
+ .addReg(NewValReg);
+ BuildMI(LoopTailMBB, DL, TII->get(RISCV::BNE))
+ .addReg(ScratchReg)
+ .addReg(RISCV::X0)
+ .addMBB(LoopHeadMBB);
+ } else {
+ // .loophead:
+ // lr.w dest, (addr)
+ // and scratch, dest, mask
+ // bne scratch, cmpval, done
+ Register MaskReg = MI.getOperand(5).getReg();
+ BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW(Ordering, Width)), DestReg)
+ .addReg(AddrReg);
+ BuildMI(LoopHeadMBB, DL, TII->get(RISCV::AND), ScratchReg)
+ .addReg(DestReg)
+ .addReg(MaskReg);
+ BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BNE))
+ .addReg(ScratchReg)
+ .addReg(CmpValReg)
+ .addMBB(DoneMBB);
+
+ // .looptail:
+ // xor scratch, dest, newval
+ // and scratch, scratch, mask
+ // xor scratch, dest, scratch
+ // sc.w scratch, scratch, (adrr)
+ // bnez scratch, loophead
+ insertMaskedMerge(TII, DL, LoopTailMBB, ScratchReg, DestReg, NewValReg,
+ MaskReg, ScratchReg);
+ BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW(Ordering, Width)), ScratchReg)
+ .addReg(AddrReg)
+ .addReg(ScratchReg);
+ BuildMI(LoopTailMBB, DL, TII->get(RISCV::BNE))
+ .addReg(ScratchReg)
+ .addReg(RISCV::X0)
+ .addMBB(LoopHeadMBB);
+ }
+
+ NextMBBI = MBB.end();
+ MI.eraseFromParent();
+
+ LivePhysRegs LiveRegs;
+ computeAndAddLiveIns(LiveRegs, *LoopHeadMBB);
+ computeAndAddLiveIns(LiveRegs, *LoopTailMBB);
+ computeAndAddLiveIns(LiveRegs, *DoneMBB);
+
+ return true;
+}
+
+} // end of anonymous namespace
+
+INITIALIZE_PASS(RISCVExpandAtomicPseudo, "riscv-expand-atomic-pseudo",
+ RISCV_EXPAND_ATOMIC_PSEUDO_NAME, false, false)
+
+namespace llvm {
+
+FunctionPass *createRISCVExpandAtomicPseudoPass() {
+ return new RISCVExpandAtomicPseudo();
+}
+
+} // end of namespace llvm
diff --git a/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp b/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp
index 84bce0f48562..5dcd294cef04 100644
--- a/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp
+++ b/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp
@@ -43,17 +43,6 @@ class RISCVExpandPseudo : public MachineFunctionPass {
bool expandMBB(MachineBasicBlock &MBB);
bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
MachineBasicBlock::iterator &NextMBBI);
- bool expandAtomicBinOp(MachineBasicBlock &MBB,
- MachineBasicBlock::iterator MBBI, AtomicRMWInst::BinOp,
- bool IsMasked, int Width,
- MachineBasicBlock::iterator &NextMBBI);
- bool expandAtomicMinMaxOp(MachineBasicBlock &MBB,
- MachineBasicBlock::iterator MBBI,
- AtomicRMWInst::BinOp, bool IsMasked, int Width,
- MachineBasicBlock::iterator &NextMBBI);
- bool expandAtomicCmpXchg(MachineBasicBlock &MBB,
- MachineBasicBlock::iterator MBBI, bool IsMasked,
- int Width, MachineBasicBlock::iterator &NextMBBI);
bool expandAuipcInstPair(MachineBasicBlock &MBB,
MachineBasicBlock::iterator MBBI,
MachineBasicBlock::iterator &NextMBBI,
@@ -99,40 +88,6 @@ bool RISCVExpandPseudo::expandMI(MachineBasicBlock &MBB,
MachineBasicBlock::iterator MBBI,
MachineBasicBlock::iterator &NextMBBI) {
switch (MBBI->getOpcode()) {
- case RISCV::PseudoAtomicLoadNand32:
- return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, false, 32,
- NextMBBI);
- case RISCV::PseudoAtomicLoadNand64:
- return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, false, 64,
- NextMBBI);
- case RISCV::PseudoMaskedAtomicSwap32:
- return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xchg, true, 32,
- NextMBBI);
- case RISCV::PseudoMaskedAtomicLoadAdd32:
- return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Add, true, 32, NextMBBI);
- case RISCV::PseudoMaskedAtomicLoadSub32:
- return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Sub, true, 32, NextMBBI);
- case RISCV::PseudoMaskedAtomicLoadNand32:
- return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, true, 32,
- NextMBBI);
- case RISCV::PseudoMaskedAtomicLoadMax32:
- return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Max, true, 32,
- NextMBBI);
- case RISCV::PseudoMaskedAtomicLoadMin32:
- return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Min, true, 32,
- NextMBBI);
- case RISCV::PseudoMaskedAtomicLoadUMax32:
- return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMax, true, 32,
- NextMBBI);
- case RISCV::PseudoMaskedAtomicLoadUMin32:
- return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMin, true, 32,
- NextMBBI);
- case RISCV::PseudoCmpXchg32:
- return expandAtomicCmpXchg(MBB, MBBI, false, 32, NextMBBI);
- case RISCV::PseudoCmpXchg64:
- return expandAtomicCmpXchg(MBB, MBBI, false, 64, NextMBBI);
- case RISCV::PseudoMaskedCmpXchg32:
- return expandAtomicCmpXchg(MBB, MBBI, true, 32, NextMBBI);
case RISCV::PseudoLLA:
return expandLoadLocalAddress(MBB, MBBI, NextMBBI);
case RISCV::PseudoLA:
@@ -146,481 +101,6 @@ bool RISCVExpandPseudo::expandMI(MachineBasicBlock &MBB,
return false;
}
-static unsigned getLRForRMW32(AtomicOrdering Ordering) {
- switch (Ordering) {
- default:
- llvm_unreachable("Unexpected AtomicOrdering");
- case AtomicOrdering::Monotonic:
- return RISCV::LR_W;
- case AtomicOrdering::Acquire:
- return RISCV::LR_W_AQ;
- case AtomicOrdering::Release:
- return RISCV::LR_W;
- case AtomicOrdering::AcquireRelease:
- return RISCV::LR_W_AQ;
- case AtomicOrdering::SequentiallyConsistent:
- return RISCV::LR_W_AQ_RL;
- }
-}
-
-static unsigned getSCForRMW32(AtomicOrdering Ordering) {
- switch (Ordering) {
- default:
- llvm_unreachable("Unexpected AtomicOrdering");
- case AtomicOrdering::Monotonic:
- return RISCV::SC_W;
- case AtomicOrdering::Acquire:
- return RISCV::SC_W;
- case AtomicOrdering::Release:
- return RISCV::SC_W_RL;
- case AtomicOrdering::AcquireRelease:
- return RISCV::SC_W_RL;
- case AtomicOrdering::SequentiallyConsistent:
- return RISCV::SC_W_AQ_RL;
- }
-}
-
-static unsigned getLRForRMW64(AtomicOrdering Ordering) {
- switch (Ordering) {
- default:
- llvm_unreachable("Unexpected AtomicOrdering");
- case AtomicOrdering::Monotonic:
- return RISCV::LR_D;
- case AtomicOrdering::Acquire:
- return RISCV::LR_D_AQ;
- case AtomicOrdering::Release:
- return RISCV::LR_D;
- case AtomicOrdering::AcquireRelease:
- return RISCV::LR_D_AQ;
- case AtomicOrdering::SequentiallyConsistent:
- return RISCV::LR_D_AQ_RL;
- }
-}
-
-static unsigned getSCForRMW64(AtomicOrdering Ordering) {
- switch (Ordering) {
- default:
- llvm_unreachable("Unexpected AtomicOrdering");
- case AtomicOrdering::Monotonic:
- return RISCV::SC_D;
- case AtomicOrdering::Acquire:
- return RISCV::SC_D;
- case AtomicOrdering::Release:
- return RISCV::SC_D_RL;
- case AtomicOrdering::AcquireRelease:
- return RISCV::SC_D_RL;
- case AtomicOrdering::SequentiallyConsistent:
- return RISCV::SC_D_AQ_RL;
- }
-}
-
-static unsigned getLRForRMW(AtomicOrdering Ordering, int Width) {
- if (Width == 32)
- return getLRForRMW32(Ordering);
- if (Width == 64)
- return getLRForRMW64(Ordering);
- llvm_unreachable("Unexpected LR width\n");
-}
-
-static unsigned getSCForRMW(AtomicOrdering Ordering, int Width) {
- if (Width == 32)
- return getSCForRMW32(Ordering);
- if (Width == 64)
- return getSCForRMW64(Ordering);
- llvm_unreachable("Unexpected SC width\n");
-}
-
-static void doAtomicBinOpExpansion(const RISCVInstrInfo *TII, MachineInstr &MI,
- DebugLoc DL, MachineBasicBlock *ThisMBB,
- MachineBasicBlock *LoopMBB,
- MachineBasicBlock *DoneMBB,
- AtomicRMWInst::BinOp BinOp, int Width) {
- Register DestReg = MI.getOperand(0).getReg();
- Register ScratchReg = MI.getOperand(1).getReg();
- Register AddrReg = MI.getOperand(2).getReg();
- Register IncrReg = MI.getOperand(3).getReg();
- AtomicOrdering Ordering =
- static_cast<AtomicOrdering>(MI.getOperand(4).getImm());
-
- // .loop:
- // lr.[w|d] dest, (addr)
- // binop scratch, dest, val
- // sc.[w|d] scratch, scratch, (addr)
- // bnez scratch, loop
- BuildMI(LoopMBB, DL, TII->get(getLRForRMW(Ordering, Width)), DestReg)
- .addReg(AddrReg);
- switch (BinOp) {
- default:
- llvm_unreachable("Unexpected AtomicRMW BinOp");
- case AtomicRMWInst::Nand:
- BuildMI(LoopMBB, DL, TII->get(RISCV::AND), ScratchReg)
- .addReg(DestReg)
- .addReg(IncrReg);
- BuildMI(LoopMBB, DL, TII->get(RISCV::XORI), ScratchReg)
- .addReg(ScratchReg)
- .addImm(-1);
- break;
- }
- BuildMI(LoopMBB, DL, TII->get(getSCForRMW(Ordering, Width)), ScratchReg)
- .addReg(AddrReg)
- .addReg(ScratchReg);
- BuildMI(LoopMBB, DL, TII->get(RISCV::BNE))
- .addReg(ScratchReg)
- .addReg(RISCV::X0)
- .addMBB(LoopMBB);
-}
-
-static void insertMaskedMerge(const RISCVInstrInfo *TII, DebugLoc DL,
- MachineBasicBlock *MBB, Register DestReg,
- Register OldValReg, Register NewValReg,
- Register MaskReg, Register ScratchReg) {
- assert(OldValReg != ScratchReg && "OldValReg and ScratchReg must be unique");
- assert(OldValReg != MaskReg && "OldValReg and MaskReg must be unique");
- assert(ScratchReg != MaskReg && "ScratchReg and MaskReg must be unique");
-
- // We select bits from newval and oldval using:
- // https://graphics.stanford.edu/~seander/bithacks.html#MaskedMerge
- // r = oldval ^ ((oldval ^ newval) & masktargetdata);
- BuildMI(MBB, DL, TII->get(RISCV::XOR), ScratchReg)
- .addReg(OldValReg)
- .addReg(NewValReg);
- BuildMI(MBB, DL, TII->get(RISCV::AND), ScratchReg)
- .addReg(ScratchReg)
- .addReg(MaskReg);
- BuildMI(MBB, DL, TII->get(RISCV::XOR), DestReg)
- .addReg(OldValReg)
- .addReg(ScratchReg);
-}
-
-static void doMaskedAtomicBinOpExpansion(
- const RISCVInstrInfo *TII, MachineInstr &MI, DebugLoc DL,
- MachineBasicBlock *ThisMBB, MachineBasicBlock *LoopMBB,
- MachineBasicBlock *DoneMBB, AtomicRMWInst::BinOp BinOp, int Width) {
- assert(Width == 32 && "Should never need to expand masked 64-bit operations");
- Register DestReg = MI.getOperand(0).getReg();
- Register ScratchReg = MI.getOperand(1).getReg();
- Register AddrReg = MI.getOperand(2).getReg();
- Register IncrReg = MI.getOperand(3).getReg();
- Register MaskReg = MI.getOperand(4).getReg();
- AtomicOrdering Ordering =
- static_cast<AtomicOrdering>(MI.getOperand(5).getImm());
-
- // .loop:
- // lr.w destreg, (alignedaddr)
- // binop scratch, destreg, incr
- // xor scratch, destreg, scratch
- // and scratch, scratch, masktargetdata
- // xor scratch, destreg, scratch
- // sc.w scratch, scratch, (alignedaddr)
- // bnez scratch, loop
- BuildMI(LoopMBB, DL, TII->get(getLRForRMW32(Ordering)), DestReg)
- .addReg(AddrReg);
- switch (BinOp) {
- default:
- llvm_unreachable("Unexpected AtomicRMW BinOp");
- case AtomicRMWInst::Xchg:
- BuildMI(LoopMBB, DL, TII->get(RISCV::ADDI), ScratchReg)
- .addReg(IncrReg)
- .addImm(0);
- break;
- case AtomicRMWInst::Add:
- BuildMI(LoopMBB, DL, TII->get(RISCV::ADD), ScratchReg)
- .addReg(DestReg)
- .addReg(IncrReg);
- break;
- case AtomicRMWInst::Sub:
- BuildMI(LoopMBB, DL, TII->get(RISCV::SUB), ScratchReg)
- .addReg(DestReg)
- .addReg(IncrReg);
- break;
- case AtomicRMWInst::Nand:
- BuildMI(LoopMBB, DL, TII->get(RISCV::AND), ScratchReg)
- .addReg(DestReg)
- .addReg(IncrReg);
- BuildMI(LoopMBB, DL, TII->get(RISCV::XORI), ScratchReg)
- .addReg(ScratchReg)
- .addImm(-1);
- break;
- }
-
- insertMaskedMerge(TII, DL, LoopMBB, ScratchReg, DestReg, ScratchReg, MaskReg,
- ScratchReg);
-
- BuildMI(LoopMBB, DL, TII->get(getSCForRMW32(Ordering)), ScratchReg)
- .addReg(AddrReg)
- .addReg(ScratchReg);
- BuildMI(LoopMBB, DL, TII->get(RISCV::BNE))
- .addReg(ScratchReg)
- .addReg(RISCV::X0)
- .addMBB(LoopMBB);
-}
-
-bool RISCVExpandPseudo::expandAtomicBinOp(
- MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
- AtomicRMWInst::BinOp BinOp, bool IsMasked, int Width,
- MachineBasicBlock::iterator &NextMBBI) {
- MachineInstr &MI = *MBBI;
- DebugLoc DL = MI.getDebugLoc();
-
- MachineFunction *MF = MBB.getParent();
- auto LoopMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
- auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
-
- // Insert new MBBs.
- MF->insert(++MBB.getIterator(), LoopMBB);
- MF->insert(++LoopMBB->getIterator(), DoneMBB);
-
- // Set up successors and transfer remaining instructions to DoneMBB.
- LoopMBB->addSuccessor(LoopMBB);
- LoopMBB->addSuccessor(DoneMBB);
- DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
- DoneMBB->transferSuccessors(&MBB);
- MBB.addSuccessor(LoopMBB);
-
- if (!IsMasked)
- doAtomicBinOpExpansion(TII, MI, DL, &MBB, LoopMBB, DoneMBB, BinOp, Width);
- else
- doMaskedAtomicBinOpExpansion(TII, MI, DL, &MBB, LoopMBB, DoneMBB, BinOp,
- Width);
-
- NextMBBI = MBB.end();
- MI.eraseFromParent();
-
- LivePhysRegs LiveRegs;
- computeAndAddLiveIns(LiveRegs, *LoopMBB);
- computeAndAddLiveIns(LiveRegs, *DoneMBB);
-
- return true;
-}
-
-static void insertSext(const RISCVInstrInfo *TII, DebugLoc DL,
- MachineBasicBlock *MBB, Register ValReg,
- Register ShamtReg) {
- BuildMI(MBB, DL, TII->get(RISCV::SLL), ValReg)
- .addReg(ValReg)
- .addReg(ShamtReg);
- BuildMI(MBB, DL, TII->get(RISCV::SRA), ValReg)
- .addReg(ValReg)
- .addReg(ShamtReg);
-}
-
-bool RISCVExpandPseudo::expandAtomicMinMaxOp(
- MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
- AtomicRMWInst::BinOp BinOp, bool IsMasked, int Width,
- MachineBasicBlock::iterator &NextMBBI) {
- assert(IsMasked == true &&
- "Should only need to expand masked atomic max/min");
- assert(Width == 32 && "Should never need to expand masked 64-bit operations");
-
- MachineInstr &MI = *MBBI;
- DebugLoc DL = MI.getDebugLoc();
- MachineFunction *MF = MBB.getParent();
- auto LoopHeadMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
- auto LoopIfBodyMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
- auto LoopTailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
- auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
-
- // Insert new MBBs.
- MF->insert(++MBB.getIterator(), LoopHeadMBB);
- MF->insert(++LoopHeadMBB->getIterator(), LoopIfBodyMBB);
- MF->insert(++LoopIfBodyMBB->getIterator(), LoopTailMBB);
- MF->insert(++LoopTailMBB->getIterator(), DoneMBB);
-
- // Set up successors and transfer remaining instructions to DoneMBB.
- LoopHeadMBB->addSuccessor(LoopIfBodyMBB);
- LoopHeadMBB->addSuccessor(LoopTailMBB);
- LoopIfBodyMBB->addSuccessor(LoopTailMBB);
- LoopTailMBB->addSuccessor(LoopHeadMBB);
- LoopTailMBB->addSuccessor(DoneMBB);
- DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
- DoneMBB->transferSuccessors(&MBB);
- MBB.addSuccessor(LoopHeadMBB);
-
- Register DestReg = MI.getOperand(0).getReg();
- Register Scratch1Reg = MI.getOperand(1).getReg();
- Register Scratch2Reg = MI.getOperand(2).getReg();
- Register AddrReg = MI.getOperand(3).getReg();
- Register IncrReg = MI.getOperand(4).getReg();
- Register MaskReg = MI.getOperand(5).getReg();
- bool IsSigned = BinOp == AtomicRMWInst::Min || BinOp == AtomicRMWInst::Max;
- AtomicOrdering Ordering =
- static_cast<AtomicOrdering>(MI.getOperand(IsSigned ? 7 : 6).getImm());
-
- //
- // .loophead:
- // lr.w destreg, (alignedaddr)
- // and scratch2, destreg, mask
- // mv scratch1, destreg
- // [sext scratch2 if signed min/max]
- // ifnochangeneeded scratch2, incr, .looptail
- BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW32(Ordering)), DestReg)
- .addReg(AddrReg);
- BuildMI(LoopHeadMBB, DL, TII->get(RISCV::AND), Scratch2Reg)
- .addReg(DestReg)
- .addReg(MaskReg);
- BuildMI(LoopHeadMBB, DL, TII->get(RISCV::ADDI), Scratch1Reg)
- .addReg(DestReg)
- .addImm(0);
-
- switch (BinOp) {
- default:
- llvm_unreachable("Unexpected AtomicRMW BinOp");
- case AtomicRMWInst::Max: {
- insertSext(TII, DL, LoopHeadMBB, Scratch2Reg, MI.getOperand(6).getReg());
- BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGE))
- .addReg(Scratch2Reg)
- .addReg(IncrReg)
- .addMBB(LoopTailMBB);
- break;
- }
- case AtomicRMWInst::Min: {
- insertSext(TII, DL, LoopHeadMBB, Scratch2Reg, MI.getOperand(6).getReg());
- BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGE))
- .addReg(IncrReg)
- .addReg(Scratch2Reg)
- .addMBB(LoopTailMBB);
- break;
- }
- case AtomicRMWInst::UMax:
- BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGEU))
- .addReg(Scratch2Reg)
- .addReg(IncrReg)
- .addMBB(LoopTailMBB);
- break;
- case AtomicRMWInst::UMin:
- BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGEU))
- .addReg(IncrReg)
- .addReg(Scratch2Reg)
- .addMBB(LoopTailMBB);
- break;
- }
-
- // .loopifbody:
- // xor scratch1, destreg, incr
- // and scratch1, scratch1, mask
- // xor scratch1, destreg, scratch1
- insertMaskedMerge(TII, DL, LoopIfBodyMBB, Scratch1Reg, DestReg, IncrReg,
- MaskReg, Scratch1Reg);
-
- // .looptail:
- // sc.w scratch1, scratch1, (addr)
- // bnez scratch1, loop
- BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW32(Ordering)), Scratch1Reg)
- .addReg(AddrReg)
- .addReg(Scratch1Reg);
- BuildMI(LoopTailMBB, DL, TII->get(RISCV::BNE))
- .addReg(Scratch1Reg)
- .addReg(RISCV::X0)
- .addMBB(LoopHeadMBB);
-
- NextMBBI = MBB.end();
- MI.eraseFromParent();
-
- LivePhysRegs LiveRegs;
- computeAndAddLiveIns(LiveRegs, *LoopHeadMBB);
- computeAndAddLiveIns(LiveRegs, *LoopIfBodyMBB);
- computeAndAddLiveIns(LiveRegs, *LoopTailMBB);
- computeAndAddLiveIns(LiveRegs, *DoneMBB);
-
- return true;
-}
-
-bool RISCVExpandPseudo::expandAtomicCmpXchg(
- MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, bool IsMasked,
- int Width, MachineBasicBlock::iterator &NextMBBI) {
- MachineInstr &MI = *MBBI;
- DebugLoc DL = MI.getDebugLoc();
- MachineFunction *MF = MBB.getParent();
- auto LoopHeadMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
- auto LoopTailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
- auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
-
- // Insert new MBBs.
- MF->insert(++MBB.getIterator(), LoopHeadMBB);
- MF->insert(++LoopHeadMBB->getIterator(), LoopTailMBB);
- MF->insert(++LoopTailMBB->getIterator(), DoneMBB);
-
- // Set up successors and transfer remaining instructions to DoneMBB.
- LoopHeadMBB->addSuccessor(LoopTailMBB);
- LoopHeadMBB->addSuccessor(DoneMBB);
- LoopTailMBB->addSuccessor(DoneMBB);
- LoopTailMBB->addSuccessor(LoopHeadMBB);
- DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
- DoneMBB->transferSuccessors(&MBB);
- MBB.addSuccessor(LoopHeadMBB);
-
- Register DestReg = MI.getOperand(0).getReg();
- Register ScratchReg = MI.getOperand(1).getReg();
- Register AddrReg = MI.getOperand(2).getReg();
- Register CmpValReg = MI.getOperand(3).getReg();
- Register NewValReg = MI.getOperand(4).getReg();
- AtomicOrdering Ordering =
- static_cast<AtomicOrdering>(MI.getOperand(IsMasked ? 6 : 5).getImm());
-
- if (!IsMasked) {
- // .loophead:
- // lr.[w|d] dest, (addr)
- // bne dest, cmpval, done
- BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW(Ordering, Width)), DestReg)
- .addReg(AddrReg);
- BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BNE))
- .addReg(DestReg)
- .addReg(CmpValReg)
- .addMBB(DoneMBB);
- // .looptail:
- // sc.[w|d] scratch, newval, (addr)
- // bnez scratch, loophead
- BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW(Ordering, Width)), ScratchReg)
- .addReg(AddrReg)
- .addReg(NewValReg);
- BuildMI(LoopTailMBB, DL, TII->get(RISCV::BNE))
- .addReg(ScratchReg)
- .addReg(RISCV::X0)
- .addMBB(LoopHeadMBB);
- } else {
- // .loophead:
- // lr.w dest, (addr)
- // and scratch, dest, mask
- // bne scratch, cmpval, done
- Register MaskReg = MI.getOperand(5).getReg();
- BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW(Ordering, Width)), DestReg)
- .addReg(AddrReg);
- BuildMI(LoopHeadMBB, DL, TII->get(RISCV::AND), ScratchReg)
- .addReg(DestReg)
- .addReg(MaskReg);
- BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BNE))
- .addReg(ScratchReg)
- .addReg(CmpValReg)
- .addMBB(DoneMBB);
-
- // .looptail:
- // xor scratch, dest, newval
- // and scratch, scratch, mask
- // xor scratch, dest, scratch
- // sc.w scratch, scratch, (adrr)
- // bnez scratch, loophead
- insertMaskedMerge(TII, DL, LoopTailMBB, ScratchReg, DestReg, NewValReg,
- MaskReg, ScratchReg);
- BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW(Ordering, Width)), ScratchReg)
- .addReg(AddrReg)
- .addReg(ScratchReg);
- BuildMI(LoopTailMBB, DL, TII->get(RISCV::BNE))
- .addReg(ScratchReg)
- .addReg(RISCV::X0)
- .addMBB(LoopHeadMBB);
- }
-
- NextMBBI = MBB.end();
- MI.eraseFromParent();
-
- LivePhysRegs LiveRegs;
- computeAndAddLiveIns(LiveRegs, *LoopHeadMBB);
- computeAndAddLiveIns(LiveRegs, *LoopTailMBB);
- computeAndAddLiveIns(LiveRegs, *DoneMBB);
-
- return true;
-}
-
bool RISCVExpandPseudo::expandAuipcInstPair(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
MachineBasicBlock::iterator &NextMBBI, unsigned FlagsHi,
diff --git a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
index de71c01753de..fe700ae713ba 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
@@ -128,6 +128,7 @@ class RISCVPassConfig : public TargetPassConfig {
bool addGlobalInstructionSelect() override;
void addPreEmitPass() override;
void addPreEmitPass2() override;
+ void addPreSched2() override;
void addPreRegAlloc() override;
};
}
@@ -167,13 +168,15 @@ bool RISCVPassConfig::addGlobalInstructionSelect() {
return false;
}
+void RISCVPassConfig::addPreSched2() { addPass(createRISCVExpandPseudoPass()); }
+
void RISCVPassConfig::addPreEmitPass() { addPass(&BranchRelaxationPassID); }
void RISCVPassConfig::addPreEmitPass2() {
// Schedule the expansion of AMOs at the last possible moment, avoiding the
// possibility for other passes to break the requirements for forward
// progress in the LR/SC block.
- addPass(createRISCVExpandPseudoPass());
+ addPass(createRISCVExpandAtomicPseudoPass());
}
void RISCVPassConfig::addPreRegAlloc() {
diff --git a/llvm/test/CodeGen/RISCV/codemodel-lowering.ll b/llvm/test/CodeGen/RISCV/codemodel-lowering.ll
index 6c172a26f050..413f2b0e45a2 100644
--- a/llvm/test/CodeGen/RISCV/codemodel-lowering.ll
+++ b/llvm/test/CodeGen/RISCV/codemodel-lowering.ll
@@ -82,23 +82,23 @@ define signext i32 @lower_blockaddress_displ(i32 signext %w) nounwind {
; RV32I-MEDIUM: # %bb.0: # %entry
; RV32I-MEDIUM-NEXT: addi sp, sp, -16
; RV32I-MEDIUM-NEXT: sw ra, 12(sp)
-; RV32I-MEDIUM-NEXT: .LBB2_5: # %entry
+; RV32I-MEDIUM-NEXT: .LBB2_1: # %entry
; RV32I-MEDIUM-NEXT: # Label of block must be emitted
; RV32I-MEDIUM-NEXT: auipc a1, %pcrel_hi(.Ltmp0)
-; RV32I-MEDIUM-NEXT: addi a1, a1, %pcrel_lo(.LBB2_5)
+; RV32I-MEDIUM-NEXT: addi a1, a1, %pcrel_lo(.LBB2_1)
; RV32I-MEDIUM-NEXT: addi a2, zero, 101
; RV32I-MEDIUM-NEXT: sw a1, 8(sp)
-; RV32I-MEDIUM-NEXT: blt a0, a2, .LBB2_3
-; RV32I-MEDIUM-NEXT: # %bb.1: # %if.then
+; RV32I-MEDIUM-NEXT: blt a0, a2, .LBB2_4
+; RV32I-MEDIUM-NEXT: # %bb.2: # %if.then
; RV32I-MEDIUM-NEXT: lw a0, 8(sp)
; RV32I-MEDIUM-NEXT: jr a0
; RV32I-MEDIUM-NEXT: .Ltmp0: # Block address taken
-; RV32I-MEDIUM-NEXT: .LBB2_2: # %return
+; RV32I-MEDIUM-NEXT: .LBB2_3: # %return
; RV32I-MEDIUM-NEXT: addi a0, zero, 4
-; RV32I-MEDIUM-NEXT: j .LBB2_4
-; RV32I-MEDIUM-NEXT: .LBB2_3: # %return.clone
+; RV32I-MEDIUM-NEXT: j .LBB2_5
+; RV32I-MEDIUM-NEXT: .LBB2_4: # %return.clone
; RV32I-MEDIUM-NEXT: addi a0, zero, 3
-; RV32I-MEDIUM-NEXT: .LBB2_4: # %.split
+; RV32I-MEDIUM-NEXT: .LBB2_5: # %.split
; RV32I-MEDIUM-NEXT: lw ra, 12(sp)
; RV32I-MEDIUM-NEXT: addi sp, sp, 16
; RV32I-MEDIUM-NEXT: ret
More information about the llvm-commits
mailing list