[llvm] r336589 - [globalisel][irtranslator] Add support for atomicrmw and (strong) cmpxchg
Daniel Sanders via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 9 12:33:40 PDT 2018
Author: dsanders
Date: Mon Jul 9 12:33:40 2018
New Revision: 336589
URL: http://llvm.org/viewvc/llvm-project?rev=336589&view=rev
Log:
[globalisel][irtranslator] Add support for atomicrmw and (strong) cmpxchg
Summary:
This patch adds support for the atomicrmw instructions and the strong
cmpxchg instruction to the IRTranslator.
I've left out weak cmpxchg because LangRef.rst isn't entirely clear on what
difference it makes to the backend. As far as I can tell from the code, it
only matters to AtomicExpandPass which is run at the LLVM-IR level.
Reviewers: ab, t.p.northover, qcolombet, rovka, aditya_nandakumar, volkan, javed.absar
Reviewed By: qcolombet
Subscribers: kristof.beyls, javed.absar, igorb, llvm-commits
Differential Revision: https://reviews.llvm.org/D40092
Modified:
llvm/trunk/include/llvm/CodeGen/GlobalISel/IRTranslator.h
llvm/trunk/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
llvm/trunk/lib/CodeGen/GlobalISel/IRTranslator.cpp
llvm/trunk/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
llvm/trunk/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll
Modified: llvm/trunk/include/llvm/CodeGen/GlobalISel/IRTranslator.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/CodeGen/GlobalISel/IRTranslator.h?rev=336589&r1=336588&r2=336589&view=diff
==============================================================================
--- llvm/trunk/include/llvm/CodeGen/GlobalISel/IRTranslator.h (original)
+++ llvm/trunk/include/llvm/CodeGen/GlobalISel/IRTranslator.h Mon Jul 9 12:33:40 2018
@@ -399,6 +399,9 @@ private:
bool translateShuffleVector(const User &U, MachineIRBuilder &MIRBuilder);
+ bool translateAtomicCmpXchg(const User &U, MachineIRBuilder &MIRBuilder);
+ bool translateAtomicRMW(const User &U, MachineIRBuilder &MIRBuilder);
+
// Stubs to keep the compiler happy while we implement the rest of the
// translation.
bool translateResume(const User &U, MachineIRBuilder &MIRBuilder) {
@@ -416,12 +419,6 @@ private:
bool translateFence(const User &U, MachineIRBuilder &MIRBuilder) {
return false;
}
- bool translateAtomicCmpXchg(const User &U, MachineIRBuilder &MIRBuilder) {
- return false;
- }
- bool translateAtomicRMW(const User &U, MachineIRBuilder &MIRBuilder) {
- return false;
- }
bool translateAddrSpaceCast(const User &U, MachineIRBuilder &MIRBuilder) {
return translateCast(TargetOpcode::G_ADDRSPACE_CAST, U, MIRBuilder);
}
Modified: llvm/trunk/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h?rev=336589&r1=336588&r2=336589&view=diff
==============================================================================
--- llvm/trunk/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h (original)
+++ llvm/trunk/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h Mon Jul 9 12:33:40 2018
@@ -717,7 +717,28 @@ public:
MachineInstrBuilder buildExtractVectorElement(unsigned Res, unsigned Val,
unsigned Idx);
- /// Build and insert `OldValRes = G_ATOMIC_CMPXCHG Addr, CmpVal, NewVal,
+ /// Build and insert `OldValRes<def>, SuccessRes<def> =
+ /// G_ATOMIC_CMPXCHG_WITH_SUCCESS Addr, CmpVal, NewVal, MMO`.
+ ///
+ /// Atomically replace the value at \p Addr with \p NewVal if it is currently
+ /// \p CmpVal otherwise leaves it unchanged. Puts the original value from \p
+ /// Addr in \p Res, along with an s1 indicating whether it was replaced.
+ ///
+ /// \pre setBasicBlock or setMI must have been called.
+ /// \pre \p OldValRes must be a generic virtual register of scalar type.
+ /// \pre \p SuccessRes must be a generic virtual register of scalar type. It
+ /// will be assigned 0 on failure and 1 on success.
+ /// \pre \p Addr must be a generic virtual register with pointer type.
+ /// \pre \p OldValRes, \p CmpVal, and \p NewVal must be generic virtual
+ /// registers of the same type.
+ ///
+ /// \return a MachineInstrBuilder for the newly created instruction.
+ MachineInstrBuilder
+ buildAtomicCmpXchgWithSuccess(unsigned OldValRes, unsigned SuccessRes,
+ unsigned Addr, unsigned CmpVal, unsigned NewVal,
+ MachineMemOperand &MMO);
+
+ /// Build and insert `OldValRes<def> = G_ATOMIC_CMPXCHG Addr, CmpVal, NewVal,
/// MMO`.
///
/// Atomically replace the value at \p Addr with \p NewVal if it is currently
@@ -734,6 +755,193 @@ public:
MachineInstrBuilder buildAtomicCmpXchg(unsigned OldValRes, unsigned Addr,
unsigned CmpVal, unsigned NewVal,
MachineMemOperand &MMO);
+
+ /// Build and insert `OldValRes<def> = G_ATOMICRMW_<Opcode> Addr, Val, MMO`.
+ ///
+ /// Atomically read-modify-update the value at \p Addr with \p Val. Puts the
+ /// original value from \p Addr in \p OldValRes. The modification is
+ /// determined by the opcode.
+ ///
+ /// \pre setBasicBlock or setMI must have been called.
+ /// \pre \p OldValRes must be a generic virtual register.
+ /// \pre \p Addr must be a generic virtual register with pointer type.
+ /// \pre \p OldValRes, and \p Val must be generic virtual registers of the
+ /// same type.
+ ///
+ /// \return a MachineInstrBuilder for the newly created instruction.
+ MachineInstrBuilder buildAtomicRMW(unsigned Opcode, unsigned OldValRes,
+ unsigned Addr, unsigned Val,
+ MachineMemOperand &MMO);
+
+ /// Build and insert `OldValRes<def> = G_ATOMICRMW_XCHG Addr, Val, MMO`.
+ ///
+ /// Atomically replace the value at \p Addr with \p Val. Puts the original
+ /// value from \p Addr in \p OldValRes.
+ ///
+ /// \pre setBasicBlock or setMI must have been called.
+ /// \pre \p OldValRes must be a generic virtual register.
+ /// \pre \p Addr must be a generic virtual register with pointer type.
+ /// \pre \p OldValRes, and \p Val must be generic virtual registers of the
+ /// same type.
+ ///
+ /// \return a MachineInstrBuilder for the newly created instruction.
+ MachineInstrBuilder buildAtomicRMWXchg(unsigned OldValRes, unsigned Addr,
+ unsigned Val, MachineMemOperand &MMO);
+
+ /// Build and insert `OldValRes<def> = G_ATOMICRMW_ADD Addr, Val, MMO`.
+ ///
+ /// Atomically replace the value at \p Addr with the addition of \p Val and
+ /// the original value. Puts the original value from \p Addr in \p OldValRes.
+ ///
+ /// \pre setBasicBlock or setMI must have been called.
+ /// \pre \p OldValRes must be a generic virtual register.
+ /// \pre \p Addr must be a generic virtual register with pointer type.
+ /// \pre \p OldValRes, and \p Val must be generic virtual registers of the
+ /// same type.
+ ///
+ /// \return a MachineInstrBuilder for the newly created instruction.
+ MachineInstrBuilder buildAtomicRMWAdd(unsigned OldValRes, unsigned Addr,
+ unsigned Val, MachineMemOperand &MMO);
+
+ /// Build and insert `OldValRes<def> = G_ATOMICRMW_SUB Addr, Val, MMO`.
+ ///
+ /// Atomically replace the value at \p Addr with the subtraction of \p Val and
+ /// the original value. Puts the original value from \p Addr in \p OldValRes.
+ ///
+ /// \pre setBasicBlock or setMI must have been called.
+ /// \pre \p OldValRes must be a generic virtual register.
+ /// \pre \p Addr must be a generic virtual register with pointer type.
+ /// \pre \p OldValRes, and \p Val must be generic virtual registers of the
+ /// same type.
+ ///
+ /// \return a MachineInstrBuilder for the newly created instruction.
+ MachineInstrBuilder buildAtomicRMWSub(unsigned OldValRes, unsigned Addr,
+ unsigned Val, MachineMemOperand &MMO);
+
+ /// Build and insert `OldValRes<def> = G_ATOMICRMW_AND Addr, Val, MMO`.
+ ///
+ /// Atomically replace the value at \p Addr with the bitwise and of \p Val and
+ /// the original value. Puts the original value from \p Addr in \p OldValRes.
+ ///
+ /// \pre setBasicBlock or setMI must have been called.
+ /// \pre \p OldValRes must be a generic virtual register.
+ /// \pre \p Addr must be a generic virtual register with pointer type.
+ /// \pre \p OldValRes, and \p Val must be generic virtual registers of the
+ /// same type.
+ ///
+ /// \return a MachineInstrBuilder for the newly created instruction.
+ MachineInstrBuilder buildAtomicRMWAnd(unsigned OldValRes, unsigned Addr,
+ unsigned Val, MachineMemOperand &MMO);
+
+ /// Build and insert `OldValRes<def> = G_ATOMICRMW_NAND Addr, Val, MMO`.
+ ///
+ /// Atomically replace the value at \p Addr with the bitwise nand of \p Val
+ /// and the original value. Puts the original value from \p Addr in \p
+ /// OldValRes.
+ ///
+ /// \pre setBasicBlock or setMI must have been called.
+ /// \pre \p OldValRes must be a generic virtual register.
+ /// \pre \p Addr must be a generic virtual register with pointer type.
+ /// \pre \p OldValRes, and \p Val must be generic virtual registers of the
+ /// same type.
+ ///
+ /// \return a MachineInstrBuilder for the newly created instruction.
+ MachineInstrBuilder buildAtomicRMWNand(unsigned OldValRes, unsigned Addr,
+ unsigned Val, MachineMemOperand &MMO);
+
+ /// Build and insert `OldValRes<def> = G_ATOMICRMW_OR Addr, Val, MMO`.
+ ///
+ /// Atomically replace the value at \p Addr with the bitwise or of \p Val and
+ /// the original value. Puts the original value from \p Addr in \p OldValRes.
+ ///
+ /// \pre setBasicBlock or setMI must have been called.
+ /// \pre \p OldValRes must be a generic virtual register.
+ /// \pre \p Addr must be a generic virtual register with pointer type.
+ /// \pre \p OldValRes, and \p Val must be generic virtual registers of the
+ /// same type.
+ ///
+ /// \return a MachineInstrBuilder for the newly created instruction.
+ MachineInstrBuilder buildAtomicRMWOr(unsigned OldValRes, unsigned Addr,
+ unsigned Val, MachineMemOperand &MMO);
+
+ /// Build and insert `OldValRes<def> = G_ATOMICRMW_XOR Addr, Val, MMO`.
+ ///
+ /// Atomically replace the value at \p Addr with the bitwise xor of \p Val and
+ /// the original value. Puts the original value from \p Addr in \p OldValRes.
+ ///
+ /// \pre setBasicBlock or setMI must have been called.
+ /// \pre \p OldValRes must be a generic virtual register.
+ /// \pre \p Addr must be a generic virtual register with pointer type.
+ /// \pre \p OldValRes, and \p Val must be generic virtual registers of the
+ /// same type.
+ ///
+ /// \return a MachineInstrBuilder for the newly created instruction.
+ MachineInstrBuilder buildAtomicRMWXor(unsigned OldValRes, unsigned Addr,
+ unsigned Val, MachineMemOperand &MMO);
+
+ /// Build and insert `OldValRes<def> = G_ATOMICRMW_MAX Addr, Val, MMO`.
+ ///
+ /// Atomically replace the value at \p Addr with the signed maximum of \p
+ /// Val and the original value. Puts the original value from \p Addr in \p
+ /// OldValRes.
+ ///
+ /// \pre setBasicBlock or setMI must have been called.
+ /// \pre \p OldValRes must be a generic virtual register.
+ /// \pre \p Addr must be a generic virtual register with pointer type.
+ /// \pre \p OldValRes, and \p Val must be generic virtual registers of the
+ /// same type.
+ ///
+ /// \return a MachineInstrBuilder for the newly created instruction.
+ MachineInstrBuilder buildAtomicRMWMax(unsigned OldValRes, unsigned Addr,
+ unsigned Val, MachineMemOperand &MMO);
+
+ /// Build and insert `OldValRes<def> = G_ATOMICRMW_MIN Addr, Val, MMO`.
+ ///
+ /// Atomically replace the value at \p Addr with the signed minimum of \p
+ /// Val and the original value. Puts the original value from \p Addr in \p
+ /// OldValRes.
+ ///
+ /// \pre setBasicBlock or setMI must have been called.
+ /// \pre \p OldValRes must be a generic virtual register.
+ /// \pre \p Addr must be a generic virtual register with pointer type.
+ /// \pre \p OldValRes, and \p Val must be generic virtual registers of the
+ /// same type.
+ ///
+ /// \return a MachineInstrBuilder for the newly created instruction.
+ MachineInstrBuilder buildAtomicRMWMin(unsigned OldValRes, unsigned Addr,
+ unsigned Val, MachineMemOperand &MMO);
+
+ /// Build and insert `OldValRes<def> = G_ATOMICRMW_UMAX Addr, Val, MMO`.
+ ///
+ /// Atomically replace the value at \p Addr with the unsigned maximum of \p
+ /// Val and the original value. Puts the original value from \p Addr in \p
+ /// OldValRes.
+ ///
+ /// \pre setBasicBlock or setMI must have been called.
+ /// \pre \p OldValRes must be a generic virtual register.
+ /// \pre \p Addr must be a generic virtual register with pointer type.
+ /// \pre \p OldValRes, and \p Val must be generic virtual registers of the
+ /// same type.
+ ///
+ /// \return a MachineInstrBuilder for the newly created instruction.
+ MachineInstrBuilder buildAtomicRMWUmax(unsigned OldValRes, unsigned Addr,
+ unsigned Val, MachineMemOperand &MMO);
+
+ /// Build and insert `OldValRes<def> = G_ATOMICRMW_UMIN Addr, Val, MMO`.
+ ///
+ /// Atomically replace the value at \p Addr with the unsigned minimum of \p
+ /// Val and the original value. Puts the original value from \p Addr in \p
+ /// OldValRes.
+ ///
+ /// \pre setBasicBlock or setMI must have been called.
+ /// \pre \p OldValRes must be a generic virtual register.
+ /// \pre \p Addr must be a generic virtual register with pointer type.
+ /// \pre \p OldValRes, and \p Val must be generic virtual registers of the
+ /// same type.
+ ///
+ /// \return a MachineInstrBuilder for the newly created instruction.
+ MachineInstrBuilder buildAtomicRMWUmin(unsigned OldValRes, unsigned Addr,
+ unsigned Val, MachineMemOperand &MMO);
};
/// A CRTP class that contains methods for building instructions that can
Modified: llvm/trunk/lib/CodeGen/GlobalISel/IRTranslator.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/GlobalISel/IRTranslator.cpp?rev=336589&r1=336588&r2=336589&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/GlobalISel/IRTranslator.cpp (original)
+++ llvm/trunk/lib/CodeGen/GlobalISel/IRTranslator.cpp Mon Jul 9 12:33:40 2018
@@ -228,6 +228,20 @@ unsigned IRTranslator::getMemOpAlignment
} else if (const LoadInst *LI = dyn_cast<LoadInst>(&I)) {
Alignment = LI->getAlignment();
ValTy = LI->getType();
+ } else if (const AtomicCmpXchgInst *AI = dyn_cast<AtomicCmpXchgInst>(&I)) {
+ // TODO(PR27168): This instruction has no alignment attribute, but unlike
+ // the default alignment for load/store, the default here is to assume
+ // it has NATURAL alignment, not DataLayout-specified alignment.
+ const DataLayout &DL = AI->getModule()->getDataLayout();
+ Alignment = DL.getTypeStoreSize(AI->getCompareOperand()->getType());
+ ValTy = AI->getCompareOperand()->getType();
+ } else if (const AtomicRMWInst *AI = dyn_cast<AtomicRMWInst>(&I)) {
+ // TODO(PR27168): This instruction has no alignment attribute, but unlike
+ // the default alignment for load/store, the default here is to assume
+ // it has NATURAL alignment, not DataLayout-specified alignment.
+ const DataLayout &DL = AI->getModule()->getDataLayout();
+ Alignment = DL.getTypeStoreSize(AI->getValOperand()->getType());
+ ValTy = AI->getType();
} else {
OptimizationRemarkMissed R("gisel-irtranslator", "", &I);
R << "unable to translate memop: " << ore::NV("Opcode", &I);
@@ -1285,6 +1299,100 @@ bool IRTranslator::translatePHI(const Us
return true;
}
+bool IRTranslator::translateAtomicCmpXchg(const User &U,
+ MachineIRBuilder &MIRBuilder) {
+ const AtomicCmpXchgInst &I = cast<AtomicCmpXchgInst>(U);
+
+ if (I.isWeak())
+ return false;
+
+ auto Flags = I.isVolatile() ? MachineMemOperand::MOVolatile
+ : MachineMemOperand::MONone;
+ Flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
+
+ Type *ResType = I.getType();
+ Type *ValType = ResType->Type::getStructElementType(0);
+
+ auto Res = getOrCreateVRegs(I);
+ unsigned OldValRes = Res[0];
+ unsigned SuccessRes = Res[1];
+ unsigned Addr = getOrCreateVReg(*I.getPointerOperand());
+ unsigned Cmp = getOrCreateVReg(*I.getCompareOperand());
+ unsigned NewVal = getOrCreateVReg(*I.getNewValOperand());
+
+ MIRBuilder.buildAtomicCmpXchgWithSuccess(
+ OldValRes, SuccessRes, Addr, Cmp, NewVal,
+ *MF->getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
+ Flags, DL->getTypeStoreSize(ValType),
+ getMemOpAlignment(I), AAMDNodes(), nullptr,
+ I.getSyncScopeID(), I.getSuccessOrdering(),
+ I.getFailureOrdering()));
+ return true;
+}
+
+bool IRTranslator::translateAtomicRMW(const User &U,
+ MachineIRBuilder &MIRBuilder) {
+ const AtomicRMWInst &I = cast<AtomicRMWInst>(U);
+
+ auto Flags = I.isVolatile() ? MachineMemOperand::MOVolatile
+ : MachineMemOperand::MONone;
+ Flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
+
+ Type *ResType = I.getType();
+
+ unsigned Res = getOrCreateVReg(I);
+ unsigned Addr = getOrCreateVReg(*I.getPointerOperand());
+ unsigned Val = getOrCreateVReg(*I.getValOperand());
+
+ unsigned Opcode = 0;
+ switch (I.getOperation()) {
+ default:
+ llvm_unreachable("Unknown atomicrmw op");
+ return false;
+ case AtomicRMWInst::Xchg:
+ Opcode = TargetOpcode::G_ATOMICRMW_XCHG;
+ break;
+ case AtomicRMWInst::Add:
+ Opcode = TargetOpcode::G_ATOMICRMW_ADD;
+ break;
+ case AtomicRMWInst::Sub:
+ Opcode = TargetOpcode::G_ATOMICRMW_SUB;
+ break;
+ case AtomicRMWInst::And:
+ Opcode = TargetOpcode::G_ATOMICRMW_AND;
+ break;
+ case AtomicRMWInst::Nand:
+ Opcode = TargetOpcode::G_ATOMICRMW_NAND;
+ break;
+ case AtomicRMWInst::Or:
+ Opcode = TargetOpcode::G_ATOMICRMW_OR;
+ break;
+ case AtomicRMWInst::Xor:
+ Opcode = TargetOpcode::G_ATOMICRMW_XOR;
+ break;
+ case AtomicRMWInst::Max:
+ Opcode = TargetOpcode::G_ATOMICRMW_MAX;
+ break;
+ case AtomicRMWInst::Min:
+ Opcode = TargetOpcode::G_ATOMICRMW_MIN;
+ break;
+ case AtomicRMWInst::UMax:
+ Opcode = TargetOpcode::G_ATOMICRMW_UMAX;
+ break;
+ case AtomicRMWInst::UMin:
+ Opcode = TargetOpcode::G_ATOMICRMW_UMIN;
+ break;
+ }
+
+ MIRBuilder.buildAtomicRMW(
+ Opcode, Res, Addr, Val,
+ *MF->getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
+ Flags, DL->getTypeStoreSize(ResType),
+ getMemOpAlignment(I), AAMDNodes(), nullptr,
+ I.getSyncScopeID(), I.getOrdering()));
+ return true;
+}
+
void IRTranslator::finishPendingPhis() {
for (auto &Phi : PendingPHIs) {
const PHINode *PI = Phi.first;
Modified: llvm/trunk/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp?rev=336589&r1=336588&r2=336589&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp (original)
+++ llvm/trunk/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp Mon Jul 9 12:33:40 2018
@@ -669,6 +669,33 @@ MachineIRBuilderBase::buildExtractVector
.addUse(Idx);
}
+MachineInstrBuilder MachineIRBuilderBase::buildAtomicCmpXchgWithSuccess(
+ unsigned OldValRes, unsigned SuccessRes, unsigned Addr, unsigned CmpVal,
+ unsigned NewVal, MachineMemOperand &MMO) {
+#ifndef NDEBUG
+ LLT OldValResTy = getMRI()->getType(OldValRes);
+ LLT SuccessResTy = getMRI()->getType(SuccessRes);
+ LLT AddrTy = getMRI()->getType(Addr);
+ LLT CmpValTy = getMRI()->getType(CmpVal);
+ LLT NewValTy = getMRI()->getType(NewVal);
+ assert(OldValResTy.isScalar() && "invalid operand type");
+ assert(SuccessResTy.isScalar() && "invalid operand type");
+ assert(AddrTy.isPointer() && "invalid operand type");
+ assert(CmpValTy.isValid() && "invalid operand type");
+ assert(NewValTy.isValid() && "invalid operand type");
+ assert(OldValResTy == CmpValTy && "type mismatch");
+ assert(OldValResTy == NewValTy && "type mismatch");
+#endif
+
+ return buildInstr(TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS)
+ .addDef(OldValRes)
+ .addDef(SuccessRes)
+ .addUse(Addr)
+ .addUse(CmpVal)
+ .addUse(NewVal)
+ .addMemOperand(&MMO);
+}
+
MachineInstrBuilder
MachineIRBuilderBase::buildAtomicCmpXchg(unsigned OldValRes, unsigned Addr,
unsigned CmpVal, unsigned NewVal,
@@ -694,6 +721,94 @@ MachineIRBuilderBase::buildAtomicCmpXchg
.addMemOperand(&MMO);
}
+MachineInstrBuilder
+MachineIRBuilderBase::buildAtomicRMW(unsigned Opcode, unsigned OldValRes,
+ unsigned Addr, unsigned Val,
+ MachineMemOperand &MMO) {
+#ifndef NDEBUG
+ LLT OldValResTy = getMRI()->getType(OldValRes);
+ LLT AddrTy = getMRI()->getType(Addr);
+ LLT ValTy = getMRI()->getType(Val);
+ assert(OldValResTy.isScalar() && "invalid operand type");
+ assert(AddrTy.isPointer() && "invalid operand type");
+ assert(ValTy.isValid() && "invalid operand type");
+ assert(OldValResTy == ValTy && "type mismatch");
+#endif
+
+ return buildInstr(Opcode)
+ .addDef(OldValRes)
+ .addUse(Addr)
+ .addUse(Val)
+ .addMemOperand(&MMO);
+}
+
+MachineInstrBuilder
+MachineIRBuilderBase::buildAtomicRMWXchg(unsigned OldValRes, unsigned Addr,
+ unsigned Val, MachineMemOperand &MMO) {
+ return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_XCHG, OldValRes, Addr, Val,
+ MMO);
+}
+MachineInstrBuilder
+MachineIRBuilderBase::buildAtomicRMWAdd(unsigned OldValRes, unsigned Addr,
+ unsigned Val, MachineMemOperand &MMO) {
+ return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_ADD, OldValRes, Addr, Val,
+ MMO);
+}
+MachineInstrBuilder
+MachineIRBuilderBase::buildAtomicRMWSub(unsigned OldValRes, unsigned Addr,
+ unsigned Val, MachineMemOperand &MMO) {
+ return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_SUB, OldValRes, Addr, Val,
+ MMO);
+}
+MachineInstrBuilder
+MachineIRBuilderBase::buildAtomicRMWAnd(unsigned OldValRes, unsigned Addr,
+ unsigned Val, MachineMemOperand &MMO) {
+ return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_AND, OldValRes, Addr, Val,
+ MMO);
+}
+MachineInstrBuilder
+MachineIRBuilderBase::buildAtomicRMWNand(unsigned OldValRes, unsigned Addr,
+ unsigned Val, MachineMemOperand &MMO) {
+ return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_NAND, OldValRes, Addr, Val,
+ MMO);
+}
+MachineInstrBuilder
+MachineIRBuilderBase::buildAtomicRMWOr(unsigned OldValRes, unsigned Addr,
+ unsigned Val, MachineMemOperand &MMO) {
+ return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_OR, OldValRes, Addr, Val,
+ MMO);
+}
+MachineInstrBuilder
+MachineIRBuilderBase::buildAtomicRMWXor(unsigned OldValRes, unsigned Addr,
+ unsigned Val, MachineMemOperand &MMO) {
+ return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_XOR, OldValRes, Addr, Val,
+ MMO);
+}
+MachineInstrBuilder
+MachineIRBuilderBase::buildAtomicRMWMax(unsigned OldValRes, unsigned Addr,
+ unsigned Val, MachineMemOperand &MMO) {
+ return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_MAX, OldValRes, Addr, Val,
+ MMO);
+}
+MachineInstrBuilder
+MachineIRBuilderBase::buildAtomicRMWMin(unsigned OldValRes, unsigned Addr,
+ unsigned Val, MachineMemOperand &MMO) {
+ return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_MIN, OldValRes, Addr, Val,
+ MMO);
+}
+MachineInstrBuilder
+MachineIRBuilderBase::buildAtomicRMWUmax(unsigned OldValRes, unsigned Addr,
+ unsigned Val, MachineMemOperand &MMO) {
+ return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_UMAX, OldValRes, Addr, Val,
+ MMO);
+}
+MachineInstrBuilder
+MachineIRBuilderBase::buildAtomicRMWUmin(unsigned OldValRes, unsigned Addr,
+ unsigned Val, MachineMemOperand &MMO) {
+ return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_UMIN, OldValRes, Addr, Val,
+ MMO);
+}
+
void MachineIRBuilderBase::validateTruncExt(unsigned Dst, unsigned Src,
bool IsExtend) {
#ifndef NDEBUG
@@ -701,7 +816,7 @@ void MachineIRBuilderBase::validateTrunc
LLT DstTy = getMRI()->getType(Dst);
if (DstTy.isVector()) {
- assert(SrcTy.isVector() && "mismatched cast between vecot and non-vector");
+ assert(SrcTy.isVector() && "mismatched cast between vector and non-vector");
assert(SrcTy.getNumElements() == DstTy.getNumElements() &&
"different number of elements in a trunc/ext");
} else
Modified: llvm/trunk/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll?rev=336589&r1=336588&r2=336589&view=diff
==============================================================================
--- llvm/trunk/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll (original)
+++ llvm/trunk/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll Mon Jul 9 12:33:40 2018
@@ -1882,3 +1882,268 @@ define i1 @return_i1_zext() {
; CHECK: RET_ReallyLR implicit $w0
ret i1 true
}
+
+; Try one cmpxchg
+define i32 @test_atomic_cmpxchg_1(i32* %addr) {
+; CHECK-LABEL: name: test_atomic_cmpxchg_1
+; CHECK: bb.1.entry:
+; CHECK-NEXT: successors: %bb.{{[^)]+}}
+; CHECK-NEXT: liveins: $x0
+; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
+; CHECK-NEXT: [[OLDVAL:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+; CHECK-NEXT: [[NEWVAL:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+; CHECK: bb.2.repeat:
+; CHECK-NEXT: successors: %bb.3({{[^)]+}}), %bb.2({{[^)]+}})
+; CHECK: [[OLDVALRES:%[0-9]+]]:_(s32), [[SUCCESS:%[0-9]+]]:_(s1) = G_ATOMIC_CMPXCHG_WITH_SUCCESS [[ADDR]](p0), [[OLDVAL]], [[NEWVAL]] :: (load store monotonic monotonic 4 on %ir.addr)
+; CHECK-NEXT: G_BRCOND [[SUCCESS]](s1), %bb.3
+; CHECK-NEXT: G_BR %bb.2
+; CHECK: bb.3.done:
+entry:
+ br label %repeat
+repeat:
+ %val_success = cmpxchg i32* %addr, i32 0, i32 1 monotonic monotonic
+ %value_loaded = extractvalue { i32, i1 } %val_success, 0
+ %success = extractvalue { i32, i1 } %val_success, 1
+ br i1 %success, label %done, label %repeat
+done:
+ ret i32 %value_loaded
+}
+
+; Try one cmpxchg with a small type and high atomic ordering.
+define i16 @test_atomic_cmpxchg_2(i16* %addr) {
+; CHECK-LABEL: name: test_atomic_cmpxchg_2
+; CHECK: bb.1.entry:
+; CHECK-NEXT: successors: %bb.2({{[^)]+}})
+; CHECK-NEXT: liveins: $x0
+; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
+; CHECK-NEXT: [[OLDVAL:%[0-9]+]]:_(s16) = G_CONSTANT i16 0
+; CHECK-NEXT: [[NEWVAL:%[0-9]+]]:_(s16) = G_CONSTANT i16 1
+; CHECK: bb.2.repeat:
+; CHECK-NEXT: successors: %bb.3({{[^)]+}}), %bb.2({{[^)]+}})
+; CHECK: [[OLDVALRES:%[0-9]+]]:_(s16), [[SUCCESS:%[0-9]+]]:_(s1) = G_ATOMIC_CMPXCHG_WITH_SUCCESS [[ADDR]](p0), [[OLDVAL]], [[NEWVAL]] :: (load store seq_cst seq_cst 2 on %ir.addr)
+; CHECK-NEXT: G_BRCOND [[SUCCESS]](s1), %bb.3
+; CHECK-NEXT: G_BR %bb.2
+; CHECK: bb.3.done:
+entry:
+ br label %repeat
+repeat:
+ %val_success = cmpxchg i16* %addr, i16 0, i16 1 seq_cst seq_cst
+ %value_loaded = extractvalue { i16, i1 } %val_success, 0
+ %success = extractvalue { i16, i1 } %val_success, 1
+ br i1 %success, label %done, label %repeat
+done:
+ ret i16 %value_loaded
+}
+
+; Try one cmpxchg where the success order and failure order differ.
+define i64 @test_atomic_cmpxchg_3(i64* %addr) {
+; CHECK-LABEL: name: test_atomic_cmpxchg_3
+; CHECK: bb.1.entry:
+; CHECK-NEXT: successors: %bb.2({{[^)]+}})
+; CHECK-NEXT: liveins: $x0
+; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
+; CHECK-NEXT: [[OLDVAL:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+; CHECK-NEXT: [[NEWVAL:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+; CHECK: bb.2.repeat:
+; CHECK-NEXT: successors: %bb.3({{[^)]+}}), %bb.2({{[^)]+}})
+; CHECK: [[OLDVALRES:%[0-9]+]]:_(s64), [[SUCCESS:%[0-9]+]]:_(s1) = G_ATOMIC_CMPXCHG_WITH_SUCCESS [[ADDR]](p0), [[OLDVAL]], [[NEWVAL]] :: (load store seq_cst acquire 8 on %ir.addr)
+; CHECK-NEXT: G_BRCOND [[SUCCESS]](s1), %bb.3
+; CHECK-NEXT: G_BR %bb.2
+; CHECK: bb.3.done:
+entry:
+ br label %repeat
+repeat:
+ %val_success = cmpxchg i64* %addr, i64 0, i64 1 seq_cst acquire
+ %value_loaded = extractvalue { i64, i1 } %val_success, 0
+ %success = extractvalue { i64, i1 } %val_success, 1
+ br i1 %success, label %done, label %repeat
+done:
+ ret i64 %value_loaded
+}
+
+; Try a monotonic atomicrmw xchg
+; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this.
+define i32 @test_atomicrmw_xchg(i256* %addr) {
+; CHECK-LABEL: name: test_atomicrmw_xchg
+; CHECK: bb.1 (%ir-block.{{[0-9]+}}):
+; CHECK-NEXT: liveins: $x0
+; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
+; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1
+; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_XCHG [[ADDR]](p0), [[VAL]] :: (load store monotonic 32 on %ir.addr)
+; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]]
+ %oldval = atomicrmw xchg i256* %addr, i256 1 monotonic
+ ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this
+ ; test so work around it by truncating to i32 for now.
+ %oldval.trunc = trunc i256 %oldval to i32
+ ret i32 %oldval.trunc
+}
+
+; Try an acquire atomicrmw add
+; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this.
+define i32 @test_atomicrmw_add(i256* %addr) {
+; CHECK-LABEL: name: test_atomicrmw_add
+; CHECK: bb.1 (%ir-block.{{[0-9]+}}):
+; CHECK-NEXT: liveins: $x0
+; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
+; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1
+; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_ADD [[ADDR]](p0), [[VAL]] :: (load store acquire 32 on %ir.addr)
+; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]]
+ %oldval = atomicrmw add i256* %addr, i256 1 acquire
+ ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this
+ ; test so work around it by truncating to i32 for now.
+ %oldval.trunc = trunc i256 %oldval to i32
+ ret i32 %oldval.trunc
+}
+
+; Try a release atomicrmw sub
+; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this.
+define i32 @test_atomicrmw_sub(i256* %addr) {
+; CHECK-LABEL: name: test_atomicrmw_sub
+; CHECK: bb.1 (%ir-block.{{[0-9]+}}):
+; CHECK-NEXT: liveins: $x0
+; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
+; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1
+; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_SUB [[ADDR]](p0), [[VAL]] :: (load store release 32 on %ir.addr)
+; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]]
+ %oldval = atomicrmw sub i256* %addr, i256 1 release
+ ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this
+ ; test so work around it by truncating to i32 for now.
+ %oldval.trunc = trunc i256 %oldval to i32
+ ret i32 %oldval.trunc
+}
+
+; Try an acq_rel atomicrmw and
+; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this.
+define i32 @test_atomicrmw_and(i256* %addr) {
+; CHECK-LABEL: name: test_atomicrmw_and
+; CHECK: bb.1 (%ir-block.{{[0-9]+}}):
+; CHECK-NEXT: liveins: $x0
+; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
+; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1
+; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_AND [[ADDR]](p0), [[VAL]] :: (load store acq_rel 32 on %ir.addr)
+; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]]
+ %oldval = atomicrmw and i256* %addr, i256 1 acq_rel
+ ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this
+ ; test so work around it by truncating to i32 for now.
+ %oldval.trunc = trunc i256 %oldval to i32
+ ret i32 %oldval.trunc
+}
+
+; Try an seq_cst atomicrmw nand
+; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this.
+define i32 @test_atomicrmw_nand(i256* %addr) {
+; CHECK-LABEL: name: test_atomicrmw_nand
+; CHECK: bb.1 (%ir-block.{{[0-9]+}}):
+; CHECK-NEXT: liveins: $x0
+; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
+; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1
+; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_NAND [[ADDR]](p0), [[VAL]] :: (load store seq_cst 32 on %ir.addr)
+; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]]
+ %oldval = atomicrmw nand i256* %addr, i256 1 seq_cst
+ ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this
+ ; test so work around it by truncating to i32 for now.
+ %oldval.trunc = trunc i256 %oldval to i32
+ ret i32 %oldval.trunc
+}
+
+; Try an seq_cst atomicrmw or
+; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this.
+define i32 @test_atomicrmw_or(i256* %addr) {
+; CHECK-LABEL: name: test_atomicrmw_or
+; CHECK: bb.1 (%ir-block.{{[0-9]+}}):
+; CHECK-NEXT: liveins: $x0
+; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
+; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1
+; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_OR [[ADDR]](p0), [[VAL]] :: (load store seq_cst 32 on %ir.addr)
+; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]]
+ %oldval = atomicrmw or i256* %addr, i256 1 seq_cst
+ ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this
+ ; test so work around it by truncating to i32 for now.
+ %oldval.trunc = trunc i256 %oldval to i32
+ ret i32 %oldval.trunc
+}
+
+; Try an seq_cst atomicrmw xor
+; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this.
+define i32 @test_atomicrmw_xor(i256* %addr) {
+; CHECK-LABEL: name: test_atomicrmw_xor
+; CHECK: bb.1 (%ir-block.{{[0-9]+}}):
+; CHECK-NEXT: liveins: $x0
+; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
+; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1
+; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_XOR [[ADDR]](p0), [[VAL]] :: (load store seq_cst 32 on %ir.addr)
+; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]]
+ %oldval = atomicrmw xor i256* %addr, i256 1 seq_cst
+ ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this
+ ; test so work around it by truncating to i32 for now.
+ %oldval.trunc = trunc i256 %oldval to i32
+ ret i32 %oldval.trunc
+}
+
+; Try an seq_cst atomicrmw min
+; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this.
+define i32 @test_atomicrmw_min(i256* %addr) {
+; CHECK-LABEL: name: test_atomicrmw_min
+; CHECK: bb.1 (%ir-block.{{[0-9]+}}):
+; CHECK-NEXT: liveins: $x0
+; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
+; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1
+; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_MIN [[ADDR]](p0), [[VAL]] :: (load store seq_cst 32 on %ir.addr)
+; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]]
+ %oldval = atomicrmw min i256* %addr, i256 1 seq_cst
+ ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this
+ ; test so work around it by truncating to i32 for now.
+ %oldval.trunc = trunc i256 %oldval to i32
+ ret i32 %oldval.trunc
+}
+
+; Try an seq_cst atomicrmw max
+; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this.
+define i32 @test_atomicrmw_max(i256* %addr) {
+; CHECK-LABEL: name: test_atomicrmw_max
+; CHECK: bb.1 (%ir-block.{{[0-9]+}}):
+; CHECK-NEXT: liveins: $x0
+; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
+; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1
+; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_MAX [[ADDR]](p0), [[VAL]] :: (load store seq_cst 32 on %ir.addr)
+; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]]
+ %oldval = atomicrmw max i256* %addr, i256 1 seq_cst
+ ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this
+ ; test so work around it by truncating to i32 for now.
+ %oldval.trunc = trunc i256 %oldval to i32
+ ret i32 %oldval.trunc
+}
+
+; Try an seq_cst atomicrmw unsigned min
+; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this.
+define i32 @test_atomicrmw_umin(i256* %addr) {
+; CHECK-LABEL: name: test_atomicrmw_umin
+; CHECK: bb.1 (%ir-block.{{[0-9]+}}):
+; CHECK-NEXT: liveins: $x0
+; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
+; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1
+; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_UMIN [[ADDR]](p0), [[VAL]] :: (load store seq_cst 32 on %ir.addr)
+; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]]
+ %oldval = atomicrmw umin i256* %addr, i256 1 seq_cst
+ ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this
+ ; test so work around it by truncating to i32 for now.
+ %oldval.trunc = trunc i256 %oldval to i32
+ ret i32 %oldval.trunc
+}
+
+; Try an seq_cst atomicrmw unsigned max
+; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this.
+define i32 @test_atomicrmw_umax(i256* %addr) {
+; CHECK-LABEL: name: test_atomicrmw_umax
+; CHECK: bb.1 (%ir-block.{{[0-9]+}}):
+; CHECK-NEXT: liveins: $x0
+; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
+; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1
+; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_UMAX [[ADDR]](p0), [[VAL]] :: (load store seq_cst 32 on %ir.addr)
+; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]]
+ %oldval = atomicrmw umax i256* %addr, i256 1 seq_cst
+ ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this
+ ; test so work around it by truncating to i32 for now.
+ %oldval.trunc = trunc i256 %oldval to i32
+ ret i32 %oldval.trunc
+}
More information about the llvm-commits
mailing list