<div class="gmail_quote">On Wed, Aug 24, 2011 at 1:50 PM, Eli Friedman <span dir="ltr"><<a href="mailto:eli.friedman@gmail.com">eli.friedman@gmail.com</a>></span> wrote:<br><blockquote class="gmail_quote" style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding-left:1ex;">
Author: efriedma<br>
Date: Wed Aug 24 15:50:09 2011<br>
New Revision: 138478<br>
<br>
URL: <a href="http://llvm.org/viewvc/llvm-project?rev=138478&view=rev" target="_blank">http://llvm.org/viewvc/llvm-project?rev=138478&view=rev</a><br>
Log:<br>
Basic x86 code generation for atomic load and store instructions.<br>
<br>
<br>
Modified:<br>
llvm/trunk/include/llvm/CodeGen/ISDOpcodes.h<br>
llvm/trunk/include/llvm/CodeGen/SelectionDAG.h<br>
llvm/trunk/include/llvm/CodeGen/SelectionDAGNodes.h<br>
llvm/trunk/include/llvm/Target/TargetSelectionDAG.td<br>
llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp<br>
llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAG.cpp<br>
llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp<br>
llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h<br>
llvm/trunk/lib/Target/X86/X86ISelLowering.cpp<br>
llvm/trunk/lib/Target/X86/X86InstrCompiler.td<br></blockquote><div><br></div><div>Wait, no tests? None at all? =[</div><div><br></div><div> </div><blockquote class="gmail_quote" style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding-left:1ex;">
<br>
Modified: llvm/trunk/include/llvm/CodeGen/ISDOpcodes.h<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/CodeGen/ISDOpcodes.h?rev=138478&r1=138477&r2=138478&view=diff" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/CodeGen/ISDOpcodes.h?rev=138478&r1=138477&r2=138478&view=diff</a><br>
==============================================================================<br>
--- llvm/trunk/include/llvm/CodeGen/ISDOpcodes.h (original)<br>
+++ llvm/trunk/include/llvm/CodeGen/ISDOpcodes.h Wed Aug 24 15:50:09 2011<br>
@@ -597,22 +597,22 @@<br>
// two integer constants: an AtomicOrdering and a SynchronizationScope.<br>
ATOMIC_FENCE,<br>
<br>
+ // Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr)<br>
+ // This corresponds to "load atomic" instruction.<br>
+ ATOMIC_LOAD,<br>
+<br>
+ // OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr, val)<br>
+ // This corresponds to "store atomic" instruction.<br>
+ ATOMIC_STORE,<br>
+<br>
// Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap)<br>
- // this corresponds to the atomic.lcs intrinsic.<br>
- // cmp is compared to *ptr, and if equal, swap is stored in *ptr.<br>
- // the return is always the original value in *ptr<br>
+ // This corresponds to the cmpxchg instruction.<br>
ATOMIC_CMP_SWAP,<br>
<br>
// Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt)<br>
- // this corresponds to the atomic.swap intrinsic.<br>
- // amt is stored to *ptr atomically.<br>
- // the return is always the original value in *ptr<br>
- ATOMIC_SWAP,<br>
-<br>
// Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amt)<br>
- // this corresponds to the atomic.load.[OpName] intrinsic.<br>
- // op(*ptr, amt) is stored to *ptr atomically.<br>
- // the return is always the original value in *ptr<br>
+ // These correspond to the atomicrmw instruction.<br>
+ ATOMIC_SWAP,<br>
ATOMIC_LOAD_ADD,<br>
ATOMIC_LOAD_SUB,<br>
ATOMIC_LOAD_AND,<br>
<br>
Modified: llvm/trunk/include/llvm/CodeGen/SelectionDAG.h<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/CodeGen/SelectionDAG.h?rev=138478&r1=138477&r2=138478&view=diff" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/CodeGen/SelectionDAG.h?rev=138478&r1=138477&r2=138478&view=diff</a><br>
==============================================================================<br>
--- llvm/trunk/include/llvm/CodeGen/SelectionDAG.h (original)<br>
+++ llvm/trunk/include/llvm/CodeGen/SelectionDAG.h Wed Aug 24 15:50:09 2011<br>
@@ -598,16 +598,26 @@<br>
AtomicOrdering Ordering,<br>
SynchronizationScope SynchScope);<br>
<br>
- /// getAtomic - Gets a node for an atomic op, produces result and chain and<br>
- /// takes 2 operands.<br>
+ /// getAtomic - Gets a node for an atomic op, produces result (if relevant)<br>
+ /// and chain and takes 2 operands.<br>
SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain,<br>
SDValue Ptr, SDValue Val, const Value* PtrVal,<br>
+ unsigned Alignment, AtomicOrdering Ordering,<br>
+ SynchronizationScope SynchScope);<br>
+ SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain,<br>
+ SDValue Ptr, SDValue Val, MachineMemOperand *MMO,<br>
+ AtomicOrdering Ordering,<br>
+ SynchronizationScope SynchScope);<br>
+<br>
+ /// getAtomic - Gets a node for an atomic op, produces result and chain and<br>
+ /// takes 1 operand.<br>
+ SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, EVT VT,<br>
+ SDValue Chain, SDValue Ptr, const Value* PtrVal,<br>
unsigned Alignment,<br>
AtomicOrdering Ordering,<br>
SynchronizationScope SynchScope);<br>
- SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain,<br>
- SDValue Ptr, SDValue Val,<br>
- MachineMemOperand *MMO,<br>
+ SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, EVT VT,<br>
+ SDValue Chain, SDValue Ptr, MachineMemOperand *MMO,<br>
AtomicOrdering Ordering,<br>
SynchronizationScope SynchScope);<br>
<br>
<br>
Modified: llvm/trunk/include/llvm/CodeGen/SelectionDAGNodes.h<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/CodeGen/SelectionDAGNodes.h?rev=138478&r1=138477&r2=138478&view=diff" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/CodeGen/SelectionDAGNodes.h?rev=138478&r1=138477&r2=138478&view=diff</a><br>
==============================================================================<br>
--- llvm/trunk/include/llvm/CodeGen/SelectionDAGNodes.h (original)<br>
+++ llvm/trunk/include/llvm/CodeGen/SelectionDAGNodes.h Wed Aug 24 15:50:09 2011<br>
@@ -976,6 +976,8 @@<br>
N->getOpcode() == ISD::ATOMIC_LOAD_MAX ||<br>
N->getOpcode() == ISD::ATOMIC_LOAD_UMIN ||<br>
N->getOpcode() == ISD::ATOMIC_LOAD_UMAX ||<br>
+ N->getOpcode() == ISD::ATOMIC_LOAD ||<br>
+ N->getOpcode() == ISD::ATOMIC_STORE ||<br>
N->isTargetMemoryOpcode();<br>
}<br>
};<br>
@@ -1025,6 +1027,14 @@<br>
InitAtomic(Ordering, SynchScope);<br>
InitOperands(Ops, Chain, Ptr, Val);<br>
}<br>
+ AtomicSDNode(unsigned Opc, DebugLoc dl, SDVTList VTL, EVT MemVT,<br>
+ SDValue Chain, SDValue Ptr,<br>
+ MachineMemOperand *MMO,<br>
+ AtomicOrdering Ordering, SynchronizationScope SynchScope)<br>
+ : MemSDNode(Opc, dl, VTL, MemVT, MMO) {<br>
+ InitAtomic(Ordering, SynchScope);<br>
+ InitOperands(Ops, Chain, Ptr);<br>
+ }<br>
<br>
const SDValue &getBasePtr() const { return getOperand(1); }<br>
const SDValue &getVal() const { return getOperand(2); }<br>
@@ -1048,7 +1058,9 @@<br>
N->getOpcode() == ISD::ATOMIC_LOAD_MIN ||<br>
N->getOpcode() == ISD::ATOMIC_LOAD_MAX ||<br>
N->getOpcode() == ISD::ATOMIC_LOAD_UMIN ||<br>
- N->getOpcode() == ISD::ATOMIC_LOAD_UMAX;<br>
+ N->getOpcode() == ISD::ATOMIC_LOAD_UMAX ||<br>
+ N->getOpcode() == ISD::ATOMIC_LOAD ||<br>
+ N->getOpcode() == ISD::ATOMIC_STORE;<br>
}<br>
};<br>
<br>
<br>
Modified: llvm/trunk/include/llvm/Target/TargetSelectionDAG.td<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/Target/TargetSelectionDAG.td?rev=138478&r1=138477&r2=138478&view=diff" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/Target/TargetSelectionDAG.td?rev=138478&r1=138477&r2=138478&view=diff</a><br>
==============================================================================<br>
--- llvm/trunk/include/llvm/Target/TargetSelectionDAG.td (original)<br>
+++ llvm/trunk/include/llvm/Target/TargetSelectionDAG.td Wed Aug 24 15:50:09 2011<br>
@@ -214,6 +214,12 @@<br>
def SDTAtomic2 : SDTypeProfile<1, 2, [<br>
SDTCisSameAs<0,2>, SDTCisInt<0>, SDTCisPtrTy<1><br>
]>;<br>
+def SDTAtomicStore : SDTypeProfile<0, 2, [<br>
+ SDTCisPtrTy<0>, SDTCisInt<1><br>
+]>;<br>
+def SDTAtomicLoad : SDTypeProfile<1, 1, [<br>
+ SDTCisInt<0>, SDTCisPtrTy<1><br>
+]>;<br>
<br>
def SDTConvertOp : SDTypeProfile<1, 5, [ //cvtss, su, us, uu, ff, fs, fu, sf, su<br>
SDTCisVT<2, OtherVT>, SDTCisVT<3, OtherVT>, SDTCisPtrTy<4>, SDTCisPtrTy<5><br>
@@ -427,6 +433,10 @@<br>
[SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;<br>
def atomic_load_umax : SDNode<"ISD::ATOMIC_LOAD_UMAX", SDTAtomic2,<br>
[SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;<br>
+def atomic_load : SDNode<"ISD::ATOMIC_LOAD", SDTAtomicLoad,<br>
+ [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;<br>
+def atomic_store : SDNode<"ISD::ATOMIC_STORE", SDTAtomicStore,<br>
+ [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;<br>
<br>
// Do not use ld, st directly. Use load, extload, sextload, zextload, store,<br>
// and truncst (see below).<br>
@@ -844,6 +854,28 @@<br>
defm atomic_load_max : binary_atomic_op<atomic_load_max>;<br>
defm atomic_load_umin : binary_atomic_op<atomic_load_umin>;<br>
defm atomic_load_umax : binary_atomic_op<atomic_load_umax>;<br>
+defm atomic_store : binary_atomic_op<atomic_store>;<br>
+<br>
+def atomic_load_8 :<br>
+ PatFrag<(ops node:$ptr),<br>
+ (atomic_load node:$ptr), [{<br>
+ return cast<AtomicSDNode>(N)->getMemoryVT() == MVT::i8;<br>
+}]>;<br>
+def atomic_load_16 :<br>
+ PatFrag<(ops node:$ptr),<br>
+ (atomic_load node:$ptr), [{<br>
+ return cast<AtomicSDNode>(N)->getMemoryVT() == MVT::i16;<br>
+}]>;<br>
+def atomic_load_32 :<br>
+ PatFrag<(ops node:$ptr),<br>
+ (atomic_load node:$ptr), [{<br>
+ return cast<AtomicSDNode>(N)->getMemoryVT() == MVT::i32;<br>
+}]>;<br>
+def atomic_load_64 :<br>
+ PatFrag<(ops node:$ptr),<br>
+ (atomic_load node:$ptr), [{<br>
+ return cast<AtomicSDNode>(N)->getMemoryVT() == MVT::i64;<br>
+}]>;<br>
<br>
//===----------------------------------------------------------------------===//<br>
// Selection DAG CONVERT_RNDSAT patterns<br>
<br>
Modified: llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp?rev=138478&r1=138477&r2=138478&view=diff" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp?rev=138478&r1=138477&r2=138478&view=diff</a><br>
==============================================================================<br>
--- llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp (original)<br>
+++ llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp Wed Aug 24 15:50:09 2011<br>
@@ -819,6 +819,11 @@<br>
Action = TLI.getOperationAction(Node->getOpcode(), InnerType);<br>
break;<br>
}<br>
+ case ISD::ATOMIC_STORE: {<br>
+ Action = TLI.getOperationAction(Node->getOpcode(),<br>
+ Node->getOperand(2).getValueType());<br>
+ break;<br>
+ }<br>
case ISD::SELECT_CC:<br>
case ISD::SETCC:<br>
case ISD::BR_CC: {<br>
<br>
Modified: llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAG.cpp<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAG.cpp?rev=138478&r1=138477&r2=138478&view=diff" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAG.cpp?rev=138478&r1=138477&r2=138478&view=diff</a><br>
==============================================================================<br>
--- llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAG.cpp (original)<br>
+++ llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAG.cpp Wed Aug 24 15:50:09 2011<br>
@@ -432,7 +432,9 @@<br>
case ISD::ATOMIC_LOAD_MIN:<br>
case ISD::ATOMIC_LOAD_MAX:<br>
case ISD::ATOMIC_LOAD_UMIN:<br>
- case ISD::ATOMIC_LOAD_UMAX: {<br>
+ case ISD::ATOMIC_LOAD_UMAX:<br>
+ case ISD::ATOMIC_LOAD:<br>
+ case ISD::ATOMIC_STORE: {<br>
const AtomicSDNode *AT = cast<AtomicSDNode>(N);<br>
ID.AddInteger(AT->getMemoryVT().getRawBits());<br>
ID.AddInteger(AT->getRawSubclassData());<br>
@@ -3904,12 +3906,14 @@<br>
Opcode == ISD::ATOMIC_LOAD_MAX ||<br>
Opcode == ISD::ATOMIC_LOAD_UMIN ||<br>
Opcode == ISD::ATOMIC_LOAD_UMAX ||<br>
- Opcode == ISD::ATOMIC_SWAP) &&<br>
+ Opcode == ISD::ATOMIC_SWAP ||<br>
+ Opcode == ISD::ATOMIC_STORE) &&<br>
"Invalid Atomic Op");<br>
<br>
EVT VT = Val.getValueType();<br>
<br>
- SDVTList VTs = getVTList(VT, MVT::Other);<br>
+ SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) :<br>
+ getVTList(VT, MVT::Other);<br>
FoldingSetNodeID ID;<br>
ID.AddInteger(MemVT.getRawBits());<br>
SDValue Ops[] = {Chain, Ptr, Val};<br>
@@ -3927,6 +3931,55 @@<br>
return SDValue(N, 0);<br>
}<br>
<br>
+SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,<br>
+ EVT VT, SDValue Chain,<br>
+ SDValue Ptr,<br>
+ const Value* PtrVal,<br>
+ unsigned Alignment,<br>
+ AtomicOrdering Ordering,<br>
+ SynchronizationScope SynchScope) {<br>
+ if (Alignment == 0) // Ensure that codegen never sees alignment 0<br>
+ Alignment = getEVTAlignment(MemVT);<br>
+<br>
+ MachineFunction &MF = getMachineFunction();<br>
+ unsigned Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;<br>
+<br>
+ // For now, atomics are considered to be volatile always.<br>
+ Flags |= MachineMemOperand::MOVolatile;<br>
+<br>
+ MachineMemOperand *MMO =<br>
+ MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags,<br>
+ MemVT.getStoreSize(), Alignment);<br>
+<br>
+ return getAtomic(Opcode, dl, MemVT, VT, Chain, Ptr, MMO,<br>
+ Ordering, SynchScope);<br>
+}<br>
+<br>
+SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,<br>
+ EVT VT, SDValue Chain,<br>
+ SDValue Ptr,<br>
+ MachineMemOperand *MMO,<br>
+ AtomicOrdering Ordering,<br>
+ SynchronizationScope SynchScope) {<br>
+ assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op");<br>
+<br>
+ SDVTList VTs = getVTList(VT, MVT::Other);<br>
+ FoldingSetNodeID ID;<br>
+ ID.AddInteger(MemVT.getRawBits());<br>
+ SDValue Ops[] = {Chain, Ptr};<br>
+ AddNodeIDNode(ID, Opcode, VTs, Ops, 2);<br>
+ void* IP = 0;<br>
+ if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {<br>
+ cast<AtomicSDNode>(E)->refineAlignment(MMO);<br>
+ return SDValue(E, 0);<br>
+ }<br>
+ SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain,<br>
+ Ptr, MMO, Ordering, SynchScope);<br>
+ CSEMap.InsertNode(N, IP);<br>
+ AllNodes.push_back(N);<br>
+ return SDValue(N, 0);<br>
+}<br>
+<br>
/// getMergeValues - Create a MERGE_VALUES node from the given operands.<br>
SDValue SelectionDAG::getMergeValues(const SDValue *Ops, unsigned NumOps,<br>
DebugLoc dl) {<br>
@@ -5795,6 +5848,8 @@<br>
case ISD::ATOMIC_LOAD_MAX: return "AtomicLoadMax";<br>
case ISD::ATOMIC_LOAD_UMIN: return "AtomicLoadUMin";<br>
case ISD::ATOMIC_LOAD_UMAX: return "AtomicLoadUMax";<br>
+ case ISD::ATOMIC_LOAD: return "AtomicLoad";<br>
+ case ISD::ATOMIC_STORE: return "AtomicStore";<br>
case ISD::PCMARKER: return "PCMarker";<br>
case ISD::READCYCLECOUNTER: return "ReadCycleCounter";<br>
case ISD::SRCVALUE: return "SrcValue";<br>
<br>
Modified: llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp?rev=138478&r1=138477&r2=138478&view=diff" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp?rev=138478&r1=138477&r2=138478&view=diff</a><br>
==============================================================================<br>
--- llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp (original)<br>
+++ llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp Wed Aug 24 15:50:09 2011<br>
@@ -3149,6 +3149,9 @@<br>
}<br>
<br>
void SelectionDAGBuilder::visitLoad(const LoadInst &I) {<br>
+ if (I.isAtomic())<br>
+ return visitAtomicLoad(I);<br>
+<br>
const Value *SV = I.getOperand(0);<br>
SDValue Ptr = getValue(SV);<br>
<br>
@@ -3226,6 +3229,9 @@<br>
}<br>
<br>
void SelectionDAGBuilder::visitStore(const StoreInst &I) {<br>
+ if (I.isAtomic())<br>
+ return visitAtomicStore(I);<br>
+<br>
const Value *SrcV = I.getOperand(0);<br>
const Value *PtrV = I.getOperand(1);<br>
<br>
@@ -3277,6 +3283,7 @@<br>
}<br>
<br>
static SDValue InsertFenceForAtomic(SDValue Chain, AtomicOrdering Order,<br>
+ SynchronizationScope Scope,<br>
bool Before, DebugLoc dl,<br>
SelectionDAG &DAG,<br>
const TargetLowering &TLI) {<br>
@@ -3294,19 +3301,21 @@<br>
}<br>
SDValue Ops[3];<br>
Ops[0] = Chain;<br>
- Ops[1] = DAG.getConstant(SequentiallyConsistent, TLI.getPointerTy());<br>
- Ops[2] = DAG.getConstant(Order, TLI.getPointerTy());<br>
+ Ops[1] = DAG.getConstant(Order, TLI.getPointerTy());<br>
+ Ops[2] = DAG.getConstant(Scope, TLI.getPointerTy());<br>
return DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops, 3);<br>
}<br>
<br>
void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {<br>
DebugLoc dl = getCurDebugLoc();<br>
AtomicOrdering Order = I.getOrdering();<br>
+ SynchronizationScope Scope = I.getSynchScope();<br>
<br>
SDValue InChain = getRoot();<br>
<br>
if (TLI.getInsertFencesForAtomic())<br>
- InChain = InsertFenceForAtomic(InChain, Order, true, dl, DAG, TLI);<br>
+ InChain = InsertFenceForAtomic(InChain, Order, Scope, true, dl,<br>
+ DAG, TLI);<br>
<br>
SDValue L =<br>
DAG.getAtomic(ISD::ATOMIC_CMP_SWAP, dl,<br>
@@ -3316,12 +3325,14 @@<br>
getValue(I.getCompareOperand()),<br>
getValue(I.getNewValOperand()),<br>
MachinePointerInfo(I.getPointerOperand()), 0 /* Alignment */,<br>
- I.getOrdering(), I.getSynchScope());<br>
+ TLI.getInsertFencesForAtomic() ? Monotonic : Order,<br>
+ Scope);<br>
<br>
SDValue OutChain = L.getValue(1);<br>
<br>
if (TLI.getInsertFencesForAtomic())<br>
- OutChain = InsertFenceForAtomic(OutChain, Order, false, dl, DAG, TLI);<br>
+ OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl,<br>
+ DAG, TLI);<br>
<br>
setValue(&I, L);<br>
DAG.setRoot(OutChain);<br>
@@ -3345,11 +3356,13 @@<br>
case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;<br>
}<br>
AtomicOrdering Order = I.getOrdering();<br>
+ SynchronizationScope Scope = I.getSynchScope();<br>
<br>
SDValue InChain = getRoot();<br>
<br>
if (TLI.getInsertFencesForAtomic())<br>
- InChain = InsertFenceForAtomic(InChain, Order, true, dl, DAG, TLI);<br>
+ InChain = InsertFenceForAtomic(InChain, Order, Scope, true, dl,<br>
+ DAG, TLI);<br>
<br>
SDValue L =<br>
DAG.getAtomic(NT, dl,<br>
@@ -3359,12 +3372,13 @@<br>
getValue(I.getValOperand()),<br>
I.getPointerOperand(), 0 /* Alignment */,<br>
TLI.getInsertFencesForAtomic() ? Monotonic : Order,<br>
- I.getSynchScope());<br>
+ Scope);<br>
<br>
SDValue OutChain = L.getValue(1);<br>
<br>
if (TLI.getInsertFencesForAtomic())<br>
- OutChain = InsertFenceForAtomic(OutChain, Order, false, dl, DAG, TLI);<br>
+ OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl,<br>
+ DAG, TLI);<br>
<br>
setValue(&I, L);<br>
DAG.setRoot(OutChain);<br>
@@ -3379,6 +3393,65 @@<br>
DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops, 3));<br>
}<br>
<br>
+void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {<br>
+ DebugLoc dl = getCurDebugLoc();<br>
+ AtomicOrdering Order = I.getOrdering();<br>
+ SynchronizationScope Scope = I.getSynchScope();<br>
+<br>
+ SDValue InChain = getRoot();<br>
+<br>
+ if (TLI.getInsertFencesForAtomic())<br>
+ InChain = InsertFenceForAtomic(InChain, Order, Scope, true, dl,<br>
+ DAG, TLI);<br>
+<br>
+ EVT VT = EVT::getEVT(I.getType());<br>
+<br>
+ SDValue L =<br>
+ DAG.getAtomic(ISD::ATOMIC_LOAD, dl, VT, VT, InChain,<br>
+ getValue(I.getPointerOperand()),<br>
+ I.getPointerOperand(), I.getAlignment(),<br>
+ TLI.getInsertFencesForAtomic() ? Monotonic : Order,<br>
+ Scope);<br>
+<br>
+ SDValue OutChain = L.getValue(1);<br>
+<br>
+ if (TLI.getInsertFencesForAtomic())<br>
+ OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl,<br>
+ DAG, TLI);<br>
+<br>
+ setValue(&I, L);<br>
+ DAG.setRoot(OutChain);<br>
+}<br>
+<br>
+void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {<br>
+ DebugLoc dl = getCurDebugLoc();<br>
+<br>
+ AtomicOrdering Order = I.getOrdering();<br>
+ SynchronizationScope Scope = I.getSynchScope();<br>
+<br>
+ SDValue InChain = getRoot();<br>
+<br>
+ if (TLI.getInsertFencesForAtomic())<br>
+ InChain = InsertFenceForAtomic(InChain, Order, Scope, true, dl,<br>
+ DAG, TLI);<br>
+<br>
+ SDValue OutChain =<br>
+ DAG.getAtomic(ISD::ATOMIC_STORE, dl,<br>
+ getValue(I.getValueOperand()).getValueType().getSimpleVT(),<br>
+ InChain,<br>
+ getValue(I.getPointerOperand()),<br>
+ getValue(I.getValueOperand()),<br>
+ I.getPointerOperand(), I.getAlignment(),<br>
+ TLI.getInsertFencesForAtomic() ? Monotonic : Order,<br>
+ Scope);<br>
+<br>
+ if (TLI.getInsertFencesForAtomic())<br>
+ OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl,<br>
+ DAG, TLI);<br>
+<br>
+ DAG.setRoot(OutChain);<br>
+}<br>
+<br>
/// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC<br>
/// node.<br>
void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,<br>
<br>
Modified: llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h?rev=138478&r1=138477&r2=138478&view=diff" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h?rev=138478&r1=138477&r2=138478&view=diff</a><br>
==============================================================================<br>
--- llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h (original)<br>
+++ llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h Wed Aug 24 15:50:09 2011<br>
@@ -526,7 +526,9 @@<br>
void visitPHI(const PHINode &I);<br>
void visitCall(const CallInst &I);<br>
bool visitMemCmpCall(const CallInst &I);<br>
-<br>
+ void visitAtomicLoad(const LoadInst &I);<br>
+ void visitAtomicStore(const StoreInst &I);<br>
+<br>
void visitInlineAsm(ImmutableCallSite CS);<br>
const char *visitIntrinsicCall(const CallInst &I, unsigned Intrinsic);<br>
void visitTargetIntrinsic(const CallInst &I, unsigned Intrinsic);<br>
<br>
Modified: llvm/trunk/lib/Target/X86/X86ISelLowering.cpp<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/X86/X86ISelLowering.cpp?rev=138478&r1=138477&r2=138478&view=diff" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/X86/X86ISelLowering.cpp?rev=138478&r1=138477&r2=138478&view=diff</a><br>
==============================================================================<br>
--- llvm/trunk/lib/Target/X86/X86ISelLowering.cpp (original)<br>
+++ llvm/trunk/lib/Target/X86/X86ISelLowering.cpp Wed Aug 24 15:50:09 2011<br>
@@ -464,6 +464,7 @@<br>
MVT VT = IntVTs[i];<br>
setOperationAction(ISD::ATOMIC_CMP_SWAP, VT, Custom);<br>
setOperationAction(ISD::ATOMIC_LOAD_SUB, VT, Custom);<br>
+ setOperationAction(ISD::ATOMIC_STORE, VT, Custom);<br>
}<br>
<br>
if (!Subtarget->is64Bit()) {<br>
@@ -9999,6 +10000,26 @@<br>
cast<AtomicSDNode>(Node)->getSynchScope());<br>
}<br>
<br>
+static SDValue LowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG) {<br>
+ SDNode *Node = Op.getNode();<br>
+ DebugLoc dl = Node->getDebugLoc();<br>
+<br>
+ // Convert seq_cst store -> xchg<br>
+ if (cast<AtomicSDNode>(Node)->getOrdering() == SequentiallyConsistent) {<br>
+ SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,<br>
+ cast<AtomicSDNode>(Node)->getMemoryVT(),<br>
+ Node->getOperand(0),<br>
+ Node->getOperand(1), Node->getOperand(2),<br>
+ cast<AtomicSDNode>(Node)->getSrcValue(),<br>
+ cast<AtomicSDNode>(Node)->getAlignment(),<br>
+ cast<AtomicSDNode>(Node)->getOrdering(),<br>
+ cast<AtomicSDNode>(Node)->getSynchScope());<br>
+ return Swap.getValue(1);<br>
+ }<br>
+ // Other atomic stores have a simple pattern.<br>
+ return Op;<br>
+}<br>
+<br>
static SDValue LowerADDC_ADDE_SUBC_SUBE(SDValue Op, SelectionDAG &DAG) {<br>
EVT VT = Op.getNode()->getValueType(0);<br>
<br>
@@ -10035,6 +10056,7 @@<br>
case ISD::ATOMIC_FENCE: return LowerATOMIC_FENCE(Op,DAG);<br>
case ISD::ATOMIC_CMP_SWAP: return LowerCMP_SWAP(Op,DAG);<br>
case ISD::ATOMIC_LOAD_SUB: return LowerLOAD_SUB(Op,DAG);<br>
+ case ISD::ATOMIC_STORE: return LowerATOMIC_STORE(Op,DAG);<br>
case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG);<br>
case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG);<br>
case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG);<br>
<br>
Modified: llvm/trunk/lib/Target/X86/X86InstrCompiler.td<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/X86/X86InstrCompiler.td?rev=138478&r1=138477&r2=138478&view=diff" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/X86/X86InstrCompiler.td?rev=138478&r1=138477&r2=138478&view=diff</a><br>
==============================================================================<br>
--- llvm/trunk/lib/Target/X86/X86InstrCompiler.td (original)<br>
+++ llvm/trunk/lib/Target/X86/X86InstrCompiler.td Wed Aug 24 15:50:09 2011<br>
@@ -1691,3 +1691,17 @@<br>
(AND64ri8 GR64:$src1, i64immSExt8:$src2)>;<br>
def : Pat<(and GR64:$src1, i64immSExt32:$src2),<br>
(AND64ri32 GR64:$src1, i64immSExt32:$src2)>;<br>
+<br>
+def : Pat<(atomic_load_8 addr:$src), (MOV8rm addr:$src)>;<br>
+def : Pat<(atomic_load_16 addr:$src), (MOV16rm addr:$src)>;<br>
+def : Pat<(atomic_load_32 addr:$src), (MOV32rm addr:$src)>;<br>
+def : Pat<(atomic_load_64 addr:$src), (MOV64rm addr:$src)>;<br>
+<br>
+def : Pat<(atomic_store_8 addr:$ptr, GR8:$val),<br>
+ (MOV8mr addr:$ptr, GR8:$val)>;<br>
+def : Pat<(atomic_store_16 addr:$ptr, GR16:$val),<br>
+ (MOV16mr addr:$ptr, GR16:$val)>;<br>
+def : Pat<(atomic_store_32 addr:$ptr, GR32:$val),<br>
+ (MOV32mr addr:$ptr, GR32:$val)>;<br>
+def : Pat<(atomic_store_64 addr:$ptr, GR64:$val),<br>
+ (MOV64mr addr:$ptr, GR64:$val)>;<br>
<br>
<br>
_______________________________________________<br>
llvm-commits mailing list<br>
<a href="mailto:llvm-commits@cs.uiuc.edu">llvm-commits@cs.uiuc.edu</a><br>
<a href="http://lists.cs.uiuc.edu/mailman/listinfo/llvm-commits" target="_blank">http://lists.cs.uiuc.edu/mailman/listinfo/llvm-commits</a><br>
</blockquote></div><br>