<div class="gmail_quote">On Wed, Aug 24, 2011 at 1:50 PM, Eli Friedman <span dir="ltr"><<a href="mailto:eli.friedman@gmail.com">eli.friedman@gmail.com</a>></span> wrote:<br><blockquote class="gmail_quote" style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding-left:1ex;">
Author: efriedma<br>
Date: Wed Aug 24 15:50:09 2011<br>
New Revision: 138478<br>
<br>
URL: <a href="http://llvm.org/viewvc/llvm-project?rev=138478&view=rev" target="_blank">http://llvm.org/viewvc/llvm-project?rev=138478&view=rev</a><br>
Log:<br>
Basic x86 code generation for atomic load and store instructions.<br>
<br>
<br>
Modified:<br>
    llvm/trunk/include/llvm/CodeGen/ISDOpcodes.h<br>
    llvm/trunk/include/llvm/CodeGen/SelectionDAG.h<br>
    llvm/trunk/include/llvm/CodeGen/SelectionDAGNodes.h<br>
    llvm/trunk/include/llvm/Target/TargetSelectionDAG.td<br>
    llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp<br>
    llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAG.cpp<br>
    llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp<br>
    llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h<br>
    llvm/trunk/lib/Target/X86/X86ISelLowering.cpp<br>
    llvm/trunk/lib/Target/X86/X86InstrCompiler.td<br></blockquote><div><br></div><div>Wait, no tests? None at all? =[</div><div><br></div><div> </div><blockquote class="gmail_quote" style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding-left:1ex;">

<br>
Modified: llvm/trunk/include/llvm/CodeGen/ISDOpcodes.h<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/CodeGen/ISDOpcodes.h?rev=138478&r1=138477&r2=138478&view=diff" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/CodeGen/ISDOpcodes.h?rev=138478&r1=138477&r2=138478&view=diff</a><br>

==============================================================================<br>
--- llvm/trunk/include/llvm/CodeGen/ISDOpcodes.h (original)<br>
+++ llvm/trunk/include/llvm/CodeGen/ISDOpcodes.h Wed Aug 24 15:50:09 2011<br>
@@ -597,22 +597,22 @@<br>
     // two integer constants: an AtomicOrdering and a SynchronizationScope.<br>
     ATOMIC_FENCE,<br>
<br>
+    // Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr)<br>
+    // This corresponds to "load atomic" instruction.<br>
+    ATOMIC_LOAD,<br>
+<br>
+    // OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr, val)<br>
+    // This corresponds to "store atomic" instruction.<br>
+    ATOMIC_STORE,<br>
+<br>
     // Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap)<br>
-    // this corresponds to the atomic.lcs intrinsic.<br>
-    // cmp is compared to *ptr, and if equal, swap is stored in *ptr.<br>
-    // the return is always the original value in *ptr<br>
+    // This corresponds to the cmpxchg instruction.<br>
     ATOMIC_CMP_SWAP,<br>
<br>
     // Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt)<br>
-    // this corresponds to the atomic.swap intrinsic.<br>
-    // amt is stored to *ptr atomically.<br>
-    // the return is always the original value in *ptr<br>
-    ATOMIC_SWAP,<br>
-<br>
     // Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amt)<br>
-    // this corresponds to the atomic.load.[OpName] intrinsic.<br>
-    // op(*ptr, amt) is stored to *ptr atomically.<br>
-    // the return is always the original value in *ptr<br>
+    // These correspond to the atomicrmw instruction.<br>
+    ATOMIC_SWAP,<br>
     ATOMIC_LOAD_ADD,<br>
     ATOMIC_LOAD_SUB,<br>
     ATOMIC_LOAD_AND,<br>
<br>
Modified: llvm/trunk/include/llvm/CodeGen/SelectionDAG.h<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/CodeGen/SelectionDAG.h?rev=138478&r1=138477&r2=138478&view=diff" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/CodeGen/SelectionDAG.h?rev=138478&r1=138477&r2=138478&view=diff</a><br>

==============================================================================<br>
--- llvm/trunk/include/llvm/CodeGen/SelectionDAG.h (original)<br>
+++ llvm/trunk/include/llvm/CodeGen/SelectionDAG.h Wed Aug 24 15:50:09 2011<br>
@@ -598,16 +598,26 @@<br>
                     AtomicOrdering Ordering,<br>
                     SynchronizationScope SynchScope);<br>
<br>
-  /// getAtomic - Gets a node for an atomic op, produces result and chain and<br>
-  /// takes 2 operands.<br>
+  /// getAtomic - Gets a node for an atomic op, produces result (if relevant)<br>
+  /// and chain and takes 2 operands.<br>
   SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain,<br>
                     SDValue Ptr, SDValue Val, const Value* PtrVal,<br>
+                    unsigned Alignment, AtomicOrdering Ordering,<br>
+                    SynchronizationScope SynchScope);<br>
+  SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain,<br>
+                    SDValue Ptr, SDValue Val, MachineMemOperand *MMO,<br>
+                    AtomicOrdering Ordering,<br>
+                    SynchronizationScope SynchScope);<br>
+<br>
+  /// getAtomic - Gets a node for an atomic op, produces result and chain and<br>
+  /// takes 1 operand.<br>
+  SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, EVT VT,<br>
+                    SDValue Chain, SDValue Ptr, const Value* PtrVal,<br>
                     unsigned Alignment,<br>
                     AtomicOrdering Ordering,<br>
                     SynchronizationScope SynchScope);<br>
-  SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain,<br>
-                    SDValue Ptr, SDValue Val,<br>
-                    MachineMemOperand *MMO,<br>
+  SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, EVT VT,<br>
+                    SDValue Chain, SDValue Ptr, MachineMemOperand *MMO,<br>
                     AtomicOrdering Ordering,<br>
                     SynchronizationScope SynchScope);<br>
<br>
<br>
Modified: llvm/trunk/include/llvm/CodeGen/SelectionDAGNodes.h<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/CodeGen/SelectionDAGNodes.h?rev=138478&r1=138477&r2=138478&view=diff" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/CodeGen/SelectionDAGNodes.h?rev=138478&r1=138477&r2=138478&view=diff</a><br>

==============================================================================<br>
--- llvm/trunk/include/llvm/CodeGen/SelectionDAGNodes.h (original)<br>
+++ llvm/trunk/include/llvm/CodeGen/SelectionDAGNodes.h Wed Aug 24 15:50:09 2011<br>
@@ -976,6 +976,8 @@<br>
            N->getOpcode() == ISD::ATOMIC_LOAD_MAX     ||<br>
            N->getOpcode() == ISD::ATOMIC_LOAD_UMIN    ||<br>
            N->getOpcode() == ISD::ATOMIC_LOAD_UMAX    ||<br>
+           N->getOpcode() == ISD::ATOMIC_LOAD         ||<br>
+           N->getOpcode() == ISD::ATOMIC_STORE        ||<br>
            N->isTargetMemoryOpcode();<br>
   }<br>
 };<br>
@@ -1025,6 +1027,14 @@<br>
     InitAtomic(Ordering, SynchScope);<br>
     InitOperands(Ops, Chain, Ptr, Val);<br>
   }<br>
+  AtomicSDNode(unsigned Opc, DebugLoc dl, SDVTList VTL, EVT MemVT,<br>
+               SDValue Chain, SDValue Ptr,<br>
+               MachineMemOperand *MMO,<br>
+               AtomicOrdering Ordering, SynchronizationScope SynchScope)<br>
+    : MemSDNode(Opc, dl, VTL, MemVT, MMO) {<br>
+    InitAtomic(Ordering, SynchScope);<br>
+    InitOperands(Ops, Chain, Ptr);<br>
+  }<br>
<br>
   const SDValue &getBasePtr() const { return getOperand(1); }<br>
   const SDValue &getVal() const { return getOperand(2); }<br>
@@ -1048,7 +1058,9 @@<br>
            N->getOpcode() == ISD::ATOMIC_LOAD_MIN     ||<br>
            N->getOpcode() == ISD::ATOMIC_LOAD_MAX     ||<br>
            N->getOpcode() == ISD::ATOMIC_LOAD_UMIN    ||<br>
-           N->getOpcode() == ISD::ATOMIC_LOAD_UMAX;<br>
+           N->getOpcode() == ISD::ATOMIC_LOAD_UMAX    ||<br>
+           N->getOpcode() == ISD::ATOMIC_LOAD         ||<br>
+           N->getOpcode() == ISD::ATOMIC_STORE;<br>
   }<br>
 };<br>
<br>
<br>
Modified: llvm/trunk/include/llvm/Target/TargetSelectionDAG.td<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/Target/TargetSelectionDAG.td?rev=138478&r1=138477&r2=138478&view=diff" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/Target/TargetSelectionDAG.td?rev=138478&r1=138477&r2=138478&view=diff</a><br>

==============================================================================<br>
--- llvm/trunk/include/llvm/Target/TargetSelectionDAG.td (original)<br>
+++ llvm/trunk/include/llvm/Target/TargetSelectionDAG.td Wed Aug 24 15:50:09 2011<br>
@@ -214,6 +214,12 @@<br>
 def SDTAtomic2 : SDTypeProfile<1, 2, [<br>
   SDTCisSameAs<0,2>, SDTCisInt<0>, SDTCisPtrTy<1><br>
 ]>;<br>
+def SDTAtomicStore : SDTypeProfile<0, 2, [<br>
+  SDTCisPtrTy<0>, SDTCisInt<1><br>
+]>;<br>
+def SDTAtomicLoad : SDTypeProfile<1, 1, [<br>
+  SDTCisInt<0>, SDTCisPtrTy<1><br>
+]>;<br>
<br>
 def SDTConvertOp : SDTypeProfile<1, 5, [ //cvtss, su, us, uu, ff, fs, fu, sf, su<br>
   SDTCisVT<2, OtherVT>, SDTCisVT<3, OtherVT>, SDTCisPtrTy<4>, SDTCisPtrTy<5><br>
@@ -427,6 +433,10 @@<br>
                     [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;<br>
 def atomic_load_umax : SDNode<"ISD::ATOMIC_LOAD_UMAX", SDTAtomic2,<br>
                     [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;<br>
+def atomic_load      : SDNode<"ISD::ATOMIC_LOAD", SDTAtomicLoad,<br>
+                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;<br>
+def atomic_store     : SDNode<"ISD::ATOMIC_STORE", SDTAtomicStore,<br>
+                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;<br>
<br>
 // Do not use ld, st directly. Use load, extload, sextload, zextload, store,<br>
 // and truncst (see below).<br>
@@ -844,6 +854,28 @@<br>
 defm atomic_load_max  : binary_atomic_op<atomic_load_max>;<br>
 defm atomic_load_umin : binary_atomic_op<atomic_load_umin>;<br>
 defm atomic_load_umax : binary_atomic_op<atomic_load_umax>;<br>
+defm atomic_store     : binary_atomic_op<atomic_store>;<br>
+<br>
+def atomic_load_8 :<br>
+  PatFrag<(ops node:$ptr),<br>
+          (atomic_load node:$ptr), [{<br>
+  return cast<AtomicSDNode>(N)->getMemoryVT() == MVT::i8;<br>
+}]>;<br>
+def atomic_load_16 :<br>
+  PatFrag<(ops node:$ptr),<br>
+          (atomic_load node:$ptr), [{<br>
+  return cast<AtomicSDNode>(N)->getMemoryVT() == MVT::i16;<br>
+}]>;<br>
+def atomic_load_32 :<br>
+  PatFrag<(ops node:$ptr),<br>
+          (atomic_load node:$ptr), [{<br>
+  return cast<AtomicSDNode>(N)->getMemoryVT() == MVT::i32;<br>
+}]>;<br>
+def atomic_load_64 :<br>
+  PatFrag<(ops node:$ptr),<br>
+          (atomic_load node:$ptr), [{<br>
+  return cast<AtomicSDNode>(N)->getMemoryVT() == MVT::i64;<br>
+}]>;<br>
<br>
 //===----------------------------------------------------------------------===//<br>
 // Selection DAG CONVERT_RNDSAT patterns<br>
<br>
Modified: llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp?rev=138478&r1=138477&r2=138478&view=diff" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp?rev=138478&r1=138477&r2=138478&view=diff</a><br>

==============================================================================<br>
--- llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp (original)<br>
+++ llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp Wed Aug 24 15:50:09 2011<br>
@@ -819,6 +819,11 @@<br>
     Action = TLI.getOperationAction(Node->getOpcode(), InnerType);<br>
     break;<br>
   }<br>
+  case ISD::ATOMIC_STORE: {<br>
+    Action = TLI.getOperationAction(Node->getOpcode(),<br>
+                                    Node->getOperand(2).getValueType());<br>
+    break;<br>
+  }<br>
   case ISD::SELECT_CC:<br>
   case ISD::SETCC:<br>
   case ISD::BR_CC: {<br>
<br>
Modified: llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAG.cpp<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAG.cpp?rev=138478&r1=138477&r2=138478&view=diff" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAG.cpp?rev=138478&r1=138477&r2=138478&view=diff</a><br>

==============================================================================<br>
--- llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAG.cpp (original)<br>
+++ llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAG.cpp Wed Aug 24 15:50:09 2011<br>
@@ -432,7 +432,9 @@<br>
   case ISD::ATOMIC_LOAD_MIN:<br>
   case ISD::ATOMIC_LOAD_MAX:<br>
   case ISD::ATOMIC_LOAD_UMIN:<br>
-  case ISD::ATOMIC_LOAD_UMAX: {<br>
+  case ISD::ATOMIC_LOAD_UMAX:<br>
+  case ISD::ATOMIC_LOAD:<br>
+  case ISD::ATOMIC_STORE: {<br>
     const AtomicSDNode *AT = cast<AtomicSDNode>(N);<br>
     ID.AddInteger(AT->getMemoryVT().getRawBits());<br>
     ID.AddInteger(AT->getRawSubclassData());<br>
@@ -3904,12 +3906,14 @@<br>
           Opcode == ISD::ATOMIC_LOAD_MAX ||<br>
           Opcode == ISD::ATOMIC_LOAD_UMIN ||<br>
           Opcode == ISD::ATOMIC_LOAD_UMAX ||<br>
-          Opcode == ISD::ATOMIC_SWAP) &&<br>
+          Opcode == ISD::ATOMIC_SWAP ||<br>
+          Opcode == ISD::ATOMIC_STORE) &&<br>
          "Invalid Atomic Op");<br>
<br>
   EVT VT = Val.getValueType();<br>
<br>
-  SDVTList VTs = getVTList(VT, MVT::Other);<br>
+  SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) :<br>
+                                               getVTList(VT, MVT::Other);<br>
   FoldingSetNodeID ID;<br>
   ID.AddInteger(MemVT.getRawBits());<br>
   SDValue Ops[] = {Chain, Ptr, Val};<br>
@@ -3927,6 +3931,55 @@<br>
   return SDValue(N, 0);<br>
 }<br>
<br>
+SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,<br>
+                                EVT VT, SDValue Chain,<br>
+                                SDValue Ptr,<br>
+                                const Value* PtrVal,<br>
+                                unsigned Alignment,<br>
+                                AtomicOrdering Ordering,<br>
+                                SynchronizationScope SynchScope) {<br>
+  if (Alignment == 0)  // Ensure that codegen never sees alignment 0<br>
+    Alignment = getEVTAlignment(MemVT);<br>
+<br>
+  MachineFunction &MF = getMachineFunction();<br>
+  unsigned Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;<br>
+<br>
+  // For now, atomics are considered to be volatile always.<br>
+  Flags |= MachineMemOperand::MOVolatile;<br>
+<br>
+  MachineMemOperand *MMO =<br>
+    MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags,<br>
+                            MemVT.getStoreSize(), Alignment);<br>
+<br>
+  return getAtomic(Opcode, dl, MemVT, VT, Chain, Ptr, MMO,<br>
+                   Ordering, SynchScope);<br>
+}<br>
+<br>
+SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,<br>
+                                EVT VT, SDValue Chain,<br>
+                                SDValue Ptr,<br>
+                                MachineMemOperand *MMO,<br>
+                                AtomicOrdering Ordering,<br>
+                                SynchronizationScope SynchScope) {<br>
+  assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op");<br>
+<br>
+  SDVTList VTs = getVTList(VT, MVT::Other);<br>
+  FoldingSetNodeID ID;<br>
+  ID.AddInteger(MemVT.getRawBits());<br>
+  SDValue Ops[] = {Chain, Ptr};<br>
+  AddNodeIDNode(ID, Opcode, VTs, Ops, 2);<br>
+  void* IP = 0;<br>
+  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {<br>
+    cast<AtomicSDNode>(E)->refineAlignment(MMO);<br>
+    return SDValue(E, 0);<br>
+  }<br>
+  SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain,<br>
+                                               Ptr, MMO, Ordering, SynchScope);<br>
+  CSEMap.InsertNode(N, IP);<br>
+  AllNodes.push_back(N);<br>
+  return SDValue(N, 0);<br>
+}<br>
+<br>
 /// getMergeValues - Create a MERGE_VALUES node from the given operands.<br>
 SDValue SelectionDAG::getMergeValues(const SDValue *Ops, unsigned NumOps,<br>
                                      DebugLoc dl) {<br>
@@ -5795,6 +5848,8 @@<br>
   case ISD::ATOMIC_LOAD_MAX:    return "AtomicLoadMax";<br>
   case ISD::ATOMIC_LOAD_UMIN:   return "AtomicLoadUMin";<br>
   case ISD::ATOMIC_LOAD_UMAX:   return "AtomicLoadUMax";<br>
+  case ISD::ATOMIC_LOAD:        return "AtomicLoad";<br>
+  case ISD::ATOMIC_STORE:       return "AtomicStore";<br>
   case ISD::PCMARKER:      return "PCMarker";<br>
   case ISD::READCYCLECOUNTER: return "ReadCycleCounter";<br>
   case ISD::SRCVALUE:      return "SrcValue";<br>
<br>
Modified: llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp?rev=138478&r1=138477&r2=138478&view=diff" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp?rev=138478&r1=138477&r2=138478&view=diff</a><br>

==============================================================================<br>
--- llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp (original)<br>
+++ llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp Wed Aug 24 15:50:09 2011<br>
@@ -3149,6 +3149,9 @@<br>
 }<br>
<br>
 void SelectionDAGBuilder::visitLoad(const LoadInst &I) {<br>
+  if (I.isAtomic())<br>
+    return visitAtomicLoad(I);<br>
+<br>
   const Value *SV = I.getOperand(0);<br>
   SDValue Ptr = getValue(SV);<br>
<br>
@@ -3226,6 +3229,9 @@<br>
 }<br>
<br>
 void SelectionDAGBuilder::visitStore(const StoreInst &I) {<br>
+  if (I.isAtomic())<br>
+    return visitAtomicStore(I);<br>
+<br>
   const Value *SrcV = I.getOperand(0);<br>
   const Value *PtrV = I.getOperand(1);<br>
<br>
@@ -3277,6 +3283,7 @@<br>
 }<br>
<br>
 static SDValue InsertFenceForAtomic(SDValue Chain, AtomicOrdering Order,<br>
+                                    SynchronizationScope Scope,<br>
                                     bool Before, DebugLoc dl,<br>
                                     SelectionDAG &DAG,<br>
                                     const TargetLowering &TLI) {<br>
@@ -3294,19 +3301,21 @@<br>
   }<br>
   SDValue Ops[3];<br>
   Ops[0] = Chain;<br>
-  Ops[1] = DAG.getConstant(SequentiallyConsistent, TLI.getPointerTy());<br>
-  Ops[2] = DAG.getConstant(Order, TLI.getPointerTy());<br>
+  Ops[1] = DAG.getConstant(Order, TLI.getPointerTy());<br>
+  Ops[2] = DAG.getConstant(Scope, TLI.getPointerTy());<br>
   return DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops, 3);<br>
 }<br>
<br>
 void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {<br>
   DebugLoc dl = getCurDebugLoc();<br>
   AtomicOrdering Order = I.getOrdering();<br>
+  SynchronizationScope Scope = I.getSynchScope();<br>
<br>
   SDValue InChain = getRoot();<br>
<br>
   if (TLI.getInsertFencesForAtomic())<br>
-    InChain = InsertFenceForAtomic(InChain, Order, true, dl, DAG, TLI);<br>
+    InChain = InsertFenceForAtomic(InChain, Order, Scope, true, dl,<br>
+                                   DAG, TLI);<br>
<br>
   SDValue L =<br>
     DAG.getAtomic(ISD::ATOMIC_CMP_SWAP, dl,<br>
@@ -3316,12 +3325,14 @@<br>
                   getValue(I.getCompareOperand()),<br>
                   getValue(I.getNewValOperand()),<br>
                   MachinePointerInfo(I.getPointerOperand()), 0 /* Alignment */,<br>
-                  I.getOrdering(), I.getSynchScope());<br>
+                  TLI.getInsertFencesForAtomic() ? Monotonic : Order,<br>
+                  Scope);<br>
<br>
   SDValue OutChain = L.getValue(1);<br>
<br>
   if (TLI.getInsertFencesForAtomic())<br>
-    OutChain = InsertFenceForAtomic(OutChain, Order, false, dl, DAG, TLI);<br>
+    OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl,<br>
+                                    DAG, TLI);<br>
<br>
   setValue(&I, L);<br>
   DAG.setRoot(OutChain);<br>
@@ -3345,11 +3356,13 @@<br>
   case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;<br>
   }<br>
   AtomicOrdering Order = I.getOrdering();<br>
+  SynchronizationScope Scope = I.getSynchScope();<br>
<br>
   SDValue InChain = getRoot();<br>
<br>
   if (TLI.getInsertFencesForAtomic())<br>
-    InChain = InsertFenceForAtomic(InChain, Order, true, dl, DAG, TLI);<br>
+    InChain = InsertFenceForAtomic(InChain, Order, Scope, true, dl,<br>
+                                   DAG, TLI);<br>
<br>
   SDValue L =<br>
     DAG.getAtomic(NT, dl,<br>
@@ -3359,12 +3372,13 @@<br>
                   getValue(I.getValOperand()),<br>
                   I.getPointerOperand(), 0 /* Alignment */,<br>
                   TLI.getInsertFencesForAtomic() ? Monotonic : Order,<br>
-                  I.getSynchScope());<br>
+                  Scope);<br>
<br>
   SDValue OutChain = L.getValue(1);<br>
<br>
   if (TLI.getInsertFencesForAtomic())<br>
-    OutChain = InsertFenceForAtomic(OutChain, Order, false, dl, DAG, TLI);<br>
+    OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl,<br>
+                                    DAG, TLI);<br>
<br>
   setValue(&I, L);<br>
   DAG.setRoot(OutChain);<br>
@@ -3379,6 +3393,65 @@<br>
   DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops, 3));<br>
 }<br>
<br>
+void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {<br>
+  DebugLoc dl = getCurDebugLoc();<br>
+  AtomicOrdering Order = I.getOrdering();<br>
+  SynchronizationScope Scope = I.getSynchScope();<br>
+<br>
+  SDValue InChain = getRoot();<br>
+<br>
+  if (TLI.getInsertFencesForAtomic())<br>
+    InChain = InsertFenceForAtomic(InChain, Order, Scope, true, dl,<br>
+                                   DAG, TLI);<br>
+<br>
+  EVT VT = EVT::getEVT(I.getType());<br>
+<br>
+  SDValue L =<br>
+    DAG.getAtomic(ISD::ATOMIC_LOAD, dl, VT, VT, InChain,<br>
+                  getValue(I.getPointerOperand()),<br>
+                  I.getPointerOperand(), I.getAlignment(),<br>
+                  TLI.getInsertFencesForAtomic() ? Monotonic : Order,<br>
+                  Scope);<br>
+<br>
+  SDValue OutChain = L.getValue(1);<br>
+<br>
+  if (TLI.getInsertFencesForAtomic())<br>
+    OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl,<br>
+                                    DAG, TLI);<br>
+<br>
+  setValue(&I, L);<br>
+  DAG.setRoot(OutChain);<br>
+}<br>
+<br>
+void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {<br>
+  DebugLoc dl = getCurDebugLoc();<br>
+<br>
+  AtomicOrdering Order = I.getOrdering();<br>
+  SynchronizationScope Scope = I.getSynchScope();<br>
+<br>
+  SDValue InChain = getRoot();<br>
+<br>
+  if (TLI.getInsertFencesForAtomic())<br>
+    InChain = InsertFenceForAtomic(InChain, Order, Scope, true, dl,<br>
+                                   DAG, TLI);<br>
+<br>
+  SDValue OutChain =<br>
+    DAG.getAtomic(ISD::ATOMIC_STORE, dl,<br>
+                  getValue(I.getValueOperand()).getValueType().getSimpleVT(),<br>
+                  InChain,<br>
+                  getValue(I.getPointerOperand()),<br>
+                  getValue(I.getValueOperand()),<br>
+                  I.getPointerOperand(), I.getAlignment(),<br>
+                  TLI.getInsertFencesForAtomic() ? Monotonic : Order,<br>
+                  Scope);<br>
+<br>
+  if (TLI.getInsertFencesForAtomic())<br>
+    OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl,<br>
+                                    DAG, TLI);<br>
+<br>
+  DAG.setRoot(OutChain);<br>
+}<br>
+<br>
 /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC<br>
 /// node.<br>
 void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,<br>
<br>
Modified: llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h?rev=138478&r1=138477&r2=138478&view=diff" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h?rev=138478&r1=138477&r2=138478&view=diff</a><br>

==============================================================================<br>
--- llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h (original)<br>
+++ llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h Wed Aug 24 15:50:09 2011<br>
@@ -526,7 +526,9 @@<br>
   void visitPHI(const PHINode &I);<br>
   void visitCall(const CallInst &I);<br>
   bool visitMemCmpCall(const CallInst &I);<br>
-<br>
+  void visitAtomicLoad(const LoadInst &I);<br>
+  void visitAtomicStore(const StoreInst &I);<br>
+<br>
   void visitInlineAsm(ImmutableCallSite CS);<br>
   const char *visitIntrinsicCall(const CallInst &I, unsigned Intrinsic);<br>
   void visitTargetIntrinsic(const CallInst &I, unsigned Intrinsic);<br>
<br>
Modified: llvm/trunk/lib/Target/X86/X86ISelLowering.cpp<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/X86/X86ISelLowering.cpp?rev=138478&r1=138477&r2=138478&view=diff" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/X86/X86ISelLowering.cpp?rev=138478&r1=138477&r2=138478&view=diff</a><br>

==============================================================================<br>
--- llvm/trunk/lib/Target/X86/X86ISelLowering.cpp (original)<br>
+++ llvm/trunk/lib/Target/X86/X86ISelLowering.cpp Wed Aug 24 15:50:09 2011<br>
@@ -464,6 +464,7 @@<br>
     MVT VT = IntVTs[i];<br>
     setOperationAction(ISD::ATOMIC_CMP_SWAP, VT, Custom);<br>
     setOperationAction(ISD::ATOMIC_LOAD_SUB, VT, Custom);<br>
+    setOperationAction(ISD::ATOMIC_STORE, VT, Custom);<br>
   }<br>
<br>
   if (!Subtarget->is64Bit()) {<br>
@@ -9999,6 +10000,26 @@<br>
                        cast<AtomicSDNode>(Node)->getSynchScope());<br>
 }<br>
<br>
+static SDValue LowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG) {<br>
+  SDNode *Node = Op.getNode();<br>
+  DebugLoc dl = Node->getDebugLoc();<br>
+<br>
+  // Convert seq_cst store -> xchg<br>
+  if (cast<AtomicSDNode>(Node)->getOrdering() == SequentiallyConsistent) {<br>
+    SDValue Swap =  DAG.getAtomic(ISD::ATOMIC_SWAP, dl,<br>
+                                  cast<AtomicSDNode>(Node)->getMemoryVT(),<br>
+                                   Node->getOperand(0),<br>
+                                   Node->getOperand(1), Node->getOperand(2),<br>
+                                   cast<AtomicSDNode>(Node)->getSrcValue(),<br>
+                                   cast<AtomicSDNode>(Node)->getAlignment(),<br>
+                                   cast<AtomicSDNode>(Node)->getOrdering(),<br>
+                                   cast<AtomicSDNode>(Node)->getSynchScope());<br>
+    return Swap.getValue(1);<br>
+  }<br>
+  // Other atomic stores have a simple pattern.<br>
+  return Op;<br>
+}<br>
+<br>
 static SDValue LowerADDC_ADDE_SUBC_SUBE(SDValue Op, SelectionDAG &DAG) {<br>
   EVT VT = Op.getNode()->getValueType(0);<br>
<br>
@@ -10035,6 +10056,7 @@<br>
   case ISD::ATOMIC_FENCE:       return LowerATOMIC_FENCE(Op,DAG);<br>
   case ISD::ATOMIC_CMP_SWAP:    return LowerCMP_SWAP(Op,DAG);<br>
   case ISD::ATOMIC_LOAD_SUB:    return LowerLOAD_SUB(Op,DAG);<br>
+  case ISD::ATOMIC_STORE:       return LowerATOMIC_STORE(Op,DAG);<br>
   case ISD::BUILD_VECTOR:       return LowerBUILD_VECTOR(Op, DAG);<br>
   case ISD::CONCAT_VECTORS:     return LowerCONCAT_VECTORS(Op, DAG);<br>
   case ISD::VECTOR_SHUFFLE:     return LowerVECTOR_SHUFFLE(Op, DAG);<br>
<br>
Modified: llvm/trunk/lib/Target/X86/X86InstrCompiler.td<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/X86/X86InstrCompiler.td?rev=138478&r1=138477&r2=138478&view=diff" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/X86/X86InstrCompiler.td?rev=138478&r1=138477&r2=138478&view=diff</a><br>

==============================================================================<br>
--- llvm/trunk/lib/Target/X86/X86InstrCompiler.td (original)<br>
+++ llvm/trunk/lib/Target/X86/X86InstrCompiler.td Wed Aug 24 15:50:09 2011<br>
@@ -1691,3 +1691,17 @@<br>
           (AND64ri8 GR64:$src1, i64immSExt8:$src2)>;<br>
 def : Pat<(and GR64:$src1, i64immSExt32:$src2),<br>
           (AND64ri32 GR64:$src1, i64immSExt32:$src2)>;<br>
+<br>
+def : Pat<(atomic_load_8 addr:$src), (MOV8rm addr:$src)>;<br>
+def : Pat<(atomic_load_16 addr:$src), (MOV16rm addr:$src)>;<br>
+def : Pat<(atomic_load_32 addr:$src), (MOV32rm addr:$src)>;<br>
+def : Pat<(atomic_load_64 addr:$src), (MOV64rm addr:$src)>;<br>
+<br>
+def : Pat<(atomic_store_8 addr:$ptr, GR8:$val),<br>
+          (MOV8mr addr:$ptr, GR8:$val)>;<br>
+def : Pat<(atomic_store_16 addr:$ptr, GR16:$val),<br>
+          (MOV16mr addr:$ptr, GR16:$val)>;<br>
+def : Pat<(atomic_store_32 addr:$ptr, GR32:$val),<br>
+          (MOV32mr addr:$ptr, GR32:$val)>;<br>
+def : Pat<(atomic_store_64 addr:$ptr, GR64:$val),<br>
+          (MOV64mr addr:$ptr, GR64:$val)>;<br>
<br>
<br>
_______________________________________________<br>
llvm-commits mailing list<br>
<a href="mailto:llvm-commits@cs.uiuc.edu">llvm-commits@cs.uiuc.edu</a><br>
<a href="http://lists.cs.uiuc.edu/mailman/listinfo/llvm-commits" target="_blank">http://lists.cs.uiuc.edu/mailman/listinfo/llvm-commits</a><br>
</blockquote></div><br>