[llvm] ad9d13d - SelectionDAG: Swap operands of atomic_store
Matt Arsenault via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 31 14:30:16 PDT 2023
Author: Matt Arsenault
Date: 2023-08-31T17:30:10-04:00
New Revision: ad9d13d5359182928f3b454d82dec92f14d0b07b
URL: https://github.com/llvm/llvm-project/commit/ad9d13d5359182928f3b454d82dec92f14d0b07b
DIFF: https://github.com/llvm/llvm-project/commit/ad9d13d5359182928f3b454d82dec92f14d0b07b.diff
LOG: SelectionDAG: Swap operands of atomic_store
Irritatingly, atomic_store had operands in the opposite order from
regular store. This made it difficult to share patterns between
regular and atomic stores.
There was a previous incomplete attempt to move atomic_store into the
regular StoreSDNode which would be better.
I think it was a mistake for all atomicrmw to swap the operand order,
so maybe it's better to take this one step further.
https://reviews.llvm.org/D123143
Added:
Modified:
llvm/include/llvm/CodeGen/SelectionDAGNodes.h
llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
llvm/include/llvm/Target/TargetSelectionDAG.td
llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
llvm/lib/Target/AArch64/AArch64InstrAtomics.td
llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
llvm/lib/Target/AMDGPU/BUFInstructions.td
llvm/lib/Target/AMDGPU/DSInstructions.td
llvm/lib/Target/AMDGPU/FLATInstructions.td
llvm/lib/Target/AMDGPU/SIInstrInfo.td
llvm/lib/Target/ARM/ARMInstrInfo.td
llvm/lib/Target/ARM/ARMInstrThumb.td
llvm/lib/Target/ARM/ARMInstrThumb2.td
llvm/lib/Target/AVR/AVRInstrInfo.td
llvm/lib/Target/Hexagon/HexagonPatterns.td
llvm/lib/Target/LoongArch/LoongArchInstrInfo.td
llvm/lib/Target/Mips/Mips64InstrInfo.td
llvm/lib/Target/Mips/MipsInstrInfo.td
llvm/lib/Target/PowerPC/PPCISelLowering.cpp
llvm/lib/Target/PowerPC/PPCInstr64Bit.td
llvm/lib/Target/PowerPC/PPCInstrInfo.td
llvm/lib/Target/PowerPC/PPCInstrP10.td
llvm/lib/Target/RISCV/RISCVInstrInfoA.td
llvm/lib/Target/Sparc/SparcInstr64Bit.td
llvm/lib/Target/Sparc/SparcInstrInfo.td
llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
llvm/lib/Target/VE/VEInstrInfo.td
llvm/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td
llvm/lib/Target/X86/X86ISelLowering.cpp
llvm/lib/Target/X86/X86InstrCompiler.td
llvm/test/TableGen/GlobalISelEmitter-atomic_store.td
llvm/utils/TableGen/GlobalISelEmitter.cpp
Removed:
################################################################################
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
index ba2222390a76cd..4d4c2673382b16 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -1385,6 +1385,7 @@ class MemSDNode : public SDNode {
const SDValue &getBasePtr() const {
switch (getOpcode()) {
case ISD::STORE:
+ case ISD::ATOMIC_STORE:
case ISD::VP_STORE:
case ISD::MSTORE:
case ISD::VP_SCATTER:
@@ -1457,8 +1458,12 @@ class AtomicSDNode : public MemSDNode {
MMO->isAtomic()) && "then why are we using an AtomicSDNode?");
}
- const SDValue &getBasePtr() const { return getOperand(1); }
- const SDValue &getVal() const { return getOperand(2); }
+ const SDValue &getBasePtr() const {
+ return getOpcode() == ISD::ATOMIC_STORE ? getOperand(2) : getOperand(1);
+ }
+ const SDValue &getVal() const {
+ return getOpcode() == ISD::ATOMIC_STORE ? getOperand(1) : getOperand(2);
+ }
/// Returns true if this SDNode represents cmpxchg atomic operation, false
/// otherwise.
diff --git a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
index 5ebd3ab9129f48..b1b122f370b26c 100644
--- a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
+++ b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
@@ -203,18 +203,16 @@ def : GINodeEquiv<G_ICMP, setcc> {
// separate nodes for them. This GINodeEquiv maps the non-atomic stores to
// G_STORE with a non-atomic MachineMemOperand.
def : GINodeEquiv<G_STORE, st> { let CheckMMOIsNonAtomic = true; }
-
-def : GINodeEquiv<G_LOAD, atomic_load> {
+def : GINodeEquiv<G_STORE, atomic_store> {
let CheckMMOIsNonAtomic = false;
let CheckMMOIsAtomic = true;
- let IfSignExtend = G_SEXTLOAD;
- let IfZeroExtend = G_ZEXTLOAD;
}
-// Operands are swapped for atomic_store vs. regular store
-def : GINodeEquiv<G_STORE, atomic_store> {
+def : GINodeEquiv<G_LOAD, atomic_load> {
let CheckMMOIsNonAtomic = false;
let CheckMMOIsAtomic = true;
+ let IfSignExtend = G_SEXTLOAD;
+ let IfZeroExtend = G_ZEXTLOAD;
}
def : GINodeEquiv<G_ATOMIC_CMPXCHG, atomic_cmp_swap>;
diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td
index 78a7fc76a64eaa..0174091d5d5bb5 100644
--- a/llvm/include/llvm/Target/TargetSelectionDAG.td
+++ b/llvm/include/llvm/Target/TargetSelectionDAG.td
@@ -315,7 +315,7 @@ def SDTFPAtomic2 : SDTypeProfile<1, 2, [
]>;
def SDTAtomicStore : SDTypeProfile<0, 2, [
- SDTCisPtrTy<0>, SDTCisInt<1>
+ SDTCisInt<0>, SDTCisPtrTy<1>
]>;
def SDTAtomicLoad : SDTypeProfile<1, 1, [
SDTCisInt<0>, SDTCisPtrTy<1>
@@ -1678,7 +1678,6 @@ defm atomic_load_min : binary_atomic_op<atomic_load_min>;
defm atomic_load_max : binary_atomic_op<atomic_load_max>;
defm atomic_load_umin : binary_atomic_op<atomic_load_umin>;
defm atomic_load_umax : binary_atomic_op<atomic_load_umax>;
-defm atomic_store : binary_atomic_op<atomic_store>;
defm atomic_cmp_swap : ternary_atomic_op<atomic_cmp_swap>;
/// Atomic load which zeroes the excess high bits.
@@ -1874,6 +1873,35 @@ def trunc_masked_scatter_i32 :
MSN->getMemoryVT().getScalarType() == MVT::i32;
}]>;
+
+def atomic_store_8 :
+ PatFrag<(ops node:$val, node:$ptr),
+ (atomic_store node:$val, node:$ptr)> {
+ let IsAtomic = true;
+ let MemoryVT = i8;
+}
+
+def atomic_store_16 :
+ PatFrag<(ops node:$val, node:$ptr),
+ (atomic_store node:$val, node:$ptr)> {
+ let IsAtomic = true;
+ let MemoryVT = i16;
+}
+
+def atomic_store_32 :
+ PatFrag<(ops node:$val, node:$ptr),
+ (atomic_store node:$val, node:$ptr)> {
+ let IsAtomic = true;
+ let MemoryVT = i32;
+}
+
+def atomic_store_64 :
+ PatFrag<(ops node:$val, node:$ptr),
+ (atomic_store node:$val, node:$ptr)> {
+ let IsAtomic = true;
+ let MemoryVT = i64;
+}
+
//===----------------------------------------------------------------------===//
// Selection DAG Pattern Support.
//
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index c53176e3a28327..a8d3732ef7306e 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -1043,7 +1043,7 @@ void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
}
case ISD::ATOMIC_STORE:
Action = TLI.getOperationAction(Node->getOpcode(),
- Node->getOperand(2).getValueType());
+ Node->getOperand(1).getValueType());
break;
case ISD::SELECT_CC:
case ISD::STRICT_FSETCC:
@@ -3080,11 +3080,10 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
}
case ISD::ATOMIC_STORE: {
// There is no libcall for atomic store; fake it with ATOMIC_SWAP.
- SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
- cast<AtomicSDNode>(Node)->getMemoryVT(),
- Node->getOperand(0),
- Node->getOperand(1), Node->getOperand(2),
- cast<AtomicSDNode>(Node)->getMemOperand());
+ SDValue Swap = DAG.getAtomic(
+ ISD::ATOMIC_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(),
+ Node->getOperand(0), Node->getOperand(2), Node->getOperand(1),
+ cast<AtomicSDNode>(Node)->getMemOperand());
Results.push_back(Swap.getValue(1));
break;
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index dc59daf5f2189a..888cb187a5b3fb 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -1956,9 +1956,9 @@ SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) {
}
SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) {
- SDValue Op2 = GetPromotedInteger(N->getOperand(2));
+ SDValue Op1 = GetPromotedInteger(N->getOperand(1));
return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(),
- N->getChain(), N->getBasePtr(), Op2, N->getMemOperand());
+ N->getChain(), Op1, N->getBasePtr(), N->getMemOperand());
}
SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) {
@@ -5505,11 +5505,10 @@ SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) {
SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) {
SDLoc dl(N);
- SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
- cast<AtomicSDNode>(N)->getMemoryVT(),
- N->getOperand(0),
- N->getOperand(1), N->getOperand(2),
- cast<AtomicSDNode>(N)->getMemOperand());
+ SDValue Swap =
+ DAG.getAtomic(ISD::ATOMIC_SWAP, dl, cast<AtomicSDNode>(N)->getMemoryVT(),
+ N->getOperand(0), N->getOperand(2), N->getOperand(1),
+ cast<AtomicSDNode>(N)->getMemOperand());
return Swap.getValue(1);
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 5424266b0923bf..899c4c247b77b2 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -4873,8 +4873,8 @@ void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
DAG.setRoot(S);
return;
}
- SDValue OutChain = DAG.getAtomic(ISD::ATOMIC_STORE, dl, MemVT, InChain,
- Ptr, Val, MMO);
+ SDValue OutChain =
+ DAG.getAtomic(ISD::ATOMIC_STORE, dl, MemVT, InChain, Val, Ptr, MMO);
setValue(&I, OutChain);
DAG.setRoot(OutChain);
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index b4ad04a3315002..b1042241859180 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -5727,7 +5727,8 @@ SDValue AArch64TargetLowering::LowerStore128(SDValue Op,
StoreNode->getMergedOrdering() == AtomicOrdering::Unordered ||
StoreNode->getMergedOrdering() == AtomicOrdering::Monotonic);
- SDValue Value = StoreNode->getOpcode() == ISD::STORE
+ SDValue Value = (StoreNode->getOpcode() == ISD::STORE ||
+ StoreNode->getOpcode() == ISD::ATOMIC_STORE)
? StoreNode->getOperand(1)
: StoreNode->getOperand(2);
SDLoc DL(Op);
diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index 1427886d71c073..fa5a8515ed92ec 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -162,14 +162,14 @@ def : Pat<(f64 (bitconvert (i64 (relaxed_load<atomic_load_64>
// A store operation that actually needs release semantics.
class releasing_store<PatFrag base>
- : PatFrag<(ops node:$ptr, node:$val), (base node:$ptr, node:$val)> {
+ : PatFrag<(ops node:$ptr, node:$val), (base node:$val, node:$ptr)> {
let IsAtomic = 1;
let IsAtomicOrderingReleaseOrStronger = 1;
}
// An atomic store operation that doesn't actually need to be atomic on AArch64.
class relaxed_store<PatFrag base>
- : PatFrag<(ops node:$ptr, node:$val), (base node:$ptr, node:$val)> {
+ : PatFrag<(ops node:$ptr, node:$val), (base node:$val, node:$ptr)> {
let IsAtomic = 1;
let IsAtomicOrderingReleaseOrStronger = 0;
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
index 2305097e3f94d5..12ccfd29f26c03 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
@@ -544,19 +544,18 @@ def truncstorei16_#as : PatFrag<(ops node:$val, node:$ptr),
def store_hi16_#as : StoreHi16 <truncstorei16, i16>;
def truncstorei8_hi16_#as : StoreHi16<truncstorei8, i8>;
def truncstorei16_hi16_#as : StoreHi16<truncstorei16, i16>;
-
} // End let IsStore = 1, AddressSpaces = ...
let IsAtomic = 1, AddressSpaces = !cast<AddressSpaceList>("StoreAddress_"#as).AddrSpaces in {
-def atomic_store_8_#as : PatFrag<(ops node:$ptr, node:$val),
- (atomic_store_8 node:$ptr, node:$val)>;
-def atomic_store_16_#as : PatFrag<(ops node:$ptr, node:$val),
- (atomic_store_16 node:$ptr, node:$val)>;
-def atomic_store_32_#as : PatFrag<(ops node:$ptr, node:$val),
- (atomic_store_32 node:$ptr, node:$val)>;
-def atomic_store_64_#as : PatFrag<(ops node:$ptr, node:$val),
- (atomic_store_64 node:$ptr, node:$val)>;
-}
+def atomic_store_8_#as : PatFrag<(ops node:$val, node:$ptr),
+ (atomic_store_8 node:$val, node:$ptr)>;
+def atomic_store_16_#as : PatFrag<(ops node:$val, node:$ptr),
+ (atomic_store_16 node:$val, node:$ptr)>;
+def atomic_store_32_#as : PatFrag<(ops node:$val, node:$ptr),
+ (atomic_store_32 node:$val, node:$ptr)>;
+def atomic_store_64_#as : PatFrag<(ops node:$val, node:$ptr),
+ (atomic_store_64 node:$val, node:$ptr)>;
+} // End let IsAtomic = 1, AddressSpaces = ...
} // End foreach as
multiclass noret_op {
diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td
index ea1578e30ae8f9..e2d62a9a2cce61 100644
--- a/llvm/lib/Target/AMDGPU/BUFInstructions.td
+++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td
@@ -1804,14 +1804,13 @@ defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D1
multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET,
ValueType vt, PatFrag atomic_st> {
- // Store follows atomic op convention so address is first
def : GCNPat <
- (atomic_st (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset), vt:$val),
+ (atomic_st vt:$val, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset)),
(Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset)
>;
def : GCNPat <
- (atomic_st (MUBUFOffset v4i32:$rsrc, i32:$soffset, i32:$offset), vt:$val),
+ (atomic_st vt:$val, (MUBUFOffset v4i32:$rsrc, i32:$soffset, i32:$offset)),
(Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset))
>;
}
diff --git a/llvm/lib/Target/AMDGPU/DSInstructions.td b/llvm/lib/Target/AMDGPU/DSInstructions.td
index c8c87c5a943e30..1a10a8fcaadca1 100644
--- a/llvm/lib/Target/AMDGPU/DSInstructions.td
+++ b/llvm/lib/Target/AMDGPU/DSInstructions.td
@@ -806,23 +806,6 @@ multiclass DSWritePat_mc <DS_Pseudo inst, ValueType vt, string frag> {
}
}
-// Irritatingly, atomic_store reverses the order of operands from a
-// normal store.
-class DSAtomicWritePat <DS_Pseudo inst, ValueType vt, PatFrag frag> : GCNPat <
- (frag (DS1Addr1Offset i32:$ptr, i32:$offset), vt:$value),
- (inst $ptr, getVregSrcForVT<vt>.ret:$value, offset:$offset, (i1 0))
->;
-
-multiclass DSAtomicWritePat_mc <DS_Pseudo inst, ValueType vt, string frag> {
- let OtherPredicates = [LDSRequiresM0Init] in {
- def : DSAtomicWritePat<inst, vt, !cast<PatFrag>(frag#"_m0")>;
- }
-
- let OtherPredicates = [NotLDSRequiresM0Init] in {
- def : DSAtomicWritePat<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt, !cast<PatFrag>(frag)>;
- }
-}
-
defm : DSWritePat_mc <DS_WRITE_B8, i32, "truncstorei8_local">;
defm : DSWritePat_mc <DS_WRITE_B16, i32, "truncstorei16_local">;
defm : DSWritePat_mc <DS_WRITE_B8, i16, "truncstorei8_local">;
@@ -832,12 +815,12 @@ foreach vt = Reg32Types.types in {
defm : DSWritePat_mc <DS_WRITE_B32, vt, "store_local">;
}
-defm : DSAtomicWritePat_mc <DS_WRITE_B8, i16, "atomic_store_8_local">;
-defm : DSAtomicWritePat_mc <DS_WRITE_B8, i32, "atomic_store_8_local">;
-defm : DSAtomicWritePat_mc <DS_WRITE_B16, i16, "atomic_store_16_local">;
-defm : DSAtomicWritePat_mc <DS_WRITE_B16, i32, "atomic_store_16_local">;
-defm : DSAtomicWritePat_mc <DS_WRITE_B32, i32, "atomic_store_32_local">;
-defm : DSAtomicWritePat_mc <DS_WRITE_B64, i64, "atomic_store_64_local">;
+defm : DSWritePat_mc <DS_WRITE_B8, i16, "atomic_store_8_local">;
+defm : DSWritePat_mc <DS_WRITE_B8, i32, "atomic_store_8_local">;
+defm : DSWritePat_mc <DS_WRITE_B16, i16, "atomic_store_16_local">;
+defm : DSWritePat_mc <DS_WRITE_B16, i32, "atomic_store_16_local">;
+defm : DSWritePat_mc <DS_WRITE_B32, i32, "atomic_store_32_local">;
+defm : DSWritePat_mc <DS_WRITE_B64, i64, "atomic_store_64_local">;
let OtherPredicates = [HasD16LoadStore] in {
def : DSWritePat <DS_WRITE_B16_D16_HI, i32, store_hi16_local>;
diff --git a/llvm/lib/Target/AMDGPU/FLATInstructions.td b/llvm/lib/Target/AMDGPU/FLATInstructions.td
index 5c86d80e7dd2b8..7d2286c5607743 100644
--- a/llvm/lib/Target/AMDGPU/FLATInstructions.td
+++ b/llvm/lib/Target/AMDGPU/FLATInstructions.td
@@ -996,12 +996,6 @@ class GlobalStoreSaddrPat <FLAT_Pseudo inst, SDPatternOperator node,
(inst $voffset, getVregSrcForVT<vt>.ret:$data, $saddr, $offset)
>;
-class GlobalAtomicStoreSaddrPat <FLAT_Pseudo inst, SDPatternOperator node,
- ValueType vt> : GCNPat <
- (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i32:$offset), vt:$data),
- (inst $voffset, getVregSrcForVT<vt>.ret:$data, $saddr, $offset)
->;
-
class GlobalAtomicSaddrPat <FLAT_Pseudo inst, SDPatternOperator node,
ValueType vt, ValueType data_vt = vt> : GCNPat <
(vt (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i32:$offset), data_vt:$data)),
@@ -1024,13 +1018,6 @@ class FlatStoreSignedPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt
(inst $vaddr, getVregSrcForVT<vt>.ret:$data, $offset)
>;
-class FlatStoreAtomicPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <
- // atomic store follows atomic binop convention so the address comes
- // first.
- (node (FlatOffset i64:$vaddr, i32:$offset), vt:$data),
- (inst $vaddr, getVregSrcForVT<vt>.ret:$data, $offset)
->;
-
class FlatStoreSignedAtomicPat <FLAT_Pseudo inst, SDPatternOperator node,
ValueType vt, ValueType data_vt = vt> : GCNPat <
// atomic store follows atomic binop convention so the address comes
@@ -1174,12 +1161,12 @@ def : FlatLoadPat <FLAT_LOAD_DWORDX4, load_flat, vt>;
def : FlatStorePat <FLAT_STORE_DWORDX4, store_flat, vt>;
}
-def : FlatStoreAtomicPat <FLAT_STORE_DWORD, atomic_store_32_flat, i32>;
-def : FlatStoreAtomicPat <FLAT_STORE_DWORDX2, atomic_store_64_flat, i64>;
-def : FlatStoreAtomicPat <FLAT_STORE_BYTE, atomic_store_8_flat, i32>;
-def : FlatStoreAtomicPat <FLAT_STORE_BYTE, atomic_store_8_flat, i16>;
-def : FlatStoreAtomicPat <FLAT_STORE_SHORT, atomic_store_16_flat, i32>;
-def : FlatStoreAtomicPat <FLAT_STORE_SHORT, atomic_store_16_flat, i16>;
+def : FlatStorePat <FLAT_STORE_DWORD, atomic_store_32_flat, i32>;
+def : FlatStorePat <FLAT_STORE_DWORDX2, atomic_store_64_flat, i64>;
+def : FlatStorePat <FLAT_STORE_BYTE, atomic_store_8_flat, i32>;
+def : FlatStorePat <FLAT_STORE_BYTE, atomic_store_8_flat, i16>;
+def : FlatStorePat <FLAT_STORE_SHORT, atomic_store_16_flat, i32>;
+def : FlatStorePat <FLAT_STORE_SHORT, atomic_store_16_flat, i16>;
foreach as = [ "flat", "global" ] in {
defm : FlatAtomicPat <"FLAT_ATOMIC_ADD", "atomic_load_add_"#as, i32>;
@@ -1269,17 +1256,6 @@ multiclass GlobalFLATStorePats<FLAT_Pseudo inst, SDPatternOperator node,
}
}
-// Deal with swapped operands for atomic_store vs. regular store
-multiclass GlobalFLATAtomicStorePats<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> {
- def : FlatStoreSignedAtomicPat <inst, node, vt> {
- let AddedComplexity = 10;
- }
-
- def : GlobalAtomicStoreSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> {
- let AddedComplexity = 11;
- }
-}
-
multiclass GlobalFLATAtomicPatsNoRtnBase<string inst, string node, ValueType vt,
ValueType data_vt = vt> {
let AddedComplexity = 11 in
@@ -1444,12 +1420,12 @@ defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16, load_d16_lo_global, v2i16>
defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16, load_d16_lo_global, v2f16>;
}
-defm : GlobalFLATAtomicStorePats <GLOBAL_STORE_BYTE, atomic_store_8_global, i32>;
-defm : GlobalFLATAtomicStorePats <GLOBAL_STORE_BYTE, atomic_store_8_global, i16>;
-defm : GlobalFLATAtomicStorePats <GLOBAL_STORE_SHORT, atomic_store_16_global, i32>;
-defm : GlobalFLATAtomicStorePats <GLOBAL_STORE_SHORT, atomic_store_16_global, i16>;
-defm : GlobalFLATAtomicStorePats <GLOBAL_STORE_DWORD, atomic_store_32_global, i32>;
-defm : GlobalFLATAtomicStorePats <GLOBAL_STORE_DWORDX2, atomic_store_64_global, i64>;
+defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, atomic_store_8_global, i32>;
+defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, atomic_store_8_global, i16>;
+defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, atomic_store_16_global, i32>;
+defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, atomic_store_16_global, i16>;
+defm : GlobalFLATStorePats <GLOBAL_STORE_DWORD, atomic_store_32_global, i32>;
+defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, atomic_store_64_global, i64>;
defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_ADD", "atomic_load_add_global", i32>;
defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SUB", "atomic_load_sub_global", i32>;
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
index 044bc4507d3a8c..9eb341212554f6 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
@@ -604,14 +604,14 @@ def atomic_store_64_glue : PatFrag <
}
let IsAtomic = 1, AddressSpaces = StoreAddress_local.AddrSpaces in {
-def atomic_store_8_local_m0 : PatFrag<(ops node:$ptr, node:$val),
- (atomic_store_8_glue node:$ptr, node:$val)>;
-def atomic_store_16_local_m0 : PatFrag<(ops node:$ptr, node:$val),
- (atomic_store_16_glue node:$ptr, node:$val)>;
-def atomic_store_32_local_m0 : PatFrag<(ops node:$ptr, node:$val),
- (atomic_store_32_glue node:$ptr, node:$val)>;
-def atomic_store_64_local_m0 : PatFrag<(ops node:$ptr, node:$val),
- (atomic_store_64_glue node:$ptr, node:$val)>;
+def atomic_store_8_local_m0 : PatFrag<(ops node:$val, node:$ptr),
+ (atomic_store_8_glue node:$val, node:$ptr)>;
+def atomic_store_16_local_m0 : PatFrag<(ops node:$val, node:$ptr),
+ (atomic_store_16_glue node:$val, node:$ptr)>;
+def atomic_store_32_local_m0 : PatFrag<(ops node:$val, node:$ptr),
+ (atomic_store_32_glue node:$val, node:$ptr)>;
+def atomic_store_64_local_m0 : PatFrag<(ops node:$val, node:$ptr),
+ (atomic_store_64_glue node:$val, node:$ptr)>;
} // End let IsAtomic = 1, AddressSpaces = StoreAddress_local.AddrSpaces
diff --git a/llvm/lib/Target/ARM/ARMInstrInfo.td b/llvm/lib/Target/ARM/ARMInstrInfo.td
index 471b706cc408a3..fd66036cc28ceb 100644
--- a/llvm/lib/Target/ARM/ARMInstrInfo.td
+++ b/llvm/lib/Target/ARM/ARMInstrInfo.td
@@ -5357,7 +5357,7 @@ def atomic_load_acquire_16 : acquiring_load<atomic_load_16>;
def atomic_load_acquire_32 : acquiring_load<atomic_load_32>;
class releasing_store<PatFrag base>
- : PatFrag<(ops node:$ptr, node:$val), (base node:$ptr, node:$val), [{
+ : PatFrag<(ops node:$ptr, node:$val), (base node:$val, node:$ptr), [{
AtomicOrdering Ordering = cast<AtomicSDNode>(N)->getSuccessOrdering();
return isReleaseOrStronger(Ordering);
}]>;
@@ -6184,15 +6184,15 @@ def : ARMPat<(atomic_load_32 ldst_so_reg:$src),
(LDRrs ldst_so_reg:$src)>;
def : ARMPat<(atomic_load_32 addrmode_imm12:$src),
(LDRi12 addrmode_imm12:$src)>;
-def : ARMPat<(atomic_store_8 ldst_so_reg:$ptr, GPR:$val),
+def : ARMPat<(atomic_store_8 GPR:$val, ldst_so_reg:$ptr),
(STRBrs GPR:$val, ldst_so_reg:$ptr)>;
-def : ARMPat<(atomic_store_8 addrmode_imm12:$ptr, GPR:$val),
+def : ARMPat<(atomic_store_8 GPR:$val, addrmode_imm12:$ptr),
(STRBi12 GPR:$val, addrmode_imm12:$ptr)>;
-def : ARMPat<(atomic_store_16 addrmode3:$ptr, GPR:$val),
+def : ARMPat<(atomic_store_16 GPR:$val, addrmode3:$ptr),
(STRH GPR:$val, addrmode3:$ptr)>;
-def : ARMPat<(atomic_store_32 ldst_so_reg:$ptr, GPR:$val),
+def : ARMPat<(atomic_store_32 GPR:$val, ldst_so_reg:$ptr),
(STRrs GPR:$val, ldst_so_reg:$ptr)>;
-def : ARMPat<(atomic_store_32 addrmode_imm12:$ptr, GPR:$val),
+def : ARMPat<(atomic_store_32 GPR:$val, addrmode_imm12:$ptr),
(STRi12 GPR:$val, addrmode_imm12:$ptr)>;
diff --git a/llvm/lib/Target/ARM/ARMInstrThumb.td b/llvm/lib/Target/ARM/ARMInstrThumb.td
index df6c129a185731..be0ca964d3f912 100644
--- a/llvm/lib/Target/ARM/ARMInstrThumb.td
+++ b/llvm/lib/Target/ARM/ARMInstrThumb.td
@@ -1713,17 +1713,17 @@ def : T1Pat<(atomic_load_32 t_addrmode_is4:$src),
(tLDRi t_addrmode_is4:$src)>;
def : T1Pat<(atomic_load_32 t_addrmode_rr:$src),
(tLDRr t_addrmode_rr:$src)>;
-def : T1Pat<(atomic_store_8 t_addrmode_is1:$ptr, tGPR:$val),
+def : T1Pat<(atomic_store_8 tGPR:$val, t_addrmode_is1:$ptr),
(tSTRBi tGPR:$val, t_addrmode_is1:$ptr)>;
-def : T1Pat<(atomic_store_8 t_addrmode_rr:$ptr, tGPR:$val),
+def : T1Pat<(atomic_store_8 tGPR:$val, t_addrmode_rr:$ptr),
(tSTRBr tGPR:$val, t_addrmode_rr:$ptr)>;
-def : T1Pat<(atomic_store_16 t_addrmode_is2:$ptr, tGPR:$val),
+def : T1Pat<(atomic_store_16 tGPR:$val, t_addrmode_is2:$ptr),
(tSTRHi tGPR:$val, t_addrmode_is2:$ptr)>;
-def : T1Pat<(atomic_store_16 t_addrmode_rr:$ptr, tGPR:$val),
+def : T1Pat<(atomic_store_16 tGPR:$val, t_addrmode_rr:$ptr),
(tSTRHr tGPR:$val, t_addrmode_rr:$ptr)>;
-def : T1Pat<(atomic_store_32 t_addrmode_is4:$ptr, tGPR:$val),
+def : T1Pat<(atomic_store_32 tGPR:$val, t_addrmode_is4:$ptr),
(tSTRi tGPR:$val, t_addrmode_is4:$ptr)>;
-def : T1Pat<(atomic_store_32 t_addrmode_rr:$ptr, tGPR:$val),
+def : T1Pat<(atomic_store_32 tGPR:$val, t_addrmode_rr:$ptr),
(tSTRr tGPR:$val, t_addrmode_rr:$ptr)>;
// Large immediate handling.
diff --git a/llvm/lib/Target/ARM/ARMInstrThumb2.td b/llvm/lib/Target/ARM/ARMInstrThumb2.td
index 610a71d68ec8c2..843ae5d7d577ba 100644
--- a/llvm/lib/Target/ARM/ARMInstrThumb2.td
+++ b/llvm/lib/Target/ARM/ARMInstrThumb2.td
@@ -4893,23 +4893,23 @@ def : T2Pat<(atomic_load_32 t2addrmode_negimm8:$addr),
(t2LDRi8 t2addrmode_negimm8:$addr)>;
def : T2Pat<(atomic_load_32 t2addrmode_so_reg:$addr),
(t2LDRs t2addrmode_so_reg:$addr)>;
-def : T2Pat<(atomic_store_8 t2addrmode_imm12:$addr, GPR:$val),
+def : T2Pat<(atomic_store_8 GPR:$val, t2addrmode_imm12:$addr),
(t2STRBi12 GPR:$val, t2addrmode_imm12:$addr)>;
-def : T2Pat<(atomic_store_8 t2addrmode_negimm8:$addr, GPR:$val),
+def : T2Pat<(atomic_store_8 GPR:$val, t2addrmode_negimm8:$addr),
(t2STRBi8 GPR:$val, t2addrmode_negimm8:$addr)>;
-def : T2Pat<(atomic_store_8 t2addrmode_so_reg:$addr, GPR:$val),
+def : T2Pat<(atomic_store_8 GPR:$val, t2addrmode_so_reg:$addr),
(t2STRBs GPR:$val, t2addrmode_so_reg:$addr)>;
-def : T2Pat<(atomic_store_16 t2addrmode_imm12:$addr, GPR:$val),
+def : T2Pat<(atomic_store_16 GPR:$val, t2addrmode_imm12:$addr),
(t2STRHi12 GPR:$val, t2addrmode_imm12:$addr)>;
-def : T2Pat<(atomic_store_16 t2addrmode_negimm8:$addr, GPR:$val),
+def : T2Pat<(atomic_store_16 GPR:$val, t2addrmode_negimm8:$addr),
(t2STRHi8 GPR:$val, t2addrmode_negimm8:$addr)>;
-def : T2Pat<(atomic_store_16 t2addrmode_so_reg:$addr, GPR:$val),
+def : T2Pat<(atomic_store_16 GPR:$val, t2addrmode_so_reg:$addr),
(t2STRHs GPR:$val, t2addrmode_so_reg:$addr)>;
-def : T2Pat<(atomic_store_32 t2addrmode_imm12:$addr, GPR:$val),
+def : T2Pat<(atomic_store_32 GPR:$val,t2addrmode_imm12:$addr),
(t2STRi12 GPR:$val, t2addrmode_imm12:$addr)>;
-def : T2Pat<(atomic_store_32 t2addrmode_negimm8:$addr, GPR:$val),
+def : T2Pat<(atomic_store_32 GPR:$val, t2addrmode_negimm8:$addr),
(t2STRi8 GPR:$val, t2addrmode_negimm8:$addr)>;
-def : T2Pat<(atomic_store_32 t2addrmode_so_reg:$addr, GPR:$val),
+def : T2Pat<(atomic_store_32 GPR:$val, t2addrmode_so_reg:$addr),
(t2STRs GPR:$val, t2addrmode_so_reg:$addr)>;
let AddedComplexity = 8, Predicates = [IsThumb, HasAcquireRelease, HasV7Clrex] in {
diff --git a/llvm/lib/Target/AVR/AVRInstrInfo.td b/llvm/lib/Target/AVR/AVRInstrInfo.td
index f93248b4940c1d..efaaec32ee6bb1 100644
--- a/llvm/lib/Target/AVR/AVRInstrInfo.td
+++ b/llvm/lib/Target/AVR/AVRInstrInfo.td
@@ -1468,9 +1468,7 @@ class AtomicStore<PatFrag Op, RegisterClass DRC, RegisterClass PTRRC>
(ins PTRRC
: $rd, DRC
: $rr),
- "atomic_op", [(Op i16
- : $rd, DRC
- : $rr)]>;
+ "atomic_op", [(Op DRC:$rr, i16:$rd)]>;
class AtomicLoadOp<PatFrag Op, RegisterClass DRC, RegisterClass PTRRC>
: Pseudo<(outs DRC:$rd),
diff --git a/llvm/lib/Target/Hexagon/HexagonPatterns.td b/llvm/lib/Target/Hexagon/HexagonPatterns.td
index d03c39d949ffc2..9de50b405445c2 100644
--- a/llvm/lib/Target/Hexagon/HexagonPatterns.td
+++ b/llvm/lib/Target/Hexagon/HexagonPatterns.td
@@ -2593,19 +2593,6 @@ class Stoream_pat<PatFrag Store, PatFrag Value, PatFrag Addr, PatFrag ValueMod,
: Pat<(Store Value:$val, Addr:$addr),
(MI Addr:$addr, (ValueMod Value:$val))>;
-// Regular stores in the DAG have two operands: value and address.
-// Atomic stores also have two, but they are reversed: address, value.
-// To use atomic stores with the patterns, they need to have their operands
-// swapped. This relies on the knowledge that the F.Fragment uses names
-// "ptr" and "val".
-class AtomSt<PatFrag F>
- : PatFrag<(ops node:$val, node:$ptr), !head(F.Fragments), F.PredicateCode,
- F.OperandTransform> {
- let IsAtomic = F.IsAtomic;
- let MemoryVT = F.MemoryVT;
-}
-
-
def IMM_BYTE : SDNodeXForm<imm, [{
// -1 can be represented as 255, etc.
// assigning to a byte restores our desired signed value.
@@ -2726,15 +2713,15 @@ let AddedComplexity = 120 in {
def: Storea_pat<store, V2I32, addrgp, S2_storerdgp>;
def: Storea_pat<store, F32, addrgp, S2_storerigp>;
def: Storea_pat<store, F64, addrgp, S2_storerdgp>;
- def: Storea_pat<AtomSt<atomic_store_8>, I32, addrgp, S2_storerbgp>;
- def: Storea_pat<AtomSt<atomic_store_16>, I32, addrgp, S2_storerhgp>;
- def: Storea_pat<AtomSt<atomic_store_32>, I32, addrgp, S2_storerigp>;
- def: Storea_pat<AtomSt<atomic_store_32>, V4I8, addrgp, S2_storerigp>;
- def: Storea_pat<AtomSt<atomic_store_32>, V2I16, addrgp, S2_storerigp>;
- def: Storea_pat<AtomSt<atomic_store_64>, I64, addrgp, S2_storerdgp>;
- def: Storea_pat<AtomSt<atomic_store_64>, V8I8, addrgp, S2_storerdgp>;
- def: Storea_pat<AtomSt<atomic_store_64>, V4I16, addrgp, S2_storerdgp>;
- def: Storea_pat<AtomSt<atomic_store_64>, V2I32, addrgp, S2_storerdgp>;
+ def: Storea_pat<atomic_store_8, I32, addrgp, S2_storerbgp>;
+ def: Storea_pat<atomic_store_16, I32, addrgp, S2_storerhgp>;
+ def: Storea_pat<atomic_store_32, I32, addrgp, S2_storerigp>;
+ def: Storea_pat<atomic_store_32, V4I8, addrgp, S2_storerigp>;
+ def: Storea_pat<atomic_store_32, V2I16, addrgp, S2_storerigp>;
+ def: Storea_pat<atomic_store_64, I64, addrgp, S2_storerdgp>;
+ def: Storea_pat<atomic_store_64, V8I8, addrgp, S2_storerdgp>;
+ def: Storea_pat<atomic_store_64, V4I16, addrgp, S2_storerdgp>;
+ def: Storea_pat<atomic_store_64, V2I32, addrgp, S2_storerdgp>;
def: Stoream_pat<truncstorei8, I64, addrgp, LoReg, S2_storerbgp>;
def: Stoream_pat<truncstorei16, I64, addrgp, LoReg, S2_storerhgp>;
@@ -2755,15 +2742,15 @@ let AddedComplexity = 110 in {
def: Storea_pat<store, V2I32, anyimm3, PS_storerdabs>;
def: Storea_pat<store, F32, anyimm2, PS_storeriabs>;
def: Storea_pat<store, F64, anyimm3, PS_storerdabs>;
- def: Storea_pat<AtomSt<atomic_store_8>, I32, anyimm0, PS_storerbabs>;
- def: Storea_pat<AtomSt<atomic_store_16>, I32, anyimm1, PS_storerhabs>;
- def: Storea_pat<AtomSt<atomic_store_32>, I32, anyimm2, PS_storeriabs>;
- def: Storea_pat<AtomSt<atomic_store_32>, V4I8, anyimm2, PS_storeriabs>;
- def: Storea_pat<AtomSt<atomic_store_32>, V2I16, anyimm2, PS_storeriabs>;
- def: Storea_pat<AtomSt<atomic_store_64>, I64, anyimm3, PS_storerdabs>;
- def: Storea_pat<AtomSt<atomic_store_64>, V8I8, anyimm3, PS_storerdabs>;
- def: Storea_pat<AtomSt<atomic_store_64>, V4I16, anyimm3, PS_storerdabs>;
- def: Storea_pat<AtomSt<atomic_store_64>, V2I32, anyimm3, PS_storerdabs>;
+ def: Storea_pat<atomic_store_8, I32, anyimm0, PS_storerbabs>;
+ def: Storea_pat<atomic_store_16, I32, anyimm1, PS_storerhabs>;
+ def: Storea_pat<atomic_store_32, I32, anyimm2, PS_storeriabs>;
+ def: Storea_pat<atomic_store_32, V4I8, anyimm2, PS_storeriabs>;
+ def: Storea_pat<atomic_store_32, V2I16, anyimm2, PS_storeriabs>;
+ def: Storea_pat<atomic_store_64, I64, anyimm3, PS_storerdabs>;
+ def: Storea_pat<atomic_store_64, V8I8, anyimm3, PS_storerdabs>;
+ def: Storea_pat<atomic_store_64, V4I16, anyimm3, PS_storerdabs>;
+ def: Storea_pat<atomic_store_64, V2I32, anyimm3, PS_storerdabs>;
def: Stoream_pat<truncstorei8, I64, anyimm0, LoReg, PS_storerbabs>;
def: Stoream_pat<truncstorei16, I64, anyimm1, LoReg, PS_storerhabs>;
@@ -2918,15 +2905,15 @@ let AddedComplexity = 40 in {
defm: Storexim_pat<truncstorei32, I64, anyimm2, LoReg, S2_storeri_io>;
defm: Storexim_pat<store, I1, anyimm0, I1toI32, S2_storerb_io>;
- defm: Storexi_pat<AtomSt<atomic_store_8>, I32, anyimm0, S2_storerb_io>;
- defm: Storexi_pat<AtomSt<atomic_store_16>, I32, anyimm1, S2_storerh_io>;
- defm: Storexi_pat<AtomSt<atomic_store_32>, I32, anyimm2, S2_storeri_io>;
- defm: Storexi_pat<AtomSt<atomic_store_32>, V4I8, anyimm2, S2_storeri_io>;
- defm: Storexi_pat<AtomSt<atomic_store_32>, V2I16, anyimm2, S2_storeri_io>;
- defm: Storexi_pat<AtomSt<atomic_store_64>, I64, anyimm3, S2_storerd_io>;
- defm: Storexi_pat<AtomSt<atomic_store_64>, V8I8, anyimm3, S2_storerd_io>;
- defm: Storexi_pat<AtomSt<atomic_store_64>, V4I16, anyimm3, S2_storerd_io>;
- defm: Storexi_pat<AtomSt<atomic_store_64>, V2I32, anyimm3, S2_storerd_io>;
+ defm: Storexi_pat<atomic_store_8, I32, anyimm0, S2_storerb_io>;
+ defm: Storexi_pat<atomic_store_16, I32, anyimm1, S2_storerh_io>;
+ defm: Storexi_pat<atomic_store_32, I32, anyimm2, S2_storeri_io>;
+ defm: Storexi_pat<atomic_store_32, V4I8, anyimm2, S2_storeri_io>;
+ defm: Storexi_pat<atomic_store_32, V2I16, anyimm2, S2_storeri_io>;
+ defm: Storexi_pat<atomic_store_64, I64, anyimm3, S2_storerd_io>;
+ defm: Storexi_pat<atomic_store_64, V8I8, anyimm3, S2_storerd_io>;
+ defm: Storexi_pat<atomic_store_64, V4I16, anyimm3, S2_storerd_io>;
+ defm: Storexi_pat<atomic_store_64, V2I32, anyimm3, S2_storerd_io>;
}
// Reg+Reg
@@ -2977,15 +2964,15 @@ let AddedComplexity = 10 in {
def: Storexim_base_pat<truncstorei32, I64, LoReg, S2_storeri_io>;
def: Storexim_base_pat<store, I1, I1toI32, S2_storerb_io>;
- def: Storexi_base_pat<AtomSt<atomic_store_8>, I32, S2_storerb_io>;
- def: Storexi_base_pat<AtomSt<atomic_store_16>, I32, S2_storerh_io>;
- def: Storexi_base_pat<AtomSt<atomic_store_32>, I32, S2_storeri_io>;
- def: Storexi_base_pat<AtomSt<atomic_store_32>, V4I8, S2_storeri_io>;
- def: Storexi_base_pat<AtomSt<atomic_store_32>, V2I16, S2_storeri_io>;
- def: Storexi_base_pat<AtomSt<atomic_store_64>, I64, S2_storerd_io>;
- def: Storexi_base_pat<AtomSt<atomic_store_64>, V8I8, S2_storerd_io>;
- def: Storexi_base_pat<AtomSt<atomic_store_64>, V4I16, S2_storerd_io>;
- def: Storexi_base_pat<AtomSt<atomic_store_64>, V2I32, S2_storerd_io>;
+ def: Storexi_base_pat<atomic_store_8, I32, S2_storerb_io>;
+ def: Storexi_base_pat<atomic_store_16, I32, S2_storerh_io>;
+ def: Storexi_base_pat<atomic_store_32, I32, S2_storeri_io>;
+ def: Storexi_base_pat<atomic_store_32, V4I8, S2_storeri_io>;
+ def: Storexi_base_pat<atomic_store_32, V2I16, S2_storeri_io>;
+ def: Storexi_base_pat<atomic_store_64, I64, S2_storerd_io>;
+ def: Storexi_base_pat<atomic_store_64, V8I8, S2_storerd_io>;
+ def: Storexi_base_pat<atomic_store_64, V4I16, S2_storerd_io>;
+ def: Storexi_base_pat<atomic_store_64, V2I32, S2_storerd_io>;
}
diff --git a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td
index cabb0c1431c8f2..586f57a20c669d 100644
--- a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td
+++ b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td
@@ -1642,13 +1642,13 @@ defm : LdPat<atomic_load_16, LD_H>;
defm : LdPat<atomic_load_32, LD_W>;
class release_seqcst_store<PatFrag base>
- : PatFrag<(ops node:$ptr, node:$val), (base node:$ptr, node:$val), [{
+ : PatFrag<(ops node:$val, node:$ptr), (base node:$val, node:$ptr), [{
AtomicOrdering Ordering = cast<AtomicSDNode>(N)->getSuccessOrdering();
return isReleaseOrStronger(Ordering);
}]>;
class unordered_monotonic_store<PatFrag base>
- : PatFrag<(ops node:$ptr, node:$val), (base node:$ptr, node:$val), [{
+ : PatFrag<(ops node:$val, node:$ptr), (base node:$val, node:$ptr), [{
AtomicOrdering Ordering = cast<AtomicSDNode>(N)->getSuccessOrdering();
return !isReleaseOrStronger(Ordering);
}]>;
@@ -1660,23 +1660,13 @@ def atomic_store_unordered_monotonic_32
def atomic_store_unordered_monotonic_64
: unordered_monotonic_store<atomic_store_64>;
-/// AtomicStores
-
-multiclass AtomicStPat<PatFrag StoreOp, LAInst Inst, RegisterClass StTy,
- ValueType vt> {
- def : Pat<(StoreOp BaseAddr:$ptr, (vt StTy:$val)),
- (Inst StTy:$val, BaseAddr:$ptr, 0)>;
- def : Pat<(StoreOp (AddLike BaseAddr:$ptr, simm12:$imm12), (vt StTy:$val)),
- (Inst StTy:$val, BaseAddr:$ptr, simm12:$imm12)>;
-}
-
-defm : AtomicStPat<atomic_store_8, ST_B, GPR, GRLenVT>;
-defm : AtomicStPat<atomic_store_16, ST_H, GPR, GRLenVT>;
-defm : AtomicStPat<atomic_store_unordered_monotonic_32, ST_W, GPR, i32>,
+defm : StPat<atomic_store_8, ST_B, GPR, GRLenVT>;
+defm : StPat<atomic_store_16, ST_H, GPR, GRLenVT>;
+defm : StPat<atomic_store_unordered_monotonic_32, ST_W, GPR, i32>,
Requires<[IsLA32]>;
def PseudoAtomicStoreW
- : Pseudo<(outs GPR:$dst), (ins GPR:$rj, GPR:$rk)>,
+ : Pseudo<(outs GPR:$dst), (ins GPR:$rk, GPR:$rj)>,
PseudoInstExpansion<(AMSWAP__DB_W R0, GPR:$rk, GPRMemAtomic:$rj)>;
def : Pat<(atomic_store_release_seqcst_32 GPR:$rj, GPR:$rk),
@@ -1684,15 +1674,15 @@ def : Pat<(atomic_store_release_seqcst_32 GPR:$rj, GPR:$rk),
let Predicates = [IsLA64] in {
def PseudoAtomicStoreD
- : Pseudo<(outs GPR:$dst), (ins GPR:$rj, GPR:$rk)>,
+ : Pseudo<(outs GPR:$dst), (ins GPR:$rk, GPR:$rj)>,
PseudoInstExpansion<(AMSWAP__DB_D R0, GPR:$rk, GPRMemAtomic:$rj)>;
def : Pat<(atomic_store_release_seqcst_64 GPR:$rj, GPR:$rk),
(PseudoAtomicStoreD GPR:$rj, GPR:$rk)>;
defm : LdPat<atomic_load_64, LD_D>;
-defm : AtomicStPat<atomic_store_unordered_monotonic_32, ST_W, GPR, i64>;
-defm : AtomicStPat<atomic_store_unordered_monotonic_64, ST_D, GPR, i64>;
+defm : StPat<atomic_store_unordered_monotonic_32, ST_W, GPR, i64>;
+defm : StPat<atomic_store_unordered_monotonic_64, ST_D, GPR, i64>;
} // Predicates = [IsLA64]
/// Atomic Ops
diff --git a/llvm/lib/Target/Mips/Mips64InstrInfo.td b/llvm/lib/Target/Mips/Mips64InstrInfo.td
index bd62a56d3008f2..ac679c4c01bc7c 100644
--- a/llvm/lib/Target/Mips/Mips64InstrInfo.td
+++ b/llvm/lib/Target/Mips/Mips64InstrInfo.td
@@ -894,13 +894,13 @@ def : MipsPat<(atomic_load_32 addr:$a), (LW64 addr:$a)>, ISA_MIPS3, GPR_64;
def : MipsPat<(atomic_load_64 addr:$a), (LD addr:$a)>, ISA_MIPS3, GPR_64;
// Atomic store patterns.
-def : MipsPat<(atomic_store_8 addr:$a, GPR64:$v), (SB64 GPR64:$v, addr:$a)>,
+def : MipsPat<(atomic_store_8 GPR64:$v, addr:$a), (SB64 GPR64:$v, addr:$a)>,
ISA_MIPS3, GPR_64;
-def : MipsPat<(atomic_store_16 addr:$a, GPR64:$v), (SH64 GPR64:$v, addr:$a)>,
+def : MipsPat<(atomic_store_16 GPR64:$v, addr:$a), (SH64 GPR64:$v, addr:$a)>,
ISA_MIPS3, GPR_64;
-def : MipsPat<(atomic_store_32 addr:$a, GPR64:$v), (SW64 GPR64:$v, addr:$a)>,
+def : MipsPat<(atomic_store_32 GPR64:$v, addr:$a), (SW64 GPR64:$v, addr:$a)>,
ISA_MIPS3, GPR_64;
-def : MipsPat<(atomic_store_64 addr:$a, GPR64:$v), (SD GPR64:$v, addr:$a)>,
+def : MipsPat<(atomic_store_64 GPR64:$v, addr:$a), (SD GPR64:$v, addr:$a)>,
ISA_MIPS3, GPR_64;
// Patterns used for matching away redundant sign extensions.
diff --git a/llvm/lib/Target/Mips/MipsInstrInfo.td b/llvm/lib/Target/Mips/MipsInstrInfo.td
index 973f40a21deebd..75270857ea1352 100644
--- a/llvm/lib/Target/Mips/MipsInstrInfo.td
+++ b/llvm/lib/Target/Mips/MipsInstrInfo.td
@@ -3357,11 +3357,11 @@ let AdditionalPredicates = [NotInMicroMips] in {
def : MipsPat<(atomic_load_32 addr:$a), (LW addr:$a)>, ISA_MIPS1;
// Atomic store patterns.
- def : MipsPat<(atomic_store_8 addr:$a, GPR32:$v), (SB GPR32:$v, addr:$a)>,
+ def : MipsPat<(atomic_store_8 GPR32:$v, addr:$a), (SB GPR32:$v, addr:$a)>,
ISA_MIPS1;
- def : MipsPat<(atomic_store_16 addr:$a, GPR32:$v), (SH GPR32:$v, addr:$a)>,
+ def : MipsPat<(atomic_store_16 GPR32:$v, addr:$a), (SH GPR32:$v, addr:$a)>,
ISA_MIPS1;
- def : MipsPat<(atomic_store_32 addr:$a, GPR32:$v), (SW GPR32:$v, addr:$a)>,
+ def : MipsPat<(atomic_store_32 GPR32:$v, addr:$a), (SW GPR32:$v, addr:$a)>,
ISA_MIPS1;
}
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 53ae80b0268865..25a98739d5fc62 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -11045,14 +11045,14 @@ SDValue PPCTargetLowering::LowerATOMIC_LOAD_STORE(SDValue Op,
SmallVector<SDValue, 4> Ops{
N->getOperand(0),
DAG.getConstant(Intrinsic::ppc_atomic_store_i128, dl, MVT::i32)};
- SDValue Val = N->getOperand(2);
+ SDValue Val = N->getOperand(1);
SDValue ValLo = DAG.getNode(ISD::TRUNCATE, dl, MVT::i64, Val);
SDValue ValHi = DAG.getNode(ISD::SRL, dl, MVT::i128, Val,
DAG.getConstant(64, dl, MVT::i32));
ValHi = DAG.getNode(ISD::TRUNCATE, dl, MVT::i64, ValHi);
Ops.push_back(ValLo);
Ops.push_back(ValHi);
- Ops.push_back(N->getOperand(1));
+ Ops.push_back(N->getOperand(2));
return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, dl, Tys, Ops, MemVT,
N->getMemOperand());
}
diff --git a/llvm/lib/Target/PowerPC/PPCInstr64Bit.td b/llvm/lib/Target/PowerPC/PPCInstr64Bit.td
index 7067239dd92802..fd436d42229835 100644
--- a/llvm/lib/Target/PowerPC/PPCInstr64Bit.td
+++ b/llvm/lib/Target/PowerPC/PPCInstr64Bit.td
@@ -1970,8 +1970,8 @@ def : Pat<(PPCaddTls i64:$in, i64:$addr),
def : Pat<(atomic_load_64 DSForm:$src), (LD memrix:$src)>;
def : Pat<(atomic_load_64 XForm:$src), (LDX memrr:$src)>;
-def : Pat<(atomic_store_64 DSForm:$ptr, i64:$val), (STD g8rc:$val, memrix:$ptr)>;
-def : Pat<(atomic_store_64 XForm:$ptr, i64:$val), (STDX g8rc:$val, memrr:$ptr)>;
+def : Pat<(atomic_store_64 i64:$val, DSForm:$ptr), (STD g8rc:$val, memrix:$ptr)>;
+def : Pat<(atomic_store_64 i64:$val, XForm:$ptr), (STDX g8rc:$val, memrr:$ptr)>;
let Predicates = [IsISA3_0, In64BitMode] in {
def : Pat<(i64 (int_ppc_cmpeqb g8rc:$a, g8rc:$b)),
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.td b/llvm/lib/Target/PowerPC/PPCInstrInfo.td
index 4a8e36e8d242fe..a71cd9efb8b5fd 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.td
@@ -5027,12 +5027,12 @@ def : Pat<(atomic_load_16 XForm:$src), (LHZX memrr:$src)>;
def : Pat<(atomic_load_32 XForm:$src), (LWZX memrr:$src)>;
// Atomic stores
-def : Pat<(atomic_store_8 DForm:$ptr, i32:$val), (STB gprc:$val, memri:$ptr)>;
-def : Pat<(atomic_store_16 DForm:$ptr, i32:$val), (STH gprc:$val, memri:$ptr)>;
-def : Pat<(atomic_store_32 DForm:$ptr, i32:$val), (STW gprc:$val, memri:$ptr)>;
-def : Pat<(atomic_store_8 XForm:$ptr, i32:$val), (STBX gprc:$val, memrr:$ptr)>;
-def : Pat<(atomic_store_16 XForm:$ptr, i32:$val), (STHX gprc:$val, memrr:$ptr)>;
-def : Pat<(atomic_store_32 XForm:$ptr, i32:$val), (STWX gprc:$val, memrr:$ptr)>;
+def : Pat<(atomic_store_8 i32:$val, DForm:$ptr), (STB gprc:$val, memri:$ptr)>;
+def : Pat<(atomic_store_16 i32:$val, DForm:$ptr), (STH gprc:$val, memri:$ptr)>;
+def : Pat<(atomic_store_32 i32:$val, DForm:$ptr), (STW gprc:$val, memri:$ptr)>;
+def : Pat<(atomic_store_8 i32:$val, XForm:$ptr), (STBX gprc:$val, memrr:$ptr)>;
+def : Pat<(atomic_store_16 i32:$val, XForm:$ptr), (STHX gprc:$val, memrr:$ptr)>;
+def : Pat<(atomic_store_32 i32:$val, XForm:$ptr), (STWX gprc:$val, memrr:$ptr)>;
let Predicates = [IsISA3_0] in {
diff --git a/llvm/lib/Target/PowerPC/PPCInstrP10.td b/llvm/lib/Target/PowerPC/PPCInstrP10.td
index 8cb8e4d91db211..fdfb762eec13e8 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrP10.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrP10.td
@@ -1236,19 +1236,19 @@ let Predicates = [PCRelativeMemops] in {
(PLDpc $ga, 0)>;
// Atomic Store
- def : Pat<(atomic_store_8 (PPCmatpcreladdr PCRelForm:$ga), i32:$RS),
+ def : Pat<(atomic_store_8 i32:$RS, (PPCmatpcreladdr PCRelForm:$ga)),
(PSTBpc $RS, $ga, 0)>;
- def : Pat<(atomic_store_16 (PPCmatpcreladdr PCRelForm:$ga), i32:$RS),
+ def : Pat<(atomic_store_16 i32:$RS, (PPCmatpcreladdr PCRelForm:$ga)),
(PSTHpc $RS, $ga, 0)>;
- def : Pat<(atomic_store_32 (PPCmatpcreladdr PCRelForm:$ga), i32:$RS),
+ def : Pat<(atomic_store_32 i32:$RS, (PPCmatpcreladdr PCRelForm:$ga)),
(PSTWpc $RS, $ga, 0)>;
- def : Pat<(atomic_store_8 (PPCmatpcreladdr PCRelForm:$ga), i64:$RS),
+ def : Pat<(atomic_store_8 i64:$RS, (PPCmatpcreladdr PCRelForm:$ga)),
(PSTB8pc $RS, $ga, 0)>;
- def : Pat<(atomic_store_16 (PPCmatpcreladdr PCRelForm:$ga), i64:$RS),
+ def : Pat<(atomic_store_16 i64:$RS, (PPCmatpcreladdr PCRelForm:$ga)),
(PSTH8pc $RS, $ga, 0)>;
- def : Pat<(atomic_store_32 (PPCmatpcreladdr PCRelForm:$ga), i64:$RS),
+ def : Pat<(atomic_store_32 i64:$RS, (PPCmatpcreladdr PCRelForm:$ga)),
(PSTW8pc $RS, $ga, 0)>;
- def : Pat<(atomic_store_64 (PPCmatpcreladdr PCRelForm:$ga), i64:$RS),
+ def : Pat<(atomic_store_64 i64:$RS, (PPCmatpcreladdr PCRelForm:$ga)),
(PSTDpc $RS, $ga, 0)>;
// Special Cases For PPCstore_scal_int_from_vsr
@@ -2276,10 +2276,10 @@ let Predicates = [PrefixInstrs] in {
def : Pat<(atomic_load_64 PDForm:$src), (PLD memri34:$src)>;
// Atomic Store
- def : Pat<(atomic_store_8 PDForm:$dst, i32:$RS), (PSTB $RS, memri34:$dst)>;
- def : Pat<(atomic_store_16 PDForm:$dst, i32:$RS), (PSTH $RS, memri34:$dst)>;
- def : Pat<(atomic_store_32 PDForm:$dst, i32:$RS), (PSTW $RS, memri34:$dst)>;
- def : Pat<(atomic_store_64 PDForm:$dst, i64:$RS), (PSTD $RS, memri34:$dst)>;
+ def : Pat<(atomic_store_8 i32:$RS, PDForm:$dst), (PSTB $RS, memri34:$dst)>;
+ def : Pat<(atomic_store_16 i32:$RS, PDForm:$dst), (PSTH $RS, memri34:$dst)>;
+ def : Pat<(atomic_store_32 i32:$RS, PDForm:$dst), (PSTW $RS, memri34:$dst)>;
+ def : Pat<(atomic_store_64 i64:$RS, PDForm:$dst), (PSTD $RS, memri34:$dst)>;
// Prefixed fpext to v2f64
def : Pat<(v4f32 (PPCldvsxlh PDForm:$src)),
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoA.td b/llvm/lib/Target/RISCV/RISCVInstrInfoA.td
index b06415b620e04e..41f2a372390e83 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoA.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoA.td
@@ -46,8 +46,7 @@ multiclass AMO_rr_aq_rl<bits<5> funct5, bits<3> funct3, string opcodestr> {
class AtomicStPat<PatFrag StoreOp, RVInst Inst, RegisterClass StTy,
ValueType vt = XLenVT>
- : Pat<(StoreOp (AddrRegImm (XLenVT GPR:$rs1), simm12:$imm12),
- (vt StTy:$rs2)),
+ : Pat<(StoreOp (vt StTy:$rs2), (AddrRegImm (XLenVT GPR:$rs1), simm12:$imm12)),
(Inst StTy:$rs2, GPR:$rs1, simm12:$imm12)>;
//===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/Sparc/SparcInstr64Bit.td b/llvm/lib/Target/Sparc/SparcInstr64Bit.td
index 0a647948741887..078c08d8d4945f 100644
--- a/llvm/lib/Target/Sparc/SparcInstr64Bit.td
+++ b/llvm/lib/Target/Sparc/SparcInstr64Bit.td
@@ -492,8 +492,8 @@ def : Pat<(i64 (atomic_load_64 ADDRrr:$src)), (LDXrr ADDRrr:$src)>;
def : Pat<(i64 (atomic_load_64 ADDRri:$src)), (LDXri ADDRri:$src)>;
// atomic_store_64 val, addr -> store val, addr
-def : Pat<(atomic_store_64 ADDRrr:$dst, i64:$val), (STXrr ADDRrr:$dst, $val)>;
-def : Pat<(atomic_store_64 ADDRri:$dst, i64:$val), (STXri ADDRri:$dst, $val)>;
+def : Pat<(atomic_store_64 i64:$val, ADDRrr:$dst), (STXrr ADDRrr:$dst, $val)>;
+def : Pat<(atomic_store_64 i64:$val, ADDRri:$dst), (STXri ADDRri:$dst, $val)>;
} // Predicates = [Is64Bit]
diff --git a/llvm/lib/Target/Sparc/SparcInstrInfo.td b/llvm/lib/Target/Sparc/SparcInstrInfo.td
index 9af8b17edcc50e..8a02c92129295b 100644
--- a/llvm/lib/Target/Sparc/SparcInstrInfo.td
+++ b/llvm/lib/Target/Sparc/SparcInstrInfo.td
@@ -1868,12 +1868,12 @@ def : Pat<(i32 (atomic_load_32 ADDRrr:$src)), (LDrr ADDRrr:$src)>;
def : Pat<(i32 (atomic_load_32 ADDRri:$src)), (LDri ADDRri:$src)>;
// atomic_store val, addr -> store val, addr
-def : Pat<(atomic_store_8 ADDRrr:$dst, i32:$val), (STBrr ADDRrr:$dst, $val)>;
-def : Pat<(atomic_store_8 ADDRri:$dst, i32:$val), (STBri ADDRri:$dst, $val)>;
-def : Pat<(atomic_store_16 ADDRrr:$dst, i32:$val), (STHrr ADDRrr:$dst, $val)>;
-def : Pat<(atomic_store_16 ADDRri:$dst, i32:$val), (STHri ADDRri:$dst, $val)>;
-def : Pat<(atomic_store_32 ADDRrr:$dst, i32:$val), (STrr ADDRrr:$dst, $val)>;
-def : Pat<(atomic_store_32 ADDRri:$dst, i32:$val), (STri ADDRri:$dst, $val)>;
+def : Pat<(atomic_store_8 i32:$val, ADDRrr:$dst), (STBrr ADDRrr:$dst, $val)>;
+def : Pat<(atomic_store_8 i32:$val, ADDRri:$dst), (STBri ADDRri:$dst, $val)>;
+def : Pat<(atomic_store_16 i32:$val, ADDRrr:$dst), (STHrr ADDRrr:$dst, $val)>;
+def : Pat<(atomic_store_16 i32:$val, ADDRri:$dst), (STHri ADDRri:$dst, $val)>;
+def : Pat<(atomic_store_32 i32:$val, ADDRrr:$dst), (STrr ADDRrr:$dst, $val)>;
+def : Pat<(atomic_store_32 i32:$val, ADDRri:$dst), (STri ADDRri:$dst, $val)>;
// A register pair with zero upper half.
// The upper part is done with ORrr instead of `COPY G0`
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index 785a08a763eb7b..6a56869ca20f89 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -5961,9 +5961,8 @@ SystemZTargetLowering::LowerOperationWrapper(SDNode *N,
case ISD::ATOMIC_STORE: {
SDLoc DL(N);
SDVTList Tys = DAG.getVTList(MVT::Other);
- SDValue Ops[] = { N->getOperand(0),
- lowerI128ToGR128(DAG, N->getOperand(2)),
- N->getOperand(1) };
+ SDValue Ops[] = {N->getOperand(0), lowerI128ToGR128(DAG, N->getOperand(1)),
+ N->getOperand(2)};
MachineMemOperand *MMO = cast<AtomicSDNode>(N)->getMemOperand();
SDValue Res = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_STORE_128,
DL, Tys, Ops, MVT::i128, MMO);
diff --git a/llvm/lib/Target/VE/VEInstrInfo.td b/llvm/lib/Target/VE/VEInstrInfo.td
index 166598cab41da8..1e548d7c101a7a 100644
--- a/llvm/lib/Target/VE/VEInstrInfo.td
+++ b/llvm/lib/Target/VE/VEInstrInfo.td
@@ -1864,10 +1864,10 @@ defm : ZXATMLD32m<atomic_load_32, LDLZXrri, LDLZXrii, LDLZXzri, LDLZXzii>;
multiclass ATMSTm<SDPatternOperator from, ValueType ty,
RM torri, RM torii,
RM tozri, RM tozii> {
- def : Pat<(from ADDRrri:$addr, ty:$src), (torri MEMrri:$addr, $src)>;
- def : Pat<(from ADDRrii:$addr, ty:$src), (torii MEMrii:$addr, $src)>;
- def : Pat<(from ADDRzri:$addr, ty:$src), (tozri MEMzri:$addr, $src)>;
- def : Pat<(from ADDRzii:$addr, ty:$src), (tozii MEMzii:$addr, $src)>;
+ def : Pat<(from ty:$src, ADDRrri:$addr), (torri MEMrri:$addr, $src)>;
+ def : Pat<(from ty:$src, ADDRrii:$addr), (torii MEMrii:$addr, $src)>;
+ def : Pat<(from ty:$src, ADDRzri:$addr), (tozri MEMzri:$addr, $src)>;
+ def : Pat<(from ty:$src, ADDRzii:$addr), (tozii MEMzii:$addr, $src)>;
}
defm : ATMSTm<atomic_store_8, i32, ST1Brri, ST1Brii, ST1Bzri, ST1Bzii>;
defm : ATMSTm<atomic_store_16, i32, ST2Brri, ST2Brii, ST2Bzri, ST2Bzii>;
@@ -1880,14 +1880,14 @@ multiclass TRATMSTm<SDPatternOperator from,
RM torii,
RM tozri,
RM tozii> {
- def : Pat<(from ADDRrri:$addr, (i32 (trunc i64:$src))),
- (torri MEMrri:$addr, (l2i $src))>;
- def : Pat<(from ADDRrii:$addr, (i32 (trunc i64:$src))),
- (torii MEMrii:$addr, (l2i $src))>;
- def : Pat<(from ADDRzri:$addr, (i32 (trunc i64:$src))),
- (tozri MEMzri:$addr, (l2i $src))>;
- def : Pat<(from ADDRzii:$addr, (i32 (trunc i64:$src))),
- (tozii MEMzii:$addr, (l2i $src))>;
+ def : Pat<(from (i32 (trunc i64:$src)), ADDRrri:$addr),
+ (torri MEMrri:$addr, (EXTRACT_SUBREG $src, sub_i32))>;
+ def : Pat<(from (i32 (trunc i64:$src)), ADDRrii:$addr),
+ (torii MEMrii:$addr, (EXTRACT_SUBREG $src, sub_i32))>;
+ def : Pat<(from (i32 (trunc i64:$src)), ADDRzri:$addr),
+ (tozri MEMzri:$addr, (EXTRACT_SUBREG $src, sub_i32))>;
+ def : Pat<(from (i32 (trunc i64:$src)), ADDRzii:$addr),
+ (tozii MEMzii:$addr, (EXTRACT_SUBREG $src, sub_i32))>;
}
defm : TRATMSTm<atomic_store_8, ST1Brri, ST1Brii, ST1Bzri, ST1Bzii>;
defm : TRATMSTm<atomic_store_16, ST2Brri, ST2Brii, ST2Bzri, ST2Bzii>;
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td
index 2dbcdd50fb8d9c..4623ce9b5c3819 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td
@@ -192,16 +192,17 @@ multiclass AtomicStore<WebAssemblyRegClass rc, string name, int atomic_op> {
defm ATOMIC_STORE_I32 : AtomicStore<I32, "i32.atomic.store", 0x17>;
defm ATOMIC_STORE_I64 : AtomicStore<I64, "i64.atomic.store", 0x18>;
-// We need an 'atomic' version of store patterns because store and atomic_store
-// nodes have
diff erent operand orders:
-// store: (store $val, $ptr)
-// atomic_store: (store $ptr, $val)
+// We used to need an 'atomic' version of store patterns because store and atomic_store
+// nodes have
diff erent operand orders.
+//
+// TODO: This is no longer true and atomic_store and store patterns
+// can be unified.
multiclass AStorePat<ValueType ty, PatFrag kind, string inst> {
- def : Pat<(kind (AddrOps32 offset32_op:$offset, I32:$addr), ty:$val),
+ def : Pat<(kind ty:$val, (AddrOps32 offset32_op:$offset, I32:$addr)),
(!cast<NI>(inst#_A32) 0, $offset, $addr, $val)>,
Requires<[HasAddr32, HasAtomics]>;
- def : Pat<(kind (AddrOps64 offset64_op:$offset, I64:$addr), ty:$val),
+ def : Pat<(kind ty:$val, (AddrOps64 offset64_op:$offset, I64:$addr)),
(!cast<NI>(inst#_A64) 0, $offset, $addr, $val)>,
Requires<[HasAddr64, HasAtomics]>;
}
@@ -221,8 +222,8 @@ defm ATOMIC_STORE32_I64 : AtomicStore<I64, "i64.atomic.store32", 0x1d>;
// instructions, we just need to match bare atomic stores. On the other hand,
// truncating stores from i64 values are once truncated to i32 first.
class trunc_astore_64<PatFrag kind> :
- PatFrag<(ops node:$addr, node:$val),
- (kind node:$addr, (i32 (trunc (i64 node:$val))))>;
+ PatFrag<(ops node:$val, node:$addr),
+ (kind (i32 (trunc (i64 node:$val))), node:$addr)>;
def trunc_astore_8_64 : trunc_astore_64<atomic_store_8>;
def trunc_astore_16_64 : trunc_astore_64<atomic_store_16>;
def trunc_astore_32_64 : trunc_astore_64<atomic_store_32>;
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 6dcb9c7d57992a..5a45629accd162 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -31168,8 +31168,8 @@ static SDValue LowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG,
if (!Subtarget.useSoftFloat() && !NoImplicitFloatOps) {
SDValue Chain;
if (Subtarget.hasSSE1()) {
- SDValue SclToVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64,
- Node->getOperand(2));
+ SDValue SclToVec =
+ DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Node->getVal());
MVT StVT = Subtarget.hasSSE2() ? MVT::v2i64 : MVT::v4f32;
SclToVec = DAG.getBitcast(StVT, SclToVec);
SDVTList Tys = DAG.getVTList(MVT::Other);
@@ -31183,9 +31183,8 @@ static SDValue LowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG,
int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
MachinePointerInfo MPI =
MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
- Chain =
- DAG.getStore(Node->getChain(), dl, Node->getOperand(2), StackPtr,
- MPI, MaybeAlign(), MachineMemOperand::MOStore);
+ Chain = DAG.getStore(Node->getChain(), dl, Node->getVal(), StackPtr,
+ MPI, MaybeAlign(), MachineMemOperand::MOStore);
SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other);
SDValue LdOps[] = {Chain, StackPtr};
SDValue Value = DAG.getMemIntrinsicNode(
@@ -31214,11 +31213,9 @@ static SDValue LowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG,
// Convert seq_cst store -> xchg
// Convert wide store -> swap (-> cmpxchg8b/cmpxchg16b)
// FIXME: 16-byte ATOMIC_SWAP isn't actually hooked up at the moment.
- SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
- Node->getMemoryVT(),
- Node->getOperand(0),
- Node->getOperand(1), Node->getOperand(2),
- Node->getMemOperand());
+ SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, Node->getMemoryVT(),
+ Node->getOperand(0), Node->getOperand(2),
+ Node->getOperand(1), Node->getMemOperand());
return Swap.getValue(1);
}
diff --git a/llvm/lib/Target/X86/X86InstrCompiler.td b/llvm/lib/Target/X86/X86InstrCompiler.td
index 52750937c42595..9e99dbd6fe8529 100644
--- a/llvm/lib/Target/X86/X86InstrCompiler.td
+++ b/llvm/lib/Target/X86/X86InstrCompiler.td
@@ -1071,30 +1071,28 @@ defm LXADD : ATOMIC_RMW_BINOP<0xc0, 0xc1, "xadd", "atomic_load_add">, TB, LOCK;
* (see below the RELEASE_MOV* / ACQUIRE_MOV* pseudo-instructions)
*/
multiclass RELEASE_BINOP_MI<string Name, SDNode op> {
- def : Pat<(atomic_store_8 addr:$dst,
- (op (atomic_load_8 addr:$dst), (i8 imm:$src))),
+ def : Pat<(atomic_store_8 (op (atomic_load_8 addr:$dst), (i8 imm:$src)),
+ addr:$dst),
(!cast<Instruction>(Name#"8mi") addr:$dst, imm:$src)>;
- def : Pat<(atomic_store_16 addr:$dst,
- (op (atomic_load_16 addr:$dst), (i16 imm:$src))),
+ def : Pat<(atomic_store_16 (op (atomic_load_16 addr:$dst), (i16 imm:$src)),
+ addr:$dst),
(!cast<Instruction>(Name#"16mi") addr:$dst, imm:$src)>;
- def : Pat<(atomic_store_32 addr:$dst,
- (op (atomic_load_32 addr:$dst), (i32 imm:$src))),
+ def : Pat<(atomic_store_32 (op (atomic_load_32 addr:$dst), (i32 imm:$src)),
+ addr:$dst),
(!cast<Instruction>(Name#"32mi") addr:$dst, imm:$src)>;
- def : Pat<(atomic_store_64 addr:$dst,
- (op (atomic_load_64 addr:$dst), (i64immSExt32:$src))),
+ def : Pat<(atomic_store_64 (op (atomic_load_64 addr:$dst), (i64immSExt32:$src)),
+ addr:$dst),
(!cast<Instruction>(Name#"64mi32") addr:$dst, (i64immSExt32:$src))>;
-
- def : Pat<(atomic_store_8 addr:$dst,
- (op (atomic_load_8 addr:$dst), (i8 GR8:$src))),
+ def : Pat<(atomic_store_8 (op (atomic_load_8 addr:$dst), (i8 GR8:$src)), addr:$dst),
(!cast<Instruction>(Name#"8mr") addr:$dst, GR8:$src)>;
- def : Pat<(atomic_store_16 addr:$dst,
- (op (atomic_load_16 addr:$dst), (i16 GR16:$src))),
+ def : Pat<(atomic_store_16 (op (atomic_load_16 addr:$dst), (i16 GR16:$src)),
+ addr:$dst),
(!cast<Instruction>(Name#"16mr") addr:$dst, GR16:$src)>;
- def : Pat<(atomic_store_32 addr:$dst,
- (op (atomic_load_32 addr:$dst), (i32 GR32:$src))),
+ def : Pat<(atomic_store_32 (op (atomic_load_32 addr:$dst), (i32 GR32:$src)),
+ addr:$dst),
(!cast<Instruction>(Name#"32mr") addr:$dst, GR32:$src)>;
- def : Pat<(atomic_store_64 addr:$dst,
- (op (atomic_load_64 addr:$dst), (i64 GR64:$src))),
+ def : Pat<(atomic_store_64 (op (atomic_load_64 addr:$dst), (i64 GR64:$src)),
+ addr:$dst),
(!cast<Instruction>(Name#"64mr") addr:$dst, GR64:$src)>;
}
defm : RELEASE_BINOP_MI<"ADD", add>;
@@ -1131,13 +1129,13 @@ defm : ATOMIC_LOAD_FP_BINOP_MI<"ADD", fadd>;
multiclass RELEASE_UNOP<string Name, dag dag8, dag dag16, dag dag32,
dag dag64> {
- def : Pat<(atomic_store_8 addr:$dst, dag8),
+ def : Pat<(atomic_store_8 dag8, addr:$dst),
(!cast<Instruction>(Name#8m) addr:$dst)>;
- def : Pat<(atomic_store_16 addr:$dst, dag16),
+ def : Pat<(atomic_store_16 dag16, addr:$dst),
(!cast<Instruction>(Name#16m) addr:$dst)>;
- def : Pat<(atomic_store_32 addr:$dst, dag32),
+ def : Pat<(atomic_store_32 dag32, addr:$dst),
(!cast<Instruction>(Name#32m) addr:$dst)>;
- def : Pat<(atomic_store_64 addr:$dst, dag64),
+ def : Pat<(atomic_store_64 dag64, addr:$dst),
(!cast<Instruction>(Name#64m) addr:$dst)>;
}
@@ -1165,22 +1163,22 @@ defm : RELEASE_UNOP<"NOT",
(not (i32 (atomic_load_32 addr:$dst))),
(not (i64 (atomic_load_64 addr:$dst)))>;
-def : Pat<(atomic_store_8 addr:$dst, (i8 imm:$src)),
+def : Pat<(atomic_store_8 (i8 imm:$src), addr:$dst),
(MOV8mi addr:$dst, imm:$src)>;
-def : Pat<(atomic_store_16 addr:$dst, (i16 imm:$src)),
+def : Pat<(atomic_store_16 (i16 imm:$src), addr:$dst),
(MOV16mi addr:$dst, imm:$src)>;
-def : Pat<(atomic_store_32 addr:$dst, (i32 imm:$src)),
+def : Pat<(atomic_store_32 (i32 imm:$src), addr:$dst),
(MOV32mi addr:$dst, imm:$src)>;
-def : Pat<(atomic_store_64 addr:$dst, (i64immSExt32:$src)),
+def : Pat<(atomic_store_64 (i64immSExt32:$src), addr:$dst),
(MOV64mi32 addr:$dst, i64immSExt32:$src)>;
-def : Pat<(atomic_store_8 addr:$dst, GR8:$src),
+def : Pat<(atomic_store_8 GR8:$src, addr:$dst),
(MOV8mr addr:$dst, GR8:$src)>;
-def : Pat<(atomic_store_16 addr:$dst, GR16:$src),
+def : Pat<(atomic_store_16 GR16:$src, addr:$dst),
(MOV16mr addr:$dst, GR16:$src)>;
-def : Pat<(atomic_store_32 addr:$dst, GR32:$src),
+def : Pat<(atomic_store_32 GR32:$src, addr:$dst),
(MOV32mr addr:$dst, GR32:$src)>;
-def : Pat<(atomic_store_64 addr:$dst, GR64:$src),
+def : Pat<(atomic_store_64 GR64:$src, addr:$dst),
(MOV64mr addr:$dst, GR64:$src)>;
def : Pat<(i8 (atomic_load_8 addr:$src)), (MOV8rm addr:$src)>;
@@ -1189,18 +1187,18 @@ def : Pat<(i32 (atomic_load_32 addr:$src)), (MOV32rm addr:$src)>;
def : Pat<(i64 (atomic_load_64 addr:$src)), (MOV64rm addr:$src)>;
// Floating point loads/stores.
-def : Pat<(atomic_store_32 addr:$dst, (i32 (bitconvert (f32 FR32:$src)))),
+def : Pat<(atomic_store_32 (i32 (bitconvert (f32 FR32:$src))), addr:$dst),
(MOVSSmr addr:$dst, FR32:$src)>, Requires<[UseSSE1]>;
-def : Pat<(atomic_store_32 addr:$dst, (i32 (bitconvert (f32 FR32:$src)))),
+def : Pat<(atomic_store_32 (i32 (bitconvert (f32 FR32:$src))), addr:$dst),
(VMOVSSmr addr:$dst, FR32:$src)>, Requires<[UseAVX]>;
-def : Pat<(atomic_store_32 addr:$dst, (i32 (bitconvert (f32 FR32:$src)))),
+def : Pat<(atomic_store_32 (i32 (bitconvert (f32 FR32:$src))), addr:$dst),
(VMOVSSZmr addr:$dst, FR32:$src)>, Requires<[HasAVX512]>;
-def : Pat<(atomic_store_64 addr:$dst, (i64 (bitconvert (f64 FR64:$src)))),
+def : Pat<(atomic_store_64 (i64 (bitconvert (f64 FR64:$src))), addr:$dst),
(MOVSDmr addr:$dst, FR64:$src)>, Requires<[UseSSE2]>;
-def : Pat<(atomic_store_64 addr:$dst, (i64 (bitconvert (f64 FR64:$src)))),
+def : Pat<(atomic_store_64 (i64 (bitconvert (f64 FR64:$src))), addr:$dst),
(VMOVSDmr addr:$dst, FR64:$src)>, Requires<[UseAVX]>;
-def : Pat<(atomic_store_64 addr:$dst, (i64 (bitconvert (f64 FR64:$src)))),
+def : Pat<(atomic_store_64 (i64 (bitconvert (f64 FR64:$src))), addr:$dst),
(VMOVSDmr addr:$dst, FR64:$src)>, Requires<[HasAVX512]>;
def : Pat<(f32 (bitconvert (i32 (atomic_load_32 addr:$src)))),
diff --git a/llvm/test/TableGen/GlobalISelEmitter-atomic_store.td b/llvm/test/TableGen/GlobalISelEmitter-atomic_store.td
index 4bcd6ed927e3a4..5263eed007bd45 100644
--- a/llvm/test/TableGen/GlobalISelEmitter-atomic_store.td
+++ b/llvm/test/TableGen/GlobalISelEmitter-atomic_store.td
@@ -5,20 +5,17 @@ include "GlobalISelEmitterCommon.td"
def ST_ATOM_B32 : I<(outs), (ins GPR32Op:$val, GPR32Op:$ptr), []>;
-// Check that the pattern for atomic_store inverts the operands to
-// match the order of G_STORE.
-
// GISEL: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_STORE,
// GISEL-NEXT: GIM_CheckMemorySizeEqualTo, /*MI*/0, /*MMO*/0, /*Size*/1,
// GISEL-NEXT: GIM_CheckAtomicOrderingOrStrongerThan, /*MI*/0, /*Order*/(int64_t)AtomicOrdering::Unordered,
-// GISEL-NEXT: // MIs[0] ptr
-// GISEL-NEXT: GIM_CheckPointerToAny, /*MI*/0, /*Op*/1, /*SizeInBits*/0,
// GISEL-NEXT: // MIs[0] val
// GISEL-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
-// GISEL-NEXT: // (atomic_store iPTR:{ *:[iPTR] }:$ptr, i32:{ *:[i32] }:$val)<<P:Predicate_atomic_store_8>> => (ST_ATOM_B32 GPR32Op:{ *:[i32] }:$val, GPR32Op:{ *:[i32] }:$ptr)
+// GISEL-NEXT: // MIs[0] ptr
+// GISEL-NEXT: GIM_CheckPointerToAny, /*MI*/0, /*Op*/1, /*SizeInBits*/0,
+// GISEL-NEXT: // (atomic_store i32:{ *:[i32] }:$val, iPTR:{ *:[iPTR] }:$ptr)<<P:Predicate_atomic_store_8>> => (ST_ATOM_B32 GPR32Op:{ *:[i32] }:$val, GPR32Op:{ *:[i32] }:$ptr)
// GISEL-NEXT: GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/MyTarget::ST_ATOM_B32,
def : Pat<
// (atomic_store_8 iPTR:$ptr, i32:$val),
- (atomic_store_8 iPTR:$ptr, i32:$val),
+ (atomic_store_8 i32:$val, iPTR:$ptr),
(ST_ATOM_B32 GPR32Op:$val, GPR32Op:$ptr)
>;
diff --git a/llvm/utils/TableGen/GlobalISelEmitter.cpp b/llvm/utils/TableGen/GlobalISelEmitter.cpp
index ef30e947a293ce..1dbd821f20fdc5 100644
--- a/llvm/utils/TableGen/GlobalISelEmitter.cpp
+++ b/llvm/utils/TableGen/GlobalISelEmitter.cpp
@@ -787,13 +787,11 @@ Expected<InstructionMatcher &> GlobalISelEmitter::createAndImportSelDAGMatcher(
}
}
- bool IsAtomic = false;
if (SrcGIEquivOrNull &&
SrcGIEquivOrNull->getValueAsBit("CheckMMOIsNonAtomic"))
InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>("NotAtomic");
else if (SrcGIEquivOrNull &&
SrcGIEquivOrNull->getValueAsBit("CheckMMOIsAtomic")) {
- IsAtomic = true;
InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(
"Unordered", AtomicOrderingMMOPredicateMatcher::AO_OrStronger);
}
@@ -847,27 +845,6 @@ Expected<InstructionMatcher &> GlobalISelEmitter::createAndImportSelDAGMatcher(
}
}
- // Hack around an unfortunate mistake in how atomic store (and really
- // atomicrmw in general) operands were ordered. A ISD::STORE used the order
- // <stored value>, <pointer> order. ISD::ATOMIC_STORE used the opposite,
- // <pointer>, <stored value>. In GlobalISel there's just the one store
- // opcode, so we need to swap the operands here to get the right type check.
- if (IsAtomic && SrcGIOrNull->TheDef->getName() == "G_STORE") {
- assert(NumChildren == 2 && "wrong operands for atomic store");
-
- const TreePatternNode *PtrChild = Src->getChild(0);
- const TreePatternNode *ValueChild = Src->getChild(1);
-
- if (auto Error = importChildMatcher(Rule, InsnMatcher, PtrChild, true,
- false, 1, TempOpIdx))
- return std::move(Error);
-
- if (auto Error = importChildMatcher(Rule, InsnMatcher, ValueChild, false,
- false, 0, TempOpIdx))
- return std::move(Error);
- return InsnMatcher;
- }
-
// Match the used operands (i.e. the children of the operator).
bool IsIntrinsic =
SrcGIOrNull->TheDef->getName() == "G_INTRINSIC" ||
More information about the llvm-commits
mailing list