[llvm] r324908 - [AArch64] Improve v8.1-A code-gen for atomic load-and
Oliver Stannard via llvm-commits
llvm-commits at lists.llvm.org
Mon Feb 12 09:03:11 PST 2018
Author: olista01
Date: Mon Feb 12 09:03:11 2018
New Revision: 324908
URL: http://llvm.org/viewvc/llvm-project?rev=324908&view=rev
Log:
[AArch64] Improve v8.1-A code-gen for atomic load-and
Armv8.1-A added an atomic load-clear instruction (which performs bitwise
and with the complement of it's operand), but not a load-and
instruction. Our current code-generation for atomic load-and always
inserts an MVN instruction to invert its argument, even if it could be
folded into a constant or another instruction.
This adds lowering early in selection DAG to convert a load-and
operation into an xor with -1 and a load-clear, allowing the normal DAG
optimisations to work on it.
To do this, I've had to add a new ISD opcode, ATOMIC_LOAD_CLR. I don't
see any easy way to do this with an AArch64-specific ISD node, because
the code-generation for atomic operations assumes the SDNodes are of
type AtomicSDNode.
I've left the old tablegen patterns in because they are still needed for
global isel.
Differential revision: https://reviews.llvm.org/D42478
Modified:
llvm/trunk/include/llvm/CodeGen/ISDOpcodes.h
llvm/trunk/include/llvm/CodeGen/SelectionDAGNodes.h
llvm/trunk/include/llvm/Target/TargetSelectionDAG.td
llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
llvm/trunk/lib/Target/AArch64/AArch64ISelLowering.cpp
llvm/trunk/lib/Target/AArch64/AArch64ISelLowering.h
llvm/trunk/lib/Target/AArch64/AArch64InstrAtomics.td
llvm/trunk/test/CodeGen/AArch64/atomic-ops-lse.ll
Modified: llvm/trunk/include/llvm/CodeGen/ISDOpcodes.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/CodeGen/ISDOpcodes.h?rev=324908&r1=324907&r2=324908&view=diff
==============================================================================
--- llvm/trunk/include/llvm/CodeGen/ISDOpcodes.h (original)
+++ llvm/trunk/include/llvm/CodeGen/ISDOpcodes.h Mon Feb 12 09:03:11 2018
@@ -779,6 +779,7 @@ namespace ISD {
ATOMIC_LOAD_ADD,
ATOMIC_LOAD_SUB,
ATOMIC_LOAD_AND,
+ ATOMIC_LOAD_CLR,
ATOMIC_LOAD_OR,
ATOMIC_LOAD_XOR,
ATOMIC_LOAD_NAND,
Modified: llvm/trunk/include/llvm/CodeGen/SelectionDAGNodes.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/CodeGen/SelectionDAGNodes.h?rev=324908&r1=324907&r2=324908&view=diff
==============================================================================
--- llvm/trunk/include/llvm/CodeGen/SelectionDAGNodes.h (original)
+++ llvm/trunk/include/llvm/CodeGen/SelectionDAGNodes.h Mon Feb 12 09:03:11 2018
@@ -1294,6 +1294,7 @@ public:
N->getOpcode() == ISD::ATOMIC_LOAD_ADD ||
N->getOpcode() == ISD::ATOMIC_LOAD_SUB ||
N->getOpcode() == ISD::ATOMIC_LOAD_AND ||
+ N->getOpcode() == ISD::ATOMIC_LOAD_CLR ||
N->getOpcode() == ISD::ATOMIC_LOAD_OR ||
N->getOpcode() == ISD::ATOMIC_LOAD_XOR ||
N->getOpcode() == ISD::ATOMIC_LOAD_NAND ||
@@ -1345,6 +1346,7 @@ public:
N->getOpcode() == ISD::ATOMIC_LOAD_ADD ||
N->getOpcode() == ISD::ATOMIC_LOAD_SUB ||
N->getOpcode() == ISD::ATOMIC_LOAD_AND ||
+ N->getOpcode() == ISD::ATOMIC_LOAD_CLR ||
N->getOpcode() == ISD::ATOMIC_LOAD_OR ||
N->getOpcode() == ISD::ATOMIC_LOAD_XOR ||
N->getOpcode() == ISD::ATOMIC_LOAD_NAND ||
Modified: llvm/trunk/include/llvm/Target/TargetSelectionDAG.td
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/Target/TargetSelectionDAG.td?rev=324908&r1=324907&r2=324908&view=diff
==============================================================================
--- llvm/trunk/include/llvm/Target/TargetSelectionDAG.td (original)
+++ llvm/trunk/include/llvm/Target/TargetSelectionDAG.td Mon Feb 12 09:03:11 2018
@@ -485,6 +485,8 @@ def atomic_load_sub : SDNode<"ISD::ATOMI
[SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
def atomic_load_and : SDNode<"ISD::ATOMIC_LOAD_AND" , SDTAtomic2,
[SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
+def atomic_load_clr : SDNode<"ISD::ATOMIC_LOAD_CLR" , SDTAtomic2,
+ [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
def atomic_load_or : SDNode<"ISD::ATOMIC_LOAD_OR" , SDTAtomic2,
[SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
def atomic_load_xor : SDNode<"ISD::ATOMIC_LOAD_XOR" , SDTAtomic2,
@@ -1244,6 +1246,7 @@ defm atomic_load_add : binary_atomic_op
defm atomic_swap : binary_atomic_op<atomic_swap>;
defm atomic_load_sub : binary_atomic_op<atomic_load_sub>;
defm atomic_load_and : binary_atomic_op<atomic_load_and>;
+defm atomic_load_clr : binary_atomic_op<atomic_load_clr>;
defm atomic_load_or : binary_atomic_op<atomic_load_or>;
defm atomic_load_xor : binary_atomic_op<atomic_load_xor>;
defm atomic_load_nand : binary_atomic_op<atomic_load_nand>;
Modified: llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp?rev=324908&r1=324907&r2=324908&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp (original)
+++ llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp Mon Feb 12 09:03:11 2018
@@ -3971,6 +3971,7 @@ void SelectionDAGLegalize::ConvertNodeTo
case ISD::ATOMIC_LOAD_ADD:
case ISD::ATOMIC_LOAD_SUB:
case ISD::ATOMIC_LOAD_AND:
+ case ISD::ATOMIC_LOAD_CLR:
case ISD::ATOMIC_LOAD_OR:
case ISD::ATOMIC_LOAD_XOR:
case ISD::ATOMIC_LOAD_NAND:
Modified: llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp?rev=324908&r1=324907&r2=324908&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp (original)
+++ llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp Mon Feb 12 09:03:11 2018
@@ -146,6 +146,7 @@ void DAGTypeLegalizer::PromoteIntegerRes
case ISD::ATOMIC_LOAD_ADD:
case ISD::ATOMIC_LOAD_SUB:
case ISD::ATOMIC_LOAD_AND:
+ case ISD::ATOMIC_LOAD_CLR:
case ISD::ATOMIC_LOAD_OR:
case ISD::ATOMIC_LOAD_XOR:
case ISD::ATOMIC_LOAD_NAND:
@@ -1402,6 +1403,7 @@ void DAGTypeLegalizer::ExpandIntegerResu
case ISD::ATOMIC_LOAD_ADD:
case ISD::ATOMIC_LOAD_SUB:
case ISD::ATOMIC_LOAD_AND:
+ case ISD::ATOMIC_LOAD_CLR:
case ISD::ATOMIC_LOAD_OR:
case ISD::ATOMIC_LOAD_XOR:
case ISD::ATOMIC_LOAD_NAND:
Modified: llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAG.cpp?rev=324908&r1=324907&r2=324908&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAG.cpp (original)
+++ llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAG.cpp Mon Feb 12 09:03:11 2018
@@ -493,6 +493,7 @@ static void AddNodeIDCustom(FoldingSetNo
case ISD::ATOMIC_LOAD_ADD:
case ISD::ATOMIC_LOAD_SUB:
case ISD::ATOMIC_LOAD_AND:
+ case ISD::ATOMIC_LOAD_CLR:
case ISD::ATOMIC_LOAD_OR:
case ISD::ATOMIC_LOAD_XOR:
case ISD::ATOMIC_LOAD_NAND:
@@ -5755,6 +5756,7 @@ SDValue SelectionDAG::getAtomic(unsigned
assert((Opcode == ISD::ATOMIC_LOAD_ADD ||
Opcode == ISD::ATOMIC_LOAD_SUB ||
Opcode == ISD::ATOMIC_LOAD_AND ||
+ Opcode == ISD::ATOMIC_LOAD_CLR ||
Opcode == ISD::ATOMIC_LOAD_OR ||
Opcode == ISD::ATOMIC_LOAD_XOR ||
Opcode == ISD::ATOMIC_LOAD_NAND ||
Modified: llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp?rev=324908&r1=324907&r2=324908&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp (original)
+++ llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp Mon Feb 12 09:03:11 2018
@@ -85,6 +85,7 @@ std::string SDNode::getOperationName(con
case ISD::ATOMIC_LOAD_ADD: return "AtomicLoadAdd";
case ISD::ATOMIC_LOAD_SUB: return "AtomicLoadSub";
case ISD::ATOMIC_LOAD_AND: return "AtomicLoadAnd";
+ case ISD::ATOMIC_LOAD_CLR: return "AtomicLoadClr";
case ISD::ATOMIC_LOAD_OR: return "AtomicLoadOr";
case ISD::ATOMIC_LOAD_XOR: return "AtomicLoadXor";
case ISD::ATOMIC_LOAD_NAND: return "AtomicLoadNand";
Modified: llvm/trunk/lib/Target/AArch64/AArch64ISelLowering.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/AArch64/AArch64ISelLowering.cpp?rev=324908&r1=324907&r2=324908&view=diff
==============================================================================
--- llvm/trunk/lib/Target/AArch64/AArch64ISelLowering.cpp (original)
+++ llvm/trunk/lib/Target/AArch64/AArch64ISelLowering.cpp Mon Feb 12 09:03:11 2018
@@ -466,6 +466,8 @@ AArch64TargetLowering::AArch64TargetLowe
setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i128, Custom);
setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i32, Custom);
setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i64, Custom);
+ setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i32, Custom);
+ setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i64, Custom);
// Lower READCYCLECOUNTER using an mrs from PMCCNTR_EL0.
// This requires the Performance Monitors extension.
@@ -2683,6 +2685,8 @@ SDValue AArch64TargetLowering::LowerOper
return LowerVECREDUCE(Op, DAG);
case ISD::ATOMIC_LOAD_SUB:
return LowerATOMIC_LOAD_SUB(Op, DAG);
+ case ISD::ATOMIC_LOAD_AND:
+ return LowerATOMIC_LOAD_AND(Op, DAG);
}
}
@@ -7305,6 +7309,23 @@ SDValue AArch64TargetLowering::LowerATOM
Op.getOperand(0), Op.getOperand(1), RHS,
AN->getMemOperand());
}
+
+SDValue AArch64TargetLowering::LowerATOMIC_LOAD_AND(SDValue Op,
+ SelectionDAG &DAG) const {
+ auto &Subtarget = static_cast<const AArch64Subtarget &>(DAG.getSubtarget());
+ if (!Subtarget.hasLSE())
+ return SDValue();
+
+ // LSE has an atomic load-clear instruction, but not a load-and.
+ SDLoc dl(Op);
+ MVT VT = Op.getSimpleValueType();
+ SDValue RHS = Op.getOperand(2);
+ AtomicSDNode *AN = cast<AtomicSDNode>(Op.getNode());
+ RHS = DAG.getNode(ISD::XOR, dl, VT, DAG.getConstant(-1ULL, dl, VT), RHS);
+ return DAG.getAtomic(ISD::ATOMIC_LOAD_CLR, dl, AN->getMemoryVT(),
+ Op.getOperand(0), Op.getOperand(1), RHS,
+ AN->getMemOperand());
+}
/// getTgtMemIntrinsic - Represent NEON load and store intrinsics as
/// MemIntrinsicNodes. The associated MachineMemOperands record the alignment
Modified: llvm/trunk/lib/Target/AArch64/AArch64ISelLowering.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/AArch64/AArch64ISelLowering.h?rev=324908&r1=324907&r2=324908&view=diff
==============================================================================
--- llvm/trunk/lib/Target/AArch64/AArch64ISelLowering.h (original)
+++ llvm/trunk/lib/Target/AArch64/AArch64ISelLowering.h Mon Feb 12 09:03:11 2018
@@ -597,6 +597,7 @@ private:
SDValue LowerFSINCOS(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerVECREDUCE(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerATOMIC_LOAD_SUB(SDValue Op, SelectionDAG &DAG) const;
+ SDValue LowerATOMIC_LOAD_AND(SDValue Op, SelectionDAG &DAG) const;
SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG,
std::vector<SDNode *> *Created) const override;
Modified: llvm/trunk/lib/Target/AArch64/AArch64InstrAtomics.td
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/AArch64/AArch64InstrAtomics.td?rev=324908&r1=324907&r2=324908&view=diff
==============================================================================
--- llvm/trunk/lib/Target/AArch64/AArch64InstrAtomics.td (original)
+++ llvm/trunk/lib/Target/AArch64/AArch64InstrAtomics.td Mon Feb 12 09:03:11 2018
@@ -409,13 +409,18 @@ let Predicates = [HasLSE] in {
defm : LDOPregister_patterns<"LDADD", "atomic_load_add">;
defm : LDOPregister_patterns<"LDSET", "atomic_load_or">;
defm : LDOPregister_patterns<"LDEOR", "atomic_load_xor">;
+ defm : LDOPregister_patterns<"LDCLR", "atomic_load_clr">;
defm : LDOPregister_patterns<"LDSMAX", "atomic_load_max">;
defm : LDOPregister_patterns<"LDSMIN", "atomic_load_min">;
defm : LDOPregister_patterns<"LDUMAX", "atomic_load_umax">;
defm : LDOPregister_patterns<"LDUMIN", "atomic_load_umin">;
defm : LDOPregister_patterns<"SWP", "atomic_swap">;
+ defm : CASregister_patterns<"CAS", "atomic_cmp_swap">;
+
+ // These two patterns are only needed for global isel, selection dag isel
+ // converts atomic load-sub into a sub and atomic load-add, and likewise for
+ // and -> clr.
defm : LDOPregister_patterns_mod<"LDADD", "atomic_load_sub", "SUB">;
defm : LDOPregister_patterns_mod<"LDCLR", "atomic_load_and", "ORN">;
- defm : CASregister_patterns<"CAS", "atomic_cmp_swap">;
}
Modified: llvm/trunk/test/CodeGen/AArch64/atomic-ops-lse.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/CodeGen/AArch64/atomic-ops-lse.ll?rev=324908&r1=324907&r2=324908&view=diff
==============================================================================
--- llvm/trunk/test/CodeGen/AArch64/atomic-ops-lse.ll (original)
+++ llvm/trunk/test/CodeGen/AArch64/atomic-ops-lse.ll Mon Feb 12 09:03:11 2018
@@ -978,6 +978,102 @@ define i64 @test_atomic_load_and_i64(i64
ret i64 %old
}
+define i8 @test_atomic_load_and_i8_inv_imm() nounwind {
+; CHECK-LABEL: test_atomic_load_and_i8_inv_imm:
+ %old = atomicrmw and i8* @var8, i8 -2 seq_cst
+; CHECK-NOT: dmb
+; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
+; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
+; CHECK: orr w[[CONST:[0-9]+]], wzr, #0x1
+; CHECK: ldclralb w[[CONST]], w[[NEW:[0-9]+]], [x[[ADDR]]]
+; CHECK-NOT: dmb
+ ret i8 %old
+}
+
+define i16 @test_atomic_load_and_i16_inv_imm() nounwind {
+; CHECK-LABEL: test_atomic_load_and_i16_inv_imm:
+ %old = atomicrmw and i16* @var16, i16 -2 seq_cst
+; CHECK-NOT: dmb
+; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
+; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
+; CHECK: orr w[[CONST:[0-9]+]], wzr, #0x1
+; CHECK: ldclralh w[[CONST]], w[[NEW:[0-9]+]], [x[[ADDR]]]
+; CHECK-NOT: dmb
+ ret i16 %old
+}
+
+define i32 @test_atomic_load_and_i32_inv_imm() nounwind {
+; CHECK-LABEL: test_atomic_load_and_i32_inv_imm:
+ %old = atomicrmw and i32* @var32, i32 -2 seq_cst
+; CHECK-NOT: dmb
+; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
+; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
+; CHECK: orr w[[CONST:[0-9]+]], wzr, #0x1
+; CHECK: ldclral w[[CONST]], w[[NEW:[0-9]+]], [x[[ADDR]]]
+; CHECK-NOT: dmb
+ ret i32 %old
+}
+
+define i64 @test_atomic_load_and_i64_inv_imm() nounwind {
+; CHECK-LABEL: test_atomic_load_and_i64_inv_imm:
+ %old = atomicrmw and i64* @var64, i64 -2 seq_cst
+; CHECK-NOT: dmb
+; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
+; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
+; CHECK: orr w[[CONST:[0-9]+]], wzr, #0x1
+; CHECK: ldclral x[[CONST]], x[[NEW:[0-9]+]], [x[[ADDR]]]
+; CHECK-NOT: dmb
+ ret i64 %old
+}
+
+define i8 @test_atomic_load_and_i8_inv_arg(i8 %offset) nounwind {
+; CHECK-LABEL: test_atomic_load_and_i8_inv_arg:
+ %inv = xor i8 %offset, -1
+ %old = atomicrmw and i8* @var8, i8 %inv seq_cst
+; CHECK-NOT: dmb
+; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
+; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
+; CHECK: ldclralb w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
+; CHECK-NOT: dmb
+ ret i8 %old
+}
+
+define i16 @test_atomic_load_and_i16_inv_arg(i16 %offset) nounwind {
+; CHECK-LABEL: test_atomic_load_and_i16_inv_arg:
+ %inv = xor i16 %offset, -1
+ %old = atomicrmw and i16* @var16, i16 %inv seq_cst
+; CHECK-NOT: dmb
+; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
+; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
+; CHECK: ldclralh w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
+; CHECK-NOT: dmb
+ ret i16 %old
+}
+
+define i32 @test_atomic_load_and_i32_inv_arg(i32 %offset) nounwind {
+; CHECK-LABEL: test_atomic_load_and_i32_inv_arg:
+ %inv = xor i32 %offset, -1
+ %old = atomicrmw and i32* @var32, i32 %inv seq_cst
+; CHECK-NOT: dmb
+; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
+; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
+; CHECK: ldclral w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
+; CHECK-NOT: dmb
+ ret i32 %old
+}
+
+define i64 @test_atomic_load_and_i64_inv_arg(i64 %offset) nounwind {
+; CHECK-LABEL: test_atomic_load_and_i64_inv_arg:
+ %inv = xor i64 %offset, -1
+ %old = atomicrmw and i64* @var64, i64 %inv seq_cst
+; CHECK-NOT: dmb
+; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
+; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
+; CHECK: ldclral x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
+; CHECK-NOT: dmb
+ ret i64 %old
+}
+
define void @test_atomic_load_and_i32_noret(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_and_i32_noret:
atomicrmw and i32* @var32, i32 %offset seq_cst
More information about the llvm-commits
mailing list