[llvm] 7334b3d - [SystemZ] Reimplement the i8/i16 compare-and-swap logic.
Jonas Paulsson via llvm-commits
llvm-commits at lists.llvm.org
Wed Mar 3 12:06:39 PST 2021
Author: Jonas Paulsson
Date: 2021-03-03T14:04:32-06:00
New Revision: 7334b3dc3ea4dbb525faaa6a930687aaecf38ae0
URL: https://github.com/llvm/llvm-project/commit/7334b3dc3ea4dbb525faaa6a930687aaecf38ae0
DIFF: https://github.com/llvm/llvm-project/commit/7334b3dc3ea4dbb525faaa6a930687aaecf38ae0.diff
LOG: [SystemZ] Reimplement the i8/i16 compare-and-swap logic.
Even though the implementation in emitAtomicCmpSwapW() was correct, it made
Valgrind report an error. Instead of using a RISBG on CmpVal, an LL[CH]R can
be made on the OldVal, and the problem is avoided.
Review: Ulrich Weigand
Differential Revision: https://reviews.llvm.org/D97604
Added:
Modified:
llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
llvm/lib/Target/SystemZ/SystemZISelLowering.h
llvm/test/CodeGen/SystemZ/cmpxchg-01.ll
llvm/test/CodeGen/SystemZ/cmpxchg-02.ll
llvm/test/CodeGen/SystemZ/cmpxchg-05.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index 94a441b03458..430837ebc59f 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -4056,7 +4056,10 @@ SDValue SystemZTargetLowering::lowerATOMIC_CMP_SWAP(SDValue Op,
SDValue Success = emitSETCC(DAG, DL, AtomicOp.getValue(1),
SystemZ::CCMASK_ICMP, SystemZ::CCMASK_CMP_EQ);
- DAG.ReplaceAllUsesOfValueWith(Op.getValue(0), AtomicOp.getValue(0));
+ // emitAtomicCmpSwapW() will zero extend the result (original value).
+ SDValue OrigVal = DAG.getNode(ISD::AssertZext, DL, WideVT, AtomicOp.getValue(0),
+ DAG.getValueType(NarrowVT));
+ DAG.ReplaceAllUsesOfValueWith(Op.getValue(0), OrigVal);
DAG.ReplaceAllUsesOfValueWith(Op.getValue(1), Success);
DAG.ReplaceAllUsesOfValueWith(Op.getValue(2), AtomicOp.getValue(2));
return SDValue();
@@ -7578,7 +7581,6 @@ MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadMinMax(
MachineBasicBlock *
SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr &MI,
MachineBasicBlock *MBB) const {
-
MachineFunction &MF = *MBB->getParent();
const SystemZInstrInfo *TII =
static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo());
@@ -7588,7 +7590,7 @@ SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr &MI,
Register Dest = MI.getOperand(0).getReg();
MachineOperand Base = earlyUseOperand(MI.getOperand(1));
int64_t Disp = MI.getOperand(2).getImm();
- Register OrigCmpVal = MI.getOperand(3).getReg();
+ Register CmpVal = MI.getOperand(3).getReg();
Register OrigSwapVal = MI.getOperand(4).getReg();
Register BitShift = MI.getOperand(5).getReg();
Register NegBitShift = MI.getOperand(6).getReg();
@@ -7597,19 +7599,19 @@ SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr &MI,
const TargetRegisterClass *RC = &SystemZ::GR32BitRegClass;
- // Get the right opcodes for the displacement.
+ // Get the right opcodes for the displacement and zero-extension.
unsigned LOpcode = TII->getOpcodeForOffset(SystemZ::L, Disp);
unsigned CSOpcode = TII->getOpcodeForOffset(SystemZ::CS, Disp);
+ unsigned ZExtOpcode = BitSize == 8 ? SystemZ::LLCR : SystemZ::LLHR;
assert(LOpcode && CSOpcode && "Displacement out of range");
// Create virtual registers for temporary results.
Register OrigOldVal = MRI.createVirtualRegister(RC);
Register OldVal = MRI.createVirtualRegister(RC);
- Register CmpVal = MRI.createVirtualRegister(RC);
Register SwapVal = MRI.createVirtualRegister(RC);
Register StoreVal = MRI.createVirtualRegister(RC);
+ Register OldValRot = MRI.createVirtualRegister(RC);
Register RetryOldVal = MRI.createVirtualRegister(RC);
- Register RetryCmpVal = MRI.createVirtualRegister(RC);
Register RetrySwapVal = MRI.createVirtualRegister(RC);
// Insert 2 basic blocks for the loop.
@@ -7631,34 +7633,32 @@ SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr &MI,
// LoopMBB:
// %OldVal = phi [ %OrigOldVal, EntryBB ], [ %RetryOldVal, SetMBB ]
- // %CmpVal = phi [ %OrigCmpVal, EntryBB ], [ %RetryCmpVal, SetMBB ]
// %SwapVal = phi [ %OrigSwapVal, EntryBB ], [ %RetrySwapVal, SetMBB ]
- // %Dest = RLL %OldVal, BitSize(%BitShift)
+ // %OldValRot = RLL %OldVal, BitSize(%BitShift)
// ^^ The low BitSize bits contain the field
// of interest.
- // %RetryCmpVal = RISBG32 %CmpVal, %Dest, 32, 63-BitSize, 0
+ // %RetrySwapVal = RISBG32 %SwapVal, %OldValRot, 32, 63-BitSize, 0
// ^^ Replace the upper 32-BitSize bits of the
- // comparison value with those that we loaded,
- // so that we can use a full word comparison.
- // CR %Dest, %RetryCmpVal
+ // swap value with those that we loaded and rotated.
+ // %Dest = LL[CH] %OldValRot
+ // CR %Dest, %CmpVal
// JNE DoneMBB
// # Fall through to SetMBB
MBB = LoopMBB;
BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal)
.addReg(OrigOldVal).addMBB(StartMBB)
.addReg(RetryOldVal).addMBB(SetMBB);
- BuildMI(MBB, DL, TII->get(SystemZ::PHI), CmpVal)
- .addReg(OrigCmpVal).addMBB(StartMBB)
- .addReg(RetryCmpVal).addMBB(SetMBB);
BuildMI(MBB, DL, TII->get(SystemZ::PHI), SwapVal)
.addReg(OrigSwapVal).addMBB(StartMBB)
.addReg(RetrySwapVal).addMBB(SetMBB);
- BuildMI(MBB, DL, TII->get(SystemZ::RLL), Dest)
+ BuildMI(MBB, DL, TII->get(SystemZ::RLL), OldValRot)
.addReg(OldVal).addReg(BitShift).addImm(BitSize);
- BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetryCmpVal)
- .addReg(CmpVal).addReg(Dest).addImm(32).addImm(63 - BitSize).addImm(0);
+ BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetrySwapVal)
+ .addReg(SwapVal).addReg(OldValRot).addImm(32).addImm(63 - BitSize).addImm(0);
+ BuildMI(MBB, DL, TII->get(ZExtOpcode), Dest)
+ .addReg(OldValRot);
BuildMI(MBB, DL, TII->get(SystemZ::CR))
- .addReg(Dest).addReg(RetryCmpVal);
+ .addReg(Dest).addReg(CmpVal);
BuildMI(MBB, DL, TII->get(SystemZ::BRC))
.addImm(SystemZ::CCMASK_ICMP)
.addImm(SystemZ::CCMASK_CMP_NE).addMBB(DoneMBB);
@@ -7666,17 +7666,12 @@ SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr &MI,
MBB->addSuccessor(SetMBB);
// SetMBB:
- // %RetrySwapVal = RISBG32 %SwapVal, %Dest, 32, 63-BitSize, 0
- // ^^ Replace the upper 32-BitSize bits of the new
- // value with those that we loaded.
- // %StoreVal = RLL %RetrySwapVal, -BitSize(%NegBitShift)
+ // %StoreVal = RLL %RetrySwapVal, -BitSize(%NegBitShift)
// ^^ Rotate the new field to its proper position.
- // %RetryOldVal = CS %Dest, %StoreVal, Disp(%Base)
+ // %RetryOldVal = CS %OldVal, %StoreVal, Disp(%Base)
// JNE LoopMBB
// # fall through to ExitMMB
MBB = SetMBB;
- BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetrySwapVal)
- .addReg(SwapVal).addReg(Dest).addImm(32).addImm(63 - BitSize).addImm(0);
BuildMI(MBB, DL, TII->get(SystemZ::RLL), StoreVal)
.addReg(RetrySwapVal).addReg(NegBitShift).addImm(-BitSize);
BuildMI(MBB, DL, TII->get(CSOpcode), RetryOldVal)
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.h b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
index 78a0e55f9b70..a9f6cd3ae8a4 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.h
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
@@ -550,7 +550,10 @@ class SystemZTargetLowering : public TargetLowering {
unsigned Depth) const override;
ISD::NodeType getExtendForAtomicOps() const override {
- return ISD::ANY_EXTEND;
+ return ISD::ZERO_EXTEND;
+ }
+ ISD::NodeType getExtendForAtomicCmpSwapArg() const override {
+ return ISD::ZERO_EXTEND;
}
bool supportSwiftError() const override {
diff --git a/llvm/test/CodeGen/SystemZ/cmpxchg-01.ll b/llvm/test/CodeGen/SystemZ/cmpxchg-01.ll
index 82c67811fb99..29dbfb2b051e 100644
--- a/llvm/test/CodeGen/SystemZ/cmpxchg-01.ll
+++ b/llvm/test/CodeGen/SystemZ/cmpxchg-01.ll
@@ -15,11 +15,12 @@ define i8 @f1(i8 %dummy, i8 *%src, i8 %cmp, i8 %swap) {
; CHECK-MAIN: risbg [[RISBG:%r[1-9]+]], %r3, 0, 189, 0{{$}}
; CHECK-MAIN-DAG: sll %r3, 3
; CHECK-MAIN-DAG: l [[OLD:%r[0-9]+]], 0([[RISBG]])
+; CHECK-MAIN-DAG: llcr %r4, %r4
; CHECK-MAIN: [[LOOP:\.[^ ]*]]:
; CHECK-MAIN: rll %r2, [[OLD]], 8(%r3)
-; CHECK-MAIN: risbg %r4, %r2, 32, 55, 0
-; CHECK-MAIN: crjlh %r2, %r4, [[EXIT:\.[^ ]*]]
; CHECK-MAIN: risbg %r5, %r2, 32, 55, 0
+; CHECK-MAIN: llcr %r2, %r2
+; CHECK-MAIN: crjlh %r2, %r4, [[EXIT:\.[^ ]*]]
; CHECK-MAIN: rll [[NEW:%r[0-9]+]], %r5, -8({{%r[1-9]+}})
; CHECK-MAIN: cs [[OLD]], [[NEW]], 0([[RISBG]])
; CHECK-MAIN: jl [[LOOP]]
@@ -48,7 +49,6 @@ define i8 @f2(i8 *%src) {
;
; CHECK-SHIFT-LABEL: f2:
; CHECK-SHIFT: lhi [[SWAP:%r[0-9]+]], 88
-; CHECK-SHIFT: risbg
; CHECK-SHIFT: risbg [[SWAP]], {{%r[0-9]+}}, 32, 55, 0
; CHECK-SHIFT: br %r14
%pair = cmpxchg i8 *%src, i8 42, i8 88 seq_cst seq_cst
@@ -62,12 +62,13 @@ define i32 @f3(i8 %dummy, i8 *%src, i8 %cmp, i8 %swap) {
; CHECK-MAIN: risbg [[RISBG:%r[1-9]+]], %r3, 0, 189, 0{{$}}
; CHECK-MAIN-DAG: sll %r3, 3
; CHECK-MAIN-DAG: l [[OLD:%r[0-9]+]], 0([[RISBG]])
+; CHECK-MAIN-DAG: llcr %r2, %r4
; CHECK-MAIN: [[LOOP:\.[^ ]*]]:
; CHECK-MAIN: rll [[TMP:%r[0-9]+]], [[OLD]], 8(%r3)
-; CHECK-MAIN: risbg %r4, [[TMP]], 32, 55, 0
-; CHECK-MAIN: cr [[TMP]], %r4
-; CHECK-MAIN: jlh [[EXIT:\.[^ ]*]]
; CHECK-MAIN: risbg %r5, [[TMP]], 32, 55, 0
+; CHECK-MAIN: llcr [[TMP2:%r[0-9]+]], [[TMP]]
+; CHECK-MAIN: cr [[TMP2]], %r2
+; CHECK-MAIN: jlh [[EXIT:\.[^ ]*]]
; CHECK-MAIN: rll [[NEW:%r[0-9]+]], %r5, -8({{%r[1-9]+}})
; CHECK-MAIN: cs [[OLD]], [[NEW]], 0([[RISBG]])
; CHECK-MAIN: jl [[LOOP]]
@@ -98,12 +99,13 @@ declare void @g()
; CHECK-MAIN: risbg [[RISBG:%r[1-9]+]], %r2, 0, 189, 0{{$}}
; CHECK-MAIN-DAG: sll %r2, 3
; CHECK-MAIN-DAG: l [[OLD:%r[0-9]+]], 0([[RISBG]])
+; CHECK-MAIN-DAG: llcr %r3, %r3
; CHECK-MAIN: [[LOOP:\.[^ ]*]]:
; CHECK-MAIN: rll [[TMP:%r[0-9]+]], [[OLD]], 8(%r2)
-; CHECK-MAIN: risbg %r3, [[TMP]], 32, 55, 0
+; CHECK-MAIN: risbg %r4, [[TMP]], 32, 55, 0
+; CHECK-MAIN: llcr [[TMP]], [[TMP]]
; CHECK-MAIN: cr [[TMP]], %r3
; CHECK-MAIN: jlh [[EXIT:\.[^ ]*]]
-; CHECK-MAIN: risbg %r4, [[TMP]], 32, 55, 0
; CHECK-MAIN: rll [[NEW:%r[0-9]+]], %r4, -8({{%r[1-9]+}})
; CHECK-MAIN: cs [[OLD]], [[NEW]], 0([[RISBG]])
; CHECK-MAIN: jl [[LOOP]]
@@ -136,12 +138,13 @@ exit:
; CHECK-MAIN: risbg [[RISBG:%r[1-9]+]], %r2, 0, 189, 0{{$}}
; CHECK-MAIN-DAG: sll %r2, 3
; CHECK-MAIN-DAG: l [[OLD:%r[0-9]+]], 0([[RISBG]])
+; CHECK-MAIN-DAG: llcr %r3, %r3
; CHECK-MAIN: [[LOOP:\.[^ ]*]]:
; CHECK-MAIN: rll [[TMP:%r[0-9]+]], [[OLD]], 8(%r2)
-; CHECK-MAIN: risbg %r3, [[TMP]], 32, 55, 0
+; CHECK-MAIN: risbg %r4, [[TMP]], 32, 55, 0
+; CHECK-MAIN: llcr [[TMP]], [[TMP]]
; CHECK-MAIN: cr [[TMP]], %r3
; CHECK-MAIN: jlh [[EXIT:\.[^ ]*]]
-; CHECK-MAIN: risbg %r4, [[TMP]], 32, 55, 0
; CHECK-MAIN: rll [[NEW:%r[0-9]+]], %r4, -8({{%r[1-9]+}})
; CHECK-MAIN: cs [[OLD]], [[NEW]], 0([[RISBG]])
; CHECK-MAIN: jl [[LOOP]]
diff --git a/llvm/test/CodeGen/SystemZ/cmpxchg-02.ll b/llvm/test/CodeGen/SystemZ/cmpxchg-02.ll
index 6e266a1308ca..f1522a235d7b 100644
--- a/llvm/test/CodeGen/SystemZ/cmpxchg-02.ll
+++ b/llvm/test/CodeGen/SystemZ/cmpxchg-02.ll
@@ -15,11 +15,12 @@ define i16 @f1(i16 %dummy, i16 *%src, i16 %cmp, i16 %swap) {
; CHECK-MAIN: risbg [[RISBG:%r[1-9]+]], %r3, 0, 189, 0{{$}}
; CHECK-MAIN-DAG: sll %r3, 3
; CHECK-MAIN-DAG: l [[OLD:%r[0-9]+]], 0([[RISBG]])
+; CHECK-MAIN-DAG: llhr %r4, %r4
; CHECK-MAIN: [[LOOP:\.[^ ]*]]:
; CHECK-MAIN: rll %r2, [[OLD]], 16(%r3)
-; CHECK-MAIN: risbg %r4, %r2, 32, 47, 0
-; CHECK-MAIN: crjlh %r2, %r4, [[EXIT:\.[^ ]*]]
; CHECK-MAIN: risbg %r5, %r2, 32, 47, 0
+; CHECK-MAIN: llhr %r2, %r2
+; CHECK-MAIN: crjlh %r2, %r4, [[EXIT:\.[^ ]*]]
; CHECK-MAIN: rll [[NEW:%r[0-9]+]], %r5, -16({{%r[1-9]+}})
; CHECK-MAIN: cs [[OLD]], [[NEW]], 0([[RISBG]])
; CHECK-MAIN: jl [[LOOP]]
@@ -48,7 +49,6 @@ define i16 @f2(i16 *%src) {
;
; CHECK-SHIFT-LABEL: f2:
; CHECK-SHIFT: lhi [[SWAP:%r[0-9]+]], 88
-; CHECK-SHIFT: risbg
; CHECK-SHIFT: risbg [[SWAP]], {{%r[0-9]+}}, 32, 47, 0
; CHECK-SHIFT: br %r14
%pair = cmpxchg i16 *%src, i16 42, i16 88 seq_cst seq_cst
@@ -62,12 +62,13 @@ define i32 @f3(i16 %dummy, i16 *%src, i16 %cmp, i16 %swap) {
; CHECK-MAIN: risbg [[RISBG:%r[1-9]+]], %r3, 0, 189, 0{{$}}
; CHECK-MAIN-DAG: sll %r3, 3
; CHECK-MAIN-DAG: l [[OLD:%r[0-9]+]], 0([[RISBG]])
+; CHECK-MAIN-DAG: llhr %r2, %r4
; CHECK-MAIN: [[LOOP:\.[^ ]*]]:
; CHECK-MAIN: rll [[TMP:%r[0-9]+]], [[OLD]], 16(%r3)
-; CHECK-MAIN: risbg %r4, [[TMP]], 32, 47, 0
-; CHECK-MAIN: cr [[TMP]], %r4
-; CHECK-MAIN: jlh [[EXIT:\.[^ ]*]]
; CHECK-MAIN: risbg %r5, [[TMP]], 32, 47, 0
+; CHECK-MAIN: llhr %r14, %r14
+; CHECK-MAIN: cr [[TMP]], %r2
+; CHECK-MAIN: jlh [[EXIT:\.[^ ]*]]
; CHECK-MAIN: rll [[NEW:%r[0-9]+]], %r5, -16({{%r[1-9]+}})
; CHECK-MAIN: cs [[OLD]], [[NEW]], 0([[RISBG]])
; CHECK-MAIN: jl [[LOOP]]
@@ -97,12 +98,13 @@ declare void @g()
; CHECK-MAIN: risbg [[RISBG:%r[1-9]+]], %r2, 0, 189, 0{{$}}
; CHECK-MAIN-DAG: sll %r2, 3
; CHECK-MAIN-DAG: l [[OLD:%r[0-9]+]], 0([[RISBG]])
+; CHECK-MAIN-DAG: llhr %r3, %r3
; CHECK-MAIN: [[LOOP:\.[^ ]*]]:
; CHECK-MAIN: rll [[TMP:%r[0-9]+]], [[OLD]], 16(%r2)
-; CHECK-MAIN: risbg %r3, [[TMP]], 32, 47, 0
+; CHECK-MAIN: risbg %r4, [[TMP]], 32, 47, 0
+; CHECK-MAIN: llhr %r14, %r14
; CHECK-MAIN: cr [[TMP]], %r3
; CHECK-MAIN: jlh [[EXIT:\.[^ ]*]]
-; CHECK-MAIN: risbg %r4, [[TMP]], 32, 47, 0
; CHECK-MAIN: rll [[NEW:%r[0-9]+]], %r4, -16({{%r[1-9]+}})
; CHECK-MAIN: cs [[OLD]], [[NEW]], 0([[RISBG]])
; CHECK-MAIN: jl [[LOOP]]
@@ -135,12 +137,13 @@ exit:
; CHECK-MAIN: risbg [[RISBG:%r[1-9]+]], %r2, 0, 189, 0{{$}}
; CHECK-MAIN-DAG: sll %r2, 3
; CHECK-MAIN-DAG: l [[OLD:%r[0-9]+]], 0([[RISBG]])
+; CHECK-MAIN-DAG: llhr %r3, %r3
; CHECK-MAIN: [[LOOP:\.[^ ]*]]:
; CHECK-MAIN: rll [[TMP:%r[0-9]+]], [[OLD]], 16(%r2)
-; CHECK-MAIN: risbg %r3, [[TMP]], 32, 47, 0
+; CHECK-MAIN: risbg %r4, [[TMP]], 32, 47, 0
+; CHECK-MAIN: llhr %r14, %r14
; CHECK-MAIN: cr [[TMP]], %r3
; CHECK-MAIN: jlh [[EXIT:\.[^ ]*]]
-; CHECK-MAIN: risbg %r4, [[TMP]], 32, 47, 0
; CHECK-MAIN: rll [[NEW:%r[0-9]+]], %r4, -16({{%r[1-9]+}})
; CHECK-MAIN: cs [[OLD]], [[NEW]], 0([[RISBG]])
; CHECK-MAIN: jl [[LOOP]]
diff --git a/llvm/test/CodeGen/SystemZ/cmpxchg-05.ll b/llvm/test/CodeGen/SystemZ/cmpxchg-05.ll
index ecfe2dabeaf5..9a6d6c9d8485 100644
--- a/llvm/test/CodeGen/SystemZ/cmpxchg-05.ll
+++ b/llvm/test/CodeGen/SystemZ/cmpxchg-05.ll
@@ -6,7 +6,7 @@
; CHECK: crjlh
; CHECK-NOT: llcr
; CHECK-NOT: cr
-; CHECK: llgcr %r2, [[RES:%r[0-9]+]]
+; CHECK: llgfr %r2, [[RES:%r[0-9]+]]
; CHECK-NOT: llcr
; CHECK-NOT: cr
define zeroext i8 @f1(i8* nocapture, i8 zeroext, i8 zeroext) {
@@ -19,7 +19,7 @@ define zeroext i8 @f1(i8* nocapture, i8 zeroext, i8 zeroext) {
; CHECK: crjlh
; CHECK-NOT: llhr
; CHECK-NOT: cr
-; CHECK: llghr %r2, [[RES:%r[0-9]+]]
+; CHECK: llgfr %r2, [[RES:%r[0-9]+]]
; CHECK-NOT: llhr
; CHECK-NOT: cr
define zeroext i16 @f2(i16* nocapture, i16 zeroext, i16 zeroext) {
@@ -53,4 +53,3 @@ define signext i16 @f4(i16* nocapture, i16 signext, i16 signext) {
%res = extractvalue { i16, i1 } %cx, 0
ret i16 %res
}
-
More information about the llvm-commits
mailing list