[llvm] 03934e7 - [SystemZ] Enable AtomicExpand pass (#70398)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Oct 31 01:51:10 PDT 2023
Author: Ilya Leoshkevich
Date: 2023-10-31T09:51:06+01:00
New Revision: 03934e70ef1cf86a6ebd1a56e0b5b46f4fec8e13
URL: https://github.com/llvm/llvm-project/commit/03934e70ef1cf86a6ebd1a56e0b5b46f4fec8e13
DIFF: https://github.com/llvm/llvm-project/commit/03934e70ef1cf86a6ebd1a56e0b5b46f4fec8e13.diff
LOG: [SystemZ] Enable AtomicExpand pass (#70398)
The upcoming OpenMP support for SystemZ requires handling of IR insns
like `atomicrmw fadd`. Normally atomic float operations are expanded by
Clang and such insns do not occur, but OpenMP generates them directly.
Other architectures handle this using the AtomicExpand pass, which
SystemZ did not need so far. Enable it.
Currently AtomicExpand treats atomic load and stores of floats
pessimistically: it casts them to integers, which SystemZ does not need,
since the floating point load and store instructions are already atomic.
However, the way Clang currently expands them is pessimistic as well, so
this change does not make things worse. Optimizing operations on atomic
floats can be a separate change in the future.
This change does not create any differences the Linux kernel build.
Added:
llvm/test/CodeGen/SystemZ/atomic-load-06.ll
llvm/test/CodeGen/SystemZ/atomic-load-07.ll
llvm/test/CodeGen/SystemZ/atomic-load-08.ll
llvm/test/CodeGen/SystemZ/atomic-store-06.ll
llvm/test/CodeGen/SystemZ/atomic-store-07.ll
llvm/test/CodeGen/SystemZ/atomic-store-08.ll
llvm/test/CodeGen/SystemZ/atomicrmw-fadd-01.ll
llvm/test/CodeGen/SystemZ/atomicrmw-fadd-02.ll
llvm/test/CodeGen/SystemZ/atomicrmw-fadd-03.ll
llvm/test/CodeGen/SystemZ/atomicrmw-fmax-01.ll
llvm/test/CodeGen/SystemZ/atomicrmw-fmax-02.ll
llvm/test/CodeGen/SystemZ/atomicrmw-fmax-03.ll
llvm/test/CodeGen/SystemZ/atomicrmw-fmin-01.ll
llvm/test/CodeGen/SystemZ/atomicrmw-fmin-02.ll
llvm/test/CodeGen/SystemZ/atomicrmw-fmin-03.ll
llvm/test/CodeGen/SystemZ/atomicrmw-fsub-01.ll
llvm/test/CodeGen/SystemZ/atomicrmw-fsub-02.ll
llvm/test/CodeGen/SystemZ/atomicrmw-fsub-03.ll
llvm/test/CodeGen/SystemZ/atomicrmw-udec_wrap.ll
llvm/test/CodeGen/SystemZ/atomicrmw-uinc_wrap.ll
llvm/test/CodeGen/SystemZ/atomicrmw-xchg-05.ll
llvm/test/CodeGen/SystemZ/atomicrmw-xchg-06.ll
llvm/test/CodeGen/SystemZ/atomicrmw-xchg-07.ll
Modified:
llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
llvm/lib/Target/SystemZ/SystemZISelLowering.h
llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index 3db777f904df0ad..e6ea4205623d31d 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -872,6 +872,15 @@ bool SystemZTargetLowering::hasInlineStackProbe(const MachineFunction &MF) const
return false;
}
+TargetLowering::AtomicExpansionKind
+SystemZTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
+ return (RMW->isFloatingPointOperation() ||
+ RMW->getOperation() == AtomicRMWInst::UIncWrap ||
+ RMW->getOperation() == AtomicRMWInst::UDecWrap)
+ ? AtomicExpansionKind::CmpXChg
+ : AtomicExpansionKind::None;
+}
+
bool SystemZTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
// We can use CGFI or CLGFI.
return isInt<32>(Imm) || isUInt<32>(Imm);
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.h b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
index 0d4b4873e9d73e6..fd951b935702aaf 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.h
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
@@ -452,6 +452,8 @@ class SystemZTargetLowering : public TargetLowering {
return VT != MVT::f64;
}
bool hasInlineStackProbe(const MachineFunction &MF) const override;
+ AtomicExpansionKind
+ shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const override;
bool isLegalICmpImmediate(int64_t Imm) const override;
bool isLegalAddImmediate(int64_t Imm) const override;
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty,
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp b/llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp
index 8235446da65c458..186494ad2ac614c 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp
@@ -226,6 +226,8 @@ void SystemZPassConfig::addIRPasses() {
addPass(createLoopDataPrefetchPass());
}
+ addPass(createAtomicExpandPass());
+
TargetPassConfig::addIRPasses();
}
diff --git a/llvm/test/CodeGen/SystemZ/atomic-load-06.ll b/llvm/test/CodeGen/SystemZ/atomic-load-06.ll
new file mode 100644
index 000000000000000..c9c5504520345cd
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomic-load-06.ll
@@ -0,0 +1,13 @@
+; Test float atomic loads.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define float @f1(ptr %src) {
+; CHECK-LABEL: f1:
+; CHECK: lgf [[R:%r[0-9]+]], 0(%r2)
+; CHECK: sllg [[R]], [[R]], 32
+; CHECK: ldgr %f0, [[R]]
+; CHECK: br %r14
+ %val = load atomic float, ptr %src seq_cst, align 4
+ ret float %val
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomic-load-07.ll b/llvm/test/CodeGen/SystemZ/atomic-load-07.ll
new file mode 100644
index 000000000000000..d183cb6af3d20d7
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomic-load-07.ll
@@ -0,0 +1,11 @@
+; Test double atomic loads.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define double @f1(ptr %src) {
+; CHECK-LABEL: f1:
+; CHECK: ld %f0, 0(%r2)
+; CHECK: br %r14
+ %val = load atomic double, ptr %src seq_cst, align 8
+ ret double %val
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomic-load-08.ll b/llvm/test/CodeGen/SystemZ/atomic-load-08.ll
new file mode 100644
index 000000000000000..069d2168e19af7a
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomic-load-08.ll
@@ -0,0 +1,20 @@
+; Test long double atomic loads. Expect a libcall.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define void @f1(ptr %ret, ptr %src) {
+; CHECK-LABEL: f1:
+; CHECK: lgr [[RET:%r[0-9]+]], %r2
+; CHECK: la %r4, 160(%r15)
+; CHECK: lghi %r2, 16
+; CHECK: lhi %r5, 5
+; CHECK: brasl %r14, __atomic_load at PLT
+; CHECK: ld [[FL:%f[0-9]+]], 160(%r15)
+; CHECK: ld [[FH:%f[0-9]+]], 168(%r15)
+; CHECK: std [[FL]], 0([[RET]])
+; CHECK: std [[FH]], 8([[RET]])
+; CHECK: br %r14
+ %val = load atomic fp128, ptr %src seq_cst, align 8
+ store fp128 %val, ptr %ret, align 8
+ ret void
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomic-store-06.ll b/llvm/test/CodeGen/SystemZ/atomic-store-06.ll
new file mode 100644
index 000000000000000..fd39793faefc8ef
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomic-store-06.ll
@@ -0,0 +1,13 @@
+; Test float atomic loads.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define void @f1(ptr %src, float %val) {
+; CHECK-LABEL: f1:
+; CHECK: lgdr [[R:%r[0-9]+]], %f0
+; CHECK: srlg [[R]], [[R]], 32
+; CHECK: st [[R]], 0(%r2)
+; CHECK: br %r14
+ store atomic float %val, ptr %src seq_cst, align 4
+ ret void
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomic-store-07.ll b/llvm/test/CodeGen/SystemZ/atomic-store-07.ll
new file mode 100644
index 000000000000000..c904b738f2c576b
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomic-store-07.ll
@@ -0,0 +1,11 @@
+; Test double atomic stores.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define void @f1(ptr %dst, double %val) {
+; CHECK-LABEL: f1:
+; CHECK: std %f0, 0(%r2)
+; CHECK: br %r14
+ store atomic double %val, ptr %dst seq_cst, align 8
+ ret void
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomic-store-08.ll b/llvm/test/CodeGen/SystemZ/atomic-store-08.ll
new file mode 100644
index 000000000000000..b33b283e8dbd76f
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomic-store-08.ll
@@ -0,0 +1,20 @@
+; Test long double atomic stores. Expect a libcall.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define void @f1(ptr %dst, ptr %src) {
+; CHECK-LABEL: f1:
+; CHECK: ld [[FL:%f[0-9]+]], 0(%r3)
+; CHECK: ld [[FH:%f[0-9]+]], 8(%r3)
+; CHECK: lgr %r3, %r2
+; CHECK: std [[FL]], 160(%r15)
+; CHECK: std [[FH]], 168(%r15)
+; CHECK: la %r4, 160(%r15)
+; CHECK: lghi %r2, 16
+; CHECK: lhi %r5, 5
+; CHECK: brasl %r14, __atomic_store at PLT
+; CHECK: br %r14
+ %val = load fp128, ptr %src, align 8
+ store atomic fp128 %val, ptr %dst seq_cst, align 8
+ ret void
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-fadd-01.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-fadd-01.ll
new file mode 100644
index 000000000000000..1bfa055781c9888
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomicrmw-fadd-01.ll
@@ -0,0 +1,22 @@
+; Test atomic float addition. Expect a compare-and-swap loop.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define float @f1(ptr %src, float %b) {
+; CHECK-LABEL: f1:
+; CHECK: le [[F:%f[0-9]+]], 0(%r2)
+; CHECK: [[L:\.L.+]]:
+; CHECK: lgdr [[RI:%r[0-9]+]], [[F]]
+; CHECK: aebr [[F]], %f0
+; CHECK: lgdr [[RO:%r[0-9]+]], [[F]]
+; CHECK: srlg [[RO]], [[RO]], 32
+; CHECK: srlg [[RI]], [[RI]], 32
+; CHECK: cs [[RI]], [[RO]], 0(%r2)
+; CHECK: sllg [[RI]], [[RI]], 32
+; CHECK: ldgr [[F]], [[RI]]
+; CHECK: jl [[L]]
+; CHECK: ler %f0, [[F]]
+; CHECK: br %r14
+ %res = atomicrmw fadd ptr %src, float %b seq_cst
+ ret float %res
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-fadd-02.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-fadd-02.ll
new file mode 100644
index 000000000000000..7f9ee3cbec22e60
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomicrmw-fadd-02.ll
@@ -0,0 +1,19 @@
+; Test atomic double addition. Expect a compare-and-swap loop.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define double @f1(ptr %src, double %b) {
+; CHECK-LABEL: f1:
+; CHECK: ld [[F:%f[0-9]+]], 0(%r2)
+; CHECK: [[L:\.L.+]]:
+; CHECK: lgdr [[RI:%r[0-9]+]], [[F]]
+; CHECK: adbr [[F]], %f0
+; CHECK: lgdr [[RO:%r[0-9]+]], [[F]]
+; CHECK: csg [[RI]], [[RO]], 0(%r2)
+; CHECK: ldgr [[F]], [[RI]]
+; CHECK: jl [[L]]
+; CHECK: ldr %f0, [[F]]
+; CHECK: br %r14
+ %res = atomicrmw fadd ptr %src, double %b seq_cst
+ ret double %res
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-fadd-03.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-fadd-03.ll
new file mode 100644
index 000000000000000..729fcbc4ac1e753
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomicrmw-fadd-03.ll
@@ -0,0 +1,30 @@
+; Test atomic long double addition. Expect a compare-and-swap loop.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define void @f1(ptr %ret, ptr %src, ptr %b) {
+; CHECK-LABEL: f1:
+; CHECK: [[FBL:%f[0-9]+]], 0(%r4)
+; CHECK: [[FBH:%f[0-9]+]], 8(%r4)
+; CHECK: [[FSL:%f[0-9]+]], 0(%r3)
+; CHECK: [[FSH:%f[0-9]+]], 8(%r3)
+; CHECK: [[LABEL:\.L.+]]:
+; CHECK: lgdr [[RISH:%r[0-9]+]], [[FSH]]
+; CHECK: lgdr [[RISL:%r[0-9]+]], [[FSL]]
+; CHECK: axbr [[FSL]], [[FBL]]
+; CHECK: lgdr [[ROSH:%r[0-9]+]], [[FSH]]
+; CHECK: lgdr [[ROSL:%r[0-9]+]], [[FSL]]
+; CHECK: cdsg [[RISL]], [[ROSL]], 0(%r3)
+; CHECK: stg [[RISH]], 168(%r15)
+; CHECK: stg [[RISL]], 160(%r15)
+; CHECK: ld [[FSL]], 160(%r15)
+; CHECK: ld [[FSH]], 168(%r15)
+; CHECK: jl [[LABEL]]
+; CHECK: std [[FSL]], 0(%r2)
+; CHECK: std [[FSH]], 8(%r2)
+; CHECK: br %r14
+ %val = load fp128, ptr %b
+ %res = atomicrmw fadd ptr %src, fp128 %val seq_cst
+ store fp128 %res, ptr %ret
+ ret void
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-fmax-01.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-fmax-01.ll
new file mode 100644
index 000000000000000..80c43137e3a03d1
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomicrmw-fmax-01.ll
@@ -0,0 +1,27 @@
+; Test atomic float maximum.
+; Expect a libcall in a compare-and-swap loop.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define float @f1(ptr %src, float %b) {
+; CHECK-LABEL: f1:
+; CHECK: lgr [[SRC:%r[0-9]+]], %r2
+; CHECK: le [[FSRC:%f[0-9]+]], 0(%r2)
+; CHECK: ler [[FB:%f[0-9]+]], %f0
+; CHECK: [[L:\.L.+]]:
+; CHECK: ler %f0, [[FSRC]]
+; CHECK: ler %f2, [[FB]]
+; CHECK: brasl %r14, fmaxf at PLT
+; CHECK: lgdr [[RO:%r[0-9]+]], %f0
+; CHECK: srlg [[RO]], [[RO]], 32
+; CHECK: lgdr [[RI:%r[0-9]+]], [[FSRC]]
+; CHECK: srlg [[RI]], [[RI]], 32
+; CHECK: cs [[RI]], [[RO]], 0([[SRC]])
+; CHECK: sllg [[RO]], [[RI]], 32
+; CHECK: ldgr [[FSRC]], [[RO]]
+; CHECK: jl [[L]]
+; CHECK: ler %f0, [[FSRC]]
+; CHECK: br %r14
+ %res = atomicrmw fmax ptr %src, float %b seq_cst
+ ret float %res
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-fmax-02.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-fmax-02.ll
new file mode 100644
index 000000000000000..8c245bb049f3f33
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomicrmw-fmax-02.ll
@@ -0,0 +1,24 @@
+; Test atomic double maximum.
+; Expect a libcall in a compare-and-swap loop.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define double @f1(ptr %src, double %b) {
+; CHECK-LABEL: f1:
+; CHECK: lgr [[RB:%r[0-9]+]], %r2
+; CHECK: ld [[FB:%f[0-9]+]], 0(%r2)
+; CHECK: ldr [[FSRC:%f[0-9]+]], %f0
+; CHECK: [[L:\.L.+]]:
+; CHECK: ldr %f0, [[FB]]
+; CHECK: ldr %f2, [[FSRC]]
+; CHECK: brasl %r14, fmax at PLT
+; CHECK: lgdr [[RO:%r[0-9]+]], %f0
+; CHECK: lgdr [[RI:%r[0-9]+]], [[FB]]
+; CHECK: csg [[RI]], [[RO]], 0([[RB]])
+; CHECK: ldgr [[FB]], [[RI]]
+; CHECK: jl [[L]]
+; CHECK: ldr %f0, [[FB]]
+; CHECK: br %r14
+ %res = atomicrmw fmax ptr %src, double %b seq_cst
+ ret double %res
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-fmax-03.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-fmax-03.ll
new file mode 100644
index 000000000000000..3c8ea19f86f8603
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomicrmw-fmax-03.ll
@@ -0,0 +1,42 @@
+; Test atomic long double maximum.
+; Expect a libcall in a compare-and-swap loop.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define void @f1(ptr %ret, ptr %src, ptr %b) {
+; CHECK-LABEL: f1:
+; CHECK: lgr [[SRC:%r[0-9]+]], %r3
+; CHECK: ld [[FBL:%f[0-9]+]], 0(%r4)
+; CHECK: ld [[FBH:%f[0-9]+]], 8(%r4)
+; CHECK: ld [[FSL:%f[0-9]+]], 0(%r3)
+; CHECK: ld [[FSH:%f[0-9]+]], 8(%r3)
+; CHECK: lgr [[RET:%r[0-9]+]], %r2
+; CHECK: [[L:\.L.+]]:
+; CHECK: std [[FBL]], 160(%r15)
+; CHECK: std [[FBH]], 168(%r15)
+; CHECK: la %r2, 192(%r15)
+; CHECK: la %r3, 176(%r15)
+; CHECK: la %r4, 160(%r15)
+; CHECK: std [[FSL]], 176(%r15)
+; CHECK: std [[FSH]], 184(%r15)
+; CHECK: brasl %r14, fmaxl at PLT
+; CHECK: ld [[FL:%f[0-9]+]], 192(%r15)
+; CHECK: ld [[FH:%f[0-9]+]], 200(%r15)
+; CHECK: lgdr [[RH:%r[0-9]+]], [[FH]]
+; CHECK: lgdr [[RL:%r[0-9]+]], [[FL]]
+; CHECK: lgdr [[RSH:%r[0-9]+]], [[FSH]]
+; CHECK: lgdr [[RSL:%r[0-9]+]], [[FSL]]
+; CHECK: cdsg [[RSL]], [[RL]], 0([[SRC]])
+; CHECK: stg [[RSH]], 216(%r15)
+; CHECK: stg [[RSL]], 208(%r15)
+; CHECK: ld [[FSL]], 208(%r15)
+; CHECK: ld [[FSH]], 216(%r15)
+; CHECK: jl [[L]]
+; CHECK: std [[FSL]], 0([[RET]])
+; CHECK: std [[FSH]], 8([[RET]])
+; CHECK: br %r14
+ %val = load fp128, ptr %b
+ %res = atomicrmw fmax ptr %src, fp128 %val seq_cst
+ store fp128 %res, ptr %ret
+ ret void
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-fmin-01.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-fmin-01.ll
new file mode 100644
index 000000000000000..c67b02e688de3b7
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomicrmw-fmin-01.ll
@@ -0,0 +1,27 @@
+; Test atomic float minimum.
+; Expect a libcall in a compare-and-swap loop.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define float @f1(ptr %src, float %b) {
+; CHECK-LABEL: f1:
+; CHECK: lgr [[SRC:%r[0-9]+]], %r2
+; CHECK: le [[FSRC:%f[0-9]+]], 0(%r2)
+; CHECK: ler [[FB:%f[0-9]+]], %f0
+; CHECK: [[L:\.L.+]]:
+; CHECK: ler %f0, [[FSRC]]
+; CHECK: ler %f2, [[FB]]
+; CHECK: brasl %r14, fminf at PLT
+; CHECK: lgdr [[RO:%r[0-9]+]], %f0
+; CHECK: srlg [[RO]], [[RO]], 32
+; CHECK: lgdr [[RI:%r[0-9]+]], [[FSRC]]
+; CHECK: srlg [[RI]], [[RI]], 32
+; CHECK: cs [[RI]], [[RO]], 0([[SRC]])
+; CHECK: sllg [[RO]], [[RI]], 32
+; CHECK: ldgr [[FSRC]], [[RO]]
+; CHECK: jl [[L]]
+; CHECK: ler %f0, [[FSRC]]
+; CHECK: br %r14
+ %res = atomicrmw fmin ptr %src, float %b seq_cst
+ ret float %res
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-fmin-02.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-fmin-02.ll
new file mode 100644
index 000000000000000..6691a8b21d11bbd
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomicrmw-fmin-02.ll
@@ -0,0 +1,24 @@
+; Test atomic double minimum.
+; Expect a libcall in a compare-and-swap loop.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define double @f1(ptr %src, double %b) {
+; CHECK-LABEL: f1:
+; CHECK: lgr [[SRC:%r[0-9]+]], %r2
+; CHECK: ld [[FSRC:%f[0-9]+]], 0(%r2)
+; CHECK: ldr [[FB:%f[0-9]+]], %f0
+; CHECK: [[L:\.L.+]]:
+; CHECK: ldr %f0, [[FSRC]]
+; CHECK: ldr %f2, [[FB]]
+; CHECK: brasl %r14, fmin at PLT
+; CHECK: lgdr [[RO:%r[0-9]+]], %f0
+; CHECK: lgdr [[RI:%r[0-9]+]], [[FSRC]]
+; CHECK: csg [[RI]], [[RO]], 0([[SRC]])
+; CHECK: ldgr [[FSRC]], [[RI]]
+; CHECK: jl [[L]]
+; CHECK: ldr %f0, [[FSRC]]
+; CHECK: br %r14
+ %res = atomicrmw fmin ptr %src, double %b seq_cst
+ ret double %res
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-fmin-03.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-fmin-03.ll
new file mode 100644
index 000000000000000..dfa2cc021d16671
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomicrmw-fmin-03.ll
@@ -0,0 +1,42 @@
+; Test atomic long double minimum.
+; Expect a libcall in a compare-and-swap loop.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define void @f1(ptr %ret, ptr %src, ptr %b) {
+; CHECK-LABEL: f1:
+; CHECK: lgr [[SRC:%r[0-9]+]], %r3
+; CHECK: ld [[FBL:%f[0-9]+]], 0(%r4)
+; CHECK: ld [[FBH:%f[0-9]+]], 8(%r4)
+; CHECK: ld [[FSL:%f[0-9]+]], 0(%r3)
+; CHECK: ld [[FSH:%f[0-9]+]], 8(%r3)
+; CHECK: lgr [[RET:%r[0-9]+]], %r2
+; CHECK: [[L:\.L.+]]:
+; CHECK: std [[FBL]], 160(%r15)
+; CHECK: std [[FBH]], 168(%r15)
+; CHECK: la %r2, 192(%r15)
+; CHECK: la %r3, 176(%r15)
+; CHECK: la %r4, 160(%r15)
+; CHECK: std [[FSL]], 176(%r15)
+; CHECK: std [[FSH]], 184(%r15)
+; CHECK: brasl %r14, fminl at PLT
+; CHECK: ld [[FL:%f[0-9]+]], 192(%r15)
+; CHECK: ld [[FH:%f[0-9]+]], 200(%r15)
+; CHECK: lgdr [[RH:%r[0-9]+]], [[FH]]
+; CHECK: lgdr [[RL:%r[0-9]+]], [[FL]]
+; CHECK: lgdr [[RSH:%r[0-9]+]], [[FSH]]
+; CHECK: lgdr [[RSL:%r[0-9]+]], [[FSL]]
+; CHECK: cdsg [[RSL]], [[RL]], 0([[SRC]])
+; CHECK: stg [[RSH]], 216(%r15)
+; CHECK: stg [[RSL]], 208(%r15)
+; CHECK: ld [[FSL]], 208(%r15)
+; CHECK: ld [[FSH]], 216(%r15)
+; CHECK: jl [[L]]
+; CHECK: std [[FSL]], 0([[RET]])
+; CHECK: std [[FSH]], 8([[RET]])
+; CHECK: br %r14
+ %val = load fp128, ptr %b
+ %res = atomicrmw fmin ptr %src, fp128 %val seq_cst
+ store fp128 %res, ptr %ret
+ ret void
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-fsub-01.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-fsub-01.ll
new file mode 100644
index 000000000000000..3f4ad31762753f6
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomicrmw-fsub-01.ll
@@ -0,0 +1,22 @@
+; Test atomic float subtraction. Expect a compare-and-swap loop.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define float @f1(ptr %src, float %b) {
+; CHECK-LABEL: f1:
+; CHECK: le [[F:%f[0-9]+]], 0(%r2)
+; CHECK: [[L:\.L.+]]:
+; CHECK: lgdr [[RI:%r[0-9]+]], [[F]]
+; CHECK: sebr [[F]], %f0
+; CHECK: lgdr [[RO:%r[0-9]+]], [[F]]
+; CHECK: srlg [[RO]], [[RO]], 32
+; CHECK: srlg [[RI]], [[RI]], 32
+; CHECK: cs [[RI]], [[RO]], 0(%r2)
+; CHECK: sllg [[RI]], [[RI]], 32
+; CHECK: ldgr [[F]], [[RI]]
+; CHECK: jl [[L]]
+; CHECK: ler %f0, [[F]]
+; CHECK: br %r14
+ %res = atomicrmw fsub ptr %src, float %b seq_cst
+ ret float %res
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-fsub-02.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-fsub-02.ll
new file mode 100644
index 000000000000000..69071f9cbe029a5
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomicrmw-fsub-02.ll
@@ -0,0 +1,19 @@
+; Test atomic double subtraction. Expect a compare-and-swap loop.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define double @f1(ptr %src, double %b) {
+; CHECK-LABEL: f1:
+; CHECK: ld [[F:%f[0-9]+]], 0(%r2)
+; CHECK: [[L:\.L.+]]:
+; CHECK: lgdr [[RI:%r[0-9]+]], [[F]]
+; CHECK: sdbr [[F]], %f0
+; CHECK: lgdr [[RO:%r[0-9]+]], [[F]]
+; CHECK: csg [[RI]], [[RO]], 0(%r2)
+; CHECK: ldgr [[F]], [[RI]]
+; CHECK: jl [[L]]
+; CHECK: ldr %f0, [[F]]
+; CHECK: br %r14
+ %res = atomicrmw fsub ptr %src, double %b seq_cst
+ ret double %res
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-fsub-03.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-fsub-03.ll
new file mode 100644
index 000000000000000..aacbc942f1a56bd
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomicrmw-fsub-03.ll
@@ -0,0 +1,30 @@
+; Test atomic long double subtraction. Expect a compare-and-swap loop.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define void @f1(ptr %ret, ptr %src, ptr %b) {
+; CHECK-LABEL: f1:
+; CHECK: [[FBL:%f[0-9]+]], 0(%r4)
+; CHECK: [[FBH:%f[0-9]+]], 8(%r4)
+; CHECK: [[FSL:%f[0-9]+]], 0(%r3)
+; CHECK: [[FSH:%f[0-9]+]], 8(%r3)
+; CHECK: [[LABEL:\.L.+]]:
+; CHECK: lgdr [[RISH:%r[0-9]+]], [[FSH]]
+; CHECK: lgdr [[RISL:%r[0-9]+]], [[FSL]]
+; CHECK: sxbr [[FSL]], [[FBL]]
+; CHECK: lgdr [[ROSH:%r[0-9]+]], [[FSH]]
+; CHECK: lgdr [[ROSL:%r[0-9]+]], [[FSL]]
+; CHECK: cdsg [[RISL]], [[ROSL]], 0(%r3)
+; CHECK: stg [[RISH]], 168(%r15)
+; CHECK: stg [[RISL]], 160(%r15)
+; CHECK: ld [[FSL]], 160(%r15)
+; CHECK: ld [[FSH]], 168(%r15)
+; CHECK: jl [[LABEL]]
+; CHECK: std [[FSL]], 0(%r2)
+; CHECK: std [[FSH]], 8(%r2)
+; CHECK: br %r14
+ %val = load fp128, ptr %b
+ %res = atomicrmw fsub ptr %src, fp128 %val seq_cst
+ store fp128 %res, ptr %ret
+ ret void
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-udec_wrap.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-udec_wrap.ll
new file mode 100644
index 000000000000000..d6427fc29f05168
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomicrmw-udec_wrap.ll
@@ -0,0 +1,28 @@
+; Test decrementing until to a minimum value. Expect a compare-and-swap loop.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define i64 @f1(ptr %src, i64 %b) {
+; CHECK-LABEL: f1:
+; CHECK: lgr [[SRC:%r[0-9]+]], %r2
+; CHECK: lg [[RI:%r[0-9]+]], 0(%r2)
+; CHECK: j [[L2:\.L.+]]
+; CHECK: [[L1:\.L.+]]:
+; CHECK: csg [[RI]], [[RO:%r[0-9]+]], 0([[SRC]])
+; CHECK: je [[L4:\.L.+]]
+; CHECK: [[L2]]:
+; CHECK: lgr [[RO]], [[RI]]
+; CHECK: slgfi [[RO]], 1
+; CHECK: lgr [[RB:%r[0-9]+]], %r3
+; CHECK: clgrjh [[RI]], %r3, [[L3:\.L.+]]
+; CHECK: lgr [[RB]], [[RO]]
+; CHECK: [[L3]]:
+; CHECK: lgr [[RO]], [[RI]]
+; CHECK: slgfi [[RO]], 1
+; CHECK: lgr [[RO]], %r3
+; CHECK: jle [[L1]]
+; CHECK: [[L4]]:
+; CHECK: br %r14
+ %res = atomicrmw udec_wrap ptr %src, i64 %b seq_cst
+ ret i64 %res
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-uinc_wrap.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-uinc_wrap.ll
new file mode 100644
index 000000000000000..cf90d756535959b
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomicrmw-uinc_wrap.ll
@@ -0,0 +1,22 @@
+; Test incrementing up to a maximum value. Expect a compare-and-swap loop.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define i64 @f1(ptr %src, i64 %b) {
+; CHECK-LABEL: f1:
+; CHECK: lgr [[SRC:%r[0-9]+]], %r2
+; CHECK: lg [[RI:%r[0-9]+]], 0(%r2)
+; CHECK: j [[L2:\.L.+]]
+; CHECK: [[L1:\.L.+]]:
+; CHECK: csg [[RI]], [[RO:%r[0-9]+]], 0([[SRC]])
+; CHECK: je [[L3:\.L.+]]
+; CHECK: [[L2]]:
+; CHECK: lghi [[RO]], 0
+; CHECK: clgrjhe [[RI]], %r3, [[L1]]
+; CHECK: la [[RO]], 1([[RI]])
+; CHECK: j [[L1]]
+; CHECK: [[L3]]:
+; CHECK: br %r14
+ %res = atomicrmw uinc_wrap ptr %src, i64 %b seq_cst
+ ret i64 %res
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-05.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-05.ll
new file mode 100644
index 000000000000000..d86e476115c3868
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-05.ll
@@ -0,0 +1,17 @@
+; Test float atomic exchange.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define float @f1(ptr %src, float %b) {
+; CHECK-LABEL: f1:
+; CHECK: l [[RI:%r[0-9]+]], 0(%r2)
+; CHECK: lgdr [[RO:%r[0-9]+]], %f0
+; CHECK: [[LABEL:\.[^:]*]]:
+; CHECK: cs [[RI]], [[RO]], 0(%r2)
+; CHECK: jl [[LABEL]]
+; CHECK: sllg [[RI]], [[RI]], 32
+; CHECK: ldgr %f0, [[RI]]
+; CHECK: br %r14
+ %res = atomicrmw xchg ptr %src, float %b seq_cst
+ ret float %res
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-06.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-06.ll
new file mode 100644
index 000000000000000..9b78e783035dd81
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-06.ll
@@ -0,0 +1,16 @@
+; Test double atomic exchange.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define double @f1(ptr %src, double %b) {
+; CHECK-LABEL: f1:
+; CHECK: lg [[RI:%r[0-9]+]], 0(%r2)
+; CHECK: lgdr [[RO:%r[0-9]+]], %f0
+; CHECK: [[LABEL:\.[^:]*]]:
+; CHECK: csg [[RI]], [[RO]], 0(%r2)
+; CHECK: jl [[LABEL]]
+; CHECK: ldgr %f0, [[RI]]
+; CHECK: br %r14
+ %res = atomicrmw xchg ptr %src, double %b seq_cst
+ ret double %res
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-07.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-07.ll
new file mode 100644
index 000000000000000..80cc85158e45e6a
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-07.ll
@@ -0,0 +1,24 @@
+; Test long double atomic exchange.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define void @f1(ptr %ret, ptr %src, ptr %b) {
+; CHECK-LABEL: f1:
+; CHECK: lg [[RH:%r[0-9]+]], 8(%r4)
+; CHECK: lgr [[RET:%r[0-9]+]], %r2
+; CHECK: lg [[RL:%r[0-9]+]], 0(%r4)
+; CHECK: stg [[RH]], 168(%r15)
+; CHECK: la %r2, 176(%r15)
+; CHECK: la %r4, 160(%r15)
+; CHECK: stg [[RL]], 160(%r15)
+; CHECK: brasl %r14, __sync_lock_test_and_set_16 at PLT
+; CHECK: lg [[RH2:%r[0-9]+]], 184(%r15)
+; CHECK: lg [[RL2:%r[0-9]+]], 176(%r15)
+; CHECK: stg [[RH]], 8([[RET]])
+; CHECK: stg [[RL]], 0([[RET]])
+; CHECK: br %r14
+ %val = load fp128, ptr %b, align 8
+ %res = atomicrmw xchg ptr %src, fp128 %val seq_cst
+ store fp128 %res, ptr %ret, align 8
+ ret void
+}
More information about the llvm-commits
mailing list