[llvm] 301b21a - [AArch64] Regenerate and add more tests for i128 atomics.
Eli Friedman via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 21 11:28:40 PDT 2021
Author: Eli Friedman
Date: 2021-07-21T11:28:27-07:00
New Revision: 301b21a941ebd94cef2f1912a547960223983887
URL: https://github.com/llvm/llvm-project/commit/301b21a941ebd94cef2f1912a547960223983887
DIFF: https://github.com/llvm/llvm-project/commit/301b21a941ebd94cef2f1912a547960223983887.diff
LOG: [AArch64] Regenerate and add more tests for i128 atomics.
Generating these tests unfortunately means a lot of junk, but it's hard
to write/update these tests by hand.
Added tests focus on atomic orderings for cmpxchg.
Actually writing out these tests showed some potentially dubious
results; we should probably consider using casp for 128-bit atomic
load/store/rmw.
Added:
Modified:
llvm/test/CodeGen/AArch64/GlobalISel/arm64-atomic-128.ll
llvm/test/CodeGen/AArch64/arm64-atomic-128.ll
Removed:
################################################################################
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-atomic-128.ll b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-atomic-128.ll
index da82f2447ef2..b5b49e86f151 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-atomic-128.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-atomic-128.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc < %s -mtriple=arm64-linux-gnu -verify-machineinstrs -global-isel -global-isel-abort=1 | FileCheck %s --check-prefix=CHECK-LLSC-O1
; RUN: llc < %s -mtriple=arm64-linux-gnu -verify-machineinstrs -mcpu=apple-a13 -global-isel -global-isel-abort=1 | FileCheck %s --check-prefix=CHECK-CAS-O1
; RUN: llc < %s -mtriple=arm64-linux-gnu -verify-machineinstrs -O0 -global-isel -global-isel-abort=1 | FileCheck %s --check-prefix=CHECK-LLSC-O0
@@ -6,44 +7,82 @@
define void @val_compare_and_swap(i128* %p, i128 %oldval, i128 %newval) {
; CHECK-LLSC-O1-LABEL: val_compare_and_swap:
-; CHECK-LLSC-O1: ldaxp {{x[0-9]+}}, {{x[0-9]+}}, [x0]
-; [... LOTS of stuff that is generic IR unrelated to atomic operations ...]
-; CHECK-LLSC-O1: stxp {{w[0-9]+}}, {{x[0-9]+}}, {{x[0-9]+}}, [x0]
+; CHECK-LLSC-O1: // %bb.0:
+; CHECK-LLSC-O1-NEXT: .LBB0_1: // =>This Inner Loop Header: Depth=1
+; CHECK-LLSC-O1-NEXT: ldaxp x8, x9, [x0]
+; CHECK-LLSC-O1-NEXT: cmp x8, x2
+; CHECK-LLSC-O1-NEXT: cset w10, ne
+; CHECK-LLSC-O1-NEXT: cmp x9, x3
+; CHECK-LLSC-O1-NEXT: cinc w10, w10, ne
+; CHECK-LLSC-O1-NEXT: cbz w10, .LBB0_3
+; CHECK-LLSC-O1-NEXT: // %bb.2: // in Loop: Header=BB0_1 Depth=1
+; CHECK-LLSC-O1-NEXT: stxp w10, x8, x9, [x0]
+; CHECK-LLSC-O1-NEXT: cbnz w10, .LBB0_1
+; CHECK-LLSC-O1-NEXT: b .LBB0_4
+; CHECK-LLSC-O1-NEXT: .LBB0_3: // in Loop: Header=BB0_1 Depth=1
+; CHECK-LLSC-O1-NEXT: stxp w10, x4, x5, [x0]
+; CHECK-LLSC-O1-NEXT: cbnz w10, .LBB0_1
+; CHECK-LLSC-O1-NEXT: .LBB0_4:
+; CHECK-LLSC-O1-NEXT: mov v0.d[0], x8
+; CHECK-LLSC-O1-NEXT: mov v0.d[1], x9
+; CHECK-LLSC-O1-NEXT: str q0, [x0]
+; CHECK-LLSC-O1-NEXT: ret
;
; CHECK-CAS-O1-LABEL: val_compare_and_swap:
-; CHECK-CAS-O1: caspa x2, x3, x4, x5, [x0]
-; CHECK-CAS-O1: mov v[[OLD:[0-9]+]].d[0], x2
-; CHECK-CAS-O1: mov v[[OLD]].d[1], x3
-; CHECK-CAS-O1: str q[[OLD]], [x0]
-
+; CHECK-CAS-O1: // %bb.0:
+; CHECK-CAS-O1-NEXT: // kill: def $x2 killed $x2 killed $x2_x3 def $x2_x3
+; CHECK-CAS-O1-NEXT: // kill: def $x4 killed $x4 killed $x4_x5 def $x4_x5
+; CHECK-CAS-O1-NEXT: // kill: def $x3 killed $x3 killed $x2_x3 def $x2_x3
+; CHECK-CAS-O1-NEXT: // kill: def $x5 killed $x5 killed $x4_x5 def $x4_x5
+; CHECK-CAS-O1-NEXT: caspa x2, x3, x4, x5, [x0]
+; CHECK-CAS-O1-NEXT: mov v0.d[0], x2
+; CHECK-CAS-O1-NEXT: mov v0.d[1], x3
+; CHECK-CAS-O1-NEXT: str q0, [x0]
+; CHECK-CAS-O1-NEXT: ret
+;
; CHECK-LLSC-O0-LABEL: val_compare_and_swap:
-; CHECK-LLSC-O0: .LBB0_1:
-; CHECK-LLSC-O0: ldaxp [[OLD_LO:x[0-9]+]], [[OLD_HI:x[0-9]+]], [x0]
-; CHECK-LLSC-O0: cmp [[OLD_LO]], x2
-; CHECK-LLSC-O0: cset [[EQUAL_TMP:w[0-9]+]], ne
-; CHECK-LLSC-O0: cmp [[OLD_HI]], x3
-; CHECK-LLSC-O0: cinc [[EQUAL:w[0-9]+]], [[EQUAL_TMP]], ne
-; CHECK-LLSC-O0: cbnz [[EQUAL]], .LBB0_3
-; CHECK-LLSC-O0: stxp [[STATUS:w[0-9]+]], x4, x5, [x0]
-; CHECK-LLSC-O0: cbnz [[STATUS]], .LBB0_1
-; CHECK-LLSC-O0: .LBB0_3:
-; CHECK-LLSC-O0: mov v[[OLD:[0-9]+]].d[0], [[OLD_LO]]
-; CHECK-LLSC-O0: mov v[[OLD]].d[1], [[OLD_HI]]
-; CHECK-LLSC-O0: str q[[OLD]], [x0]
-
-
+; CHECK-LLSC-O0: // %bb.0:
+; CHECK-LLSC-O0-NEXT: .LBB0_1: // =>This Inner Loop Header: Depth=1
+; CHECK-LLSC-O0-NEXT: ldaxp x9, x8, [x0]
+; CHECK-LLSC-O0-NEXT: cmp x9, x2
+; CHECK-LLSC-O0-NEXT: cset w10, ne
+; CHECK-LLSC-O0-NEXT: cmp x8, x3
+; CHECK-LLSC-O0-NEXT: cinc w10, w10, ne
+; CHECK-LLSC-O0-NEXT: cbnz w10, .LBB0_3
+; CHECK-LLSC-O0-NEXT: // %bb.2: // in Loop: Header=BB0_1 Depth=1
+; CHECK-LLSC-O0-NEXT: stxp w10, x4, x5, [x0]
+; CHECK-LLSC-O0-NEXT: cbnz w10, .LBB0_1
+; CHECK-LLSC-O0-NEXT: b .LBB0_4
+; CHECK-LLSC-O0-NEXT: .LBB0_3: // in Loop: Header=BB0_1 Depth=1
+; CHECK-LLSC-O0-NEXT: stxp w10, x9, x8, [x0]
+; CHECK-LLSC-O0-NEXT: cbnz w10, .LBB0_1
+; CHECK-LLSC-O0-NEXT: .LBB0_4:
+; CHECK-LLSC-O0-NEXT: // implicit-def: $q0
+; CHECK-LLSC-O0-NEXT: mov v0.d[0], x9
+; CHECK-LLSC-O0-NEXT: mov v0.d[1], x8
+; CHECK-LLSC-O0-NEXT: str q0, [x0]
+; CHECK-LLSC-O0-NEXT: ret
+;
; CHECK-CAS-O0-LABEL: val_compare_and_swap:
-; CHECK-CAS-O0: str x3, [sp, #[[SLOT:[0-9]+]]]
-; CHECK-CAS-O0: mov [[NEW_HI_TMP:x[0-9]+]], x5
-; CHECK-CAS-O0: ldr [[DESIRED_HI_TMP:x[0-9]+]], [sp, #[[SLOT]]]
-; CHECK-CAS-O0: mov [[DESIRED_HI:x[0-9]+]], [[DESIRED_HI_TMP]]
-; CHECK-CAS-O0: mov [[NEW_HI:x[0-9]+]], [[NEW_HI_TMP]]
-; CHECK-CAS-O0: caspa x2, [[DESIRED_HI]], x4, [[NEW_HI]], [x0]
-; CHECK-CAS-O0: mov [[OLD_LO:x[0-9]+]], x2
-; CHECK-CAS-O0: mov [[OLD_HI:x[0-9]+]], x3
-; CHECK-CAS-O0: mov v[[OLD:[0-9]+]].d[0], [[OLD_LO]]
-; CHECK-CAS-O0: mov v[[OLD]].d[1], [[OLD_HI]]
-; CHECK-CAS-O0: str q[[OLD]], [x0]
+; CHECK-CAS-O0: // %bb.0:
+; CHECK-CAS-O0-NEXT: sub sp, sp, #16 // =16
+; CHECK-CAS-O0-NEXT: .cfi_def_cfa_offset 16
+; CHECK-CAS-O0-NEXT: str x3, [sp, #8] // 8-byte Folded Spill
+; CHECK-CAS-O0-NEXT: mov x1, x5
+; CHECK-CAS-O0-NEXT: ldr x5, [sp, #8] // 8-byte Folded Reload
+; CHECK-CAS-O0-NEXT: // kill: def $x2 killed $x2 def $x2_x3
+; CHECK-CAS-O0-NEXT: mov x3, x5
+; CHECK-CAS-O0-NEXT: // kill: def $x4 killed $x4 def $x4_x5
+; CHECK-CAS-O0-NEXT: mov x5, x1
+; CHECK-CAS-O0-NEXT: caspa x2, x3, x4, x5, [x0]
+; CHECK-CAS-O0-NEXT: mov x9, x2
+; CHECK-CAS-O0-NEXT: mov x8, x3
+; CHECK-CAS-O0-NEXT: // implicit-def: $q0
+; CHECK-CAS-O0-NEXT: mov v0.d[0], x9
+; CHECK-CAS-O0-NEXT: mov v0.d[1], x8
+; CHECK-CAS-O0-NEXT: str q0, [x0]
+; CHECK-CAS-O0-NEXT: add sp, sp, #16 // =16
+; CHECK-CAS-O0-NEXT: ret
%pair = cmpxchg i128* %p, i128 %oldval, i128 %newval acquire acquire
%val = extractvalue { i128, i1 } %pair, 0
@@ -53,20 +92,82 @@ define void @val_compare_and_swap(i128* %p, i128 %oldval, i128 %newval) {
define void @val_compare_and_swap_monotonic_seqcst(i128* %p, i128 %oldval, i128 %newval) {
; CHECK-LLSC-O1-LABEL: val_compare_and_swap_monotonic_seqcst:
-; CHECK-LLSC-O1: ldaxp {{x[0-9]+}}, {{x[0-9]+}}, [x0]
-; [... LOTS of stuff that is generic IR unrelated to atomic operations ...]
-; CHECK-LLSC-O1: stlxp {{w[0-9]+}}, {{x[0-9]+}}, {{x[0-9]+}}, [x0]
+; CHECK-LLSC-O1: // %bb.0:
+; CHECK-LLSC-O1-NEXT: .LBB1_1: // =>This Inner Loop Header: Depth=1
+; CHECK-LLSC-O1-NEXT: ldaxp x8, x9, [x0]
+; CHECK-LLSC-O1-NEXT: cmp x8, x2
+; CHECK-LLSC-O1-NEXT: cset w10, ne
+; CHECK-LLSC-O1-NEXT: cmp x9, x3
+; CHECK-LLSC-O1-NEXT: cinc w10, w10, ne
+; CHECK-LLSC-O1-NEXT: cbz w10, .LBB1_3
+; CHECK-LLSC-O1-NEXT: // %bb.2: // in Loop: Header=BB1_1 Depth=1
+; CHECK-LLSC-O1-NEXT: stlxp w10, x8, x9, [x0]
+; CHECK-LLSC-O1-NEXT: cbnz w10, .LBB1_1
+; CHECK-LLSC-O1-NEXT: b .LBB1_4
+; CHECK-LLSC-O1-NEXT: .LBB1_3: // in Loop: Header=BB1_1 Depth=1
+; CHECK-LLSC-O1-NEXT: stlxp w10, x4, x5, [x0]
+; CHECK-LLSC-O1-NEXT: cbnz w10, .LBB1_1
+; CHECK-LLSC-O1-NEXT: .LBB1_4:
+; CHECK-LLSC-O1-NEXT: mov v0.d[0], x8
+; CHECK-LLSC-O1-NEXT: mov v0.d[1], x9
+; CHECK-LLSC-O1-NEXT: str q0, [x0]
+; CHECK-LLSC-O1-NEXT: ret
;
; CHECK-CAS-O1-LABEL: val_compare_and_swap_monotonic_seqcst:
-; CHECK-CAS-O1: caspal x2, x3, x4, x5, [x0]
-
+; CHECK-CAS-O1: // %bb.0:
+; CHECK-CAS-O1-NEXT: // kill: def $x2 killed $x2 killed $x2_x3 def $x2_x3
+; CHECK-CAS-O1-NEXT: // kill: def $x4 killed $x4 killed $x4_x5 def $x4_x5
+; CHECK-CAS-O1-NEXT: // kill: def $x3 killed $x3 killed $x2_x3 def $x2_x3
+; CHECK-CAS-O1-NEXT: // kill: def $x5 killed $x5 killed $x4_x5 def $x4_x5
+; CHECK-CAS-O1-NEXT: caspal x2, x3, x4, x5, [x0]
+; CHECK-CAS-O1-NEXT: mov v0.d[0], x2
+; CHECK-CAS-O1-NEXT: mov v0.d[1], x3
+; CHECK-CAS-O1-NEXT: str q0, [x0]
+; CHECK-CAS-O1-NEXT: ret
+;
; CHECK-LLSC-O0-LABEL: val_compare_and_swap_monotonic_seqcst:
-; CHECK-LLSC-O0: .LBB1_1:
-; CHECK-LLSC-O0: ldaxp
-; CHECK-LLSC-O0: stlxp
-
+; CHECK-LLSC-O0: // %bb.0:
+; CHECK-LLSC-O0-NEXT: .LBB1_1: // =>This Inner Loop Header: Depth=1
+; CHECK-LLSC-O0-NEXT: ldaxp x9, x8, [x0]
+; CHECK-LLSC-O0-NEXT: cmp x9, x2
+; CHECK-LLSC-O0-NEXT: cset w10, ne
+; CHECK-LLSC-O0-NEXT: cmp x8, x3
+; CHECK-LLSC-O0-NEXT: cinc w10, w10, ne
+; CHECK-LLSC-O0-NEXT: cbnz w10, .LBB1_3
+; CHECK-LLSC-O0-NEXT: // %bb.2: // in Loop: Header=BB1_1 Depth=1
+; CHECK-LLSC-O0-NEXT: stlxp w10, x4, x5, [x0]
+; CHECK-LLSC-O0-NEXT: cbnz w10, .LBB1_1
+; CHECK-LLSC-O0-NEXT: b .LBB1_4
+; CHECK-LLSC-O0-NEXT: .LBB1_3: // in Loop: Header=BB1_1 Depth=1
+; CHECK-LLSC-O0-NEXT: stlxp w10, x9, x8, [x0]
+; CHECK-LLSC-O0-NEXT: cbnz w10, .LBB1_1
+; CHECK-LLSC-O0-NEXT: .LBB1_4:
+; CHECK-LLSC-O0-NEXT: // implicit-def: $q0
+; CHECK-LLSC-O0-NEXT: mov v0.d[0], x9
+; CHECK-LLSC-O0-NEXT: mov v0.d[1], x8
+; CHECK-LLSC-O0-NEXT: str q0, [x0]
+; CHECK-LLSC-O0-NEXT: ret
+;
; CHECK-CAS-O0-LABEL: val_compare_and_swap_monotonic_seqcst:
-; CHECK-CAS-O0: caspal
+; CHECK-CAS-O0: // %bb.0:
+; CHECK-CAS-O0-NEXT: sub sp, sp, #16 // =16
+; CHECK-CAS-O0-NEXT: .cfi_def_cfa_offset 16
+; CHECK-CAS-O0-NEXT: str x3, [sp, #8] // 8-byte Folded Spill
+; CHECK-CAS-O0-NEXT: mov x1, x5
+; CHECK-CAS-O0-NEXT: ldr x5, [sp, #8] // 8-byte Folded Reload
+; CHECK-CAS-O0-NEXT: // kill: def $x2 killed $x2 def $x2_x3
+; CHECK-CAS-O0-NEXT: mov x3, x5
+; CHECK-CAS-O0-NEXT: // kill: def $x4 killed $x4 def $x4_x5
+; CHECK-CAS-O0-NEXT: mov x5, x1
+; CHECK-CAS-O0-NEXT: caspal x2, x3, x4, x5, [x0]
+; CHECK-CAS-O0-NEXT: mov x9, x2
+; CHECK-CAS-O0-NEXT: mov x8, x3
+; CHECK-CAS-O0-NEXT: // implicit-def: $q0
+; CHECK-CAS-O0-NEXT: mov v0.d[0], x9
+; CHECK-CAS-O0-NEXT: mov v0.d[1], x8
+; CHECK-CAS-O0-NEXT: str q0, [x0]
+; CHECK-CAS-O0-NEXT: add sp, sp, #16 // =16
+; CHECK-CAS-O0-NEXT: ret
%pair = cmpxchg i128* %p, i128 %oldval, i128 %newval monotonic seq_cst
%val = extractvalue { i128, i1 } %pair, 0
@@ -76,20 +177,82 @@ define void @val_compare_and_swap_monotonic_seqcst(i128* %p, i128 %oldval, i128
define void @val_compare_and_swap_release_acquire(i128* %p, i128 %oldval, i128 %newval) {
; CHECK-LLSC-O1-LABEL: val_compare_and_swap_release_acquire:
-; CHECK-LLSC-O1: ldaxp {{x[0-9]+}}, {{x[0-9]+}}, [x0]
-; [... LOTS of stuff that is generic IR unrelated to atomic operations ...]
-; CHECK-LLSC-O1: stlxp {{w[0-9]+}}, {{x[0-9]+}}, {{x[0-9]+}}, [x0]
+; CHECK-LLSC-O1: // %bb.0:
+; CHECK-LLSC-O1-NEXT: .LBB2_1: // =>This Inner Loop Header: Depth=1
+; CHECK-LLSC-O1-NEXT: ldaxp x8, x9, [x0]
+; CHECK-LLSC-O1-NEXT: cmp x8, x2
+; CHECK-LLSC-O1-NEXT: cset w10, ne
+; CHECK-LLSC-O1-NEXT: cmp x9, x3
+; CHECK-LLSC-O1-NEXT: cinc w10, w10, ne
+; CHECK-LLSC-O1-NEXT: cbz w10, .LBB2_3
+; CHECK-LLSC-O1-NEXT: // %bb.2: // in Loop: Header=BB2_1 Depth=1
+; CHECK-LLSC-O1-NEXT: stlxp w10, x8, x9, [x0]
+; CHECK-LLSC-O1-NEXT: cbnz w10, .LBB2_1
+; CHECK-LLSC-O1-NEXT: b .LBB2_4
+; CHECK-LLSC-O1-NEXT: .LBB2_3: // in Loop: Header=BB2_1 Depth=1
+; CHECK-LLSC-O1-NEXT: stlxp w10, x4, x5, [x0]
+; CHECK-LLSC-O1-NEXT: cbnz w10, .LBB2_1
+; CHECK-LLSC-O1-NEXT: .LBB2_4:
+; CHECK-LLSC-O1-NEXT: mov v0.d[0], x8
+; CHECK-LLSC-O1-NEXT: mov v0.d[1], x9
+; CHECK-LLSC-O1-NEXT: str q0, [x0]
+; CHECK-LLSC-O1-NEXT: ret
;
; CHECK-CAS-O1-LABEL: val_compare_and_swap_release_acquire:
-; CHECK-CAS-O1: caspal x2, x3, x4, x5, [x0]
-
+; CHECK-CAS-O1: // %bb.0:
+; CHECK-CAS-O1-NEXT: // kill: def $x2 killed $x2 killed $x2_x3 def $x2_x3
+; CHECK-CAS-O1-NEXT: // kill: def $x4 killed $x4 killed $x4_x5 def $x4_x5
+; CHECK-CAS-O1-NEXT: // kill: def $x3 killed $x3 killed $x2_x3 def $x2_x3
+; CHECK-CAS-O1-NEXT: // kill: def $x5 killed $x5 killed $x4_x5 def $x4_x5
+; CHECK-CAS-O1-NEXT: caspal x2, x3, x4, x5, [x0]
+; CHECK-CAS-O1-NEXT: mov v0.d[0], x2
+; CHECK-CAS-O1-NEXT: mov v0.d[1], x3
+; CHECK-CAS-O1-NEXT: str q0, [x0]
+; CHECK-CAS-O1-NEXT: ret
+;
; CHECK-LLSC-O0-LABEL: val_compare_and_swap_release_acquire:
-; CHECK-LLSC-O0: .LBB2_1:
-; CHECK-LLSC-O0: ldaxp
-; CHECK-LLSC-O0: stlxp
-
+; CHECK-LLSC-O0: // %bb.0:
+; CHECK-LLSC-O0-NEXT: .LBB2_1: // =>This Inner Loop Header: Depth=1
+; CHECK-LLSC-O0-NEXT: ldaxp x9, x8, [x0]
+; CHECK-LLSC-O0-NEXT: cmp x9, x2
+; CHECK-LLSC-O0-NEXT: cset w10, ne
+; CHECK-LLSC-O0-NEXT: cmp x8, x3
+; CHECK-LLSC-O0-NEXT: cinc w10, w10, ne
+; CHECK-LLSC-O0-NEXT: cbnz w10, .LBB2_3
+; CHECK-LLSC-O0-NEXT: // %bb.2: // in Loop: Header=BB2_1 Depth=1
+; CHECK-LLSC-O0-NEXT: stlxp w10, x4, x5, [x0]
+; CHECK-LLSC-O0-NEXT: cbnz w10, .LBB2_1
+; CHECK-LLSC-O0-NEXT: b .LBB2_4
+; CHECK-LLSC-O0-NEXT: .LBB2_3: // in Loop: Header=BB2_1 Depth=1
+; CHECK-LLSC-O0-NEXT: stlxp w10, x9, x8, [x0]
+; CHECK-LLSC-O0-NEXT: cbnz w10, .LBB2_1
+; CHECK-LLSC-O0-NEXT: .LBB2_4:
+; CHECK-LLSC-O0-NEXT: // implicit-def: $q0
+; CHECK-LLSC-O0-NEXT: mov v0.d[0], x9
+; CHECK-LLSC-O0-NEXT: mov v0.d[1], x8
+; CHECK-LLSC-O0-NEXT: str q0, [x0]
+; CHECK-LLSC-O0-NEXT: ret
+;
; CHECK-CAS-O0-LABEL: val_compare_and_swap_release_acquire:
-; CHECK-CAS-O0: caspal
+; CHECK-CAS-O0: // %bb.0:
+; CHECK-CAS-O0-NEXT: sub sp, sp, #16 // =16
+; CHECK-CAS-O0-NEXT: .cfi_def_cfa_offset 16
+; CHECK-CAS-O0-NEXT: str x3, [sp, #8] // 8-byte Folded Spill
+; CHECK-CAS-O0-NEXT: mov x1, x5
+; CHECK-CAS-O0-NEXT: ldr x5, [sp, #8] // 8-byte Folded Reload
+; CHECK-CAS-O0-NEXT: // kill: def $x2 killed $x2 def $x2_x3
+; CHECK-CAS-O0-NEXT: mov x3, x5
+; CHECK-CAS-O0-NEXT: // kill: def $x4 killed $x4 def $x4_x5
+; CHECK-CAS-O0-NEXT: mov x5, x1
+; CHECK-CAS-O0-NEXT: caspal x2, x3, x4, x5, [x0]
+; CHECK-CAS-O0-NEXT: mov x9, x2
+; CHECK-CAS-O0-NEXT: mov x8, x3
+; CHECK-CAS-O0-NEXT: // implicit-def: $q0
+; CHECK-CAS-O0-NEXT: mov v0.d[0], x9
+; CHECK-CAS-O0-NEXT: mov v0.d[1], x8
+; CHECK-CAS-O0-NEXT: str q0, [x0]
+; CHECK-CAS-O0-NEXT: add sp, sp, #16 // =16
+; CHECK-CAS-O0-NEXT: ret
%pair = cmpxchg i128* %p, i128 %oldval, i128 %newval release acquire
%val = extractvalue { i128, i1 } %pair, 0
@@ -97,22 +260,257 @@ define void @val_compare_and_swap_release_acquire(i128* %p, i128 %oldval, i128 %
ret void
}
-define void @atomic_load_relaxed(i64, i64, i128* %p, i128* %p2) {
-; CHECK-LLSC-O1-LABEL: atomic_load_relaxed
-; CHECK-LLSC-O1: ldxp
-; CHECK-LLSC-O1: stxp
-
-; CHECK-LLSC-O0-LABEL: atomic_load_relaxed
-; CHECK-LLSC-O0: ldxp
-; CHECK-LLSC-O0: stxp
+define void @val_compare_and_swap_monotonic(i128* %p, i128 %oldval, i128 %newval) {
+; CHECK-LLSC-O1-LABEL: val_compare_and_swap_monotonic:
+; CHECK-LLSC-O1: // %bb.0:
+; CHECK-LLSC-O1-NEXT: .LBB3_1: // =>This Inner Loop Header: Depth=1
+; CHECK-LLSC-O1-NEXT: ldaxp x8, x9, [x0]
+; CHECK-LLSC-O1-NEXT: cmp x8, x2
+; CHECK-LLSC-O1-NEXT: cset w10, ne
+; CHECK-LLSC-O1-NEXT: cmp x9, x3
+; CHECK-LLSC-O1-NEXT: cinc w10, w10, ne
+; CHECK-LLSC-O1-NEXT: cbz w10, .LBB3_3
+; CHECK-LLSC-O1-NEXT: // %bb.2: // in Loop: Header=BB3_1 Depth=1
+; CHECK-LLSC-O1-NEXT: stlxp w10, x8, x9, [x0]
+; CHECK-LLSC-O1-NEXT: cbnz w10, .LBB3_1
+; CHECK-LLSC-O1-NEXT: b .LBB3_4
+; CHECK-LLSC-O1-NEXT: .LBB3_3: // in Loop: Header=BB3_1 Depth=1
+; CHECK-LLSC-O1-NEXT: stlxp w10, x4, x5, [x0]
+; CHECK-LLSC-O1-NEXT: cbnz w10, .LBB3_1
+; CHECK-LLSC-O1-NEXT: .LBB3_4:
+; CHECK-LLSC-O1-NEXT: mov v0.d[0], x8
+; CHECK-LLSC-O1-NEXT: mov v0.d[1], x9
+; CHECK-LLSC-O1-NEXT: str q0, [x0]
+; CHECK-LLSC-O1-NEXT: ret
+;
+; CHECK-CAS-O1-LABEL: val_compare_and_swap_monotonic:
+; CHECK-CAS-O1: // %bb.0:
+; CHECK-CAS-O1-NEXT: // kill: def $x2 killed $x2 killed $x2_x3 def $x2_x3
+; CHECK-CAS-O1-NEXT: // kill: def $x4 killed $x4 killed $x4_x5 def $x4_x5
+; CHECK-CAS-O1-NEXT: // kill: def $x3 killed $x3 killed $x2_x3 def $x2_x3
+; CHECK-CAS-O1-NEXT: // kill: def $x5 killed $x5 killed $x4_x5 def $x4_x5
+; CHECK-CAS-O1-NEXT: caspal x2, x3, x4, x5, [x0]
+; CHECK-CAS-O1-NEXT: mov v0.d[0], x2
+; CHECK-CAS-O1-NEXT: mov v0.d[1], x3
+; CHECK-CAS-O1-NEXT: str q0, [x0]
+; CHECK-CAS-O1-NEXT: ret
+;
+; CHECK-LLSC-O0-LABEL: val_compare_and_swap_monotonic:
+; CHECK-LLSC-O0: // %bb.0:
+; CHECK-LLSC-O0-NEXT: .LBB3_1: // =>This Inner Loop Header: Depth=1
+; CHECK-LLSC-O0-NEXT: ldaxp x9, x8, [x0]
+; CHECK-LLSC-O0-NEXT: cmp x9, x2
+; CHECK-LLSC-O0-NEXT: cset w10, ne
+; CHECK-LLSC-O0-NEXT: cmp x8, x3
+; CHECK-LLSC-O0-NEXT: cinc w10, w10, ne
+; CHECK-LLSC-O0-NEXT: cbnz w10, .LBB3_3
+; CHECK-LLSC-O0-NEXT: // %bb.2: // in Loop: Header=BB3_1 Depth=1
+; CHECK-LLSC-O0-NEXT: stlxp w10, x4, x5, [x0]
+; CHECK-LLSC-O0-NEXT: cbnz w10, .LBB3_1
+; CHECK-LLSC-O0-NEXT: b .LBB3_4
+; CHECK-LLSC-O0-NEXT: .LBB3_3: // in Loop: Header=BB3_1 Depth=1
+; CHECK-LLSC-O0-NEXT: stlxp w10, x9, x8, [x0]
+; CHECK-LLSC-O0-NEXT: cbnz w10, .LBB3_1
+; CHECK-LLSC-O0-NEXT: .LBB3_4:
+; CHECK-LLSC-O0-NEXT: // implicit-def: $q0
+; CHECK-LLSC-O0-NEXT: mov v0.d[0], x9
+; CHECK-LLSC-O0-NEXT: mov v0.d[1], x8
+; CHECK-LLSC-O0-NEXT: str q0, [x0]
+; CHECK-LLSC-O0-NEXT: ret
+;
+; CHECK-CAS-O0-LABEL: val_compare_and_swap_monotonic:
+; CHECK-CAS-O0: // %bb.0:
+; CHECK-CAS-O0-NEXT: sub sp, sp, #16 // =16
+; CHECK-CAS-O0-NEXT: .cfi_def_cfa_offset 16
+; CHECK-CAS-O0-NEXT: str x3, [sp, #8] // 8-byte Folded Spill
+; CHECK-CAS-O0-NEXT: mov x1, x5
+; CHECK-CAS-O0-NEXT: ldr x5, [sp, #8] // 8-byte Folded Reload
+; CHECK-CAS-O0-NEXT: // kill: def $x2 killed $x2 def $x2_x3
+; CHECK-CAS-O0-NEXT: mov x3, x5
+; CHECK-CAS-O0-NEXT: // kill: def $x4 killed $x4 def $x4_x5
+; CHECK-CAS-O0-NEXT: mov x5, x1
+; CHECK-CAS-O0-NEXT: caspal x2, x3, x4, x5, [x0]
+; CHECK-CAS-O0-NEXT: mov x9, x2
+; CHECK-CAS-O0-NEXT: mov x8, x3
+; CHECK-CAS-O0-NEXT: // implicit-def: $q0
+; CHECK-CAS-O0-NEXT: mov v0.d[0], x9
+; CHECK-CAS-O0-NEXT: mov v0.d[1], x8
+; CHECK-CAS-O0-NEXT: str q0, [x0]
+; CHECK-CAS-O0-NEXT: add sp, sp, #16 // =16
+; CHECK-CAS-O0-NEXT: ret
+ %pair = cmpxchg i128* %p, i128 %oldval, i128 %newval release acquire
+ %val = extractvalue { i128, i1 } %pair, 0
+ store i128 %val, i128* %p
+ ret void
+}
-; CHECK-CAS-O1-LABEL: atomic_load_relaxed
-; CHECK-CAS-O1: ldxp
-; CHECK-CAS-O1: stxp
+define void @atomic_load_relaxed(i64, i64, i128* %p, i128* %p2) {
+; CHECK-LLSC-O1-LABEL: atomic_load_relaxed:
+; CHECK-LLSC-O1: // %bb.0:
+; CHECK-LLSC-O1-NEXT: mov w8, #64
+; CHECK-LLSC-O1-NEXT: sub x9, x8, #64 // =64
+; CHECK-LLSC-O1-NEXT: .LBB4_1: // %atomicrmw.start
+; CHECK-LLSC-O1-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-LLSC-O1-NEXT: ldxp x11, x10, [x2]
+; CHECK-LLSC-O1-NEXT: sub x12, x8, #64 // =64
+; CHECK-LLSC-O1-NEXT: tst wzr, #0x1
+; CHECK-LLSC-O1-NEXT: lsl x13, x10, x8
+; CHECK-LLSC-O1-NEXT: lsr x14, x10, x9
+; CHECK-LLSC-O1-NEXT: lsl x10, x10, x12
+; CHECK-LLSC-O1-NEXT: csel x10, x14, x10, ne
+; CHECK-LLSC-O1-NEXT: csel x13, x13, xzr, ne
+; CHECK-LLSC-O1-NEXT: csel x10, xzr, x10, ne
+; CHECK-LLSC-O1-NEXT: orr x11, x11, x13
+; CHECK-LLSC-O1-NEXT: lsl x13, x10, x9
+; CHECK-LLSC-O1-NEXT: lsr x12, x10, x12
+; CHECK-LLSC-O1-NEXT: orr x13, x13, x11, lsr #0
+; CHECK-LLSC-O1-NEXT: tst wzr, #0x1
+; CHECK-LLSC-O1-NEXT: csel x12, x13, x12, ne
+; CHECK-LLSC-O1-NEXT: csel x12, x11, x12, ne
+; CHECK-LLSC-O1-NEXT: stxp w13, x11, x12, [x2]
+; CHECK-LLSC-O1-NEXT: cbnz w13, .LBB4_1
+; CHECK-LLSC-O1-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-LLSC-O1-NEXT: mov v0.d[0], x11
+; CHECK-LLSC-O1-NEXT: mov v0.d[1], x10
+; CHECK-LLSC-O1-NEXT: str q0, [x3]
+; CHECK-LLSC-O1-NEXT: ret
+;
+; CHECK-CAS-O1-LABEL: atomic_load_relaxed:
+; CHECK-CAS-O1: // %bb.0:
+; CHECK-CAS-O1-NEXT: mov w8, #64
+; CHECK-CAS-O1-NEXT: sub x9, x8, #64 // =64
+; CHECK-CAS-O1-NEXT: .LBB4_1: // %atomicrmw.start
+; CHECK-CAS-O1-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-CAS-O1-NEXT: ldxp x11, x10, [x2]
+; CHECK-CAS-O1-NEXT: sub x12, x8, #64 // =64
+; CHECK-CAS-O1-NEXT: lsl x13, x10, x8
+; CHECK-CAS-O1-NEXT: lsr x14, x10, x9
+; CHECK-CAS-O1-NEXT: lsl x10, x10, x12
+; CHECK-CAS-O1-NEXT: tst wzr, #0x1
+; CHECK-CAS-O1-NEXT: csel x13, x13, xzr, ne
+; CHECK-CAS-O1-NEXT: csel x10, x14, x10, ne
+; CHECK-CAS-O1-NEXT: csel x10, xzr, x10, ne
+; CHECK-CAS-O1-NEXT: orr x11, x11, x13
+; CHECK-CAS-O1-NEXT: lsl x13, x10, x9
+; CHECK-CAS-O1-NEXT: orr x13, x13, x11, lsr #0
+; CHECK-CAS-O1-NEXT: lsr x12, x10, x12
+; CHECK-CAS-O1-NEXT: tst wzr, #0x1
+; CHECK-CAS-O1-NEXT: csel x12, x13, x12, ne
+; CHECK-CAS-O1-NEXT: csel x12, x11, x12, ne
+; CHECK-CAS-O1-NEXT: stxp w13, x11, x12, [x2]
+; CHECK-CAS-O1-NEXT: cbnz w13, .LBB4_1
+; CHECK-CAS-O1-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-CAS-O1-NEXT: mov v0.d[0], x11
+; CHECK-CAS-O1-NEXT: mov v0.d[1], x10
+; CHECK-CAS-O1-NEXT: str q0, [x3]
+; CHECK-CAS-O1-NEXT: ret
+;
+; CHECK-LLSC-O0-LABEL: atomic_load_relaxed:
+; CHECK-LLSC-O0: // %bb.0:
+; CHECK-LLSC-O0-NEXT: sub sp, sp, #64 // =64
+; CHECK-LLSC-O0-NEXT: .cfi_def_cfa_offset 64
+; CHECK-LLSC-O0-NEXT: str x2, [sp, #48] // 8-byte Folded Spill
+; CHECK-LLSC-O0-NEXT: str x3, [sp, #56] // 8-byte Folded Spill
+; CHECK-LLSC-O0-NEXT: .LBB4_1: // %atomicrmw.start
+; CHECK-LLSC-O0-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-LLSC-O0-NEXT: ldr x11, [sp, #48] // 8-byte Folded Reload
+; CHECK-LLSC-O0-NEXT: ldxp x9, x15, [x11]
+; CHECK-LLSC-O0-NEXT: mov x12, xzr
+; CHECK-LLSC-O0-NEXT: mov w8, #64
+; CHECK-LLSC-O0-NEXT: // kill: def $x8 killed $w8
+; CHECK-LLSC-O0-NEXT: mov w10, #64
+; CHECK-LLSC-O0-NEXT: // kill: def $x10 killed $w10
+; CHECK-LLSC-O0-NEXT: str x10, [sp, #8] // 8-byte Folded Spill
+; CHECK-LLSC-O0-NEXT: subs x16, x10, #64 // =64
+; CHECK-LLSC-O0-NEXT: subs x13, x8, #64 // =64
+; CHECK-LLSC-O0-NEXT: lsl x14, x15, x10
+; CHECK-LLSC-O0-NEXT: lsr x13, x15, x13
+; CHECK-LLSC-O0-NEXT: orr x13, x13, x12
+; CHECK-LLSC-O0-NEXT: lsl x15, x15, x16
+; CHECK-LLSC-O0-NEXT: subs x16, x10, #64 // =64
+; CHECK-LLSC-O0-NEXT: csel x14, x14, x12, lo
+; CHECK-LLSC-O0-NEXT: subs x16, x10, #64 // =64
+; CHECK-LLSC-O0-NEXT: csel x13, x13, x15, lo
+; CHECK-LLSC-O0-NEXT: subs x15, x10, #0 // =0
+; CHECK-LLSC-O0-NEXT: csel x13, x12, x13, eq
+; CHECK-LLSC-O0-NEXT: orr x9, x9, x14
+; CHECK-LLSC-O0-NEXT: orr x12, x12, x13
+; CHECK-LLSC-O0-NEXT: // implicit-def: $q0
+; CHECK-LLSC-O0-NEXT: mov v0.d[0], x9
+; CHECK-LLSC-O0-NEXT: str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-LLSC-O0-NEXT: mov v0.d[1], x12
+; CHECK-LLSC-O0-NEXT: str q0, [sp, #32] // 16-byte Folded Spill
+; CHECK-LLSC-O0-NEXT: subs x13, x10, #64 // =64
+; CHECK-LLSC-O0-NEXT: subs x8, x8, #64 // =64
+; CHECK-LLSC-O0-NEXT: lsl x8, x12, x8
+; CHECK-LLSC-O0-NEXT: orr x8, x8, x9, lsr #0
+; CHECK-LLSC-O0-NEXT: lsr x12, x12, x13
+; CHECK-LLSC-O0-NEXT: subs x13, x10, #64 // =64
+; CHECK-LLSC-O0-NEXT: csel x8, x8, x12, lo
+; CHECK-LLSC-O0-NEXT: subs x10, x10, #0 // =0
+; CHECK-LLSC-O0-NEXT: csel x10, x9, x8, eq
+; CHECK-LLSC-O0-NEXT: stxp w8, x9, x10, [x11]
+; CHECK-LLSC-O0-NEXT: cbnz w8, .LBB4_1
+; CHECK-LLSC-O0-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-LLSC-O0-NEXT: ldr q0, [sp, #32] // 16-byte Folded Reload
+; CHECK-LLSC-O0-NEXT: ldr x8, [sp, #56] // 8-byte Folded Reload
+; CHECK-LLSC-O0-NEXT: str q0, [x8]
+; CHECK-LLSC-O0-NEXT: add sp, sp, #64 // =64
+; CHECK-LLSC-O0-NEXT: ret
+;
+; CHECK-CAS-O0-LABEL: atomic_load_relaxed:
+; CHECK-CAS-O0: // %bb.0:
+; CHECK-CAS-O0-NEXT: sub sp, sp, #64 // =64
+; CHECK-CAS-O0-NEXT: .cfi_def_cfa_offset 64
+; CHECK-CAS-O0-NEXT: str x2, [sp, #48] // 8-byte Folded Spill
+; CHECK-CAS-O0-NEXT: str x3, [sp, #56] // 8-byte Folded Spill
+; CHECK-CAS-O0-NEXT: .LBB4_1: // %atomicrmw.start
+; CHECK-CAS-O0-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-CAS-O0-NEXT: ldr x11, [sp, #48] // 8-byte Folded Reload
+; CHECK-CAS-O0-NEXT: ldxp x9, x15, [x11]
+; CHECK-CAS-O0-NEXT: mov x12, #0
+; CHECK-CAS-O0-NEXT: mov w8, #64
+; CHECK-CAS-O0-NEXT: // kill: def $x8 killed $w8
+; CHECK-CAS-O0-NEXT: mov w10, #64
+; CHECK-CAS-O0-NEXT: // kill: def $x10 killed $w10
+; CHECK-CAS-O0-NEXT: str x10, [sp, #8] // 8-byte Folded Spill
+; CHECK-CAS-O0-NEXT: subs x16, x10, #64 // =64
+; CHECK-CAS-O0-NEXT: subs x13, x8, #64 // =64
+; CHECK-CAS-O0-NEXT: lsl x14, x15, x10
+; CHECK-CAS-O0-NEXT: lsr x13, x15, x13
+; CHECK-CAS-O0-NEXT: orr x13, x13, x12
+; CHECK-CAS-O0-NEXT: lsl x15, x15, x16
+; CHECK-CAS-O0-NEXT: subs x16, x10, #64 // =64
+; CHECK-CAS-O0-NEXT: csel x14, x14, x12, lo
+; CHECK-CAS-O0-NEXT: subs x16, x10, #64 // =64
+; CHECK-CAS-O0-NEXT: csel x13, x13, x15, lo
+; CHECK-CAS-O0-NEXT: subs x15, x10, #0 // =0
+; CHECK-CAS-O0-NEXT: csel x13, x12, x13, eq
+; CHECK-CAS-O0-NEXT: orr x9, x9, x14
+; CHECK-CAS-O0-NEXT: orr x12, x12, x13
+; CHECK-CAS-O0-NEXT: // implicit-def: $q0
+; CHECK-CAS-O0-NEXT: mov v0.d[0], x9
+; CHECK-CAS-O0-NEXT: str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-CAS-O0-NEXT: mov v0.d[1], x12
+; CHECK-CAS-O0-NEXT: str q0, [sp, #32] // 16-byte Folded Spill
+; CHECK-CAS-O0-NEXT: subs x13, x10, #64 // =64
+; CHECK-CAS-O0-NEXT: subs x8, x8, #64 // =64
+; CHECK-CAS-O0-NEXT: lsl x8, x12, x8
+; CHECK-CAS-O0-NEXT: orr x8, x8, x9, lsr #0
+; CHECK-CAS-O0-NEXT: lsr x12, x12, x13
+; CHECK-CAS-O0-NEXT: subs x13, x10, #64 // =64
+; CHECK-CAS-O0-NEXT: csel x8, x8, x12, lo
+; CHECK-CAS-O0-NEXT: subs x10, x10, #0 // =0
+; CHECK-CAS-O0-NEXT: csel x10, x9, x8, eq
+; CHECK-CAS-O0-NEXT: stxp w8, x9, x10, [x11]
+; CHECK-CAS-O0-NEXT: cbnz w8, .LBB4_1
+; CHECK-CAS-O0-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-CAS-O0-NEXT: ldr q0, [sp, #32] // 16-byte Folded Reload
+; CHECK-CAS-O0-NEXT: ldr x8, [sp, #56] // 8-byte Folded Reload
+; CHECK-CAS-O0-NEXT: str q0, [x8]
+; CHECK-CAS-O0-NEXT: add sp, sp, #64 // =64
+; CHECK-CAS-O0-NEXT: ret
-; CHECK-CAS-O0-LABEL: atomic_load_relaxed
-; CHECK-CAS-O0: ldxp
-; CHECK-CAS-O0: stxp
%r = load atomic i128, i128* %p monotonic, align 16
store i128 %r, i128* %p2
ret void
diff --git a/llvm/test/CodeGen/AArch64/arm64-atomic-128.ll b/llvm/test/CodeGen/AArch64/arm64-atomic-128.ll
index e78a6af63c2f..89ae80a71007 100644
--- a/llvm/test/CodeGen/AArch64/arm64-atomic-128.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-atomic-128.ll
@@ -1,42 +1,232 @@
-; RUN: llc < %s -mtriple=arm64-linux-gnu -verify-machineinstrs -mcpu=cyclone | FileCheck %s
-; RUN: llc < %s -mtriple=arm64-linux-gnu -verify-machineinstrs -mcpu=cyclone -mattr=+outline-atomics | FileCheck %s -check-prefix=OUTLINE-ATOMICS
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=arm64-linux-gnu -verify-machineinstrs -mcpu=cyclone | FileCheck %s -check-prefixes=CHECK,NOOUTLINE
+; RUN: llc < %s -mtriple=arm64-linux-gnu -verify-machineinstrs -mcpu=cyclone -mattr=+outline-atomics | FileCheck %s -check-prefixes=CHECK,OUTLINE
+; RUN: llc < %s -mtriple=arm64-linux-gnu -verify-machineinstrs -mcpu=cyclone -mattr=+lse | FileCheck %s -check-prefixes=CHECK,LSE
@var = global i128 0
define i128 @val_compare_and_swap(i128* %p, i128 %oldval, i128 %newval) {
-; OUTLINE-ATOMICS: bl __aarch64_cas16_acq
-; CHECK-LABEL: val_compare_and_swap:
-; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]:
-; CHECK: ldaxp [[RESULTLO:x[0-9]+]], [[RESULTHI:x[0-9]+]], [x[[ADDR:[0-9]+]]]
-; CHECK: cmp
-; CHECK: cset
-; CHECK: cmp
-; CHECK: cinc [[MISMATCH:w[0-9]+]]
-; CHECK: cbz [[MISMATCH]], [[EQUAL:.LBB[0-9]+_[0-9]+]]
-; CHECK: stxp [[SCRATCH_RES:w[0-9]+]], [[RESULTLO]], [[RESULTHI]], [x[[ADDR]]]
-; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]]
-; CHECK: b [[DONE:.LBB[0-9]+_[0-9]+]]
-; CHECK: [[EQUAL]]:
-; CHECK: stxp [[SCRATCH_RES:w[0-9]+]], x4, x5, [x[[ADDR]]]
-; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]]
-; CHECK: [[DONE]]:
+; NOOUTLINE-LABEL: val_compare_and_swap:
+; NOOUTLINE: // %bb.0:
+; NOOUTLINE-NEXT: .LBB0_1: // =>This Inner Loop Header: Depth=1
+; NOOUTLINE-NEXT: ldaxp x8, x1, [x0]
+; NOOUTLINE-NEXT: cmp x8, x2
+; NOOUTLINE-NEXT: cset w9, ne
+; NOOUTLINE-NEXT: cmp x1, x3
+; NOOUTLINE-NEXT: cinc w9, w9, ne
+; NOOUTLINE-NEXT: cbz w9, .LBB0_3
+; NOOUTLINE-NEXT: // %bb.2: // in Loop: Header=BB0_1 Depth=1
+; NOOUTLINE-NEXT: stxp w9, x8, x1, [x0]
+; NOOUTLINE-NEXT: cbnz w9, .LBB0_1
+; NOOUTLINE-NEXT: b .LBB0_4
+; NOOUTLINE-NEXT: .LBB0_3: // in Loop: Header=BB0_1 Depth=1
+; NOOUTLINE-NEXT: stxp w9, x4, x5, [x0]
+; NOOUTLINE-NEXT: cbnz w9, .LBB0_1
+; NOOUTLINE-NEXT: .LBB0_4:
+; NOOUTLINE-NEXT: mov x0, x8
+; NOOUTLINE-NEXT: ret
+;
+; OUTLINE-LABEL: val_compare_and_swap:
+; OUTLINE: // %bb.0:
+; OUTLINE-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
+; OUTLINE-NEXT: .cfi_def_cfa_offset 16
+; OUTLINE-NEXT: .cfi_offset w30, -16
+; OUTLINE-NEXT: mov x1, x3
+; OUTLINE-NEXT: mov x8, x0
+; OUTLINE-NEXT: mov x0, x2
+; OUTLINE-NEXT: mov x2, x4
+; OUTLINE-NEXT: mov x3, x5
+; OUTLINE-NEXT: mov x4, x8
+; OUTLINE-NEXT: bl __aarch64_cas16_acq
+; OUTLINE-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
+; OUTLINE-NEXT: ret
+;
+; LSE-LABEL: val_compare_and_swap:
+; LSE: // %bb.0:
+; LSE-NEXT: // kill: def $x5 killed $x5 killed $x4_x5 def $x4_x5
+; LSE-NEXT: // kill: def $x3 killed $x3 killed $x2_x3 def $x2_x3
+; LSE-NEXT: // kill: def $x4 killed $x4 killed $x4_x5 def $x4_x5
+; LSE-NEXT: // kill: def $x2 killed $x2 killed $x2_x3 def $x2_x3
+; LSE-NEXT: caspa x2, x3, x4, x5, [x0]
+; LSE-NEXT: mov x0, x2
+; LSE-NEXT: mov x1, x3
+; LSE-NEXT: ret
%pair = cmpxchg i128* %p, i128 %oldval, i128 %newval acquire acquire
%val = extractvalue { i128, i1 } %pair, 0
ret i128 %val
}
+define i128 @val_compare_and_swap_seqcst(i128* %p, i128 %oldval, i128 %newval) {
+; NOOUTLINE-LABEL: val_compare_and_swap_seqcst:
+; NOOUTLINE: // %bb.0:
+; NOOUTLINE-NEXT: .LBB1_1: // =>This Inner Loop Header: Depth=1
+; NOOUTLINE-NEXT: ldaxp x8, x1, [x0]
+; NOOUTLINE-NEXT: cmp x8, x2
+; NOOUTLINE-NEXT: cset w9, ne
+; NOOUTLINE-NEXT: cmp x1, x3
+; NOOUTLINE-NEXT: cinc w9, w9, ne
+; NOOUTLINE-NEXT: cbz w9, .LBB1_3
+; NOOUTLINE-NEXT: // %bb.2: // in Loop: Header=BB1_1 Depth=1
+; NOOUTLINE-NEXT: stlxp w9, x8, x1, [x0]
+; NOOUTLINE-NEXT: cbnz w9, .LBB1_1
+; NOOUTLINE-NEXT: b .LBB1_4
+; NOOUTLINE-NEXT: .LBB1_3: // in Loop: Header=BB1_1 Depth=1
+; NOOUTLINE-NEXT: stlxp w9, x4, x5, [x0]
+; NOOUTLINE-NEXT: cbnz w9, .LBB1_1
+; NOOUTLINE-NEXT: .LBB1_4:
+; NOOUTLINE-NEXT: mov x0, x8
+; NOOUTLINE-NEXT: ret
+;
+; OUTLINE-LABEL: val_compare_and_swap_seqcst:
+; OUTLINE: // %bb.0:
+; OUTLINE-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
+; OUTLINE-NEXT: .cfi_def_cfa_offset 16
+; OUTLINE-NEXT: .cfi_offset w30, -16
+; OUTLINE-NEXT: mov x1, x3
+; OUTLINE-NEXT: mov x8, x0
+; OUTLINE-NEXT: mov x0, x2
+; OUTLINE-NEXT: mov x2, x4
+; OUTLINE-NEXT: mov x3, x5
+; OUTLINE-NEXT: mov x4, x8
+; OUTLINE-NEXT: bl __aarch64_cas16_acq_rel
+; OUTLINE-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
+; OUTLINE-NEXT: ret
+;
+; LSE-LABEL: val_compare_and_swap_seqcst:
+; LSE: // %bb.0:
+; LSE-NEXT: // kill: def $x5 killed $x5 killed $x4_x5 def $x4_x5
+; LSE-NEXT: // kill: def $x3 killed $x3 killed $x2_x3 def $x2_x3
+; LSE-NEXT: // kill: def $x4 killed $x4 killed $x4_x5 def $x4_x5
+; LSE-NEXT: // kill: def $x2 killed $x2 killed $x2_x3 def $x2_x3
+; LSE-NEXT: caspal x2, x3, x4, x5, [x0]
+; LSE-NEXT: mov x0, x2
+; LSE-NEXT: mov x1, x3
+; LSE-NEXT: ret
+ %pair = cmpxchg i128* %p, i128 %oldval, i128 %newval seq_cst seq_cst
+ %val = extractvalue { i128, i1 } %pair, 0
+ ret i128 %val
+}
+
+define i128 @val_compare_and_swap_release(i128* %p, i128 %oldval, i128 %newval) {
+; NOOUTLINE-LABEL: val_compare_and_swap_release:
+; NOOUTLINE: // %bb.0:
+; NOOUTLINE-NEXT: .LBB2_1: // =>This Inner Loop Header: Depth=1
+; NOOUTLINE-NEXT: ldxp x8, x1, [x0]
+; NOOUTLINE-NEXT: cmp x8, x2
+; NOOUTLINE-NEXT: cset w9, ne
+; NOOUTLINE-NEXT: cmp x1, x3
+; NOOUTLINE-NEXT: cinc w9, w9, ne
+; NOOUTLINE-NEXT: cbz w9, .LBB2_3
+; NOOUTLINE-NEXT: // %bb.2: // in Loop: Header=BB2_1 Depth=1
+; NOOUTLINE-NEXT: stlxp w9, x8, x1, [x0]
+; NOOUTLINE-NEXT: cbnz w9, .LBB2_1
+; NOOUTLINE-NEXT: b .LBB2_4
+; NOOUTLINE-NEXT: .LBB2_3: // in Loop: Header=BB2_1 Depth=1
+; NOOUTLINE-NEXT: stlxp w9, x4, x5, [x0]
+; NOOUTLINE-NEXT: cbnz w9, .LBB2_1
+; NOOUTLINE-NEXT: .LBB2_4:
+; NOOUTLINE-NEXT: mov x0, x8
+; NOOUTLINE-NEXT: ret
+;
+; OUTLINE-LABEL: val_compare_and_swap_release:
+; OUTLINE: // %bb.0:
+; OUTLINE-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
+; OUTLINE-NEXT: .cfi_def_cfa_offset 16
+; OUTLINE-NEXT: .cfi_offset w30, -16
+; OUTLINE-NEXT: mov x1, x3
+; OUTLINE-NEXT: mov x8, x0
+; OUTLINE-NEXT: mov x0, x2
+; OUTLINE-NEXT: mov x2, x4
+; OUTLINE-NEXT: mov x3, x5
+; OUTLINE-NEXT: mov x4, x8
+; OUTLINE-NEXT: bl __aarch64_cas16_rel
+; OUTLINE-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
+; OUTLINE-NEXT: ret
+;
+; LSE-LABEL: val_compare_and_swap_release:
+; LSE: // %bb.0:
+; LSE-NEXT: // kill: def $x5 killed $x5 killed $x4_x5 def $x4_x5
+; LSE-NEXT: // kill: def $x3 killed $x3 killed $x2_x3 def $x2_x3
+; LSE-NEXT: // kill: def $x4 killed $x4 killed $x4_x5 def $x4_x5
+; LSE-NEXT: // kill: def $x2 killed $x2 killed $x2_x3 def $x2_x3
+; LSE-NEXT: caspl x2, x3, x4, x5, [x0]
+; LSE-NEXT: mov x0, x2
+; LSE-NEXT: mov x1, x3
+; LSE-NEXT: ret
+ %pair = cmpxchg i128* %p, i128 %oldval, i128 %newval release monotonic
+ %val = extractvalue { i128, i1 } %pair, 0
+ ret i128 %val
+}
+
+define i128 @val_compare_and_swap_monotonic(i128* %p, i128 %oldval, i128 %newval) {
+; NOOUTLINE-LABEL: val_compare_and_swap_monotonic:
+; NOOUTLINE: // %bb.0:
+; NOOUTLINE-NEXT: .LBB3_1: // =>This Inner Loop Header: Depth=1
+; NOOUTLINE-NEXT: ldxp x8, x1, [x0]
+; NOOUTLINE-NEXT: cmp x8, x2
+; NOOUTLINE-NEXT: cset w9, ne
+; NOOUTLINE-NEXT: cmp x1, x3
+; NOOUTLINE-NEXT: cinc w9, w9, ne
+; NOOUTLINE-NEXT: cbz w9, .LBB3_3
+; NOOUTLINE-NEXT: // %bb.2: // in Loop: Header=BB3_1 Depth=1
+; NOOUTLINE-NEXT: stxp w9, x8, x1, [x0]
+; NOOUTLINE-NEXT: cbnz w9, .LBB3_1
+; NOOUTLINE-NEXT: b .LBB3_4
+; NOOUTLINE-NEXT: .LBB3_3: // in Loop: Header=BB3_1 Depth=1
+; NOOUTLINE-NEXT: stxp w9, x4, x5, [x0]
+; NOOUTLINE-NEXT: cbnz w9, .LBB3_1
+; NOOUTLINE-NEXT: .LBB3_4:
+; NOOUTLINE-NEXT: mov x0, x8
+; NOOUTLINE-NEXT: ret
+;
+; OUTLINE-LABEL: val_compare_and_swap_monotonic:
+; OUTLINE: // %bb.0:
+; OUTLINE-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
+; OUTLINE-NEXT: .cfi_def_cfa_offset 16
+; OUTLINE-NEXT: .cfi_offset w30, -16
+; OUTLINE-NEXT: mov x1, x3
+; OUTLINE-NEXT: mov x8, x0
+; OUTLINE-NEXT: mov x0, x2
+; OUTLINE-NEXT: mov x2, x4
+; OUTLINE-NEXT: mov x3, x5
+; OUTLINE-NEXT: mov x4, x8
+; OUTLINE-NEXT: bl __aarch64_cas16_relax
+; OUTLINE-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
+; OUTLINE-NEXT: ret
+;
+; LSE-LABEL: val_compare_and_swap_monotonic:
+; LSE: // %bb.0:
+; LSE-NEXT: // kill: def $x5 killed $x5 killed $x4_x5 def $x4_x5
+; LSE-NEXT: // kill: def $x3 killed $x3 killed $x2_x3 def $x2_x3
+; LSE-NEXT: // kill: def $x4 killed $x4 killed $x4_x5 def $x4_x5
+; LSE-NEXT: // kill: def $x2 killed $x2 killed $x2_x3 def $x2_x3
+; LSE-NEXT: casp x2, x3, x4, x5, [x0]
+; LSE-NEXT: mov x0, x2
+; LSE-NEXT: mov x1, x3
+; LSE-NEXT: ret
+ %pair = cmpxchg i128* %p, i128 %oldval, i128 %newval monotonic monotonic
+ %val = extractvalue { i128, i1 } %pair, 0
+ ret i128 %val
+}
+
define void @fetch_and_nand(i128* %p, i128 %bits) {
; CHECK-LABEL: fetch_and_nand:
-; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]:
-; CHECK: ldxp [[DEST_REGLO:x[0-9]+]], [[DEST_REGHI:x[0-9]+]], [x0]
-; CHECK-DAG: and [[TMP_REGLO:x[0-9]+]], [[DEST_REGLO]], x2
-; CHECK-DAG: and [[TMP_REGHI:x[0-9]+]], [[DEST_REGHI]], x3
-; CHECK-DAG: mvn [[SCRATCH_REGLO:x[0-9]+]], [[TMP_REGLO]]
-; CHECK-DAG: mvn [[SCRATCH_REGHI:x[0-9]+]], [[TMP_REGHI]]
-; CHECK: stlxp [[SCRATCH_RES:w[0-9]+]], [[SCRATCH_REGLO]], [[SCRATCH_REGHI]], [x0]
-; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]]
-
-; CHECK-DAG: stp [[DEST_REGLO]], [[DEST_REGHI]]
+; CHECK: // %bb.0:
+; CHECK-NEXT: .LBB4_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldxp x9, x8, [x0]
+; CHECK-NEXT: and x10, x9, x2
+; CHECK-NEXT: and x11, x8, x3
+; CHECK-NEXT: mvn x11, x11
+; CHECK-NEXT: mvn x10, x10
+; CHECK-NEXT: stlxp w12, x10, x11, [x0]
+; CHECK-NEXT: cbnz w12, .LBB4_1
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: adrp x10, :got:var
+; CHECK-NEXT: ldr x10, [x10, :got_lo12:var]
+; CHECK-NEXT: stp x9, x8, [x10]
+; CHECK-NEXT: ret
+
%val = atomicrmw nand i128* %p, i128 %bits release
store i128 %val, i128* @var, align 16
ret void
@@ -44,14 +234,20 @@ define void @fetch_and_nand(i128* %p, i128 %bits) {
define void @fetch_and_or(i128* %p, i128 %bits) {
; CHECK-LABEL: fetch_and_or:
-; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]:
-; CHECK: ldaxp [[DEST_REGLO:x[0-9]+]], [[DEST_REGHI:x[0-9]+]], [x0]
-; CHECK-DAG: orr [[SCRATCH_REGLO:x[0-9]+]], [[DEST_REGLO]], x2
-; CHECK-DAG: orr [[SCRATCH_REGHI:x[0-9]+]], [[DEST_REGHI]], x3
-; CHECK: stlxp [[SCRATCH_RES:w[0-9]+]], [[SCRATCH_REGLO]], [[SCRATCH_REGHI]], [x0]
-; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]]
-
-; CHECK-DAG: stp [[DEST_REGLO]], [[DEST_REGHI]]
+; CHECK: // %bb.0:
+; CHECK-NEXT: .LBB5_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldaxp x9, x8, [x0]
+; CHECK-NEXT: orr x10, x8, x3
+; CHECK-NEXT: orr x11, x9, x2
+; CHECK-NEXT: stlxp w12, x11, x10, [x0]
+; CHECK-NEXT: cbnz w12, .LBB5_1
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: adrp x10, :got:var
+; CHECK-NEXT: ldr x10, [x10, :got_lo12:var]
+; CHECK-NEXT: stp x9, x8, [x10]
+; CHECK-NEXT: ret
+
%val = atomicrmw or i128* %p, i128 %bits seq_cst
store i128 %val, i128* @var, align 16
ret void
@@ -59,14 +255,20 @@ define void @fetch_and_or(i128* %p, i128 %bits) {
define void @fetch_and_add(i128* %p, i128 %bits) {
; CHECK-LABEL: fetch_and_add:
-; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]:
-; CHECK: ldaxp [[DEST_REGLO:x[0-9]+]], [[DEST_REGHI:x[0-9]+]], [x0]
-; CHECK: adds [[SCRATCH_REGLO:x[0-9]+]], [[DEST_REGLO]], x2
-; CHECK: adcs [[SCRATCH_REGHI:x[0-9]+]], [[DEST_REGHI]], x3
-; CHECK: stlxp [[SCRATCH_RES:w[0-9]+]], [[SCRATCH_REGLO]], [[SCRATCH_REGHI]], [x0]
-; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]]
-
-; CHECK-DAG: stp [[DEST_REGLO]], [[DEST_REGHI]]
+; CHECK: // %bb.0:
+; CHECK-NEXT: .LBB6_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldaxp x9, x8, [x0]
+; CHECK-NEXT: adds x10, x9, x2
+; CHECK-NEXT: adcs x11, x8, x3
+; CHECK-NEXT: stlxp w12, x10, x11, [x0]
+; CHECK-NEXT: cbnz w12, .LBB6_1
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: adrp x10, :got:var
+; CHECK-NEXT: ldr x10, [x10, :got_lo12:var]
+; CHECK-NEXT: stp x9, x8, [x10]
+; CHECK-NEXT: ret
+
%val = atomicrmw add i128* %p, i128 %bits seq_cst
store i128 %val, i128* @var, align 16
ret void
@@ -74,14 +276,20 @@ define void @fetch_and_add(i128* %p, i128 %bits) {
define void @fetch_and_sub(i128* %p, i128 %bits) {
; CHECK-LABEL: fetch_and_sub:
-; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]:
-; CHECK: ldaxp [[DEST_REGLO:x[0-9]+]], [[DEST_REGHI:x[0-9]+]], [x0]
-; CHECK: subs [[SCRATCH_REGLO:x[0-9]+]], [[DEST_REGLO]], x2
-; CHECK: sbcs [[SCRATCH_REGHI:x[0-9]+]], [[DEST_REGHI]], x3
-; CHECK: stlxp [[SCRATCH_RES:w[0-9]+]], [[SCRATCH_REGLO]], [[SCRATCH_REGHI]], [x0]
-; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]]
-
-; CHECK-DAG: stp [[DEST_REGLO]], [[DEST_REGHI]]
+; CHECK: // %bb.0:
+; CHECK-NEXT: .LBB7_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldaxp x9, x8, [x0]
+; CHECK-NEXT: subs x10, x9, x2
+; CHECK-NEXT: sbcs x11, x8, x3
+; CHECK-NEXT: stlxp w12, x10, x11, [x0]
+; CHECK-NEXT: cbnz w12, .LBB7_1
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: adrp x10, :got:var
+; CHECK-NEXT: ldr x10, [x10, :got_lo12:var]
+; CHECK-NEXT: stp x9, x8, [x10]
+; CHECK-NEXT: ret
+
%val = atomicrmw sub i128* %p, i128 %bits seq_cst
store i128 %val, i128* @var, align 16
ret void
@@ -89,20 +297,26 @@ define void @fetch_and_sub(i128* %p, i128 %bits) {
define void @fetch_and_min(i128* %p, i128 %bits) {
; CHECK-LABEL: fetch_and_min:
-; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]:
-; CHECK: ldaxp [[DEST_REGLO:x[0-9]+]], [[DEST_REGHI:x[0-9]+]], [x0]
-; CHECK: cmp [[DEST_REGLO]], x2
-; CHECK: cset [[LOCMP:w[0-9]+]], ls
-; CHECK: cmp [[DEST_REGHI:x[0-9]+]], x3
-; CHECK: cset [[HICMP:w[0-9]+]], le
-; CHECK: csel [[CMP:w[0-9]+]], [[LOCMP]], [[HICMP]], eq
-; CHECK: cmp [[CMP]], #0
-; CHECK-DAG: csel [[SCRATCH_REGHI:x[0-9]+]], [[DEST_REGHI]], x3, ne
-; CHECK-DAG: csel [[SCRATCH_REGLO:x[0-9]+]], [[DEST_REGLO]], x2, ne
-; CHECK: stlxp [[SCRATCH_RES:w[0-9]+]], [[SCRATCH_REGLO]], [[SCRATCH_REGHI]], [x0]
-; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]]
-
-; CHECK-DAG: stp [[DEST_REGLO]], [[DEST_REGHI]]
+; CHECK: // %bb.0:
+; CHECK-NEXT: .LBB8_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldaxp x9, x8, [x0]
+; CHECK-NEXT: cmp x9, x2
+; CHECK-NEXT: cset w10, ls
+; CHECK-NEXT: cmp x8, x3
+; CHECK-NEXT: cset w11, le
+; CHECK-NEXT: csel w10, w10, w11, eq
+; CHECK-NEXT: cmp w10, #0 // =0
+; CHECK-NEXT: csel x10, x8, x3, ne
+; CHECK-NEXT: csel x11, x9, x2, ne
+; CHECK-NEXT: stlxp w12, x11, x10, [x0]
+; CHECK-NEXT: cbnz w12, .LBB8_1
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: adrp x10, :got:var
+; CHECK-NEXT: ldr x10, [x10, :got_lo12:var]
+; CHECK-NEXT: stp x9, x8, [x10]
+; CHECK-NEXT: ret
+
%val = atomicrmw min i128* %p, i128 %bits seq_cst
store i128 %val, i128* @var, align 16
ret void
@@ -110,20 +324,26 @@ define void @fetch_and_min(i128* %p, i128 %bits) {
define void @fetch_and_max(i128* %p, i128 %bits) {
; CHECK-LABEL: fetch_and_max:
-; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]:
-; CHECK: ldaxp [[DEST_REGLO:x[0-9]+]], [[DEST_REGHI:x[0-9]+]], [x0]
-; CHECK: cmp [[DEST_REGLO]], x2
-; CHECK: cset [[LOCMP:w[0-9]+]], hi
-; CHECK: cmp [[DEST_REGHI:x[0-9]+]], x3
-; CHECK: cset [[HICMP:w[0-9]+]], gt
-; CHECK: csel [[CMP:w[0-9]+]], [[LOCMP]], [[HICMP]], eq
-; CHECK: cmp [[CMP]], #0
-; CHECK-DAG: csel [[SCRATCH_REGHI:x[0-9]+]], [[DEST_REGHI]], x3, ne
-; CHECK-DAG: csel [[SCRATCH_REGLO:x[0-9]+]], [[DEST_REGLO]], x2, ne
-; CHECK: stlxp [[SCRATCH_RES:w[0-9]+]], [[SCRATCH_REGLO]], [[SCRATCH_REGHI]], [x0]
-; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]]
-
-; CHECK-DAG: stp [[DEST_REGLO]], [[DEST_REGHI]]
+; CHECK: // %bb.0:
+; CHECK-NEXT: .LBB9_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldaxp x9, x8, [x0]
+; CHECK-NEXT: cmp x9, x2
+; CHECK-NEXT: cset w10, hi
+; CHECK-NEXT: cmp x8, x3
+; CHECK-NEXT: cset w11, gt
+; CHECK-NEXT: csel w10, w10, w11, eq
+; CHECK-NEXT: cmp w10, #0 // =0
+; CHECK-NEXT: csel x10, x8, x3, ne
+; CHECK-NEXT: csel x11, x9, x2, ne
+; CHECK-NEXT: stlxp w12, x11, x10, [x0]
+; CHECK-NEXT: cbnz w12, .LBB9_1
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: adrp x10, :got:var
+; CHECK-NEXT: ldr x10, [x10, :got_lo12:var]
+; CHECK-NEXT: stp x9, x8, [x10]
+; CHECK-NEXT: ret
+
%val = atomicrmw max i128* %p, i128 %bits seq_cst
store i128 %val, i128* @var, align 16
ret void
@@ -131,20 +351,26 @@ define void @fetch_and_max(i128* %p, i128 %bits) {
define void @fetch_and_umin(i128* %p, i128 %bits) {
; CHECK-LABEL: fetch_and_umin:
-; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]:
-; CHECK: ldaxp [[DEST_REGLO:x[0-9]+]], [[DEST_REGHI:x[0-9]+]], [x0]
-; CHECK: cmp [[DEST_REGLO]], x2
-; CHECK: cset [[LOCMP:w[0-9]+]], ls
-; CHECK: cmp [[DEST_REGHI:x[0-9]+]], x3
-; CHECK: cset [[HICMP:w[0-9]+]], ls
-; CHECK: csel [[CMP:w[0-9]+]], [[LOCMP]], [[HICMP]], eq
-; CHECK: cmp [[CMP]], #0
-; CHECK-DAG: csel [[SCRATCH_REGHI:x[0-9]+]], [[DEST_REGHI]], x3, ne
-; CHECK-DAG: csel [[SCRATCH_REGLO:x[0-9]+]], [[DEST_REGLO]], x2, ne
-; CHECK: stlxp [[SCRATCH_RES:w[0-9]+]], [[SCRATCH_REGLO]], [[SCRATCH_REGHI]], [x0]
-; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]]
-
-; CHECK-DAG: stp [[DEST_REGLO]], [[DEST_REGHI]]
+; CHECK: // %bb.0:
+; CHECK-NEXT: .LBB10_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldaxp x9, x8, [x0]
+; CHECK-NEXT: cmp x9, x2
+; CHECK-NEXT: cset w10, ls
+; CHECK-NEXT: cmp x8, x3
+; CHECK-NEXT: cset w11, ls
+; CHECK-NEXT: csel w10, w10, w11, eq
+; CHECK-NEXT: cmp w10, #0 // =0
+; CHECK-NEXT: csel x10, x8, x3, ne
+; CHECK-NEXT: csel x11, x9, x2, ne
+; CHECK-NEXT: stlxp w12, x11, x10, [x0]
+; CHECK-NEXT: cbnz w12, .LBB10_1
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: adrp x10, :got:var
+; CHECK-NEXT: ldr x10, [x10, :got_lo12:var]
+; CHECK-NEXT: stp x9, x8, [x10]
+; CHECK-NEXT: ret
+
%val = atomicrmw umin i128* %p, i128 %bits seq_cst
store i128 %val, i128* @var, align 16
ret void
@@ -152,20 +378,26 @@ define void @fetch_and_umin(i128* %p, i128 %bits) {
define void @fetch_and_umax(i128* %p, i128 %bits) {
; CHECK-LABEL: fetch_and_umax:
-; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]:
-; CHECK: ldaxp [[DEST_REGLO:x[0-9]+]], [[DEST_REGHI:x[0-9]+]], [x0]
-; CHECK: cmp [[DEST_REGLO]], x2
-; CHECK: cset [[LOCMP:w[0-9]+]], hi
-; CHECK: cmp [[DEST_REGHI:x[0-9]+]], x3
-; CHECK: cset [[HICMP:w[0-9]+]], hi
-; CHECK: csel [[CMP:w[0-9]+]], [[LOCMP]], [[HICMP]], eq
-; CHECK: cmp [[CMP]], #0
-; CHECK-DAG: csel [[SCRATCH_REGHI:x[0-9]+]], [[DEST_REGHI]], x3, ne
-; CHECK-DAG: csel [[SCRATCH_REGLO:x[0-9]+]], [[DEST_REGLO]], x2, ne
-; CHECK: stlxp [[SCRATCH_RES:w[0-9]+]], [[SCRATCH_REGLO]], [[SCRATCH_REGHI]], [x0]
-; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]]
-
-; CHECK-DAG: stp [[DEST_REGLO]], [[DEST_REGHI]]
+; CHECK: // %bb.0:
+; CHECK-NEXT: .LBB11_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldaxp x9, x8, [x0]
+; CHECK-NEXT: cmp x9, x2
+; CHECK-NEXT: cset w10, hi
+; CHECK-NEXT: cmp x8, x3
+; CHECK-NEXT: cset w11, hi
+; CHECK-NEXT: csel w10, w10, w11, eq
+; CHECK-NEXT: cmp w10, #0 // =0
+; CHECK-NEXT: csel x10, x8, x3, ne
+; CHECK-NEXT: csel x11, x9, x2, ne
+; CHECK-NEXT: stlxp w12, x11, x10, [x0]
+; CHECK-NEXT: cbnz w12, .LBB11_1
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: adrp x10, :got:var
+; CHECK-NEXT: ldr x10, [x10, :got_lo12:var]
+; CHECK-NEXT: stp x9, x8, [x10]
+; CHECK-NEXT: ret
+
%val = atomicrmw umax i128* %p, i128 %bits seq_cst
store i128 %val, i128* @var, align 16
ret void
@@ -173,21 +405,29 @@ define void @fetch_and_umax(i128* %p, i128 %bits) {
define i128 @atomic_load_seq_cst(i128* %p) {
; CHECK-LABEL: atomic_load_seq_cst:
-; CHECK-NOT: dmb
-; CHECK-LABEL: ldaxp
-; CHECK-NOT: dmb
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov x8, x0
+; CHECK-NEXT: .LBB12_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldaxp x0, x1, [x8]
+; CHECK-NEXT: stlxp w9, x0, x1, [x8]
+; CHECK-NEXT: cbnz w9, .LBB12_1
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: ret
%r = load atomic i128, i128* %p seq_cst, align 16
ret i128 %r
}
define i128 @atomic_load_relaxed(i64, i64, i128* %p) {
; CHECK-LABEL: atomic_load_relaxed:
-; CHECK-NOT: dmb
-; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]:
-; CHECK: ldxp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x2]
-; CHECK-NEXT: stxp [[SUCCESS:w[0-9]+]], [[LO]], [[HI]], [x2]
-; CHECK: cbnz [[SUCCESS]], [[LABEL]]
-; CHECK-NOT: dmb
+; CHECK: // %bb.0:
+; CHECK-NEXT: .LBB13_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldxp x0, x1, [x2]
+; CHECK-NEXT: stxp w8, x0, x1, [x2]
+; CHECK-NEXT: cbnz w8, .LBB13_1
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: ret
%r = load atomic i128, i128* %p monotonic, align 16
ret i128 %r
}
@@ -195,36 +435,42 @@ define i128 @atomic_load_relaxed(i64, i64, i128* %p) {
define void @atomic_store_seq_cst(i128 %in, i128* %p) {
; CHECK-LABEL: atomic_store_seq_cst:
-; CHECK-NOT: dmb
-; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]:
-; CHECK: ldaxp xzr, [[IGNORED:x[0-9]+]], [x2]
-; CHECK: stlxp [[SUCCESS:w[0-9]+]], x0, x1, [x2]
-; CHECK: cbnz [[SUCCESS]], [[LABEL]]
-; CHECK-NOT: dmb
+; CHECK: // %bb.0:
+; CHECK-NEXT: .LBB14_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldaxp xzr, x8, [x2]
+; CHECK-NEXT: stlxp w8, x0, x1, [x2]
+; CHECK-NEXT: cbnz w8, .LBB14_1
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: ret
store atomic i128 %in, i128* %p seq_cst, align 16
ret void
}
define void @atomic_store_release(i128 %in, i128* %p) {
; CHECK-LABEL: atomic_store_release:
-; CHECK-NOT: dmb
-; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]:
-; CHECK: ldxp xzr, [[IGNORED:x[0-9]+]], [x2]
-; CHECK: stlxp [[SUCCESS:w[0-9]+]], x0, x1, [x2]
-; CHECK: cbnz [[SUCCESS]], [[LABEL]]
-; CHECK-NOT: dmb
+; CHECK: // %bb.0:
+; CHECK-NEXT: .LBB15_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldxp xzr, x8, [x2]
+; CHECK-NEXT: stlxp w8, x0, x1, [x2]
+; CHECK-NEXT: cbnz w8, .LBB15_1
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: ret
store atomic i128 %in, i128* %p release, align 16
ret void
}
define void @atomic_store_relaxed(i128 %in, i128* %p) {
; CHECK-LABEL: atomic_store_relaxed:
-; CHECK-NOT: dmb
-; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]:
-; CHECK: ldxp xzr, [[IGNORED:x[0-9]+]], [x2]
-; CHECK: stxp [[SUCCESS:w[0-9]+]], x0, x1, [x2]
-; CHECK: cbnz [[SUCCESS]], [[LABEL]]
-; CHECK-NOT: dmb
+; CHECK: // %bb.0:
+; CHECK-NEXT: .LBB16_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldxp xzr, x8, [x2]
+; CHECK-NEXT: stxp w8, x0, x1, [x2]
+; CHECK-NEXT: cbnz w8, .LBB16_1
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: ret
store atomic i128 %in, i128* %p unordered, align 16
ret void
}
More information about the llvm-commits
mailing list