[llvm] [RISCV][GlobalISel] Legalize G_ATOMICRMW_XCHG (PR #209983)
Kane Wang via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 15 23:46:15 PDT 2026
https://github.com/ReVe1uv created https://github.com/llvm/llvm-project/pull/209983
This commit makes `xchg` consistent with `add/and/or/xor` across all type/extension combinations. Updates `legalizer-info-validation.mir` for the newly defined rules and adds `atomicrmw-xchg.ll` and `legalize-atomicrmw-xchg-rv{32,64}.mir` tests.
>From 2258f44cf22a43ddf82d4d99afd0195cdb4a6350 Mon Sep 17 00:00:00 2001
From: Kane Wang <wangqiang1 at kylinos.cn>
Date: Thu, 16 Jul 2026 14:04:50 +0800
Subject: [PATCH] [RISCV][GlobalISel] Legalize G_ATOMICRMW_XCHG
This commit makes xchg consistent with add/and/or/xor across all type/extension
combinations. Updates legalizer-info-validation.mir for the newly defined
rules and adds atomicrmw-xchg.ll and legalize-atomicrmw-xchg-rv{32,64}.mir tests.
---
.../Target/RISCV/GISel/RISCVLegalizerInfo.cpp | 5 +-
.../RISCV/GlobalISel/atomicrmw-xchg.ll | 1267 +++++++++++++++++
.../GlobalISel/legalizer-info-validation.mir | 5 +-
.../legalize-atomicrmw-xchg-rv32.mir | 94 ++
.../legalize-atomicrmw-xchg-rv64.mir | 125 ++
5 files changed, 1492 insertions(+), 4 deletions(-)
create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/atomicrmw-xchg.ll
create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-atomicrmw-xchg-rv32.mir
create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-atomicrmw-xchg-rv64.mir
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
index 6acb66c449dd8..8f1ac4509425d 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
@@ -737,8 +737,8 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST)
.lowerIf(all(typeInSet(0, {s8, s16, s32, s64}), typeIs(2, p0)));
getActionDefinitionsBuilder({G_ATOMIC_CMPXCHG, G_ATOMICRMW_ADD,
- G_ATOMICRMW_AND, G_ATOMICRMW_OR,
- G_ATOMICRMW_XOR})
+ G_ATOMICRMW_XCHG, G_ATOMICRMW_AND,
+ G_ATOMICRMW_OR, G_ATOMICRMW_XOR})
.legalFor(ST.hasStdExtA(), {{sXLen, p0}})
.libcallFor(!ST.hasStdExtA(), {{s8, p0}, {s16, p0}, {s32, p0}, {s64, p0}})
.clampScalar(0, sXLen, sXLen);
@@ -832,6 +832,7 @@ bool RISCVLegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
case Intrinsic::riscv_vsetvlimax:
case Intrinsic::riscv_masked_atomicrmw_add:
case Intrinsic::riscv_masked_atomicrmw_sub:
+ case Intrinsic::riscv_masked_atomicrmw_xchg:
case Intrinsic::riscv_masked_cmpxchg:
return true;
}
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/atomicrmw-xchg.ll b/llvm/test/CodeGen/RISCV/GlobalISel/atomicrmw-xchg.ll
new file mode 100644
index 0000000000000..909146854b33e
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/atomicrmw-xchg.ll
@@ -0,0 +1,1267 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -mtriple=riscv32 -mattr=+a,+zabha < %s | FileCheck %s --check-prefixes=RV32IA-ZABHA
+; RUN: llc -global-isel -mtriple=riscv32 -mattr=+a < %s | FileCheck %s --check-prefixes=RV32IA
+; RUN: llc -global-isel -mtriple=riscv32 < %s | FileCheck %s --check-prefixes=RV32I
+; RUN: llc -global-isel -mtriple=riscv64 -mattr=+a,+zabha < %s | FileCheck %s --check-prefixes=RV64IA-ZABHA
+; RUN: llc -global-isel -mtriple=riscv64 -mattr=+a < %s | FileCheck %s --check-prefixes=RV64IA
+; RUN: llc -global-isel -mtriple=riscv64 < %s | FileCheck %s --check-prefixes=RV64I
+
+define i8 @atomicrmw_xchg_i8_monotonic(ptr %ptr, i8 %val) nounwind {
+; RV32IA-ZABHA-LABEL: atomicrmw_xchg_i8_monotonic:
+; RV32IA-ZABHA: # %bb.0:
+; RV32IA-ZABHA-NEXT: amoswap.b a0, a1, (a0)
+; RV32IA-ZABHA-NEXT: ret
+;
+; RV32IA-LABEL: atomicrmw_xchg_i8_monotonic:
+; RV32IA: # %bb.0:
+; RV32IA-NEXT: li a2, 255
+; RV32IA-NEXT: andi a3, a0, -4
+; RV32IA-NEXT: andi a0, a0, 3
+; RV32IA-NEXT: slli a0, a0, 3
+; RV32IA-NEXT: zext.b a1, a1
+; RV32IA-NEXT: sll a2, a2, a0
+; RV32IA-NEXT: sll a1, a1, a0
+; RV32IA-NEXT: .LBB0_1: # =>This Inner Loop Header: Depth=1
+; RV32IA-NEXT: lr.w a4, (a3)
+; RV32IA-NEXT: mv a5, a1
+; RV32IA-NEXT: xor a5, a4, a5
+; RV32IA-NEXT: and a5, a5, a2
+; RV32IA-NEXT: xor a5, a4, a5
+; RV32IA-NEXT: sc.w a5, a5, (a3)
+; RV32IA-NEXT: bnez a5, .LBB0_1
+; RV32IA-NEXT: # %bb.2:
+; RV32IA-NEXT: srl a0, a4, a0
+; RV32IA-NEXT: ret
+;
+; RV32I-LABEL: atomicrmw_xchg_i8_monotonic:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -16
+; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: li a2, 0
+; RV32I-NEXT: call __atomic_exchange_1
+; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 16
+; RV32I-NEXT: ret
+;
+; RV64IA-ZABHA-LABEL: atomicrmw_xchg_i8_monotonic:
+; RV64IA-ZABHA: # %bb.0:
+; RV64IA-ZABHA-NEXT: amoswap.b a0, a1, (a0)
+; RV64IA-ZABHA-NEXT: ret
+;
+; RV64IA-LABEL: atomicrmw_xchg_i8_monotonic:
+; RV64IA: # %bb.0:
+; RV64IA-NEXT: li a2, 255
+; RV64IA-NEXT: andi a3, a0, -4
+; RV64IA-NEXT: andi a0, a0, 3
+; RV64IA-NEXT: slli a0, a0, 3
+; RV64IA-NEXT: zext.b a1, a1
+; RV64IA-NEXT: sllw a2, a2, a0
+; RV64IA-NEXT: sllw a1, a1, a0
+; RV64IA-NEXT: .LBB0_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT: lr.w a4, (a3)
+; RV64IA-NEXT: mv a5, a1
+; RV64IA-NEXT: xor a5, a4, a5
+; RV64IA-NEXT: and a5, a5, a2
+; RV64IA-NEXT: xor a5, a4, a5
+; RV64IA-NEXT: sc.w a5, a5, (a3)
+; RV64IA-NEXT: bnez a5, .LBB0_1
+; RV64IA-NEXT: # %bb.2:
+; RV64IA-NEXT: srlw a0, a4, a0
+; RV64IA-NEXT: ret
+;
+; RV64I-LABEL: atomicrmw_xchg_i8_monotonic:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -16
+; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: li a2, 0
+; RV64I-NEXT: call __atomic_exchange_1
+; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 16
+; RV64I-NEXT: ret
+ %res = atomicrmw xchg ptr %ptr, i8 %val monotonic
+ ret i8 %res
+}
+
+define i16 @atomicrmw_xchg_i16_monotonic(ptr %ptr, i16 %val) nounwind {
+; RV32IA-ZABHA-LABEL: atomicrmw_xchg_i16_monotonic:
+; RV32IA-ZABHA: # %bb.0:
+; RV32IA-ZABHA-NEXT: amoswap.h a0, a1, (a0)
+; RV32IA-ZABHA-NEXT: ret
+;
+; RV32IA-LABEL: atomicrmw_xchg_i16_monotonic:
+; RV32IA: # %bb.0:
+; RV32IA-NEXT: lui a2, 16
+; RV32IA-NEXT: addi a2, a2, -1
+; RV32IA-NEXT: andi a3, a0, -4
+; RV32IA-NEXT: andi a0, a0, 3
+; RV32IA-NEXT: slli a0, a0, 3
+; RV32IA-NEXT: and a1, a1, a2
+; RV32IA-NEXT: sll a2, a2, a0
+; RV32IA-NEXT: sll a1, a1, a0
+; RV32IA-NEXT: .LBB1_1: # =>This Inner Loop Header: Depth=1
+; RV32IA-NEXT: lr.w a4, (a3)
+; RV32IA-NEXT: mv a5, a1
+; RV32IA-NEXT: xor a5, a4, a5
+; RV32IA-NEXT: and a5, a5, a2
+; RV32IA-NEXT: xor a5, a4, a5
+; RV32IA-NEXT: sc.w a5, a5, (a3)
+; RV32IA-NEXT: bnez a5, .LBB1_1
+; RV32IA-NEXT: # %bb.2:
+; RV32IA-NEXT: srl a0, a4, a0
+; RV32IA-NEXT: ret
+;
+; RV32I-LABEL: atomicrmw_xchg_i16_monotonic:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -16
+; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: li a2, 0
+; RV32I-NEXT: call __atomic_exchange_2
+; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 16
+; RV32I-NEXT: ret
+;
+; RV64IA-ZABHA-LABEL: atomicrmw_xchg_i16_monotonic:
+; RV64IA-ZABHA: # %bb.0:
+; RV64IA-ZABHA-NEXT: amoswap.h a0, a1, (a0)
+; RV64IA-ZABHA-NEXT: ret
+;
+; RV64IA-LABEL: atomicrmw_xchg_i16_monotonic:
+; RV64IA: # %bb.0:
+; RV64IA-NEXT: lui a2, 16
+; RV64IA-NEXT: addi a2, a2, -1
+; RV64IA-NEXT: andi a3, a0, -4
+; RV64IA-NEXT: andi a0, a0, 3
+; RV64IA-NEXT: slli a0, a0, 3
+; RV64IA-NEXT: and a1, a1, a2
+; RV64IA-NEXT: sllw a2, a2, a0
+; RV64IA-NEXT: sllw a1, a1, a0
+; RV64IA-NEXT: .LBB1_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT: lr.w a4, (a3)
+; RV64IA-NEXT: mv a5, a1
+; RV64IA-NEXT: xor a5, a4, a5
+; RV64IA-NEXT: and a5, a5, a2
+; RV64IA-NEXT: xor a5, a4, a5
+; RV64IA-NEXT: sc.w a5, a5, (a3)
+; RV64IA-NEXT: bnez a5, .LBB1_1
+; RV64IA-NEXT: # %bb.2:
+; RV64IA-NEXT: srlw a0, a4, a0
+; RV64IA-NEXT: ret
+;
+; RV64I-LABEL: atomicrmw_xchg_i16_monotonic:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -16
+; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: li a2, 0
+; RV64I-NEXT: call __atomic_exchange_2
+; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 16
+; RV64I-NEXT: ret
+ %res = atomicrmw xchg ptr %ptr, i16 %val monotonic
+ ret i16 %res
+}
+
+define i32 @atomicrmw_xchg_i32_monotonic(ptr %ptr, i32 %val) nounwind {
+; RV32IA-ZABHA-LABEL: atomicrmw_xchg_i32_monotonic:
+; RV32IA-ZABHA: # %bb.0:
+; RV32IA-ZABHA-NEXT: amoswap.w a0, a1, (a0)
+; RV32IA-ZABHA-NEXT: ret
+;
+; RV32IA-LABEL: atomicrmw_xchg_i32_monotonic:
+; RV32IA: # %bb.0:
+; RV32IA-NEXT: amoswap.w a0, a1, (a0)
+; RV32IA-NEXT: ret
+;
+; RV32I-LABEL: atomicrmw_xchg_i32_monotonic:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -16
+; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: li a2, 0
+; RV32I-NEXT: call __atomic_exchange_4
+; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 16
+; RV32I-NEXT: ret
+;
+; RV64IA-ZABHA-LABEL: atomicrmw_xchg_i32_monotonic:
+; RV64IA-ZABHA: # %bb.0:
+; RV64IA-ZABHA-NEXT: amoswap.w a0, a1, (a0)
+; RV64IA-ZABHA-NEXT: ret
+;
+; RV64IA-LABEL: atomicrmw_xchg_i32_monotonic:
+; RV64IA: # %bb.0:
+; RV64IA-NEXT: amoswap.w a0, a1, (a0)
+; RV64IA-NEXT: ret
+;
+; RV64I-LABEL: atomicrmw_xchg_i32_monotonic:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -16
+; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: li a2, 0
+; RV64I-NEXT: call __atomic_exchange_4
+; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 16
+; RV64I-NEXT: ret
+ %res = atomicrmw xchg ptr %ptr, i32 %val monotonic
+ ret i32 %res
+}
+
+define i64 @atomicrmw_xchg_i64_monotonic(ptr %ptr, i64 %val) nounwind {
+; RV32IA-ZABHA-LABEL: atomicrmw_xchg_i64_monotonic:
+; RV32IA-ZABHA: # %bb.0:
+; RV32IA-ZABHA-NEXT: addi sp, sp, -16
+; RV32IA-ZABHA-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IA-ZABHA-NEXT: li a3, 0
+; RV32IA-ZABHA-NEXT: call __atomic_exchange_8
+; RV32IA-ZABHA-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IA-ZABHA-NEXT: addi sp, sp, 16
+; RV32IA-ZABHA-NEXT: ret
+;
+; RV32IA-LABEL: atomicrmw_xchg_i64_monotonic:
+; RV32IA: # %bb.0:
+; RV32IA-NEXT: addi sp, sp, -16
+; RV32IA-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IA-NEXT: li a3, 0
+; RV32IA-NEXT: call __atomic_exchange_8
+; RV32IA-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IA-NEXT: addi sp, sp, 16
+; RV32IA-NEXT: ret
+;
+; RV32I-LABEL: atomicrmw_xchg_i64_monotonic:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -16
+; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __atomic_exchange_8
+; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 16
+; RV32I-NEXT: ret
+;
+; RV64IA-ZABHA-LABEL: atomicrmw_xchg_i64_monotonic:
+; RV64IA-ZABHA: # %bb.0:
+; RV64IA-ZABHA-NEXT: amoswap.d a0, a1, (a0)
+; RV64IA-ZABHA-NEXT: ret
+;
+; RV64IA-LABEL: atomicrmw_xchg_i64_monotonic:
+; RV64IA: # %bb.0:
+; RV64IA-NEXT: amoswap.d a0, a1, (a0)
+; RV64IA-NEXT: ret
+;
+; RV64I-LABEL: atomicrmw_xchg_i64_monotonic:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -16
+; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: li a2, 0
+; RV64I-NEXT: call __atomic_exchange_8
+; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 16
+; RV64I-NEXT: ret
+ %res = atomicrmw xchg ptr %ptr, i64 %val monotonic
+ ret i64 %res
+}
+
+define i8 @atomicrmw_xchg_i8_acquire(ptr %ptr, i8 %val) nounwind {
+; RV32IA-ZABHA-LABEL: atomicrmw_xchg_i8_acquire:
+; RV32IA-ZABHA: # %bb.0:
+; RV32IA-ZABHA-NEXT: amoswap.b.aq a0, a1, (a0)
+; RV32IA-ZABHA-NEXT: ret
+;
+; RV32IA-LABEL: atomicrmw_xchg_i8_acquire:
+; RV32IA: # %bb.0:
+; RV32IA-NEXT: li a2, 255
+; RV32IA-NEXT: andi a3, a0, -4
+; RV32IA-NEXT: andi a0, a0, 3
+; RV32IA-NEXT: slli a0, a0, 3
+; RV32IA-NEXT: zext.b a1, a1
+; RV32IA-NEXT: sll a2, a2, a0
+; RV32IA-NEXT: sll a1, a1, a0
+; RV32IA-NEXT: .LBB4_1: # =>This Inner Loop Header: Depth=1
+; RV32IA-NEXT: lr.w.aq a4, (a3)
+; RV32IA-NEXT: mv a5, a1
+; RV32IA-NEXT: xor a5, a4, a5
+; RV32IA-NEXT: and a5, a5, a2
+; RV32IA-NEXT: xor a5, a4, a5
+; RV32IA-NEXT: sc.w a5, a5, (a3)
+; RV32IA-NEXT: bnez a5, .LBB4_1
+; RV32IA-NEXT: # %bb.2:
+; RV32IA-NEXT: srl a0, a4, a0
+; RV32IA-NEXT: ret
+;
+; RV32I-LABEL: atomicrmw_xchg_i8_acquire:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -16
+; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: li a2, 2
+; RV32I-NEXT: call __atomic_exchange_1
+; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 16
+; RV32I-NEXT: ret
+;
+; RV64IA-ZABHA-LABEL: atomicrmw_xchg_i8_acquire:
+; RV64IA-ZABHA: # %bb.0:
+; RV64IA-ZABHA-NEXT: amoswap.b.aq a0, a1, (a0)
+; RV64IA-ZABHA-NEXT: ret
+;
+; RV64IA-LABEL: atomicrmw_xchg_i8_acquire:
+; RV64IA: # %bb.0:
+; RV64IA-NEXT: li a2, 255
+; RV64IA-NEXT: andi a3, a0, -4
+; RV64IA-NEXT: andi a0, a0, 3
+; RV64IA-NEXT: slli a0, a0, 3
+; RV64IA-NEXT: zext.b a1, a1
+; RV64IA-NEXT: sllw a2, a2, a0
+; RV64IA-NEXT: sllw a1, a1, a0
+; RV64IA-NEXT: .LBB4_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT: lr.w.aq a4, (a3)
+; RV64IA-NEXT: mv a5, a1
+; RV64IA-NEXT: xor a5, a4, a5
+; RV64IA-NEXT: and a5, a5, a2
+; RV64IA-NEXT: xor a5, a4, a5
+; RV64IA-NEXT: sc.w a5, a5, (a3)
+; RV64IA-NEXT: bnez a5, .LBB4_1
+; RV64IA-NEXT: # %bb.2:
+; RV64IA-NEXT: srlw a0, a4, a0
+; RV64IA-NEXT: ret
+;
+; RV64I-LABEL: atomicrmw_xchg_i8_acquire:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -16
+; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: li a2, 2
+; RV64I-NEXT: call __atomic_exchange_1
+; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 16
+; RV64I-NEXT: ret
+ %res = atomicrmw xchg ptr %ptr, i8 %val acquire
+ ret i8 %res
+}
+
+define i16 @atomicrmw_xchg_i16_acquire(ptr %ptr, i16 %val) nounwind {
+; RV32IA-ZABHA-LABEL: atomicrmw_xchg_i16_acquire:
+; RV32IA-ZABHA: # %bb.0:
+; RV32IA-ZABHA-NEXT: amoswap.h.aq a0, a1, (a0)
+; RV32IA-ZABHA-NEXT: ret
+;
+; RV32IA-LABEL: atomicrmw_xchg_i16_acquire:
+; RV32IA: # %bb.0:
+; RV32IA-NEXT: lui a2, 16
+; RV32IA-NEXT: addi a2, a2, -1
+; RV32IA-NEXT: andi a3, a0, -4
+; RV32IA-NEXT: andi a0, a0, 3
+; RV32IA-NEXT: slli a0, a0, 3
+; RV32IA-NEXT: and a1, a1, a2
+; RV32IA-NEXT: sll a2, a2, a0
+; RV32IA-NEXT: sll a1, a1, a0
+; RV32IA-NEXT: .LBB5_1: # =>This Inner Loop Header: Depth=1
+; RV32IA-NEXT: lr.w.aq a4, (a3)
+; RV32IA-NEXT: mv a5, a1
+; RV32IA-NEXT: xor a5, a4, a5
+; RV32IA-NEXT: and a5, a5, a2
+; RV32IA-NEXT: xor a5, a4, a5
+; RV32IA-NEXT: sc.w a5, a5, (a3)
+; RV32IA-NEXT: bnez a5, .LBB5_1
+; RV32IA-NEXT: # %bb.2:
+; RV32IA-NEXT: srl a0, a4, a0
+; RV32IA-NEXT: ret
+;
+; RV32I-LABEL: atomicrmw_xchg_i16_acquire:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -16
+; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: li a2, 2
+; RV32I-NEXT: call __atomic_exchange_2
+; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 16
+; RV32I-NEXT: ret
+;
+; RV64IA-ZABHA-LABEL: atomicrmw_xchg_i16_acquire:
+; RV64IA-ZABHA: # %bb.0:
+; RV64IA-ZABHA-NEXT: amoswap.h.aq a0, a1, (a0)
+; RV64IA-ZABHA-NEXT: ret
+;
+; RV64IA-LABEL: atomicrmw_xchg_i16_acquire:
+; RV64IA: # %bb.0:
+; RV64IA-NEXT: lui a2, 16
+; RV64IA-NEXT: addi a2, a2, -1
+; RV64IA-NEXT: andi a3, a0, -4
+; RV64IA-NEXT: andi a0, a0, 3
+; RV64IA-NEXT: slli a0, a0, 3
+; RV64IA-NEXT: and a1, a1, a2
+; RV64IA-NEXT: sllw a2, a2, a0
+; RV64IA-NEXT: sllw a1, a1, a0
+; RV64IA-NEXT: .LBB5_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT: lr.w.aq a4, (a3)
+; RV64IA-NEXT: mv a5, a1
+; RV64IA-NEXT: xor a5, a4, a5
+; RV64IA-NEXT: and a5, a5, a2
+; RV64IA-NEXT: xor a5, a4, a5
+; RV64IA-NEXT: sc.w a5, a5, (a3)
+; RV64IA-NEXT: bnez a5, .LBB5_1
+; RV64IA-NEXT: # %bb.2:
+; RV64IA-NEXT: srlw a0, a4, a0
+; RV64IA-NEXT: ret
+;
+; RV64I-LABEL: atomicrmw_xchg_i16_acquire:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -16
+; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: li a2, 2
+; RV64I-NEXT: call __atomic_exchange_2
+; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 16
+; RV64I-NEXT: ret
+ %res = atomicrmw xchg ptr %ptr, i16 %val acquire
+ ret i16 %res
+}
+
+define i32 @atomicrmw_xchg_i32_acquire(ptr %ptr, i32 %val) nounwind {
+; RV32IA-ZABHA-LABEL: atomicrmw_xchg_i32_acquire:
+; RV32IA-ZABHA: # %bb.0:
+; RV32IA-ZABHA-NEXT: amoswap.w.aq a0, a1, (a0)
+; RV32IA-ZABHA-NEXT: ret
+;
+; RV32IA-LABEL: atomicrmw_xchg_i32_acquire:
+; RV32IA: # %bb.0:
+; RV32IA-NEXT: amoswap.w.aq a0, a1, (a0)
+; RV32IA-NEXT: ret
+;
+; RV32I-LABEL: atomicrmw_xchg_i32_acquire:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -16
+; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: li a2, 2
+; RV32I-NEXT: call __atomic_exchange_4
+; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 16
+; RV32I-NEXT: ret
+;
+; RV64IA-ZABHA-LABEL: atomicrmw_xchg_i32_acquire:
+; RV64IA-ZABHA: # %bb.0:
+; RV64IA-ZABHA-NEXT: amoswap.w.aq a0, a1, (a0)
+; RV64IA-ZABHA-NEXT: ret
+;
+; RV64IA-LABEL: atomicrmw_xchg_i32_acquire:
+; RV64IA: # %bb.0:
+; RV64IA-NEXT: amoswap.w.aq a0, a1, (a0)
+; RV64IA-NEXT: ret
+;
+; RV64I-LABEL: atomicrmw_xchg_i32_acquire:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -16
+; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: li a2, 2
+; RV64I-NEXT: call __atomic_exchange_4
+; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 16
+; RV64I-NEXT: ret
+ %res = atomicrmw xchg ptr %ptr, i32 %val acquire
+ ret i32 %res
+}
+
+define i64 @atomicrmw_xchg_i64_acquire(ptr %ptr, i64 %val) nounwind {
+; RV32IA-ZABHA-LABEL: atomicrmw_xchg_i64_acquire:
+; RV32IA-ZABHA: # %bb.0:
+; RV32IA-ZABHA-NEXT: addi sp, sp, -16
+; RV32IA-ZABHA-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IA-ZABHA-NEXT: li a3, 2
+; RV32IA-ZABHA-NEXT: call __atomic_exchange_8
+; RV32IA-ZABHA-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IA-ZABHA-NEXT: addi sp, sp, 16
+; RV32IA-ZABHA-NEXT: ret
+;
+; RV32IA-LABEL: atomicrmw_xchg_i64_acquire:
+; RV32IA: # %bb.0:
+; RV32IA-NEXT: addi sp, sp, -16
+; RV32IA-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IA-NEXT: li a3, 2
+; RV32IA-NEXT: call __atomic_exchange_8
+; RV32IA-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IA-NEXT: addi sp, sp, 16
+; RV32IA-NEXT: ret
+;
+; RV32I-LABEL: atomicrmw_xchg_i64_acquire:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -16
+; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: li a3, 2
+; RV32I-NEXT: call __atomic_exchange_8
+; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 16
+; RV32I-NEXT: ret
+;
+; RV64IA-ZABHA-LABEL: atomicrmw_xchg_i64_acquire:
+; RV64IA-ZABHA: # %bb.0:
+; RV64IA-ZABHA-NEXT: amoswap.d.aq a0, a1, (a0)
+; RV64IA-ZABHA-NEXT: ret
+;
+; RV64IA-LABEL: atomicrmw_xchg_i64_acquire:
+; RV64IA: # %bb.0:
+; RV64IA-NEXT: amoswap.d.aq a0, a1, (a0)
+; RV64IA-NEXT: ret
+;
+; RV64I-LABEL: atomicrmw_xchg_i64_acquire:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -16
+; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: li a2, 2
+; RV64I-NEXT: call __atomic_exchange_8
+; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 16
+; RV64I-NEXT: ret
+ %res = atomicrmw xchg ptr %ptr, i64 %val acquire
+ ret i64 %res
+}
+
+define i8 @atomicrmw_xchg_i8_release(ptr %ptr, i8 %val) nounwind {
+; RV32IA-ZABHA-LABEL: atomicrmw_xchg_i8_release:
+; RV32IA-ZABHA: # %bb.0:
+; RV32IA-ZABHA-NEXT: amoswap.b.rl a0, a1, (a0)
+; RV32IA-ZABHA-NEXT: ret
+;
+; RV32IA-LABEL: atomicrmw_xchg_i8_release:
+; RV32IA: # %bb.0:
+; RV32IA-NEXT: li a2, 255
+; RV32IA-NEXT: andi a3, a0, -4
+; RV32IA-NEXT: andi a0, a0, 3
+; RV32IA-NEXT: slli a0, a0, 3
+; RV32IA-NEXT: zext.b a1, a1
+; RV32IA-NEXT: sll a2, a2, a0
+; RV32IA-NEXT: sll a1, a1, a0
+; RV32IA-NEXT: .LBB8_1: # =>This Inner Loop Header: Depth=1
+; RV32IA-NEXT: lr.w a4, (a3)
+; RV32IA-NEXT: mv a5, a1
+; RV32IA-NEXT: xor a5, a4, a5
+; RV32IA-NEXT: and a5, a5, a2
+; RV32IA-NEXT: xor a5, a4, a5
+; RV32IA-NEXT: sc.w.rl a5, a5, (a3)
+; RV32IA-NEXT: bnez a5, .LBB8_1
+; RV32IA-NEXT: # %bb.2:
+; RV32IA-NEXT: srl a0, a4, a0
+; RV32IA-NEXT: ret
+;
+; RV32I-LABEL: atomicrmw_xchg_i8_release:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -16
+; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: li a2, 3
+; RV32I-NEXT: call __atomic_exchange_1
+; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 16
+; RV32I-NEXT: ret
+;
+; RV64IA-ZABHA-LABEL: atomicrmw_xchg_i8_release:
+; RV64IA-ZABHA: # %bb.0:
+; RV64IA-ZABHA-NEXT: amoswap.b.rl a0, a1, (a0)
+; RV64IA-ZABHA-NEXT: ret
+;
+; RV64IA-LABEL: atomicrmw_xchg_i8_release:
+; RV64IA: # %bb.0:
+; RV64IA-NEXT: li a2, 255
+; RV64IA-NEXT: andi a3, a0, -4
+; RV64IA-NEXT: andi a0, a0, 3
+; RV64IA-NEXT: slli a0, a0, 3
+; RV64IA-NEXT: zext.b a1, a1
+; RV64IA-NEXT: sllw a2, a2, a0
+; RV64IA-NEXT: sllw a1, a1, a0
+; RV64IA-NEXT: .LBB8_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT: lr.w a4, (a3)
+; RV64IA-NEXT: mv a5, a1
+; RV64IA-NEXT: xor a5, a4, a5
+; RV64IA-NEXT: and a5, a5, a2
+; RV64IA-NEXT: xor a5, a4, a5
+; RV64IA-NEXT: sc.w.rl a5, a5, (a3)
+; RV64IA-NEXT: bnez a5, .LBB8_1
+; RV64IA-NEXT: # %bb.2:
+; RV64IA-NEXT: srlw a0, a4, a0
+; RV64IA-NEXT: ret
+;
+; RV64I-LABEL: atomicrmw_xchg_i8_release:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -16
+; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: li a2, 3
+; RV64I-NEXT: call __atomic_exchange_1
+; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 16
+; RV64I-NEXT: ret
+ %res = atomicrmw xchg ptr %ptr, i8 %val release
+ ret i8 %res
+}
+
+define i16 @atomicrmw_xchg_i16_release(ptr %ptr, i16 %val) nounwind {
+; RV32IA-ZABHA-LABEL: atomicrmw_xchg_i16_release:
+; RV32IA-ZABHA: # %bb.0:
+; RV32IA-ZABHA-NEXT: amoswap.h.rl a0, a1, (a0)
+; RV32IA-ZABHA-NEXT: ret
+;
+; RV32IA-LABEL: atomicrmw_xchg_i16_release:
+; RV32IA: # %bb.0:
+; RV32IA-NEXT: lui a2, 16
+; RV32IA-NEXT: addi a2, a2, -1
+; RV32IA-NEXT: andi a3, a0, -4
+; RV32IA-NEXT: andi a0, a0, 3
+; RV32IA-NEXT: slli a0, a0, 3
+; RV32IA-NEXT: and a1, a1, a2
+; RV32IA-NEXT: sll a2, a2, a0
+; RV32IA-NEXT: sll a1, a1, a0
+; RV32IA-NEXT: .LBB9_1: # =>This Inner Loop Header: Depth=1
+; RV32IA-NEXT: lr.w a4, (a3)
+; RV32IA-NEXT: mv a5, a1
+; RV32IA-NEXT: xor a5, a4, a5
+; RV32IA-NEXT: and a5, a5, a2
+; RV32IA-NEXT: xor a5, a4, a5
+; RV32IA-NEXT: sc.w.rl a5, a5, (a3)
+; RV32IA-NEXT: bnez a5, .LBB9_1
+; RV32IA-NEXT: # %bb.2:
+; RV32IA-NEXT: srl a0, a4, a0
+; RV32IA-NEXT: ret
+;
+; RV32I-LABEL: atomicrmw_xchg_i16_release:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -16
+; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: li a2, 3
+; RV32I-NEXT: call __atomic_exchange_2
+; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 16
+; RV32I-NEXT: ret
+;
+; RV64IA-ZABHA-LABEL: atomicrmw_xchg_i16_release:
+; RV64IA-ZABHA: # %bb.0:
+; RV64IA-ZABHA-NEXT: amoswap.h.rl a0, a1, (a0)
+; RV64IA-ZABHA-NEXT: ret
+;
+; RV64IA-LABEL: atomicrmw_xchg_i16_release:
+; RV64IA: # %bb.0:
+; RV64IA-NEXT: lui a2, 16
+; RV64IA-NEXT: addi a2, a2, -1
+; RV64IA-NEXT: andi a3, a0, -4
+; RV64IA-NEXT: andi a0, a0, 3
+; RV64IA-NEXT: slli a0, a0, 3
+; RV64IA-NEXT: and a1, a1, a2
+; RV64IA-NEXT: sllw a2, a2, a0
+; RV64IA-NEXT: sllw a1, a1, a0
+; RV64IA-NEXT: .LBB9_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT: lr.w a4, (a3)
+; RV64IA-NEXT: mv a5, a1
+; RV64IA-NEXT: xor a5, a4, a5
+; RV64IA-NEXT: and a5, a5, a2
+; RV64IA-NEXT: xor a5, a4, a5
+; RV64IA-NEXT: sc.w.rl a5, a5, (a3)
+; RV64IA-NEXT: bnez a5, .LBB9_1
+; RV64IA-NEXT: # %bb.2:
+; RV64IA-NEXT: srlw a0, a4, a0
+; RV64IA-NEXT: ret
+;
+; RV64I-LABEL: atomicrmw_xchg_i16_release:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -16
+; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: li a2, 3
+; RV64I-NEXT: call __atomic_exchange_2
+; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 16
+; RV64I-NEXT: ret
+ %res = atomicrmw xchg ptr %ptr, i16 %val release
+ ret i16 %res
+}
+
+define i32 @atomicrmw_xchg_i32_release(ptr %ptr, i32 %val) nounwind {
+; RV32IA-ZABHA-LABEL: atomicrmw_xchg_i32_release:
+; RV32IA-ZABHA: # %bb.0:
+; RV32IA-ZABHA-NEXT: amoswap.w.rl a0, a1, (a0)
+; RV32IA-ZABHA-NEXT: ret
+;
+; RV32IA-LABEL: atomicrmw_xchg_i32_release:
+; RV32IA: # %bb.0:
+; RV32IA-NEXT: amoswap.w.rl a0, a1, (a0)
+; RV32IA-NEXT: ret
+;
+; RV32I-LABEL: atomicrmw_xchg_i32_release:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -16
+; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: li a2, 3
+; RV32I-NEXT: call __atomic_exchange_4
+; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 16
+; RV32I-NEXT: ret
+;
+; RV64IA-ZABHA-LABEL: atomicrmw_xchg_i32_release:
+; RV64IA-ZABHA: # %bb.0:
+; RV64IA-ZABHA-NEXT: amoswap.w.rl a0, a1, (a0)
+; RV64IA-ZABHA-NEXT: ret
+;
+; RV64IA-LABEL: atomicrmw_xchg_i32_release:
+; RV64IA: # %bb.0:
+; RV64IA-NEXT: amoswap.w.rl a0, a1, (a0)
+; RV64IA-NEXT: ret
+;
+; RV64I-LABEL: atomicrmw_xchg_i32_release:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -16
+; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: li a2, 3
+; RV64I-NEXT: call __atomic_exchange_4
+; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 16
+; RV64I-NEXT: ret
+ %res = atomicrmw xchg ptr %ptr, i32 %val release
+ ret i32 %res
+}
+
+define i64 @atomicrmw_xchg_i64_release(ptr %ptr, i64 %val) nounwind {
+; RV32IA-ZABHA-LABEL: atomicrmw_xchg_i64_release:
+; RV32IA-ZABHA: # %bb.0:
+; RV32IA-ZABHA-NEXT: addi sp, sp, -16
+; RV32IA-ZABHA-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IA-ZABHA-NEXT: li a3, 3
+; RV32IA-ZABHA-NEXT: call __atomic_exchange_8
+; RV32IA-ZABHA-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IA-ZABHA-NEXT: addi sp, sp, 16
+; RV32IA-ZABHA-NEXT: ret
+;
+; RV32IA-LABEL: atomicrmw_xchg_i64_release:
+; RV32IA: # %bb.0:
+; RV32IA-NEXT: addi sp, sp, -16
+; RV32IA-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IA-NEXT: li a3, 3
+; RV32IA-NEXT: call __atomic_exchange_8
+; RV32IA-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IA-NEXT: addi sp, sp, 16
+; RV32IA-NEXT: ret
+;
+; RV32I-LABEL: atomicrmw_xchg_i64_release:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -16
+; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: li a3, 3
+; RV32I-NEXT: call __atomic_exchange_8
+; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 16
+; RV32I-NEXT: ret
+;
+; RV64IA-ZABHA-LABEL: atomicrmw_xchg_i64_release:
+; RV64IA-ZABHA: # %bb.0:
+; RV64IA-ZABHA-NEXT: amoswap.d.rl a0, a1, (a0)
+; RV64IA-ZABHA-NEXT: ret
+;
+; RV64IA-LABEL: atomicrmw_xchg_i64_release:
+; RV64IA: # %bb.0:
+; RV64IA-NEXT: amoswap.d.rl a0, a1, (a0)
+; RV64IA-NEXT: ret
+;
+; RV64I-LABEL: atomicrmw_xchg_i64_release:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -16
+; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: li a2, 3
+; RV64I-NEXT: call __atomic_exchange_8
+; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 16
+; RV64I-NEXT: ret
+ %res = atomicrmw xchg ptr %ptr, i64 %val release
+ ret i64 %res
+}
+
+define i8 @atomicrmw_xchg_i8_acq_rel(ptr %ptr, i8 %val) nounwind {
+; RV32IA-ZABHA-LABEL: atomicrmw_xchg_i8_acq_rel:
+; RV32IA-ZABHA: # %bb.0:
+; RV32IA-ZABHA-NEXT: amoswap.b.aqrl a0, a1, (a0)
+; RV32IA-ZABHA-NEXT: ret
+;
+; RV32IA-LABEL: atomicrmw_xchg_i8_acq_rel:
+; RV32IA: # %bb.0:
+; RV32IA-NEXT: li a2, 255
+; RV32IA-NEXT: andi a3, a0, -4
+; RV32IA-NEXT: andi a0, a0, 3
+; RV32IA-NEXT: slli a0, a0, 3
+; RV32IA-NEXT: zext.b a1, a1
+; RV32IA-NEXT: sll a2, a2, a0
+; RV32IA-NEXT: sll a1, a1, a0
+; RV32IA-NEXT: .LBB12_1: # =>This Inner Loop Header: Depth=1
+; RV32IA-NEXT: lr.w.aq a4, (a3)
+; RV32IA-NEXT: mv a5, a1
+; RV32IA-NEXT: xor a5, a4, a5
+; RV32IA-NEXT: and a5, a5, a2
+; RV32IA-NEXT: xor a5, a4, a5
+; RV32IA-NEXT: sc.w.rl a5, a5, (a3)
+; RV32IA-NEXT: bnez a5, .LBB12_1
+; RV32IA-NEXT: # %bb.2:
+; RV32IA-NEXT: srl a0, a4, a0
+; RV32IA-NEXT: ret
+;
+; RV32I-LABEL: atomicrmw_xchg_i8_acq_rel:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -16
+; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: li a2, 4
+; RV32I-NEXT: call __atomic_exchange_1
+; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 16
+; RV32I-NEXT: ret
+;
+; RV64IA-ZABHA-LABEL: atomicrmw_xchg_i8_acq_rel:
+; RV64IA-ZABHA: # %bb.0:
+; RV64IA-ZABHA-NEXT: amoswap.b.aqrl a0, a1, (a0)
+; RV64IA-ZABHA-NEXT: ret
+;
+; RV64IA-LABEL: atomicrmw_xchg_i8_acq_rel:
+; RV64IA: # %bb.0:
+; RV64IA-NEXT: li a2, 255
+; RV64IA-NEXT: andi a3, a0, -4
+; RV64IA-NEXT: andi a0, a0, 3
+; RV64IA-NEXT: slli a0, a0, 3
+; RV64IA-NEXT: zext.b a1, a1
+; RV64IA-NEXT: sllw a2, a2, a0
+; RV64IA-NEXT: sllw a1, a1, a0
+; RV64IA-NEXT: .LBB12_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT: lr.w.aq a4, (a3)
+; RV64IA-NEXT: mv a5, a1
+; RV64IA-NEXT: xor a5, a4, a5
+; RV64IA-NEXT: and a5, a5, a2
+; RV64IA-NEXT: xor a5, a4, a5
+; RV64IA-NEXT: sc.w.rl a5, a5, (a3)
+; RV64IA-NEXT: bnez a5, .LBB12_1
+; RV64IA-NEXT: # %bb.2:
+; RV64IA-NEXT: srlw a0, a4, a0
+; RV64IA-NEXT: ret
+;
+; RV64I-LABEL: atomicrmw_xchg_i8_acq_rel:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -16
+; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: li a2, 4
+; RV64I-NEXT: call __atomic_exchange_1
+; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 16
+; RV64I-NEXT: ret
+ %res = atomicrmw xchg ptr %ptr, i8 %val acq_rel
+ ret i8 %res
+}
+
+define i16 @atomicrmw_xchg_i16_acq_rel(ptr %ptr, i16 %val) nounwind {
+; RV32IA-ZABHA-LABEL: atomicrmw_xchg_i16_acq_rel:
+; RV32IA-ZABHA: # %bb.0:
+; RV32IA-ZABHA-NEXT: amoswap.h.aqrl a0, a1, (a0)
+; RV32IA-ZABHA-NEXT: ret
+;
+; RV32IA-LABEL: atomicrmw_xchg_i16_acq_rel:
+; RV32IA: # %bb.0:
+; RV32IA-NEXT: lui a2, 16
+; RV32IA-NEXT: addi a2, a2, -1
+; RV32IA-NEXT: andi a3, a0, -4
+; RV32IA-NEXT: andi a0, a0, 3
+; RV32IA-NEXT: slli a0, a0, 3
+; RV32IA-NEXT: and a1, a1, a2
+; RV32IA-NEXT: sll a2, a2, a0
+; RV32IA-NEXT: sll a1, a1, a0
+; RV32IA-NEXT: .LBB13_1: # =>This Inner Loop Header: Depth=1
+; RV32IA-NEXT: lr.w.aq a4, (a3)
+; RV32IA-NEXT: mv a5, a1
+; RV32IA-NEXT: xor a5, a4, a5
+; RV32IA-NEXT: and a5, a5, a2
+; RV32IA-NEXT: xor a5, a4, a5
+; RV32IA-NEXT: sc.w.rl a5, a5, (a3)
+; RV32IA-NEXT: bnez a5, .LBB13_1
+; RV32IA-NEXT: # %bb.2:
+; RV32IA-NEXT: srl a0, a4, a0
+; RV32IA-NEXT: ret
+;
+; RV32I-LABEL: atomicrmw_xchg_i16_acq_rel:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -16
+; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: li a2, 4
+; RV32I-NEXT: call __atomic_exchange_2
+; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 16
+; RV32I-NEXT: ret
+;
+; RV64IA-ZABHA-LABEL: atomicrmw_xchg_i16_acq_rel:
+; RV64IA-ZABHA: # %bb.0:
+; RV64IA-ZABHA-NEXT: amoswap.h.aqrl a0, a1, (a0)
+; RV64IA-ZABHA-NEXT: ret
+;
+; RV64IA-LABEL: atomicrmw_xchg_i16_acq_rel:
+; RV64IA: # %bb.0:
+; RV64IA-NEXT: lui a2, 16
+; RV64IA-NEXT: addi a2, a2, -1
+; RV64IA-NEXT: andi a3, a0, -4
+; RV64IA-NEXT: andi a0, a0, 3
+; RV64IA-NEXT: slli a0, a0, 3
+; RV64IA-NEXT: and a1, a1, a2
+; RV64IA-NEXT: sllw a2, a2, a0
+; RV64IA-NEXT: sllw a1, a1, a0
+; RV64IA-NEXT: .LBB13_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT: lr.w.aq a4, (a3)
+; RV64IA-NEXT: mv a5, a1
+; RV64IA-NEXT: xor a5, a4, a5
+; RV64IA-NEXT: and a5, a5, a2
+; RV64IA-NEXT: xor a5, a4, a5
+; RV64IA-NEXT: sc.w.rl a5, a5, (a3)
+; RV64IA-NEXT: bnez a5, .LBB13_1
+; RV64IA-NEXT: # %bb.2:
+; RV64IA-NEXT: srlw a0, a4, a0
+; RV64IA-NEXT: ret
+;
+; RV64I-LABEL: atomicrmw_xchg_i16_acq_rel:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -16
+; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: li a2, 4
+; RV64I-NEXT: call __atomic_exchange_2
+; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 16
+; RV64I-NEXT: ret
+ %res = atomicrmw xchg ptr %ptr, i16 %val acq_rel
+ ret i16 %res
+}
+
+define i32 @atomicrmw_xchg_i32_acq_rel(ptr %ptr, i32 %val) nounwind {
+; RV32IA-ZABHA-LABEL: atomicrmw_xchg_i32_acq_rel:
+; RV32IA-ZABHA: # %bb.0:
+; RV32IA-ZABHA-NEXT: amoswap.w.aqrl a0, a1, (a0)
+; RV32IA-ZABHA-NEXT: ret
+;
+; RV32IA-LABEL: atomicrmw_xchg_i32_acq_rel:
+; RV32IA: # %bb.0:
+; RV32IA-NEXT: amoswap.w.aqrl a0, a1, (a0)
+; RV32IA-NEXT: ret
+;
+; RV32I-LABEL: atomicrmw_xchg_i32_acq_rel:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -16
+; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: li a2, 4
+; RV32I-NEXT: call __atomic_exchange_4
+; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 16
+; RV32I-NEXT: ret
+;
+; RV64IA-ZABHA-LABEL: atomicrmw_xchg_i32_acq_rel:
+; RV64IA-ZABHA: # %bb.0:
+; RV64IA-ZABHA-NEXT: amoswap.w.aqrl a0, a1, (a0)
+; RV64IA-ZABHA-NEXT: ret
+;
+; RV64IA-LABEL: atomicrmw_xchg_i32_acq_rel:
+; RV64IA: # %bb.0:
+; RV64IA-NEXT: amoswap.w.aqrl a0, a1, (a0)
+; RV64IA-NEXT: ret
+;
+; RV64I-LABEL: atomicrmw_xchg_i32_acq_rel:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -16
+; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: li a2, 4
+; RV64I-NEXT: call __atomic_exchange_4
+; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 16
+; RV64I-NEXT: ret
+ %res = atomicrmw xchg ptr %ptr, i32 %val acq_rel
+ ret i32 %res
+}
+
+define i64 @atomicrmw_xchg_i64_acq_rel(ptr %ptr, i64 %val) nounwind {
+; RV32IA-ZABHA-LABEL: atomicrmw_xchg_i64_acq_rel:
+; RV32IA-ZABHA: # %bb.0:
+; RV32IA-ZABHA-NEXT: addi sp, sp, -16
+; RV32IA-ZABHA-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IA-ZABHA-NEXT: li a3, 4
+; RV32IA-ZABHA-NEXT: call __atomic_exchange_8
+; RV32IA-ZABHA-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IA-ZABHA-NEXT: addi sp, sp, 16
+; RV32IA-ZABHA-NEXT: ret
+;
+; RV32IA-LABEL: atomicrmw_xchg_i64_acq_rel:
+; RV32IA: # %bb.0:
+; RV32IA-NEXT: addi sp, sp, -16
+; RV32IA-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IA-NEXT: li a3, 4
+; RV32IA-NEXT: call __atomic_exchange_8
+; RV32IA-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IA-NEXT: addi sp, sp, 16
+; RV32IA-NEXT: ret
+;
+; RV32I-LABEL: atomicrmw_xchg_i64_acq_rel:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -16
+; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: li a3, 4
+; RV32I-NEXT: call __atomic_exchange_8
+; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 16
+; RV32I-NEXT: ret
+;
+; RV64IA-ZABHA-LABEL: atomicrmw_xchg_i64_acq_rel:
+; RV64IA-ZABHA: # %bb.0:
+; RV64IA-ZABHA-NEXT: amoswap.d.aqrl a0, a1, (a0)
+; RV64IA-ZABHA-NEXT: ret
+;
+; RV64IA-LABEL: atomicrmw_xchg_i64_acq_rel:
+; RV64IA: # %bb.0:
+; RV64IA-NEXT: amoswap.d.aqrl a0, a1, (a0)
+; RV64IA-NEXT: ret
+;
+; RV64I-LABEL: atomicrmw_xchg_i64_acq_rel:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -16
+; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: li a2, 4
+; RV64I-NEXT: call __atomic_exchange_8
+; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 16
+; RV64I-NEXT: ret
+ %res = atomicrmw xchg ptr %ptr, i64 %val acq_rel
+ ret i64 %res
+}
+
+define i8 @atomicrmw_xchg_i8_seq_cst(ptr %ptr, i8 %val) nounwind {
+; RV32IA-ZABHA-LABEL: atomicrmw_xchg_i8_seq_cst:
+; RV32IA-ZABHA: # %bb.0:
+; RV32IA-ZABHA-NEXT: amoswap.b.aqrl a0, a1, (a0)
+; RV32IA-ZABHA-NEXT: ret
+;
+; RV32IA-LABEL: atomicrmw_xchg_i8_seq_cst:
+; RV32IA: # %bb.0:
+; RV32IA-NEXT: li a2, 255
+; RV32IA-NEXT: andi a3, a0, -4
+; RV32IA-NEXT: andi a0, a0, 3
+; RV32IA-NEXT: slli a0, a0, 3
+; RV32IA-NEXT: zext.b a1, a1
+; RV32IA-NEXT: sll a2, a2, a0
+; RV32IA-NEXT: sll a1, a1, a0
+; RV32IA-NEXT: .LBB16_1: # =>This Inner Loop Header: Depth=1
+; RV32IA-NEXT: lr.w.aqrl a4, (a3)
+; RV32IA-NEXT: mv a5, a1
+; RV32IA-NEXT: xor a5, a4, a5
+; RV32IA-NEXT: and a5, a5, a2
+; RV32IA-NEXT: xor a5, a4, a5
+; RV32IA-NEXT: sc.w.rl a5, a5, (a3)
+; RV32IA-NEXT: bnez a5, .LBB16_1
+; RV32IA-NEXT: # %bb.2:
+; RV32IA-NEXT: srl a0, a4, a0
+; RV32IA-NEXT: ret
+;
+; RV32I-LABEL: atomicrmw_xchg_i8_seq_cst:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -16
+; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: li a2, 5
+; RV32I-NEXT: call __atomic_exchange_1
+; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 16
+; RV32I-NEXT: ret
+;
+; RV64IA-ZABHA-LABEL: atomicrmw_xchg_i8_seq_cst:
+; RV64IA-ZABHA: # %bb.0:
+; RV64IA-ZABHA-NEXT: amoswap.b.aqrl a0, a1, (a0)
+; RV64IA-ZABHA-NEXT: ret
+;
+; RV64IA-LABEL: atomicrmw_xchg_i8_seq_cst:
+; RV64IA: # %bb.0:
+; RV64IA-NEXT: li a2, 255
+; RV64IA-NEXT: andi a3, a0, -4
+; RV64IA-NEXT: andi a0, a0, 3
+; RV64IA-NEXT: slli a0, a0, 3
+; RV64IA-NEXT: zext.b a1, a1
+; RV64IA-NEXT: sllw a2, a2, a0
+; RV64IA-NEXT: sllw a1, a1, a0
+; RV64IA-NEXT: .LBB16_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT: lr.w.aqrl a4, (a3)
+; RV64IA-NEXT: mv a5, a1
+; RV64IA-NEXT: xor a5, a4, a5
+; RV64IA-NEXT: and a5, a5, a2
+; RV64IA-NEXT: xor a5, a4, a5
+; RV64IA-NEXT: sc.w.rl a5, a5, (a3)
+; RV64IA-NEXT: bnez a5, .LBB16_1
+; RV64IA-NEXT: # %bb.2:
+; RV64IA-NEXT: srlw a0, a4, a0
+; RV64IA-NEXT: ret
+;
+; RV64I-LABEL: atomicrmw_xchg_i8_seq_cst:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -16
+; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: li a2, 5
+; RV64I-NEXT: call __atomic_exchange_1
+; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 16
+; RV64I-NEXT: ret
+ %res = atomicrmw xchg ptr %ptr, i8 %val seq_cst
+ ret i8 %res
+}
+
+define i16 @atomicrmw_xchg_i16_seq_cst(ptr %ptr, i16 %val) nounwind {
+; RV32IA-ZABHA-LABEL: atomicrmw_xchg_i16_seq_cst:
+; RV32IA-ZABHA: # %bb.0:
+; RV32IA-ZABHA-NEXT: amoswap.h.aqrl a0, a1, (a0)
+; RV32IA-ZABHA-NEXT: ret
+;
+; RV32IA-LABEL: atomicrmw_xchg_i16_seq_cst:
+; RV32IA: # %bb.0:
+; RV32IA-NEXT: lui a2, 16
+; RV32IA-NEXT: addi a2, a2, -1
+; RV32IA-NEXT: andi a3, a0, -4
+; RV32IA-NEXT: andi a0, a0, 3
+; RV32IA-NEXT: slli a0, a0, 3
+; RV32IA-NEXT: and a1, a1, a2
+; RV32IA-NEXT: sll a2, a2, a0
+; RV32IA-NEXT: sll a1, a1, a0
+; RV32IA-NEXT: .LBB17_1: # =>This Inner Loop Header: Depth=1
+; RV32IA-NEXT: lr.w.aqrl a4, (a3)
+; RV32IA-NEXT: mv a5, a1
+; RV32IA-NEXT: xor a5, a4, a5
+; RV32IA-NEXT: and a5, a5, a2
+; RV32IA-NEXT: xor a5, a4, a5
+; RV32IA-NEXT: sc.w.rl a5, a5, (a3)
+; RV32IA-NEXT: bnez a5, .LBB17_1
+; RV32IA-NEXT: # %bb.2:
+; RV32IA-NEXT: srl a0, a4, a0
+; RV32IA-NEXT: ret
+;
+; RV32I-LABEL: atomicrmw_xchg_i16_seq_cst:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -16
+; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: li a2, 5
+; RV32I-NEXT: call __atomic_exchange_2
+; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 16
+; RV32I-NEXT: ret
+;
+; RV64IA-ZABHA-LABEL: atomicrmw_xchg_i16_seq_cst:
+; RV64IA-ZABHA: # %bb.0:
+; RV64IA-ZABHA-NEXT: amoswap.h.aqrl a0, a1, (a0)
+; RV64IA-ZABHA-NEXT: ret
+;
+; RV64IA-LABEL: atomicrmw_xchg_i16_seq_cst:
+; RV64IA: # %bb.0:
+; RV64IA-NEXT: lui a2, 16
+; RV64IA-NEXT: addi a2, a2, -1
+; RV64IA-NEXT: andi a3, a0, -4
+; RV64IA-NEXT: andi a0, a0, 3
+; RV64IA-NEXT: slli a0, a0, 3
+; RV64IA-NEXT: and a1, a1, a2
+; RV64IA-NEXT: sllw a2, a2, a0
+; RV64IA-NEXT: sllw a1, a1, a0
+; RV64IA-NEXT: .LBB17_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT: lr.w.aqrl a4, (a3)
+; RV64IA-NEXT: mv a5, a1
+; RV64IA-NEXT: xor a5, a4, a5
+; RV64IA-NEXT: and a5, a5, a2
+; RV64IA-NEXT: xor a5, a4, a5
+; RV64IA-NEXT: sc.w.rl a5, a5, (a3)
+; RV64IA-NEXT: bnez a5, .LBB17_1
+; RV64IA-NEXT: # %bb.2:
+; RV64IA-NEXT: srlw a0, a4, a0
+; RV64IA-NEXT: ret
+;
+; RV64I-LABEL: atomicrmw_xchg_i16_seq_cst:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -16
+; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: li a2, 5
+; RV64I-NEXT: call __atomic_exchange_2
+; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 16
+; RV64I-NEXT: ret
+ %res = atomicrmw xchg ptr %ptr, i16 %val seq_cst
+ ret i16 %res
+}
+
+define i32 @atomicrmw_xchg_i32_seq_cst(ptr %ptr, i32 %val) nounwind {
+; RV32IA-ZABHA-LABEL: atomicrmw_xchg_i32_seq_cst:
+; RV32IA-ZABHA: # %bb.0:
+; RV32IA-ZABHA-NEXT: amoswap.w.aqrl a0, a1, (a0)
+; RV32IA-ZABHA-NEXT: ret
+;
+; RV32IA-LABEL: atomicrmw_xchg_i32_seq_cst:
+; RV32IA: # %bb.0:
+; RV32IA-NEXT: amoswap.w.aqrl a0, a1, (a0)
+; RV32IA-NEXT: ret
+;
+; RV32I-LABEL: atomicrmw_xchg_i32_seq_cst:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -16
+; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: li a2, 5
+; RV32I-NEXT: call __atomic_exchange_4
+; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 16
+; RV32I-NEXT: ret
+;
+; RV64IA-ZABHA-LABEL: atomicrmw_xchg_i32_seq_cst:
+; RV64IA-ZABHA: # %bb.0:
+; RV64IA-ZABHA-NEXT: amoswap.w.aqrl a0, a1, (a0)
+; RV64IA-ZABHA-NEXT: ret
+;
+; RV64IA-LABEL: atomicrmw_xchg_i32_seq_cst:
+; RV64IA: # %bb.0:
+; RV64IA-NEXT: amoswap.w.aqrl a0, a1, (a0)
+; RV64IA-NEXT: ret
+;
+; RV64I-LABEL: atomicrmw_xchg_i32_seq_cst:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -16
+; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: li a2, 5
+; RV64I-NEXT: call __atomic_exchange_4
+; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 16
+; RV64I-NEXT: ret
+ %res = atomicrmw xchg ptr %ptr, i32 %val seq_cst
+ ret i32 %res
+}
+
+define i64 @atomicrmw_xchg_i64_seq_cst(ptr %ptr, i64 %val) nounwind {
+; RV32IA-ZABHA-LABEL: atomicrmw_xchg_i64_seq_cst:
+; RV32IA-ZABHA: # %bb.0:
+; RV32IA-ZABHA-NEXT: addi sp, sp, -16
+; RV32IA-ZABHA-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IA-ZABHA-NEXT: li a3, 5
+; RV32IA-ZABHA-NEXT: call __atomic_exchange_8
+; RV32IA-ZABHA-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IA-ZABHA-NEXT: addi sp, sp, 16
+; RV32IA-ZABHA-NEXT: ret
+;
+; RV32IA-LABEL: atomicrmw_xchg_i64_seq_cst:
+; RV32IA: # %bb.0:
+; RV32IA-NEXT: addi sp, sp, -16
+; RV32IA-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IA-NEXT: li a3, 5
+; RV32IA-NEXT: call __atomic_exchange_8
+; RV32IA-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IA-NEXT: addi sp, sp, 16
+; RV32IA-NEXT: ret
+;
+; RV32I-LABEL: atomicrmw_xchg_i64_seq_cst:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -16
+; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: li a3, 5
+; RV32I-NEXT: call __atomic_exchange_8
+; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 16
+; RV32I-NEXT: ret
+;
+; RV64IA-ZABHA-LABEL: atomicrmw_xchg_i64_seq_cst:
+; RV64IA-ZABHA: # %bb.0:
+; RV64IA-ZABHA-NEXT: amoswap.d.aqrl a0, a1, (a0)
+; RV64IA-ZABHA-NEXT: ret
+;
+; RV64IA-LABEL: atomicrmw_xchg_i64_seq_cst:
+; RV64IA: # %bb.0:
+; RV64IA-NEXT: amoswap.d.aqrl a0, a1, (a0)
+; RV64IA-NEXT: ret
+;
+; RV64I-LABEL: atomicrmw_xchg_i64_seq_cst:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -16
+; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: li a2, 5
+; RV64I-NEXT: call __atomic_exchange_8
+; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 16
+; RV64I-NEXT: ret
+ %res = atomicrmw xchg ptr %ptr, i64 %val seq_cst
+ ret i64 %res
+}
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer-info-validation.mir
index d846ec6c1732f..488c1c6f1fe45 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer-info-validation.mir
@@ -241,8 +241,9 @@
# DEBUG-NEXT: .. the first uncovered type index: 2, OK
# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
# DEBUG-NEXT: G_ATOMICRMW_XCHG (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
+# DEBUG-NEXT: .. the first uncovered type index: 2, OK
+# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
# DEBUG-NEXT: G_ATOMICRMW_ADD (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
# DEBUG-NEXT: .. the first uncovered type index: 2, OK
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-atomicrmw-xchg-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-atomicrmw-xchg-rv32.mir
new file mode 100644
index 0000000000000..ef2dfa47c84ec
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-atomicrmw-xchg-rv32.mir
@@ -0,0 +1,94 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv32 -mattr=+a,+zabha -run-pass=legalizer %s -o - | FileCheck %s --check-prefixes=RV32IA-ZABHA
+# RUN: llc -mtriple=riscv32 -mattr=+a -run-pass=legalizer %s -o - | FileCheck %s --check-prefixes=RV32IA
+---
+name: atomicrmw_xchg_i8_monotonic
+body: |
+ bb.0.entry:
+ liveins: $x10, $x11
+
+ ; RV32IA-ZABHA-LABEL: name: atomicrmw_xchg_i8_monotonic
+ ; RV32IA-ZABHA: liveins: $x10, $x11
+ ; RV32IA-ZABHA-NEXT: {{ $}}
+ ; RV32IA-ZABHA-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+ ; RV32IA-ZABHA-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+ ; RV32IA-ZABHA-NEXT: [[ATOMICRMW_XCHG:%[0-9]+]]:_(s32) = G_ATOMICRMW_XCHG [[COPY]](p0), [[COPY1]] :: (load store monotonic (s8))
+ ; RV32IA-ZABHA-NEXT: $x10 = COPY [[ATOMICRMW_XCHG]](s32)
+ ; RV32IA-ZABHA-NEXT: PseudoRET implicit $x10
+ ;
+ ; RV32IA-LABEL: name: atomicrmw_xchg_i8_monotonic
+ ; RV32IA: liveins: $x10, $x11
+ ; RV32IA-NEXT: {{ $}}
+ ; RV32IA-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+ ; RV32IA-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+ ; RV32IA-NEXT: [[ATOMICRMW_XCHG:%[0-9]+]]:_(s32) = G_ATOMICRMW_XCHG [[COPY]](p0), [[COPY1]] :: (load store monotonic (s8))
+ ; RV32IA-NEXT: $x10 = COPY [[ATOMICRMW_XCHG]](s32)
+ ; RV32IA-NEXT: PseudoRET implicit $x10
+ %0:_(p0) = COPY $x10
+ %1:_(s32) = COPY $x11
+ %2:_(s8) = G_TRUNC %1(s32)
+ %3:_(s8) = G_ATOMICRMW_XCHG %0(p0), %2 :: (load store monotonic (s8))
+ %4:_(s32) = G_ANYEXT %3(s8)
+ $x10 = COPY %4(s32)
+ PseudoRET implicit $x10
+...
+---
+name: atomicrmw_xchg_i16_monotonic
+body: |
+ bb.0.entry:
+ liveins: $x10, $x11
+
+ ; RV32IA-ZABHA-LABEL: name: atomicrmw_xchg_i16_monotonic
+ ; RV32IA-ZABHA: liveins: $x10, $x11
+ ; RV32IA-ZABHA-NEXT: {{ $}}
+ ; RV32IA-ZABHA-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+ ; RV32IA-ZABHA-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+ ; RV32IA-ZABHA-NEXT: [[ATOMICRMW_XCHG:%[0-9]+]]:_(s32) = G_ATOMICRMW_XCHG [[COPY]](p0), [[COPY1]] :: (load store monotonic (s16))
+ ; RV32IA-ZABHA-NEXT: $x10 = COPY [[ATOMICRMW_XCHG]](s32)
+ ; RV32IA-ZABHA-NEXT: PseudoRET implicit $x10
+ ;
+ ; RV32IA-LABEL: name: atomicrmw_xchg_i16_monotonic
+ ; RV32IA: liveins: $x10, $x11
+ ; RV32IA-NEXT: {{ $}}
+ ; RV32IA-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+ ; RV32IA-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+ ; RV32IA-NEXT: [[ATOMICRMW_XCHG:%[0-9]+]]:_(s32) = G_ATOMICRMW_XCHG [[COPY]](p0), [[COPY1]] :: (load store monotonic (s16))
+ ; RV32IA-NEXT: $x10 = COPY [[ATOMICRMW_XCHG]](s32)
+ ; RV32IA-NEXT: PseudoRET implicit $x10
+ %0:_(p0) = COPY $x10
+ %1:_(s32) = COPY $x11
+ %2:_(s16) = G_TRUNC %1(s32)
+ %3:_(s16) = G_ATOMICRMW_XCHG %0(p0), %2 :: (load store monotonic (s16))
+ %4:_(s32) = G_ANYEXT %3(s16)
+ $x10 = COPY %4(s32)
+ PseudoRET implicit $x10
+...
+---
+name: atomicrmw_xchg_i32_monotonic
+body: |
+ bb.0.entry:
+ liveins: $x10, $x11
+
+ ; RV32IA-ZABHA-LABEL: name: atomicrmw_xchg_i32_monotonic
+ ; RV32IA-ZABHA: liveins: $x10, $x11
+ ; RV32IA-ZABHA-NEXT: {{ $}}
+ ; RV32IA-ZABHA-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+ ; RV32IA-ZABHA-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+ ; RV32IA-ZABHA-NEXT: [[ATOMICRMW_XCHG:%[0-9]+]]:_(s32) = G_ATOMICRMW_XCHG [[COPY]](p0), [[COPY1]] :: (load store monotonic (s32))
+ ; RV32IA-ZABHA-NEXT: $x10 = COPY [[ATOMICRMW_XCHG]](s32)
+ ; RV32IA-ZABHA-NEXT: PseudoRET implicit $x10
+ ;
+ ; RV32IA-LABEL: name: atomicrmw_xchg_i32_monotonic
+ ; RV32IA: liveins: $x10, $x11
+ ; RV32IA-NEXT: {{ $}}
+ ; RV32IA-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+ ; RV32IA-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+ ; RV32IA-NEXT: [[ATOMICRMW_XCHG:%[0-9]+]]:_(s32) = G_ATOMICRMW_XCHG [[COPY]](p0), [[COPY1]] :: (load store monotonic (s32))
+ ; RV32IA-NEXT: $x10 = COPY [[ATOMICRMW_XCHG]](s32)
+ ; RV32IA-NEXT: PseudoRET implicit $x10
+ %0:_(p0) = COPY $x10
+ %1:_(s32) = COPY $x11
+ %2:_(s32) = G_ATOMICRMW_XCHG %0(p0), %1 :: (load store monotonic (s32))
+ $x10 = COPY %2(s32)
+ PseudoRET implicit $x10
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-atomicrmw-xchg-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-atomicrmw-xchg-rv64.mir
new file mode 100644
index 0000000000000..abeb564c1b100
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-atomicrmw-xchg-rv64.mir
@@ -0,0 +1,125 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv64 -mattr=+a,+zabha -run-pass=legalizer %s -o - | FileCheck %s --check-prefixes=RV64IA-ZABHA
+# RUN: llc -mtriple=riscv64 -mattr=+a -run-pass=legalizer %s -o - | FileCheck %s --check-prefixes=RV64IA
+---
+name: atomicrmw_xchg_i8_monotonic
+body: |
+ bb.0.entry:
+ liveins: $x10, $x11
+
+ ; RV64IA-ZABHA-LABEL: name: atomicrmw_xchg_i8_monotonic
+ ; RV64IA-ZABHA: liveins: $x10, $x11
+ ; RV64IA-ZABHA-NEXT: {{ $}}
+ ; RV64IA-ZABHA-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+ ; RV64IA-ZABHA-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+ ; RV64IA-ZABHA-NEXT: [[ATOMICRMW_XCHG:%[0-9]+]]:_(s64) = G_ATOMICRMW_XCHG [[COPY]](p0), [[COPY1]] :: (load store monotonic (s8))
+ ; RV64IA-ZABHA-NEXT: $x10 = COPY [[ATOMICRMW_XCHG]](s64)
+ ; RV64IA-ZABHA-NEXT: PseudoRET implicit $x10
+ ;
+ ; RV64IA-LABEL: name: atomicrmw_xchg_i8_monotonic
+ ; RV64IA: liveins: $x10, $x11
+ ; RV64IA-NEXT: {{ $}}
+ ; RV64IA-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+ ; RV64IA-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+ ; RV64IA-NEXT: [[ATOMICRMW_XCHG:%[0-9]+]]:_(s64) = G_ATOMICRMW_XCHG [[COPY]](p0), [[COPY1]] :: (load store monotonic (s8))
+ ; RV64IA-NEXT: $x10 = COPY [[ATOMICRMW_XCHG]](s64)
+ ; RV64IA-NEXT: PseudoRET implicit $x10
+ %0:_(p0) = COPY $x10
+ %1:_(s64) = COPY $x11
+ %2:_(s8) = G_TRUNC %1(s64)
+ %3:_(s8) = G_ATOMICRMW_XCHG %0(p0), %2 :: (load store monotonic (s8))
+ %4:_(s64) = G_ANYEXT %3(s8)
+ $x10 = COPY %4(s64)
+ PseudoRET implicit $x10
+...
+---
+name: atomicrmw_xchg_i16_monotonic
+body: |
+ bb.0.entry:
+ liveins: $x10, $x11
+
+ ; RV64IA-ZABHA-LABEL: name: atomicrmw_xchg_i16_monotonic
+ ; RV64IA-ZABHA: liveins: $x10, $x11
+ ; RV64IA-ZABHA-NEXT: {{ $}}
+ ; RV64IA-ZABHA-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+ ; RV64IA-ZABHA-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+ ; RV64IA-ZABHA-NEXT: [[ATOMICRMW_XCHG:%[0-9]+]]:_(s64) = G_ATOMICRMW_XCHG [[COPY]](p0), [[COPY1]] :: (load store monotonic (s16))
+ ; RV64IA-ZABHA-NEXT: $x10 = COPY [[ATOMICRMW_XCHG]](s64)
+ ; RV64IA-ZABHA-NEXT: PseudoRET implicit $x10
+ ;
+ ; RV64IA-LABEL: name: atomicrmw_xchg_i16_monotonic
+ ; RV64IA: liveins: $x10, $x11
+ ; RV64IA-NEXT: {{ $}}
+ ; RV64IA-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+ ; RV64IA-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+ ; RV64IA-NEXT: [[ATOMICRMW_XCHG:%[0-9]+]]:_(s64) = G_ATOMICRMW_XCHG [[COPY]](p0), [[COPY1]] :: (load store monotonic (s16))
+ ; RV64IA-NEXT: $x10 = COPY [[ATOMICRMW_XCHG]](s64)
+ ; RV64IA-NEXT: PseudoRET implicit $x10
+ %0:_(p0) = COPY $x10
+ %1:_(s64) = COPY $x11
+ %2:_(s16) = G_TRUNC %1(s64)
+ %3:_(s16) = G_ATOMICRMW_XCHG %0(p0), %2 :: (load store monotonic (s16))
+ %4:_(s64) = G_ANYEXT %3(s16)
+ $x10 = COPY %4(s64)
+ PseudoRET implicit $x10
+...
+---
+name: atomicrmw_xchg_i32_monotonic
+body: |
+ bb.0.entry:
+ liveins: $x10, $x11
+
+ ; RV64IA-ZABHA-LABEL: name: atomicrmw_xchg_i32_monotonic
+ ; RV64IA-ZABHA: liveins: $x10, $x11
+ ; RV64IA-ZABHA-NEXT: {{ $}}
+ ; RV64IA-ZABHA-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+ ; RV64IA-ZABHA-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+ ; RV64IA-ZABHA-NEXT: [[ATOMICRMW_XCHG:%[0-9]+]]:_(s64) = G_ATOMICRMW_XCHG [[COPY]](p0), [[COPY1]] :: (load store monotonic (s32))
+ ; RV64IA-ZABHA-NEXT: $x10 = COPY [[ATOMICRMW_XCHG]](s64)
+ ; RV64IA-ZABHA-NEXT: PseudoRET implicit $x10
+ ;
+ ; RV64IA-LABEL: name: atomicrmw_xchg_i32_monotonic
+ ; RV64IA: liveins: $x10, $x11
+ ; RV64IA-NEXT: {{ $}}
+ ; RV64IA-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+ ; RV64IA-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+ ; RV64IA-NEXT: [[ATOMICRMW_XCHG:%[0-9]+]]:_(s64) = G_ATOMICRMW_XCHG [[COPY]](p0), [[COPY1]] :: (load store monotonic (s32))
+ ; RV64IA-NEXT: $x10 = COPY [[ATOMICRMW_XCHG]](s64)
+ ; RV64IA-NEXT: PseudoRET implicit $x10
+ %0:_(p0) = COPY $x10
+ %1:_(s64) = COPY $x11
+ %2:_(s32) = G_TRUNC %1(s64)
+ %3:_(s32) = G_ATOMICRMW_XCHG %0(p0), %2 :: (load store monotonic (s32))
+ %4:_(s64) = G_ANYEXT %3(s32)
+ $x10 = COPY %4(s64)
+ PseudoRET implicit $x10
+...
+---
+name: atomicrmw_xchg_i64_monotonic
+body: |
+ bb.0.entry:
+ liveins: $x10, $x11
+
+ ; RV64IA-ZABHA-LABEL: name: atomicrmw_xchg_i64_monotonic
+ ; RV64IA-ZABHA: liveins: $x10, $x11
+ ; RV64IA-ZABHA-NEXT: {{ $}}
+ ; RV64IA-ZABHA-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+ ; RV64IA-ZABHA-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+ ; RV64IA-ZABHA-NEXT: [[ATOMICRMW_XCHG:%[0-9]+]]:_(s64) = G_ATOMICRMW_XCHG [[COPY]](p0), [[COPY1]] :: (load store monotonic (s64))
+ ; RV64IA-ZABHA-NEXT: $x10 = COPY [[ATOMICRMW_XCHG]](s64)
+ ; RV64IA-ZABHA-NEXT: PseudoRET implicit $x10
+ ;
+ ; RV64IA-LABEL: name: atomicrmw_xchg_i64_monotonic
+ ; RV64IA: liveins: $x10, $x11
+ ; RV64IA-NEXT: {{ $}}
+ ; RV64IA-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+ ; RV64IA-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+ ; RV64IA-NEXT: [[ATOMICRMW_XCHG:%[0-9]+]]:_(s64) = G_ATOMICRMW_XCHG [[COPY]](p0), [[COPY1]] :: (load store monotonic (s64))
+ ; RV64IA-NEXT: $x10 = COPY [[ATOMICRMW_XCHG]](s64)
+ ; RV64IA-NEXT: PseudoRET implicit $x10
+ %0:_(p0) = COPY $x10
+ %1:_(s64) = COPY $x11
+ %2:_(s64) = G_ATOMICRMW_XCHG %0(p0), %1 :: (load store monotonic (s64))
+ $x10 = COPY %2(s64)
+ PseudoRET implicit $x10
+...
More information about the llvm-commits
mailing list