[Mlir-commits] [llvm] [mlir] Add usub_cond and usub_sat operations to atomicrmw (PR #105568)
llvmlistbot at llvm.org
llvmlistbot at llvm.org
Thu Aug 22 07:47:54 PDT 2024
https://github.com/anjenner updated https://github.com/llvm/llvm-project/pull/105568
>From 9191aed2028fbfee117253250830af5498ac47bf Mon Sep 17 00:00:00 2001
From: Andrew Jenner <Andrew.Jenner at amd.com>
Date: Thu, 8 Aug 2024 10:32:55 -0400
Subject: [PATCH 1/3] Add usub_cond and usub_sat operations to atomicrmw
These both perform conditional subtraction, returning the minuend and zero
respectively, if the difference is negative.
AMDGPU has instructions for these. Currently we use target intrinsics for
these, but those do not carry the ordering and syncscope. Add these to
atomicrmw so we can carry these and benefit from the regular legalization
processes.
---
llvm/bindings/ocaml/llvm/llvm.ml | 6 +
llvm/bindings/ocaml/llvm/llvm.mli | 6 +
llvm/docs/GlobalISel/GenericOpcode.rst | 4 +-
llvm/docs/LangRef.rst | 4 +
llvm/docs/ReleaseNotes.rst | 2 +
llvm/include/llvm/AsmParser/LLToken.h | 2 +
llvm/include/llvm/Bitcode/LLVMBitCodes.h | 4 +-
.../CodeGen/GlobalISel/MachineIRBuilder.h | 35 +
llvm/include/llvm/CodeGen/ISDOpcodes.h | 2 +
llvm/include/llvm/CodeGen/SelectionDAGNodes.h | 40 +-
llvm/include/llvm/IR/Instructions.h | 10 +-
llvm/include/llvm/Support/TargetOpcodes.def | 4 +-
llvm/include/llvm/Target/GenericOpcodes.td | 2 +
.../Target/GlobalISel/SelectionDAGCompat.td | 2 +
.../include/llvm/Target/TargetSelectionDAG.td | 4 +
llvm/lib/AsmParser/LLLexer.cpp | 2 +
llvm/lib/AsmParser/LLParser.cpp | 6 +
llvm/lib/Bitcode/Reader/BitcodeReader.cpp | 4 +
llvm/lib/Bitcode/Writer/BitcodeWriter.cpp | 4 +
llvm/lib/CodeGen/AtomicExpandPass.cpp | 8 +-
llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 6 +
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 24 +-
.../SelectionDAG/SelectionDAGBuilder.cpp | 6 +
.../SelectionDAG/SelectionDAGDumper.cpp | 4 +
llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp | 2 +
llvm/lib/IR/Instructions.cpp | 4 +
.../LoongArch/LoongArchISelLowering.cpp | 4 +-
llvm/lib/Target/PowerPC/PPCISelLowering.cpp | 2 +
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 4 +-
llvm/lib/Target/X86/X86ISelLowering.cpp | 2 +
.../InstCombine/InstCombineAtomicRMW.cpp | 2 +
llvm/lib/Transforms/Utils/LowerAtomic.cpp | 11 +
llvm/test/Assembler/atomic.ll | 10 +
llvm/test/Bitcode/compatibility.ll | 28 +
.../GlobalISel/legalizer-info-validation.mir | 6 +
.../AArch64/atomicrmw-cond-sub-clamp.ll | 142 ++
.../CodeGen/ARM/atomicrmw-cond-sub-clamp.ll | 186 +++
.../Hexagon/atomicrmw-cond-sub-clamp.ll | 355 +++++
.../LoongArch/atomicrmw-cond-sub-clamp.ll | 362 +++++
.../PowerPC/atomicrmw-cond-sub-clamp.ll | 396 +++++
.../CodeGen/RISCV/atomicrmw-cond-sub-clamp.ll | 1412 +++++++++++++++++
.../CodeGen/SPARC/atomicrmw-cond-sub-clamp.ll | 326 ++++
.../VE/Scalar/atomicrmw-cond-sub-clamp.ll | 240 +++
.../WebAssembly/atomicrmw-cond-sub-clamp.ll | 355 +++++
.../CodeGen/X86/atomicrmw-cond-sub-clamp.ll | 413 +++++
.../GlobalISelCombinerEmitter/match-table.td | 54 +-
llvm/test/TableGen/GlobalISelEmitter.td | 2 +-
mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td | 6 +-
.../test/Target/LLVMIR/Import/instructions.ll | 6 +-
mlir/test/Target/LLVMIR/llvmir.mlir | 6 +-
50 files changed, 4456 insertions(+), 71 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/atomicrmw-cond-sub-clamp.ll
create mode 100644 llvm/test/CodeGen/ARM/atomicrmw-cond-sub-clamp.ll
create mode 100644 llvm/test/CodeGen/Hexagon/atomicrmw-cond-sub-clamp.ll
create mode 100644 llvm/test/CodeGen/LoongArch/atomicrmw-cond-sub-clamp.ll
create mode 100644 llvm/test/CodeGen/PowerPC/atomicrmw-cond-sub-clamp.ll
create mode 100644 llvm/test/CodeGen/RISCV/atomicrmw-cond-sub-clamp.ll
create mode 100644 llvm/test/CodeGen/SPARC/atomicrmw-cond-sub-clamp.ll
create mode 100644 llvm/test/CodeGen/VE/Scalar/atomicrmw-cond-sub-clamp.ll
create mode 100644 llvm/test/CodeGen/WebAssembly/atomicrmw-cond-sub-clamp.ll
create mode 100644 llvm/test/CodeGen/X86/atomicrmw-cond-sub-clamp.ll
diff --git a/llvm/bindings/ocaml/llvm/llvm.ml b/llvm/bindings/ocaml/llvm/llvm.ml
index 908e6658a89f73..74ba31389b378e 100644
--- a/llvm/bindings/ocaml/llvm/llvm.ml
+++ b/llvm/bindings/ocaml/llvm/llvm.ml
@@ -296,6 +296,12 @@ module AtomicRMWBinOp = struct
| UMin
| FAdd
| FSub
+ | FMax
+ | FMin
+ | UInc_Wrap
+ | UDec_Wrap
+ | USub_Cond
+ | USub_Sat
end
module ValueKind = struct
diff --git a/llvm/bindings/ocaml/llvm/llvm.mli b/llvm/bindings/ocaml/llvm/llvm.mli
index b8a430adf6cf2d..076e651ba158fc 100644
--- a/llvm/bindings/ocaml/llvm/llvm.mli
+++ b/llvm/bindings/ocaml/llvm/llvm.mli
@@ -331,6 +331,12 @@ module AtomicRMWBinOp : sig
| UMin
| FAdd
| FSub
+ | FMax
+ | FMin
+ | UInc_Wrap
+ | UDec_Wrap
+ | USub_Cond
+ | USub_Sat
end
(** The kind of an [llvalue], the result of [classify_value v].
diff --git a/llvm/docs/GlobalISel/GenericOpcode.rst b/llvm/docs/GlobalISel/GenericOpcode.rst
index d32aeff5a69bb1..bba56d9a5c0ec2 100644
--- a/llvm/docs/GlobalISel/GenericOpcode.rst
+++ b/llvm/docs/GlobalISel/GenericOpcode.rst
@@ -863,7 +863,9 @@ operands.
G_ATOMICRMW_MIN, G_ATOMICRMW_UMAX,
G_ATOMICRMW_UMIN, G_ATOMICRMW_FADD,
G_ATOMICRMW_FSUB, G_ATOMICRMW_FMAX,
- G_ATOMICRMW_FMIN
+ G_ATOMICRMW_FMIN, G_ATOMICRMW_UINC_WRAP,
+ G_ATOMICRMW_UDEC_WRAP, G_ATOMICRMW_USUB_COND,
+ G_ATOMICRMW_USUB_SAT
Generic atomicrmw. Expects a MachineMemOperand in addition to explicit
operands.
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index 445980a18e7e93..d44db5999dbe2d 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -11241,6 +11241,8 @@ operation. The operation must be one of the following keywords:
- fmin
- uinc_wrap
- udec_wrap
+- usub_cond
+- usub_sat
For most of these operations, the type of '<value>' must be an integer
type whose bit width is a power of two greater than or equal to eight
@@ -11291,6 +11293,8 @@ operation argument:
- fmin: ``*ptr = minnum(*ptr, val)`` (match the `llvm.minnum.*`` intrinsic)
- uinc_wrap: ``*ptr = (*ptr u>= val) ? 0 : (*ptr + 1)`` (increment value with wraparound to zero when incremented above input value)
- udec_wrap: ``*ptr = ((*ptr == 0) || (*ptr u> val)) ? val : (*ptr - 1)`` (decrement with wraparound to input value when decremented below zero).
+- usub_cond: ``*ptr = (*ptr u>= val) ? *ptr - val : *ptr`` (subtract only if no unsigned overflow).
+- usub_sat: ``*ptr = (*ptr u>= val) ? *ptr - val : 0`` (subtract with clamping to zero).
Example:
diff --git a/llvm/docs/ReleaseNotes.rst b/llvm/docs/ReleaseNotes.rst
index 65fa21e517940b..fd52a92779ba35 100644
--- a/llvm/docs/ReleaseNotes.rst
+++ b/llvm/docs/ReleaseNotes.rst
@@ -53,6 +53,8 @@ Changes to the LLVM IR
* The ``x86_mmx`` IR type has been removed. It will be translated to
the standard vector type ``<1 x i64>`` in bitcode upgrade.
+* Added ``usub_cond`` and ``usub_sat`` operations to ``atomicrmw``.
+
Changes to LLVM infrastructure
------------------------------
diff --git a/llvm/include/llvm/AsmParser/LLToken.h b/llvm/include/llvm/AsmParser/LLToken.h
index db6780b70ca5aa..19029842a572a4 100644
--- a/llvm/include/llvm/AsmParser/LLToken.h
+++ b/llvm/include/llvm/AsmParser/LLToken.h
@@ -268,6 +268,8 @@ enum Kind {
kw_fmin,
kw_uinc_wrap,
kw_udec_wrap,
+ kw_usub_cond,
+ kw_usub_sat,
// Instruction Opcodes (Opcode in UIntVal).
kw_fneg,
diff --git a/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
index 4beac37a583445..49a48f1c1510c3 100644
--- a/llvm/include/llvm/Bitcode/LLVMBitCodes.h
+++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
@@ -485,7 +485,9 @@ enum RMWOperations {
RMW_FMAX = 13,
RMW_FMIN = 14,
RMW_UINC_WRAP = 15,
- RMW_UDEC_WRAP = 16
+ RMW_UDEC_WRAP = 16,
+ RMW_USUB_COND = 17,
+ RMW_USUB_SAT = 18
};
/// OverflowingBinaryOperatorOptionalFlags - Flags for serializing
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
index 56a77b8596a18b..fa3e95c87f5dbc 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
@@ -1636,6 +1636,41 @@ class MachineIRBuilder {
const DstOp &OldValRes, const SrcOp &Addr, const SrcOp &Val,
MachineMemOperand &MMO);
+ /// Build and insert `OldValRes<def> = G_ATOMICRMW_USUB_COND Addr, Val, MMO`.
+ ///
+ /// Atomically replace the value at \p Addr with the original value minus \p
+ /// Val if the original value is greater than or equal to \p Val, or leaves it
+ /// unchanged otherwise. Puts the original value from \p Addr in \p OldValRes.
+ ///
+ /// \pre setBasicBlock or setMI must have been called.
+ /// \pre \p OldValRes must be a generic virtual register.
+ /// \pre \p Addr must be a generic virtual register with pointer type.
+ /// \pre \p OldValRes, and \p Val must be generic virtual registers of the
+ /// same type.
+ ///
+ /// \return a MachineInstrBuilder for the newly created instruction.
+ MachineInstrBuilder buildAtomicRMWUSubCond(const DstOp &OldValRes,
+ const SrcOp &Addr,
+ const SrcOp &Val,
+ MachineMemOperand &MMO);
+
+ /// Build and insert `OldValRes<def> = G_ATOMICRMW_USUB_SAT Addr, Val, MMO`.
+ ///
+ /// Atomically replace the value at \p Addr with the original value minus \p
+ /// Val if the original value is greater than or equal to \p Val, or with zero
+ /// otherwise. Puts the original value from \p Addr in \p OldValRes.
+ ///
+ /// \pre setBasicBlock or setMI must have been called.
+ /// \pre \p OldValRes must be a generic virtual register.
+ /// \pre \p Addr must be a generic virtual register with pointer type.
+ /// \pre \p OldValRes, and \p Val must be generic virtual registers of the
+ /// same type.
+ ///
+ /// \return a MachineInstrBuilder for the newly created instruction.
+ MachineInstrBuilder buildAtomicRMWUSubSat(const DstOp &OldValRes,
+ const SrcOp &Addr, const SrcOp &Val,
+ MachineMemOperand &MMO);
+
/// Build and insert `G_FENCE Ordering, Scope`.
MachineInstrBuilder buildFence(unsigned Ordering, unsigned Scope);
diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h
index 86ff2628975942..c3fbb20dbc3b17 100644
--- a/llvm/include/llvm/CodeGen/ISDOpcodes.h
+++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h
@@ -1345,6 +1345,8 @@ enum NodeType {
ATOMIC_LOAD_FMIN,
ATOMIC_LOAD_UINC_WRAP,
ATOMIC_LOAD_UDEC_WRAP,
+ ATOMIC_LOAD_USUB_COND,
+ ATOMIC_LOAD_USUB_SAT,
/// Masked load and store - consecutive vector load and store operations
/// with additional mask operand that prevents memory accesses to the
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
index 88549d9c9a2858..6067b3b29ea181 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -1484,6 +1484,8 @@ class MemSDNode : public SDNode {
case ISD::ATOMIC_LOAD_FMIN:
case ISD::ATOMIC_LOAD_UINC_WRAP:
case ISD::ATOMIC_LOAD_UDEC_WRAP:
+ case ISD::ATOMIC_LOAD_USUB_COND:
+ case ISD::ATOMIC_LOAD_USUB_SAT:
case ISD::ATOMIC_LOAD:
case ISD::ATOMIC_STORE:
case ISD::MLOAD:
@@ -1550,27 +1552,29 @@ class AtomicSDNode : public MemSDNode {
// Methods to support isa and dyn_cast
static bool classof(const SDNode *N) {
- return N->getOpcode() == ISD::ATOMIC_CMP_SWAP ||
+ return N->getOpcode() == ISD::ATOMIC_CMP_SWAP ||
N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS ||
- N->getOpcode() == ISD::ATOMIC_SWAP ||
- N->getOpcode() == ISD::ATOMIC_LOAD_ADD ||
- N->getOpcode() == ISD::ATOMIC_LOAD_SUB ||
- N->getOpcode() == ISD::ATOMIC_LOAD_AND ||
- N->getOpcode() == ISD::ATOMIC_LOAD_CLR ||
- N->getOpcode() == ISD::ATOMIC_LOAD_OR ||
- N->getOpcode() == ISD::ATOMIC_LOAD_XOR ||
- N->getOpcode() == ISD::ATOMIC_LOAD_NAND ||
- N->getOpcode() == ISD::ATOMIC_LOAD_MIN ||
- N->getOpcode() == ISD::ATOMIC_LOAD_MAX ||
- N->getOpcode() == ISD::ATOMIC_LOAD_UMIN ||
- N->getOpcode() == ISD::ATOMIC_LOAD_UMAX ||
- N->getOpcode() == ISD::ATOMIC_LOAD_FADD ||
- N->getOpcode() == ISD::ATOMIC_LOAD_FSUB ||
- N->getOpcode() == ISD::ATOMIC_LOAD_FMAX ||
- N->getOpcode() == ISD::ATOMIC_LOAD_FMIN ||
+ N->getOpcode() == ISD::ATOMIC_SWAP ||
+ N->getOpcode() == ISD::ATOMIC_LOAD_ADD ||
+ N->getOpcode() == ISD::ATOMIC_LOAD_SUB ||
+ N->getOpcode() == ISD::ATOMIC_LOAD_AND ||
+ N->getOpcode() == ISD::ATOMIC_LOAD_CLR ||
+ N->getOpcode() == ISD::ATOMIC_LOAD_OR ||
+ N->getOpcode() == ISD::ATOMIC_LOAD_XOR ||
+ N->getOpcode() == ISD::ATOMIC_LOAD_NAND ||
+ N->getOpcode() == ISD::ATOMIC_LOAD_MIN ||
+ N->getOpcode() == ISD::ATOMIC_LOAD_MAX ||
+ N->getOpcode() == ISD::ATOMIC_LOAD_UMIN ||
+ N->getOpcode() == ISD::ATOMIC_LOAD_UMAX ||
+ N->getOpcode() == ISD::ATOMIC_LOAD_FADD ||
+ N->getOpcode() == ISD::ATOMIC_LOAD_FSUB ||
+ N->getOpcode() == ISD::ATOMIC_LOAD_FMAX ||
+ N->getOpcode() == ISD::ATOMIC_LOAD_FMIN ||
N->getOpcode() == ISD::ATOMIC_LOAD_UINC_WRAP ||
N->getOpcode() == ISD::ATOMIC_LOAD_UDEC_WRAP ||
- N->getOpcode() == ISD::ATOMIC_LOAD ||
+ N->getOpcode() == ISD::ATOMIC_LOAD_USUB_COND ||
+ N->getOpcode() == ISD::ATOMIC_LOAD_USUB_SAT ||
+ N->getOpcode() == ISD::ATOMIC_LOAD ||
N->getOpcode() == ISD::ATOMIC_STORE;
}
};
diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h
index dbd7d49a3e7672..41f4a51a782304 100644
--- a/llvm/include/llvm/IR/Instructions.h
+++ b/llvm/include/llvm/IR/Instructions.h
@@ -751,8 +751,16 @@ class AtomicRMWInst : public Instruction {
/// *p = ((old == 0) || (old u> v)) ? v : (old - 1)
UDecWrap,
+ /// Subtract only if result would be positive.
+ /// *p = (old u>= v) ? old - v : old
+ USubCond,
+
+ /// Subtract with clamping of negative results to zero.
+ /// *p = (old u>= v) ? old - v : 0
+ USubSat,
+
FIRST_BINOP = Xchg,
- LAST_BINOP = UDecWrap,
+ LAST_BINOP = USubSat,
BAD_BINOP
};
diff --git a/llvm/include/llvm/Support/TargetOpcodes.def b/llvm/include/llvm/Support/TargetOpcodes.def
index 9fb6de49fb2055..bf2ba5e352c3ce 100644
--- a/llvm/include/llvm/Support/TargetOpcodes.def
+++ b/llvm/include/llvm/Support/TargetOpcodes.def
@@ -414,12 +414,14 @@ HANDLE_TARGET_OPCODE(G_ATOMICRMW_FMAX)
HANDLE_TARGET_OPCODE(G_ATOMICRMW_FMIN)
HANDLE_TARGET_OPCODE(G_ATOMICRMW_UINC_WRAP)
HANDLE_TARGET_OPCODE(G_ATOMICRMW_UDEC_WRAP)
+HANDLE_TARGET_OPCODE(G_ATOMICRMW_USUB_COND)
+HANDLE_TARGET_OPCODE(G_ATOMICRMW_USUB_SAT)
// Marker for start of Generic AtomicRMW opcodes
HANDLE_TARGET_OPCODE_MARKER(GENERIC_ATOMICRMW_OP_START, G_ATOMICRMW_XCHG)
// Marker for end of Generic AtomicRMW opcodes
-HANDLE_TARGET_OPCODE_MARKER(GENERIC_ATOMICRMW_OP_END, G_ATOMICRMW_UDEC_WRAP)
+HANDLE_TARGET_OPCODE_MARKER(GENERIC_ATOMICRMW_OP_END, G_ATOMICRMW_USUB_SAT)
// Generic atomic fence
HANDLE_TARGET_OPCODE(G_FENCE)
diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td
index 36a0a087ba457c..f4934af4563d83 100644
--- a/llvm/include/llvm/Target/GenericOpcodes.td
+++ b/llvm/include/llvm/Target/GenericOpcodes.td
@@ -1311,6 +1311,8 @@ def G_ATOMICRMW_FMAX : G_ATOMICRMW_OP;
def G_ATOMICRMW_FMIN : G_ATOMICRMW_OP;
def G_ATOMICRMW_UINC_WRAP : G_ATOMICRMW_OP;
def G_ATOMICRMW_UDEC_WRAP : G_ATOMICRMW_OP;
+def G_ATOMICRMW_USUB_COND : G_ATOMICRMW_OP;
+def G_ATOMICRMW_USUB_SAT : G_ATOMICRMW_OP;
def G_FENCE : GenericInstruction {
let OutOperandList = (outs);
diff --git a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
index e9dbdef9fe9e7c..507716e68097db 100644
--- a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
+++ b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
@@ -259,6 +259,8 @@ def : GINodeEquiv<G_ATOMICRMW_FMAX, atomic_load_fmax>;
def : GINodeEquiv<G_ATOMICRMW_FMIN, atomic_load_fmin>;
def : GINodeEquiv<G_ATOMICRMW_UINC_WRAP, atomic_load_uinc_wrap>;
def : GINodeEquiv<G_ATOMICRMW_UDEC_WRAP, atomic_load_udec_wrap>;
+def : GINodeEquiv<G_ATOMICRMW_USUB_COND, atomic_load_usub_cond>;
+def : GINodeEquiv<G_ATOMICRMW_USUB_SAT, atomic_load_usub_sat>;
def : GINodeEquiv<G_FENCE, atomic_fence>;
def : GINodeEquiv<G_PREFETCH, prefetch>;
def : GINodeEquiv<G_TRAP, trap>;
diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td
index 172deffbd31771..f8f17036269f6e 100644
--- a/llvm/include/llvm/Target/TargetSelectionDAG.td
+++ b/llvm/include/llvm/Target/TargetSelectionDAG.td
@@ -753,6 +753,10 @@ def atomic_load_uinc_wrap : SDNode<"ISD::ATOMIC_LOAD_UINC_WRAP", SDTAtomic2,
[SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
def atomic_load_udec_wrap : SDNode<"ISD::ATOMIC_LOAD_UDEC_WRAP", SDTAtomic2,
[SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
+def atomic_load_usub_cond : SDNode<"ISD::ATOMIC_LOAD_USUB_COND", SDTAtomic2,
+ [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
+def atomic_load_usub_sat : SDNode<"ISD::ATOMIC_LOAD_USUB_SAT", SDTAtomic2,
+ [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
def atomic_load : SDNode<"ISD::ATOMIC_LOAD", SDTAtomicLoad,
[SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
diff --git a/llvm/lib/AsmParser/LLLexer.cpp b/llvm/lib/AsmParser/LLLexer.cpp
index 7c97f7afbe0933..a3e47da77fe776 100644
--- a/llvm/lib/AsmParser/LLLexer.cpp
+++ b/llvm/lib/AsmParser/LLLexer.cpp
@@ -704,6 +704,8 @@ lltok::Kind LLLexer::LexIdentifier() {
KEYWORD(umin); KEYWORD(fmax); KEYWORD(fmin);
KEYWORD(uinc_wrap);
KEYWORD(udec_wrap);
+ KEYWORD(usub_cond);
+ KEYWORD(usub_sat);
KEYWORD(splat);
KEYWORD(vscale);
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index f41907f0351257..d379393ff61ccd 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -8352,6 +8352,12 @@ int LLParser::parseAtomicRMW(Instruction *&Inst, PerFunctionState &PFS) {
case lltok::kw_udec_wrap:
Operation = AtomicRMWInst::UDecWrap;
break;
+ case lltok::kw_usub_cond:
+ Operation = AtomicRMWInst::USubCond;
+ break;
+ case lltok::kw_usub_sat:
+ Operation = AtomicRMWInst::USubSat;
+ break;
case lltok::kw_fadd:
Operation = AtomicRMWInst::FAdd;
IsFP = true;
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index d4dbab04e8ecdb..8005847f763750 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -1349,6 +1349,10 @@ static AtomicRMWInst::BinOp getDecodedRMWOperation(unsigned Val) {
return AtomicRMWInst::UIncWrap;
case bitc::RMW_UDEC_WRAP:
return AtomicRMWInst::UDecWrap;
+ case bitc::RMW_USUB_COND:
+ return AtomicRMWInst::USubCond;
+ case bitc::RMW_USUB_SAT:
+ return AtomicRMWInst::USubSat;
}
}
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index 03d0537291dada..655b4cf984d7c7 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -668,6 +668,10 @@ static unsigned getEncodedRMWOperation(AtomicRMWInst::BinOp Op) {
return bitc::RMW_UINC_WRAP;
case AtomicRMWInst::UDecWrap:
return bitc::RMW_UDEC_WRAP;
+ case AtomicRMWInst::USubCond:
+ return bitc::RMW_USUB_COND;
+ case AtomicRMWInst::USubSat:
+ return bitc::RMW_USUB_SAT;
}
}
diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index 39a705599f90cc..bd3fd12c30cdd5 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -888,7 +888,9 @@ static Value *performMaskedAtomicOp(AtomicRMWInst::BinOp Op,
case AtomicRMWInst::FMin:
case AtomicRMWInst::FMax:
case AtomicRMWInst::UIncWrap:
- case AtomicRMWInst::UDecWrap: {
+ case AtomicRMWInst::UDecWrap:
+ case AtomicRMWInst::USubCond:
+ case AtomicRMWInst::USubSat: {
// Finally, other ops will operate on the full value, so truncate down to
// the original size, and expand out again after doing the
// operation. Bitcasts will be inserted for FP values.
@@ -1562,6 +1564,8 @@ bool AtomicExpandImpl::isIdempotentRMW(AtomicRMWInst *RMWI) {
case AtomicRMWInst::Sub:
case AtomicRMWInst::Or:
case AtomicRMWInst::Xor:
+ case AtomicRMWInst::USubCond:
+ case AtomicRMWInst::USubSat:
return C->isZero();
case AtomicRMWInst::And:
return C->isMinusOne();
@@ -1803,6 +1807,8 @@ static ArrayRef<RTLIB::Libcall> GetRMWLibcall(AtomicRMWInst::BinOp Op) {
case AtomicRMWInst::FSub:
case AtomicRMWInst::UIncWrap:
case AtomicRMWInst::UDecWrap:
+ case AtomicRMWInst::USubCond:
+ case AtomicRMWInst::USubSat:
// No atomic libcalls are available for max/min/umax/umin.
return {};
}
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index f44af78cded46d..1b6c41f649bc9d 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -3301,6 +3301,12 @@ bool IRTranslator::translateAtomicRMW(const User &U,
case AtomicRMWInst::UDecWrap:
Opcode = TargetOpcode::G_ATOMICRMW_UDEC_WRAP;
break;
+ case AtomicRMWInst::USubCond:
+ Opcode = TargetOpcode::G_ATOMICRMW_USUB_COND;
+ break;
+ case AtomicRMWInst::USubSat:
+ Opcode = TargetOpcode::G_ATOMICRMW_USUB_SAT;
+ break;
}
MIRBuilder.buildAtomicRMW(
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 18a3b7bce104a7..ba7b6ce54ff0ab 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -8710,24 +8710,18 @@ SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl,
SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
SDValue Chain, SDValue Ptr, SDValue Val,
MachineMemOperand *MMO) {
- assert((Opcode == ISD::ATOMIC_LOAD_ADD ||
- Opcode == ISD::ATOMIC_LOAD_SUB ||
- Opcode == ISD::ATOMIC_LOAD_AND ||
- Opcode == ISD::ATOMIC_LOAD_CLR ||
- Opcode == ISD::ATOMIC_LOAD_OR ||
- Opcode == ISD::ATOMIC_LOAD_XOR ||
- Opcode == ISD::ATOMIC_LOAD_NAND ||
- Opcode == ISD::ATOMIC_LOAD_MIN ||
- Opcode == ISD::ATOMIC_LOAD_MAX ||
- Opcode == ISD::ATOMIC_LOAD_UMIN ||
- Opcode == ISD::ATOMIC_LOAD_UMAX ||
- Opcode == ISD::ATOMIC_LOAD_FADD ||
- Opcode == ISD::ATOMIC_LOAD_FSUB ||
- Opcode == ISD::ATOMIC_LOAD_FMAX ||
+ assert((Opcode == ISD::ATOMIC_LOAD_ADD || Opcode == ISD::ATOMIC_LOAD_SUB ||
+ Opcode == ISD::ATOMIC_LOAD_AND || Opcode == ISD::ATOMIC_LOAD_CLR ||
+ Opcode == ISD::ATOMIC_LOAD_OR || Opcode == ISD::ATOMIC_LOAD_XOR ||
+ Opcode == ISD::ATOMIC_LOAD_NAND || Opcode == ISD::ATOMIC_LOAD_MIN ||
+ Opcode == ISD::ATOMIC_LOAD_MAX || Opcode == ISD::ATOMIC_LOAD_UMIN ||
+ Opcode == ISD::ATOMIC_LOAD_UMAX || Opcode == ISD::ATOMIC_LOAD_FADD ||
+ Opcode == ISD::ATOMIC_LOAD_FSUB || Opcode == ISD::ATOMIC_LOAD_FMAX ||
Opcode == ISD::ATOMIC_LOAD_FMIN ||
Opcode == ISD::ATOMIC_LOAD_UINC_WRAP ||
Opcode == ISD::ATOMIC_LOAD_UDEC_WRAP ||
- Opcode == ISD::ATOMIC_SWAP ||
+ Opcode == ISD::ATOMIC_LOAD_USUB_COND ||
+ Opcode == ISD::ATOMIC_LOAD_USUB_SAT || Opcode == ISD::ATOMIC_SWAP ||
Opcode == ISD::ATOMIC_STORE) &&
"Invalid Atomic Op");
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 60dcb118542785..f33868b0f7e166 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -5111,6 +5111,12 @@ void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
case AtomicRMWInst::UDecWrap:
NT = ISD::ATOMIC_LOAD_UDEC_WRAP;
break;
+ case AtomicRMWInst::USubCond:
+ NT = ISD::ATOMIC_LOAD_USUB_COND;
+ break;
+ case AtomicRMWInst::USubSat:
+ NT = ISD::ATOMIC_LOAD_USUB_SAT;
+ break;
}
AtomicOrdering Ordering = I.getOrdering();
SyncScope::ID SSID = I.getSyncScopeID();
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
index 001f782f209fdb..fee2f3cb8b1918 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
@@ -103,6 +103,10 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
return "AtomicLoadUIncWrap";
case ISD::ATOMIC_LOAD_UDEC_WRAP:
return "AtomicLoadUDecWrap";
+ case ISD::ATOMIC_LOAD_USUB_COND:
+ return "AtomicLoadUSubCond";
+ case ISD::ATOMIC_LOAD_USUB_SAT:
+ return "AtomicLoadUSubSat";
case ISD::ATOMIC_LOAD: return "AtomicLoad";
case ISD::ATOMIC_STORE: return "AtomicStore";
case ISD::PCMARKER: return "PCMarker";
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 70a6e74b94d55c..fae67260477827 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -7896,6 +7896,8 @@ Value *OpenMPIRBuilder::emitRMWOpAsInstruction(Value *Src1, Value *Src2,
case AtomicRMWInst::FMin:
case AtomicRMWInst::UIncWrap:
case AtomicRMWInst::UDecWrap:
+ case AtomicRMWInst::USubCond:
+ case AtomicRMWInst::USubSat:
llvm_unreachable("Unsupported atomic update operation");
}
llvm_unreachable("Unsupported atomic update operation");
diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index 93fa635e9b4e17..19da1f60d424d2 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -1431,6 +1431,10 @@ StringRef AtomicRMWInst::getOperationName(BinOp Op) {
return "uinc_wrap";
case AtomicRMWInst::UDecWrap:
return "udec_wrap";
+ case AtomicRMWInst::USubCond:
+ return "usub_cond";
+ case AtomicRMWInst::USubSat:
+ return "usub_sat";
case AtomicRMWInst::BAD_BINOP:
return "<invalid operation>";
}
diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
index 50c6c263e966b5..be0243e441672e 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
@@ -5672,7 +5672,9 @@ LoongArchTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
// operations, use CmpXChg to expand.
if (AI->isFloatingPointOperation() ||
AI->getOperation() == AtomicRMWInst::UIncWrap ||
- AI->getOperation() == AtomicRMWInst::UDecWrap)
+ AI->getOperation() == AtomicRMWInst::UDecWrap ||
+ AI->getOperation() == AtomicRMWInst::USubCond ||
+ AI->getOperation() == AtomicRMWInst::USubSat)
return AtomicExpansionKind::CmpXChg;
unsigned Size = AI->getType()->getPrimitiveSizeInBits();
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 459a96eca1ff20..c394c5c2845b3f 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -18825,6 +18825,8 @@ PPCTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
switch (AI->getOperation()) {
case AtomicRMWInst::UIncWrap:
case AtomicRMWInst::UDecWrap:
+ case AtomicRMWInst::USubCond:
+ case AtomicRMWInst::USubSat:
return AtomicExpansionKind::CmpXChg;
default:
return TargetLowering::shouldExpandAtomicRMWInIR(AI);
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 66ea6423097ab2..deab9c817d9b41 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -20577,7 +20577,9 @@ RISCVTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
// forward-progress guarantee.
if (AI->isFloatingPointOperation() ||
AI->getOperation() == AtomicRMWInst::UIncWrap ||
- AI->getOperation() == AtomicRMWInst::UDecWrap)
+ AI->getOperation() == AtomicRMWInst::UDecWrap ||
+ AI->getOperation() == AtomicRMWInst::USubCond ||
+ AI->getOperation() == AtomicRMWInst::USubSat)
return AtomicExpansionKind::CmpXChg;
// Don't expand forced atomics, we want to have __sync libcalls instead.
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index da5ea50f80ce04..d75d8633340ead 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -30951,6 +30951,8 @@ X86TargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
case AtomicRMWInst::FMin:
case AtomicRMWInst::UIncWrap:
case AtomicRMWInst::UDecWrap:
+ case AtomicRMWInst::USubCond:
+ case AtomicRMWInst::USubSat:
default:
// These always require a non-trivial set of data operations on x86. We must
// use a cmpxchg loop.
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAtomicRMW.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAtomicRMW.cpp
index cba282cea72b8a..80a337db56d211 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAtomicRMW.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAtomicRMW.cpp
@@ -41,6 +41,8 @@ bool isIdempotentRMW(AtomicRMWInst& RMWI) {
case AtomicRMWInst::Sub:
case AtomicRMWInst::Or:
case AtomicRMWInst::Xor:
+ case AtomicRMWInst::USubCond:
+ case AtomicRMWInst::USubSat:
return C->isZero();
case AtomicRMWInst::And:
return C->isMinusOne();
diff --git a/llvm/lib/Transforms/Utils/LowerAtomic.cpp b/llvm/lib/Transforms/Utils/LowerAtomic.cpp
index f9bf419fb02252..ebe0ee854e6695 100644
--- a/llvm/lib/Transforms/Utils/LowerAtomic.cpp
+++ b/llvm/lib/Transforms/Utils/LowerAtomic.cpp
@@ -95,6 +95,17 @@ Value *llvm::buildAtomicRMWValue(AtomicRMWInst::BinOp Op,
Value *Or = Builder.CreateOr(CmpEq0, CmpOldGtVal);
return Builder.CreateSelect(Or, Val, Dec, "new");
}
+ case AtomicRMWInst::USubCond: {
+ Value *Cmp = Builder.CreateICmpUGE(Loaded, Val);
+ Value *Sub = Builder.CreateSub(Loaded, Val);
+ return Builder.CreateSelect(Cmp, Sub, Val, "new");
+ }
+ case AtomicRMWInst::USubSat: {
+ Constant *Zero = ConstantInt::get(Loaded->getType(), 0);
+ Value *Cmp = Builder.CreateICmpUGE(Loaded, Val);
+ Value *Sub = Builder.CreateSub(Loaded, Val);
+ return Builder.CreateSelect(Cmp, Sub, Zero, "new");
+ }
default:
llvm_unreachable("Unknown atomic op");
}
diff --git a/llvm/test/Assembler/atomic.ll b/llvm/test/Assembler/atomic.ll
index 32fe82ef2268c8..a44dcccc16bef1 100644
--- a/llvm/test/Assembler/atomic.ll
+++ b/llvm/test/Assembler/atomic.ll
@@ -42,6 +42,16 @@ define void @f(ptr %x) {
; CHECK: atomicrmw volatile udec_wrap ptr %x, i32 10 syncscope("agent") monotonic
atomicrmw volatile udec_wrap ptr %x, i32 10 syncscope("agent") monotonic
+ ; CHECK: atomicrmw volatile usub_cond ptr %x, i32 10 monotonic
+ atomicrmw volatile usub_cond ptr %x, i32 10 monotonic
+ ; CHECK: atomicrmw volatile usub_cond ptr %x, i32 10 syncscope("agent") monotonic
+ atomicrmw volatile usub_cond ptr %x, i32 10 syncscope("agent") monotonic
+
+ ; CHECK: atomicrmw volatile usub_sat ptr %x, i32 10 monotonic
+ atomicrmw volatile usub_sat ptr %x, i32 10 monotonic
+ ; CHECK: atomicrmw volatile usub_sat ptr %x, i32 10 syncscope("agent") monotonic
+ atomicrmw volatile usub_sat ptr %x, i32 10 syncscope("agent") monotonic
+
; CHECK: fence syncscope("singlethread") release
fence syncscope("singlethread") release
; CHECK: fence seq_cst
diff --git a/llvm/test/Bitcode/compatibility.ll b/llvm/test/Bitcode/compatibility.ll
index fd60c49a4be39b..e38c9783c9a8fe 100644
--- a/llvm/test/Bitcode/compatibility.ll
+++ b/llvm/test/Bitcode/compatibility.ll
@@ -906,6 +906,34 @@ define void @uinc_udec_wrap_atomics(ptr %word) {
ret void
}
+define void @usub_cond_usub_sat_atomics(ptr %word) {
+; CHECK: %atomicrmw.condsub0 = atomicrmw usub_cond ptr %word, i32 64 monotonic
+ %atomicrmw.condsub0 = atomicrmw usub_cond ptr %word, i32 64 monotonic
+
+; CHECK: %atomicrmw.condsub1 = atomicrmw usub_cond ptr %word, i32 128 seq_cst
+ %atomicrmw.condsub1 = atomicrmw usub_cond ptr %word, i32 128 seq_cst
+
+; CHECK: %atomicrmw.condsub2 = atomicrmw volatile usub_cond ptr %word, i32 128 seq_cst
+ %atomicrmw.condsub2 = atomicrmw volatile usub_cond ptr %word, i32 128 seq_cst
+
+; CHECK: %atomicrmw.condsub0.syncscope = atomicrmw usub_cond ptr %word, i32 27 syncscope("agent") monotonic
+ %atomicrmw.condsub0.syncscope = atomicrmw usub_cond ptr %word, i32 27 syncscope("agent") monotonic
+
+; CHECK: %atomicrmw.subclamp0 = atomicrmw usub_sat ptr %word, i32 99 monotonic
+ %atomicrmw.subclamp0 = atomicrmw usub_sat ptr %word, i32 99 monotonic
+
+; CHECK: %atomicrmw.subclamp1 = atomicrmw usub_sat ptr %word, i32 12 seq_cst
+ %atomicrmw.subclamp1 = atomicrmw usub_sat ptr %word, i32 12 seq_cst
+
+; CHECK: %atomicrmw.subclamp2 = atomicrmw volatile usub_sat ptr %word, i32 12 seq_cst
+ %atomicrmw.subclamp2 = atomicrmw volatile usub_sat ptr %word, i32 12 seq_cst
+
+; CHECK: %atomicrmw.subclamp0.syncscope = atomicrmw usub_sat ptr %word, i32 5 syncscope("system") monotonic
+ %atomicrmw.subclamp0.syncscope = atomicrmw usub_sat ptr %word, i32 5 syncscope("system") monotonic
+
+ ret void
+}
+
define void @pointer_atomics(ptr %word) {
; CHECK: %atomicrmw.xchg = atomicrmw xchg ptr %word, ptr null monotonic
%atomicrmw.xchg = atomicrmw xchg ptr %word, ptr null monotonic
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
index 87a415b45cca9a..3b13d9237a2738 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
@@ -265,6 +265,12 @@
# DEBUG-NEXT: G_ATOMICRMW_UDEC_WRAP (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: G_ATOMICRMW_USUB_COND (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
+# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: G_ATOMICRMW_USUB_SAT (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
+# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
# DEBUG-NEXT: G_FENCE (opcode {{[0-9]+}}): 0 type indices
# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
diff --git a/llvm/test/CodeGen/AArch64/atomicrmw-cond-sub-clamp.ll b/llvm/test/CodeGen/AArch64/atomicrmw-cond-sub-clamp.ll
new file mode 100644
index 00000000000000..53c8d2e37d16b7
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/atomicrmw-cond-sub-clamp.ll
@@ -0,0 +1,142 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-none-linux-gnu < %s | FileCheck %s
+
+define i8 @atomicrmw_usub_cond_i8(ptr %ptr, i8 %val) {
+; CHECK-LABEL: atomicrmw_usub_cond_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: .LBB0_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldaxrb w8, [x0]
+; CHECK-NEXT: sub w9, w8, w1
+; CHECK-NEXT: cmp w8, w1, uxtb
+; CHECK-NEXT: csel w9, w9, w1, hs
+; CHECK-NEXT: stlxrb w10, w9, [x0]
+; CHECK-NEXT: cbnz w10, .LBB0_1
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: mov w0, w8
+; CHECK-NEXT: ret
+ %result = atomicrmw usub_cond ptr %ptr, i8 %val seq_cst
+ ret i8 %result
+}
+
+define i16 @atomicrmw_usub_cond_i16(ptr %ptr, i16 %val) {
+; CHECK-LABEL: atomicrmw_usub_cond_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: .LBB1_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldaxrh w8, [x0]
+; CHECK-NEXT: sub w9, w8, w1
+; CHECK-NEXT: cmp w8, w1, uxth
+; CHECK-NEXT: csel w9, w9, w1, hs
+; CHECK-NEXT: stlxrh w10, w9, [x0]
+; CHECK-NEXT: cbnz w10, .LBB1_1
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: mov w0, w8
+; CHECK-NEXT: ret
+ %result = atomicrmw usub_cond ptr %ptr, i16 %val seq_cst
+ ret i16 %result
+}
+
+define i32 @atomicrmw_usub_cond_i32(ptr %ptr, i32 %val) {
+; CHECK-LABEL: atomicrmw_usub_cond_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: .LBB2_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldaxr w8, [x0]
+; CHECK-NEXT: subs w9, w8, w1
+; CHECK-NEXT: csel w9, w9, w1, hs
+; CHECK-NEXT: stlxr w10, w9, [x0]
+; CHECK-NEXT: cbnz w10, .LBB2_1
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: mov w0, w8
+; CHECK-NEXT: ret
+ %result = atomicrmw usub_cond ptr %ptr, i32 %val seq_cst
+ ret i32 %result
+}
+
+define i64 @atomicrmw_usub_cond_i64(ptr %ptr, i64 %val) {
+; CHECK-LABEL: atomicrmw_usub_cond_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov x8, x0
+; CHECK-NEXT: .LBB3_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldaxr x0, [x8]
+; CHECK-NEXT: subs x9, x0, x1
+; CHECK-NEXT: csel x9, x9, x1, hs
+; CHECK-NEXT: stlxr w10, x9, [x8]
+; CHECK-NEXT: cbnz w10, .LBB3_1
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: ret
+ %result = atomicrmw usub_cond ptr %ptr, i64 %val seq_cst
+ ret i64 %result
+}
+
+define i8 @atomicrmw_usub_sat_i8(ptr %ptr, i8 %val) {
+; CHECK-LABEL: atomicrmw_usub_sat_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: .LBB4_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldaxrb w8, [x0]
+; CHECK-NEXT: sub w9, w8, w1
+; CHECK-NEXT: cmp w8, w1, uxtb
+; CHECK-NEXT: csel w9, w9, wzr, hs
+; CHECK-NEXT: stlxrb w10, w9, [x0]
+; CHECK-NEXT: cbnz w10, .LBB4_1
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: mov w0, w8
+; CHECK-NEXT: ret
+ %result = atomicrmw usub_sat ptr %ptr, i8 %val seq_cst
+ ret i8 %result
+}
+
+define i16 @atomicrmw_usub_sat_i16(ptr %ptr, i16 %val) {
+; CHECK-LABEL: atomicrmw_usub_sat_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: .LBB5_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldaxrh w8, [x0]
+; CHECK-NEXT: sub w9, w8, w1
+; CHECK-NEXT: cmp w8, w1, uxth
+; CHECK-NEXT: csel w9, w9, wzr, hs
+; CHECK-NEXT: stlxrh w10, w9, [x0]
+; CHECK-NEXT: cbnz w10, .LBB5_1
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: mov w0, w8
+; CHECK-NEXT: ret
+ %result = atomicrmw usub_sat ptr %ptr, i16 %val seq_cst
+ ret i16 %result
+}
+
+define i32 @atomicrmw_usub_sat_i32(ptr %ptr, i32 %val) {
+; CHECK-LABEL: atomicrmw_usub_sat_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: .LBB6_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldaxr w8, [x0]
+; CHECK-NEXT: subs w9, w8, w1
+; CHECK-NEXT: csel w9, w9, wzr, hs
+; CHECK-NEXT: stlxr w10, w9, [x0]
+; CHECK-NEXT: cbnz w10, .LBB6_1
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: mov w0, w8
+; CHECK-NEXT: ret
+ %result = atomicrmw usub_sat ptr %ptr, i32 %val seq_cst
+ ret i32 %result
+}
+
+define i64 @atomicrmw_usub_sat_i64(ptr %ptr, i64 %val) {
+; CHECK-LABEL: atomicrmw_usub_sat_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov x8, x0
+; CHECK-NEXT: .LBB7_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldaxr x0, [x8]
+; CHECK-NEXT: subs x9, x0, x1
+; CHECK-NEXT: csel x9, x9, xzr, hs
+; CHECK-NEXT: stlxr w10, x9, [x8]
+; CHECK-NEXT: cbnz w10, .LBB7_1
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: ret
+ %result = atomicrmw usub_sat ptr %ptr, i64 %val seq_cst
+ ret i64 %result
+}
diff --git a/llvm/test/CodeGen/ARM/atomicrmw-cond-sub-clamp.ll b/llvm/test/CodeGen/ARM/atomicrmw-cond-sub-clamp.ll
new file mode 100644
index 00000000000000..a3449f233d9ae7
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/atomicrmw-cond-sub-clamp.ll
@@ -0,0 +1,186 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=arm-eabi -mcpu=cortex-a9 < %s | FileCheck %s
+
+define i8 @atomicrmw_usub_cond_i8(ptr %ptr, i8 %val) {
+; CHECK-LABEL: atomicrmw_usub_cond_i8:
+; CHECK: @ %bb.0:
+; CHECK-NEXT: dmb ish
+; CHECK-NEXT: .LBB0_1: @ %atomicrmw.start
+; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldrexb r12, [r0]
+; CHECK-NEXT: uxtb r3, r1
+; CHECK-NEXT: cmp r12, r3
+; CHECK-NEXT: mov r3, r1
+; CHECK-NEXT: subhs r3, r12, r3
+; CHECK-NEXT: strexb r2, r3, [r0]
+; CHECK-NEXT: cmp r2, #0
+; CHECK-NEXT: bne .LBB0_1
+; CHECK-NEXT: @ %bb.2: @ %atomicrmw.end
+; CHECK-NEXT: mov r0, r12
+; CHECK-NEXT: dmb ish
+; CHECK-NEXT: bx lr
+ %result = atomicrmw usub_cond ptr %ptr, i8 %val seq_cst
+ ret i8 %result
+}
+
+define i16 @atomicrmw_usub_cond_i16(ptr %ptr, i16 %val) {
+; CHECK-LABEL: atomicrmw_usub_cond_i16:
+; CHECK: @ %bb.0:
+; CHECK-NEXT: dmb ish
+; CHECK-NEXT: .LBB1_1: @ %atomicrmw.start
+; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldrexh r12, [r0]
+; CHECK-NEXT: uxth r3, r1
+; CHECK-NEXT: cmp r12, r3
+; CHECK-NEXT: mov r3, r1
+; CHECK-NEXT: subhs r3, r12, r3
+; CHECK-NEXT: strexh r2, r3, [r0]
+; CHECK-NEXT: cmp r2, #0
+; CHECK-NEXT: bne .LBB1_1
+; CHECK-NEXT: @ %bb.2: @ %atomicrmw.end
+; CHECK-NEXT: mov r0, r12
+; CHECK-NEXT: dmb ish
+; CHECK-NEXT: bx lr
+ %result = atomicrmw usub_cond ptr %ptr, i16 %val seq_cst
+ ret i16 %result
+}
+
+define i32 @atomicrmw_usub_cond_i32(ptr %ptr, i32 %val) {
+; CHECK-LABEL: atomicrmw_usub_cond_i32:
+; CHECK: @ %bb.0:
+; CHECK-NEXT: dmb ish
+; CHECK-NEXT: .LBB2_1: @ %atomicrmw.start
+; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldrex r12, [r0]
+; CHECK-NEXT: subs r3, r12, r1
+; CHECK-NEXT: movlo r3, r1
+; CHECK-NEXT: strex r2, r3, [r0]
+; CHECK-NEXT: cmp r2, #0
+; CHECK-NEXT: bne .LBB2_1
+; CHECK-NEXT: @ %bb.2: @ %atomicrmw.end
+; CHECK-NEXT: mov r0, r12
+; CHECK-NEXT: dmb ish
+; CHECK-NEXT: bx lr
+ %result = atomicrmw usub_cond ptr %ptr, i32 %val seq_cst
+ ret i32 %result
+}
+
+define i64 @atomicrmw_usub_cond_i64(ptr %ptr, i64 %val) {
+; CHECK-LABEL: atomicrmw_usub_cond_i64:
+; CHECK: @ %bb.0:
+; CHECK-NEXT: .save {r4, r5, r6, r7, r11, lr}
+; CHECK-NEXT: push {r4, r5, r6, r7, r11, lr}
+; CHECK-NEXT: dmb ish
+; CHECK-NEXT: .LBB3_1: @ %atomicrmw.start
+; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldrexd r4, r5, [r0]
+; CHECK-NEXT: mov r1, #0
+; CHECK-NEXT: subs r6, r4, r2
+; CHECK-NEXT: sbcs r7, r5, r3
+; CHECK-NEXT: movwhs r1, #1
+; CHECK-NEXT: cmp r1, #0
+; CHECK-NEXT: moveq r7, r3
+; CHECK-NEXT: moveq r6, r2
+; CHECK-NEXT: strexd r1, r6, r7, [r0]
+; CHECK-NEXT: cmp r1, #0
+; CHECK-NEXT: bne .LBB3_1
+; CHECK-NEXT: @ %bb.2: @ %atomicrmw.end
+; CHECK-NEXT: mov r0, r4
+; CHECK-NEXT: mov r1, r5
+; CHECK-NEXT: dmb ish
+; CHECK-NEXT: pop {r4, r5, r6, r7, r11, pc}
+ %result = atomicrmw usub_cond ptr %ptr, i64 %val seq_cst
+ ret i64 %result
+}
+
+define i8 @atomicrmw_usub_sat_i8(ptr %ptr, i8 %val) {
+; CHECK-LABEL: atomicrmw_usub_sat_i8:
+; CHECK: @ %bb.0:
+; CHECK-NEXT: dmb ish
+; CHECK-NEXT: .LBB4_1: @ %atomicrmw.start
+; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldrexb r12, [r0]
+; CHECK-NEXT: uxtb r3, r1
+; CHECK-NEXT: cmp r12, r3
+; CHECK-NEXT: mov r3, #0
+; CHECK-NEXT: subhs r3, r12, r1
+; CHECK-NEXT: strexb r2, r3, [r0]
+; CHECK-NEXT: cmp r2, #0
+; CHECK-NEXT: bne .LBB4_1
+; CHECK-NEXT: @ %bb.2: @ %atomicrmw.end
+; CHECK-NEXT: mov r0, r12
+; CHECK-NEXT: dmb ish
+; CHECK-NEXT: bx lr
+ %result = atomicrmw usub_sat ptr %ptr, i8 %val seq_cst
+ ret i8 %result
+}
+
+define i16 @atomicrmw_usub_sat_i16(ptr %ptr, i16 %val) {
+; CHECK-LABEL: atomicrmw_usub_sat_i16:
+; CHECK: @ %bb.0:
+; CHECK-NEXT: dmb ish
+; CHECK-NEXT: .LBB5_1: @ %atomicrmw.start
+; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldrexh r12, [r0]
+; CHECK-NEXT: uxth r3, r1
+; CHECK-NEXT: cmp r12, r3
+; CHECK-NEXT: mov r3, #0
+; CHECK-NEXT: subhs r3, r12, r1
+; CHECK-NEXT: strexh r2, r3, [r0]
+; CHECK-NEXT: cmp r2, #0
+; CHECK-NEXT: bne .LBB5_1
+; CHECK-NEXT: @ %bb.2: @ %atomicrmw.end
+; CHECK-NEXT: mov r0, r12
+; CHECK-NEXT: dmb ish
+; CHECK-NEXT: bx lr
+ %result = atomicrmw usub_sat ptr %ptr, i16 %val seq_cst
+ ret i16 %result
+}
+
+define i32 @atomicrmw_usub_sat_i32(ptr %ptr, i32 %val) {
+; CHECK-LABEL: atomicrmw_usub_sat_i32:
+; CHECK: @ %bb.0:
+; CHECK-NEXT: dmb ish
+; CHECK-NEXT: .LBB6_1: @ %atomicrmw.start
+; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldrex r12, [r0]
+; CHECK-NEXT: subs r3, r12, r1
+; CHECK-NEXT: movlo r3, #0
+; CHECK-NEXT: strex r2, r3, [r0]
+; CHECK-NEXT: cmp r2, #0
+; CHECK-NEXT: bne .LBB6_1
+; CHECK-NEXT: @ %bb.2: @ %atomicrmw.end
+; CHECK-NEXT: mov r0, r12
+; CHECK-NEXT: dmb ish
+; CHECK-NEXT: bx lr
+ %result = atomicrmw usub_sat ptr %ptr, i32 %val seq_cst
+ ret i32 %result
+}
+
+define i64 @atomicrmw_usub_sat_i64(ptr %ptr, i64 %val) {
+; CHECK-LABEL: atomicrmw_usub_sat_i64:
+; CHECK: @ %bb.0:
+; CHECK-NEXT: .save {r4, r5, r6, r7, r11, lr}
+; CHECK-NEXT: push {r4, r5, r6, r7, r11, lr}
+; CHECK-NEXT: dmb ish
+; CHECK-NEXT: .LBB7_1: @ %atomicrmw.start
+; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldrexd r4, r5, [r0]
+; CHECK-NEXT: subs r1, r4, r2
+; CHECK-NEXT: sbcs r7, r5, r3
+; CHECK-NEXT: mov r6, #0
+; CHECK-NEXT: movwhs r6, #1
+; CHECK-NEXT: cmp r6, #0
+; CHECK-NEXT: moveq r7, r6
+; CHECK-NEXT: movne r6, r1
+; CHECK-NEXT: strexd r1, r6, r7, [r0]
+; CHECK-NEXT: cmp r1, #0
+; CHECK-NEXT: bne .LBB7_1
+; CHECK-NEXT: @ %bb.2: @ %atomicrmw.end
+; CHECK-NEXT: mov r0, r4
+; CHECK-NEXT: mov r1, r5
+; CHECK-NEXT: dmb ish
+; CHECK-NEXT: pop {r4, r5, r6, r7, r11, pc}
+ %result = atomicrmw usub_sat ptr %ptr, i64 %val seq_cst
+ ret i64 %result
+}
diff --git a/llvm/test/CodeGen/Hexagon/atomicrmw-cond-sub-clamp.ll b/llvm/test/CodeGen/Hexagon/atomicrmw-cond-sub-clamp.ll
new file mode 100644
index 00000000000000..19a67df330e8dd
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/atomicrmw-cond-sub-clamp.ll
@@ -0,0 +1,355 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -march=hexagon < %s | FileCheck %s
+
+define i8 @atomicrmw_usub_cond_i8(ptr %ptr, i8 %val) {
+; CHECK-LABEL: atomicrmw_usub_cond_i8:
+; CHECK: .cfi_startproc
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: {
+; CHECK-NEXT: r0 = and(#24,asl(r0,#3))
+; CHECK-NEXT: r3 = and(r0,#-4)
+; CHECK-NEXT: r2 = #255
+; CHECK-NEXT: r4 = and(r1,#255)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: r2 = asl(r2,r0)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: r5 = sub(#-1,r2)
+; CHECK-NEXT: }
+; CHECK-NEXT: .p2align 4
+; CHECK-NEXT: .LBB0_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: {
+; CHECK-NEXT: r6 = memw_locked(r3)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: r2 = lsr(r6,r0)
+; CHECK-NEXT: r6 = and(r6,r5)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: r7 = and(r2,#255)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: p0 = cmp.gtu(r4,r7)
+; CHECK-NEXT: if (p0.new) r7 = add(r1,#0)
+; CHECK-NEXT: if (!p0.new) r7 = sub(r2,r1)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: r7 = and(r7,#255)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: r6 |= asl(r7,r0)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: memw_locked(r3,p0) = r6
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: if (!p0) jump:nt .LBB0_1
+; CHECK-NEXT: }
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: {
+; CHECK-NEXT: r0 = r2
+; CHECK-NEXT: jumpr r31
+; CHECK-NEXT: }
+ %result = atomicrmw usub_cond ptr %ptr, i8 %val seq_cst
+ ret i8 %result
+}
+
+define i16 @atomicrmw_usub_cond_i16(ptr %ptr, i16 %val) {
+; CHECK-LABEL: atomicrmw_usub_cond_i16:
+; CHECK: .cfi_startproc
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: {
+; CHECK-NEXT: r0 = and(#24,asl(r0,#3))
+; CHECK-NEXT: r3 = and(r0,#-4)
+; CHECK-NEXT: r2 = ##65535
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: r2 = asl(r2,r0)
+; CHECK-NEXT: r4 = zxth(r1)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: r5 = sub(#-1,r2)
+; CHECK-NEXT: }
+; CHECK-NEXT: .p2align 4
+; CHECK-NEXT: .LBB1_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: {
+; CHECK-NEXT: r6 = memw_locked(r3)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: r2 = lsr(r6,r0)
+; CHECK-NEXT: r6 = and(r6,r5)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: r7 = zxth(r2)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: p0 = cmp.gtu(r4,r7)
+; CHECK-NEXT: if (p0.new) r7 = add(r1,#0)
+; CHECK-NEXT: if (!p0.new) r7 = sub(r2,r1)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: r7 = zxth(r7)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: r6 |= asl(r7,r0)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: memw_locked(r3,p0) = r6
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: if (!p0) jump:nt .LBB1_1
+; CHECK-NEXT: }
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: {
+; CHECK-NEXT: r0 = r2
+; CHECK-NEXT: jumpr r31
+; CHECK-NEXT: }
+ %result = atomicrmw usub_cond ptr %ptr, i16 %val seq_cst
+ ret i16 %result
+}
+
+define i32 @atomicrmw_usub_cond_i32(ptr %ptr, i32 %val) {
+; CHECK-LABEL: atomicrmw_usub_cond_i32:
+; CHECK: .cfi_startproc
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: .p2align 4
+; CHECK-NEXT: .LBB2_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: {
+; CHECK-NEXT: r2 = memw_locked(r0)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: p0 = cmp.gtu(r1,r2)
+; CHECK-NEXT: if (p0.new) r3 = add(r1,#0)
+; CHECK-NEXT: if (!p0.new) r3 = sub(r2,r1)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: memw_locked(r0,p0) = r3
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: if (!p0) jump:nt .LBB2_1
+; CHECK-NEXT: }
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: {
+; CHECK-NEXT: r0 = r2
+; CHECK-NEXT: jumpr r31
+; CHECK-NEXT: }
+ %result = atomicrmw usub_cond ptr %ptr, i32 %val seq_cst
+ ret i32 %result
+}
+
+define i64 @atomicrmw_usub_cond_i64(ptr %ptr, i64 %val) {
+; CHECK-LABEL: atomicrmw_usub_cond_i64:
+; CHECK: .cfi_startproc
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: .p2align 4
+; CHECK-NEXT: .LBB3_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: {
+; CHECK-NEXT: r5:4 = memd_locked(r0)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: r7:6 = sub(r5:4,r3:2)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: p0 = cmp.gtu(r3:2,r5:4)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: r8 = mux(p0,r2,r6)
+; CHECK-NEXT: r9 = mux(p0,r3,r7)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: memd_locked(r0,p0) = r9:8
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: if (!p0) jump:nt .LBB3_1
+; CHECK-NEXT: }
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: {
+; CHECK-NEXT: r1:0 = combine(r5,r4)
+; CHECK-NEXT: jumpr r31
+; CHECK-NEXT: }
+ %result = atomicrmw usub_cond ptr %ptr, i64 %val seq_cst
+ ret i64 %result
+}
+
+define i8 @atomicrmw_usub_sat_i8(ptr %ptr, i8 %val) {
+; CHECK-LABEL: atomicrmw_usub_sat_i8:
+; CHECK: .cfi_startproc
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: {
+; CHECK-NEXT: r0 = and(#24,asl(r0,#3))
+; CHECK-NEXT: r3 = and(r0,#-4)
+; CHECK-NEXT: r2 = #255
+; CHECK-NEXT: r4 = and(r1,#255)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: r2 = asl(r2,r0)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: r5 = sub(#-1,r2)
+; CHECK-NEXT: }
+; CHECK-NEXT: .p2align 4
+; CHECK-NEXT: .LBB4_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: {
+; CHECK-NEXT: r6 = memw_locked(r3)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: r2 = lsr(r6,r0)
+; CHECK-NEXT: r6 = and(r6,r5)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: r7 = and(r2,#255)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: p0 = cmp.gtu(r4,r7)
+; CHECK-NEXT: if (p0.new) r7 = #0
+; CHECK-NEXT: if (!p0.new) r7 = sub(r2,r1)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: r7 = and(r7,#255)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: r6 |= asl(r7,r0)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: memw_locked(r3,p0) = r6
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: if (!p0) jump:nt .LBB4_1
+; CHECK-NEXT: }
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: {
+; CHECK-NEXT: r0 = r2
+; CHECK-NEXT: jumpr r31
+; CHECK-NEXT: }
+ %result = atomicrmw usub_sat ptr %ptr, i8 %val seq_cst
+ ret i8 %result
+}
+
+define i16 @atomicrmw_usub_sat_i16(ptr %ptr, i16 %val) {
+; CHECK-LABEL: atomicrmw_usub_sat_i16:
+; CHECK: .cfi_startproc
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: {
+; CHECK-NEXT: r0 = and(#24,asl(r0,#3))
+; CHECK-NEXT: r3 = and(r0,#-4)
+; CHECK-NEXT: r2 = ##65535
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: r2 = asl(r2,r0)
+; CHECK-NEXT: r4 = zxth(r1)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: r5 = sub(#-1,r2)
+; CHECK-NEXT: }
+; CHECK-NEXT: .p2align 4
+; CHECK-NEXT: .LBB5_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: {
+; CHECK-NEXT: r6 = memw_locked(r3)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: r2 = lsr(r6,r0)
+; CHECK-NEXT: r6 = and(r6,r5)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: r7 = zxth(r2)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: p0 = cmp.gtu(r4,r7)
+; CHECK-NEXT: if (p0.new) r7 = #0
+; CHECK-NEXT: if (!p0.new) r7 = sub(r2,r1)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: r7 = zxth(r7)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: r6 |= asl(r7,r0)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: memw_locked(r3,p0) = r6
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: if (!p0) jump:nt .LBB5_1
+; CHECK-NEXT: }
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: {
+; CHECK-NEXT: r0 = r2
+; CHECK-NEXT: jumpr r31
+; CHECK-NEXT: }
+ %result = atomicrmw usub_sat ptr %ptr, i16 %val seq_cst
+ ret i16 %result
+}
+
+define i32 @atomicrmw_usub_sat_i32(ptr %ptr, i32 %val) {
+; CHECK-LABEL: atomicrmw_usub_sat_i32:
+; CHECK: .cfi_startproc
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: .p2align 4
+; CHECK-NEXT: .LBB6_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: {
+; CHECK-NEXT: r2 = memw_locked(r0)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: p0 = cmp.gtu(r1,r2)
+; CHECK-NEXT: if (p0.new) r3 = #0
+; CHECK-NEXT: if (!p0.new) r3 = sub(r2,r1)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: memw_locked(r0,p0) = r3
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: if (!p0) jump:nt .LBB6_1
+; CHECK-NEXT: }
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: {
+; CHECK-NEXT: r0 = r2
+; CHECK-NEXT: jumpr r31
+; CHECK-NEXT: }
+ %result = atomicrmw usub_sat ptr %ptr, i32 %val seq_cst
+ ret i32 %result
+}
+
+define i64 @atomicrmw_usub_sat_i64(ptr %ptr, i64 %val) {
+; CHECK-LABEL: atomicrmw_usub_sat_i64:
+; CHECK: .cfi_startproc
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: {
+; CHECK-NEXT: r1 = #0
+; CHECK-NEXT: }
+; CHECK-NEXT: .p2align 4
+; CHECK-NEXT: .LBB7_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: {
+; CHECK-NEXT: r5:4 = memd_locked(r0)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: r7:6 = sub(r5:4,r3:2)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: p0 = cmp.gtu(r3:2,r5:4)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: r8 = mux(p0,r1,r6)
+; CHECK-NEXT: r9 = mux(p0,r1,r7)
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: memd_locked(r0,p0) = r9:8
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: if (!p0) jump:nt .LBB7_1
+; CHECK-NEXT: }
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: {
+; CHECK-NEXT: r1:0 = combine(r5,r4)
+; CHECK-NEXT: jumpr r31
+; CHECK-NEXT: }
+ %result = atomicrmw usub_sat ptr %ptr, i64 %val seq_cst
+ ret i64 %result
+}
diff --git a/llvm/test/CodeGen/LoongArch/atomicrmw-cond-sub-clamp.ll b/llvm/test/CodeGen/LoongArch/atomicrmw-cond-sub-clamp.ll
new file mode 100644
index 00000000000000..72d4a68c3945ac
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/atomicrmw-cond-sub-clamp.ll
@@ -0,0 +1,362 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc --mtriple=loongarch64 < %s | FileCheck --check-prefix=LA64 %s
+
+define i8 @atomicrmw_usub_cond_i8(ptr %ptr, i8 %val) {
+; LA64-LABEL: atomicrmw_usub_cond_i8:
+; LA64: # %bb.0:
+; LA64-NEXT: slli.d $a4, $a0, 3
+; LA64-NEXT: bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT: andi $a2, $a4, 24
+; LA64-NEXT: ori $a5, $zero, 255
+; LA64-NEXT: ld.w $a3, $a0, 0
+; LA64-NEXT: sll.w $a4, $a5, $a4
+; LA64-NEXT: nor $a4, $a4, $zero
+; LA64-NEXT: andi $a5, $a1, 255
+; LA64-NEXT: .p2align 4, , 16
+; LA64-NEXT: .LBB0_1: # %atomicrmw.start
+; LA64-NEXT: # =>This Loop Header: Depth=1
+; LA64-NEXT: # Child Loop BB0_3 Depth 2
+; LA64-NEXT: srl.w $a6, $a3, $a2
+; LA64-NEXT: addi.w $a7, $a3, 0
+; LA64-NEXT: andi $t0, $a6, 255
+; LA64-NEXT: sltu $t0, $t0, $a5
+; LA64-NEXT: xori $t0, $t0, 1
+; LA64-NEXT: sub.d $a6, $a6, $a1
+; LA64-NEXT: maskeqz $a6, $a6, $t0
+; LA64-NEXT: masknez $t0, $a1, $t0
+; LA64-NEXT: or $a6, $a6, $t0
+; LA64-NEXT: andi $a6, $a6, 255
+; LA64-NEXT: sll.w $a6, $a6, $a2
+; LA64-NEXT: and $a3, $a3, $a4
+; LA64-NEXT: or $a6, $a3, $a6
+; LA64-NEXT: .LBB0_3: # %atomicrmw.start
+; LA64-NEXT: # Parent Loop BB0_1 Depth=1
+; LA64-NEXT: # => This Inner Loop Header: Depth=2
+; LA64-NEXT: ll.w $a3, $a0, 0
+; LA64-NEXT: bne $a3, $a7, .LBB0_5
+; LA64-NEXT: # %bb.4: # %atomicrmw.start
+; LA64-NEXT: # in Loop: Header=BB0_3 Depth=2
+; LA64-NEXT: move $t0, $a6
+; LA64-NEXT: sc.w $t0, $a0, 0
+; LA64-NEXT: beqz $t0, .LBB0_3
+; LA64-NEXT: b .LBB0_6
+; LA64-NEXT: .LBB0_5: # %atomicrmw.start
+; LA64-NEXT: # in Loop: Header=BB0_1 Depth=1
+; LA64-NEXT: dbar 20
+; LA64-NEXT: .LBB0_6: # %atomicrmw.start
+; LA64-NEXT: # in Loop: Header=BB0_1 Depth=1
+; LA64-NEXT: bne $a3, $a7, .LBB0_1
+; LA64-NEXT: # %bb.2: # %atomicrmw.end
+; LA64-NEXT: srl.w $a0, $a3, $a2
+; LA64-NEXT: ret
+ %result = atomicrmw usub_cond ptr %ptr, i8 %val seq_cst
+ ret i8 %result
+}
+
+define i16 @atomicrmw_usub_cond_i16(ptr %ptr, i16 %val) {
+; LA64-LABEL: atomicrmw_usub_cond_i16:
+; LA64: # %bb.0:
+; LA64-NEXT: slli.d $a4, $a0, 3
+; LA64-NEXT: bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT: andi $a2, $a4, 24
+; LA64-NEXT: lu12i.w $a3, 15
+; LA64-NEXT: ori $a5, $a3, 4095
+; LA64-NEXT: ld.w $a3, $a0, 0
+; LA64-NEXT: sll.w $a4, $a5, $a4
+; LA64-NEXT: nor $a4, $a4, $zero
+; LA64-NEXT: bstrpick.d $a5, $a1, 15, 0
+; LA64-NEXT: .p2align 4, , 16
+; LA64-NEXT: .LBB1_1: # %atomicrmw.start
+; LA64-NEXT: # =>This Loop Header: Depth=1
+; LA64-NEXT: # Child Loop BB1_3 Depth 2
+; LA64-NEXT: srl.w $a6, $a3, $a2
+; LA64-NEXT: addi.w $a7, $a3, 0
+; LA64-NEXT: bstrpick.d $t0, $a6, 15, 0
+; LA64-NEXT: sltu $t0, $t0, $a5
+; LA64-NEXT: xori $t0, $t0, 1
+; LA64-NEXT: sub.d $a6, $a6, $a1
+; LA64-NEXT: maskeqz $a6, $a6, $t0
+; LA64-NEXT: masknez $t0, $a1, $t0
+; LA64-NEXT: or $a6, $a6, $t0
+; LA64-NEXT: bstrpick.d $a6, $a6, 15, 0
+; LA64-NEXT: sll.w $a6, $a6, $a2
+; LA64-NEXT: and $a3, $a3, $a4
+; LA64-NEXT: or $a6, $a3, $a6
+; LA64-NEXT: .LBB1_3: # %atomicrmw.start
+; LA64-NEXT: # Parent Loop BB1_1 Depth=1
+; LA64-NEXT: # => This Inner Loop Header: Depth=2
+; LA64-NEXT: ll.w $a3, $a0, 0
+; LA64-NEXT: bne $a3, $a7, .LBB1_5
+; LA64-NEXT: # %bb.4: # %atomicrmw.start
+; LA64-NEXT: # in Loop: Header=BB1_3 Depth=2
+; LA64-NEXT: move $t0, $a6
+; LA64-NEXT: sc.w $t0, $a0, 0
+; LA64-NEXT: beqz $t0, .LBB1_3
+; LA64-NEXT: b .LBB1_6
+; LA64-NEXT: .LBB1_5: # %atomicrmw.start
+; LA64-NEXT: # in Loop: Header=BB1_1 Depth=1
+; LA64-NEXT: dbar 20
+; LA64-NEXT: .LBB1_6: # %atomicrmw.start
+; LA64-NEXT: # in Loop: Header=BB1_1 Depth=1
+; LA64-NEXT: bne $a3, $a7, .LBB1_1
+; LA64-NEXT: # %bb.2: # %atomicrmw.end
+; LA64-NEXT: srl.w $a0, $a3, $a2
+; LA64-NEXT: ret
+ %result = atomicrmw usub_cond ptr %ptr, i16 %val seq_cst
+ ret i16 %result
+}
+
+define i32 @atomicrmw_usub_cond_i32(ptr %ptr, i32 %val) {
+; LA64-LABEL: atomicrmw_usub_cond_i32:
+; LA64: # %bb.0:
+; LA64-NEXT: ld.w $a2, $a0, 0
+; LA64-NEXT: addi.w $a3, $a1, 0
+; LA64-NEXT: .p2align 4, , 16
+; LA64-NEXT: .LBB2_1: # %atomicrmw.start
+; LA64-NEXT: # =>This Loop Header: Depth=1
+; LA64-NEXT: # Child Loop BB2_3 Depth 2
+; LA64-NEXT: addi.w $a4, $a2, 0
+; LA64-NEXT: sltu $a5, $a4, $a3
+; LA64-NEXT: xori $a5, $a5, 1
+; LA64-NEXT: sub.d $a2, $a2, $a1
+; LA64-NEXT: maskeqz $a2, $a2, $a5
+; LA64-NEXT: masknez $a5, $a1, $a5
+; LA64-NEXT: or $a5, $a2, $a5
+; LA64-NEXT: .LBB2_3: # %atomicrmw.start
+; LA64-NEXT: # Parent Loop BB2_1 Depth=1
+; LA64-NEXT: # => This Inner Loop Header: Depth=2
+; LA64-NEXT: ll.w $a2, $a0, 0
+; LA64-NEXT: bne $a2, $a4, .LBB2_5
+; LA64-NEXT: # %bb.4: # %atomicrmw.start
+; LA64-NEXT: # in Loop: Header=BB2_3 Depth=2
+; LA64-NEXT: move $a6, $a5
+; LA64-NEXT: sc.w $a6, $a0, 0
+; LA64-NEXT: beqz $a6, .LBB2_3
+; LA64-NEXT: b .LBB2_6
+; LA64-NEXT: .LBB2_5: # %atomicrmw.start
+; LA64-NEXT: # in Loop: Header=BB2_1 Depth=1
+; LA64-NEXT: dbar 20
+; LA64-NEXT: .LBB2_6: # %atomicrmw.start
+; LA64-NEXT: # in Loop: Header=BB2_1 Depth=1
+; LA64-NEXT: bne $a2, $a4, .LBB2_1
+; LA64-NEXT: # %bb.2: # %atomicrmw.end
+; LA64-NEXT: move $a0, $a2
+; LA64-NEXT: ret
+ %result = atomicrmw usub_cond ptr %ptr, i32 %val seq_cst
+ ret i32 %result
+}
+
+define i64 @atomicrmw_usub_cond_i64(ptr %ptr, i64 %val) {
+; LA64-LABEL: atomicrmw_usub_cond_i64:
+; LA64: # %bb.0:
+; LA64-NEXT: ld.d $a2, $a0, 0
+; LA64-NEXT: .p2align 4, , 16
+; LA64-NEXT: .LBB3_1: # %atomicrmw.start
+; LA64-NEXT: # =>This Loop Header: Depth=1
+; LA64-NEXT: # Child Loop BB3_3 Depth 2
+; LA64-NEXT: move $a3, $a2
+; LA64-NEXT: sltu $a2, $a2, $a1
+; LA64-NEXT: xori $a2, $a2, 1
+; LA64-NEXT: sub.d $a4, $a3, $a1
+; LA64-NEXT: maskeqz $a4, $a4, $a2
+; LA64-NEXT: masknez $a2, $a1, $a2
+; LA64-NEXT: or $a4, $a4, $a2
+; LA64-NEXT: .LBB3_3: # %atomicrmw.start
+; LA64-NEXT: # Parent Loop BB3_1 Depth=1
+; LA64-NEXT: # => This Inner Loop Header: Depth=2
+; LA64-NEXT: ll.d $a2, $a0, 0
+; LA64-NEXT: bne $a2, $a3, .LBB3_5
+; LA64-NEXT: # %bb.4: # %atomicrmw.start
+; LA64-NEXT: # in Loop: Header=BB3_3 Depth=2
+; LA64-NEXT: move $a5, $a4
+; LA64-NEXT: sc.d $a5, $a0, 0
+; LA64-NEXT: beqz $a5, .LBB3_3
+; LA64-NEXT: b .LBB3_6
+; LA64-NEXT: .LBB3_5: # %atomicrmw.start
+; LA64-NEXT: # in Loop: Header=BB3_1 Depth=1
+; LA64-NEXT: dbar 20
+; LA64-NEXT: .LBB3_6: # %atomicrmw.start
+; LA64-NEXT: # in Loop: Header=BB3_1 Depth=1
+; LA64-NEXT: bne $a2, $a3, .LBB3_1
+; LA64-NEXT: # %bb.2: # %atomicrmw.end
+; LA64-NEXT: move $a0, $a2
+; LA64-NEXT: ret
+ %result = atomicrmw usub_cond ptr %ptr, i64 %val seq_cst
+ ret i64 %result
+}
+
+define i8 @atomicrmw_usub_sat_i8(ptr %ptr, i8 %val) {
+; LA64-LABEL: atomicrmw_usub_sat_i8:
+; LA64: # %bb.0:
+; LA64-NEXT: slli.d $a4, $a0, 3
+; LA64-NEXT: bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT: andi $a2, $a4, 24
+; LA64-NEXT: ori $a5, $zero, 255
+; LA64-NEXT: ld.w $a3, $a0, 0
+; LA64-NEXT: sll.w $a4, $a5, $a4
+; LA64-NEXT: nor $a4, $a4, $zero
+; LA64-NEXT: andi $a5, $a1, 255
+; LA64-NEXT: .p2align 4, , 16
+; LA64-NEXT: .LBB4_1: # %atomicrmw.start
+; LA64-NEXT: # =>This Loop Header: Depth=1
+; LA64-NEXT: # Child Loop BB4_3 Depth 2
+; LA64-NEXT: srl.w $a6, $a3, $a2
+; LA64-NEXT: addi.w $a7, $a3, 0
+; LA64-NEXT: andi $t0, $a6, 255
+; LA64-NEXT: sltu $t0, $t0, $a5
+; LA64-NEXT: xori $t0, $t0, 1
+; LA64-NEXT: sub.d $a6, $a6, $a1
+; LA64-NEXT: maskeqz $a6, $a6, $t0
+; LA64-NEXT: andi $a6, $a6, 255
+; LA64-NEXT: sll.w $a6, $a6, $a2
+; LA64-NEXT: and $a3, $a3, $a4
+; LA64-NEXT: or $a6, $a3, $a6
+; LA64-NEXT: .LBB4_3: # %atomicrmw.start
+; LA64-NEXT: # Parent Loop BB4_1 Depth=1
+; LA64-NEXT: # => This Inner Loop Header: Depth=2
+; LA64-NEXT: ll.w $a3, $a0, 0
+; LA64-NEXT: bne $a3, $a7, .LBB4_5
+; LA64-NEXT: # %bb.4: # %atomicrmw.start
+; LA64-NEXT: # in Loop: Header=BB4_3 Depth=2
+; LA64-NEXT: move $t0, $a6
+; LA64-NEXT: sc.w $t0, $a0, 0
+; LA64-NEXT: beqz $t0, .LBB4_3
+; LA64-NEXT: b .LBB4_6
+; LA64-NEXT: .LBB4_5: # %atomicrmw.start
+; LA64-NEXT: # in Loop: Header=BB4_1 Depth=1
+; LA64-NEXT: dbar 20
+; LA64-NEXT: .LBB4_6: # %atomicrmw.start
+; LA64-NEXT: # in Loop: Header=BB4_1 Depth=1
+; LA64-NEXT: bne $a3, $a7, .LBB4_1
+; LA64-NEXT: # %bb.2: # %atomicrmw.end
+; LA64-NEXT: srl.w $a0, $a3, $a2
+; LA64-NEXT: ret
+ %result = atomicrmw usub_sat ptr %ptr, i8 %val seq_cst
+ ret i8 %result
+}
+
+define i16 @atomicrmw_usub_sat_i16(ptr %ptr, i16 %val) {
+; LA64-LABEL: atomicrmw_usub_sat_i16:
+; LA64: # %bb.0:
+; LA64-NEXT: slli.d $a4, $a0, 3
+; LA64-NEXT: bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT: andi $a2, $a4, 24
+; LA64-NEXT: lu12i.w $a3, 15
+; LA64-NEXT: ori $a5, $a3, 4095
+; LA64-NEXT: ld.w $a3, $a0, 0
+; LA64-NEXT: sll.w $a4, $a5, $a4
+; LA64-NEXT: nor $a4, $a4, $zero
+; LA64-NEXT: bstrpick.d $a5, $a1, 15, 0
+; LA64-NEXT: .p2align 4, , 16
+; LA64-NEXT: .LBB5_1: # %atomicrmw.start
+; LA64-NEXT: # =>This Loop Header: Depth=1
+; LA64-NEXT: # Child Loop BB5_3 Depth 2
+; LA64-NEXT: srl.w $a6, $a3, $a2
+; LA64-NEXT: addi.w $a7, $a3, 0
+; LA64-NEXT: bstrpick.d $t0, $a6, 15, 0
+; LA64-NEXT: sltu $t0, $t0, $a5
+; LA64-NEXT: xori $t0, $t0, 1
+; LA64-NEXT: sub.d $a6, $a6, $a1
+; LA64-NEXT: maskeqz $a6, $a6, $t0
+; LA64-NEXT: bstrpick.d $a6, $a6, 15, 0
+; LA64-NEXT: sll.w $a6, $a6, $a2
+; LA64-NEXT: and $a3, $a3, $a4
+; LA64-NEXT: or $a6, $a3, $a6
+; LA64-NEXT: .LBB5_3: # %atomicrmw.start
+; LA64-NEXT: # Parent Loop BB5_1 Depth=1
+; LA64-NEXT: # => This Inner Loop Header: Depth=2
+; LA64-NEXT: ll.w $a3, $a0, 0
+; LA64-NEXT: bne $a3, $a7, .LBB5_5
+; LA64-NEXT: # %bb.4: # %atomicrmw.start
+; LA64-NEXT: # in Loop: Header=BB5_3 Depth=2
+; LA64-NEXT: move $t0, $a6
+; LA64-NEXT: sc.w $t0, $a0, 0
+; LA64-NEXT: beqz $t0, .LBB5_3
+; LA64-NEXT: b .LBB5_6
+; LA64-NEXT: .LBB5_5: # %atomicrmw.start
+; LA64-NEXT: # in Loop: Header=BB5_1 Depth=1
+; LA64-NEXT: dbar 20
+; LA64-NEXT: .LBB5_6: # %atomicrmw.start
+; LA64-NEXT: # in Loop: Header=BB5_1 Depth=1
+; LA64-NEXT: bne $a3, $a7, .LBB5_1
+; LA64-NEXT: # %bb.2: # %atomicrmw.end
+; LA64-NEXT: srl.w $a0, $a3, $a2
+; LA64-NEXT: ret
+ %result = atomicrmw usub_sat ptr %ptr, i16 %val seq_cst
+ ret i16 %result
+}
+
+define i32 @atomicrmw_usub_sat_i32(ptr %ptr, i32 %val) {
+; LA64-LABEL: atomicrmw_usub_sat_i32:
+; LA64: # %bb.0:
+; LA64-NEXT: ld.w $a2, $a0, 0
+; LA64-NEXT: addi.w $a3, $a1, 0
+; LA64-NEXT: .p2align 4, , 16
+; LA64-NEXT: .LBB6_1: # %atomicrmw.start
+; LA64-NEXT: # =>This Loop Header: Depth=1
+; LA64-NEXT: # Child Loop BB6_3 Depth 2
+; LA64-NEXT: addi.w $a4, $a2, 0
+; LA64-NEXT: sltu $a5, $a4, $a3
+; LA64-NEXT: xori $a5, $a5, 1
+; LA64-NEXT: sub.d $a2, $a2, $a1
+; LA64-NEXT: maskeqz $a5, $a2, $a5
+; LA64-NEXT: .LBB6_3: # %atomicrmw.start
+; LA64-NEXT: # Parent Loop BB6_1 Depth=1
+; LA64-NEXT: # => This Inner Loop Header: Depth=2
+; LA64-NEXT: ll.w $a2, $a0, 0
+; LA64-NEXT: bne $a2, $a4, .LBB6_5
+; LA64-NEXT: # %bb.4: # %atomicrmw.start
+; LA64-NEXT: # in Loop: Header=BB6_3 Depth=2
+; LA64-NEXT: move $a6, $a5
+; LA64-NEXT: sc.w $a6, $a0, 0
+; LA64-NEXT: beqz $a6, .LBB6_3
+; LA64-NEXT: b .LBB6_6
+; LA64-NEXT: .LBB6_5: # %atomicrmw.start
+; LA64-NEXT: # in Loop: Header=BB6_1 Depth=1
+; LA64-NEXT: dbar 20
+; LA64-NEXT: .LBB6_6: # %atomicrmw.start
+; LA64-NEXT: # in Loop: Header=BB6_1 Depth=1
+; LA64-NEXT: bne $a2, $a4, .LBB6_1
+; LA64-NEXT: # %bb.2: # %atomicrmw.end
+; LA64-NEXT: move $a0, $a2
+; LA64-NEXT: ret
+ %result = atomicrmw usub_sat ptr %ptr, i32 %val seq_cst
+ ret i32 %result
+}
+
+define i64 @atomicrmw_usub_sat_i64(ptr %ptr, i64 %val) {
+; LA64-LABEL: atomicrmw_usub_sat_i64:
+; LA64: # %bb.0:
+; LA64-NEXT: ld.d $a2, $a0, 0
+; LA64-NEXT: .p2align 4, , 16
+; LA64-NEXT: .LBB7_1: # %atomicrmw.start
+; LA64-NEXT: # =>This Loop Header: Depth=1
+; LA64-NEXT: # Child Loop BB7_3 Depth 2
+; LA64-NEXT: move $a3, $a2
+; LA64-NEXT: sltu $a2, $a2, $a1
+; LA64-NEXT: xori $a2, $a2, 1
+; LA64-NEXT: sub.d $a4, $a3, $a1
+; LA64-NEXT: maskeqz $a4, $a4, $a2
+; LA64-NEXT: .LBB7_3: # %atomicrmw.start
+; LA64-NEXT: # Parent Loop BB7_1 Depth=1
+; LA64-NEXT: # => This Inner Loop Header: Depth=2
+; LA64-NEXT: ll.d $a2, $a0, 0
+; LA64-NEXT: bne $a2, $a3, .LBB7_5
+; LA64-NEXT: # %bb.4: # %atomicrmw.start
+; LA64-NEXT: # in Loop: Header=BB7_3 Depth=2
+; LA64-NEXT: move $a5, $a4
+; LA64-NEXT: sc.d $a5, $a0, 0
+; LA64-NEXT: beqz $a5, .LBB7_3
+; LA64-NEXT: b .LBB7_6
+; LA64-NEXT: .LBB7_5: # %atomicrmw.start
+; LA64-NEXT: # in Loop: Header=BB7_1 Depth=1
+; LA64-NEXT: dbar 20
+; LA64-NEXT: .LBB7_6: # %atomicrmw.start
+; LA64-NEXT: # in Loop: Header=BB7_1 Depth=1
+; LA64-NEXT: bne $a2, $a3, .LBB7_1
+; LA64-NEXT: # %bb.2: # %atomicrmw.end
+; LA64-NEXT: move $a0, $a2
+; LA64-NEXT: ret
+ %result = atomicrmw usub_sat ptr %ptr, i64 %val seq_cst
+ ret i64 %result
+}
diff --git a/llvm/test/CodeGen/PowerPC/atomicrmw-cond-sub-clamp.ll b/llvm/test/CodeGen/PowerPC/atomicrmw-cond-sub-clamp.ll
new file mode 100644
index 00000000000000..e9d598092411f4
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/atomicrmw-cond-sub-clamp.ll
@@ -0,0 +1,396 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=powerpc64-unknown-linux-gnu < %s | FileCheck %s
+
+define i8 @atomicrmw_usub_cond_i8(ptr %ptr, i8 %val) {
+; CHECK-LABEL: atomicrmw_usub_cond_i8:
+; CHECK: # %bb.0:
+; CHECK-NEXT: sync
+; CHECK-NEXT: mr 5, 3
+; CHECK-NEXT: rlwinm 7, 5, 3, 27, 28
+; CHECK-NEXT: lbz 3, 0(3)
+; CHECK-NEXT: xori 7, 7, 24
+; CHECK-NEXT: li 8, 255
+; CHECK-NEXT: clrlwi 6, 4, 24
+; CHECK-NEXT: rldicr 5, 5, 0, 61
+; CHECK-NEXT: slw 8, 8, 7
+; CHECK-NEXT: b .LBB0_2
+; CHECK-NEXT: .LBB0_1: # %atomicrmw.start
+; CHECK-NEXT: #
+; CHECK-NEXT: srw 3, 11, 7
+; CHECK-NEXT: cmplw 3, 9
+; CHECK-NEXT: beq 0, .LBB0_8
+; CHECK-NEXT: .LBB0_2: # %atomicrmw.start
+; CHECK-NEXT: # =>This Loop Header: Depth=1
+; CHECK-NEXT: # Child Loop BB0_6 Depth 2
+; CHECK-NEXT: clrlwi 9, 3, 24
+; CHECK-NEXT: cmplw 9, 6
+; CHECK-NEXT: bge 0, .LBB0_4
+; CHECK-NEXT: # %bb.3: # %atomicrmw.start
+; CHECK-NEXT: #
+; CHECK-NEXT: mr 3, 4
+; CHECK-NEXT: b .LBB0_5
+; CHECK-NEXT: .LBB0_4:
+; CHECK-NEXT: sub 3, 3, 4
+; CHECK-NEXT: .LBB0_5: # %atomicrmw.start
+; CHECK-NEXT: #
+; CHECK-NEXT: slw 3, 3, 7
+; CHECK-NEXT: slw 10, 9, 7
+; CHECK-NEXT: and 3, 3, 8
+; CHECK-NEXT: and 10, 10, 8
+; CHECK-NEXT: .LBB0_6: # %atomicrmw.start
+; CHECK-NEXT: # Parent Loop BB0_2 Depth=1
+; CHECK-NEXT: # => This Inner Loop Header: Depth=2
+; CHECK-NEXT: lwarx 12, 0, 5
+; CHECK-NEXT: and 11, 12, 8
+; CHECK-NEXT: cmpw 11, 10
+; CHECK-NEXT: bne 0, .LBB0_1
+; CHECK-NEXT: # %bb.7: # %atomicrmw.start
+; CHECK-NEXT: #
+; CHECK-NEXT: andc 12, 12, 8
+; CHECK-NEXT: or 12, 12, 3
+; CHECK-NEXT: stwcx. 12, 0, 5
+; CHECK-NEXT: bne 0, .LBB0_6
+; CHECK-NEXT: b .LBB0_1
+; CHECK-NEXT: .LBB0_8: # %atomicrmw.end
+; CHECK-NEXT: lwsync
+; CHECK-NEXT: blr
+ %result = atomicrmw usub_cond ptr %ptr, i8 %val seq_cst
+ ret i8 %result
+}
+
+define i16 @atomicrmw_usub_cond_i16(ptr %ptr, i16 %val) {
+; CHECK-LABEL: atomicrmw_usub_cond_i16:
+; CHECK: # %bb.0:
+; CHECK-NEXT: sync
+; CHECK-NEXT: mr 5, 3
+; CHECK-NEXT: li 8, 0
+; CHECK-NEXT: lhz 3, 0(3)
+; CHECK-NEXT: rlwinm 7, 5, 3, 27, 27
+; CHECK-NEXT: xori 7, 7, 16
+; CHECK-NEXT: ori 8, 8, 65535
+; CHECK-NEXT: clrlwi 6, 4, 16
+; CHECK-NEXT: rldicr 5, 5, 0, 61
+; CHECK-NEXT: slw 8, 8, 7
+; CHECK-NEXT: b .LBB1_2
+; CHECK-NEXT: .LBB1_1: # %atomicrmw.start
+; CHECK-NEXT: #
+; CHECK-NEXT: srw 3, 11, 7
+; CHECK-NEXT: cmplw 3, 9
+; CHECK-NEXT: beq 0, .LBB1_8
+; CHECK-NEXT: .LBB1_2: # %atomicrmw.start
+; CHECK-NEXT: # =>This Loop Header: Depth=1
+; CHECK-NEXT: # Child Loop BB1_6 Depth 2
+; CHECK-NEXT: clrlwi 9, 3, 16
+; CHECK-NEXT: cmplw 9, 6
+; CHECK-NEXT: bge 0, .LBB1_4
+; CHECK-NEXT: # %bb.3: # %atomicrmw.start
+; CHECK-NEXT: #
+; CHECK-NEXT: mr 3, 4
+; CHECK-NEXT: b .LBB1_5
+; CHECK-NEXT: .LBB1_4:
+; CHECK-NEXT: sub 3, 3, 4
+; CHECK-NEXT: .LBB1_5: # %atomicrmw.start
+; CHECK-NEXT: #
+; CHECK-NEXT: slw 3, 3, 7
+; CHECK-NEXT: slw 10, 9, 7
+; CHECK-NEXT: and 3, 3, 8
+; CHECK-NEXT: and 10, 10, 8
+; CHECK-NEXT: .LBB1_6: # %atomicrmw.start
+; CHECK-NEXT: # Parent Loop BB1_2 Depth=1
+; CHECK-NEXT: # => This Inner Loop Header: Depth=2
+; CHECK-NEXT: lwarx 12, 0, 5
+; CHECK-NEXT: and 11, 12, 8
+; CHECK-NEXT: cmpw 11, 10
+; CHECK-NEXT: bne 0, .LBB1_1
+; CHECK-NEXT: # %bb.7: # %atomicrmw.start
+; CHECK-NEXT: #
+; CHECK-NEXT: andc 12, 12, 8
+; CHECK-NEXT: or 12, 12, 3
+; CHECK-NEXT: stwcx. 12, 0, 5
+; CHECK-NEXT: bne 0, .LBB1_6
+; CHECK-NEXT: b .LBB1_1
+; CHECK-NEXT: .LBB1_8: # %atomicrmw.end
+; CHECK-NEXT: lwsync
+; CHECK-NEXT: blr
+ %result = atomicrmw usub_cond ptr %ptr, i16 %val seq_cst
+ ret i16 %result
+}
+
+define i32 @atomicrmw_usub_cond_i32(ptr %ptr, i32 %val) {
+; CHECK-LABEL: atomicrmw_usub_cond_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: sync
+; CHECK-NEXT: lwz 6, 0(3)
+; CHECK-NEXT: b .LBB2_2
+; CHECK-NEXT: .LBB2_1: # %atomicrmw.start
+; CHECK-NEXT: #
+; CHECK-NEXT: cmplw 5, 6
+; CHECK-NEXT: mr 6, 5
+; CHECK-NEXT: beq 0, .LBB2_7
+; CHECK-NEXT: .LBB2_2: # %atomicrmw.start
+; CHECK-NEXT: # =>This Loop Header: Depth=1
+; CHECK-NEXT: # Child Loop BB2_5 Depth 2
+; CHECK-NEXT: cmplw 6, 4
+; CHECK-NEXT: bge 0, .LBB2_4
+; CHECK-NEXT: # %bb.3: # %atomicrmw.start
+; CHECK-NEXT: #
+; CHECK-NEXT: mr 7, 4
+; CHECK-NEXT: b .LBB2_5
+; CHECK-NEXT: .LBB2_4:
+; CHECK-NEXT: sub 7, 6, 4
+; CHECK-NEXT: .LBB2_5: # %atomicrmw.start
+; CHECK-NEXT: # Parent Loop BB2_2 Depth=1
+; CHECK-NEXT: # => This Inner Loop Header: Depth=2
+; CHECK-NEXT: lwarx 5, 0, 3
+; CHECK-NEXT: cmpw 5, 6
+; CHECK-NEXT: bne 0, .LBB2_1
+; CHECK-NEXT: # %bb.6: # %atomicrmw.start
+; CHECK-NEXT: #
+; CHECK-NEXT: stwcx. 7, 0, 3
+; CHECK-NEXT: bne 0, .LBB2_5
+; CHECK-NEXT: b .LBB2_1
+; CHECK-NEXT: .LBB2_7: # %atomicrmw.end
+; CHECK-NEXT: mr 3, 5
+; CHECK-NEXT: lwsync
+; CHECK-NEXT: blr
+ %result = atomicrmw usub_cond ptr %ptr, i32 %val seq_cst
+ ret i32 %result
+}
+
+define i64 @atomicrmw_usub_cond_i64(ptr %ptr, i64 %val) {
+; CHECK-LABEL: atomicrmw_usub_cond_i64:
+; CHECK: # %bb.0:
+; CHECK-NEXT: sync
+; CHECK-NEXT: ld 6, 0(3)
+; CHECK-NEXT: b .LBB3_2
+; CHECK-NEXT: .LBB3_1: # %atomicrmw.start
+; CHECK-NEXT: #
+; CHECK-NEXT: cmpld 5, 6
+; CHECK-NEXT: mr 6, 5
+; CHECK-NEXT: beq 0, .LBB3_7
+; CHECK-NEXT: .LBB3_2: # %atomicrmw.start
+; CHECK-NEXT: # =>This Loop Header: Depth=1
+; CHECK-NEXT: # Child Loop BB3_5 Depth 2
+; CHECK-NEXT: cmpld 6, 4
+; CHECK-NEXT: bge 0, .LBB3_4
+; CHECK-NEXT: # %bb.3: # %atomicrmw.start
+; CHECK-NEXT: #
+; CHECK-NEXT: mr 7, 4
+; CHECK-NEXT: b .LBB3_5
+; CHECK-NEXT: .LBB3_4:
+; CHECK-NEXT: sub 7, 6, 4
+; CHECK-NEXT: .LBB3_5: # %atomicrmw.start
+; CHECK-NEXT: # Parent Loop BB3_2 Depth=1
+; CHECK-NEXT: # => This Inner Loop Header: Depth=2
+; CHECK-NEXT: ldarx 5, 0, 3
+; CHECK-NEXT: cmpd 5, 6
+; CHECK-NEXT: bne 0, .LBB3_1
+; CHECK-NEXT: # %bb.6: # %atomicrmw.start
+; CHECK-NEXT: #
+; CHECK-NEXT: stdcx. 7, 0, 3
+; CHECK-NEXT: bne 0, .LBB3_5
+; CHECK-NEXT: b .LBB3_1
+; CHECK-NEXT: .LBB3_7: # %atomicrmw.end
+; CHECK-NEXT: mr 3, 5
+; CHECK-NEXT: lwsync
+; CHECK-NEXT: blr
+ %result = atomicrmw usub_cond ptr %ptr, i64 %val seq_cst
+ ret i64 %result
+}
+
+define i8 @atomicrmw_usub_sat_i8(ptr %ptr, i8 %val) {
+; CHECK-LABEL: atomicrmw_usub_sat_i8:
+; CHECK: # %bb.0:
+; CHECK-NEXT: sync
+; CHECK-NEXT: mr 5, 3
+; CHECK-NEXT: rlwinm 7, 5, 3, 27, 28
+; CHECK-NEXT: lbz 3, 0(3)
+; CHECK-NEXT: xori 7, 7, 24
+; CHECK-NEXT: li 8, 255
+; CHECK-NEXT: clrlwi 6, 4, 24
+; CHECK-NEXT: rldicr 5, 5, 0, 61
+; CHECK-NEXT: slw 8, 8, 7
+; CHECK-NEXT: b .LBB4_2
+; CHECK-NEXT: .LBB4_1: # %atomicrmw.start
+; CHECK-NEXT: #
+; CHECK-NEXT: srw 3, 11, 7
+; CHECK-NEXT: cmplw 3, 9
+; CHECK-NEXT: beq 0, .LBB4_8
+; CHECK-NEXT: .LBB4_2: # %atomicrmw.start
+; CHECK-NEXT: # =>This Loop Header: Depth=1
+; CHECK-NEXT: # Child Loop BB4_6 Depth 2
+; CHECK-NEXT: clrlwi 9, 3, 24
+; CHECK-NEXT: cmplw 9, 6
+; CHECK-NEXT: bge 0, .LBB4_4
+; CHECK-NEXT: # %bb.3: # %atomicrmw.start
+; CHECK-NEXT: #
+; CHECK-NEXT: li 3, 0
+; CHECK-NEXT: b .LBB4_5
+; CHECK-NEXT: .LBB4_4:
+; CHECK-NEXT: sub 3, 3, 4
+; CHECK-NEXT: .LBB4_5: # %atomicrmw.start
+; CHECK-NEXT: #
+; CHECK-NEXT: slw 3, 3, 7
+; CHECK-NEXT: slw 10, 9, 7
+; CHECK-NEXT: and 3, 3, 8
+; CHECK-NEXT: and 10, 10, 8
+; CHECK-NEXT: .LBB4_6: # %atomicrmw.start
+; CHECK-NEXT: # Parent Loop BB4_2 Depth=1
+; CHECK-NEXT: # => This Inner Loop Header: Depth=2
+; CHECK-NEXT: lwarx 12, 0, 5
+; CHECK-NEXT: and 11, 12, 8
+; CHECK-NEXT: cmpw 11, 10
+; CHECK-NEXT: bne 0, .LBB4_1
+; CHECK-NEXT: # %bb.7: # %atomicrmw.start
+; CHECK-NEXT: #
+; CHECK-NEXT: andc 12, 12, 8
+; CHECK-NEXT: or 12, 12, 3
+; CHECK-NEXT: stwcx. 12, 0, 5
+; CHECK-NEXT: bne 0, .LBB4_6
+; CHECK-NEXT: b .LBB4_1
+; CHECK-NEXT: .LBB4_8: # %atomicrmw.end
+; CHECK-NEXT: lwsync
+; CHECK-NEXT: blr
+ %result = atomicrmw usub_sat ptr %ptr, i8 %val seq_cst
+ ret i8 %result
+}
+
+define i16 @atomicrmw_usub_sat_i16(ptr %ptr, i16 %val) {
+; CHECK-LABEL: atomicrmw_usub_sat_i16:
+; CHECK: # %bb.0:
+; CHECK-NEXT: sync
+; CHECK-NEXT: mr 5, 3
+; CHECK-NEXT: li 8, 0
+; CHECK-NEXT: lhz 3, 0(3)
+; CHECK-NEXT: rlwinm 7, 5, 3, 27, 27
+; CHECK-NEXT: xori 7, 7, 16
+; CHECK-NEXT: ori 8, 8, 65535
+; CHECK-NEXT: clrlwi 6, 4, 16
+; CHECK-NEXT: rldicr 5, 5, 0, 61
+; CHECK-NEXT: slw 8, 8, 7
+; CHECK-NEXT: b .LBB5_2
+; CHECK-NEXT: .LBB5_1: # %atomicrmw.start
+; CHECK-NEXT: #
+; CHECK-NEXT: srw 3, 11, 7
+; CHECK-NEXT: cmplw 3, 9
+; CHECK-NEXT: beq 0, .LBB5_8
+; CHECK-NEXT: .LBB5_2: # %atomicrmw.start
+; CHECK-NEXT: # =>This Loop Header: Depth=1
+; CHECK-NEXT: # Child Loop BB5_6 Depth 2
+; CHECK-NEXT: clrlwi 9, 3, 16
+; CHECK-NEXT: cmplw 9, 6
+; CHECK-NEXT: bge 0, .LBB5_4
+; CHECK-NEXT: # %bb.3: # %atomicrmw.start
+; CHECK-NEXT: #
+; CHECK-NEXT: li 3, 0
+; CHECK-NEXT: b .LBB5_5
+; CHECK-NEXT: .LBB5_4:
+; CHECK-NEXT: sub 3, 3, 4
+; CHECK-NEXT: .LBB5_5: # %atomicrmw.start
+; CHECK-NEXT: #
+; CHECK-NEXT: slw 3, 3, 7
+; CHECK-NEXT: slw 10, 9, 7
+; CHECK-NEXT: and 3, 3, 8
+; CHECK-NEXT: and 10, 10, 8
+; CHECK-NEXT: .LBB5_6: # %atomicrmw.start
+; CHECK-NEXT: # Parent Loop BB5_2 Depth=1
+; CHECK-NEXT: # => This Inner Loop Header: Depth=2
+; CHECK-NEXT: lwarx 12, 0, 5
+; CHECK-NEXT: and 11, 12, 8
+; CHECK-NEXT: cmpw 11, 10
+; CHECK-NEXT: bne 0, .LBB5_1
+; CHECK-NEXT: # %bb.7: # %atomicrmw.start
+; CHECK-NEXT: #
+; CHECK-NEXT: andc 12, 12, 8
+; CHECK-NEXT: or 12, 12, 3
+; CHECK-NEXT: stwcx. 12, 0, 5
+; CHECK-NEXT: bne 0, .LBB5_6
+; CHECK-NEXT: b .LBB5_1
+; CHECK-NEXT: .LBB5_8: # %atomicrmw.end
+; CHECK-NEXT: lwsync
+; CHECK-NEXT: blr
+ %result = atomicrmw usub_sat ptr %ptr, i16 %val seq_cst
+ ret i16 %result
+}
+
+define i32 @atomicrmw_usub_sat_i32(ptr %ptr, i32 %val) {
+; CHECK-LABEL: atomicrmw_usub_sat_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: sync
+; CHECK-NEXT: lwz 6, 0(3)
+; CHECK-NEXT: b .LBB6_2
+; CHECK-NEXT: .LBB6_1: # %atomicrmw.start
+; CHECK-NEXT: #
+; CHECK-NEXT: cmplw 5, 6
+; CHECK-NEXT: mr 6, 5
+; CHECK-NEXT: beq 0, .LBB6_7
+; CHECK-NEXT: .LBB6_2: # %atomicrmw.start
+; CHECK-NEXT: # =>This Loop Header: Depth=1
+; CHECK-NEXT: # Child Loop BB6_5 Depth 2
+; CHECK-NEXT: cmplw 6, 4
+; CHECK-NEXT: bge 0, .LBB6_4
+; CHECK-NEXT: # %bb.3: # %atomicrmw.start
+; CHECK-NEXT: #
+; CHECK-NEXT: li 7, 0
+; CHECK-NEXT: b .LBB6_5
+; CHECK-NEXT: .LBB6_4:
+; CHECK-NEXT: sub 7, 6, 4
+; CHECK-NEXT: .LBB6_5: # %atomicrmw.start
+; CHECK-NEXT: # Parent Loop BB6_2 Depth=1
+; CHECK-NEXT: # => This Inner Loop Header: Depth=2
+; CHECK-NEXT: lwarx 5, 0, 3
+; CHECK-NEXT: cmpw 5, 6
+; CHECK-NEXT: bne 0, .LBB6_1
+; CHECK-NEXT: # %bb.6: # %atomicrmw.start
+; CHECK-NEXT: #
+; CHECK-NEXT: stwcx. 7, 0, 3
+; CHECK-NEXT: bne 0, .LBB6_5
+; CHECK-NEXT: b .LBB6_1
+; CHECK-NEXT: .LBB6_7: # %atomicrmw.end
+; CHECK-NEXT: mr 3, 5
+; CHECK-NEXT: lwsync
+; CHECK-NEXT: blr
+ %result = atomicrmw usub_sat ptr %ptr, i32 %val seq_cst
+ ret i32 %result
+}
+
+define i64 @atomicrmw_usub_sat_i64(ptr %ptr, i64 %val) {
+; CHECK-LABEL: atomicrmw_usub_sat_i64:
+; CHECK: # %bb.0:
+; CHECK-NEXT: sync
+; CHECK-NEXT: ld 6, 0(3)
+; CHECK-NEXT: b .LBB7_2
+; CHECK-NEXT: .LBB7_1: # %atomicrmw.start
+; CHECK-NEXT: #
+; CHECK-NEXT: cmpld 5, 6
+; CHECK-NEXT: mr 6, 5
+; CHECK-NEXT: beq 0, .LBB7_7
+; CHECK-NEXT: .LBB7_2: # %atomicrmw.start
+; CHECK-NEXT: # =>This Loop Header: Depth=1
+; CHECK-NEXT: # Child Loop BB7_5 Depth 2
+; CHECK-NEXT: cmpld 6, 4
+; CHECK-NEXT: bge 0, .LBB7_4
+; CHECK-NEXT: # %bb.3: # %atomicrmw.start
+; CHECK-NEXT: #
+; CHECK-NEXT: li 7, 0
+; CHECK-NEXT: b .LBB7_5
+; CHECK-NEXT: .LBB7_4:
+; CHECK-NEXT: sub 7, 6, 4
+; CHECK-NEXT: .LBB7_5: # %atomicrmw.start
+; CHECK-NEXT: # Parent Loop BB7_2 Depth=1
+; CHECK-NEXT: # => This Inner Loop Header: Depth=2
+; CHECK-NEXT: ldarx 5, 0, 3
+; CHECK-NEXT: cmpd 5, 6
+; CHECK-NEXT: bne 0, .LBB7_1
+; CHECK-NEXT: # %bb.6: # %atomicrmw.start
+; CHECK-NEXT: #
+; CHECK-NEXT: stdcx. 7, 0, 3
+; CHECK-NEXT: bne 0, .LBB7_5
+; CHECK-NEXT: b .LBB7_1
+; CHECK-NEXT: .LBB7_7: # %atomicrmw.end
+; CHECK-NEXT: mr 3, 5
+; CHECK-NEXT: lwsync
+; CHECK-NEXT: blr
+ %result = atomicrmw usub_sat ptr %ptr, i64 %val seq_cst
+ ret i64 %result
+}
diff --git a/llvm/test/CodeGen/RISCV/atomicrmw-cond-sub-clamp.ll b/llvm/test/CodeGen/RISCV/atomicrmw-cond-sub-clamp.ll
new file mode 100644
index 00000000000000..84a763eb68f0cb
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/atomicrmw-cond-sub-clamp.ll
@@ -0,0 +1,1412 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
+; RUN: | FileCheck -check-prefix=RV32I %s
+; RUN: llc -mtriple=riscv32 -mattr=+a -verify-machineinstrs < %s \
+; RUN: | FileCheck -check-prefix=RV32IA %s
+; RUN: llc -mtriple=riscv32 -mattr=+a,+experimental-ztso -verify-machineinstrs < %s \
+; RUN: | FileCheck -check-prefix=RV32IA %s
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
+; RUN: | FileCheck -check-prefix=RV64I %s
+; RUN: llc -mtriple=riscv64 -mattr=+a -verify-machineinstrs < %s \
+; RUN: | FileCheck -check-prefix=RV64IA %s
+; RUN: llc -mtriple=riscv64 -mattr=+a,+experimental-ztso -verify-machineinstrs < %s \
+; RUN: | FileCheck -check-prefix=RV64IA %s
+
+
+define i8 @atomicrmw_usub_cond_i8(ptr %ptr, i8 %val) {
+; RV32I-LABEL: atomicrmw_usub_cond_i8:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -32
+; RV32I-NEXT: .cfi_def_cfa_offset 32
+; RV32I-NEXT: sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s2, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT: .cfi_offset ra, -4
+; RV32I-NEXT: .cfi_offset s0, -8
+; RV32I-NEXT: .cfi_offset s1, -12
+; RV32I-NEXT: .cfi_offset s2, -16
+; RV32I-NEXT: mv s0, a0
+; RV32I-NEXT: lbu a3, 0(a0)
+; RV32I-NEXT: mv s1, a1
+; RV32I-NEXT: andi s2, a1, 255
+; RV32I-NEXT: j .LBB0_3
+; RV32I-NEXT: .LBB0_1: # %atomicrmw.start
+; RV32I-NEXT: # in Loop: Header=BB0_3 Depth=1
+; RV32I-NEXT: mv a2, s1
+; RV32I-NEXT: .LBB0_2: # %atomicrmw.start
+; RV32I-NEXT: # in Loop: Header=BB0_3 Depth=1
+; RV32I-NEXT: sb a3, 15(sp)
+; RV32I-NEXT: addi a1, sp, 15
+; RV32I-NEXT: li a3, 5
+; RV32I-NEXT: li a4, 5
+; RV32I-NEXT: mv a0, s0
+; RV32I-NEXT: call __atomic_compare_exchange_1
+; RV32I-NEXT: lbu a3, 15(sp)
+; RV32I-NEXT: bnez a0, .LBB0_5
+; RV32I-NEXT: .LBB0_3: # %atomicrmw.start
+; RV32I-NEXT: # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT: andi a0, a3, 255
+; RV32I-NEXT: bltu a0, s2, .LBB0_1
+; RV32I-NEXT: # %bb.4: # in Loop: Header=BB0_3 Depth=1
+; RV32I-NEXT: sub a2, a3, s1
+; RV32I-NEXT: j .LBB0_2
+; RV32I-NEXT: .LBB0_5: # %atomicrmw.end
+; RV32I-NEXT: mv a0, a3
+; RV32I-NEXT: lw ra, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 32
+; RV32I-NEXT: ret
+;
+; RV32IA-LABEL: atomicrmw_usub_cond_i8:
+; RV32IA: # %bb.0:
+; RV32IA-NEXT: andi a2, a0, -4
+; RV32IA-NEXT: slli a3, a0, 3
+; RV32IA-NEXT: andi a0, a3, 24
+; RV32IA-NEXT: li a4, 255
+; RV32IA-NEXT: lw a6, 0(a2)
+; RV32IA-NEXT: sll a3, a4, a3
+; RV32IA-NEXT: not a3, a3
+; RV32IA-NEXT: andi a4, a1, 255
+; RV32IA-NEXT: j .LBB0_3
+; RV32IA-NEXT: .LBB0_1: # %atomicrmw.start
+; RV32IA-NEXT: # in Loop: Header=BB0_3 Depth=1
+; RV32IA-NEXT: mv a6, a1
+; RV32IA-NEXT: .LBB0_2: # %atomicrmw.start
+; RV32IA-NEXT: # in Loop: Header=BB0_3 Depth=1
+; RV32IA-NEXT: andi a6, a6, 255
+; RV32IA-NEXT: sll a6, a6, a0
+; RV32IA-NEXT: and a7, a5, a3
+; RV32IA-NEXT: or a7, a7, a6
+; RV32IA-NEXT: .LBB0_6: # %atomicrmw.start
+; RV32IA-NEXT: # Parent Loop BB0_3 Depth=1
+; RV32IA-NEXT: # => This Inner Loop Header: Depth=2
+; RV32IA-NEXT: lr.w.aqrl a6, (a2)
+; RV32IA-NEXT: bne a6, a5, .LBB0_8
+; RV32IA-NEXT: # %bb.7: # %atomicrmw.start
+; RV32IA-NEXT: # in Loop: Header=BB0_6 Depth=2
+; RV32IA-NEXT: sc.w.rl t0, a7, (a2)
+; RV32IA-NEXT: bnez t0, .LBB0_6
+; RV32IA-NEXT: .LBB0_8: # %atomicrmw.start
+; RV32IA-NEXT: # in Loop: Header=BB0_3 Depth=1
+; RV32IA-NEXT: beq a6, a5, .LBB0_5
+; RV32IA-NEXT: .LBB0_3: # %atomicrmw.start
+; RV32IA-NEXT: # =>This Loop Header: Depth=1
+; RV32IA-NEXT: # Child Loop BB0_6 Depth 2
+; RV32IA-NEXT: mv a5, a6
+; RV32IA-NEXT: srl a6, a6, a0
+; RV32IA-NEXT: andi a7, a6, 255
+; RV32IA-NEXT: bltu a7, a4, .LBB0_1
+; RV32IA-NEXT: # %bb.4: # in Loop: Header=BB0_3 Depth=1
+; RV32IA-NEXT: sub a6, a6, a1
+; RV32IA-NEXT: j .LBB0_2
+; RV32IA-NEXT: .LBB0_5: # %atomicrmw.end
+; RV32IA-NEXT: srl a0, a6, a0
+; RV32IA-NEXT: ret
+;
+; RV64I-LABEL: atomicrmw_usub_cond_i8:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -48
+; RV64I-NEXT: .cfi_def_cfa_offset 48
+; RV64I-NEXT: sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s2, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT: .cfi_offset ra, -8
+; RV64I-NEXT: .cfi_offset s0, -16
+; RV64I-NEXT: .cfi_offset s1, -24
+; RV64I-NEXT: .cfi_offset s2, -32
+; RV64I-NEXT: mv s0, a0
+; RV64I-NEXT: lbu a3, 0(a0)
+; RV64I-NEXT: mv s1, a1
+; RV64I-NEXT: andi s2, a1, 255
+; RV64I-NEXT: j .LBB0_3
+; RV64I-NEXT: .LBB0_1: # %atomicrmw.start
+; RV64I-NEXT: # in Loop: Header=BB0_3 Depth=1
+; RV64I-NEXT: mv a2, s1
+; RV64I-NEXT: .LBB0_2: # %atomicrmw.start
+; RV64I-NEXT: # in Loop: Header=BB0_3 Depth=1
+; RV64I-NEXT: sb a3, 15(sp)
+; RV64I-NEXT: addi a1, sp, 15
+; RV64I-NEXT: li a3, 5
+; RV64I-NEXT: li a4, 5
+; RV64I-NEXT: mv a0, s0
+; RV64I-NEXT: call __atomic_compare_exchange_1
+; RV64I-NEXT: lbu a3, 15(sp)
+; RV64I-NEXT: bnez a0, .LBB0_5
+; RV64I-NEXT: .LBB0_3: # %atomicrmw.start
+; RV64I-NEXT: # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT: andi a0, a3, 255
+; RV64I-NEXT: bltu a0, s2, .LBB0_1
+; RV64I-NEXT: # %bb.4: # in Loop: Header=BB0_3 Depth=1
+; RV64I-NEXT: sub a2, a3, s1
+; RV64I-NEXT: j .LBB0_2
+; RV64I-NEXT: .LBB0_5: # %atomicrmw.end
+; RV64I-NEXT: mv a0, a3
+; RV64I-NEXT: ld ra, 40(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 48
+; RV64I-NEXT: ret
+;
+; RV64IA-LABEL: atomicrmw_usub_cond_i8:
+; RV64IA: # %bb.0:
+; RV64IA-NEXT: andi a2, a0, -4
+; RV64IA-NEXT: slli a4, a0, 3
+; RV64IA-NEXT: andi a0, a4, 24
+; RV64IA-NEXT: li a5, 255
+; RV64IA-NEXT: lw a3, 0(a2)
+; RV64IA-NEXT: sllw a4, a5, a4
+; RV64IA-NEXT: not a4, a4
+; RV64IA-NEXT: andi a5, a1, 255
+; RV64IA-NEXT: j .LBB0_3
+; RV64IA-NEXT: .LBB0_1: # %atomicrmw.start
+; RV64IA-NEXT: # in Loop: Header=BB0_3 Depth=1
+; RV64IA-NEXT: mv a6, a1
+; RV64IA-NEXT: .LBB0_2: # %atomicrmw.start
+; RV64IA-NEXT: # in Loop: Header=BB0_3 Depth=1
+; RV64IA-NEXT: sext.w a7, a3
+; RV64IA-NEXT: andi a6, a6, 255
+; RV64IA-NEXT: sllw a6, a6, a0
+; RV64IA-NEXT: and a3, a3, a4
+; RV64IA-NEXT: or a6, a3, a6
+; RV64IA-NEXT: .LBB0_6: # %atomicrmw.start
+; RV64IA-NEXT: # Parent Loop BB0_3 Depth=1
+; RV64IA-NEXT: # => This Inner Loop Header: Depth=2
+; RV64IA-NEXT: lr.w.aqrl a3, (a2)
+; RV64IA-NEXT: bne a3, a7, .LBB0_8
+; RV64IA-NEXT: # %bb.7: # %atomicrmw.start
+; RV64IA-NEXT: # in Loop: Header=BB0_6 Depth=2
+; RV64IA-NEXT: sc.w.rl t0, a6, (a2)
+; RV64IA-NEXT: bnez t0, .LBB0_6
+; RV64IA-NEXT: .LBB0_8: # %atomicrmw.start
+; RV64IA-NEXT: # in Loop: Header=BB0_3 Depth=1
+; RV64IA-NEXT: beq a3, a7, .LBB0_5
+; RV64IA-NEXT: .LBB0_3: # %atomicrmw.start
+; RV64IA-NEXT: # =>This Loop Header: Depth=1
+; RV64IA-NEXT: # Child Loop BB0_6 Depth 2
+; RV64IA-NEXT: srlw a6, a3, a0
+; RV64IA-NEXT: andi a7, a6, 255
+; RV64IA-NEXT: bltu a7, a5, .LBB0_1
+; RV64IA-NEXT: # %bb.4: # in Loop: Header=BB0_3 Depth=1
+; RV64IA-NEXT: sub a6, a6, a1
+; RV64IA-NEXT: j .LBB0_2
+; RV64IA-NEXT: .LBB0_5: # %atomicrmw.end
+; RV64IA-NEXT: srlw a0, a3, a0
+; RV64IA-NEXT: ret
+ %result = atomicrmw usub_cond ptr %ptr, i8 %val seq_cst
+ ret i8 %result
+}
+
+define i16 @atomicrmw_usub_cond_i16(ptr %ptr, i16 %val) {
+; RV32I-LABEL: atomicrmw_usub_cond_i16:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -32
+; RV32I-NEXT: .cfi_def_cfa_offset 32
+; RV32I-NEXT: sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s2, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s3, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: .cfi_offset ra, -4
+; RV32I-NEXT: .cfi_offset s0, -8
+; RV32I-NEXT: .cfi_offset s1, -12
+; RV32I-NEXT: .cfi_offset s2, -16
+; RV32I-NEXT: .cfi_offset s3, -20
+; RV32I-NEXT: mv s0, a1
+; RV32I-NEXT: mv s1, a0
+; RV32I-NEXT: lhu a1, 0(a0)
+; RV32I-NEXT: lui s2, 16
+; RV32I-NEXT: addi s2, s2, -1
+; RV32I-NEXT: and s3, s0, s2
+; RV32I-NEXT: j .LBB1_3
+; RV32I-NEXT: .LBB1_1: # %atomicrmw.start
+; RV32I-NEXT: # in Loop: Header=BB1_3 Depth=1
+; RV32I-NEXT: mv a2, s0
+; RV32I-NEXT: .LBB1_2: # %atomicrmw.start
+; RV32I-NEXT: # in Loop: Header=BB1_3 Depth=1
+; RV32I-NEXT: sh a1, 10(sp)
+; RV32I-NEXT: addi a1, sp, 10
+; RV32I-NEXT: li a3, 5
+; RV32I-NEXT: li a4, 5
+; RV32I-NEXT: mv a0, s1
+; RV32I-NEXT: call __atomic_compare_exchange_2
+; RV32I-NEXT: lh a1, 10(sp)
+; RV32I-NEXT: bnez a0, .LBB1_5
+; RV32I-NEXT: .LBB1_3: # %atomicrmw.start
+; RV32I-NEXT: # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT: and a0, a1, s2
+; RV32I-NEXT: bltu a0, s3, .LBB1_1
+; RV32I-NEXT: # %bb.4: # in Loop: Header=BB1_3 Depth=1
+; RV32I-NEXT: sub a2, a1, s0
+; RV32I-NEXT: j .LBB1_2
+; RV32I-NEXT: .LBB1_5: # %atomicrmw.end
+; RV32I-NEXT: mv a0, a1
+; RV32I-NEXT: lw ra, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 32
+; RV32I-NEXT: ret
+;
+; RV32IA-LABEL: atomicrmw_usub_cond_i16:
+; RV32IA: # %bb.0:
+; RV32IA-NEXT: andi a2, a0, -4
+; RV32IA-NEXT: slli a4, a0, 3
+; RV32IA-NEXT: andi a0, a4, 24
+; RV32IA-NEXT: lui a3, 16
+; RV32IA-NEXT: addi a3, a3, -1
+; RV32IA-NEXT: lw a7, 0(a2)
+; RV32IA-NEXT: sll a4, a3, a4
+; RV32IA-NEXT: not a4, a4
+; RV32IA-NEXT: and a5, a1, a3
+; RV32IA-NEXT: j .LBB1_3
+; RV32IA-NEXT: .LBB1_1: # %atomicrmw.start
+; RV32IA-NEXT: # in Loop: Header=BB1_3 Depth=1
+; RV32IA-NEXT: mv a7, a1
+; RV32IA-NEXT: .LBB1_2: # %atomicrmw.start
+; RV32IA-NEXT: # in Loop: Header=BB1_3 Depth=1
+; RV32IA-NEXT: and a7, a7, a3
+; RV32IA-NEXT: sll a7, a7, a0
+; RV32IA-NEXT: and t0, a6, a4
+; RV32IA-NEXT: or t0, t0, a7
+; RV32IA-NEXT: .LBB1_6: # %atomicrmw.start
+; RV32IA-NEXT: # Parent Loop BB1_3 Depth=1
+; RV32IA-NEXT: # => This Inner Loop Header: Depth=2
+; RV32IA-NEXT: lr.w.aqrl a7, (a2)
+; RV32IA-NEXT: bne a7, a6, .LBB1_8
+; RV32IA-NEXT: # %bb.7: # %atomicrmw.start
+; RV32IA-NEXT: # in Loop: Header=BB1_6 Depth=2
+; RV32IA-NEXT: sc.w.rl t1, t0, (a2)
+; RV32IA-NEXT: bnez t1, .LBB1_6
+; RV32IA-NEXT: .LBB1_8: # %atomicrmw.start
+; RV32IA-NEXT: # in Loop: Header=BB1_3 Depth=1
+; RV32IA-NEXT: beq a7, a6, .LBB1_5
+; RV32IA-NEXT: .LBB1_3: # %atomicrmw.start
+; RV32IA-NEXT: # =>This Loop Header: Depth=1
+; RV32IA-NEXT: # Child Loop BB1_6 Depth 2
+; RV32IA-NEXT: mv a6, a7
+; RV32IA-NEXT: srl a7, a7, a0
+; RV32IA-NEXT: and t0, a7, a3
+; RV32IA-NEXT: bltu t0, a5, .LBB1_1
+; RV32IA-NEXT: # %bb.4: # in Loop: Header=BB1_3 Depth=1
+; RV32IA-NEXT: sub a7, a7, a1
+; RV32IA-NEXT: j .LBB1_2
+; RV32IA-NEXT: .LBB1_5: # %atomicrmw.end
+; RV32IA-NEXT: srl a0, a7, a0
+; RV32IA-NEXT: ret
+;
+; RV64I-LABEL: atomicrmw_usub_cond_i16:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -48
+; RV64I-NEXT: .cfi_def_cfa_offset 48
+; RV64I-NEXT: sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s2, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s3, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: .cfi_offset ra, -8
+; RV64I-NEXT: .cfi_offset s0, -16
+; RV64I-NEXT: .cfi_offset s1, -24
+; RV64I-NEXT: .cfi_offset s2, -32
+; RV64I-NEXT: .cfi_offset s3, -40
+; RV64I-NEXT: mv s0, a1
+; RV64I-NEXT: mv s1, a0
+; RV64I-NEXT: lhu a1, 0(a0)
+; RV64I-NEXT: lui s2, 16
+; RV64I-NEXT: addiw s2, s2, -1
+; RV64I-NEXT: and s3, s0, s2
+; RV64I-NEXT: j .LBB1_3
+; RV64I-NEXT: .LBB1_1: # %atomicrmw.start
+; RV64I-NEXT: # in Loop: Header=BB1_3 Depth=1
+; RV64I-NEXT: mv a2, s0
+; RV64I-NEXT: .LBB1_2: # %atomicrmw.start
+; RV64I-NEXT: # in Loop: Header=BB1_3 Depth=1
+; RV64I-NEXT: sh a1, 6(sp)
+; RV64I-NEXT: addi a1, sp, 6
+; RV64I-NEXT: li a3, 5
+; RV64I-NEXT: li a4, 5
+; RV64I-NEXT: mv a0, s1
+; RV64I-NEXT: call __atomic_compare_exchange_2
+; RV64I-NEXT: lh a1, 6(sp)
+; RV64I-NEXT: bnez a0, .LBB1_5
+; RV64I-NEXT: .LBB1_3: # %atomicrmw.start
+; RV64I-NEXT: # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT: and a0, a1, s2
+; RV64I-NEXT: bltu a0, s3, .LBB1_1
+; RV64I-NEXT: # %bb.4: # in Loop: Header=BB1_3 Depth=1
+; RV64I-NEXT: sub a2, a1, s0
+; RV64I-NEXT: j .LBB1_2
+; RV64I-NEXT: .LBB1_5: # %atomicrmw.end
+; RV64I-NEXT: mv a0, a1
+; RV64I-NEXT: ld ra, 40(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s3, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 48
+; RV64I-NEXT: ret
+;
+; RV64IA-LABEL: atomicrmw_usub_cond_i16:
+; RV64IA: # %bb.0:
+; RV64IA-NEXT: andi a2, a0, -4
+; RV64IA-NEXT: slli a5, a0, 3
+; RV64IA-NEXT: andi a0, a5, 24
+; RV64IA-NEXT: lui a3, 16
+; RV64IA-NEXT: addiw a3, a3, -1
+; RV64IA-NEXT: lw a4, 0(a2)
+; RV64IA-NEXT: sllw a5, a3, a5
+; RV64IA-NEXT: not a5, a5
+; RV64IA-NEXT: and a6, a1, a3
+; RV64IA-NEXT: j .LBB1_3
+; RV64IA-NEXT: .LBB1_1: # %atomicrmw.start
+; RV64IA-NEXT: # in Loop: Header=BB1_3 Depth=1
+; RV64IA-NEXT: mv a7, a1
+; RV64IA-NEXT: .LBB1_2: # %atomicrmw.start
+; RV64IA-NEXT: # in Loop: Header=BB1_3 Depth=1
+; RV64IA-NEXT: sext.w t0, a4
+; RV64IA-NEXT: and a7, a7, a3
+; RV64IA-NEXT: sllw a7, a7, a0
+; RV64IA-NEXT: and a4, a4, a5
+; RV64IA-NEXT: or a7, a4, a7
+; RV64IA-NEXT: .LBB1_6: # %atomicrmw.start
+; RV64IA-NEXT: # Parent Loop BB1_3 Depth=1
+; RV64IA-NEXT: # => This Inner Loop Header: Depth=2
+; RV64IA-NEXT: lr.w.aqrl a4, (a2)
+; RV64IA-NEXT: bne a4, t0, .LBB1_8
+; RV64IA-NEXT: # %bb.7: # %atomicrmw.start
+; RV64IA-NEXT: # in Loop: Header=BB1_6 Depth=2
+; RV64IA-NEXT: sc.w.rl t1, a7, (a2)
+; RV64IA-NEXT: bnez t1, .LBB1_6
+; RV64IA-NEXT: .LBB1_8: # %atomicrmw.start
+; RV64IA-NEXT: # in Loop: Header=BB1_3 Depth=1
+; RV64IA-NEXT: beq a4, t0, .LBB1_5
+; RV64IA-NEXT: .LBB1_3: # %atomicrmw.start
+; RV64IA-NEXT: # =>This Loop Header: Depth=1
+; RV64IA-NEXT: # Child Loop BB1_6 Depth 2
+; RV64IA-NEXT: srlw a7, a4, a0
+; RV64IA-NEXT: and t0, a7, a3
+; RV64IA-NEXT: bltu t0, a6, .LBB1_1
+; RV64IA-NEXT: # %bb.4: # in Loop: Header=BB1_3 Depth=1
+; RV64IA-NEXT: sub a7, a7, a1
+; RV64IA-NEXT: j .LBB1_2
+; RV64IA-NEXT: .LBB1_5: # %atomicrmw.end
+; RV64IA-NEXT: srlw a0, a4, a0
+; RV64IA-NEXT: ret
+ %result = atomicrmw usub_cond ptr %ptr, i16 %val seq_cst
+ ret i16 %result
+}
+
+define i32 @atomicrmw_usub_cond_i32(ptr %ptr, i32 %val) {
+; RV32I-LABEL: atomicrmw_usub_cond_i32:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -16
+; RV32I-NEXT: .cfi_def_cfa_offset 16
+; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s1, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT: .cfi_offset ra, -4
+; RV32I-NEXT: .cfi_offset s0, -8
+; RV32I-NEXT: .cfi_offset s1, -12
+; RV32I-NEXT: mv s0, a0
+; RV32I-NEXT: lw a3, 0(a0)
+; RV32I-NEXT: mv s1, a1
+; RV32I-NEXT: j .LBB2_3
+; RV32I-NEXT: .LBB2_1: # %atomicrmw.start
+; RV32I-NEXT: # in Loop: Header=BB2_3 Depth=1
+; RV32I-NEXT: mv a2, s1
+; RV32I-NEXT: .LBB2_2: # %atomicrmw.start
+; RV32I-NEXT: # in Loop: Header=BB2_3 Depth=1
+; RV32I-NEXT: sw a3, 0(sp)
+; RV32I-NEXT: mv a1, sp
+; RV32I-NEXT: li a3, 5
+; RV32I-NEXT: li a4, 5
+; RV32I-NEXT: mv a0, s0
+; RV32I-NEXT: call __atomic_compare_exchange_4
+; RV32I-NEXT: lw a3, 0(sp)
+; RV32I-NEXT: bnez a0, .LBB2_5
+; RV32I-NEXT: .LBB2_3: # %atomicrmw.start
+; RV32I-NEXT: # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT: bltu a3, s1, .LBB2_1
+; RV32I-NEXT: # %bb.4: # in Loop: Header=BB2_3 Depth=1
+; RV32I-NEXT: sub a2, a3, s1
+; RV32I-NEXT: j .LBB2_2
+; RV32I-NEXT: .LBB2_5: # %atomicrmw.end
+; RV32I-NEXT: mv a0, a3
+; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 16
+; RV32I-NEXT: ret
+;
+; RV32IA-LABEL: atomicrmw_usub_cond_i32:
+; RV32IA: # %bb.0:
+; RV32IA-NEXT: lw a2, 0(a0)
+; RV32IA-NEXT: j .LBB2_2
+; RV32IA-NEXT: .LBB2_1: # %atomicrmw.start
+; RV32IA-NEXT: # in Loop: Header=BB2_2 Depth=1
+; RV32IA-NEXT: mv a4, a1
+; RV32IA-NEXT: .LBB2_5: # %atomicrmw.start
+; RV32IA-NEXT: # Parent Loop BB2_2 Depth=1
+; RV32IA-NEXT: # => This Inner Loop Header: Depth=2
+; RV32IA-NEXT: lr.w.aqrl a2, (a0)
+; RV32IA-NEXT: bne a2, a3, .LBB2_7
+; RV32IA-NEXT: # %bb.6: # %atomicrmw.start
+; RV32IA-NEXT: # in Loop: Header=BB2_5 Depth=2
+; RV32IA-NEXT: sc.w.rl a5, a1, (a0)
+; RV32IA-NEXT: bnez a5, .LBB2_5
+; RV32IA-NEXT: .LBB2_7: # %atomicrmw.start
+; RV32IA-NEXT: # in Loop: Header=BB2_2 Depth=1
+; RV32IA-NEXT: beq a2, a3, .LBB2_4
+; RV32IA-NEXT: .LBB2_2: # %atomicrmw.start
+; RV32IA-NEXT: # =>This Loop Header: Depth=1
+; RV32IA-NEXT: # Child Loop BB2_8 Depth 2
+; RV32IA-NEXT: # Child Loop BB2_5 Depth 2
+; RV32IA-NEXT: mv a3, a2
+; RV32IA-NEXT: bltu a2, a1, .LBB2_1
+; RV32IA-NEXT: # %bb.3: # in Loop: Header=BB2_2 Depth=1
+; RV32IA-NEXT: sub a4, a3, a1
+; RV32IA-NEXT: .LBB2_8: # Parent Loop BB2_2 Depth=1
+; RV32IA-NEXT: # => This Inner Loop Header: Depth=2
+; RV32IA-NEXT: lr.w.aqrl a2, (a0)
+; RV32IA-NEXT: bne a2, a3, .LBB2_2
+; RV32IA-NEXT: # %bb.9: # in Loop: Header=BB2_8 Depth=2
+; RV32IA-NEXT: sc.w.rl a5, a4, (a0)
+; RV32IA-NEXT: bnez a5, .LBB2_8
+; RV32IA-NEXT: # %bb.10:
+; RV32IA-NEXT: .LBB2_4: # %atomicrmw.end
+; RV32IA-NEXT: mv a0, a2
+; RV32IA-NEXT: ret
+;
+; RV64I-LABEL: atomicrmw_usub_cond_i32:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -48
+; RV64I-NEXT: .cfi_def_cfa_offset 48
+; RV64I-NEXT: sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s2, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT: .cfi_offset ra, -8
+; RV64I-NEXT: .cfi_offset s0, -16
+; RV64I-NEXT: .cfi_offset s1, -24
+; RV64I-NEXT: .cfi_offset s2, -32
+; RV64I-NEXT: mv s0, a0
+; RV64I-NEXT: lw a3, 0(a0)
+; RV64I-NEXT: mv s1, a1
+; RV64I-NEXT: sext.w s2, a1
+; RV64I-NEXT: j .LBB2_3
+; RV64I-NEXT: .LBB2_1: # %atomicrmw.start
+; RV64I-NEXT: # in Loop: Header=BB2_3 Depth=1
+; RV64I-NEXT: mv a2, s1
+; RV64I-NEXT: .LBB2_2: # %atomicrmw.start
+; RV64I-NEXT: # in Loop: Header=BB2_3 Depth=1
+; RV64I-NEXT: sw a3, 12(sp)
+; RV64I-NEXT: addi a1, sp, 12
+; RV64I-NEXT: li a3, 5
+; RV64I-NEXT: li a4, 5
+; RV64I-NEXT: mv a0, s0
+; RV64I-NEXT: call __atomic_compare_exchange_4
+; RV64I-NEXT: lw a3, 12(sp)
+; RV64I-NEXT: bnez a0, .LBB2_5
+; RV64I-NEXT: .LBB2_3: # %atomicrmw.start
+; RV64I-NEXT: # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT: bltu a3, s2, .LBB2_1
+; RV64I-NEXT: # %bb.4: # in Loop: Header=BB2_3 Depth=1
+; RV64I-NEXT: subw a2, a3, s1
+; RV64I-NEXT: j .LBB2_2
+; RV64I-NEXT: .LBB2_5: # %atomicrmw.end
+; RV64I-NEXT: mv a0, a3
+; RV64I-NEXT: ld ra, 40(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 48
+; RV64I-NEXT: ret
+;
+; RV64IA-LABEL: atomicrmw_usub_cond_i32:
+; RV64IA: # %bb.0:
+; RV64IA-NEXT: lw a2, 0(a0)
+; RV64IA-NEXT: sext.w a3, a1
+; RV64IA-NEXT: j .LBB2_2
+; RV64IA-NEXT: .LBB2_1: # %atomicrmw.start
+; RV64IA-NEXT: # in Loop: Header=BB2_2 Depth=1
+; RV64IA-NEXT: mv a5, a1
+; RV64IA-NEXT: .LBB2_5: # %atomicrmw.start
+; RV64IA-NEXT: # Parent Loop BB2_2 Depth=1
+; RV64IA-NEXT: # => This Inner Loop Header: Depth=2
+; RV64IA-NEXT: lr.w.aqrl a2, (a0)
+; RV64IA-NEXT: bne a2, a4, .LBB2_7
+; RV64IA-NEXT: # %bb.6: # %atomicrmw.start
+; RV64IA-NEXT: # in Loop: Header=BB2_5 Depth=2
+; RV64IA-NEXT: sc.w.rl a6, a1, (a0)
+; RV64IA-NEXT: bnez a6, .LBB2_5
+; RV64IA-NEXT: .LBB2_7: # %atomicrmw.start
+; RV64IA-NEXT: # in Loop: Header=BB2_2 Depth=1
+; RV64IA-NEXT: beq a2, a4, .LBB2_4
+; RV64IA-NEXT: .LBB2_2: # %atomicrmw.start
+; RV64IA-NEXT: # =>This Loop Header: Depth=1
+; RV64IA-NEXT: # Child Loop BB2_8 Depth 2
+; RV64IA-NEXT: # Child Loop BB2_5 Depth 2
+; RV64IA-NEXT: sext.w a4, a2
+; RV64IA-NEXT: bltu a4, a3, .LBB2_1
+; RV64IA-NEXT: # %bb.3: # in Loop: Header=BB2_2 Depth=1
+; RV64IA-NEXT: subw a5, a2, a1
+; RV64IA-NEXT: .LBB2_8: # Parent Loop BB2_2 Depth=1
+; RV64IA-NEXT: # => This Inner Loop Header: Depth=2
+; RV64IA-NEXT: lr.w.aqrl a2, (a0)
+; RV64IA-NEXT: bne a2, a4, .LBB2_2
+; RV64IA-NEXT: # %bb.9: # in Loop: Header=BB2_8 Depth=2
+; RV64IA-NEXT: sc.w.rl a6, a5, (a0)
+; RV64IA-NEXT: bnez a6, .LBB2_8
+; RV64IA-NEXT: # %bb.10:
+; RV64IA-NEXT: .LBB2_4: # %atomicrmw.end
+; RV64IA-NEXT: mv a0, a2
+; RV64IA-NEXT: ret
+ %result = atomicrmw usub_cond ptr %ptr, i32 %val seq_cst
+ ret i32 %result
+}
+
+define i64 @atomicrmw_usub_cond_i64(ptr %ptr, i64 %val) {
+; RV32I-LABEL: atomicrmw_usub_cond_i64:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -32
+; RV32I-NEXT: .cfi_def_cfa_offset 32
+; RV32I-NEXT: sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s2, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT: .cfi_offset ra, -4
+; RV32I-NEXT: .cfi_offset s0, -8
+; RV32I-NEXT: .cfi_offset s1, -12
+; RV32I-NEXT: .cfi_offset s2, -16
+; RV32I-NEXT: mv s0, a0
+; RV32I-NEXT: lw a5, 4(a0)
+; RV32I-NEXT: lw a4, 0(a0)
+; RV32I-NEXT: mv s1, a2
+; RV32I-NEXT: mv s2, a1
+; RV32I-NEXT: j .LBB3_3
+; RV32I-NEXT: .LBB3_1: # %atomicrmw.start
+; RV32I-NEXT: # in Loop: Header=BB3_3 Depth=1
+; RV32I-NEXT: mv a3, s1
+; RV32I-NEXT: mv a2, s2
+; RV32I-NEXT: .LBB3_2: # %atomicrmw.start
+; RV32I-NEXT: # in Loop: Header=BB3_3 Depth=1
+; RV32I-NEXT: sw a4, 8(sp)
+; RV32I-NEXT: sw a5, 12(sp)
+; RV32I-NEXT: addi a1, sp, 8
+; RV32I-NEXT: li a4, 5
+; RV32I-NEXT: li a5, 5
+; RV32I-NEXT: mv a0, s0
+; RV32I-NEXT: call __atomic_compare_exchange_8
+; RV32I-NEXT: lw a5, 12(sp)
+; RV32I-NEXT: lw a4, 8(sp)
+; RV32I-NEXT: bnez a0, .LBB3_7
+; RV32I-NEXT: .LBB3_3: # %atomicrmw.start
+; RV32I-NEXT: # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT: sltu a0, a4, s2
+; RV32I-NEXT: mv a1, a0
+; RV32I-NEXT: beq a5, s1, .LBB3_5
+; RV32I-NEXT: # %bb.4: # %atomicrmw.start
+; RV32I-NEXT: # in Loop: Header=BB3_3 Depth=1
+; RV32I-NEXT: sltu a1, a5, s1
+; RV32I-NEXT: .LBB3_5: # %atomicrmw.start
+; RV32I-NEXT: # in Loop: Header=BB3_3 Depth=1
+; RV32I-NEXT: bnez a1, .LBB3_1
+; RV32I-NEXT: # %bb.6: # in Loop: Header=BB3_3 Depth=1
+; RV32I-NEXT: sub a3, a5, s1
+; RV32I-NEXT: sub a3, a3, a0
+; RV32I-NEXT: sub a2, a4, s2
+; RV32I-NEXT: j .LBB3_2
+; RV32I-NEXT: .LBB3_7: # %atomicrmw.end
+; RV32I-NEXT: mv a0, a4
+; RV32I-NEXT: mv a1, a5
+; RV32I-NEXT: lw ra, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 32
+; RV32I-NEXT: ret
+;
+; RV32IA-LABEL: atomicrmw_usub_cond_i64:
+; RV32IA: # %bb.0:
+; RV32IA-NEXT: addi sp, sp, -32
+; RV32IA-NEXT: .cfi_def_cfa_offset 32
+; RV32IA-NEXT: sw ra, 28(sp) # 4-byte Folded Spill
+; RV32IA-NEXT: sw s0, 24(sp) # 4-byte Folded Spill
+; RV32IA-NEXT: sw s1, 20(sp) # 4-byte Folded Spill
+; RV32IA-NEXT: sw s2, 16(sp) # 4-byte Folded Spill
+; RV32IA-NEXT: .cfi_offset ra, -4
+; RV32IA-NEXT: .cfi_offset s0, -8
+; RV32IA-NEXT: .cfi_offset s1, -12
+; RV32IA-NEXT: .cfi_offset s2, -16
+; RV32IA-NEXT: mv s0, a0
+; RV32IA-NEXT: lw a5, 4(a0)
+; RV32IA-NEXT: lw a4, 0(a0)
+; RV32IA-NEXT: mv s1, a2
+; RV32IA-NEXT: mv s2, a1
+; RV32IA-NEXT: j .LBB3_3
+; RV32IA-NEXT: .LBB3_1: # %atomicrmw.start
+; RV32IA-NEXT: # in Loop: Header=BB3_3 Depth=1
+; RV32IA-NEXT: mv a3, s1
+; RV32IA-NEXT: mv a2, s2
+; RV32IA-NEXT: .LBB3_2: # %atomicrmw.start
+; RV32IA-NEXT: # in Loop: Header=BB3_3 Depth=1
+; RV32IA-NEXT: sw a4, 8(sp)
+; RV32IA-NEXT: sw a5, 12(sp)
+; RV32IA-NEXT: addi a1, sp, 8
+; RV32IA-NEXT: li a4, 5
+; RV32IA-NEXT: li a5, 5
+; RV32IA-NEXT: mv a0, s0
+; RV32IA-NEXT: call __atomic_compare_exchange_8
+; RV32IA-NEXT: lw a5, 12(sp)
+; RV32IA-NEXT: lw a4, 8(sp)
+; RV32IA-NEXT: bnez a0, .LBB3_7
+; RV32IA-NEXT: .LBB3_3: # %atomicrmw.start
+; RV32IA-NEXT: # =>This Inner Loop Header: Depth=1
+; RV32IA-NEXT: sltu a0, a4, s2
+; RV32IA-NEXT: mv a1, a0
+; RV32IA-NEXT: beq a5, s1, .LBB3_5
+; RV32IA-NEXT: # %bb.4: # %atomicrmw.start
+; RV32IA-NEXT: # in Loop: Header=BB3_3 Depth=1
+; RV32IA-NEXT: sltu a1, a5, s1
+; RV32IA-NEXT: .LBB3_5: # %atomicrmw.start
+; RV32IA-NEXT: # in Loop: Header=BB3_3 Depth=1
+; RV32IA-NEXT: bnez a1, .LBB3_1
+; RV32IA-NEXT: # %bb.6: # in Loop: Header=BB3_3 Depth=1
+; RV32IA-NEXT: sub a3, a5, s1
+; RV32IA-NEXT: sub a3, a3, a0
+; RV32IA-NEXT: sub a2, a4, s2
+; RV32IA-NEXT: j .LBB3_2
+; RV32IA-NEXT: .LBB3_7: # %atomicrmw.end
+; RV32IA-NEXT: mv a0, a4
+; RV32IA-NEXT: mv a1, a5
+; RV32IA-NEXT: lw ra, 28(sp) # 4-byte Folded Reload
+; RV32IA-NEXT: lw s0, 24(sp) # 4-byte Folded Reload
+; RV32IA-NEXT: lw s1, 20(sp) # 4-byte Folded Reload
+; RV32IA-NEXT: lw s2, 16(sp) # 4-byte Folded Reload
+; RV32IA-NEXT: addi sp, sp, 32
+; RV32IA-NEXT: ret
+;
+; RV64I-LABEL: atomicrmw_usub_cond_i64:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -32
+; RV64I-NEXT: .cfi_def_cfa_offset 32
+; RV64I-NEXT: sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s1, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: .cfi_offset ra, -8
+; RV64I-NEXT: .cfi_offset s0, -16
+; RV64I-NEXT: .cfi_offset s1, -24
+; RV64I-NEXT: mv s0, a0
+; RV64I-NEXT: ld a3, 0(a0)
+; RV64I-NEXT: mv s1, a1
+; RV64I-NEXT: j .LBB3_3
+; RV64I-NEXT: .LBB3_1: # %atomicrmw.start
+; RV64I-NEXT: # in Loop: Header=BB3_3 Depth=1
+; RV64I-NEXT: mv a2, s1
+; RV64I-NEXT: .LBB3_2: # %atomicrmw.start
+; RV64I-NEXT: # in Loop: Header=BB3_3 Depth=1
+; RV64I-NEXT: sd a3, 0(sp)
+; RV64I-NEXT: mv a1, sp
+; RV64I-NEXT: li a3, 5
+; RV64I-NEXT: li a4, 5
+; RV64I-NEXT: mv a0, s0
+; RV64I-NEXT: call __atomic_compare_exchange_8
+; RV64I-NEXT: ld a3, 0(sp)
+; RV64I-NEXT: bnez a0, .LBB3_5
+; RV64I-NEXT: .LBB3_3: # %atomicrmw.start
+; RV64I-NEXT: # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT: bltu a3, s1, .LBB3_1
+; RV64I-NEXT: # %bb.4: # in Loop: Header=BB3_3 Depth=1
+; RV64I-NEXT: sub a2, a3, s1
+; RV64I-NEXT: j .LBB3_2
+; RV64I-NEXT: .LBB3_5: # %atomicrmw.end
+; RV64I-NEXT: mv a0, a3
+; RV64I-NEXT: ld ra, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 32
+; RV64I-NEXT: ret
+;
+; RV64IA-LABEL: atomicrmw_usub_cond_i64:
+; RV64IA: # %bb.0:
+; RV64IA-NEXT: ld a2, 0(a0)
+; RV64IA-NEXT: j .LBB3_2
+; RV64IA-NEXT: .LBB3_1: # %atomicrmw.start
+; RV64IA-NEXT: # in Loop: Header=BB3_2 Depth=1
+; RV64IA-NEXT: mv a4, a1
+; RV64IA-NEXT: .LBB3_5: # %atomicrmw.start
+; RV64IA-NEXT: # Parent Loop BB3_2 Depth=1
+; RV64IA-NEXT: # => This Inner Loop Header: Depth=2
+; RV64IA-NEXT: lr.d.aqrl a2, (a0)
+; RV64IA-NEXT: bne a2, a3, .LBB3_7
+; RV64IA-NEXT: # %bb.6: # %atomicrmw.start
+; RV64IA-NEXT: # in Loop: Header=BB3_5 Depth=2
+; RV64IA-NEXT: sc.d.rl a5, a1, (a0)
+; RV64IA-NEXT: bnez a5, .LBB3_5
+; RV64IA-NEXT: .LBB3_7: # %atomicrmw.start
+; RV64IA-NEXT: # in Loop: Header=BB3_2 Depth=1
+; RV64IA-NEXT: beq a2, a3, .LBB3_4
+; RV64IA-NEXT: .LBB3_2: # %atomicrmw.start
+; RV64IA-NEXT: # =>This Loop Header: Depth=1
+; RV64IA-NEXT: # Child Loop BB3_8 Depth 2
+; RV64IA-NEXT: # Child Loop BB3_5 Depth 2
+; RV64IA-NEXT: mv a3, a2
+; RV64IA-NEXT: bltu a2, a1, .LBB3_1
+; RV64IA-NEXT: # %bb.3: # in Loop: Header=BB3_2 Depth=1
+; RV64IA-NEXT: sub a4, a3, a1
+; RV64IA-NEXT: .LBB3_8: # Parent Loop BB3_2 Depth=1
+; RV64IA-NEXT: # => This Inner Loop Header: Depth=2
+; RV64IA-NEXT: lr.d.aqrl a2, (a0)
+; RV64IA-NEXT: bne a2, a3, .LBB3_2
+; RV64IA-NEXT: # %bb.9: # in Loop: Header=BB3_8 Depth=2
+; RV64IA-NEXT: sc.d.rl a5, a4, (a0)
+; RV64IA-NEXT: bnez a5, .LBB3_8
+; RV64IA-NEXT: # %bb.10:
+; RV64IA-NEXT: .LBB3_4: # %atomicrmw.end
+; RV64IA-NEXT: mv a0, a2
+; RV64IA-NEXT: ret
+ %result = atomicrmw usub_cond ptr %ptr, i64 %val seq_cst
+ ret i64 %result
+}
+
+define i8 @atomicrmw_usub_sat_i8(ptr %ptr, i8 %val) {
+; RV32I-LABEL: atomicrmw_usub_sat_i8:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -32
+; RV32I-NEXT: .cfi_def_cfa_offset 32
+; RV32I-NEXT: sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s2, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT: .cfi_offset ra, -4
+; RV32I-NEXT: .cfi_offset s0, -8
+; RV32I-NEXT: .cfi_offset s1, -12
+; RV32I-NEXT: .cfi_offset s2, -16
+; RV32I-NEXT: mv s0, a0
+; RV32I-NEXT: lbu a3, 0(a0)
+; RV32I-NEXT: mv s1, a1
+; RV32I-NEXT: andi s2, a1, 255
+; RV32I-NEXT: .LBB4_1: # %atomicrmw.start
+; RV32I-NEXT: # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT: andi a0, a3, 255
+; RV32I-NEXT: sltu a0, a0, s2
+; RV32I-NEXT: sub a1, a3, s1
+; RV32I-NEXT: addi a0, a0, -1
+; RV32I-NEXT: and a2, a0, a1
+; RV32I-NEXT: sb a3, 15(sp)
+; RV32I-NEXT: addi a1, sp, 15
+; RV32I-NEXT: li a3, 5
+; RV32I-NEXT: li a4, 5
+; RV32I-NEXT: mv a0, s0
+; RV32I-NEXT: call __atomic_compare_exchange_1
+; RV32I-NEXT: lbu a3, 15(sp)
+; RV32I-NEXT: beqz a0, .LBB4_1
+; RV32I-NEXT: # %bb.2: # %atomicrmw.end
+; RV32I-NEXT: mv a0, a3
+; RV32I-NEXT: lw ra, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 32
+; RV32I-NEXT: ret
+;
+; RV32IA-LABEL: atomicrmw_usub_sat_i8:
+; RV32IA: # %bb.0:
+; RV32IA-NEXT: andi a2, a0, -4
+; RV32IA-NEXT: slli a3, a0, 3
+; RV32IA-NEXT: andi a0, a3, 24
+; RV32IA-NEXT: li a4, 255
+; RV32IA-NEXT: lw a5, 0(a2)
+; RV32IA-NEXT: sll a3, a4, a3
+; RV32IA-NEXT: not a3, a3
+; RV32IA-NEXT: andi a4, a1, 255
+; RV32IA-NEXT: .LBB4_1: # %atomicrmw.start
+; RV32IA-NEXT: # =>This Loop Header: Depth=1
+; RV32IA-NEXT: # Child Loop BB4_3 Depth 2
+; RV32IA-NEXT: mv a6, a5
+; RV32IA-NEXT: srl a5, a5, a0
+; RV32IA-NEXT: andi a7, a5, 255
+; RV32IA-NEXT: sltu a7, a7, a4
+; RV32IA-NEXT: sub a5, a5, a1
+; RV32IA-NEXT: addi a7, a7, -1
+; RV32IA-NEXT: and a5, a7, a5
+; RV32IA-NEXT: andi a5, a5, 255
+; RV32IA-NEXT: sll a5, a5, a0
+; RV32IA-NEXT: and a7, a6, a3
+; RV32IA-NEXT: or a7, a7, a5
+; RV32IA-NEXT: .LBB4_3: # %atomicrmw.start
+; RV32IA-NEXT: # Parent Loop BB4_1 Depth=1
+; RV32IA-NEXT: # => This Inner Loop Header: Depth=2
+; RV32IA-NEXT: lr.w.aqrl a5, (a2)
+; RV32IA-NEXT: bne a5, a6, .LBB4_1
+; RV32IA-NEXT: # %bb.4: # %atomicrmw.start
+; RV32IA-NEXT: # in Loop: Header=BB4_3 Depth=2
+; RV32IA-NEXT: sc.w.rl t0, a7, (a2)
+; RV32IA-NEXT: bnez t0, .LBB4_3
+; RV32IA-NEXT: # %bb.5: # %atomicrmw.start
+; RV32IA-NEXT: # %bb.2: # %atomicrmw.end
+; RV32IA-NEXT: srl a0, a5, a0
+; RV32IA-NEXT: ret
+;
+; RV64I-LABEL: atomicrmw_usub_sat_i8:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -48
+; RV64I-NEXT: .cfi_def_cfa_offset 48
+; RV64I-NEXT: sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s2, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT: .cfi_offset ra, -8
+; RV64I-NEXT: .cfi_offset s0, -16
+; RV64I-NEXT: .cfi_offset s1, -24
+; RV64I-NEXT: .cfi_offset s2, -32
+; RV64I-NEXT: mv s0, a0
+; RV64I-NEXT: lbu a3, 0(a0)
+; RV64I-NEXT: mv s1, a1
+; RV64I-NEXT: andi s2, a1, 255
+; RV64I-NEXT: .LBB4_1: # %atomicrmw.start
+; RV64I-NEXT: # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT: andi a0, a3, 255
+; RV64I-NEXT: sltu a0, a0, s2
+; RV64I-NEXT: sub a1, a3, s1
+; RV64I-NEXT: addi a0, a0, -1
+; RV64I-NEXT: and a2, a0, a1
+; RV64I-NEXT: sb a3, 15(sp)
+; RV64I-NEXT: addi a1, sp, 15
+; RV64I-NEXT: li a3, 5
+; RV64I-NEXT: li a4, 5
+; RV64I-NEXT: mv a0, s0
+; RV64I-NEXT: call __atomic_compare_exchange_1
+; RV64I-NEXT: lbu a3, 15(sp)
+; RV64I-NEXT: beqz a0, .LBB4_1
+; RV64I-NEXT: # %bb.2: # %atomicrmw.end
+; RV64I-NEXT: mv a0, a3
+; RV64I-NEXT: ld ra, 40(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 48
+; RV64I-NEXT: ret
+;
+; RV64IA-LABEL: atomicrmw_usub_sat_i8:
+; RV64IA: # %bb.0:
+; RV64IA-NEXT: andi a2, a0, -4
+; RV64IA-NEXT: slli a4, a0, 3
+; RV64IA-NEXT: andi a0, a4, 24
+; RV64IA-NEXT: li a5, 255
+; RV64IA-NEXT: lw a3, 0(a2)
+; RV64IA-NEXT: sllw a4, a5, a4
+; RV64IA-NEXT: not a4, a4
+; RV64IA-NEXT: andi a5, a1, 255
+; RV64IA-NEXT: .LBB4_1: # %atomicrmw.start
+; RV64IA-NEXT: # =>This Loop Header: Depth=1
+; RV64IA-NEXT: # Child Loop BB4_3 Depth 2
+; RV64IA-NEXT: srlw a6, a3, a0
+; RV64IA-NEXT: sext.w a7, a3
+; RV64IA-NEXT: andi t0, a6, 255
+; RV64IA-NEXT: sltu t0, t0, a5
+; RV64IA-NEXT: sub a6, a6, a1
+; RV64IA-NEXT: addi t0, t0, -1
+; RV64IA-NEXT: and a6, t0, a6
+; RV64IA-NEXT: andi a6, a6, 255
+; RV64IA-NEXT: sllw a6, a6, a0
+; RV64IA-NEXT: and a3, a3, a4
+; RV64IA-NEXT: or a6, a3, a6
+; RV64IA-NEXT: .LBB4_3: # %atomicrmw.start
+; RV64IA-NEXT: # Parent Loop BB4_1 Depth=1
+; RV64IA-NEXT: # => This Inner Loop Header: Depth=2
+; RV64IA-NEXT: lr.w.aqrl a3, (a2)
+; RV64IA-NEXT: bne a3, a7, .LBB4_1
+; RV64IA-NEXT: # %bb.4: # %atomicrmw.start
+; RV64IA-NEXT: # in Loop: Header=BB4_3 Depth=2
+; RV64IA-NEXT: sc.w.rl t0, a6, (a2)
+; RV64IA-NEXT: bnez t0, .LBB4_3
+; RV64IA-NEXT: # %bb.5: # %atomicrmw.start
+; RV64IA-NEXT: # %bb.2: # %atomicrmw.end
+; RV64IA-NEXT: srlw a0, a3, a0
+; RV64IA-NEXT: ret
+ %result = atomicrmw usub_sat ptr %ptr, i8 %val seq_cst
+ ret i8 %result
+}
+
+define i16 @atomicrmw_usub_sat_i16(ptr %ptr, i16 %val) {
+; RV32I-LABEL: atomicrmw_usub_sat_i16:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -32
+; RV32I-NEXT: .cfi_def_cfa_offset 32
+; RV32I-NEXT: sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s2, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s3, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: .cfi_offset ra, -4
+; RV32I-NEXT: .cfi_offset s0, -8
+; RV32I-NEXT: .cfi_offset s1, -12
+; RV32I-NEXT: .cfi_offset s2, -16
+; RV32I-NEXT: .cfi_offset s3, -20
+; RV32I-NEXT: mv s0, a1
+; RV32I-NEXT: mv s1, a0
+; RV32I-NEXT: lhu a1, 0(a0)
+; RV32I-NEXT: lui s2, 16
+; RV32I-NEXT: addi s2, s2, -1
+; RV32I-NEXT: and s3, s0, s2
+; RV32I-NEXT: .LBB5_1: # %atomicrmw.start
+; RV32I-NEXT: # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT: and a0, a1, s2
+; RV32I-NEXT: sltu a0, a0, s3
+; RV32I-NEXT: sub a2, a1, s0
+; RV32I-NEXT: addi a0, a0, -1
+; RV32I-NEXT: and a2, a0, a2
+; RV32I-NEXT: sh a1, 10(sp)
+; RV32I-NEXT: addi a1, sp, 10
+; RV32I-NEXT: li a3, 5
+; RV32I-NEXT: li a4, 5
+; RV32I-NEXT: mv a0, s1
+; RV32I-NEXT: call __atomic_compare_exchange_2
+; RV32I-NEXT: lh a1, 10(sp)
+; RV32I-NEXT: beqz a0, .LBB5_1
+; RV32I-NEXT: # %bb.2: # %atomicrmw.end
+; RV32I-NEXT: mv a0, a1
+; RV32I-NEXT: lw ra, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 32
+; RV32I-NEXT: ret
+;
+; RV32IA-LABEL: atomicrmw_usub_sat_i16:
+; RV32IA: # %bb.0:
+; RV32IA-NEXT: andi a2, a0, -4
+; RV32IA-NEXT: slli a4, a0, 3
+; RV32IA-NEXT: andi a0, a4, 24
+; RV32IA-NEXT: lui a3, 16
+; RV32IA-NEXT: addi a3, a3, -1
+; RV32IA-NEXT: lw a6, 0(a2)
+; RV32IA-NEXT: sll a4, a3, a4
+; RV32IA-NEXT: not a4, a4
+; RV32IA-NEXT: and a5, a1, a3
+; RV32IA-NEXT: .LBB5_1: # %atomicrmw.start
+; RV32IA-NEXT: # =>This Loop Header: Depth=1
+; RV32IA-NEXT: # Child Loop BB5_3 Depth 2
+; RV32IA-NEXT: mv a7, a6
+; RV32IA-NEXT: srl a6, a6, a0
+; RV32IA-NEXT: and t0, a6, a3
+; RV32IA-NEXT: sltu t0, t0, a5
+; RV32IA-NEXT: sub a6, a6, a1
+; RV32IA-NEXT: addi t0, t0, -1
+; RV32IA-NEXT: and a6, a6, a3
+; RV32IA-NEXT: and a6, t0, a6
+; RV32IA-NEXT: sll a6, a6, a0
+; RV32IA-NEXT: and t0, a7, a4
+; RV32IA-NEXT: or t0, t0, a6
+; RV32IA-NEXT: .LBB5_3: # %atomicrmw.start
+; RV32IA-NEXT: # Parent Loop BB5_1 Depth=1
+; RV32IA-NEXT: # => This Inner Loop Header: Depth=2
+; RV32IA-NEXT: lr.w.aqrl a6, (a2)
+; RV32IA-NEXT: bne a6, a7, .LBB5_1
+; RV32IA-NEXT: # %bb.4: # %atomicrmw.start
+; RV32IA-NEXT: # in Loop: Header=BB5_3 Depth=2
+; RV32IA-NEXT: sc.w.rl t1, t0, (a2)
+; RV32IA-NEXT: bnez t1, .LBB5_3
+; RV32IA-NEXT: # %bb.5: # %atomicrmw.start
+; RV32IA-NEXT: # %bb.2: # %atomicrmw.end
+; RV32IA-NEXT: srl a0, a6, a0
+; RV32IA-NEXT: ret
+;
+; RV64I-LABEL: atomicrmw_usub_sat_i16:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -48
+; RV64I-NEXT: .cfi_def_cfa_offset 48
+; RV64I-NEXT: sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s2, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s3, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: .cfi_offset ra, -8
+; RV64I-NEXT: .cfi_offset s0, -16
+; RV64I-NEXT: .cfi_offset s1, -24
+; RV64I-NEXT: .cfi_offset s2, -32
+; RV64I-NEXT: .cfi_offset s3, -40
+; RV64I-NEXT: mv s0, a1
+; RV64I-NEXT: mv s1, a0
+; RV64I-NEXT: lhu a1, 0(a0)
+; RV64I-NEXT: lui s2, 16
+; RV64I-NEXT: addiw s2, s2, -1
+; RV64I-NEXT: and s3, s0, s2
+; RV64I-NEXT: .LBB5_1: # %atomicrmw.start
+; RV64I-NEXT: # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT: and a0, a1, s2
+; RV64I-NEXT: sltu a0, a0, s3
+; RV64I-NEXT: sub a2, a1, s0
+; RV64I-NEXT: addi a0, a0, -1
+; RV64I-NEXT: and a2, a0, a2
+; RV64I-NEXT: sh a1, 6(sp)
+; RV64I-NEXT: addi a1, sp, 6
+; RV64I-NEXT: li a3, 5
+; RV64I-NEXT: li a4, 5
+; RV64I-NEXT: mv a0, s1
+; RV64I-NEXT: call __atomic_compare_exchange_2
+; RV64I-NEXT: lh a1, 6(sp)
+; RV64I-NEXT: beqz a0, .LBB5_1
+; RV64I-NEXT: # %bb.2: # %atomicrmw.end
+; RV64I-NEXT: mv a0, a1
+; RV64I-NEXT: ld ra, 40(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s3, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 48
+; RV64I-NEXT: ret
+;
+; RV64IA-LABEL: atomicrmw_usub_sat_i16:
+; RV64IA: # %bb.0:
+; RV64IA-NEXT: andi a2, a0, -4
+; RV64IA-NEXT: slli a5, a0, 3
+; RV64IA-NEXT: andi a0, a5, 24
+; RV64IA-NEXT: lui a3, 16
+; RV64IA-NEXT: addiw a3, a3, -1
+; RV64IA-NEXT: lw a4, 0(a2)
+; RV64IA-NEXT: sllw a5, a3, a5
+; RV64IA-NEXT: not a5, a5
+; RV64IA-NEXT: and a6, a1, a3
+; RV64IA-NEXT: .LBB5_1: # %atomicrmw.start
+; RV64IA-NEXT: # =>This Loop Header: Depth=1
+; RV64IA-NEXT: # Child Loop BB5_3 Depth 2
+; RV64IA-NEXT: srlw a7, a4, a0
+; RV64IA-NEXT: sext.w t0, a4
+; RV64IA-NEXT: and t1, a7, a3
+; RV64IA-NEXT: sltu t1, t1, a6
+; RV64IA-NEXT: sub a7, a7, a1
+; RV64IA-NEXT: addi t1, t1, -1
+; RV64IA-NEXT: and a7, a7, a3
+; RV64IA-NEXT: and a7, t1, a7
+; RV64IA-NEXT: sllw a7, a7, a0
+; RV64IA-NEXT: and a4, a4, a5
+; RV64IA-NEXT: or a7, a4, a7
+; RV64IA-NEXT: .LBB5_3: # %atomicrmw.start
+; RV64IA-NEXT: # Parent Loop BB5_1 Depth=1
+; RV64IA-NEXT: # => This Inner Loop Header: Depth=2
+; RV64IA-NEXT: lr.w.aqrl a4, (a2)
+; RV64IA-NEXT: bne a4, t0, .LBB5_1
+; RV64IA-NEXT: # %bb.4: # %atomicrmw.start
+; RV64IA-NEXT: # in Loop: Header=BB5_3 Depth=2
+; RV64IA-NEXT: sc.w.rl t1, a7, (a2)
+; RV64IA-NEXT: bnez t1, .LBB5_3
+; RV64IA-NEXT: # %bb.5: # %atomicrmw.start
+; RV64IA-NEXT: # %bb.2: # %atomicrmw.end
+; RV64IA-NEXT: srlw a0, a4, a0
+; RV64IA-NEXT: ret
+ %result = atomicrmw usub_sat ptr %ptr, i16 %val seq_cst
+ ret i16 %result
+}
+
+define i32 @atomicrmw_usub_sat_i32(ptr %ptr, i32 %val) {
+; RV32I-LABEL: atomicrmw_usub_sat_i32:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -16
+; RV32I-NEXT: .cfi_def_cfa_offset 16
+; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s1, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT: .cfi_offset ra, -4
+; RV32I-NEXT: .cfi_offset s0, -8
+; RV32I-NEXT: .cfi_offset s1, -12
+; RV32I-NEXT: mv s0, a0
+; RV32I-NEXT: lw a3, 0(a0)
+; RV32I-NEXT: mv s1, a1
+; RV32I-NEXT: .LBB6_1: # %atomicrmw.start
+; RV32I-NEXT: # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT: sltu a0, a3, s1
+; RV32I-NEXT: sub a1, a3, s1
+; RV32I-NEXT: addi a0, a0, -1
+; RV32I-NEXT: and a2, a0, a1
+; RV32I-NEXT: sw a3, 0(sp)
+; RV32I-NEXT: mv a1, sp
+; RV32I-NEXT: li a3, 5
+; RV32I-NEXT: li a4, 5
+; RV32I-NEXT: mv a0, s0
+; RV32I-NEXT: call __atomic_compare_exchange_4
+; RV32I-NEXT: lw a3, 0(sp)
+; RV32I-NEXT: beqz a0, .LBB6_1
+; RV32I-NEXT: # %bb.2: # %atomicrmw.end
+; RV32I-NEXT: mv a0, a3
+; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 16
+; RV32I-NEXT: ret
+;
+; RV32IA-LABEL: atomicrmw_usub_sat_i32:
+; RV32IA: # %bb.0:
+; RV32IA-NEXT: lw a2, 0(a0)
+; RV32IA-NEXT: .LBB6_1: # %atomicrmw.start
+; RV32IA-NEXT: # =>This Loop Header: Depth=1
+; RV32IA-NEXT: # Child Loop BB6_3 Depth 2
+; RV32IA-NEXT: mv a3, a2
+; RV32IA-NEXT: sltu a2, a2, a1
+; RV32IA-NEXT: sub a4, a3, a1
+; RV32IA-NEXT: addi a2, a2, -1
+; RV32IA-NEXT: and a4, a2, a4
+; RV32IA-NEXT: .LBB6_3: # %atomicrmw.start
+; RV32IA-NEXT: # Parent Loop BB6_1 Depth=1
+; RV32IA-NEXT: # => This Inner Loop Header: Depth=2
+; RV32IA-NEXT: lr.w.aqrl a2, (a0)
+; RV32IA-NEXT: bne a2, a3, .LBB6_1
+; RV32IA-NEXT: # %bb.4: # %atomicrmw.start
+; RV32IA-NEXT: # in Loop: Header=BB6_3 Depth=2
+; RV32IA-NEXT: sc.w.rl a5, a4, (a0)
+; RV32IA-NEXT: bnez a5, .LBB6_3
+; RV32IA-NEXT: # %bb.5: # %atomicrmw.start
+; RV32IA-NEXT: # %bb.2: # %atomicrmw.end
+; RV32IA-NEXT: mv a0, a2
+; RV32IA-NEXT: ret
+;
+; RV64I-LABEL: atomicrmw_usub_sat_i32:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -48
+; RV64I-NEXT: .cfi_def_cfa_offset 48
+; RV64I-NEXT: sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s2, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT: .cfi_offset ra, -8
+; RV64I-NEXT: .cfi_offset s0, -16
+; RV64I-NEXT: .cfi_offset s1, -24
+; RV64I-NEXT: .cfi_offset s2, -32
+; RV64I-NEXT: mv s0, a0
+; RV64I-NEXT: lw a3, 0(a0)
+; RV64I-NEXT: mv s1, a1
+; RV64I-NEXT: sext.w s2, a1
+; RV64I-NEXT: .LBB6_1: # %atomicrmw.start
+; RV64I-NEXT: # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT: sltu a0, a3, s2
+; RV64I-NEXT: subw a1, a3, s1
+; RV64I-NEXT: addi a0, a0, -1
+; RV64I-NEXT: and a2, a0, a1
+; RV64I-NEXT: sw a3, 12(sp)
+; RV64I-NEXT: addi a1, sp, 12
+; RV64I-NEXT: li a3, 5
+; RV64I-NEXT: li a4, 5
+; RV64I-NEXT: mv a0, s0
+; RV64I-NEXT: call __atomic_compare_exchange_4
+; RV64I-NEXT: lw a3, 12(sp)
+; RV64I-NEXT: beqz a0, .LBB6_1
+; RV64I-NEXT: # %bb.2: # %atomicrmw.end
+; RV64I-NEXT: mv a0, a3
+; RV64I-NEXT: ld ra, 40(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 48
+; RV64I-NEXT: ret
+;
+; RV64IA-LABEL: atomicrmw_usub_sat_i32:
+; RV64IA: # %bb.0:
+; RV64IA-NEXT: lw a2, 0(a0)
+; RV64IA-NEXT: sext.w a3, a1
+; RV64IA-NEXT: .LBB6_1: # %atomicrmw.start
+; RV64IA-NEXT: # =>This Loop Header: Depth=1
+; RV64IA-NEXT: # Child Loop BB6_3 Depth 2
+; RV64IA-NEXT: sext.w a4, a2
+; RV64IA-NEXT: sltu a5, a4, a3
+; RV64IA-NEXT: subw a2, a2, a1
+; RV64IA-NEXT: addi a5, a5, -1
+; RV64IA-NEXT: and a5, a5, a2
+; RV64IA-NEXT: .LBB6_3: # %atomicrmw.start
+; RV64IA-NEXT: # Parent Loop BB6_1 Depth=1
+; RV64IA-NEXT: # => This Inner Loop Header: Depth=2
+; RV64IA-NEXT: lr.w.aqrl a2, (a0)
+; RV64IA-NEXT: bne a2, a4, .LBB6_1
+; RV64IA-NEXT: # %bb.4: # %atomicrmw.start
+; RV64IA-NEXT: # in Loop: Header=BB6_3 Depth=2
+; RV64IA-NEXT: sc.w.rl a6, a5, (a0)
+; RV64IA-NEXT: bnez a6, .LBB6_3
+; RV64IA-NEXT: # %bb.5: # %atomicrmw.start
+; RV64IA-NEXT: # %bb.2: # %atomicrmw.end
+; RV64IA-NEXT: mv a0, a2
+; RV64IA-NEXT: ret
+ %result = atomicrmw usub_sat ptr %ptr, i32 %val seq_cst
+ ret i32 %result
+}
+
+define i64 @atomicrmw_usub_sat_i64(ptr %ptr, i64 %val) {
+; RV32I-LABEL: atomicrmw_usub_sat_i64:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -32
+; RV32I-NEXT: .cfi_def_cfa_offset 32
+; RV32I-NEXT: sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s2, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT: .cfi_offset ra, -4
+; RV32I-NEXT: .cfi_offset s0, -8
+; RV32I-NEXT: .cfi_offset s1, -12
+; RV32I-NEXT: .cfi_offset s2, -16
+; RV32I-NEXT: mv s0, a0
+; RV32I-NEXT: lw a5, 4(a0)
+; RV32I-NEXT: lw a4, 0(a0)
+; RV32I-NEXT: mv s1, a2
+; RV32I-NEXT: mv s2, a1
+; RV32I-NEXT: j .LBB7_2
+; RV32I-NEXT: .LBB7_1: # %atomicrmw.start
+; RV32I-NEXT: # in Loop: Header=BB7_2 Depth=1
+; RV32I-NEXT: xori a1, a1, 1
+; RV32I-NEXT: sub a2, a5, s1
+; RV32I-NEXT: sub a2, a2, a0
+; RV32I-NEXT: sub a0, a4, s2
+; RV32I-NEXT: neg a1, a1
+; RV32I-NEXT: and a3, a1, a2
+; RV32I-NEXT: and a2, a1, a0
+; RV32I-NEXT: sw a4, 8(sp)
+; RV32I-NEXT: sw a5, 12(sp)
+; RV32I-NEXT: addi a1, sp, 8
+; RV32I-NEXT: li a4, 5
+; RV32I-NEXT: li a5, 5
+; RV32I-NEXT: mv a0, s0
+; RV32I-NEXT: call __atomic_compare_exchange_8
+; RV32I-NEXT: lw a5, 12(sp)
+; RV32I-NEXT: lw a4, 8(sp)
+; RV32I-NEXT: bnez a0, .LBB7_4
+; RV32I-NEXT: .LBB7_2: # %atomicrmw.start
+; RV32I-NEXT: # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT: sltu a0, a4, s2
+; RV32I-NEXT: mv a1, a0
+; RV32I-NEXT: beq a5, s1, .LBB7_1
+; RV32I-NEXT: # %bb.3: # %atomicrmw.start
+; RV32I-NEXT: # in Loop: Header=BB7_2 Depth=1
+; RV32I-NEXT: sltu a1, a5, s1
+; RV32I-NEXT: j .LBB7_1
+; RV32I-NEXT: .LBB7_4: # %atomicrmw.end
+; RV32I-NEXT: mv a0, a4
+; RV32I-NEXT: mv a1, a5
+; RV32I-NEXT: lw ra, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 32
+; RV32I-NEXT: ret
+;
+; RV32IA-LABEL: atomicrmw_usub_sat_i64:
+; RV32IA: # %bb.0:
+; RV32IA-NEXT: addi sp, sp, -32
+; RV32IA-NEXT: .cfi_def_cfa_offset 32
+; RV32IA-NEXT: sw ra, 28(sp) # 4-byte Folded Spill
+; RV32IA-NEXT: sw s0, 24(sp) # 4-byte Folded Spill
+; RV32IA-NEXT: sw s1, 20(sp) # 4-byte Folded Spill
+; RV32IA-NEXT: sw s2, 16(sp) # 4-byte Folded Spill
+; RV32IA-NEXT: .cfi_offset ra, -4
+; RV32IA-NEXT: .cfi_offset s0, -8
+; RV32IA-NEXT: .cfi_offset s1, -12
+; RV32IA-NEXT: .cfi_offset s2, -16
+; RV32IA-NEXT: mv s0, a0
+; RV32IA-NEXT: lw a5, 4(a0)
+; RV32IA-NEXT: lw a4, 0(a0)
+; RV32IA-NEXT: mv s1, a2
+; RV32IA-NEXT: mv s2, a1
+; RV32IA-NEXT: j .LBB7_2
+; RV32IA-NEXT: .LBB7_1: # %atomicrmw.start
+; RV32IA-NEXT: # in Loop: Header=BB7_2 Depth=1
+; RV32IA-NEXT: xori a1, a1, 1
+; RV32IA-NEXT: sub a2, a5, s1
+; RV32IA-NEXT: sub a2, a2, a0
+; RV32IA-NEXT: sub a0, a4, s2
+; RV32IA-NEXT: neg a1, a1
+; RV32IA-NEXT: and a3, a1, a2
+; RV32IA-NEXT: and a2, a1, a0
+; RV32IA-NEXT: sw a4, 8(sp)
+; RV32IA-NEXT: sw a5, 12(sp)
+; RV32IA-NEXT: addi a1, sp, 8
+; RV32IA-NEXT: li a4, 5
+; RV32IA-NEXT: li a5, 5
+; RV32IA-NEXT: mv a0, s0
+; RV32IA-NEXT: call __atomic_compare_exchange_8
+; RV32IA-NEXT: lw a5, 12(sp)
+; RV32IA-NEXT: lw a4, 8(sp)
+; RV32IA-NEXT: bnez a0, .LBB7_4
+; RV32IA-NEXT: .LBB7_2: # %atomicrmw.start
+; RV32IA-NEXT: # =>This Inner Loop Header: Depth=1
+; RV32IA-NEXT: sltu a0, a4, s2
+; RV32IA-NEXT: mv a1, a0
+; RV32IA-NEXT: beq a5, s1, .LBB7_1
+; RV32IA-NEXT: # %bb.3: # %atomicrmw.start
+; RV32IA-NEXT: # in Loop: Header=BB7_2 Depth=1
+; RV32IA-NEXT: sltu a1, a5, s1
+; RV32IA-NEXT: j .LBB7_1
+; RV32IA-NEXT: .LBB7_4: # %atomicrmw.end
+; RV32IA-NEXT: mv a0, a4
+; RV32IA-NEXT: mv a1, a5
+; RV32IA-NEXT: lw ra, 28(sp) # 4-byte Folded Reload
+; RV32IA-NEXT: lw s0, 24(sp) # 4-byte Folded Reload
+; RV32IA-NEXT: lw s1, 20(sp) # 4-byte Folded Reload
+; RV32IA-NEXT: lw s2, 16(sp) # 4-byte Folded Reload
+; RV32IA-NEXT: addi sp, sp, 32
+; RV32IA-NEXT: ret
+;
+; RV64I-LABEL: atomicrmw_usub_sat_i64:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -32
+; RV64I-NEXT: .cfi_def_cfa_offset 32
+; RV64I-NEXT: sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s1, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: .cfi_offset ra, -8
+; RV64I-NEXT: .cfi_offset s0, -16
+; RV64I-NEXT: .cfi_offset s1, -24
+; RV64I-NEXT: mv s0, a0
+; RV64I-NEXT: ld a3, 0(a0)
+; RV64I-NEXT: mv s1, a1
+; RV64I-NEXT: .LBB7_1: # %atomicrmw.start
+; RV64I-NEXT: # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT: sltu a0, a3, s1
+; RV64I-NEXT: sub a1, a3, s1
+; RV64I-NEXT: addi a0, a0, -1
+; RV64I-NEXT: and a2, a0, a1
+; RV64I-NEXT: sd a3, 0(sp)
+; RV64I-NEXT: mv a1, sp
+; RV64I-NEXT: li a3, 5
+; RV64I-NEXT: li a4, 5
+; RV64I-NEXT: mv a0, s0
+; RV64I-NEXT: call __atomic_compare_exchange_8
+; RV64I-NEXT: ld a3, 0(sp)
+; RV64I-NEXT: beqz a0, .LBB7_1
+; RV64I-NEXT: # %bb.2: # %atomicrmw.end
+; RV64I-NEXT: mv a0, a3
+; RV64I-NEXT: ld ra, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 32
+; RV64I-NEXT: ret
+;
+; RV64IA-LABEL: atomicrmw_usub_sat_i64:
+; RV64IA: # %bb.0:
+; RV64IA-NEXT: ld a2, 0(a0)
+; RV64IA-NEXT: .LBB7_1: # %atomicrmw.start
+; RV64IA-NEXT: # =>This Loop Header: Depth=1
+; RV64IA-NEXT: # Child Loop BB7_3 Depth 2
+; RV64IA-NEXT: mv a3, a2
+; RV64IA-NEXT: sltu a2, a2, a1
+; RV64IA-NEXT: sub a4, a3, a1
+; RV64IA-NEXT: addi a2, a2, -1
+; RV64IA-NEXT: and a4, a2, a4
+; RV64IA-NEXT: .LBB7_3: # %atomicrmw.start
+; RV64IA-NEXT: # Parent Loop BB7_1 Depth=1
+; RV64IA-NEXT: # => This Inner Loop Header: Depth=2
+; RV64IA-NEXT: lr.d.aqrl a2, (a0)
+; RV64IA-NEXT: bne a2, a3, .LBB7_1
+; RV64IA-NEXT: # %bb.4: # %atomicrmw.start
+; RV64IA-NEXT: # in Loop: Header=BB7_3 Depth=2
+; RV64IA-NEXT: sc.d.rl a5, a4, (a0)
+; RV64IA-NEXT: bnez a5, .LBB7_3
+; RV64IA-NEXT: # %bb.5: # %atomicrmw.start
+; RV64IA-NEXT: # %bb.2: # %atomicrmw.end
+; RV64IA-NEXT: mv a0, a2
+; RV64IA-NEXT: ret
+ %result = atomicrmw usub_sat ptr %ptr, i64 %val seq_cst
+ ret i64 %result
+}
diff --git a/llvm/test/CodeGen/SPARC/atomicrmw-cond-sub-clamp.ll b/llvm/test/CodeGen/SPARC/atomicrmw-cond-sub-clamp.ll
new file mode 100644
index 00000000000000..e092facdc5fea9
--- /dev/null
+++ b/llvm/test/CodeGen/SPARC/atomicrmw-cond-sub-clamp.ll
@@ -0,0 +1,326 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -march=sparc -mcpu=v9 < %s | FileCheck %s
+
+define i8 @atomicrmw_usub_cond_i8(ptr %ptr, i8 %val) {
+; CHECK-LABEL: atomicrmw_usub_cond_i8:
+; CHECK: .cfi_startproc
+; CHECK-NEXT: ! %bb.0:
+; CHECK-NEXT: membar #LoadLoad | #StoreLoad | #LoadStore | #StoreStore
+; CHECK-NEXT: and %o0, -4, %o2
+; CHECK-NEXT: mov 3, %o3
+; CHECK-NEXT: andn %o3, %o0, %o0
+; CHECK-NEXT: sll %o0, 3, %o0
+; CHECK-NEXT: mov 255, %o3
+; CHECK-NEXT: ld [%o2], %o5
+; CHECK-NEXT: sll %o3, %o0, %o3
+; CHECK-NEXT: xor %o3, -1, %o3
+; CHECK-NEXT: and %o1, 255, %o4
+; CHECK-NEXT: .LBB0_1: ! %atomicrmw.start
+; CHECK-NEXT: ! =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: mov %o5, %g2
+; CHECK-NEXT: srl %o5, %o0, %o5
+; CHECK-NEXT: and %o5, 255, %g3
+; CHECK-NEXT: sub %o5, %o1, %o5
+; CHECK-NEXT: cmp %g3, %o4
+; CHECK-NEXT: mov %o1, %g3
+; CHECK-NEXT: movcc %icc, %o5, %g3
+; CHECK-NEXT: and %g3, 255, %o5
+; CHECK-NEXT: sll %o5, %o0, %o5
+; CHECK-NEXT: and %g2, %o3, %g3
+; CHECK-NEXT: or %g3, %o5, %o5
+; CHECK-NEXT: cas [%o2], %g2, %o5
+; CHECK-NEXT: mov %g0, %g3
+; CHECK-NEXT: cmp %o5, %g2
+; CHECK-NEXT: move %icc, 1, %g3
+; CHECK-NEXT: cmp %g3, 1
+; CHECK-NEXT: bne %icc, .LBB0_1
+; CHECK-NEXT: nop
+; CHECK-NEXT: ! %bb.2: ! %atomicrmw.end
+; CHECK-NEXT: srl %o5, %o0, %o0
+; CHECK-NEXT: membar #LoadLoad | #StoreLoad | #LoadStore | #StoreStore
+; CHECK-NEXT: retl
+; CHECK-NEXT: nop
+ %result = atomicrmw usub_cond ptr %ptr, i8 %val seq_cst
+ ret i8 %result
+}
+
+define i16 @atomicrmw_usub_cond_i16(ptr %ptr, i16 %val) {
+; CHECK-LABEL: atomicrmw_usub_cond_i16:
+; CHECK: .cfi_startproc
+; CHECK-NEXT: ! %bb.0:
+; CHECK-NEXT: membar #LoadLoad | #StoreLoad | #LoadStore | #StoreStore
+; CHECK-NEXT: and %o0, -4, %o2
+; CHECK-NEXT: and %o0, 3, %o0
+; CHECK-NEXT: xor %o0, 2, %o0
+; CHECK-NEXT: sll %o0, 3, %o0
+; CHECK-NEXT: sethi 63, %o3
+; CHECK-NEXT: or %o3, 1023, %o3
+; CHECK-NEXT: ld [%o2], %g2
+; CHECK-NEXT: sll %o3, %o0, %o4
+; CHECK-NEXT: xor %o4, -1, %o4
+; CHECK-NEXT: and %o1, %o3, %o5
+; CHECK-NEXT: .LBB1_1: ! %atomicrmw.start
+; CHECK-NEXT: ! =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: mov %g2, %g3
+; CHECK-NEXT: srl %g2, %o0, %g2
+; CHECK-NEXT: and %g2, %o3, %g4
+; CHECK-NEXT: sub %g2, %o1, %g2
+; CHECK-NEXT: cmp %g4, %o5
+; CHECK-NEXT: mov %o1, %g4
+; CHECK-NEXT: movcc %icc, %g2, %g4
+; CHECK-NEXT: and %g4, %o3, %g2
+; CHECK-NEXT: sll %g2, %o0, %g2
+; CHECK-NEXT: and %g3, %o4, %g4
+; CHECK-NEXT: or %g4, %g2, %g2
+; CHECK-NEXT: cas [%o2], %g3, %g2
+; CHECK-NEXT: mov %g0, %g4
+; CHECK-NEXT: cmp %g2, %g3
+; CHECK-NEXT: move %icc, 1, %g4
+; CHECK-NEXT: cmp %g4, 1
+; CHECK-NEXT: bne %icc, .LBB1_1
+; CHECK-NEXT: nop
+; CHECK-NEXT: ! %bb.2: ! %atomicrmw.end
+; CHECK-NEXT: srl %g2, %o0, %o0
+; CHECK-NEXT: membar #LoadLoad | #StoreLoad | #LoadStore | #StoreStore
+; CHECK-NEXT: retl
+; CHECK-NEXT: nop
+ %result = atomicrmw usub_cond ptr %ptr, i16 %val seq_cst
+ ret i16 %result
+}
+
+define i32 @atomicrmw_usub_cond_i32(ptr %ptr, i32 %val) {
+; CHECK-LABEL: atomicrmw_usub_cond_i32:
+; CHECK: .cfi_startproc
+; CHECK-NEXT: ! %bb.0:
+; CHECK-NEXT: membar #LoadLoad | #StoreLoad | #LoadStore | #StoreStore
+; CHECK-NEXT: ld [%o0], %o2
+; CHECK-NEXT: .LBB2_1: ! %atomicrmw.start
+; CHECK-NEXT: ! =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: mov %o2, %o3
+; CHECK-NEXT: sub %o2, %o1, %o4
+; CHECK-NEXT: cmp %o2, %o1
+; CHECK-NEXT: mov %o1, %o2
+; CHECK-NEXT: movcc %icc, %o4, %o2
+; CHECK-NEXT: cas [%o0], %o3, %o2
+; CHECK-NEXT: mov %g0, %o4
+; CHECK-NEXT: cmp %o2, %o3
+; CHECK-NEXT: move %icc, 1, %o4
+; CHECK-NEXT: cmp %o4, 1
+; CHECK-NEXT: bne %icc, .LBB2_1
+; CHECK-NEXT: nop
+; CHECK-NEXT: ! %bb.2: ! %atomicrmw.end
+; CHECK-NEXT: membar #LoadLoad | #StoreLoad | #LoadStore | #StoreStore
+; CHECK-NEXT: retl
+; CHECK-NEXT: mov %o2, %o0
+ %result = atomicrmw usub_cond ptr %ptr, i32 %val seq_cst
+ ret i32 %result
+}
+
+define i64 @atomicrmw_usub_cond_i64(ptr %ptr, i64 %val) {
+; CHECK-LABEL: atomicrmw_usub_cond_i64:
+; CHECK: .cfi_startproc
+; CHECK-NEXT: ! %bb.0:
+; CHECK-NEXT: save %sp, -104, %sp
+; CHECK-NEXT: .cfi_def_cfa_register %fp
+; CHECK-NEXT: .cfi_window_save
+; CHECK-NEXT: .cfi_register %o7, %i7
+; CHECK-NEXT: ldd [%i0], %g2
+; CHECK-NEXT: add %fp, -8, %i3
+; CHECK-NEXT: mov 5, %i4
+; CHECK-NEXT: .LBB3_1: ! %atomicrmw.start
+; CHECK-NEXT: ! =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: mov %g0, %i5
+; CHECK-NEXT: mov %g0, %g4
+; CHECK-NEXT: cmp %g2, %i1
+; CHECK-NEXT: movcc %icc, 1, %i5
+; CHECK-NEXT: cmp %g3, %i2
+; CHECK-NEXT: movcc %icc, 1, %g4
+; CHECK-NEXT: cmp %g2, %i1
+; CHECK-NEXT: move %icc, %g4, %i5
+; CHECK-NEXT: subcc %g3, %i2, %g4
+; CHECK-NEXT: subxcc %g2, %i1, %l0
+; CHECK-NEXT: cmp %i5, 0
+; CHECK-NEXT: mov %i1, %o2
+; CHECK-NEXT: movne %icc, %l0, %o2
+; CHECK-NEXT: mov %i2, %o3
+; CHECK-NEXT: movne %icc, %g4, %o3
+; CHECK-NEXT: std %g2, [%fp+-8]
+; CHECK-NEXT: mov %i0, %o0
+; CHECK-NEXT: mov %i3, %o1
+; CHECK-NEXT: mov %i4, %o4
+; CHECK-NEXT: call __atomic_compare_exchange_8
+; CHECK-NEXT: mov %i4, %o5
+; CHECK-NEXT: cmp %o0, 0
+; CHECK-NEXT: be %icc, .LBB3_1
+; CHECK-NEXT: ldd [%fp+-8], %g2
+; CHECK-NEXT: ! %bb.2: ! %atomicrmw.end
+; CHECK-NEXT: mov %g2, %i0
+; CHECK-NEXT: ret
+; CHECK-NEXT: restore %g0, %g3, %o1
+ %result = atomicrmw usub_cond ptr %ptr, i64 %val seq_cst
+ ret i64 %result
+}
+
+define i8 @atomicrmw_usub_sat_i8(ptr %ptr, i8 %val) {
+; CHECK-LABEL: atomicrmw_usub_sat_i8:
+; CHECK: .cfi_startproc
+; CHECK-NEXT: ! %bb.0:
+; CHECK-NEXT: membar #LoadLoad | #StoreLoad | #LoadStore | #StoreStore
+; CHECK-NEXT: and %o0, -4, %o2
+; CHECK-NEXT: mov 3, %o3
+; CHECK-NEXT: andn %o3, %o0, %o0
+; CHECK-NEXT: sll %o0, 3, %o0
+; CHECK-NEXT: mov 255, %o3
+; CHECK-NEXT: ld [%o2], %o5
+; CHECK-NEXT: sll %o3, %o0, %o3
+; CHECK-NEXT: xor %o3, -1, %o3
+; CHECK-NEXT: and %o1, 255, %o4
+; CHECK-NEXT: .LBB4_1: ! %atomicrmw.start
+; CHECK-NEXT: ! =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: mov %g0, %g2
+; CHECK-NEXT: mov %o5, %g3
+; CHECK-NEXT: srl %o5, %o0, %o5
+; CHECK-NEXT: and %o5, 255, %g4
+; CHECK-NEXT: cmp %g4, %o4
+; CHECK-NEXT: sub %o5, %o1, %o5
+; CHECK-NEXT: movcc %icc, %o5, %g2
+; CHECK-NEXT: and %g2, 255, %o5
+; CHECK-NEXT: sll %o5, %o0, %o5
+; CHECK-NEXT: and %g3, %o3, %g2
+; CHECK-NEXT: or %g2, %o5, %o5
+; CHECK-NEXT: cas [%o2], %g3, %o5
+; CHECK-NEXT: mov %g0, %g2
+; CHECK-NEXT: cmp %o5, %g3
+; CHECK-NEXT: move %icc, 1, %g2
+; CHECK-NEXT: cmp %g2, 1
+; CHECK-NEXT: bne %icc, .LBB4_1
+; CHECK-NEXT: nop
+; CHECK-NEXT: ! %bb.2: ! %atomicrmw.end
+; CHECK-NEXT: srl %o5, %o0, %o0
+; CHECK-NEXT: membar #LoadLoad | #StoreLoad | #LoadStore | #StoreStore
+; CHECK-NEXT: retl
+; CHECK-NEXT: nop
+ %result = atomicrmw usub_sat ptr %ptr, i8 %val seq_cst
+ ret i8 %result
+}
+
+define i16 @atomicrmw_usub_sat_i16(ptr %ptr, i16 %val) {
+; CHECK-LABEL: atomicrmw_usub_sat_i16:
+; CHECK: .cfi_startproc
+; CHECK-NEXT: ! %bb.0:
+; CHECK-NEXT: save %sp, -96, %sp
+; CHECK-NEXT: .cfi_def_cfa_register %fp
+; CHECK-NEXT: .cfi_window_save
+; CHECK-NEXT: .cfi_register %o7, %i7
+; CHECK-NEXT: membar #LoadLoad | #StoreLoad | #LoadStore | #StoreStore
+; CHECK-NEXT: and %i0, -4, %i2
+; CHECK-NEXT: and %i0, 3, %i0
+; CHECK-NEXT: xor %i0, 2, %i0
+; CHECK-NEXT: sll %i0, 3, %i0
+; CHECK-NEXT: sethi 63, %i3
+; CHECK-NEXT: or %i3, 1023, %i3
+; CHECK-NEXT: ld [%i2], %g2
+; CHECK-NEXT: sll %i3, %i0, %i4
+; CHECK-NEXT: xor %i4, -1, %i4
+; CHECK-NEXT: and %i1, %i3, %i5
+; CHECK-NEXT: .LBB5_1: ! %atomicrmw.start
+; CHECK-NEXT: ! =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: mov %g0, %g3
+; CHECK-NEXT: mov %g2, %g4
+; CHECK-NEXT: srl %g2, %i0, %g2
+; CHECK-NEXT: and %g2, %i3, %l0
+; CHECK-NEXT: cmp %l0, %i5
+; CHECK-NEXT: sub %g2, %i1, %g2
+; CHECK-NEXT: movcc %icc, %g2, %g3
+; CHECK-NEXT: and %g3, %i3, %g2
+; CHECK-NEXT: sll %g2, %i0, %g2
+; CHECK-NEXT: and %g4, %i4, %g3
+; CHECK-NEXT: or %g3, %g2, %g2
+; CHECK-NEXT: cas [%i2], %g4, %g2
+; CHECK-NEXT: mov %g0, %g3
+; CHECK-NEXT: cmp %g2, %g4
+; CHECK-NEXT: move %icc, 1, %g3
+; CHECK-NEXT: cmp %g3, 1
+; CHECK-NEXT: bne %icc, .LBB5_1
+; CHECK-NEXT: nop
+; CHECK-NEXT: ! %bb.2: ! %atomicrmw.end
+; CHECK-NEXT: srl %g2, %i0, %i0
+; CHECK-NEXT: membar #LoadLoad | #StoreLoad | #LoadStore | #StoreStore
+; CHECK-NEXT: ret
+; CHECK-NEXT: restore
+ %result = atomicrmw usub_sat ptr %ptr, i16 %val seq_cst
+ ret i16 %result
+}
+
+define i32 @atomicrmw_usub_sat_i32(ptr %ptr, i32 %val) {
+; CHECK-LABEL: atomicrmw_usub_sat_i32:
+; CHECK: .cfi_startproc
+; CHECK-NEXT: ! %bb.0:
+; CHECK-NEXT: membar #LoadLoad | #StoreLoad | #LoadStore | #StoreStore
+; CHECK-NEXT: ld [%o0], %o2
+; CHECK-NEXT: .LBB6_1: ! %atomicrmw.start
+; CHECK-NEXT: ! =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: mov %o2, %o3
+; CHECK-NEXT: sub %o2, %o1, %o4
+; CHECK-NEXT: cmp %o2, %o1
+; CHECK-NEXT: mov %g0, %o2
+; CHECK-NEXT: movcc %icc, %o4, %o2
+; CHECK-NEXT: cas [%o0], %o3, %o2
+; CHECK-NEXT: mov %g0, %o4
+; CHECK-NEXT: cmp %o2, %o3
+; CHECK-NEXT: move %icc, 1, %o4
+; CHECK-NEXT: cmp %o4, 1
+; CHECK-NEXT: bne %icc, .LBB6_1
+; CHECK-NEXT: nop
+; CHECK-NEXT: ! %bb.2: ! %atomicrmw.end
+; CHECK-NEXT: membar #LoadLoad | #StoreLoad | #LoadStore | #StoreStore
+; CHECK-NEXT: retl
+; CHECK-NEXT: mov %o2, %o0
+ %result = atomicrmw usub_sat ptr %ptr, i32 %val seq_cst
+ ret i32 %result
+}
+
+define i64 @atomicrmw_usub_sat_i64(ptr %ptr, i64 %val) {
+; CHECK-LABEL: atomicrmw_usub_sat_i64:
+; CHECK: .cfi_startproc
+; CHECK-NEXT: ! %bb.0:
+; CHECK-NEXT: save %sp, -104, %sp
+; CHECK-NEXT: .cfi_def_cfa_register %fp
+; CHECK-NEXT: .cfi_window_save
+; CHECK-NEXT: .cfi_register %o7, %i7
+; CHECK-NEXT: ldd [%i0], %g2
+; CHECK-NEXT: add %fp, -8, %i3
+; CHECK-NEXT: mov 5, %i4
+; CHECK-NEXT: .LBB7_1: ! %atomicrmw.start
+; CHECK-NEXT: ! =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: mov %g0, %i5
+; CHECK-NEXT: mov %g0, %g4
+; CHECK-NEXT: mov %g0, %o2
+; CHECK-NEXT: mov %g0, %o3
+; CHECK-NEXT: cmp %g2, %i1
+; CHECK-NEXT: movcc %icc, 1, %i5
+; CHECK-NEXT: cmp %g3, %i2
+; CHECK-NEXT: movcc %icc, 1, %g4
+; CHECK-NEXT: cmp %g2, %i1
+; CHECK-NEXT: move %icc, %g4, %i5
+; CHECK-NEXT: subcc %g3, %i2, %g4
+; CHECK-NEXT: subxcc %g2, %i1, %l0
+; CHECK-NEXT: cmp %i5, 0
+; CHECK-NEXT: movne %icc, %l0, %o2
+; CHECK-NEXT: movne %icc, %g4, %o3
+; CHECK-NEXT: std %g2, [%fp+-8]
+; CHECK-NEXT: mov %i0, %o0
+; CHECK-NEXT: mov %i3, %o1
+; CHECK-NEXT: mov %i4, %o4
+; CHECK-NEXT: call __atomic_compare_exchange_8
+; CHECK-NEXT: mov %i4, %o5
+; CHECK-NEXT: cmp %o0, 0
+; CHECK-NEXT: be %icc, .LBB7_1
+; CHECK-NEXT: ldd [%fp+-8], %g2
+; CHECK-NEXT: ! %bb.2: ! %atomicrmw.end
+; CHECK-NEXT: mov %g2, %i0
+; CHECK-NEXT: ret
+; CHECK-NEXT: restore %g0, %g3, %o1
+ %result = atomicrmw usub_sat ptr %ptr, i64 %val seq_cst
+ ret i64 %result
+}
diff --git a/llvm/test/CodeGen/VE/Scalar/atomicrmw-cond-sub-clamp.ll b/llvm/test/CodeGen/VE/Scalar/atomicrmw-cond-sub-clamp.ll
new file mode 100644
index 00000000000000..58316c80326072
--- /dev/null
+++ b/llvm/test/CodeGen/VE/Scalar/atomicrmw-cond-sub-clamp.ll
@@ -0,0 +1,240 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=ve-unknown-unknown < %s | FileCheck %s
+
+define i8 @atomicrmw_usub_cond_i8(ptr %ptr, i8 %val) {
+; CHECK-LABEL: atomicrmw_usub_cond_i8:
+; CHECK: # %bb.0:
+; CHECK-NEXT: and %s1, %s1, (32)0
+; CHECK-NEXT: fencem 3
+; CHECK-NEXT: and %s2, -4, %s0
+; CHECK-NEXT: and %s0, 3, %s0
+; CHECK-NEXT: sla.w.sx %s0, %s0, 3
+; CHECK-NEXT: sla.w.sx %s3, (56)0, %s0
+; CHECK-NEXT: ldl.sx %s5, (, %s2)
+; CHECK-NEXT: xor %s3, -1, %s3
+; CHECK-NEXT: and %s3, %s3, (32)0
+; CHECK-NEXT: and %s4, %s1, (56)0
+; CHECK-NEXT: .LBB0_1: # %atomicrmw.start
+; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: or %s6, 0, %s5
+; CHECK-NEXT: and %s5, %s6, (32)0
+; CHECK-NEXT: srl %s5, %s5, %s0
+; CHECK-NEXT: and %s7, %s5, (56)0
+; CHECK-NEXT: subs.w.sx %s5, %s5, %s1
+; CHECK-NEXT: cmpu.w %s7, %s7, %s4
+; CHECK-NEXT: or %s34, 0, %s1
+; CHECK-NEXT: cmov.w.ge %s34, %s5, %s7
+; CHECK-NEXT: and %s5, %s34, (56)0
+; CHECK-NEXT: sla.w.sx %s5, %s5, %s0
+; CHECK-NEXT: and %s7, %s6, %s3
+; CHECK-NEXT: or %s5, %s7, %s5
+; CHECK-NEXT: cas.w %s5, (%s2), %s6
+; CHECK-NEXT: brne.w %s5, %s6, .LBB0_1
+; CHECK-NEXT: # %bb.2: # %atomicrmw.end
+; CHECK-NEXT: and %s1, %s5, (32)0
+; CHECK-NEXT: srl %s0, %s1, %s0
+; CHECK-NEXT: fencem 3
+; CHECK-NEXT: b.l.t (, %s10)
+ %result = atomicrmw usub_cond ptr %ptr, i8 %val seq_cst
+ ret i8 %result
+}
+
+define i16 @atomicrmw_usub_cond_i16(ptr %ptr, i16 %val) {
+; CHECK-LABEL: atomicrmw_usub_cond_i16:
+; CHECK: # %bb.0:
+; CHECK-NEXT: and %s1, %s1, (32)0
+; CHECK-NEXT: fencem 3
+; CHECK-NEXT: and %s2, -4, %s0
+; CHECK-NEXT: and %s0, 3, %s0
+; CHECK-NEXT: sla.w.sx %s0, %s0, 3
+; CHECK-NEXT: sla.w.sx %s3, (48)0, %s0
+; CHECK-NEXT: ldl.sx %s5, (, %s2)
+; CHECK-NEXT: xor %s3, -1, %s3
+; CHECK-NEXT: and %s3, %s3, (32)0
+; CHECK-NEXT: and %s4, %s1, (48)0
+; CHECK-NEXT: .LBB1_1: # %atomicrmw.start
+; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: or %s6, 0, %s5
+; CHECK-NEXT: and %s5, %s6, (32)0
+; CHECK-NEXT: srl %s5, %s5, %s0
+; CHECK-NEXT: and %s7, %s5, (48)0
+; CHECK-NEXT: subs.w.sx %s5, %s5, %s1
+; CHECK-NEXT: cmpu.w %s7, %s7, %s4
+; CHECK-NEXT: or %s34, 0, %s1
+; CHECK-NEXT: cmov.w.ge %s34, %s5, %s7
+; CHECK-NEXT: and %s5, %s34, (48)0
+; CHECK-NEXT: sla.w.sx %s5, %s5, %s0
+; CHECK-NEXT: and %s7, %s6, %s3
+; CHECK-NEXT: or %s5, %s7, %s5
+; CHECK-NEXT: cas.w %s5, (%s2), %s6
+; CHECK-NEXT: brne.w %s5, %s6, .LBB1_1
+; CHECK-NEXT: # %bb.2: # %atomicrmw.end
+; CHECK-NEXT: and %s1, %s5, (32)0
+; CHECK-NEXT: srl %s0, %s1, %s0
+; CHECK-NEXT: fencem 3
+; CHECK-NEXT: b.l.t (, %s10)
+ %result = atomicrmw usub_cond ptr %ptr, i16 %val seq_cst
+ ret i16 %result
+}
+
+define i32 @atomicrmw_usub_cond_i32(ptr %ptr, i32 %val) {
+; CHECK-LABEL: atomicrmw_usub_cond_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: fencem 3
+; CHECK-NEXT: ldl.sx %s2, (, %s0)
+; CHECK-NEXT: and %s1, %s1, (32)0
+; CHECK-NEXT: .LBB2_1: # %atomicrmw.start
+; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: or %s3, 0, %s2
+; CHECK-NEXT: subs.w.sx %s4, %s2, %s1
+; CHECK-NEXT: cmpu.w %s5, %s2, %s1
+; CHECK-NEXT: or %s2, 0, %s1
+; CHECK-NEXT: cmov.w.ge %s2, %s4, %s5
+; CHECK-NEXT: cas.w %s2, (%s0), %s3
+; CHECK-NEXT: brne.w %s2, %s3, .LBB2_1
+; CHECK-NEXT: # %bb.2: # %atomicrmw.end
+; CHECK-NEXT: fencem 3
+; CHECK-NEXT: or %s0, 0, %s2
+; CHECK-NEXT: b.l.t (, %s10)
+ %result = atomicrmw usub_cond ptr %ptr, i32 %val seq_cst
+ ret i32 %result
+}
+
+define i64 @atomicrmw_usub_cond_sub_i64(ptr %ptr, i64 %val) {
+; CHECK-LABEL: atomicrmw_usub_cond_i64:
+; CHECK: # %bb.0:
+; CHECK-NEXT: fencem 3
+; CHECK-NEXT: ld %s2, (, %s0)
+; CHECK-NEXT: .LBB3_1: # %atomicrmw.start
+; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: or %s3, 0, %s2
+; CHECK-NEXT: subs.l %s4, %s2, %s1
+; CHECK-NEXT: cmpu.l %s5, %s2, %s1
+; CHECK-NEXT: or %s2, 0, %s1
+; CHECK-NEXT: cmov.l.ge %s2, %s4, %s5
+; CHECK-NEXT: cas.l %s2, (%s0), %s3
+; CHECK-NEXT: brne.l %s2, %s3, .LBB3_1
+; CHECK-NEXT: # %bb.2: # %atomicrmw.end
+; CHECK-NEXT: fencem 3
+; CHECK-NEXT: or %s0, 0, %s2
+; CHECK-NEXT: b.l.t (, %s10)
+ %result = atomicrmw usub_cond ptr %ptr, i64 %val seq_cst
+ ret i64 %result
+}
+
+define i8 @atomicrmw_usub_sat_i8(ptr %ptr, i8 %val) {
+; CHECK-LABEL: atomicrmw_usub_sat_i8:
+; CHECK: # %bb.0:
+; CHECK-NEXT: and %s1, %s1, (32)0
+; CHECK-NEXT: fencem 3
+; CHECK-NEXT: and %s2, -4, %s0
+; CHECK-NEXT: and %s0, 3, %s0
+; CHECK-NEXT: sla.w.sx %s0, %s0, 3
+; CHECK-NEXT: sla.w.sx %s3, (56)0, %s0
+; CHECK-NEXT: ldl.sx %s5, (, %s2)
+; CHECK-NEXT: xor %s3, -1, %s3
+; CHECK-NEXT: and %s3, %s3, (32)0
+; CHECK-NEXT: and %s4, %s1, (56)0
+; CHECK-NEXT: .LBB4_1: # %atomicrmw.start
+; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: or %s6, 0, %s5
+; CHECK-NEXT: and %s5, %s6, (32)0
+; CHECK-NEXT: srl %s5, %s5, %s0
+; CHECK-NEXT: and %s7, %s5, (56)0
+; CHECK-NEXT: subs.w.sx %s5, %s5, %s1
+; CHECK-NEXT: cmpu.w %s7, %s7, %s4
+; CHECK-NEXT: cmov.w.lt %s5, (0)1, %s7
+; CHECK-NEXT: and %s5, %s5, (56)0
+; CHECK-NEXT: sla.w.sx %s5, %s5, %s0
+; CHECK-NEXT: and %s7, %s6, %s3
+; CHECK-NEXT: or %s5, %s7, %s5
+; CHECK-NEXT: cas.w %s5, (%s2), %s6
+; CHECK-NEXT: brne.w %s5, %s6, .LBB4_1
+; CHECK-NEXT: # %bb.2: # %atomicrmw.end
+; CHECK-NEXT: and %s1, %s5, (32)0
+; CHECK-NEXT: srl %s0, %s1, %s0
+; CHECK-NEXT: fencem 3
+; CHECK-NEXT: b.l.t (, %s10)
+ %result = atomicrmw usub_sat ptr %ptr, i8 %val seq_cst
+ ret i8 %result
+}
+
+define i16 @atomicrmw_usub_sat_i16(ptr %ptr, i16 %val) {
+; CHECK-LABEL: atomicrmw_usub_sat_i16:
+; CHECK: # %bb.0:
+; CHECK-NEXT: and %s1, %s1, (32)0
+; CHECK-NEXT: fencem 3
+; CHECK-NEXT: and %s2, -4, %s0
+; CHECK-NEXT: and %s0, 3, %s0
+; CHECK-NEXT: sla.w.sx %s0, %s0, 3
+; CHECK-NEXT: sla.w.sx %s3, (48)0, %s0
+; CHECK-NEXT: ldl.sx %s5, (, %s2)
+; CHECK-NEXT: xor %s3, -1, %s3
+; CHECK-NEXT: and %s3, %s3, (32)0
+; CHECK-NEXT: and %s4, %s1, (48)0
+; CHECK-NEXT: .LBB5_1: # %atomicrmw.start
+; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: or %s6, 0, %s5
+; CHECK-NEXT: and %s5, %s6, (32)0
+; CHECK-NEXT: srl %s5, %s5, %s0
+; CHECK-NEXT: and %s7, %s5, (48)0
+; CHECK-NEXT: subs.w.sx %s5, %s5, %s1
+; CHECK-NEXT: cmpu.w %s7, %s7, %s4
+; CHECK-NEXT: cmov.w.lt %s5, (0)1, %s7
+; CHECK-NEXT: and %s5, %s5, (48)0
+; CHECK-NEXT: sla.w.sx %s5, %s5, %s0
+; CHECK-NEXT: and %s7, %s6, %s3
+; CHECK-NEXT: or %s5, %s7, %s5
+; CHECK-NEXT: cas.w %s5, (%s2), %s6
+; CHECK-NEXT: brne.w %s5, %s6, .LBB5_1
+; CHECK-NEXT: # %bb.2: # %atomicrmw.end
+; CHECK-NEXT: and %s1, %s5, (32)0
+; CHECK-NEXT: srl %s0, %s1, %s0
+; CHECK-NEXT: fencem 3
+; CHECK-NEXT: b.l.t (, %s10)
+ %result = atomicrmw usub_sat ptr %ptr, i16 %val seq_cst
+ ret i16 %result
+}
+
+define i32 @atomicrmw_usub_sat_i32(ptr %ptr, i32 %val) {
+; CHECK-LABEL: atomicrmw_usub_sat_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: fencem 3
+; CHECK-NEXT: ldl.sx %s2, (, %s0)
+; CHECK-NEXT: and %s1, %s1, (32)0
+; CHECK-NEXT: .LBB6_1: # %atomicrmw.start
+; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: or %s3, 0, %s2
+; CHECK-NEXT: subs.w.sx %s2, %s2, %s1
+; CHECK-NEXT: cmpu.w %s4, %s3, %s1
+; CHECK-NEXT: cmov.w.lt %s2, (0)1, %s4
+; CHECK-NEXT: cas.w %s2, (%s0), %s3
+; CHECK-NEXT: brne.w %s2, %s3, .LBB6_1
+; CHECK-NEXT: # %bb.2: # %atomicrmw.end
+; CHECK-NEXT: fencem 3
+; CHECK-NEXT: or %s0, 0, %s2
+; CHECK-NEXT: b.l.t (, %s10)
+ %result = atomicrmw usub_sat ptr %ptr, i32 %val seq_cst
+ ret i32 %result
+}
+
+define i64 @atomicrmw_usub_sat_i64(ptr %ptr, i64 %val) {
+; CHECK-LABEL: atomicrmw_usub_sat_i64:
+; CHECK: # %bb.0:
+; CHECK-NEXT: fencem 3
+; CHECK-NEXT: ld %s2, (, %s0)
+; CHECK-NEXT: .LBB7_1: # %atomicrmw.start
+; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: or %s3, 0, %s2
+; CHECK-NEXT: subs.l %s2, %s2, %s1
+; CHECK-NEXT: cmpu.l %s4, %s3, %s1
+; CHECK-NEXT: cmov.l.lt %s2, (0)1, %s4
+; CHECK-NEXT: cas.l %s2, (%s0), %s3
+; CHECK-NEXT: brne.l %s2, %s3, .LBB7_1
+; CHECK-NEXT: # %bb.2: # %atomicrmw.end
+; CHECK-NEXT: fencem 3
+; CHECK-NEXT: or %s0, 0, %s2
+; CHECK-NEXT: b.l.t (, %s10)
+ %result = atomicrmw usub_sat ptr %ptr, i64 %val seq_cst
+ ret i64 %result
+}
diff --git a/llvm/test/CodeGen/WebAssembly/atomicrmw-cond-sub-clamp.ll b/llvm/test/CodeGen/WebAssembly/atomicrmw-cond-sub-clamp.ll
new file mode 100644
index 00000000000000..3c105fcff09a91
--- /dev/null
+++ b/llvm/test/CodeGen/WebAssembly/atomicrmw-cond-sub-clamp.ll
@@ -0,0 +1,355 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=wasm32-unknown-unknown < %s | FileCheck -check-prefix=WASM32 %s
+; RUN: llc -mtriple=wasm64-unknown-unknown < %s | FileCheck -check-prefix=WASM64 %s
+
+define i8 @atomicrmw_usub_cond_i8(ptr %ptr, i8 %val) {
+; WASM32-LABEL: atomicrmw_usub_cond_i8:
+; WASM32: .functype atomicrmw_usub_cond_i8 (i32, i32) -> (i32)
+; WASM32-NEXT: .local i32
+; WASM32-NEXT: # %bb.0:
+; WASM32-NEXT: local.get 0
+; WASM32-NEXT: local.get 0
+; WASM32-NEXT: i32.load8_u 0
+; WASM32-NEXT: local.tee 2
+; WASM32-NEXT: local.get 1
+; WASM32-NEXT: i32.sub
+; WASM32-NEXT: local.get 1
+; WASM32-NEXT: local.get 2
+; WASM32-NEXT: local.get 1
+; WASM32-NEXT: i32.const 255
+; WASM32-NEXT: i32.and
+; WASM32-NEXT: i32.ge_u
+; WASM32-NEXT: i32.select
+; WASM32-NEXT: i32.store8 0
+; WASM32-NEXT: local.get 2
+; WASM32-NEXT: # fallthrough-return
+;
+; WASM64-LABEL: atomicrmw_usub_cond_i8:
+; WASM64: .functype atomicrmw_usub_cond_i8 (i64, i32) -> (i32)
+; WASM64-NEXT: .local i32
+; WASM64-NEXT: # %bb.0:
+; WASM64-NEXT: local.get 0
+; WASM64-NEXT: local.get 0
+; WASM64-NEXT: i32.load8_u 0
+; WASM64-NEXT: local.tee 2
+; WASM64-NEXT: local.get 1
+; WASM64-NEXT: i32.sub
+; WASM64-NEXT: local.get 1
+; WASM64-NEXT: local.get 2
+; WASM64-NEXT: local.get 1
+; WASM64-NEXT: i32.const 255
+; WASM64-NEXT: i32.and
+; WASM64-NEXT: i32.ge_u
+; WASM64-NEXT: i32.select
+; WASM64-NEXT: i32.store8 0
+; WASM64-NEXT: local.get 2
+; WASM64-NEXT: # fallthrough-return
+ %result = atomicrmw usub_cond ptr %ptr, i8 %val seq_cst
+ ret i8 %result
+}
+
+define i16 @atomicrmw_usub_cond_i16(ptr %ptr, i16 %val) {
+; WASM32-LABEL: atomicrmw_usub_cond_i16:
+; WASM32: .functype atomicrmw_usub_cond_i16 (i32, i32) -> (i32)
+; WASM32-NEXT: .local i32
+; WASM32-NEXT: # %bb.0:
+; WASM32-NEXT: local.get 0
+; WASM32-NEXT: local.get 0
+; WASM32-NEXT: i32.load16_u 0
+; WASM32-NEXT: local.tee 2
+; WASM32-NEXT: local.get 1
+; WASM32-NEXT: i32.sub
+; WASM32-NEXT: local.get 1
+; WASM32-NEXT: local.get 2
+; WASM32-NEXT: local.get 1
+; WASM32-NEXT: i32.const 65535
+; WASM32-NEXT: i32.and
+; WASM32-NEXT: i32.ge_u
+; WASM32-NEXT: i32.select
+; WASM32-NEXT: i32.store16 0
+; WASM32-NEXT: local.get 2
+; WASM32-NEXT: # fallthrough-return
+;
+; WASM64-LABEL: atomicrmw_usub_cond_i16:
+; WASM64: .functype atomicrmw_usub_cond_i16 (i64, i32) -> (i32)
+; WASM64-NEXT: .local i32
+; WASM64-NEXT: # %bb.0:
+; WASM64-NEXT: local.get 0
+; WASM64-NEXT: local.get 0
+; WASM64-NEXT: i32.load16_u 0
+; WASM64-NEXT: local.tee 2
+; WASM64-NEXT: local.get 1
+; WASM64-NEXT: i32.sub
+; WASM64-NEXT: local.get 1
+; WASM64-NEXT: local.get 2
+; WASM64-NEXT: local.get 1
+; WASM64-NEXT: i32.const 65535
+; WASM64-NEXT: i32.and
+; WASM64-NEXT: i32.ge_u
+; WASM64-NEXT: i32.select
+; WASM64-NEXT: i32.store16 0
+; WASM64-NEXT: local.get 2
+; WASM64-NEXT: # fallthrough-return
+ %result = atomicrmw usub_cond ptr %ptr, i16 %val seq_cst
+ ret i16 %result
+}
+
+define i32 @atomicrmw_usub_cond_i32(ptr %ptr, i32 %val) {
+; WASM32-LABEL: atomicrmw_usub_cond_i32:
+; WASM32: .functype atomicrmw_usub_cond_i32 (i32, i32) -> (i32)
+; WASM32-NEXT: .local i32
+; WASM32-NEXT: # %bb.0:
+; WASM32-NEXT: local.get 0
+; WASM32-NEXT: local.get 0
+; WASM32-NEXT: i32.load 0
+; WASM32-NEXT: local.tee 2
+; WASM32-NEXT: local.get 1
+; WASM32-NEXT: i32.sub
+; WASM32-NEXT: local.get 1
+; WASM32-NEXT: local.get 2
+; WASM32-NEXT: local.get 1
+; WASM32-NEXT: i32.ge_u
+; WASM32-NEXT: i32.select
+; WASM32-NEXT: i32.store 0
+; WASM32-NEXT: local.get 2
+; WASM32-NEXT: # fallthrough-return
+;
+; WASM64-LABEL: atomicrmw_usub_cond_i32:
+; WASM64: .functype atomicrmw_usub_cond_i32 (i64, i32) -> (i32)
+; WASM64-NEXT: .local i32
+; WASM64-NEXT: # %bb.0:
+; WASM64-NEXT: local.get 0
+; WASM64-NEXT: local.get 0
+; WASM64-NEXT: i32.load 0
+; WASM64-NEXT: local.tee 2
+; WASM64-NEXT: local.get 1
+; WASM64-NEXT: i32.sub
+; WASM64-NEXT: local.get 1
+; WASM64-NEXT: local.get 2
+; WASM64-NEXT: local.get 1
+; WASM64-NEXT: i32.ge_u
+; WASM64-NEXT: i32.select
+; WASM64-NEXT: i32.store 0
+; WASM64-NEXT: local.get 2
+; WASM64-NEXT: # fallthrough-return
+ %result = atomicrmw usub_cond ptr %ptr, i32 %val seq_cst
+ ret i32 %result
+}
+
+define i64 @atomicrmw_usub_cond_i64(ptr %ptr, i64 %val) {
+; WASM32-LABEL: atomicrmw_usub_cond_i64:
+; WASM32: .functype atomicrmw_usub_cond_i64 (i32, i64) -> (i64)
+; WASM32-NEXT: .local i64
+; WASM32-NEXT: # %bb.0:
+; WASM32-NEXT: local.get 0
+; WASM32-NEXT: local.get 0
+; WASM32-NEXT: i64.load 0
+; WASM32-NEXT: local.tee 2
+; WASM32-NEXT: local.get 1
+; WASM32-NEXT: i64.sub
+; WASM32-NEXT: local.get 1
+; WASM32-NEXT: local.get 2
+; WASM32-NEXT: local.get 1
+; WASM32-NEXT: i64.ge_u
+; WASM32-NEXT: i64.select
+; WASM32-NEXT: i64.store 0
+; WASM32-NEXT: local.get 2
+; WASM32-NEXT: # fallthrough-return
+;
+; WASM64-LABEL: atomicrmw_usub_cond_i64:
+; WASM64: .functype atomicrmw_usub_cond_i64 (i64, i64) -> (i64)
+; WASM64-NEXT: .local i64
+; WASM64-NEXT: # %bb.0:
+; WASM64-NEXT: local.get 0
+; WASM64-NEXT: local.get 0
+; WASM64-NEXT: i64.load 0
+; WASM64-NEXT: local.tee 2
+; WASM64-NEXT: local.get 1
+; WASM64-NEXT: i64.sub
+; WASM64-NEXT: local.get 1
+; WASM64-NEXT: local.get 2
+; WASM64-NEXT: local.get 1
+; WASM64-NEXT: i64.ge_u
+; WASM64-NEXT: i64.select
+; WASM64-NEXT: i64.store 0
+; WASM64-NEXT: local.get 2
+; WASM64-NEXT: # fallthrough-return
+ %result = atomicrmw usub_cond ptr %ptr, i64 %val seq_cst
+ ret i64 %result
+}
+
+define i8 @atomicrmw_usub_sat_i8(ptr %ptr, i8 %val) {
+; WASM32-LABEL: atomicrmw_usub_sat_i8:
+; WASM32: .functype atomicrmw_usub_sat_i8 (i32, i32) -> (i32)
+; WASM32-NEXT: .local i32
+; WASM32-NEXT: # %bb.0:
+; WASM32-NEXT: local.get 0
+; WASM32-NEXT: local.get 0
+; WASM32-NEXT: i32.load8_u 0
+; WASM32-NEXT: local.tee 2
+; WASM32-NEXT: local.get 1
+; WASM32-NEXT: i32.sub
+; WASM32-NEXT: i32.const 0
+; WASM32-NEXT: local.get 2
+; WASM32-NEXT: local.get 1
+; WASM32-NEXT: i32.const 255
+; WASM32-NEXT: i32.and
+; WASM32-NEXT: i32.ge_u
+; WASM32-NEXT: i32.select
+; WASM32-NEXT: i32.store8 0
+; WASM32-NEXT: local.get 2
+; WASM32-NEXT: # fallthrough-return
+;
+; WASM64-LABEL: atomicrmw_usub_sat_i8:
+; WASM64: .functype atomicrmw_usub_sat_i8 (i64, i32) -> (i32)
+; WASM64-NEXT: .local i32
+; WASM64-NEXT: # %bb.0:
+; WASM64-NEXT: local.get 0
+; WASM64-NEXT: local.get 0
+; WASM64-NEXT: i32.load8_u 0
+; WASM64-NEXT: local.tee 2
+; WASM64-NEXT: local.get 1
+; WASM64-NEXT: i32.sub
+; WASM64-NEXT: i32.const 0
+; WASM64-NEXT: local.get 2
+; WASM64-NEXT: local.get 1
+; WASM64-NEXT: i32.const 255
+; WASM64-NEXT: i32.and
+; WASM64-NEXT: i32.ge_u
+; WASM64-NEXT: i32.select
+; WASM64-NEXT: i32.store8 0
+; WASM64-NEXT: local.get 2
+; WASM64-NEXT: # fallthrough-return
+ %result = atomicrmw usub_sat ptr %ptr, i8 %val seq_cst
+ ret i8 %result
+}
+
+define i16 @atomicrmw_usub_sat_i16(ptr %ptr, i16 %val) {
+; WASM32-LABEL: atomicrmw_usub_sat_i16:
+; WASM32: .functype atomicrmw_usub_sat_i16 (i32, i32) -> (i32)
+; WASM32-NEXT: .local i32
+; WASM32-NEXT: # %bb.0:
+; WASM32-NEXT: local.get 0
+; WASM32-NEXT: local.get 0
+; WASM32-NEXT: i32.load16_u 0
+; WASM32-NEXT: local.tee 2
+; WASM32-NEXT: local.get 1
+; WASM32-NEXT: i32.sub
+; WASM32-NEXT: i32.const 0
+; WASM32-NEXT: local.get 2
+; WASM32-NEXT: local.get 1
+; WASM32-NEXT: i32.const 65535
+; WASM32-NEXT: i32.and
+; WASM32-NEXT: i32.ge_u
+; WASM32-NEXT: i32.select
+; WASM32-NEXT: i32.store16 0
+; WASM32-NEXT: local.get 2
+; WASM32-NEXT: # fallthrough-return
+;
+; WASM64-LABEL: atomicrmw_usub_sat_i16:
+; WASM64: .functype atomicrmw_usub_sat_i16 (i64, i32) -> (i32)
+; WASM64-NEXT: .local i32
+; WASM64-NEXT: # %bb.0:
+; WASM64-NEXT: local.get 0
+; WASM64-NEXT: local.get 0
+; WASM64-NEXT: i32.load16_u 0
+; WASM64-NEXT: local.tee 2
+; WASM64-NEXT: local.get 1
+; WASM64-NEXT: i32.sub
+; WASM64-NEXT: i32.const 0
+; WASM64-NEXT: local.get 2
+; WASM64-NEXT: local.get 1
+; WASM64-NEXT: i32.const 65535
+; WASM64-NEXT: i32.and
+; WASM64-NEXT: i32.ge_u
+; WASM64-NEXT: i32.select
+; WASM64-NEXT: i32.store16 0
+; WASM64-NEXT: local.get 2
+; WASM64-NEXT: # fallthrough-return
+ %result = atomicrmw usub_sat ptr %ptr, i16 %val seq_cst
+ ret i16 %result
+}
+
+define i32 @atomicrmw_usub_sat_i32(ptr %ptr, i32 %val) {
+; WASM32-LABEL: atomicrmw_usub_sat_i32:
+; WASM32: .functype atomicrmw_usub_sat_i32 (i32, i32) -> (i32)
+; WASM32-NEXT: .local i32
+; WASM32-NEXT: # %bb.0:
+; WASM32-NEXT: local.get 0
+; WASM32-NEXT: local.get 0
+; WASM32-NEXT: i32.load 0
+; WASM32-NEXT: local.tee 2
+; WASM32-NEXT: local.get 1
+; WASM32-NEXT: i32.sub
+; WASM32-NEXT: i32.const 0
+; WASM32-NEXT: local.get 2
+; WASM32-NEXT: local.get 1
+; WASM32-NEXT: i32.ge_u
+; WASM32-NEXT: i32.select
+; WASM32-NEXT: i32.store 0
+; WASM32-NEXT: local.get 2
+; WASM32-NEXT: # fallthrough-return
+;
+; WASM64-LABEL: atomicrmw_usub_sat_i32:
+; WASM64: .functype atomicrmw_usub_sat_i32 (i64, i32) -> (i32)
+; WASM64-NEXT: .local i32
+; WASM64-NEXT: # %bb.0:
+; WASM64-NEXT: local.get 0
+; WASM64-NEXT: local.get 0
+; WASM64-NEXT: i32.load 0
+; WASM64-NEXT: local.tee 2
+; WASM64-NEXT: local.get 1
+; WASM64-NEXT: i32.sub
+; WASM64-NEXT: i32.const 0
+; WASM64-NEXT: local.get 2
+; WASM64-NEXT: local.get 1
+; WASM64-NEXT: i32.ge_u
+; WASM64-NEXT: i32.select
+; WASM64-NEXT: i32.store 0
+; WASM64-NEXT: local.get 2
+; WASM64-NEXT: # fallthrough-return
+ %result = atomicrmw usub_sat ptr %ptr, i32 %val seq_cst
+ ret i32 %result
+}
+
+define i64 @atomicrmw_usub_sat_i64(ptr %ptr, i64 %val) {
+; WASM32-LABEL: atomicrmw_usub_sat_i64:
+; WASM32: .functype atomicrmw_usub_sat_i64 (i32, i64) -> (i64)
+; WASM32-NEXT: .local i64
+; WASM32-NEXT: # %bb.0:
+; WASM32-NEXT: local.get 0
+; WASM32-NEXT: local.get 0
+; WASM32-NEXT: i64.load 0
+; WASM32-NEXT: local.tee 2
+; WASM32-NEXT: local.get 1
+; WASM32-NEXT: i64.sub
+; WASM32-NEXT: i64.const 0
+; WASM32-NEXT: local.get 2
+; WASM32-NEXT: local.get 1
+; WASM32-NEXT: i64.ge_u
+; WASM32-NEXT: i64.select
+; WASM32-NEXT: i64.store 0
+; WASM32-NEXT: local.get 2
+; WASM32-NEXT: # fallthrough-return
+;
+; WASM64-LABEL: atomicrmw_usub_sat_i64:
+; WASM64: .functype atomicrmw_usub_sat_i64 (i64, i64) -> (i64)
+; WASM64-NEXT: .local i64
+; WASM64-NEXT: # %bb.0:
+; WASM64-NEXT: local.get 0
+; WASM64-NEXT: local.get 0
+; WASM64-NEXT: i64.load 0
+; WASM64-NEXT: local.tee 2
+; WASM64-NEXT: local.get 1
+; WASM64-NEXT: i64.sub
+; WASM64-NEXT: i64.const 0
+; WASM64-NEXT: local.get 2
+; WASM64-NEXT: local.get 1
+; WASM64-NEXT: i64.ge_u
+; WASM64-NEXT: i64.select
+; WASM64-NEXT: i64.store 0
+; WASM64-NEXT: local.get 2
+; WASM64-NEXT: # fallthrough-return
+ %result = atomicrmw usub_sat ptr %ptr, i64 %val seq_cst
+ ret i64 %result
+}
diff --git a/llvm/test/CodeGen/X86/atomicrmw-cond-sub-clamp.ll b/llvm/test/CodeGen/X86/atomicrmw-cond-sub-clamp.ll
new file mode 100644
index 00000000000000..ada8c9fff0d112
--- /dev/null
+++ b/llvm/test/CodeGen/X86/atomicrmw-cond-sub-clamp.ll
@@ -0,0 +1,413 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple i686-pc-linux < %s | FileCheck %s --check-prefix=CHECK-32
+; RUN: llc -mtriple x86_64-pc-linux < %s | FileCheck %s --check-prefix=CHECK-64
+
+define i8 @atomicrmw_usub_cond_i8(ptr %ptr, i8 %val) {
+; CHECK-32-LABEL: atomicrmw_usub_cond_i8:
+; CHECK-32: # %bb.0:
+; CHECK-32-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; CHECK-32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; CHECK-32-NEXT: movzbl (%edx), %eax
+; CHECK-32-NEXT: jmp .LBB0_1
+; CHECK-32-NEXT: .p2align 4, 0x90
+; CHECK-32-NEXT: .LBB0_3: # %atomicrmw.start
+; CHECK-32-NEXT: # in Loop: Header=BB0_1 Depth=1
+; CHECK-32-NEXT: lock cmpxchgb %ah, (%edx)
+; CHECK-32-NEXT: je .LBB0_4
+; CHECK-32-NEXT: .LBB0_1: # %atomicrmw.start
+; CHECK-32-NEXT: # =>This Inner Loop Header: Depth=1
+; CHECK-32-NEXT: movb %al, %ah
+; CHECK-32-NEXT: subb %cl, %ah
+; CHECK-32-NEXT: jae .LBB0_3
+; CHECK-32-NEXT: # %bb.2: # %atomicrmw.start
+; CHECK-32-NEXT: # in Loop: Header=BB0_1 Depth=1
+; CHECK-32-NEXT: movb %cl, %ah
+; CHECK-32-NEXT: jmp .LBB0_3
+; CHECK-32-NEXT: .LBB0_4: # %atomicrmw.end
+; CHECK-32-NEXT: retl
+;
+; CHECK-64-LABEL: atomicrmw_usub_cond_i8:
+; CHECK-64: # %bb.0:
+; CHECK-64-NEXT: movzbl (%rdi), %eax
+; CHECK-64-NEXT: movzbl %sil, %ecx
+; CHECK-64-NEXT: .p2align 4, 0x90
+; CHECK-64-NEXT: .LBB0_1: # %atomicrmw.start
+; CHECK-64-NEXT: # =>This Inner Loop Header: Depth=1
+; CHECK-64-NEXT: movl %eax, %edx
+; CHECK-64-NEXT: subb %cl, %dl
+; CHECK-64-NEXT: movzbl %dl, %edx
+; CHECK-64-NEXT: cmovbl %ecx, %edx
+; CHECK-64-NEXT: lock cmpxchgb %dl, (%rdi)
+; CHECK-64-NEXT: jne .LBB0_1
+; CHECK-64-NEXT: # %bb.2: # %atomicrmw.end
+; CHECK-64-NEXT: retq
+ %result = atomicrmw usub_cond ptr %ptr, i8 %val seq_cst
+ ret i8 %result
+}
+
+define i16 @atomicrmw_usub_cond_i16(ptr %ptr, i16 %val) {
+; CHECK-32-LABEL: atomicrmw_usub_cond_i16:
+; CHECK-32: # %bb.0:
+; CHECK-32-NEXT: pushl %esi
+; CHECK-32-NEXT: .cfi_def_cfa_offset 8
+; CHECK-32-NEXT: .cfi_offset %esi, -8
+; CHECK-32-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; CHECK-32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; CHECK-32-NEXT: movzwl (%edx), %eax
+; CHECK-32-NEXT: jmp .LBB1_1
+; CHECK-32-NEXT: .p2align 4, 0x90
+; CHECK-32-NEXT: .LBB1_3: # %atomicrmw.start
+; CHECK-32-NEXT: # in Loop: Header=BB1_1 Depth=1
+; CHECK-32-NEXT: lock cmpxchgw %si, (%edx)
+; CHECK-32-NEXT: je .LBB1_4
+; CHECK-32-NEXT: .LBB1_1: # %atomicrmw.start
+; CHECK-32-NEXT: # =>This Inner Loop Header: Depth=1
+; CHECK-32-NEXT: movl %eax, %esi
+; CHECK-32-NEXT: subw %cx, %si
+; CHECK-32-NEXT: jae .LBB1_3
+; CHECK-32-NEXT: # %bb.2: # %atomicrmw.start
+; CHECK-32-NEXT: # in Loop: Header=BB1_1 Depth=1
+; CHECK-32-NEXT: movl %ecx, %esi
+; CHECK-32-NEXT: jmp .LBB1_3
+; CHECK-32-NEXT: .LBB1_4: # %atomicrmw.end
+; CHECK-32-NEXT: popl %esi
+; CHECK-32-NEXT: .cfi_def_cfa_offset 4
+; CHECK-32-NEXT: retl
+;
+; CHECK-64-LABEL: atomicrmw_usub_cond_i16:
+; CHECK-64: # %bb.0:
+; CHECK-64-NEXT: movzwl (%rdi), %eax
+; CHECK-64-NEXT: .p2align 4, 0x90
+; CHECK-64-NEXT: .LBB1_1: # %atomicrmw.start
+; CHECK-64-NEXT: # =>This Inner Loop Header: Depth=1
+; CHECK-64-NEXT: movl %eax, %ecx
+; CHECK-64-NEXT: subw %si, %cx
+; CHECK-64-NEXT: cmovbl %esi, %ecx
+; CHECK-64-NEXT: lock cmpxchgw %cx, (%rdi)
+; CHECK-64-NEXT: jne .LBB1_1
+; CHECK-64-NEXT: # %bb.2: # %atomicrmw.end
+; CHECK-64-NEXT: retq
+ %result = atomicrmw usub_cond ptr %ptr, i16 %val seq_cst
+ ret i16 %result
+}
+
+define i32 @atomicrmw_usub_cond_i32(ptr %ptr, i32 %val) {
+; CHECK-32-LABEL: atomicrmw_usub_cond_i32:
+; CHECK-32: # %bb.0:
+; CHECK-32-NEXT: pushl %esi
+; CHECK-32-NEXT: .cfi_def_cfa_offset 8
+; CHECK-32-NEXT: .cfi_offset %esi, -8
+; CHECK-32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; CHECK-32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; CHECK-32-NEXT: movl (%edx), %eax
+; CHECK-32-NEXT: jmp .LBB2_1
+; CHECK-32-NEXT: .p2align 4, 0x90
+; CHECK-32-NEXT: .LBB2_3: # %atomicrmw.start
+; CHECK-32-NEXT: # in Loop: Header=BB2_1 Depth=1
+; CHECK-32-NEXT: lock cmpxchgl %esi, (%edx)
+; CHECK-32-NEXT: je .LBB2_4
+; CHECK-32-NEXT: .LBB2_1: # %atomicrmw.start
+; CHECK-32-NEXT: # =>This Inner Loop Header: Depth=1
+; CHECK-32-NEXT: movl %eax, %esi
+; CHECK-32-NEXT: subl %ecx, %esi
+; CHECK-32-NEXT: jae .LBB2_3
+; CHECK-32-NEXT: # %bb.2: # %atomicrmw.start
+; CHECK-32-NEXT: # in Loop: Header=BB2_1 Depth=1
+; CHECK-32-NEXT: movl %ecx, %esi
+; CHECK-32-NEXT: jmp .LBB2_3
+; CHECK-32-NEXT: .LBB2_4: # %atomicrmw.end
+; CHECK-32-NEXT: popl %esi
+; CHECK-32-NEXT: .cfi_def_cfa_offset 4
+; CHECK-32-NEXT: retl
+;
+; CHECK-64-LABEL: atomicrmw_usub_cond_i32:
+; CHECK-64: # %bb.0:
+; CHECK-64-NEXT: movl (%rdi), %eax
+; CHECK-64-NEXT: .p2align 4, 0x90
+; CHECK-64-NEXT: .LBB2_1: # %atomicrmw.start
+; CHECK-64-NEXT: # =>This Inner Loop Header: Depth=1
+; CHECK-64-NEXT: movl %eax, %ecx
+; CHECK-64-NEXT: subl %esi, %ecx
+; CHECK-64-NEXT: cmovbl %esi, %ecx
+; CHECK-64-NEXT: lock cmpxchgl %ecx, (%rdi)
+; CHECK-64-NEXT: jne .LBB2_1
+; CHECK-64-NEXT: # %bb.2: # %atomicrmw.end
+; CHECK-64-NEXT: retq
+ %result = atomicrmw usub_cond ptr %ptr, i32 %val seq_cst
+ ret i32 %result
+}
+
+define i64 @atomicrmw_usub_cond_i64(ptr %ptr, i64 %val) {
+; CHECK-32-LABEL: atomicrmw_usub_cond_i64:
+; CHECK-32: # %bb.0:
+; CHECK-32-NEXT: pushl %ebp
+; CHECK-32-NEXT: .cfi_def_cfa_offset 8
+; CHECK-32-NEXT: pushl %ebx
+; CHECK-32-NEXT: .cfi_def_cfa_offset 12
+; CHECK-32-NEXT: pushl %edi
+; CHECK-32-NEXT: .cfi_def_cfa_offset 16
+; CHECK-32-NEXT: pushl %esi
+; CHECK-32-NEXT: .cfi_def_cfa_offset 20
+; CHECK-32-NEXT: .cfi_offset %esi, -20
+; CHECK-32-NEXT: .cfi_offset %edi, -16
+; CHECK-32-NEXT: .cfi_offset %ebx, -12
+; CHECK-32-NEXT: .cfi_offset %ebp, -8
+; CHECK-32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; CHECK-32-NEXT: movl {{[0-9]+}}(%esp), %edi
+; CHECK-32-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; CHECK-32-NEXT: movl (%ebp), %eax
+; CHECK-32-NEXT: movl 4(%ebp), %edx
+; CHECK-32-NEXT: jmp .LBB3_1
+; CHECK-32-NEXT: .p2align 4, 0x90
+; CHECK-32-NEXT: .LBB3_3: # %atomicrmw.start
+; CHECK-32-NEXT: # in Loop: Header=BB3_1 Depth=1
+; CHECK-32-NEXT: lock cmpxchg8b (%ebp)
+; CHECK-32-NEXT: je .LBB3_4
+; CHECK-32-NEXT: .LBB3_1: # %atomicrmw.start
+; CHECK-32-NEXT: # =>This Inner Loop Header: Depth=1
+; CHECK-32-NEXT: movl %eax, %ebx
+; CHECK-32-NEXT: subl %edi, %ebx
+; CHECK-32-NEXT: movl %edx, %ecx
+; CHECK-32-NEXT: sbbl %esi, %ecx
+; CHECK-32-NEXT: jae .LBB3_3
+; CHECK-32-NEXT: # %bb.2: # %atomicrmw.start
+; CHECK-32-NEXT: # in Loop: Header=BB3_1 Depth=1
+; CHECK-32-NEXT: movl %esi, %ecx
+; CHECK-32-NEXT: movl %edi, %ebx
+; CHECK-32-NEXT: jmp .LBB3_3
+; CHECK-32-NEXT: .LBB3_4: # %atomicrmw.end
+; CHECK-32-NEXT: popl %esi
+; CHECK-32-NEXT: .cfi_def_cfa_offset 16
+; CHECK-32-NEXT: popl %edi
+; CHECK-32-NEXT: .cfi_def_cfa_offset 12
+; CHECK-32-NEXT: popl %ebx
+; CHECK-32-NEXT: .cfi_def_cfa_offset 8
+; CHECK-32-NEXT: popl %ebp
+; CHECK-32-NEXT: .cfi_def_cfa_offset 4
+; CHECK-32-NEXT: retl
+;
+; CHECK-64-LABEL: atomicrmw_usub_cond_i64:
+; CHECK-64: # %bb.0:
+; CHECK-64-NEXT: movq (%rdi), %rax
+; CHECK-64-NEXT: .p2align 4, 0x90
+; CHECK-64-NEXT: .LBB3_1: # %atomicrmw.start
+; CHECK-64-NEXT: # =>This Inner Loop Header: Depth=1
+; CHECK-64-NEXT: movq %rax, %rcx
+; CHECK-64-NEXT: subq %rsi, %rcx
+; CHECK-64-NEXT: cmovbq %rsi, %rcx
+; CHECK-64-NEXT: lock cmpxchgq %rcx, (%rdi)
+; CHECK-64-NEXT: jne .LBB3_1
+; CHECK-64-NEXT: # %bb.2: # %atomicrmw.end
+; CHECK-64-NEXT: retq
+ %result = atomicrmw usub_cond ptr %ptr, i64 %val seq_cst
+ ret i64 %result
+}
+
+define i8 @atomicrmw_usub_sat_i8(ptr %ptr, i8 %val) {
+; CHECK-32-LABEL: atomicrmw_usub_sat_i8:
+; CHECK-32: # %bb.0:
+; CHECK-32-NEXT: pushl %ebx
+; CHECK-32-NEXT: .cfi_def_cfa_offset 8
+; CHECK-32-NEXT: .cfi_offset %ebx, -8
+; CHECK-32-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; CHECK-32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; CHECK-32-NEXT: movzbl (%edx), %eax
+; CHECK-32-NEXT: jmp .LBB4_1
+; CHECK-32-NEXT: .p2align 4, 0x90
+; CHECK-32-NEXT: .LBB4_3: # %atomicrmw.start
+; CHECK-32-NEXT: # in Loop: Header=BB4_1 Depth=1
+; CHECK-32-NEXT: lock cmpxchgb %bl, (%edx)
+; CHECK-32-NEXT: je .LBB4_4
+; CHECK-32-NEXT: .LBB4_1: # %atomicrmw.start
+; CHECK-32-NEXT: # =>This Inner Loop Header: Depth=1
+; CHECK-32-NEXT: movl %eax, %ebx
+; CHECK-32-NEXT: subb %cl, %bl
+; CHECK-32-NEXT: jae .LBB4_3
+; CHECK-32-NEXT: # %bb.2: # %atomicrmw.start
+; CHECK-32-NEXT: # in Loop: Header=BB4_1 Depth=1
+; CHECK-32-NEXT: xorl %ebx, %ebx
+; CHECK-32-NEXT: jmp .LBB4_3
+; CHECK-32-NEXT: .LBB4_4: # %atomicrmw.end
+; CHECK-32-NEXT: popl %ebx
+; CHECK-32-NEXT: .cfi_def_cfa_offset 4
+; CHECK-32-NEXT: retl
+;
+; CHECK-64-LABEL: atomicrmw_usub_sat_i8:
+; CHECK-64: # %bb.0:
+; CHECK-64-NEXT: movzbl (%rdi), %eax
+; CHECK-64-NEXT: xorl %ecx, %ecx
+; CHECK-64-NEXT: .p2align 4, 0x90
+; CHECK-64-NEXT: .LBB4_1: # %atomicrmw.start
+; CHECK-64-NEXT: # =>This Inner Loop Header: Depth=1
+; CHECK-64-NEXT: movl %eax, %edx
+; CHECK-64-NEXT: subb %sil, %dl
+; CHECK-64-NEXT: movzbl %dl, %edx
+; CHECK-64-NEXT: cmovbl %ecx, %edx
+; CHECK-64-NEXT: lock cmpxchgb %dl, (%rdi)
+; CHECK-64-NEXT: jne .LBB4_1
+; CHECK-64-NEXT: # %bb.2: # %atomicrmw.end
+; CHECK-64-NEXT: retq
+ %result = atomicrmw usub_sat ptr %ptr, i8 %val seq_cst
+ ret i8 %result
+}
+
+define i16 @atomicrmw_usub_sat_i16(ptr %ptr, i16 %val) {
+; CHECK-32-LABEL: atomicrmw_usub_sat_i16:
+; CHECK-32: # %bb.0:
+; CHECK-32-NEXT: pushl %esi
+; CHECK-32-NEXT: .cfi_def_cfa_offset 8
+; CHECK-32-NEXT: .cfi_offset %esi, -8
+; CHECK-32-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; CHECK-32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; CHECK-32-NEXT: movzwl (%edx), %eax
+; CHECK-32-NEXT: jmp .LBB5_1
+; CHECK-32-NEXT: .p2align 4, 0x90
+; CHECK-32-NEXT: .LBB5_3: # %atomicrmw.start
+; CHECK-32-NEXT: # in Loop: Header=BB5_1 Depth=1
+; CHECK-32-NEXT: lock cmpxchgw %si, (%edx)
+; CHECK-32-NEXT: je .LBB5_4
+; CHECK-32-NEXT: .LBB5_1: # %atomicrmw.start
+; CHECK-32-NEXT: # =>This Inner Loop Header: Depth=1
+; CHECK-32-NEXT: movl %eax, %esi
+; CHECK-32-NEXT: subw %cx, %si
+; CHECK-32-NEXT: jae .LBB5_3
+; CHECK-32-NEXT: # %bb.2: # %atomicrmw.start
+; CHECK-32-NEXT: # in Loop: Header=BB5_1 Depth=1
+; CHECK-32-NEXT: xorl %esi, %esi
+; CHECK-32-NEXT: jmp .LBB5_3
+; CHECK-32-NEXT: .LBB5_4: # %atomicrmw.end
+; CHECK-32-NEXT: popl %esi
+; CHECK-32-NEXT: .cfi_def_cfa_offset 4
+; CHECK-32-NEXT: retl
+;
+; CHECK-64-LABEL: atomicrmw_usub_sat_i16:
+; CHECK-64: # %bb.0:
+; CHECK-64-NEXT: movzwl (%rdi), %eax
+; CHECK-64-NEXT: xorl %ecx, %ecx
+; CHECK-64-NEXT: .p2align 4, 0x90
+; CHECK-64-NEXT: .LBB5_1: # %atomicrmw.start
+; CHECK-64-NEXT: # =>This Inner Loop Header: Depth=1
+; CHECK-64-NEXT: movl %eax, %edx
+; CHECK-64-NEXT: subw %si, %dx
+; CHECK-64-NEXT: cmovbl %ecx, %edx
+; CHECK-64-NEXT: lock cmpxchgw %dx, (%rdi)
+; CHECK-64-NEXT: jne .LBB5_1
+; CHECK-64-NEXT: # %bb.2: # %atomicrmw.end
+; CHECK-64-NEXT: retq
+ %result = atomicrmw usub_sat ptr %ptr, i16 %val seq_cst
+ ret i16 %result
+}
+
+define i32 @atomicrmw_usub_sat_i32(ptr %ptr, i32 %val) {
+; CHECK-32-LABEL: atomicrmw_usub_sat_i32:
+; CHECK-32: # %bb.0:
+; CHECK-32-NEXT: pushl %esi
+; CHECK-32-NEXT: .cfi_def_cfa_offset 8
+; CHECK-32-NEXT: .cfi_offset %esi, -8
+; CHECK-32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; CHECK-32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; CHECK-32-NEXT: movl (%edx), %eax
+; CHECK-32-NEXT: jmp .LBB6_1
+; CHECK-32-NEXT: .p2align 4, 0x90
+; CHECK-32-NEXT: .LBB6_3: # %atomicrmw.start
+; CHECK-32-NEXT: # in Loop: Header=BB6_1 Depth=1
+; CHECK-32-NEXT: lock cmpxchgl %esi, (%edx)
+; CHECK-32-NEXT: je .LBB6_4
+; CHECK-32-NEXT: .LBB6_1: # %atomicrmw.start
+; CHECK-32-NEXT: # =>This Inner Loop Header: Depth=1
+; CHECK-32-NEXT: movl %eax, %esi
+; CHECK-32-NEXT: subl %ecx, %esi
+; CHECK-32-NEXT: jae .LBB6_3
+; CHECK-32-NEXT: # %bb.2: # %atomicrmw.start
+; CHECK-32-NEXT: # in Loop: Header=BB6_1 Depth=1
+; CHECK-32-NEXT: xorl %esi, %esi
+; CHECK-32-NEXT: jmp .LBB6_3
+; CHECK-32-NEXT: .LBB6_4: # %atomicrmw.end
+; CHECK-32-NEXT: popl %esi
+; CHECK-32-NEXT: .cfi_def_cfa_offset 4
+; CHECK-32-NEXT: retl
+;
+; CHECK-64-LABEL: atomicrmw_usub_sat_i32:
+; CHECK-64: # %bb.0:
+; CHECK-64-NEXT: movl (%rdi), %eax
+; CHECK-64-NEXT: xorl %ecx, %ecx
+; CHECK-64-NEXT: .p2align 4, 0x90
+; CHECK-64-NEXT: .LBB6_1: # %atomicrmw.start
+; CHECK-64-NEXT: # =>This Inner Loop Header: Depth=1
+; CHECK-64-NEXT: movl %eax, %edx
+; CHECK-64-NEXT: subl %esi, %edx
+; CHECK-64-NEXT: cmovbl %ecx, %edx
+; CHECK-64-NEXT: lock cmpxchgl %edx, (%rdi)
+; CHECK-64-NEXT: jne .LBB6_1
+; CHECK-64-NEXT: # %bb.2: # %atomicrmw.end
+; CHECK-64-NEXT: retq
+ %result = atomicrmw usub_sat ptr %ptr, i32 %val seq_cst
+ ret i32 %result
+}
+
+define i64 @atomicrmw_usub_sat_i64(ptr %ptr, i64 %val) {
+; CHECK-32-LABEL: atomicrmw_usub_sat_i64:
+; CHECK-32: # %bb.0:
+; CHECK-32-NEXT: pushl %ebp
+; CHECK-32-NEXT: .cfi_def_cfa_offset 8
+; CHECK-32-NEXT: pushl %ebx
+; CHECK-32-NEXT: .cfi_def_cfa_offset 12
+; CHECK-32-NEXT: pushl %edi
+; CHECK-32-NEXT: .cfi_def_cfa_offset 16
+; CHECK-32-NEXT: pushl %esi
+; CHECK-32-NEXT: .cfi_def_cfa_offset 20
+; CHECK-32-NEXT: .cfi_offset %esi, -20
+; CHECK-32-NEXT: .cfi_offset %edi, -16
+; CHECK-32-NEXT: .cfi_offset %ebx, -12
+; CHECK-32-NEXT: .cfi_offset %ebp, -8
+; CHECK-32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; CHECK-32-NEXT: movl {{[0-9]+}}(%esp), %edi
+; CHECK-32-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; CHECK-32-NEXT: movl (%ebp), %eax
+; CHECK-32-NEXT: movl 4(%ebp), %edx
+; CHECK-32-NEXT: jmp .LBB7_1
+; CHECK-32-NEXT: .p2align 4, 0x90
+; CHECK-32-NEXT: .LBB7_3: # %atomicrmw.start
+; CHECK-32-NEXT: # in Loop: Header=BB7_1 Depth=1
+; CHECK-32-NEXT: lock cmpxchg8b (%ebp)
+; CHECK-32-NEXT: je .LBB7_4
+; CHECK-32-NEXT: .LBB7_1: # %atomicrmw.start
+; CHECK-32-NEXT: # =>This Inner Loop Header: Depth=1
+; CHECK-32-NEXT: movl %eax, %ebx
+; CHECK-32-NEXT: subl %edi, %ebx
+; CHECK-32-NEXT: movl %edx, %ecx
+; CHECK-32-NEXT: sbbl %esi, %ecx
+; CHECK-32-NEXT: jae .LBB7_3
+; CHECK-32-NEXT: # %bb.2: # %atomicrmw.start
+; CHECK-32-NEXT: # in Loop: Header=BB7_1 Depth=1
+; CHECK-32-NEXT: xorl %ecx, %ecx
+; CHECK-32-NEXT: xorl %ebx, %ebx
+; CHECK-32-NEXT: jmp .LBB7_3
+; CHECK-32-NEXT: .LBB7_4: # %atomicrmw.end
+; CHECK-32-NEXT: popl %esi
+; CHECK-32-NEXT: .cfi_def_cfa_offset 16
+; CHECK-32-NEXT: popl %edi
+; CHECK-32-NEXT: .cfi_def_cfa_offset 12
+; CHECK-32-NEXT: popl %ebx
+; CHECK-32-NEXT: .cfi_def_cfa_offset 8
+; CHECK-32-NEXT: popl %ebp
+; CHECK-32-NEXT: .cfi_def_cfa_offset 4
+; CHECK-32-NEXT: retl
+;
+; CHECK-64-LABEL: atomicrmw_usub_sat_i64:
+; CHECK-64: # %bb.0:
+; CHECK-64-NEXT: movq (%rdi), %rax
+; CHECK-64-NEXT: xorl %ecx, %ecx
+; CHECK-64-NEXT: .p2align 4, 0x90
+; CHECK-64-NEXT: .LBB7_1: # %atomicrmw.start
+; CHECK-64-NEXT: # =>This Inner Loop Header: Depth=1
+; CHECK-64-NEXT: movq %rax, %rdx
+; CHECK-64-NEXT: subq %rsi, %rdx
+; CHECK-64-NEXT: cmovbq %rcx, %rdx
+; CHECK-64-NEXT: lock cmpxchgq %rdx, (%rdi)
+; CHECK-64-NEXT: jne .LBB7_1
+; CHECK-64-NEXT: # %bb.2: # %atomicrmw.end
+; CHECK-64-NEXT: retq
+ %result = atomicrmw usub_sat ptr %ptr, i64 %val seq_cst
+ ret i64 %result
+}
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td
index b52849b6bc931d..5b011131ff806c 100644
--- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td
@@ -136,14 +136,14 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
// CHECK: const uint8_t *GenMyCombiner::getMatchTable() const {
// CHECK-NEXT: constexpr static uint8_t MatchTable0[] = {
// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2([[#LOWER:]]), GIMT_Encode2([[#UPPER:]]), /*)*//*default:*//*Label 6*/ GIMT_Encode4([[#DEFAULT:]]),
-// CHECK-NEXT: /*TargetOpcode::COPY*//*Label 0*/ GIMT_Encode4(470), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
-// CHECK-NEXT: /*TargetOpcode::G_AND*//*Label 1*/ GIMT_Encode4(506), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
-// CHECK-NEXT: /*TargetOpcode::G_STORE*//*Label 2*/ GIMT_Encode4(553), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
-// CHECK-NEXT: /*TargetOpcode::G_TRUNC*//*Label 3*/ GIMT_Encode4(587), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
-// CHECK-NEXT: /*TargetOpcode::G_SEXT*//*Label 4*/ GIMT_Encode4(610), GIMT_Encode4(0),
-// CHECK-NEXT: /*TargetOpcode::G_ZEXT*//*Label 5*/ GIMT_Encode4(622),
+// CHECK-NEXT: /*TargetOpcode::COPY*//*Label 0*/ GIMT_Encode4(478), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
+// CHECK-NEXT: /*TargetOpcode::G_AND*//*Label 1*/ GIMT_Encode4(514), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
+// CHECK-NEXT: /*TargetOpcode::G_STORE*//*Label 2*/ GIMT_Encode4(561), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
+// CHECK-NEXT: /*TargetOpcode::G_TRUNC*//*Label 3*/ GIMT_Encode4(595), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
+// CHECK-NEXT: /*TargetOpcode::G_SEXT*//*Label 4*/ GIMT_Encode4(618), GIMT_Encode4(0),
+// CHECK-NEXT: /*TargetOpcode::G_ZEXT*//*Label 5*/ GIMT_Encode4(630),
// CHECK-NEXT: // Label 0: @[[#%u, mul(UPPER-LOWER, 4) + 10]]
-// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 7*/ GIMT_Encode4(494), // Rule ID 4 //
+// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 7*/ GIMT_Encode4(502), // Rule ID 4 //
// CHECK-NEXT: GIM_CheckFeatures, GIMT_Encode2(GIFBS_HasAnswerToEverything),
// CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule3Enabled),
// CHECK-NEXT: // MIs[0] a
@@ -156,8 +156,8 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
// CHECK-NEXT: GIM_CheckIsSafeToFold, /*NumInsns*/1,
// CHECK-NEXT: // Combiner Rule #3: InstTest1
// CHECK-NEXT: GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner2),
-// CHECK-NEXT: // Label 7: @494
-// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 8*/ GIMT_Encode4(505), // Rule ID 3 //
+// CHECK-NEXT: // Label 7: @502
+// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 8*/ GIMT_Encode4(513), // Rule ID 3 //
// CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule2Enabled),
// CHECK-NEXT: // MIs[0] a
// CHECK-NEXT: // No operand predicates
@@ -165,10 +165,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
// CHECK-NEXT: // No operand predicates
// CHECK-NEXT: // Combiner Rule #2: InstTest0
// CHECK-NEXT: GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner1),
-// CHECK-NEXT: // Label 8: @505
+// CHECK-NEXT: // Label 8: @513
// CHECK-NEXT: GIM_Reject,
-// CHECK-NEXT: // Label 1: @506
-// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 9*/ GIMT_Encode4(552), // Rule ID 6 //
+// CHECK-NEXT: // Label 1: @514
+// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 9*/ GIMT_Encode4(560), // Rule ID 6 //
// CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule5Enabled),
// CHECK-NEXT: GIM_RootCheckType, /*Op*/2, /*Type*/GILLT_s32,
// CHECK-NEXT: // MIs[0] dst
@@ -185,10 +185,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
// CHECK-NEXT: GIR_RootToRootCopy, /*OpIdx*/0, // dst
// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/1, // z
// CHECK-NEXT: GIR_EraseRootFromParent_Done,
-// CHECK-NEXT: // Label 9: @552
+// CHECK-NEXT: // Label 9: @560
// CHECK-NEXT: GIM_Reject,
-// CHECK-NEXT: // Label 2: @553
-// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 10*/ GIMT_Encode4(586), // Rule ID 5 //
+// CHECK-NEXT: // Label 2: @561
+// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 10*/ GIMT_Encode4(594), // Rule ID 5 //
// CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule4Enabled),
// CHECK-NEXT: // MIs[0] tmp
// CHECK-NEXT: GIM_RecordInsnIgnoreCopies, /*DefineMI*/1, /*MI*/0, /*OpIdx*/0, // MIs[1]
@@ -204,29 +204,29 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
// CHECK-NEXT: GIR_RootToRootCopy, /*OpIdx*/1, // ptr
// CHECK-NEXT: GIR_MergeMemOperands, /*InsnID*/0, /*NumInsns*/2, /*MergeInsnID's*/0, 1,
// CHECK-NEXT: GIR_EraseRootFromParent_Done,
-// CHECK-NEXT: // Label 10: @586
+// CHECK-NEXT: // Label 10: @594
// CHECK-NEXT: GIM_Reject,
-// CHECK-NEXT: // Label 3: @587
-// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 11*/ GIMT_Encode4(598), // Rule ID 0 //
+// CHECK-NEXT: // Label 3: @595
+// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 11*/ GIMT_Encode4(606), // Rule ID 0 //
// CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled),
// CHECK-NEXT: // Combiner Rule #0: WipOpcodeTest0; wip_match_opcode 'G_TRUNC'
// CHECK-NEXT: GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner0),
-// CHECK-NEXT: // Label 11: @598
-// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 12*/ GIMT_Encode4(609), // Rule ID 1 //
+// CHECK-NEXT: // Label 11: @606
+// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 12*/ GIMT_Encode4(617), // Rule ID 1 //
// CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule1Enabled),
// CHECK-NEXT: // Combiner Rule #1: WipOpcodeTest1; wip_match_opcode 'G_TRUNC'
// CHECK-NEXT: GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner0),
-// CHECK-NEXT: // Label 12: @609
+// CHECK-NEXT: // Label 12: @617
// CHECK-NEXT: GIM_Reject,
-// CHECK-NEXT: // Label 4: @610
-// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 13*/ GIMT_Encode4(621), // Rule ID 2 //
+// CHECK-NEXT: // Label 4: @618
+// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 13*/ GIMT_Encode4(629), // Rule ID 2 //
// CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule1Enabled),
// CHECK-NEXT: // Combiner Rule #1: WipOpcodeTest1; wip_match_opcode 'G_SEXT'
// CHECK-NEXT: GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner0),
-// CHECK-NEXT: // Label 13: @621
+// CHECK-NEXT: // Label 13: @629
// CHECK-NEXT: GIM_Reject,
-// CHECK-NEXT: // Label 5: @622
-// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 14*/ GIMT_Encode4(656), // Rule ID 7 //
+// CHECK-NEXT: // Label 5: @630
+// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 14*/ GIMT_Encode4(664), // Rule ID 7 //
// CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule6Enabled),
// CHECK-NEXT: // MIs[0] dst
// CHECK-NEXT: // No operand predicates
@@ -240,7 +240,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
// CHECK-NEXT: GIR_RootToRootCopy, /*OpIdx*/0, // dst
// CHECK-NEXT: GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0,
// CHECK-NEXT: GIR_EraseRootFromParent_Done,
-// CHECK-NEXT: // Label 14: @656
+// CHECK-NEXT: // Label 14: @664
// CHECK-NEXT: GIM_Reject,
// CHECK-NEXT: // Label 6: @[[#%u, DEFAULT]]
// CHECK-NEXT: GIM_Reject,
diff --git a/llvm/test/TableGen/GlobalISelEmitter.td b/llvm/test/TableGen/GlobalISelEmitter.td
index 853831366fa531..b9aea33ac96aaa 100644
--- a/llvm/test/TableGen/GlobalISelEmitter.td
+++ b/llvm/test/TableGen/GlobalISelEmitter.td
@@ -513,7 +513,7 @@ def : Pat<(frag GPR32:$src1, complex:$src2, complex:$src3),
// R00O-NEXT: GIM_Reject,
// R00O: // Label [[DEFAULT_NUM]]: @[[DEFAULT]]
// R00O-NEXT: GIM_Reject,
-// R00O-NEXT: }; // Size: 1816 bytes
+// R00O-NEXT: }; // Size: 1824 bytes
def INSNBOB : I<(outs GPR32:$dst), (ins GPR32:$src1, GPR32:$src2, GPR32:$src3, GPR32:$src4),
[(set GPR32:$dst,
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td
index f41a97f9ecc818..4a43c16903394f 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td
@@ -106,6 +106,10 @@ def AtomicBinOpUIncWrap : LLVM_EnumAttrCase<"uinc_wrap",
"uinc_wrap", "UIncWrap", 15>;
def AtomicBinOpUDecWrap : LLVM_EnumAttrCase<"udec_wrap",
"udec_wrap", "UDecWrap", 16>;
+def AtomicBinOpUSubCond : LLVM_EnumAttrCase<"usub_cond",
+ "usub_cond", "USubCond", 17>;
+def AtomicBinOpUSubSat : LLVM_EnumAttrCase<"usub_sat",
+ "usub_sat", "USubSat", 18>;
// A sentinel value that has no MLIR counterpart.
def AtomicBadBinOp : LLVM_EnumAttrCase<"", "", "BAD_BINOP", 0>;
@@ -118,7 +122,7 @@ def AtomicBinOp : LLVM_EnumAttr<
AtomicBinOpNand, AtomicBinOpOr, AtomicBinOpXor, AtomicBinOpMax,
AtomicBinOpMin, AtomicBinOpUMax, AtomicBinOpUMin, AtomicBinOpFAdd,
AtomicBinOpFSub, AtomicBinOpFMax, AtomicBinOpFMin, AtomicBinOpUIncWrap,
- AtomicBinOpUDecWrap],
+ AtomicBinOpUDecWrap, AtomicBinOpUSubCond, AtomicBinOpUSubSat],
[AtomicBadBinOp]> {
let cppNamespace = "::mlir::LLVM";
}
diff --git a/mlir/test/Target/LLVMIR/Import/instructions.ll b/mlir/test/Target/LLVMIR/Import/instructions.ll
index 3b1dcee1e85c7c..f75c79ea633804 100644
--- a/mlir/test/Target/LLVMIR/Import/instructions.ll
+++ b/mlir/test/Target/LLVMIR/Import/instructions.ll
@@ -440,11 +440,15 @@ define void @atomic_rmw(ptr %ptr1, i32 %val1, ptr %ptr2, float %val2) {
%16 = atomicrmw uinc_wrap ptr %ptr1, i32 %val1 acquire
; CHECK: llvm.atomicrmw udec_wrap %[[PTR1]], %[[VAL1]] acquire
%17 = atomicrmw udec_wrap ptr %ptr1, i32 %val1 acquire
+ ; CHECK: llvm.atomicrmw usub_cond %[[PTR1]], %[[VAL1]] acquire
+ %18 = atomicrmw usub_cond ptr %ptr1, i32 %val1 acquire
+ ; CHECK: llvm.atomicrmw usub_sat %[[PTR1]], %[[VAL1]] acquire
+ %19 = atomicrmw usub_sat ptr %ptr1, i32 %val1 acquire
; CHECK: llvm.atomicrmw volatile
; CHECK-SAME: syncscope("singlethread")
; CHECK-SAME: {alignment = 8 : i64}
- %18 = atomicrmw volatile udec_wrap ptr %ptr1, i32 %val1 syncscope("singlethread") acquire, align 8
+ %20 = atomicrmw volatile udec_wrap ptr %ptr1, i32 %val1 syncscope("singlethread") acquire, align 8
ret void
}
diff --git a/mlir/test/Target/LLVMIR/llvmir.mlir b/mlir/test/Target/LLVMIR/llvmir.mlir
index 8453983aa07c33..1fcc27b01f00bc 100644
--- a/mlir/test/Target/LLVMIR/llvmir.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir.mlir
@@ -1513,11 +1513,15 @@ llvm.func @atomicrmw(
%15 = llvm.atomicrmw uinc_wrap %i32_ptr, %i32 monotonic : !llvm.ptr, i32
// CHECK: atomicrmw udec_wrap ptr %{{.*}}, i32 %{{.*}} monotonic
%16 = llvm.atomicrmw udec_wrap %i32_ptr, %i32 monotonic : !llvm.ptr, i32
+ // CHECK: atomicrmw usub_cond ptr %{{.*}}, i32 %{{.*}} monotonic
+ %17 = llvm.atomicrmw usub_cond %i32_ptr, %i32 monotonic : !llvm.ptr, i32
+ // CHECK: atomicrmw usub_sat ptr %{{.*}}, i32 %{{.*}} monotonic
+ %18 = llvm.atomicrmw usub_sat %i32_ptr, %i32 monotonic : !llvm.ptr, i32
// CHECK: atomicrmw volatile
// CHECK-SAME: syncscope("singlethread")
// CHECK-SAME: align 8
- %17 = llvm.atomicrmw volatile udec_wrap %i32_ptr, %i32 syncscope("singlethread") monotonic {alignment = 8 : i64} : !llvm.ptr, i32
+ %19 = llvm.atomicrmw volatile udec_wrap %i32_ptr, %i32 syncscope("singlethread") monotonic {alignment = 8 : i64} : !llvm.ptr, i32
llvm.return
}
>From e09ad034152eb28f9384fdc3d2e908695cfe4363 Mon Sep 17 00:00:00 2001
From: Andrew Jenner <Andrew.Jenner at amd.com>
Date: Wed, 21 Aug 2024 15:39:09 -0400
Subject: [PATCH 2/3] Update failing tests.
---
.../AArch64/atomicrmw-cond-sub-clamp.ll | 8 +-
.../LoongArch/atomicrmw-cond-sub-clamp.ll | 220 +++++++++---------
.../CodeGen/RISCV/atomicrmw-cond-sub-clamp.ll | 7 +-
.../VE/Scalar/atomicrmw-cond-sub-clamp.ll | 2 +-
4 files changed, 117 insertions(+), 120 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/atomicrmw-cond-sub-clamp.ll b/llvm/test/CodeGen/AArch64/atomicrmw-cond-sub-clamp.ll
index 53c8d2e37d16b7..f8eaef9072729c 100644
--- a/llvm/test/CodeGen/AArch64/atomicrmw-cond-sub-clamp.ll
+++ b/llvm/test/CodeGen/AArch64/atomicrmw-cond-sub-clamp.ll
@@ -127,15 +127,15 @@ define i32 @atomicrmw_usub_sat_i32(ptr %ptr, i32 %val) {
define i64 @atomicrmw_usub_sat_i64(ptr %ptr, i64 %val) {
; CHECK-LABEL: atomicrmw_usub_sat_i64:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov x8, x0
; CHECK-NEXT: .LBB7_1: // %atomicrmw.start
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ldaxr x0, [x8]
-; CHECK-NEXT: subs x9, x0, x1
+; CHECK-NEXT: ldaxr x8, [x0]
+; CHECK-NEXT: subs x9, x8, x1
; CHECK-NEXT: csel x9, x9, xzr, hs
-; CHECK-NEXT: stlxr w10, x9, [x8]
+; CHECK-NEXT: stlxr w10, x9, [x0]
; CHECK-NEXT: cbnz w10, .LBB7_1
; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: mov x0, x8
; CHECK-NEXT: ret
%result = atomicrmw usub_sat ptr %ptr, i64 %val seq_cst
ret i64 %result
diff --git a/llvm/test/CodeGen/LoongArch/atomicrmw-cond-sub-clamp.ll b/llvm/test/CodeGen/LoongArch/atomicrmw-cond-sub-clamp.ll
index 72d4a68c3945ac..44bdce6cefcbab 100644
--- a/llvm/test/CodeGen/LoongArch/atomicrmw-cond-sub-clamp.ll
+++ b/llvm/test/CodeGen/LoongArch/atomicrmw-cond-sub-clamp.ll
@@ -4,39 +4,39 @@
define i8 @atomicrmw_usub_cond_i8(ptr %ptr, i8 %val) {
; LA64-LABEL: atomicrmw_usub_cond_i8:
; LA64: # %bb.0:
-; LA64-NEXT: slli.d $a4, $a0, 3
+; LA64-NEXT: slli.d $a3, $a0, 3
; LA64-NEXT: bstrins.d $a0, $zero, 1, 0
-; LA64-NEXT: andi $a2, $a4, 24
-; LA64-NEXT: ori $a5, $zero, 255
-; LA64-NEXT: ld.w $a3, $a0, 0
-; LA64-NEXT: sll.w $a4, $a5, $a4
-; LA64-NEXT: nor $a4, $a4, $zero
-; LA64-NEXT: andi $a5, $a1, 255
+; LA64-NEXT: andi $a2, $a3, 24
+; LA64-NEXT: ori $a4, $zero, 255
+; LA64-NEXT: ld.w $a5, $a0, 0
+; LA64-NEXT: sll.w $a3, $a4, $a3
+; LA64-NEXT: nor $a3, $a3, $zero
+; LA64-NEXT: andi $a4, $a1, 255
; LA64-NEXT: .p2align 4, , 16
; LA64-NEXT: .LBB0_1: # %atomicrmw.start
; LA64-NEXT: # =>This Loop Header: Depth=1
; LA64-NEXT: # Child Loop BB0_3 Depth 2
-; LA64-NEXT: srl.w $a6, $a3, $a2
-; LA64-NEXT: addi.w $a7, $a3, 0
-; LA64-NEXT: andi $t0, $a6, 255
-; LA64-NEXT: sltu $t0, $t0, $a5
-; LA64-NEXT: xori $t0, $t0, 1
-; LA64-NEXT: sub.d $a6, $a6, $a1
-; LA64-NEXT: maskeqz $a6, $a6, $t0
-; LA64-NEXT: masknez $t0, $a1, $t0
-; LA64-NEXT: or $a6, $a6, $t0
-; LA64-NEXT: andi $a6, $a6, 255
-; LA64-NEXT: sll.w $a6, $a6, $a2
-; LA64-NEXT: and $a3, $a3, $a4
-; LA64-NEXT: or $a6, $a3, $a6
+; LA64-NEXT: move $a6, $a5
+; LA64-NEXT: srl.w $a5, $a5, $a2
+; LA64-NEXT: andi $a7, $a5, 255
+; LA64-NEXT: sltu $a7, $a7, $a4
+; LA64-NEXT: xori $a7, $a7, 1
+; LA64-NEXT: sub.d $a5, $a5, $a1
+; LA64-NEXT: maskeqz $a5, $a5, $a7
+; LA64-NEXT: masknez $a7, $a1, $a7
+; LA64-NEXT: or $a5, $a5, $a7
+; LA64-NEXT: andi $a5, $a5, 255
+; LA64-NEXT: sll.w $a5, $a5, $a2
+; LA64-NEXT: and $a7, $a6, $a3
+; LA64-NEXT: or $a7, $a7, $a5
; LA64-NEXT: .LBB0_3: # %atomicrmw.start
; LA64-NEXT: # Parent Loop BB0_1 Depth=1
; LA64-NEXT: # => This Inner Loop Header: Depth=2
-; LA64-NEXT: ll.w $a3, $a0, 0
-; LA64-NEXT: bne $a3, $a7, .LBB0_5
+; LA64-NEXT: ll.w $a5, $a0, 0
+; LA64-NEXT: bne $a5, $a6, .LBB0_5
; LA64-NEXT: # %bb.4: # %atomicrmw.start
; LA64-NEXT: # in Loop: Header=BB0_3 Depth=2
-; LA64-NEXT: move $t0, $a6
+; LA64-NEXT: move $t0, $a7
; LA64-NEXT: sc.w $t0, $a0, 0
; LA64-NEXT: beqz $t0, .LBB0_3
; LA64-NEXT: b .LBB0_6
@@ -45,9 +45,9 @@ define i8 @atomicrmw_usub_cond_i8(ptr %ptr, i8 %val) {
; LA64-NEXT: dbar 20
; LA64-NEXT: .LBB0_6: # %atomicrmw.start
; LA64-NEXT: # in Loop: Header=BB0_1 Depth=1
-; LA64-NEXT: bne $a3, $a7, .LBB0_1
+; LA64-NEXT: bne $a5, $a6, .LBB0_1
; LA64-NEXT: # %bb.2: # %atomicrmw.end
-; LA64-NEXT: srl.w $a0, $a3, $a2
+; LA64-NEXT: srl.w $a0, $a5, $a2
; LA64-NEXT: ret
%result = atomicrmw usub_cond ptr %ptr, i8 %val seq_cst
ret i8 %result
@@ -56,40 +56,40 @@ define i8 @atomicrmw_usub_cond_i8(ptr %ptr, i8 %val) {
define i16 @atomicrmw_usub_cond_i16(ptr %ptr, i16 %val) {
; LA64-LABEL: atomicrmw_usub_cond_i16:
; LA64: # %bb.0:
-; LA64-NEXT: slli.d $a4, $a0, 3
+; LA64-NEXT: slli.d $a3, $a0, 3
; LA64-NEXT: bstrins.d $a0, $zero, 1, 0
-; LA64-NEXT: andi $a2, $a4, 24
-; LA64-NEXT: lu12i.w $a3, 15
-; LA64-NEXT: ori $a5, $a3, 4095
-; LA64-NEXT: ld.w $a3, $a0, 0
-; LA64-NEXT: sll.w $a4, $a5, $a4
-; LA64-NEXT: nor $a4, $a4, $zero
-; LA64-NEXT: bstrpick.d $a5, $a1, 15, 0
+; LA64-NEXT: andi $a2, $a3, 24
+; LA64-NEXT: lu12i.w $a4, 15
+; LA64-NEXT: ori $a4, $a4, 4095
+; LA64-NEXT: ld.w $a5, $a0, 0
+; LA64-NEXT: sll.w $a3, $a4, $a3
+; LA64-NEXT: nor $a3, $a3, $zero
+; LA64-NEXT: bstrpick.d $a4, $a1, 15, 0
; LA64-NEXT: .p2align 4, , 16
; LA64-NEXT: .LBB1_1: # %atomicrmw.start
; LA64-NEXT: # =>This Loop Header: Depth=1
; LA64-NEXT: # Child Loop BB1_3 Depth 2
-; LA64-NEXT: srl.w $a6, $a3, $a2
-; LA64-NEXT: addi.w $a7, $a3, 0
-; LA64-NEXT: bstrpick.d $t0, $a6, 15, 0
-; LA64-NEXT: sltu $t0, $t0, $a5
-; LA64-NEXT: xori $t0, $t0, 1
-; LA64-NEXT: sub.d $a6, $a6, $a1
-; LA64-NEXT: maskeqz $a6, $a6, $t0
-; LA64-NEXT: masknez $t0, $a1, $t0
-; LA64-NEXT: or $a6, $a6, $t0
-; LA64-NEXT: bstrpick.d $a6, $a6, 15, 0
-; LA64-NEXT: sll.w $a6, $a6, $a2
-; LA64-NEXT: and $a3, $a3, $a4
-; LA64-NEXT: or $a6, $a3, $a6
+; LA64-NEXT: move $a6, $a5
+; LA64-NEXT: srl.w $a5, $a5, $a2
+; LA64-NEXT: bstrpick.d $a7, $a5, 15, 0
+; LA64-NEXT: sltu $a7, $a7, $a4
+; LA64-NEXT: xori $a7, $a7, 1
+; LA64-NEXT: sub.d $a5, $a5, $a1
+; LA64-NEXT: maskeqz $a5, $a5, $a7
+; LA64-NEXT: masknez $a7, $a1, $a7
+; LA64-NEXT: or $a5, $a5, $a7
+; LA64-NEXT: bstrpick.d $a5, $a5, 15, 0
+; LA64-NEXT: sll.w $a5, $a5, $a2
+; LA64-NEXT: and $a7, $a6, $a3
+; LA64-NEXT: or $a7, $a7, $a5
; LA64-NEXT: .LBB1_3: # %atomicrmw.start
; LA64-NEXT: # Parent Loop BB1_1 Depth=1
; LA64-NEXT: # => This Inner Loop Header: Depth=2
-; LA64-NEXT: ll.w $a3, $a0, 0
-; LA64-NEXT: bne $a3, $a7, .LBB1_5
+; LA64-NEXT: ll.w $a5, $a0, 0
+; LA64-NEXT: bne $a5, $a6, .LBB1_5
; LA64-NEXT: # %bb.4: # %atomicrmw.start
; LA64-NEXT: # in Loop: Header=BB1_3 Depth=2
-; LA64-NEXT: move $t0, $a6
+; LA64-NEXT: move $t0, $a7
; LA64-NEXT: sc.w $t0, $a0, 0
; LA64-NEXT: beqz $t0, .LBB1_3
; LA64-NEXT: b .LBB1_6
@@ -98,9 +98,9 @@ define i16 @atomicrmw_usub_cond_i16(ptr %ptr, i16 %val) {
; LA64-NEXT: dbar 20
; LA64-NEXT: .LBB1_6: # %atomicrmw.start
; LA64-NEXT: # in Loop: Header=BB1_1 Depth=1
-; LA64-NEXT: bne $a3, $a7, .LBB1_1
+; LA64-NEXT: bne $a5, $a6, .LBB1_1
; LA64-NEXT: # %bb.2: # %atomicrmw.end
-; LA64-NEXT: srl.w $a0, $a3, $a2
+; LA64-NEXT: srl.w $a0, $a5, $a2
; LA64-NEXT: ret
%result = atomicrmw usub_cond ptr %ptr, i16 %val seq_cst
ret i16 %result
@@ -115,13 +115,13 @@ define i32 @atomicrmw_usub_cond_i32(ptr %ptr, i32 %val) {
; LA64-NEXT: .LBB2_1: # %atomicrmw.start
; LA64-NEXT: # =>This Loop Header: Depth=1
; LA64-NEXT: # Child Loop BB2_3 Depth 2
-; LA64-NEXT: addi.w $a4, $a2, 0
-; LA64-NEXT: sltu $a5, $a4, $a3
-; LA64-NEXT: xori $a5, $a5, 1
-; LA64-NEXT: sub.d $a2, $a2, $a1
-; LA64-NEXT: maskeqz $a2, $a2, $a5
-; LA64-NEXT: masknez $a5, $a1, $a5
-; LA64-NEXT: or $a5, $a2, $a5
+; LA64-NEXT: move $a4, $a2
+; LA64-NEXT: sltu $a2, $a2, $a3
+; LA64-NEXT: xori $a2, $a2, 1
+; LA64-NEXT: sub.w $a5, $a4, $a1
+; LA64-NEXT: maskeqz $a5, $a5, $a2
+; LA64-NEXT: masknez $a2, $a1, $a2
+; LA64-NEXT: or $a5, $a5, $a2
; LA64-NEXT: .LBB2_3: # %atomicrmw.start
; LA64-NEXT: # Parent Loop BB2_1 Depth=1
; LA64-NEXT: # => This Inner Loop Header: Depth=2
@@ -188,37 +188,37 @@ define i64 @atomicrmw_usub_cond_i64(ptr %ptr, i64 %val) {
define i8 @atomicrmw_usub_sat_i8(ptr %ptr, i8 %val) {
; LA64-LABEL: atomicrmw_usub_sat_i8:
; LA64: # %bb.0:
-; LA64-NEXT: slli.d $a4, $a0, 3
+; LA64-NEXT: slli.d $a3, $a0, 3
; LA64-NEXT: bstrins.d $a0, $zero, 1, 0
-; LA64-NEXT: andi $a2, $a4, 24
-; LA64-NEXT: ori $a5, $zero, 255
-; LA64-NEXT: ld.w $a3, $a0, 0
-; LA64-NEXT: sll.w $a4, $a5, $a4
-; LA64-NEXT: nor $a4, $a4, $zero
-; LA64-NEXT: andi $a5, $a1, 255
+; LA64-NEXT: andi $a2, $a3, 24
+; LA64-NEXT: ori $a4, $zero, 255
+; LA64-NEXT: ld.w $a5, $a0, 0
+; LA64-NEXT: sll.w $a3, $a4, $a3
+; LA64-NEXT: nor $a3, $a3, $zero
+; LA64-NEXT: andi $a4, $a1, 255
; LA64-NEXT: .p2align 4, , 16
; LA64-NEXT: .LBB4_1: # %atomicrmw.start
; LA64-NEXT: # =>This Loop Header: Depth=1
; LA64-NEXT: # Child Loop BB4_3 Depth 2
-; LA64-NEXT: srl.w $a6, $a3, $a2
-; LA64-NEXT: addi.w $a7, $a3, 0
-; LA64-NEXT: andi $t0, $a6, 255
-; LA64-NEXT: sltu $t0, $t0, $a5
-; LA64-NEXT: xori $t0, $t0, 1
-; LA64-NEXT: sub.d $a6, $a6, $a1
-; LA64-NEXT: maskeqz $a6, $a6, $t0
-; LA64-NEXT: andi $a6, $a6, 255
-; LA64-NEXT: sll.w $a6, $a6, $a2
-; LA64-NEXT: and $a3, $a3, $a4
-; LA64-NEXT: or $a6, $a3, $a6
+; LA64-NEXT: move $a6, $a5
+; LA64-NEXT: srl.w $a5, $a5, $a2
+; LA64-NEXT: andi $a7, $a5, 255
+; LA64-NEXT: sltu $a7, $a7, $a4
+; LA64-NEXT: xori $a7, $a7, 1
+; LA64-NEXT: sub.d $a5, $a5, $a1
+; LA64-NEXT: maskeqz $a5, $a5, $a7
+; LA64-NEXT: andi $a5, $a5, 255
+; LA64-NEXT: sll.w $a5, $a5, $a2
+; LA64-NEXT: and $a7, $a6, $a3
+; LA64-NEXT: or $a7, $a7, $a5
; LA64-NEXT: .LBB4_3: # %atomicrmw.start
; LA64-NEXT: # Parent Loop BB4_1 Depth=1
; LA64-NEXT: # => This Inner Loop Header: Depth=2
-; LA64-NEXT: ll.w $a3, $a0, 0
-; LA64-NEXT: bne $a3, $a7, .LBB4_5
+; LA64-NEXT: ll.w $a5, $a0, 0
+; LA64-NEXT: bne $a5, $a6, .LBB4_5
; LA64-NEXT: # %bb.4: # %atomicrmw.start
; LA64-NEXT: # in Loop: Header=BB4_3 Depth=2
-; LA64-NEXT: move $t0, $a6
+; LA64-NEXT: move $t0, $a7
; LA64-NEXT: sc.w $t0, $a0, 0
; LA64-NEXT: beqz $t0, .LBB4_3
; LA64-NEXT: b .LBB4_6
@@ -227,9 +227,9 @@ define i8 @atomicrmw_usub_sat_i8(ptr %ptr, i8 %val) {
; LA64-NEXT: dbar 20
; LA64-NEXT: .LBB4_6: # %atomicrmw.start
; LA64-NEXT: # in Loop: Header=BB4_1 Depth=1
-; LA64-NEXT: bne $a3, $a7, .LBB4_1
+; LA64-NEXT: bne $a5, $a6, .LBB4_1
; LA64-NEXT: # %bb.2: # %atomicrmw.end
-; LA64-NEXT: srl.w $a0, $a3, $a2
+; LA64-NEXT: srl.w $a0, $a5, $a2
; LA64-NEXT: ret
%result = atomicrmw usub_sat ptr %ptr, i8 %val seq_cst
ret i8 %result
@@ -238,38 +238,38 @@ define i8 @atomicrmw_usub_sat_i8(ptr %ptr, i8 %val) {
define i16 @atomicrmw_usub_sat_i16(ptr %ptr, i16 %val) {
; LA64-LABEL: atomicrmw_usub_sat_i16:
; LA64: # %bb.0:
-; LA64-NEXT: slli.d $a4, $a0, 3
+; LA64-NEXT: slli.d $a3, $a0, 3
; LA64-NEXT: bstrins.d $a0, $zero, 1, 0
-; LA64-NEXT: andi $a2, $a4, 24
-; LA64-NEXT: lu12i.w $a3, 15
-; LA64-NEXT: ori $a5, $a3, 4095
-; LA64-NEXT: ld.w $a3, $a0, 0
-; LA64-NEXT: sll.w $a4, $a5, $a4
-; LA64-NEXT: nor $a4, $a4, $zero
-; LA64-NEXT: bstrpick.d $a5, $a1, 15, 0
+; LA64-NEXT: andi $a2, $a3, 24
+; LA64-NEXT: lu12i.w $a4, 15
+; LA64-NEXT: ori $a4, $a4, 4095
+; LA64-NEXT: ld.w $a5, $a0, 0
+; LA64-NEXT: sll.w $a3, $a4, $a3
+; LA64-NEXT: nor $a3, $a3, $zero
+; LA64-NEXT: bstrpick.d $a4, $a1, 15, 0
; LA64-NEXT: .p2align 4, , 16
; LA64-NEXT: .LBB5_1: # %atomicrmw.start
; LA64-NEXT: # =>This Loop Header: Depth=1
; LA64-NEXT: # Child Loop BB5_3 Depth 2
-; LA64-NEXT: srl.w $a6, $a3, $a2
-; LA64-NEXT: addi.w $a7, $a3, 0
-; LA64-NEXT: bstrpick.d $t0, $a6, 15, 0
-; LA64-NEXT: sltu $t0, $t0, $a5
-; LA64-NEXT: xori $t0, $t0, 1
-; LA64-NEXT: sub.d $a6, $a6, $a1
-; LA64-NEXT: maskeqz $a6, $a6, $t0
-; LA64-NEXT: bstrpick.d $a6, $a6, 15, 0
-; LA64-NEXT: sll.w $a6, $a6, $a2
-; LA64-NEXT: and $a3, $a3, $a4
-; LA64-NEXT: or $a6, $a3, $a6
+; LA64-NEXT: move $a6, $a5
+; LA64-NEXT: srl.w $a5, $a5, $a2
+; LA64-NEXT: bstrpick.d $a7, $a5, 15, 0
+; LA64-NEXT: sltu $a7, $a7, $a4
+; LA64-NEXT: xori $a7, $a7, 1
+; LA64-NEXT: sub.d $a5, $a5, $a1
+; LA64-NEXT: maskeqz $a5, $a5, $a7
+; LA64-NEXT: bstrpick.d $a5, $a5, 15, 0
+; LA64-NEXT: sll.w $a5, $a5, $a2
+; LA64-NEXT: and $a7, $a6, $a3
+; LA64-NEXT: or $a7, $a7, $a5
; LA64-NEXT: .LBB5_3: # %atomicrmw.start
; LA64-NEXT: # Parent Loop BB5_1 Depth=1
; LA64-NEXT: # => This Inner Loop Header: Depth=2
-; LA64-NEXT: ll.w $a3, $a0, 0
-; LA64-NEXT: bne $a3, $a7, .LBB5_5
+; LA64-NEXT: ll.w $a5, $a0, 0
+; LA64-NEXT: bne $a5, $a6, .LBB5_5
; LA64-NEXT: # %bb.4: # %atomicrmw.start
; LA64-NEXT: # in Loop: Header=BB5_3 Depth=2
-; LA64-NEXT: move $t0, $a6
+; LA64-NEXT: move $t0, $a7
; LA64-NEXT: sc.w $t0, $a0, 0
; LA64-NEXT: beqz $t0, .LBB5_3
; LA64-NEXT: b .LBB5_6
@@ -278,9 +278,9 @@ define i16 @atomicrmw_usub_sat_i16(ptr %ptr, i16 %val) {
; LA64-NEXT: dbar 20
; LA64-NEXT: .LBB5_6: # %atomicrmw.start
; LA64-NEXT: # in Loop: Header=BB5_1 Depth=1
-; LA64-NEXT: bne $a3, $a7, .LBB5_1
+; LA64-NEXT: bne $a5, $a6, .LBB5_1
; LA64-NEXT: # %bb.2: # %atomicrmw.end
-; LA64-NEXT: srl.w $a0, $a3, $a2
+; LA64-NEXT: srl.w $a0, $a5, $a2
; LA64-NEXT: ret
%result = atomicrmw usub_sat ptr %ptr, i16 %val seq_cst
ret i16 %result
@@ -295,11 +295,11 @@ define i32 @atomicrmw_usub_sat_i32(ptr %ptr, i32 %val) {
; LA64-NEXT: .LBB6_1: # %atomicrmw.start
; LA64-NEXT: # =>This Loop Header: Depth=1
; LA64-NEXT: # Child Loop BB6_3 Depth 2
-; LA64-NEXT: addi.w $a4, $a2, 0
-; LA64-NEXT: sltu $a5, $a4, $a3
-; LA64-NEXT: xori $a5, $a5, 1
-; LA64-NEXT: sub.d $a2, $a2, $a1
-; LA64-NEXT: maskeqz $a5, $a2, $a5
+; LA64-NEXT: move $a4, $a2
+; LA64-NEXT: sltu $a2, $a2, $a3
+; LA64-NEXT: xori $a2, $a2, 1
+; LA64-NEXT: sub.w $a5, $a4, $a1
+; LA64-NEXT: maskeqz $a5, $a5, $a2
; LA64-NEXT: .LBB6_3: # %atomicrmw.start
; LA64-NEXT: # Parent Loop BB6_1 Depth=1
; LA64-NEXT: # => This Inner Loop Header: Depth=2
diff --git a/llvm/test/CodeGen/RISCV/atomicrmw-cond-sub-clamp.ll b/llvm/test/CodeGen/RISCV/atomicrmw-cond-sub-clamp.ll
index 84a763eb68f0cb..2c0c34d4dbfa0b 100644
--- a/llvm/test/CodeGen/RISCV/atomicrmw-cond-sub-clamp.ll
+++ b/llvm/test/CodeGen/RISCV/atomicrmw-cond-sub-clamp.ll
@@ -449,7 +449,6 @@ define i32 @atomicrmw_usub_cond_i32(ptr %ptr, i32 %val) {
; RV32IA-NEXT: j .LBB2_2
; RV32IA-NEXT: .LBB2_1: # %atomicrmw.start
; RV32IA-NEXT: # in Loop: Header=BB2_2 Depth=1
-; RV32IA-NEXT: mv a4, a1
; RV32IA-NEXT: .LBB2_5: # %atomicrmw.start
; RV32IA-NEXT: # Parent Loop BB2_2 Depth=1
; RV32IA-NEXT: # => This Inner Loop Header: Depth=2
@@ -534,7 +533,6 @@ define i32 @atomicrmw_usub_cond_i32(ptr %ptr, i32 %val) {
; RV64IA-NEXT: j .LBB2_2
; RV64IA-NEXT: .LBB2_1: # %atomicrmw.start
; RV64IA-NEXT: # in Loop: Header=BB2_2 Depth=1
-; RV64IA-NEXT: mv a5, a1
; RV64IA-NEXT: .LBB2_5: # %atomicrmw.start
; RV64IA-NEXT: # Parent Loop BB2_2 Depth=1
; RV64IA-NEXT: # => This Inner Loop Header: Depth=2
@@ -738,7 +736,6 @@ define i64 @atomicrmw_usub_cond_i64(ptr %ptr, i64 %val) {
; RV64IA-NEXT: j .LBB3_2
; RV64IA-NEXT: .LBB3_1: # %atomicrmw.start
; RV64IA-NEXT: # in Loop: Header=BB3_2 Depth=1
-; RV64IA-NEXT: mv a4, a1
; RV64IA-NEXT: .LBB3_5: # %atomicrmw.start
; RV64IA-NEXT: # Parent Loop BB3_2 Depth=1
; RV64IA-NEXT: # => This Inner Loop Header: Depth=2
@@ -910,7 +907,7 @@ define i8 @atomicrmw_usub_sat_i8(ptr %ptr, i8 %val) {
; RV64IA-NEXT: sext.w a7, a3
; RV64IA-NEXT: andi t0, a6, 255
; RV64IA-NEXT: sltu t0, t0, a5
-; RV64IA-NEXT: sub a6, a6, a1
+; RV64IA-NEXT: subw a6, a6, a1
; RV64IA-NEXT: addi t0, t0, -1
; RV64IA-NEXT: and a6, t0, a6
; RV64IA-NEXT: andi a6, a6, 255
@@ -1082,7 +1079,7 @@ define i16 @atomicrmw_usub_sat_i16(ptr %ptr, i16 %val) {
; RV64IA-NEXT: sext.w t0, a4
; RV64IA-NEXT: and t1, a7, a3
; RV64IA-NEXT: sltu t1, t1, a6
-; RV64IA-NEXT: sub a7, a7, a1
+; RV64IA-NEXT: subw a7, a7, a1
; RV64IA-NEXT: addi t1, t1, -1
; RV64IA-NEXT: and a7, a7, a3
; RV64IA-NEXT: and a7, t1, a7
diff --git a/llvm/test/CodeGen/VE/Scalar/atomicrmw-cond-sub-clamp.ll b/llvm/test/CodeGen/VE/Scalar/atomicrmw-cond-sub-clamp.ll
index 58316c80326072..7a047fcfe3cf0c 100644
--- a/llvm/test/CodeGen/VE/Scalar/atomicrmw-cond-sub-clamp.ll
+++ b/llvm/test/CodeGen/VE/Scalar/atomicrmw-cond-sub-clamp.ll
@@ -101,7 +101,7 @@ define i32 @atomicrmw_usub_cond_i32(ptr %ptr, i32 %val) {
}
define i64 @atomicrmw_usub_cond_sub_i64(ptr %ptr, i64 %val) {
-; CHECK-LABEL: atomicrmw_usub_cond_i64:
+; CHECK-LABEL: atomicrmw_usub_cond_sub_i64:
; CHECK: # %bb.0:
; CHECK-NEXT: fencem 3
; CHECK-NEXT: ld %s2, (, %s0)
>From 5070df12fbd2c60279e9756ef2e12a2e6144af4c Mon Sep 17 00:00:00 2001
From: Andrew Jenner <Andrew.Jenner at amd.com>
Date: Thu, 22 Aug 2024 10:50:28 -0400
Subject: [PATCH 3/3] Feedback from pull request.
---
llvm/docs/LangRef.rst | 2 +-
.../CodeGen/GlobalISel/MachineIRBuilder.h | 4 +-
llvm/lib/Transforms/Utils/LowerAtomic.cpp | 12 +-
.../AArch64/atomicrmw-cond-sub-clamp.ll | 22 +-
.../LoongArch/atomicrmw-cond-sub-clamp.ll | 132 ++-
.../CodeGen/RISCV/atomicrmw-cond-sub-clamp.ll | 928 ++++++++----------
.../VE/Scalar/atomicrmw-cond-sub-clamp.ll | 123 ++-
.../CodeGen/X86/atomicrmw-cond-sub-clamp.ll | 97 +-
8 files changed, 594 insertions(+), 726 deletions(-)
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index d44db5999dbe2d..a08dcd2e41cf7d 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -11294,7 +11294,7 @@ operation argument:
- uinc_wrap: ``*ptr = (*ptr u>= val) ? 0 : (*ptr + 1)`` (increment value with wraparound to zero when incremented above input value)
- udec_wrap: ``*ptr = ((*ptr == 0) || (*ptr u> val)) ? val : (*ptr - 1)`` (decrement with wraparound to input value when decremented below zero).
- usub_cond: ``*ptr = (*ptr u>= val) ? *ptr - val : *ptr`` (subtract only if no unsigned overflow).
-- usub_sat: ``*ptr = (*ptr u>= val) ? *ptr - val : 0`` (subtract with clamping to zero).
+- usub_sat: ``*ptr = (*ptr u>= val) ? *ptr - val : 0`` (subtract with unsigned clamping to zero).
Example:
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
index fa3e95c87f5dbc..c0b9d0eac23c3f 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
@@ -1657,8 +1657,8 @@ class MachineIRBuilder {
/// Build and insert `OldValRes<def> = G_ATOMICRMW_USUB_SAT Addr, Val, MMO`.
///
/// Atomically replace the value at \p Addr with the original value minus \p
- /// Val if the original value is greater than or equal to \p Val, or with zero
- /// otherwise. Puts the original value from \p Addr in \p OldValRes.
+ /// Val, with clamping to zero if the unsigned subtraction would overflow.
+ /// Puts the original value from \p Addr in \p OldValRes.
///
/// \pre setBasicBlock or setMI must have been called.
/// \pre \p OldValRes must be a generic virtual register.
diff --git a/llvm/lib/Transforms/Utils/LowerAtomic.cpp b/llvm/lib/Transforms/Utils/LowerAtomic.cpp
index ebe0ee854e6695..8d5aa9a64ad227 100644
--- a/llvm/lib/Transforms/Utils/LowerAtomic.cpp
+++ b/llvm/lib/Transforms/Utils/LowerAtomic.cpp
@@ -98,13 +98,15 @@ Value *llvm::buildAtomicRMWValue(AtomicRMWInst::BinOp Op,
case AtomicRMWInst::USubCond: {
Value *Cmp = Builder.CreateICmpUGE(Loaded, Val);
Value *Sub = Builder.CreateSub(Loaded, Val);
- return Builder.CreateSelect(Cmp, Sub, Val, "new");
+ return Builder.CreateSelect(Cmp, Sub, Loaded, "new");
}
case AtomicRMWInst::USubSat: {
- Constant *Zero = ConstantInt::get(Loaded->getType(), 0);
- Value *Cmp = Builder.CreateICmpUGE(Loaded, Val);
- Value *Sub = Builder.CreateSub(Loaded, Val);
- return Builder.CreateSelect(Cmp, Sub, Zero, "new");
+ return Builder.CreateIntrinsic(Intrinsic::usub_sat, Loaded->getType(),
+ {Loaded, Val}, nullptr, "new");
+ // Constant *Zero = ConstantInt::get(Loaded->getType(), 0);
+ // Value *Cmp = Builder.CreateICmpUGE(Loaded, Val);
+ // Value *Sub = Builder.CreateSub(Loaded, Val);
+ // return Builder.CreateSelect(Cmp, Sub, Zero, "new");
}
default:
llvm_unreachable("Unknown atomic op");
diff --git a/llvm/test/CodeGen/AArch64/atomicrmw-cond-sub-clamp.ll b/llvm/test/CodeGen/AArch64/atomicrmw-cond-sub-clamp.ll
index f8eaef9072729c..83fe8664f72b0d 100644
--- a/llvm/test/CodeGen/AArch64/atomicrmw-cond-sub-clamp.ll
+++ b/llvm/test/CodeGen/AArch64/atomicrmw-cond-sub-clamp.ll
@@ -9,7 +9,7 @@ define i8 @atomicrmw_usub_cond_i8(ptr %ptr, i8 %val) {
; CHECK-NEXT: ldaxrb w8, [x0]
; CHECK-NEXT: sub w9, w8, w1
; CHECK-NEXT: cmp w8, w1, uxtb
-; CHECK-NEXT: csel w9, w9, w1, hs
+; CHECK-NEXT: csel w9, w9, w8, hs
; CHECK-NEXT: stlxrb w10, w9, [x0]
; CHECK-NEXT: cbnz w10, .LBB0_1
; CHECK-NEXT: // %bb.2: // %atomicrmw.end
@@ -27,7 +27,7 @@ define i16 @atomicrmw_usub_cond_i16(ptr %ptr, i16 %val) {
; CHECK-NEXT: ldaxrh w8, [x0]
; CHECK-NEXT: sub w9, w8, w1
; CHECK-NEXT: cmp w8, w1, uxth
-; CHECK-NEXT: csel w9, w9, w1, hs
+; CHECK-NEXT: csel w9, w9, w8, hs
; CHECK-NEXT: stlxrh w10, w9, [x0]
; CHECK-NEXT: cbnz w10, .LBB1_1
; CHECK-NEXT: // %bb.2: // %atomicrmw.end
@@ -44,7 +44,7 @@ define i32 @atomicrmw_usub_cond_i32(ptr %ptr, i32 %val) {
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
; CHECK-NEXT: ldaxr w8, [x0]
; CHECK-NEXT: subs w9, w8, w1
-; CHECK-NEXT: csel w9, w9, w1, hs
+; CHECK-NEXT: csel w9, w9, w8, hs
; CHECK-NEXT: stlxr w10, w9, [x0]
; CHECK-NEXT: cbnz w10, .LBB2_1
; CHECK-NEXT: // %bb.2: // %atomicrmw.end
@@ -62,7 +62,7 @@ define i64 @atomicrmw_usub_cond_i64(ptr %ptr, i64 %val) {
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
; CHECK-NEXT: ldaxr x0, [x8]
; CHECK-NEXT: subs x9, x0, x1
-; CHECK-NEXT: csel x9, x9, x1, hs
+; CHECK-NEXT: csel x9, x9, x0, hs
; CHECK-NEXT: stlxr w10, x9, [x8]
; CHECK-NEXT: cbnz w10, .LBB3_1
; CHECK-NEXT: // %bb.2: // %atomicrmw.end
@@ -77,9 +77,8 @@ define i8 @atomicrmw_usub_sat_i8(ptr %ptr, i8 %val) {
; CHECK-NEXT: .LBB4_1: // %atomicrmw.start
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
; CHECK-NEXT: ldaxrb w8, [x0]
-; CHECK-NEXT: sub w9, w8, w1
-; CHECK-NEXT: cmp w8, w1, uxtb
-; CHECK-NEXT: csel w9, w9, wzr, hs
+; CHECK-NEXT: subs w9, w8, w1, uxtb
+; CHECK-NEXT: csel w9, wzr, w9, lo
; CHECK-NEXT: stlxrb w10, w9, [x0]
; CHECK-NEXT: cbnz w10, .LBB4_1
; CHECK-NEXT: // %bb.2: // %atomicrmw.end
@@ -95,9 +94,8 @@ define i16 @atomicrmw_usub_sat_i16(ptr %ptr, i16 %val) {
; CHECK-NEXT: .LBB5_1: // %atomicrmw.start
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
; CHECK-NEXT: ldaxrh w8, [x0]
-; CHECK-NEXT: sub w9, w8, w1
-; CHECK-NEXT: cmp w8, w1, uxth
-; CHECK-NEXT: csel w9, w9, wzr, hs
+; CHECK-NEXT: subs w9, w8, w1, uxth
+; CHECK-NEXT: csel w9, wzr, w9, lo
; CHECK-NEXT: stlxrh w10, w9, [x0]
; CHECK-NEXT: cbnz w10, .LBB5_1
; CHECK-NEXT: // %bb.2: // %atomicrmw.end
@@ -114,7 +112,7 @@ define i32 @atomicrmw_usub_sat_i32(ptr %ptr, i32 %val) {
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
; CHECK-NEXT: ldaxr w8, [x0]
; CHECK-NEXT: subs w9, w8, w1
-; CHECK-NEXT: csel w9, w9, wzr, hs
+; CHECK-NEXT: csel w9, wzr, w9, lo
; CHECK-NEXT: stlxr w10, w9, [x0]
; CHECK-NEXT: cbnz w10, .LBB6_1
; CHECK-NEXT: // %bb.2: // %atomicrmw.end
@@ -131,7 +129,7 @@ define i64 @atomicrmw_usub_sat_i64(ptr %ptr, i64 %val) {
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
; CHECK-NEXT: ldaxr x8, [x0]
; CHECK-NEXT: subs x9, x8, x1
-; CHECK-NEXT: csel x9, x9, xzr, hs
+; CHECK-NEXT: csel x9, xzr, x9, lo
; CHECK-NEXT: stlxr w10, x9, [x0]
; CHECK-NEXT: cbnz w10, .LBB7_1
; CHECK-NEXT: // %bb.2: // %atomicrmw.end
diff --git a/llvm/test/CodeGen/LoongArch/atomicrmw-cond-sub-clamp.ll b/llvm/test/CodeGen/LoongArch/atomicrmw-cond-sub-clamp.ll
index 44bdce6cefcbab..95bb25c41dabcb 100644
--- a/llvm/test/CodeGen/LoongArch/atomicrmw-cond-sub-clamp.ll
+++ b/llvm/test/CodeGen/LoongArch/atomicrmw-cond-sub-clamp.ll
@@ -21,10 +21,10 @@ define i8 @atomicrmw_usub_cond_i8(ptr %ptr, i8 %val) {
; LA64-NEXT: andi $a7, $a5, 255
; LA64-NEXT: sltu $a7, $a7, $a4
; LA64-NEXT: xori $a7, $a7, 1
-; LA64-NEXT: sub.d $a5, $a5, $a1
-; LA64-NEXT: maskeqz $a5, $a5, $a7
-; LA64-NEXT: masknez $a7, $a1, $a7
-; LA64-NEXT: or $a5, $a5, $a7
+; LA64-NEXT: sub.d $t0, $a5, $a1
+; LA64-NEXT: masknez $a5, $a5, $a7
+; LA64-NEXT: maskeqz $a7, $t0, $a7
+; LA64-NEXT: or $a5, $a7, $a5
; LA64-NEXT: andi $a5, $a5, 255
; LA64-NEXT: sll.w $a5, $a5, $a2
; LA64-NEXT: and $a7, $a6, $a3
@@ -74,10 +74,10 @@ define i16 @atomicrmw_usub_cond_i16(ptr %ptr, i16 %val) {
; LA64-NEXT: bstrpick.d $a7, $a5, 15, 0
; LA64-NEXT: sltu $a7, $a7, $a4
; LA64-NEXT: xori $a7, $a7, 1
-; LA64-NEXT: sub.d $a5, $a5, $a1
-; LA64-NEXT: maskeqz $a5, $a5, $a7
-; LA64-NEXT: masknez $a7, $a1, $a7
-; LA64-NEXT: or $a5, $a5, $a7
+; LA64-NEXT: sub.d $t0, $a5, $a1
+; LA64-NEXT: masknez $a5, $a5, $a7
+; LA64-NEXT: maskeqz $a7, $t0, $a7
+; LA64-NEXT: or $a5, $a7, $a5
; LA64-NEXT: bstrpick.d $a5, $a5, 15, 0
; LA64-NEXT: sll.w $a5, $a5, $a2
; LA64-NEXT: and $a7, $a6, $a3
@@ -120,7 +120,7 @@ define i32 @atomicrmw_usub_cond_i32(ptr %ptr, i32 %val) {
; LA64-NEXT: xori $a2, $a2, 1
; LA64-NEXT: sub.w $a5, $a4, $a1
; LA64-NEXT: maskeqz $a5, $a5, $a2
-; LA64-NEXT: masknez $a2, $a1, $a2
+; LA64-NEXT: masknez $a2, $a4, $a2
; LA64-NEXT: or $a5, $a5, $a2
; LA64-NEXT: .LBB2_3: # %atomicrmw.start
; LA64-NEXT: # Parent Loop BB2_1 Depth=1
@@ -159,7 +159,7 @@ define i64 @atomicrmw_usub_cond_i64(ptr %ptr, i64 %val) {
; LA64-NEXT: xori $a2, $a2, 1
; LA64-NEXT: sub.d $a4, $a3, $a1
; LA64-NEXT: maskeqz $a4, $a4, $a2
-; LA64-NEXT: masknez $a2, $a1, $a2
+; LA64-NEXT: masknez $a2, $a3, $a2
; LA64-NEXT: or $a4, $a4, $a2
; LA64-NEXT: .LBB3_3: # %atomicrmw.start
; LA64-NEXT: # Parent Loop BB3_1 Depth=1
@@ -191,45 +191,43 @@ define i8 @atomicrmw_usub_sat_i8(ptr %ptr, i8 %val) {
; LA64-NEXT: slli.d $a3, $a0, 3
; LA64-NEXT: bstrins.d $a0, $zero, 1, 0
; LA64-NEXT: andi $a2, $a3, 24
-; LA64-NEXT: ori $a4, $zero, 255
-; LA64-NEXT: ld.w $a5, $a0, 0
-; LA64-NEXT: sll.w $a3, $a4, $a3
+; LA64-NEXT: ori $a5, $zero, 255
+; LA64-NEXT: ld.w $a4, $a0, 0
+; LA64-NEXT: sll.w $a3, $a5, $a3
; LA64-NEXT: nor $a3, $a3, $zero
-; LA64-NEXT: andi $a4, $a1, 255
+; LA64-NEXT: andi $a1, $a1, 255
; LA64-NEXT: .p2align 4, , 16
; LA64-NEXT: .LBB4_1: # %atomicrmw.start
; LA64-NEXT: # =>This Loop Header: Depth=1
; LA64-NEXT: # Child Loop BB4_3 Depth 2
-; LA64-NEXT: move $a6, $a5
-; LA64-NEXT: srl.w $a5, $a5, $a2
-; LA64-NEXT: andi $a7, $a5, 255
-; LA64-NEXT: sltu $a7, $a7, $a4
-; LA64-NEXT: xori $a7, $a7, 1
-; LA64-NEXT: sub.d $a5, $a5, $a1
-; LA64-NEXT: maskeqz $a5, $a5, $a7
-; LA64-NEXT: andi $a5, $a5, 255
-; LA64-NEXT: sll.w $a5, $a5, $a2
-; LA64-NEXT: and $a7, $a6, $a3
-; LA64-NEXT: or $a7, $a7, $a5
+; LA64-NEXT: move $a5, $a4
+; LA64-NEXT: srl.w $a4, $a4, $a2
+; LA64-NEXT: andi $a4, $a4, 255
+; LA64-NEXT: sub.d $a6, $a4, $a1
+; LA64-NEXT: sltu $a4, $a4, $a6
+; LA64-NEXT: masknez $a4, $a6, $a4
+; LA64-NEXT: sll.w $a4, $a4, $a2
+; LA64-NEXT: and $a6, $a5, $a3
+; LA64-NEXT: or $a6, $a6, $a4
; LA64-NEXT: .LBB4_3: # %atomicrmw.start
; LA64-NEXT: # Parent Loop BB4_1 Depth=1
; LA64-NEXT: # => This Inner Loop Header: Depth=2
-; LA64-NEXT: ll.w $a5, $a0, 0
-; LA64-NEXT: bne $a5, $a6, .LBB4_5
+; LA64-NEXT: ll.w $a4, $a0, 0
+; LA64-NEXT: bne $a4, $a5, .LBB4_5
; LA64-NEXT: # %bb.4: # %atomicrmw.start
; LA64-NEXT: # in Loop: Header=BB4_3 Depth=2
-; LA64-NEXT: move $t0, $a7
-; LA64-NEXT: sc.w $t0, $a0, 0
-; LA64-NEXT: beqz $t0, .LBB4_3
+; LA64-NEXT: move $a7, $a6
+; LA64-NEXT: sc.w $a7, $a0, 0
+; LA64-NEXT: beqz $a7, .LBB4_3
; LA64-NEXT: b .LBB4_6
; LA64-NEXT: .LBB4_5: # %atomicrmw.start
; LA64-NEXT: # in Loop: Header=BB4_1 Depth=1
; LA64-NEXT: dbar 20
; LA64-NEXT: .LBB4_6: # %atomicrmw.start
; LA64-NEXT: # in Loop: Header=BB4_1 Depth=1
-; LA64-NEXT: bne $a5, $a6, .LBB4_1
+; LA64-NEXT: bne $a4, $a5, .LBB4_1
; LA64-NEXT: # %bb.2: # %atomicrmw.end
-; LA64-NEXT: srl.w $a0, $a5, $a2
+; LA64-NEXT: srl.w $a0, $a4, $a2
; LA64-NEXT: ret
%result = atomicrmw usub_sat ptr %ptr, i8 %val seq_cst
ret i8 %result
@@ -242,45 +240,43 @@ define i16 @atomicrmw_usub_sat_i16(ptr %ptr, i16 %val) {
; LA64-NEXT: bstrins.d $a0, $zero, 1, 0
; LA64-NEXT: andi $a2, $a3, 24
; LA64-NEXT: lu12i.w $a4, 15
-; LA64-NEXT: ori $a4, $a4, 4095
-; LA64-NEXT: ld.w $a5, $a0, 0
-; LA64-NEXT: sll.w $a3, $a4, $a3
+; LA64-NEXT: ori $a5, $a4, 4095
+; LA64-NEXT: ld.w $a4, $a0, 0
+; LA64-NEXT: sll.w $a3, $a5, $a3
; LA64-NEXT: nor $a3, $a3, $zero
-; LA64-NEXT: bstrpick.d $a4, $a1, 15, 0
+; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0
; LA64-NEXT: .p2align 4, , 16
; LA64-NEXT: .LBB5_1: # %atomicrmw.start
; LA64-NEXT: # =>This Loop Header: Depth=1
; LA64-NEXT: # Child Loop BB5_3 Depth 2
-; LA64-NEXT: move $a6, $a5
-; LA64-NEXT: srl.w $a5, $a5, $a2
-; LA64-NEXT: bstrpick.d $a7, $a5, 15, 0
-; LA64-NEXT: sltu $a7, $a7, $a4
-; LA64-NEXT: xori $a7, $a7, 1
-; LA64-NEXT: sub.d $a5, $a5, $a1
-; LA64-NEXT: maskeqz $a5, $a5, $a7
-; LA64-NEXT: bstrpick.d $a5, $a5, 15, 0
-; LA64-NEXT: sll.w $a5, $a5, $a2
-; LA64-NEXT: and $a7, $a6, $a3
-; LA64-NEXT: or $a7, $a7, $a5
+; LA64-NEXT: move $a5, $a4
+; LA64-NEXT: srl.w $a4, $a4, $a2
+; LA64-NEXT: bstrpick.d $a4, $a4, 15, 0
+; LA64-NEXT: sub.d $a6, $a4, $a1
+; LA64-NEXT: sltu $a4, $a4, $a6
+; LA64-NEXT: masknez $a4, $a6, $a4
+; LA64-NEXT: sll.w $a4, $a4, $a2
+; LA64-NEXT: and $a6, $a5, $a3
+; LA64-NEXT: or $a6, $a6, $a4
; LA64-NEXT: .LBB5_3: # %atomicrmw.start
; LA64-NEXT: # Parent Loop BB5_1 Depth=1
; LA64-NEXT: # => This Inner Loop Header: Depth=2
-; LA64-NEXT: ll.w $a5, $a0, 0
-; LA64-NEXT: bne $a5, $a6, .LBB5_5
+; LA64-NEXT: ll.w $a4, $a0, 0
+; LA64-NEXT: bne $a4, $a5, .LBB5_5
; LA64-NEXT: # %bb.4: # %atomicrmw.start
; LA64-NEXT: # in Loop: Header=BB5_3 Depth=2
-; LA64-NEXT: move $t0, $a7
-; LA64-NEXT: sc.w $t0, $a0, 0
-; LA64-NEXT: beqz $t0, .LBB5_3
+; LA64-NEXT: move $a7, $a6
+; LA64-NEXT: sc.w $a7, $a0, 0
+; LA64-NEXT: beqz $a7, .LBB5_3
; LA64-NEXT: b .LBB5_6
; LA64-NEXT: .LBB5_5: # %atomicrmw.start
; LA64-NEXT: # in Loop: Header=BB5_1 Depth=1
; LA64-NEXT: dbar 20
; LA64-NEXT: .LBB5_6: # %atomicrmw.start
; LA64-NEXT: # in Loop: Header=BB5_1 Depth=1
-; LA64-NEXT: bne $a5, $a6, .LBB5_1
+; LA64-NEXT: bne $a4, $a5, .LBB5_1
; LA64-NEXT: # %bb.2: # %atomicrmw.end
-; LA64-NEXT: srl.w $a0, $a5, $a2
+; LA64-NEXT: srl.w $a0, $a4, $a2
; LA64-NEXT: ret
%result = atomicrmw usub_sat ptr %ptr, i16 %val seq_cst
ret i16 %result
@@ -290,33 +286,32 @@ define i32 @atomicrmw_usub_sat_i32(ptr %ptr, i32 %val) {
; LA64-LABEL: atomicrmw_usub_sat_i32:
; LA64: # %bb.0:
; LA64-NEXT: ld.w $a2, $a0, 0
-; LA64-NEXT: addi.w $a3, $a1, 0
+; LA64-NEXT: addi.w $a1, $a1, 0
; LA64-NEXT: .p2align 4, , 16
; LA64-NEXT: .LBB6_1: # %atomicrmw.start
; LA64-NEXT: # =>This Loop Header: Depth=1
; LA64-NEXT: # Child Loop BB6_3 Depth 2
-; LA64-NEXT: move $a4, $a2
-; LA64-NEXT: sltu $a2, $a2, $a3
-; LA64-NEXT: xori $a2, $a2, 1
-; LA64-NEXT: sub.w $a5, $a4, $a1
-; LA64-NEXT: maskeqz $a5, $a5, $a2
+; LA64-NEXT: move $a3, $a2
+; LA64-NEXT: sub.d $a2, $a2, $a1
+; LA64-NEXT: sltu $a4, $a3, $a2
+; LA64-NEXT: masknez $a4, $a2, $a4
; LA64-NEXT: .LBB6_3: # %atomicrmw.start
; LA64-NEXT: # Parent Loop BB6_1 Depth=1
; LA64-NEXT: # => This Inner Loop Header: Depth=2
; LA64-NEXT: ll.w $a2, $a0, 0
-; LA64-NEXT: bne $a2, $a4, .LBB6_5
+; LA64-NEXT: bne $a2, $a3, .LBB6_5
; LA64-NEXT: # %bb.4: # %atomicrmw.start
; LA64-NEXT: # in Loop: Header=BB6_3 Depth=2
-; LA64-NEXT: move $a6, $a5
-; LA64-NEXT: sc.w $a6, $a0, 0
-; LA64-NEXT: beqz $a6, .LBB6_3
+; LA64-NEXT: move $a5, $a4
+; LA64-NEXT: sc.w $a5, $a0, 0
+; LA64-NEXT: beqz $a5, .LBB6_3
; LA64-NEXT: b .LBB6_6
; LA64-NEXT: .LBB6_5: # %atomicrmw.start
; LA64-NEXT: # in Loop: Header=BB6_1 Depth=1
; LA64-NEXT: dbar 20
; LA64-NEXT: .LBB6_6: # %atomicrmw.start
; LA64-NEXT: # in Loop: Header=BB6_1 Depth=1
-; LA64-NEXT: bne $a2, $a4, .LBB6_1
+; LA64-NEXT: bne $a2, $a3, .LBB6_1
; LA64-NEXT: # %bb.2: # %atomicrmw.end
; LA64-NEXT: move $a0, $a2
; LA64-NEXT: ret
@@ -333,10 +328,9 @@ define i64 @atomicrmw_usub_sat_i64(ptr %ptr, i64 %val) {
; LA64-NEXT: # =>This Loop Header: Depth=1
; LA64-NEXT: # Child Loop BB7_3 Depth 2
; LA64-NEXT: move $a3, $a2
-; LA64-NEXT: sltu $a2, $a2, $a1
-; LA64-NEXT: xori $a2, $a2, 1
-; LA64-NEXT: sub.d $a4, $a3, $a1
-; LA64-NEXT: maskeqz $a4, $a4, $a2
+; LA64-NEXT: sub.d $a2, $a2, $a1
+; LA64-NEXT: sltu $a4, $a3, $a2
+; LA64-NEXT: masknez $a4, $a2, $a4
; LA64-NEXT: .LBB7_3: # %atomicrmw.start
; LA64-NEXT: # Parent Loop BB7_1 Depth=1
; LA64-NEXT: # => This Inner Loop Header: Depth=2
diff --git a/llvm/test/CodeGen/RISCV/atomicrmw-cond-sub-clamp.ll b/llvm/test/CodeGen/RISCV/atomicrmw-cond-sub-clamp.ll
index 2c0c34d4dbfa0b..a9c8a4be7d2b47 100644
--- a/llvm/test/CodeGen/RISCV/atomicrmw-cond-sub-clamp.ll
+++ b/llvm/test/CodeGen/RISCV/atomicrmw-cond-sub-clamp.ll
@@ -30,12 +30,13 @@ define i8 @atomicrmw_usub_cond_i8(ptr %ptr, i8 %val) {
; RV32I-NEXT: lbu a3, 0(a0)
; RV32I-NEXT: mv s1, a1
; RV32I-NEXT: andi s2, a1, 255
-; RV32I-NEXT: j .LBB0_3
; RV32I-NEXT: .LBB0_1: # %atomicrmw.start
-; RV32I-NEXT: # in Loop: Header=BB0_3 Depth=1
-; RV32I-NEXT: mv a2, s1
-; RV32I-NEXT: .LBB0_2: # %atomicrmw.start
-; RV32I-NEXT: # in Loop: Header=BB0_3 Depth=1
+; RV32I-NEXT: # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT: andi a0, a3, 255
+; RV32I-NEXT: sltu a0, a0, s2
+; RV32I-NEXT: addi a0, a0, -1
+; RV32I-NEXT: and a0, a0, s1
+; RV32I-NEXT: sub a2, a3, a0
; RV32I-NEXT: sb a3, 15(sp)
; RV32I-NEXT: addi a1, sp, 15
; RV32I-NEXT: li a3, 5
@@ -43,15 +44,8 @@ define i8 @atomicrmw_usub_cond_i8(ptr %ptr, i8 %val) {
; RV32I-NEXT: mv a0, s0
; RV32I-NEXT: call __atomic_compare_exchange_1
; RV32I-NEXT: lbu a3, 15(sp)
-; RV32I-NEXT: bnez a0, .LBB0_5
-; RV32I-NEXT: .LBB0_3: # %atomicrmw.start
-; RV32I-NEXT: # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT: andi a0, a3, 255
-; RV32I-NEXT: bltu a0, s2, .LBB0_1
-; RV32I-NEXT: # %bb.4: # in Loop: Header=BB0_3 Depth=1
-; RV32I-NEXT: sub a2, a3, s1
-; RV32I-NEXT: j .LBB0_2
-; RV32I-NEXT: .LBB0_5: # %atomicrmw.end
+; RV32I-NEXT: beqz a0, .LBB0_1
+; RV32I-NEXT: # %bb.2: # %atomicrmw.end
; RV32I-NEXT: mv a0, a3
; RV32I-NEXT: lw ra, 28(sp) # 4-byte Folded Reload
; RV32I-NEXT: lw s0, 24(sp) # 4-byte Folded Reload
@@ -66,44 +60,36 @@ define i8 @atomicrmw_usub_cond_i8(ptr %ptr, i8 %val) {
; RV32IA-NEXT: slli a3, a0, 3
; RV32IA-NEXT: andi a0, a3, 24
; RV32IA-NEXT: li a4, 255
-; RV32IA-NEXT: lw a6, 0(a2)
+; RV32IA-NEXT: lw a5, 0(a2)
; RV32IA-NEXT: sll a3, a4, a3
; RV32IA-NEXT: not a3, a3
; RV32IA-NEXT: andi a4, a1, 255
-; RV32IA-NEXT: j .LBB0_3
; RV32IA-NEXT: .LBB0_1: # %atomicrmw.start
-; RV32IA-NEXT: # in Loop: Header=BB0_3 Depth=1
-; RV32IA-NEXT: mv a6, a1
-; RV32IA-NEXT: .LBB0_2: # %atomicrmw.start
-; RV32IA-NEXT: # in Loop: Header=BB0_3 Depth=1
-; RV32IA-NEXT: andi a6, a6, 255
-; RV32IA-NEXT: sll a6, a6, a0
-; RV32IA-NEXT: and a7, a5, a3
-; RV32IA-NEXT: or a7, a7, a6
-; RV32IA-NEXT: .LBB0_6: # %atomicrmw.start
-; RV32IA-NEXT: # Parent Loop BB0_3 Depth=1
+; RV32IA-NEXT: # =>This Loop Header: Depth=1
+; RV32IA-NEXT: # Child Loop BB0_3 Depth 2
+; RV32IA-NEXT: mv a6, a5
+; RV32IA-NEXT: srl a5, a5, a0
+; RV32IA-NEXT: andi a7, a5, 255
+; RV32IA-NEXT: sltu a7, a7, a4
+; RV32IA-NEXT: addi a7, a7, -1
+; RV32IA-NEXT: and a7, a7, a1
+; RV32IA-NEXT: sub a5, a5, a7
+; RV32IA-NEXT: andi a5, a5, 255
+; RV32IA-NEXT: sll a5, a5, a0
+; RV32IA-NEXT: and a7, a6, a3
+; RV32IA-NEXT: or a7, a7, a5
+; RV32IA-NEXT: .LBB0_3: # %atomicrmw.start
+; RV32IA-NEXT: # Parent Loop BB0_1 Depth=1
; RV32IA-NEXT: # => This Inner Loop Header: Depth=2
-; RV32IA-NEXT: lr.w.aqrl a6, (a2)
-; RV32IA-NEXT: bne a6, a5, .LBB0_8
-; RV32IA-NEXT: # %bb.7: # %atomicrmw.start
-; RV32IA-NEXT: # in Loop: Header=BB0_6 Depth=2
+; RV32IA-NEXT: lr.w.aqrl a5, (a2)
+; RV32IA-NEXT: bne a5, a6, .LBB0_1
+; RV32IA-NEXT: # %bb.4: # %atomicrmw.start
+; RV32IA-NEXT: # in Loop: Header=BB0_3 Depth=2
; RV32IA-NEXT: sc.w.rl t0, a7, (a2)
-; RV32IA-NEXT: bnez t0, .LBB0_6
-; RV32IA-NEXT: .LBB0_8: # %atomicrmw.start
-; RV32IA-NEXT: # in Loop: Header=BB0_3 Depth=1
-; RV32IA-NEXT: beq a6, a5, .LBB0_5
-; RV32IA-NEXT: .LBB0_3: # %atomicrmw.start
-; RV32IA-NEXT: # =>This Loop Header: Depth=1
-; RV32IA-NEXT: # Child Loop BB0_6 Depth 2
-; RV32IA-NEXT: mv a5, a6
-; RV32IA-NEXT: srl a6, a6, a0
-; RV32IA-NEXT: andi a7, a6, 255
-; RV32IA-NEXT: bltu a7, a4, .LBB0_1
-; RV32IA-NEXT: # %bb.4: # in Loop: Header=BB0_3 Depth=1
-; RV32IA-NEXT: sub a6, a6, a1
-; RV32IA-NEXT: j .LBB0_2
-; RV32IA-NEXT: .LBB0_5: # %atomicrmw.end
-; RV32IA-NEXT: srl a0, a6, a0
+; RV32IA-NEXT: bnez t0, .LBB0_3
+; RV32IA-NEXT: # %bb.5: # %atomicrmw.start
+; RV32IA-NEXT: # %bb.2: # %atomicrmw.end
+; RV32IA-NEXT: srl a0, a5, a0
; RV32IA-NEXT: ret
;
; RV64I-LABEL: atomicrmw_usub_cond_i8:
@@ -122,12 +108,13 @@ define i8 @atomicrmw_usub_cond_i8(ptr %ptr, i8 %val) {
; RV64I-NEXT: lbu a3, 0(a0)
; RV64I-NEXT: mv s1, a1
; RV64I-NEXT: andi s2, a1, 255
-; RV64I-NEXT: j .LBB0_3
; RV64I-NEXT: .LBB0_1: # %atomicrmw.start
-; RV64I-NEXT: # in Loop: Header=BB0_3 Depth=1
-; RV64I-NEXT: mv a2, s1
-; RV64I-NEXT: .LBB0_2: # %atomicrmw.start
-; RV64I-NEXT: # in Loop: Header=BB0_3 Depth=1
+; RV64I-NEXT: # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT: andi a0, a3, 255
+; RV64I-NEXT: sltu a0, a0, s2
+; RV64I-NEXT: addi a0, a0, -1
+; RV64I-NEXT: and a0, a0, s1
+; RV64I-NEXT: sub a2, a3, a0
; RV64I-NEXT: sb a3, 15(sp)
; RV64I-NEXT: addi a1, sp, 15
; RV64I-NEXT: li a3, 5
@@ -135,15 +122,8 @@ define i8 @atomicrmw_usub_cond_i8(ptr %ptr, i8 %val) {
; RV64I-NEXT: mv a0, s0
; RV64I-NEXT: call __atomic_compare_exchange_1
; RV64I-NEXT: lbu a3, 15(sp)
-; RV64I-NEXT: bnez a0, .LBB0_5
-; RV64I-NEXT: .LBB0_3: # %atomicrmw.start
-; RV64I-NEXT: # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT: andi a0, a3, 255
-; RV64I-NEXT: bltu a0, s2, .LBB0_1
-; RV64I-NEXT: # %bb.4: # in Loop: Header=BB0_3 Depth=1
-; RV64I-NEXT: sub a2, a3, s1
-; RV64I-NEXT: j .LBB0_2
-; RV64I-NEXT: .LBB0_5: # %atomicrmw.end
+; RV64I-NEXT: beqz a0, .LBB0_1
+; RV64I-NEXT: # %bb.2: # %atomicrmw.end
; RV64I-NEXT: mv a0, a3
; RV64I-NEXT: ld ra, 40(sp) # 8-byte Folded Reload
; RV64I-NEXT: ld s0, 32(sp) # 8-byte Folded Reload
@@ -162,39 +142,31 @@ define i8 @atomicrmw_usub_cond_i8(ptr %ptr, i8 %val) {
; RV64IA-NEXT: sllw a4, a5, a4
; RV64IA-NEXT: not a4, a4
; RV64IA-NEXT: andi a5, a1, 255
-; RV64IA-NEXT: j .LBB0_3
; RV64IA-NEXT: .LBB0_1: # %atomicrmw.start
-; RV64IA-NEXT: # in Loop: Header=BB0_3 Depth=1
-; RV64IA-NEXT: mv a6, a1
-; RV64IA-NEXT: .LBB0_2: # %atomicrmw.start
-; RV64IA-NEXT: # in Loop: Header=BB0_3 Depth=1
+; RV64IA-NEXT: # =>This Loop Header: Depth=1
+; RV64IA-NEXT: # Child Loop BB0_3 Depth 2
+; RV64IA-NEXT: srlw a6, a3, a0
; RV64IA-NEXT: sext.w a7, a3
+; RV64IA-NEXT: andi t0, a6, 255
+; RV64IA-NEXT: sltu t0, t0, a5
+; RV64IA-NEXT: addi t0, t0, -1
+; RV64IA-NEXT: and t0, t0, a1
+; RV64IA-NEXT: subw a6, a6, t0
; RV64IA-NEXT: andi a6, a6, 255
; RV64IA-NEXT: sllw a6, a6, a0
; RV64IA-NEXT: and a3, a3, a4
; RV64IA-NEXT: or a6, a3, a6
-; RV64IA-NEXT: .LBB0_6: # %atomicrmw.start
-; RV64IA-NEXT: # Parent Loop BB0_3 Depth=1
+; RV64IA-NEXT: .LBB0_3: # %atomicrmw.start
+; RV64IA-NEXT: # Parent Loop BB0_1 Depth=1
; RV64IA-NEXT: # => This Inner Loop Header: Depth=2
; RV64IA-NEXT: lr.w.aqrl a3, (a2)
-; RV64IA-NEXT: bne a3, a7, .LBB0_8
-; RV64IA-NEXT: # %bb.7: # %atomicrmw.start
-; RV64IA-NEXT: # in Loop: Header=BB0_6 Depth=2
+; RV64IA-NEXT: bne a3, a7, .LBB0_1
+; RV64IA-NEXT: # %bb.4: # %atomicrmw.start
+; RV64IA-NEXT: # in Loop: Header=BB0_3 Depth=2
; RV64IA-NEXT: sc.w.rl t0, a6, (a2)
-; RV64IA-NEXT: bnez t0, .LBB0_6
-; RV64IA-NEXT: .LBB0_8: # %atomicrmw.start
-; RV64IA-NEXT: # in Loop: Header=BB0_3 Depth=1
-; RV64IA-NEXT: beq a3, a7, .LBB0_5
-; RV64IA-NEXT: .LBB0_3: # %atomicrmw.start
-; RV64IA-NEXT: # =>This Loop Header: Depth=1
-; RV64IA-NEXT: # Child Loop BB0_6 Depth 2
-; RV64IA-NEXT: srlw a6, a3, a0
-; RV64IA-NEXT: andi a7, a6, 255
-; RV64IA-NEXT: bltu a7, a5, .LBB0_1
-; RV64IA-NEXT: # %bb.4: # in Loop: Header=BB0_3 Depth=1
-; RV64IA-NEXT: sub a6, a6, a1
-; RV64IA-NEXT: j .LBB0_2
-; RV64IA-NEXT: .LBB0_5: # %atomicrmw.end
+; RV64IA-NEXT: bnez t0, .LBB0_3
+; RV64IA-NEXT: # %bb.5: # %atomicrmw.start
+; RV64IA-NEXT: # %bb.2: # %atomicrmw.end
; RV64IA-NEXT: srlw a0, a3, a0
; RV64IA-NEXT: ret
%result = atomicrmw usub_cond ptr %ptr, i8 %val seq_cst
@@ -222,12 +194,13 @@ define i16 @atomicrmw_usub_cond_i16(ptr %ptr, i16 %val) {
; RV32I-NEXT: lui s2, 16
; RV32I-NEXT: addi s2, s2, -1
; RV32I-NEXT: and s3, s0, s2
-; RV32I-NEXT: j .LBB1_3
; RV32I-NEXT: .LBB1_1: # %atomicrmw.start
-; RV32I-NEXT: # in Loop: Header=BB1_3 Depth=1
-; RV32I-NEXT: mv a2, s0
-; RV32I-NEXT: .LBB1_2: # %atomicrmw.start
-; RV32I-NEXT: # in Loop: Header=BB1_3 Depth=1
+; RV32I-NEXT: # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT: and a0, a1, s2
+; RV32I-NEXT: sltu a0, a0, s3
+; RV32I-NEXT: addi a0, a0, -1
+; RV32I-NEXT: and a0, a0, s0
+; RV32I-NEXT: sub a2, a1, a0
; RV32I-NEXT: sh a1, 10(sp)
; RV32I-NEXT: addi a1, sp, 10
; RV32I-NEXT: li a3, 5
@@ -235,15 +208,8 @@ define i16 @atomicrmw_usub_cond_i16(ptr %ptr, i16 %val) {
; RV32I-NEXT: mv a0, s1
; RV32I-NEXT: call __atomic_compare_exchange_2
; RV32I-NEXT: lh a1, 10(sp)
-; RV32I-NEXT: bnez a0, .LBB1_5
-; RV32I-NEXT: .LBB1_3: # %atomicrmw.start
-; RV32I-NEXT: # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT: and a0, a1, s2
-; RV32I-NEXT: bltu a0, s3, .LBB1_1
-; RV32I-NEXT: # %bb.4: # in Loop: Header=BB1_3 Depth=1
-; RV32I-NEXT: sub a2, a1, s0
-; RV32I-NEXT: j .LBB1_2
-; RV32I-NEXT: .LBB1_5: # %atomicrmw.end
+; RV32I-NEXT: beqz a0, .LBB1_1
+; RV32I-NEXT: # %bb.2: # %atomicrmw.end
; RV32I-NEXT: mv a0, a1
; RV32I-NEXT: lw ra, 28(sp) # 4-byte Folded Reload
; RV32I-NEXT: lw s0, 24(sp) # 4-byte Folded Reload
@@ -260,44 +226,36 @@ define i16 @atomicrmw_usub_cond_i16(ptr %ptr, i16 %val) {
; RV32IA-NEXT: andi a0, a4, 24
; RV32IA-NEXT: lui a3, 16
; RV32IA-NEXT: addi a3, a3, -1
-; RV32IA-NEXT: lw a7, 0(a2)
+; RV32IA-NEXT: lw a6, 0(a2)
; RV32IA-NEXT: sll a4, a3, a4
; RV32IA-NEXT: not a4, a4
; RV32IA-NEXT: and a5, a1, a3
-; RV32IA-NEXT: j .LBB1_3
; RV32IA-NEXT: .LBB1_1: # %atomicrmw.start
-; RV32IA-NEXT: # in Loop: Header=BB1_3 Depth=1
-; RV32IA-NEXT: mv a7, a1
-; RV32IA-NEXT: .LBB1_2: # %atomicrmw.start
-; RV32IA-NEXT: # in Loop: Header=BB1_3 Depth=1
-; RV32IA-NEXT: and a7, a7, a3
-; RV32IA-NEXT: sll a7, a7, a0
-; RV32IA-NEXT: and t0, a6, a4
-; RV32IA-NEXT: or t0, t0, a7
-; RV32IA-NEXT: .LBB1_6: # %atomicrmw.start
-; RV32IA-NEXT: # Parent Loop BB1_3 Depth=1
+; RV32IA-NEXT: # =>This Loop Header: Depth=1
+; RV32IA-NEXT: # Child Loop BB1_3 Depth 2
+; RV32IA-NEXT: mv a7, a6
+; RV32IA-NEXT: srl a6, a6, a0
+; RV32IA-NEXT: and t0, a6, a3
+; RV32IA-NEXT: sltu t0, t0, a5
+; RV32IA-NEXT: addi t0, t0, -1
+; RV32IA-NEXT: and t0, t0, a1
+; RV32IA-NEXT: sub a6, a6, t0
+; RV32IA-NEXT: and a6, a6, a3
+; RV32IA-NEXT: sll a6, a6, a0
+; RV32IA-NEXT: and t0, a7, a4
+; RV32IA-NEXT: or t0, t0, a6
+; RV32IA-NEXT: .LBB1_3: # %atomicrmw.start
+; RV32IA-NEXT: # Parent Loop BB1_1 Depth=1
; RV32IA-NEXT: # => This Inner Loop Header: Depth=2
-; RV32IA-NEXT: lr.w.aqrl a7, (a2)
-; RV32IA-NEXT: bne a7, a6, .LBB1_8
-; RV32IA-NEXT: # %bb.7: # %atomicrmw.start
-; RV32IA-NEXT: # in Loop: Header=BB1_6 Depth=2
+; RV32IA-NEXT: lr.w.aqrl a6, (a2)
+; RV32IA-NEXT: bne a6, a7, .LBB1_1
+; RV32IA-NEXT: # %bb.4: # %atomicrmw.start
+; RV32IA-NEXT: # in Loop: Header=BB1_3 Depth=2
; RV32IA-NEXT: sc.w.rl t1, t0, (a2)
-; RV32IA-NEXT: bnez t1, .LBB1_6
-; RV32IA-NEXT: .LBB1_8: # %atomicrmw.start
-; RV32IA-NEXT: # in Loop: Header=BB1_3 Depth=1
-; RV32IA-NEXT: beq a7, a6, .LBB1_5
-; RV32IA-NEXT: .LBB1_3: # %atomicrmw.start
-; RV32IA-NEXT: # =>This Loop Header: Depth=1
-; RV32IA-NEXT: # Child Loop BB1_6 Depth 2
-; RV32IA-NEXT: mv a6, a7
-; RV32IA-NEXT: srl a7, a7, a0
-; RV32IA-NEXT: and t0, a7, a3
-; RV32IA-NEXT: bltu t0, a5, .LBB1_1
-; RV32IA-NEXT: # %bb.4: # in Loop: Header=BB1_3 Depth=1
-; RV32IA-NEXT: sub a7, a7, a1
-; RV32IA-NEXT: j .LBB1_2
-; RV32IA-NEXT: .LBB1_5: # %atomicrmw.end
-; RV32IA-NEXT: srl a0, a7, a0
+; RV32IA-NEXT: bnez t1, .LBB1_3
+; RV32IA-NEXT: # %bb.5: # %atomicrmw.start
+; RV32IA-NEXT: # %bb.2: # %atomicrmw.end
+; RV32IA-NEXT: srl a0, a6, a0
; RV32IA-NEXT: ret
;
; RV64I-LABEL: atomicrmw_usub_cond_i16:
@@ -320,12 +278,13 @@ define i16 @atomicrmw_usub_cond_i16(ptr %ptr, i16 %val) {
; RV64I-NEXT: lui s2, 16
; RV64I-NEXT: addiw s2, s2, -1
; RV64I-NEXT: and s3, s0, s2
-; RV64I-NEXT: j .LBB1_3
; RV64I-NEXT: .LBB1_1: # %atomicrmw.start
-; RV64I-NEXT: # in Loop: Header=BB1_3 Depth=1
-; RV64I-NEXT: mv a2, s0
-; RV64I-NEXT: .LBB1_2: # %atomicrmw.start
-; RV64I-NEXT: # in Loop: Header=BB1_3 Depth=1
+; RV64I-NEXT: # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT: and a0, a1, s2
+; RV64I-NEXT: sltu a0, a0, s3
+; RV64I-NEXT: addi a0, a0, -1
+; RV64I-NEXT: and a0, a0, s0
+; RV64I-NEXT: sub a2, a1, a0
; RV64I-NEXT: sh a1, 6(sp)
; RV64I-NEXT: addi a1, sp, 6
; RV64I-NEXT: li a3, 5
@@ -333,15 +292,8 @@ define i16 @atomicrmw_usub_cond_i16(ptr %ptr, i16 %val) {
; RV64I-NEXT: mv a0, s1
; RV64I-NEXT: call __atomic_compare_exchange_2
; RV64I-NEXT: lh a1, 6(sp)
-; RV64I-NEXT: bnez a0, .LBB1_5
-; RV64I-NEXT: .LBB1_3: # %atomicrmw.start
-; RV64I-NEXT: # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT: and a0, a1, s2
-; RV64I-NEXT: bltu a0, s3, .LBB1_1
-; RV64I-NEXT: # %bb.4: # in Loop: Header=BB1_3 Depth=1
-; RV64I-NEXT: sub a2, a1, s0
-; RV64I-NEXT: j .LBB1_2
-; RV64I-NEXT: .LBB1_5: # %atomicrmw.end
+; RV64I-NEXT: beqz a0, .LBB1_1
+; RV64I-NEXT: # %bb.2: # %atomicrmw.end
; RV64I-NEXT: mv a0, a1
; RV64I-NEXT: ld ra, 40(sp) # 8-byte Folded Reload
; RV64I-NEXT: ld s0, 32(sp) # 8-byte Folded Reload
@@ -362,39 +314,31 @@ define i16 @atomicrmw_usub_cond_i16(ptr %ptr, i16 %val) {
; RV64IA-NEXT: sllw a5, a3, a5
; RV64IA-NEXT: not a5, a5
; RV64IA-NEXT: and a6, a1, a3
-; RV64IA-NEXT: j .LBB1_3
; RV64IA-NEXT: .LBB1_1: # %atomicrmw.start
-; RV64IA-NEXT: # in Loop: Header=BB1_3 Depth=1
-; RV64IA-NEXT: mv a7, a1
-; RV64IA-NEXT: .LBB1_2: # %atomicrmw.start
-; RV64IA-NEXT: # in Loop: Header=BB1_3 Depth=1
+; RV64IA-NEXT: # =>This Loop Header: Depth=1
+; RV64IA-NEXT: # Child Loop BB1_3 Depth 2
+; RV64IA-NEXT: srlw a7, a4, a0
; RV64IA-NEXT: sext.w t0, a4
+; RV64IA-NEXT: and t1, a7, a3
+; RV64IA-NEXT: sltu t1, t1, a6
+; RV64IA-NEXT: addi t1, t1, -1
+; RV64IA-NEXT: and t1, t1, a1
+; RV64IA-NEXT: subw a7, a7, t1
; RV64IA-NEXT: and a7, a7, a3
; RV64IA-NEXT: sllw a7, a7, a0
; RV64IA-NEXT: and a4, a4, a5
; RV64IA-NEXT: or a7, a4, a7
-; RV64IA-NEXT: .LBB1_6: # %atomicrmw.start
-; RV64IA-NEXT: # Parent Loop BB1_3 Depth=1
+; RV64IA-NEXT: .LBB1_3: # %atomicrmw.start
+; RV64IA-NEXT: # Parent Loop BB1_1 Depth=1
; RV64IA-NEXT: # => This Inner Loop Header: Depth=2
; RV64IA-NEXT: lr.w.aqrl a4, (a2)
-; RV64IA-NEXT: bne a4, t0, .LBB1_8
-; RV64IA-NEXT: # %bb.7: # %atomicrmw.start
-; RV64IA-NEXT: # in Loop: Header=BB1_6 Depth=2
+; RV64IA-NEXT: bne a4, t0, .LBB1_1
+; RV64IA-NEXT: # %bb.4: # %atomicrmw.start
+; RV64IA-NEXT: # in Loop: Header=BB1_3 Depth=2
; RV64IA-NEXT: sc.w.rl t1, a7, (a2)
-; RV64IA-NEXT: bnez t1, .LBB1_6
-; RV64IA-NEXT: .LBB1_8: # %atomicrmw.start
-; RV64IA-NEXT: # in Loop: Header=BB1_3 Depth=1
-; RV64IA-NEXT: beq a4, t0, .LBB1_5
-; RV64IA-NEXT: .LBB1_3: # %atomicrmw.start
-; RV64IA-NEXT: # =>This Loop Header: Depth=1
-; RV64IA-NEXT: # Child Loop BB1_6 Depth 2
-; RV64IA-NEXT: srlw a7, a4, a0
-; RV64IA-NEXT: and t0, a7, a3
-; RV64IA-NEXT: bltu t0, a6, .LBB1_1
-; RV64IA-NEXT: # %bb.4: # in Loop: Header=BB1_3 Depth=1
-; RV64IA-NEXT: sub a7, a7, a1
-; RV64IA-NEXT: j .LBB1_2
-; RV64IA-NEXT: .LBB1_5: # %atomicrmw.end
+; RV64IA-NEXT: bnez t1, .LBB1_3
+; RV64IA-NEXT: # %bb.5: # %atomicrmw.start
+; RV64IA-NEXT: # %bb.2: # %atomicrmw.end
; RV64IA-NEXT: srlw a0, a4, a0
; RV64IA-NEXT: ret
%result = atomicrmw usub_cond ptr %ptr, i16 %val seq_cst
@@ -415,12 +359,12 @@ define i32 @atomicrmw_usub_cond_i32(ptr %ptr, i32 %val) {
; RV32I-NEXT: mv s0, a0
; RV32I-NEXT: lw a3, 0(a0)
; RV32I-NEXT: mv s1, a1
-; RV32I-NEXT: j .LBB2_3
; RV32I-NEXT: .LBB2_1: # %atomicrmw.start
-; RV32I-NEXT: # in Loop: Header=BB2_3 Depth=1
-; RV32I-NEXT: mv a2, s1
-; RV32I-NEXT: .LBB2_2: # %atomicrmw.start
-; RV32I-NEXT: # in Loop: Header=BB2_3 Depth=1
+; RV32I-NEXT: # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT: sltu a0, a3, s1
+; RV32I-NEXT: addi a0, a0, -1
+; RV32I-NEXT: and a0, a0, s1
+; RV32I-NEXT: sub a2, a3, a0
; RV32I-NEXT: sw a3, 0(sp)
; RV32I-NEXT: mv a1, sp
; RV32I-NEXT: li a3, 5
@@ -428,14 +372,8 @@ define i32 @atomicrmw_usub_cond_i32(ptr %ptr, i32 %val) {
; RV32I-NEXT: mv a0, s0
; RV32I-NEXT: call __atomic_compare_exchange_4
; RV32I-NEXT: lw a3, 0(sp)
-; RV32I-NEXT: bnez a0, .LBB2_5
-; RV32I-NEXT: .LBB2_3: # %atomicrmw.start
-; RV32I-NEXT: # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT: bltu a3, s1, .LBB2_1
-; RV32I-NEXT: # %bb.4: # in Loop: Header=BB2_3 Depth=1
-; RV32I-NEXT: sub a2, a3, s1
-; RV32I-NEXT: j .LBB2_2
-; RV32I-NEXT: .LBB2_5: # %atomicrmw.end
+; RV32I-NEXT: beqz a0, .LBB2_1
+; RV32I-NEXT: # %bb.2: # %atomicrmw.end
; RV32I-NEXT: mv a0, a3
; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
; RV32I-NEXT: lw s0, 8(sp) # 4-byte Folded Reload
@@ -446,38 +384,25 @@ define i32 @atomicrmw_usub_cond_i32(ptr %ptr, i32 %val) {
; RV32IA-LABEL: atomicrmw_usub_cond_i32:
; RV32IA: # %bb.0:
; RV32IA-NEXT: lw a2, 0(a0)
-; RV32IA-NEXT: j .LBB2_2
; RV32IA-NEXT: .LBB2_1: # %atomicrmw.start
-; RV32IA-NEXT: # in Loop: Header=BB2_2 Depth=1
-; RV32IA-NEXT: .LBB2_5: # %atomicrmw.start
-; RV32IA-NEXT: # Parent Loop BB2_2 Depth=1
-; RV32IA-NEXT: # => This Inner Loop Header: Depth=2
-; RV32IA-NEXT: lr.w.aqrl a2, (a0)
-; RV32IA-NEXT: bne a2, a3, .LBB2_7
-; RV32IA-NEXT: # %bb.6: # %atomicrmw.start
-; RV32IA-NEXT: # in Loop: Header=BB2_5 Depth=2
-; RV32IA-NEXT: sc.w.rl a5, a1, (a0)
-; RV32IA-NEXT: bnez a5, .LBB2_5
-; RV32IA-NEXT: .LBB2_7: # %atomicrmw.start
-; RV32IA-NEXT: # in Loop: Header=BB2_2 Depth=1
-; RV32IA-NEXT: beq a2, a3, .LBB2_4
-; RV32IA-NEXT: .LBB2_2: # %atomicrmw.start
; RV32IA-NEXT: # =>This Loop Header: Depth=1
-; RV32IA-NEXT: # Child Loop BB2_8 Depth 2
-; RV32IA-NEXT: # Child Loop BB2_5 Depth 2
+; RV32IA-NEXT: # Child Loop BB2_3 Depth 2
; RV32IA-NEXT: mv a3, a2
-; RV32IA-NEXT: bltu a2, a1, .LBB2_1
-; RV32IA-NEXT: # %bb.3: # in Loop: Header=BB2_2 Depth=1
-; RV32IA-NEXT: sub a4, a3, a1
-; RV32IA-NEXT: .LBB2_8: # Parent Loop BB2_2 Depth=1
+; RV32IA-NEXT: sltu a2, a2, a1
+; RV32IA-NEXT: addi a2, a2, -1
+; RV32IA-NEXT: and a2, a2, a1
+; RV32IA-NEXT: sub a4, a3, a2
+; RV32IA-NEXT: .LBB2_3: # %atomicrmw.start
+; RV32IA-NEXT: # Parent Loop BB2_1 Depth=1
; RV32IA-NEXT: # => This Inner Loop Header: Depth=2
; RV32IA-NEXT: lr.w.aqrl a2, (a0)
-; RV32IA-NEXT: bne a2, a3, .LBB2_2
-; RV32IA-NEXT: # %bb.9: # in Loop: Header=BB2_8 Depth=2
+; RV32IA-NEXT: bne a2, a3, .LBB2_1
+; RV32IA-NEXT: # %bb.4: # %atomicrmw.start
+; RV32IA-NEXT: # in Loop: Header=BB2_3 Depth=2
; RV32IA-NEXT: sc.w.rl a5, a4, (a0)
-; RV32IA-NEXT: bnez a5, .LBB2_8
-; RV32IA-NEXT: # %bb.10:
-; RV32IA-NEXT: .LBB2_4: # %atomicrmw.end
+; RV32IA-NEXT: bnez a5, .LBB2_3
+; RV32IA-NEXT: # %bb.5: # %atomicrmw.start
+; RV32IA-NEXT: # %bb.2: # %atomicrmw.end
; RV32IA-NEXT: mv a0, a2
; RV32IA-NEXT: ret
;
@@ -497,12 +422,12 @@ define i32 @atomicrmw_usub_cond_i32(ptr %ptr, i32 %val) {
; RV64I-NEXT: lw a3, 0(a0)
; RV64I-NEXT: mv s1, a1
; RV64I-NEXT: sext.w s2, a1
-; RV64I-NEXT: j .LBB2_3
; RV64I-NEXT: .LBB2_1: # %atomicrmw.start
-; RV64I-NEXT: # in Loop: Header=BB2_3 Depth=1
-; RV64I-NEXT: mv a2, s1
-; RV64I-NEXT: .LBB2_2: # %atomicrmw.start
-; RV64I-NEXT: # in Loop: Header=BB2_3 Depth=1
+; RV64I-NEXT: # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT: sltu a0, a3, s2
+; RV64I-NEXT: addi a0, a0, -1
+; RV64I-NEXT: and a0, a0, s1
+; RV64I-NEXT: subw a2, a3, a0
; RV64I-NEXT: sw a3, 12(sp)
; RV64I-NEXT: addi a1, sp, 12
; RV64I-NEXT: li a3, 5
@@ -510,14 +435,8 @@ define i32 @atomicrmw_usub_cond_i32(ptr %ptr, i32 %val) {
; RV64I-NEXT: mv a0, s0
; RV64I-NEXT: call __atomic_compare_exchange_4
; RV64I-NEXT: lw a3, 12(sp)
-; RV64I-NEXT: bnez a0, .LBB2_5
-; RV64I-NEXT: .LBB2_3: # %atomicrmw.start
-; RV64I-NEXT: # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT: bltu a3, s2, .LBB2_1
-; RV64I-NEXT: # %bb.4: # in Loop: Header=BB2_3 Depth=1
-; RV64I-NEXT: subw a2, a3, s1
-; RV64I-NEXT: j .LBB2_2
-; RV64I-NEXT: .LBB2_5: # %atomicrmw.end
+; RV64I-NEXT: beqz a0, .LBB2_1
+; RV64I-NEXT: # %bb.2: # %atomicrmw.end
; RV64I-NEXT: mv a0, a3
; RV64I-NEXT: ld ra, 40(sp) # 8-byte Folded Reload
; RV64I-NEXT: ld s0, 32(sp) # 8-byte Folded Reload
@@ -530,38 +449,25 @@ define i32 @atomicrmw_usub_cond_i32(ptr %ptr, i32 %val) {
; RV64IA: # %bb.0:
; RV64IA-NEXT: lw a2, 0(a0)
; RV64IA-NEXT: sext.w a3, a1
-; RV64IA-NEXT: j .LBB2_2
; RV64IA-NEXT: .LBB2_1: # %atomicrmw.start
-; RV64IA-NEXT: # in Loop: Header=BB2_2 Depth=1
-; RV64IA-NEXT: .LBB2_5: # %atomicrmw.start
-; RV64IA-NEXT: # Parent Loop BB2_2 Depth=1
-; RV64IA-NEXT: # => This Inner Loop Header: Depth=2
-; RV64IA-NEXT: lr.w.aqrl a2, (a0)
-; RV64IA-NEXT: bne a2, a4, .LBB2_7
-; RV64IA-NEXT: # %bb.6: # %atomicrmw.start
-; RV64IA-NEXT: # in Loop: Header=BB2_5 Depth=2
-; RV64IA-NEXT: sc.w.rl a6, a1, (a0)
-; RV64IA-NEXT: bnez a6, .LBB2_5
-; RV64IA-NEXT: .LBB2_7: # %atomicrmw.start
-; RV64IA-NEXT: # in Loop: Header=BB2_2 Depth=1
-; RV64IA-NEXT: beq a2, a4, .LBB2_4
-; RV64IA-NEXT: .LBB2_2: # %atomicrmw.start
; RV64IA-NEXT: # =>This Loop Header: Depth=1
-; RV64IA-NEXT: # Child Loop BB2_8 Depth 2
-; RV64IA-NEXT: # Child Loop BB2_5 Depth 2
+; RV64IA-NEXT: # Child Loop BB2_3 Depth 2
; RV64IA-NEXT: sext.w a4, a2
-; RV64IA-NEXT: bltu a4, a3, .LBB2_1
-; RV64IA-NEXT: # %bb.3: # in Loop: Header=BB2_2 Depth=1
-; RV64IA-NEXT: subw a5, a2, a1
-; RV64IA-NEXT: .LBB2_8: # Parent Loop BB2_2 Depth=1
+; RV64IA-NEXT: sltu a5, a4, a3
+; RV64IA-NEXT: addi a5, a5, -1
+; RV64IA-NEXT: and a5, a5, a1
+; RV64IA-NEXT: subw a5, a2, a5
+; RV64IA-NEXT: .LBB2_3: # %atomicrmw.start
+; RV64IA-NEXT: # Parent Loop BB2_1 Depth=1
; RV64IA-NEXT: # => This Inner Loop Header: Depth=2
; RV64IA-NEXT: lr.w.aqrl a2, (a0)
-; RV64IA-NEXT: bne a2, a4, .LBB2_2
-; RV64IA-NEXT: # %bb.9: # in Loop: Header=BB2_8 Depth=2
+; RV64IA-NEXT: bne a2, a4, .LBB2_1
+; RV64IA-NEXT: # %bb.4: # %atomicrmw.start
+; RV64IA-NEXT: # in Loop: Header=BB2_3 Depth=2
; RV64IA-NEXT: sc.w.rl a6, a5, (a0)
-; RV64IA-NEXT: bnez a6, .LBB2_8
-; RV64IA-NEXT: # %bb.10:
-; RV64IA-NEXT: .LBB2_4: # %atomicrmw.end
+; RV64IA-NEXT: bnez a6, .LBB2_3
+; RV64IA-NEXT: # %bb.5: # %atomicrmw.start
+; RV64IA-NEXT: # %bb.2: # %atomicrmw.end
; RV64IA-NEXT: mv a0, a2
; RV64IA-NEXT: ret
%result = atomicrmw usub_cond ptr %ptr, i32 %val seq_cst
@@ -589,10 +495,17 @@ define i64 @atomicrmw_usub_cond_i64(ptr %ptr, i64 %val) {
; RV32I-NEXT: j .LBB3_3
; RV32I-NEXT: .LBB3_1: # %atomicrmw.start
; RV32I-NEXT: # in Loop: Header=BB3_3 Depth=1
-; RV32I-NEXT: mv a3, s1
-; RV32I-NEXT: mv a2, s2
+; RV32I-NEXT: sltu a0, a5, s1
; RV32I-NEXT: .LBB3_2: # %atomicrmw.start
; RV32I-NEXT: # in Loop: Header=BB3_3 Depth=1
+; RV32I-NEXT: xori a0, a0, 1
+; RV32I-NEXT: neg a0, a0
+; RV32I-NEXT: and a1, a0, s2
+; RV32I-NEXT: sltu a2, a4, a1
+; RV32I-NEXT: and a0, a0, s1
+; RV32I-NEXT: sub a3, a5, a0
+; RV32I-NEXT: sub a3, a3, a2
+; RV32I-NEXT: sub a2, a4, a1
; RV32I-NEXT: sw a4, 8(sp)
; RV32I-NEXT: sw a5, 12(sp)
; RV32I-NEXT: addi a1, sp, 8
@@ -602,24 +515,14 @@ define i64 @atomicrmw_usub_cond_i64(ptr %ptr, i64 %val) {
; RV32I-NEXT: call __atomic_compare_exchange_8
; RV32I-NEXT: lw a5, 12(sp)
; RV32I-NEXT: lw a4, 8(sp)
-; RV32I-NEXT: bnez a0, .LBB3_7
+; RV32I-NEXT: bnez a0, .LBB3_5
; RV32I-NEXT: .LBB3_3: # %atomicrmw.start
; RV32I-NEXT: # =>This Inner Loop Header: Depth=1
+; RV32I-NEXT: bne a5, s1, .LBB3_1
+; RV32I-NEXT: # %bb.4: # in Loop: Header=BB3_3 Depth=1
; RV32I-NEXT: sltu a0, a4, s2
-; RV32I-NEXT: mv a1, a0
-; RV32I-NEXT: beq a5, s1, .LBB3_5
-; RV32I-NEXT: # %bb.4: # %atomicrmw.start
-; RV32I-NEXT: # in Loop: Header=BB3_3 Depth=1
-; RV32I-NEXT: sltu a1, a5, s1
-; RV32I-NEXT: .LBB3_5: # %atomicrmw.start
-; RV32I-NEXT: # in Loop: Header=BB3_3 Depth=1
-; RV32I-NEXT: bnez a1, .LBB3_1
-; RV32I-NEXT: # %bb.6: # in Loop: Header=BB3_3 Depth=1
-; RV32I-NEXT: sub a3, a5, s1
-; RV32I-NEXT: sub a3, a3, a0
-; RV32I-NEXT: sub a2, a4, s2
; RV32I-NEXT: j .LBB3_2
-; RV32I-NEXT: .LBB3_7: # %atomicrmw.end
+; RV32I-NEXT: .LBB3_5: # %atomicrmw.end
; RV32I-NEXT: mv a0, a4
; RV32I-NEXT: mv a1, a5
; RV32I-NEXT: lw ra, 28(sp) # 4-byte Folded Reload
@@ -649,10 +552,17 @@ define i64 @atomicrmw_usub_cond_i64(ptr %ptr, i64 %val) {
; RV32IA-NEXT: j .LBB3_3
; RV32IA-NEXT: .LBB3_1: # %atomicrmw.start
; RV32IA-NEXT: # in Loop: Header=BB3_3 Depth=1
-; RV32IA-NEXT: mv a3, s1
-; RV32IA-NEXT: mv a2, s2
+; RV32IA-NEXT: sltu a0, a5, s1
; RV32IA-NEXT: .LBB3_2: # %atomicrmw.start
; RV32IA-NEXT: # in Loop: Header=BB3_3 Depth=1
+; RV32IA-NEXT: xori a0, a0, 1
+; RV32IA-NEXT: neg a0, a0
+; RV32IA-NEXT: and a1, a0, s2
+; RV32IA-NEXT: sltu a2, a4, a1
+; RV32IA-NEXT: and a0, a0, s1
+; RV32IA-NEXT: sub a3, a5, a0
+; RV32IA-NEXT: sub a3, a3, a2
+; RV32IA-NEXT: sub a2, a4, a1
; RV32IA-NEXT: sw a4, 8(sp)
; RV32IA-NEXT: sw a5, 12(sp)
; RV32IA-NEXT: addi a1, sp, 8
@@ -662,24 +572,14 @@ define i64 @atomicrmw_usub_cond_i64(ptr %ptr, i64 %val) {
; RV32IA-NEXT: call __atomic_compare_exchange_8
; RV32IA-NEXT: lw a5, 12(sp)
; RV32IA-NEXT: lw a4, 8(sp)
-; RV32IA-NEXT: bnez a0, .LBB3_7
+; RV32IA-NEXT: bnez a0, .LBB3_5
; RV32IA-NEXT: .LBB3_3: # %atomicrmw.start
; RV32IA-NEXT: # =>This Inner Loop Header: Depth=1
+; RV32IA-NEXT: bne a5, s1, .LBB3_1
+; RV32IA-NEXT: # %bb.4: # in Loop: Header=BB3_3 Depth=1
; RV32IA-NEXT: sltu a0, a4, s2
-; RV32IA-NEXT: mv a1, a0
-; RV32IA-NEXT: beq a5, s1, .LBB3_5
-; RV32IA-NEXT: # %bb.4: # %atomicrmw.start
-; RV32IA-NEXT: # in Loop: Header=BB3_3 Depth=1
-; RV32IA-NEXT: sltu a1, a5, s1
-; RV32IA-NEXT: .LBB3_5: # %atomicrmw.start
-; RV32IA-NEXT: # in Loop: Header=BB3_3 Depth=1
-; RV32IA-NEXT: bnez a1, .LBB3_1
-; RV32IA-NEXT: # %bb.6: # in Loop: Header=BB3_3 Depth=1
-; RV32IA-NEXT: sub a3, a5, s1
-; RV32IA-NEXT: sub a3, a3, a0
-; RV32IA-NEXT: sub a2, a4, s2
; RV32IA-NEXT: j .LBB3_2
-; RV32IA-NEXT: .LBB3_7: # %atomicrmw.end
+; RV32IA-NEXT: .LBB3_5: # %atomicrmw.end
; RV32IA-NEXT: mv a0, a4
; RV32IA-NEXT: mv a1, a5
; RV32IA-NEXT: lw ra, 28(sp) # 4-byte Folded Reload
@@ -702,12 +602,12 @@ define i64 @atomicrmw_usub_cond_i64(ptr %ptr, i64 %val) {
; RV64I-NEXT: mv s0, a0
; RV64I-NEXT: ld a3, 0(a0)
; RV64I-NEXT: mv s1, a1
-; RV64I-NEXT: j .LBB3_3
; RV64I-NEXT: .LBB3_1: # %atomicrmw.start
-; RV64I-NEXT: # in Loop: Header=BB3_3 Depth=1
-; RV64I-NEXT: mv a2, s1
-; RV64I-NEXT: .LBB3_2: # %atomicrmw.start
-; RV64I-NEXT: # in Loop: Header=BB3_3 Depth=1
+; RV64I-NEXT: # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT: sltu a0, a3, s1
+; RV64I-NEXT: addi a0, a0, -1
+; RV64I-NEXT: and a0, a0, s1
+; RV64I-NEXT: sub a2, a3, a0
; RV64I-NEXT: sd a3, 0(sp)
; RV64I-NEXT: mv a1, sp
; RV64I-NEXT: li a3, 5
@@ -715,14 +615,8 @@ define i64 @atomicrmw_usub_cond_i64(ptr %ptr, i64 %val) {
; RV64I-NEXT: mv a0, s0
; RV64I-NEXT: call __atomic_compare_exchange_8
; RV64I-NEXT: ld a3, 0(sp)
-; RV64I-NEXT: bnez a0, .LBB3_5
-; RV64I-NEXT: .LBB3_3: # %atomicrmw.start
-; RV64I-NEXT: # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT: bltu a3, s1, .LBB3_1
-; RV64I-NEXT: # %bb.4: # in Loop: Header=BB3_3 Depth=1
-; RV64I-NEXT: sub a2, a3, s1
-; RV64I-NEXT: j .LBB3_2
-; RV64I-NEXT: .LBB3_5: # %atomicrmw.end
+; RV64I-NEXT: beqz a0, .LBB3_1
+; RV64I-NEXT: # %bb.2: # %atomicrmw.end
; RV64I-NEXT: mv a0, a3
; RV64I-NEXT: ld ra, 24(sp) # 8-byte Folded Reload
; RV64I-NEXT: ld s0, 16(sp) # 8-byte Folded Reload
@@ -733,38 +627,25 @@ define i64 @atomicrmw_usub_cond_i64(ptr %ptr, i64 %val) {
; RV64IA-LABEL: atomicrmw_usub_cond_i64:
; RV64IA: # %bb.0:
; RV64IA-NEXT: ld a2, 0(a0)
-; RV64IA-NEXT: j .LBB3_2
; RV64IA-NEXT: .LBB3_1: # %atomicrmw.start
-; RV64IA-NEXT: # in Loop: Header=BB3_2 Depth=1
-; RV64IA-NEXT: .LBB3_5: # %atomicrmw.start
-; RV64IA-NEXT: # Parent Loop BB3_2 Depth=1
-; RV64IA-NEXT: # => This Inner Loop Header: Depth=2
-; RV64IA-NEXT: lr.d.aqrl a2, (a0)
-; RV64IA-NEXT: bne a2, a3, .LBB3_7
-; RV64IA-NEXT: # %bb.6: # %atomicrmw.start
-; RV64IA-NEXT: # in Loop: Header=BB3_5 Depth=2
-; RV64IA-NEXT: sc.d.rl a5, a1, (a0)
-; RV64IA-NEXT: bnez a5, .LBB3_5
-; RV64IA-NEXT: .LBB3_7: # %atomicrmw.start
-; RV64IA-NEXT: # in Loop: Header=BB3_2 Depth=1
-; RV64IA-NEXT: beq a2, a3, .LBB3_4
-; RV64IA-NEXT: .LBB3_2: # %atomicrmw.start
; RV64IA-NEXT: # =>This Loop Header: Depth=1
-; RV64IA-NEXT: # Child Loop BB3_8 Depth 2
-; RV64IA-NEXT: # Child Loop BB3_5 Depth 2
+; RV64IA-NEXT: # Child Loop BB3_3 Depth 2
; RV64IA-NEXT: mv a3, a2
-; RV64IA-NEXT: bltu a2, a1, .LBB3_1
-; RV64IA-NEXT: # %bb.3: # in Loop: Header=BB3_2 Depth=1
-; RV64IA-NEXT: sub a4, a3, a1
-; RV64IA-NEXT: .LBB3_8: # Parent Loop BB3_2 Depth=1
+; RV64IA-NEXT: sltu a2, a2, a1
+; RV64IA-NEXT: addi a2, a2, -1
+; RV64IA-NEXT: and a2, a2, a1
+; RV64IA-NEXT: sub a4, a3, a2
+; RV64IA-NEXT: .LBB3_3: # %atomicrmw.start
+; RV64IA-NEXT: # Parent Loop BB3_1 Depth=1
; RV64IA-NEXT: # => This Inner Loop Header: Depth=2
; RV64IA-NEXT: lr.d.aqrl a2, (a0)
-; RV64IA-NEXT: bne a2, a3, .LBB3_2
-; RV64IA-NEXT: # %bb.9: # in Loop: Header=BB3_8 Depth=2
+; RV64IA-NEXT: bne a2, a3, .LBB3_1
+; RV64IA-NEXT: # %bb.4: # %atomicrmw.start
+; RV64IA-NEXT: # in Loop: Header=BB3_3 Depth=2
; RV64IA-NEXT: sc.d.rl a5, a4, (a0)
-; RV64IA-NEXT: bnez a5, .LBB3_8
-; RV64IA-NEXT: # %bb.10:
-; RV64IA-NEXT: .LBB3_4: # %atomicrmw.end
+; RV64IA-NEXT: bnez a5, .LBB3_3
+; RV64IA-NEXT: # %bb.5: # %atomicrmw.start
+; RV64IA-NEXT: # %bb.2: # %atomicrmw.end
; RV64IA-NEXT: mv a0, a2
; RV64IA-NEXT: ret
%result = atomicrmw usub_cond ptr %ptr, i64 %val seq_cst
@@ -774,42 +655,38 @@ define i64 @atomicrmw_usub_cond_i64(ptr %ptr, i64 %val) {
define i8 @atomicrmw_usub_sat_i8(ptr %ptr, i8 %val) {
; RV32I-LABEL: atomicrmw_usub_sat_i8:
; RV32I: # %bb.0:
-; RV32I-NEXT: addi sp, sp, -32
-; RV32I-NEXT: .cfi_def_cfa_offset 32
-; RV32I-NEXT: sw ra, 28(sp) # 4-byte Folded Spill
-; RV32I-NEXT: sw s0, 24(sp) # 4-byte Folded Spill
-; RV32I-NEXT: sw s1, 20(sp) # 4-byte Folded Spill
-; RV32I-NEXT: sw s2, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT: addi sp, sp, -16
+; RV32I-NEXT: .cfi_def_cfa_offset 16
+; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s1, 4(sp) # 4-byte Folded Spill
; RV32I-NEXT: .cfi_offset ra, -4
; RV32I-NEXT: .cfi_offset s0, -8
; RV32I-NEXT: .cfi_offset s1, -12
-; RV32I-NEXT: .cfi_offset s2, -16
; RV32I-NEXT: mv s0, a0
; RV32I-NEXT: lbu a3, 0(a0)
-; RV32I-NEXT: mv s1, a1
-; RV32I-NEXT: andi s2, a1, 255
+; RV32I-NEXT: andi s1, a1, 255
; RV32I-NEXT: .LBB4_1: # %atomicrmw.start
; RV32I-NEXT: # =>This Inner Loop Header: Depth=1
; RV32I-NEXT: andi a0, a3, 255
-; RV32I-NEXT: sltu a0, a0, s2
-; RV32I-NEXT: sub a1, a3, s1
+; RV32I-NEXT: sub a1, a0, s1
+; RV32I-NEXT: sltu a0, a0, a1
; RV32I-NEXT: addi a0, a0, -1
; RV32I-NEXT: and a2, a0, a1
-; RV32I-NEXT: sb a3, 15(sp)
-; RV32I-NEXT: addi a1, sp, 15
+; RV32I-NEXT: sb a3, 3(sp)
+; RV32I-NEXT: addi a1, sp, 3
; RV32I-NEXT: li a3, 5
; RV32I-NEXT: li a4, 5
; RV32I-NEXT: mv a0, s0
; RV32I-NEXT: call __atomic_compare_exchange_1
-; RV32I-NEXT: lbu a3, 15(sp)
+; RV32I-NEXT: lbu a3, 3(sp)
; RV32I-NEXT: beqz a0, .LBB4_1
; RV32I-NEXT: # %bb.2: # %atomicrmw.end
; RV32I-NEXT: mv a0, a3
-; RV32I-NEXT: lw ra, 28(sp) # 4-byte Folded Reload
-; RV32I-NEXT: lw s0, 24(sp) # 4-byte Folded Reload
-; RV32I-NEXT: lw s1, 20(sp) # 4-byte Folded Reload
-; RV32I-NEXT: lw s2, 16(sp) # 4-byte Folded Reload
-; RV32I-NEXT: addi sp, sp, 32
+; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 16
; RV32I-NEXT: ret
;
; RV32IA-LABEL: atomicrmw_usub_sat_i8:
@@ -817,77 +694,72 @@ define i8 @atomicrmw_usub_sat_i8(ptr %ptr, i8 %val) {
; RV32IA-NEXT: andi a2, a0, -4
; RV32IA-NEXT: slli a3, a0, 3
; RV32IA-NEXT: andi a0, a3, 24
-; RV32IA-NEXT: li a4, 255
-; RV32IA-NEXT: lw a5, 0(a2)
-; RV32IA-NEXT: sll a3, a4, a3
+; RV32IA-NEXT: li a5, 255
+; RV32IA-NEXT: lw a4, 0(a2)
+; RV32IA-NEXT: sll a3, a5, a3
; RV32IA-NEXT: not a3, a3
-; RV32IA-NEXT: andi a4, a1, 255
+; RV32IA-NEXT: andi a1, a1, 255
; RV32IA-NEXT: .LBB4_1: # %atomicrmw.start
; RV32IA-NEXT: # =>This Loop Header: Depth=1
; RV32IA-NEXT: # Child Loop BB4_3 Depth 2
-; RV32IA-NEXT: mv a6, a5
-; RV32IA-NEXT: srl a5, a5, a0
-; RV32IA-NEXT: andi a7, a5, 255
-; RV32IA-NEXT: sltu a7, a7, a4
-; RV32IA-NEXT: sub a5, a5, a1
-; RV32IA-NEXT: addi a7, a7, -1
-; RV32IA-NEXT: and a5, a7, a5
-; RV32IA-NEXT: andi a5, a5, 255
-; RV32IA-NEXT: sll a5, a5, a0
-; RV32IA-NEXT: and a7, a6, a3
-; RV32IA-NEXT: or a7, a7, a5
+; RV32IA-NEXT: mv a5, a4
+; RV32IA-NEXT: srl a4, a4, a0
+; RV32IA-NEXT: andi a4, a4, 255
+; RV32IA-NEXT: sub a6, a4, a1
+; RV32IA-NEXT: sltu a4, a4, a6
+; RV32IA-NEXT: addi a4, a4, -1
+; RV32IA-NEXT: and a4, a4, a6
+; RV32IA-NEXT: sll a4, a4, a0
+; RV32IA-NEXT: and a6, a5, a3
+; RV32IA-NEXT: or a6, a6, a4
; RV32IA-NEXT: .LBB4_3: # %atomicrmw.start
; RV32IA-NEXT: # Parent Loop BB4_1 Depth=1
; RV32IA-NEXT: # => This Inner Loop Header: Depth=2
-; RV32IA-NEXT: lr.w.aqrl a5, (a2)
-; RV32IA-NEXT: bne a5, a6, .LBB4_1
+; RV32IA-NEXT: lr.w.aqrl a4, (a2)
+; RV32IA-NEXT: bne a4, a5, .LBB4_1
; RV32IA-NEXT: # %bb.4: # %atomicrmw.start
; RV32IA-NEXT: # in Loop: Header=BB4_3 Depth=2
-; RV32IA-NEXT: sc.w.rl t0, a7, (a2)
-; RV32IA-NEXT: bnez t0, .LBB4_3
+; RV32IA-NEXT: sc.w.rl a7, a6, (a2)
+; RV32IA-NEXT: bnez a7, .LBB4_3
; RV32IA-NEXT: # %bb.5: # %atomicrmw.start
; RV32IA-NEXT: # %bb.2: # %atomicrmw.end
-; RV32IA-NEXT: srl a0, a5, a0
+; RV32IA-NEXT: srl a0, a4, a0
; RV32IA-NEXT: ret
;
; RV64I-LABEL: atomicrmw_usub_sat_i8:
; RV64I: # %bb.0:
-; RV64I-NEXT: addi sp, sp, -48
-; RV64I-NEXT: .cfi_def_cfa_offset 48
-; RV64I-NEXT: sd ra, 40(sp) # 8-byte Folded Spill
-; RV64I-NEXT: sd s0, 32(sp) # 8-byte Folded Spill
-; RV64I-NEXT: sd s1, 24(sp) # 8-byte Folded Spill
-; RV64I-NEXT: sd s2, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT: addi sp, sp, -32
+; RV64I-NEXT: .cfi_def_cfa_offset 32
+; RV64I-NEXT: sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s1, 8(sp) # 8-byte Folded Spill
; RV64I-NEXT: .cfi_offset ra, -8
; RV64I-NEXT: .cfi_offset s0, -16
; RV64I-NEXT: .cfi_offset s1, -24
-; RV64I-NEXT: .cfi_offset s2, -32
; RV64I-NEXT: mv s0, a0
; RV64I-NEXT: lbu a3, 0(a0)
-; RV64I-NEXT: mv s1, a1
-; RV64I-NEXT: andi s2, a1, 255
+; RV64I-NEXT: andi s1, a1, 255
; RV64I-NEXT: .LBB4_1: # %atomicrmw.start
; RV64I-NEXT: # =>This Inner Loop Header: Depth=1
; RV64I-NEXT: andi a0, a3, 255
-; RV64I-NEXT: sltu a0, a0, s2
-; RV64I-NEXT: sub a1, a3, s1
+; RV64I-NEXT: sub a1, a0, s1
+; RV64I-NEXT: sltu a0, a0, a1
; RV64I-NEXT: addi a0, a0, -1
; RV64I-NEXT: and a2, a0, a1
-; RV64I-NEXT: sb a3, 15(sp)
-; RV64I-NEXT: addi a1, sp, 15
+; RV64I-NEXT: sb a3, 7(sp)
+; RV64I-NEXT: addi a1, sp, 7
; RV64I-NEXT: li a3, 5
; RV64I-NEXT: li a4, 5
; RV64I-NEXT: mv a0, s0
; RV64I-NEXT: call __atomic_compare_exchange_1
-; RV64I-NEXT: lbu a3, 15(sp)
+; RV64I-NEXT: lbu a3, 7(sp)
; RV64I-NEXT: beqz a0, .LBB4_1
; RV64I-NEXT: # %bb.2: # %atomicrmw.end
; RV64I-NEXT: mv a0, a3
-; RV64I-NEXT: ld ra, 40(sp) # 8-byte Folded Reload
-; RV64I-NEXT: ld s0, 32(sp) # 8-byte Folded Reload
-; RV64I-NEXT: ld s1, 24(sp) # 8-byte Folded Reload
-; RV64I-NEXT: ld s2, 16(sp) # 8-byte Folded Reload
-; RV64I-NEXT: addi sp, sp, 48
+; RV64I-NEXT: ld ra, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 32
; RV64I-NEXT: ret
;
; RV64IA-LABEL: atomicrmw_usub_sat_i8:
@@ -899,30 +771,29 @@ define i8 @atomicrmw_usub_sat_i8(ptr %ptr, i8 %val) {
; RV64IA-NEXT: lw a3, 0(a2)
; RV64IA-NEXT: sllw a4, a5, a4
; RV64IA-NEXT: not a4, a4
-; RV64IA-NEXT: andi a5, a1, 255
+; RV64IA-NEXT: andi a1, a1, 255
; RV64IA-NEXT: .LBB4_1: # %atomicrmw.start
; RV64IA-NEXT: # =>This Loop Header: Depth=1
; RV64IA-NEXT: # Child Loop BB4_3 Depth 2
-; RV64IA-NEXT: srlw a6, a3, a0
-; RV64IA-NEXT: sext.w a7, a3
-; RV64IA-NEXT: andi t0, a6, 255
-; RV64IA-NEXT: sltu t0, t0, a5
-; RV64IA-NEXT: subw a6, a6, a1
-; RV64IA-NEXT: addi t0, t0, -1
-; RV64IA-NEXT: and a6, t0, a6
-; RV64IA-NEXT: andi a6, a6, 255
-; RV64IA-NEXT: sllw a6, a6, a0
+; RV64IA-NEXT: srlw a5, a3, a0
+; RV64IA-NEXT: sext.w a6, a3
+; RV64IA-NEXT: andi a5, a5, 255
+; RV64IA-NEXT: sub a7, a5, a1
+; RV64IA-NEXT: sltu a5, a5, a7
+; RV64IA-NEXT: addi a5, a5, -1
+; RV64IA-NEXT: and a5, a5, a7
+; RV64IA-NEXT: sllw a5, a5, a0
; RV64IA-NEXT: and a3, a3, a4
-; RV64IA-NEXT: or a6, a3, a6
+; RV64IA-NEXT: or a5, a3, a5
; RV64IA-NEXT: .LBB4_3: # %atomicrmw.start
; RV64IA-NEXT: # Parent Loop BB4_1 Depth=1
; RV64IA-NEXT: # => This Inner Loop Header: Depth=2
; RV64IA-NEXT: lr.w.aqrl a3, (a2)
-; RV64IA-NEXT: bne a3, a7, .LBB4_1
+; RV64IA-NEXT: bne a3, a6, .LBB4_1
; RV64IA-NEXT: # %bb.4: # %atomicrmw.start
; RV64IA-NEXT: # in Loop: Header=BB4_3 Depth=2
-; RV64IA-NEXT: sc.w.rl t0, a6, (a2)
-; RV64IA-NEXT: bnez t0, .LBB4_3
+; RV64IA-NEXT: sc.w.rl a7, a5, (a2)
+; RV64IA-NEXT: bnez a7, .LBB4_3
; RV64IA-NEXT: # %bb.5: # %atomicrmw.start
; RV64IA-NEXT: # %bb.2: # %atomicrmw.end
; RV64IA-NEXT: srlw a0, a3, a0
@@ -940,40 +811,36 @@ define i16 @atomicrmw_usub_sat_i16(ptr %ptr, i16 %val) {
; RV32I-NEXT: sw s0, 24(sp) # 4-byte Folded Spill
; RV32I-NEXT: sw s1, 20(sp) # 4-byte Folded Spill
; RV32I-NEXT: sw s2, 16(sp) # 4-byte Folded Spill
-; RV32I-NEXT: sw s3, 12(sp) # 4-byte Folded Spill
; RV32I-NEXT: .cfi_offset ra, -4
; RV32I-NEXT: .cfi_offset s0, -8
; RV32I-NEXT: .cfi_offset s1, -12
; RV32I-NEXT: .cfi_offset s2, -16
-; RV32I-NEXT: .cfi_offset s3, -20
-; RV32I-NEXT: mv s0, a1
-; RV32I-NEXT: mv s1, a0
-; RV32I-NEXT: lhu a1, 0(a0)
-; RV32I-NEXT: lui s2, 16
-; RV32I-NEXT: addi s2, s2, -1
-; RV32I-NEXT: and s3, s0, s2
+; RV32I-NEXT: mv s0, a0
+; RV32I-NEXT: lhu a3, 0(a0)
+; RV32I-NEXT: lui s1, 16
+; RV32I-NEXT: addi s1, s1, -1
+; RV32I-NEXT: and s2, a1, s1
; RV32I-NEXT: .LBB5_1: # %atomicrmw.start
; RV32I-NEXT: # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT: and a0, a1, s2
-; RV32I-NEXT: sltu a0, a0, s3
-; RV32I-NEXT: sub a2, a1, s0
+; RV32I-NEXT: and a0, a3, s1
+; RV32I-NEXT: sub a1, a0, s2
+; RV32I-NEXT: sltu a0, a0, a1
; RV32I-NEXT: addi a0, a0, -1
-; RV32I-NEXT: and a2, a0, a2
-; RV32I-NEXT: sh a1, 10(sp)
-; RV32I-NEXT: addi a1, sp, 10
+; RV32I-NEXT: and a2, a0, a1
+; RV32I-NEXT: sh a3, 14(sp)
+; RV32I-NEXT: addi a1, sp, 14
; RV32I-NEXT: li a3, 5
; RV32I-NEXT: li a4, 5
-; RV32I-NEXT: mv a0, s1
+; RV32I-NEXT: mv a0, s0
; RV32I-NEXT: call __atomic_compare_exchange_2
-; RV32I-NEXT: lh a1, 10(sp)
+; RV32I-NEXT: lh a3, 14(sp)
; RV32I-NEXT: beqz a0, .LBB5_1
; RV32I-NEXT: # %bb.2: # %atomicrmw.end
-; RV32I-NEXT: mv a0, a1
+; RV32I-NEXT: mv a0, a3
; RV32I-NEXT: lw ra, 28(sp) # 4-byte Folded Reload
; RV32I-NEXT: lw s0, 24(sp) # 4-byte Folded Reload
; RV32I-NEXT: lw s1, 20(sp) # 4-byte Folded Reload
; RV32I-NEXT: lw s2, 16(sp) # 4-byte Folded Reload
-; RV32I-NEXT: lw s3, 12(sp) # 4-byte Folded Reload
; RV32I-NEXT: addi sp, sp, 32
; RV32I-NEXT: ret
;
@@ -984,36 +851,35 @@ define i16 @atomicrmw_usub_sat_i16(ptr %ptr, i16 %val) {
; RV32IA-NEXT: andi a0, a4, 24
; RV32IA-NEXT: lui a3, 16
; RV32IA-NEXT: addi a3, a3, -1
-; RV32IA-NEXT: lw a6, 0(a2)
+; RV32IA-NEXT: lw a5, 0(a2)
; RV32IA-NEXT: sll a4, a3, a4
; RV32IA-NEXT: not a4, a4
-; RV32IA-NEXT: and a5, a1, a3
+; RV32IA-NEXT: and a1, a1, a3
; RV32IA-NEXT: .LBB5_1: # %atomicrmw.start
; RV32IA-NEXT: # =>This Loop Header: Depth=1
; RV32IA-NEXT: # Child Loop BB5_3 Depth 2
-; RV32IA-NEXT: mv a7, a6
-; RV32IA-NEXT: srl a6, a6, a0
-; RV32IA-NEXT: and t0, a6, a3
-; RV32IA-NEXT: sltu t0, t0, a5
-; RV32IA-NEXT: sub a6, a6, a1
-; RV32IA-NEXT: addi t0, t0, -1
-; RV32IA-NEXT: and a6, a6, a3
-; RV32IA-NEXT: and a6, t0, a6
-; RV32IA-NEXT: sll a6, a6, a0
-; RV32IA-NEXT: and t0, a7, a4
-; RV32IA-NEXT: or t0, t0, a6
+; RV32IA-NEXT: mv a6, a5
+; RV32IA-NEXT: srl a5, a5, a0
+; RV32IA-NEXT: and a5, a5, a3
+; RV32IA-NEXT: sub a7, a5, a1
+; RV32IA-NEXT: sltu a5, a5, a7
+; RV32IA-NEXT: addi a5, a5, -1
+; RV32IA-NEXT: and a5, a5, a7
+; RV32IA-NEXT: sll a5, a5, a0
+; RV32IA-NEXT: and a7, a6, a4
+; RV32IA-NEXT: or a7, a7, a5
; RV32IA-NEXT: .LBB5_3: # %atomicrmw.start
; RV32IA-NEXT: # Parent Loop BB5_1 Depth=1
; RV32IA-NEXT: # => This Inner Loop Header: Depth=2
-; RV32IA-NEXT: lr.w.aqrl a6, (a2)
-; RV32IA-NEXT: bne a6, a7, .LBB5_1
+; RV32IA-NEXT: lr.w.aqrl a5, (a2)
+; RV32IA-NEXT: bne a5, a6, .LBB5_1
; RV32IA-NEXT: # %bb.4: # %atomicrmw.start
; RV32IA-NEXT: # in Loop: Header=BB5_3 Depth=2
-; RV32IA-NEXT: sc.w.rl t1, t0, (a2)
-; RV32IA-NEXT: bnez t1, .LBB5_3
+; RV32IA-NEXT: sc.w.rl t0, a7, (a2)
+; RV32IA-NEXT: bnez t0, .LBB5_3
; RV32IA-NEXT: # %bb.5: # %atomicrmw.start
; RV32IA-NEXT: # %bb.2: # %atomicrmw.end
-; RV32IA-NEXT: srl a0, a6, a0
+; RV32IA-NEXT: srl a0, a5, a0
; RV32IA-NEXT: ret
;
; RV64I-LABEL: atomicrmw_usub_sat_i16:
@@ -1024,40 +890,36 @@ define i16 @atomicrmw_usub_sat_i16(ptr %ptr, i16 %val) {
; RV64I-NEXT: sd s0, 32(sp) # 8-byte Folded Spill
; RV64I-NEXT: sd s1, 24(sp) # 8-byte Folded Spill
; RV64I-NEXT: sd s2, 16(sp) # 8-byte Folded Spill
-; RV64I-NEXT: sd s3, 8(sp) # 8-byte Folded Spill
; RV64I-NEXT: .cfi_offset ra, -8
; RV64I-NEXT: .cfi_offset s0, -16
; RV64I-NEXT: .cfi_offset s1, -24
; RV64I-NEXT: .cfi_offset s2, -32
-; RV64I-NEXT: .cfi_offset s3, -40
-; RV64I-NEXT: mv s0, a1
-; RV64I-NEXT: mv s1, a0
-; RV64I-NEXT: lhu a1, 0(a0)
-; RV64I-NEXT: lui s2, 16
-; RV64I-NEXT: addiw s2, s2, -1
-; RV64I-NEXT: and s3, s0, s2
+; RV64I-NEXT: mv s0, a0
+; RV64I-NEXT: lhu a3, 0(a0)
+; RV64I-NEXT: lui s1, 16
+; RV64I-NEXT: addiw s1, s1, -1
+; RV64I-NEXT: and s2, a1, s1
; RV64I-NEXT: .LBB5_1: # %atomicrmw.start
; RV64I-NEXT: # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT: and a0, a1, s2
-; RV64I-NEXT: sltu a0, a0, s3
-; RV64I-NEXT: sub a2, a1, s0
+; RV64I-NEXT: and a0, a3, s1
+; RV64I-NEXT: sub a1, a0, s2
+; RV64I-NEXT: sltu a0, a0, a1
; RV64I-NEXT: addi a0, a0, -1
-; RV64I-NEXT: and a2, a0, a2
-; RV64I-NEXT: sh a1, 6(sp)
-; RV64I-NEXT: addi a1, sp, 6
+; RV64I-NEXT: and a2, a0, a1
+; RV64I-NEXT: sh a3, 14(sp)
+; RV64I-NEXT: addi a1, sp, 14
; RV64I-NEXT: li a3, 5
; RV64I-NEXT: li a4, 5
-; RV64I-NEXT: mv a0, s1
+; RV64I-NEXT: mv a0, s0
; RV64I-NEXT: call __atomic_compare_exchange_2
-; RV64I-NEXT: lh a1, 6(sp)
+; RV64I-NEXT: lh a3, 14(sp)
; RV64I-NEXT: beqz a0, .LBB5_1
; RV64I-NEXT: # %bb.2: # %atomicrmw.end
-; RV64I-NEXT: mv a0, a1
+; RV64I-NEXT: mv a0, a3
; RV64I-NEXT: ld ra, 40(sp) # 8-byte Folded Reload
; RV64I-NEXT: ld s0, 32(sp) # 8-byte Folded Reload
; RV64I-NEXT: ld s1, 24(sp) # 8-byte Folded Reload
; RV64I-NEXT: ld s2, 16(sp) # 8-byte Folded Reload
-; RV64I-NEXT: ld s3, 8(sp) # 8-byte Folded Reload
; RV64I-NEXT: addi sp, sp, 48
; RV64I-NEXT: ret
;
@@ -1071,30 +933,29 @@ define i16 @atomicrmw_usub_sat_i16(ptr %ptr, i16 %val) {
; RV64IA-NEXT: lw a4, 0(a2)
; RV64IA-NEXT: sllw a5, a3, a5
; RV64IA-NEXT: not a5, a5
-; RV64IA-NEXT: and a6, a1, a3
+; RV64IA-NEXT: and a1, a1, a3
; RV64IA-NEXT: .LBB5_1: # %atomicrmw.start
; RV64IA-NEXT: # =>This Loop Header: Depth=1
; RV64IA-NEXT: # Child Loop BB5_3 Depth 2
-; RV64IA-NEXT: srlw a7, a4, a0
-; RV64IA-NEXT: sext.w t0, a4
-; RV64IA-NEXT: and t1, a7, a3
-; RV64IA-NEXT: sltu t1, t1, a6
-; RV64IA-NEXT: subw a7, a7, a1
-; RV64IA-NEXT: addi t1, t1, -1
-; RV64IA-NEXT: and a7, a7, a3
-; RV64IA-NEXT: and a7, t1, a7
-; RV64IA-NEXT: sllw a7, a7, a0
+; RV64IA-NEXT: srlw a6, a4, a0
+; RV64IA-NEXT: sext.w a7, a4
+; RV64IA-NEXT: and a6, a6, a3
+; RV64IA-NEXT: sub t0, a6, a1
+; RV64IA-NEXT: sltu a6, a6, t0
+; RV64IA-NEXT: addi a6, a6, -1
+; RV64IA-NEXT: and a6, a6, t0
+; RV64IA-NEXT: sllw a6, a6, a0
; RV64IA-NEXT: and a4, a4, a5
-; RV64IA-NEXT: or a7, a4, a7
+; RV64IA-NEXT: or a6, a4, a6
; RV64IA-NEXT: .LBB5_3: # %atomicrmw.start
; RV64IA-NEXT: # Parent Loop BB5_1 Depth=1
; RV64IA-NEXT: # => This Inner Loop Header: Depth=2
; RV64IA-NEXT: lr.w.aqrl a4, (a2)
-; RV64IA-NEXT: bne a4, t0, .LBB5_1
+; RV64IA-NEXT: bne a4, a7, .LBB5_1
; RV64IA-NEXT: # %bb.4: # %atomicrmw.start
; RV64IA-NEXT: # in Loop: Header=BB5_3 Depth=2
-; RV64IA-NEXT: sc.w.rl t1, a7, (a2)
-; RV64IA-NEXT: bnez t1, .LBB5_3
+; RV64IA-NEXT: sc.w.rl t0, a6, (a2)
+; RV64IA-NEXT: bnez t0, .LBB5_3
; RV64IA-NEXT: # %bb.5: # %atomicrmw.start
; RV64IA-NEXT: # %bb.2: # %atomicrmw.end
; RV64IA-NEXT: srlw a0, a4, a0
@@ -1119,10 +980,10 @@ define i32 @atomicrmw_usub_sat_i32(ptr %ptr, i32 %val) {
; RV32I-NEXT: mv s1, a1
; RV32I-NEXT: .LBB6_1: # %atomicrmw.start
; RV32I-NEXT: # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT: sltu a0, a3, s1
-; RV32I-NEXT: sub a1, a3, s1
-; RV32I-NEXT: addi a0, a0, -1
-; RV32I-NEXT: and a2, a0, a1
+; RV32I-NEXT: sub a0, a3, s1
+; RV32I-NEXT: sltu a1, a3, a0
+; RV32I-NEXT: addi a1, a1, -1
+; RV32I-NEXT: and a2, a1, a0
; RV32I-NEXT: sw a3, 0(sp)
; RV32I-NEXT: mv a1, sp
; RV32I-NEXT: li a3, 5
@@ -1146,10 +1007,10 @@ define i32 @atomicrmw_usub_sat_i32(ptr %ptr, i32 %val) {
; RV32IA-NEXT: # =>This Loop Header: Depth=1
; RV32IA-NEXT: # Child Loop BB6_3 Depth 2
; RV32IA-NEXT: mv a3, a2
-; RV32IA-NEXT: sltu a2, a2, a1
-; RV32IA-NEXT: sub a4, a3, a1
-; RV32IA-NEXT: addi a2, a2, -1
-; RV32IA-NEXT: and a4, a2, a4
+; RV32IA-NEXT: sub a2, a2, a1
+; RV32IA-NEXT: sltu a4, a3, a2
+; RV32IA-NEXT: addi a4, a4, -1
+; RV32IA-NEXT: and a4, a4, a2
; RV32IA-NEXT: .LBB6_3: # %atomicrmw.start
; RV32IA-NEXT: # Parent Loop BB6_1 Depth=1
; RV32IA-NEXT: # => This Inner Loop Header: Depth=2
@@ -1166,55 +1027,50 @@ define i32 @atomicrmw_usub_sat_i32(ptr %ptr, i32 %val) {
;
; RV64I-LABEL: atomicrmw_usub_sat_i32:
; RV64I: # %bb.0:
-; RV64I-NEXT: addi sp, sp, -48
-; RV64I-NEXT: .cfi_def_cfa_offset 48
-; RV64I-NEXT: sd ra, 40(sp) # 8-byte Folded Spill
-; RV64I-NEXT: sd s0, 32(sp) # 8-byte Folded Spill
-; RV64I-NEXT: sd s1, 24(sp) # 8-byte Folded Spill
-; RV64I-NEXT: sd s2, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT: addi sp, sp, -32
+; RV64I-NEXT: .cfi_def_cfa_offset 32
+; RV64I-NEXT: sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s1, 8(sp) # 8-byte Folded Spill
; RV64I-NEXT: .cfi_offset ra, -8
; RV64I-NEXT: .cfi_offset s0, -16
; RV64I-NEXT: .cfi_offset s1, -24
-; RV64I-NEXT: .cfi_offset s2, -32
; RV64I-NEXT: mv s0, a0
; RV64I-NEXT: lw a3, 0(a0)
; RV64I-NEXT: mv s1, a1
-; RV64I-NEXT: sext.w s2, a1
; RV64I-NEXT: .LBB6_1: # %atomicrmw.start
; RV64I-NEXT: # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT: sltu a0, a3, s2
-; RV64I-NEXT: subw a1, a3, s1
-; RV64I-NEXT: addi a0, a0, -1
-; RV64I-NEXT: and a2, a0, a1
-; RV64I-NEXT: sw a3, 12(sp)
-; RV64I-NEXT: addi a1, sp, 12
+; RV64I-NEXT: subw a0, a3, s1
+; RV64I-NEXT: sltu a1, a3, a0
+; RV64I-NEXT: addi a1, a1, -1
+; RV64I-NEXT: and a2, a1, a0
+; RV64I-NEXT: sw a3, 4(sp)
+; RV64I-NEXT: addi a1, sp, 4
; RV64I-NEXT: li a3, 5
; RV64I-NEXT: li a4, 5
; RV64I-NEXT: mv a0, s0
; RV64I-NEXT: call __atomic_compare_exchange_4
-; RV64I-NEXT: lw a3, 12(sp)
+; RV64I-NEXT: lw a3, 4(sp)
; RV64I-NEXT: beqz a0, .LBB6_1
; RV64I-NEXT: # %bb.2: # %atomicrmw.end
; RV64I-NEXT: mv a0, a3
-; RV64I-NEXT: ld ra, 40(sp) # 8-byte Folded Reload
-; RV64I-NEXT: ld s0, 32(sp) # 8-byte Folded Reload
-; RV64I-NEXT: ld s1, 24(sp) # 8-byte Folded Reload
-; RV64I-NEXT: ld s2, 16(sp) # 8-byte Folded Reload
-; RV64I-NEXT: addi sp, sp, 48
+; RV64I-NEXT: ld ra, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 32
; RV64I-NEXT: ret
;
; RV64IA-LABEL: atomicrmw_usub_sat_i32:
; RV64IA: # %bb.0:
; RV64IA-NEXT: lw a2, 0(a0)
-; RV64IA-NEXT: sext.w a3, a1
; RV64IA-NEXT: .LBB6_1: # %atomicrmw.start
; RV64IA-NEXT: # =>This Loop Header: Depth=1
; RV64IA-NEXT: # Child Loop BB6_3 Depth 2
+; RV64IA-NEXT: subw a3, a2, a1
; RV64IA-NEXT: sext.w a4, a2
-; RV64IA-NEXT: sltu a5, a4, a3
-; RV64IA-NEXT: subw a2, a2, a1
-; RV64IA-NEXT: addi a5, a5, -1
-; RV64IA-NEXT: and a5, a5, a2
+; RV64IA-NEXT: sltu a2, a4, a3
+; RV64IA-NEXT: addi a2, a2, -1
+; RV64IA-NEXT: and a3, a2, a3
; RV64IA-NEXT: .LBB6_3: # %atomicrmw.start
; RV64IA-NEXT: # Parent Loop BB6_1 Depth=1
; RV64IA-NEXT: # => This Inner Loop Header: Depth=2
@@ -1222,8 +1078,8 @@ define i32 @atomicrmw_usub_sat_i32(ptr %ptr, i32 %val) {
; RV64IA-NEXT: bne a2, a4, .LBB6_1
; RV64IA-NEXT: # %bb.4: # %atomicrmw.start
; RV64IA-NEXT: # in Loop: Header=BB6_3 Depth=2
-; RV64IA-NEXT: sc.w.rl a6, a5, (a0)
-; RV64IA-NEXT: bnez a6, .LBB6_3
+; RV64IA-NEXT: sc.w.rl a5, a3, (a0)
+; RV64IA-NEXT: bnez a5, .LBB6_3
; RV64IA-NEXT: # %bb.5: # %atomicrmw.start
; RV64IA-NEXT: # %bb.2: # %atomicrmw.end
; RV64IA-NEXT: mv a0, a2
@@ -1250,16 +1106,15 @@ define i64 @atomicrmw_usub_sat_i64(ptr %ptr, i64 %val) {
; RV32I-NEXT: lw a4, 0(a0)
; RV32I-NEXT: mv s1, a2
; RV32I-NEXT: mv s2, a1
-; RV32I-NEXT: j .LBB7_2
+; RV32I-NEXT: j .LBB7_3
; RV32I-NEXT: .LBB7_1: # %atomicrmw.start
-; RV32I-NEXT: # in Loop: Header=BB7_2 Depth=1
-; RV32I-NEXT: xori a1, a1, 1
-; RV32I-NEXT: sub a2, a5, s1
-; RV32I-NEXT: sub a2, a2, a0
-; RV32I-NEXT: sub a0, a4, s2
-; RV32I-NEXT: neg a1, a1
-; RV32I-NEXT: and a3, a1, a2
-; RV32I-NEXT: and a2, a1, a0
+; RV32I-NEXT: # in Loop: Header=BB7_3 Depth=1
+; RV32I-NEXT: sltu a2, a5, a0
+; RV32I-NEXT: .LBB7_2: # %atomicrmw.start
+; RV32I-NEXT: # in Loop: Header=BB7_3 Depth=1
+; RV32I-NEXT: addi a3, a2, -1
+; RV32I-NEXT: and a2, a3, a1
+; RV32I-NEXT: and a3, a3, a0
; RV32I-NEXT: sw a4, 8(sp)
; RV32I-NEXT: sw a5, 12(sp)
; RV32I-NEXT: addi a1, sp, 8
@@ -1269,17 +1124,18 @@ define i64 @atomicrmw_usub_sat_i64(ptr %ptr, i64 %val) {
; RV32I-NEXT: call __atomic_compare_exchange_8
; RV32I-NEXT: lw a5, 12(sp)
; RV32I-NEXT: lw a4, 8(sp)
-; RV32I-NEXT: bnez a0, .LBB7_4
-; RV32I-NEXT: .LBB7_2: # %atomicrmw.start
+; RV32I-NEXT: bnez a0, .LBB7_5
+; RV32I-NEXT: .LBB7_3: # %atomicrmw.start
; RV32I-NEXT: # =>This Inner Loop Header: Depth=1
; RV32I-NEXT: sltu a0, a4, s2
-; RV32I-NEXT: mv a1, a0
-; RV32I-NEXT: beq a5, s1, .LBB7_1
-; RV32I-NEXT: # %bb.3: # %atomicrmw.start
-; RV32I-NEXT: # in Loop: Header=BB7_2 Depth=1
-; RV32I-NEXT: sltu a1, a5, s1
-; RV32I-NEXT: j .LBB7_1
-; RV32I-NEXT: .LBB7_4: # %atomicrmw.end
+; RV32I-NEXT: sub a1, a5, s1
+; RV32I-NEXT: sub a0, a1, a0
+; RV32I-NEXT: sub a1, a4, s2
+; RV32I-NEXT: bne a0, a5, .LBB7_1
+; RV32I-NEXT: # %bb.4: # in Loop: Header=BB7_3 Depth=1
+; RV32I-NEXT: sltu a2, a4, a1
+; RV32I-NEXT: j .LBB7_2
+; RV32I-NEXT: .LBB7_5: # %atomicrmw.end
; RV32I-NEXT: mv a0, a4
; RV32I-NEXT: mv a1, a5
; RV32I-NEXT: lw ra, 28(sp) # 4-byte Folded Reload
@@ -1306,16 +1162,15 @@ define i64 @atomicrmw_usub_sat_i64(ptr %ptr, i64 %val) {
; RV32IA-NEXT: lw a4, 0(a0)
; RV32IA-NEXT: mv s1, a2
; RV32IA-NEXT: mv s2, a1
-; RV32IA-NEXT: j .LBB7_2
+; RV32IA-NEXT: j .LBB7_3
; RV32IA-NEXT: .LBB7_1: # %atomicrmw.start
-; RV32IA-NEXT: # in Loop: Header=BB7_2 Depth=1
-; RV32IA-NEXT: xori a1, a1, 1
-; RV32IA-NEXT: sub a2, a5, s1
-; RV32IA-NEXT: sub a2, a2, a0
-; RV32IA-NEXT: sub a0, a4, s2
-; RV32IA-NEXT: neg a1, a1
-; RV32IA-NEXT: and a3, a1, a2
-; RV32IA-NEXT: and a2, a1, a0
+; RV32IA-NEXT: # in Loop: Header=BB7_3 Depth=1
+; RV32IA-NEXT: sltu a2, a5, a0
+; RV32IA-NEXT: .LBB7_2: # %atomicrmw.start
+; RV32IA-NEXT: # in Loop: Header=BB7_3 Depth=1
+; RV32IA-NEXT: addi a3, a2, -1
+; RV32IA-NEXT: and a2, a3, a1
+; RV32IA-NEXT: and a3, a3, a0
; RV32IA-NEXT: sw a4, 8(sp)
; RV32IA-NEXT: sw a5, 12(sp)
; RV32IA-NEXT: addi a1, sp, 8
@@ -1325,17 +1180,18 @@ define i64 @atomicrmw_usub_sat_i64(ptr %ptr, i64 %val) {
; RV32IA-NEXT: call __atomic_compare_exchange_8
; RV32IA-NEXT: lw a5, 12(sp)
; RV32IA-NEXT: lw a4, 8(sp)
-; RV32IA-NEXT: bnez a0, .LBB7_4
-; RV32IA-NEXT: .LBB7_2: # %atomicrmw.start
+; RV32IA-NEXT: bnez a0, .LBB7_5
+; RV32IA-NEXT: .LBB7_3: # %atomicrmw.start
; RV32IA-NEXT: # =>This Inner Loop Header: Depth=1
; RV32IA-NEXT: sltu a0, a4, s2
-; RV32IA-NEXT: mv a1, a0
-; RV32IA-NEXT: beq a5, s1, .LBB7_1
-; RV32IA-NEXT: # %bb.3: # %atomicrmw.start
-; RV32IA-NEXT: # in Loop: Header=BB7_2 Depth=1
-; RV32IA-NEXT: sltu a1, a5, s1
-; RV32IA-NEXT: j .LBB7_1
-; RV32IA-NEXT: .LBB7_4: # %atomicrmw.end
+; RV32IA-NEXT: sub a1, a5, s1
+; RV32IA-NEXT: sub a0, a1, a0
+; RV32IA-NEXT: sub a1, a4, s2
+; RV32IA-NEXT: bne a0, a5, .LBB7_1
+; RV32IA-NEXT: # %bb.4: # in Loop: Header=BB7_3 Depth=1
+; RV32IA-NEXT: sltu a2, a4, a1
+; RV32IA-NEXT: j .LBB7_2
+; RV32IA-NEXT: .LBB7_5: # %atomicrmw.end
; RV32IA-NEXT: mv a0, a4
; RV32IA-NEXT: mv a1, a5
; RV32IA-NEXT: lw ra, 28(sp) # 4-byte Folded Reload
@@ -1360,10 +1216,10 @@ define i64 @atomicrmw_usub_sat_i64(ptr %ptr, i64 %val) {
; RV64I-NEXT: mv s1, a1
; RV64I-NEXT: .LBB7_1: # %atomicrmw.start
; RV64I-NEXT: # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT: sltu a0, a3, s1
-; RV64I-NEXT: sub a1, a3, s1
-; RV64I-NEXT: addi a0, a0, -1
-; RV64I-NEXT: and a2, a0, a1
+; RV64I-NEXT: sub a0, a3, s1
+; RV64I-NEXT: sltu a1, a3, a0
+; RV64I-NEXT: addi a1, a1, -1
+; RV64I-NEXT: and a2, a1, a0
; RV64I-NEXT: sd a3, 0(sp)
; RV64I-NEXT: mv a1, sp
; RV64I-NEXT: li a3, 5
@@ -1387,10 +1243,10 @@ define i64 @atomicrmw_usub_sat_i64(ptr %ptr, i64 %val) {
; RV64IA-NEXT: # =>This Loop Header: Depth=1
; RV64IA-NEXT: # Child Loop BB7_3 Depth 2
; RV64IA-NEXT: mv a3, a2
-; RV64IA-NEXT: sltu a2, a2, a1
-; RV64IA-NEXT: sub a4, a3, a1
-; RV64IA-NEXT: addi a2, a2, -1
-; RV64IA-NEXT: and a4, a2, a4
+; RV64IA-NEXT: sub a2, a2, a1
+; RV64IA-NEXT: sltu a4, a3, a2
+; RV64IA-NEXT: addi a4, a4, -1
+; RV64IA-NEXT: and a4, a4, a2
; RV64IA-NEXT: .LBB7_3: # %atomicrmw.start
; RV64IA-NEXT: # Parent Loop BB7_1 Depth=1
; RV64IA-NEXT: # => This Inner Loop Header: Depth=2
diff --git a/llvm/test/CodeGen/VE/Scalar/atomicrmw-cond-sub-clamp.ll b/llvm/test/CodeGen/VE/Scalar/atomicrmw-cond-sub-clamp.ll
index 7a047fcfe3cf0c..860c4004658dba 100644
--- a/llvm/test/CodeGen/VE/Scalar/atomicrmw-cond-sub-clamp.ll
+++ b/llvm/test/CodeGen/VE/Scalar/atomicrmw-cond-sub-clamp.ll
@@ -20,11 +20,10 @@ define i8 @atomicrmw_usub_cond_i8(ptr %ptr, i8 %val) {
; CHECK-NEXT: and %s5, %s6, (32)0
; CHECK-NEXT: srl %s5, %s5, %s0
; CHECK-NEXT: and %s7, %s5, (56)0
-; CHECK-NEXT: subs.w.sx %s5, %s5, %s1
+; CHECK-NEXT: subs.w.sx %s34, %s5, %s1
; CHECK-NEXT: cmpu.w %s7, %s7, %s4
-; CHECK-NEXT: or %s34, 0, %s1
-; CHECK-NEXT: cmov.w.ge %s34, %s5, %s7
-; CHECK-NEXT: and %s5, %s34, (56)0
+; CHECK-NEXT: cmov.w.ge %s5, %s34, %s7
+; CHECK-NEXT: and %s5, %s5, (56)0
; CHECK-NEXT: sla.w.sx %s5, %s5, %s0
; CHECK-NEXT: and %s7, %s6, %s3
; CHECK-NEXT: or %s5, %s7, %s5
@@ -58,11 +57,10 @@ define i16 @atomicrmw_usub_cond_i16(ptr %ptr, i16 %val) {
; CHECK-NEXT: and %s5, %s6, (32)0
; CHECK-NEXT: srl %s5, %s5, %s0
; CHECK-NEXT: and %s7, %s5, (48)0
-; CHECK-NEXT: subs.w.sx %s5, %s5, %s1
+; CHECK-NEXT: subs.w.sx %s34, %s5, %s1
; CHECK-NEXT: cmpu.w %s7, %s7, %s4
-; CHECK-NEXT: or %s34, 0, %s1
-; CHECK-NEXT: cmov.w.ge %s34, %s5, %s7
-; CHECK-NEXT: and %s5, %s34, (48)0
+; CHECK-NEXT: cmov.w.ge %s5, %s34, %s7
+; CHECK-NEXT: and %s5, %s5, (48)0
; CHECK-NEXT: sla.w.sx %s5, %s5, %s0
; CHECK-NEXT: and %s7, %s6, %s3
; CHECK-NEXT: or %s5, %s7, %s5
@@ -88,7 +86,7 @@ define i32 @atomicrmw_usub_cond_i32(ptr %ptr, i32 %val) {
; CHECK-NEXT: or %s3, 0, %s2
; CHECK-NEXT: subs.w.sx %s4, %s2, %s1
; CHECK-NEXT: cmpu.w %s5, %s2, %s1
-; CHECK-NEXT: or %s2, 0, %s1
+; CHECK-NEXT: or %s2, 0, %s3
; CHECK-NEXT: cmov.w.ge %s2, %s4, %s5
; CHECK-NEXT: cas.w %s2, (%s0), %s3
; CHECK-NEXT: brne.w %s2, %s3, .LBB2_1
@@ -103,20 +101,19 @@ define i32 @atomicrmw_usub_cond_i32(ptr %ptr, i32 %val) {
define i64 @atomicrmw_usub_cond_sub_i64(ptr %ptr, i64 %val) {
; CHECK-LABEL: atomicrmw_usub_cond_sub_i64:
; CHECK: # %bb.0:
+; CHECK-NEXT: or %s2, 0, %s0
; CHECK-NEXT: fencem 3
-; CHECK-NEXT: ld %s2, (, %s0)
+; CHECK-NEXT: ld %s0, (, %s0)
; CHECK-NEXT: .LBB3_1: # %atomicrmw.start
; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: or %s3, 0, %s2
-; CHECK-NEXT: subs.l %s4, %s2, %s1
-; CHECK-NEXT: cmpu.l %s5, %s2, %s1
-; CHECK-NEXT: or %s2, 0, %s1
-; CHECK-NEXT: cmov.l.ge %s2, %s4, %s5
-; CHECK-NEXT: cas.l %s2, (%s0), %s3
-; CHECK-NEXT: brne.l %s2, %s3, .LBB3_1
+; CHECK-NEXT: or %s3, 0, %s0
+; CHECK-NEXT: subs.l %s4, %s0, %s1
+; CHECK-NEXT: cmpu.l %s5, %s0, %s1
+; CHECK-NEXT: cmov.l.ge %s0, %s4, %s5
+; CHECK-NEXT: cas.l %s0, (%s2), %s3
+; CHECK-NEXT: brne.l %s0, %s3, .LBB3_1
; CHECK-NEXT: # %bb.2: # %atomicrmw.end
; CHECK-NEXT: fencem 3
-; CHECK-NEXT: or %s0, 0, %s2
; CHECK-NEXT: b.l.t (, %s10)
%result = atomicrmw usub_cond ptr %ptr, i64 %val seq_cst
ret i64 %result
@@ -125,33 +122,32 @@ define i64 @atomicrmw_usub_cond_sub_i64(ptr %ptr, i64 %val) {
define i8 @atomicrmw_usub_sat_i8(ptr %ptr, i8 %val) {
; CHECK-LABEL: atomicrmw_usub_sat_i8:
; CHECK: # %bb.0:
-; CHECK-NEXT: and %s1, %s1, (32)0
+; CHECK-NEXT: and %s3, %s1, (32)0
; CHECK-NEXT: fencem 3
-; CHECK-NEXT: and %s2, -4, %s0
+; CHECK-NEXT: and %s1, -4, %s0
; CHECK-NEXT: and %s0, 3, %s0
; CHECK-NEXT: sla.w.sx %s0, %s0, 3
-; CHECK-NEXT: sla.w.sx %s3, (56)0, %s0
-; CHECK-NEXT: ldl.sx %s5, (, %s2)
-; CHECK-NEXT: xor %s3, -1, %s3
-; CHECK-NEXT: and %s3, %s3, (32)0
-; CHECK-NEXT: and %s4, %s1, (56)0
+; CHECK-NEXT: sla.w.sx %s2, (56)0, %s0
+; CHECK-NEXT: ldl.sx %s4, (, %s1)
+; CHECK-NEXT: xor %s2, -1, %s2
+; CHECK-NEXT: and %s2, %s2, (32)0
+; CHECK-NEXT: and %s3, %s3, (56)0
; CHECK-NEXT: .LBB4_1: # %atomicrmw.start
; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: or %s6, 0, %s5
-; CHECK-NEXT: and %s5, %s6, (32)0
-; CHECK-NEXT: srl %s5, %s5, %s0
-; CHECK-NEXT: and %s7, %s5, (56)0
-; CHECK-NEXT: subs.w.sx %s5, %s5, %s1
-; CHECK-NEXT: cmpu.w %s7, %s7, %s4
-; CHECK-NEXT: cmov.w.lt %s5, (0)1, %s7
-; CHECK-NEXT: and %s5, %s5, (56)0
-; CHECK-NEXT: sla.w.sx %s5, %s5, %s0
-; CHECK-NEXT: and %s7, %s6, %s3
-; CHECK-NEXT: or %s5, %s7, %s5
-; CHECK-NEXT: cas.w %s5, (%s2), %s6
-; CHECK-NEXT: brne.w %s5, %s6, .LBB4_1
+; CHECK-NEXT: or %s5, 0, %s4
+; CHECK-NEXT: and %s4, %s5, (32)0
+; CHECK-NEXT: srl %s4, %s4, %s0
+; CHECK-NEXT: and %s4, %s4, (56)0
+; CHECK-NEXT: subs.w.sx %s6, %s4, %s3
+; CHECK-NEXT: cmpu.w %s4, %s6, %s4
+; CHECK-NEXT: cmov.w.gt %s6, (0)1, %s4
+; CHECK-NEXT: sla.w.sx %s4, %s6, %s0
+; CHECK-NEXT: and %s6, %s5, %s2
+; CHECK-NEXT: or %s4, %s6, %s4
+; CHECK-NEXT: cas.w %s4, (%s1), %s5
+; CHECK-NEXT: brne.w %s4, %s5, .LBB4_1
; CHECK-NEXT: # %bb.2: # %atomicrmw.end
-; CHECK-NEXT: and %s1, %s5, (32)0
+; CHECK-NEXT: and %s1, %s4, (32)0
; CHECK-NEXT: srl %s0, %s1, %s0
; CHECK-NEXT: fencem 3
; CHECK-NEXT: b.l.t (, %s10)
@@ -162,33 +158,32 @@ define i8 @atomicrmw_usub_sat_i8(ptr %ptr, i8 %val) {
define i16 @atomicrmw_usub_sat_i16(ptr %ptr, i16 %val) {
; CHECK-LABEL: atomicrmw_usub_sat_i16:
; CHECK: # %bb.0:
-; CHECK-NEXT: and %s1, %s1, (32)0
+; CHECK-NEXT: and %s3, %s1, (32)0
; CHECK-NEXT: fencem 3
-; CHECK-NEXT: and %s2, -4, %s0
+; CHECK-NEXT: and %s1, -4, %s0
; CHECK-NEXT: and %s0, 3, %s0
; CHECK-NEXT: sla.w.sx %s0, %s0, 3
-; CHECK-NEXT: sla.w.sx %s3, (48)0, %s0
-; CHECK-NEXT: ldl.sx %s5, (, %s2)
-; CHECK-NEXT: xor %s3, -1, %s3
-; CHECK-NEXT: and %s3, %s3, (32)0
-; CHECK-NEXT: and %s4, %s1, (48)0
+; CHECK-NEXT: sla.w.sx %s2, (48)0, %s0
+; CHECK-NEXT: ldl.sx %s4, (, %s1)
+; CHECK-NEXT: xor %s2, -1, %s2
+; CHECK-NEXT: and %s2, %s2, (32)0
+; CHECK-NEXT: and %s3, %s3, (48)0
; CHECK-NEXT: .LBB5_1: # %atomicrmw.start
; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: or %s6, 0, %s5
-; CHECK-NEXT: and %s5, %s6, (32)0
-; CHECK-NEXT: srl %s5, %s5, %s0
-; CHECK-NEXT: and %s7, %s5, (48)0
-; CHECK-NEXT: subs.w.sx %s5, %s5, %s1
-; CHECK-NEXT: cmpu.w %s7, %s7, %s4
-; CHECK-NEXT: cmov.w.lt %s5, (0)1, %s7
-; CHECK-NEXT: and %s5, %s5, (48)0
-; CHECK-NEXT: sla.w.sx %s5, %s5, %s0
-; CHECK-NEXT: and %s7, %s6, %s3
-; CHECK-NEXT: or %s5, %s7, %s5
-; CHECK-NEXT: cas.w %s5, (%s2), %s6
-; CHECK-NEXT: brne.w %s5, %s6, .LBB5_1
+; CHECK-NEXT: or %s5, 0, %s4
+; CHECK-NEXT: and %s4, %s5, (32)0
+; CHECK-NEXT: srl %s4, %s4, %s0
+; CHECK-NEXT: and %s4, %s4, (48)0
+; CHECK-NEXT: subs.w.sx %s6, %s4, %s3
+; CHECK-NEXT: cmpu.w %s4, %s6, %s4
+; CHECK-NEXT: cmov.w.gt %s6, (0)1, %s4
+; CHECK-NEXT: sla.w.sx %s4, %s6, %s0
+; CHECK-NEXT: and %s6, %s5, %s2
+; CHECK-NEXT: or %s4, %s6, %s4
+; CHECK-NEXT: cas.w %s4, (%s1), %s5
+; CHECK-NEXT: brne.w %s4, %s5, .LBB5_1
; CHECK-NEXT: # %bb.2: # %atomicrmw.end
-; CHECK-NEXT: and %s1, %s5, (32)0
+; CHECK-NEXT: and %s1, %s4, (32)0
; CHECK-NEXT: srl %s0, %s1, %s0
; CHECK-NEXT: fencem 3
; CHECK-NEXT: b.l.t (, %s10)
@@ -206,8 +201,8 @@ define i32 @atomicrmw_usub_sat_i32(ptr %ptr, i32 %val) {
; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
; CHECK-NEXT: or %s3, 0, %s2
; CHECK-NEXT: subs.w.sx %s2, %s2, %s1
-; CHECK-NEXT: cmpu.w %s4, %s3, %s1
-; CHECK-NEXT: cmov.w.lt %s2, (0)1, %s4
+; CHECK-NEXT: cmpu.w %s4, %s2, %s3
+; CHECK-NEXT: cmov.w.gt %s2, (0)1, %s4
; CHECK-NEXT: cas.w %s2, (%s0), %s3
; CHECK-NEXT: brne.w %s2, %s3, .LBB6_1
; CHECK-NEXT: # %bb.2: # %atomicrmw.end
@@ -227,8 +222,8 @@ define i64 @atomicrmw_usub_sat_i64(ptr %ptr, i64 %val) {
; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
; CHECK-NEXT: or %s3, 0, %s2
; CHECK-NEXT: subs.l %s2, %s2, %s1
-; CHECK-NEXT: cmpu.l %s4, %s3, %s1
-; CHECK-NEXT: cmov.l.lt %s2, (0)1, %s4
+; CHECK-NEXT: cmpu.l %s4, %s2, %s3
+; CHECK-NEXT: cmov.l.gt %s2, (0)1, %s4
; CHECK-NEXT: cas.l %s2, (%s0), %s3
; CHECK-NEXT: brne.l %s2, %s3, .LBB7_1
; CHECK-NEXT: # %bb.2: # %atomicrmw.end
diff --git a/llvm/test/CodeGen/X86/atomicrmw-cond-sub-clamp.ll b/llvm/test/CodeGen/X86/atomicrmw-cond-sub-clamp.ll
index ada8c9fff0d112..04bfb4e367b9d7 100644
--- a/llvm/test/CodeGen/X86/atomicrmw-cond-sub-clamp.ll
+++ b/llvm/test/CodeGen/X86/atomicrmw-cond-sub-clamp.ll
@@ -21,7 +21,7 @@ define i8 @atomicrmw_usub_cond_i8(ptr %ptr, i8 %val) {
; CHECK-32-NEXT: jae .LBB0_3
; CHECK-32-NEXT: # %bb.2: # %atomicrmw.start
; CHECK-32-NEXT: # in Loop: Header=BB0_1 Depth=1
-; CHECK-32-NEXT: movb %cl, %ah
+; CHECK-32-NEXT: movb %al, %ah
; CHECK-32-NEXT: jmp .LBB0_3
; CHECK-32-NEXT: .LBB0_4: # %atomicrmw.end
; CHECK-32-NEXT: retl
@@ -29,14 +29,14 @@ define i8 @atomicrmw_usub_cond_i8(ptr %ptr, i8 %val) {
; CHECK-64-LABEL: atomicrmw_usub_cond_i8:
; CHECK-64: # %bb.0:
; CHECK-64-NEXT: movzbl (%rdi), %eax
-; CHECK-64-NEXT: movzbl %sil, %ecx
; CHECK-64-NEXT: .p2align 4, 0x90
; CHECK-64-NEXT: .LBB0_1: # %atomicrmw.start
; CHECK-64-NEXT: # =>This Inner Loop Header: Depth=1
-; CHECK-64-NEXT: movl %eax, %edx
-; CHECK-64-NEXT: subb %cl, %dl
-; CHECK-64-NEXT: movzbl %dl, %edx
+; CHECK-64-NEXT: movzbl %al, %ecx
+; CHECK-64-NEXT: subb %sil, %al
+; CHECK-64-NEXT: movzbl %al, %edx
; CHECK-64-NEXT: cmovbl %ecx, %edx
+; CHECK-64-NEXT: movl %ecx, %eax
; CHECK-64-NEXT: lock cmpxchgb %dl, (%rdi)
; CHECK-64-NEXT: jne .LBB0_1
; CHECK-64-NEXT: # %bb.2: # %atomicrmw.end
@@ -58,7 +58,9 @@ define i16 @atomicrmw_usub_cond_i16(ptr %ptr, i16 %val) {
; CHECK-32-NEXT: .p2align 4, 0x90
; CHECK-32-NEXT: .LBB1_3: # %atomicrmw.start
; CHECK-32-NEXT: # in Loop: Header=BB1_1 Depth=1
+; CHECK-32-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-32-NEXT: lock cmpxchgw %si, (%edx)
+; CHECK-32-NEXT: # kill: def $ax killed $ax def $eax
; CHECK-32-NEXT: je .LBB1_4
; CHECK-32-NEXT: .LBB1_1: # %atomicrmw.start
; CHECK-32-NEXT: # =>This Inner Loop Header: Depth=1
@@ -67,9 +69,10 @@ define i16 @atomicrmw_usub_cond_i16(ptr %ptr, i16 %val) {
; CHECK-32-NEXT: jae .LBB1_3
; CHECK-32-NEXT: # %bb.2: # %atomicrmw.start
; CHECK-32-NEXT: # in Loop: Header=BB1_1 Depth=1
-; CHECK-32-NEXT: movl %ecx, %esi
+; CHECK-32-NEXT: movl %eax, %esi
; CHECK-32-NEXT: jmp .LBB1_3
; CHECK-32-NEXT: .LBB1_4: # %atomicrmw.end
+; CHECK-32-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-32-NEXT: popl %esi
; CHECK-32-NEXT: .cfi_def_cfa_offset 4
; CHECK-32-NEXT: retl
@@ -82,10 +85,13 @@ define i16 @atomicrmw_usub_cond_i16(ptr %ptr, i16 %val) {
; CHECK-64-NEXT: # =>This Inner Loop Header: Depth=1
; CHECK-64-NEXT: movl %eax, %ecx
; CHECK-64-NEXT: subw %si, %cx
-; CHECK-64-NEXT: cmovbl %esi, %ecx
+; CHECK-64-NEXT: cmovbl %eax, %ecx
+; CHECK-64-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-64-NEXT: lock cmpxchgw %cx, (%rdi)
+; CHECK-64-NEXT: # kill: def $ax killed $ax def $eax
; CHECK-64-NEXT: jne .LBB1_1
; CHECK-64-NEXT: # %bb.2: # %atomicrmw.end
+; CHECK-64-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-64-NEXT: retq
%result = atomicrmw usub_cond ptr %ptr, i16 %val seq_cst
ret i16 %result
@@ -113,7 +119,7 @@ define i32 @atomicrmw_usub_cond_i32(ptr %ptr, i32 %val) {
; CHECK-32-NEXT: jae .LBB2_3
; CHECK-32-NEXT: # %bb.2: # %atomicrmw.start
; CHECK-32-NEXT: # in Loop: Header=BB2_1 Depth=1
-; CHECK-32-NEXT: movl %ecx, %esi
+; CHECK-32-NEXT: movl %eax, %esi
; CHECK-32-NEXT: jmp .LBB2_3
; CHECK-32-NEXT: .LBB2_4: # %atomicrmw.end
; CHECK-32-NEXT: popl %esi
@@ -128,7 +134,7 @@ define i32 @atomicrmw_usub_cond_i32(ptr %ptr, i32 %val) {
; CHECK-64-NEXT: # =>This Inner Loop Header: Depth=1
; CHECK-64-NEXT: movl %eax, %ecx
; CHECK-64-NEXT: subl %esi, %ecx
-; CHECK-64-NEXT: cmovbl %esi, %ecx
+; CHECK-64-NEXT: cmovbl %eax, %ecx
; CHECK-64-NEXT: lock cmpxchgl %ecx, (%rdi)
; CHECK-64-NEXT: jne .LBB2_1
; CHECK-64-NEXT: # %bb.2: # %atomicrmw.end
@@ -172,8 +178,8 @@ define i64 @atomicrmw_usub_cond_i64(ptr %ptr, i64 %val) {
; CHECK-32-NEXT: jae .LBB3_3
; CHECK-32-NEXT: # %bb.2: # %atomicrmw.start
; CHECK-32-NEXT: # in Loop: Header=BB3_1 Depth=1
-; CHECK-32-NEXT: movl %esi, %ecx
-; CHECK-32-NEXT: movl %edi, %ebx
+; CHECK-32-NEXT: movl %edx, %ecx
+; CHECK-32-NEXT: movl %eax, %ebx
; CHECK-32-NEXT: jmp .LBB3_3
; CHECK-32-NEXT: .LBB3_4: # %atomicrmw.end
; CHECK-32-NEXT: popl %esi
@@ -194,7 +200,7 @@ define i64 @atomicrmw_usub_cond_i64(ptr %ptr, i64 %val) {
; CHECK-64-NEXT: # =>This Inner Loop Header: Depth=1
; CHECK-64-NEXT: movq %rax, %rcx
; CHECK-64-NEXT: subq %rsi, %rcx
-; CHECK-64-NEXT: cmovbq %rsi, %rcx
+; CHECK-64-NEXT: cmovbq %rax, %rcx
; CHECK-64-NEXT: lock cmpxchgq %rcx, (%rdi)
; CHECK-64-NEXT: jne .LBB3_1
; CHECK-64-NEXT: # %bb.2: # %atomicrmw.end
@@ -223,8 +229,7 @@ define i8 @atomicrmw_usub_sat_i8(ptr %ptr, i8 %val) {
; CHECK-32-NEXT: movl %eax, %ebx
; CHECK-32-NEXT: subb %cl, %bl
; CHECK-32-NEXT: jae .LBB4_3
-; CHECK-32-NEXT: # %bb.2: # %atomicrmw.start
-; CHECK-32-NEXT: # in Loop: Header=BB4_1 Depth=1
+; CHECK-32-NEXT: # %bb.2: # in Loop: Header=BB4_1 Depth=1
; CHECK-32-NEXT: xorl %ebx, %ebx
; CHECK-32-NEXT: jmp .LBB4_3
; CHECK-32-NEXT: .LBB4_4: # %atomicrmw.end
@@ -254,9 +259,12 @@ define i8 @atomicrmw_usub_sat_i8(ptr %ptr, i8 %val) {
define i16 @atomicrmw_usub_sat_i16(ptr %ptr, i16 %val) {
; CHECK-32-LABEL: atomicrmw_usub_sat_i16:
; CHECK-32: # %bb.0:
-; CHECK-32-NEXT: pushl %esi
+; CHECK-32-NEXT: pushl %edi
; CHECK-32-NEXT: .cfi_def_cfa_offset 8
-; CHECK-32-NEXT: .cfi_offset %esi, -8
+; CHECK-32-NEXT: pushl %esi
+; CHECK-32-NEXT: .cfi_def_cfa_offset 12
+; CHECK-32-NEXT: .cfi_offset %esi, -12
+; CHECK-32-NEXT: .cfi_offset %edi, -8
; CHECK-32-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
; CHECK-32-NEXT: movl {{[0-9]+}}(%esp), %edx
; CHECK-32-NEXT: movzwl (%edx), %eax
@@ -268,15 +276,18 @@ define i16 @atomicrmw_usub_sat_i16(ptr %ptr, i16 %val) {
; CHECK-32-NEXT: je .LBB5_4
; CHECK-32-NEXT: .LBB5_1: # %atomicrmw.start
; CHECK-32-NEXT: # =>This Inner Loop Header: Depth=1
-; CHECK-32-NEXT: movl %eax, %esi
-; CHECK-32-NEXT: subw %cx, %si
-; CHECK-32-NEXT: jae .LBB5_3
+; CHECK-32-NEXT: xorl %esi, %esi
+; CHECK-32-NEXT: movl %eax, %edi
+; CHECK-32-NEXT: subw %cx, %di
+; CHECK-32-NEXT: jb .LBB5_3
; CHECK-32-NEXT: # %bb.2: # %atomicrmw.start
; CHECK-32-NEXT: # in Loop: Header=BB5_1 Depth=1
-; CHECK-32-NEXT: xorl %esi, %esi
+; CHECK-32-NEXT: movl %edi, %esi
; CHECK-32-NEXT: jmp .LBB5_3
; CHECK-32-NEXT: .LBB5_4: # %atomicrmw.end
; CHECK-32-NEXT: popl %esi
+; CHECK-32-NEXT: .cfi_def_cfa_offset 8
+; CHECK-32-NEXT: popl %edi
; CHECK-32-NEXT: .cfi_def_cfa_offset 4
; CHECK-32-NEXT: retl
;
@@ -301,9 +312,12 @@ define i16 @atomicrmw_usub_sat_i16(ptr %ptr, i16 %val) {
define i32 @atomicrmw_usub_sat_i32(ptr %ptr, i32 %val) {
; CHECK-32-LABEL: atomicrmw_usub_sat_i32:
; CHECK-32: # %bb.0:
-; CHECK-32-NEXT: pushl %esi
+; CHECK-32-NEXT: pushl %edi
; CHECK-32-NEXT: .cfi_def_cfa_offset 8
-; CHECK-32-NEXT: .cfi_offset %esi, -8
+; CHECK-32-NEXT: pushl %esi
+; CHECK-32-NEXT: .cfi_def_cfa_offset 12
+; CHECK-32-NEXT: .cfi_offset %esi, -12
+; CHECK-32-NEXT: .cfi_offset %edi, -8
; CHECK-32-NEXT: movl {{[0-9]+}}(%esp), %ecx
; CHECK-32-NEXT: movl {{[0-9]+}}(%esp), %edx
; CHECK-32-NEXT: movl (%edx), %eax
@@ -315,15 +329,18 @@ define i32 @atomicrmw_usub_sat_i32(ptr %ptr, i32 %val) {
; CHECK-32-NEXT: je .LBB6_4
; CHECK-32-NEXT: .LBB6_1: # %atomicrmw.start
; CHECK-32-NEXT: # =>This Inner Loop Header: Depth=1
-; CHECK-32-NEXT: movl %eax, %esi
-; CHECK-32-NEXT: subl %ecx, %esi
-; CHECK-32-NEXT: jae .LBB6_3
+; CHECK-32-NEXT: xorl %esi, %esi
+; CHECK-32-NEXT: movl %eax, %edi
+; CHECK-32-NEXT: subl %ecx, %edi
+; CHECK-32-NEXT: jb .LBB6_3
; CHECK-32-NEXT: # %bb.2: # %atomicrmw.start
; CHECK-32-NEXT: # in Loop: Header=BB6_1 Depth=1
-; CHECK-32-NEXT: xorl %esi, %esi
+; CHECK-32-NEXT: movl %edi, %esi
; CHECK-32-NEXT: jmp .LBB6_3
; CHECK-32-NEXT: .LBB6_4: # %atomicrmw.end
; CHECK-32-NEXT: popl %esi
+; CHECK-32-NEXT: .cfi_def_cfa_offset 8
+; CHECK-32-NEXT: popl %edi
; CHECK-32-NEXT: .cfi_def_cfa_offset 4
; CHECK-32-NEXT: retl
;
@@ -360,30 +377,36 @@ define i64 @atomicrmw_usub_sat_i64(ptr %ptr, i64 %val) {
; CHECK-32-NEXT: .cfi_offset %edi, -16
; CHECK-32-NEXT: .cfi_offset %ebx, -12
; CHECK-32-NEXT: .cfi_offset %ebp, -8
-; CHECK-32-NEXT: movl {{[0-9]+}}(%esp), %esi
-; CHECK-32-NEXT: movl {{[0-9]+}}(%esp), %edi
; CHECK-32-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; CHECK-32-NEXT: movl (%ebp), %eax
-; CHECK-32-NEXT: movl 4(%ebp), %edx
+; CHECK-32-NEXT: movl (%ebp), %esi
+; CHECK-32-NEXT: movl 4(%ebp), %edi
; CHECK-32-NEXT: jmp .LBB7_1
; CHECK-32-NEXT: .p2align 4, 0x90
; CHECK-32-NEXT: .LBB7_3: # %atomicrmw.start
; CHECK-32-NEXT: # in Loop: Header=BB7_1 Depth=1
+; CHECK-32-NEXT: movl %esi, %eax
+; CHECK-32-NEXT: movl %edi, %edx
; CHECK-32-NEXT: lock cmpxchg8b (%ebp)
+; CHECK-32-NEXT: movl %eax, %esi
+; CHECK-32-NEXT: movl %edx, %edi
; CHECK-32-NEXT: je .LBB7_4
; CHECK-32-NEXT: .LBB7_1: # %atomicrmw.start
; CHECK-32-NEXT: # =>This Inner Loop Header: Depth=1
-; CHECK-32-NEXT: movl %eax, %ebx
-; CHECK-32-NEXT: subl %edi, %ebx
-; CHECK-32-NEXT: movl %edx, %ecx
-; CHECK-32-NEXT: sbbl %esi, %ecx
-; CHECK-32-NEXT: jae .LBB7_3
+; CHECK-32-NEXT: xorl %ecx, %ecx
+; CHECK-32-NEXT: movl %esi, %eax
+; CHECK-32-NEXT: subl {{[0-9]+}}(%esp), %eax
+; CHECK-32-NEXT: movl %edi, %edx
+; CHECK-32-NEXT: sbbl {{[0-9]+}}(%esp), %edx
+; CHECK-32-NEXT: movl $0, %ebx
+; CHECK-32-NEXT: jb .LBB7_3
; CHECK-32-NEXT: # %bb.2: # %atomicrmw.start
; CHECK-32-NEXT: # in Loop: Header=BB7_1 Depth=1
-; CHECK-32-NEXT: xorl %ecx, %ecx
-; CHECK-32-NEXT: xorl %ebx, %ebx
+; CHECK-32-NEXT: movl %edx, %ecx
+; CHECK-32-NEXT: movl %eax, %ebx
; CHECK-32-NEXT: jmp .LBB7_3
; CHECK-32-NEXT: .LBB7_4: # %atomicrmw.end
+; CHECK-32-NEXT: movl %esi, %eax
+; CHECK-32-NEXT: movl %edi, %edx
; CHECK-32-NEXT: popl %esi
; CHECK-32-NEXT: .cfi_def_cfa_offset 16
; CHECK-32-NEXT: popl %edi
More information about the Mlir-commits
mailing list