[llvm] [IR] Add storermw instruction (PR #207048)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 2 03:25:00 PDT 2026
https://github.com/gonzalobg updated https://github.com/llvm/llvm-project/pull/207048
>From edccab7464386dc8450bb026df0dc076e0c31d7f Mon Sep 17 00:00:00 2001
From: Gonzalo Brito Gadeschi <gonzalob at nvidia.com>
Date: Tue, 30 Jun 2026 16:50:34 -0700
Subject: [PATCH] [IR] Add storermw instruction
See:
https://discourse.llvm.org/t/rfc-add-ir-support-for-non-fetch-atomicrmw-c-26-p3111/90514
---
llvm/docs/Atomics.md | 20 +-
llvm/docs/LangRef.rst | 157 +-
llvm/include/llvm-c/Core.h | 137 +-
llvm/include/llvm/Analysis/AliasAnalysis.h | 3 +
llvm/include/llvm/Analysis/MemoryLocation.h | 2 +
llvm/include/llvm/AsmParser/LLParser.h | 1 +
llvm/include/llvm/AsmParser/LLToken.h | 1 +
llvm/include/llvm/Bitcode/LLVMBitCodes.h | 3 +
.../llvm/CodeGen/GlobalISel/IRTranslator.h | 6 +
llvm/include/llvm/CodeGen/ISDOpcodes.h | 27 +
llvm/include/llvm/CodeGen/SelectionDAGNodes.h | 46 +-
llvm/include/llvm/CodeGen/TargetLowering.h | 11 +
llvm/include/llvm/IR/IRBuilder.h | 12 +
llvm/include/llvm/IR/InstVisitor.h | 1 +
llvm/include/llvm/IR/Instruction.def | 3 +-
llvm/include/llvm/IR/Instructions.h | 161 +
llvm/lib/Analysis/AliasAnalysis.cpp | 17 +
llvm/lib/Analysis/MemoryLocation.cpp | 11 +
llvm/lib/AsmParser/LLLexer.cpp | 1 +
llvm/lib/AsmParser/LLParser.cpp | 167 +
llvm/lib/Bitcode/Reader/BitcodeReader.cpp | 59 +
llvm/lib/Bitcode/Writer/BitcodeWriter.cpp | 31 +-
llvm/lib/CodeGen/AtomicExpandPass.cpp | 99 +
llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 25 +
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 106 +-
.../SelectionDAG/SelectionDAGBuilder.cpp | 93 +
.../SelectionDAG/SelectionDAGBuilder.h | 2 +
.../SelectionDAG/SelectionDAGDumper.cpp | 24 +
llvm/lib/CodeGen/TargetLoweringBase.cpp | 11 +
llvm/lib/IR/AsmWriter.cpp | 21 +-
llvm/lib/IR/AutoUpgrade.cpp | 10 +
llvm/lib/IR/Instruction.cpp | 21 +
llvm/lib/IR/Instructions.cpp | 42 +
llvm/lib/IR/Verifier.cpp | 54 +
.../Inputs/reference_default_vocab_print.txt | 1 +
.../Inputs/reference_wtd1_vocab_print.txt | 1 +
.../Inputs/reference_wtd2_vocab_print.txt | 1 +
.../Assembler/invalid-storermw-elementwise.ll | 33 +
llvm/test/Assembler/invalid-storermw-types.ll | 58 +
llvm/test/Assembler/storermw.ll | 193 +
llvm/test/CodeGen/Generic/storermw.ll | 3448 +++++++++++++++++
llvm/test/CodeGen/Generic/storermw_gen.py | 206 +
.../test/Verifier/storermw-invalid/acq_rel.ll | 7 +
.../test/Verifier/storermw-invalid/acquire.ll | 7 +
.../Verifier/storermw-invalid/unordered.ll | 7 +
llvm/test/Verifier/storermw-invalid/xchg.ll | 7 +
llvm/test/tools/llvm-ir2vec/entities.ll | 89 +-
llvm/test/tools/llvm-ir2vec/triplets.ll | 58 +-
llvm/unittests/Analysis/IR2VecTest.cpp | 28 +-
49 files changed, 5309 insertions(+), 220 deletions(-)
create mode 100644 llvm/test/Assembler/invalid-storermw-elementwise.ll
create mode 100644 llvm/test/Assembler/invalid-storermw-types.ll
create mode 100644 llvm/test/Assembler/storermw.ll
create mode 100644 llvm/test/CodeGen/Generic/storermw.ll
create mode 100644 llvm/test/CodeGen/Generic/storermw_gen.py
create mode 100644 llvm/test/Verifier/storermw-invalid/acq_rel.ll
create mode 100644 llvm/test/Verifier/storermw-invalid/acquire.ll
create mode 100644 llvm/test/Verifier/storermw-invalid/unordered.ll
create mode 100644 llvm/test/Verifier/storermw-invalid/xchg.ll
diff --git a/llvm/docs/Atomics.md b/llvm/docs/Atomics.md
index 2be51fbc20d4a..9f7f1a1cfae1f 100644
--- a/llvm/docs/Atomics.md
+++ b/llvm/docs/Atomics.md
@@ -118,9 +118,9 @@ ordering; see {ref}`Atomic orderings`.
non-atomic loads and stores, but provide additional guarantees in situations
where threads and signals are involved.
-`cmpxchg` and `atomicrmw` are essentially like an atomic load followed by an
-atomic store (where the store is conditional for `cmpxchg`), but no other
-memory operation can happen on any thread between the load and store.
+`cmpxchg`, `atomicrmw`, and `storermw` are essentially like an atomic load
+followed by an atomic store (where the store is conditional for `cmpxchg`),
+but no other memory operation can happen on any thread between the load and store.
A `fence` provides Acquire and/or Release ordering which is not part
of another operation; it is normally used along with Monotonic memory
@@ -251,8 +251,8 @@ Notes for optimizers
Notes for code generation
: Code generation is essentially the same as that for unordered for loads and
- stores. No fences are required. `cmpxchg` and `atomicrmw` are required
- to appear as a single operation.
+ stores. No fences are required. `cmpxchg`, `atomicrmw`, and `storermw` are
+ required to appear as a single operation.
### Acquire
@@ -473,17 +473,17 @@ atomic constructs. Here are some lowerings it can do:
For an example of these, look at the ARM (first five lowerings) or RISC-V (last
lowering) backend.
-AtomicExpandPass supports two strategies for lowering atomicrmw/cmpxchg to
-load-linked/store-conditional (LL/SC) loops. The first expands the LL/SC loop
+AtomicExpandPass supports two strategies for lowering atomicrmw/storermw/cmpxchg
+to load-linked/store-conditional (LL/SC) loops. The first expands the LL/SC loop
in IR, calling target lowering hooks to emit intrinsics for the LL and SC
operations. However, many architectures have strict requirements for LL/SC
loops to ensure forward progress, such as restrictions on the number and type
of instructions in the loop. It isn't possible to enforce these restrictions
when the loop is expanded in LLVM IR, and so affected targets may prefer to
expand to LL/SC loops at a very late stage (i.e. after register allocation).
-AtomicExpandPass can help support lowering of part-word atomicrmw or cmpxchg
-using this strategy by producing IR for any shifting and masking that can be
-performed outside of the LL/SC loop.
+AtomicExpandPass can help support lowering of part-word atomicrmw, storermw, or
+cmpxchg using this strategy by producing IR for any shifting and masking that
+can be performed outside of the LL/SC loop.
## Libcalls: `__atomic_*`
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index 5640eec987133..5ad2d36f5502a 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -3872,7 +3872,7 @@ to infer ``captures``:
* ptrtoint: Captures address and provenance.
* call/invoke: Depends on ``captures`` on arguments. The callee is never
captured.
- * store, atomicrmw, cmpxchg: Capture the address and provenance of the value
+ * store, atomicrmw, storermw, cmpxchg: Capture the address and provenance of the value
operand. Do not capture the pointer operand (unless volatile).
* Other (including insertvalue and insertelement): Conservatively assume
address and provenance are captured.
@@ -4006,13 +4006,12 @@ exactly one write into a load that may see multiple writes.)
Atomic Memory Ordering Constraints
----------------------------------
-Atomic instructions (:ref:`cmpxchg <i_cmpxchg>`,
-:ref:`atomicrmw <i_atomicrmw>`, :ref:`fence <i_fence>`,
-:ref:`atomic load <i_load>`, and :ref:`atomic store <i_store>`) take
-ordering parameters that determine which other atomic instructions on
-the same address they *synchronize with*. These semantics implement
-the Java or C++ memory models; if these descriptions aren't precise
-enough, check those specs (see spec references in the
+Atomic instructions (:ref:`cmpxchg <i_cmpxchg>`, :ref:`atomicrmw <i_atomicrmw>`,
+:ref:`storermw <i_storermw>`, :ref:`fence <i_fence>`, :ref:`atomic load <i_load>`,
+and :ref:`atomic store <i_store>`) take ordering parameters that determine which
+other atomic instructions on the same address they *synchronize with*.
+These semantics implement the Java or C++ memory models; if these descriptions
+aren't precise enough, check those specs (see spec references in the
:doc:`atomics guide <Atomics>`). :ref:`fence <i_fence>` instructions
treat these orderings somewhat differently since they don't take an
address. See that instruction's documentation for details.
@@ -4033,11 +4032,12 @@ For a simpler introduction to the ordering constraints, see the
address. All modification orders must be compatible with the
happens-before order. There is no guarantee that the modification
orders can be combined to a global total order for the whole program
- (and this often will not be possible). If the read in an atomic
- read-modify-write operation M (:ref:`cmpxchg <i_cmpxchg>` and
- :ref:`atomicrmw <i_atomicrmw>`) reads from a ``monotonic`` (or
- stronger) write W, W must be immediately before M in the address's
- modification order. If one atomic read happens before another atomic
+ (and this often will not be possible).
+ If the read in an atomic read-modify-write operation M (:ref:`cmpxchg <i_cmpxchg>`,
+ :ref:`atomicrmw <i_atomicrmw>`, and :ref:`storermw <i_storermw>`) reads from a
+ ``monotonic`` (or stronger) write W, W must be immediately before M in the address's
+ modification order.
+ If one atomic read happens before another atomic
read of the same address and both are at least ``monotonic``, the
later read must not see an earlier value in the address's
modification order. This disallows reordering of ``monotonic`` (or
@@ -8960,9 +8960,8 @@ inlined call.
The ``noalias.addrspace`` metadata is used to identify memory
operations which cannot access objects allocated in a range of address
-spaces. It is attached to memory instructions, including
-:ref:`atomicrmw <i_atomicrmw>`, :ref:`cmpxchg <i_cmpxchg>`, and
-:ref:`call <i_call>` instructions.
+spaces.
+It is attached to memory instructions, including :ref:`atomicrmw <i_atomicrmw>`, :ref:`cmpxchg <i_cmpxchg>`, :ref:`storermw <i_storermw>`, and :ref:`call <i_call>` instructions.
This follows the same form as :ref:`range metadata <range-metadata>`,
except the field entries must be of type `i32`. The interpretation is
@@ -9000,8 +8999,8 @@ Refer to :doc:`MemoryModelRelaxationAnnotations` for more information on how thi
affects the memory model of a given target.
It is attached to memory instructions such as:
-:ref:`atomicrmw <i_atomicrmw>`, :ref:`cmpxchg <i_cmpxchg>`, :ref:`load <i_load>`,
-:ref:`store <i_store>`, :ref:`fence <i_fence>` and
+:ref:`atomicrmw <i_atomicrmw>`, :ref:`storermw <i_storermw>`, :ref:`cmpxchg <i_cmpxchg>`,
+:ref:`load <i_load>`, :ref:`store <i_store>`, :ref:`fence <i_fence>` and
:ref:`call <i_call>` instructions that read or write memory.
The metadata is structured as pairs of strings: a prefix, and suffix that form a MMRA "tag".
@@ -12280,10 +12279,11 @@ Semantics:
A fence A which has (at least) ``release`` ordering semantics
*synchronizes with* a fence B with (at least) ``acquire`` ordering
-semantics if and only if there exist atomic operations X and Y, both
-operating on some atomic object M, such that A is sequenced before X, X
-modifies M (either directly or through some side effect of a sequence
-headed by X), Y is sequenced before B, and Y observes M. This provides a
+semantics if and only if there exist atomic operations X and Y, where Y
+is not an :ref:`storermw <i_storermw>` operation, both operating on some atomic object M,
+such that A is sequenced before X, X modifies M (either directly or through
+some side effect of a sequence headed by X), Y is sequenced before B,
+and Y observes M. This provides a
*happens-before* dependency between A and B. Rather than an explicit
``fence``, one (but not both) of the atomic operations X or Y might
provide a ``release`` or ``acquire`` (resp.) ordering constraint and
@@ -12420,7 +12420,7 @@ Syntax:
Overview:
"""""""""
-The '``atomicrmw``' instruction is used to atomically modify memory.
+The '``atomicrmw``' instruction is used to atomically modify memory, returning the value at the memory location before the modification.
Arguments:
""""""""""
@@ -12519,6 +12519,117 @@ Example:
%old = atomicrmw add ptr %ptr, i32 1 acquire ; yields i32
+.. _i_storermw:
+
+'``storermw``' Instruction
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+
+::
+
+ storermw [volatile] [elementwise] <operation> ptr <pointer>, <ty> <value> [syncscope("<target-scope>")] <ordering>[, align <alignment>]
+
+Overview:
+"""""""""
+
+The '``storermw``' instruction is used to atomically modify memory without returning the value at the memory location before the modification.
+Unlike :ref:`atomicrmw <i_atomicrmw>`, ``storermw`` has store-only synchronization semantics: it acts as a release operation when ordered ``release`` or ``seq_cst``, but never as an acquire operation.
+It therefore does not synchronize-with a preceding release operation the way an acquire load, acquire :ref:`atomicrmw <i_atomicrmw>`, or acquire :ref:`fence <i_fence>` following an atomic load would.
+This semantic difference, combined with not returning a value, enables more efficient code generation on targets with native support for store-only atomic operations (e.g., PTX ``red`` instructions, ARM LSE store-atomics).
+
+Arguments:
+""""""""""
+
+There are three arguments to the '``storermw``' instruction: an operation to apply, an address whose value to modify, and an argument to the operation.
+The operation must be one of the following keywords:
+
+- add
+- sub
+- and
+- nand
+- or
+- xor
+- max
+- min
+- umax
+- umin
+- fadd
+- fsub
+- fmax
+- fmin
+- fmaximum
+- fminimum
+- fmaximumnum
+- fminimumnum
+- uinc_wrap
+- udec_wrap
+- usub_cond
+- usub_sat
+
+For all of these operations, the type of '<value>' must be a type whose bit width is a power of two greater than or equal to eight.
+For add/sub/and/nand/or/xor/max/min/umax/umin/uinc_wrap/udec_wrap/usub_cond/usub_sat, this must be an integer type, or, if the ``elementwise`` modifier is present, a fixed vector of integer type.
+For fadd/fsub/fmax/fmin/fmaximum/fminimum/fmaximumnum/fminimumnum, this must be a floating-point or fixed vector of floating-point type.
+The type of the '<pointer>' operand must be a pointer to the type of '<value>'.
+If the ``storermw`` is marked as ``volatile``, then the optimizer is not allowed to modify the number or order of execution of this ``storermw`` with other :ref:`volatile operations <volatile>`.
+
+Note: if the alignment is not greater or equal to the size of the `<value>` type, the atomic operation is likely to require a lock and have poor performance.
+
+The alignment is only optional when parsing textual IR; for in-memory IR, it is always present.
+If unspecified, the alignment is assumed to be equal to the size of the '<value>' type.
+Note that this default alignment assumption is different from the alignment used for the load/store instructions when align isn't specified.
+
+A ``storermw`` instruction can also take an optional ":ref:`syncscope <syncscope>`" argument.
+
+With the ``elementwise`` modifier, ``<value>`` must be a fixed vector type, and the instruction has per-element vector atomic semantics.
+That is, it performs the specified operation atomically to individual vector elements in an unspecified order.
+Without the ``elementwise`` modifier, vector ``storermw`` modifies the entire vector atomically.
+
+The ordering must be one of: ``monotonic``, ``release``, or ``seq_cst``.
+The orderings ``acquire`` and ``acq_rel`` are not valid for ``storermw`` because it has no acquire semantics.
+``unordered`` is not valid either: a ``storermw`` is a read-modify-write and, like ``atomicrmw``, requires at least ``monotonic`` to give the atomic read-modify-write edge a well-defined position in the location's modification order.
+
+Semantics:
+""""""""""
+
+The contents of memory at the location specified by the '``<pointer>``' operand are atomically read, modified, and written back.
+Unlike :ref:`atomicrmw <i_atomicrmw>`, the original value at the location is not returned, and the operation has no acquire semantics.
+The modification is specified by the operation argument:
+
+- add: ``*ptr = *ptr + val``
+- sub: ``*ptr = *ptr - val``
+- and: ``*ptr = *ptr & val``
+- nand: ``*ptr = ~(*ptr & val)``
+- or: ``*ptr = *ptr | val``
+- xor: ``*ptr = *ptr ^ val``
+- max: ``*ptr = *ptr > val ? *ptr : val`` (using a signed comparison)
+- min: ``*ptr = *ptr < val ? *ptr : val`` (using a signed comparison)
+- umax: ``*ptr = *ptr > val ? *ptr : val`` (using an unsigned comparison)
+- umin: ``*ptr = *ptr < val ? *ptr : val`` (using an unsigned comparison)
+- fadd: ``*ptr = *ptr + val`` (using floating point arithmetic)
+- fsub: ``*ptr = *ptr - val`` (using floating point arithmetic)
+- fmax: ``*ptr = maxnum(*ptr, val)`` (match the `llvm.maxnum.*` intrinsic)
+- fmin: ``*ptr = minnum(*ptr, val)`` (match the `llvm.minnum.*` intrinsic)
+- fmaximum: ``*ptr = maximum(*ptr, val)`` (match the `llvm.maximum.*` intrinsic)
+- fminimum: ``*ptr = minimum(*ptr, val)`` (match the `llvm.minimum.*` intrinsic)
+- fmaximumnum: ``*ptr = maximumnum(*ptr, val)`` (match the `llvm.maximumnum.*` intrinsic)
+- fminimumnum: ``*ptr = minimumnum(*ptr, val)`` (match the `llvm.minimumnum.*` intrinsic)
+- uinc_wrap: ``*ptr = (*ptr u>= val) ? 0 : (*ptr + 1)`` (increment value with wraparound to zero when incremented above input value)
+- udec_wrap: ``*ptr = ((*ptr == 0) || (*ptr u> val)) ? val : (*ptr - 1)`` (decrement with wraparound to input value when decremented below zero).
+- usub_cond: ``*ptr = (*ptr u>= val) ? *ptr - val : *ptr`` (subtract only if no unsigned overflow).
+- usub_sat: ``*ptr = (*ptr u>= val) ? *ptr - val : 0`` (subtract with unsigned clamping to zero).
+
+Example:
+""""""""
+
+.. code-block:: llvm
+
+ storermw add ptr %ptr, i32 1 monotonic, align 4
+ storermw volatile fadd ptr %fptr, float 2.0 release, align 4
+ storermw umax ptr addrspace(1) %gptr, i32 %val seq_cst, align 4
+ storermw elementwise add ptr %vptr, <4 x i32> %vval seq_cst, align 16
+
.. _i_getelementptr:
'``getelementptr``' Instruction
diff --git a/llvm/include/llvm-c/Core.h b/llvm/include/llvm-c/Core.h
index f3b98a31c5bd7..c6aae98acd62e 100644
--- a/llvm/include/llvm-c/Core.h
+++ b/llvm/include/llvm-c/Core.h
@@ -60,93 +60,94 @@ LLVM_C_EXTERN_C_BEGIN
/// to reorder them.
typedef enum {
/* Terminator Instructions */
- LLVMRet = 1,
+ LLVMRet = 1,
/* removed 2 due to API changes */
- LLVMUncondBr = 70,
- LLVMCondBr = 71,
- LLVMSwitch = 3,
- LLVMIndirectBr = 4,
- LLVMInvoke = 5,
+ LLVMUncondBr = 70,
+ LLVMCondBr = 71,
+ LLVMSwitch = 3,
+ LLVMIndirectBr = 4,
+ LLVMInvoke = 5,
/* removed 6 due to API changes */
- LLVMUnreachable = 7,
- LLVMCallBr = 67,
+ LLVMUnreachable = 7,
+ LLVMCallBr = 67,
/* Standard Unary Operators */
- LLVMFNeg = 66,
+ LLVMFNeg = 66,
/* Standard Binary Operators */
- LLVMAdd = 8,
- LLVMFAdd = 9,
- LLVMSub = 10,
- LLVMFSub = 11,
- LLVMMul = 12,
- LLVMFMul = 13,
- LLVMUDiv = 14,
- LLVMSDiv = 15,
- LLVMFDiv = 16,
- LLVMURem = 17,
- LLVMSRem = 18,
- LLVMFRem = 19,
+ LLVMAdd = 8,
+ LLVMFAdd = 9,
+ LLVMSub = 10,
+ LLVMFSub = 11,
+ LLVMMul = 12,
+ LLVMFMul = 13,
+ LLVMUDiv = 14,
+ LLVMSDiv = 15,
+ LLVMFDiv = 16,
+ LLVMURem = 17,
+ LLVMSRem = 18,
+ LLVMFRem = 19,
/* Logical Operators */
- LLVMShl = 20,
- LLVMLShr = 21,
- LLVMAShr = 22,
- LLVMAnd = 23,
- LLVMOr = 24,
- LLVMXor = 25,
+ LLVMShl = 20,
+ LLVMLShr = 21,
+ LLVMAShr = 22,
+ LLVMAnd = 23,
+ LLVMOr = 24,
+ LLVMXor = 25,
/* Memory Operators */
- LLVMAlloca = 26,
- LLVMLoad = 27,
- LLVMStore = 28,
- LLVMGetElementPtr = 29,
+ LLVMAlloca = 26,
+ LLVMLoad = 27,
+ LLVMStore = 28,
+ LLVMGetElementPtr = 29,
/* Cast Operators */
- LLVMTrunc = 30,
- LLVMZExt = 31,
- LLVMSExt = 32,
- LLVMFPToUI = 33,
- LLVMFPToSI = 34,
- LLVMUIToFP = 35,
- LLVMSIToFP = 36,
- LLVMFPTrunc = 37,
- LLVMFPExt = 38,
- LLVMPtrToInt = 39,
- LLVMPtrToAddr = 69,
- LLVMIntToPtr = 40,
- LLVMBitCast = 41,
- LLVMAddrSpaceCast = 60,
+ LLVMTrunc = 30,
+ LLVMZExt = 31,
+ LLVMSExt = 32,
+ LLVMFPToUI = 33,
+ LLVMFPToSI = 34,
+ LLVMUIToFP = 35,
+ LLVMSIToFP = 36,
+ LLVMFPTrunc = 37,
+ LLVMFPExt = 38,
+ LLVMPtrToInt = 39,
+ LLVMPtrToAddr = 69,
+ LLVMIntToPtr = 40,
+ LLVMBitCast = 41,
+ LLVMAddrSpaceCast = 60,
/* Other Operators */
- LLVMICmp = 42,
- LLVMFCmp = 43,
- LLVMPHI = 44,
- LLVMCall = 45,
- LLVMSelect = 46,
- LLVMUserOp1 = 47,
- LLVMUserOp2 = 48,
- LLVMVAArg = 49,
+ LLVMICmp = 42,
+ LLVMFCmp = 43,
+ LLVMPHI = 44,
+ LLVMCall = 45,
+ LLVMSelect = 46,
+ LLVMUserOp1 = 47,
+ LLVMUserOp2 = 48,
+ LLVMVAArg = 49,
LLVMExtractElement = 50,
- LLVMInsertElement = 51,
- LLVMShuffleVector = 52,
- LLVMExtractValue = 53,
- LLVMInsertValue = 54,
- LLVMFreeze = 68,
+ LLVMInsertElement = 51,
+ LLVMShuffleVector = 52,
+ LLVMExtractValue = 53,
+ LLVMInsertValue = 54,
+ LLVMFreeze = 68,
/* Atomic operators */
- LLVMFence = 55,
- LLVMAtomicCmpXchg = 56,
- LLVMAtomicRMW = 57,
+ LLVMFence = 55,
+ LLVMAtomicCmpXchg = 56,
+ LLVMAtomicRMW = 57,
+ LLVMStoreRMW = 72,
/* Exception Handling Operators */
- LLVMResume = 58,
- LLVMLandingPad = 59,
- LLVMCleanupRet = 61,
- LLVMCatchRet = 62,
- LLVMCatchPad = 63,
- LLVMCleanupPad = 64,
- LLVMCatchSwitch = 65
+ LLVMResume = 58,
+ LLVMLandingPad = 59,
+ LLVMCleanupRet = 61,
+ LLVMCatchRet = 62,
+ LLVMCatchPad = 63,
+ LLVMCleanupPad = 64,
+ LLVMCatchSwitch = 65
} LLVMOpcode;
typedef enum {
diff --git a/llvm/include/llvm/Analysis/AliasAnalysis.h b/llvm/include/llvm/Analysis/AliasAnalysis.h
index 4fbe8ea7b8de6..57d6b07334631 100644
--- a/llvm/include/llvm/Analysis/AliasAnalysis.h
+++ b/llvm/include/llvm/Analysis/AliasAnalysis.h
@@ -612,6 +612,9 @@ class AAResults {
LLVM_ABI ModRefInfo getModRefInfo(const AtomicRMWInst *RMW,
const MemoryLocation &Loc,
AAQueryInfo &AAQI);
+ LLVM_ABI ModRefInfo getModRefInfo(const StoreRMWInst *SI,
+ const MemoryLocation &Loc,
+ AAQueryInfo &AAQI);
LLVM_ABI ModRefInfo getModRefInfo(const CatchPadInst *I,
const MemoryLocation &Loc,
AAQueryInfo &AAQI);
diff --git a/llvm/include/llvm/Analysis/MemoryLocation.h b/llvm/include/llvm/Analysis/MemoryLocation.h
index f9ad8f9c40d98..12ec2f73612dd 100644
--- a/llvm/include/llvm/Analysis/MemoryLocation.h
+++ b/llvm/include/llvm/Analysis/MemoryLocation.h
@@ -32,6 +32,7 @@ class MemTransferInst;
class MemIntrinsic;
class AtomicCmpXchgInst;
class AtomicRMWInst;
+class StoreRMWInst;
class AnyMemTransferInst;
class AnyMemIntrinsic;
class TargetLibraryInfo;
@@ -246,6 +247,7 @@ class MemoryLocation {
LLVM_ABI static MemoryLocation get(const VAArgInst *VI);
LLVM_ABI static MemoryLocation get(const AtomicCmpXchgInst *CXI);
LLVM_ABI static MemoryLocation get(const AtomicRMWInst *RMWI);
+ LLVM_ABI static MemoryLocation get(const StoreRMWInst *SI);
static MemoryLocation get(const Instruction *Inst) {
return *MemoryLocation::getOrNone(Inst);
}
diff --git a/llvm/include/llvm/AsmParser/LLParser.h b/llvm/include/llvm/AsmParser/LLParser.h
index 788b56cb78f08..362faed5e8197 100644
--- a/llvm/include/llvm/AsmParser/LLParser.h
+++ b/llvm/include/llvm/AsmParser/LLParser.h
@@ -703,6 +703,7 @@ namespace llvm {
int parseStore(Instruction *&Inst, PerFunctionState &PFS);
int parseCmpXchg(Instruction *&Inst, PerFunctionState &PFS);
int parseAtomicRMW(Instruction *&Inst, PerFunctionState &PFS);
+ int parseStoreRMW(Instruction *&Inst, PerFunctionState &PFS);
int parseFence(Instruction *&Inst, PerFunctionState &PFS);
int parseGetElementPtr(Instruction *&Inst, PerFunctionState &PFS);
int parseExtractValue(Instruction *&Inst, PerFunctionState &PFS);
diff --git a/llvm/include/llvm/AsmParser/LLToken.h b/llvm/include/llvm/AsmParser/LLToken.h
index d2766a05ce9ba..efbb1a29185d8 100644
--- a/llvm/include/llvm/AsmParser/LLToken.h
+++ b/llvm/include/llvm/AsmParser/LLToken.h
@@ -368,6 +368,7 @@ enum Kind {
kw_fence,
kw_cmpxchg,
kw_atomicrmw,
+ kw_storermw,
kw_getelementptr,
kw_extractelement,
diff --git a/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
index af4cac996a78c..428bee39878df 100644
--- a/llvm/include/llvm/Bitcode/LLVMBitCodes.h
+++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
@@ -703,6 +703,9 @@ enum FunctionCodes {
FUNC_CODE_DEBUG_RECORD_LABEL = 65, // [DILocation, DILabel]
FUNC_CODE_DEBUG_RECORD_DECLARE_VALUE =
66, // [DILocation, DILocalVariable, DIExpression, ValueAsMetadata]
+ FUNC_CODE_INST_STORERMW = 67, // STORERMW: [ptrty, ptr, valty, val,
+ // operation, align, vol,
+ // ordering, syncscope]
};
enum UseListCodes {
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h
index a1954987571ea..a527aae54f4fb 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h
@@ -571,6 +571,12 @@ class LLVM_ABI IRTranslator : public MachineFunctionPass {
// Stubs to keep the compiler happy while we implement the rest of the
// translation.
+
+ // StoreRMW is lowered by AtomicExpandPass before reaching GlobalISel
+ bool translateStoreRMW(const User &U, MachineIRBuilder &MIRBuilder) {
+ return false; // Should not reach GlobalISel
+ }
+
bool translateResume(const User &U, MachineIRBuilder &MIRBuilder) {
return false;
}
diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h
index 224855e02f21d..da9f126d3c568 100644
--- a/llvm/include/llvm/CodeGen/ISDOpcodes.h
+++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h
@@ -1435,6 +1435,33 @@ enum NodeType {
ATOMIC_LOAD_USUB_COND,
ATOMIC_LOAD_USUB_SAT,
+ /// OUTCHAIN = ATOMIC_STORE_[OpName](INCHAIN, ptr, val)
+ /// These correspond to the storermw instruction. Like ATOMIC_STORE, they
+ /// return only a chain (no value). Unlike ATOMIC_LOAD_*, the old memory
+ /// value is discarded.
+ ATOMIC_STORE_ADD,
+ ATOMIC_STORE_SUB,
+ ATOMIC_STORE_AND,
+ ATOMIC_STORE_NAND,
+ ATOMIC_STORE_OR,
+ ATOMIC_STORE_XOR,
+ ATOMIC_STORE_MIN,
+ ATOMIC_STORE_MAX,
+ ATOMIC_STORE_UMIN,
+ ATOMIC_STORE_UMAX,
+ ATOMIC_STORE_FADD,
+ ATOMIC_STORE_FSUB,
+ ATOMIC_STORE_FMIN,
+ ATOMIC_STORE_FMAX,
+ ATOMIC_STORE_FMINIMUM,
+ ATOMIC_STORE_FMAXIMUM,
+ ATOMIC_STORE_FMINIMUMNUM,
+ ATOMIC_STORE_FMAXIMUMNUM,
+ ATOMIC_STORE_UINC_WRAP,
+ ATOMIC_STORE_UDEC_WRAP,
+ ATOMIC_STORE_USUB_COND,
+ ATOMIC_STORE_USUB_SAT,
+
/// Masked load and store - consecutive vector load and store operations
/// with additional mask operand that prevents memory accesses to the
/// masked-off lanes.
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
index b280bd48c94e9..04b9aa407b0f8 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -1625,6 +1625,28 @@ class MemSDNode : public SDNode {
case ISD::ATOMIC_LOAD_USUB_SAT:
case ISD::ATOMIC_LOAD:
case ISD::ATOMIC_STORE:
+ case ISD::ATOMIC_STORE_ADD:
+ case ISD::ATOMIC_STORE_SUB:
+ case ISD::ATOMIC_STORE_AND:
+ case ISD::ATOMIC_STORE_NAND:
+ case ISD::ATOMIC_STORE_OR:
+ case ISD::ATOMIC_STORE_XOR:
+ case ISD::ATOMIC_STORE_MIN:
+ case ISD::ATOMIC_STORE_MAX:
+ case ISD::ATOMIC_STORE_UMIN:
+ case ISD::ATOMIC_STORE_UMAX:
+ case ISD::ATOMIC_STORE_FADD:
+ case ISD::ATOMIC_STORE_FSUB:
+ case ISD::ATOMIC_STORE_FMIN:
+ case ISD::ATOMIC_STORE_FMAX:
+ case ISD::ATOMIC_STORE_FMINIMUM:
+ case ISD::ATOMIC_STORE_FMAXIMUM:
+ case ISD::ATOMIC_STORE_FMINIMUMNUM:
+ case ISD::ATOMIC_STORE_FMAXIMUMNUM:
+ case ISD::ATOMIC_STORE_UINC_WRAP:
+ case ISD::ATOMIC_STORE_UDEC_WRAP:
+ case ISD::ATOMIC_STORE_USUB_COND:
+ case ISD::ATOMIC_STORE_USUB_SAT:
case ISD::MLOAD:
case ISD::MSTORE:
case ISD::MGATHER:
@@ -1712,7 +1734,29 @@ class AtomicSDNode : public MemSDNode {
N->getOpcode() == ISD::ATOMIC_LOAD_USUB_COND ||
N->getOpcode() == ISD::ATOMIC_LOAD_USUB_SAT ||
N->getOpcode() == ISD::ATOMIC_LOAD ||
- N->getOpcode() == ISD::ATOMIC_STORE;
+ N->getOpcode() == ISD::ATOMIC_STORE ||
+ N->getOpcode() == ISD::ATOMIC_STORE_ADD ||
+ N->getOpcode() == ISD::ATOMIC_STORE_SUB ||
+ N->getOpcode() == ISD::ATOMIC_STORE_AND ||
+ N->getOpcode() == ISD::ATOMIC_STORE_NAND ||
+ N->getOpcode() == ISD::ATOMIC_STORE_OR ||
+ N->getOpcode() == ISD::ATOMIC_STORE_XOR ||
+ N->getOpcode() == ISD::ATOMIC_STORE_MIN ||
+ N->getOpcode() == ISD::ATOMIC_STORE_MAX ||
+ N->getOpcode() == ISD::ATOMIC_STORE_UMIN ||
+ N->getOpcode() == ISD::ATOMIC_STORE_UMAX ||
+ N->getOpcode() == ISD::ATOMIC_STORE_FADD ||
+ N->getOpcode() == ISD::ATOMIC_STORE_FSUB ||
+ N->getOpcode() == ISD::ATOMIC_STORE_FMIN ||
+ N->getOpcode() == ISD::ATOMIC_STORE_FMAX ||
+ N->getOpcode() == ISD::ATOMIC_STORE_FMINIMUM ||
+ N->getOpcode() == ISD::ATOMIC_STORE_FMAXIMUM ||
+ N->getOpcode() == ISD::ATOMIC_STORE_FMINIMUMNUM ||
+ N->getOpcode() == ISD::ATOMIC_STORE_FMAXIMUMNUM ||
+ N->getOpcode() == ISD::ATOMIC_STORE_UINC_WRAP ||
+ N->getOpcode() == ISD::ATOMIC_STORE_UDEC_WRAP ||
+ N->getOpcode() == ISD::ATOMIC_STORE_USUB_COND ||
+ N->getOpcode() == ISD::ATOMIC_STORE_USUB_SAT;
}
};
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 7a10a693934fd..715fac4b48fcf 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -2504,6 +2504,17 @@ class LLVM_ABI TargetLoweringBase {
return AtomicExpansionKind::None;
}
+ /// Returns how the IR-level AtomicExpand pass should expand the given
+ /// StoreRMW, if at all. Default is to expand to AtomicRMW (discard result).
+ /// Targets that can handle atomic reductions natively (e.g., NVPTX with
+ /// red.global/red.shared instructions) should return None for supported
+ /// cases.
+ virtual AtomicExpansionKind
+ shouldExpandStoreRMWInIR(const StoreRMWInst *ARI) const {
+ // Default: expand storermw to atomicrmw (discard the result)
+ return AtomicExpansionKind::Expand;
+ }
+
/// On some platforms, an AtomicRMW that never actually modifies the value
/// (such as fetch_add of 0) can be turned into a fence followed by an
/// atomic load. This may sound useless, but it makes it possible for the
diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h
index 692990f93c6fb..2675ae2f5c084 100644
--- a/llvm/include/llvm/IR/IRBuilder.h
+++ b/llvm/include/llvm/IR/IRBuilder.h
@@ -1981,6 +1981,18 @@ class IRBuilderBase {
new AtomicRMWInst(Op, Ptr, Val, *Align, Ordering, SSID, Elementwise));
}
+ StoreRMWInst *CreateStoreRMW(AtomicRMWInst::BinOp Op, Value *Ptr, Value *Val,
+ MaybeAlign Align, AtomicOrdering Ordering,
+ SyncScope::ID SSID = SyncScope::System,
+ bool Elementwise = false) {
+ if (!Align) {
+ const DataLayout &DL = BB->getDataLayout();
+ Align = llvm::Align(DL.getTypeStoreSize(Val->getType()));
+ }
+ return Insert(
+ new StoreRMWInst(Op, Ptr, Val, *Align, Ordering, SSID, Elementwise));
+ }
+
Value *CreateStructuredGEP(Type *BaseType, Value *PtrBase,
ArrayRef<Value *> Indices,
const Twine &Name = "") {
diff --git a/llvm/include/llvm/IR/InstVisitor.h b/llvm/include/llvm/IR/InstVisitor.h
index 45b2e59510b67..61a255c4f9e00 100644
--- a/llvm/include/llvm/IR/InstVisitor.h
+++ b/llvm/include/llvm/IR/InstVisitor.h
@@ -170,6 +170,7 @@ class InstVisitor {
RetTy visitStoreInst(StoreInst &I) { DELEGATE(Instruction);}
RetTy visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) { DELEGATE(Instruction);}
RetTy visitAtomicRMWInst(AtomicRMWInst &I) { DELEGATE(Instruction);}
+ RetTy visitStoreRMWInst(StoreRMWInst &I) { DELEGATE(Instruction); }
RetTy visitFenceInst(FenceInst &I) { DELEGATE(Instruction);}
RetTy visitGetElementPtrInst(GetElementPtrInst &I){ DELEGATE(Instruction);}
RetTy visitPHINode(PHINode &I) { DELEGATE(Instruction);}
diff --git a/llvm/include/llvm/IR/Instruction.def b/llvm/include/llvm/IR/Instruction.def
index 949a9d0beecb4..069d4176d6ef7 100644
--- a/llvm/include/llvm/IR/Instruction.def
+++ b/llvm/include/llvm/IR/Instruction.def
@@ -220,7 +220,8 @@ HANDLE_OTHER_INST(66, ExtractValue, ExtractValueInst)// extract from aggregate
HANDLE_OTHER_INST(67, InsertValue, InsertValueInst) // insert into aggregate
HANDLE_OTHER_INST(68, LandingPad, LandingPadInst) // Landing pad instruction.
HANDLE_OTHER_INST(69, Freeze, FreezeInst) // Freeze instruction.
- LAST_OTHER_INST(69)
+HANDLE_OTHER_INST(70, StoreRMW, StoreRMWInst) // Atomic reduction (store-only)
+ LAST_OTHER_INST(70)
#undef FIRST_TERM_INST
#undef HANDLE_TERM_INST
diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h
index f43a9b5c1acc2..8d618678b609a 100644
--- a/llvm/include/llvm/IR/Instructions.h
+++ b/llvm/include/llvm/IR/Instructions.h
@@ -950,6 +950,163 @@ struct OperandTraits<AtomicRMWInst>
DEFINE_TRANSPARENT_OPERAND_ACCESSORS(AtomicRMWInst, Value)
+//===----------------------------------------------------------------------===//
+// StoreRMWInst Class
+//===----------------------------------------------------------------------===//
+
+/// An instruction that atomically applies a binary operation to memory without
+/// returning the old value. Unlike atomicrmw, this instruction has store-only
+/// semantics and does not participate in acquire synchronization patterns.
+/// This enables more efficient code generation on targets with native support
+/// for store-only atomic operations (e.g., PTX 'red', ARM LSE store-atomics).
+class StoreRMWInst : public Instruction {
+protected:
+ friend class Instruction;
+
+ LLVM_ABI StoreRMWInst *cloneImpl() const;
+
+public:
+ /// Reuse the BinOp enumeration from AtomicRMWInst since the operations
+ /// are identical, just with different return semantics.
+ using BinOp = AtomicRMWInst::BinOp;
+
+private:
+ template <unsigned Offset>
+ using AtomicOrderingBitfieldElement =
+ typename Bitfield::Element<AtomicOrdering, Offset, 3,
+ AtomicOrdering::LAST>;
+
+ template <unsigned Offset>
+ using BinOpBitfieldElement =
+ typename Bitfield::Element<BinOp, Offset, 5, BinOp::LAST_BINOP>;
+
+ constexpr static IntrusiveOperandsAllocMarker AllocMarker{2};
+
+public:
+ LLVM_ABI StoreRMWInst(BinOp Operation, Value *Ptr, Value *Val,
+ Align Alignment, AtomicOrdering Ordering,
+ SyncScope::ID SSID, bool Elementwise = false,
+ InsertPosition InsertBefore = nullptr);
+
+ // allocate space for exactly two operands
+ void *operator new(size_t S) { return User::operator new(S, AllocMarker); }
+ void operator delete(void *Ptr) { User::operator delete(Ptr, AllocMarker); }
+
+ using VolatileField = BoolBitfieldElementT<0>;
+ using AtomicOrderingField =
+ AtomicOrderingBitfieldElementT<VolatileField::NextBit>;
+ using OperationField = BinOpBitfieldElement<AtomicOrderingField::NextBit>;
+ using AlignmentField = AlignmentBitfieldElementT<OperationField::NextBit>;
+ using ElementwiseField = BoolBitfieldElementT<AlignmentField::NextBit>;
+ static_assert(Bitfield::areContiguous<VolatileField, AtomicOrderingField,
+ OperationField, AlignmentField,
+ ElementwiseField>(),
+ "Bitfields must be contiguous");
+
+ BinOp getOperation() const { return getSubclassData<OperationField>(); }
+
+ static StringRef getOperationName(BinOp Op) {
+ return AtomicRMWInst::getOperationName(Op);
+ }
+
+ static bool isFPOperation(BinOp Op) {
+ return AtomicRMWInst::isFPOperation(Op);
+ }
+
+ void setOperation(BinOp Operation) {
+ setSubclassData<OperationField>(Operation);
+ }
+
+ /// Return the alignment of the memory that is being modified.
+ Align getAlign() const {
+ return Align(1ULL << getSubclassData<AlignmentField>());
+ }
+
+ void setAlignment(Align Align) {
+ setSubclassData<AlignmentField>(Log2(Align));
+ }
+
+ /// Return true if this is an atomic reduction on a volatile memory location.
+ bool isVolatile() const { return getSubclassData<VolatileField>(); }
+
+ /// Specify whether this is a volatile atomic reduction or not.
+ void setVolatile(bool V) { setSubclassData<VolatileField>(V); }
+
+ /// Return true if this storermw has elementwise vector semantics.
+ bool isElementwise() const { return getSubclassData<ElementwiseField>(); }
+
+ /// Specify whether this storermw has elementwise vector semantics.
+ void setElementwise(bool V) { setSubclassData<ElementwiseField>(V); }
+
+ /// Transparently provide more efficient getOperand methods.
+ DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
+
+ /// Returns the ordering constraint of this atomic reduction instruction.
+ AtomicOrdering getOrdering() const {
+ return getSubclassData<AtomicOrderingField>();
+ }
+
+ /// Sets the ordering constraint of this atomic reduction instruction.
+ /// Only store-compatible orderings are valid: unordered, monotonic,
+ /// release, and seq_cst.
+ void setOrdering(AtomicOrdering Ordering) {
+ assert(Ordering != AtomicOrdering::NotAtomic &&
+ "storermw instructions must be atomic.");
+ assert(Ordering != AtomicOrdering::Acquire &&
+ Ordering != AtomicOrdering::AcquireRelease &&
+ "storermw instructions cannot have acquire semantics.");
+ setSubclassData<AtomicOrderingField>(Ordering);
+ }
+
+ /// Returns the synchronization scope ID of this atomic reduction instruction.
+ SyncScope::ID getSyncScopeID() const { return SSID; }
+
+ /// Sets the synchronization scope ID of this atomic reduction instruction.
+ void setSyncScopeID(SyncScope::ID SSID) { this->SSID = SSID; }
+
+ Value *getPointerOperand() { return getOperand(0); }
+ const Value *getPointerOperand() const { return getOperand(0); }
+ static unsigned getPointerOperandIndex() { return 0U; }
+
+ Value *getValOperand() { return getOperand(1); }
+ const Value *getValOperand() const { return getOperand(1); }
+
+ /// Returns the address space of the pointer operand.
+ unsigned getPointerAddressSpace() const {
+ return getPointerOperand()->getType()->getPointerAddressSpace();
+ }
+
+ bool isFloatingPointOperation() const {
+ return isFPOperation(getOperation());
+ }
+
+ // Methods for support type inquiry through isa, cast, and dyn_cast:
+ static bool classof(const Instruction *I) {
+ return I->getOpcode() == Instruction::StoreRMW;
+ }
+ static bool classof(const Value *V) {
+ return isa<Instruction>(V) && classof(cast<Instruction>(V));
+ }
+
+private:
+ void Init(BinOp Operation, Value *Ptr, Value *Val, Align Align,
+ AtomicOrdering Ordering, SyncScope::ID SSID, bool Elementwise);
+
+ template <typename Bitfield>
+ void setSubclassData(typename Bitfield::Type Value) {
+ Instruction::setSubclassData<Bitfield>(Value);
+ }
+
+ /// The synchronization scope ID of this atomic reduction instruction.
+ SyncScope::ID SSID;
+};
+
+template <>
+struct OperandTraits<StoreRMWInst>
+ : public FixedNumOperandTraits<StoreRMWInst, 2> {};
+
+DEFINE_TRANSPARENT_OPERAND_ACCESSORS(StoreRMWInst, Value)
+
//===----------------------------------------------------------------------===//
// GetElementPtrInst Class
//===----------------------------------------------------------------------===//
@@ -5369,6 +5526,8 @@ inline std::optional<SyncScope::ID> getAtomicSyncScopeID(const Instruction *I) {
return AI->getSyncScopeID();
if (auto *AI = dyn_cast<AtomicRMWInst>(I))
return AI->getSyncScopeID();
+ if (auto *AI = dyn_cast<StoreRMWInst>(I))
+ return AI->getSyncScopeID();
llvm_unreachable("unhandled atomic operation");
}
@@ -5385,6 +5544,8 @@ inline void setAtomicSyncScopeID(Instruction *I, SyncScope::ID SSID) {
AI->setSyncScopeID(SSID);
else if (auto *AI = dyn_cast<AtomicRMWInst>(I))
AI->setSyncScopeID(SSID);
+ else if (auto *AI = dyn_cast<StoreRMWInst>(I))
+ AI->setSyncScopeID(SSID);
else
llvm_unreachable("unhandled atomic operation");
}
diff --git a/llvm/lib/Analysis/AliasAnalysis.cpp b/llvm/lib/Analysis/AliasAnalysis.cpp
index c7984c35e87ae..5af88681a876e 100644
--- a/llvm/lib/Analysis/AliasAnalysis.cpp
+++ b/llvm/lib/Analysis/AliasAnalysis.cpp
@@ -637,6 +637,21 @@ ModRefInfo AAResults::getModRefInfo(const AtomicRMWInst *RMW,
return ModRefInfo::ModRef;
}
+ModRefInfo AAResults::getModRefInfo(const StoreRMWInst *SI,
+ const MemoryLocation &Loc,
+ AAQueryInfo &AAQI) {
+ if (Loc.Ptr) {
+ AliasResult AR = alias(MemoryLocation::get(SI), Loc, AAQI, SI);
+ if (AR == AliasResult::NoAlias) {
+ if (isStrongerThanMonotonic(SI->getOrdering()))
+ return getSyncEffects(this, Loc, AAQI);
+ return ModRefInfo::NoModRef;
+ }
+ }
+
+ return ModRefInfo::ModRef;
+}
+
ModRefInfo AAResults::getModRefInfo(const Instruction *I,
const std::optional<MemoryLocation> &OptLoc,
AAQueryInfo &AAQIP) {
@@ -660,6 +675,8 @@ ModRefInfo AAResults::getModRefInfo(const Instruction *I,
return getModRefInfo((const AtomicCmpXchgInst *)I, Loc, AAQIP);
case Instruction::AtomicRMW:
return getModRefInfo((const AtomicRMWInst *)I, Loc, AAQIP);
+ case Instruction::StoreRMW:
+ return getModRefInfo((const StoreRMWInst *)I, Loc, AAQIP);
case Instruction::Call:
case Instruction::CallBr:
case Instruction::Invoke:
diff --git a/llvm/lib/Analysis/MemoryLocation.cpp b/llvm/lib/Analysis/MemoryLocation.cpp
index edca3871ad593..829747ea12e2d 100644
--- a/llvm/lib/Analysis/MemoryLocation.cpp
+++ b/llvm/lib/Analysis/MemoryLocation.cpp
@@ -74,6 +74,15 @@ MemoryLocation MemoryLocation::get(const AtomicRMWInst *RMWI) {
RMWI->getAAMetadata());
}
+MemoryLocation MemoryLocation::get(const StoreRMWInst *SI) {
+ const auto &DL = SI->getDataLayout();
+
+ return MemoryLocation(SI->getPointerOperand(),
+ LocationSize::precise(DL.getTypeStoreSize(
+ SI->getValOperand()->getType())),
+ SI->getAAMetadata());
+}
+
std::optional<MemoryLocation>
MemoryLocation::getOrNone(const Instruction *Inst) {
switch (Inst->getOpcode()) {
@@ -87,6 +96,8 @@ MemoryLocation::getOrNone(const Instruction *Inst) {
return get(cast<AtomicCmpXchgInst>(Inst));
case Instruction::AtomicRMW:
return get(cast<AtomicRMWInst>(Inst));
+ case Instruction::StoreRMW:
+ return get(cast<StoreRMWInst>(Inst));
default:
return std::nullopt;
}
diff --git a/llvm/lib/AsmParser/LLLexer.cpp b/llvm/lib/AsmParser/LLLexer.cpp
index 069a180056488..4067c9b05ad73 100644
--- a/llvm/lib/AsmParser/LLLexer.cpp
+++ b/llvm/lib/AsmParser/LLLexer.cpp
@@ -979,6 +979,7 @@ lltok::Kind LLLexer::LexIdentifier() {
INSTKEYWORD(store, Store);
INSTKEYWORD(cmpxchg, AtomicCmpXchg);
INSTKEYWORD(atomicrmw, AtomicRMW);
+ INSTKEYWORD(storermw, StoreRMW);
INSTKEYWORD(fence, Fence);
INSTKEYWORD(getelementptr, GetElementPtr);
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index 75e6add0ec76f..4c6e62ae1ad1e 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -7819,6 +7819,8 @@ int LLParser::parseInstruction(Instruction *&Inst, BasicBlock *BB,
return parseCmpXchg(Inst, PFS);
case lltok::kw_atomicrmw:
return parseAtomicRMW(Inst, PFS);
+ case lltok::kw_storermw:
+ return parseStoreRMW(Inst, PFS);
case lltok::kw_fence:
return parseFence(Inst, PFS);
case lltok::kw_getelementptr:
@@ -9225,6 +9227,171 @@ int LLParser::parseAtomicRMW(Instruction *&Inst, PerFunctionState &PFS) {
return AteExtraComma ? InstExtraComma : InstNormal;
}
+/// parseStoreRMW
+/// ::= 'storermw' 'volatile'? 'elementwise'? BinOp TypeAndValue ','
+/// TypeAndValue 'singlethread'? AtomicOrdering
+int LLParser::parseStoreRMW(Instruction *&Inst, PerFunctionState &PFS) {
+ Value *Ptr, *Val;
+ LocTy PtrLoc, ValLoc;
+ bool AteExtraComma = false;
+ AtomicOrdering Ordering = AtomicOrdering::NotAtomic;
+ SyncScope::ID SSID = SyncScope::System;
+ bool isVolatile = false;
+ bool IsElementwise = false;
+ bool IsFP = false;
+ AtomicRMWInst::BinOp Operation;
+ MaybeAlign Alignment;
+
+ if (EatIfPresent(lltok::kw_volatile))
+ isVolatile = true;
+ if (EatIfPresent(lltok::kw_elementwise))
+ IsElementwise = true;
+
+ switch (Lex.getKind()) {
+ default:
+ return tokError("expected binary operation in storermw");
+ case lltok::kw_xchg:
+ return tokError("xchg operation not valid for storermw (use atomicrmw)");
+ case lltok::kw_add:
+ Operation = AtomicRMWInst::Add;
+ break;
+ case lltok::kw_sub:
+ Operation = AtomicRMWInst::Sub;
+ break;
+ case lltok::kw_and:
+ Operation = AtomicRMWInst::And;
+ break;
+ case lltok::kw_nand:
+ Operation = AtomicRMWInst::Nand;
+ break;
+ case lltok::kw_or:
+ Operation = AtomicRMWInst::Or;
+ break;
+ case lltok::kw_xor:
+ Operation = AtomicRMWInst::Xor;
+ break;
+ case lltok::kw_max:
+ Operation = AtomicRMWInst::Max;
+ break;
+ case lltok::kw_min:
+ Operation = AtomicRMWInst::Min;
+ break;
+ case lltok::kw_umax:
+ Operation = AtomicRMWInst::UMax;
+ break;
+ case lltok::kw_umin:
+ Operation = AtomicRMWInst::UMin;
+ break;
+ case lltok::kw_uinc_wrap:
+ Operation = AtomicRMWInst::UIncWrap;
+ break;
+ case lltok::kw_udec_wrap:
+ Operation = AtomicRMWInst::UDecWrap;
+ break;
+ case lltok::kw_usub_cond:
+ Operation = AtomicRMWInst::USubCond;
+ break;
+ case lltok::kw_usub_sat:
+ Operation = AtomicRMWInst::USubSat;
+ break;
+ case lltok::kw_fadd:
+ Operation = AtomicRMWInst::FAdd;
+ IsFP = true;
+ break;
+ case lltok::kw_fsub:
+ Operation = AtomicRMWInst::FSub;
+ IsFP = true;
+ break;
+ case lltok::kw_fmax:
+ Operation = AtomicRMWInst::FMax;
+ IsFP = true;
+ break;
+ case lltok::kw_fmin:
+ Operation = AtomicRMWInst::FMin;
+ IsFP = true;
+ break;
+ case lltok::kw_fmaximum:
+ Operation = AtomicRMWInst::FMaximum;
+ IsFP = true;
+ break;
+ case lltok::kw_fminimum:
+ Operation = AtomicRMWInst::FMinimum;
+ IsFP = true;
+ break;
+ case lltok::kw_fmaximumnum:
+ Operation = AtomicRMWInst::FMaximumNum;
+ IsFP = true;
+ break;
+ case lltok::kw_fminimumnum:
+ Operation = AtomicRMWInst::FMinimumNum;
+ IsFP = true;
+ break;
+ }
+ Lex.Lex(); // Eat the operation.
+
+ if (parseTypeAndValue(Ptr, PtrLoc, PFS) ||
+ parseToken(lltok::comma, "expected ',' after storermw address") ||
+ parseTypeAndValue(Val, ValLoc, PFS) ||
+ parseScopeAndOrdering(true /*Always atomic*/, SSID, Ordering) ||
+ parseOptionalCommaAlign(Alignment, AteExtraComma))
+ return true;
+
+ // Validate ordering - storermw only allows store-compatible orderings
+ if (Ordering == AtomicOrdering::Acquire ||
+ Ordering == AtomicOrdering::AcquireRelease)
+ return tokError(
+ "storermw cannot have acquire semantics (only store orderings: "
+ "monotonic, release, seq_cst)");
+ if (Ordering == AtomicOrdering::NotAtomic ||
+ Ordering == AtomicOrdering::Unordered)
+ return tokError("storermw must be at least monotonic");
+
+ if (!Ptr->getType()->isPointerTy())
+ return error(PtrLoc, "storermw operand must be a pointer");
+ if (Val->getType()->isScalableTy())
+ return error(ValLoc, "storermw operand may not be scalable");
+
+ // For elementwise ops, the value must be a fixed vector type whose element
+ // type is legal for the corresponding scalar storermw operation. So assign
+ // ScalarTy the element type for elementwise ops so we can check this.
+ Type *ScalarTy = Val->getType();
+ if (IsElementwise) {
+ auto *VecTy = dyn_cast<FixedVectorType>(Val->getType());
+ if (!VecTy)
+ return error(ValLoc,
+ "storermw elementwise operand must be a fixed vector type");
+ ScalarTy = VecTy->getElementType();
+ }
+
+ if (IsFP) {
+ if (!Val->getType()->isFPOrFPVectorTy()) {
+ return error(ValLoc, "storermw " +
+ AtomicRMWInst::getOperationName(Operation) +
+ " operand must be a floating point type");
+ }
+ } else {
+ if (!ScalarTy->isIntegerTy()) {
+ return error(ValLoc, "storermw " +
+ AtomicRMWInst::getOperationName(Operation) +
+ " operand must be an integer");
+ }
+ }
+
+ unsigned Size =
+ PFS.getFunction().getDataLayout().getTypeStoreSizeInBits(Val->getType());
+ if (Size < 8 || (Size & (Size - 1)))
+ return error(ValLoc, "storermw operand must be power-of-two byte-sized"
+ " integer");
+ const Align DefaultAlignment(
+ PFS.getFunction().getDataLayout().getTypeStoreSize(Val->getType()));
+ StoreRMWInst *ARI = new StoreRMWInst(Operation, Ptr, Val,
+ Alignment.value_or(DefaultAlignment),
+ Ordering, SSID, IsElementwise);
+ ARI->setVolatile(isVolatile);
+ Inst = ARI;
+ return AteExtraComma ? InstExtraComma : InstNormal;
+}
+
/// parseFence
/// ::= 'fence' 'singlethread'? AtomicOrdering
int LLParser::parseFence(Instruction *&Inst, PerFunctionState &PFS) {
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index f6366061c9b52..97fafe5110ecf 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -6701,6 +6701,65 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
InstructionList.push_back(I);
break;
}
+ case bitc::FUNC_CODE_INST_STORERMW: {
+ // STORERMW: [ptrty, ptr, valty, val, op, vol, ordering, ssid, align?]
+ const size_t NumRecords = Record.size();
+ unsigned OpNum = 0;
+
+ Value *Ptr = nullptr;
+ unsigned PtrTypeID;
+ if (getValueTypePair(Record, OpNum, NextValueNo, Ptr, PtrTypeID, CurBB))
+ return error("Invalid storermw record");
+
+ if (!isa<PointerType>(Ptr->getType()))
+ return error("Invalid storermw record");
+
+ Value *Val = nullptr;
+ unsigned ValTypeID = InvalidTypeID;
+ if (getValueTypePair(Record, OpNum, NextValueNo, Val, ValTypeID, CurBB))
+ return error("Invalid storermw record");
+
+ if (!(NumRecords == (OpNum + 4) || NumRecords == (OpNum + 5)))
+ return error("Invalid storermw record");
+
+ bool IsElementwise = false;
+ const AtomicRMWInst::BinOp Operation =
+ getDecodedRMWOperation(Record[OpNum], IsElementwise);
+ if (Operation < AtomicRMWInst::FIRST_BINOP ||
+ Operation > AtomicRMWInst::LAST_BINOP)
+ return error("Invalid storermw record");
+
+ const bool IsVol = Record[OpNum + 1];
+
+ const AtomicOrdering Ordering = getDecodedOrdering(Record[OpNum + 2]);
+ if (Ordering == AtomicOrdering::NotAtomic ||
+ Ordering == AtomicOrdering::Unordered ||
+ Ordering == AtomicOrdering::Acquire ||
+ Ordering == AtomicOrdering::AcquireRelease)
+ return error("Invalid storermw record: must use store ordering (at "
+ "least monotonic)");
+
+ const SyncScope::ID SSID = getDecodedSyncScopeID(Record[OpNum + 3]);
+
+ MaybeAlign Alignment;
+
+ if (NumRecords == (OpNum + 5)) {
+ if (Error Err = parseAlignmentValue(Record[OpNum + 4], Alignment))
+ return Err;
+ }
+
+ if (!Alignment)
+ Alignment =
+ Align(TheModule->getDataLayout().getTypeStoreSize(Val->getType()));
+
+ I = new StoreRMWInst(Operation, Ptr, Val, *Alignment, Ordering, SSID,
+ IsElementwise);
+ ResTypeID = InvalidTypeID; // Returns void
+ cast<StoreRMWInst>(I)->setVolatile(IsVol);
+
+ InstructionList.push_back(I);
+ break;
+ }
case bitc::FUNC_CODE_INST_FENCE: { // FENCE:[ordering, ssid]
if (2 != Record.size())
return error("Invalid fence record");
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index c4058fe66ff0b..483782e20375d 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -690,9 +690,9 @@ static unsigned getEncodedBinaryOpcode(unsigned Opcode) {
}
}
-static unsigned getEncodedRMWOperation(const AtomicRMWInst &I) {
+static unsigned getEncodedRMWOperation(AtomicRMWInst::BinOp Op) {
unsigned Encoding = 0;
- switch (I.getOperation()) {
+ switch (Op) {
default: llvm_unreachable("Unknown RMW operation!");
case AtomicRMWInst::Xchg:
Encoding = bitc::RMW_XCHG;
@@ -765,8 +765,6 @@ static unsigned getEncodedRMWOperation(const AtomicRMWInst &I) {
break;
}
- if (I.isElementwise())
- Encoding |= bitc::RMW_ELEMENTWISE_FLAG;
return Encoding;
}
@@ -3603,17 +3601,38 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I,
Vals.push_back(cast<AtomicCmpXchgInst>(I).isWeak());
Vals.push_back(getEncodedAlign(cast<AtomicCmpXchgInst>(I).getAlign()));
break;
- case Instruction::AtomicRMW:
+ case Instruction::AtomicRMW: {
Code = bitc::FUNC_CODE_INST_ATOMICRMW;
pushValueAndType(I.getOperand(0), InstID, Vals); // ptrty + ptr
pushValueAndType(I.getOperand(1), InstID, Vals); // valty + val
- Vals.push_back(getEncodedRMWOperation(cast<AtomicRMWInst>(I)));
+ unsigned OpEnc =
+ getEncodedRMWOperation(cast<AtomicRMWInst>(I).getOperation());
+ if (cast<AtomicRMWInst>(I).isElementwise())
+ OpEnc |= bitc::RMW_ELEMENTWISE_FLAG;
+ Vals.push_back(OpEnc);
Vals.push_back(cast<AtomicRMWInst>(I).isVolatile());
Vals.push_back(getEncodedOrdering(cast<AtomicRMWInst>(I).getOrdering()));
Vals.push_back(
getEncodedSyncScopeID(cast<AtomicRMWInst>(I).getSyncScopeID()));
Vals.push_back(getEncodedAlign(cast<AtomicRMWInst>(I).getAlign()));
break;
+ }
+ case Instruction::StoreRMW: {
+ Code = bitc::FUNC_CODE_INST_STORERMW;
+ pushValueAndType(I.getOperand(0), InstID, Vals); // ptrty + ptr
+ pushValueAndType(I.getOperand(1), InstID, Vals); // valty + val
+ unsigned OpEnc =
+ getEncodedRMWOperation(cast<StoreRMWInst>(I).getOperation());
+ if (cast<StoreRMWInst>(I).isElementwise())
+ OpEnc |= bitc::RMW_ELEMENTWISE_FLAG;
+ Vals.push_back(OpEnc);
+ Vals.push_back(cast<StoreRMWInst>(I).isVolatile());
+ Vals.push_back(getEncodedOrdering(cast<StoreRMWInst>(I).getOrdering()));
+ Vals.push_back(
+ getEncodedSyncScopeID(cast<StoreRMWInst>(I).getSyncScopeID()));
+ Vals.push_back(getEncodedAlign(cast<StoreRMWInst>(I).getAlign()));
+ break;
+ }
case Instruction::Fence:
Code = bitc::FUNC_CODE_INST_FENCE;
Vals.push_back(getEncodedOrdering(cast<FenceInst>(I).getOrdering()));
diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index 059c0d5cb7b8b..5b898ec099ff7 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -145,6 +145,8 @@ class AtomicExpandImpl {
const Twine &AtomicOpName = "cmpxchg",
Instruction *DiagnosticInst = nullptr);
+ bool expandStoreRMWToRMW(StoreRMWInst *ARI);
+
bool expandAtomicRMWToCmpXchg(AtomicRMWInst *AI,
CreateCmpXchgInstFun CreateCmpXchg);
@@ -231,6 +233,11 @@ static unsigned getAtomicOpSize(AtomicCmpXchgInst *CASI) {
return DL.getTypeStoreSize(CASI->getCompareOperand()->getType());
}
+static unsigned getAtomicOpSize(StoreRMWInst *ARI) {
+ const DataLayout &DL = ARI->getDataLayout();
+ return DL.getTypeStoreSize(ARI->getValOperand()->getType());
+}
+
/// Copy metadata that's safe to preserve when widening atomics.
static void copyMetadataForAtomic(Instruction &Dest,
const Instruction &Source) {
@@ -335,6 +342,29 @@ bool AtomicExpandImpl::tryInsertFencesForAtomic(AtomicInst *AtomicI,
}
bool AtomicExpandImpl::processAtomicInstr(Instruction *I) {
+ // Check for atomic reduction instructions
+ if (auto *ARI = dyn_cast<StoreRMWInst>(I)) {
+ // Unsupported size (e.g. unaligned, or wider than target supports): route
+ // through the storermw -> atomicrmw conversion, which then takes the
+ // atomicrmw libcall path and emits the standard diagnostic on targets that
+ // do not provide atomic libcalls (e.g. NVPTX).
+ if (!atomicSizeSupported(TLI, ARI))
+ return expandStoreRMWToRMW(ARI);
+
+ // Ask the target how to handle this storermw.
+ auto Kind = TLI->shouldExpandStoreRMWInIR(ARI);
+ if (Kind == TargetLoweringBase::AtomicExpansionKind::None) {
+ // Target handles it natively; may still need fence splitting (e.g. NVPTX
+ // lowers seq_cst storermw to fence.sc + red.monotonic).
+ return tryInsertFencesForAtomic(
+ ARI, isReleaseOrStronger(ARI->getOrdering()),
+ TLI->atomicOperationOrderAfterFenceSplit(ARI));
+ }
+
+ // Target requested expansion -> atomicrmw with discarded result.
+ return expandStoreRMWToRMW(ARI);
+ }
+
if (auto *LI = dyn_cast<LoadInst>(I)) {
if (!LI->isAtomic())
return false;
@@ -1944,6 +1974,75 @@ void AtomicExpandImpl::expandAtomicCASToLibcall(AtomicCmpXchgInst *I,
handleUnsupportedAtomicSize(I, AtomicOpName, DiagnosticInst);
}
+/// Expand a `storermw` to an `atomicrmw` (with the result discarded) and then
+/// fully process that newly-created `atomicrmw` *inline* before returning.
+///
+/// We must process the new `atomicrmw` here rather than relying on the outer
+/// instruction loop in `processAtomicInstr` to revisit it. The outer loop
+/// uses `make_early_inc_range`, which pre-advances its iterator — instructions
+/// inserted at an earlier position are not visited again in the same pass run.
+/// Without inline processing:
+/// - unaligned (or oversized) atomicrmw would skip the
+/// `atomicSizeSupported` → `expandAtomicRMWToLibcall` route, reach
+/// instruction selection, and crash on the natural-alignment assertion
+/// in `insertRMWLLSCLoop`;
+/// - `tryInsertFencesForAtomic` / `shouldCastAtomicRMWIInIR` / idempotent
+/// simplification would all be skipped.
+/// So this function mirrors the `if (auto *RMWI = dyn_cast<AtomicRMWInst>(I))`
+/// branch of `processAtomicInstr`, applied to the newly created atomicrmw.
+bool AtomicExpandImpl::expandStoreRMWToRMW(StoreRMWInst *ARI) {
+ assert(ARI && "StoreRMWInst cannot be null");
+
+ // Extract all information from the StoreRMWInst
+ AtomicRMWInst::BinOp Op = ARI->getOperation();
+ Value *Ptr = ARI->getPointerOperand();
+ Value *Val = ARI->getValOperand();
+ Align Alignment = ARI->getAlign();
+ AtomicOrdering Ordering = ARI->getOrdering();
+ SyncScope::ID SSID = ARI->getSyncScopeID();
+ bool IsVolatile = ARI->isVolatile();
+
+ // Create the atomicrmw instruction (result discarded — storermw has none).
+ ReplacementIRBuilder Builder(ARI, *DL);
+ AtomicRMWInst *NewRMW =
+ Builder.CreateAtomicRMW(Op, Ptr, Val, Alignment, Ordering, SSID);
+ NewRMW->setVolatile(IsVolatile);
+ NewRMW->setElementwise(ARI->isElementwise());
+
+ // Remove the original storermw instruction.
+ ARI->eraseFromParent();
+
+ // Now fully process the newly created atomicrmw inline. See doc comment.
+
+ // 1. Unsupported size (e.g. unaligned, or wider than target supports) →
+ // route to libcall. This must happen here because the outer loop will
+ // not revisit this atomicrmw.
+ if (!atomicSizeSupported(TLI, NewRMW)) {
+ expandAtomicRMWToLibcall(NewRMW);
+ return true;
+ }
+
+ // 2. Cast to integer if the target requests it (e.g. FP xchg on targets
+ // that lack FP atomic xchg).
+ if (TLI->shouldCastAtomicRMWIInIR(NewRMW) ==
+ TargetLoweringBase::AtomicExpansionKind::CastToInteger) {
+ NewRMW = convertAtomicXchgToIntegerType(NewRMW);
+ }
+
+ // 3. Insert any fences the target needs around this atomic op.
+ tryInsertFencesForAtomic(NewRMW,
+ isReleaseOrStronger(NewRMW->getOrdering()) ||
+ isAcquireOrStronger(NewRMW->getOrdering()),
+ TLI->atomicOperationOrderAfterFenceSplit(NewRMW));
+
+ // 4. Try idempotent simplification first, then full RMW expansion
+ // (CmpXchg loop, LL/SC loop, partword widening, or target-specific).
+ (void)((isIdempotentRMW(NewRMW) && simplifyIdempotentRMW(NewRMW)) ||
+ tryExpandAtomicRMW(NewRMW));
+
+ return true;
+}
+
static ArrayRef<RTLIB::Libcall> GetRMWLibcall(AtomicRMWInst::BinOp Op) {
static const RTLIB::Libcall LibcallsXchg[6] = {
RTLIB::ATOMIC_EXCHANGE, RTLIB::ATOMIC_EXCHANGE_1,
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 65b8212755e6e..fd25b6f0d5c1b 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -1078,6 +1078,31 @@ void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
Action = TLI.getOperationAction(Node->getOpcode(),
Node->getOperand(1).getValueType());
break;
+ case ISD::ATOMIC_STORE_ADD:
+ case ISD::ATOMIC_STORE_SUB:
+ case ISD::ATOMIC_STORE_AND:
+ case ISD::ATOMIC_STORE_NAND:
+ case ISD::ATOMIC_STORE_OR:
+ case ISD::ATOMIC_STORE_XOR:
+ case ISD::ATOMIC_STORE_MIN:
+ case ISD::ATOMIC_STORE_MAX:
+ case ISD::ATOMIC_STORE_UMIN:
+ case ISD::ATOMIC_STORE_UMAX:
+ case ISD::ATOMIC_STORE_FADD:
+ case ISD::ATOMIC_STORE_FSUB:
+ case ISD::ATOMIC_STORE_FMIN:
+ case ISD::ATOMIC_STORE_FMAX:
+ case ISD::ATOMIC_STORE_FMINIMUM:
+ case ISD::ATOMIC_STORE_FMAXIMUM:
+ case ISD::ATOMIC_STORE_FMINIMUMNUM:
+ case ISD::ATOMIC_STORE_FMAXIMUMNUM:
+ case ISD::ATOMIC_STORE_UINC_WRAP:
+ case ISD::ATOMIC_STORE_UDEC_WRAP:
+ case ISD::ATOMIC_STORE_USUB_COND:
+ case ISD::ATOMIC_STORE_USUB_SAT:
+ Action = TLI.getOperationAction(Node->getOpcode(),
+ Node->getOperand(2).getValueType());
+ break;
case ISD::SELECT_CC:
case ISD::STRICT_FSETCC:
case ISD::STRICT_FSETCCS:
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index c542f8e7cc20b..16a7ed561ebc7 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -959,8 +959,42 @@ static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
case ISD::ATOMIC_LOAD_MAX:
case ISD::ATOMIC_LOAD_UMIN:
case ISD::ATOMIC_LOAD_UMAX:
+ case ISD::ATOMIC_LOAD_FADD:
+ case ISD::ATOMIC_LOAD_FSUB:
+ case ISD::ATOMIC_LOAD_FMAX:
+ case ISD::ATOMIC_LOAD_FMIN:
+ case ISD::ATOMIC_LOAD_FMAXIMUM:
+ case ISD::ATOMIC_LOAD_FMINIMUM:
+ case ISD::ATOMIC_LOAD_FMAXIMUMNUM:
+ case ISD::ATOMIC_LOAD_FMINIMUMNUM:
+ case ISD::ATOMIC_LOAD_UINC_WRAP:
+ case ISD::ATOMIC_LOAD_UDEC_WRAP:
+ case ISD::ATOMIC_LOAD_USUB_COND:
+ case ISD::ATOMIC_LOAD_USUB_SAT:
case ISD::ATOMIC_LOAD:
- case ISD::ATOMIC_STORE: {
+ case ISD::ATOMIC_STORE:
+ case ISD::ATOMIC_STORE_ADD:
+ case ISD::ATOMIC_STORE_SUB:
+ case ISD::ATOMIC_STORE_AND:
+ case ISD::ATOMIC_STORE_NAND:
+ case ISD::ATOMIC_STORE_OR:
+ case ISD::ATOMIC_STORE_XOR:
+ case ISD::ATOMIC_STORE_MIN:
+ case ISD::ATOMIC_STORE_MAX:
+ case ISD::ATOMIC_STORE_UMIN:
+ case ISD::ATOMIC_STORE_UMAX:
+ case ISD::ATOMIC_STORE_FADD:
+ case ISD::ATOMIC_STORE_FSUB:
+ case ISD::ATOMIC_STORE_FMIN:
+ case ISD::ATOMIC_STORE_FMAX:
+ case ISD::ATOMIC_STORE_FMINIMUM:
+ case ISD::ATOMIC_STORE_FMAXIMUM:
+ case ISD::ATOMIC_STORE_FMINIMUMNUM:
+ case ISD::ATOMIC_STORE_FMAXIMUMNUM:
+ case ISD::ATOMIC_STORE_UINC_WRAP:
+ case ISD::ATOMIC_STORE_UDEC_WRAP:
+ case ISD::ATOMIC_STORE_USUB_COND:
+ case ISD::ATOMIC_STORE_USUB_SAT: {
const AtomicSDNode *AT = cast<AtomicSDNode>(N);
ID.AddInteger(AT->getMemoryVT().getRawBits());
ID.AddInteger(AT->getRawSubclassData());
@@ -10349,27 +10383,61 @@ SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl,
SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
SDValue Chain, SDValue Ptr, SDValue Val,
MachineMemOperand *MMO) {
- assert((Opcode == ISD::ATOMIC_LOAD_ADD || Opcode == ISD::ATOMIC_LOAD_SUB ||
- Opcode == ISD::ATOMIC_LOAD_AND || Opcode == ISD::ATOMIC_LOAD_CLR ||
- Opcode == ISD::ATOMIC_LOAD_OR || Opcode == ISD::ATOMIC_LOAD_XOR ||
- Opcode == ISD::ATOMIC_LOAD_NAND || Opcode == ISD::ATOMIC_LOAD_MIN ||
- Opcode == ISD::ATOMIC_LOAD_MAX || Opcode == ISD::ATOMIC_LOAD_UMIN ||
- Opcode == ISD::ATOMIC_LOAD_UMAX || Opcode == ISD::ATOMIC_LOAD_FADD ||
- Opcode == ISD::ATOMIC_LOAD_FSUB || Opcode == ISD::ATOMIC_LOAD_FMAX ||
- Opcode == ISD::ATOMIC_LOAD_FMIN ||
- Opcode == ISD::ATOMIC_LOAD_FMINIMUM ||
- Opcode == ISD::ATOMIC_LOAD_FMAXIMUM ||
- Opcode == ISD::ATOMIC_LOAD_UINC_WRAP ||
- Opcode == ISD::ATOMIC_LOAD_UDEC_WRAP ||
- Opcode == ISD::ATOMIC_LOAD_USUB_COND ||
- Opcode == ISD::ATOMIC_LOAD_USUB_SAT || Opcode == ISD::ATOMIC_SWAP ||
- Opcode == ISD::ATOMIC_STORE) &&
- "Invalid Atomic Op");
+ assert(
+ (Opcode == ISD::ATOMIC_LOAD_ADD || Opcode == ISD::ATOMIC_LOAD_SUB ||
+ Opcode == ISD::ATOMIC_LOAD_AND || Opcode == ISD::ATOMIC_LOAD_CLR ||
+ Opcode == ISD::ATOMIC_LOAD_OR || Opcode == ISD::ATOMIC_LOAD_XOR ||
+ Opcode == ISD::ATOMIC_LOAD_NAND || Opcode == ISD::ATOMIC_LOAD_MIN ||
+ Opcode == ISD::ATOMIC_LOAD_MAX || Opcode == ISD::ATOMIC_LOAD_UMIN ||
+ Opcode == ISD::ATOMIC_LOAD_UMAX || Opcode == ISD::ATOMIC_LOAD_FADD ||
+ Opcode == ISD::ATOMIC_LOAD_FSUB || Opcode == ISD::ATOMIC_LOAD_FMAX ||
+ Opcode == ISD::ATOMIC_LOAD_FMIN || Opcode == ISD::ATOMIC_LOAD_FMINIMUM ||
+ Opcode == ISD::ATOMIC_LOAD_FMAXIMUM ||
+ Opcode == ISD::ATOMIC_LOAD_UINC_WRAP ||
+ Opcode == ISD::ATOMIC_LOAD_UDEC_WRAP ||
+ Opcode == ISD::ATOMIC_LOAD_USUB_COND ||
+ Opcode == ISD::ATOMIC_LOAD_USUB_SAT || Opcode == ISD::ATOMIC_SWAP ||
+ Opcode == ISD::ATOMIC_STORE || Opcode == ISD::ATOMIC_STORE_ADD ||
+ Opcode == ISD::ATOMIC_STORE_SUB || Opcode == ISD::ATOMIC_STORE_AND ||
+ Opcode == ISD::ATOMIC_STORE_NAND || Opcode == ISD::ATOMIC_STORE_OR ||
+ Opcode == ISD::ATOMIC_STORE_XOR || Opcode == ISD::ATOMIC_STORE_MIN ||
+ Opcode == ISD::ATOMIC_STORE_MAX || Opcode == ISD::ATOMIC_STORE_UMIN ||
+ Opcode == ISD::ATOMIC_STORE_UMAX || Opcode == ISD::ATOMIC_STORE_FADD ||
+ Opcode == ISD::ATOMIC_STORE_FSUB || Opcode == ISD::ATOMIC_STORE_FMIN ||
+ Opcode == ISD::ATOMIC_STORE_FMAX ||
+ Opcode == ISD::ATOMIC_STORE_FMINIMUM ||
+ Opcode == ISD::ATOMIC_STORE_FMAXIMUM ||
+ Opcode == ISD::ATOMIC_STORE_FMINIMUMNUM ||
+ Opcode == ISD::ATOMIC_STORE_FMAXIMUMNUM ||
+ Opcode == ISD::ATOMIC_STORE_UINC_WRAP ||
+ Opcode == ISD::ATOMIC_STORE_UDEC_WRAP ||
+ Opcode == ISD::ATOMIC_STORE_USUB_COND ||
+ Opcode == ISD::ATOMIC_STORE_USUB_SAT) &&
+ "Invalid Atomic Op");
EVT VT = Val.getValueType();
- SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) :
- getVTList(VT, MVT::Other);
+ // ATOMIC_STORE and ATOMIC_STORE_* (storermw) return only a chain; all
+ // other atomic RMW ops return the old value plus a chain.
+ bool IsStoreOnly =
+ Opcode == ISD::ATOMIC_STORE || Opcode == ISD::ATOMIC_STORE_ADD ||
+ Opcode == ISD::ATOMIC_STORE_SUB || Opcode == ISD::ATOMIC_STORE_AND ||
+ Opcode == ISD::ATOMIC_STORE_NAND || Opcode == ISD::ATOMIC_STORE_OR ||
+ Opcode == ISD::ATOMIC_STORE_XOR || Opcode == ISD::ATOMIC_STORE_MIN ||
+ Opcode == ISD::ATOMIC_STORE_MAX || Opcode == ISD::ATOMIC_STORE_UMIN ||
+ Opcode == ISD::ATOMIC_STORE_UMAX || Opcode == ISD::ATOMIC_STORE_FADD ||
+ Opcode == ISD::ATOMIC_STORE_FSUB || Opcode == ISD::ATOMIC_STORE_FMIN ||
+ Opcode == ISD::ATOMIC_STORE_FMAX ||
+ Opcode == ISD::ATOMIC_STORE_FMINIMUM ||
+ Opcode == ISD::ATOMIC_STORE_FMAXIMUM ||
+ Opcode == ISD::ATOMIC_STORE_FMINIMUMNUM ||
+ Opcode == ISD::ATOMIC_STORE_FMAXIMUMNUM ||
+ Opcode == ISD::ATOMIC_STORE_UINC_WRAP ||
+ Opcode == ISD::ATOMIC_STORE_UDEC_WRAP ||
+ Opcode == ISD::ATOMIC_STORE_USUB_COND ||
+ Opcode == ISD::ATOMIC_STORE_USUB_SAT;
+ SDVTList VTs =
+ IsStoreOnly ? getVTList(MVT::Other) : getVTList(VT, MVT::Other);
SDValue Ops[] = {Chain, Ptr, Val};
return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 8aa184423c60c..53eee067ec2d4 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -5308,6 +5308,99 @@ void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
DAG.setRoot(OutChain);
}
+void SelectionDAGBuilder::visitStoreRMW(const StoreRMWInst &I) {
+ SDLoc dl = getCurSDLoc();
+ ISD::NodeType NT;
+ switch (I.getOperation()) {
+ default:
+ llvm_unreachable("Unknown storermw operation");
+ case AtomicRMWInst::Add:
+ NT = ISD::ATOMIC_STORE_ADD;
+ break;
+ case AtomicRMWInst::Sub:
+ NT = ISD::ATOMIC_STORE_SUB;
+ break;
+ case AtomicRMWInst::And:
+ NT = ISD::ATOMIC_STORE_AND;
+ break;
+ case AtomicRMWInst::Nand:
+ NT = ISD::ATOMIC_STORE_NAND;
+ break;
+ case AtomicRMWInst::Or:
+ NT = ISD::ATOMIC_STORE_OR;
+ break;
+ case AtomicRMWInst::Xor:
+ NT = ISD::ATOMIC_STORE_XOR;
+ break;
+ case AtomicRMWInst::Max:
+ NT = ISD::ATOMIC_STORE_MAX;
+ break;
+ case AtomicRMWInst::Min:
+ NT = ISD::ATOMIC_STORE_MIN;
+ break;
+ case AtomicRMWInst::UMax:
+ NT = ISD::ATOMIC_STORE_UMAX;
+ break;
+ case AtomicRMWInst::UMin:
+ NT = ISD::ATOMIC_STORE_UMIN;
+ break;
+ case AtomicRMWInst::FAdd:
+ NT = ISD::ATOMIC_STORE_FADD;
+ break;
+ case AtomicRMWInst::FSub:
+ NT = ISD::ATOMIC_STORE_FSUB;
+ break;
+ case AtomicRMWInst::FMax:
+ NT = ISD::ATOMIC_STORE_FMAX;
+ break;
+ case AtomicRMWInst::FMin:
+ NT = ISD::ATOMIC_STORE_FMIN;
+ break;
+ case AtomicRMWInst::FMaximum:
+ NT = ISD::ATOMIC_STORE_FMAXIMUM;
+ break;
+ case AtomicRMWInst::FMinimum:
+ NT = ISD::ATOMIC_STORE_FMINIMUM;
+ break;
+ case AtomicRMWInst::FMaximumNum:
+ NT = ISD::ATOMIC_STORE_FMAXIMUMNUM;
+ break;
+ case AtomicRMWInst::FMinimumNum:
+ NT = ISD::ATOMIC_STORE_FMINIMUMNUM;
+ break;
+ case AtomicRMWInst::UIncWrap:
+ NT = ISD::ATOMIC_STORE_UINC_WRAP;
+ break;
+ case AtomicRMWInst::UDecWrap:
+ NT = ISD::ATOMIC_STORE_UDEC_WRAP;
+ break;
+ case AtomicRMWInst::USubCond:
+ NT = ISD::ATOMIC_STORE_USUB_COND;
+ break;
+ case AtomicRMWInst::USubSat:
+ NT = ISD::ATOMIC_STORE_USUB_SAT;
+ break;
+ }
+ AtomicOrdering Ordering = I.getOrdering();
+ SyncScope::ID SSID = I.getSyncScopeID();
+
+ SDValue InChain = getRoot();
+
+ auto MemVT = getValue(I.getValOperand()).getSimpleValueType();
+ const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+ auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout());
+
+ MachineFunction &MF = DAG.getMachineFunction();
+ MachineMemOperand *MMO = MF.getMachineMemOperand(
+ MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
+ I.getAlign(), AAMDNodes(), nullptr, SSID, Ordering);
+
+ SDValue L =
+ DAG.getAtomic(NT, dl, MemVT, InChain, getValue(I.getPointerOperand()),
+ getValue(I.getValOperand()), MMO);
+ DAG.setRoot(L);
+}
+
void SelectionDAGBuilder::visitFence(const FenceInst &I) {
SDLoc dl = getCurSDLoc();
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
index 21aac333a73cd..29c8f7121e6e6 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
@@ -43,6 +43,7 @@ namespace llvm {
class AAResults;
class AllocaInst;
class AtomicCmpXchgInst;
+class StoreRMWInst;
class AtomicRMWInst;
class AssumptionCache;
class BasicBlock;
@@ -624,6 +625,7 @@ class SelectionDAGBuilder {
void visitMaskedScatter(const CallInst &I);
void visitAtomicCmpXchg(const AtomicCmpXchgInst &I);
void visitAtomicRMW(const AtomicRMWInst &I);
+ void visitStoreRMW(const StoreRMWInst &I);
void visitFence(const FenceInst &I);
void visitPHI(const PHINode &I);
void visitCall(const CallInst &I);
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
index a00d7555f15f6..94dbbbf6e95d7 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
@@ -109,6 +109,8 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
case ISD::ATOMIC_LOAD_FMAX: return "AtomicLoadFMax";
case ISD::ATOMIC_LOAD_FMINIMUM: return "AtomicLoadFMinimum";
case ISD::ATOMIC_LOAD_FMAXIMUM: return "AtomicLoadFMaximum";
+ case ISD::ATOMIC_LOAD_FMINIMUMNUM: return "AtomicLoadFMinimumNum";
+ case ISD::ATOMIC_LOAD_FMAXIMUMNUM: return "AtomicLoadFMaximumNum";
case ISD::ATOMIC_LOAD_UINC_WRAP:
return "AtomicLoadUIncWrap";
case ISD::ATOMIC_LOAD_UDEC_WRAP:
@@ -119,6 +121,28 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
return "AtomicLoadUSubSat";
case ISD::ATOMIC_LOAD: return "AtomicLoad";
case ISD::ATOMIC_STORE: return "AtomicStore";
+ case ISD::ATOMIC_STORE_ADD: return "AtomicStoreAdd";
+ case ISD::ATOMIC_STORE_SUB: return "AtomicStoreSub";
+ case ISD::ATOMIC_STORE_AND: return "AtomicStoreAnd";
+ case ISD::ATOMIC_STORE_NAND: return "AtomicStoreNand";
+ case ISD::ATOMIC_STORE_OR: return "AtomicStoreOr";
+ case ISD::ATOMIC_STORE_XOR: return "AtomicStoreXor";
+ case ISD::ATOMIC_STORE_MIN: return "AtomicStoreMin";
+ case ISD::ATOMIC_STORE_MAX: return "AtomicStoreMax";
+ case ISD::ATOMIC_STORE_UMIN: return "AtomicStoreUMin";
+ case ISD::ATOMIC_STORE_UMAX: return "AtomicStoreUMax";
+ case ISD::ATOMIC_STORE_FADD: return "AtomicStoreFAdd";
+ case ISD::ATOMIC_STORE_FSUB: return "AtomicStoreFSub";
+ case ISD::ATOMIC_STORE_FMIN: return "AtomicStoreFMin";
+ case ISD::ATOMIC_STORE_FMAX: return "AtomicStoreFMax";
+ case ISD::ATOMIC_STORE_FMINIMUM: return "AtomicStoreFMinimum";
+ case ISD::ATOMIC_STORE_FMAXIMUM: return "AtomicStoreFMaximum";
+ case ISD::ATOMIC_STORE_FMINIMUMNUM: return "AtomicStoreFMinimumNum";
+ case ISD::ATOMIC_STORE_FMAXIMUMNUM: return "AtomicStoreFMaximumNum";
+ case ISD::ATOMIC_STORE_UINC_WRAP: return "AtomicStoreUIncWrap";
+ case ISD::ATOMIC_STORE_UDEC_WRAP: return "AtomicStoreUDecWrap";
+ case ISD::ATOMIC_STORE_USUB_COND: return "AtomicStoreUSubCond";
+ case ISD::ATOMIC_STORE_USUB_SAT: return "AtomicStoreUSubSat";
case ISD::PCMARKER: return "PCMarker";
case ISD::READCYCLECOUNTER: return "ReadCycleCounter";
case ISD::READSTEADYCOUNTER: return "ReadSteadyCounter";
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index 3330f43c38178..c80043b59ce56 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -2276,6 +2276,8 @@ int TargetLoweringBase::InstructionOpcodeToISD(unsigned Opcode) const {
case Fence: return 0;
case AtomicCmpXchg: return 0;
case AtomicRMW: return 0;
+ case StoreRMW:
+ return 0;
case Trunc: return ISD::TRUNCATE;
case ZExt: return ISD::ZERO_EXTEND;
case SExt: return ISD::SIGN_EXTEND;
@@ -2843,6 +2845,15 @@ TargetLoweringBase::getAtomicMemOperandFlags(const Instruction &AI,
} else if (const AtomicCmpXchgInst *CmpX = dyn_cast<AtomicCmpXchgInst>(&AI)) {
if (CmpX->isVolatile())
Flags |= MachineMemOperand::MOVolatile;
+ } else if (const StoreRMWInst *ARI = dyn_cast<StoreRMWInst>(&AI)) {
+ // For optimization purposes we model StoreRMW as a pure write (MOStore
+ // only, no MOLoad). The instruction does perform an atomic read-modify-
+ // write, but the loaded value is discarded and the operation does not
+ // participate in acquire synchronization, so alias analysis and passes
+ // that consume MMO flags can treat it as if it were a plain store.
+ Flags = MachineMemOperand::MOStore;
+ if (ARI->isVolatile())
+ Flags |= MachineMemOperand::MOVolatile;
} else
llvm_unreachable("not an atomic instruction");
diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp
index 52ed28f71f615..057426410104b 100644
--- a/llvm/lib/IR/AsmWriter.cpp
+++ b/llvm/lib/IR/AsmWriter.cpp
@@ -4447,10 +4447,11 @@ void AssemblyWriter::printInstruction(const Instruction &I) {
Out << " weak";
// If this is a volatile operation, print out the volatile marker.
- if ((isa<LoadInst>(I) && cast<LoadInst>(I).isVolatile()) ||
+ if ((isa<LoadInst>(I) && cast<LoadInst>(I).isVolatile()) ||
(isa<StoreInst>(I) && cast<StoreInst>(I).isVolatile()) ||
(isa<AtomicCmpXchgInst>(I) && cast<AtomicCmpXchgInst>(I).isVolatile()) ||
- (isa<AtomicRMWInst>(I) && cast<AtomicRMWInst>(I).isVolatile()))
+ (isa<AtomicRMWInst>(I) && cast<AtomicRMWInst>(I).isVolatile()) ||
+ (isa<StoreRMWInst>(I) && cast<StoreRMWInst>(I).isVolatile()))
Out << " volatile";
// Print out optimization information.
@@ -4467,6 +4468,13 @@ void AssemblyWriter::printInstruction(const Instruction &I) {
Out << ' ' << AtomicRMWInst::getOperationName(RMWI->getOperation());
}
+ // Print out the storermw operation
+ if (const auto *ARI = dyn_cast<StoreRMWInst>(&I)) {
+ if (ARI->isElementwise())
+ Out << " elementwise";
+ Out << ' ' << StoreRMWInst::getOperationName(ARI->getOperation());
+ }
+
// Print out the type of the operands...
const Value *Operand = I.getNumOperands() ? I.getOperand(0) : nullptr;
@@ -4804,11 +4812,11 @@ void AssemblyWriter::printInstruction(const Instruction &I) {
bool PrintAllTypes = false;
Type *TheType = Operand->getType();
- // Select, Store, ShuffleVector, CmpXchg and AtomicRMW always print all
- // types.
+ // Select, Store, ShuffleVector, CmpXchg, AtomicRMW and StoreRMW always
+ // print all types.
if (isa<SelectInst>(I) || isa<StoreInst>(I) || isa<ShuffleVectorInst>(I) ||
isa<ReturnInst>(I) || isa<AtomicCmpXchgInst>(I) ||
- isa<AtomicRMWInst>(I)) {
+ isa<AtomicRMWInst>(I) || isa<StoreRMWInst>(I)) {
PrintAllTypes = true;
} else {
for (unsigned i = 1, E = I.getNumOperands(); i != E; ++i) {
@@ -4854,6 +4862,9 @@ void AssemblyWriter::printInstruction(const Instruction &I) {
writeAtomic(RMWI->getContext(), RMWI->getOrdering(),
RMWI->getSyncScopeID());
Out << ", align " << RMWI->getAlign().value();
+ } else if (const auto *ARI = dyn_cast<StoreRMWInst>(&I)) {
+ writeAtomic(ARI->getContext(), ARI->getOrdering(), ARI->getSyncScopeID());
+ Out << ", align " << ARI->getAlign().value();
} else if (const auto *FI = dyn_cast<FenceInst>(&I)) {
writeAtomic(FI->getContext(), FI->getOrdering(), FI->getSyncScopeID());
} else if (const auto *SVI = dyn_cast<ShuffleVectorInst>(&I)) {
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index 3a823f906b012..af14d5125bd17 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -6522,6 +6522,16 @@ struct AMDGPUUnsafeFPAtomicsUpgradeVisitor
RMW.setMetadata("amdgpu.no.remote.memory.access", Empty);
RMW.setMetadata("amdgpu.ignore.denormal.mode", Empty);
}
+
+ void visitStoreRMWInst(StoreRMWInst &ARI) {
+ if (!ARI.isFloatingPointOperation())
+ return;
+
+ MDNode *Empty = MDNode::get(ARI.getContext(), {});
+ ARI.setMetadata("amdgpu.no.fine.grained.host.memory", Empty);
+ ARI.setMetadata("amdgpu.no.remote.memory.access", Empty);
+ ARI.setMetadata("amdgpu.ignore.denormal.mode", Empty);
+ }
};
} // namespace
diff --git a/llvm/lib/IR/Instruction.cpp b/llvm/lib/IR/Instruction.cpp
index 4e43de1f08fd0..bef25a79bb8ab 100644
--- a/llvm/lib/IR/Instruction.cpp
+++ b/llvm/lib/IR/Instruction.cpp
@@ -860,6 +860,8 @@ const char *Instruction::getOpcodeName(unsigned OpCode) {
case Store: return "store";
case AtomicCmpXchg: return "cmpxchg";
case AtomicRMW: return "atomicrmw";
+ case StoreRMW:
+ return "storermw";
case Fence: return "fence";
case GetElementPtr: return "getelementptr";
@@ -983,6 +985,14 @@ bool Instruction::hasSameSpecialState(const Instruction *I2,
IgnoreAlignment) &&
RMWI->getOrdering() == cast<AtomicRMWInst>(I2)->getOrdering() &&
RMWI->getSyncScopeID() == cast<AtomicRMWInst>(I2)->getSyncScopeID();
+ if (const StoreRMWInst *ARI = dyn_cast<StoreRMWInst>(I1))
+ return ARI->getOperation() == cast<StoreRMWInst>(I2)->getOperation() &&
+ ARI->isElementwise() == cast<StoreRMWInst>(I2)->isElementwise() &&
+ ARI->isVolatile() == cast<StoreRMWInst>(I2)->isVolatile() &&
+ (ARI->getAlign() == cast<StoreRMWInst>(I2)->getAlign() ||
+ IgnoreAlignment) &&
+ ARI->getOrdering() == cast<StoreRMWInst>(I2)->getOrdering() &&
+ ARI->getSyncScopeID() == cast<StoreRMWInst>(I2)->getSyncScopeID();
if (const ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(I1))
return SVI->getShuffleMask() ==
cast<ShuffleVectorInst>(I2)->getShuffleMask();
@@ -1109,6 +1119,10 @@ MemoryEffects Instruction::getMemoryEffects() const {
return GetEffects(ModRefInfo::ModRef, RMW->getOrdering(),
RMW->isVolatile());
}
+ case Instruction::StoreRMW: {
+ auto *AR = cast<StoreRMWInst>(this);
+ return GetEffects(ModRefInfo::Mod, AR->getOrdering(), AR->isVolatile());
+ }
case Instruction::AtomicCmpXchg: {
auto *CX = cast<AtomicCmpXchgInst>(this);
return GetEffects(ModRefInfo::ModRef, CX->getSuccessOrdering(),
@@ -1149,6 +1163,7 @@ bool Instruction::mayWriteToMemory() const {
case Instruction::VAArg:
case Instruction::AtomicCmpXchg:
case Instruction::AtomicRMW:
+ case Instruction::StoreRMW:
case Instruction::CatchPad:
case Instruction::CatchRet:
return true;
@@ -1167,6 +1182,7 @@ bool Instruction::isAtomic() const {
return false;
case Instruction::AtomicCmpXchg:
case Instruction::AtomicRMW:
+ case Instruction::StoreRMW:
case Instruction::Fence:
return true;
case Instruction::Load:
@@ -1206,6 +1222,8 @@ bool Instruction::isVolatile() const {
return false;
case Instruction::AtomicRMW:
return cast<AtomicRMWInst>(this)->isVolatile();
+ case Instruction::StoreRMW:
+ return cast<StoreRMWInst>(this)->isVolatile();
case Instruction::Store:
return cast<StoreInst>(this)->isVolatile();
case Instruction::Load:
@@ -1244,6 +1262,7 @@ bool Instruction::maySynchronize() const {
return FI->getSyncScopeID() != SyncScope::SingleThread;
}
case Instruction::AtomicRMW:
+ case Instruction::StoreRMW:
case Instruction::AtomicCmpXchg:
return true;
case Instruction::Store:
@@ -1261,6 +1280,8 @@ Type *Instruction::getAccessType() const {
switch (getOpcode()) {
case Instruction::Store:
return cast<StoreInst>(this)->getValueOperand()->getType();
+ case Instruction::StoreRMW:
+ return cast<StoreRMWInst>(this)->getValOperand()->getType();
case Instruction::Load:
case Instruction::AtomicRMW:
return getType();
diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index faa046729aff6..d5b4f71368caf 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -1538,6 +1538,40 @@ StringRef AtomicRMWInst::getOperationName(BinOp Op) {
llvm_unreachable("invalid atomicrmw operation");
}
+//===----------------------------------------------------------------------===//
+// StoreRMWInst Implementation
+//===----------------------------------------------------------------------===//
+
+void StoreRMWInst::Init(BinOp Operation, Value *Ptr, Value *Val,
+ Align Alignment, AtomicOrdering Ordering,
+ SyncScope::ID SSID, bool Elementwise) {
+ assert(Ordering != AtomicOrdering::NotAtomic &&
+ "storermw instructions must be atomic.");
+ assert(Ordering != AtomicOrdering::Acquire &&
+ Ordering != AtomicOrdering::AcquireRelease &&
+ "storermw instructions cannot have acquire semantics.");
+ Op<0>() = Ptr;
+ Op<1>() = Val;
+ setOperation(Operation);
+ setOrdering(Ordering);
+ setSyncScopeID(SSID);
+ setAlignment(Alignment);
+ setElementwise(Elementwise);
+
+ assert(getOperand(0) && getOperand(1) && "All operands must be non-null!");
+ assert(getOperand(0)->getType()->isPointerTy() &&
+ "Ptr must have pointer type!");
+}
+
+StoreRMWInst::StoreRMWInst(BinOp Operation, Value *Ptr, Value *Val,
+ Align Alignment, AtomicOrdering Ordering,
+ SyncScope::ID SSID, bool Elementwise,
+ InsertPosition InsertBefore)
+ : Instruction(Type::getVoidTy(Val->getContext()), StoreRMW, AllocMarker,
+ InsertBefore) {
+ Init(Operation, Ptr, Val, Alignment, Ordering, SSID, Elementwise);
+}
+
//===----------------------------------------------------------------------===//
// FenceInst Implementation
//===----------------------------------------------------------------------===//
@@ -4467,6 +4501,14 @@ AtomicRMWInst *AtomicRMWInst::cloneImpl() const {
return Result;
}
+StoreRMWInst *StoreRMWInst::cloneImpl() const {
+ StoreRMWInst *Result =
+ new StoreRMWInst(getOperation(), getOperand(0), getOperand(1), getAlign(),
+ getOrdering(), getSyncScopeID(), isElementwise());
+ Result->setVolatile(isVolatile());
+ return Result;
+}
+
FenceInst *FenceInst::cloneImpl() const {
return new FenceInst(getContext(), getOrdering(), getSyncScopeID());
}
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 648446555793b..6b612b7dc6411 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -437,6 +437,7 @@ class Verifier : public InstVisitor<Verifier>, VerifierSupport {
void visitDbgLabelIntrinsic(StringRef Kind, DbgLabelInst &DLI);
void visitAtomicCmpXchgInst(AtomicCmpXchgInst &CXI);
void visitAtomicRMWInst(AtomicRMWInst &RMWI);
+ void visitStoreRMWInst(StoreRMWInst &ARI);
void visitFenceInst(FenceInst &FI);
void visitAllocaInst(AllocaInst &AI);
void visitExtractValueInst(ExtractValueInst &EVI);
@@ -4688,6 +4689,59 @@ void Verifier::visitAtomicRMWInst(AtomicRMWInst &RMWI) {
visitInstruction(RMWI);
}
+void Verifier::visitStoreRMWInst(StoreRMWInst &ARI) {
+ AtomicOrdering Ordering = ARI.getOrdering();
+
+ // Verify store-only orderings (cannot have acquire semantics, must be at
+ // least monotonic).
+ Check(Ordering != AtomicOrdering::Acquire &&
+ Ordering != AtomicOrdering::AcquireRelease,
+ "storermw instructions cannot have acquire semantics "
+ "(only store orderings: monotonic, release, seq_cst).",
+ &ARI);
+
+ Check(Ordering != AtomicOrdering::NotAtomic &&
+ Ordering != AtomicOrdering::Unordered,
+ "storermw instructions must be at least monotonic.", &ARI);
+
+ auto Op = ARI.getOperation();
+ Type *ElTy = ARI.getValOperand()->getType();
+ Type *ScalarTy = ElTy;
+ if (ARI.isElementwise()) {
+ auto *VecTy = dyn_cast<FixedVectorType>(ElTy);
+ Check(VecTy, "storermw elementwise operand must have fixed vector type!",
+ &ARI, ElTy);
+ if (VecTy)
+ ScalarTy = VecTy->getElementType();
+ }
+
+ // Verify operation is valid (only xchg not allowed - requires return value)
+ Check(Op != AtomicRMWInst::Xchg,
+ "xchg operation not valid for storermw (use atomicrmw instead).", &ARI);
+ // Note: nand, uinc_wrap, udec_wrap, usub_cond, usub_sat are now allowed
+ // They will expand to atomicrmw on targets that don't support them natively
+
+ // Verify operand types
+ if (AtomicRMWInst::isFPOperation(Op)) {
+ Check(ElTy->isFPOrFPVectorTy() && !isa<ScalableVectorType>(ElTy),
+ "storermw " + AtomicRMWInst::getOperationName(Op) +
+ " operand must have floating-point or fixed vector of "
+ "floating-point "
+ "type!",
+ &ARI, ElTy);
+ } else {
+ Check(ScalarTy->isIntegerTy(),
+ "storermw " + AtomicRMWInst::getOperationName(Op) +
+ " operand must have integer type!",
+ &ARI, ElTy);
+ }
+
+ checkAtomicMemAccessSize(ElTy, &ARI);
+ Check(AtomicRMWInst::FIRST_BINOP <= Op && Op <= AtomicRMWInst::LAST_BINOP,
+ "Invalid binary operation!", &ARI);
+ visitInstruction(ARI);
+}
+
void Verifier::visitFenceInst(FenceInst &FI) {
const AtomicOrdering Ordering = FI.getOrdering();
Check(Ordering == AtomicOrdering::Acquire ||
diff --git a/llvm/test/Analysis/IR2Vec/Inputs/reference_default_vocab_print.txt b/llvm/test/Analysis/IR2Vec/Inputs/reference_default_vocab_print.txt
index cc36415abb00b..f6bb01987352a 100644
--- a/llvm/test/Analysis/IR2Vec/Inputs/reference_default_vocab_print.txt
+++ b/llvm/test/Analysis/IR2Vec/Inputs/reference_default_vocab_print.txt
@@ -67,6 +67,7 @@ Key: ExtractValue: [ 127.00 128.00 ]
Key: InsertValue: [ 129.00 130.00 ]
Key: LandingPad: [ 131.00 132.00 ]
Key: Freeze: [ 133.00 134.00 ]
+Key: StoreRMW: [ 0.00 0.00 ]
Key: FloatTy: [ 0.50 1.00 ]
Key: VoidTy: [ 1.50 2.00 ]
Key: LabelTy: [ 2.50 3.00 ]
diff --git a/llvm/test/Analysis/IR2Vec/Inputs/reference_wtd1_vocab_print.txt b/llvm/test/Analysis/IR2Vec/Inputs/reference_wtd1_vocab_print.txt
index 8d837e9502739..2c440c7ef354d 100644
--- a/llvm/test/Analysis/IR2Vec/Inputs/reference_wtd1_vocab_print.txt
+++ b/llvm/test/Analysis/IR2Vec/Inputs/reference_wtd1_vocab_print.txt
@@ -67,6 +67,7 @@ Key: ExtractValue: [ 63.50 64.00 ]
Key: InsertValue: [ 64.50 65.00 ]
Key: LandingPad: [ 65.50 66.00 ]
Key: Freeze: [ 66.50 67.00 ]
+Key: StoreRMW: [ 0.00 0.00 ]
Key: FloatTy: [ 0.50 1.00 ]
Key: VoidTy: [ 1.50 2.00 ]
Key: LabelTy: [ 2.50 3.00 ]
diff --git a/llvm/test/Analysis/IR2Vec/Inputs/reference_wtd2_vocab_print.txt b/llvm/test/Analysis/IR2Vec/Inputs/reference_wtd2_vocab_print.txt
index 3d31b9d9db315..7e4fd01da716b 100644
--- a/llvm/test/Analysis/IR2Vec/Inputs/reference_wtd2_vocab_print.txt
+++ b/llvm/test/Analysis/IR2Vec/Inputs/reference_wtd2_vocab_print.txt
@@ -67,6 +67,7 @@ Key: ExtractValue: [ 12.70 12.80 ]
Key: InsertValue: [ 12.90 13.00 ]
Key: LandingPad: [ 13.10 13.20 ]
Key: Freeze: [ 13.30 13.40 ]
+Key: StoreRMW: [ 0.00 0.00 ]
Key: FloatTy: [ 0.00 0.00 ]
Key: VoidTy: [ 0.00 0.00 ]
Key: LabelTy: [ 0.00 0.00 ]
diff --git a/llvm/test/Assembler/invalid-storermw-elementwise.ll b/llvm/test/Assembler/invalid-storermw-elementwise.ll
new file mode 100644
index 0000000000000..9ed6a95e94c75
--- /dev/null
+++ b/llvm/test/Assembler/invalid-storermw-elementwise.ll
@@ -0,0 +1,33 @@
+; RUN: split-file %s %t
+; RUN: not llvm-as -disable-output %t/scalar.ll 2>&1 | FileCheck %t/scalar.ll
+; RUN: not llvm-as -disable-output %t/odd-sized.ll 2>&1 | FileCheck %t/odd-sized.ll
+; RUN: not llvm-as -disable-output %t/add-must-be-integer.ll 2>&1 | FileCheck %t/add-must-be-integer.ll
+; RUN: not llvm-as -disable-output %t/fadd-must-be-fp.ll 2>&1 | FileCheck %t/fadd-must-be-fp.ll
+
+;--- scalar.ll
+; CHECK: storermw elementwise operand must be a fixed vector type
+define void @bad_scalar(ptr %p, i32 %v) {
+ storermw elementwise add ptr %p, i32 %v monotonic
+ ret void
+}
+
+;--- odd-sized.ll
+; CHECK: storermw operand must be power-of-two byte-sized integer
+define void @bad_odd_sized_vector(ptr %p, <5 x i32> %v) {
+ storermw elementwise add ptr %p, <5 x i32> %v monotonic, align 4
+ ret void
+}
+
+;--- add-must-be-integer.ll
+; CHECK: storermw add operand must be an integer
+define void @bad_add(ptr %p, <4 x float> %v) {
+ storermw elementwise add ptr %p, <4 x float> %v monotonic
+ ret void
+}
+
+;--- fadd-must-be-fp.ll
+; CHECK: storermw fadd operand must be a floating point type
+define void @bad_fadd(ptr %p, <4 x i32> %v) {
+ storermw elementwise fadd ptr %p, <4 x i32> %v monotonic
+ ret void
+}
diff --git a/llvm/test/Assembler/invalid-storermw-types.ll b/llvm/test/Assembler/invalid-storermw-types.ll
new file mode 100644
index 0000000000000..3f74badb213d2
--- /dev/null
+++ b/llvm/test/Assembler/invalid-storermw-types.ll
@@ -0,0 +1,58 @@
+; RUN: split-file %s %t
+; RUN: not llvm-as -disable-output %t/add-fp-scalar.ll 2>&1 | FileCheck %t/add-fp-scalar.ll
+; RUN: not llvm-as -disable-output %t/fadd-int-scalar.ll 2>&1 | FileCheck %t/fadd-int-scalar.ll
+; RUN: not llvm-as -disable-output %t/i4-too-narrow.ll 2>&1 | FileCheck %t/i4-too-narrow.ll
+; RUN: not llvm-as -disable-output %t/i9-not-pow2.ll 2>&1 | FileCheck %t/i9-not-pow2.ll
+; RUN: not llvm-as -disable-output %t/scalable-vector.ll 2>&1 | FileCheck %t/scalable-vector.ll
+; RUN: not llvm-as -disable-output %t/non-pointer-operand.ll 2>&1 | FileCheck %t/non-pointer-operand.ll
+
+;--- add-fp-scalar.ll
+; Non-elementwise integer op with a floating-point scalar operand.
+; CHECK: storermw add operand must be an integer
+define void @bad_add_fp(ptr %p, float %v) {
+ storermw add ptr %p, float %v monotonic, align 4
+ ret void
+}
+
+;--- fadd-int-scalar.ll
+; Non-elementwise floating-point op with an integer scalar operand.
+; CHECK: storermw fadd operand must be a floating point type
+define void @bad_fadd_int(ptr %p, i32 %v) {
+ storermw fadd ptr %p, i32 %v monotonic, align 4
+ ret void
+}
+
+;--- i4-too-narrow.ll
+; Value type must have bit width >= 8. Verifier catches this (the parser's
+; check uses getTypeStoreSizeInBits which rounds i4 up to 8).
+; CHECK: atomic memory access' size must be byte-sized
+define void @bad_i4(ptr %p, i4 %v) {
+ storermw add ptr %p, i4 %v monotonic, align 1
+ ret void
+}
+
+;--- i9-not-pow2.ll
+; Value type bit width must be a power of two. i9 has size >= 8 (so the
+; "byte-sized" check passes) but is not a power of two, exercising the
+; distinct power-of-two Verifier check.
+; CHECK: atomic memory access' operand must have a power-of-two size
+define void @bad_i9(ptr %p, i9 %v) {
+ storermw add ptr %p, i9 %v monotonic, align 2
+ ret void
+}
+
+;--- scalable-vector.ll
+; Scalable vector value operand is not allowed.
+; CHECK: storermw operand may not be scalable
+define void @bad_scalable(ptr %p, <vscale x 4 x i32> %v) {
+ storermw elementwise add ptr %p, <vscale x 4 x i32> %v monotonic, align 16
+ ret void
+}
+
+;--- non-pointer-operand.ll
+; The first operand must be a pointer, not (for example) an integer.
+; CHECK: storermw operand must be a pointer
+define void @bad_non_pointer(i32 %notptr, i32 %v) {
+ storermw add i32 %notptr, i32 %v monotonic, align 4
+ ret void
+}
diff --git a/llvm/test/Assembler/storermw.ll b/llvm/test/Assembler/storermw.ll
new file mode 100644
index 0000000000000..3a12839323e3a
--- /dev/null
+++ b/llvm/test/Assembler/storermw.ll
@@ -0,0 +1,193 @@
+; RUN: llvm-as < %s | llvm-dis | FileCheck %s
+; RUN: verify-uselistorder %s
+
+; Test atomic reduction instruction parsing and printing
+
+; Integer atomic reductions - all operations
+; CHECK-LABEL: @test_atomic_reduction_ops
+define void @test_atomic_reduction_ops(ptr %ptr, i32 %val, i64 %val64, float %fval, double %dval) {
+ ; CHECK: storermw add ptr %ptr, i32 %val seq_cst, align 4
+ storermw add ptr %ptr, i32 %val seq_cst, align 4
+
+ ; CHECK: storermw sub ptr %ptr, i32 %val seq_cst, align 4
+ storermw sub ptr %ptr, i32 %val seq_cst, align 4
+
+ ; CHECK: storermw and ptr %ptr, i32 %val seq_cst, align 4
+ storermw and ptr %ptr, i32 %val seq_cst, align 4
+
+ ; CHECK: storermw nand ptr %ptr, i32 %val seq_cst, align 4
+ storermw nand ptr %ptr, i32 %val seq_cst, align 4
+
+ ; CHECK: storermw or ptr %ptr, i32 %val seq_cst, align 4
+ storermw or ptr %ptr, i32 %val seq_cst, align 4
+
+ ; CHECK: storermw xor ptr %ptr, i32 %val seq_cst, align 4
+ storermw xor ptr %ptr, i32 %val seq_cst, align 4
+
+ ; CHECK: storermw max ptr %ptr, i32 %val seq_cst, align 4
+ storermw max ptr %ptr, i32 %val seq_cst, align 4
+
+ ; CHECK: storermw min ptr %ptr, i32 %val seq_cst, align 4
+ storermw min ptr %ptr, i32 %val seq_cst, align 4
+
+ ; CHECK: storermw umax ptr %ptr, i32 %val seq_cst, align 4
+ storermw umax ptr %ptr, i32 %val seq_cst, align 4
+
+ ; CHECK: storermw umin ptr %ptr, i32 %val seq_cst, align 4
+ storermw umin ptr %ptr, i32 %val seq_cst, align 4
+
+ ; CHECK: storermw uinc_wrap ptr %ptr, i32 %val seq_cst, align 4
+ storermw uinc_wrap ptr %ptr, i32 %val seq_cst, align 4
+
+ ; CHECK: storermw udec_wrap ptr %ptr, i32 %val seq_cst, align 4
+ storermw udec_wrap ptr %ptr, i32 %val seq_cst, align 4
+
+ ; CHECK: storermw usub_cond ptr %ptr, i32 %val seq_cst, align 4
+ storermw usub_cond ptr %ptr, i32 %val seq_cst, align 4
+
+ ; CHECK: storermw usub_sat ptr %ptr, i32 %val seq_cst, align 4
+ storermw usub_sat ptr %ptr, i32 %val seq_cst, align 4
+
+ ; 64-bit integer
+ ; CHECK: storermw add ptr %ptr, i64 %val64 seq_cst, align 8
+ storermw add ptr %ptr, i64 %val64 seq_cst, align 8
+
+ ; Floating-point atomic reductions
+ ; CHECK: storermw fadd ptr %ptr, float %fval seq_cst, align 4
+ storermw fadd ptr %ptr, float %fval seq_cst, align 4
+
+ ; CHECK: storermw fsub ptr %ptr, float %fval seq_cst, align 4
+ storermw fsub ptr %ptr, float %fval seq_cst, align 4
+
+ ; CHECK: storermw fmax ptr %ptr, float %fval seq_cst, align 4
+ storermw fmax ptr %ptr, float %fval seq_cst, align 4
+
+ ; CHECK: storermw fmin ptr %ptr, float %fval seq_cst, align 4
+ storermw fmin ptr %ptr, float %fval seq_cst, align 4
+
+ ; CHECK: storermw fmaximum ptr %ptr, float %fval seq_cst, align 4
+ storermw fmaximum ptr %ptr, float %fval seq_cst, align 4
+
+ ; CHECK: storermw fminimum ptr %ptr, float %fval seq_cst, align 4
+ storermw fminimum ptr %ptr, float %fval seq_cst, align 4
+
+ ; CHECK: storermw fmaximumnum ptr %ptr, float %fval seq_cst, align 4
+ storermw fmaximumnum ptr %ptr, float %fval seq_cst, align 4
+
+ ; CHECK: storermw fminimumnum ptr %ptr, float %fval seq_cst, align 4
+ storermw fminimumnum ptr %ptr, float %fval seq_cst, align 4
+
+ ; CHECK: storermw fmaximum ptr %ptr, double %dval seq_cst, align 8
+ storermw fmaximum ptr %ptr, double %dval seq_cst, align 8
+
+ ; CHECK: storermw fminimum ptr %ptr, double %dval seq_cst, align 8
+ storermw fminimum ptr %ptr, double %dval seq_cst, align 8
+
+ ret void
+}
+
+; Test different integer sizes (i8, i16)
+; CHECK-LABEL: @test_sizes
+define void @test_sizes(ptr %ptr, i8 %v8, i16 %v16) {
+ ; CHECK: storermw add ptr %ptr, i8 %v8 seq_cst, align 1
+ storermw add ptr %ptr, i8 %v8 seq_cst, align 1
+
+ ; CHECK: storermw add ptr %ptr, i16 %v16 seq_cst, align 2
+ storermw add ptr %ptr, i16 %v16 seq_cst, align 2
+
+ ret void
+}
+
+; Test different orderings (monotonic, release, seq_cst)
+; CHECK-LABEL: @test_orderings
+define void @test_orderings(ptr %ptr, i32 %val) {
+ ; CHECK: storermw add ptr %ptr, i32 %val monotonic, align 4
+ storermw add ptr %ptr, i32 %val monotonic, align 4
+
+ ; CHECK: storermw add ptr %ptr, i32 %val release, align 4
+ storermw add ptr %ptr, i32 %val release, align 4
+
+ ; CHECK: storermw add ptr %ptr, i32 %val seq_cst, align 4
+ storermw add ptr %ptr, i32 %val seq_cst, align 4
+
+ ret void
+}
+
+; Test volatile flag
+; CHECK-LABEL: @test_volatile
+define void @test_volatile(ptr %ptr, i32 %val) {
+ ; CHECK: storermw volatile add ptr %ptr, i32 %val seq_cst, align 4
+ storermw volatile add ptr %ptr, i32 %val seq_cst, align 4
+
+ ret void
+}
+
+; Test sync scope
+; CHECK-LABEL: @test_syncscope
+define void @test_syncscope(ptr %ptr, i32 %val) {
+ ; CHECK: storermw add ptr %ptr, i32 %val syncscope("singlethread") seq_cst, align 4
+ storermw add ptr %ptr, i32 %val syncscope("singlethread") seq_cst, align 4
+
+ ; CHECK: storermw add ptr %ptr, i32 %val syncscope("agent") monotonic, align 4
+ storermw add ptr %ptr, i32 %val syncscope("agent") monotonic, align 4
+
+ ret void
+}
+
+; Test default alignment: when the ", align N" clause is omitted, the parser
+; sets align to the size of the value type; the printer then prints it back.
+; CHECK-LABEL: @test_default_alignment
+define void @test_default_alignment(ptr %ptr, i8 %v8, i16 %v16, i32 %v32,
+ i64 %v64, float %vf, double %vd,
+ <4 x i32> %v4i32) {
+ ; CHECK: storermw add ptr %ptr, i8 %v8 monotonic, align 1
+ storermw add ptr %ptr, i8 %v8 monotonic
+
+ ; CHECK: storermw add ptr %ptr, i16 %v16 monotonic, align 2
+ storermw add ptr %ptr, i16 %v16 monotonic
+
+ ; CHECK: storermw add ptr %ptr, i32 %v32 monotonic, align 4
+ storermw add ptr %ptr, i32 %v32 monotonic
+
+ ; CHECK: storermw add ptr %ptr, i64 %v64 monotonic, align 8
+ storermw add ptr %ptr, i64 %v64 monotonic
+
+ ; CHECK: storermw fadd ptr %ptr, float %vf monotonic, align 4
+ storermw fadd ptr %ptr, float %vf monotonic
+
+ ; CHECK: storermw fadd ptr %ptr, double %vd monotonic, align 8
+ storermw fadd ptr %ptr, double %vd monotonic
+
+ ; CHECK: storermw elementwise add ptr %ptr, <4 x i32> %v4i32 monotonic, align 16
+ storermw elementwise add ptr %ptr, <4 x i32> %v4i32 monotonic
+
+ ret void
+}
+
+; Test volatile + syncscope combination
+; CHECK-LABEL: @test_volatile_syncscope
+define void @test_volatile_syncscope(ptr %ptr, i32 %val) {
+ ; CHECK: storermw volatile add ptr %ptr, i32 %val syncscope("singlethread") release, align 4
+ storermw volatile add ptr %ptr, i32 %val syncscope("singlethread") release, align 4
+
+ ret void
+}
+
+; Test elementwise modifier round-trip
+; CHECK-LABEL: @test_elementwise
+define void @test_elementwise(ptr %ptr, <4 x i32> %v4i32, <2 x i64> %v2i64,
+ <4 x float> %v4f32) {
+ ; CHECK: storermw elementwise add ptr %ptr, <4 x i32> %v4i32 seq_cst, align 16
+ storermw elementwise add ptr %ptr, <4 x i32> %v4i32 seq_cst, align 16
+
+ ; CHECK: storermw elementwise umax ptr %ptr, <2 x i64> %v2i64 monotonic, align 16
+ storermw elementwise umax ptr %ptr, <2 x i64> %v2i64 monotonic, align 16
+
+ ; CHECK: storermw elementwise fadd ptr %ptr, <4 x float> %v4f32 release, align 16
+ storermw elementwise fadd ptr %ptr, <4 x float> %v4f32 release, align 16
+
+ ; CHECK: storermw volatile elementwise add ptr %ptr, <4 x i32> %v4i32 seq_cst, align 16
+ storermw volatile elementwise add ptr %ptr, <4 x i32> %v4i32 seq_cst, align 16
+
+ ret void
+}
diff --git a/llvm/test/CodeGen/Generic/storermw.ll b/llvm/test/CodeGen/Generic/storermw.ll
new file mode 100644
index 0000000000000..033f85b5703ca
--- /dev/null
+++ b/llvm/test/CodeGen/Generic/storermw.ll
@@ -0,0 +1,3448 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -passes='require<libcall-lowering-info>,atomic-expand' -mtriple=x86_64-unknown-linux-gnu -S | FileCheck %s
+; Test AtomicExpandPass handling of storermw:
+; monotonic/release/seq_cst -> atomicrmw with dead result
+; Generated by storermw_gen.py -- do not edit function bodies by hand;
+; edit the generator and rerun it, then update-opt-checks.
+
+; ===== ordering: monotonic =====
+
+; All LangRef integer ops on i32, monotonic
+define void @test_add_i32_monotonic(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_add_i32_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw add ptr [[ADDR]], i32 [[VAL]] monotonic, align 4
+; CHECK-NEXT: ret void
+;
+ storermw add ptr %addr, i32 %val monotonic, align 4
+ ret void
+}
+
+define void @test_sub_i32_monotonic(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_sub_i32_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw sub ptr [[ADDR]], i32 [[VAL]] monotonic, align 4
+; CHECK-NEXT: ret void
+;
+ storermw sub ptr %addr, i32 %val monotonic, align 4
+ ret void
+}
+
+define void @test_and_i32_monotonic(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_and_i32_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw and ptr [[ADDR]], i32 [[VAL]] monotonic, align 4
+; CHECK-NEXT: ret void
+;
+ storermw and ptr %addr, i32 %val monotonic, align 4
+ ret void
+}
+
+define void @test_nand_i32_monotonic(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_nand_i32_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = and i32 [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[NEW:%.*]] = xor i32 [[TMP2]], -1
+; CHECK-NEXT: [[TMP3:%.*]] = cmpxchg ptr [[ADDR]], i32 [[LOADED]], i32 [[NEW]] monotonic monotonic, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw nand ptr %addr, i32 %val monotonic, align 4
+ ret void
+}
+
+define void @test_or_i32_monotonic(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_or_i32_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw or ptr [[ADDR]], i32 [[VAL]] monotonic, align 4
+; CHECK-NEXT: ret void
+;
+ storermw or ptr %addr, i32 %val monotonic, align 4
+ ret void
+}
+
+define void @test_xor_i32_monotonic(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_xor_i32_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw xor ptr [[ADDR]], i32 [[VAL]] monotonic, align 4
+; CHECK-NEXT: ret void
+;
+ storermw xor ptr %addr, i32 %val monotonic, align 4
+ ret void
+}
+
+define void @test_max_i32_monotonic(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_max_i32_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt i32 [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VAL]]
+; CHECK-NEXT: [[TMP3:%.*]] = cmpxchg ptr [[ADDR]], i32 [[LOADED]], i32 [[NEW]] monotonic monotonic, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw max ptr %addr, i32 %val monotonic, align 4
+ ret void
+}
+
+define void @test_min_i32_monotonic(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_min_i32_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp sle i32 [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VAL]]
+; CHECK-NEXT: [[TMP3:%.*]] = cmpxchg ptr [[ADDR]], i32 [[LOADED]], i32 [[NEW]] monotonic monotonic, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw min ptr %addr, i32 %val monotonic, align 4
+ ret void
+}
+
+define void @test_umax_i32_monotonic(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_umax_i32_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ugt i32 [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VAL]]
+; CHECK-NEXT: [[TMP3:%.*]] = cmpxchg ptr [[ADDR]], i32 [[LOADED]], i32 [[NEW]] monotonic monotonic, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw umax ptr %addr, i32 %val monotonic, align 4
+ ret void
+}
+
+define void @test_umin_i32_monotonic(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_umin_i32_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ule i32 [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VAL]]
+; CHECK-NEXT: [[TMP3:%.*]] = cmpxchg ptr [[ADDR]], i32 [[LOADED]], i32 [[NEW]] monotonic monotonic, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw umin ptr %addr, i32 %val monotonic, align 4
+ ret void
+}
+
+define void @test_uinc_wrap_i32_monotonic(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_uinc_wrap_i32_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[LOADED]], 1
+; CHECK-NEXT: [[TMP3:%.*]] = icmp uge i32 [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[ADDR]], i32 [[LOADED]], i32 [[NEW]] monotonic monotonic, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1
+; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP4]], 0
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw uinc_wrap ptr %addr, i32 %val monotonic, align 4
+ ret void
+}
+
+define void @test_udec_wrap_i32_monotonic(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_udec_wrap_i32_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = sub i32 [[LOADED]], 1
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[LOADED]], 0
+; CHECK-NEXT: [[TMP4:%.*]] = icmp ugt i32 [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
+; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP5]], i32 [[VAL]], i32 [[TMP2]]
+; CHECK-NEXT: [[TMP6:%.*]] = cmpxchg ptr [[ADDR]], i32 [[LOADED]], i32 [[NEW]] monotonic monotonic, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 1
+; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 0
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw udec_wrap ptr %addr, i32 %val monotonic, align 4
+ ret void
+}
+
+define void @test_usub_cond_i32_monotonic(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_usub_cond_i32_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp uge i32 [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[TMP3:%.*]] = sub i32 [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i32 [[TMP3]], i32 [[LOADED]]
+; CHECK-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[ADDR]], i32 [[LOADED]], i32 [[NEW]] monotonic monotonic, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1
+; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP4]], 0
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw usub_cond ptr %addr, i32 %val monotonic, align 4
+ ret void
+}
+
+define void @test_usub_sat_i32_monotonic(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_usub_sat_i32_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[NEW:%.*]] = call i32 @llvm.usub.sat.i32(i32 [[LOADED]], i32 [[VAL]])
+; CHECK-NEXT: [[TMP2:%.*]] = cmpxchg ptr [[ADDR]], i32 [[LOADED]], i32 [[NEW]] monotonic monotonic, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP2]], 0
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw usub_sat ptr %addr, i32 %val monotonic, align 4
+ ret void
+}
+
+; All LangRef FP ops on float, monotonic
+define void @test_fadd_f32_monotonic(ptr %addr, float %val) {
+; CHECK-LABEL: define void @test_fadd_f32_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], float [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32
+; CHECK-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[ADDR]], i32 [[TMP3]], i32 [[TMP2]] monotonic monotonic, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0
+; CHECK-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fadd ptr %addr, float %val monotonic, align 4
+ ret void
+}
+
+define void @test_fsub_f32_monotonic(ptr %addr, float %val) {
+; CHECK-LABEL: define void @test_fsub_f32_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], float [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[NEW:%.*]] = fsub float [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32
+; CHECK-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[ADDR]], i32 [[TMP3]], i32 [[TMP2]] monotonic monotonic, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0
+; CHECK-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fsub ptr %addr, float %val monotonic, align 4
+ ret void
+}
+
+define void @test_fmax_f32_monotonic(ptr %addr, float %val) {
+; CHECK-LABEL: define void @test_fmax_f32_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], float [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call float @llvm.maxnum.f32(float [[LOADED]], float [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast float [[TMP2]] to i32
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i32 [[TMP4]], i32 [[TMP3]] monotonic monotonic, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fmax ptr %addr, float %val monotonic, align 4
+ ret void
+}
+
+define void @test_fmin_f32_monotonic(ptr %addr, float %val) {
+; CHECK-LABEL: define void @test_fmin_f32_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], float [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call float @llvm.minnum.f32(float [[LOADED]], float [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast float [[TMP2]] to i32
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i32 [[TMP4]], i32 [[TMP3]] monotonic monotonic, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fmin ptr %addr, float %val monotonic, align 4
+ ret void
+}
+
+define void @test_fmaximum_f32_monotonic(ptr %addr, float %val) {
+; CHECK-LABEL: define void @test_fmaximum_f32_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], float [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call float @llvm.maximum.f32(float [[LOADED]], float [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast float [[TMP2]] to i32
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i32 [[TMP4]], i32 [[TMP3]] monotonic monotonic, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fmaximum ptr %addr, float %val monotonic, align 4
+ ret void
+}
+
+define void @test_fminimum_f32_monotonic(ptr %addr, float %val) {
+; CHECK-LABEL: define void @test_fminimum_f32_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], float [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call float @llvm.minimum.f32(float [[LOADED]], float [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast float [[TMP2]] to i32
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i32 [[TMP4]], i32 [[TMP3]] monotonic monotonic, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fminimum ptr %addr, float %val monotonic, align 4
+ ret void
+}
+
+define void @test_fmaximumnum_f32_monotonic(ptr %addr, float %val) {
+; CHECK-LABEL: define void @test_fmaximumnum_f32_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], float [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call float @llvm.maximumnum.f32(float [[LOADED]], float [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast float [[TMP2]] to i32
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i32 [[TMP4]], i32 [[TMP3]] monotonic monotonic, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fmaximumnum ptr %addr, float %val monotonic, align 4
+ ret void
+}
+
+define void @test_fminimumnum_f32_monotonic(ptr %addr, float %val) {
+; CHECK-LABEL: define void @test_fminimumnum_f32_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], float [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call float @llvm.minimumnum.f32(float [[LOADED]], float [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast float [[TMP2]] to i32
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i32 [[TMP4]], i32 [[TMP3]] monotonic monotonic, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fminimumnum ptr %addr, float %val monotonic, align 4
+ ret void
+}
+
+; <2 x i32> elementwise (all integer ops), monotonic
+define void @test_add_v2i32_elementwise_monotonic(ptr %addr, <2 x i32> %val) {
+; CHECK-LABEL: define void @test_add_v2i32_elementwise_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x i32> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw elementwise add ptr [[ADDR]], <2 x i32> [[VAL]] monotonic, align 8
+; CHECK-NEXT: ret void
+;
+ storermw elementwise add ptr %addr, <2 x i32> %val monotonic, align 8
+ ret void
+}
+
+define void @test_sub_v2i32_elementwise_monotonic(ptr %addr, <2 x i32> %val) {
+; CHECK-LABEL: define void @test_sub_v2i32_elementwise_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x i32> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw elementwise sub ptr [[ADDR]], <2 x i32> [[VAL]] monotonic, align 8
+; CHECK-NEXT: ret void
+;
+ storermw elementwise sub ptr %addr, <2 x i32> %val monotonic, align 8
+ ret void
+}
+
+define void @test_and_v2i32_elementwise_monotonic(ptr %addr, <2 x i32> %val) {
+; CHECK-LABEL: define void @test_and_v2i32_elementwise_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x i32> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw elementwise and ptr [[ADDR]], <2 x i32> [[VAL]] monotonic, align 8
+; CHECK-NEXT: ret void
+;
+ storermw elementwise and ptr %addr, <2 x i32> %val monotonic, align 8
+ ret void
+}
+
+define void @test_nand_v2i32_elementwise_monotonic(ptr %addr, <2 x i32> %val) {
+; CHECK-LABEL: define void @test_nand_v2i32_elementwise_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x i32> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x i32> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = and <2 x i32> [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[NEW:%.*]] = xor <2 x i32> [[TMP2]], splat (i32 -1)
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x i32> [[NEW]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] monotonic monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x i32>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise nand ptr %addr, <2 x i32> %val monotonic, align 8
+ ret void
+}
+
+define void @test_or_v2i32_elementwise_monotonic(ptr %addr, <2 x i32> %val) {
+; CHECK-LABEL: define void @test_or_v2i32_elementwise_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x i32> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw elementwise or ptr [[ADDR]], <2 x i32> [[VAL]] monotonic, align 8
+; CHECK-NEXT: ret void
+;
+ storermw elementwise or ptr %addr, <2 x i32> %val monotonic, align 8
+ ret void
+}
+
+define void @test_xor_v2i32_elementwise_monotonic(ptr %addr, <2 x i32> %val) {
+; CHECK-LABEL: define void @test_xor_v2i32_elementwise_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x i32> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw elementwise xor ptr [[ADDR]], <2 x i32> [[VAL]] monotonic, align 8
+; CHECK-NEXT: ret void
+;
+ storermw elementwise xor ptr %addr, <2 x i32> %val monotonic, align 8
+ ret void
+}
+
+define void @test_max_v2i32_elementwise_monotonic(ptr %addr, <2 x i32> %val) {
+; CHECK-LABEL: define void @test_max_v2i32_elementwise_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x i32> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x i32> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt <2 x i32> [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[NEW:%.*]] = select <2 x i1> [[TMP2]], <2 x i32> [[LOADED]], <2 x i32> [[VAL]]
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x i32> [[NEW]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] monotonic monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x i32>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise max ptr %addr, <2 x i32> %val monotonic, align 8
+ ret void
+}
+
+define void @test_min_v2i32_elementwise_monotonic(ptr %addr, <2 x i32> %val) {
+; CHECK-LABEL: define void @test_min_v2i32_elementwise_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x i32> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x i32> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp sle <2 x i32> [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[NEW:%.*]] = select <2 x i1> [[TMP2]], <2 x i32> [[LOADED]], <2 x i32> [[VAL]]
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x i32> [[NEW]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] monotonic monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x i32>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise min ptr %addr, <2 x i32> %val monotonic, align 8
+ ret void
+}
+
+define void @test_umax_v2i32_elementwise_monotonic(ptr %addr, <2 x i32> %val) {
+; CHECK-LABEL: define void @test_umax_v2i32_elementwise_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x i32> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x i32> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ugt <2 x i32> [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[NEW:%.*]] = select <2 x i1> [[TMP2]], <2 x i32> [[LOADED]], <2 x i32> [[VAL]]
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x i32> [[NEW]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] monotonic monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x i32>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise umax ptr %addr, <2 x i32> %val monotonic, align 8
+ ret void
+}
+
+define void @test_umin_v2i32_elementwise_monotonic(ptr %addr, <2 x i32> %val) {
+; CHECK-LABEL: define void @test_umin_v2i32_elementwise_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x i32> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x i32> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ule <2 x i32> [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[NEW:%.*]] = select <2 x i1> [[TMP2]], <2 x i32> [[LOADED]], <2 x i32> [[VAL]]
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x i32> [[NEW]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] monotonic monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x i32>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise umin ptr %addr, <2 x i32> %val monotonic, align 8
+ ret void
+}
+
+define void @test_uinc_wrap_v2i32_elementwise_monotonic(ptr %addr, <2 x i32> %val) {
+; CHECK-LABEL: define void @test_uinc_wrap_v2i32_elementwise_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x i32> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x i32> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP7:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = add <2 x i32> [[LOADED]], splat (i32 1)
+; CHECK-NEXT: [[TMP3:%.*]] = icmp uge <2 x i32> [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[NEW:%.*]] = select <2 x i1> [[TMP3]], <2 x i32> zeroinitializer, <2 x i32> [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[NEW]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast <2 x i32> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP6:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP5]], i64 [[TMP4]] monotonic monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP6]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP6]], 0
+; CHECK-NEXT: [[TMP7]] = bitcast i64 [[NEWLOADED]] to <2 x i32>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise uinc_wrap ptr %addr, <2 x i32> %val monotonic, align 8
+ ret void
+}
+
+define void @test_udec_wrap_v2i32_elementwise_monotonic(ptr %addr, <2 x i32> %val) {
+; CHECK-LABEL: define void @test_udec_wrap_v2i32_elementwise_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x i32> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x i32> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP9:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = sub <2 x i32> [[LOADED]], splat (i32 1)
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <2 x i32> [[LOADED]], zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = icmp ugt <2 x i32> [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[TMP5:%.*]] = or <2 x i1> [[TMP3]], [[TMP4]]
+; CHECK-NEXT: [[NEW:%.*]] = select <2 x i1> [[TMP5]], <2 x i32> [[VAL]], <2 x i32> [[TMP2]]
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast <2 x i32> [[NEW]] to i64
+; CHECK-NEXT: [[TMP7:%.*]] = bitcast <2 x i32> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP8:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP7]], i64 [[TMP6]] monotonic monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP8]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP8]], 0
+; CHECK-NEXT: [[TMP9]] = bitcast i64 [[NEWLOADED]] to <2 x i32>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise udec_wrap ptr %addr, <2 x i32> %val monotonic, align 8
+ ret void
+}
+
+define void @test_usub_cond_v2i32_elementwise_monotonic(ptr %addr, <2 x i32> %val) {
+; CHECK-LABEL: define void @test_usub_cond_v2i32_elementwise_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x i32> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x i32> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP7:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp uge <2 x i32> [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[TMP3:%.*]] = sub <2 x i32> [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[NEW:%.*]] = select <2 x i1> [[TMP2]], <2 x i32> [[TMP3]], <2 x i32> [[LOADED]]
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[NEW]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast <2 x i32> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP6:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP5]], i64 [[TMP4]] monotonic monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP6]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP6]], 0
+; CHECK-NEXT: [[TMP7]] = bitcast i64 [[NEWLOADED]] to <2 x i32>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise usub_cond ptr %addr, <2 x i32> %val monotonic, align 8
+ ret void
+}
+
+define void @test_usub_sat_v2i32_elementwise_monotonic(ptr %addr, <2 x i32> %val) {
+; CHECK-LABEL: define void @test_usub_sat_v2i32_elementwise_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x i32> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x i32> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[NEW:%.*]] = call <2 x i32> @llvm.usub.sat.v2i32(<2 x i32> [[LOADED]], <2 x i32> [[VAL]])
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x i32> [[NEW]] to i64
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x i32> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP3]], i64 [[TMP2]] monotonic monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; CHECK-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to <2 x i32>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise usub_sat ptr %addr, <2 x i32> %val monotonic, align 8
+ ret void
+}
+
+; <2 x float> whole-vector (all FP ops), monotonic
+define void @test_fadd_v2f32_whole_monotonic(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fadd_v2f32_whole_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[NEW:%.*]] = fadd <2 x float> [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x float> [[NEW]] to i64
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP3]], i64 [[TMP2]] monotonic monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; CHECK-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fadd ptr %addr, <2 x float> %val monotonic, align 8
+ ret void
+}
+
+define void @test_fsub_v2f32_whole_monotonic(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fsub_v2f32_whole_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[NEW:%.*]] = fsub <2 x float> [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x float> [[NEW]] to i64
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP3]], i64 [[TMP2]] monotonic monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; CHECK-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fsub ptr %addr, <2 x float> %val monotonic, align 8
+ ret void
+}
+
+define void @test_fmax_v2f32_whole_monotonic(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fmax_v2f32_whole_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call <2 x float> @llvm.maxnum.v2f32(<2 x float> [[LOADED]], <2 x float> [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[TMP2]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] monotonic monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fmax ptr %addr, <2 x float> %val monotonic, align 8
+ ret void
+}
+
+define void @test_fmin_v2f32_whole_monotonic(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fmin_v2f32_whole_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call <2 x float> @llvm.minnum.v2f32(<2 x float> [[LOADED]], <2 x float> [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[TMP2]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] monotonic monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fmin ptr %addr, <2 x float> %val monotonic, align 8
+ ret void
+}
+
+define void @test_fmaximum_v2f32_whole_monotonic(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fmaximum_v2f32_whole_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call <2 x float> @llvm.maximum.v2f32(<2 x float> [[LOADED]], <2 x float> [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[TMP2]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] monotonic monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fmaximum ptr %addr, <2 x float> %val monotonic, align 8
+ ret void
+}
+
+define void @test_fminimum_v2f32_whole_monotonic(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fminimum_v2f32_whole_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call <2 x float> @llvm.minimum.v2f32(<2 x float> [[LOADED]], <2 x float> [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[TMP2]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] monotonic monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fminimum ptr %addr, <2 x float> %val monotonic, align 8
+ ret void
+}
+
+define void @test_fmaximumnum_v2f32_whole_monotonic(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fmaximumnum_v2f32_whole_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call <2 x float> @llvm.maximumnum.v2f32(<2 x float> [[LOADED]], <2 x float> [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[TMP2]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] monotonic monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fmaximumnum ptr %addr, <2 x float> %val monotonic, align 8
+ ret void
+}
+
+define void @test_fminimumnum_v2f32_whole_monotonic(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fminimumnum_v2f32_whole_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call <2 x float> @llvm.minimumnum.v2f32(<2 x float> [[LOADED]], <2 x float> [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[TMP2]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] monotonic monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fminimumnum ptr %addr, <2 x float> %val monotonic, align 8
+ ret void
+}
+
+; <2 x float> elementwise (all FP ops), monotonic
+define void @test_fadd_v2f32_elementwise_monotonic(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fadd_v2f32_elementwise_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[NEW:%.*]] = fadd <2 x float> [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x float> [[NEW]] to i64
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP3]], i64 [[TMP2]] monotonic monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; CHECK-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise fadd ptr %addr, <2 x float> %val monotonic, align 8
+ ret void
+}
+
+define void @test_fsub_v2f32_elementwise_monotonic(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fsub_v2f32_elementwise_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[NEW:%.*]] = fsub <2 x float> [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x float> [[NEW]] to i64
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP3]], i64 [[TMP2]] monotonic monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; CHECK-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise fsub ptr %addr, <2 x float> %val monotonic, align 8
+ ret void
+}
+
+define void @test_fmax_v2f32_elementwise_monotonic(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fmax_v2f32_elementwise_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call <2 x float> @llvm.maxnum.v2f32(<2 x float> [[LOADED]], <2 x float> [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[TMP2]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] monotonic monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise fmax ptr %addr, <2 x float> %val monotonic, align 8
+ ret void
+}
+
+define void @test_fmin_v2f32_elementwise_monotonic(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fmin_v2f32_elementwise_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call <2 x float> @llvm.minnum.v2f32(<2 x float> [[LOADED]], <2 x float> [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[TMP2]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] monotonic monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise fmin ptr %addr, <2 x float> %val monotonic, align 8
+ ret void
+}
+
+define void @test_fmaximum_v2f32_elementwise_monotonic(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fmaximum_v2f32_elementwise_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call <2 x float> @llvm.maximum.v2f32(<2 x float> [[LOADED]], <2 x float> [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[TMP2]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] monotonic monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise fmaximum ptr %addr, <2 x float> %val monotonic, align 8
+ ret void
+}
+
+define void @test_fminimum_v2f32_elementwise_monotonic(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fminimum_v2f32_elementwise_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call <2 x float> @llvm.minimum.v2f32(<2 x float> [[LOADED]], <2 x float> [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[TMP2]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] monotonic monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise fminimum ptr %addr, <2 x float> %val monotonic, align 8
+ ret void
+}
+
+define void @test_fmaximumnum_v2f32_elementwise_monotonic(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fmaximumnum_v2f32_elementwise_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call <2 x float> @llvm.maximumnum.v2f32(<2 x float> [[LOADED]], <2 x float> [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[TMP2]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] monotonic monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise fmaximumnum ptr %addr, <2 x float> %val monotonic, align 8
+ ret void
+}
+
+define void @test_fminimumnum_v2f32_elementwise_monotonic(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fminimumnum_v2f32_elementwise_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call <2 x float> @llvm.minimumnum.v2f32(<2 x float> [[LOADED]], <2 x float> [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[TMP2]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] monotonic monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise fminimumnum ptr %addr, <2 x float> %val monotonic, align 8
+ ret void
+}
+
+; Alternate integer sizes for add, monotonic
+define void @test_add_i8_monotonic(ptr %addr, i8 %val) {
+; CHECK-LABEL: define void @test_add_i8_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], i8 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw add ptr [[ADDR]], i8 [[VAL]] monotonic, align 1
+; CHECK-NEXT: ret void
+;
+ storermw add ptr %addr, i8 %val monotonic, align 1
+ ret void
+}
+
+define void @test_add_i16_monotonic(ptr %addr, i16 %val) {
+; CHECK-LABEL: define void @test_add_i16_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], i16 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw add ptr [[ADDR]], i16 [[VAL]] monotonic, align 2
+; CHECK-NEXT: ret void
+;
+ storermw add ptr %addr, i16 %val monotonic, align 2
+ ret void
+}
+
+define void @test_add_i64_monotonic(ptr %addr, i64 %val) {
+; CHECK-LABEL: define void @test_add_i64_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], i64 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw add ptr [[ADDR]], i64 [[VAL]] monotonic, align 8
+; CHECK-NEXT: ret void
+;
+ storermw add ptr %addr, i64 %val monotonic, align 8
+ ret void
+}
+
+; Double for fadd, monotonic
+define void @test_fadd_f64_monotonic(ptr %addr, double %val) {
+; CHECK-LABEL: define void @test_fadd_f64_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], double [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load double, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP3]], i64 [[TMP2]] monotonic monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; CHECK-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fadd ptr %addr, double %val monotonic, align 8
+ ret void
+}
+
+; Address spaces (add i32), monotonic
+define void @test_add_i32_as0_monotonic(ptr addrspace(0) %addr, i32 %val) {
+; CHECK-LABEL: define void @test_add_i32_as0_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw add ptr [[ADDR]], i32 [[VAL]] monotonic, align 4
+; CHECK-NEXT: ret void
+;
+ storermw add ptr addrspace(0) %addr, i32 %val monotonic, align 4
+ ret void
+}
+
+define void @test_add_i32_as1_monotonic(ptr addrspace(1) %addr, i32 %val) {
+; CHECK-LABEL: define void @test_add_i32_as1_monotonic(
+; CHECK-SAME: ptr addrspace(1) [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw add ptr addrspace(1) [[ADDR]], i32 [[VAL]] monotonic, align 4
+; CHECK-NEXT: ret void
+;
+ storermw add ptr addrspace(1) %addr, i32 %val monotonic, align 4
+ ret void
+}
+
+define void @test_add_i32_as2_monotonic(ptr addrspace(2) %addr, i32 %val) {
+; CHECK-LABEL: define void @test_add_i32_as2_monotonic(
+; CHECK-SAME: ptr addrspace(2) [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw add ptr addrspace(2) [[ADDR]], i32 [[VAL]] monotonic, align 4
+; CHECK-NEXT: ret void
+;
+ storermw add ptr addrspace(2) %addr, i32 %val monotonic, align 4
+ ret void
+}
+
+define void @test_add_i32_as3_monotonic(ptr addrspace(3) %addr, i32 %val) {
+; CHECK-LABEL: define void @test_add_i32_as3_monotonic(
+; CHECK-SAME: ptr addrspace(3) [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw add ptr addrspace(3) [[ADDR]], i32 [[VAL]] monotonic, align 4
+; CHECK-NEXT: ret void
+;
+ storermw add ptr addrspace(3) %addr, i32 %val monotonic, align 4
+ ret void
+}
+
+; Volatile add i32, monotonic
+define void @test_volatile_add_i32_monotonic(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_volatile_add_i32_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw volatile add ptr [[ADDR]], i32 [[VAL]] monotonic, align 4
+; CHECK-NEXT: ret void
+;
+ storermw volatile add ptr %addr, i32 %val monotonic, align 4
+ ret void
+}
+
+; Syncscope singlethread add i32, monotonic
+define void @test_add_i32_syncscope_singlethread_monotonic(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_add_i32_syncscope_singlethread_monotonic(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw add ptr [[ADDR]], i32 [[VAL]] syncscope("singlethread") monotonic, align 4
+; CHECK-NEXT: ret void
+;
+ storermw add ptr %addr, i32 %val syncscope("singlethread") monotonic, align 4
+ ret void
+}
+
+; ===== ordering: release =====
+
+; All LangRef integer ops on i32, release
+define void @test_add_i32_release(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_add_i32_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw add ptr [[ADDR]], i32 [[VAL]] release, align 4
+; CHECK-NEXT: ret void
+;
+ storermw add ptr %addr, i32 %val release, align 4
+ ret void
+}
+
+define void @test_sub_i32_release(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_sub_i32_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw sub ptr [[ADDR]], i32 [[VAL]] release, align 4
+; CHECK-NEXT: ret void
+;
+ storermw sub ptr %addr, i32 %val release, align 4
+ ret void
+}
+
+define void @test_and_i32_release(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_and_i32_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw and ptr [[ADDR]], i32 [[VAL]] release, align 4
+; CHECK-NEXT: ret void
+;
+ storermw and ptr %addr, i32 %val release, align 4
+ ret void
+}
+
+define void @test_nand_i32_release(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_nand_i32_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = and i32 [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[NEW:%.*]] = xor i32 [[TMP2]], -1
+; CHECK-NEXT: [[TMP3:%.*]] = cmpxchg ptr [[ADDR]], i32 [[LOADED]], i32 [[NEW]] release monotonic, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw nand ptr %addr, i32 %val release, align 4
+ ret void
+}
+
+define void @test_or_i32_release(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_or_i32_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw or ptr [[ADDR]], i32 [[VAL]] release, align 4
+; CHECK-NEXT: ret void
+;
+ storermw or ptr %addr, i32 %val release, align 4
+ ret void
+}
+
+define void @test_xor_i32_release(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_xor_i32_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw xor ptr [[ADDR]], i32 [[VAL]] release, align 4
+; CHECK-NEXT: ret void
+;
+ storermw xor ptr %addr, i32 %val release, align 4
+ ret void
+}
+
+define void @test_max_i32_release(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_max_i32_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt i32 [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VAL]]
+; CHECK-NEXT: [[TMP3:%.*]] = cmpxchg ptr [[ADDR]], i32 [[LOADED]], i32 [[NEW]] release monotonic, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw max ptr %addr, i32 %val release, align 4
+ ret void
+}
+
+define void @test_min_i32_release(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_min_i32_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp sle i32 [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VAL]]
+; CHECK-NEXT: [[TMP3:%.*]] = cmpxchg ptr [[ADDR]], i32 [[LOADED]], i32 [[NEW]] release monotonic, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw min ptr %addr, i32 %val release, align 4
+ ret void
+}
+
+define void @test_umax_i32_release(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_umax_i32_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ugt i32 [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VAL]]
+; CHECK-NEXT: [[TMP3:%.*]] = cmpxchg ptr [[ADDR]], i32 [[LOADED]], i32 [[NEW]] release monotonic, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw umax ptr %addr, i32 %val release, align 4
+ ret void
+}
+
+define void @test_umin_i32_release(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_umin_i32_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ule i32 [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VAL]]
+; CHECK-NEXT: [[TMP3:%.*]] = cmpxchg ptr [[ADDR]], i32 [[LOADED]], i32 [[NEW]] release monotonic, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw umin ptr %addr, i32 %val release, align 4
+ ret void
+}
+
+define void @test_uinc_wrap_i32_release(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_uinc_wrap_i32_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[LOADED]], 1
+; CHECK-NEXT: [[TMP3:%.*]] = icmp uge i32 [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[ADDR]], i32 [[LOADED]], i32 [[NEW]] release monotonic, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1
+; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP4]], 0
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw uinc_wrap ptr %addr, i32 %val release, align 4
+ ret void
+}
+
+define void @test_udec_wrap_i32_release(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_udec_wrap_i32_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = sub i32 [[LOADED]], 1
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[LOADED]], 0
+; CHECK-NEXT: [[TMP4:%.*]] = icmp ugt i32 [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
+; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP5]], i32 [[VAL]], i32 [[TMP2]]
+; CHECK-NEXT: [[TMP6:%.*]] = cmpxchg ptr [[ADDR]], i32 [[LOADED]], i32 [[NEW]] release monotonic, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 1
+; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 0
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw udec_wrap ptr %addr, i32 %val release, align 4
+ ret void
+}
+
+define void @test_usub_cond_i32_release(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_usub_cond_i32_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp uge i32 [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[TMP3:%.*]] = sub i32 [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i32 [[TMP3]], i32 [[LOADED]]
+; CHECK-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[ADDR]], i32 [[LOADED]], i32 [[NEW]] release monotonic, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1
+; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP4]], 0
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw usub_cond ptr %addr, i32 %val release, align 4
+ ret void
+}
+
+define void @test_usub_sat_i32_release(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_usub_sat_i32_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[NEW:%.*]] = call i32 @llvm.usub.sat.i32(i32 [[LOADED]], i32 [[VAL]])
+; CHECK-NEXT: [[TMP2:%.*]] = cmpxchg ptr [[ADDR]], i32 [[LOADED]], i32 [[NEW]] release monotonic, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP2]], 0
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw usub_sat ptr %addr, i32 %val release, align 4
+ ret void
+}
+
+; All LangRef FP ops on float, release
+define void @test_fadd_f32_release(ptr %addr, float %val) {
+; CHECK-LABEL: define void @test_fadd_f32_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], float [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32
+; CHECK-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[ADDR]], i32 [[TMP3]], i32 [[TMP2]] release monotonic, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0
+; CHECK-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fadd ptr %addr, float %val release, align 4
+ ret void
+}
+
+define void @test_fsub_f32_release(ptr %addr, float %val) {
+; CHECK-LABEL: define void @test_fsub_f32_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], float [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[NEW:%.*]] = fsub float [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32
+; CHECK-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[ADDR]], i32 [[TMP3]], i32 [[TMP2]] release monotonic, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0
+; CHECK-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fsub ptr %addr, float %val release, align 4
+ ret void
+}
+
+define void @test_fmax_f32_release(ptr %addr, float %val) {
+; CHECK-LABEL: define void @test_fmax_f32_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], float [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call float @llvm.maxnum.f32(float [[LOADED]], float [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast float [[TMP2]] to i32
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i32 [[TMP4]], i32 [[TMP3]] release monotonic, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fmax ptr %addr, float %val release, align 4
+ ret void
+}
+
+define void @test_fmin_f32_release(ptr %addr, float %val) {
+; CHECK-LABEL: define void @test_fmin_f32_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], float [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call float @llvm.minnum.f32(float [[LOADED]], float [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast float [[TMP2]] to i32
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i32 [[TMP4]], i32 [[TMP3]] release monotonic, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fmin ptr %addr, float %val release, align 4
+ ret void
+}
+
+define void @test_fmaximum_f32_release(ptr %addr, float %val) {
+; CHECK-LABEL: define void @test_fmaximum_f32_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], float [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call float @llvm.maximum.f32(float [[LOADED]], float [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast float [[TMP2]] to i32
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i32 [[TMP4]], i32 [[TMP3]] release monotonic, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fmaximum ptr %addr, float %val release, align 4
+ ret void
+}
+
+define void @test_fminimum_f32_release(ptr %addr, float %val) {
+; CHECK-LABEL: define void @test_fminimum_f32_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], float [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call float @llvm.minimum.f32(float [[LOADED]], float [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast float [[TMP2]] to i32
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i32 [[TMP4]], i32 [[TMP3]] release monotonic, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fminimum ptr %addr, float %val release, align 4
+ ret void
+}
+
+define void @test_fmaximumnum_f32_release(ptr %addr, float %val) {
+; CHECK-LABEL: define void @test_fmaximumnum_f32_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], float [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call float @llvm.maximumnum.f32(float [[LOADED]], float [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast float [[TMP2]] to i32
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i32 [[TMP4]], i32 [[TMP3]] release monotonic, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fmaximumnum ptr %addr, float %val release, align 4
+ ret void
+}
+
+define void @test_fminimumnum_f32_release(ptr %addr, float %val) {
+; CHECK-LABEL: define void @test_fminimumnum_f32_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], float [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call float @llvm.minimumnum.f32(float [[LOADED]], float [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast float [[TMP2]] to i32
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i32 [[TMP4]], i32 [[TMP3]] release monotonic, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fminimumnum ptr %addr, float %val release, align 4
+ ret void
+}
+
+; <2 x i32> elementwise (all integer ops), release
+define void @test_add_v2i32_elementwise_release(ptr %addr, <2 x i32> %val) {
+; CHECK-LABEL: define void @test_add_v2i32_elementwise_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x i32> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw elementwise add ptr [[ADDR]], <2 x i32> [[VAL]] release, align 8
+; CHECK-NEXT: ret void
+;
+ storermw elementwise add ptr %addr, <2 x i32> %val release, align 8
+ ret void
+}
+
+define void @test_sub_v2i32_elementwise_release(ptr %addr, <2 x i32> %val) {
+; CHECK-LABEL: define void @test_sub_v2i32_elementwise_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x i32> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw elementwise sub ptr [[ADDR]], <2 x i32> [[VAL]] release, align 8
+; CHECK-NEXT: ret void
+;
+ storermw elementwise sub ptr %addr, <2 x i32> %val release, align 8
+ ret void
+}
+
+define void @test_and_v2i32_elementwise_release(ptr %addr, <2 x i32> %val) {
+; CHECK-LABEL: define void @test_and_v2i32_elementwise_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x i32> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw elementwise and ptr [[ADDR]], <2 x i32> [[VAL]] release, align 8
+; CHECK-NEXT: ret void
+;
+ storermw elementwise and ptr %addr, <2 x i32> %val release, align 8
+ ret void
+}
+
+define void @test_nand_v2i32_elementwise_release(ptr %addr, <2 x i32> %val) {
+; CHECK-LABEL: define void @test_nand_v2i32_elementwise_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x i32> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x i32> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = and <2 x i32> [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[NEW:%.*]] = xor <2 x i32> [[TMP2]], splat (i32 -1)
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x i32> [[NEW]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] release monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x i32>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise nand ptr %addr, <2 x i32> %val release, align 8
+ ret void
+}
+
+define void @test_or_v2i32_elementwise_release(ptr %addr, <2 x i32> %val) {
+; CHECK-LABEL: define void @test_or_v2i32_elementwise_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x i32> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw elementwise or ptr [[ADDR]], <2 x i32> [[VAL]] release, align 8
+; CHECK-NEXT: ret void
+;
+ storermw elementwise or ptr %addr, <2 x i32> %val release, align 8
+ ret void
+}
+
+define void @test_xor_v2i32_elementwise_release(ptr %addr, <2 x i32> %val) {
+; CHECK-LABEL: define void @test_xor_v2i32_elementwise_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x i32> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw elementwise xor ptr [[ADDR]], <2 x i32> [[VAL]] release, align 8
+; CHECK-NEXT: ret void
+;
+ storermw elementwise xor ptr %addr, <2 x i32> %val release, align 8
+ ret void
+}
+
+define void @test_max_v2i32_elementwise_release(ptr %addr, <2 x i32> %val) {
+; CHECK-LABEL: define void @test_max_v2i32_elementwise_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x i32> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x i32> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt <2 x i32> [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[NEW:%.*]] = select <2 x i1> [[TMP2]], <2 x i32> [[LOADED]], <2 x i32> [[VAL]]
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x i32> [[NEW]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] release monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x i32>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise max ptr %addr, <2 x i32> %val release, align 8
+ ret void
+}
+
+define void @test_min_v2i32_elementwise_release(ptr %addr, <2 x i32> %val) {
+; CHECK-LABEL: define void @test_min_v2i32_elementwise_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x i32> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x i32> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp sle <2 x i32> [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[NEW:%.*]] = select <2 x i1> [[TMP2]], <2 x i32> [[LOADED]], <2 x i32> [[VAL]]
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x i32> [[NEW]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] release monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x i32>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise min ptr %addr, <2 x i32> %val release, align 8
+ ret void
+}
+
+define void @test_umax_v2i32_elementwise_release(ptr %addr, <2 x i32> %val) {
+; CHECK-LABEL: define void @test_umax_v2i32_elementwise_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x i32> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x i32> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ugt <2 x i32> [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[NEW:%.*]] = select <2 x i1> [[TMP2]], <2 x i32> [[LOADED]], <2 x i32> [[VAL]]
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x i32> [[NEW]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] release monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x i32>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise umax ptr %addr, <2 x i32> %val release, align 8
+ ret void
+}
+
+define void @test_umin_v2i32_elementwise_release(ptr %addr, <2 x i32> %val) {
+; CHECK-LABEL: define void @test_umin_v2i32_elementwise_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x i32> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x i32> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ule <2 x i32> [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[NEW:%.*]] = select <2 x i1> [[TMP2]], <2 x i32> [[LOADED]], <2 x i32> [[VAL]]
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x i32> [[NEW]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] release monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x i32>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise umin ptr %addr, <2 x i32> %val release, align 8
+ ret void
+}
+
+define void @test_uinc_wrap_v2i32_elementwise_release(ptr %addr, <2 x i32> %val) {
+; CHECK-LABEL: define void @test_uinc_wrap_v2i32_elementwise_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x i32> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x i32> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP7:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = add <2 x i32> [[LOADED]], splat (i32 1)
+; CHECK-NEXT: [[TMP3:%.*]] = icmp uge <2 x i32> [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[NEW:%.*]] = select <2 x i1> [[TMP3]], <2 x i32> zeroinitializer, <2 x i32> [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[NEW]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast <2 x i32> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP6:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP5]], i64 [[TMP4]] release monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP6]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP6]], 0
+; CHECK-NEXT: [[TMP7]] = bitcast i64 [[NEWLOADED]] to <2 x i32>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise uinc_wrap ptr %addr, <2 x i32> %val release, align 8
+ ret void
+}
+
+define void @test_udec_wrap_v2i32_elementwise_release(ptr %addr, <2 x i32> %val) {
+; CHECK-LABEL: define void @test_udec_wrap_v2i32_elementwise_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x i32> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x i32> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP9:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = sub <2 x i32> [[LOADED]], splat (i32 1)
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <2 x i32> [[LOADED]], zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = icmp ugt <2 x i32> [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[TMP5:%.*]] = or <2 x i1> [[TMP3]], [[TMP4]]
+; CHECK-NEXT: [[NEW:%.*]] = select <2 x i1> [[TMP5]], <2 x i32> [[VAL]], <2 x i32> [[TMP2]]
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast <2 x i32> [[NEW]] to i64
+; CHECK-NEXT: [[TMP7:%.*]] = bitcast <2 x i32> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP8:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP7]], i64 [[TMP6]] release monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP8]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP8]], 0
+; CHECK-NEXT: [[TMP9]] = bitcast i64 [[NEWLOADED]] to <2 x i32>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise udec_wrap ptr %addr, <2 x i32> %val release, align 8
+ ret void
+}
+
+define void @test_usub_cond_v2i32_elementwise_release(ptr %addr, <2 x i32> %val) {
+; CHECK-LABEL: define void @test_usub_cond_v2i32_elementwise_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x i32> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x i32> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP7:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp uge <2 x i32> [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[TMP3:%.*]] = sub <2 x i32> [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[NEW:%.*]] = select <2 x i1> [[TMP2]], <2 x i32> [[TMP3]], <2 x i32> [[LOADED]]
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[NEW]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast <2 x i32> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP6:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP5]], i64 [[TMP4]] release monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP6]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP6]], 0
+; CHECK-NEXT: [[TMP7]] = bitcast i64 [[NEWLOADED]] to <2 x i32>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise usub_cond ptr %addr, <2 x i32> %val release, align 8
+ ret void
+}
+
+define void @test_usub_sat_v2i32_elementwise_release(ptr %addr, <2 x i32> %val) {
+; CHECK-LABEL: define void @test_usub_sat_v2i32_elementwise_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x i32> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x i32> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[NEW:%.*]] = call <2 x i32> @llvm.usub.sat.v2i32(<2 x i32> [[LOADED]], <2 x i32> [[VAL]])
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x i32> [[NEW]] to i64
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x i32> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP3]], i64 [[TMP2]] release monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; CHECK-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to <2 x i32>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise usub_sat ptr %addr, <2 x i32> %val release, align 8
+ ret void
+}
+
+; <2 x float> whole-vector (all FP ops), release
+define void @test_fadd_v2f32_whole_release(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fadd_v2f32_whole_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[NEW:%.*]] = fadd <2 x float> [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x float> [[NEW]] to i64
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP3]], i64 [[TMP2]] release monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; CHECK-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fadd ptr %addr, <2 x float> %val release, align 8
+ ret void
+}
+
+define void @test_fsub_v2f32_whole_release(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fsub_v2f32_whole_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[NEW:%.*]] = fsub <2 x float> [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x float> [[NEW]] to i64
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP3]], i64 [[TMP2]] release monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; CHECK-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fsub ptr %addr, <2 x float> %val release, align 8
+ ret void
+}
+
+define void @test_fmax_v2f32_whole_release(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fmax_v2f32_whole_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call <2 x float> @llvm.maxnum.v2f32(<2 x float> [[LOADED]], <2 x float> [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[TMP2]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] release monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fmax ptr %addr, <2 x float> %val release, align 8
+ ret void
+}
+
+define void @test_fmin_v2f32_whole_release(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fmin_v2f32_whole_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call <2 x float> @llvm.minnum.v2f32(<2 x float> [[LOADED]], <2 x float> [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[TMP2]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] release monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fmin ptr %addr, <2 x float> %val release, align 8
+ ret void
+}
+
+define void @test_fmaximum_v2f32_whole_release(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fmaximum_v2f32_whole_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call <2 x float> @llvm.maximum.v2f32(<2 x float> [[LOADED]], <2 x float> [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[TMP2]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] release monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fmaximum ptr %addr, <2 x float> %val release, align 8
+ ret void
+}
+
+define void @test_fminimum_v2f32_whole_release(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fminimum_v2f32_whole_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call <2 x float> @llvm.minimum.v2f32(<2 x float> [[LOADED]], <2 x float> [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[TMP2]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] release monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fminimum ptr %addr, <2 x float> %val release, align 8
+ ret void
+}
+
+define void @test_fmaximumnum_v2f32_whole_release(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fmaximumnum_v2f32_whole_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call <2 x float> @llvm.maximumnum.v2f32(<2 x float> [[LOADED]], <2 x float> [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[TMP2]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] release monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fmaximumnum ptr %addr, <2 x float> %val release, align 8
+ ret void
+}
+
+define void @test_fminimumnum_v2f32_whole_release(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fminimumnum_v2f32_whole_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call <2 x float> @llvm.minimumnum.v2f32(<2 x float> [[LOADED]], <2 x float> [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[TMP2]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] release monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fminimumnum ptr %addr, <2 x float> %val release, align 8
+ ret void
+}
+
+; <2 x float> elementwise (all FP ops), release
+define void @test_fadd_v2f32_elementwise_release(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fadd_v2f32_elementwise_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[NEW:%.*]] = fadd <2 x float> [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x float> [[NEW]] to i64
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP3]], i64 [[TMP2]] release monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; CHECK-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise fadd ptr %addr, <2 x float> %val release, align 8
+ ret void
+}
+
+define void @test_fsub_v2f32_elementwise_release(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fsub_v2f32_elementwise_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[NEW:%.*]] = fsub <2 x float> [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x float> [[NEW]] to i64
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP3]], i64 [[TMP2]] release monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; CHECK-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise fsub ptr %addr, <2 x float> %val release, align 8
+ ret void
+}
+
+define void @test_fmax_v2f32_elementwise_release(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fmax_v2f32_elementwise_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call <2 x float> @llvm.maxnum.v2f32(<2 x float> [[LOADED]], <2 x float> [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[TMP2]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] release monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise fmax ptr %addr, <2 x float> %val release, align 8
+ ret void
+}
+
+define void @test_fmin_v2f32_elementwise_release(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fmin_v2f32_elementwise_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call <2 x float> @llvm.minnum.v2f32(<2 x float> [[LOADED]], <2 x float> [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[TMP2]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] release monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise fmin ptr %addr, <2 x float> %val release, align 8
+ ret void
+}
+
+define void @test_fmaximum_v2f32_elementwise_release(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fmaximum_v2f32_elementwise_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call <2 x float> @llvm.maximum.v2f32(<2 x float> [[LOADED]], <2 x float> [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[TMP2]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] release monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise fmaximum ptr %addr, <2 x float> %val release, align 8
+ ret void
+}
+
+define void @test_fminimum_v2f32_elementwise_release(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fminimum_v2f32_elementwise_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call <2 x float> @llvm.minimum.v2f32(<2 x float> [[LOADED]], <2 x float> [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[TMP2]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] release monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise fminimum ptr %addr, <2 x float> %val release, align 8
+ ret void
+}
+
+define void @test_fmaximumnum_v2f32_elementwise_release(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fmaximumnum_v2f32_elementwise_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call <2 x float> @llvm.maximumnum.v2f32(<2 x float> [[LOADED]], <2 x float> [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[TMP2]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] release monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise fmaximumnum ptr %addr, <2 x float> %val release, align 8
+ ret void
+}
+
+define void @test_fminimumnum_v2f32_elementwise_release(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fminimumnum_v2f32_elementwise_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call <2 x float> @llvm.minimumnum.v2f32(<2 x float> [[LOADED]], <2 x float> [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[TMP2]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] release monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise fminimumnum ptr %addr, <2 x float> %val release, align 8
+ ret void
+}
+
+; Alternate integer sizes for add, release
+define void @test_add_i8_release(ptr %addr, i8 %val) {
+; CHECK-LABEL: define void @test_add_i8_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], i8 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw add ptr [[ADDR]], i8 [[VAL]] release, align 1
+; CHECK-NEXT: ret void
+;
+ storermw add ptr %addr, i8 %val release, align 1
+ ret void
+}
+
+define void @test_add_i16_release(ptr %addr, i16 %val) {
+; CHECK-LABEL: define void @test_add_i16_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], i16 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw add ptr [[ADDR]], i16 [[VAL]] release, align 2
+; CHECK-NEXT: ret void
+;
+ storermw add ptr %addr, i16 %val release, align 2
+ ret void
+}
+
+define void @test_add_i64_release(ptr %addr, i64 %val) {
+; CHECK-LABEL: define void @test_add_i64_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], i64 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw add ptr [[ADDR]], i64 [[VAL]] release, align 8
+; CHECK-NEXT: ret void
+;
+ storermw add ptr %addr, i64 %val release, align 8
+ ret void
+}
+
+; Double for fadd, release
+define void @test_fadd_f64_release(ptr %addr, double %val) {
+; CHECK-LABEL: define void @test_fadd_f64_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], double [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load double, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP3]], i64 [[TMP2]] release monotonic, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; CHECK-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fadd ptr %addr, double %val release, align 8
+ ret void
+}
+
+; Address spaces (add i32), release
+define void @test_add_i32_as0_release(ptr addrspace(0) %addr, i32 %val) {
+; CHECK-LABEL: define void @test_add_i32_as0_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw add ptr [[ADDR]], i32 [[VAL]] release, align 4
+; CHECK-NEXT: ret void
+;
+ storermw add ptr addrspace(0) %addr, i32 %val release, align 4
+ ret void
+}
+
+define void @test_add_i32_as1_release(ptr addrspace(1) %addr, i32 %val) {
+; CHECK-LABEL: define void @test_add_i32_as1_release(
+; CHECK-SAME: ptr addrspace(1) [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw add ptr addrspace(1) [[ADDR]], i32 [[VAL]] release, align 4
+; CHECK-NEXT: ret void
+;
+ storermw add ptr addrspace(1) %addr, i32 %val release, align 4
+ ret void
+}
+
+define void @test_add_i32_as2_release(ptr addrspace(2) %addr, i32 %val) {
+; CHECK-LABEL: define void @test_add_i32_as2_release(
+; CHECK-SAME: ptr addrspace(2) [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw add ptr addrspace(2) [[ADDR]], i32 [[VAL]] release, align 4
+; CHECK-NEXT: ret void
+;
+ storermw add ptr addrspace(2) %addr, i32 %val release, align 4
+ ret void
+}
+
+define void @test_add_i32_as3_release(ptr addrspace(3) %addr, i32 %val) {
+; CHECK-LABEL: define void @test_add_i32_as3_release(
+; CHECK-SAME: ptr addrspace(3) [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw add ptr addrspace(3) [[ADDR]], i32 [[VAL]] release, align 4
+; CHECK-NEXT: ret void
+;
+ storermw add ptr addrspace(3) %addr, i32 %val release, align 4
+ ret void
+}
+
+; Volatile add i32, release
+define void @test_volatile_add_i32_release(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_volatile_add_i32_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw volatile add ptr [[ADDR]], i32 [[VAL]] release, align 4
+; CHECK-NEXT: ret void
+;
+ storermw volatile add ptr %addr, i32 %val release, align 4
+ ret void
+}
+
+; Syncscope singlethread add i32, release
+define void @test_add_i32_syncscope_singlethread_release(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_add_i32_syncscope_singlethread_release(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw add ptr [[ADDR]], i32 [[VAL]] syncscope("singlethread") release, align 4
+; CHECK-NEXT: ret void
+;
+ storermw add ptr %addr, i32 %val syncscope("singlethread") release, align 4
+ ret void
+}
+
+; ===== ordering: seq_cst =====
+
+; All LangRef integer ops on i32, seq_cst
+define void @test_add_i32_seqcst(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_add_i32_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw add ptr [[ADDR]], i32 [[VAL]] seq_cst, align 4
+; CHECK-NEXT: ret void
+;
+ storermw add ptr %addr, i32 %val seq_cst, align 4
+ ret void
+}
+
+define void @test_sub_i32_seqcst(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_sub_i32_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw sub ptr [[ADDR]], i32 [[VAL]] seq_cst, align 4
+; CHECK-NEXT: ret void
+;
+ storermw sub ptr %addr, i32 %val seq_cst, align 4
+ ret void
+}
+
+define void @test_and_i32_seqcst(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_and_i32_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw and ptr [[ADDR]], i32 [[VAL]] seq_cst, align 4
+; CHECK-NEXT: ret void
+;
+ storermw and ptr %addr, i32 %val seq_cst, align 4
+ ret void
+}
+
+define void @test_nand_i32_seqcst(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_nand_i32_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = and i32 [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[NEW:%.*]] = xor i32 [[TMP2]], -1
+; CHECK-NEXT: [[TMP3:%.*]] = cmpxchg ptr [[ADDR]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw nand ptr %addr, i32 %val seq_cst, align 4
+ ret void
+}
+
+define void @test_or_i32_seqcst(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_or_i32_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw or ptr [[ADDR]], i32 [[VAL]] seq_cst, align 4
+; CHECK-NEXT: ret void
+;
+ storermw or ptr %addr, i32 %val seq_cst, align 4
+ ret void
+}
+
+define void @test_xor_i32_seqcst(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_xor_i32_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw xor ptr [[ADDR]], i32 [[VAL]] seq_cst, align 4
+; CHECK-NEXT: ret void
+;
+ storermw xor ptr %addr, i32 %val seq_cst, align 4
+ ret void
+}
+
+define void @test_max_i32_seqcst(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_max_i32_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt i32 [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VAL]]
+; CHECK-NEXT: [[TMP3:%.*]] = cmpxchg ptr [[ADDR]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw max ptr %addr, i32 %val seq_cst, align 4
+ ret void
+}
+
+define void @test_min_i32_seqcst(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_min_i32_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp sle i32 [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VAL]]
+; CHECK-NEXT: [[TMP3:%.*]] = cmpxchg ptr [[ADDR]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw min ptr %addr, i32 %val seq_cst, align 4
+ ret void
+}
+
+define void @test_umax_i32_seqcst(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_umax_i32_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ugt i32 [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VAL]]
+; CHECK-NEXT: [[TMP3:%.*]] = cmpxchg ptr [[ADDR]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw umax ptr %addr, i32 %val seq_cst, align 4
+ ret void
+}
+
+define void @test_umin_i32_seqcst(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_umin_i32_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ule i32 [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VAL]]
+; CHECK-NEXT: [[TMP3:%.*]] = cmpxchg ptr [[ADDR]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw umin ptr %addr, i32 %val seq_cst, align 4
+ ret void
+}
+
+define void @test_uinc_wrap_i32_seqcst(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_uinc_wrap_i32_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[LOADED]], 1
+; CHECK-NEXT: [[TMP3:%.*]] = icmp uge i32 [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[ADDR]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1
+; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP4]], 0
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw uinc_wrap ptr %addr, i32 %val seq_cst, align 4
+ ret void
+}
+
+define void @test_udec_wrap_i32_seqcst(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_udec_wrap_i32_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = sub i32 [[LOADED]], 1
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[LOADED]], 0
+; CHECK-NEXT: [[TMP4:%.*]] = icmp ugt i32 [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
+; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP5]], i32 [[VAL]], i32 [[TMP2]]
+; CHECK-NEXT: [[TMP6:%.*]] = cmpxchg ptr [[ADDR]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 1
+; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 0
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw udec_wrap ptr %addr, i32 %val seq_cst, align 4
+ ret void
+}
+
+define void @test_usub_cond_i32_seqcst(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_usub_cond_i32_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp uge i32 [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[TMP3:%.*]] = sub i32 [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i32 [[TMP3]], i32 [[LOADED]]
+; CHECK-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[ADDR]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1
+; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP4]], 0
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw usub_cond ptr %addr, i32 %val seq_cst, align 4
+ ret void
+}
+
+define void @test_usub_sat_i32_seqcst(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_usub_sat_i32_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[NEW:%.*]] = call i32 @llvm.usub.sat.i32(i32 [[LOADED]], i32 [[VAL]])
+; CHECK-NEXT: [[TMP2:%.*]] = cmpxchg ptr [[ADDR]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP2]], 0
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw usub_sat ptr %addr, i32 %val seq_cst, align 4
+ ret void
+}
+
+; All LangRef FP ops on float, seq_cst
+define void @test_fadd_f32_seqcst(ptr %addr, float %val) {
+; CHECK-LABEL: define void @test_fadd_f32_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], float [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32
+; CHECK-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[ADDR]], i32 [[TMP3]], i32 [[TMP2]] seq_cst seq_cst, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0
+; CHECK-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fadd ptr %addr, float %val seq_cst, align 4
+ ret void
+}
+
+define void @test_fsub_f32_seqcst(ptr %addr, float %val) {
+; CHECK-LABEL: define void @test_fsub_f32_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], float [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[NEW:%.*]] = fsub float [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32
+; CHECK-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[ADDR]], i32 [[TMP3]], i32 [[TMP2]] seq_cst seq_cst, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0
+; CHECK-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fsub ptr %addr, float %val seq_cst, align 4
+ ret void
+}
+
+define void @test_fmax_f32_seqcst(ptr %addr, float %val) {
+; CHECK-LABEL: define void @test_fmax_f32_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], float [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call float @llvm.maxnum.f32(float [[LOADED]], float [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast float [[TMP2]] to i32
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fmax ptr %addr, float %val seq_cst, align 4
+ ret void
+}
+
+define void @test_fmin_f32_seqcst(ptr %addr, float %val) {
+; CHECK-LABEL: define void @test_fmin_f32_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], float [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call float @llvm.minnum.f32(float [[LOADED]], float [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast float [[TMP2]] to i32
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fmin ptr %addr, float %val seq_cst, align 4
+ ret void
+}
+
+define void @test_fmaximum_f32_seqcst(ptr %addr, float %val) {
+; CHECK-LABEL: define void @test_fmaximum_f32_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], float [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call float @llvm.maximum.f32(float [[LOADED]], float [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast float [[TMP2]] to i32
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fmaximum ptr %addr, float %val seq_cst, align 4
+ ret void
+}
+
+define void @test_fminimum_f32_seqcst(ptr %addr, float %val) {
+; CHECK-LABEL: define void @test_fminimum_f32_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], float [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call float @llvm.minimum.f32(float [[LOADED]], float [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast float [[TMP2]] to i32
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fminimum ptr %addr, float %val seq_cst, align 4
+ ret void
+}
+
+define void @test_fmaximumnum_f32_seqcst(ptr %addr, float %val) {
+; CHECK-LABEL: define void @test_fmaximumnum_f32_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], float [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call float @llvm.maximumnum.f32(float [[LOADED]], float [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast float [[TMP2]] to i32
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fmaximumnum ptr %addr, float %val seq_cst, align 4
+ ret void
+}
+
+define void @test_fminimumnum_f32_seqcst(ptr %addr, float %val) {
+; CHECK-LABEL: define void @test_fminimumnum_f32_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], float [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[ADDR]], align 4
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call float @llvm.minimumnum.f32(float [[LOADED]], float [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast float [[TMP2]] to i32
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fminimumnum ptr %addr, float %val seq_cst, align 4
+ ret void
+}
+
+; <2 x i32> elementwise (all integer ops), seq_cst
+define void @test_add_v2i32_elementwise_seqcst(ptr %addr, <2 x i32> %val) {
+; CHECK-LABEL: define void @test_add_v2i32_elementwise_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x i32> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw elementwise add ptr [[ADDR]], <2 x i32> [[VAL]] seq_cst, align 8
+; CHECK-NEXT: ret void
+;
+ storermw elementwise add ptr %addr, <2 x i32> %val seq_cst, align 8
+ ret void
+}
+
+define void @test_sub_v2i32_elementwise_seqcst(ptr %addr, <2 x i32> %val) {
+; CHECK-LABEL: define void @test_sub_v2i32_elementwise_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x i32> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw elementwise sub ptr [[ADDR]], <2 x i32> [[VAL]] seq_cst, align 8
+; CHECK-NEXT: ret void
+;
+ storermw elementwise sub ptr %addr, <2 x i32> %val seq_cst, align 8
+ ret void
+}
+
+define void @test_and_v2i32_elementwise_seqcst(ptr %addr, <2 x i32> %val) {
+; CHECK-LABEL: define void @test_and_v2i32_elementwise_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x i32> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw elementwise and ptr [[ADDR]], <2 x i32> [[VAL]] seq_cst, align 8
+; CHECK-NEXT: ret void
+;
+ storermw elementwise and ptr %addr, <2 x i32> %val seq_cst, align 8
+ ret void
+}
+
+define void @test_nand_v2i32_elementwise_seqcst(ptr %addr, <2 x i32> %val) {
+; CHECK-LABEL: define void @test_nand_v2i32_elementwise_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x i32> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x i32> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = and <2 x i32> [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[NEW:%.*]] = xor <2 x i32> [[TMP2]], splat (i32 -1)
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x i32> [[NEW]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x i32>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise nand ptr %addr, <2 x i32> %val seq_cst, align 8
+ ret void
+}
+
+define void @test_or_v2i32_elementwise_seqcst(ptr %addr, <2 x i32> %val) {
+; CHECK-LABEL: define void @test_or_v2i32_elementwise_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x i32> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw elementwise or ptr [[ADDR]], <2 x i32> [[VAL]] seq_cst, align 8
+; CHECK-NEXT: ret void
+;
+ storermw elementwise or ptr %addr, <2 x i32> %val seq_cst, align 8
+ ret void
+}
+
+define void @test_xor_v2i32_elementwise_seqcst(ptr %addr, <2 x i32> %val) {
+; CHECK-LABEL: define void @test_xor_v2i32_elementwise_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x i32> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw elementwise xor ptr [[ADDR]], <2 x i32> [[VAL]] seq_cst, align 8
+; CHECK-NEXT: ret void
+;
+ storermw elementwise xor ptr %addr, <2 x i32> %val seq_cst, align 8
+ ret void
+}
+
+define void @test_max_v2i32_elementwise_seqcst(ptr %addr, <2 x i32> %val) {
+; CHECK-LABEL: define void @test_max_v2i32_elementwise_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x i32> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x i32> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt <2 x i32> [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[NEW:%.*]] = select <2 x i1> [[TMP2]], <2 x i32> [[LOADED]], <2 x i32> [[VAL]]
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x i32> [[NEW]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x i32>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise max ptr %addr, <2 x i32> %val seq_cst, align 8
+ ret void
+}
+
+define void @test_min_v2i32_elementwise_seqcst(ptr %addr, <2 x i32> %val) {
+; CHECK-LABEL: define void @test_min_v2i32_elementwise_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x i32> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x i32> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp sle <2 x i32> [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[NEW:%.*]] = select <2 x i1> [[TMP2]], <2 x i32> [[LOADED]], <2 x i32> [[VAL]]
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x i32> [[NEW]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x i32>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise min ptr %addr, <2 x i32> %val seq_cst, align 8
+ ret void
+}
+
+define void @test_umax_v2i32_elementwise_seqcst(ptr %addr, <2 x i32> %val) {
+; CHECK-LABEL: define void @test_umax_v2i32_elementwise_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x i32> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x i32> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ugt <2 x i32> [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[NEW:%.*]] = select <2 x i1> [[TMP2]], <2 x i32> [[LOADED]], <2 x i32> [[VAL]]
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x i32> [[NEW]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x i32>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise umax ptr %addr, <2 x i32> %val seq_cst, align 8
+ ret void
+}
+
+define void @test_umin_v2i32_elementwise_seqcst(ptr %addr, <2 x i32> %val) {
+; CHECK-LABEL: define void @test_umin_v2i32_elementwise_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x i32> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x i32> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ule <2 x i32> [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[NEW:%.*]] = select <2 x i1> [[TMP2]], <2 x i32> [[LOADED]], <2 x i32> [[VAL]]
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x i32> [[NEW]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x i32>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise umin ptr %addr, <2 x i32> %val seq_cst, align 8
+ ret void
+}
+
+define void @test_uinc_wrap_v2i32_elementwise_seqcst(ptr %addr, <2 x i32> %val) {
+; CHECK-LABEL: define void @test_uinc_wrap_v2i32_elementwise_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x i32> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x i32> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP7:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = add <2 x i32> [[LOADED]], splat (i32 1)
+; CHECK-NEXT: [[TMP3:%.*]] = icmp uge <2 x i32> [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[NEW:%.*]] = select <2 x i1> [[TMP3]], <2 x i32> zeroinitializer, <2 x i32> [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[NEW]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast <2 x i32> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP6:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP5]], i64 [[TMP4]] seq_cst seq_cst, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP6]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP6]], 0
+; CHECK-NEXT: [[TMP7]] = bitcast i64 [[NEWLOADED]] to <2 x i32>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise uinc_wrap ptr %addr, <2 x i32> %val seq_cst, align 8
+ ret void
+}
+
+define void @test_udec_wrap_v2i32_elementwise_seqcst(ptr %addr, <2 x i32> %val) {
+; CHECK-LABEL: define void @test_udec_wrap_v2i32_elementwise_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x i32> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x i32> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP9:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = sub <2 x i32> [[LOADED]], splat (i32 1)
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <2 x i32> [[LOADED]], zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = icmp ugt <2 x i32> [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[TMP5:%.*]] = or <2 x i1> [[TMP3]], [[TMP4]]
+; CHECK-NEXT: [[NEW:%.*]] = select <2 x i1> [[TMP5]], <2 x i32> [[VAL]], <2 x i32> [[TMP2]]
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast <2 x i32> [[NEW]] to i64
+; CHECK-NEXT: [[TMP7:%.*]] = bitcast <2 x i32> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP8:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP7]], i64 [[TMP6]] seq_cst seq_cst, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP8]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP8]], 0
+; CHECK-NEXT: [[TMP9]] = bitcast i64 [[NEWLOADED]] to <2 x i32>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise udec_wrap ptr %addr, <2 x i32> %val seq_cst, align 8
+ ret void
+}
+
+define void @test_usub_cond_v2i32_elementwise_seqcst(ptr %addr, <2 x i32> %val) {
+; CHECK-LABEL: define void @test_usub_cond_v2i32_elementwise_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x i32> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x i32> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP7:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp uge <2 x i32> [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[TMP3:%.*]] = sub <2 x i32> [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[NEW:%.*]] = select <2 x i1> [[TMP2]], <2 x i32> [[TMP3]], <2 x i32> [[LOADED]]
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[NEW]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast <2 x i32> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP6:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP5]], i64 [[TMP4]] seq_cst seq_cst, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP6]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP6]], 0
+; CHECK-NEXT: [[TMP7]] = bitcast i64 [[NEWLOADED]] to <2 x i32>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise usub_cond ptr %addr, <2 x i32> %val seq_cst, align 8
+ ret void
+}
+
+define void @test_usub_sat_v2i32_elementwise_seqcst(ptr %addr, <2 x i32> %val) {
+; CHECK-LABEL: define void @test_usub_sat_v2i32_elementwise_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x i32> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x i32> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[NEW:%.*]] = call <2 x i32> @llvm.usub.sat.v2i32(<2 x i32> [[LOADED]], <2 x i32> [[VAL]])
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x i32> [[NEW]] to i64
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x i32> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; CHECK-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to <2 x i32>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise usub_sat ptr %addr, <2 x i32> %val seq_cst, align 8
+ ret void
+}
+
+; <2 x float> whole-vector (all FP ops), seq_cst
+define void @test_fadd_v2f32_whole_seqcst(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fadd_v2f32_whole_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[NEW:%.*]] = fadd <2 x float> [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x float> [[NEW]] to i64
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; CHECK-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fadd ptr %addr, <2 x float> %val seq_cst, align 8
+ ret void
+}
+
+define void @test_fsub_v2f32_whole_seqcst(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fsub_v2f32_whole_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[NEW:%.*]] = fsub <2 x float> [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x float> [[NEW]] to i64
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; CHECK-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fsub ptr %addr, <2 x float> %val seq_cst, align 8
+ ret void
+}
+
+define void @test_fmax_v2f32_whole_seqcst(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fmax_v2f32_whole_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call <2 x float> @llvm.maxnum.v2f32(<2 x float> [[LOADED]], <2 x float> [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[TMP2]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fmax ptr %addr, <2 x float> %val seq_cst, align 8
+ ret void
+}
+
+define void @test_fmin_v2f32_whole_seqcst(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fmin_v2f32_whole_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call <2 x float> @llvm.minnum.v2f32(<2 x float> [[LOADED]], <2 x float> [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[TMP2]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fmin ptr %addr, <2 x float> %val seq_cst, align 8
+ ret void
+}
+
+define void @test_fmaximum_v2f32_whole_seqcst(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fmaximum_v2f32_whole_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call <2 x float> @llvm.maximum.v2f32(<2 x float> [[LOADED]], <2 x float> [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[TMP2]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fmaximum ptr %addr, <2 x float> %val seq_cst, align 8
+ ret void
+}
+
+define void @test_fminimum_v2f32_whole_seqcst(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fminimum_v2f32_whole_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call <2 x float> @llvm.minimum.v2f32(<2 x float> [[LOADED]], <2 x float> [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[TMP2]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fminimum ptr %addr, <2 x float> %val seq_cst, align 8
+ ret void
+}
+
+define void @test_fmaximumnum_v2f32_whole_seqcst(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fmaximumnum_v2f32_whole_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call <2 x float> @llvm.maximumnum.v2f32(<2 x float> [[LOADED]], <2 x float> [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[TMP2]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fmaximumnum ptr %addr, <2 x float> %val seq_cst, align 8
+ ret void
+}
+
+define void @test_fminimumnum_v2f32_whole_seqcst(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fminimumnum_v2f32_whole_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call <2 x float> @llvm.minimumnum.v2f32(<2 x float> [[LOADED]], <2 x float> [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[TMP2]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fminimumnum ptr %addr, <2 x float> %val seq_cst, align 8
+ ret void
+}
+
+; <2 x float> elementwise (all FP ops), seq_cst
+define void @test_fadd_v2f32_elementwise_seqcst(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fadd_v2f32_elementwise_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[NEW:%.*]] = fadd <2 x float> [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x float> [[NEW]] to i64
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; CHECK-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise fadd ptr %addr, <2 x float> %val seq_cst, align 8
+ ret void
+}
+
+define void @test_fsub_v2f32_elementwise_seqcst(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fsub_v2f32_elementwise_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[NEW:%.*]] = fsub <2 x float> [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x float> [[NEW]] to i64
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; CHECK-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise fsub ptr %addr, <2 x float> %val seq_cst, align 8
+ ret void
+}
+
+define void @test_fmax_v2f32_elementwise_seqcst(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fmax_v2f32_elementwise_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call <2 x float> @llvm.maxnum.v2f32(<2 x float> [[LOADED]], <2 x float> [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[TMP2]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise fmax ptr %addr, <2 x float> %val seq_cst, align 8
+ ret void
+}
+
+define void @test_fmin_v2f32_elementwise_seqcst(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fmin_v2f32_elementwise_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call <2 x float> @llvm.minnum.v2f32(<2 x float> [[LOADED]], <2 x float> [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[TMP2]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise fmin ptr %addr, <2 x float> %val seq_cst, align 8
+ ret void
+}
+
+define void @test_fmaximum_v2f32_elementwise_seqcst(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fmaximum_v2f32_elementwise_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call <2 x float> @llvm.maximum.v2f32(<2 x float> [[LOADED]], <2 x float> [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[TMP2]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise fmaximum ptr %addr, <2 x float> %val seq_cst, align 8
+ ret void
+}
+
+define void @test_fminimum_v2f32_elementwise_seqcst(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fminimum_v2f32_elementwise_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call <2 x float> @llvm.minimum.v2f32(<2 x float> [[LOADED]], <2 x float> [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[TMP2]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise fminimum ptr %addr, <2 x float> %val seq_cst, align 8
+ ret void
+}
+
+define void @test_fmaximumnum_v2f32_elementwise_seqcst(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fmaximumnum_v2f32_elementwise_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call <2 x float> @llvm.maximumnum.v2f32(<2 x float> [[LOADED]], <2 x float> [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[TMP2]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise fmaximumnum ptr %addr, <2 x float> %val seq_cst, align 8
+ ret void
+}
+
+define void @test_fminimumnum_v2f32_elementwise_seqcst(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: define void @test_fminimumnum_v2f32_elementwise_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <2 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call <2 x float> @llvm.minimumnum.v2f32(<2 x float> [[LOADED]], <2 x float> [[VAL]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x float> [[TMP2]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x float> [[LOADED]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to <2 x float>
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw elementwise fminimumnum ptr %addr, <2 x float> %val seq_cst, align 8
+ ret void
+}
+
+; Alternate integer sizes for add, seq_cst
+define void @test_add_i8_seqcst(ptr %addr, i8 %val) {
+; CHECK-LABEL: define void @test_add_i8_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], i8 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw add ptr [[ADDR]], i8 [[VAL]] seq_cst, align 1
+; CHECK-NEXT: ret void
+;
+ storermw add ptr %addr, i8 %val seq_cst, align 1
+ ret void
+}
+
+define void @test_add_i16_seqcst(ptr %addr, i16 %val) {
+; CHECK-LABEL: define void @test_add_i16_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], i16 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw add ptr [[ADDR]], i16 [[VAL]] seq_cst, align 2
+; CHECK-NEXT: ret void
+;
+ storermw add ptr %addr, i16 %val seq_cst, align 2
+ ret void
+}
+
+define void @test_add_i64_seqcst(ptr %addr, i64 %val) {
+; CHECK-LABEL: define void @test_add_i64_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], i64 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw add ptr [[ADDR]], i64 [[VAL]] seq_cst, align 8
+; CHECK-NEXT: ret void
+;
+ storermw add ptr %addr, i64 %val seq_cst, align 8
+ ret void
+}
+
+; Double for fadd, seq_cst
+define void @test_fadd_f64_seqcst(ptr %addr, double %val) {
+; CHECK-LABEL: define void @test_fadd_f64_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], double [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load double, ptr [[ADDR]], align 8
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VAL]]
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[ADDR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; CHECK-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret void
+;
+ storermw fadd ptr %addr, double %val seq_cst, align 8
+ ret void
+}
+
+; Address spaces (add i32), seq_cst
+define void @test_add_i32_as0_seqcst(ptr addrspace(0) %addr, i32 %val) {
+; CHECK-LABEL: define void @test_add_i32_as0_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw add ptr [[ADDR]], i32 [[VAL]] seq_cst, align 4
+; CHECK-NEXT: ret void
+;
+ storermw add ptr addrspace(0) %addr, i32 %val seq_cst, align 4
+ ret void
+}
+
+define void @test_add_i32_as1_seqcst(ptr addrspace(1) %addr, i32 %val) {
+; CHECK-LABEL: define void @test_add_i32_as1_seqcst(
+; CHECK-SAME: ptr addrspace(1) [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw add ptr addrspace(1) [[ADDR]], i32 [[VAL]] seq_cst, align 4
+; CHECK-NEXT: ret void
+;
+ storermw add ptr addrspace(1) %addr, i32 %val seq_cst, align 4
+ ret void
+}
+
+define void @test_add_i32_as2_seqcst(ptr addrspace(2) %addr, i32 %val) {
+; CHECK-LABEL: define void @test_add_i32_as2_seqcst(
+; CHECK-SAME: ptr addrspace(2) [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw add ptr addrspace(2) [[ADDR]], i32 [[VAL]] seq_cst, align 4
+; CHECK-NEXT: ret void
+;
+ storermw add ptr addrspace(2) %addr, i32 %val seq_cst, align 4
+ ret void
+}
+
+define void @test_add_i32_as3_seqcst(ptr addrspace(3) %addr, i32 %val) {
+; CHECK-LABEL: define void @test_add_i32_as3_seqcst(
+; CHECK-SAME: ptr addrspace(3) [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw add ptr addrspace(3) [[ADDR]], i32 [[VAL]] seq_cst, align 4
+; CHECK-NEXT: ret void
+;
+ storermw add ptr addrspace(3) %addr, i32 %val seq_cst, align 4
+ ret void
+}
+
+; Volatile add i32, seq_cst
+define void @test_volatile_add_i32_seqcst(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_volatile_add_i32_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw volatile add ptr [[ADDR]], i32 [[VAL]] seq_cst, align 4
+; CHECK-NEXT: ret void
+;
+ storermw volatile add ptr %addr, i32 %val seq_cst, align 4
+ ret void
+}
+
+; Syncscope singlethread add i32, seq_cst
+define void @test_add_i32_syncscope_singlethread_seqcst(ptr %addr, i32 %val) {
+; CHECK-LABEL: define void @test_add_i32_syncscope_singlethread_seqcst(
+; CHECK-SAME: ptr [[ADDR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw add ptr [[ADDR]], i32 [[VAL]] syncscope("singlethread") seq_cst, align 4
+; CHECK-NEXT: ret void
+;
+ storermw add ptr %addr, i32 %val syncscope("singlethread") seq_cst, align 4
+ ret void
+}
+
diff --git a/llvm/test/CodeGen/Generic/storermw_gen.py b/llvm/test/CodeGen/Generic/storermw_gen.py
new file mode 100644
index 0000000000000..ef20eede34670
--- /dev/null
+++ b/llvm/test/CodeGen/Generic/storermw_gen.py
@@ -0,0 +1,206 @@
+#!/usr/bin/env python3
+"""Generate llvm/test/CodeGen/Generic/storermw.ll.
+
+Emits IR only (no CHECK lines). After running this script, regenerate the
+FileCheck assertions with:
+ llvm/utils/update_test_checks.py --opt-binary <path-to-opt> \\
+ llvm/test/CodeGen/Generic/storermw.ll
+
+Coverage:
+ - Every LangRef op (22 total) crossed with every valid ordering
+ (monotonic, release, seq_cst) on scalar i32/float.
+ - <2 x i32> elementwise for the 14 integer ops, all 4 orderings.
+ - <2 x float> whole-vector and elementwise for the 8 FP ops, all 4 orderings.
+ - Alternate integer sizes (i8, i16, i64) and FP type (double) for `add`/`fadd`.
+ - Address spaces 0-3 for `add` on i32, all 4 orderings.
+ - Volatile + syncscope variants for `add` i32.
+"""
+
+ORDERINGS = ["monotonic", "release", "seq_cst"]
+
+INT_OPS = [
+ "add",
+ "sub",
+ "and",
+ "nand",
+ "or",
+ "xor",
+ "max",
+ "min",
+ "umax",
+ "umin",
+ "uinc_wrap",
+ "udec_wrap",
+ "usub_cond",
+ "usub_sat",
+]
+FP_OPS = [
+ "fadd",
+ "fsub",
+ "fmax",
+ "fmin",
+ "fmaximum",
+ "fminimum",
+ "fmaximumnum",
+ "fminimumnum",
+]
+
+
+def fn(
+ name,
+ ptr_ty,
+ val_ty,
+ op,
+ ordering,
+ align,
+ elementwise=False,
+ volatile=False,
+ syncscope=None,
+):
+ ew = "elementwise " if elementwise else ""
+ vol = "volatile " if volatile else ""
+ ss = f'syncscope("{syncscope}") ' if syncscope else ""
+ return (
+ f"define void @{name}({ptr_ty} %addr, {val_ty} %val) {{\n"
+ f" storermw {vol}{ew}{op} {ptr_ty} %addr, {val_ty} %val "
+ f"{ss}{ordering}, align {align}\n"
+ f" ret void\n"
+ f"}}\n\n"
+ )
+
+
+def ord_suffix(ordering):
+ return ordering.replace("_", "") # seq_cst -> seqcst
+
+
+def gen():
+ out = []
+ out.append(
+ "; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6\n"
+ )
+ out.append(
+ "; RUN: opt < %s -passes='require<libcall-lowering-info>,atomic-expand' "
+ "-mtriple=x86_64-unknown-linux-gnu -S | FileCheck %s\n"
+ )
+ out.append("; Test AtomicExpandPass handling of storermw:\n")
+ out.append("; monotonic/release/seq_cst -> atomicrmw with dead result\n")
+ out.append(
+ "; Generated by storermw_gen.py -- do not edit function bodies by hand;\n"
+ )
+ out.append("; edit the generator and rerun it, then update-opt-checks.\n\n")
+
+ for ordering in ORDERINGS:
+ suf = ord_suffix(ordering)
+ out.append(f"; ===== ordering: {ordering} =====\n\n")
+
+ out.append(f"; All LangRef integer ops on i32, {ordering}\n")
+ for op in INT_OPS:
+ out.append(fn(f"test_{op}_i32_{suf}", "ptr", "i32", op, ordering, 4))
+
+ out.append(f"; All LangRef FP ops on float, {ordering}\n")
+ for op in FP_OPS:
+ out.append(fn(f"test_{op}_f32_{suf}", "ptr", "float", op, ordering, 4))
+
+ out.append(f"; <2 x i32> elementwise (all integer ops), {ordering}\n")
+ for op in INT_OPS:
+ out.append(
+ fn(
+ f"test_{op}_v2i32_elementwise_{suf}",
+ "ptr",
+ "<2 x i32>",
+ op,
+ ordering,
+ 8,
+ elementwise=True,
+ )
+ )
+
+ out.append(f"; <2 x float> whole-vector (all FP ops), {ordering}\n")
+ for op in FP_OPS:
+ out.append(
+ fn(
+ f"test_{op}_v2f32_whole_{suf}",
+ "ptr",
+ "<2 x float>",
+ op,
+ ordering,
+ 8,
+ elementwise=False,
+ )
+ )
+
+ out.append(f"; <2 x float> elementwise (all FP ops), {ordering}\n")
+ for op in FP_OPS:
+ out.append(
+ fn(
+ f"test_{op}_v2f32_elementwise_{suf}",
+ "ptr",
+ "<2 x float>",
+ op,
+ ordering,
+ 8,
+ elementwise=True,
+ )
+ )
+
+ out.append(f"; Alternate integer sizes for add, {ordering}\n")
+ for ty, align in [("i8", 1), ("i16", 2), ("i64", 8)]:
+ out.append(fn(f"test_add_{ty}_{suf}", "ptr", ty, "add", ordering, align))
+
+ out.append(f"; Double for fadd, {ordering}\n")
+ out.append(fn(f"test_fadd_f64_{suf}", "ptr", "double", "fadd", ordering, 8))
+
+ out.append(f"; Address spaces (add i32), {ordering}\n")
+ for asn in [0, 1, 2, 3]:
+ out.append(
+ fn(
+ f"test_add_i32_as{asn}_{suf}",
+ f"ptr addrspace({asn})",
+ "i32",
+ "add",
+ ordering,
+ 4,
+ )
+ )
+
+ out.append(f"; Volatile add i32, {ordering}\n")
+ out.append(
+ fn(
+ f"test_volatile_add_i32_{suf}",
+ "ptr",
+ "i32",
+ "add",
+ ordering,
+ 4,
+ volatile=True,
+ )
+ )
+
+ out.append(f"; Syncscope singlethread add i32, {ordering}\n")
+ out.append(
+ fn(
+ f"test_add_i32_syncscope_singlethread_{suf}",
+ "ptr",
+ "i32",
+ "add",
+ ordering,
+ 4,
+ syncscope="singlethread",
+ )
+ )
+
+ return "".join(out)
+
+
+def main():
+ import os
+
+ here = os.path.dirname(os.path.abspath(__file__))
+ path = os.path.join(here, "storermw.ll")
+ with open(path, "w") as f:
+ f.write(gen())
+ print(f"wrote {path} (run update-opt-checks next)")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/llvm/test/Verifier/storermw-invalid/acq_rel.ll b/llvm/test/Verifier/storermw-invalid/acq_rel.ll
new file mode 100644
index 0000000000000..35bbfb5f75e8b
--- /dev/null
+++ b/llvm/test/Verifier/storermw-invalid/acq_rel.ll
@@ -0,0 +1,7 @@
+; RUN: not opt -S -passes=verify < %s 2>&1 | FileCheck %s
+; CHECK: storermw cannot have acquire semantics
+
+define void @test(ptr %ptr, i32 %val) {
+ storermw add ptr %ptr, i32 %val acq_rel, align 4
+ ret void
+}
diff --git a/llvm/test/Verifier/storermw-invalid/acquire.ll b/llvm/test/Verifier/storermw-invalid/acquire.ll
new file mode 100644
index 0000000000000..7cfe3838e05a9
--- /dev/null
+++ b/llvm/test/Verifier/storermw-invalid/acquire.ll
@@ -0,0 +1,7 @@
+; RUN: not opt -S -passes=verify < %s 2>&1 | FileCheck %s
+; CHECK: storermw cannot have acquire semantics
+
+define void @test(ptr %ptr, i32 %val) {
+ storermw add ptr %ptr, i32 %val acquire, align 4
+ ret void
+}
diff --git a/llvm/test/Verifier/storermw-invalid/unordered.ll b/llvm/test/Verifier/storermw-invalid/unordered.ll
new file mode 100644
index 0000000000000..3419284e42a7d
--- /dev/null
+++ b/llvm/test/Verifier/storermw-invalid/unordered.ll
@@ -0,0 +1,7 @@
+; RUN: not opt -S -passes=verify < %s 2>&1 | FileCheck %s
+; CHECK: storermw must be at least monotonic
+
+define void @test(ptr %ptr, i32 %val) {
+ storermw add ptr %ptr, i32 %val unordered, align 4
+ ret void
+}
diff --git a/llvm/test/Verifier/storermw-invalid/xchg.ll b/llvm/test/Verifier/storermw-invalid/xchg.ll
new file mode 100644
index 0000000000000..11f4fb7698cab
--- /dev/null
+++ b/llvm/test/Verifier/storermw-invalid/xchg.ll
@@ -0,0 +1,7 @@
+; RUN: not opt -S -passes=verify < %s 2>&1 | FileCheck %s
+; CHECK: xchg operation not valid for storermw
+
+define void @test(ptr %ptr, i32 %val) {
+ storermw xchg ptr %ptr, i32 %val seq_cst, align 4
+ ret void
+}
diff --git a/llvm/test/tools/llvm-ir2vec/entities.ll b/llvm/test/tools/llvm-ir2vec/entities.ll
index 002614164c128..b3e81be05731e 100644
--- a/llvm/test/tools/llvm-ir2vec/entities.ll
+++ b/llvm/test/tools/llvm-ir2vec/entities.ll
@@ -1,6 +1,6 @@
; RUN: llvm-ir2vec entities | FileCheck %s
-CHECK: 112
+CHECK: 113
CHECK-NEXT: Ret 0
CHECK-NEXT: UncondBr 1
CHECK-NEXT: CondBr 2
@@ -70,46 +70,47 @@ CHECK-NEXT: ExtractValue 65
CHECK-NEXT: InsertValue 66
CHECK-NEXT: LandingPad 67
CHECK-NEXT: Freeze 68
-CHECK-NEXT: FloatTy 69
-CHECK-NEXT: VoidTy 70
-CHECK-NEXT: LabelTy 71
-CHECK-NEXT: MetadataTy 72
-CHECK-NEXT: VectorTy 73
-CHECK-NEXT: TokenTy 74
-CHECK-NEXT: IntegerTy 75
-CHECK-NEXT: ByteTy 76
-CHECK-NEXT: FunctionTy 77
-CHECK-NEXT: PointerTy 78
-CHECK-NEXT: StructTy 79
-CHECK-NEXT: ArrayTy 80
-CHECK-NEXT: UnknownTy 81
-CHECK-NEXT: Function 82
-CHECK-NEXT: Pointer 83
-CHECK-NEXT: Constant 84
-CHECK-NEXT: Variable 85
-CHECK-NEXT: FCMP_false 86
-CHECK-NEXT: FCMP_oeq 87
-CHECK-NEXT: FCMP_ogt 88
-CHECK-NEXT: FCMP_oge 89
-CHECK-NEXT: FCMP_olt 90
-CHECK-NEXT: FCMP_ole 91
-CHECK-NEXT: FCMP_one 92
-CHECK-NEXT: FCMP_ord 93
-CHECK-NEXT: FCMP_uno 94
-CHECK-NEXT: FCMP_ueq 95
-CHECK-NEXT: FCMP_ugt 96
-CHECK-NEXT: FCMP_uge 97
-CHECK-NEXT: FCMP_ult 98
-CHECK-NEXT: FCMP_ule 99
-CHECK-NEXT: FCMP_une 100
-CHECK-NEXT: FCMP_true 101
-CHECK-NEXT: ICMP_eq 102
-CHECK-NEXT: ICMP_ne 103
-CHECK-NEXT: ICMP_ugt 104
-CHECK-NEXT: ICMP_uge 105
-CHECK-NEXT: ICMP_ult 106
-CHECK-NEXT: ICMP_ule 107
-CHECK-NEXT: ICMP_sgt 108
-CHECK-NEXT: ICMP_sge 109
-CHECK-NEXT: ICMP_slt 110
-CHECK-NEXT: ICMP_sle 111
+CHECK-NEXT: StoreRMW 69
+CHECK-NEXT: FloatTy 70
+CHECK-NEXT: VoidTy 71
+CHECK-NEXT: LabelTy 72
+CHECK-NEXT: MetadataTy 73
+CHECK-NEXT: VectorTy 74
+CHECK-NEXT: TokenTy 75
+CHECK-NEXT: IntegerTy 76
+CHECK-NEXT: ByteTy 77
+CHECK-NEXT: FunctionTy 78
+CHECK-NEXT: PointerTy 79
+CHECK-NEXT: StructTy 80
+CHECK-NEXT: ArrayTy 81
+CHECK-NEXT: UnknownTy 82
+CHECK-NEXT: Function 83
+CHECK-NEXT: Pointer 84
+CHECK-NEXT: Constant 85
+CHECK-NEXT: Variable 86
+CHECK-NEXT: FCMP_false 87
+CHECK-NEXT: FCMP_oeq 88
+CHECK-NEXT: FCMP_ogt 89
+CHECK-NEXT: FCMP_oge 90
+CHECK-NEXT: FCMP_olt 91
+CHECK-NEXT: FCMP_ole 92
+CHECK-NEXT: FCMP_one 93
+CHECK-NEXT: FCMP_ord 94
+CHECK-NEXT: FCMP_uno 95
+CHECK-NEXT: FCMP_ueq 96
+CHECK-NEXT: FCMP_ugt 97
+CHECK-NEXT: FCMP_uge 98
+CHECK-NEXT: FCMP_ult 99
+CHECK-NEXT: FCMP_ule 100
+CHECK-NEXT: FCMP_une 101
+CHECK-NEXT: FCMP_true 102
+CHECK-NEXT: ICMP_eq 103
+CHECK-NEXT: ICMP_ne 104
+CHECK-NEXT: ICMP_ugt 105
+CHECK-NEXT: ICMP_uge 106
+CHECK-NEXT: ICMP_ult 107
+CHECK-NEXT: ICMP_ule 108
+CHECK-NEXT: ICMP_sgt 109
+CHECK-NEXT: ICMP_sge 110
+CHECK-NEXT: ICMP_slt 111
+CHECK-NEXT: ICMP_sle 112
diff --git a/llvm/test/tools/llvm-ir2vec/triplets.ll b/llvm/test/tools/llvm-ir2vec/triplets.ll
index 45a158c926900..23366e90f129b 100644
--- a/llvm/test/tools/llvm-ir2vec/triplets.ll
+++ b/llvm/test/tools/llvm-ir2vec/triplets.ll
@@ -25,41 +25,41 @@ entry:
}
; TRIPLETS: MAX_RELATION=3
-; TRIPLETS-NEXT: 13 75 0
-; TRIPLETS-NEXT: 13 85 2
-; TRIPLETS-NEXT: 13 85 3
+; TRIPLETS-NEXT: 13 76 0
+; TRIPLETS-NEXT: 13 86 2
+; TRIPLETS-NEXT: 13 86 3
; TRIPLETS-NEXT: 13 0 1
-; TRIPLETS-NEXT: 0 70 0
-; TRIPLETS-NEXT: 0 85 2
-; TRIPLETS-NEXT: 17 75 0
-; TRIPLETS-NEXT: 17 85 2
-; TRIPLETS-NEXT: 17 85 3
+; TRIPLETS-NEXT: 0 71 0
+; TRIPLETS-NEXT: 0 86 2
+; TRIPLETS-NEXT: 17 76 0
+; TRIPLETS-NEXT: 17 86 2
+; TRIPLETS-NEXT: 17 86 3
; TRIPLETS-NEXT: 17 0 1
-; TRIPLETS-NEXT: 0 70 0
-; TRIPLETS-NEXT: 0 85 2
-; TRIPLETS-NEXT: 31 78 0
-; TRIPLETS-NEXT: 31 84 2
+; TRIPLETS-NEXT: 0 71 0
+; TRIPLETS-NEXT: 0 86 2
+; TRIPLETS-NEXT: 31 79 0
+; TRIPLETS-NEXT: 31 85 2
; TRIPLETS-NEXT: 31 31 1
-; TRIPLETS-NEXT: 31 78 0
-; TRIPLETS-NEXT: 31 84 2
+; TRIPLETS-NEXT: 31 79 0
+; TRIPLETS-NEXT: 31 85 2
; TRIPLETS-NEXT: 31 33 1
-; TRIPLETS-NEXT: 33 70 0
-; TRIPLETS-NEXT: 33 85 2
-; TRIPLETS-NEXT: 33 83 3
+; TRIPLETS-NEXT: 33 71 0
+; TRIPLETS-NEXT: 33 86 2
+; TRIPLETS-NEXT: 33 84 3
; TRIPLETS-NEXT: 33 33 1
-; TRIPLETS-NEXT: 33 70 0
-; TRIPLETS-NEXT: 33 85 2
-; TRIPLETS-NEXT: 33 83 3
+; TRIPLETS-NEXT: 33 71 0
+; TRIPLETS-NEXT: 33 86 2
+; TRIPLETS-NEXT: 33 84 3
; TRIPLETS-NEXT: 33 32 1
-; TRIPLETS-NEXT: 32 75 0
-; TRIPLETS-NEXT: 32 83 2
+; TRIPLETS-NEXT: 32 76 0
+; TRIPLETS-NEXT: 32 84 2
; TRIPLETS-NEXT: 32 32 1
-; TRIPLETS-NEXT: 32 75 0
-; TRIPLETS-NEXT: 32 83 2
+; TRIPLETS-NEXT: 32 76 0
+; TRIPLETS-NEXT: 32 84 2
; TRIPLETS-NEXT: 32 13 1
-; TRIPLETS-NEXT: 13 75 0
-; TRIPLETS-NEXT: 13 85 2
-; TRIPLETS-NEXT: 13 85 3
+; TRIPLETS-NEXT: 13 76 0
+; TRIPLETS-NEXT: 13 86 2
+; TRIPLETS-NEXT: 13 86 3
; TRIPLETS-NEXT: 13 0 1
-; TRIPLETS-NEXT: 0 70 0
-; TRIPLETS-NEXT: 0 85 2
+; TRIPLETS-NEXT: 0 71 0
+; TRIPLETS-NEXT: 0 86 2
diff --git a/llvm/unittests/Analysis/IR2VecTest.cpp b/llvm/unittests/Analysis/IR2VecTest.cpp
index fe67442da0f90..4678c6ef25722 100644
--- a/llvm/unittests/Analysis/IR2VecTest.cpp
+++ b/llvm/unittests/Analysis/IR2VecTest.cpp
@@ -363,8 +363,8 @@ TEST_F(IR2VecTestFixture, GetInstVec_Symbolic) {
EXPECT_EQ(AddEmb.size(), 2u);
EXPECT_EQ(RetEmb.size(), 2u);
- EXPECT_TRUE(AddEmb.approximatelyEquals(Embedding(2, 26.1)));
- EXPECT_TRUE(RetEmb.approximatelyEquals(Embedding(2, 15.8)));
+ EXPECT_TRUE(AddEmb.approximatelyEquals(Embedding(2, 26.4)));
+ EXPECT_TRUE(RetEmb.approximatelyEquals(Embedding(2, 16.0)));
}
TEST_F(IR2VecTestFixture, GetInstVec_FlowAware) {
@@ -376,8 +376,8 @@ TEST_F(IR2VecTestFixture, GetInstVec_FlowAware) {
EXPECT_EQ(AddEmb.size(), 2u);
EXPECT_EQ(RetEmb.size(), 2u);
- EXPECT_TRUE(AddEmb.approximatelyEquals(Embedding(2, 26.1)));
- EXPECT_TRUE(RetEmb.approximatelyEquals(Embedding(2, 33.3)));
+ EXPECT_TRUE(AddEmb.approximatelyEquals(Embedding(2, 26.4)));
+ EXPECT_TRUE(RetEmb.approximatelyEquals(Embedding(2, 33.7)));
}
TEST_F(IR2VecTestFixture, GetBBVector_Symbolic) {
@@ -387,9 +387,9 @@ TEST_F(IR2VecTestFixture, GetBBVector_Symbolic) {
const auto &BBVec = Emb->getBBVector(*BB);
EXPECT_EQ(BBVec.size(), 2u);
- // BB vector should be sum of add and ret: {26.1, 26.1} + {15.8, 15.8} =
- // {41.9, 41.9}
- EXPECT_TRUE(BBVec.approximatelyEquals(Embedding(2, 41.9)));
+ // BB vector should be sum of add and ret: {26.4, 26.4} + {16.0, 16.0} =
+ // {42.4, 42.4}
+ EXPECT_TRUE(BBVec.approximatelyEquals(Embedding(2, 42.4)));
}
TEST_F(IR2VecTestFixture, GetBBVector_FlowAware) {
@@ -399,9 +399,9 @@ TEST_F(IR2VecTestFixture, GetBBVector_FlowAware) {
const auto &BBVec = Emb->getBBVector(*BB);
EXPECT_EQ(BBVec.size(), 2u);
- // BB vector should be sum of add and ret: {26.1, 26.1} + {33.3, 33.3} =
- // {59.4, 59.4}
- EXPECT_TRUE(BBVec.approximatelyEquals(Embedding(2, 59.4)));
+ // BB vector should be sum of add and ret: {26.4, 26.4} + {33.7, 33.7} =
+ // {60.1, 60.1}
+ EXPECT_TRUE(BBVec.approximatelyEquals(Embedding(2, 60.1)));
}
TEST_F(IR2VecTestFixture, GetFunctionVector_Symbolic) {
@@ -412,8 +412,8 @@ TEST_F(IR2VecTestFixture, GetFunctionVector_Symbolic) {
EXPECT_EQ(FuncVec.size(), 2u);
- // Function vector should match BB vector (only one BB): {41.9, 41.9}
- EXPECT_TRUE(FuncVec.approximatelyEquals(Embedding(2, 41.9)));
+ // Function vector should match BB vector (only one BB): {42.4, 42.4}
+ EXPECT_TRUE(FuncVec.approximatelyEquals(Embedding(2, 42.4)));
}
TEST_F(IR2VecTestFixture, GetFunctionVector_FlowAware) {
@@ -423,8 +423,8 @@ TEST_F(IR2VecTestFixture, GetFunctionVector_FlowAware) {
const auto &FuncVec = Emb->getFunctionVector();
EXPECT_EQ(FuncVec.size(), 2u);
- // Function vector should match BB vector (only one BB): {59.4, 59.4}
- EXPECT_TRUE(FuncVec.approximatelyEquals(Embedding(2, 59.4)));
+ // Function vector should match BB vector (only one BB): {60.1, 60.1}
+ EXPECT_TRUE(FuncVec.approximatelyEquals(Embedding(2, 60.1)));
}
TEST_F(IR2VecTestFixture, MultipleComputeEmbeddingsConsistency_Symbolic) {
More information about the llvm-commits
mailing list