[llvm] [IR][AtomicExpand] Add elementwise modifier to atomicrmw; automatically expand for all targets (PR #189517)

Yonah Goldberg via llvm-commits llvm-commits at lists.llvm.org
Wed Apr 8 08:55:47 PDT 2026


https://github.com/YonahGoldberg updated https://github.com/llvm/llvm-project/pull/189517

>From ecb39597d2cfbf263c1c88e0c16d24f3e3bf64d8 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 31 Mar 2026 01:16:28 +0000
Subject: [PATCH 01/19] IR

---
 llvm/docs/LangRef.rst                         |  16 +-
 llvm/include/llvm/AsmParser/LLToken.h         |   1 +
 llvm/include/llvm/Bitcode/LLVMBitCodes.h      |   4 +
 llvm/include/llvm/CodeGen/TargetLowering.h    |  22 +-
 llvm/include/llvm/IR/IRBuilder.h              |   6 +-
 llvm/include/llvm/IR/Instructions.h           |  13 +-
 llvm/lib/AsmParser/LLLexer.cpp                |   1 +
 llvm/lib/AsmParser/LLParser.cpp               |  46 +-
 llvm/lib/Bitcode/Reader/BitcodeReader.cpp     |  12 +-
 llvm/lib/Bitcode/Writer/BitcodeWriter.cpp     |  96 +++-
 llvm/lib/CodeGen/AtomicExpandPass.cpp         |  82 +++
 llvm/lib/IR/AsmWriter.cpp                     |   5 +-
 llvm/lib/IR/Instruction.cpp                   |   1 +
 llvm/lib/IR/Instructions.cpp                  |  14 +-
 llvm/lib/IR/Verifier.cpp                      |  20 +-
 llvm/test/Assembler/atomic.ll                 |   6 +
 ...micrmw-elementwise-fadd-must-be-fp-type.ll |   8 +
 .../invalid-atomicrmw-elementwise-scalar.ll   |   8 +
 llvm/test/Bitcode/atomicrmw-elementwise.ll    |  16 +
 .../CodeGen/NVPTX/atomicrmw-elementwise.ll    | 506 ++++++++++++++++++
 .../AMDGPU/expand-atomic-rmw-elementwise.ll   | 138 +++++
 .../NVPTX/expand-atomic-rmw-elementwise.ll    | 262 +++++++++
 .../X86/expand-atomic-rmw-elementwise.ll      |  26 +
 23 files changed, 1244 insertions(+), 65 deletions(-)
 create mode 100644 llvm/test/Assembler/invalid-atomicrmw-elementwise-fadd-must-be-fp-type.ll
 create mode 100644 llvm/test/Assembler/invalid-atomicrmw-elementwise-scalar.ll
 create mode 100644 llvm/test/Bitcode/atomicrmw-elementwise.ll
 create mode 100644 llvm/test/CodeGen/NVPTX/atomicrmw-elementwise.ll
 create mode 100644 llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-elementwise.ll
 create mode 100644 llvm/test/Transforms/AtomicExpand/NVPTX/expand-atomic-rmw-elementwise.ll
 create mode 100644 llvm/test/Transforms/AtomicExpand/X86/expand-atomic-rmw-elementwise.ll

diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index 5584e0828d3cd..5cf3e4dbd9748 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -12098,7 +12098,7 @@ Syntax:
 
 ::
 
-      atomicrmw [volatile] <operation> ptr <pointer>, <ty> <value> [syncscope("<target-scope>")] <ordering>[, align <alignment>]  ; yields ty
+      atomicrmw [volatile] [elementwise] <operation> ptr <pointer>, <ty> <value> [syncscope("<target-scope>")] <ordering>[, align <alignment>]  ; yields ty
 
 Overview:
 """""""""
@@ -12145,7 +12145,9 @@ or fixed vector of floating-point type.  The type of the '``<pointer>``'
 operand must be a pointer to that type. If the ``atomicrmw`` is marked
 as ``volatile``, then the optimizer is not allowed to modify the
 number or order of execution of this ``atomicrmw`` with other
-:ref:`volatile operations <volatile>`.
+:ref:`volatile operations <volatile>`. If the ``elementwise`` modifier is present,
+then ``<value>`` must be a fixed vector type whose element type is legal for the
+corresponding scalar ``atomicrmw`` operation.
 
 Note: if the alignment is not greater or equal to the size of the `<value>`
 type, the atomic operation is likely to require a lock and have poor
@@ -12160,6 +12162,16 @@ isn't specified.
 An ``atomicrmw`` instruction can also take an optional
 ":ref:`syncscope <syncscope>`" argument.
 
+If the ``elementwise`` modifier is present, the instruction has per-element vector
+atomic semantics. It behaves as if it were expanded into one scalar ``atomicrmw`` per element, executed in an arbitrary order.
+Without ``elementwise``, vector ``atomicrmw`` keeps whole-value atomic semantics.
+
+Targets may implement ``atomicrmw elementwise`` either by lowering it to a
+native elementwise vector atomic, by scalarizing it into per-element scalar
+``atomicrmw`` operations, or by using an existing stronger whole-value atomic
+implementation, as long as the observable semantics are at least as strong as
+the IR definition above.
+
 Semantics:
 """"""""""
 
diff --git a/llvm/include/llvm/AsmParser/LLToken.h b/llvm/include/llvm/AsmParser/LLToken.h
index c138fb5ccc55b..eed62572d44bd 100644
--- a/llvm/include/llvm/AsmParser/LLToken.h
+++ b/llvm/include/llvm/AsmParser/LLToken.h
@@ -91,6 +91,7 @@ enum Kind {
   kw_unwind,
   kw_datalayout,
   kw_volatile,
+  kw_elementwise,
   kw_atomic,
   kw_unordered,
   kw_monotonic,
diff --git a/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
index 9162754bbfe1a..95787c595dff7 100644
--- a/llvm/include/llvm/Bitcode/LLVMBitCodes.h
+++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
@@ -524,6 +524,10 @@ enum RMWOperations {
   RMW_FMINIMUMNUM = 22,
 };
 
+enum RMWOperationFlags {
+  RMW_ELEMENTWISE_FLAG = 1 << 5,
+};
+
 /// OverflowingBinaryOperatorOptionalFlags - Flags for serializing
 /// OverflowingBinaryOperator's SubclassOptionalData contents.
 enum OverflowingBinaryOperatorOptionalFlags {
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 4d7b6ea0755e4..b04c0aa10f9d4 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -2475,13 +2475,33 @@ class LLVM_ABI TargetLoweringBase {
   }
 
   /// Returns how the IR-level AtomicExpand pass should expand the given
-  /// AtomicRMW, if at all. Default is to never expand.
+  /// AtomicRMW, if at all. Default is to never expand scalar atomics and expand
+  /// FP atomics via CmpXChg.
+  ///
+  /// Precondition: \p RMW is not elementwise. Elementwise atomicrmw
+  /// instructions are routed through \c shouldExpandAtomicRMWElementwiseInIR.
   virtual AtomicExpansionKind
   shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const {
     return RMW->isFloatingPointOperation() ?
       AtomicExpansionKind::CmpXChg : AtomicExpansionKind::None;
   }
 
+  /// Returns whether the IR-level AtomicExpand pass should expand the given
+  /// elementwise AtomicRMW into per-lane scalar atomicrmw instructions.
+  ///
+  /// Returning \c true (the default) tells AtomicExpand to first try to
+  /// conservatively drop the elementwise modifier and reuse an existing
+  /// whole-value atomicrmw lowering. If that is not possible, it scalarizes
+  /// into per-lane scalar atomicrmw instructions that are each fed back
+  /// through the normal atomic expansion pipeline.
+  ///
+  /// Targets that support native vector atomic instructions should return
+  /// \c false to preserve the elementwise atomicrmw for the backend.
+  virtual bool
+  shouldExpandAtomicRMWElementwiseInIR(const AtomicRMWInst *RMW) const {
+    return true;
+  }
+
   /// Returns how the given atomic atomicrmw should be cast by the IR-level
   /// AtomicExpand pass.
   virtual AtomicExpansionKind
diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h
index 4ed3d73c4a057..9ebac9b8c79e9 100644
--- a/llvm/include/llvm/IR/IRBuilder.h
+++ b/llvm/include/llvm/IR/IRBuilder.h
@@ -1941,13 +1941,15 @@ class IRBuilderBase {
   AtomicRMWInst *CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr,
                                  Value *Val, MaybeAlign Align,
                                  AtomicOrdering Ordering,
-                                 SyncScope::ID SSID = SyncScope::System) {
+                                 SyncScope::ID SSID = SyncScope::System,
+                                 bool Elementwise = false) {
     if (!Align) {
       const DataLayout &DL = BB->getDataLayout();
       Align = llvm::Align(DL.getTypeStoreSize(Val->getType()));
     }
 
-    return Insert(new AtomicRMWInst(Op, Ptr, Val, *Align, Ordering, SSID));
+    return Insert(
+        new AtomicRMWInst(Op, Ptr, Val, *Align, Ordering, SSID, Elementwise));
   }
 
   CallInst *CreateStructuredGEP(Type *BaseType, Value *PtrBase,
diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h
index bdc6d5bd2f5e5..9267be115b0cd 100644
--- a/llvm/include/llvm/IR/Instructions.h
+++ b/llvm/include/llvm/IR/Instructions.h
@@ -809,7 +809,7 @@ class AtomicRMWInst : public Instruction {
 public:
   LLVM_ABI AtomicRMWInst(BinOp Operation, Value *Ptr, Value *Val,
                          Align Alignment, AtomicOrdering Ordering,
-                         SyncScope::ID SSID,
+                         SyncScope::ID SSID, bool Elementwise = false,
                          InsertPosition InsertBefore = nullptr);
 
   // allocate space for exactly two operands
@@ -867,6 +867,12 @@ class AtomicRMWInst : public Instruction {
   ///
   void setVolatile(bool V) { setSubclassData<VolatileField>(V); }
 
+  /// Return true if this RMW has elementwise vector semantics.
+  bool isElementwise() const { return Elementwise; }
+
+  /// Specify whether this RMW has elementwise vector semantics.
+  void setElementwise(bool V) { Elementwise = V; }
+
   /// Transparently provide more efficient getOperand methods.
   DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
 
@@ -920,7 +926,7 @@ class AtomicRMWInst : public Instruction {
 
 private:
   void Init(BinOp Operation, Value *Ptr, Value *Val, Align Align,
-            AtomicOrdering Ordering, SyncScope::ID SSID);
+            AtomicOrdering Ordering, SyncScope::ID SSID, bool Elementwise);
 
   // Shadow Instruction::setInstructionSubclassData with a private forwarding
   // method so that subclasses cannot accidentally use it.
@@ -933,6 +939,9 @@ class AtomicRMWInst : public Instruction {
   /// room in SubClassData for everything, so synchronization scope ID gets its
   /// own field.
   SyncScope::ID SSID;
+
+  /// Whether this instruction uses per-lane vector atomic semantics.
+  bool Elementwise = false;
 };
 
 template <>
diff --git a/llvm/lib/AsmParser/LLLexer.cpp b/llvm/lib/AsmParser/LLLexer.cpp
index 60050ae477a01..7466a381f7fa3 100644
--- a/llvm/lib/AsmParser/LLLexer.cpp
+++ b/llvm/lib/AsmParser/LLLexer.cpp
@@ -600,6 +600,7 @@ lltok::Kind LLLexer::LexIdentifier() {
   KEYWORD(unwind);
   KEYWORD(datalayout);
   KEYWORD(volatile);
+  KEYWORD(elementwise);
   KEYWORD(atomic);
   KEYWORD(unordered);
   KEYWORD(monotonic);
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index 917aecb47590d..997eea293291d 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -8962,20 +8962,24 @@ int LLParser::parseCmpXchg(Instruction *&Inst, PerFunctionState &PFS) {
 }
 
 /// parseAtomicRMW
-///   ::= 'atomicrmw' 'volatile'? BinOp TypeAndValue ',' TypeAndValue
+///   ::= 'atomicrmw' 'volatile'? 'elementwise'? BinOp TypeAndValue ','
+///   TypeAndValue
 ///       'singlethread'? AtomicOrdering
 int LLParser::parseAtomicRMW(Instruction *&Inst, PerFunctionState &PFS) {
   Value *Ptr, *Val; LocTy PtrLoc, ValLoc;
   bool AteExtraComma = false;
   AtomicOrdering Ordering = AtomicOrdering::NotAtomic;
   SyncScope::ID SSID = SyncScope::System;
-  bool isVolatile = false;
+  bool IsVolatile = false;
+  bool IsElementwise = false;
   bool IsFP = false;
   AtomicRMWInst::BinOp Operation;
   MaybeAlign Alignment;
 
   if (EatIfPresent(lltok::kw_volatile))
-    isVolatile = true;
+    IsVolatile = true;
+  if (EatIfPresent(lltok::kw_elementwise))
+    IsElementwise = true;
 
   switch (Lex.getKind()) {
   default:
@@ -9052,42 +9056,54 @@ int LLParser::parseAtomicRMW(Instruction *&Inst, PerFunctionState &PFS) {
   if (Val->getType()->isScalableTy())
     return error(ValLoc, "atomicrmw operand may not be scalable");
 
+  // For elementwise ops, the value must be a fixed vector type whose element
+  // type is legal for the corresponding scalar atomicrmw operation. So assign
+  // ScalarTy the element type for elementwise ops so we can check this.
+  Type *ScalarTy = Val->getType();
+  if (IsElementwise) {
+    auto *VecTy = dyn_cast<FixedVectorType>(Val->getType());
+    if (!VecTy)
+      return error(ValLoc,
+                   "atomicrmw elementwise operand must be a fixed vector type");
+    ScalarTy = VecTy->getElementType();
+  }
+
   if (Operation == AtomicRMWInst::Xchg) {
-    if (!Val->getType()->isIntegerTy() &&
-        !Val->getType()->isFloatingPointTy() &&
-        !Val->getType()->isPointerTy()) {
+    if (!ScalarTy->isIntegerTy() && !ScalarTy->isFloatingPointTy() &&
+        !ScalarTy->isPointerTy()) {
       return error(
           ValLoc,
           "atomicrmw " + AtomicRMWInst::getOperationName(Operation) +
               " operand must be an integer, floating point, or pointer type");
     }
   } else if (IsFP) {
-    if (!Val->getType()->isFPOrFPVectorTy()) {
+    if (!ScalarTy->isFPOrFPVectorTy()) {
       return error(ValLoc, "atomicrmw " +
                                AtomicRMWInst::getOperationName(Operation) +
                                " operand must be a floating point type");
     }
   } else {
-    if (!Val->getType()->isIntegerTy()) {
+    if (!ScalarTy->isIntegerTy()) {
       return error(ValLoc, "atomicrmw " +
                                AtomicRMWInst::getOperationName(Operation) +
                                " operand must be an integer");
     }
   }
 
-  unsigned Size =
-      PFS.getFunction().getDataLayout().getTypeStoreSizeInBits(
-          Val->getType());
+  // Elementwise ops are legal on <3 x i32>, for example, because we can expand,
+  // so check the scalar type, not the vector type.
+  unsigned Size = PFS.getFunction().getDataLayout().getTypeStoreSizeInBits(
+      IsElementwise ? ScalarTy : Val->getType());
   if (Size < 8 || (Size & (Size - 1)))
     return error(ValLoc, "atomicrmw operand must be power-of-two byte-sized"
                          " integer");
   const Align DefaultAlignment(
       PFS.getFunction().getDataLayout().getTypeStoreSize(
           Val->getType()));
-  AtomicRMWInst *RMWI =
-      new AtomicRMWInst(Operation, Ptr, Val,
-                        Alignment.value_or(DefaultAlignment), Ordering, SSID);
-  RMWI->setVolatile(isVolatile);
+  AtomicRMWInst *RMWI = new AtomicRMWInst(Operation, Ptr, Val,
+                                          Alignment.value_or(DefaultAlignment),
+                                          Ordering, SSID, IsElementwise);
+  RMWI->setVolatile(IsVolatile);
   Inst = RMWI;
   return AteExtraComma ? InstExtraComma : InstNormal;
 }
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index e01cf501b8841..60c8c7812ce81 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -1364,8 +1364,10 @@ static int getDecodedBinaryOpcode(unsigned Val, Type *Ty) {
   }
 }
 
-static AtomicRMWInst::BinOp getDecodedRMWOperation(unsigned Val) {
-  switch (Val) {
+static AtomicRMWInst::BinOp getDecodedRMWOperation(unsigned Val,
+                                                   bool &IsElementwise) {
+  IsElementwise = Val & bitc::RMW_ELEMENTWISE_FLAG;
+  switch (Val & ~bitc::RMW_ELEMENTWISE_FLAG) {
   default: return AtomicRMWInst::BAD_BINOP;
   case bitc::RMW_XCHG: return AtomicRMWInst::Xchg;
   case bitc::RMW_ADD: return AtomicRMWInst::Add;
@@ -6709,8 +6711,9 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
       if (!(NumRecords == (OpNum + 4) || NumRecords == (OpNum + 5)))
         return error("Invalid atomicrmw record");
 
+      bool IsElementwise = false;
       const AtomicRMWInst::BinOp Operation =
-          getDecodedRMWOperation(Record[OpNum]);
+          getDecodedRMWOperation(Record[OpNum], IsElementwise);
       if (Operation < AtomicRMWInst::FIRST_BINOP ||
           Operation > AtomicRMWInst::LAST_BINOP)
         return error("Invalid atomicrmw record");
@@ -6735,7 +6738,8 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
         Alignment =
             Align(TheModule->getDataLayout().getTypeStoreSize(Val->getType()));
 
-      I = new AtomicRMWInst(Operation, Ptr, Val, *Alignment, Ordering, SSID);
+      I = new AtomicRMWInst(Operation, Ptr, Val, *Alignment, Ordering, SSID,
+                            IsElementwise);
       ResTypeID = ValTypeID;
       cast<AtomicRMWInst>(I)->setVolatile(IsVol);
 
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index 71d5a8bc98a4b..22052105d4d75 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -690,41 +690,84 @@ static unsigned getEncodedBinaryOpcode(unsigned Opcode) {
   }
 }
 
-static unsigned getEncodedRMWOperation(AtomicRMWInst::BinOp Op) {
-  switch (Op) {
+static unsigned getEncodedRMWOperation(const AtomicRMWInst &I) {
+  unsigned Encoding = 0;
+  switch (I.getOperation()) {
   default: llvm_unreachable("Unknown RMW operation!");
-  case AtomicRMWInst::Xchg: return bitc::RMW_XCHG;
-  case AtomicRMWInst::Add: return bitc::RMW_ADD;
-  case AtomicRMWInst::Sub: return bitc::RMW_SUB;
-  case AtomicRMWInst::And: return bitc::RMW_AND;
-  case AtomicRMWInst::Nand: return bitc::RMW_NAND;
-  case AtomicRMWInst::Or: return bitc::RMW_OR;
-  case AtomicRMWInst::Xor: return bitc::RMW_XOR;
-  case AtomicRMWInst::Max: return bitc::RMW_MAX;
-  case AtomicRMWInst::Min: return bitc::RMW_MIN;
-  case AtomicRMWInst::UMax: return bitc::RMW_UMAX;
-  case AtomicRMWInst::UMin: return bitc::RMW_UMIN;
-  case AtomicRMWInst::FAdd: return bitc::RMW_FADD;
-  case AtomicRMWInst::FSub: return bitc::RMW_FSUB;
-  case AtomicRMWInst::FMax: return bitc::RMW_FMAX;
-  case AtomicRMWInst::FMin: return bitc::RMW_FMIN;
+  case AtomicRMWInst::Xchg:
+    Encoding = bitc::RMW_XCHG;
+    break;
+  case AtomicRMWInst::Add:
+    Encoding = bitc::RMW_ADD;
+    break;
+  case AtomicRMWInst::Sub:
+    Encoding = bitc::RMW_SUB;
+    break;
+  case AtomicRMWInst::And:
+    Encoding = bitc::RMW_AND;
+    break;
+  case AtomicRMWInst::Nand:
+    Encoding = bitc::RMW_NAND;
+    break;
+  case AtomicRMWInst::Or:
+    Encoding = bitc::RMW_OR;
+    break;
+  case AtomicRMWInst::Xor:
+    Encoding = bitc::RMW_XOR;
+    break;
+  case AtomicRMWInst::Max:
+    Encoding = bitc::RMW_MAX;
+    break;
+  case AtomicRMWInst::Min:
+    Encoding = bitc::RMW_MIN;
+    break;
+  case AtomicRMWInst::UMax:
+    Encoding = bitc::RMW_UMAX;
+    break;
+  case AtomicRMWInst::UMin:
+    Encoding = bitc::RMW_UMIN;
+    break;
+  case AtomicRMWInst::FAdd:
+    Encoding = bitc::RMW_FADD;
+    break;
+  case AtomicRMWInst::FSub:
+    Encoding = bitc::RMW_FSUB;
+    break;
+  case AtomicRMWInst::FMax:
+    Encoding = bitc::RMW_FMAX;
+    break;
+  case AtomicRMWInst::FMin:
+    Encoding = bitc::RMW_FMIN;
+    break;
   case AtomicRMWInst::FMaximum:
-    return bitc::RMW_FMAXIMUM;
+    Encoding = bitc::RMW_FMAXIMUM;
+    break;
   case AtomicRMWInst::FMinimum:
-    return bitc::RMW_FMINIMUM;
+    Encoding = bitc::RMW_FMINIMUM;
+    break;
   case AtomicRMWInst::FMaximumNum:
-    return bitc::RMW_FMAXIMUMNUM;
+    Encoding = bitc::RMW_FMAXIMUMNUM;
+    break;
   case AtomicRMWInst::FMinimumNum:
-    return bitc::RMW_FMINIMUMNUM;
+    Encoding = bitc::RMW_FMINIMUMNUM;
+    break;
   case AtomicRMWInst::UIncWrap:
-    return bitc::RMW_UINC_WRAP;
+    Encoding = bitc::RMW_UINC_WRAP;
+    break;
   case AtomicRMWInst::UDecWrap:
-    return bitc::RMW_UDEC_WRAP;
+    Encoding = bitc::RMW_UDEC_WRAP;
+    break;
   case AtomicRMWInst::USubCond:
-    return bitc::RMW_USUB_COND;
+    Encoding = bitc::RMW_USUB_COND;
+    break;
   case AtomicRMWInst::USubSat:
-    return bitc::RMW_USUB_SAT;
+    Encoding = bitc::RMW_USUB_SAT;
+    break;
   }
+
+  if (I.isElementwise())
+    Encoding |= bitc::RMW_ELEMENTWISE_FLAG;
+  return Encoding;
 }
 
 static unsigned getEncodedOrdering(AtomicOrdering Ordering) {
@@ -3547,8 +3590,7 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I,
     Code = bitc::FUNC_CODE_INST_ATOMICRMW;
     pushValueAndType(I.getOperand(0), InstID, Vals); // ptrty + ptr
     pushValueAndType(I.getOperand(1), InstID, Vals); // valty + val
-    Vals.push_back(
-        getEncodedRMWOperation(cast<AtomicRMWInst>(I).getOperation()));
+    Vals.push_back(getEncodedRMWOperation(cast<AtomicRMWInst>(I)));
     Vals.push_back(cast<AtomicRMWInst>(I).isVolatile());
     Vals.push_back(getEncodedOrdering(cast<AtomicRMWInst>(I).getOrdering()));
     Vals.push_back(
diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index 7ed6c2c966683..535997e23f2eb 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -43,6 +43,7 @@
 #include "llvm/IR/Value.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/Pass.h"
+#include "llvm/Support/Alignment.h"
 #include "llvm/Support/AtomicOrdering.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/Debug.h"
@@ -99,6 +100,8 @@ class AtomicExpandImpl {
   bool tryExpandAtomicStore(StoreInst *SI);
   void expandAtomicStoreToXChg(StoreInst *SI);
   bool tryExpandAtomicRMW(AtomicRMWInst *AI);
+  bool canReuseWholeValueAtomicRMW(AtomicRMWInst *AI);
+  bool expandElementwiseAtomicRMW(AtomicRMWInst *AI);
   AtomicRMWInst *convertAtomicXchgToIntegerType(AtomicRMWInst *RMWI);
   Value *
   insertRMWLLSCLoop(IRBuilderBase &Builder, Type *ResultTy, Value *Addr,
@@ -265,6 +268,26 @@ static bool atomicSizeSupported(const TargetLowering *TLI, Inst *I) {
          Size <= TLI->getMaxAtomicSizeInBitsSupported() / 8;
 }
 
+<<<<<<< HEAD
+=======
+/// Returns true if we can lower atomicrmw elementwise using normal atomicrmw.
+bool AtomicExpandImpl::canReuseWholeValueAtomicRMW(AtomicRMWInst *AI) {
+  assert(AI->isElementwise() && "expected elementwise atomicrmw");
+
+  // Integer non-elementwise vector atomicrmw is illegal IR, so we need to be
+  // careful to reject these before removing the elementwise modifier.
+  if (!AI->isFloatingPointOperation())
+    return false;
+
+  AI->setElementwise(false);
+  bool CanReuse = atomicSizeSupported(TLI, AI) &&
+                  TLI->shouldExpandAtomicRMWInIR(AI) ==
+                      TargetLoweringBase::AtomicExpansionKind::None;
+  AI->setElementwise(true);
+  return CanReuse;
+}
+
+>>>>>>> 0075944a40cd (IR)
 bool AtomicExpandImpl::tryInsertTrailingSeqCstFence(Instruction *AtomicI) {
   if (!TLI->shouldInsertTrailingSeqCstFenceForAtomicStore(AtomicI))
     return false;
@@ -342,11 +365,31 @@ bool AtomicExpandImpl::processAtomicInstr(Instruction *I) {
 
   if (auto *RMWI = dyn_cast<AtomicRMWInst>(I)) {
     if (!atomicSizeSupported(TLI, RMWI)) {
+      if (RMWI->isElementwise())
+        return expandElementwiseAtomicRMW(RMWI);
       expandAtomicRMWToLibcall(RMWI);
       return true;
     }
 
     bool MadeChange = false;
+<<<<<<< HEAD
+=======
+
+    if (RMWI->isElementwise()) {
+      if (!TLI->shouldExpandAtomicRMWElementwiseInIR(RMWI))
+        return false;
+      if (canReuseWholeValueAtomicRMW(RMWI)) {
+        // Dropping the elementwise modifier strengthens the semantics, which is
+        // conservatively correct. Prefer the target's existing whole-value
+        // lowering over IR expansion.
+        RMWI->setElementwise(false);
+        MadeChange = true;
+      } else {
+        return expandElementwiseAtomicRMW(RMWI);
+      }
+    }
+
+>>>>>>> 0075944a40cd (IR)
     if (TLI->shouldCastAtomicRMWIInIR(RMWI) ==
         TargetLoweringBase::AtomicExpansionKind::CastToInteger) {
       RMWI = convertAtomicXchgToIntegerType(RMWI);
@@ -565,6 +608,45 @@ AtomicExpandImpl::convertAtomicXchgToIntegerType(AtomicRMWInst *RMWI) {
   return NewRMWI;
 }
 
+// Scalarize an elementwise vector atomicrmw into one scalar atomicrmw per
+// lane. Each lane keeps the original ordering/scope/volatility and is fed back
+// through processAtomicInstr() so the usual atomic expansion and fence logic
+// applies per lane rather than to the whole vector value.
+bool AtomicExpandImpl::expandElementwiseAtomicRMW(AtomicRMWInst *AI) {
+  auto *VecTy = cast<FixedVectorType>(AI->getType());
+  Type *LaneTy = VecTy->getElementType();
+  LLVMContext &Ctx = AI->getContext();
+  Value *Result = Constant::getNullValue(VecTy);
+  const uint64_t LaneSize = DL->getTypeStoreSize(LaneTy).getFixedValue();
+
+  for (unsigned Lane = 0, NumLanes = VecTy->getNumElements(); Lane != NumLanes;
+       ++Lane) {
+    ReplacementIRBuilder Builder(AI, *DL);
+    Value *Idx0 = ConstantInt::get(Type::getInt64Ty(Ctx), 0);
+    Value *Idx = ConstantInt::get(Type::getInt64Ty(Ctx), Lane);
+    Value *Indices[] = {Idx0, Idx};
+    Value *LanePtr = Builder.CreateInBoundsGEP(VecTy, AI->getPointerOperand(),
+                                               Indices, "lane.ptr");
+    Value *LaneVal =
+        Builder.CreateExtractElement(AI->getValOperand(), Idx, "lane.val");
+    auto *LaneRMW = Builder.CreateAtomicRMW(
+        AI->getOperation(), LanePtr, LaneVal,
+        commonAlignment(AI->getAlign(), LaneSize * Lane), AI->getOrdering(),
+        AI->getSyncScopeID());
+    LaneRMW->setVolatile(AI->isVolatile());
+    copyMetadataForAtomic(*LaneRMW, *AI);
+    Result = Builder.CreateInsertElement(Result, LaneRMW, Idx, "lane.old");
+
+    // Each scalar lane atomic may still need casts, fences, or further
+    // expansion, so re-run the normal atomic pipeline on it.
+    processAtomicInstr(LaneRMW);
+  }
+
+  AI->replaceAllUsesWith(Result);
+  AI->eraseFromParent();
+  return true;
+}
+
 bool AtomicExpandImpl::tryExpandAtomicLoad(LoadInst *LI) {
   switch (TLI->shouldExpandAtomicLoadInIR(LI)) {
   case TargetLoweringBase::AtomicExpansionKind::None:
diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp
index 7bff1e307237c..f8a421ba77882 100644
--- a/llvm/lib/IR/AsmWriter.cpp
+++ b/llvm/lib/IR/AsmWriter.cpp
@@ -4468,8 +4468,11 @@ void AssemblyWriter::printInstruction(const Instruction &I) {
     Out << ' ' << CI->getPredicate();
 
   // Print out the atomicrmw operation
-  if (const auto *RMWI = dyn_cast<AtomicRMWInst>(&I))
+  if (const auto *RMWI = dyn_cast<AtomicRMWInst>(&I)) {
+    if (RMWI->isElementwise())
+      Out << " elementwise";
     Out << ' ' << AtomicRMWInst::getOperationName(RMWI->getOperation());
+  }
 
   // Print out the type of the operands...
   const Value *Operand = I.getNumOperands() ? I.getOperand(0) : nullptr;
diff --git a/llvm/lib/IR/Instruction.cpp b/llvm/lib/IR/Instruction.cpp
index 5205d36a228c1..db2d959e8eb87 100644
--- a/llvm/lib/IR/Instruction.cpp
+++ b/llvm/lib/IR/Instruction.cpp
@@ -955,6 +955,7 @@ bool Instruction::hasSameSpecialState(const Instruction *I2,
                cast<AtomicCmpXchgInst>(I2)->getSyncScopeID();
   if (const AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(I1))
     return RMWI->getOperation() == cast<AtomicRMWInst>(I2)->getOperation() &&
+           RMWI->isElementwise() == cast<AtomicRMWInst>(I2)->isElementwise() &&
            RMWI->isVolatile() == cast<AtomicRMWInst>(I2)->isVolatile() &&
            (RMWI->getAlign() == cast<AtomicRMWInst>(I2)->getAlign() ||
             IgnoreAlignment) &&
diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index 8a220c48acac8..f940893ab0296 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -1436,7 +1436,7 @@ AtomicCmpXchgInst::AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal,
 
 void AtomicRMWInst::Init(BinOp Operation, Value *Ptr, Value *Val,
                          Align Alignment, AtomicOrdering Ordering,
-                         SyncScope::ID SSID) {
+                         SyncScope::ID SSID, bool Elementwise) {
   assert(Ordering != AtomicOrdering::NotAtomic &&
          "atomicrmw instructions can only be atomic.");
   assert(Ordering != AtomicOrdering::Unordered &&
@@ -1446,6 +1446,7 @@ void AtomicRMWInst::Init(BinOp Operation, Value *Ptr, Value *Val,
   setOperation(Operation);
   setOrdering(Ordering);
   setSyncScopeID(SSID);
+  setElementwise(Elementwise);
   setAlignment(Alignment);
 
   assert(getOperand(0) && getOperand(1) && "All operands must be non-null!");
@@ -1457,9 +1458,10 @@ void AtomicRMWInst::Init(BinOp Operation, Value *Ptr, Value *Val,
 
 AtomicRMWInst::AtomicRMWInst(BinOp Operation, Value *Ptr, Value *Val,
                              Align Alignment, AtomicOrdering Ordering,
-                             SyncScope::ID SSID, InsertPosition InsertBefore)
+                             SyncScope::ID SSID, bool Elementwise,
+                             InsertPosition InsertBefore)
     : Instruction(Val->getType(), AtomicRMW, AllocMarker, InsertBefore) {
-  Init(Operation, Ptr, Val, Alignment, Ordering, SSID);
+  Init(Operation, Ptr, Val, Alignment, Ordering, SSID, Elementwise);
 }
 
 StringRef AtomicRMWInst::getOperationName(BinOp Op) {
@@ -4397,9 +4399,9 @@ AtomicCmpXchgInst *AtomicCmpXchgInst::cloneImpl() const {
 }
 
 AtomicRMWInst *AtomicRMWInst::cloneImpl() const {
-  AtomicRMWInst *Result =
-      new AtomicRMWInst(getOperation(), getOperand(0), getOperand(1),
-                        getAlign(), getOrdering(), getSyncScopeID());
+  AtomicRMWInst *Result = new AtomicRMWInst(
+      getOperation(), getOperand(0), getOperand(1), getAlign(), getOrdering(),
+      getSyncScopeID(), isElementwise());
   Result->setVolatile(isVolatile());
   return Result;
 }
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index abda69304b7e4..791dfeb7c5a3b 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -4762,25 +4762,35 @@ void Verifier::visitAtomicRMWInst(AtomicRMWInst &RMWI) {
         "atomicrmw instructions cannot be unordered.", &RMWI);
   auto Op = RMWI.getOperation();
   Type *ElTy = RMWI.getOperand(1)->getType();
+  Type *ScalarTy = ElTy;
+  if (RMWI.isElementwise()) {
+    auto *VecTy = dyn_cast<FixedVectorType>(ElTy);
+    Check(VecTy, "atomicrmw elementwise operand must have fixed vector type!",
+          &RMWI, ElTy);
+    if (VecTy)
+      ScalarTy = VecTy->getElementType();
+  }
+
   if (Op == AtomicRMWInst::Xchg) {
-    Check(ElTy->isIntegerTy() || ElTy->isFloatingPointTy() ||
-              ElTy->isPointerTy(),
+    Check(ScalarTy->isIntegerTy() || ScalarTy->isFloatingPointTy() ||
+              ScalarTy->isPointerTy(),
           "atomicrmw " + AtomicRMWInst::getOperationName(Op) +
               " operand must have integer or floating point type!",
           &RMWI, ElTy);
   } else if (AtomicRMWInst::isFPOperation(Op)) {
     Check(ElTy->isFPOrFPVectorTy() && !isa<ScalableVectorType>(ElTy),
           "atomicrmw " + AtomicRMWInst::getOperationName(Op) +
-              " operand must have floating-point or fixed vector of floating-point "
+              " operand must have floating-point or fixed vector of "
+              "floating-point "
               "type!",
           &RMWI, ElTy);
   } else {
-    Check(ElTy->isIntegerTy(),
+    Check(ScalarTy->isIntegerTy(),
           "atomicrmw " + AtomicRMWInst::getOperationName(Op) +
               " operand must have integer type!",
           &RMWI, ElTy);
   }
-  checkAtomicMemAccessSize(ElTy, &RMWI);
+  checkAtomicMemAccessSize(RMWI.isElementwise() ? ScalarTy : ElTy, &RMWI);
   Check(AtomicRMWInst::FIRST_BINOP <= Op && Op <= AtomicRMWInst::LAST_BINOP,
         "Invalid binary operation!", &RMWI);
   visitInstruction(RMWI);
diff --git a/llvm/test/Assembler/atomic.ll b/llvm/test/Assembler/atomic.ll
index 0ed34f0ad98ef..611a717fa9a8d 100644
--- a/llvm/test/Assembler/atomic.ll
+++ b/llvm/test/Assembler/atomic.ll
@@ -151,5 +151,11 @@ define void @fp_vector_atomicrmw(ptr %x, <2 x half> %val) {
   ; CHECK: %atomic.fminimumnum = atomicrmw fminimumnum ptr %x, <2 x half> %val seq_cst
   %atomic.fminimumnum = atomicrmw fminimumnum ptr %x, <2 x half> %val seq_cst
 
+  ; CHECK: %atomic.elem.fadd = atomicrmw elementwise fadd ptr %x, <2 x half> %val monotonic
+  %atomic.elem.fadd = atomicrmw elementwise fadd ptr %x, <2 x half> %val monotonic
+
+  ; CHECK: %atomic.elem.fadd.vol = atomicrmw volatile elementwise fadd ptr %x, <2 x half> %val seq_cst
+  %atomic.elem.fadd.vol = atomicrmw volatile elementwise fadd ptr %x, <2 x half> %val seq_cst
+
   ret void
 }
diff --git a/llvm/test/Assembler/invalid-atomicrmw-elementwise-fadd-must-be-fp-type.ll b/llvm/test/Assembler/invalid-atomicrmw-elementwise-fadd-must-be-fp-type.ll
new file mode 100644
index 0000000000000..65a24212c20a5
--- /dev/null
+++ b/llvm/test/Assembler/invalid-atomicrmw-elementwise-fadd-must-be-fp-type.ll
@@ -0,0 +1,8 @@
+; RUN: not llvm-as %s -o /dev/null 2>&1 | FileCheck %s
+
+define <4 x i32> @bad_fadd(ptr %p, <4 x i32> %v) {
+  %old = atomicrmw elementwise fadd ptr %p, <4 x i32> %v monotonic
+  ret <4 x i32> %old
+}
+
+; CHECK: atomicrmw fadd operand must be a floating point type
diff --git a/llvm/test/Assembler/invalid-atomicrmw-elementwise-scalar.ll b/llvm/test/Assembler/invalid-atomicrmw-elementwise-scalar.ll
new file mode 100644
index 0000000000000..6037607a52eef
--- /dev/null
+++ b/llvm/test/Assembler/invalid-atomicrmw-elementwise-scalar.ll
@@ -0,0 +1,8 @@
+; RUN: not llvm-as %s -o /dev/null 2>&1 | FileCheck %s
+
+define i32 @bad_scalar(ptr %p, i32 %v) {
+  %old = atomicrmw elementwise add ptr %p, i32 %v monotonic
+  ret i32 %old
+}
+
+; CHECK: atomicrmw elementwise operand must be a fixed vector type
diff --git a/llvm/test/Bitcode/atomicrmw-elementwise.ll b/llvm/test/Bitcode/atomicrmw-elementwise.ll
new file mode 100644
index 0000000000000..db9c48a80047e
--- /dev/null
+++ b/llvm/test/Bitcode/atomicrmw-elementwise.ll
@@ -0,0 +1,16 @@
+; RUN: llvm-as %s -o - | llvm-dis | FileCheck %s
+; RUN: llvm-as %s -o - | verify-uselistorder
+
+define <4 x i32> @elem_add(ptr %p, <4 x i32> %v) {
+; CHECK-LABEL: @elem_add(
+; CHECK: %old = atomicrmw elementwise add ptr %p, <4 x i32> %v monotonic, align 16
+  %old = atomicrmw elementwise add ptr %p, <4 x i32> %v monotonic
+  ret <4 x i32> %old
+}
+
+define <4 x float> @elem_fadd(ptr %p, <4 x float> %v) {
+; CHECK-LABEL: @elem_fadd(
+; CHECK: %old = atomicrmw elementwise fadd ptr %p, <4 x float> %v seq_cst, align 16
+  %old = atomicrmw elementwise fadd ptr %p, <4 x float> %v seq_cst
+  ret <4 x float> %old
+}
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise.ll
new file mode 100644
index 0000000000000..625c2634dd584
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise.ll
@@ -0,0 +1,506 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx84 | FileCheck %s
+; RUN: %if ptxas-sm_90 && ptxas-isa-8.4 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx84 | %ptxas-verify -arch=sm_90 %}
+
+target triple = "nvptx64-nvidia-cuda"
+
+define <2 x float> @fadd_v2f32_elementwise(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: fadd_v2f32_elementwise(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-NEXT:    .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0: // %entry
+; CHECK-NEXT:    ld.param.b64 %rd1, [fadd_v2f32_elementwise_param_0];
+; CHECK-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fadd_v2f32_elementwise_param_1];
+; CHECK-NEXT:    atom.relaxed.sys.add.f32 %r3, [%rd1], %r1;
+; CHECK-NEXT:    atom.relaxed.sys.add.f32 %r4, [%rd1+4], %r2;
+; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {%r3, %r4};
+; CHECK-NEXT:    ret;
+entry:
+  %old = atomicrmw elementwise fadd ptr %addr, <2 x float> %val monotonic
+  ret <2 x float> %old
+}
+
+define <4 x float> @fadd_v4f32_elementwise(ptr %addr, <4 x float> %val) {
+; CHECK-LABEL: fadd_v4f32_elementwise(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<9>;
+; CHECK-NEXT:    .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0: // %entry
+; CHECK-NEXT:    ld.param.b64 %rd1, [fadd_v4f32_elementwise_param_0];
+; CHECK-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_v4f32_elementwise_param_1];
+; CHECK-NEXT:    atom.relaxed.sys.add.f32 %r5, [%rd1], %r1;
+; CHECK-NEXT:    atom.relaxed.sys.add.f32 %r6, [%rd1+4], %r2;
+; CHECK-NEXT:    atom.relaxed.sys.add.f32 %r7, [%rd1+8], %r3;
+; CHECK-NEXT:    atom.relaxed.sys.add.f32 %r8, [%rd1+12], %r4;
+; CHECK-NEXT:    st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; CHECK-NEXT:    ret;
+entry:
+  %old = atomicrmw elementwise fadd ptr %addr, <4 x float> %val monotonic, align 16
+  ret <4 x float> %old
+}
+
+define <4 x float> @fadd_v4f32_elementwise_volatile(ptr %addr,
+; CHECK-LABEL: fadd_v4f32_elementwise_volatile(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<9>;
+; CHECK-NEXT:    .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0: // %entry
+; CHECK-NEXT:    ld.param.b64 %rd1, [fadd_v4f32_elementwise_volatile_param_0];
+; CHECK-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_v4f32_elementwise_volatile_param_1];
+; CHECK-NEXT:    atom.relaxed.sys.add.f32 %r5, [%rd1], %r1;
+; CHECK-NEXT:    atom.relaxed.sys.add.f32 %r6, [%rd1+4], %r2;
+; CHECK-NEXT:    atom.relaxed.sys.add.f32 %r7, [%rd1+8], %r3;
+; CHECK-NEXT:    atom.relaxed.sys.add.f32 %r8, [%rd1+12], %r4;
+; CHECK-NEXT:    st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; CHECK-NEXT:    ret;
+                                                    <4 x float> %val) {
+entry:
+  %old = atomicrmw volatile elementwise fadd ptr %addr, <4 x float> %val monotonic, align 16
+  ret <4 x float> %old
+}
+
+define <4 x float> @fadd_v4f32_elementwise_block(ptr %addr, <4 x float> %val) {
+; CHECK-LABEL: fadd_v4f32_elementwise_block(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<9>;
+; CHECK-NEXT:    .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0: // %entry
+; CHECK-NEXT:    ld.param.b64 %rd1, [fadd_v4f32_elementwise_block_param_0];
+; CHECK-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_v4f32_elementwise_block_param_1];
+; CHECK-NEXT:    atom.relaxed.cta.add.f32 %r5, [%rd1], %r1;
+; CHECK-NEXT:    atom.relaxed.cta.add.f32 %r6, [%rd1+4], %r2;
+; CHECK-NEXT:    atom.relaxed.cta.add.f32 %r7, [%rd1+8], %r3;
+; CHECK-NEXT:    atom.relaxed.cta.add.f32 %r8, [%rd1+12], %r4;
+; CHECK-NEXT:    st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; CHECK-NEXT:    ret;
+entry:
+  %old = atomicrmw elementwise fadd ptr %addr, <4 x float> %val syncscope("block") monotonic, align 16
+  ret <4 x float> %old
+}
+
+define <4 x float> @fadd_v4f32_elementwise_seq_cst(ptr %addr, <4 x float> %val) {
+; CHECK-LABEL: fadd_v4f32_elementwise_seq_cst(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<9>;
+; CHECK-NEXT:    .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0: // %entry
+; CHECK-NEXT:    ld.param.b64 %rd1, [fadd_v4f32_elementwise_seq_cst_param_0];
+; CHECK-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_v4f32_elementwise_seq_cst_param_1];
+; CHECK-NEXT:    fence.sc.sys;
+; CHECK-NEXT:    atom.acquire.sys.add.f32 %r5, [%rd1], %r1;
+; CHECK-NEXT:    fence.sc.sys;
+; CHECK-NEXT:    atom.acquire.sys.add.f32 %r6, [%rd1+4], %r2;
+; CHECK-NEXT:    fence.sc.sys;
+; CHECK-NEXT:    atom.acquire.sys.add.f32 %r7, [%rd1+8], %r3;
+; CHECK-NEXT:    fence.sc.sys;
+; CHECK-NEXT:    atom.acquire.sys.add.f32 %r8, [%rd1+12], %r4;
+; CHECK-NEXT:    st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; CHECK-NEXT:    ret;
+entry:
+  %old = atomicrmw elementwise fadd ptr %addr, <4 x float> %val seq_cst, align 16
+  ret <4 x float> %old
+}
+
+define <8 x half> @fmin_v8f16_elementwise(ptr %addr, <8 x half> %val) {
+; CHECK-LABEL: fmin_v8f16_elementwise(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<9>;
+; CHECK-NEXT:    .reg .b16 %rs<41>;
+; CHECK-NEXT:    .reg .b32 %r<90>;
+; CHECK-NEXT:    .reg .b64 %rd<10>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0: // %entry
+; CHECK-NEXT:    ld.param.v4.b32 {%r25, %r26, %r27, %r28}, [fmin_v8f16_elementwise_param_1];
+; CHECK-NEXT:    ld.param.b64 %rd5, [fmin_v8f16_elementwise_param_0];
+; CHECK-NEXT:    mov.b32 {%rs1, _}, %r25;
+; CHECK-NEXT:    ld.b32 %r82, [%rd5];
+; CHECK-NEXT:  $L__BB5_1: // %atomicrmw.start
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    cvt.u16.u32 %rs9, %r82;
+; CHECK-NEXT:    min.f16 %rs10, %rs9, %rs1;
+; CHECK-NEXT:    cvt.u32.u16 %r29, %rs10;
+; CHECK-NEXT:    and.b32 %r30, %r82, -65536;
+; CHECK-NEXT:    or.b32 %r31, %r30, %r29;
+; CHECK-NEXT:    atom.relaxed.sys.cas.b32 %r1, [%rd5], %r82, %r31;
+; CHECK-NEXT:    setp.ne.b32 %p1, %r1, %r82;
+; CHECK-NEXT:    mov.b32 %r82, %r1;
+; CHECK-NEXT:    @%p1 bra $L__BB5_1;
+; CHECK-NEXT:  // %bb.2: // %atomicrmw.end
+; CHECK-NEXT:    cvt.u16.u32 %rs11, %r1;
+; CHECK-NEXT:    mov.b16 %rs12, 0x0000;
+; CHECK-NEXT:    mov.b32 %r2, {%rs11, %rs12};
+; CHECK-NEXT:    mov.b32 %r13, 0;
+; CHECK-NEXT:    add.s64 %rd6, %rd5, 2;
+; CHECK-NEXT:    mov.b32 {_, %rs2}, %r25;
+; CHECK-NEXT:    and.b64 %rd1, %rd6, -4;
+; CHECK-NEXT:    cvt.u32.u64 %r32, %rd6;
+; CHECK-NEXT:    and.b32 %r33, %r32, 3;
+; CHECK-NEXT:    shl.b32 %r3, %r33, 3;
+; CHECK-NEXT:    mov.b32 %r34, 65535;
+; CHECK-NEXT:    shl.b32 %r35, %r34, %r3;
+; CHECK-NEXT:    not.b32 %r4, %r35;
+; CHECK-NEXT:    ld.b32 %r83, [%rd1];
+; CHECK-NEXT:  $L__BB5_3: // %atomicrmw.start5
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    shr.u32 %r36, %r83, %r3;
+; CHECK-NEXT:    cvt.u16.u32 %rs13, %r36;
+; CHECK-NEXT:    min.f16 %rs14, %rs13, %rs2;
+; CHECK-NEXT:    cvt.u32.u16 %r37, %rs14;
+; CHECK-NEXT:    shl.b32 %r38, %r37, %r3;
+; CHECK-NEXT:    and.b32 %r39, %r83, %r4;
+; CHECK-NEXT:    or.b32 %r40, %r39, %r38;
+; CHECK-NEXT:    atom.relaxed.sys.cas.b32 %r5, [%rd1], %r83, %r40;
+; CHECK-NEXT:    setp.ne.b32 %p2, %r5, %r83;
+; CHECK-NEXT:    mov.b32 %r83, %r5;
+; CHECK-NEXT:    @%p2 bra $L__BB5_3;
+; CHECK-NEXT:  // %bb.4: // %atomicrmw.end4
+; CHECK-NEXT:    shr.u32 %r41, %r5, %r3;
+; CHECK-NEXT:    cvt.u16.u32 %rs15, %r41;
+; CHECK-NEXT:    mov.b32 {%rs16, _}, %r2;
+; CHECK-NEXT:    mov.b32 %r18, {%rs16, %rs15};
+; CHECK-NEXT:    mov.b32 {%rs3, _}, %r26;
+; CHECK-NEXT:    ld.b32 %r84, [%rd5+4];
+; CHECK-NEXT:  $L__BB5_5: // %atomicrmw.start20
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    cvt.u16.u32 %rs17, %r84;
+; CHECK-NEXT:    min.f16 %rs18, %rs17, %rs3;
+; CHECK-NEXT:    cvt.u32.u16 %r42, %rs18;
+; CHECK-NEXT:    and.b32 %r43, %r84, -65536;
+; CHECK-NEXT:    or.b32 %r44, %r43, %r42;
+; CHECK-NEXT:    atom.relaxed.sys.cas.b32 %r6, [%rd5+4], %r84, %r44;
+; CHECK-NEXT:    setp.ne.b32 %p3, %r6, %r84;
+; CHECK-NEXT:    mov.b32 %r84, %r6;
+; CHECK-NEXT:    @%p3 bra $L__BB5_5;
+; CHECK-NEXT:  // %bb.6: // %atomicrmw.end19
+; CHECK-NEXT:    cvt.u16.u32 %rs19, %r6;
+; CHECK-NEXT:    mov.b32 {_, %rs20}, %r13;
+; CHECK-NEXT:    mov.b32 %r7, {%rs19, %rs20};
+; CHECK-NEXT:    add.s64 %rd7, %rd5, 6;
+; CHECK-NEXT:    mov.b32 {_, %rs4}, %r26;
+; CHECK-NEXT:    and.b64 %rd2, %rd7, -4;
+; CHECK-NEXT:    cvt.u32.u64 %r45, %rd7;
+; CHECK-NEXT:    and.b32 %r46, %r45, 3;
+; CHECK-NEXT:    shl.b32 %r8, %r46, 3;
+; CHECK-NEXT:    mov.b32 %r47, 65535;
+; CHECK-NEXT:    shl.b32 %r48, %r47, %r8;
+; CHECK-NEXT:    not.b32 %r9, %r48;
+; CHECK-NEXT:    ld.b32 %r85, [%rd2];
+; CHECK-NEXT:  $L__BB5_7: // %atomicrmw.start38
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    shr.u32 %r49, %r85, %r8;
+; CHECK-NEXT:    cvt.u16.u32 %rs21, %r49;
+; CHECK-NEXT:    min.f16 %rs22, %rs21, %rs4;
+; CHECK-NEXT:    cvt.u32.u16 %r50, %rs22;
+; CHECK-NEXT:    shl.b32 %r51, %r50, %r8;
+; CHECK-NEXT:    and.b32 %r52, %r85, %r9;
+; CHECK-NEXT:    or.b32 %r53, %r52, %r51;
+; CHECK-NEXT:    atom.relaxed.sys.cas.b32 %r10, [%rd2], %r85, %r53;
+; CHECK-NEXT:    setp.ne.b32 %p4, %r10, %r85;
+; CHECK-NEXT:    mov.b32 %r85, %r10;
+; CHECK-NEXT:    @%p4 bra $L__BB5_7;
+; CHECK-NEXT:  // %bb.8: // %atomicrmw.end37
+; CHECK-NEXT:    shr.u32 %r54, %r10, %r8;
+; CHECK-NEXT:    cvt.u16.u32 %rs23, %r54;
+; CHECK-NEXT:    mov.b32 {%rs24, _}, %r7;
+; CHECK-NEXT:    mov.b32 %r19, {%rs24, %rs23};
+; CHECK-NEXT:    mov.b32 {%rs5, _}, %r27;
+; CHECK-NEXT:    ld.b32 %r86, [%rd5+8];
+; CHECK-NEXT:  $L__BB5_9: // %atomicrmw.start54
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    cvt.u16.u32 %rs25, %r86;
+; CHECK-NEXT:    min.f16 %rs26, %rs25, %rs5;
+; CHECK-NEXT:    cvt.u32.u16 %r55, %rs26;
+; CHECK-NEXT:    and.b32 %r56, %r86, -65536;
+; CHECK-NEXT:    or.b32 %r57, %r56, %r55;
+; CHECK-NEXT:    atom.relaxed.sys.cas.b32 %r11, [%rd5+8], %r86, %r57;
+; CHECK-NEXT:    setp.ne.b32 %p5, %r11, %r86;
+; CHECK-NEXT:    mov.b32 %r86, %r11;
+; CHECK-NEXT:    @%p5 bra $L__BB5_9;
+; CHECK-NEXT:  // %bb.10: // %atomicrmw.end53
+; CHECK-NEXT:    cvt.u16.u32 %rs27, %r11;
+; CHECK-NEXT:    mov.b32 {_, %rs28}, %r13;
+; CHECK-NEXT:    mov.b32 %r12, {%rs27, %rs28};
+; CHECK-NEXT:    add.s64 %rd8, %rd5, 10;
+; CHECK-NEXT:    mov.b32 {_, %rs6}, %r27;
+; CHECK-NEXT:    and.b64 %rd3, %rd8, -4;
+; CHECK-NEXT:    cvt.u32.u64 %r58, %rd8;
+; CHECK-NEXT:    and.b32 %r59, %r58, 3;
+; CHECK-NEXT:    shl.b32 %r14, %r59, 3;
+; CHECK-NEXT:    mov.b32 %r60, 65535;
+; CHECK-NEXT:    shl.b32 %r61, %r60, %r14;
+; CHECK-NEXT:    not.b32 %r15, %r61;
+; CHECK-NEXT:    ld.b32 %r87, [%rd3];
+; CHECK-NEXT:  $L__BB5_11: // %atomicrmw.start72
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    shr.u32 %r62, %r87, %r14;
+; CHECK-NEXT:    cvt.u16.u32 %rs29, %r62;
+; CHECK-NEXT:    min.f16 %rs30, %rs29, %rs6;
+; CHECK-NEXT:    cvt.u32.u16 %r63, %rs30;
+; CHECK-NEXT:    shl.b32 %r64, %r63, %r14;
+; CHECK-NEXT:    and.b32 %r65, %r87, %r15;
+; CHECK-NEXT:    or.b32 %r66, %r65, %r64;
+; CHECK-NEXT:    atom.relaxed.sys.cas.b32 %r16, [%rd3], %r87, %r66;
+; CHECK-NEXT:    setp.ne.b32 %p6, %r16, %r87;
+; CHECK-NEXT:    mov.b32 %r87, %r16;
+; CHECK-NEXT:    @%p6 bra $L__BB5_11;
+; CHECK-NEXT:  // %bb.12: // %atomicrmw.end71
+; CHECK-NEXT:    shr.u32 %r67, %r16, %r14;
+; CHECK-NEXT:    cvt.u16.u32 %rs31, %r67;
+; CHECK-NEXT:    mov.b32 {%rs32, _}, %r12;
+; CHECK-NEXT:    mov.b32 %r20, {%rs32, %rs31};
+; CHECK-NEXT:    mov.b32 {%rs7, _}, %r28;
+; CHECK-NEXT:    ld.b32 %r88, [%rd5+12];
+; CHECK-NEXT:  $L__BB5_13: // %atomicrmw.start88
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    cvt.u16.u32 %rs33, %r88;
+; CHECK-NEXT:    min.f16 %rs34, %rs33, %rs7;
+; CHECK-NEXT:    cvt.u32.u16 %r68, %rs34;
+; CHECK-NEXT:    and.b32 %r69, %r88, -65536;
+; CHECK-NEXT:    or.b32 %r70, %r69, %r68;
+; CHECK-NEXT:    atom.relaxed.sys.cas.b32 %r17, [%rd5+12], %r88, %r70;
+; CHECK-NEXT:    setp.ne.b32 %p7, %r17, %r88;
+; CHECK-NEXT:    mov.b32 %r88, %r17;
+; CHECK-NEXT:    @%p7 bra $L__BB5_13;
+; CHECK-NEXT:  // %bb.14: // %atomicrmw.end87
+; CHECK-NEXT:    cvt.u16.u32 %rs35, %r17;
+; CHECK-NEXT:    mov.b32 {_, %rs36}, %r13;
+; CHECK-NEXT:    mov.b32 %r21, {%rs35, %rs36};
+; CHECK-NEXT:    add.s64 %rd9, %rd5, 14;
+; CHECK-NEXT:    mov.b32 {_, %rs8}, %r28;
+; CHECK-NEXT:    and.b64 %rd4, %rd9, -4;
+; CHECK-NEXT:    cvt.u32.u64 %r71, %rd9;
+; CHECK-NEXT:    and.b32 %r72, %r71, 3;
+; CHECK-NEXT:    shl.b32 %r22, %r72, 3;
+; CHECK-NEXT:    mov.b32 %r73, 65535;
+; CHECK-NEXT:    shl.b32 %r74, %r73, %r22;
+; CHECK-NEXT:    not.b32 %r23, %r74;
+; CHECK-NEXT:    ld.b32 %r89, [%rd4];
+; CHECK-NEXT:  $L__BB5_15: // %atomicrmw.start106
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    shr.u32 %r75, %r89, %r22;
+; CHECK-NEXT:    cvt.u16.u32 %rs37, %r75;
+; CHECK-NEXT:    min.f16 %rs38, %rs37, %rs8;
+; CHECK-NEXT:    cvt.u32.u16 %r76, %rs38;
+; CHECK-NEXT:    shl.b32 %r77, %r76, %r22;
+; CHECK-NEXT:    and.b32 %r78, %r89, %r23;
+; CHECK-NEXT:    or.b32 %r79, %r78, %r77;
+; CHECK-NEXT:    atom.relaxed.sys.cas.b32 %r24, [%rd4], %r89, %r79;
+; CHECK-NEXT:    setp.ne.b32 %p8, %r24, %r89;
+; CHECK-NEXT:    mov.b32 %r89, %r24;
+; CHECK-NEXT:    @%p8 bra $L__BB5_15;
+; CHECK-NEXT:  // %bb.16: // %atomicrmw.end105
+; CHECK-NEXT:    shr.u32 %r80, %r24, %r22;
+; CHECK-NEXT:    cvt.u16.u32 %rs39, %r80;
+; CHECK-NEXT:    mov.b32 {%rs40, _}, %r21;
+; CHECK-NEXT:    mov.b32 %r81, {%rs40, %rs39};
+; CHECK-NEXT:    st.param.v4.b32 [func_retval0], {%r18, %r19, %r20, %r81};
+; CHECK-NEXT:    ret;
+entry:
+  %old = atomicrmw elementwise fmin ptr %addr, <8 x half> %val monotonic, align 16
+  ret <8 x half> %old
+}
+
+define <8 x half> @fmax_v8f16_elementwise(ptr %addr, <8 x half> %val) {
+; CHECK-LABEL: fmax_v8f16_elementwise(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<9>;
+; CHECK-NEXT:    .reg .b16 %rs<41>;
+; CHECK-NEXT:    .reg .b32 %r<90>;
+; CHECK-NEXT:    .reg .b64 %rd<10>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0: // %entry
+; CHECK-NEXT:    ld.param.v4.b32 {%r25, %r26, %r27, %r28}, [fmax_v8f16_elementwise_param_1];
+; CHECK-NEXT:    ld.param.b64 %rd5, [fmax_v8f16_elementwise_param_0];
+; CHECK-NEXT:    mov.b32 {%rs1, _}, %r25;
+; CHECK-NEXT:    ld.b32 %r82, [%rd5];
+; CHECK-NEXT:  $L__BB6_1: // %atomicrmw.start
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    cvt.u16.u32 %rs9, %r82;
+; CHECK-NEXT:    max.f16 %rs10, %rs9, %rs1;
+; CHECK-NEXT:    cvt.u32.u16 %r29, %rs10;
+; CHECK-NEXT:    and.b32 %r30, %r82, -65536;
+; CHECK-NEXT:    or.b32 %r31, %r30, %r29;
+; CHECK-NEXT:    atom.relaxed.sys.cas.b32 %r1, [%rd5], %r82, %r31;
+; CHECK-NEXT:    setp.ne.b32 %p1, %r1, %r82;
+; CHECK-NEXT:    mov.b32 %r82, %r1;
+; CHECK-NEXT:    @%p1 bra $L__BB6_1;
+; CHECK-NEXT:  // %bb.2: // %atomicrmw.end
+; CHECK-NEXT:    cvt.u16.u32 %rs11, %r1;
+; CHECK-NEXT:    mov.b16 %rs12, 0x0000;
+; CHECK-NEXT:    mov.b32 %r2, {%rs11, %rs12};
+; CHECK-NEXT:    mov.b32 %r13, 0;
+; CHECK-NEXT:    add.s64 %rd6, %rd5, 2;
+; CHECK-NEXT:    mov.b32 {_, %rs2}, %r25;
+; CHECK-NEXT:    and.b64 %rd1, %rd6, -4;
+; CHECK-NEXT:    cvt.u32.u64 %r32, %rd6;
+; CHECK-NEXT:    and.b32 %r33, %r32, 3;
+; CHECK-NEXT:    shl.b32 %r3, %r33, 3;
+; CHECK-NEXT:    mov.b32 %r34, 65535;
+; CHECK-NEXT:    shl.b32 %r35, %r34, %r3;
+; CHECK-NEXT:    not.b32 %r4, %r35;
+; CHECK-NEXT:    ld.b32 %r83, [%rd1];
+; CHECK-NEXT:  $L__BB6_3: // %atomicrmw.start5
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    shr.u32 %r36, %r83, %r3;
+; CHECK-NEXT:    cvt.u16.u32 %rs13, %r36;
+; CHECK-NEXT:    max.f16 %rs14, %rs13, %rs2;
+; CHECK-NEXT:    cvt.u32.u16 %r37, %rs14;
+; CHECK-NEXT:    shl.b32 %r38, %r37, %r3;
+; CHECK-NEXT:    and.b32 %r39, %r83, %r4;
+; CHECK-NEXT:    or.b32 %r40, %r39, %r38;
+; CHECK-NEXT:    atom.relaxed.sys.cas.b32 %r5, [%rd1], %r83, %r40;
+; CHECK-NEXT:    setp.ne.b32 %p2, %r5, %r83;
+; CHECK-NEXT:    mov.b32 %r83, %r5;
+; CHECK-NEXT:    @%p2 bra $L__BB6_3;
+; CHECK-NEXT:  // %bb.4: // %atomicrmw.end4
+; CHECK-NEXT:    shr.u32 %r41, %r5, %r3;
+; CHECK-NEXT:    cvt.u16.u32 %rs15, %r41;
+; CHECK-NEXT:    mov.b32 {%rs16, _}, %r2;
+; CHECK-NEXT:    mov.b32 %r18, {%rs16, %rs15};
+; CHECK-NEXT:    mov.b32 {%rs3, _}, %r26;
+; CHECK-NEXT:    ld.b32 %r84, [%rd5+4];
+; CHECK-NEXT:  $L__BB6_5: // %atomicrmw.start20
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    cvt.u16.u32 %rs17, %r84;
+; CHECK-NEXT:    max.f16 %rs18, %rs17, %rs3;
+; CHECK-NEXT:    cvt.u32.u16 %r42, %rs18;
+; CHECK-NEXT:    and.b32 %r43, %r84, -65536;
+; CHECK-NEXT:    or.b32 %r44, %r43, %r42;
+; CHECK-NEXT:    atom.relaxed.sys.cas.b32 %r6, [%rd5+4], %r84, %r44;
+; CHECK-NEXT:    setp.ne.b32 %p3, %r6, %r84;
+; CHECK-NEXT:    mov.b32 %r84, %r6;
+; CHECK-NEXT:    @%p3 bra $L__BB6_5;
+; CHECK-NEXT:  // %bb.6: // %atomicrmw.end19
+; CHECK-NEXT:    cvt.u16.u32 %rs19, %r6;
+; CHECK-NEXT:    mov.b32 {_, %rs20}, %r13;
+; CHECK-NEXT:    mov.b32 %r7, {%rs19, %rs20};
+; CHECK-NEXT:    add.s64 %rd7, %rd5, 6;
+; CHECK-NEXT:    mov.b32 {_, %rs4}, %r26;
+; CHECK-NEXT:    and.b64 %rd2, %rd7, -4;
+; CHECK-NEXT:    cvt.u32.u64 %r45, %rd7;
+; CHECK-NEXT:    and.b32 %r46, %r45, 3;
+; CHECK-NEXT:    shl.b32 %r8, %r46, 3;
+; CHECK-NEXT:    mov.b32 %r47, 65535;
+; CHECK-NEXT:    shl.b32 %r48, %r47, %r8;
+; CHECK-NEXT:    not.b32 %r9, %r48;
+; CHECK-NEXT:    ld.b32 %r85, [%rd2];
+; CHECK-NEXT:  $L__BB6_7: // %atomicrmw.start38
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    shr.u32 %r49, %r85, %r8;
+; CHECK-NEXT:    cvt.u16.u32 %rs21, %r49;
+; CHECK-NEXT:    max.f16 %rs22, %rs21, %rs4;
+; CHECK-NEXT:    cvt.u32.u16 %r50, %rs22;
+; CHECK-NEXT:    shl.b32 %r51, %r50, %r8;
+; CHECK-NEXT:    and.b32 %r52, %r85, %r9;
+; CHECK-NEXT:    or.b32 %r53, %r52, %r51;
+; CHECK-NEXT:    atom.relaxed.sys.cas.b32 %r10, [%rd2], %r85, %r53;
+; CHECK-NEXT:    setp.ne.b32 %p4, %r10, %r85;
+; CHECK-NEXT:    mov.b32 %r85, %r10;
+; CHECK-NEXT:    @%p4 bra $L__BB6_7;
+; CHECK-NEXT:  // %bb.8: // %atomicrmw.end37
+; CHECK-NEXT:    shr.u32 %r54, %r10, %r8;
+; CHECK-NEXT:    cvt.u16.u32 %rs23, %r54;
+; CHECK-NEXT:    mov.b32 {%rs24, _}, %r7;
+; CHECK-NEXT:    mov.b32 %r19, {%rs24, %rs23};
+; CHECK-NEXT:    mov.b32 {%rs5, _}, %r27;
+; CHECK-NEXT:    ld.b32 %r86, [%rd5+8];
+; CHECK-NEXT:  $L__BB6_9: // %atomicrmw.start54
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    cvt.u16.u32 %rs25, %r86;
+; CHECK-NEXT:    max.f16 %rs26, %rs25, %rs5;
+; CHECK-NEXT:    cvt.u32.u16 %r55, %rs26;
+; CHECK-NEXT:    and.b32 %r56, %r86, -65536;
+; CHECK-NEXT:    or.b32 %r57, %r56, %r55;
+; CHECK-NEXT:    atom.relaxed.sys.cas.b32 %r11, [%rd5+8], %r86, %r57;
+; CHECK-NEXT:    setp.ne.b32 %p5, %r11, %r86;
+; CHECK-NEXT:    mov.b32 %r86, %r11;
+; CHECK-NEXT:    @%p5 bra $L__BB6_9;
+; CHECK-NEXT:  // %bb.10: // %atomicrmw.end53
+; CHECK-NEXT:    cvt.u16.u32 %rs27, %r11;
+; CHECK-NEXT:    mov.b32 {_, %rs28}, %r13;
+; CHECK-NEXT:    mov.b32 %r12, {%rs27, %rs28};
+; CHECK-NEXT:    add.s64 %rd8, %rd5, 10;
+; CHECK-NEXT:    mov.b32 {_, %rs6}, %r27;
+; CHECK-NEXT:    and.b64 %rd3, %rd8, -4;
+; CHECK-NEXT:    cvt.u32.u64 %r58, %rd8;
+; CHECK-NEXT:    and.b32 %r59, %r58, 3;
+; CHECK-NEXT:    shl.b32 %r14, %r59, 3;
+; CHECK-NEXT:    mov.b32 %r60, 65535;
+; CHECK-NEXT:    shl.b32 %r61, %r60, %r14;
+; CHECK-NEXT:    not.b32 %r15, %r61;
+; CHECK-NEXT:    ld.b32 %r87, [%rd3];
+; CHECK-NEXT:  $L__BB6_11: // %atomicrmw.start72
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    shr.u32 %r62, %r87, %r14;
+; CHECK-NEXT:    cvt.u16.u32 %rs29, %r62;
+; CHECK-NEXT:    max.f16 %rs30, %rs29, %rs6;
+; CHECK-NEXT:    cvt.u32.u16 %r63, %rs30;
+; CHECK-NEXT:    shl.b32 %r64, %r63, %r14;
+; CHECK-NEXT:    and.b32 %r65, %r87, %r15;
+; CHECK-NEXT:    or.b32 %r66, %r65, %r64;
+; CHECK-NEXT:    atom.relaxed.sys.cas.b32 %r16, [%rd3], %r87, %r66;
+; CHECK-NEXT:    setp.ne.b32 %p6, %r16, %r87;
+; CHECK-NEXT:    mov.b32 %r87, %r16;
+; CHECK-NEXT:    @%p6 bra $L__BB6_11;
+; CHECK-NEXT:  // %bb.12: // %atomicrmw.end71
+; CHECK-NEXT:    shr.u32 %r67, %r16, %r14;
+; CHECK-NEXT:    cvt.u16.u32 %rs31, %r67;
+; CHECK-NEXT:    mov.b32 {%rs32, _}, %r12;
+; CHECK-NEXT:    mov.b32 %r20, {%rs32, %rs31};
+; CHECK-NEXT:    mov.b32 {%rs7, _}, %r28;
+; CHECK-NEXT:    ld.b32 %r88, [%rd5+12];
+; CHECK-NEXT:  $L__BB6_13: // %atomicrmw.start88
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    cvt.u16.u32 %rs33, %r88;
+; CHECK-NEXT:    max.f16 %rs34, %rs33, %rs7;
+; CHECK-NEXT:    cvt.u32.u16 %r68, %rs34;
+; CHECK-NEXT:    and.b32 %r69, %r88, -65536;
+; CHECK-NEXT:    or.b32 %r70, %r69, %r68;
+; CHECK-NEXT:    atom.relaxed.sys.cas.b32 %r17, [%rd5+12], %r88, %r70;
+; CHECK-NEXT:    setp.ne.b32 %p7, %r17, %r88;
+; CHECK-NEXT:    mov.b32 %r88, %r17;
+; CHECK-NEXT:    @%p7 bra $L__BB6_13;
+; CHECK-NEXT:  // %bb.14: // %atomicrmw.end87
+; CHECK-NEXT:    cvt.u16.u32 %rs35, %r17;
+; CHECK-NEXT:    mov.b32 {_, %rs36}, %r13;
+; CHECK-NEXT:    mov.b32 %r21, {%rs35, %rs36};
+; CHECK-NEXT:    add.s64 %rd9, %rd5, 14;
+; CHECK-NEXT:    mov.b32 {_, %rs8}, %r28;
+; CHECK-NEXT:    and.b64 %rd4, %rd9, -4;
+; CHECK-NEXT:    cvt.u32.u64 %r71, %rd9;
+; CHECK-NEXT:    and.b32 %r72, %r71, 3;
+; CHECK-NEXT:    shl.b32 %r22, %r72, 3;
+; CHECK-NEXT:    mov.b32 %r73, 65535;
+; CHECK-NEXT:    shl.b32 %r74, %r73, %r22;
+; CHECK-NEXT:    not.b32 %r23, %r74;
+; CHECK-NEXT:    ld.b32 %r89, [%rd4];
+; CHECK-NEXT:  $L__BB6_15: // %atomicrmw.start106
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    shr.u32 %r75, %r89, %r22;
+; CHECK-NEXT:    cvt.u16.u32 %rs37, %r75;
+; CHECK-NEXT:    max.f16 %rs38, %rs37, %rs8;
+; CHECK-NEXT:    cvt.u32.u16 %r76, %rs38;
+; CHECK-NEXT:    shl.b32 %r77, %r76, %r22;
+; CHECK-NEXT:    and.b32 %r78, %r89, %r23;
+; CHECK-NEXT:    or.b32 %r79, %r78, %r77;
+; CHECK-NEXT:    atom.relaxed.sys.cas.b32 %r24, [%rd4], %r89, %r79;
+; CHECK-NEXT:    setp.ne.b32 %p8, %r24, %r89;
+; CHECK-NEXT:    mov.b32 %r89, %r24;
+; CHECK-NEXT:    @%p8 bra $L__BB6_15;
+; CHECK-NEXT:  // %bb.16: // %atomicrmw.end105
+; CHECK-NEXT:    shr.u32 %r80, %r24, %r22;
+; CHECK-NEXT:    cvt.u16.u32 %rs39, %r80;
+; CHECK-NEXT:    mov.b32 {%rs40, _}, %r21;
+; CHECK-NEXT:    mov.b32 %r81, {%rs40, %rs39};
+; CHECK-NEXT:    st.param.v4.b32 [func_retval0], {%r18, %r19, %r20, %r81};
+; CHECK-NEXT:    ret;
+entry:
+  %old = atomicrmw elementwise fmax ptr %addr, <8 x half> %val monotonic, align 16
+  ret <8 x half> %old
+}
diff --git a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-elementwise.ll b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-elementwise.ll
new file mode 100644
index 0000000000000..e339213461ef0
--- /dev/null
+++ b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-elementwise.ll
@@ -0,0 +1,138 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -S -passes='require<libcall-lowering-info>,atomic-expand' -mcpu=gfx942 %s | FileCheck -check-prefixes=GFX942 %s
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -S -passes='require<libcall-lowering-info>,atomic-expand' -mcpu=gfx900 %s | FileCheck -check-prefixes=GFX900 %s
+
+; gfx942 has ds_pk_add_f16/bf16, so atomicrmw fadd <2 x half> is natively
+; supported as a whole-value operation. The elementwise modifier should be
+; dropped and the instruction preserved.
+;
+; gfx900 lacks ds_pk_add, so the elementwise atomicrmw must be scalarized
+; into per-lane scalar atomicrmw instructions.
+
+define <2 x half> @elementwise_fadd_v2f16_lds(ptr addrspace(3) %ptr, <2 x half> %val) {
+; GFX942-LABEL: define <2 x half> @elementwise_fadd_v2f16_lds(
+; GFX942-SAME: ptr addrspace(3) [[PTR:%.*]], <2 x half> [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {
+; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR]], <2 x half> [[VAL]] monotonic, align 4
+; GFX942-NEXT:    ret <2 x half> [[RES]]
+;
+; GFX900-LABEL: define <2 x half> @elementwise_fadd_v2f16_lds(
+; GFX900-SAME: ptr addrspace(3) [[PTR:%.*]], <2 x half> [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {
+; GFX900-NEXT:    [[LANE_VAL:%.*]] = extractelement <2 x half> [[VAL]], i64 0
+; GFX900-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(3) [[PTR]], align 4
+; GFX900-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; GFX900:       [[ATOMICRMW_START]]:
+; GFX900-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; GFX900-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[LOADED]] to i16
+; GFX900-NEXT:    [[TMP2:%.*]] = bitcast i16 [[EXTRACTED]] to half
+; GFX900-NEXT:    [[NEW:%.*]] = fadd half [[TMP2]], [[LANE_VAL]]
+; GFX900-NEXT:    [[TMP3:%.*]] = bitcast half [[NEW]] to i16
+; GFX900-NEXT:    [[EXTENDED:%.*]] = zext i16 [[TMP3]] to i32
+; GFX900-NEXT:    [[UNMASKED:%.*]] = and i32 [[LOADED]], -65536
+; GFX900-NEXT:    [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[EXTENDED]]
+; GFX900-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i32 [[LOADED]], i32 [[INSERTED]] monotonic monotonic, align 4
+; GFX900-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1
+; GFX900-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP4]], 0
+; GFX900-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; GFX900:       [[ATOMICRMW_END]]:
+; GFX900-NEXT:    [[EXTRACTED1:%.*]] = trunc i32 [[NEWLOADED]] to i16
+; GFX900-NEXT:    [[TMP5:%.*]] = bitcast i16 [[EXTRACTED1]] to half
+; GFX900-NEXT:    [[LANE_OLD:%.*]] = insertelement <2 x half> zeroinitializer, half [[TMP5]], i64 0
+; GFX900-NEXT:    [[LANE_PTR:%.*]] = getelementptr inbounds <2 x half>, ptr addrspace(3) [[PTR]], i64 0, i64 1
+; GFX900-NEXT:    [[LANE_VAL2:%.*]] = extractelement <2 x half> [[VAL]], i64 1
+; GFX900-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(3) @llvm.ptrmask.p3.i32(ptr addrspace(3) [[LANE_PTR]], i32 -4)
+; GFX900-NEXT:    [[TMP6:%.*]] = ptrtoint ptr addrspace(3) [[LANE_PTR]] to i32
+; GFX900-NEXT:    [[PTRLSB:%.*]] = and i32 [[TMP6]], 3
+; GFX900-NEXT:    [[TMP7:%.*]] = shl i32 [[PTRLSB]], 3
+; GFX900-NEXT:    [[MASK:%.*]] = shl i32 65535, [[TMP7]]
+; GFX900-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
+; GFX900-NEXT:    [[TMP8:%.*]] = load i32, ptr addrspace(3) [[ALIGNEDADDR]], align 4
+; GFX900-NEXT:    br label %[[ATOMICRMW_START5:.*]]
+; GFX900:       [[ATOMICRMW_START5]]:
+; GFX900-NEXT:    [[LOADED6:%.*]] = phi i32 [ [[TMP8]], %[[ATOMICRMW_END]] ], [ [[NEWLOADED14:%.*]], %[[ATOMICRMW_START5]] ]
+; GFX900-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[LOADED6]], [[TMP7]]
+; GFX900-NEXT:    [[EXTRACTED7:%.*]] = trunc i32 [[SHIFTED]] to i16
+; GFX900-NEXT:    [[TMP9:%.*]] = bitcast i16 [[EXTRACTED7]] to half
+; GFX900-NEXT:    [[NEW8:%.*]] = fadd half [[TMP9]], [[LANE_VAL2]]
+; GFX900-NEXT:    [[TMP10:%.*]] = bitcast half [[NEW8]] to i16
+; GFX900-NEXT:    [[EXTENDED9:%.*]] = zext i16 [[TMP10]] to i32
+; GFX900-NEXT:    [[SHIFTED10:%.*]] = shl nuw i32 [[EXTENDED9]], [[TMP7]]
+; GFX900-NEXT:    [[UNMASKED11:%.*]] = and i32 [[LOADED6]], [[INV_MASK]]
+; GFX900-NEXT:    [[INSERTED12:%.*]] = or i32 [[UNMASKED11]], [[SHIFTED10]]
+; GFX900-NEXT:    [[TMP11:%.*]] = cmpxchg ptr addrspace(3) [[ALIGNEDADDR]], i32 [[LOADED6]], i32 [[INSERTED12]] monotonic monotonic, align 4
+; GFX900-NEXT:    [[SUCCESS13:%.*]] = extractvalue { i32, i1 } [[TMP11]], 1
+; GFX900-NEXT:    [[NEWLOADED14]] = extractvalue { i32, i1 } [[TMP11]], 0
+; GFX900-NEXT:    br i1 [[SUCCESS13]], label %[[ATOMICRMW_END4:.*]], label %[[ATOMICRMW_START5]]
+; GFX900:       [[ATOMICRMW_END4]]:
+; GFX900-NEXT:    [[SHIFTED15:%.*]] = lshr i32 [[NEWLOADED14]], [[TMP7]]
+; GFX900-NEXT:    [[EXTRACTED16:%.*]] = trunc i32 [[SHIFTED15]] to i16
+; GFX900-NEXT:    [[TMP12:%.*]] = bitcast i16 [[EXTRACTED16]] to half
+; GFX900-NEXT:    [[LANE_OLD3:%.*]] = insertelement <2 x half> [[LANE_OLD]], half [[TMP12]], i64 1
+; GFX900-NEXT:    ret <2 x half> [[LANE_OLD3]]
+;
+  %res = atomicrmw elementwise fadd ptr addrspace(3) %ptr, <2 x half> %val monotonic, align 4
+  ret <2 x half> %res
+}
+
+define <2 x bfloat> @elementwise_fadd_v2bf16_lds(ptr addrspace(3) %ptr, <2 x bfloat> %val) {
+; GFX942-LABEL: define <2 x bfloat> @elementwise_fadd_v2bf16_lds(
+; GFX942-SAME: ptr addrspace(3) [[PTR:%.*]], <2 x bfloat> [[VAL:%.*]]) #[[ATTR0]] {
+; GFX942-NEXT:    [[RES:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR]], <2 x bfloat> [[VAL]] monotonic, align 4
+; GFX942-NEXT:    ret <2 x bfloat> [[RES]]
+;
+; GFX900-LABEL: define <2 x bfloat> @elementwise_fadd_v2bf16_lds(
+; GFX900-SAME: ptr addrspace(3) [[PTR:%.*]], <2 x bfloat> [[VAL:%.*]]) #[[ATTR0]] {
+; GFX900-NEXT:    [[LANE_VAL:%.*]] = extractelement <2 x bfloat> [[VAL]], i64 0
+; GFX900-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(3) [[PTR]], align 4
+; GFX900-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; GFX900:       [[ATOMICRMW_START]]:
+; GFX900-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; GFX900-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[LOADED]] to i16
+; GFX900-NEXT:    [[TMP2:%.*]] = bitcast i16 [[EXTRACTED]] to bfloat
+; GFX900-NEXT:    [[NEW:%.*]] = fadd bfloat [[TMP2]], [[LANE_VAL]]
+; GFX900-NEXT:    [[TMP3:%.*]] = bitcast bfloat [[NEW]] to i16
+; GFX900-NEXT:    [[EXTENDED:%.*]] = zext i16 [[TMP3]] to i32
+; GFX900-NEXT:    [[UNMASKED:%.*]] = and i32 [[LOADED]], -65536
+; GFX900-NEXT:    [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[EXTENDED]]
+; GFX900-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i32 [[LOADED]], i32 [[INSERTED]] monotonic monotonic, align 4
+; GFX900-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1
+; GFX900-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP4]], 0
+; GFX900-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; GFX900:       [[ATOMICRMW_END]]:
+; GFX900-NEXT:    [[EXTRACTED1:%.*]] = trunc i32 [[NEWLOADED]] to i16
+; GFX900-NEXT:    [[TMP5:%.*]] = bitcast i16 [[EXTRACTED1]] to bfloat
+; GFX900-NEXT:    [[LANE_OLD:%.*]] = insertelement <2 x bfloat> zeroinitializer, bfloat [[TMP5]], i64 0
+; GFX900-NEXT:    [[LANE_PTR:%.*]] = getelementptr inbounds <2 x bfloat>, ptr addrspace(3) [[PTR]], i64 0, i64 1
+; GFX900-NEXT:    [[LANE_VAL2:%.*]] = extractelement <2 x bfloat> [[VAL]], i64 1
+; GFX900-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(3) @llvm.ptrmask.p3.i32(ptr addrspace(3) [[LANE_PTR]], i32 -4)
+; GFX900-NEXT:    [[TMP6:%.*]] = ptrtoint ptr addrspace(3) [[LANE_PTR]] to i32
+; GFX900-NEXT:    [[PTRLSB:%.*]] = and i32 [[TMP6]], 3
+; GFX900-NEXT:    [[TMP7:%.*]] = shl i32 [[PTRLSB]], 3
+; GFX900-NEXT:    [[MASK:%.*]] = shl i32 65535, [[TMP7]]
+; GFX900-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
+; GFX900-NEXT:    [[TMP8:%.*]] = load i32, ptr addrspace(3) [[ALIGNEDADDR]], align 4
+; GFX900-NEXT:    br label %[[ATOMICRMW_START5:.*]]
+; GFX900:       [[ATOMICRMW_START5]]:
+; GFX900-NEXT:    [[LOADED6:%.*]] = phi i32 [ [[TMP8]], %[[ATOMICRMW_END]] ], [ [[NEWLOADED14:%.*]], %[[ATOMICRMW_START5]] ]
+; GFX900-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[LOADED6]], [[TMP7]]
+; GFX900-NEXT:    [[EXTRACTED7:%.*]] = trunc i32 [[SHIFTED]] to i16
+; GFX900-NEXT:    [[TMP9:%.*]] = bitcast i16 [[EXTRACTED7]] to bfloat
+; GFX900-NEXT:    [[NEW8:%.*]] = fadd bfloat [[TMP9]], [[LANE_VAL2]]
+; GFX900-NEXT:    [[TMP10:%.*]] = bitcast bfloat [[NEW8]] to i16
+; GFX900-NEXT:    [[EXTENDED9:%.*]] = zext i16 [[TMP10]] to i32
+; GFX900-NEXT:    [[SHIFTED10:%.*]] = shl nuw i32 [[EXTENDED9]], [[TMP7]]
+; GFX900-NEXT:    [[UNMASKED11:%.*]] = and i32 [[LOADED6]], [[INV_MASK]]
+; GFX900-NEXT:    [[INSERTED12:%.*]] = or i32 [[UNMASKED11]], [[SHIFTED10]]
+; GFX900-NEXT:    [[TMP11:%.*]] = cmpxchg ptr addrspace(3) [[ALIGNEDADDR]], i32 [[LOADED6]], i32 [[INSERTED12]] monotonic monotonic, align 4
+; GFX900-NEXT:    [[SUCCESS13:%.*]] = extractvalue { i32, i1 } [[TMP11]], 1
+; GFX900-NEXT:    [[NEWLOADED14]] = extractvalue { i32, i1 } [[TMP11]], 0
+; GFX900-NEXT:    br i1 [[SUCCESS13]], label %[[ATOMICRMW_END4:.*]], label %[[ATOMICRMW_START5]]
+; GFX900:       [[ATOMICRMW_END4]]:
+; GFX900-NEXT:    [[SHIFTED15:%.*]] = lshr i32 [[NEWLOADED14]], [[TMP7]]
+; GFX900-NEXT:    [[EXTRACTED16:%.*]] = trunc i32 [[SHIFTED15]] to i16
+; GFX900-NEXT:    [[TMP12:%.*]] = bitcast i16 [[EXTRACTED16]] to bfloat
+; GFX900-NEXT:    [[LANE_OLD3:%.*]] = insertelement <2 x bfloat> [[LANE_OLD]], bfloat [[TMP12]], i64 1
+; GFX900-NEXT:    ret <2 x bfloat> [[LANE_OLD3]]
+;
+  %res = atomicrmw elementwise fadd ptr addrspace(3) %ptr, <2 x bfloat> %val monotonic, align 4
+  ret <2 x bfloat> %res
+}
diff --git a/llvm/test/Transforms/AtomicExpand/NVPTX/expand-atomic-rmw-elementwise.ll b/llvm/test/Transforms/AtomicExpand/NVPTX/expand-atomic-rmw-elementwise.ll
new file mode 100644
index 0000000000000..d6137ade559aa
--- /dev/null
+++ b/llvm/test/Transforms/AtomicExpand/NVPTX/expand-atomic-rmw-elementwise.ll
@@ -0,0 +1,262 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S %s -passes='require<libcall-lowering-info>,atomic-expand' -mtriple=nvptx64-nvidia-cuda -mcpu=sm_90 -mattr=+ptx83 | FileCheck --check-prefix=SM90PTX83 %s
+; RUN: opt -S %s -passes='require<libcall-lowering-info>,atomic-expand' -mtriple=nvptx64-nvidia-cuda -mcpu=sm_90 -mattr=+ptx81 | FileCheck --check-prefix=SM90PTX81 %s
+; RUN: opt -S %s -passes='require<libcall-lowering-info>,atomic-expand' -mtriple=nvptx64-nvidia-cuda -mcpu=sm_90 | FileCheck --check-prefix=SM90 %s
+
+target triple = "nvptx64-nvidia-cuda"
+
+; v4f32 is 128 bits: preserved at PTX 83+ (.b128 support), expanded otherwise.
+define <4 x float> @fadd_v4f32_monotonic(ptr %addr, <4 x float> %val) {
+; SM90PTX83-LABEL: define <4 x float> @fadd_v4f32_monotonic(
+; SM90PTX83-SAME: ptr [[ADDR:%.*]], <4 x float> [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {
+; SM90PTX83-NEXT:  [[ENTRY:.*:]]
+; SM90PTX83-NEXT:    [[LANE_VAL:%.*]] = extractelement <4 x float> [[VAL]], i64 0
+; SM90PTX83-NEXT:    [[TMP0:%.*]] = atomicrmw fadd ptr [[ADDR]], float [[LANE_VAL]] monotonic, align 16
+; SM90PTX83-NEXT:    [[LANE_OLD:%.*]] = insertelement <4 x float> zeroinitializer, float [[TMP0]], i64 0
+; SM90PTX83-NEXT:    [[LANE_PTR:%.*]] = getelementptr inbounds <4 x float>, ptr [[ADDR]], i64 0, i64 1
+; SM90PTX83-NEXT:    [[LANE_VAL1:%.*]] = extractelement <4 x float> [[VAL]], i64 1
+; SM90PTX83-NEXT:    [[TMP1:%.*]] = atomicrmw fadd ptr [[LANE_PTR]], float [[LANE_VAL1]] monotonic, align 4
+; SM90PTX83-NEXT:    [[LANE_OLD2:%.*]] = insertelement <4 x float> [[LANE_OLD]], float [[TMP1]], i64 1
+; SM90PTX83-NEXT:    [[LANE_PTR3:%.*]] = getelementptr inbounds <4 x float>, ptr [[ADDR]], i64 0, i64 2
+; SM90PTX83-NEXT:    [[LANE_VAL4:%.*]] = extractelement <4 x float> [[VAL]], i64 2
+; SM90PTX83-NEXT:    [[TMP2:%.*]] = atomicrmw fadd ptr [[LANE_PTR3]], float [[LANE_VAL4]] monotonic, align 8
+; SM90PTX83-NEXT:    [[LANE_OLD5:%.*]] = insertelement <4 x float> [[LANE_OLD2]], float [[TMP2]], i64 2
+; SM90PTX83-NEXT:    [[LANE_PTR6:%.*]] = getelementptr inbounds <4 x float>, ptr [[ADDR]], i64 0, i64 3
+; SM90PTX83-NEXT:    [[LANE_VAL7:%.*]] = extractelement <4 x float> [[VAL]], i64 3
+; SM90PTX83-NEXT:    [[TMP3:%.*]] = atomicrmw fadd ptr [[LANE_PTR6]], float [[LANE_VAL7]] monotonic, align 4
+; SM90PTX83-NEXT:    [[OLD:%.*]] = insertelement <4 x float> [[LANE_OLD5]], float [[TMP3]], i64 3
+; SM90PTX83-NEXT:    ret <4 x float> [[OLD]]
+;
+; SM90PTX81-LABEL: define <4 x float> @fadd_v4f32_monotonic(
+; SM90PTX81-SAME: ptr [[ADDR:%.*]], <4 x float> [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {
+; SM90PTX81-NEXT:  [[ENTRY:.*:]]
+; SM90PTX81-NEXT:    [[LANE_VAL:%.*]] = extractelement <4 x float> [[VAL]], i64 0
+; SM90PTX81-NEXT:    [[TMP0:%.*]] = atomicrmw fadd ptr [[ADDR]], float [[LANE_VAL]] monotonic, align 16
+; SM90PTX81-NEXT:    [[LANE_OLD:%.*]] = insertelement <4 x float> zeroinitializer, float [[TMP0]], i64 0
+; SM90PTX81-NEXT:    [[LANE_PTR:%.*]] = getelementptr inbounds <4 x float>, ptr [[ADDR]], i64 0, i64 1
+; SM90PTX81-NEXT:    [[LANE_VAL1:%.*]] = extractelement <4 x float> [[VAL]], i64 1
+; SM90PTX81-NEXT:    [[TMP1:%.*]] = atomicrmw fadd ptr [[LANE_PTR]], float [[LANE_VAL1]] monotonic, align 4
+; SM90PTX81-NEXT:    [[LANE_OLD2:%.*]] = insertelement <4 x float> [[LANE_OLD]], float [[TMP1]], i64 1
+; SM90PTX81-NEXT:    [[LANE_PTR3:%.*]] = getelementptr inbounds <4 x float>, ptr [[ADDR]], i64 0, i64 2
+; SM90PTX81-NEXT:    [[LANE_VAL4:%.*]] = extractelement <4 x float> [[VAL]], i64 2
+; SM90PTX81-NEXT:    [[TMP2:%.*]] = atomicrmw fadd ptr [[LANE_PTR3]], float [[LANE_VAL4]] monotonic, align 8
+; SM90PTX81-NEXT:    [[LANE_OLD5:%.*]] = insertelement <4 x float> [[LANE_OLD2]], float [[TMP2]], i64 2
+; SM90PTX81-NEXT:    [[LANE_PTR6:%.*]] = getelementptr inbounds <4 x float>, ptr [[ADDR]], i64 0, i64 3
+; SM90PTX81-NEXT:    [[LANE_VAL7:%.*]] = extractelement <4 x float> [[VAL]], i64 3
+; SM90PTX81-NEXT:    [[TMP3:%.*]] = atomicrmw fadd ptr [[LANE_PTR6]], float [[LANE_VAL7]] monotonic, align 4
+; SM90PTX81-NEXT:    [[LANE_OLD8:%.*]] = insertelement <4 x float> [[LANE_OLD5]], float [[TMP3]], i64 3
+; SM90PTX81-NEXT:    ret <4 x float> [[LANE_OLD8]]
+;
+; SM90-LABEL: define <4 x float> @fadd_v4f32_monotonic(
+; SM90-SAME: ptr [[ADDR:%.*]], <4 x float> [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {
+; SM90-NEXT:  [[ENTRY:.*:]]
+; SM90-NEXT:    [[LANE_VAL:%.*]] = extractelement <4 x float> [[VAL]], i64 0
+; SM90-NEXT:    [[TMP0:%.*]] = atomicrmw fadd ptr [[ADDR]], float [[LANE_VAL]] monotonic, align 16
+; SM90-NEXT:    [[LANE_OLD:%.*]] = insertelement <4 x float> zeroinitializer, float [[TMP0]], i64 0
+; SM90-NEXT:    [[LANE_PTR:%.*]] = getelementptr inbounds <4 x float>, ptr [[ADDR]], i64 0, i64 1
+; SM90-NEXT:    [[LANE_VAL1:%.*]] = extractelement <4 x float> [[VAL]], i64 1
+; SM90-NEXT:    [[TMP1:%.*]] = atomicrmw fadd ptr [[LANE_PTR]], float [[LANE_VAL1]] monotonic, align 4
+; SM90-NEXT:    [[LANE_OLD2:%.*]] = insertelement <4 x float> [[LANE_OLD]], float [[TMP1]], i64 1
+; SM90-NEXT:    [[LANE_PTR3:%.*]] = getelementptr inbounds <4 x float>, ptr [[ADDR]], i64 0, i64 2
+; SM90-NEXT:    [[LANE_VAL4:%.*]] = extractelement <4 x float> [[VAL]], i64 2
+; SM90-NEXT:    [[TMP2:%.*]] = atomicrmw fadd ptr [[LANE_PTR3]], float [[LANE_VAL4]] monotonic, align 8
+; SM90-NEXT:    [[LANE_OLD5:%.*]] = insertelement <4 x float> [[LANE_OLD2]], float [[TMP2]], i64 2
+; SM90-NEXT:    [[LANE_PTR6:%.*]] = getelementptr inbounds <4 x float>, ptr [[ADDR]], i64 0, i64 3
+; SM90-NEXT:    [[LANE_VAL7:%.*]] = extractelement <4 x float> [[VAL]], i64 3
+; SM90-NEXT:    [[TMP3:%.*]] = atomicrmw fadd ptr [[LANE_PTR6]], float [[LANE_VAL7]] monotonic, align 4
+; SM90-NEXT:    [[LANE_OLD8:%.*]] = insertelement <4 x float> [[LANE_OLD5]], float [[TMP3]], i64 3
+; SM90-NEXT:    ret <4 x float> [[LANE_OLD8]]
+;
+entry:
+  %old = atomicrmw elementwise fadd ptr %addr, <4 x float> %val monotonic, align 16
+  ret <4 x float> %old
+}
+
+; v2f32 is 64 bits: preserved at PTX 81+ (vector atom support), expanded at PTX 78.
+define <2 x float> @fadd_v2f32_monotonic(ptr %addr, <2 x float> %val) {
+; SM90PTX83-LABEL: define <2 x float> @fadd_v2f32_monotonic(
+; SM90PTX83-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) #[[ATTR0]] {
+; SM90PTX83-NEXT:  [[ENTRY:.*:]]
+; SM90PTX83-NEXT:    [[LANE_VAL:%.*]] = extractelement <2 x float> [[VAL]], i64 0
+; SM90PTX83-NEXT:    [[TMP0:%.*]] = atomicrmw fadd ptr [[ADDR]], float [[LANE_VAL]] monotonic, align 8
+; SM90PTX83-NEXT:    [[LANE_OLD:%.*]] = insertelement <2 x float> zeroinitializer, float [[TMP0]], i64 0
+; SM90PTX83-NEXT:    [[LANE_PTR:%.*]] = getelementptr inbounds <2 x float>, ptr [[ADDR]], i64 0, i64 1
+; SM90PTX83-NEXT:    [[LANE_VAL1:%.*]] = extractelement <2 x float> [[VAL]], i64 1
+; SM90PTX83-NEXT:    [[TMP1:%.*]] = atomicrmw fadd ptr [[LANE_PTR]], float [[LANE_VAL1]] monotonic, align 4
+; SM90PTX83-NEXT:    [[OLD:%.*]] = insertelement <2 x float> [[LANE_OLD]], float [[TMP1]], i64 1
+; SM90PTX83-NEXT:    ret <2 x float> [[OLD]]
+;
+; SM90PTX81-LABEL: define <2 x float> @fadd_v2f32_monotonic(
+; SM90PTX81-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) #[[ATTR0]] {
+; SM90PTX81-NEXT:  [[ENTRY:.*:]]
+; SM90PTX81-NEXT:    [[LANE_VAL:%.*]] = extractelement <2 x float> [[VAL]], i64 0
+; SM90PTX81-NEXT:    [[TMP0:%.*]] = atomicrmw fadd ptr [[ADDR]], float [[LANE_VAL]] monotonic, align 8
+; SM90PTX81-NEXT:    [[LANE_OLD:%.*]] = insertelement <2 x float> zeroinitializer, float [[TMP0]], i64 0
+; SM90PTX81-NEXT:    [[LANE_PTR:%.*]] = getelementptr inbounds <2 x float>, ptr [[ADDR]], i64 0, i64 1
+; SM90PTX81-NEXT:    [[LANE_VAL1:%.*]] = extractelement <2 x float> [[VAL]], i64 1
+; SM90PTX81-NEXT:    [[TMP1:%.*]] = atomicrmw fadd ptr [[LANE_PTR]], float [[LANE_VAL1]] monotonic, align 4
+; SM90PTX81-NEXT:    [[OLD:%.*]] = insertelement <2 x float> [[LANE_OLD]], float [[TMP1]], i64 1
+; SM90PTX81-NEXT:    ret <2 x float> [[OLD]]
+;
+; SM90-LABEL: define <2 x float> @fadd_v2f32_monotonic(
+; SM90-SAME: ptr [[ADDR:%.*]], <2 x float> [[VAL:%.*]]) #[[ATTR0]] {
+; SM90-NEXT:  [[ENTRY:.*:]]
+; SM90-NEXT:    [[LANE_VAL:%.*]] = extractelement <2 x float> [[VAL]], i64 0
+; SM90-NEXT:    [[TMP0:%.*]] = atomicrmw fadd ptr [[ADDR]], float [[LANE_VAL]] monotonic, align 8
+; SM90-NEXT:    [[LANE_OLD:%.*]] = insertelement <2 x float> zeroinitializer, float [[TMP0]], i64 0
+; SM90-NEXT:    [[LANE_PTR:%.*]] = getelementptr inbounds <2 x float>, ptr [[ADDR]], i64 0, i64 1
+; SM90-NEXT:    [[LANE_VAL1:%.*]] = extractelement <2 x float> [[VAL]], i64 1
+; SM90-NEXT:    [[TMP1:%.*]] = atomicrmw fadd ptr [[LANE_PTR]], float [[LANE_VAL1]] monotonic, align 4
+; SM90-NEXT:    [[LANE_OLD2:%.*]] = insertelement <2 x float> [[LANE_OLD]], float [[TMP1]], i64 1
+; SM90-NEXT:    ret <2 x float> [[LANE_OLD2]]
+;
+entry:
+  %old = atomicrmw elementwise fadd ptr %addr, <2 x float> %val monotonic, align 8
+  ret <2 x float> %old
+}
+
+; seq_cst: fence splitting applies when preserved.
+define <4 x float> @fadd_v4f32_seq_cst(ptr %addr, <4 x float> %val) {
+; SM90PTX83-LABEL: define <4 x float> @fadd_v4f32_seq_cst(
+; SM90PTX83-SAME: ptr [[ADDR:%.*]], <4 x float> [[VAL:%.*]]) #[[ATTR0]] {
+; SM90PTX83-NEXT:  [[ENTRY:.*:]]
+; SM90PTX83-NEXT:    [[LANE_VAL:%.*]] = extractelement <4 x float> [[VAL]], i64 0
+; SM90PTX83-NEXT:    fence seq_cst
+; SM90PTX83-NEXT:    [[TMP0:%.*]] = atomicrmw fadd ptr [[ADDR]], float [[LANE_VAL]] acquire, align 16
+; SM90PTX83-NEXT:    [[LANE_OLD:%.*]] = insertelement <4 x float> zeroinitializer, float [[TMP0]], i64 0
+; SM90PTX83-NEXT:    [[LANE_PTR:%.*]] = getelementptr inbounds <4 x float>, ptr [[ADDR]], i64 0, i64 1
+; SM90PTX83-NEXT:    [[LANE_VAL1:%.*]] = extractelement <4 x float> [[VAL]], i64 1
+; SM90PTX83-NEXT:    fence seq_cst
+; SM90PTX83-NEXT:    [[TMP1:%.*]] = atomicrmw fadd ptr [[LANE_PTR]], float [[LANE_VAL1]] acquire, align 4
+; SM90PTX83-NEXT:    [[LANE_OLD2:%.*]] = insertelement <4 x float> [[LANE_OLD]], float [[TMP1]], i64 1
+; SM90PTX83-NEXT:    [[LANE_PTR3:%.*]] = getelementptr inbounds <4 x float>, ptr [[ADDR]], i64 0, i64 2
+; SM90PTX83-NEXT:    [[LANE_VAL4:%.*]] = extractelement <4 x float> [[VAL]], i64 2
+; SM90PTX83-NEXT:    fence seq_cst
+; SM90PTX83-NEXT:    [[TMP2:%.*]] = atomicrmw fadd ptr [[LANE_PTR3]], float [[LANE_VAL4]] acquire, align 8
+; SM90PTX83-NEXT:    [[LANE_OLD5:%.*]] = insertelement <4 x float> [[LANE_OLD2]], float [[TMP2]], i64 2
+; SM90PTX83-NEXT:    [[LANE_PTR6:%.*]] = getelementptr inbounds <4 x float>, ptr [[ADDR]], i64 0, i64 3
+; SM90PTX83-NEXT:    [[LANE_VAL7:%.*]] = extractelement <4 x float> [[VAL]], i64 3
+; SM90PTX83-NEXT:    fence seq_cst
+; SM90PTX83-NEXT:    [[TMP3:%.*]] = atomicrmw fadd ptr [[LANE_PTR6]], float [[LANE_VAL7]] acquire, align 4
+; SM90PTX83-NEXT:    [[OLD:%.*]] = insertelement <4 x float> [[LANE_OLD5]], float [[TMP3]], i64 3
+; SM90PTX83-NEXT:    ret <4 x float> [[OLD]]
+;
+; SM90PTX81-LABEL: define <4 x float> @fadd_v4f32_seq_cst(
+; SM90PTX81-SAME: ptr [[ADDR:%.*]], <4 x float> [[VAL:%.*]]) #[[ATTR0]] {
+; SM90PTX81-NEXT:  [[ENTRY:.*:]]
+; SM90PTX81-NEXT:    [[LANE_VAL:%.*]] = extractelement <4 x float> [[VAL]], i64 0
+; SM90PTX81-NEXT:    fence seq_cst
+; SM90PTX81-NEXT:    [[TMP0:%.*]] = atomicrmw fadd ptr [[ADDR]], float [[LANE_VAL]] acquire, align 16
+; SM90PTX81-NEXT:    [[LANE_OLD:%.*]] = insertelement <4 x float> zeroinitializer, float [[TMP0]], i64 0
+; SM90PTX81-NEXT:    [[LANE_PTR:%.*]] = getelementptr inbounds <4 x float>, ptr [[ADDR]], i64 0, i64 1
+; SM90PTX81-NEXT:    [[LANE_VAL1:%.*]] = extractelement <4 x float> [[VAL]], i64 1
+; SM90PTX81-NEXT:    fence seq_cst
+; SM90PTX81-NEXT:    [[TMP1:%.*]] = atomicrmw fadd ptr [[LANE_PTR]], float [[LANE_VAL1]] acquire, align 4
+; SM90PTX81-NEXT:    [[LANE_OLD2:%.*]] = insertelement <4 x float> [[LANE_OLD]], float [[TMP1]], i64 1
+; SM90PTX81-NEXT:    [[LANE_PTR3:%.*]] = getelementptr inbounds <4 x float>, ptr [[ADDR]], i64 0, i64 2
+; SM90PTX81-NEXT:    [[LANE_VAL4:%.*]] = extractelement <4 x float> [[VAL]], i64 2
+; SM90PTX81-NEXT:    fence seq_cst
+; SM90PTX81-NEXT:    [[TMP2:%.*]] = atomicrmw fadd ptr [[LANE_PTR3]], float [[LANE_VAL4]] acquire, align 8
+; SM90PTX81-NEXT:    [[LANE_OLD5:%.*]] = insertelement <4 x float> [[LANE_OLD2]], float [[TMP2]], i64 2
+; SM90PTX81-NEXT:    [[LANE_PTR6:%.*]] = getelementptr inbounds <4 x float>, ptr [[ADDR]], i64 0, i64 3
+; SM90PTX81-NEXT:    [[LANE_VAL7:%.*]] = extractelement <4 x float> [[VAL]], i64 3
+; SM90PTX81-NEXT:    fence seq_cst
+; SM90PTX81-NEXT:    [[TMP3:%.*]] = atomicrmw fadd ptr [[LANE_PTR6]], float [[LANE_VAL7]] acquire, align 4
+; SM90PTX81-NEXT:    [[LANE_OLD8:%.*]] = insertelement <4 x float> [[LANE_OLD5]], float [[TMP3]], i64 3
+; SM90PTX81-NEXT:    ret <4 x float> [[LANE_OLD8]]
+;
+; SM90-LABEL: define <4 x float> @fadd_v4f32_seq_cst(
+; SM90-SAME: ptr [[ADDR:%.*]], <4 x float> [[VAL:%.*]]) #[[ATTR0]] {
+; SM90-NEXT:  [[ENTRY:.*:]]
+; SM90-NEXT:    [[LANE_VAL:%.*]] = extractelement <4 x float> [[VAL]], i64 0
+; SM90-NEXT:    fence seq_cst
+; SM90-NEXT:    [[TMP0:%.*]] = atomicrmw fadd ptr [[ADDR]], float [[LANE_VAL]] acquire, align 16
+; SM90-NEXT:    [[LANE_OLD:%.*]] = insertelement <4 x float> zeroinitializer, float [[TMP0]], i64 0
+; SM90-NEXT:    [[LANE_PTR:%.*]] = getelementptr inbounds <4 x float>, ptr [[ADDR]], i64 0, i64 1
+; SM90-NEXT:    [[LANE_VAL1:%.*]] = extractelement <4 x float> [[VAL]], i64 1
+; SM90-NEXT:    fence seq_cst
+; SM90-NEXT:    [[TMP1:%.*]] = atomicrmw fadd ptr [[LANE_PTR]], float [[LANE_VAL1]] acquire, align 4
+; SM90-NEXT:    [[LANE_OLD2:%.*]] = insertelement <4 x float> [[LANE_OLD]], float [[TMP1]], i64 1
+; SM90-NEXT:    [[LANE_PTR3:%.*]] = getelementptr inbounds <4 x float>, ptr [[ADDR]], i64 0, i64 2
+; SM90-NEXT:    [[LANE_VAL4:%.*]] = extractelement <4 x float> [[VAL]], i64 2
+; SM90-NEXT:    fence seq_cst
+; SM90-NEXT:    [[TMP2:%.*]] = atomicrmw fadd ptr [[LANE_PTR3]], float [[LANE_VAL4]] acquire, align 8
+; SM90-NEXT:    [[LANE_OLD5:%.*]] = insertelement <4 x float> [[LANE_OLD2]], float [[TMP2]], i64 2
+; SM90-NEXT:    [[LANE_PTR6:%.*]] = getelementptr inbounds <4 x float>, ptr [[ADDR]], i64 0, i64 3
+; SM90-NEXT:    [[LANE_VAL7:%.*]] = extractelement <4 x float> [[VAL]], i64 3
+; SM90-NEXT:    fence seq_cst
+; SM90-NEXT:    [[TMP3:%.*]] = atomicrmw fadd ptr [[LANE_PTR6]], float [[LANE_VAL7]] acquire, align 4
+; SM90-NEXT:    [[LANE_OLD8:%.*]] = insertelement <4 x float> [[LANE_OLD5]], float [[TMP3]], i64 3
+; SM90-NEXT:    ret <4 x float> [[LANE_OLD8]]
+;
+entry:
+  %old = atomicrmw elementwise fadd ptr %addr, <4 x float> %val seq_cst, align 16
+  ret <4 x float> %old
+}
+
+; Integer add is never a native vector atom — always expanded.
+define <4 x i32> @add_v4i32(ptr %addr, <4 x i32> %val) {
+; SM90PTX83-LABEL: define <4 x i32> @add_v4i32(
+; SM90PTX83-SAME: ptr [[ADDR:%.*]], <4 x i32> [[VAL:%.*]]) #[[ATTR0]] {
+; SM90PTX83-NEXT:  [[ENTRY:.*:]]
+; SM90PTX83-NEXT:    [[LANE_VAL:%.*]] = extractelement <4 x i32> [[VAL]], i64 0
+; SM90PTX83-NEXT:    [[TMP0:%.*]] = atomicrmw add ptr [[ADDR]], i32 [[LANE_VAL]] monotonic, align 16
+; SM90PTX83-NEXT:    [[LANE_OLD:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[TMP0]], i64 0
+; SM90PTX83-NEXT:    [[LANE_PTR:%.*]] = getelementptr inbounds <4 x i32>, ptr [[ADDR]], i64 0, i64 1
+; SM90PTX83-NEXT:    [[LANE_VAL1:%.*]] = extractelement <4 x i32> [[VAL]], i64 1
+; SM90PTX83-NEXT:    [[TMP1:%.*]] = atomicrmw add ptr [[LANE_PTR]], i32 [[LANE_VAL1]] monotonic, align 4
+; SM90PTX83-NEXT:    [[LANE_OLD2:%.*]] = insertelement <4 x i32> [[LANE_OLD]], i32 [[TMP1]], i64 1
+; SM90PTX83-NEXT:    [[LANE_PTR3:%.*]] = getelementptr inbounds <4 x i32>, ptr [[ADDR]], i64 0, i64 2
+; SM90PTX83-NEXT:    [[LANE_VAL4:%.*]] = extractelement <4 x i32> [[VAL]], i64 2
+; SM90PTX83-NEXT:    [[TMP2:%.*]] = atomicrmw add ptr [[LANE_PTR3]], i32 [[LANE_VAL4]] monotonic, align 8
+; SM90PTX83-NEXT:    [[LANE_OLD5:%.*]] = insertelement <4 x i32> [[LANE_OLD2]], i32 [[TMP2]], i64 2
+; SM90PTX83-NEXT:    [[LANE_PTR6:%.*]] = getelementptr inbounds <4 x i32>, ptr [[ADDR]], i64 0, i64 3
+; SM90PTX83-NEXT:    [[LANE_VAL7:%.*]] = extractelement <4 x i32> [[VAL]], i64 3
+; SM90PTX83-NEXT:    [[TMP3:%.*]] = atomicrmw add ptr [[LANE_PTR6]], i32 [[LANE_VAL7]] monotonic, align 4
+; SM90PTX83-NEXT:    [[LANE_OLD8:%.*]] = insertelement <4 x i32> [[LANE_OLD5]], i32 [[TMP3]], i64 3
+; SM90PTX83-NEXT:    ret <4 x i32> [[LANE_OLD8]]
+;
+; SM90PTX81-LABEL: define <4 x i32> @add_v4i32(
+; SM90PTX81-SAME: ptr [[ADDR:%.*]], <4 x i32> [[VAL:%.*]]) #[[ATTR0]] {
+; SM90PTX81-NEXT:  [[ENTRY:.*:]]
+; SM90PTX81-NEXT:    [[LANE_VAL:%.*]] = extractelement <4 x i32> [[VAL]], i64 0
+; SM90PTX81-NEXT:    [[TMP0:%.*]] = atomicrmw add ptr [[ADDR]], i32 [[LANE_VAL]] monotonic, align 16
+; SM90PTX81-NEXT:    [[LANE_OLD:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[TMP0]], i64 0
+; SM90PTX81-NEXT:    [[LANE_PTR:%.*]] = getelementptr inbounds <4 x i32>, ptr [[ADDR]], i64 0, i64 1
+; SM90PTX81-NEXT:    [[LANE_VAL1:%.*]] = extractelement <4 x i32> [[VAL]], i64 1
+; SM90PTX81-NEXT:    [[TMP1:%.*]] = atomicrmw add ptr [[LANE_PTR]], i32 [[LANE_VAL1]] monotonic, align 4
+; SM90PTX81-NEXT:    [[LANE_OLD2:%.*]] = insertelement <4 x i32> [[LANE_OLD]], i32 [[TMP1]], i64 1
+; SM90PTX81-NEXT:    [[LANE_PTR3:%.*]] = getelementptr inbounds <4 x i32>, ptr [[ADDR]], i64 0, i64 2
+; SM90PTX81-NEXT:    [[LANE_VAL4:%.*]] = extractelement <4 x i32> [[VAL]], i64 2
+; SM90PTX81-NEXT:    [[TMP2:%.*]] = atomicrmw add ptr [[LANE_PTR3]], i32 [[LANE_VAL4]] monotonic, align 8
+; SM90PTX81-NEXT:    [[LANE_OLD5:%.*]] = insertelement <4 x i32> [[LANE_OLD2]], i32 [[TMP2]], i64 2
+; SM90PTX81-NEXT:    [[LANE_PTR6:%.*]] = getelementptr inbounds <4 x i32>, ptr [[ADDR]], i64 0, i64 3
+; SM90PTX81-NEXT:    [[LANE_VAL7:%.*]] = extractelement <4 x i32> [[VAL]], i64 3
+; SM90PTX81-NEXT:    [[TMP3:%.*]] = atomicrmw add ptr [[LANE_PTR6]], i32 [[LANE_VAL7]] monotonic, align 4
+; SM90PTX81-NEXT:    [[LANE_OLD8:%.*]] = insertelement <4 x i32> [[LANE_OLD5]], i32 [[TMP3]], i64 3
+; SM90PTX81-NEXT:    ret <4 x i32> [[LANE_OLD8]]
+;
+; SM90-LABEL: define <4 x i32> @add_v4i32(
+; SM90-SAME: ptr [[ADDR:%.*]], <4 x i32> [[VAL:%.*]]) #[[ATTR0]] {
+; SM90-NEXT:  [[ENTRY:.*:]]
+; SM90-NEXT:    [[LANE_VAL:%.*]] = extractelement <4 x i32> [[VAL]], i64 0
+; SM90-NEXT:    [[TMP0:%.*]] = atomicrmw add ptr [[ADDR]], i32 [[LANE_VAL]] monotonic, align 16
+; SM90-NEXT:    [[LANE_OLD:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[TMP0]], i64 0
+; SM90-NEXT:    [[LANE_PTR:%.*]] = getelementptr inbounds <4 x i32>, ptr [[ADDR]], i64 0, i64 1
+; SM90-NEXT:    [[LANE_VAL1:%.*]] = extractelement <4 x i32> [[VAL]], i64 1
+; SM90-NEXT:    [[TMP1:%.*]] = atomicrmw add ptr [[LANE_PTR]], i32 [[LANE_VAL1]] monotonic, align 4
+; SM90-NEXT:    [[LANE_OLD2:%.*]] = insertelement <4 x i32> [[LANE_OLD]], i32 [[TMP1]], i64 1
+; SM90-NEXT:    [[LANE_PTR3:%.*]] = getelementptr inbounds <4 x i32>, ptr [[ADDR]], i64 0, i64 2
+; SM90-NEXT:    [[LANE_VAL4:%.*]] = extractelement <4 x i32> [[VAL]], i64 2
+; SM90-NEXT:    [[TMP2:%.*]] = atomicrmw add ptr [[LANE_PTR3]], i32 [[LANE_VAL4]] monotonic, align 8
+; SM90-NEXT:    [[LANE_OLD5:%.*]] = insertelement <4 x i32> [[LANE_OLD2]], i32 [[TMP2]], i64 2
+; SM90-NEXT:    [[LANE_PTR6:%.*]] = getelementptr inbounds <4 x i32>, ptr [[ADDR]], i64 0, i64 3
+; SM90-NEXT:    [[LANE_VAL7:%.*]] = extractelement <4 x i32> [[VAL]], i64 3
+; SM90-NEXT:    [[TMP3:%.*]] = atomicrmw add ptr [[LANE_PTR6]], i32 [[LANE_VAL7]] monotonic, align 4
+; SM90-NEXT:    [[LANE_OLD8:%.*]] = insertelement <4 x i32> [[LANE_OLD5]], i32 [[TMP3]], i64 3
+; SM90-NEXT:    ret <4 x i32> [[LANE_OLD8]]
+;
+entry:
+  %old = atomicrmw elementwise add ptr %addr, <4 x i32> %val monotonic, align 16
+  ret <4 x i32> %old
+}
diff --git a/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-rmw-elementwise.ll b/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-rmw-elementwise.ll
new file mode 100644
index 0000000000000..c9e2067b5dffe
--- /dev/null
+++ b/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-rmw-elementwise.ll
@@ -0,0 +1,26 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S %s -passes='require<libcall-lowering-info>,atomic-expand' -mtriple=x86_64-linux-gnu | FileCheck %s
+
+define <4 x i32> @elem_add(ptr %p, <4 x i32> %v) {
+; CHECK-LABEL: define <4 x i32> @elem_add(
+; CHECK-SAME: ptr [[P:%.*]], <4 x i32> [[V:%.*]]) {
+; CHECK-NEXT:    [[V0:%.*]] = extractelement <4 x i32> [[V]], i64 0
+; CHECK-NEXT:    [[O0:%.*]] = atomicrmw add ptr [[P]], i32 [[V0]] monotonic, align 16
+; CHECK-NEXT:    [[R0:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[O0]], i64 0
+; CHECK-NEXT:    [[P1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[P]], i64 0, i64 1
+; CHECK-NEXT:    [[V1:%.*]] = extractelement <4 x i32> [[V]], i64 1
+; CHECK-NEXT:    [[O1:%.*]] = atomicrmw add ptr [[P1]], i32 [[V1]] monotonic, align 4
+; CHECK-NEXT:    [[R1:%.*]] = insertelement <4 x i32> [[R0]], i32 [[O1]], i64 1
+; CHECK-NEXT:    [[P2:%.*]] = getelementptr inbounds <4 x i32>, ptr [[P]], i64 0, i64 2
+; CHECK-NEXT:    [[V2:%.*]] = extractelement <4 x i32> [[V]], i64 2
+; CHECK-NEXT:    [[O2:%.*]] = atomicrmw add ptr [[P2]], i32 [[V2]] monotonic, align 8
+; CHECK-NEXT:    [[R2:%.*]] = insertelement <4 x i32> [[R1]], i32 [[O2]], i64 2
+; CHECK-NEXT:    [[P3:%.*]] = getelementptr inbounds <4 x i32>, ptr [[P]], i64 0, i64 3
+; CHECK-NEXT:    [[V3:%.*]] = extractelement <4 x i32> [[V]], i64 3
+; CHECK-NEXT:    [[O3:%.*]] = atomicrmw add ptr [[P3]], i32 [[V3]] monotonic, align 4
+; CHECK-NEXT:    [[R3:%.*]] = insertelement <4 x i32> [[R2]], i32 [[O3]], i64 3
+; CHECK-NEXT:    ret <4 x i32> [[R3]]
+;
+  %old = atomicrmw elementwise add ptr %p, <4 x i32> %v monotonic
+  ret <4 x i32> %old
+}

>From e28b8390db08ddc4df2201550ee0619d1d414613 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 31 Mar 2026 05:28:32 +0000
Subject: [PATCH 02/19] change order

---
 llvm/include/llvm/IR/IRBuilder.h          | 4 ++--
 llvm/include/llvm/IR/Instructions.h       | 5 +++--
 llvm/lib/AsmParser/LLParser.cpp           | 3 ++-
 llvm/lib/Bitcode/Reader/BitcodeReader.cpp | 2 +-
 llvm/lib/IR/Instructions.cpp              | 6 +++---
 5 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h
index 9ebac9b8c79e9..d61fdf71ffc0a 100644
--- a/llvm/include/llvm/IR/IRBuilder.h
+++ b/llvm/include/llvm/IR/IRBuilder.h
@@ -1948,8 +1948,8 @@ class IRBuilderBase {
       Align = llvm::Align(DL.getTypeStoreSize(Val->getType()));
     }
 
-    return Insert(
-        new AtomicRMWInst(Op, Ptr, Val, *Align, Ordering, SSID, Elementwise));
+    return Insert(new AtomicRMWInst(Op, Ptr, Val, *Align, Ordering, SSID,
+                                    nullptr, Elementwise));
   }
 
   CallInst *CreateStructuredGEP(Type *BaseType, Value *PtrBase,
diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h
index 9267be115b0cd..5948e0ac551bf 100644
--- a/llvm/include/llvm/IR/Instructions.h
+++ b/llvm/include/llvm/IR/Instructions.h
@@ -809,8 +809,9 @@ class AtomicRMWInst : public Instruction {
 public:
   LLVM_ABI AtomicRMWInst(BinOp Operation, Value *Ptr, Value *Val,
                          Align Alignment, AtomicOrdering Ordering,
-                         SyncScope::ID SSID, bool Elementwise = false,
-                         InsertPosition InsertBefore = nullptr);
+                         SyncScope::ID SSID,
+                         InsertPosition InsertBefore = nullptr,
+                         bool Elementwise = false);
 
   // allocate space for exactly two operands
   void *operator new(size_t S) { return User::operator new(S, AllocMarker); }
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index 997eea293291d..0f96f0c9ecf50 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -9102,7 +9102,8 @@ int LLParser::parseAtomicRMW(Instruction *&Inst, PerFunctionState &PFS) {
           Val->getType()));
   AtomicRMWInst *RMWI = new AtomicRMWInst(Operation, Ptr, Val,
                                           Alignment.value_or(DefaultAlignment),
-                                          Ordering, SSID, IsElementwise);
+                                          Ordering, SSID, nullptr,
+                                          IsElementwise);
   RMWI->setVolatile(IsVolatile);
   Inst = RMWI;
   return AteExtraComma ? InstExtraComma : InstNormal;
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index 60c8c7812ce81..c8ae658c5551a 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -6739,7 +6739,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
             Align(TheModule->getDataLayout().getTypeStoreSize(Val->getType()));
 
       I = new AtomicRMWInst(Operation, Ptr, Val, *Alignment, Ordering, SSID,
-                            IsElementwise);
+                            nullptr, IsElementwise);
       ResTypeID = ValTypeID;
       cast<AtomicRMWInst>(I)->setVolatile(IsVol);
 
diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index f940893ab0296..514377d18e8f5 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -1458,8 +1458,8 @@ void AtomicRMWInst::Init(BinOp Operation, Value *Ptr, Value *Val,
 
 AtomicRMWInst::AtomicRMWInst(BinOp Operation, Value *Ptr, Value *Val,
                              Align Alignment, AtomicOrdering Ordering,
-                             SyncScope::ID SSID, bool Elementwise,
-                             InsertPosition InsertBefore)
+                             SyncScope::ID SSID, InsertPosition InsertBefore,
+                             bool Elementwise)
     : Instruction(Val->getType(), AtomicRMW, AllocMarker, InsertBefore) {
   Init(Operation, Ptr, Val, Alignment, Ordering, SSID, Elementwise);
 }
@@ -4401,7 +4401,7 @@ AtomicCmpXchgInst *AtomicCmpXchgInst::cloneImpl() const {
 AtomicRMWInst *AtomicRMWInst::cloneImpl() const {
   AtomicRMWInst *Result = new AtomicRMWInst(
       getOperation(), getOperand(0), getOperand(1), getAlign(), getOrdering(),
-      getSyncScopeID(), isElementwise());
+      getSyncScopeID(), nullptr, isElementwise());
   Result->setVolatile(isVolatile());
   return Result;
 }

>From c6f48f0255dd6bb2927fe211835fd1f9a25799a9 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 31 Mar 2026 05:30:19 +0000
Subject: [PATCH 03/19] format

---
 llvm/lib/AsmParser/LLParser.cpp | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index 0f96f0c9ecf50..f2ff8c3c5bb0d 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -9100,10 +9100,9 @@ int LLParser::parseAtomicRMW(Instruction *&Inst, PerFunctionState &PFS) {
   const Align DefaultAlignment(
       PFS.getFunction().getDataLayout().getTypeStoreSize(
           Val->getType()));
-  AtomicRMWInst *RMWI = new AtomicRMWInst(Operation, Ptr, Val,
-                                          Alignment.value_or(DefaultAlignment),
-                                          Ordering, SSID, nullptr,
-                                          IsElementwise);
+  AtomicRMWInst *RMWI = new AtomicRMWInst(
+      Operation, Ptr, Val, Alignment.value_or(DefaultAlignment), Ordering, SSID,
+      nullptr, IsElementwise);
   RMWI->setVolatile(IsVolatile);
   Inst = RMWI;
   return AteExtraComma ? InstExtraComma : InstNormal;

>From 38b269630d3669f1e0726d9fdf128d8cf8d6d21d Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 31 Mar 2026 21:35:00 +0000
Subject: [PATCH 04/19] rebase

---
 llvm/lib/CodeGen/AtomicExpandPass.cpp | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index 535997e23f2eb..057e43427f1e2 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -268,8 +268,6 @@ static bool atomicSizeSupported(const TargetLowering *TLI, Inst *I) {
          Size <= TLI->getMaxAtomicSizeInBitsSupported() / 8;
 }
 
-<<<<<<< HEAD
-=======
 /// Returns true if we can lower atomicrmw elementwise using normal atomicrmw.
 bool AtomicExpandImpl::canReuseWholeValueAtomicRMW(AtomicRMWInst *AI) {
   assert(AI->isElementwise() && "expected elementwise atomicrmw");
@@ -287,7 +285,6 @@ bool AtomicExpandImpl::canReuseWholeValueAtomicRMW(AtomicRMWInst *AI) {
   return CanReuse;
 }
 
->>>>>>> 0075944a40cd (IR)
 bool AtomicExpandImpl::tryInsertTrailingSeqCstFence(Instruction *AtomicI) {
   if (!TLI->shouldInsertTrailingSeqCstFenceForAtomicStore(AtomicI))
     return false;
@@ -372,8 +369,6 @@ bool AtomicExpandImpl::processAtomicInstr(Instruction *I) {
     }
 
     bool MadeChange = false;
-<<<<<<< HEAD
-=======
 
     if (RMWI->isElementwise()) {
       if (!TLI->shouldExpandAtomicRMWElementwiseInIR(RMWI))
@@ -389,7 +384,6 @@ bool AtomicExpandImpl::processAtomicInstr(Instruction *I) {
       }
     }
 
->>>>>>> 0075944a40cd (IR)
     if (TLI->shouldCastAtomicRMWIInIR(RMWI) ==
         TargetLoweringBase::AtomicExpansionKind::CastToInteger) {
       RMWI = convertAtomicXchgToIntegerType(RMWI);

>From ca14376b9801c0b233d3a98d0a07ab9dfd035564 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 31 Mar 2026 22:36:58 +0000
Subject: [PATCH 05/19] update tests

---
 ...alid-atomicrmw-elementwise-add-must-be-integer-type.ll | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 llvm/test/Assembler/invalid-atomicrmw-elementwise-add-must-be-integer-type.ll

diff --git a/llvm/test/Assembler/invalid-atomicrmw-elementwise-add-must-be-integer-type.ll b/llvm/test/Assembler/invalid-atomicrmw-elementwise-add-must-be-integer-type.ll
new file mode 100644
index 0000000000000..453781d126376
--- /dev/null
+++ b/llvm/test/Assembler/invalid-atomicrmw-elementwise-add-must-be-integer-type.ll
@@ -0,0 +1,8 @@
+; RUN: not llvm-as -disable-output %s -o /dev/null 2>&1 | FileCheck %s
+
+define <4 x float> @bad_add(ptr %p, <4 x float> %v) {
+  %old = atomicrmw elementwise add ptr %p, <4 x float> %v monotonic
+  ret <4 x float> %old
+}
+
+; CHECK: atomicrmw add operand must be an integer

>From 99ed5f22802f0fd26103122ac822a38826a8c2c5 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 31 Mar 2026 22:38:39 +0000
Subject: [PATCH 06/19] update tests

---
 ...micrmw-elementwise-fadd-must-be-fp-type.ll |  2 +-
 .../invalid-atomicrmw-elementwise-scalar.ll   |  2 +-
 llvm/test/Bitcode/compatibility.ll            | 10 +++
 .../CodeGen/NVPTX/atomicrmw-elementwise.ll    |  2 -
 llvm/unittests/IR/VerifierTest.cpp            | 75 +++++++++++++++++++
 5 files changed, 87 insertions(+), 4 deletions(-)

diff --git a/llvm/test/Assembler/invalid-atomicrmw-elementwise-fadd-must-be-fp-type.ll b/llvm/test/Assembler/invalid-atomicrmw-elementwise-fadd-must-be-fp-type.ll
index 65a24212c20a5..1e80cc9c0f95e 100644
--- a/llvm/test/Assembler/invalid-atomicrmw-elementwise-fadd-must-be-fp-type.ll
+++ b/llvm/test/Assembler/invalid-atomicrmw-elementwise-fadd-must-be-fp-type.ll
@@ -1,4 +1,4 @@
-; RUN: not llvm-as %s -o /dev/null 2>&1 | FileCheck %s
+; RUN: not llvm-as -disable-output %s -o /dev/null 2>&1 | FileCheck %s
 
 define <4 x i32> @bad_fadd(ptr %p, <4 x i32> %v) {
   %old = atomicrmw elementwise fadd ptr %p, <4 x i32> %v monotonic
diff --git a/llvm/test/Assembler/invalid-atomicrmw-elementwise-scalar.ll b/llvm/test/Assembler/invalid-atomicrmw-elementwise-scalar.ll
index 6037607a52eef..bf869a0e3fe0e 100644
--- a/llvm/test/Assembler/invalid-atomicrmw-elementwise-scalar.ll
+++ b/llvm/test/Assembler/invalid-atomicrmw-elementwise-scalar.ll
@@ -1,4 +1,4 @@
-; RUN: not llvm-as %s -o /dev/null 2>&1 | FileCheck %s
+; RUN: not llvm-as -disable-output %s -o /dev/null 2>&1 | FileCheck %s
 
 define i32 @bad_scalar(ptr %p, i32 %v) {
   %old = atomicrmw elementwise add ptr %p, i32 %v monotonic
diff --git a/llvm/test/Bitcode/compatibility.ll b/llvm/test/Bitcode/compatibility.ll
index 3c154633740ee..f19b00cc79770 100644
--- a/llvm/test/Bitcode/compatibility.ll
+++ b/llvm/test/Bitcode/compatibility.ll
@@ -1026,6 +1026,16 @@ define void @pointer_atomics(ptr %word) {
   ret void
 }
 
+define void @elementwise_atomics(ptr %word, <4 x i32> %ival, <4 x float> %fval) {
+; CHECK: %atomicrmw.add = atomicrmw elementwise add ptr %word, <4 x i32> %ival monotonic, align 16
+  %atomicrmw.add = atomicrmw elementwise add ptr %word, <4 x i32> %ival monotonic, align 16
+
+; CHECK: %atomicrmw.fadd = atomicrmw elementwise fadd ptr %word, <4 x float> %fval seq_cst, align 16
+  %atomicrmw.fadd = atomicrmw elementwise fadd ptr %word, <4 x float> %fval seq_cst, align 16
+
+  ret void
+}
+
 ;; Fast Math Flags
 define void @fastmathflags_unop(float %op1) {
   %f.nnan = fneg nnan float %op1
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise.ll
index 625c2634dd584..6639d33d3db58 100644
--- a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise.ll
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise.ll
@@ -2,8 +2,6 @@
 ; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx84 | FileCheck %s
 ; RUN: %if ptxas-sm_90 && ptxas-isa-8.4 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx84 | %ptxas-verify -arch=sm_90 %}
 
-target triple = "nvptx64-nvidia-cuda"
-
 define <2 x float> @fadd_v2f32_elementwise(ptr %addr, <2 x float> %val) {
 ; CHECK-LABEL: fadd_v2f32_elementwise(
 ; CHECK:       {
diff --git a/llvm/unittests/IR/VerifierTest.cpp b/llvm/unittests/IR/VerifierTest.cpp
index 28a792a373b84..0c88fb8b1f4a6 100644
--- a/llvm/unittests/IR/VerifierTest.cpp
+++ b/llvm/unittests/IR/VerifierTest.cpp
@@ -385,6 +385,81 @@ TEST(VerifierTest, AtomicRMW) {
       << Error;
 }
 
+TEST(VerifierTest, AtomicRMWElementwiseScalar) {
+  LLVMContext C;
+  Module M("M", C);
+  FunctionType *FTy = FunctionType::get(Type::getVoidTy(C), /*isVarArg=*/false);
+  Function *F = Function::Create(FTy, Function::ExternalLinkage, "foo", M);
+  BasicBlock *Entry = BasicBlock::Create(C, "entry", F);
+  Value *Ptr = PoisonValue::get(PointerType::get(C, 0));
+
+  Type *I32Ty = Type::getInt32Ty(C);
+  Constant *CI = ConstantInt::get(I32Ty, 0);
+
+  new AtomicRMWInst(AtomicRMWInst::Add, Ptr, CI, Align(4),
+                    AtomicOrdering::Monotonic, SyncScope::System, Entry,
+                    /*Elementwise=*/true);
+  ReturnInst::Create(C, Entry);
+
+  std::string Error;
+  raw_string_ostream ErrorOS(Error);
+  EXPECT_TRUE(verifyFunction(*F, &ErrorOS));
+  EXPECT_TRUE(StringRef(Error).starts_with(
+      "atomicrmw elementwise operand must have fixed vector type!"))
+      << Error;
+}
+
+TEST(VerifierTest, AtomicRMWElementwiseIntOpOnFPVector) {
+  LLVMContext C;
+  Module M("M", C);
+  FunctionType *FTy = FunctionType::get(Type::getVoidTy(C), /*isVarArg=*/false);
+  Function *F = Function::Create(FTy, Function::ExternalLinkage, "foo", M);
+  BasicBlock *Entry = BasicBlock::Create(C, "entry", F);
+  Value *Ptr = PoisonValue::get(PointerType::get(C, 0));
+
+  Type *FPTy = Type::getFloatTy(C);
+  Constant *CV = ConstantVector::getSplat(ElementCount::getFixed(4),
+                                          ConstantFP::getZero(FPTy));
+
+  new AtomicRMWInst(AtomicRMWInst::Add, Ptr, CV, Align(16),
+                    AtomicOrdering::Monotonic, SyncScope::System, Entry,
+                    /*Elementwise=*/true);
+  ReturnInst::Create(C, Entry);
+
+  std::string Error;
+  raw_string_ostream ErrorOS(Error);
+  EXPECT_TRUE(verifyFunction(*F, &ErrorOS));
+  EXPECT_TRUE(
+      StringRef(Error).starts_with("atomicrmw add operand must have integer"))
+      << Error;
+}
+
+TEST(VerifierTest, AtomicRMWElementwiseFPOpOnIntVector) {
+  LLVMContext C;
+  Module M("M", C);
+  FunctionType *FTy = FunctionType::get(Type::getVoidTy(C), /*isVarArg=*/false);
+  Function *F = Function::Create(FTy, Function::ExternalLinkage, "foo", M);
+  BasicBlock *Entry = BasicBlock::Create(C, "entry", F);
+  Value *Ptr = PoisonValue::get(PointerType::get(C, 0));
+
+  Type *I32Ty = Type::getInt32Ty(C);
+  Constant *CV = ConstantVector::getSplat(ElementCount::getFixed(4),
+                                          ConstantInt::get(I32Ty, 0));
+
+  new AtomicRMWInst(AtomicRMWInst::FAdd, Ptr, CV, Align(16),
+                    AtomicOrdering::Monotonic, SyncScope::System, Entry,
+                    /*Elementwise=*/true);
+  ReturnInst::Create(C, Entry);
+
+  std::string Error;
+  raw_string_ostream ErrorOS(Error);
+  EXPECT_TRUE(verifyFunction(*F, &ErrorOS));
+  EXPECT_TRUE(StringRef(Error).starts_with(
+      "atomicrmw fadd operand must have floating-point or "
+      "fixed vector of floating-point type!"))
+      << Error;
+}
+
 TEST(VerifierTest, GetElementPtrInst) {
   LLVMContext C;
   Module M("M", C);

>From b63c44728b791afa2e4a0efc483421a021fdfa6b Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 31 Mar 2026 23:07:58 +0000
Subject: [PATCH 07/19] update verifier

---
 llvm/lib/AsmParser/LLParser.cpp                | 16 ++++++++--------
 llvm/lib/IR/Verifier.cpp                       | 18 +++++++++---------
 ...rmw-elementwise-add-must-be-integer-type.ll |  2 +-
 ...omicrmw-elementwise-fadd-must-be-fp-type.ll |  2 +-
 llvm/unittests/IR/VerifierTest.cpp             |  4 ++--
 5 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index f2ff8c3c5bb0d..8b0e7432a1354 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -9068,25 +9068,25 @@ int LLParser::parseAtomicRMW(Instruction *&Inst, PerFunctionState &PFS) {
     ScalarTy = VecTy->getElementType();
   }
 
+  std::string OpName =
+      (Twine("atomicrmw ") + (IsElementwise ? "elementwise " : "") +
+       AtomicRMWInst::getOperationName(Operation))
+          .str();
   if (Operation == AtomicRMWInst::Xchg) {
     if (!ScalarTy->isIntegerTy() && !ScalarTy->isFloatingPointTy() &&
         !ScalarTy->isPointerTy()) {
       return error(
           ValLoc,
-          "atomicrmw " + AtomicRMWInst::getOperationName(Operation) +
-              " operand must be an integer, floating point, or pointer type");
+          OpName + " operand must be an integer, floating point, or pointer type");
     }
   } else if (IsFP) {
     if (!ScalarTy->isFPOrFPVectorTy()) {
-      return error(ValLoc, "atomicrmw " +
-                               AtomicRMWInst::getOperationName(Operation) +
-                               " operand must be a floating point type");
+      return error(ValLoc,
+                   OpName + " operand must be a floating point type");
     }
   } else {
     if (!ScalarTy->isIntegerTy()) {
-      return error(ValLoc, "atomicrmw " +
-                               AtomicRMWInst::getOperationName(Operation) +
-                               " operand must be an integer");
+      return error(ValLoc, OpName + " operand must be an integer");
     }
   }
 
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 791dfeb7c5a3b..35081d76b074e 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -4771,23 +4771,23 @@ void Verifier::visitAtomicRMWInst(AtomicRMWInst &RMWI) {
       ScalarTy = VecTy->getElementType();
   }
 
+  std::string OpName =
+      (Twine("atomicrmw ") + (RMWI.isElementwise() ? "elementwise " : "") +
+       AtomicRMWInst::getOperationName(Op))
+          .str();
   if (Op == AtomicRMWInst::Xchg) {
     Check(ScalarTy->isIntegerTy() || ScalarTy->isFloatingPointTy() ||
               ScalarTy->isPointerTy(),
-          "atomicrmw " + AtomicRMWInst::getOperationName(Op) +
-              " operand must have integer or floating point type!",
-          &RMWI, ElTy);
+          OpName + " operand must have integer or floating point type!", &RMWI,
+          ElTy);
   } else if (AtomicRMWInst::isFPOperation(Op)) {
     Check(ElTy->isFPOrFPVectorTy() && !isa<ScalableVectorType>(ElTy),
-          "atomicrmw " + AtomicRMWInst::getOperationName(Op) +
+          OpName +
               " operand must have floating-point or fixed vector of "
-              "floating-point "
-              "type!",
+              "floating-point type!",
           &RMWI, ElTy);
   } else {
-    Check(ScalarTy->isIntegerTy(),
-          "atomicrmw " + AtomicRMWInst::getOperationName(Op) +
-              " operand must have integer type!",
+    Check(ScalarTy->isIntegerTy(), OpName + " operand must have integer type!",
           &RMWI, ElTy);
   }
   checkAtomicMemAccessSize(RMWI.isElementwise() ? ScalarTy : ElTy, &RMWI);
diff --git a/llvm/test/Assembler/invalid-atomicrmw-elementwise-add-must-be-integer-type.ll b/llvm/test/Assembler/invalid-atomicrmw-elementwise-add-must-be-integer-type.ll
index 453781d126376..399a5721a17bb 100644
--- a/llvm/test/Assembler/invalid-atomicrmw-elementwise-add-must-be-integer-type.ll
+++ b/llvm/test/Assembler/invalid-atomicrmw-elementwise-add-must-be-integer-type.ll
@@ -5,4 +5,4 @@ define <4 x float> @bad_add(ptr %p, <4 x float> %v) {
   ret <4 x float> %old
 }
 
-; CHECK: atomicrmw add operand must be an integer
+; CHECK: atomicrmw elementwise add operand must be an integer
diff --git a/llvm/test/Assembler/invalid-atomicrmw-elementwise-fadd-must-be-fp-type.ll b/llvm/test/Assembler/invalid-atomicrmw-elementwise-fadd-must-be-fp-type.ll
index 1e80cc9c0f95e..588f2c7c0dc21 100644
--- a/llvm/test/Assembler/invalid-atomicrmw-elementwise-fadd-must-be-fp-type.ll
+++ b/llvm/test/Assembler/invalid-atomicrmw-elementwise-fadd-must-be-fp-type.ll
@@ -5,4 +5,4 @@ define <4 x i32> @bad_fadd(ptr %p, <4 x i32> %v) {
   ret <4 x i32> %old
 }
 
-; CHECK: atomicrmw fadd operand must be a floating point type
+; CHECK: atomicrmw elementwise fadd operand must be a floating point type
diff --git a/llvm/unittests/IR/VerifierTest.cpp b/llvm/unittests/IR/VerifierTest.cpp
index 0c88fb8b1f4a6..2c514e6eafa4f 100644
--- a/llvm/unittests/IR/VerifierTest.cpp
+++ b/llvm/unittests/IR/VerifierTest.cpp
@@ -430,7 +430,7 @@ TEST(VerifierTest, AtomicRMWElementwiseIntOpOnFPVector) {
   raw_string_ostream ErrorOS(Error);
   EXPECT_TRUE(verifyFunction(*F, &ErrorOS));
   EXPECT_TRUE(
-      StringRef(Error).starts_with("atomicrmw add operand must have integer"))
+      StringRef(Error).starts_with("atomicrmw elementwise add operand must have integer"))
       << Error;
 }
 
@@ -455,7 +455,7 @@ TEST(VerifierTest, AtomicRMWElementwiseFPOpOnIntVector) {
   raw_string_ostream ErrorOS(Error);
   EXPECT_TRUE(verifyFunction(*F, &ErrorOS));
   EXPECT_TRUE(StringRef(Error).starts_with(
-      "atomicrmw fadd operand must have floating-point or "
+      "atomicrmw elementwise fadd operand must have floating-point or "
       "fixed vector of floating-point type!"))
       << Error;
 }

>From 4ac619453bd62d21ee28e3e36b0ad2cfcec4f841 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 31 Mar 2026 23:08:14 +0000
Subject: [PATCH 08/19] format

---
 llvm/lib/AsmParser/LLParser.cpp    | 6 +++---
 llvm/lib/IR/Verifier.cpp           | 5 ++---
 llvm/unittests/IR/VerifierTest.cpp | 4 ++--
 3 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index 8b0e7432a1354..1615df2ee5d72 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -9077,12 +9077,12 @@ int LLParser::parseAtomicRMW(Instruction *&Inst, PerFunctionState &PFS) {
         !ScalarTy->isPointerTy()) {
       return error(
           ValLoc,
-          OpName + " operand must be an integer, floating point, or pointer type");
+          OpName +
+              " operand must be an integer, floating point, or pointer type");
     }
   } else if (IsFP) {
     if (!ScalarTy->isFPOrFPVectorTy()) {
-      return error(ValLoc,
-                   OpName + " operand must be a floating point type");
+      return error(ValLoc, OpName + " operand must be a floating point type");
     }
   } else {
     if (!ScalarTy->isIntegerTy()) {
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 35081d76b074e..5589d6e0ad967 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -4782,9 +4782,8 @@ void Verifier::visitAtomicRMWInst(AtomicRMWInst &RMWI) {
           ElTy);
   } else if (AtomicRMWInst::isFPOperation(Op)) {
     Check(ElTy->isFPOrFPVectorTy() && !isa<ScalableVectorType>(ElTy),
-          OpName +
-              " operand must have floating-point or fixed vector of "
-              "floating-point type!",
+          OpName + " operand must have floating-point or fixed vector of "
+                   "floating-point type!",
           &RMWI, ElTy);
   } else {
     Check(ScalarTy->isIntegerTy(), OpName + " operand must have integer type!",
diff --git a/llvm/unittests/IR/VerifierTest.cpp b/llvm/unittests/IR/VerifierTest.cpp
index 2c514e6eafa4f..dea345114c95d 100644
--- a/llvm/unittests/IR/VerifierTest.cpp
+++ b/llvm/unittests/IR/VerifierTest.cpp
@@ -429,8 +429,8 @@ TEST(VerifierTest, AtomicRMWElementwiseIntOpOnFPVector) {
   std::string Error;
   raw_string_ostream ErrorOS(Error);
   EXPECT_TRUE(verifyFunction(*F, &ErrorOS));
-  EXPECT_TRUE(
-      StringRef(Error).starts_with("atomicrmw elementwise add operand must have integer"))
+  EXPECT_TRUE(StringRef(Error).starts_with(
+      "atomicrmw elementwise add operand must have integer"))
       << Error;
 }
 

>From 6efdeed0d086b995da7533a837216c8cd7ad5564 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Wed, 1 Apr 2026 01:25:39 +0000
Subject: [PATCH 09/19] remove elementwise from error message

---
 llvm/lib/AsmParser/LLParser.cpp               | 14 +++++++-------
 llvm/lib/IR/Verifier.cpp                      | 19 ++++++++++---------
 ...mw-elementwise-add-must-be-integer-type.ll |  2 +-
 ...micrmw-elementwise-fadd-must-be-fp-type.ll |  2 +-
 llvm/unittests/IR/VerifierTest.cpp            |  4 ++--
 5 files changed, 21 insertions(+), 20 deletions(-)

diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index 1615df2ee5d72..f2ff8c3c5bb0d 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -9068,25 +9068,25 @@ int LLParser::parseAtomicRMW(Instruction *&Inst, PerFunctionState &PFS) {
     ScalarTy = VecTy->getElementType();
   }
 
-  std::string OpName =
-      (Twine("atomicrmw ") + (IsElementwise ? "elementwise " : "") +
-       AtomicRMWInst::getOperationName(Operation))
-          .str();
   if (Operation == AtomicRMWInst::Xchg) {
     if (!ScalarTy->isIntegerTy() && !ScalarTy->isFloatingPointTy() &&
         !ScalarTy->isPointerTy()) {
       return error(
           ValLoc,
-          OpName +
+          "atomicrmw " + AtomicRMWInst::getOperationName(Operation) +
               " operand must be an integer, floating point, or pointer type");
     }
   } else if (IsFP) {
     if (!ScalarTy->isFPOrFPVectorTy()) {
-      return error(ValLoc, OpName + " operand must be a floating point type");
+      return error(ValLoc, "atomicrmw " +
+                               AtomicRMWInst::getOperationName(Operation) +
+                               " operand must be a floating point type");
     }
   } else {
     if (!ScalarTy->isIntegerTy()) {
-      return error(ValLoc, OpName + " operand must be an integer");
+      return error(ValLoc, "atomicrmw " +
+                               AtomicRMWInst::getOperationName(Operation) +
+                               " operand must be an integer");
     }
   }
 
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 5589d6e0ad967..791dfeb7c5a3b 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -4771,22 +4771,23 @@ void Verifier::visitAtomicRMWInst(AtomicRMWInst &RMWI) {
       ScalarTy = VecTy->getElementType();
   }
 
-  std::string OpName =
-      (Twine("atomicrmw ") + (RMWI.isElementwise() ? "elementwise " : "") +
-       AtomicRMWInst::getOperationName(Op))
-          .str();
   if (Op == AtomicRMWInst::Xchg) {
     Check(ScalarTy->isIntegerTy() || ScalarTy->isFloatingPointTy() ||
               ScalarTy->isPointerTy(),
-          OpName + " operand must have integer or floating point type!", &RMWI,
-          ElTy);
+          "atomicrmw " + AtomicRMWInst::getOperationName(Op) +
+              " operand must have integer or floating point type!",
+          &RMWI, ElTy);
   } else if (AtomicRMWInst::isFPOperation(Op)) {
     Check(ElTy->isFPOrFPVectorTy() && !isa<ScalableVectorType>(ElTy),
-          OpName + " operand must have floating-point or fixed vector of "
-                   "floating-point type!",
+          "atomicrmw " + AtomicRMWInst::getOperationName(Op) +
+              " operand must have floating-point or fixed vector of "
+              "floating-point "
+              "type!",
           &RMWI, ElTy);
   } else {
-    Check(ScalarTy->isIntegerTy(), OpName + " operand must have integer type!",
+    Check(ScalarTy->isIntegerTy(),
+          "atomicrmw " + AtomicRMWInst::getOperationName(Op) +
+              " operand must have integer type!",
           &RMWI, ElTy);
   }
   checkAtomicMemAccessSize(RMWI.isElementwise() ? ScalarTy : ElTy, &RMWI);
diff --git a/llvm/test/Assembler/invalid-atomicrmw-elementwise-add-must-be-integer-type.ll b/llvm/test/Assembler/invalid-atomicrmw-elementwise-add-must-be-integer-type.ll
index 399a5721a17bb..453781d126376 100644
--- a/llvm/test/Assembler/invalid-atomicrmw-elementwise-add-must-be-integer-type.ll
+++ b/llvm/test/Assembler/invalid-atomicrmw-elementwise-add-must-be-integer-type.ll
@@ -5,4 +5,4 @@ define <4 x float> @bad_add(ptr %p, <4 x float> %v) {
   ret <4 x float> %old
 }
 
-; CHECK: atomicrmw elementwise add operand must be an integer
+; CHECK: atomicrmw add operand must be an integer
diff --git a/llvm/test/Assembler/invalid-atomicrmw-elementwise-fadd-must-be-fp-type.ll b/llvm/test/Assembler/invalid-atomicrmw-elementwise-fadd-must-be-fp-type.ll
index 588f2c7c0dc21..1e80cc9c0f95e 100644
--- a/llvm/test/Assembler/invalid-atomicrmw-elementwise-fadd-must-be-fp-type.ll
+++ b/llvm/test/Assembler/invalid-atomicrmw-elementwise-fadd-must-be-fp-type.ll
@@ -5,4 +5,4 @@ define <4 x i32> @bad_fadd(ptr %p, <4 x i32> %v) {
   ret <4 x i32> %old
 }
 
-; CHECK: atomicrmw elementwise fadd operand must be a floating point type
+; CHECK: atomicrmw fadd operand must be a floating point type
diff --git a/llvm/unittests/IR/VerifierTest.cpp b/llvm/unittests/IR/VerifierTest.cpp
index dea345114c95d..7dc2ef3932a91 100644
--- a/llvm/unittests/IR/VerifierTest.cpp
+++ b/llvm/unittests/IR/VerifierTest.cpp
@@ -430,7 +430,7 @@ TEST(VerifierTest, AtomicRMWElementwiseIntOpOnFPVector) {
   raw_string_ostream ErrorOS(Error);
   EXPECT_TRUE(verifyFunction(*F, &ErrorOS));
   EXPECT_TRUE(StringRef(Error).starts_with(
-      "atomicrmw elementwise add operand must have integer"))
+      "atomicrmw add operand must have integer"))
       << Error;
 }
 
@@ -455,7 +455,7 @@ TEST(VerifierTest, AtomicRMWElementwiseFPOpOnIntVector) {
   raw_string_ostream ErrorOS(Error);
   EXPECT_TRUE(verifyFunction(*F, &ErrorOS));
   EXPECT_TRUE(StringRef(Error).starts_with(
-      "atomicrmw elementwise fadd operand must have floating-point or "
+      "atomicrmw fadd operand must have floating-point or "
       "fixed vector of floating-point type!"))
       << Error;
 }

>From 5d0f2c3aa653036580b4ff6cee0556ee6252e70b Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Wed, 1 Apr 2026 01:26:03 +0000
Subject: [PATCH 10/19] format

---
 llvm/unittests/IR/VerifierTest.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/unittests/IR/VerifierTest.cpp b/llvm/unittests/IR/VerifierTest.cpp
index 7dc2ef3932a91..0c88fb8b1f4a6 100644
--- a/llvm/unittests/IR/VerifierTest.cpp
+++ b/llvm/unittests/IR/VerifierTest.cpp
@@ -429,8 +429,8 @@ TEST(VerifierTest, AtomicRMWElementwiseIntOpOnFPVector) {
   std::string Error;
   raw_string_ostream ErrorOS(Error);
   EXPECT_TRUE(verifyFunction(*F, &ErrorOS));
-  EXPECT_TRUE(StringRef(Error).starts_with(
-      "atomicrmw add operand must have integer"))
+  EXPECT_TRUE(
+      StringRef(Error).starts_with("atomicrmw add operand must have integer"))
       << Error;
 }
 

>From 7c130a303fdaeafbe608cd13abff6fdc77e2ad77 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Thu, 2 Apr 2026 17:34:29 +0000
Subject: [PATCH 11/19] fixing nits and moving
 shouldExpandAtomicRMWElementwiseInIR to shouldExpandAtomicRMWInIR

---
 llvm/include/llvm/CodeGen/TargetLowering.h    | 33 ++++++++-----------
 llvm/lib/CodeGen/AtomicExpandPass.cpp         | 28 ++++++++--------
 .../Target/AArch64/AArch64ISelLowering.cpp    |  2 ++
 llvm/lib/Target/AMDGPU/R600ISelLowering.cpp   |  2 ++
 llvm/lib/Target/AMDGPU/SIISelLowering.cpp     |  2 ++
 llvm/lib/Target/ARM/ARMISelLowering.cpp       |  2 ++
 .../LoongArch/LoongArchISelLowering.cpp       |  2 ++
 llvm/lib/Target/M68k/M68kISelLowering.cpp     |  2 ++
 llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp   |  2 ++
 llvm/lib/Target/PowerPC/PPCISelLowering.cpp   |  2 ++
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp   |  2 ++
 llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp   |  2 ++
 llvm/lib/Target/Sparc/SparcISelLowering.cpp   |  2 ++
 .../Target/SystemZ/SystemZISelLowering.cpp    |  2 ++
 llvm/lib/Target/VE/VEISelLowering.cpp         |  2 ++
 .../WebAssembly/WebAssemblyISelLowering.cpp   |  2 ++
 llvm/lib/Target/X86/X86ISelLowering.cpp       |  2 ++
 ...mw-elementwise-add-must-be-integer-type.ll |  2 +-
 ...micrmw-elementwise-fadd-must-be-fp-type.ll |  2 +-
 .../invalid-atomicrmw-elementwise-scalar.ll   |  2 +-
 20 files changed, 59 insertions(+), 38 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index b04c0aa10f9d4..8f8f0d1590301 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -269,6 +269,8 @@ class LLVM_ABI TargetLoweringBase {
                        // operations; used by X86.
     Expand,            // Generic expansion in terms of other atomic operations.
     CustomExpand,      // Custom target-specific expansion using TLI hooks.
+    Elementwise,       // Expand an elementwise atomicrmw by first trying to drop the elementwise modifier if
+                       // the target supports a corresponding whole-value atomicrmw lowering. Otherwise, scalarize to per-lane atomics.
 
     // Rewrite to a non-atomic form for use in a known non-preemptible
     // environment.
@@ -2475,33 +2477,24 @@ class LLVM_ABI TargetLoweringBase {
   }
 
   /// Returns how the IR-level AtomicExpand pass should expand the given
-  /// AtomicRMW, if at all. Default is to never expand scalar atomics and expand
-  /// FP atomics via CmpXChg.
+  /// AtomicRMW, if at all. Default is to never expand scalar atomics, expand
+  /// FP atomics via CmpXChg, and always expand elementwise atomics.
   ///
-  /// Precondition: \p RMW is not elementwise. Elementwise atomicrmw
-  /// instructions are routed through \c shouldExpandAtomicRMWElementwiseInIR.
+  /// For elementwise atomicrmw, returning \c Elementwise tells AtomicExpand to
+  /// first try to conservatively drop the elementwise modifier and reuse an
+  /// existing whole-value atomicrmw lowering. If that is not possible, it
+  /// scalarizes into per-lane scalar atomicrmw instructions that are each fed
+  /// back through the normal atomic expansion pipeline. Targets that support
+  /// native vector atomic instructions should return \c None to preserve the
+  /// elementwise atomicrmw for the backend.
   virtual AtomicExpansionKind
   shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const {
+    if (RMW->isElementwise())
+      return AtomicExpansionKind::Elementwise;
     return RMW->isFloatingPointOperation() ?
       AtomicExpansionKind::CmpXChg : AtomicExpansionKind::None;
   }
 
-  /// Returns whether the IR-level AtomicExpand pass should expand the given
-  /// elementwise AtomicRMW into per-lane scalar atomicrmw instructions.
-  ///
-  /// Returning \c true (the default) tells AtomicExpand to first try to
-  /// conservatively drop the elementwise modifier and reuse an existing
-  /// whole-value atomicrmw lowering. If that is not possible, it scalarizes
-  /// into per-lane scalar atomicrmw instructions that are each fed back
-  /// through the normal atomic expansion pipeline.
-  ///
-  /// Targets that support native vector atomic instructions should return
-  /// \c false to preserve the elementwise atomicrmw for the backend.
-  virtual bool
-  shouldExpandAtomicRMWElementwiseInIR(const AtomicRMWInst *RMW) const {
-    return true;
-  }
-
   /// Returns how the given atomic atomicrmw should be cast by the IR-level
   /// AtomicExpand pass.
   virtual AtomicExpansionKind
diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index 057e43427f1e2..8548edf7d3d2a 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -371,17 +371,18 @@ bool AtomicExpandImpl::processAtomicInstr(Instruction *I) {
     bool MadeChange = false;
 
     if (RMWI->isElementwise()) {
-      if (!TLI->shouldExpandAtomicRMWElementwiseInIR(RMWI))
+      auto ExpansionKind = TLI->shouldExpandAtomicRMWInIR(RMWI);
+      if (ExpansionKind == TargetLoweringBase::AtomicExpansionKind::None)
         return false;
-      if (canReuseWholeValueAtomicRMW(RMWI)) {
-        // Dropping the elementwise modifier strengthens the semantics, which is
-        // conservatively correct. Prefer the target's existing whole-value
-        // lowering over IR expansion.
-        RMWI->setElementwise(false);
-        MadeChange = true;
-      } else {
+      assert(ExpansionKind == TargetLoweringBase::AtomicExpansionKind::Elementwise && "shouldExpandAtomicRMWInIR should "
+       " return ExpansionKind::Elementwise or ExpansionKind::None on elementwise atomicrmw");
+      if (!canReuseWholeValueAtomicRMW(RMWI))
         return expandElementwiseAtomicRMW(RMWI);
-      }
+      // Dropping the elementwise modifier strengthens the semantics, which is
+      // conservatively correct. Prefer the target's existing whole-value
+      // lowering over IR expansion.
+      RMWI->setElementwise(false);
+      MadeChange = true;
     }
 
     if (TLI->shouldCastAtomicRMWIInIR(RMWI) ==
@@ -609,18 +610,15 @@ AtomicExpandImpl::convertAtomicXchgToIntegerType(AtomicRMWInst *RMWI) {
 bool AtomicExpandImpl::expandElementwiseAtomicRMW(AtomicRMWInst *AI) {
   auto *VecTy = cast<FixedVectorType>(AI->getType());
   Type *LaneTy = VecTy->getElementType();
-  LLVMContext &Ctx = AI->getContext();
   Value *Result = Constant::getNullValue(VecTy);
   const uint64_t LaneSize = DL->getTypeStoreSize(LaneTy).getFixedValue();
+  ReplacementIRBuilder Builder(AI, *DL);
 
   for (unsigned Lane = 0, NumLanes = VecTy->getNumElements(); Lane != NumLanes;
        ++Lane) {
-    ReplacementIRBuilder Builder(AI, *DL);
-    Value *Idx0 = ConstantInt::get(Type::getInt64Ty(Ctx), 0);
-    Value *Idx = ConstantInt::get(Type::getInt64Ty(Ctx), Lane);
-    Value *Indices[] = {Idx0, Idx};
+    Value *Idx = Builder.getInt64(Lane);
     Value *LanePtr = Builder.CreateInBoundsGEP(VecTy, AI->getPointerOperand(),
-                                               Indices, "lane.ptr");
+                                              {Builder.getInt64(0), Idx}, "lane.ptr");
     Value *LaneVal =
         Builder.CreateExtractElement(AI->getValOperand(), Idx, "lane.val");
     auto *LaneRMW = Builder.CreateAtomicRMW(
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 38db1ac4a2fb9..241bce14280ef 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -30811,6 +30811,8 @@ static bool rmwOpMayLowerToLibcall(const AArch64Subtarget &Subtarget,
 TargetLowering::AtomicExpansionKind
 AArch64TargetLowering::shouldExpandAtomicRMWInIR(
     const AtomicRMWInst *AI) const {
+  if (AI->isElementwise())
+    return AtomicExpansionKind::Elementwise;
   Type *Ty = AI->getType();
   unsigned Size = Ty->getPrimitiveSizeInBits();
   assert(Size <= 128 && "AtomicExpandPass should've handled larger sizes.");
diff --git a/llvm/lib/Target/AMDGPU/R600ISelLowering.cpp b/llvm/lib/Target/AMDGPU/R600ISelLowering.cpp
index 4ebe154166b3f..ccbf1f7f43018 100644
--- a/llvm/lib/Target/AMDGPU/R600ISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/R600ISelLowering.cpp
@@ -2183,6 +2183,8 @@ SDNode *R600TargetLowering::PostISelFolding(MachineSDNode *Node,
 
 TargetLowering::AtomicExpansionKind
 R600TargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const {
+  if (RMW->isElementwise())
+    return AtomicExpansionKind::Elementwise;
   switch (RMW->getOperation()) {
   case AtomicRMWInst::Nand:
   case AtomicRMWInst::FAdd:
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 0bc509c4a6b29..2ec9a676c13c5 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -19444,6 +19444,8 @@ getPrivateAtomicExpansionKind(const GCNSubtarget &STI) {
 
 TargetLowering::AtomicExpansionKind
 SITargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const {
+  if (RMW->isElementwise())
+    return AtomicExpansionKind::Elementwise;
   unsigned AS = RMW->getPointerAddressSpace();
   if (AS == AMDGPUAS::PRIVATE_ADDRESS)
     return getPrivateAtomicExpansionKind(*getSubtarget());
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index 87996a41c44b6..3bb22b7b140e5 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -21289,6 +21289,8 @@ ARMTargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
 // and up to 64 bits on the non-M profiles
 TargetLowering::AtomicExpansionKind
 ARMTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
+  if (AI->isElementwise())
+    return AtomicExpansionKind::Elementwise;
   if (AI->isFloatingPointOperation())
     return AtomicExpansionKind::CmpXChg;
 
diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
index 1f9ba2610a06a..39574374de285 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
@@ -9248,6 +9248,8 @@ void LoongArchTargetLowering::emitExpandAtomicRMW(AtomicRMWInst *AI) const {
 TargetLowering::AtomicExpansionKind
 LoongArchTargetLowering::shouldExpandAtomicRMWInIR(
     const AtomicRMWInst *AI) const {
+  if (AI->isElementwise())
+    return AtomicExpansionKind::Elementwise;
   // TODO: Add more AtomicRMWInst that needs to be extended.
 
   // Since floating-point operation requires a non-trivial set of data
diff --git a/llvm/lib/Target/M68k/M68kISelLowering.cpp b/llvm/lib/Target/M68k/M68kISelLowering.cpp
index 33a6bdc10a5b1..cb9321a3f6c88 100644
--- a/llvm/lib/Target/M68k/M68kISelLowering.cpp
+++ b/llvm/lib/Target/M68k/M68kISelLowering.cpp
@@ -192,6 +192,8 @@ M68kTargetLowering::M68kTargetLowering(const M68kTargetMachine &TM,
 
 TargetLoweringBase::AtomicExpansionKind
 M68kTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const {
+  if (RMW->isElementwise())
+    return TargetLoweringBase::AtomicExpansionKind::Elementwise;
   return Subtarget.atLeastM68020()
              ? TargetLoweringBase::AtomicExpansionKind::CmpXChg
              : TargetLoweringBase::AtomicExpansionKind::None;
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index af5991ebf2c8f..bd9f18f31bd8c 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -7417,6 +7417,8 @@ void NVPTXTargetLowering::ReplaceNodeResults(
 
 NVPTXTargetLowering::AtomicExpansionKind
 NVPTXTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
+  if (AI->isElementwise())
+    return AtomicExpansionKind::Elementwise;
   Type *Ty = AI->getValOperand()->getType();
 
   if (AI->isFloatingPointOperation()) {
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 753306a8d365b..3994233c77589 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -20564,6 +20564,8 @@ bool PPCTargetLowering::shouldInlineQuadwordAtomics() const {
 
 TargetLowering::AtomicExpansionKind
 PPCTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
+  if (AI->isElementwise())
+    return AtomicExpansionKind::Elementwise;
   unsigned Size = AI->getType()->getPrimitiveSizeInBits();
   if (shouldInlineQuadwordAtomics() && Size == 128)
     return AtomicExpansionKind::MaskedIntrinsic;
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 3a8d9270fdd10..2f20c8d7fdd58 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -25719,6 +25719,8 @@ Instruction *RISCVTargetLowering::emitTrailingFence(IRBuilderBase &Builder,
 
 TargetLowering::AtomicExpansionKind
 RISCVTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
+  if (AI->isElementwise())
+    return AtomicExpansionKind::Elementwise;
   // atomicrmw {fadd,fsub} must be expanded to use compare-exchange, as floating
   // point operations can't be used in an lr/sc sequence without breaking the
   // forward-progress guarantee.
diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
index 2fcb71939322a..359f51d17ecb4 100644
--- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
@@ -639,6 +639,8 @@ bool SPIRVTargetLowering::insertLogicalCopyOnResult(
 
 TargetLowering::AtomicExpansionKind
 SPIRVTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const {
+  if (RMW->isElementwise())
+    return AtomicExpansionKind::Elementwise;
   switch (RMW->getOperation()) {
   case AtomicRMWInst::FAdd:
   case AtomicRMWInst::FSub:
diff --git a/llvm/lib/Target/Sparc/SparcISelLowering.cpp b/llvm/lib/Target/Sparc/SparcISelLowering.cpp
index bfca95c9b1a38..326692995d853 100644
--- a/llvm/lib/Target/Sparc/SparcISelLowering.cpp
+++ b/llvm/lib/Target/Sparc/SparcISelLowering.cpp
@@ -1557,6 +1557,8 @@ SparcTargetLowering::LowerCall_64(TargetLowering::CallLoweringInfo &CLI,
 
 TargetLowering::AtomicExpansionKind
 SparcTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
+  if (AI->isElementwise())
+    return AtomicExpansionKind::Elementwise;
   if (AI->getOperation() == AtomicRMWInst::Xchg &&
       AI->getType()->getPrimitiveSizeInBits() == 32)
     return AtomicExpansionKind::None; // Uses xchg instruction
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index bd9a172f9d9d7..968b9bd493aa0 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -1278,6 +1278,8 @@ SystemZTargetLowering::shouldCastAtomicStoreInIR(StoreInst *SI) const {
 TargetLowering::AtomicExpansionKind
 SystemZTargetLowering::shouldExpandAtomicRMWInIR(
     const AtomicRMWInst *RMW) const {
+  if (RMW->isElementwise())
+    return AtomicExpansionKind::Elementwise;
   // Don't expand subword operations as they require special treatment.
   if (RMW->getType()->isIntegerTy(8) || RMW->getType()->isIntegerTy(16))
     return AtomicExpansionKind::None;
diff --git a/llvm/lib/Target/VE/VEISelLowering.cpp b/llvm/lib/Target/VE/VEISelLowering.cpp
index 141196c332074..7d0b051e8b8e5 100644
--- a/llvm/lib/Target/VE/VEISelLowering.cpp
+++ b/llvm/lib/Target/VE/VEISelLowering.cpp
@@ -1094,6 +1094,8 @@ SDValue VETargetLowering::lowerATOMIC_FENCE(SDValue Op,
 
 TargetLowering::AtomicExpansionKind
 VETargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
+  if (AI->isElementwise())
+    return AtomicExpansionKind::Elementwise;
   // We have TS1AM implementation for i8/i16/i32/i64, so use it.
   if (AI->getOperation() == AtomicRMWInst::Xchg) {
     return AtomicExpansionKind::None;
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
index 47de46a6f7070..21fc4b12fec98 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
@@ -450,6 +450,8 @@ MVT WebAssemblyTargetLowering::getPointerMemTy(const DataLayout &DL,
 TargetLowering::AtomicExpansionKind
 WebAssemblyTargetLowering::shouldExpandAtomicRMWInIR(
     const AtomicRMWInst *AI) const {
+  if (AI->isElementwise())
+    return AtomicExpansionKind::Elementwise;
   // We have wasm instructions for these
   switch (AI->getOperation()) {
   case AtomicRMWInst::Add:
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 32796c3e56781..da1865ab9743d 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -32702,6 +32702,8 @@ void X86TargetLowering::emitCmpArithAtomicRMWIntrinsic(
 
 TargetLowering::AtomicExpansionKind
 X86TargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
+  if (AI->isElementwise())
+    return AtomicExpansionKind::Elementwise;
   unsigned NativeWidth = Subtarget.is64Bit() ? 64 : 32;
   Type *MemType = AI->getType();
 
diff --git a/llvm/test/Assembler/invalid-atomicrmw-elementwise-add-must-be-integer-type.ll b/llvm/test/Assembler/invalid-atomicrmw-elementwise-add-must-be-integer-type.ll
index 453781d126376..56e7580c5633b 100644
--- a/llvm/test/Assembler/invalid-atomicrmw-elementwise-add-must-be-integer-type.ll
+++ b/llvm/test/Assembler/invalid-atomicrmw-elementwise-add-must-be-integer-type.ll
@@ -1,4 +1,4 @@
-; RUN: not llvm-as -disable-output %s -o /dev/null 2>&1 | FileCheck %s
+; RUN: not llvm-as -disable-output %s | FileCheck %s
 
 define <4 x float> @bad_add(ptr %p, <4 x float> %v) {
   %old = atomicrmw elementwise add ptr %p, <4 x float> %v monotonic
diff --git a/llvm/test/Assembler/invalid-atomicrmw-elementwise-fadd-must-be-fp-type.ll b/llvm/test/Assembler/invalid-atomicrmw-elementwise-fadd-must-be-fp-type.ll
index 1e80cc9c0f95e..ec9ea18ff44b1 100644
--- a/llvm/test/Assembler/invalid-atomicrmw-elementwise-fadd-must-be-fp-type.ll
+++ b/llvm/test/Assembler/invalid-atomicrmw-elementwise-fadd-must-be-fp-type.ll
@@ -1,4 +1,4 @@
-; RUN: not llvm-as -disable-output %s -o /dev/null 2>&1 | FileCheck %s
+; RUN: not llvm-as -disable-output %s 2>&1 | FileCheck %s
 
 define <4 x i32> @bad_fadd(ptr %p, <4 x i32> %v) {
   %old = atomicrmw elementwise fadd ptr %p, <4 x i32> %v monotonic
diff --git a/llvm/test/Assembler/invalid-atomicrmw-elementwise-scalar.ll b/llvm/test/Assembler/invalid-atomicrmw-elementwise-scalar.ll
index bf869a0e3fe0e..c566dfd5f9089 100644
--- a/llvm/test/Assembler/invalid-atomicrmw-elementwise-scalar.ll
+++ b/llvm/test/Assembler/invalid-atomicrmw-elementwise-scalar.ll
@@ -1,4 +1,4 @@
-; RUN: not llvm-as -disable-output %s -o /dev/null 2>&1 | FileCheck %s
+; RUN: not llvm-as -disable-output %s 2>&1 | FileCheck %s
 
 define i32 @bad_scalar(ptr %p, i32 %v) {
   %old = atomicrmw elementwise add ptr %p, i32 %v monotonic

>From e8671ea1531b7e056f84fa55413ea19c217aca70 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Thu, 2 Apr 2026 17:35:20 +0000
Subject: [PATCH 12/19] format

---
 llvm/include/llvm/CodeGen/TargetLowering.h |  6 ++++--
 llvm/lib/CodeGen/AtomicExpandPass.cpp      | 11 +++++++----
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 8f8f0d1590301..2867dc57ab16d 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -269,8 +269,10 @@ class LLVM_ABI TargetLoweringBase {
                        // operations; used by X86.
     Expand,            // Generic expansion in terms of other atomic operations.
     CustomExpand,      // Custom target-specific expansion using TLI hooks.
-    Elementwise,       // Expand an elementwise atomicrmw by first trying to drop the elementwise modifier if
-                       // the target supports a corresponding whole-value atomicrmw lowering. Otherwise, scalarize to per-lane atomics.
+    Elementwise, // Expand an elementwise atomicrmw by first trying to drop the
+                 // elementwise modifier if the target supports a corresponding
+                 // whole-value atomicrmw lowering. Otherwise, scalarize to
+                 // per-lane atomics.
 
     // Rewrite to a non-atomic form for use in a known non-preemptible
     // environment.
diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index 8548edf7d3d2a..78dd71d0e439b 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -374,8 +374,11 @@ bool AtomicExpandImpl::processAtomicInstr(Instruction *I) {
       auto ExpansionKind = TLI->shouldExpandAtomicRMWInIR(RMWI);
       if (ExpansionKind == TargetLoweringBase::AtomicExpansionKind::None)
         return false;
-      assert(ExpansionKind == TargetLoweringBase::AtomicExpansionKind::Elementwise && "shouldExpandAtomicRMWInIR should "
-       " return ExpansionKind::Elementwise or ExpansionKind::None on elementwise atomicrmw");
+      assert(ExpansionKind ==
+                 TargetLoweringBase::AtomicExpansionKind::Elementwise &&
+             "shouldExpandAtomicRMWInIR should "
+             " return ExpansionKind::Elementwise or ExpansionKind::None on "
+             "elementwise atomicrmw");
       if (!canReuseWholeValueAtomicRMW(RMWI))
         return expandElementwiseAtomicRMW(RMWI);
       // Dropping the elementwise modifier strengthens the semantics, which is
@@ -617,8 +620,8 @@ bool AtomicExpandImpl::expandElementwiseAtomicRMW(AtomicRMWInst *AI) {
   for (unsigned Lane = 0, NumLanes = VecTy->getNumElements(); Lane != NumLanes;
        ++Lane) {
     Value *Idx = Builder.getInt64(Lane);
-    Value *LanePtr = Builder.CreateInBoundsGEP(VecTy, AI->getPointerOperand(),
-                                              {Builder.getInt64(0), Idx}, "lane.ptr");
+    Value *LanePtr = Builder.CreateInBoundsGEP(
+        VecTy, AI->getPointerOperand(), {Builder.getInt64(0), Idx}, "lane.ptr");
     Value *LaneVal =
         Builder.CreateExtractElement(AI->getValOperand(), Idx, "lane.val");
     auto *LaneRMW = Builder.CreateAtomicRMW(

>From 706bdacdd3bfa88861257a04d25892d4d6e04be6 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Thu, 2 Apr 2026 18:18:25 +0000
Subject: [PATCH 13/19] 2 >&1

---
 .../invalid-atomicrmw-elementwise-add-must-be-integer-type.ll   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/test/Assembler/invalid-atomicrmw-elementwise-add-must-be-integer-type.ll b/llvm/test/Assembler/invalid-atomicrmw-elementwise-add-must-be-integer-type.ll
index 56e7580c5633b..7397439118462 100644
--- a/llvm/test/Assembler/invalid-atomicrmw-elementwise-add-must-be-integer-type.ll
+++ b/llvm/test/Assembler/invalid-atomicrmw-elementwise-add-must-be-integer-type.ll
@@ -1,4 +1,4 @@
-; RUN: not llvm-as -disable-output %s | FileCheck %s
+; RUN: not llvm-as -disable-output %s 2>&1 | FileCheck %s
 
 define <4 x float> @bad_add(ptr %p, <4 x float> %v) {
   %old = atomicrmw elementwise add ptr %p, <4 x float> %v monotonic

>From f23d7714e3b344aa3a157d587c0e380dae5f606b Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Thu, 2 Apr 2026 19:52:55 +0000
Subject: [PATCH 14/19] swap Elementwise, insertBefore

---
 llvm/include/llvm/IR/IRBuilder.h              |  2 +-
 llvm/include/llvm/IR/Instructions.h           |  5 ++---
 llvm/lib/AsmParser/LLParser.cpp               |  2 +-
 llvm/lib/Bitcode/Reader/BitcodeReader.cpp     |  2 +-
 llvm/lib/IR/Instructions.cpp                  |  6 +++---
 llvm/unittests/Analysis/AliasAnalysisTest.cpp |  2 +-
 llvm/unittests/IR/VerifierTest.cpp            | 14 +++++++-------
 7 files changed, 16 insertions(+), 17 deletions(-)

diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h
index d61fdf71ffc0a..0b58011de1167 100644
--- a/llvm/include/llvm/IR/IRBuilder.h
+++ b/llvm/include/llvm/IR/IRBuilder.h
@@ -1949,7 +1949,7 @@ class IRBuilderBase {
     }
 
     return Insert(new AtomicRMWInst(Op, Ptr, Val, *Align, Ordering, SSID,
-                                    nullptr, Elementwise));
+                                    Elementwise));
   }
 
   CallInst *CreateStructuredGEP(Type *BaseType, Value *PtrBase,
diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h
index 5948e0ac551bf..9267be115b0cd 100644
--- a/llvm/include/llvm/IR/Instructions.h
+++ b/llvm/include/llvm/IR/Instructions.h
@@ -809,9 +809,8 @@ class AtomicRMWInst : public Instruction {
 public:
   LLVM_ABI AtomicRMWInst(BinOp Operation, Value *Ptr, Value *Val,
                          Align Alignment, AtomicOrdering Ordering,
-                         SyncScope::ID SSID,
-                         InsertPosition InsertBefore = nullptr,
-                         bool Elementwise = false);
+                         SyncScope::ID SSID, bool Elementwise = false,
+                         InsertPosition InsertBefore = nullptr);
 
   // allocate space for exactly two operands
   void *operator new(size_t S) { return User::operator new(S, AllocMarker); }
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index f2ff8c3c5bb0d..9e708c22653c2 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -9102,7 +9102,7 @@ int LLParser::parseAtomicRMW(Instruction *&Inst, PerFunctionState &PFS) {
           Val->getType()));
   AtomicRMWInst *RMWI = new AtomicRMWInst(
       Operation, Ptr, Val, Alignment.value_or(DefaultAlignment), Ordering, SSID,
-      nullptr, IsElementwise);
+      IsElementwise);
   RMWI->setVolatile(IsVolatile);
   Inst = RMWI;
   return AteExtraComma ? InstExtraComma : InstNormal;
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index c8ae658c5551a..60c8c7812ce81 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -6739,7 +6739,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
             Align(TheModule->getDataLayout().getTypeStoreSize(Val->getType()));
 
       I = new AtomicRMWInst(Operation, Ptr, Val, *Alignment, Ordering, SSID,
-                            nullptr, IsElementwise);
+                            IsElementwise);
       ResTypeID = ValTypeID;
       cast<AtomicRMWInst>(I)->setVolatile(IsVol);
 
diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index 514377d18e8f5..f940893ab0296 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -1458,8 +1458,8 @@ void AtomicRMWInst::Init(BinOp Operation, Value *Ptr, Value *Val,
 
 AtomicRMWInst::AtomicRMWInst(BinOp Operation, Value *Ptr, Value *Val,
                              Align Alignment, AtomicOrdering Ordering,
-                             SyncScope::ID SSID, InsertPosition InsertBefore,
-                             bool Elementwise)
+                             SyncScope::ID SSID, bool Elementwise,
+                             InsertPosition InsertBefore)
     : Instruction(Val->getType(), AtomicRMW, AllocMarker, InsertBefore) {
   Init(Operation, Ptr, Val, Alignment, Ordering, SSID, Elementwise);
 }
@@ -4401,7 +4401,7 @@ AtomicCmpXchgInst *AtomicCmpXchgInst::cloneImpl() const {
 AtomicRMWInst *AtomicRMWInst::cloneImpl() const {
   AtomicRMWInst *Result = new AtomicRMWInst(
       getOperation(), getOperand(0), getOperand(1), getAlign(), getOrdering(),
-      getSyncScopeID(), nullptr, isElementwise());
+      getSyncScopeID(), isElementwise());
   Result->setVolatile(isVolatile());
   return Result;
 }
diff --git a/llvm/unittests/Analysis/AliasAnalysisTest.cpp b/llvm/unittests/Analysis/AliasAnalysisTest.cpp
index a28d318ab32c8..f80328b877736 100644
--- a/llvm/unittests/Analysis/AliasAnalysisTest.cpp
+++ b/llvm/unittests/Analysis/AliasAnalysisTest.cpp
@@ -185,7 +185,7 @@ TEST_F(AliasAnalysisTest, getModRefInfo) {
       SyncScope::System, BB);
   auto *AtomicRMW = new AtomicRMWInst(
       AtomicRMWInst::Xchg, Addr, ConstantInt::get(IntType, 1), Alignment,
-      AtomicOrdering::Monotonic, SyncScope::System, BB);
+      AtomicOrdering::Monotonic, SyncScope::System, /*Elementwise=*/false, BB);
 
   FunctionType *FooBarTy = FunctionType::get(Type::getVoidTy(C), {}, false);
   Function::Create(FooBarTy, Function::ExternalLinkage, "foo", &M);
diff --git a/llvm/unittests/IR/VerifierTest.cpp b/llvm/unittests/IR/VerifierTest.cpp
index 0c88fb8b1f4a6..4f458f5af7e77 100644
--- a/llvm/unittests/IR/VerifierTest.cpp
+++ b/llvm/unittests/IR/VerifierTest.cpp
@@ -373,7 +373,7 @@ TEST(VerifierTest, AtomicRMW) {
   Constant *CV = ConstantVector::getSplat(ElementCount::getScalable(2), CF);
   new AtomicRMWInst(AtomicRMWInst::FAdd, Ptr, CV, Align(8),
                     AtomicOrdering::SequentiallyConsistent, SyncScope::System,
-                    Entry);
+                    /*Elementwise=*/false, Entry);
   ReturnInst::Create(C, Entry);
 
   std::string Error;
@@ -397,8 +397,8 @@ TEST(VerifierTest, AtomicRMWElementwiseScalar) {
   Constant *CI = ConstantInt::get(I32Ty, 0);
 
   new AtomicRMWInst(AtomicRMWInst::Add, Ptr, CI, Align(4),
-                    AtomicOrdering::Monotonic, SyncScope::System, Entry,
-                    /*Elementwise=*/true);
+                    AtomicOrdering::Monotonic, SyncScope::System,
+                    /*Elementwise=*/true, Entry);
   ReturnInst::Create(C, Entry);
 
   std::string Error;
@@ -422,8 +422,8 @@ TEST(VerifierTest, AtomicRMWElementwiseIntOpOnFPVector) {
                                           ConstantFP::getZero(FPTy));
 
   new AtomicRMWInst(AtomicRMWInst::Add, Ptr, CV, Align(16),
-                    AtomicOrdering::Monotonic, SyncScope::System, Entry,
-                    /*Elementwise=*/true);
+                    AtomicOrdering::Monotonic, SyncScope::System,
+                    /*Elementwise=*/true, Entry);
   ReturnInst::Create(C, Entry);
 
   std::string Error;
@@ -447,8 +447,8 @@ TEST(VerifierTest, AtomicRMWElementwiseFPOpOnIntVector) {
                                           ConstantInt::get(I32Ty, 0));
 
   new AtomicRMWInst(AtomicRMWInst::FAdd, Ptr, CV, Align(16),
-                    AtomicOrdering::Monotonic, SyncScope::System, Entry,
-                    /*Elementwise=*/true);
+                    AtomicOrdering::Monotonic, SyncScope::System,
+                    /*Elementwise=*/true, Entry);
   ReturnInst::Create(C, Entry);
 
   std::string Error;

>From 91314fcc95dab62817ecce9507532845b2df188b Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Thu, 2 Apr 2026 19:53:07 +0000
Subject: [PATCH 15/19] format

---
 llvm/include/llvm/IR/IRBuilder.h | 4 ++--
 llvm/lib/AsmParser/LLParser.cpp  | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h
index 0b58011de1167..9ebac9b8c79e9 100644
--- a/llvm/include/llvm/IR/IRBuilder.h
+++ b/llvm/include/llvm/IR/IRBuilder.h
@@ -1948,8 +1948,8 @@ class IRBuilderBase {
       Align = llvm::Align(DL.getTypeStoreSize(Val->getType()));
     }
 
-    return Insert(new AtomicRMWInst(Op, Ptr, Val, *Align, Ordering, SSID,
-                                    Elementwise));
+    return Insert(
+        new AtomicRMWInst(Op, Ptr, Val, *Align, Ordering, SSID, Elementwise));
   }
 
   CallInst *CreateStructuredGEP(Type *BaseType, Value *PtrBase,
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index 9e708c22653c2..997eea293291d 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -9100,9 +9100,9 @@ int LLParser::parseAtomicRMW(Instruction *&Inst, PerFunctionState &PFS) {
   const Align DefaultAlignment(
       PFS.getFunction().getDataLayout().getTypeStoreSize(
           Val->getType()));
-  AtomicRMWInst *RMWI = new AtomicRMWInst(
-      Operation, Ptr, Val, Alignment.value_or(DefaultAlignment), Ordering, SSID,
-      IsElementwise);
+  AtomicRMWInst *RMWI = new AtomicRMWInst(Operation, Ptr, Val,
+                                          Alignment.value_or(DefaultAlignment),
+                                          Ordering, SSID, IsElementwise);
   RMWI->setVolatile(IsVolatile);
   Inst = RMWI;
   return AteExtraComma ? InstExtraComma : InstNormal;

>From 460c1109ad86f2929d453db28bad87376afa6df8 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Mon, 6 Apr 2026 22:28:22 +0000
Subject: [PATCH 16/19] move elementwise to subclass data

---
 llvm/include/llvm/IR/Instructions.h | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h
index 9267be115b0cd..19cecbb90791d 100644
--- a/llvm/include/llvm/IR/Instructions.h
+++ b/llvm/include/llvm/IR/Instructions.h
@@ -821,9 +821,12 @@ class AtomicRMWInst : public Instruction {
       AtomicOrderingBitfieldElementT<VolatileField::NextBit>;
   using OperationField = BinOpBitfieldElement<AtomicOrderingField::NextBit>;
   using AlignmentField = AlignmentBitfieldElementT<OperationField::NextBit>;
-  static_assert(Bitfield::areContiguous<VolatileField, AtomicOrderingField,
-                                        OperationField, AlignmentField>(),
-                "Bitfields must be contiguous");
+  using ElementwiseField = BoolBitfieldElementT<AlignmentField::NextBit>;
+  static_assert(
+      Bitfield::areContiguous<VolatileField, AtomicOrderingField,
+                              OperationField, AlignmentField,
+                              ElementwiseField>(),
+      "Bitfields must be contiguous");
 
   BinOp getOperation() const { return getSubclassData<OperationField>(); }
 
@@ -868,10 +871,10 @@ class AtomicRMWInst : public Instruction {
   void setVolatile(bool V) { setSubclassData<VolatileField>(V); }
 
   /// Return true if this RMW has elementwise vector semantics.
-  bool isElementwise() const { return Elementwise; }
+  bool isElementwise() const { return getSubclassData<ElementwiseField>(); }
 
   /// Specify whether this RMW has elementwise vector semantics.
-  void setElementwise(bool V) { Elementwise = V; }
+  void setElementwise(bool V) { setSubclassData<ElementwiseField>(V); }
 
   /// Transparently provide more efficient getOperand methods.
   DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
@@ -939,9 +942,6 @@ class AtomicRMWInst : public Instruction {
   /// room in SubClassData for everything, so synchronization scope ID gets its
   /// own field.
   SyncScope::ID SSID;
-
-  /// Whether this instruction uses per-lane vector atomic semantics.
-  bool Elementwise = false;
 };
 
 template <>

>From 741c4a0839043ba8733625caa24689adc94d0bcb Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 7 Apr 2026 00:19:15 +0000
Subject: [PATCH 17/19] format

---
 llvm/include/llvm/IR/Instructions.h | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h
index 19cecbb90791d..15fa2488e714b 100644
--- a/llvm/include/llvm/IR/Instructions.h
+++ b/llvm/include/llvm/IR/Instructions.h
@@ -822,11 +822,10 @@ class AtomicRMWInst : public Instruction {
   using OperationField = BinOpBitfieldElement<AtomicOrderingField::NextBit>;
   using AlignmentField = AlignmentBitfieldElementT<OperationField::NextBit>;
   using ElementwiseField = BoolBitfieldElementT<AlignmentField::NextBit>;
-  static_assert(
-      Bitfield::areContiguous<VolatileField, AtomicOrderingField,
-                              OperationField, AlignmentField,
-                              ElementwiseField>(),
-      "Bitfields must be contiguous");
+  static_assert(Bitfield::areContiguous<VolatileField, AtomicOrderingField,
+                                        OperationField, AlignmentField,
+                                        ElementwiseField>(),
+                "Bitfields must be contiguous");
 
   BinOp getOperation() const { return getSubclassData<OperationField>(); }
 

>From daa6863a69e7227c5d8b6104ce04609550ad997d Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Wed, 8 Apr 2026 15:45:10 +0000
Subject: [PATCH 18/19] canonical index type

---
 llvm/lib/CodeGen/AtomicExpandPass.cpp | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index 78dd71d0e439b..9412c04665b51 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -616,12 +616,15 @@ bool AtomicExpandImpl::expandElementwiseAtomicRMW(AtomicRMWInst *AI) {
   Value *Result = Constant::getNullValue(VecTy);
   const uint64_t LaneSize = DL->getTypeStoreSize(LaneTy).getFixedValue();
   ReplacementIRBuilder Builder(AI, *DL);
+  IntegerType *IdxTy =
+      DL->getIndexType(AI->getContext(), AI->getPointerAddressSpace());
 
   for (unsigned Lane = 0, NumLanes = VecTy->getNumElements(); Lane != NumLanes;
        ++Lane) {
-    Value *Idx = Builder.getInt64(Lane);
+    Value *Idx = ConstantInt::get(IdxTy, Lane);
     Value *LanePtr = Builder.CreateInBoundsGEP(
-        VecTy, AI->getPointerOperand(), {Builder.getInt64(0), Idx}, "lane.ptr");
+        VecTy, AI->getPointerOperand(),
+        {ConstantInt::get(IdxTy, 0), Idx}, "lane.ptr");
     Value *LaneVal =
         Builder.CreateExtractElement(AI->getValOperand(), Idx, "lane.val");
     auto *LaneRMW = Builder.CreateAtomicRMW(

>From ff2b3f51687287c9cc10348b6ec5a4ee5368c637 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Wed, 8 Apr 2026 15:55:04 +0000
Subject: [PATCH 19/19] format

---
 llvm/lib/CodeGen/AtomicExpandPass.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index 9412c04665b51..ac66517130f88 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -623,8 +623,8 @@ bool AtomicExpandImpl::expandElementwiseAtomicRMW(AtomicRMWInst *AI) {
        ++Lane) {
     Value *Idx = ConstantInt::get(IdxTy, Lane);
     Value *LanePtr = Builder.CreateInBoundsGEP(
-        VecTy, AI->getPointerOperand(),
-        {ConstantInt::get(IdxTy, 0), Idx}, "lane.ptr");
+        VecTy, AI->getPointerOperand(), {ConstantInt::get(IdxTy, 0), Idx},
+        "lane.ptr");
     Value *LaneVal =
         Builder.CreateExtractElement(AI->getValOperand(), Idx, "lane.val");
     auto *LaneRMW = Builder.CreateAtomicRMW(



More information about the llvm-commits mailing list