[llvm-branch-commits] [llvm] 3b484a3 - port
Tobias Hieta via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Sat Jul 25 08:39:55 PDT 2026
Author: Yonah Goldberg
Date: 2026-07-25T17:39:46+02:00
New Revision: 3b484a37805b99e0b617b606aea1c3d45cf18765
URL: https://github.com/llvm/llvm-project/commit/3b484a37805b99e0b617b606aea1c3d45cf18765
DIFF: https://github.com/llvm/llvm-project/commit/3b484a37805b99e0b617b606aea1c3d45cf18765.diff
LOG: port
Added:
Modified:
llvm/docs/LangRef.md
llvm/lib/AsmParser/LLParser.cpp
llvm/lib/IR/Verifier.cpp
llvm/test/Assembler/atomic.ll
llvm/test/Assembler/invalid-atomicrmw-elementwise.ll
llvm/test/Bitcode/atomicrmw-elementwise.ll
llvm/test/Bitcode/compatibility.ll
llvm/unittests/IR/VerifierTest.cpp
Removed:
################################################################################
diff --git a/llvm/docs/LangRef.md b/llvm/docs/LangRef.md
index f555ca543ffdc..61d23eeee39cc 100644
--- a/llvm/docs/LangRef.md
+++ b/llvm/docs/LangRef.md
@@ -4177,6 +4177,29 @@ Otherwise, an atomic operation that is not marked
monotonic modification order with other operations that are not marked
`syncscope("singlethread")` or `syncscope("<target-scope>")`.
+(elementwise-atomics)=
+
+### Elementwise Atomic Operations
+
+The {ref}`atomicrmw <i_atomicrmw>` instruction may be marked `elementwise`.
+The access type must then be a fixed vector type whose total bit width is a
+power of two and whose element type is supported by the corresponding scalar
+atomic instruction. The {ref}`ordering <ordering>` of an `elementwise`
+instruction may not be `seq_cst`.
+
+An `elementwise` atomic instruction behaves as if it were expanded into one
+scalar version of that instruction for each vector element. Each resulting
+scalar operation has the same {ref}`ordering <ordering>` and `syncscope` as the
+original instruction. Each scalar operation occupies the original instruction's
+position in program order relative to other operations, but the scalar
+operations are not related in program order with respect to one another.
+Synchronizing with one scalar operation therefore does not, by itself,
+establish a happens-before relationship with any other scalar operation from
+the same `elementwise` instruction.
+
+Without `elementwise`, vector `atomicrmw` instructions are performed atomically
+over the entire vector operation.
+
(floatenv)=
### Floating-Point Environment
@@ -12129,9 +12152,8 @@ isn't specified.
An `atomicrmw` instruction can also take an optional
"{ref}`syncscope <syncscope>`" argument.
-If the `elementwise` modifier is present, the instruction has per-element vector
-atomic semantics. It behaves as if it were expanded into one scalar `atomicrmw` per element, that are not ordered with respect to each other.
-Without `elementwise`, vector `atomicrmw` keeps whole-value atomic semantics.
+If `atomicrmw` is marked `elementwise`, the instruction has
+{ref}`elementwise atomic semantics <elementwise-atomics>`.
##### Semantics:
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index 38d10587b104e..618abb1bd423a 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -9204,6 +9204,8 @@ int LLParser::parseAtomicRMW(Instruction *&Inst, PerFunctionState &PFS) {
if (Ordering == AtomicOrdering::Unordered)
return tokError("atomicrmw cannot be unordered");
+ if (IsElementwise && Ordering == AtomicOrdering::SequentiallyConsistent)
+ return tokError("atomicrmw elementwise cannot be sequentially consistent");
if (!Ptr->getType()->isPointerTy())
return error(PtrLoc, "atomicrmw operand must be a pointer");
if (Val->getType()->isScalableTy())
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index fa0fe7a2e6092..9a455fbc9256a 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -4706,6 +4706,8 @@ void Verifier::visitAtomicRMWInst(AtomicRMWInst &RMWI) {
Type *ElTy = RMWI.getOperand(1)->getType();
Type *ScalarTy = ElTy;
if (RMWI.isElementwise()) {
+ Check(RMWI.getOrdering() != AtomicOrdering::SequentiallyConsistent,
+ "atomicrmw elementwise cannot be sequentially consistent.", &RMWI);
auto *VecTy = dyn_cast<FixedVectorType>(ElTy);
Check(VecTy, "atomicrmw elementwise operand must have fixed vector type!",
&RMWI, ElTy);
diff --git a/llvm/test/Assembler/atomic.ll b/llvm/test/Assembler/atomic.ll
index 609cd33f61b88..fab483d1cfd27 100644
--- a/llvm/test/Assembler/atomic.ll
+++ b/llvm/test/Assembler/atomic.ll
@@ -154,8 +154,8 @@ define void @fp_vector_atomicrmw(ptr %x, <2 x half> %val) {
; CHECK: %atomic.elem.fadd = atomicrmw elementwise fadd ptr %x, <2 x half> %val monotonic
%atomic.elem.fadd = atomicrmw elementwise fadd ptr %x, <2 x half> %val monotonic
- ; CHECK: %atomic.elem.fadd.vol = atomicrmw volatile elementwise fadd ptr %x, <2 x half> %val seq_cst
- %atomic.elem.fadd.vol = atomicrmw volatile elementwise fadd ptr %x, <2 x half> %val seq_cst
+ ; CHECK: %atomic.elem.fadd.vol = atomicrmw volatile elementwise fadd ptr %x, <2 x half> %val acq_rel
+ %atomic.elem.fadd.vol = atomicrmw volatile elementwise fadd ptr %x, <2 x half> %val acq_rel
ret void
}
diff --git a/llvm/test/Assembler/invalid-atomicrmw-elementwise.ll b/llvm/test/Assembler/invalid-atomicrmw-elementwise.ll
index 2900779420b0c..3c134cbb3b8c2 100644
--- a/llvm/test/Assembler/invalid-atomicrmw-elementwise.ll
+++ b/llvm/test/Assembler/invalid-atomicrmw-elementwise.ll
@@ -3,6 +3,7 @@
; RUN: not llvm-as -disable-output %t/odd-sized.ll 2>&1 | FileCheck %t/odd-sized.ll
; RUN: not llvm-as -disable-output %t/add-must-be-integer.ll 2>&1 | FileCheck %t/add-must-be-integer.ll
; RUN: not llvm-as -disable-output %t/fadd-must-be-fp.ll 2>&1 | FileCheck %t/fadd-must-be-fp.ll
+; RUN: not llvm-as -disable-output %t/seq-cst.ll 2>&1 | FileCheck %t/seq-cst.ll
;--- scalar.ll
; CHECK: atomicrmw elementwise operand must be a fixed vector type
@@ -31,3 +32,10 @@ define <4 x i32> @bad_fadd(ptr %p, <4 x i32> %v) {
%old = atomicrmw elementwise fadd ptr %p, <4 x i32> %v monotonic
ret <4 x i32> %old
}
+
+;--- seq-cst.ll
+; CHECK: atomicrmw elementwise cannot be sequentially consistent
+define <4 x i32> @bad_seq_cst(ptr %p, <4 x i32> %v) {
+ %old = atomicrmw elementwise add ptr %p, <4 x i32> %v seq_cst
+ ret <4 x i32> %old
+}
diff --git a/llvm/test/Bitcode/atomicrmw-elementwise.ll b/llvm/test/Bitcode/atomicrmw-elementwise.ll
index db9c48a80047e..c150a7bf14470 100644
--- a/llvm/test/Bitcode/atomicrmw-elementwise.ll
+++ b/llvm/test/Bitcode/atomicrmw-elementwise.ll
@@ -10,7 +10,7 @@ define <4 x i32> @elem_add(ptr %p, <4 x i32> %v) {
define <4 x float> @elem_fadd(ptr %p, <4 x float> %v) {
; CHECK-LABEL: @elem_fadd(
-; CHECK: %old = atomicrmw elementwise fadd ptr %p, <4 x float> %v seq_cst, align 16
- %old = atomicrmw elementwise fadd ptr %p, <4 x float> %v seq_cst
+; CHECK: %old = atomicrmw elementwise fadd ptr %p, <4 x float> %v acq_rel, align 16
+ %old = atomicrmw elementwise fadd ptr %p, <4 x float> %v acq_rel
ret <4 x float> %old
}
diff --git a/llvm/test/Bitcode/compatibility.ll b/llvm/test/Bitcode/compatibility.ll
index 0ac6da0c9ed29..f2e73395617de 100644
--- a/llvm/test/Bitcode/compatibility.ll
+++ b/llvm/test/Bitcode/compatibility.ll
@@ -1033,8 +1033,8 @@ define void @elementwise_atomics(ptr %word, <4 x i32> %ival, <4 x float> %fval)
; CHECK: %atomicrmw.add = atomicrmw elementwise add ptr %word, <4 x i32> %ival monotonic, align 16
%atomicrmw.add = atomicrmw elementwise add ptr %word, <4 x i32> %ival monotonic, align 16
-; CHECK: %atomicrmw.fadd = atomicrmw elementwise fadd ptr %word, <4 x float> %fval seq_cst, align 16
- %atomicrmw.fadd = atomicrmw elementwise fadd ptr %word, <4 x float> %fval seq_cst, align 16
+; CHECK: %atomicrmw.fadd = atomicrmw elementwise fadd ptr %word, <4 x float> %fval acq_rel, align 16
+ %atomicrmw.fadd = atomicrmw elementwise fadd ptr %word, <4 x float> %fval acq_rel, align 16
ret void
}
diff --git a/llvm/unittests/IR/VerifierTest.cpp b/llvm/unittests/IR/VerifierTest.cpp
index d680dd250d66e..c74390f596093 100644
--- a/llvm/unittests/IR/VerifierTest.cpp
+++ b/llvm/unittests/IR/VerifierTest.cpp
@@ -429,6 +429,31 @@ TEST(VerifierTest, AtomicRMWElementwiseScalar) {
<< Error;
}
+TEST(VerifierTest, AtomicRMWElementwiseSequentiallyConsistent) {
+ LLVMContext C;
+ Module M("M", C);
+ FunctionType *FTy = FunctionType::get(Type::getVoidTy(C), /*isVarArg=*/false);
+ Function *F = Function::Create(FTy, Function::ExternalLinkage, "foo", M);
+ BasicBlock *Entry = BasicBlock::Create(C, "entry", F);
+ Value *Ptr = PoisonValue::get(PointerType::get(C, 0));
+
+ Type *I32Ty = Type::getInt32Ty(C);
+ Constant *CV = ConstantVector::getSplat(ElementCount::getFixed(2),
+ ConstantInt::get(I32Ty, 0));
+
+ new AtomicRMWInst(AtomicRMWInst::Add, Ptr, CV, Align(8),
+ AtomicOrdering::SequentiallyConsistent, SyncScope::System,
+ /*Elementwise=*/true, Entry);
+ ReturnInst::Create(C, Entry);
+
+ std::string Error;
+ raw_string_ostream ErrorOS(Error);
+ EXPECT_TRUE(verifyFunction(*F, &ErrorOS));
+ EXPECT_TRUE(StringRef(Error).starts_with(
+ "atomicrmw elementwise cannot be sequentially consistent."))
+ << Error;
+}
+
TEST(VerifierTest, AtomicRMWElementwiseIntOpOnFPVector) {
LLVMContext C;
Module M("M", C);
More information about the llvm-branch-commits
mailing list