[llvm] [IR] Add elementwise modifier to atomic loads (PR #204556)
Harrison Hao via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 30 00:13:31 PDT 2026
https://github.com/harrisonGPU updated https://github.com/llvm/llvm-project/pull/204556
>From da7c136219fb66972bde754c03b9c8ad7961a734 Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Thu, 18 Jun 2026 18:50:37 +0800
Subject: [PATCH 1/4] [IR] Add elementwise modifier to atomic loads
---
llvm/docs/LangRef.rst | 21 ++++++++----
llvm/include/llvm/IR/Instructions.h | 17 ++++++++--
llvm/lib/AsmParser/LLParser.cpp | 16 +++++++--
llvm/lib/Bitcode/Reader/BitcodeReader.cpp | 13 +++++---
llvm/lib/Bitcode/Writer/BitcodeWriter.cpp | 2 ++
llvm/lib/IR/AsmWriter.cpp | 3 ++
llvm/lib/IR/Instruction.cpp | 1 +
llvm/lib/IR/Instructions.cpp | 12 +++++--
llvm/lib/IR/Verifier.cpp | 22 ++++++++++---
llvm/test/Assembler/atomic.ll | 5 +++
.../invalid-load-store-atomic-elementwise.ll | 33 +++++++++++++++++++
.../Bitcode/atomic-load-store-elementwise.ll | 16 +++++++++
12 files changed, 140 insertions(+), 21 deletions(-)
create mode 100644 llvm/test/Assembler/invalid-load-store-atomic-elementwise.ll
create mode 100644 llvm/test/Bitcode/atomic-load-store-elementwise.ll
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index 287341d787b64..73e27b8a40531 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -12010,7 +12010,7 @@ Syntax:
::
<result> = load [volatile] <ty>, ptr <pointer>[, align <alignment>][, !nontemporal !<nontemp_node>][, !invariant.load !<empty_node>][, !invariant.group !<empty_node>][, !nonnull !<empty_node>][, !dereferenceable !<deref_bytes_node>][, !dereferenceable_or_null !<deref_bytes_node>][, !align !<align_node>][, !noundef !<empty_node>]
- <result> = load atomic [volatile] <ty>, ptr <pointer> [syncscope("<target-scope>")] <ordering>, align <alignment> [, !invariant.group !<empty_node>]
+ <result> = load atomic [volatile] [elementwise] <ty>, ptr <pointer> [syncscope("<target-scope>")] <ordering>, align <alignment> [, !invariant.group !<empty_node>]
!<nontemp_node> = !{ i32 1 }
!<empty_node> = !{}
!<deref_bytes_node> = !{ i64 <dereferenceable_bytes> }
@@ -12037,11 +12037,20 @@ If the ``load`` is marked as ``atomic``, it takes an extra :ref:`ordering
Atomic loads produce :ref:`defined <memmodel>` results when they may see
multiple atomic stores. The type of the pointee must be an integer, pointer,
floating-point, or vector type whose bit width is a power of two greater than
-or equal to eight. ``align`` must be
-explicitly specified on atomic loads. Note: if the alignment is not greater or
-equal to the size of the `<value>` type, the atomic operation is likely to
-require a lock and have poor performance. ``!nontemporal`` does not have any
-defined semantics for atomic loads.
+or equal to eight.
+
+If the ``elementwise`` modifier is present, the loaded type must be a fixed
+vector type, and each element type must be a valid scalar atomic load type. The
+load has per-element vector atomic semantics: it behaves as if it were expanded
+into one scalar atomic load per element, and the element loads are not ordered
+with respect to each other. Without ``elementwise``, vector atomic loads keep
+whole-value atomic semantics.
+
+``align`` must be explicitly specified on atomic loads. Note: if the alignment
+is not greater than or equal to the size of the ``<ty>`` type, or the element
+type for an ``elementwise`` load, the atomic operation is likely to require a
+lock and have poor performance. ``!nontemporal`` does not have any defined
+semantics for atomic loads.
The optional constant ``align`` argument specifies the alignment of the
operation (that is, the alignment of the memory address). It is the
diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h
index f43a9b5c1acc2..0835f576c6a7a 100644
--- a/llvm/include/llvm/IR/Instructions.h
+++ b/llvm/include/llvm/IR/Instructions.h
@@ -182,9 +182,10 @@ class LoadInst : public UnaryInstruction {
using VolatileField = BoolBitfieldElementT<0>;
using AlignmentField = AlignmentBitfieldElementT<VolatileField::NextBit>;
using OrderingField = AtomicOrderingBitfieldElementT<AlignmentField::NextBit>;
- static_assert(
- Bitfield::areContiguous<VolatileField, AlignmentField, OrderingField>(),
- "Bitfields must be contiguous");
+ using ElementWiseField = BoolBitfieldElementT<OrderingField::NextBit>;
+ static_assert(Bitfield::areContiguous<VolatileField, AlignmentField,
+ OrderingField, ElementWiseField>(),
+ "Bitfields must be contiguous");
void AssertOK();
@@ -201,9 +202,13 @@ class LoadInst : public UnaryInstruction {
InsertPosition InsertBefore);
LLVM_ABI LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
Align Align, InsertPosition InsertBefore = nullptr);
+ LLVM_ABI LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
+ Align Align, AtomicOrdering Order, SyncScope::ID SSID,
+ InsertPosition InsertBefore);
LLVM_ABI LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
Align Align, AtomicOrdering Order,
SyncScope::ID SSID = SyncScope::System,
+ bool IsElementwise = false,
InsertPosition InsertBefore = nullptr);
/// Return true if this is a load from a volatile memory location.
@@ -212,6 +217,12 @@ class LoadInst : public UnaryInstruction {
/// Specify whether this is a volatile load or not.
void setVolatile(bool V) { setSubclassData<VolatileField>(V); }
+ /// Return true if this is an elementwise atomic load.
+ bool isElementwise() const { return getSubclassData<ElementWiseField>(); }
+
+ /// Specify whether this is an elementwise atomic load or not.
+ void setElementwise(bool V) { setSubclassData<ElementWiseField>(V); }
+
/// Return the alignment of the access that is being performed.
Align getAlign() const {
return Align(1ULL << (getSubclassData<AlignmentField>()));
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index 75e6add0ec76f..b409187932db4 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -8929,7 +8929,7 @@ int LLParser::parseAlloc(Instruction *&Inst, PerFunctionState &PFS) {
/// parseLoad
/// ::= 'load' 'volatile'? TypeAndValue (',' 'align' i32)?
-/// ::= 'load' 'atomic' 'volatile'? TypeAndValue
+/// ::= 'load' 'atomic' 'volatile'? 'elementwise'? TypeAndValue
/// 'singlethread'? AtomicOrdering (',' 'align' i32)?
int LLParser::parseLoad(Instruction *&Inst, PerFunctionState &PFS) {
Value *Val; LocTy Loc;
@@ -8950,6 +8950,12 @@ int LLParser::parseLoad(Instruction *&Inst, PerFunctionState &PFS) {
Lex.Lex();
}
+ bool IsElementwise = false;
+ if (Lex.getKind() == lltok::kw_elementwise) {
+ IsElementwise = true;
+ Lex.Lex();
+ }
+
Type *Ty;
LocTy ExplicitTypeLoc = Lex.getLoc();
if (parseType(Ty) ||
@@ -8961,8 +8967,13 @@ int LLParser::parseLoad(Instruction *&Inst, PerFunctionState &PFS) {
if (!Val->getType()->isPointerTy() || !Ty->isFirstClassType())
return error(Loc, "load operand must be a pointer to a first class type");
+
+ if (IsElementwise && !isAtomic)
+ return error(Loc, "elementwise load must be atomic");
+
if (isAtomic && !Alignment)
return error(Loc, "atomic load must have explicit non-zero alignment");
+
if (Ordering == AtomicOrdering::Release ||
Ordering == AtomicOrdering::AcquireRelease)
return error(Loc, "atomic load cannot use Release ordering");
@@ -8972,7 +8983,8 @@ int LLParser::parseLoad(Instruction *&Inst, PerFunctionState &PFS) {
return error(ExplicitTypeLoc, "loading unsized types is not allowed");
if (!Alignment)
Alignment = M->getDataLayout().getABITypeAlign(Ty);
- Inst = new LoadInst(Ty, Val, "", isVolatile, *Alignment, Ordering, SSID);
+ Inst = new LoadInst(Ty, Val, "", isVolatile, *Alignment, Ordering, SSID,
+ IsElementwise);
return AteExtraComma ? InstExtraComma : InstNormal;
}
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index f46d240e7ed9b..ac9f90b32c573 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -6370,19 +6370,22 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
break;
}
case bitc::FUNC_CODE_INST_LOADATOMIC: {
- // LOADATOMIC: [opty, op, align, vol, ordering, ssid]
+ // LOADATOMIC: [opty, op, align, vol, ordering, ssid, elementwise?]
unsigned OpNum = 0;
Value *Op;
unsigned OpTypeID;
if (getValueTypePair(Record, OpNum, NextValueNo, Op, OpTypeID, CurBB) ||
- (OpNum + 4 != Record.size() && OpNum + 5 != Record.size()))
+ (OpNum + 4 != Record.size() && OpNum + 5 != Record.size() &&
+ OpNum + 6 != Record.size()))
return error("Invalid load atomic record");
if (!isa<PointerType>(Op->getType()))
return error("Load operand is not a pointer type");
Type *Ty = nullptr;
- if (OpNum + 5 == Record.size()) {
+ bool HasExplicitType =
+ Record.size() == OpNum + 5 || Record.size() == OpNum + 6;
+ if (HasExplicitType) {
ResTypeID = Record[OpNum++];
Ty = getTypeByID(ResTypeID);
} else {
@@ -6404,13 +6407,15 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
if (Ordering != AtomicOrdering::NotAtomic && Record[OpNum] == 0)
return error("Invalid load atomic record");
SyncScope::ID SSID = getDecodedSyncScopeID(Record[OpNum + 3]);
+ bool IsElementwise = Record.size() > OpNum + 4 && Record[OpNum + 4];
MaybeAlign Align;
if (Error Err = parseAlignmentValue(Record[OpNum], Align))
return Err;
if (!Align)
return error("Alignment missing from atomic load");
- I = new LoadInst(Ty, Op, "", Record[OpNum + 1], *Align, Ordering, SSID);
+ I = new LoadInst(Ty, Op, "", Record[OpNum + 1], *Align, Ordering, SSID,
+ IsElementwise);
InstructionList.push_back(I);
break;
}
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index f4857461ca58e..33f477fb1b196 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -3565,6 +3565,8 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I,
if (cast<LoadInst>(I).isAtomic()) {
Vals.push_back(getEncodedOrdering(cast<LoadInst>(I).getOrdering()));
Vals.push_back(getEncodedSyncScopeID(cast<LoadInst>(I).getSyncScopeID()));
+ if (cast<LoadInst>(I).isElementwise())
+ Vals.push_back(1);
}
break;
case Instruction::Store:
diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp
index 52ed28f71f615..31ed4f30838e8 100644
--- a/llvm/lib/IR/AsmWriter.cpp
+++ b/llvm/lib/IR/AsmWriter.cpp
@@ -4453,6 +4453,9 @@ void AssemblyWriter::printInstruction(const Instruction &I) {
(isa<AtomicRMWInst>(I) && cast<AtomicRMWInst>(I).isVolatile()))
Out << " volatile";
+ if (isa<LoadInst>(I) && cast<LoadInst>(I).isElementwise())
+ Out << " elementwise";
+
// Print out optimization information.
writeOptimizationInfo(Out, &I);
diff --git a/llvm/lib/IR/Instruction.cpp b/llvm/lib/IR/Instruction.cpp
index 4e43de1f08fd0..9b2f7d92fab39 100644
--- a/llvm/lib/IR/Instruction.cpp
+++ b/llvm/lib/IR/Instruction.cpp
@@ -926,6 +926,7 @@ bool Instruction::hasSameSpecialState(const Instruction *I2,
IgnoreAlignment);
if (const LoadInst *LI = dyn_cast<LoadInst>(I1))
return LI->isVolatile() == cast<LoadInst>(I2)->isVolatile() &&
+ LI->isElementwise() == cast<LoadInst>(I2)->isElementwise() &&
(LI->getAlign() == cast<LoadInst>(I2)->getAlign() ||
IgnoreAlignment) &&
LI->getOrdering() == cast<LoadInst>(I2)->getOrdering() &&
diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index 496a3ffa5d965..5065b23850b41 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -1360,15 +1360,22 @@ LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
Align Align, InsertPosition InsertBef)
: LoadInst(Ty, Ptr, Name, isVolatile, Align, AtomicOrdering::NotAtomic,
- SyncScope::System, InsertBef) {}
+ SyncScope::System, /*IsElementwise=*/false, InsertBef) {}
LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
Align Align, AtomicOrdering Order, SyncScope::ID SSID,
InsertPosition InsertBef)
+ : LoadInst(Ty, Ptr, Name, isVolatile, Align, Order, SSID,
+ /*IsElementwise=*/false, InsertBef) {}
+
+LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
+ Align Align, AtomicOrdering Order, SyncScope::ID SSID,
+ bool IsElementwise, InsertPosition InsertBef)
: UnaryInstruction(Ty, Load, Ptr, InsertBef) {
setVolatile(isVolatile);
setAlignment(Align);
setAtomic(Order, SSID);
+ setElementwise(IsElementwise);
AssertOK();
setName(Name);
}
@@ -4439,7 +4446,8 @@ AllocaInst *AllocaInst::cloneImpl() const {
LoadInst *LoadInst::cloneImpl() const {
return new LoadInst(getType(), getOperand(0), Twine(), isVolatile(),
- getAlign(), getOrdering(), getSyncScopeID());
+ getAlign(), getOrdering(), getSyncScopeID(),
+ isElementwise());
}
StoreInst *StoreInst::cloneImpl() const {
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 18d6578dec646..3a84ef44dd3b2 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -4783,15 +4783,29 @@ void Verifier::visitLoadInst(LoadInst &LI) {
Check(LI.getOrdering() != AtomicOrdering::Release &&
LI.getOrdering() != AtomicOrdering::AcquireRelease,
"Load cannot have Release ordering", &LI);
- Check(ElTy->getScalarType()->isIntOrPtrTy() ||
- ElTy->getScalarType()->isByteTy() ||
- ElTy->getScalarType()->isFloatingPointTy(),
+
+ Type *ScalarTy = ElTy;
+ if (LI.isElementwise()) {
+ auto *VecTy = dyn_cast<FixedVectorType>(ElTy);
+ Check(VecTy,
+ "atomic elementwise load operand must have fixed vector type!", &LI,
+ ElTy);
+ if (VecTy) {
+ checkAtomicMemAccessSize(ScalarTy, &LI);
+ ScalarTy = VecTy->getElementType();
+ }
+ }
+
+ Check(ScalarTy->getScalarType()->isIntOrPtrTy() ||
+ ScalarTy->getScalarType()->isByteTy() ||
+ ScalarTy->getScalarType()->isFloatingPointTy(),
"atomic load operand must have integer, byte, pointer, floating "
"point, or vector type!",
ElTy, &LI);
- checkAtomicMemAccessSize(ElTy, &LI);
+ checkAtomicMemAccessSize(ScalarTy, &LI);
} else {
+ Check(!LI.isElementwise(), "non-atomic load cannot be elementwise", &LI);
Check(LI.getSyncScopeID() == SyncScope::System,
"Non-atomic load cannot have SynchronizationScope specified", &LI);
}
diff --git a/llvm/test/Assembler/atomic.ll b/llvm/test/Assembler/atomic.ll
index 611a717fa9a8d..64a58249ee70d 100644
--- a/llvm/test/Assembler/atomic.ll
+++ b/llvm/test/Assembler/atomic.ll
@@ -71,6 +71,11 @@ define void @f(ptr %x) {
; CHECK : store atomic <2 x float> <float 3.0, float 4.0>, ptr %x release, align 4
store atomic <2 x float> <float 3.0, float 4.0>, ptr %x release, align 4
+ ; CHECK: load atomic elementwise <2 x float>, ptr %x syncscope("agent") monotonic, align 4
+ load atomic elementwise <2 x float>, ptr %x syncscope("agent") monotonic, align 4
+ ; CHECK: load atomic volatile elementwise <2 x i32>, ptr %x monotonic, align 4
+ load atomic volatile elementwise <2 x i32>, ptr %x monotonic, align 4
+
; CHECK: fence syncscope("singlethread") release
fence syncscope("singlethread") release
; CHECK: fence seq_cst
diff --git a/llvm/test/Assembler/invalid-load-store-atomic-elementwise.ll b/llvm/test/Assembler/invalid-load-store-atomic-elementwise.ll
new file mode 100644
index 0000000000000..53d8571a9b461
--- /dev/null
+++ b/llvm/test/Assembler/invalid-load-store-atomic-elementwise.ll
@@ -0,0 +1,33 @@
+; RUN: split-file %s %t
+; RUN: not llvm-as -disable-output %t/load-non-atomic.ll 2>&1 | FileCheck %t/load-non-atomic.ll
+; RUN: not llvm-as -disable-output %t/load-scalar.ll 2>&1 | FileCheck %t/load-scalar.ll
+; RUN: not llvm-as -disable-output %t/load-scalable.ll 2>&1 | FileCheck %t/load-scalable.ll
+; RUN: not llvm-as -disable-output %t/load-odd-sized.ll 2>&1 | FileCheck %t/load-odd-sized.ll
+
+;--- load-non-atomic.ll
+; CHECK: elementwise load must be atomic
+define <2 x float> @bad_non_atomic(ptr %p) {
+ %v = load elementwise <2 x float>, ptr %p, align 4
+ ret <2 x float> %v
+}
+
+;--- load-scalar.ll
+; CHECK: atomic elementwise load operand must have fixed vector type
+define float @bad_scalar(ptr %p) {
+ %v = load atomic elementwise float, ptr %p monotonic, align 4
+ ret float %v
+}
+
+;--- load-scalable.ll
+; CHECK: atomic elementwise load operand must have fixed vector type
+define <vscale x 2 x i32> @bad_scalable(ptr %p) {
+ %v = load atomic elementwise <vscale x 2 x i32>, ptr %p monotonic, align 4
+ ret <vscale x 2 x i32> %v
+}
+
+;--- load-odd-sized.ll
+; CHECK: atomic memory access' operand must have a power-of-two size
+define <5 x i32> @bad_odd_sized_vector(ptr %p) {
+ %v = load atomic elementwise <5 x i32>, ptr %p monotonic, align 4
+ ret <5 x i32> %v
+}
diff --git a/llvm/test/Bitcode/atomic-load-store-elementwise.ll b/llvm/test/Bitcode/atomic-load-store-elementwise.ll
new file mode 100644
index 0000000000000..6b921c199c08c
--- /dev/null
+++ b/llvm/test/Bitcode/atomic-load-store-elementwise.ll
@@ -0,0 +1,16 @@
+; RUN: llvm-as %s -o - | llvm-dis | FileCheck %s
+; RUN: llvm-as %s -o - | verify-uselistorder
+
+define <2 x float> @load_elem_f32(ptr %p) {
+; CHECK-LABEL: @load_elem_f32(
+; CHECK: %v = load atomic elementwise <2 x float>, ptr %p syncscope("agent") monotonic, align 4
+ %v = load atomic elementwise <2 x float>, ptr %p syncscope("agent") monotonic, align 4
+ ret <2 x float> %v
+}
+
+define <4 x i32> @load_elem_i32_volatile(ptr %p) {
+; CHECK-LABEL: @load_elem_i32_volatile(
+; CHECK: %v = load atomic volatile elementwise <4 x i32>, ptr %p seq_cst, align 4
+ %v = load atomic volatile elementwise <4 x i32>, ptr %p seq_cst, align 4
+ ret <4 x i32> %v
+}
>From a91a221637be87776118c0a8f6cf4c57264f2425 Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Fri, 26 Jun 2026 17:01:55 +0800
Subject: [PATCH 2/4] Update for comments.
---
llvm/docs/LangRef.rst | 12 +++++------
llvm/include/llvm/IR/Instructions.h | 7 ++-----
llvm/lib/AsmParser/LLParser.cpp | 6 +++++-
llvm/lib/Bitcode/Reader/BitcodeReader.cpp | 2 +-
llvm/lib/Bitcode/Writer/BitcodeWriter.cpp | 25 +++++++++++++----------
llvm/lib/IR/Instructions.cpp | 12 +++--------
llvm/test/Bitcode/compatibility.ll | 6 ++++++
7 files changed, 37 insertions(+), 33 deletions(-)
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index 73e27b8a40531..37fc7d804f8c1 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -12046,13 +12046,13 @@ into one scalar atomic load per element, and the element loads are not ordered
with respect to each other. Without ``elementwise``, vector atomic loads keep
whole-value atomic semantics.
-``align`` must be explicitly specified on atomic loads. Note: if the alignment
-is not greater than or equal to the size of the ``<ty>`` type, or the element
-type for an ``elementwise`` load, the atomic operation is likely to require a
-lock and have poor performance. ``!nontemporal`` does not have any defined
-semantics for atomic loads.
+``align`` must be explicitly specified on atomic loads, and is otherwise
+optional on non-atomic loads. Note: if the alignment is not greater than or equal
+to the size of the ``<ty>`` type, or the element type for an ``elementwise`` load,
+the atomic operation is likely to require a lock and have poor performance.
+``!nontemporal`` does not have any defined semantics for atomic loads.
-The optional constant ``align`` argument specifies the alignment of the
+The constant ``align`` argument specifies the alignment of the
operation (that is, the alignment of the memory address). It is the
responsibility of the code emitter to ensure that the alignment information is
correct. Overestimating the alignment results in undefined behavior.
diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h
index 0835f576c6a7a..1b09e92769e39 100644
--- a/llvm/include/llvm/IR/Instructions.h
+++ b/llvm/include/llvm/IR/Instructions.h
@@ -202,14 +202,11 @@ class LoadInst : public UnaryInstruction {
InsertPosition InsertBefore);
LLVM_ABI LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
Align Align, InsertPosition InsertBefore = nullptr);
- LLVM_ABI LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
- Align Align, AtomicOrdering Order, SyncScope::ID SSID,
- InsertPosition InsertBefore);
LLVM_ABI LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
Align Align, AtomicOrdering Order,
SyncScope::ID SSID = SyncScope::System,
- bool IsElementwise = false,
- InsertPosition InsertBefore = nullptr);
+ InsertPosition InsertBefore = nullptr,
+ bool IsElementwise = false);
/// Return true if this is a load from a volatile memory location.
bool isVolatile() const { return getSubclassData<VolatileField>(); }
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index b409187932db4..9d7975cc0f2c4 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -8971,6 +8971,10 @@ int LLParser::parseLoad(Instruction *&Inst, PerFunctionState &PFS) {
if (IsElementwise && !isAtomic)
return error(Loc, "elementwise load must be atomic");
+ if (IsElementwise && !isa<FixedVectorType>(Ty))
+ return error(ExplicitTypeLoc,
+ "atomic elementwise load operand must have fixed vector type");
+
if (isAtomic && !Alignment)
return error(Loc, "atomic load must have explicit non-zero alignment");
@@ -8984,7 +8988,7 @@ int LLParser::parseLoad(Instruction *&Inst, PerFunctionState &PFS) {
if (!Alignment)
Alignment = M->getDataLayout().getABITypeAlign(Ty);
Inst = new LoadInst(Ty, Val, "", isVolatile, *Alignment, Ordering, SSID,
- IsElementwise);
+ /*InsertBefore=*/nullptr, IsElementwise);
return AteExtraComma ? InstExtraComma : InstNormal;
}
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index ac9f90b32c573..fb54fcea7c526 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -6415,7 +6415,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
if (!Align)
return error("Alignment missing from atomic load");
I = new LoadInst(Ty, Op, "", Record[OpNum + 1], *Align, Ordering, SSID,
- IsElementwise);
+ /*InsertBefore=*/nullptr, IsElementwise);
InstructionList.push_back(I);
break;
}
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index 33f477fb1b196..d4752a39fff70 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -3550,25 +3550,28 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I,
break;
}
- case Instruction::Load:
- if (cast<LoadInst>(I).isAtomic()) {
+ case Instruction::Load: {
+ const auto &LI = cast<LoadInst>(I);
+ if (LI.isAtomic()) {
Code = bitc::FUNC_CODE_INST_LOADATOMIC;
- pushValueAndType(I.getOperand(0), InstID, Vals);
+ pushValueAndType(LI.getOperand(0), InstID, Vals);
} else {
Code = bitc::FUNC_CODE_INST_LOAD;
- if (!pushValueAndType(I.getOperand(0), InstID, Vals)) // ptr
+ if (!pushValueAndType(LI.getOperand(0), InstID, Vals)) // ptr
AbbrevToUse = FUNCTION_INST_LOAD_ABBREV;
}
- Vals.push_back(VE.getTypeID(I.getType()));
- Vals.push_back(getEncodedAlign(cast<LoadInst>(I).getAlign()));
- Vals.push_back(cast<LoadInst>(I).isVolatile());
- if (cast<LoadInst>(I).isAtomic()) {
- Vals.push_back(getEncodedOrdering(cast<LoadInst>(I).getOrdering()));
- Vals.push_back(getEncodedSyncScopeID(cast<LoadInst>(I).getSyncScopeID()));
- if (cast<LoadInst>(I).isElementwise())
+ Vals.push_back(VE.getTypeID(LI.getType()));
+ Vals.push_back(getEncodedAlign(LI.getAlign()));
+ Vals.push_back(LI.isVolatile());
+ if (LI.isAtomic()) {
+ Vals.push_back(getEncodedOrdering(LI.getOrdering()));
+ Vals.push_back(getEncodedSyncScopeID(LI.getSyncScopeID()));
+ if (LI.isElementwise())
Vals.push_back(1);
}
break;
+ }
+
case Instruction::Store:
if (cast<StoreInst>(I).isAtomic()) {
Code = bitc::FUNC_CODE_INST_STOREATOMIC;
diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index 5065b23850b41..257d6a97f5d0f 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -1360,17 +1360,11 @@ LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
Align Align, InsertPosition InsertBef)
: LoadInst(Ty, Ptr, Name, isVolatile, Align, AtomicOrdering::NotAtomic,
- SyncScope::System, /*IsElementwise=*/false, InsertBef) {}
+ SyncScope::System, InsertBef, /*IsElementwise=*/false) {}
LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
Align Align, AtomicOrdering Order, SyncScope::ID SSID,
- InsertPosition InsertBef)
- : LoadInst(Ty, Ptr, Name, isVolatile, Align, Order, SSID,
- /*IsElementwise=*/false, InsertBef) {}
-
-LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
- Align Align, AtomicOrdering Order, SyncScope::ID SSID,
- bool IsElementwise, InsertPosition InsertBef)
+ InsertPosition InsertBef, bool IsElementwise)
: UnaryInstruction(Ty, Load, Ptr, InsertBef) {
setVolatile(isVolatile);
setAlignment(Align);
@@ -4447,7 +4441,7 @@ AllocaInst *AllocaInst::cloneImpl() const {
LoadInst *LoadInst::cloneImpl() const {
return new LoadInst(getType(), getOperand(0), Twine(), isVolatile(),
getAlign(), getOrdering(), getSyncScopeID(),
- isElementwise());
+ /*InsertBefore=*/nullptr, isElementwise());
}
StoreInst *StoreInst::cloneImpl() const {
diff --git a/llvm/test/Bitcode/compatibility.ll b/llvm/test/Bitcode/compatibility.ll
index d55aa1dc496df..f84cec4f65920 100644
--- a/llvm/test/Bitcode/compatibility.ll
+++ b/llvm/test/Bitcode/compatibility.ll
@@ -1033,6 +1033,12 @@ define void @elementwise_atomics(ptr %word, <4 x i32> %ival, <4 x float> %fval)
; CHECK: %atomicrmw.fadd = atomicrmw elementwise fadd ptr %word, <4 x float> %fval seq_cst, align 16
%atomicrmw.fadd = atomicrmw elementwise fadd ptr %word, <4 x float> %fval seq_cst, align 16
+; CHECK: %load.elementwise = load atomic elementwise <4 x i32>, ptr %word monotonic, align 4
+ %load.elementwise = load atomic elementwise <4 x i32>, ptr %word monotonic, align 4
+
+; CHECK: %load.elementwise.volatile = load atomic volatile elementwise <4 x float>, ptr %word seq_cst, align 4
+ %load.elementwise.volatile = load atomic volatile elementwise <4 x float>, ptr %word seq_cst, align 4
+
ret void
}
>From 33ac0aa7959dd6e3c2560d76a7975b441337a228 Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Fri, 26 Jun 2026 17:34:38 +0800
Subject: [PATCH 3/4] Add verifier test
---
llvm/unittests/IR/VerifierTest.cpp | 71 ++++++++++++++++++++++++++++++
1 file changed, 71 insertions(+)
diff --git a/llvm/unittests/IR/VerifierTest.cpp b/llvm/unittests/IR/VerifierTest.cpp
index 8b3b31ea68395..e39955b87ca64 100644
--- a/llvm/unittests/IR/VerifierTest.cpp
+++ b/llvm/unittests/IR/VerifierTest.cpp
@@ -505,6 +505,77 @@ TEST(VerifierTest, AtomicRMWElementwiseFPOpOnIntVector) {
<< Error;
}
+TEST(VerifierTest, ElementwiseLoadNonAtomic) {
+ LLVMContext C;
+ Module M("M", C);
+ FunctionType *FTy = FunctionType::get(Type::getVoidTy(C), /*isVarArg=*/false);
+ Function *F = Function::Create(FTy, Function::ExternalLinkage, "foo", M);
+ BasicBlock *Entry = BasicBlock::Create(C, "entry", F);
+ Value *Ptr = PoisonValue::get(PointerType::get(C, 0));
+
+ Type *I32Ty = Type::getInt32Ty(C);
+ Type *VecTy = FixedVectorType::get(I32Ty, 4);
+
+ new LoadInst(VecTy, Ptr, "", /*isVolatile=*/false, Align(4),
+ AtomicOrdering::NotAtomic, SyncScope::System, Entry,
+ /*IsElementwise=*/true);
+ ReturnInst::Create(C, Entry);
+
+ std::string Error;
+ raw_string_ostream ErrorOS(Error);
+ EXPECT_TRUE(verifyFunction(*F, &ErrorOS));
+ EXPECT_TRUE(
+ StringRef(Error).starts_with("non-atomic load cannot be elementwise"))
+ << Error;
+}
+
+TEST(VerifierTest, ElementwiseLoadScalar) {
+ LLVMContext C;
+ Module M("M", C);
+ FunctionType *FTy = FunctionType::get(Type::getVoidTy(C), /*isVarArg=*/false);
+ Function *F = Function::Create(FTy, Function::ExternalLinkage, "foo", M);
+ BasicBlock *Entry = BasicBlock::Create(C, "entry", F);
+ Value *Ptr = PoisonValue::get(PointerType::get(C, 0));
+
+ Type *I32Ty = Type::getInt32Ty(C);
+
+ new LoadInst(I32Ty, Ptr, "", /*isVolatile=*/false, Align(4),
+ AtomicOrdering::Monotonic, SyncScope::System, Entry,
+ /*IsElementwise=*/true);
+ ReturnInst::Create(C, Entry);
+
+ std::string Error;
+ raw_string_ostream ErrorOS(Error);
+ EXPECT_TRUE(verifyFunction(*F, &ErrorOS));
+ EXPECT_TRUE(StringRef(Error).starts_with(
+ "atomic elementwise load operand must have fixed vector type!"))
+ << Error;
+}
+
+TEST(VerifierTest, ElementwiseLoadOddSizedVector) {
+ LLVMContext C;
+ Module M("M", C);
+ FunctionType *FTy = FunctionType::get(Type::getVoidTy(C), /*isVarArg=*/false);
+ Function *F = Function::Create(FTy, Function::ExternalLinkage, "foo", M);
+ BasicBlock *Entry = BasicBlock::Create(C, "entry", F);
+ Value *Ptr = PoisonValue::get(PointerType::get(C, 0));
+
+ Type *I32Ty = Type::getInt32Ty(C);
+ Type *VecTy = FixedVectorType::get(I32Ty, 5);
+
+ new LoadInst(VecTy, Ptr, "", /*isVolatile=*/false, Align(4),
+ AtomicOrdering::Monotonic, SyncScope::System, Entry,
+ /*IsElementwise=*/true);
+ ReturnInst::Create(C, Entry);
+
+ std::string Error;
+ raw_string_ostream ErrorOS(Error);
+ EXPECT_TRUE(verifyFunction(*F, &ErrorOS));
+ EXPECT_TRUE(StringRef(Error).starts_with(
+ "atomic memory access' operand must have a power-of-two size"))
+ << Error;
+}
+
TEST(VerifierTest, GetElementPtrInst) {
LLVMContext C;
Module M("M", C);
>From dd9a82202a9d3fd86ca525dfb4973a64cf8f1fe5 Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Mon, 29 Jun 2026 17:03:31 +0800
Subject: [PATCH 4/4] Update for comments
---
llvm/docs/LangRef.rst | 17 ++++++++++-------
llvm/lib/Bitcode/Reader/BitcodeReader.cpp | 4 +---
.../invalid-load-store-atomic-elementwise.ll | 8 ++++++++
3 files changed, 19 insertions(+), 10 deletions(-)
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index 37fc7d804f8c1..f100830ec5217 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -12035,15 +12035,18 @@ If the ``load`` is marked as ``atomic``, it takes an extra :ref:`ordering
<ordering>` and optional ``syncscope("<target-scope>")`` argument. The
``release`` and ``acq_rel`` orderings are not valid on ``load`` instructions.
Atomic loads produce :ref:`defined <memmodel>` results when they may see
-multiple atomic stores. The type of the pointee must be an integer, pointer,
-floating-point, or vector type whose bit width is a power of two greater than
-or equal to eight.
+multiple atomic stores. Atomic loads support scalar integer, pointer, and
+floating-point types whose bit width is a power of two greater than or equal
+to eight. Atomic loads also support vector types whose total bit width is a
+power of two greater than or equal to eight. That is, the entire vector is
+loaded atomically.
If the ``elementwise`` modifier is present, the loaded type must be a fixed
-vector type, and each element type must be a valid scalar atomic load type. The
-load has per-element vector atomic semantics: it behaves as if it were expanded
-into one scalar atomic load per element, and the element loads are not ordered
-with respect to each other. Without ``elementwise``, vector atomic loads keep
+vector type whose total bit width is a power of two greater than or equal to
+eight, and whose element type is supported by scalar atomic loads. The load has
+per-element atomic load semantics: it behaves as if it were expanded into
+one scalar atomic load per element, and the element loads are not ordered with
+respect to each other. Without ``elementwise``, vector atomic loads keep
whole-value atomic semantics.
``align`` must be explicitly specified on atomic loads, and is otherwise
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index fb54fcea7c526..4d53001f854bd 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -6383,9 +6383,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
return error("Load operand is not a pointer type");
Type *Ty = nullptr;
- bool HasExplicitType =
- Record.size() == OpNum + 5 || Record.size() == OpNum + 6;
- if (HasExplicitType) {
+ if (Record.size() >= OpNum + 5) {
ResTypeID = Record[OpNum++];
Ty = getTypeByID(ResTypeID);
} else {
diff --git a/llvm/test/Assembler/invalid-load-store-atomic-elementwise.ll b/llvm/test/Assembler/invalid-load-store-atomic-elementwise.ll
index 53d8571a9b461..03dbc96cb562e 100644
--- a/llvm/test/Assembler/invalid-load-store-atomic-elementwise.ll
+++ b/llvm/test/Assembler/invalid-load-store-atomic-elementwise.ll
@@ -3,6 +3,7 @@
; RUN: not llvm-as -disable-output %t/load-scalar.ll 2>&1 | FileCheck %t/load-scalar.ll
; RUN: not llvm-as -disable-output %t/load-scalable.ll 2>&1 | FileCheck %t/load-scalable.ll
; RUN: not llvm-as -disable-output %t/load-odd-sized.ll 2>&1 | FileCheck %t/load-odd-sized.ll
+; RUN: not llvm-as -disable-output %t/load-non-byte.ll 2>&1 | FileCheck %t/load-non-byte.ll
;--- load-non-atomic.ll
; CHECK: elementwise load must be atomic
@@ -31,3 +32,10 @@ define <5 x i32> @bad_odd_sized_vector(ptr %p) {
%v = load atomic elementwise <5 x i32>, ptr %p monotonic, align 4
ret <5 x i32> %v
}
+
+;--- load-non-byte.ll
+; CHECK: atomic memory access' size must be byte-sized
+define <4 x i1> @bad_non_byte(ptr %p) {
+ %v = load atomic elementwise <4 x i1>, ptr %p monotonic, align 4
+ ret <4 x i1> %v
+}
More information about the llvm-commits
mailing list