[llvm] [IR] Add elementwise modifier to atomic loads (PR #204556)
Harrison Hao via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 9 19:52:11 PDT 2026
https://github.com/harrisonGPU updated https://github.com/llvm/llvm-project/pull/204556
>From 7e7cd59d0ff6c334727580518ec618e450384ffd Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Thu, 18 Jun 2026 18:50:37 +0800
Subject: [PATCH 1/8] [IR] Add elementwise modifier to atomic loads
---
llvm/docs/LangRef.md | 21 ++++++++----
llvm/include/llvm/IR/Instructions.h | 17 ++++++++--
llvm/lib/AsmParser/LLParser.cpp | 16 +++++++--
llvm/lib/Bitcode/Reader/BitcodeReader.cpp | 13 +++++---
llvm/lib/Bitcode/Writer/BitcodeWriter.cpp | 2 ++
llvm/lib/IR/AsmWriter.cpp | 3 ++
llvm/lib/IR/Instruction.cpp | 1 +
llvm/lib/IR/Instructions.cpp | 12 +++++--
llvm/lib/IR/Verifier.cpp | 22 ++++++++++---
llvm/test/Assembler/atomic.ll | 5 +++
.../invalid-load-store-atomic-elementwise.ll | 33 +++++++++++++++++++
.../Bitcode/atomic-load-store-elementwise.ll | 16 +++++++++
12 files changed, 140 insertions(+), 21 deletions(-)
create mode 100644 llvm/test/Assembler/invalid-load-store-atomic-elementwise.ll
create mode 100644 llvm/test/Bitcode/atomic-load-store-elementwise.ll
diff --git a/llvm/docs/LangRef.md b/llvm/docs/LangRef.md
index 7c03109f56652..bcee67a3b5e7d 100644
--- a/llvm/docs/LangRef.md
+++ b/llvm/docs/LangRef.md
@@ -11709,7 +11709,7 @@ behavior.
```
<result> = load [volatile] <ty>, ptr <pointer>[, align <alignment>][, !nontemporal !<nontemp_node>][, !invariant.load !<empty_node>][, !invariant.group !<empty_node>][, !nonnull !<empty_node>][, !dereferenceable !<deref_bytes_node>][, !dereferenceable_or_null !<deref_bytes_node>][, !align !<align_node>][, !noundef !<empty_node>]
-<result> = load atomic [volatile] <ty>, ptr <pointer> [syncscope("<target-scope>")] <ordering>, align <alignment> [, !invariant.group !<empty_node>]
+<result> = load atomic [volatile] [elementwise] <ty>, ptr <pointer> [syncscope("<target-scope>")] <ordering>, align <alignment> [, !invariant.group !<empty_node>]
!<nontemp_node> = !{ i32 1 }
!<empty_node> = !{}
!<deref_bytes_node> = !{ i64 <dereferenceable_bytes> }
@@ -11734,11 +11734,20 @@ If the `load` is marked as `atomic`, it takes an extra {ref}`ordering <ordering>
Atomic loads produce {ref}`defined <memmodel>` results when they may see
multiple atomic stores. The type of the pointee must be an integer, pointer,
floating-point, or vector type whose bit width is a power of two greater than
-or equal to eight. `align` must be
-explicitly specified on atomic loads. Note: if the alignment is not greater or
-equal to the size of the `<value>` type, the atomic operation is likely to
-require a lock and have poor performance. `!nontemporal` does not have any
-defined semantics for atomic loads.
+or equal to eight.
+
+If the `elementwise` modifier is present, the loaded type must be a fixed
+vector type, and each element type must be a valid scalar atomic load type. The
+load has per-element vector atomic semantics: it behaves as if it were expanded
+into one scalar atomic load per element, and the element loads are not ordered
+with respect to each other. Without `elementwise`, vector atomic loads keep
+whole-value atomic semantics.
+
+`align` must be explicitly specified on atomic loads. Note: if the alignment
+is not greater than or equal to the size of the `<ty>` type, or the element
+type for an `elementwise` load, the atomic operation is likely to require a
+lock and have poor performance. `!nontemporal` does not have any defined
+semantics for atomic loads.
The optional constant `align` argument specifies the alignment of the
operation (that is, the alignment of the memory address). It is the
diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h
index 8ceb4cf06d15e..a56b82dc2478c 100644
--- a/llvm/include/llvm/IR/Instructions.h
+++ b/llvm/include/llvm/IR/Instructions.h
@@ -190,9 +190,10 @@ class LoadInst : public UnaryInstruction {
using VolatileField = BoolBitfieldElementT<0>;
using AlignmentField = AlignmentBitfieldElementT<VolatileField::NextBit>;
using OrderingField = AtomicOrderingBitfieldElementT<AlignmentField::NextBit>;
- static_assert(
- Bitfield::areContiguous<VolatileField, AlignmentField, OrderingField>(),
- "Bitfields must be contiguous");
+ using ElementWiseField = BoolBitfieldElementT<OrderingField::NextBit>;
+ static_assert(Bitfield::areContiguous<VolatileField, AlignmentField,
+ OrderingField, ElementWiseField>(),
+ "Bitfields must be contiguous");
void AssertOK();
@@ -209,9 +210,13 @@ class LoadInst : public UnaryInstruction {
InsertPosition InsertBefore);
LLVM_ABI LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
Align Align, InsertPosition InsertBefore = nullptr);
+ LLVM_ABI LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
+ Align Align, AtomicOrdering Order, SyncScope::ID SSID,
+ InsertPosition InsertBefore);
LLVM_ABI LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
Align Align, AtomicOrdering Order,
SyncScope::ID SSID = SyncScope::System,
+ bool IsElementwise = false,
InsertPosition InsertBefore = nullptr);
LLVM_ABI LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr,
const LoadStoreInstProperties &Props,
@@ -223,6 +228,12 @@ class LoadInst : public UnaryInstruction {
/// Specify whether this is a volatile load or not.
void setVolatile(bool V) { setSubclassData<VolatileField>(V); }
+ /// Return true if this is an elementwise atomic load.
+ bool isElementwise() const { return getSubclassData<ElementWiseField>(); }
+
+ /// Specify whether this is an elementwise atomic load or not.
+ void setElementwise(bool V) { setSubclassData<ElementWiseField>(V); }
+
/// Return the alignment of the access that is being performed.
Align getAlign() const {
return Align(1ULL << (getSubclassData<AlignmentField>()));
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index 38d10587b104e..c38214d8d1b5d 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -8962,7 +8962,7 @@ int LLParser::parseAlloc(Instruction *&Inst, PerFunctionState &PFS) {
/// parseLoad
/// ::= 'load' 'volatile'? TypeAndValue (',' 'align' i32)?
-/// ::= 'load' 'atomic' 'volatile'? TypeAndValue
+/// ::= 'load' 'atomic' 'volatile'? 'elementwise'? TypeAndValue
/// 'singlethread'? AtomicOrdering (',' 'align' i32)?
int LLParser::parseLoad(Instruction *&Inst, PerFunctionState &PFS) {
Value *Val; LocTy Loc;
@@ -8983,6 +8983,12 @@ int LLParser::parseLoad(Instruction *&Inst, PerFunctionState &PFS) {
Lex.Lex();
}
+ bool IsElementwise = false;
+ if (Lex.getKind() == lltok::kw_elementwise) {
+ IsElementwise = true;
+ Lex.Lex();
+ }
+
Type *Ty;
LocTy ExplicitTypeLoc = Lex.getLoc();
if (parseType(Ty) ||
@@ -8994,8 +9000,13 @@ int LLParser::parseLoad(Instruction *&Inst, PerFunctionState &PFS) {
if (!Val->getType()->isPointerTy() || !Ty->isFirstClassType())
return error(Loc, "load operand must be a pointer to a first class type");
+
+ if (IsElementwise && !isAtomic)
+ return error(Loc, "elementwise load must be atomic");
+
if (isAtomic && !Alignment)
return error(Loc, "atomic load must have explicit non-zero alignment");
+
if (Ordering == AtomicOrdering::Release ||
Ordering == AtomicOrdering::AcquireRelease)
return error(Loc, "atomic load cannot use Release ordering");
@@ -9005,7 +9016,8 @@ int LLParser::parseLoad(Instruction *&Inst, PerFunctionState &PFS) {
return error(ExplicitTypeLoc, "loading unsized types is not allowed");
if (!Alignment)
Alignment = M->getDataLayout().getABITypeAlign(Ty);
- Inst = new LoadInst(Ty, Val, "", isVolatile, *Alignment, Ordering, SSID);
+ Inst = new LoadInst(Ty, Val, "", isVolatile, *Alignment, Ordering, SSID,
+ IsElementwise);
return AteExtraComma ? InstExtraComma : InstNormal;
}
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index a5a11ab6221d4..0ec58c9e3a811 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -6442,19 +6442,22 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
break;
}
case bitc::FUNC_CODE_INST_LOADATOMIC: {
- // LOADATOMIC: [opty, op, align, vol, ordering, ssid]
+ // LOADATOMIC: [opty, op, align, vol, ordering, ssid, elementwise?]
unsigned OpNum = 0;
Value *Op;
unsigned OpTypeID;
if (getValueTypePair(Record, OpNum, NextValueNo, Op, OpTypeID, CurBB) ||
- (OpNum + 4 != Record.size() && OpNum + 5 != Record.size()))
+ (OpNum + 4 != Record.size() && OpNum + 5 != Record.size() &&
+ OpNum + 6 != Record.size()))
return error("Invalid load atomic record");
if (!isa<PointerType>(Op->getType()))
return error("Load operand is not a pointer type");
Type *Ty = nullptr;
- if (OpNum + 5 == Record.size()) {
+ bool HasExplicitType =
+ Record.size() == OpNum + 5 || Record.size() == OpNum + 6;
+ if (HasExplicitType) {
ResTypeID = Record[OpNum++];
Ty = getTypeByID(ResTypeID);
} else {
@@ -6476,13 +6479,15 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
if (Ordering != AtomicOrdering::NotAtomic && Record[OpNum] == 0)
return error("Invalid load atomic record");
SyncScope::ID SSID = getDecodedSyncScopeID(Record[OpNum + 3]);
+ bool IsElementwise = Record.size() > OpNum + 4 && Record[OpNum + 4];
MaybeAlign Align;
if (Error Err = parseAlignmentValue(Record[OpNum], Align))
return Err;
if (!Align)
return error("Alignment missing from atomic load");
- I = new LoadInst(Ty, Op, "", Record[OpNum + 1], *Align, Ordering, SSID);
+ I = new LoadInst(Ty, Op, "", Record[OpNum + 1], *Align, Ordering, SSID,
+ IsElementwise);
InstructionList.push_back(I);
break;
}
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index a6e7941a06699..62da7313f32d4 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -3583,6 +3583,8 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I,
if (cast<LoadInst>(I).isAtomic()) {
Vals.push_back(getEncodedOrdering(cast<LoadInst>(I).getOrdering()));
Vals.push_back(getEncodedSyncScopeID(cast<LoadInst>(I).getSyncScopeID()));
+ if (cast<LoadInst>(I).isElementwise())
+ Vals.push_back(1);
}
break;
case Instruction::Store:
diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp
index e90630a8cae59..40be6a072216e 100644
--- a/llvm/lib/IR/AsmWriter.cpp
+++ b/llvm/lib/IR/AsmWriter.cpp
@@ -4474,6 +4474,9 @@ void AssemblyWriter::printInstruction(const Instruction &I) {
(isa<AtomicRMWInst>(I) && cast<AtomicRMWInst>(I).isVolatile()))
Out << " volatile";
+ if (isa<LoadInst>(I) && cast<LoadInst>(I).isElementwise())
+ Out << " elementwise";
+
// Print out optimization information.
writeOptimizationInfo(Out, &I);
diff --git a/llvm/lib/IR/Instruction.cpp b/llvm/lib/IR/Instruction.cpp
index 112d81be3a2fa..5bade74b93c9b 100644
--- a/llvm/lib/IR/Instruction.cpp
+++ b/llvm/lib/IR/Instruction.cpp
@@ -927,6 +927,7 @@ bool Instruction::hasSameSpecialState(const Instruction *I2,
IgnoreAlignment);
if (const LoadInst *LI = dyn_cast<LoadInst>(I1))
return LI->isVolatile() == cast<LoadInst>(I2)->isVolatile() &&
+ LI->isElementwise() == cast<LoadInst>(I2)->isElementwise() &&
(LI->getAlign() == cast<LoadInst>(I2)->getAlign() ||
IgnoreAlignment) &&
LI->getOrdering() == cast<LoadInst>(I2)->getOrdering() &&
diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index ae04fb845112e..abf84a85dc918 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -1363,7 +1363,7 @@ LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
Align Align, InsertPosition InsertBef)
: LoadInst(Ty, Ptr, Name, isVolatile, Align, AtomicOrdering::NotAtomic,
- SyncScope::System, InsertBef) {}
+ SyncScope::System, /*IsElementwise=*/false, InsertBef) {}
LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name,
const LoadStoreInstProperties &Props,
@@ -1374,10 +1374,17 @@ LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name,
LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
Align Align, AtomicOrdering Order, SyncScope::ID SSID,
InsertPosition InsertBef)
+ : LoadInst(Ty, Ptr, Name, isVolatile, Align, Order, SSID,
+ /*IsElementwise=*/false, InsertBef) {}
+
+LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
+ Align Align, AtomicOrdering Order, SyncScope::ID SSID,
+ bool IsElementwise, InsertPosition InsertBef)
: UnaryInstruction(Ty, Load, Ptr, InsertBef) {
setVolatile(isVolatile);
setAlignment(Align);
setAtomic(Order, SSID);
+ setElementwise(IsElementwise);
AssertOK();
setName(Name);
}
@@ -4454,7 +4461,8 @@ AllocaInst *AllocaInst::cloneImpl() const {
LoadInst *LoadInst::cloneImpl() const {
return new LoadInst(getType(), getOperand(0), Twine(), isVolatile(),
- getAlign(), getOrdering(), getSyncScopeID());
+ getAlign(), getOrdering(), getSyncScopeID(),
+ isElementwise());
}
StoreInst *StoreInst::cloneImpl() const {
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 0744a91710b2e..0641e6e4b3dce 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -4539,15 +4539,29 @@ void Verifier::visitLoadInst(LoadInst &LI) {
Check(LI.getOrdering() != AtomicOrdering::Release &&
LI.getOrdering() != AtomicOrdering::AcquireRelease,
"Load cannot have Release ordering", &LI);
- Check(ElTy->getScalarType()->isIntOrPtrTy() ||
- ElTy->getScalarType()->isByteTy() ||
- ElTy->getScalarType()->isFloatingPointTy(),
+
+ Type *ScalarTy = ElTy;
+ if (LI.isElementwise()) {
+ auto *VecTy = dyn_cast<FixedVectorType>(ElTy);
+ Check(VecTy,
+ "atomic elementwise load operand must have fixed vector type!", &LI,
+ ElTy);
+ if (VecTy) {
+ checkAtomicMemAccessSize(ScalarTy, &LI);
+ ScalarTy = VecTy->getElementType();
+ }
+ }
+
+ Check(ScalarTy->getScalarType()->isIntOrPtrTy() ||
+ ScalarTy->getScalarType()->isByteTy() ||
+ ScalarTy->getScalarType()->isFloatingPointTy(),
"atomic load operand must have integer, byte, pointer, floating "
"point, or vector type!",
ElTy, &LI);
- checkAtomicMemAccessSize(ElTy, &LI);
+ checkAtomicMemAccessSize(ScalarTy, &LI);
} else {
+ Check(!LI.isElementwise(), "non-atomic load cannot be elementwise", &LI);
Check(LI.getSyncScopeID() == SyncScope::System,
"Non-atomic load cannot have SynchronizationScope specified", &LI);
}
diff --git a/llvm/test/Assembler/atomic.ll b/llvm/test/Assembler/atomic.ll
index 609cd33f61b88..afe4f48d3ac30 100644
--- a/llvm/test/Assembler/atomic.ll
+++ b/llvm/test/Assembler/atomic.ll
@@ -71,6 +71,11 @@ define void @f(ptr %x) {
; CHECK : store atomic <2 x float> <float 3.0, float 4.0>, ptr %x release, align 4
store atomic <2 x float> <float 3.0, float 4.0>, ptr %x release, align 4
+ ; CHECK: load atomic elementwise <2 x float>, ptr %x syncscope("agent") monotonic, align 4
+ load atomic elementwise <2 x float>, ptr %x syncscope("agent") monotonic, align 4
+ ; CHECK: load atomic volatile elementwise <2 x i32>, ptr %x monotonic, align 4
+ load atomic volatile elementwise <2 x i32>, ptr %x monotonic, align 4
+
; CHECK: fence syncscope("singlethread") release
fence syncscope("singlethread") release
; CHECK: fence seq_cst
diff --git a/llvm/test/Assembler/invalid-load-store-atomic-elementwise.ll b/llvm/test/Assembler/invalid-load-store-atomic-elementwise.ll
new file mode 100644
index 0000000000000..53d8571a9b461
--- /dev/null
+++ b/llvm/test/Assembler/invalid-load-store-atomic-elementwise.ll
@@ -0,0 +1,33 @@
+; RUN: split-file %s %t
+; RUN: not llvm-as -disable-output %t/load-non-atomic.ll 2>&1 | FileCheck %t/load-non-atomic.ll
+; RUN: not llvm-as -disable-output %t/load-scalar.ll 2>&1 | FileCheck %t/load-scalar.ll
+; RUN: not llvm-as -disable-output %t/load-scalable.ll 2>&1 | FileCheck %t/load-scalable.ll
+; RUN: not llvm-as -disable-output %t/load-odd-sized.ll 2>&1 | FileCheck %t/load-odd-sized.ll
+
+;--- load-non-atomic.ll
+; CHECK: elementwise load must be atomic
+define <2 x float> @bad_non_atomic(ptr %p) {
+ %v = load elementwise <2 x float>, ptr %p, align 4
+ ret <2 x float> %v
+}
+
+;--- load-scalar.ll
+; CHECK: atomic elementwise load operand must have fixed vector type
+define float @bad_scalar(ptr %p) {
+ %v = load atomic elementwise float, ptr %p monotonic, align 4
+ ret float %v
+}
+
+;--- load-scalable.ll
+; CHECK: atomic elementwise load operand must have fixed vector type
+define <vscale x 2 x i32> @bad_scalable(ptr %p) {
+ %v = load atomic elementwise <vscale x 2 x i32>, ptr %p monotonic, align 4
+ ret <vscale x 2 x i32> %v
+}
+
+;--- load-odd-sized.ll
+; CHECK: atomic memory access' operand must have a power-of-two size
+define <5 x i32> @bad_odd_sized_vector(ptr %p) {
+ %v = load atomic elementwise <5 x i32>, ptr %p monotonic, align 4
+ ret <5 x i32> %v
+}
diff --git a/llvm/test/Bitcode/atomic-load-store-elementwise.ll b/llvm/test/Bitcode/atomic-load-store-elementwise.ll
new file mode 100644
index 0000000000000..6b921c199c08c
--- /dev/null
+++ b/llvm/test/Bitcode/atomic-load-store-elementwise.ll
@@ -0,0 +1,16 @@
+; RUN: llvm-as %s -o - | llvm-dis | FileCheck %s
+; RUN: llvm-as %s -o - | verify-uselistorder
+
+define <2 x float> @load_elem_f32(ptr %p) {
+; CHECK-LABEL: @load_elem_f32(
+; CHECK: %v = load atomic elementwise <2 x float>, ptr %p syncscope("agent") monotonic, align 4
+ %v = load atomic elementwise <2 x float>, ptr %p syncscope("agent") monotonic, align 4
+ ret <2 x float> %v
+}
+
+define <4 x i32> @load_elem_i32_volatile(ptr %p) {
+; CHECK-LABEL: @load_elem_i32_volatile(
+; CHECK: %v = load atomic volatile elementwise <4 x i32>, ptr %p seq_cst, align 4
+ %v = load atomic volatile elementwise <4 x i32>, ptr %p seq_cst, align 4
+ ret <4 x i32> %v
+}
>From 53a802b2bd3f8c5536bcc76db70859edefd75ebd Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Fri, 26 Jun 2026 17:01:55 +0800
Subject: [PATCH 2/8] Update for comments.
---
llvm/docs/LangRef.md | 12 +++++------
llvm/include/llvm/IR/Instructions.h | 9 ++++----
llvm/lib/AsmParser/LLParser.cpp | 6 +++++-
llvm/lib/Bitcode/Reader/BitcodeReader.cpp | 2 +-
llvm/lib/Bitcode/Writer/BitcodeWriter.cpp | 25 +++++++++++++----------
llvm/lib/IR/Instructions.cpp | 12 +++--------
llvm/test/Bitcode/compatibility.ll | 6 ++++++
7 files changed, 39 insertions(+), 33 deletions(-)
diff --git a/llvm/docs/LangRef.md b/llvm/docs/LangRef.md
index bcee67a3b5e7d..415f8e3fa664c 100644
--- a/llvm/docs/LangRef.md
+++ b/llvm/docs/LangRef.md
@@ -11743,13 +11743,13 @@ into one scalar atomic load per element, and the element loads are not ordered
with respect to each other. Without `elementwise`, vector atomic loads keep
whole-value atomic semantics.
-`align` must be explicitly specified on atomic loads. Note: if the alignment
-is not greater than or equal to the size of the `<ty>` type, or the element
-type for an `elementwise` load, the atomic operation is likely to require a
-lock and have poor performance. `!nontemporal` does not have any defined
-semantics for atomic loads.
+`align` must be explicitly specified on atomic loads, and is otherwise
+optional on non-atomic loads. Note: if the alignment is not greater than or equal
+to the size of the `<ty>` type, or the element type for an `elementwise` load,
+the atomic operation is likely to require a lock and have poor performance.
+`!nontemporal` does not have any defined semantics for atomic loads.
-The optional constant `align` argument specifies the alignment of the
+The constant `align` argument specifies the alignment of the
operation (that is, the alignment of the memory address). It is the
responsibility of the code emitter to ensure that the alignment information is
correct. Overestimating the alignment results in undefined behavior.
diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h
index a56b82dc2478c..cffec9d60b0f8 100644
--- a/llvm/include/llvm/IR/Instructions.h
+++ b/llvm/include/llvm/IR/Instructions.h
@@ -182,11 +182,12 @@ struct LoadStoreInstProperties {
Align Alignment;
AtomicOrdering Ordering;
SyncScope::ID SSID;
+ bool IsElementwise;
};
/// An instruction for reading from memory. This uses the SubclassData field in
/// Value to store whether or not the load is volatile.
-class LoadInst : public UnaryInstruction {
+class LoadInst : public UnaryInstruction {
using VolatileField = BoolBitfieldElementT<0>;
using AlignmentField = AlignmentBitfieldElementT<VolatileField::NextBit>;
using OrderingField = AtomicOrderingBitfieldElementT<AlignmentField::NextBit>;
@@ -210,9 +211,6 @@ class LoadInst : public UnaryInstruction {
InsertPosition InsertBefore);
LLVM_ABI LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
Align Align, InsertPosition InsertBefore = nullptr);
- LLVM_ABI LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
- Align Align, AtomicOrdering Order, SyncScope::ID SSID,
- InsertPosition InsertBefore);
LLVM_ABI LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
Align Align, AtomicOrdering Order,
SyncScope::ID SSID = SyncScope::System,
@@ -273,7 +271,7 @@ class LoadInst : public UnaryInstruction {
/// Returns the properties of this load instruction.
LoadStoreInstProperties getProperties() const {
- return {isVolatile(), getAlign(), getOrdering(), getSyncScopeID()};
+ return {isVolatile(), getAlign(), getOrdering(), getSyncScopeID(), isElementwise()};
}
/// Sets the properties of this load instruction.
@@ -282,6 +280,7 @@ class LoadInst : public UnaryInstruction {
setAlignment(Props.Alignment);
setOrdering(Props.Ordering);
setSyncScopeID(Props.SSID);
+ setElementwise(Props.IsElementwise);
}
bool isSimple() const { return !isAtomic() && !isVolatile(); }
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index c38214d8d1b5d..f4e17d8196fc0 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -9004,6 +9004,10 @@ int LLParser::parseLoad(Instruction *&Inst, PerFunctionState &PFS) {
if (IsElementwise && !isAtomic)
return error(Loc, "elementwise load must be atomic");
+ if (IsElementwise && !isa<FixedVectorType>(Ty))
+ return error(ExplicitTypeLoc,
+ "atomic elementwise load operand must have fixed vector type");
+
if (isAtomic && !Alignment)
return error(Loc, "atomic load must have explicit non-zero alignment");
@@ -9017,7 +9021,7 @@ int LLParser::parseLoad(Instruction *&Inst, PerFunctionState &PFS) {
if (!Alignment)
Alignment = M->getDataLayout().getABITypeAlign(Ty);
Inst = new LoadInst(Ty, Val, "", isVolatile, *Alignment, Ordering, SSID,
- IsElementwise);
+ /*InsertBefore=*/nullptr, IsElementwise);
return AteExtraComma ? InstExtraComma : InstNormal;
}
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index 0ec58c9e3a811..1529df6c71e5d 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -6487,7 +6487,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
if (!Align)
return error("Alignment missing from atomic load");
I = new LoadInst(Ty, Op, "", Record[OpNum + 1], *Align, Ordering, SSID,
- IsElementwise);
+ /*InsertBefore=*/nullptr, IsElementwise);
InstructionList.push_back(I);
break;
}
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index 62da7313f32d4..0b4351fd8b49b 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -3568,25 +3568,28 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I,
break;
}
- case Instruction::Load:
- if (cast<LoadInst>(I).isAtomic()) {
+ case Instruction::Load: {
+ const auto &LI = cast<LoadInst>(I);
+ if (LI.isAtomic()) {
Code = bitc::FUNC_CODE_INST_LOADATOMIC;
- pushValueAndType(I.getOperand(0), InstID, Vals);
+ pushValueAndType(LI.getOperand(0), InstID, Vals);
} else {
Code = bitc::FUNC_CODE_INST_LOAD;
- if (!pushValueAndType(I.getOperand(0), InstID, Vals)) // ptr
+ if (!pushValueAndType(LI.getOperand(0), InstID, Vals)) // ptr
AbbrevToUse = FUNCTION_INST_LOAD_ABBREV;
}
- Vals.push_back(VE.getTypeID(I.getType()));
- Vals.push_back(getEncodedAlign(cast<LoadInst>(I).getAlign()));
- Vals.push_back(cast<LoadInst>(I).isVolatile());
- if (cast<LoadInst>(I).isAtomic()) {
- Vals.push_back(getEncodedOrdering(cast<LoadInst>(I).getOrdering()));
- Vals.push_back(getEncodedSyncScopeID(cast<LoadInst>(I).getSyncScopeID()));
- if (cast<LoadInst>(I).isElementwise())
+ Vals.push_back(VE.getTypeID(LI.getType()));
+ Vals.push_back(getEncodedAlign(LI.getAlign()));
+ Vals.push_back(LI.isVolatile());
+ if (LI.isAtomic()) {
+ Vals.push_back(getEncodedOrdering(LI.getOrdering()));
+ Vals.push_back(getEncodedSyncScopeID(LI.getSyncScopeID()));
+ if (LI.isElementwise())
Vals.push_back(1);
}
break;
+ }
+
case Instruction::Store:
if (cast<StoreInst>(I).isAtomic()) {
Code = bitc::FUNC_CODE_INST_STOREATOMIC;
diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index abf84a85dc918..3fb8d0ae41911 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -1363,7 +1363,7 @@ LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
Align Align, InsertPosition InsertBef)
: LoadInst(Ty, Ptr, Name, isVolatile, Align, AtomicOrdering::NotAtomic,
- SyncScope::System, /*IsElementwise=*/false, InsertBef) {}
+ SyncScope::System, InsertBef, /*IsElementwise=*/false) {}
LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name,
const LoadStoreInstProperties &Props,
@@ -1373,13 +1373,7 @@ LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name,
LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
Align Align, AtomicOrdering Order, SyncScope::ID SSID,
- InsertPosition InsertBef)
- : LoadInst(Ty, Ptr, Name, isVolatile, Align, Order, SSID,
- /*IsElementwise=*/false, InsertBef) {}
-
-LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
- Align Align, AtomicOrdering Order, SyncScope::ID SSID,
- bool IsElementwise, InsertPosition InsertBef)
+ InsertPosition InsertBef, bool IsElementwise)
: UnaryInstruction(Ty, Load, Ptr, InsertBef) {
setVolatile(isVolatile);
setAlignment(Align);
@@ -4462,7 +4456,7 @@ AllocaInst *AllocaInst::cloneImpl() const {
LoadInst *LoadInst::cloneImpl() const {
return new LoadInst(getType(), getOperand(0), Twine(), isVolatile(),
getAlign(), getOrdering(), getSyncScopeID(),
- isElementwise());
+ /*InsertBefore=*/nullptr, isElementwise());
}
StoreInst *StoreInst::cloneImpl() const {
diff --git a/llvm/test/Bitcode/compatibility.ll b/llvm/test/Bitcode/compatibility.ll
index 0ac6da0c9ed29..cdedb97b68813 100644
--- a/llvm/test/Bitcode/compatibility.ll
+++ b/llvm/test/Bitcode/compatibility.ll
@@ -1036,6 +1036,12 @@ define void @elementwise_atomics(ptr %word, <4 x i32> %ival, <4 x float> %fval)
; CHECK: %atomicrmw.fadd = atomicrmw elementwise fadd ptr %word, <4 x float> %fval seq_cst, align 16
%atomicrmw.fadd = atomicrmw elementwise fadd ptr %word, <4 x float> %fval seq_cst, align 16
+; CHECK: %load.elementwise = load atomic elementwise <4 x i32>, ptr %word monotonic, align 4
+ %load.elementwise = load atomic elementwise <4 x i32>, ptr %word monotonic, align 4
+
+; CHECK: %load.elementwise.volatile = load atomic volatile elementwise <4 x float>, ptr %word seq_cst, align 4
+ %load.elementwise.volatile = load atomic volatile elementwise <4 x float>, ptr %word seq_cst, align 4
+
ret void
}
>From 76b9f49eb56d3b7e9357167efd5ef7b44b0eaeb5 Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Fri, 26 Jun 2026 17:34:38 +0800
Subject: [PATCH 3/8] Add verifier test
---
llvm/unittests/IR/VerifierTest.cpp | 74 ++++++++++++++++++++++++++++++
1 file changed, 74 insertions(+)
diff --git a/llvm/unittests/IR/VerifierTest.cpp b/llvm/unittests/IR/VerifierTest.cpp
index d680dd250d66e..bd47c5117d021 100644
--- a/llvm/unittests/IR/VerifierTest.cpp
+++ b/llvm/unittests/IR/VerifierTest.cpp
@@ -521,6 +521,7 @@ TEST(VerifierTest, AtomicRMWIntVector) {
new AtomicRMWInst(AtomicRMWInst::Add, Ptr, CV, Align(8),
AtomicOrdering::SequentiallyConsistent, SyncScope::System,
/*Elementwise=*/false, Entry);
+
ReturnInst::Create(C, Entry);
std::string Error;
@@ -529,6 +530,79 @@ TEST(VerifierTest, AtomicRMWIntVector) {
EXPECT_TRUE(
StringRef(Error).starts_with("atomicrmw add operand must have integer or "
"fixed vector of integer type!"))
+}
+
+TEST(VerifierTest, ElementwiseLoadNonAtomic) {
+ LLVMContext C;
+ Module M("M", C);
+ FunctionType *FTy = FunctionType::get(Type::getVoidTy(C), /*isVarArg=*/false);
+ Function *F = Function::Create(FTy, Function::ExternalLinkage, "foo", M);
+ BasicBlock *Entry = BasicBlock::Create(C, "entry", F);
+ Value *Ptr = PoisonValue::get(PointerType::get(C, 0));
+
+ Type *I32Ty = Type::getInt32Ty(C);
+ Type *VecTy = FixedVectorType::get(I32Ty, 4);
+
+ new LoadInst(VecTy, Ptr, "",
+ LoadStoreInstProperties{/*IsVolatile=*/false, Align(4),
+ AtomicOrdering::NotAtomic,
+ SyncScope::System,
+ /*IsElementwise=*/true},
+ Entry);
+ ReturnInst::Create(C, Entry);
+
+ std::string Error;
+ raw_string_ostream ErrorOS(Error);
+ EXPECT_TRUE(verifyFunction(*F, &ErrorOS));
+ EXPECT_TRUE(
+ StringRef(Error).starts_with("non-atomic load cannot be elementwise"))
+ << Error;
+}
+
+TEST(VerifierTest, ElementwiseLoadScalar) {
+ LLVMContext C;
+ Module M("M", C);
+ FunctionType *FTy = FunctionType::get(Type::getVoidTy(C), /*isVarArg=*/false);
+ Function *F = Function::Create(FTy, Function::ExternalLinkage, "foo", M);
+ BasicBlock *Entry = BasicBlock::Create(C, "entry", F);
+ Value *Ptr = PoisonValue::get(PointerType::get(C, 0));
+
+ Type *I32Ty = Type::getInt32Ty(C);
+
+ new LoadInst(I32Ty, Ptr, "", /*isVolatile=*/false, Align(4),
+ AtomicOrdering::Monotonic, SyncScope::System, Entry,
+ /*IsElementwise=*/true);
+ ReturnInst::Create(C, Entry);
+
+ std::string Error;
+ raw_string_ostream ErrorOS(Error);
+ EXPECT_TRUE(verifyFunction(*F, &ErrorOS));
+ EXPECT_TRUE(StringRef(Error).starts_with(
+ "atomic elementwise load operand must have fixed vector type!"))
+ << Error;
+}
+
+TEST(VerifierTest, ElementwiseLoadOddSizedVector) {
+ LLVMContext C;
+ Module M("M", C);
+ FunctionType *FTy = FunctionType::get(Type::getVoidTy(C), /*isVarArg=*/false);
+ Function *F = Function::Create(FTy, Function::ExternalLinkage, "foo", M);
+ BasicBlock *Entry = BasicBlock::Create(C, "entry", F);
+ Value *Ptr = PoisonValue::get(PointerType::get(C, 0));
+
+ Type *I32Ty = Type::getInt32Ty(C);
+ Type *VecTy = FixedVectorType::get(I32Ty, 5);
+
+ new LoadInst(VecTy, Ptr, "", /*isVolatile=*/false, Align(4),
+ AtomicOrdering::Monotonic, SyncScope::System, Entry,
+ /*IsElementwise=*/true);
+ ReturnInst::Create(C, Entry);
+
+ std::string Error;
+ raw_string_ostream ErrorOS(Error);
+ EXPECT_TRUE(verifyFunction(*F, &ErrorOS));
+ EXPECT_TRUE(StringRef(Error).starts_with(
+ "atomic memory access' operand must have a power-of-two size"))
<< Error;
}
>From 57607d8735ddbc8ca33c2177f70811c787575efb Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Mon, 29 Jun 2026 17:03:31 +0800
Subject: [PATCH 4/8] Update for comments
---
llvm/docs/LangRef.md | 17 ++++++++++-------
llvm/lib/Bitcode/Reader/BitcodeReader.cpp | 4 +---
.../invalid-load-store-atomic-elementwise.ll | 8 ++++++++
3 files changed, 19 insertions(+), 10 deletions(-)
diff --git a/llvm/docs/LangRef.md b/llvm/docs/LangRef.md
index 415f8e3fa664c..4ba2462470b10 100644
--- a/llvm/docs/LangRef.md
+++ b/llvm/docs/LangRef.md
@@ -11732,15 +11732,18 @@ modify the number or order of execution of this `load` with other
If the `load` is marked as `atomic`, it takes an extra {ref}`ordering <ordering>` and optional `syncscope("<target-scope>")` argument. The
`release` and `acq_rel` orderings are not valid on `load` instructions.
Atomic loads produce {ref}`defined <memmodel>` results when they may see
-multiple atomic stores. The type of the pointee must be an integer, pointer,
-floating-point, or vector type whose bit width is a power of two greater than
-or equal to eight.
+multiple atomic stores. Atomic loads support scalar integer, pointer, and
+floating-point types whose bit width is a power of two greater than or equal
+to eight. Atomic loads also support vector types whose total bit width is a
+power of two greater than or equal to eight. That is, the entire vector is
+loaded atomically.
If the `elementwise` modifier is present, the loaded type must be a fixed
-vector type, and each element type must be a valid scalar atomic load type. The
-load has per-element vector atomic semantics: it behaves as if it were expanded
-into one scalar atomic load per element, and the element loads are not ordered
-with respect to each other. Without `elementwise`, vector atomic loads keep
+vector type whose total bit width is a power of two greater than or equal to
+eight, and whose element type is supported by scalar atomic loads. The load has
+per-element atomic load semantics: it behaves as if it were expanded into
+one scalar atomic load per element, and the element loads are not ordered with
+respect to each other. Without `elementwise`, vector atomic loads keep
whole-value atomic semantics.
`align` must be explicitly specified on atomic loads, and is otherwise
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index 1529df6c71e5d..f6e71420e59c2 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -6455,9 +6455,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
return error("Load operand is not a pointer type");
Type *Ty = nullptr;
- bool HasExplicitType =
- Record.size() == OpNum + 5 || Record.size() == OpNum + 6;
- if (HasExplicitType) {
+ if (Record.size() >= OpNum + 5) {
ResTypeID = Record[OpNum++];
Ty = getTypeByID(ResTypeID);
} else {
diff --git a/llvm/test/Assembler/invalid-load-store-atomic-elementwise.ll b/llvm/test/Assembler/invalid-load-store-atomic-elementwise.ll
index 53d8571a9b461..03dbc96cb562e 100644
--- a/llvm/test/Assembler/invalid-load-store-atomic-elementwise.ll
+++ b/llvm/test/Assembler/invalid-load-store-atomic-elementwise.ll
@@ -3,6 +3,7 @@
; RUN: not llvm-as -disable-output %t/load-scalar.ll 2>&1 | FileCheck %t/load-scalar.ll
; RUN: not llvm-as -disable-output %t/load-scalable.ll 2>&1 | FileCheck %t/load-scalable.ll
; RUN: not llvm-as -disable-output %t/load-odd-sized.ll 2>&1 | FileCheck %t/load-odd-sized.ll
+; RUN: not llvm-as -disable-output %t/load-non-byte.ll 2>&1 | FileCheck %t/load-non-byte.ll
;--- load-non-atomic.ll
; CHECK: elementwise load must be atomic
@@ -31,3 +32,10 @@ define <5 x i32> @bad_odd_sized_vector(ptr %p) {
%v = load atomic elementwise <5 x i32>, ptr %p monotonic, align 4
ret <5 x i32> %v
}
+
+;--- load-non-byte.ll
+; CHECK: atomic memory access' size must be byte-sized
+define <4 x i1> @bad_non_byte(ptr %p) {
+ %v = load atomic elementwise <4 x i1>, ptr %p monotonic, align 4
+ ret <4 x i1> %v
+}
>From e6c5e25191a87fcacb947764bca65f8cfe730184 Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Tue, 30 Jun 2026 17:45:01 +0800
Subject: [PATCH 5/8] Add element non byte
---
.../Assembler/invalid-load-store-atomic-elementwise.ll | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/llvm/test/Assembler/invalid-load-store-atomic-elementwise.ll b/llvm/test/Assembler/invalid-load-store-atomic-elementwise.ll
index 03dbc96cb562e..843564cbbc075 100644
--- a/llvm/test/Assembler/invalid-load-store-atomic-elementwise.ll
+++ b/llvm/test/Assembler/invalid-load-store-atomic-elementwise.ll
@@ -4,6 +4,7 @@
; RUN: not llvm-as -disable-output %t/load-scalable.ll 2>&1 | FileCheck %t/load-scalable.ll
; RUN: not llvm-as -disable-output %t/load-odd-sized.ll 2>&1 | FileCheck %t/load-odd-sized.ll
; RUN: not llvm-as -disable-output %t/load-non-byte.ll 2>&1 | FileCheck %t/load-non-byte.ll
+; RUN: not llvm-as -disable-output %t/load-non-byte-element.ll 2>&1 | FileCheck %t/load-non-byte-element.ll
;--- load-non-atomic.ll
; CHECK: elementwise load must be atomic
@@ -39,3 +40,10 @@ define <4 x i1> @bad_non_byte(ptr %p) {
%v = load atomic elementwise <4 x i1>, ptr %p monotonic, align 4
ret <4 x i1> %v
}
+
+;--- load-non-byte-element.ll
+; CHECK: atomic memory access' size must be byte-sized
+define <8 x i1> @bad_non_byte_element(ptr %p) {
+ %v = load atomic elementwise <8 x i1>, ptr %p monotonic, align 1
+ ret <8 x i1> %v
+}
>From 000069a0346c4e3d3b3593f793eb781e49c495ae Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Fri, 3 Jul 2026 14:29:54 +0800
Subject: [PATCH 6/8] Update doc
---
llvm/docs/LangRef.md | 10 ++++------
llvm/include/llvm/IR/Instructions.h | 9 ++++++---
llvm/lib/AsmParser/LLParser.cpp | 3 ++-
llvm/lib/Bitcode/Reader/BitcodeReader.cpp | 3 ++-
llvm/lib/IR/Instructions.cpp | 15 +++++++++++----
llvm/unittests/IR/VerifierTest.cpp | 17 +++++++----------
6 files changed, 32 insertions(+), 25 deletions(-)
diff --git a/llvm/docs/LangRef.md b/llvm/docs/LangRef.md
index 4ba2462470b10..c9fba9820023e 100644
--- a/llvm/docs/LangRef.md
+++ b/llvm/docs/LangRef.md
@@ -11732,11 +11732,9 @@ modify the number or order of execution of this `load` with other
If the `load` is marked as `atomic`, it takes an extra {ref}`ordering <ordering>` and optional `syncscope("<target-scope>")` argument. The
`release` and `acq_rel` orderings are not valid on `load` instructions.
Atomic loads produce {ref}`defined <memmodel>` results when they may see
-multiple atomic stores. Atomic loads support scalar integer, pointer, and
-floating-point types whose bit width is a power of two greater than or equal
-to eight. Atomic loads also support vector types whose total bit width is a
-power of two greater than or equal to eight. That is, the entire vector is
-loaded atomically.
+multiple atomic stores. The type of the pointee must be an integer, pointer,
+floating-point, or vector type whose bit width is a power of two greater than or
+equal to eight.
If the `elementwise` modifier is present, the loaded type must be a fixed
vector type whose total bit width is a power of two greater than or equal to
@@ -11744,7 +11742,7 @@ eight, and whose element type is supported by scalar atomic loads. The load has
per-element atomic load semantics: it behaves as if it were expanded into
one scalar atomic load per element, and the element loads are not ordered with
respect to each other. Without `elementwise`, vector atomic loads keep
-whole-value atomic semantics.
+whole-value atomic semantics. That is, the entire vector is loaded atomically.
`align` must be explicitly specified on atomic loads, and is otherwise
optional on non-atomic loads. Note: if the alignment is not greater than or equal
diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h
index cffec9d60b0f8..9378d244d3ad0 100644
--- a/llvm/include/llvm/IR/Instructions.h
+++ b/llvm/include/llvm/IR/Instructions.h
@@ -182,7 +182,8 @@ struct LoadStoreInstProperties {
Align Alignment;
AtomicOrdering Ordering;
SyncScope::ID SSID;
- bool IsElementwise;
+ // TODO: Add IsElementwise here once StoreInst also supports the elementwise
+ // modifier.
};
/// An instruction for reading from memory. This uses the SubclassData field in
@@ -211,6 +212,9 @@ class LoadInst : public UnaryInstruction {
InsertPosition InsertBefore);
LLVM_ABI LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
Align Align, InsertPosition InsertBefore = nullptr);
+ LLVM_ABI LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
+ Align Align, AtomicOrdering Order, SyncScope::ID SSID,
+ InsertPosition InsertBefore);
LLVM_ABI LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
Align Align, AtomicOrdering Order,
SyncScope::ID SSID = SyncScope::System,
@@ -271,7 +275,7 @@ class LoadInst : public UnaryInstruction {
/// Returns the properties of this load instruction.
LoadStoreInstProperties getProperties() const {
- return {isVolatile(), getAlign(), getOrdering(), getSyncScopeID(), isElementwise()};
+ return {isVolatile(), getAlign(), getOrdering(), getSyncScopeID()};
}
/// Sets the properties of this load instruction.
@@ -280,7 +284,6 @@ class LoadInst : public UnaryInstruction {
setAlignment(Props.Alignment);
setOrdering(Props.Ordering);
setSyncScopeID(Props.SSID);
- setElementwise(Props.IsElementwise);
}
bool isSimple() const { return !isAtomic() && !isVolatile(); }
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index f4e17d8196fc0..bb5c42962b8c8 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -9021,7 +9021,8 @@ int LLParser::parseLoad(Instruction *&Inst, PerFunctionState &PFS) {
if (!Alignment)
Alignment = M->getDataLayout().getABITypeAlign(Ty);
Inst = new LoadInst(Ty, Val, "", isVolatile, *Alignment, Ordering, SSID,
- /*InsertBefore=*/nullptr, IsElementwise);
+ IsElementwise,
+ /*InsertBefore=*/nullptr);
return AteExtraComma ? InstExtraComma : InstNormal;
}
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index f6e71420e59c2..11ff56134502c 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -6485,7 +6485,8 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
if (!Align)
return error("Alignment missing from atomic load");
I = new LoadInst(Ty, Op, "", Record[OpNum + 1], *Align, Ordering, SSID,
- /*InsertBefore=*/nullptr, IsElementwise);
+ IsElementwise,
+ /*InsertBefore=*/nullptr);
InstructionList.push_back(I);
break;
}
diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index 3fb8d0ae41911..73957c01fc392 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -1363,17 +1363,23 @@ LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
Align Align, InsertPosition InsertBef)
: LoadInst(Ty, Ptr, Name, isVolatile, Align, AtomicOrdering::NotAtomic,
- SyncScope::System, InsertBef, /*IsElementwise=*/false) {}
+ SyncScope::System, /*IsElementwise=*/false, InsertBef) {}
LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name,
const LoadStoreInstProperties &Props,
InsertPosition InsertBef)
: LoadInst(Ty, Ptr, Name, Props.IsVolatile, Props.Alignment, Props.Ordering,
- Props.SSID, InsertBef) {}
+ Props.SSID, /*IsElementwise=*/false, InsertBef) {}
LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
Align Align, AtomicOrdering Order, SyncScope::ID SSID,
- InsertPosition InsertBef, bool IsElementwise)
+ InsertPosition InsertBef)
+ : LoadInst(Ty, Ptr, Name, isVolatile, Align, Order, SSID,
+ /*IsElementwise=*/false, InsertBef) {}
+
+LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
+ Align Align, AtomicOrdering Order, SyncScope::ID SSID,
+ bool IsElementwise, InsertPosition InsertBef)
: UnaryInstruction(Ty, Load, Ptr, InsertBef) {
setVolatile(isVolatile);
setAlignment(Align);
@@ -4456,7 +4462,8 @@ AllocaInst *AllocaInst::cloneImpl() const {
LoadInst *LoadInst::cloneImpl() const {
return new LoadInst(getType(), getOperand(0), Twine(), isVolatile(),
getAlign(), getOrdering(), getSyncScopeID(),
- /*InsertBefore=*/nullptr, isElementwise());
+ isElementwise(),
+ /*InsertBefore=*/nullptr);
}
StoreInst *StoreInst::cloneImpl() const {
diff --git a/llvm/unittests/IR/VerifierTest.cpp b/llvm/unittests/IR/VerifierTest.cpp
index bd47c5117d021..f3753aa55dee8 100644
--- a/llvm/unittests/IR/VerifierTest.cpp
+++ b/llvm/unittests/IR/VerifierTest.cpp
@@ -543,12 +543,9 @@ TEST(VerifierTest, ElementwiseLoadNonAtomic) {
Type *I32Ty = Type::getInt32Ty(C);
Type *VecTy = FixedVectorType::get(I32Ty, 4);
- new LoadInst(VecTy, Ptr, "",
- LoadStoreInstProperties{/*IsVolatile=*/false, Align(4),
- AtomicOrdering::NotAtomic,
- SyncScope::System,
- /*IsElementwise=*/true},
- Entry);
+ new LoadInst(VecTy, Ptr, "", /*isVolatile=*/false, Align(4),
+ AtomicOrdering::NotAtomic, SyncScope::System,
+ /*IsElementwise=*/true, Entry);
ReturnInst::Create(C, Entry);
std::string Error;
@@ -570,8 +567,8 @@ TEST(VerifierTest, ElementwiseLoadScalar) {
Type *I32Ty = Type::getInt32Ty(C);
new LoadInst(I32Ty, Ptr, "", /*isVolatile=*/false, Align(4),
- AtomicOrdering::Monotonic, SyncScope::System, Entry,
- /*IsElementwise=*/true);
+ AtomicOrdering::Monotonic, SyncScope::System,
+ /*IsElementwise=*/true, Entry);
ReturnInst::Create(C, Entry);
std::string Error;
@@ -594,8 +591,8 @@ TEST(VerifierTest, ElementwiseLoadOddSizedVector) {
Type *VecTy = FixedVectorType::get(I32Ty, 5);
new LoadInst(VecTy, Ptr, "", /*isVolatile=*/false, Align(4),
- AtomicOrdering::Monotonic, SyncScope::System, Entry,
- /*IsElementwise=*/true);
+ AtomicOrdering::Monotonic, SyncScope::System,
+ /*IsElementwise=*/true, Entry);
ReturnInst::Create(C, Entry);
std::string Error;
>From d045320cf67b5dd7cee94475097c763c652d5860 Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Thu, 9 Jul 2026 09:40:39 +0800
Subject: [PATCH 7/8] Fix Typo
---
llvm/include/llvm/IR/Instructions.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h
index 9378d244d3ad0..2ea82afff22ba 100644
--- a/llvm/include/llvm/IR/Instructions.h
+++ b/llvm/include/llvm/IR/Instructions.h
@@ -188,7 +188,7 @@ struct LoadStoreInstProperties {
/// An instruction for reading from memory. This uses the SubclassData field in
/// Value to store whether or not the load is volatile.
-class LoadInst : public UnaryInstruction {
+class LoadInst : public UnaryInstruction {
using VolatileField = BoolBitfieldElementT<0>;
using AlignmentField = AlignmentBitfieldElementT<VolatileField::NextBit>;
using OrderingField = AtomicOrderingBitfieldElementT<AlignmentField::NextBit>;
>From 057002574058d39a85df9f41329251470ad07f9d Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Thu, 9 Jul 2026 22:24:19 +0800
Subject: [PATCH 8/8] Update for comment
---
llvm/include/llvm/IR/Instructions.h | 11 ++++-----
llvm/lib/AsmParser/LLParser.cpp | 5 +++--
llvm/lib/Bitcode/Reader/BitcodeReader.cpp | 8 ++++---
llvm/lib/IR/Instructions.cpp | 17 +++++---------
llvm/unittests/IR/VerifierTest.cpp | 27 +++++++++++++++--------
5 files changed, 35 insertions(+), 33 deletions(-)
diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h
index 2ea82afff22ba..5838109847845 100644
--- a/llvm/include/llvm/IR/Instructions.h
+++ b/llvm/include/llvm/IR/Instructions.h
@@ -182,8 +182,7 @@ struct LoadStoreInstProperties {
Align Alignment;
AtomicOrdering Ordering;
SyncScope::ID SSID;
- // TODO: Add IsElementwise here once StoreInst also supports the elementwise
- // modifier.
+ bool IsElementwise = false;
};
/// An instruction for reading from memory. This uses the SubclassData field in
@@ -212,13 +211,9 @@ class LoadInst : public UnaryInstruction {
InsertPosition InsertBefore);
LLVM_ABI LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
Align Align, InsertPosition InsertBefore = nullptr);
- LLVM_ABI LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
- Align Align, AtomicOrdering Order, SyncScope::ID SSID,
- InsertPosition InsertBefore);
LLVM_ABI LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
Align Align, AtomicOrdering Order,
SyncScope::ID SSID = SyncScope::System,
- bool IsElementwise = false,
InsertPosition InsertBefore = nullptr);
LLVM_ABI LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr,
const LoadStoreInstProperties &Props,
@@ -275,7 +270,8 @@ class LoadInst : public UnaryInstruction {
/// Returns the properties of this load instruction.
LoadStoreInstProperties getProperties() const {
- return {isVolatile(), getAlign(), getOrdering(), getSyncScopeID()};
+ return {isVolatile(), getAlign(), getOrdering(), getSyncScopeID(),
+ isElementwise()};
}
/// Sets the properties of this load instruction.
@@ -284,6 +280,7 @@ class LoadInst : public UnaryInstruction {
setAlignment(Props.Alignment);
setOrdering(Props.Ordering);
setSyncScopeID(Props.SSID);
+ setElementwise(Props.IsElementwise);
}
bool isSimple() const { return !isAtomic() && !isVolatile(); }
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index bb5c42962b8c8..c58b109b5ff9b 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -9020,8 +9020,9 @@ int LLParser::parseLoad(Instruction *&Inst, PerFunctionState &PFS) {
return error(ExplicitTypeLoc, "loading unsized types is not allowed");
if (!Alignment)
Alignment = M->getDataLayout().getABITypeAlign(Ty);
- Inst = new LoadInst(Ty, Val, "", isVolatile, *Alignment, Ordering, SSID,
- IsElementwise,
+ Inst = new LoadInst(Ty, Val, "",
+ LoadStoreInstProperties{isVolatile, *Alignment, Ordering,
+ SSID, IsElementwise},
/*InsertBefore=*/nullptr);
return AteExtraComma ? InstExtraComma : InstNormal;
}
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index 11ff56134502c..bb8772e22621f 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -6484,9 +6484,11 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
return Err;
if (!Align)
return error("Alignment missing from atomic load");
- I = new LoadInst(Ty, Op, "", Record[OpNum + 1], *Align, Ordering, SSID,
- IsElementwise,
- /*InsertBefore=*/nullptr);
+ I = new LoadInst(
+ Ty, Op, "",
+ LoadStoreInstProperties{/*IsVolatile=*/Record[OpNum + 1] != 0, *Align,
+ Ordering, SSID, IsElementwise},
+ /*InsertBefore=*/nullptr);
InstructionList.push_back(I);
break;
}
diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index 73957c01fc392..c8f2da73e78b2 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -1363,28 +1363,23 @@ LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
Align Align, InsertPosition InsertBef)
: LoadInst(Ty, Ptr, Name, isVolatile, Align, AtomicOrdering::NotAtomic,
- SyncScope::System, /*IsElementwise=*/false, InsertBef) {}
+ SyncScope::System, InsertBef) {}
LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name,
const LoadStoreInstProperties &Props,
InsertPosition InsertBef)
: LoadInst(Ty, Ptr, Name, Props.IsVolatile, Props.Alignment, Props.Ordering,
- Props.SSID, /*IsElementwise=*/false, InsertBef) {}
+ Props.SSID, InsertBef) {
+ setElementwise(Props.IsElementwise);
+}
LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
Align Align, AtomicOrdering Order, SyncScope::ID SSID,
InsertPosition InsertBef)
- : LoadInst(Ty, Ptr, Name, isVolatile, Align, Order, SSID,
- /*IsElementwise=*/false, InsertBef) {}
-
-LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
- Align Align, AtomicOrdering Order, SyncScope::ID SSID,
- bool IsElementwise, InsertPosition InsertBef)
: UnaryInstruction(Ty, Load, Ptr, InsertBef) {
setVolatile(isVolatile);
setAlignment(Align);
setAtomic(Order, SSID);
- setElementwise(IsElementwise);
AssertOK();
setName(Name);
}
@@ -4460,9 +4455,7 @@ AllocaInst *AllocaInst::cloneImpl() const {
}
LoadInst *LoadInst::cloneImpl() const {
- return new LoadInst(getType(), getOperand(0), Twine(), isVolatile(),
- getAlign(), getOrdering(), getSyncScopeID(),
- isElementwise(),
+ return new LoadInst(getType(), getOperand(0), Twine(), getProperties(),
/*InsertBefore=*/nullptr);
}
diff --git a/llvm/unittests/IR/VerifierTest.cpp b/llvm/unittests/IR/VerifierTest.cpp
index f3753aa55dee8..3cc3ee450e8cc 100644
--- a/llvm/unittests/IR/VerifierTest.cpp
+++ b/llvm/unittests/IR/VerifierTest.cpp
@@ -543,9 +543,12 @@ TEST(VerifierTest, ElementwiseLoadNonAtomic) {
Type *I32Ty = Type::getInt32Ty(C);
Type *VecTy = FixedVectorType::get(I32Ty, 4);
- new LoadInst(VecTy, Ptr, "", /*isVolatile=*/false, Align(4),
- AtomicOrdering::NotAtomic, SyncScope::System,
- /*IsElementwise=*/true, Entry);
+ new LoadInst(VecTy, Ptr, "",
+ LoadStoreInstProperties{/*IsVolatile=*/false, Align(4),
+ AtomicOrdering::NotAtomic,
+ SyncScope::System,
+ /*IsElementwise=*/true},
+ Entry);
ReturnInst::Create(C, Entry);
std::string Error;
@@ -566,9 +569,12 @@ TEST(VerifierTest, ElementwiseLoadScalar) {
Type *I32Ty = Type::getInt32Ty(C);
- new LoadInst(I32Ty, Ptr, "", /*isVolatile=*/false, Align(4),
- AtomicOrdering::Monotonic, SyncScope::System,
- /*IsElementwise=*/true, Entry);
+ new LoadInst(I32Ty, Ptr, "",
+ LoadStoreInstProperties{/*IsVolatile=*/false, Align(4),
+ AtomicOrdering::Monotonic,
+ SyncScope::System,
+ /*IsElementwise=*/true},
+ Entry);
ReturnInst::Create(C, Entry);
std::string Error;
@@ -590,9 +596,12 @@ TEST(VerifierTest, ElementwiseLoadOddSizedVector) {
Type *I32Ty = Type::getInt32Ty(C);
Type *VecTy = FixedVectorType::get(I32Ty, 5);
- new LoadInst(VecTy, Ptr, "", /*isVolatile=*/false, Align(4),
- AtomicOrdering::Monotonic, SyncScope::System,
- /*IsElementwise=*/true, Entry);
+ new LoadInst(VecTy, Ptr, "",
+ LoadStoreInstProperties{/*IsVolatile=*/false, Align(4),
+ AtomicOrdering::Monotonic,
+ SyncScope::System,
+ /*IsElementwise=*/true},
+ Entry);
ReturnInst::Create(C, Entry);
std::string Error;
More information about the llvm-commits
mailing list