[llvm] [IR] Add elementwise modifier to atomic loads (PR #204556)
Harrison Hao via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 8 18:18:52 PDT 2026
https://github.com/harrisonGPU updated https://github.com/llvm/llvm-project/pull/204556
>From 055c9c108d81a54eb344fc315bf2f69c64c8c553 Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Thu, 18 Jun 2026 18:50:37 +0800
Subject: [PATCH 1/6] [IR] Add elementwise modifier to atomic loads
---
llvm/docs/LangRef.rst | 21 ++++++++----
llvm/include/llvm/IR/Instructions.h | 17 ++++++++--
llvm/lib/AsmParser/LLParser.cpp | 16 +++++++--
llvm/lib/Bitcode/Reader/BitcodeReader.cpp | 13 +++++---
llvm/lib/Bitcode/Writer/BitcodeWriter.cpp | 2 ++
llvm/lib/IR/AsmWriter.cpp | 3 ++
llvm/lib/IR/Instruction.cpp | 1 +
llvm/lib/IR/Instructions.cpp | 12 +++++--
llvm/lib/IR/Verifier.cpp | 22 ++++++++++---
llvm/test/Assembler/atomic.ll | 5 +++
.../invalid-load-store-atomic-elementwise.ll | 33 +++++++++++++++++++
.../Bitcode/atomic-load-store-elementwise.ll | 16 +++++++++
12 files changed, 140 insertions(+), 21 deletions(-)
create mode 100644 llvm/test/Assembler/invalid-load-store-atomic-elementwise.ll
create mode 100644 llvm/test/Bitcode/atomic-load-store-elementwise.ll
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index 22d8467959c47..86a4fff1a2c3d 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -12150,7 +12150,7 @@ Syntax:
::
<result> = load [volatile] <ty>, ptr <pointer>[, align <alignment>][, !nontemporal !<nontemp_node>][, !invariant.load !<empty_node>][, !invariant.group !<empty_node>][, !nonnull !<empty_node>][, !dereferenceable !<deref_bytes_node>][, !dereferenceable_or_null !<deref_bytes_node>][, !align !<align_node>][, !noundef !<empty_node>]
- <result> = load atomic [volatile] <ty>, ptr <pointer> [syncscope("<target-scope>")] <ordering>, align <alignment> [, !invariant.group !<empty_node>]
+ <result> = load atomic [volatile] [elementwise] <ty>, ptr <pointer> [syncscope("<target-scope>")] <ordering>, align <alignment> [, !invariant.group !<empty_node>]
!<nontemp_node> = !{ i32 1 }
!<empty_node> = !{}
!<deref_bytes_node> = !{ i64 <dereferenceable_bytes> }
@@ -12177,11 +12177,20 @@ If the ``load`` is marked as ``atomic``, it takes an extra :ref:`ordering
Atomic loads produce :ref:`defined <memmodel>` results when they may see
multiple atomic stores. The type of the pointee must be an integer, pointer,
floating-point, or vector type whose bit width is a power of two greater than
-or equal to eight. ``align`` must be
-explicitly specified on atomic loads. Note: if the alignment is not greater or
-equal to the size of the `<value>` type, the atomic operation is likely to
-require a lock and have poor performance. ``!nontemporal`` does not have any
-defined semantics for atomic loads.
+or equal to eight.
+
+If the ``elementwise`` modifier is present, the loaded type must be a fixed
+vector type, and each element type must be a valid scalar atomic load type. The
+load has per-element vector atomic semantics: it behaves as if it were expanded
+into one scalar atomic load per element, and the element loads are not ordered
+with respect to each other. Without ``elementwise``, vector atomic loads keep
+whole-value atomic semantics.
+
+``align`` must be explicitly specified on atomic loads. Note: if the alignment
+is not greater than or equal to the size of the ``<ty>`` type, or the element
+type for an ``elementwise`` load, the atomic operation is likely to require a
+lock and have poor performance. ``!nontemporal`` does not have any defined
+semantics for atomic loads.
The optional constant ``align`` argument specifies the alignment of the
operation (that is, the alignment of the memory address). It is the
diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h
index 8ceb4cf06d15e..a56b82dc2478c 100644
--- a/llvm/include/llvm/IR/Instructions.h
+++ b/llvm/include/llvm/IR/Instructions.h
@@ -190,9 +190,10 @@ class LoadInst : public UnaryInstruction {
using VolatileField = BoolBitfieldElementT<0>;
using AlignmentField = AlignmentBitfieldElementT<VolatileField::NextBit>;
using OrderingField = AtomicOrderingBitfieldElementT<AlignmentField::NextBit>;
- static_assert(
- Bitfield::areContiguous<VolatileField, AlignmentField, OrderingField>(),
- "Bitfields must be contiguous");
+ using ElementWiseField = BoolBitfieldElementT<OrderingField::NextBit>;
+ static_assert(Bitfield::areContiguous<VolatileField, AlignmentField,
+ OrderingField, ElementWiseField>(),
+ "Bitfields must be contiguous");
void AssertOK();
@@ -209,9 +210,13 @@ class LoadInst : public UnaryInstruction {
InsertPosition InsertBefore);
LLVM_ABI LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
Align Align, InsertPosition InsertBefore = nullptr);
+ LLVM_ABI LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
+ Align Align, AtomicOrdering Order, SyncScope::ID SSID,
+ InsertPosition InsertBefore);
LLVM_ABI LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
Align Align, AtomicOrdering Order,
SyncScope::ID SSID = SyncScope::System,
+ bool IsElementwise = false,
InsertPosition InsertBefore = nullptr);
LLVM_ABI LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr,
const LoadStoreInstProperties &Props,
@@ -223,6 +228,12 @@ class LoadInst : public UnaryInstruction {
/// Specify whether this is a volatile load or not.
void setVolatile(bool V) { setSubclassData<VolatileField>(V); }
+ /// Return true if this is an elementwise atomic load.
+ bool isElementwise() const { return getSubclassData<ElementWiseField>(); }
+
+ /// Specify whether this is an elementwise atomic load or not.
+ void setElementwise(bool V) { setSubclassData<ElementWiseField>(V); }
+
/// Return the alignment of the access that is being performed.
Align getAlign() const {
return Align(1ULL << (getSubclassData<AlignmentField>()));
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index 493f46438e68f..cd36ac08ad8a0 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -8962,7 +8962,7 @@ int LLParser::parseAlloc(Instruction *&Inst, PerFunctionState &PFS) {
/// parseLoad
/// ::= 'load' 'volatile'? TypeAndValue (',' 'align' i32)?
-/// ::= 'load' 'atomic' 'volatile'? TypeAndValue
+/// ::= 'load' 'atomic' 'volatile'? 'elementwise'? TypeAndValue
/// 'singlethread'? AtomicOrdering (',' 'align' i32)?
int LLParser::parseLoad(Instruction *&Inst, PerFunctionState &PFS) {
Value *Val; LocTy Loc;
@@ -8983,6 +8983,12 @@ int LLParser::parseLoad(Instruction *&Inst, PerFunctionState &PFS) {
Lex.Lex();
}
+ bool IsElementwise = false;
+ if (Lex.getKind() == lltok::kw_elementwise) {
+ IsElementwise = true;
+ Lex.Lex();
+ }
+
Type *Ty;
LocTy ExplicitTypeLoc = Lex.getLoc();
if (parseType(Ty) ||
@@ -8994,8 +9000,13 @@ int LLParser::parseLoad(Instruction *&Inst, PerFunctionState &PFS) {
if (!Val->getType()->isPointerTy() || !Ty->isFirstClassType())
return error(Loc, "load operand must be a pointer to a first class type");
+
+ if (IsElementwise && !isAtomic)
+ return error(Loc, "elementwise load must be atomic");
+
if (isAtomic && !Alignment)
return error(Loc, "atomic load must have explicit non-zero alignment");
+
if (Ordering == AtomicOrdering::Release ||
Ordering == AtomicOrdering::AcquireRelease)
return error(Loc, "atomic load cannot use Release ordering");
@@ -9005,7 +9016,8 @@ int LLParser::parseLoad(Instruction *&Inst, PerFunctionState &PFS) {
return error(ExplicitTypeLoc, "loading unsized types is not allowed");
if (!Alignment)
Alignment = M->getDataLayout().getABITypeAlign(Ty);
- Inst = new LoadInst(Ty, Val, "", isVolatile, *Alignment, Ordering, SSID);
+ Inst = new LoadInst(Ty, Val, "", isVolatile, *Alignment, Ordering, SSID,
+ IsElementwise);
return AteExtraComma ? InstExtraComma : InstNormal;
}
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index a5a11ab6221d4..0ec58c9e3a811 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -6442,19 +6442,22 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
break;
}
case bitc::FUNC_CODE_INST_LOADATOMIC: {
- // LOADATOMIC: [opty, op, align, vol, ordering, ssid]
+ // LOADATOMIC: [opty, op, align, vol, ordering, ssid, elementwise?]
unsigned OpNum = 0;
Value *Op;
unsigned OpTypeID;
if (getValueTypePair(Record, OpNum, NextValueNo, Op, OpTypeID, CurBB) ||
- (OpNum + 4 != Record.size() && OpNum + 5 != Record.size()))
+ (OpNum + 4 != Record.size() && OpNum + 5 != Record.size() &&
+ OpNum + 6 != Record.size()))
return error("Invalid load atomic record");
if (!isa<PointerType>(Op->getType()))
return error("Load operand is not a pointer type");
Type *Ty = nullptr;
- if (OpNum + 5 == Record.size()) {
+ bool HasExplicitType =
+ Record.size() == OpNum + 5 || Record.size() == OpNum + 6;
+ if (HasExplicitType) {
ResTypeID = Record[OpNum++];
Ty = getTypeByID(ResTypeID);
} else {
@@ -6476,13 +6479,15 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
if (Ordering != AtomicOrdering::NotAtomic && Record[OpNum] == 0)
return error("Invalid load atomic record");
SyncScope::ID SSID = getDecodedSyncScopeID(Record[OpNum + 3]);
+ bool IsElementwise = Record.size() > OpNum + 4 && Record[OpNum + 4];
MaybeAlign Align;
if (Error Err = parseAlignmentValue(Record[OpNum], Align))
return Err;
if (!Align)
return error("Alignment missing from atomic load");
- I = new LoadInst(Ty, Op, "", Record[OpNum + 1], *Align, Ordering, SSID);
+ I = new LoadInst(Ty, Op, "", Record[OpNum + 1], *Align, Ordering, SSID,
+ IsElementwise);
InstructionList.push_back(I);
break;
}
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index a6e7941a06699..62da7313f32d4 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -3583,6 +3583,8 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I,
if (cast<LoadInst>(I).isAtomic()) {
Vals.push_back(getEncodedOrdering(cast<LoadInst>(I).getOrdering()));
Vals.push_back(getEncodedSyncScopeID(cast<LoadInst>(I).getSyncScopeID()));
+ if (cast<LoadInst>(I).isElementwise())
+ Vals.push_back(1);
}
break;
case Instruction::Store:
diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp
index e90630a8cae59..40be6a072216e 100644
--- a/llvm/lib/IR/AsmWriter.cpp
+++ b/llvm/lib/IR/AsmWriter.cpp
@@ -4474,6 +4474,9 @@ void AssemblyWriter::printInstruction(const Instruction &I) {
(isa<AtomicRMWInst>(I) && cast<AtomicRMWInst>(I).isVolatile()))
Out << " volatile";
+ if (isa<LoadInst>(I) && cast<LoadInst>(I).isElementwise())
+ Out << " elementwise";
+
// Print out optimization information.
writeOptimizationInfo(Out, &I);
diff --git a/llvm/lib/IR/Instruction.cpp b/llvm/lib/IR/Instruction.cpp
index 112d81be3a2fa..5bade74b93c9b 100644
--- a/llvm/lib/IR/Instruction.cpp
+++ b/llvm/lib/IR/Instruction.cpp
@@ -927,6 +927,7 @@ bool Instruction::hasSameSpecialState(const Instruction *I2,
IgnoreAlignment);
if (const LoadInst *LI = dyn_cast<LoadInst>(I1))
return LI->isVolatile() == cast<LoadInst>(I2)->isVolatile() &&
+ LI->isElementwise() == cast<LoadInst>(I2)->isElementwise() &&
(LI->getAlign() == cast<LoadInst>(I2)->getAlign() ||
IgnoreAlignment) &&
LI->getOrdering() == cast<LoadInst>(I2)->getOrdering() &&
diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index ae04fb845112e..abf84a85dc918 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -1363,7 +1363,7 @@ LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
Align Align, InsertPosition InsertBef)
: LoadInst(Ty, Ptr, Name, isVolatile, Align, AtomicOrdering::NotAtomic,
- SyncScope::System, InsertBef) {}
+ SyncScope::System, /*IsElementwise=*/false, InsertBef) {}
LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name,
const LoadStoreInstProperties &Props,
@@ -1374,10 +1374,17 @@ LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name,
LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
Align Align, AtomicOrdering Order, SyncScope::ID SSID,
InsertPosition InsertBef)
+ : LoadInst(Ty, Ptr, Name, isVolatile, Align, Order, SSID,
+ /*IsElementwise=*/false, InsertBef) {}
+
+LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
+ Align Align, AtomicOrdering Order, SyncScope::ID SSID,
+ bool IsElementwise, InsertPosition InsertBef)
: UnaryInstruction(Ty, Load, Ptr, InsertBef) {
setVolatile(isVolatile);
setAlignment(Align);
setAtomic(Order, SSID);
+ setElementwise(IsElementwise);
AssertOK();
setName(Name);
}
@@ -4454,7 +4461,8 @@ AllocaInst *AllocaInst::cloneImpl() const {
LoadInst *LoadInst::cloneImpl() const {
return new LoadInst(getType(), getOperand(0), Twine(), isVolatile(),
- getAlign(), getOrdering(), getSyncScopeID());
+ getAlign(), getOrdering(), getSyncScopeID(),
+ isElementwise());
}
StoreInst *StoreInst::cloneImpl() const {
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index f605af1d0a31b..d90b38248d5c5 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -4539,15 +4539,29 @@ void Verifier::visitLoadInst(LoadInst &LI) {
Check(LI.getOrdering() != AtomicOrdering::Release &&
LI.getOrdering() != AtomicOrdering::AcquireRelease,
"Load cannot have Release ordering", &LI);
- Check(ElTy->getScalarType()->isIntOrPtrTy() ||
- ElTy->getScalarType()->isByteTy() ||
- ElTy->getScalarType()->isFloatingPointTy(),
+
+ Type *ScalarTy = ElTy;
+ if (LI.isElementwise()) {
+ auto *VecTy = dyn_cast<FixedVectorType>(ElTy);
+ Check(VecTy,
+ "atomic elementwise load operand must have fixed vector type!", &LI,
+ ElTy);
+ if (VecTy) {
+ checkAtomicMemAccessSize(ScalarTy, &LI);
+ ScalarTy = VecTy->getElementType();
+ }
+ }
+
+ Check(ScalarTy->getScalarType()->isIntOrPtrTy() ||
+ ScalarTy->getScalarType()->isByteTy() ||
+ ScalarTy->getScalarType()->isFloatingPointTy(),
"atomic load operand must have integer, byte, pointer, floating "
"point, or vector type!",
ElTy, &LI);
- checkAtomicMemAccessSize(ElTy, &LI);
+ checkAtomicMemAccessSize(ScalarTy, &LI);
} else {
+ Check(!LI.isElementwise(), "non-atomic load cannot be elementwise", &LI);
Check(LI.getSyncScopeID() == SyncScope::System,
"Non-atomic load cannot have SynchronizationScope specified", &LI);
}
diff --git a/llvm/test/Assembler/atomic.ll b/llvm/test/Assembler/atomic.ll
index 611a717fa9a8d..64a58249ee70d 100644
--- a/llvm/test/Assembler/atomic.ll
+++ b/llvm/test/Assembler/atomic.ll
@@ -71,6 +71,11 @@ define void @f(ptr %x) {
; CHECK : store atomic <2 x float> <float 3.0, float 4.0>, ptr %x release, align 4
store atomic <2 x float> <float 3.0, float 4.0>, ptr %x release, align 4
+ ; CHECK: load atomic elementwise <2 x float>, ptr %x syncscope("agent") monotonic, align 4
+ load atomic elementwise <2 x float>, ptr %x syncscope("agent") monotonic, align 4
+ ; CHECK: load atomic volatile elementwise <2 x i32>, ptr %x monotonic, align 4
+ load atomic volatile elementwise <2 x i32>, ptr %x monotonic, align 4
+
; CHECK: fence syncscope("singlethread") release
fence syncscope("singlethread") release
; CHECK: fence seq_cst
diff --git a/llvm/test/Assembler/invalid-load-store-atomic-elementwise.ll b/llvm/test/Assembler/invalid-load-store-atomic-elementwise.ll
new file mode 100644
index 0000000000000..53d8571a9b461
--- /dev/null
+++ b/llvm/test/Assembler/invalid-load-store-atomic-elementwise.ll
@@ -0,0 +1,33 @@
+; RUN: split-file %s %t
+; RUN: not llvm-as -disable-output %t/load-non-atomic.ll 2>&1 | FileCheck %t/load-non-atomic.ll
+; RUN: not llvm-as -disable-output %t/load-scalar.ll 2>&1 | FileCheck %t/load-scalar.ll
+; RUN: not llvm-as -disable-output %t/load-scalable.ll 2>&1 | FileCheck %t/load-scalable.ll
+; RUN: not llvm-as -disable-output %t/load-odd-sized.ll 2>&1 | FileCheck %t/load-odd-sized.ll
+
+;--- load-non-atomic.ll
+; CHECK: elementwise load must be atomic
+define <2 x float> @bad_non_atomic(ptr %p) {
+ %v = load elementwise <2 x float>, ptr %p, align 4
+ ret <2 x float> %v
+}
+
+;--- load-scalar.ll
+; CHECK: atomic elementwise load operand must have fixed vector type
+define float @bad_scalar(ptr %p) {
+ %v = load atomic elementwise float, ptr %p monotonic, align 4
+ ret float %v
+}
+
+;--- load-scalable.ll
+; CHECK: atomic elementwise load operand must have fixed vector type
+define <vscale x 2 x i32> @bad_scalable(ptr %p) {
+ %v = load atomic elementwise <vscale x 2 x i32>, ptr %p monotonic, align 4
+ ret <vscale x 2 x i32> %v
+}
+
+;--- load-odd-sized.ll
+; CHECK: atomic memory access' operand must have a power-of-two size
+define <5 x i32> @bad_odd_sized_vector(ptr %p) {
+ %v = load atomic elementwise <5 x i32>, ptr %p monotonic, align 4
+ ret <5 x i32> %v
+}
diff --git a/llvm/test/Bitcode/atomic-load-store-elementwise.ll b/llvm/test/Bitcode/atomic-load-store-elementwise.ll
new file mode 100644
index 0000000000000..6b921c199c08c
--- /dev/null
+++ b/llvm/test/Bitcode/atomic-load-store-elementwise.ll
@@ -0,0 +1,16 @@
+; RUN: llvm-as %s -o - | llvm-dis | FileCheck %s
+; RUN: llvm-as %s -o - | verify-uselistorder
+
+define <2 x float> @load_elem_f32(ptr %p) {
+; CHECK-LABEL: @load_elem_f32(
+; CHECK: %v = load atomic elementwise <2 x float>, ptr %p syncscope("agent") monotonic, align 4
+ %v = load atomic elementwise <2 x float>, ptr %p syncscope("agent") monotonic, align 4
+ ret <2 x float> %v
+}
+
+define <4 x i32> @load_elem_i32_volatile(ptr %p) {
+; CHECK-LABEL: @load_elem_i32_volatile(
+; CHECK: %v = load atomic volatile elementwise <4 x i32>, ptr %p seq_cst, align 4
+ %v = load atomic volatile elementwise <4 x i32>, ptr %p seq_cst, align 4
+ ret <4 x i32> %v
+}
>From 5e97cfdb3ab4a0b72239f4c04adca2a5dfea093a Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Fri, 26 Jun 2026 17:01:55 +0800
Subject: [PATCH 2/6] Update for comments.
---
llvm/docs/LangRef.rst | 12 +++++------
llvm/include/llvm/IR/Instructions.h | 9 ++++----
llvm/lib/AsmParser/LLParser.cpp | 6 +++++-
llvm/lib/Bitcode/Reader/BitcodeReader.cpp | 2 +-
llvm/lib/Bitcode/Writer/BitcodeWriter.cpp | 25 +++++++++++++----------
llvm/lib/IR/Instructions.cpp | 12 +++--------
llvm/test/Bitcode/compatibility.ll | 6 ++++++
7 files changed, 39 insertions(+), 33 deletions(-)
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index 86a4fff1a2c3d..41c15b0e4df35 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -12186,13 +12186,13 @@ into one scalar atomic load per element, and the element loads are not ordered
with respect to each other. Without ``elementwise``, vector atomic loads keep
whole-value atomic semantics.
-``align`` must be explicitly specified on atomic loads. Note: if the alignment
-is not greater than or equal to the size of the ``<ty>`` type, or the element
-type for an ``elementwise`` load, the atomic operation is likely to require a
-lock and have poor performance. ``!nontemporal`` does not have any defined
-semantics for atomic loads.
+``align`` must be explicitly specified on atomic loads, and is otherwise
+optional on non-atomic loads. Note: if the alignment is not greater than or equal
+to the size of the ``<ty>`` type, or the element type for an ``elementwise`` load,
+the atomic operation is likely to require a lock and have poor performance.
+``!nontemporal`` does not have any defined semantics for atomic loads.
-The optional constant ``align`` argument specifies the alignment of the
+The constant ``align`` argument specifies the alignment of the
operation (that is, the alignment of the memory address). It is the
responsibility of the code emitter to ensure that the alignment information is
correct. Overestimating the alignment results in undefined behavior.
diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h
index a56b82dc2478c..cffec9d60b0f8 100644
--- a/llvm/include/llvm/IR/Instructions.h
+++ b/llvm/include/llvm/IR/Instructions.h
@@ -182,11 +182,12 @@ struct LoadStoreInstProperties {
Align Alignment;
AtomicOrdering Ordering;
SyncScope::ID SSID;
+ bool IsElementwise;
};
/// An instruction for reading from memory. This uses the SubclassData field in
/// Value to store whether or not the load is volatile.
-class LoadInst : public UnaryInstruction {
+class LoadInst : public UnaryInstruction {
using VolatileField = BoolBitfieldElementT<0>;
using AlignmentField = AlignmentBitfieldElementT<VolatileField::NextBit>;
using OrderingField = AtomicOrderingBitfieldElementT<AlignmentField::NextBit>;
@@ -210,9 +211,6 @@ class LoadInst : public UnaryInstruction {
InsertPosition InsertBefore);
LLVM_ABI LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
Align Align, InsertPosition InsertBefore = nullptr);
- LLVM_ABI LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
- Align Align, AtomicOrdering Order, SyncScope::ID SSID,
- InsertPosition InsertBefore);
LLVM_ABI LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
Align Align, AtomicOrdering Order,
SyncScope::ID SSID = SyncScope::System,
@@ -273,7 +271,7 @@ class LoadInst : public UnaryInstruction {
/// Returns the properties of this load instruction.
LoadStoreInstProperties getProperties() const {
- return {isVolatile(), getAlign(), getOrdering(), getSyncScopeID()};
+ return {isVolatile(), getAlign(), getOrdering(), getSyncScopeID(), isElementwise()};
}
/// Sets the properties of this load instruction.
@@ -282,6 +280,7 @@ class LoadInst : public UnaryInstruction {
setAlignment(Props.Alignment);
setOrdering(Props.Ordering);
setSyncScopeID(Props.SSID);
+ setElementwise(Props.IsElementwise);
}
bool isSimple() const { return !isAtomic() && !isVolatile(); }
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index cd36ac08ad8a0..43476a5ae701d 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -9004,6 +9004,10 @@ int LLParser::parseLoad(Instruction *&Inst, PerFunctionState &PFS) {
if (IsElementwise && !isAtomic)
return error(Loc, "elementwise load must be atomic");
+ if (IsElementwise && !isa<FixedVectorType>(Ty))
+ return error(ExplicitTypeLoc,
+ "atomic elementwise load operand must have fixed vector type");
+
if (isAtomic && !Alignment)
return error(Loc, "atomic load must have explicit non-zero alignment");
@@ -9017,7 +9021,7 @@ int LLParser::parseLoad(Instruction *&Inst, PerFunctionState &PFS) {
if (!Alignment)
Alignment = M->getDataLayout().getABITypeAlign(Ty);
Inst = new LoadInst(Ty, Val, "", isVolatile, *Alignment, Ordering, SSID,
- IsElementwise);
+ /*InsertBefore=*/nullptr, IsElementwise);
return AteExtraComma ? InstExtraComma : InstNormal;
}
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index 0ec58c9e3a811..1529df6c71e5d 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -6487,7 +6487,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
if (!Align)
return error("Alignment missing from atomic load");
I = new LoadInst(Ty, Op, "", Record[OpNum + 1], *Align, Ordering, SSID,
- IsElementwise);
+ /*InsertBefore=*/nullptr, IsElementwise);
InstructionList.push_back(I);
break;
}
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index 62da7313f32d4..0b4351fd8b49b 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -3568,25 +3568,28 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I,
break;
}
- case Instruction::Load:
- if (cast<LoadInst>(I).isAtomic()) {
+ case Instruction::Load: {
+ const auto &LI = cast<LoadInst>(I);
+ if (LI.isAtomic()) {
Code = bitc::FUNC_CODE_INST_LOADATOMIC;
- pushValueAndType(I.getOperand(0), InstID, Vals);
+ pushValueAndType(LI.getOperand(0), InstID, Vals);
} else {
Code = bitc::FUNC_CODE_INST_LOAD;
- if (!pushValueAndType(I.getOperand(0), InstID, Vals)) // ptr
+ if (!pushValueAndType(LI.getOperand(0), InstID, Vals)) // ptr
AbbrevToUse = FUNCTION_INST_LOAD_ABBREV;
}
- Vals.push_back(VE.getTypeID(I.getType()));
- Vals.push_back(getEncodedAlign(cast<LoadInst>(I).getAlign()));
- Vals.push_back(cast<LoadInst>(I).isVolatile());
- if (cast<LoadInst>(I).isAtomic()) {
- Vals.push_back(getEncodedOrdering(cast<LoadInst>(I).getOrdering()));
- Vals.push_back(getEncodedSyncScopeID(cast<LoadInst>(I).getSyncScopeID()));
- if (cast<LoadInst>(I).isElementwise())
+ Vals.push_back(VE.getTypeID(LI.getType()));
+ Vals.push_back(getEncodedAlign(LI.getAlign()));
+ Vals.push_back(LI.isVolatile());
+ if (LI.isAtomic()) {
+ Vals.push_back(getEncodedOrdering(LI.getOrdering()));
+ Vals.push_back(getEncodedSyncScopeID(LI.getSyncScopeID()));
+ if (LI.isElementwise())
Vals.push_back(1);
}
break;
+ }
+
case Instruction::Store:
if (cast<StoreInst>(I).isAtomic()) {
Code = bitc::FUNC_CODE_INST_STOREATOMIC;
diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index abf84a85dc918..3fb8d0ae41911 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -1363,7 +1363,7 @@ LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
Align Align, InsertPosition InsertBef)
: LoadInst(Ty, Ptr, Name, isVolatile, Align, AtomicOrdering::NotAtomic,
- SyncScope::System, /*IsElementwise=*/false, InsertBef) {}
+ SyncScope::System, InsertBef, /*IsElementwise=*/false) {}
LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name,
const LoadStoreInstProperties &Props,
@@ -1373,13 +1373,7 @@ LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name,
LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
Align Align, AtomicOrdering Order, SyncScope::ID SSID,
- InsertPosition InsertBef)
- : LoadInst(Ty, Ptr, Name, isVolatile, Align, Order, SSID,
- /*IsElementwise=*/false, InsertBef) {}
-
-LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
- Align Align, AtomicOrdering Order, SyncScope::ID SSID,
- bool IsElementwise, InsertPosition InsertBef)
+ InsertPosition InsertBef, bool IsElementwise)
: UnaryInstruction(Ty, Load, Ptr, InsertBef) {
setVolatile(isVolatile);
setAlignment(Align);
@@ -4462,7 +4456,7 @@ AllocaInst *AllocaInst::cloneImpl() const {
LoadInst *LoadInst::cloneImpl() const {
return new LoadInst(getType(), getOperand(0), Twine(), isVolatile(),
getAlign(), getOrdering(), getSyncScopeID(),
- isElementwise());
+ /*InsertBefore=*/nullptr, isElementwise());
}
StoreInst *StoreInst::cloneImpl() const {
diff --git a/llvm/test/Bitcode/compatibility.ll b/llvm/test/Bitcode/compatibility.ll
index 97889d4687516..9d8617fb7a9e3 100644
--- a/llvm/test/Bitcode/compatibility.ll
+++ b/llvm/test/Bitcode/compatibility.ll
@@ -1034,6 +1034,12 @@ define void @elementwise_atomics(ptr %word, <4 x i32> %ival, <4 x float> %fval)
; CHECK: %atomicrmw.fadd = atomicrmw elementwise fadd ptr %word, <4 x float> %fval seq_cst, align 16
%atomicrmw.fadd = atomicrmw elementwise fadd ptr %word, <4 x float> %fval seq_cst, align 16
+; CHECK: %load.elementwise = load atomic elementwise <4 x i32>, ptr %word monotonic, align 4
+ %load.elementwise = load atomic elementwise <4 x i32>, ptr %word monotonic, align 4
+
+; CHECK: %load.elementwise.volatile = load atomic volatile elementwise <4 x float>, ptr %word seq_cst, align 4
+ %load.elementwise.volatile = load atomic volatile elementwise <4 x float>, ptr %word seq_cst, align 4
+
ret void
}
>From d1975d49056be178227fb2021a4c7dbd960c122a Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Fri, 26 Jun 2026 17:34:38 +0800
Subject: [PATCH 3/6] Add verifier test
---
llvm/unittests/IR/VerifierTest.cpp | 71 ++++++++++++++++++++++++++++++
1 file changed, 71 insertions(+)
diff --git a/llvm/unittests/IR/VerifierTest.cpp b/llvm/unittests/IR/VerifierTest.cpp
index ea7ab9ec15244..190a930626e23 100644
--- a/llvm/unittests/IR/VerifierTest.cpp
+++ b/llvm/unittests/IR/VerifierTest.cpp
@@ -505,6 +505,77 @@ TEST(VerifierTest, AtomicRMWElementwiseFPOpOnIntVector) {
<< Error;
}
+TEST(VerifierTest, ElementwiseLoadNonAtomic) {
+ LLVMContext C;
+ Module M("M", C);
+ FunctionType *FTy = FunctionType::get(Type::getVoidTy(C), /*isVarArg=*/false);
+ Function *F = Function::Create(FTy, Function::ExternalLinkage, "foo", M);
+ BasicBlock *Entry = BasicBlock::Create(C, "entry", F);
+ Value *Ptr = PoisonValue::get(PointerType::get(C, 0));
+
+ Type *I32Ty = Type::getInt32Ty(C);
+ Type *VecTy = FixedVectorType::get(I32Ty, 4);
+
+ new LoadInst(VecTy, Ptr, "", /*isVolatile=*/false, Align(4),
+ AtomicOrdering::NotAtomic, SyncScope::System, Entry,
+ /*IsElementwise=*/true);
+ ReturnInst::Create(C, Entry);
+
+ std::string Error;
+ raw_string_ostream ErrorOS(Error);
+ EXPECT_TRUE(verifyFunction(*F, &ErrorOS));
+ EXPECT_TRUE(
+ StringRef(Error).starts_with("non-atomic load cannot be elementwise"))
+ << Error;
+}
+
+TEST(VerifierTest, ElementwiseLoadScalar) {
+ LLVMContext C;
+ Module M("M", C);
+ FunctionType *FTy = FunctionType::get(Type::getVoidTy(C), /*isVarArg=*/false);
+ Function *F = Function::Create(FTy, Function::ExternalLinkage, "foo", M);
+ BasicBlock *Entry = BasicBlock::Create(C, "entry", F);
+ Value *Ptr = PoisonValue::get(PointerType::get(C, 0));
+
+ Type *I32Ty = Type::getInt32Ty(C);
+
+ new LoadInst(I32Ty, Ptr, "", /*isVolatile=*/false, Align(4),
+ AtomicOrdering::Monotonic, SyncScope::System, Entry,
+ /*IsElementwise=*/true);
+ ReturnInst::Create(C, Entry);
+
+ std::string Error;
+ raw_string_ostream ErrorOS(Error);
+ EXPECT_TRUE(verifyFunction(*F, &ErrorOS));
+ EXPECT_TRUE(StringRef(Error).starts_with(
+ "atomic elementwise load operand must have fixed vector type!"))
+ << Error;
+}
+
+TEST(VerifierTest, ElementwiseLoadOddSizedVector) {
+ LLVMContext C;
+ Module M("M", C);
+ FunctionType *FTy = FunctionType::get(Type::getVoidTy(C), /*isVarArg=*/false);
+ Function *F = Function::Create(FTy, Function::ExternalLinkage, "foo", M);
+ BasicBlock *Entry = BasicBlock::Create(C, "entry", F);
+ Value *Ptr = PoisonValue::get(PointerType::get(C, 0));
+
+ Type *I32Ty = Type::getInt32Ty(C);
+ Type *VecTy = FixedVectorType::get(I32Ty, 5);
+
+ new LoadInst(VecTy, Ptr, "", /*isVolatile=*/false, Align(4),
+ AtomicOrdering::Monotonic, SyncScope::System, Entry,
+ /*IsElementwise=*/true);
+ ReturnInst::Create(C, Entry);
+
+ std::string Error;
+ raw_string_ostream ErrorOS(Error);
+ EXPECT_TRUE(verifyFunction(*F, &ErrorOS));
+ EXPECT_TRUE(StringRef(Error).starts_with(
+ "atomic memory access' operand must have a power-of-two size"))
+ << Error;
+}
+
TEST(VerifierTest, GetElementPtrInst) {
LLVMContext C;
Module M("M", C);
>From 3a606e022598da8d9c62456b9467f72d3d0c9db3 Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Mon, 29 Jun 2026 17:03:31 +0800
Subject: [PATCH 4/6] Update for comments
---
llvm/docs/LangRef.rst | 17 ++++++++++-------
llvm/lib/Bitcode/Reader/BitcodeReader.cpp | 4 +---
.../invalid-load-store-atomic-elementwise.ll | 8 ++++++++
3 files changed, 19 insertions(+), 10 deletions(-)
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index 41c15b0e4df35..a02a69b811f10 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -12175,15 +12175,18 @@ If the ``load`` is marked as ``atomic``, it takes an extra :ref:`ordering
<ordering>` and optional ``syncscope("<target-scope>")`` argument. The
``release`` and ``acq_rel`` orderings are not valid on ``load`` instructions.
Atomic loads produce :ref:`defined <memmodel>` results when they may see
-multiple atomic stores. The type of the pointee must be an integer, pointer,
-floating-point, or vector type whose bit width is a power of two greater than
-or equal to eight.
+multiple atomic stores. Atomic loads support scalar integer, pointer, and
+floating-point types whose bit width is a power of two greater than or equal
+to eight. Atomic loads also support vector types whose total bit width is a
+power of two greater than or equal to eight. That is, the entire vector is
+loaded atomically.
If the ``elementwise`` modifier is present, the loaded type must be a fixed
-vector type, and each element type must be a valid scalar atomic load type. The
-load has per-element vector atomic semantics: it behaves as if it were expanded
-into one scalar atomic load per element, and the element loads are not ordered
-with respect to each other. Without ``elementwise``, vector atomic loads keep
+vector type whose total bit width is a power of two greater than or equal to
+eight, and whose element type is supported by scalar atomic loads. The load has
+per-element atomic load semantics: it behaves as if it were expanded into
+one scalar atomic load per element, and the element loads are not ordered with
+respect to each other. Without ``elementwise``, vector atomic loads keep
whole-value atomic semantics.
``align`` must be explicitly specified on atomic loads, and is otherwise
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index 1529df6c71e5d..f6e71420e59c2 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -6455,9 +6455,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
return error("Load operand is not a pointer type");
Type *Ty = nullptr;
- bool HasExplicitType =
- Record.size() == OpNum + 5 || Record.size() == OpNum + 6;
- if (HasExplicitType) {
+ if (Record.size() >= OpNum + 5) {
ResTypeID = Record[OpNum++];
Ty = getTypeByID(ResTypeID);
} else {
diff --git a/llvm/test/Assembler/invalid-load-store-atomic-elementwise.ll b/llvm/test/Assembler/invalid-load-store-atomic-elementwise.ll
index 53d8571a9b461..03dbc96cb562e 100644
--- a/llvm/test/Assembler/invalid-load-store-atomic-elementwise.ll
+++ b/llvm/test/Assembler/invalid-load-store-atomic-elementwise.ll
@@ -3,6 +3,7 @@
; RUN: not llvm-as -disable-output %t/load-scalar.ll 2>&1 | FileCheck %t/load-scalar.ll
; RUN: not llvm-as -disable-output %t/load-scalable.ll 2>&1 | FileCheck %t/load-scalable.ll
; RUN: not llvm-as -disable-output %t/load-odd-sized.ll 2>&1 | FileCheck %t/load-odd-sized.ll
+; RUN: not llvm-as -disable-output %t/load-non-byte.ll 2>&1 | FileCheck %t/load-non-byte.ll
;--- load-non-atomic.ll
; CHECK: elementwise load must be atomic
@@ -31,3 +32,10 @@ define <5 x i32> @bad_odd_sized_vector(ptr %p) {
%v = load atomic elementwise <5 x i32>, ptr %p monotonic, align 4
ret <5 x i32> %v
}
+
+;--- load-non-byte.ll
+; CHECK: atomic memory access' size must be byte-sized
+define <4 x i1> @bad_non_byte(ptr %p) {
+ %v = load atomic elementwise <4 x i1>, ptr %p monotonic, align 4
+ ret <4 x i1> %v
+}
>From f6bcf1529e13f2009251eb926da80b750e946f94 Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Tue, 30 Jun 2026 17:45:01 +0800
Subject: [PATCH 5/6] Add element non byte
---
.../Assembler/invalid-load-store-atomic-elementwise.ll | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/llvm/test/Assembler/invalid-load-store-atomic-elementwise.ll b/llvm/test/Assembler/invalid-load-store-atomic-elementwise.ll
index 03dbc96cb562e..843564cbbc075 100644
--- a/llvm/test/Assembler/invalid-load-store-atomic-elementwise.ll
+++ b/llvm/test/Assembler/invalid-load-store-atomic-elementwise.ll
@@ -4,6 +4,7 @@
; RUN: not llvm-as -disable-output %t/load-scalable.ll 2>&1 | FileCheck %t/load-scalable.ll
; RUN: not llvm-as -disable-output %t/load-odd-sized.ll 2>&1 | FileCheck %t/load-odd-sized.ll
; RUN: not llvm-as -disable-output %t/load-non-byte.ll 2>&1 | FileCheck %t/load-non-byte.ll
+; RUN: not llvm-as -disable-output %t/load-non-byte-element.ll 2>&1 | FileCheck %t/load-non-byte-element.ll
;--- load-non-atomic.ll
; CHECK: elementwise load must be atomic
@@ -39,3 +40,10 @@ define <4 x i1> @bad_non_byte(ptr %p) {
%v = load atomic elementwise <4 x i1>, ptr %p monotonic, align 4
ret <4 x i1> %v
}
+
+;--- load-non-byte-element.ll
+; CHECK: atomic memory access' size must be byte-sized
+define <8 x i1> @bad_non_byte_element(ptr %p) {
+ %v = load atomic elementwise <8 x i1>, ptr %p monotonic, align 1
+ ret <8 x i1> %v
+}
>From 1b7d8b0ec4f7d69c2247488674c92b882cd3034c Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Fri, 3 Jul 2026 14:29:54 +0800
Subject: [PATCH 6/6] Update doc
---
llvm/docs/LangRef.rst | 10 ++++------
llvm/include/llvm/IR/Instructions.h | 9 ++++++---
llvm/lib/AsmParser/LLParser.cpp | 3 ++-
llvm/lib/Bitcode/Reader/BitcodeReader.cpp | 3 ++-
llvm/lib/IR/Instructions.cpp | 15 +++++++++++----
llvm/unittests/IR/VerifierTest.cpp | 12 ++++++------
6 files changed, 31 insertions(+), 21 deletions(-)
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index a02a69b811f10..70b79a0e38a70 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -12175,11 +12175,9 @@ If the ``load`` is marked as ``atomic``, it takes an extra :ref:`ordering
<ordering>` and optional ``syncscope("<target-scope>")`` argument. The
``release`` and ``acq_rel`` orderings are not valid on ``load`` instructions.
Atomic loads produce :ref:`defined <memmodel>` results when they may see
-multiple atomic stores. Atomic loads support scalar integer, pointer, and
-floating-point types whose bit width is a power of two greater than or equal
-to eight. Atomic loads also support vector types whose total bit width is a
-power of two greater than or equal to eight. That is, the entire vector is
-loaded atomically.
+multiple atomic stores. The type of the pointee must be an integer, pointer,
+floating-point, or vector type whose bit width is a power of two greater than or
+equal to eight.
If the ``elementwise`` modifier is present, the loaded type must be a fixed
vector type whose total bit width is a power of two greater than or equal to
@@ -12187,7 +12185,7 @@ eight, and whose element type is supported by scalar atomic loads. The load has
per-element atomic load semantics: it behaves as if it were expanded into
one scalar atomic load per element, and the element loads are not ordered with
respect to each other. Without ``elementwise``, vector atomic loads keep
-whole-value atomic semantics.
+whole-value atomic semantics. That is, the entire vector is loaded atomically.
``align`` must be explicitly specified on atomic loads, and is otherwise
optional on non-atomic loads. Note: if the alignment is not greater than or equal
diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h
index cffec9d60b0f8..9378d244d3ad0 100644
--- a/llvm/include/llvm/IR/Instructions.h
+++ b/llvm/include/llvm/IR/Instructions.h
@@ -182,7 +182,8 @@ struct LoadStoreInstProperties {
Align Alignment;
AtomicOrdering Ordering;
SyncScope::ID SSID;
- bool IsElementwise;
+ // TODO: Add IsElementwise here once StoreInst also supports the elementwise
+ // modifier.
};
/// An instruction for reading from memory. This uses the SubclassData field in
@@ -211,6 +212,9 @@ class LoadInst : public UnaryInstruction {
InsertPosition InsertBefore);
LLVM_ABI LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
Align Align, InsertPosition InsertBefore = nullptr);
+ LLVM_ABI LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
+ Align Align, AtomicOrdering Order, SyncScope::ID SSID,
+ InsertPosition InsertBefore);
LLVM_ABI LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
Align Align, AtomicOrdering Order,
SyncScope::ID SSID = SyncScope::System,
@@ -271,7 +275,7 @@ class LoadInst : public UnaryInstruction {
/// Returns the properties of this load instruction.
LoadStoreInstProperties getProperties() const {
- return {isVolatile(), getAlign(), getOrdering(), getSyncScopeID(), isElementwise()};
+ return {isVolatile(), getAlign(), getOrdering(), getSyncScopeID()};
}
/// Sets the properties of this load instruction.
@@ -280,7 +284,6 @@ class LoadInst : public UnaryInstruction {
setAlignment(Props.Alignment);
setOrdering(Props.Ordering);
setSyncScopeID(Props.SSID);
- setElementwise(Props.IsElementwise);
}
bool isSimple() const { return !isAtomic() && !isVolatile(); }
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index 43476a5ae701d..48305194112a6 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -9021,7 +9021,8 @@ int LLParser::parseLoad(Instruction *&Inst, PerFunctionState &PFS) {
if (!Alignment)
Alignment = M->getDataLayout().getABITypeAlign(Ty);
Inst = new LoadInst(Ty, Val, "", isVolatile, *Alignment, Ordering, SSID,
- /*InsertBefore=*/nullptr, IsElementwise);
+ IsElementwise,
+ /*InsertBefore=*/nullptr);
return AteExtraComma ? InstExtraComma : InstNormal;
}
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index f6e71420e59c2..11ff56134502c 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -6485,7 +6485,8 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
if (!Align)
return error("Alignment missing from atomic load");
I = new LoadInst(Ty, Op, "", Record[OpNum + 1], *Align, Ordering, SSID,
- /*InsertBefore=*/nullptr, IsElementwise);
+ IsElementwise,
+ /*InsertBefore=*/nullptr);
InstructionList.push_back(I);
break;
}
diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index 3fb8d0ae41911..73957c01fc392 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -1363,17 +1363,23 @@ LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
Align Align, InsertPosition InsertBef)
: LoadInst(Ty, Ptr, Name, isVolatile, Align, AtomicOrdering::NotAtomic,
- SyncScope::System, InsertBef, /*IsElementwise=*/false) {}
+ SyncScope::System, /*IsElementwise=*/false, InsertBef) {}
LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name,
const LoadStoreInstProperties &Props,
InsertPosition InsertBef)
: LoadInst(Ty, Ptr, Name, Props.IsVolatile, Props.Alignment, Props.Ordering,
- Props.SSID, InsertBef) {}
+ Props.SSID, /*IsElementwise=*/false, InsertBef) {}
LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
Align Align, AtomicOrdering Order, SyncScope::ID SSID,
- InsertPosition InsertBef, bool IsElementwise)
+ InsertPosition InsertBef)
+ : LoadInst(Ty, Ptr, Name, isVolatile, Align, Order, SSID,
+ /*IsElementwise=*/false, InsertBef) {}
+
+LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
+ Align Align, AtomicOrdering Order, SyncScope::ID SSID,
+ bool IsElementwise, InsertPosition InsertBef)
: UnaryInstruction(Ty, Load, Ptr, InsertBef) {
setVolatile(isVolatile);
setAlignment(Align);
@@ -4456,7 +4462,8 @@ AllocaInst *AllocaInst::cloneImpl() const {
LoadInst *LoadInst::cloneImpl() const {
return new LoadInst(getType(), getOperand(0), Twine(), isVolatile(),
getAlign(), getOrdering(), getSyncScopeID(),
- /*InsertBefore=*/nullptr, isElementwise());
+ isElementwise(),
+ /*InsertBefore=*/nullptr);
}
StoreInst *StoreInst::cloneImpl() const {
diff --git a/llvm/unittests/IR/VerifierTest.cpp b/llvm/unittests/IR/VerifierTest.cpp
index 190a930626e23..a0ad8ed3bf65e 100644
--- a/llvm/unittests/IR/VerifierTest.cpp
+++ b/llvm/unittests/IR/VerifierTest.cpp
@@ -517,8 +517,8 @@ TEST(VerifierTest, ElementwiseLoadNonAtomic) {
Type *VecTy = FixedVectorType::get(I32Ty, 4);
new LoadInst(VecTy, Ptr, "", /*isVolatile=*/false, Align(4),
- AtomicOrdering::NotAtomic, SyncScope::System, Entry,
- /*IsElementwise=*/true);
+ AtomicOrdering::NotAtomic, SyncScope::System,
+ /*IsElementwise=*/true, Entry);
ReturnInst::Create(C, Entry);
std::string Error;
@@ -540,8 +540,8 @@ TEST(VerifierTest, ElementwiseLoadScalar) {
Type *I32Ty = Type::getInt32Ty(C);
new LoadInst(I32Ty, Ptr, "", /*isVolatile=*/false, Align(4),
- AtomicOrdering::Monotonic, SyncScope::System, Entry,
- /*IsElementwise=*/true);
+ AtomicOrdering::Monotonic, SyncScope::System,
+ /*IsElementwise=*/true, Entry);
ReturnInst::Create(C, Entry);
std::string Error;
@@ -564,8 +564,8 @@ TEST(VerifierTest, ElementwiseLoadOddSizedVector) {
Type *VecTy = FixedVectorType::get(I32Ty, 5);
new LoadInst(VecTy, Ptr, "", /*isVolatile=*/false, Align(4),
- AtomicOrdering::Monotonic, SyncScope::System, Entry,
- /*IsElementwise=*/true);
+ AtomicOrdering::Monotonic, SyncScope::System,
+ /*IsElementwise=*/true, Entry);
ReturnInst::Create(C, Entry);
std::string Error;
More information about the llvm-commits
mailing list