[llvm] [IR] Allow vector atomicrmw xchg (PR #208510)
Yonah Goldberg via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 14 10:50:42 PDT 2026
https://github.com/YonahGoldberg updated https://github.com/llvm/llvm-project/pull/208510
>From 89d6f31eb40d5f7dde98726e553a3170681132fb Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Thu, 9 Jul 2026 17:03:26 +0000
Subject: [PATCH 01/13] relax constraint
---
llvm/docs/LangRef.md | 2 +-
llvm/include/llvm/CodeGen/TargetLowering.h | 5 ++--
llvm/lib/AsmParser/LLParser.cpp | 7 +++--
llvm/lib/CodeGen/AtomicExpandPass.cpp | 8 ++----
llvm/lib/IR/Verifier.cpp | 11 ++++----
llvm/test/Assembler/atomic.ll | 13 +++++++++
.../invalid-atomicrmw-xchg-fp-vector.ll | 7 -----
llvm/test/Bitcode/compatibility.ll | 2 ++
.../AtomicExpand/Mips/atomicrmw-vector.ll | 11 ++++++++
llvm/unittests/IR/VerifierTest.cpp | 28 +++++++++++++++++++
10 files changed, 69 insertions(+), 25 deletions(-)
delete mode 100644 llvm/test/Assembler/invalid-atomicrmw-xchg-fp-vector.ll
diff --git a/llvm/docs/LangRef.md b/llvm/docs/LangRef.md
index aff4297492445..c69d638ed1876 100644
--- a/llvm/docs/LangRef.md
+++ b/llvm/docs/LangRef.md
@@ -12110,7 +12110,7 @@ operation. The operation must be one of the following keywords:
For all of these operations, the type of `<value>` must be a type whose bit width is a power of two greater than or equal to eight.
For add/sub/and/nand/or/xor/max/min/umax/umin/uinc_wrap/udec_wrap/usub_cond/usub_sat, this must be an integer type or a fixed vector of integer type.
For fadd/fsub/fmax/fmin/fmaximum/fminimum/fmaximumnum/fminimumnum, this must be a floating-point or fixed vector of floating-point type.
-For xchg, this must be an integer type, floating-point type, or pointer type, or, if the `elementwise` modifier is present, a fixed vector of integer type, floating-point type, or pointer type.
+For xchg, this must be an integer type, floating-point type, pointer type, or fixed vector of integer type, floating-point type, or pointer type.
The type of the `<pointer>` operand must be a pointer to the type of `<value>`.
If the `atomicrmw` is marked as `volatile`, then the optimizer is not allowed to modify the
number or order of execution of this `atomicrmw` with other
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index f7f14fed05393..4d017c8c6b1be 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -2500,9 +2500,10 @@ class LLVM_ABI TargetLoweringBase {
/// AtomicExpand pass.
virtual AtomicExpansionKind
shouldCastAtomicRMWIInIR(AtomicRMWInst *RMWI) const {
+ Type *ValTy = RMWI->getValOperand()->getType();
if (RMWI->getOperation() == AtomicRMWInst::Xchg &&
- (RMWI->getValOperand()->getType()->isFloatingPointTy() ||
- RMWI->getValOperand()->getType()->isPointerTy()))
+ (ValTy->isFloatingPointTy() || ValTy->isPointerTy() ||
+ ValTy->isVectorTy()))
return AtomicExpansionKind::CastToInteger;
return AtomicExpansionKind::None;
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index 38d10587b104e..4b1764b03c37a 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -9222,12 +9222,13 @@ int LLParser::parseAtomicRMW(Instruction *&Inst, PerFunctionState &PFS) {
}
if (Operation == AtomicRMWInst::Xchg) {
- if (!ScalarTy->isIntegerTy() && !ScalarTy->isFloatingPointTy() &&
- !ScalarTy->isPointerTy()) {
+ if (!ScalarTy->isIntOrIntVectorTy() && !ScalarTy->isFPOrFPVectorTy() &&
+ !ScalarTy->isPtrOrPtrVectorTy()) {
return error(
ValLoc,
"atomicrmw " + AtomicRMWInst::getOperationName(Operation) +
- " operand must be an integer, floating point, or pointer type");
+ " operand must be an integer type, floating point type, pointer type, or fixed "
+ "vector of integer type, floating point type, or pointer type");
}
} else if (IsFP) {
if (!ScalarTy->isFPOrFPVectorTy()) {
diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index 6985a7a48147c..ae3c1d5e4ad0e 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -586,9 +586,7 @@ AtomicExpandImpl::convertAtomicXchgToIntegerType(AtomicRMWInst *RMWI) {
Value *Addr = RMWI->getPointerOperand();
Value *Val = RMWI->getValOperand();
- Value *NewVal = Val->getType()->isPointerTy()
- ? Builder.CreatePtrToInt(Val, NewTy)
- : Builder.CreateBitCast(Val, NewTy);
+ Value *NewVal = Builder.CreateBitOrPointerCast(Val, NewTy);
auto *NewRMWI = Builder.CreateAtomicRMW(AtomicRMWInst::Xchg, Addr, NewVal,
RMWI->getAlign(), RMWI->getOrdering(),
@@ -597,9 +595,7 @@ AtomicExpandImpl::convertAtomicXchgToIntegerType(AtomicRMWInst *RMWI) {
copyMetadataForAtomic(*NewRMWI, *RMWI);
LLVM_DEBUG(dbgs() << "Replaced " << *RMWI << " with " << *NewRMWI << "\n");
- Value *NewRVal = RMWI->getType()->isPointerTy()
- ? Builder.CreateIntToPtr(NewRMWI, RMWI->getType())
- : Builder.CreateBitCast(NewRMWI, RMWI->getType());
+ Value *NewRVal = Builder.CreateBitOrPointerCast(NewRMWI, RMWI->getType());
RMWI->replaceAllUsesWith(NewRVal);
RMWI->eraseFromParent();
return NewRMWI;
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 0744a91710b2e..02263c6536a09 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -4661,20 +4661,19 @@ void Verifier::visitAtomicRMWInst(AtomicRMWInst &RMWI) {
"atomicrmw instructions cannot be unordered.", &RMWI);
auto Op = RMWI.getOperation();
Type *ElTy = RMWI.getOperand(1)->getType();
- Type *ScalarTy = ElTy;
if (RMWI.isElementwise()) {
auto *VecTy = dyn_cast<FixedVectorType>(ElTy);
Check(VecTy, "atomicrmw elementwise operand must have fixed vector type!",
&RMWI, ElTy);
- if (VecTy)
- ScalarTy = VecTy->getElementType();
}
if (Op == AtomicRMWInst::Xchg) {
- Check(ScalarTy->isIntegerTy() || ScalarTy->isFloatingPointTy() ||
- ScalarTy->isPointerTy(),
+ Check((ElTy->isIntOrIntVectorTy() || ElTy->isFPOrFPVectorTy() ||
+ ElTy->isPtrOrPtrVectorTy()) &&
+ !isa<ScalableVectorType>(ElTy),
"atomicrmw " + AtomicRMWInst::getOperationName(Op) +
- " operand must have integer or floating point type!",
+ " operand must have integer type, floating point type, pointer type, or fixed "
+ "vector of integer type, floating point type, or pointer type!",
&RMWI, ElTy);
} else if (AtomicRMWInst::isFPOperation(Op)) {
Check(ElTy->isFPOrFPVectorTy() && !isa<ScalableVectorType>(ElTy),
diff --git a/llvm/test/Assembler/atomic.ll b/llvm/test/Assembler/atomic.ll
index 609cd33f61b88..82bbbe1905c62 100644
--- a/llvm/test/Assembler/atomic.ll
+++ b/llvm/test/Assembler/atomic.ll
@@ -160,6 +160,19 @@ define void @fp_vector_atomicrmw(ptr %x, <2 x half> %val) {
ret void
}
+define void @vector_atomicrmw_xchg(ptr %x, <2 x i16> %ival, <2 x half> %fval, <2 x ptr> %pval) {
+ ; CHECK: %atomic.xchg.int = atomicrmw xchg ptr %x, <2 x i16> %ival seq_cst
+ %atomic.xchg.int = atomicrmw xchg ptr %x, <2 x i16> %ival seq_cst
+
+ ; CHECK: %atomic.xchg.fp = atomicrmw xchg ptr %x, <2 x half> %fval seq_cst
+ %atomic.xchg.fp = atomicrmw xchg ptr %x, <2 x half> %fval seq_cst
+
+ ; CHECK: %atomic.xchg.ptr = atomicrmw xchg ptr %x, <2 x ptr> %pval seq_cst
+ %atomic.xchg.ptr = atomicrmw xchg ptr %x, <2 x ptr> %pval seq_cst
+
+ ret void
+}
+
define void @int_vector_atomicrmw(ptr %x, <2 x i16> %val) {
; CHECK: %atomic.add = atomicrmw add ptr %x, <2 x i16> %val seq_cst
%atomic.add = atomicrmw add ptr %x, <2 x i16> %val seq_cst
diff --git a/llvm/test/Assembler/invalid-atomicrmw-xchg-fp-vector.ll b/llvm/test/Assembler/invalid-atomicrmw-xchg-fp-vector.ll
deleted file mode 100644
index ea523255ee774..0000000000000
--- a/llvm/test/Assembler/invalid-atomicrmw-xchg-fp-vector.ll
+++ /dev/null
@@ -1,7 +0,0 @@
-; RUN: not llvm-as -disable-output %s 2>&1 | FileCheck %s
-
-; CHECK: error: atomicrmw xchg operand must be an integer, floating point, or pointer type
-define <2 x half> @fp_vector_atomicrmw(ptr %x, <2 x half> %val) {
- %atomic.xchg = atomicrmw xchg ptr %x, <2 x half> %val seq_cst
- ret <2 x half> %atomic.xchg
-}
diff --git a/llvm/test/Bitcode/compatibility.ll b/llvm/test/Bitcode/compatibility.ll
index 0ac6da0c9ed29..a200257065dd5 100644
--- a/llvm/test/Bitcode/compatibility.ll
+++ b/llvm/test/Bitcode/compatibility.ll
@@ -864,6 +864,8 @@ define void @atomics(ptr %word) {
;; Atomic w/o alignment
%atomicrmw_no_align.xchg = atomicrmw xchg ptr %word, i32 12 monotonic
; CHECK: %atomicrmw_no_align.xchg = atomicrmw xchg ptr %word, i32 12 monotonic
+ %atomicrmw_no_align.vector.xchg = atomicrmw xchg ptr %word, <2 x i16> <i16 12, i16 13> monotonic
+ ; CHECK: %atomicrmw_no_align.vector.xchg = atomicrmw xchg ptr %word, <2 x i16> <i16 12, i16 13> monotonic
%atomicrmw_no_align.add = atomicrmw add ptr %word, i32 13 monotonic
; CHECK: %atomicrmw_no_align.add = atomicrmw add ptr %word, i32 13 monotonic
%atomicrmw_no_align.vector.add = atomicrmw add ptr %word, <2 x i16> <i16 13, i16 14> monotonic
diff --git a/llvm/test/Transforms/AtomicExpand/Mips/atomicrmw-vector.ll b/llvm/test/Transforms/AtomicExpand/Mips/atomicrmw-vector.ll
index c16bde89200ed..c277383074aa3 100644
--- a/llvm/test/Transforms/AtomicExpand/Mips/atomicrmw-vector.ll
+++ b/llvm/test/Transforms/AtomicExpand/Mips/atomicrmw-vector.ll
@@ -23,3 +23,14 @@ define <2 x i16> @test_atomicrmw_add_v2i16(ptr %ptr, <2 x i16> %value) {
%res = atomicrmw add ptr %ptr, <2 x i16> %value seq_cst
ret <2 x i16> %res
}
+
+define <2 x i16> @test_atomicrmw_xchg_v2i16(ptr %ptr, <2 x i16> %value) {
+; CHECK-LABEL: @test_atomicrmw_xchg_v2i16(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <2 x i16> [[VALUE:%.*]] to i32
+; CHECK-NEXT: [[TMP2:%.*]] = atomicrmw xchg ptr [[PTR:%.*]], i32 [[TMP1]] monotonic, align 4
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast i32 [[TMP2]] to <2 x i16>
+; CHECK-NEXT: ret <2 x i16> [[TMP3]]
+;
+ %res = atomicrmw xchg ptr %ptr, <2 x i16> %value monotonic
+ ret <2 x i16> %res
+}
diff --git a/llvm/unittests/IR/VerifierTest.cpp b/llvm/unittests/IR/VerifierTest.cpp
index d680dd250d66e..d7d573b1c5807 100644
--- a/llvm/unittests/IR/VerifierTest.cpp
+++ b/llvm/unittests/IR/VerifierTest.cpp
@@ -532,6 +532,34 @@ TEST(VerifierTest, AtomicRMWIntVector) {
<< Error;
}
+TEST(VerifierTest, AtomicRMWXchgVector) {
+ LLVMContext C;
+ Module M("M", C);
+ FunctionType *FTy = FunctionType::get(Type::getVoidTy(C), /*isVarArg=*/false);
+ Function *F = Function::Create(FTy, Function::ExternalLinkage, "foo", M);
+ BasicBlock *Entry = BasicBlock::Create(C, "entry", F);
+ Value *Ptr = PoisonValue::get(PointerType::get(C, 0));
+
+ Type *FPTy = Type::getHalfTy(C);
+ Constant *CF = ConstantFP::getZero(FPTy);
+
+ // Invalid scalable type : atomicrmw xchg (<vscale x 2 x half>)
+ Constant *CV = ConstantVector::getSplat(ElementCount::getScalable(2), CF);
+ new AtomicRMWInst(AtomicRMWInst::Xchg, Ptr, CV, Align(8),
+ AtomicOrdering::SequentiallyConsistent, SyncScope::System,
+ /*Elementwise=*/false, Entry);
+ ReturnInst::Create(C, Entry);
+
+ std::string Error;
+ raw_string_ostream ErrorOS(Error);
+ EXPECT_TRUE(verifyFunction(*F, &ErrorOS));
+ EXPECT_TRUE(StringRef(Error).starts_with(
+ "atomicrmw xchg operand must have integer type, floating point type, "
+ "pointer type, or fixed vector of integer type, floating point type, or "
+ "pointer type!"))
+ << Error;
+}
+
TEST(VerifierTest, GetElementPtrInst) {
LLVMContext C;
Module M("M", C);
>From f32f77d937df77b3ec820ba7e710aeb1a560bf0d Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Thu, 9 Jul 2026 17:13:17 +0000
Subject: [PATCH 02/13] format
---
llvm/lib/AsmParser/LLParser.cpp | 3 ++-
llvm/lib/IR/Verifier.cpp | 3 ++-
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index 4b1764b03c37a..7a614797acefd 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -9227,7 +9227,8 @@ int LLParser::parseAtomicRMW(Instruction *&Inst, PerFunctionState &PFS) {
return error(
ValLoc,
"atomicrmw " + AtomicRMWInst::getOperationName(Operation) +
- " operand must be an integer type, floating point type, pointer type, or fixed "
+ " operand must be an integer type, floating point type, pointer "
+ "type, or fixed "
"vector of integer type, floating point type, or pointer type");
}
} else if (IsFP) {
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 02263c6536a09..e173f67be6ee2 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -4672,7 +4672,8 @@ void Verifier::visitAtomicRMWInst(AtomicRMWInst &RMWI) {
ElTy->isPtrOrPtrVectorTy()) &&
!isa<ScalableVectorType>(ElTy),
"atomicrmw " + AtomicRMWInst::getOperationName(Op) +
- " operand must have integer type, floating point type, pointer type, or fixed "
+ " operand must have integer type, floating point type, pointer "
+ "type, or fixed "
"vector of integer type, floating point type, or pointer type!",
&RMWI, ElTy);
} else if (AtomicRMWInst::isFPOperation(Op)) {
>From 2849aff55e774261c0e45c890737ae440a38a877 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Thu, 9 Jul 2026 19:17:19 +0000
Subject: [PATCH 03/13] fix LL parser
---
llvm/lib/AsmParser/LLParser.cpp | 19 +++++++------------
1 file changed, 7 insertions(+), 12 deletions(-)
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index 7a614797acefd..b0876fdddb3ad 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -9209,21 +9209,16 @@ int LLParser::parseAtomicRMW(Instruction *&Inst, PerFunctionState &PFS) {
if (Val->getType()->isScalableTy())
return error(ValLoc, "atomicrmw operand may not be scalable");
- // For elementwise ops, the value must be a fixed vector type whose element
- // type is legal for the corresponding scalar atomicrmw operation. So assign
- // ScalarTy the element type for elementwise ops so we can check this.
- Type *ScalarTy = Val->getType();
+ Type *ValTy = Val->getType();
if (IsElementwise) {
- auto *VecTy = dyn_cast<FixedVectorType>(Val->getType());
- if (!VecTy)
+ if (!isa<FixedVectorType>(Val->getType()))
return error(ValLoc,
"atomicrmw elementwise operand must be a fixed vector type");
- ScalarTy = VecTy->getElementType();
}
if (Operation == AtomicRMWInst::Xchg) {
- if (!ScalarTy->isIntOrIntVectorTy() && !ScalarTy->isFPOrFPVectorTy() &&
- !ScalarTy->isPtrOrPtrVectorTy()) {
+ if (!ValTy->isIntOrIntVectorTy() && !ValTy->isFPOrFPVectorTy() &&
+ !ValTy->isPtrOrPtrVectorTy()) {
return error(
ValLoc,
"atomicrmw " + AtomicRMWInst::getOperationName(Operation) +
@@ -9232,14 +9227,14 @@ int LLParser::parseAtomicRMW(Instruction *&Inst, PerFunctionState &PFS) {
"vector of integer type, floating point type, or pointer type");
}
} else if (IsFP) {
- if (!ScalarTy->isFPOrFPVectorTy()) {
+ if (!ValTy->isFPOrFPVectorTy()) {
return error(ValLoc, "atomicrmw " +
AtomicRMWInst::getOperationName(Operation) +
" operand must be a floating point or fixed "
"vector of floating point type");
}
} else {
- if (!ScalarTy->isIntOrIntVectorTy()) {
+ if (!ValTy->isIntOrIntVectorTy()) {
return error(
ValLoc,
"atomicrmw " + AtomicRMWInst::getOperationName(Operation) +
@@ -9248,7 +9243,7 @@ int LLParser::parseAtomicRMW(Instruction *&Inst, PerFunctionState &PFS) {
}
unsigned Size =
- PFS.getFunction().getDataLayout().getTypeStoreSizeInBits(Val->getType());
+ PFS.getFunction().getDataLayout().getTypeStoreSizeInBits(ValTy);
if (Size < 8 || (Size & (Size - 1)))
return error(ValLoc,
"atomicrmw operand must have a power-of-two byte size");
>From fe9a66a1c59c649865f9e76063e4bea32bb7c43d Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Thu, 9 Jul 2026 20:25:31 +0000
Subject: [PATCH 04/13] address comments
---
llvm/docs/LangRef.md | 2 +-
llvm/lib/AsmParser/LLParser.cpp | 5 ++---
llvm/lib/IR/Verifier.cpp | 5 ++---
llvm/test/Assembler/invalid-atomicrmw-xchg-struct.ll | 7 +++++++
llvm/unittests/IR/VerifierTest.cpp | 5 ++---
5 files changed, 14 insertions(+), 10 deletions(-)
create mode 100644 llvm/test/Assembler/invalid-atomicrmw-xchg-struct.ll
diff --git a/llvm/docs/LangRef.md b/llvm/docs/LangRef.md
index c69d638ed1876..dbdf0cf17a71c 100644
--- a/llvm/docs/LangRef.md
+++ b/llvm/docs/LangRef.md
@@ -12110,7 +12110,7 @@ operation. The operation must be one of the following keywords:
For all of these operations, the type of `<value>` must be a type whose bit width is a power of two greater than or equal to eight.
For add/sub/and/nand/or/xor/max/min/umax/umin/uinc_wrap/udec_wrap/usub_cond/usub_sat, this must be an integer type or a fixed vector of integer type.
For fadd/fsub/fmax/fmin/fmaximum/fminimum/fmaximumnum/fminimumnum, this must be a floating-point or fixed vector of floating-point type.
-For xchg, this must be an integer type, floating-point type, pointer type, or fixed vector of integer type, floating-point type, or pointer type.
+For xchg, this must be an integer type, a floating-point type, a pointer type, or a fixed vector of any of these types.
The type of the `<pointer>` operand must be a pointer to the type of `<value>`.
If the `atomicrmw` is marked as `volatile`, then the optimizer is not allowed to modify the
number or order of execution of this `atomicrmw` with other
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index b0876fdddb3ad..e089f7848b475 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -9222,9 +9222,8 @@ int LLParser::parseAtomicRMW(Instruction *&Inst, PerFunctionState &PFS) {
return error(
ValLoc,
"atomicrmw " + AtomicRMWInst::getOperationName(Operation) +
- " operand must be an integer type, floating point type, pointer "
- "type, or fixed "
- "vector of integer type, floating point type, or pointer type");
+ " operand must be an integer type, a floating-point type, a "
+ "pointer type, or a fixed vector of any of these types");
}
} else if (IsFP) {
if (!ValTy->isFPOrFPVectorTy()) {
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index e173f67be6ee2..a6d0fed42d3f4 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -4672,9 +4672,8 @@ void Verifier::visitAtomicRMWInst(AtomicRMWInst &RMWI) {
ElTy->isPtrOrPtrVectorTy()) &&
!isa<ScalableVectorType>(ElTy),
"atomicrmw " + AtomicRMWInst::getOperationName(Op) +
- " operand must have integer type, floating point type, pointer "
- "type, or fixed "
- "vector of integer type, floating point type, or pointer type!",
+ " operand must be an integer type, a floating-point type, a "
+ "pointer type, or a fixed vector of any of these types!",
&RMWI, ElTy);
} else if (AtomicRMWInst::isFPOperation(Op)) {
Check(ElTy->isFPOrFPVectorTy() && !isa<ScalableVectorType>(ElTy),
diff --git a/llvm/test/Assembler/invalid-atomicrmw-xchg-struct.ll b/llvm/test/Assembler/invalid-atomicrmw-xchg-struct.ll
new file mode 100644
index 0000000000000..3fe6583c867d3
--- /dev/null
+++ b/llvm/test/Assembler/invalid-atomicrmw-xchg-struct.ll
@@ -0,0 +1,7 @@
+; RUN: not llvm-as -disable-output %s 2>&1 | FileCheck %s
+
+; CHECK: error: atomicrmw xchg operand must be an integer type, a floating-point type, a pointer type, or a fixed vector of any of these types
+define void @f(ptr %ptr) {
+ atomicrmw xchg ptr %ptr, { i32 } zeroinitializer seq_cst
+ ret void
+}
diff --git a/llvm/unittests/IR/VerifierTest.cpp b/llvm/unittests/IR/VerifierTest.cpp
index d7d573b1c5807..2acfa3582670a 100644
--- a/llvm/unittests/IR/VerifierTest.cpp
+++ b/llvm/unittests/IR/VerifierTest.cpp
@@ -554,9 +554,8 @@ TEST(VerifierTest, AtomicRMWXchgVector) {
raw_string_ostream ErrorOS(Error);
EXPECT_TRUE(verifyFunction(*F, &ErrorOS));
EXPECT_TRUE(StringRef(Error).starts_with(
- "atomicrmw xchg operand must have integer type, floating point type, "
- "pointer type, or fixed vector of integer type, floating point type, or "
- "pointer type!"))
+ "atomicrmw xchg operand must be an integer type, a floating-point "
+ "type, a pointer type, or a fixed vector of any of these types!"))
<< Error;
}
>From b699236b11c07a2612ca85988da8e0805ce93cfe Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Thu, 9 Jul 2026 20:34:17 +0000
Subject: [PATCH 05/13] require fixed vector type
---
llvm/lib/AsmParser/LLParser.cpp | 2 +-
llvm/lib/IR/Verifier.cpp | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index e089f7848b475..2d7813458c8fa 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -9218,7 +9218,7 @@ int LLParser::parseAtomicRMW(Instruction *&Inst, PerFunctionState &PFS) {
if (Operation == AtomicRMWInst::Xchg) {
if (!ValTy->isIntOrIntVectorTy() && !ValTy->isFPOrFPVectorTy() &&
- !ValTy->isPtrOrPtrVectorTy()) {
+ !ValTy->isPtrOrPtrVectorTy() && isa<FixedVectorType>(ValTy)) {
return error(
ValLoc,
"atomicrmw " + AtomicRMWInst::getOperationName(Operation) +
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index a6d0fed42d3f4..40d91922efbbc 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -4670,7 +4670,7 @@ void Verifier::visitAtomicRMWInst(AtomicRMWInst &RMWI) {
if (Op == AtomicRMWInst::Xchg) {
Check((ElTy->isIntOrIntVectorTy() || ElTy->isFPOrFPVectorTy() ||
ElTy->isPtrOrPtrVectorTy()) &&
- !isa<ScalableVectorType>(ElTy),
+ isa<FixedVectorType>(ElTy),
"atomicrmw " + AtomicRMWInst::getOperationName(Op) +
" operand must be an integer type, a floating-point type, a "
"pointer type, or a fixed vector of any of these types!",
>From c0399a23bd5eb3d07d342a7ab2054e2cd4883f8d Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Thu, 9 Jul 2026 20:42:17 +0000
Subject: [PATCH 06/13] revert incorrect bug fix
---
llvm/lib/AsmParser/LLParser.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index 2d7813458c8fa..e089f7848b475 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -9218,7 +9218,7 @@ int LLParser::parseAtomicRMW(Instruction *&Inst, PerFunctionState &PFS) {
if (Operation == AtomicRMWInst::Xchg) {
if (!ValTy->isIntOrIntVectorTy() && !ValTy->isFPOrFPVectorTy() &&
- !ValTy->isPtrOrPtrVectorTy() && isa<FixedVectorType>(ValTy)) {
+ !ValTy->isPtrOrPtrVectorTy()) {
return error(
ValLoc,
"atomicrmw " + AtomicRMWInst::getOperationName(Operation) +
>From 73e95f93c52b2a497e1846432c5e85d3f4362807 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Fri, 10 Jul 2026 01:00:32 +0000
Subject: [PATCH 07/13] refactor
---
llvm/lib/IR/Verifier.cpp | 9 +++++----
llvm/unittests/IR/VerifierTest.cpp | 13 +++++--------
2 files changed, 10 insertions(+), 12 deletions(-)
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 40d91922efbbc..c0b691078b88b 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -4661,6 +4661,8 @@ void Verifier::visitAtomicRMWInst(AtomicRMWInst &RMWI) {
"atomicrmw instructions cannot be unordered.", &RMWI);
auto Op = RMWI.getOperation();
Type *ElTy = RMWI.getOperand(1)->getType();
+ Check(!ElTy->isScalableTy(), "atomicrmw operand may not be scalable",
+ &RMWI);
if (RMWI.isElementwise()) {
auto *VecTy = dyn_cast<FixedVectorType>(ElTy);
Check(VecTy, "atomicrmw elementwise operand must have fixed vector type!",
@@ -4669,21 +4671,20 @@ void Verifier::visitAtomicRMWInst(AtomicRMWInst &RMWI) {
if (Op == AtomicRMWInst::Xchg) {
Check((ElTy->isIntOrIntVectorTy() || ElTy->isFPOrFPVectorTy() ||
- ElTy->isPtrOrPtrVectorTy()) &&
- isa<FixedVectorType>(ElTy),
+ ElTy->isPtrOrPtrVectorTy()),
"atomicrmw " + AtomicRMWInst::getOperationName(Op) +
" operand must be an integer type, a floating-point type, a "
"pointer type, or a fixed vector of any of these types!",
&RMWI, ElTy);
} else if (AtomicRMWInst::isFPOperation(Op)) {
- Check(ElTy->isFPOrFPVectorTy() && !isa<ScalableVectorType>(ElTy),
+ Check(ElTy->isFPOrFPVectorTy(),
"atomicrmw " + AtomicRMWInst::getOperationName(Op) +
" operand must have floating-point or fixed vector of "
"floating-point "
"type!",
&RMWI, ElTy);
} else {
- Check(ElTy->isIntOrIntVectorTy() && !isa<ScalableVectorType>(ElTy),
+ Check(ElTy->isIntOrIntVectorTy(),
"atomicrmw " + AtomicRMWInst::getOperationName(Op) +
" operand must have integer or fixed vector of integer type!",
&RMWI, ElTy);
diff --git a/llvm/unittests/IR/VerifierTest.cpp b/llvm/unittests/IR/VerifierTest.cpp
index 2acfa3582670a..b81c15bccf39d 100644
--- a/llvm/unittests/IR/VerifierTest.cpp
+++ b/llvm/unittests/IR/VerifierTest.cpp
@@ -399,9 +399,8 @@ TEST(VerifierTest, AtomicRMW) {
std::string Error;
raw_string_ostream ErrorOS(Error);
EXPECT_TRUE(verifyFunction(*F, &ErrorOS));
- EXPECT_TRUE(StringRef(Error).starts_with(
- "atomicrmw fadd operand must have floating-point or "
- "fixed vector of floating-point type!"))
+ EXPECT_TRUE(
+ StringRef(Error).starts_with("atomicrmw operand may not be scalable"))
<< Error;
}
@@ -527,8 +526,7 @@ TEST(VerifierTest, AtomicRMWIntVector) {
raw_string_ostream ErrorOS(Error);
EXPECT_TRUE(verifyFunction(*F, &ErrorOS));
EXPECT_TRUE(
- StringRef(Error).starts_with("atomicrmw add operand must have integer or "
- "fixed vector of integer type!"))
+ StringRef(Error).starts_with("atomicrmw operand may not be scalable"))
<< Error;
}
@@ -553,9 +551,8 @@ TEST(VerifierTest, AtomicRMWXchgVector) {
std::string Error;
raw_string_ostream ErrorOS(Error);
EXPECT_TRUE(verifyFunction(*F, &ErrorOS));
- EXPECT_TRUE(StringRef(Error).starts_with(
- "atomicrmw xchg operand must be an integer type, a floating-point "
- "type, a pointer type, or a fixed vector of any of these types!"))
+ EXPECT_TRUE(
+ StringRef(Error).starts_with("atomicrmw operand may not be scalable"))
<< Error;
}
>From 7ccb847cc23b3cffd32db18da0749ef7b4cd4dfc Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Fri, 10 Jul 2026 01:00:44 +0000
Subject: [PATCH 08/13] format
---
llvm/lib/IR/Verifier.cpp | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index c0b691078b88b..ba410be1244ca 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -4661,8 +4661,7 @@ void Verifier::visitAtomicRMWInst(AtomicRMWInst &RMWI) {
"atomicrmw instructions cannot be unordered.", &RMWI);
auto Op = RMWI.getOperation();
Type *ElTy = RMWI.getOperand(1)->getType();
- Check(!ElTy->isScalableTy(), "atomicrmw operand may not be scalable",
- &RMWI);
+ Check(!ElTy->isScalableTy(), "atomicrmw operand may not be scalable", &RMWI);
if (RMWI.isElementwise()) {
auto *VecTy = dyn_cast<FixedVectorType>(ElTy);
Check(VecTy, "atomicrmw elementwise operand must have fixed vector type!",
>From d0443d98730b568b8f7b2f4586d39475a469a839 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Fri, 10 Jul 2026 01:22:46 +0000
Subject: [PATCH 09/13] revert atomic expand unnecessary change
---
llvm/lib/CodeGen/AtomicExpandPass.cpp | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index ae3c1d5e4ad0e..04c62a5e8123b 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -586,7 +586,9 @@ AtomicExpandImpl::convertAtomicXchgToIntegerType(AtomicRMWInst *RMWI) {
Value *Addr = RMWI->getPointerOperand();
Value *Val = RMWI->getValOperand();
- Value *NewVal = Builder.CreateBitOrPointerCast(Val, NewTy);
+ Value *NewVal = Val->getType()->isPointerTy()
+ ? Builder.CreatePtrToInt(Val, NewTy)
+ : Builder.CreateBitCast(Val, NewTy);
auto *NewRMWI = Builder.CreateAtomicRMW(AtomicRMWInst::Xchg, Addr, NewVal,
RMWI->getAlign(), RMWI->getOrdering(),
@@ -594,8 +596,10 @@ AtomicExpandImpl::convertAtomicXchgToIntegerType(AtomicRMWInst *RMWI) {
NewRMWI->setVolatile(RMWI->isVolatile());
copyMetadataForAtomic(*NewRMWI, *RMWI);
LLVM_DEBUG(dbgs() << "Replaced " << *RMWI << " with " << *NewRMWI << "\n");
-
- Value *NewRVal = Builder.CreateBitOrPointerCast(NewRMWI, RMWI->getType());
+
+ Value *NewRVal = Val->getType()->isPointerTy()
+ ? Builder.CreatePtrToInt(Val, NewTy)
+ : Builder.CreateBitCast(Val, NewTy);
RMWI->replaceAllUsesWith(NewRVal);
RMWI->eraseFromParent();
return NewRMWI;
>From e98957d0926ea5d8d44a60027f10ecf4b0692a87 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Fri, 10 Jul 2026 01:23:04 +0000
Subject: [PATCH 10/13] format
---
llvm/lib/CodeGen/AtomicExpandPass.cpp | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index 04c62a5e8123b..0fb1127bbfb63 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -587,8 +587,8 @@ AtomicExpandImpl::convertAtomicXchgToIntegerType(AtomicRMWInst *RMWI) {
Value *Addr = RMWI->getPointerOperand();
Value *Val = RMWI->getValOperand();
Value *NewVal = Val->getType()->isPointerTy()
- ? Builder.CreatePtrToInt(Val, NewTy)
- : Builder.CreateBitCast(Val, NewTy);
+ ? Builder.CreatePtrToInt(Val, NewTy)
+ : Builder.CreateBitCast(Val, NewTy);
auto *NewRMWI = Builder.CreateAtomicRMW(AtomicRMWInst::Xchg, Addr, NewVal,
RMWI->getAlign(), RMWI->getOrdering(),
@@ -596,10 +596,10 @@ AtomicExpandImpl::convertAtomicXchgToIntegerType(AtomicRMWInst *RMWI) {
NewRMWI->setVolatile(RMWI->isVolatile());
copyMetadataForAtomic(*NewRMWI, *RMWI);
LLVM_DEBUG(dbgs() << "Replaced " << *RMWI << " with " << *NewRMWI << "\n");
-
+
Value *NewRVal = Val->getType()->isPointerTy()
- ? Builder.CreatePtrToInt(Val, NewTy)
- : Builder.CreateBitCast(Val, NewTy);
+ ? Builder.CreatePtrToInt(Val, NewTy)
+ : Builder.CreateBitCast(Val, NewTy);
RMWI->replaceAllUsesWith(NewRVal);
RMWI->eraseFromParent();
return NewRMWI;
>From 2b58661317704d88aed0db44a2718385e6819e18 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Fri, 10 Jul 2026 01:25:19 +0000
Subject: [PATCH 11/13] fix
---
llvm/lib/CodeGen/AtomicExpandPass.cpp | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index 0fb1127bbfb63..6985a7a48147c 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -597,9 +597,9 @@ AtomicExpandImpl::convertAtomicXchgToIntegerType(AtomicRMWInst *RMWI) {
copyMetadataForAtomic(*NewRMWI, *RMWI);
LLVM_DEBUG(dbgs() << "Replaced " << *RMWI << " with " << *NewRMWI << "\n");
- Value *NewRVal = Val->getType()->isPointerTy()
- ? Builder.CreatePtrToInt(Val, NewTy)
- : Builder.CreateBitCast(Val, NewTy);
+ Value *NewRVal = RMWI->getType()->isPointerTy()
+ ? Builder.CreateIntToPtr(NewRMWI, RMWI->getType())
+ : Builder.CreateBitCast(NewRMWI, RMWI->getType());
RMWI->replaceAllUsesWith(NewRVal);
RMWI->eraseFromParent();
return NewRMWI;
>From dafd17436e1e90a374f71a736a457baa40d7030e Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 14 Jul 2026 15:53:56 +0000
Subject: [PATCH 12/13] fix crash
---
llvm/lib/CodeGen/AtomicExpandPass.cpp | 8 ++-----
.../AtomicExpand/Mips/atomicrmw-vector.ll | 24 +++++++++++++++++++
2 files changed, 26 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index 6985a7a48147c..2750ad16b23f1 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -586,9 +586,7 @@ AtomicExpandImpl::convertAtomicXchgToIntegerType(AtomicRMWInst *RMWI) {
Value *Addr = RMWI->getPointerOperand();
Value *Val = RMWI->getValOperand();
- Value *NewVal = Val->getType()->isPointerTy()
- ? Builder.CreatePtrToInt(Val, NewTy)
- : Builder.CreateBitCast(Val, NewTy);
+ Value *NewVal = Builder.CreateBitPreservingCastChain(*DL, Val, NewTy);
auto *NewRMWI = Builder.CreateAtomicRMW(AtomicRMWInst::Xchg, Addr, NewVal,
RMWI->getAlign(), RMWI->getOrdering(),
@@ -597,9 +595,7 @@ AtomicExpandImpl::convertAtomicXchgToIntegerType(AtomicRMWInst *RMWI) {
copyMetadataForAtomic(*NewRMWI, *RMWI);
LLVM_DEBUG(dbgs() << "Replaced " << *RMWI << " with " << *NewRMWI << "\n");
- Value *NewRVal = RMWI->getType()->isPointerTy()
- ? Builder.CreateIntToPtr(NewRMWI, RMWI->getType())
- : Builder.CreateBitCast(NewRMWI, RMWI->getType());
+ Value *NewRVal = Builder.CreateBitPreservingCastChain(*DL, NewRMWI, RMWI->getType());
RMWI->replaceAllUsesWith(NewRVal);
RMWI->eraseFromParent();
return NewRMWI;
diff --git a/llvm/test/Transforms/AtomicExpand/Mips/atomicrmw-vector.ll b/llvm/test/Transforms/AtomicExpand/Mips/atomicrmw-vector.ll
index c277383074aa3..bf8c05badd83f 100644
--- a/llvm/test/Transforms/AtomicExpand/Mips/atomicrmw-vector.ll
+++ b/llvm/test/Transforms/AtomicExpand/Mips/atomicrmw-vector.ll
@@ -34,3 +34,27 @@ define <2 x i16> @test_atomicrmw_xchg_v2i16(ptr %ptr, <2 x i16> %value) {
%res = atomicrmw xchg ptr %ptr, <2 x i16> %value monotonic
ret <2 x i16> %res
}
+
+define <2 x half> @test_atomicrmw_xchg_v2f16(ptr %ptr, <2 x half> %value) {
+; CHECK-LABEL: @test_atomicrmw_xchg_v2f16(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <2 x half> [[VALUE:%.*]] to i32
+; CHECK-NEXT: [[TMP2:%.*]] = atomicrmw xchg ptr [[PTR:%.*]], i32 [[TMP1]] monotonic, align 4
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast i32 [[TMP2]] to <2 x half>
+; CHECK-NEXT: ret <2 x half> [[TMP3]]
+;
+ %res = atomicrmw xchg ptr %ptr, <2 x half> %value monotonic
+ ret <2 x half> %res
+}
+
+define <2 x ptr> @test_atomicrmw_xchg_v2p0(ptr %ptr, <2 x ptr> %value) {
+; CHECK-LABEL: @test_atomicrmw_xchg_v2p0(
+; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint <2 x ptr> [[VALUE:%.*]] to <2 x i64>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; CHECK-NEXT: [[TMP3:%.*]] = call i128 @__atomic_exchange_16(ptr [[PTR:%.*]], i128 [[TMP2]], i32 0)
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast i128 [[TMP3]] to <2 x i64>
+; CHECK-NEXT: [[TMP5:%.*]] = inttoptr <2 x i64> [[TMP4]] to <2 x ptr>
+; CHECK-NEXT: ret <2 x ptr> [[TMP5]]
+;
+ %res = atomicrmw xchg ptr %ptr, <2 x ptr> %value monotonic
+ ret <2 x ptr> %res
+}
>From cf7a0d00a93656a3a6b2b9951d4b100fc89081d4 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 14 Jul 2026 15:54:15 +0000
Subject: [PATCH 13/13] format
---
llvm/lib/CodeGen/AtomicExpandPass.cpp | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index 2750ad16b23f1..16abf9a247af8 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -595,7 +595,8 @@ AtomicExpandImpl::convertAtomicXchgToIntegerType(AtomicRMWInst *RMWI) {
copyMetadataForAtomic(*NewRMWI, *RMWI);
LLVM_DEBUG(dbgs() << "Replaced " << *RMWI << " with " << *NewRMWI << "\n");
- Value *NewRVal = Builder.CreateBitPreservingCastChain(*DL, NewRMWI, RMWI->getType());
+ Value *NewRVal =
+ Builder.CreateBitPreservingCastChain(*DL, NewRMWI, RMWI->getType());
RMWI->replaceAllUsesWith(NewRVal);
RMWI->eraseFromParent();
return NewRMWI;
More information about the llvm-commits
mailing list