[llvm] [IR] Allow vector cmpxchg (PR #209386)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 13 23:49:42 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-transforms
@llvm/pr-subscribers-llvm-ir
Author: Yonah Goldberg (YonahGoldberg)
<details>
<summary>Changes</summary>
Previously, we only allowed vector atomicrmw xchg for elementwise atomicrmw. Relax this restriction. By default, expand these by casting to integer.
This is a follow-up on: https://github.com/llvm/llvm-project/pull/190716 and https://github.com/llvm/llvm-project/pull/208510
Assisted by AI.
---
Full diff: https://github.com/llvm/llvm-project/pull/209386.diff
9 Files Affected:
- (modified) llvm/docs/LangRef.md (+3-2)
- (modified) llvm/lib/AsmParser/LLParser.cpp (+8)
- (modified) llvm/lib/CodeGen/AtomicExpandPass.cpp (+11-11)
- (modified) llvm/lib/IR/Verifier.cpp (+6-2)
- (modified) llvm/test/Assembler/atomic.ll (+16)
- (added) llvm/test/Assembler/invalid-cmpxchg-scalable.ll (+25)
- (modified) llvm/test/Bitcode/compatibility.ll (+2)
- (added) llvm/test/Transforms/AtomicExpand/X86/expand-atomic-cmpxchg-non-integer.ll (+56)
- (modified) llvm/unittests/IR/VerifierTest.cpp (+27)
``````````diff
diff --git a/llvm/docs/LangRef.md b/llvm/docs/LangRef.md
index aff4297492445..93256d6bcf55a 100644
--- a/llvm/docs/LangRef.md
+++ b/llvm/docs/LangRef.md
@@ -11997,8 +11997,9 @@ equal, it tries to store a new value into the memory.
There are three arguments to the '`cmpxchg`' instruction: an address
to operate on, a value to compare to the value currently be at that
address, and a new value to place at that address if the compared values
-are equal. The type of `<cmp>` must be an integer or pointer type whose
-bit width is a power of two greater than or equal to eight.
+are equal. The type of `<cmp>` must be an integer type, floating-point type,
+pointer type, or fixed vector of one of these types, whose bit width is a power
+of two greater than or equal to eight.
`<cmp>` and `<new>` must
have the same type, and the type of `<pointer>` must be a pointer to
that type. If the `cmpxchg` is marked as `volatile`, then the
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index 38d10587b104e..2c95dee7a0a52 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -9099,6 +9099,14 @@ int LLParser::parseCmpXchg(Instruction *&Inst, PerFunctionState &PFS) {
return error(NewLoc, "compare value and new value type do not match");
if (!New->getType()->isFirstClassType())
return error(NewLoc, "cmpxchg operand must be a first class value");
+ if (Cmp->getType()->isScalableTy())
+ return error(CmpLoc, "cmpxchg operand may not be scalable");
+ if (!Cmp->getType()->isIntOrIntVectorTy() &&
+ !Cmp->getType()->isFPOrFPVectorTy() &&
+ !Cmp->getType()->isPtrOrPtrVectorTy())
+ return error(CmpLoc,
+ "cmpxchg operand must be an integer, floating point, "
+ "pointer, or fixed vector of one of these types");
const Align DefaultAlignment(
PFS.getFunction().getDataLayout().getTypeStoreSize(
diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index 6985a7a48147c..5a3d45c62d324 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -415,10 +415,8 @@ bool AtomicExpandImpl::processAtomicInstr(Instruction *I) {
return true;
}
- // TODO: when we're ready to make the change at the IR level, we can
- // extend convertCmpXchgToInteger for floating point too.
bool MadeChange = false;
- if (CASI->getCompareOperand()->getType()->isPointerTy()) {
+ if (!CASI->getCompareOperand()->getType()->isIntegerTy()) {
// TODO: add a TLI hook to control this so that each target can
// convert to lowering the original type one at a time.
CASI = convertCmpXchgToIntegerType(CASI);
@@ -1421,11 +1419,10 @@ Value *AtomicExpandImpl::insertRMWLLSCLoop(
return Loaded;
}
-/// Convert an atomic cmpxchg of a non-integral type to an integer cmpxchg of
-/// the equivalent bitwidth. We used to not support pointer cmpxchg in the
-/// IR. As a migration step, we convert back to what use to be the standard
-/// way to represent a pointer cmpxchg so that we can update backends one by
-/// one.
+/// Convert an atomic cmpxchg of a non-integer type to an integer cmpxchg of
+/// the equivalent bitwidth. We used to not support pointer, floating-point, or
+/// vector cmpxchg in the IR. As a migration step, convert back to what used to
+/// be the standard representation so that we can update backends one by one.
AtomicCmpXchgInst *
AtomicExpandImpl::convertCmpXchgToIntegerType(AtomicCmpXchgInst *CI) {
auto *M = CI->getModule();
@@ -1436,8 +1433,10 @@ AtomicExpandImpl::convertCmpXchgToIntegerType(AtomicCmpXchgInst *CI) {
Value *Addr = CI->getPointerOperand();
- Value *NewCmp = Builder.CreatePtrToInt(CI->getCompareOperand(), NewTy);
- Value *NewNewVal = Builder.CreatePtrToInt(CI->getNewValOperand(), NewTy);
+ Value *NewCmp =
+ Builder.CreateBitPreservingCastChain(*DL, CI->getCompareOperand(), NewTy);
+ Value *NewNewVal =
+ Builder.CreateBitPreservingCastChain(*DL, CI->getNewValOperand(), NewTy);
auto *NewCI = Builder.CreateAtomicCmpXchg(
Addr, NewCmp, NewNewVal, CI->getAlign(), CI->getSuccessOrdering(),
@@ -1449,7 +1448,8 @@ AtomicExpandImpl::convertCmpXchgToIntegerType(AtomicCmpXchgInst *CI) {
Value *OldVal = Builder.CreateExtractValue(NewCI, 0);
Value *Succ = Builder.CreateExtractValue(NewCI, 1);
- OldVal = Builder.CreateIntToPtr(OldVal, CI->getCompareOperand()->getType());
+ OldVal = Builder.CreateBitPreservingCastChain(
+ *DL, OldVal, CI->getCompareOperand()->getType());
Value *Res = PoisonValue::get(CI->getType());
Res = Builder.CreateInsertValue(Res, OldVal, 0);
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 0744a91710b2e..f5c257b4577a4 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -4650,8 +4650,12 @@ void Verifier::visitAllocaInst(AllocaInst &AI) {
void Verifier::visitAtomicCmpXchgInst(AtomicCmpXchgInst &CXI) {
Type *ElTy = CXI.getOperand(1)->getType();
- Check(ElTy->isIntOrPtrTy(),
- "cmpxchg operand must have integer or pointer type", ElTy, &CXI);
+ Check((ElTy->isIntOrIntVectorTy() || ElTy->isFPOrFPVectorTy() ||
+ ElTy->isPtrOrPtrVectorTy()) &&
+ !isa<ScalableVectorType>(ElTy),
+ "cmpxchg operand must have integer, floating point, pointer, or fixed "
+ "vector of one of these types",
+ ElTy, &CXI);
checkAtomicMemAccessSize(ElTy, &CXI);
visitInstruction(CXI);
}
diff --git a/llvm/test/Assembler/atomic.ll b/llvm/test/Assembler/atomic.ll
index 609cd33f61b88..981ca280738ef 100644
--- a/llvm/test/Assembler/atomic.ll
+++ b/llvm/test/Assembler/atomic.ll
@@ -80,6 +80,22 @@ define void @f(ptr %x) {
ret void
}
+define void @cmpxchg_non_integer(ptr %x, float %fcmp, float %fnew, <2 x i16> %icmp, <2 x i16> %inew, <2 x half> %vfcmp, <2 x half> %vfnew, <2 x ptr> %pcmp, <2 x ptr> %pnew) {
+ ; CHECK: %atomic.cmpxchg.fp = cmpxchg ptr %x, float %fcmp, float %fnew seq_cst monotonic
+ %atomic.cmpxchg.fp = cmpxchg ptr %x, float %fcmp, float %fnew seq_cst monotonic
+
+ ; CHECK: %atomic.cmpxchg.int.vector = cmpxchg ptr %x, <2 x i16> %icmp, <2 x i16> %inew seq_cst monotonic
+ %atomic.cmpxchg.int.vector = cmpxchg ptr %x, <2 x i16> %icmp, <2 x i16> %inew seq_cst monotonic
+
+ ; CHECK: %atomic.cmpxchg.fp.vector = cmpxchg ptr %x, <2 x half> %vfcmp, <2 x half> %vfnew seq_cst monotonic
+ %atomic.cmpxchg.fp.vector = cmpxchg ptr %x, <2 x half> %vfcmp, <2 x half> %vfnew seq_cst monotonic
+
+ ; CHECK: %atomic.cmpxchg.ptr.vector = cmpxchg ptr %x, <2 x ptr> %pcmp, <2 x ptr> %pnew seq_cst monotonic
+ %atomic.cmpxchg.ptr.vector = cmpxchg ptr %x, <2 x ptr> %pcmp, <2 x ptr> %pnew seq_cst monotonic
+
+ ret void
+}
+
define void @fp_atomics(ptr %x) {
; CHECK: atomicrmw fadd ptr %x, float 1.000000e+00 seq_cst
atomicrmw fadd ptr %x, float 1.0 seq_cst
diff --git a/llvm/test/Assembler/invalid-cmpxchg-scalable.ll b/llvm/test/Assembler/invalid-cmpxchg-scalable.ll
new file mode 100644
index 0000000000000..1d4f135866174
--- /dev/null
+++ b/llvm/test/Assembler/invalid-cmpxchg-scalable.ll
@@ -0,0 +1,25 @@
+; RUN: split-file %s %t --leading-lines
+; RUN: not llvm-as < %t/scalable_int_vector_cmpxchg.ll 2>&1 | FileCheck -check-prefix=ERR0 %s
+; RUN: not llvm-as < %t/scalable_fp_vector_cmpxchg.ll 2>&1 | FileCheck -check-prefix=ERR1 %s
+; RUN: not llvm-as < %t/scalable_ptr_vector_cmpxchg.ll 2>&1 | FileCheck -check-prefix=ERR2 %s
+
+;--- scalable_int_vector_cmpxchg.ll
+define void @scalable_int_vector_cmpxchg(ptr %p, <vscale x 2 x i16> %cmp, <vscale x 2 x i16> %new) {
+; ERR0: error: cmpxchg operand may not be scalable
+ %val_success = cmpxchg ptr %p, <vscale x 2 x i16> %cmp, <vscale x 2 x i16> %new seq_cst monotonic
+ ret void
+}
+
+;--- scalable_fp_vector_cmpxchg.ll
+define void @scalable_fp_vector_cmpxchg(ptr %p, <vscale x 2 x half> %cmp, <vscale x 2 x half> %new) {
+; ERR1: error: cmpxchg operand may not be scalable
+ %val_success = cmpxchg ptr %p, <vscale x 2 x half> %cmp, <vscale x 2 x half> %new seq_cst monotonic
+ ret void
+}
+
+;--- scalable_ptr_vector_cmpxchg.ll
+define void @scalable_ptr_vector_cmpxchg(ptr %p, <vscale x 2 x ptr> %cmp, <vscale x 2 x ptr> %new) {
+; ERR2: error: cmpxchg operand may not be scalable
+ %val_success = cmpxchg ptr %p, <vscale x 2 x ptr> %cmp, <vscale x 2 x ptr> %new seq_cst monotonic
+ ret void
+}
diff --git a/llvm/test/Bitcode/compatibility.ll b/llvm/test/Bitcode/compatibility.ll
index 0ac6da0c9ed29..ebbe405e44a21 100644
--- a/llvm/test/Bitcode/compatibility.ll
+++ b/llvm/test/Bitcode/compatibility.ll
@@ -828,6 +828,8 @@ define void @atomics(ptr %word) {
;; Atomic Compare And Exchange w/o alignment
%cmpxchg_no_align.0 = cmpxchg ptr %word, i32 0, i32 4 monotonic monotonic
; CHECK: %cmpxchg_no_align.0 = cmpxchg ptr %word, i32 0, i32 4 monotonic monotonic
+ %cmpxchg_no_align.vector = cmpxchg ptr %word, <2 x i16> <i16 0, i16 1>, <2 x i16> <i16 4, i16 5> monotonic monotonic
+ ; CHECK: %cmpxchg_no_align.vector = cmpxchg ptr %word, <2 x i16> <i16 0, i16 1>, <2 x i16> <i16 4, i16 5> monotonic monotonic
%cmpxchg_no_align.1 = cmpxchg ptr %word, i32 0, i32 5 acq_rel monotonic
; CHECK: %cmpxchg_no_align.1 = cmpxchg ptr %word, i32 0, i32 5 acq_rel monotonic
%cmpxchg_no_align.2 = cmpxchg ptr %word, i32 0, i32 6 acquire monotonic
diff --git a/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-cmpxchg-non-integer.ll b/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-cmpxchg-non-integer.ll
new file mode 100644
index 0000000000000..f7f92da930440
--- /dev/null
+++ b/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-cmpxchg-non-integer.ll
@@ -0,0 +1,56 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -mtriple=x86_64-linux-gnu -passes='require<libcall-lowering-info>,atomic-expand' %s | FileCheck %s
+
+define { float, i1 } @cmpxchg_float(ptr %ptr, float %cmp, float %new) {
+; CHECK-LABEL: @cmpxchg_float(
+; CHECK-SAME: ptr [[PTR:%.*]], float [[CMP:%.*]], float [[NEW:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast float [[CMP]] to i32
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32
+; CHECK-NEXT: [[TMP3:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP1]], i32 [[TMP2]] seq_cst monotonic, align 4
+; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { i32, i1 } [[TMP3]], 0
+; CHECK-NEXT: [[TMP5:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast i32 [[TMP4]] to float
+; CHECK-NEXT: [[TMP7:%.*]] = insertvalue { float, i1 } poison, float [[TMP6]], 0
+; CHECK-NEXT: [[TMP8:%.*]] = insertvalue { float, i1 } [[TMP7]], i1 [[TMP5]], 1
+; CHECK-NEXT: ret { float, i1 } [[TMP8]]
+;
+ %result = cmpxchg ptr %ptr, float %cmp, float %new seq_cst monotonic
+ ret { float, i1 } %result
+}
+
+define { <2 x i16>, i1 } @cmpxchg_v2i16(ptr %ptr, <2 x i16> %cmp, <2 x i16> %new) {
+; CHECK-LABEL: @cmpxchg_v2i16(
+; CHECK-SAME: ptr [[PTR:%.*]], <2 x i16> [[CMP:%.*]], <2 x i16> [[NEW:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <2 x i16> [[CMP]] to i32
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x i16> [[NEW]] to i32
+; CHECK-NEXT: [[TMP3:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP1]], i32 [[TMP2]] seq_cst monotonic, align 4
+; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { i32, i1 } [[TMP3]], 0
+; CHECK-NEXT: [[TMP5:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast i32 [[TMP4]] to <2 x i16>
+; CHECK-NEXT: [[TMP7:%.*]] = insertvalue { <2 x i16>, i1 } poison, <2 x i16> [[TMP6]], 0
+; CHECK-NEXT: [[TMP8:%.*]] = insertvalue { <2 x i16>, i1 } [[TMP7]], i1 [[TMP5]], 1
+; CHECK-NEXT: ret { <2 x i16>, i1 } [[TMP8]]
+;
+ %result = cmpxchg ptr %ptr, <2 x i16> %cmp, <2 x i16> %new seq_cst monotonic
+ ret { <2 x i16>, i1 } %result
+}
+
+define { <1 x ptr>, i1 } @cmpxchg_v1ptr(ptr %ptr, <1 x ptr> %cmp, <1 x ptr> %new) {
+; CHECK-LABEL: @cmpxchg_v1ptr(
+; CHECK-SAME: ptr [[PTR:%.*]], <1 x ptr> [[CMP:%.*]], <1 x ptr> [[NEW:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint <1 x ptr> [[CMP]] to <1 x i64>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <1 x i64> [[TMP1]] to i64
+; CHECK-NEXT: [[TMP3:%.*]] = ptrtoint <1 x ptr> [[NEW]] to <1 x i64>
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <1 x i64> [[TMP3]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP2]], i64 [[TMP4]] seq_cst monotonic, align 8
+; CHECK-NEXT: [[TMP6:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP7:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast i64 [[TMP6]] to <1 x i64>
+; CHECK-NEXT: [[TMP9:%.*]] = inttoptr <1 x i64> [[TMP8]] to <1 x ptr>
+; CHECK-NEXT: [[TMP10:%.*]] = insertvalue { <1 x ptr>, i1 } poison, <1 x ptr> [[TMP9]], 0
+; CHECK-NEXT: [[TMP11:%.*]] = insertvalue { <1 x ptr>, i1 } [[TMP10]], i1 [[TMP7]], 1
+; CHECK-NEXT: ret { <1 x ptr>, i1 } [[TMP11]]
+;
+ %result = cmpxchg ptr %ptr, <1 x ptr> %cmp, <1 x ptr> %new seq_cst monotonic
+ ret { <1 x ptr>, i1 } %result
+}
diff --git a/llvm/unittests/IR/VerifierTest.cpp b/llvm/unittests/IR/VerifierTest.cpp
index d680dd250d66e..d40eea6bcbf95 100644
--- a/llvm/unittests/IR/VerifierTest.cpp
+++ b/llvm/unittests/IR/VerifierTest.cpp
@@ -532,6 +532,33 @@ TEST(VerifierTest, AtomicRMWIntVector) {
<< Error;
}
+TEST(VerifierTest, AtomicCmpXchgVector) {
+ LLVMContext C;
+ Module M("M", C);
+ FunctionType *FTy = FunctionType::get(Type::getVoidTy(C), /*isVarArg=*/false);
+ Function *F = Function::Create(FTy, Function::ExternalLinkage, "foo", M);
+ BasicBlock *Entry = BasicBlock::Create(C, "entry", F);
+ Value *Ptr = PoisonValue::get(PointerType::get(C, 0));
+
+ Type *IntTy = Type::getInt16Ty(C);
+ Constant *CI = ConstantInt::get(IntTy, 0);
+
+ // Invalid scalable type : cmpxchg (<vscale x 2 x i16>)
+ Constant *CV = ConstantVector::getSplat(ElementCount::getScalable(2), CI);
+ new AtomicCmpXchgInst(Ptr, CV, CV, Align(8),
+ AtomicOrdering::SequentiallyConsistent,
+ AtomicOrdering::Monotonic, SyncScope::System, Entry);
+ ReturnInst::Create(C, Entry);
+
+ std::string Error;
+ raw_string_ostream ErrorOS(Error);
+ EXPECT_TRUE(verifyFunction(*F, &ErrorOS));
+ EXPECT_TRUE(StringRef(Error).starts_with(
+ "cmpxchg operand must have integer, floating point, pointer, or fixed "
+ "vector of one of these types"))
+ << Error;
+}
+
TEST(VerifierTest, GetElementPtrInst) {
LLVMContext C;
Module M("M", C);
``````````
</details>
https://github.com/llvm/llvm-project/pull/209386
More information about the llvm-commits
mailing list