[llvm] 18e6b82 - Allow pointer types for atomicrmw xchg
James Y Knight via llvm-commits
llvm-commits at lists.llvm.org
Wed May 25 09:24:13 PDT 2022
Author: Takafumi Arakaki
Date: 2022-05-25T16:20:26Z
New Revision: 18e6b8234a0d0b085ead20d9c26777c824bb4a3d
URL: https://github.com/llvm/llvm-project/commit/18e6b8234a0d0b085ead20d9c26777c824bb4a3d
DIFF: https://github.com/llvm/llvm-project/commit/18e6b8234a0d0b085ead20d9c26777c824bb4a3d.diff
LOG: Allow pointer types for atomicrmw xchg
This adds support for pointer types for `atomic xchg` and let us write
instructions such as `atomicrmw xchg i64** %0, i64* %1 seq_cst`. This
is similar to the patch for allowing atomicrmw xchg on floating point
types: https://reviews.llvm.org/D52416.
Differential Revision: https://reviews.llvm.org/D124728
Added:
Modified:
llvm/docs/LangRef.rst
llvm/include/llvm/CodeGen/TargetLowering.h
llvm/lib/AsmParser/LLParser.cpp
llvm/lib/CodeGen/AtomicExpandPass.cpp
llvm/lib/IR/Verifier.cpp
llvm/test/Bitcode/compatibility.ll
llvm/test/CodeGen/AMDGPU/flat_atomics_i64.ll
llvm/test/CodeGen/AMDGPU/global_atomics_i64.ll
llvm/test/CodeGen/AMDGPU/local-atomics64.ll
llvm/test/CodeGen/X86/atomic64.ll
Removed:
llvm/test/Assembler/invalid-atomicrmw-xchg-must-be-integer-or-fp-type.ll
################################################################################
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index f5ba612d2bfda..46549e1175ae6 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -10263,8 +10263,8 @@ operation. The operation must be one of the following keywords:
For most of these operations, the type of '<value>' must be an integer
type whose bit width is a power of two greater than or equal to eight
and less than or equal to a target-specific size limit. For xchg, this
-may also be a floating point type with the same size constraints as
-integers. For fadd/fsub, this must be a floating point type. The
+may also be a floating point or a pointer type with the same size constraints
+as integers. For fadd/fsub, this must be a floating point type. The
type of the '``<pointer>``' operand must be a pointer to that type. If
the ``atomicrmw`` is marked as ``volatile``, then the optimizer is not
allowed to modify the number or order of execution of this
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index a582838ff663e..8ad6c4d58817f 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -2092,7 +2092,8 @@ class TargetLoweringBase {
virtual AtomicExpansionKind
shouldCastAtomicRMWIInIR(AtomicRMWInst *RMWI) const {
if (RMWI->getOperation() == AtomicRMWInst::Xchg &&
- RMWI->getValOperand()->getType()->isFloatingPointTy())
+ (RMWI->getValOperand()->getType()->isFloatingPointTy() ||
+ RMWI->getValOperand()->getType()->isPointerTy()))
return AtomicExpansionKind::CastToInteger;
return AtomicExpansionKind::None;
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index 5076c0bff44dd..1f8d68cca3dac 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -7436,10 +7436,12 @@ int LLParser::parseAtomicRMW(Instruction *&Inst, PerFunctionState &PFS) {
if (Operation == AtomicRMWInst::Xchg) {
if (!Val->getType()->isIntegerTy() &&
- !Val->getType()->isFloatingPointTy()) {
- return error(ValLoc,
- "atomicrmw " + AtomicRMWInst::getOperationName(Operation) +
- " operand must be an integer or floating point type");
+ !Val->getType()->isFloatingPointTy() &&
+ !Val->getType()->isPointerTy()) {
+ return error(
+ ValLoc,
+ "atomicrmw " + AtomicRMWInst::getOperationName(Operation) +
+ " operand must be an integer, floating point, or pointer type");
}
} else if (IsFP) {
if (!Val->getType()->isFloatingPointTy()) {
@@ -7455,7 +7457,9 @@ int LLParser::parseAtomicRMW(Instruction *&Inst, PerFunctionState &PFS) {
}
}
- unsigned Size = Val->getType()->getPrimitiveSizeInBits();
+ unsigned Size =
+ PFS.getFunction().getParent()->getDataLayout().getTypeStoreSizeInBits(
+ Val->getType());
if (Size < 8 || (Size & (Size - 1)))
return error(ValLoc, "atomicrmw operand must be power-of-two byte-sized"
" integer");
diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index 2f4eb1f21fbf2..5ce6fbb5f6471 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -387,7 +387,9 @@ AtomicExpand::convertAtomicXchgToIntegerType(AtomicRMWInst *RMWI) {
Value *Val = RMWI->getValOperand();
Type *PT = PointerType::get(NewTy, RMWI->getPointerAddressSpace());
Value *NewAddr = Builder.CreateBitCast(Addr, PT);
- Value *NewVal = Builder.CreateBitCast(Val, NewTy);
+ Value *NewVal = Val->getType()->isPointerTy()
+ ? Builder.CreatePtrToInt(Val, NewTy)
+ : Builder.CreateBitCast(Val, NewTy);
auto *NewRMWI =
Builder.CreateAtomicRMW(AtomicRMWInst::Xchg, NewAddr, NewVal,
@@ -395,7 +397,9 @@ AtomicExpand::convertAtomicXchgToIntegerType(AtomicRMWInst *RMWI) {
NewRMWI->setVolatile(RMWI->isVolatile());
LLVM_DEBUG(dbgs() << "Replaced " << *RMWI << " with " << *NewRMWI << "\n");
- Value *NewRVal = Builder.CreateBitCast(NewRMWI, RMWI->getType());
+ Value *NewRVal = RMWI->getType()->isPointerTy()
+ ? Builder.CreateIntToPtr(NewRMWI, RMWI->getType())
+ : Builder.CreateBitCast(NewRMWI, RMWI->getType());
RMWI->replaceAllUsesWith(NewRVal);
RMWI->eraseFromParent();
return NewRMWI;
@@ -527,6 +531,7 @@ static void createCmpXchgInstFun(IRBuilder<> &Builder, Value *Addr,
Type *OrigTy = NewVal->getType();
// This code can go away when cmpxchg supports FP types.
+ assert(!OrigTy->isPointerTy());
bool NeedBitcast = OrigTy->isFloatingPointTy();
if (NeedBitcast) {
IntegerType *IntTy = Builder.getIntNTy(OrigTy->getPrimitiveSizeInBits());
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 09d47954c05e7..93063a50b39b7 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -3925,7 +3925,8 @@ void Verifier::visitAtomicRMWInst(AtomicRMWInst &RMWI) {
auto Op = RMWI.getOperation();
Type *ElTy = RMWI.getOperand(1)->getType();
if (Op == AtomicRMWInst::Xchg) {
- Check(ElTy->isIntegerTy() || ElTy->isFloatingPointTy(),
+ Check(ElTy->isIntegerTy() || ElTy->isFloatingPointTy() ||
+ ElTy->isPointerTy(),
"atomicrmw " + AtomicRMWInst::getOperationName(Op) +
" operand must have integer or floating point type!",
&RMWI, ElTy);
diff --git a/llvm/test/Assembler/invalid-atomicrmw-xchg-must-be-integer-or-fp-type.ll b/llvm/test/Assembler/invalid-atomicrmw-xchg-must-be-integer-or-fp-type.ll
deleted file mode 100644
index d0a794bd0244a..0000000000000
--- a/llvm/test/Assembler/invalid-atomicrmw-xchg-must-be-integer-or-fp-type.ll
+++ /dev/null
@@ -1,7 +0,0 @@
-; RUN: not llvm-as -disable-output %s 2>&1 | FileCheck %s
-
-; CHECK: error: atomicrmw xchg operand must be an integer or floating point type
-define void @f(i32** %ptr) {
- atomicrmw xchg i32** %ptr, i32* null seq_cst
- ret void
-}
diff --git a/llvm/test/Bitcode/compatibility.ll b/llvm/test/Bitcode/compatibility.ll
index ab9c3b1c61ab7..a87e11e0fe0d6 100644
--- a/llvm/test/Bitcode/compatibility.ll
+++ b/llvm/test/Bitcode/compatibility.ll
@@ -854,6 +854,12 @@ define void @fp_atomics(float* %word) {
ret void
}
+define void @pointer_atomics(i8** %word) {
+; CHECK: %atomicrmw.xchg = atomicrmw xchg i8** %word, i8* null monotonic
+ %atomicrmw.xchg = atomicrmw xchg i8** %word, i8* null monotonic
+ ret void
+}
+
;; Fast Math Flags
define void @fastmathflags_unop(float %op1) {
%f.nnan = fneg nnan float %op1
diff --git a/llvm/test/CodeGen/AMDGPU/flat_atomics_i64.ll b/llvm/test/CodeGen/AMDGPU/flat_atomics_i64.ll
index 29c8a0b1c695c..de5f806b91940 100644
--- a/llvm/test/CodeGen/AMDGPU/flat_atomics_i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat_atomics_i64.ll
@@ -659,6 +659,15 @@ entry:
ret void
}
+; GCN-LABEL: {{^}}atomic_xchg_pointer_offset:
+; GCN: flat_atomic_swap_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}}
+define amdgpu_kernel void @atomic_xchg_pointer_offset(i8** %out, i8* %in) {
+entry:
+ %gep = getelementptr i8*, i8** %out, i32 4
+ %val = atomicrmw volatile xchg i8** %gep, i8* %in seq_cst
+ ret void
+}
+
; GCN-LABEL: {{^}}atomic_xchg_i64_ret_offset:
; GCN: flat_atomic_swap_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}}
; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_i64.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_i64.ll
index a27361e54cd18..a2bd77fe40f8a 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_i64.ll
@@ -794,6 +794,17 @@ entry:
ret void
}
+; GCN-LABEL: {{^}}atomic_xchg_pointer_offset:
+; CIVI: buffer_atomic_swap_x2 v{{\[[0-9]+:[0-9]+\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32{{$}}
+
+; GFX9: global_atomic_swap_x2 v{{[0-9]+}}, v[{{[0-9]+:[0-9]+}}], s{{\[[0-9]+:[0-9]+\]}} offset:32{{$}}
+define amdgpu_kernel void @atomic_xchg_pointer_offset(i8* addrspace(1)* %out, i8* %in) {
+entry:
+ %gep = getelementptr i8*, i8* addrspace(1)* %out, i64 4
+ %tmp0 = atomicrmw volatile xchg i8* addrspace(1)* %gep, i8* %in seq_cst
+ ret void
+}
+
; GCN-LABEL: {{^}}atomic_xchg_i64_ret_offset:
; CIVI: buffer_atomic_swap_x2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32 glc{{$}}
; CIVI: buffer_store_dwordx2 [[RET]]
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomics64.ll b/llvm/test/CodeGen/AMDGPU/local-atomics64.ll
index 027c441a0e372..5c67b299cf134 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomics64.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomics64.ll
@@ -40,6 +40,19 @@ define amdgpu_kernel void @lds_atomic_xchg_ret_f64_offset(double addrspace(1)* %
ret void
}
+; GCN-LABEL: {{^}}lds_atomic_xchg_ret_pointer_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN: ds_wrxchg_rtn_b64 {{.*}} offset:32
+; GCN: s_endpgm
+define amdgpu_kernel void @lds_atomic_xchg_ret_pointer_offset(i8* addrspace(1)* %out, i8* addrspace(3)* %ptr) nounwind {
+ %gep = getelementptr i8*, i8* addrspace(3)* %ptr, i32 4
+ %result = atomicrmw xchg i8* addrspace(3)* %gep, i8* null seq_cst
+ store i8* %result, i8* addrspace(1)* %out, align 8
+ ret void
+}
+
; GCN-LABEL: {{^}}lds_atomic_add_ret_i64:
; SICIVI: s_mov_b32 m0
; GFX9-NOT: m0
diff --git a/llvm/test/CodeGen/X86/atomic64.ll b/llvm/test/CodeGen/X86/atomic64.ll
index 040845322857c..19899bf191f52 100644
--- a/llvm/test/CodeGen/X86/atomic64.ll
+++ b/llvm/test/CodeGen/X86/atomic64.ll
@@ -4,6 +4,7 @@
@sc64 = external dso_local global i64
@fsc64 = external dso_local global double
+ at psc64 = external dso_local global i8*
define void @atomic_fetch_add64() nounwind {
; X64-LABEL: atomic_fetch_add64:
@@ -780,3 +781,18 @@ define void @atomic_fetch_swapf64(double %x) nounwind {
%t1 = atomicrmw xchg double* @fsc64, double %x acquire
ret void
}
+
+define void @atomic_fetch_swapptr(i8* %x) nounwind {
+; X64-LABEL: atomic_fetch_swapptr:
+; X64: # %bb.0:
+; X64-NEXT: xchgq %rdi, psc64(%rip)
+; X64-NEXT: retq
+;
+; I486-LABEL: atomic_fetch_swapptr:
+; I486: # %bb.0:
+; I486-NEXT: movl {{[0-9]+}}(%esp), %eax
+; I486-NEXT: xchgl %eax, psc64
+; I486-NEXT: retl
+ %t1 = atomicrmw xchg i8** @psc64, i8* %x acquire
+ ret void
+}
More information about the llvm-commits
mailing list