[llvm] e16a713 - AMDGPU: Select global atomicrmw fadd
Matt Arsenault via llvm-commits
llvm-commits at lists.llvm.org
Wed Nov 6 16:13:43 PST 2019
Author: Matt Arsenault
Date: 2019-11-06T16:06:38-08:00
New Revision: e16a71382d689c39561e94b13af2feb4bf1b15cc
URL: https://github.com/llvm/llvm-project/commit/e16a71382d689c39561e94b13af2feb4bf1b15cc
DIFF: https://github.com/llvm/llvm-project/commit/e16a71382d689c39561e94b13af2feb4bf1b15cc.diff
LOG: AMDGPU: Select global atomicrmw fadd
This only works if there is no use of the return value.
Added:
llvm/test/CodeGen/AMDGPU/global-atomics-fp.ll
Modified:
llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
llvm/lib/Target/AMDGPU/FLATInstructions.td
llvm/lib/Target/AMDGPU/SIISelLowering.cpp
llvm/lib/Target/AMDGPU/SIInstrInfo.td
llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fadd.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index 8912ef446034..cc8c36a93fe9 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -4343,7 +4343,6 @@ const char* AMDGPUTargetLowering::getTargetNodeName(unsigned Opcode) const {
NODE_NAME_CASE(BUFFER_ATOMIC_CMPSWAP)
NODE_NAME_CASE(BUFFER_ATOMIC_FADD)
NODE_NAME_CASE(BUFFER_ATOMIC_PK_FADD)
- NODE_NAME_CASE(ATOMIC_FADD)
NODE_NAME_CASE(ATOMIC_PK_FADD)
case AMDGPUISD::LAST_AMDGPU_ISD_NUMBER: break;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
index dea0d1d4343a..8a89fe4ef588 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
@@ -531,7 +531,6 @@ enum NodeType : unsigned {
BUFFER_ATOMIC_CMPSWAP,
BUFFER_ATOMIC_FADD,
BUFFER_ATOMIC_PK_FADD,
- ATOMIC_FADD,
ATOMIC_PK_FADD,
LAST_AMDGPU_ISD_NUMBER
diff --git a/llvm/lib/Target/AMDGPU/FLATInstructions.td b/llvm/lib/Target/AMDGPU/FLATInstructions.td
index 94b038b38f5f..71cff6a572d5 100644
--- a/llvm/lib/Target/AMDGPU/FLATInstructions.td
+++ b/llvm/lib/Target/AMDGPU/FLATInstructions.td
@@ -364,11 +364,12 @@ multiclass FLAT_Global_Atomic_Pseudo<
string opName,
RegisterClass vdst_rc,
ValueType vt,
- SDPatternOperator atomic = null_frag,
+ SDPatternOperator atomic_rtn = null_frag,
+ SDPatternOperator atomic_no_rtn = null_frag,
ValueType data_vt = vt,
RegisterClass data_rc = vdst_rc> :
- FLAT_Global_Atomic_Pseudo_NO_RTN<opName, vdst_rc, vt, atomic, data_vt, data_rc>,
- FLAT_Global_Atomic_Pseudo_RTN<opName, vdst_rc, vt, atomic, data_vt, data_rc>;
+ FLAT_Global_Atomic_Pseudo_NO_RTN<opName, vdst_rc, vt, atomic_no_rtn, data_vt, data_rc>,
+ FLAT_Global_Atomic_Pseudo_RTN<opName, vdst_rc, vt, atomic_rtn, data_vt, data_rc>;
//===----------------------------------------------------------------------===//
@@ -535,11 +536,12 @@ defm GLOBAL_STORE_SHORT_D16_HI : FLAT_Global_Store_Pseudo <"global_store_short_d
let is_flat_global = 1 in {
defm GLOBAL_ATOMIC_CMPSWAP : FLAT_Global_Atomic_Pseudo <"global_atomic_cmpswap",
- VGPR_32, i32, AMDGPUatomic_cmp_swap_global_32,
+ VGPR_32, i32, AMDGPUatomic_cmp_swap_global_32, null_frag,
v2i32, VReg_64>;
defm GLOBAL_ATOMIC_CMPSWAP_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_cmpswap_x2",
VReg_64, i64, AMDGPUatomic_cmp_swap_global_64,
+ null_frag,
v2i64, VReg_128>;
defm GLOBAL_ATOMIC_SWAP : FLAT_Global_Atomic_Pseudo <"global_atomic_swap",
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 47d10afa895c..209c17a117a1 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -7094,13 +7094,16 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
EVT VT = Op.getOperand(3).getValueType();
auto *M = cast<MemSDNode>(Op);
- unsigned Opcode = VT.isVector() ? AMDGPUISD::ATOMIC_PK_FADD
- : AMDGPUISD::ATOMIC_FADD;
+ if (VT.isVector()) {
+ return DAG.getMemIntrinsicNode(
+ AMDGPUISD::ATOMIC_PK_FADD, DL, Op->getVTList(), Ops, VT,
+ M->getMemOperand());
+ }
- return DAG.getMemIntrinsicNode(Opcode, DL, Op->getVTList(), Ops, VT,
- M->getMemOperand());
+ return DAG.getAtomic(ISD::ATOMIC_LOAD_FADD, DL, VT,
+ DAG.getVTList(VT, MVT::Other), Ops,
+ M->getMemOperand()).getValue(1);
}
-
case Intrinsic::amdgcn_end_cf:
return SDValue(DAG.getMachineNode(AMDGPU::SI_END_CF, DL, MVT::Other,
Op->getOperand(2), Chain), 0);
@@ -10936,6 +10939,12 @@ SITargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
// TODO: Do have these for flat. Older targets also had them for buffers.
unsigned AS = RMW->getPointerAddressSpace();
+
+ if (AS == AMDGPUAS::GLOBAL_ADDRESS && Subtarget->hasAtomicFaddInsts()) {
+ return RMW->use_empty() ? AtomicExpansionKind::None :
+ AtomicExpansionKind::CmpXChg;
+ }
+
return (AS == AMDGPUAS::LOCAL_ADDRESS && Subtarget->hasLDSFPAtomics()) ?
AtomicExpansionKind::None : AtomicExpansionKind::CmpXChg;
}
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
index 2617372ed829..cadd4a5cc211 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
@@ -225,7 +225,6 @@ class SDGlobalAtomicNoRtn<string opcode, ValueType ty> : SDNode <opcode,
[SDNPMemOperand, SDNPHasChain, SDNPMayLoad, SDNPMayStore]
>;
-def SIglobal_atomic_fadd : SDGlobalAtomicNoRtn <"AMDGPUISD::ATOMIC_FADD", f32>;
def SIglobal_atomic_pk_fadd : SDGlobalAtomicNoRtn <"AMDGPUISD::ATOMIC_PK_FADD", v2f16>;
def SIpc_add_rel_offset : SDNode<"AMDGPUISD::PC_ADD_REL_OFFSET",
@@ -324,7 +323,7 @@ defm atomic_load_fmax_#as : binary_atomic_op<SIatomic_fmax, 0>;
def atomic_fadd_global_noret : PatFrag<
(ops node:$ptr, node:$value),
- (SIglobal_atomic_fadd node:$ptr, node:$value)> {
+ (atomic_load_fadd node:$ptr, node:$value)> {
// FIXME: Move this
let MemoryVT = f32;
let IsAtomic = 1;
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomics-fp.ll b/llvm/test/CodeGen/AMDGPU/global-atomics-fp.ll
new file mode 100644
index 000000000000..b91536eadec1
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/global-atomics-fp.ll
@@ -0,0 +1,29 @@
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX900 %s
+; RUN: llc -march=amdgcn -mcpu=gfx908 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX908 %s
+
+; GCN-LABEL: {{^}}global_atomic_fadd_ret_f32:
+; GCN: [[LOOP:BB[0-9]+_[0-9]+]]
+; GCN: v_add_f32_e32
+; GCN: global_atomic_cmpswap
+; GCN: s_andn2_b64 exec, exec,
+; GCN-NEXT: s_cbranch_execnz [[LOOP]]
+define amdgpu_kernel void @global_atomic_fadd_ret_f32(float addrspace(1)* %ptr) {
+ %result = atomicrmw fadd float addrspace(1)* %ptr, float 4.0 seq_cst
+ store float %result, float addrspace(1)* undef
+ ret void
+}
+
+; GCN-LABEL: {{^}}global_atomic_fadd_noret_f32:
+; GFX900: [[LOOP:BB[0-9]+_[0-9]+]]
+; GFX900: v_add_f32_e32
+; GFX900: global_atomic_cmpswap
+; GFX900: s_andn2_b64 exec, exec,
+; GFX900-NEXT: s_cbranch_execnz [[LOOP]]
+
+; GFX908-NOT: v_add_f32
+; GFX908: global_atomic_add_f32 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, off
+; GFX908-NOT: s_cbranch_execnz
+define amdgpu_kernel void @global_atomic_fadd_noret_f32(float addrspace(1)* %ptr) {
+ %result = atomicrmw fadd float addrspace(1)* %ptr, float 4.0 seq_cst
+ ret void
+}
diff --git a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fadd.ll b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fadd.ll
index f96cd0b29bff..42ff7aa32b57 100644
--- a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fadd.ll
+++ b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fadd.ll
@@ -1,6 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -atomic-expand %s | FileCheck -check-prefix=CI %s
; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -atomic-expand %s | FileCheck -check-prefix=GFX9 %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -atomic-expand %s | FileCheck -check-prefix=GFX908 %s
define float @test_atomicrmw_fadd_f32_flat(float* %ptr, float %value) {
; CI-LABEL: @test_atomicrmw_fadd_f32_flat(
@@ -36,6 +37,23 @@ define float @test_atomicrmw_fadd_f32_flat(float* %ptr, float %value) {
; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; GFX9: atomicrmw.end:
; GFX9-NEXT: ret float [[TMP6]]
+;
+; GFX908-LABEL: @test_atomicrmw_fadd_f32_flat(
+; GFX908-NEXT: [[TMP1:%.*]] = load float, float* [[PTR:%.*]], align 4
+; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]
+; GFX908: atomicrmw.start:
+; GFX908-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
+; GFX908-NEXT: [[TMP2:%.*]] = bitcast float* [[PTR]] to i32*
+; GFX908-NEXT: [[TMP3:%.*]] = bitcast float [[NEW]] to i32
+; GFX908-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; GFX908-NEXT: [[TMP5:%.*]] = cmpxchg i32* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst
+; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; GFX908-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908: atomicrmw.end:
+; GFX908-NEXT: ret float [[TMP6]]
;
%res = atomicrmw fadd float* %ptr, float %value seq_cst
ret float %res
@@ -75,11 +93,71 @@ define float @test_atomicrmw_fadd_f32_global(float addrspace(1)* %ptr, float %va
; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; GFX9: atomicrmw.end:
; GFX9-NEXT: ret float [[TMP6]]
+;
+; GFX908-LABEL: @test_atomicrmw_fadd_f32_global(
+; GFX908-NEXT: [[TMP1:%.*]] = load float, float addrspace(1)* [[PTR:%.*]], align 4
+; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]
+; GFX908: atomicrmw.start:
+; GFX908-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
+; GFX908-NEXT: [[TMP2:%.*]] = bitcast float addrspace(1)* [[PTR]] to i32 addrspace(1)*
+; GFX908-NEXT: [[TMP3:%.*]] = bitcast float [[NEW]] to i32
+; GFX908-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; GFX908-NEXT: [[TMP5:%.*]] = cmpxchg i32 addrspace(1)* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst
+; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; GFX908-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908: atomicrmw.end:
+; GFX908-NEXT: ret float [[TMP6]]
;
%res = atomicrmw fadd float addrspace(1)* %ptr, float %value seq_cst
ret float %res
}
+define void @test_atomicrmw_fadd_f32_global_no_use(float addrspace(1)* %ptr, float %value) {
+; CI-LABEL: @test_atomicrmw_fadd_f32_global_no_use(
+; CI-NEXT: [[TMP1:%.*]] = load float, float addrspace(1)* [[PTR:%.*]], align 4
+; CI-NEXT: br label [[ATOMICRMW_START:%.*]]
+; CI: atomicrmw.start:
+; CI-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; CI-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
+; CI-NEXT: [[TMP2:%.*]] = bitcast float addrspace(1)* [[PTR]] to i32 addrspace(1)*
+; CI-NEXT: [[TMP3:%.*]] = bitcast float [[NEW]] to i32
+; CI-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; CI-NEXT: [[TMP5:%.*]] = cmpxchg i32 addrspace(1)* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst
+; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CI-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CI: atomicrmw.end:
+; CI-NEXT: ret void
+;
+; GFX9-LABEL: @test_atomicrmw_fadd_f32_global_no_use(
+; GFX9-NEXT: [[TMP1:%.*]] = load float, float addrspace(1)* [[PTR:%.*]], align 4
+; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]
+; GFX9: atomicrmw.start:
+; GFX9-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX9-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
+; GFX9-NEXT: [[TMP2:%.*]] = bitcast float addrspace(1)* [[PTR]] to i32 addrspace(1)*
+; GFX9-NEXT: [[TMP3:%.*]] = bitcast float [[NEW]] to i32
+; GFX9-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; GFX9-NEXT: [[TMP5:%.*]] = cmpxchg i32 addrspace(1)* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst
+; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; GFX9-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX9: atomicrmw.end:
+; GFX9-NEXT: ret void
+;
+; GFX908-LABEL: @test_atomicrmw_fadd_f32_global_no_use(
+; GFX908-NEXT: [[RES:%.*]] = atomicrmw fadd float addrspace(1)* [[PTR:%.*]], float [[VALUE:%.*]] seq_cst
+; GFX908-NEXT: ret void
+;
+ %res = atomicrmw fadd float addrspace(1)* %ptr, float %value seq_cst
+ ret void
+}
+
define float @test_atomicrmw_fadd_f32_local(float addrspace(3)* %ptr, float %value) {
; CI-LABEL: @test_atomicrmw_fadd_f32_local(
; CI-NEXT: [[TMP1:%.*]] = load float, float addrspace(3)* [[PTR:%.*]], align 4
@@ -101,6 +179,10 @@ define float @test_atomicrmw_fadd_f32_local(float addrspace(3)* %ptr, float %val
; GFX9-LABEL: @test_atomicrmw_fadd_f32_local(
; GFX9-NEXT: [[RES:%.*]] = atomicrmw fadd float addrspace(3)* [[PTR:%.*]], float [[VALUE:%.*]] seq_cst
; GFX9-NEXT: ret float [[RES]]
+;
+; GFX908-LABEL: @test_atomicrmw_fadd_f32_local(
+; GFX908-NEXT: [[RES:%.*]] = atomicrmw fadd float addrspace(3)* [[PTR:%.*]], float [[VALUE:%.*]] seq_cst
+; GFX908-NEXT: ret float [[RES]]
;
%res = atomicrmw fadd float addrspace(3)* %ptr, float %value seq_cst
ret float %res
@@ -114,6 +196,10 @@ define half @test_atomicrmw_fadd_f16_flat(half* %ptr, half %value) {
; GFX9-LABEL: @test_atomicrmw_fadd_f16_flat(
; GFX9-NEXT: [[RES:%.*]] = atomicrmw fadd half* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst
; GFX9-NEXT: ret half [[RES]]
+;
+; GFX908-LABEL: @test_atomicrmw_fadd_f16_flat(
+; GFX908-NEXT: [[RES:%.*]] = atomicrmw fadd half* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst
+; GFX908-NEXT: ret half [[RES]]
;
%res = atomicrmw fadd half* %ptr, half %value seq_cst
ret half %res
@@ -127,6 +213,10 @@ define half @test_atomicrmw_fadd_f16_global(half addrspace(1)* %ptr, half %value
; GFX9-LABEL: @test_atomicrmw_fadd_f16_global(
; GFX9-NEXT: [[RES:%.*]] = atomicrmw fadd half addrspace(1)* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst
; GFX9-NEXT: ret half [[RES]]
+;
+; GFX908-LABEL: @test_atomicrmw_fadd_f16_global(
+; GFX908-NEXT: [[RES:%.*]] = atomicrmw fadd half addrspace(1)* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst
+; GFX908-NEXT: ret half [[RES]]
;
%res = atomicrmw fadd half addrspace(1)* %ptr, half %value seq_cst
ret half %res
@@ -140,6 +230,10 @@ define half @test_atomicrmw_fadd_f16_local(half addrspace(3)* %ptr, half %value)
; GFX9-LABEL: @test_atomicrmw_fadd_f16_local(
; GFX9-NEXT: [[RES:%.*]] = atomicrmw fadd half addrspace(3)* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst
; GFX9-NEXT: ret half [[RES]]
+;
+; GFX908-LABEL: @test_atomicrmw_fadd_f16_local(
+; GFX908-NEXT: [[RES:%.*]] = atomicrmw fadd half addrspace(3)* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst
+; GFX908-NEXT: ret half [[RES]]
;
%res = atomicrmw fadd half addrspace(3)* %ptr, half %value seq_cst
ret half %res
@@ -179,6 +273,23 @@ define double @test_atomicrmw_fadd_f64_flat(double* %ptr, double %value) {
; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; GFX9: atomicrmw.end:
; GFX9-NEXT: ret double [[TMP6]]
+;
+; GFX908-LABEL: @test_atomicrmw_fadd_f64_flat(
+; GFX908-NEXT: [[TMP1:%.*]] = load double, double* [[PTR:%.*]], align 8
+; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]
+; GFX908: atomicrmw.start:
+; GFX908-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]
+; GFX908-NEXT: [[TMP2:%.*]] = bitcast double* [[PTR]] to i64*
+; GFX908-NEXT: [[TMP3:%.*]] = bitcast double [[NEW]] to i64
+; GFX908-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX908-NEXT: [[TMP5:%.*]] = cmpxchg i64* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst
+; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX908-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908: atomicrmw.end:
+; GFX908-NEXT: ret double [[TMP6]]
;
%res = atomicrmw fadd double* %ptr, double %value seq_cst
ret double %res
@@ -218,6 +329,23 @@ define double @test_atomicrmw_fadd_f64_global(double addrspace(1)* %ptr, double
; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; GFX9: atomicrmw.end:
; GFX9-NEXT: ret double [[TMP6]]
+;
+; GFX908-LABEL: @test_atomicrmw_fadd_f64_global(
+; GFX908-NEXT: [[TMP1:%.*]] = load double, double addrspace(1)* [[PTR:%.*]], align 8
+; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]
+; GFX908: atomicrmw.start:
+; GFX908-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]
+; GFX908-NEXT: [[TMP2:%.*]] = bitcast double addrspace(1)* [[PTR]] to i64 addrspace(1)*
+; GFX908-NEXT: [[TMP3:%.*]] = bitcast double [[NEW]] to i64
+; GFX908-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX908-NEXT: [[TMP5:%.*]] = cmpxchg i64 addrspace(1)* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst
+; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX908-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908: atomicrmw.end:
+; GFX908-NEXT: ret double [[TMP6]]
;
%res = atomicrmw fadd double addrspace(1)* %ptr, double %value seq_cst
ret double %res
@@ -257,6 +385,23 @@ define double @test_atomicrmw_fadd_f64_local(double addrspace(3)* %ptr, double %
; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; GFX9: atomicrmw.end:
; GFX9-NEXT: ret double [[TMP6]]
+;
+; GFX908-LABEL: @test_atomicrmw_fadd_f64_local(
+; GFX908-NEXT: [[TMP1:%.*]] = load double, double addrspace(3)* [[PTR:%.*]], align 8
+; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]
+; GFX908: atomicrmw.start:
+; GFX908-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]
+; GFX908-NEXT: [[TMP2:%.*]] = bitcast double addrspace(3)* [[PTR]] to i64 addrspace(3)*
+; GFX908-NEXT: [[TMP3:%.*]] = bitcast double [[NEW]] to i64
+; GFX908-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX908-NEXT: [[TMP5:%.*]] = cmpxchg i64 addrspace(3)* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst
+; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX908-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908: atomicrmw.end:
+; GFX908-NEXT: ret double [[TMP6]]
;
%res = atomicrmw fadd double addrspace(3)* %ptr, double %value seq_cst
ret double %res
More information about the llvm-commits
mailing list