[llvm] AMDGPU: Flat instructions do not have signed offsets gfx7-gfx11 (PR #95852)
Matt Arsenault via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 18 03:01:08 PDT 2024
https://github.com/arsenm updated https://github.com/llvm/llvm-project/pull/95852
>From dac4f5e1a33795680c07492fffd93d060e090f35 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Fri, 14 Jun 2024 22:04:21 +0200
Subject: [PATCH 1/2] AMDGPU: Flat instructions do not have signed offsets
gfx7-gfx11
Fixes some atomicrmw fadd and intrinsic cases. There are a few
more broken places using signed offsets.
---
llvm/lib/Target/AMDGPU/AMDGPUInstructions.td | 1 +
llvm/lib/Target/AMDGPU/FLATInstructions.td | 45 ++++++-----
.../CodeGen/AMDGPU/flat-atomicrmw-fadd.ll | 74 +++++++++++++++----
llvm/test/CodeGen/AMDGPU/fp-atomics-gfx940.ll | 30 ++++++--
.../CodeGen/AMDGPU/fp64-atomics-gfx90a.ll | 54 +++++++++++---
.../InferAddressSpaces/AMDGPU/flat_atomic.ll | 6 +-
6 files changed, 151 insertions(+), 59 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
index 783bc9d7ef593..65439f2e55c07 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
@@ -616,6 +616,7 @@ multiclass local_addr_space_atomic_op {
}
}
+defm int_amdgcn_flat_atomic_fadd : noret_op;
defm int_amdgcn_flat_atomic_fadd : flat_addr_space_atomic_op;
defm int_amdgcn_flat_atomic_fadd_v2bf16 : noret_op;
defm int_amdgcn_flat_atomic_fmin : noret_op;
diff --git a/llvm/lib/Target/AMDGPU/FLATInstructions.td b/llvm/lib/Target/AMDGPU/FLATInstructions.td
index 2b1a0c9eaad4e..7a4c8988fd76e 100644
--- a/llvm/lib/Target/AMDGPU/FLATInstructions.td
+++ b/llvm/lib/Target/AMDGPU/FLATInstructions.td
@@ -1585,16 +1585,16 @@ let OtherPredicates = [isGFX12Plus] in {
let OtherPredicates = [isGFX10Plus] in {
defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_FMIN", "atomic_load_fmin_global", f32>;
defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_FMAX", "atomic_load_fmax_global", f32>;
-defm : FlatSignedAtomicPat <"FLAT_ATOMIC_FMIN", "atomic_load_fmin_flat", f32>;
-defm : FlatSignedAtomicPat <"FLAT_ATOMIC_FMAX", "atomic_load_fmax_flat", f32>;
+defm : FlatAtomicPat <"FLAT_ATOMIC_FMIN", "atomic_load_fmin_flat", f32>;
+defm : FlatAtomicPat <"FLAT_ATOMIC_FMAX", "atomic_load_fmax_flat", f32>;
}
let OtherPredicates = [isGFX10GFX11] in {
defm : GlobalFLATAtomicIntrPats <"GLOBAL_ATOMIC_FMIN", "int_amdgcn_global_atomic_fmin", f32>;
defm : GlobalFLATAtomicIntrPats <"GLOBAL_ATOMIC_FMAX", "int_amdgcn_global_atomic_fmax", f32>;
-defm : FlatSignedAtomicIntrPat <"FLAT_ATOMIC_FMIN", "int_amdgcn_flat_atomic_fmin", f32>;
-defm : FlatSignedAtomicIntrPat <"FLAT_ATOMIC_FMAX", "int_amdgcn_flat_atomic_fmax", f32>;
+defm : FlatAtomicIntrPat <"FLAT_ATOMIC_FMIN", "int_amdgcn_flat_atomic_fmin", f32>;
+defm : FlatAtomicIntrPat <"FLAT_ATOMIC_FMAX", "int_amdgcn_flat_atomic_fmax", f32>;
}
let OtherPredicates = [isGFX10Only] in {
@@ -1602,17 +1602,17 @@ defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_MIN_F64", "atomic_load_fmin_global",
defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_MAX_F64", "atomic_load_fmax_global", f64>;
defm : GlobalFLATAtomicIntrPats <"GLOBAL_ATOMIC_MIN_F64", "int_amdgcn_global_atomic_fmin", f64>;
defm : GlobalFLATAtomicIntrPats <"GLOBAL_ATOMIC_MAX_F64", "int_amdgcn_global_atomic_fmax", f64>;
-defm : FlatSignedAtomicPat <"FLAT_ATOMIC_MIN_F64", "atomic_load_fmin_flat", f64>;
-defm : FlatSignedAtomicPat <"FLAT_ATOMIC_MAX_F64", "atomic_load_fmax_flat", f64>;
-defm : FlatSignedAtomicIntrPat <"FLAT_ATOMIC_MIN_F64", "int_amdgcn_flat_atomic_fmin", f64>;
-defm : FlatSignedAtomicIntrPat <"FLAT_ATOMIC_MAX_F64", "int_amdgcn_flat_atomic_fmax", f64>;
+defm : FlatAtomicPat <"FLAT_ATOMIC_MIN_F64", "atomic_load_fmin_flat", f64>;
+defm : FlatAtomicPat <"FLAT_ATOMIC_MAX_F64", "atomic_load_fmax_flat", f64>;
+defm : FlatAtomicIntrPat <"FLAT_ATOMIC_MIN_F64", "int_amdgcn_flat_atomic_fmin", f64>;
+defm : FlatAtomicIntrPat <"FLAT_ATOMIC_MAX_F64", "int_amdgcn_flat_atomic_fmax", f64>;
}
let OtherPredicates = [isGFX12Only] in {
defm : GlobalFLATAtomicIntrPats <"GLOBAL_ATOMIC_FMIN", "int_amdgcn_global_atomic_fmin_num", f32>;
defm : GlobalFLATAtomicIntrPats <"GLOBAL_ATOMIC_FMAX", "int_amdgcn_global_atomic_fmax_num", f32>;
- defm : FlatSignedAtomicIntrPat <"FLAT_ATOMIC_FMIN", "int_amdgcn_flat_atomic_fmin_num", f32>;
- defm : FlatSignedAtomicIntrPat <"FLAT_ATOMIC_FMAX", "int_amdgcn_flat_atomic_fmax_num", f32>;
+ defm : FlatAtomicIntrPat <"FLAT_ATOMIC_FMIN", "int_amdgcn_flat_atomic_fmin_num", f32>;
+ defm : FlatAtomicIntrPat <"FLAT_ATOMIC_FMAX", "int_amdgcn_flat_atomic_fmax_num", f32>;
}
let OtherPredicates = [HasAtomicFaddNoRtnInsts] in {
@@ -1646,25 +1646,24 @@ defm : GlobalFLATAtomicPatsWithAddrSpace<"GLOBAL_ATOMIC_ADD_F64", "int_amdgcn_fl
defm : GlobalFLATAtomicPatsWithAddrSpace<"GLOBAL_ATOMIC_ADD_F64", "int_amdgcn_global_atomic_fadd", "global_addrspace", f64>;
defm : GlobalFLATAtomicIntrPats <"GLOBAL_ATOMIC_MIN_F64", "int_amdgcn_global_atomic_fmin", f64>;
defm : GlobalFLATAtomicIntrPats <"GLOBAL_ATOMIC_MAX_F64", "int_amdgcn_global_atomic_fmax", f64>;
-defm : FlatSignedAtomicPat <"FLAT_ATOMIC_ADD_F64", "atomic_load_fadd_flat", f64>;
-defm : FlatSignedAtomicPat <"FLAT_ATOMIC_MIN_F64", "atomic_load_fmin_flat", f64>;
-defm : FlatSignedAtomicPat <"FLAT_ATOMIC_MAX_F64", "atomic_load_fmax_flat", f64>;
-defm : FlatSignedAtomicPatWithAddrSpace <"FLAT_ATOMIC_ADD_F64", "int_amdgcn_flat_atomic_fadd", "flat_addrspace", f64>;
-defm : FlatSignedAtomicIntrPat <"FLAT_ATOMIC_MIN_F64", "int_amdgcn_flat_atomic_fmin", f64>;
-defm : FlatSignedAtomicIntrPat <"FLAT_ATOMIC_MAX_F64", "int_amdgcn_flat_atomic_fmax", f64>;
+defm : FlatAtomicPat <"FLAT_ATOMIC_ADD_F64", "atomic_load_fadd_flat", f64>;
+defm : FlatAtomicPat <"FLAT_ATOMIC_MIN_F64", "atomic_load_fmin_flat", f64>;
+defm : FlatAtomicPat <"FLAT_ATOMIC_MAX_F64", "atomic_load_fmax_flat", f64>;
+defm : FlatAtomicIntrPat <"FLAT_ATOMIC_ADD_F64", "int_amdgcn_flat_atomic_fadd", f64>;
+defm : FlatAtomicIntrPat <"FLAT_ATOMIC_MIN_F64", "int_amdgcn_flat_atomic_fmin", f64>;
+defm : FlatAtomicIntrPat <"FLAT_ATOMIC_MAX_F64", "int_amdgcn_flat_atomic_fmax", f64>;
}
let OtherPredicates = [HasFlatAtomicFaddF32Inst] in {
-defm : FlatSignedAtomicPat <"FLAT_ATOMIC_ADD_F32", "atomic_load_fadd_flat", f32>;
-defm : FlatSignedAtomicPatWithAddrSpace <"FLAT_ATOMIC_ADD_F32", "int_amdgcn_flat_atomic_fadd", "flat_addrspace", f32>;
+defm : FlatAtomicPat <"FLAT_ATOMIC_ADD_F32", "atomic_load_fadd_flat", f32>;
+defm : FlatAtomicIntrPat <"FLAT_ATOMIC_ADD_F32", "int_amdgcn_flat_atomic_fadd", f32>;
}
let OtherPredicates = [HasAtomicFlatPkAdd16Insts] in {
-// FIXME: These do not have signed offsets
-defm : FlatSignedAtomicPatWithAddrSpace <"FLAT_ATOMIC_PK_ADD_F16", "int_amdgcn_flat_atomic_fadd", "flat_addrspace", v2f16>;
-defm : FlatSignedAtomicIntrPat <"FLAT_ATOMIC_PK_ADD_BF16", "int_amdgcn_flat_atomic_fadd_v2bf16", v2i16>;
-defm : FlatSignedAtomicPat <"FLAT_ATOMIC_PK_ADD_F16", "atomic_load_fadd_flat", v2f16>;
-defm : FlatSignedAtomicPat <"FLAT_ATOMIC_PK_ADD_BF16", "atomic_load_fadd_flat", v2bf16>;
+defm : FlatAtomicIntrPat <"FLAT_ATOMIC_PK_ADD_F16", "int_amdgcn_flat_atomic_fadd", v2f16>;
+defm : FlatAtomicIntrPat <"FLAT_ATOMIC_PK_ADD_BF16", "int_amdgcn_flat_atomic_fadd_v2bf16", v2i16>;
+defm : FlatAtomicPat <"FLAT_ATOMIC_PK_ADD_F16", "atomic_load_fadd_flat", v2f16>;
+defm : FlatAtomicPat <"FLAT_ATOMIC_PK_ADD_BF16", "atomic_load_fadd_flat", v2bf16>;
}
let OtherPredicates = [HasAtomicGlobalPkAddBF16Inst] in
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
index bc2a23966c9ca..458f966b0f75f 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
@@ -391,8 +391,11 @@ define float @flat_agent_atomic_fadd_ret_f32__offset12b_neg(ptr %ptr, float %val
; GFX940-LABEL: flat_agent_atomic_fadd_ret_f32__offset12b_neg:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX940-NEXT: s_nop 1
+; GFX940-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: flat_atomic_add_f32 v0, v[0:1], v2 offset:63488 sc0
+; GFX940-NEXT: flat_atomic_add_f32 v0, v[0:1], v2 sc0
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: s_setpc_b64 s[30:31]
@@ -400,8 +403,10 @@ define float @flat_agent_atomic_fadd_ret_f32__offset12b_neg(ptr %ptr, float %val
; GFX11-LABEL: flat_agent_atomic_fadd_ret_f32__offset12b_neg:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
+; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_add_f32 v0, v[0:1], v2 offset:63488 glc
+; GFX11-NEXT: flat_atomic_add_f32 v0, v[0:1], v2 glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
@@ -1003,8 +1008,11 @@ define void @flat_agent_atomic_fadd_noret_f32__offset12b_neg(ptr %ptr, float %va
; GFX940-LABEL: flat_agent_atomic_fadd_noret_f32__offset12b_neg:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX940-NEXT: s_nop 1
+; GFX940-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: flat_atomic_add_f32 v[0:1], v2 offset:63488
+; GFX940-NEXT: flat_atomic_add_f32 v[0:1], v2
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: s_setpc_b64 s[30:31]
@@ -1012,8 +1020,10 @@ define void @flat_agent_atomic_fadd_noret_f32__offset12b_neg(ptr %ptr, float %va
; GFX11-LABEL: flat_agent_atomic_fadd_noret_f32__offset12b_neg:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
+; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_add_f32 v[0:1], v2 offset:63488
+; GFX11-NEXT: flat_atomic_add_f32 v[0:1], v2
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl1_inv
@@ -1952,8 +1962,11 @@ define float @flat_agent_atomic_fadd_ret_f32__offset12b_neg__ftz(ptr %ptr, float
; GFX940-LABEL: flat_agent_atomic_fadd_ret_f32__offset12b_neg__ftz:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX940-NEXT: s_nop 1
+; GFX940-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: flat_atomic_add_f32 v0, v[0:1], v2 offset:63488 sc0
+; GFX940-NEXT: flat_atomic_add_f32 v0, v[0:1], v2 sc0
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: s_setpc_b64 s[30:31]
@@ -1961,8 +1974,10 @@ define float @flat_agent_atomic_fadd_ret_f32__offset12b_neg__ftz(ptr %ptr, float
; GFX11-LABEL: flat_agent_atomic_fadd_ret_f32__offset12b_neg__ftz:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
+; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_add_f32 v0, v[0:1], v2 offset:63488 glc
+; GFX11-NEXT: flat_atomic_add_f32 v0, v[0:1], v2 glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
@@ -2564,8 +2579,11 @@ define void @flat_agent_atomic_fadd_noret_f32__offset12b_neg__ftz(ptr %ptr, floa
; GFX940-LABEL: flat_agent_atomic_fadd_noret_f32__offset12b_neg__ftz:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX940-NEXT: s_nop 1
+; GFX940-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: flat_atomic_add_f32 v[0:1], v2 offset:63488
+; GFX940-NEXT: flat_atomic_add_f32 v[0:1], v2
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: s_setpc_b64 s[30:31]
@@ -2573,8 +2591,10 @@ define void @flat_agent_atomic_fadd_noret_f32__offset12b_neg__ftz(ptr %ptr, floa
; GFX11-LABEL: flat_agent_atomic_fadd_noret_f32__offset12b_neg__ftz:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
+; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_add_f32 v[0:1], v2 offset:63488
+; GFX11-NEXT: flat_atomic_add_f32 v[0:1], v2
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl1_inv
@@ -3528,8 +3548,11 @@ define double @flat_agent_atomic_fadd_ret_f64__offset12b_neg(ptr %ptr, double %v
; GFX940-LABEL: flat_agent_atomic_fadd_ret_f64__offset12b_neg:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX940-NEXT: s_nop 1
+; GFX940-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: flat_atomic_add_f64 v[0:1], v[0:1], v[2:3] offset:63488 sc0
+; GFX940-NEXT: flat_atomic_add_f64 v[0:1], v[0:1], v[2:3] sc0
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: s_setpc_b64 s[30:31]
@@ -3594,7 +3617,9 @@ define double @flat_agent_atomic_fadd_ret_f64__offset12b_neg(ptr %ptr, double %v
; GFX90A-LABEL: flat_agent_atomic_fadd_ret_f64__offset12b_neg:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_atomic_add_f64 v[0:1], v[0:1], v[2:3] offset:63488 glc
+; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX90A-NEXT: flat_atomic_add_f64 v[0:1], v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -4057,8 +4082,11 @@ define void @flat_agent_atomic_fadd_noret_f64__offset12b_neg(ptr %ptr, double %v
; GFX940-LABEL: flat_agent_atomic_fadd_noret_f64__offset12b_neg:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX940-NEXT: s_nop 1
+; GFX940-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: flat_atomic_add_f64 v[0:1], v[2:3] offset:63488
+; GFX940-NEXT: flat_atomic_add_f64 v[0:1], v[2:3]
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: s_setpc_b64 s[30:31]
@@ -4120,7 +4148,9 @@ define void @flat_agent_atomic_fadd_noret_f64__offset12b_neg(ptr %ptr, double %v
; GFX90A-LABEL: flat_agent_atomic_fadd_noret_f64__offset12b_neg:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_atomic_add_f64 v[0:1], v[2:3] offset:63488
+; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX90A-NEXT: flat_atomic_add_f64 v[0:1], v[2:3]
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -10679,8 +10709,11 @@ define <2 x half> @flat_agent_atomic_fadd_ret_v2f16__offset12b_neg(ptr %ptr, <2
; GFX940-LABEL: flat_agent_atomic_fadd_ret_v2f16__offset12b_neg:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX940-NEXT: s_nop 1
+; GFX940-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: flat_atomic_pk_add_f16 v0, v[0:1], v2 offset:63488 sc0
+; GFX940-NEXT: flat_atomic_pk_add_f16 v0, v[0:1], v2 sc0
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: s_setpc_b64 s[30:31]
@@ -11245,8 +11278,11 @@ define void @flat_agent_atomic_fadd_noret_v2f16__offset12b_neg(ptr %ptr, <2 x ha
; GFX940-LABEL: flat_agent_atomic_fadd_noret_v2f16__offset12b_neg:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX940-NEXT: s_nop 1
+; GFX940-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: flat_atomic_pk_add_f16 v[0:1], v2 offset:63488
+; GFX940-NEXT: flat_atomic_pk_add_f16 v[0:1], v2
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: s_setpc_b64 s[30:31]
@@ -12380,8 +12416,11 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX940-LABEL: flat_agent_atomic_fadd_ret_v2bf16__offset12b_neg:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX940-NEXT: s_nop 1
+; GFX940-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: flat_atomic_pk_add_bf16 v0, v[0:1], v2 offset:63488 sc0
+; GFX940-NEXT: flat_atomic_pk_add_bf16 v0, v[0:1], v2 sc0
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: s_setpc_b64 s[30:31]
@@ -13210,8 +13249,11 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX940-LABEL: flat_agent_atomic_fadd_noret_v2bf16__offset12b_neg:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX940-NEXT: s_nop 1
+; GFX940-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
; GFX940-NEXT: buffer_wbl2 sc1
-; GFX940-NEXT: flat_atomic_pk_add_bf16 v[0:1], v2 offset:63488
+; GFX940-NEXT: flat_atomic_pk_add_bf16 v[0:1], v2
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: buffer_inv sc1
; GFX940-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/fp-atomics-gfx940.ll b/llvm/test/CodeGen/AMDGPU/fp-atomics-gfx940.ll
index 55f77bbadc56d..5761c198e20ba 100644
--- a/llvm/test/CodeGen/AMDGPU/fp-atomics-gfx940.ll
+++ b/llvm/test/CodeGen/AMDGPU/fp-atomics-gfx940.ll
@@ -435,7 +435,10 @@ define float @flat_atomic_fadd_f32_intrinsic_ret__negoffset(ptr %ptr, float %dat
; GFX940-LABEL: flat_atomic_fadd_f32_intrinsic_ret__negoffset:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT: flat_atomic_add_f32 v0, v[0:1], v2 offset:64512 sc0
+; GFX940-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffffc00, v0
+; GFX940-NEXT: s_nop 1
+; GFX940-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX940-NEXT: flat_atomic_add_f32 v0, v[0:1], v2 sc0
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
@@ -481,7 +484,10 @@ define void @flat_atomic_fadd_f32_intrinsic_noret__negoffset(ptr %ptr, float %da
; GFX940-LABEL: flat_atomic_fadd_f32_intrinsic_noret__negoffset:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT: flat_atomic_add_f32 v[0:1], v2 offset:64512
+; GFX940-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffffc00, v0
+; GFX940-NEXT: s_nop 1
+; GFX940-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX940-NEXT: flat_atomic_add_f32 v[0:1], v2
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
@@ -527,7 +533,10 @@ define <2 x half> @flat_atomic_fadd_v2f16_intrinsic_ret__negoffset(ptr %ptr, <2
; GFX940-LABEL: flat_atomic_fadd_v2f16_intrinsic_ret__negoffset:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT: flat_atomic_pk_add_f16 v0, v[0:1], v2 offset:64512 sc0
+; GFX940-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffffc00, v0
+; GFX940-NEXT: s_nop 1
+; GFX940-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX940-NEXT: flat_atomic_pk_add_f16 v0, v[0:1], v2 sc0
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
@@ -573,7 +582,10 @@ define void @flat_atomic_fadd_v2f16_intrinsic_noret__negoffset(ptr %ptr, <2 x ha
; GFX940-LABEL: flat_atomic_fadd_v2f16_intrinsic_noret__negoffset:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT: flat_atomic_pk_add_f16 v[0:1], v2 offset:64512
+; GFX940-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffffc00, v0
+; GFX940-NEXT: s_nop 1
+; GFX940-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX940-NEXT: flat_atomic_pk_add_f16 v[0:1], v2
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
@@ -619,7 +631,10 @@ define <2 x i16> @flat_atomic_fadd_v2bf16_intrinsic_ret__negoffset(ptr %ptr, <2
; GFX940-LABEL: flat_atomic_fadd_v2bf16_intrinsic_ret__negoffset:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT: flat_atomic_pk_add_bf16 v0, v[0:1], v2 offset:64512 sc0
+; GFX940-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffffc00, v0
+; GFX940-NEXT: s_nop 1
+; GFX940-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX940-NEXT: flat_atomic_pk_add_bf16 v0, v[0:1], v2 sc0
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
@@ -665,7 +680,10 @@ define void @flat_atomic_fadd_v2bf16_intrinsic_noret__negoffset(ptr %ptr, <2 x i
; GFX940-LABEL: flat_atomic_fadd_v2bf16_intrinsic_noret__negoffset:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT: flat_atomic_pk_add_bf16 v[0:1], v2 offset:64512
+; GFX940-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffffc00, v0
+; GFX940-NEXT: s_nop 1
+; GFX940-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX940-NEXT: flat_atomic_pk_add_bf16 v[0:1], v2
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll b/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll
index 7a2f1222f2cb4..bf3dbec6fa9ef 100644
--- a/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll
+++ b/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll
@@ -2257,14 +2257,19 @@ define double @flat_atomic_fadd_f64_intrinsic_rtn__negoffset(ptr %ptr, double %d
; GFX90A-LABEL: flat_atomic_fadd_f64_intrinsic_rtn__negoffset:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_atomic_add_f64 v[0:1], v[0:1], v[2:3] offset:61448 glc
+; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff008, v0
+; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX90A-NEXT: flat_atomic_add_f64 v[0:1], v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX940-LABEL: flat_atomic_fadd_f64_intrinsic_rtn__negoffset:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT: flat_atomic_add_f64 v[0:1], v[0:1], v[2:3] offset:61448 sc0
+; GFX940-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff008, v0
+; GFX940-NEXT: s_nop 1
+; GFX940-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX940-NEXT: flat_atomic_add_f64 v[0:1], v[0:1], v[2:3] sc0
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr double, ptr %ptr, i64 -511
@@ -2295,14 +2300,19 @@ define void @flat_atomic_fadd_f64_intrinsic_noret__negoffset(ptr %ptr, double %d
; GFX90A-LABEL: flat_atomic_fadd_f64_intrinsic_noret__negoffset:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_atomic_add_f64 v[0:1], v[2:3] offset:61448
+; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff008, v0
+; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX90A-NEXT: flat_atomic_add_f64 v[0:1], v[2:3]
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX940-LABEL: flat_atomic_fadd_f64_intrinsic_noret__negoffset:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT: flat_atomic_add_f64 v[0:1], v[2:3] offset:61448
+; GFX940-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff008, v0
+; GFX940-NEXT: s_nop 1
+; GFX940-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX940-NEXT: flat_atomic_add_f64 v[0:1], v[2:3]
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr double, ptr %ptr, i64 -511
@@ -2333,14 +2343,19 @@ define double @flat_atomic_fmin_f64_intrinsic_rtn__negoffset(ptr %ptr, double %d
; GFX90A-LABEL: flat_atomic_fmin_f64_intrinsic_rtn__negoffset:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_atomic_min_f64 v[0:1], v[0:1], v[2:3] offset:61448 glc
+; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff008, v0
+; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX90A-NEXT: flat_atomic_min_f64 v[0:1], v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX940-LABEL: flat_atomic_fmin_f64_intrinsic_rtn__negoffset:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT: flat_atomic_min_f64 v[0:1], v[0:1], v[2:3] offset:61448 sc0
+; GFX940-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff008, v0
+; GFX940-NEXT: s_nop 1
+; GFX940-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX940-NEXT: flat_atomic_min_f64 v[0:1], v[0:1], v[2:3] sc0
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr double, ptr %ptr, i64 -511
@@ -2371,14 +2386,19 @@ define void @flat_atomic_fmin_f64_intrinsic_noret__negoffset(ptr %ptr, double %d
; GFX90A-LABEL: flat_atomic_fmin_f64_intrinsic_noret__negoffset:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_atomic_min_f64 v[0:1], v[2:3] offset:61448
+; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff008, v0
+; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX90A-NEXT: flat_atomic_min_f64 v[0:1], v[2:3]
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX940-LABEL: flat_atomic_fmin_f64_intrinsic_noret__negoffset:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT: flat_atomic_min_f64 v[0:1], v[2:3] offset:61448
+; GFX940-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff008, v0
+; GFX940-NEXT: s_nop 1
+; GFX940-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX940-NEXT: flat_atomic_min_f64 v[0:1], v[2:3]
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr double, ptr %ptr, i64 -511
@@ -2409,14 +2429,19 @@ define double @flat_atomic_fmax_f64_intrinsic_rtn__negoffset(ptr %ptr, double %d
; GFX90A-LABEL: flat_atomic_fmax_f64_intrinsic_rtn__negoffset:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_atomic_max_f64 v[0:1], v[0:1], v[2:3] offset:61448 glc
+; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff008, v0
+; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX90A-NEXT: flat_atomic_max_f64 v[0:1], v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX940-LABEL: flat_atomic_fmax_f64_intrinsic_rtn__negoffset:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT: flat_atomic_max_f64 v[0:1], v[0:1], v[2:3] offset:61448 sc0
+; GFX940-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff008, v0
+; GFX940-NEXT: s_nop 1
+; GFX940-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX940-NEXT: flat_atomic_max_f64 v[0:1], v[0:1], v[2:3] sc0
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr double, ptr %ptr, i64 -511
@@ -2447,14 +2472,19 @@ define void @flat_atomic_fmax_f64_intrinsic_noret__negoffset(ptr %ptr, double %d
; GFX90A-LABEL: flat_atomic_fmax_f64_intrinsic_noret__negoffset:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_atomic_max_f64 v[0:1], v[2:3] offset:61448
+; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff008, v0
+; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX90A-NEXT: flat_atomic_max_f64 v[0:1], v[2:3]
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX940-LABEL: flat_atomic_fmax_f64_intrinsic_noret__negoffset:
; GFX940: ; %bb.0:
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT: flat_atomic_max_f64 v[0:1], v[2:3] offset:61448
+; GFX940-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff008, v0
+; GFX940-NEXT: s_nop 1
+; GFX940-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX940-NEXT: flat_atomic_max_f64 v[0:1], v[2:3]
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX940-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr double, ptr %ptr, i64 -511
diff --git a/llvm/test/Transforms/InferAddressSpaces/AMDGPU/flat_atomic.ll b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/flat_atomic.ll
index ddc14b6a25c26..f8d97c81698cd 100644
--- a/llvm/test/Transforms/InferAddressSpaces/AMDGPU/flat_atomic.ll
+++ b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/flat_atomic.ll
@@ -15,10 +15,12 @@ define protected amdgpu_kernel void @InferNothing(i32 %a, ptr %b, double %c) {
; CHECK-NEXT: s_lshl_b64 s[0:1], s[2:3], 3
; CHECK-NEXT: s_add_u32 s0, s0, s4
; CHECK-NEXT: s_addc_u32 s1, s1, s5
+; CHECK-NEXT: v_mov_b32_e32 v3, s1
+; CHECK-NEXT: v_add_co_u32_e64 v2, vcc, -8, s0
; CHECK-NEXT: v_mov_b32_e32 v0, s6
; CHECK-NEXT: v_mov_b32_e32 v1, s7
-; CHECK-NEXT: v_pk_mov_b32 v[2:3], s[0:1], s[0:1] op_sel:[0,1]
-; CHECK-NEXT: flat_atomic_add_f64 v[2:3], v[0:1] offset:65528
+; CHECK-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc
+; CHECK-NEXT: flat_atomic_add_f64 v[2:3], v[0:1]
; CHECK-NEXT: s_endpgm
entry:
%i = add nsw i32 %a, -1
>From cf7e6c466a2bba52b49f98a45eb4147102eec033 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Tue, 18 Jun 2024 11:53:13 +0200
Subject: [PATCH 2/2] Remove now unused pattern classes
---
llvm/lib/Target/AMDGPU/FLATInstructions.td | 15 ---------------
1 file changed, 15 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/FLATInstructions.td b/llvm/lib/Target/AMDGPU/FLATInstructions.td
index 7a4c8988fd76e..3d2a74adab965 100644
--- a/llvm/lib/Target/AMDGPU/FLATInstructions.td
+++ b/llvm/lib/Target/AMDGPU/FLATInstructions.td
@@ -1155,21 +1155,6 @@ multiclass FlatSignedAtomicPat <string inst, string node, ValueType vt,
def : FlatSignedAtomicPatBase<!cast<FLAT_Pseudo>(inst), noRtnNode, vt, data_vt>;
}
-multiclass FlatSignedAtomicIntrPat <string inst, string node, ValueType vt,
- ValueType data_vt = vt> {
- defm : FlatSignedAtomicPat<inst, node, vt, data_vt, /* complexity */ 0, /* isIntr */ 1>;
-}
-
-multiclass FlatSignedAtomicPatWithAddrSpace<string inst, string intr, string addrSpaceSuffix,
- ValueType vt, ValueType data_vt = vt> {
- defvar noRtnNode = !cast<PatFrags>(intr # "_noret_" # addrSpaceSuffix);
- defvar rtnNode = !cast<PatFrags>(intr # "_" # addrSpaceSuffix);
-
- let AddedComplexity = 1 in
- def : FlatSignedAtomicPatBase<!cast<FLAT_Pseudo>(inst), noRtnNode, vt, data_vt>;
- def : FlatSignedAtomicPatBase<!cast<FLAT_Pseudo>(inst#"_RTN"), rtnNode, vt, data_vt>;
-}
-
class ScratchLoadSignedPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <
(vt (node (ScratchOffset (i32 VGPR_32:$vaddr), i32:$offset))),
(inst $vaddr, $offset)
More information about the llvm-commits
mailing list