[llvm] AMDGPU: Fix creating illegally typed readfirstlane in atomic optimizer (PR #128388)
Matt Arsenault via llvm-commits
llvm-commits at lists.llvm.org
Sat Feb 22 18:57:58 PST 2025
https://github.com/arsenm created https://github.com/llvm/llvm-project/pull/128388
We need to promote 8/16-bit cases to 32-bit. Unfortunately we are
missing demanded bits optimizations on readfirstlane, so we end up emitting
an and instruction on the input.
>From 6b92c78c07203ddf73e5ca135b5a50f98b4903d7 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Sun, 23 Feb 2025 09:51:16 +0700
Subject: [PATCH] AMDGPU: Fix creating illegally typed readfirstlane in atomic
optimizer
We need to promote 8/16-bit cases to 32-bit. Unfortunately we are
missing demanded bits optimizations on readfirstlane, so we end up emitting
an and instruction on the input.
---
.../Target/AMDGPU/AMDGPUAtomicOptimizer.cpp | 11 +-
.../AMDGPU/atomic-optimizer-promote-i8.ll | 176 +
.../atomic_optimizations_global_pointer.ll | 4241 +++++++++++++++++
3 files changed, 4426 insertions(+), 2 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/atomic-optimizer-promote-i8.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
index 02f5ce2d18ff6..e46d0587e7943 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
@@ -898,8 +898,15 @@ void AMDGPUAtomicOptimizerImpl::optimizeAtomic(Instruction &I,
// We need to broadcast the value who was the lowest active lane (the first
// lane) to all other lanes in the wavefront.
- Value *BroadcastI = nullptr;
- BroadcastI = B.CreateIntrinsic(Ty, Intrinsic::amdgcn_readfirstlane, PHI);
+
+ Value *ReadlaneVal = PHI;
+ if (TyBitWidth < 32)
+ ReadlaneVal = B.CreateZExt(PHI, B.getInt32Ty());
+
+ Value *BroadcastI = B.CreateIntrinsic(
+ ReadlaneVal->getType(), Intrinsic::amdgcn_readfirstlane, ReadlaneVal);
+ if (TyBitWidth < 32)
+ BroadcastI = B.CreateTrunc(BroadcastI, Ty);
// Now that we have the result of our single atomic operation, we need to
// get our individual lane's slice into the result. We use the lane offset
diff --git a/llvm/test/CodeGen/AMDGPU/atomic-optimizer-promote-i8.ll b/llvm/test/CodeGen/AMDGPU/atomic-optimizer-promote-i8.ll
new file mode 100644
index 0000000000000..d3e591634503f
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/atomic-optimizer-promote-i8.ll
@@ -0,0 +1,176 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -passes=amdgpu-atomic-optimizer %s | FileCheck %s
+
+define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace(1) %uniform.ptr, i8 %val) {
+; CHECK-LABEL: define amdgpu_kernel void @uniform_or_i8(
+; CHECK-SAME: ptr addrspace(1) [[RESULT:%.*]], ptr addrspace(1) [[UNIFORM_PTR:%.*]], i8 [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; CHECK-NEXT: [[TMP2:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT: [[TMP3:%.*]] = lshr i64 [[TMP1]], 32
+; CHECK-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0)
+; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP4]], i32 [[TMP5]])
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[TMP6]], 0
+; CHECK-NEXT: br i1 [[TMP7]], label %[[BB8:.*]], label %[[BB10:.*]]
+; CHECK: [[BB8]]:
+; CHECK-NEXT: [[TMP9:%.*]] = atomicrmw or ptr addrspace(1) [[UNIFORM_PTR]], i8 [[VAL]] monotonic, align 1
+; CHECK-NEXT: br label %[[BB10]]
+; CHECK: [[BB10]]:
+; CHECK-NEXT: [[TMP11:%.*]] = phi i8 [ poison, [[TMP0:%.*]] ], [ [[TMP9]], %[[BB8]] ]
+; CHECK-NEXT: [[TMP16:%.*]] = zext i8 [[TMP11]] to i32
+; CHECK-NEXT: [[TMP17:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP16]])
+; CHECK-NEXT: [[TMP12:%.*]] = trunc i32 [[TMP17]] to i8
+; CHECK-NEXT: [[TMP13:%.*]] = trunc i32 [[TMP6]] to i8
+; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP7]], i8 0, i8 [[VAL]]
+; CHECK-NEXT: [[TMP15:%.*]] = or i8 [[TMP12]], [[TMP14]]
+; CHECK-NEXT: store i8 [[TMP15]], ptr addrspace(1) [[RESULT]], align 1
+; CHECK-NEXT: ret void
+;
+ %rmw = atomicrmw or ptr addrspace(1) %uniform.ptr, i8 %val monotonic, align 1
+ store i8 %rmw, ptr addrspace(1) %result
+ ret void
+}
+
+define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspace(1) %uniform.ptr, i8 %val) {
+; CHECK-LABEL: define amdgpu_kernel void @uniform_add_i8(
+; CHECK-SAME: ptr addrspace(1) [[RESULT:%.*]], ptr addrspace(1) [[UNIFORM_PTR:%.*]], i8 [[VAL:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; CHECK-NEXT: [[TMP2:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT: [[TMP3:%.*]] = lshr i64 [[TMP1]], 32
+; CHECK-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0)
+; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP4]], i32 [[TMP5]])
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP1]])
+; CHECK-NEXT: [[TMP8:%.*]] = trunc i64 [[TMP7]] to i8
+; CHECK-NEXT: [[TMP9:%.*]] = mul i8 [[VAL]], [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP6]], 0
+; CHECK-NEXT: br i1 [[TMP10]], label %[[BB11:.*]], label %[[BB13:.*]]
+; CHECK: [[BB11]]:
+; CHECK-NEXT: [[TMP12:%.*]] = atomicrmw add ptr addrspace(1) [[UNIFORM_PTR]], i8 [[TMP9]] monotonic, align 1
+; CHECK-NEXT: br label %[[BB13]]
+; CHECK: [[BB13]]:
+; CHECK-NEXT: [[TMP14:%.*]] = phi i8 [ poison, [[TMP0:%.*]] ], [ [[TMP12]], %[[BB11]] ]
+; CHECK-NEXT: [[TMP19:%.*]] = zext i8 [[TMP14]] to i32
+; CHECK-NEXT: [[TMP20:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP19]])
+; CHECK-NEXT: [[TMP15:%.*]] = trunc i32 [[TMP20]] to i8
+; CHECK-NEXT: [[TMP16:%.*]] = trunc i32 [[TMP6]] to i8
+; CHECK-NEXT: [[TMP17:%.*]] = mul i8 [[VAL]], [[TMP16]]
+; CHECK-NEXT: [[TMP18:%.*]] = add i8 [[TMP15]], [[TMP17]]
+; CHECK-NEXT: store i8 [[TMP18]], ptr addrspace(1) [[RESULT]], align 1
+; CHECK-NEXT: ret void
+;
+ %rmw = atomicrmw add ptr addrspace(1) %uniform.ptr, i8 %val monotonic, align 1
+ store i8 %rmw, ptr addrspace(1) %result
+ ret void
+}
+
+define amdgpu_kernel void @uniform_xchg_i8(ptr addrspace(1) %result, ptr addrspace(1) %uniform.ptr, i8 %val) {
+; CHECK-LABEL: define amdgpu_kernel void @uniform_xchg_i8(
+; CHECK-SAME: ptr addrspace(1) [[RESULT:%.*]], ptr addrspace(1) [[UNIFORM_PTR:%.*]], i8 [[VAL:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[RMW:%.*]] = atomicrmw xchg ptr addrspace(1) [[UNIFORM_PTR]], i8 [[VAL]] monotonic, align 1
+; CHECK-NEXT: store i8 [[RMW]], ptr addrspace(1) [[RESULT]], align 1
+; CHECK-NEXT: ret void
+;
+ %rmw = atomicrmw xchg ptr addrspace(1) %uniform.ptr, i8 %val monotonic, align 1
+ store i8 %rmw, ptr addrspace(1) %result
+ ret void
+}
+
+define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspace(1) %uniform.ptr, i16 %val) {
+; CHECK-LABEL: define amdgpu_kernel void @uniform_or_i16(
+; CHECK-SAME: ptr addrspace(1) [[RESULT:%.*]], ptr addrspace(1) [[UNIFORM_PTR:%.*]], i16 [[VAL:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; CHECK-NEXT: [[TMP2:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT: [[TMP3:%.*]] = lshr i64 [[TMP1]], 32
+; CHECK-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0)
+; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP4]], i32 [[TMP5]])
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[TMP6]], 0
+; CHECK-NEXT: br i1 [[TMP7]], label %[[BB8:.*]], label %[[BB10:.*]]
+; CHECK: [[BB8]]:
+; CHECK-NEXT: [[TMP9:%.*]] = atomicrmw or ptr addrspace(1) [[UNIFORM_PTR]], i16 [[VAL]] monotonic, align 2
+; CHECK-NEXT: br label %[[BB10]]
+; CHECK: [[BB10]]:
+; CHECK-NEXT: [[TMP11:%.*]] = phi i16 [ poison, [[TMP0:%.*]] ], [ [[TMP9]], %[[BB8]] ]
+; CHECK-NEXT: [[TMP16:%.*]] = zext i16 [[TMP11]] to i32
+; CHECK-NEXT: [[TMP17:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP16]])
+; CHECK-NEXT: [[TMP12:%.*]] = trunc i32 [[TMP17]] to i16
+; CHECK-NEXT: [[TMP13:%.*]] = trunc i32 [[TMP6]] to i16
+; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP7]], i16 0, i16 [[VAL]]
+; CHECK-NEXT: [[TMP15:%.*]] = or i16 [[TMP12]], [[TMP14]]
+; CHECK-NEXT: store i16 [[TMP15]], ptr addrspace(1) [[RESULT]], align 2
+; CHECK-NEXT: ret void
+;
+ %rmw = atomicrmw or ptr addrspace(1) %uniform.ptr, i16 %val monotonic, align 2
+ store i16 %rmw, ptr addrspace(1) %result
+ ret void
+}
+
+define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspace(1) %uniform.ptr, i16 %val) {
+; CHECK-LABEL: define amdgpu_kernel void @uniform_add_i16(
+; CHECK-SAME: ptr addrspace(1) [[RESULT:%.*]], ptr addrspace(1) [[UNIFORM_PTR:%.*]], i16 [[VAL:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; CHECK-NEXT: [[TMP2:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT: [[TMP3:%.*]] = lshr i64 [[TMP1]], 32
+; CHECK-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0)
+; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP4]], i32 [[TMP5]])
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP1]])
+; CHECK-NEXT: [[TMP8:%.*]] = trunc i64 [[TMP7]] to i16
+; CHECK-NEXT: [[TMP9:%.*]] = mul i16 [[VAL]], [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP6]], 0
+; CHECK-NEXT: br i1 [[TMP10]], label %[[BB11:.*]], label %[[BB13:.*]]
+; CHECK: [[BB11]]:
+; CHECK-NEXT: [[TMP12:%.*]] = atomicrmw add ptr addrspace(1) [[UNIFORM_PTR]], i16 [[TMP9]] monotonic, align 2
+; CHECK-NEXT: br label %[[BB13]]
+; CHECK: [[BB13]]:
+; CHECK-NEXT: [[TMP14:%.*]] = phi i16 [ poison, [[TMP0:%.*]] ], [ [[TMP12]], %[[BB11]] ]
+; CHECK-NEXT: [[TMP19:%.*]] = zext i16 [[TMP14]] to i32
+; CHECK-NEXT: [[TMP20:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP19]])
+; CHECK-NEXT: [[TMP15:%.*]] = trunc i32 [[TMP20]] to i16
+; CHECK-NEXT: [[TMP16:%.*]] = trunc i32 [[TMP6]] to i16
+; CHECK-NEXT: [[TMP17:%.*]] = mul i16 [[VAL]], [[TMP16]]
+; CHECK-NEXT: [[TMP18:%.*]] = add i16 [[TMP15]], [[TMP17]]
+; CHECK-NEXT: store i16 [[TMP18]], ptr addrspace(1) [[RESULT]], align 2
+; CHECK-NEXT: ret void
+;
+ %rmw = atomicrmw add ptr addrspace(1) %uniform.ptr, i16 %val monotonic, align 2
+ store i16 %rmw, ptr addrspace(1) %result
+ ret void
+}
+
+define amdgpu_kernel void @uniform_xchg_i16(ptr addrspace(1) %result, ptr addrspace(1) %uniform.ptr, i16 %val) {
+; CHECK-LABEL: define amdgpu_kernel void @uniform_xchg_i16(
+; CHECK-SAME: ptr addrspace(1) [[RESULT:%.*]], ptr addrspace(1) [[UNIFORM_PTR:%.*]], i16 [[VAL:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[RMW:%.*]] = atomicrmw xchg ptr addrspace(1) [[UNIFORM_PTR]], i16 [[VAL]] monotonic, align 2
+; CHECK-NEXT: store i16 [[RMW]], ptr addrspace(1) [[RESULT]], align 2
+; CHECK-NEXT: ret void
+;
+ %rmw = atomicrmw xchg ptr addrspace(1) %uniform.ptr, i16 %val monotonic, align 2
+ store i16 %rmw, ptr addrspace(1) %result
+ ret void
+}
+
+define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrspace(1) %uniform.ptr, half %val) {
+; CHECK-LABEL: define amdgpu_kernel void @uniform_fadd_f16(
+; CHECK-SAME: ptr addrspace(1) [[RESULT:%.*]], ptr addrspace(1) [[UNIFORM_PTR:%.*]], half [[VAL:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[RMW:%.*]] = atomicrmw fadd ptr addrspace(1) [[UNIFORM_PTR]], half [[VAL]] monotonic, align 2
+; CHECK-NEXT: store half [[RMW]], ptr addrspace(1) [[RESULT]], align 2
+; CHECK-NEXT: ret void
+;
+ %rmw = atomicrmw fadd ptr addrspace(1) %uniform.ptr, half %val monotonic, align 2
+ store half %rmw, ptr addrspace(1) %result
+ ret void
+}
+
+define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrspace(1) %uniform.ptr, bfloat %val) {
+; CHECK-LABEL: define amdgpu_kernel void @uniform_fadd_bf16(
+; CHECK-SAME: ptr addrspace(1) [[RESULT:%.*]], ptr addrspace(1) [[UNIFORM_PTR:%.*]], bfloat [[VAL:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[RMW:%.*]] = atomicrmw fadd ptr addrspace(1) [[UNIFORM_PTR]], bfloat [[VAL]] monotonic, align 2
+; CHECK-NEXT: store bfloat [[RMW]], ptr addrspace(1) [[RESULT]], align 2
+; CHECK-NEXT: ret void
+;
+ %rmw = atomicrmw fadd ptr addrspace(1) %uniform.ptr, bfloat %val monotonic, align 2
+ store bfloat %rmw, ptr addrspace(1) %result
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
index bc89a186db010..3737cc414c58f 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
@@ -7149,3 +7149,4244 @@ entry:
store i64 %old, ptr addrspace(1) %out
ret void
}
+
+define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace(1) %uniform.ptr, i8 %val) {
+; GFX7LESS-LABEL: uniform_or_i8:
+; GFX7LESS: ; %bb.0:
+; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7LESS-NEXT: s_load_dword s6, s[4:5], 0xd
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: ; implicit-def: $vgpr0
+; GFX7LESS-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX7LESS-NEXT: s_cbranch_execz .LBB12_2
+; GFX7LESS-NEXT: ; %bb.1:
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: s_and_b32 s8, s2, -4
+; GFX7LESS-NEXT: s_mov_b32 s11, 0xf000
+; GFX7LESS-NEXT: s_and_b32 s2, s2, 3
+; GFX7LESS-NEXT: s_lshl_b32 s2, s2, 3
+; GFX7LESS-NEXT: s_and_b32 s7, s6, 0xff
+; GFX7LESS-NEXT: s_lshl_b32 s7, s7, s2
+; GFX7LESS-NEXT: s_mov_b32 s10, -1
+; GFX7LESS-NEXT: s_mov_b32 s9, s3
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s7
+; GFX7LESS-NEXT: buffer_atomic_or v0, off, s[8:11], 0 glc
+; GFX7LESS-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, s2, v0
+; GFX7LESS-NEXT: .LBB12_2:
+; GFX7LESS-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s6
+; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v0
+; GFX7LESS-NEXT: v_cndmask_b32_e64 v0, v1, 0, vcc
+; GFX7LESS-NEXT: v_or_b32_e32 v0, s4, v0
+; GFX7LESS-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; GFX7LESS-NEXT: s_endpgm
+;
+; GFX8-LABEL: uniform_or_i8:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX8-NEXT: ; implicit-def: $vgpr0
+; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX8-NEXT: s_cbranch_execz .LBB12_2
+; GFX8-NEXT: ; %bb.1:
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: s_and_b32 s8, s2, -4
+; GFX8-NEXT: s_and_b32 s2, s2, 3
+; GFX8-NEXT: s_mov_b32 s9, s3
+; GFX8-NEXT: s_lshl_b32 s2, s2, 3
+; GFX8-NEXT: s_and_b32 s3, s6, 0xff
+; GFX8-NEXT: s_lshl_b32 s3, s3, s2
+; GFX8-NEXT: s_mov_b32 s11, 0xf000
+; GFX8-NEXT: s_mov_b32 s10, -1
+; GFX8-NEXT: v_mov_b32_e32 v0, s3
+; GFX8-NEXT: buffer_atomic_or v0, off, s[8:11], 0 glc
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, s2, v0
+; GFX8-NEXT: .LBB12_2:
+; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX8-NEXT: v_readfirstlane_b32 s4, v0
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v0, s6
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc
+; GFX8-NEXT: s_mov_b32 s3, 0xf000
+; GFX8-NEXT: s_mov_b32 s2, -1
+; GFX8-NEXT: v_or_b32_e32 v0, s4, v0
+; GFX8-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; GFX8-NEXT: s_endpgm
+;
+; GFX9-LABEL: uniform_or_i8:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: ; implicit-def: $vgpr0
+; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT: s_cbranch_execz .LBB12_2
+; GFX9-NEXT: ; %bb.1:
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_and_b32 s8, s2, -4
+; GFX9-NEXT: s_and_b32 s2, s2, 3
+; GFX9-NEXT: s_mov_b32 s9, s3
+; GFX9-NEXT: s_lshl_b32 s2, s2, 3
+; GFX9-NEXT: s_and_b32 s3, s6, 0xff
+; GFX9-NEXT: s_lshl_b32 s3, s3, s2
+; GFX9-NEXT: s_mov_b32 s11, 0xf000
+; GFX9-NEXT: s_mov_b32 s10, -1
+; GFX9-NEXT: v_mov_b32_e32 v0, s3
+; GFX9-NEXT: buffer_atomic_or v0, off, s[8:11], 0 glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, s2, v0
+; GFX9-NEXT: .LBB12_2:
+; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX9-NEXT: v_readfirstlane_b32 s4, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v0, s6
+; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc
+; GFX9-NEXT: s_mov_b32 s3, 0xf000
+; GFX9-NEXT: s_mov_b32 s2, -1
+; GFX9-NEXT: v_or_b32_e32 v0, s4, v0
+; GFX9-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; GFX9-NEXT: s_endpgm
+;
+; GFX1064-LABEL: uniform_or_i8:
+; GFX1064: ; %bb.0:
+; GFX1064-NEXT: s_clause 0x1
+; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX1064-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: ; implicit-def: $vgpr0
+; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX1064-NEXT: s_cbranch_execz .LBB12_2
+; GFX1064-NEXT: ; %bb.1:
+; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT: s_and_b32 s7, s2, 3
+; GFX1064-NEXT: s_and_b32 s8, s6, 0xff
+; GFX1064-NEXT: s_lshl_b32 s7, s7, 3
+; GFX1064-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1064-NEXT: s_lshl_b32 s9, s8, s7
+; GFX1064-NEXT: s_and_b32 s8, s2, -4
+; GFX1064-NEXT: v_mov_b32_e32 v0, s9
+; GFX1064-NEXT: s_mov_b32 s10, -1
+; GFX1064-NEXT: s_mov_b32 s9, s3
+; GFX1064-NEXT: buffer_atomic_or v0, off, s[8:11], 0 glc
+; GFX1064-NEXT: s_waitcnt vmcnt(0)
+; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s7, v0
+; GFX1064-NEXT: .LBB12_2:
+; GFX1064-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX1064-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1064-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1064-NEXT: v_cndmask_b32_e64 v0, s6, 0, vcc
+; GFX1064-NEXT: v_or_b32_e32 v0, s2, v0
+; GFX1064-NEXT: s_mov_b32 s2, -1
+; GFX1064-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; GFX1064-NEXT: s_endpgm
+;
+; GFX1032-LABEL: uniform_or_i8:
+; GFX1032: ; %bb.0:
+; GFX1032-NEXT: s_clause 0x1
+; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX1032-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT: ; implicit-def: $vgpr0
+; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo
+; GFX1032-NEXT: s_cbranch_execz .LBB12_2
+; GFX1032-NEXT: ; %bb.1:
+; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT: s_and_b32 s5, s2, 3
+; GFX1032-NEXT: s_and_b32 s7, s6, 0xff
+; GFX1032-NEXT: s_lshl_b32 s5, s5, 3
+; GFX1032-NEXT: s_and_b32 s8, s2, -4
+; GFX1032-NEXT: s_lshl_b32 s7, s7, s5
+; GFX1032-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1032-NEXT: v_mov_b32_e32 v0, s7
+; GFX1032-NEXT: s_mov_b32 s10, -1
+; GFX1032-NEXT: s_mov_b32 s9, s3
+; GFX1032-NEXT: buffer_atomic_or v0, off, s[8:11], 0 glc
+; GFX1032-NEXT: s_waitcnt vmcnt(0)
+; GFX1032-NEXT: v_lshrrev_b32_e32 v0, s5, v0
+; GFX1032-NEXT: .LBB12_2:
+; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX1032-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1032-NEXT: v_cndmask_b32_e64 v0, s6, 0, vcc_lo
+; GFX1032-NEXT: v_or_b32_e32 v0, s2, v0
+; GFX1032-NEXT: s_mov_b32 s2, -1
+; GFX1032-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; GFX1032-NEXT: s_endpgm
+;
+; GFX1164-LABEL: uniform_or_i8:
+; GFX1164: ; %bb.0:
+; GFX1164-NEXT: s_clause 0x1
+; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1164-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1164-NEXT: ; implicit-def: $vgpr0
+; GFX1164-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX1164-NEXT: s_cbranch_execz .LBB12_2
+; GFX1164-NEXT: ; %bb.1:
+; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-NEXT: s_and_b32 s7, s2, 3
+; GFX1164-NEXT: s_and_b32 s8, s6, 0xff
+; GFX1164-NEXT: s_lshl_b32 s7, s7, 3
+; GFX1164-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1164-NEXT: s_lshl_b32 s9, s8, s7
+; GFX1164-NEXT: s_and_b32 s8, s2, -4
+; GFX1164-NEXT: v_mov_b32_e32 v0, s9
+; GFX1164-NEXT: s_mov_b32 s10, -1
+; GFX1164-NEXT: s_mov_b32 s9, s3
+; GFX1164-NEXT: buffer_atomic_or_b32 v0, off, s[8:11], 0 glc
+; GFX1164-NEXT: s_waitcnt vmcnt(0)
+; GFX1164-NEXT: v_lshrrev_b32_e32 v0, s7, v0
+; GFX1164-NEXT: .LBB12_2:
+; GFX1164-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1164-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1164-NEXT: v_cndmask_b32_e64 v0, s6, 0, vcc
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_or_b32_e32 v0, s2, v0
+; GFX1164-NEXT: s_mov_b32 s2, -1
+; GFX1164-NEXT: buffer_store_b8 v0, off, s[0:3], 0
+; GFX1164-NEXT: s_endpgm
+;
+; GFX1132-LABEL: uniform_or_i8:
+; GFX1132: ; %bb.0:
+; GFX1132-NEXT: s_clause 0x1
+; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1132-NEXT: s_load_b32 s4, s[4:5], 0x34
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1132-NEXT: ; implicit-def: $vgpr0
+; GFX1132-NEXT: s_and_saveexec_b32 s5, vcc_lo
+; GFX1132-NEXT: s_cbranch_execz .LBB12_2
+; GFX1132-NEXT: ; %bb.1:
+; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-NEXT: s_and_b32 s6, s2, 3
+; GFX1132-NEXT: s_and_b32 s7, s4, 0xff
+; GFX1132-NEXT: s_lshl_b32 s6, s6, 3
+; GFX1132-NEXT: s_and_b32 s8, s2, -4
+; GFX1132-NEXT: s_lshl_b32 s7, s7, s6
+; GFX1132-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1132-NEXT: v_mov_b32_e32 v0, s7
+; GFX1132-NEXT: s_mov_b32 s10, -1
+; GFX1132-NEXT: s_mov_b32 s9, s3
+; GFX1132-NEXT: buffer_atomic_or_b32 v0, off, s[8:11], 0 glc
+; GFX1132-NEXT: s_waitcnt vmcnt(0)
+; GFX1132-NEXT: v_lshrrev_b32_e32 v0, s6, v0
+; GFX1132-NEXT: .LBB12_2:
+; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1132-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1132-NEXT: v_cndmask_b32_e64 v0, s4, 0, vcc_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: v_or_b32_e32 v0, s2, v0
+; GFX1132-NEXT: s_mov_b32 s2, -1
+; GFX1132-NEXT: buffer_store_b8 v0, off, s[0:3], 0
+; GFX1132-NEXT: s_endpgm
+;
+; GFX1264-LABEL: uniform_or_i8:
+; GFX1264: ; %bb.0:
+; GFX1264-NEXT: s_clause 0x1
+; GFX1264-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1264-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1264-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1264-NEXT: ; implicit-def: $vgpr0
+; GFX1264-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX1264-NEXT: s_cbranch_execz .LBB12_2
+; GFX1264-NEXT: ; %bb.1:
+; GFX1264-NEXT: s_wait_kmcnt 0x0
+; GFX1264-NEXT: s_and_b32 s7, s2, 3
+; GFX1264-NEXT: s_and_b32 s8, s6, 0xff
+; GFX1264-NEXT: s_lshl_b32 s7, s7, 3
+; GFX1264-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1264-NEXT: s_lshl_b32 s9, s8, s7
+; GFX1264-NEXT: s_and_b32 s8, s2, -4
+; GFX1264-NEXT: v_mov_b32_e32 v0, s9
+; GFX1264-NEXT: s_mov_b32 s10, -1
+; GFX1264-NEXT: s_mov_b32 s9, s3
+; GFX1264-NEXT: buffer_atomic_or_b32 v0, off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1264-NEXT: s_wait_loadcnt 0x0
+; GFX1264-NEXT: v_lshrrev_b32_e32 v0, s7, v0
+; GFX1264-NEXT: .LBB12_2:
+; GFX1264-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1264-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX1264-NEXT: s_wait_kmcnt 0x0
+; GFX1264-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1264-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1264-NEXT: v_cndmask_b32_e64 v0, s6, 0, vcc
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1264-NEXT: v_or_b32_e32 v0, s2, v0
+; GFX1264-NEXT: s_mov_b32 s2, -1
+; GFX1264-NEXT: buffer_store_b8 v0, off, s[0:3], null
+; GFX1264-NEXT: s_endpgm
+;
+; GFX1232-LABEL: uniform_or_i8:
+; GFX1232: ; %bb.0:
+; GFX1232-NEXT: s_clause 0x1
+; GFX1232-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1232-NEXT: s_load_b32 s4, s[4:5], 0x34
+; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1232-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1232-NEXT: ; implicit-def: $vgpr0
+; GFX1232-NEXT: s_and_saveexec_b32 s5, vcc_lo
+; GFX1232-NEXT: s_cbranch_execz .LBB12_2
+; GFX1232-NEXT: ; %bb.1:
+; GFX1232-NEXT: s_wait_kmcnt 0x0
+; GFX1232-NEXT: s_and_b32 s6, s2, 3
+; GFX1232-NEXT: s_and_b32 s7, s4, 0xff
+; GFX1232-NEXT: s_lshl_b32 s6, s6, 3
+; GFX1232-NEXT: s_and_b32 s8, s2, -4
+; GFX1232-NEXT: s_lshl_b32 s7, s7, s6
+; GFX1232-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1232-NEXT: v_mov_b32_e32 v0, s7
+; GFX1232-NEXT: s_mov_b32 s10, -1
+; GFX1232-NEXT: s_mov_b32 s9, s3
+; GFX1232-NEXT: buffer_atomic_or_b32 v0, off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1232-NEXT: s_wait_loadcnt 0x0
+; GFX1232-NEXT: v_lshrrev_b32_e32 v0, s6, v0
+; GFX1232-NEXT: .LBB12_2:
+; GFX1232-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1232-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX1232-NEXT: s_wait_kmcnt 0x0
+; GFX1232-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1232-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1232-NEXT: v_cndmask_b32_e64 v0, s4, 0, vcc_lo
+; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1232-NEXT: v_or_b32_e32 v0, s2, v0
+; GFX1232-NEXT: s_mov_b32 s2, -1
+; GFX1232-NEXT: buffer_store_b8 v0, off, s[0:3], null
+; GFX1232-NEXT: s_endpgm
+ %rmw = atomicrmw or ptr addrspace(1) %uniform.ptr, i8 %val monotonic, align 1
+ store i8 %rmw, ptr addrspace(1) %result
+ ret void
+}
+
+define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspace(1) %uniform.ptr, i8 %val) {
+; GFX7LESS-LABEL: uniform_add_i8:
+; GFX7LESS: ; %bb.0:
+; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec
+; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7LESS-NEXT: s_load_dword s10, s[4:5], 0xd
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v4, s7, v0
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
+; GFX7LESS-NEXT: ; implicit-def: $vgpr0
+; GFX7LESS-NEXT: s_and_saveexec_b64 s[8:9], vcc
+; GFX7LESS-NEXT: s_cbranch_execz .LBB13_4
+; GFX7LESS-NEXT: ; %bb.1:
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: s_and_b32 s4, s2, -4
+; GFX7LESS-NEXT: s_mov_b32 s5, s3
+; GFX7LESS-NEXT: s_and_b32 s11, s2, 3
+; GFX7LESS-NEXT: s_mov_b64 s[2:3], 0
+; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT: s_mul_i32 s6, s10, s6
+; GFX7LESS-NEXT: s_lshl_b32 s11, s11, 3
+; GFX7LESS-NEXT: s_load_dword s15, s[4:5], 0x0
+; GFX7LESS-NEXT: s_lshl_b32 s12, 0xff, s11
+; GFX7LESS-NEXT: s_and_b32 s6, s6, 0xff
+; GFX7LESS-NEXT: s_not_b32 s13, s12
+; GFX7LESS-NEXT: s_lshl_b32 s14, s6, s11
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s15
+; GFX7LESS-NEXT: s_mov_b32 s6, -1
+; GFX7LESS-NEXT: .LBB13_2: ; %atomicrmw.start
+; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: v_add_i32_e32 v0, vcc, s14, v1
+; GFX7LESS-NEXT: v_and_b32_e32 v0, s12, v0
+; GFX7LESS-NEXT: s_waitcnt expcnt(0)
+; GFX7LESS-NEXT: v_and_b32_e32 v2, s13, v1
+; GFX7LESS-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
+; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX7LESS-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX7LESS-NEXT: s_cbranch_execnz .LBB13_2
+; GFX7LESS-NEXT: ; %bb.3: ; %atomicrmw.end
+; GFX7LESS-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, s11, v2
+; GFX7LESS-NEXT: .LBB13_4: ; %Flow
+; GFX7LESS-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v0
+; GFX7LESS-NEXT: s_and_b32 s5, s10, 0xff
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s4
+; GFX7LESS-NEXT: v_mad_u32_u24 v0, s5, v4, v0
+; GFX7LESS-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; GFX7LESS-NEXT: s_endpgm
+;
+; GFX8-LABEL: uniform_add_i8:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT: s_load_dword s10, s[4:5], 0x34
+; GFX8-NEXT: s_mov_b64 s[6:7], exec
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
+; GFX8-NEXT: ; implicit-def: $vgpr0
+; GFX8-NEXT: s_and_saveexec_b64 s[8:9], vcc
+; GFX8-NEXT: s_cbranch_execz .LBB13_4
+; GFX8-NEXT: ; %bb.1:
+; GFX8-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: s_mul_i32 s6, s10, s4
+; GFX8-NEXT: s_and_b32 s4, s2, -4
+; GFX8-NEXT: s_mov_b32 s5, s3
+; GFX8-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX8-NEXT: s_and_b32 s2, s2, 3
+; GFX8-NEXT: s_lshl_b32 s11, s2, 3
+; GFX8-NEXT: s_lshl_b32 s12, 0xff, s11
+; GFX8-NEXT: s_and_b32 s2, s6, 0xff
+; GFX8-NEXT: s_not_b32 s13, s12
+; GFX8-NEXT: s_lshl_b32 s14, s2, s11
+; GFX8-NEXT: s_mov_b64 s[2:3], 0
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-NEXT: s_mov_b32 s6, -1
+; GFX8-NEXT: .LBB13_2: ; %atomicrmw.start
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s14, v1
+; GFX8-NEXT: v_and_b32_e32 v2, s13, v1
+; GFX8-NEXT: v_and_b32_e32 v0, s12, v0
+; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX8-NEXT: v_mov_b32_e32 v3, v1
+; GFX8-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX8-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX8-NEXT: v_mov_b32_e32 v1, v2
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX8-NEXT: s_cbranch_execnz .LBB13_2
+; GFX8-NEXT: ; %bb.3: ; %atomicrmw.end
+; GFX8-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, s11, v2
+; GFX8-NEXT: .LBB13_4: ; %Flow
+; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX8-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX8-NEXT: v_readfirstlane_b32 s4, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s4
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: s_mov_b32 s3, 0xf000
+; GFX8-NEXT: s_mov_b32 s2, -1
+; GFX8-NEXT: v_mad_u16 v0, s10, v4, v0
+; GFX8-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; GFX8-NEXT: s_endpgm
+;
+; GFX9-LABEL: uniform_add_i8:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_load_dword s10, s[4:5], 0x34
+; GFX9-NEXT: s_mov_b64 s[6:7], exec
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
+; GFX9-NEXT: ; implicit-def: $vgpr0
+; GFX9-NEXT: s_and_saveexec_b64 s[8:9], vcc
+; GFX9-NEXT: s_cbranch_execz .LBB13_4
+; GFX9-NEXT: ; %bb.1:
+; GFX9-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_mul_i32 s6, s10, s4
+; GFX9-NEXT: s_and_b32 s4, s2, -4
+; GFX9-NEXT: s_mov_b32 s5, s3
+; GFX9-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX9-NEXT: s_and_b32 s2, s2, 3
+; GFX9-NEXT: s_lshl_b32 s11, s2, 3
+; GFX9-NEXT: s_lshl_b32 s12, 0xff, s11
+; GFX9-NEXT: s_and_b32 s2, s6, 0xff
+; GFX9-NEXT: s_not_b32 s13, s12
+; GFX9-NEXT: s_lshl_b32 s14, s2, s11
+; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v1, s7
+; GFX9-NEXT: s_mov_b32 s7, 0xf000
+; GFX9-NEXT: s_mov_b32 s6, -1
+; GFX9-NEXT: .LBB13_2: ; %atomicrmw.start
+; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: v_add_u32_e32 v0, s14, v1
+; GFX9-NEXT: v_and_b32_e32 v0, s12, v0
+; GFX9-NEXT: v_and_or_b32 v0, v1, s13, v0
+; GFX9-NEXT: v_mov_b32_e32 v3, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, v0
+; GFX9-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: v_mov_b32_e32 v1, v2
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_cbranch_execnz .LBB13_2
+; GFX9-NEXT: ; %bb.3: ; %atomicrmw.end
+; GFX9-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, s11, v2
+; GFX9-NEXT: .LBB13_4: ; %Flow
+; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX9-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX9-NEXT: v_readfirstlane_b32 s4, v0
+; GFX9-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 s3, 0xf000
+; GFX9-NEXT: s_mov_b32 s2, -1
+; GFX9-NEXT: v_mad_legacy_u16 v0, s10, v4, v0
+; GFX9-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; GFX9-NEXT: s_endpgm
+;
+; GFX1064-LABEL: uniform_add_i8:
+; GFX1064: ; %bb.0:
+; GFX1064-NEXT: s_clause 0x1
+; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX1064-NEXT: s_load_dword s10, s[4:5], 0x34
+; GFX1064-NEXT: s_mov_b64 s[6:7], exec
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX1064-NEXT: ; implicit-def: $vgpr0
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
+; GFX1064-NEXT: s_and_saveexec_b64 s[8:9], vcc
+; GFX1064-NEXT: s_cbranch_execz .LBB13_4
+; GFX1064-NEXT: ; %bb.1:
+; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT: s_and_b32 s4, s2, -4
+; GFX1064-NEXT: s_mov_b32 s5, s3
+; GFX1064-NEXT: s_and_b32 s2, s2, 3
+; GFX1064-NEXT: s_load_dword s3, s[4:5], 0x0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1064-NEXT: s_lshl_b32 s11, s2, 3
+; GFX1064-NEXT: s_mul_i32 s2, s10, s6
+; GFX1064-NEXT: s_lshl_b32 s12, 0xff, s11
+; GFX1064-NEXT: s_and_b32 s2, s2, 0xff
+; GFX1064-NEXT: s_not_b32 s13, s12
+; GFX1064-NEXT: s_lshl_b32 s14, s2, s11
+; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1064-NEXT: s_mov_b32 s6, -1
+; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT: v_mov_b32_e32 v1, s3
+; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: .LBB13_2: ; %atomicrmw.start
+; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: v_add_nc_u32_e32 v0, s14, v1
+; GFX1064-NEXT: v_and_b32_e32 v0, s12, v0
+; GFX1064-NEXT: v_and_or_b32 v0, v1, s13, v0
+; GFX1064-NEXT: v_mov_b32_e32 v3, v1
+; GFX1064-NEXT: v_mov_b32_e32 v2, v0
+; GFX1064-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
+; GFX1064-NEXT: s_waitcnt vmcnt(0)
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX1064-NEXT: v_mov_b32_e32 v1, v2
+; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_cbranch_execnz .LBB13_2
+; GFX1064-NEXT: ; %bb.3: ; %atomicrmw.end
+; GFX1064-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s11, v2
+; GFX1064-NEXT: .LBB13_4: ; %Flow
+; GFX1064-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX1064-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1064-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1064-NEXT: v_mad_u16 v0, s10, v4, s2
+; GFX1064-NEXT: s_mov_b32 s2, -1
+; GFX1064-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; GFX1064-NEXT: s_endpgm
+;
+; GFX1032-LABEL: uniform_add_i8:
+; GFX1032: ; %bb.0:
+; GFX1032-NEXT: s_clause 0x1
+; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX1032-NEXT: s_load_dword s8, s[4:5], 0x34
+; GFX1032-NEXT: s_mov_b32 s6, exec_lo
+; GFX1032-NEXT: s_mov_b32 s10, 0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
+; GFX1032-NEXT: ; implicit-def: $vgpr0
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v4
+; GFX1032-NEXT: s_and_saveexec_b32 s9, vcc_lo
+; GFX1032-NEXT: s_cbranch_execz .LBB13_4
+; GFX1032-NEXT: ; %bb.1:
+; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT: s_and_b32 s4, s2, -4
+; GFX1032-NEXT: s_mov_b32 s5, s3
+; GFX1032-NEXT: s_and_b32 s2, s2, 3
+; GFX1032-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s6
+; GFX1032-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1032-NEXT: s_mul_i32 s6, s8, s6
+; GFX1032-NEXT: s_lshl_b32 s3, 0xff, s2
+; GFX1032-NEXT: s_and_b32 s6, s6, 0xff
+; GFX1032-NEXT: s_not_b32 s11, s3
+; GFX1032-NEXT: s_lshl_b32 s12, s6, s2
+; GFX1032-NEXT: s_mov_b32 s6, -1
+; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT: v_mov_b32_e32 v1, s7
+; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1032-NEXT: .LBB13_2: ; %atomicrmw.start
+; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: v_add_nc_u32_e32 v0, s12, v1
+; GFX1032-NEXT: v_and_b32_e32 v0, s3, v0
+; GFX1032-NEXT: v_and_or_b32 v0, v1, s11, v0
+; GFX1032-NEXT: v_mov_b32_e32 v3, v1
+; GFX1032-NEXT: v_mov_b32_e32 v2, v0
+; GFX1032-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
+; GFX1032-NEXT: s_waitcnt vmcnt(0)
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX1032-NEXT: v_mov_b32_e32 v1, v2
+; GFX1032-NEXT: s_or_b32 s10, vcc_lo, s10
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s10
+; GFX1032-NEXT: s_cbranch_execnz .LBB13_2
+; GFX1032-NEXT: ; %bb.3: ; %atomicrmw.end
+; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s10
+; GFX1032-NEXT: v_lshrrev_b32_e32 v0, s2, v2
+; GFX1032-NEXT: .LBB13_4: ; %Flow
+; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s9
+; GFX1032-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1032-NEXT: v_mad_u16 v0, s8, v4, s2
+; GFX1032-NEXT: s_mov_b32 s2, -1
+; GFX1032-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; GFX1032-NEXT: s_endpgm
+;
+; GFX1164-LABEL: uniform_add_i8:
+; GFX1164: ; %bb.0:
+; GFX1164-NEXT: s_clause 0x1
+; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1164-NEXT: s_load_b32 s10, s[4:5], 0x34
+; GFX1164-NEXT: s_mov_b64 s[6:7], exec
+; GFX1164-NEXT: s_mov_b64 s[8:9], exec
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX1164-NEXT: ; implicit-def: $vgpr0
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1164-NEXT: s_cbranch_execz .LBB13_4
+; GFX1164-NEXT: ; %bb.1:
+; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-NEXT: s_and_b32 s4, s2, -4
+; GFX1164-NEXT: s_mov_b32 s5, s3
+; GFX1164-NEXT: s_and_b32 s2, s2, 3
+; GFX1164-NEXT: s_load_b32 s3, s[4:5], 0x0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1164-NEXT: s_lshl_b32 s11, s2, 3
+; GFX1164-NEXT: s_mul_i32 s2, s10, s6
+; GFX1164-NEXT: s_lshl_b32 s12, 0xff, s11
+; GFX1164-NEXT: s_and_b32 s2, s2, 0xff
+; GFX1164-NEXT: s_not_b32 s13, s12
+; GFX1164-NEXT: s_lshl_b32 s14, s2, s11
+; GFX1164-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1164-NEXT: s_mov_b32 s6, -1
+; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-NEXT: v_mov_b32_e32 v1, s3
+; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: .LBB13_2: ; %atomicrmw.start
+; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: v_add_nc_u32_e32 v0, s14, v1
+; GFX1164-NEXT: v_and_b32_e32 v0, s12, v0
+; GFX1164-NEXT: s_waitcnt_depctr 0xfff
+; GFX1164-NEXT: v_and_or_b32 v0, v1, s13, v0
+; GFX1164-NEXT: v_mov_b32_e32 v3, v1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1164-NEXT: v_mov_b32_e32 v2, v0
+; GFX1164-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
+; GFX1164-NEXT: s_waitcnt vmcnt(0)
+; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX1164-NEXT: v_mov_b32_e32 v1, v2
+; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_cbranch_execnz .LBB13_2
+; GFX1164-NEXT: ; %bb.3: ; %atomicrmw.end
+; GFX1164-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: v_lshrrev_b32_e32 v0, s11, v2
+; GFX1164-NEXT: .LBB13_4: ; %Flow
+; GFX1164-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1164-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mad_u16 v0, s10, v4, s2
+; GFX1164-NEXT: s_mov_b32 s2, -1
+; GFX1164-NEXT: buffer_store_b8 v0, off, s[0:3], 0
+; GFX1164-NEXT: s_endpgm
+;
+; GFX1132-LABEL: uniform_add_i8:
+; GFX1132: ; %bb.0:
+; GFX1132-NEXT: s_clause 0x1
+; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1132-NEXT: s_load_b32 s8, s[4:5], 0x34
+; GFX1132-NEXT: s_mov_b32 s6, exec_lo
+; GFX1132-NEXT: s_mov_b32 s10, 0
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
+; GFX1132-NEXT: s_mov_b32 s9, exec_lo
+; GFX1132-NEXT: ; implicit-def: $vgpr0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1132-NEXT: s_cbranch_execz .LBB13_4
+; GFX1132-NEXT: ; %bb.1:
+; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-NEXT: s_and_b32 s4, s2, -4
+; GFX1132-NEXT: s_mov_b32 s5, s3
+; GFX1132-NEXT: s_and_b32 s2, s2, 3
+; GFX1132-NEXT: s_load_b32 s7, s[4:5], 0x0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s6, s6
+; GFX1132-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1132-NEXT: s_mul_i32 s6, s8, s6
+; GFX1132-NEXT: s_lshl_b32 s3, 0xff, s2
+; GFX1132-NEXT: s_and_b32 s6, s6, 0xff
+; GFX1132-NEXT: s_not_b32 s11, s3
+; GFX1132-NEXT: s_lshl_b32 s12, s6, s2
+; GFX1132-NEXT: s_mov_b32 s6, -1
+; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-NEXT: v_mov_b32_e32 v1, s7
+; GFX1132-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1132-NEXT: .LBB13_2: ; %atomicrmw.start
+; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_add_nc_u32_e32 v0, s12, v1
+; GFX1132-NEXT: v_and_b32_e32 v0, s3, v0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_and_or_b32 v0, v1, s11, v0
+; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
+; GFX1132-NEXT: s_waitcnt vmcnt(0)
+; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX1132-NEXT: v_mov_b32_e32 v1, v2
+; GFX1132-NEXT: s_or_b32 s10, vcc_lo, s10
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s10
+; GFX1132-NEXT: s_cbranch_execnz .LBB13_2
+; GFX1132-NEXT: ; %bb.3: ; %atomicrmw.end
+; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s10
+; GFX1132-NEXT: v_lshrrev_b32_e32 v0, s2, v2
+; GFX1132-NEXT: .LBB13_4: ; %Flow
+; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s9
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1132-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: v_mad_u16 v0, s8, v4, s2
+; GFX1132-NEXT: s_mov_b32 s2, -1
+; GFX1132-NEXT: buffer_store_b8 v0, off, s[0:3], 0
+; GFX1132-NEXT: s_endpgm
+;
+; GFX1264-LABEL: uniform_add_i8:
+; GFX1264: ; %bb.0:
+; GFX1264-NEXT: s_clause 0x1
+; GFX1264-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1264-NEXT: s_load_b32 s10, s[4:5], 0x34
+; GFX1264-NEXT: s_mov_b64 s[6:7], exec
+; GFX1264-NEXT: s_mov_b64 s[8:9], exec
+; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX1264-NEXT: ; implicit-def: $vgpr0
+; GFX1264-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1264-NEXT: s_cbranch_execz .LBB13_4
+; GFX1264-NEXT: ; %bb.1:
+; GFX1264-NEXT: s_wait_kmcnt 0x0
+; GFX1264-NEXT: s_and_b32 s4, s2, -4
+; GFX1264-NEXT: s_mov_b32 s5, s3
+; GFX1264-NEXT: s_and_b32 s2, s2, 3
+; GFX1264-NEXT: s_load_b32 s3, s[4:5], 0x0
+; GFX1264-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1264-NEXT: s_lshl_b32 s11, s2, 3
+; GFX1264-NEXT: s_wait_alu 0xfffe
+; GFX1264-NEXT: s_mul_i32 s2, s10, s6
+; GFX1264-NEXT: s_lshl_b32 s12, 0xff, s11
+; GFX1264-NEXT: s_and_b32 s2, s2, 0xff
+; GFX1264-NEXT: s_not_b32 s13, s12
+; GFX1264-NEXT: s_lshl_b32 s14, s2, s11
+; GFX1264-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1264-NEXT: s_mov_b32 s6, -1
+; GFX1264-NEXT: s_wait_kmcnt 0x0
+; GFX1264-NEXT: v_mov_b32_e32 v1, s3
+; GFX1264-NEXT: s_mov_b64 s[2:3], 0
+; GFX1264-NEXT: .LBB13_2: ; %atomicrmw.start
+; GFX1264-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1264-NEXT: v_add_nc_u32_e32 v0, s14, v1
+; GFX1264-NEXT: v_and_b32_e32 v0, s12, v0
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1264-NEXT: v_and_or_b32 v0, v1, s13, v0
+; GFX1264-NEXT: v_mov_b32_e32 v3, v1
+; GFX1264-NEXT: v_mov_b32_e32 v2, v0
+; GFX1264-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1264-NEXT: s_wait_loadcnt 0x0
+; GFX1264-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX1264-NEXT: v_mov_b32_e32 v1, v2
+; GFX1264-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1264-NEXT: s_wait_alu 0xfffe
+; GFX1264-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1264-NEXT: s_cbranch_execnz .LBB13_2
+; GFX1264-NEXT: ; %bb.3: ; %atomicrmw.end
+; GFX1264-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX1264-NEXT: v_lshrrev_b32_e32 v0, s11, v2
+; GFX1264-NEXT: .LBB13_4: ; %Flow
+; GFX1264-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1264-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX1264-NEXT: s_wait_kmcnt 0x0
+; GFX1264-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1264-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1264-NEXT: s_wait_alu 0xf1ff
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1264-NEXT: v_mad_u16 v0, s10, v4, s2
+; GFX1264-NEXT: s_mov_b32 s2, -1
+; GFX1264-NEXT: buffer_store_b8 v0, off, s[0:3], null
+; GFX1264-NEXT: s_endpgm
+;
+; GFX1232-LABEL: uniform_add_i8:
+; GFX1232: ; %bb.0:
+; GFX1232-NEXT: s_clause 0x1
+; GFX1232-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1232-NEXT: s_load_b32 s8, s[4:5], 0x34
+; GFX1232-NEXT: s_mov_b32 s6, exec_lo
+; GFX1232-NEXT: s_mov_b32 s10, 0
+; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
+; GFX1232-NEXT: s_mov_b32 s9, exec_lo
+; GFX1232-NEXT: ; implicit-def: $vgpr0
+; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1232-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1232-NEXT: s_cbranch_execz .LBB13_4
+; GFX1232-NEXT: ; %bb.1:
+; GFX1232-NEXT: s_wait_kmcnt 0x0
+; GFX1232-NEXT: s_and_b32 s4, s2, -4
+; GFX1232-NEXT: s_mov_b32 s5, s3
+; GFX1232-NEXT: s_and_b32 s2, s2, 3
+; GFX1232-NEXT: s_load_b32 s7, s[4:5], 0x0
+; GFX1232-NEXT: s_bcnt1_i32_b32 s6, s6
+; GFX1232-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1232-NEXT: s_wait_alu 0xfffe
+; GFX1232-NEXT: s_mul_i32 s6, s8, s6
+; GFX1232-NEXT: s_lshl_b32 s3, 0xff, s2
+; GFX1232-NEXT: s_wait_alu 0xfffe
+; GFX1232-NEXT: s_and_b32 s6, s6, 0xff
+; GFX1232-NEXT: s_not_b32 s11, s3
+; GFX1232-NEXT: s_wait_alu 0xfffe
+; GFX1232-NEXT: s_lshl_b32 s12, s6, s2
+; GFX1232-NEXT: s_mov_b32 s6, -1
+; GFX1232-NEXT: s_wait_kmcnt 0x0
+; GFX1232-NEXT: v_mov_b32_e32 v1, s7
+; GFX1232-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1232-NEXT: .LBB13_2: ; %atomicrmw.start
+; GFX1232-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1232-NEXT: v_add_nc_u32_e32 v0, s12, v1
+; GFX1232-NEXT: v_and_b32_e32 v0, s3, v0
+; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1232-NEXT: v_and_or_b32 v0, v1, s11, v0
+; GFX1232-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1232-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1232-NEXT: s_wait_loadcnt 0x0
+; GFX1232-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX1232-NEXT: v_mov_b32_e32 v1, v2
+; GFX1232-NEXT: s_or_b32 s10, vcc_lo, s10
+; GFX1232-NEXT: s_wait_alu 0xfffe
+; GFX1232-NEXT: s_and_not1_b32 exec_lo, exec_lo, s10
+; GFX1232-NEXT: s_cbranch_execnz .LBB13_2
+; GFX1232-NEXT: ; %bb.3: ; %atomicrmw.end
+; GFX1232-NEXT: s_or_b32 exec_lo, exec_lo, s10
+; GFX1232-NEXT: v_lshrrev_b32_e32 v0, s2, v2
+; GFX1232-NEXT: .LBB13_4: ; %Flow
+; GFX1232-NEXT: s_or_b32 exec_lo, exec_lo, s9
+; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1232-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX1232-NEXT: s_wait_kmcnt 0x0
+; GFX1232-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1232-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1232-NEXT: s_wait_alu 0xf1ff
+; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1232-NEXT: v_mad_u16 v0, s8, v4, s2
+; GFX1232-NEXT: s_mov_b32 s2, -1
+; GFX1232-NEXT: buffer_store_b8 v0, off, s[0:3], null
+; GFX1232-NEXT: s_endpgm
+ %rmw = atomicrmw add ptr addrspace(1) %uniform.ptr, i8 %val monotonic, align 1
+ store i8 %rmw, ptr addrspace(1) %result
+ ret void
+}
+
+define amdgpu_kernel void @uniform_xchg_i8(ptr addrspace(1) %result, ptr addrspace(1) %uniform.ptr, i8 %val) {
+; GFX7LESS-LABEL: uniform_xchg_i8:
+; GFX7LESS: ; %bb.0:
+; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7LESS-NEXT: s_load_dword s6, s[4:5], 0xd
+; GFX7LESS-NEXT: s_mov_b64 s[8:9], 0
+; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: s_and_b32 s4, s2, -4
+; GFX7LESS-NEXT: s_mov_b32 s5, s3
+; GFX7LESS-NEXT: s_and_b32 s2, s2, 3
+; GFX7LESS-NEXT: s_and_b32 s3, s6, 0xff
+; GFX7LESS-NEXT: s_load_dword s6, s[4:5], 0x0
+; GFX7LESS-NEXT: s_lshl_b32 s10, s2, 3
+; GFX7LESS-NEXT: s_lshl_b32 s11, 0xff, s10
+; GFX7LESS-NEXT: s_lshl_b32 s2, s3, s10
+; GFX7LESS-NEXT: s_not_b32 s3, s11
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s6
+; GFX7LESS-NEXT: s_mov_b32 s6, -1
+; GFX7LESS-NEXT: .LBB14_1: ; %atomicrmw.start
+; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: v_and_b32_e32 v0, s3, v1
+; GFX7LESS-NEXT: v_or_b32_e32 v0, s2, v0
+; GFX7LESS-NEXT: s_waitcnt expcnt(0)
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
+; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX7LESS-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX7LESS-NEXT: s_cbranch_execnz .LBB14_1
+; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX7LESS-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, s10, v2
+; GFX7LESS-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; GFX7LESS-NEXT: s_endpgm
+;
+; GFX8-LABEL: uniform_xchg_i8:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: s_and_b32 s4, s2, -4
+; GFX8-NEXT: s_mov_b32 s5, s3
+; GFX8-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX8-NEXT: s_and_b32 s2, s2, 3
+; GFX8-NEXT: s_lshl_b32 s8, s2, 3
+; GFX8-NEXT: s_lshl_b32 s2, 0xff, s8
+; GFX8-NEXT: s_not_b32 s9, s2
+; GFX8-NEXT: s_and_b32 s2, s6, 0xff
+; GFX8-NEXT: s_lshl_b32 s10, s2, s8
+; GFX8-NEXT: s_mov_b64 s[2:3], 0
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-NEXT: s_mov_b32 s6, -1
+; GFX8-NEXT: .LBB14_1: ; %atomicrmw.start
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: v_and_b32_e32 v0, s9, v1
+; GFX8-NEXT: v_or_b32_e32 v0, s10, v0
+; GFX8-NEXT: v_mov_b32_e32 v3, v1
+; GFX8-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX8-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX8-NEXT: v_mov_b32_e32 v1, v2
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX8-NEXT: s_cbranch_execnz .LBB14_1
+; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX8-NEXT: s_mov_b32 s3, 0xf000
+; GFX8-NEXT: s_mov_b32 s2, -1
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, s8, v2
+; GFX8-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; GFX8-NEXT: s_endpgm
+;
+; GFX9-LABEL: uniform_xchg_i8:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_and_b32 s4, s2, -4
+; GFX9-NEXT: s_mov_b32 s5, s3
+; GFX9-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX9-NEXT: s_and_b32 s2, s2, 3
+; GFX9-NEXT: s_lshl_b32 s8, s2, 3
+; GFX9-NEXT: s_lshl_b32 s2, 0xff, s8
+; GFX9-NEXT: s_not_b32 s9, s2
+; GFX9-NEXT: s_and_b32 s2, s6, 0xff
+; GFX9-NEXT: s_lshl_b32 s10, s2, s8
+; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v1, s7
+; GFX9-NEXT: s_mov_b32 s7, 0xf000
+; GFX9-NEXT: s_mov_b32 s6, -1
+; GFX9-NEXT: .LBB14_1: ; %atomicrmw.start
+; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: v_and_b32_e32 v0, s9, v1
+; GFX9-NEXT: v_or_b32_e32 v0, s10, v0
+; GFX9-NEXT: v_mov_b32_e32 v3, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, v0
+; GFX9-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: v_mov_b32_e32 v1, v2
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_cbranch_execnz .LBB14_1
+; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_mov_b32 s3, 0xf000
+; GFX9-NEXT: s_mov_b32 s2, -1
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, s8, v2
+; GFX9-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; GFX9-NEXT: s_endpgm
+;
+; GFX1064-LABEL: uniform_xchg_i8:
+; GFX1064: ; %bb.0:
+; GFX1064-NEXT: s_clause 0x1
+; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX1064-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT: s_and_b32 s4, s2, -4
+; GFX1064-NEXT: s_mov_b32 s5, s3
+; GFX1064-NEXT: s_and_b32 s2, s2, 3
+; GFX1064-NEXT: s_load_dword s3, s[4:5], 0x0
+; GFX1064-NEXT: s_lshl_b32 s8, s2, 3
+; GFX1064-NEXT: s_and_b32 s6, s6, 0xff
+; GFX1064-NEXT: s_lshl_b32 s2, 0xff, s8
+; GFX1064-NEXT: s_lshl_b32 s10, s6, s8
+; GFX1064-NEXT: s_not_b32 s9, s2
+; GFX1064-NEXT: s_mov_b32 s6, -1
+; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT: v_mov_b32_e32 v1, s3
+; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: .LBB14_1: ; %atomicrmw.start
+; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: v_and_or_b32 v0, v1, s9, s10
+; GFX1064-NEXT: v_mov_b32_e32 v3, v1
+; GFX1064-NEXT: v_mov_b32_e32 v2, v0
+; GFX1064-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
+; GFX1064-NEXT: s_waitcnt vmcnt(0)
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX1064-NEXT: v_mov_b32_e32 v1, v2
+; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_cbranch_execnz .LBB14_1
+; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1064-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s8, v2
+; GFX1064-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1064-NEXT: s_mov_b32 s2, -1
+; GFX1064-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; GFX1064-NEXT: s_endpgm
+;
+; GFX1032-LABEL: uniform_xchg_i8:
+; GFX1032: ; %bb.0:
+; GFX1032-NEXT: s_clause 0x1
+; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX1032-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX1032-NEXT: s_mov_b32 s9, 0
+; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT: s_and_b32 s4, s2, -4
+; GFX1032-NEXT: s_mov_b32 s5, s3
+; GFX1032-NEXT: s_and_b32 s2, s2, 3
+; GFX1032-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX1032-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1032-NEXT: s_and_b32 s6, s6, 0xff
+; GFX1032-NEXT: s_lshl_b32 s3, 0xff, s2
+; GFX1032-NEXT: s_lshl_b32 s8, s6, s2
+; GFX1032-NEXT: s_not_b32 s3, s3
+; GFX1032-NEXT: s_mov_b32 s6, -1
+; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT: v_mov_b32_e32 v1, s7
+; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1032-NEXT: .LBB14_1: ; %atomicrmw.start
+; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: v_and_or_b32 v0, v1, s3, s8
+; GFX1032-NEXT: v_mov_b32_e32 v3, v1
+; GFX1032-NEXT: v_mov_b32_e32 v2, v0
+; GFX1032-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
+; GFX1032-NEXT: s_waitcnt vmcnt(0)
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX1032-NEXT: v_mov_b32_e32 v1, v2
+; GFX1032-NEXT: s_or_b32 s9, vcc_lo, s9
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s9
+; GFX1032-NEXT: s_cbranch_execnz .LBB14_1
+; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s9
+; GFX1032-NEXT: v_lshrrev_b32_e32 v0, s2, v2
+; GFX1032-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1032-NEXT: s_mov_b32 s2, -1
+; GFX1032-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; GFX1032-NEXT: s_endpgm
+;
+; GFX1164-LABEL: uniform_xchg_i8:
+; GFX1164: ; %bb.0:
+; GFX1164-NEXT: s_clause 0x1
+; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1164-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX1164-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-NEXT: s_and_b32 s4, s2, -4
+; GFX1164-NEXT: s_mov_b32 s5, s3
+; GFX1164-NEXT: s_and_b32 s2, s2, 3
+; GFX1164-NEXT: s_load_b32 s3, s[4:5], 0x0
+; GFX1164-NEXT: s_lshl_b32 s8, s2, 3
+; GFX1164-NEXT: s_and_b32 s6, s6, 0xff
+; GFX1164-NEXT: s_lshl_b32 s2, 0xff, s8
+; GFX1164-NEXT: s_lshl_b32 s10, s6, s8
+; GFX1164-NEXT: s_not_b32 s9, s2
+; GFX1164-NEXT: s_mov_b32 s6, -1
+; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-NEXT: v_mov_b32_e32 v1, s3
+; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: .LBB14_1: ; %atomicrmw.start
+; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_and_or_b32 v0, v1, s9, s10
+; GFX1164-NEXT: v_mov_b32_e32 v3, v1
+; GFX1164-NEXT: v_mov_b32_e32 v2, v0
+; GFX1164-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
+; GFX1164-NEXT: s_waitcnt vmcnt(0)
+; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX1164-NEXT: v_mov_b32_e32 v1, v2
+; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_cbranch_execnz .LBB14_1
+; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1164-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: v_lshrrev_b32_e32 v0, s8, v2
+; GFX1164-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1164-NEXT: s_mov_b32 s2, -1
+; GFX1164-NEXT: buffer_store_b8 v0, off, s[0:3], 0
+; GFX1164-NEXT: s_endpgm
+;
+; GFX1132-LABEL: uniform_xchg_i8:
+; GFX1132: ; %bb.0:
+; GFX1132-NEXT: s_clause 0x1
+; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1132-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX1132-NEXT: s_mov_b32 s9, 0
+; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-NEXT: s_and_b32 s4, s2, -4
+; GFX1132-NEXT: s_mov_b32 s5, s3
+; GFX1132-NEXT: s_and_b32 s2, s2, 3
+; GFX1132-NEXT: s_load_b32 s7, s[4:5], 0x0
+; GFX1132-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1132-NEXT: s_and_b32 s6, s6, 0xff
+; GFX1132-NEXT: s_lshl_b32 s3, 0xff, s2
+; GFX1132-NEXT: s_lshl_b32 s8, s6, s2
+; GFX1132-NEXT: s_not_b32 s3, s3
+; GFX1132-NEXT: s_mov_b32 s6, -1
+; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-NEXT: v_mov_b32_e32 v1, s7
+; GFX1132-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1132-NEXT: .LBB14_1: ; %atomicrmw.start
+; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_and_or_b32 v0, v1, s3, s8
+; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
+; GFX1132-NEXT: s_waitcnt vmcnt(0)
+; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX1132-NEXT: v_mov_b32_e32 v1, v2
+; GFX1132-NEXT: s_or_b32 s9, vcc_lo, s9
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s9
+; GFX1132-NEXT: s_cbranch_execnz .LBB14_1
+; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s9
+; GFX1132-NEXT: v_lshrrev_b32_e32 v0, s2, v2
+; GFX1132-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1132-NEXT: s_mov_b32 s2, -1
+; GFX1132-NEXT: buffer_store_b8 v0, off, s[0:3], 0
+; GFX1132-NEXT: s_endpgm
+;
+; GFX1264-LABEL: uniform_xchg_i8:
+; GFX1264: ; %bb.0:
+; GFX1264-NEXT: s_clause 0x1
+; GFX1264-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1264-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX1264-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1264-NEXT: s_wait_kmcnt 0x0
+; GFX1264-NEXT: s_and_b32 s4, s2, -4
+; GFX1264-NEXT: s_mov_b32 s5, s3
+; GFX1264-NEXT: s_and_b32 s2, s2, 3
+; GFX1264-NEXT: s_load_b32 s3, s[4:5], 0x0
+; GFX1264-NEXT: s_lshl_b32 s8, s2, 3
+; GFX1264-NEXT: s_and_b32 s6, s6, 0xff
+; GFX1264-NEXT: s_lshl_b32 s2, 0xff, s8
+; GFX1264-NEXT: s_lshl_b32 s10, s6, s8
+; GFX1264-NEXT: s_not_b32 s9, s2
+; GFX1264-NEXT: s_mov_b32 s6, -1
+; GFX1264-NEXT: s_wait_kmcnt 0x0
+; GFX1264-NEXT: v_mov_b32_e32 v1, s3
+; GFX1264-NEXT: s_mov_b64 s[2:3], 0
+; GFX1264-NEXT: .LBB14_1: ; %atomicrmw.start
+; GFX1264-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1264-NEXT: v_and_or_b32 v0, v1, s9, s10
+; GFX1264-NEXT: v_mov_b32_e32 v3, v1
+; GFX1264-NEXT: v_mov_b32_e32 v2, v0
+; GFX1264-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1264-NEXT: s_wait_loadcnt 0x0
+; GFX1264-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX1264-NEXT: v_mov_b32_e32 v1, v2
+; GFX1264-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1264-NEXT: s_wait_alu 0xfffe
+; GFX1264-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1264-NEXT: s_cbranch_execnz .LBB14_1
+; GFX1264-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1264-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX1264-NEXT: v_lshrrev_b32_e32 v0, s8, v2
+; GFX1264-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1264-NEXT: s_mov_b32 s2, -1
+; GFX1264-NEXT: buffer_store_b8 v0, off, s[0:3], null
+; GFX1264-NEXT: s_endpgm
+;
+; GFX1232-LABEL: uniform_xchg_i8:
+; GFX1232: ; %bb.0:
+; GFX1232-NEXT: s_clause 0x1
+; GFX1232-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1232-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX1232-NEXT: s_mov_b32 s9, 0
+; GFX1232-NEXT: s_wait_kmcnt 0x0
+; GFX1232-NEXT: s_and_b32 s4, s2, -4
+; GFX1232-NEXT: s_mov_b32 s5, s3
+; GFX1232-NEXT: s_and_b32 s2, s2, 3
+; GFX1232-NEXT: s_load_b32 s7, s[4:5], 0x0
+; GFX1232-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1232-NEXT: s_and_b32 s6, s6, 0xff
+; GFX1232-NEXT: s_lshl_b32 s3, 0xff, s2
+; GFX1232-NEXT: s_lshl_b32 s8, s6, s2
+; GFX1232-NEXT: s_not_b32 s3, s3
+; GFX1232-NEXT: s_mov_b32 s6, -1
+; GFX1232-NEXT: s_wait_kmcnt 0x0
+; GFX1232-NEXT: v_mov_b32_e32 v1, s7
+; GFX1232-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1232-NEXT: .LBB14_1: ; %atomicrmw.start
+; GFX1232-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1232-NEXT: v_and_or_b32 v0, v1, s3, s8
+; GFX1232-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1232-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1232-NEXT: s_wait_loadcnt 0x0
+; GFX1232-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX1232-NEXT: v_mov_b32_e32 v1, v2
+; GFX1232-NEXT: s_or_b32 s9, vcc_lo, s9
+; GFX1232-NEXT: s_wait_alu 0xfffe
+; GFX1232-NEXT: s_and_not1_b32 exec_lo, exec_lo, s9
+; GFX1232-NEXT: s_cbranch_execnz .LBB14_1
+; GFX1232-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1232-NEXT: s_or_b32 exec_lo, exec_lo, s9
+; GFX1232-NEXT: v_lshrrev_b32_e32 v0, s2, v2
+; GFX1232-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1232-NEXT: s_mov_b32 s2, -1
+; GFX1232-NEXT: buffer_store_b8 v0, off, s[0:3], null
+; GFX1232-NEXT: s_endpgm
+ %rmw = atomicrmw xchg ptr addrspace(1) %uniform.ptr, i8 %val monotonic, align 1
+ store i8 %rmw, ptr addrspace(1) %result
+ ret void
+}
+
+define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspace(1) %uniform.ptr, i16 %val) {
+; GFX7LESS-LABEL: uniform_or_i16:
+; GFX7LESS: ; %bb.0:
+; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7LESS-NEXT: s_load_dword s6, s[4:5], 0xd
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, 0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX7LESS-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX7LESS-NEXT: s_cbranch_execz .LBB15_2
+; GFX7LESS-NEXT: ; %bb.1:
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: s_and_b32 s8, s2, -4
+; GFX7LESS-NEXT: s_mov_b32 s11, 0xf000
+; GFX7LESS-NEXT: s_and_b32 s2, s2, 3
+; GFX7LESS-NEXT: s_lshl_b32 s2, s2, 3
+; GFX7LESS-NEXT: s_and_b32 s7, s6, 0xffff
+; GFX7LESS-NEXT: s_lshl_b32 s7, s7, s2
+; GFX7LESS-NEXT: s_mov_b32 s10, -1
+; GFX7LESS-NEXT: s_mov_b32 s9, s3
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s7
+; GFX7LESS-NEXT: buffer_atomic_or v0, off, s[8:11], 0 glc
+; GFX7LESS-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GFX7LESS-NEXT: v_bfe_u32 v0, v0, s2, 16
+; GFX7LESS-NEXT: .LBB15_2:
+; GFX7LESS-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s6
+; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v0
+; GFX7LESS-NEXT: v_cndmask_b32_e64 v0, v1, 0, vcc
+; GFX7LESS-NEXT: v_or_b32_e32 v0, s4, v0
+; GFX7LESS-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX7LESS-NEXT: s_endpgm
+;
+; GFX8-LABEL: uniform_or_i16:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX8-NEXT: ; implicit-def: $vgpr0
+; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX8-NEXT: s_cbranch_execz .LBB15_2
+; GFX8-NEXT: ; %bb.1:
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: s_and_b32 s8, s2, -4
+; GFX8-NEXT: s_and_b32 s2, s2, 3
+; GFX8-NEXT: s_mov_b32 s9, s3
+; GFX8-NEXT: s_lshl_b32 s2, s2, 3
+; GFX8-NEXT: s_and_b32 s3, 0xffff, s6
+; GFX8-NEXT: s_lshl_b32 s3, s3, s2
+; GFX8-NEXT: s_mov_b32 s11, 0xf000
+; GFX8-NEXT: s_mov_b32 s10, -1
+; GFX8-NEXT: v_mov_b32_e32 v0, s3
+; GFX8-NEXT: buffer_atomic_or v0, off, s[8:11], 0 glc
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, s2, v0
+; GFX8-NEXT: .LBB15_2:
+; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX8-NEXT: v_readfirstlane_b32 s4, v0
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v0, s6
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc
+; GFX8-NEXT: s_mov_b32 s3, 0xf000
+; GFX8-NEXT: s_mov_b32 s2, -1
+; GFX8-NEXT: v_or_b32_e32 v0, s4, v0
+; GFX8-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX8-NEXT: s_endpgm
+;
+; GFX9-LABEL: uniform_or_i16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: ; implicit-def: $vgpr0
+; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT: s_cbranch_execz .LBB15_2
+; GFX9-NEXT: ; %bb.1:
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_and_b32 s8, s2, -4
+; GFX9-NEXT: s_and_b32 s2, s2, 3
+; GFX9-NEXT: s_mov_b32 s9, s3
+; GFX9-NEXT: s_lshl_b32 s2, s2, 3
+; GFX9-NEXT: s_and_b32 s3, 0xffff, s6
+; GFX9-NEXT: s_lshl_b32 s3, s3, s2
+; GFX9-NEXT: s_mov_b32 s11, 0xf000
+; GFX9-NEXT: s_mov_b32 s10, -1
+; GFX9-NEXT: v_mov_b32_e32 v0, s3
+; GFX9-NEXT: buffer_atomic_or v0, off, s[8:11], 0 glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, s2, v0
+; GFX9-NEXT: .LBB15_2:
+; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_readfirstlane_b32 s4, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v0, s6
+; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc
+; GFX9-NEXT: s_mov_b32 s3, 0xf000
+; GFX9-NEXT: s_mov_b32 s2, -1
+; GFX9-NEXT: v_or_b32_e32 v0, s4, v0
+; GFX9-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX9-NEXT: s_endpgm
+;
+; GFX1064-LABEL: uniform_or_i16:
+; GFX1064: ; %bb.0:
+; GFX1064-NEXT: s_clause 0x1
+; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX1064-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: ; implicit-def: $vgpr0
+; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX1064-NEXT: s_cbranch_execz .LBB15_2
+; GFX1064-NEXT: ; %bb.1:
+; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT: s_and_b32 s7, s2, 3
+; GFX1064-NEXT: s_and_b32 s8, 0xffff, s6
+; GFX1064-NEXT: s_lshl_b32 s7, s7, 3
+; GFX1064-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1064-NEXT: s_lshl_b32 s9, s8, s7
+; GFX1064-NEXT: s_and_b32 s8, s2, -4
+; GFX1064-NEXT: v_mov_b32_e32 v0, s9
+; GFX1064-NEXT: s_mov_b32 s10, -1
+; GFX1064-NEXT: s_mov_b32 s9, s3
+; GFX1064-NEXT: buffer_atomic_or v0, off, s[8:11], 0 glc
+; GFX1064-NEXT: s_waitcnt vmcnt(0)
+; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s7, v0
+; GFX1064-NEXT: .LBB15_2:
+; GFX1064-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX1064-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1064-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1064-NEXT: v_cndmask_b32_e64 v0, s6, 0, vcc
+; GFX1064-NEXT: v_or_b32_e32 v0, s2, v0
+; GFX1064-NEXT: s_mov_b32 s2, -1
+; GFX1064-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX1064-NEXT: s_endpgm
+;
+; GFX1032-LABEL: uniform_or_i16:
+; GFX1032: ; %bb.0:
+; GFX1032-NEXT: s_clause 0x1
+; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX1032-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT: ; implicit-def: $vgpr0
+; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo
+; GFX1032-NEXT: s_cbranch_execz .LBB15_2
+; GFX1032-NEXT: ; %bb.1:
+; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT: s_and_b32 s5, s2, 3
+; GFX1032-NEXT: s_and_b32 s7, 0xffff, s6
+; GFX1032-NEXT: s_lshl_b32 s5, s5, 3
+; GFX1032-NEXT: s_and_b32 s8, s2, -4
+; GFX1032-NEXT: s_lshl_b32 s7, s7, s5
+; GFX1032-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1032-NEXT: v_mov_b32_e32 v0, s7
+; GFX1032-NEXT: s_mov_b32 s10, -1
+; GFX1032-NEXT: s_mov_b32 s9, s3
+; GFX1032-NEXT: buffer_atomic_or v0, off, s[8:11], 0 glc
+; GFX1032-NEXT: s_waitcnt vmcnt(0)
+; GFX1032-NEXT: v_lshrrev_b32_e32 v0, s5, v0
+; GFX1032-NEXT: .LBB15_2:
+; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX1032-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1032-NEXT: v_cndmask_b32_e64 v0, s6, 0, vcc_lo
+; GFX1032-NEXT: v_or_b32_e32 v0, s2, v0
+; GFX1032-NEXT: s_mov_b32 s2, -1
+; GFX1032-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX1032-NEXT: s_endpgm
+;
+; GFX1164-LABEL: uniform_or_i16:
+; GFX1164: ; %bb.0:
+; GFX1164-NEXT: s_clause 0x1
+; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1164-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1164-NEXT: ; implicit-def: $vgpr0
+; GFX1164-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX1164-NEXT: s_cbranch_execz .LBB15_2
+; GFX1164-NEXT: ; %bb.1:
+; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-NEXT: s_and_b32 s7, s2, 3
+; GFX1164-NEXT: s_and_b32 s8, 0xffff, s6
+; GFX1164-NEXT: s_lshl_b32 s7, s7, 3
+; GFX1164-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1164-NEXT: s_lshl_b32 s9, s8, s7
+; GFX1164-NEXT: s_and_b32 s8, s2, -4
+; GFX1164-NEXT: v_mov_b32_e32 v0, s9
+; GFX1164-NEXT: s_mov_b32 s10, -1
+; GFX1164-NEXT: s_mov_b32 s9, s3
+; GFX1164-NEXT: buffer_atomic_or_b32 v0, off, s[8:11], 0 glc
+; GFX1164-NEXT: s_waitcnt vmcnt(0)
+; GFX1164-NEXT: v_lshrrev_b32_e32 v0, s7, v0
+; GFX1164-NEXT: .LBB15_2:
+; GFX1164-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1164-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1164-NEXT: v_cndmask_b32_e64 v0, s6, 0, vcc
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_or_b32_e32 v0, s2, v0
+; GFX1164-NEXT: s_mov_b32 s2, -1
+; GFX1164-NEXT: buffer_store_b16 v0, off, s[0:3], 0
+; GFX1164-NEXT: s_endpgm
+;
+; GFX1132-LABEL: uniform_or_i16:
+; GFX1132: ; %bb.0:
+; GFX1132-NEXT: s_clause 0x1
+; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1132-NEXT: s_load_b32 s4, s[4:5], 0x34
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1132-NEXT: ; implicit-def: $vgpr0
+; GFX1132-NEXT: s_and_saveexec_b32 s5, vcc_lo
+; GFX1132-NEXT: s_cbranch_execz .LBB15_2
+; GFX1132-NEXT: ; %bb.1:
+; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-NEXT: s_and_b32 s6, s2, 3
+; GFX1132-NEXT: s_and_b32 s7, 0xffff, s4
+; GFX1132-NEXT: s_lshl_b32 s6, s6, 3
+; GFX1132-NEXT: s_and_b32 s8, s2, -4
+; GFX1132-NEXT: s_lshl_b32 s7, s7, s6
+; GFX1132-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1132-NEXT: v_mov_b32_e32 v0, s7
+; GFX1132-NEXT: s_mov_b32 s10, -1
+; GFX1132-NEXT: s_mov_b32 s9, s3
+; GFX1132-NEXT: buffer_atomic_or_b32 v0, off, s[8:11], 0 glc
+; GFX1132-NEXT: s_waitcnt vmcnt(0)
+; GFX1132-NEXT: v_lshrrev_b32_e32 v0, s6, v0
+; GFX1132-NEXT: .LBB15_2:
+; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1132-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1132-NEXT: v_cndmask_b32_e64 v0, s4, 0, vcc_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: v_or_b32_e32 v0, s2, v0
+; GFX1132-NEXT: s_mov_b32 s2, -1
+; GFX1132-NEXT: buffer_store_b16 v0, off, s[0:3], 0
+; GFX1132-NEXT: s_endpgm
+;
+; GFX1264-LABEL: uniform_or_i16:
+; GFX1264: ; %bb.0:
+; GFX1264-NEXT: s_clause 0x1
+; GFX1264-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1264-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1264-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1264-NEXT: ; implicit-def: $vgpr0
+; GFX1264-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX1264-NEXT: s_cbranch_execz .LBB15_2
+; GFX1264-NEXT: ; %bb.1:
+; GFX1264-NEXT: s_wait_kmcnt 0x0
+; GFX1264-NEXT: s_and_b32 s7, s2, 3
+; GFX1264-NEXT: s_and_b32 s8, 0xffff, s6
+; GFX1264-NEXT: s_lshl_b32 s7, s7, 3
+; GFX1264-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1264-NEXT: s_lshl_b32 s9, s8, s7
+; GFX1264-NEXT: s_and_b32 s8, s2, -4
+; GFX1264-NEXT: v_mov_b32_e32 v0, s9
+; GFX1264-NEXT: s_mov_b32 s10, -1
+; GFX1264-NEXT: s_mov_b32 s9, s3
+; GFX1264-NEXT: buffer_atomic_or_b32 v0, off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1264-NEXT: s_wait_loadcnt 0x0
+; GFX1264-NEXT: v_lshrrev_b32_e32 v0, s7, v0
+; GFX1264-NEXT: .LBB15_2:
+; GFX1264-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1264-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX1264-NEXT: s_wait_kmcnt 0x0
+; GFX1264-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1264-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1264-NEXT: v_cndmask_b32_e64 v0, s6, 0, vcc
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1264-NEXT: v_or_b32_e32 v0, s2, v0
+; GFX1264-NEXT: s_mov_b32 s2, -1
+; GFX1264-NEXT: buffer_store_b16 v0, off, s[0:3], null
+; GFX1264-NEXT: s_endpgm
+;
+; GFX1232-LABEL: uniform_or_i16:
+; GFX1232: ; %bb.0:
+; GFX1232-NEXT: s_clause 0x1
+; GFX1232-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1232-NEXT: s_load_b32 s4, s[4:5], 0x34
+; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1232-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1232-NEXT: ; implicit-def: $vgpr0
+; GFX1232-NEXT: s_and_saveexec_b32 s5, vcc_lo
+; GFX1232-NEXT: s_cbranch_execz .LBB15_2
+; GFX1232-NEXT: ; %bb.1:
+; GFX1232-NEXT: s_wait_kmcnt 0x0
+; GFX1232-NEXT: s_and_b32 s6, s2, 3
+; GFX1232-NEXT: s_and_b32 s7, 0xffff, s4
+; GFX1232-NEXT: s_lshl_b32 s6, s6, 3
+; GFX1232-NEXT: s_and_b32 s8, s2, -4
+; GFX1232-NEXT: s_lshl_b32 s7, s7, s6
+; GFX1232-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1232-NEXT: v_mov_b32_e32 v0, s7
+; GFX1232-NEXT: s_mov_b32 s10, -1
+; GFX1232-NEXT: s_mov_b32 s9, s3
+; GFX1232-NEXT: buffer_atomic_or_b32 v0, off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1232-NEXT: s_wait_loadcnt 0x0
+; GFX1232-NEXT: v_lshrrev_b32_e32 v0, s6, v0
+; GFX1232-NEXT: .LBB15_2:
+; GFX1232-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1232-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX1232-NEXT: s_wait_kmcnt 0x0
+; GFX1232-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1232-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1232-NEXT: v_cndmask_b32_e64 v0, s4, 0, vcc_lo
+; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1232-NEXT: v_or_b32_e32 v0, s2, v0
+; GFX1232-NEXT: s_mov_b32 s2, -1
+; GFX1232-NEXT: buffer_store_b16 v0, off, s[0:3], null
+; GFX1232-NEXT: s_endpgm
+ %rmw = atomicrmw or ptr addrspace(1) %uniform.ptr, i16 %val monotonic, align 2
+ store i16 %rmw, ptr addrspace(1) %result
+ ret void
+}
+
+define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspace(1) %uniform.ptr, i16 %val) {
+; GFX7LESS-LABEL: uniform_add_i16:
+; GFX7LESS: ; %bb.0:
+; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec
+; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7LESS-NEXT: s_load_dword s10, s[4:5], 0xd
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, 0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, s6, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v4, s7, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
+; GFX7LESS-NEXT: s_and_saveexec_b64 s[8:9], vcc
+; GFX7LESS-NEXT: s_cbranch_execz .LBB16_4
+; GFX7LESS-NEXT: ; %bb.1:
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: s_and_b32 s4, s2, -4
+; GFX7LESS-NEXT: s_mov_b32 s5, s3
+; GFX7LESS-NEXT: s_and_b32 s11, s2, 3
+; GFX7LESS-NEXT: s_mov_b64 s[2:3], 0
+; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT: s_mul_i32 s6, s10, s6
+; GFX7LESS-NEXT: s_lshl_b32 s11, s11, 3
+; GFX7LESS-NEXT: s_load_dword s15, s[4:5], 0x0
+; GFX7LESS-NEXT: s_lshl_b32 s12, 0xffff, s11
+; GFX7LESS-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX7LESS-NEXT: s_not_b32 s13, s12
+; GFX7LESS-NEXT: s_lshl_b32 s14, s6, s11
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s15
+; GFX7LESS-NEXT: s_mov_b32 s6, -1
+; GFX7LESS-NEXT: .LBB16_2: ; %atomicrmw.start
+; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: v_add_i32_e32 v0, vcc, s14, v1
+; GFX7LESS-NEXT: v_and_b32_e32 v0, s12, v0
+; GFX7LESS-NEXT: s_waitcnt expcnt(0)
+; GFX7LESS-NEXT: v_and_b32_e32 v2, s13, v1
+; GFX7LESS-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
+; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX7LESS-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX7LESS-NEXT: s_cbranch_execnz .LBB16_2
+; GFX7LESS-NEXT: ; %bb.3: ; %atomicrmw.end
+; GFX7LESS-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX7LESS-NEXT: v_bfe_u32 v0, v2, s11, 16
+; GFX7LESS-NEXT: .LBB16_4: ; %Flow
+; GFX7LESS-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v0
+; GFX7LESS-NEXT: s_and_b32 s5, s10, 0xffff
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s4
+; GFX7LESS-NEXT: v_mad_u32_u24 v0, s5, v4, v0
+; GFX7LESS-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX7LESS-NEXT: s_endpgm
+;
+; GFX8-LABEL: uniform_add_i16:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT: s_load_dword s10, s[4:5], 0x34
+; GFX8-NEXT: s_mov_b64 s[6:7], exec
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
+; GFX8-NEXT: ; implicit-def: $vgpr0
+; GFX8-NEXT: s_and_saveexec_b64 s[8:9], vcc
+; GFX8-NEXT: s_cbranch_execz .LBB16_4
+; GFX8-NEXT: ; %bb.1:
+; GFX8-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: s_mul_i32 s6, s10, s4
+; GFX8-NEXT: s_and_b32 s4, s2, -4
+; GFX8-NEXT: s_mov_b32 s5, s3
+; GFX8-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX8-NEXT: s_and_b32 s2, s2, 3
+; GFX8-NEXT: s_lshl_b32 s11, s2, 3
+; GFX8-NEXT: s_lshl_b32 s12, 0xffff, s11
+; GFX8-NEXT: s_and_b32 s2, s6, 0xffff
+; GFX8-NEXT: s_not_b32 s13, s12
+; GFX8-NEXT: s_lshl_b32 s14, s2, s11
+; GFX8-NEXT: s_mov_b64 s[2:3], 0
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-NEXT: s_mov_b32 s6, -1
+; GFX8-NEXT: .LBB16_2: ; %atomicrmw.start
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s14, v1
+; GFX8-NEXT: v_and_b32_e32 v2, s13, v1
+; GFX8-NEXT: v_and_b32_e32 v0, s12, v0
+; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX8-NEXT: v_mov_b32_e32 v3, v1
+; GFX8-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX8-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX8-NEXT: v_mov_b32_e32 v1, v2
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX8-NEXT: s_cbranch_execnz .LBB16_2
+; GFX8-NEXT: ; %bb.3: ; %atomicrmw.end
+; GFX8-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, s11, v2
+; GFX8-NEXT: .LBB16_4: ; %Flow
+; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX8-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX8-NEXT: v_readfirstlane_b32 s4, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s4
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: s_mov_b32 s3, 0xf000
+; GFX8-NEXT: s_mov_b32 s2, -1
+; GFX8-NEXT: v_mad_u16 v0, s10, v4, v0
+; GFX8-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX8-NEXT: s_endpgm
+;
+; GFX9-LABEL: uniform_add_i16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_load_dword s10, s[4:5], 0x34
+; GFX9-NEXT: s_mov_b64 s[6:7], exec
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
+; GFX9-NEXT: ; implicit-def: $vgpr0
+; GFX9-NEXT: s_and_saveexec_b64 s[8:9], vcc
+; GFX9-NEXT: s_cbranch_execz .LBB16_4
+; GFX9-NEXT: ; %bb.1:
+; GFX9-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_mul_i32 s6, s10, s4
+; GFX9-NEXT: s_and_b32 s4, s2, -4
+; GFX9-NEXT: s_mov_b32 s5, s3
+; GFX9-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX9-NEXT: s_and_b32 s2, s2, 3
+; GFX9-NEXT: s_lshl_b32 s11, s2, 3
+; GFX9-NEXT: s_lshl_b32 s12, 0xffff, s11
+; GFX9-NEXT: s_and_b32 s2, s6, 0xffff
+; GFX9-NEXT: s_not_b32 s13, s12
+; GFX9-NEXT: s_lshl_b32 s14, s2, s11
+; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v1, s7
+; GFX9-NEXT: s_mov_b32 s7, 0xf000
+; GFX9-NEXT: s_mov_b32 s6, -1
+; GFX9-NEXT: .LBB16_2: ; %atomicrmw.start
+; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: v_add_u32_e32 v0, s14, v1
+; GFX9-NEXT: v_and_b32_e32 v0, s12, v0
+; GFX9-NEXT: v_and_or_b32 v0, v1, s13, v0
+; GFX9-NEXT: v_mov_b32_e32 v3, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, v0
+; GFX9-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: v_mov_b32_e32 v1, v2
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_cbranch_execnz .LBB16_2
+; GFX9-NEXT: ; %bb.3: ; %atomicrmw.end
+; GFX9-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, s11, v2
+; GFX9-NEXT: .LBB16_4: ; %Flow
+; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_readfirstlane_b32 s4, v0
+; GFX9-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 s3, 0xf000
+; GFX9-NEXT: s_mov_b32 s2, -1
+; GFX9-NEXT: v_mad_legacy_u16 v0, s10, v4, v0
+; GFX9-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX9-NEXT: s_endpgm
+;
+; GFX1064-LABEL: uniform_add_i16:
+; GFX1064: ; %bb.0:
+; GFX1064-NEXT: s_clause 0x1
+; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX1064-NEXT: s_load_dword s10, s[4:5], 0x34
+; GFX1064-NEXT: s_mov_b64 s[6:7], exec
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX1064-NEXT: ; implicit-def: $vgpr0
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
+; GFX1064-NEXT: s_and_saveexec_b64 s[8:9], vcc
+; GFX1064-NEXT: s_cbranch_execz .LBB16_4
+; GFX1064-NEXT: ; %bb.1:
+; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT: s_and_b32 s4, s2, -4
+; GFX1064-NEXT: s_mov_b32 s5, s3
+; GFX1064-NEXT: s_and_b32 s2, s2, 3
+; GFX1064-NEXT: s_load_dword s3, s[4:5], 0x0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1064-NEXT: s_lshl_b32 s11, s2, 3
+; GFX1064-NEXT: s_mul_i32 s2, s10, s6
+; GFX1064-NEXT: s_lshl_b32 s12, 0xffff, s11
+; GFX1064-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX1064-NEXT: s_not_b32 s13, s12
+; GFX1064-NEXT: s_lshl_b32 s14, s2, s11
+; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1064-NEXT: s_mov_b32 s6, -1
+; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT: v_mov_b32_e32 v1, s3
+; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: .LBB16_2: ; %atomicrmw.start
+; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: v_add_nc_u32_e32 v0, s14, v1
+; GFX1064-NEXT: v_and_b32_e32 v0, s12, v0
+; GFX1064-NEXT: v_and_or_b32 v0, v1, s13, v0
+; GFX1064-NEXT: v_mov_b32_e32 v3, v1
+; GFX1064-NEXT: v_mov_b32_e32 v2, v0
+; GFX1064-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
+; GFX1064-NEXT: s_waitcnt vmcnt(0)
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX1064-NEXT: v_mov_b32_e32 v1, v2
+; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_cbranch_execnz .LBB16_2
+; GFX1064-NEXT: ; %bb.3: ; %atomicrmw.end
+; GFX1064-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s11, v2
+; GFX1064-NEXT: .LBB16_4: ; %Flow
+; GFX1064-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX1064-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1064-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1064-NEXT: v_mad_u16 v0, s10, v4, s2
+; GFX1064-NEXT: s_mov_b32 s2, -1
+; GFX1064-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX1064-NEXT: s_endpgm
+;
+; GFX1032-LABEL: uniform_add_i16:
+; GFX1032: ; %bb.0:
+; GFX1032-NEXT: s_clause 0x1
+; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX1032-NEXT: s_load_dword s8, s[4:5], 0x34
+; GFX1032-NEXT: s_mov_b32 s6, exec_lo
+; GFX1032-NEXT: s_mov_b32 s10, 0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
+; GFX1032-NEXT: ; implicit-def: $vgpr0
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v4
+; GFX1032-NEXT: s_and_saveexec_b32 s9, vcc_lo
+; GFX1032-NEXT: s_cbranch_execz .LBB16_4
+; GFX1032-NEXT: ; %bb.1:
+; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT: s_and_b32 s4, s2, -4
+; GFX1032-NEXT: s_mov_b32 s5, s3
+; GFX1032-NEXT: s_and_b32 s2, s2, 3
+; GFX1032-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s6
+; GFX1032-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1032-NEXT: s_mul_i32 s6, s8, s6
+; GFX1032-NEXT: s_lshl_b32 s3, 0xffff, s2
+; GFX1032-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX1032-NEXT: s_not_b32 s11, s3
+; GFX1032-NEXT: s_lshl_b32 s12, s6, s2
+; GFX1032-NEXT: s_mov_b32 s6, -1
+; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT: v_mov_b32_e32 v1, s7
+; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1032-NEXT: .LBB16_2: ; %atomicrmw.start
+; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: v_add_nc_u32_e32 v0, s12, v1
+; GFX1032-NEXT: v_and_b32_e32 v0, s3, v0
+; GFX1032-NEXT: v_and_or_b32 v0, v1, s11, v0
+; GFX1032-NEXT: v_mov_b32_e32 v3, v1
+; GFX1032-NEXT: v_mov_b32_e32 v2, v0
+; GFX1032-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
+; GFX1032-NEXT: s_waitcnt vmcnt(0)
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX1032-NEXT: v_mov_b32_e32 v1, v2
+; GFX1032-NEXT: s_or_b32 s10, vcc_lo, s10
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s10
+; GFX1032-NEXT: s_cbranch_execnz .LBB16_2
+; GFX1032-NEXT: ; %bb.3: ; %atomicrmw.end
+; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s10
+; GFX1032-NEXT: v_lshrrev_b32_e32 v0, s2, v2
+; GFX1032-NEXT: .LBB16_4: ; %Flow
+; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s9
+; GFX1032-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1032-NEXT: v_mad_u16 v0, s8, v4, s2
+; GFX1032-NEXT: s_mov_b32 s2, -1
+; GFX1032-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX1032-NEXT: s_endpgm
+;
+; GFX1164-LABEL: uniform_add_i16:
+; GFX1164: ; %bb.0:
+; GFX1164-NEXT: s_clause 0x1
+; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1164-NEXT: s_load_b32 s10, s[4:5], 0x34
+; GFX1164-NEXT: s_mov_b64 s[6:7], exec
+; GFX1164-NEXT: s_mov_b64 s[8:9], exec
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX1164-NEXT: ; implicit-def: $vgpr0
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1164-NEXT: s_cbranch_execz .LBB16_4
+; GFX1164-NEXT: ; %bb.1:
+; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-NEXT: s_and_b32 s4, s2, -4
+; GFX1164-NEXT: s_mov_b32 s5, s3
+; GFX1164-NEXT: s_and_b32 s2, s2, 3
+; GFX1164-NEXT: s_load_b32 s3, s[4:5], 0x0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1164-NEXT: s_lshl_b32 s11, s2, 3
+; GFX1164-NEXT: s_mul_i32 s2, s10, s6
+; GFX1164-NEXT: s_lshl_b32 s12, 0xffff, s11
+; GFX1164-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX1164-NEXT: s_not_b32 s13, s12
+; GFX1164-NEXT: s_lshl_b32 s14, s2, s11
+; GFX1164-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1164-NEXT: s_mov_b32 s6, -1
+; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-NEXT: v_mov_b32_e32 v1, s3
+; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: .LBB16_2: ; %atomicrmw.start
+; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: v_add_nc_u32_e32 v0, s14, v1
+; GFX1164-NEXT: v_and_b32_e32 v0, s12, v0
+; GFX1164-NEXT: s_waitcnt_depctr 0xfff
+; GFX1164-NEXT: v_and_or_b32 v0, v1, s13, v0
+; GFX1164-NEXT: v_mov_b32_e32 v3, v1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1164-NEXT: v_mov_b32_e32 v2, v0
+; GFX1164-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
+; GFX1164-NEXT: s_waitcnt vmcnt(0)
+; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX1164-NEXT: v_mov_b32_e32 v1, v2
+; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_cbranch_execnz .LBB16_2
+; GFX1164-NEXT: ; %bb.3: ; %atomicrmw.end
+; GFX1164-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: v_lshrrev_b32_e32 v0, s11, v2
+; GFX1164-NEXT: .LBB16_4: ; %Flow
+; GFX1164-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1164-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mad_u16 v0, s10, v4, s2
+; GFX1164-NEXT: s_mov_b32 s2, -1
+; GFX1164-NEXT: buffer_store_b16 v0, off, s[0:3], 0
+; GFX1164-NEXT: s_endpgm
+;
+; GFX1132-LABEL: uniform_add_i16:
+; GFX1132: ; %bb.0:
+; GFX1132-NEXT: s_clause 0x1
+; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1132-NEXT: s_load_b32 s8, s[4:5], 0x34
+; GFX1132-NEXT: s_mov_b32 s6, exec_lo
+; GFX1132-NEXT: s_mov_b32 s10, 0
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
+; GFX1132-NEXT: s_mov_b32 s9, exec_lo
+; GFX1132-NEXT: ; implicit-def: $vgpr0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1132-NEXT: s_cbranch_execz .LBB16_4
+; GFX1132-NEXT: ; %bb.1:
+; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-NEXT: s_and_b32 s4, s2, -4
+; GFX1132-NEXT: s_mov_b32 s5, s3
+; GFX1132-NEXT: s_and_b32 s2, s2, 3
+; GFX1132-NEXT: s_load_b32 s7, s[4:5], 0x0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s6, s6
+; GFX1132-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1132-NEXT: s_mul_i32 s6, s8, s6
+; GFX1132-NEXT: s_lshl_b32 s3, 0xffff, s2
+; GFX1132-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX1132-NEXT: s_not_b32 s11, s3
+; GFX1132-NEXT: s_lshl_b32 s12, s6, s2
+; GFX1132-NEXT: s_mov_b32 s6, -1
+; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-NEXT: v_mov_b32_e32 v1, s7
+; GFX1132-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1132-NEXT: .LBB16_2: ; %atomicrmw.start
+; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_add_nc_u32_e32 v0, s12, v1
+; GFX1132-NEXT: v_and_b32_e32 v0, s3, v0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_and_or_b32 v0, v1, s11, v0
+; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
+; GFX1132-NEXT: s_waitcnt vmcnt(0)
+; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX1132-NEXT: v_mov_b32_e32 v1, v2
+; GFX1132-NEXT: s_or_b32 s10, vcc_lo, s10
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s10
+; GFX1132-NEXT: s_cbranch_execnz .LBB16_2
+; GFX1132-NEXT: ; %bb.3: ; %atomicrmw.end
+; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s10
+; GFX1132-NEXT: v_lshrrev_b32_e32 v0, s2, v2
+; GFX1132-NEXT: .LBB16_4: ; %Flow
+; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s9
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1132-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: v_mad_u16 v0, s8, v4, s2
+; GFX1132-NEXT: s_mov_b32 s2, -1
+; GFX1132-NEXT: buffer_store_b16 v0, off, s[0:3], 0
+; GFX1132-NEXT: s_endpgm
+;
+; GFX1264-LABEL: uniform_add_i16:
+; GFX1264: ; %bb.0:
+; GFX1264-NEXT: s_clause 0x1
+; GFX1264-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1264-NEXT: s_load_b32 s10, s[4:5], 0x34
+; GFX1264-NEXT: s_mov_b64 s[6:7], exec
+; GFX1264-NEXT: s_mov_b64 s[8:9], exec
+; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX1264-NEXT: ; implicit-def: $vgpr0
+; GFX1264-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1264-NEXT: s_cbranch_execz .LBB16_4
+; GFX1264-NEXT: ; %bb.1:
+; GFX1264-NEXT: s_wait_kmcnt 0x0
+; GFX1264-NEXT: s_and_b32 s4, s2, -4
+; GFX1264-NEXT: s_mov_b32 s5, s3
+; GFX1264-NEXT: s_and_b32 s2, s2, 3
+; GFX1264-NEXT: s_load_b32 s3, s[4:5], 0x0
+; GFX1264-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1264-NEXT: s_lshl_b32 s11, s2, 3
+; GFX1264-NEXT: s_wait_alu 0xfffe
+; GFX1264-NEXT: s_mul_i32 s2, s10, s6
+; GFX1264-NEXT: s_lshl_b32 s12, 0xffff, s11
+; GFX1264-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX1264-NEXT: s_not_b32 s13, s12
+; GFX1264-NEXT: s_lshl_b32 s14, s2, s11
+; GFX1264-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1264-NEXT: s_mov_b32 s6, -1
+; GFX1264-NEXT: s_wait_kmcnt 0x0
+; GFX1264-NEXT: v_mov_b32_e32 v1, s3
+; GFX1264-NEXT: s_mov_b64 s[2:3], 0
+; GFX1264-NEXT: .LBB16_2: ; %atomicrmw.start
+; GFX1264-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1264-NEXT: v_add_nc_u32_e32 v0, s14, v1
+; GFX1264-NEXT: v_and_b32_e32 v0, s12, v0
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1264-NEXT: v_and_or_b32 v0, v1, s13, v0
+; GFX1264-NEXT: v_mov_b32_e32 v3, v1
+; GFX1264-NEXT: v_mov_b32_e32 v2, v0
+; GFX1264-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1264-NEXT: s_wait_loadcnt 0x0
+; GFX1264-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX1264-NEXT: v_mov_b32_e32 v1, v2
+; GFX1264-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1264-NEXT: s_wait_alu 0xfffe
+; GFX1264-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1264-NEXT: s_cbranch_execnz .LBB16_2
+; GFX1264-NEXT: ; %bb.3: ; %atomicrmw.end
+; GFX1264-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX1264-NEXT: v_lshrrev_b32_e32 v0, s11, v2
+; GFX1264-NEXT: .LBB16_4: ; %Flow
+; GFX1264-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1264-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX1264-NEXT: s_wait_kmcnt 0x0
+; GFX1264-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1264-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1264-NEXT: s_wait_alu 0xf1ff
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1264-NEXT: v_mad_u16 v0, s10, v4, s2
+; GFX1264-NEXT: s_mov_b32 s2, -1
+; GFX1264-NEXT: buffer_store_b16 v0, off, s[0:3], null
+; GFX1264-NEXT: s_endpgm
+;
+; GFX1232-LABEL: uniform_add_i16:
+; GFX1232: ; %bb.0:
+; GFX1232-NEXT: s_clause 0x1
+; GFX1232-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1232-NEXT: s_load_b32 s8, s[4:5], 0x34
+; GFX1232-NEXT: s_mov_b32 s6, exec_lo
+; GFX1232-NEXT: s_mov_b32 s10, 0
+; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
+; GFX1232-NEXT: s_mov_b32 s9, exec_lo
+; GFX1232-NEXT: ; implicit-def: $vgpr0
+; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1232-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1232-NEXT: s_cbranch_execz .LBB16_4
+; GFX1232-NEXT: ; %bb.1:
+; GFX1232-NEXT: s_wait_kmcnt 0x0
+; GFX1232-NEXT: s_and_b32 s4, s2, -4
+; GFX1232-NEXT: s_mov_b32 s5, s3
+; GFX1232-NEXT: s_and_b32 s2, s2, 3
+; GFX1232-NEXT: s_load_b32 s7, s[4:5], 0x0
+; GFX1232-NEXT: s_bcnt1_i32_b32 s6, s6
+; GFX1232-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1232-NEXT: s_wait_alu 0xfffe
+; GFX1232-NEXT: s_mul_i32 s6, s8, s6
+; GFX1232-NEXT: s_lshl_b32 s3, 0xffff, s2
+; GFX1232-NEXT: s_wait_alu 0xfffe
+; GFX1232-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX1232-NEXT: s_not_b32 s11, s3
+; GFX1232-NEXT: s_wait_alu 0xfffe
+; GFX1232-NEXT: s_lshl_b32 s12, s6, s2
+; GFX1232-NEXT: s_mov_b32 s6, -1
+; GFX1232-NEXT: s_wait_kmcnt 0x0
+; GFX1232-NEXT: v_mov_b32_e32 v1, s7
+; GFX1232-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1232-NEXT: .LBB16_2: ; %atomicrmw.start
+; GFX1232-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1232-NEXT: v_add_nc_u32_e32 v0, s12, v1
+; GFX1232-NEXT: v_and_b32_e32 v0, s3, v0
+; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1232-NEXT: v_and_or_b32 v0, v1, s11, v0
+; GFX1232-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1232-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1232-NEXT: s_wait_loadcnt 0x0
+; GFX1232-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX1232-NEXT: v_mov_b32_e32 v1, v2
+; GFX1232-NEXT: s_or_b32 s10, vcc_lo, s10
+; GFX1232-NEXT: s_wait_alu 0xfffe
+; GFX1232-NEXT: s_and_not1_b32 exec_lo, exec_lo, s10
+; GFX1232-NEXT: s_cbranch_execnz .LBB16_2
+; GFX1232-NEXT: ; %bb.3: ; %atomicrmw.end
+; GFX1232-NEXT: s_or_b32 exec_lo, exec_lo, s10
+; GFX1232-NEXT: v_lshrrev_b32_e32 v0, s2, v2
+; GFX1232-NEXT: .LBB16_4: ; %Flow
+; GFX1232-NEXT: s_or_b32 exec_lo, exec_lo, s9
+; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1232-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX1232-NEXT: s_wait_kmcnt 0x0
+; GFX1232-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1232-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1232-NEXT: s_wait_alu 0xf1ff
+; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1232-NEXT: v_mad_u16 v0, s8, v4, s2
+; GFX1232-NEXT: s_mov_b32 s2, -1
+; GFX1232-NEXT: buffer_store_b16 v0, off, s[0:3], null
+; GFX1232-NEXT: s_endpgm
+ %rmw = atomicrmw add ptr addrspace(1) %uniform.ptr, i16 %val monotonic, align 2
+ store i16 %rmw, ptr addrspace(1) %result
+ ret void
+}
+
+define amdgpu_kernel void @uniform_xchg_i16(ptr addrspace(1) %result, ptr addrspace(1) %uniform.ptr, i16 %val) {
+; GFX7LESS-LABEL: uniform_xchg_i16:
+; GFX7LESS: ; %bb.0:
+; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7LESS-NEXT: s_load_dword s6, s[4:5], 0xd
+; GFX7LESS-NEXT: s_mov_b64 s[8:9], 0
+; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: s_and_b32 s4, s2, -4
+; GFX7LESS-NEXT: s_mov_b32 s5, s3
+; GFX7LESS-NEXT: s_and_b32 s2, s2, 3
+; GFX7LESS-NEXT: s_and_b32 s3, s6, 0xffff
+; GFX7LESS-NEXT: s_load_dword s6, s[4:5], 0x0
+; GFX7LESS-NEXT: s_lshl_b32 s10, s2, 3
+; GFX7LESS-NEXT: s_lshl_b32 s11, 0xffff, s10
+; GFX7LESS-NEXT: s_lshl_b32 s2, s3, s10
+; GFX7LESS-NEXT: s_not_b32 s3, s11
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s6
+; GFX7LESS-NEXT: s_mov_b32 s6, -1
+; GFX7LESS-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: v_and_b32_e32 v0, s3, v1
+; GFX7LESS-NEXT: v_or_b32_e32 v0, s2, v0
+; GFX7LESS-NEXT: s_waitcnt expcnt(0)
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
+; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX7LESS-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX7LESS-NEXT: s_cbranch_execnz .LBB17_1
+; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX7LESS-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, s10, v2
+; GFX7LESS-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX7LESS-NEXT: s_endpgm
+;
+; GFX8-LABEL: uniform_xchg_i16:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: s_and_b32 s4, s2, -4
+; GFX8-NEXT: s_mov_b32 s5, s3
+; GFX8-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX8-NEXT: s_and_b32 s2, s2, 3
+; GFX8-NEXT: s_lshl_b32 s8, s2, 3
+; GFX8-NEXT: s_lshl_b32 s2, 0xffff, s8
+; GFX8-NEXT: s_not_b32 s9, s2
+; GFX8-NEXT: s_and_b32 s2, s6, 0xffff
+; GFX8-NEXT: s_lshl_b32 s10, s2, s8
+; GFX8-NEXT: s_mov_b64 s[2:3], 0
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-NEXT: s_mov_b32 s6, -1
+; GFX8-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: v_and_b32_e32 v0, s9, v1
+; GFX8-NEXT: v_or_b32_e32 v0, s10, v0
+; GFX8-NEXT: v_mov_b32_e32 v3, v1
+; GFX8-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX8-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX8-NEXT: v_mov_b32_e32 v1, v2
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX8-NEXT: s_cbranch_execnz .LBB17_1
+; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX8-NEXT: s_mov_b32 s3, 0xf000
+; GFX8-NEXT: s_mov_b32 s2, -1
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, s8, v2
+; GFX8-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX8-NEXT: s_endpgm
+;
+; GFX9-LABEL: uniform_xchg_i16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_and_b32 s4, s2, -4
+; GFX9-NEXT: s_mov_b32 s5, s3
+; GFX9-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX9-NEXT: s_and_b32 s2, s2, 3
+; GFX9-NEXT: s_lshl_b32 s8, s2, 3
+; GFX9-NEXT: s_lshl_b32 s2, 0xffff, s8
+; GFX9-NEXT: s_not_b32 s9, s2
+; GFX9-NEXT: s_and_b32 s2, s6, 0xffff
+; GFX9-NEXT: s_lshl_b32 s10, s2, s8
+; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v1, s7
+; GFX9-NEXT: s_mov_b32 s7, 0xf000
+; GFX9-NEXT: s_mov_b32 s6, -1
+; GFX9-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: v_and_b32_e32 v0, s9, v1
+; GFX9-NEXT: v_or_b32_e32 v0, s10, v0
+; GFX9-NEXT: v_mov_b32_e32 v3, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, v0
+; GFX9-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: v_mov_b32_e32 v1, v2
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_cbranch_execnz .LBB17_1
+; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_mov_b32 s3, 0xf000
+; GFX9-NEXT: s_mov_b32 s2, -1
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, s8, v2
+; GFX9-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX9-NEXT: s_endpgm
+;
+; GFX1064-LABEL: uniform_xchg_i16:
+; GFX1064: ; %bb.0:
+; GFX1064-NEXT: s_clause 0x1
+; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX1064-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT: s_and_b32 s4, s2, -4
+; GFX1064-NEXT: s_mov_b32 s5, s3
+; GFX1064-NEXT: s_and_b32 s2, s2, 3
+; GFX1064-NEXT: s_load_dword s3, s[4:5], 0x0
+; GFX1064-NEXT: s_lshl_b32 s8, s2, 3
+; GFX1064-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX1064-NEXT: s_lshl_b32 s2, 0xffff, s8
+; GFX1064-NEXT: s_lshl_b32 s10, s6, s8
+; GFX1064-NEXT: s_not_b32 s9, s2
+; GFX1064-NEXT: s_mov_b32 s6, -1
+; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT: v_mov_b32_e32 v1, s3
+; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: v_and_or_b32 v0, v1, s9, s10
+; GFX1064-NEXT: v_mov_b32_e32 v3, v1
+; GFX1064-NEXT: v_mov_b32_e32 v2, v0
+; GFX1064-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
+; GFX1064-NEXT: s_waitcnt vmcnt(0)
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX1064-NEXT: v_mov_b32_e32 v1, v2
+; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_cbranch_execnz .LBB17_1
+; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1064-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s8, v2
+; GFX1064-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1064-NEXT: s_mov_b32 s2, -1
+; GFX1064-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX1064-NEXT: s_endpgm
+;
+; GFX1032-LABEL: uniform_xchg_i16:
+; GFX1032: ; %bb.0:
+; GFX1032-NEXT: s_clause 0x1
+; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX1032-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX1032-NEXT: s_mov_b32 s9, 0
+; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT: s_and_b32 s4, s2, -4
+; GFX1032-NEXT: s_mov_b32 s5, s3
+; GFX1032-NEXT: s_and_b32 s2, s2, 3
+; GFX1032-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX1032-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1032-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX1032-NEXT: s_lshl_b32 s3, 0xffff, s2
+; GFX1032-NEXT: s_lshl_b32 s8, s6, s2
+; GFX1032-NEXT: s_not_b32 s3, s3
+; GFX1032-NEXT: s_mov_b32 s6, -1
+; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT: v_mov_b32_e32 v1, s7
+; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1032-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: v_and_or_b32 v0, v1, s3, s8
+; GFX1032-NEXT: v_mov_b32_e32 v3, v1
+; GFX1032-NEXT: v_mov_b32_e32 v2, v0
+; GFX1032-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
+; GFX1032-NEXT: s_waitcnt vmcnt(0)
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX1032-NEXT: v_mov_b32_e32 v1, v2
+; GFX1032-NEXT: s_or_b32 s9, vcc_lo, s9
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s9
+; GFX1032-NEXT: s_cbranch_execnz .LBB17_1
+; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s9
+; GFX1032-NEXT: v_lshrrev_b32_e32 v0, s2, v2
+; GFX1032-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1032-NEXT: s_mov_b32 s2, -1
+; GFX1032-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX1032-NEXT: s_endpgm
+;
+; GFX1164-LABEL: uniform_xchg_i16:
+; GFX1164: ; %bb.0:
+; GFX1164-NEXT: s_clause 0x1
+; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1164-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX1164-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-NEXT: s_and_b32 s4, s2, -4
+; GFX1164-NEXT: s_mov_b32 s5, s3
+; GFX1164-NEXT: s_and_b32 s2, s2, 3
+; GFX1164-NEXT: s_load_b32 s3, s[4:5], 0x0
+; GFX1164-NEXT: s_lshl_b32 s8, s2, 3
+; GFX1164-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX1164-NEXT: s_lshl_b32 s2, 0xffff, s8
+; GFX1164-NEXT: s_lshl_b32 s10, s6, s8
+; GFX1164-NEXT: s_not_b32 s9, s2
+; GFX1164-NEXT: s_mov_b32 s6, -1
+; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-NEXT: v_mov_b32_e32 v1, s3
+; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_and_or_b32 v0, v1, s9, s10
+; GFX1164-NEXT: v_mov_b32_e32 v3, v1
+; GFX1164-NEXT: v_mov_b32_e32 v2, v0
+; GFX1164-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
+; GFX1164-NEXT: s_waitcnt vmcnt(0)
+; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX1164-NEXT: v_mov_b32_e32 v1, v2
+; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_cbranch_execnz .LBB17_1
+; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1164-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: v_lshrrev_b32_e32 v0, s8, v2
+; GFX1164-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1164-NEXT: s_mov_b32 s2, -1
+; GFX1164-NEXT: buffer_store_b16 v0, off, s[0:3], 0
+; GFX1164-NEXT: s_endpgm
+;
+; GFX1132-LABEL: uniform_xchg_i16:
+; GFX1132: ; %bb.0:
+; GFX1132-NEXT: s_clause 0x1
+; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1132-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX1132-NEXT: s_mov_b32 s9, 0
+; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-NEXT: s_and_b32 s4, s2, -4
+; GFX1132-NEXT: s_mov_b32 s5, s3
+; GFX1132-NEXT: s_and_b32 s2, s2, 3
+; GFX1132-NEXT: s_load_b32 s7, s[4:5], 0x0
+; GFX1132-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1132-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX1132-NEXT: s_lshl_b32 s3, 0xffff, s2
+; GFX1132-NEXT: s_lshl_b32 s8, s6, s2
+; GFX1132-NEXT: s_not_b32 s3, s3
+; GFX1132-NEXT: s_mov_b32 s6, -1
+; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-NEXT: v_mov_b32_e32 v1, s7
+; GFX1132-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1132-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_and_or_b32 v0, v1, s3, s8
+; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
+; GFX1132-NEXT: s_waitcnt vmcnt(0)
+; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX1132-NEXT: v_mov_b32_e32 v1, v2
+; GFX1132-NEXT: s_or_b32 s9, vcc_lo, s9
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s9
+; GFX1132-NEXT: s_cbranch_execnz .LBB17_1
+; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s9
+; GFX1132-NEXT: v_lshrrev_b32_e32 v0, s2, v2
+; GFX1132-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1132-NEXT: s_mov_b32 s2, -1
+; GFX1132-NEXT: buffer_store_b16 v0, off, s[0:3], 0
+; GFX1132-NEXT: s_endpgm
+;
+; GFX1264-LABEL: uniform_xchg_i16:
+; GFX1264: ; %bb.0:
+; GFX1264-NEXT: s_clause 0x1
+; GFX1264-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1264-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX1264-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1264-NEXT: s_wait_kmcnt 0x0
+; GFX1264-NEXT: s_and_b32 s4, s2, -4
+; GFX1264-NEXT: s_mov_b32 s5, s3
+; GFX1264-NEXT: s_and_b32 s2, s2, 3
+; GFX1264-NEXT: s_load_b32 s3, s[4:5], 0x0
+; GFX1264-NEXT: s_lshl_b32 s8, s2, 3
+; GFX1264-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX1264-NEXT: s_lshl_b32 s2, 0xffff, s8
+; GFX1264-NEXT: s_lshl_b32 s10, s6, s8
+; GFX1264-NEXT: s_not_b32 s9, s2
+; GFX1264-NEXT: s_mov_b32 s6, -1
+; GFX1264-NEXT: s_wait_kmcnt 0x0
+; GFX1264-NEXT: v_mov_b32_e32 v1, s3
+; GFX1264-NEXT: s_mov_b64 s[2:3], 0
+; GFX1264-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX1264-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1264-NEXT: v_and_or_b32 v0, v1, s9, s10
+; GFX1264-NEXT: v_mov_b32_e32 v3, v1
+; GFX1264-NEXT: v_mov_b32_e32 v2, v0
+; GFX1264-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1264-NEXT: s_wait_loadcnt 0x0
+; GFX1264-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX1264-NEXT: v_mov_b32_e32 v1, v2
+; GFX1264-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1264-NEXT: s_wait_alu 0xfffe
+; GFX1264-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1264-NEXT: s_cbranch_execnz .LBB17_1
+; GFX1264-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1264-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX1264-NEXT: v_lshrrev_b32_e32 v0, s8, v2
+; GFX1264-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1264-NEXT: s_mov_b32 s2, -1
+; GFX1264-NEXT: buffer_store_b16 v0, off, s[0:3], null
+; GFX1264-NEXT: s_endpgm
+;
+; GFX1232-LABEL: uniform_xchg_i16:
+; GFX1232: ; %bb.0:
+; GFX1232-NEXT: s_clause 0x1
+; GFX1232-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1232-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX1232-NEXT: s_mov_b32 s9, 0
+; GFX1232-NEXT: s_wait_kmcnt 0x0
+; GFX1232-NEXT: s_and_b32 s4, s2, -4
+; GFX1232-NEXT: s_mov_b32 s5, s3
+; GFX1232-NEXT: s_and_b32 s2, s2, 3
+; GFX1232-NEXT: s_load_b32 s7, s[4:5], 0x0
+; GFX1232-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1232-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX1232-NEXT: s_lshl_b32 s3, 0xffff, s2
+; GFX1232-NEXT: s_lshl_b32 s8, s6, s2
+; GFX1232-NEXT: s_not_b32 s3, s3
+; GFX1232-NEXT: s_mov_b32 s6, -1
+; GFX1232-NEXT: s_wait_kmcnt 0x0
+; GFX1232-NEXT: v_mov_b32_e32 v1, s7
+; GFX1232-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1232-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX1232-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1232-NEXT: v_and_or_b32 v0, v1, s3, s8
+; GFX1232-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1232-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1232-NEXT: s_wait_loadcnt 0x0
+; GFX1232-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX1232-NEXT: v_mov_b32_e32 v1, v2
+; GFX1232-NEXT: s_or_b32 s9, vcc_lo, s9
+; GFX1232-NEXT: s_wait_alu 0xfffe
+; GFX1232-NEXT: s_and_not1_b32 exec_lo, exec_lo, s9
+; GFX1232-NEXT: s_cbranch_execnz .LBB17_1
+; GFX1232-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1232-NEXT: s_or_b32 exec_lo, exec_lo, s9
+; GFX1232-NEXT: v_lshrrev_b32_e32 v0, s2, v2
+; GFX1232-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1232-NEXT: s_mov_b32 s2, -1
+; GFX1232-NEXT: buffer_store_b16 v0, off, s[0:3], null
+; GFX1232-NEXT: s_endpgm
+ %rmw = atomicrmw xchg ptr addrspace(1) %uniform.ptr, i16 %val monotonic, align 2
+ store i16 %rmw, ptr addrspace(1) %result
+ ret void
+}
+
+define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrspace(1) %uniform.ptr, half %val) {
+; GFX7LESS-LABEL: uniform_fadd_f16:
+; GFX7LESS: ; %bb.0:
+; GFX7LESS-NEXT: s_load_dword s6, s[4:5], 0xd
+; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7LESS-NEXT: s_mov_b64 s[8:9], 0
+; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: v_cvt_f32_f16_e32 v0, s6
+; GFX7LESS-NEXT: s_and_b32 s4, s2, -4
+; GFX7LESS-NEXT: s_mov_b32 s5, s3
+; GFX7LESS-NEXT: s_and_b32 s2, s2, 3
+; GFX7LESS-NEXT: s_lshl_b32 s10, s2, 3
+; GFX7LESS-NEXT: s_load_dword s3, s[4:5], 0x0
+; GFX7LESS-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX7LESS-NEXT: s_lshl_b32 s2, 0xffff, s10
+; GFX7LESS-NEXT: v_cvt_f32_f16_e32 v4, v0
+; GFX7LESS-NEXT: s_not_b32 s2, s2
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s3
+; GFX7LESS-NEXT: s_mov_b32 s6, -1
+; GFX7LESS-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, s10, v1
+; GFX7LESS-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX7LESS-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX7LESS-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX7LESS-NEXT: v_lshlrev_b32_e32 v0, s10, v0
+; GFX7LESS-NEXT: s_waitcnt expcnt(0)
+; GFX7LESS-NEXT: v_and_b32_e32 v2, s2, v1
+; GFX7LESS-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
+; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX7LESS-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX7LESS-NEXT: s_cbranch_execnz .LBB18_1
+; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX7LESS-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, s10, v2
+; GFX7LESS-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX7LESS-NEXT: s_endpgm
+;
+; GFX8-LABEL: uniform_fadd_f16:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT: s_load_dword s11, s[4:5], 0x34
+; GFX8-NEXT: s_mov_b64 s[8:9], 0
+; GFX8-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-NEXT: s_mov_b32 s6, -1
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: s_and_b32 s4, s2, -4
+; GFX8-NEXT: s_mov_b32 s5, s3
+; GFX8-NEXT: s_load_dword s3, s[4:5], 0x0
+; GFX8-NEXT: s_and_b32 s2, s2, 3
+; GFX8-NEXT: s_lshl_b32 s10, s2, 3
+; GFX8-NEXT: s_lshl_b32 s2, 0xffff, s10
+; GFX8-NEXT: s_not_b32 s2, s2
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, s10, v1
+; GFX8-NEXT: v_add_f16_e32 v0, s11, v0
+; GFX8-NEXT: v_and_b32_e32 v2, s2, v1
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, s10, v0
+; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX8-NEXT: v_mov_b32_e32 v3, v1
+; GFX8-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX8-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX8-NEXT: v_mov_b32_e32 v1, v2
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX8-NEXT: s_cbranch_execnz .LBB18_1
+; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX8-NEXT: s_mov_b32 s3, 0xf000
+; GFX8-NEXT: s_mov_b32 s2, -1
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, s10, v2
+; GFX8-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX8-NEXT: s_endpgm
+;
+; GFX9-LABEL: uniform_fadd_f16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_load_dword s11, s[4:5], 0x34
+; GFX9-NEXT: s_mov_b64 s[8:9], 0
+; GFX9-NEXT: s_mov_b32 s7, 0xf000
+; GFX9-NEXT: s_mov_b32 s6, -1
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_and_b32 s4, s2, -4
+; GFX9-NEXT: s_mov_b32 s5, s3
+; GFX9-NEXT: s_load_dword s3, s[4:5], 0x0
+; GFX9-NEXT: s_and_b32 s2, s2, 3
+; GFX9-NEXT: s_lshl_b32 s10, s2, 3
+; GFX9-NEXT: s_lshl_b32 s2, 0xffff, s10
+; GFX9-NEXT: s_not_b32 s2, s2
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, s10, v1
+; GFX9-NEXT: v_add_f16_e32 v0, s11, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, s10, v0
+; GFX9-NEXT: v_and_or_b32 v0, v1, s2, v0
+; GFX9-NEXT: v_mov_b32_e32 v3, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, v0
+; GFX9-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX9-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX9-NEXT: v_mov_b32_e32 v1, v2
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX9-NEXT: s_cbranch_execnz .LBB18_1
+; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX9-NEXT: s_mov_b32 s3, 0xf000
+; GFX9-NEXT: s_mov_b32 s2, -1
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, s10, v2
+; GFX9-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX9-NEXT: s_endpgm
+;
+; GFX1064-LABEL: uniform_fadd_f16:
+; GFX1064: ; %bb.0:
+; GFX1064-NEXT: s_clause 0x1
+; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX1064-NEXT: s_load_dword s8, s[4:5], 0x34
+; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1064-NEXT: s_mov_b32 s6, -1
+; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT: s_and_b32 s4, s2, -4
+; GFX1064-NEXT: s_mov_b32 s5, s3
+; GFX1064-NEXT: s_and_b32 s2, s2, 3
+; GFX1064-NEXT: s_load_dword s3, s[4:5], 0x0
+; GFX1064-NEXT: s_lshl_b32 s9, s2, 3
+; GFX1064-NEXT: s_lshl_b32 s2, 0xffff, s9
+; GFX1064-NEXT: s_not_b32 s10, s2
+; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT: v_mov_b32_e32 v1, s3
+; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s9, v1
+; GFX1064-NEXT: v_add_f16_e32 v0, s8, v0
+; GFX1064-NEXT: v_lshlrev_b32_sdwa v0, s9, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX1064-NEXT: v_and_or_b32 v0, v1, s10, v0
+; GFX1064-NEXT: v_mov_b32_e32 v3, v1
+; GFX1064-NEXT: v_mov_b32_e32 v2, v0
+; GFX1064-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
+; GFX1064-NEXT: s_waitcnt vmcnt(0)
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX1064-NEXT: v_mov_b32_e32 v1, v2
+; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_cbranch_execnz .LBB18_1
+; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1064-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s9, v2
+; GFX1064-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1064-NEXT: s_mov_b32 s2, -1
+; GFX1064-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX1064-NEXT: s_endpgm
+;
+; GFX1032-LABEL: uniform_fadd_f16:
+; GFX1032: ; %bb.0:
+; GFX1032-NEXT: s_clause 0x1
+; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX1032-NEXT: s_load_dword s8, s[4:5], 0x34
+; GFX1032-NEXT: s_mov_b32 s9, 0
+; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT: s_and_b32 s4, s2, -4
+; GFX1032-NEXT: s_mov_b32 s5, s3
+; GFX1032-NEXT: s_and_b32 s2, s2, 3
+; GFX1032-NEXT: s_load_dword s6, s[4:5], 0x0
+; GFX1032-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1032-NEXT: s_lshl_b32 s3, 0xffff, s2
+; GFX1032-NEXT: s_not_b32 s3, s3
+; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT: v_mov_b32_e32 v1, s6
+; GFX1032-NEXT: s_mov_b32 s6, -1
+; GFX1032-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: v_lshrrev_b32_e32 v0, s2, v1
+; GFX1032-NEXT: v_add_f16_e32 v0, s8, v0
+; GFX1032-NEXT: v_lshlrev_b32_sdwa v0, s2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX1032-NEXT: v_and_or_b32 v0, v1, s3, v0
+; GFX1032-NEXT: v_mov_b32_e32 v3, v1
+; GFX1032-NEXT: v_mov_b32_e32 v2, v0
+; GFX1032-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
+; GFX1032-NEXT: s_waitcnt vmcnt(0)
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX1032-NEXT: v_mov_b32_e32 v1, v2
+; GFX1032-NEXT: s_or_b32 s9, vcc_lo, s9
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s9
+; GFX1032-NEXT: s_cbranch_execnz .LBB18_1
+; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s9
+; GFX1032-NEXT: v_lshrrev_b32_e32 v0, s2, v2
+; GFX1032-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1032-NEXT: s_mov_b32 s2, -1
+; GFX1032-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX1032-NEXT: s_endpgm
+;
+; GFX1164-LABEL: uniform_fadd_f16:
+; GFX1164: ; %bb.0:
+; GFX1164-NEXT: s_clause 0x1
+; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1164-NEXT: s_load_b32 s8, s[4:5], 0x34
+; GFX1164-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1164-NEXT: s_mov_b32 s6, -1
+; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-NEXT: s_and_b32 s4, s2, -4
+; GFX1164-NEXT: s_mov_b32 s5, s3
+; GFX1164-NEXT: s_and_b32 s2, s2, 3
+; GFX1164-NEXT: s_load_b32 s3, s[4:5], 0x0
+; GFX1164-NEXT: s_lshl_b32 s9, s2, 3
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_lshl_b32 s2, 0xffff, s9
+; GFX1164-NEXT: s_not_b32 s10, s2
+; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-NEXT: v_mov_b32_e32 v1, s3
+; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: v_lshrrev_b32_e32 v0, s9, v1
+; GFX1164-NEXT: v_add_f16_e32 v0, s8, v0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX1164-NEXT: v_lshlrev_b32_e32 v0, s9, v0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_and_or_b32 v0, v1, s10, v0
+; GFX1164-NEXT: v_mov_b32_e32 v3, v1
+; GFX1164-NEXT: v_mov_b32_e32 v2, v0
+; GFX1164-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
+; GFX1164-NEXT: s_waitcnt vmcnt(0)
+; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX1164-NEXT: v_mov_b32_e32 v1, v2
+; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_cbranch_execnz .LBB18_1
+; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1164-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: v_lshrrev_b32_e32 v0, s9, v2
+; GFX1164-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1164-NEXT: s_mov_b32 s2, -1
+; GFX1164-NEXT: buffer_store_b16 v0, off, s[0:3], 0
+; GFX1164-NEXT: s_endpgm
+;
+; GFX1132-LABEL: uniform_fadd_f16:
+; GFX1132: ; %bb.0:
+; GFX1132-NEXT: s_clause 0x1
+; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1132-NEXT: s_load_b32 s8, s[4:5], 0x34
+; GFX1132-NEXT: s_mov_b32 s9, 0
+; GFX1132-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-NEXT: s_and_b32 s4, s2, -4
+; GFX1132-NEXT: s_mov_b32 s5, s3
+; GFX1132-NEXT: s_and_b32 s2, s2, 3
+; GFX1132-NEXT: s_load_b32 s6, s[4:5], 0x0
+; GFX1132-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: s_lshl_b32 s3, 0xffff, s2
+; GFX1132-NEXT: s_not_b32 s3, s3
+; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-NEXT: v_mov_b32_e32 v1, s6
+; GFX1132-NEXT: s_mov_b32 s6, -1
+; GFX1132-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_lshrrev_b32_e32 v0, s2, v1
+; GFX1132-NEXT: v_add_f16_e32 v0, s8, v0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX1132-NEXT: v_lshlrev_b32_e32 v0, s2, v0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_and_or_b32 v0, v1, s3, v0
+; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
+; GFX1132-NEXT: s_waitcnt vmcnt(0)
+; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX1132-NEXT: v_mov_b32_e32 v1, v2
+; GFX1132-NEXT: s_or_b32 s9, vcc_lo, s9
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s9
+; GFX1132-NEXT: s_cbranch_execnz .LBB18_1
+; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s9
+; GFX1132-NEXT: v_lshrrev_b32_e32 v0, s2, v2
+; GFX1132-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1132-NEXT: s_mov_b32 s2, -1
+; GFX1132-NEXT: buffer_store_b16 v0, off, s[0:3], 0
+; GFX1132-NEXT: s_endpgm
+;
+; GFX1264-LABEL: uniform_fadd_f16:
+; GFX1264: ; %bb.0:
+; GFX1264-NEXT: s_clause 0x1
+; GFX1264-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1264-NEXT: s_load_b32 s8, s[4:5], 0x34
+; GFX1264-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1264-NEXT: s_mov_b32 s6, -1
+; GFX1264-NEXT: s_wait_kmcnt 0x0
+; GFX1264-NEXT: s_and_b32 s4, s2, -4
+; GFX1264-NEXT: s_mov_b32 s5, s3
+; GFX1264-NEXT: s_and_b32 s2, s2, 3
+; GFX1264-NEXT: s_load_b32 s3, s[4:5], 0x0
+; GFX1264-NEXT: s_lshl_b32 s9, s2, 3
+; GFX1264-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1264-NEXT: s_lshl_b32 s2, 0xffff, s9
+; GFX1264-NEXT: s_not_b32 s10, s2
+; GFX1264-NEXT: s_wait_kmcnt 0x0
+; GFX1264-NEXT: v_mov_b32_e32 v1, s3
+; GFX1264-NEXT: s_mov_b64 s[2:3], 0
+; GFX1264-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX1264-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1264-NEXT: v_lshrrev_b32_e32 v0, s9, v1
+; GFX1264-NEXT: v_add_f16_e32 v0, s8, v0
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1264-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX1264-NEXT: v_lshlrev_b32_e32 v0, s9, v0
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1264-NEXT: v_and_or_b32 v0, v1, s10, v0
+; GFX1264-NEXT: v_mov_b32_e32 v3, v1
+; GFX1264-NEXT: v_mov_b32_e32 v2, v0
+; GFX1264-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1264-NEXT: s_wait_loadcnt 0x0
+; GFX1264-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX1264-NEXT: v_mov_b32_e32 v1, v2
+; GFX1264-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1264-NEXT: s_wait_alu 0xfffe
+; GFX1264-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1264-NEXT: s_cbranch_execnz .LBB18_1
+; GFX1264-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1264-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX1264-NEXT: v_lshrrev_b32_e32 v0, s9, v2
+; GFX1264-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1264-NEXT: s_mov_b32 s2, -1
+; GFX1264-NEXT: buffer_store_b16 v0, off, s[0:3], null
+; GFX1264-NEXT: s_endpgm
+;
+; GFX1232-LABEL: uniform_fadd_f16:
+; GFX1232: ; %bb.0:
+; GFX1232-NEXT: s_clause 0x1
+; GFX1232-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1232-NEXT: s_load_b32 s8, s[4:5], 0x34
+; GFX1232-NEXT: s_mov_b32 s9, 0
+; GFX1232-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1232-NEXT: s_wait_kmcnt 0x0
+; GFX1232-NEXT: s_and_b32 s4, s2, -4
+; GFX1232-NEXT: s_mov_b32 s5, s3
+; GFX1232-NEXT: s_and_b32 s2, s2, 3
+; GFX1232-NEXT: s_load_b32 s6, s[4:5], 0x0
+; GFX1232-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1232-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1232-NEXT: s_lshl_b32 s3, 0xffff, s2
+; GFX1232-NEXT: s_not_b32 s3, s3
+; GFX1232-NEXT: s_wait_kmcnt 0x0
+; GFX1232-NEXT: v_mov_b32_e32 v1, s6
+; GFX1232-NEXT: s_mov_b32 s6, -1
+; GFX1232-NEXT: .LBB18_1: ; %atomicrmw.start
+; GFX1232-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1232-NEXT: v_lshrrev_b32_e32 v0, s2, v1
+; GFX1232-NEXT: v_add_f16_e32 v0, s8, v0
+; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1232-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX1232-NEXT: v_lshlrev_b32_e32 v0, s2, v0
+; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1232-NEXT: v_and_or_b32 v0, v1, s3, v0
+; GFX1232-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1232-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1232-NEXT: s_wait_loadcnt 0x0
+; GFX1232-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX1232-NEXT: v_mov_b32_e32 v1, v2
+; GFX1232-NEXT: s_or_b32 s9, vcc_lo, s9
+; GFX1232-NEXT: s_wait_alu 0xfffe
+; GFX1232-NEXT: s_and_not1_b32 exec_lo, exec_lo, s9
+; GFX1232-NEXT: s_cbranch_execnz .LBB18_1
+; GFX1232-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1232-NEXT: s_or_b32 exec_lo, exec_lo, s9
+; GFX1232-NEXT: v_lshrrev_b32_e32 v0, s2, v2
+; GFX1232-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1232-NEXT: s_mov_b32 s2, -1
+; GFX1232-NEXT: buffer_store_b16 v0, off, s[0:3], null
+; GFX1232-NEXT: s_endpgm
+ %rmw = atomicrmw fadd ptr addrspace(1) %uniform.ptr, half %val monotonic, align 2
+ store half %rmw, ptr addrspace(1) %result
+ ret void
+}
+
+define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrspace(1) %uniform.ptr, bfloat %val) {
+; GFX7LESS-LABEL: uniform_fadd_bf16:
+; GFX7LESS: ; %bb.0:
+; GFX7LESS-NEXT: s_load_dword s6, s[4:5], 0xd
+; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7LESS-NEXT: s_mov_b64 s[8:9], 0
+; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: s_lshl_b32 s6, s6, 16
+; GFX7LESS-NEXT: s_and_b32 s4, s2, -4
+; GFX7LESS-NEXT: s_mov_b32 s5, s3
+; GFX7LESS-NEXT: s_and_b32 s2, s2, 3
+; GFX7LESS-NEXT: s_lshl_b32 s10, s2, 3
+; GFX7LESS-NEXT: s_load_dword s3, s[4:5], 0x0
+; GFX7LESS-NEXT: v_mul_f32_e64 v0, 1.0, s6
+; GFX7LESS-NEXT: s_lshl_b32 s2, 0xffff, s10
+; GFX7LESS-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
+; GFX7LESS-NEXT: s_not_b32 s2, s2
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s3
+; GFX7LESS-NEXT: s_mov_b32 s6, -1
+; GFX7LESS-NEXT: .LBB19_1: ; %atomicrmw.start
+; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, s10, v1
+; GFX7LESS-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7LESS-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7LESS-NEXT: v_lshlrev_b32_e32 v0, s10, v0
+; GFX7LESS-NEXT: s_waitcnt expcnt(0)
+; GFX7LESS-NEXT: v_and_b32_e32 v2, s2, v1
+; GFX7LESS-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
+; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX7LESS-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX7LESS-NEXT: s_cbranch_execnz .LBB19_1
+; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX7LESS-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, s10, v2
+; GFX7LESS-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX7LESS-NEXT: s_endpgm
+;
+; GFX8-LABEL: uniform_fadd_bf16:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX8-NEXT: s_mov_b64 s[8:9], 0
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: s_and_b32 s4, s2, -4
+; GFX8-NEXT: s_mov_b32 s5, s3
+; GFX8-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX8-NEXT: s_and_b32 s3, s2, 3
+; GFX8-NEXT: s_lshl_b32 s10, s3, 3
+; GFX8-NEXT: s_lshl_b32 s3, 0xffff, s10
+; GFX8-NEXT: s_lshl_b32 s2, s6, 16
+; GFX8-NEXT: s_not_b32 s3, s3
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-NEXT: s_mov_b32 s6, -1
+; GFX8-NEXT: .LBB19_1: ; %atomicrmw.start
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: v_mov_b32_e32 v0, s10
+; GFX8-NEXT: v_lshrrev_b32_sdwa v3, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_add_f32_e32 v3, s2, v3
+; GFX8-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v3
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fff, v4
+; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v3
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX8-NEXT: v_and_b32_e32 v2, s3, v1
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX8-NEXT: v_mov_b32_e32 v3, v1
+; GFX8-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX8-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX8-NEXT: v_mov_b32_e32 v1, v2
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX8-NEXT: s_cbranch_execnz .LBB19_1
+; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX8-NEXT: s_mov_b32 s3, 0xf000
+; GFX8-NEXT: s_mov_b32 s2, -1
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, s10, v2
+; GFX8-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX8-NEXT: s_endpgm
+;
+; GFX9-LABEL: uniform_fadd_bf16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX9-NEXT: s_mov_b64 s[8:9], 0
+; GFX9-NEXT: s_movk_i32 s11, 0x7fff
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_and_b32 s4, s2, -4
+; GFX9-NEXT: s_mov_b32 s5, s3
+; GFX9-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX9-NEXT: s_and_b32 s3, s2, 3
+; GFX9-NEXT: s_lshl_b32 s10, s3, 3
+; GFX9-NEXT: s_lshl_b32 s3, 0xffff, s10
+; GFX9-NEXT: s_lshl_b32 s2, s6, 16
+; GFX9-NEXT: s_not_b32 s3, s3
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v1, s7
+; GFX9-NEXT: s_mov_b32 s7, 0xf000
+; GFX9-NEXT: s_mov_b32 s6, -1
+; GFX9-NEXT: .LBB19_1: ; %atomicrmw.start
+; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: v_lshrrev_b32_sdwa v0, s10, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_f32_e32 v0, s2, v0
+; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v2, v2, v0, s11
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_sdwa v0, s10, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_and_or_b32 v0, v1, s3, v0
+; GFX9-NEXT: v_mov_b32_e32 v3, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, v0
+; GFX9-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX9-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX9-NEXT: v_mov_b32_e32 v1, v2
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX9-NEXT: s_cbranch_execnz .LBB19_1
+; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX9-NEXT: s_mov_b32 s3, 0xf000
+; GFX9-NEXT: s_mov_b32 s2, -1
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, s10, v2
+; GFX9-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX9-NEXT: s_endpgm
+;
+; GFX1064-LABEL: uniform_fadd_bf16:
+; GFX1064: ; %bb.0:
+; GFX1064-NEXT: s_clause 0x1
+; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX1064-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT: s_and_b32 s4, s2, -4
+; GFX1064-NEXT: s_mov_b32 s5, s3
+; GFX1064-NEXT: s_and_b32 s2, s2, 3
+; GFX1064-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX1064-NEXT: s_lshl_b32 s8, s2, 3
+; GFX1064-NEXT: s_lshl_b32 s10, s6, 16
+; GFX1064-NEXT: s_lshl_b32 s2, 0xffff, s8
+; GFX1064-NEXT: s_mov_b32 s6, -1
+; GFX1064-NEXT: s_not_b32 s9, s2
+; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT: v_mov_b32_e32 v1, s7
+; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1064-NEXT: .LBB19_1: ; %atomicrmw.start
+; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: v_lshrrev_b32_sdwa v0, s8, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX1064-NEXT: v_add_f32_e32 v0, s10, v0
+; GFX1064-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX1064-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX1064-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX1064-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX1064-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
+; GFX1064-NEXT: v_lshlrev_b32_sdwa v0, s8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX1064-NEXT: v_and_or_b32 v0, v1, s9, v0
+; GFX1064-NEXT: v_mov_b32_e32 v3, v1
+; GFX1064-NEXT: v_mov_b32_e32 v2, v0
+; GFX1064-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
+; GFX1064-NEXT: s_waitcnt vmcnt(0)
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX1064-NEXT: v_mov_b32_e32 v1, v2
+; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_cbranch_execnz .LBB19_1
+; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1064-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s8, v2
+; GFX1064-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1064-NEXT: s_mov_b32 s2, -1
+; GFX1064-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX1064-NEXT: s_endpgm
+;
+; GFX1032-LABEL: uniform_fadd_bf16:
+; GFX1032: ; %bb.0:
+; GFX1032-NEXT: s_clause 0x1
+; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX1032-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT: s_and_b32 s4, s2, -4
+; GFX1032-NEXT: s_mov_b32 s5, s3
+; GFX1032-NEXT: s_and_b32 s2, s2, 3
+; GFX1032-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX1032-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1032-NEXT: s_lshl_b32 s9, s6, 16
+; GFX1032-NEXT: s_lshl_b32 s3, 0xffff, s2
+; GFX1032-NEXT: s_mov_b32 s6, -1
+; GFX1032-NEXT: s_not_b32 s8, s3
+; GFX1032-NEXT: s_mov_b32 s3, 0
+; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT: v_mov_b32_e32 v1, s7
+; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1032-NEXT: .LBB19_1: ; %atomicrmw.start
+; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: v_lshrrev_b32_sdwa v0, s2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX1032-NEXT: v_add_f32_e32 v0, s9, v0
+; GFX1032-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX1032-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX1032-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX1032-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX1032-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX1032-NEXT: v_lshlrev_b32_sdwa v0, s2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX1032-NEXT: v_and_or_b32 v0, v1, s8, v0
+; GFX1032-NEXT: v_mov_b32_e32 v3, v1
+; GFX1032-NEXT: v_mov_b32_e32 v2, v0
+; GFX1032-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
+; GFX1032-NEXT: s_waitcnt vmcnt(0)
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX1032-NEXT: v_mov_b32_e32 v1, v2
+; GFX1032-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
+; GFX1032-NEXT: s_cbranch_execnz .LBB19_1
+; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX1032-NEXT: v_lshrrev_b32_e32 v0, s2, v2
+; GFX1032-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1032-NEXT: s_mov_b32 s2, -1
+; GFX1032-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX1032-NEXT: s_endpgm
+;
+; GFX1164-LABEL: uniform_fadd_bf16:
+; GFX1164: ; %bb.0:
+; GFX1164-NEXT: s_clause 0x1
+; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1164-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-NEXT: s_and_b32 s4, s2, -4
+; GFX1164-NEXT: s_mov_b32 s5, s3
+; GFX1164-NEXT: s_and_b32 s2, s2, 3
+; GFX1164-NEXT: s_load_b32 s7, s[4:5], 0x0
+; GFX1164-NEXT: s_lshl_b32 s8, s2, 3
+; GFX1164-NEXT: s_lshl_b32 s10, s6, 16
+; GFX1164-NEXT: s_lshl_b32 s2, 0xffff, s8
+; GFX1164-NEXT: s_mov_b32 s6, -1
+; GFX1164-NEXT: s_not_b32 s9, s2
+; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-NEXT: v_mov_b32_e32 v1, s7
+; GFX1164-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1164-NEXT: .p2align 6
+; GFX1164-NEXT: .LBB19_1: ; %atomicrmw.start
+; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: v_lshrrev_b32_e32 v0, s8, v1
+; GFX1164-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: v_add_f32_e32 v0, s10, v0
+; GFX1164-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX1164-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX1164-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX1164-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX1164-NEXT: v_lshlrev_b32_e32 v0, s8, v0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_and_or_b32 v0, v1, s9, v0
+; GFX1164-NEXT: v_mov_b32_e32 v3, v1
+; GFX1164-NEXT: v_mov_b32_e32 v2, v0
+; GFX1164-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
+; GFX1164-NEXT: s_waitcnt vmcnt(0)
+; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX1164-NEXT: v_mov_b32_e32 v1, v2
+; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_cbranch_execnz .LBB19_1
+; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1164-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: v_lshrrev_b32_e32 v0, s8, v2
+; GFX1164-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1164-NEXT: s_mov_b32 s2, -1
+; GFX1164-NEXT: buffer_store_b16 v0, off, s[0:3], 0
+; GFX1164-NEXT: s_endpgm
+;
+; GFX1132-LABEL: uniform_fadd_bf16:
+; GFX1132: ; %bb.0:
+; GFX1132-NEXT: s_clause 0x1
+; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1132-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-NEXT: s_and_b32 s4, s2, -4
+; GFX1132-NEXT: s_mov_b32 s5, s3
+; GFX1132-NEXT: s_and_b32 s2, s2, 3
+; GFX1132-NEXT: s_load_b32 s7, s[4:5], 0x0
+; GFX1132-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1132-NEXT: s_lshl_b32 s9, s6, 16
+; GFX1132-NEXT: s_lshl_b32 s3, 0xffff, s2
+; GFX1132-NEXT: s_mov_b32 s6, -1
+; GFX1132-NEXT: s_not_b32 s8, s3
+; GFX1132-NEXT: s_mov_b32 s3, 0
+; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-NEXT: v_mov_b32_e32 v1, s7
+; GFX1132-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1132-NEXT: .p2align 6
+; GFX1132-NEXT: .LBB19_1: ; %atomicrmw.start
+; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_lshrrev_b32_e32 v0, s2, v1
+; GFX1132-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_add_f32_e32 v0, s9, v0
+; GFX1132-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX1132-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX1132-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX1132-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX1132-NEXT: v_lshlrev_b32_e32 v0, s2, v0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_and_or_b32 v0, v1, s8, v0
+; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
+; GFX1132-NEXT: s_waitcnt vmcnt(0)
+; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX1132-NEXT: v_mov_b32_e32 v1, v2
+; GFX1132-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
+; GFX1132-NEXT: s_cbranch_execnz .LBB19_1
+; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX1132-NEXT: v_lshrrev_b32_e32 v0, s2, v2
+; GFX1132-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1132-NEXT: s_mov_b32 s2, -1
+; GFX1132-NEXT: buffer_store_b16 v0, off, s[0:3], 0
+; GFX1132-NEXT: s_endpgm
+;
+; GFX1264-LABEL: uniform_fadd_bf16:
+; GFX1264: ; %bb.0:
+; GFX1264-NEXT: s_clause 0x1
+; GFX1264-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1264-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX1264-NEXT: s_wait_kmcnt 0x0
+; GFX1264-NEXT: s_and_b32 s4, s2, -4
+; GFX1264-NEXT: s_mov_b32 s5, s3
+; GFX1264-NEXT: s_and_b32 s2, s2, 3
+; GFX1264-NEXT: s_load_b32 s7, s[4:5], 0x0
+; GFX1264-NEXT: s_lshl_b32 s8, s2, 3
+; GFX1264-NEXT: s_lshl_b32 s10, s6, 16
+; GFX1264-NEXT: s_lshl_b32 s2, 0xffff, s8
+; GFX1264-NEXT: s_mov_b32 s6, -1
+; GFX1264-NEXT: s_not_b32 s9, s2
+; GFX1264-NEXT: s_mov_b64 s[2:3], 0
+; GFX1264-NEXT: s_wait_kmcnt 0x0
+; GFX1264-NEXT: v_mov_b32_e32 v1, s7
+; GFX1264-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1264-NEXT: .LBB19_1: ; %atomicrmw.start
+; GFX1264-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1264-NEXT: v_lshrrev_b32_e32 v0, s8, v1
+; GFX1264-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1264-NEXT: v_add_f32_e32 v0, s10, v0
+; GFX1264-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX1264-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX1264-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1264-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX1264-NEXT: s_wait_alu 0xfffd
+; GFX1264-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1264-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX1264-NEXT: v_lshlrev_b32_e32 v0, s8, v0
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1264-NEXT: v_and_or_b32 v0, v1, s9, v0
+; GFX1264-NEXT: v_mov_b32_e32 v3, v1
+; GFX1264-NEXT: v_mov_b32_e32 v2, v0
+; GFX1264-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1264-NEXT: s_wait_loadcnt 0x0
+; GFX1264-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX1264-NEXT: v_mov_b32_e32 v1, v2
+; GFX1264-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1264-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1264-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1264-NEXT: s_cbranch_execnz .LBB19_1
+; GFX1264-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1264-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX1264-NEXT: v_lshrrev_b32_e32 v0, s8, v2
+; GFX1264-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1264-NEXT: s_mov_b32 s2, -1
+; GFX1264-NEXT: buffer_store_b16 v0, off, s[0:3], null
+; GFX1264-NEXT: s_endpgm
+;
+; GFX1232-LABEL: uniform_fadd_bf16:
+; GFX1232: ; %bb.0:
+; GFX1232-NEXT: s_clause 0x1
+; GFX1232-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1232-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX1232-NEXT: s_wait_kmcnt 0x0
+; GFX1232-NEXT: s_and_b32 s4, s2, -4
+; GFX1232-NEXT: s_mov_b32 s5, s3
+; GFX1232-NEXT: s_and_b32 s2, s2, 3
+; GFX1232-NEXT: s_load_b32 s7, s[4:5], 0x0
+; GFX1232-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1232-NEXT: s_lshl_b32 s9, s6, 16
+; GFX1232-NEXT: s_lshl_b32 s3, 0xffff, s2
+; GFX1232-NEXT: s_mov_b32 s6, -1
+; GFX1232-NEXT: s_not_b32 s8, s3
+; GFX1232-NEXT: s_mov_b32 s3, 0
+; GFX1232-NEXT: s_wait_kmcnt 0x0
+; GFX1232-NEXT: v_mov_b32_e32 v1, s7
+; GFX1232-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1232-NEXT: .LBB19_1: ; %atomicrmw.start
+; GFX1232-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1232-NEXT: v_lshrrev_b32_e32 v0, s2, v1
+; GFX1232-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1232-NEXT: v_add_f32_e32 v0, s9, v0
+; GFX1232-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX1232-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX1232-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1232-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX1232-NEXT: s_wait_alu 0xfffd
+; GFX1232-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1232-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX1232-NEXT: v_lshlrev_b32_e32 v0, s2, v0
+; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1232-NEXT: v_and_or_b32 v0, v1, s8, v0
+; GFX1232-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1232-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1232-NEXT: s_wait_loadcnt 0x0
+; GFX1232-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX1232-NEXT: v_mov_b32_e32 v1, v2
+; GFX1232-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1232-NEXT: s_wait_alu 0xfffe
+; GFX1232-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
+; GFX1232-NEXT: s_cbranch_execnz .LBB19_1
+; GFX1232-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1232-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX1232-NEXT: v_lshrrev_b32_e32 v0, s2, v2
+; GFX1232-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1232-NEXT: s_mov_b32 s2, -1
+; GFX1232-NEXT: buffer_store_b16 v0, off, s[0:3], null
+; GFX1232-NEXT: s_endpgm
+ %rmw = atomicrmw fadd ptr addrspace(1) %uniform.ptr, bfloat %val monotonic, align 2
+ store bfloat %rmw, ptr addrspace(1) %result
+ ret void
+}
+
+define amdgpu_kernel void @uniform_fadd_v2f16(ptr addrspace(1) %result, ptr addrspace(1) %uniform.ptr, <2 x half> %val) {
+; GFX7LESS-LABEL: uniform_fadd_v2f16:
+; GFX7LESS: ; %bb.0:
+; GFX7LESS-NEXT: s_load_dword s6, s[4:5], 0xd
+; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7LESS-NEXT: s_mov_b64 s[8:9], 0
+; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: s_lshr_b32 s4, s6, 16
+; GFX7LESS-NEXT: v_cvt_f32_f16_e32 v0, s6
+; GFX7LESS-NEXT: s_load_dword s5, s[2:3], 0x0
+; GFX7LESS-NEXT: v_cvt_f32_f16_e32 v1, s4
+; GFX7LESS-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX7LESS-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX7LESS-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX7LESS-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: v_cvt_f32_f16_e32 v2, s5
+; GFX7LESS-NEXT: s_lshr_b32 s4, s5, 16
+; GFX7LESS-NEXT: v_cvt_f32_f16_e32 v3, s4
+; GFX7LESS-NEXT: s_mov_b32 s6, -1
+; GFX7LESS-NEXT: s_mov_b32 s4, s2
+; GFX7LESS-NEXT: s_mov_b32 s5, s3
+; GFX7LESS-NEXT: .LBB20_1: ; %atomicrmw.start
+; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX7LESS-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX7LESS-NEXT: s_waitcnt expcnt(0)
+; GFX7LESS-NEXT: v_cvt_f32_f16_e32 v4, v3
+; GFX7LESS-NEXT: v_cvt_f32_f16_e32 v5, v2
+; GFX7LESS-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX7LESS-NEXT: v_add_f32_e32 v4, v4, v1
+; GFX7LESS-NEXT: v_add_f32_e32 v5, v5, v0
+; GFX7LESS-NEXT: v_or_b32_e32 v3, v2, v3
+; GFX7LESS-NEXT: v_cvt_f16_f32_e32 v2, v4
+; GFX7LESS-NEXT: v_cvt_f16_f32_e32 v4, v5
+; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7LESS-NEXT: v_or_b32_e32 v2, v4, v2
+; GFX7LESS-NEXT: v_mov_b32_e32 v5, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, v2
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[4:5], off, s[4:7], 0 glc
+; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
+; GFX7LESS-NEXT: v_lshrrev_b32_e32 v3, 16, v4
+; GFX7LESS-NEXT: v_cvt_f32_f16_e32 v2, v4
+; GFX7LESS-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX7LESS-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX7LESS-NEXT: s_cbranch_execnz .LBB20_1
+; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX7LESS-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-NEXT: v_cvt_f16_f32_e32 v0, v3
+; GFX7LESS-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7LESS-NEXT: v_cvt_f16_f32_e32 v1, v2
+; GFX7LESS-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX7LESS-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX7LESS-NEXT: s_endpgm
+;
+; GFX8-LABEL: uniform_fadd_v2f16:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT: s_load_dword s10, s[4:5], 0x34
+; GFX8-NEXT: s_mov_b64 s[8:9], 0
+; GFX8-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-NEXT: s_mov_b32 s6, -1
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: s_load_dword s5, s[2:3], 0x0
+; GFX8-NEXT: s_lshr_b32 s11, s10, 16
+; GFX8-NEXT: s_mov_b32 s4, s2
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: s_mov_b32 s5, s3
+; GFX8-NEXT: .LBB20_1: ; %atomicrmw.start
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: v_mov_b32_e32 v0, s11
+; GFX8-NEXT: v_add_f16_e32 v2, s10, v1
+; GFX8-NEXT: v_add_f16_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX8-NEXT: v_mov_b32_e32 v3, v1
+; GFX8-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX8-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX8-NEXT: v_mov_b32_e32 v1, v2
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX8-NEXT: s_cbranch_execnz .LBB20_1
+; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX8-NEXT: s_mov_b32 s3, 0xf000
+; GFX8-NEXT: s_mov_b32 s2, -1
+; GFX8-NEXT: buffer_store_dword v2, off, s[0:3], 0
+; GFX8-NEXT: s_endpgm
+;
+; GFX9-LABEL: uniform_fadd_v2f16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_load_dword s10, s[4:5], 0x34
+; GFX9-NEXT: s_mov_b64 s[8:9], 0
+; GFX9-NEXT: s_mov_b32 s7, 0xf000
+; GFX9-NEXT: s_mov_b32 s6, -1
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dword s5, s[2:3], 0x0
+; GFX9-NEXT: s_mov_b32 s4, s2
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v1, s5
+; GFX9-NEXT: s_mov_b32 s5, s3
+; GFX9-NEXT: .LBB20_1: ; %atomicrmw.start
+; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: v_pk_add_f16 v0, v1, s10
+; GFX9-NEXT: v_mov_b32_e32 v3, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, v0
+; GFX9-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX9-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX9-NEXT: v_mov_b32_e32 v1, v2
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX9-NEXT: s_cbranch_execnz .LBB20_1
+; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX9-NEXT: s_mov_b32 s3, 0xf000
+; GFX9-NEXT: s_mov_b32 s2, -1
+; GFX9-NEXT: buffer_store_dword v2, off, s[0:3], 0
+; GFX9-NEXT: s_endpgm
+;
+; GFX1064-LABEL: uniform_fadd_v2f16:
+; GFX1064: ; %bb.0:
+; GFX1064-NEXT: s_clause 0x1
+; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX1064-NEXT: s_load_dword s10, s[4:5], 0x34
+; GFX1064-NEXT: s_mov_b64 s[8:9], 0
+; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1064-NEXT: s_mov_b32 s6, -1
+; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT: s_load_dword s4, s[2:3], 0x0
+; GFX1064-NEXT: s_mov_b32 s5, s3
+; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT: v_mov_b32_e32 v1, s4
+; GFX1064-NEXT: s_mov_b32 s4, s2
+; GFX1064-NEXT: .LBB20_1: ; %atomicrmw.start
+; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: v_pk_add_f16 v0, v1, s10
+; GFX1064-NEXT: v_mov_b32_e32 v3, v1
+; GFX1064-NEXT: v_mov_b32_e32 v2, v0
+; GFX1064-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
+; GFX1064-NEXT: s_waitcnt vmcnt(0)
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX1064-NEXT: v_mov_b32_e32 v1, v2
+; GFX1064-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX1064-NEXT: s_cbranch_execnz .LBB20_1
+; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1064-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX1064-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1064-NEXT: s_mov_b32 s2, -1
+; GFX1064-NEXT: buffer_store_dword v2, off, s[0:3], 0
+; GFX1064-NEXT: s_endpgm
+;
+; GFX1032-LABEL: uniform_fadd_v2f16:
+; GFX1032: ; %bb.0:
+; GFX1032-NEXT: s_clause 0x1
+; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX1032-NEXT: s_load_dword s8, s[4:5], 0x34
+; GFX1032-NEXT: s_mov_b32 s9, 0
+; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1032-NEXT: s_mov_b32 s6, -1
+; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT: s_load_dword s4, s[2:3], 0x0
+; GFX1032-NEXT: s_mov_b32 s5, s3
+; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT: v_mov_b32_e32 v1, s4
+; GFX1032-NEXT: s_mov_b32 s4, s2
+; GFX1032-NEXT: .LBB20_1: ; %atomicrmw.start
+; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: v_pk_add_f16 v0, v1, s8
+; GFX1032-NEXT: v_mov_b32_e32 v3, v1
+; GFX1032-NEXT: v_mov_b32_e32 v2, v0
+; GFX1032-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
+; GFX1032-NEXT: s_waitcnt vmcnt(0)
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX1032-NEXT: v_mov_b32_e32 v1, v2
+; GFX1032-NEXT: s_or_b32 s9, vcc_lo, s9
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s9
+; GFX1032-NEXT: s_cbranch_execnz .LBB20_1
+; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s9
+; GFX1032-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1032-NEXT: s_mov_b32 s2, -1
+; GFX1032-NEXT: buffer_store_dword v2, off, s[0:3], 0
+; GFX1032-NEXT: s_endpgm
+;
+; GFX1164-LABEL: uniform_fadd_v2f16:
+; GFX1164: ; %bb.0:
+; GFX1164-NEXT: s_clause 0x1
+; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1164-NEXT: s_load_b32 s10, s[4:5], 0x34
+; GFX1164-NEXT: s_mov_b64 s[8:9], 0
+; GFX1164-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1164-NEXT: s_mov_b32 s6, -1
+; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-NEXT: s_load_b32 s4, s[2:3], 0x0
+; GFX1164-NEXT: s_mov_b32 s5, s3
+; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-NEXT: v_mov_b32_e32 v1, s4
+; GFX1164-NEXT: s_mov_b32 s4, s2
+; GFX1164-NEXT: .LBB20_1: ; %atomicrmw.start
+; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_pk_add_f16 v0, v1, s10
+; GFX1164-NEXT: v_mov_b32_e32 v3, v1
+; GFX1164-NEXT: v_mov_b32_e32 v2, v0
+; GFX1164-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
+; GFX1164-NEXT: s_waitcnt vmcnt(0)
+; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX1164-NEXT: v_mov_b32_e32 v1, v2
+; GFX1164-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[8:9]
+; GFX1164-NEXT: s_cbranch_execnz .LBB20_1
+; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1164-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX1164-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1164-NEXT: s_mov_b32 s2, -1
+; GFX1164-NEXT: buffer_store_b32 v2, off, s[0:3], 0
+; GFX1164-NEXT: s_endpgm
+;
+; GFX1132-LABEL: uniform_fadd_v2f16:
+; GFX1132: ; %bb.0:
+; GFX1132-NEXT: s_clause 0x1
+; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1132-NEXT: s_load_b32 s8, s[4:5], 0x34
+; GFX1132-NEXT: s_mov_b32 s9, 0
+; GFX1132-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1132-NEXT: s_mov_b32 s6, -1
+; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-NEXT: s_load_b32 s4, s[2:3], 0x0
+; GFX1132-NEXT: s_mov_b32 s5, s3
+; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-NEXT: v_mov_b32_e32 v1, s4
+; GFX1132-NEXT: s_mov_b32 s4, s2
+; GFX1132-NEXT: .LBB20_1: ; %atomicrmw.start
+; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_pk_add_f16 v0, v1, s8
+; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
+; GFX1132-NEXT: s_waitcnt vmcnt(0)
+; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX1132-NEXT: v_mov_b32_e32 v1, v2
+; GFX1132-NEXT: s_or_b32 s9, vcc_lo, s9
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s9
+; GFX1132-NEXT: s_cbranch_execnz .LBB20_1
+; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s9
+; GFX1132-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1132-NEXT: s_mov_b32 s2, -1
+; GFX1132-NEXT: buffer_store_b32 v2, off, s[0:3], 0
+; GFX1132-NEXT: s_endpgm
+;
+; GFX1264-LABEL: uniform_fadd_v2f16:
+; GFX1264: ; %bb.0:
+; GFX1264-NEXT: s_clause 0x1
+; GFX1264-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1264-NEXT: s_load_b32 s10, s[4:5], 0x34
+; GFX1264-NEXT: s_mov_b64 s[8:9], 0
+; GFX1264-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1264-NEXT: s_mov_b32 s6, -1
+; GFX1264-NEXT: s_wait_kmcnt 0x0
+; GFX1264-NEXT: s_load_b32 s4, s[2:3], 0x0
+; GFX1264-NEXT: s_mov_b32 s5, s3
+; GFX1264-NEXT: s_wait_kmcnt 0x0
+; GFX1264-NEXT: v_mov_b32_e32 v1, s4
+; GFX1264-NEXT: s_mov_b32 s4, s2
+; GFX1264-NEXT: .LBB20_1: ; %atomicrmw.start
+; GFX1264-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1264-NEXT: v_pk_add_f16 v0, v1, s10
+; GFX1264-NEXT: v_mov_b32_e32 v3, v1
+; GFX1264-NEXT: v_mov_b32_e32 v2, v0
+; GFX1264-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1264-NEXT: s_wait_loadcnt 0x0
+; GFX1264-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX1264-NEXT: v_mov_b32_e32 v1, v2
+; GFX1264-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX1264-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1264-NEXT: s_and_not1_b64 exec, exec, s[8:9]
+; GFX1264-NEXT: s_cbranch_execnz .LBB20_1
+; GFX1264-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1264-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX1264-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1264-NEXT: s_mov_b32 s2, -1
+; GFX1264-NEXT: buffer_store_b32 v2, off, s[0:3], null
+; GFX1264-NEXT: s_endpgm
+;
+; GFX1232-LABEL: uniform_fadd_v2f16:
+; GFX1232: ; %bb.0:
+; GFX1232-NEXT: s_clause 0x1
+; GFX1232-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1232-NEXT: s_load_b32 s8, s[4:5], 0x34
+; GFX1232-NEXT: s_mov_b32 s9, 0
+; GFX1232-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1232-NEXT: s_mov_b32 s6, -1
+; GFX1232-NEXT: s_wait_kmcnt 0x0
+; GFX1232-NEXT: s_load_b32 s4, s[2:3], 0x0
+; GFX1232-NEXT: s_mov_b32 s5, s3
+; GFX1232-NEXT: s_wait_kmcnt 0x0
+; GFX1232-NEXT: v_mov_b32_e32 v1, s4
+; GFX1232-NEXT: s_mov_b32 s4, s2
+; GFX1232-NEXT: .LBB20_1: ; %atomicrmw.start
+; GFX1232-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1232-NEXT: v_pk_add_f16 v0, v1, s8
+; GFX1232-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1232-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1232-NEXT: s_wait_loadcnt 0x0
+; GFX1232-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX1232-NEXT: v_mov_b32_e32 v1, v2
+; GFX1232-NEXT: s_or_b32 s9, vcc_lo, s9
+; GFX1232-NEXT: s_wait_alu 0xfffe
+; GFX1232-NEXT: s_and_not1_b32 exec_lo, exec_lo, s9
+; GFX1232-NEXT: s_cbranch_execnz .LBB20_1
+; GFX1232-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1232-NEXT: s_or_b32 exec_lo, exec_lo, s9
+; GFX1232-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1232-NEXT: s_mov_b32 s2, -1
+; GFX1232-NEXT: buffer_store_b32 v2, off, s[0:3], null
+; GFX1232-NEXT: s_endpgm
+ %rmw = atomicrmw fadd ptr addrspace(1) %uniform.ptr, <2 x half> %val monotonic, align 4
+ store <2 x half> %rmw, ptr addrspace(1) %result
+ ret void
+}
+
+define amdgpu_kernel void @uniform_fadd_v2bf16(ptr addrspace(1) %result, ptr addrspace(1) %uniform.ptr, <2 x bfloat> %val) {
+; GFX7LESS-LABEL: uniform_fadd_v2bf16:
+; GFX7LESS: ; %bb.0:
+; GFX7LESS-NEXT: s_load_dword s6, s[4:5], 0xd
+; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7LESS-NEXT: s_mov_b64 s[8:9], 0
+; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: s_and_b32 s4, s6, 0xffff0000
+; GFX7LESS-NEXT: s_lshl_b32 s5, s6, 16
+; GFX7LESS-NEXT: s_load_dword s6, s[2:3], 0x0
+; GFX7LESS-NEXT: v_mul_f32_e64 v0, 1.0, s5
+; GFX7LESS-NEXT: v_mul_f32_e64 v1, 1.0, s4
+; GFX7LESS-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: s_and_b32 s4, s6, 0xffff0000
+; GFX7LESS-NEXT: s_lshl_b32 s5, s6, 16
+; GFX7LESS-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s5
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s4
+; GFX7LESS-NEXT: s_mov_b32 s6, -1
+; GFX7LESS-NEXT: s_mov_b32 s4, s2
+; GFX7LESS-NEXT: s_mov_b32 s5, s3
+; GFX7LESS-NEXT: .LBB21_1: ; %atomicrmw.start
+; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7LESS-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7LESS-NEXT: s_waitcnt expcnt(0)
+; GFX7LESS-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
+; GFX7LESS-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
+; GFX7LESS-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX7LESS-NEXT: v_add_f32_e32 v4, v4, v0
+; GFX7LESS-NEXT: v_add_f32_e32 v5, v5, v1
+; GFX7LESS-NEXT: v_alignbit_b32 v3, v2, v3, 16
+; GFX7LESS-NEXT: v_lshrrev_b32_e32 v2, 16, v5
+; GFX7LESS-NEXT: v_alignbit_b32 v2, v2, v4, 16
+; GFX7LESS-NEXT: v_mov_b32_e32 v5, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, v2
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[4:5], off, s[4:7], 0 glc
+; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
+; GFX7LESS-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX7LESS-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX7LESS-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX7LESS-NEXT: s_cbranch_execnz .LBB21_1
+; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX7LESS-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-NEXT: v_mul_f32_e32 v0, 1.0, v2
+; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7LESS-NEXT: v_mul_f32_e32 v1, 1.0, v3
+; GFX7LESS-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7LESS-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX7LESS-NEXT: s_endpgm
+;
+; GFX8-LABEL: uniform_fadd_v2bf16:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX8-NEXT: s_load_dword s0, s[4:5], 0x34
+; GFX8-NEXT: s_mov_b64 s[2:3], 0
+; GFX8-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-NEXT: s_mov_b32 s6, -1
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: s_load_dword s1, s[10:11], 0x0
+; GFX8-NEXT: s_lshl_b32 s12, s0, 16
+; GFX8-NEXT: s_and_b32 s13, s0, 0xffff0000
+; GFX8-NEXT: s_mov_b32 s4, s10
+; GFX8-NEXT: s_mov_b32 s5, s11
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: .LBB21_1: ; %atomicrmw.start
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX8-NEXT: v_add_f32_e32 v0, s12, v0
+; GFX8-NEXT: v_add_f32_e32 v2, s13, v2
+; GFX8-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX8-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v0
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v2
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fff, v3
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5
+; GFX8-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX8-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX8-NEXT: v_cmp_u_f32_e64 s[0:1], v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v3, v4, s[0:1]
+; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX8-NEXT: v_alignbit_b32 v0, v2, v0, 16
+; GFX8-NEXT: v_mov_b32_e32 v3, v1
+; GFX8-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX8-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX8-NEXT: v_mov_b32_e32 v1, v2
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX8-NEXT: s_cbranch_execnz .LBB21_1
+; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX8-NEXT: s_mov_b32 s11, 0xf000
+; GFX8-NEXT: s_mov_b32 s10, -1
+; GFX8-NEXT: buffer_store_dword v2, off, s[8:11], 0
+; GFX8-NEXT: s_endpgm
+;
+; GFX9-LABEL: uniform_fadd_v2bf16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX9-NEXT: s_load_dword s0, s[4:5], 0x34
+; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: s_movk_i32 s12, 0x7fff
+; GFX9-NEXT: s_mov_b32 s13, 0x7060302
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dword s1, s[10:11], 0x0
+; GFX9-NEXT: s_mov_b32 s7, 0xf000
+; GFX9-NEXT: s_mov_b32 s6, -1
+; GFX9-NEXT: s_lshl_b32 s14, s0, 16
+; GFX9-NEXT: s_and_b32 s15, s0, 0xffff0000
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v1, s1
+; GFX9-NEXT: s_mov_b32 s4, s10
+; GFX9-NEXT: s_mov_b32 s5, s11
+; GFX9-NEXT: .LBB21_1: ; %atomicrmw.start
+; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX9-NEXT: v_add_f32_e32 v0, s14, v0
+; GFX9-NEXT: v_add_f32_e32 v2, s15, v2
+; GFX9-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX9-NEXT: v_add3_u32 v3, v3, v0, s12
+; GFX9-NEXT: v_add3_u32 v5, v5, v2, s12
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: v_cmp_u_f32_e64 s[0:1], v0, v0
+; GFX9-NEXT: v_cndmask_b32_e64 v0, v3, v4, s[0:1]
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
+; GFX9-NEXT: v_perm_b32 v0, v2, v0, s13
+; GFX9-NEXT: v_mov_b32_e32 v3, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, v0
+; GFX9-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: v_mov_b32_e32 v1, v2
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_cbranch_execnz .LBB21_1
+; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_mov_b32 s11, 0xf000
+; GFX9-NEXT: s_mov_b32 s10, -1
+; GFX9-NEXT: buffer_store_dword v2, off, s[8:11], 0
+; GFX9-NEXT: s_endpgm
+;
+; GFX1064-LABEL: uniform_fadd_v2bf16:
+; GFX1064: ; %bb.0:
+; GFX1064-NEXT: s_clause 0x1
+; GFX1064-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX1064-NEXT: s_load_dword s0, s[4:5], 0x34
+; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1064-NEXT: s_mov_b32 s6, -1
+; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT: s_load_dword s1, s[10:11], 0x0
+; GFX1064-NEXT: s_lshl_b32 s12, s0, 16
+; GFX1064-NEXT: s_and_b32 s13, s0, 0xffff0000
+; GFX1064-NEXT: s_mov_b32 s4, s10
+; GFX1064-NEXT: s_mov_b32 s5, s11
+; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT: v_mov_b32_e32 v1, s1
+; GFX1064-NEXT: .LBB21_1: ; %atomicrmw.start
+; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX1064-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX1064-NEXT: v_add_f32_e32 v0, s12, v0
+; GFX1064-NEXT: v_add_f32_e32 v2, s13, v2
+; GFX1064-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX1064-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX1064-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX1064-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX1064-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX1064-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX1064-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX1064-NEXT: v_cmp_u_f32_e64 s[0:1], v0, v0
+; GFX1064-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc
+; GFX1064-NEXT: v_cndmask_b32_e64 v0, v3, v5, s[0:1]
+; GFX1064-NEXT: v_perm_b32 v0, v2, v0, 0x7060302
+; GFX1064-NEXT: v_mov_b32_e32 v3, v1
+; GFX1064-NEXT: v_mov_b32_e32 v2, v0
+; GFX1064-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
+; GFX1064-NEXT: s_waitcnt vmcnt(0)
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX1064-NEXT: v_mov_b32_e32 v1, v2
+; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_cbranch_execnz .LBB21_1
+; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1064-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1064-NEXT: s_mov_b32 s10, -1
+; GFX1064-NEXT: buffer_store_dword v2, off, s[8:11], 0
+; GFX1064-NEXT: s_endpgm
+;
+; GFX1032-LABEL: uniform_fadd_v2bf16:
+; GFX1032: ; %bb.0:
+; GFX1032-NEXT: s_clause 0x1
+; GFX1032-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX1032-NEXT: s_load_dword s0, s[4:5], 0x34
+; GFX1032-NEXT: s_mov_b32 s1, 0
+; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1032-NEXT: s_mov_b32 s6, -1
+; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT: s_load_dword s4, s[10:11], 0x0
+; GFX1032-NEXT: s_lshl_b32 s2, s0, 16
+; GFX1032-NEXT: s_and_b32 s3, s0, 0xffff0000
+; GFX1032-NEXT: s_mov_b32 s5, s11
+; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT: v_mov_b32_e32 v1, s4
+; GFX1032-NEXT: s_mov_b32 s4, s10
+; GFX1032-NEXT: .LBB21_1: ; %atomicrmw.start
+; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX1032-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX1032-NEXT: v_add_f32_e32 v0, s2, v0
+; GFX1032-NEXT: v_add_f32_e32 v2, s3, v2
+; GFX1032-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX1032-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX1032-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX1032-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX1032-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX1032-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX1032-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX1032-NEXT: v_cmp_u_f32_e64 s0, v0, v0
+; GFX1032-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
+; GFX1032-NEXT: v_cndmask_b32_e64 v0, v3, v5, s0
+; GFX1032-NEXT: v_perm_b32 v0, v2, v0, 0x7060302
+; GFX1032-NEXT: v_mov_b32_e32 v3, v1
+; GFX1032-NEXT: v_mov_b32_e32 v2, v0
+; GFX1032-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
+; GFX1032-NEXT: s_waitcnt vmcnt(0)
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX1032-NEXT: v_mov_b32_e32 v1, v2
+; GFX1032-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s1
+; GFX1032-NEXT: s_cbranch_execnz .LBB21_1
+; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX1032-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1032-NEXT: s_mov_b32 s10, -1
+; GFX1032-NEXT: buffer_store_dword v2, off, s[8:11], 0
+; GFX1032-NEXT: s_endpgm
+;
+; GFX1164-LABEL: uniform_fadd_v2bf16:
+; GFX1164: ; %bb.0:
+; GFX1164-NEXT: s_clause 0x1
+; GFX1164-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
+; GFX1164-NEXT: s_load_b32 s0, s[4:5], 0x34
+; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1164-NEXT: s_mov_b32 s6, -1
+; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-NEXT: s_load_b32 s1, s[10:11], 0x0
+; GFX1164-NEXT: s_lshl_b32 s12, s0, 16
+; GFX1164-NEXT: s_and_b32 s13, s0, 0xffff0000
+; GFX1164-NEXT: s_mov_b32 s4, s10
+; GFX1164-NEXT: s_mov_b32 s5, s11
+; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-NEXT: v_mov_b32_e32 v1, s1
+; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1
+; GFX1164-NEXT: .p2align 6
+; GFX1164-NEXT: .LBB21_1: ; %atomicrmw.start
+; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX1164-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX1164-NEXT: v_add_f32_e32 v0, s12, v0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_add_f32_e32 v2, s13, v2
+; GFX1164-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1164-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX1164-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX1164-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX1164-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX1164-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX1164-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX1164-NEXT: v_cmp_u_f32_e64 s[0:1], v0, v0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc
+; GFX1164-NEXT: v_cndmask_b32_e64 v0, v3, v5, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_perm_b32 v0, v2, v0, 0x7060302
+; GFX1164-NEXT: v_mov_b32_e32 v3, v1
+; GFX1164-NEXT: v_mov_b32_e32 v2, v0
+; GFX1164-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
+; GFX1164-NEXT: s_waitcnt vmcnt(0)
+; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX1164-NEXT: v_mov_b32_e32 v1, v2
+; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_cbranch_execnz .LBB21_1
+; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2
+; GFX1164-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1164-NEXT: s_mov_b32 s10, -1
+; GFX1164-NEXT: buffer_store_b32 v2, off, s[8:11], 0
+; GFX1164-NEXT: s_endpgm
+;
+; GFX1132-LABEL: uniform_fadd_v2bf16:
+; GFX1132: ; %bb.0:
+; GFX1132-NEXT: s_clause 0x1
+; GFX1132-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
+; GFX1132-NEXT: s_load_b32 s0, s[4:5], 0x34
+; GFX1132-NEXT: s_mov_b32 s1, 0
+; GFX1132-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1132-NEXT: s_mov_b32 s6, -1
+; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-NEXT: s_load_b32 s4, s[10:11], 0x0
+; GFX1132-NEXT: s_lshl_b32 s2, s0, 16
+; GFX1132-NEXT: s_and_b32 s3, s0, 0xffff0000
+; GFX1132-NEXT: s_mov_b32 s5, s11
+; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-NEXT: v_mov_b32_e32 v1, s4
+; GFX1132-NEXT: s_mov_b32 s4, s10
+; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1
+; GFX1132-NEXT: .p2align 6
+; GFX1132-NEXT: .LBB21_1: ; %atomicrmw.start
+; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1132-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX1132-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX1132-NEXT: v_add_f32_e32 v0, s2, v0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-NEXT: v_add_f32_e32 v2, s3, v2
+; GFX1132-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX1132-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX1132-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX1132-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX1132-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX1132-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX1132-NEXT: v_cmp_u_f32_e64 s0, v0, v0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
+; GFX1132-NEXT: v_cndmask_b32_e64 v0, v3, v5, s0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_perm_b32 v0, v2, v0, 0x7060302
+; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
+; GFX1132-NEXT: s_waitcnt vmcnt(0)
+; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX1132-NEXT: v_mov_b32_e32 v1, v2
+; GFX1132-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX1132-NEXT: s_cbranch_execnz .LBB21_1
+; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2
+; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX1132-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1132-NEXT: s_mov_b32 s10, -1
+; GFX1132-NEXT: buffer_store_b32 v2, off, s[8:11], 0
+; GFX1132-NEXT: s_endpgm
+;
+; GFX1264-LABEL: uniform_fadd_v2bf16:
+; GFX1264: ; %bb.0:
+; GFX1264-NEXT: s_clause 0x1
+; GFX1264-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
+; GFX1264-NEXT: s_load_b32 s0, s[4:5], 0x34
+; GFX1264-NEXT: s_mov_b64 s[2:3], 0
+; GFX1264-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1264-NEXT: s_mov_b32 s6, -1
+; GFX1264-NEXT: s_wait_kmcnt 0x0
+; GFX1264-NEXT: s_load_b32 s1, s[10:11], 0x0
+; GFX1264-NEXT: s_lshl_b32 s12, s0, 16
+; GFX1264-NEXT: s_and_b32 s13, s0, 0xffff0000
+; GFX1264-NEXT: s_mov_b32 s4, s10
+; GFX1264-NEXT: s_mov_b32 s5, s11
+; GFX1264-NEXT: s_wait_kmcnt 0x0
+; GFX1264-NEXT: v_mov_b32_e32 v1, s1
+; GFX1264-NEXT: .LBB21_1: ; %atomicrmw.start
+; GFX1264-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1264-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX1264-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX1264-NEXT: v_add_f32_e32 v0, s12, v0
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1264-NEXT: v_add_f32_e32 v2, s13, v2
+; GFX1264-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1264-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX1264-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX1264-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX1264-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX1264-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX1264-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX1264-NEXT: v_cmp_u_f32_e64 s[0:1], v0, v0
+; GFX1264-NEXT: s_wait_alu 0xfffd
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1264-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc
+; GFX1264-NEXT: s_wait_alu 0xf1ff
+; GFX1264-NEXT: v_cndmask_b32_e64 v0, v3, v5, s[0:1]
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1264-NEXT: v_perm_b32 v0, v2, v0, 0x7060302
+; GFX1264-NEXT: v_mov_b32_e32 v3, v1
+; GFX1264-NEXT: v_mov_b32_e32 v2, v0
+; GFX1264-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1264-NEXT: s_wait_loadcnt 0x0
+; GFX1264-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX1264-NEXT: v_mov_b32_e32 v1, v2
+; GFX1264-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1264-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1264-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1264-NEXT: s_cbranch_execnz .LBB21_1
+; GFX1264-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1264-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX1264-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1264-NEXT: s_mov_b32 s10, -1
+; GFX1264-NEXT: buffer_store_b32 v2, off, s[8:11], null
+; GFX1264-NEXT: s_endpgm
+;
+; GFX1232-LABEL: uniform_fadd_v2bf16:
+; GFX1232: ; %bb.0:
+; GFX1232-NEXT: s_clause 0x1
+; GFX1232-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
+; GFX1232-NEXT: s_load_b32 s0, s[4:5], 0x34
+; GFX1232-NEXT: s_mov_b32 s1, 0
+; GFX1232-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1232-NEXT: s_mov_b32 s6, -1
+; GFX1232-NEXT: s_wait_kmcnt 0x0
+; GFX1232-NEXT: s_load_b32 s4, s[10:11], 0x0
+; GFX1232-NEXT: s_lshl_b32 s2, s0, 16
+; GFX1232-NEXT: s_and_b32 s3, s0, 0xffff0000
+; GFX1232-NEXT: s_mov_b32 s5, s11
+; GFX1232-NEXT: s_wait_kmcnt 0x0
+; GFX1232-NEXT: v_mov_b32_e32 v1, s4
+; GFX1232-NEXT: s_mov_b32 s4, s10
+; GFX1232-NEXT: .LBB21_1: ; %atomicrmw.start
+; GFX1232-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1232-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX1232-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX1232-NEXT: v_add_f32_e32 v0, s2, v0
+; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1232-NEXT: v_add_f32_e32 v2, s3, v2
+; GFX1232-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1232-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX1232-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX1232-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX1232-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX1232-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX1232-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX1232-NEXT: v_cmp_u_f32_e64 s0, v0, v0
+; GFX1232-NEXT: s_wait_alu 0xfffd
+; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1232-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
+; GFX1232-NEXT: s_wait_alu 0xf1ff
+; GFX1232-NEXT: v_cndmask_b32_e64 v0, v3, v5, s0
+; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1232-NEXT: v_perm_b32 v0, v2, v0, 0x7060302
+; GFX1232-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1232-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1232-NEXT: s_wait_loadcnt 0x0
+; GFX1232-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX1232-NEXT: v_mov_b32_e32 v1, v2
+; GFX1232-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX1232-NEXT: s_wait_alu 0xfffe
+; GFX1232-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX1232-NEXT: s_cbranch_execnz .LBB21_1
+; GFX1232-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1232-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX1232-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1232-NEXT: s_mov_b32 s10, -1
+; GFX1232-NEXT: buffer_store_b32 v2, off, s[8:11], null
+; GFX1232-NEXT: s_endpgm
+ %rmw = atomicrmw fadd ptr addrspace(1) %uniform.ptr, <2 x bfloat> %val monotonic, align 4
+ store <2 x bfloat> %rmw, ptr addrspace(1) %result
+ ret void
+}
More information about the llvm-commits
mailing list