[llvm] f3881d6 - [AMDGPU] Generate test checks. NFC.
Jay Foad via llvm-commits
llvm-commits at lists.llvm.org
Mon Nov 2 06:01:16 PST 2020
Author: Jay Foad
Date: 2020-11-02T13:56:46Z
New Revision: f3881d6517f2ae6850037134261ad775d471f18b
URL: https://github.com/llvm/llvm-project/commit/f3881d6517f2ae6850037134261ad775d471f18b
DIFF: https://github.com/llvm/llvm-project/commit/f3881d6517f2ae6850037134261ad775d471f18b.diff
LOG: [AMDGPU] Generate test checks. NFC.
Added:
Modified:
llvm/test/CodeGen/AMDGPU/ds-sub-offset.ll
llvm/test/CodeGen/AMDGPU/ds_read2.ll
llvm/test/CodeGen/AMDGPU/ds_write2.ll
Removed:
################################################################################
diff --git a/llvm/test/CodeGen/AMDGPU/ds-sub-offset.ll b/llvm/test/CodeGen/AMDGPU/ds-sub-offset.ll
index 7487cd98e0aa..2a4ede05acd6 100644
--- a/llvm/test/CodeGen/AMDGPU/ds-sub-offset.ll
+++ b/llvm/test/CodeGen/AMDGPU/ds-sub-offset.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CI %s
; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s
@@ -5,12 +6,23 @@ declare i32 @llvm.amdgcn.workitem.id.x() #0
@lds.obj = addrspace(3) global [256 x i32] undef, align 4
-; GCN-LABEL: {{^}}write_ds_sub0_offset0_global:
-; GCN: v_lshlrev_b32_e32 [[SHL:v[0-9]+]], 2, v0
-; GCN: v_sub_{{[iu]}}32_e32 [[BASEPTR:v[0-9]+]], {{(vcc, )?}}lds.obj at abs32@lo, [[SHL]]
-; GCN: v_mov_b32_e32 [[VAL:v[0-9]+]], 0x7b
-; GCN: ds_write_b32 [[BASEPTR]], [[VAL]] offset:12
define amdgpu_kernel void @write_ds_sub0_offset0_global() #0 {
+; CI-LABEL: write_ds_sub0_offset0_global:
+; CI: ; %bb.0: ; %entry
+; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT: v_sub_i32_e32 v0, vcc, lds.obj at abs32@lo, v0
+; CI-NEXT: v_mov_b32_e32 v1, 0x7b
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: ds_write_b32 v0, v1 offset:12
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: write_ds_sub0_offset0_global:
+; GFX9: ; %bb.0: ; %entry
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: v_sub_u32_e32 v0, lds.obj at abs32@lo, v0
+; GFX9-NEXT: v_mov_b32_e32 v1, 0x7b
+; GFX9-NEXT: ds_write_b32 v0, v1 offset:12
+; GFX9-NEXT: s_endpgm
entry:
%x.i = call i32 @llvm.amdgcn.workitem.id.x() #1
%sub1 = sub i32 0, %x.i
@@ -20,10 +32,41 @@ entry:
ret void
}
-; GFX9-LABEL: {{^}}write_ds_sub0_offset0_global_clamp_bit:
-; GFX9: v_sub_u32
-; GFX9: s_endpgm
define amdgpu_kernel void @write_ds_sub0_offset0_global_clamp_bit(float %dummy.val) #0 {
+; CI-LABEL: write_ds_sub0_offset0_global_clamp_bit:
+; CI: ; %bb.0: ; %entry
+; CI-NEXT: s_load_dword s0, s[0:1], 0x9
+; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT: v_sub_i32_e32 v0, vcc, lds.obj at abs32@lo, v0
+; CI-NEXT: s_mov_b64 vcc, 0
+; CI-NEXT: v_mov_b32_e32 v2, 0x7b
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_mov_b32_e32 v1, s0
+; CI-NEXT: s_mov_b32 s0, 0
+; CI-NEXT: v_div_fmas_f32 v1, v1, v1, v1
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: s_mov_b32 s2, -1
+; CI-NEXT: s_mov_b32 s1, s0
+; CI-NEXT: ds_write_b32 v0, v2 offset:12
+; CI-NEXT: buffer_store_dword v1, off, s[0:3], 0
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: write_ds_sub0_offset0_global_clamp_bit:
+; GFX9: ; %bb.0: ; %entry
+; GFX9-NEXT: s_load_dword s0, s[0:1], 0x24
+; GFX9-NEXT: s_mov_b64 vcc, 0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: v_sub_u32_e32 v0, lds.obj at abs32@lo, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v1, s0
+; GFX9-NEXT: v_div_fmas_f32 v2, v1, v1, v1
+; GFX9-NEXT: v_mov_b32_e32 v1, 0x7b
+; GFX9-NEXT: ds_write_b32 v0, v1 offset:12
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: global_store_dword v[0:1], v2, off
+; GFX9-NEXT: s_endpgm
entry:
%x.i = call i32 @llvm.amdgcn.workitem.id.x() #1
%sub1 = sub i32 0, %x.i
@@ -35,13 +78,23 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}add_x_shl_neg_to_sub_max_offset:
-; GCN-DAG: v_lshlrev_b32_e32 [[SCALED:v[0-9]+]], 2, v0
-; CI-DAG: v_sub_i32_e32 [[NEG:v[0-9]+]], vcc, 0, [[SCALED]]
-; GFX9-DAG: v_sub_u32_e32 [[NEG:v[0-9]+]], 0, [[SCALED]]
-; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 13
-; GCN: ds_write_b8 [[NEG]], [[K]] offset:65535
define amdgpu_kernel void @add_x_shl_neg_to_sub_max_offset() #1 {
+; CI-LABEL: add_x_shl_neg_to_sub_max_offset:
+; CI: ; %bb.0:
+; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT: v_sub_i32_e32 v0, vcc, 0, v0
+; CI-NEXT: v_mov_b32_e32 v1, 13
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: ds_write_b8 v0, v1 offset:65535
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: add_x_shl_neg_to_sub_max_offset:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: v_sub_u32_e32 v0, 0, v0
+; GFX9-NEXT: v_mov_b32_e32 v1, 13
+; GFX9-NEXT: ds_write_b8 v0, v1 offset:65535
+; GFX9-NEXT: s_endpgm
%x.i = call i32 @llvm.amdgcn.workitem.id.x() #0
%neg = sub i32 0, %x.i
%shl = shl i32 %neg, 2
@@ -51,13 +104,23 @@ define amdgpu_kernel void @add_x_shl_neg_to_sub_max_offset() #1 {
ret void
}
-; GCN-LABEL: {{^}}add_x_shl_neg_to_sub_max_offset_p1:
-; GCN-DAG: v_lshlrev_b32_e32 [[SCALED:v[0-9]+]], 2, v0
-; CI-DAG: v_sub_i32_e32 [[NEG:v[0-9]+]], vcc, 0x10000, [[SCALED]]
-; GFX9-DAG: v_sub_u32_e32 [[NEG:v[0-9]+]], 0x10000, [[SCALED]]
-; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 13
-; GCN: ds_write_b8 [[NEG]], [[K]]{{$}}
define amdgpu_kernel void @add_x_shl_neg_to_sub_max_offset_p1() #1 {
+; CI-LABEL: add_x_shl_neg_to_sub_max_offset_p1:
+; CI: ; %bb.0:
+; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT: v_sub_i32_e32 v0, vcc, 0x10000, v0
+; CI-NEXT: v_mov_b32_e32 v1, 13
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: ds_write_b8 v0, v1
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: add_x_shl_neg_to_sub_max_offset_p1:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: v_sub_u32_e32 v0, 0x10000, v0
+; GFX9-NEXT: v_mov_b32_e32 v1, 13
+; GFX9-NEXT: ds_write_b8 v0, v1
+; GFX9-NEXT: s_endpgm
%x.i = call i32 @llvm.amdgcn.workitem.id.x() #0
%neg = sub i32 0, %x.i
%shl = shl i32 %neg, 2
@@ -67,17 +130,25 @@ define amdgpu_kernel void @add_x_shl_neg_to_sub_max_offset_p1() #1 {
ret void
}
-; GCN-LABEL: {{^}}add_x_shl_neg_to_sub_multi_use:
-; GCN-DAG: v_lshlrev_b32_e32 [[SCALED:v[0-9]+]], 2, v0
-; CI-DAG: v_sub_i32_e32 [[NEG:v[0-9]+]], vcc, 0, [[SCALED]]
-; GFX9-DAG: v_sub_u32_e32 [[NEG:v[0-9]+]], 0, [[SCALED]]
-; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 13
-; GCN-NOT: v_sub
-; GCN: ds_write_b32 [[NEG]], [[K]] offset:123{{$}}
-; GCN-NOT: v_sub
-; GCN: ds_write_b32 [[NEG]], [[K]] offset:456{{$}}
-; GCN: s_endpgm
define amdgpu_kernel void @add_x_shl_neg_to_sub_multi_use() #1 {
+; CI-LABEL: add_x_shl_neg_to_sub_multi_use:
+; CI: ; %bb.0:
+; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT: v_sub_i32_e32 v0, vcc, 0, v0
+; CI-NEXT: v_mov_b32_e32 v1, 13
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: ds_write_b32 v0, v1 offset:123
+; CI-NEXT: ds_write_b32 v0, v1 offset:456
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: add_x_shl_neg_to_sub_multi_use:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: v_sub_u32_e32 v0, 0, v0
+; GFX9-NEXT: v_mov_b32_e32 v1, 13
+; GFX9-NEXT: ds_write_b32 v0, v1 offset:123
+; GFX9-NEXT: ds_write_b32 v0, v1 offset:456
+; GFX9-NEXT: s_endpgm
%x.i = call i32 @llvm.amdgcn.workitem.id.x() #0
%neg = sub i32 0, %x.i
%shl = shl i32 %neg, 2
@@ -90,17 +161,25 @@ define amdgpu_kernel void @add_x_shl_neg_to_sub_multi_use() #1 {
ret void
}
-; GCN-LABEL: {{^}}add_x_shl_neg_to_sub_multi_use_same_offset:
-; GCN-DAG: v_lshlrev_b32_e32 [[SCALED:v[0-9]+]], 2, v0
-; CI-DAG: v_sub_i32_e32 [[NEG:v[0-9]+]], vcc, 0, [[SCALED]]
-; GFX9-DAG: v_sub_u32_e32 [[NEG:v[0-9]+]], 0, [[SCALED]]
-; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 13
-; GCN-NOT: v_sub
-; GCN: ds_write_b32 [[NEG]], [[K]] offset:123{{$}}
-; GCN-NOT: v_sub
-; GCN: ds_write_b32 [[NEG]], [[K]] offset:123{{$}}
-; GCN: s_endpgm
define amdgpu_kernel void @add_x_shl_neg_to_sub_multi_use_same_offset() #1 {
+; CI-LABEL: add_x_shl_neg_to_sub_multi_use_same_offset:
+; CI: ; %bb.0:
+; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT: v_sub_i32_e32 v0, vcc, 0, v0
+; CI-NEXT: v_mov_b32_e32 v1, 13
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: ds_write_b32 v0, v1 offset:123
+; CI-NEXT: ds_write_b32 v0, v1 offset:123
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: add_x_shl_neg_to_sub_multi_use_same_offset:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: v_sub_u32_e32 v0, 0, v0
+; GFX9-NEXT: v_mov_b32_e32 v1, 13
+; GFX9-NEXT: ds_write_b32 v0, v1 offset:123
+; GFX9-NEXT: ds_write_b32 v0, v1 offset:123
+; GFX9-NEXT: s_endpgm
%x.i = call i32 @llvm.amdgcn.workitem.id.x() #0
%neg = sub i32 0, %x.i
%shl = shl i32 %neg, 2
@@ -111,12 +190,25 @@ define amdgpu_kernel void @add_x_shl_neg_to_sub_multi_use_same_offset() #1 {
ret void
}
-; GCN-LABEL: {{^}}add_x_shl_neg_to_sub_misaligned_i64_max_offset:
-; GCN-DAG: v_lshlrev_b32_e32 [[SCALED:v[0-9]+]], 2, v0
-; CI-DAG: v_sub_i32_e32 [[NEG:v[0-9]+]], vcc, 0, [[SCALED]]
-; GFX9-DAG: v_sub_u32_e32 [[NEG:v[0-9]+]], 0, [[SCALED]]
-; GCN: ds_write2_b32 [[NEG]], {{v[0-9]+}}, {{v[0-9]+}} offset0:254 offset1:255
define amdgpu_kernel void @add_x_shl_neg_to_sub_misaligned_i64_max_offset() #1 {
+; CI-LABEL: add_x_shl_neg_to_sub_misaligned_i64_max_offset:
+; CI: ; %bb.0:
+; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT: v_sub_i32_e32 v0, vcc, 0, v0
+; CI-NEXT: v_mov_b32_e32 v1, 0x7b
+; CI-NEXT: v_mov_b32_e32 v2, 0
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: ds_write2_b32 v0, v1, v2 offset0:254 offset1:255
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: add_x_shl_neg_to_sub_misaligned_i64_max_offset:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: v_sub_u32_e32 v0, 0, v0
+; GFX9-NEXT: v_mov_b32_e32 v1, 0x7b
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: ds_write2_b32 v0, v1, v2 offset0:254 offset1:255
+; GFX9-NEXT: s_endpgm
%x.i = call i32 @llvm.amdgcn.workitem.id.x() #0
%neg = sub i32 0, %x.i
%shl = shl i32 %neg, 2
@@ -126,10 +218,43 @@ define amdgpu_kernel void @add_x_shl_neg_to_sub_misaligned_i64_max_offset() #1 {
ret void
}
-; GFX9-LABEL: {{^}}add_x_shl_neg_to_sub_misaligned_i64_max_offset_clamp_bit:
-; GFX9: v_sub_u32
-; GFX9: s_endpgm
define amdgpu_kernel void @add_x_shl_neg_to_sub_misaligned_i64_max_offset_clamp_bit(float %dummy.val) #1 {
+; CI-LABEL: add_x_shl_neg_to_sub_misaligned_i64_max_offset_clamp_bit:
+; CI: ; %bb.0:
+; CI-NEXT: s_load_dword s0, s[0:1], 0x9
+; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT: v_sub_i32_e32 v0, vcc, 0, v0
+; CI-NEXT: s_mov_b64 vcc, 0
+; CI-NEXT: v_mov_b32_e32 v2, 0x7b
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_mov_b32_e32 v1, s0
+; CI-NEXT: s_mov_b32 s0, 0
+; CI-NEXT: v_div_fmas_f32 v1, v1, v1, v1
+; CI-NEXT: v_mov_b32_e32 v3, 0
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: s_mov_b32 s2, -1
+; CI-NEXT: s_mov_b32 s1, s0
+; CI-NEXT: ds_write2_b32 v0, v2, v3 offset0:254 offset1:255
+; CI-NEXT: buffer_store_dword v1, off, s[0:3], 0
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: add_x_shl_neg_to_sub_misaligned_i64_max_offset_clamp_bit:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dword s0, s[0:1], 0x24
+; GFX9-NEXT: s_mov_b64 vcc, 0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: v_sub_u32_e32 v0, 0, v0
+; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v1, s0
+; GFX9-NEXT: v_div_fmas_f32 v2, v1, v1, v1
+; GFX9-NEXT: v_mov_b32_e32 v1, 0x7b
+; GFX9-NEXT: ds_write2_b32 v0, v1, v3 offset0:254 offset1:255
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: global_store_dword v[0:1], v2, off
+; GFX9-NEXT: s_endpgm
%x.i = call i32 @llvm.amdgcn.workitem.id.x() #0
%neg = sub i32 0, %x.i
%shl = shl i32 %neg, 2
@@ -141,12 +266,25 @@ define amdgpu_kernel void @add_x_shl_neg_to_sub_misaligned_i64_max_offset_clamp_
ret void
}
-; GCN-LABEL: {{^}}add_x_shl_neg_to_sub_misaligned_i64_max_offset_p1:
-; GCN-DAG: v_lshlrev_b32_e32 [[SCALED:v[0-9]+]], 2, v0
-; CI-DAG: v_sub_i32_e32 [[NEG:v[0-9]+]], vcc, 0x3fc, [[SCALED]]
-; GFX9-DAG: v_sub_u32_e32 [[NEG:v[0-9]+]], 0x3fc, [[SCALED]]
-; GCN: ds_write2_b32 [[NEG]], {{v[0-9]+}}, {{v[0-9]+}} offset1:1{{$}}
define amdgpu_kernel void @add_x_shl_neg_to_sub_misaligned_i64_max_offset_p1() #1 {
+; CI-LABEL: add_x_shl_neg_to_sub_misaligned_i64_max_offset_p1:
+; CI: ; %bb.0:
+; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT: v_sub_i32_e32 v0, vcc, 0x3fc, v0
+; CI-NEXT: v_mov_b32_e32 v1, 0x7b
+; CI-NEXT: v_mov_b32_e32 v2, 0
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: ds_write2_b32 v0, v1, v2 offset1:1
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: add_x_shl_neg_to_sub_misaligned_i64_max_offset_p1:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: v_sub_u32_e32 v0, 0x3fc, v0
+; GFX9-NEXT: v_mov_b32_e32 v1, 0x7b
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: ds_write2_b32 v0, v1, v2 offset1:1
+; GFX9-NEXT: s_endpgm
%x.i = call i32 @llvm.amdgcn.workitem.id.x() #0
%neg = sub i32 0, %x.i
%shl = shl i32 %neg, 2
diff --git a/llvm/test/CodeGen/AMDGPU/ds_read2.ll b/llvm/test/CodeGen/AMDGPU/ds_read2.ll
index d01d41955449..eb445a3d095a 100644
--- a/llvm/test/CodeGen/AMDGPU/ds_read2.ll
+++ b/llvm/test/CodeGen/AMDGPU/ds_read2.ll
@@ -1,6 +1,7 @@
-; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -mattr=+load-store-opt < %s | FileCheck -enable-var-scope -strict-whitespace -check-prefixes=GCN,CI %s
-; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -mattr=+load-store-opt,+flat-for-global,-unaligned-access-mode < %s | FileCheck -enable-var-scope -strict-whitespace -check-prefixes=GCN,GFX9,GFX9-ALIGNED %s
-; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -mattr=+load-store-opt,+flat-for-global,+unaligned-access-mode < %s | FileCheck -enable-var-scope -strict-whitespace -check-prefixes=GCN,GFX9,GFX9-UNALIGNED %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -mattr=+load-store-opt < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CI %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -mattr=+load-store-opt,-unaligned-access-mode < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9,GFX9-ALIGNED %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -mattr=+load-store-opt,+unaligned-access-mode < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9,GFX9-UNALIGNED %s
; FIXME: We don't get cases where the address was an SGPR because we
; get a copy to the address register for each one.
@@ -8,17 +9,32 @@
@lds = addrspace(3) global [512 x float] undef, align 4
@lds.f64 = addrspace(3) global [512 x double] undef, align 8
-; GCN-LABEL: {{^}}simple_read2_f32:
-; CI-DAG: s_mov_b32 m0
-; GFX9-NOT: m0
-
-; GCN: ds_read2_b32 v{{\[}}[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]{{\]}}, v{{[0-9]+}} offset1:8
-; GCN: s_waitcnt lgkmcnt(0)
-; GCN: v_add_f32_e32 [[RESULT:v[0-9]+]], v[[LO_VREG]], v[[HI_VREG]]
-; CI: buffer_store_dword [[RESULT]]
-; GFX9: global_store_dword v{{[0-9]+}}, [[RESULT]], s{{\[[0-9]+:[0-9]+\]}}
-; GCN: s_endpgm
define amdgpu_kernel void @simple_read2_f32(float addrspace(1)* %out) #0 {
+; CI-LABEL: simple_read2_f32:
+; CI: ; %bb.0:
+; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT: v_add_i32_e32 v1, vcc, lds at abs32@lo, v0
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: ds_read2_b32 v[1:2], v1 offset1:8
+; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9
+; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: s_mov_b32 s2, 0
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_add_f32_e32 v2, v1, v2
+; CI-NEXT: v_mov_b32_e32 v1, 0
+; CI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: simple_read2_f32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; GFX9-NEXT: v_add_u32_e32 v0, lds at abs32@lo, v2
+; GFX9-NEXT: ds_read2_b32 v[0:1], v0 offset1:8
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: global_store_dword v2, v0, s[0:1]
+; GFX9-NEXT: s_endpgm
%x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
%arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %x.i
%val0 = load float, float addrspace(3)* %arrayidx0, align 4
@@ -31,17 +47,32 @@ define amdgpu_kernel void @simple_read2_f32(float addrspace(1)* %out) #0 {
ret void
}
-; GCN-LABEL: {{^}}simple_read2_f32_max_offset:
-; CI-DAG: s_mov_b32 m0
-; GFX9-NOT: m0
-
-; GCN: ds_read2_b32 v{{\[}}[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]{{\]}}, v{{[0-9]+}} offset1:255
-; GCN: s_waitcnt lgkmcnt(0)
-; GCN: v_add_f32_e32 [[RESULT:v[0-9]+]], v[[LO_VREG]], v[[HI_VREG]]
-
-; CI: buffer_store_dword [[RESULT]]
-; GFX9: global_store_dword v{{[0-9]+}}, [[RESULT]], s{{\[[0-9]+:[0-9]+\]}}
define amdgpu_kernel void @simple_read2_f32_max_offset(float addrspace(1)* %out) #0 {
+; CI-LABEL: simple_read2_f32_max_offset:
+; CI: ; %bb.0:
+; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT: v_add_i32_e32 v1, vcc, lds at abs32@lo, v0
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: ds_read2_b32 v[1:2], v1 offset1:255
+; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9
+; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: s_mov_b32 s2, 0
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_add_f32_e32 v2, v1, v2
+; CI-NEXT: v_mov_b32_e32 v1, 0
+; CI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: simple_read2_f32_max_offset:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; GFX9-NEXT: v_add_u32_e32 v0, lds at abs32@lo, v2
+; GFX9-NEXT: ds_read2_b32 v[0:1], v0 offset1:255
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: global_store_dword v2, v0, s[0:1]
+; GFX9-NEXT: s_endpgm
%x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
%arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %x.i
%val0 = load float, float addrspace(3)* %arrayidx0, align 4
@@ -54,15 +85,34 @@ define amdgpu_kernel void @simple_read2_f32_max_offset(float addrspace(1)* %out)
ret void
}
-; GCN-LABEL: @simple_read2_f32_too_far
-; CI-DAG: s_mov_b32 m0
-; GFX9-NOT: m0
-
-; GCN-NOT: ds_read2_b32
-; GCN: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}}
-; GCN: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}} offset:1028
-; GCN: s_endpgm
define amdgpu_kernel void @simple_read2_f32_too_far(float addrspace(1)* %out) #0 {
+; CI-LABEL: simple_read2_f32_too_far:
+; CI: ; %bb.0:
+; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT: v_add_i32_e32 v1, vcc, lds at abs32@lo, v0
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: ds_read_b32 v2, v1
+; CI-NEXT: ds_read_b32 v1, v1 offset:1028
+; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9
+; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: s_mov_b32 s2, 0
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_add_f32_e32 v2, v2, v1
+; CI-NEXT: v_mov_b32_e32 v1, 0
+; CI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: simple_read2_f32_too_far:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: v_add_u32_e32 v1, lds at abs32@lo, v0
+; GFX9-NEXT: ds_read_b32 v2, v1
+; GFX9-NEXT: ds_read_b32 v1, v1 offset:1028
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_f32_e32 v1, v2, v1
+; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX9-NEXT: s_endpgm
%x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
%arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %x.i
%val0 = load float, float addrspace(3)* %arrayidx0, align 4
@@ -75,14 +125,39 @@ define amdgpu_kernel void @simple_read2_f32_too_far(float addrspace(1)* %out) #0
ret void
}
-; GCN-LABEL: @simple_read2_f32_x2
-; CI-DAG: s_mov_b32 m0
-; GFX9-NOT: m0
-
-; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASEADDR:v[0-9]+]] offset1:8
-; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASEADDR]] offset0:11 offset1:27
-; GCN: s_endpgm
define amdgpu_kernel void @simple_read2_f32_x2(float addrspace(1)* %out) #0 {
+; CI-LABEL: simple_read2_f32_x2:
+; CI: ; %bb.0:
+; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT: v_add_i32_e32 v3, vcc, lds at abs32@lo, v0
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: ds_read2_b32 v[1:2], v3 offset1:8
+; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9
+; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: s_mov_b32 s2, 0
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_add_f32_e32 v4, v1, v2
+; CI-NEXT: ds_read2_b32 v[1:2], v3 offset0:11 offset1:27
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_add_f32_e32 v1, v1, v2
+; CI-NEXT: v_add_f32_e32 v2, v4, v1
+; CI-NEXT: v_mov_b32_e32 v1, 0
+; CI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: simple_read2_f32_x2:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 2, v0
+; GFX9-NEXT: v_add_u32_e32 v2, lds at abs32@lo, v4
+; GFX9-NEXT: ds_read2_b32 v[0:1], v2 offset1:8
+; GFX9-NEXT: ds_read2_b32 v[2:3], v2 offset0:11 offset1:27
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_add_f32_e32 v1, v2, v3
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: global_store_dword v4, v0, s[0:1]
+; GFX9-NEXT: s_endpgm
%tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1
%idx.0 = add nsw i32 %tid.x, 0
%arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.0
@@ -109,15 +184,42 @@ define amdgpu_kernel void @simple_read2_f32_x2(float addrspace(1)* %out) #0 {
}
; Make sure there is an instruction between the two sets of reads.
-; GCN-LABEL: @simple_read2_f32_x2_barrier
-; CI-DAG: s_mov_b32 m0
-; GFX9-NOT: m0
-
-; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASEADDR:v[0-9]+]] offset1:8
-; GCN: s_barrier
-; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASEADDR]] offset0:11 offset1:27
-; GCN: s_endpgm
define amdgpu_kernel void @simple_read2_f32_x2_barrier(float addrspace(1)* %out) #0 {
+; CI-LABEL: simple_read2_f32_x2_barrier:
+; CI: ; %bb.0:
+; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT: v_add_i32_e32 v3, vcc, lds at abs32@lo, v0
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: ds_read2_b32 v[1:2], v3 offset1:8
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: s_barrier
+; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9
+; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: s_mov_b32 s2, 0
+; CI-NEXT: v_add_f32_e32 v4, v1, v2
+; CI-NEXT: ds_read2_b32 v[1:2], v3 offset0:11 offset1:27
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_add_f32_e32 v1, v1, v2
+; CI-NEXT: v_add_f32_e32 v2, v4, v1
+; CI-NEXT: v_mov_b32_e32 v1, 0
+; CI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: simple_read2_f32_x2_barrier:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 2, v0
+; GFX9-NEXT: v_add_u32_e32 v2, lds at abs32@lo, v4
+; GFX9-NEXT: ds_read2_b32 v[0:1], v2 offset1:8
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_barrier
+; GFX9-NEXT: ds_read2_b32 v[2:3], v2 offset0:11 offset1:27
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_f32_e32 v1, v2, v3
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: global_store_dword v4, v0, s[0:1]
+; GFX9-NEXT: s_endpgm
%tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1
%idx.0 = add nsw i32 %tid.x, 0
%arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.0
@@ -147,15 +249,39 @@ define amdgpu_kernel void @simple_read2_f32_x2_barrier(float addrspace(1)* %out)
; For some reason adding something to the base address for the first
; element results in only folding the inner pair.
-
-; GCN-LABEL: @simple_read2_f32_x2_nonzero_base
-; CI-DAG: s_mov_b32 m0
-; GFX9-NOT: m0
-
-; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASEADDR:v[0-9]+]] offset0:2 offset1:8
-; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASEADDR]] offset0:11 offset1:27
-; GCN: s_endpgm
define amdgpu_kernel void @simple_read2_f32_x2_nonzero_base(float addrspace(1)* %out) #0 {
+; CI-LABEL: simple_read2_f32_x2_nonzero_base:
+; CI: ; %bb.0:
+; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT: v_add_i32_e32 v3, vcc, lds at abs32@lo, v0
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: ds_read2_b32 v[1:2], v3 offset0:2 offset1:8
+; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9
+; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: s_mov_b32 s2, 0
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_add_f32_e32 v4, v1, v2
+; CI-NEXT: ds_read2_b32 v[1:2], v3 offset0:11 offset1:27
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_add_f32_e32 v1, v1, v2
+; CI-NEXT: v_add_f32_e32 v2, v4, v1
+; CI-NEXT: v_mov_b32_e32 v1, 0
+; CI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 offset:8
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: simple_read2_f32_x2_nonzero_base:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 2, v0
+; GFX9-NEXT: v_add_u32_e32 v2, lds at abs32@lo, v4
+; GFX9-NEXT: ds_read2_b32 v[0:1], v2 offset0:2 offset1:8
+; GFX9-NEXT: ds_read2_b32 v[2:3], v2 offset0:11 offset1:27
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_add_f32_e32 v1, v2, v3
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: global_store_dword v4, v0, s[0:1] offset:8
+; GFX9-NEXT: s_endpgm
%tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1
%idx.0 = add nsw i32 %tid.x, 2
%arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.0
@@ -186,16 +312,40 @@ define amdgpu_kernel void @simple_read2_f32_x2_nonzero_base(float addrspace(1)*
; merge.
; Base pointers come from
diff erent subregister of same super
; register. We can't safely merge this.
-
-; GCN-LABEL: @read2_ptr_is_subreg_arg_f32
-; CI-DAG: s_mov_b32 m0
-; GFX9-NOT: m0
-
-; GCN-NOT: ds_read2_b32
-; GCN: ds_read_b32
-; GCN: ds_read_b32
-; GCN: s_endpgm
define amdgpu_kernel void @read2_ptr_is_subreg_arg_f32(float addrspace(1)* %out, <2 x float addrspace(3)*> %lds.ptr) #0 {
+; CI-LABEL: read2_ptr_is_subreg_arg_f32:
+; CI: ; %bb.0:
+; CI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9
+; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xb
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: s_mov_b32 s7, 0xf000
+; CI-NEXT: s_mov_b32 s6, 0
+; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_mov_b32_e32 v1, s0
+; CI-NEXT: v_mov_b32_e32 v2, s1
+; CI-NEXT: ds_read_b32 v1, v1 offset:32
+; CI-NEXT: ds_read_b32 v2, v2
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_add_f32_e32 v2, v1, v2
+; CI-NEXT: v_mov_b32_e32 v1, 0
+; CI-NEXT: buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: read2_ptr_is_subreg_arg_f32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2c
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v1, s0
+; GFX9-NEXT: v_mov_b32_e32 v2, s1
+; GFX9-NEXT: ds_read_b32 v1, v1 offset:32
+; GFX9-NEXT: ds_read_b32 v2, v2
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX9-NEXT: global_store_dword v0, v1, s[2:3]
+; GFX9-NEXT: s_endpgm
%x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
%index.0 = insertelement <2 x i32> undef, i32 %x.i, i32 0
%index.1 = insertelement <2 x i32> %index.0, i32 8, i32 0
@@ -215,16 +365,40 @@ define amdgpu_kernel void @read2_ptr_is_subreg_arg_f32(float addrspace(1)* %out,
; are rejecting merges that have the same, constant 0 offset, so make
; sure we are really rejecting it because of the
diff erent
; subregisters.
-
-; GCN-LABEL: @read2_ptr_is_subreg_arg_offset_f32
-; CI-DAG: s_mov_b32 m0
-; GFX9-NOT: m0
-
-; GCN-NOT: ds_read2_b32
-; GCN: ds_read_b32
-; GCN: ds_read_b32
-; GCN: s_endpgm
define amdgpu_kernel void @read2_ptr_is_subreg_arg_offset_f32(float addrspace(1)* %out, <2 x float addrspace(3)*> %lds.ptr) #0 {
+; CI-LABEL: read2_ptr_is_subreg_arg_offset_f32:
+; CI: ; %bb.0:
+; CI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9
+; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xb
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: s_mov_b32 s7, 0xf000
+; CI-NEXT: s_mov_b32 s6, 0
+; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_mov_b32_e32 v1, s0
+; CI-NEXT: v_mov_b32_e32 v2, s1
+; CI-NEXT: ds_read_b32 v1, v1 offset:32
+; CI-NEXT: ds_read_b32 v2, v2 offset:32
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_add_f32_e32 v2, v1, v2
+; CI-NEXT: v_mov_b32_e32 v1, 0
+; CI-NEXT: buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: read2_ptr_is_subreg_arg_offset_f32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2c
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v1, s0
+; GFX9-NEXT: v_mov_b32_e32 v2, s1
+; GFX9-NEXT: ds_read_b32 v1, v1 offset:32
+; GFX9-NEXT: ds_read_b32 v2, v2 offset:32
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX9-NEXT: global_store_dword v0, v1, s[2:3]
+; GFX9-NEXT: s_endpgm
%x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
%index.0 = insertelement <2 x i32> undef, i32 %x.i, i32 0
%index.1 = insertelement <2 x i32> %index.0, i32 8, i32 0
@@ -244,13 +418,32 @@ define amdgpu_kernel void @read2_ptr_is_subreg_arg_offset_f32(float addrspace(1)
ret void
}
-; GCN-LABEL: {{^}}read2_ptr_is_subreg_f32:
-; CI-DAG: s_mov_b32 m0
-; GFX9-NOT: m0
-
-; GCN: ds_read2_b32 {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}} offset1:8{{$}}
-; GCN: s_endpgm
define amdgpu_kernel void @read2_ptr_is_subreg_f32(float addrspace(1)* %out) #0 {
+; CI-LABEL: read2_ptr_is_subreg_f32:
+; CI: ; %bb.0:
+; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT: v_add_i32_e32 v1, vcc, lds at abs32@lo, v0
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: ds_read2_b32 v[1:2], v1 offset1:8
+; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9
+; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: s_mov_b32 s2, 0
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_add_f32_e32 v2, v1, v2
+; CI-NEXT: v_mov_b32_e32 v1, 0
+; CI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: read2_ptr_is_subreg_f32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; GFX9-NEXT: v_add_u32_e32 v0, lds at abs32@lo, v2
+; GFX9-NEXT: ds_read2_b32 v[0:1], v0 offset1:8
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: global_store_dword v2, v0, s[0:1]
+; GFX9-NEXT: s_endpgm
%x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
%ptr.0 = insertelement <2 x [512 x float] addrspace(3)*> undef, [512 x float] addrspace(3)* @lds, i32 0
%ptr.1 = insertelement <2 x [512 x float] addrspace(3)*> %ptr.0, [512 x float] addrspace(3)* @lds, i32 1
@@ -269,15 +462,34 @@ define amdgpu_kernel void @read2_ptr_is_subreg_f32(float addrspace(1)* %out) #0
ret void
}
-; GCN-LABEL: @simple_read2_f32_volatile_0
-; CI-DAG: s_mov_b32 m0
-; GFX9-NOT: m0
-
-; GCN-NOT: ds_read2_b32
-; GCN: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}}
-; GCN: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}} offset:32
-; GCN: s_endpgm
define amdgpu_kernel void @simple_read2_f32_volatile_0(float addrspace(1)* %out) #0 {
+; CI-LABEL: simple_read2_f32_volatile_0:
+; CI: ; %bb.0:
+; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT: v_add_i32_e32 v1, vcc, lds at abs32@lo, v0
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: ds_read_b32 v2, v1
+; CI-NEXT: ds_read_b32 v1, v1 offset:32
+; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9
+; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: s_mov_b32 s2, 0
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_add_f32_e32 v2, v2, v1
+; CI-NEXT: v_mov_b32_e32 v1, 0
+; CI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: simple_read2_f32_volatile_0:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: v_add_u32_e32 v1, lds at abs32@lo, v0
+; GFX9-NEXT: ds_read_b32 v2, v1
+; GFX9-NEXT: ds_read_b32 v1, v1 offset:32
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_f32_e32 v1, v2, v1
+; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX9-NEXT: s_endpgm
%x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
%arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %x.i
%val0 = load volatile float, float addrspace(3)* %arrayidx0, align 4
@@ -290,15 +502,34 @@ define amdgpu_kernel void @simple_read2_f32_volatile_0(float addrspace(1)* %out)
ret void
}
-; GCN-LABEL: @simple_read2_f32_volatile_1
-; CI-DAG: s_mov_b32 m0
-; GFX9-NOT: m0
-
-; GCN-NOT: ds_read2_b32
-; GCN: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}}
-; GCN: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}} offset:32
-; GCN: s_endpgm
define amdgpu_kernel void @simple_read2_f32_volatile_1(float addrspace(1)* %out) #0 {
+; CI-LABEL: simple_read2_f32_volatile_1:
+; CI: ; %bb.0:
+; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT: v_add_i32_e32 v1, vcc, lds at abs32@lo, v0
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: ds_read_b32 v2, v1
+; CI-NEXT: ds_read_b32 v1, v1 offset:32
+; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9
+; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: s_mov_b32 s2, 0
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_add_f32_e32 v2, v2, v1
+; CI-NEXT: v_mov_b32_e32 v1, 0
+; CI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: simple_read2_f32_volatile_1:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: v_add_u32_e32 v1, lds at abs32@lo, v0
+; GFX9-NEXT: ds_read_b32 v2, v1
+; GFX9-NEXT: ds_read_b32 v1, v1 offset:32
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_f32_e32 v1, v2, v1
+; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX9-NEXT: s_endpgm
%x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
%arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %x.i
%val0 = load float, float addrspace(3)* %arrayidx0, align 4
@@ -312,15 +543,88 @@ define amdgpu_kernel void @simple_read2_f32_volatile_1(float addrspace(1)* %out)
}
; Can't fold since not correctly aligned.
-; GCN-LABEL: @unaligned_read2_f32
-; CI-DAG: s_mov_b32 m0
-; GFX9-NOT: m0
-
-; CI-COUNT-4: ds_read_u8
-; GFX9-ALIGNED-4: ds_read_u8
-; GFX9-UNALIGNED-4: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}}}
-; GCN: s_endpgm
define amdgpu_kernel void @unaligned_read2_f32(float addrspace(1)* %out, float addrspace(3)* %lds) #0 {
+; CI-LABEL: unaligned_read2_f32:
+; CI: ; %bb.0:
+; CI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9
+; CI-NEXT: s_load_dword s0, s[0:1], 0xb
+; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: s_mov_b32 s7, 0xf000
+; CI-NEXT: s_mov_b32 s6, 0
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_add_i32_e32 v1, vcc, s0, v0
+; CI-NEXT: ds_read_u8 v2, v1 offset:35
+; CI-NEXT: ds_read_u8 v3, v1 offset:34
+; CI-NEXT: ds_read_u8 v4, v1 offset:33
+; CI-NEXT: ds_read_u8 v5, v1 offset:32
+; CI-NEXT: ds_read_u8 v6, v1 offset:3
+; CI-NEXT: ds_read_u8 v7, v1 offset:2
+; CI-NEXT: ds_read_u8 v8, v1 offset:1
+; CI-NEXT: ds_read_u8 v1, v1
+; CI-NEXT: s_waitcnt lgkmcnt(7)
+; CI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; CI-NEXT: s_waitcnt lgkmcnt(3)
+; CI-NEXT: v_lshlrev_b32_e32 v6, 8, v6
+; CI-NEXT: s_waitcnt lgkmcnt(2)
+; CI-NEXT: v_or_b32_e32 v6, v6, v7
+; CI-NEXT: v_lshlrev_b32_e32 v4, 8, v4
+; CI-NEXT: v_or_b32_e32 v2, v2, v3
+; CI-NEXT: s_waitcnt lgkmcnt(1)
+; CI-NEXT: v_lshlrev_b32_e32 v8, 8, v8
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_or_b32_e32 v1, v8, v1
+; CI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; CI-NEXT: v_or_b32_e32 v4, v4, v5
+; CI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; CI-NEXT: v_or_b32_e32 v1, v6, v1
+; CI-NEXT: v_or_b32_e32 v2, v2, v4
+; CI-NEXT: v_add_f32_e32 v2, v1, v2
+; CI-NEXT: v_mov_b32_e32 v1, 0
+; CI-NEXT: buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
+; CI-NEXT: s_endpgm
+;
+; GFX9-ALIGNED-LABEL: unaligned_read2_f32:
+; GFX9-ALIGNED: ; %bb.0:
+; GFX9-ALIGNED-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
+; GFX9-ALIGNED-NEXT: s_load_dword s0, s[0:1], 0x2c
+; GFX9-ALIGNED-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-ALIGNED-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-ALIGNED-NEXT: v_add_u32_e32 v1, s0, v0
+; GFX9-ALIGNED-NEXT: ds_read_u8 v2, v1
+; GFX9-ALIGNED-NEXT: ds_read_u8 v3, v1 offset:1
+; GFX9-ALIGNED-NEXT: ds_read_u8 v4, v1 offset:2
+; GFX9-ALIGNED-NEXT: ds_read_u8 v5, v1 offset:3
+; GFX9-ALIGNED-NEXT: ds_read_u8 v6, v1 offset:32
+; GFX9-ALIGNED-NEXT: ds_read_u8 v7, v1 offset:33
+; GFX9-ALIGNED-NEXT: ds_read_u8 v8, v1 offset:34
+; GFX9-ALIGNED-NEXT: ds_read_u8 v1, v1 offset:35
+; GFX9-ALIGNED-NEXT: s_waitcnt lgkmcnt(6)
+; GFX9-ALIGNED-NEXT: v_lshl_or_b32 v2, v3, 8, v2
+; GFX9-ALIGNED-NEXT: s_waitcnt lgkmcnt(4)
+; GFX9-ALIGNED-NEXT: v_lshl_or_b32 v3, v5, 8, v4
+; GFX9-ALIGNED-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX9-ALIGNED-NEXT: s_waitcnt lgkmcnt(2)
+; GFX9-ALIGNED-NEXT: v_lshl_or_b32 v3, v7, 8, v6
+; GFX9-ALIGNED-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-ALIGNED-NEXT: v_lshl_or_b32 v1, v1, 8, v8
+; GFX9-ALIGNED-NEXT: v_lshl_or_b32 v1, v1, 16, v3
+; GFX9-ALIGNED-NEXT: v_add_f32_e32 v1, v2, v1
+; GFX9-ALIGNED-NEXT: global_store_dword v0, v1, s[2:3]
+; GFX9-ALIGNED-NEXT: s_endpgm
+;
+; GFX9-UNALIGNED-LABEL: unaligned_read2_f32:
+; GFX9-UNALIGNED: ; %bb.0:
+; GFX9-UNALIGNED-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
+; GFX9-UNALIGNED-NEXT: s_load_dword s0, s[0:1], 0x2c
+; GFX9-UNALIGNED-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; GFX9-UNALIGNED-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-UNALIGNED-NEXT: v_add_u32_e32 v0, s0, v2
+; GFX9-UNALIGNED-NEXT: ds_read2_b32 v[0:1], v0 offset1:8
+; GFX9-UNALIGNED-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-UNALIGNED-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-UNALIGNED-NEXT: global_store_dword v2, v0, s[2:3]
+; GFX9-UNALIGNED-NEXT: s_endpgm
%x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
%arrayidx0 = getelementptr inbounds float, float addrspace(3)* %lds, i32 %x.i
%val0 = load float, float addrspace(3)* %arrayidx0, align 1
@@ -333,15 +637,65 @@ define amdgpu_kernel void @unaligned_read2_f32(float addrspace(1)* %out, float a
ret void
}
-; GCN-LABEL: @misaligned_2_simple_read2_f32
-; CI-DAG: s_mov_b32 m0
-; GFX9-NOT: m0
-
-; CI-COUNT-2: ds_read_u16
-; GFX9-ALIGNED-2: ds_read_u16
-; GFX9-UNALIGNED-4: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}}}
-; GCN: s_endpgm
define amdgpu_kernel void @misaligned_2_simple_read2_f32(float addrspace(1)* %out, float addrspace(3)* %lds) #0 {
+; CI-LABEL: misaligned_2_simple_read2_f32:
+; CI: ; %bb.0:
+; CI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9
+; CI-NEXT: s_load_dword s0, s[0:1], 0xb
+; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: s_mov_b32 s7, 0xf000
+; CI-NEXT: s_mov_b32 s6, 0
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_add_i32_e32 v1, vcc, s0, v0
+; CI-NEXT: ds_read_u16 v2, v1 offset:34
+; CI-NEXT: ds_read_u16 v3, v1 offset:32
+; CI-NEXT: ds_read_u16 v4, v1 offset:2
+; CI-NEXT: ds_read_u16 v1, v1
+; CI-NEXT: s_waitcnt lgkmcnt(3)
+; CI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; CI-NEXT: s_waitcnt lgkmcnt(2)
+; CI-NEXT: v_or_b32_e32 v2, v2, v3
+; CI-NEXT: s_waitcnt lgkmcnt(1)
+; CI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_or_b32_e32 v1, v4, v1
+; CI-NEXT: v_add_f32_e32 v2, v1, v2
+; CI-NEXT: v_mov_b32_e32 v1, 0
+; CI-NEXT: buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
+; CI-NEXT: s_endpgm
+;
+; GFX9-ALIGNED-LABEL: misaligned_2_simple_read2_f32:
+; GFX9-ALIGNED: ; %bb.0:
+; GFX9-ALIGNED-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
+; GFX9-ALIGNED-NEXT: s_load_dword s0, s[0:1], 0x2c
+; GFX9-ALIGNED-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-ALIGNED-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-ALIGNED-NEXT: v_add_u32_e32 v1, s0, v0
+; GFX9-ALIGNED-NEXT: ds_read_u16 v2, v1
+; GFX9-ALIGNED-NEXT: ds_read_u16 v3, v1 offset:2
+; GFX9-ALIGNED-NEXT: ds_read_u16 v4, v1 offset:32
+; GFX9-ALIGNED-NEXT: ds_read_u16 v1, v1 offset:34
+; GFX9-ALIGNED-NEXT: s_waitcnt lgkmcnt(2)
+; GFX9-ALIGNED-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX9-ALIGNED-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-ALIGNED-NEXT: v_lshl_or_b32 v1, v1, 16, v4
+; GFX9-ALIGNED-NEXT: v_add_f32_e32 v1, v2, v1
+; GFX9-ALIGNED-NEXT: global_store_dword v0, v1, s[2:3]
+; GFX9-ALIGNED-NEXT: s_endpgm
+;
+; GFX9-UNALIGNED-LABEL: misaligned_2_simple_read2_f32:
+; GFX9-UNALIGNED: ; %bb.0:
+; GFX9-UNALIGNED-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
+; GFX9-UNALIGNED-NEXT: s_load_dword s0, s[0:1], 0x2c
+; GFX9-UNALIGNED-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; GFX9-UNALIGNED-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-UNALIGNED-NEXT: v_add_u32_e32 v0, s0, v2
+; GFX9-UNALIGNED-NEXT: ds_read2_b32 v[0:1], v0 offset1:8
+; GFX9-UNALIGNED-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-UNALIGNED-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-UNALIGNED-NEXT: global_store_dword v2, v0, s[2:3]
+; GFX9-UNALIGNED-NEXT: s_endpgm
%x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
%arrayidx0 = getelementptr inbounds float, float addrspace(3)* %lds, i32 %x.i
%val0 = load float, float addrspace(3)* %arrayidx0, align 2
@@ -354,18 +708,32 @@ define amdgpu_kernel void @misaligned_2_simple_read2_f32(float addrspace(1)* %ou
ret void
}
-; GCN-LABEL: @simple_read2_f64
-; CI-DAG: s_mov_b32 m0
-; GFX9-NOT: m0
-
-; GCN-DAG: v_lshlrev_b32_e32 [[VOFS:v[0-9]+]], 3, {{v[0-9]+}}
-; GCN-DAG: v_add_{{[iu]}}32_e32 [[VPTR:v[0-9]+]], {{(vcc, )?}}lds.f64 at abs32@lo, [[VOFS]]
-; GCN: ds_read2_b64 v{{\[}}[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]{{\]}}, [[VPTR]] offset1:8
-; GCN: v_add_f64 [[RESULT:v\[[0-9]+:[0-9]+\]]], v{{\[}}[[LO_VREG]]:{{[0-9]+\]}}, v{{\[[0-9]+}}:[[HI_VREG]]{{\]}}
-
-; CI: buffer_store_dwordx2 [[RESULT]]
-; GFX9: global_store_dwordx2 v{{[0-9]+}}, [[RESULT]], s{{\[[0-9]+:[0-9]+\]}}
define amdgpu_kernel void @simple_read2_f64(double addrspace(1)* %out) #0 {
+; CI-LABEL: simple_read2_f64:
+; CI: ; %bb.0:
+; CI-NEXT: v_lshlrev_b32_e32 v4, 3, v0
+; CI-NEXT: v_add_i32_e32 v0, vcc, lds.f64 at abs32@lo, v4
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: ds_read2_b64 v[0:3], v0 offset1:8
+; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9
+; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: s_mov_b32 s2, 0
+; CI-NEXT: v_mov_b32_e32 v5, 0
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_add_f64 v[0:1], v[0:1], v[2:3]
+; CI-NEXT: buffer_store_dwordx2 v[0:1], v[4:5], s[0:3], 0 addr64
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: simple_read2_f64:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 3, v0
+; GFX9-NEXT: v_add_u32_e32 v0, lds.f64 at abs32@lo, v4
+; GFX9-NEXT: ds_read2_b64 v[0:3], v0 offset1:8
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_f64 v[0:1], v[0:1], v[2:3]
+; GFX9-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
+; GFX9-NEXT: s_endpgm
%x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
%arrayidx0 = getelementptr inbounds [512 x double], [512 x double] addrspace(3)* @lds.f64, i32 0, i32 %x.i
%val0 = load double, double addrspace(3)* %arrayidx0, align 8
@@ -378,13 +746,32 @@ define amdgpu_kernel void @simple_read2_f64(double addrspace(1)* %out) #0 {
ret void
}
-; GCN-LABEL: @simple_read2_f64_max_offset
-; CI-DAG: s_mov_b32 m0
-; GFX9-NOT: m0
-
-; GCN: ds_read2_b64 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:255
-; GCN: s_endpgm
define amdgpu_kernel void @simple_read2_f64_max_offset(double addrspace(1)* %out) #0 {
+; CI-LABEL: simple_read2_f64_max_offset:
+; CI: ; %bb.0:
+; CI-NEXT: v_lshlrev_b32_e32 v4, 3, v0
+; CI-NEXT: v_add_i32_e32 v0, vcc, lds.f64 at abs32@lo, v4
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: ds_read2_b64 v[0:3], v0 offset1:255
+; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9
+; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: s_mov_b32 s2, 0
+; CI-NEXT: v_mov_b32_e32 v5, 0
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_add_f64 v[0:1], v[0:1], v[2:3]
+; CI-NEXT: buffer_store_dwordx2 v[0:1], v[4:5], s[0:3], 0 addr64
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: simple_read2_f64_max_offset:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 3, v0
+; GFX9-NEXT: v_add_u32_e32 v0, lds.f64 at abs32@lo, v4
+; GFX9-NEXT: ds_read2_b64 v[0:3], v0 offset1:255
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_f64 v[0:1], v[0:1], v[2:3]
+; GFX9-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
+; GFX9-NEXT: s_endpgm
%x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
%arrayidx0 = getelementptr inbounds [512 x double], [512 x double] addrspace(3)* @lds.f64, i32 0, i32 %x.i
%val0 = load double, double addrspace(3)* %arrayidx0, align 8
@@ -397,15 +784,34 @@ define amdgpu_kernel void @simple_read2_f64_max_offset(double addrspace(1)* %out
ret void
}
-; GCN-LABEL: @simple_read2_f64_too_far
-; CI-DAG: s_mov_b32 m0
-; GFX9-NOT: m0
-
-; GCN-NOT: ds_read2_b64
-; GCN: ds_read_b64 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}
-; GCN: ds_read_b64 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset:2056
-; GCN: s_endpgm
define amdgpu_kernel void @simple_read2_f64_too_far(double addrspace(1)* %out) #0 {
+; CI-LABEL: simple_read2_f64_too_far:
+; CI: ; %bb.0:
+; CI-NEXT: v_lshlrev_b32_e32 v0, 3, v0
+; CI-NEXT: v_add_i32_e32 v3, vcc, lds.f64 at abs32@lo, v0
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: ds_read_b64 v[1:2], v3
+; CI-NEXT: ds_read_b64 v[3:4], v3 offset:2056
+; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9
+; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: s_mov_b32 s2, 0
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_add_f64 v[2:3], v[1:2], v[3:4]
+; CI-NEXT: v_mov_b32_e32 v1, 0
+; CI-NEXT: buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: simple_read2_f64_too_far:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 3, v0
+; GFX9-NEXT: v_add_u32_e32 v2, lds.f64 at abs32@lo, v4
+; GFX9-NEXT: ds_read_b64 v[0:1], v2
+; GFX9-NEXT: ds_read_b64 v[2:3], v2 offset:2056
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_f64 v[0:1], v[0:1], v[2:3]
+; GFX9-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
+; GFX9-NEXT: s_endpgm
%x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
%arrayidx0 = getelementptr inbounds [512 x double], [512 x double] addrspace(3)* @lds.f64, i32 0, i32 %x.i
%val0 = load double, double addrspace(3)* %arrayidx0, align 8
@@ -419,14 +825,38 @@ define amdgpu_kernel void @simple_read2_f64_too_far(double addrspace(1)* %out) #
}
; Alignment only 4
-; GCN-LABEL: @misaligned_read2_f64
-; CI-DAG: s_mov_b32 m0
-; GFX9-NOT: m0
-
-; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}} offset1:1
-; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}} offset0:14 offset1:15
-; GCN: s_endpgm
define amdgpu_kernel void @misaligned_read2_f64(double addrspace(1)* %out, double addrspace(3)* %lds) #0 {
+; CI-LABEL: misaligned_read2_f64:
+; CI: ; %bb.0:
+; CI-NEXT: s_load_dword s2, s[0:1], 0xb
+; CI-NEXT: v_lshlrev_b32_e32 v0, 3, v0
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9
+; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_add_i32_e32 v3, vcc, s2, v0
+; CI-NEXT: ds_read2_b32 v[1:2], v3 offset1:1
+; CI-NEXT: ds_read2_b32 v[3:4], v3 offset0:14 offset1:15
+; CI-NEXT: s_mov_b32 s2, 0
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_add_f64 v[2:3], v[1:2], v[3:4]
+; CI-NEXT: v_mov_b32_e32 v1, 0
+; CI-NEXT: buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: misaligned_read2_f64:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dword s2, s[0:1], 0x2c
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 3, v0
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_u32_e32 v2, s2, v4
+; GFX9-NEXT: ds_read2_b32 v[0:1], v2 offset1:1
+; GFX9-NEXT: ds_read2_b32 v[2:3], v2 offset0:14 offset1:15
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_f64 v[0:1], v[0:1], v[2:3]
+; GFX9-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
+; GFX9-NEXT: s_endpgm
%x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
%arrayidx0 = getelementptr inbounds double, double addrspace(3)* %lds, i32 %x.i
%val0 = load double, double addrspace(3)* %arrayidx0, align 4
@@ -441,13 +871,31 @@ define amdgpu_kernel void @misaligned_read2_f64(double addrspace(1)* %out, doubl
@foo = addrspace(3) global [4 x i32] undef, align 4
-; GCN-LABEL: @load_constant_adjacent_offsets
-; CI-DAG: s_mov_b32 m0
-; GFX9-NOT: m0
-
-; GCN-DAG: v_mov_b32_e32 [[PTR:v[0-9]+]], foo at abs32@lo{{$}}
-; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[PTR]] offset1:1
define amdgpu_kernel void @load_constant_adjacent_offsets(i32 addrspace(1)* %out) {
+; CI-LABEL: load_constant_adjacent_offsets:
+; CI: ; %bb.0:
+; CI-NEXT: v_mov_b32_e32 v0, foo at abs32@lo
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: ds_read2_b32 v[0:1], v0 offset1:1
+; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9
+; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: s_mov_b32 s2, -1
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_add_i32_e32 v0, vcc, v1, v0
+; CI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: load_constant_adjacent_offsets:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_mov_b32_e32 v0, foo at abs32@lo
+; GFX9-NEXT: ds_read2_b32 v[0:1], v0 offset1:1
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_u32_e32 v2, v0, v1
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: v_mov_b32_e32 v1, s1
+; GFX9-NEXT: global_store_dword v[0:1], v2, off
+; GFX9-NEXT: s_endpgm
%val0 = load i32, i32 addrspace(3)* getelementptr inbounds ([4 x i32], [4 x i32] addrspace(3)* @foo, i32 0, i32 0), align 4
%val1 = load i32, i32 addrspace(3)* getelementptr inbounds ([4 x i32], [4 x i32] addrspace(3)* @foo, i32 0, i32 1), align 4
%sum = add i32 %val0, %val1
@@ -455,13 +903,31 @@ define amdgpu_kernel void @load_constant_adjacent_offsets(i32 addrspace(1)* %out
ret void
}
-; GCN-LABEL: @load_constant_disjoint_offsets
-; CI-DAG: s_mov_b32 m0
-; GFX9-NOT: m0
-
-; GCN-DAG: v_mov_b32_e32 [[PTR:v[0-9]+]], foo at abs32@lo{{$}}
-; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[PTR]] offset1:2
define amdgpu_kernel void @load_constant_disjoint_offsets(i32 addrspace(1)* %out) {
+; CI-LABEL: load_constant_disjoint_offsets:
+; CI: ; %bb.0:
+; CI-NEXT: v_mov_b32_e32 v0, foo at abs32@lo
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: ds_read2_b32 v[0:1], v0 offset1:2
+; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9
+; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: s_mov_b32 s2, -1
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_add_i32_e32 v0, vcc, v1, v0
+; CI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: load_constant_disjoint_offsets:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_mov_b32_e32 v0, foo at abs32@lo
+; GFX9-NEXT: ds_read2_b32 v[0:1], v0 offset1:2
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_u32_e32 v2, v0, v1
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: v_mov_b32_e32 v1, s1
+; GFX9-NEXT: global_store_dword v[0:1], v2, off
+; GFX9-NEXT: s_endpgm
%val0 = load i32, i32 addrspace(3)* getelementptr inbounds ([4 x i32], [4 x i32] addrspace(3)* @foo, i32 0, i32 0), align 4
%val1 = load i32, i32 addrspace(3)* getelementptr inbounds ([4 x i32], [4 x i32] addrspace(3)* @foo, i32 0, i32 2), align 4
%sum = add i32 %val0, %val1
@@ -471,19 +937,48 @@ define amdgpu_kernel void @load_constant_disjoint_offsets(i32 addrspace(1)* %out
@bar = addrspace(3) global [4 x i64] undef, align 4
-; GCN-LABEL: @load_misaligned64_constant_offsets
-; CI-DAG: s_mov_b32 m0
-; GFX9-NOT: m0
-
-; GCN-DAG: v_mov_b32_e32 [[PTR:v[0-9]+]], bar at abs32@lo{{$}}
-; CI: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[PTR]] offset1:1
-; CI: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[PTR]] offset0:2 offset1:3
-
-; GFX9-ALIGNED-DAG: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[PTR]] offset1:1
-; GFX9-ALIGNED-DAG: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[PTR]] offset0:2 offset1:3
-
-; GFX9-UNALIGNED: ds_read_b128 v{{\[[0-9]+:[0-9]+\]}}, [[PTR]]
define amdgpu_kernel void @load_misaligned64_constant_offsets(i64 addrspace(1)* %out) {
+; CI-LABEL: load_misaligned64_constant_offsets:
+; CI: ; %bb.0:
+; CI-NEXT: v_mov_b32_e32 v2, bar at abs32@lo
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: ds_read2_b32 v[0:1], v2 offset1:1
+; CI-NEXT: ds_read2_b32 v[2:3], v2 offset0:2 offset1:3
+; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9
+; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: s_mov_b32 s2, -1
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_add_i32_e32 v0, vcc, v0, v2
+; CI-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc
+; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; CI-NEXT: s_endpgm
+;
+; GFX9-ALIGNED-LABEL: load_misaligned64_constant_offsets:
+; GFX9-ALIGNED: ; %bb.0:
+; GFX9-ALIGNED-NEXT: v_mov_b32_e32 v2, bar at abs32@lo
+; GFX9-ALIGNED-NEXT: ds_read2_b32 v[0:1], v2 offset1:1
+; GFX9-ALIGNED-NEXT: ds_read2_b32 v[2:3], v2 offset0:2 offset1:3
+; GFX9-ALIGNED-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-ALIGNED-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-ALIGNED-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2
+; GFX9-ALIGNED-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX9-ALIGNED-NEXT: v_mov_b32_e32 v3, s1
+; GFX9-ALIGNED-NEXT: v_mov_b32_e32 v2, s0
+; GFX9-ALIGNED-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX9-ALIGNED-NEXT: s_endpgm
+;
+; GFX9-UNALIGNED-LABEL: load_misaligned64_constant_offsets:
+; GFX9-UNALIGNED: ; %bb.0:
+; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v0, bar at abs32@lo
+; GFX9-UNALIGNED-NEXT: ds_read_b128 v[0:3], v0
+; GFX9-UNALIGNED-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-UNALIGNED-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-UNALIGNED-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2
+; GFX9-UNALIGNED-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v3, s1
+; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v2, s0
+; GFX9-UNALIGNED-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX9-UNALIGNED-NEXT: s_endpgm
%val0 = load i64, i64 addrspace(3)* getelementptr inbounds ([4 x i64], [4 x i64] addrspace(3)* @bar, i32 0, i32 0), align 4
%val1 = load i64, i64 addrspace(3)* getelementptr inbounds ([4 x i64], [4 x i64] addrspace(3)* @bar, i32 0, i32 1), align 4
%sum = add i64 %val0, %val1
@@ -493,19 +988,43 @@ define amdgpu_kernel void @load_misaligned64_constant_offsets(i64 addrspace(1)*
@bar.large = addrspace(3) global [4096 x i64] undef, align 4
-; GCN-LABEL: @load_misaligned64_constant_large_offsets
-; CI-DAG: s_mov_b32 m0
-; GFX9-NOT: m0
-
-; GCN-DAG: s_mov_b32 [[SBASE0:s[0-9]+]], bar.large at abs32@lo
-; GCN-DAG: s_add_i32 [[SBASE1:s[0-9]+]], [[SBASE0]], 0x4000{{$}}
-; GCN-DAG: s_addk_i32 [[SBASE0]], 0x7ff8{{$}}
-; GCN-DAG: v_mov_b32_e32 [[VBASE0:v[0-9]+]], [[SBASE0]]
-; GCN-DAG: v_mov_b32_e32 [[VBASE1:v[0-9]+]], [[SBASE1]]
-; GCN-DAG: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[VBASE0]] offset1:1
-; GCN-DAG: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[VBASE1]] offset1:1
-; GCN: s_endpgm
define amdgpu_kernel void @load_misaligned64_constant_large_offsets(i64 addrspace(1)* %out) {
+; CI-LABEL: load_misaligned64_constant_large_offsets:
+; CI: ; %bb.0:
+; CI-NEXT: s_mov_b32 s4, bar.large at abs32@lo
+; CI-NEXT: s_add_i32 s5, s4, 0x4000
+; CI-NEXT: s_addk_i32 s4, 0x7ff8
+; CI-NEXT: v_mov_b32_e32 v0, s5
+; CI-NEXT: v_mov_b32_e32 v2, s4
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: ds_read2_b32 v[0:1], v0 offset1:1
+; CI-NEXT: ds_read2_b32 v[2:3], v2 offset1:1
+; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9
+; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: s_mov_b32 s2, -1
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_add_i32_e32 v0, vcc, v0, v2
+; CI-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc
+; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: load_misaligned64_constant_large_offsets:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_mov_b32 s2, bar.large at abs32@lo
+; GFX9-NEXT: s_add_i32 s3, s2, 0x4000
+; GFX9-NEXT: s_addk_i32 s2, 0x7ff8
+; GFX9-NEXT: v_mov_b32_e32 v0, s3
+; GFX9-NEXT: v_mov_b32_e32 v2, s2
+; GFX9-NEXT: ds_read2_b32 v[0:1], v0 offset1:1
+; GFX9-NEXT: ds_read2_b32 v[2:3], v2 offset1:1
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX9-NEXT: v_mov_b32_e32 v3, s1
+; GFX9-NEXT: v_mov_b32_e32 v2, s0
+; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX9-NEXT: s_endpgm
%val0 = load i64, i64 addrspace(3)* getelementptr inbounds ([4096 x i64], [4096 x i64] addrspace(3)* @bar.large, i32 0, i32 2048), align 4
%val1 = load i64, i64 addrspace(3)* getelementptr inbounds ([4096 x i64], [4096 x i64] addrspace(3)* @bar.large, i32 0, i32 4095), align 4
%sum = add i64 %val0, %val1
@@ -516,11 +1035,65 @@ define amdgpu_kernel void @load_misaligned64_constant_large_offsets(i64 addrspac
@sgemm.lA = internal unnamed_addr addrspace(3) global [264 x float] undef, align 4
@sgemm.lB = internal unnamed_addr addrspace(3) global [776 x float] undef, align 4
-; GCN-LABEL: {{^}}sgemm_inner_loop_read2_sequence:
-; CI-DAG: s_mov_b32 m0
-; GFX9-NOT: m0
-
define amdgpu_kernel void @sgemm_inner_loop_read2_sequence(float addrspace(1)* %C, i32 %lda, i32 %ldb) #0 {
+; CI-LABEL: sgemm_inner_loop_read2_sequence:
+; CI: ; %bb.0:
+; CI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9
+; CI-NEXT: s_lshl_b32 s0, s2, 2
+; CI-NEXT: s_add_i32 s1, s0, 0xc20
+; CI-NEXT: s_addk_i32 s0, 0xc60
+; CI-NEXT: v_mov_b32_e32 v0, s1
+; CI-NEXT: v_mov_b32_e32 v4, s0
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: ds_read2_b32 v[2:3], v0 offset1:1
+; CI-NEXT: ds_read2_b32 v[4:5], v4 offset1:1
+; CI-NEXT: v_lshlrev_b32_e32 v8, 2, v1
+; CI-NEXT: ds_read2_b32 v[0:1], v8 offset1:1
+; CI-NEXT: ds_read2_b32 v[6:7], v8 offset0:32 offset1:33
+; CI-NEXT: ds_read2_b32 v[8:9], v8 offset0:64 offset1:65
+; CI-NEXT: s_mov_b32 s7, 0xf000
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_add_f32_e32 v2, v2, v3
+; CI-NEXT: v_add_f32_e32 v2, v2, v4
+; CI-NEXT: v_add_f32_e32 v2, v2, v5
+; CI-NEXT: v_add_f32_e32 v0, v2, v0
+; CI-NEXT: v_add_f32_e32 v0, v0, v1
+; CI-NEXT: v_add_f32_e32 v0, v0, v6
+; CI-NEXT: v_add_f32_e32 v0, v0, v7
+; CI-NEXT: v_add_f32_e32 v0, v0, v8
+; CI-NEXT: s_mov_b32 s6, -1
+; CI-NEXT: v_add_f32_e32 v0, v0, v9
+; CI-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: sgemm_inner_loop_read2_sequence:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_lshl_b32 s2, s2, 2
+; GFX9-NEXT: s_add_i32 s3, s2, 0xc20
+; GFX9-NEXT: s_addk_i32 s2, 0xc60
+; GFX9-NEXT: v_mov_b32_e32 v0, s3
+; GFX9-NEXT: v_mov_b32_e32 v4, s2
+; GFX9-NEXT: ds_read2_b32 v[2:3], v0 offset1:1
+; GFX9-NEXT: ds_read2_b32 v[4:5], v4 offset1:1
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 2, v1
+; GFX9-NEXT: ds_read2_b32 v[0:1], v8 offset1:1
+; GFX9-NEXT: ds_read2_b32 v[6:7], v8 offset0:32 offset1:33
+; GFX9-NEXT: ds_read2_b32 v[8:9], v8 offset0:64 offset1:65
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_f32_e32 v2, v2, v3
+; GFX9-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX9-NEXT: v_add_f32_e32 v2, v2, v5
+; GFX9-NEXT: v_add_f32_e32 v0, v2, v0
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v6
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v7
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v8
+; GFX9-NEXT: v_add_f32_e32 v2, v0, v9
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: v_mov_b32_e32 v1, s1
+; GFX9-NEXT: global_store_dword v[0:1], v2, off
+; GFX9-NEXT: s_endpgm
%x.i = tail call i32 @llvm.amdgcn.workgroup.id.x() #1
%y.i = tail call i32 @llvm.amdgcn.workitem.id.y() #1
%arrayidx44 = getelementptr inbounds [264 x float], [264 x float] addrspace(3)* @sgemm.lA, i32 0, i32 %x.i
@@ -564,30 +1137,131 @@ define amdgpu_kernel void @sgemm_inner_loop_read2_sequence(float addrspace(1)* %
ret void
}
-; GCN-LABEL: {{^}}misaligned_read2_v2i32:
-; CI-DAG: s_mov_b32 m0
-; GFX9-NOT: m0
define amdgpu_kernel void @misaligned_read2_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> addrspace(3)* %in) #0 {
+; CI-LABEL: misaligned_read2_v2i32:
+; CI: ; %bb.0:
+; CI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9
+; CI-NEXT: s_load_dword s0, s[0:1], 0xb
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: s_mov_b32 s7, 0xf000
+; CI-NEXT: s_mov_b32 s6, -1
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_mov_b32_e32 v0, s0
+; CI-NEXT: ds_read2_b32 v[0:1], v0 offset1:1
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: misaligned_read2_v2i32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
+; GFX9-NEXT: s_load_dword s0, s[0:1], 0x2c
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v2, s2
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: ds_read2_b32 v[0:1], v0 offset1:1
+; GFX9-NEXT: v_mov_b32_e32 v3, s3
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX9-NEXT: s_endpgm
%load = load <2 x i32>, <2 x i32> addrspace(3)* %in, align 4
store <2 x i32> %load, <2 x i32> addrspace(1)* %out, align 8
ret void
}
-; GCN-LABEL: {{^}}misaligned_read2_i64:
-; CI-DAG: s_mov_b32 m0
-; GFX9-NOT: m0
define amdgpu_kernel void @misaligned_read2_i64(i64 addrspace(1)* %out, i64 addrspace(3)* %in) #0 {
+; CI-LABEL: misaligned_read2_i64:
+; CI: ; %bb.0:
+; CI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9
+; CI-NEXT: s_load_dword s0, s[0:1], 0xb
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: s_mov_b32 s7, 0xf000
+; CI-NEXT: s_mov_b32 s6, -1
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_mov_b32_e32 v0, s0
+; CI-NEXT: ds_read2_b32 v[0:1], v0 offset1:1
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: misaligned_read2_i64:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
+; GFX9-NEXT: s_load_dword s0, s[0:1], 0x2c
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v2, s2
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: ds_read2_b32 v[0:1], v0 offset1:1
+; GFX9-NEXT: v_mov_b32_e32 v3, s3
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX9-NEXT: s_endpgm
%load = load i64, i64 addrspace(3)* %in, align 4
store i64 %load, i64 addrspace(1)* %out, align 8
ret void
}
-; GCN-LABEL: ds_read_
diff _base_interleaving
-; CI-DAG: s_mov_b32 m0
-; GFX9-NOT: m0
-
-; GCN-NOT: ds_read_b32
define amdgpu_kernel void @ds_read_
diff _base_interleaving(
+; CI-LABEL: ds_read_
diff _base_interleaving:
+; CI: ; %bb.0: ; %bb
+; CI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9
+; CI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0xb
+; CI-NEXT: v_lshlrev_b32_e32 v1, 4, v1
+; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: s_mov_b32 s7, 0xf000
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_add_i32_e32 v2, vcc, s0, v1
+; CI-NEXT: v_add_i32_e32 v4, vcc, s1, v0
+; CI-NEXT: v_add_i32_e32 v3, vcc, s2, v1
+; CI-NEXT: v_add_i32_e32 v6, vcc, s3, v0
+; CI-NEXT: ds_read2_b32 v[0:1], v2 offset1:1
+; CI-NEXT: ds_read2_b32 v[2:3], v3 offset1:1
+; CI-NEXT: ds_read2_b32 v[4:5], v4 offset1:4
+; CI-NEXT: s_mov_b32 s6, -1
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_mul_f32_e32 v0, v0, v4
+; CI-NEXT: v_add_f32_e32 v4, 2.0, v0
+; CI-NEXT: v_mul_f32_e32 v5, v1, v5
+; CI-NEXT: ds_read2_b32 v[0:1], v6 offset1:4
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_mul_f32_e32 v0, v2, v0
+; CI-NEXT: v_sub_f32_e32 v0, v4, v0
+; CI-NEXT: v_sub_f32_e32 v0, v0, v5
+; CI-NEXT: v_mul_f32_e32 v1, v3, v1
+; CI-NEXT: v_sub_f32_e32 v0, v0, v1
+; CI-NEXT: buffer_store_dword v0, off, s[4:7], 0 offset:40
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: ds_read_
diff _base_interleaving:
+; GFX9: ; %bb.0: ; %bb
+; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x2c
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 4, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_u32_e32 v2, s0, v1
+; GFX9-NEXT: v_add_u32_e32 v3, s1, v0
+; GFX9-NEXT: v_add_u32_e32 v4, s2, v1
+; GFX9-NEXT: v_add_u32_e32 v6, s3, v0
+; GFX9-NEXT: ds_read2_b32 v[0:1], v2 offset1:1
+; GFX9-NEXT: ds_read2_b32 v[2:3], v3 offset1:4
+; GFX9-NEXT: ds_read2_b32 v[4:5], v4 offset1:1
+; GFX9-NEXT: ds_read2_b32 v[6:7], v6 offset1:4
+; GFX9-NEXT: s_waitcnt lgkmcnt(2)
+; GFX9-NEXT: v_mul_f32_e32 v0, v0, v2
+; GFX9-NEXT: v_add_f32_e32 v0, 2.0, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mul_f32_e32 v2, v4, v6
+; GFX9-NEXT: v_sub_f32_e32 v0, v0, v2
+; GFX9-NEXT: v_mul_f32_e32 v1, v1, v3
+; GFX9-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_mul_f32_e32 v1, v5, v7
+; GFX9-NEXT: v_sub_f32_e32 v2, v0, v1
+; GFX9-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-NEXT: v_mov_b32_e32 v1, s5
+; GFX9-NEXT: global_store_dword v[0:1], v2, off offset:40
+; GFX9-NEXT: s_endpgm
float addrspace(1)* nocapture %arg,
[4 x [4 x float]] addrspace(3)* %arg1,
[4 x [4 x float]] addrspace(3)* %arg2,
@@ -625,11 +1299,68 @@ bb:
ret void
}
-; GCN-LABEL: ds_read_call_read:
-; GCN: ds_read_b32
-; GCN: s_swappc_b64
-; GCN: ds_read_b32
define amdgpu_kernel void @ds_read_call_read(i32 addrspace(1)* %out, i32 addrspace(3)* %arg) {
+; CI-LABEL: ds_read_call_read:
+; CI: ; %bb.0:
+; CI-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0
+; CI-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1
+; CI-NEXT: s_load_dwordx2 s[36:37], s[0:1], 0x9
+; CI-NEXT: s_load_dword s0, s[0:1], 0xb
+; CI-NEXT: s_mov_b32 s42, -1
+; CI-NEXT: s_mov_b32 s43, 0xe8f000
+; CI-NEXT: s_add_u32 s40, s40, s3
+; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT: s_addc_u32 s41, s41, 0
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_add_i32_e32 v40, vcc, s0, v0
+; CI-NEXT: s_getpc_b64 s[0:1]
+; CI-NEXT: s_add_u32 s0, s0, void_func_void at gotpcrel32@lo+4
+; CI-NEXT: s_addc_u32 s1, s1, void_func_void at gotpcrel32@hi+12
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
+; CI-NEXT: ds_read_b32 v41, v40
+; CI-NEXT: s_mov_b64 s[0:1], s[40:41]
+; CI-NEXT: s_mov_b64 s[2:3], s[42:43]
+; CI-NEXT: s_mov_b32 s32, 0
+; CI-NEXT: s_mov_b32 s39, 0xf000
+; CI-NEXT: s_mov_b32 s38, -1
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: s_swappc_b64 s[30:31], s[4:5]
+; CI-NEXT: ds_read_b32 v0, v40 offset:4
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_add_i32_e32 v0, vcc, v41, v0
+; CI-NEXT: buffer_store_dword v0, off, s[36:39], 0
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: ds_read_call_read:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT: s_mov_b32 s38, -1
+; GFX9-NEXT: s_load_dwordx2 s[34:35], s[0:1], 0x24
+; GFX9-NEXT: s_load_dword s2, s[0:1], 0x2c
+; GFX9-NEXT: s_mov_b32 s39, 0xe00000
+; GFX9-NEXT: s_add_u32 s36, s36, s3
+; GFX9-NEXT: s_addc_u32 s37, s37, 0
+; GFX9-NEXT: s_getpc_b64 s[0:1]
+; GFX9-NEXT: s_add_u32 s0, s0, void_func_void at gotpcrel32@lo+4
+; GFX9-NEXT: s_addc_u32 s1, s1, void_func_void at gotpcrel32@hi+12
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_lshl_add_u32 v40, v0, 2, s2
+; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
+; GFX9-NEXT: ds_read_b32 v41, v40
+; GFX9-NEXT: s_mov_b64 s[0:1], s[36:37]
+; GFX9-NEXT: s_mov_b64 s[2:3], s[38:39]
+; GFX9-NEXT: s_mov_b32 s32, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_swappc_b64 s[30:31], s[4:5]
+; GFX9-NEXT: ds_read_b32 v0, v40 offset:4
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_u32_e32 v2, v41, v0
+; GFX9-NEXT: v_mov_b32_e32 v0, s34
+; GFX9-NEXT: v_mov_b32_e32 v1, s35
+; GFX9-NEXT: global_store_dword v[0:1], v2, off
+; GFX9-NEXT: s_endpgm
%x = call i32 @llvm.amdgcn.workitem.id.x()
%arrayidx0 = getelementptr i32, i32 addrspace(3)* %arg, i32 %x
%arrayidx1 = getelementptr i32, i32 addrspace(3)* %arrayidx0, i32 1
@@ -641,17 +1372,29 @@ define amdgpu_kernel void @ds_read_call_read(i32 addrspace(1)* %out, i32 addrspa
ret void
}
-; GCN-LABEL: {{^}}ds_read_interp_read:
-; CI: s_mov_b32 m0, -1
-; CI: ds_read_b32
-; CI: s_mov_b32 m0, s0
-; CI: v_interp_mov_f32
-; CI: s_mov_b32 m0, -1
-; CI: ds_read_b32
-; GFX9: ds_read2_b32 v[0:1], v0 offset1:4
-; GFX9: s_mov_b32 m0, s0
-; GFX9: v_interp_mov_f32
define amdgpu_ps <2 x float> @ds_read_interp_read(i32 inreg %prims, float addrspace(3)* %inptr) {
+; CI-LABEL: ds_read_interp_read:
+; CI: ; %bb.0:
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: ds_read_b32 v2, v0
+; CI-NEXT: s_mov_b32 m0, s0
+; CI-NEXT: v_interp_mov_f32 v1, p10, attr0.x
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: ds_read_b32 v0, v0 offset:16
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_add_f32_e32 v1, v0, v1
+; CI-NEXT: v_mov_b32_e32 v0, v2
+; CI-NEXT: ; return to shader part epilog
+;
+; GFX9-LABEL: ds_read_interp_read:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: ds_read2_b32 v[0:1], v0 offset1:4
+; GFX9-NEXT: s_mov_b32 m0, s0
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: v_interp_mov_f32_e32 v2, p10, attr0.x
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX9-NEXT: ; return to shader part epilog
%v0 = load float, float addrspace(3)* %inptr, align 4
%intrp = call float @llvm.amdgcn.interp.mov(i32 0, i32 0, i32 0, i32 %prims)
%ptr1 = getelementptr float, float addrspace(3)* %inptr, i32 4
@@ -664,14 +1407,80 @@ define amdgpu_ps <2 x float> @ds_read_interp_read(i32 inreg %prims, float addrsp
@v2i32_align1 = internal addrspace(3) global [100 x <2 x i32>] undef, align 1
-; GCN-LABEL: {{^}}read2_v2i32_align1_odd_offset:
-; CI-COUNT-8: ds_read_u8
-
-; GFX9-ALIGNED-COUNT-8: ds_read_u8
-
-; GFX9-UNALIGNED: v_mov_b32_e32 [[BASE_ADDR:v[0-9]+]], 0x41{{$}}
-; GFX9-UNALIGNED: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASE_ADDR]] offset1:1{{$}}
define amdgpu_kernel void @read2_v2i32_align1_odd_offset(<2 x i32> addrspace(1)* %out) {
+; CI-LABEL: read2_v2i32_align1_odd_offset:
+; CI: ; %bb.0: ; %entry
+; CI-NEXT: v_mov_b32_e32 v0, 0
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: ds_read_u8 v1, v0 offset:72
+; CI-NEXT: ds_read_u8 v2, v0 offset:71
+; CI-NEXT: ds_read_u8 v3, v0 offset:70
+; CI-NEXT: ds_read_u8 v4, v0 offset:69
+; CI-NEXT: ds_read_u8 v5, v0 offset:68
+; CI-NEXT: s_waitcnt lgkmcnt(4)
+; CI-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; CI-NEXT: s_waitcnt lgkmcnt(3)
+; CI-NEXT: v_or_b32_e32 v1, v1, v2
+; CI-NEXT: s_waitcnt lgkmcnt(2)
+; CI-NEXT: v_lshlrev_b32_e32 v3, 8, v3
+; CI-NEXT: s_waitcnt lgkmcnt(1)
+; CI-NEXT: v_or_b32_e32 v3, v3, v4
+; CI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; CI-NEXT: v_or_b32_e32 v1, v1, v3
+; CI-NEXT: ds_read_u8 v2, v0 offset:67
+; CI-NEXT: ds_read_u8 v3, v0 offset:66
+; CI-NEXT: ds_read_u8 v0, v0 offset:65
+; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9
+; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: s_mov_b32 s2, -1
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_lshlrev_b32_e32 v3, 8, v3
+; CI-NEXT: v_or_b32_e32 v0, v3, v0
+; CI-NEXT: v_lshlrev_b32_e32 v3, 8, v5
+; CI-NEXT: v_or_b32_e32 v2, v3, v2
+; CI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; CI-NEXT: v_or_b32_e32 v0, v2, v0
+; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; CI-NEXT: s_endpgm
+;
+; GFX9-ALIGNED-LABEL: read2_v2i32_align1_odd_offset:
+; GFX9-ALIGNED: ; %bb.0: ; %entry
+; GFX9-ALIGNED-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-ALIGNED-NEXT: ds_read_u8 v2, v0 offset:65
+; GFX9-ALIGNED-NEXT: ds_read_u8 v3, v0 offset:66
+; GFX9-ALIGNED-NEXT: ds_read_u8 v4, v0 offset:67
+; GFX9-ALIGNED-NEXT: ds_read_u8 v5, v0 offset:68
+; GFX9-ALIGNED-NEXT: ds_read_u8 v1, v0 offset:69
+; GFX9-ALIGNED-NEXT: ds_read_u8 v6, v0 offset:70
+; GFX9-ALIGNED-NEXT: ds_read_u8 v7, v0 offset:71
+; GFX9-ALIGNED-NEXT: ds_read_u8 v0, v0 offset:72
+; GFX9-ALIGNED-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-ALIGNED-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-ALIGNED-NEXT: v_lshlrev_b32_e32 v6, 8, v6
+; GFX9-ALIGNED-NEXT: v_or_b32_e32 v1, v6, v1
+; GFX9-ALIGNED-NEXT: v_lshlrev_b32_e32 v0, 8, v0
+; GFX9-ALIGNED-NEXT: v_or_b32_sdwa v0, v0, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-ALIGNED-NEXT: v_or_b32_e32 v1, v0, v1
+; GFX9-ALIGNED-NEXT: v_lshlrev_b32_e32 v0, 8, v3
+; GFX9-ALIGNED-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX9-ALIGNED-NEXT: v_lshlrev_b32_e32 v2, 8, v5
+; GFX9-ALIGNED-NEXT: v_or_b32_sdwa v2, v2, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-ALIGNED-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX9-ALIGNED-NEXT: v_mov_b32_e32 v3, s1
+; GFX9-ALIGNED-NEXT: v_mov_b32_e32 v2, s0
+; GFX9-ALIGNED-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX9-ALIGNED-NEXT: s_endpgm
+;
+; GFX9-UNALIGNED-LABEL: read2_v2i32_align1_odd_offset:
+; GFX9-UNALIGNED: ; %bb.0: ; %entry
+; GFX9-UNALIGNED-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v0, 0x41
+; GFX9-UNALIGNED-NEXT: ds_read2_b32 v[0:1], v0 offset1:1
+; GFX9-UNALIGNED-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v3, s1
+; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v2, s0
+; GFX9-UNALIGNED-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX9-UNALIGNED-NEXT: s_endpgm
entry:
%load = load <2 x i32>, <2 x i32> addrspace(3)* bitcast (i8 addrspace(3)* getelementptr (i8, i8 addrspace(3)* bitcast ([100 x <2 x i32>] addrspace(3)* @v2i32_align1 to i8 addrspace(3)*), i32 65) to <2 x i32> addrspace(3)*), align 1
store <2 x i32> %load, <2 x i32> addrspace(1)* %out
diff --git a/llvm/test/CodeGen/AMDGPU/ds_write2.ll b/llvm/test/CodeGen/AMDGPU/ds_write2.ll
index a4a5d47c9e05..f4657490d34d 100644
--- a/llvm/test/CodeGen/AMDGPU/ds_write2.ll
+++ b/llvm/test/CodeGen/AMDGPU/ds_write2.ll
@@ -1,20 +1,37 @@
-; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -mattr=+load-store-opt < %s | FileCheck -enable-var-scope -strict-whitespace -check-prefixes=GCN,CI %s
-; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -mattr=+load-store-opt,+flat-for-global,-unaligned-access-mode < %s | FileCheck -enable-var-scope -strict-whitespace -check-prefixes=GCN,GFX9,GFX9-ALIGNED %s
-; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -mattr=+load-store-opt,+flat-for-global,+unaligned-access-mode < %s | FileCheck -enable-var-scope -strict-whitespace -check-prefixes=GCN,GFX9,GFX9-UNALIGNED %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -mattr=+load-store-opt < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CI %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -mattr=+load-store-opt,-unaligned-access-mode < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9,GFX9-ALIGNED %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -mattr=+load-store-opt,+unaligned-access-mode < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9,GFX9-UNALIGNED %s
@lds = addrspace(3) global [512 x float] undef, align 4
@lds.f64 = addrspace(3) global [512 x double] undef, align 8
-; GCN-LABEL: {{^}}simple_write2_one_val_f32:
-; CI-DAG: s_mov_b32 m0
-; GFX9-NOT: m0
-
-; GCN-DAG: {{buffer|flat|global}}_load_dword [[VAL:v[0-9]+]]
-; GCN-DAG: v_lshlrev_b32_e32 [[VBASE:v[0-9]+]], 2, v{{[0-9]+}}
-; GCN-DAG: v_add_{{[ui]}}32_e32 [[VPTR:v[0-9]+]], {{(vcc, )?}}lds at abs32@lo, [[VBASE]]
-; GCN: ds_write2_b32 [[VPTR]], [[VAL]], [[VAL]] offset1:8
-; GCN: s_endpgm
define amdgpu_kernel void @simple_write2_one_val_f32(float addrspace(1)* %C, float addrspace(1)* %in) #0 {
+; CI-LABEL: simple_write2_one_val_f32:
+; CI: ; %bb.0:
+; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xb
+; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: s_mov_b32 s2, 0
+; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT: v_mov_b32_e32 v1, 0
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: buffer_load_dword v1, v[0:1], s[0:3], 0 addr64
+; CI-NEXT: v_add_i32_e32 v0, vcc, lds at abs32@lo, v0
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: s_waitcnt vmcnt(0)
+; CI-NEXT: ds_write2_b32 v0, v1, v1 offset1:8
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: simple_write2_one_val_f32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2c
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dword v1, v0, s[0:1]
+; GFX9-NEXT: v_add_u32_e32 v0, lds at abs32@lo, v0
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: ds_write2_b32 v0, v1, v1 offset1:8
+; GFX9-NEXT: s_endpgm
%x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
%in.gep = getelementptr float, float addrspace(1)* %in, i32 %x.i
%val = load float, float addrspace(1)* %in.gep, align 4
@@ -26,21 +43,34 @@ define amdgpu_kernel void @simple_write2_one_val_f32(float addrspace(1)* %C, flo
ret void
}
-; GCN-LABEL: {{^}}simple_write2_two_val_f32:
-; CI-DAG: s_mov_b32 m0
-; GFX9-NOT: m0
-
-; CI-DAG: buffer_load_dword [[VAL0:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
-; CI-DAG: buffer_load_dword [[VAL1:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4
-
-; GFX9-DAG: global_load_dword [[VAL0:v[0-9]+]], v{{[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}{{$}}
-; GFX9-DAG: global_load_dword [[VAL1:v[0-9]+]], v{{[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}} offset:4{{$}}
-
-; GCN-DAG: v_lshlrev_b32_e32 [[VBASE:v[0-9]+]], 2, v{{[0-9]+}}
-; GCN-DAG: v_add_{{[ui]}}32_e32 [[VPTR:v[0-9]+]], {{(vcc, )?}}lds at abs32@lo, [[VBASE]]
-; GCN: ds_write2_b32 [[VPTR]], [[VAL0]], [[VAL1]] offset1:8
-; GCN: s_endpgm
define amdgpu_kernel void @simple_write2_two_val_f32(float addrspace(1)* %C, float addrspace(1)* %in) #0 {
+; CI-LABEL: simple_write2_two_val_f32:
+; CI: ; %bb.0:
+; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xb
+; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: s_mov_b32 s2, 0
+; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT: v_mov_b32_e32 v1, 0
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: buffer_load_dword v2, v[0:1], s[0:3], 0 addr64
+; CI-NEXT: buffer_load_dword v1, v[0:1], s[0:3], 0 addr64 offset:4
+; CI-NEXT: v_add_i32_e32 v0, vcc, lds at abs32@lo, v0
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: s_waitcnt vmcnt(0)
+; CI-NEXT: ds_write2_b32 v0, v2, v1 offset1:8
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: simple_write2_two_val_f32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2c
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dword v1, v0, s[0:1]
+; GFX9-NEXT: global_load_dword v2, v0, s[0:1] offset:4
+; GFX9-NEXT: v_add_u32_e32 v0, lds at abs32@lo, v0
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: ds_write2_b32 v0, v1, v2 offset1:8
+; GFX9-NEXT: s_endpgm
%x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
%in.gep.0 = getelementptr float, float addrspace(1)* %in, i32 %x.i
%in.gep.1 = getelementptr float, float addrspace(1)* %in.gep.0, i32 1
@@ -54,15 +84,41 @@ define amdgpu_kernel void @simple_write2_two_val_f32(float addrspace(1)* %C, flo
ret void
}
-; GCN-LABEL: @simple_write2_two_val_f32_volatile_0
-; CI-DAG: s_mov_b32 m0
-; GFX9-NOT: m0
-
-; GCN-NOT: ds_write2_b32
-; GCN: ds_write_b32 {{v[0-9]+}}, {{v[0-9]+}}
-; GCN: ds_write_b32 {{v[0-9]+}}, {{v[0-9]+}} offset:32
-; GCN: s_endpgm
define amdgpu_kernel void @simple_write2_two_val_f32_volatile_0(float addrspace(1)* %C, float addrspace(1)* %in0, float addrspace(1)* %in1) #0 {
+; CI-LABEL: simple_write2_two_val_f32_volatile_0:
+; CI: ; %bb.0:
+; CI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0xb
+; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: s_mov_b32 s2, 0
+; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT: v_mov_b32_e32 v1, 0
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: s_mov_b64 s[0:1], s[4:5]
+; CI-NEXT: s_mov_b64 s[4:5], s[6:7]
+; CI-NEXT: s_mov_b64 s[6:7], s[2:3]
+; CI-NEXT: buffer_load_dword v2, v[0:1], s[0:3], 0 addr64
+; CI-NEXT: buffer_load_dword v1, v[0:1], s[4:7], 0 addr64
+; CI-NEXT: v_add_i32_e32 v0, vcc, lds at abs32@lo, v0
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: s_waitcnt vmcnt(1)
+; CI-NEXT: ds_write_b32 v0, v2
+; CI-NEXT: s_waitcnt vmcnt(0)
+; CI-NEXT: ds_write_b32 v0, v1 offset:32
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: simple_write2_two_val_f32_volatile_0:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x2c
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dword v1, v0, s[0:1]
+; GFX9-NEXT: global_load_dword v2, v0, s[2:3]
+; GFX9-NEXT: v_add_u32_e32 v0, lds at abs32@lo, v0
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: ds_write_b32 v0, v1
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: ds_write_b32 v0, v2 offset:32
+; GFX9-NEXT: s_endpgm
%x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
%in0.gep = getelementptr float, float addrspace(1)* %in0, i32 %x.i
%in1.gep = getelementptr float, float addrspace(1)* %in1, i32 %x.i
@@ -76,15 +132,41 @@ define amdgpu_kernel void @simple_write2_two_val_f32_volatile_0(float addrspace(
ret void
}
-; GCN-LABEL: @simple_write2_two_val_f32_volatile_1
-; CI-DAG: s_mov_b32 m0
-; GFX9-NOT: m0
-
-; GCN-NOT: ds_write2_b32
-; GCN: ds_write_b32 {{v[0-9]+}}, {{v[0-9]+}}
-; GCN: ds_write_b32 {{v[0-9]+}}, {{v[0-9]+}} offset:32
-; GCN: s_endpgm
define amdgpu_kernel void @simple_write2_two_val_f32_volatile_1(float addrspace(1)* %C, float addrspace(1)* %in0, float addrspace(1)* %in1) #0 {
+; CI-LABEL: simple_write2_two_val_f32_volatile_1:
+; CI: ; %bb.0:
+; CI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0xb
+; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: s_mov_b32 s2, 0
+; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT: v_mov_b32_e32 v1, 0
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: s_mov_b64 s[0:1], s[4:5]
+; CI-NEXT: s_mov_b64 s[4:5], s[6:7]
+; CI-NEXT: s_mov_b64 s[6:7], s[2:3]
+; CI-NEXT: buffer_load_dword v2, v[0:1], s[0:3], 0 addr64
+; CI-NEXT: buffer_load_dword v1, v[0:1], s[4:7], 0 addr64
+; CI-NEXT: v_add_i32_e32 v0, vcc, lds at abs32@lo, v0
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: s_waitcnt vmcnt(1)
+; CI-NEXT: ds_write_b32 v0, v2
+; CI-NEXT: s_waitcnt vmcnt(0)
+; CI-NEXT: ds_write_b32 v0, v1 offset:32
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: simple_write2_two_val_f32_volatile_1:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x2c
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dword v1, v0, s[0:1]
+; GFX9-NEXT: global_load_dword v2, v0, s[2:3]
+; GFX9-NEXT: v_add_u32_e32 v0, lds at abs32@lo, v0
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: ds_write_b32 v0, v1
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: ds_write_b32 v0, v2 offset:32
+; GFX9-NEXT: s_endpgm
%x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
%in0.gep = getelementptr float, float addrspace(1)* %in0, i32 %x.i
%in1.gep = getelementptr float, float addrspace(1)* %in1, i32 %x.i
@@ -99,28 +181,41 @@ define amdgpu_kernel void @simple_write2_two_val_f32_volatile_1(float addrspace(
}
; 2 data subregisters from
diff erent super registers.
-; GCN-LABEL: {{^}}simple_write2_two_val_subreg2_mixed_f32:
-; GFX9-NOT: m0
-
-; CI: buffer_load_dwordx2 v{{\[}}[[VAL0:[0-9]+]]:{{[0-9]+\]}}
-; CI: buffer_load_dwordx2 v{{\[[0-9]+}}:[[VAL1:[0-9]+]]{{\]}}
-; CI-DAG: s_mov_b32 m0
-
-; CI-DAG: v_lshlrev_b32_e32 [[VOFS:v[0-9]+]], 2, v{{[0-9]+}}
-; CI-DAG: v_add_i32_e32 [[VPTR:v[0-9]+]], vcc, lds at abs32@lo, [[VOFS]]
-;
-; TODO: This should be an s_mov_b32. The v_mov_b32 gets introduced by an
+; TODO: GFX9 has v_mov_b32_e32 v2, lds at abs32@lo
+; This should be an s_mov_b32. The v_mov_b32 gets introduced by an
; early legalization of the constant bus constraint on the v_lshl_add_u32,
; and then SIFoldOperands folds in an unlucky order.
-; GFX9-DAG: v_mov_b32_e32 [[VBASE:v[0-9]+]], lds at abs32@lo
-; GFX9-DAG: v_lshl_add_u32 [[VPTR:v[0-9]+]], {{v[0-9]+}}, 2, [[VBASE]]
-
-; GFX9-DAG: global_load_dwordx2 v{{\[}}[[VAL0:[0-9]+]]:{{[0-9]+\]}}
-; GFX9-DAG: global_load_dwordx2 v{{\[[0-9]+}}:[[VAL1:[0-9]+]]{{\]}}
-
-; GCN: ds_write2_b32 [[VPTR]], v[[VAL0]], v[[VAL1]] offset1:8
-; GCN: s_endpgm
define amdgpu_kernel void @simple_write2_two_val_subreg2_mixed_f32(float addrspace(1)* %C, <2 x float> addrspace(1)* %in) #0 {
+; CI-LABEL: simple_write2_two_val_subreg2_mixed_f32:
+; CI: ; %bb.0:
+; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xb
+; CI-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: s_mov_b32 s2, 0
+; CI-NEXT: v_mov_b32_e32 v2, 0
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: buffer_load_dwordx2 v[3:4], v[1:2], s[0:3], 0 addr64
+; CI-NEXT: buffer_load_dwordx2 v[1:2], v[1:2], s[0:3], 0 addr64 offset:8
+; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT: v_add_i32_e32 v0, vcc, lds at abs32@lo, v0
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: s_waitcnt vmcnt(0)
+; CI-NEXT: ds_write2_b32 v0, v3, v2 offset1:8
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: simple_write2_two_val_subreg2_mixed_f32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2c
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 3, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dwordx2 v[1:2], v3, s[0:1]
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v3, s[0:1] offset:8
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v2, lds at abs32@lo
+; GFX9-NEXT: v_lshl_add_u32 v0, v0, 2, v2
+; GFX9-NEXT: ds_write2_b32 v0, v1, v3 offset1:8
+; GFX9-NEXT: s_endpgm
%x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
%in.gep.0 = getelementptr <2 x float>, <2 x float> addrspace(1)* %in, i32 %x.i
%in.gep.1 = getelementptr <2 x float>, <2 x float> addrspace(1)* %in.gep.0, i32 1
@@ -136,20 +231,34 @@ define amdgpu_kernel void @simple_write2_two_val_subreg2_mixed_f32(float addrspa
ret void
}
-; GCN-LABEL: @simple_write2_two_val_subreg2_f32
-; CI-DAG: s_mov_b32 m0
-; GFX9-NOT: m0
-
-; GCN-DAG: {{buffer|global}}_load_dwordx2 v{{\[}}[[VAL0:[0-9]+]]:[[VAL1:[0-9]+]]{{\]}}
-
-; CI-DAG: v_lshlrev_b32_e32 [[VOFS:v[0-9]+]], 2, v{{[0-9]+}}
-; CI-DAG: v_add_i32_e32 [[VPTR:v[0-9]+]], vcc, lds at abs32@lo, [[VOFS]]
-; GFX9-DAG: v_mov_b32_e32 [[VBASE:v[0-9]+]], lds at abs32@lo
-; GFX9-DAG: v_lshl_add_u32 [[VPTR:v[0-9]+]], v{{[0-9]+}}, 2, [[VBASE]]
-
-; GCN: ds_write2_b32 [[VPTR]], v[[VAL0]], v[[VAL1]] offset1:8
-; GCN: s_endpgm
define amdgpu_kernel void @simple_write2_two_val_subreg2_f32(float addrspace(1)* %C, <2 x float> addrspace(1)* %in) #0 {
+; CI-LABEL: simple_write2_two_val_subreg2_f32:
+; CI: ; %bb.0:
+; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xb
+; CI-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: s_mov_b32 s2, 0
+; CI-NEXT: v_mov_b32_e32 v2, 0
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: buffer_load_dwordx2 v[1:2], v[1:2], s[0:3], 0 addr64
+; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT: v_add_i32_e32 v0, vcc, lds at abs32@lo, v0
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: s_waitcnt vmcnt(0)
+; CI-NEXT: ds_write2_b32 v0, v1, v2 offset1:8
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: simple_write2_two_val_subreg2_f32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2c
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX9-NEXT: v_mov_b32_e32 v3, lds at abs32@lo
+; GFX9-NEXT: v_lshl_add_u32 v0, v0, 2, v3
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dwordx2 v[1:2], v1, s[0:1]
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: ds_write2_b32 v0, v1, v2 offset1:8
+; GFX9-NEXT: s_endpgm
%x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
%in.gep = getelementptr <2 x float>, <2 x float> addrspace(1)* %in, i32 %x.i
%val = load <2 x float>, <2 x float> addrspace(1)* %in.gep, align 8
@@ -163,20 +272,34 @@ define amdgpu_kernel void @simple_write2_two_val_subreg2_f32(float addrspace(1)*
ret void
}
-; GCN-LABEL: @simple_write2_two_val_subreg4_f32
-; CI-DAG: s_mov_b32 m0
-; GFX9-NOT: m0
-
-; GCN-DAG: {{buffer|global}}_load_dwordx4 v{{\[}}[[VAL0:[0-9]+]]:[[VAL1:[0-9]+]]{{\]}}
-
-; CI-DAG: v_lshlrev_b32_e32 [[VOFS:v[0-9]+]], 2, v{{[0-9]+}}
-; CI-DAG: v_add_i32_e32 [[VPTR:v[0-9]+]], vcc, lds at abs32@lo, [[VOFS]]
-; GFX9-DAG: v_mov_b32_e32 [[VBASE:v[0-9]+]], lds at abs32@lo
-; GFX9-DAG: v_lshl_add_u32 [[VPTR:v[0-9]+]], v{{[0-9]+}}, 2, [[VBASE]]
-
-; GCN: ds_write2_b32 [[VPTR]], v[[VAL0]], v[[VAL1]] offset1:8
-; GCN: s_endpgm
define amdgpu_kernel void @simple_write2_two_val_subreg4_f32(float addrspace(1)* %C, <4 x float> addrspace(1)* %in) #0 {
+; CI-LABEL: simple_write2_two_val_subreg4_f32:
+; CI: ; %bb.0:
+; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xb
+; CI-NEXT: v_lshlrev_b32_e32 v1, 4, v0
+; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: s_mov_b32 s2, 0
+; CI-NEXT: v_mov_b32_e32 v2, 0
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: buffer_load_dwordx4 v[1:4], v[1:2], s[0:3], 0 addr64
+; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT: v_add_i32_e32 v0, vcc, lds at abs32@lo, v0
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: s_waitcnt vmcnt(0)
+; CI-NEXT: ds_write2_b32 v0, v1, v4 offset1:8
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: simple_write2_two_val_subreg4_f32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2c
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 4, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dwordx4 v[1:4], v1, s[0:1]
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v2, lds at abs32@lo
+; GFX9-NEXT: v_lshl_add_u32 v0, v0, 2, v2
+; GFX9-NEXT: ds_write2_b32 v0, v1, v4 offset1:8
+; GFX9-NEXT: s_endpgm
%x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
%in.gep = getelementptr <4 x float>, <4 x float> addrspace(1)* %in, i32 %x.i
%val = load <4 x float>, <4 x float> addrspace(1)* %in.gep, align 16
@@ -190,22 +313,34 @@ define amdgpu_kernel void @simple_write2_two_val_subreg4_f32(float addrspace(1)*
ret void
}
-; GCN-LABEL: @simple_write2_two_val_max_offset_f32
-; CI-DAG: s_mov_b32 m0
-; GFX9-NOT: m0
-
-; CI-DAG: buffer_load_dword [[VAL0:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
-; CI-DAG: buffer_load_dword [[VAL1:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4
-
-; GFX9-DAG: global_load_dword [[VAL0:v[0-9]+]], v{{[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}{{$}}
-; GFX9-DAG: global_load_dword [[VAL1:v[0-9]+]], v{{[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}} offset:4{{$}}
-
-; GCN-DAG: v_lshlrev_b32_e32 [[VBASE:v[0-9]+]], 2, v{{[0-9]+}}
-; GCN-DAG: v_add_{{[ui]}}32_e32 [[VPTR:v[0-9]+]], {{(vcc, )?}}lds at abs32@lo, [[VBASE]]
-
-; GCN: ds_write2_b32 [[VPTR]], [[VAL0]], [[VAL1]] offset1:255
-; GCN: s_endpgm
define amdgpu_kernel void @simple_write2_two_val_max_offset_f32(float addrspace(1)* %C, float addrspace(1)* %in) #0 {
+; CI-LABEL: simple_write2_two_val_max_offset_f32:
+; CI: ; %bb.0:
+; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xb
+; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: s_mov_b32 s2, 0
+; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT: v_mov_b32_e32 v1, 0
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: buffer_load_dword v2, v[0:1], s[0:3], 0 addr64
+; CI-NEXT: buffer_load_dword v1, v[0:1], s[0:3], 0 addr64 offset:4
+; CI-NEXT: v_add_i32_e32 v0, vcc, lds at abs32@lo, v0
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: s_waitcnt vmcnt(0)
+; CI-NEXT: ds_write2_b32 v0, v2, v1 offset1:255
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: simple_write2_two_val_max_offset_f32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2c
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dword v1, v0, s[0:1]
+; GFX9-NEXT: global_load_dword v2, v0, s[0:1] offset:4
+; GFX9-NEXT: v_add_u32_e32 v0, lds at abs32@lo, v0
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: ds_write2_b32 v0, v1, v2 offset1:255
+; GFX9-NEXT: s_endpgm
%x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
%in.gep.0 = getelementptr float, float addrspace(1)* %in, i32 %x.i
%in.gep.1 = getelementptr float, float addrspace(1)* %in.gep.0, i32 1
@@ -219,14 +354,41 @@ define amdgpu_kernel void @simple_write2_two_val_max_offset_f32(float addrspace(
ret void
}
-; GCN-LABEL: @simple_write2_two_val_too_far_f32
-; CI-DAG: s_mov_b32 m0
-; GFX9-NOT: m0
-
-; GCN: ds_write_b32 v{{[0-9]+}}, v{{[0-9]+}}
-; GCN: ds_write_b32 v{{[0-9]+}}, v{{[0-9]+}} offset:1028
-; GCN: s_endpgm
define amdgpu_kernel void @simple_write2_two_val_too_far_f32(float addrspace(1)* %C, float addrspace(1)* %in0, float addrspace(1)* %in1) #0 {
+; CI-LABEL: simple_write2_two_val_too_far_f32:
+; CI: ; %bb.0:
+; CI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0xb
+; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: s_mov_b32 s2, 0
+; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT: v_mov_b32_e32 v1, 0
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: s_mov_b64 s[0:1], s[4:5]
+; CI-NEXT: s_mov_b64 s[4:5], s[6:7]
+; CI-NEXT: s_mov_b64 s[6:7], s[2:3]
+; CI-NEXT: buffer_load_dword v2, v[0:1], s[0:3], 0 addr64
+; CI-NEXT: buffer_load_dword v1, v[0:1], s[4:7], 0 addr64
+; CI-NEXT: v_add_i32_e32 v0, vcc, lds at abs32@lo, v0
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: s_waitcnt vmcnt(1)
+; CI-NEXT: ds_write_b32 v0, v2
+; CI-NEXT: s_waitcnt vmcnt(0)
+; CI-NEXT: ds_write_b32 v0, v1 offset:1028
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: simple_write2_two_val_too_far_f32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x2c
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dword v1, v0, s[0:1]
+; GFX9-NEXT: global_load_dword v2, v0, s[2:3]
+; GFX9-NEXT: v_add_u32_e32 v0, lds at abs32@lo, v0
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: ds_write_b32 v0, v1
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: ds_write_b32 v0, v2 offset:1028
+; GFX9-NEXT: s_endpgm
%x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
%in0.gep = getelementptr float, float addrspace(1)* %in0, i32 %x.i
%in1.gep = getelementptr float, float addrspace(1)* %in1, i32 %x.i
@@ -240,14 +402,39 @@ define amdgpu_kernel void @simple_write2_two_val_too_far_f32(float addrspace(1)*
ret void
}
-; GCN-LABEL: @simple_write2_two_val_f32_x2
-; CI-DAG: s_mov_b32 m0
-; GFX9-NOT: m0
-
-; GCN: ds_write2_b32 [[BASEADDR:v[0-9]+]], [[VAL0:v[0-9]+]], [[VAL1:v[0-9]+]] offset1:8
-; GCN: ds_write2_b32 [[BASEADDR:v[0-9]+]], [[VAL0]], [[VAL1]] offset0:11 offset1:27
-; GCN: s_endpgm
define amdgpu_kernel void @simple_write2_two_val_f32_x2(float addrspace(1)* %C, float addrspace(1)* %in0, float addrspace(1)* %in1) #0 {
+; CI-LABEL: simple_write2_two_val_f32_x2:
+; CI: ; %bb.0:
+; CI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0xb
+; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: s_mov_b32 s2, 0
+; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT: v_mov_b32_e32 v1, 0
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: s_mov_b64 s[0:1], s[4:5]
+; CI-NEXT: s_mov_b64 s[4:5], s[6:7]
+; CI-NEXT: s_mov_b64 s[6:7], s[2:3]
+; CI-NEXT: buffer_load_dword v2, v[0:1], s[0:3], 0 addr64
+; CI-NEXT: buffer_load_dword v1, v[0:1], s[4:7], 0 addr64
+; CI-NEXT: v_add_i32_e32 v0, vcc, lds at abs32@lo, v0
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: s_waitcnt vmcnt(0)
+; CI-NEXT: ds_write2_b32 v0, v2, v1 offset1:8
+; CI-NEXT: ds_write2_b32 v0, v2, v1 offset0:11 offset1:27
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: simple_write2_two_val_f32_x2:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x2c
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dword v1, v0, s[0:1]
+; GFX9-NEXT: global_load_dword v2, v0, s[2:3]
+; GFX9-NEXT: v_add_u32_e32 v0, lds at abs32@lo, v0
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: ds_write2_b32 v0, v1, v2 offset1:8
+; GFX9-NEXT: ds_write2_b32 v0, v1, v2 offset0:11 offset1:27
+; GFX9-NEXT: s_endpgm
%tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1
%in0.gep = getelementptr float, float addrspace(1)* %in0, i32 %tid.x
%in1.gep = getelementptr float, float addrspace(1)* %in1, i32 %tid.x
@@ -273,14 +460,39 @@ define amdgpu_kernel void @simple_write2_two_val_f32_x2(float addrspace(1)* %C,
ret void
}
-; GCN-LABEL: @simple_write2_two_val_f32_x2_nonzero_base
-; CI-DAG: s_mov_b32 m0
-; GFX9-NOT: m0
-
-; GCN: ds_write2_b32 [[BASEADDR:v[0-9]+]], [[VAL0:v[0-9]+]], [[VAL1:v[0-9]+]] offset0:3 offset1:8
-; GCN: ds_write2_b32 [[BASEADDR:v[0-9]+]], [[VAL0]], [[VAL1]] offset0:11 offset1:27
-; GCN: s_endpgm
define amdgpu_kernel void @simple_write2_two_val_f32_x2_nonzero_base(float addrspace(1)* %C, float addrspace(1)* %in0, float addrspace(1)* %in1) #0 {
+; CI-LABEL: simple_write2_two_val_f32_x2_nonzero_base:
+; CI: ; %bb.0:
+; CI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0xb
+; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: s_mov_b32 s2, 0
+; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT: v_mov_b32_e32 v1, 0
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: s_mov_b64 s[0:1], s[4:5]
+; CI-NEXT: s_mov_b64 s[4:5], s[6:7]
+; CI-NEXT: s_mov_b64 s[6:7], s[2:3]
+; CI-NEXT: buffer_load_dword v2, v[0:1], s[0:3], 0 addr64
+; CI-NEXT: buffer_load_dword v1, v[0:1], s[4:7], 0 addr64
+; CI-NEXT: v_add_i32_e32 v0, vcc, lds at abs32@lo, v0
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: s_waitcnt vmcnt(0)
+; CI-NEXT: ds_write2_b32 v0, v2, v1 offset0:3 offset1:8
+; CI-NEXT: ds_write2_b32 v0, v2, v1 offset0:11 offset1:27
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: simple_write2_two_val_f32_x2_nonzero_base:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x2c
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dword v1, v0, s[0:1]
+; GFX9-NEXT: global_load_dword v2, v0, s[2:3]
+; GFX9-NEXT: v_add_u32_e32 v0, lds at abs32@lo, v0
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: ds_write2_b32 v0, v1, v2 offset0:3 offset1:8
+; GFX9-NEXT: ds_write2_b32 v0, v1, v2 offset0:11 offset1:27
+; GFX9-NEXT: s_endpgm
%tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1
%in0.gep = getelementptr float, float addrspace(1)* %in0, i32 %tid.x
%in1.gep = getelementptr float, float addrspace(1)* %in1, i32 %tid.x
@@ -306,15 +518,45 @@ define amdgpu_kernel void @simple_write2_two_val_f32_x2_nonzero_base(float addrs
ret void
}
-; GCN-LABEL: @write2_ptr_subreg_arg_two_val_f32
-; CI-DAG: s_mov_b32 m0
-; GFX9-NOT: m0
-
-; GCN-NOT: ds_write2_b32
-; GCN: ds_write_b32
-; GCN: ds_write_b32
-; GCN: s_endpgm
define amdgpu_kernel void @write2_ptr_subreg_arg_two_val_f32(float addrspace(1)* %C, float addrspace(1)* %in0, float addrspace(1)* %in1, <2 x float addrspace(3)*> %lds.ptr) #0 {
+; CI-LABEL: write2_ptr_subreg_arg_two_val_f32:
+; CI: ; %bb.0:
+; CI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0xb
+; CI-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0xf
+; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: s_mov_b32 s2, 0
+; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: s_mov_b64 s[0:1], s[4:5]
+; CI-NEXT: v_mov_b32_e32 v1, 0
+; CI-NEXT: s_mov_b64 s[4:5], s[6:7]
+; CI-NEXT: s_mov_b64 s[6:7], s[2:3]
+; CI-NEXT: buffer_load_dword v2, v[0:1], s[0:3], 0 addr64
+; CI-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
+; CI-NEXT: v_mov_b32_e32 v1, s8
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: v_mov_b32_e32 v3, s9
+; CI-NEXT: s_waitcnt vmcnt(1)
+; CI-NEXT: ds_write_b32 v1, v2 offset:32
+; CI-NEXT: s_waitcnt vmcnt(0)
+; CI-NEXT: ds_write_b32 v3, v0 offset:32
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: write2_ptr_subreg_arg_two_val_f32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x2c
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x3c
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX9-NEXT: global_load_dword v0, v0, s[6:7]
+; GFX9-NEXT: v_mov_b32_e32 v2, s0
+; GFX9-NEXT: v_mov_b32_e32 v3, s1
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: ds_write_b32 v2, v1 offset:32
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: ds_write_b32 v3, v0 offset:32
+; GFX9-NEXT: s_endpgm
%x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
%in0.gep = getelementptr float, float addrspace(1)* %in0, i32 %x.i
%in1.gep = getelementptr float, float addrspace(1)* %in1, i32 %x.i
@@ -336,15 +578,32 @@ define amdgpu_kernel void @write2_ptr_subreg_arg_two_val_f32(float addrspace(1)*
ret void
}
-; GCN-LABEL: @simple_write2_one_val_f64
-; CI-DAG: s_mov_b32 m0
-; GFX9-NOT: m0
-
-; GCN-DAG: {{buffer|global}}_load_dwordx2 [[VAL:v\[[0-9]+:[0-9]+\]]],
-; GCN-DAG: v_lshlrev_b32_e32 [[VPTR:v[0-9]+]], 3, v{{[0-9]+}}
-; GCN: ds_write2_b64 [[VPTR]], [[VAL]], [[VAL]] offset1:8
-; GCN: s_endpgm
define amdgpu_kernel void @simple_write2_one_val_f64(double addrspace(1)* %C, double addrspace(1)* %in) #0 {
+; CI-LABEL: simple_write2_one_val_f64:
+; CI: ; %bb.0:
+; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xb
+; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: s_mov_b32 s2, 0
+; CI-NEXT: v_lshlrev_b32_e32 v0, 3, v0
+; CI-NEXT: v_mov_b32_e32 v1, 0
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: buffer_load_dwordx2 v[1:2], v[0:1], s[0:3], 0 addr64
+; CI-NEXT: v_add_i32_e32 v0, vcc, lds.f64 at abs32@lo, v0
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: s_waitcnt vmcnt(0)
+; CI-NEXT: ds_write2_b64 v0, v[1:2], v[1:2] offset1:8
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: simple_write2_one_val_f64:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2c
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 3, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[0:1]
+; GFX9-NEXT: v_add_u32_e32 v2, lds.f64 at abs32@lo, v2
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: ds_write2_b64 v2, v[0:1], v[0:1] offset1:8
+; GFX9-NEXT: s_endpgm
%x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
%in.gep = getelementptr double, double addrspace(1)* %in, i32 %x.i
%val = load double, double addrspace(1)* %in.gep, align 8
@@ -356,16 +615,36 @@ define amdgpu_kernel void @simple_write2_one_val_f64(double addrspace(1)* %C, do
ret void
}
-; GCN-LABEL: @misaligned_simple_write2_one_val_f64
-; CI-DAG: s_mov_b32 m0
-; GFX9-NOT: m0
-
-; GCN-DAG: {{buffer|global}}_load_dwordx2 v{{\[}}[[VAL0:[0-9]+]]:[[VAL1:[0-9]+]]{{\]}}
-; GCN-DAG: v_lshlrev_b32_e32 [[VPTR:v[0-9]+]], 3, v{{[0-9]+}}
-; GCN: ds_write2_b32 [[VPTR]], v[[VAL0]], v[[VAL1]] offset1:1
-; GCN: ds_write2_b32 [[VPTR]], v[[VAL0]], v[[VAL1]] offset0:14 offset1:15
-; GCN: s_endpgm
define amdgpu_kernel void @misaligned_simple_write2_one_val_f64(double addrspace(1)* %C, double addrspace(1)* %in, double addrspace(3)* %lds) #0 {
+; CI-LABEL: misaligned_simple_write2_one_val_f64:
+; CI: ; %bb.0:
+; CI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0xb
+; CI-NEXT: s_load_dword s0, s[0:1], 0xd
+; CI-NEXT: s_mov_b32 s7, 0xf000
+; CI-NEXT: s_mov_b32 s6, 0
+; CI-NEXT: v_lshlrev_b32_e32 v0, 3, v0
+; CI-NEXT: v_mov_b32_e32 v1, 0
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: buffer_load_dwordx2 v[1:2], v[0:1], s[4:7], 0 addr64
+; CI-NEXT: v_add_i32_e32 v0, vcc, s0, v0
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: s_waitcnt vmcnt(0)
+; CI-NEXT: ds_write2_b32 v0, v1, v2 offset1:1
+; CI-NEXT: ds_write2_b32 v0, v1, v2 offset0:14 offset1:15
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: misaligned_simple_write2_one_val_f64:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x2c
+; GFX9-NEXT: s_load_dword s0, s[0:1], 0x34
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 3, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[2:3]
+; GFX9-NEXT: v_add_u32_e32 v2, s0, v2
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: ds_write2_b32 v2, v0, v1 offset1:1
+; GFX9-NEXT: ds_write2_b32 v2, v0, v1 offset0:14 offset1:15
+; GFX9-NEXT: s_endpgm
%x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
%in.gep = getelementptr double, double addrspace(1)* %in, i32 %x.i
%val = load double, double addrspace(1)* %in.gep, align 8
@@ -377,21 +656,34 @@ define amdgpu_kernel void @misaligned_simple_write2_one_val_f64(double addrspace
ret void
}
-; GCN-LABEL: @simple_write2_two_val_f64
-; CI-DAG: s_mov_b32 m0
-; GFX9-NOT: m0
-
-; CI-DAG: buffer_load_dwordx2 [[VAL0:v\[[0-9]+:[0-9]+\]]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
-; CI-DAG: buffer_load_dwordx2 [[VAL1:v\[[0-9]+:[0-9]+\]]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8
-
-; GFX9-DAG: global_load_dwordx2 [[VAL0:v\[[0-9]+:[0-9]+\]]], v{{[0-9]+}}, {{s\[[0-9]+:[0-9]+\]$}}
-; GFX9-DAG: global_load_dwordx2 [[VAL1:v\[[0-9]+:[0-9]+\]]], v{{[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}} offset:8
-
-; GCN-DAG: v_lshlrev_b32_e32 [[VBASE:v[0-9]+]], 3, v{{[0-9]+}}
-; GCN-DAG: v_add_{{[ui]}}32_e32 [[VPTR:v[0-9]+]], {{(vcc, )?}}lds.f64 at abs32@lo, [[VBASE]]
-; GCN: ds_write2_b64 [[VPTR]], [[VAL0]], [[VAL1]] offset1:8
-; GCN: s_endpgm
define amdgpu_kernel void @simple_write2_two_val_f64(double addrspace(1)* %C, double addrspace(1)* %in) #0 {
+; CI-LABEL: simple_write2_two_val_f64:
+; CI: ; %bb.0:
+; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xb
+; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: s_mov_b32 s2, 0
+; CI-NEXT: v_lshlrev_b32_e32 v0, 3, v0
+; CI-NEXT: v_mov_b32_e32 v1, 0
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: buffer_load_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
+; CI-NEXT: buffer_load_dwordx2 v[4:5], v[0:1], s[0:3], 0 addr64 offset:8
+; CI-NEXT: v_add_i32_e32 v0, vcc, lds.f64 at abs32@lo, v0
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: s_waitcnt vmcnt(0)
+; CI-NEXT: ds_write2_b64 v0, v[2:3], v[4:5] offset1:8
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: simple_write2_two_val_f64:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2c
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 3, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dwordx2 v[0:1], v4, s[0:1]
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[0:1] offset:8
+; GFX9-NEXT: v_add_u32_e32 v4, lds.f64 at abs32@lo, v4
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: ds_write2_b64 v4, v[0:1], v[2:3] offset1:8
+; GFX9-NEXT: s_endpgm
%x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
%in.gep.0 = getelementptr double, double addrspace(1)* %in, i32 %x.i
%in.gep.1 = getelementptr double, double addrspace(1)* %in.gep.0, i32 1
@@ -407,26 +699,45 @@ define amdgpu_kernel void @simple_write2_two_val_f64(double addrspace(1)* %C, do
@foo = addrspace(3) global [4 x i32] undef, align 4
-; GCN-LABEL: @store_constant_adjacent_offsets
-; CI-DAG: s_mov_b32 m0
-; GFX9-NOT: m0
-
-; GCN-DAG: v_mov_b32_e32 [[PTR:v[0-9]+]], foo at abs32@lo{{$}}
-; GCN: ds_write2_b32 [[PTR]], v{{[0-9]+}}, v{{[0-9]+}} offset1:1
define amdgpu_kernel void @store_constant_adjacent_offsets() {
+; CI-LABEL: store_constant_adjacent_offsets:
+; CI: ; %bb.0:
+; CI-NEXT: s_movk_i32 s0, 0x7b
+; CI-NEXT: v_mov_b32_e32 v0, foo at abs32@lo
+; CI-NEXT: v_mov_b32_e32 v1, s0
+; CI-NEXT: v_mov_b32_e32 v2, s0
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: ds_write2_b32 v0, v1, v2 offset1:1
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: store_constant_adjacent_offsets:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_movk_i32 s0, 0x7b
+; GFX9-NEXT: v_mov_b32_e32 v0, foo at abs32@lo
+; GFX9-NEXT: v_mov_b32_e32 v1, s0
+; GFX9-NEXT: v_mov_b32_e32 v2, s0
+; GFX9-NEXT: ds_write2_b32 v0, v1, v2 offset1:1
+; GFX9-NEXT: s_endpgm
store i32 123, i32 addrspace(3)* getelementptr inbounds ([4 x i32], [4 x i32] addrspace(3)* @foo, i32 0, i32 0), align 4
store i32 123, i32 addrspace(3)* getelementptr inbounds ([4 x i32], [4 x i32] addrspace(3)* @foo, i32 0, i32 1), align 4
ret void
}
-; GCN-LABEL: @store_constant_disjoint_offsets
-; CI-DAG: s_mov_b32 m0
-; GFX9-NOT: m0
-
-; GCN-DAG: v_mov_b32_e32 [[VAL:v[0-9]+]], 0x7b{{$}}
-; GCN-DAG: v_mov_b32_e32 [[PTR:v[0-9]+]], foo at abs32@lo{{$}}
-; GCN: ds_write2_b32 [[PTR]], [[VAL]], [[VAL]] offset1:2
define amdgpu_kernel void @store_constant_disjoint_offsets() {
+; CI-LABEL: store_constant_disjoint_offsets:
+; CI: ; %bb.0:
+; CI-NEXT: v_mov_b32_e32 v0, foo at abs32@lo
+; CI-NEXT: v_mov_b32_e32 v1, 0x7b
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: ds_write2_b32 v0, v1, v1 offset1:2
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: store_constant_disjoint_offsets:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_mov_b32_e32 v0, foo at abs32@lo
+; GFX9-NEXT: v_mov_b32_e32 v1, 0x7b
+; GFX9-NEXT: ds_write2_b32 v0, v1, v1 offset1:2
+; GFX9-NEXT: s_endpgm
store i32 123, i32 addrspace(3)* getelementptr inbounds ([4 x i32], [4 x i32] addrspace(3)* @foo, i32 0, i32 0), align 4
store i32 123, i32 addrspace(3)* getelementptr inbounds ([4 x i32], [4 x i32] addrspace(3)* @foo, i32 0, i32 2), align 4
ret void
@@ -434,21 +745,35 @@ define amdgpu_kernel void @store_constant_disjoint_offsets() {
@bar = addrspace(3) global [4 x i64] undef, align 4
-; GCN-LABEL: @store_misaligned64_constant_offsets
-; CI-DAG: s_mov_b32 m0
-; GFX9-NOT: m0
-
-; GCN-DAG: v_mov_b32_e32 [[PTR:v[0-9]+]], bar at abs32@lo{{$}}
-; CI-DAG: ds_write2_b32 [[PTR]], v{{[0-9]+}}, v{{[0-9]+}} offset1:1
-; CI-DAG: ds_write2_b32 [[PTR]], v{{[0-9]+}}, v{{[0-9]+}} offset0:2 offset1:3
-
-; GFX9-ALIGNED-DAG: ds_write2_b32 [[PTR]], v{{[0-9]+}}, v{{[0-9]+}} offset1:1
-; GFX9-ALIGNED-DAG: ds_write2_b32 [[PTR]], v{{[0-9]+}}, v{{[0-9]+}} offset0:2 offset1:3
-
-; GFX9-UNALIGNED: ds_write_b128 [[PTR]], {{v\[[0-9]+:[0-9]+\]}}
-
-; GCN: s_endpgm
define amdgpu_kernel void @store_misaligned64_constant_offsets() {
+; CI-LABEL: store_misaligned64_constant_offsets:
+; CI: ; %bb.0:
+; CI-NEXT: v_mov_b32_e32 v0, bar at abs32@lo
+; CI-NEXT: v_mov_b32_e32 v1, 0x7b
+; CI-NEXT: v_mov_b32_e32 v2, 0
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: ds_write2_b32 v0, v1, v2 offset1:1
+; CI-NEXT: ds_write2_b32 v0, v1, v2 offset0:2 offset1:3
+; CI-NEXT: s_endpgm
+;
+; GFX9-ALIGNED-LABEL: store_misaligned64_constant_offsets:
+; GFX9-ALIGNED: ; %bb.0:
+; GFX9-ALIGNED-NEXT: v_mov_b32_e32 v0, bar at abs32@lo
+; GFX9-ALIGNED-NEXT: v_mov_b32_e32 v1, 0x7b
+; GFX9-ALIGNED-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-ALIGNED-NEXT: ds_write2_b32 v0, v1, v2 offset1:1
+; GFX9-ALIGNED-NEXT: ds_write2_b32 v0, v1, v2 offset0:2 offset1:3
+; GFX9-ALIGNED-NEXT: s_endpgm
+;
+; GFX9-UNALIGNED-LABEL: store_misaligned64_constant_offsets:
+; GFX9-UNALIGNED: ; %bb.0:
+; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v0, 0x7b
+; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v2, v0
+; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v3, v1
+; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v4, bar at abs32@lo
+; GFX9-UNALIGNED-NEXT: ds_write_b128 v4, v[0:3]
+; GFX9-UNALIGNED-NEXT: s_endpgm
store i64 123, i64 addrspace(3)* getelementptr inbounds ([4 x i64], [4 x i64] addrspace(3)* @bar, i32 0, i32 0), align 4
store i64 123, i64 addrspace(3)* getelementptr inbounds ([4 x i64], [4 x i64] addrspace(3)* @bar, i32 0, i32 1), align 4
ret void
@@ -456,19 +781,33 @@ define amdgpu_kernel void @store_misaligned64_constant_offsets() {
@bar.large = addrspace(3) global [4096 x i64] undef, align 4
-; GCN-LABEL: @store_misaligned64_constant_large_offsets
-; CI-DAG: s_mov_b32 m0
-; GFX9-NOT: m0
-
-; GCN-DAG: s_mov_b32 [[SBASE0:s[0-9]+]], bar.large at abs32@lo
-; GCN-DAG: s_add_i32 [[SBASE1:s[0-9]+]], [[SBASE0]], 0x4000{{$}}
-; GCN-DAG: s_addk_i32 [[SBASE0]], 0x7ff8{{$}}
-; GCN-DAG: v_mov_b32_e32 [[VBASE0:v[0-9]+]], [[SBASE0]]{{$}}
-; GCN-DAG: v_mov_b32_e32 [[VBASE1:v[0-9]+]], [[SBASE1]]{{$}}
-; GCN-DAG: ds_write2_b32 [[VBASE0]], v{{[0-9]+}}, v{{[0-9]+}} offset1:1
-; GCN-DAG: ds_write2_b32 [[VBASE1]], v{{[0-9]+}}, v{{[0-9]+}} offset1:1
-; GCN: s_endpgm
define amdgpu_kernel void @store_misaligned64_constant_large_offsets() {
+; CI-LABEL: store_misaligned64_constant_large_offsets:
+; CI: ; %bb.0:
+; CI-NEXT: s_mov_b32 s0, bar.large at abs32@lo
+; CI-NEXT: s_add_i32 s1, s0, 0x4000
+; CI-NEXT: v_mov_b32_e32 v0, s1
+; CI-NEXT: v_mov_b32_e32 v1, 0x7b
+; CI-NEXT: v_mov_b32_e32 v2, 0
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: s_addk_i32 s0, 0x7ff8
+; CI-NEXT: ds_write2_b32 v0, v1, v2 offset1:1
+; CI-NEXT: v_mov_b32_e32 v0, s0
+; CI-NEXT: ds_write2_b32 v0, v1, v2 offset1:1
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: store_misaligned64_constant_large_offsets:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_mov_b32 s0, bar.large at abs32@lo
+; GFX9-NEXT: s_add_i32 s1, s0, 0x4000
+; GFX9-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-NEXT: v_mov_b32_e32 v1, 0x7b
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: s_addk_i32 s0, 0x7ff8
+; GFX9-NEXT: ds_write2_b32 v0, v1, v2 offset1:1
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: ds_write2_b32 v0, v1, v2 offset1:1
+; GFX9-NEXT: s_endpgm
store i64 123, i64 addrspace(3)* getelementptr inbounds ([4096 x i64], [4096 x i64] addrspace(3)* @bar.large, i32 0, i32 2048), align 4
store i64 123, i64 addrspace(3)* getelementptr inbounds ([4096 x i64], [4096 x i64] addrspace(3)* @bar.large, i32 0, i32 4095), align 4
ret void
@@ -478,6 +817,48 @@ define amdgpu_kernel void @store_misaligned64_constant_large_offsets() {
@sgemm.lB = internal unnamed_addr addrspace(3) global [776 x float] undef, align 4
define amdgpu_kernel void @write2_sgemm_sequence(float addrspace(1)* %C, i32 %lda, i32 %ldb, float addrspace(1)* %in) #0 {
+; CI-LABEL: write2_sgemm_sequence:
+; CI: ; %bb.0:
+; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
+; CI-NEXT: s_lshl_b32 s2, s2, 2
+; CI-NEXT: s_add_i32 s3, s2, 0xc20
+; CI-NEXT: v_mov_b32_e32 v0, s3
+; CI-NEXT: s_addk_i32 s2, 0xc60
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: s_load_dword s0, s[0:1], 0x0
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_mov_b32_e32 v2, s0
+; CI-NEXT: v_mov_b32_e32 v3, s0
+; CI-NEXT: ds_write2_b32 v0, v2, v3 offset1:1
+; CI-NEXT: v_mov_b32_e32 v0, s2
+; CI-NEXT: ds_write2_b32 v0, v2, v3 offset1:1
+; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v1
+; CI-NEXT: ds_write2_b32 v0, v2, v3 offset1:1
+; CI-NEXT: ds_write2_b32 v0, v2, v3 offset0:32 offset1:33
+; CI-NEXT: ds_write2_b32 v0, v2, v3 offset0:64 offset1:65
+; CI-NEXT: s_endpgm
+;
+; GFX9-LABEL: write2_sgemm_sequence:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX9-NEXT: s_lshl_b32 s2, s2, 2
+; GFX9-NEXT: s_add_i32 s3, s2, 0xc20
+; GFX9-NEXT: s_addk_i32 s2, 0xc60
+; GFX9-NEXT: v_mov_b32_e32 v0, s3
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dword s0, s[0:1], 0x0
+; GFX9-NEXT: v_mov_b32_e32 v2, s2
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v3, s0
+; GFX9-NEXT: v_mov_b32_e32 v4, s0
+; GFX9-NEXT: ds_write2_b32 v0, v3, v4 offset1:1
+; GFX9-NEXT: ds_write2_b32 v2, v3, v4 offset1:1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v1
+; GFX9-NEXT: ds_write2_b32 v0, v3, v4 offset1:1
+; GFX9-NEXT: ds_write2_b32 v0, v3, v4 offset0:32 offset1:33
+; GFX9-NEXT: ds_write2_b32 v0, v3, v4 offset0:64 offset1:65
+; GFX9-NEXT: s_endpgm
%x.i = tail call i32 @llvm.amdgcn.workgroup.id.x() #1
%y.i = tail call i32 @llvm.amdgcn.workitem.id.y() #1
%val = load float, float addrspace(1)* %in
@@ -512,18 +893,55 @@ define amdgpu_kernel void @write2_sgemm_sequence(float addrspace(1)* %C, i32 %ld
ret void
}
-; GCN-LABEL: {{^}}simple_write2_v4f32_superreg_align4:
-; CI: s_mov_b32 m0
-; GFX9-NOT: m0
-
-; CI: ds_write2_b32 {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}} offset1:1{{$}}
-; CI: ds_write2_b32 {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}} offset0:2 offset1:3{{$}}
-
-; GFX9-ALIGNED-DAG: ds_write2_b32 {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}} offset1:1{{$}}
-; GFX9-ALIGNED-DAG: ds_write2_b32 {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}} offset0:2 offset1:3{{$}}
-
-; GFX9-UNALIGNED: ds_write_b128 {{v[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}
define amdgpu_kernel void @simple_write2_v4f32_superreg_align4(<4 x float> addrspace(3)* %out, <4 x float> addrspace(1)* %in) #0 {
+; CI-LABEL: simple_write2_v4f32_superreg_align4:
+; CI: ; %bb.0:
+; CI-NEXT: s_load_dword s4, s[0:1], 0x9
+; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xb
+; CI-NEXT: v_lshlrev_b32_e32 v0, 4, v0
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_add_i32_e32 v0, vcc, s4, v0
+; CI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_mov_b32_e32 v1, s0
+; CI-NEXT: v_mov_b32_e32 v2, s1
+; CI-NEXT: ds_write2_b32 v0, v1, v2 offset1:1
+; CI-NEXT: v_mov_b32_e32 v3, s2
+; CI-NEXT: v_mov_b32_e32 v1, s3
+; CI-NEXT: ds_write2_b32 v0, v3, v1 offset0:2 offset1:3
+; CI-NEXT: s_endpgm
+;
+; GFX9-ALIGNED-LABEL: simple_write2_v4f32_superreg_align4:
+; GFX9-ALIGNED: ; %bb.0:
+; GFX9-ALIGNED-NEXT: s_load_dword s4, s[0:1], 0x24
+; GFX9-ALIGNED-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2c
+; GFX9-ALIGNED-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-ALIGNED-NEXT: v_lshl_add_u32 v0, v0, 4, s4
+; GFX9-ALIGNED-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
+; GFX9-ALIGNED-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-ALIGNED-NEXT: v_mov_b32_e32 v1, s0
+; GFX9-ALIGNED-NEXT: v_mov_b32_e32 v2, s1
+; GFX9-ALIGNED-NEXT: v_mov_b32_e32 v3, s2
+; GFX9-ALIGNED-NEXT: v_mov_b32_e32 v4, s3
+; GFX9-ALIGNED-NEXT: ds_write2_b32 v0, v1, v2 offset1:1
+; GFX9-ALIGNED-NEXT: ds_write2_b32 v0, v3, v4 offset0:2 offset1:3
+; GFX9-ALIGNED-NEXT: s_endpgm
+;
+; GFX9-UNALIGNED-LABEL: simple_write2_v4f32_superreg_align4:
+; GFX9-UNALIGNED: ; %bb.0:
+; GFX9-UNALIGNED-NEXT: s_load_dword s4, s[0:1], 0x24
+; GFX9-UNALIGNED-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2c
+; GFX9-UNALIGNED-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-UNALIGNED-NEXT: v_lshl_add_u32 v4, v0, 4, s4
+; GFX9-UNALIGNED-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
+; GFX9-UNALIGNED-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v1, s1
+; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v2, s2
+; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v3, s3
+; GFX9-UNALIGNED-NEXT: ds_write_b128 v4, v[0:3]
+; GFX9-UNALIGNED-NEXT: s_endpgm
%x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
%in.gep = getelementptr inbounds <4 x float>, <4 x float> addrspace(1)* %in
%val0 = load <4 x float>, <4 x float> addrspace(1)* %in.gep, align 4
@@ -534,14 +952,47 @@ define amdgpu_kernel void @simple_write2_v4f32_superreg_align4(<4 x float> addrs
@v2i32_align1 = internal addrspace(3) global [100 x <2 x i32>] undef, align 1
-; GCN-LABEL: {{^}}write2_v2i32_align1_odd_offset:
-; CI-COUNT-8: ds_write_b8
-
-; GFX9-ALIGNED-COUNT-8: ds_write_b8
-
-; GFX9-UNALIGNED: v_mov_b32_e32 [[BASE_ADDR:v[0-9]+]], 0x41{{$}}
-; GFX9-UNALIGNED: ds_write2_b32 [[BASE_ADDR]], v{{[0-9]+}}, v{{[0-9]+}} offset1:1{{$}}
define amdgpu_kernel void @write2_v2i32_align1_odd_offset() {
+; CI-LABEL: write2_v2i32_align1_odd_offset:
+; CI: ; %bb.0: ; %entry
+; CI-NEXT: v_mov_b32_e32 v0, 0x7b
+; CI-NEXT: v_mov_b32_e32 v1, 0
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: ds_write_b8 v1, v0 offset:65
+; CI-NEXT: v_mov_b32_e32 v0, 1
+; CI-NEXT: ds_write_b8 v1, v0 offset:70
+; CI-NEXT: v_mov_b32_e32 v0, 0xc8
+; CI-NEXT: ds_write_b8 v1, v0 offset:69
+; CI-NEXT: ds_write_b8 v1, v1 offset:68
+; CI-NEXT: ds_write_b8 v1, v1 offset:67
+; CI-NEXT: ds_write_b8 v1, v1 offset:66
+; CI-NEXT: ds_write_b8 v1, v1 offset:72
+; CI-NEXT: ds_write_b8 v1, v1 offset:71
+; CI-NEXT: s_endpgm
+;
+; GFX9-ALIGNED-LABEL: write2_v2i32_align1_odd_offset:
+; GFX9-ALIGNED: ; %bb.0: ; %entry
+; GFX9-ALIGNED-NEXT: v_mov_b32_e32 v0, 0x7b
+; GFX9-ALIGNED-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-ALIGNED-NEXT: ds_write_b8 v1, v0 offset:65
+; GFX9-ALIGNED-NEXT: v_mov_b32_e32 v0, 1
+; GFX9-ALIGNED-NEXT: ds_write_b8 v1, v0 offset:70
+; GFX9-ALIGNED-NEXT: v_mov_b32_e32 v0, 0xc8
+; GFX9-ALIGNED-NEXT: ds_write_b8 v1, v0 offset:69
+; GFX9-ALIGNED-NEXT: ds_write_b8 v1, v1 offset:68
+; GFX9-ALIGNED-NEXT: ds_write_b8 v1, v1 offset:67
+; GFX9-ALIGNED-NEXT: ds_write_b8 v1, v1 offset:66
+; GFX9-ALIGNED-NEXT: ds_write_b8 v1, v1 offset:72
+; GFX9-ALIGNED-NEXT: ds_write_b8 v1, v1 offset:71
+; GFX9-ALIGNED-NEXT: s_endpgm
+;
+; GFX9-UNALIGNED-LABEL: write2_v2i32_align1_odd_offset:
+; GFX9-UNALIGNED: ; %bb.0: ; %entry
+; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v0, 0x41
+; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v1, 0x7b
+; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v2, 0x1c8
+; GFX9-UNALIGNED-NEXT: ds_write2_b32 v0, v1, v2 offset1:1
+; GFX9-UNALIGNED-NEXT: s_endpgm
entry:
store <2 x i32> <i32 123, i32 456>, <2 x i32> addrspace(3)* bitcast (i8 addrspace(3)* getelementptr (i8, i8 addrspace(3)* bitcast ([100 x <2 x i32>] addrspace(3)* @v2i32_align1 to i8 addrspace(3)*), i32 65) to <2 x i32> addrspace(3)*), align 1
ret void
More information about the llvm-commits
mailing list