[llvm] c3a146a - AMDGPU/GlobalISel: RegBankLegalize rules for sched barriers intrinsics (#203425)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Jun 12 08:08:26 PDT 2026
Author: vangthao95
Date: 2026-06-12T08:08:21-07:00
New Revision: c3a146a4e8fab800c621acc401dbdf1d0c960be8
URL: https://github.com/llvm/llvm-project/commit/c3a146a4e8fab800c621acc401dbdf1d0c960be8
DIFF: https://github.com/llvm/llvm-project/commit/c3a146a4e8fab800c621acc401dbdf1d0c960be8.diff
LOG: AMDGPU/GlobalISel: RegBankLegalize rules for sched barriers intrinsics (#203425)
Add rules for sched barrier intrinsics. Note, there are regressions due
to AGPR results being copied back to VGPR un-necessarily. That will be
addressed in a future follow-up patch.
Added:
Modified:
llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.barrier.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
index 76a7ed6e54299..73cf73211780b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
@@ -1775,6 +1775,7 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
// Intrinsics with no register operands.
addRulesForIOpcs({amdgcn_asyncmark,
amdgcn_endpgm,
+ amdgcn_iglp_opt,
amdgcn_init_exec,
amdgcn_s_barrier,
amdgcn_s_barrier_leave,
@@ -1799,6 +1800,8 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
amdgcn_s_wait_storecnt,
amdgcn_s_wait_tensorcnt,
amdgcn_s_waitcnt,
+ amdgcn_sched_barrier,
+ amdgcn_sched_group_barrier,
amdgcn_unreachable,
amdgcn_wait_asyncmark,
amdgcn_wave_barrier})
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.ll
index a10c99070d8e1..6bf7e08ce9b34 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -amdgpu-mfma-vgpr-form=0 < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx90a -amdgpu-mfma-vgpr-form=0 < %s | FileCheck -check-prefixes=GCN,SDAG %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx90a -amdgpu-mfma-vgpr-form=0 < %s | FileCheck -check-prefixes=GCN,GISEL %s
define amdgpu_kernel void @test_iglp_opt() #0 {
; GCN-LABEL: test_iglp_opt:
@@ -12,111 +13,378 @@ entry:
}
define amdgpu_kernel void @test_iglp_opt_mfma_gemm(ptr addrspace(3) noalias %in, ptr addrspace(3) noalias %out) #0 {
-; GCN-LABEL: test_iglp_opt_mfma_gemm:
-; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: v_lshlrev_b32_e32 v0, 7, v0
-; GCN-NEXT: v_and_b32_e32 v0, 0x1ff80, v0
-; GCN-NEXT: v_mov_b32_e32 v3, 2.0
-; GCN-NEXT: ; iglp_opt mask(0x00000000)
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_add_u32_e32 v1, s0, v0
-; GCN-NEXT: v_add_u32_e32 v2, 0x6000, v1
-; GCN-NEXT: ds_read_b128 a[28:31], v2 offset:57456
-; GCN-NEXT: ds_read_b128 a[24:27], v2 offset:57440
-; GCN-NEXT: ds_read_b128 a[20:23], v2 offset:57424
-; GCN-NEXT: ds_read_b128 a[16:19], v2 offset:57408
-; GCN-NEXT: ds_read_b128 a[0:3], v2 offset:57344
-; GCN-NEXT: ds_read_b128 a[4:7], v2 offset:57360
-; GCN-NEXT: ds_read_b128 a[8:11], v2 offset:57376
-; GCN-NEXT: ds_read_b128 a[12:15], v2 offset:57392
-; GCN-NEXT: v_mov_b32_e32 v2, 1.0
-; GCN-NEXT: ds_read_b128 a[60:63], v1 offset:49264
-; GCN-NEXT: ds_read_b128 a[56:59], v1 offset:49248
-; GCN-NEXT: ds_read_b128 a[52:55], v1 offset:49232
-; GCN-NEXT: ds_read_b128 a[48:51], v1 offset:49216
-; GCN-NEXT: ds_read_b128 a[44:47], v1 offset:49200
-; GCN-NEXT: ds_read_b128 a[40:43], v1 offset:49184
-; GCN-NEXT: ds_read_b128 a[36:39], v1 offset:49168
-; GCN-NEXT: ds_read_b128 a[32:35], v1 offset:49152
-; GCN-NEXT: s_waitcnt lgkmcnt(8)
-; GCN-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
-; GCN-NEXT: ds_read_b128 a[156:159], v1 offset:112
-; GCN-NEXT: ds_read_b128 a[152:155], v1 offset:96
-; GCN-NEXT: ds_read_b128 a[68:71], v1 offset:24592
-; GCN-NEXT: ds_read_b128 a[64:67], v1 offset:24576
-; GCN-NEXT: v_add_u32_e32 v0, s1, v0
-; GCN-NEXT: s_waitcnt lgkmcnt(4)
-; GCN-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v2, v3, a[32:63]
-; GCN-NEXT: ds_read_b128 a[148:151], v1 offset:80
-; GCN-NEXT: ds_read_b128 a[144:147], v1 offset:64
-; GCN-NEXT: ds_read_b128 a[128:131], v1
-; GCN-NEXT: ds_read_b128 a[132:135], v1 offset:16
-; GCN-NEXT: ds_read_b128 a[136:139], v1 offset:32
-; GCN-NEXT: ds_read_b128 a[140:143], v1 offset:48
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v2, v3, a[128:159]
-; GCN-NEXT: ds_read_b128 a[124:127], v1 offset:8304
-; GCN-NEXT: ds_read_b128 a[120:123], v1 offset:8288
-; GCN-NEXT: ds_read_b128 a[116:119], v1 offset:8272
-; GCN-NEXT: ds_read_b128 a[112:115], v1 offset:8256
-; GCN-NEXT: ds_read_b128 a[108:111], v1 offset:8240
-; GCN-NEXT: ds_read_b128 a[104:107], v1 offset:8224
-; GCN-NEXT: ds_read_b128 a[100:103], v1 offset:8208
-; GCN-NEXT: ds_read_b128 a[96:99], v1 offset:8192
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v2, v3, a[96:127]
-; GCN-NEXT: ds_read_b128 a[92:95], v1 offset:24688
-; GCN-NEXT: ds_read_b128 a[88:91], v1 offset:24672
-; GCN-NEXT: ds_read_b128 a[84:87], v1 offset:24656
-; GCN-NEXT: ds_read_b128 a[80:83], v1 offset:24640
-; GCN-NEXT: ds_read_b128 a[76:79], v1 offset:24624
-; GCN-NEXT: ds_read_b128 a[72:75], v1 offset:24608
-; GCN-NEXT: s_nop 2
-; GCN-NEXT: ds_write_b128 v0, a[156:159] offset:112
-; GCN-NEXT: ds_write_b128 v0, a[152:155] offset:96
-; GCN-NEXT: ds_write_b128 v0, a[148:151] offset:80
-; GCN-NEXT: ds_write_b128 v0, a[144:147] offset:64
-; GCN-NEXT: ds_write_b128 v0, a[140:143] offset:48
-; GCN-NEXT: ds_write_b128 v0, a[136:139] offset:32
-; GCN-NEXT: ds_write_b128 v0, a[132:135] offset:16
-; GCN-NEXT: ds_write_b128 v0, a[128:131]
-; GCN-NEXT: v_mov_b32_e32 v0, s1
-; GCN-NEXT: s_waitcnt lgkmcnt(8)
-; GCN-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v2, v3, a[64:95]
-; GCN-NEXT: ds_write_b128 v0, a[56:59] offset:24672
-; GCN-NEXT: ds_write_b128 v0, a[60:63] offset:24688
-; GCN-NEXT: ds_write_b128 v0, a[48:51] offset:24640
-; GCN-NEXT: ds_write_b128 v0, a[120:123] offset:8288
-; GCN-NEXT: ds_write_b128 v0, a[124:127] offset:8304
-; GCN-NEXT: ds_write_b128 v0, a[112:115] offset:8256
-; GCN-NEXT: ds_write_b128 v0, a[116:119] offset:8272
-; GCN-NEXT: ds_write_b128 v0, a[104:107] offset:8224
-; GCN-NEXT: ds_write_b128 v0, a[108:111] offset:8240
-; GCN-NEXT: ds_write_b128 v0, a[96:99] offset:8192
-; GCN-NEXT: ds_write_b128 v0, a[100:103] offset:8208
-; GCN-NEXT: ds_write_b128 v0, a[52:55] offset:24656
-; GCN-NEXT: ds_write_b128 v0, a[40:43] offset:24608
-; GCN-NEXT: ds_write_b128 v0, a[44:47] offset:24624
-; GCN-NEXT: ds_write_b128 v0, a[32:35] offset:24576
-; GCN-NEXT: ds_write_b128 v0, a[36:39] offset:24592
-; GCN-NEXT: ds_write_b128 v0, a[24:27] offset:32864
-; GCN-NEXT: ds_write_b128 v0, a[28:31] offset:32880
-; GCN-NEXT: ds_write_b128 v0, a[16:19] offset:32832
-; GCN-NEXT: ds_write_b128 v0, a[88:91] offset:16480
-; GCN-NEXT: ds_write_b128 v0, a[92:95] offset:16496
-; GCN-NEXT: ds_write_b128 v0, a[80:83] offset:16448
-; GCN-NEXT: ds_write_b128 v0, a[84:87] offset:16464
-; GCN-NEXT: ds_write_b128 v0, a[72:75] offset:16416
-; GCN-NEXT: ds_write_b128 v0, a[76:79] offset:16432
-; GCN-NEXT: ds_write_b128 v0, a[64:67] offset:16384
-; GCN-NEXT: ds_write_b128 v0, a[68:71] offset:16400
-; GCN-NEXT: ds_write_b128 v0, a[20:23] offset:32848
-; GCN-NEXT: ds_write_b128 v0, a[8:11] offset:32800
-; GCN-NEXT: ds_write_b128 v0, a[12:15] offset:32816
-; GCN-NEXT: ds_write_b128 v0, a[0:3] offset:32768
-; GCN-NEXT: ds_write_b128 v0, a[4:7] offset:32784
-; GCN-NEXT: s_endpgm
+; SDAG-LABEL: test_iglp_opt_mfma_gemm:
+; SDAG: ; %bb.0: ; %entry
+; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-NEXT: v_lshlrev_b32_e32 v0, 7, v0
+; SDAG-NEXT: v_and_b32_e32 v0, 0x1ff80, v0
+; SDAG-NEXT: v_mov_b32_e32 v3, 2.0
+; SDAG-NEXT: ; iglp_opt mask(0x00000000)
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_add_u32_e32 v1, s0, v0
+; SDAG-NEXT: v_add_u32_e32 v2, 0x6000, v1
+; SDAG-NEXT: ds_read_b128 a[28:31], v2 offset:57456
+; SDAG-NEXT: ds_read_b128 a[24:27], v2 offset:57440
+; SDAG-NEXT: ds_read_b128 a[20:23], v2 offset:57424
+; SDAG-NEXT: ds_read_b128 a[16:19], v2 offset:57408
+; SDAG-NEXT: ds_read_b128 a[0:3], v2 offset:57344
+; SDAG-NEXT: ds_read_b128 a[4:7], v2 offset:57360
+; SDAG-NEXT: ds_read_b128 a[8:11], v2 offset:57376
+; SDAG-NEXT: ds_read_b128 a[12:15], v2 offset:57392
+; SDAG-NEXT: v_mov_b32_e32 v2, 1.0
+; SDAG-NEXT: ds_read_b128 a[60:63], v1 offset:49264
+; SDAG-NEXT: ds_read_b128 a[56:59], v1 offset:49248
+; SDAG-NEXT: ds_read_b128 a[52:55], v1 offset:49232
+; SDAG-NEXT: ds_read_b128 a[48:51], v1 offset:49216
+; SDAG-NEXT: ds_read_b128 a[44:47], v1 offset:49200
+; SDAG-NEXT: ds_read_b128 a[40:43], v1 offset:49184
+; SDAG-NEXT: ds_read_b128 a[36:39], v1 offset:49168
+; SDAG-NEXT: ds_read_b128 a[32:35], v1 offset:49152
+; SDAG-NEXT: s_waitcnt lgkmcnt(8)
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
+; SDAG-NEXT: ds_read_b128 a[156:159], v1 offset:112
+; SDAG-NEXT: ds_read_b128 a[152:155], v1 offset:96
+; SDAG-NEXT: ds_read_b128 a[68:71], v1 offset:24592
+; SDAG-NEXT: ds_read_b128 a[64:67], v1 offset:24576
+; SDAG-NEXT: v_add_u32_e32 v0, s1, v0
+; SDAG-NEXT: s_waitcnt lgkmcnt(4)
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v2, v3, a[32:63]
+; SDAG-NEXT: ds_read_b128 a[148:151], v1 offset:80
+; SDAG-NEXT: ds_read_b128 a[144:147], v1 offset:64
+; SDAG-NEXT: ds_read_b128 a[128:131], v1
+; SDAG-NEXT: ds_read_b128 a[132:135], v1 offset:16
+; SDAG-NEXT: ds_read_b128 a[136:139], v1 offset:32
+; SDAG-NEXT: ds_read_b128 a[140:143], v1 offset:48
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v2, v3, a[128:159]
+; SDAG-NEXT: ds_read_b128 a[124:127], v1 offset:8304
+; SDAG-NEXT: ds_read_b128 a[120:123], v1 offset:8288
+; SDAG-NEXT: ds_read_b128 a[116:119], v1 offset:8272
+; SDAG-NEXT: ds_read_b128 a[112:115], v1 offset:8256
+; SDAG-NEXT: ds_read_b128 a[108:111], v1 offset:8240
+; SDAG-NEXT: ds_read_b128 a[104:107], v1 offset:8224
+; SDAG-NEXT: ds_read_b128 a[100:103], v1 offset:8208
+; SDAG-NEXT: ds_read_b128 a[96:99], v1 offset:8192
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v2, v3, a[96:127]
+; SDAG-NEXT: ds_read_b128 a[92:95], v1 offset:24688
+; SDAG-NEXT: ds_read_b128 a[88:91], v1 offset:24672
+; SDAG-NEXT: ds_read_b128 a[84:87], v1 offset:24656
+; SDAG-NEXT: ds_read_b128 a[80:83], v1 offset:24640
+; SDAG-NEXT: ds_read_b128 a[76:79], v1 offset:24624
+; SDAG-NEXT: ds_read_b128 a[72:75], v1 offset:24608
+; SDAG-NEXT: s_nop 2
+; SDAG-NEXT: ds_write_b128 v0, a[156:159] offset:112
+; SDAG-NEXT: ds_write_b128 v0, a[152:155] offset:96
+; SDAG-NEXT: ds_write_b128 v0, a[148:151] offset:80
+; SDAG-NEXT: ds_write_b128 v0, a[144:147] offset:64
+; SDAG-NEXT: ds_write_b128 v0, a[140:143] offset:48
+; SDAG-NEXT: ds_write_b128 v0, a[136:139] offset:32
+; SDAG-NEXT: ds_write_b128 v0, a[132:135] offset:16
+; SDAG-NEXT: ds_write_b128 v0, a[128:131]
+; SDAG-NEXT: v_mov_b32_e32 v0, s1
+; SDAG-NEXT: s_waitcnt lgkmcnt(8)
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v2, v3, a[64:95]
+; SDAG-NEXT: ds_write_b128 v0, a[56:59] offset:24672
+; SDAG-NEXT: ds_write_b128 v0, a[60:63] offset:24688
+; SDAG-NEXT: ds_write_b128 v0, a[48:51] offset:24640
+; SDAG-NEXT: ds_write_b128 v0, a[120:123] offset:8288
+; SDAG-NEXT: ds_write_b128 v0, a[124:127] offset:8304
+; SDAG-NEXT: ds_write_b128 v0, a[112:115] offset:8256
+; SDAG-NEXT: ds_write_b128 v0, a[116:119] offset:8272
+; SDAG-NEXT: ds_write_b128 v0, a[104:107] offset:8224
+; SDAG-NEXT: ds_write_b128 v0, a[108:111] offset:8240
+; SDAG-NEXT: ds_write_b128 v0, a[96:99] offset:8192
+; SDAG-NEXT: ds_write_b128 v0, a[100:103] offset:8208
+; SDAG-NEXT: ds_write_b128 v0, a[52:55] offset:24656
+; SDAG-NEXT: ds_write_b128 v0, a[40:43] offset:24608
+; SDAG-NEXT: ds_write_b128 v0, a[44:47] offset:24624
+; SDAG-NEXT: ds_write_b128 v0, a[32:35] offset:24576
+; SDAG-NEXT: ds_write_b128 v0, a[36:39] offset:24592
+; SDAG-NEXT: ds_write_b128 v0, a[24:27] offset:32864
+; SDAG-NEXT: ds_write_b128 v0, a[28:31] offset:32880
+; SDAG-NEXT: ds_write_b128 v0, a[16:19] offset:32832
+; SDAG-NEXT: ds_write_b128 v0, a[88:91] offset:16480
+; SDAG-NEXT: ds_write_b128 v0, a[92:95] offset:16496
+; SDAG-NEXT: ds_write_b128 v0, a[80:83] offset:16448
+; SDAG-NEXT: ds_write_b128 v0, a[84:87] offset:16464
+; SDAG-NEXT: ds_write_b128 v0, a[72:75] offset:16416
+; SDAG-NEXT: ds_write_b128 v0, a[76:79] offset:16432
+; SDAG-NEXT: ds_write_b128 v0, a[64:67] offset:16384
+; SDAG-NEXT: ds_write_b128 v0, a[68:71] offset:16400
+; SDAG-NEXT: ds_write_b128 v0, a[20:23] offset:32848
+; SDAG-NEXT: ds_write_b128 v0, a[8:11] offset:32800
+; SDAG-NEXT: ds_write_b128 v0, a[12:15] offset:32816
+; SDAG-NEXT: ds_write_b128 v0, a[0:3] offset:32768
+; SDAG-NEXT: ds_write_b128 v0, a[4:7] offset:32784
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: test_iglp_opt_mfma_gemm:
+; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GISEL-NEXT: v_lshlrev_b32_e32 v33, 7, v0
+; GISEL-NEXT: v_mov_b32_e32 v32, 1.0
+; GISEL-NEXT: v_mov_b32_e32 v34, 2.0
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: v_add_u32_e32 v0, s0, v33
+; GISEL-NEXT: ds_read_b128 a[96:99], v0 offset:8192
+; GISEL-NEXT: ds_read_b128 a[100:103], v0 offset:8208
+; GISEL-NEXT: ds_read_b128 a[104:107], v0 offset:8224
+; GISEL-NEXT: ds_read_b128 a[108:111], v0 offset:8240
+; GISEL-NEXT: ds_read_b128 a[112:115], v0 offset:8256
+; GISEL-NEXT: ds_read_b128 a[116:119], v0 offset:8272
+; GISEL-NEXT: ds_read_b128 a[120:123], v0 offset:8288
+; GISEL-NEXT: ds_read_b128 a[124:127], v0 offset:8304
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v32, v34, a[96:127]
+; GISEL-NEXT: ds_read_b128 a[0:3], v0 offset:24576
+; GISEL-NEXT: ds_read_b128 a[4:7], v0 offset:24592
+; GISEL-NEXT: ds_read_b128 a[8:11], v0 offset:24608
+; GISEL-NEXT: ds_read_b128 a[12:15], v0 offset:24624
+; GISEL-NEXT: ds_read_b128 a[16:19], v0 offset:24640
+; GISEL-NEXT: ds_read_b128 a[20:23], v0 offset:24656
+; GISEL-NEXT: ds_read_b128 a[24:27], v0 offset:24672
+; GISEL-NEXT: ds_read_b128 a[28:31], v0 offset:24688
+; GISEL-NEXT: v_add_u32_e32 v1, 0xc000, v0
+; GISEL-NEXT: ds_read_b128 a[152:155], v0 offset:96
+; GISEL-NEXT: ds_read_b128 a[156:159], v0 offset:112
+; GISEL-NEXT: v_add_u32_e32 v33, s1, v33
+; GISEL-NEXT: ; iglp_opt mask(0x00000000)
+; GISEL-NEXT: s_waitcnt lgkmcnt(2)
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v32, v34, a[0:31]
+; GISEL-NEXT: ds_read_b128 a[32:35], v0 offset:49152
+; GISEL-NEXT: ds_read_b128 a[36:39], v0 offset:49168
+; GISEL-NEXT: ds_read_b128 a[40:43], v0 offset:49184
+; GISEL-NEXT: ds_read_b128 a[44:47], v0 offset:49200
+; GISEL-NEXT: ds_read_b128 a[48:51], v0 offset:49216
+; GISEL-NEXT: ds_read_b128 a[52:55], v0 offset:49232
+; GISEL-NEXT: ds_read_b128 a[56:59], v0 offset:49248
+; GISEL-NEXT: ds_read_b128 a[60:63], v0 offset:49264
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v32, v34, a[32:63]
+; GISEL-NEXT: ds_read_b128 a[64:67], v1 offset:32768
+; GISEL-NEXT: ds_read_b128 a[68:71], v1 offset:32784
+; GISEL-NEXT: ds_read_b128 a[72:75], v1 offset:32800
+; GISEL-NEXT: ds_read_b128 a[76:79], v1 offset:32816
+; GISEL-NEXT: ds_read_b128 a[80:83], v1 offset:32832
+; GISEL-NEXT: ds_read_b128 a[84:87], v1 offset:32848
+; GISEL-NEXT: ds_read_b128 a[88:91], v1 offset:32864
+; GISEL-NEXT: ds_read_b128 a[92:95], v1 offset:32880
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v32, v34, a[64:95]
+; GISEL-NEXT: ds_read_b128 a[128:131], v0
+; GISEL-NEXT: ds_read_b128 a[132:135], v0 offset:16
+; GISEL-NEXT: ds_read_b128 a[136:139], v0 offset:32
+; GISEL-NEXT: ds_read_b128 a[140:143], v0 offset:48
+; GISEL-NEXT: ds_read_b128 a[144:147], v0 offset:64
+; GISEL-NEXT: ds_read_b128 a[148:151], v0 offset:80
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v32, v34, a[128:159]
+; GISEL-NEXT: v_mov_b32_e32 v32, s1
+; GISEL-NEXT: s_nop 15
+; GISEL-NEXT: s_nop 1
+; GISEL-NEXT: v_accvgpr_read_b32 v0, a128
+; GISEL-NEXT: v_accvgpr_read_b32 v1, a129
+; GISEL-NEXT: v_accvgpr_read_b32 v2, a130
+; GISEL-NEXT: v_accvgpr_read_b32 v3, a131
+; GISEL-NEXT: v_accvgpr_read_b32 v4, a132
+; GISEL-NEXT: v_accvgpr_read_b32 v5, a133
+; GISEL-NEXT: v_accvgpr_read_b32 v6, a134
+; GISEL-NEXT: v_accvgpr_read_b32 v7, a135
+; GISEL-NEXT: v_accvgpr_read_b32 v8, a136
+; GISEL-NEXT: v_accvgpr_read_b32 v9, a137
+; GISEL-NEXT: v_accvgpr_read_b32 v10, a138
+; GISEL-NEXT: v_accvgpr_read_b32 v11, a139
+; GISEL-NEXT: v_accvgpr_read_b32 v12, a140
+; GISEL-NEXT: v_accvgpr_read_b32 v13, a141
+; GISEL-NEXT: v_accvgpr_read_b32 v14, a142
+; GISEL-NEXT: v_accvgpr_read_b32 v15, a143
+; GISEL-NEXT: v_accvgpr_read_b32 v16, a144
+; GISEL-NEXT: v_accvgpr_read_b32 v17, a145
+; GISEL-NEXT: v_accvgpr_read_b32 v18, a146
+; GISEL-NEXT: v_accvgpr_read_b32 v19, a147
+; GISEL-NEXT: v_accvgpr_read_b32 v20, a148
+; GISEL-NEXT: v_accvgpr_read_b32 v21, a149
+; GISEL-NEXT: v_accvgpr_read_b32 v22, a150
+; GISEL-NEXT: v_accvgpr_read_b32 v23, a151
+; GISEL-NEXT: v_accvgpr_read_b32 v24, a152
+; GISEL-NEXT: v_accvgpr_read_b32 v25, a153
+; GISEL-NEXT: v_accvgpr_read_b32 v26, a154
+; GISEL-NEXT: v_accvgpr_read_b32 v27, a155
+; GISEL-NEXT: v_accvgpr_read_b32 v28, a156
+; GISEL-NEXT: v_accvgpr_read_b32 v29, a157
+; GISEL-NEXT: v_accvgpr_read_b32 v30, a158
+; GISEL-NEXT: v_accvgpr_read_b32 v31, a159
+; GISEL-NEXT: ds_write_b128 v33, v[0:3]
+; GISEL-NEXT: ds_write_b128 v33, v[4:7] offset:16
+; GISEL-NEXT: ds_write_b128 v33, v[8:11] offset:32
+; GISEL-NEXT: ds_write_b128 v33, v[12:15] offset:48
+; GISEL-NEXT: ds_write_b128 v33, v[16:19] offset:64
+; GISEL-NEXT: ds_write_b128 v33, v[20:23] offset:80
+; GISEL-NEXT: ds_write_b128 v33, v[24:27] offset:96
+; GISEL-NEXT: ds_write_b128 v33, v[28:31] offset:112
+; GISEL-NEXT: v_accvgpr_read_b32 v0, a96
+; GISEL-NEXT: v_accvgpr_read_b32 v1, a97
+; GISEL-NEXT: v_accvgpr_read_b32 v2, a98
+; GISEL-NEXT: v_accvgpr_read_b32 v3, a99
+; GISEL-NEXT: v_accvgpr_read_b32 v4, a100
+; GISEL-NEXT: v_accvgpr_read_b32 v5, a101
+; GISEL-NEXT: v_accvgpr_read_b32 v6, a102
+; GISEL-NEXT: v_accvgpr_read_b32 v7, a103
+; GISEL-NEXT: v_accvgpr_read_b32 v8, a104
+; GISEL-NEXT: v_accvgpr_read_b32 v9, a105
+; GISEL-NEXT: v_accvgpr_read_b32 v10, a106
+; GISEL-NEXT: v_accvgpr_read_b32 v11, a107
+; GISEL-NEXT: v_accvgpr_read_b32 v12, a108
+; GISEL-NEXT: v_accvgpr_read_b32 v13, a109
+; GISEL-NEXT: v_accvgpr_read_b32 v14, a110
+; GISEL-NEXT: v_accvgpr_read_b32 v15, a111
+; GISEL-NEXT: v_accvgpr_read_b32 v16, a112
+; GISEL-NEXT: v_accvgpr_read_b32 v17, a113
+; GISEL-NEXT: v_accvgpr_read_b32 v18, a114
+; GISEL-NEXT: v_accvgpr_read_b32 v19, a115
+; GISEL-NEXT: v_accvgpr_read_b32 v20, a116
+; GISEL-NEXT: v_accvgpr_read_b32 v21, a117
+; GISEL-NEXT: v_accvgpr_read_b32 v22, a118
+; GISEL-NEXT: v_accvgpr_read_b32 v23, a119
+; GISEL-NEXT: v_accvgpr_read_b32 v24, a120
+; GISEL-NEXT: v_accvgpr_read_b32 v25, a121
+; GISEL-NEXT: v_accvgpr_read_b32 v26, a122
+; GISEL-NEXT: v_accvgpr_read_b32 v27, a123
+; GISEL-NEXT: v_accvgpr_read_b32 v28, a124
+; GISEL-NEXT: v_accvgpr_read_b32 v29, a125
+; GISEL-NEXT: v_accvgpr_read_b32 v30, a126
+; GISEL-NEXT: v_accvgpr_read_b32 v31, a127
+; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:8192
+; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:8208
+; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:8224
+; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:8240
+; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:8256
+; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:8272
+; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:8288
+; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:8304
+; GISEL-NEXT: v_accvgpr_read_b32 v0, a0
+; GISEL-NEXT: v_accvgpr_read_b32 v1, a1
+; GISEL-NEXT: v_accvgpr_read_b32 v2, a2
+; GISEL-NEXT: v_accvgpr_read_b32 v3, a3
+; GISEL-NEXT: v_accvgpr_read_b32 v4, a4
+; GISEL-NEXT: v_accvgpr_read_b32 v5, a5
+; GISEL-NEXT: v_accvgpr_read_b32 v6, a6
+; GISEL-NEXT: v_accvgpr_read_b32 v7, a7
+; GISEL-NEXT: v_accvgpr_read_b32 v8, a8
+; GISEL-NEXT: v_accvgpr_read_b32 v9, a9
+; GISEL-NEXT: v_accvgpr_read_b32 v10, a10
+; GISEL-NEXT: v_accvgpr_read_b32 v11, a11
+; GISEL-NEXT: v_accvgpr_read_b32 v12, a12
+; GISEL-NEXT: v_accvgpr_read_b32 v13, a13
+; GISEL-NEXT: v_accvgpr_read_b32 v14, a14
+; GISEL-NEXT: v_accvgpr_read_b32 v15, a15
+; GISEL-NEXT: v_accvgpr_read_b32 v16, a16
+; GISEL-NEXT: v_accvgpr_read_b32 v17, a17
+; GISEL-NEXT: v_accvgpr_read_b32 v18, a18
+; GISEL-NEXT: v_accvgpr_read_b32 v19, a19
+; GISEL-NEXT: v_accvgpr_read_b32 v20, a20
+; GISEL-NEXT: v_accvgpr_read_b32 v21, a21
+; GISEL-NEXT: v_accvgpr_read_b32 v22, a22
+; GISEL-NEXT: v_accvgpr_read_b32 v23, a23
+; GISEL-NEXT: v_accvgpr_read_b32 v24, a24
+; GISEL-NEXT: v_accvgpr_read_b32 v25, a25
+; GISEL-NEXT: v_accvgpr_read_b32 v26, a26
+; GISEL-NEXT: v_accvgpr_read_b32 v27, a27
+; GISEL-NEXT: v_accvgpr_read_b32 v28, a28
+; GISEL-NEXT: v_accvgpr_read_b32 v29, a29
+; GISEL-NEXT: v_accvgpr_read_b32 v30, a30
+; GISEL-NEXT: v_accvgpr_read_b32 v31, a31
+; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:16384
+; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:16400
+; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:16416
+; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:16432
+; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:16448
+; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:16464
+; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:16480
+; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:16496
+; GISEL-NEXT: v_accvgpr_read_b32 v0, a32
+; GISEL-NEXT: v_accvgpr_read_b32 v1, a33
+; GISEL-NEXT: v_accvgpr_read_b32 v2, a34
+; GISEL-NEXT: v_accvgpr_read_b32 v3, a35
+; GISEL-NEXT: v_accvgpr_read_b32 v4, a36
+; GISEL-NEXT: v_accvgpr_read_b32 v5, a37
+; GISEL-NEXT: v_accvgpr_read_b32 v6, a38
+; GISEL-NEXT: v_accvgpr_read_b32 v7, a39
+; GISEL-NEXT: v_accvgpr_read_b32 v8, a40
+; GISEL-NEXT: v_accvgpr_read_b32 v9, a41
+; GISEL-NEXT: v_accvgpr_read_b32 v10, a42
+; GISEL-NEXT: v_accvgpr_read_b32 v11, a43
+; GISEL-NEXT: v_accvgpr_read_b32 v12, a44
+; GISEL-NEXT: v_accvgpr_read_b32 v13, a45
+; GISEL-NEXT: v_accvgpr_read_b32 v14, a46
+; GISEL-NEXT: v_accvgpr_read_b32 v15, a47
+; GISEL-NEXT: v_accvgpr_read_b32 v16, a48
+; GISEL-NEXT: v_accvgpr_read_b32 v17, a49
+; GISEL-NEXT: v_accvgpr_read_b32 v18, a50
+; GISEL-NEXT: v_accvgpr_read_b32 v19, a51
+; GISEL-NEXT: v_accvgpr_read_b32 v20, a52
+; GISEL-NEXT: v_accvgpr_read_b32 v21, a53
+; GISEL-NEXT: v_accvgpr_read_b32 v22, a54
+; GISEL-NEXT: v_accvgpr_read_b32 v23, a55
+; GISEL-NEXT: v_accvgpr_read_b32 v24, a56
+; GISEL-NEXT: v_accvgpr_read_b32 v25, a57
+; GISEL-NEXT: v_accvgpr_read_b32 v26, a58
+; GISEL-NEXT: v_accvgpr_read_b32 v27, a59
+; GISEL-NEXT: v_accvgpr_read_b32 v28, a60
+; GISEL-NEXT: v_accvgpr_read_b32 v29, a61
+; GISEL-NEXT: v_accvgpr_read_b32 v30, a62
+; GISEL-NEXT: v_accvgpr_read_b32 v31, a63
+; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:24576
+; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:24592
+; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:24608
+; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:24624
+; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:24640
+; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:24656
+; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:24672
+; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:24688
+; GISEL-NEXT: v_accvgpr_read_b32 v0, a64
+; GISEL-NEXT: v_accvgpr_read_b32 v1, a65
+; GISEL-NEXT: v_accvgpr_read_b32 v2, a66
+; GISEL-NEXT: v_accvgpr_read_b32 v3, a67
+; GISEL-NEXT: v_accvgpr_read_b32 v4, a68
+; GISEL-NEXT: v_accvgpr_read_b32 v5, a69
+; GISEL-NEXT: v_accvgpr_read_b32 v6, a70
+; GISEL-NEXT: v_accvgpr_read_b32 v7, a71
+; GISEL-NEXT: v_accvgpr_read_b32 v8, a72
+; GISEL-NEXT: v_accvgpr_read_b32 v9, a73
+; GISEL-NEXT: v_accvgpr_read_b32 v10, a74
+; GISEL-NEXT: v_accvgpr_read_b32 v11, a75
+; GISEL-NEXT: v_accvgpr_read_b32 v12, a76
+; GISEL-NEXT: v_accvgpr_read_b32 v13, a77
+; GISEL-NEXT: v_accvgpr_read_b32 v14, a78
+; GISEL-NEXT: v_accvgpr_read_b32 v15, a79
+; GISEL-NEXT: v_accvgpr_read_b32 v16, a80
+; GISEL-NEXT: v_accvgpr_read_b32 v17, a81
+; GISEL-NEXT: v_accvgpr_read_b32 v18, a82
+; GISEL-NEXT: v_accvgpr_read_b32 v19, a83
+; GISEL-NEXT: v_accvgpr_read_b32 v20, a84
+; GISEL-NEXT: v_accvgpr_read_b32 v21, a85
+; GISEL-NEXT: v_accvgpr_read_b32 v22, a86
+; GISEL-NEXT: v_accvgpr_read_b32 v23, a87
+; GISEL-NEXT: v_accvgpr_read_b32 v24, a88
+; GISEL-NEXT: v_accvgpr_read_b32 v25, a89
+; GISEL-NEXT: v_accvgpr_read_b32 v26, a90
+; GISEL-NEXT: v_accvgpr_read_b32 v27, a91
+; GISEL-NEXT: v_accvgpr_read_b32 v28, a92
+; GISEL-NEXT: v_accvgpr_read_b32 v29, a93
+; GISEL-NEXT: v_accvgpr_read_b32 v30, a94
+; GISEL-NEXT: v_accvgpr_read_b32 v31, a95
+; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:32768
+; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:32784
+; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:32800
+; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:32816
+; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:32832
+; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:32848
+; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:32864
+; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:32880
+; GISEL-NEXT: s_endpgm
entry:
call void @llvm.amdgcn.iglp.opt(i32 0)
%idx = call i32 @llvm.amdgcn.workitem.id.x()
@@ -150,110 +418,375 @@ entry:
define amdgpu_kernel void @test_iglp_opt_rev_mfma_gemm(ptr addrspace(3) noalias %in, ptr addrspace(3) noalias %out) #0 {
-; GCN-LABEL: test_iglp_opt_rev_mfma_gemm:
-; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: v_lshlrev_b32_e32 v0, 7, v0
-; GCN-NEXT: v_and_b32_e32 v0, 0x1ff80, v0
-; GCN-NEXT: v_mov_b32_e32 v2, 1.0
-; GCN-NEXT: v_mov_b32_e32 v1, 2.0
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_add_u32_e32 v3, s0, v0
-; GCN-NEXT: ds_read_b128 a[28:31], v3 offset:112
-; GCN-NEXT: ds_read_b128 a[24:27], v3 offset:96
-; GCN-NEXT: ds_read_b128 a[20:23], v3 offset:80
-; GCN-NEXT: ds_read_b128 a[16:19], v3 offset:64
-; GCN-NEXT: ds_read_b128 a[0:3], v3
-; GCN-NEXT: ds_read_b128 a[4:7], v3 offset:16
-; GCN-NEXT: ds_read_b128 a[8:11], v3 offset:32
-; GCN-NEXT: ds_read_b128 a[12:15], v3 offset:48
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v1, a[0:31]
-; GCN-NEXT: ds_read_b128 a[156:159], v3 offset:8304
-; GCN-NEXT: ds_read_b128 a[152:155], v3 offset:8288
-; GCN-NEXT: ds_read_b128 a[148:151], v3 offset:8272
-; GCN-NEXT: ds_read_b128 a[144:147], v3 offset:8256
-; GCN-NEXT: ds_read_b128 a[140:143], v3 offset:8240
-; GCN-NEXT: ds_read_b128 a[136:139], v3 offset:8224
-; GCN-NEXT: ds_read_b128 a[132:135], v3 offset:8208
-; GCN-NEXT: ds_read_b128 a[128:131], v3 offset:8192
-; GCN-NEXT: v_add_u32_e32 v4, 0x6000, v3
-; GCN-NEXT: v_add_u32_e32 v0, s1, v0
-; GCN-NEXT: ; iglp_opt mask(0x00000001)
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v2, v1, a[128:159]
-; GCN-NEXT: ds_read_b128 a[124:127], v3 offset:24688
-; GCN-NEXT: ds_read_b128 a[120:123], v3 offset:24672
-; GCN-NEXT: ds_read_b128 a[116:119], v3 offset:24656
-; GCN-NEXT: ds_read_b128 a[112:115], v3 offset:24640
-; GCN-NEXT: ds_read_b128 a[108:111], v3 offset:24624
-; GCN-NEXT: ds_read_b128 a[104:107], v3 offset:24608
-; GCN-NEXT: ds_read_b128 a[100:103], v3 offset:24592
-; GCN-NEXT: ds_read_b128 a[96:99], v3 offset:24576
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v2, v1, a[96:127]
-; GCN-NEXT: ds_read_b128 a[92:95], v3 offset:49264
-; GCN-NEXT: ds_read_b128 a[88:91], v3 offset:49248
-; GCN-NEXT: ds_read_b128 a[84:87], v3 offset:49232
-; GCN-NEXT: ds_read_b128 a[80:83], v3 offset:49216
-; GCN-NEXT: ds_read_b128 a[76:79], v3 offset:49200
-; GCN-NEXT: ds_read_b128 a[72:75], v3 offset:49184
-; GCN-NEXT: ds_read_b128 a[68:71], v3 offset:49168
-; GCN-NEXT: ds_read_b128 a[64:67], v3 offset:49152
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v2, v1, a[64:95]
-; GCN-NEXT: ds_read_b128 a[60:63], v4 offset:57456
-; GCN-NEXT: ds_read_b128 a[56:59], v4 offset:57440
-; GCN-NEXT: ds_read_b128 a[52:55], v4 offset:57424
-; GCN-NEXT: ds_read_b128 a[48:51], v4 offset:57408
-; GCN-NEXT: ds_read_b128 a[32:35], v4 offset:57344
-; GCN-NEXT: ds_read_b128 a[36:39], v4 offset:57360
-; GCN-NEXT: ds_read_b128 a[40:43], v4 offset:57376
-; GCN-NEXT: ds_read_b128 a[44:47], v4 offset:57392
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v2, v1, a[32:63]
-; GCN-NEXT: ds_write_b128 v0, a[28:31] offset:112
-; GCN-NEXT: ds_write_b128 v0, a[24:27] offset:96
-; GCN-NEXT: ds_write_b128 v0, a[20:23] offset:80
-; GCN-NEXT: ds_write_b128 v0, a[16:19] offset:64
-; GCN-NEXT: ds_write_b128 v0, a[12:15] offset:48
-; GCN-NEXT: ds_write_b128 v0, a[8:11] offset:32
-; GCN-NEXT: ds_write_b128 v0, a[4:7] offset:16
-; GCN-NEXT: ds_write_b128 v0, a[0:3]
-; GCN-NEXT: v_mov_b32_e32 v0, s1
-; GCN-NEXT: ds_write_b128 v0, a[152:155] offset:8288
-; GCN-NEXT: ds_write_b128 v0, a[156:159] offset:8304
-; GCN-NEXT: ds_write_b128 v0, a[144:147] offset:8256
-; GCN-NEXT: ds_write_b128 v0, a[148:151] offset:8272
-; GCN-NEXT: ds_write_b128 v0, a[136:139] offset:8224
-; GCN-NEXT: ds_write_b128 v0, a[140:143] offset:8240
-; GCN-NEXT: ds_write_b128 v0, a[128:131] offset:8192
-; GCN-NEXT: ds_write_b128 v0, a[132:135] offset:8208
-; GCN-NEXT: ds_write_b128 v0, a[120:123] offset:16480
-; GCN-NEXT: ds_write_b128 v0, a[124:127] offset:16496
-; GCN-NEXT: ds_write_b128 v0, a[112:115] offset:16448
-; GCN-NEXT: ds_write_b128 v0, a[116:119] offset:16464
-; GCN-NEXT: ds_write_b128 v0, a[104:107] offset:16416
-; GCN-NEXT: ds_write_b128 v0, a[108:111] offset:16432
-; GCN-NEXT: ds_write_b128 v0, a[96:99] offset:16384
-; GCN-NEXT: ds_write_b128 v0, a[100:103] offset:16400
-; GCN-NEXT: ds_write_b128 v0, a[88:91] offset:24672
-; GCN-NEXT: ds_write_b128 v0, a[92:95] offset:24688
-; GCN-NEXT: ds_write_b128 v0, a[80:83] offset:24640
-; GCN-NEXT: ds_write_b128 v0, a[84:87] offset:24656
-; GCN-NEXT: ds_write_b128 v0, a[72:75] offset:24608
-; GCN-NEXT: ds_write_b128 v0, a[76:79] offset:24624
-; GCN-NEXT: ds_write_b128 v0, a[64:67] offset:24576
-; GCN-NEXT: ds_write_b128 v0, a[68:71] offset:24592
-; GCN-NEXT: ds_write_b128 v0, a[56:59] offset:32864
-; GCN-NEXT: ds_write_b128 v0, a[60:63] offset:32880
-; GCN-NEXT: ds_write_b128 v0, a[48:51] offset:32832
-; GCN-NEXT: ds_write_b128 v0, a[52:55] offset:32848
-; GCN-NEXT: ds_write_b128 v0, a[40:43] offset:32800
-; GCN-NEXT: ds_write_b128 v0, a[44:47] offset:32816
-; GCN-NEXT: ds_write_b128 v0, a[32:35] offset:32768
-; GCN-NEXT: ds_write_b128 v0, a[36:39] offset:32784
-; GCN-NEXT: s_endpgm
+; SDAG-LABEL: test_iglp_opt_rev_mfma_gemm:
+; SDAG: ; %bb.0: ; %entry
+; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-NEXT: v_lshlrev_b32_e32 v0, 7, v0
+; SDAG-NEXT: v_and_b32_e32 v0, 0x1ff80, v0
+; SDAG-NEXT: v_mov_b32_e32 v2, 1.0
+; SDAG-NEXT: v_mov_b32_e32 v1, 2.0
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_add_u32_e32 v3, s0, v0
+; SDAG-NEXT: ds_read_b128 a[28:31], v3 offset:112
+; SDAG-NEXT: ds_read_b128 a[24:27], v3 offset:96
+; SDAG-NEXT: ds_read_b128 a[20:23], v3 offset:80
+; SDAG-NEXT: ds_read_b128 a[16:19], v3 offset:64
+; SDAG-NEXT: ds_read_b128 a[0:3], v3
+; SDAG-NEXT: ds_read_b128 a[4:7], v3 offset:16
+; SDAG-NEXT: ds_read_b128 a[8:11], v3 offset:32
+; SDAG-NEXT: ds_read_b128 a[12:15], v3 offset:48
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v1, a[0:31]
+; SDAG-NEXT: ds_read_b128 a[156:159], v3 offset:8304
+; SDAG-NEXT: ds_read_b128 a[152:155], v3 offset:8288
+; SDAG-NEXT: ds_read_b128 a[148:151], v3 offset:8272
+; SDAG-NEXT: ds_read_b128 a[144:147], v3 offset:8256
+; SDAG-NEXT: ds_read_b128 a[140:143], v3 offset:8240
+; SDAG-NEXT: ds_read_b128 a[136:139], v3 offset:8224
+; SDAG-NEXT: ds_read_b128 a[132:135], v3 offset:8208
+; SDAG-NEXT: ds_read_b128 a[128:131], v3 offset:8192
+; SDAG-NEXT: v_add_u32_e32 v4, 0x6000, v3
+; SDAG-NEXT: v_add_u32_e32 v0, s1, v0
+; SDAG-NEXT: ; iglp_opt mask(0x00000001)
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v2, v1, a[128:159]
+; SDAG-NEXT: ds_read_b128 a[124:127], v3 offset:24688
+; SDAG-NEXT: ds_read_b128 a[120:123], v3 offset:24672
+; SDAG-NEXT: ds_read_b128 a[116:119], v3 offset:24656
+; SDAG-NEXT: ds_read_b128 a[112:115], v3 offset:24640
+; SDAG-NEXT: ds_read_b128 a[108:111], v3 offset:24624
+; SDAG-NEXT: ds_read_b128 a[104:107], v3 offset:24608
+; SDAG-NEXT: ds_read_b128 a[100:103], v3 offset:24592
+; SDAG-NEXT: ds_read_b128 a[96:99], v3 offset:24576
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v2, v1, a[96:127]
+; SDAG-NEXT: ds_read_b128 a[92:95], v3 offset:49264
+; SDAG-NEXT: ds_read_b128 a[88:91], v3 offset:49248
+; SDAG-NEXT: ds_read_b128 a[84:87], v3 offset:49232
+; SDAG-NEXT: ds_read_b128 a[80:83], v3 offset:49216
+; SDAG-NEXT: ds_read_b128 a[76:79], v3 offset:49200
+; SDAG-NEXT: ds_read_b128 a[72:75], v3 offset:49184
+; SDAG-NEXT: ds_read_b128 a[68:71], v3 offset:49168
+; SDAG-NEXT: ds_read_b128 a[64:67], v3 offset:49152
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v2, v1, a[64:95]
+; SDAG-NEXT: ds_read_b128 a[60:63], v4 offset:57456
+; SDAG-NEXT: ds_read_b128 a[56:59], v4 offset:57440
+; SDAG-NEXT: ds_read_b128 a[52:55], v4 offset:57424
+; SDAG-NEXT: ds_read_b128 a[48:51], v4 offset:57408
+; SDAG-NEXT: ds_read_b128 a[32:35], v4 offset:57344
+; SDAG-NEXT: ds_read_b128 a[36:39], v4 offset:57360
+; SDAG-NEXT: ds_read_b128 a[40:43], v4 offset:57376
+; SDAG-NEXT: ds_read_b128 a[44:47], v4 offset:57392
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v2, v1, a[32:63]
+; SDAG-NEXT: ds_write_b128 v0, a[28:31] offset:112
+; SDAG-NEXT: ds_write_b128 v0, a[24:27] offset:96
+; SDAG-NEXT: ds_write_b128 v0, a[20:23] offset:80
+; SDAG-NEXT: ds_write_b128 v0, a[16:19] offset:64
+; SDAG-NEXT: ds_write_b128 v0, a[12:15] offset:48
+; SDAG-NEXT: ds_write_b128 v0, a[8:11] offset:32
+; SDAG-NEXT: ds_write_b128 v0, a[4:7] offset:16
+; SDAG-NEXT: ds_write_b128 v0, a[0:3]
+; SDAG-NEXT: v_mov_b32_e32 v0, s1
+; SDAG-NEXT: ds_write_b128 v0, a[152:155] offset:8288
+; SDAG-NEXT: ds_write_b128 v0, a[156:159] offset:8304
+; SDAG-NEXT: ds_write_b128 v0, a[144:147] offset:8256
+; SDAG-NEXT: ds_write_b128 v0, a[148:151] offset:8272
+; SDAG-NEXT: ds_write_b128 v0, a[136:139] offset:8224
+; SDAG-NEXT: ds_write_b128 v0, a[140:143] offset:8240
+; SDAG-NEXT: ds_write_b128 v0, a[128:131] offset:8192
+; SDAG-NEXT: ds_write_b128 v0, a[132:135] offset:8208
+; SDAG-NEXT: ds_write_b128 v0, a[120:123] offset:16480
+; SDAG-NEXT: ds_write_b128 v0, a[124:127] offset:16496
+; SDAG-NEXT: ds_write_b128 v0, a[112:115] offset:16448
+; SDAG-NEXT: ds_write_b128 v0, a[116:119] offset:16464
+; SDAG-NEXT: ds_write_b128 v0, a[104:107] offset:16416
+; SDAG-NEXT: ds_write_b128 v0, a[108:111] offset:16432
+; SDAG-NEXT: ds_write_b128 v0, a[96:99] offset:16384
+; SDAG-NEXT: ds_write_b128 v0, a[100:103] offset:16400
+; SDAG-NEXT: ds_write_b128 v0, a[88:91] offset:24672
+; SDAG-NEXT: ds_write_b128 v0, a[92:95] offset:24688
+; SDAG-NEXT: ds_write_b128 v0, a[80:83] offset:24640
+; SDAG-NEXT: ds_write_b128 v0, a[84:87] offset:24656
+; SDAG-NEXT: ds_write_b128 v0, a[72:75] offset:24608
+; SDAG-NEXT: ds_write_b128 v0, a[76:79] offset:24624
+; SDAG-NEXT: ds_write_b128 v0, a[64:67] offset:24576
+; SDAG-NEXT: ds_write_b128 v0, a[68:71] offset:24592
+; SDAG-NEXT: ds_write_b128 v0, a[56:59] offset:32864
+; SDAG-NEXT: ds_write_b128 v0, a[60:63] offset:32880
+; SDAG-NEXT: ds_write_b128 v0, a[48:51] offset:32832
+; SDAG-NEXT: ds_write_b128 v0, a[52:55] offset:32848
+; SDAG-NEXT: ds_write_b128 v0, a[40:43] offset:32800
+; SDAG-NEXT: ds_write_b128 v0, a[44:47] offset:32816
+; SDAG-NEXT: ds_write_b128 v0, a[32:35] offset:32768
+; SDAG-NEXT: ds_write_b128 v0, a[36:39] offset:32784
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: test_iglp_opt_rev_mfma_gemm:
+; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GISEL-NEXT: v_lshlrev_b32_e32 v32, 7, v0
+; GISEL-NEXT: v_mov_b32_e32 v33, 1.0
+; GISEL-NEXT: v_mov_b32_e32 v34, 2.0
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: v_add_u32_e32 v0, s0, v32
+; GISEL-NEXT: ds_read_b128 a[0:3], v0
+; GISEL-NEXT: ds_read_b128 a[4:7], v0 offset:16
+; GISEL-NEXT: ds_read_b128 a[8:11], v0 offset:32
+; GISEL-NEXT: ds_read_b128 a[12:15], v0 offset:48
+; GISEL-NEXT: ds_read_b128 a[16:19], v0 offset:64
+; GISEL-NEXT: ds_read_b128 a[20:23], v0 offset:80
+; GISEL-NEXT: ds_read_b128 a[24:27], v0 offset:96
+; GISEL-NEXT: ds_read_b128 a[28:31], v0 offset:112
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v33, v34, a[0:31]
+; GISEL-NEXT: ds_read_b128 a[32:35], v0 offset:8192
+; GISEL-NEXT: ds_read_b128 a[36:39], v0 offset:8208
+; GISEL-NEXT: ds_read_b128 a[40:43], v0 offset:8224
+; GISEL-NEXT: ds_read_b128 a[44:47], v0 offset:8240
+; GISEL-NEXT: ds_read_b128 a[48:51], v0 offset:8256
+; GISEL-NEXT: ds_read_b128 a[52:55], v0 offset:8272
+; GISEL-NEXT: ds_read_b128 a[56:59], v0 offset:8288
+; GISEL-NEXT: ds_read_b128 a[60:63], v0 offset:8304
+; GISEL-NEXT: v_add_u32_e32 v1, 0xc000, v0
+; GISEL-NEXT: v_add_u32_e32 v32, s1, v32
+; GISEL-NEXT: ; iglp_opt mask(0x00000001)
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v33, v34, a[32:63]
+; GISEL-NEXT: ds_read_b128 a[64:67], v0 offset:24576
+; GISEL-NEXT: ds_read_b128 a[68:71], v0 offset:24592
+; GISEL-NEXT: ds_read_b128 a[72:75], v0 offset:24608
+; GISEL-NEXT: ds_read_b128 a[76:79], v0 offset:24624
+; GISEL-NEXT: ds_read_b128 a[80:83], v0 offset:24640
+; GISEL-NEXT: ds_read_b128 a[84:87], v0 offset:24656
+; GISEL-NEXT: ds_read_b128 a[88:91], v0 offset:24672
+; GISEL-NEXT: ds_read_b128 a[92:95], v0 offset:24688
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v33, v34, a[64:95]
+; GISEL-NEXT: ds_read_b128 a[96:99], v0 offset:49152
+; GISEL-NEXT: ds_read_b128 a[100:103], v0 offset:49168
+; GISEL-NEXT: ds_read_b128 a[104:107], v0 offset:49184
+; GISEL-NEXT: ds_read_b128 a[108:111], v0 offset:49200
+; GISEL-NEXT: ds_read_b128 a[112:115], v0 offset:49216
+; GISEL-NEXT: ds_read_b128 a[116:119], v0 offset:49232
+; GISEL-NEXT: ds_read_b128 a[120:123], v0 offset:49248
+; GISEL-NEXT: ds_read_b128 a[124:127], v0 offset:49264
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v33, v34, a[96:127]
+; GISEL-NEXT: ds_read_b128 a[128:131], v1 offset:32768
+; GISEL-NEXT: ds_read_b128 a[132:135], v1 offset:32784
+; GISEL-NEXT: ds_read_b128 a[136:139], v1 offset:32800
+; GISEL-NEXT: ds_read_b128 a[140:143], v1 offset:32816
+; GISEL-NEXT: ds_read_b128 a[144:147], v1 offset:32832
+; GISEL-NEXT: ds_read_b128 a[148:151], v1 offset:32848
+; GISEL-NEXT: ds_read_b128 a[152:155], v1 offset:32864
+; GISEL-NEXT: ds_read_b128 a[156:159], v1 offset:32880
+; GISEL-NEXT: v_accvgpr_read_b32 v0, a0
+; GISEL-NEXT: v_accvgpr_read_b32 v1, a1
+; GISEL-NEXT: v_accvgpr_read_b32 v2, a2
+; GISEL-NEXT: v_accvgpr_read_b32 v3, a3
+; GISEL-NEXT: v_accvgpr_read_b32 v4, a4
+; GISEL-NEXT: v_accvgpr_read_b32 v5, a5
+; GISEL-NEXT: v_accvgpr_read_b32 v6, a6
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v33, v34, a[128:159]
+; GISEL-NEXT: v_accvgpr_read_b32 v7, a7
+; GISEL-NEXT: v_accvgpr_read_b32 v8, a8
+; GISEL-NEXT: v_accvgpr_read_b32 v9, a9
+; GISEL-NEXT: v_accvgpr_read_b32 v10, a10
+; GISEL-NEXT: v_accvgpr_read_b32 v11, a11
+; GISEL-NEXT: v_accvgpr_read_b32 v12, a12
+; GISEL-NEXT: v_accvgpr_read_b32 v13, a13
+; GISEL-NEXT: v_accvgpr_read_b32 v14, a14
+; GISEL-NEXT: v_accvgpr_read_b32 v15, a15
+; GISEL-NEXT: v_accvgpr_read_b32 v16, a16
+; GISEL-NEXT: v_accvgpr_read_b32 v17, a17
+; GISEL-NEXT: v_accvgpr_read_b32 v18, a18
+; GISEL-NEXT: v_accvgpr_read_b32 v19, a19
+; GISEL-NEXT: v_accvgpr_read_b32 v20, a20
+; GISEL-NEXT: v_accvgpr_read_b32 v21, a21
+; GISEL-NEXT: v_accvgpr_read_b32 v22, a22
+; GISEL-NEXT: v_accvgpr_read_b32 v23, a23
+; GISEL-NEXT: v_accvgpr_read_b32 v24, a24
+; GISEL-NEXT: v_accvgpr_read_b32 v25, a25
+; GISEL-NEXT: v_accvgpr_read_b32 v26, a26
+; GISEL-NEXT: v_accvgpr_read_b32 v27, a27
+; GISEL-NEXT: v_accvgpr_read_b32 v28, a28
+; GISEL-NEXT: v_accvgpr_read_b32 v29, a29
+; GISEL-NEXT: v_accvgpr_read_b32 v30, a30
+; GISEL-NEXT: v_accvgpr_read_b32 v31, a31
+; GISEL-NEXT: ds_write_b128 v32, v[0:3]
+; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:16
+; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:32
+; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:48
+; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:64
+; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:80
+; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:96
+; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:112
+; GISEL-NEXT: v_accvgpr_read_b32 v0, a32
+; GISEL-NEXT: v_accvgpr_read_b32 v1, a33
+; GISEL-NEXT: v_accvgpr_read_b32 v2, a34
+; GISEL-NEXT: v_accvgpr_read_b32 v3, a35
+; GISEL-NEXT: v_accvgpr_read_b32 v4, a36
+; GISEL-NEXT: v_accvgpr_read_b32 v5, a37
+; GISEL-NEXT: v_accvgpr_read_b32 v6, a38
+; GISEL-NEXT: v_accvgpr_read_b32 v7, a39
+; GISEL-NEXT: v_accvgpr_read_b32 v8, a40
+; GISEL-NEXT: v_accvgpr_read_b32 v9, a41
+; GISEL-NEXT: v_accvgpr_read_b32 v10, a42
+; GISEL-NEXT: v_accvgpr_read_b32 v11, a43
+; GISEL-NEXT: v_accvgpr_read_b32 v12, a44
+; GISEL-NEXT: v_accvgpr_read_b32 v13, a45
+; GISEL-NEXT: v_accvgpr_read_b32 v14, a46
+; GISEL-NEXT: v_accvgpr_read_b32 v15, a47
+; GISEL-NEXT: v_accvgpr_read_b32 v16, a48
+; GISEL-NEXT: v_accvgpr_read_b32 v17, a49
+; GISEL-NEXT: v_accvgpr_read_b32 v18, a50
+; GISEL-NEXT: v_accvgpr_read_b32 v19, a51
+; GISEL-NEXT: v_accvgpr_read_b32 v20, a52
+; GISEL-NEXT: v_accvgpr_read_b32 v21, a53
+; GISEL-NEXT: v_accvgpr_read_b32 v22, a54
+; GISEL-NEXT: v_accvgpr_read_b32 v23, a55
+; GISEL-NEXT: v_accvgpr_read_b32 v24, a56
+; GISEL-NEXT: v_accvgpr_read_b32 v25, a57
+; GISEL-NEXT: v_accvgpr_read_b32 v26, a58
+; GISEL-NEXT: v_accvgpr_read_b32 v27, a59
+; GISEL-NEXT: v_accvgpr_read_b32 v28, a60
+; GISEL-NEXT: v_accvgpr_read_b32 v29, a61
+; GISEL-NEXT: v_accvgpr_read_b32 v30, a62
+; GISEL-NEXT: v_accvgpr_read_b32 v31, a63
+; GISEL-NEXT: v_mov_b32_e32 v32, s1
+; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:8192
+; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:8208
+; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:8224
+; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:8240
+; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:8256
+; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:8272
+; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:8288
+; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:8304
+; GISEL-NEXT: v_accvgpr_read_b32 v0, a64
+; GISEL-NEXT: v_accvgpr_read_b32 v1, a65
+; GISEL-NEXT: v_accvgpr_read_b32 v2, a66
+; GISEL-NEXT: v_accvgpr_read_b32 v3, a67
+; GISEL-NEXT: v_accvgpr_read_b32 v4, a68
+; GISEL-NEXT: v_accvgpr_read_b32 v5, a69
+; GISEL-NEXT: v_accvgpr_read_b32 v6, a70
+; GISEL-NEXT: v_accvgpr_read_b32 v7, a71
+; GISEL-NEXT: v_accvgpr_read_b32 v8, a72
+; GISEL-NEXT: v_accvgpr_read_b32 v9, a73
+; GISEL-NEXT: v_accvgpr_read_b32 v10, a74
+; GISEL-NEXT: v_accvgpr_read_b32 v11, a75
+; GISEL-NEXT: v_accvgpr_read_b32 v12, a76
+; GISEL-NEXT: v_accvgpr_read_b32 v13, a77
+; GISEL-NEXT: v_accvgpr_read_b32 v14, a78
+; GISEL-NEXT: v_accvgpr_read_b32 v15, a79
+; GISEL-NEXT: v_accvgpr_read_b32 v16, a80
+; GISEL-NEXT: v_accvgpr_read_b32 v17, a81
+; GISEL-NEXT: v_accvgpr_read_b32 v18, a82
+; GISEL-NEXT: v_accvgpr_read_b32 v19, a83
+; GISEL-NEXT: v_accvgpr_read_b32 v20, a84
+; GISEL-NEXT: v_accvgpr_read_b32 v21, a85
+; GISEL-NEXT: v_accvgpr_read_b32 v22, a86
+; GISEL-NEXT: v_accvgpr_read_b32 v23, a87
+; GISEL-NEXT: v_accvgpr_read_b32 v24, a88
+; GISEL-NEXT: v_accvgpr_read_b32 v25, a89
+; GISEL-NEXT: v_accvgpr_read_b32 v26, a90
+; GISEL-NEXT: v_accvgpr_read_b32 v27, a91
+; GISEL-NEXT: v_accvgpr_read_b32 v28, a92
+; GISEL-NEXT: v_accvgpr_read_b32 v29, a93
+; GISEL-NEXT: v_accvgpr_read_b32 v30, a94
+; GISEL-NEXT: v_accvgpr_read_b32 v31, a95
+; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:16384
+; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:16400
+; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:16416
+; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:16432
+; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:16448
+; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:16464
+; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:16480
+; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:16496
+; GISEL-NEXT: v_accvgpr_read_b32 v0, a96
+; GISEL-NEXT: v_accvgpr_read_b32 v1, a97
+; GISEL-NEXT: v_accvgpr_read_b32 v2, a98
+; GISEL-NEXT: v_accvgpr_read_b32 v3, a99
+; GISEL-NEXT: v_accvgpr_read_b32 v4, a100
+; GISEL-NEXT: v_accvgpr_read_b32 v5, a101
+; GISEL-NEXT: v_accvgpr_read_b32 v6, a102
+; GISEL-NEXT: v_accvgpr_read_b32 v7, a103
+; GISEL-NEXT: v_accvgpr_read_b32 v8, a104
+; GISEL-NEXT: v_accvgpr_read_b32 v9, a105
+; GISEL-NEXT: v_accvgpr_read_b32 v10, a106
+; GISEL-NEXT: v_accvgpr_read_b32 v11, a107
+; GISEL-NEXT: v_accvgpr_read_b32 v12, a108
+; GISEL-NEXT: v_accvgpr_read_b32 v13, a109
+; GISEL-NEXT: v_accvgpr_read_b32 v14, a110
+; GISEL-NEXT: v_accvgpr_read_b32 v15, a111
+; GISEL-NEXT: v_accvgpr_read_b32 v16, a112
+; GISEL-NEXT: v_accvgpr_read_b32 v17, a113
+; GISEL-NEXT: v_accvgpr_read_b32 v18, a114
+; GISEL-NEXT: v_accvgpr_read_b32 v19, a115
+; GISEL-NEXT: v_accvgpr_read_b32 v20, a116
+; GISEL-NEXT: v_accvgpr_read_b32 v21, a117
+; GISEL-NEXT: v_accvgpr_read_b32 v22, a118
+; GISEL-NEXT: v_accvgpr_read_b32 v23, a119
+; GISEL-NEXT: v_accvgpr_read_b32 v24, a120
+; GISEL-NEXT: v_accvgpr_read_b32 v25, a121
+; GISEL-NEXT: v_accvgpr_read_b32 v26, a122
+; GISEL-NEXT: v_accvgpr_read_b32 v27, a123
+; GISEL-NEXT: v_accvgpr_read_b32 v28, a124
+; GISEL-NEXT: v_accvgpr_read_b32 v29, a125
+; GISEL-NEXT: v_accvgpr_read_b32 v30, a126
+; GISEL-NEXT: v_accvgpr_read_b32 v31, a127
+; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:24576
+; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:24592
+; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:24608
+; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:24624
+; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:24640
+; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:24656
+; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:24672
+; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:24688
+; GISEL-NEXT: v_accvgpr_read_b32 v0, a128
+; GISEL-NEXT: v_accvgpr_read_b32 v1, a129
+; GISEL-NEXT: v_accvgpr_read_b32 v2, a130
+; GISEL-NEXT: v_accvgpr_read_b32 v3, a131
+; GISEL-NEXT: v_accvgpr_read_b32 v4, a132
+; GISEL-NEXT: v_accvgpr_read_b32 v5, a133
+; GISEL-NEXT: v_accvgpr_read_b32 v6, a134
+; GISEL-NEXT: v_accvgpr_read_b32 v7, a135
+; GISEL-NEXT: v_accvgpr_read_b32 v8, a136
+; GISEL-NEXT: v_accvgpr_read_b32 v9, a137
+; GISEL-NEXT: v_accvgpr_read_b32 v10, a138
+; GISEL-NEXT: v_accvgpr_read_b32 v11, a139
+; GISEL-NEXT: v_accvgpr_read_b32 v12, a140
+; GISEL-NEXT: v_accvgpr_read_b32 v13, a141
+; GISEL-NEXT: v_accvgpr_read_b32 v14, a142
+; GISEL-NEXT: v_accvgpr_read_b32 v15, a143
+; GISEL-NEXT: v_accvgpr_read_b32 v16, a144
+; GISEL-NEXT: v_accvgpr_read_b32 v17, a145
+; GISEL-NEXT: v_accvgpr_read_b32 v18, a146
+; GISEL-NEXT: v_accvgpr_read_b32 v19, a147
+; GISEL-NEXT: v_accvgpr_read_b32 v20, a148
+; GISEL-NEXT: v_accvgpr_read_b32 v21, a149
+; GISEL-NEXT: v_accvgpr_read_b32 v22, a150
+; GISEL-NEXT: v_accvgpr_read_b32 v23, a151
+; GISEL-NEXT: v_accvgpr_read_b32 v24, a152
+; GISEL-NEXT: v_accvgpr_read_b32 v25, a153
+; GISEL-NEXT: v_accvgpr_read_b32 v26, a154
+; GISEL-NEXT: v_accvgpr_read_b32 v27, a155
+; GISEL-NEXT: v_accvgpr_read_b32 v28, a156
+; GISEL-NEXT: v_accvgpr_read_b32 v29, a157
+; GISEL-NEXT: v_accvgpr_read_b32 v30, a158
+; GISEL-NEXT: v_accvgpr_read_b32 v31, a159
+; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:32768
+; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:32784
+; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:32800
+; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:32816
+; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:32832
+; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:32848
+; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:32864
+; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:32880
+; GISEL-NEXT: s_endpgm
entry:
call void @llvm.amdgcn.iglp.opt(i32 1)
%idx = call i32 @llvm.amdgcn.workitem.id.x()
@@ -286,26 +819,47 @@ entry:
}
define amdgpu_kernel void @test_iglp_opt_asm_sideeffect(ptr addrspace(3) noalias %in, ptr addrspace(3) noalias %out) #0 {
-; GCN-LABEL: test_iglp_opt_asm_sideeffect:
-; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GCN-NEXT: v_and_b32_e32 v0, 0xffc, v0
-; GCN-NEXT: ; iglp_opt mask(0x00000000)
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_add_u32_e32 v1, s0, v0
-; GCN-NEXT: ds_read_b32 v2, v1
-; GCN-NEXT: v_add_u32_e32 v0, s1, v0
-; GCN-NEXT: v_mov_b32_e32 v1, s0
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: ds_write_b32 v0, v2
-; GCN-NEXT: ;;#ASMSTART
-; GCN-NEXT: ;;#ASMEND
-; GCN-NEXT: ds_read_b32 v1, v1 offset:256
-; GCN-NEXT: v_mov_b32_e32 v0, s1
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: ds_write_b32 v0, v1 offset:256
-; GCN-NEXT: s_endpgm
+; SDAG-LABEL: test_iglp_opt_asm_sideeffect:
+; SDAG: ; %bb.0: ; %entry
+; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; SDAG-NEXT: v_and_b32_e32 v0, 0xffc, v0
+; SDAG-NEXT: ; iglp_opt mask(0x00000000)
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_add_u32_e32 v1, s0, v0
+; SDAG-NEXT: ds_read_b32 v2, v1
+; SDAG-NEXT: v_add_u32_e32 v0, s1, v0
+; SDAG-NEXT: v_mov_b32_e32 v1, s0
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: ds_write_b32 v0, v2
+; SDAG-NEXT: ;;#ASMSTART
+; SDAG-NEXT: ;;#ASMEND
+; SDAG-NEXT: ds_read_b32 v1, v1 offset:256
+; SDAG-NEXT: v_mov_b32_e32 v0, s1
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: ds_write_b32 v0, v1 offset:256
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: test_iglp_opt_asm_sideeffect:
+; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GISEL-NEXT: ; iglp_opt mask(0x00000000)
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: v_add_u32_e32 v1, s0, v0
+; GISEL-NEXT: ds_read_b32 v1, v1
+; GISEL-NEXT: v_mov_b32_e32 v2, s0
+; GISEL-NEXT: v_add_u32_e32 v0, s1, v0
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: ds_write_b32 v0, v1
+; GISEL-NEXT: ;;#ASMSTART
+; GISEL-NEXT: ;;#ASMEND
+; GISEL-NEXT: ds_read_b32 v0, v2 offset:256
+; GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: ds_write_b32 v1, v0 offset:256
+; GISEL-NEXT: s_endpgm
entry:
%idx = call i32 @llvm.amdgcn.workitem.id.x()
%load.0.addr = getelementptr float, ptr addrspace(3) %in, i32 %idx
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.barrier.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.barrier.ll
index 9b88a1076a7c7..b4dbbfbb422b2 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.barrier.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.barrier.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=amdgcn < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn < %s | FileCheck -check-prefix=GCN %s
define amdgpu_kernel void @test_sched_barrier() #0 {
; GCN-LABEL: test_sched_barrier:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.ll
index 996c76488080f..3b18be77fcb24 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.ll
@@ -1,6 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -misched-cluster=0 -amdgpu-mfma-vgpr-form=0 < %s | FileCheck -check-prefix=GCN %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -misched-cluster=0 -amdgpu-mfma-vgpr-form=0 -amdgpu-igrouplp-exact-solver-max-branches=250000 < %s | FileCheck -check-prefix=EXACTCUTOFF %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx90a -misched-cluster=0 -amdgpu-mfma-vgpr-form=0 < %s | FileCheck -check-prefixes=GCN,SDAG %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx90a -misched-cluster=0 -amdgpu-mfma-vgpr-form=0 < %s | FileCheck -check-prefixes=GCN,GISEL %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx90a -misched-cluster=0 -amdgpu-mfma-vgpr-form=0 -amdgpu-igrouplp-exact-solver-max-branches=250000 < %s | FileCheck -check-prefix=EXACTCUTOFF %s
define amdgpu_kernel void @test_sched_group_barrier() #0 {
; GCN-LABEL: test_sched_group_barrier:
@@ -27,73 +28,141 @@ entry:
}
define amdgpu_kernel void @test_sched_group_barrier_pipeline_READ_VALU_WRITE(ptr addrspace(1) noalias %in, ptr addrspace(1) noalias %out) #0 {
-; GCN-LABEL: test_sched_group_barrier_pipeline_READ_VALU_WRITE:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCN-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GCN-NEXT: v_lshlrev_b32_e32 v32, 7, v0
-; GCN-NEXT: ; kill: killed $sgpr0_sgpr1
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: global_load_dwordx4 v[0:3], v32, s[0:1]
-; GCN-NEXT: global_load_dwordx4 v[4:7], v32, s[0:1] offset:16
-; GCN-NEXT: global_load_dwordx4 v[8:11], v32, s[0:1] offset:32
-; GCN-NEXT: global_load_dwordx4 v[12:15], v32, s[0:1] offset:48
-; GCN-NEXT: global_load_dwordx4 v[16:19], v32, s[0:1] offset:64
-; GCN-NEXT: global_load_dwordx4 v[20:23], v32, s[0:1] offset:80
-; GCN-NEXT: global_load_dwordx4 v[24:27], v32, s[0:1] offset:96
-; GCN-NEXT: global_load_dwordx4 v[28:31], v32, s[0:1] offset:112
-; GCN-NEXT: ; sched_group_barrier mask(0x00000020) size(8) SyncID(0)
-; GCN-NEXT: s_waitcnt vmcnt(7)
-; GCN-NEXT: v_mul_lo_u32 v3, v3, v3
-; GCN-NEXT: v_mul_lo_u32 v2, v2, v2
-; GCN-NEXT: v_mul_lo_u32 v1, v1, v1
-; GCN-NEXT: v_mul_lo_u32 v0, v0, v0
-; GCN-NEXT: s_waitcnt vmcnt(6)
-; GCN-NEXT: v_mul_lo_u32 v7, v7, v7
-; GCN-NEXT: v_mul_lo_u32 v6, v6, v6
-; GCN-NEXT: v_mul_lo_u32 v5, v5, v5
-; GCN-NEXT: v_mul_lo_u32 v4, v4, v4
-; GCN-NEXT: s_waitcnt vmcnt(5)
-; GCN-NEXT: v_mul_lo_u32 v11, v11, v11
-; GCN-NEXT: v_mul_lo_u32 v10, v10, v10
-; GCN-NEXT: v_mul_lo_u32 v9, v9, v9
-; GCN-NEXT: v_mul_lo_u32 v8, v8, v8
-; GCN-NEXT: s_waitcnt vmcnt(4)
-; GCN-NEXT: v_mul_lo_u32 v15, v15, v15
-; GCN-NEXT: v_mul_lo_u32 v14, v14, v14
-; GCN-NEXT: v_mul_lo_u32 v13, v13, v13
-; GCN-NEXT: v_mul_lo_u32 v12, v12, v12
-; GCN-NEXT: s_waitcnt vmcnt(3)
-; GCN-NEXT: v_mul_lo_u32 v19, v19, v19
-; GCN-NEXT: v_mul_lo_u32 v18, v18, v18
-; GCN-NEXT: v_mul_lo_u32 v17, v17, v17
-; GCN-NEXT: v_mul_lo_u32 v16, v16, v16
-; GCN-NEXT: s_waitcnt vmcnt(2)
-; GCN-NEXT: v_mul_lo_u32 v23, v23, v23
-; GCN-NEXT: v_mul_lo_u32 v22, v22, v22
-; GCN-NEXT: v_mul_lo_u32 v21, v21, v21
-; GCN-NEXT: v_mul_lo_u32 v20, v20, v20
-; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_mul_lo_u32 v27, v27, v27
-; GCN-NEXT: v_mul_lo_u32 v26, v26, v26
-; GCN-NEXT: v_mul_lo_u32 v25, v25, v25
-; GCN-NEXT: v_mul_lo_u32 v24, v24, v24
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_mul_lo_u32 v31, v31, v31
-; GCN-NEXT: v_mul_lo_u32 v30, v30, v30
-; GCN-NEXT: v_mul_lo_u32 v29, v29, v29
-; GCN-NEXT: v_mul_lo_u32 v28, v28, v28
-; GCN-NEXT: global_store_dwordx4 v32, v[28:31], s[2:3] offset:112
-; GCN-NEXT: global_store_dwordx4 v32, v[24:27], s[2:3] offset:96
-; GCN-NEXT: global_store_dwordx4 v32, v[20:23], s[2:3] offset:80
-; GCN-NEXT: global_store_dwordx4 v32, v[16:19], s[2:3] offset:64
-; GCN-NEXT: global_store_dwordx4 v32, v[12:15], s[2:3] offset:48
-; GCN-NEXT: global_store_dwordx4 v32, v[8:11], s[2:3] offset:32
-; GCN-NEXT: global_store_dwordx4 v32, v[4:7], s[2:3] offset:16
-; GCN-NEXT: global_store_dwordx4 v32, v[0:3], s[2:3]
-; GCN-NEXT: ; sched_group_barrier mask(0x00000002) size(30) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000040) size(8) SyncID(0)
-; GCN-NEXT: s_endpgm
+; SDAG-LABEL: test_sched_group_barrier_pipeline_READ_VALU_WRITE:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; SDAG-NEXT: v_lshlrev_b32_e32 v32, 7, v0
+; SDAG-NEXT: ; kill: killed $sgpr0_sgpr1
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: global_load_dwordx4 v[0:3], v32, s[0:1]
+; SDAG-NEXT: global_load_dwordx4 v[4:7], v32, s[0:1] offset:16
+; SDAG-NEXT: global_load_dwordx4 v[8:11], v32, s[0:1] offset:32
+; SDAG-NEXT: global_load_dwordx4 v[12:15], v32, s[0:1] offset:48
+; SDAG-NEXT: global_load_dwordx4 v[16:19], v32, s[0:1] offset:64
+; SDAG-NEXT: global_load_dwordx4 v[20:23], v32, s[0:1] offset:80
+; SDAG-NEXT: global_load_dwordx4 v[24:27], v32, s[0:1] offset:96
+; SDAG-NEXT: global_load_dwordx4 v[28:31], v32, s[0:1] offset:112
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000020) size(8) SyncID(0)
+; SDAG-NEXT: s_waitcnt vmcnt(7)
+; SDAG-NEXT: v_mul_lo_u32 v3, v3, v3
+; SDAG-NEXT: v_mul_lo_u32 v2, v2, v2
+; SDAG-NEXT: v_mul_lo_u32 v1, v1, v1
+; SDAG-NEXT: v_mul_lo_u32 v0, v0, v0
+; SDAG-NEXT: s_waitcnt vmcnt(6)
+; SDAG-NEXT: v_mul_lo_u32 v7, v7, v7
+; SDAG-NEXT: v_mul_lo_u32 v6, v6, v6
+; SDAG-NEXT: v_mul_lo_u32 v5, v5, v5
+; SDAG-NEXT: v_mul_lo_u32 v4, v4, v4
+; SDAG-NEXT: s_waitcnt vmcnt(5)
+; SDAG-NEXT: v_mul_lo_u32 v11, v11, v11
+; SDAG-NEXT: v_mul_lo_u32 v10, v10, v10
+; SDAG-NEXT: v_mul_lo_u32 v9, v9, v9
+; SDAG-NEXT: v_mul_lo_u32 v8, v8, v8
+; SDAG-NEXT: s_waitcnt vmcnt(4)
+; SDAG-NEXT: v_mul_lo_u32 v15, v15, v15
+; SDAG-NEXT: v_mul_lo_u32 v14, v14, v14
+; SDAG-NEXT: v_mul_lo_u32 v13, v13, v13
+; SDAG-NEXT: v_mul_lo_u32 v12, v12, v12
+; SDAG-NEXT: s_waitcnt vmcnt(3)
+; SDAG-NEXT: v_mul_lo_u32 v19, v19, v19
+; SDAG-NEXT: v_mul_lo_u32 v18, v18, v18
+; SDAG-NEXT: v_mul_lo_u32 v17, v17, v17
+; SDAG-NEXT: v_mul_lo_u32 v16, v16, v16
+; SDAG-NEXT: s_waitcnt vmcnt(2)
+; SDAG-NEXT: v_mul_lo_u32 v23, v23, v23
+; SDAG-NEXT: v_mul_lo_u32 v22, v22, v22
+; SDAG-NEXT: v_mul_lo_u32 v21, v21, v21
+; SDAG-NEXT: v_mul_lo_u32 v20, v20, v20
+; SDAG-NEXT: s_waitcnt vmcnt(1)
+; SDAG-NEXT: v_mul_lo_u32 v27, v27, v27
+; SDAG-NEXT: v_mul_lo_u32 v26, v26, v26
+; SDAG-NEXT: v_mul_lo_u32 v25, v25, v25
+; SDAG-NEXT: v_mul_lo_u32 v24, v24, v24
+; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: v_mul_lo_u32 v31, v31, v31
+; SDAG-NEXT: v_mul_lo_u32 v30, v30, v30
+; SDAG-NEXT: v_mul_lo_u32 v29, v29, v29
+; SDAG-NEXT: v_mul_lo_u32 v28, v28, v28
+; SDAG-NEXT: global_store_dwordx4 v32, v[28:31], s[2:3] offset:112
+; SDAG-NEXT: global_store_dwordx4 v32, v[24:27], s[2:3] offset:96
+; SDAG-NEXT: global_store_dwordx4 v32, v[20:23], s[2:3] offset:80
+; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[2:3] offset:64
+; SDAG-NEXT: global_store_dwordx4 v32, v[12:15], s[2:3] offset:48
+; SDAG-NEXT: global_store_dwordx4 v32, v[8:11], s[2:3] offset:32
+; SDAG-NEXT: global_store_dwordx4 v32, v[4:7], s[2:3] offset:16
+; SDAG-NEXT: global_store_dwordx4 v32, v[0:3], s[2:3]
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000002) size(30) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000040) size(8) SyncID(0)
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: test_sched_group_barrier_pipeline_READ_VALU_WRITE:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GISEL-NEXT: v_lshlrev_b32_e32 v32, 7, v0
+; GISEL-NEXT: ; kill: killed $sgpr0_sgpr1
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: global_load_dwordx4 v[0:3], v32, s[0:1]
+; GISEL-NEXT: global_load_dwordx4 v[4:7], v32, s[0:1] offset:16
+; GISEL-NEXT: global_load_dwordx4 v[8:11], v32, s[0:1] offset:32
+; GISEL-NEXT: global_load_dwordx4 v[12:15], v32, s[0:1] offset:48
+; GISEL-NEXT: global_load_dwordx4 v[16:19], v32, s[0:1] offset:64
+; GISEL-NEXT: global_load_dwordx4 v[20:23], v32, s[0:1] offset:80
+; GISEL-NEXT: global_load_dwordx4 v[24:27], v32, s[0:1] offset:96
+; GISEL-NEXT: global_load_dwordx4 v[28:31], v32, s[0:1] offset:112
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000020) size(8) SyncID(0)
+; GISEL-NEXT: s_waitcnt vmcnt(7)
+; GISEL-NEXT: v_mul_lo_u32 v0, v0, v0
+; GISEL-NEXT: v_mul_lo_u32 v1, v1, v1
+; GISEL-NEXT: v_mul_lo_u32 v2, v2, v2
+; GISEL-NEXT: v_mul_lo_u32 v3, v3, v3
+; GISEL-NEXT: s_waitcnt vmcnt(6)
+; GISEL-NEXT: v_mul_lo_u32 v4, v4, v4
+; GISEL-NEXT: v_mul_lo_u32 v5, v5, v5
+; GISEL-NEXT: v_mul_lo_u32 v6, v6, v6
+; GISEL-NEXT: v_mul_lo_u32 v7, v7, v7
+; GISEL-NEXT: s_waitcnt vmcnt(5)
+; GISEL-NEXT: v_mul_lo_u32 v8, v8, v8
+; GISEL-NEXT: v_mul_lo_u32 v9, v9, v9
+; GISEL-NEXT: v_mul_lo_u32 v10, v10, v10
+; GISEL-NEXT: v_mul_lo_u32 v11, v11, v11
+; GISEL-NEXT: s_waitcnt vmcnt(4)
+; GISEL-NEXT: v_mul_lo_u32 v12, v12, v12
+; GISEL-NEXT: v_mul_lo_u32 v13, v13, v13
+; GISEL-NEXT: v_mul_lo_u32 v14, v14, v14
+; GISEL-NEXT: v_mul_lo_u32 v15, v15, v15
+; GISEL-NEXT: s_waitcnt vmcnt(3)
+; GISEL-NEXT: v_mul_lo_u32 v16, v16, v16
+; GISEL-NEXT: v_mul_lo_u32 v17, v17, v17
+; GISEL-NEXT: v_mul_lo_u32 v18, v18, v18
+; GISEL-NEXT: v_mul_lo_u32 v19, v19, v19
+; GISEL-NEXT: s_waitcnt vmcnt(2)
+; GISEL-NEXT: v_mul_lo_u32 v20, v20, v20
+; GISEL-NEXT: v_mul_lo_u32 v21, v21, v21
+; GISEL-NEXT: v_mul_lo_u32 v22, v22, v22
+; GISEL-NEXT: v_mul_lo_u32 v23, v23, v23
+; GISEL-NEXT: s_waitcnt vmcnt(1)
+; GISEL-NEXT: v_mul_lo_u32 v24, v24, v24
+; GISEL-NEXT: v_mul_lo_u32 v25, v25, v25
+; GISEL-NEXT: v_mul_lo_u32 v26, v26, v26
+; GISEL-NEXT: v_mul_lo_u32 v27, v27, v27
+; GISEL-NEXT: s_waitcnt vmcnt(0)
+; GISEL-NEXT: v_mul_lo_u32 v28, v28, v28
+; GISEL-NEXT: v_mul_lo_u32 v29, v29, v29
+; GISEL-NEXT: v_mul_lo_u32 v30, v30, v30
+; GISEL-NEXT: v_mul_lo_u32 v31, v31, v31
+; GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[2:3]
+; GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[2:3] offset:16
+; GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[2:3] offset:32
+; GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[2:3] offset:48
+; GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[2:3] offset:64
+; GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[2:3] offset:80
+; GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[2:3] offset:96
+; GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[2:3] offset:112
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000002) size(30) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000040) size(8) SyncID(0)
+; GISEL-NEXT: s_endpgm
;
; EXACTCUTOFF-LABEL: test_sched_group_barrier_pipeline_READ_VALU_WRITE:
; EXACTCUTOFF: ; %bb.0:
@@ -178,86 +247,167 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_READ_VALU_WRITE(ptr
}
define amdgpu_kernel void @test_sched_group_barrier_pipeline_alternating_READ_VALU(ptr addrspace(1) noalias %in, ptr addrspace(1) noalias %out) #0 {
-; GCN-LABEL: test_sched_group_barrier_pipeline_alternating_READ_VALU:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCN-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GCN-NEXT: v_lshlrev_b32_e32 v32, 7, v0
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: global_load_dwordx4 v[28:31], v32, s[0:1] offset:16
-; GCN-NEXT: global_load_dwordx4 v[0:3], v32, s[0:1]
-; GCN-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
-; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_mul_lo_u32 v29, v29, v29
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_mul_lo_u32 v3, v3, v3
-; GCN-NEXT: v_mul_lo_u32 v2, v2, v2
-; GCN-NEXT: global_load_dwordx4 v[4:7], v32, s[0:1] offset:112
-; GCN-NEXT: v_mul_lo_u32 v1, v1, v1
-; GCN-NEXT: v_mul_lo_u32 v0, v0, v0
-; GCN-NEXT: global_load_dwordx4 v[8:11], v32, s[0:1] offset:96
-; GCN-NEXT: v_mul_lo_u32 v28, v28, v28
-; GCN-NEXT: v_mul_lo_u32 v31, v31, v31
-; GCN-NEXT: v_mul_lo_u32 v30, v30, v30
-; GCN-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
-; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_mul_lo_u32 v7, v7, v7
-; GCN-NEXT: v_mul_lo_u32 v6, v6, v6
-; GCN-NEXT: global_load_dwordx4 v[12:15], v32, s[0:1] offset:80
-; GCN-NEXT: v_mul_lo_u32 v5, v5, v5
-; GCN-NEXT: v_mul_lo_u32 v4, v4, v4
-; GCN-NEXT: global_load_dwordx4 v[16:19], v32, s[0:1] offset:48
-; GCN-NEXT: s_waitcnt vmcnt(2)
-; GCN-NEXT: v_mul_lo_u32 v11, v11, v11
-; GCN-NEXT: v_mul_lo_u32 v10, v10, v10
-; GCN-NEXT: v_mul_lo_u32 v9, v9, v9
-; GCN-NEXT: v_mul_lo_u32 v8, v8, v8
-; GCN-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
-; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_mul_lo_u32 v15, v15, v15
-; GCN-NEXT: v_mul_lo_u32 v14, v14, v14
-; GCN-NEXT: v_mul_lo_u32 v13, v13, v13
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_mul_lo_u32 v19, v19, v19
-; GCN-NEXT: v_mul_lo_u32 v18, v18, v18
-; GCN-NEXT: global_load_dwordx4 v[20:23], v32, s[0:1] offset:64
-; GCN-NEXT: global_load_dwordx4 v[24:27], v32, s[0:1] offset:32
-; GCN-NEXT: v_mul_lo_u32 v17, v17, v17
-; GCN-NEXT: v_mul_lo_u32 v16, v16, v16
-; GCN-NEXT: v_mul_lo_u32 v12, v12, v12
-; GCN-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
-; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_mul_lo_u32 v23, v23, v23
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_mul_lo_u32 v27, v27, v27
-; GCN-NEXT: v_mul_lo_u32 v26, v26, v26
-; GCN-NEXT: v_mul_lo_u32 v25, v25, v25
-; GCN-NEXT: v_mul_lo_u32 v24, v24, v24
-; GCN-NEXT: v_mul_lo_u32 v22, v22, v22
-; GCN-NEXT: v_mul_lo_u32 v21, v21, v21
-; GCN-NEXT: v_mul_lo_u32 v20, v20, v20
-; GCN-NEXT: global_store_dwordx4 v32, v[4:7], s[2:3] offset:112
-; GCN-NEXT: global_store_dwordx4 v32, v[8:11], s[2:3] offset:96
-; GCN-NEXT: global_store_dwordx4 v32, v[12:15], s[2:3] offset:80
-; GCN-NEXT: global_store_dwordx4 v32, v[20:23], s[2:3] offset:64
-; GCN-NEXT: global_store_dwordx4 v32, v[16:19], s[2:3] offset:48
-; GCN-NEXT: global_store_dwordx4 v32, v[24:27], s[2:3] offset:32
-; GCN-NEXT: global_store_dwordx4 v32, v[28:31], s[2:3] offset:16
-; GCN-NEXT: global_store_dwordx4 v32, v[0:3], s[2:3]
-; GCN-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000040) size(8) SyncID(0)
-; GCN-NEXT: s_endpgm
+; SDAG-LABEL: test_sched_group_barrier_pipeline_alternating_READ_VALU:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; SDAG-NEXT: v_lshlrev_b32_e32 v32, 7, v0
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: global_load_dwordx4 v[28:31], v32, s[0:1] offset:16
+; SDAG-NEXT: global_load_dwordx4 v[0:3], v32, s[0:1]
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
+; SDAG-NEXT: s_waitcnt vmcnt(1)
+; SDAG-NEXT: v_mul_lo_u32 v29, v29, v29
+; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: v_mul_lo_u32 v3, v3, v3
+; SDAG-NEXT: v_mul_lo_u32 v2, v2, v2
+; SDAG-NEXT: global_load_dwordx4 v[4:7], v32, s[0:1] offset:112
+; SDAG-NEXT: v_mul_lo_u32 v1, v1, v1
+; SDAG-NEXT: v_mul_lo_u32 v0, v0, v0
+; SDAG-NEXT: global_load_dwordx4 v[8:11], v32, s[0:1] offset:96
+; SDAG-NEXT: v_mul_lo_u32 v28, v28, v28
+; SDAG-NEXT: v_mul_lo_u32 v31, v31, v31
+; SDAG-NEXT: v_mul_lo_u32 v30, v30, v30
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
+; SDAG-NEXT: s_waitcnt vmcnt(1)
+; SDAG-NEXT: v_mul_lo_u32 v7, v7, v7
+; SDAG-NEXT: v_mul_lo_u32 v6, v6, v6
+; SDAG-NEXT: global_load_dwordx4 v[12:15], v32, s[0:1] offset:80
+; SDAG-NEXT: v_mul_lo_u32 v5, v5, v5
+; SDAG-NEXT: v_mul_lo_u32 v4, v4, v4
+; SDAG-NEXT: global_load_dwordx4 v[16:19], v32, s[0:1] offset:48
+; SDAG-NEXT: s_waitcnt vmcnt(2)
+; SDAG-NEXT: v_mul_lo_u32 v11, v11, v11
+; SDAG-NEXT: v_mul_lo_u32 v10, v10, v10
+; SDAG-NEXT: v_mul_lo_u32 v9, v9, v9
+; SDAG-NEXT: v_mul_lo_u32 v8, v8, v8
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
+; SDAG-NEXT: s_waitcnt vmcnt(1)
+; SDAG-NEXT: v_mul_lo_u32 v15, v15, v15
+; SDAG-NEXT: v_mul_lo_u32 v14, v14, v14
+; SDAG-NEXT: v_mul_lo_u32 v13, v13, v13
+; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: v_mul_lo_u32 v19, v19, v19
+; SDAG-NEXT: v_mul_lo_u32 v18, v18, v18
+; SDAG-NEXT: global_load_dwordx4 v[20:23], v32, s[0:1] offset:64
+; SDAG-NEXT: global_load_dwordx4 v[24:27], v32, s[0:1] offset:32
+; SDAG-NEXT: v_mul_lo_u32 v17, v17, v17
+; SDAG-NEXT: v_mul_lo_u32 v16, v16, v16
+; SDAG-NEXT: v_mul_lo_u32 v12, v12, v12
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
+; SDAG-NEXT: s_waitcnt vmcnt(1)
+; SDAG-NEXT: v_mul_lo_u32 v23, v23, v23
+; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: v_mul_lo_u32 v27, v27, v27
+; SDAG-NEXT: v_mul_lo_u32 v26, v26, v26
+; SDAG-NEXT: v_mul_lo_u32 v25, v25, v25
+; SDAG-NEXT: v_mul_lo_u32 v24, v24, v24
+; SDAG-NEXT: v_mul_lo_u32 v22, v22, v22
+; SDAG-NEXT: v_mul_lo_u32 v21, v21, v21
+; SDAG-NEXT: v_mul_lo_u32 v20, v20, v20
+; SDAG-NEXT: global_store_dwordx4 v32, v[4:7], s[2:3] offset:112
+; SDAG-NEXT: global_store_dwordx4 v32, v[8:11], s[2:3] offset:96
+; SDAG-NEXT: global_store_dwordx4 v32, v[12:15], s[2:3] offset:80
+; SDAG-NEXT: global_store_dwordx4 v32, v[20:23], s[2:3] offset:64
+; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[2:3] offset:48
+; SDAG-NEXT: global_store_dwordx4 v32, v[24:27], s[2:3] offset:32
+; SDAG-NEXT: global_store_dwordx4 v32, v[28:31], s[2:3] offset:16
+; SDAG-NEXT: global_store_dwordx4 v32, v[0:3], s[2:3]
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000040) size(8) SyncID(0)
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: test_sched_group_barrier_pipeline_alternating_READ_VALU:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GISEL-NEXT: v_lshlrev_b32_e32 v32, 7, v0
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: global_load_dwordx4 v[28:31], v32, s[0:1] offset:96
+; GISEL-NEXT: global_load_dwordx4 v[0:3], v32, s[0:1] offset:112
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
+; GISEL-NEXT: s_waitcnt vmcnt(1)
+; GISEL-NEXT: v_mul_lo_u32 v28, v28, v28
+; GISEL-NEXT: s_waitcnt vmcnt(0)
+; GISEL-NEXT: v_mul_lo_u32 v0, v0, v0
+; GISEL-NEXT: v_mul_lo_u32 v1, v1, v1
+; GISEL-NEXT: global_load_dwordx4 v[4:7], v32, s[0:1]
+; GISEL-NEXT: v_mul_lo_u32 v2, v2, v2
+; GISEL-NEXT: v_mul_lo_u32 v3, v3, v3
+; GISEL-NEXT: v_mul_lo_u32 v29, v29, v29
+; GISEL-NEXT: v_mul_lo_u32 v30, v30, v30
+; GISEL-NEXT: v_mul_lo_u32 v31, v31, v31
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
+; GISEL-NEXT: s_waitcnt vmcnt(0)
+; GISEL-NEXT: v_mul_lo_u32 v4, v4, v4
+; GISEL-NEXT: v_mul_lo_u32 v5, v5, v5
+; GISEL-NEXT: global_load_dwordx4 v[8:11], v32, s[0:1] offset:16
+; GISEL-NEXT: v_mul_lo_u32 v6, v6, v6
+; GISEL-NEXT: v_mul_lo_u32 v7, v7, v7
+; GISEL-NEXT: global_load_dwordx4 v[12:15], v32, s[0:1] offset:32
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
+; GISEL-NEXT: s_waitcnt vmcnt(1)
+; GISEL-NEXT: v_mul_lo_u32 v8, v8, v8
+; GISEL-NEXT: v_mul_lo_u32 v9, v9, v9
+; GISEL-NEXT: global_load_dwordx4 v[16:19], v32, s[0:1] offset:48
+; GISEL-NEXT: v_mul_lo_u32 v10, v10, v10
+; GISEL-NEXT: v_mul_lo_u32 v11, v11, v11
+; GISEL-NEXT: global_load_dwordx4 v[20:23], v32, s[0:1] offset:64
+; GISEL-NEXT: s_waitcnt vmcnt(2)
+; GISEL-NEXT: v_mul_lo_u32 v12, v12, v12
+; GISEL-NEXT: v_mul_lo_u32 v13, v13, v13
+; GISEL-NEXT: global_load_dwordx4 v[24:27], v32, s[0:1] offset:80
+; GISEL-NEXT: v_mul_lo_u32 v14, v14, v14
+; GISEL-NEXT: v_mul_lo_u32 v15, v15, v15
+; GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[2:3]
+; GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[2:3] offset:16
+; GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[2:3] offset:32
+; GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[2:3] offset:96
+; GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[2:3] offset:112
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
+; GISEL-NEXT: s_waitcnt vmcnt(7)
+; GISEL-NEXT: v_mul_lo_u32 v16, v16, v16
+; GISEL-NEXT: v_mul_lo_u32 v17, v17, v17
+; GISEL-NEXT: v_mul_lo_u32 v18, v18, v18
+; GISEL-NEXT: v_mul_lo_u32 v19, v19, v19
+; GISEL-NEXT: s_waitcnt vmcnt(6)
+; GISEL-NEXT: v_mul_lo_u32 v20, v20, v20
+; GISEL-NEXT: v_mul_lo_u32 v21, v21, v21
+; GISEL-NEXT: v_mul_lo_u32 v22, v22, v22
+; GISEL-NEXT: v_mul_lo_u32 v23, v23, v23
+; GISEL-NEXT: s_waitcnt vmcnt(5)
+; GISEL-NEXT: v_mul_lo_u32 v24, v24, v24
+; GISEL-NEXT: v_mul_lo_u32 v25, v25, v25
+; GISEL-NEXT: v_mul_lo_u32 v26, v26, v26
+; GISEL-NEXT: v_mul_lo_u32 v27, v27, v27
+; GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[2:3] offset:48
+; GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[2:3] offset:64
+; GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[2:3] offset:80
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000040) size(8) SyncID(0)
+; GISEL-NEXT: s_endpgm
;
; EXACTCUTOFF-LABEL: test_sched_group_barrier_pipeline_alternating_READ_VALU:
; EXACTCUTOFF: ; %bb.0:
@@ -383,94 +533,182 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_alternating_READ_VA
}
define amdgpu_kernel void @test_sched_group_barrier_pipeline_alternating_READ_VALU_WRITE(ptr addrspace(1) noalias %in, ptr addrspace(1) noalias %out) #0 {
-; GCN-LABEL: test_sched_group_barrier_pipeline_alternating_READ_VALU_WRITE:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCN-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GCN-NEXT: v_lshlrev_b32_e32 v16, 7, v0
-; GCN-NEXT: ; kill: killed $sgpr0_sgpr1
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: global_load_dwordx4 v[12:15], v16, s[0:1] offset:32
-; GCN-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_mul_lo_u32 v13, v13, v13
-; GCN-NEXT: v_mul_lo_u32 v12, v12, v12
-; GCN-NEXT: v_mul_lo_u32 v15, v15, v15
-; GCN-NEXT: v_mul_lo_u32 v14, v14, v14
-; GCN-NEXT: global_store_dwordx4 v16, v[12:15], s[2:3] offset:32
-; GCN-NEXT: global_load_dwordx4 v[0:3], v16, s[0:1]
-; GCN-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_mul_lo_u32 v3, v3, v3
-; GCN-NEXT: v_mul_lo_u32 v2, v2, v2
-; GCN-NEXT: v_mul_lo_u32 v1, v1, v1
-; GCN-NEXT: v_mul_lo_u32 v0, v0, v0
-; GCN-NEXT: global_store_dwordx4 v16, v[0:3], s[2:3]
-; GCN-NEXT: global_load_dwordx4 v[0:3], v16, s[0:1] offset:112
-; GCN-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_mul_lo_u32 v3, v3, v3
-; GCN-NEXT: v_mul_lo_u32 v2, v2, v2
-; GCN-NEXT: v_mul_lo_u32 v1, v1, v1
-; GCN-NEXT: v_mul_lo_u32 v0, v0, v0
-; GCN-NEXT: global_store_dwordx4 v16, v[0:3], s[2:3] offset:112
-; GCN-NEXT: global_load_dwordx4 v[0:3], v16, s[0:1] offset:96
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_mul_lo_u32 v3, v3, v3
-; GCN-NEXT: v_mul_lo_u32 v2, v2, v2
-; GCN-NEXT: v_mul_lo_u32 v1, v1, v1
-; GCN-NEXT: v_mul_lo_u32 v0, v0, v0
-; GCN-NEXT: global_store_dwordx4 v16, v[0:3], s[2:3] offset:96
-; GCN-NEXT: global_load_dwordx4 v[0:3], v16, s[0:1] offset:80
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_mul_lo_u32 v3, v3, v3
-; GCN-NEXT: v_mul_lo_u32 v2, v2, v2
-; GCN-NEXT: v_mul_lo_u32 v1, v1, v1
-; GCN-NEXT: v_mul_lo_u32 v0, v0, v0
-; GCN-NEXT: global_store_dwordx4 v16, v[0:3], s[2:3] offset:80
-; GCN-NEXT: global_load_dwordx4 v[4:7], v16, s[0:1] offset:48
-; GCN-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_mul_lo_u32 v7, v7, v7
-; GCN-NEXT: v_mul_lo_u32 v6, v6, v6
-; GCN-NEXT: v_mul_lo_u32 v5, v5, v5
-; GCN-NEXT: v_mul_lo_u32 v4, v4, v4
-; GCN-NEXT: global_store_dwordx4 v16, v[4:7], s[2:3] offset:48
-; GCN-NEXT: global_load_dwordx4 v[8:11], v16, s[0:1] offset:16
-; GCN-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_mul_lo_u32 v9, v9, v9
-; GCN-NEXT: v_mul_lo_u32 v8, v8, v8
-; GCN-NEXT: v_mul_lo_u32 v11, v11, v11
-; GCN-NEXT: v_mul_lo_u32 v10, v10, v10
-; GCN-NEXT: global_store_dwordx4 v16, v[8:11], s[2:3] offset:16
-; GCN-NEXT: global_load_dwordx4 v[8:11], v16, s[0:1] offset:64
-; GCN-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_mul_lo_u32 v11, v11, v11
-; GCN-NEXT: v_mul_lo_u32 v10, v10, v10
-; GCN-NEXT: v_mul_lo_u32 v9, v9, v9
-; GCN-NEXT: v_mul_lo_u32 v8, v8, v8
-; GCN-NEXT: global_store_dwordx4 v16, v[8:11], s[2:3] offset:64
-; GCN-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
-; GCN-NEXT: s_endpgm
+; SDAG-LABEL: test_sched_group_barrier_pipeline_alternating_READ_VALU_WRITE:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; SDAG-NEXT: v_lshlrev_b32_e32 v16, 7, v0
+; SDAG-NEXT: ; kill: killed $sgpr0_sgpr1
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: global_load_dwordx4 v[12:15], v16, s[0:1] offset:32
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
+; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: v_mul_lo_u32 v13, v13, v13
+; SDAG-NEXT: v_mul_lo_u32 v12, v12, v12
+; SDAG-NEXT: v_mul_lo_u32 v15, v15, v15
+; SDAG-NEXT: v_mul_lo_u32 v14, v14, v14
+; SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[2:3] offset:32
+; SDAG-NEXT: global_load_dwordx4 v[0:3], v16, s[0:1]
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
+; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: v_mul_lo_u32 v3, v3, v3
+; SDAG-NEXT: v_mul_lo_u32 v2, v2, v2
+; SDAG-NEXT: v_mul_lo_u32 v1, v1, v1
+; SDAG-NEXT: v_mul_lo_u32 v0, v0, v0
+; SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[2:3]
+; SDAG-NEXT: global_load_dwordx4 v[0:3], v16, s[0:1] offset:112
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
+; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: v_mul_lo_u32 v3, v3, v3
+; SDAG-NEXT: v_mul_lo_u32 v2, v2, v2
+; SDAG-NEXT: v_mul_lo_u32 v1, v1, v1
+; SDAG-NEXT: v_mul_lo_u32 v0, v0, v0
+; SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[2:3] offset:112
+; SDAG-NEXT: global_load_dwordx4 v[0:3], v16, s[0:1] offset:96
+; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: v_mul_lo_u32 v3, v3, v3
+; SDAG-NEXT: v_mul_lo_u32 v2, v2, v2
+; SDAG-NEXT: v_mul_lo_u32 v1, v1, v1
+; SDAG-NEXT: v_mul_lo_u32 v0, v0, v0
+; SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[2:3] offset:96
+; SDAG-NEXT: global_load_dwordx4 v[0:3], v16, s[0:1] offset:80
+; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: v_mul_lo_u32 v3, v3, v3
+; SDAG-NEXT: v_mul_lo_u32 v2, v2, v2
+; SDAG-NEXT: v_mul_lo_u32 v1, v1, v1
+; SDAG-NEXT: v_mul_lo_u32 v0, v0, v0
+; SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[2:3] offset:80
+; SDAG-NEXT: global_load_dwordx4 v[4:7], v16, s[0:1] offset:48
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
+; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: v_mul_lo_u32 v7, v7, v7
+; SDAG-NEXT: v_mul_lo_u32 v6, v6, v6
+; SDAG-NEXT: v_mul_lo_u32 v5, v5, v5
+; SDAG-NEXT: v_mul_lo_u32 v4, v4, v4
+; SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[2:3] offset:48
+; SDAG-NEXT: global_load_dwordx4 v[8:11], v16, s[0:1] offset:16
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
+; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: v_mul_lo_u32 v9, v9, v9
+; SDAG-NEXT: v_mul_lo_u32 v8, v8, v8
+; SDAG-NEXT: v_mul_lo_u32 v11, v11, v11
+; SDAG-NEXT: v_mul_lo_u32 v10, v10, v10
+; SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[2:3] offset:16
+; SDAG-NEXT: global_load_dwordx4 v[8:11], v16, s[0:1] offset:64
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
+; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: v_mul_lo_u32 v11, v11, v11
+; SDAG-NEXT: v_mul_lo_u32 v10, v10, v10
+; SDAG-NEXT: v_mul_lo_u32 v9, v9, v9
+; SDAG-NEXT: v_mul_lo_u32 v8, v8, v8
+; SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[2:3] offset:64
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: test_sched_group_barrier_pipeline_alternating_READ_VALU_WRITE:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GISEL-NEXT: v_lshlrev_b32_e32 v12, 7, v0
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: global_load_dwordx4 v[4:7], v12, s[0:1] offset:96
+; GISEL-NEXT: s_waitcnt vmcnt(0)
+; GISEL-NEXT: v_mul_lo_u32 v4, v4, v4
+; GISEL-NEXT: v_mul_lo_u32 v5, v5, v5
+; GISEL-NEXT: v_mul_lo_u32 v6, v6, v6
+; GISEL-NEXT: v_mul_lo_u32 v7, v7, v7
+; GISEL-NEXT: global_store_dwordx4 v12, v[4:7], s[2:3] offset:96
+; GISEL-NEXT: global_load_dwordx4 v[4:7], v12, s[0:1] offset:16
+; GISEL-NEXT: s_waitcnt vmcnt(0)
+; GISEL-NEXT: v_mul_lo_u32 v4, v4, v4
+; GISEL-NEXT: v_mul_lo_u32 v5, v5, v5
+; GISEL-NEXT: v_mul_lo_u32 v6, v6, v6
+; GISEL-NEXT: v_mul_lo_u32 v7, v7, v7
+; GISEL-NEXT: global_store_dwordx4 v12, v[4:7], s[2:3] offset:16
+; GISEL-NEXT: global_load_dwordx4 v[4:7], v12, s[0:1] offset:64
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
+; GISEL-NEXT: s_waitcnt vmcnt(0)
+; GISEL-NEXT: v_mul_lo_u32 v4, v4, v4
+; GISEL-NEXT: global_load_dwordx4 v[8:11], v12, s[0:1] offset:80
+; GISEL-NEXT: v_mul_lo_u32 v5, v5, v5
+; GISEL-NEXT: v_mul_lo_u32 v6, v6, v6
+; GISEL-NEXT: v_mul_lo_u32 v7, v7, v7
+; GISEL-NEXT: s_waitcnt vmcnt(0)
+; GISEL-NEXT: v_mul_lo_u32 v8, v8, v8
+; GISEL-NEXT: v_mul_lo_u32 v9, v9, v9
+; GISEL-NEXT: v_mul_lo_u32 v10, v10, v10
+; GISEL-NEXT: v_mul_lo_u32 v11, v11, v11
+; GISEL-NEXT: global_store_dwordx4 v12, v[8:11], s[2:3] offset:80
+; GISEL-NEXT: global_load_dwordx4 v[8:11], v12, s[0:1] offset:48
+; GISEL-NEXT: s_waitcnt vmcnt(0)
+; GISEL-NEXT: v_mul_lo_u32 v10, v10, v10
+; GISEL-NEXT: v_mul_lo_u32 v8, v8, v8
+; GISEL-NEXT: v_mul_lo_u32 v9, v9, v9
+; GISEL-NEXT: v_mul_lo_u32 v11, v11, v11
+; GISEL-NEXT: global_store_dwordx4 v12, v[8:11], s[2:3] offset:48
+; GISEL-NEXT: global_load_dwordx4 v[0:3], v12, s[0:1] offset:112
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
+; GISEL-NEXT: s_waitcnt vmcnt(0)
+; GISEL-NEXT: v_mul_lo_u32 v0, v0, v0
+; GISEL-NEXT: v_mul_lo_u32 v1, v1, v1
+; GISEL-NEXT: v_mul_lo_u32 v2, v2, v2
+; GISEL-NEXT: v_mul_lo_u32 v3, v3, v3
+; GISEL-NEXT: global_store_dwordx4 v12, v[0:3], s[2:3] offset:112
+; GISEL-NEXT: global_load_dwordx4 v[0:3], v12, s[0:1]
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
+; GISEL-NEXT: s_waitcnt vmcnt(0)
+; GISEL-NEXT: v_mul_lo_u32 v0, v0, v0
+; GISEL-NEXT: v_mul_lo_u32 v1, v1, v1
+; GISEL-NEXT: v_mul_lo_u32 v2, v2, v2
+; GISEL-NEXT: v_mul_lo_u32 v3, v3, v3
+; GISEL-NEXT: global_store_dwordx4 v12, v[0:3], s[2:3]
+; GISEL-NEXT: global_load_dwordx4 v[0:3], v12, s[0:1] offset:32
+; GISEL-NEXT: s_waitcnt vmcnt(0)
+; GISEL-NEXT: v_mul_lo_u32 v0, v0, v0
+; GISEL-NEXT: v_mul_lo_u32 v1, v1, v1
+; GISEL-NEXT: v_mul_lo_u32 v2, v2, v2
+; GISEL-NEXT: v_mul_lo_u32 v3, v3, v3
+; GISEL-NEXT: global_store_dwordx4 v12, v[0:3], s[2:3] offset:32
+; GISEL-NEXT: global_store_dwordx4 v12, v[4:7], s[2:3] offset:64
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
+; GISEL-NEXT: s_endpgm
;
; EXACTCUTOFF-LABEL: test_sched_group_barrier_pipeline_alternating_READ_VALU_WRITE:
; EXACTCUTOFF: ; %bb.0:
@@ -618,111 +856,378 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_alternating_READ_VA
}
define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_cluster(ptr addrspace(3) noalias %in, ptr addrspace(3) noalias %out) #0 {
-; GCN-LABEL: test_sched_group_barrier_pipeline_MFMA_cluster:
-; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: v_lshlrev_b32_e32 v0, 7, v0
-; GCN-NEXT: v_and_b32_e32 v0, 0x1ff80, v0
-; GCN-NEXT: v_mov_b32_e32 v2, 1.0
-; GCN-NEXT: v_mov_b32_e32 v1, 2.0
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_add_u32_e32 v3, s0, v0
-; GCN-NEXT: ds_read_b128 a[60:63], v3 offset:112
-; GCN-NEXT: ds_read_b128 a[56:59], v3 offset:96
-; GCN-NEXT: ds_read_b128 a[52:55], v3 offset:80
-; GCN-NEXT: ds_read_b128 a[48:51], v3 offset:64
-; GCN-NEXT: ds_read_b128 a[32:35], v3
-; GCN-NEXT: ds_read_b128 a[36:39], v3 offset:16
-; GCN-NEXT: ds_read_b128 a[40:43], v3 offset:32
-; GCN-NEXT: ds_read_b128 a[44:47], v3 offset:48
-; GCN-NEXT: v_add_u32_e32 v4, 0x6000, v3
-; GCN-NEXT: ds_read_b128 a[28:31], v3 offset:8304
-; GCN-NEXT: ds_read_b128 a[24:27], v3 offset:8288
-; GCN-NEXT: ds_read_b128 a[20:23], v3 offset:8272
-; GCN-NEXT: ds_read_b128 a[16:19], v3 offset:8256
-; GCN-NEXT: ds_read_b128 a[12:15], v3 offset:8240
-; GCN-NEXT: ds_read_b128 a[8:11], v3 offset:8224
-; GCN-NEXT: ds_read_b128 a[4:7], v3 offset:8208
-; GCN-NEXT: ds_read_b128 a[0:3], v3 offset:8192
-; GCN-NEXT: ds_read_b128 a[92:95], v3 offset:24688
-; GCN-NEXT: ds_read_b128 a[88:91], v3 offset:24672
-; GCN-NEXT: ds_read_b128 a[84:87], v3 offset:24656
-; GCN-NEXT: ds_read_b128 a[80:83], v3 offset:24640
-; GCN-NEXT: ds_read_b128 a[76:79], v3 offset:24624
-; GCN-NEXT: ds_read_b128 a[72:75], v3 offset:24608
-; GCN-NEXT: ds_read_b128 a[68:71], v3 offset:24592
-; GCN-NEXT: ds_read_b128 a[64:67], v3 offset:24576
-; GCN-NEXT: ds_read_b128 a[124:127], v3 offset:49264
-; GCN-NEXT: ds_read_b128 a[120:123], v3 offset:49248
-; GCN-NEXT: ds_read_b128 a[116:119], v3 offset:49232
-; GCN-NEXT: ds_read_b128 a[112:115], v3 offset:49216
-; GCN-NEXT: ds_read_b128 a[108:111], v3 offset:49200
-; GCN-NEXT: ds_read_b128 a[104:107], v3 offset:49184
-; GCN-NEXT: ds_read_b128 a[100:103], v3 offset:49168
-; GCN-NEXT: ds_read_b128 a[96:99], v3 offset:49152
-; GCN-NEXT: ds_read_b128 a[156:159], v4 offset:57456
-; GCN-NEXT: ds_read_b128 a[152:155], v4 offset:57440
-; GCN-NEXT: ds_read_b128 a[148:151], v4 offset:57424
-; GCN-NEXT: ds_read_b128 a[144:147], v4 offset:57408
-; GCN-NEXT: ds_read_b128 a[128:131], v4 offset:57344
-; GCN-NEXT: ds_read_b128 a[132:135], v4 offset:57360
-; GCN-NEXT: ds_read_b128 a[136:139], v4 offset:57376
-; GCN-NEXT: ds_read_b128 a[140:143], v4 offset:57392
-; GCN-NEXT: s_waitcnt lgkmcnt(14)
-; GCN-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v2, v1, a[32:63]
-; GCN-NEXT: v_add_u32_e32 v0, s1, v0
-; GCN-NEXT: ; sched_group_barrier mask(0x00000100) size(40) SyncID(0)
-; GCN-NEXT: s_waitcnt lgkmcnt(8)
-; GCN-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v2, v1, a[96:127]
-; GCN-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v2, v1, a[64:95]
-; GCN-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v1, a[0:31]
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v2, v1, a[128:159]
-; GCN-NEXT: s_nop 11
-; GCN-NEXT: ds_write_b128 v0, a[60:63] offset:112
-; GCN-NEXT: ds_write_b128 v0, a[56:59] offset:96
-; GCN-NEXT: ds_write_b128 v0, a[52:55] offset:80
-; GCN-NEXT: ds_write_b128 v0, a[48:51] offset:64
-; GCN-NEXT: ds_write_b128 v0, a[44:47] offset:48
-; GCN-NEXT: ds_write_b128 v0, a[40:43] offset:32
-; GCN-NEXT: ds_write_b128 v0, a[36:39] offset:16
-; GCN-NEXT: ds_write_b128 v0, a[32:35]
-; GCN-NEXT: v_mov_b32_e32 v0, s1
-; GCN-NEXT: ds_write_b128 v0, a[24:27] offset:8288
-; GCN-NEXT: ds_write_b128 v0, a[28:31] offset:8304
-; GCN-NEXT: ds_write_b128 v0, a[16:19] offset:8256
-; GCN-NEXT: ds_write_b128 v0, a[20:23] offset:8272
-; GCN-NEXT: ds_write_b128 v0, a[8:11] offset:8224
-; GCN-NEXT: ds_write_b128 v0, a[12:15] offset:8240
-; GCN-NEXT: ds_write_b128 v0, a[0:3] offset:8192
-; GCN-NEXT: ds_write_b128 v0, a[4:7] offset:8208
-; GCN-NEXT: ds_write_b128 v0, a[88:91] offset:16480
-; GCN-NEXT: ds_write_b128 v0, a[92:95] offset:16496
-; GCN-NEXT: ds_write_b128 v0, a[80:83] offset:16448
-; GCN-NEXT: ds_write_b128 v0, a[84:87] offset:16464
-; GCN-NEXT: ds_write_b128 v0, a[72:75] offset:16416
-; GCN-NEXT: ds_write_b128 v0, a[76:79] offset:16432
-; GCN-NEXT: ds_write_b128 v0, a[64:67] offset:16384
-; GCN-NEXT: ds_write_b128 v0, a[68:71] offset:16400
-; GCN-NEXT: ds_write_b128 v0, a[120:123] offset:24672
-; GCN-NEXT: ds_write_b128 v0, a[124:127] offset:24688
-; GCN-NEXT: ds_write_b128 v0, a[112:115] offset:24640
-; GCN-NEXT: ds_write_b128 v0, a[116:119] offset:24656
-; GCN-NEXT: ds_write_b128 v0, a[104:107] offset:24608
-; GCN-NEXT: ds_write_b128 v0, a[108:111] offset:24624
-; GCN-NEXT: ds_write_b128 v0, a[96:99] offset:24576
-; GCN-NEXT: ds_write_b128 v0, a[100:103] offset:24592
-; GCN-NEXT: ds_write_b128 v0, a[152:155] offset:32864
-; GCN-NEXT: ds_write_b128 v0, a[156:159] offset:32880
-; GCN-NEXT: ds_write_b128 v0, a[144:147] offset:32832
-; GCN-NEXT: ds_write_b128 v0, a[148:151] offset:32848
-; GCN-NEXT: ds_write_b128 v0, a[136:139] offset:32800
-; GCN-NEXT: ds_write_b128 v0, a[140:143] offset:32816
-; GCN-NEXT: ds_write_b128 v0, a[128:131] offset:32768
-; GCN-NEXT: ds_write_b128 v0, a[132:135] offset:32784
-; GCN-NEXT: ; sched_group_barrier mask(0x00000008) size(5) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000200) size(40) SyncID(0)
-; GCN-NEXT: s_endpgm
+; SDAG-LABEL: test_sched_group_barrier_pipeline_MFMA_cluster:
+; SDAG: ; %bb.0: ; %entry
+; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-NEXT: v_lshlrev_b32_e32 v0, 7, v0
+; SDAG-NEXT: v_and_b32_e32 v0, 0x1ff80, v0
+; SDAG-NEXT: v_mov_b32_e32 v2, 1.0
+; SDAG-NEXT: v_mov_b32_e32 v1, 2.0
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_add_u32_e32 v3, s0, v0
+; SDAG-NEXT: ds_read_b128 a[60:63], v3 offset:112
+; SDAG-NEXT: ds_read_b128 a[56:59], v3 offset:96
+; SDAG-NEXT: ds_read_b128 a[52:55], v3 offset:80
+; SDAG-NEXT: ds_read_b128 a[48:51], v3 offset:64
+; SDAG-NEXT: ds_read_b128 a[32:35], v3
+; SDAG-NEXT: ds_read_b128 a[36:39], v3 offset:16
+; SDAG-NEXT: ds_read_b128 a[40:43], v3 offset:32
+; SDAG-NEXT: ds_read_b128 a[44:47], v3 offset:48
+; SDAG-NEXT: v_add_u32_e32 v4, 0x6000, v3
+; SDAG-NEXT: ds_read_b128 a[28:31], v3 offset:8304
+; SDAG-NEXT: ds_read_b128 a[24:27], v3 offset:8288
+; SDAG-NEXT: ds_read_b128 a[20:23], v3 offset:8272
+; SDAG-NEXT: ds_read_b128 a[16:19], v3 offset:8256
+; SDAG-NEXT: ds_read_b128 a[12:15], v3 offset:8240
+; SDAG-NEXT: ds_read_b128 a[8:11], v3 offset:8224
+; SDAG-NEXT: ds_read_b128 a[4:7], v3 offset:8208
+; SDAG-NEXT: ds_read_b128 a[0:3], v3 offset:8192
+; SDAG-NEXT: ds_read_b128 a[92:95], v3 offset:24688
+; SDAG-NEXT: ds_read_b128 a[88:91], v3 offset:24672
+; SDAG-NEXT: ds_read_b128 a[84:87], v3 offset:24656
+; SDAG-NEXT: ds_read_b128 a[80:83], v3 offset:24640
+; SDAG-NEXT: ds_read_b128 a[76:79], v3 offset:24624
+; SDAG-NEXT: ds_read_b128 a[72:75], v3 offset:24608
+; SDAG-NEXT: ds_read_b128 a[68:71], v3 offset:24592
+; SDAG-NEXT: ds_read_b128 a[64:67], v3 offset:24576
+; SDAG-NEXT: ds_read_b128 a[124:127], v3 offset:49264
+; SDAG-NEXT: ds_read_b128 a[120:123], v3 offset:49248
+; SDAG-NEXT: ds_read_b128 a[116:119], v3 offset:49232
+; SDAG-NEXT: ds_read_b128 a[112:115], v3 offset:49216
+; SDAG-NEXT: ds_read_b128 a[108:111], v3 offset:49200
+; SDAG-NEXT: ds_read_b128 a[104:107], v3 offset:49184
+; SDAG-NEXT: ds_read_b128 a[100:103], v3 offset:49168
+; SDAG-NEXT: ds_read_b128 a[96:99], v3 offset:49152
+; SDAG-NEXT: ds_read_b128 a[156:159], v4 offset:57456
+; SDAG-NEXT: ds_read_b128 a[152:155], v4 offset:57440
+; SDAG-NEXT: ds_read_b128 a[148:151], v4 offset:57424
+; SDAG-NEXT: ds_read_b128 a[144:147], v4 offset:57408
+; SDAG-NEXT: ds_read_b128 a[128:131], v4 offset:57344
+; SDAG-NEXT: ds_read_b128 a[132:135], v4 offset:57360
+; SDAG-NEXT: ds_read_b128 a[136:139], v4 offset:57376
+; SDAG-NEXT: ds_read_b128 a[140:143], v4 offset:57392
+; SDAG-NEXT: s_waitcnt lgkmcnt(14)
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v2, v1, a[32:63]
+; SDAG-NEXT: v_add_u32_e32 v0, s1, v0
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000100) size(40) SyncID(0)
+; SDAG-NEXT: s_waitcnt lgkmcnt(8)
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v2, v1, a[96:127]
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v2, v1, a[64:95]
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v1, a[0:31]
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v2, v1, a[128:159]
+; SDAG-NEXT: s_nop 11
+; SDAG-NEXT: ds_write_b128 v0, a[60:63] offset:112
+; SDAG-NEXT: ds_write_b128 v0, a[56:59] offset:96
+; SDAG-NEXT: ds_write_b128 v0, a[52:55] offset:80
+; SDAG-NEXT: ds_write_b128 v0, a[48:51] offset:64
+; SDAG-NEXT: ds_write_b128 v0, a[44:47] offset:48
+; SDAG-NEXT: ds_write_b128 v0, a[40:43] offset:32
+; SDAG-NEXT: ds_write_b128 v0, a[36:39] offset:16
+; SDAG-NEXT: ds_write_b128 v0, a[32:35]
+; SDAG-NEXT: v_mov_b32_e32 v0, s1
+; SDAG-NEXT: ds_write_b128 v0, a[24:27] offset:8288
+; SDAG-NEXT: ds_write_b128 v0, a[28:31] offset:8304
+; SDAG-NEXT: ds_write_b128 v0, a[16:19] offset:8256
+; SDAG-NEXT: ds_write_b128 v0, a[20:23] offset:8272
+; SDAG-NEXT: ds_write_b128 v0, a[8:11] offset:8224
+; SDAG-NEXT: ds_write_b128 v0, a[12:15] offset:8240
+; SDAG-NEXT: ds_write_b128 v0, a[0:3] offset:8192
+; SDAG-NEXT: ds_write_b128 v0, a[4:7] offset:8208
+; SDAG-NEXT: ds_write_b128 v0, a[88:91] offset:16480
+; SDAG-NEXT: ds_write_b128 v0, a[92:95] offset:16496
+; SDAG-NEXT: ds_write_b128 v0, a[80:83] offset:16448
+; SDAG-NEXT: ds_write_b128 v0, a[84:87] offset:16464
+; SDAG-NEXT: ds_write_b128 v0, a[72:75] offset:16416
+; SDAG-NEXT: ds_write_b128 v0, a[76:79] offset:16432
+; SDAG-NEXT: ds_write_b128 v0, a[64:67] offset:16384
+; SDAG-NEXT: ds_write_b128 v0, a[68:71] offset:16400
+; SDAG-NEXT: ds_write_b128 v0, a[120:123] offset:24672
+; SDAG-NEXT: ds_write_b128 v0, a[124:127] offset:24688
+; SDAG-NEXT: ds_write_b128 v0, a[112:115] offset:24640
+; SDAG-NEXT: ds_write_b128 v0, a[116:119] offset:24656
+; SDAG-NEXT: ds_write_b128 v0, a[104:107] offset:24608
+; SDAG-NEXT: ds_write_b128 v0, a[108:111] offset:24624
+; SDAG-NEXT: ds_write_b128 v0, a[96:99] offset:24576
+; SDAG-NEXT: ds_write_b128 v0, a[100:103] offset:24592
+; SDAG-NEXT: ds_write_b128 v0, a[152:155] offset:32864
+; SDAG-NEXT: ds_write_b128 v0, a[156:159] offset:32880
+; SDAG-NEXT: ds_write_b128 v0, a[144:147] offset:32832
+; SDAG-NEXT: ds_write_b128 v0, a[148:151] offset:32848
+; SDAG-NEXT: ds_write_b128 v0, a[136:139] offset:32800
+; SDAG-NEXT: ds_write_b128 v0, a[140:143] offset:32816
+; SDAG-NEXT: ds_write_b128 v0, a[128:131] offset:32768
+; SDAG-NEXT: ds_write_b128 v0, a[132:135] offset:32784
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000008) size(5) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000200) size(40) SyncID(0)
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: test_sched_group_barrier_pipeline_MFMA_cluster:
+; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GISEL-NEXT: v_lshlrev_b32_e32 v32, 7, v0
+; GISEL-NEXT: v_mov_b32_e32 v33, 1.0
+; GISEL-NEXT: v_mov_b32_e32 v34, 2.0
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: v_add_u32_e32 v0, s0, v32
+; GISEL-NEXT: ds_read_b128 a[96:99], v0
+; GISEL-NEXT: ds_read_b128 a[100:103], v0 offset:16
+; GISEL-NEXT: ds_read_b128 a[104:107], v0 offset:32
+; GISEL-NEXT: ds_read_b128 a[108:111], v0 offset:48
+; GISEL-NEXT: ds_read_b128 a[112:115], v0 offset:64
+; GISEL-NEXT: ds_read_b128 a[116:119], v0 offset:80
+; GISEL-NEXT: ds_read_b128 a[120:123], v0 offset:96
+; GISEL-NEXT: ds_read_b128 a[124:127], v0 offset:112
+; GISEL-NEXT: ds_read_b128 a[0:3], v0 offset:8192
+; GISEL-NEXT: ds_read_b128 a[4:7], v0 offset:8208
+; GISEL-NEXT: ds_read_b128 a[8:11], v0 offset:8224
+; GISEL-NEXT: ds_read_b128 a[12:15], v0 offset:8240
+; GISEL-NEXT: ds_read_b128 a[16:19], v0 offset:8256
+; GISEL-NEXT: ds_read_b128 a[20:23], v0 offset:8272
+; GISEL-NEXT: ds_read_b128 a[24:27], v0 offset:8288
+; GISEL-NEXT: ds_read_b128 a[28:31], v0 offset:8304
+; GISEL-NEXT: ds_read_b128 a[32:35], v0 offset:24576
+; GISEL-NEXT: ds_read_b128 a[36:39], v0 offset:24592
+; GISEL-NEXT: ds_read_b128 a[40:43], v0 offset:24608
+; GISEL-NEXT: ds_read_b128 a[44:47], v0 offset:24624
+; GISEL-NEXT: ds_read_b128 a[48:51], v0 offset:24640
+; GISEL-NEXT: ds_read_b128 a[52:55], v0 offset:24656
+; GISEL-NEXT: ds_read_b128 a[56:59], v0 offset:24672
+; GISEL-NEXT: ds_read_b128 a[60:63], v0 offset:24688
+; GISEL-NEXT: v_add_u32_e32 v1, 0xc000, v0
+; GISEL-NEXT: ds_read_b128 a[64:67], v0 offset:49152
+; GISEL-NEXT: ds_read_b128 a[68:71], v0 offset:49168
+; GISEL-NEXT: ds_read_b128 a[72:75], v0 offset:49184
+; GISEL-NEXT: ds_read_b128 a[76:79], v0 offset:49200
+; GISEL-NEXT: ds_read_b128 a[80:83], v0 offset:49216
+; GISEL-NEXT: ds_read_b128 a[84:87], v0 offset:49232
+; GISEL-NEXT: ds_read_b128 a[88:91], v0 offset:49248
+; GISEL-NEXT: ds_read_b128 a[92:95], v0 offset:49264
+; GISEL-NEXT: ds_read_b128 a[128:131], v1 offset:32768
+; GISEL-NEXT: ds_read_b128 a[132:135], v1 offset:32784
+; GISEL-NEXT: ds_read_b128 a[136:139], v1 offset:32800
+; GISEL-NEXT: ds_read_b128 a[140:143], v1 offset:32816
+; GISEL-NEXT: ds_read_b128 a[144:147], v1 offset:32832
+; GISEL-NEXT: ds_read_b128 a[148:151], v1 offset:32848
+; GISEL-NEXT: ds_read_b128 a[152:155], v1 offset:32864
+; GISEL-NEXT: ds_read_b128 a[156:159], v1 offset:32880
+; GISEL-NEXT: s_waitcnt lgkmcnt(14)
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v33, v34, a[96:127]
+; GISEL-NEXT: v_add_u32_e32 v32, s1, v32
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000100) size(40) SyncID(0)
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v33, v34, a[0:31]
+; GISEL-NEXT: s_nop 15
+; GISEL-NEXT: s_nop 0
+; GISEL-NEXT: v_accvgpr_read_b32 v0, a96
+; GISEL-NEXT: v_accvgpr_read_b32 v1, a97
+; GISEL-NEXT: v_accvgpr_read_b32 v2, a98
+; GISEL-NEXT: v_accvgpr_read_b32 v3, a99
+; GISEL-NEXT: v_accvgpr_read_b32 v4, a100
+; GISEL-NEXT: v_accvgpr_read_b32 v5, a101
+; GISEL-NEXT: v_accvgpr_read_b32 v6, a102
+; GISEL-NEXT: v_accvgpr_read_b32 v7, a103
+; GISEL-NEXT: v_accvgpr_read_b32 v8, a104
+; GISEL-NEXT: v_accvgpr_read_b32 v9, a105
+; GISEL-NEXT: v_accvgpr_read_b32 v10, a106
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v33, v34, a[32:63]
+; GISEL-NEXT: v_accvgpr_read_b32 v11, a107
+; GISEL-NEXT: v_accvgpr_read_b32 v12, a108
+; GISEL-NEXT: v_accvgpr_read_b32 v13, a109
+; GISEL-NEXT: v_accvgpr_read_b32 v14, a110
+; GISEL-NEXT: v_accvgpr_read_b32 v15, a111
+; GISEL-NEXT: v_accvgpr_read_b32 v16, a112
+; GISEL-NEXT: v_accvgpr_read_b32 v17, a113
+; GISEL-NEXT: v_accvgpr_read_b32 v18, a114
+; GISEL-NEXT: v_accvgpr_read_b32 v19, a115
+; GISEL-NEXT: v_accvgpr_read_b32 v20, a116
+; GISEL-NEXT: v_accvgpr_read_b32 v21, a117
+; GISEL-NEXT: v_accvgpr_read_b32 v22, a118
+; GISEL-NEXT: v_accvgpr_read_b32 v23, a119
+; GISEL-NEXT: v_accvgpr_read_b32 v24, a120
+; GISEL-NEXT: v_accvgpr_read_b32 v25, a121
+; GISEL-NEXT: s_waitcnt lgkmcnt(8)
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v33, v34, a[64:95]
+; GISEL-NEXT: v_accvgpr_read_b32 v26, a122
+; GISEL-NEXT: v_accvgpr_read_b32 v27, a123
+; GISEL-NEXT: v_accvgpr_read_b32 v28, a124
+; GISEL-NEXT: v_accvgpr_read_b32 v29, a125
+; GISEL-NEXT: v_accvgpr_read_b32 v30, a126
+; GISEL-NEXT: v_accvgpr_read_b32 v31, a127
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v33, v34, a[128:159]
+; GISEL-NEXT: ds_write_b128 v32, v[0:3]
+; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:16
+; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:32
+; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:48
+; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:64
+; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:80
+; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:96
+; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:112
+; GISEL-NEXT: v_accvgpr_read_b32 v0, a0
+; GISEL-NEXT: v_accvgpr_read_b32 v1, a1
+; GISEL-NEXT: v_accvgpr_read_b32 v2, a2
+; GISEL-NEXT: v_accvgpr_read_b32 v3, a3
+; GISEL-NEXT: v_accvgpr_read_b32 v4, a4
+; GISEL-NEXT: v_accvgpr_read_b32 v5, a5
+; GISEL-NEXT: v_accvgpr_read_b32 v6, a6
+; GISEL-NEXT: v_accvgpr_read_b32 v7, a7
+; GISEL-NEXT: v_accvgpr_read_b32 v8, a8
+; GISEL-NEXT: v_accvgpr_read_b32 v9, a9
+; GISEL-NEXT: v_accvgpr_read_b32 v10, a10
+; GISEL-NEXT: v_accvgpr_read_b32 v11, a11
+; GISEL-NEXT: v_accvgpr_read_b32 v12, a12
+; GISEL-NEXT: v_accvgpr_read_b32 v13, a13
+; GISEL-NEXT: v_accvgpr_read_b32 v14, a14
+; GISEL-NEXT: v_accvgpr_read_b32 v15, a15
+; GISEL-NEXT: v_accvgpr_read_b32 v16, a16
+; GISEL-NEXT: v_accvgpr_read_b32 v17, a17
+; GISEL-NEXT: v_accvgpr_read_b32 v18, a18
+; GISEL-NEXT: v_accvgpr_read_b32 v19, a19
+; GISEL-NEXT: v_accvgpr_read_b32 v20, a20
+; GISEL-NEXT: v_accvgpr_read_b32 v21, a21
+; GISEL-NEXT: v_accvgpr_read_b32 v22, a22
+; GISEL-NEXT: v_accvgpr_read_b32 v23, a23
+; GISEL-NEXT: v_accvgpr_read_b32 v24, a24
+; GISEL-NEXT: v_accvgpr_read_b32 v25, a25
+; GISEL-NEXT: v_accvgpr_read_b32 v26, a26
+; GISEL-NEXT: v_accvgpr_read_b32 v27, a27
+; GISEL-NEXT: v_accvgpr_read_b32 v28, a28
+; GISEL-NEXT: v_accvgpr_read_b32 v29, a29
+; GISEL-NEXT: v_accvgpr_read_b32 v30, a30
+; GISEL-NEXT: v_accvgpr_read_b32 v31, a31
+; GISEL-NEXT: v_mov_b32_e32 v32, s1
+; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:8192
+; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:8208
+; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:8224
+; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:8240
+; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:8256
+; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:8272
+; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:8288
+; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:8304
+; GISEL-NEXT: v_accvgpr_read_b32 v0, a32
+; GISEL-NEXT: v_accvgpr_read_b32 v1, a33
+; GISEL-NEXT: v_accvgpr_read_b32 v2, a34
+; GISEL-NEXT: v_accvgpr_read_b32 v3, a35
+; GISEL-NEXT: v_accvgpr_read_b32 v4, a36
+; GISEL-NEXT: v_accvgpr_read_b32 v5, a37
+; GISEL-NEXT: v_accvgpr_read_b32 v6, a38
+; GISEL-NEXT: v_accvgpr_read_b32 v7, a39
+; GISEL-NEXT: v_accvgpr_read_b32 v8, a40
+; GISEL-NEXT: v_accvgpr_read_b32 v9, a41
+; GISEL-NEXT: v_accvgpr_read_b32 v10, a42
+; GISEL-NEXT: v_accvgpr_read_b32 v11, a43
+; GISEL-NEXT: v_accvgpr_read_b32 v12, a44
+; GISEL-NEXT: v_accvgpr_read_b32 v13, a45
+; GISEL-NEXT: v_accvgpr_read_b32 v14, a46
+; GISEL-NEXT: v_accvgpr_read_b32 v15, a47
+; GISEL-NEXT: v_accvgpr_read_b32 v16, a48
+; GISEL-NEXT: v_accvgpr_read_b32 v17, a49
+; GISEL-NEXT: v_accvgpr_read_b32 v18, a50
+; GISEL-NEXT: v_accvgpr_read_b32 v19, a51
+; GISEL-NEXT: v_accvgpr_read_b32 v20, a52
+; GISEL-NEXT: v_accvgpr_read_b32 v21, a53
+; GISEL-NEXT: v_accvgpr_read_b32 v22, a54
+; GISEL-NEXT: v_accvgpr_read_b32 v23, a55
+; GISEL-NEXT: v_accvgpr_read_b32 v24, a56
+; GISEL-NEXT: v_accvgpr_read_b32 v25, a57
+; GISEL-NEXT: v_accvgpr_read_b32 v26, a58
+; GISEL-NEXT: v_accvgpr_read_b32 v27, a59
+; GISEL-NEXT: v_accvgpr_read_b32 v28, a60
+; GISEL-NEXT: v_accvgpr_read_b32 v29, a61
+; GISEL-NEXT: v_accvgpr_read_b32 v30, a62
+; GISEL-NEXT: v_accvgpr_read_b32 v31, a63
+; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:16384
+; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:16400
+; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:16416
+; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:16432
+; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:16448
+; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:16464
+; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:16480
+; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:16496
+; GISEL-NEXT: v_accvgpr_read_b32 v0, a64
+; GISEL-NEXT: v_accvgpr_read_b32 v1, a65
+; GISEL-NEXT: v_accvgpr_read_b32 v2, a66
+; GISEL-NEXT: v_accvgpr_read_b32 v3, a67
+; GISEL-NEXT: v_accvgpr_read_b32 v4, a68
+; GISEL-NEXT: v_accvgpr_read_b32 v5, a69
+; GISEL-NEXT: v_accvgpr_read_b32 v6, a70
+; GISEL-NEXT: v_accvgpr_read_b32 v7, a71
+; GISEL-NEXT: v_accvgpr_read_b32 v8, a72
+; GISEL-NEXT: v_accvgpr_read_b32 v9, a73
+; GISEL-NEXT: v_accvgpr_read_b32 v10, a74
+; GISEL-NEXT: v_accvgpr_read_b32 v11, a75
+; GISEL-NEXT: v_accvgpr_read_b32 v12, a76
+; GISEL-NEXT: v_accvgpr_read_b32 v13, a77
+; GISEL-NEXT: v_accvgpr_read_b32 v14, a78
+; GISEL-NEXT: v_accvgpr_read_b32 v15, a79
+; GISEL-NEXT: v_accvgpr_read_b32 v16, a80
+; GISEL-NEXT: v_accvgpr_read_b32 v17, a81
+; GISEL-NEXT: v_accvgpr_read_b32 v18, a82
+; GISEL-NEXT: v_accvgpr_read_b32 v19, a83
+; GISEL-NEXT: v_accvgpr_read_b32 v20, a84
+; GISEL-NEXT: v_accvgpr_read_b32 v21, a85
+; GISEL-NEXT: v_accvgpr_read_b32 v22, a86
+; GISEL-NEXT: v_accvgpr_read_b32 v23, a87
+; GISEL-NEXT: v_accvgpr_read_b32 v24, a88
+; GISEL-NEXT: v_accvgpr_read_b32 v25, a89
+; GISEL-NEXT: v_accvgpr_read_b32 v26, a90
+; GISEL-NEXT: v_accvgpr_read_b32 v27, a91
+; GISEL-NEXT: v_accvgpr_read_b32 v28, a92
+; GISEL-NEXT: v_accvgpr_read_b32 v29, a93
+; GISEL-NEXT: v_accvgpr_read_b32 v30, a94
+; GISEL-NEXT: v_accvgpr_read_b32 v31, a95
+; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:24576
+; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:24592
+; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:24608
+; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:24624
+; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:24640
+; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:24656
+; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:24672
+; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:24688
+; GISEL-NEXT: v_accvgpr_read_b32 v0, a128
+; GISEL-NEXT: v_accvgpr_read_b32 v1, a129
+; GISEL-NEXT: v_accvgpr_read_b32 v2, a130
+; GISEL-NEXT: v_accvgpr_read_b32 v3, a131
+; GISEL-NEXT: v_accvgpr_read_b32 v4, a132
+; GISEL-NEXT: v_accvgpr_read_b32 v5, a133
+; GISEL-NEXT: v_accvgpr_read_b32 v6, a134
+; GISEL-NEXT: v_accvgpr_read_b32 v7, a135
+; GISEL-NEXT: v_accvgpr_read_b32 v8, a136
+; GISEL-NEXT: v_accvgpr_read_b32 v9, a137
+; GISEL-NEXT: v_accvgpr_read_b32 v10, a138
+; GISEL-NEXT: v_accvgpr_read_b32 v11, a139
+; GISEL-NEXT: v_accvgpr_read_b32 v12, a140
+; GISEL-NEXT: v_accvgpr_read_b32 v13, a141
+; GISEL-NEXT: v_accvgpr_read_b32 v14, a142
+; GISEL-NEXT: v_accvgpr_read_b32 v15, a143
+; GISEL-NEXT: v_accvgpr_read_b32 v16, a144
+; GISEL-NEXT: v_accvgpr_read_b32 v17, a145
+; GISEL-NEXT: v_accvgpr_read_b32 v18, a146
+; GISEL-NEXT: v_accvgpr_read_b32 v19, a147
+; GISEL-NEXT: v_accvgpr_read_b32 v20, a148
+; GISEL-NEXT: v_accvgpr_read_b32 v21, a149
+; GISEL-NEXT: v_accvgpr_read_b32 v22, a150
+; GISEL-NEXT: v_accvgpr_read_b32 v23, a151
+; GISEL-NEXT: v_accvgpr_read_b32 v24, a152
+; GISEL-NEXT: v_accvgpr_read_b32 v25, a153
+; GISEL-NEXT: v_accvgpr_read_b32 v26, a154
+; GISEL-NEXT: v_accvgpr_read_b32 v27, a155
+; GISEL-NEXT: v_accvgpr_read_b32 v28, a156
+; GISEL-NEXT: v_accvgpr_read_b32 v29, a157
+; GISEL-NEXT: v_accvgpr_read_b32 v30, a158
+; GISEL-NEXT: v_accvgpr_read_b32 v31, a159
+; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:32768
+; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:32784
+; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:32800
+; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:32816
+; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:32832
+; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:32848
+; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:32864
+; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:32880
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000008) size(5) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000200) size(40) SyncID(0)
+; GISEL-NEXT: s_endpgm
;
; EXACTCUTOFF-LABEL: test_sched_group_barrier_pipeline_MFMA_cluster:
; EXACTCUTOFF: ; %bb.0: ; %entry
@@ -866,134 +1371,423 @@ entry:
}
define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_interleave(ptr addrspace(3) noalias %in, ptr addrspace(3) noalias %out) #0 {
-; GCN-LABEL: test_sched_group_barrier_pipeline_MFMA_interleave:
-; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: v_lshlrev_b32_e32 v0, 7, v0
-; GCN-NEXT: v_and_b32_e32 v0, 0x1ff80, v0
-; GCN-NEXT: v_mov_b32_e32 v2, 1.0
-; GCN-NEXT: v_mov_b32_e32 v3, 2.0
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_add_u32_e32 v1, s0, v0
-; GCN-NEXT: ds_read_b128 a[28:31], v1 offset:112
-; GCN-NEXT: ds_read_b128 a[24:27], v1 offset:96
-; GCN-NEXT: ds_read_b128 a[20:23], v1 offset:80
-; GCN-NEXT: ds_read_b128 a[16:19], v1 offset:64
-; GCN-NEXT: ds_read_b128 a[0:3], v1
-; GCN-NEXT: ds_read_b128 a[4:7], v1 offset:16
-; GCN-NEXT: ds_read_b128 a[8:11], v1 offset:32
-; GCN-NEXT: ds_read_b128 a[12:15], v1 offset:48
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
-; GCN-NEXT: v_add_u32_e32 v0, s1, v0
-; GCN-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
-; GCN-NEXT: s_nop 15
-; GCN-NEXT: s_nop 1
-; GCN-NEXT: ds_write_b128 v0, a[28:31] offset:112
-; GCN-NEXT: ds_write_b128 v0, a[24:27] offset:96
-; GCN-NEXT: ds_write_b128 v0, a[20:23] offset:80
-; GCN-NEXT: ds_write_b128 v0, a[16:19] offset:64
-; GCN-NEXT: ds_write_b128 v0, a[12:15] offset:48
-; GCN-NEXT: ds_write_b128 v0, a[8:11] offset:32
-; GCN-NEXT: ds_write_b128 v0, a[4:7] offset:16
-; GCN-NEXT: ds_write_b128 v0, a[0:3]
-; GCN-NEXT: ds_read_b128 a[28:31], v1 offset:8304
-; GCN-NEXT: ds_read_b128 a[24:27], v1 offset:8288
-; GCN-NEXT: ds_read_b128 a[20:23], v1 offset:8272
-; GCN-NEXT: ds_read_b128 a[16:19], v1 offset:8256
-; GCN-NEXT: ds_read_b128 a[12:15], v1 offset:8240
-; GCN-NEXT: ds_read_b128 a[8:11], v1 offset:8224
-; GCN-NEXT: ds_read_b128 a[4:7], v1 offset:8208
-; GCN-NEXT: ds_read_b128 a[0:3], v1 offset:8192
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
-; GCN-NEXT: v_mov_b32_e32 v0, s1
-; GCN-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
-; GCN-NEXT: s_nop 15
-; GCN-NEXT: s_nop 1
-; GCN-NEXT: ds_write_b128 v0, a[24:27] offset:8288
-; GCN-NEXT: ds_write_b128 v0, a[28:31] offset:8304
-; GCN-NEXT: ds_write_b128 v0, a[16:19] offset:8256
-; GCN-NEXT: ds_write_b128 v0, a[20:23] offset:8272
-; GCN-NEXT: ds_write_b128 v0, a[8:11] offset:8224
-; GCN-NEXT: ds_write_b128 v0, a[12:15] offset:8240
-; GCN-NEXT: ds_write_b128 v0, a[0:3] offset:8192
-; GCN-NEXT: ds_write_b128 v0, a[4:7] offset:8208
-; GCN-NEXT: ds_read_b128 a[28:31], v1 offset:24688
-; GCN-NEXT: ds_read_b128 a[24:27], v1 offset:24672
-; GCN-NEXT: ds_read_b128 a[20:23], v1 offset:24656
-; GCN-NEXT: ds_read_b128 a[16:19], v1 offset:24640
-; GCN-NEXT: ds_read_b128 a[12:15], v1 offset:24624
-; GCN-NEXT: ds_read_b128 a[8:11], v1 offset:24608
-; GCN-NEXT: ds_read_b128 a[4:7], v1 offset:24592
-; GCN-NEXT: ds_read_b128 a[0:3], v1 offset:24576
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
-; GCN-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
-; GCN-NEXT: s_nop 15
-; GCN-NEXT: s_nop 2
-; GCN-NEXT: ds_write_b128 v0, a[24:27] offset:16480
-; GCN-NEXT: ds_write_b128 v0, a[28:31] offset:16496
-; GCN-NEXT: ds_write_b128 v0, a[16:19] offset:16448
-; GCN-NEXT: ds_write_b128 v0, a[20:23] offset:16464
-; GCN-NEXT: ds_write_b128 v0, a[8:11] offset:16416
-; GCN-NEXT: ds_write_b128 v0, a[12:15] offset:16432
-; GCN-NEXT: ds_write_b128 v0, a[0:3] offset:16384
-; GCN-NEXT: ds_write_b128 v0, a[4:7] offset:16400
-; GCN-NEXT: ds_read_b128 a[28:31], v1 offset:49264
-; GCN-NEXT: ds_read_b128 a[24:27], v1 offset:49248
-; GCN-NEXT: ds_read_b128 a[20:23], v1 offset:49232
-; GCN-NEXT: ds_read_b128 a[16:19], v1 offset:49216
-; GCN-NEXT: ds_read_b128 a[12:15], v1 offset:49200
-; GCN-NEXT: ds_read_b128 a[8:11], v1 offset:49184
-; GCN-NEXT: ds_read_b128 a[4:7], v1 offset:49168
-; GCN-NEXT: ds_read_b128 a[0:3], v1 offset:49152
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
-; GCN-NEXT: v_add_u32_e32 v1, 0x6000, v1
-; GCN-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
-; GCN-NEXT: s_nop 15
-; GCN-NEXT: s_nop 1
-; GCN-NEXT: ds_write_b128 v0, a[24:27] offset:24672
-; GCN-NEXT: ds_write_b128 v0, a[28:31] offset:24688
-; GCN-NEXT: ds_write_b128 v0, a[16:19] offset:24640
-; GCN-NEXT: ds_write_b128 v0, a[20:23] offset:24656
-; GCN-NEXT: ds_write_b128 v0, a[8:11] offset:24608
-; GCN-NEXT: ds_write_b128 v0, a[12:15] offset:24624
-; GCN-NEXT: ds_write_b128 v0, a[0:3] offset:24576
-; GCN-NEXT: ds_write_b128 v0, a[4:7] offset:24592
-; GCN-NEXT: ds_read_b128 a[28:31], v1 offset:57456
-; GCN-NEXT: ds_read_b128 a[24:27], v1 offset:57440
-; GCN-NEXT: ds_read_b128 a[20:23], v1 offset:57424
-; GCN-NEXT: ds_read_b128 a[16:19], v1 offset:57408
-; GCN-NEXT: ds_read_b128 a[0:3], v1 offset:57344
-; GCN-NEXT: ds_read_b128 a[4:7], v1 offset:57360
-; GCN-NEXT: ds_read_b128 a[8:11], v1 offset:57376
-; GCN-NEXT: ds_read_b128 a[12:15], v1 offset:57392
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
-; GCN-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
-; GCN-NEXT: s_nop 15
-; GCN-NEXT: s_nop 2
-; GCN-NEXT: ds_write_b128 v0, a[24:27] offset:32864
-; GCN-NEXT: ds_write_b128 v0, a[28:31] offset:32880
-; GCN-NEXT: ds_write_b128 v0, a[16:19] offset:32832
-; GCN-NEXT: ds_write_b128 v0, a[20:23] offset:32848
-; GCN-NEXT: ds_write_b128 v0, a[8:11] offset:32800
-; GCN-NEXT: ds_write_b128 v0, a[12:15] offset:32816
-; GCN-NEXT: ds_write_b128 v0, a[0:3] offset:32768
-; GCN-NEXT: ds_write_b128 v0, a[4:7] offset:32784
-; GCN-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
-; GCN-NEXT: s_endpgm
+; SDAG-LABEL: test_sched_group_barrier_pipeline_MFMA_interleave:
+; SDAG: ; %bb.0: ; %entry
+; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-NEXT: v_lshlrev_b32_e32 v0, 7, v0
+; SDAG-NEXT: v_and_b32_e32 v0, 0x1ff80, v0
+; SDAG-NEXT: v_mov_b32_e32 v2, 1.0
+; SDAG-NEXT: v_mov_b32_e32 v3, 2.0
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_add_u32_e32 v1, s0, v0
+; SDAG-NEXT: ds_read_b128 a[28:31], v1 offset:112
+; SDAG-NEXT: ds_read_b128 a[24:27], v1 offset:96
+; SDAG-NEXT: ds_read_b128 a[20:23], v1 offset:80
+; SDAG-NEXT: ds_read_b128 a[16:19], v1 offset:64
+; SDAG-NEXT: ds_read_b128 a[0:3], v1
+; SDAG-NEXT: ds_read_b128 a[4:7], v1 offset:16
+; SDAG-NEXT: ds_read_b128 a[8:11], v1 offset:32
+; SDAG-NEXT: ds_read_b128 a[12:15], v1 offset:48
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
+; SDAG-NEXT: v_add_u32_e32 v0, s1, v0
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
+; SDAG-NEXT: s_nop 15
+; SDAG-NEXT: s_nop 1
+; SDAG-NEXT: ds_write_b128 v0, a[28:31] offset:112
+; SDAG-NEXT: ds_write_b128 v0, a[24:27] offset:96
+; SDAG-NEXT: ds_write_b128 v0, a[20:23] offset:80
+; SDAG-NEXT: ds_write_b128 v0, a[16:19] offset:64
+; SDAG-NEXT: ds_write_b128 v0, a[12:15] offset:48
+; SDAG-NEXT: ds_write_b128 v0, a[8:11] offset:32
+; SDAG-NEXT: ds_write_b128 v0, a[4:7] offset:16
+; SDAG-NEXT: ds_write_b128 v0, a[0:3]
+; SDAG-NEXT: ds_read_b128 a[28:31], v1 offset:8304
+; SDAG-NEXT: ds_read_b128 a[24:27], v1 offset:8288
+; SDAG-NEXT: ds_read_b128 a[20:23], v1 offset:8272
+; SDAG-NEXT: ds_read_b128 a[16:19], v1 offset:8256
+; SDAG-NEXT: ds_read_b128 a[12:15], v1 offset:8240
+; SDAG-NEXT: ds_read_b128 a[8:11], v1 offset:8224
+; SDAG-NEXT: ds_read_b128 a[4:7], v1 offset:8208
+; SDAG-NEXT: ds_read_b128 a[0:3], v1 offset:8192
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
+; SDAG-NEXT: v_mov_b32_e32 v0, s1
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
+; SDAG-NEXT: s_nop 15
+; SDAG-NEXT: s_nop 1
+; SDAG-NEXT: ds_write_b128 v0, a[24:27] offset:8288
+; SDAG-NEXT: ds_write_b128 v0, a[28:31] offset:8304
+; SDAG-NEXT: ds_write_b128 v0, a[16:19] offset:8256
+; SDAG-NEXT: ds_write_b128 v0, a[20:23] offset:8272
+; SDAG-NEXT: ds_write_b128 v0, a[8:11] offset:8224
+; SDAG-NEXT: ds_write_b128 v0, a[12:15] offset:8240
+; SDAG-NEXT: ds_write_b128 v0, a[0:3] offset:8192
+; SDAG-NEXT: ds_write_b128 v0, a[4:7] offset:8208
+; SDAG-NEXT: ds_read_b128 a[28:31], v1 offset:24688
+; SDAG-NEXT: ds_read_b128 a[24:27], v1 offset:24672
+; SDAG-NEXT: ds_read_b128 a[20:23], v1 offset:24656
+; SDAG-NEXT: ds_read_b128 a[16:19], v1 offset:24640
+; SDAG-NEXT: ds_read_b128 a[12:15], v1 offset:24624
+; SDAG-NEXT: ds_read_b128 a[8:11], v1 offset:24608
+; SDAG-NEXT: ds_read_b128 a[4:7], v1 offset:24592
+; SDAG-NEXT: ds_read_b128 a[0:3], v1 offset:24576
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
+; SDAG-NEXT: s_nop 15
+; SDAG-NEXT: s_nop 2
+; SDAG-NEXT: ds_write_b128 v0, a[24:27] offset:16480
+; SDAG-NEXT: ds_write_b128 v0, a[28:31] offset:16496
+; SDAG-NEXT: ds_write_b128 v0, a[16:19] offset:16448
+; SDAG-NEXT: ds_write_b128 v0, a[20:23] offset:16464
+; SDAG-NEXT: ds_write_b128 v0, a[8:11] offset:16416
+; SDAG-NEXT: ds_write_b128 v0, a[12:15] offset:16432
+; SDAG-NEXT: ds_write_b128 v0, a[0:3] offset:16384
+; SDAG-NEXT: ds_write_b128 v0, a[4:7] offset:16400
+; SDAG-NEXT: ds_read_b128 a[28:31], v1 offset:49264
+; SDAG-NEXT: ds_read_b128 a[24:27], v1 offset:49248
+; SDAG-NEXT: ds_read_b128 a[20:23], v1 offset:49232
+; SDAG-NEXT: ds_read_b128 a[16:19], v1 offset:49216
+; SDAG-NEXT: ds_read_b128 a[12:15], v1 offset:49200
+; SDAG-NEXT: ds_read_b128 a[8:11], v1 offset:49184
+; SDAG-NEXT: ds_read_b128 a[4:7], v1 offset:49168
+; SDAG-NEXT: ds_read_b128 a[0:3], v1 offset:49152
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
+; SDAG-NEXT: v_add_u32_e32 v1, 0x6000, v1
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
+; SDAG-NEXT: s_nop 15
+; SDAG-NEXT: s_nop 1
+; SDAG-NEXT: ds_write_b128 v0, a[24:27] offset:24672
+; SDAG-NEXT: ds_write_b128 v0, a[28:31] offset:24688
+; SDAG-NEXT: ds_write_b128 v0, a[16:19] offset:24640
+; SDAG-NEXT: ds_write_b128 v0, a[20:23] offset:24656
+; SDAG-NEXT: ds_write_b128 v0, a[8:11] offset:24608
+; SDAG-NEXT: ds_write_b128 v0, a[12:15] offset:24624
+; SDAG-NEXT: ds_write_b128 v0, a[0:3] offset:24576
+; SDAG-NEXT: ds_write_b128 v0, a[4:7] offset:24592
+; SDAG-NEXT: ds_read_b128 a[28:31], v1 offset:57456
+; SDAG-NEXT: ds_read_b128 a[24:27], v1 offset:57440
+; SDAG-NEXT: ds_read_b128 a[20:23], v1 offset:57424
+; SDAG-NEXT: ds_read_b128 a[16:19], v1 offset:57408
+; SDAG-NEXT: ds_read_b128 a[0:3], v1 offset:57344
+; SDAG-NEXT: ds_read_b128 a[4:7], v1 offset:57360
+; SDAG-NEXT: ds_read_b128 a[8:11], v1 offset:57376
+; SDAG-NEXT: ds_read_b128 a[12:15], v1 offset:57392
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
+; SDAG-NEXT: s_nop 15
+; SDAG-NEXT: s_nop 2
+; SDAG-NEXT: ds_write_b128 v0, a[24:27] offset:32864
+; SDAG-NEXT: ds_write_b128 v0, a[28:31] offset:32880
+; SDAG-NEXT: ds_write_b128 v0, a[16:19] offset:32832
+; SDAG-NEXT: ds_write_b128 v0, a[20:23] offset:32848
+; SDAG-NEXT: ds_write_b128 v0, a[8:11] offset:32800
+; SDAG-NEXT: ds_write_b128 v0, a[12:15] offset:32816
+; SDAG-NEXT: ds_write_b128 v0, a[0:3] offset:32768
+; SDAG-NEXT: ds_write_b128 v0, a[4:7] offset:32784
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: test_sched_group_barrier_pipeline_MFMA_interleave:
+; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GISEL-NEXT: v_lshlrev_b32_e32 v0, 7, v0
+; GISEL-NEXT: v_mov_b32_e32 v33, 1.0
+; GISEL-NEXT: v_mov_b32_e32 v34, 2.0
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: v_add_u32_e32 v32, s0, v0
+; GISEL-NEXT: ds_read_b128 a[0:3], v32
+; GISEL-NEXT: ds_read_b128 a[4:7], v32 offset:16
+; GISEL-NEXT: ds_read_b128 a[8:11], v32 offset:32
+; GISEL-NEXT: ds_read_b128 a[12:15], v32 offset:48
+; GISEL-NEXT: ds_read_b128 a[16:19], v32 offset:64
+; GISEL-NEXT: ds_read_b128 a[20:23], v32 offset:80
+; GISEL-NEXT: ds_read_b128 a[24:27], v32 offset:96
+; GISEL-NEXT: ds_read_b128 a[28:31], v32 offset:112
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v33, v34, a[0:31]
+; GISEL-NEXT: v_add_u32_e32 v35, s1, v0
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
+; GISEL-NEXT: s_nop 15
+; GISEL-NEXT: s_nop 1
+; GISEL-NEXT: v_accvgpr_read_b32 v0, a0
+; GISEL-NEXT: v_accvgpr_read_b32 v1, a1
+; GISEL-NEXT: v_accvgpr_read_b32 v2, a2
+; GISEL-NEXT: v_accvgpr_read_b32 v3, a3
+; GISEL-NEXT: v_accvgpr_read_b32 v4, a4
+; GISEL-NEXT: v_accvgpr_read_b32 v5, a5
+; GISEL-NEXT: v_accvgpr_read_b32 v6, a6
+; GISEL-NEXT: v_accvgpr_read_b32 v7, a7
+; GISEL-NEXT: v_accvgpr_read_b32 v8, a8
+; GISEL-NEXT: v_accvgpr_read_b32 v9, a9
+; GISEL-NEXT: v_accvgpr_read_b32 v10, a10
+; GISEL-NEXT: v_accvgpr_read_b32 v11, a11
+; GISEL-NEXT: v_accvgpr_read_b32 v12, a12
+; GISEL-NEXT: v_accvgpr_read_b32 v13, a13
+; GISEL-NEXT: v_accvgpr_read_b32 v14, a14
+; GISEL-NEXT: v_accvgpr_read_b32 v15, a15
+; GISEL-NEXT: v_accvgpr_read_b32 v16, a16
+; GISEL-NEXT: v_accvgpr_read_b32 v17, a17
+; GISEL-NEXT: v_accvgpr_read_b32 v18, a18
+; GISEL-NEXT: v_accvgpr_read_b32 v19, a19
+; GISEL-NEXT: v_accvgpr_read_b32 v20, a20
+; GISEL-NEXT: v_accvgpr_read_b32 v21, a21
+; GISEL-NEXT: v_accvgpr_read_b32 v22, a22
+; GISEL-NEXT: v_accvgpr_read_b32 v23, a23
+; GISEL-NEXT: v_accvgpr_read_b32 v24, a24
+; GISEL-NEXT: v_accvgpr_read_b32 v25, a25
+; GISEL-NEXT: v_accvgpr_read_b32 v26, a26
+; GISEL-NEXT: v_accvgpr_read_b32 v27, a27
+; GISEL-NEXT: v_accvgpr_read_b32 v28, a28
+; GISEL-NEXT: v_accvgpr_read_b32 v29, a29
+; GISEL-NEXT: v_accvgpr_read_b32 v30, a30
+; GISEL-NEXT: v_accvgpr_read_b32 v31, a31
+; GISEL-NEXT: ds_write_b128 v35, v[0:3]
+; GISEL-NEXT: ds_write_b128 v35, v[4:7] offset:16
+; GISEL-NEXT: ds_write_b128 v35, v[8:11] offset:32
+; GISEL-NEXT: ds_write_b128 v35, v[12:15] offset:48
+; GISEL-NEXT: ds_write_b128 v35, v[16:19] offset:64
+; GISEL-NEXT: ds_write_b128 v35, v[20:23] offset:80
+; GISEL-NEXT: ds_write_b128 v35, v[24:27] offset:96
+; GISEL-NEXT: ds_write_b128 v35, v[28:31] offset:112
+; GISEL-NEXT: ds_read_b128 a[0:3], v32 offset:8192
+; GISEL-NEXT: ds_read_b128 a[4:7], v32 offset:8208
+; GISEL-NEXT: ds_read_b128 a[8:11], v32 offset:8224
+; GISEL-NEXT: ds_read_b128 a[12:15], v32 offset:8240
+; GISEL-NEXT: ds_read_b128 a[16:19], v32 offset:8256
+; GISEL-NEXT: ds_read_b128 a[20:23], v32 offset:8272
+; GISEL-NEXT: ds_read_b128 a[24:27], v32 offset:8288
+; GISEL-NEXT: ds_read_b128 a[28:31], v32 offset:8304
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v33, v34, a[0:31]
+; GISEL-NEXT: v_mov_b32_e32 v35, s1
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
+; GISEL-NEXT: s_nop 15
+; GISEL-NEXT: s_nop 1
+; GISEL-NEXT: v_accvgpr_read_b32 v0, a0
+; GISEL-NEXT: v_accvgpr_read_b32 v1, a1
+; GISEL-NEXT: v_accvgpr_read_b32 v2, a2
+; GISEL-NEXT: v_accvgpr_read_b32 v3, a3
+; GISEL-NEXT: v_accvgpr_read_b32 v4, a4
+; GISEL-NEXT: v_accvgpr_read_b32 v5, a5
+; GISEL-NEXT: v_accvgpr_read_b32 v6, a6
+; GISEL-NEXT: v_accvgpr_read_b32 v7, a7
+; GISEL-NEXT: v_accvgpr_read_b32 v8, a8
+; GISEL-NEXT: v_accvgpr_read_b32 v9, a9
+; GISEL-NEXT: v_accvgpr_read_b32 v10, a10
+; GISEL-NEXT: v_accvgpr_read_b32 v11, a11
+; GISEL-NEXT: v_accvgpr_read_b32 v12, a12
+; GISEL-NEXT: v_accvgpr_read_b32 v13, a13
+; GISEL-NEXT: v_accvgpr_read_b32 v14, a14
+; GISEL-NEXT: v_accvgpr_read_b32 v15, a15
+; GISEL-NEXT: v_accvgpr_read_b32 v16, a16
+; GISEL-NEXT: v_accvgpr_read_b32 v17, a17
+; GISEL-NEXT: v_accvgpr_read_b32 v18, a18
+; GISEL-NEXT: v_accvgpr_read_b32 v19, a19
+; GISEL-NEXT: v_accvgpr_read_b32 v20, a20
+; GISEL-NEXT: v_accvgpr_read_b32 v21, a21
+; GISEL-NEXT: v_accvgpr_read_b32 v22, a22
+; GISEL-NEXT: v_accvgpr_read_b32 v23, a23
+; GISEL-NEXT: v_accvgpr_read_b32 v24, a24
+; GISEL-NEXT: v_accvgpr_read_b32 v25, a25
+; GISEL-NEXT: v_accvgpr_read_b32 v26, a26
+; GISEL-NEXT: v_accvgpr_read_b32 v27, a27
+; GISEL-NEXT: v_accvgpr_read_b32 v28, a28
+; GISEL-NEXT: v_accvgpr_read_b32 v29, a29
+; GISEL-NEXT: v_accvgpr_read_b32 v30, a30
+; GISEL-NEXT: v_accvgpr_read_b32 v31, a31
+; GISEL-NEXT: ds_write_b128 v35, v[0:3] offset:8192
+; GISEL-NEXT: ds_write_b128 v35, v[4:7] offset:8208
+; GISEL-NEXT: ds_write_b128 v35, v[8:11] offset:8224
+; GISEL-NEXT: ds_write_b128 v35, v[12:15] offset:8240
+; GISEL-NEXT: ds_write_b128 v35, v[16:19] offset:8256
+; GISEL-NEXT: ds_write_b128 v35, v[20:23] offset:8272
+; GISEL-NEXT: ds_write_b128 v35, v[24:27] offset:8288
+; GISEL-NEXT: ds_write_b128 v35, v[28:31] offset:8304
+; GISEL-NEXT: ds_read_b128 a[0:3], v32 offset:24576
+; GISEL-NEXT: ds_read_b128 a[4:7], v32 offset:24592
+; GISEL-NEXT: ds_read_b128 a[8:11], v32 offset:24608
+; GISEL-NEXT: ds_read_b128 a[12:15], v32 offset:24624
+; GISEL-NEXT: ds_read_b128 a[16:19], v32 offset:24640
+; GISEL-NEXT: ds_read_b128 a[20:23], v32 offset:24656
+; GISEL-NEXT: ds_read_b128 a[24:27], v32 offset:24672
+; GISEL-NEXT: ds_read_b128 a[28:31], v32 offset:24688
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v33, v34, a[0:31]
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
+; GISEL-NEXT: s_nop 15
+; GISEL-NEXT: s_nop 2
+; GISEL-NEXT: v_accvgpr_read_b32 v0, a0
+; GISEL-NEXT: v_accvgpr_read_b32 v1, a1
+; GISEL-NEXT: v_accvgpr_read_b32 v2, a2
+; GISEL-NEXT: v_accvgpr_read_b32 v3, a3
+; GISEL-NEXT: v_accvgpr_read_b32 v4, a4
+; GISEL-NEXT: v_accvgpr_read_b32 v5, a5
+; GISEL-NEXT: v_accvgpr_read_b32 v6, a6
+; GISEL-NEXT: v_accvgpr_read_b32 v7, a7
+; GISEL-NEXT: v_accvgpr_read_b32 v8, a8
+; GISEL-NEXT: v_accvgpr_read_b32 v9, a9
+; GISEL-NEXT: v_accvgpr_read_b32 v10, a10
+; GISEL-NEXT: v_accvgpr_read_b32 v11, a11
+; GISEL-NEXT: v_accvgpr_read_b32 v12, a12
+; GISEL-NEXT: v_accvgpr_read_b32 v13, a13
+; GISEL-NEXT: v_accvgpr_read_b32 v14, a14
+; GISEL-NEXT: v_accvgpr_read_b32 v15, a15
+; GISEL-NEXT: v_accvgpr_read_b32 v16, a16
+; GISEL-NEXT: v_accvgpr_read_b32 v17, a17
+; GISEL-NEXT: v_accvgpr_read_b32 v18, a18
+; GISEL-NEXT: v_accvgpr_read_b32 v19, a19
+; GISEL-NEXT: v_accvgpr_read_b32 v20, a20
+; GISEL-NEXT: v_accvgpr_read_b32 v21, a21
+; GISEL-NEXT: v_accvgpr_read_b32 v22, a22
+; GISEL-NEXT: v_accvgpr_read_b32 v23, a23
+; GISEL-NEXT: v_accvgpr_read_b32 v24, a24
+; GISEL-NEXT: v_accvgpr_read_b32 v25, a25
+; GISEL-NEXT: v_accvgpr_read_b32 v26, a26
+; GISEL-NEXT: v_accvgpr_read_b32 v27, a27
+; GISEL-NEXT: v_accvgpr_read_b32 v28, a28
+; GISEL-NEXT: v_accvgpr_read_b32 v29, a29
+; GISEL-NEXT: v_accvgpr_read_b32 v30, a30
+; GISEL-NEXT: v_accvgpr_read_b32 v31, a31
+; GISEL-NEXT: ds_write_b128 v35, v[0:3] offset:16384
+; GISEL-NEXT: ds_write_b128 v35, v[4:7] offset:16400
+; GISEL-NEXT: ds_write_b128 v35, v[8:11] offset:16416
+; GISEL-NEXT: ds_write_b128 v35, v[12:15] offset:16432
+; GISEL-NEXT: ds_write_b128 v35, v[16:19] offset:16448
+; GISEL-NEXT: ds_write_b128 v35, v[20:23] offset:16464
+; GISEL-NEXT: ds_write_b128 v35, v[24:27] offset:16480
+; GISEL-NEXT: ds_write_b128 v35, v[28:31] offset:16496
+; GISEL-NEXT: ds_read_b128 a[0:3], v32 offset:49152
+; GISEL-NEXT: ds_read_b128 a[4:7], v32 offset:49168
+; GISEL-NEXT: ds_read_b128 a[8:11], v32 offset:49184
+; GISEL-NEXT: ds_read_b128 a[12:15], v32 offset:49200
+; GISEL-NEXT: ds_read_b128 a[16:19], v32 offset:49216
+; GISEL-NEXT: ds_read_b128 a[20:23], v32 offset:49232
+; GISEL-NEXT: ds_read_b128 a[24:27], v32 offset:49248
+; GISEL-NEXT: ds_read_b128 a[28:31], v32 offset:49264
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v33, v34, a[0:31]
+; GISEL-NEXT: v_add_u32_e32 v32, 0xc000, v32
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
+; GISEL-NEXT: s_nop 15
+; GISEL-NEXT: s_nop 1
+; GISEL-NEXT: v_accvgpr_read_b32 v0, a0
+; GISEL-NEXT: v_accvgpr_read_b32 v1, a1
+; GISEL-NEXT: v_accvgpr_read_b32 v2, a2
+; GISEL-NEXT: v_accvgpr_read_b32 v3, a3
+; GISEL-NEXT: v_accvgpr_read_b32 v4, a4
+; GISEL-NEXT: v_accvgpr_read_b32 v5, a5
+; GISEL-NEXT: v_accvgpr_read_b32 v6, a6
+; GISEL-NEXT: v_accvgpr_read_b32 v7, a7
+; GISEL-NEXT: v_accvgpr_read_b32 v8, a8
+; GISEL-NEXT: v_accvgpr_read_b32 v9, a9
+; GISEL-NEXT: v_accvgpr_read_b32 v10, a10
+; GISEL-NEXT: v_accvgpr_read_b32 v11, a11
+; GISEL-NEXT: v_accvgpr_read_b32 v12, a12
+; GISEL-NEXT: v_accvgpr_read_b32 v13, a13
+; GISEL-NEXT: v_accvgpr_read_b32 v14, a14
+; GISEL-NEXT: v_accvgpr_read_b32 v15, a15
+; GISEL-NEXT: v_accvgpr_read_b32 v16, a16
+; GISEL-NEXT: v_accvgpr_read_b32 v17, a17
+; GISEL-NEXT: v_accvgpr_read_b32 v18, a18
+; GISEL-NEXT: v_accvgpr_read_b32 v19, a19
+; GISEL-NEXT: v_accvgpr_read_b32 v20, a20
+; GISEL-NEXT: v_accvgpr_read_b32 v21, a21
+; GISEL-NEXT: v_accvgpr_read_b32 v22, a22
+; GISEL-NEXT: v_accvgpr_read_b32 v23, a23
+; GISEL-NEXT: v_accvgpr_read_b32 v24, a24
+; GISEL-NEXT: v_accvgpr_read_b32 v25, a25
+; GISEL-NEXT: v_accvgpr_read_b32 v26, a26
+; GISEL-NEXT: v_accvgpr_read_b32 v27, a27
+; GISEL-NEXT: v_accvgpr_read_b32 v28, a28
+; GISEL-NEXT: v_accvgpr_read_b32 v29, a29
+; GISEL-NEXT: v_accvgpr_read_b32 v30, a30
+; GISEL-NEXT: v_accvgpr_read_b32 v31, a31
+; GISEL-NEXT: ds_write_b128 v35, v[0:3] offset:24576
+; GISEL-NEXT: ds_write_b128 v35, v[4:7] offset:24592
+; GISEL-NEXT: ds_write_b128 v35, v[8:11] offset:24608
+; GISEL-NEXT: ds_write_b128 v35, v[12:15] offset:24624
+; GISEL-NEXT: ds_write_b128 v35, v[16:19] offset:24640
+; GISEL-NEXT: ds_write_b128 v35, v[20:23] offset:24656
+; GISEL-NEXT: ds_write_b128 v35, v[24:27] offset:24672
+; GISEL-NEXT: ds_write_b128 v35, v[28:31] offset:24688
+; GISEL-NEXT: ds_read_b128 a[0:3], v32 offset:32768
+; GISEL-NEXT: ds_read_b128 a[4:7], v32 offset:32784
+; GISEL-NEXT: ds_read_b128 a[8:11], v32 offset:32800
+; GISEL-NEXT: ds_read_b128 a[12:15], v32 offset:32816
+; GISEL-NEXT: ds_read_b128 a[16:19], v32 offset:32832
+; GISEL-NEXT: ds_read_b128 a[20:23], v32 offset:32848
+; GISEL-NEXT: ds_read_b128 a[24:27], v32 offset:32864
+; GISEL-NEXT: ds_read_b128 a[28:31], v32 offset:32880
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v33, v34, a[0:31]
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
+; GISEL-NEXT: s_nop 15
+; GISEL-NEXT: s_nop 2
+; GISEL-NEXT: v_accvgpr_read_b32 v0, a0
+; GISEL-NEXT: v_accvgpr_read_b32 v1, a1
+; GISEL-NEXT: v_accvgpr_read_b32 v2, a2
+; GISEL-NEXT: v_accvgpr_read_b32 v3, a3
+; GISEL-NEXT: v_accvgpr_read_b32 v4, a4
+; GISEL-NEXT: v_accvgpr_read_b32 v5, a5
+; GISEL-NEXT: v_accvgpr_read_b32 v6, a6
+; GISEL-NEXT: v_accvgpr_read_b32 v7, a7
+; GISEL-NEXT: v_accvgpr_read_b32 v8, a8
+; GISEL-NEXT: v_accvgpr_read_b32 v9, a9
+; GISEL-NEXT: v_accvgpr_read_b32 v10, a10
+; GISEL-NEXT: v_accvgpr_read_b32 v11, a11
+; GISEL-NEXT: v_accvgpr_read_b32 v12, a12
+; GISEL-NEXT: v_accvgpr_read_b32 v13, a13
+; GISEL-NEXT: v_accvgpr_read_b32 v14, a14
+; GISEL-NEXT: v_accvgpr_read_b32 v15, a15
+; GISEL-NEXT: v_accvgpr_read_b32 v16, a16
+; GISEL-NEXT: v_accvgpr_read_b32 v17, a17
+; GISEL-NEXT: v_accvgpr_read_b32 v18, a18
+; GISEL-NEXT: v_accvgpr_read_b32 v19, a19
+; GISEL-NEXT: v_accvgpr_read_b32 v20, a20
+; GISEL-NEXT: v_accvgpr_read_b32 v21, a21
+; GISEL-NEXT: v_accvgpr_read_b32 v22, a22
+; GISEL-NEXT: v_accvgpr_read_b32 v23, a23
+; GISEL-NEXT: v_accvgpr_read_b32 v24, a24
+; GISEL-NEXT: v_accvgpr_read_b32 v25, a25
+; GISEL-NEXT: v_accvgpr_read_b32 v26, a26
+; GISEL-NEXT: v_accvgpr_read_b32 v27, a27
+; GISEL-NEXT: v_accvgpr_read_b32 v28, a28
+; GISEL-NEXT: v_accvgpr_read_b32 v29, a29
+; GISEL-NEXT: v_accvgpr_read_b32 v30, a30
+; GISEL-NEXT: v_accvgpr_read_b32 v31, a31
+; GISEL-NEXT: ds_write_b128 v35, v[0:3] offset:32768
+; GISEL-NEXT: ds_write_b128 v35, v[4:7] offset:32784
+; GISEL-NEXT: ds_write_b128 v35, v[8:11] offset:32800
+; GISEL-NEXT: ds_write_b128 v35, v[12:15] offset:32816
+; GISEL-NEXT: ds_write_b128 v35, v[16:19] offset:32832
+; GISEL-NEXT: ds_write_b128 v35, v[20:23] offset:32848
+; GISEL-NEXT: ds_write_b128 v35, v[24:27] offset:32864
+; GISEL-NEXT: ds_write_b128 v35, v[28:31] offset:32880
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
+; GISEL-NEXT: s_endpgm
;
; EXACTCUTOFF-LABEL: test_sched_group_barrier_pipeline_MFMA_interleave:
; EXACTCUTOFF: ; %bb.0: ; %entry
@@ -1184,190 +1978,555 @@ entry:
}
define amdgpu_kernel void @test_sched_group_barrier_pipeline_interleave_EXP_MFMA(ptr addrspace(3) noalias %in, ptr addrspace(3) noalias %out, <5 x float> %in1) #0 {
-; GCN-LABEL: test_sched_group_barrier_pipeline_interleave_EXP_MFMA:
-; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x44
-; GCN-NEXT: v_mov_b32_e32 v3, 0x3fb8aa3b
-; GCN-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GCN-NEXT: v_mov_b32_e32 v7, 0x32a5705f
-; GCN-NEXT: v_lshlrev_b32_e32 v0, 7, v0
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mul_f32_e32 v4, s0, v3
-; GCN-NEXT: v_rndne_f32_e32 v5, v4
-; GCN-NEXT: v_sub_f32_e32 v6, v4, v5
-; GCN-NEXT: v_fma_f32 v4, s0, v3, -v4
-; GCN-NEXT: v_fmac_f32_e32 v4, s0, v7
-; GCN-NEXT: v_add_f32_e32 v4, v6, v4
-; GCN-NEXT: v_exp_f32_e32 v4, v4
-; GCN-NEXT: v_cvt_i32_f32_e32 v5, v5
-; GCN-NEXT: v_and_b32_e32 v0, 0x1ff80, v0
-; GCN-NEXT: v_add_u32_e32 v1, s6, v0
-; GCN-NEXT: ds_read_b128 a[60:63], v1 offset:112
-; GCN-NEXT: ds_read_b128 a[56:59], v1 offset:96
-; GCN-NEXT: ds_read_b128 a[52:55], v1 offset:80
-; GCN-NEXT: ds_read_b128 a[48:51], v1 offset:64
-; GCN-NEXT: ds_read_b128 a[32:35], v1
-; GCN-NEXT: ds_read_b128 a[36:39], v1 offset:16
-; GCN-NEXT: ds_read_b128 a[40:43], v1 offset:32
-; GCN-NEXT: ds_read_b128 a[44:47], v1 offset:48
-; GCN-NEXT: v_mov_b32_e32 v9, 1.0
-; GCN-NEXT: v_ldexp_f32 v4, v4, v5
-; GCN-NEXT: v_mov_b32_e32 v5, 0xc2ce8ed0
-; GCN-NEXT: v_mul_f32_e32 v10, s1, v3
-; GCN-NEXT: v_cmp_nlt_f32_e32 vcc, s0, v5
-; GCN-NEXT: v_mov_b32_e32 v6, 0x42b17218
-; GCN-NEXT: v_rndne_f32_e32 v11, v10
-; GCN-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
-; GCN-NEXT: v_cmp_ngt_f32_e32 vcc, s0, v6
-; GCN-NEXT: v_mov_b32_e32 v8, 0x7f800000
-; GCN-NEXT: v_sub_f32_e32 v12, v10, v11
-; GCN-NEXT: v_fma_f32 v10, s1, v3, -v10
-; GCN-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc
-; GCN-NEXT: v_fmac_f32_e32 v10, s1, v7
-; GCN-NEXT: ds_read_b128 a[28:31], v1 offset:8304
-; GCN-NEXT: s_waitcnt lgkmcnt(1)
-; GCN-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v9, v4, a[32:63]
-; GCN-NEXT: v_add_f32_e32 v4, v12, v10
-; GCN-NEXT: v_exp_f32_e32 v4, v4
-; GCN-NEXT: v_cvt_i32_f32_e32 v10, v11
-; GCN-NEXT: ds_read_b128 a[24:27], v1 offset:8288
-; GCN-NEXT: ds_read_b128 a[20:23], v1 offset:8272
-; GCN-NEXT: ds_read_b128 a[16:19], v1 offset:8256
-; GCN-NEXT: ds_read_b128 a[12:15], v1 offset:8240
-; GCN-NEXT: ds_read_b128 a[8:11], v1 offset:8224
-; GCN-NEXT: ds_read_b128 a[4:7], v1 offset:8208
-; GCN-NEXT: ds_read_b128 a[0:3], v1 offset:8192
-; GCN-NEXT: v_ldexp_f32 v4, v4, v10
-; GCN-NEXT: v_cmp_nlt_f32_e32 vcc, s1, v5
-; GCN-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
-; GCN-NEXT: v_cmp_ngt_f32_e32 vcc, s1, v6
-; GCN-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc
-; GCN-NEXT: v_mul_f32_e32 v10, s2, v3
-; GCN-NEXT: v_rndne_f32_e32 v11, v10
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v9, v4, a[0:31]
-; GCN-NEXT: v_fma_f32 v4, s2, v3, -v10
-; GCN-NEXT: v_sub_f32_e32 v12, v10, v11
-; GCN-NEXT: v_fmac_f32_e32 v4, s2, v7
-; GCN-NEXT: v_add_f32_e32 v4, v12, v4
-; GCN-NEXT: v_exp_f32_e32 v4, v4
-; GCN-NEXT: v_cvt_i32_f32_e32 v10, v11
-; GCN-NEXT: ds_read_b128 a[92:95], v1 offset:24688
-; GCN-NEXT: ds_read_b128 a[88:91], v1 offset:24672
-; GCN-NEXT: ds_read_b128 a[84:87], v1 offset:24656
-; GCN-NEXT: ds_read_b128 a[80:83], v1 offset:24640
-; GCN-NEXT: ds_read_b128 a[76:79], v1 offset:24624
-; GCN-NEXT: ds_read_b128 a[72:75], v1 offset:24608
-; GCN-NEXT: ds_read_b128 a[68:71], v1 offset:24592
-; GCN-NEXT: ds_read_b128 a[64:67], v1 offset:24576
-; GCN-NEXT: v_add_u32_e32 v2, 0x6000, v1
-; GCN-NEXT: ds_read_b128 a[124:127], v1 offset:49264
-; GCN-NEXT: ds_read_b128 a[120:123], v1 offset:49248
-; GCN-NEXT: ds_read_b128 a[116:119], v1 offset:49232
-; GCN-NEXT: ds_read_b128 a[112:115], v1 offset:49216
-; GCN-NEXT: ds_read_b128 a[108:111], v1 offset:49200
-; GCN-NEXT: ds_read_b128 a[104:107], v1 offset:49184
-; GCN-NEXT: ds_read_b128 a[100:103], v1 offset:49168
-; GCN-NEXT: ds_read_b128 a[96:99], v1 offset:49152
-; GCN-NEXT: v_ldexp_f32 v1, v4, v10
-; GCN-NEXT: v_cmp_nlt_f32_e32 vcc, s2, v5
-; GCN-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; GCN-NEXT: v_cmp_ngt_f32_e32 vcc, s2, v6
-; GCN-NEXT: v_mul_f32_e32 v4, s3, v3
-; GCN-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc
-; GCN-NEXT: v_rndne_f32_e32 v10, v4
-; GCN-NEXT: s_load_dword s8, s[4:5], 0x54
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v9, v1, a[64:95]
-; GCN-NEXT: v_sub_f32_e32 v1, v4, v10
-; GCN-NEXT: v_fma_f32 v4, s3, v3, -v4
-; GCN-NEXT: v_fmac_f32_e32 v4, s3, v7
-; GCN-NEXT: v_add_f32_e32 v1, v1, v4
-; GCN-NEXT: v_exp_f32_e32 v1, v1
-; GCN-NEXT: v_cvt_i32_f32_e32 v4, v10
-; GCN-NEXT: v_cmp_nlt_f32_e32 vcc, s3, v5
-; GCN-NEXT: ds_read_b128 a[156:159], v2 offset:57456
-; GCN-NEXT: ds_read_b128 a[152:155], v2 offset:57440
-; GCN-NEXT: v_ldexp_f32 v1, v1, v4
-; GCN-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; GCN-NEXT: v_cmp_ngt_f32_e32 vcc, s3, v6
-; GCN-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc
-; GCN-NEXT: v_mul_f32_e32 v4, s8, v3
-; GCN-NEXT: v_fma_f32 v3, s8, v3, -v4
-; GCN-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v9, v1, a[96:127]
-; GCN-NEXT: v_rndne_f32_e32 v1, v4
-; GCN-NEXT: v_sub_f32_e32 v10, v4, v1
-; GCN-NEXT: v_fmac_f32_e32 v3, s8, v7
-; GCN-NEXT: v_add_f32_e32 v3, v10, v3
-; GCN-NEXT: v_exp_f32_e32 v3, v3
-; GCN-NEXT: v_cvt_i32_f32_e32 v1, v1
-; GCN-NEXT: ds_read_b128 a[148:151], v2 offset:57424
-; GCN-NEXT: ds_read_b128 a[144:147], v2 offset:57408
-; GCN-NEXT: ds_read_b128 a[128:131], v2 offset:57344
-; GCN-NEXT: ds_read_b128 a[132:135], v2 offset:57360
-; GCN-NEXT: ds_read_b128 a[136:139], v2 offset:57376
-; GCN-NEXT: ds_read_b128 a[140:143], v2 offset:57392
-; GCN-NEXT: v_ldexp_f32 v1, v3, v1
-; GCN-NEXT: v_cmp_nlt_f32_e32 vcc, s8, v5
-; GCN-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; GCN-NEXT: v_cmp_ngt_f32_e32 vcc, s8, v6
-; GCN-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc
-; GCN-NEXT: v_add_u32_e32 v0, s7, v0
-; GCN-NEXT: ds_write_b128 v0, a[60:63] offset:112
-; GCN-NEXT: s_waitcnt lgkmcnt(1)
-; GCN-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v9, v1, a[128:159]
-; GCN-NEXT: ds_write_b128 v0, a[56:59] offset:96
-; GCN-NEXT: ds_write_b128 v0, a[52:55] offset:80
-; GCN-NEXT: ds_write_b128 v0, a[48:51] offset:64
-; GCN-NEXT: ds_write_b128 v0, a[44:47] offset:48
-; GCN-NEXT: ds_write_b128 v0, a[40:43] offset:32
-; GCN-NEXT: ds_write_b128 v0, a[36:39] offset:16
-; GCN-NEXT: ds_write_b128 v0, a[32:35]
-; GCN-NEXT: v_mov_b32_e32 v0, s7
-; GCN-NEXT: ; kill: killed $sgpr4_sgpr5
-; GCN-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
-; GCN-NEXT: ds_write_b128 v0, a[24:27] offset:8288
-; GCN-NEXT: ds_write_b128 v0, a[28:31] offset:8304
-; GCN-NEXT: ds_write_b128 v0, a[16:19] offset:8256
-; GCN-NEXT: ds_write_b128 v0, a[20:23] offset:8272
-; GCN-NEXT: ds_write_b128 v0, a[8:11] offset:8224
-; GCN-NEXT: ds_write_b128 v0, a[12:15] offset:8240
-; GCN-NEXT: ds_write_b128 v0, a[0:3] offset:8192
-; GCN-NEXT: ds_write_b128 v0, a[4:7] offset:8208
-; GCN-NEXT: ds_write_b128 v0, a[88:91] offset:16480
-; GCN-NEXT: ds_write_b128 v0, a[92:95] offset:16496
-; GCN-NEXT: ds_write_b128 v0, a[80:83] offset:16448
-; GCN-NEXT: ds_write_b128 v0, a[84:87] offset:16464
-; GCN-NEXT: ds_write_b128 v0, a[72:75] offset:16416
-; GCN-NEXT: ds_write_b128 v0, a[76:79] offset:16432
-; GCN-NEXT: ds_write_b128 v0, a[64:67] offset:16384
-; GCN-NEXT: ds_write_b128 v0, a[68:71] offset:16400
-; GCN-NEXT: ds_write_b128 v0, a[120:123] offset:24672
-; GCN-NEXT: ds_write_b128 v0, a[124:127] offset:24688
-; GCN-NEXT: ds_write_b128 v0, a[112:115] offset:24640
-; GCN-NEXT: ds_write_b128 v0, a[116:119] offset:24656
-; GCN-NEXT: ds_write_b128 v0, a[104:107] offset:24608
-; GCN-NEXT: ds_write_b128 v0, a[108:111] offset:24624
-; GCN-NEXT: ds_write_b128 v0, a[96:99] offset:24576
-; GCN-NEXT: ds_write_b128 v0, a[100:103] offset:24592
-; GCN-NEXT: ds_write_b128 v0, a[152:155] offset:32864
-; GCN-NEXT: ds_write_b128 v0, a[156:159] offset:32880
-; GCN-NEXT: ds_write_b128 v0, a[144:147] offset:32832
-; GCN-NEXT: ds_write_b128 v0, a[148:151] offset:32848
-; GCN-NEXT: ds_write_b128 v0, a[136:139] offset:32800
-; GCN-NEXT: ds_write_b128 v0, a[140:143] offset:32816
-; GCN-NEXT: ds_write_b128 v0, a[128:131] offset:32768
-; GCN-NEXT: ds_write_b128 v0, a[132:135] offset:32784
-; GCN-NEXT: s_endpgm
+; SDAG-LABEL: test_sched_group_barrier_pipeline_interleave_EXP_MFMA:
+; SDAG: ; %bb.0: ; %entry
+; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x44
+; SDAG-NEXT: v_mov_b32_e32 v3, 0x3fb8aa3b
+; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; SDAG-NEXT: v_mov_b32_e32 v7, 0x32a5705f
+; SDAG-NEXT: v_lshlrev_b32_e32 v0, 7, v0
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mul_f32_e32 v4, s0, v3
+; SDAG-NEXT: v_rndne_f32_e32 v5, v4
+; SDAG-NEXT: v_sub_f32_e32 v6, v4, v5
+; SDAG-NEXT: v_fma_f32 v4, s0, v3, -v4
+; SDAG-NEXT: v_fmac_f32_e32 v4, s0, v7
+; SDAG-NEXT: v_add_f32_e32 v4, v6, v4
+; SDAG-NEXT: v_exp_f32_e32 v4, v4
+; SDAG-NEXT: v_cvt_i32_f32_e32 v5, v5
+; SDAG-NEXT: v_and_b32_e32 v0, 0x1ff80, v0
+; SDAG-NEXT: v_add_u32_e32 v1, s6, v0
+; SDAG-NEXT: ds_read_b128 a[60:63], v1 offset:112
+; SDAG-NEXT: ds_read_b128 a[56:59], v1 offset:96
+; SDAG-NEXT: ds_read_b128 a[52:55], v1 offset:80
+; SDAG-NEXT: ds_read_b128 a[48:51], v1 offset:64
+; SDAG-NEXT: ds_read_b128 a[32:35], v1
+; SDAG-NEXT: ds_read_b128 a[36:39], v1 offset:16
+; SDAG-NEXT: ds_read_b128 a[40:43], v1 offset:32
+; SDAG-NEXT: ds_read_b128 a[44:47], v1 offset:48
+; SDAG-NEXT: v_mov_b32_e32 v9, 1.0
+; SDAG-NEXT: v_ldexp_f32 v4, v4, v5
+; SDAG-NEXT: v_mov_b32_e32 v5, 0xc2ce8ed0
+; SDAG-NEXT: v_mul_f32_e32 v10, s1, v3
+; SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s0, v5
+; SDAG-NEXT: v_mov_b32_e32 v6, 0x42b17218
+; SDAG-NEXT: v_rndne_f32_e32 v11, v10
+; SDAG-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
+; SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s0, v6
+; SDAG-NEXT: v_mov_b32_e32 v8, 0x7f800000
+; SDAG-NEXT: v_sub_f32_e32 v12, v10, v11
+; SDAG-NEXT: v_fma_f32 v10, s1, v3, -v10
+; SDAG-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc
+; SDAG-NEXT: v_fmac_f32_e32 v10, s1, v7
+; SDAG-NEXT: ds_read_b128 a[28:31], v1 offset:8304
+; SDAG-NEXT: s_waitcnt lgkmcnt(1)
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v9, v4, a[32:63]
+; SDAG-NEXT: v_add_f32_e32 v4, v12, v10
+; SDAG-NEXT: v_exp_f32_e32 v4, v4
+; SDAG-NEXT: v_cvt_i32_f32_e32 v10, v11
+; SDAG-NEXT: ds_read_b128 a[24:27], v1 offset:8288
+; SDAG-NEXT: ds_read_b128 a[20:23], v1 offset:8272
+; SDAG-NEXT: ds_read_b128 a[16:19], v1 offset:8256
+; SDAG-NEXT: ds_read_b128 a[12:15], v1 offset:8240
+; SDAG-NEXT: ds_read_b128 a[8:11], v1 offset:8224
+; SDAG-NEXT: ds_read_b128 a[4:7], v1 offset:8208
+; SDAG-NEXT: ds_read_b128 a[0:3], v1 offset:8192
+; SDAG-NEXT: v_ldexp_f32 v4, v4, v10
+; SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s1, v5
+; SDAG-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
+; SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s1, v6
+; SDAG-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc
+; SDAG-NEXT: v_mul_f32_e32 v10, s2, v3
+; SDAG-NEXT: v_rndne_f32_e32 v11, v10
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v9, v4, a[0:31]
+; SDAG-NEXT: v_fma_f32 v4, s2, v3, -v10
+; SDAG-NEXT: v_sub_f32_e32 v12, v10, v11
+; SDAG-NEXT: v_fmac_f32_e32 v4, s2, v7
+; SDAG-NEXT: v_add_f32_e32 v4, v12, v4
+; SDAG-NEXT: v_exp_f32_e32 v4, v4
+; SDAG-NEXT: v_cvt_i32_f32_e32 v10, v11
+; SDAG-NEXT: ds_read_b128 a[92:95], v1 offset:24688
+; SDAG-NEXT: ds_read_b128 a[88:91], v1 offset:24672
+; SDAG-NEXT: ds_read_b128 a[84:87], v1 offset:24656
+; SDAG-NEXT: ds_read_b128 a[80:83], v1 offset:24640
+; SDAG-NEXT: ds_read_b128 a[76:79], v1 offset:24624
+; SDAG-NEXT: ds_read_b128 a[72:75], v1 offset:24608
+; SDAG-NEXT: ds_read_b128 a[68:71], v1 offset:24592
+; SDAG-NEXT: ds_read_b128 a[64:67], v1 offset:24576
+; SDAG-NEXT: v_add_u32_e32 v2, 0x6000, v1
+; SDAG-NEXT: ds_read_b128 a[124:127], v1 offset:49264
+; SDAG-NEXT: ds_read_b128 a[120:123], v1 offset:49248
+; SDAG-NEXT: ds_read_b128 a[116:119], v1 offset:49232
+; SDAG-NEXT: ds_read_b128 a[112:115], v1 offset:49216
+; SDAG-NEXT: ds_read_b128 a[108:111], v1 offset:49200
+; SDAG-NEXT: ds_read_b128 a[104:107], v1 offset:49184
+; SDAG-NEXT: ds_read_b128 a[100:103], v1 offset:49168
+; SDAG-NEXT: ds_read_b128 a[96:99], v1 offset:49152
+; SDAG-NEXT: v_ldexp_f32 v1, v4, v10
+; SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s2, v5
+; SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
+; SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s2, v6
+; SDAG-NEXT: v_mul_f32_e32 v4, s3, v3
+; SDAG-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc
+; SDAG-NEXT: v_rndne_f32_e32 v10, v4
+; SDAG-NEXT: s_load_dword s8, s[4:5], 0x54
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v9, v1, a[64:95]
+; SDAG-NEXT: v_sub_f32_e32 v1, v4, v10
+; SDAG-NEXT: v_fma_f32 v4, s3, v3, -v4
+; SDAG-NEXT: v_fmac_f32_e32 v4, s3, v7
+; SDAG-NEXT: v_add_f32_e32 v1, v1, v4
+; SDAG-NEXT: v_exp_f32_e32 v1, v1
+; SDAG-NEXT: v_cvt_i32_f32_e32 v4, v10
+; SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s3, v5
+; SDAG-NEXT: ds_read_b128 a[156:159], v2 offset:57456
+; SDAG-NEXT: ds_read_b128 a[152:155], v2 offset:57440
+; SDAG-NEXT: v_ldexp_f32 v1, v1, v4
+; SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
+; SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s3, v6
+; SDAG-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc
+; SDAG-NEXT: v_mul_f32_e32 v4, s8, v3
+; SDAG-NEXT: v_fma_f32 v3, s8, v3, -v4
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v9, v1, a[96:127]
+; SDAG-NEXT: v_rndne_f32_e32 v1, v4
+; SDAG-NEXT: v_sub_f32_e32 v10, v4, v1
+; SDAG-NEXT: v_fmac_f32_e32 v3, s8, v7
+; SDAG-NEXT: v_add_f32_e32 v3, v10, v3
+; SDAG-NEXT: v_exp_f32_e32 v3, v3
+; SDAG-NEXT: v_cvt_i32_f32_e32 v1, v1
+; SDAG-NEXT: ds_read_b128 a[148:151], v2 offset:57424
+; SDAG-NEXT: ds_read_b128 a[144:147], v2 offset:57408
+; SDAG-NEXT: ds_read_b128 a[128:131], v2 offset:57344
+; SDAG-NEXT: ds_read_b128 a[132:135], v2 offset:57360
+; SDAG-NEXT: ds_read_b128 a[136:139], v2 offset:57376
+; SDAG-NEXT: ds_read_b128 a[140:143], v2 offset:57392
+; SDAG-NEXT: v_ldexp_f32 v1, v3, v1
+; SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s8, v5
+; SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
+; SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s8, v6
+; SDAG-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc
+; SDAG-NEXT: v_add_u32_e32 v0, s7, v0
+; SDAG-NEXT: ds_write_b128 v0, a[60:63] offset:112
+; SDAG-NEXT: s_waitcnt lgkmcnt(1)
+; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v9, v1, a[128:159]
+; SDAG-NEXT: ds_write_b128 v0, a[56:59] offset:96
+; SDAG-NEXT: ds_write_b128 v0, a[52:55] offset:80
+; SDAG-NEXT: ds_write_b128 v0, a[48:51] offset:64
+; SDAG-NEXT: ds_write_b128 v0, a[44:47] offset:48
+; SDAG-NEXT: ds_write_b128 v0, a[40:43] offset:32
+; SDAG-NEXT: ds_write_b128 v0, a[36:39] offset:16
+; SDAG-NEXT: ds_write_b128 v0, a[32:35]
+; SDAG-NEXT: v_mov_b32_e32 v0, s7
+; SDAG-NEXT: ; kill: killed $sgpr4_sgpr5
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
+; SDAG-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
+; SDAG-NEXT: ds_write_b128 v0, a[24:27] offset:8288
+; SDAG-NEXT: ds_write_b128 v0, a[28:31] offset:8304
+; SDAG-NEXT: ds_write_b128 v0, a[16:19] offset:8256
+; SDAG-NEXT: ds_write_b128 v0, a[20:23] offset:8272
+; SDAG-NEXT: ds_write_b128 v0, a[8:11] offset:8224
+; SDAG-NEXT: ds_write_b128 v0, a[12:15] offset:8240
+; SDAG-NEXT: ds_write_b128 v0, a[0:3] offset:8192
+; SDAG-NEXT: ds_write_b128 v0, a[4:7] offset:8208
+; SDAG-NEXT: ds_write_b128 v0, a[88:91] offset:16480
+; SDAG-NEXT: ds_write_b128 v0, a[92:95] offset:16496
+; SDAG-NEXT: ds_write_b128 v0, a[80:83] offset:16448
+; SDAG-NEXT: ds_write_b128 v0, a[84:87] offset:16464
+; SDAG-NEXT: ds_write_b128 v0, a[72:75] offset:16416
+; SDAG-NEXT: ds_write_b128 v0, a[76:79] offset:16432
+; SDAG-NEXT: ds_write_b128 v0, a[64:67] offset:16384
+; SDAG-NEXT: ds_write_b128 v0, a[68:71] offset:16400
+; SDAG-NEXT: ds_write_b128 v0, a[120:123] offset:24672
+; SDAG-NEXT: ds_write_b128 v0, a[124:127] offset:24688
+; SDAG-NEXT: ds_write_b128 v0, a[112:115] offset:24640
+; SDAG-NEXT: ds_write_b128 v0, a[116:119] offset:24656
+; SDAG-NEXT: ds_write_b128 v0, a[104:107] offset:24608
+; SDAG-NEXT: ds_write_b128 v0, a[108:111] offset:24624
+; SDAG-NEXT: ds_write_b128 v0, a[96:99] offset:24576
+; SDAG-NEXT: ds_write_b128 v0, a[100:103] offset:24592
+; SDAG-NEXT: ds_write_b128 v0, a[152:155] offset:32864
+; SDAG-NEXT: ds_write_b128 v0, a[156:159] offset:32880
+; SDAG-NEXT: ds_write_b128 v0, a[144:147] offset:32832
+; SDAG-NEXT: ds_write_b128 v0, a[148:151] offset:32848
+; SDAG-NEXT: ds_write_b128 v0, a[136:139] offset:32800
+; SDAG-NEXT: ds_write_b128 v0, a[140:143] offset:32816
+; SDAG-NEXT: ds_write_b128 v0, a[128:131] offset:32768
+; SDAG-NEXT: ds_write_b128 v0, a[132:135] offset:32784
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: test_sched_group_barrier_pipeline_interleave_EXP_MFMA:
+; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x44
+; GISEL-NEXT: v_mov_b32_e32 v2, 0x3fb8aa3b
+; GISEL-NEXT: v_mov_b32_e32 v5, 0x32a5705f
+; GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: v_mul_f32_e32 v3, s0, v2
+; GISEL-NEXT: v_fma_f32 v4, s0, v2, -v3
+; GISEL-NEXT: v_rndne_f32_e32 v6, v3
+; GISEL-NEXT: v_fmac_f32_e32 v4, s0, v5
+; GISEL-NEXT: v_sub_f32_e32 v3, v3, v6
+; GISEL-NEXT: v_add_f32_e32 v3, v3, v4
+; GISEL-NEXT: v_cvt_i32_f32_e32 v4, v6
+; GISEL-NEXT: v_exp_f32_e32 v3, v3
+; GISEL-NEXT: v_lshlrev_b32_e32 v32, 7, v0
+; GISEL-NEXT: s_load_dword s8, s[4:5], 0x54
+; GISEL-NEXT: ; kill: killed $sgpr4_sgpr5
+; GISEL-NEXT: v_add_u32_e32 v0, s6, v32
+; GISEL-NEXT: v_ldexp_f32 v3, v3, v4
+; GISEL-NEXT: v_readfirstlane_b32 s4, v3
+; GISEL-NEXT: v_mov_b32_e32 v3, 0xc2ce8ed0
+; GISEL-NEXT: ds_read_b128 a[64:67], v0
+; GISEL-NEXT: ds_read_b128 a[68:71], v0 offset:16
+; GISEL-NEXT: ds_read_b128 a[72:75], v0 offset:32
+; GISEL-NEXT: ds_read_b128 a[76:79], v0 offset:48
+; GISEL-NEXT: ds_read_b128 a[80:83], v0 offset:64
+; GISEL-NEXT: ds_read_b128 a[84:87], v0 offset:80
+; GISEL-NEXT: ds_read_b128 a[88:91], v0 offset:96
+; GISEL-NEXT: ds_read_b128 a[92:95], v0 offset:112
+; GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v3
+; GISEL-NEXT: v_mov_b32_e32 v4, 0x42b17218
+; GISEL-NEXT: v_mov_b32_e32 v33, 1.0
+; GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GISEL-NEXT: v_cmp_gt_f32_e32 vcc, s0, v4
+; GISEL-NEXT: v_mul_f32_e32 v6, s1, v2
+; GISEL-NEXT: s_cselect_b32 s4, 0, s4
+; GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GISEL-NEXT: v_fma_f32 v7, s1, v2, -v6
+; GISEL-NEXT: v_rndne_f32_e32 v8, v6
+; GISEL-NEXT: s_cselect_b32 s0, 0x7f800000, s4
+; GISEL-NEXT: v_fmac_f32_e32 v7, s1, v5
+; GISEL-NEXT: v_sub_f32_e32 v6, v6, v8
+; GISEL-NEXT: v_add_f32_e32 v6, v6, v7
+; GISEL-NEXT: v_cvt_i32_f32_e32 v7, v8
+; GISEL-NEXT: v_mov_b32_e32 v8, s0
+; GISEL-NEXT: ds_read_b128 a[0:3], v0 offset:8192
+; GISEL-NEXT: ds_read_b128 a[4:7], v0 offset:8208
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v33, v8, a[64:95]
+; GISEL-NEXT: v_exp_f32_e32 v6, v6
+; GISEL-NEXT: ds_read_b128 a[8:11], v0 offset:8224
+; GISEL-NEXT: ds_read_b128 a[12:15], v0 offset:8240
+; GISEL-NEXT: ds_read_b128 a[16:19], v0 offset:8256
+; GISEL-NEXT: ds_read_b128 a[20:23], v0 offset:8272
+; GISEL-NEXT: ds_read_b128 a[24:27], v0 offset:8288
+; GISEL-NEXT: ds_read_b128 a[28:31], v0 offset:8304
+; GISEL-NEXT: v_ldexp_f32 v6, v6, v7
+; GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s1, v3
+; GISEL-NEXT: v_readfirstlane_b32 s0, v6
+; GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GISEL-NEXT: v_cmp_gt_f32_e32 vcc, s1, v4
+; GISEL-NEXT: v_mul_f32_e32 v6, s2, v2
+; GISEL-NEXT: s_cselect_b32 s0, 0, s0
+; GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GISEL-NEXT: v_fma_f32 v7, s2, v2, -v6
+; GISEL-NEXT: v_rndne_f32_e32 v8, v6
+; GISEL-NEXT: s_cselect_b32 s0, 0x7f800000, s0
+; GISEL-NEXT: v_fmac_f32_e32 v7, s2, v5
+; GISEL-NEXT: v_sub_f32_e32 v6, v6, v8
+; GISEL-NEXT: v_add_f32_e32 v6, v6, v7
+; GISEL-NEXT: v_cvt_i32_f32_e32 v7, v8
+; GISEL-NEXT: v_mov_b32_e32 v8, s0
+; GISEL-NEXT: ds_read_b128 a[32:35], v0 offset:24576
+; GISEL-NEXT: ds_read_b128 a[36:39], v0 offset:24592
+; GISEL-NEXT: s_waitcnt lgkmcnt(2)
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v33, v8, a[0:31]
+; GISEL-NEXT: v_exp_f32_e32 v6, v6
+; GISEL-NEXT: ds_read_b128 a[40:43], v0 offset:24608
+; GISEL-NEXT: ds_read_b128 a[44:47], v0 offset:24624
+; GISEL-NEXT: ds_read_b128 a[48:51], v0 offset:24640
+; GISEL-NEXT: ds_read_b128 a[52:55], v0 offset:24656
+; GISEL-NEXT: ds_read_b128 a[56:59], v0 offset:24672
+; GISEL-NEXT: ds_read_b128 a[60:63], v0 offset:24688
+; GISEL-NEXT: v_add_u32_e32 v1, 0xc000, v0
+; GISEL-NEXT: ds_read_b128 a[96:99], v0 offset:49152
+; GISEL-NEXT: ds_read_b128 a[100:103], v0 offset:49168
+; GISEL-NEXT: ds_read_b128 a[104:107], v0 offset:49184
+; GISEL-NEXT: ds_read_b128 a[108:111], v0 offset:49200
+; GISEL-NEXT: ds_read_b128 a[112:115], v0 offset:49216
+; GISEL-NEXT: ds_read_b128 a[116:119], v0 offset:49232
+; GISEL-NEXT: ds_read_b128 a[120:123], v0 offset:49248
+; GISEL-NEXT: ds_read_b128 a[124:127], v0 offset:49264
+; GISEL-NEXT: v_ldexp_f32 v0, v6, v7
+; GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v3
+; GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GISEL-NEXT: v_cmp_gt_f32_e32 vcc, s2, v4
+; GISEL-NEXT: v_mul_f32_e32 v0, s3, v2
+; GISEL-NEXT: s_cselect_b32 s0, 0, s0
+; GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GISEL-NEXT: v_fma_f32 v6, s3, v2, -v0
+; GISEL-NEXT: v_rndne_f32_e32 v7, v0
+; GISEL-NEXT: s_cselect_b32 s0, 0x7f800000, s0
+; GISEL-NEXT: v_fmac_f32_e32 v6, s3, v5
+; GISEL-NEXT: v_sub_f32_e32 v0, v0, v7
+; GISEL-NEXT: v_add_f32_e32 v0, v0, v6
+; GISEL-NEXT: v_cvt_i32_f32_e32 v6, v7
+; GISEL-NEXT: v_mov_b32_e32 v7, s0
+; GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s3, v3
+; GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GISEL-NEXT: s_waitcnt lgkmcnt(8)
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v33, v7, a[32:63]
+; GISEL-NEXT: v_exp_f32_e32 v0, v0
+; GISEL-NEXT: v_cmp_gt_f32_e32 vcc, s3, v4
+; GISEL-NEXT: ds_read_b128 a[128:131], v1 offset:32768
+; GISEL-NEXT: ds_read_b128 a[132:135], v1 offset:32784
+; GISEL-NEXT: v_ldexp_f32 v0, v0, v6
+; GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GISEL-NEXT: v_mul_f32_e32 v0, s8, v2
+; GISEL-NEXT: v_fma_f32 v2, s8, v2, -v0
+; GISEL-NEXT: s_cselect_b32 s0, 0, s0
+; GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GISEL-NEXT: v_fmac_f32_e32 v2, s8, v5
+; GISEL-NEXT: v_rndne_f32_e32 v5, v0
+; GISEL-NEXT: s_cselect_b32 s0, 0x7f800000, s0
+; GISEL-NEXT: v_sub_f32_e32 v0, v0, v5
+; GISEL-NEXT: v_add_f32_e32 v0, v0, v2
+; GISEL-NEXT: v_cvt_i32_f32_e32 v2, v5
+; GISEL-NEXT: v_mov_b32_e32 v5, s0
+; GISEL-NEXT: ds_read_b128 a[136:139], v1 offset:32800
+; GISEL-NEXT: ds_read_b128 a[140:143], v1 offset:32816
+; GISEL-NEXT: s_waitcnt lgkmcnt(4)
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v33, v5, a[96:127]
+; GISEL-NEXT: v_exp_f32_e32 v0, v0
+; GISEL-NEXT: ds_read_b128 a[144:147], v1 offset:32832
+; GISEL-NEXT: ds_read_b128 a[148:151], v1 offset:32848
+; GISEL-NEXT: ds_read_b128 a[152:155], v1 offset:32864
+; GISEL-NEXT: ds_read_b128 a[156:159], v1 offset:32880
+; GISEL-NEXT: v_ldexp_f32 v0, v0, v2
+; GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s8, v3
+; GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GISEL-NEXT: v_cmp_gt_f32_e32 vcc, s8, v4
+; GISEL-NEXT: s_cselect_b32 s0, 0, s0
+; GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GISEL-NEXT: s_cselect_b32 s0, 0x7f800000, s0
+; GISEL-NEXT: v_mov_b32_e32 v34, s0
+; GISEL-NEXT: v_accvgpr_read_b32 v0, a64
+; GISEL-NEXT: v_accvgpr_read_b32 v1, a65
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v33, v34, a[128:159]
+; GISEL-NEXT: v_accvgpr_read_b32 v2, a66
+; GISEL-NEXT: v_accvgpr_read_b32 v3, a67
+; GISEL-NEXT: v_accvgpr_read_b32 v4, a68
+; GISEL-NEXT: v_accvgpr_read_b32 v5, a69
+; GISEL-NEXT: v_accvgpr_read_b32 v6, a70
+; GISEL-NEXT: v_accvgpr_read_b32 v7, a71
+; GISEL-NEXT: v_accvgpr_read_b32 v8, a72
+; GISEL-NEXT: v_accvgpr_read_b32 v9, a73
+; GISEL-NEXT: v_accvgpr_read_b32 v10, a74
+; GISEL-NEXT: v_accvgpr_read_b32 v11, a75
+; GISEL-NEXT: v_accvgpr_read_b32 v12, a76
+; GISEL-NEXT: v_accvgpr_read_b32 v13, a77
+; GISEL-NEXT: v_accvgpr_read_b32 v14, a78
+; GISEL-NEXT: v_accvgpr_read_b32 v15, a79
+; GISEL-NEXT: v_accvgpr_read_b32 v16, a80
+; GISEL-NEXT: v_accvgpr_read_b32 v17, a81
+; GISEL-NEXT: v_accvgpr_read_b32 v18, a82
+; GISEL-NEXT: v_accvgpr_read_b32 v19, a83
+; GISEL-NEXT: v_accvgpr_read_b32 v20, a84
+; GISEL-NEXT: v_accvgpr_read_b32 v21, a85
+; GISEL-NEXT: v_accvgpr_read_b32 v22, a86
+; GISEL-NEXT: v_accvgpr_read_b32 v23, a87
+; GISEL-NEXT: v_accvgpr_read_b32 v24, a88
+; GISEL-NEXT: v_accvgpr_read_b32 v25, a89
+; GISEL-NEXT: v_accvgpr_read_b32 v26, a90
+; GISEL-NEXT: v_accvgpr_read_b32 v27, a91
+; GISEL-NEXT: v_accvgpr_read_b32 v28, a92
+; GISEL-NEXT: v_accvgpr_read_b32 v29, a93
+; GISEL-NEXT: v_accvgpr_read_b32 v30, a94
+; GISEL-NEXT: v_accvgpr_read_b32 v31, a95
+; GISEL-NEXT: v_add_u32_e32 v32, s7, v32
+; GISEL-NEXT: ds_write_b128 v32, v[0:3]
+; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:16
+; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:32
+; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:48
+; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:64
+; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:80
+; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:96
+; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:112
+; GISEL-NEXT: v_accvgpr_read_b32 v0, a0
+; GISEL-NEXT: v_accvgpr_read_b32 v1, a1
+; GISEL-NEXT: v_accvgpr_read_b32 v2, a2
+; GISEL-NEXT: v_accvgpr_read_b32 v3, a3
+; GISEL-NEXT: v_accvgpr_read_b32 v4, a4
+; GISEL-NEXT: v_accvgpr_read_b32 v5, a5
+; GISEL-NEXT: v_accvgpr_read_b32 v6, a6
+; GISEL-NEXT: v_accvgpr_read_b32 v7, a7
+; GISEL-NEXT: v_accvgpr_read_b32 v8, a8
+; GISEL-NEXT: v_accvgpr_read_b32 v9, a9
+; GISEL-NEXT: v_accvgpr_read_b32 v10, a10
+; GISEL-NEXT: v_accvgpr_read_b32 v11, a11
+; GISEL-NEXT: v_accvgpr_read_b32 v12, a12
+; GISEL-NEXT: v_accvgpr_read_b32 v13, a13
+; GISEL-NEXT: v_accvgpr_read_b32 v14, a14
+; GISEL-NEXT: v_accvgpr_read_b32 v15, a15
+; GISEL-NEXT: v_accvgpr_read_b32 v16, a16
+; GISEL-NEXT: v_accvgpr_read_b32 v17, a17
+; GISEL-NEXT: v_accvgpr_read_b32 v18, a18
+; GISEL-NEXT: v_accvgpr_read_b32 v19, a19
+; GISEL-NEXT: v_accvgpr_read_b32 v20, a20
+; GISEL-NEXT: v_accvgpr_read_b32 v21, a21
+; GISEL-NEXT: v_accvgpr_read_b32 v22, a22
+; GISEL-NEXT: v_accvgpr_read_b32 v23, a23
+; GISEL-NEXT: v_accvgpr_read_b32 v24, a24
+; GISEL-NEXT: v_accvgpr_read_b32 v25, a25
+; GISEL-NEXT: v_accvgpr_read_b32 v26, a26
+; GISEL-NEXT: v_accvgpr_read_b32 v27, a27
+; GISEL-NEXT: v_accvgpr_read_b32 v28, a28
+; GISEL-NEXT: v_accvgpr_read_b32 v29, a29
+; GISEL-NEXT: v_accvgpr_read_b32 v30, a30
+; GISEL-NEXT: v_accvgpr_read_b32 v31, a31
+; GISEL-NEXT: v_mov_b32_e32 v32, s7
+; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:8192
+; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:8208
+; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:8224
+; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:8240
+; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:8256
+; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:8272
+; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:8288
+; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:8304
+; GISEL-NEXT: v_accvgpr_read_b32 v0, a32
+; GISEL-NEXT: v_accvgpr_read_b32 v1, a33
+; GISEL-NEXT: v_accvgpr_read_b32 v2, a34
+; GISEL-NEXT: v_accvgpr_read_b32 v3, a35
+; GISEL-NEXT: v_accvgpr_read_b32 v4, a36
+; GISEL-NEXT: v_accvgpr_read_b32 v5, a37
+; GISEL-NEXT: v_accvgpr_read_b32 v6, a38
+; GISEL-NEXT: v_accvgpr_read_b32 v7, a39
+; GISEL-NEXT: v_accvgpr_read_b32 v8, a40
+; GISEL-NEXT: v_accvgpr_read_b32 v9, a41
+; GISEL-NEXT: v_accvgpr_read_b32 v10, a42
+; GISEL-NEXT: v_accvgpr_read_b32 v11, a43
+; GISEL-NEXT: v_accvgpr_read_b32 v12, a44
+; GISEL-NEXT: v_accvgpr_read_b32 v13, a45
+; GISEL-NEXT: v_accvgpr_read_b32 v14, a46
+; GISEL-NEXT: v_accvgpr_read_b32 v15, a47
+; GISEL-NEXT: v_accvgpr_read_b32 v16, a48
+; GISEL-NEXT: v_accvgpr_read_b32 v17, a49
+; GISEL-NEXT: v_accvgpr_read_b32 v18, a50
+; GISEL-NEXT: v_accvgpr_read_b32 v19, a51
+; GISEL-NEXT: v_accvgpr_read_b32 v20, a52
+; GISEL-NEXT: v_accvgpr_read_b32 v21, a53
+; GISEL-NEXT: v_accvgpr_read_b32 v22, a54
+; GISEL-NEXT: v_accvgpr_read_b32 v23, a55
+; GISEL-NEXT: v_accvgpr_read_b32 v24, a56
+; GISEL-NEXT: v_accvgpr_read_b32 v25, a57
+; GISEL-NEXT: v_accvgpr_read_b32 v26, a58
+; GISEL-NEXT: v_accvgpr_read_b32 v27, a59
+; GISEL-NEXT: v_accvgpr_read_b32 v28, a60
+; GISEL-NEXT: v_accvgpr_read_b32 v29, a61
+; GISEL-NEXT: v_accvgpr_read_b32 v30, a62
+; GISEL-NEXT: v_accvgpr_read_b32 v31, a63
+; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:16384
+; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:16400
+; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:16416
+; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:16432
+; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:16448
+; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:16464
+; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:16480
+; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:16496
+; GISEL-NEXT: v_accvgpr_read_b32 v0, a96
+; GISEL-NEXT: v_accvgpr_read_b32 v1, a97
+; GISEL-NEXT: v_accvgpr_read_b32 v2, a98
+; GISEL-NEXT: v_accvgpr_read_b32 v3, a99
+; GISEL-NEXT: v_accvgpr_read_b32 v4, a100
+; GISEL-NEXT: v_accvgpr_read_b32 v5, a101
+; GISEL-NEXT: v_accvgpr_read_b32 v6, a102
+; GISEL-NEXT: v_accvgpr_read_b32 v7, a103
+; GISEL-NEXT: v_accvgpr_read_b32 v8, a104
+; GISEL-NEXT: v_accvgpr_read_b32 v9, a105
+; GISEL-NEXT: v_accvgpr_read_b32 v10, a106
+; GISEL-NEXT: v_accvgpr_read_b32 v11, a107
+; GISEL-NEXT: v_accvgpr_read_b32 v12, a108
+; GISEL-NEXT: v_accvgpr_read_b32 v13, a109
+; GISEL-NEXT: v_accvgpr_read_b32 v14, a110
+; GISEL-NEXT: v_accvgpr_read_b32 v15, a111
+; GISEL-NEXT: v_accvgpr_read_b32 v16, a112
+; GISEL-NEXT: v_accvgpr_read_b32 v17, a113
+; GISEL-NEXT: v_accvgpr_read_b32 v18, a114
+; GISEL-NEXT: v_accvgpr_read_b32 v19, a115
+; GISEL-NEXT: v_accvgpr_read_b32 v20, a116
+; GISEL-NEXT: v_accvgpr_read_b32 v21, a117
+; GISEL-NEXT: v_accvgpr_read_b32 v22, a118
+; GISEL-NEXT: v_accvgpr_read_b32 v23, a119
+; GISEL-NEXT: v_accvgpr_read_b32 v24, a120
+; GISEL-NEXT: v_accvgpr_read_b32 v25, a121
+; GISEL-NEXT: v_accvgpr_read_b32 v26, a122
+; GISEL-NEXT: v_accvgpr_read_b32 v27, a123
+; GISEL-NEXT: v_accvgpr_read_b32 v28, a124
+; GISEL-NEXT: v_accvgpr_read_b32 v29, a125
+; GISEL-NEXT: v_accvgpr_read_b32 v30, a126
+; GISEL-NEXT: v_accvgpr_read_b32 v31, a127
+; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:24576
+; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:24592
+; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:24608
+; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:24624
+; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:24640
+; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:24656
+; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:24672
+; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:24688
+; GISEL-NEXT: v_accvgpr_read_b32 v0, a128
+; GISEL-NEXT: v_accvgpr_read_b32 v1, a129
+; GISEL-NEXT: v_accvgpr_read_b32 v2, a130
+; GISEL-NEXT: v_accvgpr_read_b32 v3, a131
+; GISEL-NEXT: v_accvgpr_read_b32 v4, a132
+; GISEL-NEXT: v_accvgpr_read_b32 v5, a133
+; GISEL-NEXT: v_accvgpr_read_b32 v6, a134
+; GISEL-NEXT: v_accvgpr_read_b32 v7, a135
+; GISEL-NEXT: v_accvgpr_read_b32 v8, a136
+; GISEL-NEXT: v_accvgpr_read_b32 v9, a137
+; GISEL-NEXT: v_accvgpr_read_b32 v10, a138
+; GISEL-NEXT: v_accvgpr_read_b32 v11, a139
+; GISEL-NEXT: v_accvgpr_read_b32 v12, a140
+; GISEL-NEXT: v_accvgpr_read_b32 v13, a141
+; GISEL-NEXT: v_accvgpr_read_b32 v14, a142
+; GISEL-NEXT: v_accvgpr_read_b32 v15, a143
+; GISEL-NEXT: v_accvgpr_read_b32 v16, a144
+; GISEL-NEXT: v_accvgpr_read_b32 v17, a145
+; GISEL-NEXT: v_accvgpr_read_b32 v18, a146
+; GISEL-NEXT: v_accvgpr_read_b32 v19, a147
+; GISEL-NEXT: v_accvgpr_read_b32 v20, a148
+; GISEL-NEXT: v_accvgpr_read_b32 v21, a149
+; GISEL-NEXT: v_accvgpr_read_b32 v22, a150
+; GISEL-NEXT: v_accvgpr_read_b32 v23, a151
+; GISEL-NEXT: v_accvgpr_read_b32 v24, a152
+; GISEL-NEXT: v_accvgpr_read_b32 v25, a153
+; GISEL-NEXT: v_accvgpr_read_b32 v26, a154
+; GISEL-NEXT: v_accvgpr_read_b32 v27, a155
+; GISEL-NEXT: v_accvgpr_read_b32 v28, a156
+; GISEL-NEXT: v_accvgpr_read_b32 v29, a157
+; GISEL-NEXT: v_accvgpr_read_b32 v30, a158
+; GISEL-NEXT: v_accvgpr_read_b32 v31, a159
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
+; GISEL-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
+; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:32768
+; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:32784
+; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:32800
+; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:32816
+; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:32832
+; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:32848
+; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:32864
+; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:32880
+; GISEL-NEXT: s_endpgm
;
; EXACTCUTOFF-LABEL: test_sched_group_barrier_pipeline_interleave_EXP_MFMA:
; EXACTCUTOFF: ; %bb.0: ; %entry
More information about the llvm-commits
mailing list