[llvm] 5ce3273 - [AMDGPU] Scheduler: Don't revert the schedule if the register pressure isn't changed for a region
Valery Pykhtin via llvm-commits
llvm-commits at lists.llvm.org
Fri Dec 2 06:59:50 PST 2022
Author: Valery Pykhtin
Date: 2022-12-02T15:59:38+01:00
New Revision: 5ce3273ebf5be4df72b3167698336af5db80ea48
URL: https://github.com/llvm/llvm-project/commit/5ce3273ebf5be4df72b3167698336af5db80ea48
DIFF: https://github.com/llvm/llvm-project/commit/5ce3273ebf5be4df72b3167698336af5db80ea48.diff
LOG: [AMDGPU] Scheduler: Don't revert the schedule if the register pressure isn't changed for a region
This one-linear fix improves compilation time for about ~40% on ASAN enabled code.
Reviewed By: arsenm
Differential Revision: https://reviews.llvm.org/D136069
Added:
Modified:
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
llvm/test/CodeGen/AMDGPU/load-global-i16.ll
llvm/test/CodeGen/AMDGPU/machine-scheduler-sink-trivial-remats.mir
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index 29c42495d93a5..9b76122b77d8b 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -933,6 +933,9 @@ bool GCNSchedStage::shouldRevertScheduling(unsigned WavesAfter) {
}
bool OccInitialScheduleStage::shouldRevertScheduling(unsigned WavesAfter) {
+ if (PressureAfter == PressureBefore)
+ return false;
+
if (GCNSchedStage::shouldRevertScheduling(WavesAfter))
return true;
@@ -956,6 +959,9 @@ bool UnclusteredHighRPStage::shouldRevertScheduling(unsigned WavesAfter) {
}
bool ClusteredLowOccStage::shouldRevertScheduling(unsigned WavesAfter) {
+ if (PressureAfter == PressureBefore)
+ return false;
+
if (GCNSchedStage::shouldRevertScheduling(WavesAfter))
return true;
diff --git a/llvm/test/CodeGen/AMDGPU/load-global-i16.ll b/llvm/test/CodeGen/AMDGPU/load-global-i16.ll
index 8eeee8d653afb..2f3a6f01e2912 100644
--- a/llvm/test/CodeGen/AMDGPU/load-global-i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-global-i16.ll
@@ -3476,122 +3476,103 @@ define amdgpu_kernel void @global_zextload_v64i16_to_v64i32(<64 x i32> addrspace
; GCN-NOHSA-SI-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NOHSA-SI-NEXT: s_mov_b32 s8, s6
; GCN-NOHSA-SI-NEXT: s_mov_b32 s9, s7
-; GCN-NOHSA-SI-NEXT: buffer_load_dwordx4 v[12:15], off, s[8:11], 0
-; GCN-NOHSA-SI-NEXT: buffer_load_dwordx4 v[16:19], off, s[8:11], 0 offset:16
-; GCN-NOHSA-SI-NEXT: s_waitcnt vmcnt(1)
-; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v3, 16, v15
-; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v1, 16, v14
-; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v6, 16, v13
-; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v4, 16, v12
-; GCN-NOHSA-SI-NEXT: s_waitcnt vmcnt(0)
-; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v11, 16, v19
-; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v9, 16, v18
-; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v2, 0xffff, v15
-; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v0, 0xffff, v14
-; GCN-NOHSA-SI-NEXT: buffer_store_dword v0, off, s[12:15], 0 offset:4 ; 4-byte Folded Spill
-; GCN-NOHSA-SI-NEXT: s_waitcnt vmcnt(0)
-; GCN-NOHSA-SI-NEXT: buffer_store_dword v1, off, s[12:15], 0 offset:8 ; 4-byte Folded Spill
-; GCN-NOHSA-SI-NEXT: buffer_store_dword v2, off, s[12:15], 0 offset:12 ; 4-byte Folded Spill
-; GCN-NOHSA-SI-NEXT: buffer_store_dword v3, off, s[12:15], 0 offset:16 ; 4-byte Folded Spill
-; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v5, 0xffff, v13
-; GCN-NOHSA-SI-NEXT: s_waitcnt expcnt(0)
-; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v3, 0xffff, v12
-; GCN-NOHSA-SI-NEXT: buffer_store_dword v3, off, s[12:15], 0 offset:20 ; 4-byte Folded Spill
-; GCN-NOHSA-SI-NEXT: s_waitcnt vmcnt(0)
-; GCN-NOHSA-SI-NEXT: buffer_store_dword v4, off, s[12:15], 0 offset:24 ; 4-byte Folded Spill
-; GCN-NOHSA-SI-NEXT: buffer_store_dword v5, off, s[12:15], 0 offset:28 ; 4-byte Folded Spill
-; GCN-NOHSA-SI-NEXT: buffer_store_dword v6, off, s[12:15], 0 offset:32 ; 4-byte Folded Spill
-; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v15, 16, v17
-; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v13, 16, v16
-; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v10, 0xffff, v19
-; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v8, 0xffff, v18
-; GCN-NOHSA-SI-NEXT: buffer_load_dwordx4 v[18:21], off, s[8:11], 0 offset:32
-; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v14, 0xffff, v17
-; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v12, 0xffff, v16
-; GCN-NOHSA-SI-NEXT: buffer_load_dwordx4 v[22:25], off, s[8:11], 0 offset:48
-; GCN-NOHSA-SI-NEXT: s_waitcnt vmcnt(1)
-; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v29, 16, v21
-; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v27, 16, v20
-; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v33, 16, v19
-; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v31, 16, v18
-; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v28, 0xffff, v21
-; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v26, 0xffff, v20
-; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v32, 0xffff, v19
-; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v30, 0xffff, v18
-; GCN-NOHSA-SI-NEXT: s_waitcnt vmcnt(0)
-; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v19, 16, v25
-; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v17, 16, v24
-; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v37, 16, v23
-; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v35, 16, v22
-; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v18, 0xffff, v25
-; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v16, 0xffff, v24
-; GCN-NOHSA-SI-NEXT: buffer_load_dwordx4 v[38:41], off, s[8:11], 0 offset:64
-; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v36, 0xffff, v23
-; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v34, 0xffff, v22
+; GCN-NOHSA-SI-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0
+; GCN-NOHSA-SI-NEXT: buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:16
+; GCN-NOHSA-SI-NEXT: buffer_load_dwordx4 v[8:11], off, s[8:11], 0 offset:32
+; GCN-NOHSA-SI-NEXT: buffer_load_dwordx4 v[12:15], off, s[8:11], 0 offset:48
+; GCN-NOHSA-SI-NEXT: buffer_load_dwordx4 v[16:19], off, s[8:11], 0 offset:64
; GCN-NOHSA-SI-NEXT: buffer_load_dwordx4 v[20:23], off, s[8:11], 0 offset:80
-; GCN-NOHSA-SI-NEXT: s_waitcnt vmcnt(1)
-; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v44, 16, v41
-; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v42, 16, v40
-; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v48, 16, v39
-; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v46, 16, v38
-; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v43, 0xffff, v41
-; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v41, 0xffff, v40
-; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v47, 0xffff, v39
-; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v45, 0xffff, v38
+; GCN-NOHSA-SI-NEXT: buffer_load_dwordx4 v[24:27], off, s[8:11], 0 offset:96
+; GCN-NOHSA-SI-NEXT: buffer_load_dwordx4 v[28:31], off, s[8:11], 0 offset:112
+; GCN-NOHSA-SI-NEXT: s_waitcnt vmcnt(7)
+; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v35, 16, v3
+; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v33, 16, v2
+; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v39, 16, v1
+; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v37, 16, v0
+; GCN-NOHSA-SI-NEXT: s_waitcnt vmcnt(6)
+; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v43, 16, v7
+; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v41, 16, v6
+; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v34, 0xffff, v3
+; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v32, 0xffff, v2
+; GCN-NOHSA-SI-NEXT: buffer_store_dword v32, off, s[12:15], 0 offset:4 ; 4-byte Folded Spill
; GCN-NOHSA-SI-NEXT: s_waitcnt vmcnt(0)
-; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v52, 16, v23
-; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v50, 16, v22
-; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v56, 16, v21
-; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v54, 16, v20
-; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v51, 0xffff, v23
-; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v49, 0xffff, v22
-; GCN-NOHSA-SI-NEXT: buffer_load_dwordx4 v[22:25], off, s[8:11], 0 offset:96
-; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v55, 0xffff, v21
-; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v53, 0xffff, v20
-; GCN-NOHSA-SI-NEXT: buffer_load_dwordx4 v[57:60], off, s[8:11], 0 offset:112
-; GCN-NOHSA-SI-NEXT: s_waitcnt vmcnt(1)
-; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v7, 16, v25
-; GCN-NOHSA-SI-NEXT: s_waitcnt expcnt(1)
-; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v5, 16, v24
-; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v3, 16, v23
-; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v1, 16, v22
+; GCN-NOHSA-SI-NEXT: buffer_store_dword v33, off, s[12:15], 0 offset:8 ; 4-byte Folded Spill
+; GCN-NOHSA-SI-NEXT: buffer_store_dword v34, off, s[12:15], 0 offset:12 ; 4-byte Folded Spill
+; GCN-NOHSA-SI-NEXT: buffer_store_dword v35, off, s[12:15], 0 offset:16 ; 4-byte Folded Spill
+; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v38, 0xffff, v1
+; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v36, 0xffff, v0
; GCN-NOHSA-SI-NEXT: s_waitcnt expcnt(0)
-; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v6, 0xffff, v25
-; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v4, 0xffff, v24
-; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v2, 0xffff, v23
-; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v0, 0xffff, v22
-; GCN-NOHSA-SI-NEXT: s_waitcnt vmcnt(0)
-; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v23, 16, v60
-; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v21, 16, v59
-; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v63, 16, v58
-; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v61, 16, v57
-; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v22, 0xffff, v60
-; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v20, 0xffff, v59
-; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v62, 0xffff, v58
-; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v60, 0xffff, v57
+; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v35, 16, v5
+; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v33, 16, v4
+; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v42, 0xffff, v7
+; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v40, 0xffff, v6
+; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v34, 0xffff, v5
+; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v32, 0xffff, v4
+; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v7, 16, v11
+; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v5, 16, v10
+; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v47, 16, v9
+; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v45, 16, v8
+; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v6, 0xffff, v11
+; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v4, 0xffff, v10
+; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v46, 0xffff, v9
+; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v44, 0xffff, v8
+; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v11, 16, v15
+; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v9, 16, v14
+; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v51, 16, v13
+; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v49, 16, v12
+; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v10, 0xffff, v15
+; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v8, 0xffff, v14
+; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v50, 0xffff, v13
+; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v48, 0xffff, v12
+; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v15, 16, v19
+; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v13, 16, v18
+; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v55, 16, v17
+; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v53, 16, v16
+; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v14, 0xffff, v19
+; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v12, 0xffff, v18
+; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v54, 0xffff, v17
+; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v52, 0xffff, v16
+; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v19, 16, v23
+; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v17, 16, v22
+; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v59, 16, v21
+; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v57, 16, v20
+; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v18, 0xffff, v23
+; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v16, 0xffff, v22
+; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v58, 0xffff, v21
+; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v56, 0xffff, v20
+; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v23, 16, v27
+; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v21, 16, v26
+; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v63, 16, v25
+; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v61, 16, v24
+; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v22, 0xffff, v27
+; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v20, 0xffff, v26
+; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v62, 0xffff, v25
+; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v60, 0xffff, v24
+; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v27, 16, v31
+; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v25, 16, v30
+; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v3, 16, v29
+; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v1, 16, v28
+; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v26, 0xffff, v31
+; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v24, 0xffff, v30
+; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v2, 0xffff, v29
+; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v0, 0xffff, v28
; GCN-NOHSA-SI-NEXT: s_mov_b32 s0, s4
; GCN-NOHSA-SI-NEXT: s_mov_b32 s1, s5
-; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[60:63], off, s[0:3], 0 offset:224
-; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[20:23], off, s[0:3], 0 offset:240
-; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:192
-; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:208
-; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[53:56], off, s[0:3], 0 offset:160
-; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[49:52], off, s[0:3], 0 offset:176
-; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[45:48], off, s[0:3], 0 offset:128
-; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[41:44], off, s[0:3], 0 offset:144
-; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[34:37], off, s[0:3], 0 offset:96
-; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:112
-; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[30:33], off, s[0:3], 0 offset:64
-; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[26:29], off, s[0:3], 0 offset:80
-; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:32
-; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:48
-; GCN-NOHSA-SI-NEXT: buffer_load_dword v0, off, s[12:15], 0 offset:20 ; 4-byte Folded Reload
-; GCN-NOHSA-SI-NEXT: buffer_load_dword v1, off, s[12:15], 0 offset:24 ; 4-byte Folded Reload
-; GCN-NOHSA-SI-NEXT: buffer_load_dword v2, off, s[12:15], 0 offset:28 ; 4-byte Folded Reload
-; GCN-NOHSA-SI-NEXT: buffer_load_dword v3, off, s[12:15], 0 offset:32 ; 4-byte Folded Reload
-; GCN-NOHSA-SI-NEXT: s_waitcnt vmcnt(0)
-; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
-; GCN-NOHSA-SI-NEXT: s_waitcnt expcnt(0)
+; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:224
+; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[24:27], off, s[0:3], 0 offset:240
+; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[60:63], off, s[0:3], 0 offset:192
+; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[20:23], off, s[0:3], 0 offset:208
+; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[56:59], off, s[0:3], 0 offset:160
+; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:176
+; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[52:55], off, s[0:3], 0 offset:128
+; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:144
+; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[48:51], off, s[0:3], 0 offset:96
+; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:112
+; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[44:47], off, s[0:3], 0 offset:64
+; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:80
+; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[32:35], off, s[0:3], 0 offset:32
+; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[40:43], off, s[0:3], 0 offset:48
+; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[36:39], off, s[0:3], 0
; GCN-NOHSA-SI-NEXT: buffer_load_dword v0, off, s[12:15], 0 offset:4 ; 4-byte Folded Reload
; GCN-NOHSA-SI-NEXT: buffer_load_dword v1, off, s[12:15], 0 offset:8 ; 4-byte Folded Reload
; GCN-NOHSA-SI-NEXT: buffer_load_dword v2, off, s[12:15], 0 offset:12 ; 4-byte Folded Reload
@@ -3808,108 +3789,102 @@ define amdgpu_kernel void @global_zextload_v64i16_to_v64i32(<64 x i32> addrspace
; GCN-NOHSA-VI-NEXT: s_mov_b32 s8, s6
; GCN-NOHSA-VI-NEXT: s_mov_b32 s9, s7
; GCN-NOHSA-VI-NEXT: s_mov_b32 s11, s3
-; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[12:15], off, s[8:11], 0
-; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[16:19], off, s[8:11], 0 offset:16
+; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:16
+; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[8:11], off, s[8:11], 0 offset:32
+; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[12:15], off, s[8:11], 0 offset:48
+; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[16:19], off, s[8:11], 0 offset:64
+; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[20:23], off, s[8:11], 0 offset:80
+; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[24:27], off, s[8:11], 0 offset:96
+; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[28:31], off, s[8:11], 0 offset:112
; GCN-NOHSA-VI-NEXT: s_addc_u32 s89, s89, 0
; GCN-NOHSA-VI-NEXT: s_mov_b32 s0, s4
; GCN-NOHSA-VI-NEXT: s_mov_b32 s1, s5
-; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(1)
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v3, 16, v15
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v1, 16, v14
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v2, 0xffff, v15
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v0, 0xffff, v14
-; GCN-NOHSA-VI-NEXT: buffer_store_dword v0, off, s[88:91], 0 offset:4 ; 4-byte Folded Spill
-; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(0)
-; GCN-NOHSA-VI-NEXT: buffer_store_dword v1, off, s[88:91], 0 offset:8 ; 4-byte Folded Spill
-; GCN-NOHSA-VI-NEXT: buffer_store_dword v2, off, s[88:91], 0 offset:12 ; 4-byte Folded Spill
-; GCN-NOHSA-VI-NEXT: buffer_store_dword v3, off, s[88:91], 0 offset:16 ; 4-byte Folded Spill
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v7, 16, v13
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v5, 16, v12
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v11, 16, v19
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v9, 16, v18
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v6, 0xffff, v13
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v4, 0xffff, v12
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v15, 16, v17
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v13, 16, v16
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v10, 0xffff, v19
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v8, 0xffff, v18
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v14, 0xffff, v17
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v12, 0xffff, v16
-; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[16:19], off, s[8:11], 0 offset:32
-; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[20:23], off, s[8:11], 0 offset:48
-; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(1)
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v27, 16, v19
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v25, 16, v18
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v31, 16, v17
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v29, 16, v16
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v26, 0xffff, v19
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v24, 0xffff, v18
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v30, 0xffff, v17
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v28, 0xffff, v16
+; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(7)
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v35, 16, v3
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v33, 16, v2
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v34, 0xffff, v3
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v32, 0xffff, v2
+; GCN-NOHSA-VI-NEXT: buffer_store_dword v32, off, s[88:91], 0 offset:4 ; 4-byte Folded Spill
; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT: buffer_store_dword v33, off, s[88:91], 0 offset:8 ; 4-byte Folded Spill
+; GCN-NOHSA-VI-NEXT: buffer_store_dword v34, off, s[88:91], 0 offset:12 ; 4-byte Folded Spill
+; GCN-NOHSA-VI-NEXT: buffer_store_dword v35, off, s[88:91], 0 offset:16 ; 4-byte Folded Spill
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v39, 16, v1
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v37, 16, v0
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v38, 0xffff, v1
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v36, 0xffff, v0
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v3, 16, v29
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v1, 16, v28
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v2, 0xffff, v29
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v0, 0xffff, v28
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v43, 16, v7
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v41, 16, v6
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v35, 16, v5
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v33, 16, v4
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v42, 0xffff, v7
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v40, 0xffff, v6
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v34, 0xffff, v5
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v32, 0xffff, v4
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v7, 16, v11
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v5, 16, v10
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v47, 16, v9
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v45, 16, v8
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v6, 0xffff, v11
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v4, 0xffff, v10
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v46, 0xffff, v9
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v44, 0xffff, v8
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v11, 16, v15
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v9, 16, v14
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v51, 16, v13
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v49, 16, v12
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v10, 0xffff, v15
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v8, 0xffff, v14
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v50, 0xffff, v13
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v48, 0xffff, v12
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v15, 16, v19
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v13, 16, v18
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v55, 16, v17
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v53, 16, v16
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v14, 0xffff, v19
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v12, 0xffff, v18
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v54, 0xffff, v17
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v52, 0xffff, v16
; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v19, 16, v23
; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v17, 16, v22
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v35, 16, v21
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v33, 16, v20
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v59, 16, v21
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v57, 16, v20
; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v18, 0xffff, v23
; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v16, 0xffff, v22
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v34, 0xffff, v21
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v32, 0xffff, v20
-; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[20:23], off, s[8:11], 0 offset:64
-; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[36:39], off, s[8:11], 0 offset:80
-; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(1)
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v43, 16, v23
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v41, 16, v22
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v47, 16, v21
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v45, 16, v20
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v42, 0xffff, v23
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v40, 0xffff, v22
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v46, 0xffff, v21
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v44, 0xffff, v20
-; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(0)
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v23, 16, v39
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v21, 16, v38
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v51, 16, v37
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v49, 16, v36
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v22, 0xffff, v39
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v20, 0xffff, v38
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v50, 0xffff, v37
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v48, 0xffff, v36
-; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[36:39], off, s[8:11], 0 offset:96
-; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[52:55], off, s[8:11], 0 offset:112
-; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(1)
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v59, 16, v39
-; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(0)
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v3, 16, v53
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v1, 16, v52
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v2, 0xffff, v53
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v0, 0xffff, v52
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v57, 16, v38
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v63, 16, v37
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v61, 16, v36
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v58, 0xffff, v39
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v56, 0xffff, v38
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v62, 0xffff, v37
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v60, 0xffff, v36
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v39, 16, v55
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v37, 16, v54
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v38, 0xffff, v55
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v36, 0xffff, v54
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v58, 0xffff, v21
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v56, 0xffff, v20
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v23, 16, v27
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v21, 16, v26
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v63, 16, v25
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v61, 16, v24
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v22, 0xffff, v27
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v20, 0xffff, v26
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v62, 0xffff, v25
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v60, 0xffff, v24
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v27, 16, v31
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v25, 16, v30
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v26, 0xffff, v31
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v24, 0xffff, v30
; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:224
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[36:39], off, s[0:3], 0 offset:240
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[24:27], off, s[0:3], 0 offset:240
; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[60:63], off, s[0:3], 0 offset:192
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[56:59], off, s[0:3], 0 offset:208
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[48:51], off, s[0:3], 0 offset:160
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[20:23], off, s[0:3], 0 offset:176
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[44:47], off, s[0:3], 0 offset:128
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[40:43], off, s[0:3], 0 offset:144
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[32:35], off, s[0:3], 0 offset:96
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:112
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[28:31], off, s[0:3], 0 offset:64
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[24:27], off, s[0:3], 0 offset:80
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:32
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:48
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[20:23], off, s[0:3], 0 offset:208
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[56:59], off, s[0:3], 0 offset:160
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:176
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[52:55], off, s[0:3], 0 offset:128
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:144
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[48:51], off, s[0:3], 0 offset:96
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:112
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[44:47], off, s[0:3], 0 offset:64
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:80
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[32:35], off, s[0:3], 0 offset:32
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[40:43], off, s[0:3], 0 offset:48
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[36:39], off, s[0:3], 0
; GCN-NOHSA-VI-NEXT: buffer_load_dword v0, off, s[88:91], 0 offset:4 ; 4-byte Folded Reload
; GCN-NOHSA-VI-NEXT: buffer_load_dword v1, off, s[88:91], 0 offset:8 ; 4-byte Folded Reload
; GCN-NOHSA-VI-NEXT: buffer_load_dword v2, off, s[88:91], 0 offset:12 ; 4-byte Folded Reload
@@ -4601,110 +4576,107 @@ define amdgpu_kernel void @global_sextload_v64i16_to_v64i32(<64 x i32> addrspace
; GCN-NOHSA-VI-NEXT: s_add_u32 s88, s88, s3
; GCN-NOHSA-VI-NEXT: s_mov_b32 s3, 0xf000
; GCN-NOHSA-VI-NEXT: s_mov_b32 s2, -1
+; GCN-NOHSA-VI-NEXT: s_mov_b32 s10, s2
; GCN-NOHSA-VI-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NOHSA-VI-NEXT: s_mov_b32 s8, s6
+; GCN-NOHSA-VI-NEXT: s_mov_b32 s9, s7
+; GCN-NOHSA-VI-NEXT: s_mov_b32 s11, s3
+; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:16
+; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[8:11], off, s[8:11], 0 offset:32
+; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[12:15], off, s[8:11], 0 offset:48
+; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[16:19], off, s[8:11], 0 offset:64
+; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[20:23], off, s[8:11], 0 offset:80
+; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[24:27], off, s[8:11], 0 offset:96
+; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[28:31], off, s[8:11], 0 offset:112
+; GCN-NOHSA-VI-NEXT: s_addc_u32 s89, s89, 0
; GCN-NOHSA-VI-NEXT: s_mov_b32 s0, s4
; GCN-NOHSA-VI-NEXT: s_mov_b32 s1, s5
-; GCN-NOHSA-VI-NEXT: s_mov_b32 s4, s6
-; GCN-NOHSA-VI-NEXT: s_mov_b32 s5, s7
-; GCN-NOHSA-VI-NEXT: s_mov_b32 s6, s2
-; GCN-NOHSA-VI-NEXT: s_mov_b32 s7, s3
-; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[8:11], off, s[4:7], 0 offset:112
-; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[12:15], off, s[4:7], 0 offset:96
-; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[16:19], off, s[4:7], 0 offset:80
-; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[20:23], off, s[4:7], 0 offset:64
-; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[24:27], off, s[4:7], 0
-; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[28:31], off, s[4:7], 0 offset:16
-; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[32:35], off, s[4:7], 0 offset:32
-; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[36:39], off, s[4:7], 0 offset:48
-; GCN-NOHSA-VI-NEXT: s_addc_u32 s89, s89, 0
-; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(6)
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v63, 16, v13
-; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(5)
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v59, 16, v17
-; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(4)
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v55, 16, v21
-; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(3)
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v3, 16, v27
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v1, 16, v26
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v2, v27, 0, 16
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v0, v26, 0, 16
-; GCN-NOHSA-VI-NEXT: buffer_store_dword v0, off, s[88:91], 0 offset:4 ; 4-byte Folded Spill
+; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(7)
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v35, 16, v3
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v33, 16, v2
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v34, v3, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v32, v2, 0, 16
+; GCN-NOHSA-VI-NEXT: buffer_store_dword v32, off, s[88:91], 0 offset:4 ; 4-byte Folded Spill
; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(0)
-; GCN-NOHSA-VI-NEXT: buffer_store_dword v1, off, s[88:91], 0 offset:8 ; 4-byte Folded Spill
-; GCN-NOHSA-VI-NEXT: buffer_store_dword v2, off, s[88:91], 0 offset:12 ; 4-byte Folded Spill
-; GCN-NOHSA-VI-NEXT: buffer_store_dword v3, off, s[88:91], 0 offset:16 ; 4-byte Folded Spill
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v3, 16, v9
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v1, 16, v8
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v2, v9, 0, 16
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v0, v8, 0, 16
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v7, 16, v25
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v5, 16, v24
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v6, v25, 0, 16
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v4, v24, 0, 16
+; GCN-NOHSA-VI-NEXT: buffer_store_dword v33, off, s[88:91], 0 offset:8 ; 4-byte Folded Spill
+; GCN-NOHSA-VI-NEXT: buffer_store_dword v34, off, s[88:91], 0 offset:12 ; 4-byte Folded Spill
+; GCN-NOHSA-VI-NEXT: buffer_store_dword v35, off, s[88:91], 0 offset:16 ; 4-byte Folded Spill
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v39, 16, v1
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v37, 16, v0
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v38, v1, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v36, v0, 0, 16
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v3, 16, v29
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v1, 16, v28
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v2, v29, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v0, v28, 0, 16
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v35, 16, v7
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v33, 16, v6
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v34, v7, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v32, v6, 0, 16
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v43, 16, v5
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v41, 16, v4
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v42, v5, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v40, v4, 0, 16
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v7, 16, v11
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v5, 16, v10
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v6, v11, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v4, v10, 0, 16
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v47, 16, v9
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v45, 16, v8
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v46, v9, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v44, v8, 0, 16
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v11, 16, v15
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v9, 16, v14
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v10, v15, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v8, v14, 0, 16
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v51, 16, v13
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v49, 16, v12
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v50, v13, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v48, v12, 0, 16
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v15, 16, v19
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v13, 16, v18
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v14, v19, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v12, v18, 0, 16
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v55, 16, v17
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v53, 16, v16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v54, v17, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v52, v16, 0, 16
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v19, 16, v23
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v17, 16, v22
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v18, v23, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v16, v22, 0, 16
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v59, 16, v21
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v57, 16, v20
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v58, v21, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v56, v20, 0, 16
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v23, 16, v27
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v21, 16, v26
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v22, v27, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v20, v26, 0, 16
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v63, 16, v25
+; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v61, 16, v24
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v62, v25, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v60, v24, 0, 16
; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v27, 16, v31
; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v25, 16, v30
; GCN-NOHSA-VI-NEXT: v_bfe_i32 v26, v31, 0, 16
; GCN-NOHSA-VI-NEXT: v_bfe_i32 v24, v30, 0, 16
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v43, 16, v29
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v41, 16, v28
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v42, v29, 0, 16
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v40, v28, 0, 16
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v31, 16, v35
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v29, 16, v34
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v30, v35, 0, 16
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v28, v34, 0, 16
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v47, 16, v33
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v45, 16, v32
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v46, v33, 0, 16
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v44, v32, 0, 16
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v35, 16, v39
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v33, 16, v38
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v34, v39, 0, 16
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v32, v38, 0, 16
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v51, 16, v37
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v49, 16, v36
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v50, v37, 0, 16
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v48, v36, 0, 16
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v39, 16, v23
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v37, 16, v22
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v38, v23, 0, 16
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v36, v22, 0, 16
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v53, 16, v20
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v54, v21, 0, 16
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v52, v20, 0, 16
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v22, 16, v19
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v20, 16, v18
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v21, v19, 0, 16
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v19, v18, 0, 16
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v57, 16, v16
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v58, v17, 0, 16
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v56, v16, 0, 16
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v18, 16, v15
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v16, 16, v14
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v17, v15, 0, 16
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v15, v14, 0, 16
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v61, 16, v12
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v62, v13, 0, 16
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v60, v12, 0, 16
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v14, 16, v11
-; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v12, 16, v10
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v13, v11, 0, 16
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v11, v10, 0, 16
; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:224
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[11:14], off, s[0:3], 0 offset:240
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[24:27], off, s[0:3], 0 offset:240
; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[60:63], off, s[0:3], 0 offset:192
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[15:18], off, s[0:3], 0 offset:208
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[20:23], off, s[0:3], 0 offset:208
; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[56:59], off, s[0:3], 0 offset:160
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[19:22], off, s[0:3], 0 offset:176
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:176
; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[52:55], off, s[0:3], 0 offset:128
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[36:39], off, s[0:3], 0 offset:144
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:144
; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[48:51], off, s[0:3], 0 offset:96
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[32:35], off, s[0:3], 0 offset:112
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:112
; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[44:47], off, s[0:3], 0 offset:64
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[28:31], off, s[0:3], 0 offset:80
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:80
; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[40:43], off, s[0:3], 0 offset:32
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[24:27], off, s[0:3], 0 offset:48
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[32:35], off, s[0:3], 0 offset:48
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[36:39], off, s[0:3], 0
; GCN-NOHSA-VI-NEXT: buffer_load_dword v0, off, s[88:91], 0 offset:4 ; 4-byte Folded Reload
; GCN-NOHSA-VI-NEXT: buffer_load_dword v1, off, s[88:91], 0 offset:8 ; 4-byte Folded Reload
; GCN-NOHSA-VI-NEXT: buffer_load_dword v2, off, s[88:91], 0 offset:12 ; 4-byte Folded Reload
diff --git a/llvm/test/CodeGen/AMDGPU/machine-scheduler-sink-trivial-remats.mir b/llvm/test/CodeGen/AMDGPU/machine-scheduler-sink-trivial-remats.mir
index 7f739bd9243de..1a1765ff6bee0 100644
--- a/llvm/test/CodeGen/AMDGPU/machine-scheduler-sink-trivial-remats.mir
+++ b/llvm/test/CodeGen/AMDGPU/machine-scheduler-sink-trivial-remats.mir
@@ -4146,10 +4146,10 @@ body: |
; GFX908-NEXT: [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr0_sgpr1
; GFX908-NEXT: [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
; GFX908-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]](p4), 52, 0 :: (dereferenceable invariant load (s64), align 4, addrspace 4)
- ; GFX908-NEXT: [[V_CMP_GT_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_GT_U32_e64 [[S_LOAD_DWORDX2_IMM]].sub0, [[COPY1]](s32), implicit $exec
; GFX908-NEXT: undef %4.sub1:sreg_64 = S_MOV_B32 0
- ; GFX908-NEXT: undef %4.sub0:sreg_64 = COPY [[S_LOAD_DWORDX2_IMM]].sub1
; GFX908-NEXT: %5:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 0, implicit $exec, implicit $mode, implicit-def $m0
+ ; GFX908-NEXT: [[V_CMP_GT_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_GT_U32_e64 [[S_LOAD_DWORDX2_IMM]].sub0, [[COPY1]](s32), implicit $exec
+ ; GFX908-NEXT: undef %4.sub0:sreg_64 = COPY [[S_LOAD_DWORDX2_IMM]].sub1
; GFX908-NEXT: %6:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 1, implicit $exec, implicit $mode, implicit-def $m0
; GFX908-NEXT: %7:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 2, implicit $exec, implicit $mode, implicit-def $m0
; GFX908-NEXT: %8:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 3, implicit $exec, implicit $mode, implicit-def $m0
@@ -4488,10 +4488,10 @@ body: |
; GFX908-NEXT: [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr0_sgpr1
; GFX908-NEXT: [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
; GFX908-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]](p4), 52, 0 :: (dereferenceable invariant load (s64), align 4, addrspace 4)
- ; GFX908-NEXT: [[V_CMP_GT_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_GT_U32_e64 [[S_LOAD_DWORDX2_IMM]].sub0, [[COPY1]](s32), implicit $exec
; GFX908-NEXT: undef %4.sub1:sreg_64 = S_MOV_B32 0
- ; GFX908-NEXT: undef %4.sub0:sreg_64 = COPY [[S_LOAD_DWORDX2_IMM]].sub1
; GFX908-NEXT: %5:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 0, implicit $exec, implicit $mode, implicit-def $m0
+ ; GFX908-NEXT: [[V_CMP_GT_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_GT_U32_e64 [[S_LOAD_DWORDX2_IMM]].sub0, [[COPY1]](s32), implicit $exec
+ ; GFX908-NEXT: undef %4.sub0:sreg_64 = COPY [[S_LOAD_DWORDX2_IMM]].sub1
; GFX908-NEXT: %6:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 1, implicit $exec, implicit $mode, implicit-def $m0
; GFX908-NEXT: %7:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 2, implicit $exec, implicit $mode, implicit-def $m0
; GFX908-NEXT: %8:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 3, implicit $exec, implicit $mode, implicit-def $m0
More information about the llvm-commits
mailing list