[llvm-branch-commits] [llvm] GlobalISel: Use extended LLTs in lshr narrow combine (PR #210298)
Petar Avramovic via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Mon Jul 20 05:45:27 PDT 2026
https://github.com/petar-avramovic updated https://github.com/llvm/llvm-project/pull/210298
>From 660b489f3d1640ef6fd4b29562a92b85f5e63057 Mon Sep 17 00:00:00 2001
From: Petar Avramovic <Petar.Avramovic at amd.com>
Date: Fri, 17 Jul 2026 12:42:15 +0200
Subject: [PATCH] GlobalISel: Use extended LLTs in lshr narrow combine
---
.../lib/CodeGen/GlobalISel/CombinerHelper.cpp | 2 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll | 259 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll | 4379 +++++++++-------
llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll | 4478 ++++++++++-------
.../CodeGen/AMDGPU/GlobalISel/llvm.abs.ll | 16 +-
.../llvm.amdgcn.make.buffer.rsrc.ll | 225 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/lshr.ll | 86 +-
.../AMDGPU/GlobalISel/minmaxabs-i64.ll | 14 +-
.../CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll | 371 +-
.../CodeGen/AMDGPU/GlobalISel/sext_inreg.ll | 176 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/shl.ll | 23 +-
.../CodeGen/AMDGPU/GlobalISel/srem.i64.ll | 310 +-
llvm/test/CodeGen/AMDGPU/div_i128.ll | 3867 +++++++-------
llvm/test/CodeGen/AMDGPU/div_v2i128.ll | 2741 +++++-----
llvm/test/CodeGen/AMDGPU/fptoi.i128.ll | 290 +-
llvm/test/CodeGen/AMDGPU/itofp.i128.ll | 2413 ++++++---
16 files changed, 11113 insertions(+), 8537 deletions(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index b30caed0af8a3..a9a203af200ed 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -2579,7 +2579,7 @@ void CombinerHelper::applyCombineShiftToUnmerge(
unsigned HalfSize = Size / 2;
assert(ShiftVal >= HalfSize);
- LLT HalfTy = LLT::scalar(HalfSize);
+ LLT HalfTy = Ty.changeElementSize(HalfSize);
auto Unmerge = Builder.buildUnmerge(HalfTy, SrcReg);
unsigned NarrowShiftAmt = ShiftVal - HalfSize;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll
index af02638e03189..5a8027bf75881 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll
@@ -1,10 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu6.00-amd-amdpal < %s | FileCheck -check-prefixes=GCN,GFX6 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-amd-amdpal < %s | FileCheck -check-prefixes=GCN,GFX8 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00-amd-amdpal < %s | FileCheck -check-prefixes=GCN,GFX9 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-amd-amdpal < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-TRUE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-FAKE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu6.00-amd-amdpal < %s | FileCheck -check-prefixes=GCN,GFX6 %s
+; RUN: llc -global-isel -mtriple=amdgpu8.03-amd-amdpal < %s | FileCheck -check-prefixes=GCN,GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00-amd-amdpal < %s | FileCheck -check-prefixes=GCN,GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgpu10.10-amd-amdpal < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-TRUE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-FAKE16 %s
define i8 @v_ashr_i8(i8 %value, i8 %amount) {
; GFX6-LABEL: v_ashr_i8:
@@ -1750,15 +1750,19 @@ define i65 @v_ashr_i65(i65 %value, i65 %amount) {
; GFX6-NEXT: v_sub_i32_e32 v8, vcc, 64, v3
; GFX6-NEXT: v_lshr_b64 v[6:7], v[0:1], v3
; GFX6-NEXT: v_lshl_b64 v[8:9], v[4:5], v8
-; GFX6-NEXT: v_cmp_gt_u32_e32 vcc, 64, v3
-; GFX6-NEXT: v_or_b32_e32 v7, v7, v9
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 0xffffffc0, v3
+; GFX6-NEXT: v_ashr_i64 v[10:11], v[4:5], v3
; GFX6-NEXT: v_or_b32_e32 v6, v6, v8
-; GFX6-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc
+; GFX6-NEXT: v_ashrrev_i32_e32 v8, 31, v5
+; GFX6-NEXT: v_ashr_i64 v[4:5], v[4:5], v2
+; GFX6-NEXT: v_or_b32_e32 v7, v7, v9
+; GFX6-NEXT: v_cmp_gt_u32_e32 vcc, 64, v3
+; GFX6-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v4, v5, v7, vcc
; GFX6-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v3
-; GFX6-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc
-; GFX6-NEXT: v_and_b32_e32 v2, 1, v2
-; GFX6-NEXT: v_cndmask_b32_e64 v0, v4, v0, s[4:5]
-; GFX6-NEXT: v_cndmask_b32_e64 v1, v3, v1, s[4:5]
+; GFX6-NEXT: v_cndmask_b32_e64 v0, v2, v0, s[4:5]
+; GFX6-NEXT: v_cndmask_b32_e64 v1, v4, v1, s[4:5]
+; GFX6-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_ashr_i65:
@@ -1769,15 +1773,19 @@ define i65 @v_ashr_i65(i65 %value, i65 %amount) {
; GFX8-NEXT: v_sub_u32_e32 v8, vcc, 64, v3
; GFX8-NEXT: v_lshrrev_b64 v[6:7], v3, v[0:1]
; GFX8-NEXT: v_lshlrev_b64 v[8:9], v8, v[4:5]
-; GFX8-NEXT: v_cmp_gt_u32_e32 vcc, 64, v3
-; GFX8-NEXT: v_or_b32_e32 v7, v7, v9
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 0xffffffc0, v3
+; GFX8-NEXT: v_ashrrev_i64 v[10:11], v3, v[4:5]
; GFX8-NEXT: v_or_b32_e32 v6, v6, v8
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc
+; GFX8-NEXT: v_ashrrev_i32_e32 v8, 31, v5
+; GFX8-NEXT: v_ashrrev_i64 v[4:5], v2, v[4:5]
+; GFX8-NEXT: v_or_b32_e32 v7, v7, v9
+; GFX8-NEXT: v_cmp_gt_u32_e32 vcc, 64, v3
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v4, v5, v7, vcc
; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc
-; GFX8-NEXT: v_and_b32_e32 v2, 1, v2
-; GFX8-NEXT: v_cndmask_b32_e64 v0, v4, v0, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e64 v1, v3, v1, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v2, v0, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e64 v1, v4, v1, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_ashr_i65:
@@ -1788,51 +1796,64 @@ define i65 @v_ashr_i65(i65 %value, i65 %amount) {
; GFX9-NEXT: v_sub_u32_e32 v8, 64, v3
; GFX9-NEXT: v_lshrrev_b64 v[6:7], v3, v[0:1]
; GFX9-NEXT: v_lshlrev_b64 v[8:9], v8, v[4:5]
-; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, 64, v3
-; GFX9-NEXT: v_or_b32_e32 v7, v7, v9
+; GFX9-NEXT: v_add_u32_e32 v2, 0xffffffc0, v3
+; GFX9-NEXT: v_ashrrev_i64 v[10:11], v3, v[4:5]
; GFX9-NEXT: v_or_b32_e32 v6, v6, v8
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc
+; GFX9-NEXT: v_ashrrev_i32_e32 v8, 31, v5
+; GFX9-NEXT: v_ashrrev_i64 v[4:5], v2, v[4:5]
+; GFX9-NEXT: v_or_b32_e32 v7, v7, v9
+; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, 64, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v7, vcc
; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v3
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc
-; GFX9-NEXT: v_and_b32_e32 v2, 1, v2
-; GFX9-NEXT: v_cndmask_b32_e64 v0, v4, v0, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_e64 v1, v3, v1, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e64 v0, v2, v0, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e64 v1, v4, v1, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_ashr_i65:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_bfe_i32 v4, v2, 0, 1
-; GFX10-NEXT: v_sub_nc_u32_e32 v8, 64, v3
+; GFX10-NEXT: v_sub_nc_u32_e32 v2, 64, v3
+; GFX10-NEXT: v_add_nc_u32_e32 v10, 0xffffffc0, v3
; GFX10-NEXT: v_lshrrev_b64 v[6:7], v3, v[0:1]
; GFX10-NEXT: v_cmp_gt_u32_e32 vcc_lo, 64, v3
-; GFX10-NEXT: v_and_b32_e32 v2, 1, v2
; GFX10-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX10-NEXT: v_lshlrev_b64 v[8:9], v8, v[4:5]
-; GFX10-NEXT: v_or_b32_e32 v7, v7, v9
-; GFX10-NEXT: v_or_b32_e32 v6, v6, v8
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo
+; GFX10-NEXT: v_cmp_eq_u32_e64 s4, 0, v3
+; GFX10-NEXT: v_lshlrev_b64 v[8:9], v2, v[4:5]
+; GFX10-NEXT: v_ashrrev_i64 v[10:11], v10, v[4:5]
+; GFX10-NEXT: v_or_b32_e32 v2, v6, v8
+; GFX10-NEXT: v_or_b32_e32 v8, v7, v9
+; GFX10-NEXT: v_ashrrev_i64 v[6:7], v3, v[4:5]
+; GFX10-NEXT: v_ashrrev_i32_e32 v4, 31, v5
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v10, v2, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e32 v3, v11, v8, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v2, v0, s4
+; GFX10-NEXT: v_cndmask_b32_e64 v1, v3, v1, s4
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_ashr_i65:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_bfe_i32 v4, v2, 0, 1
-; GFX11-NEXT: v_sub_nc_u32_e32 v8, 64, v3
+; GFX11-NEXT: v_sub_nc_u32_e32 v2, 64, v3
; GFX11-NEXT: v_lshrrev_b64 v[6:7], v3, v[0:1]
; GFX11-NEXT: v_cmp_gt_u32_e32 vcc_lo, 64, v3
-; GFX11-NEXT: v_and_b32_e32 v2, 1, v2
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 0, v3
; GFX11-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX11-NEXT: v_lshlrev_b64 v[8:9], v8, v[4:5]
-; GFX11-NEXT: v_or_b32_e32 v7, v7, v9
-; GFX11-NEXT: v_or_b32_e32 v6, v6, v8
-; GFX11-NEXT: v_dual_cndmask_b32 v5, v5, v7 :: v_dual_cndmask_b32 v4, v4, v6
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX11-NEXT: v_dual_cndmask_b32 v0, v4, v0 :: v_dual_cndmask_b32 v1, v5, v1
+; GFX11-NEXT: v_lshlrev_b64 v[8:9], v2, v[4:5]
+; GFX11-NEXT: v_or_b32_e32 v2, v6, v8
+; GFX11-NEXT: v_or_b32_e32 v8, v7, v9
+; GFX11-NEXT: v_add_nc_u32_e32 v10, 0xffffffc0, v3
+; GFX11-NEXT: v_ashrrev_i64 v[6:7], v3, v[4:5]
+; GFX11-NEXT: v_ashrrev_i64 v[10:11], v10, v[4:5]
+; GFX11-NEXT: v_ashrrev_i32_e32 v4, 31, v5
+; GFX11-NEXT: v_dual_cndmask_b32 v2, v10, v2 :: v_dual_cndmask_b32 v3, v11, v8
+; GFX11-NEXT: v_cndmask_b32_e64 v0, v2, v0, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v1, v3, v1, s0
+; GFX11-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = ashr i65 %value, %amount
ret i65 %result
@@ -1842,61 +1863,50 @@ define i65 @v_ashr_i65_33(i65 %value) {
; GFX6-LABEL: v_ashr_i65_33:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_bfe_i32 v0, v2, 0, 1
; GFX6-NEXT: v_mov_b32_e32 v3, v1
-; GFX6-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX6-NEXT: v_lshl_b64 v[4:5], v[0:1], 31
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 1, v3
-; GFX6-NEXT: v_and_b32_e32 v2, 1, v2
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX6-NEXT: v_bfe_i32 v1, v2, 0, 1
+; GFX6-NEXT: v_ashrrev_i32_e32 v2, 31, v1
+; GFX6-NEXT: v_lshl_b64 v[0:1], v[1:2], 31
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 1, v3
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v3
+; GFX6-NEXT: v_ashrrev_i32_e32 v2, 1, v2
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_ashr_i65_33:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_bfe_i32 v0, v2, 0, 1
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX8-NEXT: v_lshlrev_b64 v[4:5], 31, v[0:1]
-; GFX8-NEXT: v_lshrrev_b32_e32 v0, 1, v3
-; GFX8-NEXT: v_and_b32_e32 v2, 1, v2
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX8-NEXT: v_bfe_i32 v1, v2, 0, 1
+; GFX8-NEXT: v_ashrrev_i32_e32 v2, 31, v1
+; GFX8-NEXT: v_lshlrev_b64 v[0:1], 31, v[1:2]
+; GFX8-NEXT: v_lshrrev_b32_e32 v3, 1, v3
+; GFX8-NEXT: v_or_b32_e32 v0, v0, v3
+; GFX8-NEXT: v_ashrrev_i32_e32 v2, 1, v2
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_ashr_i65_33:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_bfe_i32 v0, v2, 0, 1
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX9-NEXT: v_lshlrev_b64 v[4:5], 31, v[0:1]
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 1, v3
-; GFX9-NEXT: v_and_b32_e32 v2, 1, v2
-; GFX9-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX9-NEXT: v_bfe_i32 v1, v2, 0, 1
+; GFX9-NEXT: v_ashrrev_i32_e32 v2, 31, v1
+; GFX9-NEXT: v_lshlrev_b64 v[0:1], 31, v[1:2]
+; GFX9-NEXT: v_lshrrev_b32_e32 v3, 1, v3
+; GFX9-NEXT: v_or_b32_e32 v0, v0, v3
+; GFX9-NEXT: v_ashrrev_i32_e32 v2, 1, v2
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: v_ashr_i65_33:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_bfe_i32 v0, v2, 0, 1
-; GFX10-NEXT: v_mov_b32_e32 v3, v1
-; GFX10-NEXT: v_and_b32_e32 v2, 1, v2
-; GFX10-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX10-NEXT: v_lshlrev_b64 v[4:5], 31, v[0:1]
-; GFX10-NEXT: v_lshrrev_b32_e32 v0, 1, v3
-; GFX10-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: v_ashr_i65_33:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_bfe_i32 v0, v2, 0, 1
-; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_and_b32 v2, 1, v2
-; GFX11-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX11-NEXT: v_lshlrev_b64 v[4:5], 31, v[0:1]
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, 1, v3
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX10PLUS-LABEL: v_ashr_i65_33:
+; GFX10PLUS: ; %bb.0:
+; GFX10PLUS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10PLUS-NEXT: v_mov_b32_e32 v3, v1
+; GFX10PLUS-NEXT: v_bfe_i32 v1, v2, 0, 1
+; GFX10PLUS-NEXT: v_lshrrev_b32_e32 v3, 1, v3
+; GFX10PLUS-NEXT: v_ashrrev_i32_e32 v2, 31, v1
+; GFX10PLUS-NEXT: v_lshlrev_b64 v[0:1], 31, v[1:2]
+; GFX10PLUS-NEXT: v_ashrrev_i32_e32 v2, 1, v2
+; GFX10PLUS-NEXT: v_or_b32_e32 v0, v0, v3
+; GFX10PLUS-NEXT: s_setpc_b64 s[30:31]
%result = ashr i65 %value, 33
ret i65 %result
}
@@ -1904,35 +1914,48 @@ define i65 @v_ashr_i65_33(i65 %value) {
define amdgpu_ps i65 @s_ashr_i65(i65 inreg %value, i65 inreg %amount) {
; GCN-LABEL: s_ashr_i65:
; GCN: ; %bb.0:
-; GCN-NEXT: s_and_b32 s4, s2, 1
-; GCN-NEXT: s_bfe_i64 s[6:7], s[2:3], 0x10000
-; GCN-NEXT: s_sub_i32 s2, 64, s3
-; GCN-NEXT: s_lshr_b64 s[8:9], s[0:1], s3
-; GCN-NEXT: s_lshl_b64 s[10:11], s[6:7], s2
-; GCN-NEXT: s_or_b64 s[8:9], s[8:9], s[10:11]
+; GCN-NEXT: s_bfe_i64 s[4:5], s[2:3], 0x10000
+; GCN-NEXT: s_sub_i32 s10, s3, 64
+; GCN-NEXT: s_sub_i32 s8, 64, s3
; GCN-NEXT: s_cmp_lt_u32 s3, 64
-; GCN-NEXT: s_cselect_b32 s2, s8, s6
-; GCN-NEXT: s_cselect_b32 s5, s9, s7
+; GCN-NEXT: s_cselect_b32 s11, 1, 0
; GCN-NEXT: s_cmp_eq_u32 s3, 0
-; GCN-NEXT: s_cselect_b32 s1, s1, s5
-; GCN-NEXT: s_cselect_b32 s0, s0, s2
-; GCN-NEXT: s_mov_b32 s2, s4
+; GCN-NEXT: s_cselect_b32 s12, 1, 0
+; GCN-NEXT: s_ashr_i64 s[6:7], s[4:5], s3
+; GCN-NEXT: s_lshr_b64 s[2:3], s[0:1], s3
+; GCN-NEXT: s_lshl_b64 s[8:9], s[4:5], s8
+; GCN-NEXT: s_or_b64 s[2:3], s[2:3], s[8:9]
+; GCN-NEXT: s_ashr_i32 s7, s5, 31
+; GCN-NEXT: s_ashr_i64 s[4:5], s[4:5], s10
+; GCN-NEXT: s_cmp_lg_u32 s11, 0
+; GCN-NEXT: s_cselect_b64 s[2:3], s[2:3], s[4:5]
+; GCN-NEXT: s_cmp_lg_u32 s12, 0
+; GCN-NEXT: s_cselect_b64 s[0:1], s[0:1], s[2:3]
+; GCN-NEXT: s_cmp_lg_u32 s11, 0
+; GCN-NEXT: s_cselect_b32 s2, s6, s7
; GCN-NEXT: ; return to shader part epilog
;
; GFX10PLUS-LABEL: s_ashr_i65:
; GFX10PLUS: ; %bb.0:
; GFX10PLUS-NEXT: s_bfe_i64 s[4:5], s[2:3], 0x10000
-; GFX10PLUS-NEXT: s_sub_i32 s8, 64, s3
-; GFX10PLUS-NEXT: s_lshr_b64 s[6:7], s[0:1], s3
-; GFX10PLUS-NEXT: s_lshl_b64 s[8:9], s[4:5], s8
-; GFX10PLUS-NEXT: s_and_b32 s2, s2, 1
-; GFX10PLUS-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
+; GFX10PLUS-NEXT: s_sub_i32 s10, s3, 64
+; GFX10PLUS-NEXT: s_sub_i32 s2, 64, s3
; GFX10PLUS-NEXT: s_cmp_lt_u32 s3, 64
-; GFX10PLUS-NEXT: s_cselect_b32 s4, s6, s4
-; GFX10PLUS-NEXT: s_cselect_b32 s5, s7, s5
+; GFX10PLUS-NEXT: s_cselect_b32 s11, 1, 0
; GFX10PLUS-NEXT: s_cmp_eq_u32 s3, 0
-; GFX10PLUS-NEXT: s_cselect_b32 s0, s0, s4
-; GFX10PLUS-NEXT: s_cselect_b32 s1, s1, s5
+; GFX10PLUS-NEXT: s_cselect_b32 s12, 1, 0
+; GFX10PLUS-NEXT: s_lshr_b64 s[6:7], s[0:1], s3
+; GFX10PLUS-NEXT: s_lshl_b64 s[8:9], s[4:5], s2
+; GFX10PLUS-NEXT: s_ashr_i64 s[2:3], s[4:5], s3
+; GFX10PLUS-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
+; GFX10PLUS-NEXT: s_ashr_i32 s3, s5, 31
+; GFX10PLUS-NEXT: s_ashr_i64 s[4:5], s[4:5], s10
+; GFX10PLUS-NEXT: s_cmp_lg_u32 s11, 0
+; GFX10PLUS-NEXT: s_cselect_b64 s[4:5], s[6:7], s[4:5]
+; GFX10PLUS-NEXT: s_cmp_lg_u32 s12, 0
+; GFX10PLUS-NEXT: s_cselect_b64 s[0:1], s[0:1], s[4:5]
+; GFX10PLUS-NEXT: s_cmp_lg_u32 s11, 0
+; GFX10PLUS-NEXT: s_cselect_b32 s2, s2, s3
; GFX10PLUS-NEXT: ; return to shader part epilog
%result = ashr i65 %value, %amount
ret i65 %result
@@ -1941,26 +1964,20 @@ define amdgpu_ps i65 @s_ashr_i65(i65 inreg %value, i65 inreg %amount) {
define amdgpu_ps i65 @s_ashr_i65_33(i65 inreg %value) {
; GCN-LABEL: s_ashr_i65_33:
; GCN: ; %bb.0:
-; GCN-NEXT: s_mov_b32 s3, s1
-; GCN-NEXT: s_bfe_i64 s[0:1], s[2:3], 0x10000
-; GCN-NEXT: s_lshl_b64 s[4:5], s[0:1], 31
-; GCN-NEXT: s_lshr_b32 s6, s3, 1
-; GCN-NEXT: s_mov_b32 s7, 0
-; GCN-NEXT: s_or_b64 s[4:5], s[6:7], s[4:5]
-; GCN-NEXT: s_and_b32 s2, s2, 1
-; GCN-NEXT: s_mov_b32 s0, s4
+; GCN-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x10000
+; GCN-NEXT: s_lshr_b32 s4, s1, 1
+; GCN-NEXT: s_lshl_b64 s[0:1], s[2:3], 31
+; GCN-NEXT: s_or_b32 s0, s0, s4
+; GCN-NEXT: s_ashr_i32 s2, s3, 1
; GCN-NEXT: ; return to shader part epilog
;
; GFX10PLUS-LABEL: s_ashr_i65_33:
; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: s_mov_b32 s3, s1
-; GFX10PLUS-NEXT: s_mov_b32 s5, 0
-; GFX10PLUS-NEXT: s_bfe_i64 s[0:1], s[2:3], 0x10000
-; GFX10PLUS-NEXT: s_lshr_b32 s4, s3, 1
-; GFX10PLUS-NEXT: s_lshl_b64 s[6:7], s[0:1], 31
-; GFX10PLUS-NEXT: s_and_b32 s2, s2, 1
-; GFX10PLUS-NEXT: s_or_b64 s[4:5], s[4:5], s[6:7]
-; GFX10PLUS-NEXT: s_mov_b32 s0, s4
+; GFX10PLUS-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x10000
+; GFX10PLUS-NEXT: s_lshr_b32 s4, s1, 1
+; GFX10PLUS-NEXT: s_lshl_b64 s[0:1], s[2:3], 31
+; GFX10PLUS-NEXT: s_ashr_i32 s2, s3, 1
+; GFX10PLUS-NEXT: s_or_b32 s0, s0, s4
; GFX10PLUS-NEXT: ; return to shader part epilog
%result = ashr i65 %value, 33
ret i65 %result
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll
index c814e1c4ec7f1..1e54d1b362133 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll
@@ -1,9 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu6.00-amd-amdpal -o - %s | FileCheck -check-prefixes=GCN,GFX6 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-amd-amdpal -o - %s | FileCheck -check-prefixes=GCN,GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgpu6.00-amd-amdpal -o - %s | FileCheck -check-prefixes=GCN,GFX6 %s
+; RUN: llc -global-isel -mtriple=amdgpu8.03-amd-amdpal -o - %s | FileCheck -check-prefixes=GCN,GFX8 %s
; xUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00-amd-amdpal -o - %s | FileCheck -check-prefixes=GCN,GFX9 %s
; xUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-amd-amdpal -o - %s | FileCheck -check-prefixes=GCN,GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=+real-true16 -o - %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal -mattr=+real-true16 -o - %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16 %s
; xUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s
define amdgpu_ps i7 @s_fshl_i7(i7 inreg %lhs, i7 inreg %rhs, i7 inreg %amt) {
@@ -79,47 +79,6 @@ define amdgpu_ps i7 @s_fshl_i7(i7 inreg %lhs, i7 inreg %rhs, i7 inreg %amt) {
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshl_i7:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, 7
-; GFX11-NEXT: s_and_b32 s2, s2, 0x7f
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_rcp_iflag_f32_e32 v0, v0
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
-; GFX11-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_readfirstlane_b32 s3, v0
-; GFX11-NEXT: s_mul_i32 s4, s3, -7
-; GFX11-NEXT: s_mul_hi_u32 s4, s3, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_i32 s3, s3, s4
-; GFX11-NEXT: s_mul_hi_u32 s3, s2, s3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_mul_i32 s3, s3, -7
-; GFX11-NEXT: s_add_i32 s2, s2, s3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_cmp_ge_u32 s2, 7
-; GFX11-NEXT: s_cselect_b32 s3, 1, 0
-; GFX11-NEXT: s_add_i32 s4, s2, -7
-; GFX11-NEXT: s_cmp_lg_u32 s3, 0
-; GFX11-NEXT: s_cselect_b32 s2, s4, s2
-; GFX11-NEXT: s_cmp_ge_u32 s2, 7
-; GFX11-NEXT: s_cselect_b32 s3, 1, 0
-; GFX11-NEXT: s_add_i32 s4, s2, -7
-; GFX11-NEXT: s_cmp_lg_u32 s3, 0
-; GFX11-NEXT: s_cselect_b32 s2, s4, s2
-; GFX11-NEXT: s_and_b32 s1, s1, 0x7f
-; GFX11-NEXT: s_sub_i32 s3, 6, s2
-; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX11-NEXT: s_and_b32 s2, s2, 0x7f
-; GFX11-NEXT: s_lshr_b32 s1, s1, 1
-; GFX11-NEXT: s_and_b32 s3, s3, 0x7f
-; GFX11-NEXT: s_lshl_b32 s0, s0, s2
-; GFX11-NEXT: s_lshr_b32 s1, s1, s3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshl_i7:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v0, 7
@@ -154,6 +113,7 @@ define amdgpu_ps i7 @s_fshl_i7(i7 inreg %lhs, i7 inreg %rhs, i7 inreg %amt) {
; GFX9-NEXT: s_lshr_b32 s1, s1, s2
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: s_fshl_i7:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, 7
@@ -188,6 +148,48 @@ define amdgpu_ps i7 @s_fshl_i7(i7 inreg %lhs, i7 inreg %rhs, i7 inreg %amt) {
; GFX10-NEXT: s_lshr_b32 s1, s1, s3
; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: s_fshl_i7:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, 7
+; GFX11-NEXT: s_and_b32 s2, s2, 0x7f
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX11-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_readfirstlane_b32 s3, v0
+; GFX11-NEXT: s_mul_i32 s4, s3, -7
+; GFX11-NEXT: s_mul_hi_u32 s4, s3, s4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_add_i32 s3, s3, s4
+; GFX11-NEXT: s_mul_hi_u32 s3, s2, s3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_mul_i32 s3, s3, -7
+; GFX11-NEXT: s_add_i32 s2, s2, s3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_cmp_ge_u32 s2, 7
+; GFX11-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-NEXT: s_add_i32 s4, s2, -7
+; GFX11-NEXT: s_cmp_lg_u32 s3, 0
+; GFX11-NEXT: s_cselect_b32 s2, s4, s2
+; GFX11-NEXT: s_cmp_ge_u32 s2, 7
+; GFX11-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-NEXT: s_add_i32 s4, s2, -7
+; GFX11-NEXT: s_cmp_lg_u32 s3, 0
+; GFX11-NEXT: s_cselect_b32 s2, s4, s2
+; GFX11-NEXT: s_and_b32 s1, s1, 0x7f
+; GFX11-NEXT: s_sub_i32 s3, 6, s2
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_and_b32 s2, s2, 0x7f
+; GFX11-NEXT: s_lshr_b32 s1, s1, 1
+; GFX11-NEXT: s_and_b32 s3, s3, 0x7f
+; GFX11-NEXT: s_lshl_b32 s0, s0, s2
+; GFX11-NEXT: s_lshr_b32 s1, s1, s3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: ; return to shader part epilog
%result = call i7 @llvm.fshl.i7(i7 %lhs, i7 %rhs, i7 %amt)
ret i7 %result
}
@@ -256,45 +258,6 @@ define i7 @v_fshl_i7(i7 %lhs, i7 %rhs, i7 %amt) {
; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshl_i7:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v3, 7
-; GFX11-NEXT: v_and_b32_e32 v2, 0x7f, v2
-; GFX11-NEXT: v_and_b16 v0.h, 0x7f, v1.l
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_rcp_iflag_f32_e32 v3, v3
-; GFX11-NEXT: v_lshrrev_b16 v0.h, 1, v0.h
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_mul_f32_e32 v3, 0x4f7ffffe, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX11-NEXT: v_readfirstlane_b32 s0, v3
-; GFX11-NEXT: s_mul_i32 s1, s0, -7
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_mul_hi_u32 s1, s0, s1
-; GFX11-NEXT: s_add_i32 s0, s0, s1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_mul_hi_u32 v5, v2, s0
-; GFX11-NEXT: v_mad_u64_u32 v[3:4], null, v5, -7, v[2:3]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_add_nc_u32_e32 v2, -7, v3
-; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 7, v3
-; GFX11-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_add_nc_u32_e32 v3, -7, v2
-; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 7, v2
-; GFX11-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_sub_nc_u16 v1.l, 6, v2.l
-; GFX11-NEXT: v_and_b16 v1.h, 0x7f, v2.l
-; GFX11-NEXT: v_and_b16 v1.l, 0x7f, v1.l
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshlrev_b16 v0.l, v1.h, v0.l
-; GFX11-NEXT: v_lshrrev_b16 v0.h, v1.l, v0.h
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_or_b16 v0.l, v0.l, v0.h
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_i7:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -324,6 +287,7 @@ define i7 @v_fshl_i7(i7 %lhs, i7 %rhs, i7 %amt) {
; GFX9-NEXT: v_lshrrev_b16_e32 v1, v2, v1
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshl_i7:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -353,6 +317,47 @@ define i7 @v_fshl_i7(i7 %lhs, i7 %rhs, i7 %amt) {
; GFX10-NEXT: v_lshrrev_b16 v1, v3, v1
; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_fshl_i7:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_cvt_f32_ubyte0_e32 v3, 7
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0x7f, v2
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0x7f, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_rcp_iflag_f32_e32 v3, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, 1, v0.h
+; GFX11-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-TRUE16-NEXT: v_mul_f32_e32 v3, 0x4f7ffffe, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v3, v3
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v3
+; GFX11-TRUE16-NEXT: s_mul_i32 s1, s0, -7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_mul_hi_u32 s1, s0, s1
+; GFX11-TRUE16-NEXT: s_add_i32 s0, s0, s1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mul_hi_u32 v5, v2, s0
+; GFX11-TRUE16-NEXT: v_mad_u64_u32 v[3:4], null, v5, -7, v[2:3]
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, -7, v3
+; GFX11-TRUE16-NEXT: v_cmp_le_u32_e32 vcc_lo, 7, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, -7, v2
+; GFX11-TRUE16-NEXT: v_cmp_le_u32_e32 vcc_lo, 7, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_sub_nc_u16 v1.l, 6, v2.l
+; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0x7f, v2.l
+; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0x7f, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, v1.h, v0.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, v1.l, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-FAKE16-LABEL: v_fshl_i7:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -419,18 +424,6 @@ define amdgpu_ps i8 @s_fshl_i8(i8 inreg %lhs, i8 inreg %rhs, i8 inreg %amt) {
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshl_i8:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s1, s1, 0xff
-; GFX11-NEXT: s_and_b32 s3, s2, 7
-; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX11-NEXT: s_and_not1_b32 s2, 7, s2
-; GFX11-NEXT: s_lshr_b32 s1, s1, 1
-; GFX11-NEXT: s_lshl_b32 s0, s0, s3
-; GFX11-NEXT: s_lshr_b32 s1, s1, s2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshl_i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s1, s1, 0xff
@@ -442,6 +435,7 @@ define amdgpu_ps i8 @s_fshl_i8(i8 inreg %lhs, i8 inreg %rhs, i8 inreg %amt) {
; GFX9-NEXT: s_lshr_b32 s1, s1, s2
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: s_fshl_i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s1, s1, 0xff
@@ -453,6 +447,19 @@ define amdgpu_ps i8 @s_fshl_i8(i8 inreg %lhs, i8 inreg %rhs, i8 inreg %amt) {
; GFX10-NEXT: s_lshr_b32 s1, s1, s2
; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: s_fshl_i8:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-NEXT: s_and_b32 s3, s2, 7
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_and_not1_b32 s2, 7, s2
+; GFX11-NEXT: s_lshr_b32 s1, s1, 1
+; GFX11-NEXT: s_lshl_b32 s0, s0, s3
+; GFX11-NEXT: s_lshr_b32 s1, s1, s2
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: ; return to shader part epilog
%result = call i8 @llvm.fshl.i8(i8 %lhs, i8 %rhs, i8 %amt)
ret i8 %result
}
@@ -482,21 +489,6 @@ define i8 @v_fshl_i8(i8 %lhs, i8 %rhs, i8 %amt) {
; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshl_i8:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_and_b16 v0.h, 0xff, v1.l
-; GFX11-NEXT: v_xor_b16 v1.l, v2.l, -1
-; GFX11-NEXT: v_and_b16 v1.h, v2.l, 7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_lshrrev_b16 v0.h, 1, v0.h
-; GFX11-NEXT: v_and_b16 v1.l, v1.l, 7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshlrev_b16 v0.l, v1.h, v0.l
-; GFX11-NEXT: v_lshrrev_b16 v0.h, v1.l, v0.h
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_or_b16 v0.l, v0.l, v0.h
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -509,6 +501,7 @@ define i8 @v_fshl_i8(i8 %lhs, i8 %rhs, i8 %amt) {
; GFX9-NEXT: v_lshrrev_b16_e32 v1, v2, v1
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshl_i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -521,6 +514,23 @@ define i8 @v_fshl_i8(i8 %lhs, i8 %rhs, i8 %amt) {
; GFX10-NEXT: v_lshrrev_b16 v1, v3, v1
; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_fshl_i8:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.l
+; GFX11-TRUE16-NEXT: v_xor_b16 v1.l, v2.l, -1
+; GFX11-TRUE16-NEXT: v_and_b16 v1.h, v2.l, 7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, 1, v0.h
+; GFX11-TRUE16-NEXT: v_and_b16 v1.l, v1.l, 7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, v1.h, v0.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, v1.l, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-FAKE16-LABEL: v_fshl_i8:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -557,15 +567,6 @@ define amdgpu_ps i8 @s_fshl_i8_4(i8 inreg %lhs, i8 inreg %rhs) {
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshl_i8_4:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s1, s1, 0xff
-; GFX11-NEXT: s_lshl_b32 s0, s0, 4
-; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshr_b32 s1, s1, 4
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshl_i8_4:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s1, s1, 0xff
@@ -574,6 +575,7 @@ define amdgpu_ps i8 @s_fshl_i8_4(i8 inreg %lhs, i8 inreg %rhs) {
; GFX9-NEXT: s_lshr_b32 s1, s1, 4
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: s_fshl_i8_4:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s1, s1, 0xff
@@ -582,6 +584,16 @@ define amdgpu_ps i8 @s_fshl_i8_4(i8 inreg %lhs, i8 inreg %rhs) {
; GFX10-NEXT: s_lshr_b32 s1, s1, 4
; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: s_fshl_i8_4:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-NEXT: s_lshl_b32 s0, s0, 4
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_lshr_b32 s1, s1, 4
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: ; return to shader part epilog
%result = call i8 @llvm.fshl.i8(i8 %lhs, i8 %rhs, i8 4)
ret i8 %result
}
@@ -604,15 +616,6 @@ define i8 @v_fshl_i8_4(i8 %lhs, i8 %rhs) {
; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshl_i8_4:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_and_b16 v0.h, 0xff, v1.l
-; GFX11-NEXT: v_lshlrev_b16 v0.l, 4, v0.l
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b16 v0.h, 4, v0.h
-; GFX11-NEXT: v_or_b16 v0.l, v0.l, v0.h
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_i8_4:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -621,6 +624,7 @@ define i8 @v_fshl_i8_4(i8 %lhs, i8 %rhs) {
; GFX9-NEXT: v_lshrrev_b16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshl_i8_4:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -629,6 +633,17 @@ define i8 @v_fshl_i8_4(i8 %lhs, i8 %rhs) {
; GFX10-NEXT: v_lshrrev_b16 v1, 4, v1
; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_fshl_i8_4:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 4, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, 4, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-FAKE16-LABEL: v_fshl_i8_4:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -659,15 +674,6 @@ define amdgpu_ps i8 @s_fshl_i8_5(i8 inreg %lhs, i8 inreg %rhs) {
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshl_i8_5:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s1, s1, 0xff
-; GFX11-NEXT: s_lshl_b32 s0, s0, 5
-; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshr_b32 s1, s1, 3
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshl_i8_5:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s1, s1, 0xff
@@ -676,6 +682,7 @@ define amdgpu_ps i8 @s_fshl_i8_5(i8 inreg %lhs, i8 inreg %rhs) {
; GFX9-NEXT: s_lshr_b32 s1, s1, 3
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: s_fshl_i8_5:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s1, s1, 0xff
@@ -684,6 +691,16 @@ define amdgpu_ps i8 @s_fshl_i8_5(i8 inreg %lhs, i8 inreg %rhs) {
; GFX10-NEXT: s_lshr_b32 s1, s1, 3
; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: s_fshl_i8_5:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-NEXT: s_lshl_b32 s0, s0, 5
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_lshr_b32 s1, s1, 3
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: ; return to shader part epilog
%result = call i8 @llvm.fshl.i8(i8 %lhs, i8 %rhs, i8 5)
ret i8 %result
}
@@ -706,15 +723,6 @@ define i8 @v_fshl_i8_5(i8 %lhs, i8 %rhs) {
; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshl_i8_5:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_and_b16 v0.h, 0xff, v1.l
-; GFX11-NEXT: v_lshlrev_b16 v0.l, 5, v0.l
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b16 v0.h, 3, v0.h
-; GFX11-NEXT: v_or_b16 v0.l, v0.l, v0.h
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_i8_5:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -723,6 +731,7 @@ define i8 @v_fshl_i8_5(i8 %lhs, i8 %rhs) {
; GFX9-NEXT: v_lshrrev_b16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshl_i8_5:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -731,6 +740,17 @@ define i8 @v_fshl_i8_5(i8 %lhs, i8 %rhs) {
; GFX10-NEXT: v_lshrrev_b16 v1, 3, v1
; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_fshl_i8_5:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 5, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, 3, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-FAKE16-LABEL: v_fshl_i8_5:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -795,33 +815,6 @@ define amdgpu_ps i16 @s_fshl_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg, i16 in
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshl_v2i8:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_lshr_b32 s4, s1, 8
-; GFX11-NEXT: s_lshr_b32 s5, s2, 8
-; GFX11-NEXT: s_and_b32 s4, s4, 0xff
-; GFX11-NEXT: s_and_b32 s1, s1, 0xff
-; GFX11-NEXT: s_and_b32 s6, s2, 7
-; GFX11-NEXT: s_and_b32 s4, 0xffff, s4
-; GFX11-NEXT: s_lshr_b32 s3, s0, 8
-; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX11-NEXT: s_lshl_b32 s0, s0, s6
-; GFX11-NEXT: s_and_b32 s6, s5, 7
-; GFX11-NEXT: s_lshr_b32 s4, s4, 1
-; GFX11-NEXT: s_and_not1_b32 s5, 7, s5
-; GFX11-NEXT: s_lshr_b32 s1, s1, 1
-; GFX11-NEXT: s_and_not1_b32 s2, 7, s2
-; GFX11-NEXT: s_lshl_b32 s3, s3, s6
-; GFX11-NEXT: s_lshr_b32 s4, s4, s5
-; GFX11-NEXT: s_lshr_b32 s1, s1, s2
-; GFX11-NEXT: s_or_b32 s2, s3, s4
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: s_and_b32 s1, s2, 0xff
-; GFX11-NEXT: s_and_b32 s0, s0, 0xff
-; GFX11-NEXT: s_lshl_b32 s1, s1, 8
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshl_v2i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_lshr_b32 s4, s1, 8
@@ -848,6 +841,7 @@ define amdgpu_ps i16 @s_fshl_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg, i16 in
; GFX9-NEXT: s_lshl_b32 s1, s1, 8
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: s_fshl_v2i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_lshr_b32 s4, s1, 8
@@ -874,6 +868,34 @@ define amdgpu_ps i16 @s_fshl_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg, i16 in
; GFX10-NEXT: s_lshl_b32 s1, s1, 8
; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: s_fshl_v2i8:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_lshr_b32 s4, s1, 8
+; GFX11-NEXT: s_lshr_b32 s5, s2, 8
+; GFX11-NEXT: s_and_b32 s4, s4, 0xff
+; GFX11-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-NEXT: s_and_b32 s6, s2, 7
+; GFX11-NEXT: s_and_b32 s4, 0xffff, s4
+; GFX11-NEXT: s_lshr_b32 s3, s0, 8
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_lshl_b32 s0, s0, s6
+; GFX11-NEXT: s_and_b32 s6, s5, 7
+; GFX11-NEXT: s_lshr_b32 s4, s4, 1
+; GFX11-NEXT: s_and_not1_b32 s5, 7, s5
+; GFX11-NEXT: s_lshr_b32 s1, s1, 1
+; GFX11-NEXT: s_and_not1_b32 s2, 7, s2
+; GFX11-NEXT: s_lshl_b32 s3, s3, s6
+; GFX11-NEXT: s_lshr_b32 s4, s4, s5
+; GFX11-NEXT: s_lshr_b32 s1, s1, s2
+; GFX11-NEXT: s_or_b32 s2, s3, s4
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: s_and_b32 s1, s2, 0xff
+; GFX11-NEXT: s_and_b32 s0, s0, 0xff
+; GFX11-NEXT: s_lshl_b32 s1, s1, 8
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: ; return to shader part epilog
%lhs = bitcast i16 %lhs.arg to <2 x i8>
%rhs = bitcast i16 %rhs.arg to <2 x i8>
%amt = bitcast i16 %amt.arg to <2 x i8>
@@ -933,37 +955,6 @@ define i16 @v_fshl_v2i8(i16 %lhs.arg, i16 %rhs.arg, i16 %amt.arg) {
; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshl_v2i8:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, 8, v1
-; GFX11-NEXT: v_lshrrev_b32_e32 v4, 8, v2
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 8, v0
-; GFX11-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-NEXT: v_xor_b16 v2.h, v2.l, -1
-; GFX11-NEXT: v_and_b16 v0.h, 0xff, v3.l
-; GFX11-NEXT: v_xor_b16 v1.h, v4.l, -1
-; GFX11-NEXT: v_and_b16 v3.l, v4.l, 7
-; GFX11-NEXT: v_and_b16 v2.l, v2.l, 7
-; GFX11-NEXT: v_lshrrev_b16 v1.l, 1, v1.l
-; GFX11-NEXT: v_lshrrev_b16 v0.h, 1, v0.h
-; GFX11-NEXT: v_and_b16 v1.h, v1.h, 7
-; GFX11-NEXT: v_and_b16 v2.h, v2.h, 7
-; GFX11-NEXT: v_lshlrev_b16 v3.l, v3.l, v5.l
-; GFX11-NEXT: v_lshlrev_b16 v0.l, v2.l, v0.l
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_lshrrev_b16 v0.h, v1.h, v0.h
-; GFX11-NEXT: v_lshrrev_b16 v1.l, v2.h, v1.l
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_or_b16 v0.h, v3.l, v0.h
-; GFX11-NEXT: v_or_b16 v0.l, v0.l, v1.l
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
-; GFX11-NEXT: v_or_b16 v0.l, v0.l, v0.h
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_v2i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -989,6 +980,7 @@ define i16 @v_fshl_v2i8(i16 %lhs.arg, i16 %rhs.arg, i16 %amt.arg) {
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshl_v2i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1015,6 +1007,39 @@ define i16 @v_fshl_v2i8(i16 %lhs.arg, i16 %rhs.arg, i16 %amt.arg) {
; GFX10-NEXT: v_and_b32_sdwa v1, v2, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX10-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_fshl_v2i8:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 8, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v0
+; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
+; GFX11-TRUE16-NEXT: v_xor_b16 v2.h, v2.l, -1
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v3.l
+; GFX11-TRUE16-NEXT: v_xor_b16 v1.h, v4.l, -1
+; GFX11-TRUE16-NEXT: v_and_b16 v3.l, v4.l, 7
+; GFX11-TRUE16-NEXT: v_and_b16 v2.l, v2.l, 7
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v1.l, 1, v1.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, 1, v0.h
+; GFX11-TRUE16-NEXT: v_and_b16 v1.h, v1.h, 7
+; GFX11-TRUE16-NEXT: v_and_b16 v2.h, v2.h, 7
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, v3.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, v2.l, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, v1.h, v0.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v1.l, v2.h, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v3.l, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
+; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-FAKE16-LABEL: v_fshl_v2i8:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1153,58 +1178,6 @@ define amdgpu_ps i32 @s_fshl_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg, i32 in
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshl_v4i8:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s10, s1, 0xff
-; GFX11-NEXT: s_lshr_b32 s6, s1, 8
-; GFX11-NEXT: s_and_b32 s10, 0xffff, s10
-; GFX11-NEXT: s_lshr_b32 s8, s2, 8
-; GFX11-NEXT: s_lshr_b32 s9, s2, 16
-; GFX11-NEXT: s_lshr_b32 s11, s2, 24
-; GFX11-NEXT: s_and_b32 s12, s2, 7
-; GFX11-NEXT: s_lshr_b32 s10, s10, 1
-; GFX11-NEXT: s_and_not1_b32 s2, 7, s2
-; GFX11-NEXT: s_lshr_b32 s3, s0, 8
-; GFX11-NEXT: s_lshr_b32 s4, s0, 16
-; GFX11-NEXT: s_lshr_b32 s5, s0, 24
-; GFX11-NEXT: s_lshl_b32 s0, s0, s12
-; GFX11-NEXT: s_lshr_b32 s2, s10, s2
-; GFX11-NEXT: s_and_b32 s6, s6, 0xff
-; GFX11-NEXT: s_or_b32 s0, s0, s2
-; GFX11-NEXT: s_and_b32 s2, 0xffff, s6
-; GFX11-NEXT: s_lshr_b32 s7, s1, 16
-; GFX11-NEXT: s_and_b32 s6, s8, 7
-; GFX11-NEXT: s_lshr_b32 s2, s2, 1
-; GFX11-NEXT: s_and_not1_b32 s8, 7, s8
-; GFX11-NEXT: s_lshl_b32 s3, s3, s6
-; GFX11-NEXT: s_lshr_b32 s2, s2, s8
-; GFX11-NEXT: s_and_b32 s6, s7, 0xff
-; GFX11-NEXT: s_or_b32 s2, s3, s2
-; GFX11-NEXT: s_and_b32 s3, 0xffff, s6
-; GFX11-NEXT: s_and_b32 s6, s9, 7
-; GFX11-NEXT: s_lshr_b32 s3, s3, 1
-; GFX11-NEXT: s_and_not1_b32 s7, 7, s9
-; GFX11-NEXT: s_lshl_b32 s4, s4, s6
-; GFX11-NEXT: s_lshr_b32 s3, s3, s7
-; GFX11-NEXT: s_and_b32 s6, s11, 7
-; GFX11-NEXT: s_lshr_b32 s1, s1, 25
-; GFX11-NEXT: s_and_not1_b32 s7, 7, s11
-; GFX11-NEXT: s_lshl_b32 s5, s5, s6
-; GFX11-NEXT: s_lshr_b32 s1, s1, s7
-; GFX11-NEXT: s_or_b32 s3, s4, s3
-; GFX11-NEXT: s_and_b32 s2, s2, 0xff
-; GFX11-NEXT: s_or_b32 s1, s5, s1
-; GFX11-NEXT: s_and_b32 s0, s0, 0xff
-; GFX11-NEXT: s_lshl_b32 s2, s2, 8
-; GFX11-NEXT: s_and_b32 s3, s3, 0xff
-; GFX11-NEXT: s_or_b32 s0, s0, s2
-; GFX11-NEXT: s_lshl_b32 s2, s3, 16
-; GFX11-NEXT: s_and_b32 s1, s1, 0xff
-; GFX11-NEXT: s_or_b32 s0, s0, s2
-; GFX11-NEXT: s_lshl_b32 s1, s1, 24
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshl_v4i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s11, s2, 7
@@ -1256,6 +1229,7 @@ define amdgpu_ps i32 @s_fshl_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg, i32 in
; GFX9-NEXT: s_lshl_b32 s1, s1, 24
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: s_fshl_v4i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s10, s1, 0xff
@@ -1307,6 +1281,59 @@ define amdgpu_ps i32 @s_fshl_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg, i32 in
; GFX10-NEXT: s_lshl_b32 s1, s1, 24
; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: s_fshl_v4i8:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s10, s1, 0xff
+; GFX11-NEXT: s_lshr_b32 s6, s1, 8
+; GFX11-NEXT: s_and_b32 s10, 0xffff, s10
+; GFX11-NEXT: s_lshr_b32 s8, s2, 8
+; GFX11-NEXT: s_lshr_b32 s9, s2, 16
+; GFX11-NEXT: s_lshr_b32 s11, s2, 24
+; GFX11-NEXT: s_and_b32 s12, s2, 7
+; GFX11-NEXT: s_lshr_b32 s10, s10, 1
+; GFX11-NEXT: s_and_not1_b32 s2, 7, s2
+; GFX11-NEXT: s_lshr_b32 s3, s0, 8
+; GFX11-NEXT: s_lshr_b32 s4, s0, 16
+; GFX11-NEXT: s_lshr_b32 s5, s0, 24
+; GFX11-NEXT: s_lshl_b32 s0, s0, s12
+; GFX11-NEXT: s_lshr_b32 s2, s10, s2
+; GFX11-NEXT: s_and_b32 s6, s6, 0xff
+; GFX11-NEXT: s_or_b32 s0, s0, s2
+; GFX11-NEXT: s_and_b32 s2, 0xffff, s6
+; GFX11-NEXT: s_lshr_b32 s7, s1, 16
+; GFX11-NEXT: s_and_b32 s6, s8, 7
+; GFX11-NEXT: s_lshr_b32 s2, s2, 1
+; GFX11-NEXT: s_and_not1_b32 s8, 7, s8
+; GFX11-NEXT: s_lshl_b32 s3, s3, s6
+; GFX11-NEXT: s_lshr_b32 s2, s2, s8
+; GFX11-NEXT: s_and_b32 s6, s7, 0xff
+; GFX11-NEXT: s_or_b32 s2, s3, s2
+; GFX11-NEXT: s_and_b32 s3, 0xffff, s6
+; GFX11-NEXT: s_and_b32 s6, s9, 7
+; GFX11-NEXT: s_lshr_b32 s3, s3, 1
+; GFX11-NEXT: s_and_not1_b32 s7, 7, s9
+; GFX11-NEXT: s_lshl_b32 s4, s4, s6
+; GFX11-NEXT: s_lshr_b32 s3, s3, s7
+; GFX11-NEXT: s_and_b32 s6, s11, 7
+; GFX11-NEXT: s_lshr_b32 s1, s1, 25
+; GFX11-NEXT: s_and_not1_b32 s7, 7, s11
+; GFX11-NEXT: s_lshl_b32 s5, s5, s6
+; GFX11-NEXT: s_lshr_b32 s1, s1, s7
+; GFX11-NEXT: s_or_b32 s3, s4, s3
+; GFX11-NEXT: s_and_b32 s2, s2, 0xff
+; GFX11-NEXT: s_or_b32 s1, s5, s1
+; GFX11-NEXT: s_and_b32 s0, s0, 0xff
+; GFX11-NEXT: s_lshl_b32 s2, s2, 8
+; GFX11-NEXT: s_and_b32 s3, s3, 0xff
+; GFX11-NEXT: s_or_b32 s0, s0, s2
+; GFX11-NEXT: s_lshl_b32 s2, s3, 16
+; GFX11-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-NEXT: s_or_b32 s0, s0, s2
+; GFX11-NEXT: s_lshl_b32 s1, s1, 24
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: ; return to shader part epilog
%lhs = bitcast i32 %lhs.arg to <4 x i8>
%rhs = bitcast i32 %rhs.arg to <4 x i8>
%amt = bitcast i32 %amt.arg to <4 x i8>
@@ -1413,61 +1440,6 @@ define i32 @v_fshl_v4i8(i32 %lhs.arg, i32 %rhs.arg, i32 %amt.arg) {
; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshl_v4i8:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, 8, v1
-; GFX11-NEXT: v_lshrrev_b32_e32 v4, 8, v2
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 8, v0
-; GFX11-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; GFX11-NEXT: v_lshrrev_b32_e32 v9, 16, v2
-; GFX11-NEXT: v_lshrrev_b32_e32 v10, 24, v2
-; GFX11-NEXT: v_and_b16 v3.l, 0xff, v3.l
-; GFX11-NEXT: v_xor_b16 v3.h, v4.l, -1
-; GFX11-NEXT: v_and_b16 v4.l, v4.l, 7
-; GFX11-NEXT: v_and_b16 v4.h, 0xff, v6.l
-; GFX11-NEXT: v_xor_b16 v5.h, v10.l, -1
-; GFX11-NEXT: v_lshrrev_b16 v3.l, 1, v3.l
-; GFX11-NEXT: v_and_b16 v3.h, v3.h, 7
-; GFX11-NEXT: v_lshlrev_b16 v4.l, v4.l, v5.l
-; GFX11-NEXT: v_xor_b16 v5.l, v9.l, -1
-; GFX11-NEXT: v_lshrrev_b32_e32 v7, 16, v0
-; GFX11-NEXT: v_lshrrev_b32_e32 v8, 24, v0
-; GFX11-NEXT: v_and_b16 v0.h, v2.l, 7
-; GFX11-NEXT: v_and_b16 v2.h, 0xff, v1.l
-; GFX11-NEXT: v_xor_b16 v2.l, v2.l, -1
-; GFX11-NEXT: v_lshrrev_b16 v3.l, v3.h, v3.l
-; GFX11-NEXT: v_and_b16 v3.h, v9.l, 7
-; GFX11-NEXT: v_lshrrev_b16 v4.h, 1, v4.h
-; GFX11-NEXT: v_and_b16 v5.l, v5.l, 7
-; GFX11-NEXT: v_and_b16 v6.l, v10.l, 7
-; GFX11-NEXT: v_lshrrev_b32_e32 v9, 25, v1
-; GFX11-NEXT: v_and_b16 v1.l, v5.h, 7
-; GFX11-NEXT: v_lshrrev_b16 v2.h, 1, v2.h
-; GFX11-NEXT: v_and_b16 v2.l, v2.l, 7
-; GFX11-NEXT: v_or_b16 v4.l, v4.l, v3.l
-; GFX11-NEXT: v_lshlrev_b16 v1.h, v3.h, v7.l
-; GFX11-NEXT: v_lshrrev_b16 v3.l, v5.l, v4.h
-; GFX11-NEXT: v_lshlrev_b16 v3.h, v6.l, v8.l
-; GFX11-NEXT: v_lshrrev_b16 v1.l, v1.l, v9.l
-; GFX11-NEXT: v_lshlrev_b16 v0.l, v0.h, v0.l
-; GFX11-NEXT: v_lshrrev_b16 v0.h, v2.l, v2.h
-; GFX11-NEXT: v_and_b32_e32 v2, 0xff, v4
-; GFX11-NEXT: v_or_b16 v3.l, v1.h, v3.l
-; GFX11-NEXT: v_or_b16 v1.l, v3.h, v1.l
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_or_b16 v0.l, v0.l, v0.h
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 8, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_and_b32_e32 v3, 0xff, v3
-; GFX11-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_and_or_b32 v0, 0xff, v0, v2
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v1, 24, v1
-; GFX11-NEXT: v_or3_b32 v0, v0, v2, v1
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_v4i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1516,6 +1488,7 @@ define i32 @v_fshl_v4i8(i32 %lhs.arg, i32 %rhs.arg, i32 %amt.arg) {
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 24, v0
; GFX9-NEXT: v_or3_b32 v0, v1, v2, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshl_v4i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1567,6 +1540,63 @@ define i32 @v_fshl_v4i8(i32 %lhs.arg, i32 %rhs.arg, i32 %amt.arg) {
; GFX10-NEXT: v_lshlrev_b32_e32 v1, 24, v1
; GFX10-NEXT: v_or3_b32 v0, v0, v2, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_fshl_v4i8:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 8, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 24, v2
+; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l
+; GFX11-TRUE16-NEXT: v_xor_b16 v3.h, v4.l, -1
+; GFX11-TRUE16-NEXT: v_and_b16 v4.l, v4.l, 7
+; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v6.l
+; GFX11-TRUE16-NEXT: v_xor_b16 v5.h, v10.l, -1
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v3.l, 1, v3.l
+; GFX11-TRUE16-NEXT: v_and_b16 v3.h, v3.h, 7
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, v4.l, v5.l
+; GFX11-TRUE16-NEXT: v_xor_b16 v5.l, v9.l, -1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 24, v0
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v2.l, 7
+; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v1.l
+; GFX11-TRUE16-NEXT: v_xor_b16 v2.l, v2.l, -1
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v3.l, v3.h, v3.l
+; GFX11-TRUE16-NEXT: v_and_b16 v3.h, v9.l, 7
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v4.h, 1, v4.h
+; GFX11-TRUE16-NEXT: v_and_b16 v5.l, v5.l, 7
+; GFX11-TRUE16-NEXT: v_and_b16 v6.l, v10.l, 7
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 25, v1
+; GFX11-TRUE16-NEXT: v_and_b16 v1.l, v5.h, 7
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v2.h, 1, v2.h
+; GFX11-TRUE16-NEXT: v_and_b16 v2.l, v2.l, 7
+; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v3.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, v3.h, v7.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v3.l, v5.l, v4.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, v6.l, v8.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v1.l, v1.l, v9.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, v2.l, v2.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xff, v4
+; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v1.h, v3.l
+; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v3.h, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v0, 0xff, v0, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 24, v1
+; GFX11-TRUE16-NEXT: v_or3_b32 v0, v0, v2, v1
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-FAKE16-LABEL: v_fshl_v4i8:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1631,77 +1661,74 @@ define i32 @v_fshl_v4i8(i32 %lhs.arg, i32 %rhs.arg, i32 %amt.arg) {
}
define amdgpu_ps i24 @s_fshl_i24(i24 inreg %lhs, i24 inreg %rhs, i24 inreg %amt) {
-; GCN-LABEL: s_fshl_i24:
-; GCN: ; %bb.0:
-; GCN-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
-; GCN-NEXT: v_rcp_iflag_f32_e32 v0, v0
-; GCN-NEXT: s_and_b32 s2, s2, 0xffffff
-; GCN-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
-; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GCN-NEXT: v_readfirstlane_b32 s3, v0
-; GCN-NEXT: s_mul_i32 s4, s3, 0xffffffe8
-; GCN-NEXT: v_mul_hi_u32 v0, v0, s4
-; GCN-NEXT: v_readfirstlane_b32 s4, v0
-; GCN-NEXT: s_add_i32 s3, s3, s4
-; GCN-NEXT: v_mov_b32_e32 v0, s3
-; GCN-NEXT: v_mul_hi_u32 v0, s2, v0
-; GCN-NEXT: v_readfirstlane_b32 s3, v0
-; GCN-NEXT: s_mulk_i32 s3, 0xffe8
-; GCN-NEXT: s_add_i32 s2, s2, s3
-; GCN-NEXT: s_cmp_ge_u32 s2, 24
-; GCN-NEXT: s_cselect_b32 s3, 1, 0
-; GCN-NEXT: s_sub_i32 s4, s2, 24
-; GCN-NEXT: s_cmp_lg_u32 s3, 0
-; GCN-NEXT: s_cselect_b32 s2, s4, s2
-; GCN-NEXT: s_cmp_ge_u32 s2, 24
-; GCN-NEXT: s_cselect_b32 s3, 1, 0
-; GCN-NEXT: s_sub_i32 s4, s2, 24
-; GCN-NEXT: s_cmp_lg_u32 s3, 0
-; GCN-NEXT: s_cselect_b32 s2, s4, s2
-; GCN-NEXT: s_sub_i32 s3, 23, s2
-; GCN-NEXT: s_bfe_u32 s1, s1, 0x170001
-; GCN-NEXT: s_lshl_b32 s0, s0, s2
-; GCN-NEXT: s_lshr_b32 s1, s1, s3
-; GCN-NEXT: s_or_b32 s0, s0, s1
-; GCN-NEXT: ; return to shader part epilog
+; GFX6-LABEL: s_fshl_i24:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
+; GFX6-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; GFX6-NEXT: s_and_b32 s2, s2, 0xffffff
+; GFX6-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX6-NEXT: v_readfirstlane_b32 s3, v0
+; GFX6-NEXT: s_mul_i32 s4, s3, 0xffffffe8
+; GFX6-NEXT: v_mul_hi_u32 v0, v0, s4
+; GFX6-NEXT: v_readfirstlane_b32 s4, v0
+; GFX6-NEXT: s_add_i32 s3, s3, s4
+; GFX6-NEXT: v_mov_b32_e32 v0, s3
+; GFX6-NEXT: v_mul_hi_u32 v0, s2, v0
+; GFX6-NEXT: v_readfirstlane_b32 s3, v0
+; GFX6-NEXT: s_mulk_i32 s3, 0xffe8
+; GFX6-NEXT: s_add_i32 s2, s2, s3
+; GFX6-NEXT: s_cmp_ge_u32 s2, 24
+; GFX6-NEXT: s_cselect_b32 s3, 1, 0
+; GFX6-NEXT: s_sub_i32 s4, s2, 24
+; GFX6-NEXT: s_cmp_lg_u32 s3, 0
+; GFX6-NEXT: s_cselect_b32 s2, s4, s2
+; GFX6-NEXT: s_cmp_ge_u32 s2, 24
+; GFX6-NEXT: s_cselect_b32 s3, 1, 0
+; GFX6-NEXT: s_sub_i32 s4, s2, 24
+; GFX6-NEXT: s_cmp_lg_u32 s3, 0
+; GFX6-NEXT: s_cselect_b32 s2, s4, s2
+; GFX6-NEXT: s_sub_i32 s3, 23, s2
+; GFX6-NEXT: s_bfe_u32 s1, s1, 0x170001
+; GFX6-NEXT: s_lshl_b32 s0, s0, s2
+; GFX6-NEXT: s_lshr_b32 s1, s1, s3
+; GFX6-NEXT: s_or_b32 s0, s0, s1
+; GFX6-NEXT: ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshl_i24:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
+; GFX8-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; GFX8-NEXT: s_and_b32 s2, s2, 0xffffff
+; GFX8-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX8-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX8-NEXT: v_readfirstlane_b32 s3, v0
+; GFX8-NEXT: s_mul_i32 s4, s3, 0xffffffe8
+; GFX8-NEXT: v_mul_hi_u32 v0, v0, s4
+; GFX8-NEXT: v_readfirstlane_b32 s4, v0
+; GFX8-NEXT: s_add_i32 s3, s3, s4
+; GFX8-NEXT: v_mov_b32_e32 v0, s3
+; GFX8-NEXT: v_mul_hi_u32 v0, s2, v0
+; GFX8-NEXT: v_readfirstlane_b32 s3, v0
+; GFX8-NEXT: s_mulk_i32 s3, 0xffe8
+; GFX8-NEXT: s_add_i32 s2, s2, s3
+; GFX8-NEXT: s_cmp_ge_u32 s2, 24
+; GFX8-NEXT: s_cselect_b32 s3, 1, 0
+; GFX8-NEXT: s_sub_i32 s4, s2, 24
+; GFX8-NEXT: s_cmp_lg_u32 s3, 0
+; GFX8-NEXT: s_cselect_b32 s2, s4, s2
+; GFX8-NEXT: s_cmp_ge_u32 s2, 24
+; GFX8-NEXT: s_cselect_b32 s3, 1, 0
+; GFX8-NEXT: s_sub_i32 s4, s2, 24
+; GFX8-NEXT: s_cmp_lg_u32 s3, 0
+; GFX8-NEXT: s_cselect_b32 s2, s4, s2
+; GFX8-NEXT: s_sub_i32 s3, 23, s2
+; GFX8-NEXT: s_bfe_u32 s1, s1, 0x170001
+; GFX8-NEXT: s_lshl_b32 s0, s0, s2
+; GFX8-NEXT: s_lshr_b32 s1, s1, s3
+; GFX8-NEXT: s_or_b32 s0, s0, s1
+; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshl_i24:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
-; GFX11-NEXT: s_and_b32 s2, s2, 0xffffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_rcp_iflag_f32_e32 v0, v0
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
-; GFX11-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_readfirstlane_b32 s3, v0
-; GFX11-NEXT: s_mul_i32 s4, s3, 0xffffffe8
-; GFX11-NEXT: s_mul_hi_u32 s4, s3, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_i32 s3, s3, s4
-; GFX11-NEXT: s_mul_hi_u32 s3, s2, s3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_mulk_i32 s3, 0xffe8
-; GFX11-NEXT: s_add_i32 s2, s2, s3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_cmp_ge_u32 s2, 24
-; GFX11-NEXT: s_cselect_b32 s3, 1, 0
-; GFX11-NEXT: s_sub_i32 s4, s2, 24
-; GFX11-NEXT: s_cmp_lg_u32 s3, 0
-; GFX11-NEXT: s_cselect_b32 s2, s4, s2
-; GFX11-NEXT: s_cmp_ge_u32 s2, 24
-; GFX11-NEXT: s_cselect_b32 s3, 1, 0
-; GFX11-NEXT: s_sub_i32 s4, s2, 24
-; GFX11-NEXT: s_cmp_lg_u32 s3, 0
-; GFX11-NEXT: s_cselect_b32 s2, s4, s2
-; GFX11-NEXT: s_bfe_u32 s1, s1, 0x170001
-; GFX11-NEXT: s_sub_i32 s3, 23, s2
-; GFX11-NEXT: s_lshl_b32 s0, s0, s2
-; GFX11-NEXT: s_lshr_b32 s1, s1, s3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshl_i24:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
@@ -1732,6 +1759,7 @@ define amdgpu_ps i24 @s_fshl_i24(i24 inreg %lhs, i24 inreg %rhs, i24 inreg %amt)
; GFX9-NEXT: s_lshr_b32 s1, s1, s3
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: s_fshl_i24:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
@@ -1762,6 +1790,44 @@ define amdgpu_ps i24 @s_fshl_i24(i24 inreg %lhs, i24 inreg %rhs, i24 inreg %amt)
; GFX10-NEXT: s_lshr_b32 s1, s1, s3
; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: s_fshl_i24:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
+; GFX11-NEXT: s_and_b32 s2, s2, 0xffffff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX11-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_readfirstlane_b32 s3, v0
+; GFX11-NEXT: s_mul_i32 s4, s3, 0xffffffe8
+; GFX11-NEXT: s_mul_hi_u32 s4, s3, s4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_add_i32 s3, s3, s4
+; GFX11-NEXT: s_mul_hi_u32 s3, s2, s3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_mulk_i32 s3, 0xffe8
+; GFX11-NEXT: s_add_i32 s2, s2, s3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_cmp_ge_u32 s2, 24
+; GFX11-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-NEXT: s_sub_i32 s4, s2, 24
+; GFX11-NEXT: s_cmp_lg_u32 s3, 0
+; GFX11-NEXT: s_cselect_b32 s2, s4, s2
+; GFX11-NEXT: s_cmp_ge_u32 s2, 24
+; GFX11-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-NEXT: s_sub_i32 s4, s2, 24
+; GFX11-NEXT: s_cmp_lg_u32 s3, 0
+; GFX11-NEXT: s_cselect_b32 s2, s4, s2
+; GFX11-NEXT: s_bfe_u32 s1, s1, 0x170001
+; GFX11-NEXT: s_sub_i32 s3, 23, s2
+; GFX11-NEXT: s_lshl_b32 s0, s0, s2
+; GFX11-NEXT: s_lshr_b32 s1, s1, s3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: ; return to shader part epilog
%result = call i24 @llvm.fshl.i24(i24 %lhs, i24 %rhs, i24 %amt)
ret i24 %result
}
@@ -1831,42 +1897,6 @@ define i24 @v_fshl_i24(i24 %lhs, i24 %rhs, i24 %amt) {
; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshl_i24:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v3, 24
-; GFX11-NEXT: v_and_b32_e32 v2, 0xffffff, v2
-; GFX11-NEXT: v_bfe_u32 v1, v1, 1, 23
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_rcp_iflag_f32_e32 v3, v3
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_mul_f32_e32 v3, 0x4f7ffffe, v3
-; GFX11-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_readfirstlane_b32 s0, v3
-; GFX11-NEXT: s_mul_i32 s1, s0, 0xffffffe8
-; GFX11-NEXT: s_mul_hi_u32 s1, s0, s1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_i32 s0, s0, s1
-; GFX11-NEXT: v_mul_hi_u32 v5, v2, s0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_mad_u64_u32 v[3:4], null, 0xffffffe8, v5, v[2:3]
-; GFX11-NEXT: v_add_nc_u32_e32 v2, 0xffffffe8, v3
-; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc_lo
-; GFX11-NEXT: v_add_nc_u32_e32 v3, 0xffffffe8, v2
-; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc_lo
-; GFX11-NEXT: v_sub_nc_u32_e32 v3, 23, v2
-; GFX11-NEXT: v_and_b32_e32 v2, 0xffffff, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v3, 0xffffff, v3
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, v3, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_lshl_or_b32 v0, v0, v2, v1
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_i24:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1895,6 +1925,7 @@ define i24 @v_fshl_i24(i24 %lhs, i24 %rhs, i24 %amt) {
; GFX9-NEXT: v_lshrrev_b32_e32 v1, v3, v1
; GFX9-NEXT: v_lshl_or_b32 v0, v0, v2, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshl_i24:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1922,6 +1953,43 @@ define i24 @v_fshl_i24(i24 %lhs, i24 %rhs, i24 %amt) {
; GFX10-NEXT: v_lshrrev_b32_e32 v1, v3, v1
; GFX10-NEXT: v_lshl_or_b32 v0, v0, v2, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_fshl_i24:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v3, 24
+; GFX11-NEXT: v_and_b32_e32 v2, 0xffffff, v2
+; GFX11-NEXT: v_bfe_u32 v1, v1, 1, 23
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_rcp_iflag_f32_e32 v3, v3
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-NEXT: v_mul_f32_e32 v3, 0x4f7ffffe, v3
+; GFX11-NEXT: v_cvt_u32_f32_e32 v3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_readfirstlane_b32 s0, v3
+; GFX11-NEXT: s_mul_i32 s1, s0, 0xffffffe8
+; GFX11-NEXT: s_mul_hi_u32 s1, s0, s1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_add_i32 s0, s0, s1
+; GFX11-NEXT: v_mul_hi_u32 v5, v2, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_mad_u64_u32 v[3:4], null, 0xffffffe8, v5, v[2:3]
+; GFX11-NEXT: v_add_nc_u32_e32 v2, 0xffffffe8, v3
+; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc_lo
+; GFX11-NEXT: v_add_nc_u32_e32 v3, 0xffffffe8, v2
+; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc_lo
+; GFX11-NEXT: v_sub_nc_u32_e32 v3, 23, v2
+; GFX11-NEXT: v_and_b32_e32 v2, 0xffffff, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_and_b32_e32 v3, 0xffffff, v3
+; GFX11-NEXT: v_lshrrev_b32_e32 v1, v3, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshl_or_b32 v0, v0, v2, v1
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call i24 @llvm.fshl.i24(i24 %lhs, i24 %rhs, i24 %amt)
ret i24 %result
}
@@ -2181,114 +2249,6 @@ define amdgpu_ps i48 @s_fshl_v2i24(i48 inreg %lhs.arg, i48 inreg %rhs.arg, i48 i
; GFX8-NEXT: s_or_b32 s1, s2, s1
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshl_v2i24:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
-; GFX11-NEXT: s_lshr_b32 s9, s1, 8
-; GFX11-NEXT: s_and_b32 s1, s1, 0xff
-; GFX11-NEXT: s_lshr_b32 s6, s0, 8
-; GFX11-NEXT: s_lshr_b32 s8, s0, 24
-; GFX11-NEXT: v_rcp_iflag_f32_e32 v0, v0
-; GFX11-NEXT: s_lshl_b32 s1, s1, 8
-; GFX11-NEXT: s_lshr_b32 s11, s3, 8
-; GFX11-NEXT: s_and_b32 s3, s3, 0xff
-; GFX11-NEXT: s_and_b32 s6, s6, 0xff
-; GFX11-NEXT: s_or_b32 s1, s8, s1
-; GFX11-NEXT: s_lshr_b32 s8, s2, 8
-; GFX11-NEXT: s_lshr_b32 s10, s2, 24
-; GFX11-NEXT: s_lshl_b32 s3, s3, 8
-; GFX11-NEXT: s_lshr_b32 s7, s0, 16
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
-; GFX11-NEXT: s_and_b32 s0, s0, 0xff
-; GFX11-NEXT: s_lshl_b32 s6, s6, 8
-; GFX11-NEXT: s_and_b32 s8, s8, 0xff
-; GFX11-NEXT: s_or_b32 s3, s10, s3
-; GFX11-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX11-NEXT: s_lshr_b32 s10, s4, 8
-; GFX11-NEXT: s_or_b32 s0, s0, s6
-; GFX11-NEXT: s_and_b32 s6, s7, 0xff
-; GFX11-NEXT: s_and_b32 s7, s9, 0xff
-; GFX11-NEXT: v_readfirstlane_b32 s13, v0
-; GFX11-NEXT: s_lshr_b32 s9, s2, 16
-; GFX11-NEXT: s_and_b32 s2, s2, 0xff
-; GFX11-NEXT: s_lshl_b32 s8, s8, 8
-; GFX11-NEXT: s_and_b32 s10, s10, 0xff
-; GFX11-NEXT: s_or_b32 s2, s2, s8
-; GFX11-NEXT: s_and_b32 s8, s9, 0xff
-; GFX11-NEXT: s_and_b32 s9, s11, 0xff
-; GFX11-NEXT: s_lshr_b32 s11, s4, 16
-; GFX11-NEXT: s_lshr_b32 s12, s4, 24
-; GFX11-NEXT: s_and_b32 s4, s4, 0xff
-; GFX11-NEXT: s_lshl_b32 s10, s10, 8
-; GFX11-NEXT: s_and_b32 s11, s11, 0xff
-; GFX11-NEXT: s_or_b32 s4, s4, s10
-; GFX11-NEXT: s_mul_i32 s10, s13, 0xffffffe8
-; GFX11-NEXT: s_pack_ll_b32_b16 s4, s4, s11
-; GFX11-NEXT: s_mul_hi_u32 s10, s13, s10
-; GFX11-NEXT: s_lshr_b32 s14, s5, 8
-; GFX11-NEXT: s_add_i32 s13, s13, s10
-; GFX11-NEXT: s_and_b32 s5, s5, 0xff
-; GFX11-NEXT: s_mul_hi_u32 s10, s4, s13
-; GFX11-NEXT: s_lshl_b32 s5, s5, 8
-; GFX11-NEXT: s_mulk_i32 s10, 0xffe8
-; GFX11-NEXT: s_or_b32 s5, s12, s5
-; GFX11-NEXT: s_add_i32 s4, s4, s10
-; GFX11-NEXT: s_and_b32 s10, s14, 0xff
-; GFX11-NEXT: s_cmp_ge_u32 s4, 24
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s6
-; GFX11-NEXT: s_cselect_b32 s11, 1, 0
-; GFX11-NEXT: s_sub_i32 s12, s4, 24
-; GFX11-NEXT: s_cmp_lg_u32 s11, 0
-; GFX11-NEXT: s_pack_ll_b32_b16 s2, s2, s8
-; GFX11-NEXT: s_cselect_b32 s4, s12, s4
-; GFX11-NEXT: s_pack_ll_b32_b16 s5, s5, s10
-; GFX11-NEXT: s_cmp_ge_u32 s4, 24
-; GFX11-NEXT: s_pack_ll_b32_b16 s3, s3, s9
-; GFX11-NEXT: s_cselect_b32 s6, 1, 0
-; GFX11-NEXT: s_sub_i32 s8, s4, 24
-; GFX11-NEXT: s_cmp_lg_u32 s6, 0
-; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s7
-; GFX11-NEXT: s_cselect_b32 s4, s8, s4
-; GFX11-NEXT: s_lshr_b32 s2, s2, 1
-; GFX11-NEXT: s_sub_i32 s6, 23, s4
-; GFX11-NEXT: s_lshl_b32 s0, s0, s4
-; GFX11-NEXT: s_mul_hi_u32 s4, s5, s13
-; GFX11-NEXT: s_lshr_b32 s2, s2, s6
-; GFX11-NEXT: s_mulk_i32 s4, 0xffe8
-; GFX11-NEXT: s_or_b32 s0, s0, s2
-; GFX11-NEXT: s_add_i32 s5, s5, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_cmp_ge_u32 s5, 24
-; GFX11-NEXT: s_cselect_b32 s2, 1, 0
-; GFX11-NEXT: s_sub_i32 s4, s5, 24
-; GFX11-NEXT: s_cmp_lg_u32 s2, 0
-; GFX11-NEXT: s_cselect_b32 s2, s4, s5
-; GFX11-NEXT: s_cmp_ge_u32 s2, 24
-; GFX11-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-NEXT: s_sub_i32 s5, s2, 24
-; GFX11-NEXT: s_cmp_lg_u32 s4, 0
-; GFX11-NEXT: s_cselect_b32 s2, s5, s2
-; GFX11-NEXT: s_lshr_b32 s3, s3, 1
-; GFX11-NEXT: s_sub_i32 s4, 23, s2
-; GFX11-NEXT: s_lshl_b32 s1, s1, s2
-; GFX11-NEXT: s_lshr_b32 s2, s3, s4
-; GFX11-NEXT: s_and_b32 s3, s0, 0xff
-; GFX11-NEXT: s_or_b32 s1, s1, s2
-; GFX11-NEXT: s_bfe_u32 s2, s0, 0x80008
-; GFX11-NEXT: s_bfe_u32 s0, s0, 0x80010
-; GFX11-NEXT: s_lshl_b32 s2, s2, 8
-; GFX11-NEXT: s_lshl_b32 s0, s0, 16
-; GFX11-NEXT: s_or_b32 s2, s3, s2
-; GFX11-NEXT: s_and_b32 s3, s1, 0xff
-; GFX11-NEXT: s_or_b32 s0, s2, s0
-; GFX11-NEXT: s_lshl_b32 s2, s3, 24
-; GFX11-NEXT: s_bfe_u32 s3, s1, 0x80010
-; GFX11-NEXT: s_bfe_u32 s1, s1, 0x80008
-; GFX11-NEXT: s_lshl_b32 s3, s3, 8
-; GFX11-NEXT: s_or_b32 s0, s0, s2
-; GFX11-NEXT: s_or_b32 s1, s1, s3
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshl_v2i24:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_lshr_b32 s6, s0, 8
@@ -2413,6 +2373,7 @@ define amdgpu_ps i48 @s_fshl_v2i24(i48 inreg %lhs.arg, i48 inreg %rhs.arg, i48 i
; GFX9-NEXT: s_lshl_b32 s1, s1, 8
; GFX9-NEXT: s_or_b32 s1, s2, s1
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: s_fshl_v2i24:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_lshr_b32 s6, s0, 8
@@ -2537,6 +2498,116 @@ define amdgpu_ps i48 @s_fshl_v2i24(i48 inreg %lhs.arg, i48 inreg %rhs.arg, i48 i
; GFX10-NEXT: s_or_b32 s0, s0, s2
; GFX10-NEXT: s_or_b32 s1, s1, s3
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-TRUE16-LABEL: s_fshl_v2i24:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s1, 8
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s0, 24
+; GFX11-TRUE16-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s3, 8
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s3, 0xff
+; GFX11-TRUE16-NEXT: s_and_b32 s6, s6, 0xff
+; GFX11-TRUE16-NEXT: s_or_b32 s1, s8, s1
+; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s2, 24
+; GFX11-TRUE16-NEXT: s_lshl_b32 s3, s3, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s0, 16
+; GFX11-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-TRUE16-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, 0xff
+; GFX11-TRUE16-NEXT: s_lshl_b32 s6, s6, 8
+; GFX11-TRUE16-NEXT: s_and_b32 s8, s8, 0xff
+; GFX11-TRUE16-NEXT: s_or_b32 s3, s10, s3
+; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s4, 8
+; GFX11-TRUE16-NEXT: s_or_b32 s0, s0, s6
+; GFX11-TRUE16-NEXT: s_and_b32 s6, s7, 0xff
+; GFX11-TRUE16-NEXT: s_and_b32 s7, s9, 0xff
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s13, v0
+; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s2, 16
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, 0xff
+; GFX11-TRUE16-NEXT: s_lshl_b32 s8, s8, 8
+; GFX11-TRUE16-NEXT: s_and_b32 s10, s10, 0xff
+; GFX11-TRUE16-NEXT: s_or_b32 s2, s2, s8
+; GFX11-TRUE16-NEXT: s_and_b32 s8, s9, 0xff
+; GFX11-TRUE16-NEXT: s_and_b32 s9, s11, 0xff
+; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s4, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s4, 24
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s4, 0xff
+; GFX11-TRUE16-NEXT: s_lshl_b32 s10, s10, 8
+; GFX11-TRUE16-NEXT: s_and_b32 s11, s11, 0xff
+; GFX11-TRUE16-NEXT: s_or_b32 s4, s4, s10
+; GFX11-TRUE16-NEXT: s_mul_i32 s10, s13, 0xffffffe8
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s4, s4, s11
+; GFX11-TRUE16-NEXT: s_mul_hi_u32 s10, s13, s10
+; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s5, 8
+; GFX11-TRUE16-NEXT: s_add_i32 s13, s13, s10
+; GFX11-TRUE16-NEXT: s_and_b32 s5, s5, 0xff
+; GFX11-TRUE16-NEXT: s_mul_hi_u32 s10, s4, s13
+; GFX11-TRUE16-NEXT: s_lshl_b32 s5, s5, 8
+; GFX11-TRUE16-NEXT: s_mulk_i32 s10, 0xffe8
+; GFX11-TRUE16-NEXT: s_or_b32 s5, s12, s5
+; GFX11-TRUE16-NEXT: s_add_i32 s4, s4, s10
+; GFX11-TRUE16-NEXT: s_and_b32 s10, s14, 0xff
+; GFX11-TRUE16-NEXT: s_cmp_ge_u32 s4, 24
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s0, s0, s6
+; GFX11-TRUE16-NEXT: s_cselect_b32 s11, 1, 0
+; GFX11-TRUE16-NEXT: s_sub_i32 s12, s4, 24
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s11, 0
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s2, s2, s8
+; GFX11-TRUE16-NEXT: s_cselect_b32 s4, s12, s4
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s5, s5, s10
+; GFX11-TRUE16-NEXT: s_cmp_ge_u32 s4, 24
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s3, s3, s9
+; GFX11-TRUE16-NEXT: s_cselect_b32 s6, 1, 0
+; GFX11-TRUE16-NEXT: s_sub_i32 s8, s4, 24
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s6, 0
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s1, s1, s7
+; GFX11-TRUE16-NEXT: s_cselect_b32 s4, s8, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s2, 1
+; GFX11-TRUE16-NEXT: s_sub_i32 s6, 23, s4
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, s4
+; GFX11-TRUE16-NEXT: s_mul_hi_u32 s4, s5, s13
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s2, s6
+; GFX11-TRUE16-NEXT: s_mulk_i32 s4, 0xffe8
+; GFX11-TRUE16-NEXT: s_or_b32 s0, s0, s2
+; GFX11-TRUE16-NEXT: s_add_i32 s5, s5, s4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_cmp_ge_u32 s5, 24
+; GFX11-TRUE16-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-TRUE16-NEXT: s_sub_i32 s4, s5, 24
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
+; GFX11-TRUE16-NEXT: s_cselect_b32 s2, s4, s5
+; GFX11-TRUE16-NEXT: s_cmp_ge_u32 s2, 24
+; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-TRUE16-NEXT: s_sub_i32 s5, s2, 24
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 0
+; GFX11-TRUE16-NEXT: s_cselect_b32 s2, s5, s2
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s3, 1
+; GFX11-TRUE16-NEXT: s_sub_i32 s4, 23, s2
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, s2
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s3, s4
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s0, 0xff
+; GFX11-TRUE16-NEXT: s_or_b32 s1, s1, s2
+; GFX11-TRUE16-NEXT: s_bfe_u32 s2, s0, 0x80008
+; GFX11-TRUE16-NEXT: s_bfe_u32 s0, s0, 0x80010
+; GFX11-TRUE16-NEXT: s_lshl_b32 s2, s2, 8
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
+; GFX11-TRUE16-NEXT: s_or_b32 s2, s3, s2
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s1, 0xff
+; GFX11-TRUE16-NEXT: s_or_b32 s0, s2, s0
+; GFX11-TRUE16-NEXT: s_lshl_b32 s2, s3, 24
+; GFX11-TRUE16-NEXT: s_bfe_u32 s3, s1, 0x80010
+; GFX11-TRUE16-NEXT: s_bfe_u32 s1, s1, 0x80008
+; GFX11-TRUE16-NEXT: s_lshl_b32 s3, s3, 8
+; GFX11-TRUE16-NEXT: s_or_b32 s0, s0, s2
+; GFX11-TRUE16-NEXT: s_or_b32 s1, s1, s3
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
; GFX11-FAKE16-LABEL: s_fshl_v2i24:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_lshr_b32 s6, s0, 8
@@ -2771,58 +2842,6 @@ define <2 x i24> @v_fshl_v2i24(<2 x i24> %lhs, <2 x i24> %rhs, <2 x i24> %amt) {
; GFX8-NEXT: v_or_b32_e32 v1, v1, v2
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshl_v2i24:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v6, 24
-; GFX11-NEXT: v_and_b32_e32 v4, 0xffffff, v4
-; GFX11-NEXT: v_and_b32_e32 v5, 0xffffff, v5
-; GFX11-NEXT: v_bfe_u32 v2, v2, 1, 23
-; GFX11-NEXT: v_bfe_u32 v3, v3, 1, 23
-; GFX11-NEXT: v_rcp_iflag_f32_e32 v6, v6
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_mul_f32_e32 v6, 0x4f7ffffe, v6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cvt_u32_f32_e32 v6, v6
-; GFX11-NEXT: v_readfirstlane_b32 s0, v6
-; GFX11-NEXT: s_mul_i32 s1, s0, 0xffffffe8
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_mul_hi_u32 s1, s0, s1
-; GFX11-NEXT: s_add_i32 s0, s0, s1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_mul_hi_u32 v8, v4, s0
-; GFX11-NEXT: v_mad_u64_u32 v[6:7], null, 0xffffffe8, v8, v[4:5]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_add_nc_u32_e32 v4, 0xffffffe8, v6
-; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v6
-; GFX11-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc_lo
-; GFX11-NEXT: v_mul_hi_u32 v9, v5, s0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_mad_u64_u32 v[7:8], null, 0xffffffe8, v9, v[5:6]
-; GFX11-NEXT: v_add_nc_u32_e32 v6, 0xffffffe8, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_add_nc_u32_e32 v5, 0xffffffe8, v7
-; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v7
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc_lo
-; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_dual_cndmask_b32 v4, v4, v6 :: v_dual_add_nc_u32 v7, 0xffffffe8, v5
-; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v5
-; GFX11-NEXT: v_sub_nc_u32_e32 v6, 23, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_dual_cndmask_b32 v5, v5, v7 :: v_dual_and_b32 v4, 0xffffff, v4
-; GFX11-NEXT: v_and_b32_e32 v6, 0xffffff, v6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_sub_nc_u32_e32 v7, 23, v5
-; GFX11-NEXT: v_and_b32_e32 v5, 0xffffff, v5
-; GFX11-NEXT: v_lshrrev_b32_e32 v2, v6, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_and_b32_e32 v7, 0xffffff, v7
-; GFX11-NEXT: v_lshl_or_b32 v0, v0, v4, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, v7, v3
-; GFX11-NEXT: v_lshl_or_b32 v1, v1, v5, v3
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_v2i24:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2866,6 +2885,7 @@ define <2 x i24> @v_fshl_v2i24(<2 x i24> %lhs, <2 x i24> %rhs, <2 x i24> %amt) {
; GFX9-NEXT: v_lshrrev_b32_e32 v3, v4, v3
; GFX9-NEXT: v_lshl_or_b32 v1, v1, v2, v3
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshl_v2i24:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2908,33 +2928,86 @@ define <2 x i24> @v_fshl_v2i24(<2 x i24> %lhs, <2 x i24> %rhs, <2 x i24> %amt) {
; GFX10-NEXT: v_lshrrev_b32_e32 v3, v7, v3
; GFX10-NEXT: v_lshl_or_b32 v1, v1, v4, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_fshl_v2i24:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v6, 24
+; GFX11-NEXT: v_and_b32_e32 v4, 0xffffff, v4
+; GFX11-NEXT: v_and_b32_e32 v5, 0xffffff, v5
+; GFX11-NEXT: v_bfe_u32 v2, v2, 1, 23
+; GFX11-NEXT: v_bfe_u32 v3, v3, 1, 23
+; GFX11-NEXT: v_rcp_iflag_f32_e32 v6, v6
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-NEXT: v_mul_f32_e32 v6, 0x4f7ffffe, v6
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cvt_u32_f32_e32 v6, v6
+; GFX11-NEXT: v_readfirstlane_b32 s0, v6
+; GFX11-NEXT: s_mul_i32 s1, s0, 0xffffffe8
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_mul_hi_u32 s1, s0, s1
+; GFX11-NEXT: s_add_i32 s0, s0, s1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_mul_hi_u32 v8, v4, s0
+; GFX11-NEXT: v_mad_u64_u32 v[6:7], null, 0xffffffe8, v8, v[4:5]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_add_nc_u32_e32 v4, 0xffffffe8, v6
+; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v6
+; GFX11-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc_lo
+; GFX11-NEXT: v_mul_hi_u32 v9, v5, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_mad_u64_u32 v[7:8], null, 0xffffffe8, v9, v[5:6]
+; GFX11-NEXT: v_add_nc_u32_e32 v6, 0xffffffe8, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_add_nc_u32_e32 v5, 0xffffffe8, v7
+; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v7
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc_lo
+; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_dual_cndmask_b32 v4, v4, v6 :: v_dual_add_nc_u32 v7, 0xffffffe8, v5
+; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v5
+; GFX11-NEXT: v_sub_nc_u32_e32 v6, 23, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_dual_cndmask_b32 v5, v5, v7 :: v_dual_and_b32 v4, 0xffffff, v4
+; GFX11-NEXT: v_and_b32_e32 v6, 0xffffff, v6
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_sub_nc_u32_e32 v7, 23, v5
+; GFX11-NEXT: v_and_b32_e32 v5, 0xffffff, v5
+; GFX11-NEXT: v_lshrrev_b32_e32 v2, v6, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_and_b32_e32 v7, 0xffffff, v7
+; GFX11-NEXT: v_lshl_or_b32 v0, v0, v4, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v3, v7, v3
+; GFX11-NEXT: v_lshl_or_b32 v1, v1, v5, v3
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call <2 x i24> @llvm.fshl.v2i24(<2 x i24> %lhs, <2 x i24> %rhs, <2 x i24> %amt)
ret <2 x i24> %result
}
define amdgpu_ps i32 @s_fshl_i32(i32 inreg %lhs, i32 inreg %rhs, i32 inreg %amt) {
-; GCN-LABEL: s_fshl_i32:
-; GCN: ; %bb.0:
-; GCN-NEXT: v_mov_b32_e32 v0, s1
-; GCN-NEXT: s_not_b32 s1, s2
-; GCN-NEXT: v_alignbit_b32 v0, s0, v0, 1
-; GCN-NEXT: s_lshr_b32 s0, s0, 1
-; GCN-NEXT: v_mov_b32_e32 v1, s1
-; GCN-NEXT: v_alignbit_b32 v0, s0, v0, v1
-; GCN-NEXT: v_readfirstlane_b32 s0, v0
-; GCN-NEXT: ; return to shader part epilog
+; GFX6-LABEL: s_fshl_i32:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: v_mov_b32_e32 v0, s1
+; GFX6-NEXT: s_not_b32 s1, s2
+; GFX6-NEXT: v_alignbit_b32 v0, s0, v0, 1
+; GFX6-NEXT: s_lshr_b32 s0, s0, 1
+; GFX6-NEXT: v_mov_b32_e32 v1, s1
+; GFX6-NEXT: v_alignbit_b32 v0, s0, v0, v1
+; GFX6-NEXT: v_readfirstlane_b32 s0, v0
+; GFX6-NEXT: ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshl_i32:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: v_mov_b32_e32 v0, s1
+; GFX8-NEXT: s_not_b32 s1, s2
+; GFX8-NEXT: v_alignbit_b32 v0, s0, v0, 1
+; GFX8-NEXT: s_lshr_b32 s0, s0, 1
+; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: v_alignbit_b32 v0, s0, v0, v1
+; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshl_i32:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_not_b32 s2, s2
-; GFX11-NEXT: v_alignbit_b32 v0, s0, s1, 1
-; GFX11-NEXT: v_mov_b32_e32 v1, s2
-; GFX11-NEXT: s_lshr_b32 s0, s0, 1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_alignbit_b32 v0, s0, v0, v1.l
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshl_i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_mov_b32_e32 v0, s1
@@ -2945,6 +3018,7 @@ define amdgpu_ps i32 @s_fshl_i32(i32 inreg %lhs, i32 inreg %rhs, i32 inreg %amt)
; GFX9-NEXT: v_alignbit_b32 v0, s0, v0, v1
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: s_fshl_i32:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_alignbit_b32 v0, s0, s1, 1
@@ -2953,6 +3027,19 @@ define amdgpu_ps i32 @s_fshl_i32(i32 inreg %lhs, i32 inreg %rhs, i32 inreg %amt)
; GFX10-NEXT: v_alignbit_b32 v0, s0, v0, s1
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-TRUE16-LABEL: s_fshl_i32:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_not_b32 s2, s2
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, s0, s1, 1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s2
+; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, s0, v0, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
; GFX11-FAKE16-LABEL: s_fshl_i32:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, s0, s1, 1
@@ -2968,59 +3055,77 @@ define amdgpu_ps i32 @s_fshl_i32(i32 inreg %lhs, i32 inreg %rhs, i32 inreg %amt)
}
define amdgpu_ps i32 @s_fshl_i32_5(i32 inreg %lhs, i32 inreg %rhs) {
-; GCN-LABEL: s_fshl_i32_5:
-; GCN: ; %bb.0:
-; GCN-NEXT: v_mov_b32_e32 v0, s1
-; GCN-NEXT: v_alignbit_b32 v0, s0, v0, 27
-; GCN-NEXT: v_readfirstlane_b32 s0, v0
-; GCN-NEXT: ; return to shader part epilog
+; GFX6-LABEL: s_fshl_i32_5:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: v_mov_b32_e32 v0, s1
+; GFX6-NEXT: v_alignbit_b32 v0, s0, v0, 27
+; GFX6-NEXT: v_readfirstlane_b32 s0, v0
+; GFX6-NEXT: ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshl_i32_5:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: v_mov_b32_e32 v0, s1
+; GFX8-NEXT: v_alignbit_b32 v0, s0, v0, 27
+; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshl_i32_5:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_alignbit_b32 v0, s0, s1, 27
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshl_i32_5:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_mov_b32_e32 v0, s1
; GFX9-NEXT: v_alignbit_b32 v0, s0, v0, 27
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: s_fshl_i32_5:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_alignbit_b32 v0, s0, s1, 27
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: s_fshl_i32_5:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_alignbit_b32 v0, s0, s1, 27
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: ; return to shader part epilog
%result = call i32 @llvm.fshl.i32(i32 %lhs, i32 %rhs, i32 5)
ret i32 %result
}
define amdgpu_ps i32 @s_fshl_i32_8(i32 inreg %lhs, i32 inreg %rhs) {
-; GCN-LABEL: s_fshl_i32_8:
-; GCN: ; %bb.0:
-; GCN-NEXT: v_mov_b32_e32 v0, s1
-; GCN-NEXT: v_alignbit_b32 v0, s0, v0, 24
-; GCN-NEXT: v_readfirstlane_b32 s0, v0
-; GCN-NEXT: ; return to shader part epilog
+; GFX6-LABEL: s_fshl_i32_8:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: v_mov_b32_e32 v0, s1
+; GFX6-NEXT: v_alignbit_b32 v0, s0, v0, 24
+; GFX6-NEXT: v_readfirstlane_b32 s0, v0
+; GFX6-NEXT: ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshl_i32_8:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: v_mov_b32_e32 v0, s1
+; GFX8-NEXT: v_alignbit_b32 v0, s0, v0, 24
+; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshl_i32_8:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_alignbit_b32 v0, s0, s1, 24
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshl_i32_8:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_mov_b32_e32 v0, s1
; GFX9-NEXT: v_alignbit_b32 v0, s0, v0, 24
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: s_fshl_i32_8:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_alignbit_b32 v0, s0, s1, 24
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: s_fshl_i32_8:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_alignbit_b32 v0, s0, s1, 24
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: ; return to shader part epilog
%result = call i32 @llvm.fshl.i32(i32 %lhs, i32 %rhs, i32 8)
ret i32 %result
}
@@ -3035,15 +3140,16 @@ define i32 @v_fshl_i32(i32 %lhs, i32 %rhs, i32 %amt) {
; GCN-NEXT: v_alignbit_b32 v0, v0, v1, v2
; GCN-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshl_i32:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_alignbit_b32 v1, v0, v1, 1
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, 1, v0
-; GFX11-NEXT: v_not_b32_e32 v2, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_alignbit_b32 v0, v0, v1, v2.l
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: v_fshl_i32:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, v0, v1, 1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 1, v0
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, v0, v1, v2.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-FAKE16-LABEL: v_fshl_i32:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3090,23 +3196,24 @@ define i32 @v_fshl_i32_8(i32 %lhs, i32 %rhs) {
}
define amdgpu_ps float @v_fshl_i32_ssv(i32 inreg %lhs, i32 inreg %rhs, i32 %amt) {
-; GCN-LABEL: v_fshl_i32_ssv:
-; GCN: ; %bb.0:
-; GCN-NEXT: v_mov_b32_e32 v1, s1
-; GCN-NEXT: v_alignbit_b32 v1, s0, v1, 1
-; GCN-NEXT: s_lshr_b32 s0, s0, 1
-; GCN-NEXT: v_not_b32_e32 v0, v0
-; GCN-NEXT: v_alignbit_b32 v0, s0, v1, v0
-; GCN-NEXT: ; return to shader part epilog
+; GFX6-LABEL: v_fshl_i32_ssv:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: v_mov_b32_e32 v1, s1
+; GFX6-NEXT: v_alignbit_b32 v1, s0, v1, 1
+; GFX6-NEXT: s_lshr_b32 s0, s0, 1
+; GFX6-NEXT: v_not_b32_e32 v0, v0
+; GFX6-NEXT: v_alignbit_b32 v0, s0, v1, v0
+; GFX6-NEXT: ; return to shader part epilog
+;
+; GFX8-LABEL: v_fshl_i32_ssv:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: v_alignbit_b32 v1, s0, v1, 1
+; GFX8-NEXT: s_lshr_b32 s0, s0, 1
+; GFX8-NEXT: v_not_b32_e32 v0, v0
+; GFX8-NEXT: v_alignbit_b32 v0, s0, v1, v0
+; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: v_fshl_i32_ssv:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_alignbit_b32 v1, s0, s1, 1
-; GFX11-NEXT: v_not_b32_e32 v0, v0
-; GFX11-NEXT: s_lshr_b32 s0, s0, 1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_alignbit_b32 v0, s0, v1, v0.l
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshl_i32_ssv:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_mov_b32_e32 v1, s1
@@ -3115,6 +3222,7 @@ define amdgpu_ps float @v_fshl_i32_ssv(i32 inreg %lhs, i32 inreg %rhs, i32 %amt)
; GFX9-NEXT: v_not_b32_e32 v0, v0
; GFX9-NEXT: v_alignbit_b32 v0, s0, v1, v0
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: v_fshl_i32_ssv:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_alignbit_b32 v1, s0, s1, 1
@@ -3122,6 +3230,16 @@ define amdgpu_ps float @v_fshl_i32_ssv(i32 inreg %lhs, i32 inreg %rhs, i32 %amt)
; GFX10-NEXT: s_lshr_b32 s0, s0, 1
; GFX10-NEXT: v_alignbit_b32 v0, s0, v1, v0
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-TRUE16-LABEL: v_fshl_i32_ssv:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, s0, s1, 1
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v0, v0
+; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, s0, v1, v0.l
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
; GFX11-FAKE16-LABEL: v_fshl_i32_ssv:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, s0, s1, 1
@@ -3136,24 +3254,24 @@ define amdgpu_ps float @v_fshl_i32_ssv(i32 inreg %lhs, i32 inreg %rhs, i32 %amt)
}
define amdgpu_ps float @v_fshl_i32_svs(i32 inreg %lhs, i32 %rhs, i32 inreg %amt) {
-; GCN-LABEL: v_fshl_i32_svs:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_not_b32 s1, s1
-; GCN-NEXT: v_alignbit_b32 v0, s0, v0, 1
-; GCN-NEXT: s_lshr_b32 s0, s0, 1
-; GCN-NEXT: v_mov_b32_e32 v1, s1
-; GCN-NEXT: v_alignbit_b32 v0, s0, v0, v1
-; GCN-NEXT: ; return to shader part epilog
+; GFX6-LABEL: v_fshl_i32_svs:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_not_b32 s1, s1
+; GFX6-NEXT: v_alignbit_b32 v0, s0, v0, 1
+; GFX6-NEXT: s_lshr_b32 s0, s0, 1
+; GFX6-NEXT: v_mov_b32_e32 v1, s1
+; GFX6-NEXT: v_alignbit_b32 v0, s0, v0, v1
+; GFX6-NEXT: ; return to shader part epilog
+;
+; GFX8-LABEL: v_fshl_i32_svs:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_not_b32 s1, s1
+; GFX8-NEXT: v_alignbit_b32 v0, s0, v0, 1
+; GFX8-NEXT: s_lshr_b32 s0, s0, 1
+; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: v_alignbit_b32 v0, s0, v0, v1
+; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: v_fshl_i32_svs:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_not_b32 s1, s1
-; GFX11-NEXT: v_alignbit_b32 v0, s0, v0, 1
-; GFX11-NEXT: v_mov_b32_e32 v1, s1
-; GFX11-NEXT: s_lshr_b32 s0, s0, 1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_alignbit_b32 v0, s0, v0, v1.l
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshl_i32_svs:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_not_b32 s1, s1
@@ -3162,6 +3280,7 @@ define amdgpu_ps float @v_fshl_i32_svs(i32 inreg %lhs, i32 %rhs, i32 inreg %amt)
; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: v_alignbit_b32 v0, s0, v0, v1
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: v_fshl_i32_svs:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_alignbit_b32 v0, s0, v0, 1
@@ -3169,6 +3288,17 @@ define amdgpu_ps float @v_fshl_i32_svs(i32 inreg %lhs, i32 %rhs, i32 inreg %amt)
; GFX10-NEXT: s_not_b32 s1, s1
; GFX10-NEXT: v_alignbit_b32 v0, s0, v0, s1
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-TRUE16-LABEL: v_fshl_i32_svs:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_not_b32 s1, s1
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, s0, v0, 1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s1
+; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, s0, v0, v1.l
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
; GFX11-FAKE16-LABEL: v_fshl_i32_svs:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, s0, v0, 1
@@ -3183,25 +3313,26 @@ define amdgpu_ps float @v_fshl_i32_svs(i32 inreg %lhs, i32 %rhs, i32 inreg %amt)
}
define amdgpu_ps float @v_fshl_i32_vss(i32 inreg %lhs, i32 inreg %rhs, i32 inreg %amt) {
-; GCN-LABEL: v_fshl_i32_vss:
-; GCN: ; %bb.0:
-; GCN-NEXT: v_mov_b32_e32 v0, s1
-; GCN-NEXT: s_not_b32 s1, s2
-; GCN-NEXT: v_alignbit_b32 v0, s0, v0, 1
-; GCN-NEXT: s_lshr_b32 s0, s0, 1
-; GCN-NEXT: v_mov_b32_e32 v1, s1
-; GCN-NEXT: v_alignbit_b32 v0, s0, v0, v1
-; GCN-NEXT: ; return to shader part epilog
+; GFX6-LABEL: v_fshl_i32_vss:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: v_mov_b32_e32 v0, s1
+; GFX6-NEXT: s_not_b32 s1, s2
+; GFX6-NEXT: v_alignbit_b32 v0, s0, v0, 1
+; GFX6-NEXT: s_lshr_b32 s0, s0, 1
+; GFX6-NEXT: v_mov_b32_e32 v1, s1
+; GFX6-NEXT: v_alignbit_b32 v0, s0, v0, v1
+; GFX6-NEXT: ; return to shader part epilog
+;
+; GFX8-LABEL: v_fshl_i32_vss:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: v_mov_b32_e32 v0, s1
+; GFX8-NEXT: s_not_b32 s1, s2
+; GFX8-NEXT: v_alignbit_b32 v0, s0, v0, 1
+; GFX8-NEXT: s_lshr_b32 s0, s0, 1
+; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: v_alignbit_b32 v0, s0, v0, v1
+; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: v_fshl_i32_vss:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_not_b32 s2, s2
-; GFX11-NEXT: v_alignbit_b32 v0, s0, s1, 1
-; GFX11-NEXT: v_mov_b32_e32 v1, s2
-; GFX11-NEXT: s_lshr_b32 s0, s0, 1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_alignbit_b32 v0, s0, v0, v1.l
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshl_i32_vss:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_mov_b32_e32 v0, s1
@@ -3211,6 +3342,7 @@ define amdgpu_ps float @v_fshl_i32_vss(i32 inreg %lhs, i32 inreg %rhs, i32 inreg
; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: v_alignbit_b32 v0, s0, v0, v1
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: v_fshl_i32_vss:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_alignbit_b32 v0, s0, s1, 1
@@ -3218,6 +3350,17 @@ define amdgpu_ps float @v_fshl_i32_vss(i32 inreg %lhs, i32 inreg %rhs, i32 inreg
; GFX10-NEXT: s_not_b32 s1, s2
; GFX10-NEXT: v_alignbit_b32 v0, s0, v0, s1
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-TRUE16-LABEL: v_fshl_i32_vss:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_not_b32 s2, s2
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, s0, s1, 1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s2
+; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, s0, v0, v1.l
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
; GFX11-FAKE16-LABEL: v_fshl_i32_vss:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, s0, s1, 1
@@ -3232,32 +3375,32 @@ define amdgpu_ps float @v_fshl_i32_vss(i32 inreg %lhs, i32 inreg %rhs, i32 inreg
}
define <2 x i32> @v_fshl_v2i32(<2 x i32> %lhs, <2 x i32> %rhs, <2 x i32> %amt) {
-; GCN-LABEL: v_fshl_v2i32:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_alignbit_b32 v2, v0, v2, 1
-; GCN-NEXT: v_lshrrev_b32_e32 v0, 1, v0
-; GCN-NEXT: v_not_b32_e32 v4, v4
-; GCN-NEXT: v_alignbit_b32 v0, v0, v2, v4
-; GCN-NEXT: v_alignbit_b32 v2, v1, v3, 1
-; GCN-NEXT: v_lshrrev_b32_e32 v1, 1, v1
-; GCN-NEXT: v_not_b32_e32 v3, v5
-; GCN-NEXT: v_alignbit_b32 v1, v1, v2, v3
-; GCN-NEXT: s_setpc_b64 s[30:31]
+; GFX6-LABEL: v_fshl_v2i32:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_alignbit_b32 v2, v0, v2, 1
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, 1, v0
+; GFX6-NEXT: v_not_b32_e32 v4, v4
+; GFX6-NEXT: v_alignbit_b32 v0, v0, v2, v4
+; GFX6-NEXT: v_alignbit_b32 v2, v1, v3, 1
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, 1, v1
+; GFX6-NEXT: v_not_b32_e32 v3, v5
+; GFX6-NEXT: v_alignbit_b32 v1, v1, v2, v3
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshl_v2i32:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_alignbit_b32 v2, v0, v2, 1
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, 1, v0
+; GFX8-NEXT: v_not_b32_e32 v4, v4
+; GFX8-NEXT: v_alignbit_b32 v0, v0, v2, v4
+; GFX8-NEXT: v_alignbit_b32 v2, v1, v3, 1
+; GFX8-NEXT: v_lshrrev_b32_e32 v1, 1, v1
+; GFX8-NEXT: v_not_b32_e32 v3, v5
+; GFX8-NEXT: v_alignbit_b32 v1, v1, v2, v3
+; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshl_v2i32:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_alignbit_b32 v2, v0, v2, 1
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, 1, v0
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: v_alignbit_b32 v3, v1, v3, 1
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, 1, v1
-; GFX11-NEXT: v_not_b32_e32 v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_alignbit_b32 v0, v0, v2, v4.l
-; GFX11-NEXT: v_alignbit_b32 v1, v1, v3, v5.l
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_v2i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3270,6 +3413,7 @@ define <2 x i32> @v_fshl_v2i32(<2 x i32> %lhs, <2 x i32> %rhs, <2 x i32> %amt) {
; GFX9-NEXT: v_not_b32_e32 v3, v5
; GFX9-NEXT: v_alignbit_b32 v1, v1, v2, v3
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshl_v2i32:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3282,6 +3426,21 @@ define <2 x i32> @v_fshl_v2i32(<2 x i32> %lhs, <2 x i32> %rhs, <2 x i32> %amt) {
; GFX10-NEXT: v_alignbit_b32 v0, v0, v2, v4
; GFX10-NEXT: v_alignbit_b32 v1, v1, v3, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_fshl_v2i32:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v2, v0, v2, 1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 1, v0
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v3, v1, v3, 1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 1, v1
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, v0, v2, v4.l
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, v1, v3, v5.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-FAKE16-LABEL: v_fshl_v2i32:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3300,40 +3459,40 @@ define <2 x i32> @v_fshl_v2i32(<2 x i32> %lhs, <2 x i32> %rhs, <2 x i32> %amt) {
}
define <3 x i32> @v_fshl_v3i32(<3 x i32> %lhs, <3 x i32> %rhs, <3 x i32> %amt) {
-; GCN-LABEL: v_fshl_v3i32:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_alignbit_b32 v3, v0, v3, 1
-; GCN-NEXT: v_lshrrev_b32_e32 v0, 1, v0
-; GCN-NEXT: v_not_b32_e32 v6, v6
-; GCN-NEXT: v_alignbit_b32 v0, v0, v3, v6
-; GCN-NEXT: v_alignbit_b32 v3, v1, v4, 1
-; GCN-NEXT: v_lshrrev_b32_e32 v1, 1, v1
-; GCN-NEXT: v_not_b32_e32 v4, v7
-; GCN-NEXT: v_alignbit_b32 v1, v1, v3, v4
-; GCN-NEXT: v_alignbit_b32 v3, v2, v5, 1
-; GCN-NEXT: v_lshrrev_b32_e32 v2, 1, v2
-; GCN-NEXT: v_not_b32_e32 v4, v8
-; GCN-NEXT: v_alignbit_b32 v2, v2, v3, v4
-; GCN-NEXT: s_setpc_b64 s[30:31]
+; GFX6-LABEL: v_fshl_v3i32:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_alignbit_b32 v3, v0, v3, 1
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, 1, v0
+; GFX6-NEXT: v_not_b32_e32 v6, v6
+; GFX6-NEXT: v_alignbit_b32 v0, v0, v3, v6
+; GFX6-NEXT: v_alignbit_b32 v3, v1, v4, 1
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, 1, v1
+; GFX6-NEXT: v_not_b32_e32 v4, v7
+; GFX6-NEXT: v_alignbit_b32 v1, v1, v3, v4
+; GFX6-NEXT: v_alignbit_b32 v3, v2, v5, 1
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 1, v2
+; GFX6-NEXT: v_not_b32_e32 v4, v8
+; GFX6-NEXT: v_alignbit_b32 v2, v2, v3, v4
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshl_v3i32:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_alignbit_b32 v3, v0, v3, 1
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, 1, v0
+; GFX8-NEXT: v_not_b32_e32 v6, v6
+; GFX8-NEXT: v_alignbit_b32 v0, v0, v3, v6
+; GFX8-NEXT: v_alignbit_b32 v3, v1, v4, 1
+; GFX8-NEXT: v_lshrrev_b32_e32 v1, 1, v1
+; GFX8-NEXT: v_not_b32_e32 v4, v7
+; GFX8-NEXT: v_alignbit_b32 v1, v1, v3, v4
+; GFX8-NEXT: v_alignbit_b32 v3, v2, v5, 1
+; GFX8-NEXT: v_lshrrev_b32_e32 v2, 1, v2
+; GFX8-NEXT: v_not_b32_e32 v4, v8
+; GFX8-NEXT: v_alignbit_b32 v2, v2, v3, v4
+; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshl_v3i32:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_alignbit_b32 v3, v0, v3, 1
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, 1, v0
-; GFX11-NEXT: v_not_b32_e32 v6, v6
-; GFX11-NEXT: v_alignbit_b32 v4, v1, v4, 1
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, 1, v1
-; GFX11-NEXT: v_not_b32_e32 v7, v7
-; GFX11-NEXT: v_alignbit_b32 v5, v2, v5, 1
-; GFX11-NEXT: v_lshrrev_b32_e32 v2, 1, v2
-; GFX11-NEXT: v_not_b32_e32 v8, v8
-; GFX11-NEXT: v_alignbit_b32 v0, v0, v3, v6.l
-; GFX11-NEXT: v_alignbit_b32 v1, v1, v4, v7.l
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-NEXT: v_alignbit_b32 v2, v2, v5, v8.l
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_v3i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3350,6 +3509,7 @@ define <3 x i32> @v_fshl_v3i32(<3 x i32> %lhs, <3 x i32> %rhs, <3 x i32> %amt) {
; GFX9-NEXT: v_not_b32_e32 v4, v8
; GFX9-NEXT: v_alignbit_b32 v2, v2, v3, v4
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshl_v3i32:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3366,6 +3526,25 @@ define <3 x i32> @v_fshl_v3i32(<3 x i32> %lhs, <3 x i32> %rhs, <3 x i32> %amt) {
; GFX10-NEXT: v_alignbit_b32 v1, v1, v4, v7
; GFX10-NEXT: v_alignbit_b32 v2, v2, v5, v8
; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_fshl_v3i32:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v3, v0, v3, 1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 1, v0
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v6
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v4, v1, v4, 1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 1, v1
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v7, v7
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v5, v2, v5, 1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 1, v2
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v8, v8
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, v0, v3, v6.l
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, v1, v4, v7.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v2, v2, v5, v8.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-FAKE16-LABEL: v_fshl_v3i32:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3388,48 +3567,48 @@ define <3 x i32> @v_fshl_v3i32(<3 x i32> %lhs, <3 x i32> %rhs, <3 x i32> %amt) {
}
define <4 x i32> @v_fshl_v4i32(<4 x i32> %lhs, <4 x i32> %rhs, <4 x i32> %amt) {
-; GCN-LABEL: v_fshl_v4i32:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_alignbit_b32 v4, v0, v4, 1
-; GCN-NEXT: v_lshrrev_b32_e32 v0, 1, v0
-; GCN-NEXT: v_not_b32_e32 v8, v8
-; GCN-NEXT: v_alignbit_b32 v0, v0, v4, v8
-; GCN-NEXT: v_alignbit_b32 v4, v1, v5, 1
-; GCN-NEXT: v_lshrrev_b32_e32 v1, 1, v1
-; GCN-NEXT: v_not_b32_e32 v5, v9
-; GCN-NEXT: v_alignbit_b32 v1, v1, v4, v5
-; GCN-NEXT: v_alignbit_b32 v4, v2, v6, 1
-; GCN-NEXT: v_lshrrev_b32_e32 v2, 1, v2
-; GCN-NEXT: v_not_b32_e32 v5, v10
-; GCN-NEXT: v_alignbit_b32 v2, v2, v4, v5
-; GCN-NEXT: v_alignbit_b32 v4, v3, v7, 1
-; GCN-NEXT: v_lshrrev_b32_e32 v3, 1, v3
-; GCN-NEXT: v_not_b32_e32 v5, v11
-; GCN-NEXT: v_alignbit_b32 v3, v3, v4, v5
-; GCN-NEXT: s_setpc_b64 s[30:31]
+; GFX6-LABEL: v_fshl_v4i32:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_alignbit_b32 v4, v0, v4, 1
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, 1, v0
+; GFX6-NEXT: v_not_b32_e32 v8, v8
+; GFX6-NEXT: v_alignbit_b32 v0, v0, v4, v8
+; GFX6-NEXT: v_alignbit_b32 v4, v1, v5, 1
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, 1, v1
+; GFX6-NEXT: v_not_b32_e32 v5, v9
+; GFX6-NEXT: v_alignbit_b32 v1, v1, v4, v5
+; GFX6-NEXT: v_alignbit_b32 v4, v2, v6, 1
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 1, v2
+; GFX6-NEXT: v_not_b32_e32 v5, v10
+; GFX6-NEXT: v_alignbit_b32 v2, v2, v4, v5
+; GFX6-NEXT: v_alignbit_b32 v4, v3, v7, 1
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 1, v3
+; GFX6-NEXT: v_not_b32_e32 v5, v11
+; GFX6-NEXT: v_alignbit_b32 v3, v3, v4, v5
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fshl_v4i32:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_alignbit_b32 v4, v0, v4, 1
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, 1, v0
+; GFX8-NEXT: v_not_b32_e32 v8, v8
+; GFX8-NEXT: v_alignbit_b32 v0, v0, v4, v8
+; GFX8-NEXT: v_alignbit_b32 v4, v1, v5, 1
+; GFX8-NEXT: v_lshrrev_b32_e32 v1, 1, v1
+; GFX8-NEXT: v_not_b32_e32 v5, v9
+; GFX8-NEXT: v_alignbit_b32 v1, v1, v4, v5
+; GFX8-NEXT: v_alignbit_b32 v4, v2, v6, 1
+; GFX8-NEXT: v_lshrrev_b32_e32 v2, 1, v2
+; GFX8-NEXT: v_not_b32_e32 v5, v10
+; GFX8-NEXT: v_alignbit_b32 v2, v2, v4, v5
+; GFX8-NEXT: v_alignbit_b32 v4, v3, v7, 1
+; GFX8-NEXT: v_lshrrev_b32_e32 v3, 1, v3
+; GFX8-NEXT: v_not_b32_e32 v5, v11
+; GFX8-NEXT: v_alignbit_b32 v3, v3, v4, v5
+; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshl_v4i32:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_alignbit_b32 v4, v0, v4, 1
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, 1, v0
-; GFX11-NEXT: v_not_b32_e32 v8, v8
-; GFX11-NEXT: v_alignbit_b32 v5, v1, v5, 1
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, 1, v1
-; GFX11-NEXT: v_not_b32_e32 v9, v9
-; GFX11-NEXT: v_alignbit_b32 v6, v2, v6, 1
-; GFX11-NEXT: v_lshrrev_b32_e32 v2, 1, v2
-; GFX11-NEXT: v_not_b32_e32 v10, v10
-; GFX11-NEXT: v_alignbit_b32 v7, v3, v7, 1
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, 1, v3
-; GFX11-NEXT: v_not_b32_e32 v11, v11
-; GFX11-NEXT: v_alignbit_b32 v0, v0, v4, v8.l
-; GFX11-NEXT: v_alignbit_b32 v1, v1, v5, v9.l
-; GFX11-NEXT: v_alignbit_b32 v2, v2, v6, v10.l
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT: v_alignbit_b32 v3, v3, v7, v11.l
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_v4i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3450,6 +3629,7 @@ define <4 x i32> @v_fshl_v4i32(<4 x i32> %lhs, <4 x i32> %rhs, <4 x i32> %amt) {
; GFX9-NEXT: v_not_b32_e32 v5, v11
; GFX9-NEXT: v_alignbit_b32 v3, v3, v4, v5
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshl_v4i32:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3470,6 +3650,29 @@ define <4 x i32> @v_fshl_v4i32(<4 x i32> %lhs, <4 x i32> %rhs, <4 x i32> %amt) {
; GFX10-NEXT: v_alignbit_b32 v2, v2, v6, v10
; GFX10-NEXT: v_alignbit_b32 v3, v3, v7, v11
; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_fshl_v4i32:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v4, v0, v4, 1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 1, v0
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v8, v8
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v5, v1, v5, 1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 1, v1
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v9, v9
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v6, v2, v6, 1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 1, v2
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v10, v10
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v7, v3, v7, 1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 1, v3
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v11, v11
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, v0, v4, v8.l
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, v1, v5, v9.l
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v2, v2, v6, v10.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v3, v3, v7, v11.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-FAKE16-LABEL: v_fshl_v4i32:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3517,17 +3720,6 @@ define amdgpu_ps i16 @s_fshl_i16(i16 inreg %lhs, i16 inreg %rhs, i16 inreg %amt)
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshl_i16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX11-NEXT: s_and_b32 s3, s2, 15
-; GFX11-NEXT: s_and_not1_b32 s2, 15, s2
-; GFX11-NEXT: s_lshr_b32 s1, s1, 1
-; GFX11-NEXT: s_lshl_b32 s0, s0, s3
-; GFX11-NEXT: s_lshr_b32 s1, s1, s2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshl_i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s1, 0xffff, s1
@@ -3538,6 +3730,7 @@ define amdgpu_ps i16 @s_fshl_i16(i16 inreg %lhs, i16 inreg %rhs, i16 inreg %amt)
; GFX9-NEXT: s_lshr_b32 s1, s1, s2
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: s_fshl_i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s1, 0xffff, s1
@@ -3548,6 +3741,18 @@ define amdgpu_ps i16 @s_fshl_i16(i16 inreg %lhs, i16 inreg %rhs, i16 inreg %amt)
; GFX10-NEXT: s_lshr_b32 s1, s1, s2
; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: s_fshl_i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_and_b32 s3, s2, 15
+; GFX11-NEXT: s_and_not1_b32 s2, 15, s2
+; GFX11-NEXT: s_lshr_b32 s1, s1, 1
+; GFX11-NEXT: s_lshl_b32 s0, s0, s3
+; GFX11-NEXT: s_lshr_b32 s1, s1, s2
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: ; return to shader part epilog
%result = call i16 @llvm.fshl.i16(i16 %lhs, i16 %rhs, i16 %amt)
ret i16 %result
}
@@ -3568,14 +3773,6 @@ define amdgpu_ps i16 @s_fshl_i16_4(i16 inreg %lhs, i16 inreg %rhs) {
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshl_i16_4:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX11-NEXT: s_lshl_b32 s0, s0, 4
-; GFX11-NEXT: s_lshr_b32 s1, s1, 12
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshl_i16_4:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s1, 0xffff, s1
@@ -3583,6 +3780,7 @@ define amdgpu_ps i16 @s_fshl_i16_4(i16 inreg %lhs, i16 inreg %rhs) {
; GFX9-NEXT: s_lshr_b32 s1, s1, 12
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: s_fshl_i16_4:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s1, 0xffff, s1
@@ -3590,6 +3788,15 @@ define amdgpu_ps i16 @s_fshl_i16_4(i16 inreg %lhs, i16 inreg %rhs) {
; GFX10-NEXT: s_lshr_b32 s1, s1, 12
; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: s_fshl_i16_4:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_lshl_b32 s0, s0, 4
+; GFX11-NEXT: s_lshr_b32 s1, s1, 12
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: ; return to shader part epilog
%result = call i16 @llvm.fshl.i16(i16 %lhs, i16 %rhs, i16 4)
ret i16 %result
}
@@ -3610,14 +3817,6 @@ define amdgpu_ps i16 @s_fshl_i16_5(i16 inreg %lhs, i16 inreg %rhs) {
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshl_i16_5:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX11-NEXT: s_lshl_b32 s0, s0, 5
-; GFX11-NEXT: s_lshr_b32 s1, s1, 11
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshl_i16_5:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s1, 0xffff, s1
@@ -3625,6 +3824,7 @@ define amdgpu_ps i16 @s_fshl_i16_5(i16 inreg %lhs, i16 inreg %rhs) {
; GFX9-NEXT: s_lshr_b32 s1, s1, 11
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: s_fshl_i16_5:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s1, 0xffff, s1
@@ -3632,6 +3832,15 @@ define amdgpu_ps i16 @s_fshl_i16_5(i16 inreg %lhs, i16 inreg %rhs) {
; GFX10-NEXT: s_lshr_b32 s1, s1, 11
; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: s_fshl_i16_5:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_lshl_b32 s0, s0, 5
+; GFX11-NEXT: s_lshr_b32 s1, s1, 11
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: ; return to shader part epilog
%result = call i16 @llvm.fshl.i16(i16 %lhs, i16 %rhs, i16 5)
ret i16 %result
}
@@ -3661,19 +3870,6 @@ define i16 @v_fshl_i16(i16 %lhs, i16 %rhs, i16 %amt) {
; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshl_i16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_xor_b16 v0.h, v2.l, -1
-; GFX11-NEXT: v_and_b16 v1.h, v2.l, 15
-; GFX11-NEXT: v_lshrrev_b16 v1.l, 1, v1.l
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_and_b16 v0.h, v0.h, 15
-; GFX11-NEXT: v_lshlrev_b16 v0.l, v1.h, v0.l
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b16 v0.h, v0.h, v1.l
-; GFX11-NEXT: v_or_b16 v0.l, v0.l, v0.h
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3685,6 +3881,7 @@ define i16 @v_fshl_i16(i16 %lhs, i16 %rhs, i16 %amt) {
; GFX9-NEXT: v_lshrrev_b16_e32 v1, v2, v1
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshl_i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3696,6 +3893,21 @@ define i16 @v_fshl_i16(i16 %lhs, i16 %rhs, i16 %amt) {
; GFX10-NEXT: v_lshrrev_b16 v1, v3, v1
; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_fshl_i16:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_xor_b16 v0.h, v2.l, -1
+; GFX11-TRUE16-NEXT: v_and_b16 v1.h, v2.l, 15
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v1.l, 1, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v0.h, 15
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, v1.h, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, v0.h, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-FAKE16-LABEL: v_fshl_i16:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3730,14 +3942,6 @@ define i16 @v_fshl_i16_4(i16 %lhs, i16 %rhs) {
; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshl_i16_4:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshlrev_b16 v0.l, 4, v0.l
-; GFX11-NEXT: v_lshrrev_b16 v0.h, 12, v1.l
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_or_b16 v0.l, v0.l, v0.h
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_i16_4:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3745,6 +3949,7 @@ define i16 @v_fshl_i16_4(i16 %lhs, i16 %rhs) {
; GFX9-NEXT: v_lshrrev_b16_e32 v1, 12, v1
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshl_i16_4:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3752,6 +3957,16 @@ define i16 @v_fshl_i16_4(i16 %lhs, i16 %rhs) {
; GFX10-NEXT: v_lshrrev_b16 v1, 12, v1
; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_fshl_i16_4:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 4, v0.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, 12, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-FAKE16-LABEL: v_fshl_i16_4:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3781,14 +3996,6 @@ define i16 @v_fshl_i16_5(i16 %lhs, i16 %rhs) {
; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshl_i16_5:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshlrev_b16 v0.l, 5, v0.l
-; GFX11-NEXT: v_lshrrev_b16 v0.h, 11, v1.l
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_or_b16 v0.l, v0.l, v0.h
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_i16_5:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3796,6 +4003,7 @@ define i16 @v_fshl_i16_5(i16 %lhs, i16 %rhs) {
; GFX9-NEXT: v_lshrrev_b16_e32 v1, 11, v1
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshl_i16_5:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3803,6 +4011,16 @@ define i16 @v_fshl_i16_5(i16 %lhs, i16 %rhs) {
; GFX10-NEXT: v_lshrrev_b16 v1, 11, v1
; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_fshl_i16_5:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 5, v0.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, 11, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-FAKE16-LABEL: v_fshl_i16_5:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3839,20 +4057,6 @@ define amdgpu_ps half @v_fshl_i16_ssv(i16 inreg %lhs, i16 inreg %rhs, i16 %amt)
; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: v_fshl_i16_ssv:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_xor_b16 v0.h, v0.l, -1
-; GFX11-NEXT: v_and_b16 v0.l, v0.l, 15
-; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: s_lshr_b32 s1, s1, 1
-; GFX11-NEXT: v_and_b16 v0.h, v0.h, 15
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshlrev_b16 v0.l, v0.l, s0
-; GFX11-NEXT: v_lshrrev_b16 v0.h, v0.h, s1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_or_b16 v0.l, v0.l, v0.h
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshl_i16_ssv:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_and_b32_e32 v1, 15, v0
@@ -3864,6 +4068,7 @@ define amdgpu_ps half @v_fshl_i16_ssv(i16 inreg %lhs, i16 inreg %rhs, i16 %amt)
; GFX9-NEXT: v_lshrrev_b16_e64 v0, v0, s0
; GFX9-NEXT: v_or_b32_e32 v0, v1, v0
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: v_fshl_i16_ssv:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_xor_b32_e32 v1, -1, v0
@@ -3875,6 +4080,22 @@ define amdgpu_ps half @v_fshl_i16_ssv(i16 inreg %lhs, i16 inreg %rhs, i16 %amt)
; GFX10-NEXT: v_lshrrev_b16 v1, v1, s1
; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-TRUE16-LABEL: v_fshl_i16_ssv:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: v_xor_b16 v0.h, v0.l, -1
+; GFX11-TRUE16-NEXT: v_and_b16 v0.l, v0.l, 15
+; GFX11-TRUE16-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s1, 1
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v0.h, 15
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, v0.l, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, v0.h, s1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
; GFX11-FAKE16-LABEL: v_fshl_i16_ssv:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: v_xor_b32_e32 v1, -1, v0
@@ -3915,17 +4136,6 @@ define amdgpu_ps half @v_fshl_i16_svs(i16 inreg %lhs, i16 %rhs, i16 inreg %amt)
; GFX8-NEXT: v_or_b32_e32 v0, s0, v0
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: v_fshl_i16_svs:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_lshrrev_b16 v0.l, 1, v0.l
-; GFX11-NEXT: s_and_not1_b32 s2, 15, s1
-; GFX11-NEXT: s_and_b32 s1, s1, 15
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: s_lshl_b32 s0, s0, s1
-; GFX11-NEXT: v_lshrrev_b16 v0.l, s2, v0.l
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_or_b16 v0.l, s0, v0.l
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshl_i16_svs:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s2, s1, 15
@@ -3935,6 +4145,7 @@ define amdgpu_ps half @v_fshl_i16_svs(i16 inreg %lhs, i16 %rhs, i16 inreg %amt)
; GFX9-NEXT: v_lshrrev_b16_e32 v0, s1, v0
; GFX9-NEXT: v_or_b32_e32 v0, s0, v0
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: v_fshl_i16_svs:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_lshrrev_b16 v0, 1, v0
@@ -3944,6 +4155,19 @@ define amdgpu_ps half @v_fshl_i16_svs(i16 inreg %lhs, i16 %rhs, i16 inreg %amt)
; GFX10-NEXT: v_lshrrev_b16 v0, s2, v0
; GFX10-NEXT: v_or_b32_e32 v0, s0, v0
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-TRUE16-LABEL: v_fshl_i16_svs:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.l, 1, v0.l
+; GFX11-TRUE16-NEXT: s_and_not1_b32 s2, 15, s1
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, 15
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, s1
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.l, s2, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, s0, v0.l
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
; GFX11-FAKE16-LABEL: v_fshl_i16_svs:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: v_lshrrev_b16 v0, 1, v0
@@ -3982,18 +4206,6 @@ define amdgpu_ps half @v_fshl_i16_vss(i16 %lhs, i16 inreg %rhs, i16 inreg %amt)
; GFX8-NEXT: v_or_b32_e32 v0, s0, v0
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: v_fshl_i16_vss:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s2, s1, 15
-; GFX11-NEXT: s_and_b32 s0, 0xffff, s0
-; GFX11-NEXT: v_lshlrev_b16 v0.l, s2, v0.l
-; GFX11-NEXT: s_and_not1_b32 s1, 15, s1
-; GFX11-NEXT: s_lshr_b32 s0, s0, 1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshr_b32 s0, s0, s1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_or_b16 v0.l, v0.l, s0
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshl_i16_vss:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s0, 0xffff, s0
@@ -4004,6 +4216,7 @@ define amdgpu_ps half @v_fshl_i16_vss(i16 %lhs, i16 inreg %rhs, i16 inreg %amt)
; GFX9-NEXT: s_lshr_b32 s0, s0, s1
; GFX9-NEXT: v_or_b32_e32 v0, s0, v0
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: v_fshl_i16_vss:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s2, s1, 15
@@ -4014,6 +4227,20 @@ define amdgpu_ps half @v_fshl_i16_vss(i16 %lhs, i16 inreg %rhs, i16 inreg %amt)
; GFX10-NEXT: s_lshr_b32 s0, s0, s1
; GFX10-NEXT: v_or_b32_e32 v0, s0, v0
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-TRUE16-LABEL: v_fshl_i16_vss:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s1, 15
+; GFX11-TRUE16-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, s2, v0.l
+; GFX11-TRUE16-NEXT: s_and_not1_b32 s1, 15, s1
+; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, s1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, s0
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
; GFX11-FAKE16-LABEL: v_fshl_i16_vss:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_and_b32 s2, s1, 15
@@ -4077,29 +4304,6 @@ define amdgpu_ps i32 @s_fshl_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs, <
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshl_v2i16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s6, s1, 0xffff
-; GFX11-NEXT: s_lshr_b32 s1, s1, 16
-; GFX11-NEXT: s_and_b32 s3, s2, 0xf000f
-; GFX11-NEXT: s_lshr_b32 s6, s6, 0x10001
-; GFX11-NEXT: s_lshr_b32 s1, s1, 1
-; GFX11-NEXT: s_and_not1_b32 s2, 0xf000f, s2
-; GFX11-NEXT: s_lshr_b32 s4, s0, 16
-; GFX11-NEXT: s_lshr_b32 s5, s3, 16
-; GFX11-NEXT: s_pack_ll_b32_b16 s1, s6, s1
-; GFX11-NEXT: s_lshl_b32 s0, s0, s3
-; GFX11-NEXT: s_lshl_b32 s3, s4, s5
-; GFX11-NEXT: s_and_b32 s4, s1, 0xffff
-; GFX11-NEXT: s_lshr_b32 s1, s1, 16
-; GFX11-NEXT: s_lshr_b32 s5, s2, 16
-; GFX11-NEXT: s_lshr_b32 s2, s4, s2
-; GFX11-NEXT: s_lshr_b32 s1, s1, s5
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s3
-; GFX11-NEXT: s_pack_ll_b32_b16 s1, s2, s1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshl_v2i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s3, s2, 0xf000f
@@ -4122,6 +4326,7 @@ define amdgpu_ps i32 @s_fshl_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs, <
; GFX9-NEXT: s_pack_ll_b32_b16 s1, s2, s1
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: s_fshl_v2i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s6, s1, 0xffff
@@ -4144,11 +4349,35 @@ define amdgpu_ps i32 @s_fshl_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs, <
; GFX10-NEXT: s_pack_ll_b32_b16 s1, s2, s1
; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
- %result = call <2 x i16> @llvm.fshl.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt)
- %cast = bitcast <2 x i16> %result to i32
- ret i32 %cast
-}
-
+;
+; GFX11-LABEL: s_fshl_v2i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s6, s1, 0xffff
+; GFX11-NEXT: s_lshr_b32 s1, s1, 16
+; GFX11-NEXT: s_and_b32 s3, s2, 0xf000f
+; GFX11-NEXT: s_lshr_b32 s6, s6, 0x10001
+; GFX11-NEXT: s_lshr_b32 s1, s1, 1
+; GFX11-NEXT: s_and_not1_b32 s2, 0xf000f, s2
+; GFX11-NEXT: s_lshr_b32 s4, s0, 16
+; GFX11-NEXT: s_lshr_b32 s5, s3, 16
+; GFX11-NEXT: s_pack_ll_b32_b16 s1, s6, s1
+; GFX11-NEXT: s_lshl_b32 s0, s0, s3
+; GFX11-NEXT: s_lshl_b32 s3, s4, s5
+; GFX11-NEXT: s_and_b32 s4, s1, 0xffff
+; GFX11-NEXT: s_lshr_b32 s1, s1, 16
+; GFX11-NEXT: s_lshr_b32 s5, s2, 16
+; GFX11-NEXT: s_lshr_b32 s2, s4, s2
+; GFX11-NEXT: s_lshr_b32 s1, s1, s5
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s3
+; GFX11-NEXT: s_pack_ll_b32_b16 s1, s2, s1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: ; return to shader part epilog
+ %result = call <2 x i16> @llvm.fshl.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt)
+ %cast = bitcast <2 x i16> %result to i32
+ ret i32 %cast
+}
+
define <2 x i16> @v_fshl_v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt) {
; GFX6-LABEL: v_fshl_v2i16:
; GFX6: ; %bb.0:
@@ -4199,19 +4428,6 @@ define <2 x i16> @v_fshl_v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt) {
; GFX8-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshl_v2i16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_xor_b32_e32 v3, -1, v2
-; GFX11-NEXT: v_and_b32_e32 v2, 0xf000f, v2
-; GFX11-NEXT: v_pk_lshrrev_b16 v1, 1, v1 op_sel_hi:[0,1]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_and_b32_e32 v3, 0xf000f, v3
-; GFX11-NEXT: v_pk_lshlrev_b16 v0, v2, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_pk_lshrrev_b16 v1, v3, v1
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_v2i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -4223,6 +4439,7 @@ define <2 x i16> @v_fshl_v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt) {
; GFX9-NEXT: v_pk_lshrrev_b16 v1, v2, v1
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshl_v2i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -4234,6 +4451,20 @@ define <2 x i16> @v_fshl_v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt) {
; GFX10-NEXT: v_pk_lshrrev_b16 v1, v3, v1
; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_fshl_v2i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_xor_b32_e32 v3, -1, v2
+; GFX11-NEXT: v_and_b32_e32 v2, 0xf000f, v2
+; GFX11-NEXT: v_pk_lshrrev_b16 v1, 1, v1 op_sel_hi:[0,1]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_and_b32_e32 v3, 0xf000f, v3
+; GFX11-NEXT: v_pk_lshlrev_b16 v0, v2, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_pk_lshrrev_b16 v1, v3, v1
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call <2 x i16> @llvm.fshl.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt)
ret <2 x i16> %result
}
@@ -4268,14 +4499,6 @@ define <2 x i16> @v_fshl_v2i16_4_8(<2 x i16> %lhs, <2 x i16> %rhs) {
; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshl_v2i16_4_8:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_pk_lshlrev_b16 v0, 0x80004, v0
-; GFX11-NEXT: v_pk_lshrrev_b16 v1, 0x8000c, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_v2i16_4_8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -4285,6 +4508,7 @@ define <2 x i16> @v_fshl_v2i16_4_8(<2 x i16> %lhs, <2 x i16> %rhs) {
; GFX9-NEXT: v_pk_lshrrev_b16 v1, v2, v1
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshl_v2i16_4_8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -4292,6 +4516,15 @@ define <2 x i16> @v_fshl_v2i16_4_8(<2 x i16> %lhs, <2 x i16> %rhs) {
; GFX10-NEXT: v_pk_lshrrev_b16 v1, 0x8000c, v1
; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_fshl_v2i16_4_8:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_pk_lshlrev_b16 v0, 0x80004, v0
+; GFX11-NEXT: v_pk_lshrrev_b16 v1, 0x8000c, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call <2 x i16> @llvm.fshl.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> <i16 4, i16 8>)
ret <2 x i16> %result
}
@@ -4346,22 +4579,6 @@ define amdgpu_ps float @v_fshl_v2i16_ssv(<2 x i16> inreg %lhs, <2 x i16> inreg %
; GFX8-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: v_fshl_v2i16_ssv:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_xor_b32_e32 v1, -1, v0
-; GFX11-NEXT: s_and_b32 s2, s1, 0xffff
-; GFX11-NEXT: s_lshr_b32 s1, s1, 16
-; GFX11-NEXT: v_and_b32_e32 v0, 0xf000f, v0
-; GFX11-NEXT: s_lshr_b32 s2, s2, 0x10001
-; GFX11-NEXT: v_and_b32_e32 v1, 0xf000f, v1
-; GFX11-NEXT: s_lshr_b32 s1, s1, 1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: s_pack_ll_b32_b16 s1, s2, s1
-; GFX11-NEXT: v_pk_lshlrev_b16 v0, v0, s0
-; GFX11-NEXT: v_pk_lshrrev_b16 v1, v1, s1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshl_v2i16_ssv:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_and_b32_e32 v1, 0xf000f, v0
@@ -4376,6 +4593,7 @@ define amdgpu_ps float @v_fshl_v2i16_ssv(<2 x i16> inreg %lhs, <2 x i16> inreg %
; GFX9-NEXT: v_pk_lshrrev_b16 v0, v0, s0
; GFX9-NEXT: v_or_b32_e32 v0, v1, v0
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: v_fshl_v2i16_ssv:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_xor_b32_e32 v1, -1, v0
@@ -4390,6 +4608,23 @@ define amdgpu_ps float @v_fshl_v2i16_ssv(<2 x i16> inreg %lhs, <2 x i16> inreg %
; GFX10-NEXT: v_pk_lshrrev_b16 v1, v1, s1
; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: v_fshl_v2i16_ssv:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_xor_b32_e32 v1, -1, v0
+; GFX11-NEXT: s_and_b32 s2, s1, 0xffff
+; GFX11-NEXT: s_lshr_b32 s1, s1, 16
+; GFX11-NEXT: v_and_b32_e32 v0, 0xf000f, v0
+; GFX11-NEXT: s_lshr_b32 s2, s2, 0x10001
+; GFX11-NEXT: v_and_b32_e32 v1, 0xf000f, v1
+; GFX11-NEXT: s_lshr_b32 s1, s1, 1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: s_pack_ll_b32_b16 s1, s2, s1
+; GFX11-NEXT: v_pk_lshlrev_b16 v0, v0, s0
+; GFX11-NEXT: v_pk_lshrrev_b16 v1, v1, s1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-NEXT: ; return to shader part epilog
%result = call <2 x i16> @llvm.fshl.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt)
%cast = bitcast <2 x i16> %result to float
ret float %cast
@@ -4439,21 +4674,6 @@ define amdgpu_ps float @v_fshl_v2i16_svs(<2 x i16> inreg %lhs, <2 x i16> %rhs, <
; GFX8-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: v_fshl_v2i16_svs:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_pk_lshrrev_b16 v0, 1, v0 op_sel_hi:[0,1]
-; GFX11-NEXT: s_and_b32 s2, s1, 0xf000f
-; GFX11-NEXT: s_and_not1_b32 s1, 0xf000f, s1
-; GFX11-NEXT: s_lshr_b32 s3, s0, 16
-; GFX11-NEXT: s_lshr_b32 s4, s2, 16
-; GFX11-NEXT: v_pk_lshrrev_b16 v0, s1, v0
-; GFX11-NEXT: s_lshl_b32 s0, s0, s2
-; GFX11-NEXT: s_lshl_b32 s1, s3, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_or_b32_e32 v0, s0, v0
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshl_v2i16_svs:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s2, s1, 0xf000f
@@ -4467,6 +4687,7 @@ define amdgpu_ps float @v_fshl_v2i16_svs(<2 x i16> inreg %lhs, <2 x i16> %rhs, <
; GFX9-NEXT: v_pk_lshrrev_b16 v0, s1, v0
; GFX9-NEXT: v_or_b32_e32 v0, s0, v0
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: v_fshl_v2i16_svs:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_pk_lshrrev_b16 v0, 1, v0 op_sel_hi:[0,1]
@@ -4480,6 +4701,22 @@ define amdgpu_ps float @v_fshl_v2i16_svs(<2 x i16> inreg %lhs, <2 x i16> %rhs, <
; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX10-NEXT: v_or_b32_e32 v0, s0, v0
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: v_fshl_v2i16_svs:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_pk_lshrrev_b16 v0, 1, v0 op_sel_hi:[0,1]
+; GFX11-NEXT: s_and_b32 s2, s1, 0xf000f
+; GFX11-NEXT: s_and_not1_b32 s1, 0xf000f, s1
+; GFX11-NEXT: s_lshr_b32 s3, s0, 16
+; GFX11-NEXT: s_lshr_b32 s4, s2, 16
+; GFX11-NEXT: v_pk_lshrrev_b16 v0, s1, v0
+; GFX11-NEXT: s_lshl_b32 s0, s0, s2
+; GFX11-NEXT: s_lshl_b32 s1, s3, s4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX11-NEXT: ; return to shader part epilog
%result = call <2 x i16> @llvm.fshl.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt)
%cast = bitcast <2 x i16> %result to float
ret float %cast
@@ -4530,25 +4767,6 @@ define amdgpu_ps float @v_fshl_v2i16_vss(<2 x i16> %lhs, <2 x i16> inreg %rhs, <
; GFX8-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: v_fshl_v2i16_vss:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s3, s0, 0xffff
-; GFX11-NEXT: s_lshr_b32 s0, s0, 16
-; GFX11-NEXT: s_lshr_b32 s3, s3, 0x10001
-; GFX11-NEXT: s_lshr_b32 s0, s0, 1
-; GFX11-NEXT: s_and_b32 s2, s1, 0xf000f
-; GFX11-NEXT: s_and_not1_b32 s1, 0xf000f, s1
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s3, s0
-; GFX11-NEXT: v_pk_lshlrev_b16 v0, s2, v0
-; GFX11-NEXT: s_and_b32 s2, s0, 0xffff
-; GFX11-NEXT: s_lshr_b32 s0, s0, 16
-; GFX11-NEXT: s_lshr_b32 s3, s1, 16
-; GFX11-NEXT: s_lshr_b32 s1, s2, s1
-; GFX11-NEXT: s_lshr_b32 s0, s0, s3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s1, s0
-; GFX11-NEXT: v_or_b32_e32 v0, s0, v0
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshl_v2i16_vss:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s2, s1, 0xf000f
@@ -4567,6 +4785,7 @@ define amdgpu_ps float @v_fshl_v2i16_vss(<2 x i16> %lhs, <2 x i16> inreg %rhs, <
; GFX9-NEXT: s_pack_ll_b32_b16 s0, s1, s0
; GFX9-NEXT: v_or_b32_e32 v0, s0, v0
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: v_fshl_v2i16_vss:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s3, s0, 0xffff
@@ -4585,6 +4804,26 @@ define amdgpu_ps float @v_fshl_v2i16_vss(<2 x i16> %lhs, <2 x i16> inreg %rhs, <
; GFX10-NEXT: s_pack_ll_b32_b16 s0, s1, s0
; GFX10-NEXT: v_or_b32_e32 v0, s0, v0
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: v_fshl_v2i16_vss:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s3, s0, 0xffff
+; GFX11-NEXT: s_lshr_b32 s0, s0, 16
+; GFX11-NEXT: s_lshr_b32 s3, s3, 0x10001
+; GFX11-NEXT: s_lshr_b32 s0, s0, 1
+; GFX11-NEXT: s_and_b32 s2, s1, 0xf000f
+; GFX11-NEXT: s_and_not1_b32 s1, 0xf000f, s1
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s3, s0
+; GFX11-NEXT: v_pk_lshlrev_b16 v0, s2, v0
+; GFX11-NEXT: s_and_b32 s2, s0, 0xffff
+; GFX11-NEXT: s_lshr_b32 s0, s0, 16
+; GFX11-NEXT: s_lshr_b32 s3, s1, 16
+; GFX11-NEXT: s_lshr_b32 s1, s2, s1
+; GFX11-NEXT: s_lshr_b32 s0, s0, s3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s1, s0
+; GFX11-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX11-NEXT: ; return to shader part epilog
%result = call <2 x i16> @llvm.fshl.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt)
%cast = bitcast <2 x i16> %result to float
ret float %cast
@@ -4652,47 +4891,6 @@ define amdgpu_ps i48 @s_fshl_v3i16(<3 x i16> inreg %lhs, <3 x i16> inreg %rhs, <
; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshl_v3i16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s9, s2, 0xffff
-; GFX11-NEXT: s_lshr_b32 s2, s2, 16
-; GFX11-NEXT: s_and_b32 s6, s4, 0xf000f
-; GFX11-NEXT: s_lshr_b32 s9, s9, 0x10001
-; GFX11-NEXT: s_lshr_b32 s2, s2, 1
-; GFX11-NEXT: s_and_not1_b32 s4, 0xf000f, s4
-; GFX11-NEXT: s_lshr_b32 s7, s0, 16
-; GFX11-NEXT: s_lshr_b32 s8, s6, 16
-; GFX11-NEXT: s_pack_ll_b32_b16 s2, s9, s2
-; GFX11-NEXT: s_lshl_b32 s0, s0, s6
-; GFX11-NEXT: s_lshl_b32 s6, s7, s8
-; GFX11-NEXT: s_and_b32 s7, s2, 0xffff
-; GFX11-NEXT: s_lshr_b32 s2, s2, 16
-; GFX11-NEXT: s_lshr_b32 s8, s4, 16
-; GFX11-NEXT: s_lshr_b32 s4, s7, s4
-; GFX11-NEXT: s_lshr_b32 s2, s2, s8
-; GFX11-NEXT: s_and_b32 s8, s3, 0xffff
-; GFX11-NEXT: s_lshr_b32 s3, s3, 16
-; GFX11-NEXT: s_pack_ll_b32_b16 s2, s4, s2
-; GFX11-NEXT: s_and_b32 s4, s5, 0xf000f
-; GFX11-NEXT: s_lshr_b32 s8, s8, 0x10001
-; GFX11-NEXT: s_lshr_b32 s3, s3, 1
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s6
-; GFX11-NEXT: s_and_not1_b32 s5, 0xf000f, s5
-; GFX11-NEXT: s_lshr_b32 s6, s1, 16
-; GFX11-NEXT: s_lshr_b32 s7, s4, 16
-; GFX11-NEXT: s_pack_ll_b32_b16 s3, s8, s3
-; GFX11-NEXT: s_lshl_b32 s1, s1, s4
-; GFX11-NEXT: s_lshl_b32 s4, s6, s7
-; GFX11-NEXT: s_and_b32 s6, s3, 0xffff
-; GFX11-NEXT: s_lshr_b32 s3, s3, 16
-; GFX11-NEXT: s_lshr_b32 s7, s5, 16
-; GFX11-NEXT: s_lshr_b32 s5, s6, s5
-; GFX11-NEXT: s_lshr_b32 s3, s3, s7
-; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s4
-; GFX11-NEXT: s_pack_ll_b32_b16 s3, s5, s3
-; GFX11-NEXT: s_or_b32 s0, s0, s2
-; GFX11-NEXT: s_or_b32 s1, s1, s3
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshl_v3i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s6, s4, 0xf000f
@@ -4739,6 +4937,7 @@ define amdgpu_ps i48 @s_fshl_v3i16(<3 x i16> inreg %lhs, <3 x i16> inreg %rhs, <
; GFX9-NEXT: s_or_b32 s0, s0, s2
; GFX9-NEXT: s_and_b32 s1, s1, 0xffff
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: s_fshl_v3i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s9, s2, 0xffff
@@ -4785,6 +4984,49 @@ define amdgpu_ps i48 @s_fshl_v3i16(<3 x i16> inreg %lhs, <3 x i16> inreg %rhs, <
; GFX10-NEXT: s_or_b32 s0, s0, s3
; GFX10-NEXT: s_and_b32 s1, s1, 0xffff
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-TRUE16-LABEL: s_fshl_v3i16:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_and_b32 s9, s2, 0xffff
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s2, 16
+; GFX11-TRUE16-NEXT: s_and_b32 s6, s4, 0xf000f
+; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s9, 0x10001
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s2, 1
+; GFX11-TRUE16-NEXT: s_and_not1_b32 s4, 0xf000f, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s6, 16
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s2, s9, s2
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, s6
+; GFX11-TRUE16-NEXT: s_lshl_b32 s6, s7, s8
+; GFX11-TRUE16-NEXT: s_and_b32 s7, s2, 0xffff
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s4, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s7, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s2, s8
+; GFX11-TRUE16-NEXT: s_and_b32 s8, s3, 0xffff
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s3, 16
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s2, s4, s2
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s5, 0xf000f
+; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s8, 0x10001
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s3, 1
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s0, s0, s6
+; GFX11-TRUE16-NEXT: s_and_not1_b32 s5, 0xf000f, s5
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s4, 16
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s3, s8, s3
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, s4
+; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s6, s7
+; GFX11-TRUE16-NEXT: s_and_b32 s6, s3, 0xffff
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s3, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s5, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s6, s5
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s3, s7
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s1, s1, s4
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s3, s5, s3
+; GFX11-TRUE16-NEXT: s_or_b32 s0, s0, s2
+; GFX11-TRUE16-NEXT: s_or_b32 s1, s1, s3
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
; GFX11-FAKE16-LABEL: s_fshl_v3i16:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_and_b32 s9, s2, 0xffff
@@ -4902,26 +5144,6 @@ define <3 x half> @v_fshl_v3i16(<3 x i16> %lhs, <3 x i16> %rhs, <3 x i16> %amt)
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshl_v3i16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_xor_b32_e32 v6, -1, v4
-; GFX11-NEXT: v_xor_b32_e32 v7, -1, v5
-; GFX11-NEXT: v_and_b32_e32 v4, 0xf000f, v4
-; GFX11-NEXT: v_pk_lshrrev_b16 v2, 1, v2 op_sel_hi:[0,1]
-; GFX11-NEXT: v_and_b32_e32 v5, 0xf000f, v5
-; GFX11-NEXT: v_and_b32_e32 v6, 0xf000f, v6
-; GFX11-NEXT: v_pk_lshrrev_b16 v3, 1, v3 op_sel_hi:[0,1]
-; GFX11-NEXT: v_and_b32_e32 v7, 0xf000f, v7
-; GFX11-NEXT: v_pk_lshlrev_b16 v0, v4, v0
-; GFX11-NEXT: v_pk_lshlrev_b16 v1, v5, v1
-; GFX11-NEXT: v_pk_lshrrev_b16 v2, v6, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_pk_lshrrev_b16 v3, v7, v3
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_v3i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -4940,6 +5162,7 @@ define <3 x half> @v_fshl_v3i16(<3 x i16> %lhs, <3 x i16> %rhs, <3 x i16> %amt)
; GFX9-NEXT: v_pk_lshrrev_b16 v2, v4, v2
; GFX9-NEXT: v_or_b32_e32 v1, v1, v2
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshl_v3i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -4958,17 +5181,38 @@ define <3 x half> @v_fshl_v3i16(<3 x i16> %lhs, <3 x i16> %rhs, <3 x i16> %amt)
; GFX10-NEXT: v_or_b32_e32 v0, v0, v2
; GFX10-NEXT: v_or_b32_e32 v1, v1, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
- %result = call <3 x i16> @llvm.fshl.v3i16(<3 x i16> %lhs, <3 x i16> %rhs, <3 x i16> %amt)
- %cast.result = bitcast <3 x i16> %result to <3 x half>
- ret <3 x half> %cast.result
-}
-
-define amdgpu_ps <2 x i32> @s_fshl_v4i16(<4 x i16> inreg %lhs, <4 x i16> inreg %rhs, <4 x i16> inreg %amt) {
-; GFX6-LABEL: s_fshl_v4i16:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: s_and_b32 s10, s4, 15
-; GFX6-NEXT: s_lshr_b32 s6, s0, 16
-; GFX6-NEXT: s_lshr_b32 s8, s4, 16
+;
+; GFX11-LABEL: v_fshl_v3i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_xor_b32_e32 v6, -1, v4
+; GFX11-NEXT: v_xor_b32_e32 v7, -1, v5
+; GFX11-NEXT: v_and_b32_e32 v4, 0xf000f, v4
+; GFX11-NEXT: v_pk_lshrrev_b16 v2, 1, v2 op_sel_hi:[0,1]
+; GFX11-NEXT: v_and_b32_e32 v5, 0xf000f, v5
+; GFX11-NEXT: v_and_b32_e32 v6, 0xf000f, v6
+; GFX11-NEXT: v_pk_lshrrev_b16 v3, 1, v3 op_sel_hi:[0,1]
+; GFX11-NEXT: v_and_b32_e32 v7, 0xf000f, v7
+; GFX11-NEXT: v_pk_lshlrev_b16 v0, v4, v0
+; GFX11-NEXT: v_pk_lshlrev_b16 v1, v5, v1
+; GFX11-NEXT: v_pk_lshrrev_b16 v2, v6, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_pk_lshrrev_b16 v3, v7, v3
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %result = call <3 x i16> @llvm.fshl.v3i16(<3 x i16> %lhs, <3 x i16> %rhs, <3 x i16> %amt)
+ %cast.result = bitcast <3 x i16> %result to <3 x half>
+ ret <3 x half> %cast.result
+}
+
+define amdgpu_ps <2 x i32> @s_fshl_v4i16(<4 x i16> inreg %lhs, <4 x i16> inreg %rhs, <4 x i16> inreg %amt) {
+; GFX6-LABEL: s_fshl_v4i16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_and_b32 s10, s4, 15
+; GFX6-NEXT: s_lshr_b32 s6, s0, 16
+; GFX6-NEXT: s_lshr_b32 s8, s4, 16
; GFX6-NEXT: s_andn2_b32 s4, 15, s4
; GFX6-NEXT: s_lshl_b32 s0, s0, s10
; GFX6-NEXT: s_bfe_u32 s10, s2, 0xf0001
@@ -5046,47 +5290,6 @@ define amdgpu_ps <2 x i32> @s_fshl_v4i16(<4 x i16> inreg %lhs, <4 x i16> inreg %
; GFX8-NEXT: s_or_b32 s1, s1, s2
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshl_v4i16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s9, s2, 0xffff
-; GFX11-NEXT: s_lshr_b32 s2, s2, 16
-; GFX11-NEXT: s_and_b32 s6, s4, 0xf000f
-; GFX11-NEXT: s_lshr_b32 s9, s9, 0x10001
-; GFX11-NEXT: s_lshr_b32 s2, s2, 1
-; GFX11-NEXT: s_and_not1_b32 s4, 0xf000f, s4
-; GFX11-NEXT: s_lshr_b32 s7, s0, 16
-; GFX11-NEXT: s_lshr_b32 s8, s6, 16
-; GFX11-NEXT: s_pack_ll_b32_b16 s2, s9, s2
-; GFX11-NEXT: s_lshl_b32 s0, s0, s6
-; GFX11-NEXT: s_lshl_b32 s6, s7, s8
-; GFX11-NEXT: s_and_b32 s7, s2, 0xffff
-; GFX11-NEXT: s_lshr_b32 s2, s2, 16
-; GFX11-NEXT: s_lshr_b32 s8, s4, 16
-; GFX11-NEXT: s_lshr_b32 s4, s7, s4
-; GFX11-NEXT: s_lshr_b32 s2, s2, s8
-; GFX11-NEXT: s_and_b32 s8, s3, 0xffff
-; GFX11-NEXT: s_lshr_b32 s3, s3, 16
-; GFX11-NEXT: s_pack_ll_b32_b16 s2, s4, s2
-; GFX11-NEXT: s_and_b32 s4, s5, 0xf000f
-; GFX11-NEXT: s_lshr_b32 s8, s8, 0x10001
-; GFX11-NEXT: s_lshr_b32 s3, s3, 1
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s6
-; GFX11-NEXT: s_and_not1_b32 s5, 0xf000f, s5
-; GFX11-NEXT: s_lshr_b32 s6, s1, 16
-; GFX11-NEXT: s_lshr_b32 s7, s4, 16
-; GFX11-NEXT: s_pack_ll_b32_b16 s3, s8, s3
-; GFX11-NEXT: s_lshl_b32 s1, s1, s4
-; GFX11-NEXT: s_lshl_b32 s4, s6, s7
-; GFX11-NEXT: s_and_b32 s6, s3, 0xffff
-; GFX11-NEXT: s_lshr_b32 s3, s3, 16
-; GFX11-NEXT: s_lshr_b32 s7, s5, 16
-; GFX11-NEXT: s_lshr_b32 s5, s6, s5
-; GFX11-NEXT: s_lshr_b32 s3, s3, s7
-; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s4
-; GFX11-NEXT: s_pack_ll_b32_b16 s3, s5, s3
-; GFX11-NEXT: s_or_b32 s0, s0, s2
-; GFX11-NEXT: s_or_b32 s1, s1, s3
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshl_v4i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s6, s4, 0xf000f
@@ -5128,6 +5331,7 @@ define amdgpu_ps <2 x i32> @s_fshl_v4i16(<4 x i16> inreg %lhs, <4 x i16> inreg %
; GFX9-NEXT: s_pack_ll_b32_b16 s2, s3, s2
; GFX9-NEXT: s_or_b32 s1, s1, s2
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: s_fshl_v4i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s9, s2, 0xffff
@@ -5169,6 +5373,48 @@ define amdgpu_ps <2 x i32> @s_fshl_v4i16(<4 x i16> inreg %lhs, <4 x i16> inreg %
; GFX10-NEXT: s_or_b32 s0, s0, s2
; GFX10-NEXT: s_or_b32 s1, s1, s3
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: s_fshl_v4i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s9, s2, 0xffff
+; GFX11-NEXT: s_lshr_b32 s2, s2, 16
+; GFX11-NEXT: s_and_b32 s6, s4, 0xf000f
+; GFX11-NEXT: s_lshr_b32 s9, s9, 0x10001
+; GFX11-NEXT: s_lshr_b32 s2, s2, 1
+; GFX11-NEXT: s_and_not1_b32 s4, 0xf000f, s4
+; GFX11-NEXT: s_lshr_b32 s7, s0, 16
+; GFX11-NEXT: s_lshr_b32 s8, s6, 16
+; GFX11-NEXT: s_pack_ll_b32_b16 s2, s9, s2
+; GFX11-NEXT: s_lshl_b32 s0, s0, s6
+; GFX11-NEXT: s_lshl_b32 s6, s7, s8
+; GFX11-NEXT: s_and_b32 s7, s2, 0xffff
+; GFX11-NEXT: s_lshr_b32 s2, s2, 16
+; GFX11-NEXT: s_lshr_b32 s8, s4, 16
+; GFX11-NEXT: s_lshr_b32 s4, s7, s4
+; GFX11-NEXT: s_lshr_b32 s2, s2, s8
+; GFX11-NEXT: s_and_b32 s8, s3, 0xffff
+; GFX11-NEXT: s_lshr_b32 s3, s3, 16
+; GFX11-NEXT: s_pack_ll_b32_b16 s2, s4, s2
+; GFX11-NEXT: s_and_b32 s4, s5, 0xf000f
+; GFX11-NEXT: s_lshr_b32 s8, s8, 0x10001
+; GFX11-NEXT: s_lshr_b32 s3, s3, 1
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s6
+; GFX11-NEXT: s_and_not1_b32 s5, 0xf000f, s5
+; GFX11-NEXT: s_lshr_b32 s6, s1, 16
+; GFX11-NEXT: s_lshr_b32 s7, s4, 16
+; GFX11-NEXT: s_pack_ll_b32_b16 s3, s8, s3
+; GFX11-NEXT: s_lshl_b32 s1, s1, s4
+; GFX11-NEXT: s_lshl_b32 s4, s6, s7
+; GFX11-NEXT: s_and_b32 s6, s3, 0xffff
+; GFX11-NEXT: s_lshr_b32 s3, s3, 16
+; GFX11-NEXT: s_lshr_b32 s7, s5, 16
+; GFX11-NEXT: s_lshr_b32 s5, s6, s5
+; GFX11-NEXT: s_lshr_b32 s3, s3, s7
+; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s4
+; GFX11-NEXT: s_pack_ll_b32_b16 s3, s5, s3
+; GFX11-NEXT: s_or_b32 s0, s0, s2
+; GFX11-NEXT: s_or_b32 s1, s1, s3
+; GFX11-NEXT: ; return to shader part epilog
%result = call <4 x i16> @llvm.fshl.v4i16(<4 x i16> %lhs, <4 x i16> %rhs, <4 x i16> %amt)
%cast.result = bitcast <4 x i16> %result to <2 x i32>
ret <2 x i32> %cast.result
@@ -5261,26 +5507,6 @@ define <4 x half> @v_fshl_v4i16(<4 x i16> %lhs, <4 x i16> %rhs, <4 x i16> %amt)
; GFX8-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshl_v4i16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_xor_b32_e32 v6, -1, v4
-; GFX11-NEXT: v_xor_b32_e32 v7, -1, v5
-; GFX11-NEXT: v_and_b32_e32 v4, 0xf000f, v4
-; GFX11-NEXT: v_pk_lshrrev_b16 v2, 1, v2 op_sel_hi:[0,1]
-; GFX11-NEXT: v_and_b32_e32 v5, 0xf000f, v5
-; GFX11-NEXT: v_and_b32_e32 v6, 0xf000f, v6
-; GFX11-NEXT: v_pk_lshrrev_b16 v3, 1, v3 op_sel_hi:[0,1]
-; GFX11-NEXT: v_and_b32_e32 v7, 0xf000f, v7
-; GFX11-NEXT: v_pk_lshlrev_b16 v0, v4, v0
-; GFX11-NEXT: v_pk_lshlrev_b16 v1, v5, v1
-; GFX11-NEXT: v_pk_lshrrev_b16 v2, v6, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_pk_lshrrev_b16 v3, v7, v3
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_v4i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -5299,6 +5525,7 @@ define <4 x half> @v_fshl_v4i16(<4 x i16> %lhs, <4 x i16> %rhs, <4 x i16> %amt)
; GFX9-NEXT: v_pk_lshrrev_b16 v2, v4, v2
; GFX9-NEXT: v_or_b32_e32 v1, v1, v2
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshl_v4i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -5317,30 +5544,51 @@ define <4 x half> @v_fshl_v4i16(<4 x i16> %lhs, <4 x i16> %rhs, <4 x i16> %amt)
; GFX10-NEXT: v_or_b32_e32 v0, v0, v2
; GFX10-NEXT: v_or_b32_e32 v1, v1, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_fshl_v4i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_xor_b32_e32 v6, -1, v4
+; GFX11-NEXT: v_xor_b32_e32 v7, -1, v5
+; GFX11-NEXT: v_and_b32_e32 v4, 0xf000f, v4
+; GFX11-NEXT: v_pk_lshrrev_b16 v2, 1, v2 op_sel_hi:[0,1]
+; GFX11-NEXT: v_and_b32_e32 v5, 0xf000f, v5
+; GFX11-NEXT: v_and_b32_e32 v6, 0xf000f, v6
+; GFX11-NEXT: v_pk_lshrrev_b16 v3, 1, v3 op_sel_hi:[0,1]
+; GFX11-NEXT: v_and_b32_e32 v7, 0xf000f, v7
+; GFX11-NEXT: v_pk_lshlrev_b16 v0, v4, v0
+; GFX11-NEXT: v_pk_lshlrev_b16 v1, v5, v1
+; GFX11-NEXT: v_pk_lshrrev_b16 v2, v6, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_pk_lshrrev_b16 v3, v7, v3
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x i16> @llvm.fshl.v4i16(<4 x i16> %lhs, <4 x i16> %rhs, <4 x i16> %amt)
%cast.result = bitcast <4 x i16> %result to <4 x half>
ret <4 x half> %cast.result
}
define amdgpu_ps i64 @s_fshl_i64(i64 inreg %lhs, i64 inreg %rhs, i64 inreg %amt) {
-; GCN-LABEL: s_fshl_i64:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_lshl_b64 s[0:1], s[0:1], s4
-; GCN-NEXT: s_lshr_b64 s[2:3], s[2:3], 1
-; GCN-NEXT: s_not_b32 s4, s4
-; GCN-NEXT: s_lshr_b64 s[2:3], s[2:3], s4
-; GCN-NEXT: s_or_b64 s[0:1], s[0:1], s[2:3]
-; GCN-NEXT: ; return to shader part epilog
+; GFX6-LABEL: s_fshl_i64:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_lshl_b64 s[0:1], s[0:1], s4
+; GFX6-NEXT: s_lshr_b64 s[2:3], s[2:3], 1
+; GFX6-NEXT: s_not_b32 s4, s4
+; GFX6-NEXT: s_lshr_b64 s[2:3], s[2:3], s4
+; GFX6-NEXT: s_or_b64 s[0:1], s[0:1], s[2:3]
+; GFX6-NEXT: ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshl_i64:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_lshl_b64 s[0:1], s[0:1], s4
+; GFX8-NEXT: s_lshr_b64 s[2:3], s[2:3], 1
+; GFX8-NEXT: s_not_b32 s4, s4
+; GFX8-NEXT: s_lshr_b64 s[2:3], s[2:3], s4
+; GFX8-NEXT: s_or_b64 s[0:1], s[0:1], s[2:3]
+; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshl_i64:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], 1
-; GFX11-NEXT: s_not_b32 s5, s4
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], s4
-; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], s5
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b64 s[0:1], s[0:1], s[2:3]
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshl_i64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], s4
@@ -5349,6 +5597,7 @@ define amdgpu_ps i64 @s_fshl_i64(i64 inreg %lhs, i64 inreg %rhs, i64 inreg %amt)
; GFX9-NEXT: s_lshr_b64 s[2:3], s[2:3], s4
; GFX9-NEXT: s_or_b64 s[0:1], s[0:1], s[2:3]
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: s_fshl_i64:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_lshr_b64 s[2:3], s[2:3], 1
@@ -5357,6 +5606,16 @@ define amdgpu_ps i64 @s_fshl_i64(i64 inreg %lhs, i64 inreg %rhs, i64 inreg %amt)
; GFX10-NEXT: s_lshr_b64 s[2:3], s[2:3], s5
; GFX10-NEXT: s_or_b64 s[0:1], s[0:1], s[2:3]
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: s_fshl_i64:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], 1
+; GFX11-NEXT: s_not_b32 s5, s4
+; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], s4
+; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], s5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b64 s[0:1], s[0:1], s[2:3]
+; GFX11-NEXT: ; return to shader part epilog
%result = call i64 @llvm.fshl.i64(i64 %lhs, i64 %rhs, i64 %amt)
ret i64 %result
}
@@ -5366,19 +5625,15 @@ define amdgpu_ps i64 @s_fshl_i64_5(i64 inreg %lhs, i64 inreg %rhs) {
; GCN: ; %bb.0:
; GCN-NEXT: s_lshl_b64 s[0:1], s[0:1], 5
; GCN-NEXT: s_lshr_b32 s2, s3, 27
-; GCN-NEXT: s_mov_b32 s3, 0
-; GCN-NEXT: s_or_b64 s[2:3], s[0:1], s[2:3]
-; GCN-NEXT: s_mov_b32 s0, s2
+; GCN-NEXT: s_or_b32 s0, s0, s2
; GCN-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: s_fshl_i64_5:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 5
; GFX11-NEXT: s_lshr_b32 s2, s3, 27
-; GFX11-NEXT: s_mov_b32 s3, 0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b64 s[2:3], s[0:1], s[2:3]
-; GFX11-NEXT: s_mov_b32 s0, s2
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, s2
; GFX11-NEXT: ; return to shader part epilog
%result = call i64 @llvm.fshl.i64(i64 %lhs, i64 %rhs, i64 5)
ret i64 %result
@@ -5403,23 +5658,19 @@ define amdgpu_ps i64 @s_fshl_i64_32(i64 inreg %lhs, i64 inreg %rhs) {
define amdgpu_ps i64 @s_fshl_i64_48(i64 inreg %lhs, i64 inreg %rhs) {
; GCN-LABEL: s_fshl_i64_48:
; GCN: ; %bb.0:
-; GCN-NEXT: s_mov_b32 s4, s0
-; GCN-NEXT: s_lshr_b64 s[0:1], s[2:3], 16
-; GCN-NEXT: s_lshl_b32 s3, s4, 16
-; GCN-NEXT: s_mov_b32 s2, 0
-; GCN-NEXT: s_or_b64 s[2:3], s[2:3], s[0:1]
-; GCN-NEXT: s_mov_b32 s1, s3
+; GCN-NEXT: s_lshl_b32 s1, s0, 16
+; GCN-NEXT: s_mov_b32 s0, 0
+; GCN-NEXT: s_lshr_b64 s[2:3], s[2:3], 16
+; GCN-NEXT: s_or_b64 s[0:1], s[0:1], s[2:3]
; GCN-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: s_fshl_i64_48:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_mov_b32 s4, s0
-; GFX11-NEXT: s_lshr_b64 s[0:1], s[2:3], 16
-; GFX11-NEXT: s_lshl_b32 s3, s4, 16
-; GFX11-NEXT: s_mov_b32 s2, 0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b64 s[2:3], s[2:3], s[0:1]
-; GFX11-NEXT: s_mov_b32 s1, s3
+; GFX11-NEXT: s_lshl_b32 s1, s0, 16
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], 16
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b64 s[0:1], s[0:1], s[2:3]
; GFX11-NEXT: ; return to shader part epilog
%result = call i64 @llvm.fshl.i64(i64 %lhs, i64 %rhs, i64 48)
ret i64 %result
@@ -5450,19 +5701,6 @@ define i64 @v_fshl_i64(i64 %lhs, i64 %rhs, i64 %amt) {
; GFX8-NEXT: v_or_b32_e32 v1, v1, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshl_i64:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshrrev_b64 v[2:3], 1, v[2:3]
-; GFX11-NEXT: v_and_b32_e32 v5, 63, v4
-; GFX11-NEXT: v_bfi_b32 v4, v4, 0, 63
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshlrev_b64 v[0:1], v5, v[0:1]
-; GFX11-NEXT: v_lshrrev_b64 v[2:3], v4, v[2:3]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_i64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -5474,6 +5712,7 @@ define i64 @v_fshl_i64(i64 %lhs, i64 %rhs, i64 %amt) {
; GFX9-NEXT: v_or_b32_e32 v0, v0, v2
; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshl_i64:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -5485,6 +5724,20 @@ define i64 @v_fshl_i64(i64 %lhs, i64 %rhs, i64 %amt) {
; GFX10-NEXT: v_or_b32_e32 v0, v0, v2
; GFX10-NEXT: v_or_b32_e32 v1, v1, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_fshl_i64:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshrrev_b64 v[2:3], 1, v[2:3]
+; GFX11-NEXT: v_and_b32_e32 v5, 63, v4
+; GFX11-NEXT: v_bfi_b32 v4, v4, 0, 63
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], v5, v[0:1]
+; GFX11-NEXT: v_lshrrev_b64 v[2:3], v4, v[2:3]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call i64 @llvm.fshl.i64(i64 %lhs, i64 %rhs, i64 %amt)
ret i64 %result
}
@@ -5506,14 +5759,6 @@ define i64 @v_fshl_i64_5(i64 %lhs, i64 %rhs) {
; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshl_i64_5:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshlrev_b64 v[0:1], 5, v[0:1]
-; GFX11-NEXT: v_lshrrev_b32_e32 v2, 27, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_i64_5:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -5521,6 +5766,7 @@ define i64 @v_fshl_i64_5(i64 %lhs, i64 %rhs) {
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 27, v3
; GFX9-NEXT: v_or_b32_e32 v0, v0, v2
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshl_i64_5:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -5528,6 +5774,15 @@ define i64 @v_fshl_i64_5(i64 %lhs, i64 %rhs) {
; GFX10-NEXT: v_lshrrev_b32_e32 v2, 27, v3
; GFX10-NEXT: v_or_b32_e32 v0, v0, v2
; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_fshl_i64_5:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], 5, v[0:1]
+; GFX11-NEXT: v_lshrrev_b32_e32 v2, 27, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call i64 @llvm.fshl.i64(i64 %lhs, i64 %rhs, i64 5)
ret i64 %result
}
@@ -5568,14 +5823,6 @@ define i64 @v_fshl_i64_48(i64 %lhs, i64 %rhs) {
; GFX8-NEXT: v_or_b32_e32 v1, v2, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshl_i64_48:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v4, v0
-; GFX11-NEXT: v_lshrrev_b64 v[0:1], 16, v[2:3]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_lshl_or_b32 v1, v4, 16, v1
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_i64_48:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -5583,6 +5830,7 @@ define i64 @v_fshl_i64_48(i64 %lhs, i64 %rhs) {
; GFX9-NEXT: v_lshrrev_b64 v[0:1], 16, v[2:3]
; GFX9-NEXT: v_lshl_or_b32 v1, v4, 16, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshl_i64_48:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -5590,14 +5838,23 @@ define i64 @v_fshl_i64_48(i64 %lhs, i64 %rhs) {
; GFX10-NEXT: v_lshrrev_b64 v[0:1], 16, v[2:3]
; GFX10-NEXT: v_lshl_or_b32 v1, v4, 16, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
- %result = call i64 @llvm.fshl.i64(i64 %lhs, i64 %rhs, i64 48)
- ret i64 %result
-}
-
-define amdgpu_ps <2 x float> @v_fshl_i64_ssv(i64 inreg %lhs, i64 inreg %rhs, i64 %amt) {
-; GFX6-LABEL: v_fshl_i64_ssv:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: v_and_b32_e32 v1, 63, v0
+;
+; GFX11-LABEL: v_fshl_i64_48:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v4, v0
+; GFX11-NEXT: v_lshrrev_b64 v[0:1], 16, v[2:3]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshl_or_b32 v1, v4, 16, v1
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %result = call i64 @llvm.fshl.i64(i64 %lhs, i64 %rhs, i64 48)
+ ret i64 %result
+}
+
+define amdgpu_ps <2 x float> @v_fshl_i64_ssv(i64 inreg %lhs, i64 inreg %rhs, i64 %amt) {
+; GFX6-LABEL: v_fshl_i64_ssv:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: v_and_b32_e32 v1, 63, v0
; GFX6-NEXT: v_lshl_b64 v[1:2], s[0:1], v1
; GFX6-NEXT: s_lshr_b64 s[0:1], s[2:3], 1
; GFX6-NEXT: v_bfi_b32 v0, v0, 0, 63
@@ -5617,18 +5874,6 @@ define amdgpu_ps <2 x float> @v_fshl_i64_ssv(i64 inreg %lhs, i64 inreg %rhs, i64
; GFX8-NEXT: v_or_b32_e32 v1, v2, v4
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: v_fshl_i64_ssv:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_and_b32_e32 v1, 63, v0
-; GFX11-NEXT: v_bfi_b32 v2, v0, 0, 63
-; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], 1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshlrev_b64 v[0:1], v1, s[0:1]
-; GFX11-NEXT: v_lshrrev_b64 v[2:3], v2, s[2:3]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshl_i64_ssv:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_and_b32_e32 v1, 63, v0
@@ -5639,6 +5884,7 @@ define amdgpu_ps <2 x float> @v_fshl_i64_ssv(i64 inreg %lhs, i64 inreg %rhs, i64
; GFX9-NEXT: v_or_b32_e32 v0, v1, v3
; GFX9-NEXT: v_or_b32_e32 v1, v2, v4
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: v_fshl_i64_ssv:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_and_b32_e32 v1, 63, v0
@@ -5649,6 +5895,19 @@ define amdgpu_ps <2 x float> @v_fshl_i64_ssv(i64 inreg %lhs, i64 inreg %rhs, i64
; GFX10-NEXT: v_or_b32_e32 v0, v0, v2
; GFX10-NEXT: v_or_b32_e32 v1, v1, v3
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: v_fshl_i64_ssv:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_and_b32_e32 v1, 63, v0
+; GFX11-NEXT: v_bfi_b32 v2, v0, 0, 63
+; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], v1, s[0:1]
+; GFX11-NEXT: v_lshrrev_b64 v[2:3], v2, s[2:3]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX11-NEXT: ; return to shader part epilog
%result = call i64 @llvm.fshl.i64(i64 %lhs, i64 %rhs, i64 %amt)
%cast = bitcast i64 %result to <2 x float>
ret <2 x float> %cast
@@ -5679,18 +5938,6 @@ define amdgpu_ps <2 x float> @v_fshl_i64_svs(i64 inreg %lhs, i64 %rhs, i64 inreg
; GFX8-NEXT: v_or_b32_e32 v1, v3, v1
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: v_fshl_i64_svs:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_lshrrev_b64 v[0:1], 1, v[0:1]
-; GFX11-NEXT: s_and_not1_b32 s3, 63, s2
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], s2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
-; GFX11-NEXT: v_lshrrev_b64 v[0:1], s3, v[0:1]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX11-NEXT: v_or_b32_e32 v1, v3, v1
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshl_i64_svs:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_lshrrev_b64 v[0:1], 1, v[0:1]
@@ -5702,6 +5949,7 @@ define amdgpu_ps <2 x float> @v_fshl_i64_svs(i64 inreg %lhs, i64 %rhs, i64 inreg
; GFX9-NEXT: v_or_b32_e32 v0, v2, v0
; GFX9-NEXT: v_or_b32_e32 v1, v3, v1
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: v_fshl_i64_svs:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_lshrrev_b64 v[0:1], 1, v[0:1]
@@ -5713,6 +5961,19 @@ define amdgpu_ps <2 x float> @v_fshl_i64_svs(i64 inreg %lhs, i64 %rhs, i64 inreg
; GFX10-NEXT: v_or_b32_e32 v0, v2, v0
; GFX10-NEXT: v_or_b32_e32 v1, v3, v1
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: v_fshl_i64_svs:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_lshrrev_b64 v[0:1], 1, v[0:1]
+; GFX11-NEXT: s_and_not1_b32 s3, 63, s2
+; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], s2
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
+; GFX11-NEXT: v_lshrrev_b64 v[0:1], s3, v[0:1]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX11-NEXT: v_or_b32_e32 v1, v3, v1
+; GFX11-NEXT: ; return to shader part epilog
%result = call i64 @llvm.fshl.i64(i64 %lhs, i64 %rhs, i64 %amt)
%cast = bitcast i64 %result to <2 x float>
ret <2 x float> %cast
@@ -5745,18 +6006,6 @@ define amdgpu_ps <2 x float> @v_fshl_i64_vss(i64 %lhs, i64 inreg %rhs, i64 inreg
; GFX8-NEXT: v_or_b32_e32 v1, v1, v3
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: v_fshl_i64_vss:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], 1
-; GFX11-NEXT: s_not_b32 s3, s2
-; GFX11-NEXT: s_and_b32 s2, s2, 63
-; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], s3
-; GFX11-NEXT: v_lshlrev_b64 v[0:1], s2, v[0:1]
-; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshl_i64_vss:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s3, s2, 63
@@ -5769,6 +6018,7 @@ define amdgpu_ps <2 x float> @v_fshl_i64_vss(i64 %lhs, i64 inreg %rhs, i64 inreg
; GFX9-NEXT: v_or_b32_e32 v0, v0, v2
; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: v_fshl_i64_vss:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_lshr_b64 s[0:1], s[0:1], 1
@@ -5781,39 +6031,53 @@ define amdgpu_ps <2 x float> @v_fshl_i64_vss(i64 %lhs, i64 inreg %rhs, i64 inreg
; GFX10-NEXT: v_or_b32_e32 v1, v1, v3
; GFX10-NEXT: v_or_b32_e32 v0, v0, v2
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: v_fshl_i64_vss:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], 1
+; GFX11-NEXT: s_not_b32 s3, s2
+; GFX11-NEXT: s_and_b32 s2, s2, 63
+; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], s3
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], s2, v[0:1]
+; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX11-NEXT: ; return to shader part epilog
%result = call i64 @llvm.fshl.i64(i64 %lhs, i64 %rhs, i64 %amt)
%cast = bitcast i64 %result to <2 x float>
ret <2 x float> %cast
}
define amdgpu_ps <2 x i64> @s_fshl_v2i64(<2 x i64> inreg %lhs, <2 x i64> inreg %rhs, <2 x i64> inreg %amt) {
-; GCN-LABEL: s_fshl_v2i64:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_lshl_b64 s[0:1], s[0:1], s8
-; GCN-NEXT: s_lshr_b64 s[4:5], s[4:5], 1
-; GCN-NEXT: s_not_b32 s8, s8
-; GCN-NEXT: s_lshr_b64 s[4:5], s[4:5], s8
-; GCN-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
-; GCN-NEXT: s_lshr_b64 s[4:5], s[6:7], 1
-; GCN-NEXT: s_not_b32 s6, s10
-; GCN-NEXT: s_lshl_b64 s[2:3], s[2:3], s10
-; GCN-NEXT: s_lshr_b64 s[4:5], s[4:5], s6
-; GCN-NEXT: s_or_b64 s[2:3], s[2:3], s[4:5]
-; GCN-NEXT: ; return to shader part epilog
+; GFX6-LABEL: s_fshl_v2i64:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_lshl_b64 s[0:1], s[0:1], s8
+; GFX6-NEXT: s_lshr_b64 s[4:5], s[4:5], 1
+; GFX6-NEXT: s_not_b32 s8, s8
+; GFX6-NEXT: s_lshr_b64 s[4:5], s[4:5], s8
+; GFX6-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
+; GFX6-NEXT: s_lshr_b64 s[4:5], s[6:7], 1
+; GFX6-NEXT: s_not_b32 s6, s10
+; GFX6-NEXT: s_lshl_b64 s[2:3], s[2:3], s10
+; GFX6-NEXT: s_lshr_b64 s[4:5], s[4:5], s6
+; GFX6-NEXT: s_or_b64 s[2:3], s[2:3], s[4:5]
+; GFX6-NEXT: ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshl_v2i64:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_lshl_b64 s[0:1], s[0:1], s8
+; GFX8-NEXT: s_lshr_b64 s[4:5], s[4:5], 1
+; GFX8-NEXT: s_not_b32 s8, s8
+; GFX8-NEXT: s_lshr_b64 s[4:5], s[4:5], s8
+; GFX8-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
+; GFX8-NEXT: s_lshr_b64 s[4:5], s[6:7], 1
+; GFX8-NEXT: s_not_b32 s6, s10
+; GFX8-NEXT: s_lshl_b64 s[2:3], s[2:3], s10
+; GFX8-NEXT: s_lshr_b64 s[4:5], s[4:5], s6
+; GFX8-NEXT: s_or_b64 s[2:3], s[2:3], s[4:5]
+; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshl_v2i64:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_lshr_b64 s[4:5], s[4:5], 1
-; GFX11-NEXT: s_not_b32 s9, s8
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], s8
-; GFX11-NEXT: s_lshr_b64 s[6:7], s[6:7], 1
-; GFX11-NEXT: s_not_b32 s8, s10
-; GFX11-NEXT: s_lshr_b64 s[4:5], s[4:5], s9
-; GFX11-NEXT: s_lshl_b64 s[2:3], s[2:3], s10
-; GFX11-NEXT: s_lshr_b64 s[6:7], s[6:7], s8
-; GFX11-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
-; GFX11-NEXT: s_or_b64 s[2:3], s[2:3], s[6:7]
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshl_v2i64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], s8
@@ -5827,6 +6091,7 @@ define amdgpu_ps <2 x i64> @s_fshl_v2i64(<2 x i64> inreg %lhs, <2 x i64> inreg %
; GFX9-NEXT: s_lshr_b64 s[4:5], s[4:5], s6
; GFX9-NEXT: s_or_b64 s[2:3], s[2:3], s[4:5]
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: s_fshl_v2i64:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_lshr_b64 s[4:5], s[4:5], 1
@@ -5840,6 +6105,20 @@ define amdgpu_ps <2 x i64> @s_fshl_v2i64(<2 x i64> inreg %lhs, <2 x i64> inreg %
; GFX10-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
; GFX10-NEXT: s_or_b64 s[2:3], s[2:3], s[6:7]
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: s_fshl_v2i64:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_lshr_b64 s[4:5], s[4:5], 1
+; GFX11-NEXT: s_not_b32 s9, s8
+; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], s8
+; GFX11-NEXT: s_lshr_b64 s[6:7], s[6:7], 1
+; GFX11-NEXT: s_not_b32 s8, s10
+; GFX11-NEXT: s_lshr_b64 s[4:5], s[4:5], s9
+; GFX11-NEXT: s_lshl_b64 s[2:3], s[2:3], s10
+; GFX11-NEXT: s_lshr_b64 s[6:7], s[6:7], s8
+; GFX11-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
+; GFX11-NEXT: s_or_b64 s[2:3], s[2:3], s[6:7]
+; GFX11-NEXT: ; return to shader part epilog
%result = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %lhs, <2 x i64> %rhs, <2 x i64> %amt)
ret <2 x i64> %result
}
@@ -5883,28 +6162,6 @@ define <2 x i64> @v_fshl_v2i64(<2 x i64> %lhs, <2 x i64> %rhs, <2 x i64> %amt) {
; GFX8-NEXT: v_or_b32_e32 v3, v3, v7
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshl_v2i64:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshrrev_b64 v[4:5], 1, v[4:5]
-; GFX11-NEXT: v_lshrrev_b64 v[6:7], 1, v[6:7]
-; GFX11-NEXT: v_and_b32_e32 v9, 63, v8
-; GFX11-NEXT: v_bfi_b32 v8, v8, 0, 63
-; GFX11-NEXT: v_and_b32_e32 v11, 63, v10
-; GFX11-NEXT: v_bfi_b32 v10, v10, 0, 63
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_lshlrev_b64 v[0:1], v9, v[0:1]
-; GFX11-NEXT: v_lshrrev_b64 v[4:5], v8, v[4:5]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_lshlrev_b64 v[2:3], v11, v[2:3]
-; GFX11-NEXT: v_lshrrev_b64 v[6:7], v10, v[6:7]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX11-NEXT: v_or_b32_e32 v1, v1, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX11-NEXT: v_or_b32_e32 v3, v3, v7
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_v2i64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -5923,6 +6180,7 @@ define <2 x i64> @v_fshl_v2i64(<2 x i64> %lhs, <2 x i64> %rhs, <2 x i64> %amt) {
; GFX9-NEXT: v_or_b32_e32 v2, v2, v6
; GFX9-NEXT: v_or_b32_e32 v3, v3, v7
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshl_v2i64:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -5941,6 +6199,29 @@ define <2 x i64> @v_fshl_v2i64(<2 x i64> %lhs, <2 x i64> %rhs, <2 x i64> %amt) {
; GFX10-NEXT: v_or_b32_e32 v2, v2, v6
; GFX10-NEXT: v_or_b32_e32 v3, v3, v7
; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_fshl_v2i64:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshrrev_b64 v[4:5], 1, v[4:5]
+; GFX11-NEXT: v_lshrrev_b64 v[6:7], 1, v[6:7]
+; GFX11-NEXT: v_and_b32_e32 v9, 63, v8
+; GFX11-NEXT: v_bfi_b32 v8, v8, 0, 63
+; GFX11-NEXT: v_and_b32_e32 v11, 63, v10
+; GFX11-NEXT: v_bfi_b32 v10, v10, 0, 63
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], v9, v[0:1]
+; GFX11-NEXT: v_lshrrev_b64 v[4:5], v8, v[4:5]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_lshlrev_b64 v[2:3], v11, v[2:3]
+; GFX11-NEXT: v_lshrrev_b64 v[6:7], v10, v[6:7]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX11-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX11-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %lhs, <2 x i64> %rhs, <2 x i64> %amt)
ret <2 x i64> %result
}
@@ -5948,73 +6229,98 @@ define <2 x i64> @v_fshl_v2i64(<2 x i64> %lhs, <2 x i64> %rhs, <2 x i64> %amt) {
define amdgpu_ps i128 @s_fshl_i128(i128 inreg %lhs, i128 inreg %rhs, i128 inreg %amt) {
; GFX6-LABEL: s_fshl_i128:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_and_b32 s10, s8, 64
-; GFX6-NEXT: s_mov_b32 s11, 0
-; GFX6-NEXT: v_cmp_ne_u64_e64 s[10:11], s[10:11], 0
-; GFX6-NEXT: s_and_b64 s[12:13], s[10:11], exec
-; GFX6-NEXT: s_cselect_b32 s3, s1, s3
-; GFX6-NEXT: s_cselect_b32 s2, s0, s2
-; GFX6-NEXT: s_lshl_b64 s[2:3], s[2:3], s8
-; GFX6-NEXT: s_and_b64 s[12:13], s[10:11], exec
-; GFX6-NEXT: s_cselect_b32 s1, s7, s1
-; GFX6-NEXT: s_cselect_b32 s0, s6, s0
-; GFX6-NEXT: s_lshr_b64 s[12:13], s[0:1], 1
-; GFX6-NEXT: s_not_b32 s14, s8
-; GFX6-NEXT: s_lshr_b64 s[12:13], s[12:13], s14
-; GFX6-NEXT: s_or_b64 s[2:3], s[2:3], s[12:13]
-; GFX6-NEXT: s_lshl_b64 s[0:1], s[0:1], s8
-; GFX6-NEXT: s_and_b64 s[8:9], s[10:11], exec
-; GFX6-NEXT: s_cselect_b32 s5, s5, s7
-; GFX6-NEXT: s_cselect_b32 s4, s4, s6
-; GFX6-NEXT: s_lshr_b64 s[4:5], s[4:5], 1
+; GFX6-NEXT: s_and_b32 s9, s8, 0x7f
+; GFX6-NEXT: s_sub_i32 s11, s9, 64
+; GFX6-NEXT: s_sub_i32 s14, 64, s9
+; GFX6-NEXT: s_cmp_lt_u32 s9, 64
+; GFX6-NEXT: s_cselect_b32 s18, 1, 0
+; GFX6-NEXT: s_cmp_eq_u32 s9, 0
+; GFX6-NEXT: s_cselect_b32 s9, 1, 0
+; GFX6-NEXT: s_lshr_b64 s[14:15], s[0:1], s14
+; GFX6-NEXT: s_lshl_b64 s[16:17], s[2:3], s8
+; GFX6-NEXT: s_lshl_b64 s[12:13], s[0:1], s8
+; GFX6-NEXT: s_or_b64 s[14:15], s[14:15], s[16:17]
+; GFX6-NEXT: s_lshl_b64 s[0:1], s[0:1], s11
+; GFX6-NEXT: s_cmp_lg_u32 s18, 0
+; GFX6-NEXT: s_cselect_b64 s[12:13], s[12:13], 0
+; GFX6-NEXT: s_cselect_b64 s[0:1], s[14:15], s[0:1]
+; GFX6-NEXT: s_cmp_lg_u32 s9, 0
+; GFX6-NEXT: s_mov_b32 s10, 0
+; GFX6-NEXT: s_cselect_b64 s[2:3], s[2:3], s[0:1]
+; GFX6-NEXT: s_lshr_b64 s[0:1], s[4:5], 1
+; GFX6-NEXT: s_lshl_b32 s11, s6, 31
+; GFX6-NEXT: s_lshr_b64 s[4:5], s[6:7], 1
+; GFX6-NEXT: s_andn2_b32 s6, 0x7f, s8
+; GFX6-NEXT: s_or_b64 s[0:1], s[0:1], s[10:11]
+; GFX6-NEXT: s_not_b32 s9, s8
+; GFX6-NEXT: s_sub_i32 s14, s6, 64
+; GFX6-NEXT: s_sub_i32 s10, 64, s6
+; GFX6-NEXT: s_cmp_lt_u32 s6, 64
+; GFX6-NEXT: s_cselect_b32 s15, 1, 0
+; GFX6-NEXT: s_cmp_eq_u32 s6, 0
+; GFX6-NEXT: s_cselect_b32 s16, 1, 0
+; GFX6-NEXT: s_lshr_b64 s[6:7], s[4:5], s9
+; GFX6-NEXT: s_lshr_b64 s[8:9], s[0:1], s9
+; GFX6-NEXT: s_lshl_b64 s[10:11], s[4:5], s10
+; GFX6-NEXT: s_or_b64 s[8:9], s[8:9], s[10:11]
; GFX6-NEXT: s_lshr_b64 s[4:5], s[4:5], s14
-; GFX6-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
+; GFX6-NEXT: s_cmp_lg_u32 s15, 0
+; GFX6-NEXT: s_cselect_b64 s[4:5], s[8:9], s[4:5]
+; GFX6-NEXT: s_cmp_lg_u32 s16, 0
+; GFX6-NEXT: s_cselect_b64 s[0:1], s[0:1], s[4:5]
+; GFX6-NEXT: s_cmp_lg_u32 s15, 0
+; GFX6-NEXT: s_cselect_b64 s[4:5], s[6:7], 0
+; GFX6-NEXT: s_or_b64 s[0:1], s[12:13], s[0:1]
+; GFX6-NEXT: s_or_b64 s[2:3], s[2:3], s[4:5]
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_fshl_i128:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_and_b32 s10, s8, 64
-; GFX8-NEXT: s_mov_b32 s11, 0
-; GFX8-NEXT: s_cmp_lg_u64 s[10:11], 0
-; GFX8-NEXT: s_cselect_b32 s11, s7, s1
-; GFX8-NEXT: s_cselect_b32 s10, s6, s0
-; GFX8-NEXT: s_cselect_b32 s1, s1, s3
-; GFX8-NEXT: s_cselect_b32 s0, s0, s2
-; GFX8-NEXT: s_cselect_b32 s5, s5, s7
-; GFX8-NEXT: s_cselect_b32 s4, s4, s6
-; GFX8-NEXT: s_lshr_b64 s[2:3], s[10:11], 1
-; GFX8-NEXT: s_not_b32 s6, s8
-; GFX8-NEXT: s_lshl_b64 s[0:1], s[0:1], s8
-; GFX8-NEXT: s_lshr_b64 s[2:3], s[2:3], s6
-; GFX8-NEXT: s_lshr_b64 s[4:5], s[4:5], 1
-; GFX8-NEXT: s_or_b64 s[2:3], s[0:1], s[2:3]
-; GFX8-NEXT: s_lshl_b64 s[0:1], s[10:11], s8
-; GFX8-NEXT: s_lshr_b64 s[4:5], s[4:5], s6
-; GFX8-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
+; GFX8-NEXT: s_and_b32 s9, s8, 0x7f
+; GFX8-NEXT: s_sub_i32 s11, s9, 64
+; GFX8-NEXT: s_sub_i32 s14, 64, s9
+; GFX8-NEXT: s_cmp_lt_u32 s9, 64
+; GFX8-NEXT: s_cselect_b32 s18, 1, 0
+; GFX8-NEXT: s_cmp_eq_u32 s9, 0
+; GFX8-NEXT: s_cselect_b32 s9, 1, 0
+; GFX8-NEXT: s_lshr_b64 s[14:15], s[0:1], s14
+; GFX8-NEXT: s_lshl_b64 s[16:17], s[2:3], s8
+; GFX8-NEXT: s_lshl_b64 s[12:13], s[0:1], s8
+; GFX8-NEXT: s_or_b64 s[14:15], s[14:15], s[16:17]
+; GFX8-NEXT: s_lshl_b64 s[0:1], s[0:1], s11
+; GFX8-NEXT: s_cmp_lg_u32 s18, 0
+; GFX8-NEXT: s_cselect_b64 s[12:13], s[12:13], 0
+; GFX8-NEXT: s_cselect_b64 s[0:1], s[14:15], s[0:1]
+; GFX8-NEXT: s_cmp_lg_u32 s9, 0
+; GFX8-NEXT: s_mov_b32 s10, 0
+; GFX8-NEXT: s_cselect_b64 s[2:3], s[2:3], s[0:1]
+; GFX8-NEXT: s_lshr_b64 s[0:1], s[4:5], 1
+; GFX8-NEXT: s_lshl_b32 s11, s6, 31
+; GFX8-NEXT: s_lshr_b64 s[4:5], s[6:7], 1
+; GFX8-NEXT: s_andn2_b32 s6, 0x7f, s8
+; GFX8-NEXT: s_or_b64 s[0:1], s[0:1], s[10:11]
+; GFX8-NEXT: s_not_b32 s9, s8
+; GFX8-NEXT: s_sub_i32 s14, s6, 64
+; GFX8-NEXT: s_sub_i32 s10, 64, s6
+; GFX8-NEXT: s_cmp_lt_u32 s6, 64
+; GFX8-NEXT: s_cselect_b32 s15, 1, 0
+; GFX8-NEXT: s_cmp_eq_u32 s6, 0
+; GFX8-NEXT: s_cselect_b32 s16, 1, 0
+; GFX8-NEXT: s_lshr_b64 s[6:7], s[4:5], s9
+; GFX8-NEXT: s_lshr_b64 s[8:9], s[0:1], s9
+; GFX8-NEXT: s_lshl_b64 s[10:11], s[4:5], s10
+; GFX8-NEXT: s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX8-NEXT: s_lshr_b64 s[4:5], s[4:5], s14
+; GFX8-NEXT: s_cmp_lg_u32 s15, 0
+; GFX8-NEXT: s_cselect_b64 s[4:5], s[8:9], s[4:5]
+; GFX8-NEXT: s_cmp_lg_u32 s16, 0
+; GFX8-NEXT: s_cselect_b64 s[0:1], s[0:1], s[4:5]
+; GFX8-NEXT: s_cmp_lg_u32 s15, 0
+; GFX8-NEXT: s_cselect_b64 s[4:5], s[6:7], 0
+; GFX8-NEXT: s_or_b64 s[0:1], s[12:13], s[0:1]
+; GFX8-NEXT: s_or_b64 s[2:3], s[2:3], s[4:5]
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshl_i128:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s10, s8, 64
-; GFX11-NEXT: s_mov_b32 s11, 0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_cmp_lg_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s11, s7, s1
-; GFX11-NEXT: s_cselect_b32 s10, s6, s0
-; GFX11-NEXT: s_cselect_b32 s1, s1, s3
-; GFX11-NEXT: s_cselect_b32 s0, s0, s2
-; GFX11-NEXT: s_cselect_b32 s3, s5, s7
-; GFX11-NEXT: s_cselect_b32 s2, s4, s6
-; GFX11-NEXT: s_lshl_b64 s[4:5], s[0:1], s8
-; GFX11-NEXT: s_lshr_b64 s[0:1], s[10:11], 1
-; GFX11-NEXT: s_not_b32 s9, s8
-; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], 1
-; GFX11-NEXT: s_lshl_b64 s[6:7], s[10:11], s8
-; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], s9
-; GFX11-NEXT: s_lshr_b64 s[8:9], s[0:1], s9
-; GFX11-NEXT: s_or_b64 s[0:1], s[6:7], s[2:3]
-; GFX11-NEXT: s_or_b64 s[2:3], s[4:5], s[8:9]
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshl_i128:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s9, s8, 0x7f
@@ -6061,6 +6367,7 @@ define amdgpu_ps i128 @s_fshl_i128(i128 inreg %lhs, i128 inreg %rhs, i128 inreg
; GFX9-NEXT: s_or_b64 s[0:1], s[12:13], s[0:1]
; GFX9-NEXT: s_or_b64 s[2:3], s[2:3], s[4:5]
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: s_fshl_i128:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s9, s8, 0x7f
@@ -6107,6 +6414,53 @@ define amdgpu_ps i128 @s_fshl_i128(i128 inreg %lhs, i128 inreg %rhs, i128 inreg
; GFX10-NEXT: s_or_b64 s[0:1], s[14:15], s[0:1]
; GFX10-NEXT: s_or_b64 s[2:3], s[2:3], s[4:5]
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: s_fshl_i128:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s9, s8, 0x7f
+; GFX11-NEXT: s_mov_b32 s10, 0
+; GFX11-NEXT: s_sub_i32 s11, s9, 64
+; GFX11-NEXT: s_sub_i32 s12, 64, s9
+; GFX11-NEXT: s_cmp_lt_u32 s9, 64
+; GFX11-NEXT: s_cselect_b32 s18, 1, 0
+; GFX11-NEXT: s_cmp_eq_u32 s9, 0
+; GFX11-NEXT: s_cselect_b32 s9, 1, 0
+; GFX11-NEXT: s_lshr_b64 s[12:13], s[0:1], s12
+; GFX11-NEXT: s_lshl_b64 s[14:15], s[2:3], s8
+; GFX11-NEXT: s_lshl_b64 s[16:17], s[0:1], s8
+; GFX11-NEXT: s_or_b64 s[12:13], s[12:13], s[14:15]
+; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], s11
+; GFX11-NEXT: s_cmp_lg_u32 s18, 0
+; GFX11-NEXT: s_cselect_b64 s[14:15], s[16:17], 0
+; GFX11-NEXT: s_cselect_b64 s[0:1], s[12:13], s[0:1]
+; GFX11-NEXT: s_cmp_lg_u32 s9, 0
+; GFX11-NEXT: s_cselect_b64 s[2:3], s[2:3], s[0:1]
+; GFX11-NEXT: s_lshr_b64 s[0:1], s[4:5], 1
+; GFX11-NEXT: s_lshl_b32 s11, s6, 31
+; GFX11-NEXT: s_lshr_b64 s[4:5], s[6:7], 1
+; GFX11-NEXT: s_and_not1_b32 s6, 0x7f, s8
+; GFX11-NEXT: s_or_b64 s[0:1], s[0:1], s[10:11]
+; GFX11-NEXT: s_not_b32 s10, s8
+; GFX11-NEXT: s_sub_i32 s12, s6, 64
+; GFX11-NEXT: s_sub_i32 s8, 64, s6
+; GFX11-NEXT: s_cmp_lt_u32 s6, 64
+; GFX11-NEXT: s_cselect_b32 s13, 1, 0
+; GFX11-NEXT: s_cmp_eq_u32 s6, 0
+; GFX11-NEXT: s_cselect_b32 s16, 1, 0
+; GFX11-NEXT: s_lshr_b64 s[6:7], s[0:1], s10
+; GFX11-NEXT: s_lshl_b64 s[8:9], s[4:5], s8
+; GFX11-NEXT: s_lshr_b64 s[10:11], s[4:5], s10
+; GFX11-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
+; GFX11-NEXT: s_lshr_b64 s[4:5], s[4:5], s12
+; GFX11-NEXT: s_cmp_lg_u32 s13, 0
+; GFX11-NEXT: s_cselect_b64 s[4:5], s[6:7], s[4:5]
+; GFX11-NEXT: s_cmp_lg_u32 s16, 0
+; GFX11-NEXT: s_cselect_b64 s[0:1], s[0:1], s[4:5]
+; GFX11-NEXT: s_cmp_lg_u32 s13, 0
+; GFX11-NEXT: s_cselect_b64 s[4:5], s[10:11], 0
+; GFX11-NEXT: s_or_b64 s[0:1], s[14:15], s[0:1]
+; GFX11-NEXT: s_or_b64 s[2:3], s[2:3], s[4:5]
+; GFX11-NEXT: ; return to shader part epilog
%result = call i128 @llvm.fshl.i128(i128 %lhs, i128 %rhs, i128 %amt)
ret i128 %result
}
@@ -6192,72 +6546,24 @@ define i128 @v_fshl_i128(i128 %lhs, i128 %rhs, i128 %amt) {
; GFX8-NEXT: v_add_u32_e32 v15, vcc, v14, v17
; GFX8-NEXT: v_lshrrev_b64 v[4:5], v14, v[0:1]
; GFX8-NEXT: v_lshlrev_b64 v[6:7], v6, v[2:3]
-; GFX8-NEXT: v_lshrrev_b64 v[8:9], v14, v[2:3]
-; GFX8-NEXT: v_lshrrev_b64 v[2:3], v15, v[2:3]
-; GFX8-NEXT: v_or_b32_e32 v4, v4, v6
-; GFX8-NEXT: v_or_b32_e32 v5, v5, v7
-; GFX8-NEXT: v_cmp_gt_u32_e32 vcc, 64, v14
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v14
-; GFX8-NEXT: v_cndmask_b32_e64 v0, v2, v0, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e64 v1, v3, v1, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e32 v2, 0, v8, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v3, 0, v9, vcc
-; GFX8-NEXT: v_or_b32_e32 v0, v11, v0
-; GFX8-NEXT: v_or_b32_e32 v1, v12, v1
-; GFX8-NEXT: v_or_b32_e32 v2, v10, v2
-; GFX8-NEXT: v_or_b32_e32 v3, v13, v3
-; GFX8-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: v_fshl_i128:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_and_b32_e32 v17, 0x7f, v8
-; GFX11-NEXT: v_lshrrev_b64 v[4:5], 1, v[4:5]
-; GFX11-NEXT: v_bfi_b32 v18, v8, 0, 0x7f
-; GFX11-NEXT: v_lshrrev_b64 v[9:10], 1, v[6:7]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT: v_sub_nc_u32_e32 v11, 64, v17
-; GFX11-NEXT: v_lshlrev_b64 v[7:8], v17, v[2:3]
-; GFX11-NEXT: v_add_nc_u32_e32 v15, 0xffffffc0, v17
-; GFX11-NEXT: v_lshl_or_b32 v5, v6, 31, v5
-; GFX11-NEXT: v_sub_nc_u32_e32 v16, 64, v18
-; GFX11-NEXT: v_lshrrev_b64 v[11:12], v11, v[0:1]
-; GFX11-NEXT: v_lshlrev_b64 v[13:14], v17, v[0:1]
-; GFX11-NEXT: v_lshlrev_b64 v[0:1], v15, v[0:1]
-; GFX11-NEXT: v_cmp_gt_u32_e32 vcc_lo, 64, v17
-; GFX11-NEXT: v_add_nc_u32_e32 v19, 0xffffffc0, v18
-; GFX11-NEXT: v_lshlrev_b64 v[15:16], v16, v[9:10]
-; GFX11-NEXT: v_or_b32_e32 v11, v11, v7
-; GFX11-NEXT: v_lshrrev_b64 v[6:7], v18, v[4:5]
-; GFX11-NEXT: v_or_b32_e32 v8, v12, v8
-; GFX11-NEXT: v_cmp_gt_u32_e64 s1, 64, v18
-; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 0, v18
-; GFX11-NEXT: v_cndmask_b32_e32 v20, v0, v11, vcc_lo
-; GFX11-NEXT: v_lshrrev_b64 v[11:12], v19, v[9:10]
-; GFX11-NEXT: v_or_b32_e32 v0, v6, v15
-; GFX11-NEXT: v_or_b32_e32 v6, v7, v16
-; GFX11-NEXT: v_cndmask_b32_e32 v7, v1, v8, vcc_lo
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 0, v17
-; GFX11-NEXT: v_cndmask_b32_e32 v13, 0, v13, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e64 v8, v11, v0, s1
-; GFX11-NEXT: v_lshrrev_b64 v[0:1], v18, v[9:10]
-; GFX11-NEXT: v_cndmask_b32_e64 v6, v12, v6, s1
-; GFX11-NEXT: v_cndmask_b32_e32 v9, 0, v14, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e64 v2, v20, v2, s0
-; GFX11-NEXT: v_cndmask_b32_e64 v3, v7, v3, s0
-; GFX11-NEXT: v_cndmask_b32_e64 v4, v8, v4, s2
-; GFX11-NEXT: v_cndmask_b32_e64 v5, v6, v5, s2
-; GFX11-NEXT: v_cndmask_b32_e64 v6, 0, v0, s1
-; GFX11-NEXT: v_cndmask_b32_e64 v7, 0, v1, s1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v0, v13, v4
-; GFX11-NEXT: v_or_b32_e32 v1, v9, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX11-NEXT: v_or_b32_e32 v3, v3, v7
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX8-NEXT: v_lshrrev_b64 v[8:9], v14, v[2:3]
+; GFX8-NEXT: v_lshrrev_b64 v[2:3], v15, v[2:3]
+; GFX8-NEXT: v_or_b32_e32 v4, v4, v6
+; GFX8-NEXT: v_or_b32_e32 v5, v5, v7
+; GFX8-NEXT: v_cmp_gt_u32_e32 vcc, 64, v14
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v14
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v2, v0, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e64 v1, v3, v1, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e32 v2, 0, v8, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v3, 0, v9, vcc
+; GFX8-NEXT: v_or_b32_e32 v0, v11, v0
+; GFX8-NEXT: v_or_b32_e32 v1, v12, v1
+; GFX8-NEXT: v_or_b32_e32 v2, v10, v2
+; GFX8-NEXT: v_or_b32_e32 v3, v13, v3
+; GFX8-NEXT: s_setpc_b64 s[30:31]
+;
; GFX9-LABEL: v_fshl_i128:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -6304,6 +6610,7 @@ define i128 @v_fshl_i128(i128 %lhs, i128 %rhs, i128 %amt) {
; GFX9-NEXT: v_or_b32_e32 v2, v10, v2
; GFX9-NEXT: v_or_b32_e32 v3, v13, v3
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshl_i128:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -6349,6 +6656,55 @@ define i128 @v_fshl_i128(i128 %lhs, i128 %rhs, i128 %amt) {
; GFX10-NEXT: v_or_b32_e32 v2, v2, v6
; GFX10-NEXT: v_or_b32_e32 v3, v3, v7
; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_fshl_i128:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_and_b32_e32 v17, 0x7f, v8
+; GFX11-NEXT: v_lshrrev_b64 v[4:5], 1, v[4:5]
+; GFX11-NEXT: v_bfi_b32 v18, v8, 0, 0x7f
+; GFX11-NEXT: v_lshrrev_b64 v[9:10], 1, v[6:7]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT: v_sub_nc_u32_e32 v11, 64, v17
+; GFX11-NEXT: v_lshlrev_b64 v[7:8], v17, v[2:3]
+; GFX11-NEXT: v_add_nc_u32_e32 v15, 0xffffffc0, v17
+; GFX11-NEXT: v_lshl_or_b32 v5, v6, 31, v5
+; GFX11-NEXT: v_sub_nc_u32_e32 v16, 64, v18
+; GFX11-NEXT: v_lshrrev_b64 v[11:12], v11, v[0:1]
+; GFX11-NEXT: v_lshlrev_b64 v[13:14], v17, v[0:1]
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], v15, v[0:1]
+; GFX11-NEXT: v_cmp_gt_u32_e32 vcc_lo, 64, v17
+; GFX11-NEXT: v_add_nc_u32_e32 v19, 0xffffffc0, v18
+; GFX11-NEXT: v_lshlrev_b64 v[15:16], v16, v[9:10]
+; GFX11-NEXT: v_or_b32_e32 v11, v11, v7
+; GFX11-NEXT: v_lshrrev_b64 v[6:7], v18, v[4:5]
+; GFX11-NEXT: v_or_b32_e32 v8, v12, v8
+; GFX11-NEXT: v_cmp_gt_u32_e64 s1, 64, v18
+; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 0, v18
+; GFX11-NEXT: v_cndmask_b32_e32 v20, v0, v11, vcc_lo
+; GFX11-NEXT: v_lshrrev_b64 v[11:12], v19, v[9:10]
+; GFX11-NEXT: v_or_b32_e32 v0, v6, v15
+; GFX11-NEXT: v_or_b32_e32 v6, v7, v16
+; GFX11-NEXT: v_cndmask_b32_e32 v7, v1, v8, vcc_lo
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 0, v17
+; GFX11-NEXT: v_cndmask_b32_e32 v13, 0, v13, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e64 v8, v11, v0, s1
+; GFX11-NEXT: v_lshrrev_b64 v[0:1], v18, v[9:10]
+; GFX11-NEXT: v_cndmask_b32_e64 v6, v12, v6, s1
+; GFX11-NEXT: v_cndmask_b32_e32 v9, 0, v14, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v20, v2, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v3, v7, v3, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v4, v8, v4, s2
+; GFX11-NEXT: v_cndmask_b32_e64 v5, v6, v5, s2
+; GFX11-NEXT: v_cndmask_b32_e64 v6, 0, v0, s1
+; GFX11-NEXT: v_cndmask_b32_e64 v7, 0, v1, s1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v0, v13, v4
+; GFX11-NEXT: v_or_b32_e32 v1, v9, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX11-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call i128 @llvm.fshl.i128(i128 %lhs, i128 %rhs, i128 %amt)
ret i128 %result
}
@@ -6356,101 +6712,110 @@ define i128 @v_fshl_i128(i128 %lhs, i128 %rhs, i128 %amt) {
define amdgpu_ps <4 x float> @v_fshl_i128_ssv(i128 inreg %lhs, i128 inreg %rhs, i128 %amt) {
; GFX6-LABEL: v_fshl_i128_ssv:
; GFX6: ; %bb.0:
-; GFX6-NEXT: v_and_b32_e32 v1, 64, v0
-; GFX6-NEXT: v_mov_b32_e32 v2, 0
-; GFX6-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[1:2]
-; GFX6-NEXT: v_mov_b32_e32 v8, s1
-; GFX6-NEXT: v_mov_b32_e32 v6, s7
-; GFX6-NEXT: v_mov_b32_e32 v7, s5
-; GFX6-NEXT: v_cndmask_b32_e32 v3, v8, v6, vcc
-; GFX6-NEXT: v_mov_b32_e32 v1, s6
-; GFX6-NEXT: v_cndmask_b32_e32 v7, v6, v7, vcc
-; GFX6-NEXT: v_mov_b32_e32 v6, s4
-; GFX6-NEXT: v_cndmask_b32_e32 v6, v1, v6, vcc
-; GFX6-NEXT: v_mov_b32_e32 v9, s0
-; GFX6-NEXT: v_lshr_b64 v[6:7], v[6:7], 1
-; GFX6-NEXT: v_cndmask_b32_e32 v2, v9, v1, vcc
-; GFX6-NEXT: v_not_b32_e32 v10, v0
-; GFX6-NEXT: v_lshl_b64 v[4:5], v[2:3], v0
-; GFX6-NEXT: v_lshr_b64 v[6:7], v[6:7], v10
-; GFX6-NEXT: v_lshr_b64 v[2:3], v[2:3], 1
-; GFX6-NEXT: v_or_b32_e32 v1, v5, v7
-; GFX6-NEXT: v_mov_b32_e32 v5, s3
-; GFX6-NEXT: v_cndmask_b32_e32 v8, v5, v8, vcc
-; GFX6-NEXT: v_mov_b32_e32 v5, s2
-; GFX6-NEXT: v_cndmask_b32_e32 v7, v5, v9, vcc
-; GFX6-NEXT: v_lshl_b64 v[7:8], v[7:8], v0
-; GFX6-NEXT: v_lshr_b64 v[2:3], v[2:3], v10
-; GFX6-NEXT: v_or_b32_e32 v0, v4, v6
-; GFX6-NEXT: v_or_b32_e32 v3, v8, v3
-; GFX6-NEXT: v_or_b32_e32 v2, v7, v2
+; GFX6-NEXT: v_and_b32_e32 v8, 0x7f, v0
+; GFX6-NEXT: v_sub_i32_e32 v1, vcc, 64, v8
+; GFX6-NEXT: v_lshr_b64 v[1:2], s[0:1], v1
+; GFX6-NEXT: v_lshl_b64 v[3:4], s[2:3], v8
+; GFX6-NEXT: v_add_i32_e32 v10, vcc, 0xffffffc0, v8
+; GFX6-NEXT: v_lshl_b64 v[5:6], s[0:1], v8
+; GFX6-NEXT: v_or_b32_e32 v3, v1, v3
+; GFX6-NEXT: v_or_b32_e32 v4, v2, v4
+; GFX6-NEXT: v_lshl_b64 v[1:2], s[0:1], v10
+; GFX6-NEXT: v_mov_b32_e32 v7, 0x7f
+; GFX6-NEXT: v_cmp_gt_u32_e32 vcc, 64, v8
+; GFX6-NEXT: s_mov_b32 s8, 0
+; GFX6-NEXT: v_cndmask_b32_e32 v10, 0, v5, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v11, 0, v6, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX6-NEXT: v_mov_b32_e32 v3, s2
+; GFX6-NEXT: v_mov_b32_e32 v4, s3
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v8
+; GFX6-NEXT: s_lshr_b64 s[0:1], s[4:5], 1
+; GFX6-NEXT: s_lshl_b32 s9, s6, 31
+; GFX6-NEXT: v_bfi_b32 v13, v0, 0, v7
+; GFX6-NEXT: v_cndmask_b32_e32 v8, v1, v3, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v12, v2, v4, vcc
+; GFX6-NEXT: s_or_b64 s[0:1], s[0:1], s[8:9]
+; GFX6-NEXT: s_lshr_b64 s[2:3], s[6:7], 1
+; GFX6-NEXT: v_sub_i32_e32 v4, vcc, 64, v13
+; GFX6-NEXT: v_not_b32_e32 v9, 63
+; GFX6-NEXT: v_lshr_b64 v[2:3], s[0:1], v13
+; GFX6-NEXT: v_lshl_b64 v[4:5], s[2:3], v4
+; GFX6-NEXT: v_add_i32_e32 v9, vcc, v13, v9
+; GFX6-NEXT: v_or_b32_e32 v4, v2, v4
+; GFX6-NEXT: v_or_b32_e32 v5, v3, v5
+; GFX6-NEXT: v_lshr_b64 v[2:3], s[2:3], v9
+; GFX6-NEXT: v_lshr_b64 v[0:1], s[2:3], v13
+; GFX6-NEXT: v_cmp_gt_u32_e32 vcc, 64, v13
+; GFX6-NEXT: v_mov_b32_e32 v7, s1
+; GFX6-NEXT: v_mov_b32_e32 v6, s0
+; GFX6-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc
+; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v13
+; GFX6-NEXT: v_cndmask_b32_e64 v2, v2, v6, s[0:1]
+; GFX6-NEXT: v_cndmask_b32_e64 v3, v3, v7, s[0:1]
+; GFX6-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc
+; GFX6-NEXT: v_or_b32_e32 v0, v10, v2
+; GFX6-NEXT: v_or_b32_e32 v1, v11, v3
+; GFX6-NEXT: v_or_b32_e32 v2, v8, v4
+; GFX6-NEXT: v_or_b32_e32 v3, v12, v5
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: v_fshl_i128_ssv:
; GFX8: ; %bb.0:
-; GFX8-NEXT: v_and_b32_e32 v1, 64, v0
-; GFX8-NEXT: v_mov_b32_e32 v2, 0
-; GFX8-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[1:2]
-; GFX8-NEXT: v_mov_b32_e32 v8, s1
-; GFX8-NEXT: v_mov_b32_e32 v6, s7
-; GFX8-NEXT: v_mov_b32_e32 v7, s5
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v8, v6, vcc
-; GFX8-NEXT: v_mov_b32_e32 v1, s6
-; GFX8-NEXT: v_cndmask_b32_e32 v7, v6, v7, vcc
-; GFX8-NEXT: v_mov_b32_e32 v6, s4
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v1, v6, vcc
-; GFX8-NEXT: v_mov_b32_e32 v9, s0
-; GFX8-NEXT: v_lshrrev_b64 v[6:7], 1, v[6:7]
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v9, v1, vcc
-; GFX8-NEXT: v_not_b32_e32 v10, v0
-; GFX8-NEXT: v_lshlrev_b64 v[4:5], v0, v[2:3]
-; GFX8-NEXT: v_lshrrev_b64 v[6:7], v10, v[6:7]
-; GFX8-NEXT: v_lshrrev_b64 v[2:3], 1, v[2:3]
-; GFX8-NEXT: v_or_b32_e32 v1, v5, v7
-; GFX8-NEXT: v_mov_b32_e32 v5, s3
-; GFX8-NEXT: v_cndmask_b32_e32 v8, v5, v8, vcc
-; GFX8-NEXT: v_mov_b32_e32 v5, s2
-; GFX8-NEXT: v_cndmask_b32_e32 v7, v5, v9, vcc
-; GFX8-NEXT: v_lshlrev_b64 v[7:8], v0, v[7:8]
-; GFX8-NEXT: v_lshrrev_b64 v[2:3], v10, v[2:3]
-; GFX8-NEXT: v_or_b32_e32 v0, v4, v6
-; GFX8-NEXT: v_or_b32_e32 v3, v8, v3
-; GFX8-NEXT: v_or_b32_e32 v2, v7, v2
+; GFX8-NEXT: v_and_b32_e32 v8, 0x7f, v0
+; GFX8-NEXT: v_sub_u32_e32 v1, vcc, 64, v8
+; GFX8-NEXT: v_lshrrev_b64 v[1:2], v1, s[0:1]
+; GFX8-NEXT: v_lshlrev_b64 v[3:4], v8, s[2:3]
+; GFX8-NEXT: v_add_u32_e32 v10, vcc, 0xffffffc0, v8
+; GFX8-NEXT: v_lshlrev_b64 v[5:6], v8, s[0:1]
+; GFX8-NEXT: v_or_b32_e32 v3, v1, v3
+; GFX8-NEXT: v_or_b32_e32 v4, v2, v4
+; GFX8-NEXT: v_lshlrev_b64 v[1:2], v10, s[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v7, 0x7f
+; GFX8-NEXT: v_cmp_gt_u32_e32 vcc, 64, v8
+; GFX8-NEXT: s_mov_b32 s8, 0
+; GFX8-NEXT: v_cndmask_b32_e32 v10, 0, v5, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v11, 0, v6, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX8-NEXT: v_mov_b32_e32 v3, s2
+; GFX8-NEXT: v_mov_b32_e32 v4, s3
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v8
+; GFX8-NEXT: s_lshr_b64 s[0:1], s[4:5], 1
+; GFX8-NEXT: s_lshl_b32 s9, s6, 31
+; GFX8-NEXT: v_bfi_b32 v13, v0, 0, v7
+; GFX8-NEXT: v_cndmask_b32_e32 v8, v1, v3, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v12, v2, v4, vcc
+; GFX8-NEXT: s_or_b64 s[0:1], s[0:1], s[8:9]
+; GFX8-NEXT: s_lshr_b64 s[2:3], s[6:7], 1
+; GFX8-NEXT: v_sub_u32_e32 v4, vcc, 64, v13
+; GFX8-NEXT: v_not_b32_e32 v9, 63
+; GFX8-NEXT: v_lshrrev_b64 v[2:3], v13, s[0:1]
+; GFX8-NEXT: v_lshlrev_b64 v[4:5], v4, s[2:3]
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v13, v9
+; GFX8-NEXT: v_or_b32_e32 v4, v2, v4
+; GFX8-NEXT: v_or_b32_e32 v5, v3, v5
+; GFX8-NEXT: v_lshrrev_b64 v[2:3], v9, s[2:3]
+; GFX8-NEXT: v_lshrrev_b64 v[0:1], v13, s[2:3]
+; GFX8-NEXT: v_cmp_gt_u32_e32 vcc, 64, v13
+; GFX8-NEXT: v_mov_b32_e32 v7, s1
+; GFX8-NEXT: v_mov_b32_e32 v6, s0
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v13
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v2, v6, s[0:1]
+; GFX8-NEXT: v_cndmask_b32_e64 v3, v3, v7, s[0:1]
+; GFX8-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc
+; GFX8-NEXT: v_or_b32_e32 v0, v10, v2
+; GFX8-NEXT: v_or_b32_e32 v1, v11, v3
+; GFX8-NEXT: v_or_b32_e32 v2, v8, v4
+; GFX8-NEXT: v_or_b32_e32 v3, v12, v5
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: v_fshl_i128_ssv:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v1, 64, v0
-; GFX11-NEXT: v_dual_mov_b32 v3, s7 :: v_dual_mov_b32 v6, s6
-; GFX11-NEXT: v_mov_b32_e32 v5, s4
-; GFX11-NEXT: v_mov_b32_e32 v7, s0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[1:2]
-; GFX11-NEXT: v_mov_b32_e32 v1, s5
-; GFX11-NEXT: v_not_b32_e32 v9, v0
-; GFX11-NEXT: v_cndmask_b32_e32 v2, s1, v3, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e32 v4, s7, v1, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e32 v3, s6, v5, vcc_lo
-; GFX11-NEXT: v_mov_b32_e32 v5, s1
-; GFX11-NEXT: v_cndmask_b32_e32 v1, s0, v6, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_lshrrev_b64 v[3:4], 1, v[3:4]
-; GFX11-NEXT: v_cndmask_b32_e32 v6, s3, v5, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e32 v5, s2, v7, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_lshrrev_b64 v[7:8], 1, v[1:2]
-; GFX11-NEXT: v_lshlrev_b64 v[1:2], v0, v[1:2]
-; GFX11-NEXT: v_lshrrev_b64 v[3:4], v9, v[3:4]
-; GFX11-NEXT: v_lshlrev_b64 v[5:6], v0, v[5:6]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_lshrrev_b64 v[7:8], v9, v[7:8]
-; GFX11-NEXT: v_or_b32_e32 v0, v1, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_or_b32_e32 v1, v2, v4
-; GFX11-NEXT: v_or_b32_e32 v2, v5, v7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v3, v6, v8
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshl_i128_ssv:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_and_b32_e32 v8, 0x7f, v0
@@ -6502,6 +6867,7 @@ define amdgpu_ps <4 x float> @v_fshl_i128_ssv(i128 inreg %lhs, i128 inreg %rhs,
; GFX9-NEXT: v_or_b32_e32 v2, v8, v4
; GFX9-NEXT: v_or_b32_e32 v3, v11, v5
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: v_fshl_i128_ssv:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_and_b32_e32 v11, 0x7f, v0
@@ -6550,6 +6916,60 @@ define amdgpu_ps <4 x float> @v_fshl_i128_ssv(i128 inreg %lhs, i128 inreg %rhs,
; GFX10-NEXT: v_or_b32_e32 v2, v7, v2
; GFX10-NEXT: v_or_b32_e32 v3, v8, v3
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: v_fshl_i128_ssv:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_and_b32_e32 v11, 0x7f, v0
+; GFX11-NEXT: v_bfi_b32 v12, v0, 0, 0x7f
+; GFX11-NEXT: s_mov_b32 s8, 0
+; GFX11-NEXT: s_lshr_b64 s[4:5], s[4:5], 1
+; GFX11-NEXT: s_lshl_b32 s9, s6, 31
+; GFX11-NEXT: v_lshlrev_b64 v[5:6], v11, s[0:1]
+; GFX11-NEXT: v_cmp_gt_u32_e32 vcc_lo, 64, v11
+; GFX11-NEXT: v_sub_nc_u32_e32 v9, 64, v12
+; GFX11-NEXT: s_or_b64 s[4:5], s[4:5], s[8:9]
+; GFX11-NEXT: s_lshr_b64 s[6:7], s[6:7], 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_dual_cndmask_b32 v14, 0, v5 :: v_dual_add_nc_u32 v13, 0xffffffc0, v12
+; GFX11-NEXT: v_sub_nc_u32_e32 v3, 64, v11
+; GFX11-NEXT: v_lshlrev_b64 v[1:2], v11, s[2:3]
+; GFX11-NEXT: v_lshlrev_b64 v[9:10], v9, s[6:7]
+; GFX11-NEXT: v_cndmask_b32_e32 v6, 0, v6, vcc_lo
+; GFX11-NEXT: v_lshrrev_b64 v[3:4], v3, s[0:1]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_or_b32_e32 v4, v4, v2
+; GFX11-NEXT: v_add_nc_u32_e32 v7, 0xffffffc0, v11
+; GFX11-NEXT: v_or_b32_e32 v3, v3, v1
+; GFX11-NEXT: v_lshrrev_b64 v[0:1], v12, s[4:5]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_lshlrev_b64 v[7:8], v7, s[0:1]
+; GFX11-NEXT: v_cmp_gt_u32_e64 s1, 64, v12
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 0, v11
+; GFX11-NEXT: v_or_b32_e32 v5, v0, v9
+; GFX11-NEXT: v_mov_b32_e32 v0, s4
+; GFX11-NEXT: v_or_b32_e32 v9, v1, v10
+; GFX11-NEXT: v_cndmask_b32_e32 v7, v7, v3, vcc_lo
+; GFX11-NEXT: v_lshrrev_b64 v[2:3], v13, s[6:7]
+; GFX11-NEXT: v_dual_cndmask_b32 v8, v8, v4 :: v_dual_mov_b32 v1, s5
+; GFX11-NEXT: v_cmp_eq_u32_e64 s4, 0, v12
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_cndmask_b32_e64 v7, v7, s2, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, v5, s1
+; GFX11-NEXT: v_lshrrev_b64 v[4:5], v12, s[6:7]
+; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, v9, s1
+; GFX11-NEXT: v_cndmask_b32_e64 v8, v8, s3, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e64 v0, v2, v0, s4
+; GFX11-NEXT: v_cndmask_b32_e64 v1, v3, v1, s4
+; GFX11-NEXT: v_cndmask_b32_e64 v2, 0, v4, s1
+; GFX11-NEXT: v_cndmask_b32_e64 v3, 0, v5, s1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v0, v14, v0
+; GFX11-NEXT: v_or_b32_e32 v1, v6, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v2, v7, v2
+; GFX11-NEXT: v_or_b32_e32 v3, v8, v3
+; GFX11-NEXT: ; return to shader part epilog
%result = call i128 @llvm.fshl.i128(i128 %lhs, i128 %rhs, i128 %amt)
%cast.result = bitcast i128 %result to <4 x float>
ret <4 x float> %cast.result
@@ -6672,61 +7092,6 @@ define amdgpu_ps <4 x float> @v_fshl_i128_svs(i128 inreg %lhs, i128 %rhs, i128 i
; GFX8-NEXT: v_or_b32_e32 v3, v3, v5
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: v_fshl_i128_svs:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s5, s4, 0x7f
-; GFX11-NEXT: v_lshrrev_b64 v[0:1], 1, v[0:1]
-; GFX11-NEXT: s_sub_i32 s12, s5, 64
-; GFX11-NEXT: s_sub_i32 s6, 64, s5
-; GFX11-NEXT: s_cmp_lt_u32 s5, 64
-; GFX11-NEXT: s_cselect_b32 s13, 1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s5, 0
-; GFX11-NEXT: v_lshl_or_b32 v1, v2, 31, v1
-; GFX11-NEXT: s_cselect_b32 s5, 1, 0
-; GFX11-NEXT: s_lshr_b64 s[6:7], s[0:1], s6
-; GFX11-NEXT: s_lshl_b64 s[8:9], s[2:3], s4
-; GFX11-NEXT: s_lshl_b64 s[10:11], s[0:1], s4
-; GFX11-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], s12
-; GFX11-NEXT: s_cmp_lg_u32 s13, 0
-; GFX11-NEXT: v_lshrrev_b64 v[2:3], 1, v[2:3]
-; GFX11-NEXT: s_cselect_b64 s[8:9], s[10:11], 0
-; GFX11-NEXT: s_cselect_b64 s[0:1], s[6:7], s[0:1]
-; GFX11-NEXT: s_cmp_lg_u32 s5, 0
-; GFX11-NEXT: s_cselect_b64 s[0:1], s[2:3], s[0:1]
-; GFX11-NEXT: s_and_not1_b32 s2, 0x7f, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_sub_i32 s4, 64, s2
-; GFX11-NEXT: v_lshrrev_b64 v[4:5], s2, v[0:1]
-; GFX11-NEXT: v_lshlrev_b64 v[6:7], s4, v[2:3]
-; GFX11-NEXT: s_sub_i32 s3, s2, 64
-; GFX11-NEXT: s_cmp_lt_u32 s2, 64
-; GFX11-NEXT: v_lshrrev_b64 v[8:9], s2, v[2:3]
-; GFX11-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s2, 0
-; GFX11-NEXT: v_lshrrev_b64 v[2:3], s3, v[2:3]
-; GFX11-NEXT: v_or_b32_e32 v4, v4, v6
-; GFX11-NEXT: v_or_b32_e32 v5, v5, v7
-; GFX11-NEXT: s_cselect_b32 s5, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s4, 0
-; GFX11-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
-; GFX11-NEXT: s_cmp_lg_u32 s5, 0
-; GFX11-NEXT: v_dual_cndmask_b32 v2, v2, v4 :: v_dual_cndmask_b32 v3, v3, v5
-; GFX11-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
-; GFX11-NEXT: s_cmp_lg_u32 s4, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_dual_cndmask_b32 v4, v2, v0 :: v_dual_cndmask_b32 v5, v3, v1
-; GFX11-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
-; GFX11-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
-; GFX11-NEXT: v_dual_cndmask_b32 v6, 0, v8 :: v_dual_mov_b32 v3, s1
-; GFX11-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_cndmask_b32 v7, 0, v9
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_or_b32_e32 v1, v1, v5
-; GFX11-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v3, v3, v7
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshl_i128_svs:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s5, s4, 0x7f
@@ -6783,6 +7148,7 @@ define amdgpu_ps <4 x float> @v_fshl_i128_svs(i128 inreg %lhs, i128 %rhs, i128 i
; GFX9-NEXT: v_or_b32_e32 v2, v2, v4
; GFX9-NEXT: v_or_b32_e32 v3, v3, v5
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: v_fshl_i128_svs:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s5, s4, 0x7f
@@ -6839,6 +7205,62 @@ define amdgpu_ps <4 x float> @v_fshl_i128_svs(i128 inreg %lhs, i128 %rhs, i128 i
; GFX10-NEXT: v_or_b32_e32 v2, v2, v6
; GFX10-NEXT: v_or_b32_e32 v3, v3, v7
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: v_fshl_i128_svs:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s5, s4, 0x7f
+; GFX11-NEXT: v_lshrrev_b64 v[0:1], 1, v[0:1]
+; GFX11-NEXT: s_sub_i32 s12, s5, 64
+; GFX11-NEXT: s_sub_i32 s6, 64, s5
+; GFX11-NEXT: s_cmp_lt_u32 s5, 64
+; GFX11-NEXT: s_cselect_b32 s13, 1, 0
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: v_lshl_or_b32 v1, v2, 31, v1
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: s_lshr_b64 s[6:7], s[0:1], s6
+; GFX11-NEXT: s_lshl_b64 s[8:9], s[2:3], s4
+; GFX11-NEXT: s_lshl_b64 s[10:11], s[0:1], s4
+; GFX11-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
+; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], s12
+; GFX11-NEXT: s_cmp_lg_u32 s13, 0
+; GFX11-NEXT: v_lshrrev_b64 v[2:3], 1, v[2:3]
+; GFX11-NEXT: s_cselect_b64 s[8:9], s[10:11], 0
+; GFX11-NEXT: s_cselect_b64 s[0:1], s[6:7], s[0:1]
+; GFX11-NEXT: s_cmp_lg_u32 s5, 0
+; GFX11-NEXT: s_cselect_b64 s[0:1], s[2:3], s[0:1]
+; GFX11-NEXT: s_and_not1_b32 s2, 0x7f, s4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_sub_i32 s4, 64, s2
+; GFX11-NEXT: v_lshrrev_b64 v[4:5], s2, v[0:1]
+; GFX11-NEXT: v_lshlrev_b64 v[6:7], s4, v[2:3]
+; GFX11-NEXT: s_sub_i32 s3, s2, 64
+; GFX11-NEXT: s_cmp_lt_u32 s2, 64
+; GFX11-NEXT: v_lshrrev_b64 v[8:9], s2, v[2:3]
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_eq_u32 s2, 0
+; GFX11-NEXT: v_lshrrev_b64 v[2:3], s3, v[2:3]
+; GFX11-NEXT: v_or_b32_e32 v4, v4, v6
+; GFX11-NEXT: v_or_b32_e32 v5, v5, v7
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 0
+; GFX11-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
+; GFX11-NEXT: s_cmp_lg_u32 s5, 0
+; GFX11-NEXT: v_dual_cndmask_b32 v2, v2, v4 :: v_dual_cndmask_b32 v3, v3, v5
+; GFX11-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_dual_cndmask_b32 v4, v2, v0 :: v_dual_cndmask_b32 v5, v3, v1
+; GFX11-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
+; GFX11-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
+; GFX11-NEXT: v_dual_cndmask_b32 v6, 0, v8 :: v_dual_mov_b32 v3, s1
+; GFX11-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_cndmask_b32 v7, 0, v9
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX11-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX11-NEXT: ; return to shader part epilog
%result = call i128 @llvm.fshl.i128(i128 %lhs, i128 %rhs, i128 %amt)
%cast.result = bitcast i128 %result to <4 x float>
ret <4 x float> %cast.result
@@ -6847,89 +7269,124 @@ define amdgpu_ps <4 x float> @v_fshl_i128_svs(i128 inreg %lhs, i128 %rhs, i128 i
define amdgpu_ps <4 x float> @v_fshl_i128_vss(i128 %lhs, i128 inreg %rhs, i128 inreg %amt) {
; GFX6-LABEL: v_fshl_i128_vss:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_and_b32 s6, s4, 64
-; GFX6-NEXT: s_mov_b32 s7, 0
-; GFX6-NEXT: v_cmp_ne_u64_e64 vcc, s[6:7], 0
-; GFX6-NEXT: v_mov_b32_e32 v4, s3
-; GFX6-NEXT: v_cndmask_b32_e32 v6, v1, v4, vcc
-; GFX6-NEXT: v_mov_b32_e32 v4, s2
-; GFX6-NEXT: v_cndmask_b32_e32 v5, v0, v4, vcc
-; GFX6-NEXT: s_and_b64 s[6:7], vcc, exec
-; GFX6-NEXT: s_cselect_b32 s1, s1, s3
-; GFX6-NEXT: s_cselect_b32 s0, s0, s2
-; GFX6-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX6-NEXT: v_lshr_b64 v[2:3], v[5:6], 1
-; GFX6-NEXT: v_lshl_b64 v[7:8], v[5:6], s4
+; GFX6-NEXT: s_and_b32 s5, s4, 0x7f
+; GFX6-NEXT: s_sub_i32 s7, s5, 64
+; GFX6-NEXT: s_sub_i32 s8, 64, s5
+; GFX6-NEXT: s_cmp_lt_u32 s5, 64
+; GFX6-NEXT: s_cselect_b32 s9, 1, 0
+; GFX6-NEXT: s_cmp_eq_u32 s5, 0
+; GFX6-NEXT: s_cselect_b32 s10, 1, 0
+; GFX6-NEXT: v_lshr_b64 v[4:5], v[0:1], s8
+; GFX6-NEXT: v_lshl_b64 v[6:7], v[2:3], s5
+; GFX6-NEXT: v_lshl_b64 v[8:9], v[0:1], s5
+; GFX6-NEXT: s_cmp_lg_u32 s9, 0
+; GFX6-NEXT: v_lshl_b64 v[0:1], v[0:1], s7
+; GFX6-NEXT: s_cselect_b64 vcc, exec, 0
+; GFX6-NEXT: v_or_b32_e32 v4, v4, v6
+; GFX6-NEXT: v_or_b32_e32 v5, v5, v7
+; GFX6-NEXT: v_cndmask_b32_e32 v6, 0, v8, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v7, 0, v9, vcc
+; GFX6-NEXT: s_cselect_b64 vcc, exec, 0
+; GFX6-NEXT: s_cmp_lg_u32 s10, 0
+; GFX6-NEXT: s_mov_b32 s6, 0
+; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc
+; GFX6-NEXT: s_cselect_b64 vcc, exec, 0
; GFX6-NEXT: s_lshr_b64 s[0:1], s[0:1], 1
-; GFX6-NEXT: s_not_b32 s2, s4
-; GFX6-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; GFX6-NEXT: v_lshl_b64 v[5:6], v[0:1], s4
-; GFX6-NEXT: v_lshr_b64 v[1:2], v[2:3], s2
-; GFX6-NEXT: v_or_b32_e32 v4, s1, v8
-; GFX6-NEXT: v_or_b32_e32 v0, s0, v7
-; GFX6-NEXT: v_or_b32_e32 v3, v6, v2
-; GFX6-NEXT: v_or_b32_e32 v2, v5, v1
-; GFX6-NEXT: v_mov_b32_e32 v1, v4
+; GFX6-NEXT: s_lshl_b32 s7, s2, 31
+; GFX6-NEXT: s_or_b64 s[0:1], s[0:1], s[6:7]
+; GFX6-NEXT: s_not_b32 s6, s4
+; GFX6-NEXT: s_andn2_b32 s4, 0x7f, s4
+; GFX6-NEXT: s_lshr_b64 s[2:3], s[2:3], 1
+; GFX6-NEXT: s_sub_i32 s10, s4, 64
+; GFX6-NEXT: s_sub_i32 s8, 64, s4
+; GFX6-NEXT: s_cmp_lt_u32 s4, 64
+; GFX6-NEXT: s_cselect_b32 s11, 1, 0
+; GFX6-NEXT: s_cmp_eq_u32 s4, 0
+; GFX6-NEXT: s_cselect_b32 s12, 1, 0
+; GFX6-NEXT: s_lshr_b64 s[4:5], s[2:3], s6
+; GFX6-NEXT: s_lshr_b64 s[6:7], s[0:1], s6
+; GFX6-NEXT: s_lshl_b64 s[8:9], s[2:3], s8
+; GFX6-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
+; GFX6-NEXT: s_lshr_b64 s[2:3], s[2:3], s10
+; GFX6-NEXT: s_cmp_lg_u32 s11, 0
+; GFX6-NEXT: s_cselect_b64 s[2:3], s[6:7], s[2:3]
+; GFX6-NEXT: s_cmp_lg_u32 s12, 0
+; GFX6-NEXT: s_cselect_b64 s[0:1], s[0:1], s[2:3]
+; GFX6-NEXT: s_cmp_lg_u32 s11, 0
+; GFX6-NEXT: s_cselect_b64 s[2:3], s[4:5], 0
+; GFX6-NEXT: v_cndmask_b32_e32 v4, v0, v2, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v5, v1, v3, vcc
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: v_mov_b32_e32 v1, s1
+; GFX6-NEXT: v_mov_b32_e32 v2, s2
+; GFX6-NEXT: v_mov_b32_e32 v3, s3
+; GFX6-NEXT: v_or_b32_e32 v0, v6, v0
+; GFX6-NEXT: v_or_b32_e32 v1, v7, v1
+; GFX6-NEXT: v_or_b32_e32 v2, v4, v2
+; GFX6-NEXT: v_or_b32_e32 v3, v5, v3
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: v_fshl_i128_vss:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_and_b32 s6, s4, 64
-; GFX8-NEXT: s_mov_b32 s7, 0
-; GFX8-NEXT: s_cmp_lg_u64 s[6:7], 0
-; GFX8-NEXT: v_mov_b32_e32 v4, s3
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v1, v4, vcc
-; GFX8-NEXT: v_mov_b32_e32 v4, s2
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v0, v4, vcc
-; GFX8-NEXT: s_and_b64 s[6:7], vcc, exec
-; GFX8-NEXT: s_cselect_b32 s1, s1, s3
-; GFX8-NEXT: s_cselect_b32 s0, s0, s2
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX8-NEXT: v_lshrrev_b64 v[2:3], 1, v[5:6]
-; GFX8-NEXT: v_lshlrev_b64 v[7:8], s4, v[5:6]
+; GFX8-NEXT: s_and_b32 s5, s4, 0x7f
+; GFX8-NEXT: s_sub_i32 s7, s5, 64
+; GFX8-NEXT: s_sub_i32 s8, 64, s5
+; GFX8-NEXT: s_cmp_lt_u32 s5, 64
+; GFX8-NEXT: s_cselect_b32 s9, 1, 0
+; GFX8-NEXT: s_cmp_eq_u32 s5, 0
+; GFX8-NEXT: s_cselect_b32 s10, 1, 0
+; GFX8-NEXT: v_lshrrev_b64 v[4:5], s8, v[0:1]
+; GFX8-NEXT: v_lshlrev_b64 v[6:7], s5, v[2:3]
+; GFX8-NEXT: v_lshlrev_b64 v[8:9], s5, v[0:1]
+; GFX8-NEXT: s_cmp_lg_u32 s9, 0
+; GFX8-NEXT: v_lshlrev_b64 v[0:1], s7, v[0:1]
+; GFX8-NEXT: s_cselect_b64 vcc, exec, 0
+; GFX8-NEXT: v_or_b32_e32 v4, v4, v6
+; GFX8-NEXT: v_or_b32_e32 v5, v5, v7
+; GFX8-NEXT: v_cndmask_b32_e32 v6, 0, v8, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v7, 0, v9, vcc
+; GFX8-NEXT: s_cselect_b64 vcc, exec, 0
+; GFX8-NEXT: s_cmp_lg_u32 s10, 0
+; GFX8-NEXT: s_mov_b32 s6, 0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc
+; GFX8-NEXT: s_cselect_b64 vcc, exec, 0
; GFX8-NEXT: s_lshr_b64 s[0:1], s[0:1], 1
-; GFX8-NEXT: s_not_b32 s2, s4
-; GFX8-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; GFX8-NEXT: v_lshlrev_b64 v[5:6], s4, v[0:1]
-; GFX8-NEXT: v_lshrrev_b64 v[1:2], s2, v[2:3]
-; GFX8-NEXT: v_or_b32_e32 v4, s1, v8
-; GFX8-NEXT: v_or_b32_e32 v0, s0, v7
-; GFX8-NEXT: v_or_b32_e32 v3, v6, v2
-; GFX8-NEXT: v_or_b32_e32 v2, v5, v1
-; GFX8-NEXT: v_mov_b32_e32 v1, v4
+; GFX8-NEXT: s_lshl_b32 s7, s2, 31
+; GFX8-NEXT: s_or_b64 s[0:1], s[0:1], s[6:7]
+; GFX8-NEXT: s_not_b32 s6, s4
+; GFX8-NEXT: s_andn2_b32 s4, 0x7f, s4
+; GFX8-NEXT: s_lshr_b64 s[2:3], s[2:3], 1
+; GFX8-NEXT: s_sub_i32 s10, s4, 64
+; GFX8-NEXT: s_sub_i32 s8, 64, s4
+; GFX8-NEXT: s_cmp_lt_u32 s4, 64
+; GFX8-NEXT: s_cselect_b32 s11, 1, 0
+; GFX8-NEXT: s_cmp_eq_u32 s4, 0
+; GFX8-NEXT: s_cselect_b32 s12, 1, 0
+; GFX8-NEXT: s_lshr_b64 s[4:5], s[2:3], s6
+; GFX8-NEXT: s_lshr_b64 s[6:7], s[0:1], s6
+; GFX8-NEXT: s_lshl_b64 s[8:9], s[2:3], s8
+; GFX8-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
+; GFX8-NEXT: s_lshr_b64 s[2:3], s[2:3], s10
+; GFX8-NEXT: s_cmp_lg_u32 s11, 0
+; GFX8-NEXT: s_cselect_b64 s[2:3], s[6:7], s[2:3]
+; GFX8-NEXT: s_cmp_lg_u32 s12, 0
+; GFX8-NEXT: s_cselect_b64 s[0:1], s[0:1], s[2:3]
+; GFX8-NEXT: s_cmp_lg_u32 s11, 0
+; GFX8-NEXT: s_cselect_b64 s[2:3], s[4:5], 0
+; GFX8-NEXT: v_cndmask_b32_e32 v4, v0, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v1, v3, vcc
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: v_mov_b32_e32 v2, s2
+; GFX8-NEXT: v_mov_b32_e32 v3, s3
+; GFX8-NEXT: v_or_b32_e32 v0, v6, v0
+; GFX8-NEXT: v_or_b32_e32 v1, v7, v1
+; GFX8-NEXT: v_or_b32_e32 v2, v4, v2
+; GFX8-NEXT: v_or_b32_e32 v3, v5, v3
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: v_fshl_i128_vss:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s6, s4, 64
-; GFX11-NEXT: s_mov_b32 s7, 0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_cmp_lg_u64 s[6:7], 0
-; GFX11-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX11-NEXT: v_cndmask_b32_e64 v5, v1, s3, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e64 v4, v0, s2, vcc_lo
-; GFX11-NEXT: v_dual_cndmask_b32 v1, v3, v1 :: v_dual_cndmask_b32 v0, v2, v0
-; GFX11-NEXT: s_and_b32 s5, vcc_lo, exec_lo
-; GFX11-NEXT: s_cselect_b32 s1, s1, s3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_lshrrev_b64 v[2:3], 1, v[4:5]
-; GFX11-NEXT: s_cselect_b32 s0, s0, s2
-; GFX11-NEXT: s_not_b32 s2, s4
-; GFX11-NEXT: v_lshlrev_b64 v[4:5], s4, v[4:5]
-; GFX11-NEXT: v_lshlrev_b64 v[6:7], s4, v[0:1]
-; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], 1
-; GFX11-NEXT: v_lshrrev_b64 v[2:3], s2, v[2:3]
-; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_or_b32_e32 v0, s0, v4
-; GFX11-NEXT: v_or_b32_e32 v1, s1, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v2, v6, v2
-; GFX11-NEXT: v_or_b32_e32 v3, v7, v3
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshl_i128_vss:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s5, s4, 0x7f
@@ -6989,6 +7446,7 @@ define amdgpu_ps <4 x float> @v_fshl_i128_vss(i128 %lhs, i128 inreg %rhs, i128 i
; GFX9-NEXT: v_or_b32_e32 v2, v4, v2
; GFX9-NEXT: v_or_b32_e32 v3, v5, v3
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: v_fshl_i128_vss:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s5, s4, 0x7f
@@ -7048,38 +7506,92 @@ define amdgpu_ps <4 x float> @v_fshl_i128_vss(i128 %lhs, i128 inreg %rhs, i128 i
; GFX10-NEXT: v_or_b32_e32 v2, v2, v4
; GFX10-NEXT: v_or_b32_e32 v3, v3, v5
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: v_fshl_i128_vss:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s5, s4, 0x7f
+; GFX11-NEXT: s_mov_b32 s8, 0
+; GFX11-NEXT: s_sub_i32 s6, 64, s5
+; GFX11-NEXT: s_sub_i32 s7, s5, 64
+; GFX11-NEXT: s_cmp_lt_u32 s5, 64
+; GFX11-NEXT: v_lshrrev_b64 v[4:5], s6, v[0:1]
+; GFX11-NEXT: s_cselect_b32 s6, 1, 0
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: v_lshlrev_b64 v[6:7], s5, v[2:3]
+; GFX11-NEXT: s_cselect_b32 s9, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s6, 0
+; GFX11-NEXT: v_lshlrev_b64 v[8:9], s5, v[0:1]
+; GFX11-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
+; GFX11-NEXT: s_cselect_b32 s5, exec_lo, 0
+; GFX11-NEXT: s_cmp_lg_u32 s9, 0
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], s7, v[0:1]
+; GFX11-NEXT: s_cselect_b32 s6, exec_lo, 0
+; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], 1
+; GFX11-NEXT: s_lshl_b32 s9, s2, 31
+; GFX11-NEXT: s_and_not1_b32 s7, 0x7f, s4
+; GFX11-NEXT: v_or_b32_e32 v4, v4, v6
+; GFX11-NEXT: v_or_b32_e32 v5, v5, v7
+; GFX11-NEXT: s_or_b64 s[0:1], s[0:1], s[8:9]
+; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], 1
+; GFX11-NEXT: s_not_b32 s10, s4
+; GFX11-NEXT: s_sub_i32 s12, s7, 64
+; GFX11-NEXT: s_sub_i32 s8, 64, s7
+; GFX11-NEXT: s_cmp_lt_u32 s7, 64
+; GFX11-NEXT: v_cndmask_b32_e64 v4, v0, v4, s5
+; GFX11-NEXT: s_cselect_b32 s13, 1, 0
+; GFX11-NEXT: s_cmp_eq_u32 s7, 0
+; GFX11-NEXT: v_dual_cndmask_b32 v6, 0, v8 :: v_dual_cndmask_b32 v7, 0, v9
+; GFX11-NEXT: s_cselect_b32 s7, 1, 0
+; GFX11-NEXT: v_cndmask_b32_e64 v8, v1, v5, s5
+; GFX11-NEXT: s_lshr_b64 s[4:5], s[0:1], s10
+; GFX11-NEXT: s_lshl_b64 s[8:9], s[2:3], s8
+; GFX11-NEXT: s_lshr_b64 s[10:11], s[2:3], s10
+; GFX11-NEXT: s_or_b64 s[4:5], s[4:5], s[8:9]
+; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], s12
+; GFX11-NEXT: s_cmp_lg_u32 s13, 0
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v4, v2, s6
+; GFX11-NEXT: s_cselect_b64 s[2:3], s[4:5], s[2:3]
+; GFX11-NEXT: s_cmp_lg_u32 s7, 0
+; GFX11-NEXT: v_cndmask_b32_e64 v3, v8, v3, s6
+; GFX11-NEXT: s_cselect_b64 s[0:1], s[0:1], s[2:3]
+; GFX11-NEXT: s_cmp_lg_u32 s13, 0
+; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT: s_cselect_b64 s[2:3], s[10:11], 0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_dual_mov_b32 v5, s3 :: v_dual_mov_b32 v4, s2
+; GFX11-NEXT: v_or_b32_e32 v0, v6, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_or_b32_e32 v1, v7, v1
+; GFX11-NEXT: v_or_b32_e32 v3, v3, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v2, v2, v4
+; GFX11-NEXT: ; return to shader part epilog
%result = call i128 @llvm.fshl.i128(i128 %lhs, i128 %rhs, i128 %amt)
%cast.result = bitcast i128 %result to <4 x float>
ret <4 x float> %cast.result
}
define amdgpu_ps i128 @s_fshl_i128_65(i128 inreg %lhs, i128 inreg %rhs) {
-; GCN-LABEL: s_fshl_i128_65:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_lshl_b64 s[2:3], s[0:1], 1
-; GCN-NEXT: s_lshr_b32 s10, s7, 31
-; GCN-NEXT: s_mov_b32 s11, 0
-; GCN-NEXT: s_or_b64 s[8:9], s[2:3], s[10:11]
-; GCN-NEXT: s_lshl_b64 s[0:1], s[6:7], 1
-; GCN-NEXT: s_lshr_b32 s10, s5, 31
-; GCN-NEXT: s_or_b64 s[6:7], s[0:1], s[10:11]
-; GCN-NEXT: s_mov_b32 s0, s6
-; GCN-NEXT: s_mov_b32 s2, s8
-; GCN-NEXT: ; return to shader part epilog
+; GFX6-LABEL: s_fshl_i128_65:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_lshl_b64 s[2:3], s[0:1], 1
+; GFX6-NEXT: s_lshr_b32 s4, s5, 31
+; GFX6-NEXT: s_lshl_b64 s[0:1], s[6:7], 1
+; GFX6-NEXT: s_or_b32 s0, s0, s4
+; GFX6-NEXT: s_lshr_b32 s4, s7, 31
+; GFX6-NEXT: s_or_b32 s2, s2, s4
+; GFX6-NEXT: ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshl_i128_65:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_lshl_b64 s[2:3], s[0:1], 1
+; GFX8-NEXT: s_lshr_b32 s4, s5, 31
+; GFX8-NEXT: s_lshl_b64 s[0:1], s[6:7], 1
+; GFX8-NEXT: s_or_b32 s0, s0, s4
+; GFX8-NEXT: s_lshr_b32 s4, s7, 31
+; GFX8-NEXT: s_or_b32 s2, s2, s4
+; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshl_i128_65:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_lshl_b64 s[2:3], s[0:1], 1
-; GFX11-NEXT: s_lshr_b32 s10, s7, 31
-; GFX11-NEXT: s_mov_b32 s11, 0
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[6:7], 1
-; GFX11-NEXT: s_or_b64 s[8:9], s[2:3], s[10:11]
-; GFX11-NEXT: s_lshr_b32 s10, s5, 31
-; GFX11-NEXT: s_mov_b32 s2, s8
-; GFX11-NEXT: s_or_b64 s[6:7], s[0:1], s[10:11]
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_mov_b32 s0, s6
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshl_i128_65:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_lshl_b64 s[2:3], s[0:1], 1
@@ -7089,6 +7601,7 @@ define amdgpu_ps i128 @s_fshl_i128_65(i128 inreg %lhs, i128 inreg %rhs) {
; GFX9-NEXT: s_lshr_b32 s4, s7, 31
; GFX9-NEXT: s_or_b32 s2, s2, s4
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: s_fshl_i128_65:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_lshl_b64 s[2:3], s[0:1], 1
@@ -7098,6 +7611,16 @@ define amdgpu_ps i128 @s_fshl_i128_65(i128 inreg %lhs, i128 inreg %rhs) {
; GFX10-NEXT: s_or_b32 s0, s0, s4
; GFX10-NEXT: s_or_b32 s2, s2, s5
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: s_fshl_i128_65:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_lshl_b64 s[2:3], s[0:1], 1
+; GFX11-NEXT: s_lshr_b32 s4, s5, 31
+; GFX11-NEXT: s_lshl_b64 s[0:1], s[6:7], 1
+; GFX11-NEXT: s_lshr_b32 s5, s7, 31
+; GFX11-NEXT: s_or_b32 s0, s0, s4
+; GFX11-NEXT: s_or_b32 s2, s2, s5
+; GFX11-NEXT: ; return to shader part epilog
%result = call i128 @llvm.fshl.i128(i128 %lhs, i128 %rhs, i128 65)
ret i128 %result
}
@@ -7107,35 +7630,24 @@ define i128 @v_fshl_i128_65(i128 %lhs, i128 %rhs) {
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_lshl_b64 v[2:3], v[0:1], 1
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 31, v7
-; GFX6-NEXT: v_or_b32_e32 v2, v2, v0
; GFX6-NEXT: v_lshl_b64 v[0:1], v[6:7], 1
; GFX6-NEXT: v_lshrrev_b32_e32 v4, 31, v5
; GFX6-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 31, v7
+; GFX6-NEXT: v_or_b32_e32 v2, v2, v4
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_fshl_i128_65:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b64 v[2:3], 1, v[0:1]
-; GFX8-NEXT: v_lshrrev_b32_e32 v0, 31, v7
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v0
; GFX8-NEXT: v_lshlrev_b64 v[0:1], 1, v[6:7]
; GFX8-NEXT: v_lshrrev_b32_e32 v4, 31, v5
; GFX8-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX8-NEXT: v_lshrrev_b32_e32 v4, 31, v7
+; GFX8-NEXT: v_or_b32_e32 v2, v2, v4
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshl_i128_65:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshlrev_b64 v[2:3], 1, v[0:1]
-; GFX11-NEXT: v_lshlrev_b64 v[0:1], 1, v[6:7]
-; GFX11-NEXT: v_lshrrev_b32_e32 v4, 31, v5
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 31, v7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX11-NEXT: v_or_b32_e32 v2, v2, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_i128_65:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -7146,6 +7658,7 @@ define i128 @v_fshl_i128_65(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: v_lshrrev_b32_e32 v4, 31, v7
; GFX9-NEXT: v_or_b32_e32 v2, v2, v4
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshl_i128_65:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -7156,6 +7669,18 @@ define i128 @v_fshl_i128_65(i128 %lhs, i128 %rhs) {
; GFX10-NEXT: v_or_b32_e32 v0, v0, v4
; GFX10-NEXT: v_or_b32_e32 v2, v2, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_fshl_i128_65:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b64 v[2:3], 1, v[0:1]
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], 1, v[6:7]
+; GFX11-NEXT: v_lshrrev_b32_e32 v4, 31, v5
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, 31, v7
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX11-NEXT: v_or_b32_e32 v2, v2, v5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call i128 @llvm.fshl.i128(i128 %lhs, i128 %rhs, i128 65)
ret i128 %result
}
@@ -7163,128 +7688,182 @@ define i128 @v_fshl_i128_65(i128 %lhs, i128 %rhs) {
define amdgpu_ps <2 x i128> @s_fshl_v2i128(<2 x i128> inreg %lhs, <2 x i128> inreg %rhs, <2 x i128> inreg %amt) {
; GFX6-LABEL: s_fshl_v2i128:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_and_b32 s18, s20, 64
-; GFX6-NEXT: s_mov_b32 s19, 0
-; GFX6-NEXT: v_cmp_ne_u64_e64 s[22:23], s[18:19], 0
-; GFX6-NEXT: s_and_b64 s[24:25], s[22:23], exec
-; GFX6-NEXT: s_cselect_b32 s7, s5, s7
-; GFX6-NEXT: s_cselect_b32 s6, s4, s6
-; GFX6-NEXT: s_lshl_b64 s[6:7], s[6:7], s20
-; GFX6-NEXT: s_and_b64 s[24:25], s[22:23], exec
-; GFX6-NEXT: s_cselect_b32 s5, s15, s5
-; GFX6-NEXT: s_cselect_b32 s4, s14, s4
-; GFX6-NEXT: s_lshr_b64 s[24:25], s[4:5], 1
-; GFX6-NEXT: s_not_b32 s17, s20
-; GFX6-NEXT: s_lshr_b64 s[24:25], s[24:25], s17
-; GFX6-NEXT: s_or_b64 s[6:7], s[6:7], s[24:25]
-; GFX6-NEXT: s_lshl_b64 s[4:5], s[4:5], s20
-; GFX6-NEXT: s_and_b64 s[20:21], s[22:23], exec
-; GFX6-NEXT: s_cselect_b32 s13, s13, s15
-; GFX6-NEXT: s_cselect_b32 s12, s12, s14
-; GFX6-NEXT: s_lshr_b64 s[12:13], s[12:13], 1
-; GFX6-NEXT: s_and_b32 s18, s16, 64
-; GFX6-NEXT: s_lshr_b64 s[12:13], s[12:13], s17
-; GFX6-NEXT: v_cmp_ne_u64_e64 s[14:15], s[18:19], 0
-; GFX6-NEXT: s_or_b64 s[4:5], s[4:5], s[12:13]
-; GFX6-NEXT: s_and_b64 s[12:13], s[14:15], exec
-; GFX6-NEXT: s_cselect_b32 s3, s1, s3
-; GFX6-NEXT: s_cselect_b32 s2, s0, s2
-; GFX6-NEXT: s_lshl_b64 s[2:3], s[2:3], s16
-; GFX6-NEXT: s_and_b64 s[12:13], s[14:15], exec
-; GFX6-NEXT: s_cselect_b32 s1, s11, s1
-; GFX6-NEXT: s_cselect_b32 s0, s10, s0
-; GFX6-NEXT: s_lshr_b64 s[12:13], s[0:1], 1
+; GFX6-NEXT: s_and_b32 s17, s16, 0x7f
+; GFX6-NEXT: s_sub_i32 s19, s17, 64
+; GFX6-NEXT: s_sub_i32 s21, 64, s17
+; GFX6-NEXT: s_cmp_lt_u32 s17, 64
+; GFX6-NEXT: s_cselect_b32 s28, 1, 0
+; GFX6-NEXT: s_cmp_eq_u32 s17, 0
+; GFX6-NEXT: s_cselect_b32 s17, 1, 0
+; GFX6-NEXT: s_lshr_b64 s[24:25], s[0:1], s21
+; GFX6-NEXT: s_lshl_b64 s[26:27], s[2:3], s16
+; GFX6-NEXT: s_lshl_b64 s[22:23], s[0:1], s16
+; GFX6-NEXT: s_or_b64 s[24:25], s[24:25], s[26:27]
+; GFX6-NEXT: s_lshl_b64 s[0:1], s[0:1], s19
+; GFX6-NEXT: s_cmp_lg_u32 s28, 0
+; GFX6-NEXT: s_cselect_b64 s[22:23], s[22:23], 0
+; GFX6-NEXT: s_cselect_b64 s[0:1], s[24:25], s[0:1]
+; GFX6-NEXT: s_cmp_lg_u32 s17, 0
+; GFX6-NEXT: s_mov_b32 s18, 0
+; GFX6-NEXT: s_cselect_b64 s[2:3], s[2:3], s[0:1]
+; GFX6-NEXT: s_lshr_b64 s[0:1], s[8:9], 1
+; GFX6-NEXT: s_lshl_b32 s19, s10, 31
+; GFX6-NEXT: s_lshr_b64 s[8:9], s[10:11], 1
+; GFX6-NEXT: s_andn2_b32 s10, 0x7f, s16
+; GFX6-NEXT: s_or_b64 s[0:1], s[0:1], s[18:19]
; GFX6-NEXT: s_not_b32 s17, s16
-; GFX6-NEXT: s_lshr_b64 s[12:13], s[12:13], s17
-; GFX6-NEXT: s_or_b64 s[2:3], s[2:3], s[12:13]
-; GFX6-NEXT: s_lshl_b64 s[0:1], s[0:1], s16
-; GFX6-NEXT: s_and_b64 s[12:13], s[14:15], exec
-; GFX6-NEXT: s_cselect_b32 s9, s9, s11
-; GFX6-NEXT: s_cselect_b32 s8, s8, s10
-; GFX6-NEXT: s_lshr_b64 s[8:9], s[8:9], 1
-; GFX6-NEXT: s_lshr_b64 s[8:9], s[8:9], s17
-; GFX6-NEXT: s_or_b64 s[0:1], s[0:1], s[8:9]
+; GFX6-NEXT: s_sub_i32 s19, s10, 64
+; GFX6-NEXT: s_sub_i32 s21, 64, s10
+; GFX6-NEXT: s_cmp_lt_u32 s10, 64
+; GFX6-NEXT: s_cselect_b32 s26, 1, 0
+; GFX6-NEXT: s_cmp_eq_u32 s10, 0
+; GFX6-NEXT: s_cselect_b32 s27, 1, 0
+; GFX6-NEXT: s_lshr_b64 s[10:11], s[8:9], s17
+; GFX6-NEXT: s_lshr_b64 s[16:17], s[0:1], s17
+; GFX6-NEXT: s_lshl_b64 s[24:25], s[8:9], s21
+; GFX6-NEXT: s_or_b64 s[16:17], s[16:17], s[24:25]
+; GFX6-NEXT: s_lshr_b64 s[8:9], s[8:9], s19
+; GFX6-NEXT: s_cmp_lg_u32 s26, 0
+; GFX6-NEXT: s_cselect_b64 s[8:9], s[16:17], s[8:9]
+; GFX6-NEXT: s_cmp_lg_u32 s27, 0
+; GFX6-NEXT: s_cselect_b64 s[0:1], s[0:1], s[8:9]
+; GFX6-NEXT: s_cmp_lg_u32 s26, 0
+; GFX6-NEXT: s_cselect_b64 s[8:9], s[10:11], 0
+; GFX6-NEXT: s_or_b64 s[2:3], s[2:3], s[8:9]
+; GFX6-NEXT: s_and_b32 s8, s20, 0x7f
+; GFX6-NEXT: s_or_b64 s[0:1], s[22:23], s[0:1]
+; GFX6-NEXT: s_sub_i32 s19, s8, 64
+; GFX6-NEXT: s_sub_i32 s10, 64, s8
+; GFX6-NEXT: s_cmp_lt_u32 s8, 64
+; GFX6-NEXT: s_cselect_b32 s21, 1, 0
+; GFX6-NEXT: s_cmp_eq_u32 s8, 0
+; GFX6-NEXT: s_cselect_b32 s22, 1, 0
+; GFX6-NEXT: s_lshr_b64 s[10:11], s[4:5], s10
+; GFX6-NEXT: s_lshl_b64 s[16:17], s[6:7], s20
+; GFX6-NEXT: s_lshl_b64 s[8:9], s[4:5], s20
+; GFX6-NEXT: s_or_b64 s[10:11], s[10:11], s[16:17]
+; GFX6-NEXT: s_lshl_b64 s[4:5], s[4:5], s19
+; GFX6-NEXT: s_cmp_lg_u32 s21, 0
+; GFX6-NEXT: s_cselect_b64 s[8:9], s[8:9], 0
+; GFX6-NEXT: s_cselect_b64 s[4:5], s[10:11], s[4:5]
+; GFX6-NEXT: s_cmp_lg_u32 s22, 0
+; GFX6-NEXT: s_cselect_b64 s[6:7], s[6:7], s[4:5]
+; GFX6-NEXT: s_lshr_b64 s[4:5], s[12:13], 1
+; GFX6-NEXT: s_lshl_b32 s19, s14, 31
+; GFX6-NEXT: s_andn2_b32 s12, 0x7f, s20
+; GFX6-NEXT: s_or_b64 s[4:5], s[4:5], s[18:19]
+; GFX6-NEXT: s_lshr_b64 s[10:11], s[14:15], 1
+; GFX6-NEXT: s_not_b32 s14, s20
+; GFX6-NEXT: s_sub_i32 s18, s12, 64
+; GFX6-NEXT: s_sub_i32 s16, 64, s12
+; GFX6-NEXT: s_cmp_lt_u32 s12, 64
+; GFX6-NEXT: s_cselect_b32 s19, 1, 0
+; GFX6-NEXT: s_cmp_eq_u32 s12, 0
+; GFX6-NEXT: s_cselect_b32 s20, 1, 0
+; GFX6-NEXT: s_lshr_b64 s[12:13], s[10:11], s14
+; GFX6-NEXT: s_lshr_b64 s[14:15], s[4:5], s14
+; GFX6-NEXT: s_lshl_b64 s[16:17], s[10:11], s16
+; GFX6-NEXT: s_or_b64 s[14:15], s[14:15], s[16:17]
+; GFX6-NEXT: s_lshr_b64 s[10:11], s[10:11], s18
+; GFX6-NEXT: s_cmp_lg_u32 s19, 0
+; GFX6-NEXT: s_cselect_b64 s[10:11], s[14:15], s[10:11]
+; GFX6-NEXT: s_cmp_lg_u32 s20, 0
+; GFX6-NEXT: s_cselect_b64 s[4:5], s[4:5], s[10:11]
+; GFX6-NEXT: s_cmp_lg_u32 s19, 0
+; GFX6-NEXT: s_cselect_b64 s[10:11], s[12:13], 0
+; GFX6-NEXT: s_or_b64 s[4:5], s[8:9], s[4:5]
+; GFX6-NEXT: s_or_b64 s[6:7], s[6:7], s[10:11]
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_fshl_v2i128:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_and_b32 s18, s20, 64
-; GFX8-NEXT: s_mov_b32 s19, 0
-; GFX8-NEXT: s_cmp_lg_u64 s[18:19], 0
-; GFX8-NEXT: s_cselect_b32 s23, s15, s5
-; GFX8-NEXT: s_cselect_b32 s22, s14, s4
-; GFX8-NEXT: s_cselect_b32 s5, s5, s7
-; GFX8-NEXT: s_cselect_b32 s4, s4, s6
-; GFX8-NEXT: s_cselect_b32 s13, s13, s15
-; GFX8-NEXT: s_cselect_b32 s12, s12, s14
-; GFX8-NEXT: s_lshr_b64 s[6:7], s[22:23], 1
+; GFX8-NEXT: s_and_b32 s17, s16, 0x7f
+; GFX8-NEXT: s_sub_i32 s19, s17, 64
+; GFX8-NEXT: s_sub_i32 s21, 64, s17
+; GFX8-NEXT: s_cmp_lt_u32 s17, 64
+; GFX8-NEXT: s_cselect_b32 s28, 1, 0
+; GFX8-NEXT: s_cmp_eq_u32 s17, 0
+; GFX8-NEXT: s_cselect_b32 s17, 1, 0
+; GFX8-NEXT: s_lshr_b64 s[24:25], s[0:1], s21
+; GFX8-NEXT: s_lshl_b64 s[26:27], s[2:3], s16
+; GFX8-NEXT: s_lshl_b64 s[22:23], s[0:1], s16
+; GFX8-NEXT: s_or_b64 s[24:25], s[24:25], s[26:27]
+; GFX8-NEXT: s_lshl_b64 s[0:1], s[0:1], s19
+; GFX8-NEXT: s_cmp_lg_u32 s28, 0
+; GFX8-NEXT: s_cselect_b64 s[22:23], s[22:23], 0
+; GFX8-NEXT: s_cselect_b64 s[0:1], s[24:25], s[0:1]
+; GFX8-NEXT: s_cmp_lg_u32 s17, 0
+; GFX8-NEXT: s_mov_b32 s18, 0
+; GFX8-NEXT: s_cselect_b64 s[2:3], s[2:3], s[0:1]
+; GFX8-NEXT: s_lshr_b64 s[0:1], s[8:9], 1
+; GFX8-NEXT: s_lshl_b32 s19, s10, 31
+; GFX8-NEXT: s_lshr_b64 s[8:9], s[10:11], 1
+; GFX8-NEXT: s_andn2_b32 s10, 0x7f, s16
+; GFX8-NEXT: s_or_b64 s[0:1], s[0:1], s[18:19]
+; GFX8-NEXT: s_not_b32 s17, s16
+; GFX8-NEXT: s_sub_i32 s19, s10, 64
+; GFX8-NEXT: s_sub_i32 s21, 64, s10
+; GFX8-NEXT: s_cmp_lt_u32 s10, 64
+; GFX8-NEXT: s_cselect_b32 s26, 1, 0
+; GFX8-NEXT: s_cmp_eq_u32 s10, 0
+; GFX8-NEXT: s_cselect_b32 s27, 1, 0
+; GFX8-NEXT: s_lshr_b64 s[10:11], s[8:9], s17
+; GFX8-NEXT: s_lshr_b64 s[16:17], s[0:1], s17
+; GFX8-NEXT: s_lshl_b64 s[24:25], s[8:9], s21
+; GFX8-NEXT: s_or_b64 s[16:17], s[16:17], s[24:25]
+; GFX8-NEXT: s_lshr_b64 s[8:9], s[8:9], s19
+; GFX8-NEXT: s_cmp_lg_u32 s26, 0
+; GFX8-NEXT: s_cselect_b64 s[8:9], s[16:17], s[8:9]
+; GFX8-NEXT: s_cmp_lg_u32 s27, 0
+; GFX8-NEXT: s_cselect_b64 s[0:1], s[0:1], s[8:9]
+; GFX8-NEXT: s_cmp_lg_u32 s26, 0
+; GFX8-NEXT: s_cselect_b64 s[8:9], s[10:11], 0
+; GFX8-NEXT: s_or_b64 s[2:3], s[2:3], s[8:9]
+; GFX8-NEXT: s_and_b32 s8, s20, 0x7f
+; GFX8-NEXT: s_or_b64 s[0:1], s[22:23], s[0:1]
+; GFX8-NEXT: s_sub_i32 s19, s8, 64
+; GFX8-NEXT: s_sub_i32 s10, 64, s8
+; GFX8-NEXT: s_cmp_lt_u32 s8, 64
+; GFX8-NEXT: s_cselect_b32 s21, 1, 0
+; GFX8-NEXT: s_cmp_eq_u32 s8, 0
+; GFX8-NEXT: s_cselect_b32 s22, 1, 0
+; GFX8-NEXT: s_lshr_b64 s[10:11], s[4:5], s10
+; GFX8-NEXT: s_lshl_b64 s[16:17], s[6:7], s20
+; GFX8-NEXT: s_lshl_b64 s[8:9], s[4:5], s20
+; GFX8-NEXT: s_or_b64 s[10:11], s[10:11], s[16:17]
+; GFX8-NEXT: s_lshl_b64 s[4:5], s[4:5], s19
+; GFX8-NEXT: s_cmp_lg_u32 s21, 0
+; GFX8-NEXT: s_cselect_b64 s[8:9], s[8:9], 0
+; GFX8-NEXT: s_cselect_b64 s[4:5], s[10:11], s[4:5]
+; GFX8-NEXT: s_cmp_lg_u32 s22, 0
+; GFX8-NEXT: s_cselect_b64 s[6:7], s[6:7], s[4:5]
+; GFX8-NEXT: s_lshr_b64 s[4:5], s[12:13], 1
+; GFX8-NEXT: s_lshl_b32 s19, s14, 31
+; GFX8-NEXT: s_andn2_b32 s12, 0x7f, s20
+; GFX8-NEXT: s_or_b64 s[4:5], s[4:5], s[18:19]
+; GFX8-NEXT: s_lshr_b64 s[10:11], s[14:15], 1
; GFX8-NEXT: s_not_b32 s14, s20
-; GFX8-NEXT: s_lshl_b64 s[4:5], s[4:5], s20
-; GFX8-NEXT: s_lshr_b64 s[6:7], s[6:7], s14
-; GFX8-NEXT: s_lshr_b64 s[12:13], s[12:13], 1
-; GFX8-NEXT: s_or_b64 s[6:7], s[4:5], s[6:7]
-; GFX8-NEXT: s_lshl_b64 s[4:5], s[22:23], s20
-; GFX8-NEXT: s_lshr_b64 s[12:13], s[12:13], s14
-; GFX8-NEXT: s_or_b64 s[4:5], s[4:5], s[12:13]
-; GFX8-NEXT: s_and_b32 s18, s16, 64
-; GFX8-NEXT: s_cmp_lg_u64 s[18:19], 0
-; GFX8-NEXT: s_cselect_b32 s13, s11, s1
-; GFX8-NEXT: s_cselect_b32 s12, s10, s0
-; GFX8-NEXT: s_cselect_b32 s1, s1, s3
-; GFX8-NEXT: s_cselect_b32 s0, s0, s2
-; GFX8-NEXT: s_cselect_b32 s9, s9, s11
-; GFX8-NEXT: s_cselect_b32 s8, s8, s10
-; GFX8-NEXT: s_lshr_b64 s[2:3], s[12:13], 1
-; GFX8-NEXT: s_not_b32 s10, s16
-; GFX8-NEXT: s_lshl_b64 s[0:1], s[0:1], s16
-; GFX8-NEXT: s_lshr_b64 s[2:3], s[2:3], s10
-; GFX8-NEXT: s_lshr_b64 s[8:9], s[8:9], 1
-; GFX8-NEXT: s_or_b64 s[2:3], s[0:1], s[2:3]
-; GFX8-NEXT: s_lshl_b64 s[0:1], s[12:13], s16
-; GFX8-NEXT: s_lshr_b64 s[8:9], s[8:9], s10
-; GFX8-NEXT: s_or_b64 s[0:1], s[0:1], s[8:9]
+; GFX8-NEXT: s_sub_i32 s18, s12, 64
+; GFX8-NEXT: s_sub_i32 s16, 64, s12
+; GFX8-NEXT: s_cmp_lt_u32 s12, 64
+; GFX8-NEXT: s_cselect_b32 s19, 1, 0
+; GFX8-NEXT: s_cmp_eq_u32 s12, 0
+; GFX8-NEXT: s_cselect_b32 s20, 1, 0
+; GFX8-NEXT: s_lshr_b64 s[12:13], s[10:11], s14
+; GFX8-NEXT: s_lshr_b64 s[14:15], s[4:5], s14
+; GFX8-NEXT: s_lshl_b64 s[16:17], s[10:11], s16
+; GFX8-NEXT: s_or_b64 s[14:15], s[14:15], s[16:17]
+; GFX8-NEXT: s_lshr_b64 s[10:11], s[10:11], s18
+; GFX8-NEXT: s_cmp_lg_u32 s19, 0
+; GFX8-NEXT: s_cselect_b64 s[10:11], s[14:15], s[10:11]
+; GFX8-NEXT: s_cmp_lg_u32 s20, 0
+; GFX8-NEXT: s_cselect_b64 s[4:5], s[4:5], s[10:11]
+; GFX8-NEXT: s_cmp_lg_u32 s19, 0
+; GFX8-NEXT: s_cselect_b64 s[10:11], s[12:13], 0
+; GFX8-NEXT: s_or_b64 s[4:5], s[8:9], s[4:5]
+; GFX8-NEXT: s_or_b64 s[6:7], s[6:7], s[10:11]
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshl_v2i128:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s18, s20, 64
-; GFX11-NEXT: s_mov_b32 s19, 0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_cmp_lg_u64 s[18:19], 0
-; GFX11-NEXT: s_cselect_b32 s23, s15, s5
-; GFX11-NEXT: s_cselect_b32 s22, s14, s4
-; GFX11-NEXT: s_cselect_b32 s5, s5, s7
-; GFX11-NEXT: s_cselect_b32 s4, s4, s6
-; GFX11-NEXT: s_cselect_b32 s13, s13, s15
-; GFX11-NEXT: s_cselect_b32 s12, s12, s14
-; GFX11-NEXT: s_lshr_b64 s[6:7], s[22:23], 1
-; GFX11-NEXT: s_not_b32 s14, s20
-; GFX11-NEXT: s_lshl_b64 s[4:5], s[4:5], s20
-; GFX11-NEXT: s_lshr_b64 s[6:7], s[6:7], s14
-; GFX11-NEXT: s_and_b32 s18, s16, 64
-; GFX11-NEXT: s_or_b64 s[6:7], s[4:5], s[6:7]
-; GFX11-NEXT: s_lshr_b64 s[4:5], s[12:13], 1
-; GFX11-NEXT: s_lshl_b64 s[12:13], s[22:23], s20
-; GFX11-NEXT: s_lshr_b64 s[4:5], s[4:5], s14
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b64 s[4:5], s[12:13], s[4:5]
-; GFX11-NEXT: s_cmp_lg_u64 s[18:19], 0
-; GFX11-NEXT: s_cselect_b32 s13, s11, s1
-; GFX11-NEXT: s_cselect_b32 s12, s10, s0
-; GFX11-NEXT: s_cselect_b32 s1, s1, s3
-; GFX11-NEXT: s_cselect_b32 s0, s0, s2
-; GFX11-NEXT: s_cselect_b32 s3, s9, s11
-; GFX11-NEXT: s_cselect_b32 s2, s8, s10
-; GFX11-NEXT: s_lshl_b64 s[8:9], s[0:1], s16
-; GFX11-NEXT: s_lshr_b64 s[0:1], s[12:13], 1
-; GFX11-NEXT: s_not_b32 s14, s16
-; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], 1
-; GFX11-NEXT: s_lshl_b64 s[10:11], s[12:13], s16
-; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], s14
-; GFX11-NEXT: s_lshr_b64 s[12:13], s[0:1], s14
-; GFX11-NEXT: s_or_b64 s[0:1], s[10:11], s[2:3]
-; GFX11-NEXT: s_or_b64 s[2:3], s[8:9], s[12:13]
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshl_v2i128:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s17, s16, 0x7f
@@ -7373,6 +7952,7 @@ define amdgpu_ps <2 x i128> @s_fshl_v2i128(<2 x i128> inreg %lhs, <2 x i128> inr
; GFX9-NEXT: s_or_b64 s[4:5], s[8:9], s[4:5]
; GFX9-NEXT: s_or_b64 s[6:7], s[6:7], s[10:11]
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: s_fshl_v2i128:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s17, s16, 0x7f
@@ -7461,6 +8041,95 @@ define amdgpu_ps <2 x i128> @s_fshl_v2i128(<2 x i128> inreg %lhs, <2 x i128> inr
; GFX10-NEXT: s_or_b64 s[4:5], s[10:11], s[4:5]
; GFX10-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: s_fshl_v2i128:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s17, s16, 0x7f
+; GFX11-NEXT: s_mov_b32 s18, 0
+; GFX11-NEXT: s_sub_i32 s19, s17, 64
+; GFX11-NEXT: s_sub_i32 s21, 64, s17
+; GFX11-NEXT: s_cmp_lt_u32 s17, 64
+; GFX11-NEXT: s_cselect_b32 s28, 1, 0
+; GFX11-NEXT: s_cmp_eq_u32 s17, 0
+; GFX11-NEXT: s_cselect_b32 s17, 1, 0
+; GFX11-NEXT: s_lshr_b64 s[22:23], s[0:1], s21
+; GFX11-NEXT: s_lshl_b64 s[24:25], s[2:3], s16
+; GFX11-NEXT: s_lshl_b64 s[26:27], s[0:1], s16
+; GFX11-NEXT: s_or_b64 s[22:23], s[22:23], s[24:25]
+; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], s19
+; GFX11-NEXT: s_cmp_lg_u32 s28, 0
+; GFX11-NEXT: s_cselect_b64 s[24:25], s[26:27], 0
+; GFX11-NEXT: s_cselect_b64 s[0:1], s[22:23], s[0:1]
+; GFX11-NEXT: s_cmp_lg_u32 s17, 0
+; GFX11-NEXT: s_cselect_b64 s[2:3], s[2:3], s[0:1]
+; GFX11-NEXT: s_lshr_b64 s[0:1], s[8:9], 1
+; GFX11-NEXT: s_lshl_b32 s19, s10, 31
+; GFX11-NEXT: s_lshr_b64 s[8:9], s[10:11], 1
+; GFX11-NEXT: s_and_not1_b32 s10, 0x7f, s16
+; GFX11-NEXT: s_or_b64 s[0:1], s[0:1], s[18:19]
+; GFX11-NEXT: s_not_b32 s19, s16
+; GFX11-NEXT: s_sub_i32 s21, s10, 64
+; GFX11-NEXT: s_sub_i32 s16, 64, s10
+; GFX11-NEXT: s_cmp_lt_u32 s10, 64
+; GFX11-NEXT: s_cselect_b32 s26, 1, 0
+; GFX11-NEXT: s_cmp_eq_u32 s10, 0
+; GFX11-NEXT: s_cselect_b32 s27, 1, 0
+; GFX11-NEXT: s_lshr_b64 s[10:11], s[0:1], s19
+; GFX11-NEXT: s_lshl_b64 s[16:17], s[8:9], s16
+; GFX11-NEXT: s_lshr_b64 s[22:23], s[8:9], s19
+; GFX11-NEXT: s_or_b64 s[10:11], s[10:11], s[16:17]
+; GFX11-NEXT: s_lshr_b64 s[8:9], s[8:9], s21
+; GFX11-NEXT: s_cmp_lg_u32 s26, 0
+; GFX11-NEXT: s_cselect_b64 s[8:9], s[10:11], s[8:9]
+; GFX11-NEXT: s_cmp_lg_u32 s27, 0
+; GFX11-NEXT: s_cselect_b64 s[0:1], s[0:1], s[8:9]
+; GFX11-NEXT: s_cmp_lg_u32 s26, 0
+; GFX11-NEXT: s_cselect_b64 s[8:9], s[22:23], 0
+; GFX11-NEXT: s_and_b32 s10, s20, 0x7f
+; GFX11-NEXT: s_or_b64 s[0:1], s[24:25], s[0:1]
+; GFX11-NEXT: s_or_b64 s[2:3], s[2:3], s[8:9]
+; GFX11-NEXT: s_sub_i32 s19, s10, 64
+; GFX11-NEXT: s_sub_i32 s8, 64, s10
+; GFX11-NEXT: s_cmp_lt_u32 s10, 64
+; GFX11-NEXT: s_cselect_b32 s21, 1, 0
+; GFX11-NEXT: s_cmp_eq_u32 s10, 0
+; GFX11-NEXT: s_cselect_b32 s22, 1, 0
+; GFX11-NEXT: s_lshr_b64 s[8:9], s[4:5], s8
+; GFX11-NEXT: s_lshl_b64 s[10:11], s[6:7], s20
+; GFX11-NEXT: s_lshl_b64 s[16:17], s[4:5], s20
+; GFX11-NEXT: s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[4:5], s19
+; GFX11-NEXT: s_cmp_lg_u32 s21, 0
+; GFX11-NEXT: s_cselect_b64 s[10:11], s[16:17], 0
+; GFX11-NEXT: s_cselect_b64 s[4:5], s[8:9], s[4:5]
+; GFX11-NEXT: s_cmp_lg_u32 s22, 0
+; GFX11-NEXT: s_cselect_b64 s[6:7], s[6:7], s[4:5]
+; GFX11-NEXT: s_lshr_b64 s[4:5], s[12:13], 1
+; GFX11-NEXT: s_lshl_b32 s19, s14, 31
+; GFX11-NEXT: s_and_not1_b32 s12, 0x7f, s20
+; GFX11-NEXT: s_or_b64 s[4:5], s[4:5], s[18:19]
+; GFX11-NEXT: s_lshr_b64 s[8:9], s[14:15], 1
+; GFX11-NEXT: s_not_b32 s16, s20
+; GFX11-NEXT: s_sub_i32 s18, s12, 64
+; GFX11-NEXT: s_sub_i32 s14, 64, s12
+; GFX11-NEXT: s_cmp_lt_u32 s12, 64
+; GFX11-NEXT: s_cselect_b32 s19, 1, 0
+; GFX11-NEXT: s_cmp_eq_u32 s12, 0
+; GFX11-NEXT: s_cselect_b32 s20, 1, 0
+; GFX11-NEXT: s_lshr_b64 s[12:13], s[4:5], s16
+; GFX11-NEXT: s_lshl_b64 s[14:15], s[8:9], s14
+; GFX11-NEXT: s_lshr_b64 s[16:17], s[8:9], s16
+; GFX11-NEXT: s_or_b64 s[12:13], s[12:13], s[14:15]
+; GFX11-NEXT: s_lshr_b64 s[8:9], s[8:9], s18
+; GFX11-NEXT: s_cmp_lg_u32 s19, 0
+; GFX11-NEXT: s_cselect_b64 s[8:9], s[12:13], s[8:9]
+; GFX11-NEXT: s_cmp_lg_u32 s20, 0
+; GFX11-NEXT: s_cselect_b64 s[4:5], s[4:5], s[8:9]
+; GFX11-NEXT: s_cmp_lg_u32 s19, 0
+; GFX11-NEXT: s_cselect_b64 s[8:9], s[16:17], 0
+; GFX11-NEXT: s_or_b64 s[4:5], s[10:11], s[4:5]
+; GFX11-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
+; GFX11-NEXT: ; return to shader part epilog
%result = call <2 x i128> @llvm.fshl.v2i128(<2 x i128> %lhs, <2 x i128> %rhs, <2 x i128> %amt)
ret <2 x i128> %result
}
@@ -7648,96 +8317,6 @@ define <2 x i128> @v_fshl_v2i128(<2 x i128> %lhs, <2 x i128> %rhs, <2 x i128> %a
; GFX8-NEXT: v_or_b32_e32 v7, v7, v11
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshl_v2i128:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_and_b32_e32 v19, 0x7f, v16
-; GFX11-NEXT: v_lshrrev_b64 v[8:9], 1, v[8:9]
-; GFX11-NEXT: v_bfi_b32 v29, v16, 0, 0x7f
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_lshlrev_b64 v[17:18], v19, v[0:1]
-; GFX11-NEXT: v_cmp_gt_u32_e32 vcc_lo, 64, v19
-; GFX11-NEXT: v_lshl_or_b32 v9, v10, 31, v9
-; GFX11-NEXT: v_lshrrev_b64 v[10:11], 1, v[10:11]
-; GFX11-NEXT: v_sub_nc_u32_e32 v16, 64, v29
-; GFX11-NEXT: v_cmp_gt_u32_e64 s1, 64, v29
-; GFX11-NEXT: v_cndmask_b32_e32 v30, 0, v17, vcc_lo
-; GFX11-NEXT: v_sub_nc_u32_e32 v21, 64, v19
-; GFX11-NEXT: v_dual_cndmask_b32 v18, 0, v18 :: v_dual_add_nc_u32 v25, 0xffffffc0, v19
-; GFX11-NEXT: v_lshlrev_b64 v[23:24], v19, v[2:3]
-; GFX11-NEXT: v_lshlrev_b64 v[27:28], v16, v[10:11]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_lshrrev_b64 v[21:22], v21, v[0:1]
-; GFX11-NEXT: v_lshlrev_b64 v[0:1], v25, v[0:1]
-; GFX11-NEXT: v_lshrrev_b64 v[25:26], v29, v[8:9]
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 0, v19
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v21, v21, v23
-; GFX11-NEXT: v_add_nc_u32_e32 v23, 0xffffffc0, v29
-; GFX11-NEXT: v_or_b32_e32 v19, v25, v27
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_cndmask_b32_e32 v21, v0, v21, vcc_lo
-; GFX11-NEXT: v_or_b32_e32 v0, v22, v24
-; GFX11-NEXT: v_lshrrev_b64 v[16:17], v23, v[10:11]
-; GFX11-NEXT: v_or_b32_e32 v22, v26, v28
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_cndmask_b32_e64 v21, v21, v2, s0
-; GFX11-NEXT: v_cndmask_b32_e32 v23, v1, v0, vcc_lo
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v29
-; GFX11-NEXT: v_cndmask_b32_e64 v16, v16, v19, s1
-; GFX11-NEXT: v_lshrrev_b64 v[0:1], v29, v[10:11]
-; GFX11-NEXT: v_cndmask_b32_e64 v10, v17, v22, s1
-; GFX11-NEXT: v_cndmask_b32_e64 v22, v23, v3, s0
-; GFX11-NEXT: v_and_b32_e32 v23, 0x7f, v20
-; GFX11-NEXT: v_cndmask_b32_e32 v2, v16, v8, vcc_lo
-; GFX11-NEXT: v_bfi_b32 v20, v20, 0, 0x7f
-; GFX11-NEXT: v_cndmask_b32_e32 v3, v10, v9, vcc_lo
-; GFX11-NEXT: v_lshrrev_b64 v[8:9], 1, v[12:13]
-; GFX11-NEXT: v_sub_nc_u32_e32 v10, 64, v23
-; GFX11-NEXT: v_cndmask_b32_e64 v24, 0, v0, s1
-; GFX11-NEXT: v_cndmask_b32_e64 v25, 0, v1, s1
-; GFX11-NEXT: v_or_b32_e32 v0, v30, v2
-; GFX11-NEXT: v_add_nc_u32_e32 v16, 0xffffffc0, v23
-; GFX11-NEXT: v_lshl_or_b32 v9, v14, 31, v9
-; GFX11-NEXT: v_lshrrev_b64 v[14:15], 1, v[14:15]
-; GFX11-NEXT: v_add_nc_u32_e32 v26, 0xffffffc0, v20
-; GFX11-NEXT: v_or_b32_e32 v1, v18, v3
-; GFX11-NEXT: v_lshrrev_b64 v[2:3], v10, v[4:5]
-; GFX11-NEXT: v_lshlrev_b64 v[10:11], v23, v[6:7]
-; GFX11-NEXT: v_sub_nc_u32_e32 v18, 64, v20
-; GFX11-NEXT: v_lshlrev_b64 v[12:13], v23, v[4:5]
-; GFX11-NEXT: v_lshlrev_b64 v[4:5], v16, v[4:5]
-; GFX11-NEXT: v_cmp_gt_u32_e32 vcc_lo, 64, v23
-; GFX11-NEXT: v_lshrrev_b64 v[16:17], v20, v[8:9]
-; GFX11-NEXT: v_or_b32_e32 v10, v2, v10
-; GFX11-NEXT: v_lshlrev_b64 v[18:19], v18, v[14:15]
-; GFX11-NEXT: v_or_b32_e32 v2, v21, v24
-; GFX11-NEXT: v_or_b32_e32 v11, v3, v11
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT: v_dual_cndmask_b32 v12, 0, v12 :: v_dual_cndmask_b32 v21, v4, v10
-; GFX11-NEXT: v_lshrrev_b64 v[3:4], v26, v[14:15]
-; GFX11-NEXT: v_or_b32_e32 v10, v16, v18
-; GFX11-NEXT: v_cmp_gt_u32_e64 s1, 64, v20
-; GFX11-NEXT: v_or_b32_e32 v16, v17, v19
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v5, v11, vcc_lo
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 0, v23
-; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 0, v20
-; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, v10, s1
-; GFX11-NEXT: v_lshrrev_b64 v[10:11], v20, v[14:15]
-; GFX11-NEXT: v_cndmask_b32_e64 v4, v4, v16, s1
-; GFX11-NEXT: v_cndmask_b32_e32 v13, 0, v13, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e64 v6, v21, v6, s0
-; GFX11-NEXT: v_cndmask_b32_e64 v7, v5, v7, s0
-; GFX11-NEXT: v_cndmask_b32_e64 v5, v3, v8, s2
-; GFX11-NEXT: v_cndmask_b32_e64 v8, v4, v9, s2
-; GFX11-NEXT: v_cndmask_b32_e64 v9, 0, v10, s1
-; GFX11-NEXT: v_cndmask_b32_e64 v10, 0, v11, s1
-; GFX11-NEXT: v_or_b32_e32 v3, v22, v25
-; GFX11-NEXT: v_or_b32_e32 v4, v12, v5
-; GFX11-NEXT: v_or_b32_e32 v5, v13, v8
-; GFX11-NEXT: v_or_b32_e32 v6, v6, v9
-; GFX11-NEXT: v_or_b32_e32 v7, v7, v10
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshl_v2i128:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -7825,6 +8404,7 @@ define <2 x i128> @v_fshl_v2i128(<2 x i128> %lhs, <2 x i128> %rhs, <2 x i128> %a
; GFX9-NEXT: v_or_b32_e32 v6, v17, v6
; GFX9-NEXT: v_or_b32_e32 v7, v12, v7
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshl_v2i128:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -7911,6 +8491,97 @@ define <2 x i128> @v_fshl_v2i128(<2 x i128> %lhs, <2 x i128> %rhs, <2 x i128> %a
; GFX10-NEXT: v_or_b32_e32 v6, v6, v9
; GFX10-NEXT: v_or_b32_e32 v7, v7, v10
; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_fshl_v2i128:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_and_b32_e32 v19, 0x7f, v16
+; GFX11-NEXT: v_lshrrev_b64 v[8:9], 1, v[8:9]
+; GFX11-NEXT: v_bfi_b32 v29, v16, 0, 0x7f
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_lshlrev_b64 v[17:18], v19, v[0:1]
+; GFX11-NEXT: v_cmp_gt_u32_e32 vcc_lo, 64, v19
+; GFX11-NEXT: v_lshl_or_b32 v9, v10, 31, v9
+; GFX11-NEXT: v_lshrrev_b64 v[10:11], 1, v[10:11]
+; GFX11-NEXT: v_sub_nc_u32_e32 v16, 64, v29
+; GFX11-NEXT: v_cmp_gt_u32_e64 s1, 64, v29
+; GFX11-NEXT: v_cndmask_b32_e32 v30, 0, v17, vcc_lo
+; GFX11-NEXT: v_sub_nc_u32_e32 v21, 64, v19
+; GFX11-NEXT: v_dual_cndmask_b32 v18, 0, v18 :: v_dual_add_nc_u32 v25, 0xffffffc0, v19
+; GFX11-NEXT: v_lshlrev_b64 v[23:24], v19, v[2:3]
+; GFX11-NEXT: v_lshlrev_b64 v[27:28], v16, v[10:11]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_lshrrev_b64 v[21:22], v21, v[0:1]
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], v25, v[0:1]
+; GFX11-NEXT: v_lshrrev_b64 v[25:26], v29, v[8:9]
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 0, v19
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v21, v21, v23
+; GFX11-NEXT: v_add_nc_u32_e32 v23, 0xffffffc0, v29
+; GFX11-NEXT: v_or_b32_e32 v19, v25, v27
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_cndmask_b32_e32 v21, v0, v21, vcc_lo
+; GFX11-NEXT: v_or_b32_e32 v0, v22, v24
+; GFX11-NEXT: v_lshrrev_b64 v[16:17], v23, v[10:11]
+; GFX11-NEXT: v_or_b32_e32 v22, v26, v28
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_cndmask_b32_e64 v21, v21, v2, s0
+; GFX11-NEXT: v_cndmask_b32_e32 v23, v1, v0, vcc_lo
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v29
+; GFX11-NEXT: v_cndmask_b32_e64 v16, v16, v19, s1
+; GFX11-NEXT: v_lshrrev_b64 v[0:1], v29, v[10:11]
+; GFX11-NEXT: v_cndmask_b32_e64 v10, v17, v22, s1
+; GFX11-NEXT: v_cndmask_b32_e64 v22, v23, v3, s0
+; GFX11-NEXT: v_and_b32_e32 v23, 0x7f, v20
+; GFX11-NEXT: v_cndmask_b32_e32 v2, v16, v8, vcc_lo
+; GFX11-NEXT: v_bfi_b32 v20, v20, 0, 0x7f
+; GFX11-NEXT: v_cndmask_b32_e32 v3, v10, v9, vcc_lo
+; GFX11-NEXT: v_lshrrev_b64 v[8:9], 1, v[12:13]
+; GFX11-NEXT: v_sub_nc_u32_e32 v10, 64, v23
+; GFX11-NEXT: v_cndmask_b32_e64 v24, 0, v0, s1
+; GFX11-NEXT: v_cndmask_b32_e64 v25, 0, v1, s1
+; GFX11-NEXT: v_or_b32_e32 v0, v30, v2
+; GFX11-NEXT: v_add_nc_u32_e32 v16, 0xffffffc0, v23
+; GFX11-NEXT: v_lshl_or_b32 v9, v14, 31, v9
+; GFX11-NEXT: v_lshrrev_b64 v[14:15], 1, v[14:15]
+; GFX11-NEXT: v_add_nc_u32_e32 v26, 0xffffffc0, v20
+; GFX11-NEXT: v_or_b32_e32 v1, v18, v3
+; GFX11-NEXT: v_lshrrev_b64 v[2:3], v10, v[4:5]
+; GFX11-NEXT: v_lshlrev_b64 v[10:11], v23, v[6:7]
+; GFX11-NEXT: v_sub_nc_u32_e32 v18, 64, v20
+; GFX11-NEXT: v_lshlrev_b64 v[12:13], v23, v[4:5]
+; GFX11-NEXT: v_lshlrev_b64 v[4:5], v16, v[4:5]
+; GFX11-NEXT: v_cmp_gt_u32_e32 vcc_lo, 64, v23
+; GFX11-NEXT: v_lshrrev_b64 v[16:17], v20, v[8:9]
+; GFX11-NEXT: v_or_b32_e32 v10, v2, v10
+; GFX11-NEXT: v_lshlrev_b64 v[18:19], v18, v[14:15]
+; GFX11-NEXT: v_or_b32_e32 v2, v21, v24
+; GFX11-NEXT: v_or_b32_e32 v11, v3, v11
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT: v_dual_cndmask_b32 v12, 0, v12 :: v_dual_cndmask_b32 v21, v4, v10
+; GFX11-NEXT: v_lshrrev_b64 v[3:4], v26, v[14:15]
+; GFX11-NEXT: v_or_b32_e32 v10, v16, v18
+; GFX11-NEXT: v_cmp_gt_u32_e64 s1, 64, v20
+; GFX11-NEXT: v_or_b32_e32 v16, v17, v19
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v5, v11, vcc_lo
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 0, v23
+; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 0, v20
+; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, v10, s1
+; GFX11-NEXT: v_lshrrev_b64 v[10:11], v20, v[14:15]
+; GFX11-NEXT: v_cndmask_b32_e64 v4, v4, v16, s1
+; GFX11-NEXT: v_cndmask_b32_e32 v13, 0, v13, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e64 v6, v21, v6, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v7, v5, v7, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v5, v3, v8, s2
+; GFX11-NEXT: v_cndmask_b32_e64 v8, v4, v9, s2
+; GFX11-NEXT: v_cndmask_b32_e64 v9, 0, v10, s1
+; GFX11-NEXT: v_cndmask_b32_e64 v10, 0, v11, s1
+; GFX11-NEXT: v_or_b32_e32 v3, v22, v25
+; GFX11-NEXT: v_or_b32_e32 v4, v12, v5
+; GFX11-NEXT: v_or_b32_e32 v5, v13, v8
+; GFX11-NEXT: v_or_b32_e32 v6, v6, v9
+; GFX11-NEXT: v_or_b32_e32 v7, v7, v10
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call <2 x i128> @llvm.fshl.v2i128(<2 x i128> %lhs, <2 x i128> %rhs, <2 x i128> %amt)
ret <2 x i128> %result
}
@@ -7947,5 +8618,3 @@ declare i128 @llvm.fshl.i128(i128, i128, i128) #0
declare <2 x i128> @llvm.fshl.v2i128(<2 x i128>, <2 x i128>, <2 x i128>) #0
attributes #0 = { nounwind readnone speculatable willreturn }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX11-TRUE16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
index f48a6fe809160..82dd5756972ca 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
@@ -1,9 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu6.00-amd-amdpal -o - %s | FileCheck -check-prefixes=GCN,GFX6 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-amd-amdpal -o - %s | FileCheck -check-prefixes=GCN,GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgpu6.00-amd-amdpal -o - %s | FileCheck -check-prefixes=GCN,GFX6 %s
+; RUN: llc -global-isel -mtriple=amdgpu8.03-amd-amdpal -o - %s | FileCheck -check-prefixes=GCN,GFX8 %s
; xUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00-amd-amdpal -o - %s | FileCheck -check-prefixes=GCN,GFX9 %s
; xUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-amd-amdpal -o - %s | FileCheck -check-prefixes=GCN,GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=+real-true16 -o - %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal -mattr=+real-true16 -o - %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16 %s
; xUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s
define amdgpu_ps i7 @s_fshr_i7(i7 inreg %lhs, i7 inreg %rhs, i7 inreg %amt) {
@@ -80,47 +80,6 @@ define amdgpu_ps i7 @s_fshr_i7(i7 inreg %lhs, i7 inreg %rhs, i7 inreg %amt) {
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshr_i7:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, 7
-; GFX11-NEXT: s_and_b32 s2, s2, 0x7f
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_rcp_iflag_f32_e32 v0, v0
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
-; GFX11-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_readfirstlane_b32 s3, v0
-; GFX11-NEXT: s_mul_i32 s4, s3, -7
-; GFX11-NEXT: s_mul_hi_u32 s4, s3, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_i32 s3, s3, s4
-; GFX11-NEXT: s_mul_hi_u32 s3, s2, s3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_mul_i32 s3, s3, -7
-; GFX11-NEXT: s_add_i32 s2, s2, s3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_cmp_ge_u32 s2, 7
-; GFX11-NEXT: s_cselect_b32 s3, 1, 0
-; GFX11-NEXT: s_add_i32 s4, s2, -7
-; GFX11-NEXT: s_cmp_lg_u32 s3, 0
-; GFX11-NEXT: s_cselect_b32 s2, s4, s2
-; GFX11-NEXT: s_cmp_ge_u32 s2, 7
-; GFX11-NEXT: s_cselect_b32 s3, 1, 0
-; GFX11-NEXT: s_add_i32 s4, s2, -7
-; GFX11-NEXT: s_cmp_lg_u32 s3, 0
-; GFX11-NEXT: s_cselect_b32 s2, s4, s2
-; GFX11-NEXT: s_and_b32 s1, s1, 0x7f
-; GFX11-NEXT: s_sub_i32 s3, 6, s2
-; GFX11-NEXT: s_lshl_b32 s0, s0, 1
-; GFX11-NEXT: s_and_b32 s2, s2, 0x7f
-; GFX11-NEXT: s_and_b32 s3, s3, 0x7f
-; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX11-NEXT: s_lshl_b32 s0, s0, s3
-; GFX11-NEXT: s_lshr_b32 s1, s1, s2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshr_i7:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v0, 7
@@ -155,6 +114,7 @@ define amdgpu_ps i7 @s_fshr_i7(i7 inreg %lhs, i7 inreg %rhs, i7 inreg %amt) {
; GFX9-NEXT: s_lshr_b32 s1, s1, s2
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: s_fshr_i7:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, 7
@@ -189,6 +149,48 @@ define amdgpu_ps i7 @s_fshr_i7(i7 inreg %lhs, i7 inreg %rhs, i7 inreg %amt) {
; GFX10-NEXT: s_lshr_b32 s1, s1, s2
; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: s_fshr_i7:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, 7
+; GFX11-NEXT: s_and_b32 s2, s2, 0x7f
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX11-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_readfirstlane_b32 s3, v0
+; GFX11-NEXT: s_mul_i32 s4, s3, -7
+; GFX11-NEXT: s_mul_hi_u32 s4, s3, s4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_add_i32 s3, s3, s4
+; GFX11-NEXT: s_mul_hi_u32 s3, s2, s3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_mul_i32 s3, s3, -7
+; GFX11-NEXT: s_add_i32 s2, s2, s3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_cmp_ge_u32 s2, 7
+; GFX11-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-NEXT: s_add_i32 s4, s2, -7
+; GFX11-NEXT: s_cmp_lg_u32 s3, 0
+; GFX11-NEXT: s_cselect_b32 s2, s4, s2
+; GFX11-NEXT: s_cmp_ge_u32 s2, 7
+; GFX11-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-NEXT: s_add_i32 s4, s2, -7
+; GFX11-NEXT: s_cmp_lg_u32 s3, 0
+; GFX11-NEXT: s_cselect_b32 s2, s4, s2
+; GFX11-NEXT: s_and_b32 s1, s1, 0x7f
+; GFX11-NEXT: s_sub_i32 s3, 6, s2
+; GFX11-NEXT: s_lshl_b32 s0, s0, 1
+; GFX11-NEXT: s_and_b32 s2, s2, 0x7f
+; GFX11-NEXT: s_and_b32 s3, s3, 0x7f
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_lshl_b32 s0, s0, s3
+; GFX11-NEXT: s_lshr_b32 s1, s1, s2
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: ; return to shader part epilog
%result = call i7 @llvm.fshr.i7(i7 %lhs, i7 %rhs, i7 %amt)
ret i7 %result
}
@@ -258,44 +260,6 @@ define i7 @v_fshr_i7(i7 %lhs, i7 %rhs, i7 %amt) {
; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshr_i7:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v3, 7
-; GFX11-NEXT: v_and_b32_e32 v2, 0x7f, v2
-; GFX11-NEXT: v_lshlrev_b16 v0.l, 1, v0.l
-; GFX11-NEXT: v_and_b16 v1.l, 0x7f, v1.l
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_rcp_iflag_f32_e32 v3, v3
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_mul_f32_e32 v3, 0x4f7ffffe, v3
-; GFX11-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_readfirstlane_b32 s0, v3
-; GFX11-NEXT: s_mul_i32 s1, s0, -7
-; GFX11-NEXT: s_mul_hi_u32 s1, s0, s1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_i32 s0, s0, s1
-; GFX11-NEXT: v_mul_hi_u32 v5, v2, s0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_mad_u64_u32 v[3:4], null, v5, -7, v[2:3]
-; GFX11-NEXT: v_add_nc_u32_e32 v2, -7, v3
-; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 7, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc_lo
-; GFX11-NEXT: v_add_nc_u32_e32 v3, -7, v2
-; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 7, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc_lo
-; GFX11-NEXT: v_sub_nc_u16 v0.h, 6, v2.l
-; GFX11-NEXT: v_and_b16 v1.h, 0x7f, v2.l
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b16 v0.h, 0x7f, v0.h
-; GFX11-NEXT: v_lshlrev_b16 v0.l, v0.h, v0.l
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b16 v0.h, v1.h, v1.l
-; GFX11-NEXT: v_or_b16 v0.l, v0.l, v0.h
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshr_i7:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -325,6 +289,7 @@ define i7 @v_fshr_i7(i7 %lhs, i7 %rhs, i7 %amt) {
; GFX9-NEXT: v_lshrrev_b16_e32 v1, v2, v1
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshr_i7:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -354,6 +319,46 @@ define i7 @v_fshr_i7(i7 %lhs, i7 %rhs, i7 %amt) {
; GFX10-NEXT: v_lshlrev_b16 v0, v3, v0
; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_fshr_i7:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_cvt_f32_ubyte0_e32 v3, 7
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0x7f, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 1, v0.l
+; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0x7f, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_rcp_iflag_f32_e32 v3, v3
+; GFX11-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-TRUE16-NEXT: v_mul_f32_e32 v3, 0x4f7ffffe, v3
+; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v3
+; GFX11-TRUE16-NEXT: s_mul_i32 s1, s0, -7
+; GFX11-TRUE16-NEXT: s_mul_hi_u32 s1, s0, s1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_add_i32 s0, s0, s1
+; GFX11-TRUE16-NEXT: v_mul_hi_u32 v5, v2, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mad_u64_u32 v[3:4], null, v5, -7, v[2:3]
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, -7, v3
+; GFX11-TRUE16-NEXT: v_cmp_le_u32_e32 vcc_lo, 7, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, -7, v2
+; GFX11-TRUE16-NEXT: v_cmp_le_u32_e32 vcc_lo, 7, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_sub_nc_u16 v0.h, 6, v2.l
+; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0x7f, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0x7f, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, v0.h, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, v1.h, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-FAKE16-LABEL: v_fshr_i7:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -420,18 +425,6 @@ define amdgpu_ps i8 @s_fshr_i8(i8 inreg %lhs, i8 inreg %rhs, i8 inreg %amt) {
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshr_i8:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s1, s1, 0xff
-; GFX11-NEXT: s_lshl_b32 s0, s0, 1
-; GFX11-NEXT: s_and_not1_b32 s3, 7, s2
-; GFX11-NEXT: s_and_b32 s2, s2, 7
-; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX11-NEXT: s_lshl_b32 s0, s0, s3
-; GFX11-NEXT: s_lshr_b32 s1, s1, s2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshr_i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s1, s1, 0xff
@@ -443,6 +436,7 @@ define amdgpu_ps i8 @s_fshr_i8(i8 inreg %lhs, i8 inreg %rhs, i8 inreg %amt) {
; GFX9-NEXT: s_lshr_b32 s1, s1, s2
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: s_fshr_i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s1, s1, 0xff
@@ -454,6 +448,19 @@ define amdgpu_ps i8 @s_fshr_i8(i8 inreg %lhs, i8 inreg %rhs, i8 inreg %amt) {
; GFX10-NEXT: s_lshr_b32 s1, s1, s2
; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: s_fshr_i8:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-NEXT: s_lshl_b32 s0, s0, 1
+; GFX11-NEXT: s_and_not1_b32 s3, 7, s2
+; GFX11-NEXT: s_and_b32 s2, s2, 7
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_lshl_b32 s0, s0, s3
+; GFX11-NEXT: s_lshr_b32 s1, s1, s2
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: ; return to shader part epilog
%result = call i8 @llvm.fshr.i8(i8 %lhs, i8 %rhs, i8 %amt)
ret i8 %result
}
@@ -483,20 +490,6 @@ define i8 @v_fshr_i8(i8 %lhs, i8 %rhs, i8 %amt) {
; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshr_i8:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_xor_b16 v0.h, v2.l, -1
-; GFX11-NEXT: v_lshlrev_b16 v0.l, 1, v0.l
-; GFX11-NEXT: v_and_b16 v1.h, v2.l, 7
-; GFX11-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b16 v0.h, v0.h, 7
-; GFX11-NEXT: v_lshlrev_b16 v0.l, v0.h, v0.l
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b16 v0.h, v1.h, v1.l
-; GFX11-NEXT: v_or_b16 v0.l, v0.l, v0.h
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshr_i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -508,6 +501,7 @@ define i8 @v_fshr_i8(i8 %lhs, i8 %rhs, i8 %amt) {
; GFX9-NEXT: v_lshrrev_b16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshr_i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -520,6 +514,22 @@ define i8 @v_fshr_i8(i8 %lhs, i8 %rhs, i8 %amt) {
; GFX10-NEXT: v_lshlrev_b16 v0, v3, v0
; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_fshr_i8:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_xor_b16 v0.h, v2.l, -1
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 1, v0.l
+; GFX11-TRUE16-NEXT: v_and_b16 v1.h, v2.l, 7
+; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v0.h, 7
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, v0.h, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, v1.h, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-FAKE16-LABEL: v_fshr_i8:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -555,15 +565,6 @@ define amdgpu_ps i8 @s_fshr_i8_4(i8 inreg %lhs, i8 inreg %rhs) {
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshr_i8_4:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s1, s1, 0xff
-; GFX11-NEXT: s_lshl_b32 s0, s0, 4
-; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshr_b32 s1, s1, 4
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshr_i8_4:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s1, s1, 0xff
@@ -572,6 +573,7 @@ define amdgpu_ps i8 @s_fshr_i8_4(i8 inreg %lhs, i8 inreg %rhs) {
; GFX9-NEXT: s_lshr_b32 s1, s1, 4
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: s_fshr_i8_4:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s1, s1, 0xff
@@ -580,6 +582,16 @@ define amdgpu_ps i8 @s_fshr_i8_4(i8 inreg %lhs, i8 inreg %rhs) {
; GFX10-NEXT: s_lshr_b32 s1, s1, 4
; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: s_fshr_i8_4:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-NEXT: s_lshl_b32 s0, s0, 4
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_lshr_b32 s1, s1, 4
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: ; return to shader part epilog
%result = call i8 @llvm.fshr.i8(i8 %lhs, i8 %rhs, i8 4)
ret i8 %result
}
@@ -602,15 +614,6 @@ define i8 @v_fshr_i8_4(i8 %lhs, i8 %rhs) {
; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshr_i8_4:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_and_b16 v0.h, 0xff, v1.l
-; GFX11-NEXT: v_lshlrev_b16 v0.l, 4, v0.l
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b16 v0.h, 4, v0.h
-; GFX11-NEXT: v_or_b16 v0.l, v0.l, v0.h
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshr_i8_4:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -619,6 +622,7 @@ define i8 @v_fshr_i8_4(i8 %lhs, i8 %rhs) {
; GFX9-NEXT: v_lshrrev_b16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshr_i8_4:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -627,6 +631,17 @@ define i8 @v_fshr_i8_4(i8 %lhs, i8 %rhs) {
; GFX10-NEXT: v_lshrrev_b16 v1, 4, v1
; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_fshr_i8_4:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 4, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, 4, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-FAKE16-LABEL: v_fshr_i8_4:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -657,15 +672,6 @@ define amdgpu_ps i8 @s_fshr_i8_5(i8 inreg %lhs, i8 inreg %rhs) {
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshr_i8_5:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s1, s1, 0xff
-; GFX11-NEXT: s_lshl_b32 s0, s0, 3
-; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshr_b32 s1, s1, 5
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshr_i8_5:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s1, s1, 0xff
@@ -674,6 +680,7 @@ define amdgpu_ps i8 @s_fshr_i8_5(i8 inreg %lhs, i8 inreg %rhs) {
; GFX9-NEXT: s_lshr_b32 s1, s1, 5
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: s_fshr_i8_5:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s1, s1, 0xff
@@ -682,6 +689,16 @@ define amdgpu_ps i8 @s_fshr_i8_5(i8 inreg %lhs, i8 inreg %rhs) {
; GFX10-NEXT: s_lshr_b32 s1, s1, 5
; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: s_fshr_i8_5:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-NEXT: s_lshl_b32 s0, s0, 3
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_lshr_b32 s1, s1, 5
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: ; return to shader part epilog
%result = call i8 @llvm.fshr.i8(i8 %lhs, i8 %rhs, i8 5)
ret i8 %result
}
@@ -704,15 +721,6 @@ define i8 @v_fshr_i8_5(i8 %lhs, i8 %rhs) {
; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshr_i8_5:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_and_b16 v0.h, 0xff, v1.l
-; GFX11-NEXT: v_lshlrev_b16 v0.l, 3, v0.l
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b16 v0.h, 5, v0.h
-; GFX11-NEXT: v_or_b16 v0.l, v0.l, v0.h
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshr_i8_5:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -721,6 +729,7 @@ define i8 @v_fshr_i8_5(i8 %lhs, i8 %rhs) {
; GFX9-NEXT: v_lshrrev_b16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshr_i8_5:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -729,6 +738,17 @@ define i8 @v_fshr_i8_5(i8 %lhs, i8 %rhs) {
; GFX10-NEXT: v_lshrrev_b16 v1, 5, v1
; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_fshr_i8_5:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 3, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, 5, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-FAKE16-LABEL: v_fshr_i8_5:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -794,33 +814,6 @@ define amdgpu_ps i16 @s_fshr_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg, i16 in
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshr_v2i8:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_lshr_b32 s4, s1, 8
-; GFX11-NEXT: s_lshr_b32 s3, s0, 8
-; GFX11-NEXT: s_lshl_b32 s0, s0, 1
-; GFX11-NEXT: s_and_not1_b32 s5, 7, s2
-; GFX11-NEXT: s_lshr_b32 s6, s2, 8
-; GFX11-NEXT: s_and_b32 s4, s4, 0xff
-; GFX11-NEXT: s_lshl_b32 s0, s0, s5
-; GFX11-NEXT: s_and_b32 s1, s1, 0xff
-; GFX11-NEXT: s_lshl_b32 s3, s3, 1
-; GFX11-NEXT: s_and_not1_b32 s5, 7, s6
-; GFX11-NEXT: s_and_b32 s6, s6, 7
-; GFX11-NEXT: s_and_b32 s4, 0xffff, s4
-; GFX11-NEXT: s_and_b32 s2, s2, 7
-; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX11-NEXT: s_lshl_b32 s3, s3, s5
-; GFX11-NEXT: s_lshr_b32 s4, s4, s6
-; GFX11-NEXT: s_lshr_b32 s1, s1, s2
-; GFX11-NEXT: s_or_b32 s2, s3, s4
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: s_and_b32 s1, s2, 0xff
-; GFX11-NEXT: s_and_b32 s0, s0, 0xff
-; GFX11-NEXT: s_lshl_b32 s1, s1, 8
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshr_v2i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_lshr_b32 s4, s1, 8
@@ -847,6 +840,7 @@ define amdgpu_ps i16 @s_fshr_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg, i16 in
; GFX9-NEXT: s_lshl_b32 s1, s1, 8
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: s_fshr_v2i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_lshr_b32 s4, s1, 8
@@ -873,6 +867,34 @@ define amdgpu_ps i16 @s_fshr_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg, i16 in
; GFX10-NEXT: s_lshl_b32 s1, s1, 8
; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: s_fshr_v2i8:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_lshr_b32 s4, s1, 8
+; GFX11-NEXT: s_lshr_b32 s3, s0, 8
+; GFX11-NEXT: s_lshl_b32 s0, s0, 1
+; GFX11-NEXT: s_and_not1_b32 s5, 7, s2
+; GFX11-NEXT: s_lshr_b32 s6, s2, 8
+; GFX11-NEXT: s_and_b32 s4, s4, 0xff
+; GFX11-NEXT: s_lshl_b32 s0, s0, s5
+; GFX11-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-NEXT: s_lshl_b32 s3, s3, 1
+; GFX11-NEXT: s_and_not1_b32 s5, 7, s6
+; GFX11-NEXT: s_and_b32 s6, s6, 7
+; GFX11-NEXT: s_and_b32 s4, 0xffff, s4
+; GFX11-NEXT: s_and_b32 s2, s2, 7
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_lshl_b32 s3, s3, s5
+; GFX11-NEXT: s_lshr_b32 s4, s4, s6
+; GFX11-NEXT: s_lshr_b32 s1, s1, s2
+; GFX11-NEXT: s_or_b32 s2, s3, s4
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: s_and_b32 s1, s2, 0xff
+; GFX11-NEXT: s_and_b32 s0, s0, 0xff
+; GFX11-NEXT: s_lshl_b32 s1, s1, 8
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: ; return to shader part epilog
%lhs = bitcast i16 %lhs.arg to <2 x i8>
%rhs = bitcast i16 %rhs.arg to <2 x i8>
%amt = bitcast i16 %amt.arg to <2 x i8>
@@ -932,37 +954,6 @@ define i16 @v_fshr_v2i8(i16 %lhs.arg, i16 %rhs.arg, i16 %amt.arg) {
; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshr_v2i8:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, 8, v2
-; GFX11-NEXT: v_lshrrev_b32_e32 v4, 8, v0
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 8, v1
-; GFX11-NEXT: v_xor_b16 v1.h, v2.l, -1
-; GFX11-NEXT: v_lshlrev_b16 v0.l, 1, v0.l
-; GFX11-NEXT: v_xor_b16 v0.h, v3.l, -1
-; GFX11-NEXT: v_lshlrev_b16 v2.h, 1, v4.l
-; GFX11-NEXT: v_and_b16 v3.l, v3.l, 7
-; GFX11-NEXT: v_and_b16 v3.h, 0xff, v5.l
-; GFX11-NEXT: v_and_b16 v2.l, v2.l, 7
-; GFX11-NEXT: v_and_b16 v0.h, v0.h, 7
-; GFX11-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-NEXT: v_and_b16 v1.h, v1.h, 7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_lshlrev_b16 v0.h, v0.h, v2.h
-; GFX11-NEXT: v_lshrrev_b16 v2.h, v3.l, v3.h
-; GFX11-NEXT: v_lshlrev_b16 v0.l, v1.h, v0.l
-; GFX11-NEXT: v_lshrrev_b16 v1.l, v2.l, v1.l
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_or_b16 v0.h, v0.h, v2.h
-; GFX11-NEXT: v_or_b16 v0.l, v0.l, v1.l
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
-; GFX11-NEXT: v_or_b16 v0.l, v0.l, v0.h
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshr_v2i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -987,6 +978,7 @@ define i16 @v_fshr_v2i8(i16 %lhs.arg, i16 %rhs.arg, i16 %amt.arg) {
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
; GFX9-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshr_v2i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1013,6 +1005,39 @@ define i16 @v_fshr_v2i8(i16 %lhs.arg, i16 %rhs.arg, i16 %amt.arg) {
; GFX10-NEXT: v_and_b32_sdwa v1, v2, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX10-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_fshr_v2i8:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 8, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v1
+; GFX11-TRUE16-NEXT: v_xor_b16 v1.h, v2.l, -1
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 1, v0.l
+; GFX11-TRUE16-NEXT: v_xor_b16 v0.h, v3.l, -1
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 1, v4.l
+; GFX11-TRUE16-NEXT: v_and_b16 v3.l, v3.l, 7
+; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v5.l
+; GFX11-TRUE16-NEXT: v_and_b16 v2.l, v2.l, 7
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v0.h, 7
+; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
+; GFX11-TRUE16-NEXT: v_and_b16 v1.h, v1.h, 7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, v0.h, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v2.h, v3.l, v3.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, v1.h, v0.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v1.l, v2.l, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v0.h, v2.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
+; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-FAKE16-LABEL: v_fshr_v2i8:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1155,59 +1180,6 @@ define amdgpu_ps i32 @s_fshr_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg, i32 in
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshr_v4i8:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_lshr_b32 s6, s1, 8
-; GFX11-NEXT: s_lshr_b32 s7, s1, 16
-; GFX11-NEXT: s_lshr_b32 s8, s1, 24
-; GFX11-NEXT: s_and_b32 s1, s1, 0xff
-; GFX11-NEXT: s_lshr_b32 s3, s0, 8
-; GFX11-NEXT: s_lshr_b32 s9, s2, 8
-; GFX11-NEXT: s_lshr_b32 s10, s2, 16
-; GFX11-NEXT: s_lshr_b32 s11, s2, 24
-; GFX11-NEXT: s_and_not1_b32 s12, 7, s2
-; GFX11-NEXT: s_and_b32 s2, s2, 7
-; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX11-NEXT: s_and_b32 s6, s6, 0xff
-; GFX11-NEXT: s_lshr_b32 s4, s0, 16
-; GFX11-NEXT: s_lshr_b32 s5, s0, 24
-; GFX11-NEXT: s_lshl_b32 s0, s0, 1
-; GFX11-NEXT: s_lshr_b32 s1, s1, s2
-; GFX11-NEXT: s_lshl_b32 s2, s3, 1
-; GFX11-NEXT: s_and_not1_b32 s3, 7, s9
-; GFX11-NEXT: s_and_b32 s9, s9, 7
-; GFX11-NEXT: s_and_b32 s6, 0xffff, s6
-; GFX11-NEXT: s_lshl_b32 s0, s0, s12
-; GFX11-NEXT: s_lshl_b32 s2, s2, s3
-; GFX11-NEXT: s_lshr_b32 s3, s6, s9
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: s_or_b32 s1, s2, s3
-; GFX11-NEXT: s_lshl_b32 s2, s4, 1
-; GFX11-NEXT: s_and_b32 s4, s7, 0xff
-; GFX11-NEXT: s_and_not1_b32 s3, 7, s10
-; GFX11-NEXT: s_and_b32 s6, s10, 7
-; GFX11-NEXT: s_and_b32 s4, 0xffff, s4
-; GFX11-NEXT: s_lshl_b32 s2, s2, s3
-; GFX11-NEXT: s_lshr_b32 s3, s4, s6
-; GFX11-NEXT: s_lshl_b32 s4, s5, 1
-; GFX11-NEXT: s_and_not1_b32 s5, 7, s11
-; GFX11-NEXT: s_and_b32 s6, s11, 7
-; GFX11-NEXT: s_lshl_b32 s4, s4, s5
-; GFX11-NEXT: s_lshr_b32 s5, s8, s6
-; GFX11-NEXT: s_or_b32 s2, s2, s3
-; GFX11-NEXT: s_and_b32 s1, s1, 0xff
-; GFX11-NEXT: s_or_b32 s3, s4, s5
-; GFX11-NEXT: s_and_b32 s0, s0, 0xff
-; GFX11-NEXT: s_lshl_b32 s1, s1, 8
-; GFX11-NEXT: s_and_b32 s2, s2, 0xff
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: s_lshl_b32 s1, s2, 16
-; GFX11-NEXT: s_and_b32 s2, s3, 0xff
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: s_lshl_b32 s1, s2, 24
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshr_v4i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_lshr_b32 s6, s1, 8
@@ -1260,6 +1232,7 @@ define amdgpu_ps i32 @s_fshr_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg, i32 in
; GFX9-NEXT: s_lshl_b32 s1, s1, 24
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: s_fshr_v4i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_lshr_b32 s6, s1, 8
@@ -1312,6 +1285,60 @@ define amdgpu_ps i32 @s_fshr_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg, i32 in
; GFX10-NEXT: s_lshl_b32 s1, s2, 24
; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: s_fshr_v4i8:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_lshr_b32 s6, s1, 8
+; GFX11-NEXT: s_lshr_b32 s7, s1, 16
+; GFX11-NEXT: s_lshr_b32 s8, s1, 24
+; GFX11-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-NEXT: s_lshr_b32 s3, s0, 8
+; GFX11-NEXT: s_lshr_b32 s9, s2, 8
+; GFX11-NEXT: s_lshr_b32 s10, s2, 16
+; GFX11-NEXT: s_lshr_b32 s11, s2, 24
+; GFX11-NEXT: s_and_not1_b32 s12, 7, s2
+; GFX11-NEXT: s_and_b32 s2, s2, 7
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_and_b32 s6, s6, 0xff
+; GFX11-NEXT: s_lshr_b32 s4, s0, 16
+; GFX11-NEXT: s_lshr_b32 s5, s0, 24
+; GFX11-NEXT: s_lshl_b32 s0, s0, 1
+; GFX11-NEXT: s_lshr_b32 s1, s1, s2
+; GFX11-NEXT: s_lshl_b32 s2, s3, 1
+; GFX11-NEXT: s_and_not1_b32 s3, 7, s9
+; GFX11-NEXT: s_and_b32 s9, s9, 7
+; GFX11-NEXT: s_and_b32 s6, 0xffff, s6
+; GFX11-NEXT: s_lshl_b32 s0, s0, s12
+; GFX11-NEXT: s_lshl_b32 s2, s2, s3
+; GFX11-NEXT: s_lshr_b32 s3, s6, s9
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: s_or_b32 s1, s2, s3
+; GFX11-NEXT: s_lshl_b32 s2, s4, 1
+; GFX11-NEXT: s_and_b32 s4, s7, 0xff
+; GFX11-NEXT: s_and_not1_b32 s3, 7, s10
+; GFX11-NEXT: s_and_b32 s6, s10, 7
+; GFX11-NEXT: s_and_b32 s4, 0xffff, s4
+; GFX11-NEXT: s_lshl_b32 s2, s2, s3
+; GFX11-NEXT: s_lshr_b32 s3, s4, s6
+; GFX11-NEXT: s_lshl_b32 s4, s5, 1
+; GFX11-NEXT: s_and_not1_b32 s5, 7, s11
+; GFX11-NEXT: s_and_b32 s6, s11, 7
+; GFX11-NEXT: s_lshl_b32 s4, s4, s5
+; GFX11-NEXT: s_lshr_b32 s5, s8, s6
+; GFX11-NEXT: s_or_b32 s2, s2, s3
+; GFX11-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-NEXT: s_or_b32 s3, s4, s5
+; GFX11-NEXT: s_and_b32 s0, s0, 0xff
+; GFX11-NEXT: s_lshl_b32 s1, s1, 8
+; GFX11-NEXT: s_and_b32 s2, s2, 0xff
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: s_lshl_b32 s1, s2, 16
+; GFX11-NEXT: s_and_b32 s2, s3, 0xff
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: s_lshl_b32 s1, s2, 24
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: ; return to shader part epilog
%lhs = bitcast i32 %lhs.arg to <4 x i8>
%rhs = bitcast i32 %rhs.arg to <4 x i8>
%amt = bitcast i32 %amt.arg to <4 x i8>
@@ -1420,62 +1447,6 @@ define i32 @v_fshr_v4i8(i32 %lhs.arg, i32 %rhs.arg, i32 %amt.arg) {
; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshr_v4i8:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v7, 8, v2
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, 8, v0
-; GFX11-NEXT: v_lshrrev_b32_e32 v4, 8, v1
-; GFX11-NEXT: v_lshrrev_b32_e32 v8, 16, v1
-; GFX11-NEXT: v_lshrrev_b32_e32 v9, 24, v1
-; GFX11-NEXT: v_xor_b16 v1.h, v7.l, -1
-; GFX11-NEXT: v_lshrrev_b32_e32 v10, 16, v2
-; GFX11-NEXT: v_lshrrev_b32_e32 v11, 24, v2
-; GFX11-NEXT: v_lshlrev_b16 v2.h, 1, v3.l
-; GFX11-NEXT: v_and_b16 v3.l, v7.l, 7
-; GFX11-NEXT: v_and_b16 v1.h, v1.h, 7
-; GFX11-NEXT: v_and_b16 v3.h, 0xff, v4.l
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX11-NEXT: v_lshrrev_b32_e32 v6, 24, v0
-; GFX11-NEXT: v_xor_b16 v4.l, v10.l, -1
-; GFX11-NEXT: v_lshlrev_b16 v1.h, v1.h, v2.h
-; GFX11-NEXT: v_lshrrev_b16 v2.h, v3.l, v3.h
-; GFX11-NEXT: v_xor_b16 v3.h, v11.l, -1
-; GFX11-NEXT: v_xor_b16 v0.h, v2.l, -1
-; GFX11-NEXT: v_lshlrev_b16 v3.l, 1, v5.l
-; GFX11-NEXT: v_and_b16 v4.l, v4.l, 7
-; GFX11-NEXT: v_and_b16 v4.h, v10.l, 7
-; GFX11-NEXT: v_and_b16 v5.l, 0xff, v8.l
-; GFX11-NEXT: v_lshlrev_b16 v5.h, 1, v6.l
-; GFX11-NEXT: v_and_b16 v3.h, v3.h, 7
-; GFX11-NEXT: v_and_b16 v6.l, v11.l, 7
-; GFX11-NEXT: v_lshlrev_b16 v0.l, 1, v0.l
-; GFX11-NEXT: v_and_b16 v0.h, v0.h, 7
-; GFX11-NEXT: v_and_b16 v2.l, v2.l, 7
-; GFX11-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-NEXT: v_or_b16 v7.l, v1.h, v2.h
-; GFX11-NEXT: v_lshlrev_b16 v1.h, v4.l, v3.l
-; GFX11-NEXT: v_lshrrev_b16 v2.h, v4.h, v5.l
-; GFX11-NEXT: v_lshlrev_b16 v3.l, v3.h, v5.h
-; GFX11-NEXT: v_lshrrev_b16 v3.h, v6.l, v9.l
-; GFX11-NEXT: v_lshlrev_b16 v0.l, v0.h, v0.l
-; GFX11-NEXT: v_lshrrev_b16 v0.h, v2.l, v1.l
-; GFX11-NEXT: v_and_b32_e32 v4, 0xff, v7
-; GFX11-NEXT: v_or_b16 v1.l, v1.h, v2.h
-; GFX11-NEXT: v_or_b16 v2.l, v3.l, v3.h
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_or_b16 v0.l, v0.l, v0.h
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 8, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX11-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_and_or_b32 v0, 0xff, v0, v3
-; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 24, v2
-; GFX11-NEXT: v_or3_b32 v0, v0, v1, v2
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshr_v4i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1524,6 +1495,7 @@ define i32 @v_fshr_v4i8(i32 %lhs.arg, i32 %rhs.arg, i32 %amt.arg) {
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 24, v0
; GFX9-NEXT: v_or3_b32 v0, v1, v2, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshr_v4i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1576,6 +1548,64 @@ define i32 @v_fshr_v4i8(i32 %lhs.arg, i32 %rhs.arg, i32 %amt.arg) {
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 24, v2
; GFX10-NEXT: v_or3_b32 v0, v0, v1, v2
; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_fshr_v4i8:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 8, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 8, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 24, v1
+; GFX11-TRUE16-NEXT: v_xor_b16 v1.h, v7.l, -1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 24, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 1, v3.l
+; GFX11-TRUE16-NEXT: v_and_b16 v3.l, v7.l, 7
+; GFX11-TRUE16-NEXT: v_and_b16 v1.h, v1.h, 7
+; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v4.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 24, v0
+; GFX11-TRUE16-NEXT: v_xor_b16 v4.l, v10.l, -1
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, v1.h, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v2.h, v3.l, v3.h
+; GFX11-TRUE16-NEXT: v_xor_b16 v3.h, v11.l, -1
+; GFX11-TRUE16-NEXT: v_xor_b16 v0.h, v2.l, -1
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 1, v5.l
+; GFX11-TRUE16-NEXT: v_and_b16 v4.l, v4.l, 7
+; GFX11-TRUE16-NEXT: v_and_b16 v4.h, v10.l, 7
+; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v8.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 1, v6.l
+; GFX11-TRUE16-NEXT: v_and_b16 v3.h, v3.h, 7
+; GFX11-TRUE16-NEXT: v_and_b16 v6.l, v11.l, 7
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 1, v0.l
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v0.h, 7
+; GFX11-TRUE16-NEXT: v_and_b16 v2.l, v2.l, 7
+; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v1.h, v2.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, v4.l, v3.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v2.h, v4.h, v5.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, v3.h, v5.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v3.h, v6.l, v9.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, v2.l, v1.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xff, v7
+; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v2.h
+; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v3.l, v3.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 8, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v0, 0xff, v0, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 24, v2
+; GFX11-TRUE16-NEXT: v_or3_b32 v0, v0, v1, v2
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-FAKE16-LABEL: v_fshr_v4i8:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1641,79 +1671,76 @@ define i32 @v_fshr_v4i8(i32 %lhs.arg, i32 %rhs.arg, i32 %amt.arg) {
}
define amdgpu_ps i24 @s_fshr_i24(i24 inreg %lhs, i24 inreg %rhs, i24 inreg %amt) {
-; GCN-LABEL: s_fshr_i24:
-; GCN: ; %bb.0:
-; GCN-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
-; GCN-NEXT: v_rcp_iflag_f32_e32 v0, v0
-; GCN-NEXT: s_and_b32 s2, s2, 0xffffff
-; GCN-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
-; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GCN-NEXT: v_readfirstlane_b32 s3, v0
-; GCN-NEXT: s_mul_i32 s4, s3, 0xffffffe8
-; GCN-NEXT: v_mul_hi_u32 v0, v0, s4
-; GCN-NEXT: v_readfirstlane_b32 s4, v0
-; GCN-NEXT: s_add_i32 s3, s3, s4
-; GCN-NEXT: v_mov_b32_e32 v0, s3
-; GCN-NEXT: v_mul_hi_u32 v0, s2, v0
-; GCN-NEXT: v_readfirstlane_b32 s3, v0
-; GCN-NEXT: s_mulk_i32 s3, 0xffe8
-; GCN-NEXT: s_add_i32 s2, s2, s3
-; GCN-NEXT: s_cmp_ge_u32 s2, 24
-; GCN-NEXT: s_cselect_b32 s3, 1, 0
-; GCN-NEXT: s_sub_i32 s4, s2, 24
-; GCN-NEXT: s_cmp_lg_u32 s3, 0
-; GCN-NEXT: s_cselect_b32 s2, s4, s2
-; GCN-NEXT: s_cmp_ge_u32 s2, 24
-; GCN-NEXT: s_cselect_b32 s3, 1, 0
-; GCN-NEXT: s_sub_i32 s4, s2, 24
-; GCN-NEXT: s_cmp_lg_u32 s3, 0
-; GCN-NEXT: s_cselect_b32 s2, s4, s2
-; GCN-NEXT: s_lshl_b32 s0, s0, 1
-; GCN-NEXT: s_and_b32 s1, s1, 0xffffff
-; GCN-NEXT: s_sub_i32 s3, 23, s2
-; GCN-NEXT: s_lshl_b32 s0, s0, s3
-; GCN-NEXT: s_lshr_b32 s1, s1, s2
-; GCN-NEXT: s_or_b32 s0, s0, s1
-; GCN-NEXT: ; return to shader part epilog
+; GFX6-LABEL: s_fshr_i24:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
+; GFX6-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; GFX6-NEXT: s_and_b32 s2, s2, 0xffffff
+; GFX6-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX6-NEXT: v_readfirstlane_b32 s3, v0
+; GFX6-NEXT: s_mul_i32 s4, s3, 0xffffffe8
+; GFX6-NEXT: v_mul_hi_u32 v0, v0, s4
+; GFX6-NEXT: v_readfirstlane_b32 s4, v0
+; GFX6-NEXT: s_add_i32 s3, s3, s4
+; GFX6-NEXT: v_mov_b32_e32 v0, s3
+; GFX6-NEXT: v_mul_hi_u32 v0, s2, v0
+; GFX6-NEXT: v_readfirstlane_b32 s3, v0
+; GFX6-NEXT: s_mulk_i32 s3, 0xffe8
+; GFX6-NEXT: s_add_i32 s2, s2, s3
+; GFX6-NEXT: s_cmp_ge_u32 s2, 24
+; GFX6-NEXT: s_cselect_b32 s3, 1, 0
+; GFX6-NEXT: s_sub_i32 s4, s2, 24
+; GFX6-NEXT: s_cmp_lg_u32 s3, 0
+; GFX6-NEXT: s_cselect_b32 s2, s4, s2
+; GFX6-NEXT: s_cmp_ge_u32 s2, 24
+; GFX6-NEXT: s_cselect_b32 s3, 1, 0
+; GFX6-NEXT: s_sub_i32 s4, s2, 24
+; GFX6-NEXT: s_cmp_lg_u32 s3, 0
+; GFX6-NEXT: s_cselect_b32 s2, s4, s2
+; GFX6-NEXT: s_lshl_b32 s0, s0, 1
+; GFX6-NEXT: s_and_b32 s1, s1, 0xffffff
+; GFX6-NEXT: s_sub_i32 s3, 23, s2
+; GFX6-NEXT: s_lshl_b32 s0, s0, s3
+; GFX6-NEXT: s_lshr_b32 s1, s1, s2
+; GFX6-NEXT: s_or_b32 s0, s0, s1
+; GFX6-NEXT: ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshr_i24:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
+; GFX8-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; GFX8-NEXT: s_and_b32 s2, s2, 0xffffff
+; GFX8-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX8-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX8-NEXT: v_readfirstlane_b32 s3, v0
+; GFX8-NEXT: s_mul_i32 s4, s3, 0xffffffe8
+; GFX8-NEXT: v_mul_hi_u32 v0, v0, s4
+; GFX8-NEXT: v_readfirstlane_b32 s4, v0
+; GFX8-NEXT: s_add_i32 s3, s3, s4
+; GFX8-NEXT: v_mov_b32_e32 v0, s3
+; GFX8-NEXT: v_mul_hi_u32 v0, s2, v0
+; GFX8-NEXT: v_readfirstlane_b32 s3, v0
+; GFX8-NEXT: s_mulk_i32 s3, 0xffe8
+; GFX8-NEXT: s_add_i32 s2, s2, s3
+; GFX8-NEXT: s_cmp_ge_u32 s2, 24
+; GFX8-NEXT: s_cselect_b32 s3, 1, 0
+; GFX8-NEXT: s_sub_i32 s4, s2, 24
+; GFX8-NEXT: s_cmp_lg_u32 s3, 0
+; GFX8-NEXT: s_cselect_b32 s2, s4, s2
+; GFX8-NEXT: s_cmp_ge_u32 s2, 24
+; GFX8-NEXT: s_cselect_b32 s3, 1, 0
+; GFX8-NEXT: s_sub_i32 s4, s2, 24
+; GFX8-NEXT: s_cmp_lg_u32 s3, 0
+; GFX8-NEXT: s_cselect_b32 s2, s4, s2
+; GFX8-NEXT: s_lshl_b32 s0, s0, 1
+; GFX8-NEXT: s_and_b32 s1, s1, 0xffffff
+; GFX8-NEXT: s_sub_i32 s3, 23, s2
+; GFX8-NEXT: s_lshl_b32 s0, s0, s3
+; GFX8-NEXT: s_lshr_b32 s1, s1, s2
+; GFX8-NEXT: s_or_b32 s0, s0, s1
+; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshr_i24:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
-; GFX11-NEXT: s_and_b32 s2, s2, 0xffffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_rcp_iflag_f32_e32 v0, v0
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
-; GFX11-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_readfirstlane_b32 s3, v0
-; GFX11-NEXT: s_mul_i32 s4, s3, 0xffffffe8
-; GFX11-NEXT: s_mul_hi_u32 s4, s3, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_i32 s3, s3, s4
-; GFX11-NEXT: s_mul_hi_u32 s3, s2, s3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_mulk_i32 s3, 0xffe8
-; GFX11-NEXT: s_add_i32 s2, s2, s3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_cmp_ge_u32 s2, 24
-; GFX11-NEXT: s_cselect_b32 s3, 1, 0
-; GFX11-NEXT: s_sub_i32 s4, s2, 24
-; GFX11-NEXT: s_cmp_lg_u32 s3, 0
-; GFX11-NEXT: s_cselect_b32 s2, s4, s2
-; GFX11-NEXT: s_cmp_ge_u32 s2, 24
-; GFX11-NEXT: s_cselect_b32 s3, 1, 0
-; GFX11-NEXT: s_sub_i32 s4, s2, 24
-; GFX11-NEXT: s_cmp_lg_u32 s3, 0
-; GFX11-NEXT: s_cselect_b32 s2, s4, s2
-; GFX11-NEXT: s_lshl_b32 s0, s0, 1
-; GFX11-NEXT: s_sub_i32 s3, 23, s2
-; GFX11-NEXT: s_and_b32 s1, s1, 0xffffff
-; GFX11-NEXT: s_lshl_b32 s0, s0, s3
-; GFX11-NEXT: s_lshr_b32 s1, s1, s2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshr_i24:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
@@ -1745,6 +1772,7 @@ define amdgpu_ps i24 @s_fshr_i24(i24 inreg %lhs, i24 inreg %rhs, i24 inreg %amt)
; GFX9-NEXT: s_lshr_b32 s1, s1, s2
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: s_fshr_i24:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
@@ -1776,6 +1804,45 @@ define amdgpu_ps i24 @s_fshr_i24(i24 inreg %lhs, i24 inreg %rhs, i24 inreg %amt)
; GFX10-NEXT: s_lshr_b32 s1, s1, s2
; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: s_fshr_i24:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
+; GFX11-NEXT: s_and_b32 s2, s2, 0xffffff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX11-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_readfirstlane_b32 s3, v0
+; GFX11-NEXT: s_mul_i32 s4, s3, 0xffffffe8
+; GFX11-NEXT: s_mul_hi_u32 s4, s3, s4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_add_i32 s3, s3, s4
+; GFX11-NEXT: s_mul_hi_u32 s3, s2, s3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_mulk_i32 s3, 0xffe8
+; GFX11-NEXT: s_add_i32 s2, s2, s3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_cmp_ge_u32 s2, 24
+; GFX11-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-NEXT: s_sub_i32 s4, s2, 24
+; GFX11-NEXT: s_cmp_lg_u32 s3, 0
+; GFX11-NEXT: s_cselect_b32 s2, s4, s2
+; GFX11-NEXT: s_cmp_ge_u32 s2, 24
+; GFX11-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-NEXT: s_sub_i32 s4, s2, 24
+; GFX11-NEXT: s_cmp_lg_u32 s3, 0
+; GFX11-NEXT: s_cselect_b32 s2, s4, s2
+; GFX11-NEXT: s_lshl_b32 s0, s0, 1
+; GFX11-NEXT: s_sub_i32 s3, 23, s2
+; GFX11-NEXT: s_and_b32 s1, s1, 0xffffff
+; GFX11-NEXT: s_lshl_b32 s0, s0, s3
+; GFX11-NEXT: s_lshr_b32 s1, s1, s2
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: ; return to shader part epilog
%result = call i24 @llvm.fshr.i24(i24 %lhs, i24 %rhs, i24 %amt)
ret i24 %result
}
@@ -1847,43 +1914,6 @@ define i24 @v_fshr_i24(i24 %lhs, i24 %rhs, i24 %amt) {
; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshr_i24:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v3, 24
-; GFX11-NEXT: v_and_b32_e32 v2, 0xffffff, v2
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 1, v0
-; GFX11-NEXT: v_and_b32_e32 v1, 0xffffff, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_rcp_iflag_f32_e32 v3, v3
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_mul_f32_e32 v3, 0x4f7ffffe, v3
-; GFX11-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_readfirstlane_b32 s0, v3
-; GFX11-NEXT: s_mul_i32 s1, s0, 0xffffffe8
-; GFX11-NEXT: s_mul_hi_u32 s1, s0, s1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_i32 s0, s0, s1
-; GFX11-NEXT: v_mul_hi_u32 v5, v2, s0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_mad_u64_u32 v[3:4], null, 0xffffffe8, v5, v[2:3]
-; GFX11-NEXT: v_add_nc_u32_e32 v2, 0xffffffe8, v3
-; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc_lo
-; GFX11-NEXT: v_add_nc_u32_e32 v3, 0xffffffe8, v2
-; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc_lo
-; GFX11-NEXT: v_sub_nc_u32_e32 v3, 23, v2
-; GFX11-NEXT: v_and_b32_e32 v2, 0xffffff, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_and_b32_e32 v3, 0xffffff, v3
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, v2, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_lshl_or_b32 v0, v0, v3, v1
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshr_i24:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1913,6 +1943,7 @@ define i24 @v_fshr_i24(i24 %lhs, i24 %rhs, i24 %amt) {
; GFX9-NEXT: v_lshrrev_b32_e32 v1, v2, v1
; GFX9-NEXT: v_lshl_or_b32 v0, v0, v3, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshr_i24:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1941,6 +1972,44 @@ define i24 @v_fshr_i24(i24 %lhs, i24 %rhs, i24 %amt) {
; GFX10-NEXT: v_lshrrev_b32_e32 v1, v2, v1
; GFX10-NEXT: v_lshl_or_b32 v0, v0, v3, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_fshr_i24:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v3, 24
+; GFX11-NEXT: v_and_b32_e32 v2, 0xffffff, v2
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 1, v0
+; GFX11-NEXT: v_and_b32_e32 v1, 0xffffff, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_rcp_iflag_f32_e32 v3, v3
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-NEXT: v_mul_f32_e32 v3, 0x4f7ffffe, v3
+; GFX11-NEXT: v_cvt_u32_f32_e32 v3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_readfirstlane_b32 s0, v3
+; GFX11-NEXT: s_mul_i32 s1, s0, 0xffffffe8
+; GFX11-NEXT: s_mul_hi_u32 s1, s0, s1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_add_i32 s0, s0, s1
+; GFX11-NEXT: v_mul_hi_u32 v5, v2, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_mad_u64_u32 v[3:4], null, 0xffffffe8, v5, v[2:3]
+; GFX11-NEXT: v_add_nc_u32_e32 v2, 0xffffffe8, v3
+; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc_lo
+; GFX11-NEXT: v_add_nc_u32_e32 v3, 0xffffffe8, v2
+; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc_lo
+; GFX11-NEXT: v_sub_nc_u32_e32 v3, 23, v2
+; GFX11-NEXT: v_and_b32_e32 v2, 0xffffff, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_and_b32_e32 v3, 0xffffff, v3
+; GFX11-NEXT: v_lshrrev_b32_e32 v1, v2, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshl_or_b32 v0, v0, v3, v1
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call i24 @llvm.fshr.i24(i24 %lhs, i24 %rhs, i24 %amt)
ret i24 %result
}
@@ -2200,114 +2269,6 @@ define amdgpu_ps i48 @s_fshr_v2i24(i48 inreg %lhs.arg, i48 inreg %rhs.arg, i48 i
; GFX8-NEXT: s_or_b32 s1, s2, s1
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshr_v2i24:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
-; GFX11-NEXT: s_lshr_b32 s9, s1, 8
-; GFX11-NEXT: s_and_b32 s1, s1, 0xff
-; GFX11-NEXT: s_lshr_b32 s6, s0, 8
-; GFX11-NEXT: s_lshr_b32 s8, s0, 24
-; GFX11-NEXT: v_rcp_iflag_f32_e32 v0, v0
-; GFX11-NEXT: s_lshl_b32 s1, s1, 8
-; GFX11-NEXT: s_lshr_b32 s11, s3, 8
-; GFX11-NEXT: s_and_b32 s3, s3, 0xff
-; GFX11-NEXT: s_and_b32 s6, s6, 0xff
-; GFX11-NEXT: s_or_b32 s1, s8, s1
-; GFX11-NEXT: s_lshr_b32 s8, s2, 8
-; GFX11-NEXT: s_lshr_b32 s10, s2, 24
-; GFX11-NEXT: s_lshl_b32 s3, s3, 8
-; GFX11-NEXT: s_lshr_b32 s7, s0, 16
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
-; GFX11-NEXT: s_and_b32 s0, s0, 0xff
-; GFX11-NEXT: s_lshl_b32 s6, s6, 8
-; GFX11-NEXT: s_and_b32 s8, s8, 0xff
-; GFX11-NEXT: s_or_b32 s3, s10, s3
-; GFX11-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX11-NEXT: s_lshr_b32 s10, s4, 8
-; GFX11-NEXT: s_or_b32 s0, s0, s6
-; GFX11-NEXT: s_and_b32 s6, s7, 0xff
-; GFX11-NEXT: s_and_b32 s7, s9, 0xff
-; GFX11-NEXT: v_readfirstlane_b32 s13, v0
-; GFX11-NEXT: s_lshr_b32 s9, s2, 16
-; GFX11-NEXT: s_and_b32 s2, s2, 0xff
-; GFX11-NEXT: s_lshl_b32 s8, s8, 8
-; GFX11-NEXT: s_and_b32 s10, s10, 0xff
-; GFX11-NEXT: s_or_b32 s2, s2, s8
-; GFX11-NEXT: s_and_b32 s8, s9, 0xff
-; GFX11-NEXT: s_and_b32 s9, s11, 0xff
-; GFX11-NEXT: s_lshr_b32 s11, s4, 16
-; GFX11-NEXT: s_lshr_b32 s12, s4, 24
-; GFX11-NEXT: s_and_b32 s4, s4, 0xff
-; GFX11-NEXT: s_lshl_b32 s10, s10, 8
-; GFX11-NEXT: s_and_b32 s11, s11, 0xff
-; GFX11-NEXT: s_or_b32 s4, s4, s10
-; GFX11-NEXT: s_mul_i32 s10, s13, 0xffffffe8
-; GFX11-NEXT: s_pack_ll_b32_b16 s4, s4, s11
-; GFX11-NEXT: s_mul_hi_u32 s10, s13, s10
-; GFX11-NEXT: s_lshr_b32 s14, s5, 8
-; GFX11-NEXT: s_add_i32 s13, s13, s10
-; GFX11-NEXT: s_and_b32 s5, s5, 0xff
-; GFX11-NEXT: s_mul_hi_u32 s10, s4, s13
-; GFX11-NEXT: s_lshl_b32 s5, s5, 8
-; GFX11-NEXT: s_mulk_i32 s10, 0xffe8
-; GFX11-NEXT: s_or_b32 s5, s12, s5
-; GFX11-NEXT: s_add_i32 s4, s4, s10
-; GFX11-NEXT: s_and_b32 s10, s14, 0xff
-; GFX11-NEXT: s_cmp_ge_u32 s4, 24
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s6
-; GFX11-NEXT: s_cselect_b32 s11, 1, 0
-; GFX11-NEXT: s_sub_i32 s12, s4, 24
-; GFX11-NEXT: s_cmp_lg_u32 s11, 0
-; GFX11-NEXT: s_pack_ll_b32_b16 s2, s2, s8
-; GFX11-NEXT: s_cselect_b32 s4, s12, s4
-; GFX11-NEXT: s_pack_ll_b32_b16 s5, s5, s10
-; GFX11-NEXT: s_cmp_ge_u32 s4, 24
-; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s7
-; GFX11-NEXT: s_cselect_b32 s6, 1, 0
-; GFX11-NEXT: s_sub_i32 s8, s4, 24
-; GFX11-NEXT: s_cmp_lg_u32 s6, 0
-; GFX11-NEXT: s_pack_ll_b32_b16 s3, s3, s9
-; GFX11-NEXT: s_cselect_b32 s4, s8, s4
-; GFX11-NEXT: s_mul_hi_u32 s8, s5, s13
-; GFX11-NEXT: s_sub_i32 s6, 23, s4
-; GFX11-NEXT: s_lshl_b32 s0, s0, 1
-; GFX11-NEXT: s_mulk_i32 s8, 0xffe8
-; GFX11-NEXT: s_lshl_b32 s0, s0, s6
-; GFX11-NEXT: s_lshr_b32 s2, s2, s4
-; GFX11-NEXT: s_add_i32 s5, s5, s8
-; GFX11-NEXT: s_or_b32 s0, s0, s2
-; GFX11-NEXT: s_cmp_ge_u32 s5, 24
-; GFX11-NEXT: s_cselect_b32 s2, 1, 0
-; GFX11-NEXT: s_sub_i32 s4, s5, 24
-; GFX11-NEXT: s_cmp_lg_u32 s2, 0
-; GFX11-NEXT: s_cselect_b32 s2, s4, s5
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_cmp_ge_u32 s2, 24
-; GFX11-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-NEXT: s_sub_i32 s5, s2, 24
-; GFX11-NEXT: s_cmp_lg_u32 s4, 0
-; GFX11-NEXT: s_cselect_b32 s2, s5, s2
-; GFX11-NEXT: s_lshl_b32 s1, s1, 1
-; GFX11-NEXT: s_sub_i32 s4, 23, s2
-; GFX11-NEXT: s_lshr_b32 s2, s3, s2
-; GFX11-NEXT: s_lshl_b32 s1, s1, s4
-; GFX11-NEXT: s_and_b32 s3, s0, 0xff
-; GFX11-NEXT: s_or_b32 s1, s1, s2
-; GFX11-NEXT: s_bfe_u32 s2, s0, 0x80008
-; GFX11-NEXT: s_bfe_u32 s0, s0, 0x80010
-; GFX11-NEXT: s_lshl_b32 s2, s2, 8
-; GFX11-NEXT: s_lshl_b32 s0, s0, 16
-; GFX11-NEXT: s_or_b32 s2, s3, s2
-; GFX11-NEXT: s_and_b32 s3, s1, 0xff
-; GFX11-NEXT: s_or_b32 s0, s2, s0
-; GFX11-NEXT: s_lshl_b32 s2, s3, 24
-; GFX11-NEXT: s_bfe_u32 s3, s1, 0x80010
-; GFX11-NEXT: s_bfe_u32 s1, s1, 0x80008
-; GFX11-NEXT: s_lshl_b32 s3, s3, 8
-; GFX11-NEXT: s_or_b32 s0, s0, s2
-; GFX11-NEXT: s_or_b32 s1, s1, s3
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshr_v2i24:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_lshr_b32 s9, s1, 8
@@ -2432,6 +2393,7 @@ define amdgpu_ps i48 @s_fshr_v2i24(i48 inreg %lhs.arg, i48 inreg %rhs.arg, i48 i
; GFX9-NEXT: s_lshl_b32 s1, s1, 8
; GFX9-NEXT: s_or_b32 s1, s2, s1
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: s_fshr_v2i24:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
@@ -2556,6 +2518,116 @@ define amdgpu_ps i48 @s_fshr_v2i24(i48 inreg %lhs.arg, i48 inreg %rhs.arg, i48 i
; GFX10-NEXT: s_or_b32 s0, s0, s2
; GFX10-NEXT: s_or_b32 s1, s1, s3
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-TRUE16-LABEL: s_fshr_v2i24:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s1, 8
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s0, 24
+; GFX11-TRUE16-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s3, 8
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s3, 0xff
+; GFX11-TRUE16-NEXT: s_and_b32 s6, s6, 0xff
+; GFX11-TRUE16-NEXT: s_or_b32 s1, s8, s1
+; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s2, 24
+; GFX11-TRUE16-NEXT: s_lshl_b32 s3, s3, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s0, 16
+; GFX11-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-TRUE16-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, 0xff
+; GFX11-TRUE16-NEXT: s_lshl_b32 s6, s6, 8
+; GFX11-TRUE16-NEXT: s_and_b32 s8, s8, 0xff
+; GFX11-TRUE16-NEXT: s_or_b32 s3, s10, s3
+; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s4, 8
+; GFX11-TRUE16-NEXT: s_or_b32 s0, s0, s6
+; GFX11-TRUE16-NEXT: s_and_b32 s6, s7, 0xff
+; GFX11-TRUE16-NEXT: s_and_b32 s7, s9, 0xff
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s13, v0
+; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s2, 16
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, 0xff
+; GFX11-TRUE16-NEXT: s_lshl_b32 s8, s8, 8
+; GFX11-TRUE16-NEXT: s_and_b32 s10, s10, 0xff
+; GFX11-TRUE16-NEXT: s_or_b32 s2, s2, s8
+; GFX11-TRUE16-NEXT: s_and_b32 s8, s9, 0xff
+; GFX11-TRUE16-NEXT: s_and_b32 s9, s11, 0xff
+; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s4, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s4, 24
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s4, 0xff
+; GFX11-TRUE16-NEXT: s_lshl_b32 s10, s10, 8
+; GFX11-TRUE16-NEXT: s_and_b32 s11, s11, 0xff
+; GFX11-TRUE16-NEXT: s_or_b32 s4, s4, s10
+; GFX11-TRUE16-NEXT: s_mul_i32 s10, s13, 0xffffffe8
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s4, s4, s11
+; GFX11-TRUE16-NEXT: s_mul_hi_u32 s10, s13, s10
+; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s5, 8
+; GFX11-TRUE16-NEXT: s_add_i32 s13, s13, s10
+; GFX11-TRUE16-NEXT: s_and_b32 s5, s5, 0xff
+; GFX11-TRUE16-NEXT: s_mul_hi_u32 s10, s4, s13
+; GFX11-TRUE16-NEXT: s_lshl_b32 s5, s5, 8
+; GFX11-TRUE16-NEXT: s_mulk_i32 s10, 0xffe8
+; GFX11-TRUE16-NEXT: s_or_b32 s5, s12, s5
+; GFX11-TRUE16-NEXT: s_add_i32 s4, s4, s10
+; GFX11-TRUE16-NEXT: s_and_b32 s10, s14, 0xff
+; GFX11-TRUE16-NEXT: s_cmp_ge_u32 s4, 24
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s0, s0, s6
+; GFX11-TRUE16-NEXT: s_cselect_b32 s11, 1, 0
+; GFX11-TRUE16-NEXT: s_sub_i32 s12, s4, 24
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s11, 0
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s2, s2, s8
+; GFX11-TRUE16-NEXT: s_cselect_b32 s4, s12, s4
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s5, s5, s10
+; GFX11-TRUE16-NEXT: s_cmp_ge_u32 s4, 24
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s1, s1, s7
+; GFX11-TRUE16-NEXT: s_cselect_b32 s6, 1, 0
+; GFX11-TRUE16-NEXT: s_sub_i32 s8, s4, 24
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s6, 0
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s3, s3, s9
+; GFX11-TRUE16-NEXT: s_cselect_b32 s4, s8, s4
+; GFX11-TRUE16-NEXT: s_mul_hi_u32 s8, s5, s13
+; GFX11-TRUE16-NEXT: s_sub_i32 s6, 23, s4
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 1
+; GFX11-TRUE16-NEXT: s_mulk_i32 s8, 0xffe8
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, s6
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s2, s4
+; GFX11-TRUE16-NEXT: s_add_i32 s5, s5, s8
+; GFX11-TRUE16-NEXT: s_or_b32 s0, s0, s2
+; GFX11-TRUE16-NEXT: s_cmp_ge_u32 s5, 24
+; GFX11-TRUE16-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-TRUE16-NEXT: s_sub_i32 s4, s5, 24
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
+; GFX11-TRUE16-NEXT: s_cselect_b32 s2, s4, s5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_cmp_ge_u32 s2, 24
+; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-TRUE16-NEXT: s_sub_i32 s5, s2, 24
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 0
+; GFX11-TRUE16-NEXT: s_cselect_b32 s2, s5, s2
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 1
+; GFX11-TRUE16-NEXT: s_sub_i32 s4, 23, s2
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s3, s2
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, s4
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s0, 0xff
+; GFX11-TRUE16-NEXT: s_or_b32 s1, s1, s2
+; GFX11-TRUE16-NEXT: s_bfe_u32 s2, s0, 0x80008
+; GFX11-TRUE16-NEXT: s_bfe_u32 s0, s0, 0x80010
+; GFX11-TRUE16-NEXT: s_lshl_b32 s2, s2, 8
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
+; GFX11-TRUE16-NEXT: s_or_b32 s2, s3, s2
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s1, 0xff
+; GFX11-TRUE16-NEXT: s_or_b32 s0, s2, s0
+; GFX11-TRUE16-NEXT: s_lshl_b32 s2, s3, 24
+; GFX11-TRUE16-NEXT: s_bfe_u32 s3, s1, 0x80010
+; GFX11-TRUE16-NEXT: s_bfe_u32 s1, s1, 0x80008
+; GFX11-TRUE16-NEXT: s_lshl_b32 s3, s3, 8
+; GFX11-TRUE16-NEXT: s_or_b32 s0, s0, s2
+; GFX11-TRUE16-NEXT: s_or_b32 s1, s1, s3
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
; GFX11-FAKE16-LABEL: s_fshr_v2i24:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
@@ -2794,59 +2866,6 @@ define <2 x i24> @v_fshr_v2i24(<2 x i24> %lhs, <2 x i24> %rhs, <2 x i24> %amt) {
; GFX8-NEXT: v_or_b32_e32 v1, v1, v2
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshr_v2i24:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v6, 24
-; GFX11-NEXT: v_and_b32_e32 v4, 0xffffff, v4
-; GFX11-NEXT: v_and_b32_e32 v5, 0xffffff, v5
-; GFX11-NEXT: v_and_b32_e32 v2, 0xffffff, v2
-; GFX11-NEXT: v_and_b32_e32 v3, 0xffffff, v3
-; GFX11-NEXT: v_rcp_iflag_f32_e32 v6, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 1, v0
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_dual_mul_f32 v6, 0x4f7ffffe, v6 :: v_dual_lshlrev_b32 v1, 1, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cvt_u32_f32_e32 v6, v6
-; GFX11-NEXT: v_readfirstlane_b32 s0, v6
-; GFX11-NEXT: s_mul_i32 s1, s0, 0xffffffe8
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_mul_hi_u32 s1, s0, s1
-; GFX11-NEXT: s_add_i32 s0, s0, s1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_mul_hi_u32 v8, v4, s0
-; GFX11-NEXT: v_mad_u64_u32 v[6:7], null, 0xffffffe8, v8, v[4:5]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_add_nc_u32_e32 v4, 0xffffffe8, v6
-; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v6
-; GFX11-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc_lo
-; GFX11-NEXT: v_mul_hi_u32 v9, v5, s0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_mad_u64_u32 v[7:8], null, 0xffffffe8, v9, v[5:6]
-; GFX11-NEXT: v_add_nc_u32_e32 v6, 0xffffffe8, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_add_nc_u32_e32 v5, 0xffffffe8, v7
-; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v7
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc_lo
-; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_dual_cndmask_b32 v4, v4, v6 :: v_dual_add_nc_u32 v7, 0xffffffe8, v5
-; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v5
-; GFX11-NEXT: v_sub_nc_u32_e32 v6, 23, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_dual_cndmask_b32 v5, v5, v7 :: v_dual_and_b32 v4, 0xffffff, v4
-; GFX11-NEXT: v_and_b32_e32 v6, 0xffffff, v6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_sub_nc_u32_e32 v7, 23, v5
-; GFX11-NEXT: v_and_b32_e32 v5, 0xffffff, v5
-; GFX11-NEXT: v_lshrrev_b32_e32 v2, v4, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_and_b32_e32 v4, 0xffffff, v7
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshl_or_b32 v0, v0, v6, v2
-; GFX11-NEXT: v_lshl_or_b32 v1, v1, v4, v3
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshr_v2i24:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2892,6 +2911,7 @@ define <2 x i24> @v_fshr_v2i24(<2 x i24> %lhs, <2 x i24> %rhs, <2 x i24> %amt) {
; GFX9-NEXT: v_lshrrev_b32_e32 v2, v2, v3
; GFX9-NEXT: v_lshl_or_b32 v1, v1, v4, v2
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshr_v2i24:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2936,26 +2956,81 @@ define <2 x i24> @v_fshr_v2i24(<2 x i24> %lhs, <2 x i24> %rhs, <2 x i24> %amt) {
; GFX10-NEXT: v_lshl_or_b32 v0, v0, v6, v2
; GFX10-NEXT: v_lshl_or_b32 v1, v1, v5, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_fshr_v2i24:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v6, 24
+; GFX11-NEXT: v_and_b32_e32 v4, 0xffffff, v4
+; GFX11-NEXT: v_and_b32_e32 v5, 0xffffff, v5
+; GFX11-NEXT: v_and_b32_e32 v2, 0xffffff, v2
+; GFX11-NEXT: v_and_b32_e32 v3, 0xffffff, v3
+; GFX11-NEXT: v_rcp_iflag_f32_e32 v6, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 1, v0
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-NEXT: v_dual_mul_f32 v6, 0x4f7ffffe, v6 :: v_dual_lshlrev_b32 v1, 1, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cvt_u32_f32_e32 v6, v6
+; GFX11-NEXT: v_readfirstlane_b32 s0, v6
+; GFX11-NEXT: s_mul_i32 s1, s0, 0xffffffe8
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_mul_hi_u32 s1, s0, s1
+; GFX11-NEXT: s_add_i32 s0, s0, s1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_mul_hi_u32 v8, v4, s0
+; GFX11-NEXT: v_mad_u64_u32 v[6:7], null, 0xffffffe8, v8, v[4:5]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_add_nc_u32_e32 v4, 0xffffffe8, v6
+; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v6
+; GFX11-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc_lo
+; GFX11-NEXT: v_mul_hi_u32 v9, v5, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_mad_u64_u32 v[7:8], null, 0xffffffe8, v9, v[5:6]
+; GFX11-NEXT: v_add_nc_u32_e32 v6, 0xffffffe8, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_add_nc_u32_e32 v5, 0xffffffe8, v7
+; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v7
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc_lo
+; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_dual_cndmask_b32 v4, v4, v6 :: v_dual_add_nc_u32 v7, 0xffffffe8, v5
+; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v5
+; GFX11-NEXT: v_sub_nc_u32_e32 v6, 23, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_dual_cndmask_b32 v5, v5, v7 :: v_dual_and_b32 v4, 0xffffff, v4
+; GFX11-NEXT: v_and_b32_e32 v6, 0xffffff, v6
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_sub_nc_u32_e32 v7, 23, v5
+; GFX11-NEXT: v_and_b32_e32 v5, 0xffffff, v5
+; GFX11-NEXT: v_lshrrev_b32_e32 v2, v4, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_and_b32_e32 v4, 0xffffff, v7
+; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_lshl_or_b32 v0, v0, v6, v2
+; GFX11-NEXT: v_lshl_or_b32 v1, v1, v4, v3
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call <2 x i24> @llvm.fshr.v2i24(<2 x i24> %lhs, <2 x i24> %rhs, <2 x i24> %amt)
ret <2 x i24> %result
}
define amdgpu_ps i32 @s_fshr_i32(i32 inreg %lhs, i32 inreg %rhs, i32 inreg %amt) {
-; GCN-LABEL: s_fshr_i32:
-; GCN: ; %bb.0:
-; GCN-NEXT: v_mov_b32_e32 v0, s1
-; GCN-NEXT: v_mov_b32_e32 v1, s2
-; GCN-NEXT: v_alignbit_b32 v0, s0, v0, v1
-; GCN-NEXT: v_readfirstlane_b32 s0, v0
-; GCN-NEXT: ; return to shader part epilog
+; GFX6-LABEL: s_fshr_i32:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: v_mov_b32_e32 v0, s1
+; GFX6-NEXT: v_mov_b32_e32 v1, s2
+; GFX6-NEXT: v_alignbit_b32 v0, s0, v0, v1
+; GFX6-NEXT: v_readfirstlane_b32 s0, v0
+; GFX6-NEXT: ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshr_i32:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: v_mov_b32_e32 v0, s1
+; GFX8-NEXT: v_mov_b32_e32 v1, s2
+; GFX8-NEXT: v_alignbit_b32 v0, s0, v0, v1
+; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshr_i32:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_alignbit_b32 v0, s0, s1, v0.l
-; GFX11-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshr_i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_mov_b32_e32 v0, s1
@@ -2963,12 +3038,22 @@ define amdgpu_ps i32 @s_fshr_i32(i32 inreg %lhs, i32 inreg %rhs, i32 inreg %amt)
; GFX9-NEXT: v_alignbit_b32 v0, s0, v0, v1
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: s_fshr_i32:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_mov_b32_e32 v0, s2
; GFX10-NEXT: v_alignbit_b32 v0, s0, s1, v0
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-TRUE16-LABEL: s_fshr_i32:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, s0, s1, v0.l
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
; GFX11-FAKE16-LABEL: s_fshr_i32:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s2
@@ -2981,59 +3066,77 @@ define amdgpu_ps i32 @s_fshr_i32(i32 inreg %lhs, i32 inreg %rhs, i32 inreg %amt)
}
define amdgpu_ps i32 @s_fshr_i32_5(i32 inreg %lhs, i32 inreg %rhs) {
-; GCN-LABEL: s_fshr_i32_5:
-; GCN: ; %bb.0:
-; GCN-NEXT: v_mov_b32_e32 v0, s1
-; GCN-NEXT: v_alignbit_b32 v0, s0, v0, 5
-; GCN-NEXT: v_readfirstlane_b32 s0, v0
-; GCN-NEXT: ; return to shader part epilog
+; GFX6-LABEL: s_fshr_i32_5:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: v_mov_b32_e32 v0, s1
+; GFX6-NEXT: v_alignbit_b32 v0, s0, v0, 5
+; GFX6-NEXT: v_readfirstlane_b32 s0, v0
+; GFX6-NEXT: ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshr_i32_5:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: v_mov_b32_e32 v0, s1
+; GFX8-NEXT: v_alignbit_b32 v0, s0, v0, 5
+; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshr_i32_5:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_alignbit_b32 v0, s0, s1, 5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshr_i32_5:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_mov_b32_e32 v0, s1
; GFX9-NEXT: v_alignbit_b32 v0, s0, v0, 5
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: s_fshr_i32_5:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_alignbit_b32 v0, s0, s1, 5
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: s_fshr_i32_5:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_alignbit_b32 v0, s0, s1, 5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: ; return to shader part epilog
%result = call i32 @llvm.fshr.i32(i32 %lhs, i32 %rhs, i32 5)
ret i32 %result
}
define amdgpu_ps i32 @s_fshr_i32_8(i32 inreg %lhs, i32 inreg %rhs) {
-; GCN-LABEL: s_fshr_i32_8:
-; GCN: ; %bb.0:
-; GCN-NEXT: v_mov_b32_e32 v0, s1
-; GCN-NEXT: v_alignbit_b32 v0, s0, v0, 8
-; GCN-NEXT: v_readfirstlane_b32 s0, v0
-; GCN-NEXT: ; return to shader part epilog
+; GFX6-LABEL: s_fshr_i32_8:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: v_mov_b32_e32 v0, s1
+; GFX6-NEXT: v_alignbit_b32 v0, s0, v0, 8
+; GFX6-NEXT: v_readfirstlane_b32 s0, v0
+; GFX6-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshr_i32_8:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_alignbit_b32 v0, s0, s1, 8
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-NEXT: ; return to shader part epilog
-; GFX9-LABEL: s_fshr_i32_8:
+; GFX8-LABEL: s_fshr_i32_8:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: v_mov_b32_e32 v0, s1
+; GFX8-NEXT: v_alignbit_b32 v0, s0, v0, 8
+; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: ; return to shader part epilog
+;
+; GFX9-LABEL: s_fshr_i32_8:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_mov_b32_e32 v0, s1
; GFX9-NEXT: v_alignbit_b32 v0, s0, v0, 8
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: s_fshr_i32_8:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_alignbit_b32 v0, s0, s1, 8
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: s_fshr_i32_8:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_alignbit_b32 v0, s0, s1, 8
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: ; return to shader part epilog
%result = call i32 @llvm.fshr.i32(i32 %lhs, i32 %rhs, i32 8)
ret i32 %result
}
@@ -3045,11 +3148,12 @@ define i32 @v_fshr_i32(i32 %lhs, i32 %rhs, i32 %amt) {
; GCN-NEXT: v_alignbit_b32 v0, v0, v1, v2
; GCN-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshr_i32:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_alignbit_b32 v0, v0, v1, v2.l
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: v_fshr_i32:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, v0, v1, v2.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-FAKE16-LABEL: v_fshr_i32:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3092,25 +3196,34 @@ define i32 @v_fshr_i32_8(i32 %lhs, i32 %rhs) {
}
define amdgpu_ps float @v_fshr_i32_ssv(i32 inreg %lhs, i32 inreg %rhs, i32 %amt) {
-; GCN-LABEL: v_fshr_i32_ssv:
-; GCN: ; %bb.0:
-; GCN-NEXT: v_mov_b32_e32 v1, s1
-; GCN-NEXT: v_alignbit_b32 v0, s0, v1, v0
-; GCN-NEXT: ; return to shader part epilog
+; GFX6-LABEL: v_fshr_i32_ssv:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: v_mov_b32_e32 v1, s1
+; GFX6-NEXT: v_alignbit_b32 v0, s0, v1, v0
+; GFX6-NEXT: ; return to shader part epilog
+;
+; GFX8-LABEL: v_fshr_i32_ssv:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: v_alignbit_b32 v0, s0, v1, v0
+; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: v_fshr_i32_ssv:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_alignbit_b32 v0, s0, s1, v0.l
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshr_i32_ssv:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: v_alignbit_b32 v0, s0, v1, v0
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: v_fshr_i32_ssv:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_alignbit_b32 v0, s0, s1, v0
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-TRUE16-LABEL: v_fshr_i32_ssv:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, s0, s1, v0.l
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
; GFX11-FAKE16-LABEL: v_fshr_i32_ssv:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, s0, s1, v0
@@ -3121,27 +3234,36 @@ define amdgpu_ps float @v_fshr_i32_ssv(i32 inreg %lhs, i32 inreg %rhs, i32 %amt)
}
define amdgpu_ps float @v_fshr_i32_svs(i32 inreg %lhs, i32 %rhs, i32 inreg %amt) {
-; GCN-LABEL: v_fshr_i32_svs:
-; GCN: ; %bb.0:
-; GCN-NEXT: v_mov_b32_e32 v1, s1
-; GCN-NEXT: v_alignbit_b32 v0, s0, v0, v1
-; GCN-NEXT: ; return to shader part epilog
+; GFX6-LABEL: v_fshr_i32_svs:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: v_mov_b32_e32 v1, s1
+; GFX6-NEXT: v_alignbit_b32 v0, s0, v0, v1
+; GFX6-NEXT: ; return to shader part epilog
+;
+; GFX8-LABEL: v_fshr_i32_svs:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: v_alignbit_b32 v0, s0, v0, v1
+; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: v_fshr_i32_svs:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_mov_b32_e32 v1, s1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_alignbit_b32 v0, s0, v0, v1.l
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshr_i32_svs:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: v_alignbit_b32 v0, s0, v0, v1
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: v_fshr_i32_svs:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_alignbit_b32 v0, s0, v0, s1
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-TRUE16-LABEL: v_fshr_i32_svs:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, s0, v0, v1.l
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
; GFX11-FAKE16-LABEL: v_fshr_i32_svs:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, s0, v0, s1
@@ -3152,30 +3274,40 @@ define amdgpu_ps float @v_fshr_i32_svs(i32 inreg %lhs, i32 %rhs, i32 inreg %amt)
}
define amdgpu_ps float @v_fshr_i32_vss(i32 inreg %lhs, i32 inreg %rhs, i32 inreg %amt) {
-; GCN-LABEL: v_fshr_i32_vss:
-; GCN: ; %bb.0:
-; GCN-NEXT: v_mov_b32_e32 v0, s1
-; GCN-NEXT: v_mov_b32_e32 v1, s2
-; GCN-NEXT: v_alignbit_b32 v0, s0, v0, v1
-; GCN-NEXT: ; return to shader part epilog
+; GFX6-LABEL: v_fshr_i32_vss:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: v_mov_b32_e32 v0, s1
+; GFX6-NEXT: v_mov_b32_e32 v1, s2
+; GFX6-NEXT: v_alignbit_b32 v0, s0, v0, v1
+; GFX6-NEXT: ; return to shader part epilog
+;
+; GFX8-LABEL: v_fshr_i32_vss:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: v_mov_b32_e32 v0, s1
+; GFX8-NEXT: v_mov_b32_e32 v1, s2
+; GFX8-NEXT: v_alignbit_b32 v0, s0, v0, v1
+; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: v_fshr_i32_vss:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_alignbit_b32 v0, s0, s1, v0.l
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshr_i32_vss:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_mov_b32_e32 v0, s1
; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: v_alignbit_b32 v0, s0, v0, v1
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: v_fshr_i32_vss:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_mov_b32_e32 v0, s2
; GFX10-NEXT: v_alignbit_b32 v0, s0, s1, v0
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-TRUE16-LABEL: v_fshr_i32_vss:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, s0, s1, v0.l
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
; GFX11-FAKE16-LABEL: v_fshr_i32_vss:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s2
@@ -3195,12 +3327,13 @@ define <2 x i32> @v_fshr_v2i32(<2 x i32> %lhs, <2 x i32> %rhs, <2 x i32> %amt) {
; GCN-NEXT: v_alignbit_b32 v1, v1, v3, v5
; GCN-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshr_v2i32:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_alignbit_b32 v0, v0, v2, v4.l
-; GFX11-NEXT: v_alignbit_b32 v1, v1, v3, v5.l
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: v_fshr_v2i32:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, v0, v2, v4.l
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, v1, v3, v5.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-FAKE16-LABEL: v_fshr_v2i32:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3220,13 +3353,14 @@ define <3 x i32> @v_fshr_v3i32(<3 x i32> %lhs, <3 x i32> %rhs, <3 x i32> %amt) {
; GCN-NEXT: v_alignbit_b32 v2, v2, v5, v8
; GCN-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshr_v3i32:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_alignbit_b32 v0, v0, v3, v6.l
-; GFX11-NEXT: v_alignbit_b32 v1, v1, v4, v7.l
-; GFX11-NEXT: v_alignbit_b32 v2, v2, v5, v8.l
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: v_fshr_v3i32:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, v0, v3, v6.l
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, v1, v4, v7.l
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v2, v2, v5, v8.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-FAKE16-LABEL: v_fshr_v3i32:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3248,14 +3382,15 @@ define <4 x i32> @v_fshr_v4i32(<4 x i32> %lhs, <4 x i32> %rhs, <4 x i32> %amt) {
; GCN-NEXT: v_alignbit_b32 v3, v3, v7, v11
; GCN-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshr_v4i32:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_alignbit_b32 v0, v0, v4, v8.l
-; GFX11-NEXT: v_alignbit_b32 v1, v1, v5, v9.l
-; GFX11-NEXT: v_alignbit_b32 v2, v2, v6, v10.l
-; GFX11-NEXT: v_alignbit_b32 v3, v3, v7, v11.l
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: v_fshr_v4i32:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, v0, v4, v8.l
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, v1, v5, v9.l
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v2, v2, v6, v10.l
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v3, v3, v7, v11.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-FAKE16-LABEL: v_fshr_v4i32:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3291,17 +3426,6 @@ define amdgpu_ps i16 @s_fshr_i16(i16 inreg %lhs, i16 inreg %rhs, i16 inreg %amt)
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshr_i16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s3, s2, 15
-; GFX11-NEXT: s_and_not1_b32 s2, 15, s2
-; GFX11-NEXT: s_lshl_b32 s0, s0, 1
-; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX11-NEXT: s_lshl_b32 s0, s0, s2
-; GFX11-NEXT: s_lshr_b32 s1, s1, s3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshr_i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s3, s2, 15
@@ -3312,6 +3436,7 @@ define amdgpu_ps i16 @s_fshr_i16(i16 inreg %lhs, i16 inreg %rhs, i16 inreg %amt)
; GFX9-NEXT: s_lshr_b32 s1, s1, s3
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: s_fshr_i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s3, s2, 15
@@ -3322,6 +3447,18 @@ define amdgpu_ps i16 @s_fshr_i16(i16 inreg %lhs, i16 inreg %rhs, i16 inreg %amt)
; GFX10-NEXT: s_lshr_b32 s1, s1, s3
; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: s_fshr_i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s3, s2, 15
+; GFX11-NEXT: s_and_not1_b32 s2, 15, s2
+; GFX11-NEXT: s_lshl_b32 s0, s0, 1
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_lshl_b32 s0, s0, s2
+; GFX11-NEXT: s_lshr_b32 s1, s1, s3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: ; return to shader part epilog
%result = call i16 @llvm.fshr.i16(i16 %lhs, i16 %rhs, i16 %amt)
ret i16 %result
}
@@ -3342,14 +3479,6 @@ define amdgpu_ps i16 @s_fshr_i16_4(i16 inreg %lhs, i16 inreg %rhs) {
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshr_i16_4:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX11-NEXT: s_lshl_b32 s0, s0, 12
-; GFX11-NEXT: s_lshr_b32 s1, s1, 4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshr_i16_4:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s1, 0xffff, s1
@@ -3357,6 +3486,7 @@ define amdgpu_ps i16 @s_fshr_i16_4(i16 inreg %lhs, i16 inreg %rhs) {
; GFX9-NEXT: s_lshr_b32 s1, s1, 4
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: s_fshr_i16_4:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s1, 0xffff, s1
@@ -3364,6 +3494,15 @@ define amdgpu_ps i16 @s_fshr_i16_4(i16 inreg %lhs, i16 inreg %rhs) {
; GFX10-NEXT: s_lshr_b32 s1, s1, 4
; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: s_fshr_i16_4:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_lshl_b32 s0, s0, 12
+; GFX11-NEXT: s_lshr_b32 s1, s1, 4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: ; return to shader part epilog
%result = call i16 @llvm.fshr.i16(i16 %lhs, i16 %rhs, i16 4)
ret i16 %result
}
@@ -3384,14 +3523,6 @@ define amdgpu_ps i16 @s_fshr_i16_5(i16 inreg %lhs, i16 inreg %rhs) {
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshr_i16_5:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX11-NEXT: s_lshl_b32 s0, s0, 11
-; GFX11-NEXT: s_lshr_b32 s1, s1, 5
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshr_i16_5:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s1, 0xffff, s1
@@ -3399,6 +3530,7 @@ define amdgpu_ps i16 @s_fshr_i16_5(i16 inreg %lhs, i16 inreg %rhs) {
; GFX9-NEXT: s_lshr_b32 s1, s1, 5
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: s_fshr_i16_5:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s1, 0xffff, s1
@@ -3406,6 +3538,15 @@ define amdgpu_ps i16 @s_fshr_i16_5(i16 inreg %lhs, i16 inreg %rhs) {
; GFX10-NEXT: s_lshr_b32 s1, s1, 5
; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: s_fshr_i16_5:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_lshl_b32 s0, s0, 11
+; GFX11-NEXT: s_lshr_b32 s1, s1, 5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: ; return to shader part epilog
%result = call i16 @llvm.fshr.i16(i16 %lhs, i16 %rhs, i16 5)
ret i16 %result
}
@@ -3436,19 +3577,6 @@ define i16 @v_fshr_i16(i16 %lhs, i16 %rhs, i16 %amt) {
; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshr_i16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_xor_b16 v0.h, v2.l, -1
-; GFX11-NEXT: v_lshlrev_b16 v0.l, 1, v0.l
-; GFX11-NEXT: v_and_b16 v1.h, v2.l, 15
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b16 v0.h, v0.h, 15
-; GFX11-NEXT: v_lshlrev_b16 v0.l, v0.h, v0.l
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b16 v0.h, v1.h, v1.l
-; GFX11-NEXT: v_or_b16 v0.l, v0.l, v0.h
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshr_i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3460,6 +3588,7 @@ define i16 @v_fshr_i16(i16 %lhs, i16 %rhs, i16 %amt) {
; GFX9-NEXT: v_lshrrev_b16_e32 v1, v3, v1
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshr_i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3471,6 +3600,21 @@ define i16 @v_fshr_i16(i16 %lhs, i16 %rhs, i16 %amt) {
; GFX10-NEXT: v_lshlrev_b16 v0, v3, v0
; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_fshr_i16:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_xor_b16 v0.h, v2.l, -1
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 1, v0.l
+; GFX11-TRUE16-NEXT: v_and_b16 v1.h, v2.l, 15
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v0.h, 15
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, v0.h, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, v1.h, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-FAKE16-LABEL: v_fshr_i16:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3505,14 +3649,6 @@ define i16 @v_fshr_i16_4(i16 %lhs, i16 %rhs) {
; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshr_i16_4:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshlrev_b16 v0.l, 12, v0.l
-; GFX11-NEXT: v_lshrrev_b16 v0.h, 4, v1.l
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_or_b16 v0.l, v0.l, v0.h
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshr_i16_4:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3520,6 +3656,7 @@ define i16 @v_fshr_i16_4(i16 %lhs, i16 %rhs) {
; GFX9-NEXT: v_lshrrev_b16_e32 v1, 4, v1
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshr_i16_4:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3527,6 +3664,16 @@ define i16 @v_fshr_i16_4(i16 %lhs, i16 %rhs) {
; GFX10-NEXT: v_lshrrev_b16 v1, 4, v1
; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_fshr_i16_4:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 12, v0.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, 4, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-FAKE16-LABEL: v_fshr_i16_4:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3556,14 +3703,6 @@ define i16 @v_fshr_i16_5(i16 %lhs, i16 %rhs) {
; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshr_i16_5:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshlrev_b16 v0.l, 11, v0.l
-; GFX11-NEXT: v_lshrrev_b16 v0.h, 5, v1.l
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_or_b16 v0.l, v0.l, v0.h
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshr_i16_5:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3571,6 +3710,7 @@ define i16 @v_fshr_i16_5(i16 %lhs, i16 %rhs) {
; GFX9-NEXT: v_lshrrev_b16_e32 v1, 5, v1
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshr_i16_5:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3578,6 +3718,16 @@ define i16 @v_fshr_i16_5(i16 %lhs, i16 %rhs) {
; GFX10-NEXT: v_lshrrev_b16 v1, 5, v1
; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_fshr_i16_5:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 11, v0.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, 5, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-FAKE16-LABEL: v_fshr_i16_5:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3614,18 +3764,6 @@ define amdgpu_ps half @v_fshr_i16_ssv(i16 inreg %lhs, i16 inreg %rhs, i16 %amt)
; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: v_fshr_i16_ssv:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_xor_b16 v0.h, v0.l, -1
-; GFX11-NEXT: v_and_b16 v0.l, v0.l, 15
-; GFX11-NEXT: s_lshl_b32 s0, s0, 1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_and_b16 v0.h, v0.h, 15
-; GFX11-NEXT: v_lshrrev_b16 v0.l, v0.l, s1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b16 v0.h, v0.h, s0
-; GFX11-NEXT: v_or_b16 v0.l, v0.h, v0.l
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshr_i16_ssv:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_and_b32_e32 v1, 15, v0
@@ -3636,6 +3774,7 @@ define amdgpu_ps half @v_fshr_i16_ssv(i16 inreg %lhs, i16 inreg %rhs, i16 %amt)
; GFX9-NEXT: v_lshrrev_b16_e64 v1, v1, s1
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: v_fshr_i16_ssv:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_xor_b32_e32 v1, -1, v0
@@ -3646,6 +3785,20 @@ define amdgpu_ps half @v_fshr_i16_ssv(i16 inreg %lhs, i16 inreg %rhs, i16 %amt)
; GFX10-NEXT: v_lshlrev_b16 v1, v1, s0
; GFX10-NEXT: v_or_b32_e32 v0, v1, v0
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-TRUE16-LABEL: v_fshr_i16_ssv:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: v_xor_b16 v0.h, v0.l, -1
+; GFX11-TRUE16-NEXT: v_and_b16 v0.l, v0.l, 15
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v0.h, 15
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.l, v0.l, s1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, v0.h, s0
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v0.l
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
; GFX11-FAKE16-LABEL: v_fshr_i16_ssv:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: v_xor_b32_e32 v1, -1, v0
@@ -3685,17 +3838,6 @@ define amdgpu_ps half @v_fshr_i16_svs(i16 inreg %lhs, i16 %rhs, i16 inreg %amt)
; GFX8-NEXT: v_or_b32_e32 v0, s0, v0
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: v_fshr_i16_svs:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s2, s1, 15
-; GFX11-NEXT: s_and_not1_b32 s1, 15, s1
-; GFX11-NEXT: v_lshrrev_b16 v0.l, s2, v0.l
-; GFX11-NEXT: s_lshl_b32 s0, s0, 1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshl_b32 s0, s0, s1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_or_b16 v0.l, s0, v0.l
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshr_i16_svs:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s2, s1, 15
@@ -3705,6 +3847,7 @@ define amdgpu_ps half @v_fshr_i16_svs(i16 inreg %lhs, i16 %rhs, i16 inreg %amt)
; GFX9-NEXT: v_lshrrev_b16_e32 v0, s2, v0
; GFX9-NEXT: v_or_b32_e32 v0, s0, v0
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: v_fshr_i16_svs:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s2, s1, 15
@@ -3714,6 +3857,19 @@ define amdgpu_ps half @v_fshr_i16_svs(i16 inreg %lhs, i16 %rhs, i16 inreg %amt)
; GFX10-NEXT: s_lshl_b32 s0, s0, s1
; GFX10-NEXT: v_or_b32_e32 v0, s0, v0
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-TRUE16-LABEL: v_fshr_i16_svs:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s1, 15
+; GFX11-TRUE16-NEXT: s_and_not1_b32 s1, 15, s1
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.l, s2, v0.l
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, s1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, s0, v0.l
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
; GFX11-FAKE16-LABEL: v_fshr_i16_svs:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_and_b32 s2, s1, 15
@@ -3753,18 +3909,6 @@ define amdgpu_ps half @v_fshr_i16_vss(i16 %lhs, i16 inreg %rhs, i16 inreg %amt)
; GFX8-NEXT: v_or_b32_e32 v0, s0, v0
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: v_fshr_i16_vss:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_lshlrev_b16 v0.l, 1, v0.l
-; GFX11-NEXT: s_and_not1_b32 s2, 15, s1
-; GFX11-NEXT: s_and_b32 s1, s1, 15
-; GFX11-NEXT: s_and_b32 s0, 0xffff, s0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b16 v0.l, s2, v0.l
-; GFX11-NEXT: s_lshr_b32 s0, s0, s1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_or_b16 v0.l, v0.l, s0
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshr_i16_vss:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s2, s1, 15
@@ -3775,6 +3919,7 @@ define amdgpu_ps half @v_fshr_i16_vss(i16 %lhs, i16 inreg %rhs, i16 inreg %amt)
; GFX9-NEXT: s_lshr_b32 s0, s0, s2
; GFX9-NEXT: v_or_b32_e32 v0, s0, v0
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: v_fshr_i16_vss:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_lshlrev_b16 v0, 1, v0
@@ -3785,6 +3930,20 @@ define amdgpu_ps half @v_fshr_i16_vss(i16 %lhs, i16 inreg %rhs, i16 inreg %amt)
; GFX10-NEXT: v_lshlrev_b16 v0, s2, v0
; GFX10-NEXT: v_or_b32_e32 v0, s0, v0
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-TRUE16-LABEL: v_fshr_i16_vss:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 1, v0.l
+; GFX11-TRUE16-NEXT: s_and_not1_b32 s2, 15, s1
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, 15
+; GFX11-TRUE16-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, s2, v0.l
+; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, s1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, s0
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
; GFX11-FAKE16-LABEL: v_fshr_i16_vss:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, 1, v0
@@ -3850,28 +4009,6 @@ define amdgpu_ps i32 @s_fshr_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs, <
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshr_v2i16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_lshr_b32 s3, s0, 16
-; GFX11-NEXT: s_lshl_b32 s0, s0, 0x10001
-; GFX11-NEXT: s_lshl_b32 s3, s3, 1
-; GFX11-NEXT: s_and_b32 s4, s2, 0xf000f
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s3
-; GFX11-NEXT: s_and_not1_b32 s2, 0xf000f, s2
-; GFX11-NEXT: s_lshr_b32 s3, s0, 16
-; GFX11-NEXT: s_lshr_b32 s5, s2, 16
-; GFX11-NEXT: s_lshl_b32 s0, s0, s2
-; GFX11-NEXT: s_lshl_b32 s2, s3, s5
-; GFX11-NEXT: s_and_b32 s3, s1, 0xffff
-; GFX11-NEXT: s_lshr_b32 s1, s1, 16
-; GFX11-NEXT: s_lshr_b32 s5, s4, 16
-; GFX11-NEXT: s_lshr_b32 s3, s3, s4
-; GFX11-NEXT: s_lshr_b32 s1, s1, s5
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s2
-; GFX11-NEXT: s_pack_ll_b32_b16 s1, s3, s1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s0, s0, s1
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshr_v2i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_lshr_b32 s4, s0, 16
@@ -3893,6 +4030,7 @@ define amdgpu_ps i32 @s_fshr_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs, <
; GFX9-NEXT: s_pack_ll_b32_b16 s1, s2, s1
; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: s_fshr_v2i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_lshr_b32 s3, s0, 16
@@ -3914,6 +4052,29 @@ define amdgpu_ps i32 @s_fshr_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs, <
; GFX10-NEXT: s_pack_ll_b32_b16 s1, s3, s1
; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: s_fshr_v2i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_lshr_b32 s3, s0, 16
+; GFX11-NEXT: s_lshl_b32 s0, s0, 0x10001
+; GFX11-NEXT: s_lshl_b32 s3, s3, 1
+; GFX11-NEXT: s_and_b32 s4, s2, 0xf000f
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s3
+; GFX11-NEXT: s_and_not1_b32 s2, 0xf000f, s2
+; GFX11-NEXT: s_lshr_b32 s3, s0, 16
+; GFX11-NEXT: s_lshr_b32 s5, s2, 16
+; GFX11-NEXT: s_lshl_b32 s0, s0, s2
+; GFX11-NEXT: s_lshl_b32 s2, s3, s5
+; GFX11-NEXT: s_and_b32 s3, s1, 0xffff
+; GFX11-NEXT: s_lshr_b32 s1, s1, 16
+; GFX11-NEXT: s_lshr_b32 s5, s4, 16
+; GFX11-NEXT: s_lshr_b32 s3, s3, s4
+; GFX11-NEXT: s_lshr_b32 s1, s1, s5
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX11-NEXT: s_pack_ll_b32_b16 s1, s3, s1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: ; return to shader part epilog
%result = call <2 x i16> @llvm.fshr.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt)
%cast = bitcast <2 x i16> %result to i32
ret i32 %cast
@@ -3972,19 +4133,6 @@ define <2 x i16> @v_fshr_v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt) {
; GFX8-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshr_v2i16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_xor_b32_e32 v3, -1, v2
-; GFX11-NEXT: v_pk_lshlrev_b16 v0, 1, v0 op_sel_hi:[0,1]
-; GFX11-NEXT: v_and_b32_e32 v2, 0xf000f, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_and_b32_e32 v3, 0xf000f, v3
-; GFX11-NEXT: v_pk_lshrrev_b16 v1, v2, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_pk_lshlrev_b16 v0, v3, v0
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshr_v2i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3996,6 +4144,7 @@ define <2 x i16> @v_fshr_v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt) {
; GFX9-NEXT: v_pk_lshrrev_b16 v1, v3, v1
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshr_v2i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -4007,6 +4156,20 @@ define <2 x i16> @v_fshr_v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt) {
; GFX10-NEXT: v_pk_lshlrev_b16 v0, v3, v0
; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_fshr_v2i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_xor_b32_e32 v3, -1, v2
+; GFX11-NEXT: v_pk_lshlrev_b16 v0, 1, v0 op_sel_hi:[0,1]
+; GFX11-NEXT: v_and_b32_e32 v2, 0xf000f, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_and_b32_e32 v3, 0xf000f, v3
+; GFX11-NEXT: v_pk_lshrrev_b16 v1, v2, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_pk_lshlrev_b16 v0, v3, v0
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call <2 x i16> @llvm.fshr.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt)
ret <2 x i16> %result
}
@@ -4040,14 +4203,6 @@ define <2 x i16> @v_fshr_v2i16_4_8(<2 x i16> %lhs, <2 x i16> %rhs) {
; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshr_v2i16_4_8:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_pk_lshlrev_b16 v0, 0x8000c, v0
-; GFX11-NEXT: v_pk_lshrrev_b16 v1, 0x80004, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshr_v2i16_4_8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -4057,6 +4212,7 @@ define <2 x i16> @v_fshr_v2i16_4_8(<2 x i16> %lhs, <2 x i16> %rhs) {
; GFX9-NEXT: v_pk_lshrrev_b16 v1, v2, v1
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshr_v2i16_4_8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -4064,9 +4220,18 @@ define <2 x i16> @v_fshr_v2i16_4_8(<2 x i16> %lhs, <2 x i16> %rhs) {
; GFX10-NEXT: v_pk_lshrrev_b16 v1, 0x80004, v1
; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
- %result = call <2 x i16> @llvm.fshr.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> <i16 4, i16 8>)
- ret <2 x i16> %result
-}
+;
+; GFX11-LABEL: v_fshr_v2i16_4_8:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_pk_lshlrev_b16 v0, 0x8000c, v0
+; GFX11-NEXT: v_pk_lshrrev_b16 v1, 0x80004, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %result = call <2 x i16> @llvm.fshr.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> <i16 4, i16 8>)
+ ret <2 x i16> %result
+}
define amdgpu_ps float @v_fshr_v2i16_ssv(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs, <2 x i16> %amt) {
; GFX6-LABEL: v_fshr_v2i16_ssv:
@@ -4120,20 +4285,6 @@ define amdgpu_ps float @v_fshr_v2i16_ssv(<2 x i16> inreg %lhs, <2 x i16> inreg %
; GFX8-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: v_fshr_v2i16_ssv:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_xor_b32_e32 v1, -1, v0
-; GFX11-NEXT: s_lshr_b32 s2, s0, 16
-; GFX11-NEXT: v_and_b32_e32 v0, 0xf000f, v0
-; GFX11-NEXT: s_lshl_b32 s0, s0, 0x10001
-; GFX11-NEXT: s_lshl_b32 s2, s2, 1
-; GFX11-NEXT: v_and_b32_e32 v1, 0xf000f, v1
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s2
-; GFX11-NEXT: v_pk_lshrrev_b16 v0, v0, s1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_pk_lshlrev_b16 v1, v1, s0
-; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshr_v2i16_ssv:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_lshr_b32 s2, s0, 16
@@ -4147,6 +4298,7 @@ define amdgpu_ps float @v_fshr_v2i16_ssv(<2 x i16> inreg %lhs, <2 x i16> inreg %
; GFX9-NEXT: v_pk_lshrrev_b16 v1, v1, s1
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: v_fshr_v2i16_ssv:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_xor_b32_e32 v1, -1, v0
@@ -4160,6 +4312,21 @@ define amdgpu_ps float @v_fshr_v2i16_ssv(<2 x i16> inreg %lhs, <2 x i16> inreg %
; GFX10-NEXT: v_pk_lshlrev_b16 v1, v1, s0
; GFX10-NEXT: v_or_b32_e32 v0, v1, v0
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: v_fshr_v2i16_ssv:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_xor_b32_e32 v1, -1, v0
+; GFX11-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-NEXT: v_and_b32_e32 v0, 0xf000f, v0
+; GFX11-NEXT: s_lshl_b32 s0, s0, 0x10001
+; GFX11-NEXT: s_lshl_b32 s2, s2, 1
+; GFX11-NEXT: v_and_b32_e32 v1, 0xf000f, v1
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX11-NEXT: v_pk_lshrrev_b16 v0, v0, s1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_pk_lshlrev_b16 v1, v1, s0
+; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-NEXT: ; return to shader part epilog
%result = call <2 x i16> @llvm.fshr.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt)
%cast = bitcast <2 x i16> %result to float
ret float %cast
@@ -4212,24 +4379,6 @@ define amdgpu_ps float @v_fshr_v2i16_svs(<2 x i16> inreg %lhs, <2 x i16> %rhs, <
; GFX8-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: v_fshr_v2i16_svs:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_lshr_b32 s2, s0, 16
-; GFX11-NEXT: s_lshl_b32 s0, s0, 0x10001
-; GFX11-NEXT: s_lshl_b32 s2, s2, 1
-; GFX11-NEXT: s_and_b32 s3, s1, 0xf000f
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s2
-; GFX11-NEXT: s_and_not1_b32 s1, 0xf000f, s1
-; GFX11-NEXT: s_lshr_b32 s2, s0, 16
-; GFX11-NEXT: s_lshr_b32 s4, s1, 16
-; GFX11-NEXT: v_pk_lshrrev_b16 v0, s3, v0
-; GFX11-NEXT: s_lshl_b32 s0, s0, s1
-; GFX11-NEXT: s_lshl_b32 s1, s2, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_or_b32_e32 v0, s0, v0
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshr_v2i16_svs:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_lshr_b32 s3, s0, 16
@@ -4246,6 +4395,7 @@ define amdgpu_ps float @v_fshr_v2i16_svs(<2 x i16> inreg %lhs, <2 x i16> %rhs, <
; GFX9-NEXT: v_pk_lshrrev_b16 v0, s2, v0
; GFX9-NEXT: v_or_b32_e32 v0, s0, v0
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: v_fshr_v2i16_svs:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_lshr_b32 s2, s0, 16
@@ -4262,6 +4412,25 @@ define amdgpu_ps float @v_fshr_v2i16_svs(<2 x i16> inreg %lhs, <2 x i16> %rhs, <
; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s1
; GFX10-NEXT: v_or_b32_e32 v0, s0, v0
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: v_fshr_v2i16_svs:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-NEXT: s_lshl_b32 s0, s0, 0x10001
+; GFX11-NEXT: s_lshl_b32 s2, s2, 1
+; GFX11-NEXT: s_and_b32 s3, s1, 0xf000f
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX11-NEXT: s_and_not1_b32 s1, 0xf000f, s1
+; GFX11-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-NEXT: s_lshr_b32 s4, s1, 16
+; GFX11-NEXT: v_pk_lshrrev_b16 v0, s3, v0
+; GFX11-NEXT: s_lshl_b32 s0, s0, s1
+; GFX11-NEXT: s_lshl_b32 s1, s2, s4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX11-NEXT: ; return to shader part epilog
%result = call <2 x i16> @llvm.fshr.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt)
%cast = bitcast <2 x i16> %result to float
ret float %cast
@@ -4315,22 +4484,6 @@ define amdgpu_ps float @v_fshr_v2i16_vss(<2 x i16> %lhs, <2 x i16> inreg %rhs, <
; GFX8-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: v_fshr_v2i16_vss:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_pk_lshlrev_b16 v0, 1, v0 op_sel_hi:[0,1]
-; GFX11-NEXT: s_and_b32 s2, s1, 0xf000f
-; GFX11-NEXT: s_and_not1_b32 s1, 0xf000f, s1
-; GFX11-NEXT: s_and_b32 s3, s0, 0xffff
-; GFX11-NEXT: s_lshr_b32 s0, s0, 16
-; GFX11-NEXT: s_lshr_b32 s4, s2, 16
-; GFX11-NEXT: v_pk_lshlrev_b16 v0, s1, v0
-; GFX11-NEXT: s_lshr_b32 s1, s3, s2
-; GFX11-NEXT: s_lshr_b32 s0, s0, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s1, s0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_or_b32_e32 v0, s0, v0
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshr_v2i16_vss:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s2, s1, 0xf000f
@@ -4345,6 +4498,7 @@ define amdgpu_ps float @v_fshr_v2i16_vss(<2 x i16> %lhs, <2 x i16> inreg %rhs, <
; GFX9-NEXT: s_pack_ll_b32_b16 s0, s1, s0
; GFX9-NEXT: v_or_b32_e32 v0, s0, v0
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: v_fshr_v2i16_vss:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_pk_lshlrev_b16 v0, 1, v0 op_sel_hi:[0,1]
@@ -4359,6 +4513,23 @@ define amdgpu_ps float @v_fshr_v2i16_vss(<2 x i16> %lhs, <2 x i16> inreg %rhs, <
; GFX10-NEXT: s_pack_ll_b32_b16 s0, s1, s0
; GFX10-NEXT: v_or_b32_e32 v0, s0, v0
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: v_fshr_v2i16_vss:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_pk_lshlrev_b16 v0, 1, v0 op_sel_hi:[0,1]
+; GFX11-NEXT: s_and_b32 s2, s1, 0xf000f
+; GFX11-NEXT: s_and_not1_b32 s1, 0xf000f, s1
+; GFX11-NEXT: s_and_b32 s3, s0, 0xffff
+; GFX11-NEXT: s_lshr_b32 s0, s0, 16
+; GFX11-NEXT: s_lshr_b32 s4, s2, 16
+; GFX11-NEXT: v_pk_lshlrev_b16 v0, s1, v0
+; GFX11-NEXT: s_lshr_b32 s1, s3, s2
+; GFX11-NEXT: s_lshr_b32 s0, s0, s4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s1, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX11-NEXT: ; return to shader part epilog
%result = call <2 x i16> @llvm.fshr.v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt)
%cast = bitcast <2 x i16> %result to float
ret float %cast
@@ -4429,45 +4600,6 @@ define amdgpu_ps i48 @s_fshr_v3i16(<3 x i16> inreg %lhs, <3 x i16> inreg %rhs, <
; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshr_v3i16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_lshr_b32 s6, s0, 16
-; GFX11-NEXT: s_lshl_b32 s0, s0, 0x10001
-; GFX11-NEXT: s_lshl_b32 s6, s6, 1
-; GFX11-NEXT: s_and_b32 s7, s4, 0xf000f
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s6
-; GFX11-NEXT: s_and_not1_b32 s4, 0xf000f, s4
-; GFX11-NEXT: s_lshr_b32 s6, s0, 16
-; GFX11-NEXT: s_lshr_b32 s8, s4, 16
-; GFX11-NEXT: s_lshl_b32 s0, s0, s4
-; GFX11-NEXT: s_lshl_b32 s4, s6, s8
-; GFX11-NEXT: s_and_b32 s6, s2, 0xffff
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s4
-; GFX11-NEXT: s_lshr_b32 s4, s1, 16
-; GFX11-NEXT: s_lshr_b32 s2, s2, 16
-; GFX11-NEXT: s_lshr_b32 s8, s7, 16
-; GFX11-NEXT: s_lshl_b32 s1, s1, 0x10001
-; GFX11-NEXT: s_lshl_b32 s4, s4, 1
-; GFX11-NEXT: s_lshr_b32 s6, s6, s7
-; GFX11-NEXT: s_lshr_b32 s2, s2, s8
-; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s4
-; GFX11-NEXT: s_and_not1_b32 s4, 0xf000f, s5
-; GFX11-NEXT: s_pack_ll_b32_b16 s2, s6, s2
-; GFX11-NEXT: s_and_b32 s6, s5, 0xf000f
-; GFX11-NEXT: s_lshr_b32 s5, s1, 16
-; GFX11-NEXT: s_lshr_b32 s7, s4, 16
-; GFX11-NEXT: s_lshl_b32 s1, s1, s4
-; GFX11-NEXT: s_lshl_b32 s4, s5, s7
-; GFX11-NEXT: s_and_b32 s5, s3, 0xffff
-; GFX11-NEXT: s_lshr_b32 s3, s3, 16
-; GFX11-NEXT: s_lshr_b32 s7, s6, 16
-; GFX11-NEXT: s_lshr_b32 s5, s5, s6
-; GFX11-NEXT: s_lshr_b32 s3, s3, s7
-; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s4
-; GFX11-NEXT: s_pack_ll_b32_b16 s3, s5, s3
-; GFX11-NEXT: s_or_b32 s0, s0, s2
-; GFX11-NEXT: s_or_b32 s1, s1, s3
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshr_v3i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_lshr_b32 s7, s0, 16
@@ -4512,6 +4644,7 @@ define amdgpu_ps i48 @s_fshr_v3i16(<3 x i16> inreg %lhs, <3 x i16> inreg %rhs, <
; GFX9-NEXT: s_or_b32 s0, s0, s2
; GFX9-NEXT: s_and_b32 s1, s1, 0xffff
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: s_fshr_v3i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_lshr_b32 s6, s0, 16
@@ -4556,6 +4689,47 @@ define amdgpu_ps i48 @s_fshr_v3i16(<3 x i16> inreg %lhs, <3 x i16> inreg %rhs, <
; GFX10-NEXT: s_or_b32 s0, s0, s3
; GFX10-NEXT: s_and_b32 s1, s1, 0xffff
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-TRUE16-LABEL: s_fshr_v3i16:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 16
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 0x10001
+; GFX11-TRUE16-NEXT: s_lshl_b32 s6, s6, 1
+; GFX11-TRUE16-NEXT: s_and_b32 s7, s4, 0xf000f
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s0, s0, s6
+; GFX11-TRUE16-NEXT: s_and_not1_b32 s4, 0xf000f, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s4, 16
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, s4
+; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s6, s8
+; GFX11-TRUE16-NEXT: s_and_b32 s6, s2, 0xffff
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s0, s0, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s7, 16
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 0x10001
+; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 1
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s6, s7
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s2, s8
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s1, s1, s4
+; GFX11-TRUE16-NEXT: s_and_not1_b32 s4, 0xf000f, s5
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s2, s6, s2
+; GFX11-TRUE16-NEXT: s_and_b32 s6, s5, 0xf000f
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s4, 16
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, s4
+; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s5, s7
+; GFX11-TRUE16-NEXT: s_and_b32 s5, s3, 0xffff
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s3, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s6, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s5, s6
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s3, s7
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s1, s1, s4
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s3, s5, s3
+; GFX11-TRUE16-NEXT: s_or_b32 s0, s0, s2
+; GFX11-TRUE16-NEXT: s_or_b32 s1, s1, s3
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
; GFX11-FAKE16-LABEL: s_fshr_v3i16:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_lshr_b32 s6, s0, 16
@@ -4676,26 +4850,6 @@ define <3 x half> @v_fshr_v3i16(<3 x i16> %lhs, <3 x i16> %rhs, <3 x i16> %amt)
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshr_v3i16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_xor_b32_e32 v6, -1, v4
-; GFX11-NEXT: v_xor_b32_e32 v7, -1, v5
-; GFX11-NEXT: v_and_b32_e32 v4, 0xf000f, v4
-; GFX11-NEXT: v_pk_lshlrev_b16 v0, 1, v0 op_sel_hi:[0,1]
-; GFX11-NEXT: v_and_b32_e32 v5, 0xf000f, v5
-; GFX11-NEXT: v_and_b32_e32 v6, 0xf000f, v6
-; GFX11-NEXT: v_pk_lshlrev_b16 v1, 1, v1 op_sel_hi:[0,1]
-; GFX11-NEXT: v_and_b32_e32 v7, 0xf000f, v7
-; GFX11-NEXT: v_pk_lshrrev_b16 v2, v4, v2
-; GFX11-NEXT: v_pk_lshrrev_b16 v3, v5, v3
-; GFX11-NEXT: v_pk_lshlrev_b16 v0, v6, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_pk_lshlrev_b16 v1, v7, v1
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshr_v3i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -4714,6 +4868,7 @@ define <3 x half> @v_fshr_v3i16(<3 x i16> %lhs, <3 x i16> %rhs, <3 x i16> %amt)
; GFX9-NEXT: v_pk_lshrrev_b16 v2, v2, v3
; GFX9-NEXT: v_or_b32_e32 v1, v1, v2
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshr_v3i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -4732,6 +4887,27 @@ define <3 x half> @v_fshr_v3i16(<3 x i16> %lhs, <3 x i16> %rhs, <3 x i16> %amt)
; GFX10-NEXT: v_or_b32_e32 v0, v0, v2
; GFX10-NEXT: v_or_b32_e32 v1, v1, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_fshr_v3i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_xor_b32_e32 v6, -1, v4
+; GFX11-NEXT: v_xor_b32_e32 v7, -1, v5
+; GFX11-NEXT: v_and_b32_e32 v4, 0xf000f, v4
+; GFX11-NEXT: v_pk_lshlrev_b16 v0, 1, v0 op_sel_hi:[0,1]
+; GFX11-NEXT: v_and_b32_e32 v5, 0xf000f, v5
+; GFX11-NEXT: v_and_b32_e32 v6, 0xf000f, v6
+; GFX11-NEXT: v_pk_lshlrev_b16 v1, 1, v1 op_sel_hi:[0,1]
+; GFX11-NEXT: v_and_b32_e32 v7, 0xf000f, v7
+; GFX11-NEXT: v_pk_lshrrev_b16 v2, v4, v2
+; GFX11-NEXT: v_pk_lshrrev_b16 v3, v5, v3
+; GFX11-NEXT: v_pk_lshlrev_b16 v0, v6, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_pk_lshlrev_b16 v1, v7, v1
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call <3 x i16> @llvm.fshr.v3i16(<3 x i16> %lhs, <3 x i16> %rhs, <3 x i16> %amt)
%cast.result = bitcast <3 x i16> %result to <3 x half>
ret <3 x half> %cast.result
@@ -4826,45 +5002,6 @@ define amdgpu_ps <2 x i32> @s_fshr_v4i16(<4 x i16> inreg %lhs, <4 x i16> inreg %
; GFX8-NEXT: s_or_b32 s1, s1, s2
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshr_v4i16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_lshr_b32 s6, s0, 16
-; GFX11-NEXT: s_lshl_b32 s0, s0, 0x10001
-; GFX11-NEXT: s_lshl_b32 s6, s6, 1
-; GFX11-NEXT: s_and_b32 s7, s4, 0xf000f
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s6
-; GFX11-NEXT: s_and_not1_b32 s4, 0xf000f, s4
-; GFX11-NEXT: s_lshr_b32 s6, s0, 16
-; GFX11-NEXT: s_lshr_b32 s8, s4, 16
-; GFX11-NEXT: s_lshl_b32 s0, s0, s4
-; GFX11-NEXT: s_lshl_b32 s4, s6, s8
-; GFX11-NEXT: s_and_b32 s6, s2, 0xffff
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s4
-; GFX11-NEXT: s_lshr_b32 s4, s1, 16
-; GFX11-NEXT: s_lshr_b32 s2, s2, 16
-; GFX11-NEXT: s_lshr_b32 s8, s7, 16
-; GFX11-NEXT: s_lshl_b32 s1, s1, 0x10001
-; GFX11-NEXT: s_lshl_b32 s4, s4, 1
-; GFX11-NEXT: s_lshr_b32 s6, s6, s7
-; GFX11-NEXT: s_lshr_b32 s2, s2, s8
-; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s4
-; GFX11-NEXT: s_and_not1_b32 s4, 0xf000f, s5
-; GFX11-NEXT: s_pack_ll_b32_b16 s2, s6, s2
-; GFX11-NEXT: s_and_b32 s6, s5, 0xf000f
-; GFX11-NEXT: s_lshr_b32 s5, s1, 16
-; GFX11-NEXT: s_lshr_b32 s7, s4, 16
-; GFX11-NEXT: s_lshl_b32 s1, s1, s4
-; GFX11-NEXT: s_lshl_b32 s4, s5, s7
-; GFX11-NEXT: s_and_b32 s5, s3, 0xffff
-; GFX11-NEXT: s_lshr_b32 s3, s3, 16
-; GFX11-NEXT: s_lshr_b32 s7, s6, 16
-; GFX11-NEXT: s_lshr_b32 s5, s5, s6
-; GFX11-NEXT: s_lshr_b32 s3, s3, s7
-; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s4
-; GFX11-NEXT: s_pack_ll_b32_b16 s3, s5, s3
-; GFX11-NEXT: s_or_b32 s0, s0, s2
-; GFX11-NEXT: s_or_b32 s1, s1, s3
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshr_v4i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_lshr_b32 s7, s0, 16
@@ -4904,6 +5041,7 @@ define amdgpu_ps <2 x i32> @s_fshr_v4i16(<4 x i16> inreg %lhs, <4 x i16> inreg %
; GFX9-NEXT: s_pack_ll_b32_b16 s2, s2, s3
; GFX9-NEXT: s_or_b32 s1, s1, s2
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: s_fshr_v4i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_lshr_b32 s6, s0, 16
@@ -4943,25 +5081,65 @@ define amdgpu_ps <2 x i32> @s_fshr_v4i16(<4 x i16> inreg %lhs, <4 x i16> inreg %
; GFX10-NEXT: s_or_b32 s0, s0, s2
; GFX10-NEXT: s_or_b32 s1, s1, s3
; GFX10-NEXT: ; return to shader part epilog
- %result = call <4 x i16> @llvm.fshr.v4i16(<4 x i16> %lhs, <4 x i16> %rhs, <4 x i16> %amt)
- %cast.result = bitcast <4 x i16> %result to <2 x i32>
- ret <2 x i32> %cast.result
-}
-
-define <4 x half> @v_fshr_v4i16(<4 x i16> %lhs, <4 x i16> %rhs, <4 x i16> %amt) {
-; GFX6-LABEL: v_fshr_v4i16:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v10, 16, v4
-; GFX6-NEXT: v_and_b32_e32 v12, 15, v4
-; GFX6-NEXT: v_xor_b32_e32 v4, -1, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v6, 16, v0
-; GFX6-NEXT: v_lshrrev_b32_e32 v8, 16, v2
-; GFX6-NEXT: v_and_b32_e32 v4, 15, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 1, v0
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, v4, v0
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, v12, v2
+;
+; GFX11-LABEL: s_fshr_v4i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_lshr_b32 s6, s0, 16
+; GFX11-NEXT: s_lshl_b32 s0, s0, 0x10001
+; GFX11-NEXT: s_lshl_b32 s6, s6, 1
+; GFX11-NEXT: s_and_b32 s7, s4, 0xf000f
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s6
+; GFX11-NEXT: s_and_not1_b32 s4, 0xf000f, s4
+; GFX11-NEXT: s_lshr_b32 s6, s0, 16
+; GFX11-NEXT: s_lshr_b32 s8, s4, 16
+; GFX11-NEXT: s_lshl_b32 s0, s0, s4
+; GFX11-NEXT: s_lshl_b32 s4, s6, s8
+; GFX11-NEXT: s_and_b32 s6, s2, 0xffff
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s4
+; GFX11-NEXT: s_lshr_b32 s4, s1, 16
+; GFX11-NEXT: s_lshr_b32 s2, s2, 16
+; GFX11-NEXT: s_lshr_b32 s8, s7, 16
+; GFX11-NEXT: s_lshl_b32 s1, s1, 0x10001
+; GFX11-NEXT: s_lshl_b32 s4, s4, 1
+; GFX11-NEXT: s_lshr_b32 s6, s6, s7
+; GFX11-NEXT: s_lshr_b32 s2, s2, s8
+; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s4
+; GFX11-NEXT: s_and_not1_b32 s4, 0xf000f, s5
+; GFX11-NEXT: s_pack_ll_b32_b16 s2, s6, s2
+; GFX11-NEXT: s_and_b32 s6, s5, 0xf000f
+; GFX11-NEXT: s_lshr_b32 s5, s1, 16
+; GFX11-NEXT: s_lshr_b32 s7, s4, 16
+; GFX11-NEXT: s_lshl_b32 s1, s1, s4
+; GFX11-NEXT: s_lshl_b32 s4, s5, s7
+; GFX11-NEXT: s_and_b32 s5, s3, 0xffff
+; GFX11-NEXT: s_lshr_b32 s3, s3, 16
+; GFX11-NEXT: s_lshr_b32 s7, s6, 16
+; GFX11-NEXT: s_lshr_b32 s5, s5, s6
+; GFX11-NEXT: s_lshr_b32 s3, s3, s7
+; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s4
+; GFX11-NEXT: s_pack_ll_b32_b16 s3, s5, s3
+; GFX11-NEXT: s_or_b32 s0, s0, s2
+; GFX11-NEXT: s_or_b32 s1, s1, s3
+; GFX11-NEXT: ; return to shader part epilog
+ %result = call <4 x i16> @llvm.fshr.v4i16(<4 x i16> %lhs, <4 x i16> %rhs, <4 x i16> %amt)
+ %cast.result = bitcast <4 x i16> %result to <2 x i32>
+ ret <2 x i32> %cast.result
+}
+
+define <4 x half> @v_fshr_v4i16(<4 x i16> %lhs, <4 x i16> %rhs, <4 x i16> %amt) {
+; GFX6-LABEL: v_fshr_v4i16:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v10, 16, v4
+; GFX6-NEXT: v_and_b32_e32 v12, 15, v4
+; GFX6-NEXT: v_xor_b32_e32 v4, -1, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v8, 16, v2
+; GFX6-NEXT: v_and_b32_e32 v4, 15, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 1, v0
+; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, v4, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, v12, v2
; GFX6-NEXT: v_xor_b32_e32 v4, -1, v10
; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
; GFX6-NEXT: v_and_b32_e32 v2, 15, v10
@@ -5040,26 +5218,6 @@ define <4 x half> @v_fshr_v4i16(<4 x i16> %lhs, <4 x i16> %rhs, <4 x i16> %amt)
; GFX8-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshr_v4i16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_xor_b32_e32 v6, -1, v4
-; GFX11-NEXT: v_xor_b32_e32 v7, -1, v5
-; GFX11-NEXT: v_and_b32_e32 v4, 0xf000f, v4
-; GFX11-NEXT: v_pk_lshlrev_b16 v0, 1, v0 op_sel_hi:[0,1]
-; GFX11-NEXT: v_and_b32_e32 v5, 0xf000f, v5
-; GFX11-NEXT: v_and_b32_e32 v6, 0xf000f, v6
-; GFX11-NEXT: v_pk_lshlrev_b16 v1, 1, v1 op_sel_hi:[0,1]
-; GFX11-NEXT: v_and_b32_e32 v7, 0xf000f, v7
-; GFX11-NEXT: v_pk_lshrrev_b16 v2, v4, v2
-; GFX11-NEXT: v_pk_lshrrev_b16 v3, v5, v3
-; GFX11-NEXT: v_pk_lshlrev_b16 v0, v6, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_pk_lshlrev_b16 v1, v7, v1
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshr_v4i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -5078,6 +5236,7 @@ define <4 x half> @v_fshr_v4i16(<4 x i16> %lhs, <4 x i16> %rhs, <4 x i16> %amt)
; GFX9-NEXT: v_pk_lshrrev_b16 v2, v2, v3
; GFX9-NEXT: v_or_b32_e32 v1, v1, v2
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshr_v4i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -5096,30 +5255,51 @@ define <4 x half> @v_fshr_v4i16(<4 x i16> %lhs, <4 x i16> %rhs, <4 x i16> %amt)
; GFX10-NEXT: v_or_b32_e32 v0, v0, v2
; GFX10-NEXT: v_or_b32_e32 v1, v1, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_fshr_v4i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_xor_b32_e32 v6, -1, v4
+; GFX11-NEXT: v_xor_b32_e32 v7, -1, v5
+; GFX11-NEXT: v_and_b32_e32 v4, 0xf000f, v4
+; GFX11-NEXT: v_pk_lshlrev_b16 v0, 1, v0 op_sel_hi:[0,1]
+; GFX11-NEXT: v_and_b32_e32 v5, 0xf000f, v5
+; GFX11-NEXT: v_and_b32_e32 v6, 0xf000f, v6
+; GFX11-NEXT: v_pk_lshlrev_b16 v1, 1, v1 op_sel_hi:[0,1]
+; GFX11-NEXT: v_and_b32_e32 v7, 0xf000f, v7
+; GFX11-NEXT: v_pk_lshrrev_b16 v2, v4, v2
+; GFX11-NEXT: v_pk_lshrrev_b16 v3, v5, v3
+; GFX11-NEXT: v_pk_lshlrev_b16 v0, v6, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_pk_lshlrev_b16 v1, v7, v1
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x i16> @llvm.fshr.v4i16(<4 x i16> %lhs, <4 x i16> %rhs, <4 x i16> %amt)
%cast.result = bitcast <4 x i16> %result to <4 x half>
ret <4 x half> %cast.result
}
define amdgpu_ps i64 @s_fshr_i64(i64 inreg %lhs, i64 inreg %rhs, i64 inreg %amt) {
-; GCN-LABEL: s_fshr_i64:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
-; GCN-NEXT: s_not_b32 s5, s4
-; GCN-NEXT: s_lshl_b64 s[0:1], s[0:1], s5
-; GCN-NEXT: s_lshr_b64 s[2:3], s[2:3], s4
-; GCN-NEXT: s_or_b64 s[0:1], s[0:1], s[2:3]
-; GCN-NEXT: ; return to shader part epilog
+; GFX6-LABEL: s_fshr_i64:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
+; GFX6-NEXT: s_not_b32 s5, s4
+; GFX6-NEXT: s_lshl_b64 s[0:1], s[0:1], s5
+; GFX6-NEXT: s_lshr_b64 s[2:3], s[2:3], s4
+; GFX6-NEXT: s_or_b64 s[0:1], s[0:1], s[2:3]
+; GFX6-NEXT: ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshr_i64:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
+; GFX8-NEXT: s_not_b32 s5, s4
+; GFX8-NEXT: s_lshl_b64 s[0:1], s[0:1], s5
+; GFX8-NEXT: s_lshr_b64 s[2:3], s[2:3], s4
+; GFX8-NEXT: s_or_b64 s[0:1], s[0:1], s[2:3]
+; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshr_i64:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
-; GFX11-NEXT: s_not_b32 s5, s4
-; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], s4
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], s5
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b64 s[0:1], s[0:1], s[2:3]
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshr_i64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
@@ -5128,6 +5308,7 @@ define amdgpu_ps i64 @s_fshr_i64(i64 inreg %lhs, i64 inreg %rhs, i64 inreg %amt)
; GFX9-NEXT: s_lshr_b64 s[2:3], s[2:3], s4
; GFX9-NEXT: s_or_b64 s[0:1], s[0:1], s[2:3]
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: s_fshr_i64:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
@@ -5136,6 +5317,16 @@ define amdgpu_ps i64 @s_fshr_i64(i64 inreg %lhs, i64 inreg %rhs, i64 inreg %amt)
; GFX10-NEXT: s_lshl_b64 s[0:1], s[0:1], s5
; GFX10-NEXT: s_or_b64 s[0:1], s[0:1], s[2:3]
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: s_fshr_i64:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
+; GFX11-NEXT: s_not_b32 s5, s4
+; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], s4
+; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], s5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b64 s[0:1], s[0:1], s[2:3]
+; GFX11-NEXT: ; return to shader part epilog
%result = call i64 @llvm.fshr.i64(i64 %lhs, i64 %rhs, i64 %amt)
ret i64 %result
}
@@ -5143,23 +5334,19 @@ define amdgpu_ps i64 @s_fshr_i64(i64 inreg %lhs, i64 inreg %rhs, i64 inreg %amt)
define amdgpu_ps i64 @s_fshr_i64_5(i64 inreg %lhs, i64 inreg %rhs) {
; GCN-LABEL: s_fshr_i64_5:
; GCN: ; %bb.0:
-; GCN-NEXT: s_mov_b32 s4, s0
-; GCN-NEXT: s_lshr_b64 s[0:1], s[2:3], 5
-; GCN-NEXT: s_lshl_b32 s3, s4, 27
-; GCN-NEXT: s_mov_b32 s2, 0
-; GCN-NEXT: s_or_b64 s[2:3], s[2:3], s[0:1]
-; GCN-NEXT: s_mov_b32 s1, s3
+; GCN-NEXT: s_lshl_b32 s1, s0, 27
+; GCN-NEXT: s_mov_b32 s0, 0
+; GCN-NEXT: s_lshr_b64 s[2:3], s[2:3], 5
+; GCN-NEXT: s_or_b64 s[0:1], s[0:1], s[2:3]
; GCN-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: s_fshr_i64_5:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_mov_b32 s4, s0
-; GFX11-NEXT: s_lshr_b64 s[0:1], s[2:3], 5
-; GFX11-NEXT: s_lshl_b32 s3, s4, 27
-; GFX11-NEXT: s_mov_b32 s2, 0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b64 s[2:3], s[2:3], s[0:1]
-; GFX11-NEXT: s_mov_b32 s1, s3
+; GFX11-NEXT: s_lshl_b32 s1, s0, 27
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], 5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b64 s[0:1], s[0:1], s[2:3]
; GFX11-NEXT: ; return to shader part epilog
%result = call i64 @llvm.fshr.i64(i64 %lhs, i64 %rhs, i64 5)
ret i64 %result
@@ -5186,19 +5373,15 @@ define amdgpu_ps i64 @s_fshr_i64_48(i64 inreg %lhs, i64 inreg %rhs) {
; GCN: ; %bb.0:
; GCN-NEXT: s_lshl_b64 s[0:1], s[0:1], 16
; GCN-NEXT: s_lshr_b32 s2, s3, 16
-; GCN-NEXT: s_mov_b32 s3, 0
-; GCN-NEXT: s_or_b64 s[2:3], s[0:1], s[2:3]
-; GCN-NEXT: s_mov_b32 s0, s2
+; GCN-NEXT: s_or_b32 s0, s0, s2
; GCN-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: s_fshr_i64_48:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 16
; GFX11-NEXT: s_lshr_b32 s2, s3, 16
-; GFX11-NEXT: s_mov_b32 s3, 0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b64 s[2:3], s[0:1], s[2:3]
-; GFX11-NEXT: s_mov_b32 s0, s2
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, s2
; GFX11-NEXT: ; return to shader part epilog
%result = call i64 @llvm.fshr.i64(i64 %lhs, i64 %rhs, i64 48)
ret i64 %result
@@ -5229,19 +5412,6 @@ define i64 @v_fshr_i64(i64 %lhs, i64 %rhs, i64 %amt) {
; GFX8-NEXT: v_or_b32_e32 v1, v1, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshr_i64:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
-; GFX11-NEXT: v_and_b32_e32 v5, 63, v4
-; GFX11-NEXT: v_bfi_b32 v4, v4, 0, 63
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshrrev_b64 v[2:3], v5, v[2:3]
-; GFX11-NEXT: v_lshlrev_b64 v[0:1], v4, v[0:1]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshr_i64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -5253,6 +5423,7 @@ define i64 @v_fshr_i64(i64 %lhs, i64 %rhs, i64 %amt) {
; GFX9-NEXT: v_or_b32_e32 v0, v0, v2
; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshr_i64:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -5264,6 +5435,20 @@ define i64 @v_fshr_i64(i64 %lhs, i64 %rhs, i64 %amt) {
; GFX10-NEXT: v_or_b32_e32 v0, v0, v2
; GFX10-NEXT: v_or_b32_e32 v1, v1, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_fshr_i64:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
+; GFX11-NEXT: v_and_b32_e32 v5, 63, v4
+; GFX11-NEXT: v_bfi_b32 v4, v4, 0, 63
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_lshrrev_b64 v[2:3], v5, v[2:3]
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], v4, v[0:1]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call i64 @llvm.fshr.i64(i64 %lhs, i64 %rhs, i64 %amt)
ret i64 %result
}
@@ -5287,14 +5472,6 @@ define i64 @v_fshr_i64_5(i64 %lhs, i64 %rhs) {
; GFX8-NEXT: v_or_b32_e32 v1, v2, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshr_i64_5:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v4, v0
-; GFX11-NEXT: v_lshrrev_b64 v[0:1], 5, v[2:3]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_lshl_or_b32 v1, v4, 27, v1
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshr_i64_5:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -5302,6 +5479,7 @@ define i64 @v_fshr_i64_5(i64 %lhs, i64 %rhs) {
; GFX9-NEXT: v_lshrrev_b64 v[0:1], 5, v[2:3]
; GFX9-NEXT: v_lshl_or_b32 v1, v4, 27, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshr_i64_5:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -5309,6 +5487,15 @@ define i64 @v_fshr_i64_5(i64 %lhs, i64 %rhs) {
; GFX10-NEXT: v_lshrrev_b64 v[0:1], 5, v[2:3]
; GFX10-NEXT: v_lshl_or_b32 v1, v4, 27, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_fshr_i64_5:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v4, v0
+; GFX11-NEXT: v_lshrrev_b64 v[0:1], 5, v[2:3]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshl_or_b32 v1, v4, 27, v1
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call i64 @llvm.fshr.i64(i64 %lhs, i64 %rhs, i64 5)
ret i64 %result
}
@@ -5346,27 +5533,28 @@ define i64 @v_fshr_i64_48(i64 %lhs, i64 %rhs) {
; GFX8-NEXT: v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshr_i64_48:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshlrev_b64 v[0:1], 16, v[0:1]
-; GFX11-NEXT: v_mov_b16_e32 v2.h, 0
-; GFX11-NEXT: v_mov_b16_e32 v2.l, v3.h
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshr_i64_48:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_lshlrev_b64 v[0:1], 16, v[0:1]
; GFX9-NEXT: v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshr_i64_48:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshlrev_b64 v[0:1], 16, v[0:1]
; GFX10-NEXT: v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_fshr_i64_48:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], 16, v[0:1]
+; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call i64 @llvm.fshr.i64(i64 %lhs, i64 %rhs, i64 48)
ret i64 %result
}
@@ -5394,19 +5582,6 @@ define amdgpu_ps <2 x float> @v_fshr_i64_ssv(i64 inreg %lhs, i64 inreg %rhs, i64
; GFX8-NEXT: v_or_b32_e32 v1, v2, v4
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: v_fshr_i64_ssv:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_bfi_b32 v1, v0, 0, 63
-; GFX11-NEXT: v_and_b32_e32 v2, 63, v0
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_lshlrev_b64 v[0:1], v1, s[0:1]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b64 v[2:3], v2, s[2:3]
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshr_i64_ssv:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
@@ -5417,6 +5592,7 @@ define amdgpu_ps <2 x float> @v_fshr_i64_ssv(i64 inreg %lhs, i64 inreg %rhs, i64
; GFX9-NEXT: v_or_b32_e32 v0, v1, v3
; GFX9-NEXT: v_or_b32_e32 v1, v2, v4
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: v_fshr_i64_ssv:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_bfi_b32 v1, v0, 0, 63
@@ -5427,6 +5603,20 @@ define amdgpu_ps <2 x float> @v_fshr_i64_ssv(i64 inreg %lhs, i64 inreg %rhs, i64
; GFX10-NEXT: v_or_b32_e32 v0, v0, v2
; GFX10-NEXT: v_or_b32_e32 v1, v1, v3
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: v_fshr_i64_ssv:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_bfi_b32 v1, v0, 0, 63
+; GFX11-NEXT: v_and_b32_e32 v2, 63, v0
+; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], v1, s[0:1]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b64 v[2:3], v2, s[2:3]
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX11-NEXT: ; return to shader part epilog
%result = call i64 @llvm.fshr.i64(i64 %lhs, i64 %rhs, i64 %amt)
%cast = bitcast i64 %result to <2 x float>
ret <2 x float> %cast
@@ -5459,18 +5649,6 @@ define amdgpu_ps <2 x float> @v_fshr_i64_svs(i64 inreg %lhs, i64 %rhs, i64 inreg
; GFX8-NEXT: v_or_b32_e32 v1, v3, v1
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: v_fshr_i64_svs:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
-; GFX11-NEXT: s_not_b32 s3, s2
-; GFX11-NEXT: s_and_b32 s2, s2, 63
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], s3
-; GFX11-NEXT: v_lshrrev_b64 v[0:1], s2, v[0:1]
-; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v1, v3, v1
-; GFX11-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshr_i64_svs:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
@@ -5483,6 +5661,7 @@ define amdgpu_ps <2 x float> @v_fshr_i64_svs(i64 inreg %lhs, i64 %rhs, i64 inreg
; GFX9-NEXT: v_or_b32_e32 v0, v2, v0
; GFX9-NEXT: v_or_b32_e32 v1, v3, v1
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: v_fshr_i64_svs:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
@@ -5495,6 +5674,19 @@ define amdgpu_ps <2 x float> @v_fshr_i64_svs(i64 inreg %lhs, i64 %rhs, i64 inreg
; GFX10-NEXT: v_or_b32_e32 v1, v3, v1
; GFX10-NEXT: v_or_b32_e32 v0, v2, v0
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: v_fshr_i64_svs:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
+; GFX11-NEXT: s_not_b32 s3, s2
+; GFX11-NEXT: s_and_b32 s2, s2, 63
+; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], s3
+; GFX11-NEXT: v_lshrrev_b64 v[0:1], s2, v[0:1]
+; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v1, v3, v1
+; GFX11-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX11-NEXT: ; return to shader part epilog
%result = call i64 @llvm.fshr.i64(i64 %lhs, i64 %rhs, i64 %amt)
%cast = bitcast i64 %result to <2 x float>
ret <2 x float> %cast
@@ -5525,18 +5717,6 @@ define amdgpu_ps <2 x float> @v_fshr_i64_vss(i64 %lhs, i64 inreg %rhs, i64 inreg
; GFX8-NEXT: v_or_b32_e32 v1, v1, v3
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: v_fshr_i64_vss:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
-; GFX11-NEXT: s_and_not1_b32 s3, 63, s2
-; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
-; GFX11-NEXT: v_lshlrev_b64 v[0:1], s3, v[0:1]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshr_i64_vss:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
@@ -5548,6 +5728,7 @@ define amdgpu_ps <2 x float> @v_fshr_i64_vss(i64 %lhs, i64 inreg %rhs, i64 inreg
; GFX9-NEXT: v_or_b32_e32 v0, v0, v2
; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: v_fshr_i64_vss:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
@@ -5559,39 +5740,53 @@ define amdgpu_ps <2 x float> @v_fshr_i64_vss(i64 %lhs, i64 inreg %rhs, i64 inreg
; GFX10-NEXT: v_or_b32_e32 v0, v0, v2
; GFX10-NEXT: v_or_b32_e32 v1, v1, v3
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: v_fshr_i64_vss:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
+; GFX11-NEXT: s_and_not1_b32 s3, 63, s2
+; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], s3, v[0:1]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX11-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX11-NEXT: ; return to shader part epilog
%result = call i64 @llvm.fshr.i64(i64 %lhs, i64 %rhs, i64 %amt)
%cast = bitcast i64 %result to <2 x float>
ret <2 x float> %cast
}
define amdgpu_ps <2 x i64> @s_fshr_v2i64(<2 x i64> inreg %lhs, <2 x i64> inreg %rhs, <2 x i64> inreg %amt) {
-; GCN-LABEL: s_fshr_v2i64:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
-; GCN-NEXT: s_not_b32 s9, s8
-; GCN-NEXT: s_lshl_b64 s[0:1], s[0:1], s9
-; GCN-NEXT: s_lshr_b64 s[4:5], s[4:5], s8
-; GCN-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
-; GCN-NEXT: s_lshl_b64 s[2:3], s[2:3], 1
-; GCN-NEXT: s_not_b32 s4, s10
-; GCN-NEXT: s_lshl_b64 s[2:3], s[2:3], s4
-; GCN-NEXT: s_lshr_b64 s[4:5], s[6:7], s10
-; GCN-NEXT: s_or_b64 s[2:3], s[2:3], s[4:5]
-; GCN-NEXT: ; return to shader part epilog
+; GFX6-LABEL: s_fshr_v2i64:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
+; GFX6-NEXT: s_not_b32 s9, s8
+; GFX6-NEXT: s_lshl_b64 s[0:1], s[0:1], s9
+; GFX6-NEXT: s_lshr_b64 s[4:5], s[4:5], s8
+; GFX6-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
+; GFX6-NEXT: s_lshl_b64 s[2:3], s[2:3], 1
+; GFX6-NEXT: s_not_b32 s4, s10
+; GFX6-NEXT: s_lshl_b64 s[2:3], s[2:3], s4
+; GFX6-NEXT: s_lshr_b64 s[4:5], s[6:7], s10
+; GFX6-NEXT: s_or_b64 s[2:3], s[2:3], s[4:5]
+; GFX6-NEXT: ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshr_v2i64:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
+; GFX8-NEXT: s_not_b32 s9, s8
+; GFX8-NEXT: s_lshl_b64 s[0:1], s[0:1], s9
+; GFX8-NEXT: s_lshr_b64 s[4:5], s[4:5], s8
+; GFX8-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
+; GFX8-NEXT: s_lshl_b64 s[2:3], s[2:3], 1
+; GFX8-NEXT: s_not_b32 s4, s10
+; GFX8-NEXT: s_lshl_b64 s[2:3], s[2:3], s4
+; GFX8-NEXT: s_lshr_b64 s[4:5], s[6:7], s10
+; GFX8-NEXT: s_or_b64 s[2:3], s[2:3], s[4:5]
+; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshr_v2i64:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
-; GFX11-NEXT: s_not_b32 s9, s8
-; GFX11-NEXT: s_lshl_b64 s[2:3], s[2:3], 1
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], s9
-; GFX11-NEXT: s_not_b32 s9, s10
-; GFX11-NEXT: s_lshr_b64 s[4:5], s[4:5], s8
-; GFX11-NEXT: s_lshl_b64 s[2:3], s[2:3], s9
-; GFX11-NEXT: s_lshr_b64 s[6:7], s[6:7], s10
-; GFX11-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
-; GFX11-NEXT: s_or_b64 s[2:3], s[2:3], s[6:7]
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshr_v2i64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
@@ -5605,6 +5800,7 @@ define amdgpu_ps <2 x i64> @s_fshr_v2i64(<2 x i64> inreg %lhs, <2 x i64> inreg %
; GFX9-NEXT: s_lshr_b64 s[4:5], s[6:7], s10
; GFX9-NEXT: s_or_b64 s[2:3], s[2:3], s[4:5]
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: s_fshr_v2i64:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
@@ -5618,6 +5814,20 @@ define amdgpu_ps <2 x i64> @s_fshr_v2i64(<2 x i64> inreg %lhs, <2 x i64> inreg %
; GFX10-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
; GFX10-NEXT: s_or_b64 s[2:3], s[2:3], s[6:7]
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: s_fshr_v2i64:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
+; GFX11-NEXT: s_not_b32 s9, s8
+; GFX11-NEXT: s_lshl_b64 s[2:3], s[2:3], 1
+; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], s9
+; GFX11-NEXT: s_not_b32 s9, s10
+; GFX11-NEXT: s_lshr_b64 s[4:5], s[4:5], s8
+; GFX11-NEXT: s_lshl_b64 s[2:3], s[2:3], s9
+; GFX11-NEXT: s_lshr_b64 s[6:7], s[6:7], s10
+; GFX11-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
+; GFX11-NEXT: s_or_b64 s[2:3], s[2:3], s[6:7]
+; GFX11-NEXT: ; return to shader part epilog
%result = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %lhs, <2 x i64> %rhs, <2 x i64> %amt)
ret <2 x i64> %result
}
@@ -5661,28 +5871,6 @@ define <2 x i64> @v_fshr_v2i64(<2 x i64> %lhs, <2 x i64> %rhs, <2 x i64> %amt) {
; GFX8-NEXT: v_or_b32_e32 v3, v3, v7
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshr_v2i64:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
-; GFX11-NEXT: v_lshlrev_b64 v[2:3], 1, v[2:3]
-; GFX11-NEXT: v_bfi_b32 v9, v8, 0, 63
-; GFX11-NEXT: v_and_b32_e32 v8, 63, v8
-; GFX11-NEXT: v_bfi_b32 v11, v10, 0, 63
-; GFX11-NEXT: v_and_b32_e32 v10, 63, v10
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_lshlrev_b64 v[0:1], v9, v[0:1]
-; GFX11-NEXT: v_lshrrev_b64 v[4:5], v8, v[4:5]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_lshlrev_b64 v[2:3], v11, v[2:3]
-; GFX11-NEXT: v_lshrrev_b64 v[6:7], v10, v[6:7]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX11-NEXT: v_or_b32_e32 v1, v1, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX11-NEXT: v_or_b32_e32 v3, v3, v7
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshr_v2i64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -5701,6 +5889,7 @@ define <2 x i64> @v_fshr_v2i64(<2 x i64> %lhs, <2 x i64> %rhs, <2 x i64> %amt) {
; GFX9-NEXT: v_or_b32_e32 v2, v2, v6
; GFX9-NEXT: v_or_b32_e32 v3, v3, v7
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshr_v2i64:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -5719,6 +5908,29 @@ define <2 x i64> @v_fshr_v2i64(<2 x i64> %lhs, <2 x i64> %rhs, <2 x i64> %amt) {
; GFX10-NEXT: v_or_b32_e32 v2, v2, v6
; GFX10-NEXT: v_or_b32_e32 v3, v3, v7
; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_fshr_v2i64:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
+; GFX11-NEXT: v_lshlrev_b64 v[2:3], 1, v[2:3]
+; GFX11-NEXT: v_bfi_b32 v9, v8, 0, 63
+; GFX11-NEXT: v_and_b32_e32 v8, 63, v8
+; GFX11-NEXT: v_bfi_b32 v11, v10, 0, 63
+; GFX11-NEXT: v_and_b32_e32 v10, 63, v10
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], v9, v[0:1]
+; GFX11-NEXT: v_lshrrev_b64 v[4:5], v8, v[4:5]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_lshlrev_b64 v[2:3], v11, v[2:3]
+; GFX11-NEXT: v_lshrrev_b64 v[6:7], v10, v[6:7]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX11-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX11-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %lhs, <2 x i64> %rhs, <2 x i64> %amt)
ret <2 x i64> %result
}
@@ -5726,73 +5938,96 @@ define <2 x i64> @v_fshr_v2i64(<2 x i64> %lhs, <2 x i64> %rhs, <2 x i64> %amt) {
define amdgpu_ps i128 @s_fshr_i128(i128 inreg %lhs, i128 inreg %rhs, i128 inreg %amt) {
; GFX6-LABEL: s_fshr_i128:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_and_b32 s10, s8, 64
-; GFX6-NEXT: s_mov_b32 s11, 0
-; GFX6-NEXT: v_cmp_eq_u64_e64 s[10:11], s[10:11], 0
-; GFX6-NEXT: s_and_b64 s[12:13], s[10:11], exec
-; GFX6-NEXT: s_cselect_b32 s13, s7, s1
-; GFX6-NEXT: s_cselect_b32 s12, s6, s0
-; GFX6-NEXT: s_lshr_b64 s[14:15], s[12:13], s8
-; GFX6-NEXT: s_and_b64 s[16:17], s[10:11], exec
-; GFX6-NEXT: s_cselect_b32 s1, s1, s3
-; GFX6-NEXT: s_cselect_b32 s0, s0, s2
-; GFX6-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
+; GFX6-NEXT: s_lshl_b64 s[10:11], s[0:1], 1
+; GFX6-NEXT: s_lshl_b64 s[2:3], s[2:3], 1
+; GFX6-NEXT: s_lshr_b32 s0, s1, 31
+; GFX6-NEXT: s_or_b32 s2, s2, s0
+; GFX6-NEXT: s_andn2_b32 s0, 0x7f, s8
; GFX6-NEXT: s_not_b32 s9, s8
-; GFX6-NEXT: s_lshl_b64 s[0:1], s[0:1], s9
-; GFX6-NEXT: s_or_b64 s[2:3], s[0:1], s[14:15]
-; GFX6-NEXT: s_and_b64 s[0:1], s[10:11], exec
-; GFX6-NEXT: s_cselect_b32 s1, s5, s7
-; GFX6-NEXT: s_cselect_b32 s0, s4, s6
-; GFX6-NEXT: s_lshl_b64 s[4:5], s[12:13], 1
-; GFX6-NEXT: s_lshr_b64 s[0:1], s[0:1], s8
-; GFX6-NEXT: s_lshl_b64 s[4:5], s[4:5], s9
-; GFX6-NEXT: s_or_b64 s[0:1], s[4:5], s[0:1]
+; GFX6-NEXT: s_sub_i32 s16, s0, 64
+; GFX6-NEXT: s_sub_i32 s12, 64, s0
+; GFX6-NEXT: s_cmp_lt_u32 s0, 64
+; GFX6-NEXT: s_cselect_b32 s17, 1, 0
+; GFX6-NEXT: s_cmp_eq_u32 s0, 0
+; GFX6-NEXT: s_cselect_b32 s18, 1, 0
+; GFX6-NEXT: s_lshr_b64 s[12:13], s[10:11], s12
+; GFX6-NEXT: s_lshl_b64 s[14:15], s[2:3], s9
+; GFX6-NEXT: s_lshl_b64 s[0:1], s[10:11], s9
+; GFX6-NEXT: s_or_b64 s[12:13], s[12:13], s[14:15]
+; GFX6-NEXT: s_lshl_b64 s[10:11], s[10:11], s16
+; GFX6-NEXT: s_cmp_lg_u32 s17, 0
+; GFX6-NEXT: s_cselect_b64 s[0:1], s[0:1], 0
+; GFX6-NEXT: s_cselect_b64 s[10:11], s[12:13], s[10:11]
+; GFX6-NEXT: s_cmp_lg_u32 s18, 0
+; GFX6-NEXT: s_cselect_b64 s[2:3], s[2:3], s[10:11]
+; GFX6-NEXT: s_and_b32 s9, s8, 0x7f
+; GFX6-NEXT: s_sub_i32 s14, s9, 64
+; GFX6-NEXT: s_sub_i32 s12, 64, s9
+; GFX6-NEXT: s_cmp_lt_u32 s9, 64
+; GFX6-NEXT: s_cselect_b32 s15, 1, 0
+; GFX6-NEXT: s_cmp_eq_u32 s9, 0
+; GFX6-NEXT: s_cselect_b32 s16, 1, 0
+; GFX6-NEXT: s_lshr_b64 s[10:11], s[6:7], s8
+; GFX6-NEXT: s_lshr_b64 s[8:9], s[4:5], s8
+; GFX6-NEXT: s_lshl_b64 s[12:13], s[6:7], s12
+; GFX6-NEXT: s_or_b64 s[8:9], s[8:9], s[12:13]
+; GFX6-NEXT: s_lshr_b64 s[6:7], s[6:7], s14
+; GFX6-NEXT: s_cmp_lg_u32 s15, 0
+; GFX6-NEXT: s_cselect_b64 s[6:7], s[8:9], s[6:7]
+; GFX6-NEXT: s_cmp_lg_u32 s16, 0
+; GFX6-NEXT: s_cselect_b64 s[4:5], s[4:5], s[6:7]
+; GFX6-NEXT: s_cmp_lg_u32 s15, 0
+; GFX6-NEXT: s_cselect_b64 s[6:7], s[10:11], 0
+; GFX6-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
+; GFX6-NEXT: s_or_b64 s[2:3], s[2:3], s[6:7]
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_fshr_i128:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_and_b32 s10, s8, 64
-; GFX8-NEXT: s_mov_b32 s11, 0
-; GFX8-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX8-NEXT: s_cselect_b32 s3, s1, s3
-; GFX8-NEXT: s_cselect_b32 s2, s0, s2
-; GFX8-NEXT: s_cselect_b32 s1, s7, s1
-; GFX8-NEXT: s_cselect_b32 s0, s6, s0
-; GFX8-NEXT: s_cselect_b32 s4, s4, s6
-; GFX8-NEXT: s_cselect_b32 s5, s5, s7
-; GFX8-NEXT: s_lshr_b64 s[6:7], s[0:1], s8
+; GFX8-NEXT: s_lshl_b64 s[10:11], s[0:1], 1
; GFX8-NEXT: s_lshl_b64 s[2:3], s[2:3], 1
+; GFX8-NEXT: s_lshr_b32 s0, s1, 31
+; GFX8-NEXT: s_or_b32 s2, s2, s0
+; GFX8-NEXT: s_andn2_b32 s0, 0x7f, s8
; GFX8-NEXT: s_not_b32 s9, s8
-; GFX8-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
-; GFX8-NEXT: s_lshl_b64 s[2:3], s[2:3], s9
-; GFX8-NEXT: s_lshr_b64 s[4:5], s[4:5], s8
-; GFX8-NEXT: s_lshl_b64 s[0:1], s[0:1], s9
-; GFX8-NEXT: s_or_b64 s[2:3], s[2:3], s[6:7]
+; GFX8-NEXT: s_sub_i32 s16, s0, 64
+; GFX8-NEXT: s_sub_i32 s12, 64, s0
+; GFX8-NEXT: s_cmp_lt_u32 s0, 64
+; GFX8-NEXT: s_cselect_b32 s17, 1, 0
+; GFX8-NEXT: s_cmp_eq_u32 s0, 0
+; GFX8-NEXT: s_cselect_b32 s18, 1, 0
+; GFX8-NEXT: s_lshr_b64 s[12:13], s[10:11], s12
+; GFX8-NEXT: s_lshl_b64 s[14:15], s[2:3], s9
+; GFX8-NEXT: s_lshl_b64 s[0:1], s[10:11], s9
+; GFX8-NEXT: s_or_b64 s[12:13], s[12:13], s[14:15]
+; GFX8-NEXT: s_lshl_b64 s[10:11], s[10:11], s16
+; GFX8-NEXT: s_cmp_lg_u32 s17, 0
+; GFX8-NEXT: s_cselect_b64 s[0:1], s[0:1], 0
+; GFX8-NEXT: s_cselect_b64 s[10:11], s[12:13], s[10:11]
+; GFX8-NEXT: s_cmp_lg_u32 s18, 0
+; GFX8-NEXT: s_cselect_b64 s[2:3], s[2:3], s[10:11]
+; GFX8-NEXT: s_and_b32 s9, s8, 0x7f
+; GFX8-NEXT: s_sub_i32 s14, s9, 64
+; GFX8-NEXT: s_sub_i32 s12, 64, s9
+; GFX8-NEXT: s_cmp_lt_u32 s9, 64
+; GFX8-NEXT: s_cselect_b32 s15, 1, 0
+; GFX8-NEXT: s_cmp_eq_u32 s9, 0
+; GFX8-NEXT: s_cselect_b32 s16, 1, 0
+; GFX8-NEXT: s_lshr_b64 s[10:11], s[6:7], s8
+; GFX8-NEXT: s_lshr_b64 s[8:9], s[4:5], s8
+; GFX8-NEXT: s_lshl_b64 s[12:13], s[6:7], s12
+; GFX8-NEXT: s_or_b64 s[8:9], s[8:9], s[12:13]
+; GFX8-NEXT: s_lshr_b64 s[6:7], s[6:7], s14
+; GFX8-NEXT: s_cmp_lg_u32 s15, 0
+; GFX8-NEXT: s_cselect_b64 s[6:7], s[8:9], s[6:7]
+; GFX8-NEXT: s_cmp_lg_u32 s16, 0
+; GFX8-NEXT: s_cselect_b64 s[4:5], s[4:5], s[6:7]
+; GFX8-NEXT: s_cmp_lg_u32 s15, 0
+; GFX8-NEXT: s_cselect_b64 s[6:7], s[10:11], 0
; GFX8-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
+; GFX8-NEXT: s_or_b64 s[2:3], s[2:3], s[6:7]
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshr_i128:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s10, s8, 64
-; GFX11-NEXT: s_mov_b32 s11, 0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_cmp_eq_u64 s[10:11], 0
-; GFX11-NEXT: s_cselect_b32 s3, s1, s3
-; GFX11-NEXT: s_cselect_b32 s2, s0, s2
-; GFX11-NEXT: s_cselect_b32 s1, s7, s1
-; GFX11-NEXT: s_cselect_b32 s0, s6, s0
-; GFX11-NEXT: s_cselect_b32 s4, s4, s6
-; GFX11-NEXT: s_cselect_b32 s5, s5, s7
-; GFX11-NEXT: s_lshr_b64 s[6:7], s[0:1], s8
-; GFX11-NEXT: s_lshl_b64 s[2:3], s[2:3], 1
-; GFX11-NEXT: s_not_b32 s9, s8
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
-; GFX11-NEXT: s_lshr_b64 s[4:5], s[4:5], s8
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], s9
-; GFX11-NEXT: s_lshl_b64 s[2:3], s[2:3], s9
-; GFX11-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
-; GFX11-NEXT: s_or_b64 s[2:3], s[2:3], s[6:7]
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshr_i128:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_lshl_b64 s[10:11], s[0:1], 1
@@ -5838,6 +6073,7 @@ define amdgpu_ps i128 @s_fshr_i128(i128 inreg %lhs, i128 inreg %rhs, i128 inreg
; GFX9-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
; GFX9-NEXT: s_or_b64 s[2:3], s[2:3], s[6:7]
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: s_fshr_i128:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_lshl_b64 s[2:3], s[2:3], 1
@@ -5883,6 +6119,53 @@ define amdgpu_ps i128 @s_fshr_i128(i128 inreg %lhs, i128 inreg %rhs, i128 inreg
; GFX10-NEXT: s_or_b64 s[0:1], s[12:13], s[0:1]
; GFX10-NEXT: s_or_b64 s[2:3], s[2:3], s[4:5]
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: s_fshr_i128:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_lshl_b64 s[2:3], s[2:3], 1
+; GFX11-NEXT: s_lshr_b32 s9, s1, 31
+; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
+; GFX11-NEXT: s_or_b32 s2, s2, s9
+; GFX11-NEXT: s_and_not1_b32 s9, 0x7f, s8
+; GFX11-NEXT: s_not_b32 s14, s8
+; GFX11-NEXT: s_sub_i32 s16, s9, 64
+; GFX11-NEXT: s_sub_i32 s10, 64, s9
+; GFX11-NEXT: s_cmp_lt_u32 s9, 64
+; GFX11-NEXT: s_cselect_b32 s17, 1, 0
+; GFX11-NEXT: s_cmp_eq_u32 s9, 0
+; GFX11-NEXT: s_cselect_b32 s9, 1, 0
+; GFX11-NEXT: s_lshr_b64 s[10:11], s[0:1], s10
+; GFX11-NEXT: s_lshl_b64 s[12:13], s[2:3], s14
+; GFX11-NEXT: s_lshl_b64 s[14:15], s[0:1], s14
+; GFX11-NEXT: s_or_b64 s[10:11], s[10:11], s[12:13]
+; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], s16
+; GFX11-NEXT: s_cmp_lg_u32 s17, 0
+; GFX11-NEXT: s_cselect_b64 s[12:13], s[14:15], 0
+; GFX11-NEXT: s_cselect_b64 s[0:1], s[10:11], s[0:1]
+; GFX11-NEXT: s_cmp_lg_u32 s9, 0
+; GFX11-NEXT: s_cselect_b64 s[2:3], s[2:3], s[0:1]
+; GFX11-NEXT: s_and_b32 s0, s8, 0x7f
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_sub_i32 s14, s0, 64
+; GFX11-NEXT: s_sub_i32 s9, 64, s0
+; GFX11-NEXT: s_cmp_lt_u32 s0, 64
+; GFX11-NEXT: s_cselect_b32 s15, 1, 0
+; GFX11-NEXT: s_cmp_eq_u32 s0, 0
+; GFX11-NEXT: s_cselect_b32 s16, 1, 0
+; GFX11-NEXT: s_lshr_b64 s[0:1], s[4:5], s8
+; GFX11-NEXT: s_lshl_b64 s[10:11], s[6:7], s9
+; GFX11-NEXT: s_lshr_b64 s[8:9], s[6:7], s8
+; GFX11-NEXT: s_or_b64 s[0:1], s[0:1], s[10:11]
+; GFX11-NEXT: s_lshr_b64 s[6:7], s[6:7], s14
+; GFX11-NEXT: s_cmp_lg_u32 s15, 0
+; GFX11-NEXT: s_cselect_b64 s[0:1], s[0:1], s[6:7]
+; GFX11-NEXT: s_cmp_lg_u32 s16, 0
+; GFX11-NEXT: s_cselect_b64 s[0:1], s[4:5], s[0:1]
+; GFX11-NEXT: s_cmp_lg_u32 s15, 0
+; GFX11-NEXT: s_cselect_b64 s[4:5], s[8:9], 0
+; GFX11-NEXT: s_or_b64 s[0:1], s[12:13], s[0:1]
+; GFX11-NEXT: s_or_b64 s[2:3], s[2:3], s[4:5]
+; GFX11-NEXT: ; return to shader part epilog
%result = call i128 @llvm.fshr.i128(i128 %lhs, i128 %rhs, i128 %amt)
ret i128 %result
}
@@ -5891,80 +6174,101 @@ define i128 @v_fshr_i128(i128 %lhs, i128 %rhs, i128 %amt) {
; GFX6-LABEL: v_fshr_i128:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v9, 64, v8
-; GFX6-NEXT: v_mov_b32_e32 v10, 0
-; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[9:10]
-; GFX6-NEXT: v_not_b32_e32 v13, v8
-; GFX6-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v7, v1, v7, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v6, v0, v6, vcc
-; GFX6-NEXT: v_lshr_b64 v[9:10], v[4:5], v8
-; GFX6-NEXT: v_lshl_b64 v[4:5], v[6:7], 1
-; GFX6-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX6-NEXT: v_lshl_b64 v[0:1], v[0:1], 1
-; GFX6-NEXT: v_lshl_b64 v[11:12], v[4:5], v13
-; GFX6-NEXT: v_lshr_b64 v[2:3], v[6:7], v8
-; GFX6-NEXT: v_lshl_b64 v[5:6], v[0:1], v13
-; GFX6-NEXT: v_or_b32_e32 v4, v12, v10
-; GFX6-NEXT: v_or_b32_e32 v0, v11, v9
-; GFX6-NEXT: v_or_b32_e32 v3, v6, v3
-; GFX6-NEXT: v_or_b32_e32 v2, v5, v2
-; GFX6-NEXT: v_mov_b32_e32 v1, v4
+; GFX6-NEXT: v_lshl_b64 v[2:3], v[2:3], 1
+; GFX6-NEXT: v_lshl_b64 v[9:10], v[0:1], 1
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, 31, v1
+; GFX6-NEXT: v_or_b32_e32 v2, v2, v0
+; GFX6-NEXT: v_mov_b32_e32 v0, 0x7f
+; GFX6-NEXT: v_bfi_b32 v15, v8, 0, v0
+; GFX6-NEXT: v_sub_i32_e32 v0, vcc, 64, v15
+; GFX6-NEXT: v_not_b32_e32 v16, 63
+; GFX6-NEXT: v_lshr_b64 v[0:1], v[9:10], v0
+; GFX6-NEXT: v_lshl_b64 v[11:12], v[2:3], v15
+; GFX6-NEXT: v_add_i32_e32 v17, vcc, v15, v16
+; GFX6-NEXT: v_lshl_b64 v[13:14], v[9:10], v15
+; GFX6-NEXT: v_or_b32_e32 v11, v0, v11
+; GFX6-NEXT: v_or_b32_e32 v12, v1, v12
+; GFX6-NEXT: v_lshl_b64 v[0:1], v[9:10], v17
+; GFX6-NEXT: v_cmp_gt_u32_e32 vcc, 64, v15
+; GFX6-NEXT: v_cndmask_b32_e32 v10, 0, v13, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v13, 0, v14, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v11, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v1, v1, v12, vcc
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v15
+; GFX6-NEXT: v_and_b32_e32 v14, 0x7f, v8
+; GFX6-NEXT: v_cndmask_b32_e32 v11, v0, v2, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v12, v1, v3, vcc
+; GFX6-NEXT: v_sub_i32_e32 v2, vcc, 64, v14
+; GFX6-NEXT: v_lshr_b64 v[0:1], v[4:5], v14
+; GFX6-NEXT: v_lshl_b64 v[2:3], v[6:7], v2
+; GFX6-NEXT: v_add_i32_e32 v15, vcc, v14, v16
+; GFX6-NEXT: v_or_b32_e32 v2, v0, v2
+; GFX6-NEXT: v_or_b32_e32 v3, v1, v3
+; GFX6-NEXT: v_lshr_b64 v[0:1], v[6:7], v15
+; GFX6-NEXT: v_lshr_b64 v[8:9], v[6:7], v14
+; GFX6-NEXT: v_cmp_gt_u32_e32 vcc, 64, v14
+; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX6-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v14
+; GFX6-NEXT: v_cndmask_b32_e64 v0, v0, v4, s[4:5]
+; GFX6-NEXT: v_cndmask_b32_e64 v1, v1, v5, s[4:5]
+; GFX6-NEXT: v_cndmask_b32_e32 v2, 0, v8, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v3, 0, v9, vcc
+; GFX6-NEXT: v_or_b32_e32 v0, v10, v0
+; GFX6-NEXT: v_or_b32_e32 v1, v13, v1
+; GFX6-NEXT: v_or_b32_e32 v2, v11, v2
+; GFX6-NEXT: v_or_b32_e32 v3, v12, v3
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_fshr_i128:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_and_b32_e32 v9, 64, v8
-; GFX8-NEXT: v_mov_b32_e32 v10, 0
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[9:10]
-; GFX8-NEXT: v_not_b32_e32 v13, v8
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v7, v1, v7, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v0, v6, vcc
-; GFX8-NEXT: v_lshrrev_b64 v[9:10], v8, v[4:5]
-; GFX8-NEXT: v_lshlrev_b64 v[4:5], 1, v[6:7]
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX8-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
-; GFX8-NEXT: v_lshlrev_b64 v[11:12], v13, v[4:5]
-; GFX8-NEXT: v_lshrrev_b64 v[2:3], v8, v[6:7]
-; GFX8-NEXT: v_lshlrev_b64 v[5:6], v13, v[0:1]
-; GFX8-NEXT: v_or_b32_e32 v4, v12, v10
-; GFX8-NEXT: v_or_b32_e32 v0, v11, v9
-; GFX8-NEXT: v_or_b32_e32 v3, v6, v3
-; GFX8-NEXT: v_or_b32_e32 v2, v5, v2
-; GFX8-NEXT: v_mov_b32_e32 v1, v4
+; GFX8-NEXT: v_lshlrev_b64 v[2:3], 1, v[2:3]
+; GFX8-NEXT: v_lshlrev_b64 v[9:10], 1, v[0:1]
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, 31, v1
+; GFX8-NEXT: v_or_b32_e32 v2, v2, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, 0x7f
+; GFX8-NEXT: v_bfi_b32 v15, v8, 0, v0
+; GFX8-NEXT: v_sub_u32_e32 v0, vcc, 64, v15
+; GFX8-NEXT: v_not_b32_e32 v16, 63
+; GFX8-NEXT: v_lshrrev_b64 v[0:1], v0, v[9:10]
+; GFX8-NEXT: v_lshlrev_b64 v[11:12], v15, v[2:3]
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, v15, v16
+; GFX8-NEXT: v_lshlrev_b64 v[13:14], v15, v[9:10]
+; GFX8-NEXT: v_or_b32_e32 v11, v0, v11
+; GFX8-NEXT: v_or_b32_e32 v12, v1, v12
+; GFX8-NEXT: v_lshlrev_b64 v[0:1], v17, v[9:10]
+; GFX8-NEXT: v_cmp_gt_u32_e32 vcc, 64, v15
+; GFX8-NEXT: v_cndmask_b32_e32 v10, 0, v13, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v13, 0, v14, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v11, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v12, vcc
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v15
+; GFX8-NEXT: v_and_b32_e32 v14, 0x7f, v8
+; GFX8-NEXT: v_cndmask_b32_e32 v11, v0, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v12, v1, v3, vcc
+; GFX8-NEXT: v_sub_u32_e32 v2, vcc, 64, v14
+; GFX8-NEXT: v_lshrrev_b64 v[0:1], v14, v[4:5]
+; GFX8-NEXT: v_lshlrev_b64 v[2:3], v2, v[6:7]
+; GFX8-NEXT: v_add_u32_e32 v15, vcc, v14, v16
+; GFX8-NEXT: v_or_b32_e32 v2, v0, v2
+; GFX8-NEXT: v_or_b32_e32 v3, v1, v3
+; GFX8-NEXT: v_lshrrev_b64 v[0:1], v15, v[6:7]
+; GFX8-NEXT: v_lshrrev_b64 v[8:9], v14, v[6:7]
+; GFX8-NEXT: v_cmp_gt_u32_e32 vcc, 64, v14
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v14
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v0, v4, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e64 v1, v1, v5, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e32 v2, 0, v8, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v3, 0, v9, vcc
+; GFX8-NEXT: v_or_b32_e32 v0, v10, v0
+; GFX8-NEXT: v_or_b32_e32 v1, v13, v1
+; GFX8-NEXT: v_or_b32_e32 v2, v11, v2
+; GFX8-NEXT: v_or_b32_e32 v3, v12, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshr_i128:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_and_b32 v9, 64, v8
-; GFX11-NEXT: v_not_b32_e32 v11, v8
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[9:10]
-; GFX11-NEXT: v_dual_cndmask_b32 v5, v7, v5 :: v_dual_cndmask_b32 v10, v1, v7
-; GFX11-NEXT: v_dual_cndmask_b32 v9, v0, v6 :: v_dual_cndmask_b32 v0, v2, v0
-; GFX11-NEXT: v_dual_cndmask_b32 v1, v3, v1 :: v_dual_cndmask_b32 v4, v6, v4
-; GFX11-NEXT: v_lshlrev_b64 v[2:3], 1, v[9:10]
-; GFX11-NEXT: v_lshrrev_b64 v[6:7], v8, v[9:10]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
-; GFX11-NEXT: v_lshrrev_b64 v[4:5], v8, v[4:5]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_lshlrev_b64 v[2:3], v11, v[2:3]
-; GFX11-NEXT: v_lshlrev_b64 v[8:9], v11, v[0:1]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_or_b32_e32 v0, v2, v4
-; GFX11-NEXT: v_or_b32_e32 v1, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v2, v8, v6
-; GFX11-NEXT: v_or_b32_e32 v3, v9, v7
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshr_i128:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -6012,6 +6316,7 @@ define i128 @v_fshr_i128(i128 %lhs, i128 %rhs, i128 %amt) {
; GFX9-NEXT: v_or_b32_e32 v2, v11, v2
; GFX9-NEXT: v_or_b32_e32 v3, v12, v3
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshr_i128:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -6058,6 +6363,61 @@ define i128 @v_fshr_i128(i128 %lhs, i128 %rhs, i128 %amt) {
; GFX10-NEXT: v_or_b32_e32 v2, v2, v6
; GFX10-NEXT: v_or_b32_e32 v3, v3, v8
; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_fshr_i128:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b64 v[2:3], 1, v[2:3]
+; GFX11-NEXT: v_bfi_b32 v18, v8, 0, 0x7f
+; GFX11-NEXT: v_lshrrev_b32_e32 v9, 31, v1
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
+; GFX11-NEXT: v_and_b32_e32 v19, 0x7f, v8
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_sub_nc_u32_e32 v10, 64, v18
+; GFX11-NEXT: v_or_b32_e32 v2, v2, v9
+; GFX11-NEXT: v_cmp_gt_u32_e32 vcc_lo, 64, v18
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT: v_sub_nc_u32_e32 v16, 64, v19
+; GFX11-NEXT: v_lshlrev_b64 v[12:13], v18, v[0:1]
+; GFX11-NEXT: v_lshrrev_b64 v[8:9], v10, v[0:1]
+; GFX11-NEXT: v_lshlrev_b64 v[10:11], v18, v[2:3]
+; GFX11-NEXT: v_cmp_gt_u32_e64 s1, 64, v19
+; GFX11-NEXT: v_lshlrev_b64 v[16:17], v16, v[6:7]
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 0, v18
+; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 0, v19
+; GFX11-NEXT: v_cndmask_b32_e32 v12, 0, v12, vcc_lo
+; GFX11-NEXT: v_or_b32_e32 v11, v9, v11
+; GFX11-NEXT: v_add_nc_u32_e32 v14, 0xffffffc0, v18
+; GFX11-NEXT: v_or_b32_e32 v8, v8, v10
+; GFX11-NEXT: v_add_nc_u32_e32 v10, 0xffffffc0, v19
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], v14, v[0:1]
+; GFX11-NEXT: v_lshrrev_b64 v[14:15], v19, v[4:5]
+; GFX11-NEXT: v_cndmask_b32_e32 v20, v0, v8, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_lshrrev_b64 v[8:9], v10, v[6:7]
+; GFX11-NEXT: v_or_b32_e32 v0, v14, v16
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v10, v15, v17
+; GFX11-NEXT: v_cndmask_b32_e32 v11, v1, v11, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v20, v2, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v8, v8, v0, s1
+; GFX11-NEXT: v_lshrrev_b64 v[0:1], v19, v[6:7]
+; GFX11-NEXT: v_cndmask_b32_e64 v6, v9, v10, s1
+; GFX11-NEXT: v_cndmask_b32_e32 v7, 0, v13, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e64 v3, v11, v3, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v4, v8, v4, s2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_cndmask_b32_e64 v5, v6, v5, s2
+; GFX11-NEXT: v_cndmask_b32_e64 v6, 0, v0, s1
+; GFX11-NEXT: v_cndmask_b32_e64 v8, 0, v1, s1
+; GFX11-NEXT: v_or_b32_e32 v0, v12, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v1, v7, v5
+; GFX11-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v3, v3, v8
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call i128 @llvm.fshr.i128(i128 %lhs, i128 %rhs, i128 %amt)
ret i128 %result
}
@@ -6065,100 +6425,108 @@ define i128 @v_fshr_i128(i128 %lhs, i128 %rhs, i128 %amt) {
define amdgpu_ps <4 x float> @v_fshr_i128_ssv(i128 inreg %lhs, i128 inreg %rhs, i128 %amt) {
; GFX6-LABEL: v_fshr_i128_ssv:
; GFX6: ; %bb.0:
-; GFX6-NEXT: v_and_b32_e32 v1, 64, v0
-; GFX6-NEXT: v_mov_b32_e32 v2, 0
-; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[1:2]
-; GFX6-NEXT: v_mov_b32_e32 v4, s7
-; GFX6-NEXT: v_mov_b32_e32 v3, s5
-; GFX6-NEXT: v_mov_b32_e32 v6, s6
-; GFX6-NEXT: v_mov_b32_e32 v8, s1
-; GFX6-NEXT: v_mov_b32_e32 v9, s0
-; GFX6-NEXT: v_cndmask_b32_e32 v2, v4, v3, vcc
-; GFX6-NEXT: v_mov_b32_e32 v1, s4
-; GFX6-NEXT: v_cndmask_b32_e32 v5, v8, v4, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v4, v9, v6, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v1, v6, v1, vcc
-; GFX6-NEXT: v_lshl_b64 v[6:7], v[4:5], 1
-; GFX6-NEXT: v_not_b32_e32 v10, v0
-; GFX6-NEXT: v_lshr_b64 v[2:3], v[1:2], v0
-; GFX6-NEXT: v_lshl_b64 v[6:7], v[6:7], v10
-; GFX6-NEXT: v_lshr_b64 v[4:5], v[4:5], v0
-; GFX6-NEXT: v_or_b32_e32 v1, v7, v3
-; GFX6-NEXT: v_mov_b32_e32 v3, s3
-; GFX6-NEXT: v_cndmask_b32_e32 v8, v3, v8, vcc
+; GFX6-NEXT: v_mov_b32_e32 v1, 0x7f
+; GFX6-NEXT: s_lshl_b64 s[8:9], s[0:1], 1
+; GFX6-NEXT: s_lshl_b64 s[2:3], s[2:3], 1
+; GFX6-NEXT: s_lshr_b32 s0, s1, 31
+; GFX6-NEXT: v_bfi_b32 v7, v0, 0, v1
+; GFX6-NEXT: s_or_b32 s2, s2, s0
+; GFX6-NEXT: v_sub_i32_e32 v1, vcc, 64, v7
+; GFX6-NEXT: v_not_b32_e32 v8, 63
+; GFX6-NEXT: v_lshr_b64 v[1:2], s[8:9], v1
+; GFX6-NEXT: v_lshl_b64 v[3:4], s[2:3], v7
+; GFX6-NEXT: v_add_i32_e32 v9, vcc, v7, v8
+; GFX6-NEXT: v_lshl_b64 v[5:6], s[8:9], v7
+; GFX6-NEXT: v_or_b32_e32 v3, v1, v3
+; GFX6-NEXT: v_or_b32_e32 v4, v2, v4
+; GFX6-NEXT: v_lshl_b64 v[1:2], s[8:9], v9
+; GFX6-NEXT: v_cmp_gt_u32_e32 vcc, 64, v7
+; GFX6-NEXT: v_cndmask_b32_e32 v9, 0, v5, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v6, 0, v6, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
; GFX6-NEXT: v_mov_b32_e32 v3, s2
-; GFX6-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
-; GFX6-NEXT: v_lshl_b64 v[7:8], v[7:8], 1
-; GFX6-NEXT: v_or_b32_e32 v0, v6, v2
-; GFX6-NEXT: v_lshl_b64 v[7:8], v[7:8], v10
-; GFX6-NEXT: v_or_b32_e32 v3, v8, v5
-; GFX6-NEXT: v_or_b32_e32 v2, v7, v4
+; GFX6-NEXT: v_mov_b32_e32 v4, s3
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
+; GFX6-NEXT: v_and_b32_e32 v11, 0x7f, v0
+; GFX6-NEXT: v_cndmask_b32_e32 v7, v1, v3, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v10, v2, v4, vcc
+; GFX6-NEXT: v_sub_i32_e32 v2, vcc, 64, v11
+; GFX6-NEXT: v_lshr_b64 v[0:1], s[4:5], v11
+; GFX6-NEXT: v_lshl_b64 v[2:3], s[6:7], v2
+; GFX6-NEXT: v_add_i32_e32 v8, vcc, v11, v8
+; GFX6-NEXT: v_or_b32_e32 v2, v0, v2
+; GFX6-NEXT: v_or_b32_e32 v3, v1, v3
+; GFX6-NEXT: v_lshr_b64 v[0:1], s[6:7], v8
+; GFX6-NEXT: v_lshr_b64 v[4:5], s[6:7], v11
+; GFX6-NEXT: v_cmp_gt_u32_e32 vcc, 64, v11
+; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX6-NEXT: v_mov_b32_e32 v2, s4
+; GFX6-NEXT: v_mov_b32_e32 v3, s5
+; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v11
+; GFX6-NEXT: v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; GFX6-NEXT: v_cndmask_b32_e64 v1, v1, v3, s[0:1]
+; GFX6-NEXT: v_cndmask_b32_e32 v2, 0, v4, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v3, 0, v5, vcc
+; GFX6-NEXT: v_or_b32_e32 v0, v9, v0
+; GFX6-NEXT: v_or_b32_e32 v1, v6, v1
+; GFX6-NEXT: v_or_b32_e32 v2, v7, v2
+; GFX6-NEXT: v_or_b32_e32 v3, v10, v3
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: v_fshr_i128_ssv:
; GFX8: ; %bb.0:
-; GFX8-NEXT: v_and_b32_e32 v1, 64, v0
-; GFX8-NEXT: v_mov_b32_e32 v2, 0
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[1:2]
-; GFX8-NEXT: v_mov_b32_e32 v4, s7
-; GFX8-NEXT: v_mov_b32_e32 v3, s5
-; GFX8-NEXT: v_mov_b32_e32 v6, s6
-; GFX8-NEXT: v_mov_b32_e32 v8, s1
-; GFX8-NEXT: v_mov_b32_e32 v9, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v4, v3, vcc
-; GFX8-NEXT: v_mov_b32_e32 v1, s4
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v4, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v9, v6, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v1, vcc
-; GFX8-NEXT: v_lshlrev_b64 v[6:7], 1, v[4:5]
-; GFX8-NEXT: v_not_b32_e32 v10, v0
-; GFX8-NEXT: v_lshrrev_b64 v[2:3], v0, v[1:2]
-; GFX8-NEXT: v_lshlrev_b64 v[6:7], v10, v[6:7]
-; GFX8-NEXT: v_lshrrev_b64 v[4:5], v0, v[4:5]
-; GFX8-NEXT: v_or_b32_e32 v1, v7, v3
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: v_cndmask_b32_e32 v8, v3, v8, vcc
+; GFX8-NEXT: v_mov_b32_e32 v1, 0x7f
+; GFX8-NEXT: s_lshl_b64 s[8:9], s[0:1], 1
+; GFX8-NEXT: s_lshl_b64 s[2:3], s[2:3], 1
+; GFX8-NEXT: s_lshr_b32 s0, s1, 31
+; GFX8-NEXT: v_bfi_b32 v7, v0, 0, v1
+; GFX8-NEXT: s_or_b32 s2, s2, s0
+; GFX8-NEXT: v_sub_u32_e32 v1, vcc, 64, v7
+; GFX8-NEXT: v_not_b32_e32 v8, 63
+; GFX8-NEXT: v_lshrrev_b64 v[1:2], v1, s[8:9]
+; GFX8-NEXT: v_lshlrev_b64 v[3:4], v7, s[2:3]
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v7, v8
+; GFX8-NEXT: v_lshlrev_b64 v[5:6], v7, s[8:9]
+; GFX8-NEXT: v_or_b32_e32 v3, v1, v3
+; GFX8-NEXT: v_or_b32_e32 v4, v2, v4
+; GFX8-NEXT: v_lshlrev_b64 v[1:2], v9, s[8:9]
+; GFX8-NEXT: v_cmp_gt_u32_e32 vcc, 64, v7
+; GFX8-NEXT: v_cndmask_b32_e32 v9, 0, v5, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v6, 0, v6, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
; GFX8-NEXT: v_mov_b32_e32 v3, s2
-; GFX8-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
-; GFX8-NEXT: v_lshlrev_b64 v[7:8], 1, v[7:8]
-; GFX8-NEXT: v_or_b32_e32 v0, v6, v2
-; GFX8-NEXT: v_lshlrev_b64 v[7:8], v10, v[7:8]
-; GFX8-NEXT: v_or_b32_e32 v3, v8, v5
-; GFX8-NEXT: v_or_b32_e32 v2, v7, v4
+; GFX8-NEXT: v_mov_b32_e32 v4, s3
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
+; GFX8-NEXT: v_and_b32_e32 v11, 0x7f, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v1, v3, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v10, v2, v4, vcc
+; GFX8-NEXT: v_sub_u32_e32 v2, vcc, 64, v11
+; GFX8-NEXT: v_lshrrev_b64 v[0:1], v11, s[4:5]
+; GFX8-NEXT: v_lshlrev_b64 v[2:3], v2, s[6:7]
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, v11, v8
+; GFX8-NEXT: v_or_b32_e32 v2, v0, v2
+; GFX8-NEXT: v_or_b32_e32 v3, v1, v3
+; GFX8-NEXT: v_lshrrev_b64 v[0:1], v8, s[6:7]
+; GFX8-NEXT: v_lshrrev_b64 v[4:5], v11, s[6:7]
+; GFX8-NEXT: v_cmp_gt_u32_e32 vcc, 64, v11
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX8-NEXT: v_mov_b32_e32 v2, s4
+; GFX8-NEXT: v_mov_b32_e32 v3, s5
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v11
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; GFX8-NEXT: v_cndmask_b32_e64 v1, v1, v3, s[0:1]
+; GFX8-NEXT: v_cndmask_b32_e32 v2, 0, v4, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v3, 0, v5, vcc
+; GFX8-NEXT: v_or_b32_e32 v0, v9, v0
+; GFX8-NEXT: v_or_b32_e32 v1, v6, v1
+; GFX8-NEXT: v_or_b32_e32 v2, v7, v2
+; GFX8-NEXT: v_or_b32_e32 v3, v10, v3
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: v_fshr_i128_ssv:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v1, 64, v0
-; GFX11-NEXT: v_dual_mov_b32 v3, s5 :: v_dual_mov_b32 v8, s4
-; GFX11-NEXT: v_mov_b32_e32 v7, s0
-; GFX11-NEXT: v_not_b32_e32 v9, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[1:2]
-; GFX11-NEXT: v_mov_b32_e32 v1, s7
-; GFX11-NEXT: v_dual_mov_b32 v5, s1 :: v_dual_cndmask_b32 v4, s1, v1
-; GFX11-NEXT: v_cndmask_b32_e32 v2, s7, v3, vcc_lo
-; GFX11-NEXT: v_mov_b32_e32 v3, s6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_cndmask_b32_e32 v6, s3, v5, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e32 v5, s2, v7, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e32 v1, s6, v8, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e32 v3, s0, v3, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_lshlrev_b64 v[5:6], 1, v[5:6]
-; GFX11-NEXT: v_lshrrev_b64 v[1:2], v0, v[1:2]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_lshlrev_b64 v[7:8], 1, v[3:4]
-; GFX11-NEXT: v_lshrrev_b64 v[3:4], v0, v[3:4]
-; GFX11-NEXT: v_lshlrev_b64 v[5:6], v9, v[5:6]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b64 v[7:8], v9, v[7:8]
-; GFX11-NEXT: v_or_b32_e32 v0, v7, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v1, v8, v2
-; GFX11-NEXT: v_or_b32_e32 v2, v5, v3
-; GFX11-NEXT: v_or_b32_e32 v3, v6, v4
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshr_i128_ssv:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_mov_b32_e32 v1, 0x7f
@@ -6209,6 +6577,7 @@ define amdgpu_ps <4 x float> @v_fshr_i128_ssv(i128 inreg %lhs, i128 inreg %rhs,
; GFX9-NEXT: v_or_b32_e32 v2, v7, v2
; GFX9-NEXT: v_or_b32_e32 v3, v9, v3
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: v_fshr_i128_ssv:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_bfi_b32 v11, v0, 0, 0x7f
@@ -6254,6 +6623,56 @@ define amdgpu_ps <4 x float> @v_fshr_i128_ssv(i128 inreg %lhs, i128 inreg %rhs,
; GFX10-NEXT: v_or_b32_e32 v2, v7, v8
; GFX10-NEXT: v_or_b32_e32 v3, v4, v9
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: v_fshr_i128_ssv:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_bfi_b32 v11, v0, 0, 0x7f
+; GFX11-NEXT: s_lshl_b64 s[8:9], s[2:3], 1
+; GFX11-NEXT: s_lshr_b32 s2, s1, 31
+; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
+; GFX11-NEXT: s_or_b32 s8, s8, s2
+; GFX11-NEXT: v_sub_nc_u32_e32 v1, 64, v11
+; GFX11-NEXT: v_lshlrev_b64 v[5:6], v11, s[0:1]
+; GFX11-NEXT: v_cmp_gt_u32_e32 vcc_lo, 64, v11
+; GFX11-NEXT: v_and_b32_e32 v12, 0x7f, v0
+; GFX11-NEXT: v_lshlrev_b64 v[3:4], v11, s[8:9]
+; GFX11-NEXT: v_lshrrev_b64 v[1:2], v1, s[0:1]
+; GFX11-NEXT: v_dual_cndmask_b32 v5, 0, v5 :: v_dual_add_nc_u32 v0, 0xffffffc0, v11
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT: v_sub_nc_u32_e32 v9, 64, v12
+; GFX11-NEXT: v_lshrrev_b64 v[7:8], v12, s[4:5]
+; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 0, v12
+; GFX11-NEXT: v_or_b32_e32 v3, v1, v3
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], v0, s[0:1]
+; GFX11-NEXT: v_add_nc_u32_e32 v13, 0xffffffc0, v12
+; GFX11-NEXT: v_lshlrev_b64 v[9:10], v9, s[6:7]
+; GFX11-NEXT: v_or_b32_e32 v4, v2, v4
+; GFX11-NEXT: v_cmp_gt_u32_e64 s1, 64, v12
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 0, v11
+; GFX11-NEXT: v_cndmask_b32_e32 v14, v0, v3, vcc_lo
+; GFX11-NEXT: v_lshrrev_b64 v[2:3], v13, s[6:7]
+; GFX11-NEXT: v_or_b32_e32 v0, v7, v9
+; GFX11-NEXT: v_or_b32_e32 v7, v8, v10
+; GFX11-NEXT: v_cndmask_b32_e32 v4, v1, v4, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e32 v6, 0, v6, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, v0, s1
+; GFX11-NEXT: v_lshrrev_b64 v[0:1], v12, s[6:7]
+; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, v7, s1
+; GFX11-NEXT: v_cndmask_b32_e64 v7, v14, s8, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v4, v4, s9, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, s4, s2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, s5, s2
+; GFX11-NEXT: v_cndmask_b32_e64 v8, 0, v0, s1
+; GFX11-NEXT: v_cndmask_b32_e64 v9, 0, v1, s1
+; GFX11-NEXT: v_or_b32_e32 v0, v5, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v1, v6, v3
+; GFX11-NEXT: v_or_b32_e32 v2, v7, v8
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v3, v4, v9
+; GFX11-NEXT: ; return to shader part epilog
%result = call i128 @llvm.fshr.i128(i128 %lhs, i128 %rhs, i128 %amt)
%cast.result = bitcast i128 %result to <4 x float>
ret <4 x float> %cast.result
@@ -6262,85 +6681,122 @@ define amdgpu_ps <4 x float> @v_fshr_i128_ssv(i128 inreg %lhs, i128 inreg %rhs,
define amdgpu_ps <4 x float> @v_fshr_i128_svs(i128 inreg %lhs, i128 %rhs, i128 inreg %amt) {
; GFX6-LABEL: v_fshr_i128_svs:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_and_b32 s6, s4, 64
-; GFX6-NEXT: s_mov_b32 s7, 0
-; GFX6-NEXT: v_cmp_eq_u64_e64 vcc, s[6:7], 0
-; GFX6-NEXT: v_mov_b32_e32 v4, s1
-; GFX6-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
-; GFX6-NEXT: v_mov_b32_e32 v4, s0
-; GFX6-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX6-NEXT: s_not_b32 s6, s4
-; GFX6-NEXT: v_lshr_b64 v[0:1], v[0:1], s4
-; GFX6-NEXT: v_lshl_b64 v[4:5], v[2:3], 1
-; GFX6-NEXT: v_lshr_b64 v[2:3], v[2:3], s4
-; GFX6-NEXT: s_and_b64 s[4:5], vcc, exec
-; GFX6-NEXT: s_cselect_b32 s1, s1, s3
-; GFX6-NEXT: s_cselect_b32 s0, s0, s2
-; GFX6-NEXT: v_lshl_b64 v[4:5], v[4:5], s6
-; GFX6-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
-; GFX6-NEXT: s_lshl_b64 s[0:1], s[0:1], s6
-; GFX6-NEXT: v_or_b32_e32 v1, v5, v1
-; GFX6-NEXT: v_or_b32_e32 v0, v4, v0
-; GFX6-NEXT: v_or_b32_e32 v3, s1, v3
-; GFX6-NEXT: v_or_b32_e32 v2, s0, v2
+; GFX6-NEXT: s_lshl_b64 s[6:7], s[0:1], 1
+; GFX6-NEXT: s_lshl_b64 s[2:3], s[2:3], 1
+; GFX6-NEXT: s_lshr_b32 s0, s1, 31
+; GFX6-NEXT: s_or_b32 s2, s2, s0
+; GFX6-NEXT: s_andn2_b32 s0, 0x7f, s4
+; GFX6-NEXT: s_not_b32 s5, s4
+; GFX6-NEXT: s_sub_i32 s12, s0, 64
+; GFX6-NEXT: s_sub_i32 s8, 64, s0
+; GFX6-NEXT: s_cmp_lt_u32 s0, 64
+; GFX6-NEXT: s_cselect_b32 s13, 1, 0
+; GFX6-NEXT: s_cmp_eq_u32 s0, 0
+; GFX6-NEXT: s_cselect_b32 s14, 1, 0
+; GFX6-NEXT: s_lshr_b64 s[8:9], s[6:7], s8
+; GFX6-NEXT: s_lshl_b64 s[10:11], s[2:3], s5
+; GFX6-NEXT: s_lshl_b64 s[0:1], s[6:7], s5
+; GFX6-NEXT: s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX6-NEXT: s_lshl_b64 s[6:7], s[6:7], s12
+; GFX6-NEXT: s_cmp_lg_u32 s13, 0
+; GFX6-NEXT: s_cselect_b64 s[0:1], s[0:1], 0
+; GFX6-NEXT: s_cselect_b64 s[6:7], s[8:9], s[6:7]
+; GFX6-NEXT: s_cmp_lg_u32 s14, 0
+; GFX6-NEXT: s_cselect_b64 s[2:3], s[2:3], s[6:7]
+; GFX6-NEXT: s_and_b32 s4, s4, 0x7f
+; GFX6-NEXT: s_sub_i32 s5, s4, 64
+; GFX6-NEXT: s_sub_i32 s6, 64, s4
+; GFX6-NEXT: s_cmp_lt_u32 s4, 64
+; GFX6-NEXT: s_cselect_b32 s7, 1, 0
+; GFX6-NEXT: s_cmp_eq_u32 s4, 0
+; GFX6-NEXT: v_lshr_b64 v[4:5], v[0:1], s4
+; GFX6-NEXT: v_lshl_b64 v[6:7], v[2:3], s6
+; GFX6-NEXT: s_cselect_b32 s8, 1, 0
+; GFX6-NEXT: v_lshr_b64 v[8:9], v[2:3], s4
+; GFX6-NEXT: v_lshr_b64 v[2:3], v[2:3], s5
+; GFX6-NEXT: s_cmp_lg_u32 s7, 0
+; GFX6-NEXT: v_or_b32_e32 v4, v4, v6
+; GFX6-NEXT: v_or_b32_e32 v5, v5, v7
+; GFX6-NEXT: s_cselect_b64 vcc, exec, 0
+; GFX6-NEXT: s_cmp_lg_u32 s8, 0
+; GFX6-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc
+; GFX6-NEXT: s_cselect_b64 vcc, exec, 0
+; GFX6-NEXT: s_cmp_lg_u32 s7, 0
+; GFX6-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
+; GFX6-NEXT: s_cselect_b64 vcc, exec, 0
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: v_mov_b32_e32 v1, s1
+; GFX6-NEXT: v_cndmask_b32_e32 v4, 0, v8, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v5, 0, v9, vcc
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX6-NEXT: v_mov_b32_e32 v2, s2
+; GFX6-NEXT: v_mov_b32_e32 v3, s3
+; GFX6-NEXT: v_or_b32_e32 v2, v2, v4
+; GFX6-NEXT: v_or_b32_e32 v3, v3, v5
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: v_fshr_i128_svs:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_not_b32 s8, s4
-; GFX8-NEXT: s_and_b32 s6, s4, 64
-; GFX8-NEXT: s_mov_b32 s7, 0
-; GFX8-NEXT: s_cmp_eq_u64 s[6:7], 0
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: v_mov_b32_e32 v4, s1
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX8-NEXT: v_lshlrev_b64 v[4:5], 1, v[2:3]
-; GFX8-NEXT: v_lshrrev_b64 v[0:1], s4, v[0:1]
-; GFX8-NEXT: v_lshrrev_b64 v[2:3], s4, v[2:3]
-; GFX8-NEXT: s_and_b64 s[4:5], vcc, exec
-; GFX8-NEXT: s_cselect_b32 s1, s1, s3
-; GFX8-NEXT: s_cselect_b32 s0, s0, s2
-; GFX8-NEXT: v_lshlrev_b64 v[4:5], s8, v[4:5]
-; GFX8-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
-; GFX8-NEXT: s_lshl_b64 s[0:1], s[0:1], s8
-; GFX8-NEXT: v_or_b32_e32 v1, v5, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v4, v0
-; GFX8-NEXT: v_or_b32_e32 v3, s1, v3
-; GFX8-NEXT: v_or_b32_e32 v2, s0, v2
+; GFX8-NEXT: s_lshl_b64 s[6:7], s[0:1], 1
+; GFX8-NEXT: s_lshl_b64 s[2:3], s[2:3], 1
+; GFX8-NEXT: s_lshr_b32 s0, s1, 31
+; GFX8-NEXT: s_or_b32 s2, s2, s0
+; GFX8-NEXT: s_andn2_b32 s0, 0x7f, s4
+; GFX8-NEXT: s_not_b32 s5, s4
+; GFX8-NEXT: s_sub_i32 s12, s0, 64
+; GFX8-NEXT: s_sub_i32 s8, 64, s0
+; GFX8-NEXT: s_cmp_lt_u32 s0, 64
+; GFX8-NEXT: s_cselect_b32 s13, 1, 0
+; GFX8-NEXT: s_cmp_eq_u32 s0, 0
+; GFX8-NEXT: s_cselect_b32 s14, 1, 0
+; GFX8-NEXT: s_lshr_b64 s[8:9], s[6:7], s8
+; GFX8-NEXT: s_lshl_b64 s[10:11], s[2:3], s5
+; GFX8-NEXT: s_lshl_b64 s[0:1], s[6:7], s5
+; GFX8-NEXT: s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX8-NEXT: s_lshl_b64 s[6:7], s[6:7], s12
+; GFX8-NEXT: s_cmp_lg_u32 s13, 0
+; GFX8-NEXT: s_cselect_b64 s[0:1], s[0:1], 0
+; GFX8-NEXT: s_cselect_b64 s[6:7], s[8:9], s[6:7]
+; GFX8-NEXT: s_cmp_lg_u32 s14, 0
+; GFX8-NEXT: s_cselect_b64 s[2:3], s[2:3], s[6:7]
+; GFX8-NEXT: s_and_b32 s4, s4, 0x7f
+; GFX8-NEXT: s_sub_i32 s5, s4, 64
+; GFX8-NEXT: s_sub_i32 s6, 64, s4
+; GFX8-NEXT: s_cmp_lt_u32 s4, 64
+; GFX8-NEXT: s_cselect_b32 s7, 1, 0
+; GFX8-NEXT: s_cmp_eq_u32 s4, 0
+; GFX8-NEXT: v_lshrrev_b64 v[4:5], s4, v[0:1]
+; GFX8-NEXT: v_lshlrev_b64 v[6:7], s6, v[2:3]
+; GFX8-NEXT: s_cselect_b32 s8, 1, 0
+; GFX8-NEXT: v_lshrrev_b64 v[8:9], s4, v[2:3]
+; GFX8-NEXT: v_lshrrev_b64 v[2:3], s5, v[2:3]
+; GFX8-NEXT: s_cmp_lg_u32 s7, 0
+; GFX8-NEXT: v_or_b32_e32 v4, v4, v6
+; GFX8-NEXT: v_or_b32_e32 v5, v5, v7
+; GFX8-NEXT: s_cselect_b64 vcc, exec, 0
+; GFX8-NEXT: s_cmp_lg_u32 s8, 0
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc
+; GFX8-NEXT: s_cselect_b64 vcc, exec, 0
+; GFX8-NEXT: s_cmp_lg_u32 s7, 0
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
+; GFX8-NEXT: s_cselect_b64 vcc, exec, 0
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: v_cndmask_b32_e32 v4, 0, v8, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v5, 0, v9, vcc
+; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX8-NEXT: v_mov_b32_e32 v2, s2
+; GFX8-NEXT: v_mov_b32_e32 v3, s3
+; GFX8-NEXT: v_or_b32_e32 v2, v2, v4
+; GFX8-NEXT: v_or_b32_e32 v3, v3, v5
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: v_fshr_i128_svs:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s6, s4, 64
-; GFX11-NEXT: s_mov_b32 s7, 0
-; GFX11-NEXT: s_not_b32 s5, s4
-; GFX11-NEXT: s_cmp_eq_u64 s[6:7], 0
-; GFX11-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX11-NEXT: v_dual_cndmask_b32 v1, v3, v1 :: v_dual_cndmask_b32 v4, s1, v3
-; GFX11-NEXT: v_dual_cndmask_b32 v3, s0, v2 :: v_dual_cndmask_b32 v0, v2, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshlrev_b64 v[5:6], 1, v[3:4]
-; GFX11-NEXT: v_lshrrev_b64 v[0:1], s4, v[0:1]
-; GFX11-NEXT: v_lshrrev_b64 v[2:3], s4, v[3:4]
-; GFX11-NEXT: s_and_b32 s4, vcc_lo, exec_lo
-; GFX11-NEXT: s_cselect_b32 s1, s1, s3
-; GFX11-NEXT: s_cselect_b32 s0, s0, s2
-; GFX11-NEXT: v_lshlrev_b64 v[5:6], s5, v[5:6]
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], s5
-; GFX11-NEXT: v_or_b32_e32 v2, s0, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_or_b32_e32 v0, v5, v0
-; GFX11-NEXT: v_or_b32_e32 v1, v6, v1
-; GFX11-NEXT: v_or_b32_e32 v3, s1, v3
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshr_i128_svs:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_lshl_b64 s[6:7], s[0:1], 1
@@ -6399,6 +6855,7 @@ define amdgpu_ps <4 x float> @v_fshr_i128_svs(i128 inreg %lhs, i128 %rhs, i128 i
; GFX9-NEXT: v_or_b32_e32 v2, v2, v4
; GFX9-NEXT: v_or_b32_e32 v3, v3, v5
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: v_fshr_i128_svs:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_lshl_b64 s[2:3], s[2:3], 1
@@ -6457,6 +6914,64 @@ define amdgpu_ps <4 x float> @v_fshr_i128_svs(i128 inreg %lhs, i128 %rhs, i128 i
; GFX10-NEXT: v_or_b32_e32 v2, v2, v6
; GFX10-NEXT: v_or_b32_e32 v3, v3, v7
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: v_fshr_i128_svs:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_lshl_b64 s[2:3], s[2:3], 1
+; GFX11-NEXT: s_lshr_b32 s5, s1, 31
+; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
+; GFX11-NEXT: s_or_b32 s2, s2, s5
+; GFX11-NEXT: s_and_not1_b32 s5, 0x7f, s4
+; GFX11-NEXT: s_not_b32 s10, s4
+; GFX11-NEXT: s_sub_i32 s12, s5, 64
+; GFX11-NEXT: s_sub_i32 s6, 64, s5
+; GFX11-NEXT: s_cmp_lt_u32 s5, 64
+; GFX11-NEXT: s_cselect_b32 s13, 1, 0
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: s_lshr_b64 s[6:7], s[0:1], s6
+; GFX11-NEXT: s_lshl_b64 s[8:9], s[2:3], s10
+; GFX11-NEXT: s_lshl_b64 s[10:11], s[0:1], s10
+; GFX11-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
+; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], s12
+; GFX11-NEXT: s_cmp_lg_u32 s13, 0
+; GFX11-NEXT: s_cselect_b64 s[8:9], s[10:11], 0
+; GFX11-NEXT: s_cselect_b64 s[0:1], s[6:7], s[0:1]
+; GFX11-NEXT: s_cmp_lg_u32 s5, 0
+; GFX11-NEXT: s_cselect_b64 s[0:1], s[2:3], s[0:1]
+; GFX11-NEXT: s_and_b32 s2, s4, 0x7f
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_sub_i32 s4, 64, s2
+; GFX11-NEXT: v_lshrrev_b64 v[4:5], s2, v[0:1]
+; GFX11-NEXT: v_lshlrev_b64 v[6:7], s4, v[2:3]
+; GFX11-NEXT: s_sub_i32 s3, s2, 64
+; GFX11-NEXT: s_cmp_lt_u32 s2, 64
+; GFX11-NEXT: v_lshrrev_b64 v[8:9], s2, v[2:3]
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_eq_u32 s2, 0
+; GFX11-NEXT: v_lshrrev_b64 v[2:3], s3, v[2:3]
+; GFX11-NEXT: v_or_b32_e32 v4, v4, v6
+; GFX11-NEXT: v_or_b32_e32 v5, v5, v7
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 0
+; GFX11-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
+; GFX11-NEXT: s_cmp_lg_u32 s5, 0
+; GFX11-NEXT: v_dual_cndmask_b32 v2, v2, v4 :: v_dual_cndmask_b32 v3, v3, v5
+; GFX11-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_dual_cndmask_b32 v4, v2, v0 :: v_dual_cndmask_b32 v5, v3, v1
+; GFX11-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
+; GFX11-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
+; GFX11-NEXT: v_dual_cndmask_b32 v6, 0, v8 :: v_dual_mov_b32 v3, s1
+; GFX11-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_cndmask_b32 v7, 0, v9
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX11-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX11-NEXT: ; return to shader part epilog
%result = call i128 @llvm.fshr.i128(i128 %lhs, i128 %rhs, i128 %amt)
%cast.result = bitcast i128 %result to <4 x float>
ret <4 x float> %cast.result
@@ -6465,90 +6980,120 @@ define amdgpu_ps <4 x float> @v_fshr_i128_svs(i128 inreg %lhs, i128 %rhs, i128 i
define amdgpu_ps <4 x float> @v_fshr_i128_vss(i128 %lhs, i128 inreg %rhs, i128 inreg %amt) {
; GFX6-LABEL: v_fshr_i128_vss:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_and_b32 s6, s4, 64
-; GFX6-NEXT: s_mov_b32 s7, 0
-; GFX6-NEXT: v_cmp_eq_u64_e64 vcc, s[6:7], 0
-; GFX6-NEXT: v_mov_b32_e32 v4, s3
-; GFX6-NEXT: v_cndmask_b32_e32 v6, v1, v4, vcc
-; GFX6-NEXT: v_mov_b32_e32 v4, s2
-; GFX6-NEXT: v_cndmask_b32_e32 v5, v0, v4, vcc
-; GFX6-NEXT: s_and_b64 s[6:7], vcc, exec
-; GFX6-NEXT: v_lshl_b64 v[7:8], v[5:6], 1
-; GFX6-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX6-NEXT: s_cselect_b32 s1, s1, s3
-; GFX6-NEXT: s_cselect_b32 s0, s0, s2
-; GFX6-NEXT: s_not_b32 s2, s4
-; GFX6-NEXT: v_lshl_b64 v[0:1], v[0:1], 1
-; GFX6-NEXT: v_lshl_b64 v[7:8], v[7:8], s2
-; GFX6-NEXT: s_lshr_b64 s[0:1], s[0:1], s4
-; GFX6-NEXT: v_lshr_b64 v[2:3], v[5:6], s4
-; GFX6-NEXT: v_lshl_b64 v[5:6], v[0:1], s2
-; GFX6-NEXT: v_or_b32_e32 v4, s1, v8
-; GFX6-NEXT: v_or_b32_e32 v0, s0, v7
-; GFX6-NEXT: v_or_b32_e32 v3, v6, v3
-; GFX6-NEXT: v_or_b32_e32 v2, v5, v2
-; GFX6-NEXT: v_mov_b32_e32 v1, v4
+; GFX6-NEXT: v_lshl_b64 v[2:3], v[2:3], 1
+; GFX6-NEXT: s_andn2_b32 s5, 0x7f, s4
+; GFX6-NEXT: v_lshl_b64 v[4:5], v[0:1], 1
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, 31, v1
+; GFX6-NEXT: s_sub_i32 s6, s5, 64
+; GFX6-NEXT: s_sub_i32 s7, 64, s5
+; GFX6-NEXT: v_or_b32_e32 v2, v2, v0
+; GFX6-NEXT: s_cmp_lt_u32 s5, 64
+; GFX6-NEXT: s_cselect_b32 s8, 1, 0
+; GFX6-NEXT: s_cmp_eq_u32 s5, 0
+; GFX6-NEXT: v_lshr_b64 v[0:1], v[4:5], s7
+; GFX6-NEXT: v_lshl_b64 v[6:7], v[2:3], s5
+; GFX6-NEXT: s_cselect_b32 s9, 1, 0
+; GFX6-NEXT: v_lshl_b64 v[8:9], v[4:5], s5
+; GFX6-NEXT: s_cmp_lg_u32 s8, 0
+; GFX6-NEXT: v_or_b32_e32 v6, v0, v6
+; GFX6-NEXT: v_or_b32_e32 v7, v1, v7
+; GFX6-NEXT: v_lshl_b64 v[0:1], v[4:5], s6
+; GFX6-NEXT: s_cselect_b64 vcc, exec, 0
+; GFX6-NEXT: v_cndmask_b32_e32 v4, 0, v8, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v5, 0, v9, vcc
+; GFX6-NEXT: s_cselect_b64 vcc, exec, 0
+; GFX6-NEXT: s_cmp_lg_u32 s9, 0
+; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v1, v1, v7, vcc
+; GFX6-NEXT: s_cselect_b64 vcc, exec, 0
+; GFX6-NEXT: s_and_b32 s5, s4, 0x7f
+; GFX6-NEXT: s_sub_i32 s10, s5, 64
+; GFX6-NEXT: s_sub_i32 s8, 64, s5
+; GFX6-NEXT: s_cmp_lt_u32 s5, 64
+; GFX6-NEXT: s_cselect_b32 s11, 1, 0
+; GFX6-NEXT: s_cmp_eq_u32 s5, 0
+; GFX6-NEXT: s_cselect_b32 s12, 1, 0
+; GFX6-NEXT: s_lshr_b64 s[6:7], s[2:3], s4
+; GFX6-NEXT: s_lshr_b64 s[4:5], s[0:1], s4
+; GFX6-NEXT: s_lshl_b64 s[8:9], s[2:3], s8
+; GFX6-NEXT: s_or_b64 s[4:5], s[4:5], s[8:9]
+; GFX6-NEXT: s_lshr_b64 s[2:3], s[2:3], s10
+; GFX6-NEXT: s_cmp_lg_u32 s11, 0
+; GFX6-NEXT: s_cselect_b64 s[2:3], s[4:5], s[2:3]
+; GFX6-NEXT: s_cmp_lg_u32 s12, 0
+; GFX6-NEXT: s_cselect_b64 s[0:1], s[0:1], s[2:3]
+; GFX6-NEXT: s_cmp_lg_u32 s11, 0
+; GFX6-NEXT: s_cselect_b64 s[2:3], s[6:7], 0
+; GFX6-NEXT: v_cndmask_b32_e32 v6, v0, v2, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v7, v1, v3, vcc
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: v_mov_b32_e32 v1, s1
+; GFX6-NEXT: v_mov_b32_e32 v2, s2
+; GFX6-NEXT: v_mov_b32_e32 v3, s3
+; GFX6-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX6-NEXT: v_or_b32_e32 v1, v5, v1
+; GFX6-NEXT: v_or_b32_e32 v2, v6, v2
+; GFX6-NEXT: v_or_b32_e32 v3, v7, v3
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: v_fshr_i128_vss:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_and_b32 s6, s4, 64
-; GFX8-NEXT: s_mov_b32 s7, 0
-; GFX8-NEXT: s_cmp_eq_u64 s[6:7], 0
-; GFX8-NEXT: v_mov_b32_e32 v4, s3
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v1, v4, vcc
-; GFX8-NEXT: v_mov_b32_e32 v4, s2
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v0, v4, vcc
-; GFX8-NEXT: s_and_b64 s[6:7], vcc, exec
-; GFX8-NEXT: v_lshlrev_b64 v[7:8], 1, v[5:6]
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX8-NEXT: s_cselect_b32 s1, s1, s3
-; GFX8-NEXT: s_cselect_b32 s0, s0, s2
-; GFX8-NEXT: s_not_b32 s2, s4
-; GFX8-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
-; GFX8-NEXT: v_lshlrev_b64 v[7:8], s2, v[7:8]
-; GFX8-NEXT: s_lshr_b64 s[0:1], s[0:1], s4
-; GFX8-NEXT: v_lshrrev_b64 v[2:3], s4, v[5:6]
-; GFX8-NEXT: v_lshlrev_b64 v[5:6], s2, v[0:1]
-; GFX8-NEXT: v_or_b32_e32 v4, s1, v8
-; GFX8-NEXT: v_or_b32_e32 v0, s0, v7
-; GFX8-NEXT: v_or_b32_e32 v3, v6, v3
-; GFX8-NEXT: v_or_b32_e32 v2, v5, v2
-; GFX8-NEXT: v_mov_b32_e32 v1, v4
+; GFX8-NEXT: v_lshlrev_b64 v[2:3], 1, v[2:3]
+; GFX8-NEXT: s_andn2_b32 s5, 0x7f, s4
+; GFX8-NEXT: v_lshlrev_b64 v[4:5], 1, v[0:1]
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, 31, v1
+; GFX8-NEXT: s_sub_i32 s6, s5, 64
+; GFX8-NEXT: s_sub_i32 s7, 64, s5
+; GFX8-NEXT: v_or_b32_e32 v2, v2, v0
+; GFX8-NEXT: s_cmp_lt_u32 s5, 64
+; GFX8-NEXT: s_cselect_b32 s8, 1, 0
+; GFX8-NEXT: s_cmp_eq_u32 s5, 0
+; GFX8-NEXT: v_lshrrev_b64 v[0:1], s7, v[4:5]
+; GFX8-NEXT: v_lshlrev_b64 v[6:7], s5, v[2:3]
+; GFX8-NEXT: s_cselect_b32 s9, 1, 0
+; GFX8-NEXT: v_lshlrev_b64 v[8:9], s5, v[4:5]
+; GFX8-NEXT: s_cmp_lg_u32 s8, 0
+; GFX8-NEXT: v_or_b32_e32 v6, v0, v6
+; GFX8-NEXT: v_or_b32_e32 v7, v1, v7
+; GFX8-NEXT: v_lshlrev_b64 v[0:1], s6, v[4:5]
+; GFX8-NEXT: s_cselect_b64 vcc, exec, 0
+; GFX8-NEXT: v_cndmask_b32_e32 v4, 0, v8, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v5, 0, v9, vcc
+; GFX8-NEXT: s_cselect_b64 vcc, exec, 0
+; GFX8-NEXT: s_cmp_lg_u32 s9, 0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v7, vcc
+; GFX8-NEXT: s_cselect_b64 vcc, exec, 0
+; GFX8-NEXT: s_and_b32 s5, s4, 0x7f
+; GFX8-NEXT: s_sub_i32 s10, s5, 64
+; GFX8-NEXT: s_sub_i32 s8, 64, s5
+; GFX8-NEXT: s_cmp_lt_u32 s5, 64
+; GFX8-NEXT: s_cselect_b32 s11, 1, 0
+; GFX8-NEXT: s_cmp_eq_u32 s5, 0
+; GFX8-NEXT: s_cselect_b32 s12, 1, 0
+; GFX8-NEXT: s_lshr_b64 s[6:7], s[2:3], s4
+; GFX8-NEXT: s_lshr_b64 s[4:5], s[0:1], s4
+; GFX8-NEXT: s_lshl_b64 s[8:9], s[2:3], s8
+; GFX8-NEXT: s_or_b64 s[4:5], s[4:5], s[8:9]
+; GFX8-NEXT: s_lshr_b64 s[2:3], s[2:3], s10
+; GFX8-NEXT: s_cmp_lg_u32 s11, 0
+; GFX8-NEXT: s_cselect_b64 s[2:3], s[4:5], s[2:3]
+; GFX8-NEXT: s_cmp_lg_u32 s12, 0
+; GFX8-NEXT: s_cselect_b64 s[0:1], s[0:1], s[2:3]
+; GFX8-NEXT: s_cmp_lg_u32 s11, 0
+; GFX8-NEXT: s_cselect_b64 s[2:3], s[6:7], 0
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v0, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v1, v3, vcc
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: v_mov_b32_e32 v2, s2
+; GFX8-NEXT: v_mov_b32_e32 v3, s3
+; GFX8-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX8-NEXT: v_or_b32_e32 v1, v5, v1
+; GFX8-NEXT: v_or_b32_e32 v2, v6, v2
+; GFX8-NEXT: v_or_b32_e32 v3, v7, v3
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: v_fshr_i128_vss:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s6, s4, 64
-; GFX11-NEXT: s_mov_b32 s7, 0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_cmp_eq_u64 s[6:7], 0
-; GFX11-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX11-NEXT: v_cndmask_b32_e64 v5, v1, s3, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e64 v4, v0, s2, vcc_lo
-; GFX11-NEXT: v_dual_cndmask_b32 v1, v3, v1 :: v_dual_cndmask_b32 v0, v2, v0
-; GFX11-NEXT: s_and_b32 s5, vcc_lo, exec_lo
-; GFX11-NEXT: s_cselect_b32 s1, s1, s3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_lshlrev_b64 v[2:3], 1, v[4:5]
-; GFX11-NEXT: s_cselect_b32 s0, s0, s2
-; GFX11-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
-; GFX11-NEXT: s_not_b32 s2, s4
-; GFX11-NEXT: v_lshrrev_b64 v[4:5], s4, v[4:5]
-; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], s4
-; GFX11-NEXT: v_lshlrev_b64 v[2:3], s2, v[2:3]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshlrev_b64 v[6:7], s2, v[0:1]
-; GFX11-NEXT: v_or_b32_e32 v0, s0, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_or_b32_e32 v1, s1, v3
-; GFX11-NEXT: v_or_b32_e32 v2, v6, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v3, v7, v5
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: v_fshr_i128_vss:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_lshlrev_b64 v[2:3], 1, v[2:3]
@@ -6606,6 +7151,7 @@ define amdgpu_ps <4 x float> @v_fshr_i128_vss(i128 %lhs, i128 inreg %rhs, i128 i
; GFX9-NEXT: v_or_b32_e32 v2, v6, v2
; GFX9-NEXT: v_or_b32_e32 v3, v7, v3
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: v_fshr_i128_vss:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_lshlrev_b64 v[2:3], 1, v[2:3]
@@ -6663,39 +7209,95 @@ define amdgpu_ps <4 x float> @v_fshr_i128_vss(i128 %lhs, i128 inreg %rhs, i128 i
; GFX10-NEXT: v_or_b32_e32 v2, v2, v4
; GFX10-NEXT: v_or_b32_e32 v3, v3, v5
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: v_fshr_i128_vss:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_lshlrev_b64 v[2:3], 1, v[2:3]
+; GFX11-NEXT: v_lshrrev_b32_e32 v4, 31, v1
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
+; GFX11-NEXT: s_and_not1_b32 s5, 0x7f, s4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_sub_i32 s6, 64, s5
+; GFX11-NEXT: s_sub_i32 s7, s5, 64
+; GFX11-NEXT: s_cmp_lt_u32 s5, 64
+; GFX11-NEXT: v_or_b32_e32 v2, v2, v4
+; GFX11-NEXT: v_lshrrev_b64 v[4:5], s6, v[0:1]
+; GFX11-NEXT: s_cselect_b32 s6, 1, 0
+; GFX11-NEXT: s_cmp_eq_u32 s5, 0
+; GFX11-NEXT: v_lshlrev_b64 v[8:9], s5, v[0:1]
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s6, 0
+; GFX11-NEXT: v_lshlrev_b64 v[6:7], s5, v[2:3]
+; GFX11-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
+; GFX11-NEXT: s_cselect_b32 s5, exec_lo, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 0
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], s7, v[0:1]
+; GFX11-NEXT: s_cselect_b32 s6, exec_lo, 0
+; GFX11-NEXT: s_and_b32 s7, s4, 0x7f
+; GFX11-NEXT: v_or_b32_e32 v4, v4, v6
+; GFX11-NEXT: s_sub_i32 s12, s7, 64
+; GFX11-NEXT: s_sub_i32 s10, 64, s7
+; GFX11-NEXT: s_cmp_lt_u32 s7, 64
+; GFX11-NEXT: v_or_b32_e32 v5, v5, v7
+; GFX11-NEXT: s_cselect_b32 s13, 1, 0
+; GFX11-NEXT: s_cmp_eq_u32 s7, 0
+; GFX11-NEXT: v_cndmask_b32_e64 v4, v0, v4, s5
+; GFX11-NEXT: s_cselect_b32 s7, 1, 0
+; GFX11-NEXT: s_lshr_b64 s[8:9], s[0:1], s4
+; GFX11-NEXT: s_lshl_b64 s[10:11], s[2:3], s10
+; GFX11-NEXT: v_dual_cndmask_b32 v6, 0, v8 :: v_dual_cndmask_b32 v7, 0, v9
+; GFX11-NEXT: v_cndmask_b32_e64 v8, v1, v5, s5
+; GFX11-NEXT: s_lshr_b64 s[4:5], s[2:3], s4
+; GFX11-NEXT: s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], s12
+; GFX11-NEXT: s_cmp_lg_u32 s13, 0
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v4, v2, s6
+; GFX11-NEXT: s_cselect_b64 s[2:3], s[8:9], s[2:3]
+; GFX11-NEXT: s_cmp_lg_u32 s7, 0
+; GFX11-NEXT: v_cndmask_b32_e64 v3, v8, v3, s6
+; GFX11-NEXT: s_cselect_b64 s[0:1], s[0:1], s[2:3]
+; GFX11-NEXT: s_cmp_lg_u32 s13, 0
+; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT: s_cselect_b64 s[2:3], s[4:5], 0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_dual_mov_b32 v5, s3 :: v_dual_mov_b32 v4, s2
+; GFX11-NEXT: v_or_b32_e32 v0, v6, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_or_b32_e32 v1, v7, v1
+; GFX11-NEXT: v_or_b32_e32 v3, v3, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v2, v2, v4
+; GFX11-NEXT: ; return to shader part epilog
%result = call i128 @llvm.fshr.i128(i128 %lhs, i128 %rhs, i128 %amt)
%cast.result = bitcast i128 %result to <4 x float>
ret <4 x float> %cast.result
}
define amdgpu_ps i128 @s_fshr_i128_65(i128 inreg %lhs, i128 inreg %rhs) {
-; GCN-LABEL: s_fshr_i128_65:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_mov_b32 s4, s2
-; GCN-NEXT: s_lshr_b64 s[2:3], s[0:1], 1
-; GCN-NEXT: s_lshl_b32 s9, s4, 31
-; GCN-NEXT: s_mov_b32 s8, 0
-; GCN-NEXT: s_or_b64 s[4:5], s[8:9], s[2:3]
-; GCN-NEXT: s_lshr_b64 s[6:7], s[6:7], 1
-; GCN-NEXT: s_lshl_b32 s9, s0, 31
-; GCN-NEXT: s_or_b64 s[0:1], s[8:9], s[6:7]
-; GCN-NEXT: s_mov_b32 s0, s6
-; GCN-NEXT: s_mov_b32 s3, s5
-; GCN-NEXT: ; return to shader part epilog
+; GFX6-LABEL: s_fshr_i128_65:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_mov_b32 s4, 0
+; GFX6-NEXT: s_lshl_b32 s5, s0, 31
+; GFX6-NEXT: s_lshl_b32 s3, s2, 31
+; GFX6-NEXT: s_mov_b32 s2, s4
+; GFX6-NEXT: s_lshr_b64 s[0:1], s[0:1], 1
+; GFX6-NEXT: s_or_b64 s[2:3], s[2:3], s[0:1]
+; GFX6-NEXT: s_lshr_b64 s[0:1], s[6:7], 1
+; GFX6-NEXT: s_or_b64 s[0:1], s[4:5], s[0:1]
+; GFX6-NEXT: ; return to shader part epilog
+;
+; GFX8-LABEL: s_fshr_i128_65:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_mov_b32 s4, 0
+; GFX8-NEXT: s_lshl_b32 s5, s0, 31
+; GFX8-NEXT: s_lshl_b32 s3, s2, 31
+; GFX8-NEXT: s_mov_b32 s2, s4
+; GFX8-NEXT: s_lshr_b64 s[0:1], s[0:1], 1
+; GFX8-NEXT: s_or_b64 s[2:3], s[2:3], s[0:1]
+; GFX8-NEXT: s_lshr_b64 s[0:1], s[6:7], 1
+; GFX8-NEXT: s_or_b64 s[0:1], s[4:5], s[0:1]
+; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshr_i128_65:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_mov_b32 s4, s2
-; GFX11-NEXT: s_lshr_b64 s[2:3], s[0:1], 1
-; GFX11-NEXT: s_lshl_b32 s9, s4, 31
-; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_lshr_b64 s[6:7], s[6:7], 1
-; GFX11-NEXT: s_or_b64 s[4:5], s[8:9], s[2:3]
-; GFX11-NEXT: s_lshl_b32 s9, s0, 31
-; GFX11-NEXT: s_mov_b32 s3, s5
-; GFX11-NEXT: s_or_b64 s[0:1], s[8:9], s[6:7]
-; GFX11-NEXT: s_mov_b32 s0, s6
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshr_i128_65:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_mov_b32 s4, 0
@@ -6707,6 +7309,7 @@ define amdgpu_ps i128 @s_fshr_i128_65(i128 inreg %lhs, i128 inreg %rhs) {
; GFX9-NEXT: s_lshr_b64 s[0:1], s[6:7], 1
; GFX9-NEXT: s_or_b64 s[0:1], s[4:5], s[0:1]
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: s_fshr_i128_65:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_mov_b32 s4, 0
@@ -6718,6 +7321,18 @@ define amdgpu_ps i128 @s_fshr_i128_65(i128 inreg %lhs, i128 inreg %rhs) {
; GFX10-NEXT: s_or_b64 s[0:1], s[4:5], s[6:7]
; GFX10-NEXT: s_or_b64 s[2:3], s[2:3], s[8:9]
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: s_fshr_i128_65:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_lshl_b32 s5, s0, 31
+; GFX11-NEXT: s_lshl_b32 s3, s2, 31
+; GFX11-NEXT: s_mov_b32 s2, s4
+; GFX11-NEXT: s_lshr_b64 s[6:7], s[6:7], 1
+; GFX11-NEXT: s_lshr_b64 s[8:9], s[0:1], 1
+; GFX11-NEXT: s_or_b64 s[0:1], s[4:5], s[6:7]
+; GFX11-NEXT: s_or_b64 s[2:3], s[2:3], s[8:9]
+; GFX11-NEXT: ; return to shader part epilog
%result = call i128 @llvm.fshr.i128(i128 %lhs, i128 %rhs, i128 65)
ret i128 %result
}
@@ -6745,18 +7360,6 @@ define i128 @v_fshr_i128_65(i128 %lhs, i128 %rhs) {
; GFX8-NEXT: v_or_b32_e32 v1, v4, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshr_i128_65:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v8, v2
-; GFX11-NEXT: v_lshrrev_b64 v[4:5], 1, v[6:7]
-; GFX11-NEXT: v_lshrrev_b64 v[2:3], 1, v[0:1]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshl_or_b32 v1, v0, 31, v5
-; GFX11-NEXT: v_lshl_or_b32 v3, v8, 31, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT: v_mov_b32_e32 v0, v4
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshr_i128_65:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -6767,6 +7370,7 @@ define i128 @v_fshr_i128_65(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: v_lshl_or_b32 v1, v0, 31, v5
; GFX9-NEXT: v_mov_b32_e32 v0, v4
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshr_i128_65:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -6777,6 +7381,19 @@ define i128 @v_fshr_i128_65(i128 %lhs, i128 %rhs) {
; GFX10-NEXT: v_lshl_or_b32 v3, v8, 31, v3
; GFX10-NEXT: v_mov_b32_e32 v0, v4
; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_fshr_i128_65:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v8, v2
+; GFX11-NEXT: v_lshrrev_b64 v[4:5], 1, v[6:7]
+; GFX11-NEXT: v_lshrrev_b64 v[2:3], 1, v[0:1]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_lshl_or_b32 v1, v0, 31, v5
+; GFX11-NEXT: v_lshl_or_b32 v3, v8, 31, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT: v_mov_b32_e32 v0, v4
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call i128 @llvm.fshr.i128(i128 %lhs, i128 %rhs, i128 65)
ret i128 %result
}
@@ -6784,127 +7401,180 @@ define i128 @v_fshr_i128_65(i128 %lhs, i128 %rhs) {
define amdgpu_ps <2 x i128> @s_fshr_v2i128(<2 x i128> inreg %lhs, <2 x i128> inreg %rhs, <2 x i128> inreg %amt) {
; GFX6-LABEL: s_fshr_v2i128:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_and_b32 s18, s20, 64
-; GFX6-NEXT: s_mov_b32 s19, 0
-; GFX6-NEXT: v_cmp_eq_u64_e64 s[22:23], s[18:19], 0
-; GFX6-NEXT: s_and_b64 s[24:25], s[22:23], exec
-; GFX6-NEXT: s_cselect_b32 s25, s15, s5
-; GFX6-NEXT: s_cselect_b32 s24, s14, s4
-; GFX6-NEXT: s_lshr_b64 s[26:27], s[24:25], s20
-; GFX6-NEXT: s_and_b64 s[28:29], s[22:23], exec
-; GFX6-NEXT: s_cselect_b32 s5, s5, s7
-; GFX6-NEXT: s_cselect_b32 s4, s4, s6
-; GFX6-NEXT: s_lshl_b64 s[4:5], s[4:5], 1
-; GFX6-NEXT: s_not_b32 s17, s20
-; GFX6-NEXT: s_lshl_b64 s[4:5], s[4:5], s17
-; GFX6-NEXT: s_or_b64 s[6:7], s[4:5], s[26:27]
-; GFX6-NEXT: s_and_b64 s[4:5], s[22:23], exec
-; GFX6-NEXT: s_cselect_b32 s5, s13, s15
-; GFX6-NEXT: s_cselect_b32 s4, s12, s14
-; GFX6-NEXT: s_lshl_b64 s[12:13], s[24:25], 1
-; GFX6-NEXT: s_and_b32 s18, s16, 64
-; GFX6-NEXT: s_lshr_b64 s[4:5], s[4:5], s20
-; GFX6-NEXT: s_lshl_b64 s[12:13], s[12:13], s17
-; GFX6-NEXT: v_cmp_eq_u64_e64 s[14:15], s[18:19], 0
-; GFX6-NEXT: s_or_b64 s[4:5], s[12:13], s[4:5]
-; GFX6-NEXT: s_and_b64 s[12:13], s[14:15], exec
-; GFX6-NEXT: s_cselect_b32 s13, s11, s1
-; GFX6-NEXT: s_cselect_b32 s12, s10, s0
-; GFX6-NEXT: s_lshr_b64 s[18:19], s[12:13], s16
-; GFX6-NEXT: s_and_b64 s[20:21], s[14:15], exec
-; GFX6-NEXT: s_cselect_b32 s1, s1, s3
-; GFX6-NEXT: s_cselect_b32 s0, s0, s2
-; GFX6-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
+; GFX6-NEXT: s_lshl_b64 s[18:19], s[0:1], 1
+; GFX6-NEXT: s_lshl_b64 s[2:3], s[2:3], 1
+; GFX6-NEXT: s_lshr_b32 s0, s1, 31
+; GFX6-NEXT: s_or_b32 s2, s2, s0
+; GFX6-NEXT: s_andn2_b32 s0, 0x7f, s16
; GFX6-NEXT: s_not_b32 s17, s16
-; GFX6-NEXT: s_lshl_b64 s[0:1], s[0:1], s17
-; GFX6-NEXT: s_or_b64 s[2:3], s[0:1], s[18:19]
-; GFX6-NEXT: s_and_b64 s[0:1], s[14:15], exec
-; GFX6-NEXT: s_cselect_b32 s1, s9, s11
-; GFX6-NEXT: s_cselect_b32 s0, s8, s10
-; GFX6-NEXT: s_lshl_b64 s[8:9], s[12:13], 1
-; GFX6-NEXT: s_lshr_b64 s[0:1], s[0:1], s16
-; GFX6-NEXT: s_lshl_b64 s[8:9], s[8:9], s17
-; GFX6-NEXT: s_or_b64 s[0:1], s[8:9], s[0:1]
+; GFX6-NEXT: s_sub_i32 s21, s0, 64
+; GFX6-NEXT: s_sub_i32 s22, 64, s0
+; GFX6-NEXT: s_cmp_lt_u32 s0, 64
+; GFX6-NEXT: s_cselect_b32 s26, 1, 0
+; GFX6-NEXT: s_cmp_eq_u32 s0, 0
+; GFX6-NEXT: s_cselect_b32 s27, 1, 0
+; GFX6-NEXT: s_lshr_b64 s[22:23], s[18:19], s22
+; GFX6-NEXT: s_lshl_b64 s[24:25], s[2:3], s17
+; GFX6-NEXT: s_lshl_b64 s[0:1], s[18:19], s17
+; GFX6-NEXT: s_or_b64 s[22:23], s[22:23], s[24:25]
+; GFX6-NEXT: s_lshl_b64 s[18:19], s[18:19], s21
+; GFX6-NEXT: s_cmp_lg_u32 s26, 0
+; GFX6-NEXT: s_cselect_b64 s[0:1], s[0:1], 0
+; GFX6-NEXT: s_cselect_b64 s[18:19], s[22:23], s[18:19]
+; GFX6-NEXT: s_cmp_lg_u32 s27, 0
+; GFX6-NEXT: s_cselect_b64 s[2:3], s[2:3], s[18:19]
+; GFX6-NEXT: s_and_b32 s17, s16, 0x7f
+; GFX6-NEXT: s_sub_i32 s21, s17, 64
+; GFX6-NEXT: s_sub_i32 s22, 64, s17
+; GFX6-NEXT: s_cmp_lt_u32 s17, 64
+; GFX6-NEXT: s_cselect_b32 s24, 1, 0
+; GFX6-NEXT: s_cmp_eq_u32 s17, 0
+; GFX6-NEXT: s_cselect_b32 s25, 1, 0
+; GFX6-NEXT: s_lshr_b64 s[18:19], s[10:11], s16
+; GFX6-NEXT: s_lshr_b64 s[16:17], s[8:9], s16
+; GFX6-NEXT: s_lshl_b64 s[22:23], s[10:11], s22
+; GFX6-NEXT: s_or_b64 s[16:17], s[16:17], s[22:23]
+; GFX6-NEXT: s_lshr_b64 s[10:11], s[10:11], s21
+; GFX6-NEXT: s_cmp_lg_u32 s24, 0
+; GFX6-NEXT: s_cselect_b64 s[10:11], s[16:17], s[10:11]
+; GFX6-NEXT: s_cmp_lg_u32 s25, 0
+; GFX6-NEXT: s_cselect_b64 s[8:9], s[8:9], s[10:11]
+; GFX6-NEXT: s_cmp_lg_u32 s24, 0
+; GFX6-NEXT: s_cselect_b64 s[10:11], s[18:19], 0
+; GFX6-NEXT: s_or_b64 s[0:1], s[0:1], s[8:9]
+; GFX6-NEXT: s_lshl_b64 s[8:9], s[4:5], 1
+; GFX6-NEXT: s_lshl_b64 s[6:7], s[6:7], 1
+; GFX6-NEXT: s_lshr_b32 s4, s5, 31
+; GFX6-NEXT: s_or_b32 s6, s6, s4
+; GFX6-NEXT: s_andn2_b32 s4, 0x7f, s20
+; GFX6-NEXT: s_or_b64 s[2:3], s[2:3], s[10:11]
+; GFX6-NEXT: s_not_b32 s16, s20
+; GFX6-NEXT: s_sub_i32 s18, s4, 64
+; GFX6-NEXT: s_sub_i32 s10, 64, s4
+; GFX6-NEXT: s_cmp_lt_u32 s4, 64
+; GFX6-NEXT: s_cselect_b32 s19, 1, 0
+; GFX6-NEXT: s_cmp_eq_u32 s4, 0
+; GFX6-NEXT: s_cselect_b32 s21, 1, 0
+; GFX6-NEXT: s_lshl_b64 s[4:5], s[8:9], s16
+; GFX6-NEXT: s_lshr_b64 s[10:11], s[8:9], s10
+; GFX6-NEXT: s_lshl_b64 s[16:17], s[6:7], s16
+; GFX6-NEXT: s_or_b64 s[10:11], s[10:11], s[16:17]
+; GFX6-NEXT: s_lshl_b64 s[8:9], s[8:9], s18
+; GFX6-NEXT: s_cmp_lg_u32 s19, 0
+; GFX6-NEXT: s_cselect_b64 s[4:5], s[4:5], 0
+; GFX6-NEXT: s_cselect_b64 s[8:9], s[10:11], s[8:9]
+; GFX6-NEXT: s_cmp_lg_u32 s21, 0
+; GFX6-NEXT: s_cselect_b64 s[6:7], s[6:7], s[8:9]
+; GFX6-NEXT: s_and_b32 s8, s20, 0x7f
+; GFX6-NEXT: s_sub_i32 s18, s8, 64
+; GFX6-NEXT: s_sub_i32 s16, 64, s8
+; GFX6-NEXT: s_cmp_lt_u32 s8, 64
+; GFX6-NEXT: s_cselect_b32 s19, 1, 0
+; GFX6-NEXT: s_cmp_eq_u32 s8, 0
+; GFX6-NEXT: s_cselect_b32 s21, 1, 0
+; GFX6-NEXT: s_lshr_b64 s[10:11], s[12:13], s20
+; GFX6-NEXT: s_lshl_b64 s[16:17], s[14:15], s16
+; GFX6-NEXT: s_lshr_b64 s[8:9], s[14:15], s20
+; GFX6-NEXT: s_or_b64 s[10:11], s[10:11], s[16:17]
+; GFX6-NEXT: s_lshr_b64 s[14:15], s[14:15], s18
+; GFX6-NEXT: s_cmp_lg_u32 s19, 0
+; GFX6-NEXT: s_cselect_b64 s[10:11], s[10:11], s[14:15]
+; GFX6-NEXT: s_cmp_lg_u32 s21, 0
+; GFX6-NEXT: s_cselect_b64 s[10:11], s[12:13], s[10:11]
+; GFX6-NEXT: s_cmp_lg_u32 s19, 0
+; GFX6-NEXT: s_cselect_b64 s[8:9], s[8:9], 0
+; GFX6-NEXT: s_or_b64 s[4:5], s[4:5], s[10:11]
+; GFX6-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_fshr_v2i128:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_and_b32 s18, s20, 64
-; GFX8-NEXT: s_mov_b32 s19, 0
-; GFX8-NEXT: s_cmp_eq_u64 s[18:19], 0
-; GFX8-NEXT: s_cselect_b32 s7, s5, s7
-; GFX8-NEXT: s_cselect_b32 s6, s4, s6
-; GFX8-NEXT: s_cselect_b32 s5, s15, s5
-; GFX8-NEXT: s_cselect_b32 s4, s14, s4
-; GFX8-NEXT: s_cselect_b32 s12, s12, s14
-; GFX8-NEXT: s_cselect_b32 s13, s13, s15
-; GFX8-NEXT: s_lshr_b64 s[14:15], s[4:5], s20
-; GFX8-NEXT: s_lshl_b64 s[6:7], s[6:7], 1
-; GFX8-NEXT: s_not_b32 s17, s20
-; GFX8-NEXT: s_lshl_b64 s[4:5], s[4:5], 1
-; GFX8-NEXT: s_lshl_b64 s[6:7], s[6:7], s17
-; GFX8-NEXT: s_lshr_b64 s[12:13], s[12:13], s20
-; GFX8-NEXT: s_lshl_b64 s[4:5], s[4:5], s17
-; GFX8-NEXT: s_or_b64 s[6:7], s[6:7], s[14:15]
-; GFX8-NEXT: s_or_b64 s[4:5], s[4:5], s[12:13]
-; GFX8-NEXT: s_and_b32 s18, s16, 64
-; GFX8-NEXT: s_cmp_eq_u64 s[18:19], 0
-; GFX8-NEXT: s_cselect_b32 s3, s1, s3
-; GFX8-NEXT: s_cselect_b32 s2, s0, s2
-; GFX8-NEXT: s_cselect_b32 s1, s11, s1
-; GFX8-NEXT: s_cselect_b32 s0, s10, s0
-; GFX8-NEXT: s_cselect_b32 s8, s8, s10
-; GFX8-NEXT: s_cselect_b32 s9, s9, s11
-; GFX8-NEXT: s_lshr_b64 s[10:11], s[0:1], s16
+; GFX8-NEXT: s_lshl_b64 s[18:19], s[0:1], 1
; GFX8-NEXT: s_lshl_b64 s[2:3], s[2:3], 1
-; GFX8-NEXT: s_not_b32 s12, s16
-; GFX8-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
-; GFX8-NEXT: s_lshl_b64 s[2:3], s[2:3], s12
-; GFX8-NEXT: s_lshr_b64 s[8:9], s[8:9], s16
-; GFX8-NEXT: s_lshl_b64 s[0:1], s[0:1], s12
-; GFX8-NEXT: s_or_b64 s[2:3], s[2:3], s[10:11]
+; GFX8-NEXT: s_lshr_b32 s0, s1, 31
+; GFX8-NEXT: s_or_b32 s2, s2, s0
+; GFX8-NEXT: s_andn2_b32 s0, 0x7f, s16
+; GFX8-NEXT: s_not_b32 s17, s16
+; GFX8-NEXT: s_sub_i32 s21, s0, 64
+; GFX8-NEXT: s_sub_i32 s22, 64, s0
+; GFX8-NEXT: s_cmp_lt_u32 s0, 64
+; GFX8-NEXT: s_cselect_b32 s26, 1, 0
+; GFX8-NEXT: s_cmp_eq_u32 s0, 0
+; GFX8-NEXT: s_cselect_b32 s27, 1, 0
+; GFX8-NEXT: s_lshr_b64 s[22:23], s[18:19], s22
+; GFX8-NEXT: s_lshl_b64 s[24:25], s[2:3], s17
+; GFX8-NEXT: s_lshl_b64 s[0:1], s[18:19], s17
+; GFX8-NEXT: s_or_b64 s[22:23], s[22:23], s[24:25]
+; GFX8-NEXT: s_lshl_b64 s[18:19], s[18:19], s21
+; GFX8-NEXT: s_cmp_lg_u32 s26, 0
+; GFX8-NEXT: s_cselect_b64 s[0:1], s[0:1], 0
+; GFX8-NEXT: s_cselect_b64 s[18:19], s[22:23], s[18:19]
+; GFX8-NEXT: s_cmp_lg_u32 s27, 0
+; GFX8-NEXT: s_cselect_b64 s[2:3], s[2:3], s[18:19]
+; GFX8-NEXT: s_and_b32 s17, s16, 0x7f
+; GFX8-NEXT: s_sub_i32 s21, s17, 64
+; GFX8-NEXT: s_sub_i32 s22, 64, s17
+; GFX8-NEXT: s_cmp_lt_u32 s17, 64
+; GFX8-NEXT: s_cselect_b32 s24, 1, 0
+; GFX8-NEXT: s_cmp_eq_u32 s17, 0
+; GFX8-NEXT: s_cselect_b32 s25, 1, 0
+; GFX8-NEXT: s_lshr_b64 s[18:19], s[10:11], s16
+; GFX8-NEXT: s_lshr_b64 s[16:17], s[8:9], s16
+; GFX8-NEXT: s_lshl_b64 s[22:23], s[10:11], s22
+; GFX8-NEXT: s_or_b64 s[16:17], s[16:17], s[22:23]
+; GFX8-NEXT: s_lshr_b64 s[10:11], s[10:11], s21
+; GFX8-NEXT: s_cmp_lg_u32 s24, 0
+; GFX8-NEXT: s_cselect_b64 s[10:11], s[16:17], s[10:11]
+; GFX8-NEXT: s_cmp_lg_u32 s25, 0
+; GFX8-NEXT: s_cselect_b64 s[8:9], s[8:9], s[10:11]
+; GFX8-NEXT: s_cmp_lg_u32 s24, 0
+; GFX8-NEXT: s_cselect_b64 s[10:11], s[18:19], 0
; GFX8-NEXT: s_or_b64 s[0:1], s[0:1], s[8:9]
+; GFX8-NEXT: s_lshl_b64 s[8:9], s[4:5], 1
+; GFX8-NEXT: s_lshl_b64 s[6:7], s[6:7], 1
+; GFX8-NEXT: s_lshr_b32 s4, s5, 31
+; GFX8-NEXT: s_or_b32 s6, s6, s4
+; GFX8-NEXT: s_andn2_b32 s4, 0x7f, s20
+; GFX8-NEXT: s_or_b64 s[2:3], s[2:3], s[10:11]
+; GFX8-NEXT: s_not_b32 s16, s20
+; GFX8-NEXT: s_sub_i32 s18, s4, 64
+; GFX8-NEXT: s_sub_i32 s10, 64, s4
+; GFX8-NEXT: s_cmp_lt_u32 s4, 64
+; GFX8-NEXT: s_cselect_b32 s19, 1, 0
+; GFX8-NEXT: s_cmp_eq_u32 s4, 0
+; GFX8-NEXT: s_cselect_b32 s21, 1, 0
+; GFX8-NEXT: s_lshl_b64 s[4:5], s[8:9], s16
+; GFX8-NEXT: s_lshr_b64 s[10:11], s[8:9], s10
+; GFX8-NEXT: s_lshl_b64 s[16:17], s[6:7], s16
+; GFX8-NEXT: s_or_b64 s[10:11], s[10:11], s[16:17]
+; GFX8-NEXT: s_lshl_b64 s[8:9], s[8:9], s18
+; GFX8-NEXT: s_cmp_lg_u32 s19, 0
+; GFX8-NEXT: s_cselect_b64 s[4:5], s[4:5], 0
+; GFX8-NEXT: s_cselect_b64 s[8:9], s[10:11], s[8:9]
+; GFX8-NEXT: s_cmp_lg_u32 s21, 0
+; GFX8-NEXT: s_cselect_b64 s[6:7], s[6:7], s[8:9]
+; GFX8-NEXT: s_and_b32 s8, s20, 0x7f
+; GFX8-NEXT: s_sub_i32 s18, s8, 64
+; GFX8-NEXT: s_sub_i32 s16, 64, s8
+; GFX8-NEXT: s_cmp_lt_u32 s8, 64
+; GFX8-NEXT: s_cselect_b32 s19, 1, 0
+; GFX8-NEXT: s_cmp_eq_u32 s8, 0
+; GFX8-NEXT: s_cselect_b32 s21, 1, 0
+; GFX8-NEXT: s_lshr_b64 s[10:11], s[12:13], s20
+; GFX8-NEXT: s_lshl_b64 s[16:17], s[14:15], s16
+; GFX8-NEXT: s_lshr_b64 s[8:9], s[14:15], s20
+; GFX8-NEXT: s_or_b64 s[10:11], s[10:11], s[16:17]
+; GFX8-NEXT: s_lshr_b64 s[14:15], s[14:15], s18
+; GFX8-NEXT: s_cmp_lg_u32 s19, 0
+; GFX8-NEXT: s_cselect_b64 s[10:11], s[10:11], s[14:15]
+; GFX8-NEXT: s_cmp_lg_u32 s21, 0
+; GFX8-NEXT: s_cselect_b64 s[10:11], s[12:13], s[10:11]
+; GFX8-NEXT: s_cmp_lg_u32 s19, 0
+; GFX8-NEXT: s_cselect_b64 s[8:9], s[8:9], 0
+; GFX8-NEXT: s_or_b64 s[4:5], s[4:5], s[10:11]
+; GFX8-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX8-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_fshr_v2i128:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s18, s20, 64
-; GFX11-NEXT: s_mov_b32 s19, 0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_cmp_eq_u64 s[18:19], 0
-; GFX11-NEXT: s_cselect_b32 s7, s5, s7
-; GFX11-NEXT: s_cselect_b32 s6, s4, s6
-; GFX11-NEXT: s_cselect_b32 s5, s15, s5
-; GFX11-NEXT: s_cselect_b32 s4, s14, s4
-; GFX11-NEXT: s_cselect_b32 s12, s12, s14
-; GFX11-NEXT: s_cselect_b32 s13, s13, s15
-; GFX11-NEXT: s_lshl_b64 s[6:7], s[6:7], 1
-; GFX11-NEXT: s_not_b32 s17, s20
-; GFX11-NEXT: s_lshr_b64 s[14:15], s[4:5], s20
-; GFX11-NEXT: s_lshl_b64 s[4:5], s[4:5], 1
-; GFX11-NEXT: s_lshl_b64 s[6:7], s[6:7], s17
-; GFX11-NEXT: s_lshr_b64 s[12:13], s[12:13], s20
-; GFX11-NEXT: s_lshl_b64 s[4:5], s[4:5], s17
-; GFX11-NEXT: s_and_b32 s18, s16, 64
-; GFX11-NEXT: s_or_b64 s[6:7], s[6:7], s[14:15]
-; GFX11-NEXT: s_or_b64 s[4:5], s[4:5], s[12:13]
-; GFX11-NEXT: s_cmp_eq_u64 s[18:19], 0
-; GFX11-NEXT: s_cselect_b32 s3, s1, s3
-; GFX11-NEXT: s_cselect_b32 s2, s0, s2
-; GFX11-NEXT: s_cselect_b32 s1, s11, s1
-; GFX11-NEXT: s_cselect_b32 s0, s10, s0
-; GFX11-NEXT: s_cselect_b32 s8, s8, s10
-; GFX11-NEXT: s_cselect_b32 s9, s9, s11
-; GFX11-NEXT: s_lshr_b64 s[10:11], s[0:1], s16
-; GFX11-NEXT: s_lshl_b64 s[2:3], s[2:3], 1
-; GFX11-NEXT: s_not_b32 s12, s16
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
-; GFX11-NEXT: s_lshr_b64 s[8:9], s[8:9], s16
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], s12
-; GFX11-NEXT: s_lshl_b64 s[2:3], s[2:3], s12
-; GFX11-NEXT: s_or_b64 s[0:1], s[0:1], s[8:9]
-; GFX11-NEXT: s_or_b64 s[2:3], s[2:3], s[10:11]
-; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshr_v2i128:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_lshl_b64 s[18:19], s[0:1], 1
@@ -6992,6 +7662,7 @@ define amdgpu_ps <2 x i128> @s_fshr_v2i128(<2 x i128> inreg %lhs, <2 x i128> inr
; GFX9-NEXT: s_or_b64 s[4:5], s[4:5], s[10:11]
; GFX9-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX9-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: s_fshr_v2i128:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_lshl_b64 s[2:3], s[2:3], 1
@@ -7079,6 +7750,96 @@ define amdgpu_ps <2 x i128> @s_fshr_v2i128(<2 x i128> inreg %lhs, <2 x i128> inr
; GFX10-NEXT: s_or_b64 s[4:5], s[10:11], s[4:5]
; GFX10-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: s_fshr_v2i128:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_lshl_b64 s[2:3], s[2:3], 1
+; GFX11-NEXT: s_lshr_b32 s17, s1, 31
+; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
+; GFX11-NEXT: s_or_b32 s2, s2, s17
+; GFX11-NEXT: s_and_not1_b32 s17, 0x7f, s16
+; GFX11-NEXT: s_not_b32 s21, s16
+; GFX11-NEXT: s_sub_i32 s26, s17, 64
+; GFX11-NEXT: s_sub_i32 s18, 64, s17
+; GFX11-NEXT: s_cmp_lt_u32 s17, 64
+; GFX11-NEXT: s_cselect_b32 s27, 1, 0
+; GFX11-NEXT: s_cmp_eq_u32 s17, 0
+; GFX11-NEXT: s_cselect_b32 s17, 1, 0
+; GFX11-NEXT: s_lshr_b64 s[18:19], s[0:1], s18
+; GFX11-NEXT: s_lshl_b64 s[22:23], s[2:3], s21
+; GFX11-NEXT: s_lshl_b64 s[24:25], s[0:1], s21
+; GFX11-NEXT: s_or_b64 s[18:19], s[18:19], s[22:23]
+; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], s26
+; GFX11-NEXT: s_cmp_lg_u32 s27, 0
+; GFX11-NEXT: s_cselect_b64 s[22:23], s[24:25], 0
+; GFX11-NEXT: s_cselect_b64 s[0:1], s[18:19], s[0:1]
+; GFX11-NEXT: s_cmp_lg_u32 s17, 0
+; GFX11-NEXT: s_cselect_b64 s[2:3], s[2:3], s[0:1]
+; GFX11-NEXT: s_and_b32 s0, s16, 0x7f
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_sub_i32 s21, s0, 64
+; GFX11-NEXT: s_sub_i32 s17, 64, s0
+; GFX11-NEXT: s_cmp_lt_u32 s0, 64
+; GFX11-NEXT: s_cselect_b32 s24, 1, 0
+; GFX11-NEXT: s_cmp_eq_u32 s0, 0
+; GFX11-NEXT: s_cselect_b32 s25, 1, 0
+; GFX11-NEXT: s_lshr_b64 s[0:1], s[8:9], s16
+; GFX11-NEXT: s_lshl_b64 s[18:19], s[10:11], s17
+; GFX11-NEXT: s_lshr_b64 s[16:17], s[10:11], s16
+; GFX11-NEXT: s_or_b64 s[0:1], s[0:1], s[18:19]
+; GFX11-NEXT: s_lshr_b64 s[10:11], s[10:11], s21
+; GFX11-NEXT: s_cmp_lg_u32 s24, 0
+; GFX11-NEXT: s_cselect_b64 s[0:1], s[0:1], s[10:11]
+; GFX11-NEXT: s_cmp_lg_u32 s25, 0
+; GFX11-NEXT: s_cselect_b64 s[0:1], s[8:9], s[0:1]
+; GFX11-NEXT: s_cmp_lg_u32 s24, 0
+; GFX11-NEXT: s_cselect_b64 s[8:9], s[16:17], 0
+; GFX11-NEXT: s_lshl_b64 s[6:7], s[6:7], 1
+; GFX11-NEXT: s_or_b64 s[2:3], s[2:3], s[8:9]
+; GFX11-NEXT: s_lshr_b32 s8, s5, 31
+; GFX11-NEXT: s_or_b64 s[0:1], s[22:23], s[0:1]
+; GFX11-NEXT: s_or_b32 s6, s6, s8
+; GFX11-NEXT: s_and_not1_b32 s8, 0x7f, s20
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[4:5], 1
+; GFX11-NEXT: s_not_b32 s16, s20
+; GFX11-NEXT: s_sub_i32 s18, s8, 64
+; GFX11-NEXT: s_sub_i32 s9, 64, s8
+; GFX11-NEXT: s_cmp_lt_u32 s8, 64
+; GFX11-NEXT: s_cselect_b32 s19, 1, 0
+; GFX11-NEXT: s_cmp_eq_u32 s8, 0
+; GFX11-NEXT: s_cselect_b32 s21, 1, 0
+; GFX11-NEXT: s_lshr_b64 s[8:9], s[4:5], s9
+; GFX11-NEXT: s_lshl_b64 s[10:11], s[6:7], s16
+; GFX11-NEXT: s_lshl_b64 s[16:17], s[4:5], s16
+; GFX11-NEXT: s_or_b64 s[8:9], s[8:9], s[10:11]
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[4:5], s18
+; GFX11-NEXT: s_cmp_lg_u32 s19, 0
+; GFX11-NEXT: s_cselect_b64 s[10:11], s[16:17], 0
+; GFX11-NEXT: s_cselect_b64 s[4:5], s[8:9], s[4:5]
+; GFX11-NEXT: s_cmp_lg_u32 s21, 0
+; GFX11-NEXT: s_cselect_b64 s[6:7], s[6:7], s[4:5]
+; GFX11-NEXT: s_and_b32 s4, s20, 0x7f
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_sub_i32 s18, s4, 64
+; GFX11-NEXT: s_sub_i32 s8, 64, s4
+; GFX11-NEXT: s_cmp_lt_u32 s4, 64
+; GFX11-NEXT: s_cselect_b32 s19, 1, 0
+; GFX11-NEXT: s_cmp_eq_u32 s4, 0
+; GFX11-NEXT: s_cselect_b32 s21, 1, 0
+; GFX11-NEXT: s_lshr_b64 s[4:5], s[12:13], s20
+; GFX11-NEXT: s_lshl_b64 s[8:9], s[14:15], s8
+; GFX11-NEXT: s_lshr_b64 s[16:17], s[14:15], s20
+; GFX11-NEXT: s_or_b64 s[4:5], s[4:5], s[8:9]
+; GFX11-NEXT: s_lshr_b64 s[8:9], s[14:15], s18
+; GFX11-NEXT: s_cmp_lg_u32 s19, 0
+; GFX11-NEXT: s_cselect_b64 s[4:5], s[4:5], s[8:9]
+; GFX11-NEXT: s_cmp_lg_u32 s21, 0
+; GFX11-NEXT: s_cselect_b64 s[4:5], s[12:13], s[4:5]
+; GFX11-NEXT: s_cmp_lg_u32 s19, 0
+; GFX11-NEXT: s_cselect_b64 s[8:9], s[16:17], 0
+; GFX11-NEXT: s_or_b64 s[4:5], s[10:11], s[4:5]
+; GFX11-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
+; GFX11-NEXT: ; return to shader part epilog
%result = call <2 x i128> @llvm.fshr.v2i128(<2 x i128> %lhs, <2 x i128> %rhs, <2 x i128> %amt)
ret <2 x i128> %result
}
@@ -7087,134 +7848,185 @@ define <2 x i128> @v_fshr_v2i128(<2 x i128> %lhs, <2 x i128> %rhs, <2 x i128> %a
; GFX6-LABEL: v_fshr_v2i128:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v17, 64, v16
-; GFX6-NEXT: v_mov_b32_e32 v18, 0
-; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[17:18]
-; GFX6-NEXT: v_cndmask_b32_e32 v9, v11, v9, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v8, v10, v8, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v11, v1, v11, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v10, v0, v10, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX6-NEXT: v_lshl_b64 v[2:3], v[10:11], 1
-; GFX6-NEXT: v_lshl_b64 v[0:1], v[0:1], 1
-; GFX6-NEXT: v_lshr_b64 v[8:9], v[8:9], v16
-; GFX6-NEXT: v_lshr_b64 v[10:11], v[10:11], v16
-; GFX6-NEXT: v_not_b32_e32 v16, v16
-; GFX6-NEXT: v_lshl_b64 v[2:3], v[2:3], v16
-; GFX6-NEXT: v_lshl_b64 v[16:17], v[0:1], v16
-; GFX6-NEXT: v_or_b32_e32 v1, v3, v9
-; GFX6-NEXT: v_or_b32_e32 v3, v17, v11
-; GFX6-NEXT: v_and_b32_e32 v17, 64, v20
-; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[17:18]
-; GFX6-NEXT: v_or_b32_e32 v0, v2, v8
-; GFX6-NEXT: v_cndmask_b32_e32 v9, v15, v13, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v8, v14, v12, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v15, v5, v15, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v14, v4, v14, vcc
-; GFX6-NEXT: v_lshr_b64 v[11:12], v[8:9], v20
-; GFX6-NEXT: v_lshl_b64 v[8:9], v[14:15], 1
-; GFX6-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc
-; GFX6-NEXT: v_not_b32_e32 v13, v20
-; GFX6-NEXT: v_lshl_b64 v[4:5], v[4:5], 1
-; GFX6-NEXT: v_lshl_b64 v[17:18], v[8:9], v13
-; GFX6-NEXT: v_or_b32_e32 v2, v16, v10
-; GFX6-NEXT: v_lshr_b64 v[6:7], v[14:15], v20
-; GFX6-NEXT: v_lshl_b64 v[9:10], v[4:5], v13
-; GFX6-NEXT: v_or_b32_e32 v8, v18, v12
-; GFX6-NEXT: v_or_b32_e32 v4, v17, v11
-; GFX6-NEXT: v_or_b32_e32 v7, v10, v7
-; GFX6-NEXT: v_or_b32_e32 v6, v9, v6
-; GFX6-NEXT: v_mov_b32_e32 v5, v8
+; GFX6-NEXT: v_lshl_b64 v[2:3], v[2:3], 1
+; GFX6-NEXT: v_mov_b32_e32 v18, 0x7f
+; GFX6-NEXT: v_lshl_b64 v[21:22], v[0:1], 1
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, 31, v1
+; GFX6-NEXT: v_bfi_b32 v19, v16, 0, v18
+; GFX6-NEXT: v_or_b32_e32 v2, v2, v0
+; GFX6-NEXT: v_not_b32_e32 v17, 63
+; GFX6-NEXT: v_sub_i32_e32 v23, vcc, 64, v19
+; GFX6-NEXT: v_add_i32_e32 v27, vcc, v19, v17
+; GFX6-NEXT: v_lshr_b64 v[23:24], v[21:22], v23
+; GFX6-NEXT: v_lshl_b64 v[25:26], v[2:3], v19
+; GFX6-NEXT: v_lshl_b64 v[0:1], v[21:22], v19
+; GFX6-NEXT: v_lshl_b64 v[21:22], v[21:22], v27
+; GFX6-NEXT: v_cmp_gt_u32_e32 vcc, 64, v19
+; GFX6-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v19
+; GFX6-NEXT: v_or_b32_e32 v19, v23, v25
+; GFX6-NEXT: v_or_b32_e32 v23, v24, v26
+; GFX6-NEXT: v_cndmask_b32_e32 v24, 0, v0, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v0, v21, v19, vcc
+; GFX6-NEXT: v_cndmask_b32_e64 v19, v0, v2, s[4:5]
+; GFX6-NEXT: v_and_b32_e32 v2, 0x7f, v16
+; GFX6-NEXT: v_cndmask_b32_e32 v25, 0, v1, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v1, v22, v23, vcc
+; GFX6-NEXT: v_add_i32_e32 v16, vcc, v2, v17
+; GFX6-NEXT: v_sub_i32_e32 v21, vcc, 64, v2
+; GFX6-NEXT: v_cndmask_b32_e64 v23, v1, v3, s[4:5]
+; GFX6-NEXT: v_cmp_gt_u32_e32 vcc, 64, v2
+; GFX6-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v2
+; GFX6-NEXT: v_lshr_b64 v[0:1], v[10:11], v2
+; GFX6-NEXT: v_lshr_b64 v[2:3], v[8:9], v2
+; GFX6-NEXT: v_lshl_b64 v[21:22], v[10:11], v21
+; GFX6-NEXT: v_lshl_b64 v[6:7], v[6:7], 1
+; GFX6-NEXT: v_or_b32_e32 v21, v2, v21
+; GFX6-NEXT: v_or_b32_e32 v22, v3, v22
+; GFX6-NEXT: v_lshr_b64 v[2:3], v[10:11], v16
+; GFX6-NEXT: v_bfi_b32 v16, v20, 0, v18
+; GFX6-NEXT: v_cndmask_b32_e32 v2, v2, v21, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v3, v3, v22, vcc
+; GFX6-NEXT: v_cndmask_b32_e64 v2, v2, v8, s[4:5]
+; GFX6-NEXT: v_cndmask_b32_e64 v3, v3, v9, s[4:5]
+; GFX6-NEXT: v_cndmask_b32_e32 v8, 0, v0, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v9, 0, v1, vcc
+; GFX6-NEXT: v_or_b32_e32 v0, v24, v2
+; GFX6-NEXT: v_or_b32_e32 v1, v25, v3
+; GFX6-NEXT: v_or_b32_e32 v2, v19, v8
+; GFX6-NEXT: v_or_b32_e32 v3, v23, v9
+; GFX6-NEXT: v_lshl_b64 v[8:9], v[4:5], 1
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 31, v5
+; GFX6-NEXT: v_or_b32_e32 v6, v6, v4
+; GFX6-NEXT: v_sub_i32_e32 v10, vcc, 64, v16
+; GFX6-NEXT: v_add_i32_e32 v21, vcc, v16, v17
+; GFX6-NEXT: v_lshr_b64 v[10:11], v[8:9], v10
+; GFX6-NEXT: v_lshl_b64 v[18:19], v[6:7], v16
+; GFX6-NEXT: v_lshl_b64 v[4:5], v[8:9], v16
+; GFX6-NEXT: v_lshl_b64 v[8:9], v[8:9], v21
+; GFX6-NEXT: v_cmp_gt_u32_e32 vcc, 64, v16
+; GFX6-NEXT: v_or_b32_e32 v10, v10, v18
+; GFX6-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v16
+; GFX6-NEXT: v_cndmask_b32_e32 v16, 0, v4, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v4, v8, v10, vcc
+; GFX6-NEXT: v_or_b32_e32 v11, v11, v19
+; GFX6-NEXT: v_cndmask_b32_e64 v10, v4, v6, s[4:5]
+; GFX6-NEXT: v_and_b32_e32 v6, 0x7f, v20
+; GFX6-NEXT: v_cndmask_b32_e32 v18, 0, v5, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v5, v9, v11, vcc
+; GFX6-NEXT: v_add_i32_e32 v17, vcc, v6, v17
+; GFX6-NEXT: v_sub_i32_e32 v8, vcc, 64, v6
+; GFX6-NEXT: v_cndmask_b32_e64 v11, v5, v7, s[4:5]
+; GFX6-NEXT: v_cmp_gt_u32_e32 vcc, 64, v6
+; GFX6-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v6
+; GFX6-NEXT: v_lshr_b64 v[4:5], v[14:15], v6
+; GFX6-NEXT: v_lshr_b64 v[6:7], v[12:13], v6
+; GFX6-NEXT: v_lshl_b64 v[8:9], v[14:15], v8
+; GFX6-NEXT: v_or_b32_e32 v8, v6, v8
+; GFX6-NEXT: v_or_b32_e32 v9, v7, v9
+; GFX6-NEXT: v_lshr_b64 v[6:7], v[14:15], v17
+; GFX6-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc
+; GFX6-NEXT: v_cndmask_b32_e64 v6, v6, v12, s[4:5]
+; GFX6-NEXT: v_cndmask_b32_e64 v7, v7, v13, s[4:5]
+; GFX6-NEXT: v_cndmask_b32_e32 v8, 0, v4, vcc
+; GFX6-NEXT: v_cndmask_b32_e32 v9, 0, v5, vcc
+; GFX6-NEXT: v_or_b32_e32 v4, v16, v6
+; GFX6-NEXT: v_or_b32_e32 v5, v18, v7
+; GFX6-NEXT: v_or_b32_e32 v6, v10, v8
+; GFX6-NEXT: v_or_b32_e32 v7, v11, v9
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_fshr_v2i128:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_and_b32_e32 v17, 64, v16
-; GFX8-NEXT: v_mov_b32_e32 v18, 0
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[17:18]
-; GFX8-NEXT: v_cndmask_b32_e32 v9, v11, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v8, v10, v8, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v11, v1, v11, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v10, v0, v10, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX8-NEXT: v_lshlrev_b64 v[2:3], 1, v[10:11]
-; GFX8-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
-; GFX8-NEXT: v_lshrrev_b64 v[8:9], v16, v[8:9]
-; GFX8-NEXT: v_lshrrev_b64 v[10:11], v16, v[10:11]
-; GFX8-NEXT: v_not_b32_e32 v16, v16
-; GFX8-NEXT: v_lshlrev_b64 v[2:3], v16, v[2:3]
-; GFX8-NEXT: v_lshlrev_b64 v[16:17], v16, v[0:1]
-; GFX8-NEXT: v_or_b32_e32 v1, v3, v9
-; GFX8-NEXT: v_or_b32_e32 v3, v17, v11
-; GFX8-NEXT: v_and_b32_e32 v17, 64, v20
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[17:18]
-; GFX8-NEXT: v_or_b32_e32 v0, v2, v8
-; GFX8-NEXT: v_cndmask_b32_e32 v9, v15, v13, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v8, v14, v12, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v15, v5, v15, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v14, v4, v14, vcc
-; GFX8-NEXT: v_lshrrev_b64 v[11:12], v20, v[8:9]
-; GFX8-NEXT: v_lshlrev_b64 v[8:9], 1, v[14:15]
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc
-; GFX8-NEXT: v_not_b32_e32 v13, v20
-; GFX8-NEXT: v_lshlrev_b64 v[4:5], 1, v[4:5]
-; GFX8-NEXT: v_lshlrev_b64 v[17:18], v13, v[8:9]
-; GFX8-NEXT: v_or_b32_e32 v2, v16, v10
-; GFX8-NEXT: v_lshrrev_b64 v[6:7], v20, v[14:15]
-; GFX8-NEXT: v_lshlrev_b64 v[9:10], v13, v[4:5]
-; GFX8-NEXT: v_or_b32_e32 v8, v18, v12
-; GFX8-NEXT: v_or_b32_e32 v4, v17, v11
-; GFX8-NEXT: v_or_b32_e32 v7, v10, v7
-; GFX8-NEXT: v_or_b32_e32 v6, v9, v6
-; GFX8-NEXT: v_mov_b32_e32 v5, v8
+; GFX8-NEXT: v_lshlrev_b64 v[2:3], 1, v[2:3]
+; GFX8-NEXT: v_mov_b32_e32 v18, 0x7f
+; GFX8-NEXT: v_lshlrev_b64 v[21:22], 1, v[0:1]
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, 31, v1
+; GFX8-NEXT: v_bfi_b32 v19, v16, 0, v18
+; GFX8-NEXT: v_or_b32_e32 v2, v2, v0
+; GFX8-NEXT: v_not_b32_e32 v17, 63
+; GFX8-NEXT: v_sub_u32_e32 v23, vcc, 64, v19
+; GFX8-NEXT: v_add_u32_e32 v27, vcc, v19, v17
+; GFX8-NEXT: v_lshrrev_b64 v[23:24], v23, v[21:22]
+; GFX8-NEXT: v_lshlrev_b64 v[25:26], v19, v[2:3]
+; GFX8-NEXT: v_lshlrev_b64 v[0:1], v19, v[21:22]
+; GFX8-NEXT: v_lshlrev_b64 v[21:22], v27, v[21:22]
+; GFX8-NEXT: v_cmp_gt_u32_e32 vcc, 64, v19
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v19
+; GFX8-NEXT: v_or_b32_e32 v19, v23, v25
+; GFX8-NEXT: v_or_b32_e32 v23, v24, v26
+; GFX8-NEXT: v_cndmask_b32_e32 v24, 0, v0, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v21, v19, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v19, v0, v2, s[4:5]
+; GFX8-NEXT: v_and_b32_e32 v2, 0x7f, v16
+; GFX8-NEXT: v_cndmask_b32_e32 v25, 0, v1, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v22, v23, vcc
+; GFX8-NEXT: v_add_u32_e32 v16, vcc, v2, v17
+; GFX8-NEXT: v_sub_u32_e32 v21, vcc, 64, v2
+; GFX8-NEXT: v_cndmask_b32_e64 v23, v1, v3, s[4:5]
+; GFX8-NEXT: v_cmp_gt_u32_e32 vcc, 64, v2
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v2
+; GFX8-NEXT: v_lshrrev_b64 v[0:1], v2, v[10:11]
+; GFX8-NEXT: v_lshrrev_b64 v[2:3], v2, v[8:9]
+; GFX8-NEXT: v_lshlrev_b64 v[21:22], v21, v[10:11]
+; GFX8-NEXT: v_lshlrev_b64 v[6:7], 1, v[6:7]
+; GFX8-NEXT: v_or_b32_e32 v21, v2, v21
+; GFX8-NEXT: v_or_b32_e32 v22, v3, v22
+; GFX8-NEXT: v_lshrrev_b64 v[2:3], v16, v[10:11]
+; GFX8-NEXT: v_bfi_b32 v16, v20, 0, v18
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v21, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v22, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v2, v8, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e64 v3, v3, v9, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e32 v8, 0, v0, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v9, 0, v1, vcc
+; GFX8-NEXT: v_or_b32_e32 v0, v24, v2
+; GFX8-NEXT: v_or_b32_e32 v1, v25, v3
+; GFX8-NEXT: v_or_b32_e32 v2, v19, v8
+; GFX8-NEXT: v_or_b32_e32 v3, v23, v9
+; GFX8-NEXT: v_lshlrev_b64 v[8:9], 1, v[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v4, 31, v5
+; GFX8-NEXT: v_or_b32_e32 v6, v6, v4
+; GFX8-NEXT: v_sub_u32_e32 v10, vcc, 64, v16
+; GFX8-NEXT: v_add_u32_e32 v21, vcc, v16, v17
+; GFX8-NEXT: v_lshrrev_b64 v[10:11], v10, v[8:9]
+; GFX8-NEXT: v_lshlrev_b64 v[18:19], v16, v[6:7]
+; GFX8-NEXT: v_lshlrev_b64 v[4:5], v16, v[8:9]
+; GFX8-NEXT: v_lshlrev_b64 v[8:9], v21, v[8:9]
+; GFX8-NEXT: v_cmp_gt_u32_e32 vcc, 64, v16
+; GFX8-NEXT: v_or_b32_e32 v10, v10, v18
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v16
+; GFX8-NEXT: v_cndmask_b32_e32 v16, 0, v4, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v4, v8, v10, vcc
+; GFX8-NEXT: v_or_b32_e32 v11, v11, v19
+; GFX8-NEXT: v_cndmask_b32_e64 v10, v4, v6, s[4:5]
+; GFX8-NEXT: v_and_b32_e32 v6, 0x7f, v20
+; GFX8-NEXT: v_cndmask_b32_e32 v18, 0, v5, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v11, vcc
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, v6, v17
+; GFX8-NEXT: v_sub_u32_e32 v8, vcc, 64, v6
+; GFX8-NEXT: v_cndmask_b32_e64 v11, v5, v7, s[4:5]
+; GFX8-NEXT: v_cmp_gt_u32_e32 vcc, 64, v6
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v6
+; GFX8-NEXT: v_lshrrev_b64 v[4:5], v6, v[14:15]
+; GFX8-NEXT: v_lshrrev_b64 v[6:7], v6, v[12:13]
+; GFX8-NEXT: v_lshlrev_b64 v[8:9], v8, v[14:15]
+; GFX8-NEXT: v_or_b32_e32 v8, v6, v8
+; GFX8-NEXT: v_or_b32_e32 v9, v7, v9
+; GFX8-NEXT: v_lshrrev_b64 v[6:7], v17, v[14:15]
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v6, v6, v12, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e64 v7, v7, v13, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e32 v8, 0, v4, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v9, 0, v5, vcc
+; GFX8-NEXT: v_or_b32_e32 v4, v16, v6
+; GFX8-NEXT: v_or_b32_e32 v5, v18, v7
+; GFX8-NEXT: v_or_b32_e32 v6, v10, v8
+; GFX8-NEXT: v_or_b32_e32 v7, v11, v9
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_fshr_v2i128:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v18, 0 :: v_dual_and_b32 v17, 64, v16
-; GFX11-NEXT: v_not_b32_e32 v19, v16
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[17:18]
-; GFX11-NEXT: v_and_b32_e32 v17, 64, v20
-; GFX11-NEXT: v_cmp_eq_u64_e64 s0, 0, v[17:18]
-; GFX11-NEXT: v_dual_cndmask_b32 v9, v11, v9 :: v_dual_cndmask_b32 v22, v1, v11
-; GFX11-NEXT: v_dual_cndmask_b32 v21, v0, v10 :: v_dual_cndmask_b32 v8, v10, v8
-; GFX11-NEXT: v_dual_cndmask_b32 v3, v3, v1 :: v_dual_cndmask_b32 v2, v2, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_cndmask_b32_e64 v18, v5, v15, s0
-; GFX11-NEXT: v_lshlrev_b64 v[10:11], 1, v[21:22]
-; GFX11-NEXT: v_cndmask_b32_e64 v17, v4, v14, s0
-; GFX11-NEXT: v_cndmask_b32_e64 v5, v7, v5, s0
-; GFX11-NEXT: v_cndmask_b32_e64 v4, v6, v4, s0
-; GFX11-NEXT: v_lshrrev_b64 v[8:9], v16, v[8:9]
-; GFX11-NEXT: v_lshlrev_b64 v[2:3], 1, v[2:3]
-; GFX11-NEXT: v_lshlrev_b64 v[10:11], v19, v[10:11]
-; GFX11-NEXT: v_lshlrev_b64 v[6:7], 1, v[17:18]
-; GFX11-NEXT: v_lshlrev_b64 v[4:5], 1, v[4:5]
-; GFX11-NEXT: v_cndmask_b32_e64 v13, v15, v13, s0
-; GFX11-NEXT: v_cndmask_b32_e64 v12, v14, v12, s0
-; GFX11-NEXT: v_not_b32_e32 v0, v20
-; GFX11-NEXT: v_or_b32_e32 v1, v11, v9
-; GFX11-NEXT: v_lshrrev_b64 v[14:15], v16, v[21:22]
-; GFX11-NEXT: v_lshlrev_b64 v[2:3], v19, v[2:3]
-; GFX11-NEXT: v_lshrrev_b64 v[11:12], v20, v[12:13]
-; GFX11-NEXT: v_lshlrev_b64 v[6:7], v0, v[6:7]
-; GFX11-NEXT: v_lshrrev_b64 v[16:17], v20, v[17:18]
-; GFX11-NEXT: v_lshlrev_b64 v[18:19], v0, v[4:5]
-; GFX11-NEXT: v_or_b32_e32 v0, v10, v8
-; GFX11-NEXT: v_or_b32_e32 v3, v3, v15
-; GFX11-NEXT: v_or_b32_e32 v2, v2, v14
-; GFX11-NEXT: v_or_b32_e32 v5, v7, v12
-; GFX11-NEXT: v_or_b32_e32 v4, v6, v11
-; GFX11-NEXT: v_or_b32_e32 v7, v19, v17
-; GFX11-NEXT: v_or_b32_e32 v6, v18, v16
-; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-LABEL: v_fshr_v2i128:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -7304,6 +8116,7 @@ define <2 x i128> @v_fshr_v2i128(<2 x i128> %lhs, <2 x i128> %rhs, <2 x i128> %a
; GFX9-NEXT: v_or_b32_e32 v6, v8, v10
; GFX9-NEXT: v_or_b32_e32 v7, v9, v11
; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: v_fshr_v2i128:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -7392,6 +8205,101 @@ define <2 x i128> @v_fshr_v2i128(<2 x i128> %lhs, <2 x i128> %rhs, <2 x i128> %a
; GFX10-NEXT: v_or_b32_e32 v6, v6, v8
; GFX10-NEXT: v_or_b32_e32 v7, v7, v9
; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_fshr_v2i128:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b64 v[2:3], 1, v[2:3]
+; GFX11-NEXT: v_bfi_b32 v25, v16, 0, 0x7f
+; GFX11-NEXT: v_lshrrev_b32_e32 v17, 31, v1
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
+; GFX11-NEXT: v_lshlrev_b64 v[6:7], 1, v[6:7]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_sub_nc_u32_e32 v18, 64, v25
+; GFX11-NEXT: v_or_b32_e32 v2, v2, v17
+; GFX11-NEXT: v_cmp_gt_u32_e32 vcc_lo, 64, v25
+; GFX11-NEXT: v_lshlrev_b64 v[23:24], v25, v[0:1]
+; GFX11-NEXT: v_and_b32_e32 v26, 0x7f, v16
+; GFX11-NEXT: v_lshrrev_b64 v[17:18], v18, v[0:1]
+; GFX11-NEXT: v_lshlrev_b64 v[21:22], v25, v[2:3]
+; GFX11-NEXT: v_add_nc_u32_e32 v19, 0xffffffc0, v25
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 0, v25
+; GFX11-NEXT: v_dual_cndmask_b32 v23, 0, v23 :: v_dual_cndmask_b32 v24, 0, v24
+; GFX11-NEXT: v_bfi_b32 v25, v20, 0, 0x7f
+; GFX11-NEXT: v_or_b32_e32 v22, v18, v22
+; GFX11-NEXT: v_sub_nc_u32_e32 v18, 64, v26
+; GFX11-NEXT: v_or_b32_e32 v21, v17, v21
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], v19, v[0:1]
+; GFX11-NEXT: v_lshrrev_b64 v[16:17], v26, v[8:9]
+; GFX11-NEXT: v_and_b32_e32 v20, 0x7f, v20
+; GFX11-NEXT: v_lshlrev_b64 v[18:19], v18, v[10:11]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_dual_cndmask_b32 v21, v0, v21 :: v_dual_cndmask_b32 v22, v1, v22
+; GFX11-NEXT: v_cmp_gt_u32_e32 vcc_lo, 64, v26
+; GFX11-NEXT: v_or_b32_e32 v16, v16, v18
+; GFX11-NEXT: v_add_nc_u32_e32 v27, 0xffffffc0, v26
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT: v_cndmask_b32_e64 v21, v21, v2, s0
+; GFX11-NEXT: v_or_b32_e32 v17, v17, v19
+; GFX11-NEXT: v_cndmask_b32_e64 v22, v22, v3, s0
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 0, v26
+; GFX11-NEXT: v_lshrrev_b64 v[0:1], v27, v[10:11]
+; GFX11-NEXT: v_lshrrev_b64 v[2:3], v26, v[10:11]
+; GFX11-NEXT: v_lshrrev_b32_e32 v10, 31, v5
+; GFX11-NEXT: v_lshlrev_b64 v[4:5], 1, v[4:5]
+; GFX11-NEXT: v_sub_nc_u32_e32 v18, 64, v20
+; GFX11-NEXT: v_cmp_gt_u32_e64 s1, 64, v20
+; GFX11-NEXT: v_dual_cndmask_b32 v0, v0, v16 :: v_dual_cndmask_b32 v1, v1, v17
+; GFX11-NEXT: v_cndmask_b32_e32 v26, 0, v2, vcc_lo
+; GFX11-NEXT: v_or_b32_e32 v6, v6, v10
+; GFX11-NEXT: v_cndmask_b32_e32 v27, 0, v3, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, v8, s0
+; GFX11-NEXT: v_sub_nc_u32_e32 v8, 64, v25
+; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v9, s0
+; GFX11-NEXT: v_cmp_gt_u32_e32 vcc_lo, 64, v25
+; GFX11-NEXT: v_lshlrev_b64 v[10:11], v25, v[4:5]
+; GFX11-NEXT: v_or_b32_e32 v0, v23, v0
+; GFX11-NEXT: v_lshrrev_b64 v[2:3], v8, v[4:5]
+; GFX11-NEXT: v_lshlrev_b64 v[8:9], v25, v[6:7]
+; GFX11-NEXT: v_add_nc_u32_e32 v23, 0xffffffc0, v20
+; GFX11-NEXT: v_lshlrev_b64 v[18:19], v18, v[14:15]
+; GFX11-NEXT: v_cndmask_b32_e32 v10, 0, v10, vcc_lo
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 0, v25
+; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 0, v20
+; GFX11-NEXT: v_or_b32_e32 v8, v2, v8
+; GFX11-NEXT: v_add_nc_u32_e32 v16, 0xffffffc0, v25
+; GFX11-NEXT: v_or_b32_e32 v2, v21, v26
+; GFX11-NEXT: v_or_b32_e32 v9, v3, v9
+; GFX11-NEXT: v_cndmask_b32_e32 v11, 0, v11, vcc_lo
+; GFX11-NEXT: v_or_b32_e32 v1, v24, v1
+; GFX11-NEXT: v_lshlrev_b64 v[4:5], v16, v[4:5]
+; GFX11-NEXT: v_lshrrev_b64 v[16:17], v20, v[12:13]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e32 v21, v4, v8, vcc_lo
+; GFX11-NEXT: v_lshrrev_b64 v[3:4], v23, v[14:15]
+; GFX11-NEXT: v_or_b32_e32 v8, v16, v18
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v16, v17, v19
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v5, v9, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e64 v6, v21, v6, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, v8, s1
+; GFX11-NEXT: v_lshrrev_b64 v[8:9], v20, v[14:15]
+; GFX11-NEXT: v_cndmask_b32_e64 v4, v4, v16, s1
+; GFX11-NEXT: v_cndmask_b32_e64 v7, v5, v7, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_cndmask_b32_e64 v5, v3, v12, s2
+; GFX11-NEXT: v_or_b32_e32 v3, v22, v27
+; GFX11-NEXT: v_cndmask_b32_e64 v12, v4, v13, s2
+; GFX11-NEXT: v_cndmask_b32_e64 v8, 0, v8, s1
+; GFX11-NEXT: v_cndmask_b32_e64 v9, 0, v9, s1
+; GFX11-NEXT: v_or_b32_e32 v4, v10, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v5, v11, v12
+; GFX11-NEXT: v_or_b32_e32 v6, v6, v8
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v7, v7, v9
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call <2 x i128> @llvm.fshr.v2i128(<2 x i128> %lhs, <2 x i128> %rhs, <2 x i128> %amt)
ret <2 x i128> %result
}
@@ -7428,5 +8336,3 @@ declare i128 @llvm.fshr.i128(i128, i128, i128) #0
declare <2 x i128> @llvm.fshr.v2i128(<2 x i128>, <2 x i128>, <2 x i128>) #0
attributes #0 = { nounwind readnone speculatable willreturn }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX11-TRUE16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.abs.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.abs.ll
index ecc0f74b2efef..2ea447a47dc54 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.abs.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.abs.ll
@@ -2,8 +2,8 @@
; RUN: llc -global-isel -mtriple=amdgpu6.00 -o - < %s | FileCheck %s --check-prefixes=GFX,GFX6
; RUN: llc -global-isel -mtriple=amdgpu8.03 -o - < %s | FileCheck %s --check-prefixes=GFX,GFX8
; RUN: llc -global-isel -mtriple=amdgpu10.10 -o - < %s | FileCheck %s --check-prefixes=GFX,GFX10
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.50 -mattr=-real-true16 -o - < %s | FileCheck %s --check-prefixes=GFX1250,GFX1250-FAKE16
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.50 -mattr=+real-true16 -o - < %s | FileCheck %s --check-prefixes=GFX1250,GFX1250-REAL16
+; RUN: llc -global-isel -mtriple=amdgpu12.50 -mattr=-real-true16 -o - < %s | FileCheck %s --check-prefixes=GFX1250,GFX1250-FAKE16
+; RUN: llc -global-isel -mtriple=amdgpu12.50 -mattr=+real-true16 -o - < %s | FileCheck %s --check-prefixes=GFX1250,GFX1250-REAL16
declare i16 @llvm.abs.i16(i16, i1)
declare i32 @llvm.abs.i32(i32, i1)
@@ -66,12 +66,12 @@ define amdgpu_cs i64 @abs_sgpr_i64(i64 inreg %arg) {
; GFX1250-NEXT: global_wb
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_sub_nc_u64 s[2:3], 0, s[0:1]
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_max_i64 v[0:1], s[0:1], s[2:3]
-; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1250-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1250-NEXT: s_ashr_i32 s2, s1, 31
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: s_mov_b32 s3, s2
+; GFX1250-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[2:3]
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: s_xor_b64 s[0:1], s[0:1], s[2:3]
; GFX1250-NEXT: ; return to shader part epilog
%res = call i64 @llvm.abs.i64(i64 %arg, i1 false)
ret i64 %res
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.make.buffer.rsrc.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.make.buffer.rsrc.ll
index 0d238788031bc..a2f2bde2c710c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.make.buffer.rsrc.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.make.buffer.rsrc.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 2
; RUN: llc -global-isel -mtriple=amdgpu9.00 -stop-after=instruction-select < %s | FileCheck %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.50 -stop-after=instruction-select < %s | FileCheck --check-prefix=CHECK45 %s
+; RUN: llc -global-isel -mtriple=amdgpu12.50 -stop-after=instruction-select < %s | FileCheck --check-prefix=CHECK45 %s
define amdgpu_ps ptr addrspace(8) @basic_raw_buffer(ptr inreg %p) {
; CHECK-LABEL: name: basic_raw_buffer
@@ -193,45 +193,44 @@ define amdgpu_ps ptr addrspace(8) @variable_top_half(ptr inreg %p, i64 inreg %nu
; CHECK45: bb.1 (%ir-block.0):
; CHECK45-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4
; CHECK45-NEXT: {{ $}}
- ; CHECK45-NEXT: [[COPY:%[0-9]+]]:sgpr(i32) = COPY $sgpr0
- ; CHECK45-NEXT: [[COPY1:%[0-9]+]]:sgpr(i32) = COPY $sgpr1
- ; CHECK45-NEXT: [[MV:%[0-9]+]]:sgpr(p0) = G_MERGE_VALUES [[COPY]](i32), [[COPY1]](i32)
- ; CHECK45-NEXT: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
- ; CHECK45-NEXT: [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
- ; CHECK45-NEXT: [[MV1:%[0-9]+]]:sreg_64(i64) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
- ; CHECK45-NEXT: [[COPY4:%[0-9]+]]:sreg_32(i32) = COPY $sgpr4
- ; CHECK45-NEXT: [[C:%[0-9]+]]:sreg_32(i32) = G_CONSTANT i32 0
- ; CHECK45-NEXT: [[PTRTOINT:%[0-9]+]]:sreg_64(i64) = G_PTRTOINT [[MV]](p0)
- ; CHECK45-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 25
- ; CHECK45-NEXT: [[SHL:%[0-9]+]]:sreg_32(s32) = G_SHL [[COPY2]], [[C1]](s32)
- ; CHECK45-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32(s32) = S_MOV_B32 0
- ; CHECK45-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64(i64) = REG_SEQUENCE [[S_MOV_B32_]](s32), %subreg.sub0, [[SHL]](s32), %subreg.sub1
- ; CHECK45-NEXT: [[S_OR_B64_:%[0-9]+]]:sreg_64(i64) = S_OR_B64 [[PTRTOINT]](i64), [[REG_SEQUENCE]](i64), implicit-def dead $scc
- ; CHECK45-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32(i32) = S_MOV_B32 7
- ; CHECK45-NEXT: [[S_LSHR_B64_:%[0-9]+]]:sreg_64(i64) = S_LSHR_B64 [[MV1]](i64), [[S_MOV_B32_1]](i32), implicit-def dead $scc
- ; CHECK45-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32(i32) = S_MOV_B32 16384
- ; CHECK45-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sreg_64(i64) = REG_SEQUENCE [[C]](i32), %subreg.sub0, [[S_MOV_B32_2]](i32), %subreg.sub1
- ; CHECK45-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32(i32) = S_MOV_B32 28
- ; CHECK45-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32(i32) = S_LSHL_B32 [[COPY4]](i32), [[S_MOV_B32_3]](i32), implicit-def dead $scc
- ; CHECK45-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sreg_64(i64) = REG_SEQUENCE [[C]](i32), %subreg.sub0, [[S_LSHL_B32_]](i32), %subreg.sub1
- ; CHECK45-NEXT: [[S_OR_B64_1:%[0-9]+]]:sreg_64(i64) = S_OR_B64 [[S_LSHR_B64_]](i64), [[REG_SEQUENCE1]](i64), implicit-def dead $scc
- ; CHECK45-NEXT: [[S_OR_B64_2:%[0-9]+]]:sreg_64(i64) = S_OR_B64 [[S_OR_B64_1]](i64), [[REG_SEQUENCE2]](i64), implicit-def dead $scc
- ; CHECK45-NEXT: [[COPY5:%[0-9]+]]:sreg_32(i32) = COPY [[S_OR_B64_]].sub0(i64)
- ; CHECK45-NEXT: [[COPY6:%[0-9]+]]:sreg_32(i32) = COPY [[S_OR_B64_]].sub1(i64)
- ; CHECK45-NEXT: [[COPY7:%[0-9]+]]:sreg_32(i32) = COPY [[S_OR_B64_2]].sub0(i64)
- ; CHECK45-NEXT: [[COPY8:%[0-9]+]]:sreg_32(i32) = COPY [[S_OR_B64_2]].sub1(i64)
- ; CHECK45-NEXT: [[COPY9:%[0-9]+]]:vgpr_32(i32) = COPY [[COPY5]](i32)
- ; CHECK45-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY9]](i32), implicit $exec
- ; CHECK45-NEXT: $sgpr0 = COPY [[V_READFIRSTLANE_B32_]](i32)
- ; CHECK45-NEXT: [[COPY10:%[0-9]+]]:vgpr_32(i32) = COPY [[COPY6]](i32)
- ; CHECK45-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY10]](i32), implicit $exec
- ; CHECK45-NEXT: $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]](i32)
- ; CHECK45-NEXT: [[COPY11:%[0-9]+]]:vgpr_32(i32) = COPY [[COPY7]](i32)
- ; CHECK45-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY11]](i32), implicit $exec
- ; CHECK45-NEXT: $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]](i32)
- ; CHECK45-NEXT: [[COPY12:%[0-9]+]]:vgpr_32(i32) = COPY [[COPY8]](i32)
- ; CHECK45-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY12]](i32), implicit $exec
- ; CHECK45-NEXT: $sgpr3 = COPY [[V_READFIRSTLANE_B32_3]](i32)
+ ; CHECK45-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; CHECK45-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+ ; CHECK45-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+ ; CHECK45-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; CHECK45-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; CHECK45-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1
+ ; CHECK45-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; CHECK45-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; CHECK45-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 25
+ ; CHECK45-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY2]], [[S_MOV_B32_1]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; CHECK45-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_2]], %subreg.sub0, [[S_LSHL_B32_]], %subreg.sub1
+ ; CHECK45-NEXT: [[S_OR_B64_:%[0-9]+]]:sreg_64 = S_OR_B64 [[REG_SEQUENCE]], [[REG_SEQUENCE2]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 7
+ ; CHECK45-NEXT: [[S_LSHR_B64_:%[0-9]+]]:sreg_64 = S_LSHR_B64 [[REG_SEQUENCE1]], [[S_MOV_B32_3]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 16384
+ ; CHECK45-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_4]], %subreg.sub1
+ ; CHECK45-NEXT: [[S_MOV_B32_5:%[0-9]+]]:sreg_32 = S_MOV_B32 28
+ ; CHECK45-NEXT: [[S_LSHL_B32_1:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY4]], [[S_MOV_B32_5]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_LSHL_B32_1]], %subreg.sub1
+ ; CHECK45-NEXT: [[S_OR_B64_1:%[0-9]+]]:sreg_64 = S_OR_B64 [[S_LSHR_B64_]], [[REG_SEQUENCE3]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[S_OR_B64_2:%[0-9]+]]:sreg_64 = S_OR_B64 [[S_OR_B64_1]], [[REG_SEQUENCE4]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY [[S_OR_B64_]].sub0
+ ; CHECK45-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY [[S_OR_B64_]].sub1
+ ; CHECK45-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY [[S_OR_B64_2]].sub0
+ ; CHECK45-NEXT: [[COPY8:%[0-9]+]]:sreg_32 = COPY [[S_OR_B64_2]].sub1
+ ; CHECK45-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+ ; CHECK45-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+ ; CHECK45-NEXT: $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+ ; CHECK45-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+ ; CHECK45-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
+ ; CHECK45-NEXT: $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+ ; CHECK45-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[COPY7]]
+ ; CHECK45-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
+ ; CHECK45-NEXT: $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
+ ; CHECK45-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY8]]
+ ; CHECK45-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY12]], implicit $exec
+ ; CHECK45-NEXT: $sgpr3 = COPY [[V_READFIRSTLANE_B32_3]]
; CHECK45-NEXT: SI_RETURN_TO_EPILOG implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $sgpr3
%rsrc = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p0(ptr %p, i16 4, i64 %numVals, i32 %flags)
ret ptr addrspace(8) %rsrc
@@ -270,47 +269,46 @@ define amdgpu_ps ptr addrspace(8) @general_case(ptr inreg %p, i16 inreg %stride,
; CHECK45: bb.1 (%ir-block.0):
; CHECK45-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4, $sgpr5
; CHECK45-NEXT: {{ $}}
- ; CHECK45-NEXT: [[COPY:%[0-9]+]]:sgpr(i32) = COPY $sgpr0
- ; CHECK45-NEXT: [[COPY1:%[0-9]+]]:sgpr(i32) = COPY $sgpr1
- ; CHECK45-NEXT: [[MV:%[0-9]+]]:sgpr(p0) = G_MERGE_VALUES [[COPY]](i32), [[COPY1]](i32)
- ; CHECK45-NEXT: [[COPY2:%[0-9]+]]:sreg_32(i32) = COPY $sgpr2
- ; CHECK45-NEXT: [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
- ; CHECK45-NEXT: [[COPY4:%[0-9]+]]:sgpr(s32) = COPY $sgpr4
- ; CHECK45-NEXT: [[MV1:%[0-9]+]]:sreg_64(i64) = G_MERGE_VALUES [[COPY3]](s32), [[COPY4]](s32)
- ; CHECK45-NEXT: [[COPY5:%[0-9]+]]:sreg_32(i32) = COPY $sgpr5
- ; CHECK45-NEXT: [[C:%[0-9]+]]:sreg_32(i32) = G_CONSTANT i32 0
- ; CHECK45-NEXT: [[PTRTOINT:%[0-9]+]]:sreg_64(i64) = G_PTRTOINT [[MV]](p0)
- ; CHECK45-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 25
- ; CHECK45-NEXT: [[SHL:%[0-9]+]]:sreg_32(s32) = G_SHL [[COPY3]], [[C1]](s32)
- ; CHECK45-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32(s32) = S_MOV_B32 0
- ; CHECK45-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64(i64) = REG_SEQUENCE [[S_MOV_B32_]](s32), %subreg.sub0, [[SHL]](s32), %subreg.sub1
- ; CHECK45-NEXT: [[S_OR_B64_:%[0-9]+]]:sreg_64(i64) = S_OR_B64 [[PTRTOINT]](i64), [[REG_SEQUENCE]](i64), implicit-def dead $scc
- ; CHECK45-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32(i32) = S_MOV_B32 7
- ; CHECK45-NEXT: [[S_LSHR_B64_:%[0-9]+]]:sreg_64(i64) = S_LSHR_B64 [[MV1]](i64), [[S_MOV_B32_1]](i32), implicit-def dead $scc
- ; CHECK45-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32(i32) = S_MOV_B32 12
- ; CHECK45-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32(i32) = S_LSHL_B32 [[COPY2]](i32), [[S_MOV_B32_2]](i32), implicit-def dead $scc
- ; CHECK45-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sreg_64(i64) = REG_SEQUENCE [[C]](i32), %subreg.sub0, [[S_LSHL_B32_]](i32), %subreg.sub1
- ; CHECK45-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32(i32) = S_MOV_B32 28
- ; CHECK45-NEXT: [[S_LSHL_B32_1:%[0-9]+]]:sreg_32(i32) = S_LSHL_B32 [[COPY5]](i32), [[S_MOV_B32_3]](i32), implicit-def dead $scc
- ; CHECK45-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sreg_64(i64) = REG_SEQUENCE [[C]](i32), %subreg.sub0, [[S_LSHL_B32_1]](i32), %subreg.sub1
- ; CHECK45-NEXT: [[S_OR_B64_1:%[0-9]+]]:sreg_64(i64) = S_OR_B64 [[S_LSHR_B64_]](i64), [[REG_SEQUENCE1]](i64), implicit-def dead $scc
- ; CHECK45-NEXT: [[S_OR_B64_2:%[0-9]+]]:sreg_64(i64) = S_OR_B64 [[S_OR_B64_1]](i64), [[REG_SEQUENCE2]](i64), implicit-def dead $scc
- ; CHECK45-NEXT: [[COPY6:%[0-9]+]]:sreg_32(i32) = COPY [[S_OR_B64_]].sub0(i64)
- ; CHECK45-NEXT: [[COPY7:%[0-9]+]]:sreg_32(i32) = COPY [[S_OR_B64_]].sub1(i64)
- ; CHECK45-NEXT: [[COPY8:%[0-9]+]]:sreg_32(i32) = COPY [[S_OR_B64_2]].sub0(i64)
- ; CHECK45-NEXT: [[COPY9:%[0-9]+]]:sreg_32(i32) = COPY [[S_OR_B64_2]].sub1(i64)
- ; CHECK45-NEXT: [[COPY10:%[0-9]+]]:vgpr_32(i32) = COPY [[COPY6]](i32)
- ; CHECK45-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY10]](i32), implicit $exec
- ; CHECK45-NEXT: $sgpr0 = COPY [[V_READFIRSTLANE_B32_]](i32)
- ; CHECK45-NEXT: [[COPY11:%[0-9]+]]:vgpr_32(i32) = COPY [[COPY7]](i32)
- ; CHECK45-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY11]](i32), implicit $exec
- ; CHECK45-NEXT: $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]](i32)
- ; CHECK45-NEXT: [[COPY12:%[0-9]+]]:vgpr_32(i32) = COPY [[COPY8]](i32)
- ; CHECK45-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY12]](i32), implicit $exec
- ; CHECK45-NEXT: $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]](i32)
- ; CHECK45-NEXT: [[COPY13:%[0-9]+]]:vgpr_32(i32) = COPY [[COPY9]](i32)
- ; CHECK45-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY13]](i32), implicit $exec
- ; CHECK45-NEXT: $sgpr3 = COPY [[V_READFIRSTLANE_B32_3]](i32)
+ ; CHECK45-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; CHECK45-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+ ; CHECK45-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+ ; CHECK45-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; CHECK45-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; CHECK45-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; CHECK45-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY4]], %subreg.sub1
+ ; CHECK45-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; CHECK45-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; CHECK45-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 25
+ ; CHECK45-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY3]], [[S_MOV_B32_1]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; CHECK45-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_2]], %subreg.sub0, [[S_LSHL_B32_]], %subreg.sub1
+ ; CHECK45-NEXT: [[S_OR_B64_:%[0-9]+]]:sreg_64 = S_OR_B64 [[REG_SEQUENCE]], [[REG_SEQUENCE2]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 7
+ ; CHECK45-NEXT: [[S_LSHR_B64_:%[0-9]+]]:sreg_64 = S_LSHR_B64 [[REG_SEQUENCE1]], [[S_MOV_B32_3]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 12
+ ; CHECK45-NEXT: [[S_LSHL_B32_1:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY2]], [[S_MOV_B32_4]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_LSHL_B32_1]], %subreg.sub1
+ ; CHECK45-NEXT: [[S_MOV_B32_5:%[0-9]+]]:sreg_32 = S_MOV_B32 28
+ ; CHECK45-NEXT: [[S_LSHL_B32_2:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY5]], [[S_MOV_B32_5]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_LSHL_B32_2]], %subreg.sub1
+ ; CHECK45-NEXT: [[S_OR_B64_1:%[0-9]+]]:sreg_64 = S_OR_B64 [[S_LSHR_B64_]], [[REG_SEQUENCE3]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[S_OR_B64_2:%[0-9]+]]:sreg_64 = S_OR_B64 [[S_OR_B64_1]], [[REG_SEQUENCE4]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY [[S_OR_B64_]].sub0
+ ; CHECK45-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY [[S_OR_B64_]].sub1
+ ; CHECK45-NEXT: [[COPY8:%[0-9]+]]:sreg_32 = COPY [[S_OR_B64_2]].sub0
+ ; CHECK45-NEXT: [[COPY9:%[0-9]+]]:sreg_32 = COPY [[S_OR_B64_2]].sub1
+ ; CHECK45-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+ ; CHECK45-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
+ ; CHECK45-NEXT: $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+ ; CHECK45-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[COPY7]]
+ ; CHECK45-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
+ ; CHECK45-NEXT: $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+ ; CHECK45-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY8]]
+ ; CHECK45-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY12]], implicit $exec
+ ; CHECK45-NEXT: $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
+ ; CHECK45-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[COPY9]]
+ ; CHECK45-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY13]], implicit $exec
+ ; CHECK45-NEXT: $sgpr3 = COPY [[V_READFIRSTLANE_B32_3]]
; CHECK45-NEXT: SI_RETURN_TO_EPILOG implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $sgpr3
%rsrc = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p0(ptr %p, i16 %stride, i64 %numVals, i32 %flags)
ret ptr addrspace(8) %rsrc
@@ -342,39 +340,38 @@ define amdgpu_ps float @general_case_load(ptr inreg %p, i16 inreg %stride, i64 i
; CHECK45: bb.1 (%ir-block.0):
; CHECK45-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4, $sgpr5
; CHECK45-NEXT: {{ $}}
- ; CHECK45-NEXT: [[COPY:%[0-9]+]]:sgpr(i32) = COPY $sgpr0
- ; CHECK45-NEXT: [[COPY1:%[0-9]+]]:sgpr(i32) = COPY $sgpr1
- ; CHECK45-NEXT: [[MV:%[0-9]+]]:sgpr(p0) = G_MERGE_VALUES [[COPY]](i32), [[COPY1]](i32)
- ; CHECK45-NEXT: [[COPY2:%[0-9]+]]:sreg_32(i32) = COPY $sgpr2
- ; CHECK45-NEXT: [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
- ; CHECK45-NEXT: [[COPY4:%[0-9]+]]:sgpr(s32) = COPY $sgpr4
- ; CHECK45-NEXT: [[MV1:%[0-9]+]]:sreg_64(i64) = G_MERGE_VALUES [[COPY3]](s32), [[COPY4]](s32)
- ; CHECK45-NEXT: [[COPY5:%[0-9]+]]:sreg_32(i32) = COPY $sgpr5
- ; CHECK45-NEXT: [[C:%[0-9]+]]:sreg_32(i32) = G_CONSTANT i32 0
- ; CHECK45-NEXT: [[PTRTOINT:%[0-9]+]]:sreg_64(i64) = G_PTRTOINT [[MV]](p0)
- ; CHECK45-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 25
- ; CHECK45-NEXT: [[SHL:%[0-9]+]]:sreg_32(s32) = G_SHL [[COPY3]], [[C1]](s32)
- ; CHECK45-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32(s32) = S_MOV_B32 0
- ; CHECK45-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64(i64) = REG_SEQUENCE [[S_MOV_B32_]](s32), %subreg.sub0, [[SHL]](s32), %subreg.sub1
- ; CHECK45-NEXT: [[S_OR_B64_:%[0-9]+]]:sreg_64(i64) = S_OR_B64 [[PTRTOINT]](i64), [[REG_SEQUENCE]](i64), implicit-def dead $scc
- ; CHECK45-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32(i32) = S_MOV_B32 7
- ; CHECK45-NEXT: [[S_LSHR_B64_:%[0-9]+]]:sreg_64(i64) = S_LSHR_B64 [[MV1]](i64), [[S_MOV_B32_1]](i32), implicit-def dead $scc
- ; CHECK45-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32(i32) = S_MOV_B32 12
- ; CHECK45-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32(i32) = S_LSHL_B32 [[COPY2]](i32), [[S_MOV_B32_2]](i32), implicit-def dead $scc
- ; CHECK45-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sreg_64(i64) = REG_SEQUENCE [[C]](i32), %subreg.sub0, [[S_LSHL_B32_]](i32), %subreg.sub1
- ; CHECK45-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32(i32) = S_MOV_B32 28
- ; CHECK45-NEXT: [[S_LSHL_B32_1:%[0-9]+]]:sreg_32(i32) = S_LSHL_B32 [[COPY5]](i32), [[S_MOV_B32_3]](i32), implicit-def dead $scc
- ; CHECK45-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sreg_64(i64) = REG_SEQUENCE [[C]](i32), %subreg.sub0, [[S_LSHL_B32_1]](i32), %subreg.sub1
- ; CHECK45-NEXT: [[S_OR_B64_1:%[0-9]+]]:sreg_64(i64) = S_OR_B64 [[S_LSHR_B64_]](i64), [[REG_SEQUENCE1]](i64), implicit-def dead $scc
- ; CHECK45-NEXT: [[S_OR_B64_2:%[0-9]+]]:sreg_64(i64) = S_OR_B64 [[S_OR_B64_1]](i64), [[REG_SEQUENCE2]](i64), implicit-def dead $scc
- ; CHECK45-NEXT: [[COPY6:%[0-9]+]]:sreg_32(i32) = COPY [[S_OR_B64_]].sub0(i64)
- ; CHECK45-NEXT: [[COPY7:%[0-9]+]]:sreg_32(i32) = COPY [[S_OR_B64_]].sub1(i64)
- ; CHECK45-NEXT: [[COPY8:%[0-9]+]]:sreg_32(i32) = COPY [[S_OR_B64_2]].sub0(i64)
- ; CHECK45-NEXT: [[COPY9:%[0-9]+]]:sreg_32(i32) = COPY [[S_OR_B64_2]].sub1(i64)
- ; CHECK45-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128(<4 x i32>) = REG_SEQUENCE [[COPY6]](i32), %subreg.sub0, [[COPY7]](i32), %subreg.sub1, [[COPY8]](i32), %subreg.sub2, [[COPY9]](i32), %subreg.sub3
- ; CHECK45-NEXT: [[COPY10:%[0-9]+]]:vgpr_32(i32) = COPY [[C]](i32)
- ; CHECK45-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_IDXEN:%[0-9]+]]:vgpr_32(f32) = BUFFER_LOAD_DWORD_VBUFFER_IDXEN [[COPY10]](i32), [[REG_SEQUENCE3]](<4 x i32>), $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (f32) from %ir.rsrc, align 1, addrspace 8)
- ; CHECK45-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_IDXEN]](f32)
+ ; CHECK45-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; CHECK45-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+ ; CHECK45-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+ ; CHECK45-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; CHECK45-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; CHECK45-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; CHECK45-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY4]], %subreg.sub1
+ ; CHECK45-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; CHECK45-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; CHECK45-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 25
+ ; CHECK45-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY3]], [[S_MOV_B32_1]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; CHECK45-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_2]], %subreg.sub0, [[S_LSHL_B32_]], %subreg.sub1
+ ; CHECK45-NEXT: [[S_OR_B64_:%[0-9]+]]:sreg_64 = S_OR_B64 [[REG_SEQUENCE]], [[REG_SEQUENCE2]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 7
+ ; CHECK45-NEXT: [[S_LSHR_B64_:%[0-9]+]]:sreg_64 = S_LSHR_B64 [[REG_SEQUENCE1]], [[S_MOV_B32_3]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 12
+ ; CHECK45-NEXT: [[S_LSHL_B32_1:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY2]], [[S_MOV_B32_4]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_LSHL_B32_1]], %subreg.sub1
+ ; CHECK45-NEXT: [[S_MOV_B32_5:%[0-9]+]]:sreg_32 = S_MOV_B32 28
+ ; CHECK45-NEXT: [[S_LSHL_B32_2:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY5]], [[S_MOV_B32_5]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_LSHL_B32_2]], %subreg.sub1
+ ; CHECK45-NEXT: [[S_OR_B64_1:%[0-9]+]]:sreg_64 = S_OR_B64 [[S_LSHR_B64_]], [[REG_SEQUENCE3]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[S_OR_B64_2:%[0-9]+]]:sreg_64 = S_OR_B64 [[S_OR_B64_1]], [[REG_SEQUENCE4]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY [[S_OR_B64_]].sub0
+ ; CHECK45-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY [[S_OR_B64_]].sub1
+ ; CHECK45-NEXT: [[COPY8:%[0-9]+]]:sreg_32 = COPY [[S_OR_B64_2]].sub0
+ ; CHECK45-NEXT: [[COPY9:%[0-9]+]]:sreg_32 = COPY [[S_OR_B64_2]].sub1
+ ; CHECK45-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1, [[COPY8]], %subreg.sub2, [[COPY9]], %subreg.sub3
+ ; CHECK45-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; CHECK45-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_IDXEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_IDXEN [[COPY10]], [[REG_SEQUENCE5]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (f32) from %ir.rsrc, align 1, addrspace 8)
+ ; CHECK45-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_IDXEN]]
; CHECK45-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%rsrc = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p0(ptr %p, i16 %stride, i64 %numVals, i32 %flags)
%value = call float @llvm.amdgcn.struct.ptr.buffer.load(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/lshr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/lshr.ll
index 864037d713ad2..df0f8a9e6a9cb 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/lshr.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/lshr.ll
@@ -1,9 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu6.00-amd-amdpal < %s | FileCheck -check-prefixes=GCN,GFX6 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-amd-amdpal < %s | FileCheck -check-prefixes=GCN,GFX8 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00-amd-amdpal < %s | FileCheck -check-prefixes=GCN,GFX9 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-amd-amdpal < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11 %s
+; RUN: llc -global-isel -mtriple=amdgpu6.00-amd-amdpal < %s | FileCheck -check-prefixes=GCN,GFX6 %s
+; RUN: llc -global-isel -mtriple=amdgpu8.03-amd-amdpal < %s | FileCheck -check-prefixes=GCN,GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00-amd-amdpal < %s | FileCheck -check-prefixes=GCN,GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgpu10.10-amd-amdpal < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11 %s
define i8 @v_lshr_i8(i8 %value, i8 %amount) {
; GFX6-LABEL: v_lshr_i8:
@@ -1752,27 +1752,67 @@ define i65 @v_lshr_i65(i65 %value, i65 %amount) {
}
define i65 @v_lshr_i65_33(i65 %value) {
-; GCN-LABEL: v_lshr_i65_33:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_alignbit_b32 v0, v2, v1, 1
-; GCN-NEXT: v_mov_b32_e32 v1, 0
-; GCN-NEXT: v_mov_b32_e32 v2, 0
-; GCN-NEXT: s_setpc_b64 s[30:31]
+; GFX6-LABEL: v_lshr_i65_33:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v0, 1
+; GFX6-NEXT: v_mov_b32_e32 v3, v1
+; GFX6-NEXT: v_mov_b32_e32 v1, 0
+; GFX6-NEXT: v_and_b32_e32 v0, 1, v2
+; GFX6-NEXT: v_lshl_b64 v[0:1], v[0:1], 31
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 1, v3
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT: v_mov_b32_e32 v2, 0
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_lshr_i65_33:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v0, 1
+; GFX8-NEXT: v_mov_b32_e32 v3, v1
+; GFX8-NEXT: v_mov_b32_e32 v1, 0
+; GFX8-NEXT: v_and_b32_e32 v0, 1, v2
+; GFX8-NEXT: v_lshlrev_b64 v[0:1], 31, v[0:1]
+; GFX8-NEXT: v_lshrrev_b32_e32 v2, 1, v3
+; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT: v_mov_b32_e32 v2, 0
+; GFX8-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_lshr_i65_33:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v0, 1
+; GFX9-NEXT: v_mov_b32_e32 v3, v1
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: v_and_b32_e32 v0, 1, v2
+; GFX9-NEXT: v_lshlrev_b64 v[0:1], 31, v[0:1]
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 1, v3
+; GFX9-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_lshr_i65_33:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_alignbit_b32 v0, v2, v1, 1
+; GFX10-NEXT: v_mov_b32_e32 v0, 1
+; GFX10-NEXT: v_mov_b32_e32 v3, v1
; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: v_and_b32_e32 v0, 1, v2
+; GFX10-NEXT: v_lshrrev_b32_e32 v2, 1, v3
+; GFX10-NEXT: v_lshlrev_b64 v[0:1], 31, v[0:1]
+; GFX10-NEXT: v_or_b32_e32 v0, v0, v2
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_lshr_i65_33:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_alignbit_b32 v0, v2, v1, 1
-; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 0
+; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v0, 1
+; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 1, v2
+; GFX11-NEXT: v_lshrrev_b32_e32 v2, 1, v3
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], 31, v[0:1]
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX11-NEXT: v_mov_b32_e32 v2, 0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = lshr i65 %value, 33
ret i65 %result
@@ -1829,20 +1869,20 @@ define amdgpu_ps i65 @s_lshr_i65(i65 inreg %value, i65 inreg %amount) {
define amdgpu_ps i65 @s_lshr_i65_33(i65 inreg %value) {
; GCN-LABEL: s_lshr_i65_33:
; GCN: ; %bb.0:
-; GCN-NEXT: s_mov_b32 s3, s2
-; GCN-NEXT: s_mov_b32 s2, s1
-; GCN-NEXT: s_lshr_b64 s[0:1], s[2:3], 1
-; GCN-NEXT: s_mov_b32 s1, 0
+; GCN-NEXT: s_and_b64 s[2:3], s[2:3], 1
+; GCN-NEXT: s_lshr_b32 s4, s1, 1
+; GCN-NEXT: s_lshl_b64 s[0:1], s[2:3], 31
+; GCN-NEXT: s_or_b32 s0, s0, s4
; GCN-NEXT: s_mov_b32 s2, 0
; GCN-NEXT: ; return to shader part epilog
;
; GFX10PLUS-LABEL: s_lshr_i65_33:
; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: s_mov_b32 s3, s2
-; GFX10PLUS-NEXT: s_mov_b32 s2, s1
-; GFX10PLUS-NEXT: s_lshr_b64 s[0:1], s[2:3], 1
-; GFX10PLUS-NEXT: s_mov_b32 s1, 0
+; GFX10PLUS-NEXT: s_and_b64 s[2:3], s[2:3], 1
+; GFX10PLUS-NEXT: s_lshr_b32 s4, s1, 1
+; GFX10PLUS-NEXT: s_lshl_b64 s[0:1], s[2:3], 31
; GFX10PLUS-NEXT: s_mov_b32 s2, 0
+; GFX10PLUS-NEXT: s_or_b32 s0, s0, s4
; GFX10PLUS-NEXT: ; return to shader part epilog
%result = lshr i65 %value, 33
ret i65 %result
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/minmaxabs-i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/minmaxabs-i64.ll
index 78c4b2d4c0737..e3564933dc75f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/minmaxabs-i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/minmaxabs-i64.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.50-unknown-amdpal < %s | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgpu12.50-unknown-amdpal < %s | FileCheck %s
declare i64 @llvm.umin.i64(i64, i64)
declare i64 @llvm.umax.i64(i64, i64)
@@ -195,12 +195,12 @@ define amdgpu_ps i64 @test_abs_i64_s(i64 inreg %a) {
; CHECK-NEXT: global_wb
; CHECK-NEXT: v_nop
; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; CHECK-NEXT: s_sub_nc_u64 s[2:3], 0, s[0:1]
-; CHECK-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; CHECK-NEXT: v_max_i64 v[0:1], s[0:1], s[2:3]
-; CHECK-NEXT: v_readfirstlane_b32 s0, v0
-; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; CHECK-NEXT: v_readfirstlane_b32 s1, v1
+; CHECK-NEXT: s_ashr_i32 s2, s1, 31
+; CHECK-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; CHECK-NEXT: s_mov_b32 s3, s2
+; CHECK-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[2:3]
+; CHECK-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; CHECK-NEXT: s_xor_b64 s[0:1], s[0:1], s[2:3]
; CHECK-NEXT: ; return to shader part epilog
%r = call i64 @llvm.abs.i64(i64 %a, i1 0)
ret i64 %r
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll
index 7f4f7c7575db1..7375704ae8c42 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -amdgpu-codegenprepare-disable-idiv-expansion=1 -mtriple=amdgpu7.01-mesa-mesa3d -denormal-fp-math-f32=preserve-sign < %s | FileCheck -check-prefixes=CHECK,GISEL %s
-; RUN: llc -global-isel -global-isel-abort=2 -amdgpu-codegenprepare-disable-idiv-expansion=0 -mtriple=amdgpu7.01-mesa-mesa3d -denormal-fp-math-f32=preserve-sign < %s | FileCheck -check-prefixes=CHECK,CGP %s
+; RUN: llc -global-isel -amdgpu-codegenprepare-disable-idiv-expansion=1 -mtriple=amdgpu7.01-mesa-mesa3d -denormal-fp-math-f32=preserve-sign < %s | FileCheck -check-prefixes=CHECK,GISEL %s
+; RUN: llc -global-isel -amdgpu-codegenprepare-disable-idiv-expansion=0 -mtriple=amdgpu7.01-mesa-mesa3d -denormal-fp-math-f32=preserve-sign < %s | FileCheck -check-prefixes=CHECK,CGP %s
; The same 32-bit expansion is implemented in the legalizer and in AMDGPUCodeGenPrepare.
@@ -193,136 +193,183 @@ define amdgpu_ps i64 @s_sdiv_i64(i64 inreg %num, i64 inreg %den) {
; CHECK-LABEL: s_sdiv_i64:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_or_b64 s[0:1], s[2:3], s[4:5]
-; CHECK-NEXT: s_cmp_lg_u32 s1, 0
+; CHECK-NEXT: s_mov_b32 s0, 0
+; CHECK-NEXT: v_cmp_ne_u64_e64 s[0:1], s[0:1], 0
+; CHECK-NEXT: s_mov_b32 s6, 1
+; CHECK-NEXT: s_or_b64 s[0:1], s[0:1], s[0:1]
; CHECK-NEXT: s_cbranch_scc0 .LBB1_2
; CHECK-NEXT: ; %bb.1:
-; CHECK-NEXT: s_ashr_i32 s0, s5, 31
-; CHECK-NEXT: s_add_u32 s6, s4, s0
+; CHECK-NEXT: s_ashr_i32 s0, s3, 31
+; CHECK-NEXT: s_ashr_i32 s6, s5, 31
+; CHECK-NEXT: s_add_u32 s10, s2, s0
+; CHECK-NEXT: s_addc_u32 s11, s3, s0
+; CHECK-NEXT: s_add_u32 s8, s4, s6
+; CHECK-NEXT: s_mov_b32 s7, s6
+; CHECK-NEXT: s_addc_u32 s9, s5, s6
+; CHECK-NEXT: s_xor_b64 s[8:9], s[8:9], s[6:7]
+; CHECK-NEXT: v_cvt_f32_u32_e32 v0, s8
+; CHECK-NEXT: v_cvt_f32_u32_e32 v1, s9
; CHECK-NEXT: s_mov_b32 s1, s0
-; CHECK-NEXT: s_addc_u32 s7, s5, s0
-; CHECK-NEXT: s_xor_b64 s[6:7], s[6:7], s[0:1]
-; CHECK-NEXT: v_cvt_f32_u32_e32 v0, s6
-; CHECK-NEXT: v_cvt_f32_u32_e32 v1, s7
-; CHECK-NEXT: s_sub_u32 s5, 0, s6
-; CHECK-NEXT: s_subb_u32 s10, 0, s7
-; CHECK-NEXT: v_madmk_f32 v0, v1, 0x4f800000, v0
-; CHECK-NEXT: v_rcp_f32_e32 v0, v0
+; CHECK-NEXT: s_xor_b64 s[10:11], s[10:11], s[0:1]
+; CHECK-NEXT: s_sub_u32 s3, 0, s8
+; CHECK-NEXT: v_mac_f32_e32 v0, 0x4f800000, v1
+; CHECK-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; CHECK-NEXT: s_subb_u32 s5, 0, s9
; CHECK-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0
; CHECK-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0
; CHECK-NEXT: v_trunc_f32_e32 v1, v1
-; CHECK-NEXT: v_madmk_f32 v0, v1, 0xcf800000, v0
-; CHECK-NEXT: v_cvt_u32_f32_e32 v7, v1
-; CHECK-NEXT: v_cvt_u32_f32_e32 v8, v0
-; CHECK-NEXT: v_mul_lo_u32 v2, s5, v7
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[8:9], s5, v8, 0
-; CHECK-NEXT: v_mul_lo_u32 v3, s10, v8
-; CHECK-NEXT: v_add_i32_e32 v1, vcc, v1, v2
-; CHECK-NEXT: v_add_i32_e32 v5, vcc, v1, v3
-; CHECK-NEXT: v_mul_hi_u32 v9, v8, v0
-; CHECK-NEXT: v_mad_u64_u32 v[1:2], s[8:9], v8, v5, 0
-; CHECK-NEXT: v_mad_u64_u32 v[3:4], s[8:9], v7, v0, 0
-; CHECK-NEXT: v_mad_u64_u32 v[5:6], s[8:9], v7, v5, 0
-; CHECK-NEXT: v_add_i32_e32 v0, vcc, v9, v1
-; CHECK-NEXT: v_addc_u32_e32 v1, vcc, 0, v2, vcc
-; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v3
-; CHECK-NEXT: v_addc_u32_e32 v0, vcc, v1, v4, vcc
-; CHECK-NEXT: v_addc_u32_e32 v1, vcc, 0, v6, vcc
-; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v5
-; CHECK-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; CHECK-NEXT: v_add_i32_e32 v5, vcc, v8, v0
-; CHECK-NEXT: v_addc_u32_e32 v6, vcc, v7, v1, vcc
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[8:9], s5, v5, 0
-; CHECK-NEXT: v_mul_lo_u32 v2, s5, v6
-; CHECK-NEXT: v_mul_lo_u32 v3, s10, v5
-; CHECK-NEXT: s_ashr_i32 s10, s3, 31
-; CHECK-NEXT: v_mul_hi_u32 v8, v5, v0
-; CHECK-NEXT: v_add_i32_e32 v1, vcc, v2, v1
-; CHECK-NEXT: v_add_i32_e32 v7, vcc, v3, v1
-; CHECK-NEXT: v_mad_u64_u32 v[1:2], s[8:9], v5, v7, 0
-; CHECK-NEXT: v_mad_u64_u32 v[3:4], s[8:9], v6, v0, 0
-; CHECK-NEXT: s_mov_b32 s11, s10
-; CHECK-NEXT: v_add_i32_e32 v8, vcc, v8, v1
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[8:9], v6, v7, 0
-; CHECK-NEXT: v_addc_u32_e32 v2, vcc, 0, v2, vcc
-; CHECK-NEXT: v_add_i32_e32 v3, vcc, v8, v3
-; CHECK-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc
-; CHECK-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; CHECK-NEXT: v_add_i32_e32 v0, vcc, v2, v0
-; CHECK-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; CHECK-NEXT: s_add_u32 s8, s2, s10
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v5, v0
-; CHECK-NEXT: s_addc_u32 s9, s3, s10
-; CHECK-NEXT: v_addc_u32_e32 v3, vcc, v6, v1, vcc
-; CHECK-NEXT: s_xor_b64 s[12:13], s[8:9], s[10:11]
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[8:9], s12, v3, 0
-; CHECK-NEXT: v_mul_hi_u32 v4, s12, v2
-; CHECK-NEXT: v_readfirstlane_b32 s3, v1
+; CHECK-NEXT: v_mac_f32_e32 v0, 0xcf800000, v1
+; CHECK-NEXT: v_cvt_u32_f32_e32 v0, v0
+; CHECK-NEXT: v_cvt_u32_f32_e32 v1, v1
+; CHECK-NEXT: v_mul_hi_u32 v2, s3, v0
+; CHECK-NEXT: v_readfirstlane_b32 s13, v0
+; CHECK-NEXT: s_mul_i32 s15, s3, s13
+; CHECK-NEXT: v_readfirstlane_b32 s12, v1
+; CHECK-NEXT: v_readfirstlane_b32 s16, v2
+; CHECK-NEXT: v_mul_hi_u32 v2, v0, s15
+; CHECK-NEXT: s_mul_i32 s14, s3, s12
+; CHECK-NEXT: s_add_u32 s14, s14, s16
+; CHECK-NEXT: s_mul_i32 s16, s5, s13
+; CHECK-NEXT: s_add_u32 s14, s16, s14
+; CHECK-NEXT: v_readfirstlane_b32 s16, v2
+; CHECK-NEXT: s_mul_i32 s17, s13, s14
+; CHECK-NEXT: v_mul_hi_u32 v2, v1, s15
+; CHECK-NEXT: v_mul_hi_u32 v0, v0, s14
+; CHECK-NEXT: s_mul_i32 s18, s12, s15
+; CHECK-NEXT: s_add_u32 s16, s16, s17
+; CHECK-NEXT: s_cselect_b32 s17, 1, 0
+; CHECK-NEXT: s_add_u32 s16, s16, s18
+; CHECK-NEXT: s_cselect_b32 s15, 1, 0
+; CHECK-NEXT: s_add_i32 s17, s17, s15
+; CHECK-NEXT: v_readfirstlane_b32 s15, v2
+; CHECK-NEXT: v_readfirstlane_b32 s16, v0
+; CHECK-NEXT: s_mul_i32 s18, s12, s14
+; CHECK-NEXT: s_add_u32 s15, s15, s16
+; CHECK-NEXT: s_cselect_b32 s16, 1, 0
+; CHECK-NEXT: s_add_u32 s15, s15, s18
+; CHECK-NEXT: s_cselect_b32 s18, 1, 0
+; CHECK-NEXT: v_mul_hi_u32 v0, v1, s14
+; CHECK-NEXT: s_add_i32 s16, s16, s18
+; CHECK-NEXT: s_add_u32 s14, s15, s17
+; CHECK-NEXT: s_cselect_b32 s15, 1, 0
+; CHECK-NEXT: s_add_i32 s16, s16, s15
+; CHECK-NEXT: v_readfirstlane_b32 s15, v0
+; CHECK-NEXT: s_add_i32 s15, s15, s16
+; CHECK-NEXT: s_add_u32 s13, s13, s14
+; CHECK-NEXT: v_mov_b32_e32 v0, s13
+; CHECK-NEXT: v_mul_hi_u32 v0, s3, v0
+; CHECK-NEXT: s_mul_i32 s14, s3, s13
+; CHECK-NEXT: s_addc_u32 s12, s12, s15
+; CHECK-NEXT: s_mul_i32 s3, s3, s12
+; CHECK-NEXT: v_readfirstlane_b32 s15, v0
+; CHECK-NEXT: v_mov_b32_e32 v0, s14
+; CHECK-NEXT: v_mul_hi_u32 v1, s13, v0
+; CHECK-NEXT: s_add_u32 s3, s3, s15
+; CHECK-NEXT: s_mul_i32 s5, s5, s13
+; CHECK-NEXT: s_add_u32 s3, s5, s3
+; CHECK-NEXT: v_readfirstlane_b32 s5, v1
+; CHECK-NEXT: v_mov_b32_e32 v1, s3
+; CHECK-NEXT: s_mul_i32 s15, s13, s3
+; CHECK-NEXT: v_mul_hi_u32 v0, s12, v0
+; CHECK-NEXT: v_mul_hi_u32 v2, s13, v1
+; CHECK-NEXT: s_mul_i32 s14, s12, s14
+; CHECK-NEXT: s_add_u32 s5, s5, s15
+; CHECK-NEXT: s_cselect_b32 s15, 1, 0
+; CHECK-NEXT: s_add_u32 s5, s5, s14
+; CHECK-NEXT: s_cselect_b32 s5, 1, 0
+; CHECK-NEXT: s_add_i32 s15, s15, s5
; CHECK-NEXT: v_readfirstlane_b32 s5, v0
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[8:9], s13, v2, 0
-; CHECK-NEXT: v_readfirstlane_b32 s8, v4
-; CHECK-NEXT: s_add_u32 s5, s8, s5
-; CHECK-NEXT: s_addc_u32 s3, 0, s3
-; CHECK-NEXT: v_readfirstlane_b32 s14, v1
-; CHECK-NEXT: v_mad_u64_u32 v[1:2], s[8:9], s13, v3, 0
-; CHECK-NEXT: v_readfirstlane_b32 s8, v0
-; CHECK-NEXT: s_add_u32 s5, s5, s8
-; CHECK-NEXT: s_addc_u32 s3, s3, s14
-; CHECK-NEXT: v_readfirstlane_b32 s5, v2
-; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_readfirstlane_b32 s8, v1
-; CHECK-NEXT: s_add_u32 s3, s3, s8
-; CHECK-NEXT: v_mov_b32_e32 v0, s3
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[8:9], s6, v0, 0
-; CHECK-NEXT: s_addc_u32 s5, 0, s5
-; CHECK-NEXT: s_mul_i32 s14, s6, s5
-; CHECK-NEXT: s_mov_b64 s[8:9], 0
-; CHECK-NEXT: v_readfirstlane_b32 s15, v1
-; CHECK-NEXT: s_add_i32 s14, s15, s14
-; CHECK-NEXT: s_mul_i32 s15, s7, s3
-; CHECK-NEXT: s_add_i32 s16, s14, s15
-; CHECK-NEXT: s_sub_i32 s17, s13, s16
+; CHECK-NEXT: v_readfirstlane_b32 s14, v2
+; CHECK-NEXT: s_mul_i32 s3, s12, s3
+; CHECK-NEXT: s_add_u32 s5, s5, s14
+; CHECK-NEXT: s_cselect_b32 s14, 1, 0
+; CHECK-NEXT: s_add_u32 s3, s5, s3
+; CHECK-NEXT: s_cselect_b32 s5, 1, 0
+; CHECK-NEXT: v_mul_hi_u32 v0, s12, v1
+; CHECK-NEXT: s_add_i32 s5, s14, s5
+; CHECK-NEXT: s_add_u32 s3, s3, s15
+; CHECK-NEXT: s_cselect_b32 s14, 1, 0
+; CHECK-NEXT: s_add_i32 s5, s5, s14
; CHECK-NEXT: v_readfirstlane_b32 s14, v0
-; CHECK-NEXT: s_sub_u32 s12, s12, s14
-; CHECK-NEXT: s_cselect_b64 s[14:15], -1, 0
-; CHECK-NEXT: s_subb_u32 s17, s17, s7
-; CHECK-NEXT: s_sub_u32 s18, s12, s6
-; CHECK-NEXT: s_subb_u32 s17, s17, 0
-; CHECK-NEXT: s_cmp_ge_u32 s17, s7
-; CHECK-NEXT: s_cselect_b32 s19, -1, 0
-; CHECK-NEXT: s_cmp_ge_u32 s18, s6
-; CHECK-NEXT: s_cselect_b32 s18, -1, 0
-; CHECK-NEXT: s_cmp_eq_u32 s17, s7
-; CHECK-NEXT: s_cselect_b32 s17, s18, s19
-; CHECK-NEXT: s_add_u32 s18, s3, 1
-; CHECK-NEXT: s_addc_u32 s19, s5, 0
-; CHECK-NEXT: s_add_u32 s20, s3, 2
-; CHECK-NEXT: s_addc_u32 s21, s5, 0
-; CHECK-NEXT: s_cmp_lg_u32 s17, 0
-; CHECK-NEXT: s_cselect_b32 s17, s20, s18
-; CHECK-NEXT: s_cselect_b32 s18, s21, s19
-; CHECK-NEXT: s_or_b32 s14, s14, s15
-; CHECK-NEXT: s_subb_u32 s13, s13, s16
-; CHECK-NEXT: s_cmp_ge_u32 s13, s7
-; CHECK-NEXT: s_cselect_b32 s14, -1, 0
-; CHECK-NEXT: s_cmp_ge_u32 s12, s6
-; CHECK-NEXT: s_cselect_b32 s6, -1, 0
-; CHECK-NEXT: s_cmp_eq_u32 s13, s7
-; CHECK-NEXT: s_cselect_b32 s6, s6, s14
-; CHECK-NEXT: s_cmp_lg_u32 s6, 0
-; CHECK-NEXT: s_cselect_b32 s7, s18, s5
-; CHECK-NEXT: s_cselect_b32 s6, s17, s3
-; CHECK-NEXT: s_xor_b64 s[0:1], s[10:11], s[0:1]
-; CHECK-NEXT: s_xor_b64 s[6:7], s[6:7], s[0:1]
+; CHECK-NEXT: s_add_i32 s14, s14, s5
+; CHECK-NEXT: s_add_u32 s3, s13, s3
+; CHECK-NEXT: v_mov_b32_e32 v0, s3
+; CHECK-NEXT: v_mul_hi_u32 v0, s10, v0
+; CHECK-NEXT: s_addc_u32 s5, s12, s14
+; CHECK-NEXT: v_mov_b32_e32 v1, s5
+; CHECK-NEXT: s_mul_i32 s13, s10, s5
+; CHECK-NEXT: v_readfirstlane_b32 s12, v0
+; CHECK-NEXT: v_mov_b32_e32 v0, s11
+; CHECK-NEXT: v_mul_hi_u32 v0, v0, s3
+; CHECK-NEXT: v_mul_hi_u32 v2, s10, v1
+; CHECK-NEXT: s_mul_i32 s14, s11, s3
+; CHECK-NEXT: s_add_u32 s12, s12, s13
+; CHECK-NEXT: s_cselect_b32 s13, 1, 0
+; CHECK-NEXT: s_add_u32 s12, s12, s14
+; CHECK-NEXT: s_cselect_b32 s3, 1, 0
+; CHECK-NEXT: s_add_i32 s13, s13, s3
+; CHECK-NEXT: v_readfirstlane_b32 s3, v0
+; CHECK-NEXT: v_readfirstlane_b32 s12, v2
+; CHECK-NEXT: s_mul_i32 s5, s11, s5
+; CHECK-NEXT: s_add_u32 s3, s3, s12
+; CHECK-NEXT: s_cselect_b32 s12, 1, 0
+; CHECK-NEXT: s_add_u32 s3, s3, s5
+; CHECK-NEXT: s_cselect_b32 s5, 1, 0
+; CHECK-NEXT: s_add_i32 s5, s12, s5
+; CHECK-NEXT: s_add_u32 s12, s3, s13
+; CHECK-NEXT: v_mul_hi_u32 v0, s11, v1
+; CHECK-NEXT: v_mov_b32_e32 v1, s12
+; CHECK-NEXT: v_mul_hi_u32 v1, s8, v1
+; CHECK-NEXT: s_cselect_b32 s3, 1, 0
+; CHECK-NEXT: s_add_i32 s5, s5, s3
+; CHECK-NEXT: v_readfirstlane_b32 s3, v0
+; CHECK-NEXT: s_add_i32 s13, s3, s5
+; CHECK-NEXT: v_readfirstlane_b32 s5, v1
+; CHECK-NEXT: s_mul_i32 s14, s8, s13
+; CHECK-NEXT: s_add_u32 s5, s14, s5
+; CHECK-NEXT: s_mul_i32 s14, s9, s12
+; CHECK-NEXT: s_mul_i32 s3, s8, s12
+; CHECK-NEXT: s_add_u32 s5, s14, s5
+; CHECK-NEXT: s_sub_u32 s3, s10, s3
+; CHECK-NEXT: s_cselect_b32 s10, 1, 0
+; CHECK-NEXT: s_subb_u32 s14, s11, s5
+; CHECK-NEXT: s_sub_i32 s5, s11, s5
+; CHECK-NEXT: s_cmp_ge_u32 s14, s9
+; CHECK-NEXT: s_cselect_b32 s11, -1, 0
+; CHECK-NEXT: s_cmp_ge_u32 s3, s8
+; CHECK-NEXT: s_cselect_b32 s15, -1, 0
+; CHECK-NEXT: s_cmp_eq_u32 s14, s9
+; CHECK-NEXT: s_cselect_b32 s14, s15, s11
+; CHECK-NEXT: s_sub_u32 s3, s3, s8
+; CHECK-NEXT: s_cselect_b32 s11, 1, 0
+; CHECK-NEXT: s_cmp_lg_u32 s10, 0
+; CHECK-NEXT: s_subb_u32 s5, s5, s9
+; CHECK-NEXT: s_cmp_lg_u32 s11, 0
+; CHECK-NEXT: s_subb_u32 s5, s5, 0
+; CHECK-NEXT: s_add_u32 s10, s12, 1
+; CHECK-NEXT: s_addc_u32 s11, s13, 0
+; CHECK-NEXT: s_cmp_ge_u32 s5, s9
+; CHECK-NEXT: s_cselect_b32 s15, -1, 0
+; CHECK-NEXT: s_cmp_ge_u32 s3, s8
+; CHECK-NEXT: s_cselect_b32 s3, -1, 0
+; CHECK-NEXT: s_cmp_eq_u32 s5, s9
+; CHECK-NEXT: s_cselect_b32 s3, s3, s15
+; CHECK-NEXT: s_add_u32 s8, s10, 1
+; CHECK-NEXT: s_addc_u32 s9, s11, 0
+; CHECK-NEXT: s_cmp_lg_u32 s3, 0
+; CHECK-NEXT: s_cselect_b64 s[8:9], s[8:9], s[10:11]
+; CHECK-NEXT: s_cmp_lg_u32 s14, 0
+; CHECK-NEXT: s_cselect_b64 s[8:9], s[8:9], s[12:13]
+; CHECK-NEXT: s_xor_b64 s[0:1], s[0:1], s[6:7]
+; CHECK-NEXT: s_xor_b64 s[6:7], s[8:9], s[0:1]
; CHECK-NEXT: s_sub_u32 s0, s6, s0
-; CHECK-NEXT: s_subb_u32 s1, s7, s1
+; CHECK-NEXT: s_mov_b32 s6, 0
; CHECK-NEXT: s_branch .LBB1_3
; CHECK-NEXT: .LBB1_2:
-; CHECK-NEXT: s_mov_b64 s[8:9], -1
; CHECK-NEXT: ; implicit-def: $sgpr0_sgpr1
; CHECK-NEXT: .LBB1_3: ; %Flow
-; CHECK-NEXT: s_and_b64 s[6:7], s[8:9], exec
-; CHECK-NEXT: s_cselect_b32 s3, 1, 0
-; CHECK-NEXT: s_cmp_lg_u32 s3, 1
+; CHECK-NEXT: s_xor_b32 s1, s6, 1
+; CHECK-NEXT: s_cmp_lg_u32 s1, 0
; CHECK-NEXT: s_cbranch_scc1 .LBB1_5
; CHECK-NEXT: ; %bb.4:
; CHECK-NEXT: v_cvt_f32_u32_e32 v0, s4
@@ -330,29 +377,32 @@ define amdgpu_ps i64 @s_sdiv_i64(i64 inreg %num, i64 inreg %den) {
; CHECK-NEXT: v_rcp_iflag_f32_e32 v0, v0
; CHECK-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; CHECK-NEXT: v_cvt_u32_f32_e32 v0, v0
-; CHECK-NEXT: v_mul_lo_u32 v1, s0, v0
-; CHECK-NEXT: v_mul_hi_u32 v1, v0, v1
-; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v1
-; CHECK-NEXT: v_mul_hi_u32 v0, s2, v0
+; CHECK-NEXT: v_readfirstlane_b32 s1, v0
+; CHECK-NEXT: s_mul_i32 s0, s0, s1
+; CHECK-NEXT: v_mul_hi_u32 v0, v0, s0
; CHECK-NEXT: v_readfirstlane_b32 s0, v0
-; CHECK-NEXT: s_mul_i32 s0, s0, s4
-; CHECK-NEXT: s_sub_i32 s0, s2, s0
-; CHECK-NEXT: s_sub_i32 s1, s0, s4
-; CHECK-NEXT: v_add_i32_e32 v1, vcc, 1, v0
-; CHECK-NEXT: s_cmp_ge_u32 s0, s4
-; CHECK-NEXT: s_cselect_b64 vcc, -1, 0
-; CHECK-NEXT: s_cselect_b32 s0, s1, s0
-; CHECK-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; CHECK-NEXT: v_add_i32_e32 v1, vcc, 1, v0
-; CHECK-NEXT: s_cmp_ge_u32 s0, s4
-; CHECK-NEXT: s_cselect_b64 vcc, -1, 0
-; CHECK-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; CHECK-NEXT: s_branch .LBB1_6
-; CHECK-NEXT: .LBB1_5:
+; CHECK-NEXT: s_add_i32 s0, s1, s0
; CHECK-NEXT: v_mov_b32_e32 v0, s0
-; CHECK-NEXT: v_mov_b32_e32 v1, s1
-; CHECK-NEXT: .LBB1_6: ; %.split
+; CHECK-NEXT: v_mul_hi_u32 v0, s2, v0
+; CHECK-NEXT: s_mov_b32 s1, 0
; CHECK-NEXT: v_readfirstlane_b32 s0, v0
+; CHECK-NEXT: s_mul_i32 s3, s0, s4
+; CHECK-NEXT: s_sub_i32 s3, s2, s3
+; CHECK-NEXT: s_cmp_ge_u32 s3, s4
+; CHECK-NEXT: s_cselect_b32 s5, 1, 0
+; CHECK-NEXT: s_add_i32 s2, s0, 1
+; CHECK-NEXT: s_cmp_lg_u32 s5, 0
+; CHECK-NEXT: s_cselect_b32 s2, s2, s0
+; CHECK-NEXT: s_sub_i32 s0, s3, s4
+; CHECK-NEXT: s_cmp_lg_u32 s5, 0
+; CHECK-NEXT: s_cselect_b32 s0, s0, s3
+; CHECK-NEXT: s_cmp_ge_u32 s0, s4
+; CHECK-NEXT: s_cselect_b32 s4, 1, 0
+; CHECK-NEXT: s_add_i32 s0, s2, 1
+; CHECK-NEXT: s_mov_b32 s3, s1
+; CHECK-NEXT: s_cmp_lg_u32 s4, 0
+; CHECK-NEXT: s_cselect_b64 s[0:1], s[0:1], s[2:3]
+; CHECK-NEXT: .LBB1_5: ; %.split
; CHECK-NEXT: s_mov_b32 s1, s0
; CHECK-NEXT: ; return to shader part epilog
%result = sdiv i64 %num, %den
@@ -1000,20 +1050,35 @@ define i64 @v_sdiv_i64_pow2k_denom(i64 %num) {
}
define <2 x i64> @v_sdiv_v2i64_pow2k_denom(<2 x i64> %num) {
-; CHECK-LABEL: v_sdiv_v2i64_pow2k_denom:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_ashrrev_i32_e32 v4, 31, v1
-; CHECK-NEXT: v_lshrrev_b32_e32 v4, 20, v4
-; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v4
-; CHECK-NEXT: v_ashrrev_i32_e32 v4, 31, v3
-; CHECK-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; CHECK-NEXT: v_lshrrev_b32_e32 v4, 20, v4
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v4
-; CHECK-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; CHECK-NEXT: v_ashr_i64 v[0:1], v[0:1], 12
-; CHECK-NEXT: v_ashr_i64 v[2:3], v[2:3], 12
-; CHECK-NEXT: s_setpc_b64 s[30:31]
+; GISEL-LABEL: v_sdiv_v2i64_pow2k_denom:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT: v_ashrrev_i32_e32 v4, 31, v1
+; GISEL-NEXT: v_lshrrev_b32_e32 v4, 20, v4
+; GISEL-NEXT: v_ashrrev_i32_e32 v5, 31, v3
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v4
+; GISEL-NEXT: v_lshrrev_b32_e32 v5, 20, v5
+; GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v5
+; GISEL-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GISEL-NEXT: v_ashr_i64 v[0:1], v[0:1], 12
+; GISEL-NEXT: v_ashr_i64 v[2:3], v[2:3], 12
+; GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; CGP-LABEL: v_sdiv_v2i64_pow2k_denom:
+; CGP: ; %bb.0:
+; CGP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CGP-NEXT: v_ashrrev_i32_e32 v4, 31, v1
+; CGP-NEXT: v_lshrrev_b32_e32 v4, 20, v4
+; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v4
+; CGP-NEXT: v_ashrrev_i32_e32 v4, 31, v3
+; CGP-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CGP-NEXT: v_lshrrev_b32_e32 v4, 20, v4
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v4
+; CGP-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; CGP-NEXT: v_ashr_i64 v[0:1], v[0:1], 12
+; CGP-NEXT: v_ashr_i64 v[2:3], v[2:3], 12
+; CGP-NEXT: s_setpc_b64 s[30:31]
%result = sdiv <2 x i64> %num, <i64 4096, i64 4096>
ret <2 x i64> %result
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/sext_inreg.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/sext_inreg.ll
index c94a950c85dde..9edfc92609ace 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/sext_inreg.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/sext_inreg.ll
@@ -1,9 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu6.00-amd-amdpal < %s | FileCheck -check-prefixes=GCN,GFX6 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-amd-amdpal < %s | FileCheck -check-prefixes=GCN,GFX8 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00-amd-amdpal < %s | FileCheck -check-prefixes=GCN,GFX9 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-amd-amdpal < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11 %s
+; RUN: llc -global-isel -mtriple=amdgpu6.00-amd-amdpal < %s | FileCheck -check-prefixes=GCN,GFX6 %s
+; RUN: llc -global-isel -mtriple=amdgpu8.03-amd-amdpal < %s | FileCheck -check-prefixes=GCN,GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00-amd-amdpal < %s | FileCheck -check-prefixes=GCN,GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgpu10.10-amd-amdpal < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11 %s
define i8 @v_sext_inreg_i8_4(i8 %value) {
; GCN-LABEL: v_sext_inreg_i8_4:
@@ -1325,33 +1325,55 @@ define i65 @v_sext_inreg_i65_22(i65 %value) {
; GFX6-LABEL: v_sext_inreg_i65_22:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_lshl_b64 v[1:2], v[0:1], 21
-; GFX6-NEXT: v_ashr_i64 v[0:1], v[1:2], 21
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 31, v2
+; GFX6-NEXT: v_lshl_b64 v[2:3], v[2:3], 22
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 10, v1
+; GFX6-NEXT: v_or_b32_e32 v2, v2, v3
+; GFX6-NEXT: v_bfe_i32 v2, v2, 0, 1
+; GFX6-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX6-NEXT: v_bfe_u32 v1, v1, 0, 10
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 10, v2
+; GFX6-NEXT: v_ashr_i64 v[2:3], v[2:3], 22
+; GFX6-NEXT: v_or_b32_e32 v1, v1, v4
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_sext_inreg_i65_22:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshlrev_b64 v[1:2], 21, v[0:1]
-; GFX8-NEXT: v_ashrrev_i64 v[0:1], 21, v[1:2]
-; GFX8-NEXT: v_lshrrev_b32_e32 v2, 31, v2
+; GFX8-NEXT: v_lshlrev_b64 v[2:3], 22, v[2:3]
+; GFX8-NEXT: v_lshrrev_b32_e32 v3, 10, v1
+; GFX8-NEXT: v_or_b32_e32 v2, v2, v3
+; GFX8-NEXT: v_bfe_i32 v2, v2, 0, 1
+; GFX8-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX8-NEXT: v_bfe_u32 v1, v1, 0, 10
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 10, v2
+; GFX8-NEXT: v_ashrrev_i64 v[2:3], 22, v[2:3]
+; GFX8-NEXT: v_or_b32_e32 v1, v1, v4
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_sext_inreg_i65_22:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_lshlrev_b64 v[1:2], 21, v[0:1]
-; GFX9-NEXT: v_ashrrev_i64 v[0:1], 21, v[1:2]
-; GFX9-NEXT: v_lshrrev_b32_e32 v2, 31, v2
+; GFX9-NEXT: v_lshlrev_b64 v[2:3], 22, v[2:3]
+; GFX9-NEXT: v_lshrrev_b32_e32 v3, 10, v1
+; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
+; GFX9-NEXT: v_bfe_i32 v2, v2, 0, 1
+; GFX9-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX9-NEXT: v_bfe_u32 v1, v1, 0, 10
+; GFX9-NEXT: v_lshl_or_b32 v1, v2, 10, v1
+; GFX9-NEXT: v_ashrrev_i64 v[2:3], 22, v[2:3]
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10PLUS-LABEL: v_sext_inreg_i65_22:
; GFX10PLUS: ; %bb.0:
; GFX10PLUS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10PLUS-NEXT: v_lshlrev_b64 v[1:2], 21, v[0:1]
-; GFX10PLUS-NEXT: v_ashrrev_i64 v[0:1], 21, v[1:2]
-; GFX10PLUS-NEXT: v_lshrrev_b32_e32 v2, 31, v2
+; GFX10PLUS-NEXT: v_lshlrev_b64 v[2:3], 22, v[2:3]
+; GFX10PLUS-NEXT: v_lshrrev_b32_e32 v3, 10, v1
+; GFX10PLUS-NEXT: v_bfe_u32 v1, v1, 0, 10
+; GFX10PLUS-NEXT: v_or_b32_e32 v2, v2, v3
+; GFX10PLUS-NEXT: v_bfe_i32 v2, v2, 0, 1
+; GFX10PLUS-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX10PLUS-NEXT: v_lshl_or_b32 v1, v2, 10, v1
+; GFX10PLUS-NEXT: v_ashrrev_i64 v[2:3], 22, v[2:3]
; GFX10PLUS-NEXT: s_setpc_b64 s[30:31]
%shl = shl i65 %value, 22
%ashr = ashr i65 %shl, 22
@@ -1362,61 +1384,50 @@ define i65 @v_sext_inreg_i65_33(i65 %value) {
; GFX6-LABEL: v_sext_inreg_i65_33:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_bfe_i32 v0, v2, 0, 1
; GFX6-NEXT: v_mov_b32_e32 v3, v1
-; GFX6-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX6-NEXT: v_lshl_b64 v[4:5], v[0:1], 31
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 1, v3
-; GFX6-NEXT: v_and_b32_e32 v2, 1, v2
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX6-NEXT: v_bfe_i32 v1, v2, 0, 1
+; GFX6-NEXT: v_ashrrev_i32_e32 v2, 31, v1
+; GFX6-NEXT: v_lshl_b64 v[0:1], v[1:2], 31
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 1, v3
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v3
+; GFX6-NEXT: v_ashrrev_i32_e32 v2, 1, v2
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_sext_inreg_i65_33:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_bfe_i32 v0, v2, 0, 1
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX8-NEXT: v_lshlrev_b64 v[4:5], 31, v[0:1]
-; GFX8-NEXT: v_lshrrev_b32_e32 v0, 1, v3
-; GFX8-NEXT: v_and_b32_e32 v2, 1, v2
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX8-NEXT: v_bfe_i32 v1, v2, 0, 1
+; GFX8-NEXT: v_ashrrev_i32_e32 v2, 31, v1
+; GFX8-NEXT: v_lshlrev_b64 v[0:1], 31, v[1:2]
+; GFX8-NEXT: v_lshrrev_b32_e32 v3, 1, v3
+; GFX8-NEXT: v_or_b32_e32 v0, v0, v3
+; GFX8-NEXT: v_ashrrev_i32_e32 v2, 1, v2
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_sext_inreg_i65_33:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_bfe_i32 v0, v2, 0, 1
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX9-NEXT: v_lshlrev_b64 v[4:5], 31, v[0:1]
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 1, v3
-; GFX9-NEXT: v_and_b32_e32 v2, 1, v2
-; GFX9-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX9-NEXT: v_bfe_i32 v1, v2, 0, 1
+; GFX9-NEXT: v_ashrrev_i32_e32 v2, 31, v1
+; GFX9-NEXT: v_lshlrev_b64 v[0:1], 31, v[1:2]
+; GFX9-NEXT: v_lshrrev_b32_e32 v3, 1, v3
+; GFX9-NEXT: v_or_b32_e32 v0, v0, v3
+; GFX9-NEXT: v_ashrrev_i32_e32 v2, 1, v2
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: v_sext_inreg_i65_33:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_bfe_i32 v0, v2, 0, 1
-; GFX10-NEXT: v_mov_b32_e32 v3, v1
-; GFX10-NEXT: v_and_b32_e32 v2, 1, v2
-; GFX10-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX10-NEXT: v_lshlrev_b64 v[4:5], 31, v[0:1]
-; GFX10-NEXT: v_lshrrev_b32_e32 v0, 1, v3
-; GFX10-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: v_sext_inreg_i65_33:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_bfe_i32 v0, v2, 0, 1
-; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_and_b32 v2, 1, v2
-; GFX11-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX11-NEXT: v_lshlrev_b64 v[4:5], 31, v[0:1]
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, 1, v3
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX10PLUS-LABEL: v_sext_inreg_i65_33:
+; GFX10PLUS: ; %bb.0:
+; GFX10PLUS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10PLUS-NEXT: v_mov_b32_e32 v3, v1
+; GFX10PLUS-NEXT: v_bfe_i32 v1, v2, 0, 1
+; GFX10PLUS-NEXT: v_lshrrev_b32_e32 v3, 1, v3
+; GFX10PLUS-NEXT: v_ashrrev_i32_e32 v2, 31, v1
+; GFX10PLUS-NEXT: v_lshlrev_b64 v[0:1], 31, v[1:2]
+; GFX10PLUS-NEXT: v_ashrrev_i32_e32 v2, 1, v2
+; GFX10PLUS-NEXT: v_or_b32_e32 v0, v0, v3
+; GFX10PLUS-NEXT: s_setpc_b64 s[30:31]
%shl = shl i65 %value, 33
%ashr = ashr i65 %value, 33
ret i65 %ashr
@@ -1425,16 +1436,28 @@ define i65 @v_sext_inreg_i65_33(i65 %value) {
define amdgpu_ps i65 @s_sext_inreg_i65_18(i65 inreg %value) {
; GCN-LABEL: s_sext_inreg_i65_18:
; GCN: ; %bb.0:
-; GCN-NEXT: s_lshl_b64 s[0:1], s[0:1], 17
-; GCN-NEXT: s_lshr_b32 s2, s1, 31
-; GCN-NEXT: s_ashr_i64 s[0:1], s[0:1], 17
+; GCN-NEXT: s_lshl_b64 s[2:3], s[2:3], 18
+; GCN-NEXT: s_lshr_b32 s3, s1, 14
+; GCN-NEXT: s_or_b32 s2, s2, s3
+; GCN-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x10000
+; GCN-NEXT: s_bfe_u64 s[0:1], s[0:1], 0x2e0000
+; GCN-NEXT: s_lshl_b32 s5, s2, 14
+; GCN-NEXT: s_mov_b32 s4, 0
+; GCN-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
+; GCN-NEXT: s_ashr_i64 s[2:3], s[2:3], 18
; GCN-NEXT: ; return to shader part epilog
;
; GFX10PLUS-LABEL: s_sext_inreg_i65_18:
; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: s_lshl_b64 s[2:3], s[0:1], 17
-; GFX10PLUS-NEXT: s_ashr_i64 s[0:1], s[2:3], 17
-; GFX10PLUS-NEXT: s_lshr_b32 s2, s3, 31
+; GFX10PLUS-NEXT: s_lshl_b64 s[2:3], s[2:3], 18
+; GFX10PLUS-NEXT: s_lshr_b32 s3, s1, 14
+; GFX10PLUS-NEXT: s_bfe_u64 s[0:1], s[0:1], 0x2e0000
+; GFX10PLUS-NEXT: s_or_b32 s2, s2, s3
+; GFX10PLUS-NEXT: s_mov_b32 s4, 0
+; GFX10PLUS-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x10000
+; GFX10PLUS-NEXT: s_lshl_b32 s5, s2, 14
+; GFX10PLUS-NEXT: s_ashr_i64 s[2:3], s[2:3], 18
+; GFX10PLUS-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
; GFX10PLUS-NEXT: ; return to shader part epilog
%shl = shl i65 %value, 18
%ashr = ashr i65 %shl, 18
@@ -1444,16 +1467,28 @@ define amdgpu_ps i65 @s_sext_inreg_i65_18(i65 inreg %value) {
define amdgpu_ps i65 @s_sext_inreg_i65_33(i65 inreg %value) {
; GCN-LABEL: s_sext_inreg_i65_33:
; GCN: ; %bb.0:
-; GCN-NEXT: s_bfe_i64 s[2:3], s[0:1], 0x200000
-; GCN-NEXT: s_lshr_b32 s2, s3, 31
-; GCN-NEXT: s_mov_b32 s1, s3
+; GCN-NEXT: s_lshl_b32 s3, s2, 1
+; GCN-NEXT: s_mov_b32 s2, 0
+; GCN-NEXT: s_lshr_b64 s[4:5], s[0:1], 31
+; GCN-NEXT: s_or_b64 s[2:3], s[2:3], s[4:5]
+; GCN-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x10000
+; GCN-NEXT: s_bfe_u32 s4, s0, 0x1f0000
+; GCN-NEXT: s_lshl_b64 s[0:1], s[2:3], 31
+; GCN-NEXT: s_or_b32 s0, s0, s4
+; GCN-NEXT: s_ashr_i32 s2, s3, 1
; GCN-NEXT: ; return to shader part epilog
;
; GFX10PLUS-LABEL: s_sext_inreg_i65_33:
; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: s_bfe_i64 s[2:3], s[0:1], 0x200000
-; GFX10PLUS-NEXT: s_lshr_b32 s2, s3, 31
-; GFX10PLUS-NEXT: s_mov_b32 s1, s3
+; GFX10PLUS-NEXT: s_lshl_b32 s3, s2, 1
+; GFX10PLUS-NEXT: s_mov_b32 s2, 0
+; GFX10PLUS-NEXT: s_lshr_b64 s[4:5], s[0:1], 31
+; GFX10PLUS-NEXT: s_or_b64 s[2:3], s[2:3], s[4:5]
+; GFX10PLUS-NEXT: s_bfe_u32 s4, s0, 0x1f0000
+; GFX10PLUS-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x10000
+; GFX10PLUS-NEXT: s_lshl_b64 s[0:1], s[2:3], 31
+; GFX10PLUS-NEXT: s_ashr_i32 s2, s3, 1
+; GFX10PLUS-NEXT: s_or_b32 s0, s0, s4
; GFX10PLUS-NEXT: ; return to shader part epilog
%shl = shl i65 %value, 33
%ashr = ashr i65 %shl, 33
@@ -1472,3 +1507,6 @@ define amdgpu_ps i65 @s_sext_inreg_i65_33(i65 inreg %value) {
; %ashr = ashrshl <2 x i65> %shl, <i65 36, i65 36>
; ret <2 x i65> %ashr
; }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX10: {{.*}}
+; GFX11: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/shl.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/shl.ll
index 17667e2abdb7f..d1df5a02dfc75 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/shl.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/shl.ll
@@ -1,9 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu6.00-amd-amdpal < %s | FileCheck -check-prefixes=GCN,GFX6 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-amd-amdpal < %s | FileCheck -check-prefixes=GCN,GFX8 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00-amd-amdpal < %s | FileCheck -check-prefixes=GCN,GFX9 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-amd-amdpal < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11 %s
+; RUN: llc -global-isel -mtriple=amdgpu6.00-amd-amdpal < %s | FileCheck -check-prefixes=GCN,GFX6 %s
+; RUN: llc -global-isel -mtriple=amdgpu8.03-amd-amdpal < %s | FileCheck -check-prefixes=GCN,GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00-amd-amdpal < %s | FileCheck -check-prefixes=GCN,GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgpu10.10-amd-amdpal < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11 %s
define i8 @v_shl_i8(i8 %value, i8 %amount) {
; GFX6-LABEL: v_shl_i8:
@@ -1781,15 +1781,22 @@ define amdgpu_ps i65 @s_shl_i65(i65 inreg %value, i65 inreg %amount) {
define amdgpu_ps i65 @s_shl_i65_33(i65 inreg %value) {
; GCN-LABEL: s_shl_i65_33:
; GCN: ; %bb.0:
-; GCN-NEXT: s_lshr_b32 s2, s0, 31
-; GCN-NEXT: s_lshl_b32 s1, s0, 1
+; GCN-NEXT: s_lshl_b32 s4, s0, 1
+; GCN-NEXT: s_mov_b32 s6, 0
+; GCN-NEXT: s_lshl_b32 s7, s2, 1
+; GCN-NEXT: s_lshr_b64 s[0:1], s[0:1], 31
+; GCN-NEXT: s_or_b64 s[2:3], s[6:7], s[0:1]
; GCN-NEXT: s_mov_b32 s0, 0
+; GCN-NEXT: s_mov_b32 s1, s4
; GCN-NEXT: ; return to shader part epilog
;
; GFX10PLUS-LABEL: s_shl_i65_33:
; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: s_lshr_b32 s2, s0, 31
+; GFX10PLUS-NEXT: s_mov_b32 s4, 0
+; GFX10PLUS-NEXT: s_lshl_b32 s5, s2, 1
+; GFX10PLUS-NEXT: s_lshr_b64 s[2:3], s[0:1], 31
; GFX10PLUS-NEXT: s_lshl_b32 s1, s0, 1
+; GFX10PLUS-NEXT: s_or_b64 s[2:3], s[4:5], s[2:3]
; GFX10PLUS-NEXT: s_mov_b32 s0, 0
; GFX10PLUS-NEXT: ; return to shader part epilog
%result = shl i65 %value, 33
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll
index 4314387b5cc5f..a739ede299c06 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -amdgpu-codegenprepare-disable-idiv-expansion=1 -mtriple=amdgpu7.01-mesa-mesa3d -denormal-fp-math-f32=preserve-sign < %s | FileCheck -check-prefixes=CHECK,GISEL %s
-; RUN: llc -global-isel -global-isel-abort=2 -amdgpu-codegenprepare-disable-idiv-expansion=0 -mtriple=amdgpu7.01-mesa-mesa3d -denormal-fp-math-f32=preserve-sign < %s | FileCheck -check-prefixes=CHECK,CGP %s
+; RUN: llc -global-isel -amdgpu-codegenprepare-disable-idiv-expansion=1 -mtriple=amdgpu7.01-mesa-mesa3d -denormal-fp-math-f32=preserve-sign < %s | FileCheck -check-prefixes=CHECK,GISEL %s
+; RUN: llc -global-isel -amdgpu-codegenprepare-disable-idiv-expansion=0 -mtriple=amdgpu7.01-mesa-mesa3d -denormal-fp-math-f32=preserve-sign < %s | FileCheck -check-prefixes=CHECK,CGP %s
; The same 32-bit expansion is implemented in the legalizer and in AMDGPUCodeGenPrepare.
@@ -189,135 +189,184 @@ define amdgpu_ps i64 @s_srem_i64(i64 inreg %num, i64 inreg %den) {
; CHECK-LABEL: s_srem_i64:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_or_b64 s[0:1], s[2:3], s[4:5]
-; CHECK-NEXT: s_cmp_lg_u32 s1, 0
+; CHECK-NEXT: s_mov_b32 s0, 0
+; CHECK-NEXT: v_cmp_ne_u64_e64 s[0:1], s[0:1], 0
+; CHECK-NEXT: s_mov_b32 s6, 1
+; CHECK-NEXT: s_or_b64 s[0:1], s[0:1], s[0:1]
; CHECK-NEXT: s_cbranch_scc0 .LBB1_2
; CHECK-NEXT: ; %bb.1:
-; CHECK-NEXT: s_ashr_i32 s0, s5, 31
-; CHECK-NEXT: s_add_u32 s6, s4, s0
+; CHECK-NEXT: s_ashr_i32 s0, s3, 31
+; CHECK-NEXT: s_ashr_i32 s6, s5, 31
+; CHECK-NEXT: s_add_u32 s8, s2, s0
+; CHECK-NEXT: s_addc_u32 s9, s3, s0
+; CHECK-NEXT: s_add_u32 s10, s4, s6
+; CHECK-NEXT: s_mov_b32 s7, s6
+; CHECK-NEXT: s_addc_u32 s11, s5, s6
+; CHECK-NEXT: s_xor_b64 s[6:7], s[10:11], s[6:7]
+; CHECK-NEXT: v_cvt_f32_u32_e32 v0, s6
+; CHECK-NEXT: v_cvt_f32_u32_e32 v1, s7
; CHECK-NEXT: s_mov_b32 s1, s0
-; CHECK-NEXT: s_addc_u32 s7, s5, s0
-; CHECK-NEXT: s_xor_b64 s[0:1], s[6:7], s[0:1]
-; CHECK-NEXT: v_cvt_f32_u32_e32 v0, s0
-; CHECK-NEXT: v_cvt_f32_u32_e32 v1, s1
-; CHECK-NEXT: s_sub_u32 s5, 0, s0
-; CHECK-NEXT: s_subb_u32 s8, 0, s1
-; CHECK-NEXT: v_madmk_f32 v0, v1, 0x4f800000, v0
-; CHECK-NEXT: v_rcp_f32_e32 v0, v0
+; CHECK-NEXT: s_xor_b64 s[8:9], s[8:9], s[0:1]
+; CHECK-NEXT: s_sub_u32 s3, 0, s6
+; CHECK-NEXT: v_mac_f32_e32 v0, 0x4f800000, v1
+; CHECK-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; CHECK-NEXT: s_subb_u32 s5, 0, s7
; CHECK-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0
; CHECK-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0
; CHECK-NEXT: v_trunc_f32_e32 v1, v1
-; CHECK-NEXT: v_madmk_f32 v0, v1, 0xcf800000, v0
-; CHECK-NEXT: v_cvt_u32_f32_e32 v7, v1
-; CHECK-NEXT: v_cvt_u32_f32_e32 v8, v0
-; CHECK-NEXT: v_mul_lo_u32 v2, s5, v7
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[6:7], s5, v8, 0
-; CHECK-NEXT: v_mul_lo_u32 v3, s8, v8
-; CHECK-NEXT: v_add_i32_e32 v1, vcc, v1, v2
-; CHECK-NEXT: v_add_i32_e32 v5, vcc, v1, v3
-; CHECK-NEXT: v_mul_hi_u32 v9, v8, v0
-; CHECK-NEXT: v_mad_u64_u32 v[1:2], s[6:7], v8, v5, 0
-; CHECK-NEXT: v_mad_u64_u32 v[3:4], s[6:7], v7, v0, 0
-; CHECK-NEXT: v_mad_u64_u32 v[5:6], s[6:7], v7, v5, 0
-; CHECK-NEXT: v_add_i32_e32 v0, vcc, v9, v1
-; CHECK-NEXT: v_addc_u32_e32 v1, vcc, 0, v2, vcc
-; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v3
-; CHECK-NEXT: v_addc_u32_e32 v0, vcc, v1, v4, vcc
-; CHECK-NEXT: v_addc_u32_e32 v1, vcc, 0, v6, vcc
-; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v5
-; CHECK-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; CHECK-NEXT: v_add_i32_e32 v5, vcc, v8, v0
-; CHECK-NEXT: v_addc_u32_e32 v6, vcc, v7, v1, vcc
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[6:7], s5, v5, 0
-; CHECK-NEXT: v_mul_lo_u32 v2, s5, v6
-; CHECK-NEXT: v_mul_lo_u32 v3, s8, v5
-; CHECK-NEXT: s_ashr_i32 s8, s3, 31
-; CHECK-NEXT: v_mul_hi_u32 v8, v5, v0
-; CHECK-NEXT: v_add_i32_e32 v1, vcc, v2, v1
-; CHECK-NEXT: v_add_i32_e32 v7, vcc, v3, v1
-; CHECK-NEXT: v_mad_u64_u32 v[1:2], s[6:7], v5, v7, 0
-; CHECK-NEXT: v_mad_u64_u32 v[3:4], s[6:7], v6, v0, 0
-; CHECK-NEXT: s_mov_b32 s9, s8
-; CHECK-NEXT: v_add_i32_e32 v8, vcc, v8, v1
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[6:7], v6, v7, 0
-; CHECK-NEXT: v_addc_u32_e32 v2, vcc, 0, v2, vcc
-; CHECK-NEXT: v_add_i32_e32 v3, vcc, v8, v3
-; CHECK-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc
-; CHECK-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; CHECK-NEXT: v_add_i32_e32 v0, vcc, v2, v0
-; CHECK-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; CHECK-NEXT: s_add_u32 s6, s2, s8
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v5, v0
-; CHECK-NEXT: s_addc_u32 s7, s3, s8
-; CHECK-NEXT: v_addc_u32_e32 v3, vcc, v6, v1, vcc
-; CHECK-NEXT: s_xor_b64 s[10:11], s[6:7], s[8:9]
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[6:7], s10, v3, 0
-; CHECK-NEXT: v_mul_hi_u32 v4, s10, v2
-; CHECK-NEXT: v_readfirstlane_b32 s3, v1
+; CHECK-NEXT: v_mac_f32_e32 v0, 0xcf800000, v1
+; CHECK-NEXT: v_cvt_u32_f32_e32 v0, v0
+; CHECK-NEXT: v_cvt_u32_f32_e32 v1, v1
+; CHECK-NEXT: v_mul_hi_u32 v2, s3, v0
+; CHECK-NEXT: v_readfirstlane_b32 s11, v0
+; CHECK-NEXT: s_mul_i32 s13, s3, s11
+; CHECK-NEXT: v_readfirstlane_b32 s10, v1
+; CHECK-NEXT: v_readfirstlane_b32 s14, v2
+; CHECK-NEXT: v_mul_hi_u32 v2, v0, s13
+; CHECK-NEXT: s_mul_i32 s12, s3, s10
+; CHECK-NEXT: s_add_u32 s12, s12, s14
+; CHECK-NEXT: s_mul_i32 s14, s5, s11
+; CHECK-NEXT: s_add_u32 s12, s14, s12
+; CHECK-NEXT: v_readfirstlane_b32 s14, v2
+; CHECK-NEXT: s_mul_i32 s15, s11, s12
+; CHECK-NEXT: v_mul_hi_u32 v2, v1, s13
+; CHECK-NEXT: v_mul_hi_u32 v0, v0, s12
+; CHECK-NEXT: s_mul_i32 s16, s10, s13
+; CHECK-NEXT: s_add_u32 s14, s14, s15
+; CHECK-NEXT: s_cselect_b32 s15, 1, 0
+; CHECK-NEXT: s_add_u32 s14, s14, s16
+; CHECK-NEXT: s_cselect_b32 s13, 1, 0
+; CHECK-NEXT: s_add_i32 s15, s15, s13
+; CHECK-NEXT: v_readfirstlane_b32 s13, v2
+; CHECK-NEXT: v_readfirstlane_b32 s14, v0
+; CHECK-NEXT: s_mul_i32 s16, s10, s12
+; CHECK-NEXT: s_add_u32 s13, s13, s14
+; CHECK-NEXT: s_cselect_b32 s14, 1, 0
+; CHECK-NEXT: s_add_u32 s13, s13, s16
+; CHECK-NEXT: s_cselect_b32 s16, 1, 0
+; CHECK-NEXT: v_mul_hi_u32 v0, v1, s12
+; CHECK-NEXT: s_add_i32 s14, s14, s16
+; CHECK-NEXT: s_add_u32 s12, s13, s15
+; CHECK-NEXT: s_cselect_b32 s13, 1, 0
+; CHECK-NEXT: s_add_i32 s14, s14, s13
+; CHECK-NEXT: v_readfirstlane_b32 s13, v0
+; CHECK-NEXT: s_add_i32 s13, s13, s14
+; CHECK-NEXT: s_add_u32 s11, s11, s12
+; CHECK-NEXT: v_mov_b32_e32 v0, s11
+; CHECK-NEXT: v_mul_hi_u32 v0, s3, v0
+; CHECK-NEXT: s_mul_i32 s12, s3, s11
+; CHECK-NEXT: s_addc_u32 s10, s10, s13
+; CHECK-NEXT: s_mul_i32 s3, s3, s10
+; CHECK-NEXT: v_readfirstlane_b32 s13, v0
+; CHECK-NEXT: v_mov_b32_e32 v0, s12
+; CHECK-NEXT: v_mul_hi_u32 v1, s11, v0
+; CHECK-NEXT: s_add_u32 s3, s3, s13
+; CHECK-NEXT: s_mul_i32 s5, s5, s11
+; CHECK-NEXT: s_add_u32 s3, s5, s3
+; CHECK-NEXT: v_readfirstlane_b32 s5, v1
+; CHECK-NEXT: v_mov_b32_e32 v1, s3
+; CHECK-NEXT: s_mul_i32 s13, s11, s3
+; CHECK-NEXT: v_mul_hi_u32 v0, s10, v0
+; CHECK-NEXT: v_mul_hi_u32 v2, s11, v1
+; CHECK-NEXT: s_mul_i32 s12, s10, s12
+; CHECK-NEXT: s_add_u32 s5, s5, s13
+; CHECK-NEXT: s_cselect_b32 s13, 1, 0
+; CHECK-NEXT: s_add_u32 s5, s5, s12
+; CHECK-NEXT: s_cselect_b32 s5, 1, 0
+; CHECK-NEXT: s_add_i32 s13, s13, s5
; CHECK-NEXT: v_readfirstlane_b32 s5, v0
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[6:7], s11, v2, 0
-; CHECK-NEXT: v_readfirstlane_b32 s6, v4
-; CHECK-NEXT: s_add_u32 s5, s6, s5
-; CHECK-NEXT: s_addc_u32 s3, 0, s3
-; CHECK-NEXT: v_readfirstlane_b32 s12, v1
-; CHECK-NEXT: v_mad_u64_u32 v[1:2], s[6:7], s11, v3, 0
-; CHECK-NEXT: v_readfirstlane_b32 s6, v0
-; CHECK-NEXT: s_add_u32 s5, s5, s6
-; CHECK-NEXT: s_addc_u32 s3, s3, s12
-; CHECK-NEXT: v_readfirstlane_b32 s5, v2
-; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_readfirstlane_b32 s6, v1
-; CHECK-NEXT: s_add_u32 s3, s3, s6
-; CHECK-NEXT: v_mov_b32_e32 v0, s3
-; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[6:7], s0, v0, 0
-; CHECK-NEXT: s_addc_u32 s5, 0, s5
-; CHECK-NEXT: s_mul_i32 s5, s0, s5
-; CHECK-NEXT: s_mul_i32 s3, s1, s3
-; CHECK-NEXT: v_readfirstlane_b32 s12, v1
+; CHECK-NEXT: v_readfirstlane_b32 s12, v2
+; CHECK-NEXT: s_mul_i32 s3, s10, s3
+; CHECK-NEXT: s_add_u32 s5, s5, s12
+; CHECK-NEXT: s_cselect_b32 s12, 1, 0
+; CHECK-NEXT: s_add_u32 s3, s5, s3
+; CHECK-NEXT: s_cselect_b32 s5, 1, 0
+; CHECK-NEXT: v_mul_hi_u32 v0, s10, v1
; CHECK-NEXT: s_add_i32 s5, s12, s5
-; CHECK-NEXT: s_add_i32 s5, s5, s3
-; CHECK-NEXT: s_sub_i32 s3, s11, s5
+; CHECK-NEXT: s_add_u32 s3, s3, s13
+; CHECK-NEXT: s_cselect_b32 s12, 1, 0
+; CHECK-NEXT: s_add_i32 s5, s5, s12
; CHECK-NEXT: v_readfirstlane_b32 s12, v0
-; CHECK-NEXT: s_sub_u32 s10, s10, s12
-; CHECK-NEXT: s_cselect_b64 s[12:13], -1, 0
-; CHECK-NEXT: s_subb_u32 s3, s3, s1
-; CHECK-NEXT: s_sub_u32 s16, s10, s0
-; CHECK-NEXT: s_cselect_b64 s[14:15], -1, 0
-; CHECK-NEXT: s_subb_u32 s17, s3, 0
-; CHECK-NEXT: s_cmp_ge_u32 s17, s1
-; CHECK-NEXT: s_cselect_b32 s18, -1, 0
-; CHECK-NEXT: s_cmp_ge_u32 s16, s0
-; CHECK-NEXT: s_cselect_b32 s19, -1, 0
-; CHECK-NEXT: s_cmp_eq_u32 s17, s1
-; CHECK-NEXT: s_cselect_b32 s18, s19, s18
-; CHECK-NEXT: s_or_b32 s14, s14, s15
-; CHECK-NEXT: s_subb_u32 s3, s3, s1
-; CHECK-NEXT: s_sub_u32 s14, s16, s0
-; CHECK-NEXT: s_subb_u32 s3, s3, 0
-; CHECK-NEXT: s_cmp_lg_u32 s18, 0
-; CHECK-NEXT: s_cselect_b32 s14, s14, s16
-; CHECK-NEXT: s_cselect_b32 s3, s3, s17
-; CHECK-NEXT: s_or_b32 s12, s12, s13
-; CHECK-NEXT: s_subb_u32 s5, s11, s5
-; CHECK-NEXT: s_cmp_ge_u32 s5, s1
-; CHECK-NEXT: s_cselect_b32 s11, -1, 0
-; CHECK-NEXT: s_cmp_ge_u32 s10, s0
-; CHECK-NEXT: s_cselect_b32 s0, -1, 0
-; CHECK-NEXT: s_cmp_eq_u32 s5, s1
-; CHECK-NEXT: s_cselect_b32 s0, s0, s11
-; CHECK-NEXT: s_cmp_lg_u32 s0, 0
-; CHECK-NEXT: s_cselect_b32 s1, s3, s5
-; CHECK-NEXT: s_cselect_b32 s0, s14, s10
-; CHECK-NEXT: s_xor_b64 s[0:1], s[0:1], s[8:9]
-; CHECK-NEXT: s_mov_b64 s[6:7], 0
-; CHECK-NEXT: s_sub_u32 s0, s0, s8
+; CHECK-NEXT: s_add_i32 s12, s12, s5
+; CHECK-NEXT: s_add_u32 s3, s11, s3
+; CHECK-NEXT: v_mov_b32_e32 v0, s3
+; CHECK-NEXT: v_mul_hi_u32 v0, s8, v0
+; CHECK-NEXT: s_addc_u32 s5, s10, s12
+; CHECK-NEXT: v_mov_b32_e32 v1, s5
+; CHECK-NEXT: s_mul_i32 s11, s8, s5
+; CHECK-NEXT: v_readfirstlane_b32 s10, v0
+; CHECK-NEXT: v_mov_b32_e32 v0, s9
+; CHECK-NEXT: v_mul_hi_u32 v0, v0, s3
+; CHECK-NEXT: v_mul_hi_u32 v2, s8, v1
+; CHECK-NEXT: s_mul_i32 s12, s9, s3
+; CHECK-NEXT: s_add_u32 s10, s10, s11
+; CHECK-NEXT: s_cselect_b32 s11, 1, 0
+; CHECK-NEXT: s_add_u32 s10, s10, s12
+; CHECK-NEXT: s_cselect_b32 s3, 1, 0
+; CHECK-NEXT: s_add_i32 s11, s11, s3
+; CHECK-NEXT: v_readfirstlane_b32 s3, v0
+; CHECK-NEXT: v_readfirstlane_b32 s10, v2
+; CHECK-NEXT: s_mul_i32 s5, s9, s5
+; CHECK-NEXT: s_add_u32 s3, s3, s10
+; CHECK-NEXT: s_cselect_b32 s10, 1, 0
+; CHECK-NEXT: s_add_u32 s3, s3, s5
+; CHECK-NEXT: s_cselect_b32 s5, 1, 0
+; CHECK-NEXT: s_add_i32 s5, s10, s5
+; CHECK-NEXT: s_add_u32 s3, s3, s11
+; CHECK-NEXT: v_mul_hi_u32 v0, s9, v1
+; CHECK-NEXT: v_mov_b32_e32 v1, s3
+; CHECK-NEXT: v_mul_hi_u32 v1, s6, v1
+; CHECK-NEXT: s_cselect_b32 s10, 1, 0
+; CHECK-NEXT: s_add_i32 s5, s5, s10
+; CHECK-NEXT: v_readfirstlane_b32 s10, v0
+; CHECK-NEXT: s_add_i32 s10, s10, s5
+; CHECK-NEXT: v_readfirstlane_b32 s11, v1
+; CHECK-NEXT: s_mul_i32 s10, s6, s10
+; CHECK-NEXT: s_mul_i32 s5, s6, s3
+; CHECK-NEXT: s_add_u32 s10, s10, s11
+; CHECK-NEXT: s_mul_i32 s3, s7, s3
+; CHECK-NEXT: s_add_u32 s3, s3, s10
+; CHECK-NEXT: s_sub_u32 s10, s8, s5
+; CHECK-NEXT: s_cselect_b32 s5, 1, 0
+; CHECK-NEXT: s_subb_u32 s11, s9, s3
+; CHECK-NEXT: s_sub_i32 s3, s9, s3
+; CHECK-NEXT: s_cmp_ge_u32 s11, s7
+; CHECK-NEXT: s_cselect_b32 s8, -1, 0
+; CHECK-NEXT: s_cmp_ge_u32 s10, s6
+; CHECK-NEXT: s_cselect_b32 s9, -1, 0
+; CHECK-NEXT: s_cmp_eq_u32 s11, s7
+; CHECK-NEXT: s_cselect_b32 s12, s9, s8
+; CHECK-NEXT: s_sub_u32 s8, s10, s6
+; CHECK-NEXT: s_cselect_b32 s13, 1, 0
+; CHECK-NEXT: s_cmp_lg_u32 s5, 0
+; CHECK-NEXT: s_subb_u32 s3, s3, s7
+; CHECK-NEXT: s_cmp_lg_u32 s13, 0
+; CHECK-NEXT: s_subb_u32 s9, s3, 0
+; CHECK-NEXT: s_cmp_ge_u32 s9, s7
+; CHECK-NEXT: s_cselect_b32 s5, -1, 0
+; CHECK-NEXT: s_cmp_ge_u32 s8, s6
+; CHECK-NEXT: s_cselect_b32 s14, -1, 0
+; CHECK-NEXT: s_cmp_eq_u32 s9, s7
+; CHECK-NEXT: s_cselect_b32 s5, s14, s5
+; CHECK-NEXT: s_sub_u32 s6, s8, s6
+; CHECK-NEXT: s_cselect_b32 s14, 1, 0
+; CHECK-NEXT: s_cmp_lg_u32 s13, 0
+; CHECK-NEXT: s_subb_u32 s3, s3, s7
+; CHECK-NEXT: s_cmp_lg_u32 s14, 0
+; CHECK-NEXT: s_subb_u32 s7, s3, 0
+; CHECK-NEXT: s_cmp_lg_u32 s5, 0
+; CHECK-NEXT: s_cselect_b64 s[6:7], s[6:7], s[8:9]
+; CHECK-NEXT: s_cmp_lg_u32 s12, 0
+; CHECK-NEXT: s_cselect_b64 s[6:7], s[6:7], s[10:11]
+; CHECK-NEXT: s_xor_b64 s[6:7], s[6:7], s[0:1]
+; CHECK-NEXT: s_sub_u32 s0, s6, s0
+; CHECK-NEXT: s_mov_b32 s6, 0
; CHECK-NEXT: s_branch .LBB1_3
; CHECK-NEXT: .LBB1_2:
-; CHECK-NEXT: s_mov_b64 s[6:7], -1
; CHECK-NEXT: ; implicit-def: $sgpr0_sgpr1
; CHECK-NEXT: .LBB1_3: ; %Flow
-; CHECK-NEXT: s_and_b64 s[6:7], s[6:7], exec
-; CHECK-NEXT: s_cselect_b32 s1, 1, 0
-; CHECK-NEXT: s_cmp_lg_u32 s1, 1
+; CHECK-NEXT: s_xor_b32 s1, s6, 1
+; CHECK-NEXT: s_cmp_lg_u32 s1, 0
; CHECK-NEXT: s_cbranch_scc1 .LBB1_5
; CHECK-NEXT: ; %bb.4:
; CHECK-NEXT: v_cvt_f32_u32_e32 v0, s4
@@ -325,19 +374,28 @@ define amdgpu_ps i64 @s_srem_i64(i64 inreg %num, i64 inreg %den) {
; CHECK-NEXT: v_rcp_iflag_f32_e32 v0, v0
; CHECK-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; CHECK-NEXT: v_cvt_u32_f32_e32 v0, v0
-; CHECK-NEXT: v_mul_lo_u32 v1, s0, v0
-; CHECK-NEXT: v_mul_hi_u32 v1, v0, v1
-; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; CHECK-NEXT: v_readfirstlane_b32 s1, v0
+; CHECK-NEXT: s_mul_i32 s0, s0, s1
+; CHECK-NEXT: v_mul_hi_u32 v0, v0, s0
+; CHECK-NEXT: v_readfirstlane_b32 s0, v0
+; CHECK-NEXT: s_add_i32 s0, s1, s0
+; CHECK-NEXT: v_mov_b32_e32 v0, s0
; CHECK-NEXT: v_mul_hi_u32 v0, s2, v0
+; CHECK-NEXT: s_mov_b32 s1, 0
; CHECK-NEXT: v_readfirstlane_b32 s0, v0
; CHECK-NEXT: s_mul_i32 s0, s0, s4
; CHECK-NEXT: s_sub_i32 s0, s2, s0
-; CHECK-NEXT: s_sub_i32 s1, s0, s4
-; CHECK-NEXT: s_cmp_ge_u32 s0, s4
-; CHECK-NEXT: s_cselect_b32 s0, s1, s0
-; CHECK-NEXT: s_sub_i32 s1, s0, s4
; CHECK-NEXT: s_cmp_ge_u32 s0, s4
-; CHECK-NEXT: s_cselect_b32 s0, s1, s0
+; CHECK-NEXT: s_cselect_b32 s2, 1, 0
+; CHECK-NEXT: s_sub_i32 s3, s0, s4
+; CHECK-NEXT: s_cmp_lg_u32 s2, 0
+; CHECK-NEXT: s_cselect_b32 s2, s3, s0
+; CHECK-NEXT: s_cmp_ge_u32 s2, s4
+; CHECK-NEXT: s_cselect_b32 s5, 1, 0
+; CHECK-NEXT: s_sub_i32 s0, s2, s4
+; CHECK-NEXT: s_mov_b32 s3, s1
+; CHECK-NEXT: s_cmp_lg_u32 s5, 0
+; CHECK-NEXT: s_cselect_b64 s[0:1], s[0:1], s[2:3]
; CHECK-NEXT: .LBB1_5: ; %.split
; CHECK-NEXT: s_mov_b32 s1, s0
; CHECK-NEXT: ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/div_i128.ll b/llvm/test/CodeGen/AMDGPU/div_i128.ll
index 0b067ebbee0e3..5e7b30735a67a 100644
--- a/llvm/test/CodeGen/AMDGPU/div_i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/div_i128.ll
@@ -2,8 +2,8 @@
; RUN: llc -global-isel=0 -mtriple=amdgpu9.00-amd-amdhsa -o - %s | FileCheck -check-prefixes=GFX9 %s
; RUN: llc -O0 -global-isel=0 -mtriple=amdgpu9.00-amd-amdhsa -o - %s | FileCheck -check-prefixes=GFX9-O0 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.00-amd-amdhsa -o - %s | FileCheck -check-prefixes=GFX9-G %s
-; RUN: llc -O0 -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.00-amd-amdhsa -o - %s | FileCheck -check-prefixes=GFX9-G-O0 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.00-amd-amdhsa -o - %s | FileCheck -check-prefixes=GFX9-G %s
+; RUN: llc -O0 -global-isel=1 -mtriple=amdgpu9.00-amd-amdhsa -o - %s | FileCheck -check-prefixes=GFX9-G-O0 %s
define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-LABEL: v_sdiv_i128_vv:
@@ -1171,202 +1171,203 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-LABEL: v_sdiv_i128_vv:
; GFX9-G: ; %bb.0: ; %_udiv-special-cases
; GFX9-G-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-G-NEXT: v_ashrrev_i32_e32 v17, 31, v3
-; GFX9-G-NEXT: v_xor_b32_e32 v0, v0, v17
-; GFX9-G-NEXT: v_xor_b32_e32 v1, v1, v17
-; GFX9-G-NEXT: v_sub_co_u32_e32 v8, vcc, v0, v17
-; GFX9-G-NEXT: v_subb_co_u32_e32 v9, vcc, v1, v17, vcc
-; GFX9-G-NEXT: v_xor_b32_e32 v1, v2, v17
-; GFX9-G-NEXT: v_ashrrev_i32_e32 v18, 31, v7
-; GFX9-G-NEXT: v_xor_b32_e32 v0, v3, v17
-; GFX9-G-NEXT: v_subb_co_u32_e32 v10, vcc, v1, v17, vcc
-; GFX9-G-NEXT: v_subb_co_u32_e32 v11, vcc, v0, v17, vcc
-; GFX9-G-NEXT: v_xor_b32_e32 v1, v4, v18
-; GFX9-G-NEXT: v_xor_b32_e32 v0, v5, v18
-; GFX9-G-NEXT: v_sub_co_u32_e32 v21, vcc, v1, v18
-; GFX9-G-NEXT: v_subb_co_u32_e32 v22, vcc, v0, v18, vcc
-; GFX9-G-NEXT: v_xor_b32_e32 v0, v6, v18
-; GFX9-G-NEXT: v_xor_b32_e32 v1, v7, v18
-; GFX9-G-NEXT: v_subb_co_u32_e32 v0, vcc, v0, v18, vcc
-; GFX9-G-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v18, vcc
-; GFX9-G-NEXT: v_or_b32_e32 v3, v22, v1
-; GFX9-G-NEXT: v_or_b32_e32 v2, v21, v0
-; GFX9-G-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3]
-; GFX9-G-NEXT: v_or_b32_e32 v3, v9, v11
-; GFX9-G-NEXT: v_or_b32_e32 v2, v8, v10
-; GFX9-G-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[2:3]
-; GFX9-G-NEXT: v_ffbh_u32_e32 v2, v0
+; GFX9-G-NEXT: v_ashrrev_i32_e32 v16, 31, v3
+; GFX9-G-NEXT: v_xor_b32_e32 v0, v16, v0
+; GFX9-G-NEXT: v_xor_b32_e32 v1, v16, v1
+; GFX9-G-NEXT: v_sub_co_u32_e32 v8, vcc, v0, v16
+; GFX9-G-NEXT: v_xor_b32_e32 v2, v16, v2
+; GFX9-G-NEXT: v_subb_co_u32_e32 v9, vcc, v1, v16, vcc
+; GFX9-G-NEXT: v_ashrrev_i32_e32 v17, 31, v7
+; GFX9-G-NEXT: v_xor_b32_e32 v3, v16, v3
+; GFX9-G-NEXT: v_subb_co_u32_e32 v10, vcc, v2, v16, vcc
+; GFX9-G-NEXT: v_subb_co_u32_e32 v11, vcc, v3, v16, vcc
+; GFX9-G-NEXT: v_xor_b32_e32 v0, v17, v4
+; GFX9-G-NEXT: v_xor_b32_e32 v1, v17, v5
+; GFX9-G-NEXT: v_sub_co_u32_e32 v18, vcc, v0, v17
+; GFX9-G-NEXT: v_xor_b32_e32 v2, v17, v6
+; GFX9-G-NEXT: v_subb_co_u32_e32 v19, vcc, v1, v17, vcc
+; GFX9-G-NEXT: v_xor_b32_e32 v3, v17, v7
+; GFX9-G-NEXT: v_subb_co_u32_e32 v4, vcc, v2, v17, vcc
+; GFX9-G-NEXT: v_subb_co_u32_e32 v5, vcc, v3, v17, vcc
+; GFX9-G-NEXT: v_or_b32_e32 v0, v18, v4
+; GFX9-G-NEXT: v_or_b32_e32 v1, v19, v5
+; GFX9-G-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
+; GFX9-G-NEXT: v_or_b32_e32 v0, v8, v10
+; GFX9-G-NEXT: v_or_b32_e32 v1, v9, v11
+; GFX9-G-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[0:1]
+; GFX9-G-NEXT: v_ffbh_u32_e32 v1, v18
+; GFX9-G-NEXT: v_ffbh_u32_e32 v0, v19
+; GFX9-G-NEXT: v_add_u32_e32 v1, 32, v1
+; GFX9-G-NEXT: v_ffbh_u32_e32 v2, v4
+; GFX9-G-NEXT: s_or_b64 s[10:11], vcc, s[4:5]
+; GFX9-G-NEXT: v_min_u32_e32 v0, v0, v1
+; GFX9-G-NEXT: v_ffbh_u32_e32 v1, v5
+; GFX9-G-NEXT: v_add_u32_e32 v2, 32, v2
+; GFX9-G-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[4:5]
+; GFX9-G-NEXT: v_add_u32_e32 v0, 64, v0
+; GFX9-G-NEXT: v_min_u32_e32 v1, v1, v2
+; GFX9-G-NEXT: v_ffbh_u32_e32 v2, v8
+; GFX9-G-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX9-G-NEXT: v_ffbh_u32_e32 v1, v9
; GFX9-G-NEXT: v_add_u32_e32 v2, 32, v2
-; GFX9-G-NEXT: v_ffbh_u32_e32 v3, v1
-; GFX9-G-NEXT: v_min_u32_e32 v2, v2, v3
-; GFX9-G-NEXT: v_ffbh_u32_e32 v3, v21
-; GFX9-G-NEXT: v_add_u32_e32 v3, 32, v3
-; GFX9-G-NEXT: v_ffbh_u32_e32 v4, v22
-; GFX9-G-NEXT: v_min_u32_e32 v3, v3, v4
-; GFX9-G-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-G-NEXT: v_add_co_u32_e32 v3, vcc, 64, v3
-; GFX9-G-NEXT: v_addc_co_u32_e64 v4, s[6:7], 0, 0, vcc
-; GFX9-G-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
-; GFX9-G-NEXT: v_ffbh_u32_e32 v5, v11
-; GFX9-G-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc
; GFX9-G-NEXT: v_ffbh_u32_e32 v3, v10
+; GFX9-G-NEXT: v_min_u32_e32 v1, v1, v2
+; GFX9-G-NEXT: v_ffbh_u32_e32 v2, v11
; GFX9-G-NEXT: v_add_u32_e32 v3, 32, v3
-; GFX9-G-NEXT: v_min_u32_e32 v3, v3, v5
-; GFX9-G-NEXT: v_ffbh_u32_e32 v5, v8
-; GFX9-G-NEXT: v_add_u32_e32 v5, 32, v5
-; GFX9-G-NEXT: v_ffbh_u32_e32 v6, v9
-; GFX9-G-NEXT: v_min_u32_e32 v5, v5, v6
-; GFX9-G-NEXT: v_cndmask_b32_e64 v4, v4, 0, vcc
-; GFX9-G-NEXT: v_add_co_u32_e32 v5, vcc, 64, v5
-; GFX9-G-NEXT: v_addc_co_u32_e64 v6, s[6:7], 0, 0, vcc
-; GFX9-G-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
-; GFX9-G-NEXT: v_mov_b32_e32 v19, v17
-; GFX9-G-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
-; GFX9-G-NEXT: v_cndmask_b32_e64 v6, v6, 0, vcc
-; GFX9-G-NEXT: v_sub_co_u32_e32 v2, vcc, v2, v3
-; GFX9-G-NEXT: v_subb_co_u32_e32 v3, vcc, v4, v6, vcc
-; GFX9-G-NEXT: v_subb_co_u32_e64 v4, s[6:7], 0, 0, vcc
-; GFX9-G-NEXT: v_subb_co_u32_e64 v5, s[6:7], 0, 0, s[6:7]
-; GFX9-G-NEXT: s_mov_b64 s[6:7], 0x7f
-; GFX9-G-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GFX9-G-NEXT: v_mov_b32_e32 v20, v18
+; GFX9-G-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[10:11]
+; GFX9-G-NEXT: v_add_u32_e32 v1, 64, v1
+; GFX9-G-NEXT: v_min_u32_e32 v2, v2, v3
+; GFX9-G-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX9-G-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v1
+; GFX9-G-NEXT: v_subb_co_u32_e64 v1, s[4:5], 0, 0, vcc
+; GFX9-G-NEXT: v_mov_b32_e32 v6, 0x7f
+; GFX9-G-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-G-NEXT: v_subb_co_u32_e64 v2, s[4:5], 0, 0, s[4:5]
+; GFX9-G-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[6:7]
+; GFX9-G-NEXT: v_subb_co_u32_e64 v3, s[4:5], 0, 0, s[4:5]
; GFX9-G-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; GFX9-G-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; GFX9-G-NEXT: v_cmp_lt_u64_e32 vcc, 0, v[2:3]
+; GFX9-G-NEXT: s_mov_b64 s[6:7], exec
; GFX9-G-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; GFX9-G-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[4:5]
+; GFX9-G-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3]
+; GFX9-G-NEXT: s_mov_b64 s[8:9], 0
; GFX9-G-NEXT: v_cndmask_b32_e32 v6, v7, v6, vcc
; GFX9-G-NEXT: v_and_b32_e32 v6, 1, v6
-; GFX9-G-NEXT: v_cmp_eq_u32_e32 vcc, 1, v6
-; GFX9-G-NEXT: v_xor_b32_e32 v6, 0x7f, v2
-; GFX9-G-NEXT: v_or_b32_e32 v6, v6, v4
-; GFX9-G-NEXT: v_or_b32_e32 v7, v3, v5
-; GFX9-G-NEXT: s_or_b64 s[4:5], s[4:5], vcc
-; GFX9-G-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
-; GFX9-G-NEXT: s_xor_b64 s[6:7], s[4:5], -1
-; GFX9-G-NEXT: v_cndmask_b32_e64 v13, v11, 0, s[4:5]
-; GFX9-G-NEXT: v_cndmask_b32_e64 v12, v10, 0, s[4:5]
-; GFX9-G-NEXT: v_cndmask_b32_e64 v7, v9, 0, s[4:5]
-; GFX9-G-NEXT: v_cndmask_b32_e64 v6, v8, 0, s[4:5]
-; GFX9-G-NEXT: s_and_b64 s[4:5], s[6:7], vcc
+; GFX9-G-NEXT: v_cmp_ne_u16_e32 vcc, 0, v6
+; GFX9-G-NEXT: v_xor_b32_e32 v6, 0x7f, v0
+; GFX9-G-NEXT: v_or_b32_e32 v6, v6, v2
+; GFX9-G-NEXT: v_or_b32_e32 v7, v1, v3
+; GFX9-G-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[6:7]
+; GFX9-G-NEXT: s_or_b64 s[10:11], s[10:11], vcc
+; GFX9-G-NEXT: s_or_b64 s[4:5], s[10:11], s[4:5]
+; GFX9-G-NEXT: v_cndmask_b32_e64 v6, v8, 0, s[10:11]
+; GFX9-G-NEXT: v_cndmask_b32_e64 v7, v9, 0, s[10:11]
+; GFX9-G-NEXT: v_cndmask_b32_e64 v12, v10, 0, s[10:11]
+; GFX9-G-NEXT: v_cndmask_b32_e64 v13, v11, 0, s[10:11]
+; GFX9-G-NEXT: s_xor_b64 s[4:5], s[4:5], s[6:7]
; GFX9-G-NEXT: s_and_saveexec_b64 s[6:7], s[4:5]
; GFX9-G-NEXT: s_cbranch_execz .LBB0_6
; GFX9-G-NEXT: ; %bb.1: ; %udiv-bb1
-; GFX9-G-NEXT: v_add_co_u32_e32 v23, vcc, 1, v2
-; GFX9-G-NEXT: v_addc_co_u32_e32 v24, vcc, 0, v3, vcc
-; GFX9-G-NEXT: v_addc_co_u32_e32 v25, vcc, 0, v4, vcc
-; GFX9-G-NEXT: v_sub_u32_e32 v7, 0x7f, v2
-; GFX9-G-NEXT: v_addc_co_u32_e32 v26, vcc, 0, v5, vcc
-; GFX9-G-NEXT: v_sub_u32_e32 v5, 64, v7
-; GFX9-G-NEXT: v_lshlrev_b64 v[3:4], v7, v[10:11]
-; GFX9-G-NEXT: v_lshrrev_b64 v[5:6], v5, v[8:9]
-; GFX9-G-NEXT: v_sub_u32_e32 v2, 63, v2
-; GFX9-G-NEXT: v_or_b32_e32 v5, v3, v5
-; GFX9-G-NEXT: v_lshlrev_b64 v[2:3], v2, v[8:9]
-; GFX9-G-NEXT: s_xor_b64 s[8:9], vcc, -1
-; GFX9-G-NEXT: v_or_b32_e32 v4, v4, v6
-; GFX9-G-NEXT: v_cmp_gt_u32_e32 vcc, 64, v7
-; GFX9-G-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
-; GFX9-G-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX9-G-NEXT: v_lshlrev_b64 v[4:5], v7, v[8:9]
-; GFX9-G-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v7
-; GFX9-G-NEXT: v_cndmask_b32_e64 v3, v3, v11, s[4:5]
-; GFX9-G-NEXT: v_cndmask_b32_e64 v2, v2, v10, s[4:5]
-; GFX9-G-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc
-; GFX9-G-NEXT: v_mov_b32_e32 v6, 0
-; GFX9-G-NEXT: v_mov_b32_e32 v7, 0
-; GFX9-G-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
-; GFX9-G-NEXT: s_and_saveexec_b64 s[4:5], s[8:9]
-; GFX9-G-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
+; GFX9-G-NEXT: v_add_co_u32_e32 v20, vcc, 1, v0
+; GFX9-G-NEXT: v_addc_co_u32_e32 v21, vcc, 0, v1, vcc
+; GFX9-G-NEXT: v_addc_co_u32_e32 v22, vcc, 0, v2, vcc
+; GFX9-G-NEXT: s_mov_b64 s[4:5], exec
+; GFX9-G-NEXT: v_addc_co_u32_e32 v23, vcc, 0, v3, vcc
+; GFX9-G-NEXT: s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX9-G-NEXT: v_sub_co_u32_e32 v12, vcc, 0x7f, v0
+; GFX9-G-NEXT: v_sub_u32_e32 v0, 64, v12
+; GFX9-G-NEXT: v_lshrrev_b64 v[0:1], v0, v[8:9]
+; GFX9-G-NEXT: v_lshlrev_b64 v[2:3], v12, v[10:11]
+; GFX9-G-NEXT: v_add_u32_e32 v13, 0xffffffc0, v12
+; GFX9-G-NEXT: v_lshlrev_b64 v[6:7], v12, v[8:9]
+; GFX9-G-NEXT: v_or_b32_e32 v2, v0, v2
+; GFX9-G-NEXT: v_or_b32_e32 v3, v1, v3
+; GFX9-G-NEXT: v_lshlrev_b64 v[0:1], v13, v[8:9]
+; GFX9-G-NEXT: v_cmp_gt_u32_e32 vcc, 64, v12
+; GFX9-G-NEXT: v_cndmask_b32_e32 v6, 0, v6, vcc
+; GFX9-G-NEXT: v_cndmask_b32_e32 v7, 0, v7, vcc
+; GFX9-G-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-G-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX9-G-NEXT: v_cmp_eq_u32_e32 vcc, 0, v12
+; GFX9-G-NEXT: v_cndmask_b32_e32 v12, v0, v10, vcc
+; GFX9-G-NEXT: v_cndmask_b32_e32 v13, v1, v11, vcc
+; GFX9-G-NEXT: s_mov_b64 s[10:11], s[8:9]
+; GFX9-G-NEXT: v_mov_b32_e32 v0, s8
+; GFX9-G-NEXT: v_mov_b32_e32 v1, s9
+; GFX9-G-NEXT: v_mov_b32_e32 v2, s10
+; GFX9-G-NEXT: v_mov_b32_e32 v3, s11
+; GFX9-G-NEXT: s_and_saveexec_b64 s[8:9], s[4:5]
+; GFX9-G-NEXT: s_xor_b64 s[12:13], exec, s[8:9]
; GFX9-G-NEXT: s_cbranch_execz .LBB0_5
; GFX9-G-NEXT: ; %bb.2: ; %udiv-preheader
-; GFX9-G-NEXT: v_sub_u32_e32 v12, 64, v23
-; GFX9-G-NEXT: v_lshrrev_b64 v[6:7], v23, v[8:9]
-; GFX9-G-NEXT: v_lshlrev_b64 v[12:13], v12, v[10:11]
-; GFX9-G-NEXT: v_cmp_gt_u32_e32 vcc, 64, v23
-; GFX9-G-NEXT: v_or_b32_e32 v14, v6, v12
-; GFX9-G-NEXT: v_subrev_u32_e32 v6, 64, v23
-; GFX9-G-NEXT: v_or_b32_e32 v13, v7, v13
-; GFX9-G-NEXT: v_lshrrev_b64 v[6:7], v6, v[10:11]
-; GFX9-G-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v23
-; GFX9-G-NEXT: v_cndmask_b32_e32 v7, v7, v13, vcc
-; GFX9-G-NEXT: v_cndmask_b32_e64 v12, v7, v9, s[4:5]
-; GFX9-G-NEXT: v_cndmask_b32_e32 v9, v6, v14, vcc
-; GFX9-G-NEXT: v_lshrrev_b64 v[6:7], v23, v[10:11]
-; GFX9-G-NEXT: v_cndmask_b32_e64 v11, v9, v8, s[4:5]
-; GFX9-G-NEXT: v_cndmask_b32_e32 v14, 0, v7, vcc
-; GFX9-G-NEXT: v_cndmask_b32_e32 v13, 0, v6, vcc
-; GFX9-G-NEXT: v_add_co_u32_e32 v10, vcc, -1, v21
-; GFX9-G-NEXT: v_addc_co_u32_e32 v27, vcc, -1, v22, vcc
-; GFX9-G-NEXT: v_addc_co_u32_e32 v28, vcc, -1, v0, vcc
-; GFX9-G-NEXT: v_addc_co_u32_e32 v29, vcc, -1, v1, vcc
-; GFX9-G-NEXT: v_mov_b32_e32 v8, 0
+; GFX9-G-NEXT: v_sub_u32_e32 v2, 64, v20
+; GFX9-G-NEXT: v_lshrrev_b64 v[0:1], v20, v[8:9]
+; GFX9-G-NEXT: v_lshlrev_b64 v[2:3], v2, v[10:11]
+; GFX9-G-NEXT: v_add_u32_e32 v24, 0xffffffc0, v20
+; GFX9-G-NEXT: v_lshrrev_b64 v[14:15], v20, v[10:11]
+; GFX9-G-NEXT: v_or_b32_e32 v2, v0, v2
+; GFX9-G-NEXT: v_or_b32_e32 v3, v1, v3
+; GFX9-G-NEXT: v_lshrrev_b64 v[0:1], v24, v[10:11]
+; GFX9-G-NEXT: v_cmp_gt_u32_e32 vcc, 64, v20
+; GFX9-G-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-G-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX9-G-NEXT: v_cndmask_b32_e32 v14, 0, v14, vcc
+; GFX9-G-NEXT: v_cndmask_b32_e32 v15, 0, v15, vcc
+; GFX9-G-NEXT: v_add_co_u32_e32 v24, vcc, -1, v18
+; GFX9-G-NEXT: v_addc_co_u32_e32 v25, vcc, -1, v19, vcc
+; GFX9-G-NEXT: s_mov_b64 s[8:9], 0
+; GFX9-G-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v20
+; GFX9-G-NEXT: v_addc_co_u32_e32 v26, vcc, -1, v4, vcc
+; GFX9-G-NEXT: v_cndmask_b32_e64 v10, v0, v8, s[4:5]
+; GFX9-G-NEXT: v_cndmask_b32_e64 v11, v1, v9, s[4:5]
+; GFX9-G-NEXT: v_addc_co_u32_e32 v27, vcc, -1, v5, vcc
+; GFX9-G-NEXT: s_mov_b64 s[10:11], s[8:9]
; GFX9-G-NEXT: v_mov_b32_e32 v9, 0
-; GFX9-G-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-G-NEXT: v_mov_b32_e32 v15, 0
-; GFX9-G-NEXT: v_mov_b32_e32 v16, 0
-; GFX9-G-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-G-NEXT: v_mov_b32_e32 v0, s8
+; GFX9-G-NEXT: v_mov_b32_e32 v1, s9
+; GFX9-G-NEXT: v_mov_b32_e32 v8, 1
+; GFX9-G-NEXT: v_mov_b32_e32 v2, s10
+; GFX9-G-NEXT: v_mov_b32_e32 v3, s11
; GFX9-G-NEXT: .LBB0_3: ; %udiv-do-while
; GFX9-G-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-G-NEXT: v_lshlrev_b64 v[30:31], 1, v[4:5]
-; GFX9-G-NEXT: v_lshlrev_b64 v[13:14], 1, v[13:14]
-; GFX9-G-NEXT: v_or_b32_e32 v4, v15, v30
-; GFX9-G-NEXT: v_lshrrev_b32_e32 v15, 31, v12
-; GFX9-G-NEXT: v_lshlrev_b64 v[11:12], 1, v[11:12]
-; GFX9-G-NEXT: v_or_b32_e32 v13, v13, v15
-; GFX9-G-NEXT: v_lshrrev_b32_e32 v15, 31, v3
-; GFX9-G-NEXT: v_or_b32_e32 v11, v11, v15
-; GFX9-G-NEXT: v_sub_co_u32_e32 v15, vcc, v10, v11
-; GFX9-G-NEXT: v_subb_co_u32_e32 v15, vcc, v27, v12, vcc
-; GFX9-G-NEXT: v_subb_co_u32_e32 v15, vcc, v28, v13, vcc
-; GFX9-G-NEXT: v_subb_co_u32_e32 v15, vcc, v29, v14, vcc
-; GFX9-G-NEXT: v_ashrrev_i32_e32 v30, 31, v15
-; GFX9-G-NEXT: v_and_b32_e32 v15, v30, v21
-; GFX9-G-NEXT: v_lshrrev_b32_e32 v6, 31, v5
-; GFX9-G-NEXT: v_or_b32_e32 v5, v16, v31
-; GFX9-G-NEXT: v_and_b32_e32 v16, v30, v22
-; GFX9-G-NEXT: v_sub_co_u32_e32 v11, vcc, v11, v15
-; GFX9-G-NEXT: v_subb_co_u32_e32 v12, vcc, v12, v16, vcc
-; GFX9-G-NEXT: v_and_b32_e32 v15, v30, v0
-; GFX9-G-NEXT: v_and_b32_e32 v16, v30, v1
-; GFX9-G-NEXT: v_subb_co_u32_e32 v13, vcc, v13, v15, vcc
-; GFX9-G-NEXT: v_subb_co_u32_e32 v14, vcc, v14, v16, vcc
-; GFX9-G-NEXT: v_add_co_u32_e32 v23, vcc, -1, v23
-; GFX9-G-NEXT: v_addc_co_u32_e32 v24, vcc, -1, v24, vcc
-; GFX9-G-NEXT: v_addc_co_u32_e32 v25, vcc, -1, v25, vcc
-; GFX9-G-NEXT: v_addc_co_u32_e32 v26, vcc, -1, v26, vcc
-; GFX9-G-NEXT: v_lshlrev_b64 v[2:3], 1, v[2:3]
-; GFX9-G-NEXT: v_or_b32_e32 v15, v23, v25
-; GFX9-G-NEXT: v_or_b32_e32 v16, v24, v26
-; GFX9-G-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[15:16]
-; GFX9-G-NEXT: v_or3_b32 v2, v2, v6, v8
-; GFX9-G-NEXT: v_and_b32_e32 v6, 1, v30
-; GFX9-G-NEXT: v_or3_b32 v3, v3, 0, v9
-; GFX9-G-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-G-NEXT: v_mov_b32_e32 v16, v7
-; GFX9-G-NEXT: v_mov_b32_e32 v15, v6
-; GFX9-G-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-G-NEXT: v_lshlrev_b64 v[2:3], 1, v[6:7]
+; GFX9-G-NEXT: v_lshrrev_b32_e32 v8, 31, v7
+; GFX9-G-NEXT: v_or_b32_e32 v6, v0, v2
+; GFX9-G-NEXT: v_or_b32_e32 v7, v1, v3
+; GFX9-G-NEXT: v_lshlrev_b64 v[2:3], 1, v[10:11]
+; GFX9-G-NEXT: v_lshrrev_b32_e32 v10, 31, v13
+; GFX9-G-NEXT: v_lshlrev_b64 v[0:1], 1, v[14:15]
+; GFX9-G-NEXT: v_or_b32_e32 v2, v2, v10
+; GFX9-G-NEXT: v_lshrrev_b32_e32 v14, 31, v11
+; GFX9-G-NEXT: v_sub_co_u32_e32 v10, vcc, v24, v2
+; GFX9-G-NEXT: v_or_b32_e32 v0, v0, v14
+; GFX9-G-NEXT: v_subb_co_u32_e32 v10, vcc, v25, v3, vcc
+; GFX9-G-NEXT: v_subb_co_u32_e32 v10, vcc, v26, v0, vcc
+; GFX9-G-NEXT: v_subb_co_u32_e32 v10, vcc, v27, v1, vcc
+; GFX9-G-NEXT: v_ashrrev_i32_e32 v28, 31, v10
+; GFX9-G-NEXT: v_and_b32_e32 v10, v28, v18
+; GFX9-G-NEXT: v_and_b32_e32 v11, v28, v19
+; GFX9-G-NEXT: v_sub_co_u32_e32 v10, vcc, v2, v10
+; GFX9-G-NEXT: v_subb_co_u32_e32 v11, vcc, v3, v11, vcc
+; GFX9-G-NEXT: v_and_b32_e32 v2, v28, v4
+; GFX9-G-NEXT: v_and_b32_e32 v3, v28, v5
+; GFX9-G-NEXT: v_subb_co_u32_e32 v14, vcc, v0, v2, vcc
+; GFX9-G-NEXT: v_subb_co_u32_e32 v15, vcc, v1, v3, vcc
+; GFX9-G-NEXT: v_add_co_u32_e32 v20, vcc, -1, v20
+; GFX9-G-NEXT: v_addc_co_u32_e32 v21, vcc, -1, v21, vcc
+; GFX9-G-NEXT: v_addc_co_u32_e32 v22, vcc, -1, v22, vcc
+; GFX9-G-NEXT: v_addc_co_u32_e32 v23, vcc, -1, v23, vcc
+; GFX9-G-NEXT: v_or_b32_e32 v0, v20, v22
+; GFX9-G-NEXT: v_or_b32_e32 v1, v21, v23
+; GFX9-G-NEXT: v_lshlrev_b64 v[12:13], 1, v[12:13]
+; GFX9-G-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
+; GFX9-G-NEXT: v_or_b32_e32 v12, v12, v8
+; GFX9-G-NEXT: v_and_b32_e32 v8, 1, v28
+; GFX9-G-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX9-G-NEXT: v_mov_b32_e32 v0, v8
+; GFX9-G-NEXT: v_mov_b32_e32 v1, v9
+; GFX9-G-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX9-G-NEXT: s_cbranch_execnz .LBB0_3
; GFX9-G-NEXT: ; %bb.4: ; %Flow
-; GFX9-G-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-G-NEXT: .LBB0_5: ; %Flow2
; GFX9-G-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX9-G-NEXT: v_lshlrev_b64 v[12:13], 1, v[2:3]
-; GFX9-G-NEXT: v_lshlrev_b64 v[0:1], 1, v[4:5]
-; GFX9-G-NEXT: v_lshrrev_b32_e32 v2, 31, v5
-; GFX9-G-NEXT: v_or_b32_e32 v12, v12, v2
-; GFX9-G-NEXT: v_or_b32_e32 v7, v7, v1
-; GFX9-G-NEXT: v_or_b32_e32 v6, v6, v0
+; GFX9-G-NEXT: .LBB0_5: ; %Flow2
+; GFX9-G-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX9-G-NEXT: v_lshlrev_b64 v[2:3], 1, v[6:7]
+; GFX9-G-NEXT: v_lshlrev_b64 v[12:13], 1, v[12:13]
+; GFX9-G-NEXT: v_lshrrev_b32_e32 v4, 31, v7
+; GFX9-G-NEXT: v_or_b32_e32 v12, v12, v4
+; GFX9-G-NEXT: v_or_b32_e32 v6, v0, v2
+; GFX9-G-NEXT: v_or_b32_e32 v7, v1, v3
; GFX9-G-NEXT: .LBB0_6: ; %Flow3
; GFX9-G-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX9-G-NEXT: v_xor_b32_e32 v2, v18, v17
-; GFX9-G-NEXT: v_xor_b32_e32 v3, v20, v19
-; GFX9-G-NEXT: v_xor_b32_e32 v0, v6, v2
+; GFX9-G-NEXT: v_xor_b32_e32 v3, v17, v16
+; GFX9-G-NEXT: v_xor_b32_e32 v0, v6, v3
; GFX9-G-NEXT: v_xor_b32_e32 v1, v7, v3
-; GFX9-G-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v2
-; GFX9-G-NEXT: v_xor_b32_e32 v5, v12, v2
+; GFX9-G-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v3
+; GFX9-G-NEXT: v_xor_b32_e32 v2, v12, v3
; GFX9-G-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v3, vcc
; GFX9-G-NEXT: v_xor_b32_e32 v4, v13, v3
-; GFX9-G-NEXT: v_subb_co_u32_e32 v2, vcc, v5, v2, vcc
+; GFX9-G-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v3, vcc
; GFX9-G-NEXT: v_subb_co_u32_e32 v3, vcc, v4, v3, vcc
; GFX9-G-NEXT: s_setpc_b64 s[30:31]
;
@@ -1374,297 +1375,234 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0: ; %bb.0: ; %_udiv-special-cases
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-G-O0-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GFX9-G-O0-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v4
-; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v2
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v1
-; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v0
-; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 def $vgpr11_vgpr12 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v5
-; GFX9-G-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v2
-; GFX9-G-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v4
-; GFX9-G-O0-NEXT: ; kill: def $vgpr3 killed $vgpr3 def $vgpr3_vgpr4 killed $exec
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v0
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 63
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v4
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v2
; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v3
-; GFX9-G-O0-NEXT: v_ashrrev_i64 v[8:9], s4, v[8:9]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v2
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v1
-; GFX9-G-O0-NEXT: v_ashrrev_i64 v[13:14], s4, v[13:14]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v4
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v9
-; GFX9-G-O0-NEXT: v_xor_b32_e64 v0, v5, v0
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v3
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v8
-; GFX9-G-O0-NEXT: v_xor_b32_e64 v15, v3, v4
-; GFX9-G-O0-NEXT: ; kill: def $vgpr15 killed $vgpr15 def $vgpr15_vgpr16 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, v0
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v7
-; GFX9-G-O0-NEXT: v_xor_b32_e64 v0, v5, v0
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v6
-; GFX9-G-O0-NEXT: v_xor_b32_e64 v8, v3, v4
-; GFX9-G-O0-NEXT: ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v0
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v8
-; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v7, s[4:5], v0, v3
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v9
-; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v0, s[4:5], v0, v5, s[4:5]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v15
-; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v9, s[4:5], v4, v3, s[4:5]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v16
-; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v4, s[4:5], v4, v5, s[4:5]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v4
-; GFX9-G-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v0
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v2
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v14
-; GFX9-G-O0-NEXT: v_xor_b32_e64 v0, v4, v0
-; GFX9-G-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr1_vgpr2 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v13
-; GFX9-G-O0-NEXT: v_xor_b32_e64 v13, v2, v1
-; GFX9-G-O0-NEXT: ; kill: def $vgpr13 killed $vgpr13 def $vgpr13_vgpr14 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v0
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v12
-; GFX9-G-O0-NEXT: v_xor_b32_e64 v0, v4, v0
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v11
-; GFX9-G-O0-NEXT: v_xor_b32_e64 v11, v2, v1
-; GFX9-G-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 def $vgpr11_vgpr12 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v0
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v11
-; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v0, s[4:5], v0, v2
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v12
-; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v6, s[4:5], v1, v4, s[4:5]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v13
-; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v12, s[4:5], v1, v2, s[4:5]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v14
-; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v1, s[4:5], v1, v4, s[4:5]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr12 killed $vgpr12 def $vgpr12_vgpr13 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v1
-; GFX9-G-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v6
-; GFX9-G-O0-NEXT: v_xor_b32_e64 v4, v4, v5
-; GFX9-G-O0-NEXT: v_xor_b32_e64 v2, v2, v3
-; GFX9-G-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v4
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v3
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v2
-; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v12
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v13
-; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v7
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v8
-; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v10
; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v9
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v10
-; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v8
+; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v13
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v1
-; GFX9-G-O0-NEXT: v_or_b32_e64 v2, v11, v6
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v12
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v0
-; GFX9-G-O0-NEXT: v_or_b32_e64 v0, v5, v4
-; GFX9-G-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v2
+; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v4
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v5
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v6
+; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: ; kill: def $vgpr12 killed $vgpr12 def $vgpr12_vgpr13_vgpr14_vgpr15 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v0
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, v7
; GFX9-G-O0-NEXT: s_mov_b64 s[8:9], 0
; GFX9-G-O0-NEXT: ; implicit-def: $vgpr31 : SGPR spill to VGPR lane
; GFX9-G-O0-NEXT: v_writelane_b32 v31, s8, 0
; GFX9-G-O0-NEXT: v_writelane_b32 v31, s9, 1
-; GFX9-G-O0-NEXT: v_cmp_eq_u64_e64 s[4:5], v[0:1], s[8:9]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v10
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v8
-; GFX9-G-O0-NEXT: v_or_b32_e64 v14, v3, v1
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v9
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v7
-; GFX9-G-O0-NEXT: v_or_b32_e64 v7, v2, v0
-; GFX9-G-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v14
-; GFX9-G-O0-NEXT: v_cmp_eq_u64_e64 s[6:7], v[7:8], s[8:9]
-; GFX9-G-O0-NEXT: s_or_b64 s[4:5], s[4:5], s[6:7]
-; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v4, v4
-; GFX9-G-O0-NEXT: s_mov_b32 s10, 32
-; GFX9-G-O0-NEXT: v_add_u32_e64 v4, v4, s10
-; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v6, v6
-; GFX9-G-O0-NEXT: v_min_u32_e64 v6, v4, v6
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-G-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v4
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v7
-; GFX9-G-O0-NEXT: v_cmp_ne_u64_e64 s[6:7], v[12:13], s[8:9]
+; GFX9-G-O0-NEXT: s_mov_b64 s[6:7], 0x7f
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v5
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v6
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v1
+; GFX9-G-O0-NEXT: s_mov_b32 s4, 31
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s4
+; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v11, v2, v7
+; GFX9-G-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr0_vgpr1 killed $exec
+; GFX9-G-O0-NEXT: s_mov_b32 s4, 31
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v9, v0, v1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v14
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v15
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v1
+; GFX9-G-O0-NEXT: s_mov_b32 s4, 31
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s4
+; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v10, v2, v7
+; GFX9-G-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr0_vgpr1 killed $exec
+; GFX9-G-O0-NEXT: s_mov_b32 s4, 31
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v8, v0, v1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v3
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v4
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v5
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v6
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v2
+; GFX9-G-O0-NEXT: v_xor_b32_e64 v0, v11, v0
+; GFX9-G-O0-NEXT: v_xor_b32_e64 v1, v11, v1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v4
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v5
+; GFX9-G-O0-NEXT: v_xor_b32_e64 v3, v9, v3
+; GFX9-G-O0-NEXT: v_xor_b32_e64 v2, v9, v2
+; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v0, s[4:5], v0, v11
+; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v1, s[4:5], v1, v11, s[4:5]
+; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v4, s[4:5], v3, v9, s[4:5]
+; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v2, s[4:5], v2, v9, s[4:5]
+; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v12
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v13
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v14
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v15
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v6
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v7
+; GFX9-G-O0-NEXT: v_xor_b32_e64 v7, v10, v5
+; GFX9-G-O0-NEXT: v_xor_b32_e64 v5, v10, v3
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v12
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v13
+; GFX9-G-O0-NEXT: v_xor_b32_e64 v6, v8, v6
+; GFX9-G-O0-NEXT: v_xor_b32_e64 v3, v8, v3
+; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v7, s[4:5], v7, v10
+; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v5, s[4:5], v5, v10, s[4:5]
+; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v6, s[4:5], v6, v8, s[4:5]
+; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v3, s[4:5], v3, v8, s[4:5]
+; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: v_xor_b32_e64 v12, v10, v11
+; GFX9-G-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: v_xor_b32_e64 v10, v10, v11
+; GFX9-G-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: v_xor_b32_e64 v10, v8, v9
+; GFX9-G-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: v_xor_b32_e64 v8, v8, v9
+; GFX9-G-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: v_or_b32_e64 v8, v7, v6
+; GFX9-G-O0-NEXT: v_or_b32_e64 v10, v5, v3
+; GFX9-G-O0-NEXT: ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, s9
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, s8
+; GFX9-G-O0-NEXT: v_cmp_eq_u64_e64 s[4:5], v[8:9], v[10:11]
+; GFX9-G-O0-NEXT: v_or_b32_e64 v8, v0, v4
+; GFX9-G-O0-NEXT: v_or_b32_e64 v10, v1, v2
+; GFX9-G-O0-NEXT: ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, s9
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, s8
+; GFX9-G-O0-NEXT: v_cmp_eq_u64_e64 s[10:11], v[8:9], v[10:11]
+; GFX9-G-O0-NEXT: s_or_b64 s[4:5], s[4:5], s[10:11]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v6
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v3
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, s9
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, s8
+; GFX9-G-O0-NEXT: v_cmp_eq_u64_e64 s[10:11], v[8:9], v[10:11]
; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v5, v5
-; GFX9-G-O0-NEXT: v_add_u32_e64 v5, v5, s10
-; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v11, v11
-; GFX9-G-O0-NEXT: v_min_u32_e64 v12, v5, v11
-; GFX9-G-O0-NEXT: ; kill: def $vgpr12 killed $vgpr12 def $vgpr12_vgpr13 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v4
-; GFX9-G-O0-NEXT: s_mov_b64 s[12:13], 64
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v12
-; GFX9-G-O0-NEXT: s_mov_b32 s14, s12
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v13
-; GFX9-G-O0-NEXT: s_mov_b32 s11, s13
-; GFX9-G-O0-NEXT: v_add_co_u32_e64 v11, s[14:15], v11, s14
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, s11
-; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v5, s[14:15], v5, v12, s[14:15]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 def $vgpr11_vgpr12 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v5
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v12
-; GFX9-G-O0-NEXT: s_mov_b64 s[14:15], s[6:7]
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v5, v5, v8, s[14:15]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v6
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v11
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v7, v6, v7, s[6:7]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v5
-; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v5, v0
-; GFX9-G-O0-NEXT: v_add_u32_e64 v5, v5, s10
-; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v6, v1
+; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v7, v7
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, 32
+; GFX9-G-O0-NEXT: v_add_u32_e64 v7, v7, v8
+; GFX9-G-O0-NEXT: v_min_u32_e64 v5, v5, v7
+; GFX9-G-O0-NEXT: s_mov_b32 s12, 64
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s12
+; GFX9-G-O0-NEXT: v_add_u32_e64 v5, v5, v7
+; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v3, v3
+; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v6, v6
+; GFX9-G-O0-NEXT: v_add_u32_e64 v6, v6, v8
+; GFX9-G-O0-NEXT: v_min_u32_e64 v3, v3, v6
+; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v3, v3, v5, s[10:11]
+; GFX9-G-O0-NEXT: s_mov_b32 s16, 0
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v4
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v2
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, s9
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, s8
+; GFX9-G-O0-NEXT: v_cmp_eq_u64_e64 s[10:11], v[5:6], v[9:10]
+; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v5, v1
+; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v6, v0
+; GFX9-G-O0-NEXT: v_add_u32_e64 v6, v6, v8
; GFX9-G-O0-NEXT: v_min_u32_e64 v5, v5, v6
-; GFX9-G-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v4
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v6
-; GFX9-G-O0-NEXT: v_cmp_ne_u64_e64 s[6:7], v[9:10], s[8:9]
-; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v9, v2
-; GFX9-G-O0-NEXT: v_add_u32_e64 v9, v9, s10
-; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v10, v3
-; GFX9-G-O0-NEXT: v_min_u32_e64 v12, v9, v10
-; GFX9-G-O0-NEXT: ; kill: def $vgpr12 killed $vgpr12 def $vgpr12_vgpr13 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v4
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v12
-; GFX9-G-O0-NEXT: s_mov_b32 s10, s12
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v13
-; GFX9-G-O0-NEXT: s_mov_b32 s12, s13
-; GFX9-G-O0-NEXT: v_add_co_u32_e64 v9, s[10:11], v9, s10
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, s12
-; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v4, s[10:11], v4, v10, s[10:11]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v4
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v10
-; GFX9-G-O0-NEXT: s_mov_b64 s[10:11], s[6:7]
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v4, v4, v11, s[10:11]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v5
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v9
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v5, v5, v6, s[6:7]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v4
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v5
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v7
-; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v4, s[6:7], v4, v9
-; GFX9-G-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 killed $vgpr5_vgpr6 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v8
-; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v6, s[10:11], v5, v6, s[6:7]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v6
-; GFX9-G-O0-NEXT: s_mov_b32 s7, s8
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s7
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s7
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s12
+; GFX9-G-O0-NEXT: v_add_u32_e64 v6, v5, v6
+; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v5, v2
+; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v7, v4
+; GFX9-G-O0-NEXT: v_add_u32_e64 v7, v7, v8
+; GFX9-G-O0-NEXT: v_min_u32_e64 v5, v5, v7
+; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v5, v5, v6, s[10:11]
+; GFX9-G-O0-NEXT: s_mov_b32 s15, 0
+; GFX9-G-O0-NEXT: s_mov_b32 s13, 0
+; GFX9-G-O0-NEXT: s_mov_b32 s14, 0
+; GFX9-G-O0-NEXT: s_mov_b32 s12, 0
+; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v5, s[10:11], v3, v5
+; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, s16
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s16
+; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v3, s[10:11], v3, v6, s[10:11]
+; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s15
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s14
; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v7, s[10:11], v6, v7, s[10:11]
-; GFX9-G-O0-NEXT: s_mov_b32 s14, s9
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s14
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, s14
+; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s13
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, s12
; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v6, s[10:11], v6, v8, s[10:11]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v6
+; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v5
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v4
-; GFX9-G-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v8
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v7
-; GFX9-G-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_cmp_eq_u64_e64 s[10:11], v[7:8], s[8:9]
-; GFX9-G-O0-NEXT: s_mov_b64 s[12:13], 0x7f
-; GFX9-G-O0-NEXT: v_cmp_gt_u64_e64 s[16:17], v[4:5], s[12:13]
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v9, 0, 1, s[16:17]
-; GFX9-G-O0-NEXT: v_cmp_ne_u64_e64 s[16:17], v[7:8], s[8:9]
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v6, 0, 1, s[16:17]
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v6, v6, v9, s[10:11]
-; GFX9-G-O0-NEXT: v_and_b32_e64 v6, 1, v6
-; GFX9-G-O0-NEXT: v_cmp_eq_u32_e64 s[10:11], v6, 1
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v6, 0, 1, s[10:11]
-; GFX9-G-O0-NEXT: s_mov_b32 s6, 1
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, s6
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v6, v6, v9, s[4:5]
-; GFX9-G-O0-NEXT: v_and_b32_e64 v6, 1, v6
-; GFX9-G-O0-NEXT: v_cmp_eq_u32_e64 s[4:5], v6, 1
-; GFX9-G-O0-NEXT: s_mov_b64 s[10:11], s[4:5]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v5
-; GFX9-G-O0-NEXT: s_mov_b32 s15, s13
-; GFX9-G-O0-NEXT: v_xor_b32_e64 v6, v6, s15
-; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 killed $vgpr4_vgpr5 killed $exec
-; GFX9-G-O0-NEXT: ; kill: def $sgpr12 killed $sgpr12 killed $sgpr12_sgpr13
-; GFX9-G-O0-NEXT: v_xor_b32_e64 v4, v4, s12
-; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v6
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v5
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v8
-; GFX9-G-O0-NEXT: v_or_b32_e64 v6, v6, v9
-; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 killed $vgpr4_vgpr5 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v7
-; GFX9-G-O0-NEXT: v_or_b32_e64 v4, v4, v5
-; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v3
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v7
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v6
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, s7
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, s6
+; GFX9-G-O0-NEXT: v_cmp_gt_u64_e64 s[14:15], v[10:11], v[12:13]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, s9
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, s8
+; GFX9-G-O0-NEXT: v_cmp_gt_u64_e64 s[12:13], v[8:9], v[10:11]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, s9
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, s8
+; GFX9-G-O0-NEXT: v_cmp_eq_u64_e64 s[10:11], v[8:9], v[10:11]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, 1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, 0
+; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v11, v9, v10, s[14:15]
+; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v8, v9, v10, s[12:13]
+; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v8, v8, v11, s[10:11]
+; GFX9-G-O0-NEXT: v_and_b32_e64 v8, v8, v10
+; GFX9-G-O0-NEXT: v_cmp_ne_u16_e64 s[10:11], v8, v9
+; GFX9-G-O0-NEXT: s_or_b64 s[4:5], s[4:5], s[10:11]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, s7
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, s6
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v11
+; GFX9-G-O0-NEXT: v_xor_b32_e64 v5, v5, v9
+; GFX9-G-O0-NEXT: v_xor_b32_e64 v3, v3, v8
+; GFX9-G-O0-NEXT: v_or_b32_e64 v5, v5, v7
+; GFX9-G-O0-NEXT: v_or_b32_e64 v3, v3, v6
+; GFX9-G-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v3
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s8
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, s9
+; GFX9-G-O0-NEXT: v_cmp_eq_u64_e64 s[6:7], v[5:6], v[7:8]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s8
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s9
; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v6
-; GFX9-G-O0-NEXT: v_cmp_eq_u64_e64 s[8:9], v[4:5], s[8:9]
-; GFX9-G-O0-NEXT: s_mov_b64 s[12:13], s[10:11]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, s14
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v4, v1, v4, s[12:13]
-; GFX9-G-O0-NEXT: s_mov_b64 s[12:13], s[10:11]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, s7
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v0, v0, v1, s[12:13]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v7
+; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v0, v0, v5, s[4:5]
+; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v3, v1, v3, s[4:5]
; GFX9-G-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v4
-; GFX9-G-O0-NEXT: s_mov_b64 s[12:13], s[10:11]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, s14
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v4, v3, v4, s[12:13]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, s7
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v2, v2, v3, s[10:11]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v4
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v4, 0, 1, s[8:9]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, s6
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v3
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s8
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s9
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v6
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v7
; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v4, v4, v5, s[4:5]
-; GFX9-G-O0-NEXT: v_and_b32_e64 v4, 1, v4
-; GFX9-G-O0-NEXT: v_cmp_eq_u32_e64 s[4:5], v4, 1
-; GFX9-G-O0-NEXT: s_mov_b64 s[6:7], -1
+; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v2, v2, v3, s[4:5]
+; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v2
+; GFX9-G-O0-NEXT: ; kill: def $vgpr0_vgpr1 killed $vgpr0_vgpr1 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v4
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v5
+; GFX9-G-O0-NEXT: s_or_b64 s[4:5], s[4:5], s[6:7]
+; GFX9-G-O0-NEXT: s_mov_b64 s[6:7], exec
; GFX9-G-O0-NEXT: s_xor_b64 s[6:7], s[4:5], s[6:7]
-; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_nop 0
; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_mov_b64 s[4:5], exec
; GFX9-G-O0-NEXT: v_writelane_b32 v31, s4, 2
; GFX9-G-O0-NEXT: v_writelane_b32 v31, s5, 3
@@ -1673,8 +1611,8 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
; GFX9-G-O0-NEXT: s_and_b64 s[4:5], s[4:5], s[6:7]
; GFX9-G-O0-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-G-O0-NEXT: s_cbranch_execz .LBB0_3
-; GFX9-G-O0-NEXT: s_branch .LBB0_8
+; GFX9-G-O0-NEXT: s_cbranch_execz .LBB0_2
+; GFX9-G-O0-NEXT: s_branch .LBB0_7
; GFX9-G-O0-NEXT: .LBB0_1: ; %Flow
; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-G-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 ; 4-byte Folded Reload
@@ -1683,33 +1621,29 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_readlane_b32 s4, v31, 4
; GFX9-G-O0-NEXT: v_readlane_b32 s5, v31, 5
; GFX9-G-O0-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-G-O0-NEXT: ; %bb.2: ; %Flow
-; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
-; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
-; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
-; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
-; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
-; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
-; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
-; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
-; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_branch .LBB0_5
-; GFX9-G-O0-NEXT: .LBB0_3: ; %Flow2
+; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_nop 0
+; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_branch .LBB0_4
+; GFX9-G-O0-NEXT: .LBB0_2: ; %Flow2
; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-G-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
@@ -1717,59 +1651,73 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_readlane_b32 s4, v31, 2
; GFX9-G-O0-NEXT: v_readlane_b32 s5, v31, 3
; GFX9-G-O0-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(1)
-; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(1)
-; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_branch .LBB0_9
-; GFX9-G-O0-NEXT: .LBB0_4: ; %udiv-loop-exit
-; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 1
+; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_branch .LBB0_8
+; GFX9-G-O0-NEXT: .LBB0_3: ; %udiv-loop-exit
+; GFX9-G-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(2)
-; GFX9-G-O0-NEXT: v_lshlrev_b64 v[2:3], s4, v[0:1]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v4
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v5
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
-; GFX9-G-O0-NEXT: v_lshlrev_b64 v[9:10], s4, v[9:10]
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 63
-; GFX9-G-O0-NEXT: v_lshrrev_b64 v[0:1], s4, v[0:1]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v1
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v10
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v8
-; GFX9-G-O0-NEXT: v_or3_b32 v4, v4, v11, v12
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v0
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v9
-; GFX9-G-O0-NEXT: v_or3_b32 v0, v0, v1, v7
-; GFX9-G-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v4
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v3
; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v6
-; GFX9-G-O0-NEXT: v_or_b32_e64 v4, v4, v7
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v2
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v5
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v7
+; GFX9-G-O0-NEXT: s_mov_b32 s4, 1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-G-O0-NEXT: v_lshlrev_b64 v[9:10], v0, v[2:3]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-G-O0-NEXT: v_lshlrev_b64 v[0:1], v0, v[4:5]
+; GFX9-G-O0-NEXT: ; kill: def $vgpr3 killed $vgpr3 killed $vgpr2_vgpr3 killed $exec
+; GFX9-G-O0-NEXT: s_mov_b32 s4, 31
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s4
+; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v5, v2, v3
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v0
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v11
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v12
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v13
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v14
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v2
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v9
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v10
+; GFX9-G-O0-NEXT: v_or_b32_e64 v0, v0, v6
+; GFX9-G-O0-NEXT: v_or_b32_e64 v2, v1, v2
+; GFX9-G-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v2
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v7
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v8
+; GFX9-G-O0-NEXT: v_or3_b32 v4, v4, v5, v6
; GFX9-G-O0-NEXT: v_or_b32_e64 v2, v2, v3
-; GFX9-G-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v4
-; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v2
+; GFX9-G-O0-NEXT: ; kill: def $vgpr0_vgpr1 killed $vgpr0_vgpr1 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v4
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v5
; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_nop 0
; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_branch .LBB0_3
-; GFX9-G-O0-NEXT: .LBB0_5: ; %Flow1
+; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_branch .LBB0_2
+; GFX9-G-O0-NEXT: .LBB0_4: ; %Flow1
; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-G-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
@@ -1777,29 +1725,29 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_readlane_b32 s4, v31, 6
; GFX9-G-O0-NEXT: v_readlane_b32 s5, v31, 7
; GFX9-G-O0-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(1)
-; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(1)
-; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_branch .LBB0_4
-; GFX9-G-O0-NEXT: .LBB0_6: ; %udiv-do-while
+; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_branch .LBB0_3
+; GFX9-G-O0-NEXT: .LBB0_5: ; %udiv-do-while
; GFX9-G-O0-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-G-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 ; 4-byte Folded Reload
@@ -1807,467 +1755,433 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-G-O0-NEXT: v_readlane_b32 s6, v31, 8
; GFX9-G-O0-NEXT: v_readlane_b32 s7, v31, 9
-; GFX9-G-O0-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 63
+; GFX9-G-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: s_mov_b64 s[8:9], 1
+; GFX9-G-O0-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(18)
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v2
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v3
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(16)
-; GFX9-G-O0-NEXT: v_lshrrev_b64 v[7:8], s4, v[2:3]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v8
-; GFX9-G-O0-NEXT: s_mov_b32 s5, 1
-; GFX9-G-O0-NEXT: v_lshlrev_b64 v[14:15], s5, v[14:15]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v15
-; GFX9-G-O0-NEXT: v_or_b32_e64 v6, v6, v11
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v7
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v14
-; GFX9-G-O0-NEXT: v_or_b32_e64 v14, v7, v8
-; GFX9-G-O0-NEXT: ; kill: def $vgpr14 killed $vgpr14 def $vgpr14_vgpr15 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, v6
-; GFX9-G-O0-NEXT: v_lshlrev_b64 v[29:30], s5, v[2:3]
-; GFX9-G-O0-NEXT: v_lshrrev_b64 v[6:7], s4, v[27:28]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v30
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v7
-; GFX9-G-O0-NEXT: v_or_b32_e64 v2, v2, v3
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v29
-; GFX9-G-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 killed $vgpr6_vgpr7 killed $exec
-; GFX9-G-O0-NEXT: v_or_b32_e64 v7, v3, v6
-; GFX9-G-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v2
-; GFX9-G-O0-NEXT: v_lshlrev_b64 v[2:3], s5, v[0:1]
-; GFX9-G-O0-NEXT: v_lshlrev_b64 v[27:28], s5, v[27:28]
-; GFX9-G-O0-NEXT: v_lshrrev_b64 v[0:1], s4, v[0:1]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v1
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v28
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, v5
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v4
+; GFX9-G-O0-NEXT: s_mov_b32 s10, 1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s10
+; GFX9-G-O0-NEXT: v_lshlrev_b64 v[2:3], v2, v[0:1]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, s10
+; GFX9-G-O0-NEXT: v_lshlrev_b64 v[4:5], v4, v[14:15]
+; GFX9-G-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr0_vgpr1 killed $exec
+; GFX9-G-O0-NEXT: s_mov_b32 s11, 31
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s11
+; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v1, v0, v1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v4
+; GFX9-G-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 killed $vgpr4_vgpr5 killed $exec
+; GFX9-G-O0-NEXT: v_or_b32_e64 v7, v0, v1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v24
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v25
+; GFX9-G-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr0_vgpr1 killed $exec
+; GFX9-G-O0-NEXT: s_mov_b32 s11, 31
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s11
+; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v1, v0, v1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v2
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v3
+; GFX9-G-O0-NEXT: v_or_b32_e64 v4, v0, v1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v22
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v23
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v24
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, v25
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s10
+; GFX9-G-O0-NEXT: v_lshlrev_b64 v[25:26], v0, v[2:3]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s10
+; GFX9-G-O0-NEXT: v_lshlrev_b64 v[0:1], v0, v[14:15]
+; GFX9-G-O0-NEXT: ; kill: def $vgpr3 killed $vgpr3 killed $vgpr2_vgpr3 killed $exec
+; GFX9-G-O0-NEXT: s_mov_b32 s10, 31
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s10
+; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v15, v2, v3
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v0
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v1
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(10)
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, v26
-; GFX9-G-O0-NEXT: v_or3_b32 v6, v6, v11, v16
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v0
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v27
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v25
-; GFX9-G-O0-NEXT: v_or3_b32 v0, v0, v1, v11
-; GFX9-G-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v6
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v3
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v27
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v28
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(8)
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v24
-; GFX9-G-O0-NEXT: v_or_b32_e64 v6, v6, v11
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v2
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v23
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v23, v29
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v24, v30
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v2
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v22, v25
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v26
+; GFX9-G-O0-NEXT: v_or_b32_e64 v0, v0, v22
+; GFX9-G-O0-NEXT: v_or_b32_e64 v2, v1, v2
+; GFX9-G-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v2
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v22, v23
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v24
+; GFX9-G-O0-NEXT: v_or3_b32 v14, v14, v15, v22
; GFX9-G-O0-NEXT: v_or_b32_e64 v2, v2, v3
-; GFX9-G-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v6
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v7
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(1)
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v4
-; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v11, s[8:9], v11, v6
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v8
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v5
-; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v4, s[8:9], v4, v7, s[8:9]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v14
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v12
-; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v11, s[8:9], v5, v4, s[8:9]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v15
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v13
-; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v8, s[8:9], v8, v5, s[8:9]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 def $vgpr11_vgpr12 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v8
-; GFX9-G-O0-NEXT: v_ashrrev_i64 v[13:14], s4, v[11:12]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v14
-; GFX9-G-O0-NEXT: s_mov_b64 s[4:5], 1
-; GFX9-G-O0-NEXT: s_mov_b32 s8, s5
-; GFX9-G-O0-NEXT: v_and_b32_e64 v12, v8, s8
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v13
-; GFX9-G-O0-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
-; GFX9-G-O0-NEXT: v_and_b32_e64 v14, v11, s4
; GFX9-G-O0-NEXT: ; kill: def $vgpr14 killed $vgpr14 def $vgpr14_vgpr15 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, v12
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, 0
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, 0
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, v20
-; GFX9-G-O0-NEXT: v_and_b32_e64 v16, v8, v16
-; GFX9-G-O0-NEXT: v_and_b32_e64 v19, v11, v19
-; GFX9-G-O0-NEXT: ; kill: def $vgpr19 killed $vgpr19 def $vgpr19_vgpr20 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v20, v16
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, v22
-; GFX9-G-O0-NEXT: v_and_b32_e64 v8, v8, v16
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, v21
-; GFX9-G-O0-NEXT: v_and_b32_e64 v21, v11, v16
-; GFX9-G-O0-NEXT: ; kill: def $vgpr21 killed $vgpr21 def $vgpr21_vgpr22 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v22, v8
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v21
-; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v6, s[4:5], v6, v8
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v22
-; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v8, s[4:5], v7, v8, s[4:5]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v19
-; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v4, s[4:5], v4, v7, s[4:5]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v20
-; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v7, s[4:5], v5, v7, s[4:5]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v7
-; GFX9-G-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, v2
+; GFX9-G-O0-NEXT: ; kill: def $vgpr0_vgpr1 killed $vgpr0_vgpr1 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v14
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v15
+; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_nop 0
+; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(4)
+; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v13, s[10:11], v13, v4
+; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v12, s[10:11], v12, v9, s[10:11]
+; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v10, s[10:11], v10, v7, s[10:11]
+; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v12, s[10:11], v6, v5, s[10:11]
+; GFX9-G-O0-NEXT: s_mov_b32 s10, 31
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s10
+; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v10, v6, v12
+; GFX9-G-O0-NEXT: s_mov_b32 s10, 31
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s10
+; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v6, v6, v12
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, s9
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, s8
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v13
+; GFX9-G-O0-NEXT: ; kill: def $vgpr14 killed $vgpr14 killed $vgpr13_vgpr14 killed $exec
+; GFX9-G-O0-NEXT: v_and_b32_e64 v12, v10, v12
+; GFX9-G-O0-NEXT: ; kill: def $vgpr12 killed $vgpr12 def $vgpr12_vgpr13 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v14
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v23, s5
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v22, s4
+; GFX9-G-O0-NEXT: ; kill: def $vgpr12_vgpr13 killed $vgpr12_vgpr13 def $vgpr12_vgpr13_vgpr14_vgpr15 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v22
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, v23
+; GFX9-G-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_nop 0
+; GFX9-G-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: v_and_b32_e64 v11, v10, v11
+; GFX9-G-O0-NEXT: v_and_b32_e64 v10, v10, v21
+; GFX9-G-O0-NEXT: v_and_b32_e64 v8, v6, v8
+; GFX9-G-O0-NEXT: v_and_b32_e64 v6, v6, v20
+; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v4, s[8:9], v4, v11
+; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v10, s[8:9], v9, v10, s[8:9]
+; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v9, s[8:9], v7, v8, s[8:9]
+; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v8, s[8:9], v5, v6, s[8:9]
+; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5_vgpr6_vgpr7 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v9
; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v8
-; GFX9-G-O0-NEXT: s_mov_b64 s[10:11], -1
-; GFX9-G-O0-NEXT: s_mov_b32 s9, s10
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v17
-; GFX9-G-O0-NEXT: v_add_co_u32_e64 v16, s[4:5], v8, s9
-; GFX9-G-O0-NEXT: s_mov_b32 s8, s11
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v18
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, s8
-; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v8, s[4:5], v8, v11, s[4:5]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v9
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v17, s9
-; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v19, s[4:5], v11, v17, s[4:5]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v10
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, s8
-; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v9, s[4:5], v9, v10, s[4:5]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr19 killed $vgpr19 def $vgpr19_vgpr20 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v20, v9
-; GFX9-G-O0-NEXT: ; kill: def $vgpr16 killed $vgpr16 def $vgpr16_vgpr17 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v17, v8
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v16
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v17
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v16
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v17
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v18
; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v19
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v20
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v18, v17
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v21, v20
-; GFX9-G-O0-NEXT: v_or_b32_e64 v18, v18, v21
-; GFX9-G-O0-NEXT: ; kill: def $vgpr16 killed $vgpr16 killed $vgpr16_vgpr17 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v17, v19
-; GFX9-G-O0-NEXT: v_or_b32_e64 v16, v16, v17
+; GFX9-G-O0-NEXT: s_mov_b32 s8, -1
+; GFX9-G-O0-NEXT: s_mov_b32 s12, -1
+; GFX9-G-O0-NEXT: s_mov_b32 s11, -1
+; GFX9-G-O0-NEXT: s_mov_b32 s10, -1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, s8
+; GFX9-G-O0-NEXT: v_add_co_u32_e64 v16, s[8:9], v11, v16
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, s12
+; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v17, s[8:9], v10, v11, s[8:9]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, s11
+; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v19, s[8:9], v9, v10, s[8:9]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, s10
+; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v18, s[8:9], v8, v9, s[8:9]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v16
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v17
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v19
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v18
+; GFX9-G-O0-NEXT: v_or_b32_e64 v16, v16, v19
+; GFX9-G-O0-NEXT: v_or_b32_e64 v18, v17, v18
; GFX9-G-O0-NEXT: ; kill: def $vgpr16 killed $vgpr16 def $vgpr16_vgpr17 killed $exec
; GFX9-G-O0-NEXT: v_mov_b32_e32 v17, v18
-; GFX9-G-O0-NEXT: v_cmp_eq_u64_e64 s[4:5], v[16:17], v[12:13]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v19, s5
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v18, s4
+; GFX9-G-O0-NEXT: v_cmp_eq_u64_e64 s[4:5], v[16:17], v[18:19]
; GFX9-G-O0-NEXT: s_or_b64 s[4:5], s[4:5], s[6:7]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v17, v3
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, v2
-; GFX9-G-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v17, v1
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, v0
-; GFX9-G-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v17, v15
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, v14
-; GFX9-G-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v17, v13
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, v12
-; GFX9-G-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_mov_b64 s[6:7], s[4:5]
-; GFX9-G-O0-NEXT: v_writelane_b32 v31, s6, 4
-; GFX9-G-O0-NEXT: v_writelane_b32 v31, s7, 5
+; GFX9-G-O0-NEXT: v_writelane_b32 v31, s4, 4
+; GFX9-G-O0-NEXT: v_writelane_b32 v31, s5, 5
; GFX9-G-O0-NEXT: s_mov_b64 s[6:7], s[4:5]
; GFX9-G-O0-NEXT: v_writelane_b32 v31, s6, 8
; GFX9-G-O0-NEXT: v_writelane_b32 v31, s7, 9
; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-G-O0-NEXT: buffer_store_dword v31, off, s[0:3], s32 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
-; GFX9-G-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX9-G-O0-NEXT: s_cbranch_execnz .LBB0_6
+; GFX9-G-O0-NEXT: s_cbranch_execnz .LBB0_5
; GFX9-G-O0-NEXT: s_branch .LBB0_1
-; GFX9-G-O0-NEXT: .LBB0_7: ; %udiv-preheader
+; GFX9-G-O0-NEXT: .LBB0_6: ; %udiv-preheader
; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-G-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
-; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(9)
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v10
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
-; GFX9-G-O0-NEXT: v_lshrrev_b64 v[6:7], v4, v[18:19]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v7
+; GFX9-G-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: s_mov_b32 s4, 64
-; GFX9-G-O0-NEXT: v_sub_u32_e64 v20, s4, v4
-; GFX9-G-O0-NEXT: v_lshlrev_b64 v[20:21], v20, v[16:17]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v22, v21
-; GFX9-G-O0-NEXT: v_or_b32_e64 v5, v5, v22
-; GFX9-G-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 killed $vgpr6_vgpr7 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v20
-; GFX9-G-O0-NEXT: v_or_b32_e64 v6, v6, v7
-; GFX9-G-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v5
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v22, v7
-; GFX9-G-O0-NEXT: v_cmp_lt_u32_e64 s[6:7], v4, s4
-; GFX9-G-O0-NEXT: v_sub_u32_e64 v5, v4, s4
-; GFX9-G-O0-NEXT: v_lshrrev_b64 v[20:21], v5, v[16:17]
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v22, v17
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v23, v16
+; GFX9-G-O0-NEXT: ; kill: def $vgpr20 killed $vgpr20 def $vgpr20_vgpr21 killed $exec
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(1)
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v21, v4
+; GFX9-G-O0-NEXT: s_mov_b32 s5, 0xffffffc0
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, s5
+; GFX9-G-O0-NEXT: v_add_u32_e64 v4, v18, v4
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, s4
+; GFX9-G-O0-NEXT: v_sub_u32_e64 v5, v5, v18
+; GFX9-G-O0-NEXT: s_mov_b32 s6, 0
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s4
+; GFX9-G-O0-NEXT: v_cmp_lt_u32_e64 s[4:5], v18, v6
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s6
+; GFX9-G-O0-NEXT: v_cmp_eq_u32_e64 s[6:7], v18, v6
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-G-O0-NEXT: v_lshrrev_b64 v[6:7], v18, v[20:21]
+; GFX9-G-O0-NEXT: v_lshrrev_b64 v[25:26], v18, v[22:23]
+; GFX9-G-O0-NEXT: v_lshlrev_b64 v[23:24], v5, v[20:21]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v19, v25
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v26
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v22, v23
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v18, v24
+; GFX9-G-O0-NEXT: v_or_b32_e64 v19, v19, v22
+; GFX9-G-O0-NEXT: v_or_b32_e64 v18, v5, v18
+; GFX9-G-O0-NEXT: s_mov_b64 s[8:9], 0
+; GFX9-G-O0-NEXT: v_lshrrev_b64 v[20:21], v4, v[20:21]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v20
; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v21
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v5, v5, v22, s[6:7]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v6
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v20
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[6:7]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v5
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v7
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 0
-; GFX9-G-O0-NEXT: v_cmp_eq_u32_e64 s[4:5], v4, s4
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v20, v19
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v5, v5, v20, s[4:5]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 killed $vgpr6_vgpr7 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v18
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[4:5]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v5
-; GFX9-G-O0-NEXT: v_lshrrev_b64 v[4:5], v4, v[16:17]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v17, v5
-; GFX9-G-O0-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-G-O0-NEXT: s_mov_b32 s8, s5
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, s8
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v16, v16, v17, s[6:7]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v4
-; GFX9-G-O0-NEXT: s_mov_b32 s8, s4
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, s8
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v4, v4, v5, s[6:7]
+; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v4, v4, v19, s[4:5]
+; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v5, v5, v18, s[4:5]
+; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v4, v4, v17, s[6:7]
+; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v16, v5, v16, s[6:7]
; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v16
-; GFX9-G-O0-NEXT: s_mov_b64 s[10:11], -1
-; GFX9-G-O0-NEXT: s_mov_b32 s9, s10
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, v14
-; GFX9-G-O0-NEXT: v_add_co_u32_e64 v18, s[6:7], v16, s9
-; GFX9-G-O0-NEXT: s_mov_b32 s8, s11
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v15
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, s8
-; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v14, s[6:7], v14, v15, s[6:7]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr18 killed $vgpr18 def $vgpr18_vgpr19 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v19, v14
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v12
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, s9
-; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v16, s[6:7], v14, v15, s[6:7]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v13
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, s8
-; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v12, s[6:7], v12, v13, s[6:7]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v19, s9
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v18, s8
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v17, v6
+; GFX9-G-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 killed $vgpr6_vgpr7 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, v18
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v19
+; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v16, v16, v17, s[4:5]
+; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[4:5]
; GFX9-G-O0-NEXT: ; kill: def $vgpr16 killed $vgpr16 def $vgpr16_vgpr17 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v17, v12
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, s5
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, s4
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, s5
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, s4
-; GFX9-G-O0-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_writelane_b32 v31, s4, 8
-; GFX9-G-O0-NEXT: v_writelane_b32 v31, s5, 9
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v17, v6
+; GFX9-G-O0-NEXT: ; kill: def $vgpr4_vgpr5 killed $vgpr4_vgpr5 def $vgpr4_vgpr5_vgpr6_vgpr7 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v16
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v17
+; GFX9-G-O0-NEXT: s_mov_b32 s4, -1
+; GFX9-G-O0-NEXT: s_mov_b32 s10, -1
+; GFX9-G-O0-NEXT: s_mov_b32 s7, -1
+; GFX9-G-O0-NEXT: s_mov_b32 s6, -1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, s4
+; GFX9-G-O0-NEXT: v_add_co_u32_e64 v15, s[4:5], v15, v16
+; GFX9-G-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, s10
+; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v14, s[4:5], v14, v15, s[4:5]
+; GFX9-G-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, s7
+; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v13, s[4:5], v13, v14, s[4:5]
+; GFX9-G-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, s6
+; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v12, s[4:5], v12, v13, s[4:5]
+; GFX9-G-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_mov_b64 s[4:5], s[8:9]
+; GFX9-G-O0-NEXT: s_mov_b64 s[6:7], s[8:9]
+; GFX9-G-O0-NEXT: v_writelane_b32 v31, s8, 8
+; GFX9-G-O0-NEXT: v_writelane_b32 v31, s9, 9
; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-G-O0-NEXT: buffer_store_dword v31, off, s[0:3], s32 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
-; GFX9-G-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, s7
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, s6
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, s5
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, s4
+; GFX9-G-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_branch .LBB0_6
-; GFX9-G-O0-NEXT: .LBB0_8: ; %udiv-bb1
+; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_branch .LBB0_5
+; GFX9-G-O0-NEXT: .LBB0_7: ; %udiv-bb1
; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-G-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
-; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_mov_b64 s[6:7], 1
-; GFX9-G-O0-NEXT: s_mov_b32 s4, s6
+; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-G-O0-NEXT: s_mov_b32 s6, 1
+; GFX9-G-O0-NEXT: s_mov_b32 s10, 0
+; GFX9-G-O0-NEXT: s_mov_b32 s9, 0
+; GFX9-G-O0-NEXT: s_mov_b32 s8, 0
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, s6
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: v_add_co_u32_e64 v4, s[6:7], v2, v4
+; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s10
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(1)
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v3
-; GFX9-G-O0-NEXT: v_add_co_u32_e64 v0, s[4:5], v2, s4
-; GFX9-G-O0-NEXT: s_mov_b32 s6, s7
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, s6
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
-; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v3, s[4:5], v4, v1, s[4:5]
-; GFX9-G-O0-NEXT: s_mov_b64 s[6:7], 0
-; GFX9-G-O0-NEXT: s_mov_b32 s8, s6
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v12
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, s8
-; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v8, s[4:5], v1, v5, s[4:5]
-; GFX9-G-O0-NEXT: s_mov_b32 s9, s7
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v13
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, s9
-; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v1, s[4:5], v1, v5, s[4:5]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v1
-; GFX9-G-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v3
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v1
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v0
-; GFX9-G-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v9
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v8
-; GFX9-G-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_mov_b64 s[10:11], 0x7f
-; GFX9-G-O0-NEXT: s_mov_b32 s4, s10
-; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v2, s[4:5], s4, v2
-; GFX9-G-O0-NEXT: s_mov_b32 s10, s11
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, s10
-; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v4, s[4:5], v3, v4, s[4:5]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v4
-; GFX9-G-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 killed $vgpr2_vgpr3 killed $exec
-; GFX9-G-O0-NEXT: v_lshlrev_b64 v[4:5], v2, v[10:11]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v5
-; GFX9-G-O0-NEXT: s_mov_b32 s10, 64
-; GFX9-G-O0-NEXT: v_sub_u32_e64 v12, s10, v2
-; GFX9-G-O0-NEXT: v_lshrrev_b64 v[12:13], v12, v[6:7]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v13
-; GFX9-G-O0-NEXT: v_or_b32_e64 v3, v3, v14
-; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 killed $vgpr4_vgpr5 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v12
-; GFX9-G-O0-NEXT: v_or_b32_e64 v4, v4, v5
-; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v5, s[6:7], v5, v7, s[6:7]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s9
+; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v7, s[6:7], v6, v7, s[6:7]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s8
+; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v6, s[6:7], v1, v6, s[6:7]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v4
; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v5
-; GFX9-G-O0-NEXT: v_cmp_lt_u32_e64 s[4:5], v2, s10
-; GFX9-G-O0-NEXT: v_sub_u32_e64 v3, v2, s10
-; GFX9-G-O0-NEXT: v_lshlrev_b64 v[12:13], v3, v[6:7]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v13
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v3, v3, v14, s[4:5]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v4
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v12
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v4, v4, v5, s[4:5]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v3
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v5
-; GFX9-G-O0-NEXT: s_mov_b32 s10, 0
-; GFX9-G-O0-NEXT: v_cmp_eq_u32_e64 s[10:11], v2, s10
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v11
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v3, v3, v12, s[10:11]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 killed $vgpr4_vgpr5 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v10
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v4, v4, v5, s[10:11]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v3
-; GFX9-G-O0-NEXT: v_lshlrev_b64 v[6:7], v2, v[6:7]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v7
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s9
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v2, v2, v3, s[4:5]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 killed $vgpr6_vgpr7 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, s8
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v6, v3, v6, s[4:5]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v2
-; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, v7
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, v6
+; GFX9-G-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_mov_b32 s6, 0x7f
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v8, s[6:7], v1, v2
+; GFX9-G-O0-NEXT: s_mov_b32 s7, 64
+; GFX9-G-O0-NEXT: ; kill: def $vgpr12 killed $vgpr12 def $vgpr12_vgpr13 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v0
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v9
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v3
+; GFX9-G-O0-NEXT: s_mov_b32 s6, 0xffffffc0
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s6
+; GFX9-G-O0-NEXT: v_add_u32_e64 v2, v8, v0
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s7
+; GFX9-G-O0-NEXT: v_sub_u32_e64 v14, v0, v8
+; GFX9-G-O0-NEXT: s_mov_b32 s6, 0
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s7
+; GFX9-G-O0-NEXT: v_cmp_lt_u32_e64 s[8:9], v8, v0
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s6
+; GFX9-G-O0-NEXT: v_cmp_eq_u32_e64 s[6:7], v8, v0
+; GFX9-G-O0-NEXT: v_lshlrev_b64 v[0:1], v8, v[12:13]
+; GFX9-G-O0-NEXT: v_lshrrev_b64 v[17:18], v14, v[12:13]
+; GFX9-G-O0-NEXT: v_lshlrev_b64 v[15:16], v8, v[10:11]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v17
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v18
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v15
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v16
+; GFX9-G-O0-NEXT: v_or_b32_e64 v11, v11, v14
+; GFX9-G-O0-NEXT: v_or_b32_e64 v10, v8, v10
+; GFX9-G-O0-NEXT: v_lshlrev_b64 v[12:13], v2, v[12:13]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, s5
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, s4
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v0
; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v1
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v9
-; GFX9-G-O0-NEXT: v_or_b32_e64 v2, v2, v3
-; GFX9-G-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v8
-; GFX9-G-O0-NEXT: v_or_b32_e64 v0, v0, v1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v14
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v15
+; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v0, v0, v8, s[8:9]
+; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v2, v1, v2, s[8:9]
; GFX9-G-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v2
-; GFX9-G-O0-NEXT: v_cmp_ne_u64_e64 s[4:5], v[0:1], s[6:7]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s6
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, s7
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s6
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, s7
-; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v12
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v13
+; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v8, v8, v11, s[8:9]
+; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v2, v2, v10, s[8:9]
+; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v8, v8, v9, s[6:7]
+; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v2, v2, v3, s[6:7]
+; GFX9-G-O0-NEXT: ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v2
+; GFX9-G-O0-NEXT: ; kill: def $vgpr0_vgpr1 killed $vgpr0_vgpr1 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v8
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v9
+; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX9-G-O0-NEXT: s_mov_b64 s[10:11], s[4:5]
+; GFX9-G-O0-NEXT: v_or_b32_e64 v4, v4, v7
+; GFX9-G-O0-NEXT: v_or_b32_e64 v6, v5, v6
+; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v6
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s5
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s4
+; GFX9-G-O0-NEXT: v_cmp_ne_u64_e64 s[4:5], v[4:5], v[6:7]
+; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s8
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, s9
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, s11
+; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_mov_b64 s[6:7], exec
; GFX9-G-O0-NEXT: s_and_b64 s[4:5], s[6:7], s[4:5]
; GFX9-G-O0-NEXT: s_xor_b64 s[6:7], s[4:5], s[6:7]
@@ -2277,55 +2191,37 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: buffer_store_dword v31, off, s[0:3], s32 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
; GFX9-G-O0-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-G-O0-NEXT: s_cbranch_execz .LBB0_5
-; GFX9-G-O0-NEXT: s_branch .LBB0_7
-; GFX9-G-O0-NEXT: .LBB0_9: ; %udiv-end
-; GFX9-G-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: s_cbranch_execz .LBB0_4
+; GFX9-G-O0-NEXT: s_branch .LBB0_6
+; GFX9-G-O0-NEXT: .LBB0_8: ; %udiv-end
+; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(2)
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v8
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v9
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v2
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v5
-; GFX9-G-O0-NEXT: v_xor_b32_e64 v0, v0, v3
-; GFX9-G-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr1_vgpr2 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v4
-; GFX9-G-O0-NEXT: v_xor_b32_e64 v5, v2, v1
-; GFX9-G-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v0
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v10
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v8
-; GFX9-G-O0-NEXT: v_xor_b32_e64 v0, v0, v2
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v9
-; GFX9-G-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 killed $vgpr7_vgpr8 killed $exec
-; GFX9-G-O0-NEXT: v_xor_b32_e64 v7, v7, v4
-; GFX9-G-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v0
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v7
-; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v4, s[4:5], v0, v4
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v8
-; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v2, s[4:5], v0, v2, s[4:5]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v5
-; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v0, s[4:5], v0, v1, s[4:5]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v6
-; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v3, s[4:5], v1, v3, s[4:5]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v3
-; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v2
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 32
-; GFX9-G-O0-NEXT: v_lshrrev_b64 v[0:1], s4, v[0:1]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v0
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v4
-; GFX9-G-O0-NEXT: v_lshrrev_b64 v[4:5], s4, v[4:5]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v4
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v11
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v2
+; GFX9-G-O0-NEXT: v_xor_b32_e64 v0, v0, v7
+; GFX9-G-O0-NEXT: v_xor_b32_e64 v1, v1, v6
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v8
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v9
+; GFX9-G-O0-NEXT: v_xor_b32_e64 v2, v2, v5
+; GFX9-G-O0-NEXT: v_xor_b32_e64 v3, v3, v4
+; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v0, s[4:5], v0, v7
+; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v1, s[4:5], v1, v6, s[4:5]
+; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v2, s[4:5], v2, v5, s[4:5]
+; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v3, s[4:5], v3, v4, s[4:5]
; GFX9-G-O0-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GFX9-G-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-G-O0-NEXT: s_setpc_b64 s[30:31]
@@ -3363,175 +3259,179 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-LABEL: v_udiv_i128_vv:
; GFX9-G: ; %bb.0: ; %_udiv-special-cases
; GFX9-G-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-G-NEXT: v_or_b32_e32 v9, v5, v7
; GFX9-G-NEXT: v_or_b32_e32 v8, v4, v6
+; GFX9-G-NEXT: v_or_b32_e32 v9, v5, v7
; GFX9-G-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9]
-; GFX9-G-NEXT: v_or_b32_e32 v9, v1, v3
; GFX9-G-NEXT: v_or_b32_e32 v8, v0, v2
+; GFX9-G-NEXT: v_or_b32_e32 v9, v1, v3
; GFX9-G-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[8:9]
-; GFX9-G-NEXT: v_ffbh_u32_e32 v8, v6
-; GFX9-G-NEXT: v_add_u32_e32 v8, 32, v8
-; GFX9-G-NEXT: v_ffbh_u32_e32 v9, v7
-; GFX9-G-NEXT: v_min_u32_e32 v8, v8, v9
; GFX9-G-NEXT: v_ffbh_u32_e32 v9, v4
+; GFX9-G-NEXT: v_ffbh_u32_e32 v8, v5
; GFX9-G-NEXT: v_add_u32_e32 v9, 32, v9
-; GFX9-G-NEXT: v_ffbh_u32_e32 v10, v5
+; GFX9-G-NEXT: v_ffbh_u32_e32 v10, v6
+; GFX9-G-NEXT: s_or_b64 s[10:11], vcc, s[4:5]
+; GFX9-G-NEXT: v_min_u32_e32 v8, v8, v9
+; GFX9-G-NEXT: v_ffbh_u32_e32 v9, v7
+; GFX9-G-NEXT: v_add_u32_e32 v10, 32, v10
+; GFX9-G-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7]
+; GFX9-G-NEXT: v_add_u32_e32 v8, 64, v8
; GFX9-G-NEXT: v_min_u32_e32 v9, v9, v10
-; GFX9-G-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-G-NEXT: v_add_co_u32_e32 v9, vcc, 64, v9
-; GFX9-G-NEXT: v_addc_co_u32_e64 v10, s[6:7], 0, 0, vcc
-; GFX9-G-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
-; GFX9-G-NEXT: v_ffbh_u32_e32 v11, v3
+; GFX9-G-NEXT: v_ffbh_u32_e32 v10, v0
; GFX9-G-NEXT: v_cndmask_b32_e32 v8, v9, v8, vcc
-; GFX9-G-NEXT: v_ffbh_u32_e32 v9, v2
-; GFX9-G-NEXT: v_add_u32_e32 v9, 32, v9
-; GFX9-G-NEXT: v_min_u32_e32 v9, v9, v11
-; GFX9-G-NEXT: v_ffbh_u32_e32 v11, v0
+; GFX9-G-NEXT: v_ffbh_u32_e32 v9, v1
+; GFX9-G-NEXT: v_add_u32_e32 v10, 32, v10
+; GFX9-G-NEXT: v_ffbh_u32_e32 v11, v2
+; GFX9-G-NEXT: v_min_u32_e32 v9, v9, v10
+; GFX9-G-NEXT: v_ffbh_u32_e32 v10, v3
; GFX9-G-NEXT: v_add_u32_e32 v11, 32, v11
-; GFX9-G-NEXT: v_ffbh_u32_e32 v12, v1
-; GFX9-G-NEXT: v_min_u32_e32 v11, v11, v12
-; GFX9-G-NEXT: v_cndmask_b32_e64 v10, v10, 0, vcc
-; GFX9-G-NEXT: v_add_co_u32_e32 v11, vcc, 64, v11
-; GFX9-G-NEXT: v_addc_co_u32_e64 v12, s[6:7], 0, 0, vcc
-; GFX9-G-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
-; GFX9-G-NEXT: v_cndmask_b32_e32 v9, v11, v9, vcc
-; GFX9-G-NEXT: v_cndmask_b32_e64 v13, v12, 0, vcc
+; GFX9-G-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3]
+; GFX9-G-NEXT: v_add_u32_e32 v9, 64, v9
+; GFX9-G-NEXT: v_min_u32_e32 v10, v10, v11
+; GFX9-G-NEXT: v_cndmask_b32_e32 v9, v10, v9, vcc
; GFX9-G-NEXT: v_sub_co_u32_e32 v12, vcc, v8, v9
-; GFX9-G-NEXT: v_subb_co_u32_e32 v13, vcc, v10, v13, vcc
-; GFX9-G-NEXT: v_subb_co_u32_e64 v14, s[6:7], 0, 0, vcc
-; GFX9-G-NEXT: v_subb_co_u32_e64 v15, s[6:7], 0, 0, s[6:7]
-; GFX9-G-NEXT: s_mov_b64 s[6:7], 0x7f
-; GFX9-G-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[12:13]
-; GFX9-G-NEXT: v_or_b32_e32 v11, v13, v15
+; GFX9-G-NEXT: v_subb_co_u32_e64 v13, s[4:5], 0, 0, vcc
+; GFX9-G-NEXT: v_mov_b32_e32 v8, 0x7f
+; GFX9-G-NEXT: v_mov_b32_e32 v9, 0
+; GFX9-G-NEXT: v_subb_co_u32_e64 v14, s[4:5], 0, 0, s[4:5]
+; GFX9-G-NEXT: v_cmp_gt_u64_e32 vcc, v[12:13], v[8:9]
+; GFX9-G-NEXT: v_subb_co_u32_e64 v15, s[4:5], 0, 0, s[4:5]
; GFX9-G-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; GFX9-G-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[14:15]
+; GFX9-G-NEXT: v_cmp_lt_u64_e32 vcc, 0, v[14:15]
+; GFX9-G-NEXT: s_mov_b64 s[6:7], exec
; GFX9-G-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
; GFX9-G-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[14:15]
+; GFX9-G-NEXT: s_mov_b64 s[8:9], 0
; GFX9-G-NEXT: v_cndmask_b32_e32 v8, v9, v8, vcc
; GFX9-G-NEXT: v_and_b32_e32 v8, 1, v8
-; GFX9-G-NEXT: v_cmp_eq_u32_e32 vcc, 1, v8
+; GFX9-G-NEXT: v_cmp_ne_u16_e32 vcc, 0, v8
; GFX9-G-NEXT: v_xor_b32_e32 v8, 0x7f, v12
-; GFX9-G-NEXT: v_or_b32_e32 v10, v8, v14
-; GFX9-G-NEXT: s_or_b64 s[4:5], s[4:5], vcc
-; GFX9-G-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
-; GFX9-G-NEXT: s_xor_b64 s[6:7], s[4:5], -1
-; GFX9-G-NEXT: v_cndmask_b32_e64 v9, v3, 0, s[4:5]
-; GFX9-G-NEXT: v_cndmask_b32_e64 v8, v2, 0, s[4:5]
-; GFX9-G-NEXT: v_cndmask_b32_e64 v10, v1, 0, s[4:5]
-; GFX9-G-NEXT: v_cndmask_b32_e64 v11, v0, 0, s[4:5]
-; GFX9-G-NEXT: s_and_b64 s[4:5], s[6:7], vcc
+; GFX9-G-NEXT: v_or_b32_e32 v8, v8, v14
+; GFX9-G-NEXT: v_or_b32_e32 v9, v13, v15
+; GFX9-G-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[8:9]
+; GFX9-G-NEXT: s_or_b64 s[10:11], s[10:11], vcc
+; GFX9-G-NEXT: s_or_b64 s[4:5], s[10:11], s[4:5]
+; GFX9-G-NEXT: v_cndmask_b32_e64 v10, v0, 0, s[10:11]
+; GFX9-G-NEXT: v_cndmask_b32_e64 v11, v1, 0, s[10:11]
+; GFX9-G-NEXT: v_cndmask_b32_e64 v8, v2, 0, s[10:11]
+; GFX9-G-NEXT: v_cndmask_b32_e64 v9, v3, 0, s[10:11]
+; GFX9-G-NEXT: s_xor_b64 s[4:5], s[4:5], s[6:7]
; GFX9-G-NEXT: s_and_saveexec_b64 s[6:7], s[4:5]
; GFX9-G-NEXT: s_cbranch_execz .LBB1_6
; GFX9-G-NEXT: ; %bb.1: ; %udiv-bb1
; GFX9-G-NEXT: v_add_co_u32_e32 v18, vcc, 1, v12
; GFX9-G-NEXT: v_addc_co_u32_e32 v19, vcc, 0, v13, vcc
-; GFX9-G-NEXT: v_sub_u32_e32 v13, 0x7f, v12
-; GFX9-G-NEXT: v_sub_u32_e32 v10, 64, v13
-; GFX9-G-NEXT: v_lshlrev_b64 v[8:9], v13, v[2:3]
-; GFX9-G-NEXT: v_lshrrev_b64 v[10:11], v10, v[0:1]
; GFX9-G-NEXT: v_addc_co_u32_e32 v20, vcc, 0, v14, vcc
-; GFX9-G-NEXT: v_or_b32_e32 v10, v8, v10
-; GFX9-G-NEXT: v_sub_u32_e32 v8, 63, v12
+; GFX9-G-NEXT: s_mov_b64 s[4:5], exec
; GFX9-G-NEXT: v_addc_co_u32_e32 v21, vcc, 0, v15, vcc
+; GFX9-G-NEXT: s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX9-G-NEXT: v_sub_co_u32_e32 v16, vcc, 0x7f, v12
+; GFX9-G-NEXT: v_sub_u32_e32 v8, 64, v16
+; GFX9-G-NEXT: v_lshrrev_b64 v[8:9], v8, v[0:1]
+; GFX9-G-NEXT: v_lshlrev_b64 v[10:11], v16, v[2:3]
+; GFX9-G-NEXT: v_add_u32_e32 v14, 0xffffffc0, v16
+; GFX9-G-NEXT: v_lshlrev_b64 v[12:13], v16, v[0:1]
+; GFX9-G-NEXT: v_or_b32_e32 v10, v8, v10
; GFX9-G-NEXT: v_or_b32_e32 v11, v9, v11
-; GFX9-G-NEXT: v_lshlrev_b64 v[8:9], v8, v[0:1]
-; GFX9-G-NEXT: s_xor_b64 s[8:9], vcc, -1
-; GFX9-G-NEXT: v_cmp_gt_u32_e32 vcc, 64, v13
-; GFX9-G-NEXT: v_cndmask_b32_e32 v9, v9, v11, vcc
+; GFX9-G-NEXT: v_lshlrev_b64 v[8:9], v14, v[0:1]
+; GFX9-G-NEXT: v_cmp_gt_u32_e32 vcc, 64, v16
+; GFX9-G-NEXT: v_cndmask_b32_e32 v14, 0, v12, vcc
+; GFX9-G-NEXT: v_cndmask_b32_e32 v15, 0, v13, vcc
; GFX9-G-NEXT: v_cndmask_b32_e32 v8, v8, v10, vcc
-; GFX9-G-NEXT: v_lshlrev_b64 v[10:11], v13, v[0:1]
-; GFX9-G-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v13
-; GFX9-G-NEXT: v_cndmask_b32_e64 v9, v9, v3, s[4:5]
-; GFX9-G-NEXT: v_cndmask_b32_e64 v8, v8, v2, s[4:5]
-; GFX9-G-NEXT: v_cndmask_b32_e32 v11, 0, v11, vcc
-; GFX9-G-NEXT: v_mov_b32_e32 v12, 0
-; GFX9-G-NEXT: v_mov_b32_e32 v13, 0
-; GFX9-G-NEXT: v_cndmask_b32_e32 v10, 0, v10, vcc
-; GFX9-G-NEXT: s_and_saveexec_b64 s[4:5], s[8:9]
-; GFX9-G-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
+; GFX9-G-NEXT: v_cndmask_b32_e32 v9, v9, v11, vcc
+; GFX9-G-NEXT: v_cmp_eq_u32_e32 vcc, 0, v16
+; GFX9-G-NEXT: v_cndmask_b32_e32 v8, v8, v2, vcc
+; GFX9-G-NEXT: v_cndmask_b32_e32 v9, v9, v3, vcc
+; GFX9-G-NEXT: s_mov_b64 s[10:11], s[8:9]
+; GFX9-G-NEXT: v_mov_b32_e32 v11, s9
+; GFX9-G-NEXT: v_mov_b32_e32 v10, s8
+; GFX9-G-NEXT: v_mov_b32_e32 v13, s11
+; GFX9-G-NEXT: v_mov_b32_e32 v12, s10
+; GFX9-G-NEXT: s_and_saveexec_b64 s[8:9], s[4:5]
+; GFX9-G-NEXT: s_xor_b64 s[12:13], exec, s[8:9]
; GFX9-G-NEXT: s_cbranch_execz .LBB1_5
; GFX9-G-NEXT: ; %bb.2: ; %udiv-preheader
-; GFX9-G-NEXT: v_sub_u32_e32 v14, 64, v18
-; GFX9-G-NEXT: v_lshrrev_b64 v[12:13], v18, v[0:1]
-; GFX9-G-NEXT: v_lshlrev_b64 v[14:15], v14, v[2:3]
+; GFX9-G-NEXT: v_sub_u32_e32 v12, 64, v18
+; GFX9-G-NEXT: v_add_u32_e32 v22, 0xffffffc0, v18
+; GFX9-G-NEXT: v_lshrrev_b64 v[10:11], v18, v[0:1]
+; GFX9-G-NEXT: v_lshlrev_b64 v[12:13], v12, v[2:3]
+; GFX9-G-NEXT: v_lshrrev_b64 v[16:17], v18, v[2:3]
+; GFX9-G-NEXT: v_lshrrev_b64 v[2:3], v22, v[2:3]
+; GFX9-G-NEXT: v_or_b32_e32 v10, v10, v12
+; GFX9-G-NEXT: v_or_b32_e32 v11, v11, v13
; GFX9-G-NEXT: v_cmp_gt_u32_e32 vcc, 64, v18
-; GFX9-G-NEXT: v_or_b32_e32 v14, v12, v14
-; GFX9-G-NEXT: v_subrev_u32_e32 v12, 64, v18
-; GFX9-G-NEXT: v_or_b32_e32 v15, v13, v15
-; GFX9-G-NEXT: v_lshrrev_b64 v[12:13], v12, v[2:3]
-; GFX9-G-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v18
-; GFX9-G-NEXT: v_cndmask_b32_e32 v13, v13, v15, vcc
-; GFX9-G-NEXT: v_cndmask_b32_e64 v15, v13, v1, s[4:5]
-; GFX9-G-NEXT: v_lshrrev_b64 v[1:2], v18, v[2:3]
-; GFX9-G-NEXT: v_cndmask_b32_e32 v12, v12, v14, vcc
-; GFX9-G-NEXT: v_cndmask_b32_e32 v3, 0, v2, vcc
-; GFX9-G-NEXT: v_cndmask_b32_e32 v2, 0, v1, vcc
+; GFX9-G-NEXT: v_cndmask_b32_e32 v2, v2, v10, vcc
+; GFX9-G-NEXT: v_cndmask_b32_e32 v3, v3, v11, vcc
+; GFX9-G-NEXT: v_cndmask_b32_e32 v16, 0, v16, vcc
+; GFX9-G-NEXT: v_cndmask_b32_e32 v17, 0, v17, vcc
; GFX9-G-NEXT: v_add_co_u32_e32 v22, vcc, -1, v4
; GFX9-G-NEXT: v_addc_co_u32_e32 v23, vcc, -1, v5, vcc
+; GFX9-G-NEXT: s_mov_b64 s[8:9], 0
+; GFX9-G-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v18
; GFX9-G-NEXT: v_addc_co_u32_e32 v24, vcc, -1, v6, vcc
-; GFX9-G-NEXT: v_cndmask_b32_e64 v14, v12, v0, s[4:5]
+; GFX9-G-NEXT: v_cndmask_b32_e64 v2, v2, v0, s[4:5]
+; GFX9-G-NEXT: v_cndmask_b32_e64 v3, v3, v1, s[4:5]
; GFX9-G-NEXT: v_addc_co_u32_e32 v25, vcc, -1, v7, vcc
-; GFX9-G-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-G-NEXT: s_mov_b64 s[10:11], s[8:9]
; GFX9-G-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-G-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-G-NEXT: v_mov_b32_e32 v16, 0
-; GFX9-G-NEXT: v_mov_b32_e32 v17, 0
-; GFX9-G-NEXT: v_mov_b32_e32 v13, 0
+; GFX9-G-NEXT: v_mov_b32_e32 v11, s9
+; GFX9-G-NEXT: v_mov_b32_e32 v10, s8
+; GFX9-G-NEXT: v_mov_b32_e32 v0, 1
+; GFX9-G-NEXT: v_mov_b32_e32 v13, s11
+; GFX9-G-NEXT: v_mov_b32_e32 v12, s10
; GFX9-G-NEXT: .LBB1_3: ; %udiv-do-while
; GFX9-G-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-G-NEXT: v_lshrrev_b32_e32 v12, 31, v11
-; GFX9-G-NEXT: v_lshlrev_b64 v[10:11], 1, v[10:11]
+; GFX9-G-NEXT: v_lshlrev_b64 v[12:13], 1, v[14:15]
+; GFX9-G-NEXT: v_lshrrev_b32_e32 v0, 31, v15
+; GFX9-G-NEXT: v_or_b32_e32 v14, v10, v12
+; GFX9-G-NEXT: v_or_b32_e32 v15, v11, v13
+; GFX9-G-NEXT: v_lshlrev_b64 v[10:11], 1, v[16:17]
+; GFX9-G-NEXT: v_lshrrev_b32_e32 v12, 31, v3
; GFX9-G-NEXT: v_lshlrev_b64 v[2:3], 1, v[2:3]
-; GFX9-G-NEXT: v_or_b32_e32 v10, v16, v10
-; GFX9-G-NEXT: v_lshrrev_b32_e32 v16, 31, v15
-; GFX9-G-NEXT: v_lshlrev_b64 v[14:15], 1, v[14:15]
-; GFX9-G-NEXT: v_or_b32_e32 v2, v2, v16
-; GFX9-G-NEXT: v_lshrrev_b32_e32 v16, 31, v9
-; GFX9-G-NEXT: v_or_b32_e32 v14, v14, v16
-; GFX9-G-NEXT: v_sub_co_u32_e32 v16, vcc, v22, v14
-; GFX9-G-NEXT: v_subb_co_u32_e32 v16, vcc, v23, v15, vcc
-; GFX9-G-NEXT: v_subb_co_u32_e32 v16, vcc, v24, v2, vcc
-; GFX9-G-NEXT: v_subb_co_u32_e32 v16, vcc, v25, v3, vcc
-; GFX9-G-NEXT: v_ashrrev_i32_e32 v26, 31, v16
-; GFX9-G-NEXT: v_and_b32_e32 v16, v26, v4
-; GFX9-G-NEXT: v_or_b32_e32 v11, v17, v11
-; GFX9-G-NEXT: v_and_b32_e32 v17, v26, v5
-; GFX9-G-NEXT: v_sub_co_u32_e32 v14, vcc, v14, v16
-; GFX9-G-NEXT: v_subb_co_u32_e32 v15, vcc, v15, v17, vcc
-; GFX9-G-NEXT: v_and_b32_e32 v16, v26, v6
-; GFX9-G-NEXT: v_and_b32_e32 v17, v26, v7
-; GFX9-G-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v16, vcc
-; GFX9-G-NEXT: v_subb_co_u32_e32 v3, vcc, v3, v17, vcc
+; GFX9-G-NEXT: v_or_b32_e32 v10, v10, v12
+; GFX9-G-NEXT: v_lshrrev_b32_e32 v12, 31, v9
+; GFX9-G-NEXT: v_or_b32_e32 v2, v2, v12
+; GFX9-G-NEXT: v_sub_co_u32_e32 v12, vcc, v22, v2
+; GFX9-G-NEXT: v_subb_co_u32_e32 v12, vcc, v23, v3, vcc
+; GFX9-G-NEXT: v_subb_co_u32_e32 v12, vcc, v24, v10, vcc
+; GFX9-G-NEXT: v_subb_co_u32_e32 v12, vcc, v25, v11, vcc
+; GFX9-G-NEXT: v_ashrrev_i32_e32 v12, 31, v12
+; GFX9-G-NEXT: v_and_b32_e32 v13, v12, v4
+; GFX9-G-NEXT: v_and_b32_e32 v16, v12, v5
+; GFX9-G-NEXT: v_sub_co_u32_e32 v2, vcc, v2, v13
+; GFX9-G-NEXT: v_subb_co_u32_e32 v3, vcc, v3, v16, vcc
+; GFX9-G-NEXT: v_and_b32_e32 v13, v12, v6
+; GFX9-G-NEXT: v_and_b32_e32 v17, v12, v7
+; GFX9-G-NEXT: v_subb_co_u32_e32 v16, vcc, v10, v13, vcc
+; GFX9-G-NEXT: v_subb_co_u32_e32 v17, vcc, v11, v17, vcc
; GFX9-G-NEXT: v_add_co_u32_e32 v18, vcc, -1, v18
; GFX9-G-NEXT: v_addc_co_u32_e32 v19, vcc, -1, v19, vcc
; GFX9-G-NEXT: v_addc_co_u32_e32 v20, vcc, -1, v20, vcc
; GFX9-G-NEXT: v_addc_co_u32_e32 v21, vcc, -1, v21, vcc
+; GFX9-G-NEXT: v_or_b32_e32 v10, v18, v20
+; GFX9-G-NEXT: v_or_b32_e32 v11, v19, v21
; GFX9-G-NEXT: v_lshlrev_b64 v[8:9], 1, v[8:9]
-; GFX9-G-NEXT: v_or_b32_e32 v16, v18, v20
-; GFX9-G-NEXT: v_or_b32_e32 v17, v19, v21
-; GFX9-G-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[16:17]
-; GFX9-G-NEXT: v_or3_b32 v8, v8, v12, v0
-; GFX9-G-NEXT: v_and_b32_e32 v12, 1, v26
-; GFX9-G-NEXT: v_or3_b32 v9, v9, 0, v1
-; GFX9-G-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-G-NEXT: v_mov_b32_e32 v17, v13
-; GFX9-G-NEXT: v_mov_b32_e32 v16, v12
-; GFX9-G-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-G-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[10:11]
+; GFX9-G-NEXT: v_or_b32_e32 v8, v8, v0
+; GFX9-G-NEXT: v_and_b32_e32 v0, 1, v12
+; GFX9-G-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX9-G-NEXT: v_mov_b32_e32 v11, v1
+; GFX9-G-NEXT: v_mov_b32_e32 v10, v0
+; GFX9-G-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX9-G-NEXT: s_cbranch_execnz .LBB1_3
; GFX9-G-NEXT: ; %bb.4: ; %Flow
-; GFX9-G-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-G-NEXT: .LBB1_5: ; %Flow2
; GFX9-G-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX9-G-NEXT: .LBB1_5: ; %Flow2
+; GFX9-G-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX9-G-NEXT: v_lshlrev_b64 v[0:1], 1, v[14:15]
; GFX9-G-NEXT: v_lshlrev_b64 v[8:9], 1, v[8:9]
-; GFX9-G-NEXT: v_lshlrev_b64 v[0:1], 1, v[10:11]
-; GFX9-G-NEXT: v_lshrrev_b32_e32 v2, 31, v11
+; GFX9-G-NEXT: v_lshrrev_b32_e32 v2, 31, v15
; GFX9-G-NEXT: v_or_b32_e32 v8, v8, v2
-; GFX9-G-NEXT: v_or_b32_e32 v10, v13, v1
-; GFX9-G-NEXT: v_or_b32_e32 v11, v12, v0
+; GFX9-G-NEXT: v_or_b32_e32 v10, v10, v0
+; GFX9-G-NEXT: v_or_b32_e32 v11, v11, v1
; GFX9-G-NEXT: .LBB1_6: ; %Flow3
; GFX9-G-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX9-G-NEXT: v_mov_b32_e32 v0, v11
-; GFX9-G-NEXT: v_mov_b32_e32 v1, v10
+; GFX9-G-NEXT: v_mov_b32_e32 v0, v10
+; GFX9-G-NEXT: v_mov_b32_e32 v1, v11
; GFX9-G-NEXT: v_mov_b32_e32 v2, v8
; GFX9-G-NEXT: v_mov_b32_e32 v3, v9
; GFX9-G-NEXT: s_setpc_b64 s[30:31]
@@ -3540,848 +3440,818 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0: ; %bb.0: ; %_udiv-special-cases
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-G-O0-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GFX9-G-O0-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v6
-; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v2
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v0
-; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec
; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v1
-; GFX9-G-O0-NEXT: ; kill: def $vgpr12 killed $vgpr12 def $vgpr12_vgpr13 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v3
-; GFX9-G-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v5
-; GFX9-G-O0-NEXT: ; kill: def $vgpr14 killed $vgpr14 def $vgpr14_vgpr15 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, v7
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v14
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v15
-; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v2
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v3
+; GFX9-G-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v9
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v8
+; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_nop 0
+; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(2)
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v12
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v13
-; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v9
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v10
-; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v4
+; GFX9-G-O0-NEXT: ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9_vgpr10_vgpr11 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v5
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v6
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v7
+; GFX9-G-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v15
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v1
-; GFX9-G-O0-NEXT: v_or_b32_e64 v2, v8, v6
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v14
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v0
-; GFX9-G-O0-NEXT: v_or_b32_e64 v0, v5, v4
-; GFX9-G-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v2
+; GFX9-G-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_mov_b64 s[8:9], 0
-; GFX9-G-O0-NEXT: ; implicit-def: $vgpr31 : SGPR spill to VGPR lane
-; GFX9-G-O0-NEXT: v_writelane_b32 v31, s8, 0
-; GFX9-G-O0-NEXT: v_writelane_b32 v31, s9, 1
-; GFX9-G-O0-NEXT: v_cmp_eq_u64_e64 s[4:5], v[0:1], s[8:9]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v13
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v10
-; GFX9-G-O0-NEXT: v_or_b32_e64 v7, v3, v1
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v12
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v9
-; GFX9-G-O0-NEXT: v_or_b32_e64 v9, v2, v0
-; GFX9-G-O0-NEXT: ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v7
-; GFX9-G-O0-NEXT: v_cmp_eq_u64_e64 s[6:7], v[9:10], s[8:9]
+; GFX9-G-O0-NEXT: ; implicit-def: $vgpr32 : SGPR spill to VGPR lane
+; GFX9-G-O0-NEXT: v_writelane_b32 v32, s8, 0
+; GFX9-G-O0-NEXT: v_writelane_b32 v32, s9, 1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v8
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v9
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v11
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v5
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v6
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v12
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v13
+; GFX9-G-O0-NEXT: v_or_b32_e64 v4, v4, v7
+; GFX9-G-O0-NEXT: v_or_b32_e64 v6, v5, v6
+; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v6
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s8
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s9
+; GFX9-G-O0-NEXT: v_cmp_eq_u64_e64 s[4:5], v[4:5], v[6:7]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v0
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v3
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v2
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v5
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v6
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v12
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v13
+; GFX9-G-O0-NEXT: v_or_b32_e64 v4, v4, v7
+; GFX9-G-O0-NEXT: v_or_b32_e64 v6, v5, v6
+; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v6
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s8
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s9
+; GFX9-G-O0-NEXT: v_cmp_eq_u64_e64 s[6:7], v[4:5], v[6:7]
; GFX9-G-O0-NEXT: s_or_b64 s[4:5], s[4:5], s[6:7]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v8
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v9
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v11
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, s8
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, s9
+; GFX9-G-O0-NEXT: v_cmp_eq_u64_e64 s[6:7], v[9:10], v[4:5]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v6
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v7
+; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v4, v4
+; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v5, v5
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, 32
+; GFX9-G-O0-NEXT: v_add_u32_e64 v5, v5, v8
+; GFX9-G-O0-NEXT: v_min_u32_e64 v4, v4, v5
+; GFX9-G-O0-NEXT: s_mov_b32 s10, 64
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, s10
+; GFX9-G-O0-NEXT: v_add_u32_e64 v5, v4, v5
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v9
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v10
; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v4, v4
-; GFX9-G-O0-NEXT: s_mov_b32 s6, 32
-; GFX9-G-O0-NEXT: v_add_u32_e64 v4, v4, s6
; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v6, v6
-; GFX9-G-O0-NEXT: v_min_u32_e64 v6, v4, v6
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-G-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v4
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v7
+; GFX9-G-O0-NEXT: v_add_u32_e64 v6, v6, v8
+; GFX9-G-O0-NEXT: v_min_u32_e64 v4, v4, v6
+; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v4, v4, v5, s[6:7]
+; GFX9-G-O0-NEXT: s_mov_b32 s14, 0
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v0
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v3
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v2
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, s8
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s9
+; GFX9-G-O0-NEXT: v_cmp_eq_u64_e64 s[6:7], v[9:10], v[5:6]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v11
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v12
; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v5, v5
-; GFX9-G-O0-NEXT: v_add_u32_e64 v5, v5, s6
-; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v8, v8
-; GFX9-G-O0-NEXT: v_min_u32_e64 v16, v5, v8
-; GFX9-G-O0-NEXT: ; kill: def $vgpr16 killed $vgpr16 def $vgpr16_vgpr17 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v17, v4
-; GFX9-G-O0-NEXT: s_mov_b64 s[10:11], 64
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v16
-; GFX9-G-O0-NEXT: s_mov_b32 s12, s10
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v17
-; GFX9-G-O0-NEXT: s_mov_b32 s7, s11
-; GFX9-G-O0-NEXT: v_add_co_u32_e64 v8, s[12:13], v8, s12
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, s7
-; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v5, s[12:13], v5, v9, s[12:13]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v5
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v9
-; GFX9-G-O0-NEXT: s_mov_b64 s[12:13], s[8:9]
-; GFX9-G-O0-NEXT: v_cmp_ne_u64_e64 s[12:13], v[14:15], s[12:13]
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v5, v5, v10, s[12:13]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v6
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v8
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v7, v6, v7, s[12:13]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v5
-; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v5, v0
-; GFX9-G-O0-NEXT: v_add_u32_e64 v5, v5, s6
-; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v6, v1
+; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v6, v6
+; GFX9-G-O0-NEXT: v_add_u32_e64 v6, v6, v8
; GFX9-G-O0-NEXT: v_min_u32_e64 v5, v5, v6
-; GFX9-G-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v4
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v6
-; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v9, v2
-; GFX9-G-O0-NEXT: v_add_u32_e64 v9, v9, s6
-; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v10, v3
-; GFX9-G-O0-NEXT: v_min_u32_e64 v14, v9, v10
-; GFX9-G-O0-NEXT: ; kill: def $vgpr14 killed $vgpr14 def $vgpr14_vgpr15 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, v4
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v14
-; GFX9-G-O0-NEXT: s_mov_b32 s6, s10
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v15
-; GFX9-G-O0-NEXT: s_mov_b32 s10, s11
-; GFX9-G-O0-NEXT: v_add_co_u32_e64 v9, s[6:7], v9, s6
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, s10
-; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v4, s[6:7], v4, v10, s[6:7]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v4
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v10
-; GFX9-G-O0-NEXT: s_mov_b64 s[6:7], s[8:9]
-; GFX9-G-O0-NEXT: v_cmp_ne_u64_e64 s[6:7], v[12:13], s[6:7]
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v4, v4, v11, s[6:7]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v5
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v9
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s10
+; GFX9-G-O0-NEXT: v_add_u32_e64 v6, v5, v6
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v9
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v10
+; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v5, v5
+; GFX9-G-O0-NEXT: v_ffbh_u32_e64 v7, v7
+; GFX9-G-O0-NEXT: v_add_u32_e64 v7, v7, v8
+; GFX9-G-O0-NEXT: v_min_u32_e64 v5, v5, v7
; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v5, v5, v6, s[6:7]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v4
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v5
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v7
-; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v4, s[6:7], v4, v9
-; GFX9-G-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 killed $vgpr5_vgpr6 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v8
-; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v6, s[10:11], v5, v6, s[6:7]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v6
-; GFX9-G-O0-NEXT: s_mov_b32 s7, s8
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s7
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s7
-; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v7, s[10:11], v6, v7, s[10:11]
-; GFX9-G-O0-NEXT: s_mov_b32 s14, s9
+; GFX9-G-O0-NEXT: s_mov_b32 s13, 0
+; GFX9-G-O0-NEXT: s_mov_b32 s11, 0
+; GFX9-G-O0-NEXT: s_mov_b32 s12, 0
+; GFX9-G-O0-NEXT: s_mov_b32 s10, 0
+; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v4, s[6:7], v4, v5
+; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, s14
; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s14
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, s14
-; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v6, s[10:11], v6, v8, s[10:11]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v6
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v5
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v4
-; GFX9-G-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v8
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v7
-; GFX9-G-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_cmp_eq_u64_e64 s[10:11], v[7:8], s[8:9]
-; GFX9-G-O0-NEXT: s_mov_b64 s[12:13], 0x7f
-; GFX9-G-O0-NEXT: v_cmp_gt_u64_e64 s[16:17], v[4:5], s[12:13]
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v9, 0, 1, s[16:17]
-; GFX9-G-O0-NEXT: v_cmp_ne_u64_e64 s[16:17], v[7:8], s[8:9]
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v6, 0, 1, s[16:17]
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v6, v6, v9, s[10:11]
-; GFX9-G-O0-NEXT: v_and_b32_e64 v6, 1, v6
-; GFX9-G-O0-NEXT: v_cmp_eq_u32_e64 s[10:11], v6, 1
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v6, 0, 1, s[10:11]
-; GFX9-G-O0-NEXT: s_mov_b32 s6, 1
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, s6
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v6, v6, v9, s[4:5]
-; GFX9-G-O0-NEXT: v_and_b32_e64 v6, 1, v6
-; GFX9-G-O0-NEXT: v_cmp_eq_u32_e64 s[4:5], v6, 1
-; GFX9-G-O0-NEXT: s_mov_b64 s[10:11], s[4:5]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v5
-; GFX9-G-O0-NEXT: s_mov_b32 s15, s13
-; GFX9-G-O0-NEXT: v_xor_b32_e64 v6, v6, s15
-; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 killed $vgpr4_vgpr5 killed $exec
-; GFX9-G-O0-NEXT: ; kill: def $sgpr12 killed $sgpr12 killed $sgpr12_sgpr13
-; GFX9-G-O0-NEXT: v_xor_b32_e64 v4, v4, s12
-; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v6
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v5
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v8
-; GFX9-G-O0-NEXT: v_or_b32_e64 v6, v6, v9
-; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 killed $vgpr4_vgpr5 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v7
-; GFX9-G-O0-NEXT: v_or_b32_e64 v4, v4, v5
+; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v5, s[6:7], v5, v6, s[6:7]
+; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s13
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s12
+; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v7, s[6:7], v6, v7, s[6:7]
+; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s11
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, s10
+; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v6, s[6:7], v6, v8, s[6:7]
+; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_mov_b64 s[6:7], 0x7f
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v4
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v5
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v7
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v6
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, s7
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, s6
+; GFX9-G-O0-NEXT: v_cmp_gt_u64_e64 s[14:15], v[10:11], v[12:13]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, s9
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, s8
+; GFX9-G-O0-NEXT: v_cmp_gt_u64_e64 s[12:13], v[8:9], v[10:11]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, s9
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, s8
+; GFX9-G-O0-NEXT: v_cmp_eq_u64_e64 s[10:11], v[8:9], v[10:11]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, 1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, 0
+; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v11, v9, v10, s[14:15]
+; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v8, v9, v10, s[12:13]
+; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v8, v8, v11, s[10:11]
+; GFX9-G-O0-NEXT: v_and_b32_e64 v8, v8, v10
+; GFX9-G-O0-NEXT: v_cmp_ne_u16_e64 s[10:11], v8, v9
+; GFX9-G-O0-NEXT: s_or_b64 s[4:5], s[4:5], s[10:11]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, s7
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, s6
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v11
+; GFX9-G-O0-NEXT: v_xor_b32_e64 v4, v4, v9
+; GFX9-G-O0-NEXT: v_xor_b32_e64 v5, v5, v8
+; GFX9-G-O0-NEXT: v_or_b32_e64 v4, v4, v7
+; GFX9-G-O0-NEXT: v_or_b32_e64 v6, v5, v6
; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v6
-; GFX9-G-O0-NEXT: v_cmp_eq_u64_e64 s[8:9], v[4:5], s[8:9]
-; GFX9-G-O0-NEXT: s_mov_b64 s[12:13], s[10:11]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, s14
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v4, v1, v4, s[12:13]
-; GFX9-G-O0-NEXT: s_mov_b64 s[12:13], s[10:11]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, s7
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v0, v0, v1, s[12:13]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s8
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s9
+; GFX9-G-O0-NEXT: v_cmp_eq_u64_e64 s[6:7], v[4:5], v[6:7]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v0
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v3
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v2
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s8
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, s9
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v0
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v4
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v5
+; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v0, v0, v3, s[4:5]
+; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v2, v1, v2, s[4:5]
; GFX9-G-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v4
-; GFX9-G-O0-NEXT: s_mov_b64 s[12:13], s[10:11]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, s14
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v4, v3, v4, s[12:13]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, s7
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v2, v2, v3, s[10:11]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v4
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v4, 0, 1, s[8:9]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, s6
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v2
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s8
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, s9
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v2
+; GFX9-G-O0-NEXT: ; kill: def $vgpr3 killed $vgpr3 killed $vgpr2_vgpr3 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v6
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v7
; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v4, v4, v5, s[4:5]
-; GFX9-G-O0-NEXT: v_and_b32_e64 v4, 1, v4
-; GFX9-G-O0-NEXT: v_cmp_eq_u32_e64 s[4:5], v4, 1
-; GFX9-G-O0-NEXT: s_mov_b64 s[6:7], -1
+; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v2, v2, v3, s[4:5]
+; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v2
+; GFX9-G-O0-NEXT: ; kill: def $vgpr0_vgpr1 killed $vgpr0_vgpr1 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v4
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v5
+; GFX9-G-O0-NEXT: s_or_b64 s[4:5], s[4:5], s[6:7]
+; GFX9-G-O0-NEXT: s_mov_b64 s[6:7], exec
; GFX9-G-O0-NEXT: s_xor_b64 s[6:7], s[4:5], s[6:7]
-; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_nop 0
; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_mov_b64 s[4:5], exec
-; GFX9-G-O0-NEXT: v_writelane_b32 v31, s4, 2
-; GFX9-G-O0-NEXT: v_writelane_b32 v31, s5, 3
-; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
-; GFX9-G-O0-NEXT: buffer_store_dword v31, off, s[0:3], s32 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
+; GFX9-G-O0-NEXT: v_writelane_b32 v32, s4, 2
+; GFX9-G-O0-NEXT: v_writelane_b32 v32, s5, 3
+; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
+; GFX9-G-O0-NEXT: buffer_store_dword v32, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-G-O0-NEXT: s_and_b64 s[4:5], s[4:5], s[6:7]
; GFX9-G-O0-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-G-O0-NEXT: s_cbranch_execz .LBB1_3
-; GFX9-G-O0-NEXT: s_branch .LBB1_8
+; GFX9-G-O0-NEXT: s_cbranch_execz .LBB1_2
+; GFX9-G-O0-NEXT: s_branch .LBB1_7
; GFX9-G-O0-NEXT: .LBB1_1: ; %Flow
-; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
-; GFX9-G-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
+; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
+; GFX9-G-O0-NEXT: buffer_load_dword v32, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
-; GFX9-G-O0-NEXT: v_readlane_b32 s4, v31, 4
-; GFX9-G-O0-NEXT: v_readlane_b32 s5, v31, 5
+; GFX9-G-O0-NEXT: v_readlane_b32 s4, v32, 4
+; GFX9-G-O0-NEXT: v_readlane_b32 s5, v32, 5
; GFX9-G-O0-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-G-O0-NEXT: ; %bb.2: ; %Flow
-; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
-; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
-; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
-; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
-; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
-; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
-; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
-; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
-; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_branch .LBB1_5
-; GFX9-G-O0-NEXT: .LBB1_3: ; %Flow2
-; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
-; GFX9-G-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
+; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_nop 0
+; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_branch .LBB1_4
+; GFX9-G-O0-NEXT: .LBB1_2: ; %Flow2
+; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
+; GFX9-G-O0-NEXT: buffer_load_dword v32, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
-; GFX9-G-O0-NEXT: v_readlane_b32 s4, v31, 2
-; GFX9-G-O0-NEXT: v_readlane_b32 s5, v31, 3
+; GFX9-G-O0-NEXT: v_readlane_b32 s4, v32, 2
+; GFX9-G-O0-NEXT: v_readlane_b32 s5, v32, 3
; GFX9-G-O0-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(1)
-; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(1)
-; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_branch .LBB1_9
-; GFX9-G-O0-NEXT: .LBB1_4: ; %udiv-loop-exit
-; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 1
+; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_branch .LBB1_8
+; GFX9-G-O0-NEXT: .LBB1_3: ; %udiv-loop-exit
+; GFX9-G-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(2)
-; GFX9-G-O0-NEXT: v_lshlrev_b64 v[2:3], s4, v[0:1]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v4
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v5
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
-; GFX9-G-O0-NEXT: v_lshlrev_b64 v[9:10], s4, v[9:10]
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 63
-; GFX9-G-O0-NEXT: v_lshrrev_b64 v[0:1], s4, v[0:1]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v1
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v10
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v8
-; GFX9-G-O0-NEXT: v_or3_b32 v4, v4, v11, v12
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v0
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v9
-; GFX9-G-O0-NEXT: v_or3_b32 v0, v0, v1, v7
-; GFX9-G-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v4
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v3
; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v6
-; GFX9-G-O0-NEXT: v_or_b32_e64 v4, v4, v7
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v2
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v5
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v7
+; GFX9-G-O0-NEXT: s_mov_b32 s4, 1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-G-O0-NEXT: v_lshlrev_b64 v[9:10], v0, v[2:3]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-G-O0-NEXT: v_lshlrev_b64 v[0:1], v0, v[4:5]
+; GFX9-G-O0-NEXT: ; kill: def $vgpr3 killed $vgpr3 killed $vgpr2_vgpr3 killed $exec
+; GFX9-G-O0-NEXT: s_mov_b32 s4, 31
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s4
+; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v5, v2, v3
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v0
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v11
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v12
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v13
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v14
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v2
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v9
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v10
+; GFX9-G-O0-NEXT: v_or_b32_e64 v0, v0, v6
+; GFX9-G-O0-NEXT: v_or_b32_e64 v2, v1, v2
+; GFX9-G-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v2
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v7
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v8
+; GFX9-G-O0-NEXT: v_or3_b32 v4, v4, v5, v6
; GFX9-G-O0-NEXT: v_or_b32_e64 v2, v2, v3
-; GFX9-G-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v4
-; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v2
+; GFX9-G-O0-NEXT: ; kill: def $vgpr0_vgpr1 killed $vgpr0_vgpr1 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v4
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v5
; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_nop 0
; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_branch .LBB1_3
-; GFX9-G-O0-NEXT: .LBB1_5: ; %Flow1
-; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
-; GFX9-G-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
+; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_branch .LBB1_2
+; GFX9-G-O0-NEXT: .LBB1_4: ; %Flow1
+; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
+; GFX9-G-O0-NEXT: buffer_load_dword v32, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
-; GFX9-G-O0-NEXT: v_readlane_b32 s4, v31, 6
-; GFX9-G-O0-NEXT: v_readlane_b32 s5, v31, 7
+; GFX9-G-O0-NEXT: v_readlane_b32 s4, v32, 6
+; GFX9-G-O0-NEXT: v_readlane_b32 s5, v32, 7
; GFX9-G-O0-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(1)
-; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(1)
-; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_branch .LBB1_4
-; GFX9-G-O0-NEXT: .LBB1_6: ; %udiv-do-while
+; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_branch .LBB1_3
+; GFX9-G-O0-NEXT: .LBB1_5: ; %udiv-do-while
; GFX9-G-O0-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
-; GFX9-G-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
+; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
+; GFX9-G-O0-NEXT: buffer_load_dword v32, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
-; GFX9-G-O0-NEXT: v_readlane_b32 s6, v31, 8
-; GFX9-G-O0-NEXT: v_readlane_b32 s7, v31, 9
-; GFX9-G-O0-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 63
+; GFX9-G-O0-NEXT: v_readlane_b32 s6, v32, 8
+; GFX9-G-O0-NEXT: v_readlane_b32 s7, v32, 9
+; GFX9-G-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: s_mov_b64 s[8:9], 1
+; GFX9-G-O0-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(18)
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v2
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v3
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(16)
-; GFX9-G-O0-NEXT: v_lshrrev_b64 v[7:8], s4, v[2:3]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v8
-; GFX9-G-O0-NEXT: s_mov_b32 s5, 1
-; GFX9-G-O0-NEXT: v_lshlrev_b64 v[14:15], s5, v[14:15]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v15
-; GFX9-G-O0-NEXT: v_or_b32_e64 v6, v6, v11
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v7
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v14
-; GFX9-G-O0-NEXT: v_or_b32_e64 v14, v7, v8
-; GFX9-G-O0-NEXT: ; kill: def $vgpr14 killed $vgpr14 def $vgpr14_vgpr15 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, v6
-; GFX9-G-O0-NEXT: v_lshlrev_b64 v[29:30], s5, v[2:3]
-; GFX9-G-O0-NEXT: v_lshrrev_b64 v[6:7], s4, v[27:28]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v30
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v7
-; GFX9-G-O0-NEXT: v_or_b32_e64 v2, v2, v3
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v29
-; GFX9-G-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 killed $vgpr6_vgpr7 killed $exec
-; GFX9-G-O0-NEXT: v_or_b32_e64 v7, v3, v6
-; GFX9-G-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v2
-; GFX9-G-O0-NEXT: v_lshlrev_b64 v[2:3], s5, v[0:1]
-; GFX9-G-O0-NEXT: v_lshlrev_b64 v[27:28], s5, v[27:28]
-; GFX9-G-O0-NEXT: v_lshrrev_b64 v[0:1], s4, v[0:1]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v1
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v28
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v21, v5
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v20, v4
+; GFX9-G-O0-NEXT: s_mov_b32 s10, 1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s10
+; GFX9-G-O0-NEXT: v_lshlrev_b64 v[2:3], v2, v[0:1]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, s10
+; GFX9-G-O0-NEXT: v_lshlrev_b64 v[4:5], v4, v[20:21]
+; GFX9-G-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr0_vgpr1 killed $exec
+; GFX9-G-O0-NEXT: s_mov_b32 s11, 31
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s11
+; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v1, v0, v1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v4
+; GFX9-G-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 killed $vgpr4_vgpr5 killed $exec
+; GFX9-G-O0-NEXT: v_or_b32_e64 v7, v0, v1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v14
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v15
+; GFX9-G-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr0_vgpr1 killed $exec
+; GFX9-G-O0-NEXT: s_mov_b32 s11, 31
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s11
+; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v1, v0, v1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v2
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v3
+; GFX9-G-O0-NEXT: v_or_b32_e64 v4, v0, v1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v12
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v13
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v14
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v15
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s10
+; GFX9-G-O0-NEXT: v_lshlrev_b64 v[22:23], v0, v[2:3]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s10
+; GFX9-G-O0-NEXT: v_lshlrev_b64 v[0:1], v0, v[12:13]
+; GFX9-G-O0-NEXT: ; kill: def $vgpr3 killed $vgpr3 killed $vgpr2_vgpr3 killed $exec
+; GFX9-G-O0-NEXT: s_mov_b32 s10, 31
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s10
+; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v13, v2, v3
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v0
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v1
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(10)
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, v26
-; GFX9-G-O0-NEXT: v_or3_b32 v6, v6, v11, v16
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v0
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v27
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v25
-; GFX9-G-O0-NEXT: v_or3_b32 v0, v0, v1, v11
-; GFX9-G-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v6
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v3
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v28
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v29
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(8)
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v24
-; GFX9-G-O0-NEXT: v_or_b32_e64 v6, v6, v11
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v2
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v20, v30
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v21, v31
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v2
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v22
; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v23
-; GFX9-G-O0-NEXT: v_or_b32_e64 v2, v2, v3
-; GFX9-G-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v6
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v7
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(1)
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v4
-; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v11, s[8:9], v11, v6
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v8
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v5
-; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v4, s[8:9], v4, v7, s[8:9]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v14
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v12
-; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v11, s[8:9], v5, v4, s[8:9]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v15
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v13
-; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v8, s[8:9], v8, v5, s[8:9]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 def $vgpr11_vgpr12 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v8
-; GFX9-G-O0-NEXT: v_ashrrev_i64 v[13:14], s4, v[11:12]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v14
-; GFX9-G-O0-NEXT: s_mov_b64 s[4:5], 1
-; GFX9-G-O0-NEXT: s_mov_b32 s8, s5
-; GFX9-G-O0-NEXT: v_and_b32_e64 v12, v8, s8
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v13
-; GFX9-G-O0-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
-; GFX9-G-O0-NEXT: v_and_b32_e64 v14, v11, s4
-; GFX9-G-O0-NEXT: ; kill: def $vgpr14 killed $vgpr14 def $vgpr14_vgpr15 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, v12
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, 0
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, 0
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, v20
-; GFX9-G-O0-NEXT: v_and_b32_e64 v16, v8, v16
-; GFX9-G-O0-NEXT: v_and_b32_e64 v19, v11, v19
-; GFX9-G-O0-NEXT: ; kill: def $vgpr19 killed $vgpr19 def $vgpr19_vgpr20 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v20, v16
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, v22
-; GFX9-G-O0-NEXT: v_and_b32_e64 v8, v8, v16
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, v21
-; GFX9-G-O0-NEXT: v_and_b32_e64 v21, v11, v16
-; GFX9-G-O0-NEXT: ; kill: def $vgpr21 killed $vgpr21 def $vgpr21_vgpr22 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v22, v8
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v21
-; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v6, s[4:5], v6, v8
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v22
-; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v8, s[4:5], v7, v8, s[4:5]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v19
-; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v4, s[4:5], v4, v7, s[4:5]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v20
-; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v7, s[4:5], v5, v7, s[4:5]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v7
-; GFX9-G-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec
+; GFX9-G-O0-NEXT: v_or_b32_e64 v0, v0, v14
+; GFX9-G-O0-NEXT: v_or_b32_e64 v2, v1, v2
+; GFX9-G-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v2
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v20
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v21
+; GFX9-G-O0-NEXT: v_or3_b32 v12, v12, v13, v14
+; GFX9-G-O0-NEXT: v_or_b32_e64 v2, v2, v3
+; GFX9-G-O0-NEXT: ; kill: def $vgpr12 killed $vgpr12 def $vgpr12_vgpr13 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v2
+; GFX9-G-O0-NEXT: ; kill: def $vgpr0_vgpr1 killed $vgpr0_vgpr1 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v12
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v13
+; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_nop 0
+; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(4)
+; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v11, s[10:11], v11, v4
+; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v10, s[10:11], v10, v9, s[10:11]
+; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v8, s[10:11], v8, v7, s[10:11]
+; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v10, s[10:11], v6, v5, s[10:11]
+; GFX9-G-O0-NEXT: s_mov_b32 s10, 31
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s10
+; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v8, v6, v10
+; GFX9-G-O0-NEXT: s_mov_b32 s10, 31
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s10
+; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v6, v6, v10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, s9
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, s8
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v12
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v13
+; GFX9-G-O0-NEXT: v_and_b32_e64 v12, v8, v11
+; GFX9-G-O0-NEXT: ; kill: def $vgpr12 killed $vgpr12 def $vgpr12_vgpr13 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, s5
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, s4
+; GFX9-G-O0-NEXT: ; kill: def $vgpr12_vgpr13 killed $vgpr12_vgpr13 def $vgpr12_vgpr13_vgpr14_vgpr15 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, v11
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v10
+; GFX9-G-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_nop 0
+; GFX9-G-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v22, v24
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v23, v25
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v20, v26
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v21, v27
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v22
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v23
+; GFX9-G-O0-NEXT: v_and_b32_e64 v11, v8, v11
+; GFX9-G-O0-NEXT: v_and_b32_e64 v10, v8, v10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v20
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v20, v21
+; GFX9-G-O0-NEXT: v_and_b32_e64 v8, v6, v8
+; GFX9-G-O0-NEXT: v_and_b32_e64 v6, v6, v20
+; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v4, s[8:9], v4, v11
+; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v10, s[8:9], v9, v10, s[8:9]
+; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v9, s[8:9], v7, v8, s[8:9]
+; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v8, s[8:9], v5, v6, s[8:9]
+; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5_vgpr6_vgpr7 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v9
; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v8
-; GFX9-G-O0-NEXT: s_mov_b64 s[10:11], -1
-; GFX9-G-O0-NEXT: s_mov_b32 s9, s10
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v17
-; GFX9-G-O0-NEXT: v_add_co_u32_e64 v16, s[4:5], v8, s9
-; GFX9-G-O0-NEXT: s_mov_b32 s8, s11
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v18
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, s8
-; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v8, s[4:5], v8, v11, s[4:5]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v9
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v17, s9
-; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v19, s[4:5], v11, v17, s[4:5]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v10
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, s8
-; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v9, s[4:5], v9, v10, s[4:5]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr19 killed $vgpr19 def $vgpr19_vgpr20 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v20, v9
-; GFX9-G-O0-NEXT: ; kill: def $vgpr16 killed $vgpr16 def $vgpr16_vgpr17 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v17, v8
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v16
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v17
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v16
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v17
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v18
; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v19
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v20
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v18, v17
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v21, v20
-; GFX9-G-O0-NEXT: v_or_b32_e64 v18, v18, v21
-; GFX9-G-O0-NEXT: ; kill: def $vgpr16 killed $vgpr16 killed $vgpr16_vgpr17 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v17, v19
-; GFX9-G-O0-NEXT: v_or_b32_e64 v16, v16, v17
+; GFX9-G-O0-NEXT: s_mov_b32 s8, -1
+; GFX9-G-O0-NEXT: s_mov_b32 s12, -1
+; GFX9-G-O0-NEXT: s_mov_b32 s11, -1
+; GFX9-G-O0-NEXT: s_mov_b32 s10, -1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, s8
+; GFX9-G-O0-NEXT: v_add_co_u32_e64 v16, s[8:9], v11, v16
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, s12
+; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v17, s[8:9], v10, v11, s[8:9]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, s11
+; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v19, s[8:9], v9, v10, s[8:9]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, s10
+; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v18, s[8:9], v8, v9, s[8:9]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v16
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v17
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v19
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v18
+; GFX9-G-O0-NEXT: v_or_b32_e64 v16, v16, v19
+; GFX9-G-O0-NEXT: v_or_b32_e64 v18, v17, v18
; GFX9-G-O0-NEXT: ; kill: def $vgpr16 killed $vgpr16 def $vgpr16_vgpr17 killed $exec
; GFX9-G-O0-NEXT: v_mov_b32_e32 v17, v18
-; GFX9-G-O0-NEXT: v_cmp_eq_u64_e64 s[4:5], v[16:17], v[12:13]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v19, s5
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v18, s4
+; GFX9-G-O0-NEXT: v_cmp_eq_u64_e64 s[4:5], v[16:17], v[18:19]
; GFX9-G-O0-NEXT: s_or_b64 s[4:5], s[4:5], s[6:7]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v17, v3
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, v2
-; GFX9-G-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v17, v1
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, v0
-; GFX9-G-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v17, v15
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, v14
-; GFX9-G-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v17, v13
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, v12
-; GFX9-G-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_mov_b64 s[6:7], s[4:5]
-; GFX9-G-O0-NEXT: v_writelane_b32 v31, s6, 4
-; GFX9-G-O0-NEXT: v_writelane_b32 v31, s7, 5
+; GFX9-G-O0-NEXT: v_writelane_b32 v32, s4, 4
+; GFX9-G-O0-NEXT: v_writelane_b32 v32, s5, 5
; GFX9-G-O0-NEXT: s_mov_b64 s[6:7], s[4:5]
-; GFX9-G-O0-NEXT: v_writelane_b32 v31, s6, 8
-; GFX9-G-O0-NEXT: v_writelane_b32 v31, s7, 9
-; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
-; GFX9-G-O0-NEXT: buffer_store_dword v31, off, s[0:3], s32 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
-; GFX9-G-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: v_writelane_b32 v32, s6, 8
+; GFX9-G-O0-NEXT: v_writelane_b32 v32, s7, 9
+; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
+; GFX9-G-O0-NEXT: buffer_store_dword v32, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19]
+; GFX9-G-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX9-G-O0-NEXT: s_cbranch_execnz .LBB1_6
+; GFX9-G-O0-NEXT: s_cbranch_execnz .LBB1_5
; GFX9-G-O0-NEXT: s_branch .LBB1_1
-; GFX9-G-O0-NEXT: .LBB1_7: ; %udiv-preheader
-; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
-; GFX9-G-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
-; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(9)
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v10
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
-; GFX9-G-O0-NEXT: v_lshrrev_b64 v[6:7], v4, v[18:19]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v7
+; GFX9-G-O0-NEXT: .LBB1_6: ; %udiv-preheader
+; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
+; GFX9-G-O0-NEXT: buffer_load_dword v32, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19]
+; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: s_mov_b32 s4, 64
-; GFX9-G-O0-NEXT: v_sub_u32_e64 v20, s4, v4
-; GFX9-G-O0-NEXT: v_lshlrev_b64 v[20:21], v20, v[16:17]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v22, v21
-; GFX9-G-O0-NEXT: v_or_b32_e64 v5, v5, v22
-; GFX9-G-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 killed $vgpr6_vgpr7 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v20
-; GFX9-G-O0-NEXT: v_or_b32_e64 v6, v6, v7
-; GFX9-G-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v5
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v22, v7
-; GFX9-G-O0-NEXT: v_cmp_lt_u32_e64 s[6:7], v4, s4
-; GFX9-G-O0-NEXT: v_sub_u32_e64 v5, v4, s4
-; GFX9-G-O0-NEXT: v_lshrrev_b64 v[20:21], v5, v[16:17]
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(2)
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, v5
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v4
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v21, v7
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v20, v6
+; GFX9-G-O0-NEXT: s_mov_b32 s5, 0xffffffc0
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, s5
+; GFX9-G-O0-NEXT: v_add_u32_e64 v4, v12, v4
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, s4
+; GFX9-G-O0-NEXT: v_sub_u32_e64 v5, v5, v12
+; GFX9-G-O0-NEXT: s_mov_b32 s6, 0
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s4
+; GFX9-G-O0-NEXT: v_cmp_lt_u32_e64 s[4:5], v12, v6
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s6
+; GFX9-G-O0-NEXT: v_cmp_eq_u32_e64 s[6:7], v12, v6
+; GFX9-G-O0-NEXT: v_lshrrev_b64 v[6:7], v12, v[20:21]
+; GFX9-G-O0-NEXT: v_lshrrev_b64 v[25:26], v12, v[14:15]
+; GFX9-G-O0-NEXT: v_lshlrev_b64 v[23:24], v5, v[20:21]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v25
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v26
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v22, v23
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v24
+; GFX9-G-O0-NEXT: v_or_b32_e64 v13, v13, v22
+; GFX9-G-O0-NEXT: v_or_b32_e64 v12, v5, v12
+; GFX9-G-O0-NEXT: s_mov_b64 s[8:9], 0
+; GFX9-G-O0-NEXT: v_lshrrev_b64 v[20:21], v4, v[20:21]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v20
; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v21
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v5, v5, v22, s[6:7]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v6
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v20
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[6:7]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v5
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v7
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 0
-; GFX9-G-O0-NEXT: v_cmp_eq_u32_e64 s[4:5], v4, s4
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v20, v19
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v5, v5, v20, s[4:5]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 killed $vgpr6_vgpr7 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v18
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[4:5]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v5
-; GFX9-G-O0-NEXT: v_lshrrev_b64 v[4:5], v4, v[16:17]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v17, v5
-; GFX9-G-O0-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-G-O0-NEXT: s_mov_b32 s8, s5
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, s8
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v16, v16, v17, s[6:7]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v4
-; GFX9-G-O0-NEXT: s_mov_b32 s8, s4
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, s8
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v4, v4, v5, s[6:7]
+; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v4, v4, v13, s[4:5]
+; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v5, v5, v12, s[4:5]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v14
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v15
+; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v4, v4, v13, s[6:7]
+; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v12, v5, v12, s[6:7]
; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v16
-; GFX9-G-O0-NEXT: s_mov_b64 s[10:11], -1
-; GFX9-G-O0-NEXT: s_mov_b32 s9, s10
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, v14
-; GFX9-G-O0-NEXT: v_add_co_u32_e64 v18, s[6:7], v16, s9
-; GFX9-G-O0-NEXT: s_mov_b32 s8, s11
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v15
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, s8
-; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v14, s[6:7], v14, v15, s[6:7]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr18 killed $vgpr18 def $vgpr18_vgpr19 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v19, v14
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v12
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v12
; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, s9
-; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v16, s[6:7], v14, v15, s[6:7]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v13
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, s8
-; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v12, s[6:7], v12, v13, s[6:7]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr16 killed $vgpr16 def $vgpr16_vgpr17 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v17, v12
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, s8
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v6
+; GFX9-G-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 killed $vgpr6_vgpr7 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v14
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v15
+; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v12, v12, v13, s[4:5]
+; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[4:5]
+; GFX9-G-O0-NEXT: ; kill: def $vgpr12 killed $vgpr12 def $vgpr12_vgpr13 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v6
+; GFX9-G-O0-NEXT: ; kill: def $vgpr4_vgpr5 killed $vgpr4_vgpr5 def $vgpr4_vgpr5_vgpr6_vgpr7 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v12
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v13
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, v16
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v17
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v18
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v19
+; GFX9-G-O0-NEXT: s_mov_b32 s4, -1
+; GFX9-G-O0-NEXT: s_mov_b32 s10, -1
+; GFX9-G-O0-NEXT: s_mov_b32 s7, -1
+; GFX9-G-O0-NEXT: s_mov_b32 s6, -1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, s4
+; GFX9-G-O0-NEXT: v_add_co_u32_e64 v15, s[4:5], v15, v16
+; GFX9-G-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, s10
+; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v14, s[4:5], v14, v15, s[4:5]
+; GFX9-G-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, s7
+; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v13, s[4:5], v13, v14, s[4:5]
+; GFX9-G-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, s6
+; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v12, s[4:5], v12, v13, s[4:5]
+; GFX9-G-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_mov_b64 s[4:5], s[8:9]
+; GFX9-G-O0-NEXT: s_mov_b64 s[6:7], s[8:9]
+; GFX9-G-O0-NEXT: v_writelane_b32 v32, s8, 8
+; GFX9-G-O0-NEXT: v_writelane_b32 v32, s9, 9
+; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
+; GFX9-G-O0-NEXT: buffer_store_dword v32, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, s7
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, s6
; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, s5
; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, s4
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, s5
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, s4
-; GFX9-G-O0-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_writelane_b32 v31, s4, 8
-; GFX9-G-O0-NEXT: v_writelane_b32 v31, s5, 9
-; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
-; GFX9-G-O0-NEXT: buffer_store_dword v31, off, s[0:3], s32 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
-; GFX9-G-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_branch .LBB1_6
-; GFX9-G-O0-NEXT: .LBB1_8: ; %udiv-bb1
-; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
-; GFX9-G-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
-; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_branch .LBB1_5
+; GFX9-G-O0-NEXT: .LBB1_7: ; %udiv-bb1
+; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
+; GFX9-G-O0-NEXT: buffer_load_dword v32, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19]
+; GFX9-G-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_mov_b64 s[6:7], 1
-; GFX9-G-O0-NEXT: s_mov_b32 s4, s6
+; GFX9-G-O0-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-G-O0-NEXT: s_mov_b32 s6, 1
+; GFX9-G-O0-NEXT: s_mov_b32 s10, 0
+; GFX9-G-O0-NEXT: s_mov_b32 s9, 0
+; GFX9-G-O0-NEXT: s_mov_b32 s8, 0
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, s6
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: v_add_co_u32_e64 v4, s[6:7], v1, v4
+; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, s10
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(1)
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v3
-; GFX9-G-O0-NEXT: v_add_co_u32_e64 v0, s[4:5], v2, s4
-; GFX9-G-O0-NEXT: s_mov_b32 s6, s7
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, s6
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
-; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v3, s[4:5], v4, v1, s[4:5]
-; GFX9-G-O0-NEXT: s_mov_b64 s[6:7], 0
-; GFX9-G-O0-NEXT: s_mov_b32 s8, s6
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v12
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, s8
-; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v8, s[4:5], v1, v5, s[4:5]
-; GFX9-G-O0-NEXT: s_mov_b32 s9, s7
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v13
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, s9
-; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v1, s[4:5], v1, v5, s[4:5]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v1
-; GFX9-G-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v3
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v1
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v0
-; GFX9-G-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v5, s[6:7], v3, v5, s[6:7]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, s9
+; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v7, s[6:7], v2, v3, s[6:7]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s8
+; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v6, s[6:7], v0, v2, s[6:7]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v4
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v5
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v7
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, v6
+; GFX9-G-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_mov_b32 s6, 0x7f
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s6
+; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v3, s[6:7], v0, v1
+; GFX9-G-O0-NEXT: s_mov_b32 s7, 64
; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v9
; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v8
-; GFX9-G-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_mov_b64 s[10:11], 0x7f
-; GFX9-G-O0-NEXT: s_mov_b32 s4, s10
-; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v2, s[4:5], s4, v2
-; GFX9-G-O0-NEXT: s_mov_b32 s10, s11
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, s10
-; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v4, s[4:5], v3, v4, s[4:5]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v4
-; GFX9-G-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 killed $vgpr2_vgpr3 killed $exec
-; GFX9-G-O0-NEXT: v_lshlrev_b64 v[4:5], v2, v[10:11]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v5
-; GFX9-G-O0-NEXT: s_mov_b32 s10, 64
-; GFX9-G-O0-NEXT: v_sub_u32_e64 v12, s10, v2
-; GFX9-G-O0-NEXT: v_lshrrev_b64 v[12:13], v12, v[6:7]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v13
-; GFX9-G-O0-NEXT: v_or_b32_e64 v3, v3, v14
-; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 killed $vgpr4_vgpr5 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v12
-; GFX9-G-O0-NEXT: v_or_b32_e64 v4, v4, v5
-; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v3
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v5
-; GFX9-G-O0-NEXT: v_cmp_lt_u32_e64 s[4:5], v2, s10
-; GFX9-G-O0-NEXT: v_sub_u32_e64 v3, v2, s10
-; GFX9-G-O0-NEXT: v_lshlrev_b64 v[12:13], v3, v[6:7]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v13
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v3, v3, v14, s[4:5]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v4
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v12
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v4, v4, v5, s[4:5]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v3
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v5
-; GFX9-G-O0-NEXT: s_mov_b32 s10, 0
-; GFX9-G-O0-NEXT: v_cmp_eq_u32_e64 s[10:11], v2, s10
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v11
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v3, v3, v12, s[10:11]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 killed $vgpr4_vgpr5 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v10
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v4, v4, v5, s[10:11]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v3
-; GFX9-G-O0-NEXT: v_lshlrev_b64 v[6:7], v2, v[6:7]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v7
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s9
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v2, v2, v3, s[4:5]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 killed $vgpr6_vgpr7 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, s8
-; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v6, v3, v6, s[4:5]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v2
-; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_mov_b32 s6, 0xffffffc0
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s6
+; GFX9-G-O0-NEXT: v_add_u32_e64 v2, v3, v0
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s7
+; GFX9-G-O0-NEXT: v_sub_u32_e64 v8, v0, v3
+; GFX9-G-O0-NEXT: s_mov_b32 s6, 0
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s7
+; GFX9-G-O0-NEXT: v_cmp_lt_u32_e64 s[8:9], v3, v0
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s6
+; GFX9-G-O0-NEXT: v_cmp_eq_u32_e64 s[6:7], v3, v0
+; GFX9-G-O0-NEXT: v_lshlrev_b64 v[0:1], v3, v[12:13]
+; GFX9-G-O0-NEXT: v_lshrrev_b64 v[17:18], v8, v[12:13]
+; GFX9-G-O0-NEXT: v_lshlrev_b64 v[15:16], v3, v[10:11]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v17
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v18
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v15
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v16
+; GFX9-G-O0-NEXT: v_or_b32_e64 v9, v9, v14
+; GFX9-G-O0-NEXT: v_or_b32_e64 v3, v3, v8
+; GFX9-G-O0-NEXT: v_lshlrev_b64 v[12:13], v2, v[12:13]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, s5
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, s4
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v0
; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v1
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v9
-; GFX9-G-O0-NEXT: v_or_b32_e64 v2, v2, v3
-; GFX9-G-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v8
-; GFX9-G-O0-NEXT: v_or_b32_e64 v0, v0, v1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v14
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v15
+; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v0, v0, v8, s[8:9]
+; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v2, v1, v2, s[8:9]
; GFX9-G-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v2
-; GFX9-G-O0-NEXT: v_cmp_ne_u64_e64 s[4:5], v[0:1], s[6:7]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s6
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, s7
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s6
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, s7
-; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v12
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v13
+; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v8, v8, v9, s[8:9]
+; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v2, v2, v3, s[8:9]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v11
+; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v8, v8, v9, s[6:7]
+; GFX9-G-O0-NEXT: v_cndmask_b32_e64 v2, v2, v3, s[6:7]
+; GFX9-G-O0-NEXT: ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v2
+; GFX9-G-O0-NEXT: ; kill: def $vgpr0_vgpr1 killed $vgpr0_vgpr1 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v8
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v9
+; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX9-G-O0-NEXT: s_mov_b64 s[10:11], s[4:5]
+; GFX9-G-O0-NEXT: v_or_b32_e64 v4, v4, v7
+; GFX9-G-O0-NEXT: v_or_b32_e64 v6, v5, v6
+; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v6
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s5
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s4
+; GFX9-G-O0-NEXT: v_cmp_ne_u64_e64 s[4:5], v[4:5], v[6:7]
+; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s8
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, s9
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, s11
+; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_nop 0
-; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_mov_b64 s[6:7], exec
; GFX9-G-O0-NEXT: s_and_b64 s[4:5], s[6:7], s[4:5]
; GFX9-G-O0-NEXT: s_xor_b64 s[6:7], s[4:5], s[6:7]
-; GFX9-G-O0-NEXT: v_writelane_b32 v31, s6, 6
-; GFX9-G-O0-NEXT: v_writelane_b32 v31, s7, 7
-; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
-; GFX9-G-O0-NEXT: buffer_store_dword v31, off, s[0:3], s32 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
+; GFX9-G-O0-NEXT: v_writelane_b32 v32, s6, 6
+; GFX9-G-O0-NEXT: v_writelane_b32 v32, s7, 7
+; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
+; GFX9-G-O0-NEXT: buffer_store_dword v32, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-G-O0-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-G-O0-NEXT: s_cbranch_execz .LBB1_5
-; GFX9-G-O0-NEXT: s_branch .LBB1_7
-; GFX9-G-O0-NEXT: .LBB1_9: ; %udiv-end
-; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 32
+; GFX9-G-O0-NEXT: s_cbranch_execz .LBB1_4
+; GFX9-G-O0-NEXT: s_branch .LBB1_6
+; GFX9-G-O0-NEXT: .LBB1_8: ; %udiv-end
+; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v3
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(2)
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v4
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(1)
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v5
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
-; GFX9-G-O0-NEXT: v_lshrrev_b64 v[0:1], s4, v[6:7]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v0
-; GFX9-G-O0-NEXT: v_lshrrev_b64 v[2:3], s4, v[4:5]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v2
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v6
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v4
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v6
; GFX9-G-O0-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GFX9-G-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-G-O0-NEXT: s_setpc_b64 s[30:31]
@@ -4470,68 +4340,64 @@ define i128 @v_sdiv_i128_v_pow2k(i128 %lhs) {
; GFX9-G-NEXT: v_lshrrev_b64 v[4:5], 31, v[4:5]
; GFX9-G-NEXT: v_add_co_u32_e32 v0, vcc, v0, v4
; GFX9-G-NEXT: v_addc_co_u32_e32 v4, vcc, v1, v5, vcc
-; GFX9-G-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v2, vcc
-; GFX9-G-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX9-G-NEXT: v_lshlrev_b64 v[0:1], 31, v[2:3]
-; GFX9-G-NEXT: v_lshrrev_b32_e32 v2, 1, v4
-; GFX9-G-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX9-G-NEXT: v_ashrrev_i32_e32 v2, 1, v3
-; GFX9-G-NEXT: v_ashrrev_i32_e32 v3, 31, v3
+; GFX9-G-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v2, vcc
+; GFX9-G-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v3, vcc
+; GFX9-G-NEXT: v_lshlrev_b64 v[0:1], 31, v[1:2]
+; GFX9-G-NEXT: v_lshrrev_b32_e32 v3, 1, v4
+; GFX9-G-NEXT: v_or_b32_e32 v0, v0, v3
+; GFX9-G-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX9-G-NEXT: v_ashrrev_i32_e32 v2, 1, v2
; GFX9-G-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-G-O0-LABEL: v_sdiv_i128_v_pow2k:
; GFX9-G-O0: ; %bb.0:
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v2
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v0
-; GFX9-G-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v1
-; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v3
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 63
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v4
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v5
-; GFX9-G-O0-NEXT: v_ashrrev_i64 v[0:1], s4, v[0:1]
-; GFX9-G-O0-NEXT: s_mov_b32 s5, 31
-; GFX9-G-O0-NEXT: v_lshrrev_b64 v[2:3], s5, v[0:1]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v6
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v2
-; GFX9-G-O0-NEXT: v_add_co_u32_e64 v0, s[6:7], v0, v1
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v7
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v3
-; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v2, s[6:7], v1, v2, s[6:7]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v4
-; GFX9-G-O0-NEXT: s_mov_b64 s[8:9], 0
-; GFX9-G-O0-NEXT: s_mov_b32 s4, s8
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, s4
-; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v3, s[6:7], v1, v3, s[6:7]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v5
-; GFX9-G-O0-NEXT: s_mov_b32 s4, s9
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v1, s[6:7], v1, v4, s[6:7]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr3 killed $vgpr3 def $vgpr3_vgpr4 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v1
-; GFX9-G-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v2
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 33
-; GFX9-G-O0-NEXT: v_lshrrev_b64 v[1:2], s4, v[0:1]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v2
-; GFX9-G-O0-NEXT: v_lshlrev_b64 v[5:6], s5, v[3:4]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v6
-; GFX9-G-O0-NEXT: v_or_b32_e64 v0, v0, v7
-; GFX9-G-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr1_vgpr2 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v5
-; GFX9-G-O0-NEXT: v_or_b32_e64 v1, v1, v2
-; GFX9-G-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v0
+; GFX9-G-O0-NEXT: ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9_vgpr10_vgpr11 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v2
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v3
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v11
+; GFX9-G-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr0_vgpr1 killed $exec
+; GFX9-G-O0-NEXT: s_mov_b32 s4, 31
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v0, v0, v1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v0
; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-G-O0-NEXT: v_ashrrev_i64 v[3:4], s4, v[3:4]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v1
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 32
-; GFX9-G-O0-NEXT: v_lshrrev_b64 v[1:2], s4, v[1:2]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr1_vgpr2 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v3
-; GFX9-G-O0-NEXT: v_lshrrev_b64 v[3:4], s4, v[3:4]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr3 killed $vgpr3 killed $vgpr3_vgpr4 killed $exec
+; GFX9-G-O0-NEXT: s_mov_b32 s4, 31
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-G-O0-NEXT: v_lshrrev_b64 v[6:7], v0, v[1:2]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v8
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v9
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v11
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v6
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v7
+; GFX9-G-O0-NEXT: s_mov_b32 s8, 0
+; GFX9-G-O0-NEXT: s_mov_b32 s5, 0
+; GFX9-G-O0-NEXT: v_add_co_u32_e64 v4, s[6:7], v4, v5
+; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v1, s[6:7], v1, v3, s[6:7]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, s8
+; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v5, s[6:7], v2, v3, s[6:7]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s5
+; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v4, s[6:7], v0, v2, s[6:7]
+; GFX9-G-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-G-O0-NEXT: s_mov_b32 s5, 1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s5
+; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v2, v0, v1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-G-O0-NEXT: v_lshlrev_b64 v[5:6], v0, v[5:6]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v5
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v6
+; GFX9-G-O0-NEXT: v_or_b32_e64 v0, v0, v2
+; GFX9-G-O0-NEXT: s_mov_b32 s4, 31
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s4
+; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v3, v2, v4
+; GFX9-G-O0-NEXT: s_mov_b32 s4, 1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s4
+; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v2, v2, v4
; GFX9-G-O0-NEXT: s_setpc_b64 s[30:31]
%div = sdiv i128 %lhs, 8589934592
ret i128 %div
@@ -4591,45 +4457,34 @@ define i128 @v_sdiv_exact_i128_v_pow2k(i128 %lhs) {
; GFX9-G-NEXT: v_mov_b32_e32 v4, v1
; GFX9-G-NEXT: v_lshlrev_b64 v[0:1], 31, v[2:3]
; GFX9-G-NEXT: v_lshrrev_b32_e32 v2, 1, v4
-; GFX9-G-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX9-G-NEXT: v_ashrrev_i32_e32 v4, 31, v3
+; GFX9-G-NEXT: v_or_b32_e32 v0, v0, v2
; GFX9-G-NEXT: v_ashrrev_i32_e32 v2, 1, v3
-; GFX9-G-NEXT: v_ashrrev_i32_e32 v3, 31, v3
+; GFX9-G-NEXT: v_mov_b32_e32 v3, v4
; GFX9-G-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-G-O0-LABEL: v_sdiv_exact_i128_v_pow2k:
; GFX9-G-O0: ; %bb.0:
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v2
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v0
-; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v2
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v3
; GFX9-G-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v1
-; GFX9-G-O0-NEXT: ; kill: def $vgpr3 killed $vgpr3 def $vgpr3_vgpr4 killed $exec
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v0
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-G-O0-NEXT: s_mov_b32 s4, 1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v2, v0, v1
; GFX9-G-O0-NEXT: s_mov_b32 s4, 31
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v3
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v4
-; GFX9-G-O0-NEXT: v_lshlrev_b64 v[1:2], s4, v[0:1]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v2
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 33
-; GFX9-G-O0-NEXT: v_lshrrev_b64 v[5:6], s4, v[5:6]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v6
-; GFX9-G-O0-NEXT: v_or_b32_e64 v0, v0, v7
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v1
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v5
-; GFX9-G-O0-NEXT: v_or_b32_e64 v1, v1, v2
-; GFX9-G-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-G-O0-NEXT: v_ashrrev_i64 v[3:4], s4, v[3:4]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v1
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 32
-; GFX9-G-O0-NEXT: v_lshrrev_b64 v[1:2], s4, v[1:2]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr1_vgpr2 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v3
-; GFX9-G-O0-NEXT: v_lshrrev_b64 v[3:4], s4, v[3:4]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr3 killed $vgpr3 killed $vgpr3_vgpr4 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-G-O0-NEXT: v_lshlrev_b64 v[5:6], v0, v[5:6]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v5
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v6
+; GFX9-G-O0-NEXT: v_or_b32_e64 v0, v0, v2
+; GFX9-G-O0-NEXT: s_mov_b32 s4, 31
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s4
+; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v3, v2, v4
+; GFX9-G-O0-NEXT: s_mov_b32 s4, 1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s4
+; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v2, v2, v4
; GFX9-G-O0-NEXT: s_setpc_b64 s[30:31]
%div = sdiv exact i128 %lhs, 8589934592
ret i128 %div
@@ -4689,7 +4544,7 @@ define i128 @v_udiv_i128_v_pow2k(i128 %lhs) {
; GFX9-G-NEXT: v_mov_b32_e32 v4, v1
; GFX9-G-NEXT: v_lshlrev_b64 v[0:1], 31, v[2:3]
; GFX9-G-NEXT: v_lshrrev_b32_e32 v2, 1, v4
-; GFX9-G-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX9-G-NEXT: v_or_b32_e32 v0, v0, v2
; GFX9-G-NEXT: v_lshrrev_b32_e32 v2, 1, v3
; GFX9-G-NEXT: v_mov_b32_e32 v3, 0
; GFX9-G-NEXT: s_setpc_b64 s[30:31]
@@ -4697,37 +4552,23 @@ define i128 @v_udiv_i128_v_pow2k(i128 %lhs) {
; GFX9-G-O0-LABEL: v_udiv_i128_v_pow2k:
; GFX9-G-O0: ; %bb.0:
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v2
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v0
-; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v1
-; GFX9-G-O0-NEXT: ; kill: def $vgpr3 killed $vgpr3 def $vgpr3_vgpr4 killed $exec
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v0
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v2
+; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-G-O0-NEXT: s_mov_b32 s4, 1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v2, v0, v1
; GFX9-G-O0-NEXT: s_mov_b32 s4, 31
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v3
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v4
-; GFX9-G-O0-NEXT: v_lshlrev_b64 v[1:2], s4, v[0:1]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v2
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 33
-; GFX9-G-O0-NEXT: v_lshrrev_b64 v[5:6], s4, v[5:6]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v6
-; GFX9-G-O0-NEXT: v_or_b32_e64 v0, v0, v7
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-G-O0-NEXT: v_lshlrev_b64 v[4:5], v0, v[4:5]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v4
; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v5
-; GFX9-G-O0-NEXT: v_or_b32_e64 v1, v1, v2
-; GFX9-G-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-G-O0-NEXT: v_lshrrev_b64 v[3:4], s4, v[3:4]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v1
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 32
-; GFX9-G-O0-NEXT: v_lshrrev_b64 v[1:2], s4, v[1:2]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr1_vgpr2 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v3
-; GFX9-G-O0-NEXT: v_lshrrev_b64 v[3:4], s4, v[3:4]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr3 killed $vgpr3 killed $vgpr3_vgpr4 killed $exec
+; GFX9-G-O0-NEXT: v_or_b32_e64 v0, v0, v2
+; GFX9-G-O0-NEXT: s_mov_b32 s4, 1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s4
+; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v2, v2, v3
+; GFX9-G-O0-NEXT: s_mov_b32 s4, 0
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, s4
; GFX9-G-O0-NEXT: s_setpc_b64 s[30:31]
%div = udiv i128 %lhs, 8589934592
ret i128 %div
@@ -4787,7 +4628,7 @@ define i128 @v_udiv_exact_i128_v_pow2k(i128 %lhs) {
; GFX9-G-NEXT: v_mov_b32_e32 v4, v1
; GFX9-G-NEXT: v_lshlrev_b64 v[0:1], 31, v[2:3]
; GFX9-G-NEXT: v_lshrrev_b32_e32 v2, 1, v4
-; GFX9-G-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX9-G-NEXT: v_or_b32_e32 v0, v0, v2
; GFX9-G-NEXT: v_lshrrev_b32_e32 v2, 1, v3
; GFX9-G-NEXT: v_mov_b32_e32 v3, 0
; GFX9-G-NEXT: s_setpc_b64 s[30:31]
@@ -4795,37 +4636,23 @@ define i128 @v_udiv_exact_i128_v_pow2k(i128 %lhs) {
; GFX9-G-O0-LABEL: v_udiv_exact_i128_v_pow2k:
; GFX9-G-O0: ; %bb.0:
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v2
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v0
-; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v1
-; GFX9-G-O0-NEXT: ; kill: def $vgpr3 killed $vgpr3 def $vgpr3_vgpr4 killed $exec
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v0
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v2
+; GFX9-G-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-G-O0-NEXT: s_mov_b32 s4, 1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v2, v0, v1
; GFX9-G-O0-NEXT: s_mov_b32 s4, 31
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v3
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v4
-; GFX9-G-O0-NEXT: v_lshlrev_b64 v[1:2], s4, v[0:1]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v2
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 33
-; GFX9-G-O0-NEXT: v_lshrrev_b64 v[5:6], s4, v[5:6]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v6
-; GFX9-G-O0-NEXT: v_or_b32_e64 v0, v0, v7
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-G-O0-NEXT: v_lshlrev_b64 v[4:5], v0, v[4:5]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v4
; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v5
-; GFX9-G-O0-NEXT: v_or_b32_e64 v1, v1, v2
-; GFX9-G-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-G-O0-NEXT: v_lshrrev_b64 v[3:4], s4, v[3:4]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v1
-; GFX9-G-O0-NEXT: s_mov_b32 s4, 32
-; GFX9-G-O0-NEXT: v_lshrrev_b64 v[1:2], s4, v[1:2]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr1_vgpr2 killed $exec
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v3
-; GFX9-G-O0-NEXT: v_lshrrev_b64 v[3:4], s4, v[3:4]
-; GFX9-G-O0-NEXT: ; kill: def $vgpr3 killed $vgpr3 killed $vgpr3_vgpr4 killed $exec
+; GFX9-G-O0-NEXT: v_or_b32_e64 v0, v0, v2
+; GFX9-G-O0-NEXT: s_mov_b32 s4, 1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s4
+; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v2, v2, v3
+; GFX9-G-O0-NEXT: s_mov_b32 s4, 0
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, s4
; GFX9-G-O0-NEXT: s_setpc_b64 s[30:31]
%div = udiv exact i128 %lhs, 8589934592
ret i128 %div
diff --git a/llvm/test/CodeGen/AMDGPU/div_v2i128.ll b/llvm/test/CodeGen/AMDGPU/div_v2i128.ll
index df15c2d9e0601..a4ccb6ca116dc 100644
--- a/llvm/test/CodeGen/AMDGPU/div_v2i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/div_v2i128.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
; RUN: llc -global-isel=0 -mtriple=amdgpu7.00-amd-amdhsa -o - %s | FileCheck -check-prefix=SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu7.00-amd-amdhsa -o - %s | FileCheck -check-prefix=GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu7.00-amd-amdhsa -o - %s | FileCheck -check-prefix=GISEL %s
define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-LABEL: v_sdiv_v2i128_vv:
@@ -410,402 +410,402 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL: ; %bb.0: ; %_udiv-special-cases_udiv-special-cases
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GISEL-NEXT: v_ashrrev_i32_e32 v24, 31, v3
-; GISEL-NEXT: v_xor_b32_e32 v0, v0, v24
-; GISEL-NEXT: v_xor_b32_e32 v1, v1, v24
+; GISEL-NEXT: v_xor_b32_e32 v0, v24, v0
+; GISEL-NEXT: v_xor_b32_e32 v1, v24, v1
; GISEL-NEXT: v_sub_i32_e32 v16, vcc, v0, v24
+; GISEL-NEXT: v_xor_b32_e32 v2, v24, v2
; GISEL-NEXT: v_subb_u32_e32 v17, vcc, v1, v24, vcc
-; GISEL-NEXT: v_xor_b32_e32 v1, v2, v24
; GISEL-NEXT: v_ashrrev_i32_e32 v25, 31, v11
-; GISEL-NEXT: v_xor_b32_e32 v0, v3, v24
-; GISEL-NEXT: v_subb_u32_e32 v18, vcc, v1, v24, vcc
-; GISEL-NEXT: v_subb_u32_e32 v19, vcc, v0, v24, vcc
-; GISEL-NEXT: v_xor_b32_e32 v1, v8, v25
-; GISEL-NEXT: v_xor_b32_e32 v0, v9, v25
-; GISEL-NEXT: v_sub_i32_e32 v28, vcc, v1, v25
-; GISEL-NEXT: v_subb_u32_e32 v29, vcc, v0, v25, vcc
-; GISEL-NEXT: v_xor_b32_e32 v1, v10, v25
-; GISEL-NEXT: v_xor_b32_e32 v0, v11, v25
-; GISEL-NEXT: v_subb_u32_e32 v2, vcc, v1, v25, vcc
-; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v0, v25, vcc
-; GISEL-NEXT: v_or_b32_e32 v1, v29, v3
-; GISEL-NEXT: v_or_b32_e32 v0, v28, v2
+; GISEL-NEXT: v_xor_b32_e32 v3, v24, v3
+; GISEL-NEXT: v_subb_u32_e32 v18, vcc, v2, v24, vcc
+; GISEL-NEXT: v_subb_u32_e32 v19, vcc, v3, v24, vcc
+; GISEL-NEXT: v_xor_b32_e32 v0, v25, v8
+; GISEL-NEXT: v_xor_b32_e32 v1, v25, v9
+; GISEL-NEXT: v_sub_i32_e32 v26, vcc, v0, v25
+; GISEL-NEXT: v_xor_b32_e32 v2, v25, v10
+; GISEL-NEXT: v_subb_u32_e32 v27, vcc, v1, v25, vcc
+; GISEL-NEXT: v_xor_b32_e32 v3, v25, v11
+; GISEL-NEXT: v_subb_u32_e32 v10, vcc, v2, v25, vcc
+; GISEL-NEXT: v_subb_u32_e32 v11, vcc, v3, v25, vcc
+; GISEL-NEXT: v_or_b32_e32 v0, v26, v10
+; GISEL-NEXT: v_or_b32_e32 v1, v27, v11
; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
-; GISEL-NEXT: v_or_b32_e32 v1, v17, v19
; GISEL-NEXT: v_or_b32_e32 v0, v16, v18
+; GISEL-NEXT: v_or_b32_e32 v1, v17, v19
; GISEL-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[0:1]
-; GISEL-NEXT: v_ffbh_u32_e32 v0, v2
-; GISEL-NEXT: v_add_i32_e64 v0, s[6:7], 32, v0
-; GISEL-NEXT: v_ffbh_u32_e32 v1, v3
+; GISEL-NEXT: v_ffbh_u32_e32 v1, v26
+; GISEL-NEXT: v_ffbh_u32_e32 v0, v27
+; GISEL-NEXT: v_add_i32_e64 v1, s[6:7], 32, v1
; GISEL-NEXT: v_min_u32_e32 v0, v0, v1
-; GISEL-NEXT: v_ffbh_u32_e32 v1, v28
+; GISEL-NEXT: v_ffbh_u32_e32 v1, v10
; GISEL-NEXT: v_add_i32_e64 v1, s[6:7], 32, v1
-; GISEL-NEXT: v_ffbh_u32_e32 v8, v29
-; GISEL-NEXT: v_min_u32_e32 v1, v1, v8
-; GISEL-NEXT: v_add_i32_e64 v1, s[6:7], 64, v1
-; GISEL-NEXT: v_addc_u32_e64 v8, s[6:7], 0, 0, s[6:7]
-; GISEL-NEXT: v_cmp_ne_u64_e64 s[6:7], 0, v[2:3]
-; GISEL-NEXT: v_ffbh_u32_e32 v10, v17
+; GISEL-NEXT: v_add_i32_e64 v0, s[6:7], 64, v0
+; GISEL-NEXT: v_ffbh_u32_e32 v2, v11
+; GISEL-NEXT: v_cmp_eq_u64_e64 s[6:7], 0, v[10:11]
+; GISEL-NEXT: v_min_u32_e32 v1, v2, v1
; GISEL-NEXT: v_cndmask_b32_e64 v0, v1, v0, s[6:7]
-; GISEL-NEXT: v_ffbh_u32_e32 v1, v18
-; GISEL-NEXT: v_cndmask_b32_e64 v9, v8, 0, s[6:7]
+; GISEL-NEXT: v_ffbh_u32_e32 v1, v16
; GISEL-NEXT: v_add_i32_e64 v1, s[6:7], 32, v1
-; GISEL-NEXT: v_ffbh_u32_e32 v8, v19
-; GISEL-NEXT: v_min_u32_e32 v1, v1, v8
-; GISEL-NEXT: v_ffbh_u32_e32 v8, v16
-; GISEL-NEXT: v_add_i32_e64 v8, s[6:7], 32, v8
-; GISEL-NEXT: v_min_u32_e32 v8, v8, v10
-; GISEL-NEXT: v_add_i32_e64 v8, s[6:7], 64, v8
-; GISEL-NEXT: v_addc_u32_e64 v10, s[6:7], 0, 0, s[6:7]
-; GISEL-NEXT: v_cmp_ne_u64_e64 s[6:7], 0, v[18:19]
-; GISEL-NEXT: s_mov_b64 s[8:9], 0x7f
-; GISEL-NEXT: v_cndmask_b32_e64 v1, v8, v1, s[6:7]
-; GISEL-NEXT: v_cndmask_b32_e64 v10, v10, 0, s[6:7]
-; GISEL-NEXT: v_sub_i32_e64 v8, s[6:7], v0, v1
-; GISEL-NEXT: v_subb_u32_e64 v9, s[6:7], v9, v10, s[6:7]
-; GISEL-NEXT: v_subb_u32_e64 v10, s[6:7], 0, 0, s[6:7]
-; GISEL-NEXT: v_subb_u32_e64 v11, s[6:7], 0, 0, s[6:7]
-; GISEL-NEXT: v_xor_b32_e32 v0, 0x7f, v8
-; GISEL-NEXT: v_cmp_lt_u64_e64 s[8:9], s[8:9], v[8:9]
-; GISEL-NEXT: v_or_b32_e32 v0, v0, v10
-; GISEL-NEXT: v_or_b32_e32 v1, v9, v11
-; GISEL-NEXT: v_cmp_ne_u64_e64 s[6:7], 0, v[0:1]
-; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[8:9]
-; GISEL-NEXT: v_cmp_ne_u64_e64 s[8:9], 0, v[10:11]
+; GISEL-NEXT: v_ffbh_u32_e32 v2, v17
+; GISEL-NEXT: v_min_u32_e32 v1, v2, v1
+; GISEL-NEXT: v_ffbh_u32_e32 v2, v18
+; GISEL-NEXT: v_add_i32_e64 v2, s[6:7], 32, v2
+; GISEL-NEXT: v_add_i32_e64 v1, s[6:7], 64, v1
+; GISEL-NEXT: v_ffbh_u32_e32 v3, v19
+; GISEL-NEXT: v_cmp_eq_u64_e64 s[6:7], 0, v[18:19]
+; GISEL-NEXT: v_min_u32_e32 v2, v3, v2
+; GISEL-NEXT: v_cndmask_b32_e64 v1, v2, v1, s[6:7]
+; GISEL-NEXT: v_sub_i32_e64 v0, s[6:7], v0, v1
+; GISEL-NEXT: v_subb_u32_e64 v1, s[6:7], 0, 0, s[6:7]
+; GISEL-NEXT: v_mov_b32_e32 v8, 0x7f
+; GISEL-NEXT: v_mov_b32_e32 v9, 0
+; GISEL-NEXT: v_subb_u32_e64 v2, s[6:7], 0, 0, s[6:7]
+; GISEL-NEXT: v_cmp_gt_u64_e64 s[8:9], v[0:1], v[8:9]
+; GISEL-NEXT: v_subb_u32_e64 v3, s[6:7], 0, 0, s[6:7]
+; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, s[8:9]
+; GISEL-NEXT: v_cmp_lt_u64_e64 s[8:9], 0, v[2:3]
+; GISEL-NEXT: v_xor_b32_e32 v20, 0x7f, v0
+; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, s[8:9]
+; GISEL-NEXT: v_cmp_eq_u64_e64 s[8:9], 0, v[2:3]
+; GISEL-NEXT: v_or_b32_e32 v20, v20, v2
+; GISEL-NEXT: v_cndmask_b32_e64 v8, v9, v8, s[8:9]
+; GISEL-NEXT: v_or_b32_e32 v21, v1, v3
+; GISEL-NEXT: v_and_b32_e32 v8, 1, v8
+; GISEL-NEXT: v_cmp_eq_u64_e64 s[6:7], 0, v[20:21]
+; GISEL-NEXT: v_cmp_ne_u32_e64 s[8:9], 0, v8
; GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[8:9]
-; GISEL-NEXT: v_cmp_eq_u64_e64 s[8:9], 0, v[10:11]
-; GISEL-NEXT: v_mov_b32_e32 v26, v24
-; GISEL-NEXT: v_cndmask_b32_e64 v0, v1, v0, s[8:9]
-; GISEL-NEXT: v_and_b32_e32 v0, 1, v0
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[8:9], 1, v0
; GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[8:9]
-; GISEL-NEXT: s_xor_b64 s[10:11], s[4:5], -1
-; GISEL-NEXT: v_cndmask_b32_e64 v1, v19, 0, s[4:5]
-; GISEL-NEXT: s_movk_i32 s8, 0x7f
-; GISEL-NEXT: v_cndmask_b32_e64 v0, v18, 0, s[4:5]
-; GISEL-NEXT: s_and_b64 s[10:11], s[10:11], s[6:7]
-; GISEL-NEXT: v_mov_b32_e32 v27, v25
-; GISEL-NEXT: v_cndmask_b32_e64 v20, v17, 0, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e64 v21, v16, 0, s[4:5]
-; GISEL-NEXT: s_and_saveexec_b64 s[6:7], s[10:11]
+; GISEL-NEXT: s_mov_b64 s[10:11], exec
+; GISEL-NEXT: s_or_b64 s[6:7], s[4:5], s[6:7]
+; GISEL-NEXT: s_mov_b64 s[12:13], 0
+; GISEL-NEXT: v_cndmask_b32_e64 v20, v16, 0, s[4:5]
+; GISEL-NEXT: v_cndmask_b32_e64 v8, v18, 0, s[4:5]
+; GISEL-NEXT: v_cndmask_b32_e64 v9, v19, 0, s[4:5]
+; GISEL-NEXT: s_xor_b64 s[8:9], s[6:7], s[10:11]
+; GISEL-NEXT: v_cndmask_b32_e64 v21, v17, 0, s[4:5]
+; GISEL-NEXT: s_and_saveexec_b64 s[6:7], s[8:9]
; GISEL-NEXT: s_cbranch_execz .LBB0_6
; GISEL-NEXT: ; %bb.1: ; %udiv-bb15
-; GISEL-NEXT: v_add_i32_e32 v30, vcc, 1, v8
-; GISEL-NEXT: v_addc_u32_e32 v31, vcc, 0, v9, vcc
-; GISEL-NEXT: v_addc_u32_e32 v32, vcc, 0, v10, vcc
-; GISEL-NEXT: v_addc_u32_e64 v33, s[4:5], 0, v11, vcc
-; GISEL-NEXT: v_sub_i32_e32 v11, vcc, s8, v8
-; GISEL-NEXT: v_sub_i32_e32 v9, vcc, 64, v11
-; GISEL-NEXT: v_lshl_b64 v[0:1], v[18:19], v11
-; GISEL-NEXT: v_lshr_b64 v[9:10], v[16:17], v9
-; GISEL-NEXT: s_xor_b64 s[8:9], s[4:5], -1
-; GISEL-NEXT: v_or_b32_e32 v9, v0, v9
-; GISEL-NEXT: v_sub_i32_e32 v0, vcc, 63, v8
-; GISEL-NEXT: v_or_b32_e32 v10, v1, v10
-; GISEL-NEXT: v_lshl_b64 v[0:1], v[16:17], v0
-; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v11
-; GISEL-NEXT: v_cndmask_b32_e32 v0, v0, v9, vcc
-; GISEL-NEXT: v_lshl_b64 v[8:9], v[16:17], v11
-; GISEL-NEXT: v_cndmask_b32_e32 v1, v1, v10, vcc
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v11
-; GISEL-NEXT: v_cndmask_b32_e64 v1, v1, v19, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e64 v0, v0, v18, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e32 v9, 0, v9, vcc
-; GISEL-NEXT: v_mov_b32_e32 v10, 0
-; GISEL-NEXT: v_mov_b32_e32 v11, 0
-; GISEL-NEXT: v_cndmask_b32_e32 v8, 0, v8, vcc
-; GISEL-NEXT: s_and_saveexec_b64 s[4:5], s[8:9]
-; GISEL-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
+; GISEL-NEXT: v_add_i32_e32 v28, vcc, 1, v0
+; GISEL-NEXT: v_addc_u32_e32 v29, vcc, 0, v1, vcc
+; GISEL-NEXT: v_addc_u32_e32 v30, vcc, 0, v2, vcc
+; GISEL-NEXT: s_mov_b64 s[4:5], exec
+; GISEL-NEXT: v_addc_u32_e32 v31, vcc, 0, v3, vcc
+; GISEL-NEXT: s_xor_b64 s[4:5], vcc, s[4:5]
+; GISEL-NEXT: v_sub_i32_e32 v22, vcc, 0x7f, v0
+; GISEL-NEXT: v_not_b32_e32 v0, 63
+; GISEL-NEXT: v_add_i32_e32 v20, vcc, v22, v0
+; GISEL-NEXT: v_sub_i32_e32 v0, vcc, 64, v22
+; GISEL-NEXT: v_lshr_b64 v[0:1], v[16:17], v0
+; GISEL-NEXT: v_lshl_b64 v[2:3], v[18:19], v22
+; GISEL-NEXT: v_lshl_b64 v[8:9], v[16:17], v22
+; GISEL-NEXT: v_or_b32_e32 v2, v0, v2
+; GISEL-NEXT: v_or_b32_e32 v3, v1, v3
+; GISEL-NEXT: v_lshl_b64 v[0:1], v[16:17], v20
+; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v22
+; GISEL-NEXT: v_cndmask_b32_e32 v20, 0, v8, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v21, 0, v9, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v22
+; GISEL-NEXT: v_cndmask_b32_e32 v8, v0, v18, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v9, v1, v19, vcc
+; GISEL-NEXT: s_mov_b64 s[14:15], s[12:13]
+; GISEL-NEXT: v_mov_b32_e32 v0, s12
+; GISEL-NEXT: v_mov_b32_e32 v1, s13
+; GISEL-NEXT: v_mov_b32_e32 v2, s14
+; GISEL-NEXT: v_mov_b32_e32 v3, s15
+; GISEL-NEXT: s_and_saveexec_b64 s[8:9], s[4:5]
+; GISEL-NEXT: s_xor_b64 s[12:13], exec, s[8:9]
; GISEL-NEXT: s_cbranch_execz .LBB0_5
; GISEL-NEXT: ; %bb.2: ; %udiv-preheader4
-; GISEL-NEXT: v_sub_i32_e32 v20, vcc, 64, v30
-; GISEL-NEXT: v_lshr_b64 v[10:11], v[16:17], v30
-; GISEL-NEXT: v_lshl_b64 v[20:21], v[18:19], v20
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v30
-; GISEL-NEXT: v_or_b32_e32 v20, v10, v20
-; GISEL-NEXT: v_subrev_i32_e32 v10, vcc, 64, v30
-; GISEL-NEXT: v_or_b32_e32 v21, v11, v21
-; GISEL-NEXT: v_lshr_b64 v[10:11], v[18:19], v10
-; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v30
-; GISEL-NEXT: v_cndmask_b32_e32 v11, v11, v21, vcc
-; GISEL-NEXT: v_cndmask_b32_e64 v17, v11, v17, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e32 v20, v10, v20, vcc
-; GISEL-NEXT: v_lshr_b64 v[10:11], v[18:19], v30
-; GISEL-NEXT: v_cndmask_b32_e64 v16, v20, v16, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e32 v21, 0, v11, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v20, 0, v10, vcc
-; GISEL-NEXT: v_add_i32_e32 v34, vcc, -1, v28
-; GISEL-NEXT: v_addc_u32_e32 v35, vcc, -1, v29, vcc
-; GISEL-NEXT: v_addc_u32_e32 v36, vcc, -1, v2, vcc
-; GISEL-NEXT: v_addc_u32_e32 v37, vcc, -1, v3, vcc
-; GISEL-NEXT: v_mov_b32_e32 v18, 0
-; GISEL-NEXT: v_mov_b32_e32 v19, 0
-; GISEL-NEXT: s_mov_b64 s[4:5], 0
-; GISEL-NEXT: v_mov_b32_e32 v22, 0
-; GISEL-NEXT: v_mov_b32_e32 v23, 0
-; GISEL-NEXT: v_mov_b32_e32 v11, 0
+; GISEL-NEXT: v_sub_i32_e32 v2, vcc, 64, v28
+; GISEL-NEXT: v_lshr_b64 v[0:1], v[16:17], v28
+; GISEL-NEXT: v_lshl_b64 v[2:3], v[18:19], v2
+; GISEL-NEXT: v_add_i32_e32 v32, vcc, 0xffffffc0, v28
+; GISEL-NEXT: v_lshr_b64 v[22:23], v[18:19], v28
+; GISEL-NEXT: v_or_b32_e32 v2, v0, v2
+; GISEL-NEXT: v_or_b32_e32 v3, v1, v3
+; GISEL-NEXT: v_lshr_b64 v[0:1], v[18:19], v32
+; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v28
+; GISEL-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v22, 0, v22, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v23, 0, v23, vcc
+; GISEL-NEXT: v_add_i32_e32 v32, vcc, -1, v26
+; GISEL-NEXT: v_addc_u32_e32 v33, vcc, -1, v27, vcc
+; GISEL-NEXT: s_mov_b64 s[8:9], 0
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v28
+; GISEL-NEXT: v_addc_u32_e32 v34, vcc, -1, v10, vcc
+; GISEL-NEXT: v_cndmask_b32_e64 v18, v0, v16, s[4:5]
+; GISEL-NEXT: v_cndmask_b32_e64 v19, v1, v17, s[4:5]
+; GISEL-NEXT: v_addc_u32_e32 v35, vcc, -1, v11, vcc
+; GISEL-NEXT: s_mov_b64 s[10:11], s[8:9]
+; GISEL-NEXT: v_mov_b32_e32 v17, 0
+; GISEL-NEXT: v_mov_b32_e32 v0, s8
+; GISEL-NEXT: v_mov_b32_e32 v1, s9
+; GISEL-NEXT: v_mov_b32_e32 v16, 1
+; GISEL-NEXT: v_mov_b32_e32 v2, s10
+; GISEL-NEXT: v_mov_b32_e32 v3, s11
; GISEL-NEXT: .LBB0_3: ; %udiv-do-while3
; GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GISEL-NEXT: v_lshl_b64 v[38:39], v[8:9], 1
-; GISEL-NEXT: v_lshl_b64 v[20:21], v[20:21], 1
-; GISEL-NEXT: v_or_b32_e32 v8, v22, v38
-; GISEL-NEXT: v_lshrrev_b32_e32 v22, 31, v17
-; GISEL-NEXT: v_lshl_b64 v[16:17], v[16:17], 1
-; GISEL-NEXT: v_or_b32_e32 v20, v20, v22
-; GISEL-NEXT: v_lshrrev_b32_e32 v22, 31, v1
-; GISEL-NEXT: v_or_b32_e32 v16, v16, v22
-; GISEL-NEXT: v_sub_i32_e32 v22, vcc, v34, v16
-; GISEL-NEXT: v_subb_u32_e32 v22, vcc, v35, v17, vcc
-; GISEL-NEXT: v_subb_u32_e32 v22, vcc, v36, v20, vcc
-; GISEL-NEXT: v_subb_u32_e32 v22, vcc, v37, v21, vcc
-; GISEL-NEXT: v_ashrrev_i32_e32 v38, 31, v22
-; GISEL-NEXT: v_lshrrev_b32_e32 v10, 31, v9
-; GISEL-NEXT: v_or_b32_e32 v9, v23, v39
-; GISEL-NEXT: v_and_b32_e32 v23, v38, v28
-; GISEL-NEXT: v_and_b32_e32 v22, v38, v29
-; GISEL-NEXT: v_sub_i32_e32 v16, vcc, v16, v23
-; GISEL-NEXT: v_subb_u32_e32 v17, vcc, v17, v22, vcc
-; GISEL-NEXT: v_and_b32_e32 v23, v38, v2
-; GISEL-NEXT: v_and_b32_e32 v22, v38, v3
-; GISEL-NEXT: v_subb_u32_e32 v20, vcc, v20, v23, vcc
-; GISEL-NEXT: v_subb_u32_e32 v21, vcc, v21, v22, vcc
-; GISEL-NEXT: v_add_i32_e32 v30, vcc, -1, v30
+; GISEL-NEXT: v_lshl_b64 v[2:3], v[20:21], 1
+; GISEL-NEXT: v_lshrrev_b32_e32 v16, 31, v21
+; GISEL-NEXT: v_or_b32_e32 v20, v0, v2
+; GISEL-NEXT: v_or_b32_e32 v21, v1, v3
+; GISEL-NEXT: v_lshl_b64 v[2:3], v[18:19], 1
+; GISEL-NEXT: v_lshrrev_b32_e32 v18, 31, v9
+; GISEL-NEXT: v_lshl_b64 v[0:1], v[22:23], 1
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v18
+; GISEL-NEXT: v_lshrrev_b32_e32 v22, 31, v19
+; GISEL-NEXT: v_sub_i32_e32 v18, vcc, v32, v2
+; GISEL-NEXT: v_or_b32_e32 v0, v0, v22
+; GISEL-NEXT: v_subb_u32_e32 v18, vcc, v33, v3, vcc
+; GISEL-NEXT: v_subb_u32_e32 v18, vcc, v34, v0, vcc
+; GISEL-NEXT: v_subb_u32_e32 v18, vcc, v35, v1, vcc
+; GISEL-NEXT: v_ashrrev_i32_e32 v36, 31, v18
+; GISEL-NEXT: v_and_b32_e32 v18, v36, v26
+; GISEL-NEXT: v_and_b32_e32 v19, v36, v27
+; GISEL-NEXT: v_sub_i32_e32 v18, vcc, v2, v18
+; GISEL-NEXT: v_subb_u32_e32 v19, vcc, v3, v19, vcc
+; GISEL-NEXT: v_and_b32_e32 v2, v36, v10
+; GISEL-NEXT: v_and_b32_e32 v3, v36, v11
+; GISEL-NEXT: v_subb_u32_e32 v22, vcc, v0, v2, vcc
+; GISEL-NEXT: v_subb_u32_e32 v23, vcc, v1, v3, vcc
+; GISEL-NEXT: v_add_i32_e32 v28, vcc, -1, v28
+; GISEL-NEXT: v_addc_u32_e32 v29, vcc, -1, v29, vcc
+; GISEL-NEXT: v_addc_u32_e32 v30, vcc, -1, v30, vcc
; GISEL-NEXT: v_addc_u32_e32 v31, vcc, -1, v31, vcc
-; GISEL-NEXT: v_addc_u32_e32 v32, vcc, -1, v32, vcc
-; GISEL-NEXT: v_addc_u32_e32 v33, vcc, -1, v33, vcc
-; GISEL-NEXT: v_or_b32_e32 v23, v31, v33
-; GISEL-NEXT: v_or_b32_e32 v22, v30, v32
-; GISEL-NEXT: v_lshl_b64 v[0:1], v[0:1], 1
-; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[22:23]
-; GISEL-NEXT: v_or_b32_e32 v0, v0, v10
-; GISEL-NEXT: v_and_b32_e32 v10, 1, v38
-; GISEL-NEXT: v_or_b32_e32 v1, v19, v1
-; GISEL-NEXT: v_or_b32_e32 v0, v18, v0
-; GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GISEL-NEXT: v_mov_b32_e32 v23, v11
-; GISEL-NEXT: v_mov_b32_e32 v22, v10
-; GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GISEL-NEXT: v_or_b32_e32 v0, v28, v30
+; GISEL-NEXT: v_or_b32_e32 v1, v29, v31
+; GISEL-NEXT: v_lshl_b64 v[8:9], v[8:9], 1
+; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
+; GISEL-NEXT: v_or_b32_e32 v8, v8, v16
+; GISEL-NEXT: v_and_b32_e32 v16, 1, v36
+; GISEL-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GISEL-NEXT: v_mov_b32_e32 v0, v16
+; GISEL-NEXT: v_mov_b32_e32 v1, v17
+; GISEL-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GISEL-NEXT: s_cbranch_execnz .LBB0_3
; GISEL-NEXT: ; %bb.4: ; %Flow13
-; GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
-; GISEL-NEXT: .LBB0_5: ; %Flow14
; GISEL-NEXT: s_or_b64 exec, exec, s[8:9]
-; GISEL-NEXT: v_lshl_b64 v[0:1], v[0:1], 1
-; GISEL-NEXT: v_lshl_b64 v[2:3], v[8:9], 1
-; GISEL-NEXT: v_lshrrev_b32_e32 v16, 31, v9
-; GISEL-NEXT: v_or_b32_e32 v0, v0, v16
-; GISEL-NEXT: v_or_b32_e32 v20, v11, v3
-; GISEL-NEXT: v_or_b32_e32 v21, v10, v2
+; GISEL-NEXT: .LBB0_5: ; %Flow14
+; GISEL-NEXT: s_or_b64 exec, exec, s[12:13]
+; GISEL-NEXT: v_lshl_b64 v[2:3], v[20:21], 1
+; GISEL-NEXT: v_lshl_b64 v[8:9], v[8:9], 1
+; GISEL-NEXT: v_lshrrev_b32_e32 v10, 31, v21
+; GISEL-NEXT: v_or_b32_e32 v8, v8, v10
+; GISEL-NEXT: v_or_b32_e32 v20, v0, v2
+; GISEL-NEXT: v_or_b32_e32 v21, v1, v3
; GISEL-NEXT: .LBB0_6: ; %Flow16
; GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
; GISEL-NEXT: v_ashrrev_i32_e32 v18, 31, v7
-; GISEL-NEXT: v_xor_b32_e32 v3, v4, v18
-; GISEL-NEXT: v_xor_b32_e32 v2, v5, v18
-; GISEL-NEXT: v_sub_i32_e32 v8, vcc, v3, v18
-; GISEL-NEXT: v_subb_u32_e32 v9, vcc, v2, v18, vcc
-; GISEL-NEXT: v_xor_b32_e32 v3, v6, v18
+; GISEL-NEXT: v_xor_b32_e32 v0, v18, v4
+; GISEL-NEXT: v_xor_b32_e32 v1, v18, v5
+; GISEL-NEXT: v_xor_b32_e32 v2, v18, v6
+; GISEL-NEXT: v_sub_i32_e32 v6, vcc, v0, v18
+; GISEL-NEXT: v_xor_b32_e32 v3, v18, v7
+; GISEL-NEXT: v_subb_u32_e32 v7, vcc, v1, v18, vcc
; GISEL-NEXT: v_ashrrev_i32_e32 v19, 31, v15
-; GISEL-NEXT: v_xor_b32_e32 v2, v7, v18
-; GISEL-NEXT: v_subb_u32_e32 v6, vcc, v3, v18, vcc
-; GISEL-NEXT: v_subb_u32_e32 v7, vcc, v2, v18, vcc
-; GISEL-NEXT: v_xor_b32_e32 v3, v12, v19
-; GISEL-NEXT: v_xor_b32_e32 v2, v13, v19
-; GISEL-NEXT: v_sub_i32_e32 v28, vcc, v3, v19
-; GISEL-NEXT: v_subb_u32_e32 v29, vcc, v2, v19, vcc
-; GISEL-NEXT: v_xor_b32_e32 v2, v14, v19
-; GISEL-NEXT: v_xor_b32_e32 v3, v15, v19
-; GISEL-NEXT: v_subb_u32_e32 v2, vcc, v2, v19, vcc
-; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v3, v19, vcc
-; GISEL-NEXT: v_or_b32_e32 v5, v29, v3
-; GISEL-NEXT: v_or_b32_e32 v4, v28, v2
+; GISEL-NEXT: v_subb_u32_e32 v10, vcc, v2, v18, vcc
+; GISEL-NEXT: v_subb_u32_e32 v11, vcc, v3, v18, vcc
+; GISEL-NEXT: v_xor_b32_e32 v0, v19, v12
+; GISEL-NEXT: v_xor_b32_e32 v1, v19, v13
+; GISEL-NEXT: v_sub_i32_e32 v22, vcc, v0, v19
+; GISEL-NEXT: v_xor_b32_e32 v2, v19, v14
+; GISEL-NEXT: v_subb_u32_e32 v23, vcc, v1, v19, vcc
+; GISEL-NEXT: v_xor_b32_e32 v3, v19, v15
+; GISEL-NEXT: v_subb_u32_e32 v4, vcc, v2, v19, vcc
+; GISEL-NEXT: v_subb_u32_e32 v5, vcc, v3, v19, vcc
+; GISEL-NEXT: v_or_b32_e32 v0, v22, v4
+; GISEL-NEXT: v_or_b32_e32 v1, v23, v5
+; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
+; GISEL-NEXT: v_or_b32_e32 v0, v6, v10
+; GISEL-NEXT: v_or_b32_e32 v1, v7, v11
+; GISEL-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[0:1]
+; GISEL-NEXT: v_ffbh_u32_e32 v1, v22
+; GISEL-NEXT: s_or_b64 s[10:11], vcc, s[4:5]
+; GISEL-NEXT: v_ffbh_u32_e32 v0, v23
+; GISEL-NEXT: v_add_i32_e32 v1, vcc, 32, v1
+; GISEL-NEXT: v_min_u32_e32 v0, v0, v1
+; GISEL-NEXT: v_ffbh_u32_e32 v2, v4
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, 64, v0
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, 32, v2
+; GISEL-NEXT: v_ffbh_u32_e32 v1, v5
; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[4:5]
-; GISEL-NEXT: v_or_b32_e32 v5, v9, v7
-; GISEL-NEXT: v_or_b32_e32 v4, v8, v6
-; GISEL-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[4:5]
-; GISEL-NEXT: v_ffbh_u32_e32 v4, v2
-; GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GISEL-NEXT: v_add_i32_e32 v4, vcc, 32, v4
-; GISEL-NEXT: v_ffbh_u32_e32 v5, v3
-; GISEL-NEXT: v_min_u32_e32 v4, v4, v5
-; GISEL-NEXT: v_ffbh_u32_e32 v5, v28
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, 32, v5
-; GISEL-NEXT: v_ffbh_u32_e32 v10, v29
-; GISEL-NEXT: v_min_u32_e32 v5, v5, v10
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, 64, v5
-; GISEL-NEXT: v_addc_u32_e64 v10, s[6:7], 0, 0, vcc
-; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
-; GISEL-NEXT: v_ffbh_u32_e32 v11, v7
-; GISEL-NEXT: v_cndmask_b32_e32 v4, v5, v4, vcc
-; GISEL-NEXT: v_ffbh_u32_e32 v5, v6
-; GISEL-NEXT: v_cndmask_b32_e64 v10, v10, 0, vcc
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, 32, v5
-; GISEL-NEXT: v_min_u32_e32 v5, v5, v11
-; GISEL-NEXT: v_ffbh_u32_e32 v11, v8
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, 32, v11
-; GISEL-NEXT: v_ffbh_u32_e32 v12, v9
-; GISEL-NEXT: v_min_u32_e32 v11, v11, v12
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, 64, v11
-; GISEL-NEXT: v_addc_u32_e64 v12, s[6:7], 0, 0, vcc
-; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
-; GISEL-NEXT: v_mov_b32_e32 v22, v18
-; GISEL-NEXT: v_cndmask_b32_e32 v5, v11, v5, vcc
-; GISEL-NEXT: v_cndmask_b32_e64 v12, v12, 0, vcc
-; GISEL-NEXT: v_sub_i32_e32 v4, vcc, v4, v5
-; GISEL-NEXT: v_subb_u32_e32 v5, vcc, v10, v12, vcc
-; GISEL-NEXT: v_subb_u32_e64 v12, s[6:7], 0, 0, vcc
-; GISEL-NEXT: v_subb_u32_e64 v13, s[6:7], 0, 0, s[6:7]
-; GISEL-NEXT: s_mov_b64 s[6:7], 0x7f
-; GISEL-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[4:5]
-; GISEL-NEXT: v_or_b32_e32 v15, v5, v13
-; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[12:13]
-; GISEL-NEXT: v_mov_b32_e32 v23, v19
-; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[12:13]
-; GISEL-NEXT: s_movk_i32 s8, 0x7f
-; GISEL-NEXT: v_cndmask_b32_e32 v10, v11, v10, vcc
-; GISEL-NEXT: v_and_b32_e32 v10, 1, v10
-; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 1, v10
-; GISEL-NEXT: v_xor_b32_e32 v10, 0x7f, v4
-; GISEL-NEXT: v_or_b32_e32 v14, v10, v12
-; GISEL-NEXT: s_or_b64 s[4:5], s[4:5], vcc
-; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[14:15]
-; GISEL-NEXT: s_xor_b64 s[6:7], s[4:5], -1
-; GISEL-NEXT: v_cndmask_b32_e64 v11, v7, 0, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e64 v10, v6, 0, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e64 v14, v9, 0, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e64 v15, v8, 0, s[4:5]
-; GISEL-NEXT: s_and_b64 s[4:5], s[6:7], vcc
+; GISEL-NEXT: v_min_u32_e32 v1, v1, v2
+; GISEL-NEXT: v_ffbh_u32_e32 v2, v6
+; GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
+; GISEL-NEXT: v_ffbh_u32_e32 v1, v7
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, 32, v2
+; GISEL-NEXT: v_min_u32_e32 v1, v1, v2
+; GISEL-NEXT: v_ffbh_u32_e32 v3, v10
+; GISEL-NEXT: v_add_i32_e32 v1, vcc, 64, v1
+; GISEL-NEXT: v_add_i32_e32 v3, vcc, 32, v3
+; GISEL-NEXT: v_ffbh_u32_e32 v2, v11
+; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[10:11]
+; GISEL-NEXT: v_min_u32_e32 v2, v2, v3
+; GISEL-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
+; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v0, v1
+; GISEL-NEXT: v_subb_u32_e64 v1, s[4:5], 0, 0, vcc
+; GISEL-NEXT: v_mov_b32_e32 v12, 0x7f
+; GISEL-NEXT: v_mov_b32_e32 v13, 0
+; GISEL-NEXT: v_subb_u32_e64 v2, s[4:5], 0, 0, s[4:5]
+; GISEL-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[12:13]
+; GISEL-NEXT: v_subb_u32_e64 v3, s[4:5], 0, 0, s[4:5]
+; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT: v_cmp_lt_u64_e32 vcc, 0, v[2:3]
+; GISEL-NEXT: s_mov_b64 s[6:7], exec
+; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
+; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3]
+; GISEL-NEXT: s_mov_b64 s[8:9], 0
+; GISEL-NEXT: v_cndmask_b32_e32 v12, v13, v12, vcc
+; GISEL-NEXT: v_and_b32_e32 v12, 1, v12
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v12
+; GISEL-NEXT: v_xor_b32_e32 v12, 0x7f, v0
+; GISEL-NEXT: v_or_b32_e32 v12, v12, v2
+; GISEL-NEXT: v_or_b32_e32 v13, v1, v3
+; GISEL-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[12:13]
+; GISEL-NEXT: s_or_b64 s[10:11], s[10:11], vcc
+; GISEL-NEXT: s_or_b64 s[4:5], s[10:11], s[4:5]
+; GISEL-NEXT: v_cndmask_b32_e64 v12, v6, 0, s[10:11]
+; GISEL-NEXT: v_cndmask_b32_e64 v13, v7, 0, s[10:11]
+; GISEL-NEXT: v_cndmask_b32_e64 v14, v10, 0, s[10:11]
+; GISEL-NEXT: v_cndmask_b32_e64 v15, v11, 0, s[10:11]
+; GISEL-NEXT: s_xor_b64 s[4:5], s[4:5], s[6:7]
; GISEL-NEXT: s_and_saveexec_b64 s[6:7], s[4:5]
; GISEL-NEXT: s_cbranch_execz .LBB0_12
; GISEL-NEXT: ; %bb.7: ; %udiv-bb1
-; GISEL-NEXT: v_add_i32_e32 v30, vcc, 1, v4
-; GISEL-NEXT: v_addc_u32_e32 v31, vcc, 0, v5, vcc
-; GISEL-NEXT: v_addc_u32_e32 v32, vcc, 0, v12, vcc
-; GISEL-NEXT: v_addc_u32_e64 v33, s[4:5], 0, v13, vcc
-; GISEL-NEXT: v_sub_i32_e32 v14, vcc, s8, v4
-; GISEL-NEXT: v_sub_i32_e32 v5, vcc, 64, v14
-; GISEL-NEXT: v_lshl_b64 v[10:11], v[6:7], v14
-; GISEL-NEXT: v_lshr_b64 v[12:13], v[8:9], v5
-; GISEL-NEXT: v_sub_i32_e32 v4, vcc, 63, v4
-; GISEL-NEXT: v_lshl_b64 v[4:5], v[8:9], v4
-; GISEL-NEXT: v_or_b32_e32 v11, v11, v13
+; GISEL-NEXT: v_add_i32_e32 v26, vcc, 1, v0
+; GISEL-NEXT: v_addc_u32_e32 v27, vcc, 0, v1, vcc
+; GISEL-NEXT: v_addc_u32_e32 v28, vcc, 0, v2, vcc
+; GISEL-NEXT: s_mov_b64 s[4:5], exec
+; GISEL-NEXT: v_addc_u32_e32 v29, vcc, 0, v3, vcc
+; GISEL-NEXT: s_xor_b64 s[4:5], vcc, s[4:5]
+; GISEL-NEXT: v_sub_i32_e32 v14, vcc, 0x7f, v0
+; GISEL-NEXT: v_not_b32_e32 v0, 63
+; GISEL-NEXT: v_add_i32_e32 v15, vcc, v14, v0
+; GISEL-NEXT: v_sub_i32_e32 v0, vcc, 64, v14
+; GISEL-NEXT: v_lshr_b64 v[0:1], v[6:7], v0
+; GISEL-NEXT: v_lshl_b64 v[2:3], v[10:11], v14
+; GISEL-NEXT: v_lshl_b64 v[12:13], v[6:7], v14
+; GISEL-NEXT: v_or_b32_e32 v2, v0, v2
+; GISEL-NEXT: v_or_b32_e32 v3, v1, v3
+; GISEL-NEXT: v_lshl_b64 v[0:1], v[6:7], v15
; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v14
-; GISEL-NEXT: s_xor_b64 s[8:9], s[4:5], -1
-; GISEL-NEXT: v_or_b32_e32 v10, v10, v12
-; GISEL-NEXT: v_cndmask_b32_e32 v5, v5, v11, vcc
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v14
-; GISEL-NEXT: v_cndmask_b32_e64 v11, v5, v7, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e32 v10, v4, v10, vcc
-; GISEL-NEXT: v_lshl_b64 v[4:5], v[8:9], v14
-; GISEL-NEXT: v_cndmask_b32_e64 v10, v10, v6, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc
-; GISEL-NEXT: v_mov_b32_e32 v12, 0
-; GISEL-NEXT: v_mov_b32_e32 v13, 0
-; GISEL-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
-; GISEL-NEXT: s_and_saveexec_b64 s[4:5], s[8:9]
-; GISEL-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
+; GISEL-NEXT: v_cndmask_b32_e32 v12, 0, v12, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v13, 0, v13, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v14
+; GISEL-NEXT: v_cndmask_b32_e32 v14, v0, v10, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v15, v1, v11, vcc
+; GISEL-NEXT: s_mov_b64 s[10:11], s[8:9]
+; GISEL-NEXT: v_mov_b32_e32 v0, s8
+; GISEL-NEXT: v_mov_b32_e32 v1, s9
+; GISEL-NEXT: v_mov_b32_e32 v2, s10
+; GISEL-NEXT: v_mov_b32_e32 v3, s11
+; GISEL-NEXT: s_and_saveexec_b64 s[8:9], s[4:5]
+; GISEL-NEXT: s_xor_b64 s[12:13], exec, s[8:9]
; GISEL-NEXT: s_cbranch_execz .LBB0_11
; GISEL-NEXT: ; %bb.8: ; %udiv-preheader
-; GISEL-NEXT: v_sub_i32_e32 v14, vcc, 64, v30
-; GISEL-NEXT: v_lshr_b64 v[12:13], v[8:9], v30
-; GISEL-NEXT: v_lshl_b64 v[14:15], v[6:7], v14
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v30
-; GISEL-NEXT: v_or_b32_e32 v14, v12, v14
-; GISEL-NEXT: v_subrev_i32_e32 v12, vcc, 64, v30
-; GISEL-NEXT: v_or_b32_e32 v15, v13, v15
-; GISEL-NEXT: v_lshr_b64 v[12:13], v[6:7], v12
-; GISEL-NEXT: v_lshr_b64 v[6:7], v[6:7], v30
-; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v30
-; GISEL-NEXT: v_cndmask_b32_e32 v13, v13, v15, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v12, v12, v14, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v17, 0, v7, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v16, 0, v6, vcc
-; GISEL-NEXT: v_add_i32_e32 v34, vcc, -1, v28
-; GISEL-NEXT: v_addc_u32_e32 v35, vcc, -1, v29, vcc
-; GISEL-NEXT: v_addc_u32_e32 v36, vcc, -1, v2, vcc
-; GISEL-NEXT: v_cndmask_b32_e64 v9, v13, v9, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e64 v8, v12, v8, s[4:5]
-; GISEL-NEXT: v_addc_u32_e32 v37, vcc, -1, v3, vcc
-; GISEL-NEXT: v_mov_b32_e32 v6, 0
+; GISEL-NEXT: v_sub_i32_e32 v2, vcc, 64, v26
+; GISEL-NEXT: v_lshr_b64 v[0:1], v[6:7], v26
+; GISEL-NEXT: v_lshl_b64 v[2:3], v[10:11], v2
+; GISEL-NEXT: v_add_i32_e32 v30, vcc, 0xffffffc0, v26
+; GISEL-NEXT: v_lshr_b64 v[16:17], v[10:11], v26
+; GISEL-NEXT: v_or_b32_e32 v2, v0, v2
+; GISEL-NEXT: v_or_b32_e32 v3, v1, v3
+; GISEL-NEXT: v_lshr_b64 v[0:1], v[10:11], v30
+; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v26
+; GISEL-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v16, 0, v16, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v17, 0, v17, vcc
+; GISEL-NEXT: v_add_i32_e32 v30, vcc, -1, v22
+; GISEL-NEXT: v_addc_u32_e32 v31, vcc, -1, v23, vcc
+; GISEL-NEXT: s_mov_b64 s[8:9], 0
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v26
+; GISEL-NEXT: v_addc_u32_e32 v32, vcc, -1, v4, vcc
+; GISEL-NEXT: v_cndmask_b32_e64 v10, v0, v6, s[4:5]
+; GISEL-NEXT: v_cndmask_b32_e64 v11, v1, v7, s[4:5]
+; GISEL-NEXT: v_addc_u32_e32 v33, vcc, -1, v5, vcc
+; GISEL-NEXT: s_mov_b64 s[10:11], s[8:9]
; GISEL-NEXT: v_mov_b32_e32 v7, 0
-; GISEL-NEXT: s_mov_b64 s[10:11], 0
-; GISEL-NEXT: v_mov_b32_e32 v14, 0
-; GISEL-NEXT: v_mov_b32_e32 v15, 0
-; GISEL-NEXT: v_mov_b32_e32 v13, 0
+; GISEL-NEXT: v_mov_b32_e32 v0, s8
+; GISEL-NEXT: v_mov_b32_e32 v1, s9
+; GISEL-NEXT: v_mov_b32_e32 v6, 1
+; GISEL-NEXT: v_mov_b32_e32 v2, s10
+; GISEL-NEXT: v_mov_b32_e32 v3, s11
; GISEL-NEXT: .LBB0_9: ; %udiv-do-while
; GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GISEL-NEXT: v_lshl_b64 v[38:39], v[4:5], 1
-; GISEL-NEXT: v_add_i32_e32 v30, vcc, -1, v30
-; GISEL-NEXT: v_or_b32_e32 v4, v14, v38
-; GISEL-NEXT: v_lshrrev_b32_e32 v14, 31, v9
-; GISEL-NEXT: v_lshl_b64 v[8:9], v[8:9], 1
-; GISEL-NEXT: v_addc_u32_e32 v31, vcc, -1, v31, vcc
-; GISEL-NEXT: v_lshrrev_b32_e32 v12, 31, v5
-; GISEL-NEXT: v_or_b32_e32 v5, v15, v39
-; GISEL-NEXT: v_lshrrev_b32_e32 v15, 31, v11
-; GISEL-NEXT: v_lshl_b64 v[10:11], v[10:11], 1
-; GISEL-NEXT: v_addc_u32_e32 v32, vcc, -1, v32, vcc
; GISEL-NEXT: v_lshl_b64 v[16:17], v[16:17], 1
-; GISEL-NEXT: v_or_b32_e32 v8, v8, v15
-; GISEL-NEXT: v_addc_u32_e32 v33, vcc, -1, v33, vcc
-; GISEL-NEXT: v_or_b32_e32 v10, v10, v12
-; GISEL-NEXT: v_sub_i32_e32 v12, vcc, v34, v8
-; GISEL-NEXT: v_or_b32_e32 v16, v16, v14
-; GISEL-NEXT: v_subb_u32_e32 v12, vcc, v35, v9, vcc
-; GISEL-NEXT: v_subb_u32_e32 v12, vcc, v36, v16, vcc
-; GISEL-NEXT: v_or_b32_e32 v14, v30, v32
-; GISEL-NEXT: v_or_b32_e32 v15, v31, v33
-; GISEL-NEXT: v_subb_u32_e32 v12, vcc, v37, v17, vcc
-; GISEL-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[14:15]
-; GISEL-NEXT: v_ashrrev_i32_e32 v14, 31, v12
-; GISEL-NEXT: v_and_b32_e32 v38, v14, v28
-; GISEL-NEXT: v_and_b32_e32 v15, v14, v29
-; GISEL-NEXT: v_sub_i32_e32 v8, vcc, v8, v38
-; GISEL-NEXT: v_subb_u32_e32 v9, vcc, v9, v15, vcc
-; GISEL-NEXT: v_and_b32_e32 v39, v14, v2
-; GISEL-NEXT: v_and_b32_e32 v12, 1, v14
-; GISEL-NEXT: v_and_b32_e32 v38, v14, v3
-; GISEL-NEXT: v_subb_u32_e32 v16, vcc, v16, v39, vcc
-; GISEL-NEXT: v_or_b32_e32 v11, v7, v11
-; GISEL-NEXT: v_or_b32_e32 v10, v6, v10
-; GISEL-NEXT: s_or_b64 s[10:11], s[4:5], s[10:11]
-; GISEL-NEXT: v_mov_b32_e32 v15, v13
-; GISEL-NEXT: v_mov_b32_e32 v14, v12
-; GISEL-NEXT: v_subb_u32_e32 v17, vcc, v17, v38, vcc
-; GISEL-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GISEL-NEXT: v_lshl_b64 v[2:3], v[10:11], 1
+; GISEL-NEXT: v_lshrrev_b32_e32 v6, 31, v11
+; GISEL-NEXT: v_or_b32_e32 v16, v16, v6
+; GISEL-NEXT: v_lshrrev_b32_e32 v6, 31, v15
+; GISEL-NEXT: v_lshl_b64 v[10:11], v[12:13], 1
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v6
+; GISEL-NEXT: v_or_b32_e32 v12, v0, v10
+; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v30, v2
+; GISEL-NEXT: v_subb_u32_e32 v0, vcc, v31, v3, vcc
+; GISEL-NEXT: v_subb_u32_e32 v0, vcc, v32, v16, vcc
+; GISEL-NEXT: v_subb_u32_e32 v0, vcc, v33, v17, vcc
+; GISEL-NEXT: v_add_i32_e32 v26, vcc, -1, v26
+; GISEL-NEXT: v_lshl_b64 v[14:15], v[14:15], 1
+; GISEL-NEXT: v_addc_u32_e32 v27, vcc, -1, v27, vcc
+; GISEL-NEXT: v_lshrrev_b32_e32 v6, 31, v13
+; GISEL-NEXT: v_ashrrev_i32_e32 v34, 31, v0
+; GISEL-NEXT: v_addc_u32_e32 v28, vcc, -1, v28, vcc
+; GISEL-NEXT: v_or_b32_e32 v14, v14, v6
+; GISEL-NEXT: v_and_b32_e32 v6, 1, v34
+; GISEL-NEXT: v_addc_u32_e32 v29, vcc, -1, v29, vcc
+; GISEL-NEXT: v_or_b32_e32 v13, v1, v11
+; GISEL-NEXT: v_mov_b32_e32 v0, v6
+; GISEL-NEXT: v_mov_b32_e32 v1, v7
+; GISEL-NEXT: v_and_b32_e32 v6, v34, v22
+; GISEL-NEXT: v_or_b32_e32 v10, v26, v28
+; GISEL-NEXT: v_or_b32_e32 v11, v27, v29
+; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[10:11]
+; GISEL-NEXT: v_and_b32_e32 v11, v34, v23
+; GISEL-NEXT: v_sub_i32_e64 v10, s[4:5], v2, v6
+; GISEL-NEXT: v_and_b32_e32 v35, v34, v4
+; GISEL-NEXT: v_subb_u32_e64 v11, s[4:5], v3, v11, s[4:5]
+; GISEL-NEXT: v_and_b32_e32 v34, v34, v5
+; GISEL-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GISEL-NEXT: v_subb_u32_e64 v16, vcc, v16, v35, s[4:5]
+; GISEL-NEXT: v_subb_u32_e32 v17, vcc, v17, v34, vcc
+; GISEL-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GISEL-NEXT: s_cbranch_execnz .LBB0_9
; GISEL-NEXT: ; %bb.10: ; %Flow
-; GISEL-NEXT: s_or_b64 exec, exec, s[10:11]
-; GISEL-NEXT: .LBB0_11: ; %Flow11
; GISEL-NEXT: s_or_b64 exec, exec, s[8:9]
-; GISEL-NEXT: v_lshl_b64 v[10:11], v[10:11], 1
-; GISEL-NEXT: v_lshl_b64 v[2:3], v[4:5], 1
-; GISEL-NEXT: v_lshrrev_b32_e32 v6, 31, v5
-; GISEL-NEXT: v_or_b32_e32 v10, v10, v6
-; GISEL-NEXT: v_or_b32_e32 v14, v13, v3
-; GISEL-NEXT: v_or_b32_e32 v15, v12, v2
+; GISEL-NEXT: .LBB0_11: ; %Flow11
+; GISEL-NEXT: s_or_b64 exec, exec, s[12:13]
+; GISEL-NEXT: v_lshl_b64 v[2:3], v[12:13], 1
+; GISEL-NEXT: v_lshl_b64 v[14:15], v[14:15], 1
+; GISEL-NEXT: v_lshrrev_b32_e32 v4, 31, v13
+; GISEL-NEXT: v_or_b32_e32 v14, v14, v4
+; GISEL-NEXT: v_or_b32_e32 v12, v0, v2
+; GISEL-NEXT: v_or_b32_e32 v13, v1, v3
; GISEL-NEXT: .LBB0_12: ; %Flow12
; GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
-; GISEL-NEXT: v_xor_b32_e32 v2, v25, v24
-; GISEL-NEXT: v_xor_b32_e32 v3, v27, v26
-; GISEL-NEXT: v_xor_b32_e32 v5, v0, v2
-; GISEL-NEXT: v_xor_b32_e32 v0, v21, v2
-; GISEL-NEXT: v_xor_b32_e32 v4, v1, v3
-; GISEL-NEXT: v_xor_b32_e32 v1, v20, v3
-; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v0, v2
+; GISEL-NEXT: v_xor_b32_e32 v3, v25, v24
+; GISEL-NEXT: v_xor_b32_e32 v0, v20, v3
+; GISEL-NEXT: v_xor_b32_e32 v1, v21, v3
+; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v0, v3
+; GISEL-NEXT: v_xor_b32_e32 v2, v8, v3
; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v1, v3, vcc
-; GISEL-NEXT: v_subb_u32_e32 v2, vcc, v5, v2, vcc
-; GISEL-NEXT: v_xor_b32_e32 v6, v19, v18
+; GISEL-NEXT: v_xor_b32_e32 v4, v9, v3
+; GISEL-NEXT: v_subb_u32_e32 v2, vcc, v2, v3, vcc
+; GISEL-NEXT: v_xor_b32_e32 v7, v19, v18
; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v4, v3, vcc
-; GISEL-NEXT: v_xor_b32_e32 v7, v23, v22
-; GISEL-NEXT: v_xor_b32_e32 v4, v15, v6
-; GISEL-NEXT: v_xor_b32_e32 v5, v14, v7
-; GISEL-NEXT: v_sub_i32_e32 v4, vcc, v4, v6
-; GISEL-NEXT: v_xor_b32_e32 v9, v10, v6
+; GISEL-NEXT: v_xor_b32_e32 v4, v12, v7
+; GISEL-NEXT: v_xor_b32_e32 v5, v13, v7
+; GISEL-NEXT: v_sub_i32_e32 v4, vcc, v4, v7
+; GISEL-NEXT: v_xor_b32_e32 v6, v14, v7
; GISEL-NEXT: v_subb_u32_e32 v5, vcc, v5, v7, vcc
-; GISEL-NEXT: v_xor_b32_e32 v8, v11, v7
-; GISEL-NEXT: v_subb_u32_e32 v6, vcc, v9, v6, vcc
+; GISEL-NEXT: v_xor_b32_e32 v8, v15, v7
+; GISEL-NEXT: v_subb_u32_e32 v6, vcc, v6, v7, vcc
; GISEL-NEXT: v_subb_u32_e32 v7, vcc, v8, v7, vcc
; GISEL-NEXT: s_setpc_b64 s[30:31]
%shl = sdiv <2 x i128> %lhs, %rhs
@@ -847,24 +847,26 @@ define <2 x i128> @v_sdiv_v2i128_v_pow2k(<2 x i128> %lhs) {
; GISEL-NEXT: v_mov_b32_e32 v9, v8
; GISEL-NEXT: v_lshr_b64 v[8:9], v[8:9], 31
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v8
+; GISEL-NEXT: v_addc_u32_e32 v8, vcc, v1, v9, vcc
+; GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v2, vcc
+; GISEL-NEXT: v_addc_u32_e32 v2, vcc, 0, v3, vcc
+; GISEL-NEXT: v_lshrrev_b32_e32 v3, 1, v8
; GISEL-NEXT: v_ashrrev_i32_e32 v8, 31, v7
-; GISEL-NEXT: v_addc_u32_e32 v0, vcc, v1, v9, vcc
; GISEL-NEXT: v_mov_b32_e32 v9, v8
-; GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v2, vcc
; GISEL-NEXT: v_lshr_b64 v[8:9], v[8:9], 31
-; GISEL-NEXT: v_addc_u32_e32 v10, vcc, 0, v3, vcc
+; GISEL-NEXT: v_lshl_b64 v[0:1], v[1:2], 31
; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v8
-; GISEL-NEXT: v_addc_u32_e32 v4, vcc, v5, v9, vcc
+; GISEL-NEXT: v_addc_u32_e32 v8, vcc, v5, v9, vcc
; GISEL-NEXT: v_addc_u32_e32 v5, vcc, 0, v6, vcc
-; GISEL-NEXT: v_addc_u32_e32 v8, vcc, 0, v7, vcc
-; GISEL-NEXT: v_alignbit_b32 v0, v1, v0, 1
-; GISEL-NEXT: v_ashrrev_i32_e32 v2, 1, v10
-; GISEL-NEXT: v_ashrrev_i32_e32 v3, 31, v10
-; GISEL-NEXT: v_alignbit_b32 v4, v5, v4, 1
-; GISEL-NEXT: v_ashrrev_i32_e32 v6, 1, v8
-; GISEL-NEXT: v_ashrrev_i32_e32 v7, 31, v8
-; GISEL-NEXT: v_alignbit_b32 v1, v10, v1, 1
-; GISEL-NEXT: v_alignbit_b32 v5, v8, v5, 1
+; GISEL-NEXT: v_addc_u32_e32 v6, vcc, 0, v7, vcc
+; GISEL-NEXT: v_lshl_b64 v[4:5], v[5:6], 31
+; GISEL-NEXT: v_lshrrev_b32_e32 v7, 1, v8
+; GISEL-NEXT: v_or_b32_e32 v0, v0, v3
+; GISEL-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GISEL-NEXT: v_ashrrev_i32_e32 v2, 1, v2
+; GISEL-NEXT: v_or_b32_e32 v4, v4, v7
+; GISEL-NEXT: v_ashrrev_i32_e32 v7, 31, v6
+; GISEL-NEXT: v_ashrrev_i32_e32 v6, 1, v6
; GISEL-NEXT: s_setpc_b64 s[30:31]
%shl = sdiv <2 x i128> %lhs, <i128 8589934592, i128 8589934592>
ret <2 x i128> %shl
@@ -1225,350 +1227,356 @@ define <2 x i128> @v_udiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-LABEL: v_udiv_v2i128_vv:
; GISEL: ; %bb.0: ; %_udiv-special-cases_udiv-special-cases
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT: v_mov_b32_e32 v19, v3
-; GISEL-NEXT: v_mov_b32_e32 v18, v2
-; GISEL-NEXT: v_or_b32_e32 v3, v9, v11
+; GISEL-NEXT: v_mov_b32_e32 v16, v2
+; GISEL-NEXT: v_mov_b32_e32 v17, v3
; GISEL-NEXT: v_or_b32_e32 v2, v8, v10
+; GISEL-NEXT: v_or_b32_e32 v3, v9, v11
; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3]
-; GISEL-NEXT: v_or_b32_e32 v3, v1, v19
-; GISEL-NEXT: v_or_b32_e32 v2, v0, v18
+; GISEL-NEXT: v_or_b32_e32 v2, v0, v16
+; GISEL-NEXT: v_or_b32_e32 v3, v1, v17
; GISEL-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[2:3]
-; GISEL-NEXT: v_ffbh_u32_e32 v2, v10
-; GISEL-NEXT: v_add_i32_e64 v2, s[6:7], 32, v2
-; GISEL-NEXT: v_ffbh_u32_e32 v3, v11
-; GISEL-NEXT: v_min_u32_e32 v2, v2, v3
; GISEL-NEXT: v_ffbh_u32_e32 v3, v8
+; GISEL-NEXT: v_ffbh_u32_e32 v2, v9
; GISEL-NEXT: v_add_i32_e64 v3, s[6:7], 32, v3
-; GISEL-NEXT: v_ffbh_u32_e32 v16, v9
-; GISEL-NEXT: v_min_u32_e32 v3, v3, v16
-; GISEL-NEXT: v_add_i32_e64 v3, s[6:7], 64, v3
-; GISEL-NEXT: v_addc_u32_e64 v16, s[6:7], 0, 0, s[6:7]
-; GISEL-NEXT: v_cmp_ne_u64_e64 s[6:7], 0, v[10:11]
-; GISEL-NEXT: v_ffbh_u32_e32 v17, v19
-; GISEL-NEXT: v_cndmask_b32_e64 v2, v3, v2, s[6:7]
-; GISEL-NEXT: v_ffbh_u32_e32 v3, v18
-; GISEL-NEXT: v_cndmask_b32_e64 v16, v16, 0, s[6:7]
+; GISEL-NEXT: v_min_u32_e32 v2, v2, v3
+; GISEL-NEXT: v_ffbh_u32_e32 v18, v10
+; GISEL-NEXT: v_add_i32_e64 v2, s[6:7], 64, v2
+; GISEL-NEXT: v_add_i32_e64 v18, s[6:7], 32, v18
+; GISEL-NEXT: v_ffbh_u32_e32 v3, v11
+; GISEL-NEXT: v_cmp_eq_u64_e64 s[6:7], 0, v[10:11]
+; GISEL-NEXT: v_min_u32_e32 v3, v3, v18
+; GISEL-NEXT: v_cndmask_b32_e64 v18, v3, v2, s[6:7]
+; GISEL-NEXT: v_ffbh_u32_e32 v3, v0
+; GISEL-NEXT: v_ffbh_u32_e32 v2, v1
; GISEL-NEXT: v_add_i32_e64 v3, s[6:7], 32, v3
-; GISEL-NEXT: v_min_u32_e32 v3, v3, v17
-; GISEL-NEXT: v_ffbh_u32_e32 v17, v0
-; GISEL-NEXT: v_add_i32_e64 v17, s[6:7], 32, v17
-; GISEL-NEXT: v_ffbh_u32_e32 v20, v1
-; GISEL-NEXT: v_min_u32_e32 v17, v17, v20
-; GISEL-NEXT: v_add_i32_e64 v17, s[6:7], 64, v17
-; GISEL-NEXT: v_addc_u32_e64 v20, s[6:7], 0, 0, s[6:7]
-; GISEL-NEXT: v_cmp_ne_u64_e64 s[6:7], 0, v[18:19]
-; GISEL-NEXT: s_mov_b64 s[8:9], 0x7f
-; GISEL-NEXT: v_cndmask_b32_e64 v3, v17, v3, s[6:7]
-; GISEL-NEXT: v_cndmask_b32_e64 v21, v20, 0, s[6:7]
-; GISEL-NEXT: v_sub_i32_e64 v20, s[6:7], v2, v3
-; GISEL-NEXT: v_subb_u32_e64 v21, s[6:7], v16, v21, s[6:7]
+; GISEL-NEXT: v_min_u32_e32 v2, v2, v3
+; GISEL-NEXT: v_ffbh_u32_e32 v19, v16
+; GISEL-NEXT: v_add_i32_e64 v2, s[6:7], 64, v2
+; GISEL-NEXT: v_add_i32_e64 v19, s[6:7], 32, v19
+; GISEL-NEXT: v_ffbh_u32_e32 v3, v17
+; GISEL-NEXT: v_cmp_eq_u64_e64 s[6:7], 0, v[16:17]
+; GISEL-NEXT: v_min_u32_e32 v3, v3, v19
+; GISEL-NEXT: v_cndmask_b32_e64 v19, v3, v2, s[6:7]
+; GISEL-NEXT: v_sub_i32_e64 v20, s[6:7], v18, v19
+; GISEL-NEXT: v_mov_b32_e32 v2, 0x7f
+; GISEL-NEXT: v_mov_b32_e32 v3, 0
+; GISEL-NEXT: v_subb_u32_e64 v21, s[6:7], 0, 0, s[6:7]
; GISEL-NEXT: v_subb_u32_e64 v22, s[6:7], 0, 0, s[6:7]
+; GISEL-NEXT: v_cmp_gt_u64_e64 s[8:9], v[20:21], v[2:3]
; GISEL-NEXT: v_subb_u32_e64 v23, s[6:7], 0, 0, s[6:7]
-; GISEL-NEXT: v_xor_b32_e32 v2, 0x7f, v20
-; GISEL-NEXT: v_cmp_lt_u64_e64 s[8:9], s[8:9], v[20:21]
-; GISEL-NEXT: v_or_b32_e32 v2, v2, v22
-; GISEL-NEXT: v_or_b32_e32 v3, v21, v23
-; GISEL-NEXT: v_cmp_ne_u64_e64 s[6:7], 0, v[2:3]
; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[8:9]
-; GISEL-NEXT: v_cmp_ne_u64_e64 s[8:9], 0, v[22:23]
-; GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GISEL-NEXT: v_cmp_lt_u64_e64 s[8:9], 0, v[22:23]
+; GISEL-NEXT: v_xor_b32_e32 v18, 0x7f, v20
; GISEL-NEXT: v_cndmask_b32_e64 v3, 0, 1, s[8:9]
; GISEL-NEXT: v_cmp_eq_u64_e64 s[8:9], 0, v[22:23]
+; GISEL-NEXT: v_or_b32_e32 v18, v18, v22
; GISEL-NEXT: v_cndmask_b32_e64 v2, v3, v2, s[8:9]
+; GISEL-NEXT: v_or_b32_e32 v19, v21, v23
; GISEL-NEXT: v_and_b32_e32 v2, 1, v2
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[8:9], 1, v2
+; GISEL-NEXT: v_cmp_eq_u64_e64 s[6:7], 0, v[18:19]
+; GISEL-NEXT: v_cmp_ne_u32_e64 s[8:9], 0, v2
+; GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[8:9]
-; GISEL-NEXT: s_xor_b64 s[10:11], s[4:5], -1
-; GISEL-NEXT: v_cndmask_b32_e64 v3, v19, 0, s[4:5]
-; GISEL-NEXT: s_movk_i32 s8, 0x7f
-; GISEL-NEXT: v_cndmask_b32_e64 v2, v18, 0, s[4:5]
-; GISEL-NEXT: s_and_b64 s[10:11], s[10:11], s[6:7]
-; GISEL-NEXT: v_cndmask_b32_e64 v16, v1, 0, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e64 v17, v0, 0, s[4:5]
-; GISEL-NEXT: s_and_saveexec_b64 s[6:7], s[10:11]
+; GISEL-NEXT: s_mov_b64 s[10:11], exec
+; GISEL-NEXT: s_or_b64 s[6:7], s[4:5], s[6:7]
+; GISEL-NEXT: s_mov_b64 s[12:13], 0
+; GISEL-NEXT: v_cndmask_b32_e64 v18, v0, 0, s[4:5]
+; GISEL-NEXT: v_cndmask_b32_e64 v2, v16, 0, s[4:5]
+; GISEL-NEXT: v_cndmask_b32_e64 v3, v17, 0, s[4:5]
+; GISEL-NEXT: s_xor_b64 s[8:9], s[6:7], s[10:11]
+; GISEL-NEXT: v_cndmask_b32_e64 v19, v1, 0, s[4:5]
+; GISEL-NEXT: s_and_saveexec_b64 s[6:7], s[8:9]
; GISEL-NEXT: s_cbranch_execz .LBB2_6
; GISEL-NEXT: ; %bb.1: ; %udiv-bb15
; GISEL-NEXT: v_add_i32_e32 v26, vcc, 1, v20
; GISEL-NEXT: v_addc_u32_e32 v27, vcc, 0, v21, vcc
; GISEL-NEXT: v_addc_u32_e32 v28, vcc, 0, v22, vcc
-; GISEL-NEXT: v_addc_u32_e64 v29, s[4:5], 0, v23, vcc
-; GISEL-NEXT: v_sub_i32_e32 v21, vcc, s8, v20
-; GISEL-NEXT: v_sub_i32_e32 v16, vcc, 64, v21
-; GISEL-NEXT: v_lshl_b64 v[2:3], v[18:19], v21
-; GISEL-NEXT: v_lshr_b64 v[16:17], v[0:1], v16
-; GISEL-NEXT: s_xor_b64 s[8:9], s[4:5], -1
-; GISEL-NEXT: v_or_b32_e32 v16, v2, v16
-; GISEL-NEXT: v_sub_i32_e32 v2, vcc, 63, v20
-; GISEL-NEXT: v_or_b32_e32 v17, v3, v17
-; GISEL-NEXT: v_lshl_b64 v[2:3], v[0:1], v2
-; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v21
-; GISEL-NEXT: v_cndmask_b32_e32 v3, v3, v17, vcc
+; GISEL-NEXT: s_mov_b64 s[4:5], exec
+; GISEL-NEXT: v_addc_u32_e32 v29, vcc, 0, v23, vcc
+; GISEL-NEXT: s_xor_b64 s[4:5], vcc, s[4:5]
+; GISEL-NEXT: v_sub_i32_e32 v24, vcc, 0x7f, v20
+; GISEL-NEXT: v_not_b32_e32 v2, 63
+; GISEL-NEXT: v_add_i32_e32 v22, vcc, v24, v2
+; GISEL-NEXT: v_sub_i32_e32 v2, vcc, 64, v24
+; GISEL-NEXT: v_lshr_b64 v[2:3], v[0:1], v2
+; GISEL-NEXT: v_lshl_b64 v[18:19], v[16:17], v24
+; GISEL-NEXT: v_lshl_b64 v[20:21], v[0:1], v24
+; GISEL-NEXT: v_or_b32_e32 v18, v2, v18
+; GISEL-NEXT: v_or_b32_e32 v19, v3, v19
+; GISEL-NEXT: v_lshl_b64 v[2:3], v[0:1], v22
+; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v24
+; GISEL-NEXT: v_cndmask_b32_e32 v22, 0, v20, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v23, 0, v21, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v2, v2, v18, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v3, v3, v19, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v24
; GISEL-NEXT: v_cndmask_b32_e32 v2, v2, v16, vcc
-; GISEL-NEXT: v_lshl_b64 v[16:17], v[0:1], v21
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v21
-; GISEL-NEXT: v_cndmask_b32_e64 v3, v3, v19, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e64 v2, v2, v18, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e32 v17, 0, v17, vcc
-; GISEL-NEXT: v_mov_b32_e32 v20, 0
-; GISEL-NEXT: v_mov_b32_e32 v21, 0
-; GISEL-NEXT: v_cndmask_b32_e32 v16, 0, v16, vcc
-; GISEL-NEXT: s_and_saveexec_b64 s[4:5], s[8:9]
-; GISEL-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
+; GISEL-NEXT: v_cndmask_b32_e32 v3, v3, v17, vcc
+; GISEL-NEXT: s_mov_b64 s[14:15], s[12:13]
+; GISEL-NEXT: v_mov_b32_e32 v19, s13
+; GISEL-NEXT: v_mov_b32_e32 v18, s12
+; GISEL-NEXT: v_mov_b32_e32 v21, s15
+; GISEL-NEXT: v_mov_b32_e32 v20, s14
+; GISEL-NEXT: s_and_saveexec_b64 s[8:9], s[4:5]
+; GISEL-NEXT: s_xor_b64 s[12:13], exec, s[8:9]
; GISEL-NEXT: s_cbranch_execz .LBB2_5
; GISEL-NEXT: ; %bb.2: ; %udiv-preheader4
-; GISEL-NEXT: v_sub_i32_e32 v22, vcc, 64, v26
-; GISEL-NEXT: v_lshr_b64 v[20:21], v[0:1], v26
-; GISEL-NEXT: v_lshl_b64 v[22:23], v[18:19], v22
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v26
-; GISEL-NEXT: v_or_b32_e32 v22, v20, v22
-; GISEL-NEXT: v_subrev_i32_e32 v20, vcc, 64, v26
-; GISEL-NEXT: v_or_b32_e32 v23, v21, v23
-; GISEL-NEXT: v_lshr_b64 v[20:21], v[18:19], v20
+; GISEL-NEXT: v_sub_i32_e32 v20, vcc, 64, v26
+; GISEL-NEXT: v_add_i32_e32 v30, vcc, 0xffffffc0, v26
+; GISEL-NEXT: v_lshr_b64 v[18:19], v[0:1], v26
+; GISEL-NEXT: v_lshl_b64 v[20:21], v[16:17], v20
+; GISEL-NEXT: v_lshr_b64 v[24:25], v[16:17], v26
+; GISEL-NEXT: v_lshr_b64 v[16:17], v[16:17], v30
+; GISEL-NEXT: v_or_b32_e32 v18, v18, v20
+; GISEL-NEXT: v_or_b32_e32 v19, v19, v21
; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v26
-; GISEL-NEXT: v_lshr_b64 v[18:19], v[18:19], v26
-; GISEL-NEXT: v_cndmask_b32_e32 v21, v21, v23, vcc
-; GISEL-NEXT: v_cndmask_b32_e64 v23, v21, v1, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e32 v1, v20, v22, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v19, 0, v19, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v18, 0, v18, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v16, v16, v18, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v17, v17, v19, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v24, 0, v24, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v25, 0, v25, vcc
; GISEL-NEXT: v_add_i32_e32 v30, vcc, -1, v8
; GISEL-NEXT: v_addc_u32_e32 v31, vcc, -1, v9, vcc
+; GISEL-NEXT: s_mov_b64 s[8:9], 0
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v26
; GISEL-NEXT: v_addc_u32_e32 v32, vcc, -1, v10, vcc
-; GISEL-NEXT: v_cndmask_b32_e64 v22, v1, v0, s[4:5]
+; GISEL-NEXT: v_cndmask_b32_e64 v16, v16, v0, s[4:5]
+; GISEL-NEXT: v_cndmask_b32_e64 v17, v17, v1, s[4:5]
; GISEL-NEXT: v_addc_u32_e32 v33, vcc, -1, v11, vcc
-; GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GISEL-NEXT: s_mov_b64 s[10:11], s[8:9]
; GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GISEL-NEXT: s_mov_b64 s[4:5], 0
-; GISEL-NEXT: v_mov_b32_e32 v24, 0
-; GISEL-NEXT: v_mov_b32_e32 v25, 0
-; GISEL-NEXT: v_mov_b32_e32 v21, 0
+; GISEL-NEXT: v_mov_b32_e32 v19, s9
+; GISEL-NEXT: v_mov_b32_e32 v18, s8
+; GISEL-NEXT: v_mov_b32_e32 v0, 1
+; GISEL-NEXT: v_mov_b32_e32 v21, s11
+; GISEL-NEXT: v_mov_b32_e32 v20, s10
; GISEL-NEXT: .LBB2_3: ; %udiv-do-while3
; GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GISEL-NEXT: v_lshl_b64 v[20:21], v[22:23], 1
+; GISEL-NEXT: v_lshrrev_b32_e32 v0, 31, v23
+; GISEL-NEXT: v_or_b32_e32 v22, v18, v20
+; GISEL-NEXT: v_or_b32_e32 v23, v19, v21
+; GISEL-NEXT: v_lshl_b64 v[18:19], v[24:25], 1
; GISEL-NEXT: v_lshrrev_b32_e32 v20, 31, v17
; GISEL-NEXT: v_lshl_b64 v[16:17], v[16:17], 1
-; GISEL-NEXT: v_lshl_b64 v[18:19], v[18:19], 1
-; GISEL-NEXT: v_or_b32_e32 v16, v24, v16
-; GISEL-NEXT: v_lshrrev_b32_e32 v24, 31, v23
-; GISEL-NEXT: v_lshl_b64 v[22:23], v[22:23], 1
-; GISEL-NEXT: v_or_b32_e32 v18, v18, v24
-; GISEL-NEXT: v_lshrrev_b32_e32 v24, 31, v3
-; GISEL-NEXT: v_or_b32_e32 v22, v22, v24
-; GISEL-NEXT: v_sub_i32_e32 v24, vcc, v30, v22
-; GISEL-NEXT: v_subb_u32_e32 v24, vcc, v31, v23, vcc
-; GISEL-NEXT: v_subb_u32_e32 v24, vcc, v32, v18, vcc
-; GISEL-NEXT: v_subb_u32_e32 v24, vcc, v33, v19, vcc
-; GISEL-NEXT: v_ashrrev_i32_e32 v34, 31, v24
-; GISEL-NEXT: v_and_b32_e32 v24, v34, v8
-; GISEL-NEXT: v_sub_i32_e32 v22, vcc, v22, v24
-; GISEL-NEXT: v_and_b32_e32 v24, v34, v9
-; GISEL-NEXT: v_subb_u32_e32 v23, vcc, v23, v24, vcc
-; GISEL-NEXT: v_and_b32_e32 v24, v34, v10
-; GISEL-NEXT: v_subb_u32_e32 v18, vcc, v18, v24, vcc
-; GISEL-NEXT: v_and_b32_e32 v24, v34, v11
-; GISEL-NEXT: v_subb_u32_e32 v19, vcc, v19, v24, vcc
+; GISEL-NEXT: v_or_b32_e32 v18, v18, v20
+; GISEL-NEXT: v_lshrrev_b32_e32 v20, 31, v3
+; GISEL-NEXT: v_or_b32_e32 v16, v16, v20
+; GISEL-NEXT: v_sub_i32_e32 v20, vcc, v30, v16
+; GISEL-NEXT: v_subb_u32_e32 v20, vcc, v31, v17, vcc
+; GISEL-NEXT: v_subb_u32_e32 v20, vcc, v32, v18, vcc
+; GISEL-NEXT: v_subb_u32_e32 v20, vcc, v33, v19, vcc
+; GISEL-NEXT: v_ashrrev_i32_e32 v20, 31, v20
+; GISEL-NEXT: v_and_b32_e32 v21, v20, v8
+; GISEL-NEXT: v_and_b32_e32 v24, v20, v9
+; GISEL-NEXT: v_sub_i32_e32 v16, vcc, v16, v21
+; GISEL-NEXT: v_subb_u32_e32 v17, vcc, v17, v24, vcc
+; GISEL-NEXT: v_and_b32_e32 v21, v20, v10
+; GISEL-NEXT: v_and_b32_e32 v25, v20, v11
+; GISEL-NEXT: v_subb_u32_e32 v24, vcc, v18, v21, vcc
+; GISEL-NEXT: v_subb_u32_e32 v25, vcc, v19, v25, vcc
; GISEL-NEXT: v_add_i32_e32 v26, vcc, -1, v26
; GISEL-NEXT: v_addc_u32_e32 v27, vcc, -1, v27, vcc
; GISEL-NEXT: v_addc_u32_e32 v28, vcc, -1, v28, vcc
; GISEL-NEXT: v_addc_u32_e32 v29, vcc, -1, v29, vcc
-; GISEL-NEXT: v_or_b32_e32 v17, v25, v17
-; GISEL-NEXT: v_or_b32_e32 v24, v26, v28
-; GISEL-NEXT: v_or_b32_e32 v25, v27, v29
+; GISEL-NEXT: v_or_b32_e32 v18, v26, v28
+; GISEL-NEXT: v_or_b32_e32 v19, v27, v29
; GISEL-NEXT: v_lshl_b64 v[2:3], v[2:3], 1
-; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[24:25]
-; GISEL-NEXT: v_or_b32_e32 v2, v2, v20
-; GISEL-NEXT: v_and_b32_e32 v20, 1, v34
-; GISEL-NEXT: v_or_b32_e32 v3, v1, v3
-; GISEL-NEXT: v_or_b32_e32 v2, v0, v2
-; GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GISEL-NEXT: v_mov_b32_e32 v25, v21
-; GISEL-NEXT: v_mov_b32_e32 v24, v20
-; GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[18:19]
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v0
+; GISEL-NEXT: v_and_b32_e32 v0, 1, v20
+; GISEL-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GISEL-NEXT: v_mov_b32_e32 v19, v1
+; GISEL-NEXT: v_mov_b32_e32 v18, v0
+; GISEL-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GISEL-NEXT: s_cbranch_execnz .LBB2_3
; GISEL-NEXT: ; %bb.4: ; %Flow13
-; GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
-; GISEL-NEXT: .LBB2_5: ; %Flow14
; GISEL-NEXT: s_or_b64 exec, exec, s[8:9]
+; GISEL-NEXT: .LBB2_5: ; %Flow14
+; GISEL-NEXT: s_or_b64 exec, exec, s[12:13]
+; GISEL-NEXT: v_lshl_b64 v[0:1], v[22:23], 1
; GISEL-NEXT: v_lshl_b64 v[2:3], v[2:3], 1
-; GISEL-NEXT: v_lshl_b64 v[0:1], v[16:17], 1
-; GISEL-NEXT: v_lshrrev_b32_e32 v8, 31, v17
+; GISEL-NEXT: v_lshrrev_b32_e32 v8, 31, v23
; GISEL-NEXT: v_or_b32_e32 v2, v2, v8
-; GISEL-NEXT: v_or_b32_e32 v16, v21, v1
-; GISEL-NEXT: v_or_b32_e32 v17, v20, v0
+; GISEL-NEXT: v_or_b32_e32 v18, v18, v0
+; GISEL-NEXT: v_or_b32_e32 v19, v19, v1
; GISEL-NEXT: .LBB2_6: ; %Flow16
; GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
-; GISEL-NEXT: v_or_b32_e32 v1, v13, v15
; GISEL-NEXT: v_or_b32_e32 v0, v12, v14
+; GISEL-NEXT: v_or_b32_e32 v1, v13, v15
; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
-; GISEL-NEXT: v_or_b32_e32 v1, v5, v7
; GISEL-NEXT: v_or_b32_e32 v0, v4, v6
+; GISEL-NEXT: v_or_b32_e32 v1, v5, v7
; GISEL-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[0:1]
-; GISEL-NEXT: v_ffbh_u32_e32 v0, v14
-; GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, 32, v0
-; GISEL-NEXT: v_ffbh_u32_e32 v1, v15
-; GISEL-NEXT: v_min_u32_e32 v0, v0, v1
; GISEL-NEXT: v_ffbh_u32_e32 v1, v12
+; GISEL-NEXT: s_or_b64 s[10:11], vcc, s[4:5]
+; GISEL-NEXT: v_ffbh_u32_e32 v0, v13
; GISEL-NEXT: v_add_i32_e32 v1, vcc, 32, v1
-; GISEL-NEXT: v_ffbh_u32_e32 v8, v13
+; GISEL-NEXT: v_min_u32_e32 v0, v0, v1
+; GISEL-NEXT: v_ffbh_u32_e32 v8, v14
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, 64, v0
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, 32, v8
+; GISEL-NEXT: v_ffbh_u32_e32 v1, v15
+; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[14:15]
; GISEL-NEXT: v_min_u32_e32 v1, v1, v8
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, 64, v1
-; GISEL-NEXT: v_addc_u32_e64 v8, s[6:7], 0, 0, vcc
-; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[14:15]
-; GISEL-NEXT: v_ffbh_u32_e32 v9, v7
+; GISEL-NEXT: v_ffbh_u32_e32 v8, v4
; GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
-; GISEL-NEXT: v_ffbh_u32_e32 v1, v6
-; GISEL-NEXT: v_cndmask_b32_e64 v8, v8, 0, vcc
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, 32, v1
-; GISEL-NEXT: v_min_u32_e32 v1, v1, v9
-; GISEL-NEXT: v_ffbh_u32_e32 v9, v4
+; GISEL-NEXT: v_ffbh_u32_e32 v1, v5
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, 32, v8
+; GISEL-NEXT: v_min_u32_e32 v1, v1, v8
+; GISEL-NEXT: v_ffbh_u32_e32 v9, v6
+; GISEL-NEXT: v_add_i32_e32 v1, vcc, 64, v1
; GISEL-NEXT: v_add_i32_e32 v9, vcc, 32, v9
-; GISEL-NEXT: v_ffbh_u32_e32 v10, v5
-; GISEL-NEXT: v_min_u32_e32 v9, v9, v10
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, 64, v9
-; GISEL-NEXT: v_addc_u32_e64 v10, s[6:7], 0, 0, vcc
-; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
-; GISEL-NEXT: s_movk_i32 s8, 0x7f
-; GISEL-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc
-; GISEL-NEXT: v_cndmask_b32_e64 v10, v10, 0, vcc
+; GISEL-NEXT: v_ffbh_u32_e32 v8, v7
+; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7]
+; GISEL-NEXT: v_min_u32_e32 v8, v8, v9
+; GISEL-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc
; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v0, v1
-; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v8, v10, vcc
-; GISEL-NEXT: v_subb_u32_e64 v18, s[6:7], 0, 0, vcc
-; GISEL-NEXT: v_subb_u32_e64 v19, s[6:7], 0, 0, s[6:7]
-; GISEL-NEXT: s_mov_b64 s[6:7], 0x7f
-; GISEL-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1]
-; GISEL-NEXT: v_or_b32_e32 v11, v1, v19
+; GISEL-NEXT: v_subb_u32_e64 v1, s[4:5], 0, 0, vcc
+; GISEL-NEXT: v_mov_b32_e32 v8, 0x7f
+; GISEL-NEXT: v_mov_b32_e32 v9, 0
+; GISEL-NEXT: v_subb_u32_e64 v16, s[4:5], 0, 0, s[4:5]
+; GISEL-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[8:9]
+; GISEL-NEXT: v_subb_u32_e64 v17, s[4:5], 0, 0, s[4:5]
; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[18:19]
+; GISEL-NEXT: v_cmp_lt_u64_e32 vcc, 0, v[16:17]
+; GISEL-NEXT: s_mov_b64 s[6:7], exec
; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[18:19]
+; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[16:17]
+; GISEL-NEXT: s_mov_b64 s[8:9], 0
; GISEL-NEXT: v_cndmask_b32_e32 v8, v9, v8, vcc
; GISEL-NEXT: v_and_b32_e32 v8, 1, v8
-; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 1, v8
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v8
; GISEL-NEXT: v_xor_b32_e32 v8, 0x7f, v0
-; GISEL-NEXT: v_or_b32_e32 v10, v8, v18
-; GISEL-NEXT: s_or_b64 s[4:5], s[4:5], vcc
-; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
-; GISEL-NEXT: s_xor_b64 s[6:7], s[4:5], -1
-; GISEL-NEXT: v_cndmask_b32_e64 v9, v7, 0, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e64 v8, v6, 0, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e64 v10, v5, 0, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e64 v11, v4, 0, s[4:5]
-; GISEL-NEXT: s_and_b64 s[4:5], s[6:7], vcc
+; GISEL-NEXT: v_or_b32_e32 v8, v8, v16
+; GISEL-NEXT: v_or_b32_e32 v9, v1, v17
+; GISEL-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[8:9]
+; GISEL-NEXT: s_or_b64 s[10:11], s[10:11], vcc
+; GISEL-NEXT: s_or_b64 s[4:5], s[10:11], s[4:5]
+; GISEL-NEXT: v_cndmask_b32_e64 v10, v4, 0, s[10:11]
+; GISEL-NEXT: v_cndmask_b32_e64 v11, v5, 0, s[10:11]
+; GISEL-NEXT: v_cndmask_b32_e64 v8, v6, 0, s[10:11]
+; GISEL-NEXT: v_cndmask_b32_e64 v9, v7, 0, s[10:11]
+; GISEL-NEXT: s_xor_b64 s[4:5], s[4:5], s[6:7]
; GISEL-NEXT: s_and_saveexec_b64 s[6:7], s[4:5]
; GISEL-NEXT: s_cbranch_execz .LBB2_12
; GISEL-NEXT: ; %bb.7: ; %udiv-bb1
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, 1, v0
-; GISEL-NEXT: v_addc_u32_e32 v24, vcc, 0, v1, vcc
-; GISEL-NEXT: v_addc_u32_e32 v25, vcc, 0, v18, vcc
-; GISEL-NEXT: v_addc_u32_e64 v26, s[4:5], 0, v19, vcc
-; GISEL-NEXT: v_sub_i32_e32 v8, vcc, s8, v0
-; GISEL-NEXT: v_sub_i32_e32 v1, vcc, 64, v8
-; GISEL-NEXT: v_lshl_b64 v[10:11], v[6:7], v8
-; GISEL-NEXT: v_lshr_b64 v[18:19], v[4:5], v1
-; GISEL-NEXT: v_sub_i32_e32 v0, vcc, 63, v0
-; GISEL-NEXT: v_lshl_b64 v[0:1], v[4:5], v0
-; GISEL-NEXT: v_or_b32_e32 v11, v11, v19
-; GISEL-NEXT: v_or_b32_e32 v10, v10, v18
-; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v8
-; GISEL-NEXT: v_cndmask_b32_e32 v1, v1, v11, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v0, v0, v10, vcc
-; GISEL-NEXT: v_lshl_b64 v[10:11], v[4:5], v8
-; GISEL-NEXT: s_xor_b64 s[8:9], s[4:5], -1
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v8
-; GISEL-NEXT: v_cndmask_b32_e64 v1, v1, v7, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e64 v0, v0, v6, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e32 v11, 0, v11, vcc
-; GISEL-NEXT: v_mov_b32_e32 v18, 0
-; GISEL-NEXT: v_mov_b32_e32 v19, 0
-; GISEL-NEXT: v_cndmask_b32_e32 v10, 0, v10, vcc
-; GISEL-NEXT: s_and_saveexec_b64 s[4:5], s[8:9]
-; GISEL-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
+; GISEL-NEXT: v_add_i32_e32 v24, vcc, 1, v0
+; GISEL-NEXT: v_addc_u32_e32 v25, vcc, 0, v1, vcc
+; GISEL-NEXT: v_addc_u32_e32 v16, vcc, 0, v16, vcc
+; GISEL-NEXT: s_mov_b64 s[4:5], exec
+; GISEL-NEXT: v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; GISEL-NEXT: s_xor_b64 s[4:5], vcc, s[4:5]
+; GISEL-NEXT: v_sub_i32_e32 v20, vcc, 0x7f, v0
+; GISEL-NEXT: v_not_b32_e32 v0, 63
+; GISEL-NEXT: v_add_i32_e32 v21, vcc, v20, v0
+; GISEL-NEXT: v_sub_i32_e32 v0, vcc, 64, v20
+; GISEL-NEXT: v_lshr_b64 v[0:1], v[4:5], v0
+; GISEL-NEXT: v_lshl_b64 v[8:9], v[6:7], v20
+; GISEL-NEXT: v_lshl_b64 v[10:11], v[4:5], v20
+; GISEL-NEXT: v_or_b32_e32 v22, v0, v8
+; GISEL-NEXT: v_or_b32_e32 v23, v1, v9
+; GISEL-NEXT: v_lshl_b64 v[8:9], v[4:5], v21
+; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v20
+; GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v10, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v11, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v8, v8, v22, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v9, v9, v23, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v20
+; GISEL-NEXT: v_cndmask_b32_e32 v8, v8, v6, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v9, v9, v7, vcc
+; GISEL-NEXT: s_mov_b64 s[10:11], s[8:9]
+; GISEL-NEXT: v_mov_b32_e32 v21, s9
+; GISEL-NEXT: v_mov_b32_e32 v20, s8
+; GISEL-NEXT: v_mov_b32_e32 v23, s11
+; GISEL-NEXT: v_mov_b32_e32 v22, s10
+; GISEL-NEXT: s_and_saveexec_b64 s[8:9], s[4:5]
+; GISEL-NEXT: s_xor_b64 s[12:13], exec, s[8:9]
; GISEL-NEXT: s_cbranch_execz .LBB2_11
; GISEL-NEXT: ; %bb.8: ; %udiv-preheader
-; GISEL-NEXT: v_sub_i32_e32 v8, vcc, 64, v9
-; GISEL-NEXT: v_lshr_b64 v[18:19], v[4:5], v9
-; GISEL-NEXT: v_lshl_b64 v[20:21], v[6:7], v8
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v9
-; GISEL-NEXT: v_or_b32_e32 v20, v18, v20
-; GISEL-NEXT: v_subrev_i32_e32 v18, vcc, 64, v9
-; GISEL-NEXT: v_or_b32_e32 v8, v19, v21
-; GISEL-NEXT: v_lshr_b64 v[18:19], v[6:7], v18
-; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v9
-; GISEL-NEXT: v_cndmask_b32_e32 v8, v19, v8, vcc
-; GISEL-NEXT: v_cndmask_b32_e64 v8, v8, v5, s[4:5]
-; GISEL-NEXT: v_lshr_b64 v[5:6], v[6:7], v9
-; GISEL-NEXT: v_cndmask_b32_e32 v18, v18, v20, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v21, 0, v6, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v20, 0, v5, vcc
-; GISEL-NEXT: v_add_i32_e32 v6, vcc, -1, v12
+; GISEL-NEXT: v_sub_i32_e32 v20, vcc, 64, v24
+; GISEL-NEXT: v_add_i32_e32 v26, vcc, 0xffffffc0, v24
+; GISEL-NEXT: v_lshr_b64 v[10:11], v[4:5], v24
+; GISEL-NEXT: v_lshl_b64 v[20:21], v[6:7], v20
+; GISEL-NEXT: v_lshr_b64 v[22:23], v[6:7], v24
+; GISEL-NEXT: v_lshr_b64 v[6:7], v[6:7], v26
+; GISEL-NEXT: v_or_b32_e32 v10, v10, v20
+; GISEL-NEXT: v_or_b32_e32 v11, v11, v21
+; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v24
+; GISEL-NEXT: v_cndmask_b32_e32 v6, v6, v10, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v7, v7, v11, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v10, 0, v22, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v11, 0, v23, vcc
+; GISEL-NEXT: v_add_i32_e32 v26, vcc, -1, v12
; GISEL-NEXT: v_addc_u32_e32 v27, vcc, -1, v13, vcc
+; GISEL-NEXT: s_mov_b64 s[8:9], 0
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v24
; GISEL-NEXT: v_addc_u32_e32 v28, vcc, -1, v14, vcc
-; GISEL-NEXT: v_cndmask_b32_e64 v7, v18, v4, s[4:5]
+; GISEL-NEXT: v_cndmask_b32_e64 v6, v6, v4, s[4:5]
+; GISEL-NEXT: v_cndmask_b32_e64 v7, v7, v5, s[4:5]
; GISEL-NEXT: v_addc_u32_e32 v29, vcc, -1, v15, vcc
-; GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GISEL-NEXT: s_mov_b64 s[10:11], s[8:9]
; GISEL-NEXT: v_mov_b32_e32 v5, 0
-; GISEL-NEXT: s_mov_b64 s[10:11], 0
-; GISEL-NEXT: v_mov_b32_e32 v22, 0
-; GISEL-NEXT: v_mov_b32_e32 v23, 0
-; GISEL-NEXT: v_mov_b32_e32 v19, 0
+; GISEL-NEXT: v_mov_b32_e32 v21, s9
+; GISEL-NEXT: v_mov_b32_e32 v20, s8
+; GISEL-NEXT: v_mov_b32_e32 v4, 1
+; GISEL-NEXT: v_mov_b32_e32 v23, s11
+; GISEL-NEXT: v_mov_b32_e32 v22, s10
; GISEL-NEXT: .LBB2_9: ; %udiv-do-while
; GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GISEL-NEXT: v_lshl_b64 v[20:21], v[20:21], 1
-; GISEL-NEXT: v_lshrrev_b32_e32 v18, 31, v8
-; GISEL-NEXT: v_lshl_b64 v[7:8], v[7:8], 1
-; GISEL-NEXT: v_or_b32_e32 v20, v20, v18
-; GISEL-NEXT: v_lshrrev_b32_e32 v18, 31, v1
-; GISEL-NEXT: v_lshl_b64 v[0:1], v[0:1], 1
-; GISEL-NEXT: v_or_b32_e32 v7, v7, v18
-; GISEL-NEXT: v_lshrrev_b32_e32 v18, 31, v11
-; GISEL-NEXT: v_or_b32_e32 v0, v0, v18
-; GISEL-NEXT: v_sub_i32_e32 v18, vcc, v6, v7
-; GISEL-NEXT: v_subb_u32_e32 v18, vcc, v27, v8, vcc
; GISEL-NEXT: v_lshl_b64 v[10:11], v[10:11], 1
-; GISEL-NEXT: v_subb_u32_e32 v18, vcc, v28, v20, vcc
-; GISEL-NEXT: v_subb_u32_e32 v18, vcc, v29, v21, vcc
-; GISEL-NEXT: v_add_i32_e64 v9, s[4:5], -1, v9
-; GISEL-NEXT: v_or_b32_e32 v10, v22, v10
-; GISEL-NEXT: v_ashrrev_i32_e32 v22, 31, v18
-; GISEL-NEXT: v_addc_u32_e64 v24, s[4:5], -1, v24, s[4:5]
-; GISEL-NEXT: v_or_b32_e32 v11, v23, v11
-; GISEL-NEXT: v_and_b32_e32 v18, 1, v22
-; GISEL-NEXT: v_and_b32_e32 v30, v22, v15
-; GISEL-NEXT: v_and_b32_e32 v31, v22, v14
-; GISEL-NEXT: v_and_b32_e32 v23, v22, v13
-; GISEL-NEXT: v_and_b32_e32 v22, v22, v12
-; GISEL-NEXT: v_addc_u32_e64 v25, s[4:5], -1, v25, s[4:5]
-; GISEL-NEXT: v_sub_i32_e32 v7, vcc, v7, v22
-; GISEL-NEXT: v_addc_u32_e64 v26, s[4:5], -1, v26, s[4:5]
-; GISEL-NEXT: v_subb_u32_e32 v8, vcc, v8, v23, vcc
-; GISEL-NEXT: v_or_b32_e32 v23, v24, v26
-; GISEL-NEXT: v_or_b32_e32 v22, v9, v25
-; GISEL-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[22:23]
-; GISEL-NEXT: v_subb_u32_e32 v20, vcc, v20, v31, vcc
-; GISEL-NEXT: v_or_b32_e32 v1, v5, v1
-; GISEL-NEXT: v_or_b32_e32 v0, v4, v0
-; GISEL-NEXT: v_subb_u32_e32 v21, vcc, v21, v30, vcc
-; GISEL-NEXT: s_or_b64 s[10:11], s[4:5], s[10:11]
-; GISEL-NEXT: v_mov_b32_e32 v23, v19
-; GISEL-NEXT: v_mov_b32_e32 v22, v18
-; GISEL-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GISEL-NEXT: v_lshl_b64 v[22:23], v[6:7], 1
+; GISEL-NEXT: v_lshrrev_b32_e32 v4, 31, v7
+; GISEL-NEXT: v_or_b32_e32 v10, v10, v4
+; GISEL-NEXT: v_lshrrev_b32_e32 v4, 31, v9
+; GISEL-NEXT: v_or_b32_e32 v22, v22, v4
+; GISEL-NEXT: v_sub_i32_e32 v4, vcc, v26, v22
+; GISEL-NEXT: v_subb_u32_e32 v4, vcc, v27, v23, vcc
+; GISEL-NEXT: v_subb_u32_e32 v4, vcc, v28, v10, vcc
+; GISEL-NEXT: v_subb_u32_e32 v4, vcc, v29, v11, vcc
+; GISEL-NEXT: v_add_i32_e32 v24, vcc, -1, v24
+; GISEL-NEXT: v_addc_u32_e32 v25, vcc, -1, v25, vcc
+; GISEL-NEXT: v_lshl_b64 v[6:7], v[0:1], 1
+; GISEL-NEXT: v_lshl_b64 v[8:9], v[8:9], 1
+; GISEL-NEXT: v_ashrrev_i32_e32 v30, 31, v4
+; GISEL-NEXT: v_addc_u32_e32 v16, vcc, -1, v16, vcc
+; GISEL-NEXT: v_lshrrev_b32_e32 v0, 31, v1
+; GISEL-NEXT: v_and_b32_e32 v4, 1, v30
+; GISEL-NEXT: v_addc_u32_e32 v17, vcc, -1, v17, vcc
+; GISEL-NEXT: v_or_b32_e32 v8, v8, v0
+; GISEL-NEXT: v_or_b32_e32 v0, v20, v6
+; GISEL-NEXT: v_or_b32_e32 v1, v21, v7
+; GISEL-NEXT: v_mov_b32_e32 v21, v5
+; GISEL-NEXT: v_mov_b32_e32 v20, v4
+; GISEL-NEXT: v_and_b32_e32 v4, v30, v12
+; GISEL-NEXT: v_or_b32_e32 v6, v24, v16
+; GISEL-NEXT: v_or_b32_e32 v7, v25, v17
+; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7]
+; GISEL-NEXT: v_and_b32_e32 v7, v30, v13
+; GISEL-NEXT: v_sub_i32_e64 v6, s[4:5], v22, v4
+; GISEL-NEXT: v_and_b32_e32 v31, v30, v14
+; GISEL-NEXT: v_subb_u32_e64 v7, s[4:5], v23, v7, s[4:5]
+; GISEL-NEXT: v_and_b32_e32 v30, v30, v15
+; GISEL-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GISEL-NEXT: v_subb_u32_e64 v10, vcc, v10, v31, s[4:5]
+; GISEL-NEXT: v_subb_u32_e32 v11, vcc, v11, v30, vcc
+; GISEL-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GISEL-NEXT: s_cbranch_execnz .LBB2_9
; GISEL-NEXT: ; %bb.10: ; %Flow
-; GISEL-NEXT: s_or_b64 exec, exec, s[10:11]
-; GISEL-NEXT: .LBB2_11: ; %Flow11
; GISEL-NEXT: s_or_b64 exec, exec, s[8:9]
-; GISEL-NEXT: v_lshl_b64 v[8:9], v[0:1], 1
-; GISEL-NEXT: v_lshl_b64 v[0:1], v[10:11], 1
-; GISEL-NEXT: v_lshrrev_b32_e32 v4, 31, v11
-; GISEL-NEXT: v_or_b32_e32 v8, v8, v4
-; GISEL-NEXT: v_or_b32_e32 v10, v19, v1
-; GISEL-NEXT: v_or_b32_e32 v11, v18, v0
+; GISEL-NEXT: .LBB2_11: ; %Flow11
+; GISEL-NEXT: s_or_b64 exec, exec, s[12:13]
+; GISEL-NEXT: v_lshl_b64 v[4:5], v[0:1], 1
+; GISEL-NEXT: v_lshl_b64 v[8:9], v[8:9], 1
+; GISEL-NEXT: v_lshrrev_b32_e32 v0, 31, v1
+; GISEL-NEXT: v_or_b32_e32 v8, v8, v0
+; GISEL-NEXT: v_or_b32_e32 v10, v20, v4
+; GISEL-NEXT: v_or_b32_e32 v11, v21, v5
; GISEL-NEXT: .LBB2_12: ; %Flow12
; GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
-; GISEL-NEXT: v_mov_b32_e32 v0, v17
-; GISEL-NEXT: v_mov_b32_e32 v1, v16
-; GISEL-NEXT: v_mov_b32_e32 v4, v11
-; GISEL-NEXT: v_mov_b32_e32 v5, v10
+; GISEL-NEXT: v_mov_b32_e32 v0, v18
+; GISEL-NEXT: v_mov_b32_e32 v1, v19
+; GISEL-NEXT: v_mov_b32_e32 v4, v10
+; GISEL-NEXT: v_mov_b32_e32 v5, v11
; GISEL-NEXT: v_mov_b32_e32 v6, v8
; GISEL-NEXT: v_mov_b32_e32 v7, v9
; GISEL-NEXT: s_setpc_b64 s[30:31]
@@ -1595,15 +1603,17 @@ define <2 x i128> @v_udiv_v2i128_v_pow2k(<2 x i128> %lhs) {
; GISEL-LABEL: v_udiv_v2i128_v_pow2k:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT: v_lshrrev_b32_e32 v8, 1, v3
-; GISEL-NEXT: v_lshrrev_b32_e32 v9, 1, v7
-; GISEL-NEXT: v_alignbit_b32 v0, v2, v1, 1
-; GISEL-NEXT: v_alignbit_b32 v4, v6, v5, 1
-; GISEL-NEXT: v_alignbit_b32 v1, v3, v2, 1
-; GISEL-NEXT: v_alignbit_b32 v5, v7, v6, 1
-; GISEL-NEXT: v_mov_b32_e32 v2, v8
+; GISEL-NEXT: v_mov_b32_e32 v4, v1
+; GISEL-NEXT: v_lshl_b64 v[0:1], v[2:3], 31
+; GISEL-NEXT: v_mov_b32_e32 v8, v5
+; GISEL-NEXT: v_lshrrev_b32_e32 v2, 1, v4
+; GISEL-NEXT: v_lshl_b64 v[4:5], v[6:7], 31
+; GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GISEL-NEXT: v_lshrrev_b32_e32 v2, 1, v3
+; GISEL-NEXT: v_lshrrev_b32_e32 v3, 1, v8
+; GISEL-NEXT: v_or_b32_e32 v4, v4, v3
+; GISEL-NEXT: v_lshrrev_b32_e32 v6, 1, v7
; GISEL-NEXT: v_mov_b32_e32 v3, 0
-; GISEL-NEXT: v_mov_b32_e32 v6, v9
; GISEL-NEXT: v_mov_b32_e32 v7, 0
; GISEL-NEXT: s_setpc_b64 s[30:31]
%shl = udiv <2 x i128> %lhs, <i128 8589934592, i128 8589934592>
@@ -2061,448 +2071,434 @@ define <2 x i128> @v_srem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-LABEL: v_srem_v2i128_vv:
; GISEL: ; %bb.0: ; %_udiv-special-cases_udiv-special-cases
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT: buffer_store_dword v40, off, s[0:3], s32 ; 4-byte Folded Spill
-; GISEL-NEXT: v_ashrrev_i32_e32 v28, 31, v3
-; GISEL-NEXT: v_xor_b32_e32 v0, v0, v28
-; GISEL-NEXT: v_xor_b32_e32 v1, v1, v28
-; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v0, v28
-; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v1, v28, vcc
-; GISEL-NEXT: v_xor_b32_e32 v2, v2, v28
-; GISEL-NEXT: v_xor_b32_e32 v3, v3, v28
-; GISEL-NEXT: v_subb_u32_e32 v2, vcc, v2, v28, vcc
-; GISEL-NEXT: v_ashrrev_i32_e32 v16, 31, v11
-; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v3, v28, vcc
-; GISEL-NEXT: v_xor_b32_e32 v8, v8, v16
-; GISEL-NEXT: v_xor_b32_e32 v9, v9, v16
-; GISEL-NEXT: v_sub_i32_e32 v30, vcc, v8, v16
-; GISEL-NEXT: v_subb_u32_e32 v29, vcc, v9, v16, vcc
-; GISEL-NEXT: v_xor_b32_e32 v8, v10, v16
-; GISEL-NEXT: v_xor_b32_e32 v9, v11, v16
-; GISEL-NEXT: v_subb_u32_e32 v8, vcc, v8, v16, vcc
-; GISEL-NEXT: v_subb_u32_e32 v9, vcc, v9, v16, vcc
-; GISEL-NEXT: v_or_b32_e32 v11, v29, v9
-; GISEL-NEXT: v_or_b32_e32 v10, v30, v8
-; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[10:11]
-; GISEL-NEXT: v_or_b32_e32 v11, v1, v3
-; GISEL-NEXT: v_or_b32_e32 v10, v0, v2
-; GISEL-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[10:11]
-; GISEL-NEXT: v_ffbh_u32_e32 v10, v8
-; GISEL-NEXT: v_add_i32_e64 v10, s[6:7], 32, v10
-; GISEL-NEXT: v_ffbh_u32_e32 v11, v9
-; GISEL-NEXT: v_min_u32_e32 v10, v10, v11
-; GISEL-NEXT: v_ffbh_u32_e32 v11, v30
-; GISEL-NEXT: v_add_i32_e64 v11, s[6:7], 32, v11
-; GISEL-NEXT: v_ffbh_u32_e32 v16, v29
-; GISEL-NEXT: v_min_u32_e32 v11, v11, v16
-; GISEL-NEXT: v_add_i32_e64 v11, s[6:7], 64, v11
-; GISEL-NEXT: v_addc_u32_e64 v16, s[6:7], 0, 0, s[6:7]
-; GISEL-NEXT: v_cmp_ne_u64_e64 s[6:7], 0, v[8:9]
-; GISEL-NEXT: v_ffbh_u32_e32 v18, v1
-; GISEL-NEXT: v_cndmask_b32_e64 v10, v11, v10, s[6:7]
-; GISEL-NEXT: v_ffbh_u32_e32 v11, v2
-; GISEL-NEXT: v_cndmask_b32_e64 v17, v16, 0, s[6:7]
-; GISEL-NEXT: v_add_i32_e64 v11, s[6:7], 32, v11
-; GISEL-NEXT: v_ffbh_u32_e32 v16, v3
-; GISEL-NEXT: v_min_u32_e32 v11, v11, v16
-; GISEL-NEXT: v_ffbh_u32_e32 v16, v0
-; GISEL-NEXT: v_add_i32_e64 v16, s[6:7], 32, v16
-; GISEL-NEXT: v_min_u32_e32 v16, v16, v18
-; GISEL-NEXT: v_add_i32_e64 v16, s[6:7], 64, v16
-; GISEL-NEXT: v_addc_u32_e64 v18, s[6:7], 0, 0, s[6:7]
-; GISEL-NEXT: v_cmp_ne_u64_e64 s[6:7], 0, v[2:3]
-; GISEL-NEXT: s_mov_b64 s[8:9], 0x7f
-; GISEL-NEXT: v_cndmask_b32_e64 v11, v16, v11, s[6:7]
-; GISEL-NEXT: v_cndmask_b32_e64 v18, v18, 0, s[6:7]
-; GISEL-NEXT: v_sub_i32_e64 v16, s[6:7], v10, v11
-; GISEL-NEXT: v_subb_u32_e64 v17, s[6:7], v17, v18, s[6:7]
-; GISEL-NEXT: v_subb_u32_e64 v18, s[6:7], 0, 0, s[6:7]
-; GISEL-NEXT: v_subb_u32_e64 v19, s[6:7], 0, 0, s[6:7]
-; GISEL-NEXT: v_xor_b32_e32 v10, 0x7f, v16
-; GISEL-NEXT: v_cmp_lt_u64_e64 s[8:9], s[8:9], v[16:17]
-; GISEL-NEXT: v_or_b32_e32 v10, v10, v18
-; GISEL-NEXT: v_or_b32_e32 v11, v17, v19
-; GISEL-NEXT: v_cmp_ne_u64_e64 s[6:7], 0, v[10:11]
-; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, s[8:9]
-; GISEL-NEXT: v_cmp_ne_u64_e64 s[8:9], 0, v[18:19]
+; GISEL-NEXT: v_ashrrev_i32_e32 v29, 31, v3
+; GISEL-NEXT: v_ashrrev_i32_e32 v17, 31, v11
+; GISEL-NEXT: v_xor_b32_e32 v0, v0, v29
+; GISEL-NEXT: v_xor_b32_e32 v1, v1, v29
+; GISEL-NEXT: v_xor_b32_e32 v19, v10, v17
+; GISEL-NEXT: v_sub_i32_e32 v10, vcc, v0, v29
+; GISEL-NEXT: v_xor_b32_e32 v2, v2, v29
+; GISEL-NEXT: v_xor_b32_e32 v20, v11, v17
+; GISEL-NEXT: v_subb_u32_e32 v11, vcc, v1, v29, vcc
+; GISEL-NEXT: v_xor_b32_e32 v3, v3, v29
+; GISEL-NEXT: v_xor_b32_e32 v16, v8, v17
+; GISEL-NEXT: v_subb_u32_e32 v8, vcc, v2, v29, vcc
+; GISEL-NEXT: v_xor_b32_e32 v18, v9, v17
+; GISEL-NEXT: v_subb_u32_e32 v9, vcc, v3, v29, vcc
+; GISEL-NEXT: v_sub_i32_e32 v31, vcc, v16, v17
+; GISEL-NEXT: v_subb_u32_e32 v30, vcc, v18, v17, vcc
+; GISEL-NEXT: v_subb_u32_e32 v16, vcc, v19, v17, vcc
+; GISEL-NEXT: v_subb_u32_e32 v17, vcc, v20, v17, vcc
+; GISEL-NEXT: v_or_b32_e32 v0, v31, v16
+; GISEL-NEXT: v_or_b32_e32 v1, v30, v17
+; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
+; GISEL-NEXT: v_or_b32_e32 v0, v10, v8
+; GISEL-NEXT: v_or_b32_e32 v1, v11, v9
+; GISEL-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[0:1]
+; GISEL-NEXT: v_ffbh_u32_e32 v1, v31
+; GISEL-NEXT: v_ffbh_u32_e32 v0, v30
+; GISEL-NEXT: v_add_i32_e64 v1, s[6:7], 32, v1
+; GISEL-NEXT: v_min_u32_e32 v0, v0, v1
+; GISEL-NEXT: v_ffbh_u32_e32 v2, v16
+; GISEL-NEXT: v_add_i32_e64 v0, s[6:7], 64, v0
+; GISEL-NEXT: v_add_i32_e64 v2, s[6:7], 32, v2
+; GISEL-NEXT: v_ffbh_u32_e32 v1, v17
+; GISEL-NEXT: v_cmp_eq_u64_e64 s[6:7], 0, v[16:17]
+; GISEL-NEXT: v_min_u32_e32 v1, v1, v2
+; GISEL-NEXT: v_ffbh_u32_e32 v2, v10
+; GISEL-NEXT: v_cndmask_b32_e64 v0, v1, v0, s[6:7]
+; GISEL-NEXT: v_ffbh_u32_e32 v1, v11
+; GISEL-NEXT: v_add_i32_e64 v2, s[6:7], 32, v2
+; GISEL-NEXT: v_min_u32_e32 v1, v1, v2
+; GISEL-NEXT: v_ffbh_u32_e32 v2, v8
+; GISEL-NEXT: v_add_i32_e64 v2, s[6:7], 32, v2
+; GISEL-NEXT: v_add_i32_e64 v1, s[6:7], 64, v1
+; GISEL-NEXT: v_ffbh_u32_e32 v3, v9
+; GISEL-NEXT: v_cmp_eq_u64_e64 s[6:7], 0, v[8:9]
+; GISEL-NEXT: v_min_u32_e32 v2, v3, v2
+; GISEL-NEXT: v_cndmask_b32_e64 v1, v2, v1, s[6:7]
+; GISEL-NEXT: v_sub_i32_e64 v0, s[6:7], v0, v1
+; GISEL-NEXT: v_subb_u32_e64 v1, s[6:7], 0, 0, s[6:7]
+; GISEL-NEXT: v_mov_b32_e32 v18, 0x7f
+; GISEL-NEXT: v_mov_b32_e32 v19, 0
+; GISEL-NEXT: v_subb_u32_e64 v2, s[6:7], 0, 0, s[6:7]
+; GISEL-NEXT: v_cmp_gt_u64_e64 s[8:9], v[0:1], v[18:19]
+; GISEL-NEXT: v_subb_u32_e64 v3, s[6:7], 0, 0, s[6:7]
+; GISEL-NEXT: v_cndmask_b32_e64 v18, 0, 1, s[8:9]
+; GISEL-NEXT: v_cmp_lt_u64_e64 s[8:9], 0, v[2:3]
+; GISEL-NEXT: v_xor_b32_e32 v20, 0x7f, v0
+; GISEL-NEXT: v_cndmask_b32_e64 v19, 0, 1, s[8:9]
+; GISEL-NEXT: v_cmp_eq_u64_e64 s[8:9], 0, v[2:3]
+; GISEL-NEXT: v_or_b32_e32 v20, v20, v2
+; GISEL-NEXT: v_cndmask_b32_e64 v18, v19, v18, s[8:9]
+; GISEL-NEXT: v_or_b32_e32 v21, v1, v3
+; GISEL-NEXT: v_and_b32_e32 v18, 1, v18
+; GISEL-NEXT: v_cmp_eq_u64_e64 s[6:7], 0, v[20:21]
+; GISEL-NEXT: v_cmp_ne_u32_e64 s[8:9], 0, v18
; GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, s[8:9]
-; GISEL-NEXT: v_cmp_eq_u64_e64 s[8:9], 0, v[18:19]
-; GISEL-NEXT: v_mov_b32_e32 v31, v28
-; GISEL-NEXT: v_cndmask_b32_e64 v10, v11, v10, s[8:9]
-; GISEL-NEXT: v_and_b32_e32 v10, 1, v10
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[8:9], 1, v10
; GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[8:9]
-; GISEL-NEXT: s_xor_b64 s[10:11], s[4:5], -1
-; GISEL-NEXT: v_cndmask_b32_e64 v11, v3, 0, s[4:5]
-; GISEL-NEXT: s_movk_i32 s8, 0x7f
-; GISEL-NEXT: v_cndmask_b32_e64 v10, v2, 0, s[4:5]
-; GISEL-NEXT: s_and_b64 s[10:11], s[10:11], s[6:7]
-; GISEL-NEXT: v_cndmask_b32_e64 v35, v1, 0, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e64 v37, v0, 0, s[4:5]
-; GISEL-NEXT: s_and_saveexec_b64 s[6:7], s[10:11]
+; GISEL-NEXT: s_mov_b64 s[10:11], exec
+; GISEL-NEXT: s_or_b64 s[6:7], s[4:5], s[6:7]
+; GISEL-NEXT: s_mov_b64 s[12:13], 0
+; GISEL-NEXT: v_cndmask_b32_e64 v28, v10, 0, s[4:5]
+; GISEL-NEXT: v_cndmask_b32_e64 v18, v8, 0, s[4:5]
+; GISEL-NEXT: v_cndmask_b32_e64 v19, v9, 0, s[4:5]
+; GISEL-NEXT: s_xor_b64 s[8:9], s[6:7], s[10:11]
+; GISEL-NEXT: v_cndmask_b32_e64 v33, v11, 0, s[4:5]
+; GISEL-NEXT: s_and_saveexec_b64 s[6:7], s[8:9]
; GISEL-NEXT: s_cbranch_execz .LBB4_6
; GISEL-NEXT: ; %bb.1: ; %udiv-bb15
-; GISEL-NEXT: v_add_i32_e32 v32, vcc, 1, v16
-; GISEL-NEXT: v_addc_u32_e32 v33, vcc, 0, v17, vcc
-; GISEL-NEXT: v_addc_u32_e32 v34, vcc, 0, v18, vcc
-; GISEL-NEXT: v_addc_u32_e64 v35, s[4:5], 0, v19, vcc
-; GISEL-NEXT: v_sub_i32_e32 v19, vcc, s8, v16
-; GISEL-NEXT: v_sub_i32_e32 v17, vcc, 64, v19
-; GISEL-NEXT: v_lshl_b64 v[10:11], v[2:3], v19
-; GISEL-NEXT: v_lshr_b64 v[17:18], v[0:1], v17
-; GISEL-NEXT: s_xor_b64 s[8:9], s[4:5], -1
-; GISEL-NEXT: v_or_b32_e32 v17, v10, v17
-; GISEL-NEXT: v_sub_i32_e32 v10, vcc, 63, v16
-; GISEL-NEXT: v_or_b32_e32 v18, v11, v18
-; GISEL-NEXT: v_lshl_b64 v[10:11], v[0:1], v10
-; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v19
-; GISEL-NEXT: v_cndmask_b32_e32 v10, v10, v17, vcc
-; GISEL-NEXT: v_lshl_b64 v[16:17], v[0:1], v19
-; GISEL-NEXT: v_cndmask_b32_e32 v11, v11, v18, vcc
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v19
-; GISEL-NEXT: v_cndmask_b32_e64 v11, v11, v3, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e64 v10, v10, v2, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e32 v17, 0, v17, vcc
-; GISEL-NEXT: v_mov_b32_e32 v18, 0
-; GISEL-NEXT: v_mov_b32_e32 v19, 0
-; GISEL-NEXT: v_cndmask_b32_e32 v16, 0, v16, vcc
-; GISEL-NEXT: s_and_saveexec_b64 s[4:5], s[8:9]
-; GISEL-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
+; GISEL-NEXT: v_add_i32_e32 v18, vcc, 1, v0
+; GISEL-NEXT: v_addc_u32_e32 v32, vcc, 0, v1, vcc
+; GISEL-NEXT: v_addc_u32_e32 v33, vcc, 0, v2, vcc
+; GISEL-NEXT: s_mov_b64 s[4:5], exec
+; GISEL-NEXT: v_addc_u32_e32 v34, vcc, 0, v3, vcc
+; GISEL-NEXT: s_xor_b64 s[4:5], vcc, s[4:5]
+; GISEL-NEXT: v_sub_i32_e32 v21, vcc, 0x7f, v0
+; GISEL-NEXT: v_not_b32_e32 v0, 63
+; GISEL-NEXT: v_add_i32_e32 v22, vcc, v21, v0
+; GISEL-NEXT: v_sub_i32_e32 v0, vcc, 64, v21
+; GISEL-NEXT: v_lshr_b64 v[0:1], v[10:11], v0
+; GISEL-NEXT: v_lshl_b64 v[2:3], v[8:9], v21
+; GISEL-NEXT: v_lshl_b64 v[19:20], v[10:11], v21
+; GISEL-NEXT: v_or_b32_e32 v2, v0, v2
+; GISEL-NEXT: v_or_b32_e32 v3, v1, v3
+; GISEL-NEXT: v_lshl_b64 v[0:1], v[10:11], v22
+; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v21
+; GISEL-NEXT: v_cndmask_b32_e32 v19, 0, v19, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v20, 0, v20, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v21
+; GISEL-NEXT: v_cndmask_b32_e32 v21, v0, v8, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v22, v1, v9, vcc
+; GISEL-NEXT: s_mov_b64 s[14:15], s[12:13]
+; GISEL-NEXT: v_mov_b32_e32 v0, s12
+; GISEL-NEXT: v_mov_b32_e32 v1, s13
+; GISEL-NEXT: v_mov_b32_e32 v2, s14
+; GISEL-NEXT: v_mov_b32_e32 v3, s15
+; GISEL-NEXT: s_and_saveexec_b64 s[8:9], s[4:5]
+; GISEL-NEXT: s_xor_b64 s[12:13], exec, s[8:9]
; GISEL-NEXT: s_cbranch_execz .LBB4_5
; GISEL-NEXT: ; %bb.2: ; %udiv-preheader4
-; GISEL-NEXT: v_sub_i32_e32 v20, vcc, 64, v32
-; GISEL-NEXT: v_lshr_b64 v[18:19], v[0:1], v32
-; GISEL-NEXT: v_lshl_b64 v[20:21], v[2:3], v20
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v32
-; GISEL-NEXT: v_or_b32_e32 v20, v18, v20
-; GISEL-NEXT: v_subrev_i32_e32 v18, vcc, 64, v32
-; GISEL-NEXT: v_or_b32_e32 v21, v19, v21
-; GISEL-NEXT: v_lshr_b64 v[18:19], v[2:3], v18
-; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v32
-; GISEL-NEXT: v_cndmask_b32_e32 v19, v19, v21, vcc
-; GISEL-NEXT: v_cndmask_b32_e64 v21, v19, v1, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e32 v20, v18, v20, vcc
-; GISEL-NEXT: v_lshr_b64 v[18:19], v[2:3], v32
-; GISEL-NEXT: v_cndmask_b32_e64 v20, v20, v0, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e32 v25, 0, v19, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v24, 0, v18, vcc
-; GISEL-NEXT: v_add_i32_e32 v36, vcc, -1, v30
-; GISEL-NEXT: v_addc_u32_e32 v37, vcc, -1, v29, vcc
-; GISEL-NEXT: v_addc_u32_e32 v38, vcc, -1, v8, vcc
-; GISEL-NEXT: v_addc_u32_e32 v39, vcc, -1, v9, vcc
-; GISEL-NEXT: v_mov_b32_e32 v22, 0
-; GISEL-NEXT: v_mov_b32_e32 v23, 0
-; GISEL-NEXT: s_mov_b64 s[4:5], 0
-; GISEL-NEXT: v_mov_b32_e32 v26, 0
-; GISEL-NEXT: v_mov_b32_e32 v27, 0
-; GISEL-NEXT: v_mov_b32_e32 v19, 0
+; GISEL-NEXT: v_sub_i32_e32 v2, vcc, 64, v18
+; GISEL-NEXT: v_lshr_b64 v[0:1], v[10:11], v18
+; GISEL-NEXT: v_lshl_b64 v[2:3], v[8:9], v2
+; GISEL-NEXT: v_add_i32_e32 v25, vcc, 0xffffffc0, v18
+; GISEL-NEXT: v_lshr_b64 v[23:24], v[8:9], v18
+; GISEL-NEXT: v_or_b32_e32 v2, v0, v2
+; GISEL-NEXT: v_or_b32_e32 v3, v1, v3
+; GISEL-NEXT: v_lshr_b64 v[0:1], v[8:9], v25
+; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v18
+; GISEL-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v27, 0, v23, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v28, 0, v24, vcc
+; GISEL-NEXT: v_add_i32_e32 v35, vcc, -1, v31
+; GISEL-NEXT: v_addc_u32_e32 v36, vcc, -1, v30, vcc
+; GISEL-NEXT: s_mov_b64 s[8:9], 0
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v18
+; GISEL-NEXT: v_addc_u32_e32 v37, vcc, -1, v16, vcc
+; GISEL-NEXT: v_cndmask_b32_e64 v25, v0, v10, s[4:5]
+; GISEL-NEXT: v_cndmask_b32_e64 v26, v1, v11, s[4:5]
+; GISEL-NEXT: v_addc_u32_e32 v38, vcc, -1, v17, vcc
+; GISEL-NEXT: s_mov_b64 s[10:11], s[8:9]
+; GISEL-NEXT: v_mov_b32_e32 v24, 0
+; GISEL-NEXT: v_mov_b32_e32 v0, s8
+; GISEL-NEXT: v_mov_b32_e32 v1, s9
+; GISEL-NEXT: v_mov_b32_e32 v23, 1
+; GISEL-NEXT: v_mov_b32_e32 v2, s10
+; GISEL-NEXT: v_mov_b32_e32 v3, s11
; GISEL-NEXT: .LBB4_3: ; %udiv-do-while3
; GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GISEL-NEXT: v_lshl_b64 v[48:49], v[16:17], 1
-; GISEL-NEXT: v_lshl_b64 v[24:25], v[24:25], 1
-; GISEL-NEXT: v_or_b32_e32 v16, v26, v48
-; GISEL-NEXT: v_lshrrev_b32_e32 v26, 31, v21
-; GISEL-NEXT: v_lshl_b64 v[20:21], v[20:21], 1
-; GISEL-NEXT: v_or_b32_e32 v24, v24, v26
-; GISEL-NEXT: v_lshrrev_b32_e32 v26, 31, v11
-; GISEL-NEXT: v_or_b32_e32 v20, v20, v26
-; GISEL-NEXT: v_sub_i32_e32 v26, vcc, v36, v20
-; GISEL-NEXT: v_subb_u32_e32 v26, vcc, v37, v21, vcc
-; GISEL-NEXT: v_subb_u32_e32 v26, vcc, v38, v24, vcc
-; GISEL-NEXT: v_subb_u32_e32 v26, vcc, v39, v25, vcc
-; GISEL-NEXT: v_ashrrev_i32_e32 v48, 31, v26
-; GISEL-NEXT: v_lshrrev_b32_e32 v18, 31, v17
-; GISEL-NEXT: v_or_b32_e32 v17, v27, v49
-; GISEL-NEXT: v_and_b32_e32 v27, v48, v30
-; GISEL-NEXT: v_and_b32_e32 v26, v48, v29
-; GISEL-NEXT: v_sub_i32_e32 v20, vcc, v20, v27
-; GISEL-NEXT: v_subb_u32_e32 v21, vcc, v21, v26, vcc
-; GISEL-NEXT: v_and_b32_e32 v27, v48, v8
-; GISEL-NEXT: v_and_b32_e32 v26, v48, v9
-; GISEL-NEXT: v_subb_u32_e32 v24, vcc, v24, v27, vcc
-; GISEL-NEXT: v_subb_u32_e32 v25, vcc, v25, v26, vcc
-; GISEL-NEXT: v_add_i32_e32 v32, vcc, -1, v32
+; GISEL-NEXT: v_lshl_b64 v[2:3], v[19:20], 1
+; GISEL-NEXT: v_lshrrev_b32_e32 v23, 31, v20
+; GISEL-NEXT: v_or_b32_e32 v19, v0, v2
+; GISEL-NEXT: v_or_b32_e32 v20, v1, v3
+; GISEL-NEXT: v_lshl_b64 v[2:3], v[25:26], 1
+; GISEL-NEXT: v_lshrrev_b32_e32 v25, 31, v22
+; GISEL-NEXT: v_lshl_b64 v[0:1], v[27:28], 1
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v25
+; GISEL-NEXT: v_lshrrev_b32_e32 v27, 31, v26
+; GISEL-NEXT: v_sub_i32_e32 v25, vcc, v35, v2
+; GISEL-NEXT: v_or_b32_e32 v0, v0, v27
+; GISEL-NEXT: v_subb_u32_e32 v25, vcc, v36, v3, vcc
+; GISEL-NEXT: v_subb_u32_e32 v25, vcc, v37, v0, vcc
+; GISEL-NEXT: v_subb_u32_e32 v25, vcc, v38, v1, vcc
+; GISEL-NEXT: v_ashrrev_i32_e32 v39, 31, v25
+; GISEL-NEXT: v_and_b32_e32 v25, v39, v31
+; GISEL-NEXT: v_and_b32_e32 v26, v39, v30
+; GISEL-NEXT: v_sub_i32_e32 v25, vcc, v2, v25
+; GISEL-NEXT: v_subb_u32_e32 v26, vcc, v3, v26, vcc
+; GISEL-NEXT: v_and_b32_e32 v2, v39, v16
+; GISEL-NEXT: v_and_b32_e32 v3, v39, v17
+; GISEL-NEXT: v_subb_u32_e32 v27, vcc, v0, v2, vcc
+; GISEL-NEXT: v_subb_u32_e32 v28, vcc, v1, v3, vcc
+; GISEL-NEXT: v_add_i32_e32 v18, vcc, -1, v18
+; GISEL-NEXT: v_addc_u32_e32 v32, vcc, -1, v32, vcc
; GISEL-NEXT: v_addc_u32_e32 v33, vcc, -1, v33, vcc
; GISEL-NEXT: v_addc_u32_e32 v34, vcc, -1, v34, vcc
-; GISEL-NEXT: v_addc_u32_e32 v35, vcc, -1, v35, vcc
-; GISEL-NEXT: v_or_b32_e32 v27, v33, v35
-; GISEL-NEXT: v_or_b32_e32 v26, v32, v34
-; GISEL-NEXT: v_lshl_b64 v[10:11], v[10:11], 1
-; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[26:27]
-; GISEL-NEXT: v_or_b32_e32 v10, v10, v18
-; GISEL-NEXT: v_and_b32_e32 v18, 1, v48
-; GISEL-NEXT: v_or_b32_e32 v11, v23, v11
-; GISEL-NEXT: v_or_b32_e32 v10, v22, v10
-; GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GISEL-NEXT: v_mov_b32_e32 v27, v19
-; GISEL-NEXT: v_mov_b32_e32 v26, v18
-; GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GISEL-NEXT: v_or_b32_e32 v0, v18, v33
+; GISEL-NEXT: v_or_b32_e32 v1, v32, v34
+; GISEL-NEXT: v_lshl_b64 v[21:22], v[21:22], 1
+; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
+; GISEL-NEXT: v_or_b32_e32 v21, v21, v23
+; GISEL-NEXT: v_and_b32_e32 v23, 1, v39
+; GISEL-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GISEL-NEXT: v_mov_b32_e32 v0, v23
+; GISEL-NEXT: v_mov_b32_e32 v1, v24
+; GISEL-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GISEL-NEXT: s_cbranch_execnz .LBB4_3
; GISEL-NEXT: ; %bb.4: ; %Flow13
-; GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
-; GISEL-NEXT: .LBB4_5: ; %Flow14
; GISEL-NEXT: s_or_b64 exec, exec, s[8:9]
-; GISEL-NEXT: v_lshl_b64 v[10:11], v[10:11], 1
-; GISEL-NEXT: v_lshrrev_b32_e32 v20, 31, v17
-; GISEL-NEXT: v_lshl_b64 v[16:17], v[16:17], 1
-; GISEL-NEXT: v_or_b32_e32 v10, v10, v20
-; GISEL-NEXT: v_or_b32_e32 v35, v19, v17
-; GISEL-NEXT: v_or_b32_e32 v37, v18, v16
+; GISEL-NEXT: .LBB4_5: ; %Flow14
+; GISEL-NEXT: s_or_b64 exec, exec, s[12:13]
+; GISEL-NEXT: v_lshl_b64 v[2:3], v[19:20], 1
+; GISEL-NEXT: v_lshl_b64 v[18:19], v[21:22], 1
+; GISEL-NEXT: v_lshrrev_b32_e32 v20, 31, v20
+; GISEL-NEXT: v_or_b32_e32 v18, v18, v20
+; GISEL-NEXT: v_or_b32_e32 v28, v0, v2
+; GISEL-NEXT: v_or_b32_e32 v33, v1, v3
; GISEL-NEXT: .LBB4_6: ; %Flow16
; GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
; GISEL-NEXT: v_ashrrev_i32_e32 v32, 31, v7
-; GISEL-NEXT: v_xor_b32_e32 v4, v4, v32
-; GISEL-NEXT: v_xor_b32_e32 v5, v5, v32
-; GISEL-NEXT: v_sub_i32_e32 v4, vcc, v4, v32
+; GISEL-NEXT: v_xor_b32_e32 v1, v4, v32
+; GISEL-NEXT: v_xor_b32_e32 v2, v5, v32
+; GISEL-NEXT: v_xor_b32_e32 v3, v6, v32
+; GISEL-NEXT: v_sub_i32_e32 v6, vcc, v1, v32
+; GISEL-NEXT: v_xor_b32_e32 v5, v7, v32
+; GISEL-NEXT: v_subb_u32_e32 v7, vcc, v2, v32, vcc
+; GISEL-NEXT: v_ashrrev_i32_e32 v0, 31, v15
+; GISEL-NEXT: v_subb_u32_e32 v4, vcc, v3, v32, vcc
+; GISEL-NEXT: v_xor_b32_e32 v12, v12, v0
; GISEL-NEXT: v_subb_u32_e32 v5, vcc, v5, v32, vcc
-; GISEL-NEXT: v_xor_b32_e32 v6, v6, v32
-; GISEL-NEXT: v_xor_b32_e32 v7, v7, v32
-; GISEL-NEXT: v_subb_u32_e32 v6, vcc, v6, v32, vcc
-; GISEL-NEXT: v_ashrrev_i32_e32 v16, 31, v15
-; GISEL-NEXT: v_subb_u32_e32 v7, vcc, v7, v32, vcc
-; GISEL-NEXT: v_xor_b32_e32 v12, v12, v16
-; GISEL-NEXT: v_xor_b32_e32 v13, v13, v16
-; GISEL-NEXT: v_sub_i32_e32 v36, vcc, v12, v16
-; GISEL-NEXT: v_subb_u32_e32 v33, vcc, v13, v16, vcc
-; GISEL-NEXT: v_xor_b32_e32 v12, v14, v16
-; GISEL-NEXT: v_xor_b32_e32 v13, v15, v16
-; GISEL-NEXT: v_subb_u32_e32 v12, vcc, v12, v16, vcc
-; GISEL-NEXT: v_subb_u32_e32 v13, vcc, v13, v16, vcc
-; GISEL-NEXT: v_or_b32_e32 v15, v33, v13
-; GISEL-NEXT: v_or_b32_e32 v14, v36, v12
+; GISEL-NEXT: v_xor_b32_e32 v13, v13, v0
+; GISEL-NEXT: v_sub_i32_e32 v35, vcc, v12, v0
+; GISEL-NEXT: v_xor_b32_e32 v14, v14, v0
+; GISEL-NEXT: v_subb_u32_e32 v34, vcc, v13, v0, vcc
+; GISEL-NEXT: v_xor_b32_e32 v15, v15, v0
+; GISEL-NEXT: v_subb_u32_e32 v12, vcc, v14, v0, vcc
+; GISEL-NEXT: v_subb_u32_e32 v13, vcc, v15, v0, vcc
+; GISEL-NEXT: v_or_b32_e32 v0, v35, v12
+; GISEL-NEXT: v_or_b32_e32 v1, v34, v13
+; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
+; GISEL-NEXT: v_or_b32_e32 v0, v6, v4
+; GISEL-NEXT: v_or_b32_e32 v1, v7, v5
+; GISEL-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[0:1]
+; GISEL-NEXT: v_ffbh_u32_e32 v1, v35
+; GISEL-NEXT: s_or_b64 s[10:11], vcc, s[4:5]
+; GISEL-NEXT: v_ffbh_u32_e32 v0, v34
+; GISEL-NEXT: v_add_i32_e32 v1, vcc, 32, v1
+; GISEL-NEXT: v_min_u32_e32 v0, v0, v1
+; GISEL-NEXT: v_ffbh_u32_e32 v2, v12
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, 64, v0
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, 32, v2
+; GISEL-NEXT: v_ffbh_u32_e32 v1, v13
+; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[12:13]
+; GISEL-NEXT: v_min_u32_e32 v1, v1, v2
+; GISEL-NEXT: v_ffbh_u32_e32 v2, v6
+; GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
+; GISEL-NEXT: v_ffbh_u32_e32 v1, v7
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, 32, v2
+; GISEL-NEXT: v_min_u32_e32 v1, v1, v2
+; GISEL-NEXT: v_ffbh_u32_e32 v3, v4
+; GISEL-NEXT: v_add_i32_e32 v1, vcc, 64, v1
+; GISEL-NEXT: v_add_i32_e32 v3, vcc, 32, v3
+; GISEL-NEXT: v_ffbh_u32_e32 v2, v5
+; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[4:5]
+; GISEL-NEXT: v_min_u32_e32 v2, v2, v3
+; GISEL-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
+; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v0, v1
+; GISEL-NEXT: v_subb_u32_e64 v1, s[4:5], 0, 0, vcc
+; GISEL-NEXT: v_mov_b32_e32 v2, 0x7f
+; GISEL-NEXT: v_mov_b32_e32 v3, 0
+; GISEL-NEXT: v_subb_u32_e64 v14, s[4:5], 0, 0, s[4:5]
+; GISEL-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3]
+; GISEL-NEXT: v_subb_u32_e64 v15, s[4:5], 0, 0, s[4:5]
+; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GISEL-NEXT: v_cmp_lt_u64_e32 vcc, 0, v[14:15]
+; GISEL-NEXT: s_mov_b64 s[6:7], exec
+; GISEL-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[14:15]
-; GISEL-NEXT: v_or_b32_e32 v15, v5, v7
-; GISEL-NEXT: v_or_b32_e32 v14, v4, v6
-; GISEL-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[14:15]
-; GISEL-NEXT: v_ffbh_u32_e32 v14, v12
-; GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GISEL-NEXT: v_add_i32_e32 v14, vcc, 32, v14
-; GISEL-NEXT: v_ffbh_u32_e32 v15, v13
-; GISEL-NEXT: v_min_u32_e32 v14, v14, v15
-; GISEL-NEXT: v_ffbh_u32_e32 v15, v36
-; GISEL-NEXT: v_add_i32_e32 v15, vcc, 32, v15
-; GISEL-NEXT: v_ffbh_u32_e32 v16, v33
-; GISEL-NEXT: v_min_u32_e32 v15, v15, v16
-; GISEL-NEXT: v_add_i32_e32 v15, vcc, 64, v15
-; GISEL-NEXT: v_addc_u32_e64 v16, s[6:7], 0, 0, vcc
-; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[12:13]
-; GISEL-NEXT: v_ffbh_u32_e32 v17, v7
-; GISEL-NEXT: v_cndmask_b32_e32 v14, v15, v14, vcc
-; GISEL-NEXT: v_ffbh_u32_e32 v15, v6
-; GISEL-NEXT: v_cndmask_b32_e64 v16, v16, 0, vcc
-; GISEL-NEXT: v_add_i32_e32 v15, vcc, 32, v15
-; GISEL-NEXT: v_min_u32_e32 v15, v15, v17
-; GISEL-NEXT: v_ffbh_u32_e32 v17, v4
-; GISEL-NEXT: v_add_i32_e32 v17, vcc, 32, v17
-; GISEL-NEXT: v_ffbh_u32_e32 v18, v5
-; GISEL-NEXT: v_min_u32_e32 v17, v17, v18
-; GISEL-NEXT: v_add_i32_e32 v17, vcc, 64, v17
-; GISEL-NEXT: v_addc_u32_e64 v18, s[6:7], 0, 0, vcc
-; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
-; GISEL-NEXT: v_mov_b32_e32 v34, v32
-; GISEL-NEXT: v_cndmask_b32_e32 v15, v17, v15, vcc
-; GISEL-NEXT: v_cndmask_b32_e64 v18, v18, 0, vcc
-; GISEL-NEXT: v_sub_i32_e32 v14, vcc, v14, v15
-; GISEL-NEXT: v_subb_u32_e32 v15, vcc, v16, v18, vcc
-; GISEL-NEXT: v_subb_u32_e64 v18, s[6:7], 0, 0, vcc
-; GISEL-NEXT: v_subb_u32_e64 v19, s[6:7], 0, 0, s[6:7]
-; GISEL-NEXT: s_mov_b64 s[6:7], 0x7f
-; GISEL-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[14:15]
-; GISEL-NEXT: v_or_b32_e32 v21, v15, v19
-; GISEL-NEXT: v_cndmask_b32_e64 v16, 0, 1, vcc
-; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[18:19]
-; GISEL-NEXT: s_movk_i32 s8, 0x7f
-; GISEL-NEXT: v_cndmask_b32_e64 v17, 0, 1, vcc
-; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[18:19]
-; GISEL-NEXT: v_cndmask_b32_e32 v16, v17, v16, vcc
-; GISEL-NEXT: v_and_b32_e32 v16, 1, v16
-; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 1, v16
-; GISEL-NEXT: v_xor_b32_e32 v16, 0x7f, v14
-; GISEL-NEXT: v_or_b32_e32 v20, v16, v18
-; GISEL-NEXT: s_or_b64 s[4:5], s[4:5], vcc
-; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[20:21]
-; GISEL-NEXT: s_xor_b64 s[6:7], s[4:5], -1
-; GISEL-NEXT: v_cndmask_b32_e64 v17, v7, 0, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e64 v16, v6, 0, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e64 v20, v5, 0, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e64 v21, v4, 0, s[4:5]
-; GISEL-NEXT: s_and_b64 s[4:5], s[6:7], vcc
+; GISEL-NEXT: s_mov_b64 s[8:9], 0
+; GISEL-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc
+; GISEL-NEXT: v_and_b32_e32 v2, 1, v2
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
+; GISEL-NEXT: v_xor_b32_e32 v2, 0x7f, v0
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v14
+; GISEL-NEXT: v_or_b32_e32 v3, v1, v15
+; GISEL-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[2:3]
+; GISEL-NEXT: s_or_b64 s[10:11], s[10:11], vcc
+; GISEL-NEXT: s_or_b64 s[4:5], s[10:11], s[4:5]
+; GISEL-NEXT: v_cndmask_b32_e64 v20, v6, 0, s[10:11]
+; GISEL-NEXT: v_cndmask_b32_e64 v21, v7, 0, s[10:11]
+; GISEL-NEXT: v_cndmask_b32_e64 v2, v4, 0, s[10:11]
+; GISEL-NEXT: v_cndmask_b32_e64 v3, v5, 0, s[10:11]
+; GISEL-NEXT: s_xor_b64 s[4:5], s[4:5], s[6:7]
; GISEL-NEXT: s_and_saveexec_b64 s[6:7], s[4:5]
; GISEL-NEXT: s_cbranch_execz .LBB4_12
; GISEL-NEXT: ; %bb.7: ; %udiv-bb1
-; GISEL-NEXT: v_add_i32_e32 v38, vcc, 1, v14
+; GISEL-NEXT: v_add_i32_e32 v36, vcc, 1, v0
+; GISEL-NEXT: v_addc_u32_e32 v37, vcc, 0, v1, vcc
+; GISEL-NEXT: v_addc_u32_e32 v38, vcc, 0, v14, vcc
+; GISEL-NEXT: s_mov_b64 s[4:5], exec
; GISEL-NEXT: v_addc_u32_e32 v39, vcc, 0, v15, vcc
-; GISEL-NEXT: v_addc_u32_e32 v48, vcc, 0, v18, vcc
-; GISEL-NEXT: v_addc_u32_e64 v49, s[4:5], 0, v19, vcc
-; GISEL-NEXT: v_sub_i32_e32 v19, vcc, s8, v14
-; GISEL-NEXT: v_sub_i32_e32 v17, vcc, 64, v19
-; GISEL-NEXT: v_lshl_b64 v[15:16], v[6:7], v19
-; GISEL-NEXT: v_lshr_b64 v[17:18], v[4:5], v17
-; GISEL-NEXT: v_sub_i32_e32 v14, vcc, 63, v14
-; GISEL-NEXT: v_or_b32_e32 v16, v16, v18
-; GISEL-NEXT: v_or_b32_e32 v18, v15, v17
-; GISEL-NEXT: v_lshl_b64 v[14:15], v[4:5], v14
-; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v19
-; GISEL-NEXT: s_xor_b64 s[8:9], s[4:5], -1
-; GISEL-NEXT: v_cndmask_b32_e32 v15, v15, v16, vcc
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v19
-; GISEL-NEXT: v_cndmask_b32_e64 v17, v15, v7, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e32 v16, v14, v18, vcc
-; GISEL-NEXT: v_lshl_b64 v[14:15], v[4:5], v19
-; GISEL-NEXT: v_cndmask_b32_e64 v16, v16, v6, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e32 v15, 0, v15, vcc
-; GISEL-NEXT: v_mov_b32_e32 v18, 0
-; GISEL-NEXT: v_mov_b32_e32 v19, 0
+; GISEL-NEXT: s_xor_b64 s[4:5], vcc, s[4:5]
+; GISEL-NEXT: v_sub_i32_e32 v20, vcc, 0x7f, v0
+; GISEL-NEXT: v_not_b32_e32 v0, 63
+; GISEL-NEXT: v_add_i32_e32 v21, vcc, v20, v0
+; GISEL-NEXT: v_sub_i32_e32 v0, vcc, 64, v20
+; GISEL-NEXT: v_lshr_b64 v[0:1], v[6:7], v0
+; GISEL-NEXT: v_lshl_b64 v[2:3], v[4:5], v20
+; GISEL-NEXT: v_lshl_b64 v[14:15], v[6:7], v20
+; GISEL-NEXT: v_or_b32_e32 v2, v0, v2
+; GISEL-NEXT: v_or_b32_e32 v3, v1, v3
+; GISEL-NEXT: v_lshl_b64 v[0:1], v[6:7], v21
+; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v20
; GISEL-NEXT: v_cndmask_b32_e32 v14, 0, v14, vcc
-; GISEL-NEXT: s_and_saveexec_b64 s[4:5], s[8:9]
-; GISEL-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
+; GISEL-NEXT: v_cndmask_b32_e32 v15, 0, v15, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v20
+; GISEL-NEXT: v_cndmask_b32_e32 v20, v0, v4, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v21, v1, v5, vcc
+; GISEL-NEXT: s_mov_b64 s[10:11], s[8:9]
+; GISEL-NEXT: v_mov_b32_e32 v0, s8
+; GISEL-NEXT: v_mov_b32_e32 v1, s9
+; GISEL-NEXT: v_mov_b32_e32 v2, s10
+; GISEL-NEXT: v_mov_b32_e32 v3, s11
+; GISEL-NEXT: s_and_saveexec_b64 s[8:9], s[4:5]
+; GISEL-NEXT: s_xor_b64 s[12:13], exec, s[8:9]
; GISEL-NEXT: s_cbranch_execz .LBB4_11
; GISEL-NEXT: ; %bb.8: ; %udiv-preheader
-; GISEL-NEXT: v_sub_i32_e32 v20, vcc, 64, v38
-; GISEL-NEXT: v_lshr_b64 v[18:19], v[4:5], v38
-; GISEL-NEXT: v_lshl_b64 v[20:21], v[6:7], v20
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v38
-; GISEL-NEXT: v_or_b32_e32 v20, v18, v20
-; GISEL-NEXT: v_subrev_i32_e32 v18, vcc, 64, v38
-; GISEL-NEXT: v_or_b32_e32 v21, v19, v21
-; GISEL-NEXT: v_lshr_b64 v[18:19], v[6:7], v18
-; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v38
-; GISEL-NEXT: v_cndmask_b32_e32 v19, v19, v21, vcc
-; GISEL-NEXT: v_cndmask_b32_e64 v25, v19, v5, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e32 v20, v18, v20, vcc
-; GISEL-NEXT: v_lshr_b64 v[18:19], v[6:7], v38
-; GISEL-NEXT: v_cndmask_b32_e64 v24, v20, v4, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e32 v27, 0, v19, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v26, 0, v18, vcc
-; GISEL-NEXT: v_add_i32_e32 v50, vcc, -1, v36
-; GISEL-NEXT: v_addc_u32_e32 v51, vcc, -1, v33, vcc
-; GISEL-NEXT: v_addc_u32_e32 v52, vcc, -1, v12, vcc
-; GISEL-NEXT: v_addc_u32_e32 v53, vcc, -1, v13, vcc
-; GISEL-NEXT: v_mov_b32_e32 v20, 0
-; GISEL-NEXT: v_mov_b32_e32 v21, 0
-; GISEL-NEXT: s_mov_b64 s[10:11], 0
-; GISEL-NEXT: v_mov_b32_e32 v22, 0
+; GISEL-NEXT: v_sub_i32_e32 v2, vcc, 64, v36
+; GISEL-NEXT: v_lshr_b64 v[0:1], v[6:7], v36
+; GISEL-NEXT: v_lshl_b64 v[2:3], v[4:5], v2
+; GISEL-NEXT: v_add_i32_e32 v24, vcc, 0xffffffc0, v36
+; GISEL-NEXT: v_lshr_b64 v[22:23], v[4:5], v36
+; GISEL-NEXT: v_or_b32_e32 v2, v0, v2
+; GISEL-NEXT: v_or_b32_e32 v3, v1, v3
+; GISEL-NEXT: v_lshr_b64 v[0:1], v[4:5], v24
+; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v36
+; GISEL-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v26, 0, v22, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v27, 0, v23, vcc
+; GISEL-NEXT: v_add_i32_e32 v48, vcc, -1, v35
+; GISEL-NEXT: v_addc_u32_e32 v49, vcc, -1, v34, vcc
+; GISEL-NEXT: s_mov_b64 s[8:9], 0
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v36
+; GISEL-NEXT: v_addc_u32_e32 v50, vcc, -1, v12, vcc
+; GISEL-NEXT: v_cndmask_b32_e64 v24, v0, v6, s[4:5]
+; GISEL-NEXT: v_cndmask_b32_e64 v25, v1, v7, s[4:5]
+; GISEL-NEXT: v_addc_u32_e32 v51, vcc, -1, v13, vcc
+; GISEL-NEXT: s_mov_b64 s[10:11], s[8:9]
; GISEL-NEXT: v_mov_b32_e32 v23, 0
-; GISEL-NEXT: v_mov_b32_e32 v19, 0
+; GISEL-NEXT: v_mov_b32_e32 v0, s8
+; GISEL-NEXT: v_mov_b32_e32 v1, s9
+; GISEL-NEXT: v_mov_b32_e32 v22, 1
+; GISEL-NEXT: v_mov_b32_e32 v2, s10
+; GISEL-NEXT: v_mov_b32_e32 v3, s11
; GISEL-NEXT: .LBB4_9: ; %udiv-do-while
; GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GISEL-NEXT: v_lshrrev_b32_e32 v18, 31, v25
-; GISEL-NEXT: v_lshl_b64 v[24:25], v[24:25], 1
-; GISEL-NEXT: v_add_i32_e32 v38, vcc, -1, v38
-; GISEL-NEXT: v_lshl_b64 v[26:27], v[26:27], 1
-; GISEL-NEXT: v_lshrrev_b32_e32 v54, 31, v17
+; GISEL-NEXT: v_lshl_b64 v[2:3], v[14:15], 1
+; GISEL-NEXT: v_add_i32_e32 v36, vcc, -1, v36
+; GISEL-NEXT: v_lshrrev_b32_e32 v22, 31, v15
+; GISEL-NEXT: v_or_b32_e32 v14, v0, v2
+; GISEL-NEXT: v_or_b32_e32 v15, v1, v3
+; GISEL-NEXT: v_lshl_b64 v[0:1], v[24:25], 1
+; GISEL-NEXT: v_addc_u32_e32 v37, vcc, -1, v37, vcc
+; GISEL-NEXT: v_lshrrev_b32_e32 v24, 31, v25
+; GISEL-NEXT: v_lshrrev_b32_e32 v25, 31, v21
+; GISEL-NEXT: v_lshl_b64 v[20:21], v[20:21], 1
+; GISEL-NEXT: v_addc_u32_e32 v38, vcc, -1, v38, vcc
+; GISEL-NEXT: v_lshl_b64 v[2:3], v[26:27], 1
+; GISEL-NEXT: v_or_b32_e32 v0, v0, v25
; GISEL-NEXT: v_addc_u32_e32 v39, vcc, -1, v39, vcc
-; GISEL-NEXT: v_or_b32_e32 v24, v24, v54
-; GISEL-NEXT: v_addc_u32_e32 v48, vcc, -1, v48, vcc
-; GISEL-NEXT: v_or_b32_e32 v26, v26, v18
-; GISEL-NEXT: v_sub_i32_e64 v18, s[4:5], v50, v24
-; GISEL-NEXT: v_addc_u32_e32 v49, vcc, -1, v49, vcc
-; GISEL-NEXT: v_lshrrev_b32_e32 v55, 31, v15
-; GISEL-NEXT: v_lshl_b64 v[14:15], v[14:15], 1
-; GISEL-NEXT: v_subb_u32_e64 v18, vcc, v51, v25, s[4:5]
-; GISEL-NEXT: v_subb_u32_e32 v18, vcc, v52, v26, vcc
-; GISEL-NEXT: v_lshl_b64 v[16:17], v[16:17], 1
-; GISEL-NEXT: v_or_b32_e32 v15, v23, v15
-; GISEL-NEXT: v_or_b32_e32 v14, v22, v14
-; GISEL-NEXT: v_or_b32_e32 v22, v38, v48
-; GISEL-NEXT: v_or_b32_e32 v23, v39, v49
-; GISEL-NEXT: v_subb_u32_e32 v18, vcc, v53, v27, vcc
-; GISEL-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[22:23]
-; GISEL-NEXT: v_ashrrev_i32_e32 v22, 31, v18
-; GISEL-NEXT: v_or_b32_e32 v16, v16, v55
-; GISEL-NEXT: v_and_b32_e32 v18, 1, v22
-; GISEL-NEXT: v_and_b32_e32 v54, v22, v13
-; GISEL-NEXT: v_and_b32_e32 v55, v22, v12
-; GISEL-NEXT: v_and_b32_e32 v40, v22, v33
-; GISEL-NEXT: v_and_b32_e32 v22, v22, v36
-; GISEL-NEXT: v_sub_i32_e32 v24, vcc, v24, v22
-; GISEL-NEXT: v_subb_u32_e32 v25, vcc, v25, v40, vcc
-; GISEL-NEXT: v_subb_u32_e32 v26, vcc, v26, v55, vcc
-; GISEL-NEXT: v_or_b32_e32 v17, v21, v17
-; GISEL-NEXT: v_or_b32_e32 v16, v20, v16
-; GISEL-NEXT: s_or_b64 s[10:11], s[4:5], s[10:11]
-; GISEL-NEXT: v_mov_b32_e32 v23, v19
-; GISEL-NEXT: v_mov_b32_e32 v22, v18
-; GISEL-NEXT: v_subb_u32_e32 v27, vcc, v27, v54, vcc
-; GISEL-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GISEL-NEXT: v_or_b32_e32 v20, v20, v22
+; GISEL-NEXT: v_sub_i32_e32 v22, vcc, v48, v0
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v24
+; GISEL-NEXT: v_subb_u32_e32 v22, vcc, v49, v1, vcc
+; GISEL-NEXT: v_subb_u32_e32 v22, vcc, v50, v2, vcc
+; GISEL-NEXT: v_or_b32_e32 v24, v36, v38
+; GISEL-NEXT: v_or_b32_e32 v25, v37, v39
+; GISEL-NEXT: v_subb_u32_e32 v22, vcc, v51, v3, vcc
+; GISEL-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[24:25]
+; GISEL-NEXT: v_ashrrev_i32_e32 v24, 31, v22
+; GISEL-NEXT: v_and_b32_e32 v25, v24, v35
+; GISEL-NEXT: v_and_b32_e32 v22, 1, v24
+; GISEL-NEXT: v_and_b32_e32 v26, v24, v34
+; GISEL-NEXT: v_and_b32_e32 v27, v24, v12
+; GISEL-NEXT: v_and_b32_e32 v52, v24, v13
+; GISEL-NEXT: v_sub_i32_e32 v24, vcc, v0, v25
+; GISEL-NEXT: v_subb_u32_e32 v25, vcc, v1, v26, vcc
+; GISEL-NEXT: v_subb_u32_e32 v26, vcc, v2, v27, vcc
+; GISEL-NEXT: s_or_b64 s[8:9], s[4:5], s[8:9]
+; GISEL-NEXT: v_mov_b32_e32 v0, v22
+; GISEL-NEXT: v_mov_b32_e32 v1, v23
+; GISEL-NEXT: v_subb_u32_e32 v27, vcc, v3, v52, vcc
+; GISEL-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GISEL-NEXT: s_cbranch_execnz .LBB4_9
; GISEL-NEXT: ; %bb.10: ; %Flow
-; GISEL-NEXT: s_or_b64 exec, exec, s[10:11]
-; GISEL-NEXT: .LBB4_11: ; %Flow11
; GISEL-NEXT: s_or_b64 exec, exec, s[8:9]
-; GISEL-NEXT: v_lshl_b64 v[16:17], v[16:17], 1
-; GISEL-NEXT: v_lshrrev_b32_e32 v20, 31, v15
-; GISEL-NEXT: v_lshl_b64 v[14:15], v[14:15], 1
-; GISEL-NEXT: v_or_b32_e32 v16, v16, v20
-; GISEL-NEXT: v_or_b32_e32 v20, v19, v15
-; GISEL-NEXT: v_or_b32_e32 v21, v18, v14
+; GISEL-NEXT: .LBB4_11: ; %Flow11
+; GISEL-NEXT: s_or_b64 exec, exec, s[12:13]
+; GISEL-NEXT: v_lshl_b64 v[22:23], v[14:15], 1
+; GISEL-NEXT: v_lshl_b64 v[2:3], v[20:21], 1
+; GISEL-NEXT: v_lshrrev_b32_e32 v14, 31, v15
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v14
+; GISEL-NEXT: v_or_b32_e32 v20, v0, v22
+; GISEL-NEXT: v_or_b32_e32 v21, v1, v23
; GISEL-NEXT: .LBB4_12: ; %Flow12
; GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
-; GISEL-NEXT: buffer_load_dword v40, off, s[0:3], s32 ; 4-byte Folded Reload
-; GISEL-NEXT: v_mul_lo_u32 v9, v37, v9
-; GISEL-NEXT: v_mad_u64_u32 v[14:15], s[4:5], v37, v8, 0
-; GISEL-NEXT: v_mad_u64_u32 v[22:23], s[4:5], v30, v37, 0
-; GISEL-NEXT: v_mul_lo_u32 v18, v35, v8
-; GISEL-NEXT: v_mov_b32_e32 v24, 0
-; GISEL-NEXT: v_add_i32_e32 v15, vcc, v15, v9
-; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v29, v37, v[23:24]
-; GISEL-NEXT: v_add_i32_e32 v15, vcc, v15, v18
-; GISEL-NEXT: v_mad_u64_u32 v[14:15], s[4:5], v10, v30, v[14:15]
-; GISEL-NEXT: v_mul_lo_u32 v11, v11, v30
-; GISEL-NEXT: v_mov_b32_e32 v23, v8
-; GISEL-NEXT: v_mad_u64_u32 v[18:19], s[4:5], v30, v35, v[23:24]
-; GISEL-NEXT: v_mul_lo_u32 v10, v10, v29
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v15
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v19
-; GISEL-NEXT: v_addc_u32_e64 v9, s[4:5], 0, 0, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v29, v35, v[8:9]
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v11
-; GISEL-NEXT: v_mul_lo_u32 v15, v16, v33
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v14
-; GISEL-NEXT: v_addc_u32_e32 v9, vcc, v9, v10, vcc
-; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v0, v22
-; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v1, v18, vcc
-; GISEL-NEXT: v_subb_u32_e32 v2, vcc, v2, v8, vcc
-; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v3, v9, vcc
-; GISEL-NEXT: v_xor_b32_e32 v0, v0, v28
-; GISEL-NEXT: v_mul_lo_u32 v10, v21, v13
-; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v21, v12, 0
-; GISEL-NEXT: v_mad_u64_u32 v[22:23], s[4:5], v36, v21, 0
-; GISEL-NEXT: v_xor_b32_e32 v1, v1, v31
-; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v0, v28
-; GISEL-NEXT: v_xor_b32_e32 v2, v2, v28
-; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v1, v31, vcc
-; GISEL-NEXT: v_xor_b32_e32 v3, v3, v31
-; GISEL-NEXT: v_subb_u32_e32 v2, vcc, v2, v28, vcc
-; GISEL-NEXT: v_mul_lo_u32 v12, v20, v12
-; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v3, v31, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
-; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v33, v21, v[23:24]
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v12
-; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v16, v36, v[8:9]
-; GISEL-NEXT: v_mul_lo_u32 v14, v17, v36
-; GISEL-NEXT: v_mov_b32_e32 v23, v10
-; GISEL-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v36, v20, v[23:24]
-; GISEL-NEXT: v_add_i32_e32 v14, vcc, v14, v9
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v11, v13
-; GISEL-NEXT: v_addc_u32_e64 v10, s[4:5], 0, 0, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v33, v20, v[9:10]
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v15, v14
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v8
-; GISEL-NEXT: v_addc_u32_e32 v9, vcc, v10, v11, vcc
-; GISEL-NEXT: v_sub_i32_e32 v4, vcc, v4, v22
-; GISEL-NEXT: v_subb_u32_e32 v5, vcc, v5, v12, vcc
-; GISEL-NEXT: v_subb_u32_e32 v6, vcc, v6, v8, vcc
-; GISEL-NEXT: v_subb_u32_e32 v7, vcc, v7, v9, vcc
-; GISEL-NEXT: v_xor_b32_e32 v4, v4, v32
-; GISEL-NEXT: v_xor_b32_e32 v5, v5, v34
-; GISEL-NEXT: v_sub_i32_e32 v4, vcc, v4, v32
+; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v31, v18, 0
+; GISEL-NEXT: v_mad_u64_u32 v[22:23], s[4:5], v31, v28, 0
+; GISEL-NEXT: v_mul_lo_u32 v36, v30, v18
+; GISEL-NEXT: v_mul_lo_u32 v3, v35, v3
+; GISEL-NEXT: v_mad_u64_u32 v[14:15], s[4:5], v30, v33, v[0:1]
+; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v35, v2, 0
+; GISEL-NEXT: v_mul_lo_u32 v2, v34, v2
+; GISEL-NEXT: v_mad_u64_u32 v[24:25], s[4:5], v16, v28, v[14:15]
+; GISEL-NEXT: v_mad_u64_u32 v[14:15], s[4:5], v35, v20, 0
+; GISEL-NEXT: v_mad_u64_u32 v[26:27], vcc, v31, v33, v[23:24]
+; GISEL-NEXT: v_mul_lo_u32 v23, v31, v19
+; GISEL-NEXT: v_mad_u64_u32 v[18:19], s[4:5], v34, v21, v[0:1]
+; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v30, v28, v[26:27]
+; GISEL-NEXT: v_addc_u32_e64 v23, s[4:5], v25, v23, s[4:5]
+; GISEL-NEXT: v_addc_u32_e32 v23, vcc, v23, v36, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[24:25], s[4:5], v16, v33, v[23:24]
+; GISEL-NEXT: v_sub_i32_e32 v10, vcc, v10, v22
+; GISEL-NEXT: v_subb_u32_e32 v0, vcc, v11, v0, vcc
+; GISEL-NEXT: v_xor_b32_e32 v22, v10, v29
+; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v17, v28, v[24:25]
+; GISEL-NEXT: v_mad_u64_u32 v[16:17], s[4:5], v12, v20, v[18:19]
+; GISEL-NEXT: v_xor_b32_e32 v23, v0, v29
+; GISEL-NEXT: v_subb_u32_e32 v0, vcc, v8, v1, vcc
+; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v9, v10, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[8:9], vcc, v35, v21, v[15:16]
+; GISEL-NEXT: v_xor_b32_e32 v18, v0, v29
+; GISEL-NEXT: v_sub_i32_e64 v0, s[4:5], v22, v29
+; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[6:7], v34, v20, v[8:9]
+; GISEL-NEXT: v_addc_u32_e64 v3, s[6:7], v17, v3, s[6:7]
+; GISEL-NEXT: v_addc_u32_e32 v2, vcc, v3, v2, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[6:7], v12, v21, v[2:3]
+; GISEL-NEXT: v_xor_b32_e32 v19, v1, v29
+; GISEL-NEXT: v_subb_u32_e64 v1, s[4:5], v23, v29, s[4:5]
+; GISEL-NEXT: v_subb_u32_e64 v2, vcc, v18, v29, s[4:5]
+; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v19, v29, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[15:16], s[4:5], v13, v20, v[8:9]
+; GISEL-NEXT: v_sub_i32_e32 v6, vcc, v6, v14
+; GISEL-NEXT: v_subb_u32_e32 v7, vcc, v7, v10, vcc
+; GISEL-NEXT: v_subb_u32_e32 v4, vcc, v4, v11, vcc
+; GISEL-NEXT: v_subb_u32_e32 v5, vcc, v5, v15, vcc
; GISEL-NEXT: v_xor_b32_e32 v6, v6, v32
-; GISEL-NEXT: v_subb_u32_e32 v5, vcc, v5, v34, vcc
-; GISEL-NEXT: v_xor_b32_e32 v7, v7, v34
-; GISEL-NEXT: v_subb_u32_e32 v6, vcc, v6, v32, vcc
-; GISEL-NEXT: v_subb_u32_e32 v7, vcc, v7, v34, vcc
-; GISEL-NEXT: s_waitcnt vmcnt(0)
+; GISEL-NEXT: v_xor_b32_e32 v7, v7, v32
+; GISEL-NEXT: v_xor_b32_e32 v8, v4, v32
+; GISEL-NEXT: v_sub_i32_e32 v4, vcc, v6, v32
+; GISEL-NEXT: v_xor_b32_e32 v9, v5, v32
+; GISEL-NEXT: v_subb_u32_e32 v5, vcc, v7, v32, vcc
+; GISEL-NEXT: v_subb_u32_e32 v6, vcc, v8, v32, vcc
+; GISEL-NEXT: v_subb_u32_e32 v7, vcc, v9, v32, vcc
; GISEL-NEXT: s_setpc_b64 s[30:31]
%shl = srem <2 x i128> %lhs, %rhs
ret <2 x i128> %shl
@@ -2902,391 +2898,382 @@ define <2 x i128> @v_urem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-LABEL: v_urem_v2i128_vv:
; GISEL: ; %bb.0: ; %_udiv-special-cases_udiv-special-cases
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT: v_or_b32_e32 v17, v9, v11
; GISEL-NEXT: v_or_b32_e32 v16, v8, v10
+; GISEL-NEXT: v_or_b32_e32 v17, v9, v11
; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[16:17]
-; GISEL-NEXT: v_or_b32_e32 v17, v1, v3
; GISEL-NEXT: v_or_b32_e32 v16, v0, v2
+; GISEL-NEXT: v_or_b32_e32 v17, v1, v3
; GISEL-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[16:17]
-; GISEL-NEXT: v_ffbh_u32_e32 v16, v10
-; GISEL-NEXT: v_add_i32_e64 v16, s[6:7], 32, v16
-; GISEL-NEXT: v_ffbh_u32_e32 v17, v11
-; GISEL-NEXT: v_min_u32_e32 v16, v16, v17
; GISEL-NEXT: v_ffbh_u32_e32 v17, v8
+; GISEL-NEXT: v_ffbh_u32_e32 v16, v9
; GISEL-NEXT: v_add_i32_e64 v17, s[6:7], 32, v17
-; GISEL-NEXT: v_ffbh_u32_e32 v18, v9
-; GISEL-NEXT: v_min_u32_e32 v17, v17, v18
-; GISEL-NEXT: v_add_i32_e64 v17, s[6:7], 64, v17
-; GISEL-NEXT: v_addc_u32_e64 v18, s[6:7], 0, 0, s[6:7]
-; GISEL-NEXT: v_cmp_ne_u64_e64 s[6:7], 0, v[10:11]
-; GISEL-NEXT: v_ffbh_u32_e32 v20, v1
-; GISEL-NEXT: v_cndmask_b32_e64 v16, v17, v16, s[6:7]
-; GISEL-NEXT: v_ffbh_u32_e32 v17, v2
-; GISEL-NEXT: v_cndmask_b32_e64 v19, v18, 0, s[6:7]
-; GISEL-NEXT: v_add_i32_e64 v17, s[6:7], 32, v17
-; GISEL-NEXT: v_ffbh_u32_e32 v18, v3
+; GISEL-NEXT: v_min_u32_e32 v16, v16, v17
+; GISEL-NEXT: v_ffbh_u32_e32 v18, v10
+; GISEL-NEXT: v_add_i32_e64 v16, s[6:7], 64, v16
+; GISEL-NEXT: v_add_i32_e64 v18, s[6:7], 32, v18
+; GISEL-NEXT: v_ffbh_u32_e32 v17, v11
+; GISEL-NEXT: v_cmp_eq_u64_e64 s[6:7], 0, v[10:11]
; GISEL-NEXT: v_min_u32_e32 v17, v17, v18
; GISEL-NEXT: v_ffbh_u32_e32 v18, v0
+; GISEL-NEXT: v_cndmask_b32_e64 v16, v17, v16, s[6:7]
+; GISEL-NEXT: v_ffbh_u32_e32 v17, v1
; GISEL-NEXT: v_add_i32_e64 v18, s[6:7], 32, v18
-; GISEL-NEXT: v_min_u32_e32 v18, v18, v20
-; GISEL-NEXT: v_add_i32_e64 v18, s[6:7], 64, v18
-; GISEL-NEXT: v_addc_u32_e64 v20, s[6:7], 0, 0, s[6:7]
-; GISEL-NEXT: v_cmp_ne_u64_e64 s[6:7], 0, v[2:3]
-; GISEL-NEXT: s_mov_b64 s[8:9], 0x7f
+; GISEL-NEXT: v_min_u32_e32 v17, v17, v18
+; GISEL-NEXT: v_ffbh_u32_e32 v19, v2
+; GISEL-NEXT: v_add_i32_e64 v17, s[6:7], 64, v17
+; GISEL-NEXT: v_add_i32_e64 v19, s[6:7], 32, v19
+; GISEL-NEXT: v_ffbh_u32_e32 v18, v3
+; GISEL-NEXT: v_cmp_eq_u64_e64 s[6:7], 0, v[2:3]
+; GISEL-NEXT: v_min_u32_e32 v18, v18, v19
; GISEL-NEXT: v_cndmask_b32_e64 v17, v18, v17, s[6:7]
-; GISEL-NEXT: v_cndmask_b32_e64 v20, v20, 0, s[6:7]
-; GISEL-NEXT: v_sub_i32_e64 v18, s[6:7], v16, v17
-; GISEL-NEXT: v_subb_u32_e64 v19, s[6:7], v19, v20, s[6:7]
-; GISEL-NEXT: v_subb_u32_e64 v20, s[6:7], 0, 0, s[6:7]
-; GISEL-NEXT: v_subb_u32_e64 v21, s[6:7], 0, 0, s[6:7]
-; GISEL-NEXT: v_xor_b32_e32 v16, 0x7f, v18
-; GISEL-NEXT: v_cmp_lt_u64_e64 s[8:9], s[8:9], v[18:19]
-; GISEL-NEXT: v_or_b32_e32 v16, v16, v20
-; GISEL-NEXT: v_or_b32_e32 v17, v19, v21
-; GISEL-NEXT: v_cmp_ne_u64_e64 s[6:7], 0, v[16:17]
-; GISEL-NEXT: v_cndmask_b32_e64 v16, 0, 1, s[8:9]
-; GISEL-NEXT: v_cmp_ne_u64_e64 s[8:9], 0, v[20:21]
+; GISEL-NEXT: v_sub_i32_e64 v16, s[6:7], v16, v17
+; GISEL-NEXT: v_mov_b32_e32 v20, 0x7f
+; GISEL-NEXT: v_mov_b32_e32 v21, 0
+; GISEL-NEXT: v_subb_u32_e64 v17, s[6:7], 0, 0, s[6:7]
+; GISEL-NEXT: v_subb_u32_e64 v18, s[6:7], 0, 0, s[6:7]
+; GISEL-NEXT: v_cmp_gt_u64_e64 s[8:9], v[16:17], v[20:21]
+; GISEL-NEXT: v_subb_u32_e64 v19, s[6:7], 0, 0, s[6:7]
+; GISEL-NEXT: v_cndmask_b32_e64 v20, 0, 1, s[8:9]
+; GISEL-NEXT: v_cmp_lt_u64_e64 s[8:9], 0, v[18:19]
+; GISEL-NEXT: v_xor_b32_e32 v22, 0x7f, v16
+; GISEL-NEXT: v_cndmask_b32_e64 v21, 0, 1, s[8:9]
+; GISEL-NEXT: v_cmp_eq_u64_e64 s[8:9], 0, v[18:19]
+; GISEL-NEXT: v_or_b32_e32 v22, v22, v18
+; GISEL-NEXT: v_cndmask_b32_e64 v20, v21, v20, s[8:9]
+; GISEL-NEXT: v_or_b32_e32 v23, v17, v19
+; GISEL-NEXT: v_and_b32_e32 v20, 1, v20
+; GISEL-NEXT: v_cmp_eq_u64_e64 s[6:7], 0, v[22:23]
+; GISEL-NEXT: v_cmp_ne_u32_e64 s[8:9], 0, v20
; GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e64 v17, 0, 1, s[8:9]
-; GISEL-NEXT: v_cmp_eq_u64_e64 s[8:9], 0, v[20:21]
-; GISEL-NEXT: v_cndmask_b32_e64 v16, v17, v16, s[8:9]
-; GISEL-NEXT: v_and_b32_e32 v16, 1, v16
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[8:9], 1, v16
; GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[8:9]
-; GISEL-NEXT: s_xor_b64 s[10:11], s[4:5], -1
-; GISEL-NEXT: v_cndmask_b32_e64 v17, v3, 0, s[4:5]
-; GISEL-NEXT: s_movk_i32 s8, 0x7f
-; GISEL-NEXT: v_cndmask_b32_e64 v16, v2, 0, s[4:5]
-; GISEL-NEXT: s_and_b64 s[10:11], s[10:11], s[6:7]
-; GISEL-NEXT: v_cndmask_b32_e64 v32, v1, 0, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e64 v33, v0, 0, s[4:5]
-; GISEL-NEXT: s_and_saveexec_b64 s[6:7], s[10:11]
+; GISEL-NEXT: s_mov_b64 s[10:11], exec
+; GISEL-NEXT: s_or_b64 s[6:7], s[4:5], s[6:7]
+; GISEL-NEXT: s_mov_b64 s[12:13], 0
+; GISEL-NEXT: v_cndmask_b32_e64 v32, v0, 0, s[4:5]
+; GISEL-NEXT: v_cndmask_b32_e64 v20, v2, 0, s[4:5]
+; GISEL-NEXT: v_cndmask_b32_e64 v21, v3, 0, s[4:5]
+; GISEL-NEXT: s_xor_b64 s[8:9], s[6:7], s[10:11]
+; GISEL-NEXT: v_cndmask_b32_e64 v33, v1, 0, s[4:5]
+; GISEL-NEXT: s_and_saveexec_b64 s[6:7], s[8:9]
; GISEL-NEXT: s_cbranch_execz .LBB5_6
; GISEL-NEXT: ; %bb.1: ; %udiv-bb15
-; GISEL-NEXT: v_add_i32_e32 v30, vcc, 1, v18
-; GISEL-NEXT: v_addc_u32_e32 v31, vcc, 0, v19, vcc
-; GISEL-NEXT: v_addc_u32_e32 v32, vcc, 0, v20, vcc
-; GISEL-NEXT: v_addc_u32_e64 v33, s[4:5], 0, v21, vcc
-; GISEL-NEXT: v_sub_i32_e32 v21, vcc, s8, v18
-; GISEL-NEXT: v_sub_i32_e32 v19, vcc, 64, v21
-; GISEL-NEXT: v_lshl_b64 v[16:17], v[2:3], v21
-; GISEL-NEXT: v_lshr_b64 v[19:20], v[0:1], v19
-; GISEL-NEXT: s_xor_b64 s[8:9], s[4:5], -1
-; GISEL-NEXT: v_or_b32_e32 v19, v16, v19
-; GISEL-NEXT: v_sub_i32_e32 v16, vcc, 63, v18
-; GISEL-NEXT: v_or_b32_e32 v20, v17, v20
-; GISEL-NEXT: v_lshl_b64 v[16:17], v[0:1], v16
-; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v21
-; GISEL-NEXT: v_cndmask_b32_e32 v16, v16, v19, vcc
-; GISEL-NEXT: v_lshl_b64 v[18:19], v[0:1], v21
-; GISEL-NEXT: v_cndmask_b32_e32 v17, v17, v20, vcc
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v21
-; GISEL-NEXT: v_cndmask_b32_e64 v17, v17, v3, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e64 v16, v16, v2, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e32 v19, 0, v19, vcc
-; GISEL-NEXT: v_mov_b32_e32 v20, 0
-; GISEL-NEXT: v_mov_b32_e32 v21, 0
-; GISEL-NEXT: v_cndmask_b32_e32 v18, 0, v18, vcc
-; GISEL-NEXT: s_and_saveexec_b64 s[4:5], s[8:9]
-; GISEL-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
+; GISEL-NEXT: v_add_i32_e32 v29, vcc, 1, v16
+; GISEL-NEXT: v_addc_u32_e32 v30, vcc, 0, v17, vcc
+; GISEL-NEXT: v_addc_u32_e32 v31, vcc, 0, v18, vcc
+; GISEL-NEXT: s_mov_b64 s[4:5], exec
+; GISEL-NEXT: v_addc_u32_e32 v32, vcc, 0, v19, vcc
+; GISEL-NEXT: s_xor_b64 s[4:5], vcc, s[4:5]
+; GISEL-NEXT: v_sub_i32_e32 v20, vcc, 0x7f, v16
+; GISEL-NEXT: v_not_b32_e32 v16, 63
+; GISEL-NEXT: v_add_i32_e32 v23, vcc, v20, v16
+; GISEL-NEXT: v_sub_i32_e32 v16, vcc, 64, v20
+; GISEL-NEXT: v_lshr_b64 v[16:17], v[0:1], v16
+; GISEL-NEXT: v_lshl_b64 v[18:19], v[2:3], v20
+; GISEL-NEXT: v_lshl_b64 v[21:22], v[0:1], v20
+; GISEL-NEXT: v_or_b32_e32 v18, v16, v18
+; GISEL-NEXT: v_or_b32_e32 v19, v17, v19
+; GISEL-NEXT: v_lshl_b64 v[16:17], v[0:1], v23
+; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v20
+; GISEL-NEXT: v_cndmask_b32_e32 v21, 0, v21, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v22, 0, v22, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v16, v16, v18, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v17, v17, v19, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v20
+; GISEL-NEXT: v_cndmask_b32_e32 v23, v16, v2, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v24, v17, v3, vcc
+; GISEL-NEXT: s_mov_b64 s[14:15], s[12:13]
+; GISEL-NEXT: v_mov_b32_e32 v17, s13
+; GISEL-NEXT: v_mov_b32_e32 v16, s12
+; GISEL-NEXT: v_mov_b32_e32 v19, s15
+; GISEL-NEXT: v_mov_b32_e32 v18, s14
+; GISEL-NEXT: s_and_saveexec_b64 s[8:9], s[4:5]
+; GISEL-NEXT: s_xor_b64 s[12:13], exec, s[8:9]
; GISEL-NEXT: s_cbranch_execz .LBB5_5
; GISEL-NEXT: ; %bb.2: ; %udiv-preheader4
-; GISEL-NEXT: v_sub_i32_e32 v22, vcc, 64, v30
-; GISEL-NEXT: v_lshr_b64 v[20:21], v[0:1], v30
-; GISEL-NEXT: v_lshl_b64 v[22:23], v[2:3], v22
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v30
-; GISEL-NEXT: v_or_b32_e32 v22, v20, v22
-; GISEL-NEXT: v_subrev_i32_e32 v20, vcc, 64, v30
-; GISEL-NEXT: v_or_b32_e32 v23, v21, v23
-; GISEL-NEXT: v_lshr_b64 v[20:21], v[2:3], v20
-; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v30
-; GISEL-NEXT: v_cndmask_b32_e32 v21, v21, v23, vcc
-; GISEL-NEXT: v_cndmask_b32_e64 v23, v21, v1, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e32 v22, v20, v22, vcc
-; GISEL-NEXT: v_lshr_b64 v[20:21], v[2:3], v30
-; GISEL-NEXT: v_cndmask_b32_e64 v22, v22, v0, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e32 v27, 0, v21, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v26, 0, v20, vcc
-; GISEL-NEXT: v_add_i32_e32 v34, vcc, -1, v8
-; GISEL-NEXT: v_addc_u32_e32 v35, vcc, -1, v9, vcc
-; GISEL-NEXT: v_addc_u32_e32 v36, vcc, -1, v10, vcc
-; GISEL-NEXT: v_addc_u32_e32 v37, vcc, -1, v11, vcc
-; GISEL-NEXT: v_mov_b32_e32 v24, 0
-; GISEL-NEXT: v_mov_b32_e32 v25, 0
-; GISEL-NEXT: s_mov_b64 s[4:5], 0
-; GISEL-NEXT: v_mov_b32_e32 v28, 0
-; GISEL-NEXT: v_mov_b32_e32 v29, 0
-; GISEL-NEXT: v_mov_b32_e32 v21, 0
+; GISEL-NEXT: v_sub_i32_e32 v18, vcc, 64, v29
+; GISEL-NEXT: v_lshr_b64 v[16:17], v[0:1], v29
+; GISEL-NEXT: v_lshl_b64 v[18:19], v[2:3], v18
+; GISEL-NEXT: v_add_i32_e32 v20, vcc, 0xffffffc0, v29
+; GISEL-NEXT: v_lshr_b64 v[27:28], v[2:3], v29
+; GISEL-NEXT: v_or_b32_e32 v18, v16, v18
+; GISEL-NEXT: v_or_b32_e32 v19, v17, v19
+; GISEL-NEXT: v_lshr_b64 v[16:17], v[2:3], v20
+; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v29
+; GISEL-NEXT: v_cndmask_b32_e32 v16, v16, v18, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v17, v17, v19, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v27, 0, v27, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v28, 0, v28, vcc
+; GISEL-NEXT: v_add_i32_e32 v33, vcc, -1, v8
+; GISEL-NEXT: v_addc_u32_e32 v34, vcc, -1, v9, vcc
+; GISEL-NEXT: s_mov_b64 s[8:9], 0
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v29
+; GISEL-NEXT: v_addc_u32_e32 v35, vcc, -1, v10, vcc
+; GISEL-NEXT: v_cndmask_b32_e64 v25, v16, v0, s[4:5]
+; GISEL-NEXT: v_cndmask_b32_e64 v26, v17, v1, s[4:5]
+; GISEL-NEXT: v_addc_u32_e32 v36, vcc, -1, v11, vcc
+; GISEL-NEXT: s_mov_b64 s[10:11], s[8:9]
+; GISEL-NEXT: v_mov_b32_e32 v19, 1
+; GISEL-NEXT: v_mov_b32_e32 v20, 0
+; GISEL-NEXT: v_mov_b32_e32 v17, s9
+; GISEL-NEXT: v_mov_b32_e32 v16, s8
+; GISEL-NEXT: v_mov_b32_e32 v19, s11
+; GISEL-NEXT: v_mov_b32_e32 v18, s10
; GISEL-NEXT: .LBB5_3: ; %udiv-do-while3
; GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GISEL-NEXT: v_lshl_b64 v[38:39], v[18:19], 1
-; GISEL-NEXT: v_lshl_b64 v[26:27], v[26:27], 1
-; GISEL-NEXT: v_or_b32_e32 v18, v28, v38
-; GISEL-NEXT: v_lshrrev_b32_e32 v28, 31, v23
-; GISEL-NEXT: v_lshl_b64 v[22:23], v[22:23], 1
-; GISEL-NEXT: v_or_b32_e32 v26, v26, v28
-; GISEL-NEXT: v_lshrrev_b32_e32 v28, 31, v17
-; GISEL-NEXT: v_or_b32_e32 v22, v22, v28
-; GISEL-NEXT: v_sub_i32_e32 v28, vcc, v34, v22
-; GISEL-NEXT: v_subb_u32_e32 v28, vcc, v35, v23, vcc
-; GISEL-NEXT: v_subb_u32_e32 v28, vcc, v36, v26, vcc
-; GISEL-NEXT: v_subb_u32_e32 v28, vcc, v37, v27, vcc
-; GISEL-NEXT: v_ashrrev_i32_e32 v38, 31, v28
-; GISEL-NEXT: v_lshrrev_b32_e32 v20, 31, v19
-; GISEL-NEXT: v_or_b32_e32 v19, v29, v39
-; GISEL-NEXT: v_and_b32_e32 v29, v38, v8
-; GISEL-NEXT: v_and_b32_e32 v28, v38, v9
-; GISEL-NEXT: v_sub_i32_e32 v22, vcc, v22, v29
-; GISEL-NEXT: v_subb_u32_e32 v23, vcc, v23, v28, vcc
-; GISEL-NEXT: v_and_b32_e32 v29, v38, v10
-; GISEL-NEXT: v_and_b32_e32 v28, v38, v11
-; GISEL-NEXT: v_subb_u32_e32 v26, vcc, v26, v29, vcc
-; GISEL-NEXT: v_subb_u32_e32 v27, vcc, v27, v28, vcc
-; GISEL-NEXT: v_add_i32_e32 v30, vcc, -1, v30
+; GISEL-NEXT: v_lshl_b64 v[18:19], v[21:22], 1
+; GISEL-NEXT: v_lshrrev_b32_e32 v37, 31, v22
+; GISEL-NEXT: v_or_b32_e32 v21, v16, v18
+; GISEL-NEXT: v_or_b32_e32 v22, v17, v19
+; GISEL-NEXT: v_lshl_b64 v[18:19], v[25:26], 1
+; GISEL-NEXT: v_lshrrev_b32_e32 v25, 31, v24
+; GISEL-NEXT: v_lshl_b64 v[16:17], v[27:28], 1
+; GISEL-NEXT: v_or_b32_e32 v18, v18, v25
+; GISEL-NEXT: v_lshrrev_b32_e32 v27, 31, v26
+; GISEL-NEXT: v_sub_i32_e32 v25, vcc, v33, v18
+; GISEL-NEXT: v_or_b32_e32 v16, v16, v27
+; GISEL-NEXT: v_subb_u32_e32 v25, vcc, v34, v19, vcc
+; GISEL-NEXT: v_subb_u32_e32 v25, vcc, v35, v16, vcc
+; GISEL-NEXT: v_subb_u32_e32 v25, vcc, v36, v17, vcc
+; GISEL-NEXT: v_ashrrev_i32_e32 v38, 31, v25
+; GISEL-NEXT: v_and_b32_e32 v25, v38, v8
+; GISEL-NEXT: v_and_b32_e32 v26, v38, v9
+; GISEL-NEXT: v_sub_i32_e32 v25, vcc, v18, v25
+; GISEL-NEXT: v_subb_u32_e32 v26, vcc, v19, v26, vcc
+; GISEL-NEXT: v_and_b32_e32 v18, v38, v10
+; GISEL-NEXT: v_and_b32_e32 v19, v38, v11
+; GISEL-NEXT: v_subb_u32_e32 v27, vcc, v16, v18, vcc
+; GISEL-NEXT: v_subb_u32_e32 v28, vcc, v17, v19, vcc
+; GISEL-NEXT: v_add_i32_e32 v29, vcc, -1, v29
+; GISEL-NEXT: v_addc_u32_e32 v30, vcc, -1, v30, vcc
; GISEL-NEXT: v_addc_u32_e32 v31, vcc, -1, v31, vcc
; GISEL-NEXT: v_addc_u32_e32 v32, vcc, -1, v32, vcc
-; GISEL-NEXT: v_addc_u32_e32 v33, vcc, -1, v33, vcc
-; GISEL-NEXT: v_or_b32_e32 v29, v31, v33
-; GISEL-NEXT: v_or_b32_e32 v28, v30, v32
-; GISEL-NEXT: v_lshl_b64 v[16:17], v[16:17], 1
-; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[28:29]
-; GISEL-NEXT: v_or_b32_e32 v16, v16, v20
-; GISEL-NEXT: v_and_b32_e32 v20, 1, v38
-; GISEL-NEXT: v_or_b32_e32 v17, v25, v17
-; GISEL-NEXT: v_or_b32_e32 v16, v24, v16
-; GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GISEL-NEXT: v_mov_b32_e32 v29, v21
-; GISEL-NEXT: v_mov_b32_e32 v28, v20
-; GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GISEL-NEXT: v_or_b32_e32 v16, v29, v31
+; GISEL-NEXT: v_or_b32_e32 v17, v30, v32
+; GISEL-NEXT: v_lshl_b64 v[23:24], v[23:24], 1
+; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[16:17]
+; GISEL-NEXT: v_and_b32_e32 v19, 1, v38
+; GISEL-NEXT: v_or_b32_e32 v23, v23, v37
+; GISEL-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GISEL-NEXT: v_mov_b32_e32 v16, v19
+; GISEL-NEXT: v_mov_b32_e32 v17, v20
+; GISEL-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GISEL-NEXT: s_cbranch_execnz .LBB5_3
; GISEL-NEXT: ; %bb.4: ; %Flow13
-; GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
-; GISEL-NEXT: .LBB5_5: ; %Flow14
; GISEL-NEXT: s_or_b64 exec, exec, s[8:9]
-; GISEL-NEXT: v_lshl_b64 v[16:17], v[16:17], 1
-; GISEL-NEXT: v_lshrrev_b32_e32 v22, 31, v19
-; GISEL-NEXT: v_lshl_b64 v[18:19], v[18:19], 1
-; GISEL-NEXT: v_or_b32_e32 v16, v16, v22
-; GISEL-NEXT: v_or_b32_e32 v32, v21, v19
-; GISEL-NEXT: v_or_b32_e32 v33, v20, v18
+; GISEL-NEXT: .LBB5_5: ; %Flow14
+; GISEL-NEXT: s_or_b64 exec, exec, s[12:13]
+; GISEL-NEXT: v_lshl_b64 v[18:19], v[21:22], 1
+; GISEL-NEXT: v_lshl_b64 v[20:21], v[23:24], 1
+; GISEL-NEXT: v_lshrrev_b32_e32 v22, 31, v22
+; GISEL-NEXT: v_or_b32_e32 v20, v20, v22
+; GISEL-NEXT: v_or_b32_e32 v32, v16, v18
+; GISEL-NEXT: v_or_b32_e32 v33, v17, v19
; GISEL-NEXT: .LBB5_6: ; %Flow16
; GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
-; GISEL-NEXT: v_or_b32_e32 v19, v13, v15
-; GISEL-NEXT: v_or_b32_e32 v18, v12, v14
-; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[18:19]
-; GISEL-NEXT: v_or_b32_e32 v19, v5, v7
-; GISEL-NEXT: v_or_b32_e32 v18, v4, v6
-; GISEL-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[18:19]
+; GISEL-NEXT: v_or_b32_e32 v16, v12, v14
+; GISEL-NEXT: v_or_b32_e32 v17, v13, v15
+; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[16:17]
+; GISEL-NEXT: v_or_b32_e32 v16, v4, v6
+; GISEL-NEXT: v_or_b32_e32 v17, v5, v7
+; GISEL-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[16:17]
+; GISEL-NEXT: v_ffbh_u32_e32 v17, v12
+; GISEL-NEXT: s_or_b64 s[10:11], vcc, s[4:5]
+; GISEL-NEXT: v_ffbh_u32_e32 v16, v13
+; GISEL-NEXT: v_add_i32_e32 v17, vcc, 32, v17
+; GISEL-NEXT: v_min_u32_e32 v16, v16, v17
; GISEL-NEXT: v_ffbh_u32_e32 v18, v14
-; GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GISEL-NEXT: v_add_i32_e32 v16, vcc, 64, v16
; GISEL-NEXT: v_add_i32_e32 v18, vcc, 32, v18
-; GISEL-NEXT: v_ffbh_u32_e32 v19, v15
-; GISEL-NEXT: v_min_u32_e32 v18, v18, v19
-; GISEL-NEXT: v_ffbh_u32_e32 v19, v12
-; GISEL-NEXT: v_add_i32_e32 v19, vcc, 32, v19
-; GISEL-NEXT: v_ffbh_u32_e32 v20, v13
-; GISEL-NEXT: v_min_u32_e32 v19, v19, v20
-; GISEL-NEXT: v_add_i32_e32 v19, vcc, 64, v19
-; GISEL-NEXT: v_addc_u32_e64 v20, s[6:7], 0, 0, vcc
-; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[14:15]
-; GISEL-NEXT: v_ffbh_u32_e32 v21, v7
-; GISEL-NEXT: v_cndmask_b32_e32 v18, v19, v18, vcc
+; GISEL-NEXT: v_ffbh_u32_e32 v17, v15
+; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[14:15]
+; GISEL-NEXT: v_min_u32_e32 v17, v17, v18
+; GISEL-NEXT: v_ffbh_u32_e32 v18, v4
+; GISEL-NEXT: v_cndmask_b32_e32 v16, v17, v16, vcc
+; GISEL-NEXT: v_ffbh_u32_e32 v17, v5
+; GISEL-NEXT: v_add_i32_e32 v18, vcc, 32, v18
+; GISEL-NEXT: v_min_u32_e32 v17, v17, v18
; GISEL-NEXT: v_ffbh_u32_e32 v19, v6
-; GISEL-NEXT: v_cndmask_b32_e64 v20, v20, 0, vcc
+; GISEL-NEXT: v_add_i32_e32 v17, vcc, 64, v17
; GISEL-NEXT: v_add_i32_e32 v19, vcc, 32, v19
-; GISEL-NEXT: v_min_u32_e32 v19, v19, v21
-; GISEL-NEXT: v_ffbh_u32_e32 v21, v4
-; GISEL-NEXT: v_add_i32_e32 v21, vcc, 32, v21
-; GISEL-NEXT: v_ffbh_u32_e32 v22, v5
-; GISEL-NEXT: v_min_u32_e32 v21, v21, v22
-; GISEL-NEXT: v_add_i32_e32 v21, vcc, 64, v21
-; GISEL-NEXT: v_addc_u32_e64 v22, s[6:7], 0, 0, vcc
-; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
-; GISEL-NEXT: s_movk_i32 s8, 0x7f
-; GISEL-NEXT: v_cndmask_b32_e32 v19, v21, v19, vcc
-; GISEL-NEXT: v_cndmask_b32_e64 v22, v22, 0, vcc
-; GISEL-NEXT: v_sub_i32_e32 v18, vcc, v18, v19
-; GISEL-NEXT: v_subb_u32_e32 v19, vcc, v20, v22, vcc
-; GISEL-NEXT: v_subb_u32_e64 v22, s[6:7], 0, 0, vcc
-; GISEL-NEXT: v_subb_u32_e64 v23, s[6:7], 0, 0, s[6:7]
-; GISEL-NEXT: s_mov_b64 s[6:7], 0x7f
-; GISEL-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[18:19]
-; GISEL-NEXT: v_or_b32_e32 v25, v19, v23
-; GISEL-NEXT: v_cndmask_b32_e64 v20, 0, 1, vcc
-; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[22:23]
-; GISEL-NEXT: v_cndmask_b32_e64 v21, 0, 1, vcc
+; GISEL-NEXT: v_ffbh_u32_e32 v18, v7
+; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7]
+; GISEL-NEXT: v_min_u32_e32 v18, v18, v19
+; GISEL-NEXT: v_cndmask_b32_e32 v17, v18, v17, vcc
+; GISEL-NEXT: v_sub_i32_e32 v16, vcc, v16, v17
+; GISEL-NEXT: v_subb_u32_e64 v17, s[4:5], 0, 0, vcc
+; GISEL-NEXT: v_mov_b32_e32 v18, 0x7f
+; GISEL-NEXT: v_mov_b32_e32 v19, 0
+; GISEL-NEXT: v_subb_u32_e64 v22, s[4:5], 0, 0, s[4:5]
+; GISEL-NEXT: v_cmp_gt_u64_e32 vcc, v[16:17], v[18:19]
+; GISEL-NEXT: v_subb_u32_e64 v23, s[4:5], 0, 0, s[4:5]
+; GISEL-NEXT: v_cndmask_b32_e64 v18, 0, 1, vcc
+; GISEL-NEXT: v_cmp_lt_u64_e32 vcc, 0, v[22:23]
+; GISEL-NEXT: s_mov_b64 s[6:7], exec
+; GISEL-NEXT: v_cndmask_b32_e64 v19, 0, 1, vcc
; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[22:23]
-; GISEL-NEXT: v_cndmask_b32_e32 v20, v21, v20, vcc
-; GISEL-NEXT: v_and_b32_e32 v20, 1, v20
-; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 1, v20
-; GISEL-NEXT: v_xor_b32_e32 v20, 0x7f, v18
-; GISEL-NEXT: v_or_b32_e32 v24, v20, v22
-; GISEL-NEXT: s_or_b64 s[4:5], s[4:5], vcc
-; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[24:25]
-; GISEL-NEXT: s_xor_b64 s[6:7], s[4:5], -1
-; GISEL-NEXT: v_cndmask_b32_e64 v21, v7, 0, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e64 v20, v6, 0, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e64 v24, v5, 0, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e64 v25, v4, 0, s[4:5]
-; GISEL-NEXT: s_and_b64 s[4:5], s[6:7], vcc
+; GISEL-NEXT: s_mov_b64 s[8:9], 0
+; GISEL-NEXT: v_cndmask_b32_e32 v18, v19, v18, vcc
+; GISEL-NEXT: v_and_b32_e32 v18, 1, v18
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v18
+; GISEL-NEXT: v_xor_b32_e32 v18, 0x7f, v16
+; GISEL-NEXT: v_or_b32_e32 v18, v18, v22
+; GISEL-NEXT: v_or_b32_e32 v19, v17, v23
+; GISEL-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[18:19]
+; GISEL-NEXT: s_or_b64 s[10:11], s[10:11], vcc
+; GISEL-NEXT: s_or_b64 s[4:5], s[10:11], s[4:5]
+; GISEL-NEXT: v_cndmask_b32_e64 v24, v4, 0, s[10:11]
+; GISEL-NEXT: v_cndmask_b32_e64 v25, v5, 0, s[10:11]
+; GISEL-NEXT: v_cndmask_b32_e64 v18, v6, 0, s[10:11]
+; GISEL-NEXT: v_cndmask_b32_e64 v19, v7, 0, s[10:11]
+; GISEL-NEXT: s_xor_b64 s[4:5], s[4:5], s[6:7]
; GISEL-NEXT: s_and_saveexec_b64 s[6:7], s[4:5]
; GISEL-NEXT: s_cbranch_execz .LBB5_12
; GISEL-NEXT: ; %bb.7: ; %udiv-bb1
-; GISEL-NEXT: v_add_i32_e32 v34, vcc, 1, v18
-; GISEL-NEXT: v_addc_u32_e32 v35, vcc, 0, v19, vcc
+; GISEL-NEXT: v_add_i32_e32 v34, vcc, 1, v16
+; GISEL-NEXT: v_addc_u32_e32 v35, vcc, 0, v17, vcc
; GISEL-NEXT: v_addc_u32_e32 v36, vcc, 0, v22, vcc
-; GISEL-NEXT: v_addc_u32_e64 v37, s[4:5], 0, v23, vcc
-; GISEL-NEXT: v_sub_i32_e32 v23, vcc, s8, v18
-; GISEL-NEXT: v_sub_i32_e32 v21, vcc, 64, v23
-; GISEL-NEXT: v_lshl_b64 v[19:20], v[6:7], v23
-; GISEL-NEXT: v_lshr_b64 v[21:22], v[4:5], v21
-; GISEL-NEXT: v_sub_i32_e32 v18, vcc, 63, v18
-; GISEL-NEXT: v_or_b32_e32 v20, v20, v22
-; GISEL-NEXT: v_or_b32_e32 v22, v19, v21
-; GISEL-NEXT: v_lshl_b64 v[18:19], v[4:5], v18
-; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v23
-; GISEL-NEXT: s_xor_b64 s[8:9], s[4:5], -1
-; GISEL-NEXT: v_cndmask_b32_e32 v19, v19, v20, vcc
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v23
-; GISEL-NEXT: v_cndmask_b32_e64 v21, v19, v7, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e32 v20, v18, v22, vcc
-; GISEL-NEXT: v_lshl_b64 v[18:19], v[4:5], v23
-; GISEL-NEXT: v_cndmask_b32_e64 v20, v20, v6, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e32 v19, 0, v19, vcc
-; GISEL-NEXT: v_mov_b32_e32 v22, 0
-; GISEL-NEXT: v_mov_b32_e32 v23, 0
-; GISEL-NEXT: v_cndmask_b32_e32 v18, 0, v18, vcc
-; GISEL-NEXT: s_and_saveexec_b64 s[4:5], s[8:9]
-; GISEL-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
+; GISEL-NEXT: s_mov_b64 s[4:5], exec
+; GISEL-NEXT: v_addc_u32_e32 v37, vcc, 0, v23, vcc
+; GISEL-NEXT: s_xor_b64 s[4:5], vcc, s[4:5]
+; GISEL-NEXT: v_sub_i32_e32 v24, vcc, 0x7f, v16
+; GISEL-NEXT: v_not_b32_e32 v16, 63
+; GISEL-NEXT: v_add_i32_e32 v25, vcc, v24, v16
+; GISEL-NEXT: v_sub_i32_e32 v16, vcc, 64, v24
+; GISEL-NEXT: v_lshr_b64 v[16:17], v[4:5], v16
+; GISEL-NEXT: v_lshl_b64 v[18:19], v[6:7], v24
+; GISEL-NEXT: v_lshl_b64 v[22:23], v[4:5], v24
+; GISEL-NEXT: v_or_b32_e32 v18, v16, v18
+; GISEL-NEXT: v_or_b32_e32 v19, v17, v19
+; GISEL-NEXT: v_lshl_b64 v[16:17], v[4:5], v25
+; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v24
+; GISEL-NEXT: v_cndmask_b32_e32 v22, 0, v22, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v23, 0, v23, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v16, v16, v18, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v17, v17, v19, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v24
+; GISEL-NEXT: v_cndmask_b32_e32 v24, v16, v6, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v25, v17, v7, vcc
+; GISEL-NEXT: s_mov_b64 s[10:11], s[8:9]
+; GISEL-NEXT: v_mov_b32_e32 v17, s9
+; GISEL-NEXT: v_mov_b32_e32 v16, s8
+; GISEL-NEXT: v_mov_b32_e32 v19, s11
+; GISEL-NEXT: v_mov_b32_e32 v18, s10
+; GISEL-NEXT: s_and_saveexec_b64 s[8:9], s[4:5]
+; GISEL-NEXT: s_xor_b64 s[12:13], exec, s[8:9]
; GISEL-NEXT: s_cbranch_execz .LBB5_11
; GISEL-NEXT: ; %bb.8: ; %udiv-preheader
-; GISEL-NEXT: v_sub_i32_e32 v24, vcc, 64, v34
-; GISEL-NEXT: v_lshr_b64 v[22:23], v[4:5], v34
-; GISEL-NEXT: v_lshl_b64 v[24:25], v[6:7], v24
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v34
-; GISEL-NEXT: v_or_b32_e32 v24, v22, v24
-; GISEL-NEXT: v_subrev_i32_e32 v22, vcc, 64, v34
-; GISEL-NEXT: v_or_b32_e32 v25, v23, v25
-; GISEL-NEXT: v_lshr_b64 v[22:23], v[6:7], v22
+; GISEL-NEXT: v_sub_i32_e32 v18, vcc, 64, v34
+; GISEL-NEXT: v_lshr_b64 v[16:17], v[4:5], v34
+; GISEL-NEXT: v_lshl_b64 v[18:19], v[6:7], v18
+; GISEL-NEXT: v_add_i32_e32 v28, vcc, 0xffffffc0, v34
+; GISEL-NEXT: v_lshr_b64 v[26:27], v[6:7], v34
+; GISEL-NEXT: v_or_b32_e32 v18, v16, v18
+; GISEL-NEXT: v_or_b32_e32 v19, v17, v19
+; GISEL-NEXT: v_lshr_b64 v[16:17], v[6:7], v28
; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v34
-; GISEL-NEXT: v_cndmask_b32_e32 v23, v23, v25, vcc
-; GISEL-NEXT: v_cndmask_b32_e64 v27, v23, v5, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e32 v24, v22, v24, vcc
-; GISEL-NEXT: v_lshr_b64 v[22:23], v[6:7], v34
-; GISEL-NEXT: v_cndmask_b32_e64 v26, v24, v4, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e32 v31, 0, v23, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v30, 0, v22, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v16, v16, v18, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v17, v17, v19, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v30, 0, v26, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v31, 0, v27, vcc
; GISEL-NEXT: v_add_i32_e32 v38, vcc, -1, v12
; GISEL-NEXT: v_addc_u32_e32 v39, vcc, -1, v13, vcc
+; GISEL-NEXT: s_mov_b64 s[8:9], 0
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v34
; GISEL-NEXT: v_addc_u32_e32 v48, vcc, -1, v14, vcc
+; GISEL-NEXT: v_cndmask_b32_e64 v28, v16, v4, s[4:5]
+; GISEL-NEXT: v_cndmask_b32_e64 v29, v17, v5, s[4:5]
; GISEL-NEXT: v_addc_u32_e32 v49, vcc, -1, v15, vcc
-; GISEL-NEXT: v_mov_b32_e32 v24, 0
-; GISEL-NEXT: v_mov_b32_e32 v25, 0
-; GISEL-NEXT: s_mov_b64 s[10:11], 0
-; GISEL-NEXT: v_mov_b32_e32 v28, 0
-; GISEL-NEXT: v_mov_b32_e32 v29, 0
-; GISEL-NEXT: v_mov_b32_e32 v23, 0
+; GISEL-NEXT: s_mov_b64 s[10:11], s[8:9]
+; GISEL-NEXT: v_mov_b32_e32 v27, 0
+; GISEL-NEXT: v_mov_b32_e32 v17, s9
+; GISEL-NEXT: v_mov_b32_e32 v16, s8
+; GISEL-NEXT: v_mov_b32_e32 v26, 1
+; GISEL-NEXT: v_mov_b32_e32 v19, s11
+; GISEL-NEXT: v_mov_b32_e32 v18, s10
; GISEL-NEXT: .LBB5_9: ; %udiv-do-while
; GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GISEL-NEXT: v_lshl_b64 v[50:51], v[18:19], 1
+; GISEL-NEXT: v_lshl_b64 v[18:19], v[22:23], 1
; GISEL-NEXT: v_add_i32_e32 v34, vcc, -1, v34
-; GISEL-NEXT: v_or_b32_e32 v18, v28, v50
-; GISEL-NEXT: v_lshrrev_b32_e32 v28, 31, v27
-; GISEL-NEXT: v_lshl_b64 v[26:27], v[26:27], 1
+; GISEL-NEXT: v_lshrrev_b32_e32 v26, 31, v23
+; GISEL-NEXT: v_or_b32_e32 v22, v16, v18
+; GISEL-NEXT: v_or_b32_e32 v23, v17, v19
+; GISEL-NEXT: v_lshl_b64 v[16:17], v[28:29], 1
; GISEL-NEXT: v_addc_u32_e32 v35, vcc, -1, v35, vcc
-; GISEL-NEXT: v_lshrrev_b32_e32 v22, 31, v19
-; GISEL-NEXT: v_or_b32_e32 v19, v29, v51
-; GISEL-NEXT: v_lshrrev_b32_e32 v29, 31, v21
-; GISEL-NEXT: v_lshl_b64 v[20:21], v[20:21], 1
+; GISEL-NEXT: v_lshrrev_b32_e32 v28, 31, v29
+; GISEL-NEXT: v_lshrrev_b32_e32 v29, 31, v25
+; GISEL-NEXT: v_lshl_b64 v[24:25], v[24:25], 1
; GISEL-NEXT: v_addc_u32_e32 v36, vcc, -1, v36, vcc
-; GISEL-NEXT: v_lshl_b64 v[30:31], v[30:31], 1
-; GISEL-NEXT: v_or_b32_e32 v26, v26, v29
+; GISEL-NEXT: v_lshl_b64 v[18:19], v[30:31], 1
+; GISEL-NEXT: v_or_b32_e32 v16, v16, v29
; GISEL-NEXT: v_addc_u32_e32 v37, vcc, -1, v37, vcc
-; GISEL-NEXT: v_or_b32_e32 v20, v20, v22
-; GISEL-NEXT: v_sub_i32_e32 v22, vcc, v38, v26
-; GISEL-NEXT: v_or_b32_e32 v30, v30, v28
-; GISEL-NEXT: v_subb_u32_e32 v22, vcc, v39, v27, vcc
-; GISEL-NEXT: v_subb_u32_e32 v22, vcc, v48, v30, vcc
+; GISEL-NEXT: v_or_b32_e32 v24, v24, v26
+; GISEL-NEXT: v_sub_i32_e32 v26, vcc, v38, v16
+; GISEL-NEXT: v_or_b32_e32 v18, v18, v28
+; GISEL-NEXT: v_subb_u32_e32 v26, vcc, v39, v17, vcc
+; GISEL-NEXT: v_subb_u32_e32 v26, vcc, v48, v18, vcc
; GISEL-NEXT: v_or_b32_e32 v28, v34, v36
; GISEL-NEXT: v_or_b32_e32 v29, v35, v37
-; GISEL-NEXT: v_subb_u32_e32 v22, vcc, v49, v31, vcc
+; GISEL-NEXT: v_subb_u32_e32 v26, vcc, v49, v19, vcc
; GISEL-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[28:29]
-; GISEL-NEXT: v_ashrrev_i32_e32 v28, 31, v22
-; GISEL-NEXT: v_and_b32_e32 v50, v28, v12
-; GISEL-NEXT: v_and_b32_e32 v29, v28, v13
-; GISEL-NEXT: v_sub_i32_e32 v26, vcc, v26, v50
-; GISEL-NEXT: v_subb_u32_e32 v27, vcc, v27, v29, vcc
-; GISEL-NEXT: v_and_b32_e32 v51, v28, v14
-; GISEL-NEXT: v_and_b32_e32 v22, 1, v28
+; GISEL-NEXT: v_ashrrev_i32_e32 v28, 31, v26
+; GISEL-NEXT: v_and_b32_e32 v29, v28, v12
+; GISEL-NEXT: v_and_b32_e32 v26, 1, v28
+; GISEL-NEXT: v_and_b32_e32 v30, v28, v13
+; GISEL-NEXT: v_and_b32_e32 v31, v28, v14
; GISEL-NEXT: v_and_b32_e32 v50, v28, v15
-; GISEL-NEXT: v_subb_u32_e32 v30, vcc, v30, v51, vcc
-; GISEL-NEXT: v_or_b32_e32 v21, v25, v21
-; GISEL-NEXT: v_or_b32_e32 v20, v24, v20
-; GISEL-NEXT: s_or_b64 s[10:11], s[4:5], s[10:11]
-; GISEL-NEXT: v_mov_b32_e32 v29, v23
-; GISEL-NEXT: v_mov_b32_e32 v28, v22
-; GISEL-NEXT: v_subb_u32_e32 v31, vcc, v31, v50, vcc
-; GISEL-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GISEL-NEXT: v_sub_i32_e32 v28, vcc, v16, v29
+; GISEL-NEXT: v_subb_u32_e32 v29, vcc, v17, v30, vcc
+; GISEL-NEXT: v_subb_u32_e32 v30, vcc, v18, v31, vcc
+; GISEL-NEXT: s_or_b64 s[8:9], s[4:5], s[8:9]
+; GISEL-NEXT: v_mov_b32_e32 v16, v26
+; GISEL-NEXT: v_mov_b32_e32 v17, v27
+; GISEL-NEXT: v_subb_u32_e32 v31, vcc, v19, v50, vcc
+; GISEL-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GISEL-NEXT: s_cbranch_execnz .LBB5_9
; GISEL-NEXT: ; %bb.10: ; %Flow
-; GISEL-NEXT: s_or_b64 exec, exec, s[10:11]
-; GISEL-NEXT: .LBB5_11: ; %Flow11
; GISEL-NEXT: s_or_b64 exec, exec, s[8:9]
-; GISEL-NEXT: v_lshl_b64 v[20:21], v[20:21], 1
-; GISEL-NEXT: v_lshrrev_b32_e32 v24, 31, v19
-; GISEL-NEXT: v_lshl_b64 v[18:19], v[18:19], 1
-; GISEL-NEXT: v_or_b32_e32 v20, v20, v24
-; GISEL-NEXT: v_or_b32_e32 v24, v23, v19
-; GISEL-NEXT: v_or_b32_e32 v25, v22, v18
+; GISEL-NEXT: .LBB5_11: ; %Flow11
+; GISEL-NEXT: s_or_b64 exec, exec, s[12:13]
+; GISEL-NEXT: v_lshl_b64 v[26:27], v[22:23], 1
+; GISEL-NEXT: v_lshl_b64 v[18:19], v[24:25], 1
+; GISEL-NEXT: v_lshrrev_b32_e32 v22, 31, v23
+; GISEL-NEXT: v_or_b32_e32 v18, v18, v22
+; GISEL-NEXT: v_or_b32_e32 v24, v16, v26
+; GISEL-NEXT: v_or_b32_e32 v25, v17, v27
; GISEL-NEXT: .LBB5_12: ; %Flow12
; GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
-; GISEL-NEXT: v_mul_lo_u32 v11, v33, v11
-; GISEL-NEXT: v_mad_u64_u32 v[18:19], s[4:5], v33, v10, 0
-; GISEL-NEXT: v_mad_u64_u32 v[26:27], s[4:5], v8, v33, 0
-; GISEL-NEXT: v_mul_lo_u32 v22, v32, v10
-; GISEL-NEXT: v_mov_b32_e32 v28, 0
-; GISEL-NEXT: v_add_i32_e32 v19, vcc, v19, v11
-; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v9, v33, v[27:28]
-; GISEL-NEXT: v_add_i32_e32 v19, vcc, v19, v22
-; GISEL-NEXT: v_mad_u64_u32 v[18:19], s[4:5], v16, v8, v[18:19]
-; GISEL-NEXT: v_mul_lo_u32 v17, v17, v8
-; GISEL-NEXT: v_mov_b32_e32 v27, v10
-; GISEL-NEXT: v_mad_u64_u32 v[22:23], s[4:5], v8, v32, v[27:28]
-; GISEL-NEXT: v_mul_lo_u32 v16, v16, v9
-; GISEL-NEXT: v_add_i32_e32 v17, vcc, v17, v19
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v11, v23
-; GISEL-NEXT: v_addc_u32_e64 v11, s[4:5], 0, 0, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v9, v32, v[10:11]
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v16, v17
-; GISEL-NEXT: v_mul_lo_u32 v16, v21, v12
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v18
-; GISEL-NEXT: v_addc_u32_e32 v9, vcc, v9, v10, vcc
-; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v0, v26
-; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v1, v22, vcc
-; GISEL-NEXT: v_subb_u32_e32 v2, vcc, v2, v8, vcc
-; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v3, v9, vcc
-; GISEL-NEXT: v_mul_lo_u32 v10, v25, v15
-; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v25, v14, 0
-; GISEL-NEXT: v_mad_u64_u32 v[26:27], s[4:5], v12, v25, 0
-; GISEL-NEXT: v_mul_lo_u32 v14, v24, v14
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
-; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v13, v25, v[27:28]
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v14
-; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v20, v12, v[8:9]
-; GISEL-NEXT: v_mov_b32_e32 v27, v10
-; GISEL-NEXT: v_mad_u64_u32 v[14:15], s[4:5], v12, v24, v[27:28]
-; GISEL-NEXT: v_add_i32_e32 v12, vcc, v16, v9
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v11, v15
-; GISEL-NEXT: v_mul_lo_u32 v16, v20, v13
-; GISEL-NEXT: v_addc_u32_e64 v10, s[4:5], 0, 0, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v13, v24, v[9:10]
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v16, v12
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v8
-; GISEL-NEXT: v_addc_u32_e32 v9, vcc, v10, v11, vcc
-; GISEL-NEXT: v_sub_i32_e32 v4, vcc, v4, v26
-; GISEL-NEXT: v_subb_u32_e32 v5, vcc, v5, v14, vcc
-; GISEL-NEXT: v_subb_u32_e32 v6, vcc, v6, v8, vcc
-; GISEL-NEXT: v_subb_u32_e32 v7, vcc, v7, v9, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[16:17], s[4:5], v8, v20, 0
+; GISEL-NEXT: v_mad_u64_u32 v[26:27], s[4:5], v8, v32, 0
+; GISEL-NEXT: v_mul_lo_u32 v34, v9, v20
+; GISEL-NEXT: v_mad_u64_u32 v[22:23], s[4:5], v9, v33, v[16:17]
+; GISEL-NEXT: v_mad_u64_u32 v[16:17], s[4:5], v12, v18, 0
+; GISEL-NEXT: v_mad_u64_u32 v[28:29], s[4:5], v10, v32, v[22:23]
+; GISEL-NEXT: v_mad_u64_u32 v[30:31], s[4:5], v13, v25, v[16:17]
+; GISEL-NEXT: v_mad_u64_u32 v[16:17], vcc, v8, v33, v[27:28]
+; GISEL-NEXT: v_mul_lo_u32 v8, v8, v21
+; GISEL-NEXT: v_mad_u64_u32 v[20:21], s[4:5], v12, v24, 0
+; GISEL-NEXT: v_mad_u64_u32 v[22:23], s[4:5], v14, v24, v[30:31]
+; GISEL-NEXT: v_mad_u64_u32 v[27:28], s[4:5], v9, v32, v[16:17]
+; GISEL-NEXT: v_addc_u32_e64 v8, s[4:5], v29, v8, s[4:5]
+; GISEL-NEXT: v_addc_u32_e32 v8, vcc, v8, v34, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[16:17], s[4:5], v10, v33, v[8:9]
+; GISEL-NEXT: v_mad_u64_u32 v[8:9], vcc, v12, v25, v[21:22]
+; GISEL-NEXT: v_mul_lo_u32 v29, v12, v19
+; GISEL-NEXT: v_mul_lo_u32 v30, v13, v18
+; GISEL-NEXT: v_mad_u64_u32 v[18:19], s[4:5], v11, v32, v[16:17]
+; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[6:7], v13, v24, v[8:9]
+; GISEL-NEXT: v_addc_u32_e64 v8, s[6:7], v23, v29, s[6:7]
+; GISEL-NEXT: v_addc_u32_e32 v8, vcc, v8, v30, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[12:13], s[6:7], v14, v25, v[8:9]
+; GISEL-NEXT: v_sub_i32_e64 v0, s[4:5], v0, v26
+; GISEL-NEXT: v_subb_u32_e64 v1, s[4:5], v1, v27, s[4:5]
+; GISEL-NEXT: v_subb_u32_e64 v2, vcc, v2, v28, s[4:5]
+; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v3, v18, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v15, v24, v[12:13]
+; GISEL-NEXT: v_sub_i32_e32 v4, vcc, v4, v20
+; GISEL-NEXT: v_subb_u32_e32 v5, vcc, v5, v10, vcc
+; GISEL-NEXT: v_subb_u32_e32 v6, vcc, v6, v11, vcc
+; GISEL-NEXT: v_subb_u32_e32 v7, vcc, v7, v8, vcc
; GISEL-NEXT: s_setpc_b64 s[30:31]
%shl = urem <2 x i128> %lhs, %rhs
ret <2 x i128> %shl
diff --git a/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll b/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll
index 9f639666f2ca5..954aba0f812aa 100644
--- a/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
; RUN: llc -global-isel=0 -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -check-prefixes=GCN,SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -check-prefixes=GCN,GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -check-prefixes=GCN,GISEL %s
define i128 @fptosi_f64_to_i128(double %x) {
; SDAG-LABEL: fptosi_f64_to_i128:
@@ -712,86 +712,214 @@ define i128 @fptoui_f16_to_i128(half %x) {
}
define i128 @fptosi_bf16_to_i128(bfloat %x) {
-; GCN-LABEL: fptosi_bf16_to_i128:
-; GCN: ; %bb.0: ; %fp-to-i-entry
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v4, v0
-; GCN-NEXT: v_bfe_u32 v8, v4, 7, 8
-; GCN-NEXT: s_movk_i32 s4, 0x7e
-; GCN-NEXT: v_mov_b32_e32 v0, 0
-; GCN-NEXT: v_mov_b32_e32 v1, 0
-; GCN-NEXT: v_mov_b32_e32 v2, 0
-; GCN-NEXT: v_mov_b32_e32 v3, 0
-; GCN-NEXT: v_cmp_lt_u16_e32 vcc, s4, v8
-; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; GCN-NEXT: s_cbranch_execz .LBB6_6
-; GCN-NEXT: ; %bb.1: ; %fp-to-i-if-check.exp.size
-; GCN-NEXT: v_ashrrev_i16_e32 v0, 15, v4
-; GCN-NEXT: v_bfe_i32 v7, v0, 0, 16
-; GCN-NEXT: v_and_b32_e32 v0, 0x7f, v4
-; GCN-NEXT: s_movk_i32 s5, 0x85
-; GCN-NEXT: s_mov_b32 s4, 0
-; GCN-NEXT: v_ashrrev_i32_e32 v6, 31, v7
-; GCN-NEXT: v_or_b32_e32 v5, 1, v7
-; GCN-NEXT: v_or_b32_e32 v4, 0x80, v0
-; GCN-NEXT: v_cmp_lt_u16_e32 vcc, s5, v8
-; GCN-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GCN-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GCN-NEXT: s_and_saveexec_b64 s[8:9], vcc
-; GCN-NEXT: s_xor_b64 s[8:9], exec, s[8:9]
-; GCN-NEXT: s_cbranch_execz .LBB6_3
-; GCN-NEXT: ; %bb.2: ; %fp-to-i-if-exp.large
-; GCN-NEXT: v_add_u16_e32 v10, 0xff7a, v8
-; GCN-NEXT: v_and_b32_e32 v0, 0xffff, v4
-; GCN-NEXT: v_mov_b32_e32 v1, s4
-; GCN-NEXT: v_sub_u32_e32 v2, 64, v10
-; GCN-NEXT: v_lshrrev_b64 v[3:4], v2, v[0:1]
-; GCN-NEXT: v_subrev_u32_e32 v2, 64, v10
-; GCN-NEXT: v_lshlrev_b64 v[8:9], v2, v[0:1]
-; GCN-NEXT: v_cmp_gt_u16_e32 vcc, 64, v10
-; GCN-NEXT: v_cndmask_b32_e32 v4, v9, v4, vcc
-; GCN-NEXT: v_cmp_ne_u16_e64 s[4:5], 0, v10
-; GCN-NEXT: v_cndmask_b32_e64 v9, 0, v4, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e32 v8, v8, v3, vcc
-; GCN-NEXT: v_lshlrev_b64 v[3:4], v10, v[0:1]
-; GCN-NEXT: v_cndmask_b32_e64 v8, 0, v8, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e32 v10, 0, v3, vcc
-; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v10, v5, 0
-; GCN-NEXT: v_mov_b32_e32 v2, 0
-; GCN-NEXT: v_cndmask_b32_e32 v13, 0, v4, vcc
-; GCN-NEXT: v_mul_lo_u32 v11, v6, v8
-; GCN-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v13, v5, v[1:2]
-; GCN-NEXT: v_mul_lo_u32 v12, v5, v9
-; GCN-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v5, v8, 0
-; GCN-NEXT: v_mul_lo_u32 v5, v7, v13
-; GCN-NEXT: v_mov_b32_e32 v1, v3
-; GCN-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v10, v6, v[1:2]
-; GCN-NEXT: v_add3_u32 v9, v9, v12, v11
-; GCN-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v7, v10, v[8:9]
-; GCN-NEXT: v_mul_lo_u32 v7, v7, v10
-; GCN-NEXT: v_add_co_u32_e32 v2, vcc, v4, v2
-; GCN-NEXT: v_addc_co_u32_e64 v3, s[4:5], 0, 0, vcc
-; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v13, v6, v[2:3]
-; GCN-NEXT: v_add3_u32 v4, v7, v9, v5
-; GCN-NEXT: ; implicit-def: $vgpr5
-; GCN-NEXT: v_add_co_u32_e32 v2, vcc, v2, v8
-; GCN-NEXT: v_addc_co_u32_e32 v3, vcc, v3, v4, vcc
-; GCN-NEXT: ; implicit-def: $vgpr8
-; GCN-NEXT: ; implicit-def: $vgpr4
-; GCN-NEXT: .LBB6_3: ; %Flow
-; GCN-NEXT: s_andn2_saveexec_b64 s[4:5], s[8:9]
-; GCN-NEXT: ; %bb.4: ; %fp-to-i-if-exp.small
-; GCN-NEXT: v_sub_u16_e32 v0, 0x86, v8
-; GCN-NEXT: v_lshrrev_b16_e32 v0, v0, v4
-; GCN-NEXT: v_mul_hi_i32_i24_e32 v1, v0, v5
-; GCN-NEXT: v_ashrrev_i32_e32 v2, 31, v1
-; GCN-NEXT: v_mul_i32_i24_e32 v0, v0, v5
-; GCN-NEXT: v_mov_b32_e32 v3, v2
-; GCN-NEXT: ; %bb.5: ; %Flow1
-; GCN-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN-NEXT: .LBB6_6: ; %fp-to-i-cleanup
-; GCN-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN-NEXT: s_setpc_b64 s[30:31]
+; SDAG-LABEL: fptosi_bf16_to_i128:
+; SDAG: ; %bb.0: ; %fp-to-i-entry
+; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-NEXT: v_mov_b32_e32 v4, v0
+; SDAG-NEXT: v_bfe_u32 v8, v4, 7, 8
+; SDAG-NEXT: s_movk_i32 s4, 0x7e
+; SDAG-NEXT: v_mov_b32_e32 v0, 0
+; SDAG-NEXT: v_mov_b32_e32 v1, 0
+; SDAG-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-NEXT: v_mov_b32_e32 v3, 0
+; SDAG-NEXT: v_cmp_lt_u16_e32 vcc, s4, v8
+; SDAG-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; SDAG-NEXT: s_cbranch_execz .LBB6_6
+; SDAG-NEXT: ; %bb.1: ; %fp-to-i-if-check.exp.size
+; SDAG-NEXT: v_ashrrev_i16_e32 v0, 15, v4
+; SDAG-NEXT: v_bfe_i32 v7, v0, 0, 16
+; SDAG-NEXT: v_and_b32_e32 v0, 0x7f, v4
+; SDAG-NEXT: s_movk_i32 s5, 0x85
+; SDAG-NEXT: s_mov_b32 s4, 0
+; SDAG-NEXT: v_ashrrev_i32_e32 v6, 31, v7
+; SDAG-NEXT: v_or_b32_e32 v5, 1, v7
+; SDAG-NEXT: v_or_b32_e32 v4, 0x80, v0
+; SDAG-NEXT: v_cmp_lt_u16_e32 vcc, s5, v8
+; SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
+; SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
+; SDAG-NEXT: s_and_saveexec_b64 s[8:9], vcc
+; SDAG-NEXT: s_xor_b64 s[8:9], exec, s[8:9]
+; SDAG-NEXT: s_cbranch_execz .LBB6_3
+; SDAG-NEXT: ; %bb.2: ; %fp-to-i-if-exp.large
+; SDAG-NEXT: v_add_u16_e32 v10, 0xff7a, v8
+; SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; SDAG-NEXT: v_mov_b32_e32 v1, s4
+; SDAG-NEXT: v_sub_u32_e32 v2, 64, v10
+; SDAG-NEXT: v_lshrrev_b64 v[3:4], v2, v[0:1]
+; SDAG-NEXT: v_subrev_u32_e32 v2, 64, v10
+; SDAG-NEXT: v_lshlrev_b64 v[8:9], v2, v[0:1]
+; SDAG-NEXT: v_cmp_gt_u16_e32 vcc, 64, v10
+; SDAG-NEXT: v_cndmask_b32_e32 v4, v9, v4, vcc
+; SDAG-NEXT: v_cmp_ne_u16_e64 s[4:5], 0, v10
+; SDAG-NEXT: v_cndmask_b32_e64 v9, 0, v4, s[4:5]
+; SDAG-NEXT: v_cndmask_b32_e32 v8, v8, v3, vcc
+; SDAG-NEXT: v_lshlrev_b64 v[3:4], v10, v[0:1]
+; SDAG-NEXT: v_cndmask_b32_e64 v8, 0, v8, s[4:5]
+; SDAG-NEXT: v_cndmask_b32_e32 v10, 0, v3, vcc
+; SDAG-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v10, v5, 0
+; SDAG-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-NEXT: v_cndmask_b32_e32 v13, 0, v4, vcc
+; SDAG-NEXT: v_mul_lo_u32 v11, v6, v8
+; SDAG-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v13, v5, v[1:2]
+; SDAG-NEXT: v_mul_lo_u32 v12, v5, v9
+; SDAG-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v5, v8, 0
+; SDAG-NEXT: v_mul_lo_u32 v5, v7, v13
+; SDAG-NEXT: v_mov_b32_e32 v1, v3
+; SDAG-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v10, v6, v[1:2]
+; SDAG-NEXT: v_add3_u32 v9, v9, v12, v11
+; SDAG-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v7, v10, v[8:9]
+; SDAG-NEXT: v_mul_lo_u32 v7, v7, v10
+; SDAG-NEXT: v_add_co_u32_e32 v2, vcc, v4, v2
+; SDAG-NEXT: v_addc_co_u32_e64 v3, s[4:5], 0, 0, vcc
+; SDAG-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v13, v6, v[2:3]
+; SDAG-NEXT: v_add3_u32 v4, v7, v9, v5
+; SDAG-NEXT: ; implicit-def: $vgpr5
+; SDAG-NEXT: v_add_co_u32_e32 v2, vcc, v2, v8
+; SDAG-NEXT: v_addc_co_u32_e32 v3, vcc, v3, v4, vcc
+; SDAG-NEXT: ; implicit-def: $vgpr8
+; SDAG-NEXT: ; implicit-def: $vgpr4
+; SDAG-NEXT: .LBB6_3: ; %Flow
+; SDAG-NEXT: s_andn2_saveexec_b64 s[4:5], s[8:9]
+; SDAG-NEXT: ; %bb.4: ; %fp-to-i-if-exp.small
+; SDAG-NEXT: v_sub_u16_e32 v0, 0x86, v8
+; SDAG-NEXT: v_lshrrev_b16_e32 v0, v0, v4
+; SDAG-NEXT: v_mul_hi_i32_i24_e32 v1, v0, v5
+; SDAG-NEXT: v_ashrrev_i32_e32 v2, 31, v1
+; SDAG-NEXT: v_mul_i32_i24_e32 v0, v0, v5
+; SDAG-NEXT: v_mov_b32_e32 v3, v2
+; SDAG-NEXT: ; %bb.5: ; %Flow1
+; SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
+; SDAG-NEXT: .LBB6_6: ; %fp-to-i-cleanup
+; SDAG-NEXT: s_or_b64 exec, exec, s[6:7]
+; SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-LABEL: fptosi_bf16_to_i128:
+; GISEL: ; %bb.0: ; %fp-to-i-entry
+; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT: v_mov_b32_e32 v4, v0
+; GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GISEL-NEXT: v_lshrrev_b16_e32 v6, 7, v4
+; GISEL-NEXT: v_mov_b32_e32 v0, 0x7f
+; GISEL-NEXT: s_mov_b64 s[6:7], s[4:5]
+; GISEL-NEXT: v_cmp_ge_u16_sdwa s[8:9], v6, v0 src0_sel:BYTE_0 src1_sel:DWORD
+; GISEL-NEXT: v_mov_b32_e32 v0, s4
+; GISEL-NEXT: v_mov_b32_e32 v1, s5
+; GISEL-NEXT: v_mov_b32_e32 v2, s6
+; GISEL-NEXT: v_mov_b32_e32 v3, s7
+; GISEL-NEXT: s_and_saveexec_b64 s[10:11], s[8:9]
+; GISEL-NEXT: s_cbranch_execz .LBB6_6
+; GISEL-NEXT: ; %bb.1: ; %fp-to-i-if-check.exp.size
+; GISEL-NEXT: v_cmp_ge_i16_e32 vcc, -1, v4
+; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc
+; GISEL-NEXT: v_and_b32_e32 v0, 1, v0
+; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GISEL-NEXT: v_lshlrev_b16_e32 v2, 1, v0
+; GISEL-NEXT: v_or_b32_e32 v1, v1, v2
+; GISEL-NEXT: v_lshlrev_b16_e32 v3, 2, v0
+; GISEL-NEXT: v_or_b32_e32 v2, v0, v2
+; GISEL-NEXT: v_or_b32_e32 v1, v1, v3
+; GISEL-NEXT: v_lshlrev_b16_e32 v7, 3, v0
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v3
+; GISEL-NEXT: v_or_b32_e32 v1, v1, v7
+; GISEL-NEXT: v_lshlrev_b16_e32 v8, 4, v0
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v7
+; GISEL-NEXT: v_or_b32_e32 v1, v1, v8
+; GISEL-NEXT: v_lshlrev_b16_e32 v9, 5, v0
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v8
+; GISEL-NEXT: v_or_b32_e32 v1, v1, v9
+; GISEL-NEXT: v_lshlrev_b16_e32 v10, 6, v0
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v9
+; GISEL-NEXT: v_or_b32_e32 v1, v1, v10
+; GISEL-NEXT: v_lshlrev_b16_e32 v11, 7, v0
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v10
+; GISEL-NEXT: v_or_b32_e32 v1, v1, v11
+; GISEL-NEXT: v_lshlrev_b16_e32 v12, 8, v0
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v11
+; GISEL-NEXT: v_or_b32_e32 v1, v1, v12
+; GISEL-NEXT: v_lshlrev_b16_e32 v13, 9, v0
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v12
+; GISEL-NEXT: v_or_b32_e32 v1, v1, v13
+; GISEL-NEXT: v_lshlrev_b16_e32 v14, 10, v0
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v13
+; GISEL-NEXT: v_or_b32_e32 v1, v1, v14
+; GISEL-NEXT: v_lshlrev_b16_e32 v15, 11, v0
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v14
+; GISEL-NEXT: v_or_b32_e32 v1, v1, v15
+; GISEL-NEXT: v_lshlrev_b16_e32 v16, 12, v0
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v15
+; GISEL-NEXT: v_or_b32_e32 v1, v1, v16
+; GISEL-NEXT: v_lshlrev_b16_e32 v17, 13, v0
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v16
+; GISEL-NEXT: v_or_b32_e32 v1, v1, v17
+; GISEL-NEXT: v_lshlrev_b16_e32 v18, 14, v0
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v17
+; GISEL-NEXT: v_or_b32_e32 v1, v1, v18
+; GISEL-NEXT: v_lshlrev_b16_e32 v0, 15, v0
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v18
+; GISEL-NEXT: v_or_b32_e32 v1, v1, v0
+; GISEL-NEXT: v_or_b32_e32 v0, v2, v0
+; GISEL-NEXT: v_and_b32_e32 v9, 0xffff, v0
+; GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v9
+; GISEL-NEXT: v_or3_b32 v7, v1, v0, 1
+; GISEL-NEXT: v_or3_b32 v8, v9, v0, 0
+; GISEL-NEXT: v_and_b32_e32 v0, 0x7f, v4
+; GISEL-NEXT: v_or_b32_e32 v4, 0x80, v0
+; GISEL-NEXT: v_mov_b32_e32 v0, 0x86
+; GISEL-NEXT: v_mov_b32_e32 v5, 0
+; GISEL-NEXT: v_cmp_ge_u16_sdwa s[4:5], v6, v0 src0_sel:BYTE_0 src1_sel:DWORD
+; GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GISEL-NEXT: s_and_saveexec_b64 s[6:7], s[4:5]
+; GISEL-NEXT: s_xor_b64 s[12:13], exec, s[6:7]
+; GISEL-NEXT: s_cbranch_execz .LBB6_3
+; GISEL-NEXT: ; %bb.2: ; %fp-to-i-if-exp.large
+; GISEL-NEXT: v_mov_b32_e32 v0, 0xffffff7a
+; GISEL-NEXT: v_add_u16_sdwa v6, v6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GISEL-NEXT: v_lshlrev_b64 v[0:1], v6, v[4:5]
+; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v6
+; GISEL-NEXT: v_lshl_or_b32 v13, v9, 16, v9
+; GISEL-NEXT: v_sub_u32_e32 v2, 64, v6
+; GISEL-NEXT: v_cndmask_b32_e32 v14, 0, v0, vcc
+; GISEL-NEXT: v_lshrrev_b64 v[9:10], v2, v[4:5]
+; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v14, v13, 0
+; GISEL-NEXT: v_add_u32_e32 v11, 0xffffffc0, v6
+; GISEL-NEXT: v_cndmask_b32_e32 v15, 0, v1, vcc
+; GISEL-NEXT: v_lshlrev_b64 v[4:5], v11, v[4:5]
+; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v15, v8, v[2:3]
+; GISEL-NEXT: v_cndmask_b32_e32 v0, v4, v9, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v6
+; GISEL-NEXT: v_cndmask_b32_e64 v4, v0, 0, s[4:5]
+; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[6:7], v14, v7, 0
+; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[6:7], v4, v7, v[11:12]
+; GISEL-NEXT: v_mul_lo_u32 v9, v14, v13
+; GISEL-NEXT: v_mul_lo_u32 v6, v15, v13
+; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[6:7], v14, v8, v[1:2]
+; GISEL-NEXT: v_mad_u64_u32 v[1:2], s[8:9], v15, v7, v[11:12]
+; GISEL-NEXT: v_addc_co_u32_e64 v3, s[8:9], v3, v9, s[8:9]
+; GISEL-NEXT: v_addc_co_u32_e64 v3, s[6:7], v3, v6, s[6:7]
+; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[6:7], v4, v8, v[3:4]
+; GISEL-NEXT: v_cndmask_b32_e32 v3, v5, v10, vcc
+; GISEL-NEXT: v_cndmask_b32_e64 v5, v3, 0, s[4:5]
+; GISEL-NEXT: ; implicit-def: $vgpr6
+; GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v5, v7, v[11:12]
+; GISEL-NEXT: ; implicit-def: $vgpr4
+; GISEL-NEXT: ; implicit-def: $vgpr7
+; GISEL-NEXT: .LBB6_3: ; %Flow
+; GISEL-NEXT: s_andn2_saveexec_b64 s[4:5], s[12:13]
+; GISEL-NEXT: ; %bb.4: ; %fp-to-i-if-exp.small
+; GISEL-NEXT: v_mov_b32_e32 v0, 0x86
+; GISEL-NEXT: v_sub_u16_sdwa v0, v0, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GISEL-NEXT: v_lshrrev_b16_e32 v0, v0, v4
+; GISEL-NEXT: v_mul_hi_i32_i24_e32 v1, v0, v7
+; GISEL-NEXT: v_ashrrev_i32_e32 v2, 31, v1
+; GISEL-NEXT: v_mul_i32_i24_e32 v0, v0, v7
+; GISEL-NEXT: v_mov_b32_e32 v3, v2
+; GISEL-NEXT: ; %bb.5: ; %Flow1
+; GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GISEL-NEXT: .LBB6_6: ; %fp-to-i-cleanup
+; GISEL-NEXT: s_or_b64 exec, exec, s[10:11]
+; GISEL-NEXT: s_setpc_b64 s[30:31]
%cvt = fptosi bfloat %x to i128
ret i128 %cvt
}
diff --git a/llvm/test/CodeGen/AMDGPU/itofp.i128.ll b/llvm/test/CodeGen/AMDGPU/itofp.i128.ll
index 93b5997ef81fb..5efb825a7a90a 100644
--- a/llvm/test/CodeGen/AMDGPU/itofp.i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/itofp.i128.ll
@@ -1,822 +1,1659 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
; RUN: llc -global-isel=0 -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -check-prefixes=GCN,SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -check-prefixes=GCN,GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -check-prefixes=GCN,GISEL %s
define float @sitofp_i128_to_f32(i128 %x) {
-; GCN-LABEL: sitofp_i128_to_f32:
-; GCN: ; %bb.0: ; %itofp-entry
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_or_b32_e32 v5, v1, v3
-; GCN-NEXT: v_or_b32_e32 v4, v0, v2
-; GCN-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GCN-NEXT: v_mov_b32_e32 v4, 0
-; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; GCN-NEXT: s_cbranch_execz .LBB0_12
-; GCN-NEXT: ; %bb.1: ; %itofp-if-end
-; GCN-NEXT: v_ashrrev_i32_e32 v5, 31, v3
-; GCN-NEXT: v_xor_b32_e32 v0, v0, v5
-; GCN-NEXT: v_xor_b32_e32 v1, v1, v5
-; GCN-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v5
-; GCN-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v5, vcc
-; GCN-NEXT: v_xor_b32_e32 v2, v2, v5
-; GCN-NEXT: v_xor_b32_e32 v6, v3, v5
-; GCN-NEXT: v_subb_co_u32_e32 v4, vcc, v2, v5, vcc
-; GCN-NEXT: v_subb_co_u32_e32 v5, vcc, v6, v5, vcc
-; GCN-NEXT: v_ffbh_u32_e32 v2, v4
-; GCN-NEXT: v_add_u32_e32 v2, 32, v2
-; GCN-NEXT: v_ffbh_u32_e32 v6, v5
-; GCN-NEXT: v_min_u32_e32 v2, v2, v6
-; GCN-NEXT: v_ffbh_u32_e32 v6, v0
-; GCN-NEXT: v_add_u32_e32 v6, 32, v6
-; GCN-NEXT: v_ffbh_u32_e32 v7, v1
-; GCN-NEXT: v_min_u32_e32 v6, v6, v7
-; GCN-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GCN-NEXT: v_add_u32_e32 v6, 64, v6
-; GCN-NEXT: v_cndmask_b32_e32 v7, v6, v2, vcc
-; GCN-NEXT: v_sub_u32_e32 v2, 0x80, v7
-; GCN-NEXT: v_cmp_gt_i32_e32 vcc, 25, v2
-; GCN-NEXT: ; implicit-def: $vgpr8
-; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GCN-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; GCN-NEXT: s_cbranch_execnz .LBB0_13
-; GCN-NEXT: .LBB0_2: ; %Flow3
-; GCN-NEXT: s_or_saveexec_b64 s[8:9], s[4:5]
-; GCN-NEXT: v_sub_u32_e32 v6, 0x7f, v7
-; GCN-NEXT: s_xor_b64 exec, exec, s[8:9]
-; GCN-NEXT: s_cbranch_execz .LBB0_11
-; GCN-NEXT: ; %bb.3: ; %NodeBlock
-; GCN-NEXT: v_cmp_lt_i32_e32 vcc, 25, v2
-; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GCN-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
-; GCN-NEXT: s_cbranch_execz .LBB0_7
-; GCN-NEXT: ; %bb.4: ; %LeafBlock
-; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 26, v2
-; GCN-NEXT: s_and_saveexec_b64 s[12:13], vcc
-; GCN-NEXT: s_cbranch_execz .LBB0_6
-; GCN-NEXT: ; %bb.5: ; %itofp-sw-default
-; GCN-NEXT: v_sub_u32_e32 v12, 0x66, v7
-; GCN-NEXT: v_sub_u32_e32 v10, 64, v12
-; GCN-NEXT: v_lshrrev_b64 v[8:9], v12, v[0:1]
-; GCN-NEXT: v_lshlrev_b64 v[10:11], v10, v[4:5]
-; GCN-NEXT: v_sub_u32_e32 v13, 38, v7
-; GCN-NEXT: v_or_b32_e32 v11, v9, v11
-; GCN-NEXT: v_or_b32_e32 v10, v8, v10
-; GCN-NEXT: v_lshrrev_b64 v[8:9], v13, v[4:5]
-; GCN-NEXT: v_cmp_gt_u32_e32 vcc, 64, v12
-; GCN-NEXT: v_add_u32_e32 v14, 26, v7
-; GCN-NEXT: v_cndmask_b32_e32 v9, v9, v11, vcc
-; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v12
-; GCN-NEXT: v_cndmask_b32_e32 v8, v8, v10, vcc
-; GCN-NEXT: v_lshrrev_b64 v[10:11], v13, v[0:1]
-; GCN-NEXT: v_lshlrev_b64 v[12:13], v14, v[4:5]
-; GCN-NEXT: v_subrev_u32_e32 v7, 38, v7
-; GCN-NEXT: v_cndmask_b32_e64 v15, v8, v0, s[4:5]
-; GCN-NEXT: v_lshlrev_b64 v[7:8], v7, v[0:1]
-; GCN-NEXT: v_cndmask_b32_e64 v9, v9, v1, s[4:5]
-; GCN-NEXT: v_or_b32_e32 v11, v13, v11
-; GCN-NEXT: v_or_b32_e32 v10, v12, v10
-; GCN-NEXT: v_cmp_gt_u32_e32 vcc, 64, v14
-; GCN-NEXT: v_lshlrev_b64 v[0:1], v14, v[0:1]
-; GCN-NEXT: v_cndmask_b32_e32 v8, v8, v11, vcc
-; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v14
-; GCN-NEXT: v_cndmask_b32_e32 v7, v7, v10, vcc
-; GCN-NEXT: v_cndmask_b32_e64 v5, v8, v5, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e64 v4, v7, v4, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; GCN-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GCN-NEXT: v_or_b32_e32 v1, v1, v5
-; GCN-NEXT: v_or_b32_e32 v0, v0, v4
-; GCN-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
-; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GCN-NEXT: v_or_b32_e32 v8, v15, v0
-; GCN-NEXT: v_mov_b32_e32 v0, v8
-; GCN-NEXT: v_mov_b32_e32 v1, v9
-; GCN-NEXT: .LBB0_6: ; %Flow1
-; GCN-NEXT: s_or_b64 exec, exec, s[12:13]
-; GCN-NEXT: .LBB0_7: ; %Flow2
-; GCN-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11]
-; GCN-NEXT: ; %bb.8: ; %itofp-sw-bb
-; GCN-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
-; GCN-NEXT: ; %bb.9: ; %itofp-sw-epilog
-; GCN-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN-NEXT: v_lshrrev_b32_e32 v4, 2, v0
-; GCN-NEXT: v_and_or_b32 v0, v4, 1, v0
-; GCN-NEXT: v_add_co_u32_e32 v0, vcc, 1, v0
-; GCN-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GCN-NEXT: v_and_b32_e32 v4, 0x4000000, v0
-; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
-; GCN-NEXT: v_alignbit_b32 v8, v1, v0, 2
-; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GCN-NEXT: s_cbranch_execnz .LBB0_14
-; GCN-NEXT: .LBB0_10: ; %Flow
-; GCN-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN-NEXT: .LBB0_11: ; %Flow4
-; GCN-NEXT: s_or_b64 exec, exec, s[8:9]
-; GCN-NEXT: v_and_b32_e32 v0, 0x80000000, v3
-; GCN-NEXT: v_lshl_add_u32 v1, v6, 23, 1.0
-; GCN-NEXT: v_and_b32_e32 v2, 0x7fffff, v8
-; GCN-NEXT: v_or3_b32 v4, v2, v0, v1
-; GCN-NEXT: .LBB0_12: ; %Flow5
-; GCN-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN-NEXT: v_mov_b32_e32 v0, v4
-; GCN-NEXT: s_setpc_b64 s[30:31]
-; GCN-NEXT: .LBB0_13: ; %itofp-if-else
-; GCN-NEXT: v_add_u32_e32 v2, 0xffffff98, v7
-; GCN-NEXT: v_lshlrev_b64 v[0:1], v2, v[0:1]
-; GCN-NEXT: v_cmp_gt_u32_e32 vcc, 64, v2
-; GCN-NEXT: v_cndmask_b32_e32 v8, 0, v0, vcc
-; GCN-NEXT: ; implicit-def: $vgpr2
-; GCN-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GCN-NEXT: ; implicit-def: $vgpr4_vgpr5
-; GCN-NEXT: s_branch .LBB0_2
-; GCN-NEXT: .LBB0_14: ; %itofp-if-then20
-; GCN-NEXT: v_alignbit_b32 v8, v1, v0, 3
-; GCN-NEXT: v_mov_b32_e32 v6, v2
-; GCN-NEXT: s_branch .LBB0_10
+; SDAG-LABEL: sitofp_i128_to_f32:
+; SDAG: ; %bb.0: ; %itofp-entry
+; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-NEXT: v_or_b32_e32 v5, v1, v3
+; SDAG-NEXT: v_or_b32_e32 v4, v0, v2
+; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SDAG-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; SDAG-NEXT: s_cbranch_execz .LBB0_12
+; SDAG-NEXT: ; %bb.1: ; %itofp-if-end
+; SDAG-NEXT: v_ashrrev_i32_e32 v5, 31, v3
+; SDAG-NEXT: v_xor_b32_e32 v0, v0, v5
+; SDAG-NEXT: v_xor_b32_e32 v1, v1, v5
+; SDAG-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v5
+; SDAG-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v5, vcc
+; SDAG-NEXT: v_xor_b32_e32 v2, v2, v5
+; SDAG-NEXT: v_xor_b32_e32 v6, v3, v5
+; SDAG-NEXT: v_subb_co_u32_e32 v4, vcc, v2, v5, vcc
+; SDAG-NEXT: v_subb_co_u32_e32 v5, vcc, v6, v5, vcc
+; SDAG-NEXT: v_ffbh_u32_e32 v2, v4
+; SDAG-NEXT: v_add_u32_e32 v2, 32, v2
+; SDAG-NEXT: v_ffbh_u32_e32 v6, v5
+; SDAG-NEXT: v_min_u32_e32 v2, v2, v6
+; SDAG-NEXT: v_ffbh_u32_e32 v6, v0
+; SDAG-NEXT: v_add_u32_e32 v6, 32, v6
+; SDAG-NEXT: v_ffbh_u32_e32 v7, v1
+; SDAG-NEXT: v_min_u32_e32 v6, v6, v7
+; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; SDAG-NEXT: v_add_u32_e32 v6, 64, v6
+; SDAG-NEXT: v_cndmask_b32_e32 v7, v6, v2, vcc
+; SDAG-NEXT: v_sub_u32_e32 v2, 0x80, v7
+; SDAG-NEXT: v_cmp_gt_i32_e32 vcc, 25, v2
+; SDAG-NEXT: ; implicit-def: $vgpr8
+; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; SDAG-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
+; SDAG-NEXT: s_cbranch_execnz .LBB0_13
+; SDAG-NEXT: .LBB0_2: ; %Flow3
+; SDAG-NEXT: s_or_saveexec_b64 s[8:9], s[4:5]
+; SDAG-NEXT: v_sub_u32_e32 v6, 0x7f, v7
+; SDAG-NEXT: s_xor_b64 exec, exec, s[8:9]
+; SDAG-NEXT: s_cbranch_execz .LBB0_11
+; SDAG-NEXT: ; %bb.3: ; %NodeBlock
+; SDAG-NEXT: v_cmp_lt_i32_e32 vcc, 25, v2
+; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; SDAG-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
+; SDAG-NEXT: s_cbranch_execz .LBB0_7
+; SDAG-NEXT: ; %bb.4: ; %LeafBlock
+; SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 26, v2
+; SDAG-NEXT: s_and_saveexec_b64 s[12:13], vcc
+; SDAG-NEXT: s_cbranch_execz .LBB0_6
+; SDAG-NEXT: ; %bb.5: ; %itofp-sw-default
+; SDAG-NEXT: v_sub_u32_e32 v12, 0x66, v7
+; SDAG-NEXT: v_sub_u32_e32 v10, 64, v12
+; SDAG-NEXT: v_lshrrev_b64 v[8:9], v12, v[0:1]
+; SDAG-NEXT: v_lshlrev_b64 v[10:11], v10, v[4:5]
+; SDAG-NEXT: v_sub_u32_e32 v13, 38, v7
+; SDAG-NEXT: v_or_b32_e32 v11, v9, v11
+; SDAG-NEXT: v_or_b32_e32 v10, v8, v10
+; SDAG-NEXT: v_lshrrev_b64 v[8:9], v13, v[4:5]
+; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v12
+; SDAG-NEXT: v_add_u32_e32 v14, 26, v7
+; SDAG-NEXT: v_cndmask_b32_e32 v9, v9, v11, vcc
+; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v12
+; SDAG-NEXT: v_cndmask_b32_e32 v8, v8, v10, vcc
+; SDAG-NEXT: v_lshrrev_b64 v[10:11], v13, v[0:1]
+; SDAG-NEXT: v_lshlrev_b64 v[12:13], v14, v[4:5]
+; SDAG-NEXT: v_subrev_u32_e32 v7, 38, v7
+; SDAG-NEXT: v_cndmask_b32_e64 v15, v8, v0, s[4:5]
+; SDAG-NEXT: v_lshlrev_b64 v[7:8], v7, v[0:1]
+; SDAG-NEXT: v_cndmask_b32_e64 v9, v9, v1, s[4:5]
+; SDAG-NEXT: v_or_b32_e32 v11, v13, v11
+; SDAG-NEXT: v_or_b32_e32 v10, v12, v10
+; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v14
+; SDAG-NEXT: v_lshlrev_b64 v[0:1], v14, v[0:1]
+; SDAG-NEXT: v_cndmask_b32_e32 v8, v8, v11, vcc
+; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v14
+; SDAG-NEXT: v_cndmask_b32_e32 v7, v7, v10, vcc
+; SDAG-NEXT: v_cndmask_b32_e64 v5, v8, v5, s[4:5]
+; SDAG-NEXT: v_cndmask_b32_e64 v4, v7, v4, s[4:5]
+; SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
+; SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
+; SDAG-NEXT: v_or_b32_e32 v1, v1, v5
+; SDAG-NEXT: v_or_b32_e32 v0, v0, v4
+; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; SDAG-NEXT: v_or_b32_e32 v8, v15, v0
+; SDAG-NEXT: v_mov_b32_e32 v0, v8
+; SDAG-NEXT: v_mov_b32_e32 v1, v9
+; SDAG-NEXT: .LBB0_6: ; %Flow1
+; SDAG-NEXT: s_or_b64 exec, exec, s[12:13]
+; SDAG-NEXT: .LBB0_7: ; %Flow2
+; SDAG-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11]
+; SDAG-NEXT: ; %bb.8: ; %itofp-sw-bb
+; SDAG-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
+; SDAG-NEXT: ; %bb.9: ; %itofp-sw-epilog
+; SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
+; SDAG-NEXT: v_lshrrev_b32_e32 v4, 2, v0
+; SDAG-NEXT: v_and_or_b32 v0, v4, 1, v0
+; SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 1, v0
+; SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; SDAG-NEXT: v_and_b32_e32 v4, 0x4000000, v0
+; SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
+; SDAG-NEXT: v_alignbit_b32 v8, v1, v0, 2
+; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; SDAG-NEXT: s_cbranch_execnz .LBB0_14
+; SDAG-NEXT: .LBB0_10: ; %Flow
+; SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
+; SDAG-NEXT: .LBB0_11: ; %Flow4
+; SDAG-NEXT: s_or_b64 exec, exec, s[8:9]
+; SDAG-NEXT: v_and_b32_e32 v0, 0x80000000, v3
+; SDAG-NEXT: v_lshl_add_u32 v1, v6, 23, 1.0
+; SDAG-NEXT: v_and_b32_e32 v2, 0x7fffff, v8
+; SDAG-NEXT: v_or3_b32 v4, v2, v0, v1
+; SDAG-NEXT: .LBB0_12: ; %Flow5
+; SDAG-NEXT: s_or_b64 exec, exec, s[6:7]
+; SDAG-NEXT: v_mov_b32_e32 v0, v4
+; SDAG-NEXT: s_setpc_b64 s[30:31]
+; SDAG-NEXT: .LBB0_13: ; %itofp-if-else
+; SDAG-NEXT: v_add_u32_e32 v2, 0xffffff98, v7
+; SDAG-NEXT: v_lshlrev_b64 v[0:1], v2, v[0:1]
+; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v2
+; SDAG-NEXT: v_cndmask_b32_e32 v8, 0, v0, vcc
+; SDAG-NEXT: ; implicit-def: $vgpr2
+; SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
+; SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5
+; SDAG-NEXT: s_branch .LBB0_2
+; SDAG-NEXT: .LBB0_14: ; %itofp-if-then20
+; SDAG-NEXT: v_alignbit_b32 v8, v1, v0, 3
+; SDAG-NEXT: v_mov_b32_e32 v6, v2
+; SDAG-NEXT: s_branch .LBB0_10
+;
+; GISEL-LABEL: sitofp_i128_to_f32:
+; GISEL: ; %bb.0: ; %itofp-entry
+; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT: v_or_b32_e32 v4, v0, v2
+; GISEL-NEXT: v_or_b32_e32 v5, v1, v3
+; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; GISEL-NEXT: s_mov_b32 s4, 0
+; GISEL-NEXT: v_mov_b32_e32 v4, s4
+; GISEL-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GISEL-NEXT: s_cbranch_execz .LBB0_12
+; GISEL-NEXT: ; %bb.1: ; %itofp-if-end
+; GISEL-NEXT: v_ashrrev_i32_e32 v6, 31, v3
+; GISEL-NEXT: v_xor_b32_e32 v0, v6, v0
+; GISEL-NEXT: v_xor_b32_e32 v1, v6, v1
+; GISEL-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v6
+; GISEL-NEXT: v_xor_b32_e32 v2, v6, v2
+; GISEL-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v6, vcc
+; GISEL-NEXT: v_xor_b32_e32 v3, v6, v3
+; GISEL-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v6, vcc
+; GISEL-NEXT: v_ffbh_u32_e32 v5, v0
+; GISEL-NEXT: v_subb_co_u32_e32 v3, vcc, v3, v6, vcc
+; GISEL-NEXT: v_ffbh_u32_e32 v4, v1
+; GISEL-NEXT: v_add_u32_e32 v5, 32, v5
+; GISEL-NEXT: v_ffbh_u32_e32 v7, v2
+; GISEL-NEXT: v_min_u32_e32 v4, v4, v5
+; GISEL-NEXT: v_ffbh_u32_e32 v5, v3
+; GISEL-NEXT: v_add_u32_e32 v7, 32, v7
+; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3]
+; GISEL-NEXT: v_add_u32_e32 v4, 64, v4
+; GISEL-NEXT: v_min_u32_e32 v5, v5, v7
+; GISEL-NEXT: v_cndmask_b32_e32 v5, v5, v4, vcc
+; GISEL-NEXT: v_sub_u32_e32 v7, 0x80, v5
+; GISEL-NEXT: v_cmp_ge_i32_e32 vcc, 24, v7
+; GISEL-NEXT: ; implicit-def: $vgpr4
+; GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
+; GISEL-NEXT: s_cbranch_execnz .LBB0_13
+; GISEL-NEXT: .LBB0_2: ; %Flow3
+; GISEL-NEXT: s_or_saveexec_b64 s[8:9], s[4:5]
+; GISEL-NEXT: v_sub_u32_e32 v8, 0x7f, v5
+; GISEL-NEXT: s_xor_b64 exec, exec, s[8:9]
+; GISEL-NEXT: s_cbranch_execz .LBB0_11
+; GISEL-NEXT: ; %bb.3: ; %NodeBlock
+; GISEL-NEXT: v_cmp_le_i32_e32 vcc, 26, v7
+; GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GISEL-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
+; GISEL-NEXT: s_cbranch_execz .LBB0_7
+; GISEL-NEXT: ; %bb.4: ; %LeafBlock
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 26, v7
+; GISEL-NEXT: s_and_saveexec_b64 s[12:13], vcc
+; GISEL-NEXT: s_cbranch_execz .LBB0_6
+; GISEL-NEXT: ; %bb.5: ; %itofp-sw-default
+; GISEL-NEXT: v_sub_u32_e32 v4, 0x66, v5
+; GISEL-NEXT: v_sub_u32_e32 v11, 64, v4
+; GISEL-NEXT: v_lshrrev_b64 v[9:10], v4, v[0:1]
+; GISEL-NEXT: v_lshlrev_b64 v[11:12], v11, v[2:3]
+; GISEL-NEXT: v_add_u32_e32 v13, 0xffffffc0, v4
+; GISEL-NEXT: v_or_b32_e32 v11, v9, v11
+; GISEL-NEXT: v_or_b32_e32 v12, v10, v12
+; GISEL-NEXT: v_lshrrev_b64 v[9:10], v13, v[2:3]
+; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v4
+; GISEL-NEXT: v_add_u32_e32 v14, 26, v5
+; GISEL-NEXT: v_cndmask_b32_e32 v9, v9, v11, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v10, v10, v12, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
+; GISEL-NEXT: v_sub_u32_e32 v11, 64, v14
+; GISEL-NEXT: v_cndmask_b32_e32 v13, v9, v0, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v4, v10, v1, vcc
+; GISEL-NEXT: v_lshrrev_b64 v[9:10], v14, -1
+; GISEL-NEXT: v_lshlrev_b64 v[11:12], v11, -1
+; GISEL-NEXT: v_add_u32_e32 v5, 0xffffffda, v5
+; GISEL-NEXT: v_or_b32_e32 v15, v9, v11
+; GISEL-NEXT: v_or_b32_e32 v16, v10, v12
+; GISEL-NEXT: v_lshrrev_b64 v[11:12], v5, -1
+; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v14
+; GISEL-NEXT: v_cndmask_b32_e32 v5, v11, v15, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v11, v12, v16, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v14
+; GISEL-NEXT: v_cndmask_b32_e32 v9, 0, v9, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v10, 0, v10, vcc
+; GISEL-NEXT: v_cndmask_b32_e64 v5, v5, -1, s[4:5]
+; GISEL-NEXT: v_cndmask_b32_e64 v11, v11, -1, s[4:5]
+; GISEL-NEXT: v_and_b32_e32 v2, v9, v2
+; GISEL-NEXT: v_and_b32_e32 v3, v10, v3
+; GISEL-NEXT: v_and_or_b32 v0, v5, v0, v2
+; GISEL-NEXT: v_and_or_b32 v1, v11, v1, v3
+; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GISEL-NEXT: v_or_b32_e32 v3, v13, v0
+; GISEL-NEXT: v_mov_b32_e32 v0, v3
+; GISEL-NEXT: v_mov_b32_e32 v1, v4
+; GISEL-NEXT: v_mov_b32_e32 v2, v5
+; GISEL-NEXT: v_mov_b32_e32 v3, v6
+; GISEL-NEXT: .LBB0_6: ; %Flow1
+; GISEL-NEXT: s_or_b64 exec, exec, s[12:13]
+; GISEL-NEXT: .LBB0_7: ; %Flow2
+; GISEL-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11]
+; GISEL-NEXT: ; %bb.8: ; %itofp-sw-bb
+; GISEL-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
+; GISEL-NEXT: ; %bb.9: ; %itofp-sw-epilog
+; GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GISEL-NEXT: v_bfe_u32 v2, v0, 2, 1
+; GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 1, v0
+; GISEL-NEXT: v_bfrev_b32_e32 v2, 32
+; GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GISEL-NEXT: v_mov_b32_e32 v3, 0
+; GISEL-NEXT: v_and_b32_e32 v2, 0x4000000, v0
+; GISEL-NEXT: v_lshrrev_b64 v[4:5], 2, v[0:1]
+; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GISEL-NEXT: s_cbranch_execnz .LBB0_14
+; GISEL-NEXT: .LBB0_10: ; %Flow
+; GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GISEL-NEXT: .LBB0_11: ; %Flow4
+; GISEL-NEXT: s_or_b64 exec, exec, s[8:9]
+; GISEL-NEXT: v_and_b32_e32 v0, 0x80000000, v6
+; GISEL-NEXT: v_lshl_add_u32 v1, v8, 23, 1.0
+; GISEL-NEXT: v_and_b32_e32 v2, 0x7fffff, v4
+; GISEL-NEXT: v_or3_b32 v4, v2, v0, v1
+; GISEL-NEXT: .LBB0_12: ; %Flow5
+; GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
+; GISEL-NEXT: v_mov_b32_e32 v0, v4
+; GISEL-NEXT: s_setpc_b64 s[30:31]
+; GISEL-NEXT: .LBB0_13: ; %itofp-if-else
+; GISEL-NEXT: v_add_u32_e32 v2, 0xffffff98, v5
+; GISEL-NEXT: v_lshlrev_b64 v[0:1], v2, v[0:1]
+; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v2
+; GISEL-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc
+; GISEL-NEXT: ; implicit-def: $vgpr7
+; GISEL-NEXT: ; implicit-def: $vgpr0
+; GISEL-NEXT: ; implicit-def: $vgpr2
+; GISEL-NEXT: s_branch .LBB0_2
+; GISEL-NEXT: .LBB0_14: ; %itofp-if-then20
+; GISEL-NEXT: v_lshrrev_b64 v[4:5], 3, v[0:1]
+; GISEL-NEXT: v_mov_b32_e32 v8, v7
+; GISEL-NEXT: s_branch .LBB0_10
%cvt = sitofp i128 %x to float
ret float %cvt
}
define float @uitofp_i128_to_f32(i128 %x) {
-; GCN-LABEL: uitofp_i128_to_f32:
-; GCN: ; %bb.0: ; %itofp-entry
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_or_b32_e32 v5, v1, v3
-; GCN-NEXT: v_or_b32_e32 v4, v0, v2
-; GCN-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GCN-NEXT: v_mov_b32_e32 v4, 0
-; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; GCN-NEXT: s_cbranch_execz .LBB1_12
-; GCN-NEXT: ; %bb.1: ; %itofp-if-end
-; GCN-NEXT: v_ffbh_u32_e32 v4, v2
-; GCN-NEXT: v_add_u32_e32 v4, 32, v4
-; GCN-NEXT: v_ffbh_u32_e32 v5, v3
-; GCN-NEXT: v_min_u32_e32 v4, v4, v5
-; GCN-NEXT: v_ffbh_u32_e32 v5, v0
-; GCN-NEXT: v_add_u32_e32 v5, 32, v5
-; GCN-NEXT: v_ffbh_u32_e32 v6, v1
-; GCN-NEXT: v_min_u32_e32 v5, v5, v6
-; GCN-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
-; GCN-NEXT: v_add_u32_e32 v5, 64, v5
-; GCN-NEXT: v_cndmask_b32_e32 v6, v5, v4, vcc
-; GCN-NEXT: v_sub_u32_e32 v4, 0x80, v6
-; GCN-NEXT: v_cmp_gt_i32_e32 vcc, 25, v4
-; GCN-NEXT: ; implicit-def: $vgpr7
-; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GCN-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; GCN-NEXT: s_cbranch_execnz .LBB1_13
-; GCN-NEXT: .LBB1_2: ; %Flow3
-; GCN-NEXT: s_or_saveexec_b64 s[8:9], s[4:5]
-; GCN-NEXT: v_sub_u32_e32 v5, 0x7f, v6
-; GCN-NEXT: s_xor_b64 exec, exec, s[8:9]
-; GCN-NEXT: s_cbranch_execz .LBB1_11
-; GCN-NEXT: ; %bb.3: ; %NodeBlock
-; GCN-NEXT: v_cmp_lt_i32_e32 vcc, 25, v4
-; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GCN-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
-; GCN-NEXT: s_cbranch_execz .LBB1_7
-; GCN-NEXT: ; %bb.4: ; %LeafBlock
-; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 26, v4
-; GCN-NEXT: s_and_saveexec_b64 s[12:13], vcc
-; GCN-NEXT: s_cbranch_execz .LBB1_6
-; GCN-NEXT: ; %bb.5: ; %itofp-sw-default
-; GCN-NEXT: v_sub_u32_e32 v11, 0x66, v6
-; GCN-NEXT: v_sub_u32_e32 v9, 64, v11
-; GCN-NEXT: v_lshrrev_b64 v[7:8], v11, v[0:1]
-; GCN-NEXT: v_lshlrev_b64 v[9:10], v9, v[2:3]
-; GCN-NEXT: v_sub_u32_e32 v12, 38, v6
-; GCN-NEXT: v_or_b32_e32 v10, v8, v10
-; GCN-NEXT: v_or_b32_e32 v9, v7, v9
-; GCN-NEXT: v_lshrrev_b64 v[7:8], v12, v[2:3]
-; GCN-NEXT: v_cmp_gt_u32_e32 vcc, 64, v11
-; GCN-NEXT: v_add_u32_e32 v13, 26, v6
-; GCN-NEXT: v_cndmask_b32_e32 v8, v8, v10, vcc
-; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v11
-; GCN-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc
-; GCN-NEXT: v_lshrrev_b64 v[9:10], v12, v[0:1]
-; GCN-NEXT: v_lshlrev_b64 v[11:12], v13, v[2:3]
-; GCN-NEXT: v_subrev_u32_e32 v6, 38, v6
-; GCN-NEXT: v_cndmask_b32_e64 v14, v7, v0, s[4:5]
-; GCN-NEXT: v_lshlrev_b64 v[6:7], v6, v[0:1]
-; GCN-NEXT: v_cndmask_b32_e64 v8, v8, v1, s[4:5]
-; GCN-NEXT: v_or_b32_e32 v10, v12, v10
-; GCN-NEXT: v_or_b32_e32 v9, v11, v9
-; GCN-NEXT: v_cmp_gt_u32_e32 vcc, 64, v13
-; GCN-NEXT: v_lshlrev_b64 v[0:1], v13, v[0:1]
-; GCN-NEXT: v_cndmask_b32_e32 v7, v7, v10, vcc
-; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v13
-; GCN-NEXT: v_cndmask_b32_e32 v6, v6, v9, vcc
-; GCN-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e64 v2, v6, v2, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; GCN-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GCN-NEXT: v_or_b32_e32 v1, v1, v3
-; GCN-NEXT: v_or_b32_e32 v0, v0, v2
-; GCN-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
-; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GCN-NEXT: v_or_b32_e32 v7, v14, v0
-; GCN-NEXT: v_mov_b32_e32 v0, v7
-; GCN-NEXT: v_mov_b32_e32 v1, v8
-; GCN-NEXT: .LBB1_6: ; %Flow1
-; GCN-NEXT: s_or_b64 exec, exec, s[12:13]
-; GCN-NEXT: .LBB1_7: ; %Flow2
-; GCN-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11]
-; GCN-NEXT: ; %bb.8: ; %itofp-sw-bb
-; GCN-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
-; GCN-NEXT: ; %bb.9: ; %itofp-sw-epilog
-; GCN-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN-NEXT: v_lshrrev_b32_e32 v2, 2, v0
-; GCN-NEXT: v_and_or_b32 v0, v2, 1, v0
-; GCN-NEXT: v_add_co_u32_e32 v0, vcc, 1, v0
-; GCN-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GCN-NEXT: v_and_b32_e32 v2, 0x4000000, v0
-; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
-; GCN-NEXT: v_alignbit_b32 v7, v1, v0, 2
-; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GCN-NEXT: s_cbranch_execnz .LBB1_14
-; GCN-NEXT: .LBB1_10: ; %Flow
-; GCN-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN-NEXT: .LBB1_11: ; %Flow4
-; GCN-NEXT: s_or_b64 exec, exec, s[8:9]
-; GCN-NEXT: v_and_b32_e32 v0, 0x7fffff, v7
-; GCN-NEXT: v_lshl_or_b32 v0, v5, 23, v0
-; GCN-NEXT: v_add_u32_e32 v4, 1.0, v0
-; GCN-NEXT: .LBB1_12: ; %Flow5
-; GCN-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN-NEXT: v_mov_b32_e32 v0, v4
-; GCN-NEXT: s_setpc_b64 s[30:31]
-; GCN-NEXT: .LBB1_13: ; %itofp-if-else
-; GCN-NEXT: v_add_u32_e32 v2, 0xffffff98, v6
-; GCN-NEXT: v_lshlrev_b64 v[0:1], v2, v[0:1]
-; GCN-NEXT: v_cmp_gt_u32_e32 vcc, 64, v2
-; GCN-NEXT: v_cndmask_b32_e32 v7, 0, v0, vcc
-; GCN-NEXT: ; implicit-def: $vgpr4
-; GCN-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GCN-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GCN-NEXT: s_branch .LBB1_2
-; GCN-NEXT: .LBB1_14: ; %itofp-if-then20
-; GCN-NEXT: v_alignbit_b32 v7, v1, v0, 3
-; GCN-NEXT: v_mov_b32_e32 v5, v4
-; GCN-NEXT: s_branch .LBB1_10
+; SDAG-LABEL: uitofp_i128_to_f32:
+; SDAG: ; %bb.0: ; %itofp-entry
+; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-NEXT: v_or_b32_e32 v5, v1, v3
+; SDAG-NEXT: v_or_b32_e32 v4, v0, v2
+; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SDAG-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; SDAG-NEXT: s_cbranch_execz .LBB1_12
+; SDAG-NEXT: ; %bb.1: ; %itofp-if-end
+; SDAG-NEXT: v_ffbh_u32_e32 v4, v2
+; SDAG-NEXT: v_add_u32_e32 v4, 32, v4
+; SDAG-NEXT: v_ffbh_u32_e32 v5, v3
+; SDAG-NEXT: v_min_u32_e32 v4, v4, v5
+; SDAG-NEXT: v_ffbh_u32_e32 v5, v0
+; SDAG-NEXT: v_add_u32_e32 v5, 32, v5
+; SDAG-NEXT: v_ffbh_u32_e32 v6, v1
+; SDAG-NEXT: v_min_u32_e32 v5, v5, v6
+; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; SDAG-NEXT: v_add_u32_e32 v5, 64, v5
+; SDAG-NEXT: v_cndmask_b32_e32 v6, v5, v4, vcc
+; SDAG-NEXT: v_sub_u32_e32 v4, 0x80, v6
+; SDAG-NEXT: v_cmp_gt_i32_e32 vcc, 25, v4
+; SDAG-NEXT: ; implicit-def: $vgpr7
+; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; SDAG-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
+; SDAG-NEXT: s_cbranch_execnz .LBB1_13
+; SDAG-NEXT: .LBB1_2: ; %Flow3
+; SDAG-NEXT: s_or_saveexec_b64 s[8:9], s[4:5]
+; SDAG-NEXT: v_sub_u32_e32 v5, 0x7f, v6
+; SDAG-NEXT: s_xor_b64 exec, exec, s[8:9]
+; SDAG-NEXT: s_cbranch_execz .LBB1_11
+; SDAG-NEXT: ; %bb.3: ; %NodeBlock
+; SDAG-NEXT: v_cmp_lt_i32_e32 vcc, 25, v4
+; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; SDAG-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
+; SDAG-NEXT: s_cbranch_execz .LBB1_7
+; SDAG-NEXT: ; %bb.4: ; %LeafBlock
+; SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 26, v4
+; SDAG-NEXT: s_and_saveexec_b64 s[12:13], vcc
+; SDAG-NEXT: s_cbranch_execz .LBB1_6
+; SDAG-NEXT: ; %bb.5: ; %itofp-sw-default
+; SDAG-NEXT: v_sub_u32_e32 v11, 0x66, v6
+; SDAG-NEXT: v_sub_u32_e32 v9, 64, v11
+; SDAG-NEXT: v_lshrrev_b64 v[7:8], v11, v[0:1]
+; SDAG-NEXT: v_lshlrev_b64 v[9:10], v9, v[2:3]
+; SDAG-NEXT: v_sub_u32_e32 v12, 38, v6
+; SDAG-NEXT: v_or_b32_e32 v10, v8, v10
+; SDAG-NEXT: v_or_b32_e32 v9, v7, v9
+; SDAG-NEXT: v_lshrrev_b64 v[7:8], v12, v[2:3]
+; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v11
+; SDAG-NEXT: v_add_u32_e32 v13, 26, v6
+; SDAG-NEXT: v_cndmask_b32_e32 v8, v8, v10, vcc
+; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v11
+; SDAG-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc
+; SDAG-NEXT: v_lshrrev_b64 v[9:10], v12, v[0:1]
+; SDAG-NEXT: v_lshlrev_b64 v[11:12], v13, v[2:3]
+; SDAG-NEXT: v_subrev_u32_e32 v6, 38, v6
+; SDAG-NEXT: v_cndmask_b32_e64 v14, v7, v0, s[4:5]
+; SDAG-NEXT: v_lshlrev_b64 v[6:7], v6, v[0:1]
+; SDAG-NEXT: v_cndmask_b32_e64 v8, v8, v1, s[4:5]
+; SDAG-NEXT: v_or_b32_e32 v10, v12, v10
+; SDAG-NEXT: v_or_b32_e32 v9, v11, v9
+; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v13
+; SDAG-NEXT: v_lshlrev_b64 v[0:1], v13, v[0:1]
+; SDAG-NEXT: v_cndmask_b32_e32 v7, v7, v10, vcc
+; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v13
+; SDAG-NEXT: v_cndmask_b32_e32 v6, v6, v9, vcc
+; SDAG-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[4:5]
+; SDAG-NEXT: v_cndmask_b32_e64 v2, v6, v2, s[4:5]
+; SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
+; SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
+; SDAG-NEXT: v_or_b32_e32 v1, v1, v3
+; SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; SDAG-NEXT: v_or_b32_e32 v7, v14, v0
+; SDAG-NEXT: v_mov_b32_e32 v0, v7
+; SDAG-NEXT: v_mov_b32_e32 v1, v8
+; SDAG-NEXT: .LBB1_6: ; %Flow1
+; SDAG-NEXT: s_or_b64 exec, exec, s[12:13]
+; SDAG-NEXT: .LBB1_7: ; %Flow2
+; SDAG-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11]
+; SDAG-NEXT: ; %bb.8: ; %itofp-sw-bb
+; SDAG-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
+; SDAG-NEXT: ; %bb.9: ; %itofp-sw-epilog
+; SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
+; SDAG-NEXT: v_lshrrev_b32_e32 v2, 2, v0
+; SDAG-NEXT: v_and_or_b32 v0, v2, 1, v0
+; SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 1, v0
+; SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; SDAG-NEXT: v_and_b32_e32 v2, 0x4000000, v0
+; SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
+; SDAG-NEXT: v_alignbit_b32 v7, v1, v0, 2
+; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; SDAG-NEXT: s_cbranch_execnz .LBB1_14
+; SDAG-NEXT: .LBB1_10: ; %Flow
+; SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
+; SDAG-NEXT: .LBB1_11: ; %Flow4
+; SDAG-NEXT: s_or_b64 exec, exec, s[8:9]
+; SDAG-NEXT: v_and_b32_e32 v0, 0x7fffff, v7
+; SDAG-NEXT: v_lshl_or_b32 v0, v5, 23, v0
+; SDAG-NEXT: v_add_u32_e32 v4, 1.0, v0
+; SDAG-NEXT: .LBB1_12: ; %Flow5
+; SDAG-NEXT: s_or_b64 exec, exec, s[6:7]
+; SDAG-NEXT: v_mov_b32_e32 v0, v4
+; SDAG-NEXT: s_setpc_b64 s[30:31]
+; SDAG-NEXT: .LBB1_13: ; %itofp-if-else
+; SDAG-NEXT: v_add_u32_e32 v2, 0xffffff98, v6
+; SDAG-NEXT: v_lshlrev_b64 v[0:1], v2, v[0:1]
+; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v2
+; SDAG-NEXT: v_cndmask_b32_e32 v7, 0, v0, vcc
+; SDAG-NEXT: ; implicit-def: $vgpr4
+; SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
+; SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
+; SDAG-NEXT: s_branch .LBB1_2
+; SDAG-NEXT: .LBB1_14: ; %itofp-if-then20
+; SDAG-NEXT: v_alignbit_b32 v7, v1, v0, 3
+; SDAG-NEXT: v_mov_b32_e32 v5, v4
+; SDAG-NEXT: s_branch .LBB1_10
+;
+; GISEL-LABEL: uitofp_i128_to_f32:
+; GISEL: ; %bb.0: ; %itofp-entry
+; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT: v_or_b32_e32 v4, v0, v2
+; GISEL-NEXT: v_or_b32_e32 v5, v1, v3
+; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; GISEL-NEXT: s_mov_b32 s4, 0
+; GISEL-NEXT: v_mov_b32_e32 v4, s4
+; GISEL-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GISEL-NEXT: s_cbranch_execz .LBB1_12
+; GISEL-NEXT: ; %bb.1: ; %itofp-if-end
+; GISEL-NEXT: v_ffbh_u32_e32 v5, v0
+; GISEL-NEXT: v_ffbh_u32_e32 v4, v1
+; GISEL-NEXT: v_add_u32_e32 v5, 32, v5
+; GISEL-NEXT: v_ffbh_u32_e32 v6, v2
+; GISEL-NEXT: v_min_u32_e32 v4, v4, v5
+; GISEL-NEXT: v_ffbh_u32_e32 v5, v3
+; GISEL-NEXT: v_add_u32_e32 v6, 32, v6
+; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3]
+; GISEL-NEXT: v_add_u32_e32 v4, 64, v4
+; GISEL-NEXT: v_min_u32_e32 v5, v5, v6
+; GISEL-NEXT: v_cndmask_b32_e32 v5, v5, v4, vcc
+; GISEL-NEXT: v_sub_u32_e32 v6, 0x80, v5
+; GISEL-NEXT: v_cmp_ge_i32_e32 vcc, 24, v6
+; GISEL-NEXT: ; implicit-def: $vgpr4
+; GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
+; GISEL-NEXT: s_cbranch_execnz .LBB1_13
+; GISEL-NEXT: .LBB1_2: ; %Flow3
+; GISEL-NEXT: s_or_saveexec_b64 s[8:9], s[4:5]
+; GISEL-NEXT: v_sub_u32_e32 v7, 0x7f, v5
+; GISEL-NEXT: s_xor_b64 exec, exec, s[8:9]
+; GISEL-NEXT: s_cbranch_execz .LBB1_11
+; GISEL-NEXT: ; %bb.3: ; %NodeBlock
+; GISEL-NEXT: v_cmp_le_i32_e32 vcc, 26, v6
+; GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GISEL-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
+; GISEL-NEXT: s_cbranch_execz .LBB1_7
+; GISEL-NEXT: ; %bb.4: ; %LeafBlock
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 26, v6
+; GISEL-NEXT: s_and_saveexec_b64 s[12:13], vcc
+; GISEL-NEXT: s_cbranch_execz .LBB1_6
+; GISEL-NEXT: ; %bb.5: ; %itofp-sw-default
+; GISEL-NEXT: v_sub_u32_e32 v4, 0x66, v5
+; GISEL-NEXT: v_sub_u32_e32 v10, 64, v4
+; GISEL-NEXT: v_lshrrev_b64 v[8:9], v4, v[0:1]
+; GISEL-NEXT: v_lshlrev_b64 v[10:11], v10, v[2:3]
+; GISEL-NEXT: v_add_u32_e32 v12, 0xffffffc0, v4
+; GISEL-NEXT: v_or_b32_e32 v10, v8, v10
+; GISEL-NEXT: v_or_b32_e32 v11, v9, v11
+; GISEL-NEXT: v_lshrrev_b64 v[8:9], v12, v[2:3]
+; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v4
+; GISEL-NEXT: v_add_u32_e32 v13, 26, v5
+; GISEL-NEXT: v_cndmask_b32_e32 v8, v8, v10, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v9, v9, v11, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
+; GISEL-NEXT: v_sub_u32_e32 v10, 64, v13
+; GISEL-NEXT: v_cndmask_b32_e32 v12, v8, v0, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v4, v9, v1, vcc
+; GISEL-NEXT: v_lshrrev_b64 v[8:9], v13, -1
+; GISEL-NEXT: v_lshlrev_b64 v[10:11], v10, -1
+; GISEL-NEXT: v_add_u32_e32 v5, 0xffffffda, v5
+; GISEL-NEXT: v_or_b32_e32 v14, v8, v10
+; GISEL-NEXT: v_or_b32_e32 v15, v9, v11
+; GISEL-NEXT: v_lshrrev_b64 v[10:11], v5, -1
+; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v13
+; GISEL-NEXT: v_cndmask_b32_e32 v5, v10, v14, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v10, v11, v15, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v13
+; GISEL-NEXT: v_cndmask_b32_e32 v8, 0, v8, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v9, 0, v9, vcc
+; GISEL-NEXT: v_cndmask_b32_e64 v5, v5, -1, s[4:5]
+; GISEL-NEXT: v_cndmask_b32_e64 v10, v10, -1, s[4:5]
+; GISEL-NEXT: v_and_b32_e32 v2, v8, v2
+; GISEL-NEXT: v_and_b32_e32 v3, v9, v3
+; GISEL-NEXT: v_and_or_b32 v0, v5, v0, v2
+; GISEL-NEXT: v_and_or_b32 v1, v10, v1, v3
+; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GISEL-NEXT: v_or_b32_e32 v3, v12, v0
+; GISEL-NEXT: v_mov_b32_e32 v0, v3
+; GISEL-NEXT: v_mov_b32_e32 v1, v4
+; GISEL-NEXT: v_mov_b32_e32 v2, v5
+; GISEL-NEXT: v_mov_b32_e32 v3, v6
+; GISEL-NEXT: .LBB1_6: ; %Flow1
+; GISEL-NEXT: s_or_b64 exec, exec, s[12:13]
+; GISEL-NEXT: .LBB1_7: ; %Flow2
+; GISEL-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11]
+; GISEL-NEXT: ; %bb.8: ; %itofp-sw-bb
+; GISEL-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
+; GISEL-NEXT: ; %bb.9: ; %itofp-sw-epilog
+; GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GISEL-NEXT: v_bfe_u32 v2, v0, 2, 1
+; GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 1, v0
+; GISEL-NEXT: v_bfrev_b32_e32 v2, 32
+; GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GISEL-NEXT: v_mov_b32_e32 v3, 0
+; GISEL-NEXT: v_and_b32_e32 v2, 0x4000000, v0
+; GISEL-NEXT: v_lshrrev_b64 v[4:5], 2, v[0:1]
+; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GISEL-NEXT: s_cbranch_execnz .LBB1_14
+; GISEL-NEXT: .LBB1_10: ; %Flow
+; GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GISEL-NEXT: .LBB1_11: ; %Flow4
+; GISEL-NEXT: s_or_b64 exec, exec, s[8:9]
+; GISEL-NEXT: v_lshl_add_u32 v0, v7, 23, 1.0
+; GISEL-NEXT: v_mov_b32_e32 v1, 0x7fffff
+; GISEL-NEXT: v_and_or_b32 v4, v4, v1, v0
+; GISEL-NEXT: .LBB1_12: ; %Flow5
+; GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
+; GISEL-NEXT: v_mov_b32_e32 v0, v4
+; GISEL-NEXT: s_setpc_b64 s[30:31]
+; GISEL-NEXT: .LBB1_13: ; %itofp-if-else
+; GISEL-NEXT: v_add_u32_e32 v2, 0xffffff98, v5
+; GISEL-NEXT: v_lshlrev_b64 v[0:1], v2, v[0:1]
+; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v2
+; GISEL-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc
+; GISEL-NEXT: ; implicit-def: $vgpr6
+; GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GISEL-NEXT: s_branch .LBB1_2
+; GISEL-NEXT: .LBB1_14: ; %itofp-if-then20
+; GISEL-NEXT: v_lshrrev_b64 v[4:5], 3, v[0:1]
+; GISEL-NEXT: v_mov_b32_e32 v7, v6
+; GISEL-NEXT: s_branch .LBB1_10
%cvt = uitofp i128 %x to float
ret float %cvt
}
define double @sitofp_i128_to_f64(i128 %x) {
-; GCN-LABEL: sitofp_i128_to_f64:
-; GCN: ; %bb.0: ; %itofp-entry
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v5, v1
-; GCN-NEXT: v_mov_b32_e32 v4, v0
-; GCN-NEXT: v_or_b32_e32 v1, v5, v3
-; GCN-NEXT: v_or_b32_e32 v0, v4, v2
-; GCN-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
-; GCN-NEXT: v_mov_b32_e32 v0, 0
-; GCN-NEXT: v_mov_b32_e32 v1, 0
-; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; GCN-NEXT: s_cbranch_execz .LBB2_12
-; GCN-NEXT: ; %bb.1: ; %itofp-if-end
-; GCN-NEXT: v_ashrrev_i32_e32 v0, 31, v3
-; GCN-NEXT: v_xor_b32_e32 v4, v4, v0
-; GCN-NEXT: v_xor_b32_e32 v1, v5, v0
-; GCN-NEXT: v_sub_co_u32_e32 v4, vcc, v4, v0
-; GCN-NEXT: v_subb_co_u32_e32 v5, vcc, v1, v0, vcc
-; GCN-NEXT: v_xor_b32_e32 v2, v2, v0
-; GCN-NEXT: v_xor_b32_e32 v1, v3, v0
-; GCN-NEXT: v_subb_co_u32_e32 v6, vcc, v2, v0, vcc
-; GCN-NEXT: v_subb_co_u32_e32 v7, vcc, v1, v0, vcc
-; GCN-NEXT: v_ffbh_u32_e32 v0, v6
-; GCN-NEXT: v_add_u32_e32 v0, 32, v0
-; GCN-NEXT: v_ffbh_u32_e32 v1, v7
-; GCN-NEXT: v_min_u32_e32 v0, v0, v1
-; GCN-NEXT: v_ffbh_u32_e32 v1, v4
-; GCN-NEXT: v_add_u32_e32 v1, 32, v1
-; GCN-NEXT: v_ffbh_u32_e32 v2, v5
-; GCN-NEXT: v_min_u32_e32 v1, v1, v2
-; GCN-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
-; GCN-NEXT: v_add_u32_e32 v1, 64, v1
-; GCN-NEXT: v_cndmask_b32_e32 v9, v1, v0, vcc
-; GCN-NEXT: v_sub_u32_e32 v2, 0x80, v9
-; GCN-NEXT: v_cmp_gt_i32_e32 vcc, 54, v2
-; GCN-NEXT: ; implicit-def: $vgpr10
-; GCN-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GCN-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; GCN-NEXT: s_cbranch_execnz .LBB2_13
-; GCN-NEXT: .LBB2_2: ; %Flow3
-; GCN-NEXT: s_or_saveexec_b64 s[8:9], s[4:5]
-; GCN-NEXT: v_sub_u32_e32 v8, 0x7f, v9
-; GCN-NEXT: s_xor_b64 exec, exec, s[8:9]
-; GCN-NEXT: s_cbranch_execz .LBB2_11
-; GCN-NEXT: ; %bb.3: ; %NodeBlock
-; GCN-NEXT: v_cmp_lt_i32_e32 vcc, 54, v2
-; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GCN-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
-; GCN-NEXT: s_cbranch_execz .LBB2_7
-; GCN-NEXT: ; %bb.4: ; %LeafBlock
-; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 55, v2
-; GCN-NEXT: s_and_saveexec_b64 s[12:13], vcc
-; GCN-NEXT: s_cbranch_execz .LBB2_6
-; GCN-NEXT: ; %bb.5: ; %itofp-sw-default
-; GCN-NEXT: v_sub_u32_e32 v12, 0x49, v9
-; GCN-NEXT: v_sub_u32_e32 v10, 64, v12
-; GCN-NEXT: v_lshrrev_b64 v[0:1], v12, v[4:5]
-; GCN-NEXT: v_lshlrev_b64 v[10:11], v10, v[6:7]
-; GCN-NEXT: v_sub_u32_e32 v13, 9, v9
-; GCN-NEXT: v_or_b32_e32 v11, v1, v11
-; GCN-NEXT: v_or_b32_e32 v10, v0, v10
-; GCN-NEXT: v_lshrrev_b64 v[0:1], v13, v[6:7]
-; GCN-NEXT: v_cmp_gt_u32_e32 vcc, 64, v12
-; GCN-NEXT: v_add_u32_e32 v16, 55, v9
-; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v11, vcc
-; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v12
-; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v10, vcc
-; GCN-NEXT: v_lshrrev_b64 v[10:11], v12, v[6:7]
-; GCN-NEXT: v_lshrrev_b64 v[12:13], v13, v[4:5]
-; GCN-NEXT: v_lshlrev_b64 v[14:15], v16, v[6:7]
-; GCN-NEXT: v_add_u32_e32 v9, -9, v9
-; GCN-NEXT: v_or_b32_e32 v15, v15, v13
-; GCN-NEXT: v_or_b32_e32 v14, v14, v12
-; GCN-NEXT: v_lshlrev_b64 v[12:13], v9, v[4:5]
-; GCN-NEXT: v_cndmask_b32_e32 v11, 0, v11, vcc
-; GCN-NEXT: v_cndmask_b32_e32 v10, 0, v10, vcc
-; GCN-NEXT: v_cmp_gt_u32_e32 vcc, 64, v16
-; GCN-NEXT: v_cndmask_b32_e64 v1, v1, v5, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e64 v0, v0, v4, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e32 v9, v13, v15, vcc
-; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v16
-; GCN-NEXT: v_lshlrev_b64 v[4:5], v16, v[4:5]
-; GCN-NEXT: v_cndmask_b32_e64 v7, v9, v7, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e32 v9, v12, v14, vcc
-; GCN-NEXT: v_cndmask_b32_e64 v6, v9, v6, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc
-; GCN-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
-; GCN-NEXT: v_or_b32_e32 v5, v5, v7
-; GCN-NEXT: v_or_b32_e32 v4, v4, v6
-; GCN-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GCN-NEXT: v_mov_b32_e32 v6, v10
-; GCN-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; GCN-NEXT: v_or_b32_e32 v0, v0, v4
-; GCN-NEXT: v_mov_b32_e32 v5, v1
-; GCN-NEXT: v_mov_b32_e32 v4, v0
-; GCN-NEXT: v_mov_b32_e32 v7, v11
-; GCN-NEXT: .LBB2_6: ; %Flow1
-; GCN-NEXT: s_or_b64 exec, exec, s[12:13]
-; GCN-NEXT: .LBB2_7: ; %Flow2
-; GCN-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11]
-; GCN-NEXT: ; %bb.8: ; %itofp-sw-bb
-; GCN-NEXT: v_lshlrev_b64 v[6:7], 1, v[6:7]
-; GCN-NEXT: v_lshrrev_b32_e32 v0, 31, v5
-; GCN-NEXT: v_lshlrev_b64 v[4:5], 1, v[4:5]
-; GCN-NEXT: v_or_b32_e32 v6, v6, v0
-; GCN-NEXT: ; %bb.9: ; %itofp-sw-epilog
-; GCN-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN-NEXT: v_lshrrev_b32_e32 v0, 2, v4
-; GCN-NEXT: v_and_or_b32 v0, v0, 1, v4
-; GCN-NEXT: v_add_co_u32_e32 v4, vcc, 1, v0
-; GCN-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
-; GCN-NEXT: v_addc_co_u32_e32 v6, vcc, 0, v6, vcc
-; GCN-NEXT: v_lshrrev_b64 v[0:1], 2, v[4:5]
-; GCN-NEXT: v_lshlrev_b32_e32 v7, 30, v6
-; GCN-NEXT: v_or_b32_e32 v10, v1, v7
-; GCN-NEXT: v_and_b32_e32 v1, 0x800000, v5
-; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
-; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GCN-NEXT: s_cbranch_execnz .LBB2_14
-; GCN-NEXT: .LBB2_10: ; %Flow
-; GCN-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN-NEXT: .LBB2_11: ; %Flow4
-; GCN-NEXT: s_or_b64 exec, exec, s[8:9]
-; GCN-NEXT: v_mov_b32_e32 v2, 0x3ff00000
-; GCN-NEXT: v_and_b32_e32 v1, 0x80000000, v3
-; GCN-NEXT: v_lshl_add_u32 v2, v8, 20, v2
-; GCN-NEXT: v_and_b32_e32 v3, 0xfffff, v10
-; GCN-NEXT: v_or3_b32 v1, v3, v1, v2
-; GCN-NEXT: .LBB2_12: ; %Flow5
-; GCN-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN-NEXT: s_setpc_b64 s[30:31]
-; GCN-NEXT: .LBB2_13: ; %itofp-if-else
-; GCN-NEXT: v_add_u32_e32 v2, 0xffffffb5, v9
-; GCN-NEXT: v_lshlrev_b64 v[0:1], v2, v[4:5]
-; GCN-NEXT: v_cmp_gt_u32_e32 vcc, 64, v2
-; GCN-NEXT: v_cndmask_b32_e32 v10, 0, v1, vcc
-; GCN-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GCN-NEXT: ; implicit-def: $vgpr2
-; GCN-NEXT: ; implicit-def: $vgpr6_vgpr7
-; GCN-NEXT: ; implicit-def: $vgpr4_vgpr5
-; GCN-NEXT: s_branch .LBB2_2
-; GCN-NEXT: .LBB2_14: ; %itofp-if-then20
-; GCN-NEXT: v_lshrrev_b64 v[0:1], 3, v[4:5]
-; GCN-NEXT: v_lshlrev_b32_e32 v4, 29, v6
-; GCN-NEXT: v_or_b32_e32 v10, v1, v4
-; GCN-NEXT: v_mov_b32_e32 v8, v2
-; GCN-NEXT: s_branch .LBB2_10
+; SDAG-LABEL: sitofp_i128_to_f64:
+; SDAG: ; %bb.0: ; %itofp-entry
+; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-NEXT: v_mov_b32_e32 v5, v1
+; SDAG-NEXT: v_mov_b32_e32 v4, v0
+; SDAG-NEXT: v_or_b32_e32 v1, v5, v3
+; SDAG-NEXT: v_or_b32_e32 v0, v4, v2
+; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; SDAG-NEXT: v_mov_b32_e32 v0, 0
+; SDAG-NEXT: v_mov_b32_e32 v1, 0
+; SDAG-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; SDAG-NEXT: s_cbranch_execz .LBB2_12
+; SDAG-NEXT: ; %bb.1: ; %itofp-if-end
+; SDAG-NEXT: v_ashrrev_i32_e32 v0, 31, v3
+; SDAG-NEXT: v_xor_b32_e32 v4, v4, v0
+; SDAG-NEXT: v_xor_b32_e32 v1, v5, v0
+; SDAG-NEXT: v_sub_co_u32_e32 v4, vcc, v4, v0
+; SDAG-NEXT: v_subb_co_u32_e32 v5, vcc, v1, v0, vcc
+; SDAG-NEXT: v_xor_b32_e32 v2, v2, v0
+; SDAG-NEXT: v_xor_b32_e32 v1, v3, v0
+; SDAG-NEXT: v_subb_co_u32_e32 v6, vcc, v2, v0, vcc
+; SDAG-NEXT: v_subb_co_u32_e32 v7, vcc, v1, v0, vcc
+; SDAG-NEXT: v_ffbh_u32_e32 v0, v6
+; SDAG-NEXT: v_add_u32_e32 v0, 32, v0
+; SDAG-NEXT: v_ffbh_u32_e32 v1, v7
+; SDAG-NEXT: v_min_u32_e32 v0, v0, v1
+; SDAG-NEXT: v_ffbh_u32_e32 v1, v4
+; SDAG-NEXT: v_add_u32_e32 v1, 32, v1
+; SDAG-NEXT: v_ffbh_u32_e32 v2, v5
+; SDAG-NEXT: v_min_u32_e32 v1, v1, v2
+; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; SDAG-NEXT: v_add_u32_e32 v1, 64, v1
+; SDAG-NEXT: v_cndmask_b32_e32 v9, v1, v0, vcc
+; SDAG-NEXT: v_sub_u32_e32 v2, 0x80, v9
+; SDAG-NEXT: v_cmp_gt_i32_e32 vcc, 54, v2
+; SDAG-NEXT: ; implicit-def: $vgpr10
+; SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
+; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; SDAG-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
+; SDAG-NEXT: s_cbranch_execnz .LBB2_13
+; SDAG-NEXT: .LBB2_2: ; %Flow3
+; SDAG-NEXT: s_or_saveexec_b64 s[8:9], s[4:5]
+; SDAG-NEXT: v_sub_u32_e32 v8, 0x7f, v9
+; SDAG-NEXT: s_xor_b64 exec, exec, s[8:9]
+; SDAG-NEXT: s_cbranch_execz .LBB2_11
+; SDAG-NEXT: ; %bb.3: ; %NodeBlock
+; SDAG-NEXT: v_cmp_lt_i32_e32 vcc, 54, v2
+; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; SDAG-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
+; SDAG-NEXT: s_cbranch_execz .LBB2_7
+; SDAG-NEXT: ; %bb.4: ; %LeafBlock
+; SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 55, v2
+; SDAG-NEXT: s_and_saveexec_b64 s[12:13], vcc
+; SDAG-NEXT: s_cbranch_execz .LBB2_6
+; SDAG-NEXT: ; %bb.5: ; %itofp-sw-default
+; SDAG-NEXT: v_sub_u32_e32 v12, 0x49, v9
+; SDAG-NEXT: v_sub_u32_e32 v10, 64, v12
+; SDAG-NEXT: v_lshrrev_b64 v[0:1], v12, v[4:5]
+; SDAG-NEXT: v_lshlrev_b64 v[10:11], v10, v[6:7]
+; SDAG-NEXT: v_sub_u32_e32 v13, 9, v9
+; SDAG-NEXT: v_or_b32_e32 v11, v1, v11
+; SDAG-NEXT: v_or_b32_e32 v10, v0, v10
+; SDAG-NEXT: v_lshrrev_b64 v[0:1], v13, v[6:7]
+; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v12
+; SDAG-NEXT: v_add_u32_e32 v16, 55, v9
+; SDAG-NEXT: v_cndmask_b32_e32 v1, v1, v11, vcc
+; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v12
+; SDAG-NEXT: v_cndmask_b32_e32 v0, v0, v10, vcc
+; SDAG-NEXT: v_lshrrev_b64 v[10:11], v12, v[6:7]
+; SDAG-NEXT: v_lshrrev_b64 v[12:13], v13, v[4:5]
+; SDAG-NEXT: v_lshlrev_b64 v[14:15], v16, v[6:7]
+; SDAG-NEXT: v_add_u32_e32 v9, -9, v9
+; SDAG-NEXT: v_or_b32_e32 v15, v15, v13
+; SDAG-NEXT: v_or_b32_e32 v14, v14, v12
+; SDAG-NEXT: v_lshlrev_b64 v[12:13], v9, v[4:5]
+; SDAG-NEXT: v_cndmask_b32_e32 v11, 0, v11, vcc
+; SDAG-NEXT: v_cndmask_b32_e32 v10, 0, v10, vcc
+; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v16
+; SDAG-NEXT: v_cndmask_b32_e64 v1, v1, v5, s[4:5]
+; SDAG-NEXT: v_cndmask_b32_e64 v0, v0, v4, s[4:5]
+; SDAG-NEXT: v_cndmask_b32_e32 v9, v13, v15, vcc
+; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v16
+; SDAG-NEXT: v_lshlrev_b64 v[4:5], v16, v[4:5]
+; SDAG-NEXT: v_cndmask_b32_e64 v7, v9, v7, s[4:5]
+; SDAG-NEXT: v_cndmask_b32_e32 v9, v12, v14, vcc
+; SDAG-NEXT: v_cndmask_b32_e64 v6, v9, v6, s[4:5]
+; SDAG-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc
+; SDAG-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
+; SDAG-NEXT: v_or_b32_e32 v5, v5, v7
+; SDAG-NEXT: v_or_b32_e32 v4, v4, v6
+; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; SDAG-NEXT: v_mov_b32_e32 v6, v10
+; SDAG-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
+; SDAG-NEXT: v_or_b32_e32 v0, v0, v4
+; SDAG-NEXT: v_mov_b32_e32 v5, v1
+; SDAG-NEXT: v_mov_b32_e32 v4, v0
+; SDAG-NEXT: v_mov_b32_e32 v7, v11
+; SDAG-NEXT: .LBB2_6: ; %Flow1
+; SDAG-NEXT: s_or_b64 exec, exec, s[12:13]
+; SDAG-NEXT: .LBB2_7: ; %Flow2
+; SDAG-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11]
+; SDAG-NEXT: ; %bb.8: ; %itofp-sw-bb
+; SDAG-NEXT: v_lshlrev_b64 v[6:7], 1, v[6:7]
+; SDAG-NEXT: v_lshrrev_b32_e32 v0, 31, v5
+; SDAG-NEXT: v_lshlrev_b64 v[4:5], 1, v[4:5]
+; SDAG-NEXT: v_or_b32_e32 v6, v6, v0
+; SDAG-NEXT: ; %bb.9: ; %itofp-sw-epilog
+; SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
+; SDAG-NEXT: v_lshrrev_b32_e32 v0, 2, v4
+; SDAG-NEXT: v_and_or_b32 v0, v0, 1, v4
+; SDAG-NEXT: v_add_co_u32_e32 v4, vcc, 1, v0
+; SDAG-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; SDAG-NEXT: v_addc_co_u32_e32 v6, vcc, 0, v6, vcc
+; SDAG-NEXT: v_lshrrev_b64 v[0:1], 2, v[4:5]
+; SDAG-NEXT: v_lshlrev_b32_e32 v7, 30, v6
+; SDAG-NEXT: v_or_b32_e32 v10, v1, v7
+; SDAG-NEXT: v_and_b32_e32 v1, 0x800000, v5
+; SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
+; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; SDAG-NEXT: s_cbranch_execnz .LBB2_14
+; SDAG-NEXT: .LBB2_10: ; %Flow
+; SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
+; SDAG-NEXT: .LBB2_11: ; %Flow4
+; SDAG-NEXT: s_or_b64 exec, exec, s[8:9]
+; SDAG-NEXT: v_mov_b32_e32 v2, 0x3ff00000
+; SDAG-NEXT: v_and_b32_e32 v1, 0x80000000, v3
+; SDAG-NEXT: v_lshl_add_u32 v2, v8, 20, v2
+; SDAG-NEXT: v_and_b32_e32 v3, 0xfffff, v10
+; SDAG-NEXT: v_or3_b32 v1, v3, v1, v2
+; SDAG-NEXT: .LBB2_12: ; %Flow5
+; SDAG-NEXT: s_or_b64 exec, exec, s[6:7]
+; SDAG-NEXT: s_setpc_b64 s[30:31]
+; SDAG-NEXT: .LBB2_13: ; %itofp-if-else
+; SDAG-NEXT: v_add_u32_e32 v2, 0xffffffb5, v9
+; SDAG-NEXT: v_lshlrev_b64 v[0:1], v2, v[4:5]
+; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v2
+; SDAG-NEXT: v_cndmask_b32_e32 v10, 0, v1, vcc
+; SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
+; SDAG-NEXT: ; implicit-def: $vgpr2
+; SDAG-NEXT: ; implicit-def: $vgpr6_vgpr7
+; SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5
+; SDAG-NEXT: s_branch .LBB2_2
+; SDAG-NEXT: .LBB2_14: ; %itofp-if-then20
+; SDAG-NEXT: v_lshrrev_b64 v[0:1], 3, v[4:5]
+; SDAG-NEXT: v_lshlrev_b32_e32 v4, 29, v6
+; SDAG-NEXT: v_or_b32_e32 v10, v1, v4
+; SDAG-NEXT: v_mov_b32_e32 v8, v2
+; SDAG-NEXT: s_branch .LBB2_10
+;
+; GISEL-LABEL: sitofp_i128_to_f64:
+; GISEL: ; %bb.0: ; %itofp-entry
+; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT: v_mov_b32_e32 v4, v0
+; GISEL-NEXT: v_mov_b32_e32 v5, v1
+; GISEL-NEXT: v_or_b32_e32 v0, v4, v2
+; GISEL-NEXT: v_or_b32_e32 v1, v5, v3
+; GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GISEL-NEXT: v_mov_b32_e32 v0, s4
+; GISEL-NEXT: v_mov_b32_e32 v1, s5
+; GISEL-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GISEL-NEXT: s_cbranch_execz .LBB2_12
+; GISEL-NEXT: ; %bb.1: ; %itofp-if-end
+; GISEL-NEXT: v_ashrrev_i32_e32 v6, 31, v3
+; GISEL-NEXT: v_xor_b32_e32 v0, v6, v4
+; GISEL-NEXT: v_xor_b32_e32 v1, v6, v5
+; GISEL-NEXT: v_xor_b32_e32 v4, v6, v2
+; GISEL-NEXT: v_sub_co_u32_e32 v2, vcc, v0, v6
+; GISEL-NEXT: v_xor_b32_e32 v5, v6, v3
+; GISEL-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v6, vcc
+; GISEL-NEXT: v_subb_co_u32_e32 v4, vcc, v4, v6, vcc
+; GISEL-NEXT: v_ffbh_u32_e32 v1, v2
+; GISEL-NEXT: v_subb_co_u32_e32 v5, vcc, v5, v6, vcc
+; GISEL-NEXT: v_ffbh_u32_e32 v0, v3
+; GISEL-NEXT: v_add_u32_e32 v1, 32, v1
+; GISEL-NEXT: v_ffbh_u32_e32 v7, v4
+; GISEL-NEXT: v_min_u32_e32 v0, v0, v1
+; GISEL-NEXT: v_ffbh_u32_e32 v1, v5
+; GISEL-NEXT: v_add_u32_e32 v7, 32, v7
+; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[4:5]
+; GISEL-NEXT: v_add_u32_e32 v0, 64, v0
+; GISEL-NEXT: v_min_u32_e32 v1, v1, v7
+; GISEL-NEXT: v_cndmask_b32_e32 v9, v1, v0, vcc
+; GISEL-NEXT: v_sub_u32_e32 v7, 0x80, v9
+; GISEL-NEXT: v_cmp_ge_i32_e32 vcc, 53, v7
+; GISEL-NEXT: ; implicit-def: $vgpr10
+; GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
+; GISEL-NEXT: s_cbranch_execnz .LBB2_13
+; GISEL-NEXT: .LBB2_2: ; %Flow3
+; GISEL-NEXT: s_or_saveexec_b64 s[8:9], s[4:5]
+; GISEL-NEXT: v_sub_u32_e32 v8, 0x7f, v9
+; GISEL-NEXT: s_xor_b64 exec, exec, s[8:9]
+; GISEL-NEXT: s_cbranch_execz .LBB2_11
+; GISEL-NEXT: ; %bb.3: ; %NodeBlock
+; GISEL-NEXT: v_cmp_le_i32_e32 vcc, 55, v7
+; GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GISEL-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
+; GISEL-NEXT: s_cbranch_execz .LBB2_7
+; GISEL-NEXT: ; %bb.4: ; %LeafBlock
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 55, v7
+; GISEL-NEXT: s_and_saveexec_b64 s[12:13], vcc
+; GISEL-NEXT: s_cbranch_execz .LBB2_6
+; GISEL-NEXT: ; %bb.5: ; %itofp-sw-default
+; GISEL-NEXT: v_sub_u32_e32 v14, 0x49, v9
+; GISEL-NEXT: v_sub_u32_e32 v10, 64, v14
+; GISEL-NEXT: v_lshrrev_b64 v[0:1], v14, v[2:3]
+; GISEL-NEXT: v_lshlrev_b64 v[10:11], v10, v[4:5]
+; GISEL-NEXT: v_add_u32_e32 v15, 0xffffffc0, v14
+; GISEL-NEXT: v_lshrrev_b64 v[12:13], v14, v[4:5]
+; GISEL-NEXT: v_or_b32_e32 v10, v0, v10
+; GISEL-NEXT: v_or_b32_e32 v11, v1, v11
+; GISEL-NEXT: v_lshrrev_b64 v[0:1], v15, v[4:5]
+; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v14
+; GISEL-NEXT: v_add_u32_e32 v15, 55, v9
+; GISEL-NEXT: v_cndmask_b32_e32 v0, v0, v10, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v1, v1, v11, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v14
+; GISEL-NEXT: v_cndmask_b32_e32 v11, 0, v12, vcc
+; GISEL-NEXT: v_sub_u32_e32 v12, 64, v15
+; GISEL-NEXT: v_cndmask_b32_e64 v14, v0, v2, s[4:5]
+; GISEL-NEXT: v_cndmask_b32_e64 v10, v1, v3, s[4:5]
+; GISEL-NEXT: v_lshrrev_b64 v[0:1], v15, -1
+; GISEL-NEXT: v_lshlrev_b64 v[12:13], v12, -1
+; GISEL-NEXT: v_add_u32_e32 v9, -9, v9
+; GISEL-NEXT: v_or_b32_e32 v16, v0, v12
+; GISEL-NEXT: v_or_b32_e32 v17, v1, v13
+; GISEL-NEXT: v_lshrrev_b64 v[12:13], v9, -1
+; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v15
+; GISEL-NEXT: v_cndmask_b32_e32 v9, v12, v16, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v12, v13, v17, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v15
+; GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
+; GISEL-NEXT: v_cndmask_b32_e64 v9, v9, -1, s[4:5]
+; GISEL-NEXT: v_cndmask_b32_e64 v12, v12, -1, s[4:5]
+; GISEL-NEXT: v_and_b32_e32 v0, v0, v4
+; GISEL-NEXT: v_and_b32_e32 v1, v1, v5
+; GISEL-NEXT: v_and_or_b32 v0, v9, v2, v0
+; GISEL-NEXT: v_and_or_b32 v1, v12, v3, v1
+; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GISEL-NEXT: v_or_b32_e32 v9, v14, v0
+; GISEL-NEXT: v_mov_b32_e32 v2, v9
+; GISEL-NEXT: v_mov_b32_e32 v3, v10
+; GISEL-NEXT: v_mov_b32_e32 v4, v11
+; GISEL-NEXT: v_mov_b32_e32 v5, v12
+; GISEL-NEXT: .LBB2_6: ; %Flow1
+; GISEL-NEXT: s_or_b64 exec, exec, s[12:13]
+; GISEL-NEXT: .LBB2_7: ; %Flow2
+; GISEL-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11]
+; GISEL-NEXT: s_cbranch_execz .LBB2_9
+; GISEL-NEXT: ; %bb.8: ; %itofp-sw-bb
+; GISEL-NEXT: v_lshlrev_b64 v[4:5], 1, v[4:5]
+; GISEL-NEXT: v_lshlrev_b64 v[0:1], 1, v[2:3]
+; GISEL-NEXT: v_lshrrev_b32_e32 v2, 31, v3
+; GISEL-NEXT: v_or_b32_e32 v2, v4, v2
+; GISEL-NEXT: v_mov_b32_e32 v5, v3
+; GISEL-NEXT: v_mov_b32_e32 v4, v2
+; GISEL-NEXT: v_mov_b32_e32 v3, v1
+; GISEL-NEXT: v_mov_b32_e32 v2, v0
+; GISEL-NEXT: .LBB2_9: ; %itofp-sw-epilog
+; GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GISEL-NEXT: v_bfe_u32 v0, v2, 2, 1
+; GISEL-NEXT: v_or_b32_e32 v0, v2, v0
+; GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 1, v0
+; GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GISEL-NEXT: v_mov_b32_e32 v10, 0x800000
+; GISEL-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v4, vcc
+; GISEL-NEXT: v_lshrrev_b64 v[0:1], 2, v[2:3]
+; GISEL-NEXT: v_mov_b32_e32 v9, 0
+; GISEL-NEXT: v_and_b32_e32 v10, 0x800000, v3
+; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[9:10]
+; GISEL-NEXT: v_lshl_or_b32 v10, v4, 30, v1
+; GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GISEL-NEXT: s_cbranch_execnz .LBB2_14
+; GISEL-NEXT: .LBB2_10: ; %Flow
+; GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GISEL-NEXT: .LBB2_11: ; %Flow4
+; GISEL-NEXT: s_or_b64 exec, exec, s[8:9]
+; GISEL-NEXT: v_mov_b32_e32 v2, 0x3ff00000
+; GISEL-NEXT: v_and_b32_e32 v1, 0x80000000, v6
+; GISEL-NEXT: v_lshl_add_u32 v2, v8, 20, v2
+; GISEL-NEXT: v_and_b32_e32 v3, 0xfffff, v10
+; GISEL-NEXT: v_or3_b32 v1, v3, v1, v2
+; GISEL-NEXT: .LBB2_12: ; %Flow5
+; GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
+; GISEL-NEXT: s_setpc_b64 s[30:31]
+; GISEL-NEXT: .LBB2_13: ; %itofp-if-else
+; GISEL-NEXT: v_add_u32_e32 v4, 0xffffffb5, v9
+; GISEL-NEXT: v_lshlrev_b64 v[0:1], v4, v[2:3]
+; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v4
+; GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v10, 0, v1, vcc
+; GISEL-NEXT: ; implicit-def: $vgpr7
+; GISEL-NEXT: ; implicit-def: $vgpr2
+; GISEL-NEXT: s_branch .LBB2_2
+; GISEL-NEXT: .LBB2_14: ; %itofp-if-then20
+; GISEL-NEXT: v_lshrrev_b64 v[0:1], 3, v[2:3]
+; GISEL-NEXT: v_mov_b32_e32 v8, v7
+; GISEL-NEXT: v_lshl_or_b32 v10, v4, 29, v1
+; GISEL-NEXT: s_branch .LBB2_10
%cvt = sitofp i128 %x to double
ret double %cvt
}
define double @uitofp_i128_to_f64(i128 %x) {
-; GCN-LABEL: uitofp_i128_to_f64:
-; GCN: ; %bb.0: ; %itofp-entry
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_or_b32_e32 v5, v1, v3
-; GCN-NEXT: v_or_b32_e32 v4, v0, v2
-; GCN-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GCN-NEXT: v_mov_b32_e32 v4, 0
-; GCN-NEXT: v_mov_b32_e32 v5, 0
-; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; GCN-NEXT: s_cbranch_execz .LBB3_12
-; GCN-NEXT: ; %bb.1: ; %itofp-if-end
-; GCN-NEXT: v_ffbh_u32_e32 v4, v2
-; GCN-NEXT: v_add_u32_e32 v4, 32, v4
-; GCN-NEXT: v_ffbh_u32_e32 v5, v3
-; GCN-NEXT: v_min_u32_e32 v4, v4, v5
-; GCN-NEXT: v_ffbh_u32_e32 v5, v0
-; GCN-NEXT: v_add_u32_e32 v5, 32, v5
-; GCN-NEXT: v_ffbh_u32_e32 v6, v1
-; GCN-NEXT: v_min_u32_e32 v5, v5, v6
-; GCN-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
-; GCN-NEXT: v_add_u32_e32 v5, 64, v5
-; GCN-NEXT: v_cndmask_b32_e32 v8, v5, v4, vcc
-; GCN-NEXT: v_sub_u32_e32 v6, 0x80, v8
-; GCN-NEXT: v_cmp_gt_i32_e32 vcc, 54, v6
-; GCN-NEXT: ; implicit-def: $vgpr9
-; GCN-NEXT: ; implicit-def: $vgpr4_vgpr5
-; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GCN-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; GCN-NEXT: s_cbranch_execnz .LBB3_13
-; GCN-NEXT: .LBB3_2: ; %Flow3
-; GCN-NEXT: s_or_saveexec_b64 s[8:9], s[4:5]
-; GCN-NEXT: v_sub_u32_e32 v7, 0x7f, v8
-; GCN-NEXT: s_xor_b64 exec, exec, s[8:9]
-; GCN-NEXT: s_cbranch_execz .LBB3_11
-; GCN-NEXT: ; %bb.3: ; %NodeBlock
-; GCN-NEXT: v_cmp_lt_i32_e32 vcc, 54, v6
-; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GCN-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
-; GCN-NEXT: s_cbranch_execz .LBB3_7
-; GCN-NEXT: ; %bb.4: ; %LeafBlock
-; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 55, v6
-; GCN-NEXT: s_and_saveexec_b64 s[12:13], vcc
-; GCN-NEXT: s_cbranch_execz .LBB3_6
-; GCN-NEXT: ; %bb.5: ; %itofp-sw-default
-; GCN-NEXT: v_sub_u32_e32 v11, 0x49, v8
-; GCN-NEXT: v_sub_u32_e32 v9, 64, v11
-; GCN-NEXT: v_lshrrev_b64 v[4:5], v11, v[0:1]
-; GCN-NEXT: v_lshlrev_b64 v[9:10], v9, v[2:3]
-; GCN-NEXT: v_sub_u32_e32 v12, 9, v8
-; GCN-NEXT: v_or_b32_e32 v10, v5, v10
-; GCN-NEXT: v_or_b32_e32 v9, v4, v9
-; GCN-NEXT: v_lshrrev_b64 v[4:5], v12, v[2:3]
-; GCN-NEXT: v_cmp_gt_u32_e32 vcc, 64, v11
-; GCN-NEXT: v_add_u32_e32 v15, 55, v8
-; GCN-NEXT: v_cndmask_b32_e32 v5, v5, v10, vcc
-; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v11
-; GCN-NEXT: v_cndmask_b32_e32 v4, v4, v9, vcc
-; GCN-NEXT: v_lshrrev_b64 v[9:10], v11, v[2:3]
-; GCN-NEXT: v_lshrrev_b64 v[11:12], v12, v[0:1]
-; GCN-NEXT: v_lshlrev_b64 v[13:14], v15, v[2:3]
-; GCN-NEXT: v_add_u32_e32 v8, -9, v8
-; GCN-NEXT: v_or_b32_e32 v14, v14, v12
-; GCN-NEXT: v_or_b32_e32 v13, v13, v11
-; GCN-NEXT: v_lshlrev_b64 v[11:12], v8, v[0:1]
-; GCN-NEXT: v_cndmask_b32_e32 v10, 0, v10, vcc
-; GCN-NEXT: v_cndmask_b32_e32 v9, 0, v9, vcc
-; GCN-NEXT: v_cmp_gt_u32_e32 vcc, 64, v15
-; GCN-NEXT: v_cndmask_b32_e64 v5, v5, v1, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e64 v4, v4, v0, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e32 v8, v12, v14, vcc
-; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v15
-; GCN-NEXT: v_lshlrev_b64 v[0:1], v15, v[0:1]
-; GCN-NEXT: v_cndmask_b32_e64 v3, v8, v3, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e32 v8, v11, v13, vcc
-; GCN-NEXT: v_cndmask_b32_e64 v2, v8, v2, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; GCN-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GCN-NEXT: v_or_b32_e32 v1, v1, v3
-; GCN-NEXT: v_or_b32_e32 v0, v0, v2
-; GCN-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
-; GCN-NEXT: v_mov_b32_e32 v2, v9
-; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GCN-NEXT: v_or_b32_e32 v4, v4, v0
-; GCN-NEXT: v_mov_b32_e32 v0, v4
-; GCN-NEXT: v_mov_b32_e32 v1, v5
-; GCN-NEXT: v_mov_b32_e32 v3, v10
-; GCN-NEXT: .LBB3_6: ; %Flow1
-; GCN-NEXT: s_or_b64 exec, exec, s[12:13]
-; GCN-NEXT: .LBB3_7: ; %Flow2
-; GCN-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11]
-; GCN-NEXT: ; %bb.8: ; %itofp-sw-bb
-; GCN-NEXT: v_lshlrev_b64 v[2:3], 1, v[2:3]
-; GCN-NEXT: v_lshrrev_b32_e32 v3, 31, v1
-; GCN-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
-; GCN-NEXT: v_or_b32_e32 v2, v2, v3
-; GCN-NEXT: ; %bb.9: ; %itofp-sw-epilog
-; GCN-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN-NEXT: v_lshrrev_b32_e32 v3, 2, v0
-; GCN-NEXT: v_and_or_b32 v0, v3, 1, v0
-; GCN-NEXT: v_add_co_u32_e32 v0, vcc, 1, v0
-; GCN-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GCN-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v2, vcc
-; GCN-NEXT: v_lshrrev_b64 v[4:5], 2, v[0:1]
-; GCN-NEXT: v_and_b32_e32 v3, 0x800000, v1
-; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 0, v3
-; GCN-NEXT: v_alignbit_b32 v9, v2, v1, 2
-; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GCN-NEXT: s_cbranch_execnz .LBB3_14
-; GCN-NEXT: .LBB3_10: ; %Flow
-; GCN-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN-NEXT: .LBB3_11: ; %Flow4
-; GCN-NEXT: s_or_b64 exec, exec, s[8:9]
-; GCN-NEXT: v_and_b32_e32 v0, 0xfffff, v9
-; GCN-NEXT: v_lshl_or_b32 v0, v7, 20, v0
-; GCN-NEXT: v_add_u32_e32 v5, 0x3ff00000, v0
-; GCN-NEXT: .LBB3_12: ; %Flow5
-; GCN-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN-NEXT: v_mov_b32_e32 v0, v4
-; GCN-NEXT: v_mov_b32_e32 v1, v5
-; GCN-NEXT: s_setpc_b64 s[30:31]
-; GCN-NEXT: .LBB3_13: ; %itofp-if-else
-; GCN-NEXT: v_add_u32_e32 v2, 0xffffffb5, v8
-; GCN-NEXT: v_lshlrev_b64 v[0:1], v2, v[0:1]
-; GCN-NEXT: v_cmp_gt_u32_e32 vcc, 64, v2
-; GCN-NEXT: v_cndmask_b32_e32 v9, 0, v1, vcc
-; GCN-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc
-; GCN-NEXT: ; implicit-def: $vgpr6
-; GCN-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GCN-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GCN-NEXT: s_branch .LBB3_2
-; GCN-NEXT: .LBB3_14: ; %itofp-if-then20
-; GCN-NEXT: v_lshrrev_b64 v[4:5], 3, v[0:1]
-; GCN-NEXT: v_alignbit_b32 v9, v2, v1, 3
-; GCN-NEXT: v_mov_b32_e32 v7, v6
-; GCN-NEXT: s_branch .LBB3_10
+; SDAG-LABEL: uitofp_i128_to_f64:
+; SDAG: ; %bb.0: ; %itofp-entry
+; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-NEXT: v_or_b32_e32 v5, v1, v3
+; SDAG-NEXT: v_or_b32_e32 v4, v0, v2
+; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SDAG-NEXT: v_mov_b32_e32 v5, 0
+; SDAG-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; SDAG-NEXT: s_cbranch_execz .LBB3_12
+; SDAG-NEXT: ; %bb.1: ; %itofp-if-end
+; SDAG-NEXT: v_ffbh_u32_e32 v4, v2
+; SDAG-NEXT: v_add_u32_e32 v4, 32, v4
+; SDAG-NEXT: v_ffbh_u32_e32 v5, v3
+; SDAG-NEXT: v_min_u32_e32 v4, v4, v5
+; SDAG-NEXT: v_ffbh_u32_e32 v5, v0
+; SDAG-NEXT: v_add_u32_e32 v5, 32, v5
+; SDAG-NEXT: v_ffbh_u32_e32 v6, v1
+; SDAG-NEXT: v_min_u32_e32 v5, v5, v6
+; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; SDAG-NEXT: v_add_u32_e32 v5, 64, v5
+; SDAG-NEXT: v_cndmask_b32_e32 v8, v5, v4, vcc
+; SDAG-NEXT: v_sub_u32_e32 v6, 0x80, v8
+; SDAG-NEXT: v_cmp_gt_i32_e32 vcc, 54, v6
+; SDAG-NEXT: ; implicit-def: $vgpr9
+; SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5
+; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; SDAG-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
+; SDAG-NEXT: s_cbranch_execnz .LBB3_13
+; SDAG-NEXT: .LBB3_2: ; %Flow3
+; SDAG-NEXT: s_or_saveexec_b64 s[8:9], s[4:5]
+; SDAG-NEXT: v_sub_u32_e32 v7, 0x7f, v8
+; SDAG-NEXT: s_xor_b64 exec, exec, s[8:9]
+; SDAG-NEXT: s_cbranch_execz .LBB3_11
+; SDAG-NEXT: ; %bb.3: ; %NodeBlock
+; SDAG-NEXT: v_cmp_lt_i32_e32 vcc, 54, v6
+; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; SDAG-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
+; SDAG-NEXT: s_cbranch_execz .LBB3_7
+; SDAG-NEXT: ; %bb.4: ; %LeafBlock
+; SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 55, v6
+; SDAG-NEXT: s_and_saveexec_b64 s[12:13], vcc
+; SDAG-NEXT: s_cbranch_execz .LBB3_6
+; SDAG-NEXT: ; %bb.5: ; %itofp-sw-default
+; SDAG-NEXT: v_sub_u32_e32 v11, 0x49, v8
+; SDAG-NEXT: v_sub_u32_e32 v9, 64, v11
+; SDAG-NEXT: v_lshrrev_b64 v[4:5], v11, v[0:1]
+; SDAG-NEXT: v_lshlrev_b64 v[9:10], v9, v[2:3]
+; SDAG-NEXT: v_sub_u32_e32 v12, 9, v8
+; SDAG-NEXT: v_or_b32_e32 v10, v5, v10
+; SDAG-NEXT: v_or_b32_e32 v9, v4, v9
+; SDAG-NEXT: v_lshrrev_b64 v[4:5], v12, v[2:3]
+; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v11
+; SDAG-NEXT: v_add_u32_e32 v15, 55, v8
+; SDAG-NEXT: v_cndmask_b32_e32 v5, v5, v10, vcc
+; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v11
+; SDAG-NEXT: v_cndmask_b32_e32 v4, v4, v9, vcc
+; SDAG-NEXT: v_lshrrev_b64 v[9:10], v11, v[2:3]
+; SDAG-NEXT: v_lshrrev_b64 v[11:12], v12, v[0:1]
+; SDAG-NEXT: v_lshlrev_b64 v[13:14], v15, v[2:3]
+; SDAG-NEXT: v_add_u32_e32 v8, -9, v8
+; SDAG-NEXT: v_or_b32_e32 v14, v14, v12
+; SDAG-NEXT: v_or_b32_e32 v13, v13, v11
+; SDAG-NEXT: v_lshlrev_b64 v[11:12], v8, v[0:1]
+; SDAG-NEXT: v_cndmask_b32_e32 v10, 0, v10, vcc
+; SDAG-NEXT: v_cndmask_b32_e32 v9, 0, v9, vcc
+; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v15
+; SDAG-NEXT: v_cndmask_b32_e64 v5, v5, v1, s[4:5]
+; SDAG-NEXT: v_cndmask_b32_e64 v4, v4, v0, s[4:5]
+; SDAG-NEXT: v_cndmask_b32_e32 v8, v12, v14, vcc
+; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v15
+; SDAG-NEXT: v_lshlrev_b64 v[0:1], v15, v[0:1]
+; SDAG-NEXT: v_cndmask_b32_e64 v3, v8, v3, s[4:5]
+; SDAG-NEXT: v_cndmask_b32_e32 v8, v11, v13, vcc
+; SDAG-NEXT: v_cndmask_b32_e64 v2, v8, v2, s[4:5]
+; SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
+; SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
+; SDAG-NEXT: v_or_b32_e32 v1, v1, v3
+; SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; SDAG-NEXT: v_mov_b32_e32 v2, v9
+; SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; SDAG-NEXT: v_or_b32_e32 v4, v4, v0
+; SDAG-NEXT: v_mov_b32_e32 v0, v4
+; SDAG-NEXT: v_mov_b32_e32 v1, v5
+; SDAG-NEXT: v_mov_b32_e32 v3, v10
+; SDAG-NEXT: .LBB3_6: ; %Flow1
+; SDAG-NEXT: s_or_b64 exec, exec, s[12:13]
+; SDAG-NEXT: .LBB3_7: ; %Flow2
+; SDAG-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11]
+; SDAG-NEXT: ; %bb.8: ; %itofp-sw-bb
+; SDAG-NEXT: v_lshlrev_b64 v[2:3], 1, v[2:3]
+; SDAG-NEXT: v_lshrrev_b32_e32 v3, 31, v1
+; SDAG-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
+; SDAG-NEXT: v_or_b32_e32 v2, v2, v3
+; SDAG-NEXT: ; %bb.9: ; %itofp-sw-epilog
+; SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
+; SDAG-NEXT: v_lshrrev_b32_e32 v3, 2, v0
+; SDAG-NEXT: v_and_or_b32 v0, v3, 1, v0
+; SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 1, v0
+; SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; SDAG-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v2, vcc
+; SDAG-NEXT: v_lshrrev_b64 v[4:5], 2, v[0:1]
+; SDAG-NEXT: v_and_b32_e32 v3, 0x800000, v1
+; SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 0, v3
+; SDAG-NEXT: v_alignbit_b32 v9, v2, v1, 2
+; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; SDAG-NEXT: s_cbranch_execnz .LBB3_14
+; SDAG-NEXT: .LBB3_10: ; %Flow
+; SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
+; SDAG-NEXT: .LBB3_11: ; %Flow4
+; SDAG-NEXT: s_or_b64 exec, exec, s[8:9]
+; SDAG-NEXT: v_and_b32_e32 v0, 0xfffff, v9
+; SDAG-NEXT: v_lshl_or_b32 v0, v7, 20, v0
+; SDAG-NEXT: v_add_u32_e32 v5, 0x3ff00000, v0
+; SDAG-NEXT: .LBB3_12: ; %Flow5
+; SDAG-NEXT: s_or_b64 exec, exec, s[6:7]
+; SDAG-NEXT: v_mov_b32_e32 v0, v4
+; SDAG-NEXT: v_mov_b32_e32 v1, v5
+; SDAG-NEXT: s_setpc_b64 s[30:31]
+; SDAG-NEXT: .LBB3_13: ; %itofp-if-else
+; SDAG-NEXT: v_add_u32_e32 v2, 0xffffffb5, v8
+; SDAG-NEXT: v_lshlrev_b64 v[0:1], v2, v[0:1]
+; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v2
+; SDAG-NEXT: v_cndmask_b32_e32 v9, 0, v1, vcc
+; SDAG-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc
+; SDAG-NEXT: ; implicit-def: $vgpr6
+; SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
+; SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
+; SDAG-NEXT: s_branch .LBB3_2
+; SDAG-NEXT: .LBB3_14: ; %itofp-if-then20
+; SDAG-NEXT: v_lshrrev_b64 v[4:5], 3, v[0:1]
+; SDAG-NEXT: v_alignbit_b32 v9, v2, v1, 3
+; SDAG-NEXT: v_mov_b32_e32 v7, v6
+; SDAG-NEXT: s_branch .LBB3_10
+;
+; GISEL-LABEL: uitofp_i128_to_f64:
+; GISEL: ; %bb.0: ; %itofp-entry
+; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT: v_or_b32_e32 v4, v0, v2
+; GISEL-NEXT: v_or_b32_e32 v5, v1, v3
+; GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; GISEL-NEXT: v_mov_b32_e32 v4, s4
+; GISEL-NEXT: v_mov_b32_e32 v5, s5
+; GISEL-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GISEL-NEXT: s_cbranch_execz .LBB3_12
+; GISEL-NEXT: ; %bb.1: ; %itofp-if-end
+; GISEL-NEXT: v_ffbh_u32_e32 v5, v0
+; GISEL-NEXT: v_ffbh_u32_e32 v4, v1
+; GISEL-NEXT: v_add_u32_e32 v5, 32, v5
+; GISEL-NEXT: v_ffbh_u32_e32 v6, v2
+; GISEL-NEXT: v_min_u32_e32 v4, v4, v5
+; GISEL-NEXT: v_ffbh_u32_e32 v5, v3
+; GISEL-NEXT: v_add_u32_e32 v6, 32, v6
+; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3]
+; GISEL-NEXT: v_add_u32_e32 v4, 64, v4
+; GISEL-NEXT: v_min_u32_e32 v5, v5, v6
+; GISEL-NEXT: v_cndmask_b32_e32 v8, v5, v4, vcc
+; GISEL-NEXT: v_sub_u32_e32 v6, 0x80, v8
+; GISEL-NEXT: v_cmp_ge_i32_e32 vcc, 53, v6
+; GISEL-NEXT: ; implicit-def: $vgpr9
+; GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
+; GISEL-NEXT: s_cbranch_execnz .LBB3_13
+; GISEL-NEXT: .LBB3_2: ; %Flow3
+; GISEL-NEXT: s_or_saveexec_b64 s[8:9], s[4:5]
+; GISEL-NEXT: v_sub_u32_e32 v7, 0x7f, v8
+; GISEL-NEXT: s_xor_b64 exec, exec, s[8:9]
+; GISEL-NEXT: s_cbranch_execz .LBB3_11
+; GISEL-NEXT: ; %bb.3: ; %NodeBlock
+; GISEL-NEXT: v_cmp_le_i32_e32 vcc, 55, v6
+; GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GISEL-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
+; GISEL-NEXT: s_cbranch_execz .LBB3_7
+; GISEL-NEXT: ; %bb.4: ; %LeafBlock
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 55, v6
+; GISEL-NEXT: s_and_saveexec_b64 s[12:13], vcc
+; GISEL-NEXT: s_cbranch_execz .LBB3_6
+; GISEL-NEXT: ; %bb.5: ; %itofp-sw-default
+; GISEL-NEXT: v_sub_u32_e32 v13, 0x49, v8
+; GISEL-NEXT: v_sub_u32_e32 v9, 64, v13
+; GISEL-NEXT: v_lshrrev_b64 v[4:5], v13, v[0:1]
+; GISEL-NEXT: v_lshlrev_b64 v[9:10], v9, v[2:3]
+; GISEL-NEXT: v_add_u32_e32 v14, 0xffffffc0, v13
+; GISEL-NEXT: v_lshrrev_b64 v[11:12], v13, v[2:3]
+; GISEL-NEXT: v_or_b32_e32 v9, v4, v9
+; GISEL-NEXT: v_or_b32_e32 v10, v5, v10
+; GISEL-NEXT: v_lshrrev_b64 v[4:5], v14, v[2:3]
+; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v13
+; GISEL-NEXT: v_add_u32_e32 v15, 55, v8
+; GISEL-NEXT: v_cndmask_b32_e32 v4, v4, v9, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v5, v5, v10, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v13
+; GISEL-NEXT: v_cndmask_b32_e32 v10, 0, v11, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v11, 0, v12, vcc
+; GISEL-NEXT: v_sub_u32_e32 v12, 64, v15
+; GISEL-NEXT: v_cndmask_b32_e64 v14, v4, v0, s[4:5]
+; GISEL-NEXT: v_cndmask_b32_e64 v9, v5, v1, s[4:5]
+; GISEL-NEXT: v_lshrrev_b64 v[4:5], v15, -1
+; GISEL-NEXT: v_lshlrev_b64 v[12:13], v12, -1
+; GISEL-NEXT: v_add_u32_e32 v8, -9, v8
+; GISEL-NEXT: v_or_b32_e32 v16, v4, v12
+; GISEL-NEXT: v_or_b32_e32 v17, v5, v13
+; GISEL-NEXT: v_lshrrev_b64 v[12:13], v8, -1
+; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v15
+; GISEL-NEXT: v_cndmask_b32_e32 v8, v12, v16, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v12, v13, v17, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v15
+; GISEL-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc
+; GISEL-NEXT: v_cndmask_b32_e64 v8, v8, -1, s[4:5]
+; GISEL-NEXT: v_cndmask_b32_e64 v12, v12, -1, s[4:5]
+; GISEL-NEXT: v_and_b32_e32 v2, v4, v2
+; GISEL-NEXT: v_and_b32_e32 v3, v5, v3
+; GISEL-NEXT: v_and_or_b32 v0, v8, v0, v2
+; GISEL-NEXT: v_and_or_b32 v1, v12, v1, v3
+; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GISEL-NEXT: v_or_b32_e32 v8, v14, v0
+; GISEL-NEXT: v_mov_b32_e32 v0, v8
+; GISEL-NEXT: v_mov_b32_e32 v1, v9
+; GISEL-NEXT: v_mov_b32_e32 v2, v10
+; GISEL-NEXT: v_mov_b32_e32 v3, v11
+; GISEL-NEXT: .LBB3_6: ; %Flow1
+; GISEL-NEXT: s_or_b64 exec, exec, s[12:13]
+; GISEL-NEXT: .LBB3_7: ; %Flow2
+; GISEL-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11]
+; GISEL-NEXT: s_cbranch_execz .LBB3_9
+; GISEL-NEXT: ; %bb.8: ; %itofp-sw-bb
+; GISEL-NEXT: v_lshlrev_b64 v[10:11], 1, v[2:3]
+; GISEL-NEXT: v_lshlrev_b64 v[8:9], 1, v[0:1]
+; GISEL-NEXT: v_lshrrev_b32_e32 v0, 31, v1
+; GISEL-NEXT: v_or_b32_e32 v10, v10, v0
+; GISEL-NEXT: v_mov_b32_e32 v0, v8
+; GISEL-NEXT: v_mov_b32_e32 v1, v9
+; GISEL-NEXT: v_mov_b32_e32 v2, v10
+; GISEL-NEXT: v_mov_b32_e32 v3, v11
+; GISEL-NEXT: .LBB3_9: ; %itofp-sw-epilog
+; GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GISEL-NEXT: v_bfe_u32 v4, v0, 2, 1
+; GISEL-NEXT: v_or_b32_e32 v0, v0, v4
+; GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 1, v0
+; GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GISEL-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v2, vcc
+; GISEL-NEXT: v_mov_b32_e32 v9, 0x800000
+; GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GISEL-NEXT: v_and_b32_e32 v9, 0x800000, v1
+; GISEL-NEXT: v_lshrrev_b64 v[4:5], 2, v[0:1]
+; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
+; GISEL-NEXT: v_lshlrev_b64 v[8:9], 30, v[2:3]
+; GISEL-NEXT: v_lshrrev_b32_e32 v5, 2, v1
+; GISEL-NEXT: v_or_b32_e32 v9, v8, v5
+; GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GISEL-NEXT: s_cbranch_execnz .LBB3_14
+; GISEL-NEXT: .LBB3_10: ; %Flow
+; GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GISEL-NEXT: .LBB3_11: ; %Flow4
+; GISEL-NEXT: s_or_b64 exec, exec, s[8:9]
+; GISEL-NEXT: v_mov_b32_e32 v0, 0x3ff00000
+; GISEL-NEXT: v_lshl_add_u32 v0, v7, 20, v0
+; GISEL-NEXT: v_mov_b32_e32 v1, 0xfffff
+; GISEL-NEXT: v_and_or_b32 v5, v9, v1, v0
+; GISEL-NEXT: .LBB3_12: ; %Flow5
+; GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
+; GISEL-NEXT: v_mov_b32_e32 v0, v4
+; GISEL-NEXT: v_mov_b32_e32 v1, v5
+; GISEL-NEXT: s_setpc_b64 s[30:31]
+; GISEL-NEXT: .LBB3_13: ; %itofp-if-else
+; GISEL-NEXT: v_add_u32_e32 v2, 0xffffffb5, v8
+; GISEL-NEXT: v_lshlrev_b64 v[0:1], v2, v[0:1]
+; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v2
+; GISEL-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v9, 0, v1, vcc
+; GISEL-NEXT: ; implicit-def: $vgpr6
+; GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GISEL-NEXT: s_branch .LBB3_2
+; GISEL-NEXT: .LBB3_14: ; %itofp-if-then20
+; GISEL-NEXT: v_lshlrev_b64 v[2:3], 29, v[2:3]
+; GISEL-NEXT: v_lshrrev_b64 v[4:5], 3, v[0:1]
+; GISEL-NEXT: v_lshrrev_b32_e32 v0, 3, v1
+; GISEL-NEXT: v_or_b32_e32 v9, v2, v0
+; GISEL-NEXT: v_mov_b32_e32 v7, v6
+; GISEL-NEXT: s_branch .LBB3_10
%cvt = uitofp i128 %x to double
ret double %cvt
}
define half @sitofp_i128_to_f16(i128 %x) {
-; GCN-LABEL: sitofp_i128_to_f16:
-; GCN: ; %bb.0: ; %itofp-entry
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_or_b32_e32 v5, v1, v3
-; GCN-NEXT: v_or_b32_e32 v4, v0, v2
-; GCN-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GCN-NEXT: v_mov_b32_e32 v4, 0
-; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; GCN-NEXT: s_cbranch_execz .LBB4_12
-; GCN-NEXT: ; %bb.1: ; %itofp-if-end
-; GCN-NEXT: v_ashrrev_i32_e32 v5, 31, v3
-; GCN-NEXT: v_xor_b32_e32 v0, v0, v5
-; GCN-NEXT: v_xor_b32_e32 v1, v1, v5
-; GCN-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v5
-; GCN-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v5, vcc
-; GCN-NEXT: v_xor_b32_e32 v2, v2, v5
-; GCN-NEXT: v_xor_b32_e32 v6, v3, v5
-; GCN-NEXT: v_subb_co_u32_e32 v4, vcc, v2, v5, vcc
-; GCN-NEXT: v_subb_co_u32_e32 v5, vcc, v6, v5, vcc
-; GCN-NEXT: v_ffbh_u32_e32 v2, v4
-; GCN-NEXT: v_add_u32_e32 v2, 32, v2
-; GCN-NEXT: v_ffbh_u32_e32 v6, v5
-; GCN-NEXT: v_min_u32_e32 v2, v2, v6
-; GCN-NEXT: v_ffbh_u32_e32 v6, v0
-; GCN-NEXT: v_add_u32_e32 v6, 32, v6
-; GCN-NEXT: v_ffbh_u32_e32 v7, v1
-; GCN-NEXT: v_min_u32_e32 v6, v6, v7
-; GCN-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GCN-NEXT: v_add_u32_e32 v6, 64, v6
-; GCN-NEXT: v_cndmask_b32_e32 v7, v6, v2, vcc
-; GCN-NEXT: v_sub_u32_e32 v2, 0x80, v7
-; GCN-NEXT: v_cmp_gt_i32_e32 vcc, 25, v2
-; GCN-NEXT: ; implicit-def: $vgpr8
-; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GCN-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; GCN-NEXT: s_cbranch_execnz .LBB4_13
-; GCN-NEXT: .LBB4_2: ; %Flow3
-; GCN-NEXT: s_or_saveexec_b64 s[8:9], s[4:5]
-; GCN-NEXT: v_sub_u32_e32 v6, 0x7f, v7
-; GCN-NEXT: s_xor_b64 exec, exec, s[8:9]
-; GCN-NEXT: s_cbranch_execz .LBB4_11
-; GCN-NEXT: ; %bb.3: ; %NodeBlock
-; GCN-NEXT: v_cmp_lt_i32_e32 vcc, 25, v2
-; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GCN-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
-; GCN-NEXT: s_cbranch_execz .LBB4_7
-; GCN-NEXT: ; %bb.4: ; %LeafBlock
-; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 26, v2
-; GCN-NEXT: s_and_saveexec_b64 s[12:13], vcc
-; GCN-NEXT: s_cbranch_execz .LBB4_6
-; GCN-NEXT: ; %bb.5: ; %itofp-sw-default
-; GCN-NEXT: v_sub_u32_e32 v12, 0x66, v7
-; GCN-NEXT: v_sub_u32_e32 v10, 64, v12
-; GCN-NEXT: v_lshrrev_b64 v[8:9], v12, v[0:1]
-; GCN-NEXT: v_lshlrev_b64 v[10:11], v10, v[4:5]
-; GCN-NEXT: v_sub_u32_e32 v13, 38, v7
-; GCN-NEXT: v_or_b32_e32 v11, v9, v11
-; GCN-NEXT: v_or_b32_e32 v10, v8, v10
-; GCN-NEXT: v_lshrrev_b64 v[8:9], v13, v[4:5]
-; GCN-NEXT: v_cmp_gt_u32_e32 vcc, 64, v12
-; GCN-NEXT: v_add_u32_e32 v14, 26, v7
-; GCN-NEXT: v_cndmask_b32_e32 v9, v9, v11, vcc
-; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v12
-; GCN-NEXT: v_cndmask_b32_e32 v8, v8, v10, vcc
-; GCN-NEXT: v_lshrrev_b64 v[10:11], v13, v[0:1]
-; GCN-NEXT: v_lshlrev_b64 v[12:13], v14, v[4:5]
-; GCN-NEXT: v_subrev_u32_e32 v7, 38, v7
-; GCN-NEXT: v_cndmask_b32_e64 v15, v8, v0, s[4:5]
-; GCN-NEXT: v_lshlrev_b64 v[7:8], v7, v[0:1]
-; GCN-NEXT: v_cndmask_b32_e64 v9, v9, v1, s[4:5]
-; GCN-NEXT: v_or_b32_e32 v11, v13, v11
-; GCN-NEXT: v_or_b32_e32 v10, v12, v10
-; GCN-NEXT: v_cmp_gt_u32_e32 vcc, 64, v14
-; GCN-NEXT: v_lshlrev_b64 v[0:1], v14, v[0:1]
-; GCN-NEXT: v_cndmask_b32_e32 v8, v8, v11, vcc
-; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v14
-; GCN-NEXT: v_cndmask_b32_e32 v7, v7, v10, vcc
-; GCN-NEXT: v_cndmask_b32_e64 v5, v8, v5, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e64 v4, v7, v4, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; GCN-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GCN-NEXT: v_or_b32_e32 v1, v1, v5
-; GCN-NEXT: v_or_b32_e32 v0, v0, v4
-; GCN-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
-; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GCN-NEXT: v_or_b32_e32 v8, v15, v0
-; GCN-NEXT: v_mov_b32_e32 v0, v8
-; GCN-NEXT: v_mov_b32_e32 v1, v9
-; GCN-NEXT: .LBB4_6: ; %Flow1
-; GCN-NEXT: s_or_b64 exec, exec, s[12:13]
-; GCN-NEXT: .LBB4_7: ; %Flow2
-; GCN-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11]
-; GCN-NEXT: ; %bb.8: ; %itofp-sw-bb
-; GCN-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
-; GCN-NEXT: ; %bb.9: ; %itofp-sw-epilog
-; GCN-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN-NEXT: v_lshrrev_b32_e32 v4, 2, v0
-; GCN-NEXT: v_and_or_b32 v0, v4, 1, v0
-; GCN-NEXT: v_add_co_u32_e32 v0, vcc, 1, v0
-; GCN-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GCN-NEXT: v_and_b32_e32 v4, 0x4000000, v0
-; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
-; GCN-NEXT: v_alignbit_b32 v8, v1, v0, 2
-; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GCN-NEXT: s_cbranch_execnz .LBB4_14
-; GCN-NEXT: .LBB4_10: ; %Flow
-; GCN-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN-NEXT: .LBB4_11: ; %Flow4
-; GCN-NEXT: s_or_b64 exec, exec, s[8:9]
-; GCN-NEXT: v_and_b32_e32 v0, 0x80000000, v3
-; GCN-NEXT: v_lshl_add_u32 v1, v6, 23, 1.0
-; GCN-NEXT: v_and_b32_e32 v2, 0x7fffff, v8
-; GCN-NEXT: v_or3_b32 v0, v2, v0, v1
-; GCN-NEXT: v_cvt_f16_f32_e32 v4, v0
-; GCN-NEXT: .LBB4_12: ; %Flow5
-; GCN-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN-NEXT: v_mov_b32_e32 v0, v4
-; GCN-NEXT: s_setpc_b64 s[30:31]
-; GCN-NEXT: .LBB4_13: ; %itofp-if-else
-; GCN-NEXT: v_add_u32_e32 v2, 0xffffff98, v7
-; GCN-NEXT: v_lshlrev_b64 v[0:1], v2, v[0:1]
-; GCN-NEXT: v_cmp_gt_u32_e32 vcc, 64, v2
-; GCN-NEXT: v_cndmask_b32_e32 v8, 0, v0, vcc
-; GCN-NEXT: ; implicit-def: $vgpr2
-; GCN-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GCN-NEXT: ; implicit-def: $vgpr4_vgpr5
-; GCN-NEXT: s_branch .LBB4_2
-; GCN-NEXT: .LBB4_14: ; %itofp-if-then20
-; GCN-NEXT: v_alignbit_b32 v8, v1, v0, 3
-; GCN-NEXT: v_mov_b32_e32 v6, v2
-; GCN-NEXT: s_branch .LBB4_10
+; SDAG-LABEL: sitofp_i128_to_f16:
+; SDAG: ; %bb.0: ; %itofp-entry
+; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-NEXT: v_or_b32_e32 v5, v1, v3
+; SDAG-NEXT: v_or_b32_e32 v4, v0, v2
+; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SDAG-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; SDAG-NEXT: s_cbranch_execz .LBB4_12
+; SDAG-NEXT: ; %bb.1: ; %itofp-if-end
+; SDAG-NEXT: v_ashrrev_i32_e32 v5, 31, v3
+; SDAG-NEXT: v_xor_b32_e32 v0, v0, v5
+; SDAG-NEXT: v_xor_b32_e32 v1, v1, v5
+; SDAG-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v5
+; SDAG-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v5, vcc
+; SDAG-NEXT: v_xor_b32_e32 v2, v2, v5
+; SDAG-NEXT: v_xor_b32_e32 v6, v3, v5
+; SDAG-NEXT: v_subb_co_u32_e32 v4, vcc, v2, v5, vcc
+; SDAG-NEXT: v_subb_co_u32_e32 v5, vcc, v6, v5, vcc
+; SDAG-NEXT: v_ffbh_u32_e32 v2, v4
+; SDAG-NEXT: v_add_u32_e32 v2, 32, v2
+; SDAG-NEXT: v_ffbh_u32_e32 v6, v5
+; SDAG-NEXT: v_min_u32_e32 v2, v2, v6
+; SDAG-NEXT: v_ffbh_u32_e32 v6, v0
+; SDAG-NEXT: v_add_u32_e32 v6, 32, v6
+; SDAG-NEXT: v_ffbh_u32_e32 v7, v1
+; SDAG-NEXT: v_min_u32_e32 v6, v6, v7
+; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; SDAG-NEXT: v_add_u32_e32 v6, 64, v6
+; SDAG-NEXT: v_cndmask_b32_e32 v7, v6, v2, vcc
+; SDAG-NEXT: v_sub_u32_e32 v2, 0x80, v7
+; SDAG-NEXT: v_cmp_gt_i32_e32 vcc, 25, v2
+; SDAG-NEXT: ; implicit-def: $vgpr8
+; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; SDAG-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
+; SDAG-NEXT: s_cbranch_execnz .LBB4_13
+; SDAG-NEXT: .LBB4_2: ; %Flow3
+; SDAG-NEXT: s_or_saveexec_b64 s[8:9], s[4:5]
+; SDAG-NEXT: v_sub_u32_e32 v6, 0x7f, v7
+; SDAG-NEXT: s_xor_b64 exec, exec, s[8:9]
+; SDAG-NEXT: s_cbranch_execz .LBB4_11
+; SDAG-NEXT: ; %bb.3: ; %NodeBlock
+; SDAG-NEXT: v_cmp_lt_i32_e32 vcc, 25, v2
+; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; SDAG-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
+; SDAG-NEXT: s_cbranch_execz .LBB4_7
+; SDAG-NEXT: ; %bb.4: ; %LeafBlock
+; SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 26, v2
+; SDAG-NEXT: s_and_saveexec_b64 s[12:13], vcc
+; SDAG-NEXT: s_cbranch_execz .LBB4_6
+; SDAG-NEXT: ; %bb.5: ; %itofp-sw-default
+; SDAG-NEXT: v_sub_u32_e32 v12, 0x66, v7
+; SDAG-NEXT: v_sub_u32_e32 v10, 64, v12
+; SDAG-NEXT: v_lshrrev_b64 v[8:9], v12, v[0:1]
+; SDAG-NEXT: v_lshlrev_b64 v[10:11], v10, v[4:5]
+; SDAG-NEXT: v_sub_u32_e32 v13, 38, v7
+; SDAG-NEXT: v_or_b32_e32 v11, v9, v11
+; SDAG-NEXT: v_or_b32_e32 v10, v8, v10
+; SDAG-NEXT: v_lshrrev_b64 v[8:9], v13, v[4:5]
+; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v12
+; SDAG-NEXT: v_add_u32_e32 v14, 26, v7
+; SDAG-NEXT: v_cndmask_b32_e32 v9, v9, v11, vcc
+; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v12
+; SDAG-NEXT: v_cndmask_b32_e32 v8, v8, v10, vcc
+; SDAG-NEXT: v_lshrrev_b64 v[10:11], v13, v[0:1]
+; SDAG-NEXT: v_lshlrev_b64 v[12:13], v14, v[4:5]
+; SDAG-NEXT: v_subrev_u32_e32 v7, 38, v7
+; SDAG-NEXT: v_cndmask_b32_e64 v15, v8, v0, s[4:5]
+; SDAG-NEXT: v_lshlrev_b64 v[7:8], v7, v[0:1]
+; SDAG-NEXT: v_cndmask_b32_e64 v9, v9, v1, s[4:5]
+; SDAG-NEXT: v_or_b32_e32 v11, v13, v11
+; SDAG-NEXT: v_or_b32_e32 v10, v12, v10
+; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v14
+; SDAG-NEXT: v_lshlrev_b64 v[0:1], v14, v[0:1]
+; SDAG-NEXT: v_cndmask_b32_e32 v8, v8, v11, vcc
+; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v14
+; SDAG-NEXT: v_cndmask_b32_e32 v7, v7, v10, vcc
+; SDAG-NEXT: v_cndmask_b32_e64 v5, v8, v5, s[4:5]
+; SDAG-NEXT: v_cndmask_b32_e64 v4, v7, v4, s[4:5]
+; SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
+; SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
+; SDAG-NEXT: v_or_b32_e32 v1, v1, v5
+; SDAG-NEXT: v_or_b32_e32 v0, v0, v4
+; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; SDAG-NEXT: v_or_b32_e32 v8, v15, v0
+; SDAG-NEXT: v_mov_b32_e32 v0, v8
+; SDAG-NEXT: v_mov_b32_e32 v1, v9
+; SDAG-NEXT: .LBB4_6: ; %Flow1
+; SDAG-NEXT: s_or_b64 exec, exec, s[12:13]
+; SDAG-NEXT: .LBB4_7: ; %Flow2
+; SDAG-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11]
+; SDAG-NEXT: ; %bb.8: ; %itofp-sw-bb
+; SDAG-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
+; SDAG-NEXT: ; %bb.9: ; %itofp-sw-epilog
+; SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
+; SDAG-NEXT: v_lshrrev_b32_e32 v4, 2, v0
+; SDAG-NEXT: v_and_or_b32 v0, v4, 1, v0
+; SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 1, v0
+; SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; SDAG-NEXT: v_and_b32_e32 v4, 0x4000000, v0
+; SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
+; SDAG-NEXT: v_alignbit_b32 v8, v1, v0, 2
+; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; SDAG-NEXT: s_cbranch_execnz .LBB4_14
+; SDAG-NEXT: .LBB4_10: ; %Flow
+; SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
+; SDAG-NEXT: .LBB4_11: ; %Flow4
+; SDAG-NEXT: s_or_b64 exec, exec, s[8:9]
+; SDAG-NEXT: v_and_b32_e32 v0, 0x80000000, v3
+; SDAG-NEXT: v_lshl_add_u32 v1, v6, 23, 1.0
+; SDAG-NEXT: v_and_b32_e32 v2, 0x7fffff, v8
+; SDAG-NEXT: v_or3_b32 v0, v2, v0, v1
+; SDAG-NEXT: v_cvt_f16_f32_e32 v4, v0
+; SDAG-NEXT: .LBB4_12: ; %Flow5
+; SDAG-NEXT: s_or_b64 exec, exec, s[6:7]
+; SDAG-NEXT: v_mov_b32_e32 v0, v4
+; SDAG-NEXT: s_setpc_b64 s[30:31]
+; SDAG-NEXT: .LBB4_13: ; %itofp-if-else
+; SDAG-NEXT: v_add_u32_e32 v2, 0xffffff98, v7
+; SDAG-NEXT: v_lshlrev_b64 v[0:1], v2, v[0:1]
+; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v2
+; SDAG-NEXT: v_cndmask_b32_e32 v8, 0, v0, vcc
+; SDAG-NEXT: ; implicit-def: $vgpr2
+; SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
+; SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5
+; SDAG-NEXT: s_branch .LBB4_2
+; SDAG-NEXT: .LBB4_14: ; %itofp-if-then20
+; SDAG-NEXT: v_alignbit_b32 v8, v1, v0, 3
+; SDAG-NEXT: v_mov_b32_e32 v6, v2
+; SDAG-NEXT: s_branch .LBB4_10
+;
+; GISEL-LABEL: sitofp_i128_to_f16:
+; GISEL: ; %bb.0: ; %itofp-entry
+; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT: v_or_b32_e32 v4, v0, v2
+; GISEL-NEXT: v_or_b32_e32 v5, v1, v3
+; GISEL-NEXT: s_mov_b32 s4, 0
+; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; GISEL-NEXT: v_mov_b32_e32 v4, s4
+; GISEL-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GISEL-NEXT: s_cbranch_execz .LBB4_12
+; GISEL-NEXT: ; %bb.1: ; %itofp-if-end
+; GISEL-NEXT: v_ashrrev_i32_e32 v6, 31, v3
+; GISEL-NEXT: v_xor_b32_e32 v0, v6, v0
+; GISEL-NEXT: v_xor_b32_e32 v1, v6, v1
+; GISEL-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v6
+; GISEL-NEXT: v_xor_b32_e32 v2, v6, v2
+; GISEL-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v6, vcc
+; GISEL-NEXT: v_xor_b32_e32 v3, v6, v3
+; GISEL-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v6, vcc
+; GISEL-NEXT: v_ffbh_u32_e32 v5, v0
+; GISEL-NEXT: v_subb_co_u32_e32 v3, vcc, v3, v6, vcc
+; GISEL-NEXT: v_ffbh_u32_e32 v4, v1
+; GISEL-NEXT: v_add_u32_e32 v5, 32, v5
+; GISEL-NEXT: v_ffbh_u32_e32 v7, v2
+; GISEL-NEXT: v_min_u32_e32 v4, v4, v5
+; GISEL-NEXT: v_ffbh_u32_e32 v5, v3
+; GISEL-NEXT: v_add_u32_e32 v7, 32, v7
+; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3]
+; GISEL-NEXT: v_add_u32_e32 v4, 64, v4
+; GISEL-NEXT: v_min_u32_e32 v5, v5, v7
+; GISEL-NEXT: v_cndmask_b32_e32 v5, v5, v4, vcc
+; GISEL-NEXT: v_sub_u32_e32 v7, 0x80, v5
+; GISEL-NEXT: v_cmp_ge_i32_e32 vcc, 24, v7
+; GISEL-NEXT: ; implicit-def: $vgpr4
+; GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
+; GISEL-NEXT: s_cbranch_execnz .LBB4_13
+; GISEL-NEXT: .LBB4_2: ; %Flow3
+; GISEL-NEXT: s_or_saveexec_b64 s[8:9], s[4:5]
+; GISEL-NEXT: v_sub_u32_e32 v8, 0x7f, v5
+; GISEL-NEXT: s_xor_b64 exec, exec, s[8:9]
+; GISEL-NEXT: s_cbranch_execz .LBB4_11
+; GISEL-NEXT: ; %bb.3: ; %NodeBlock
+; GISEL-NEXT: v_cmp_le_i32_e32 vcc, 26, v7
+; GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GISEL-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
+; GISEL-NEXT: s_cbranch_execz .LBB4_7
+; GISEL-NEXT: ; %bb.4: ; %LeafBlock
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 26, v7
+; GISEL-NEXT: s_and_saveexec_b64 s[12:13], vcc
+; GISEL-NEXT: s_cbranch_execz .LBB4_6
+; GISEL-NEXT: ; %bb.5: ; %itofp-sw-default
+; GISEL-NEXT: v_sub_u32_e32 v4, 0x66, v5
+; GISEL-NEXT: v_sub_u32_e32 v11, 64, v4
+; GISEL-NEXT: v_lshrrev_b64 v[9:10], v4, v[0:1]
+; GISEL-NEXT: v_lshlrev_b64 v[11:12], v11, v[2:3]
+; GISEL-NEXT: v_add_u32_e32 v13, 0xffffffc0, v4
+; GISEL-NEXT: v_or_b32_e32 v11, v9, v11
+; GISEL-NEXT: v_or_b32_e32 v12, v10, v12
+; GISEL-NEXT: v_lshrrev_b64 v[9:10], v13, v[2:3]
+; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v4
+; GISEL-NEXT: v_add_u32_e32 v14, 26, v5
+; GISEL-NEXT: v_cndmask_b32_e32 v9, v9, v11, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v10, v10, v12, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
+; GISEL-NEXT: v_sub_u32_e32 v11, 64, v14
+; GISEL-NEXT: v_cndmask_b32_e32 v13, v9, v0, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v4, v10, v1, vcc
+; GISEL-NEXT: v_lshrrev_b64 v[9:10], v14, -1
+; GISEL-NEXT: v_lshlrev_b64 v[11:12], v11, -1
+; GISEL-NEXT: v_add_u32_e32 v5, 0xffffffda, v5
+; GISEL-NEXT: v_or_b32_e32 v15, v9, v11
+; GISEL-NEXT: v_or_b32_e32 v16, v10, v12
+; GISEL-NEXT: v_lshrrev_b64 v[11:12], v5, -1
+; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v14
+; GISEL-NEXT: v_cndmask_b32_e32 v5, v11, v15, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v11, v12, v16, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v14
+; GISEL-NEXT: v_cndmask_b32_e32 v9, 0, v9, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v10, 0, v10, vcc
+; GISEL-NEXT: v_cndmask_b32_e64 v5, v5, -1, s[4:5]
+; GISEL-NEXT: v_cndmask_b32_e64 v11, v11, -1, s[4:5]
+; GISEL-NEXT: v_and_b32_e32 v2, v9, v2
+; GISEL-NEXT: v_and_b32_e32 v3, v10, v3
+; GISEL-NEXT: v_and_or_b32 v0, v5, v0, v2
+; GISEL-NEXT: v_and_or_b32 v1, v11, v1, v3
+; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GISEL-NEXT: v_or_b32_e32 v3, v13, v0
+; GISEL-NEXT: v_mov_b32_e32 v0, v3
+; GISEL-NEXT: v_mov_b32_e32 v1, v4
+; GISEL-NEXT: v_mov_b32_e32 v2, v5
+; GISEL-NEXT: v_mov_b32_e32 v3, v6
+; GISEL-NEXT: .LBB4_6: ; %Flow1
+; GISEL-NEXT: s_or_b64 exec, exec, s[12:13]
+; GISEL-NEXT: .LBB4_7: ; %Flow2
+; GISEL-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11]
+; GISEL-NEXT: ; %bb.8: ; %itofp-sw-bb
+; GISEL-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
+; GISEL-NEXT: ; %bb.9: ; %itofp-sw-epilog
+; GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GISEL-NEXT: v_bfe_u32 v2, v0, 2, 1
+; GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 1, v0
+; GISEL-NEXT: v_bfrev_b32_e32 v2, 32
+; GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GISEL-NEXT: v_mov_b32_e32 v3, 0
+; GISEL-NEXT: v_and_b32_e32 v2, 0x4000000, v0
+; GISEL-NEXT: v_lshrrev_b64 v[4:5], 2, v[0:1]
+; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GISEL-NEXT: s_cbranch_execnz .LBB4_14
+; GISEL-NEXT: .LBB4_10: ; %Flow
+; GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GISEL-NEXT: .LBB4_11: ; %Flow4
+; GISEL-NEXT: s_or_b64 exec, exec, s[8:9]
+; GISEL-NEXT: v_and_b32_e32 v0, 0x80000000, v6
+; GISEL-NEXT: v_lshl_add_u32 v1, v8, 23, 1.0
+; GISEL-NEXT: v_and_b32_e32 v2, 0x7fffff, v4
+; GISEL-NEXT: v_or3_b32 v0, v2, v0, v1
+; GISEL-NEXT: v_cvt_f16_f32_e32 v4, v0
+; GISEL-NEXT: .LBB4_12: ; %Flow5
+; GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
+; GISEL-NEXT: v_mov_b32_e32 v0, v4
+; GISEL-NEXT: s_setpc_b64 s[30:31]
+; GISEL-NEXT: .LBB4_13: ; %itofp-if-else
+; GISEL-NEXT: v_add_u32_e32 v2, 0xffffff98, v5
+; GISEL-NEXT: v_lshlrev_b64 v[0:1], v2, v[0:1]
+; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v2
+; GISEL-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc
+; GISEL-NEXT: ; implicit-def: $vgpr7
+; GISEL-NEXT: ; implicit-def: $vgpr0
+; GISEL-NEXT: ; implicit-def: $vgpr2
+; GISEL-NEXT: s_branch .LBB4_2
+; GISEL-NEXT: .LBB4_14: ; %itofp-if-then20
+; GISEL-NEXT: v_lshrrev_b64 v[4:5], 3, v[0:1]
+; GISEL-NEXT: v_mov_b32_e32 v8, v7
+; GISEL-NEXT: s_branch .LBB4_10
%cvt = sitofp i128 %x to half
ret half %cvt
}
define half @uitofp_i128_to_f16(i128 %x) {
-; GCN-LABEL: uitofp_i128_to_f16:
-; GCN: ; %bb.0: ; %itofp-entry
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_or_b32_e32 v5, v1, v3
-; GCN-NEXT: v_or_b32_e32 v4, v0, v2
-; GCN-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GCN-NEXT: v_mov_b32_e32 v4, 0
-; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; GCN-NEXT: s_cbranch_execz .LBB5_12
-; GCN-NEXT: ; %bb.1: ; %itofp-if-end
-; GCN-NEXT: v_ffbh_u32_e32 v4, v2
-; GCN-NEXT: v_add_u32_e32 v4, 32, v4
-; GCN-NEXT: v_ffbh_u32_e32 v5, v3
-; GCN-NEXT: v_min_u32_e32 v4, v4, v5
-; GCN-NEXT: v_ffbh_u32_e32 v5, v0
-; GCN-NEXT: v_add_u32_e32 v5, 32, v5
-; GCN-NEXT: v_ffbh_u32_e32 v6, v1
-; GCN-NEXT: v_min_u32_e32 v5, v5, v6
-; GCN-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
-; GCN-NEXT: v_add_u32_e32 v5, 64, v5
-; GCN-NEXT: v_cndmask_b32_e32 v6, v5, v4, vcc
-; GCN-NEXT: v_sub_u32_e32 v4, 0x80, v6
-; GCN-NEXT: v_cmp_gt_i32_e32 vcc, 25, v4
-; GCN-NEXT: ; implicit-def: $vgpr7
-; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GCN-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; GCN-NEXT: s_cbranch_execnz .LBB5_13
-; GCN-NEXT: .LBB5_2: ; %Flow3
-; GCN-NEXT: s_or_saveexec_b64 s[8:9], s[4:5]
-; GCN-NEXT: v_sub_u32_e32 v5, 0x7f, v6
-; GCN-NEXT: s_xor_b64 exec, exec, s[8:9]
-; GCN-NEXT: s_cbranch_execz .LBB5_11
-; GCN-NEXT: ; %bb.3: ; %NodeBlock
-; GCN-NEXT: v_cmp_lt_i32_e32 vcc, 25, v4
-; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GCN-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
-; GCN-NEXT: s_cbranch_execz .LBB5_7
-; GCN-NEXT: ; %bb.4: ; %LeafBlock
-; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 26, v4
-; GCN-NEXT: s_and_saveexec_b64 s[12:13], vcc
-; GCN-NEXT: s_cbranch_execz .LBB5_6
-; GCN-NEXT: ; %bb.5: ; %itofp-sw-default
-; GCN-NEXT: v_sub_u32_e32 v11, 0x66, v6
-; GCN-NEXT: v_sub_u32_e32 v9, 64, v11
-; GCN-NEXT: v_lshrrev_b64 v[7:8], v11, v[0:1]
-; GCN-NEXT: v_lshlrev_b64 v[9:10], v9, v[2:3]
-; GCN-NEXT: v_sub_u32_e32 v12, 38, v6
-; GCN-NEXT: v_or_b32_e32 v10, v8, v10
-; GCN-NEXT: v_or_b32_e32 v9, v7, v9
-; GCN-NEXT: v_lshrrev_b64 v[7:8], v12, v[2:3]
-; GCN-NEXT: v_cmp_gt_u32_e32 vcc, 64, v11
-; GCN-NEXT: v_add_u32_e32 v13, 26, v6
-; GCN-NEXT: v_cndmask_b32_e32 v8, v8, v10, vcc
-; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v11
-; GCN-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc
-; GCN-NEXT: v_lshrrev_b64 v[9:10], v12, v[0:1]
-; GCN-NEXT: v_lshlrev_b64 v[11:12], v13, v[2:3]
-; GCN-NEXT: v_subrev_u32_e32 v6, 38, v6
-; GCN-NEXT: v_cndmask_b32_e64 v14, v7, v0, s[4:5]
-; GCN-NEXT: v_lshlrev_b64 v[6:7], v6, v[0:1]
-; GCN-NEXT: v_cndmask_b32_e64 v8, v8, v1, s[4:5]
-; GCN-NEXT: v_or_b32_e32 v10, v12, v10
-; GCN-NEXT: v_or_b32_e32 v9, v11, v9
-; GCN-NEXT: v_cmp_gt_u32_e32 vcc, 64, v13
-; GCN-NEXT: v_lshlrev_b64 v[0:1], v13, v[0:1]
-; GCN-NEXT: v_cndmask_b32_e32 v7, v7, v10, vcc
-; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v13
-; GCN-NEXT: v_cndmask_b32_e32 v6, v6, v9, vcc
-; GCN-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e64 v2, v6, v2, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; GCN-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GCN-NEXT: v_or_b32_e32 v1, v1, v3
-; GCN-NEXT: v_or_b32_e32 v0, v0, v2
-; GCN-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
-; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GCN-NEXT: v_or_b32_e32 v7, v14, v0
-; GCN-NEXT: v_mov_b32_e32 v0, v7
-; GCN-NEXT: v_mov_b32_e32 v1, v8
-; GCN-NEXT: .LBB5_6: ; %Flow1
-; GCN-NEXT: s_or_b64 exec, exec, s[12:13]
-; GCN-NEXT: .LBB5_7: ; %Flow2
-; GCN-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11]
-; GCN-NEXT: ; %bb.8: ; %itofp-sw-bb
-; GCN-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
-; GCN-NEXT: ; %bb.9: ; %itofp-sw-epilog
-; GCN-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN-NEXT: v_lshrrev_b32_e32 v2, 2, v0
-; GCN-NEXT: v_and_or_b32 v0, v2, 1, v0
-; GCN-NEXT: v_add_co_u32_e32 v0, vcc, 1, v0
-; GCN-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GCN-NEXT: v_and_b32_e32 v2, 0x4000000, v0
-; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
-; GCN-NEXT: v_alignbit_b32 v7, v1, v0, 2
-; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GCN-NEXT: s_cbranch_execnz .LBB5_14
-; GCN-NEXT: .LBB5_10: ; %Flow
-; GCN-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN-NEXT: .LBB5_11: ; %Flow4
-; GCN-NEXT: s_or_b64 exec, exec, s[8:9]
-; GCN-NEXT: v_and_b32_e32 v0, 0x7fffff, v7
-; GCN-NEXT: v_lshl_or_b32 v0, v5, 23, v0
-; GCN-NEXT: v_add_u32_e32 v0, 1.0, v0
-; GCN-NEXT: v_cvt_f16_f32_e32 v4, v0
-; GCN-NEXT: .LBB5_12: ; %Flow5
-; GCN-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN-NEXT: v_mov_b32_e32 v0, v4
-; GCN-NEXT: s_setpc_b64 s[30:31]
-; GCN-NEXT: .LBB5_13: ; %itofp-if-else
-; GCN-NEXT: v_add_u32_e32 v2, 0xffffff98, v6
-; GCN-NEXT: v_lshlrev_b64 v[0:1], v2, v[0:1]
-; GCN-NEXT: v_cmp_gt_u32_e32 vcc, 64, v2
-; GCN-NEXT: v_cndmask_b32_e32 v7, 0, v0, vcc
-; GCN-NEXT: ; implicit-def: $vgpr4
-; GCN-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GCN-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GCN-NEXT: s_branch .LBB5_2
-; GCN-NEXT: .LBB5_14: ; %itofp-if-then20
-; GCN-NEXT: v_alignbit_b32 v7, v1, v0, 3
-; GCN-NEXT: v_mov_b32_e32 v5, v4
-; GCN-NEXT: s_branch .LBB5_10
+; SDAG-LABEL: uitofp_i128_to_f16:
+; SDAG: ; %bb.0: ; %itofp-entry
+; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-NEXT: v_or_b32_e32 v5, v1, v3
+; SDAG-NEXT: v_or_b32_e32 v4, v0, v2
+; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SDAG-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; SDAG-NEXT: s_cbranch_execz .LBB5_12
+; SDAG-NEXT: ; %bb.1: ; %itofp-if-end
+; SDAG-NEXT: v_ffbh_u32_e32 v4, v2
+; SDAG-NEXT: v_add_u32_e32 v4, 32, v4
+; SDAG-NEXT: v_ffbh_u32_e32 v5, v3
+; SDAG-NEXT: v_min_u32_e32 v4, v4, v5
+; SDAG-NEXT: v_ffbh_u32_e32 v5, v0
+; SDAG-NEXT: v_add_u32_e32 v5, 32, v5
+; SDAG-NEXT: v_ffbh_u32_e32 v6, v1
+; SDAG-NEXT: v_min_u32_e32 v5, v5, v6
+; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; SDAG-NEXT: v_add_u32_e32 v5, 64, v5
+; SDAG-NEXT: v_cndmask_b32_e32 v6, v5, v4, vcc
+; SDAG-NEXT: v_sub_u32_e32 v4, 0x80, v6
+; SDAG-NEXT: v_cmp_gt_i32_e32 vcc, 25, v4
+; SDAG-NEXT: ; implicit-def: $vgpr7
+; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; SDAG-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
+; SDAG-NEXT: s_cbranch_execnz .LBB5_13
+; SDAG-NEXT: .LBB5_2: ; %Flow3
+; SDAG-NEXT: s_or_saveexec_b64 s[8:9], s[4:5]
+; SDAG-NEXT: v_sub_u32_e32 v5, 0x7f, v6
+; SDAG-NEXT: s_xor_b64 exec, exec, s[8:9]
+; SDAG-NEXT: s_cbranch_execz .LBB5_11
+; SDAG-NEXT: ; %bb.3: ; %NodeBlock
+; SDAG-NEXT: v_cmp_lt_i32_e32 vcc, 25, v4
+; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; SDAG-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
+; SDAG-NEXT: s_cbranch_execz .LBB5_7
+; SDAG-NEXT: ; %bb.4: ; %LeafBlock
+; SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 26, v4
+; SDAG-NEXT: s_and_saveexec_b64 s[12:13], vcc
+; SDAG-NEXT: s_cbranch_execz .LBB5_6
+; SDAG-NEXT: ; %bb.5: ; %itofp-sw-default
+; SDAG-NEXT: v_sub_u32_e32 v11, 0x66, v6
+; SDAG-NEXT: v_sub_u32_e32 v9, 64, v11
+; SDAG-NEXT: v_lshrrev_b64 v[7:8], v11, v[0:1]
+; SDAG-NEXT: v_lshlrev_b64 v[9:10], v9, v[2:3]
+; SDAG-NEXT: v_sub_u32_e32 v12, 38, v6
+; SDAG-NEXT: v_or_b32_e32 v10, v8, v10
+; SDAG-NEXT: v_or_b32_e32 v9, v7, v9
+; SDAG-NEXT: v_lshrrev_b64 v[7:8], v12, v[2:3]
+; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v11
+; SDAG-NEXT: v_add_u32_e32 v13, 26, v6
+; SDAG-NEXT: v_cndmask_b32_e32 v8, v8, v10, vcc
+; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v11
+; SDAG-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc
+; SDAG-NEXT: v_lshrrev_b64 v[9:10], v12, v[0:1]
+; SDAG-NEXT: v_lshlrev_b64 v[11:12], v13, v[2:3]
+; SDAG-NEXT: v_subrev_u32_e32 v6, 38, v6
+; SDAG-NEXT: v_cndmask_b32_e64 v14, v7, v0, s[4:5]
+; SDAG-NEXT: v_lshlrev_b64 v[6:7], v6, v[0:1]
+; SDAG-NEXT: v_cndmask_b32_e64 v8, v8, v1, s[4:5]
+; SDAG-NEXT: v_or_b32_e32 v10, v12, v10
+; SDAG-NEXT: v_or_b32_e32 v9, v11, v9
+; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v13
+; SDAG-NEXT: v_lshlrev_b64 v[0:1], v13, v[0:1]
+; SDAG-NEXT: v_cndmask_b32_e32 v7, v7, v10, vcc
+; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v13
+; SDAG-NEXT: v_cndmask_b32_e32 v6, v6, v9, vcc
+; SDAG-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[4:5]
+; SDAG-NEXT: v_cndmask_b32_e64 v2, v6, v2, s[4:5]
+; SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
+; SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
+; SDAG-NEXT: v_or_b32_e32 v1, v1, v3
+; SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; SDAG-NEXT: v_or_b32_e32 v7, v14, v0
+; SDAG-NEXT: v_mov_b32_e32 v0, v7
+; SDAG-NEXT: v_mov_b32_e32 v1, v8
+; SDAG-NEXT: .LBB5_6: ; %Flow1
+; SDAG-NEXT: s_or_b64 exec, exec, s[12:13]
+; SDAG-NEXT: .LBB5_7: ; %Flow2
+; SDAG-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11]
+; SDAG-NEXT: ; %bb.8: ; %itofp-sw-bb
+; SDAG-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
+; SDAG-NEXT: ; %bb.9: ; %itofp-sw-epilog
+; SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
+; SDAG-NEXT: v_lshrrev_b32_e32 v2, 2, v0
+; SDAG-NEXT: v_and_or_b32 v0, v2, 1, v0
+; SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 1, v0
+; SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; SDAG-NEXT: v_and_b32_e32 v2, 0x4000000, v0
+; SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
+; SDAG-NEXT: v_alignbit_b32 v7, v1, v0, 2
+; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; SDAG-NEXT: s_cbranch_execnz .LBB5_14
+; SDAG-NEXT: .LBB5_10: ; %Flow
+; SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
+; SDAG-NEXT: .LBB5_11: ; %Flow4
+; SDAG-NEXT: s_or_b64 exec, exec, s[8:9]
+; SDAG-NEXT: v_and_b32_e32 v0, 0x7fffff, v7
+; SDAG-NEXT: v_lshl_or_b32 v0, v5, 23, v0
+; SDAG-NEXT: v_add_u32_e32 v0, 1.0, v0
+; SDAG-NEXT: v_cvt_f16_f32_e32 v4, v0
+; SDAG-NEXT: .LBB5_12: ; %Flow5
+; SDAG-NEXT: s_or_b64 exec, exec, s[6:7]
+; SDAG-NEXT: v_mov_b32_e32 v0, v4
+; SDAG-NEXT: s_setpc_b64 s[30:31]
+; SDAG-NEXT: .LBB5_13: ; %itofp-if-else
+; SDAG-NEXT: v_add_u32_e32 v2, 0xffffff98, v6
+; SDAG-NEXT: v_lshlrev_b64 v[0:1], v2, v[0:1]
+; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v2
+; SDAG-NEXT: v_cndmask_b32_e32 v7, 0, v0, vcc
+; SDAG-NEXT: ; implicit-def: $vgpr4
+; SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
+; SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
+; SDAG-NEXT: s_branch .LBB5_2
+; SDAG-NEXT: .LBB5_14: ; %itofp-if-then20
+; SDAG-NEXT: v_alignbit_b32 v7, v1, v0, 3
+; SDAG-NEXT: v_mov_b32_e32 v5, v4
+; SDAG-NEXT: s_branch .LBB5_10
+;
+; GISEL-LABEL: uitofp_i128_to_f16:
+; GISEL: ; %bb.0: ; %itofp-entry
+; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT: v_or_b32_e32 v4, v0, v2
+; GISEL-NEXT: v_or_b32_e32 v5, v1, v3
+; GISEL-NEXT: s_mov_b32 s4, 0
+; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; GISEL-NEXT: v_mov_b32_e32 v4, s4
+; GISEL-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GISEL-NEXT: s_cbranch_execz .LBB5_12
+; GISEL-NEXT: ; %bb.1: ; %itofp-if-end
+; GISEL-NEXT: v_ffbh_u32_e32 v5, v0
+; GISEL-NEXT: v_ffbh_u32_e32 v4, v1
+; GISEL-NEXT: v_add_u32_e32 v5, 32, v5
+; GISEL-NEXT: v_ffbh_u32_e32 v6, v2
+; GISEL-NEXT: v_min_u32_e32 v4, v4, v5
+; GISEL-NEXT: v_ffbh_u32_e32 v5, v3
+; GISEL-NEXT: v_add_u32_e32 v6, 32, v6
+; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3]
+; GISEL-NEXT: v_add_u32_e32 v4, 64, v4
+; GISEL-NEXT: v_min_u32_e32 v5, v5, v6
+; GISEL-NEXT: v_cndmask_b32_e32 v5, v5, v4, vcc
+; GISEL-NEXT: v_sub_u32_e32 v6, 0x80, v5
+; GISEL-NEXT: v_cmp_ge_i32_e32 vcc, 24, v6
+; GISEL-NEXT: ; implicit-def: $vgpr4
+; GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
+; GISEL-NEXT: s_cbranch_execnz .LBB5_13
+; GISEL-NEXT: .LBB5_2: ; %Flow3
+; GISEL-NEXT: s_or_saveexec_b64 s[8:9], s[4:5]
+; GISEL-NEXT: v_sub_u32_e32 v7, 0x7f, v5
+; GISEL-NEXT: s_xor_b64 exec, exec, s[8:9]
+; GISEL-NEXT: s_cbranch_execz .LBB5_11
+; GISEL-NEXT: ; %bb.3: ; %NodeBlock
+; GISEL-NEXT: v_cmp_le_i32_e32 vcc, 26, v6
+; GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GISEL-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
+; GISEL-NEXT: s_cbranch_execz .LBB5_7
+; GISEL-NEXT: ; %bb.4: ; %LeafBlock
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 26, v6
+; GISEL-NEXT: s_and_saveexec_b64 s[12:13], vcc
+; GISEL-NEXT: s_cbranch_execz .LBB5_6
+; GISEL-NEXT: ; %bb.5: ; %itofp-sw-default
+; GISEL-NEXT: v_sub_u32_e32 v4, 0x66, v5
+; GISEL-NEXT: v_sub_u32_e32 v10, 64, v4
+; GISEL-NEXT: v_lshrrev_b64 v[8:9], v4, v[0:1]
+; GISEL-NEXT: v_lshlrev_b64 v[10:11], v10, v[2:3]
+; GISEL-NEXT: v_add_u32_e32 v12, 0xffffffc0, v4
+; GISEL-NEXT: v_or_b32_e32 v10, v8, v10
+; GISEL-NEXT: v_or_b32_e32 v11, v9, v11
+; GISEL-NEXT: v_lshrrev_b64 v[8:9], v12, v[2:3]
+; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v4
+; GISEL-NEXT: v_add_u32_e32 v13, 26, v5
+; GISEL-NEXT: v_cndmask_b32_e32 v8, v8, v10, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v9, v9, v11, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
+; GISEL-NEXT: v_sub_u32_e32 v10, 64, v13
+; GISEL-NEXT: v_cndmask_b32_e32 v12, v8, v0, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v4, v9, v1, vcc
+; GISEL-NEXT: v_lshrrev_b64 v[8:9], v13, -1
+; GISEL-NEXT: v_lshlrev_b64 v[10:11], v10, -1
+; GISEL-NEXT: v_add_u32_e32 v5, 0xffffffda, v5
+; GISEL-NEXT: v_or_b32_e32 v14, v8, v10
+; GISEL-NEXT: v_or_b32_e32 v15, v9, v11
+; GISEL-NEXT: v_lshrrev_b64 v[10:11], v5, -1
+; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v13
+; GISEL-NEXT: v_cndmask_b32_e32 v5, v10, v14, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v10, v11, v15, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v13
+; GISEL-NEXT: v_cndmask_b32_e32 v8, 0, v8, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v9, 0, v9, vcc
+; GISEL-NEXT: v_cndmask_b32_e64 v5, v5, -1, s[4:5]
+; GISEL-NEXT: v_cndmask_b32_e64 v10, v10, -1, s[4:5]
+; GISEL-NEXT: v_and_b32_e32 v2, v8, v2
+; GISEL-NEXT: v_and_b32_e32 v3, v9, v3
+; GISEL-NEXT: v_and_or_b32 v0, v5, v0, v2
+; GISEL-NEXT: v_and_or_b32 v1, v10, v1, v3
+; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GISEL-NEXT: v_or_b32_e32 v3, v12, v0
+; GISEL-NEXT: v_mov_b32_e32 v0, v3
+; GISEL-NEXT: v_mov_b32_e32 v1, v4
+; GISEL-NEXT: v_mov_b32_e32 v2, v5
+; GISEL-NEXT: v_mov_b32_e32 v3, v6
+; GISEL-NEXT: .LBB5_6: ; %Flow1
+; GISEL-NEXT: s_or_b64 exec, exec, s[12:13]
+; GISEL-NEXT: .LBB5_7: ; %Flow2
+; GISEL-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11]
+; GISEL-NEXT: ; %bb.8: ; %itofp-sw-bb
+; GISEL-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
+; GISEL-NEXT: ; %bb.9: ; %itofp-sw-epilog
+; GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GISEL-NEXT: v_bfe_u32 v2, v0, 2, 1
+; GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 1, v0
+; GISEL-NEXT: v_bfrev_b32_e32 v2, 32
+; GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GISEL-NEXT: v_mov_b32_e32 v3, 0
+; GISEL-NEXT: v_and_b32_e32 v2, 0x4000000, v0
+; GISEL-NEXT: v_lshrrev_b64 v[4:5], 2, v[0:1]
+; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GISEL-NEXT: s_cbranch_execnz .LBB5_14
+; GISEL-NEXT: .LBB5_10: ; %Flow
+; GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GISEL-NEXT: .LBB5_11: ; %Flow4
+; GISEL-NEXT: s_or_b64 exec, exec, s[8:9]
+; GISEL-NEXT: v_lshl_add_u32 v0, v7, 23, 1.0
+; GISEL-NEXT: v_mov_b32_e32 v1, 0x7fffff
+; GISEL-NEXT: v_and_or_b32 v0, v4, v1, v0
+; GISEL-NEXT: v_cvt_f16_f32_e32 v4, v0
+; GISEL-NEXT: .LBB5_12: ; %Flow5
+; GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
+; GISEL-NEXT: v_mov_b32_e32 v0, v4
+; GISEL-NEXT: s_setpc_b64 s[30:31]
+; GISEL-NEXT: .LBB5_13: ; %itofp-if-else
+; GISEL-NEXT: v_add_u32_e32 v2, 0xffffff98, v5
+; GISEL-NEXT: v_lshlrev_b64 v[0:1], v2, v[0:1]
+; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v2
+; GISEL-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc
+; GISEL-NEXT: ; implicit-def: $vgpr6
+; GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GISEL-NEXT: s_branch .LBB5_2
+; GISEL-NEXT: .LBB5_14: ; %itofp-if-then20
+; GISEL-NEXT: v_lshrrev_b64 v[4:5], 3, v[0:1]
+; GISEL-NEXT: v_mov_b32_e32 v7, v6
+; GISEL-NEXT: s_branch .LBB5_10
%cvt = uitofp i128 %x to half
ret half %cvt
}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GISEL: {{.*}}
-; SDAG: {{.*}}
+; GCN: {{.*}}
More information about the llvm-branch-commits
mailing list