[llvm-branch-commits] [llvm] [SelectionDAG] Select the FREEZE target opcode (PR #240476)

Mitch Briles via llvm-branch-commits llvm-branch-commits at lists.llvm.org
Mon Nov 9 13:47:06 PST 2026


https://github.com/MitchBriles updated https://github.com/llvm/llvm-project/pull/240476

>From ad4dc40aa05d4ae29f3b728cb3a8c30c7271175b Mon Sep 17 00:00:00 2001
From: Mitch <mitchbriles at gmail.com>
Date: Fri, 23 Oct 2026 06:40:58 -0600
Subject: [PATCH 1/2] Precommit miscompiling FREEZE tests

Co-Authored-By: Claude Opus 5.5 <noreply at anthropic.com>
---
 llvm/test/CodeGen/AArch64/freeze-multi-use.ll | 117 +++++++++++++++
 llvm/test/CodeGen/AArch64/freeze.ll           |  14 ++
 llvm/test/CodeGen/AMDGPU/freeze-multi-use.ll  |  94 ++++++++++++
 llvm/test/CodeGen/RISCV/freeze-multi-use.ll   | 137 ++++++++++++++++++
 .../CodeGen/RISCV/rvv/freeze-multi-use.ll     |  43 ++++++
 llvm/test/CodeGen/X86/freeze-multi-use.ll     |  85 +++++++++++
 llvm/test/CodeGen/X86/freeze.ll               |   8 +
 7 files changed, 498 insertions(+)
 create mode 100644 llvm/test/CodeGen/AArch64/freeze-multi-use.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/freeze-multi-use.ll
 create mode 100644 llvm/test/CodeGen/RISCV/freeze-multi-use.ll
 create mode 100644 llvm/test/CodeGen/RISCV/rvv/freeze-multi-use.ll
 create mode 100644 llvm/test/CodeGen/X86/freeze-multi-use.ll

diff --git a/llvm/test/CodeGen/AArch64/freeze-multi-use.ll b/llvm/test/CodeGen/AArch64/freeze-multi-use.ll
new file mode 100644
index 0000000000000..822ec50d0a571
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/freeze-multi-use.ll
@@ -0,0 +1,117 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=aarch64-- < %s | FileCheck %s
+
+; Every use of a frozen value must observe the same value.
+
+; The multiply and the shift must read the same value.
+define i32 @ashr_mul(i32 %x) {
+; CHECK-LABEL: ashr_mul:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    asr w8, w8, #3
+; CHECK-NEXT:    mul w0, w8, w8
+; CHECK-NEXT:    ret
+  %p = ashr i32 %x, 32
+  %w = freeze i32 %p
+  %s = ashr i32 %w, 3
+  %r = mul i32 %w, %s
+  ret i32 %r
+}
+
+; Both halves of the funnel shift must use the same shift amount.
+define i8 @fshr(i32 %x) {
+; CHECK-LABEL: fshr:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mvn w8, w8
+; CHECK-NEXT:    mov w9, #2 // =0x2
+; CHECK-NEXT:    lsr w10, w0, w8
+; CHECK-NEXT:    lsl w8, w9, w8
+; CHECK-NEXT:    orr w0, w8, w10
+; CHECK-NEXT:    ret
+  %w = freeze i32 poison
+  %f = call i32 @llvm.fshr.i32(i32 1, i32 %x, i32 %w)
+  %r = trunc i32 %f to i8
+  ret i8 %r
+}
+
+; The select condition and the zext must read the same bit.
+define i64 @select_zext(i64 %a, i64 %b) {
+; CHECK-LABEL: select_zext:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    tst w8, #0x1
+; CHECK-NEXT:    csel x8, x0, x1, ne
+; CHECK-NEXT:    and x9, x8, #0x1
+; CHECK-NEXT:    add x0, x8, x9
+; CHECK-NEXT:    ret
+  %w = freeze i1 poison
+  %s = select i1 %w, i64 %a, i64 %b
+  %z = zext i1 %w to i64
+  %r = add i64 %s, %z
+  ret i64 %r
+}
+
+; The fcmp and the bitcast must see the same float, so the result is 0.
+define i32 @float_bits() {
+; CHECK-LABEL: float_bits:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcmp s0, s0
+; CHECK-NEXT:    and w8, w8, #0x7fffffff
+; CHECK-NEXT:    mov w9, #2139095040 // =0x7f800000
+; CHECK-NEXT:    cset w10, vs
+; CHECK-NEXT:    cmp w8, w9
+; CHECK-NEXT:    cset w8, hi
+; CHECK-NEXT:    eor w0, w10, w8
+; CHECK-NEXT:    ret
+  %w = freeze float poison
+  %c = fcmp uno float %w, 0.0
+  %b = bitcast float %w to i32
+  %e = and i32 %b, 2147483647
+  %n = icmp ugt i32 %e, 2139095040
+  %x = xor i1 %c, %n
+  %r = zext i1 %x to i32
+  ret i32 %r
+}
+
+; The result must be element 0 of %x.
+define i32 @vector_lane_and_whole(<4 x i32> %x) {
+; CHECK-LABEL: vector_lane_and_whole:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    add v0.4s, v0.4s, v0.4s
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    sub w0, w8, w8
+; CHECK-NEXT:    ret
+  %w = freeze <4 x i32> poison
+  %e0 = extractelement <4 x i32> %w, i32 0
+  %y = add <4 x i32> %w, %x
+  %e1 = extractelement <4 x i32> %y, i32 0
+  %r = sub i32 %e1, %e0
+  ret i32 %r
+}
+
+; The returned value must be the one passed to @use.
+define i32 @across_call() nounwind {
+; CHECK-LABEL: across_call:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    bl use
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %w = freeze i32 poison
+  call void @use(i32 %w)
+  ret i32 %w
+}
+
+; The returned value must be the one passed to @usev.
+define <4 x i32> @vec_across_call() nounwind {
+; CHECK-LABEL: vec_across_call:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    bl usev
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %w = freeze <4 x i32> poison
+  call void @usev(<4 x i32> %w)
+  ret <4 x i32> %w
+}
+
+declare void @use(i32)
+declare void @usev(<4 x i32>)
diff --git a/llvm/test/CodeGen/AArch64/freeze.ll b/llvm/test/CodeGen/AArch64/freeze.ll
index a660c52c567ed..fff659e1f21b9 100644
--- a/llvm/test/CodeGen/AArch64/freeze.ll
+++ b/llvm/test/CodeGen/AArch64/freeze.ll
@@ -588,3 +588,17 @@ define i32 @freeze_ucmp(i32 %a0) nounwind {
   %z = call i32 @llvm.ucmp.i32(i32 %y, i32 1)
   ret i32 %z
 }
+
+define i128 @freeze_i128() {
+; CHECK-SD-LABEL: freeze_i128:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: freeze_i128:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    mov d0, v0.d[1]
+; CHECK-GI-NEXT:    fmov x1, d0
+; CHECK-GI-NEXT:    ret
+  %y1 = freeze i128 poison
+  ret i128 %y1
+}
diff --git a/llvm/test/CodeGen/AMDGPU/freeze-multi-use.ll b/llvm/test/CodeGen/AMDGPU/freeze-multi-use.ll
new file mode 100644
index 0000000000000..08422bf946c3b
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/freeze-multi-use.ll
@@ -0,0 +1,94 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgpu11.00-amd-amdhsa < %s | FileCheck %s
+
+; Every use of a frozen value must observe the same value.
+
+; The multiply and the shift must read the same value.
+define i32 @ashr_mul(i32 %x) {
+; CHECK-LABEL: ashr_mul:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_ashr_i32 s0, s0, 3
+; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; CHECK-NEXT:    s_mul_i32 s0, s0, s0
+; CHECK-NEXT:    v_mov_b32_e32 v0, s0
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %p = ashr i32 %x, 32
+  %w = freeze i32 %p
+  %s = ashr i32 %w, 3
+  %r = mul i32 %w, %s
+  ret i32 %r
+}
+
+; The returned value must be the one passed to @use.
+define i32 @across_call() nounwind {
+; CHECK-LABEL: across_call:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_mov_b32 s0, s33
+; CHECK-NEXT:    s_mov_b32 s33, s32
+; CHECK-NEXT:    s_or_saveexec_b32 s1, -1
+; CHECK-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
+; CHECK-NEXT:    s_mov_b32 exec_lo, s1
+; CHECK-NEXT:    v_writelane_b32 v40, s0, 2
+; CHECK-NEXT:    s_add_i32 s32, s32, 16
+; CHECK-NEXT:    v_writelane_b32 v40, s30, 0
+; CHECK-NEXT:    v_writelane_b32 v40, s31, 1
+; CHECK-NEXT:    s_getpc_b64 s[0:1]
+; CHECK-NEXT:    s_add_u32 s0, s0, use at gotpcrel32@lo+4
+; CHECK-NEXT:    s_addc_u32 s1, s1, use at gotpcrel32@hi+12
+; CHECK-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; CHECK-NEXT:    v_readlane_b32 s30, v40, 0
+; CHECK-NEXT:    v_readlane_b32 s31, v40, 1
+; CHECK-NEXT:    s_mov_b32 s32, s33
+; CHECK-NEXT:    v_readlane_b32 s0, v40, 2
+; CHECK-NEXT:    s_or_saveexec_b32 s1, -1
+; CHECK-NEXT:    scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
+; CHECK-NEXT:    s_mov_b32 exec_lo, s1
+; CHECK-NEXT:    s_mov_b32 s33, s0
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %w = freeze i32 poison
+  call void @use(i32 %w)
+  ret i32 %w
+}
+
+; The returned value must be the one passed to @usev.
+define <4 x i32> @vec_across_call() nounwind {
+; CHECK-LABEL: vec_across_call:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_mov_b32 s0, s33
+; CHECK-NEXT:    s_mov_b32 s33, s32
+; CHECK-NEXT:    s_or_saveexec_b32 s1, -1
+; CHECK-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
+; CHECK-NEXT:    s_mov_b32 exec_lo, s1
+; CHECK-NEXT:    v_writelane_b32 v40, s0, 2
+; CHECK-NEXT:    s_add_i32 s32, s32, 16
+; CHECK-NEXT:    v_writelane_b32 v40, s30, 0
+; CHECK-NEXT:    v_writelane_b32 v40, s31, 1
+; CHECK-NEXT:    s_getpc_b64 s[0:1]
+; CHECK-NEXT:    s_add_u32 s0, s0, usev at gotpcrel32@lo+4
+; CHECK-NEXT:    s_addc_u32 s1, s1, usev at gotpcrel32@hi+12
+; CHECK-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; CHECK-NEXT:    v_readlane_b32 s30, v40, 0
+; CHECK-NEXT:    v_readlane_b32 s31, v40, 1
+; CHECK-NEXT:    s_mov_b32 s32, s33
+; CHECK-NEXT:    v_readlane_b32 s0, v40, 2
+; CHECK-NEXT:    s_or_saveexec_b32 s1, -1
+; CHECK-NEXT:    scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
+; CHECK-NEXT:    s_mov_b32 exec_lo, s1
+; CHECK-NEXT:    s_mov_b32 s33, s0
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %w = freeze <4 x i32> poison
+  call void @usev(<4 x i32> %w)
+  ret <4 x i32> %w
+}
+
+declare void @use(i32)
+declare void @usev(<4 x i32>)
diff --git a/llvm/test/CodeGen/RISCV/freeze-multi-use.ll b/llvm/test/CodeGen/RISCV/freeze-multi-use.ll
new file mode 100644
index 0000000000000..18b7644740660
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/freeze-multi-use.ll
@@ -0,0 +1,137 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=riscv32 < %s | FileCheck %s --check-prefixes=CHECK,RV32
+; RUN: llc -mtriple=riscv64 < %s | FileCheck %s --check-prefixes=CHECK,RV64
+
+; Every use of a frozen value must observe the same value.
+
+; The result must have the same parity as %n.
+define i32 @add_twice(i32 %n) {
+; RV32-LABEL: add_twice:
+; RV32:       # %bb.0:
+; RV32-NEXT:    add a0, a0, a0
+; RV32-NEXT:    add a0, a0, a0
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: add_twice:
+; RV64:       # %bb.0:
+; RV64-NEXT:    add a0, a0, a0
+; RV64-NEXT:    addw a0, a0, a0
+; RV64-NEXT:    ret
+  %w = freeze i32 poison
+  %a = add i32 %n, %w
+  %r = add i32 %a, %w
+  ret i32 %r
+}
+
+; The result must be %x.
+define i32 @cross_block(i1 %c, i32 %x) {
+; RV32-LABEL: cross_block:
+; RV32:       # %bb.0: # %entry
+; RV32-NEXT:    andi a0, a0, 1
+; RV32-NEXT:    beqz a0, .LBB1_2
+; RV32-NEXT:  # %bb.1: # %t
+; RV32-NEXT:    add a0, a0, a1
+; RV32-NEXT:    sub a0, a0, a0
+; RV32-NEXT:    ret
+; RV32-NEXT:  .LBB1_2: # %e
+; RV32-NEXT:    mv a0, a1
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: cross_block:
+; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    andi a0, a0, 1
+; RV64-NEXT:    beqz a0, .LBB1_2
+; RV64-NEXT:  # %bb.1: # %t
+; RV64-NEXT:    add a0, a0, a1
+; RV64-NEXT:    subw a0, a0, a0
+; RV64-NEXT:    ret
+; RV64-NEXT:  .LBB1_2: # %e
+; RV64-NEXT:    mv a0, a1
+; RV64-NEXT:    ret
+entry:
+  %w = freeze i32 poison
+  %a = add i32 %w, %x
+  br i1 %c, label %t, label %e
+t:
+  %s = sub i32 %a, %w
+  ret i32 %s
+e:
+  ret i32 %x
+}
+
+; The fcmp and the bitcast must see the same float, so the result is 0.
+define i32 @float_bits() {
+; RV32-LABEL: float_bits:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -16
+; RV32-NEXT:    .cfi_def_cfa_offset 16
+; RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32-NEXT:    .cfi_offset ra, -4
+; RV32-NEXT:    call __unordsf2
+; RV32-NEXT:    slli a1, a0, 1
+; RV32-NEXT:    srli a1, a1, 1
+; RV32-NEXT:    lui a2, 522240
+; RV32-NEXT:    snez a0, a0
+; RV32-NEXT:    sltu a1, a2, a1
+; RV32-NEXT:    xor a0, a0, a1
+; RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32-NEXT:    .cfi_restore ra
+; RV32-NEXT:    addi sp, sp, 16
+; RV32-NEXT:    .cfi_def_cfa_offset 0
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: float_bits:
+; RV64:       # %bb.0:
+; RV64-NEXT:    addi sp, sp, -16
+; RV64-NEXT:    .cfi_def_cfa_offset 16
+; RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64-NEXT:    .cfi_offset ra, -8
+; RV64-NEXT:    call __unordsf2
+; RV64-NEXT:    slli a1, a0, 33
+; RV64-NEXT:    srli a1, a1, 33
+; RV64-NEXT:    lui a2, 522240
+; RV64-NEXT:    snez a0, a0
+; RV64-NEXT:    sltu a1, a2, a1
+; RV64-NEXT:    xor a0, a0, a1
+; RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64-NEXT:    .cfi_restore ra
+; RV64-NEXT:    addi sp, sp, 16
+; RV64-NEXT:    .cfi_def_cfa_offset 0
+; RV64-NEXT:    ret
+  %w = freeze float poison
+  %c = fcmp uno float %w, 0.0
+  %b = bitcast float %w to i32
+  %e = and i32 %b, 2147483647
+  %n = icmp ugt i32 %e, 2139095040
+  %x = xor i1 %c, %n
+  %r = zext i1 %x to i32
+  ret i32 %r
+}
+
+; The returned value must be the one passed to @use.
+define i32 @across_call() nounwind {
+; RV32-LABEL: across_call:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -16
+; RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32-NEXT:    call use
+; RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32-NEXT:    addi sp, sp, 16
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: across_call:
+; RV64:       # %bb.0:
+; RV64-NEXT:    addi sp, sp, -16
+; RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64-NEXT:    call use
+; RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64-NEXT:    addi sp, sp, 16
+; RV64-NEXT:    ret
+  %w = freeze i32 poison
+  call void @use(i32 %w)
+  ret i32 %w
+}
+
+declare void @use(i32)
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
diff --git a/llvm/test/CodeGen/RISCV/rvv/freeze-multi-use.ll b/llvm/test/CodeGen/RISCV/rvv/freeze-multi-use.ll
new file mode 100644
index 0000000000000..71d79b5f18b4e
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/freeze-multi-use.ll
@@ -0,0 +1,43 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=riscv64 -mattr=+v < %s | FileCheck %s
+
+; Every use of a frozen value must observe the same value.
+
+; Element 1 is poison before the freeze. The result must be 0.
+define i32 @partial_vector(i32 %x) {
+; CHECK-LABEL: partial_vector:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vmv.v.x v8, a0
+; CHECK-NEXT:    vadd.vx v8, v8, a0
+; CHECK-NEXT:    vslidedown.vi v8, v8, 1
+; CHECK-NEXT:    vmv.x.s a0, v8
+; CHECK-NEXT:    add a1, a0, a0
+; CHECK-NEXT:    subw a0, a0, a1
+; CHECK-NEXT:    ret
+  %v = insertelement <4 x i32> poison, i32 %x, i32 0
+  %w = freeze <4 x i32> %v
+  %y = add <4 x i32> %w, %w
+  %e = extractelement <4 x i32> %w, i32 1
+  %e2 = extractelement <4 x i32> %y, i32 1
+  %r = sub i32 %e2, %e
+  %q = sub i32 %r, %e
+  ret i32 %q
+}
+
+; The returned value must be the one passed to @usev.
+define <4 x i32> @vec_across_call() nounwind {
+; CHECK-LABEL: vec_across_call:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    call usev
+; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %w = freeze <4 x i32> poison
+  call void @usev(<4 x i32> %w)
+  ret <4 x i32> %w
+}
+
+declare void @usev(<4 x i32>)
diff --git a/llvm/test/CodeGen/X86/freeze-multi-use.ll b/llvm/test/CodeGen/X86/freeze-multi-use.ll
new file mode 100644
index 0000000000000..08bee89cec667
--- /dev/null
+++ b/llvm/test/CodeGen/X86/freeze-multi-use.ll
@@ -0,0 +1,85 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=x86_64-- < %s | FileCheck %s
+
+; Every use of a frozen value must observe the same value.
+
+; The result must have the same parity as %n.
+define i32 @add_twice(i32 %n) {
+; CHECK-LABEL: add_twice:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    addl %eax, %eax
+; CHECK-NEXT:    addl %eax, %eax
+; CHECK-NEXT:    retq
+  %w = freeze i32 poison
+  %a = add i32 %n, %w
+  %r = add i32 %a, %w
+  ret i32 %r
+}
+
+; The multiply and the shift must read the same value.
+define i32 @ashr_mul(i32 %x) {
+; CHECK-LABEL: ashr_mul:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    sarl $3, %eax
+; CHECK-NEXT:    imull %eax, %eax
+; CHECK-NEXT:    retq
+  %p = ashr i32 %x, 32
+  %w = freeze i32 %p
+  %s = ashr i32 %w, 3
+  %r = mul i32 %w, %s
+  ret i32 %r
+}
+
+; The result must be %x.
+define i32 @cross_block(i1 %c, i32 %x) {
+; CHECK-LABEL: cross_block:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    testb $1, %dil
+; CHECK-NEXT:    je .LBB2_2
+; CHECK-NEXT:  # %bb.1: # %t
+; CHECK-NEXT:    addl %esi, %eax
+; CHECK-NEXT:    subl %eax, %eax
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB2_2: # %e
+; CHECK-NEXT:    movl %esi, %eax
+; CHECK-NEXT:    retq
+entry:
+  %w = freeze i32 poison
+  %a = add i32 %w, %x
+  br i1 %c, label %t, label %e
+t:
+  %s = sub i32 %a, %w
+  ret i32 %s
+e:
+  ret i32 %x
+}
+
+; The returned value must be the one passed to @use.
+define i32 @across_call() nounwind {
+; CHECK-LABEL: across_call:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq use at PLT
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+  %w = freeze i32 poison
+  call void @use(i32 %w)
+  ret i32 %w
+}
+
+; The returned value must be the one passed to @usev.
+define <4 x i32> @vec_across_call() nounwind {
+; CHECK-LABEL: vec_across_call:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    callq usev at PLT
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+  %w = freeze <4 x i32> poison
+  call void @usev(<4 x i32> %w)
+  ret <4 x i32> %w
+}
+
+declare void @use(i32)
+declare void @usev(<4 x i32>)
diff --git a/llvm/test/CodeGen/X86/freeze.ll b/llvm/test/CodeGen/X86/freeze.ll
index 5ea78f884e980..2609b2fafac54 100644
--- a/llvm/test/CodeGen/X86/freeze.ll
+++ b/llvm/test/CodeGen/X86/freeze.ll
@@ -203,3 +203,11 @@ entry:
   %and = and i64 %sel2, %shl
   ret i64 %and
 }
+
+define i128 @freeze_i128() {
+; X86ASM-LABEL: freeze_i128:
+; X86ASM:       # %bb.0:
+; X86ASM-NEXT:    retq
+  %y1 = freeze i128 poison
+  ret i128 %y1
+}

>From 0d168cb42f0002c9b526a6fd1b62d7b3bda6c9fd Mon Sep 17 00:00:00 2001
From: Mitch <mitchbriles at gmail.com>
Date: Fri, 23 Oct 2026 15:19:06 -0600
Subject: [PATCH 2/2] [SelectionDAG] Select the FREEZE target opcode

Selecting freeze as a COPY is wrong if the value may be undefined.
Now we can select ISD::FREEZE as FREEZE instead. Targets that don't
have physical registers can keep selecting COPY since there is no
regalloc to observe the difference.

Fixes #58321, #60429, #144780.

Assisted-By: Claude Opus 5.5 <noreply at anthropic.com>
---
 llvm/docs/CodeGenerator.md                    |    4 +-
 .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp |   22 +-
 .../CodeGen/SelectionDAG/SelectionDAGISel.cpp |   10 +-
 .../AArch64/concat-vector-add-combine.ll      |    1 +
 llvm/test/CodeGen/AArch64/fma-fneg-combine.ll |    6 +-
 llvm/test/CodeGen/AArch64/freeze-multi-use.ll |   51 +-
 llvm/test/CodeGen/AArch64/freeze.ll           |  279 +-
 .../AArch64/neon-shuffle-vector-tbl.ll        |   70 +-
 llvm/test/CodeGen/AArch64/vector-compress.ll  |   12 +-
 .../AArch64/vector-extract-last-active.ll     |   70 +-
 .../AMDGPU/agpr-copy-no-free-registers.ll     |   14 +-
 llvm/test/CodeGen/AMDGPU/andorbitset.ll       |    7 +-
 .../test/CodeGen/AMDGPU/carryout-selection.ll |   40 +-
 .../AMDGPU/divergence-driven-trunc-to-i1.ll   |   48 +-
 .../AMDGPU/float-to-arbitrary-fp-fp8-hw.ll    |   22 +-
 llvm/test/CodeGen/AMDGPU/fold-fabs.ll         |    1 +
 llvm/test/CodeGen/AMDGPU/freeze-multi-use.ll  |   47 +-
 llvm/test/CodeGen/AMDGPU/freeze.ll            |   39 +-
 .../CodeGen/AMDGPU/llvm.amdgcn.permlane.ll    | 3806 +++++++++++------
 .../AMDGPU/llvm.amdgcn.permlane16.var.ll      |   80 +-
 llvm/test/CodeGen/AMDGPU/mad_64_32.ll         |    4 +-
 .../CodeGen/AMDGPU/mdt-preserving-crash.ll    |    3 +-
 llvm/test/CodeGen/AMDGPU/mul_int24.ll         |    8 +-
 llvm/test/CodeGen/AMDGPU/permute_i8.ll        |   24 +-
 ...-alloca-vector-dynamic-idx-bitcasts-llc.ll |  638 ++-
 .../AMDGPU/promote-alloca-vector-to-vector.ll |    5 +-
 llvm/test/CodeGen/AMDGPU/sad.ll               |    7 +-
 llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll     |   12 +-
 llvm/test/CodeGen/AMDGPU/select-undef.ll      |    2 +
 .../AMDGPU/splitkit-getsubrangeformask.ll     |    7 +-
 llvm/test/CodeGen/AMDGPU/udiv64.ll            |    3 +-
 llvm/test/CodeGen/Hexagon/pr196966.ll         |   33 +-
 llvm/test/CodeGen/PowerPC/sms-grp-order.ll    |   15 +-
 .../CodeGen/PowerPC/widen-vec-correctly-be.ll |    6 +-
 llvm/test/CodeGen/RISCV/freeze-multi-use.ll   |   60 +-
 .../CodeGen/RISCV/rvv/freeze-multi-use.ll     |   24 +-
 llvm/test/CodeGen/SystemZ/pr60413.ll          |   10 +-
 llvm/test/CodeGen/X86/bit-manip-i256.ll       |   37 +-
 llvm/test/CodeGen/X86/combine-urem.ll         |   12 +-
 llvm/test/CodeGen/X86/freeze-binary.ll        |    6 +
 llvm/test/CodeGen/X86/freeze-multi-use.ll     |   33 +-
 llvm/test/CodeGen/X86/freeze-vector.ll        |   26 +-
 llvm/test/CodeGen/X86/freeze.ll               |   28 +-
 llvm/test/CodeGen/X86/pr169485.ll             |    5 +-
 llvm/test/CodeGen/X86/pr57402.ll              |   18 +-
 llvm/test/CodeGen/X86/pr57554.ll              |    1 +
 llvm/test/CodeGen/X86/var-permute-256.ll      |    7 +-
 .../CodeGen/X86/vector-compress-freeze.ll     |    5 +-
 llvm/test/CodeGen/X86/vector-compress.ll      |  574 +--
 49 files changed, 4014 insertions(+), 2228 deletions(-)

diff --git a/llvm/docs/CodeGenerator.md b/llvm/docs/CodeGenerator.md
index 227b16fe9b01d..f0cff99a17d1f 100644
--- a/llvm/docs/CodeGenerator.md
+++ b/llvm/docs/CodeGenerator.md
@@ -481,7 +481,9 @@ register allocation, a `FREEZE` of a defined value becomes a `COPY`, and a
 `FREEZE` of an undefined value stays until code emission, where it emits no
 code.
 
-Selectors still lower `freeze` to a `COPY`. If the copied value is undefined,
+SelectionDAG selects `freeze` to `FREEZE`, except on targets that don't use
+physical registers for values, such as WebAssembly. FastISel and the GlobalISel
+selectors still lower it to a `COPY`. If the copied value is undefined,
 different uses can observe different values, which violates the IR semantics of
 `freeze`.
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 805c45b9a7928..8aa132f231a45 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -1063,19 +1063,30 @@ bool SDNodeKeyInfo::isEqual(const SDNodeKey &Key, const SDNode &N) {
 //                              SelectionDAG Class
 //===----------------------------------------------------------------------===//
 
+/// Return true if \p Opc with operand \p Op is a freeze of undef, before or
+/// after instruction selection.
+static bool isFreezeOfUndef(unsigned Opc, SDValue Op) {
+  if (Opc != ISD::FREEZE && Opc != ~(unsigned)TargetOpcode::FREEZE)
+    return false;
+  return Op.isUndef() || (Op.isMachineOpcode() &&
+                          Op.getMachineOpcode() == TargetOpcode::IMPLICIT_DEF);
+}
+
 /// doNotCSE - Return true if CSE should not be performed for this node.
 static bool doNotCSE(SDNode *N) {
   if (N->getValueType(0) == MVT::Glue)
     return true; // Never CSE anything that produces a glue result.
 
+  // Each freeze of undef is a distinct value.
+  if (N->getNumOperands() == 1 &&
+      isFreezeOfUndef(N->getOpcode(), N->getOperand(0)))
+    return true;
+
   switch (N->getOpcode()) {
   default: break;
   case ISD::HANDLENODE:
   case ISD::EH_LABEL:
     return true;   // Never CSE these nodes.
-  case ISD::FREEZE:
-    // Each freeze of undef is a distinct value.
-    return N->getOperand(0).isUndef();
   }
 
   // Check that remaining values produced are not flags.
@@ -7600,7 +7611,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
   SDVTList VTs = getVTList(VT);
   SDValue Ops[] = {N1};
   // Don't CSE glue producing nodes or freezes of undef.
-  if (VT != MVT::Glue && !(Opcode == ISD::FREEZE && N1.isUndef())) {
+  if (VT != MVT::Glue && !isFreezeOfUndef(Opcode, N1)) {
     SDNodeKey ID(Opcode, VTs, Ops);
     FoldingSetInsertToken InsertToken;
     if (SDNode *E = lookupNode(ID, DL, InsertToken)) {
@@ -12721,7 +12732,8 @@ SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
                                   SDVTList VTs, ArrayRef<SDValue> Ops) {
   // If an identical node already exists, use it.
   FoldingSetInsertToken InsertToken;
-  if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
+  if (VTs.VTs[VTs.NumVTs - 1] != MVT::Glue &&
+      !(Ops.size() == 1 && isFreezeOfUndef(Opc, Ops[0]))) {
     SDNodeKey ID(Opc, VTs, Ops);
     AddNodeIDCustom(ID.Tail, N, Opc);
     if (SDNode *ON = lookupNode(ID, SDLoc(N), InsertToken))
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index 6ed863774072e..46504e80a6a85 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -2575,11 +2575,11 @@ void SelectionDAGISel::Select_RELOC_NONE(SDNode *N) {
 }
 
 void SelectionDAGISel::Select_FREEZE(SDNode *N) {
-  // TODO: We don't have FREEZE pseudo-instruction in MachineInstr-level now.
-  // If FREEZE instruction is added later, the code below must be changed as
-  // well.
-  CurDAG->SelectNodeTo(N, TargetOpcode::COPY, N->getValueType(0),
-                       N->getOperand(0));
+  // Without physical registers, different uses of an undefined value can't
+  // observe different values.
+  unsigned Opc =
+      TM.usesPhysRegsForValues() ? TargetOpcode::FREEZE : TargetOpcode::COPY;
+  CurDAG->SelectNodeTo(N, Opc, N->getValueType(0), N->getOperand(0));
 }
 
 void SelectionDAGISel::Select_ARITH_FENCE(SDNode *N) {
diff --git a/llvm/test/CodeGen/AArch64/concat-vector-add-combine.ll b/llvm/test/CodeGen/AArch64/concat-vector-add-combine.ll
index 171f74149c905..e233238bee905 100644
--- a/llvm/test/CodeGen/AArch64/concat-vector-add-combine.ll
+++ b/llvm/test/CodeGen/AArch64/concat-vector-add-combine.ll
@@ -94,6 +94,7 @@ define i32 @combine_undef_add_8xi32(i32 %a, i32 %b, i32 %c, i32 %d) local_unname
 ; CHECK-LABEL: combine_undef_add_8xi32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmov s1, w0
+; CHECK-NEXT:    // freeze: $w8
 ; CHECK-NEXT:    dup v0.4s, w8
 ; CHECK-NEXT:    mov v1.s[1], w1
 ; CHECK-NEXT:    mov v1.s[2], w2
diff --git a/llvm/test/CodeGen/AArch64/fma-fneg-combine.ll b/llvm/test/CodeGen/AArch64/fma-fneg-combine.ll
index 936cc6a7d366f..58117d5bcad9c 100644
--- a/llvm/test/CodeGen/AArch64/fma-fneg-combine.ll
+++ b/llvm/test/CodeGen/AArch64/fma-fneg-combine.ll
@@ -15,7 +15,8 @@ define float @positive_case_fma(float %a0, float %a1, float %a2) {
 define float @negative_case_fma(float %a0, float %a1) {
 ; CHECK-LABEL: negative_case_fma:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    fnmadd s0, s0, s1, s0
+; CHECK-NEXT:    // freeze: $s2
+; CHECK-NEXT:    fnmadd s0, s0, s1, s2
 ; CHECK-NEXT:    ret
   %fma = call float @llvm.fma.f32(float %a0, float %a1, float poison)
   %freeze = freeze float %fma
@@ -37,7 +38,8 @@ define float @positive_case_fmad(float %a0, float %a1, float %a2) {
 define float @negative_case_fmad(float %a0, float %a1) {
 ; CHECK-LABEL: negative_case_fmad:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    fnmadd s0, s0, s1, s0
+; CHECK-NEXT:    // freeze: $s2
+; CHECK-NEXT:    fnmadd s0, s0, s1, s2
 ; CHECK-NEXT:    ret
   %fmad = call float @llvm.fmuladd.f32(float %a0, float %a1, float poison)
   %freeze = freeze float %fmad
diff --git a/llvm/test/CodeGen/AArch64/freeze-multi-use.ll b/llvm/test/CodeGen/AArch64/freeze-multi-use.ll
index 822ec50d0a571..0fbf4c61db5bf 100644
--- a/llvm/test/CodeGen/AArch64/freeze-multi-use.ll
+++ b/llvm/test/CodeGen/AArch64/freeze-multi-use.ll
@@ -7,8 +7,9 @@
 define i32 @ashr_mul(i32 %x) {
 ; CHECK-LABEL: ashr_mul:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    asr w8, w8, #3
-; CHECK-NEXT:    mul w0, w8, w8
+; CHECK-NEXT:    // freeze: $w0
+; CHECK-NEXT:    asr w8, w0, #3
+; CHECK-NEXT:    mul w0, w0, w8
 ; CHECK-NEXT:    ret
   %p = ashr i32 %x, 32
   %w = freeze i32 %p
@@ -21,11 +22,12 @@ define i32 @ashr_mul(i32 %x) {
 define i8 @fshr(i32 %x) {
 ; CHECK-LABEL: fshr:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mvn w8, w8
-; CHECK-NEXT:    mov w9, #2 // =0x2
-; CHECK-NEXT:    lsr w10, w0, w8
-; CHECK-NEXT:    lsl w8, w9, w8
-; CHECK-NEXT:    orr w0, w8, w10
+; CHECK-NEXT:    // freeze: $w8
+; CHECK-NEXT:    mvn w9, w8
+; CHECK-NEXT:    mov w10, #2 // =0x2
+; CHECK-NEXT:    lsr w8, w0, w8
+; CHECK-NEXT:    lsl w9, w10, w9
+; CHECK-NEXT:    orr w0, w9, w8
 ; CHECK-NEXT:    ret
   %w = freeze i32 poison
   %f = call i32 @llvm.fshr.i32(i32 1, i32 %x, i32 %w)
@@ -37,10 +39,11 @@ define i8 @fshr(i32 %x) {
 define i64 @select_zext(i64 %a, i64 %b) {
 ; CHECK-LABEL: select_zext:
 ; CHECK:       // %bb.0:
+; CHECK-NEXT:    // freeze: $w8
 ; CHECK-NEXT:    tst w8, #0x1
-; CHECK-NEXT:    csel x8, x0, x1, ne
-; CHECK-NEXT:    and x9, x8, #0x1
-; CHECK-NEXT:    add x0, x8, x9
+; CHECK-NEXT:    csel x9, x0, x1, ne
+; CHECK-NEXT:    and x8, x8, #0x1
+; CHECK-NEXT:    add x0, x9, x8
 ; CHECK-NEXT:    ret
   %w = freeze i1 poison
   %s = select i1 %w, i64 %a, i64 %b
@@ -53,9 +56,11 @@ define i64 @select_zext(i64 %a, i64 %b) {
 define i32 @float_bits() {
 ; CHECK-LABEL: float_bits:
 ; CHECK:       // %bb.0:
+; CHECK-NEXT:    // freeze: $s0
+; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    fcmp s0, s0
-; CHECK-NEXT:    and w8, w8, #0x7fffffff
 ; CHECK-NEXT:    mov w9, #2139095040 // =0x7f800000
+; CHECK-NEXT:    and w8, w8, #0x7fffffff
 ; CHECK-NEXT:    cset w10, vs
 ; CHECK-NEXT:    cmp w8, w9
 ; CHECK-NEXT:    cset w8, hi
@@ -75,9 +80,11 @@ define i32 @float_bits() {
 define i32 @vector_lane_and_whole(<4 x i32> %x) {
 ; CHECK-LABEL: vector_lane_and_whole:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    add v0.4s, v0.4s, v0.4s
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    sub w0, w8, w8
+; CHECK-NEXT:    // freeze: $q1
+; CHECK-NEXT:    add v0.4s, v1.4s, v0.4s
+; CHECK-NEXT:    fmov w8, s1
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    sub w0, w9, w8
 ; CHECK-NEXT:    ret
   %w = freeze <4 x i32> poison
   %e0 = extractelement <4 x i32> %w, i32 0
@@ -91,9 +98,12 @@ define i32 @vector_lane_and_whole(<4 x i32> %x) {
 define i32 @across_call() nounwind {
 ; CHECK-LABEL: across_call:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEXT:    // freeze: $w19
+; CHECK-NEXT:    mov w0, w19
 ; CHECK-NEXT:    bl use
-; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    mov w0, w19
+; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
   %w = freeze i32 poison
   call void @use(i32 %w)
@@ -104,9 +114,14 @@ define i32 @across_call() nounwind {
 define <4 x i32> @vec_across_call() nounwind {
 ; CHECK-LABEL: vec_across_call:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    sub sp, sp, #32
+; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Spill
+; CHECK-NEXT:    // freeze: $q0
+; CHECK-NEXT:    str q0, [sp] // 16-byte Spill
 ; CHECK-NEXT:    bl usev
-; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Reload
+; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Reload
+; CHECK-NEXT:    add sp, sp, #32
 ; CHECK-NEXT:    ret
   %w = freeze <4 x i32> poison
   call void @usev(<4 x i32> %w)
diff --git a/llvm/test/CodeGen/AArch64/freeze.ll b/llvm/test/CodeGen/AArch64/freeze.ll
index fff659e1f21b9..de3317d58a5f1 100644
--- a/llvm/test/CodeGen/AArch64/freeze.ll
+++ b/llvm/test/CodeGen/AArch64/freeze.ll
@@ -7,30 +7,48 @@
 %struct.T = type { i32, i32 }
 
 define i32 @freeze_int() {
-; CHECK-LABEL: freeze_int:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    mul w0, w8, w8
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: freeze_int:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    // freeze: $w0
+; CHECK-SD-NEXT:    mul w0, w0, w0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: freeze_int:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    mul w0, w8, w8
+; CHECK-GI-NEXT:    ret
   %y1 = freeze i32 undef
   %t1 = mul i32 %y1, %y1
   ret i32 %t1
 }
 
 define i5 @freeze_int2() {
-; CHECK-LABEL: freeze_int2:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    mul w0, w8, w8
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: freeze_int2:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    // freeze: $w0
+; CHECK-SD-NEXT:    mul w0, w0, w0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: freeze_int2:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    mul w0, w8, w8
+; CHECK-GI-NEXT:    ret
   %y1 = freeze i5 undef
   %t1 = mul i5 %y1, %y1
   ret i5 %t1
 }
 
 define float @freeze_float() {
-; CHECK-LABEL: freeze_float:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    fadd s0, s0, s0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: freeze_float:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    // freeze: $s0
+; CHECK-SD-NEXT:    fadd s0, s0, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: freeze_float:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    fadd s0, s0, s0
+; CHECK-GI-NEXT:    ret
   %y1 = freeze float undef
   %t1 = fadd float %y1, %y1
   ret float %t1
@@ -39,6 +57,7 @@ define float @freeze_float() {
 define <2 x i8> @freeze_v2i8() {
 ; CHECK-LABEL: freeze_v2i8:
 ; CHECK:       // %bb.0:
+; CHECK-NEXT:    // freeze: $d0
 ; CHECK-NEXT:    add v0.2s, v0.2s, v0.2s
 ; CHECK-NEXT:    ret
   %y1 = freeze <2 x i8> undef
@@ -49,6 +68,7 @@ define <2 x i8> @freeze_v2i8() {
 define <3 x i8> @freeze_v3i8() {
 ; CHECK-SD-LABEL: freeze_v3i8:
 ; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    // freeze: $d0
 ; CHECK-SD-NEXT:    add v0.4h, v0.4h, v0.4h
 ; CHECK-SD-NEXT:    umov w0, v0.h[0]
 ; CHECK-SD-NEXT:    umov w1, v0.h[1]
@@ -76,6 +96,7 @@ define <3 x i8> @freeze_v3i8() {
 define <4 x i8> @freeze_v4i8() {
 ; CHECK-SD-LABEL: freeze_v4i8:
 ; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    // freeze: $d0
 ; CHECK-SD-NEXT:    add v0.4h, v0.4h, v0.4h
 ; CHECK-SD-NEXT:    ret
 ;
@@ -98,20 +119,32 @@ define <4 x i8> @freeze_v4i8() {
 }
 
 define <8 x i8> @freeze_v8i8() {
-; CHECK-LABEL: freeze_v8i8:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    add v0.8b, v0.8b, v0.8b
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: freeze_v8i8:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    // freeze: $d0
+; CHECK-SD-NEXT:    add v0.8b, v0.8b, v0.8b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: freeze_v8i8:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    add v0.8b, v0.8b, v0.8b
+; CHECK-GI-NEXT:    ret
   %y1 = freeze <8 x i8> undef
   %t1 = add <8 x i8> %y1, %y1
   ret <8 x i8> %t1
 }
 
 define <16 x i8> @freeze_v16i8() {
-; CHECK-LABEL: freeze_v16i8:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    add v0.16b, v0.16b, v0.16b
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: freeze_v16i8:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    // freeze: $q0
+; CHECK-SD-NEXT:    add v0.16b, v0.16b, v0.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: freeze_v16i8:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    add v0.16b, v0.16b, v0.16b
+; CHECK-GI-NEXT:    ret
   %y1 = freeze <16 x i8> undef
   %t1 = add <16 x i8> %y1, %y1
   ret <16 x i8> %t1
@@ -120,8 +153,10 @@ define <16 x i8> @freeze_v16i8() {
 define <32 x i8> @freeze_v32i8() {
 ; CHECK-SD-LABEL: freeze_v32i8:
 ; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    // freeze: $q1
+; CHECK-SD-NEXT:    // freeze: $q0
 ; CHECK-SD-NEXT:    add v0.16b, v0.16b, v0.16b
-; CHECK-SD-NEXT:    add v1.16b, v0.16b, v0.16b
+; CHECK-SD-NEXT:    add v1.16b, v1.16b, v1.16b
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: freeze_v32i8:
@@ -137,6 +172,7 @@ define <32 x i8> @freeze_v32i8() {
 define <2 x i16> @freeze_v2i16() {
 ; CHECK-SD-LABEL: freeze_v2i16:
 ; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    // freeze: $d0
 ; CHECK-SD-NEXT:    add v0.2s, v0.2s, v0.2s
 ; CHECK-SD-NEXT:    ret
 ;
@@ -153,30 +189,48 @@ define <2 x i16> @freeze_v2i16() {
 }
 
 define <3 x i16> @freeze_v3i16() {
-; CHECK-LABEL: freeze_v3i16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    add v0.4h, v0.4h, v0.4h
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: freeze_v3i16:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    // freeze: $d0
+; CHECK-SD-NEXT:    add v0.4h, v0.4h, v0.4h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: freeze_v3i16:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    add v0.4h, v0.4h, v0.4h
+; CHECK-GI-NEXT:    ret
   %y1 = freeze <3 x i16> undef
   %t1 = add <3 x i16> %y1, %y1
   ret <3 x i16> %t1
 }
 
 define <4 x i16> @freeze_v4i16() {
-; CHECK-LABEL: freeze_v4i16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    add v0.4h, v0.4h, v0.4h
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: freeze_v4i16:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    // freeze: $d0
+; CHECK-SD-NEXT:    add v0.4h, v0.4h, v0.4h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: freeze_v4i16:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    add v0.4h, v0.4h, v0.4h
+; CHECK-GI-NEXT:    ret
   %y1 = freeze <4 x i16> undef
   %t1 = add <4 x i16> %y1, %y1
   ret <4 x i16> %t1
 }
 
 define <8 x i16> @freeze_v8i16() {
-; CHECK-LABEL: freeze_v8i16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    add v0.8h, v0.8h, v0.8h
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: freeze_v8i16:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    // freeze: $q0
+; CHECK-SD-NEXT:    add v0.8h, v0.8h, v0.8h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: freeze_v8i16:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    add v0.8h, v0.8h, v0.8h
+; CHECK-GI-NEXT:    ret
   %y1 = freeze <8 x i16> undef
   %t1 = add <8 x i16> %y1, %y1
   ret <8 x i16> %t1
@@ -185,8 +239,10 @@ define <8 x i16> @freeze_v8i16() {
 define <16 x i16> @freeze_v16i16() {
 ; CHECK-SD-LABEL: freeze_v16i16:
 ; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    // freeze: $q1
+; CHECK-SD-NEXT:    // freeze: $q0
 ; CHECK-SD-NEXT:    add v0.8h, v0.8h, v0.8h
-; CHECK-SD-NEXT:    add v1.8h, v0.8h, v0.8h
+; CHECK-SD-NEXT:    add v1.8h, v1.8h, v1.8h
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: freeze_v16i16:
@@ -200,30 +256,48 @@ define <16 x i16> @freeze_v16i16() {
 }
 
 define <2 x i32> @freeze_v2i32() {
-; CHECK-LABEL: freeze_v2i32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    add v0.2s, v0.2s, v0.2s
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: freeze_v2i32:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    // freeze: $d0
+; CHECK-SD-NEXT:    add v0.2s, v0.2s, v0.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: freeze_v2i32:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    add v0.2s, v0.2s, v0.2s
+; CHECK-GI-NEXT:    ret
   %y1 = freeze <2 x i32> undef
   %t1 = add <2 x i32> %y1, %y1
   ret <2 x i32> %t1
 }
 
 define <3 x i32> @freeze_v3i32() {
-; CHECK-LABEL: freeze_v3i32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    add v0.4s, v0.4s, v0.4s
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: freeze_v3i32:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    // freeze: $q0
+; CHECK-SD-NEXT:    add v0.4s, v0.4s, v0.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: freeze_v3i32:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    add v0.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT:    ret
   %y1 = freeze <3 x i32> undef
   %t1 = add <3 x i32> %y1, %y1
   ret <3 x i32> %t1
 }
 
 define <4 x i32> @freeze_v4i32() {
-; CHECK-LABEL: freeze_v4i32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    add v0.4s, v0.4s, v0.4s
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: freeze_v4i32:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    // freeze: $q0
+; CHECK-SD-NEXT:    add v0.4s, v0.4s, v0.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: freeze_v4i32:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    add v0.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT:    ret
   %y1 = freeze <4 x i32> undef
   %t1 = add <4 x i32> %y1, %y1
   ret <4 x i32> %t1
@@ -232,8 +306,10 @@ define <4 x i32> @freeze_v4i32() {
 define <8 x i32> @freeze_v8i32() {
 ; CHECK-SD-LABEL: freeze_v8i32:
 ; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    // freeze: $q1
+; CHECK-SD-NEXT:    // freeze: $q0
 ; CHECK-SD-NEXT:    add v0.4s, v0.4s, v0.4s
-; CHECK-SD-NEXT:    add v1.4s, v0.4s, v0.4s
+; CHECK-SD-NEXT:    add v1.4s, v1.4s, v1.4s
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: freeze_v8i32:
@@ -247,10 +323,16 @@ define <8 x i32> @freeze_v8i32() {
 }
 
 define <2 x i64> @freeze_v2i64() {
-; CHECK-LABEL: freeze_v2i64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    add v0.2d, v0.2d, v0.2d
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: freeze_v2i64:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    // freeze: $q0
+; CHECK-SD-NEXT:    add v0.2d, v0.2d, v0.2d
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: freeze_v2i64:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    add v0.2d, v0.2d, v0.2d
+; CHECK-GI-NEXT:    ret
   %y1 = freeze <2 x i64> undef
   %t1 = add <2 x i64> %y1, %y1
   ret <2 x i64> %t1
@@ -259,10 +341,12 @@ define <2 x i64> @freeze_v2i64() {
 define <3 x i64> @freeze_v3i64() {
 ; CHECK-SD-LABEL: freeze_v3i64:
 ; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    // freeze: $q0
 ; CHECK-SD-NEXT:    add v0.2d, v0.2d, v0.2d
 ; CHECK-SD-NEXT:    mov d1, v0.d[1]
-; CHECK-SD-NEXT:    add d2, d0, d0
+; CHECK-SD-NEXT:    // freeze: $q2
 ; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    add d2, d2, d2
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: freeze_v3i64:
@@ -281,8 +365,10 @@ define <3 x i64> @freeze_v3i64() {
 define <4 x i64> @freeze_v4i64() {
 ; CHECK-SD-LABEL: freeze_v4i64:
 ; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    // freeze: $q1
+; CHECK-SD-NEXT:    // freeze: $q0
 ; CHECK-SD-NEXT:    add v0.2d, v0.2d, v0.2d
-; CHECK-SD-NEXT:    add v1.2d, v0.2d, v0.2d
+; CHECK-SD-NEXT:    add v1.2d, v1.2d, v1.2d
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: freeze_v4i64:
@@ -300,7 +386,8 @@ define <2 x ptr> @freeze_v2p0() {
 ; CHECK-SD:       // %bb.0:
 ; CHECK-SD-NEXT:    mov w8, #4 // =0x4
 ; CHECK-SD-NEXT:    dup v0.2d, x8
-; CHECK-SD-NEXT:    add v0.2d, v0.2d, v0.2d
+; CHECK-SD-NEXT:    // freeze: $q1
+; CHECK-SD-NEXT:    add v0.2d, v1.2d, v0.2d
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: freeze_v2p0:
@@ -318,10 +405,12 @@ define <3 x ptr> @freeze_v3p0() {
 ; CHECK-SD-LABEL: freeze_v3p0:
 ; CHECK-SD:       // %bb.0:
 ; CHECK-SD-NEXT:    mov w8, #4 // =0x4
+; CHECK-SD-NEXT:    // freeze: $q0
 ; CHECK-SD-NEXT:    dup v2.2d, x8
 ; CHECK-SD-NEXT:    add v0.2d, v0.2d, v2.2d
 ; CHECK-SD-NEXT:    mov d1, v0.d[1]
-; CHECK-SD-NEXT:    add d2, d0, d2
+; CHECK-SD-NEXT:    // freeze: $q3
+; CHECK-SD-NEXT:    add d2, d3, d2
 ; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-SD-NEXT:    ret
 ;
@@ -343,9 +432,11 @@ define <4 x ptr> @freeze_v4p0() {
 ; CHECK-SD-LABEL: freeze_v4p0:
 ; CHECK-SD:       // %bb.0:
 ; CHECK-SD-NEXT:    mov w8, #4 // =0x4
+; CHECK-SD-NEXT:    // freeze: $q2
+; CHECK-SD-NEXT:    // freeze: $q0
 ; CHECK-SD-NEXT:    dup v1.2d, x8
 ; CHECK-SD-NEXT:    add v0.2d, v0.2d, v1.2d
-; CHECK-SD-NEXT:    add v1.2d, v0.2d, v1.2d
+; CHECK-SD-NEXT:    add v1.2d, v2.2d, v1.2d
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: freeze_v4p0:
@@ -361,20 +452,33 @@ define <4 x ptr> @freeze_v4p0() {
 }
 
 define ptr @freeze_ptr() {
-; CHECK-LABEL: freeze_ptr:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    add x0, x8, #4
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: freeze_ptr:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    // freeze: $x0
+; CHECK-SD-NEXT:    add x0, x0, #4
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: freeze_ptr:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    add x0, x8, #4
+; CHECK-GI-NEXT:    ret
   %y1 = freeze ptr undef
   %t1 = getelementptr i8, ptr %y1, i64 4
   ret ptr %t1
 }
 
 define i32 @freeze_struct() {
-; CHECK-LABEL: freeze_struct:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    add w0, w8, w8
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: freeze_struct:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    // freeze: $w0
+; CHECK-SD-NEXT:    // freeze: $w8
+; CHECK-SD-NEXT:    add w0, w8, w0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: freeze_struct:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    add w0, w8, w8
+; CHECK-GI-NEXT:    ret
   %y1 = freeze %struct.T undef
   %v1 = extractvalue %struct.T %y1, 0
   %v2 = extractvalue %struct.T %y1, 1
@@ -383,10 +487,17 @@ define i32 @freeze_struct() {
 }
 
 define i32 @freeze_anonstruct() {
-; CHECK-LABEL: freeze_anonstruct:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    add w0, w8, w8
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: freeze_anonstruct:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    // freeze: $w0
+; CHECK-SD-NEXT:    // freeze: $w8
+; CHECK-SD-NEXT:    add w0, w8, w0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: freeze_anonstruct:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    add w0, w8, w8
+; CHECK-GI-NEXT:    ret
   %y1 = freeze {i32, i32} undef
   %v1 = extractvalue {i32, i32} %y1, 0
   %v2 = extractvalue {i32, i32} %y1, 1
@@ -395,10 +506,17 @@ define i32 @freeze_anonstruct() {
 }
 
 define i32 @freeze_anonstruct2() {
-; CHECK-LABEL: freeze_anonstruct2:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    add w0, w8, w8, uxth
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: freeze_anonstruct2:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    // freeze: $w0
+; CHECK-SD-NEXT:    // freeze: $w8
+; CHECK-SD-NEXT:    add w0, w8, w0, uxth
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: freeze_anonstruct2:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    add w0, w8, w8, uxth
+; CHECK-GI-NEXT:    ret
   %y1 = freeze {i32, i16} undef
   %v1 = extractvalue {i32, i16} %y1, 0
   %v2 = extractvalue {i32, i16} %y1, 1
@@ -408,10 +526,17 @@ define i32 @freeze_anonstruct2() {
 }
 
 define i64 @freeze_array() {
-; CHECK-LABEL: freeze_array:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    add x0, x8, x8
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: freeze_array:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    // freeze: $x0
+; CHECK-SD-NEXT:    // freeze: $x8
+; CHECK-SD-NEXT:    add x0, x8, x0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: freeze_array:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    add x0, x8, x8
+; CHECK-GI-NEXT:    ret
   %y1 = freeze [2 x i64] undef
   %v1 = extractvalue [2 x i64] %y1, 0
   %v2 = extractvalue [2 x i64] %y1, 1
@@ -592,6 +717,8 @@ define i32 @freeze_ucmp(i32 %a0) nounwind {
 define i128 @freeze_i128() {
 ; CHECK-SD-LABEL: freeze_i128:
 ; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    // freeze: $x0
+; CHECK-SD-NEXT:    // freeze: $x1
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: freeze_i128:
diff --git a/llvm/test/CodeGen/AArch64/neon-shuffle-vector-tbl.ll b/llvm/test/CodeGen/AArch64/neon-shuffle-vector-tbl.ll
index 9315a5b831aa1..d35e4e7f2353c 100644
--- a/llvm/test/CodeGen/AArch64/neon-shuffle-vector-tbl.ll
+++ b/llvm/test/CodeGen/AArch64/neon-shuffle-vector-tbl.ll
@@ -565,68 +565,68 @@ define <16 x i8> @no_shuffle_with_two_mask_sources(<16 x i8> %src, <16 x i8> %ma
 ; CHECK-SD:       // %bb.0:
 ; CHECK-SD-NEXT:    sub sp, sp, #16
 ; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-SD-NEXT:    umov.b w8, v1[0]
-; CHECK-SD-NEXT:    umov.b w10, v1[8]
-; CHECK-SD-NEXT:    mov x11, sp
-; CHECK-SD-NEXT:    umov.b w9, v2[1]
+; CHECK-SD-NEXT:    umov.b w9, v1[0]
+; CHECK-SD-NEXT:    umov.b w11, v1[8]
+; CHECK-SD-NEXT:    mov x13, sp
+; CHECK-SD-NEXT:    umov.b w10, v2[1]
 ; CHECK-SD-NEXT:    umov.b w12, v2[9]
 ; CHECK-SD-NEXT:    mov x14, sp
-; CHECK-SD-NEXT:    mov x13, sp
+; CHECK-SD-NEXT:    mov x8, sp
 ; CHECK-SD-NEXT:    str q0, [sp]
 ; CHECK-SD-NEXT:    umov.b w15, v2[11]
-; CHECK-SD-NEXT:    bfxil x11, x8, #0, #4
-; CHECK-SD-NEXT:    bfxil x14, x10, #0, #4
-; CHECK-SD-NEXT:    umov.b w8, v1[2]
 ; CHECK-SD-NEXT:    bfxil x13, x9, #0, #4
-; CHECK-SD-NEXT:    mov x9, sp
-; CHECK-SD-NEXT:    umov.b w10, v1[10]
-; CHECK-SD-NEXT:    bfxil x9, x12, #0, #4
-; CHECK-SD-NEXT:    ldr b0, [x11]
+; CHECK-SD-NEXT:    bfxil x14, x11, #0, #4
+; CHECK-SD-NEXT:    umov.b w9, v1[2]
+; CHECK-SD-NEXT:    bfxil x8, x10, #0, #4
+; CHECK-SD-NEXT:    mov x10, sp
+; CHECK-SD-NEXT:    umov.b w11, v1[10]
+; CHECK-SD-NEXT:    bfxil x10, x12, #0, #4
+; CHECK-SD-NEXT:    ldr b0, [x13]
 ; CHECK-SD-NEXT:    ldr b3, [x14]
-; CHECK-SD-NEXT:    mov x11, sp
-; CHECK-SD-NEXT:    umov.b w12, v2[3]
+; CHECK-SD-NEXT:    mov x12, sp
+; CHECK-SD-NEXT:    umov.b w13, v2[3]
 ; CHECK-SD-NEXT:    mov x14, sp
-; CHECK-SD-NEXT:    ld1.b { v0 }[1], [x13]
-; CHECK-SD-NEXT:    ld1.b { v3 }[1], [x9]
-; CHECK-SD-NEXT:    bfxil x11, x8, #0, #4
-; CHECK-SD-NEXT:    bfxil x14, x10, #0, #4
+; CHECK-SD-NEXT:    ld1.b { v0 }[1], [x8]
+; CHECK-SD-NEXT:    ld1.b { v3 }[1], [x10]
+; CHECK-SD-NEXT:    bfxil x12, x9, #0, #4
+; CHECK-SD-NEXT:    bfxil x14, x11, #0, #4
 ; CHECK-SD-NEXT:    mov x8, sp
 ; CHECK-SD-NEXT:    umov.b w9, v1[4]
 ; CHECK-SD-NEXT:    mov x10, sp
-; CHECK-SD-NEXT:    umov.b w13, v1[12]
-; CHECK-SD-NEXT:    ld1.b { v0 }[2], [x11]
+; CHECK-SD-NEXT:    umov.b w11, v1[12]
+; CHECK-SD-NEXT:    ld1.b { v0 }[2], [x12]
 ; CHECK-SD-NEXT:    ld1.b { v3 }[2], [x14]
-; CHECK-SD-NEXT:    bfxil x8, x12, #0, #4
+; CHECK-SD-NEXT:    bfxil x8, x13, #0, #4
 ; CHECK-SD-NEXT:    bfxil x10, x15, #0, #4
-; CHECK-SD-NEXT:    mov x11, sp
-; CHECK-SD-NEXT:    umov.b w12, v2[5]
+; CHECK-SD-NEXT:    mov x12, sp
+; CHECK-SD-NEXT:    umov.b w13, v2[5]
 ; CHECK-SD-NEXT:    mov x14, sp
 ; CHECK-SD-NEXT:    umov.b w15, v2[13]
-; CHECK-SD-NEXT:    bfxil x11, x9, #0, #4
+; CHECK-SD-NEXT:    bfxil x12, x9, #0, #4
 ; CHECK-SD-NEXT:    ld1.b { v0 }[3], [x8]
 ; CHECK-SD-NEXT:    ld1.b { v3 }[3], [x10]
-; CHECK-SD-NEXT:    bfxil x14, x13, #0, #4
+; CHECK-SD-NEXT:    bfxil x14, x11, #0, #4
 ; CHECK-SD-NEXT:    umov.b w8, v1[6]
 ; CHECK-SD-NEXT:    umov.b w9, v1[14]
 ; CHECK-SD-NEXT:    mov x10, sp
-; CHECK-SD-NEXT:    mov x13, sp
-; CHECK-SD-NEXT:    bfxil x10, x12, #0, #4
-; CHECK-SD-NEXT:    umov.b w12, v2[7]
-; CHECK-SD-NEXT:    ld1.b { v0 }[4], [x11]
-; CHECK-SD-NEXT:    ld1.b { v3 }[4], [x14]
-; CHECK-SD-NEXT:    bfxil x13, x15, #0, #4
 ; CHECK-SD-NEXT:    mov x11, sp
+; CHECK-SD-NEXT:    bfxil x10, x13, #0, #4
+; CHECK-SD-NEXT:    umov.b w13, v2[7]
+; CHECK-SD-NEXT:    ld1.b { v0 }[4], [x12]
+; CHECK-SD-NEXT:    ld1.b { v3 }[4], [x14]
+; CHECK-SD-NEXT:    bfxil x11, x15, #0, #4
+; CHECK-SD-NEXT:    mov x12, sp
 ; CHECK-SD-NEXT:    mov x14, sp
 ; CHECK-SD-NEXT:    umov.b w15, v2[15]
-; CHECK-SD-NEXT:    bfxil x11, x8, #0, #4
+; CHECK-SD-NEXT:    bfxil x12, x8, #0, #4
 ; CHECK-SD-NEXT:    bfxil x14, x9, #0, #4
 ; CHECK-SD-NEXT:    mov x8, sp
 ; CHECK-SD-NEXT:    ld1.b { v0 }[5], [x10]
-; CHECK-SD-NEXT:    ld1.b { v3 }[5], [x13]
+; CHECK-SD-NEXT:    ld1.b { v3 }[5], [x11]
 ; CHECK-SD-NEXT:    mov x9, sp
-; CHECK-SD-NEXT:    bfxil x9, x12, #0, #4
+; CHECK-SD-NEXT:    bfxil x9, x13, #0, #4
 ; CHECK-SD-NEXT:    bfxil x8, x15, #0, #4
-; CHECK-SD-NEXT:    ld1.b { v0 }[6], [x11]
+; CHECK-SD-NEXT:    ld1.b { v0 }[6], [x12]
 ; CHECK-SD-NEXT:    ld1.b { v3 }[6], [x14]
 ; CHECK-SD-NEXT:    ld1.b { v0 }[7], [x9]
 ; CHECK-SD-NEXT:    ld1.b { v3 }[7], [x8]
diff --git a/llvm/test/CodeGen/AArch64/vector-compress.ll b/llvm/test/CodeGen/AArch64/vector-compress.ll
index 56f6a1cd341d2..7a4d14b4ab6bd 100644
--- a/llvm/test/CodeGen/AArch64/vector-compress.ll
+++ b/llvm/test/CodeGen/AArch64/vector-compress.ll
@@ -357,10 +357,10 @@ define <4 x i8> @test_compress_small(<4 x i8> %vec, <4 x i1> %mask) {
 ; CHECK-NEXT:    umov.h w9, v1[0]
 ; CHECK-NEXT:    umov.h w10, v1[1]
 ; CHECK-NEXT:    umov.h w11, v1[2]
-; CHECK-NEXT:    bfi x8, x9, #1, #1
 ; CHECK-NEXT:    and x10, x10, #0x1
-; CHECK-NEXT:    and x9, x9, #0x1
-; CHECK-NEXT:    add x9, x9, x10
+; CHECK-NEXT:    and x12, x9, #0x1
+; CHECK-NEXT:    bfi x8, x9, #1, #1
+; CHECK-NEXT:    add x9, x12, x10
 ; CHECK-NEXT:    add x10, sp, #8
 ; CHECK-NEXT:    and x11, x11, #0x1
 ; CHECK-NEXT:    bfi x10, x9, #1, #2
@@ -390,10 +390,10 @@ define <4 x i4> @test_compress_illegal_element_type(<4 x i4> %vec, <4 x i1> %mas
 ; CHECK-NEXT:    umov.h w9, v1[0]
 ; CHECK-NEXT:    umov.h w10, v1[1]
 ; CHECK-NEXT:    umov.h w11, v1[2]
-; CHECK-NEXT:    bfi x8, x9, #1, #1
 ; CHECK-NEXT:    and x10, x10, #0x1
-; CHECK-NEXT:    and x9, x9, #0x1
-; CHECK-NEXT:    add x9, x9, x10
+; CHECK-NEXT:    and x12, x9, #0x1
+; CHECK-NEXT:    bfi x8, x9, #1, #1
+; CHECK-NEXT:    add x9, x12, x10
 ; CHECK-NEXT:    add x10, sp, #8
 ; CHECK-NEXT:    and x11, x11, #0x1
 ; CHECK-NEXT:    bfi x10, x9, #1, #2
diff --git a/llvm/test/CodeGen/AArch64/vector-extract-last-active.ll b/llvm/test/CodeGen/AArch64/vector-extract-last-active.ll
index 21f306bb129d7..6a3012351d561 100644
--- a/llvm/test/CodeGen/AArch64/vector-extract-last-active.ll
+++ b/llvm/test/CodeGen/AArch64/vector-extract-last-active.ll
@@ -11,16 +11,16 @@ define i8 @extract_last_i8(<16 x i8> %data, <16 x i8> %mask, i8 %passthru) {
 ; NEON-FIXED-NEXT:    adrp x8, .LCPI0_0
 ; NEON-FIXED-NEXT:    cmtst v1.16b, v1.16b, v1.16b
 ; NEON-FIXED-NEXT:    ldr q3, [x8, :lo12:.LCPI0_0]
-; NEON-FIXED-NEXT:    mov x9, sp
+; NEON-FIXED-NEXT:    mov x8, sp
 ; NEON-FIXED-NEXT:    str q0, [sp]
 ; NEON-FIXED-NEXT:    bic v2.16b, v3.16b, v2.16b
 ; NEON-FIXED-NEXT:    umaxv b1, v1.16b
 ; NEON-FIXED-NEXT:    umaxv b2, v2.16b
-; NEON-FIXED-NEXT:    fmov w8, s2
-; NEON-FIXED-NEXT:    bfxil x9, x8, #0, #4
-; NEON-FIXED-NEXT:    ldrb w8, [x9]
 ; NEON-FIXED-NEXT:    fmov w9, s1
+; NEON-FIXED-NEXT:    fmov w1, s2
 ; NEON-FIXED-NEXT:    tst w9, #0x1
+; NEON-FIXED-NEXT:    bfxil x8, x1, #0, #4
+; NEON-FIXED-NEXT:    ldrb w8, [x8]
 ; NEON-FIXED-NEXT:    csel w0, w8, w0, ne
 ; NEON-FIXED-NEXT:    add sp, sp, #16
 ; NEON-FIXED-NEXT:    ret
@@ -49,18 +49,18 @@ define i16 @extract_last_i16(<8 x i16> %data, <8 x i16> %mask, i16 %passthru) {
 ; NEON-FIXED-NEXT:    .cfi_def_cfa_offset 16
 ; NEON-FIXED-NEXT:    cmtst v1.8h, v1.8h, v1.8h
 ; NEON-FIXED-NEXT:    adrp x8, .LCPI1_0
-; NEON-FIXED-NEXT:    mov x9, sp
-; NEON-FIXED-NEXT:    ldr d3, [x8, :lo12:.LCPI1_0]
 ; NEON-FIXED-NEXT:    str q0, [sp]
+; NEON-FIXED-NEXT:    ldr d3, [x8, :lo12:.LCPI1_0]
+; NEON-FIXED-NEXT:    mov x8, sp
 ; NEON-FIXED-NEXT:    xtn v2.8b, v1.8h
 ; NEON-FIXED-NEXT:    umaxv h1, v1.8h
 ; NEON-FIXED-NEXT:    and v2.8b, v2.8b, v3.8b
-; NEON-FIXED-NEXT:    umaxv b2, v2.8b
-; NEON-FIXED-NEXT:    fmov w8, s2
-; NEON-FIXED-NEXT:    bfi x9, x8, #1, #3
-; NEON-FIXED-NEXT:    ldrh w8, [x9]
 ; NEON-FIXED-NEXT:    fmov w9, s1
+; NEON-FIXED-NEXT:    umaxv b2, v2.8b
 ; NEON-FIXED-NEXT:    tst w9, #0x1
+; NEON-FIXED-NEXT:    fmov w1, s2
+; NEON-FIXED-NEXT:    bfi x8, x1, #1, #3
+; NEON-FIXED-NEXT:    ldrh w8, [x8]
 ; NEON-FIXED-NEXT:    csel w0, w8, w0, ne
 ; NEON-FIXED-NEXT:    add sp, sp, #16
 ; NEON-FIXED-NEXT:    ret
@@ -89,18 +89,18 @@ define i32 @extract_last_i32(<4 x i32> %data, <4 x i32> %mask, i32 %passthru) {
 ; NEON-FIXED-NEXT:    .cfi_def_cfa_offset 16
 ; NEON-FIXED-NEXT:    cmtst v1.4s, v1.4s, v1.4s
 ; NEON-FIXED-NEXT:    adrp x8, .LCPI2_0
-; NEON-FIXED-NEXT:    mov x9, sp
-; NEON-FIXED-NEXT:    ldr d3, [x8, :lo12:.LCPI2_0]
 ; NEON-FIXED-NEXT:    str q0, [sp]
+; NEON-FIXED-NEXT:    ldr d3, [x8, :lo12:.LCPI2_0]
+; NEON-FIXED-NEXT:    mov x8, sp
 ; NEON-FIXED-NEXT:    xtn v2.4h, v1.4s
 ; NEON-FIXED-NEXT:    umaxv s1, v1.4s
 ; NEON-FIXED-NEXT:    and v2.8b, v2.8b, v3.8b
-; NEON-FIXED-NEXT:    umaxv h2, v2.4h
-; NEON-FIXED-NEXT:    fmov w8, s2
-; NEON-FIXED-NEXT:    bfi x9, x8, #2, #2
-; NEON-FIXED-NEXT:    ldr w8, [x9]
 ; NEON-FIXED-NEXT:    fmov w9, s1
+; NEON-FIXED-NEXT:    umaxv h2, v2.4h
 ; NEON-FIXED-NEXT:    tst w9, #0x1
+; NEON-FIXED-NEXT:    fmov w1, s2
+; NEON-FIXED-NEXT:    bfi x8, x1, #2, #2
+; NEON-FIXED-NEXT:    ldr w8, [x8]
 ; NEON-FIXED-NEXT:    csel w0, w8, w0, ne
 ; NEON-FIXED-NEXT:    add sp, sp, #16
 ; NEON-FIXED-NEXT:    ret
@@ -171,18 +171,18 @@ define half @extract_last_half(<8 x half> %data, <8 x i16> %mask, half %passthru
 ; NEON-FIXED-NEXT:    .cfi_def_cfa_offset 16
 ; NEON-FIXED-NEXT:    cmtst v1.8h, v1.8h, v1.8h
 ; NEON-FIXED-NEXT:    adrp x8, .LCPI4_0
-; NEON-FIXED-NEXT:    mov x9, sp
-; NEON-FIXED-NEXT:    ldr d4, [x8, :lo12:.LCPI4_0]
 ; NEON-FIXED-NEXT:    str q0, [sp]
+; NEON-FIXED-NEXT:    ldr d4, [x8, :lo12:.LCPI4_0]
+; NEON-FIXED-NEXT:    mov x8, sp
 ; NEON-FIXED-NEXT:    // kill: def $h2 killed $h2 def $s2
 ; NEON-FIXED-NEXT:    xtn v3.8b, v1.8h
 ; NEON-FIXED-NEXT:    umaxv h1, v1.8h
 ; NEON-FIXED-NEXT:    and v3.8b, v3.8b, v4.8b
 ; NEON-FIXED-NEXT:    umaxv b3, v3.8b
-; NEON-FIXED-NEXT:    fmov w8, s3
-; NEON-FIXED-NEXT:    bfi x9, x8, #1, #3
+; NEON-FIXED-NEXT:    fmov w0, s3
+; NEON-FIXED-NEXT:    bfi x8, x0, #1, #3
+; NEON-FIXED-NEXT:    ldr h0, [x8]
 ; NEON-FIXED-NEXT:    fmov w8, s1
-; NEON-FIXED-NEXT:    ldr h0, [x9]
 ; NEON-FIXED-NEXT:    tst w8, #0x1
 ; NEON-FIXED-NEXT:    fcsel s0, s0, s2, ne
 ; NEON-FIXED-NEXT:    // kill: def $h0 killed $h0 killed $s0
@@ -214,18 +214,18 @@ define bfloat @extract_last_bfloat(<8 x bfloat> %data, <8 x i16> %mask, bfloat %
 ; NEON-FIXED-NEXT:    .cfi_def_cfa_offset 16
 ; NEON-FIXED-NEXT:    cmtst v1.8h, v1.8h, v1.8h
 ; NEON-FIXED-NEXT:    adrp x8, .LCPI5_0
-; NEON-FIXED-NEXT:    mov x9, sp
-; NEON-FIXED-NEXT:    ldr d4, [x8, :lo12:.LCPI5_0]
 ; NEON-FIXED-NEXT:    str q0, [sp]
+; NEON-FIXED-NEXT:    ldr d4, [x8, :lo12:.LCPI5_0]
+; NEON-FIXED-NEXT:    mov x8, sp
 ; NEON-FIXED-NEXT:    // kill: def $h2 killed $h2 def $s2
 ; NEON-FIXED-NEXT:    xtn v3.8b, v1.8h
 ; NEON-FIXED-NEXT:    umaxv h1, v1.8h
 ; NEON-FIXED-NEXT:    and v3.8b, v3.8b, v4.8b
 ; NEON-FIXED-NEXT:    umaxv b3, v3.8b
-; NEON-FIXED-NEXT:    fmov w8, s3
-; NEON-FIXED-NEXT:    bfi x9, x8, #1, #3
+; NEON-FIXED-NEXT:    fmov w0, s3
+; NEON-FIXED-NEXT:    bfi x8, x0, #1, #3
+; NEON-FIXED-NEXT:    ldr h0, [x8]
 ; NEON-FIXED-NEXT:    fmov w8, s1
-; NEON-FIXED-NEXT:    ldr h0, [x9]
 ; NEON-FIXED-NEXT:    tst w8, #0x1
 ; NEON-FIXED-NEXT:    fcsel s0, s0, s2, ne
 ; NEON-FIXED-NEXT:    // kill: def $h0 killed $h0 killed $s0
@@ -257,17 +257,17 @@ define float @extract_last_float(<4 x float> %data, <4 x i32> %mask, float %pass
 ; NEON-FIXED-NEXT:    .cfi_def_cfa_offset 16
 ; NEON-FIXED-NEXT:    cmtst v1.4s, v1.4s, v1.4s
 ; NEON-FIXED-NEXT:    adrp x8, .LCPI6_0
-; NEON-FIXED-NEXT:    mov x9, sp
-; NEON-FIXED-NEXT:    ldr d4, [x8, :lo12:.LCPI6_0]
 ; NEON-FIXED-NEXT:    str q0, [sp]
+; NEON-FIXED-NEXT:    ldr d4, [x8, :lo12:.LCPI6_0]
+; NEON-FIXED-NEXT:    mov x8, sp
 ; NEON-FIXED-NEXT:    xtn v3.4h, v1.4s
 ; NEON-FIXED-NEXT:    umaxv s1, v1.4s
 ; NEON-FIXED-NEXT:    and v3.8b, v3.8b, v4.8b
 ; NEON-FIXED-NEXT:    umaxv h3, v3.4h
-; NEON-FIXED-NEXT:    fmov w8, s3
-; NEON-FIXED-NEXT:    bfi x9, x8, #2, #2
+; NEON-FIXED-NEXT:    fmov w0, s3
+; NEON-FIXED-NEXT:    bfi x8, x0, #2, #2
+; NEON-FIXED-NEXT:    ldr s0, [x8]
 ; NEON-FIXED-NEXT:    fmov w8, s1
-; NEON-FIXED-NEXT:    ldr s0, [x9]
 ; NEON-FIXED-NEXT:    tst w8, #0x1
 ; NEON-FIXED-NEXT:    fcsel s0, s0, s2, ne
 ; NEON-FIXED-NEXT:    add sp, sp, #16
@@ -441,7 +441,7 @@ define i32 @extract_last_active_v3i32(<3 x i32> %a, <3 x i1> %c) {
 ; NEON-FIXED-NEXT:    .cfi_def_cfa_offset 16
 ; NEON-FIXED-NEXT:    movi v1.2d, #0000000000000000
 ; NEON-FIXED-NEXT:    adrp x8, .LCPI18_0
-; NEON-FIXED-NEXT:    mov x11, sp
+; NEON-FIXED-NEXT:    mov x10, sp
 ; NEON-FIXED-NEXT:    ldr d3, [x8, :lo12:.LCPI18_0]
 ; NEON-FIXED-NEXT:    str q0, [sp]
 ; NEON-FIXED-NEXT:    mov v1.h[0], w0
@@ -456,9 +456,9 @@ define i32 @extract_last_active_v3i32(<3 x i32> %a, <3 x i1> %c) {
 ; NEON-FIXED-NEXT:    orr w8, w9, w8, lsr #16
 ; NEON-FIXED-NEXT:    tst w8, #0x1
 ; NEON-FIXED-NEXT:    umaxv h2, v2.4h
-; NEON-FIXED-NEXT:    fmov w10, s2
-; NEON-FIXED-NEXT:    bfi x11, x10, #2, #2
-; NEON-FIXED-NEXT:    ldr w9, [x11]
+; NEON-FIXED-NEXT:    fmov w0, s2
+; NEON-FIXED-NEXT:    bfi x10, x0, #2, #2
+; NEON-FIXED-NEXT:    ldr w9, [x10]
 ; NEON-FIXED-NEXT:    csinv w0, w9, wzr, ne
 ; NEON-FIXED-NEXT:    add sp, sp, #16
 ; NEON-FIXED-NEXT:    ret
diff --git a/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll b/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
index d37e7695ff786..ace95d0caf4b8 100644
--- a/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
+++ b/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
@@ -515,12 +515,12 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
 ; GFX908-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x10
 ; GFX908-NEXT:    s_load_dword s7, s[8:9], 0x18
 ; GFX908-NEXT:    s_mov_b32 s6, 0
-; GFX908-NEXT:    s_mov_b32 s9, s6
+; GFX908-NEXT:    ; freeze: $sgpr18_sgpr19
 ; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX908-NEXT:    v_cvt_f32_u32_e32 v1, s3
 ; GFX908-NEXT:    s_sub_i32 s8, 0, s3
 ; GFX908-NEXT:    v_cvt_f32_f16_e32 v12, s7
-; GFX908-NEXT:    s_mov_b64 s[18:19], 0
+; GFX908-NEXT:    s_mov_b32 s9, s6
 ; GFX908-NEXT:    v_rcp_f32_e32 v1, v1
 ; GFX908-NEXT:    v_mov_b32_e32 v14, 0
 ; GFX908-NEXT:    v_mul_f32_e32 v1, 0x4f7ffffe, v1
@@ -553,7 +553,8 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
 ; GFX908-NEXT:    s_lshl_b64 s[12:13], s[4:5], 5
 ; GFX908-NEXT:    s_lshl_b64 s[14:15], s[8:9], 5
 ; GFX908-NEXT:    s_lshl_b64 s[16:17], s[2:3], 5
-; GFX908-NEXT:    s_and_b64 s[0:1], exec, s[0:1]
+; GFX908-NEXT:    s_and_b64 s[0:1], exec, s[18:19]
+; GFX908-NEXT:    s_mov_b64 s[18:19], 0
 ; GFX908-NEXT:    s_branch .LBB3_2
 ; GFX908-NEXT:  .LBB3_1: ; in Loop: Header=BB3_2 Depth=1
 ; GFX908-NEXT:    s_branch .LBB3_9
@@ -654,11 +655,11 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
 ; GFX90A-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x10
 ; GFX90A-NEXT:    s_load_dword s7, s[8:9], 0x18
 ; GFX90A-NEXT:    s_mov_b32 s6, 0
-; GFX90A-NEXT:    s_mov_b32 s9, s6
+; GFX90A-NEXT:    ; freeze: $sgpr18_sgpr19
 ; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90A-NEXT:    v_cvt_f32_u32_e32 v0, s3
 ; GFX90A-NEXT:    s_sub_i32 s8, 0, s3
-; GFX90A-NEXT:    s_mov_b64 s[18:19], 0
+; GFX90A-NEXT:    s_mov_b32 s9, s6
 ; GFX90A-NEXT:    v_mov_b32_e32 v14, 0
 ; GFX90A-NEXT:    v_rcp_f32_e32 v0, v0
 ; GFX90A-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
@@ -692,7 +693,8 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
 ; GFX90A-NEXT:    s_lshl_b64 s[12:13], s[4:5], 5
 ; GFX90A-NEXT:    s_lshl_b64 s[14:15], s[8:9], 5
 ; GFX90A-NEXT:    s_lshl_b64 s[16:17], s[2:3], 5
-; GFX90A-NEXT:    s_and_b64 s[0:1], exec, s[0:1]
+; GFX90A-NEXT:    s_and_b64 s[0:1], exec, s[18:19]
+; GFX90A-NEXT:    s_mov_b64 s[18:19], 0
 ; GFX90A-NEXT:    s_branch .LBB3_2
 ; GFX90A-NEXT:  .LBB3_1: ; in Loop: Header=BB3_2 Depth=1
 ; GFX90A-NEXT:    s_branch .LBB3_9
diff --git a/llvm/test/CodeGen/AMDGPU/andorbitset.ll b/llvm/test/CodeGen/AMDGPU/andorbitset.ll
index 03499b184d565..e7aa64ce127e2 100644
--- a/llvm/test/CodeGen/AMDGPU/andorbitset.ll
+++ b/llvm/test/CodeGen/AMDGPU/andorbitset.ll
@@ -109,20 +109,21 @@ define amdgpu_kernel void @s_set_midbit(ptr addrspace(1) %out, i32 %in) {
 define void @bitset_verifier_error_freeze_poison() local_unnamed_addr #0 {
 ; SI-LABEL: bitset_verifier_error_freeze_poison:
 ; SI:       ; %bb.0: ; %bb
+; SI-NEXT:    ; freeze: $sgpr8
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_getpc_b64 s[4:5]
 ; SI-NEXT:    s_add_u32 s4, s4, gv at gotpcrel32@lo+4
 ; SI-NEXT:    s_addc_u32 s5, s5, gv at gotpcrel32@hi+12
 ; SI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
+; SI-NEXT:    s_and_b32 s9, s8, 0x7fffffff
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    v_mov_b32_e32 v0, s9
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    s_and_b32 s8, s4, 0x7fffffff
-; SI-NEXT:    v_mov_b32_e32 v0, s8
 ; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v0, 0x3f7fbe77
-; SI-NEXT:    v_cmp_ge_f32_e64 vcc, |s4|, v0
+; SI-NEXT:    v_cmp_ge_f32_e64 vcc, |s8|, v0
 ; SI-NEXT:    s_cbranch_vccnz .LBB6_2
 ; SI-NEXT:  ; %bb.1: ; %bb5
 ; SI-NEXT:  .LBB6_2: ; %bb6
diff --git a/llvm/test/CodeGen/AMDGPU/carryout-selection.ll b/llvm/test/CodeGen/AMDGPU/carryout-selection.ll
index 3f0618de15a32..a2f5fa25bf822 100644
--- a/llvm/test/CodeGen/AMDGPU/carryout-selection.ll
+++ b/llvm/test/CodeGen/AMDGPU/carryout-selection.ll
@@ -4230,19 +4230,19 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
 ; GCN-ISEL-NEXT:   [[S_CSELECT_B32_4:%[0-9]+]]:sreg_32 = S_CSELECT_B32 [[S_MOV_B32_10]], [[S_MOV_B32_9]], implicit $scc
 ; GCN-ISEL-NEXT:   S_CMP_EQ_U32 [[S_SUB_C2]], [[COPY22]], implicit-def $scc
 ; GCN-ISEL-NEXT:   [[S_CSELECT_B32_5:%[0-9]+]]:sreg_32 = S_CSELECT_B32 [[S_CSELECT_B32_4]], [[S_CSELECT_B32_3]], implicit $scc
-; GCN-ISEL-NEXT:   [[COPY72:%[0-9]+]]:sreg_32 = COPY [[S_CSELECT_B32_5]]
+; GCN-ISEL-NEXT:   [[FREEZE:%[0-9]+]]:sreg_32 = FREEZE [[S_CSELECT_B32_5]]
 ; GCN-ISEL-NEXT:   [[REG_SEQUENCE22:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY69]], %subreg.sub0, [[COPY68]], %subreg.sub1
 ; GCN-ISEL-NEXT:   [[S_MOV_B64_1:%[0-9]+]]:sreg_64 = S_MOV_B64 1
 ; GCN-ISEL-NEXT:   [[S_ADD_U6:%[0-9]+]]:sreg_64 = S_ADD_U64_PSEUDO [[REG_SEQUENCE22]], [[S_MOV_B64_1]], implicit-def dead $scc
-; GCN-ISEL-NEXT:   [[COPY73:%[0-9]+]]:sreg_32 = COPY [[S_ADD_U6]].sub0
+; GCN-ISEL-NEXT:   [[COPY72:%[0-9]+]]:sreg_32 = COPY [[S_ADD_U6]].sub0
 ; GCN-ISEL-NEXT:   [[S_MOV_B64_2:%[0-9]+]]:sreg_64 = S_MOV_B64 2
 ; GCN-ISEL-NEXT:   [[S_ADD_U7:%[0-9]+]]:sreg_64 = S_ADD_U64_PSEUDO [[REG_SEQUENCE22]], [[S_MOV_B64_2]], implicit-def dead $scc
-; GCN-ISEL-NEXT:   [[COPY74:%[0-9]+]]:sreg_32 = COPY [[S_ADD_U7]].sub0
-; GCN-ISEL-NEXT:   S_CMP_LG_U32 [[COPY72]], [[S_MOV_B32_9]], implicit-def $scc
-; GCN-ISEL-NEXT:   [[S_CSELECT_B32_6:%[0-9]+]]:sreg_32 = S_CSELECT_B32 [[COPY74]], [[COPY73]], implicit $scc
-; GCN-ISEL-NEXT:   [[COPY75:%[0-9]+]]:sreg_32 = COPY [[S_ADD_U6]].sub1
-; GCN-ISEL-NEXT:   [[COPY76:%[0-9]+]]:sreg_32 = COPY [[S_ADD_U7]].sub1
-; GCN-ISEL-NEXT:   [[S_CSELECT_B32_7:%[0-9]+]]:sreg_32 = S_CSELECT_B32 [[COPY76]], [[COPY75]], implicit $scc
+; GCN-ISEL-NEXT:   [[COPY73:%[0-9]+]]:sreg_32 = COPY [[S_ADD_U7]].sub0
+; GCN-ISEL-NEXT:   S_CMP_LG_U32 [[FREEZE]], [[S_MOV_B32_9]], implicit-def $scc
+; GCN-ISEL-NEXT:   [[S_CSELECT_B32_6:%[0-9]+]]:sreg_32 = S_CSELECT_B32 [[COPY73]], [[COPY72]], implicit $scc
+; GCN-ISEL-NEXT:   [[COPY74:%[0-9]+]]:sreg_32 = COPY [[S_ADD_U6]].sub1
+; GCN-ISEL-NEXT:   [[COPY75:%[0-9]+]]:sreg_32 = COPY [[S_ADD_U7]].sub1
+; GCN-ISEL-NEXT:   [[S_CSELECT_B32_7:%[0-9]+]]:sreg_32 = S_CSELECT_B32 [[COPY75]], [[COPY74]], implicit $scc
 ; GCN-ISEL-NEXT:   [[S_SUB_C4:%[0-9]+]]:sreg_32, [[S_SUB_C5:%[0-9]+]]:sreg_64_xexec = S_SUB_CO_PSEUDO [[COPY60]], [[S_ADD_I32_8]], [[S_USUBO1]], implicit-def dead $scc
 ; GCN-ISEL-NEXT:   S_CMP_GE_U32 [[S_SUB_C4]], [[COPY22]], implicit-def $scc
 ; GCN-ISEL-NEXT:   [[S_CSELECT_B32_8:%[0-9]+]]:sreg_32 = S_CSELECT_B32 [[S_MOV_B32_10]], [[S_MOV_B32_9]], implicit $scc
@@ -4250,25 +4250,25 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
 ; GCN-ISEL-NEXT:   [[S_CSELECT_B32_9:%[0-9]+]]:sreg_32 = S_CSELECT_B32 [[S_MOV_B32_10]], [[S_MOV_B32_9]], implicit $scc
 ; GCN-ISEL-NEXT:   S_CMP_EQ_U32 [[S_SUB_C4]], [[COPY22]], implicit-def $scc
 ; GCN-ISEL-NEXT:   [[S_CSELECT_B32_10:%[0-9]+]]:sreg_32 = S_CSELECT_B32 [[S_CSELECT_B32_9]], [[S_CSELECT_B32_8]], implicit $scc
-; GCN-ISEL-NEXT:   [[COPY77:%[0-9]+]]:sreg_32 = COPY [[S_CSELECT_B32_10]]
-; GCN-ISEL-NEXT:   S_CMP_LG_U32 [[COPY77]], [[S_MOV_B32_9]], implicit-def $scc
+; GCN-ISEL-NEXT:   [[FREEZE1:%[0-9]+]]:sreg_32 = FREEZE [[S_CSELECT_B32_10]]
+; GCN-ISEL-NEXT:   S_CMP_LG_U32 [[FREEZE1]], [[S_MOV_B32_9]], implicit-def $scc
 ; GCN-ISEL-NEXT:   [[S_CSELECT_B32_11:%[0-9]+]]:sreg_32 = S_CSELECT_B32 [[S_CSELECT_B32_7]], [[COPY68]], implicit $scc
 ; GCN-ISEL-NEXT:   [[S_CSELECT_B32_12:%[0-9]+]]:sreg_32 = S_CSELECT_B32 [[S_CSELECT_B32_6]], [[COPY69]], implicit $scc
 ; GCN-ISEL-NEXT:   [[REG_SEQUENCE23:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_CSELECT_B32_12]], %subreg.sub0, [[S_CSELECT_B32_11]], %subreg.sub1
-; GCN-ISEL-NEXT:   [[COPY78:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE23]]
+; GCN-ISEL-NEXT:   [[COPY76:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE23]]
 ; GCN-ISEL-NEXT: {{  $}}
 ; GCN-ISEL-NEXT: bb.3..split:
-; GCN-ISEL-NEXT:   [[PHI:%[0-9]+]]:sreg_64 = PHI [[COPY78]], %bb.2, [[COPY20]], %bb.1
-; GCN-ISEL-NEXT:   [[COPY79:%[0-9]+]]:sreg_32 = COPY [[COPY7]].sub1
-; GCN-ISEL-NEXT:   [[COPY80:%[0-9]+]]:sreg_32 = COPY [[COPY7]].sub0
-; GCN-ISEL-NEXT:   [[REG_SEQUENCE24:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY80]], %subreg.sub0, [[COPY79]], %subreg.sub1
-; GCN-ISEL-NEXT:   [[COPY81:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE24]].sub1
-; GCN-ISEL-NEXT:   [[COPY82:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE24]].sub0
+; GCN-ISEL-NEXT:   [[PHI:%[0-9]+]]:sreg_64 = PHI [[COPY76]], %bb.2, [[COPY20]], %bb.1
+; GCN-ISEL-NEXT:   [[COPY77:%[0-9]+]]:sreg_32 = COPY [[COPY7]].sub1
+; GCN-ISEL-NEXT:   [[COPY78:%[0-9]+]]:sreg_32 = COPY [[COPY7]].sub0
+; GCN-ISEL-NEXT:   [[REG_SEQUENCE24:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY78]], %subreg.sub0, [[COPY77]], %subreg.sub1
+; GCN-ISEL-NEXT:   [[COPY79:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE24]].sub1
+; GCN-ISEL-NEXT:   [[COPY80:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE24]].sub0
 ; GCN-ISEL-NEXT:   [[S_MOV_B32_11:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
 ; GCN-ISEL-NEXT:   [[S_MOV_B32_12:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
-; GCN-ISEL-NEXT:   [[REG_SEQUENCE25:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY82]], %subreg.sub0, [[COPY81]], %subreg.sub1, [[S_MOV_B32_12]], %subreg.sub2, [[S_MOV_B32_11]], %subreg.sub3
-; GCN-ISEL-NEXT:   [[COPY83:%[0-9]+]]:vreg_64 = COPY [[PHI]]
-; GCN-ISEL-NEXT:   BUFFER_STORE_DWORDX2_OFFSET [[COPY83]], [[REG_SEQUENCE25]], 0, 0, 0, 0, implicit $exec :: (store (s64) into %ir.13, addrspace 1)
+; GCN-ISEL-NEXT:   [[REG_SEQUENCE25:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY80]], %subreg.sub0, [[COPY79]], %subreg.sub1, [[S_MOV_B32_12]], %subreg.sub2, [[S_MOV_B32_11]], %subreg.sub3
+; GCN-ISEL-NEXT:   [[COPY81:%[0-9]+]]:vreg_64 = COPY [[PHI]]
+; GCN-ISEL-NEXT:   BUFFER_STORE_DWORDX2_OFFSET [[COPY81]], [[REG_SEQUENCE25]], 0, 0, 0, 0, implicit $exec :: (store (s64) into %ir.13, addrspace 1)
 ; GCN-ISEL-NEXT:   S_ENDPGM 0
   %result = udiv i64 %x, %y
   store i64 %result, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/divergence-driven-trunc-to-i1.ll b/llvm/test/CodeGen/AMDGPU/divergence-driven-trunc-to-i1.ll
index 5589311e90d67..cd8d0a2606d2b 100644
--- a/llvm/test/CodeGen/AMDGPU/divergence-driven-trunc-to-i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/divergence-driven-trunc-to-i1.ll
@@ -17,18 +17,18 @@ define amdgpu_kernel void @uniform_trunc_i16_to_i1(ptr addrspace(1) %out, i16 %x
   ; GCN-NEXT:   [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 16
   ; GCN-NEXT:   [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[S_LOAD_DWORD_IMM]], [[S_MOV_B32_2]], implicit-def dead $scc
   ; GCN-NEXT:   [[S_LSHR_B32_:%[0-9]+]]:sreg_32 = S_LSHR_B32 [[S_LOAD_DWORD_IMM]], [[S_MOV_B32_2]], implicit-def dead $scc
-  ; GCN-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY [[S_LSHR_B32_]]
-  ; GCN-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 1, [[COPY3]], implicit-def dead $scc
+  ; GCN-NEXT:   [[FREEZE:%[0-9]+]]:sreg_32 = FREEZE [[S_LSHR_B32_]]
+  ; GCN-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 1, [[FREEZE]], implicit-def dead $scc
   ; GCN-NEXT:   S_CMP_EQ_U32 [[S_AND_B32_]], 1, implicit-def $scc
-  ; GCN-NEXT:   [[COPY4:%[0-9]+]]:sreg_64 = COPY $scc
+  ; GCN-NEXT:   [[COPY3:%[0-9]+]]:sreg_64 = COPY $scc
   ; GCN-NEXT:   [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 0
   ; GCN-NEXT:   S_CMP_LT_I32 [[S_LSHL_B32_]], [[S_MOV_B32_3]], implicit-def $scc
-  ; GCN-NEXT:   [[COPY5:%[0-9]+]]:sreg_64 = COPY $scc
-  ; GCN-NEXT:   [[S_OR_B64_:%[0-9]+]]:sreg_64 = S_OR_B64 [[COPY5]], [[COPY4]], implicit-def dead $scc
+  ; GCN-NEXT:   [[COPY4:%[0-9]+]]:sreg_64 = COPY $scc
+  ; GCN-NEXT:   [[S_OR_B64_:%[0-9]+]]:sreg_64 = S_OR_B64 [[COPY4]], [[COPY3]], implicit-def dead $scc
   ; GCN-NEXT:   $scc = COPY [[S_OR_B64_]]
   ; GCN-NEXT:   [[S_CSELECT_B32_:%[0-9]+]]:sreg_32 = S_CSELECT_B32 1, 0, implicit $scc
-  ; GCN-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[S_CSELECT_B32_]]
-  ; GCN-NEXT:   BUFFER_STORE_BYTE_OFFSET [[COPY6]], [[REG_SEQUENCE]], 0, 0, 0, 0, implicit $exec :: (store (s8) into %ir.out.load, addrspace 1)
+  ; GCN-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[S_CSELECT_B32_]]
+  ; GCN-NEXT:   BUFFER_STORE_BYTE_OFFSET [[COPY5]], [[REG_SEQUENCE]], 0, 0, 0, 0, implicit $exec :: (store (s8) into %ir.out.load, addrspace 1)
   ; GCN-NEXT:   S_ENDPGM 0
   %setcc = icmp slt i16 %x, 0
   %select = select i1 %setcc, i1 true, i1 %z
@@ -71,20 +71,20 @@ define amdgpu_kernel void @uniform_trunc_i32_to_i1(ptr addrspace(1) %out, i32 %x
   ; GCN-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
   ; GCN-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
   ; GCN-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[S_MOV_B32_1]], %subreg.sub2, [[S_MOV_B32_]], %subreg.sub3
-  ; GCN-NEXT:   [[COPY3:%[0-9]+]]:sreg_64 = COPY [[S_LOAD_DWORDX2_IMM1]]
-  ; GCN-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY [[COPY3]].sub0
-  ; GCN-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY [[COPY3]].sub1
-  ; GCN-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 1, [[COPY5]], implicit-def dead $scc
+  ; GCN-NEXT:   [[FREEZE:%[0-9]+]]:sreg_64 = FREEZE [[S_LOAD_DWORDX2_IMM1]]
+  ; GCN-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY [[FREEZE]].sub0
+  ; GCN-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY [[FREEZE]].sub1
+  ; GCN-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 1, [[COPY4]], implicit-def dead $scc
   ; GCN-NEXT:   S_CMP_EQ_U32 [[S_AND_B32_]], 1, implicit-def $scc
-  ; GCN-NEXT:   [[COPY6:%[0-9]+]]:sreg_64 = COPY $scc
+  ; GCN-NEXT:   [[COPY5:%[0-9]+]]:sreg_64 = COPY $scc
   ; GCN-NEXT:   [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 0
-  ; GCN-NEXT:   S_CMP_LT_I32 [[COPY4]], [[S_MOV_B32_2]], implicit-def $scc
-  ; GCN-NEXT:   [[COPY7:%[0-9]+]]:sreg_64 = COPY $scc
-  ; GCN-NEXT:   [[S_OR_B64_:%[0-9]+]]:sreg_64 = S_OR_B64 [[COPY7]], [[COPY6]], implicit-def dead $scc
+  ; GCN-NEXT:   S_CMP_LT_I32 [[COPY3]], [[S_MOV_B32_2]], implicit-def $scc
+  ; GCN-NEXT:   [[COPY6:%[0-9]+]]:sreg_64 = COPY $scc
+  ; GCN-NEXT:   [[S_OR_B64_:%[0-9]+]]:sreg_64 = S_OR_B64 [[COPY6]], [[COPY5]], implicit-def dead $scc
   ; GCN-NEXT:   $scc = COPY [[S_OR_B64_]]
   ; GCN-NEXT:   [[S_CSELECT_B32_:%[0-9]+]]:sreg_32 = S_CSELECT_B32 1, 0, implicit $scc
-  ; GCN-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_CSELECT_B32_]]
-  ; GCN-NEXT:   BUFFER_STORE_BYTE_OFFSET [[COPY8]], [[REG_SEQUENCE]], 0, 0, 0, 0, implicit $exec :: (store (s8) into %ir.out.load, addrspace 1)
+  ; GCN-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_CSELECT_B32_]]
+  ; GCN-NEXT:   BUFFER_STORE_BYTE_OFFSET [[COPY7]], [[REG_SEQUENCE]], 0, 0, 0, 0, implicit $exec :: (store (s8) into %ir.out.load, addrspace 1)
   ; GCN-NEXT:   S_ENDPGM 0
   %setcc = icmp slt i32 %x, 0
   %select = select i1 %setcc, i1 true, i1 %z
@@ -129,18 +129,18 @@ define amdgpu_kernel void @uniform_trunc_i64_to_i1(ptr addrspace(1) %out, i64 %x
   ; GCN-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
   ; GCN-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[S_MOV_B32_1]], %subreg.sub2, [[S_MOV_B32_]], %subreg.sub3
   ; GCN-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub3
-  ; GCN-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORD_IMM]]
-  ; GCN-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 1, [[COPY6]], implicit-def dead $scc
+  ; GCN-NEXT:   [[FREEZE:%[0-9]+]]:sreg_32 = FREEZE [[S_LOAD_DWORD_IMM]]
+  ; GCN-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 1, [[FREEZE]], implicit-def dead $scc
   ; GCN-NEXT:   S_CMP_EQ_U32 [[S_AND_B32_]], 1, implicit-def $scc
-  ; GCN-NEXT:   [[COPY7:%[0-9]+]]:sreg_64 = COPY $scc
+  ; GCN-NEXT:   [[COPY6:%[0-9]+]]:sreg_64 = COPY $scc
   ; GCN-NEXT:   [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 0
   ; GCN-NEXT:   S_CMP_LT_I32 [[COPY5]], [[S_MOV_B32_2]], implicit-def $scc
-  ; GCN-NEXT:   [[COPY8:%[0-9]+]]:sreg_64 = COPY $scc
-  ; GCN-NEXT:   [[S_OR_B64_:%[0-9]+]]:sreg_64 = S_OR_B64 [[COPY8]], [[COPY7]], implicit-def dead $scc
+  ; GCN-NEXT:   [[COPY7:%[0-9]+]]:sreg_64 = COPY $scc
+  ; GCN-NEXT:   [[S_OR_B64_:%[0-9]+]]:sreg_64 = S_OR_B64 [[COPY7]], [[COPY6]], implicit-def dead $scc
   ; GCN-NEXT:   $scc = COPY [[S_OR_B64_]]
   ; GCN-NEXT:   [[S_CSELECT_B32_:%[0-9]+]]:sreg_32 = S_CSELECT_B32 1, 0, implicit $scc
-  ; GCN-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[S_CSELECT_B32_]]
-  ; GCN-NEXT:   BUFFER_STORE_BYTE_OFFSET [[COPY9]], [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec :: (store (s8) into %ir.2, addrspace 1)
+  ; GCN-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_CSELECT_B32_]]
+  ; GCN-NEXT:   BUFFER_STORE_BYTE_OFFSET [[COPY8]], [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec :: (store (s8) into %ir.2, addrspace 1)
   ; GCN-NEXT:   S_ENDPGM 0
   %setcc = icmp slt i64 %x, 0
   %select = select i1 %setcc, i1 true, i1 %z
diff --git a/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-hw.ll b/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-hw.ll
index 954d6a3f54cec..d2298cc92c401 100644
--- a/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-hw.ll
+++ b/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-hw.ll
@@ -1165,21 +1165,19 @@ define i8 @to_fp8_f64(double %x) {
 ; GFX950-NEXT:    v_lshrrev_b64 v[12:13], v6, v[12:13]
 ; GFX950-NEXT:    v_and_b32_e32 v6, v12, v14
 ; GFX950-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GFX950-NEXT:    s_movk_i32 s1, 0x80
 ; GFX950-NEXT:    s_mov_b32 s0, 0
+; GFX950-NEXT:    s_movk_i32 s1, 0x80
 ; GFX950-NEXT:    v_cndmask_b32_e32 v6, 0, v6, vcc
 ; GFX950-NEXT:    v_lshl_add_u64 v[4:5], v[8:9], 0, v[6:7]
 ; GFX950-NEXT:    v_cmp_lt_i64_e32 vcc, 3, v[4:5]
-; GFX950-NEXT:    v_and_b32_sdwa v8, v1, s1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX950-NEXT:    v_lshrrev_b32_e32 v6, 17, v11
-; GFX950-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc
-; GFX950-NEXT:    v_cndmask_b32_e64 v5, 0, 4, vcc
-; GFX950-NEXT:    v_or3_b32 v9, v8, v5, v4
 ; GFX950-NEXT:    v_and_b32_e32 v5, 0x1ffff, v11
+; GFX950-NEXT:    v_lshrrev_b32_e32 v6, 17, v11
+; GFX950-NEXT:    v_cndmask_b32_e64 v8, v4, 0, vcc
 ; GFX950-NEXT:    v_mov_b32_e32 v4, v10
+; GFX950-NEXT:    v_cndmask_b32_e64 v9, 0, 4, vcc
 ; GFX950-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GFX950-NEXT:    v_bfe_u32 v4, v11, 18, 2
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    v_and_b32_sdwa v12, v1, s1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
 ; GFX950-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
 ; GFX950-NEXT:    v_and_or_b32 v5, v4, 1, v5
 ; GFX950-NEXT:    v_and_b32_e32 v6, v6, v5
@@ -1187,29 +1185,29 @@ define i8 @to_fp8_f64(double %x) {
 ; GFX950-NEXT:    v_lshl_add_u64 v[4:5], v[4:5], 0, v[6:7]
 ; GFX950-NEXT:    v_cmp_lt_i64_e32 vcc, 3, v[4:5]
 ; GFX950-NEXT:    v_mov_b32_e32 v7, s0
-; GFX950-NEXT:    s_nop 0
+; GFX950-NEXT:    v_or3_b32 v8, v12, v9, v8
 ; GFX950-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
 ; GFX950-NEXT:    v_lshl_add_u64 v[2:3], v[2:3], 0, v[6:7]
 ; GFX950-NEXT:    v_lshl_add_u64 v[2:3], v[2:3], 0, 14
 ; GFX950-NEXT:    v_lshlrev_b64 v[6:7], 2, v[2:3]
 ; GFX950-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc
-; GFX950-NEXT:    v_or_b32_e32 v6, v6, v8
+; GFX950-NEXT:    v_or_b32_e32 v6, v6, v12
 ; GFX950-NEXT:    v_cndmask_b32_e64 v5, v5, 0, vcc
 ; GFX950-NEXT:    v_or_b32_e32 v6, v6, v4
 ; GFX950-NEXT:    v_cmp_gt_i64_e32 vcc, 1, v[2:3]
 ; GFX950-NEXT:    v_cmp_eq_u64_e64 s[0:1], 30, v[2:3]
 ; GFX950-NEXT:    s_nop 0
-; GFX950-NEXT:    v_cndmask_b32_e32 v6, v6, v9, vcc
+; GFX950-NEXT:    v_cndmask_b32_e32 v6, v6, v8, vcc
 ; GFX950-NEXT:    v_cmp_lt_i64_e32 vcc, 3, v[4:5]
 ; GFX950-NEXT:    s_and_b64 s[0:1], s[0:1], vcc
 ; GFX950-NEXT:    v_cmp_lt_i64_e32 vcc, 30, v[2:3]
-; GFX950-NEXT:    v_or_b32_e32 v2, 0x7c, v8
+; GFX950-NEXT:    v_or_b32_e32 v2, 0x7c, v12
 ; GFX950-NEXT:    s_or_b64 vcc, vcc, s[0:1]
 ; GFX950-NEXT:    v_cndmask_b32_e32 v3, v6, v2, vcc
 ; GFX950-NEXT:    v_cmp_eq_f64_e32 vcc, 0, v[0:1]
 ; GFX950-NEXT:    s_movk_i32 s0, 0x204
 ; GFX950-NEXT:    s_nop 0
-; GFX950-NEXT:    v_cndmask_b32_e32 v3, v3, v8, vcc
+; GFX950-NEXT:    v_cndmask_b32_e32 v3, v3, v12, vcc
 ; GFX950-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], s0
 ; GFX950-NEXT:    s_nop 1
 ; GFX950-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
diff --git a/llvm/test/CodeGen/AMDGPU/fold-fabs.ll b/llvm/test/CodeGen/AMDGPU/fold-fabs.ll
index 0dbbae3c94f38..608afa627df76 100644
--- a/llvm/test/CodeGen/AMDGPU/fold-fabs.ll
+++ b/llvm/test/CodeGen/AMDGPU/fold-fabs.ll
@@ -70,6 +70,7 @@ exit:
 define float @fold_abs_in_branch_undef(float %arg1, float %arg2) {
 ; GFX10-LABEL: fold_abs_in_branch_undef:
 ; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    ; freeze: $sgpr4
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_add_f32_e64 v0, |s4|, |s4|
 ; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, 1.0, v0
diff --git a/llvm/test/CodeGen/AMDGPU/freeze-multi-use.ll b/llvm/test/CodeGen/AMDGPU/freeze-multi-use.ll
index 08422bf946c3b..a0ce9aec62f12 100644
--- a/llvm/test/CodeGen/AMDGPU/freeze-multi-use.ll
+++ b/llvm/test/CodeGen/AMDGPU/freeze-multi-use.ll
@@ -7,10 +7,11 @@
 define i32 @ashr_mul(i32 %x) {
 ; CHECK-LABEL: ashr_mul:
 ; CHECK:       ; %bb.0:
+; CHECK-NEXT:    ; freeze: $sgpr0
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_ashr_i32 s0, s0, 3
+; CHECK-NEXT:    s_ashr_i32 s1, s0, 3
 ; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; CHECK-NEXT:    s_mul_i32 s0, s0, s0
+; CHECK-NEXT:    s_mul_i32 s0, s0, s1
 ; CHECK-NEXT:    v_mov_b32_e32 v0, s0
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
   %p = ashr i32 %x, 32
@@ -30,20 +31,25 @@ define i32 @across_call() nounwind {
 ; CHECK-NEXT:    s_or_saveexec_b32 s1, -1
 ; CHECK-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
 ; CHECK-NEXT:    s_mov_b32 exec_lo, s1
-; CHECK-NEXT:    v_writelane_b32 v40, s0, 2
+; CHECK-NEXT:    v_writelane_b32 v40, s0, 3
 ; CHECK-NEXT:    s_add_i32 s32, s32, 16
-; CHECK-NEXT:    v_writelane_b32 v40, s30, 0
-; CHECK-NEXT:    v_writelane_b32 v40, s31, 1
+; CHECK-NEXT:    v_writelane_b32 v40, s34, 0
+; CHECK-NEXT:    v_writelane_b32 v40, s30, 1
+; CHECK-NEXT:    v_writelane_b32 v40, s31, 2
 ; CHECK-NEXT:    s_getpc_b64 s[0:1]
 ; CHECK-NEXT:    s_add_u32 s0, s0, use at gotpcrel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s1, s1, use at gotpcrel32@hi+12
 ; CHECK-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
+; CHECK-NEXT:    ; freeze: $sgpr34
+; CHECK-NEXT:    v_mov_b32_e32 v0, s34
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; CHECK-NEXT:    v_readlane_b32 s30, v40, 0
-; CHECK-NEXT:    v_readlane_b32 s31, v40, 1
+; CHECK-NEXT:    v_readlane_b32 s30, v40, 1
+; CHECK-NEXT:    v_mov_b32_e32 v0, s34
+; CHECK-NEXT:    v_readlane_b32 s31, v40, 2
+; CHECK-NEXT:    v_readlane_b32 s34, v40, 0
 ; CHECK-NEXT:    s_mov_b32 s32, s33
-; CHECK-NEXT:    v_readlane_b32 s0, v40, 2
+; CHECK-NEXT:    v_readlane_b32 s0, v40, 3
 ; CHECK-NEXT:    s_or_saveexec_b32 s1, -1
 ; CHECK-NEXT:    scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
 ; CHECK-NEXT:    s_mov_b32 exec_lo, s1
@@ -65,20 +71,33 @@ define <4 x i32> @vec_across_call() nounwind {
 ; CHECK-NEXT:    s_or_saveexec_b32 s1, -1
 ; CHECK-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
 ; CHECK-NEXT:    s_mov_b32 exec_lo, s1
-; CHECK-NEXT:    v_writelane_b32 v40, s0, 2
+; CHECK-NEXT:    v_writelane_b32 v40, s0, 6
 ; CHECK-NEXT:    s_add_i32 s32, s32, 16
-; CHECK-NEXT:    v_writelane_b32 v40, s30, 0
-; CHECK-NEXT:    v_writelane_b32 v40, s31, 1
+; CHECK-NEXT:    v_writelane_b32 v40, s36, 0
+; CHECK-NEXT:    v_writelane_b32 v40, s37, 1
+; CHECK-NEXT:    v_writelane_b32 v40, s38, 2
+; CHECK-NEXT:    v_writelane_b32 v40, s39, 3
+; CHECK-NEXT:    v_writelane_b32 v40, s30, 4
+; CHECK-NEXT:    v_writelane_b32 v40, s31, 5
 ; CHECK-NEXT:    s_getpc_b64 s[0:1]
 ; CHECK-NEXT:    s_add_u32 s0, s0, usev at gotpcrel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s1, s1, usev at gotpcrel32@hi+12
 ; CHECK-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
+; CHECK-NEXT:    ; freeze: $sgpr36_sgpr37_sgpr38_sgpr39
+; CHECK-NEXT:    v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
+; CHECK-NEXT:    v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; CHECK-NEXT:    v_readlane_b32 s30, v40, 0
-; CHECK-NEXT:    v_readlane_b32 s31, v40, 1
+; CHECK-NEXT:    v_readlane_b32 s30, v40, 4
+; CHECK-NEXT:    v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
+; CHECK-NEXT:    v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
+; CHECK-NEXT:    v_readlane_b32 s31, v40, 5
+; CHECK-NEXT:    v_readlane_b32 s39, v40, 3
+; CHECK-NEXT:    v_readlane_b32 s38, v40, 2
+; CHECK-NEXT:    v_readlane_b32 s37, v40, 1
+; CHECK-NEXT:    v_readlane_b32 s36, v40, 0
 ; CHECK-NEXT:    s_mov_b32 s32, s33
-; CHECK-NEXT:    v_readlane_b32 s0, v40, 2
+; CHECK-NEXT:    v_readlane_b32 s0, v40, 6
 ; CHECK-NEXT:    s_or_saveexec_b32 s1, -1
 ; CHECK-NEXT:    scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
 ; CHECK-NEXT:    s_mov_b32 exec_lo, s1
diff --git a/llvm/test/CodeGen/AMDGPU/freeze.ll b/llvm/test/CodeGen/AMDGPU/freeze.ll
index ae9c97cc8d139..8f171d12b8bf9 100644
--- a/llvm/test/CodeGen/AMDGPU/freeze.ll
+++ b/llvm/test/CodeGen/AMDGPU/freeze.ll
@@ -16908,9 +16908,11 @@ define double @freeze_fabs_double(float %a, double %b, double %c) {
 ; GFX6-SDAG-LABEL: freeze_fabs_double:
 ; GFX6-SDAG:       ; %bb.0:
 ; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-SDAG-NEXT:    v_mov_b32_e32 v5, v0
-; GFX6-SDAG-NEXT:    v_add_f64 v[0:1], |v[4:5]|, v[1:2]
-; GFX6-SDAG-NEXT:    v_add_f64 v[2:3], |v[4:5]|, v[3:4]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v6, v0
+; GFX6-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v5, s4
+; GFX6-SDAG-NEXT:    v_add_f64 v[0:1], |v[5:6]|, v[1:2]
+; GFX6-SDAG-NEXT:    v_add_f64 v[2:3], |v[5:6]|, v[3:4]
 ; GFX6-SDAG-NEXT:    v_add_f64 v[0:1], v[0:1], v[2:3]
 ; GFX6-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -16926,9 +16928,11 @@ define double @freeze_fabs_double(float %a, double %b, double %c) {
 ; GFX7-SDAG-LABEL: freeze_fabs_double:
 ; GFX7-SDAG:       ; %bb.0:
 ; GFX7-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-SDAG-NEXT:    v_mov_b32_e32 v5, v0
-; GFX7-SDAG-NEXT:    v_add_f64 v[0:1], |v[4:5]|, v[1:2]
-; GFX7-SDAG-NEXT:    v_add_f64 v[2:3], |v[4:5]|, v[3:4]
+; GFX7-SDAG-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX7-SDAG-NEXT:    v_mov_b32_e32 v5, s4
+; GFX7-SDAG-NEXT:    v_add_f64 v[0:1], |v[5:6]|, v[1:2]
+; GFX7-SDAG-NEXT:    v_add_f64 v[2:3], |v[5:6]|, v[3:4]
 ; GFX7-SDAG-NEXT:    v_add_f64 v[0:1], v[0:1], v[2:3]
 ; GFX7-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -16944,9 +16948,11 @@ define double @freeze_fabs_double(float %a, double %b, double %c) {
 ; GFX8-SDAG-LABEL: freeze_fabs_double:
 ; GFX8-SDAG:       ; %bb.0:
 ; GFX8-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-SDAG-NEXT:    v_mov_b32_e32 v5, v0
-; GFX8-SDAG-NEXT:    v_add_f64 v[0:1], |v[4:5]|, v[1:2]
-; GFX8-SDAG-NEXT:    v_add_f64 v[2:3], |v[4:5]|, v[3:4]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v6, v0
+; GFX8-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v5, s4
+; GFX8-SDAG-NEXT:    v_add_f64 v[0:1], |v[5:6]|, v[1:2]
+; GFX8-SDAG-NEXT:    v_add_f64 v[2:3], |v[5:6]|, v[3:4]
 ; GFX8-SDAG-NEXT:    v_add_f64 v[0:1], v[0:1], v[2:3]
 ; GFX8-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -16971,9 +16977,11 @@ define double @freeze_fabs_double(float %a, double %b, double %c) {
 ; GFX10-SDAG-LABEL: freeze_fabs_double:
 ; GFX10-SDAG:       ; %bb.0:
 ; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v5, v0
-; GFX10-SDAG-NEXT:    v_add_f64 v[0:1], |v[4:5]|, v[1:2]
-; GFX10-SDAG-NEXT:    v_add_f64 v[2:3], |v[4:5]|, v[3:4]
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v6, v0
+; GFX10-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v5, s4
+; GFX10-SDAG-NEXT:    v_add_f64 v[0:1], |v[5:6]|, v[1:2]
+; GFX10-SDAG-NEXT:    v_add_f64 v[2:3], |v[5:6]|, v[3:4]
 ; GFX10-SDAG-NEXT:    v_add_f64 v[0:1], v[0:1], v[2:3]
 ; GFX10-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -16988,10 +16996,11 @@ define double @freeze_fabs_double(float %a, double %b, double %c) {
 ;
 ; GFX11-SDAG-LABEL: freeze_fabs_double:
 ; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    ; freeze: $sgpr0
 ; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-NEXT:    v_mov_b32_e32 v5, v0
-; GFX11-SDAG-NEXT:    v_add_f64 v[0:1], |v[4:5]|, v[1:2]
-; GFX11-SDAG-NEXT:    v_add_f64 v[2:3], |v[4:5]|, v[3:4]
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v6, v0 :: v_dual_mov_b32 v5, s0
+; GFX11-SDAG-NEXT:    v_add_f64 v[0:1], |v[5:6]|, v[1:2]
+; GFX11-SDAG-NEXT:    v_add_f64 v[2:3], |v[5:6]|, v[3:4]
 ; GFX11-SDAG-NEXT:    v_add_f64 v[0:1], v[0:1], v[2:3]
 ; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane.ll
index ae1f9ff278c30..7a9f4603a976b 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane.ll
@@ -7349,369 +7349,499 @@ define amdgpu_kernel void @v_permlane16_b32_tid_tid_f64(ptr addrspace(1) %out, f
 }
 
 define amdgpu_kernel void @v_permlane16_b32_undef_tid_i32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-LABEL: v_permlane16_b32_undef_tid_i32:
-; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_clause 0x1
-; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    v_permlane16_b32 v0, v0, s0, s1
-; GFX10-NEXT:    global_store_dword v1, v0, s[2:3]
-; GFX10-NEXT:    s_endpgm
-;
-; GFX11-LABEL: v_permlane16_b32_undef_tid_i32:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
-; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_permlane16_b32 v0, v0, s0, s1
-; GFX11-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX11-NEXT:    s_endpgm
-;
-; GFX12-LABEL: v_permlane16_b32_undef_tid_i32:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_clause 0x1
-; GFX12-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
-; GFX12-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT:    v_permlane16_b32 v0, v0, s0, s1
-; GFX12-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX12-NEXT:    s_endpgm
-;
-; GFX13-LABEL: v_permlane16_b32_undef_tid_i32:
-; GFX13:       ; %bb.0:
-; GFX13-NEXT:    s_clause 0x1
-; GFX13-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-NEXT:    s_mov_b32 s4, 0x3ff
-; GFX13-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
-; GFX13-NEXT:    s_wait_kmcnt 0x0
-; GFX13-NEXT:    v_permlane16_b32 v0, v0, s0, s1
-; GFX13-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX13-NEXT:    s_endpgm
-  %tidx = call i32 @llvm.amdgcn.workitem.id.x()
-  %undef = freeze i32 poison
-  %v = call i32 @llvm.amdgcn.permlane16.i32(i32 %undef, i32 %tidx, i32 %src1, i32 %src2, i1 false, i1 false)
-  store i32 %v, ptr addrspace(1) %out
-  ret void
-}
-
-define amdgpu_kernel void @v_permlane16_b32_undef_tid_f32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-LABEL: v_permlane16_b32_undef_tid_f32:
-; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_clause 0x1
-; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    v_permlane16_b32 v0, v0, s0, s1
-; GFX10-NEXT:    global_store_dword v1, v0, s[2:3]
-; GFX10-NEXT:    s_endpgm
-;
-; GFX11-LABEL: v_permlane16_b32_undef_tid_f32:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
-; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_permlane16_b32 v0, v0, s0, s1
-; GFX11-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX11-NEXT:    s_endpgm
-;
-; GFX12-LABEL: v_permlane16_b32_undef_tid_f32:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_clause 0x1
-; GFX12-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
-; GFX12-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT:    v_permlane16_b32 v0, v0, s0, s1
-; GFX12-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX12-NEXT:    s_endpgm
-;
-; GFX13-LABEL: v_permlane16_b32_undef_tid_f32:
-; GFX13:       ; %bb.0:
-; GFX13-NEXT:    s_clause 0x1
-; GFX13-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-NEXT:    s_mov_b32 s4, 0x3ff
-; GFX13-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
-; GFX13-NEXT:    s_wait_kmcnt 0x0
-; GFX13-NEXT:    v_permlane16_b32 v0, v0, s0, s1
-; GFX13-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX13-NEXT:    s_endpgm
-  %tidx = call i32 @llvm.amdgcn.workitem.id.x()
-  %tidx_f32 = bitcast i32 %tidx to float
-  %undef = freeze float poison
-  %v = call float @llvm.amdgcn.permlane16.f32(float %undef, float %tidx_f32, i32 %src1, i32 %src2, i1 false, i1 false)
-  store float %v, ptr addrspace(1) %out
-  ret void
-}
-
-define amdgpu_kernel void @v_permlane16_b32_undef_tid_i64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_undef_tid_i64:
+; GFX10-SDAG-LABEL: v_permlane16_b32_undef_tid_i32:
 ; GFX10-SDAG:       ; %bb.0:
 ; GFX10-SDAG-NEXT:    s_clause 0x1
 ; GFX10-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
 ; GFX10-SDAG-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
 ; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v1, s4
 ; GFX10-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_permlane16_b32 v1, v2, s0, s1
-; GFX10-SDAG-NEXT:    v_permlane16_b32 v0, v0, s0, s1
-; GFX10-SDAG-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1
+; GFX10-SDAG-NEXT:    global_store_dword v2, v1, s[2:3]
 ; GFX10-SDAG-NEXT:    s_endpgm
 ;
-; GFX10-GISEL-LABEL: v_permlane16_b32_undef_tid_i64:
+; GFX10-GISEL-LABEL: v_permlane16_b32_undef_tid_i32:
 ; GFX10-GISEL:       ; %bb.0:
 ; GFX10-GISEL-NEXT:    s_clause 0x1
 ; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
 ; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-GISEL-NEXT:    v_permlane16_b32 v0, v0, s0, s1
-; GFX10-GISEL-NEXT:    v_permlane16_b32 v1, v2, s0, s1
-; GFX10-GISEL-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-GISEL-NEXT:    global_store_dword v1, v0, s[2:3]
 ; GFX10-GISEL-NEXT:    s_endpgm
 ;
-; GFX11-SDAG-LABEL: v_permlane16_b32_undef_tid_i64:
+; GFX11-SDAG-LABEL: v_permlane16_b32_undef_tid_i32:
 ; GFX11-SDAG:       ; %bb.0:
 ; GFX11-SDAG-NEXT:    s_clause 0x1
 ; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
 ; GFX11-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
 ; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, 0
-; GFX11-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT:    v_permlane16_b32 v1, v2, s0, s1
-; GFX11-SDAG-NEXT:    v_permlane16_b32 v0, v0, s0, s1
-; GFX11-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1
+; GFX11-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
-; GFX11-GISEL-LABEL: v_permlane16_b32_undef_tid_i64:
+; GFX11-GISEL-LABEL: v_permlane16_b32_undef_tid_i32:
 ; GFX11-GISEL:       ; %bb.0:
 ; GFX11-GISEL-NEXT:    s_clause 0x1
 ; GFX11-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
 ; GFX11-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-GISEL-NEXT:    v_permlane16_b32 v0, v0, s0, s1
-; GFX11-GISEL-NEXT:    v_permlane16_b32 v1, v2, s0, s1
-; GFX11-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
 ; GFX11-GISEL-NEXT:    s_endpgm
 ;
-; GFX12-SDAG-LABEL: v_permlane16_b32_undef_tid_i64:
+; GFX12-SDAG-LABEL: v_permlane16_b32_undef_tid_i32:
 ; GFX12-SDAG:       ; %bb.0:
 ; GFX12-SDAG-NEXT:    s_clause 0x1
 ; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
 ; GFX12-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v2, 0
-; GFX12-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-NEXT:    v_permlane16_b32 v1, v2, s0, s1
-; GFX12-SDAG-NEXT:    v_permlane16_b32 v0, v0, s0, s1
-; GFX12-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1
+; GFX12-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
 ; GFX12-SDAG-NEXT:    s_endpgm
 ;
-; GFX12-GISEL-LABEL: v_permlane16_b32_undef_tid_i64:
+; GFX12-GISEL-LABEL: v_permlane16_b32_undef_tid_i32:
 ; GFX12-GISEL:       ; %bb.0:
 ; GFX12-GISEL-NEXT:    s_clause 0x1
 ; GFX12-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
 ; GFX12-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX12-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_permlane16_b32 v0, v0, s0, s1
-; GFX12-GISEL-NEXT:    v_permlane16_b32 v1, v2, s0, s1
-; GFX12-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
 ; GFX12-GISEL-NEXT:    s_endpgm
 ;
-; GFX13-SDAG-LABEL: v_permlane16_b32_undef_tid_i64:
+; GFX13-SDAG-LABEL: v_permlane16_b32_undef_tid_i32:
 ; GFX13-SDAG:       ; %bb.0:
 ; GFX13-SDAG-NEXT:    s_clause 0x1
 ; GFX13-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
 ; GFX13-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT:    s_mov_b32 s4, 0x3ff
+; GFX13-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX13-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX13-SDAG-NEXT:    s_mov_b32 s5, 0x3ff
 ; GFX13-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-SDAG-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
 ; GFX13-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT:    v_permlane16_b32 v1, v2, s0, s1
-; GFX13-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX13-SDAG-NEXT:    v_permlane16_b32 v0, v0, s0, s1
-; GFX13-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1
+; GFX13-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
 ; GFX13-SDAG-NEXT:    s_endpgm
 ;
-; GFX13-GISEL-LABEL: v_permlane16_b32_undef_tid_i64:
+; GFX13-GISEL-LABEL: v_permlane16_b32_undef_tid_i32:
 ; GFX13-GISEL:       ; %bb.0:
 ; GFX13-GISEL-NEXT:    s_clause 0x1
 ; GFX13-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
 ; GFX13-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
 ; GFX13-GISEL-NEXT:    s_mov_b32 s4, 0x3ff
 ; GFX13-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-GISEL-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
 ; GFX13-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX13-GISEL-NEXT:    v_permlane16_b32 v0, v0, s0, s1
-; GFX13-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX13-GISEL-NEXT:    v_permlane16_b32 v1, v2, s0, s1
-; GFX13-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
 ; GFX13-GISEL-NEXT:    s_endpgm
   %tidx = call i32 @llvm.amdgcn.workitem.id.x()
-  %tidx_i64 = zext i32 %tidx to i64
-  %undef = freeze i64 poison
-  %v = call i64 @llvm.amdgcn.permlane16.i64(i64 %undef, i64 %tidx_i64, i32 %src1, i32 %src2, i1 false, i1 false)
-  store i64 %v, ptr addrspace(1) %out
+  %undef = freeze i32 poison
+  %v = call i32 @llvm.amdgcn.permlane16.i32(i32 %undef, i32 %tidx, i32 %src1, i32 %src2, i1 false, i1 false)
+  store i32 %v, ptr addrspace(1) %out
   ret void
 }
 
-define amdgpu_kernel void @v_permlane16_b32_undef_tid_f64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_undef_tid_f64:
+define amdgpu_kernel void @v_permlane16_b32_undef_tid_f32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
+; GFX10-SDAG-LABEL: v_permlane16_b32_undef_tid_f32:
 ; GFX10-SDAG:       ; %bb.0:
-; GFX10-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
 ; GFX10-SDAG-NEXT:    s_clause 0x1
 ; GFX10-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
 ; GFX10-SDAG-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
 ; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v1, s4
 ; GFX10-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_permlane16_b32 v1, v1, s0, s1
-; GFX10-SDAG-NEXT:    v_permlane16_b32 v0, v0, s0, s1
-; GFX10-SDAG-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1
+; GFX10-SDAG-NEXT:    global_store_dword v2, v1, s[2:3]
 ; GFX10-SDAG-NEXT:    s_endpgm
 ;
-; GFX10-GISEL-LABEL: v_permlane16_b32_undef_tid_f64:
+; GFX10-GISEL-LABEL: v_permlane16_b32_undef_tid_f32:
 ; GFX10-GISEL:       ; %bb.0:
-; GFX10-GISEL-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
 ; GFX10-GISEL-NEXT:    s_clause 0x1
 ; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
 ; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-GISEL-NEXT:    v_permlane16_b32 v0, v0, s0, s1
-; GFX10-GISEL-NEXT:    v_permlane16_b32 v1, v1, s0, s1
-; GFX10-GISEL-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-GISEL-NEXT:    global_store_dword v1, v0, s[2:3]
 ; GFX10-GISEL-NEXT:    s_endpgm
 ;
-; GFX11-SDAG-LABEL: v_permlane16_b32_undef_tid_f64:
+; GFX11-SDAG-LABEL: v_permlane16_b32_undef_tid_f32:
 ; GFX11-SDAG:       ; %bb.0:
-; GFX11-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GFX11-SDAG-NEXT:    s_clause 0x1
 ; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
 ; GFX11-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
 ; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, 0
-; GFX11-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX11-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT:    v_permlane16_b32 v1, v1, s0, s1
-; GFX11-SDAG-NEXT:    v_permlane16_b32 v0, v0, s0, s1
-; GFX11-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1
+; GFX11-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
-; GFX11-GISEL-LABEL: v_permlane16_b32_undef_tid_f64:
+; GFX11-GISEL-LABEL: v_permlane16_b32_undef_tid_f32:
 ; GFX11-GISEL:       ; %bb.0:
-; GFX11-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GFX11-GISEL-NEXT:    s_clause 0x1
 ; GFX11-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
 ; GFX11-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-GISEL-NEXT:    v_mov_b32_e32 v2, 0
-; GFX11-GISEL-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX11-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-GISEL-NEXT:    v_permlane16_b32 v0, v0, s0, s1
-; GFX11-GISEL-NEXT:    v_permlane16_b32 v1, v1, s0, s1
-; GFX11-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
 ; GFX11-GISEL-NEXT:    s_endpgm
 ;
-; GFX12-SDAG-LABEL: v_permlane16_b32_undef_tid_f64:
+; GFX12-SDAG-LABEL: v_permlane16_b32_undef_tid_f32:
 ; GFX12-SDAG:       ; %bb.0:
-; GFX12-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GFX12-SDAG-NEXT:    s_clause 0x1
 ; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
 ; GFX12-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v2, 0
-; GFX12-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX12-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-NEXT:    v_permlane16_b32 v1, v1, s0, s1
-; GFX12-SDAG-NEXT:    v_permlane16_b32 v0, v0, s0, s1
-; GFX12-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1
+; GFX12-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
 ; GFX12-SDAG-NEXT:    s_endpgm
 ;
-; GFX12-GISEL-LABEL: v_permlane16_b32_undef_tid_f64:
+; GFX12-GISEL-LABEL: v_permlane16_b32_undef_tid_f32:
 ; GFX12-GISEL:       ; %bb.0:
-; GFX12-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GFX12-GISEL-NEXT:    s_clause 0x1
 ; GFX12-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
 ; GFX12-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-GISEL-NEXT:    v_mov_b32_e32 v2, 0
-; GFX12-GISEL-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX12-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_permlane16_b32 v0, v0, s0, s1
-; GFX12-GISEL-NEXT:    v_permlane16_b32 v1, v1, s0, s1
-; GFX12-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
 ; GFX12-GISEL-NEXT:    s_endpgm
 ;
-; GFX13-SDAG-LABEL: v_permlane16_b32_undef_tid_f64:
+; GFX13-SDAG-LABEL: v_permlane16_b32_undef_tid_f32:
 ; GFX13-SDAG:       ; %bb.0:
-; GFX13-SDAG-NEXT:    s_mov_b32 s0, 0x3ff
-; GFX13-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX13-SDAG-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s0, v0 bitop3:0x40
 ; GFX13-SDAG-NEXT:    s_clause 0x1
 ; GFX13-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
 ; GFX13-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX13-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX13-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX13-SDAG-NEXT:    s_mov_b32 s5, 0x3ff
+; GFX13-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
 ; GFX13-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT:    v_permlane16_b32 v1, v1, s0, s1
-; GFX13-SDAG-NEXT:    v_permlane16_b32 v0, v0, s0, s1
-; GFX13-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1
+; GFX13-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
 ; GFX13-SDAG-NEXT:    s_endpgm
 ;
-; GFX13-GISEL-LABEL: v_permlane16_b32_undef_tid_f64:
+; GFX13-GISEL-LABEL: v_permlane16_b32_undef_tid_f32:
 ; GFX13-GISEL:       ; %bb.0:
-; GFX13-GISEL-NEXT:    s_mov_b32 s0, 0x3ff
-; GFX13-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX13-GISEL-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s0, v0 bitop3:0x40
 ; GFX13-GISEL-NEXT:    s_clause 0x1
 ; GFX13-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
 ; GFX13-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-GISEL-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX13-GISEL-NEXT:    s_mov_b32 s4, 0x3ff
+; GFX13-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
 ; GFX13-GISEL-NEXT:    s_wait_kmcnt 0x0
-; GFX13-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX13-GISEL-NEXT:    v_permlane16_b32 v0, v0, s0, s1
-; GFX13-GISEL-NEXT:    v_permlane16_b32 v1, v1, s0, s1
-; GFX13-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
 ; GFX13-GISEL-NEXT:    s_endpgm
   %tidx = call i32 @llvm.amdgcn.workitem.id.x()
   %tidx_f32 = bitcast i32 %tidx to float
-  %tidx_f64 = fpext float %tidx_f32 to double
-  %undef = freeze double poison
-  %v = call double @llvm.amdgcn.permlane16.f64(double %undef, double %tidx_f64, i32 %src1, i32 %src2, i1 false, i1 false)
-  store double %v, ptr addrspace(1) %out
+  %undef = freeze float poison
+  %v = call float @llvm.amdgcn.permlane16.f32(float %undef, float %tidx_f32, i32 %src1, i32 %src2, i1 false, i1 false)
+  store float %v, ptr addrspace(1) %out
   ret void
 }
 
-define amdgpu_kernel void @v_permlane16_b32_i_tid_i32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_i_tid_i32:
+define amdgpu_kernel void @v_permlane16_b32_undef_tid_i64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
+; GFX10-SDAG-LABEL: v_permlane16_b32_undef_tid_i64:
 ; GFX10-SDAG:       ; %bb.0:
 ; GFX10-SDAG-NEXT:    s_clause 0x1
 ; GFX10-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
 ; GFX10-SDAG-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v1, 0x3039
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v3, 0
+; GFX10-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, s5
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v1, s4
+; GFX10-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v2, v3, s0, s1
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1
+; GFX10-SDAG-NEXT:    global_store_dwordx2 v3, v[1:2], s[2:3]
+; GFX10-SDAG-NEXT:    s_endpgm
+;
+; GFX10-GISEL-LABEL: v_permlane16_b32_undef_tid_i64:
+; GFX10-GISEL:       ; %bb.0:
+; GFX10-GISEL-NEXT:    s_clause 0x1
+; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT:    v_permlane16_b32 v0, v0, s0, s1
+; GFX10-GISEL-NEXT:    v_permlane16_b32 v1, v2, s0, s1
+; GFX10-GISEL-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-GISEL-NEXT:    s_endpgm
+;
+; GFX11-SDAG-LABEL: v_permlane16_b32_undef_tid_i64:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_clause 0x1
+; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v2, v3, s0, s1
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1
+; GFX11-SDAG-NEXT:    global_store_b64 v3, v[1:2], s[2:3]
+; GFX11-SDAG-NEXT:    s_endpgm
+;
+; GFX11-GISEL-LABEL: v_permlane16_b32_undef_tid_i64:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_clause 0x1
+; GFX11-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT:    v_permlane16_b32 v0, v0, s0, s1
+; GFX11-GISEL-NEXT:    v_permlane16_b32 v1, v2, s0, s1
+; GFX11-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-GISEL-NEXT:    s_endpgm
+;
+; GFX12-SDAG-LABEL: v_permlane16_b32_undef_tid_i64:
+; GFX12-SDAG:       ; %bb.0:
+; GFX12-SDAG-NEXT:    s_clause 0x1
+; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v2, v3, s0, s1
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1
+; GFX12-SDAG-NEXT:    global_store_b64 v3, v[1:2], s[2:3]
+; GFX12-SDAG-NEXT:    s_endpgm
+;
+; GFX12-GISEL-LABEL: v_permlane16_b32_undef_tid_i64:
+; GFX12-GISEL:       ; %bb.0:
+; GFX12-GISEL-NEXT:    s_clause 0x1
+; GFX12-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT:    v_permlane16_b32 v0, v0, s0, s1
+; GFX12-GISEL-NEXT:    v_permlane16_b32 v1, v2, s0, s1
+; GFX12-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-GISEL-NEXT:    s_endpgm
+;
+; GFX13-SDAG-LABEL: v_permlane16_b32_undef_tid_i64:
+; GFX13-SDAG:       ; %bb.0:
+; GFX13-SDAG-NEXT:    s_clause 0x1
+; GFX13-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX13-SDAG-NEXT:    s_mov_b32 s5, 0x3ff
+; GFX13-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
+; GFX13-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX13-SDAG-NEXT:    v_permlane16_b32 v2, v3, s0, s1
+; GFX13-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX13-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1
+; GFX13-SDAG-NEXT:    global_store_b64 v3, v[1:2], s[2:3]
+; GFX13-SDAG-NEXT:    s_endpgm
+;
+; GFX13-GISEL-LABEL: v_permlane16_b32_undef_tid_i64:
+; GFX13-GISEL:       ; %bb.0:
+; GFX13-GISEL-NEXT:    s_clause 0x1
+; GFX13-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-GISEL-NEXT:    s_mov_b32 s4, 0x3ff
+; GFX13-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-GISEL-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX13-GISEL-NEXT:    v_permlane16_b32 v0, v0, s0, s1
+; GFX13-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX13-GISEL-NEXT:    v_permlane16_b32 v1, v2, s0, s1
+; GFX13-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-GISEL-NEXT:    s_endpgm
+  %tidx = call i32 @llvm.amdgcn.workitem.id.x()
+  %tidx_i64 = zext i32 %tidx to i64
+  %undef = freeze i64 poison
+  %v = call i64 @llvm.amdgcn.permlane16.i64(i64 %undef, i64 %tidx_i64, i32 %src1, i32 %src2, i1 false, i1 false)
+  store i64 %v, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @v_permlane16_b32_undef_tid_f64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
+; GFX10-SDAG-LABEL: v_permlane16_b32_undef_tid_f64:
+; GFX10-SDAG:       ; %bb.0:
+; GFX10-SDAG-NEXT:    s_clause 0x1
+; GFX10-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-SDAG-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v4, 0
+; GFX10-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v3, s5
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, s4
+; GFX10-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v3, v1, s0, s1
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v2, v0, s0, s1
+; GFX10-SDAG-NEXT:    global_store_dwordx2 v4, v[2:3], s[2:3]
+; GFX10-SDAG-NEXT:    s_endpgm
+;
+; GFX10-GISEL-LABEL: v_permlane16_b32_undef_tid_f64:
+; GFX10-GISEL:       ; %bb.0:
+; GFX10-GISEL-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-GISEL-NEXT:    s_clause 0x1
+; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT:    v_permlane16_b32 v0, v0, s0, s1
+; GFX10-GISEL-NEXT:    v_permlane16_b32 v1, v1, s0, s1
+; GFX10-GISEL-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-GISEL-NEXT:    s_endpgm
+;
+; GFX11-SDAG-LABEL: v_permlane16_b32_undef_tid_f64:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_clause 0x1
+; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT:    v_mov_b32_e32 v4, 0
+; GFX11-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v3, s5 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v3, v1, s0, s1
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v2, v0, s0, s1
+; GFX11-SDAG-NEXT:    global_store_b64 v4, v[2:3], s[2:3]
+; GFX11-SDAG-NEXT:    s_endpgm
+;
+; GFX11-GISEL-LABEL: v_permlane16_b32_undef_tid_f64:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT:    s_clause 0x1
+; GFX11-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT:    v_permlane16_b32 v0, v0, s0, s1
+; GFX11-GISEL-NEXT:    v_permlane16_b32 v1, v1, s0, s1
+; GFX11-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-GISEL-NEXT:    s_endpgm
+;
+; GFX12-SDAG-LABEL: v_permlane16_b32_undef_tid_f64:
+; GFX12-SDAG:       ; %bb.0:
+; GFX12-SDAG-NEXT:    s_clause 0x1
+; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-SDAG-NEXT:    v_mov_b32_e32 v4, 0
+; GFX12-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v3, s5 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT:    v_mov_b32_e32 v2, s4
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v3, v1, s0, s1
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v2, v0, s0, s1
+; GFX12-SDAG-NEXT:    global_store_b64 v4, v[2:3], s[2:3]
+; GFX12-SDAG-NEXT:    s_endpgm
+;
+; GFX12-GISEL-LABEL: v_permlane16_b32_undef_tid_f64:
+; GFX12-GISEL:       ; %bb.0:
+; GFX12-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-NEXT:    s_clause 0x1
+; GFX12-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX12-GISEL-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT:    v_permlane16_b32 v0, v0, s0, s1
+; GFX12-GISEL-NEXT:    v_permlane16_b32 v1, v1, s0, s1
+; GFX12-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-GISEL-NEXT:    s_endpgm
+;
+; GFX13-SDAG-LABEL: v_permlane16_b32_undef_tid_f64:
+; GFX13-SDAG:       ; %bb.0:
+; GFX13-SDAG-NEXT:    s_clause 0x1
+; GFX13-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX13-SDAG-NEXT:    s_mov_b32 s6, 0x3ff
+; GFX13-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v3, s5 :: v_dual_bitop2_b32 v0, s6, v0 bitop3:0x40
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v4, 0
+; GFX13-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX13-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX13-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-SDAG-NEXT:    v_permlane16_b32 v3, v1, s0, s1
+; GFX13-SDAG-NEXT:    v_permlane16_b32 v2, v0, s0, s1
+; GFX13-SDAG-NEXT:    global_store_b64 v4, v[2:3], s[2:3]
+; GFX13-SDAG-NEXT:    s_endpgm
+;
+; GFX13-GISEL-LABEL: v_permlane16_b32_undef_tid_f64:
+; GFX13-GISEL:       ; %bb.0:
+; GFX13-GISEL-NEXT:    s_mov_b32 s0, 0x3ff
+; GFX13-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX13-GISEL-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s0, v0 bitop3:0x40
+; GFX13-GISEL-NEXT:    s_clause 0x1
+; GFX13-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-GISEL-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX13-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX13-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-GISEL-NEXT:    v_permlane16_b32 v0, v0, s0, s1
+; GFX13-GISEL-NEXT:    v_permlane16_b32 v1, v1, s0, s1
+; GFX13-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-GISEL-NEXT:    s_endpgm
+  %tidx = call i32 @llvm.amdgcn.workitem.id.x()
+  %tidx_f32 = bitcast i32 %tidx to float
+  %tidx_f64 = fpext float %tidx_f32 to double
+  %undef = freeze double poison
+  %v = call double @llvm.amdgcn.permlane16.f64(double %undef, double %tidx_f64, i32 %src1, i32 %src2, i1 false, i1 false)
+  store double %v, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @v_permlane16_b32_i_tid_i32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
+; GFX10-SDAG-LABEL: v_permlane16_b32_i_tid_i32:
+; GFX10-SDAG:       ; %bb.0:
+; GFX10-SDAG-NEXT:    s_clause 0x1
+; GFX10-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-SDAG-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v1, 0x3039
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX10-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1
 ; GFX10-SDAG-NEXT:    global_store_dword v2, v1, s[2:3]
@@ -8186,53 +8316,109 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_f64(ptr addrspace(1) %out, i32
 }
 
 define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_i32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-LABEL: v_permlane16_b32_i_tid_fi_i32:
-; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_clause 0x1
-; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX10-NEXT:    global_store_dword v1, v0, s[2:3]
-; GFX10-NEXT:    s_endpgm
+; GFX10-SDAG-LABEL: v_permlane16_b32_i_tid_fi_i32:
+; GFX10-SDAG:       ; %bb.0:
+; GFX10-SDAG-NEXT:    s_clause 0x1
+; GFX10-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-SDAG-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v1, s4
+; GFX10-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX10-SDAG-NEXT:    global_store_dword v2, v1, s[2:3]
+; GFX10-SDAG-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: v_permlane16_b32_i_tid_fi_i32:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
-; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX11-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX11-NEXT:    s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_i_tid_fi_i32:
+; GFX10-GISEL:       ; %bb.0:
+; GFX10-GISEL-NEXT:    s_clause 0x1
+; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, 0
+; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX10-GISEL-NEXT:    global_store_dword v1, v0, s[2:3]
+; GFX10-GISEL-NEXT:    s_endpgm
 ;
-; GFX12-LABEL: v_permlane16_b32_i_tid_fi_i32:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_clause 0x1
-; GFX12-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
-; GFX12-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX12-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX12-NEXT:    s_endpgm
+; GFX11-SDAG-LABEL: v_permlane16_b32_i_tid_fi_i32:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_clause 0x1
+; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX11-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
+; GFX11-SDAG-NEXT:    s_endpgm
 ;
-; GFX13-LABEL: v_permlane16_b32_i_tid_fi_i32:
-; GFX13:       ; %bb.0:
-; GFX13-NEXT:    s_clause 0x1
-; GFX13-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-NEXT:    s_mov_b32 s4, 0x3ff
-; GFX13-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
-; GFX13-NEXT:    s_wait_kmcnt 0x0
-; GFX13-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX13-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX13-NEXT:    s_endpgm
+; GFX11-GISEL-LABEL: v_permlane16_b32_i_tid_fi_i32:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_clause 0x1
+; GFX11-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX11-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
+; GFX11-GISEL-NEXT:    s_endpgm
+;
+; GFX12-SDAG-LABEL: v_permlane16_b32_i_tid_fi_i32:
+; GFX12-SDAG:       ; %bb.0:
+; GFX12-SDAG-NEXT:    s_clause 0x1
+; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX12-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX12-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
+; GFX12-SDAG-NEXT:    s_endpgm
+;
+; GFX12-GISEL-LABEL: v_permlane16_b32_i_tid_fi_i32:
+; GFX12-GISEL:       ; %bb.0:
+; GFX12-GISEL-NEXT:    s_clause 0x1
+; GFX12-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX12-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
+; GFX12-GISEL-NEXT:    s_endpgm
+;
+; GFX13-SDAG-LABEL: v_permlane16_b32_i_tid_fi_i32:
+; GFX13-SDAG:       ; %bb.0:
+; GFX13-SDAG-NEXT:    s_clause 0x1
+; GFX13-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX13-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX13-SDAG-NEXT:    s_mov_b32 s5, 0x3ff
+; GFX13-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
+; GFX13-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX13-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX13-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
+; GFX13-SDAG-NEXT:    s_endpgm
+;
+; GFX13-GISEL-LABEL: v_permlane16_b32_i_tid_fi_i32:
+; GFX13-GISEL:       ; %bb.0:
+; GFX13-GISEL-NEXT:    s_clause 0x1
+; GFX13-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-GISEL-NEXT:    s_mov_b32 s4, 0x3ff
+; GFX13-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX13-GISEL-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX13-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
+; GFX13-GISEL-NEXT:    s_endpgm
   %tidx = call i32 @llvm.amdgcn.workitem.id.x()
   %undef = freeze i32 poison
   %v = call i32 @llvm.amdgcn.permlane16.i32(i32 %undef, i32 %tidx, i32 %src1, i32 %src2, i1 true, i1 false)
@@ -8241,53 +8427,109 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_i32(ptr addrspace(1) %out,
 }
 
 define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_f32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-LABEL: v_permlane16_b32_i_tid_fi_f32:
-; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_clause 0x1
-; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX10-NEXT:    global_store_dword v1, v0, s[2:3]
-; GFX10-NEXT:    s_endpgm
+; GFX10-SDAG-LABEL: v_permlane16_b32_i_tid_fi_f32:
+; GFX10-SDAG:       ; %bb.0:
+; GFX10-SDAG-NEXT:    s_clause 0x1
+; GFX10-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-SDAG-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v1, s4
+; GFX10-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX10-SDAG-NEXT:    global_store_dword v2, v1, s[2:3]
+; GFX10-SDAG-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: v_permlane16_b32_i_tid_fi_f32:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
-; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX11-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX11-NEXT:    s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_i_tid_fi_f32:
+; GFX10-GISEL:       ; %bb.0:
+; GFX10-GISEL-NEXT:    s_clause 0x1
+; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, 0
+; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX10-GISEL-NEXT:    global_store_dword v1, v0, s[2:3]
+; GFX10-GISEL-NEXT:    s_endpgm
 ;
-; GFX12-LABEL: v_permlane16_b32_i_tid_fi_f32:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_clause 0x1
-; GFX12-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
-; GFX12-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX12-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX12-NEXT:    s_endpgm
+; GFX11-SDAG-LABEL: v_permlane16_b32_i_tid_fi_f32:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_clause 0x1
+; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX11-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
+; GFX11-SDAG-NEXT:    s_endpgm
 ;
-; GFX13-LABEL: v_permlane16_b32_i_tid_fi_f32:
-; GFX13:       ; %bb.0:
-; GFX13-NEXT:    s_clause 0x1
-; GFX13-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-NEXT:    s_mov_b32 s4, 0x3ff
-; GFX13-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
-; GFX13-NEXT:    s_wait_kmcnt 0x0
-; GFX13-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX13-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX13-NEXT:    s_endpgm
+; GFX11-GISEL-LABEL: v_permlane16_b32_i_tid_fi_f32:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_clause 0x1
+; GFX11-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX11-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
+; GFX11-GISEL-NEXT:    s_endpgm
+;
+; GFX12-SDAG-LABEL: v_permlane16_b32_i_tid_fi_f32:
+; GFX12-SDAG:       ; %bb.0:
+; GFX12-SDAG-NEXT:    s_clause 0x1
+; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX12-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX12-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
+; GFX12-SDAG-NEXT:    s_endpgm
+;
+; GFX12-GISEL-LABEL: v_permlane16_b32_i_tid_fi_f32:
+; GFX12-GISEL:       ; %bb.0:
+; GFX12-GISEL-NEXT:    s_clause 0x1
+; GFX12-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX12-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
+; GFX12-GISEL-NEXT:    s_endpgm
+;
+; GFX13-SDAG-LABEL: v_permlane16_b32_i_tid_fi_f32:
+; GFX13-SDAG:       ; %bb.0:
+; GFX13-SDAG-NEXT:    s_clause 0x1
+; GFX13-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX13-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX13-SDAG-NEXT:    s_mov_b32 s5, 0x3ff
+; GFX13-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
+; GFX13-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX13-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX13-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
+; GFX13-SDAG-NEXT:    s_endpgm
+;
+; GFX13-GISEL-LABEL: v_permlane16_b32_i_tid_fi_f32:
+; GFX13-GISEL:       ; %bb.0:
+; GFX13-GISEL-NEXT:    s_clause 0x1
+; GFX13-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-GISEL-NEXT:    s_mov_b32 s4, 0x3ff
+; GFX13-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX13-GISEL-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX13-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
+; GFX13-GISEL-NEXT:    s_endpgm
   %tidx = call i32 @llvm.amdgcn.workitem.id.x()
   %tidx_f32 = bitcast i32 %tidx to float
   %undef = freeze float poison
@@ -8302,11 +8544,14 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_i64(ptr addrspace(1) %out,
 ; GFX10-SDAG-NEXT:    s_clause 0x1
 ; GFX10-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
 ; GFX10-SDAG-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v3, 0
+; GFX10-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, s5
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v1, s4
 ; GFX10-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_permlane16_b32 v1, v2, s0, s1 op_sel:[1,0]
-; GFX10-SDAG-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX10-SDAG-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v2, v3, s0, s1 op_sel:[1,0]
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX10-SDAG-NEXT:    global_store_dwordx2 v3, v[1:2], s[2:3]
 ; GFX10-SDAG-NEXT:    s_endpgm
 ;
 ; GFX10-GISEL-LABEL: v_permlane16_b32_i_tid_fi_i64:
@@ -8326,13 +8571,14 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_i64(ptr addrspace(1) %out,
 ; GFX11-SDAG-NEXT:    s_clause 0x1
 ; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
 ; GFX11-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, 0
-; GFX11-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT:    v_permlane16_b32 v1, v2, s0, s1 op_sel:[1,0]
-; GFX11-SDAG-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX11-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v2, v3, s0, s1 op_sel:[1,0]
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX11-SDAG-NEXT:    global_store_b64 v3, v[1:2], s[2:3]
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
 ; GFX11-GISEL-LABEL: v_permlane16_b32_i_tid_fi_i64:
@@ -8354,13 +8600,14 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_i64(ptr addrspace(1) %out,
 ; GFX12-SDAG-NEXT:    s_clause 0x1
 ; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
 ; GFX12-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-SDAG-NEXT:    v_mov_b32_e32 v2, 0
-; GFX12-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-NEXT:    v_permlane16_b32 v1, v2, s0, s1 op_sel:[1,0]
-; GFX12-SDAG-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX12-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v2, v3, s0, s1 op_sel:[1,0]
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX12-SDAG-NEXT:    global_store_b64 v3, v[1:2], s[2:3]
 ; GFX12-SDAG-NEXT:    s_endpgm
 ;
 ; GFX12-GISEL-LABEL: v_permlane16_b32_i_tid_fi_i64:
@@ -8382,14 +8629,16 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_i64(ptr addrspace(1) %out,
 ; GFX13-SDAG-NEXT:    s_clause 0x1
 ; GFX13-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
 ; GFX13-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT:    s_mov_b32 s4, 0x3ff
-; GFX13-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-SDAG-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX13-SDAG-NEXT:    s_mov_b32 s5, 0x3ff
+; GFX13-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
 ; GFX13-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT:    v_permlane16_b32 v1, v2, s0, s1 op_sel:[1,0]
+; GFX13-SDAG-NEXT:    v_permlane16_b32 v2, v3, s0, s1 op_sel:[1,0]
 ; GFX13-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX13-SDAG-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX13-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX13-SDAG-NEXT:    global_store_b64 v3, v[1:2], s[2:3]
 ; GFX13-SDAG-NEXT:    s_endpgm
 ;
 ; GFX13-GISEL-LABEL: v_permlane16_b32_i_tid_fi_i64:
@@ -8417,15 +8666,18 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_i64(ptr addrspace(1) %out,
 define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_f64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
 ; GFX10-SDAG-LABEL: v_permlane16_b32_i_tid_fi_f64:
 ; GFX10-SDAG:       ; %bb.0:
-; GFX10-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
 ; GFX10-SDAG-NEXT:    s_clause 0x1
 ; GFX10-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
 ; GFX10-SDAG-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v4, 0
+; GFX10-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v3, s5
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, s4
 ; GFX10-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_permlane16_b32 v1, v1, s0, s1 op_sel:[1,0]
-; GFX10-SDAG-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX10-SDAG-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v3, v1, s0, s1 op_sel:[1,0]
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v2, v0, s0, s1 op_sel:[1,0]
+; GFX10-SDAG-NEXT:    global_store_dwordx2 v4, v[2:3], s[2:3]
 ; GFX10-SDAG-NEXT:    s_endpgm
 ;
 ; GFX10-GISEL-LABEL: v_permlane16_b32_i_tid_fi_f64:
@@ -8443,17 +8695,20 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_f64(ptr addrspace(1) %out,
 ;
 ; GFX11-SDAG-LABEL: v_permlane16_b32_i_tid_fi_f64:
 ; GFX11-SDAG:       ; %bb.0:
-; GFX11-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GFX11-SDAG-NEXT:    s_clause 0x1
 ; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
 ; GFX11-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT:    v_mov_b32_e32 v4, 0
+; GFX11-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v3, s5 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX11-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
 ; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT:    v_permlane16_b32 v1, v1, s0, s1 op_sel:[1,0]
-; GFX11-SDAG-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX11-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v3, v1, s0, s1 op_sel:[1,0]
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v2, v0, s0, s1 op_sel:[1,0]
+; GFX11-SDAG-NEXT:    global_store_b64 v4, v[2:3], s[2:3]
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
 ; GFX11-GISEL-LABEL: v_permlane16_b32_i_tid_fi_f64:
@@ -8473,17 +8728,20 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_f64(ptr addrspace(1) %out,
 ;
 ; GFX12-SDAG-LABEL: v_permlane16_b32_i_tid_fi_f64:
 ; GFX12-SDAG:       ; %bb.0:
-; GFX12-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GFX12-SDAG-NEXT:    s_clause 0x1
 ; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
 ; GFX12-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX12-SDAG-NEXT:    v_mov_b32_e32 v4, 0
+; GFX12-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v3, s5 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT:    v_mov_b32_e32 v2, s4
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-NEXT:    v_permlane16_b32 v1, v1, s0, s1 op_sel:[1,0]
-; GFX12-SDAG-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX12-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v3, v1, s0, s1 op_sel:[1,0]
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v2, v0, s0, s1 op_sel:[1,0]
+; GFX12-SDAG-NEXT:    global_store_b64 v4, v[2:3], s[2:3]
 ; GFX12-SDAG-NEXT:    s_endpgm
 ;
 ; GFX12-GISEL-LABEL: v_permlane16_b32_i_tid_fi_f64:
@@ -8503,18 +8761,20 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_f64(ptr addrspace(1) %out,
 ;
 ; GFX13-SDAG-LABEL: v_permlane16_b32_i_tid_fi_f64:
 ; GFX13-SDAG:       ; %bb.0:
-; GFX13-SDAG-NEXT:    s_mov_b32 s0, 0x3ff
-; GFX13-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX13-SDAG-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s0, v0 bitop3:0x40
 ; GFX13-SDAG-NEXT:    s_clause 0x1
 ; GFX13-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
 ; GFX13-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX13-SDAG-NEXT:    s_mov_b32 s6, 0x3ff
+; GFX13-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v3, s5 :: v_dual_bitop2_b32 v0, s6, v0 bitop3:0x40
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v4, 0
 ; GFX13-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
 ; GFX13-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX13-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT:    v_permlane16_b32 v1, v1, s0, s1 op_sel:[1,0]
-; GFX13-SDAG-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX13-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-SDAG-NEXT:    v_permlane16_b32 v3, v1, s0, s1 op_sel:[1,0]
+; GFX13-SDAG-NEXT:    v_permlane16_b32 v2, v0, s0, s1 op_sel:[1,0]
+; GFX13-SDAG-NEXT:    global_store_b64 v4, v[2:3], s[2:3]
 ; GFX13-SDAG-NEXT:    s_endpgm
 ;
 ; GFX13-GISEL-LABEL: v_permlane16_b32_i_tid_fi_f64:
@@ -8542,53 +8802,109 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_f64(ptr addrspace(1) %out,
 }
 
 define amdgpu_kernel void @v_permlane16_b32_i_tid_bc_i32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-LABEL: v_permlane16_b32_i_tid_bc_i32:
-; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_clause 0x1
-; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX10-NEXT:    global_store_dword v1, v0, s[2:3]
-; GFX10-NEXT:    s_endpgm
-;
-; GFX11-LABEL: v_permlane16_b32_i_tid_bc_i32:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
-; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX11-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX11-NEXT:    s_endpgm
+; GFX10-SDAG-LABEL: v_permlane16_b32_i_tid_bc_i32:
+; GFX10-SDAG:       ; %bb.0:
+; GFX10-SDAG-NEXT:    s_clause 0x1
+; GFX10-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-SDAG-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v1, s4
+; GFX10-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX10-SDAG-NEXT:    global_store_dword v2, v1, s[2:3]
+; GFX10-SDAG-NEXT:    s_endpgm
 ;
-; GFX12-LABEL: v_permlane16_b32_i_tid_bc_i32:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_clause 0x1
-; GFX12-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
-; GFX12-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX12-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX12-NEXT:    s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_i_tid_bc_i32:
+; GFX10-GISEL:       ; %bb.0:
+; GFX10-GISEL-NEXT:    s_clause 0x1
+; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, 0
+; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX10-GISEL-NEXT:    global_store_dword v1, v0, s[2:3]
+; GFX10-GISEL-NEXT:    s_endpgm
 ;
-; GFX13-LABEL: v_permlane16_b32_i_tid_bc_i32:
-; GFX13:       ; %bb.0:
-; GFX13-NEXT:    s_clause 0x1
-; GFX13-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-NEXT:    s_mov_b32 s4, 0x3ff
-; GFX13-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
-; GFX13-NEXT:    s_wait_kmcnt 0x0
-; GFX13-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX13-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX13-NEXT:    s_endpgm
+; GFX11-SDAG-LABEL: v_permlane16_b32_i_tid_bc_i32:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_clause 0x1
+; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX11-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
+; GFX11-SDAG-NEXT:    s_endpgm
+;
+; GFX11-GISEL-LABEL: v_permlane16_b32_i_tid_bc_i32:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_clause 0x1
+; GFX11-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX11-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
+; GFX11-GISEL-NEXT:    s_endpgm
+;
+; GFX12-SDAG-LABEL: v_permlane16_b32_i_tid_bc_i32:
+; GFX12-SDAG:       ; %bb.0:
+; GFX12-SDAG-NEXT:    s_clause 0x1
+; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX12-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX12-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
+; GFX12-SDAG-NEXT:    s_endpgm
+;
+; GFX12-GISEL-LABEL: v_permlane16_b32_i_tid_bc_i32:
+; GFX12-GISEL:       ; %bb.0:
+; GFX12-GISEL-NEXT:    s_clause 0x1
+; GFX12-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX12-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
+; GFX12-GISEL-NEXT:    s_endpgm
+;
+; GFX13-SDAG-LABEL: v_permlane16_b32_i_tid_bc_i32:
+; GFX13-SDAG:       ; %bb.0:
+; GFX13-SDAG-NEXT:    s_clause 0x1
+; GFX13-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX13-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX13-SDAG-NEXT:    s_mov_b32 s5, 0x3ff
+; GFX13-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
+; GFX13-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX13-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX13-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
+; GFX13-SDAG-NEXT:    s_endpgm
+;
+; GFX13-GISEL-LABEL: v_permlane16_b32_i_tid_bc_i32:
+; GFX13-GISEL:       ; %bb.0:
+; GFX13-GISEL-NEXT:    s_clause 0x1
+; GFX13-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-GISEL-NEXT:    s_mov_b32 s4, 0x3ff
+; GFX13-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX13-GISEL-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX13-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
+; GFX13-GISEL-NEXT:    s_endpgm
   %tidx = call i32 @llvm.amdgcn.workitem.id.x()
   %undef = freeze i32 poison
   %v = call i32 @llvm.amdgcn.permlane16.i32(i32 %undef, i32 %tidx, i32 %src1, i32 %src2, i1 false, i1 true)
@@ -8597,53 +8913,109 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_bc_i32(ptr addrspace(1) %out,
 }
 
 define amdgpu_kernel void @v_permlane16_b32_i_tid_bc_f32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-LABEL: v_permlane16_b32_i_tid_bc_f32:
-; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_clause 0x1
-; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX10-NEXT:    global_store_dword v1, v0, s[2:3]
-; GFX10-NEXT:    s_endpgm
+; GFX10-SDAG-LABEL: v_permlane16_b32_i_tid_bc_f32:
+; GFX10-SDAG:       ; %bb.0:
+; GFX10-SDAG-NEXT:    s_clause 0x1
+; GFX10-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-SDAG-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v1, s4
+; GFX10-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX10-SDAG-NEXT:    global_store_dword v2, v1, s[2:3]
+; GFX10-SDAG-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: v_permlane16_b32_i_tid_bc_f32:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
-; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX11-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX11-NEXT:    s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_i_tid_bc_f32:
+; GFX10-GISEL:       ; %bb.0:
+; GFX10-GISEL-NEXT:    s_clause 0x1
+; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, 0
+; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX10-GISEL-NEXT:    global_store_dword v1, v0, s[2:3]
+; GFX10-GISEL-NEXT:    s_endpgm
 ;
-; GFX12-LABEL: v_permlane16_b32_i_tid_bc_f32:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_clause 0x1
-; GFX12-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
-; GFX12-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX12-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX12-NEXT:    s_endpgm
+; GFX11-SDAG-LABEL: v_permlane16_b32_i_tid_bc_f32:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_clause 0x1
+; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX11-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
+; GFX11-SDAG-NEXT:    s_endpgm
 ;
-; GFX13-LABEL: v_permlane16_b32_i_tid_bc_f32:
-; GFX13:       ; %bb.0:
-; GFX13-NEXT:    s_clause 0x1
-; GFX13-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-NEXT:    s_mov_b32 s4, 0x3ff
-; GFX13-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
-; GFX13-NEXT:    s_wait_kmcnt 0x0
-; GFX13-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX13-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX13-NEXT:    s_endpgm
+; GFX11-GISEL-LABEL: v_permlane16_b32_i_tid_bc_f32:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_clause 0x1
+; GFX11-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX11-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
+; GFX11-GISEL-NEXT:    s_endpgm
+;
+; GFX12-SDAG-LABEL: v_permlane16_b32_i_tid_bc_f32:
+; GFX12-SDAG:       ; %bb.0:
+; GFX12-SDAG-NEXT:    s_clause 0x1
+; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX12-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX12-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
+; GFX12-SDAG-NEXT:    s_endpgm
+;
+; GFX12-GISEL-LABEL: v_permlane16_b32_i_tid_bc_f32:
+; GFX12-GISEL:       ; %bb.0:
+; GFX12-GISEL-NEXT:    s_clause 0x1
+; GFX12-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX12-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
+; GFX12-GISEL-NEXT:    s_endpgm
+;
+; GFX13-SDAG-LABEL: v_permlane16_b32_i_tid_bc_f32:
+; GFX13-SDAG:       ; %bb.0:
+; GFX13-SDAG-NEXT:    s_clause 0x1
+; GFX13-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX13-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX13-SDAG-NEXT:    s_mov_b32 s5, 0x3ff
+; GFX13-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
+; GFX13-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX13-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX13-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
+; GFX13-SDAG-NEXT:    s_endpgm
+;
+; GFX13-GISEL-LABEL: v_permlane16_b32_i_tid_bc_f32:
+; GFX13-GISEL:       ; %bb.0:
+; GFX13-GISEL-NEXT:    s_clause 0x1
+; GFX13-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-GISEL-NEXT:    s_mov_b32 s4, 0x3ff
+; GFX13-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX13-GISEL-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX13-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
+; GFX13-GISEL-NEXT:    s_endpgm
   %tidx = call i32 @llvm.amdgcn.workitem.id.x()
   %tidx_f32 = bitcast i32 %tidx to float
   %undef = freeze float poison
@@ -8658,11 +9030,14 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_bc_i64(ptr addrspace(1) %out,
 ; GFX10-SDAG-NEXT:    s_clause 0x1
 ; GFX10-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
 ; GFX10-SDAG-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v3, 0
+; GFX10-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, s5
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v1, s4
 ; GFX10-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_permlane16_b32 v1, v2, s0, s1 op_sel:[0,1]
-; GFX10-SDAG-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX10-SDAG-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v2, v3, s0, s1 op_sel:[0,1]
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX10-SDAG-NEXT:    global_store_dwordx2 v3, v[1:2], s[2:3]
 ; GFX10-SDAG-NEXT:    s_endpgm
 ;
 ; GFX10-GISEL-LABEL: v_permlane16_b32_i_tid_bc_i64:
@@ -8682,13 +9057,14 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_bc_i64(ptr addrspace(1) %out,
 ; GFX11-SDAG-NEXT:    s_clause 0x1
 ; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
 ; GFX11-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, 0
-; GFX11-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT:    v_permlane16_b32 v1, v2, s0, s1 op_sel:[0,1]
-; GFX11-SDAG-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX11-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v2, v3, s0, s1 op_sel:[0,1]
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX11-SDAG-NEXT:    global_store_b64 v3, v[1:2], s[2:3]
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
 ; GFX11-GISEL-LABEL: v_permlane16_b32_i_tid_bc_i64:
@@ -8710,13 +9086,14 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_bc_i64(ptr addrspace(1) %out,
 ; GFX12-SDAG-NEXT:    s_clause 0x1
 ; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
 ; GFX12-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-SDAG-NEXT:    v_mov_b32_e32 v2, 0
-; GFX12-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-NEXT:    v_permlane16_b32 v1, v2, s0, s1 op_sel:[0,1]
-; GFX12-SDAG-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX12-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v2, v3, s0, s1 op_sel:[0,1]
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX12-SDAG-NEXT:    global_store_b64 v3, v[1:2], s[2:3]
 ; GFX12-SDAG-NEXT:    s_endpgm
 ;
 ; GFX12-GISEL-LABEL: v_permlane16_b32_i_tid_bc_i64:
@@ -8738,14 +9115,16 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_bc_i64(ptr addrspace(1) %out,
 ; GFX13-SDAG-NEXT:    s_clause 0x1
 ; GFX13-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
 ; GFX13-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT:    s_mov_b32 s4, 0x3ff
-; GFX13-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-SDAG-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX13-SDAG-NEXT:    s_mov_b32 s5, 0x3ff
+; GFX13-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
 ; GFX13-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT:    v_permlane16_b32 v1, v2, s0, s1 op_sel:[0,1]
+; GFX13-SDAG-NEXT:    v_permlane16_b32 v2, v3, s0, s1 op_sel:[0,1]
 ; GFX13-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX13-SDAG-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX13-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX13-SDAG-NEXT:    global_store_b64 v3, v[1:2], s[2:3]
 ; GFX13-SDAG-NEXT:    s_endpgm
 ;
 ; GFX13-GISEL-LABEL: v_permlane16_b32_i_tid_bc_i64:
@@ -8773,15 +9152,18 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_bc_i64(ptr addrspace(1) %out,
 define amdgpu_kernel void @v_permlane16_b32_i_tid_bc_f64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
 ; GFX10-SDAG-LABEL: v_permlane16_b32_i_tid_bc_f64:
 ; GFX10-SDAG:       ; %bb.0:
-; GFX10-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
 ; GFX10-SDAG-NEXT:    s_clause 0x1
 ; GFX10-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
 ; GFX10-SDAG-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v4, 0
+; GFX10-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v3, s5
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, s4
 ; GFX10-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_permlane16_b32 v1, v1, s0, s1 op_sel:[0,1]
-; GFX10-SDAG-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX10-SDAG-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v3, v1, s0, s1 op_sel:[0,1]
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v2, v0, s0, s1 op_sel:[0,1]
+; GFX10-SDAG-NEXT:    global_store_dwordx2 v4, v[2:3], s[2:3]
 ; GFX10-SDAG-NEXT:    s_endpgm
 ;
 ; GFX10-GISEL-LABEL: v_permlane16_b32_i_tid_bc_f64:
@@ -8799,17 +9181,20 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_bc_f64(ptr addrspace(1) %out,
 ;
 ; GFX11-SDAG-LABEL: v_permlane16_b32_i_tid_bc_f64:
 ; GFX11-SDAG:       ; %bb.0:
-; GFX11-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GFX11-SDAG-NEXT:    s_clause 0x1
 ; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
 ; GFX11-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT:    v_mov_b32_e32 v4, 0
+; GFX11-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v3, s5 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX11-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
 ; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT:    v_permlane16_b32 v1, v1, s0, s1 op_sel:[0,1]
-; GFX11-SDAG-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX11-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v3, v1, s0, s1 op_sel:[0,1]
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v2, v0, s0, s1 op_sel:[0,1]
+; GFX11-SDAG-NEXT:    global_store_b64 v4, v[2:3], s[2:3]
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
 ; GFX11-GISEL-LABEL: v_permlane16_b32_i_tid_bc_f64:
@@ -8829,17 +9214,20 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_bc_f64(ptr addrspace(1) %out,
 ;
 ; GFX12-SDAG-LABEL: v_permlane16_b32_i_tid_bc_f64:
 ; GFX12-SDAG:       ; %bb.0:
-; GFX12-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GFX12-SDAG-NEXT:    s_clause 0x1
 ; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
 ; GFX12-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX12-SDAG-NEXT:    v_mov_b32_e32 v4, 0
+; GFX12-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v3, s5 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT:    v_mov_b32_e32 v2, s4
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-NEXT:    v_permlane16_b32 v1, v1, s0, s1 op_sel:[0,1]
-; GFX12-SDAG-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX12-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v3, v1, s0, s1 op_sel:[0,1]
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v2, v0, s0, s1 op_sel:[0,1]
+; GFX12-SDAG-NEXT:    global_store_b64 v4, v[2:3], s[2:3]
 ; GFX12-SDAG-NEXT:    s_endpgm
 ;
 ; GFX12-GISEL-LABEL: v_permlane16_b32_i_tid_bc_f64:
@@ -8859,18 +9247,20 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_bc_f64(ptr addrspace(1) %out,
 ;
 ; GFX13-SDAG-LABEL: v_permlane16_b32_i_tid_bc_f64:
 ; GFX13-SDAG:       ; %bb.0:
-; GFX13-SDAG-NEXT:    s_mov_b32 s0, 0x3ff
-; GFX13-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX13-SDAG-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s0, v0 bitop3:0x40
 ; GFX13-SDAG-NEXT:    s_clause 0x1
 ; GFX13-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
 ; GFX13-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX13-SDAG-NEXT:    s_mov_b32 s6, 0x3ff
+; GFX13-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v3, s5 :: v_dual_bitop2_b32 v0, s6, v0 bitop3:0x40
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v4, 0
 ; GFX13-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
 ; GFX13-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX13-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT:    v_permlane16_b32 v1, v1, s0, s1 op_sel:[0,1]
-; GFX13-SDAG-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX13-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-SDAG-NEXT:    v_permlane16_b32 v3, v1, s0, s1 op_sel:[0,1]
+; GFX13-SDAG-NEXT:    v_permlane16_b32 v2, v0, s0, s1 op_sel:[0,1]
+; GFX13-SDAG-NEXT:    global_store_b64 v4, v[2:3], s[2:3]
 ; GFX13-SDAG-NEXT:    s_endpgm
 ;
 ; GFX13-GISEL-LABEL: v_permlane16_b32_i_tid_bc_f64:
@@ -8898,53 +9288,109 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_bc_f64(ptr addrspace(1) %out,
 }
 
 define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_bc_i32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-LABEL: v_permlane16_b32_i_tid_fi_bc_i32:
-; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_clause 0x1
-; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX10-NEXT:    global_store_dword v1, v0, s[2:3]
-; GFX10-NEXT:    s_endpgm
+; GFX10-SDAG-LABEL: v_permlane16_b32_i_tid_fi_bc_i32:
+; GFX10-SDAG:       ; %bb.0:
+; GFX10-SDAG-NEXT:    s_clause 0x1
+; GFX10-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-SDAG-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v1, s4
+; GFX10-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX10-SDAG-NEXT:    global_store_dword v2, v1, s[2:3]
+; GFX10-SDAG-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: v_permlane16_b32_i_tid_fi_bc_i32:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
-; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX11-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX11-NEXT:    s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_i_tid_fi_bc_i32:
+; GFX10-GISEL:       ; %bb.0:
+; GFX10-GISEL-NEXT:    s_clause 0x1
+; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, 0
+; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
+; GFX10-GISEL-NEXT:    global_store_dword v1, v0, s[2:3]
+; GFX10-GISEL-NEXT:    s_endpgm
 ;
-; GFX12-LABEL: v_permlane16_b32_i_tid_fi_bc_i32:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_clause 0x1
-; GFX12-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
-; GFX12-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX12-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX12-NEXT:    s_endpgm
+; GFX11-SDAG-LABEL: v_permlane16_b32_i_tid_fi_bc_i32:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_clause 0x1
+; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX11-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
+; GFX11-SDAG-NEXT:    s_endpgm
 ;
-; GFX13-LABEL: v_permlane16_b32_i_tid_fi_bc_i32:
-; GFX13:       ; %bb.0:
-; GFX13-NEXT:    s_clause 0x1
-; GFX13-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-NEXT:    s_mov_b32 s4, 0x3ff
-; GFX13-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
-; GFX13-NEXT:    s_wait_kmcnt 0x0
-; GFX13-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX13-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX13-NEXT:    s_endpgm
+; GFX11-GISEL-LABEL: v_permlane16_b32_i_tid_fi_bc_i32:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_clause 0x1
+; GFX11-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
+; GFX11-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
+; GFX11-GISEL-NEXT:    s_endpgm
+;
+; GFX12-SDAG-LABEL: v_permlane16_b32_i_tid_fi_bc_i32:
+; GFX12-SDAG:       ; %bb.0:
+; GFX12-SDAG-NEXT:    s_clause 0x1
+; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX12-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX12-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
+; GFX12-SDAG-NEXT:    s_endpgm
+;
+; GFX12-GISEL-LABEL: v_permlane16_b32_i_tid_fi_bc_i32:
+; GFX12-GISEL:       ; %bb.0:
+; GFX12-GISEL-NEXT:    s_clause 0x1
+; GFX12-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
+; GFX12-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
+; GFX12-GISEL-NEXT:    s_endpgm
+;
+; GFX13-SDAG-LABEL: v_permlane16_b32_i_tid_fi_bc_i32:
+; GFX13-SDAG:       ; %bb.0:
+; GFX13-SDAG-NEXT:    s_clause 0x1
+; GFX13-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX13-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX13-SDAG-NEXT:    s_mov_b32 s5, 0x3ff
+; GFX13-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
+; GFX13-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX13-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX13-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
+; GFX13-SDAG-NEXT:    s_endpgm
+;
+; GFX13-GISEL-LABEL: v_permlane16_b32_i_tid_fi_bc_i32:
+; GFX13-GISEL:       ; %bb.0:
+; GFX13-GISEL-NEXT:    s_clause 0x1
+; GFX13-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-GISEL-NEXT:    s_mov_b32 s4, 0x3ff
+; GFX13-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX13-GISEL-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
+; GFX13-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
+; GFX13-GISEL-NEXT:    s_endpgm
   %tidx = call i32 @llvm.amdgcn.workitem.id.x()
   %undef = freeze i32 poison
   %v = call i32 @llvm.amdgcn.permlane16.i32(i32 %undef, i32 %tidx, i32 %src1, i32 %src2, i1 true, i1 true)
@@ -8953,53 +9399,109 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_bc_i32(ptr addrspace(1) %ou
 }
 
 define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_bc_f32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-LABEL: v_permlane16_b32_i_tid_fi_bc_f32:
-; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_clause 0x1
-; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX10-NEXT:    global_store_dword v1, v0, s[2:3]
-; GFX10-NEXT:    s_endpgm
+; GFX10-SDAG-LABEL: v_permlane16_b32_i_tid_fi_bc_f32:
+; GFX10-SDAG:       ; %bb.0:
+; GFX10-SDAG-NEXT:    s_clause 0x1
+; GFX10-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-SDAG-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v1, s4
+; GFX10-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX10-SDAG-NEXT:    global_store_dword v2, v1, s[2:3]
+; GFX10-SDAG-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: v_permlane16_b32_i_tid_fi_bc_f32:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
-; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX11-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX11-NEXT:    s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_i_tid_fi_bc_f32:
+; GFX10-GISEL:       ; %bb.0:
+; GFX10-GISEL-NEXT:    s_clause 0x1
+; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, 0
+; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
+; GFX10-GISEL-NEXT:    global_store_dword v1, v0, s[2:3]
+; GFX10-GISEL-NEXT:    s_endpgm
 ;
-; GFX12-LABEL: v_permlane16_b32_i_tid_fi_bc_f32:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_clause 0x1
-; GFX12-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
-; GFX12-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX12-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX12-NEXT:    s_endpgm
+; GFX11-SDAG-LABEL: v_permlane16_b32_i_tid_fi_bc_f32:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_clause 0x1
+; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX11-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
+; GFX11-SDAG-NEXT:    s_endpgm
 ;
-; GFX13-LABEL: v_permlane16_b32_i_tid_fi_bc_f32:
-; GFX13:       ; %bb.0:
-; GFX13-NEXT:    s_clause 0x1
-; GFX13-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-NEXT:    s_mov_b32 s4, 0x3ff
-; GFX13-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
-; GFX13-NEXT:    s_wait_kmcnt 0x0
-; GFX13-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX13-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX13-NEXT:    s_endpgm
+; GFX11-GISEL-LABEL: v_permlane16_b32_i_tid_fi_bc_f32:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_clause 0x1
+; GFX11-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
+; GFX11-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
+; GFX11-GISEL-NEXT:    s_endpgm
+;
+; GFX12-SDAG-LABEL: v_permlane16_b32_i_tid_fi_bc_f32:
+; GFX12-SDAG:       ; %bb.0:
+; GFX12-SDAG-NEXT:    s_clause 0x1
+; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX12-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX12-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
+; GFX12-SDAG-NEXT:    s_endpgm
+;
+; GFX12-GISEL-LABEL: v_permlane16_b32_i_tid_fi_bc_f32:
+; GFX12-GISEL:       ; %bb.0:
+; GFX12-GISEL-NEXT:    s_clause 0x1
+; GFX12-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
+; GFX12-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
+; GFX12-GISEL-NEXT:    s_endpgm
+;
+; GFX13-SDAG-LABEL: v_permlane16_b32_i_tid_fi_bc_f32:
+; GFX13-SDAG:       ; %bb.0:
+; GFX13-SDAG-NEXT:    s_clause 0x1
+; GFX13-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX13-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX13-SDAG-NEXT:    s_mov_b32 s5, 0x3ff
+; GFX13-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
+; GFX13-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX13-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX13-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
+; GFX13-SDAG-NEXT:    s_endpgm
+;
+; GFX13-GISEL-LABEL: v_permlane16_b32_i_tid_fi_bc_f32:
+; GFX13-GISEL:       ; %bb.0:
+; GFX13-GISEL-NEXT:    s_clause 0x1
+; GFX13-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-GISEL-NEXT:    s_mov_b32 s4, 0x3ff
+; GFX13-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX13-GISEL-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
+; GFX13-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
+; GFX13-GISEL-NEXT:    s_endpgm
   %tidx = call i32 @llvm.amdgcn.workitem.id.x()
   %tidx_f32 = bitcast i32 %tidx to float
   %undef = freeze float poison
@@ -9014,11 +9516,14 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_bc_i64(ptr addrspace(1) %ou
 ; GFX10-SDAG-NEXT:    s_clause 0x1
 ; GFX10-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
 ; GFX10-SDAG-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v3, 0
+; GFX10-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, s5
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v1, s4
 ; GFX10-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_permlane16_b32 v1, v2, s0, s1 op_sel:[1,1]
-; GFX10-SDAG-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX10-SDAG-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v2, v3, s0, s1 op_sel:[1,1]
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX10-SDAG-NEXT:    global_store_dwordx2 v3, v[1:2], s[2:3]
 ; GFX10-SDAG-NEXT:    s_endpgm
 ;
 ; GFX10-GISEL-LABEL: v_permlane16_b32_i_tid_fi_bc_i64:
@@ -9038,13 +9543,14 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_bc_i64(ptr addrspace(1) %ou
 ; GFX11-SDAG-NEXT:    s_clause 0x1
 ; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
 ; GFX11-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, 0
-; GFX11-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT:    v_permlane16_b32 v1, v2, s0, s1 op_sel:[1,1]
-; GFX11-SDAG-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX11-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v2, v3, s0, s1 op_sel:[1,1]
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX11-SDAG-NEXT:    global_store_b64 v3, v[1:2], s[2:3]
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
 ; GFX11-GISEL-LABEL: v_permlane16_b32_i_tid_fi_bc_i64:
@@ -9066,13 +9572,14 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_bc_i64(ptr addrspace(1) %ou
 ; GFX12-SDAG-NEXT:    s_clause 0x1
 ; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
 ; GFX12-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-SDAG-NEXT:    v_mov_b32_e32 v2, 0
-; GFX12-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-NEXT:    v_permlane16_b32 v1, v2, s0, s1 op_sel:[1,1]
-; GFX12-SDAG-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX12-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v2, v3, s0, s1 op_sel:[1,1]
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX12-SDAG-NEXT:    global_store_b64 v3, v[1:2], s[2:3]
 ; GFX12-SDAG-NEXT:    s_endpgm
 ;
 ; GFX12-GISEL-LABEL: v_permlane16_b32_i_tid_fi_bc_i64:
@@ -9094,14 +9601,16 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_bc_i64(ptr addrspace(1) %ou
 ; GFX13-SDAG-NEXT:    s_clause 0x1
 ; GFX13-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
 ; GFX13-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT:    s_mov_b32 s4, 0x3ff
-; GFX13-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-SDAG-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX13-SDAG-NEXT:    s_mov_b32 s5, 0x3ff
+; GFX13-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
 ; GFX13-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT:    v_permlane16_b32 v1, v2, s0, s1 op_sel:[1,1]
+; GFX13-SDAG-NEXT:    v_permlane16_b32 v2, v3, s0, s1 op_sel:[1,1]
 ; GFX13-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX13-SDAG-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX13-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-SDAG-NEXT:    v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX13-SDAG-NEXT:    global_store_b64 v3, v[1:2], s[2:3]
 ; GFX13-SDAG-NEXT:    s_endpgm
 ;
 ; GFX13-GISEL-LABEL: v_permlane16_b32_i_tid_fi_bc_i64:
@@ -9129,15 +9638,18 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_bc_i64(ptr addrspace(1) %ou
 define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_bc_f64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
 ; GFX10-SDAG-LABEL: v_permlane16_b32_i_tid_fi_bc_f64:
 ; GFX10-SDAG:       ; %bb.0:
-; GFX10-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
 ; GFX10-SDAG-NEXT:    s_clause 0x1
 ; GFX10-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
 ; GFX10-SDAG-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v4, 0
+; GFX10-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v3, s5
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, s4
 ; GFX10-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_permlane16_b32 v1, v1, s0, s1 op_sel:[1,1]
-; GFX10-SDAG-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX10-SDAG-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v3, v1, s0, s1 op_sel:[1,1]
+; GFX10-SDAG-NEXT:    v_permlane16_b32 v2, v0, s0, s1 op_sel:[1,1]
+; GFX10-SDAG-NEXT:    global_store_dwordx2 v4, v[2:3], s[2:3]
 ; GFX10-SDAG-NEXT:    s_endpgm
 ;
 ; GFX10-GISEL-LABEL: v_permlane16_b32_i_tid_fi_bc_f64:
@@ -9155,17 +9667,20 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_bc_f64(ptr addrspace(1) %ou
 ;
 ; GFX11-SDAG-LABEL: v_permlane16_b32_i_tid_fi_bc_f64:
 ; GFX11-SDAG:       ; %bb.0:
-; GFX11-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GFX11-SDAG-NEXT:    s_clause 0x1
 ; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
 ; GFX11-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT:    v_mov_b32_e32 v4, 0
+; GFX11-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v3, s5 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX11-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
 ; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT:    v_permlane16_b32 v1, v1, s0, s1 op_sel:[1,1]
-; GFX11-SDAG-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX11-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v3, v1, s0, s1 op_sel:[1,1]
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-SDAG-NEXT:    v_permlane16_b32 v2, v0, s0, s1 op_sel:[1,1]
+; GFX11-SDAG-NEXT:    global_store_b64 v4, v[2:3], s[2:3]
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
 ; GFX11-GISEL-LABEL: v_permlane16_b32_i_tid_fi_bc_f64:
@@ -9185,17 +9700,20 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_bc_f64(ptr addrspace(1) %ou
 ;
 ; GFX12-SDAG-LABEL: v_permlane16_b32_i_tid_fi_bc_f64:
 ; GFX12-SDAG:       ; %bb.0:
-; GFX12-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GFX12-SDAG-NEXT:    s_clause 0x1
 ; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
 ; GFX12-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX12-SDAG-NEXT:    v_mov_b32_e32 v4, 0
+; GFX12-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v3, s5 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT:    v_mov_b32_e32 v2, s4
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-NEXT:    v_permlane16_b32 v1, v1, s0, s1 op_sel:[1,1]
-; GFX12-SDAG-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX12-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v3, v1, s0, s1 op_sel:[1,1]
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT:    v_permlane16_b32 v2, v0, s0, s1 op_sel:[1,1]
+; GFX12-SDAG-NEXT:    global_store_b64 v4, v[2:3], s[2:3]
 ; GFX12-SDAG-NEXT:    s_endpgm
 ;
 ; GFX12-GISEL-LABEL: v_permlane16_b32_i_tid_fi_bc_f64:
@@ -9215,18 +9733,20 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_bc_f64(ptr addrspace(1) %ou
 ;
 ; GFX13-SDAG-LABEL: v_permlane16_b32_i_tid_fi_bc_f64:
 ; GFX13-SDAG:       ; %bb.0:
-; GFX13-SDAG-NEXT:    s_mov_b32 s0, 0x3ff
-; GFX13-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX13-SDAG-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s0, v0 bitop3:0x40
 ; GFX13-SDAG-NEXT:    s_clause 0x1
 ; GFX13-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
 ; GFX13-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX13-SDAG-NEXT:    s_mov_b32 s6, 0x3ff
+; GFX13-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v3, s5 :: v_dual_bitop2_b32 v0, s6, v0 bitop3:0x40
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v4, 0
 ; GFX13-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
 ; GFX13-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX13-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT:    v_permlane16_b32 v1, v1, s0, s1 op_sel:[1,1]
-; GFX13-SDAG-NEXT:    v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX13-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-SDAG-NEXT:    v_permlane16_b32 v3, v1, s0, s1 op_sel:[1,1]
+; GFX13-SDAG-NEXT:    v_permlane16_b32 v2, v0, s0, s1 op_sel:[1,1]
+; GFX13-SDAG-NEXT:    global_store_b64 v4, v[2:3], s[2:3]
 ; GFX13-SDAG-NEXT:    s_endpgm
 ;
 ; GFX13-GISEL-LABEL: v_permlane16_b32_i_tid_fi_bc_f64:
@@ -9608,358 +10128,488 @@ define amdgpu_kernel void @v_permlanex16_b32_tid_tid_f64(ptr addrspace(1) %out,
 }
 
 define amdgpu_kernel void @v_permlanex16_b32_undef_tid_i32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-LABEL: v_permlanex16_b32_undef_tid_i32:
-; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_clause 0x1
-; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    v_permlanex16_b32 v0, v0, s0, s1
-; GFX10-NEXT:    global_store_dword v1, v0, s[2:3]
-; GFX10-NEXT:    s_endpgm
-;
-; GFX11-LABEL: v_permlanex16_b32_undef_tid_i32:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
-; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_permlanex16_b32 v0, v0, s0, s1
-; GFX11-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX11-NEXT:    s_endpgm
-;
-; GFX12-LABEL: v_permlanex16_b32_undef_tid_i32:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_clause 0x1
-; GFX12-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
-; GFX12-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT:    v_permlanex16_b32 v0, v0, s0, s1
-; GFX12-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX12-NEXT:    s_endpgm
-;
-; GFX13-LABEL: v_permlanex16_b32_undef_tid_i32:
-; GFX13:       ; %bb.0:
-; GFX13-NEXT:    s_clause 0x1
-; GFX13-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-NEXT:    s_mov_b32 s4, 0x3ff
-; GFX13-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
-; GFX13-NEXT:    s_wait_kmcnt 0x0
-; GFX13-NEXT:    v_permlanex16_b32 v0, v0, s0, s1
-; GFX13-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX13-NEXT:    s_endpgm
-  %tidx = call i32 @llvm.amdgcn.workitem.id.x()
-  %undef = freeze i32 poison
-  %v = call i32 @llvm.amdgcn.permlanex16.i32(i32 %undef, i32 %tidx, i32 %src1, i32 %src2, i1 false, i1 false)
-  store i32 %v, ptr addrspace(1) %out
-  ret void
-}
-
-define amdgpu_kernel void @v_permlanex16_b32_undef_tid_f32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-LABEL: v_permlanex16_b32_undef_tid_f32:
-; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_clause 0x1
-; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    v_permlanex16_b32 v0, v0, s0, s1
-; GFX10-NEXT:    global_store_dword v1, v0, s[2:3]
-; GFX10-NEXT:    s_endpgm
-;
-; GFX11-LABEL: v_permlanex16_b32_undef_tid_f32:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
-; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_permlanex16_b32 v0, v0, s0, s1
-; GFX11-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX11-NEXT:    s_endpgm
-;
-; GFX12-LABEL: v_permlanex16_b32_undef_tid_f32:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_clause 0x1
-; GFX12-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
-; GFX12-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT:    v_permlanex16_b32 v0, v0, s0, s1
-; GFX12-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX12-NEXT:    s_endpgm
-;
-; GFX13-LABEL: v_permlanex16_b32_undef_tid_f32:
-; GFX13:       ; %bb.0:
-; GFX13-NEXT:    s_clause 0x1
-; GFX13-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-NEXT:    s_mov_b32 s4, 0x3ff
-; GFX13-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
-; GFX13-NEXT:    s_wait_kmcnt 0x0
-; GFX13-NEXT:    v_permlanex16_b32 v0, v0, s0, s1
-; GFX13-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX13-NEXT:    s_endpgm
-  %tidx = call i32 @llvm.amdgcn.workitem.id.x()
-  %tidx_f32 = bitcast i32 %tidx to float
-  %undef = freeze float poison
-  %v = call float @llvm.amdgcn.permlanex16.f32(float %undef, float %tidx_f32, i32 %src1, i32 %src2, i1 false, i1 false)
-  store float %v, ptr addrspace(1) %out
-  ret void
-}
-
-define amdgpu_kernel void @v_permlanex16_b32_undef_tid_i64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_undef_tid_i64:
+; GFX10-SDAG-LABEL: v_permlanex16_b32_undef_tid_i32:
 ; GFX10-SDAG:       ; %bb.0:
 ; GFX10-SDAG-NEXT:    s_clause 0x1
 ; GFX10-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
 ; GFX10-SDAG-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
 ; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v1, s4
 ; GFX10-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_permlanex16_b32 v1, v2, s0, s1
-; GFX10-SDAG-NEXT:    v_permlanex16_b32 v0, v0, s0, s1
-; GFX10-SDAG-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1
+; GFX10-SDAG-NEXT:    global_store_dword v2, v1, s[2:3]
 ; GFX10-SDAG-NEXT:    s_endpgm
 ;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_undef_tid_i64:
+; GFX10-GISEL-LABEL: v_permlanex16_b32_undef_tid_i32:
 ; GFX10-GISEL:       ; %bb.0:
 ; GFX10-GISEL-NEXT:    s_clause 0x1
 ; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
 ; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1
-; GFX10-GISEL-NEXT:    v_permlanex16_b32 v1, v2, s0, s1
-; GFX10-GISEL-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-GISEL-NEXT:    global_store_dword v1, v0, s[2:3]
 ; GFX10-GISEL-NEXT:    s_endpgm
 ;
-; GFX11-SDAG-LABEL: v_permlanex16_b32_undef_tid_i64:
+; GFX11-SDAG-LABEL: v_permlanex16_b32_undef_tid_i32:
 ; GFX11-SDAG:       ; %bb.0:
 ; GFX11-SDAG-NEXT:    s_clause 0x1
 ; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
 ; GFX11-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
 ; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, 0
-; GFX11-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT:    v_permlanex16_b32 v1, v2, s0, s1
-; GFX11-SDAG-NEXT:    v_permlanex16_b32 v0, v0, s0, s1
-; GFX11-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1
+; GFX11-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
-; GFX11-GISEL-LABEL: v_permlanex16_b32_undef_tid_i64:
+; GFX11-GISEL-LABEL: v_permlanex16_b32_undef_tid_i32:
 ; GFX11-GISEL:       ; %bb.0:
 ; GFX11-GISEL-NEXT:    s_clause 0x1
 ; GFX11-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
 ; GFX11-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1
-; GFX11-GISEL-NEXT:    v_permlanex16_b32 v1, v2, s0, s1
-; GFX11-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
 ; GFX11-GISEL-NEXT:    s_endpgm
 ;
-; GFX12-SDAG-LABEL: v_permlanex16_b32_undef_tid_i64:
+; GFX12-SDAG-LABEL: v_permlanex16_b32_undef_tid_i32:
 ; GFX12-SDAG:       ; %bb.0:
 ; GFX12-SDAG-NEXT:    s_clause 0x1
 ; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
 ; GFX12-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v2, 0
-; GFX12-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-NEXT:    v_permlanex16_b32 v1, v2, s0, s1
-; GFX12-SDAG-NEXT:    v_permlanex16_b32 v0, v0, s0, s1
-; GFX12-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1
+; GFX12-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
 ; GFX12-SDAG-NEXT:    s_endpgm
 ;
-; GFX12-GISEL-LABEL: v_permlanex16_b32_undef_tid_i64:
+; GFX12-GISEL-LABEL: v_permlanex16_b32_undef_tid_i32:
 ; GFX12-GISEL:       ; %bb.0:
 ; GFX12-GISEL-NEXT:    s_clause 0x1
 ; GFX12-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
 ; GFX12-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX12-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1
-; GFX12-GISEL-NEXT:    v_permlanex16_b32 v1, v2, s0, s1
-; GFX12-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
 ; GFX12-GISEL-NEXT:    s_endpgm
 ;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_undef_tid_i64:
+; GFX13-SDAG-LABEL: v_permlanex16_b32_undef_tid_i32:
 ; GFX13-SDAG:       ; %bb.0:
 ; GFX13-SDAG-NEXT:    s_clause 0x1
 ; GFX13-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
 ; GFX13-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT:    s_mov_b32 s4, 0x3ff
+; GFX13-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX13-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX13-SDAG-NEXT:    s_mov_b32 s5, 0x3ff
 ; GFX13-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-SDAG-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
 ; GFX13-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT:    v_permlanex16_b32 v1, v2, s0, s1
-; GFX13-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX13-SDAG-NEXT:    v_permlanex16_b32 v0, v0, s0, s1
-; GFX13-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1
+; GFX13-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
 ; GFX13-SDAG-NEXT:    s_endpgm
 ;
-; GFX13-GISEL-LABEL: v_permlanex16_b32_undef_tid_i64:
+; GFX13-GISEL-LABEL: v_permlanex16_b32_undef_tid_i32:
 ; GFX13-GISEL:       ; %bb.0:
 ; GFX13-GISEL-NEXT:    s_clause 0x1
 ; GFX13-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
 ; GFX13-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
 ; GFX13-GISEL-NEXT:    s_mov_b32 s4, 0x3ff
 ; GFX13-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-GISEL-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
 ; GFX13-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX13-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1
-; GFX13-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX13-GISEL-NEXT:    v_permlanex16_b32 v1, v2, s0, s1
-; GFX13-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
 ; GFX13-GISEL-NEXT:    s_endpgm
   %tidx = call i32 @llvm.amdgcn.workitem.id.x()
-  %tidx_i64 = zext i32 %tidx to i64
-  %undef = freeze i64 poison
-  %v = call i64 @llvm.amdgcn.permlanex16.i64(i64 %undef, i64 %tidx_i64, i32 %src1, i32 %src2, i1 false, i1 false)
-  store i64 %v, ptr addrspace(1) %out
+  %undef = freeze i32 poison
+  %v = call i32 @llvm.amdgcn.permlanex16.i32(i32 %undef, i32 %tidx, i32 %src1, i32 %src2, i1 false, i1 false)
+  store i32 %v, ptr addrspace(1) %out
   ret void
 }
 
-define amdgpu_kernel void @v_permlanex16_b32_undef_tid_f64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_undef_tid_f64:
+define amdgpu_kernel void @v_permlanex16_b32_undef_tid_f32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
+; GFX10-SDAG-LABEL: v_permlanex16_b32_undef_tid_f32:
 ; GFX10-SDAG:       ; %bb.0:
-; GFX10-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
 ; GFX10-SDAG-NEXT:    s_clause 0x1
 ; GFX10-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
 ; GFX10-SDAG-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
 ; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v1, s4
 ; GFX10-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_permlanex16_b32 v1, v1, s0, s1
-; GFX10-SDAG-NEXT:    v_permlanex16_b32 v0, v0, s0, s1
-; GFX10-SDAG-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1
+; GFX10-SDAG-NEXT:    global_store_dword v2, v1, s[2:3]
 ; GFX10-SDAG-NEXT:    s_endpgm
 ;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_undef_tid_f64:
+; GFX10-GISEL-LABEL: v_permlanex16_b32_undef_tid_f32:
 ; GFX10-GISEL:       ; %bb.0:
-; GFX10-GISEL-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
 ; GFX10-GISEL-NEXT:    s_clause 0x1
 ; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
 ; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1
-; GFX10-GISEL-NEXT:    v_permlanex16_b32 v1, v1, s0, s1
-; GFX10-GISEL-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-GISEL-NEXT:    global_store_dword v1, v0, s[2:3]
 ; GFX10-GISEL-NEXT:    s_endpgm
 ;
-; GFX11-SDAG-LABEL: v_permlanex16_b32_undef_tid_f64:
+; GFX11-SDAG-LABEL: v_permlanex16_b32_undef_tid_f32:
 ; GFX11-SDAG:       ; %bb.0:
-; GFX11-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GFX11-SDAG-NEXT:    s_clause 0x1
 ; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
 ; GFX11-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
 ; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, 0
-; GFX11-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX11-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT:    v_permlanex16_b32 v1, v1, s0, s1
-; GFX11-SDAG-NEXT:    v_permlanex16_b32 v0, v0, s0, s1
-; GFX11-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1
+; GFX11-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
-; GFX11-GISEL-LABEL: v_permlanex16_b32_undef_tid_f64:
+; GFX11-GISEL-LABEL: v_permlanex16_b32_undef_tid_f32:
 ; GFX11-GISEL:       ; %bb.0:
-; GFX11-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GFX11-GISEL-NEXT:    s_clause 0x1
 ; GFX11-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
 ; GFX11-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-GISEL-NEXT:    v_mov_b32_e32 v2, 0
-; GFX11-GISEL-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX11-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1
-; GFX11-GISEL-NEXT:    v_permlanex16_b32 v1, v1, s0, s1
-; GFX11-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
 ; GFX11-GISEL-NEXT:    s_endpgm
 ;
-; GFX12-SDAG-LABEL: v_permlanex16_b32_undef_tid_f64:
+; GFX12-SDAG-LABEL: v_permlanex16_b32_undef_tid_f32:
 ; GFX12-SDAG:       ; %bb.0:
-; GFX12-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GFX12-SDAG-NEXT:    s_clause 0x1
 ; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
 ; GFX12-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v2, 0
-; GFX12-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX12-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-NEXT:    v_permlanex16_b32 v1, v1, s0, s1
-; GFX12-SDAG-NEXT:    v_permlanex16_b32 v0, v0, s0, s1
-; GFX12-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1
+; GFX12-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
 ; GFX12-SDAG-NEXT:    s_endpgm
 ;
-; GFX12-GISEL-LABEL: v_permlanex16_b32_undef_tid_f64:
+; GFX12-GISEL-LABEL: v_permlanex16_b32_undef_tid_f32:
 ; GFX12-GISEL:       ; %bb.0:
-; GFX12-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GFX12-GISEL-NEXT:    s_clause 0x1
 ; GFX12-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
 ; GFX12-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-GISEL-NEXT:    v_mov_b32_e32 v2, 0
-; GFX12-GISEL-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX12-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1
-; GFX12-GISEL-NEXT:    v_permlanex16_b32 v1, v1, s0, s1
-; GFX12-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
 ; GFX12-GISEL-NEXT:    s_endpgm
 ;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_undef_tid_f64:
+; GFX13-SDAG-LABEL: v_permlanex16_b32_undef_tid_f32:
 ; GFX13-SDAG:       ; %bb.0:
-; GFX13-SDAG-NEXT:    s_mov_b32 s0, 0x3ff
-; GFX13-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX13-SDAG-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s0, v0 bitop3:0x40
 ; GFX13-SDAG-NEXT:    s_clause 0x1
 ; GFX13-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
 ; GFX13-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX13-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX13-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX13-SDAG-NEXT:    s_mov_b32 s5, 0x3ff
+; GFX13-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
 ; GFX13-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT:    v_permlanex16_b32 v1, v1, s0, s1
-; GFX13-SDAG-NEXT:    v_permlanex16_b32 v0, v0, s0, s1
-; GFX13-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1
+; GFX13-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
 ; GFX13-SDAG-NEXT:    s_endpgm
 ;
-; GFX13-GISEL-LABEL: v_permlanex16_b32_undef_tid_f64:
+; GFX13-GISEL-LABEL: v_permlanex16_b32_undef_tid_f32:
 ; GFX13-GISEL:       ; %bb.0:
-; GFX13-GISEL-NEXT:    s_mov_b32 s0, 0x3ff
-; GFX13-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX13-GISEL-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s0, v0 bitop3:0x40
 ; GFX13-GISEL-NEXT:    s_clause 0x1
 ; GFX13-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
 ; GFX13-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-GISEL-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX13-GISEL-NEXT:    s_mov_b32 s4, 0x3ff
+; GFX13-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
 ; GFX13-GISEL-NEXT:    s_wait_kmcnt 0x0
-; GFX13-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX13-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1
-; GFX13-GISEL-NEXT:    v_permlanex16_b32 v1, v1, s0, s1
-; GFX13-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
 ; GFX13-GISEL-NEXT:    s_endpgm
   %tidx = call i32 @llvm.amdgcn.workitem.id.x()
   %tidx_f32 = bitcast i32 %tidx to float
-  %tidx_f64 = fpext float %tidx_f32 to double
-  %undef = freeze double poison
-  %v = call double @llvm.amdgcn.permlanex16.f64(double %undef, double %tidx_f64, i32 %src1, i32 %src2, i1 false, i1 false)
-  store double %v, ptr addrspace(1) %out
+  %undef = freeze float poison
+  %v = call float @llvm.amdgcn.permlanex16.f32(float %undef, float %tidx_f32, i32 %src1, i32 %src2, i1 false, i1 false)
+  store float %v, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @v_permlanex16_b32_undef_tid_i64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
+; GFX10-SDAG-LABEL: v_permlanex16_b32_undef_tid_i64:
+; GFX10-SDAG:       ; %bb.0:
+; GFX10-SDAG-NEXT:    s_clause 0x1
+; GFX10-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-SDAG-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v3, 0
+; GFX10-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, s5
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v1, s4
+; GFX10-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v2, v3, s0, s1
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1
+; GFX10-SDAG-NEXT:    global_store_dwordx2 v3, v[1:2], s[2:3]
+; GFX10-SDAG-NEXT:    s_endpgm
+;
+; GFX10-GISEL-LABEL: v_permlanex16_b32_undef_tid_i64:
+; GFX10-GISEL:       ; %bb.0:
+; GFX10-GISEL-NEXT:    s_clause 0x1
+; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v1, v2, s0, s1
+; GFX10-GISEL-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-GISEL-NEXT:    s_endpgm
+;
+; GFX11-SDAG-LABEL: v_permlanex16_b32_undef_tid_i64:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_clause 0x1
+; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v2, v3, s0, s1
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1
+; GFX11-SDAG-NEXT:    global_store_b64 v3, v[1:2], s[2:3]
+; GFX11-SDAG-NEXT:    s_endpgm
+;
+; GFX11-GISEL-LABEL: v_permlanex16_b32_undef_tid_i64:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_clause 0x1
+; GFX11-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v1, v2, s0, s1
+; GFX11-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-GISEL-NEXT:    s_endpgm
+;
+; GFX12-SDAG-LABEL: v_permlanex16_b32_undef_tid_i64:
+; GFX12-SDAG:       ; %bb.0:
+; GFX12-SDAG-NEXT:    s_clause 0x1
+; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v2, v3, s0, s1
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1
+; GFX12-SDAG-NEXT:    global_store_b64 v3, v[1:2], s[2:3]
+; GFX12-SDAG-NEXT:    s_endpgm
+;
+; GFX12-GISEL-LABEL: v_permlanex16_b32_undef_tid_i64:
+; GFX12-GISEL:       ; %bb.0:
+; GFX12-GISEL-NEXT:    s_clause 0x1
+; GFX12-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v1, v2, s0, s1
+; GFX12-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-GISEL-NEXT:    s_endpgm
+;
+; GFX13-SDAG-LABEL: v_permlanex16_b32_undef_tid_i64:
+; GFX13-SDAG:       ; %bb.0:
+; GFX13-SDAG-NEXT:    s_clause 0x1
+; GFX13-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX13-SDAG-NEXT:    s_mov_b32 s5, 0x3ff
+; GFX13-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
+; GFX13-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX13-SDAG-NEXT:    v_permlanex16_b32 v2, v3, s0, s1
+; GFX13-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX13-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1
+; GFX13-SDAG-NEXT:    global_store_b64 v3, v[1:2], s[2:3]
+; GFX13-SDAG-NEXT:    s_endpgm
+;
+; GFX13-GISEL-LABEL: v_permlanex16_b32_undef_tid_i64:
+; GFX13-GISEL:       ; %bb.0:
+; GFX13-GISEL-NEXT:    s_clause 0x1
+; GFX13-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-GISEL-NEXT:    s_mov_b32 s4, 0x3ff
+; GFX13-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-GISEL-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX13-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1
+; GFX13-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX13-GISEL-NEXT:    v_permlanex16_b32 v1, v2, s0, s1
+; GFX13-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-GISEL-NEXT:    s_endpgm
+  %tidx = call i32 @llvm.amdgcn.workitem.id.x()
+  %tidx_i64 = zext i32 %tidx to i64
+  %undef = freeze i64 poison
+  %v = call i64 @llvm.amdgcn.permlanex16.i64(i64 %undef, i64 %tidx_i64, i32 %src1, i32 %src2, i1 false, i1 false)
+  store i64 %v, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @v_permlanex16_b32_undef_tid_f64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
+; GFX10-SDAG-LABEL: v_permlanex16_b32_undef_tid_f64:
+; GFX10-SDAG:       ; %bb.0:
+; GFX10-SDAG-NEXT:    s_clause 0x1
+; GFX10-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-SDAG-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v4, 0
+; GFX10-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v3, s5
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, s4
+; GFX10-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v3, v1, s0, s1
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v2, v0, s0, s1
+; GFX10-SDAG-NEXT:    global_store_dwordx2 v4, v[2:3], s[2:3]
+; GFX10-SDAG-NEXT:    s_endpgm
+;
+; GFX10-GISEL-LABEL: v_permlanex16_b32_undef_tid_f64:
+; GFX10-GISEL:       ; %bb.0:
+; GFX10-GISEL-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-GISEL-NEXT:    s_clause 0x1
+; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v1, v1, s0, s1
+; GFX10-GISEL-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-GISEL-NEXT:    s_endpgm
+;
+; GFX11-SDAG-LABEL: v_permlanex16_b32_undef_tid_f64:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_clause 0x1
+; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT:    v_mov_b32_e32 v4, 0
+; GFX11-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v3, s5 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v3, v1, s0, s1
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v2, v0, s0, s1
+; GFX11-SDAG-NEXT:    global_store_b64 v4, v[2:3], s[2:3]
+; GFX11-SDAG-NEXT:    s_endpgm
+;
+; GFX11-GISEL-LABEL: v_permlanex16_b32_undef_tid_f64:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT:    s_clause 0x1
+; GFX11-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v1, v1, s0, s1
+; GFX11-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-GISEL-NEXT:    s_endpgm
+;
+; GFX12-SDAG-LABEL: v_permlanex16_b32_undef_tid_f64:
+; GFX12-SDAG:       ; %bb.0:
+; GFX12-SDAG-NEXT:    s_clause 0x1
+; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-SDAG-NEXT:    v_mov_b32_e32 v4, 0
+; GFX12-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v3, s5 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT:    v_mov_b32_e32 v2, s4
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v3, v1, s0, s1
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v2, v0, s0, s1
+; GFX12-SDAG-NEXT:    global_store_b64 v4, v[2:3], s[2:3]
+; GFX12-SDAG-NEXT:    s_endpgm
+;
+; GFX12-GISEL-LABEL: v_permlanex16_b32_undef_tid_f64:
+; GFX12-GISEL:       ; %bb.0:
+; GFX12-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-NEXT:    s_clause 0x1
+; GFX12-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX12-GISEL-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v1, v1, s0, s1
+; GFX12-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-GISEL-NEXT:    s_endpgm
+;
+; GFX13-SDAG-LABEL: v_permlanex16_b32_undef_tid_f64:
+; GFX13-SDAG:       ; %bb.0:
+; GFX13-SDAG-NEXT:    s_clause 0x1
+; GFX13-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX13-SDAG-NEXT:    s_mov_b32 s6, 0x3ff
+; GFX13-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v3, s5 :: v_dual_bitop2_b32 v0, s6, v0 bitop3:0x40
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v4, 0
+; GFX13-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX13-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX13-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-SDAG-NEXT:    v_permlanex16_b32 v3, v1, s0, s1
+; GFX13-SDAG-NEXT:    v_permlanex16_b32 v2, v0, s0, s1
+; GFX13-SDAG-NEXT:    global_store_b64 v4, v[2:3], s[2:3]
+; GFX13-SDAG-NEXT:    s_endpgm
+;
+; GFX13-GISEL-LABEL: v_permlanex16_b32_undef_tid_f64:
+; GFX13-GISEL:       ; %bb.0:
+; GFX13-GISEL-NEXT:    s_mov_b32 s0, 0x3ff
+; GFX13-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX13-GISEL-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s0, v0 bitop3:0x40
+; GFX13-GISEL-NEXT:    s_clause 0x1
+; GFX13-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-GISEL-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX13-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX13-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1
+; GFX13-GISEL-NEXT:    v_permlanex16_b32 v1, v1, s0, s1
+; GFX13-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-GISEL-NEXT:    s_endpgm
+  %tidx = call i32 @llvm.amdgcn.workitem.id.x()
+  %tidx_f32 = bitcast i32 %tidx to float
+  %tidx_f64 = fpext float %tidx_f32 to double
+  %undef = freeze double poison
+  %v = call double @llvm.amdgcn.permlanex16.f64(double %undef, double %tidx_f64, i32 %src1, i32 %src2, i1 false, i1 false)
+  store double %v, ptr addrspace(1) %out
   ret void
 }
 
@@ -10445,53 +11095,109 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_f64(ptr addrspace(1) %out, i3
 }
 
 define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_i32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-LABEL: v_permlanex16_b32_i_tid_fi_i32:
-; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_clause 0x1
-; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX10-NEXT:    global_store_dword v1, v0, s[2:3]
-; GFX10-NEXT:    s_endpgm
+; GFX10-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_i32:
+; GFX10-SDAG:       ; %bb.0:
+; GFX10-SDAG-NEXT:    s_clause 0x1
+; GFX10-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-SDAG-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v1, s4
+; GFX10-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX10-SDAG-NEXT:    global_store_dword v2, v1, s[2:3]
+; GFX10-SDAG-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: v_permlanex16_b32_i_tid_fi_i32:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
-; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX11-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX11-NEXT:    s_endpgm
+; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_i32:
+; GFX10-GISEL:       ; %bb.0:
+; GFX10-GISEL-NEXT:    s_clause 0x1
+; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, 0
+; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX10-GISEL-NEXT:    global_store_dword v1, v0, s[2:3]
+; GFX10-GISEL-NEXT:    s_endpgm
 ;
-; GFX12-LABEL: v_permlanex16_b32_i_tid_fi_i32:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_clause 0x1
-; GFX12-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
-; GFX12-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX12-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX12-NEXT:    s_endpgm
+; GFX11-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_i32:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_clause 0x1
+; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX11-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
+; GFX11-SDAG-NEXT:    s_endpgm
 ;
-; GFX13-LABEL: v_permlanex16_b32_i_tid_fi_i32:
-; GFX13:       ; %bb.0:
-; GFX13-NEXT:    s_clause 0x1
-; GFX13-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-NEXT:    s_mov_b32 s4, 0x3ff
-; GFX13-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
-; GFX13-NEXT:    s_wait_kmcnt 0x0
-; GFX13-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX13-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX13-NEXT:    s_endpgm
+; GFX11-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_i32:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_clause 0x1
+; GFX11-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX11-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
+; GFX11-GISEL-NEXT:    s_endpgm
+;
+; GFX12-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_i32:
+; GFX12-SDAG:       ; %bb.0:
+; GFX12-SDAG-NEXT:    s_clause 0x1
+; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX12-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX12-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
+; GFX12-SDAG-NEXT:    s_endpgm
+;
+; GFX12-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_i32:
+; GFX12-GISEL:       ; %bb.0:
+; GFX12-GISEL-NEXT:    s_clause 0x1
+; GFX12-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX12-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
+; GFX12-GISEL-NEXT:    s_endpgm
+;
+; GFX13-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_i32:
+; GFX13-SDAG:       ; %bb.0:
+; GFX13-SDAG-NEXT:    s_clause 0x1
+; GFX13-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX13-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX13-SDAG-NEXT:    s_mov_b32 s5, 0x3ff
+; GFX13-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
+; GFX13-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX13-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX13-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
+; GFX13-SDAG-NEXT:    s_endpgm
+;
+; GFX13-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_i32:
+; GFX13-GISEL:       ; %bb.0:
+; GFX13-GISEL-NEXT:    s_clause 0x1
+; GFX13-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-GISEL-NEXT:    s_mov_b32 s4, 0x3ff
+; GFX13-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX13-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX13-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
+; GFX13-GISEL-NEXT:    s_endpgm
   %tidx = call i32 @llvm.amdgcn.workitem.id.x()
   %undef = freeze i32 poison
   %v = call i32 @llvm.amdgcn.permlanex16.i32(i32 %undef, i32 %tidx, i32 %src1, i32 %src2, i1 true, i1 false)
@@ -10500,101 +11206,647 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_i32(ptr addrspace(1) %out,
 }
 
 define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_f32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-LABEL: v_permlanex16_b32_i_tid_fi_f32:
-; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_clause 0x1
-; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX10-NEXT:    global_store_dword v1, v0, s[2:3]
-; GFX10-NEXT:    s_endpgm
+; GFX10-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_f32:
+; GFX10-SDAG:       ; %bb.0:
+; GFX10-SDAG-NEXT:    s_clause 0x1
+; GFX10-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-SDAG-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v1, s4
+; GFX10-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX10-SDAG-NEXT:    global_store_dword v2, v1, s[2:3]
+; GFX10-SDAG-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: v_permlanex16_b32_i_tid_fi_f32:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
-; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX11-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX11-NEXT:    s_endpgm
+; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_f32:
+; GFX10-GISEL:       ; %bb.0:
+; GFX10-GISEL-NEXT:    s_clause 0x1
+; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, 0
+; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX10-GISEL-NEXT:    global_store_dword v1, v0, s[2:3]
+; GFX10-GISEL-NEXT:    s_endpgm
 ;
-; GFX12-LABEL: v_permlanex16_b32_i_tid_fi_f32:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_clause 0x1
-; GFX12-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
-; GFX12-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX12-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX12-NEXT:    s_endpgm
+; GFX11-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_f32:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_clause 0x1
+; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX11-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
+; GFX11-SDAG-NEXT:    s_endpgm
+;
+; GFX11-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_f32:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_clause 0x1
+; GFX11-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX11-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
+; GFX11-GISEL-NEXT:    s_endpgm
+;
+; GFX12-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_f32:
+; GFX12-SDAG:       ; %bb.0:
+; GFX12-SDAG-NEXT:    s_clause 0x1
+; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX12-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX12-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
+; GFX12-SDAG-NEXT:    s_endpgm
+;
+; GFX12-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_f32:
+; GFX12-GISEL:       ; %bb.0:
+; GFX12-GISEL-NEXT:    s_clause 0x1
+; GFX12-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX12-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
+; GFX12-GISEL-NEXT:    s_endpgm
+;
+; GFX13-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_f32:
+; GFX13-SDAG:       ; %bb.0:
+; GFX13-SDAG-NEXT:    s_clause 0x1
+; GFX13-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX13-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX13-SDAG-NEXT:    s_mov_b32 s5, 0x3ff
+; GFX13-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
+; GFX13-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX13-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX13-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
+; GFX13-SDAG-NEXT:    s_endpgm
+;
+; GFX13-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_f32:
+; GFX13-GISEL:       ; %bb.0:
+; GFX13-GISEL-NEXT:    s_clause 0x1
+; GFX13-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-GISEL-NEXT:    s_mov_b32 s4, 0x3ff
+; GFX13-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX13-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX13-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
+; GFX13-GISEL-NEXT:    s_endpgm
+  %tidx = call i32 @llvm.amdgcn.workitem.id.x()
+  %tidx_f32 = bitcast i32 %tidx to float
+  %undef = freeze float poison
+  %v = call float @llvm.amdgcn.permlanex16.f32(float %undef, float %tidx_f32, i32 %src1, i32 %src2, i1 true, i1 false)
+  store float %v, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_i64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
+; GFX10-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_i64:
+; GFX10-SDAG:       ; %bb.0:
+; GFX10-SDAG-NEXT:    s_clause 0x1
+; GFX10-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-SDAG-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v3, 0
+; GFX10-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, s5
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v1, s4
+; GFX10-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v2, v3, s0, s1 op_sel:[1,0]
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX10-SDAG-NEXT:    global_store_dwordx2 v3, v[1:2], s[2:3]
+; GFX10-SDAG-NEXT:    s_endpgm
+;
+; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_i64:
+; GFX10-GISEL:       ; %bb.0:
+; GFX10-GISEL-NEXT:    s_clause 0x1
+; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v1, v2, s0, s1 op_sel:[1,0]
+; GFX10-GISEL-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-GISEL-NEXT:    s_endpgm
+;
+; GFX11-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_i64:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_clause 0x1
+; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v2, v3, s0, s1 op_sel:[1,0]
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX11-SDAG-NEXT:    global_store_b64 v3, v[1:2], s[2:3]
+; GFX11-SDAG-NEXT:    s_endpgm
+;
+; GFX11-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_i64:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_clause 0x1
+; GFX11-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v1, v2, s0, s1 op_sel:[1,0]
+; GFX11-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-GISEL-NEXT:    s_endpgm
+;
+; GFX12-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_i64:
+; GFX12-SDAG:       ; %bb.0:
+; GFX12-SDAG-NEXT:    s_clause 0x1
+; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v2, v3, s0, s1 op_sel:[1,0]
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX12-SDAG-NEXT:    global_store_b64 v3, v[1:2], s[2:3]
+; GFX12-SDAG-NEXT:    s_endpgm
+;
+; GFX12-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_i64:
+; GFX12-GISEL:       ; %bb.0:
+; GFX12-GISEL-NEXT:    s_clause 0x1
+; GFX12-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v1, v2, s0, s1 op_sel:[1,0]
+; GFX12-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-GISEL-NEXT:    s_endpgm
+;
+; GFX13-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_i64:
+; GFX13-SDAG:       ; %bb.0:
+; GFX13-SDAG-NEXT:    s_clause 0x1
+; GFX13-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX13-SDAG-NEXT:    s_mov_b32 s5, 0x3ff
+; GFX13-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
+; GFX13-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX13-SDAG-NEXT:    v_permlanex16_b32 v2, v3, s0, s1 op_sel:[1,0]
+; GFX13-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX13-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX13-SDAG-NEXT:    global_store_b64 v3, v[1:2], s[2:3]
+; GFX13-SDAG-NEXT:    s_endpgm
+;
+; GFX13-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_i64:
+; GFX13-GISEL:       ; %bb.0:
+; GFX13-GISEL-NEXT:    s_clause 0x1
+; GFX13-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-GISEL-NEXT:    s_mov_b32 s4, 0x3ff
+; GFX13-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-GISEL-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX13-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX13-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX13-GISEL-NEXT:    v_permlanex16_b32 v1, v2, s0, s1 op_sel:[1,0]
+; GFX13-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-GISEL-NEXT:    s_endpgm
+  %tidx = call i32 @llvm.amdgcn.workitem.id.x()
+  %tidx_i64 = zext i32 %tidx to i64
+  %undef = freeze i64 poison
+  %v = call i64 @llvm.amdgcn.permlanex16.i64(i64 %undef, i64 %tidx_i64, i32 %src1, i32 %src2, i1 true, i1 false)
+  store i64 %v, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_f64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
+; GFX10-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_f64:
+; GFX10-SDAG:       ; %bb.0:
+; GFX10-SDAG-NEXT:    s_clause 0x1
+; GFX10-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-SDAG-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v4, 0
+; GFX10-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v3, s5
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, s4
+; GFX10-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v3, v1, s0, s1 op_sel:[1,0]
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v2, v0, s0, s1 op_sel:[1,0]
+; GFX10-SDAG-NEXT:    global_store_dwordx2 v4, v[2:3], s[2:3]
+; GFX10-SDAG-NEXT:    s_endpgm
+;
+; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_f64:
+; GFX10-GISEL:       ; %bb.0:
+; GFX10-GISEL-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-GISEL-NEXT:    s_clause 0x1
+; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v1, v1, s0, s1 op_sel:[1,0]
+; GFX10-GISEL-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-GISEL-NEXT:    s_endpgm
+;
+; GFX11-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_f64:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_clause 0x1
+; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT:    v_mov_b32_e32 v4, 0
+; GFX11-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v3, s5 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v3, v1, s0, s1 op_sel:[1,0]
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v2, v0, s0, s1 op_sel:[1,0]
+; GFX11-SDAG-NEXT:    global_store_b64 v4, v[2:3], s[2:3]
+; GFX11-SDAG-NEXT:    s_endpgm
+;
+; GFX11-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_f64:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT:    s_clause 0x1
+; GFX11-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v1, v1, s0, s1 op_sel:[1,0]
+; GFX11-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-GISEL-NEXT:    s_endpgm
+;
+; GFX12-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_f64:
+; GFX12-SDAG:       ; %bb.0:
+; GFX12-SDAG-NEXT:    s_clause 0x1
+; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-SDAG-NEXT:    v_mov_b32_e32 v4, 0
+; GFX12-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v3, s5 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT:    v_mov_b32_e32 v2, s4
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v3, v1, s0, s1 op_sel:[1,0]
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v2, v0, s0, s1 op_sel:[1,0]
+; GFX12-SDAG-NEXT:    global_store_b64 v4, v[2:3], s[2:3]
+; GFX12-SDAG-NEXT:    s_endpgm
+;
+; GFX12-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_f64:
+; GFX12-GISEL:       ; %bb.0:
+; GFX12-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-NEXT:    s_clause 0x1
+; GFX12-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX12-GISEL-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v1, v1, s0, s1 op_sel:[1,0]
+; GFX12-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-GISEL-NEXT:    s_endpgm
+;
+; GFX13-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_f64:
+; GFX13-SDAG:       ; %bb.0:
+; GFX13-SDAG-NEXT:    s_clause 0x1
+; GFX13-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX13-SDAG-NEXT:    s_mov_b32 s6, 0x3ff
+; GFX13-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v3, s5 :: v_dual_bitop2_b32 v0, s6, v0 bitop3:0x40
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v4, 0
+; GFX13-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX13-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX13-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-SDAG-NEXT:    v_permlanex16_b32 v3, v1, s0, s1 op_sel:[1,0]
+; GFX13-SDAG-NEXT:    v_permlanex16_b32 v2, v0, s0, s1 op_sel:[1,0]
+; GFX13-SDAG-NEXT:    global_store_b64 v4, v[2:3], s[2:3]
+; GFX13-SDAG-NEXT:    s_endpgm
+;
+; GFX13-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_f64:
+; GFX13-GISEL:       ; %bb.0:
+; GFX13-GISEL-NEXT:    s_mov_b32 s0, 0x3ff
+; GFX13-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX13-GISEL-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s0, v0 bitop3:0x40
+; GFX13-GISEL-NEXT:    s_clause 0x1
+; GFX13-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-GISEL-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX13-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX13-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX13-GISEL-NEXT:    v_permlanex16_b32 v1, v1, s0, s1 op_sel:[1,0]
+; GFX13-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-GISEL-NEXT:    s_endpgm
+  %tidx = call i32 @llvm.amdgcn.workitem.id.x()
+  %tidx_f32 = bitcast i32 %tidx to float
+  %tidx_f64 = fpext float %tidx_f32 to double
+  %undef = freeze double poison
+  %v = call double @llvm.amdgcn.permlanex16.f64(double %undef, double %tidx_f64, i32 %src1, i32 %src2, i1 true, i1 false)
+  store double %v, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @v_permlanex16_b32_i_tid_bc_i32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
+; GFX10-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_i32:
+; GFX10-SDAG:       ; %bb.0:
+; GFX10-SDAG-NEXT:    s_clause 0x1
+; GFX10-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-SDAG-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v1, s4
+; GFX10-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX10-SDAG-NEXT:    global_store_dword v2, v1, s[2:3]
+; GFX10-SDAG-NEXT:    s_endpgm
+;
+; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_i32:
+; GFX10-GISEL:       ; %bb.0:
+; GFX10-GISEL-NEXT:    s_clause 0x1
+; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, 0
+; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX10-GISEL-NEXT:    global_store_dword v1, v0, s[2:3]
+; GFX10-GISEL-NEXT:    s_endpgm
+;
+; GFX11-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_i32:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_clause 0x1
+; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX11-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
+; GFX11-SDAG-NEXT:    s_endpgm
+;
+; GFX11-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_i32:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_clause 0x1
+; GFX11-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX11-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
+; GFX11-GISEL-NEXT:    s_endpgm
+;
+; GFX12-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_i32:
+; GFX12-SDAG:       ; %bb.0:
+; GFX12-SDAG-NEXT:    s_clause 0x1
+; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX12-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX12-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
+; GFX12-SDAG-NEXT:    s_endpgm
+;
+; GFX12-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_i32:
+; GFX12-GISEL:       ; %bb.0:
+; GFX12-GISEL-NEXT:    s_clause 0x1
+; GFX12-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX12-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
+; GFX12-GISEL-NEXT:    s_endpgm
+;
+; GFX13-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_i32:
+; GFX13-SDAG:       ; %bb.0:
+; GFX13-SDAG-NEXT:    s_clause 0x1
+; GFX13-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX13-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX13-SDAG-NEXT:    s_mov_b32 s5, 0x3ff
+; GFX13-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
+; GFX13-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX13-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX13-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
+; GFX13-SDAG-NEXT:    s_endpgm
+;
+; GFX13-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_i32:
+; GFX13-GISEL:       ; %bb.0:
+; GFX13-GISEL-NEXT:    s_clause 0x1
+; GFX13-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-GISEL-NEXT:    s_mov_b32 s4, 0x3ff
+; GFX13-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX13-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX13-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
+; GFX13-GISEL-NEXT:    s_endpgm
+  %tidx = call i32 @llvm.amdgcn.workitem.id.x()
+  %undef = freeze i32 poison
+  %v = call i32 @llvm.amdgcn.permlanex16.i32(i32 %undef, i32 %tidx, i32 %src1, i32 %src2, i1 false, i1 true)
+  store i32 %v, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @v_permlanex16_b32_i_tid_bc_f32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
+; GFX10-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_f32:
+; GFX10-SDAG:       ; %bb.0:
+; GFX10-SDAG-NEXT:    s_clause 0x1
+; GFX10-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-SDAG-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v1, s4
+; GFX10-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX10-SDAG-NEXT:    global_store_dword v2, v1, s[2:3]
+; GFX10-SDAG-NEXT:    s_endpgm
+;
+; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_f32:
+; GFX10-GISEL:       ; %bb.0:
+; GFX10-GISEL-NEXT:    s_clause 0x1
+; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, 0
+; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX10-GISEL-NEXT:    global_store_dword v1, v0, s[2:3]
+; GFX10-GISEL-NEXT:    s_endpgm
+;
+; GFX11-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_f32:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_clause 0x1
+; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX11-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
+; GFX11-SDAG-NEXT:    s_endpgm
+;
+; GFX11-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_f32:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_clause 0x1
+; GFX11-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX11-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
+; GFX11-GISEL-NEXT:    s_endpgm
+;
+; GFX12-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_f32:
+; GFX12-SDAG:       ; %bb.0:
+; GFX12-SDAG-NEXT:    s_clause 0x1
+; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX12-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX12-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
+; GFX12-SDAG-NEXT:    s_endpgm
+;
+; GFX12-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_f32:
+; GFX12-GISEL:       ; %bb.0:
+; GFX12-GISEL-NEXT:    s_clause 0x1
+; GFX12-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX12-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
+; GFX12-GISEL-NEXT:    s_endpgm
+;
+; GFX13-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_f32:
+; GFX13-SDAG:       ; %bb.0:
+; GFX13-SDAG-NEXT:    s_clause 0x1
+; GFX13-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX13-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX13-SDAG-NEXT:    s_mov_b32 s5, 0x3ff
+; GFX13-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
+; GFX13-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX13-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX13-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
+; GFX13-SDAG-NEXT:    s_endpgm
 ;
-; GFX13-LABEL: v_permlanex16_b32_i_tid_fi_f32:
-; GFX13:       ; %bb.0:
-; GFX13-NEXT:    s_clause 0x1
-; GFX13-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-NEXT:    s_mov_b32 s4, 0x3ff
-; GFX13-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
-; GFX13-NEXT:    s_wait_kmcnt 0x0
-; GFX13-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX13-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX13-NEXT:    s_endpgm
+; GFX13-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_f32:
+; GFX13-GISEL:       ; %bb.0:
+; GFX13-GISEL-NEXT:    s_clause 0x1
+; GFX13-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-GISEL-NEXT:    s_mov_b32 s4, 0x3ff
+; GFX13-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX13-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX13-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
+; GFX13-GISEL-NEXT:    s_endpgm
   %tidx = call i32 @llvm.amdgcn.workitem.id.x()
   %tidx_f32 = bitcast i32 %tidx to float
   %undef = freeze float poison
-  %v = call float @llvm.amdgcn.permlanex16.f32(float %undef, float %tidx_f32, i32 %src1, i32 %src2, i1 true, i1 false)
+  %v = call float @llvm.amdgcn.permlanex16.f32(float %undef, float %tidx_f32, i32 %src1, i32 %src2, i1 false, i1 true)
   store float %v, ptr addrspace(1) %out
   ret void
 }
 
-define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_i64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_i64:
+define amdgpu_kernel void @v_permlanex16_b32_i_tid_bc_i64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
+; GFX10-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_i64:
 ; GFX10-SDAG:       ; %bb.0:
 ; GFX10-SDAG-NEXT:    s_clause 0x1
 ; GFX10-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
 ; GFX10-SDAG-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v3, 0
+; GFX10-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, s5
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v1, s4
 ; GFX10-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_permlanex16_b32 v1, v2, s0, s1 op_sel:[1,0]
-; GFX10-SDAG-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX10-SDAG-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v2, v3, s0, s1 op_sel:[0,1]
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX10-SDAG-NEXT:    global_store_dwordx2 v3, v[1:2], s[2:3]
 ; GFX10-SDAG-NEXT:    s_endpgm
 ;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_i64:
+; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_i64:
 ; GFX10-GISEL:       ; %bb.0:
 ; GFX10-GISEL-NEXT:    s_clause 0x1
 ; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
 ; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
 ; GFX10-GISEL-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX10-GISEL-NEXT:    v_permlanex16_b32 v1, v2, s0, s1 op_sel:[1,0]
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v1, v2, s0, s1 op_sel:[0,1]
 ; GFX10-GISEL-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
 ; GFX10-GISEL-NEXT:    s_endpgm
 ;
-; GFX11-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_i64:
+; GFX11-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_i64:
 ; GFX11-SDAG:       ; %bb.0:
 ; GFX11-SDAG-NEXT:    s_clause 0x1
 ; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
 ; GFX11-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, 0
-; GFX11-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT:    v_permlanex16_b32 v1, v2, s0, s1 op_sel:[1,0]
-; GFX11-SDAG-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX11-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v2, v3, s0, s1 op_sel:[0,1]
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX11-SDAG-NEXT:    global_store_b64 v3, v[1:2], s[2:3]
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
-; GFX11-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_i64:
+; GFX11-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_i64:
 ; GFX11-GISEL:       ; %bb.0:
 ; GFX11-GISEL-NEXT:    s_clause 0x1
 ; GFX11-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
@@ -10603,26 +11855,27 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_i64(ptr addrspace(1) %out,
 ; GFX11-GISEL-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX11-GISEL-NEXT:    v_permlanex16_b32 v1, v2, s0, s1 op_sel:[1,0]
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v1, v2, s0, s1 op_sel:[0,1]
 ; GFX11-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
 ; GFX11-GISEL-NEXT:    s_endpgm
 ;
-; GFX12-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_i64:
+; GFX12-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_i64:
 ; GFX12-SDAG:       ; %bb.0:
 ; GFX12-SDAG-NEXT:    s_clause 0x1
 ; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
 ; GFX12-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-SDAG-NEXT:    v_mov_b32_e32 v2, 0
-; GFX12-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-NEXT:    v_permlanex16_b32 v1, v2, s0, s1 op_sel:[1,0]
-; GFX12-SDAG-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX12-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v2, v3, s0, s1 op_sel:[0,1]
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX12-SDAG-NEXT:    global_store_b64 v3, v[1:2], s[2:3]
 ; GFX12-SDAG-NEXT:    s_endpgm
 ;
-; GFX12-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_i64:
+; GFX12-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_i64:
 ; GFX12-GISEL:       ; %bb.0:
 ; GFX12-GISEL-NEXT:    s_clause 0x1
 ; GFX12-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
@@ -10631,27 +11884,29 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_i64(ptr addrspace(1) %out,
 ; GFX12-GISEL-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX12-GISEL-NEXT:    v_permlanex16_b32 v1, v2, s0, s1 op_sel:[1,0]
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v1, v2, s0, s1 op_sel:[0,1]
 ; GFX12-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
 ; GFX12-GISEL-NEXT:    s_endpgm
 ;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_i64:
+; GFX13-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_i64:
 ; GFX13-SDAG:       ; %bb.0:
 ; GFX13-SDAG-NEXT:    s_clause 0x1
 ; GFX13-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
 ; GFX13-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT:    s_mov_b32 s4, 0x3ff
-; GFX13-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-SDAG-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX13-SDAG-NEXT:    s_mov_b32 s5, 0x3ff
+; GFX13-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
 ; GFX13-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT:    v_permlanex16_b32 v1, v2, s0, s1 op_sel:[1,0]
+; GFX13-SDAG-NEXT:    v_permlanex16_b32 v2, v3, s0, s1 op_sel:[0,1]
 ; GFX13-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX13-SDAG-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX13-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX13-SDAG-NEXT:    global_store_b64 v3, v[1:2], s[2:3]
 ; GFX13-SDAG-NEXT:    s_endpgm
 ;
-; GFX13-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_i64:
+; GFX13-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_i64:
 ; GFX13-GISEL:       ; %bb.0:
 ; GFX13-GISEL-NEXT:    s_clause 0x1
 ; GFX13-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
@@ -10660,34 +11915,37 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_i64(ptr addrspace(1) %out,
 ; GFX13-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX13-GISEL-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
 ; GFX13-GISEL-NEXT:    s_wait_kmcnt 0x0
-; GFX13-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX13-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
 ; GFX13-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX13-GISEL-NEXT:    v_permlanex16_b32 v1, v2, s0, s1 op_sel:[1,0]
+; GFX13-GISEL-NEXT:    v_permlanex16_b32 v1, v2, s0, s1 op_sel:[0,1]
 ; GFX13-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
 ; GFX13-GISEL-NEXT:    s_endpgm
   %tidx = call i32 @llvm.amdgcn.workitem.id.x()
   %tidx_i64 = zext i32 %tidx to i64
   %undef = freeze i64 poison
-  %v = call i64 @llvm.amdgcn.permlanex16.i64(i64 %undef, i64 %tidx_i64, i32 %src1, i32 %src2, i1 true, i1 false)
+  %v = call i64 @llvm.amdgcn.permlanex16.i64(i64 %undef, i64 %tidx_i64, i32 %src1, i32 %src2, i1 false, i1 true)
   store i64 %v, ptr addrspace(1) %out
   ret void
 }
 
-define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_f64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_f64:
+define amdgpu_kernel void @v_permlanex16_b32_i_tid_bc_f64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
+; GFX10-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_f64:
 ; GFX10-SDAG:       ; %bb.0:
-; GFX10-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
 ; GFX10-SDAG-NEXT:    s_clause 0x1
 ; GFX10-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
 ; GFX10-SDAG-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v4, 0
+; GFX10-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v3, s5
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, s4
 ; GFX10-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_permlanex16_b32 v1, v1, s0, s1 op_sel:[1,0]
-; GFX10-SDAG-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX10-SDAG-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v3, v1, s0, s1 op_sel:[0,1]
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v2, v0, s0, s1 op_sel:[0,1]
+; GFX10-SDAG-NEXT:    global_store_dwordx2 v4, v[2:3], s[2:3]
 ; GFX10-SDAG-NEXT:    s_endpgm
 ;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_f64:
+; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_f64:
 ; GFX10-GISEL:       ; %bb.0:
 ; GFX10-GISEL-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
 ; GFX10-GISEL-NEXT:    s_clause 0x1
@@ -10695,27 +11953,30 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_f64(ptr addrspace(1) %out,
 ; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
 ; GFX10-GISEL-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX10-GISEL-NEXT:    v_permlanex16_b32 v1, v1, s0, s1 op_sel:[1,0]
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v1, v1, s0, s1 op_sel:[0,1]
 ; GFX10-GISEL-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
 ; GFX10-GISEL-NEXT:    s_endpgm
 ;
-; GFX11-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_f64:
+; GFX11-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_f64:
 ; GFX11-SDAG:       ; %bb.0:
-; GFX11-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GFX11-SDAG-NEXT:    s_clause 0x1
 ; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
 ; GFX11-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT:    v_mov_b32_e32 v4, 0
+; GFX11-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v3, s5 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX11-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
 ; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT:    v_permlanex16_b32 v1, v1, s0, s1 op_sel:[1,0]
-; GFX11-SDAG-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX11-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v3, v1, s0, s1 op_sel:[0,1]
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v2, v0, s0, s1 op_sel:[0,1]
+; GFX11-SDAG-NEXT:    global_store_b64 v4, v[2:3], s[2:3]
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
-; GFX11-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_f64:
+; GFX11-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_f64:
 ; GFX11-GISEL:       ; %bb.0:
 ; GFX11-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GFX11-GISEL-NEXT:    s_clause 0x1
@@ -10725,27 +11986,30 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_f64(ptr addrspace(1) %out,
 ; GFX11-GISEL-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
 ; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX11-GISEL-NEXT:    v_permlanex16_b32 v1, v1, s0, s1 op_sel:[1,0]
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v1, v1, s0, s1 op_sel:[0,1]
 ; GFX11-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
 ; GFX11-GISEL-NEXT:    s_endpgm
 ;
-; GFX12-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_f64:
+; GFX12-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_f64:
 ; GFX12-SDAG:       ; %bb.0:
-; GFX12-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GFX12-SDAG-NEXT:    s_clause 0x1
 ; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
 ; GFX12-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX12-SDAG-NEXT:    v_mov_b32_e32 v4, 0
+; GFX12-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v3, s5 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT:    v_mov_b32_e32 v2, s4
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-NEXT:    v_permlanex16_b32 v1, v1, s0, s1 op_sel:[1,0]
-; GFX12-SDAG-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX12-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v3, v1, s0, s1 op_sel:[0,1]
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v2, v0, s0, s1 op_sel:[0,1]
+; GFX12-SDAG-NEXT:    global_store_b64 v4, v[2:3], s[2:3]
 ; GFX12-SDAG-NEXT:    s_endpgm
 ;
-; GFX12-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_f64:
+; GFX12-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_f64:
 ; GFX12-GISEL:       ; %bb.0:
 ; GFX12-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GFX12-GISEL-NEXT:    s_clause 0x1
@@ -10755,28 +12019,30 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_f64(ptr addrspace(1) %out,
 ; GFX12-GISEL-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX12-GISEL-NEXT:    v_permlanex16_b32 v1, v1, s0, s1 op_sel:[1,0]
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v1, v1, s0, s1 op_sel:[0,1]
 ; GFX12-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
 ; GFX12-GISEL-NEXT:    s_endpgm
 ;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_f64:
+; GFX13-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_f64:
 ; GFX13-SDAG:       ; %bb.0:
-; GFX13-SDAG-NEXT:    s_mov_b32 s0, 0x3ff
-; GFX13-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX13-SDAG-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s0, v0 bitop3:0x40
 ; GFX13-SDAG-NEXT:    s_clause 0x1
 ; GFX13-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
 ; GFX13-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX13-SDAG-NEXT:    s_mov_b32 s6, 0x3ff
+; GFX13-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v3, s5 :: v_dual_bitop2_b32 v0, s6, v0 bitop3:0x40
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v4, 0
 ; GFX13-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
 ; GFX13-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX13-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT:    v_permlanex16_b32 v1, v1, s0, s1 op_sel:[1,0]
-; GFX13-SDAG-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX13-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-SDAG-NEXT:    v_permlanex16_b32 v3, v1, s0, s1 op_sel:[0,1]
+; GFX13-SDAG-NEXT:    v_permlanex16_b32 v2, v0, s0, s1 op_sel:[0,1]
+; GFX13-SDAG-NEXT:    global_store_b64 v4, v[2:3], s[2:3]
 ; GFX13-SDAG-NEXT:    s_endpgm
 ;
-; GFX13-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_f64:
+; GFX13-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_f64:
 ; GFX13-GISEL:       ; %bb.0:
 ; GFX13-GISEL-NEXT:    s_mov_b32 s0, 0x3ff
 ; GFX13-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
@@ -10787,478 +12053,234 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_f64(ptr addrspace(1) %out,
 ; GFX13-GISEL-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
 ; GFX13-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX13-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX13-GISEL-NEXT:    v_permlanex16_b32 v1, v1, s0, s1 op_sel:[1,0]
+; GFX13-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX13-GISEL-NEXT:    v_permlanex16_b32 v1, v1, s0, s1 op_sel:[0,1]
 ; GFX13-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
 ; GFX13-GISEL-NEXT:    s_endpgm
   %tidx = call i32 @llvm.amdgcn.workitem.id.x()
   %tidx_f32 = bitcast i32 %tidx to float
   %tidx_f64 = fpext float %tidx_f32 to double
   %undef = freeze double poison
-  %v = call double @llvm.amdgcn.permlanex16.f64(double %undef, double %tidx_f64, i32 %src1, i32 %src2, i1 true, i1 false)
+  %v = call double @llvm.amdgcn.permlanex16.f64(double %undef, double %tidx_f64, i32 %src1, i32 %src2, i1 false, i1 true)
   store double %v, ptr addrspace(1) %out
   ret void
 }
 
-define amdgpu_kernel void @v_permlanex16_b32_i_tid_bc_i32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-LABEL: v_permlanex16_b32_i_tid_bc_i32:
-; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_clause 0x1
-; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX10-NEXT:    global_store_dword v1, v0, s[2:3]
-; GFX10-NEXT:    s_endpgm
-;
-; GFX11-LABEL: v_permlanex16_b32_i_tid_bc_i32:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
-; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX11-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX11-NEXT:    s_endpgm
-;
-; GFX12-LABEL: v_permlanex16_b32_i_tid_bc_i32:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_clause 0x1
-; GFX12-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
-; GFX12-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX12-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX12-NEXT:    s_endpgm
-;
-; GFX13-LABEL: v_permlanex16_b32_i_tid_bc_i32:
-; GFX13:       ; %bb.0:
-; GFX13-NEXT:    s_clause 0x1
-; GFX13-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-NEXT:    s_mov_b32 s4, 0x3ff
-; GFX13-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
-; GFX13-NEXT:    s_wait_kmcnt 0x0
-; GFX13-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX13-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX13-NEXT:    s_endpgm
-  %tidx = call i32 @llvm.amdgcn.workitem.id.x()
-  %undef = freeze i32 poison
-  %v = call i32 @llvm.amdgcn.permlanex16.i32(i32 %undef, i32 %tidx, i32 %src1, i32 %src2, i1 false, i1 true)
-  store i32 %v, ptr addrspace(1) %out
-  ret void
-}
-
-define amdgpu_kernel void @v_permlanex16_b32_i_tid_bc_f32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-LABEL: v_permlanex16_b32_i_tid_bc_f32:
-; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_clause 0x1
-; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX10-NEXT:    global_store_dword v1, v0, s[2:3]
-; GFX10-NEXT:    s_endpgm
-;
-; GFX11-LABEL: v_permlanex16_b32_i_tid_bc_f32:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
-; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX11-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX11-NEXT:    s_endpgm
-;
-; GFX12-LABEL: v_permlanex16_b32_i_tid_bc_f32:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_clause 0x1
-; GFX12-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
-; GFX12-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX12-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX12-NEXT:    s_endpgm
-;
-; GFX13-LABEL: v_permlanex16_b32_i_tid_bc_f32:
-; GFX13:       ; %bb.0:
-; GFX13-NEXT:    s_clause 0x1
-; GFX13-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-NEXT:    s_mov_b32 s4, 0x3ff
-; GFX13-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
-; GFX13-NEXT:    s_wait_kmcnt 0x0
-; GFX13-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX13-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX13-NEXT:    s_endpgm
-  %tidx = call i32 @llvm.amdgcn.workitem.id.x()
-  %tidx_f32 = bitcast i32 %tidx to float
-  %undef = freeze float poison
-  %v = call float @llvm.amdgcn.permlanex16.f32(float %undef, float %tidx_f32, i32 %src1, i32 %src2, i1 false, i1 true)
-  store float %v, ptr addrspace(1) %out
-  ret void
-}
-
-define amdgpu_kernel void @v_permlanex16_b32_i_tid_bc_i64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_i64:
+define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_bc_i32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
+; GFX10-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_bc_i32:
 ; GFX10-SDAG:       ; %bb.0:
 ; GFX10-SDAG-NEXT:    s_clause 0x1
 ; GFX10-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
 ; GFX10-SDAG-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
 ; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v1, s4
 ; GFX10-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_permlanex16_b32 v1, v2, s0, s1 op_sel:[0,1]
-; GFX10-SDAG-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX10-SDAG-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX10-SDAG-NEXT:    global_store_dword v2, v1, s[2:3]
 ; GFX10-SDAG-NEXT:    s_endpgm
 ;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_i64:
+; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_bc_i32:
 ; GFX10-GISEL:       ; %bb.0:
 ; GFX10-GISEL-NEXT:    s_clause 0x1
 ; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
 ; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX10-GISEL-NEXT:    v_permlanex16_b32 v1, v2, s0, s1 op_sel:[0,1]
-; GFX10-GISEL-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
+; GFX10-GISEL-NEXT:    global_store_dword v1, v0, s[2:3]
 ; GFX10-GISEL-NEXT:    s_endpgm
 ;
-; GFX11-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_i64:
+; GFX11-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_bc_i32:
 ; GFX11-SDAG:       ; %bb.0:
 ; GFX11-SDAG-NEXT:    s_clause 0x1
 ; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
 ; GFX11-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
 ; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, 0
-; GFX11-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT:    v_permlanex16_b32 v1, v2, s0, s1 op_sel:[0,1]
-; GFX11-SDAG-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX11-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX11-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
-; GFX11-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_i64:
+; GFX11-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_bc_i32:
 ; GFX11-GISEL:       ; %bb.0:
 ; GFX11-GISEL-NEXT:    s_clause 0x1
 ; GFX11-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
 ; GFX11-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX11-GISEL-NEXT:    v_permlanex16_b32 v1, v2, s0, s1 op_sel:[0,1]
-; GFX11-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
+; GFX11-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
 ; GFX11-GISEL-NEXT:    s_endpgm
 ;
-; GFX12-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_i64:
+; GFX12-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_bc_i32:
 ; GFX12-SDAG:       ; %bb.0:
 ; GFX12-SDAG-NEXT:    s_clause 0x1
 ; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
 ; GFX12-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v2, 0
-; GFX12-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-NEXT:    v_permlanex16_b32 v1, v2, s0, s1 op_sel:[0,1]
-; GFX12-SDAG-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX12-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX12-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
 ; GFX12-SDAG-NEXT:    s_endpgm
 ;
-; GFX12-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_i64:
+; GFX12-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_bc_i32:
 ; GFX12-GISEL:       ; %bb.0:
 ; GFX12-GISEL-NEXT:    s_clause 0x1
 ; GFX12-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
 ; GFX12-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX12-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX12-GISEL-NEXT:    v_permlanex16_b32 v1, v2, s0, s1 op_sel:[0,1]
-; GFX12-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
+; GFX12-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
 ; GFX12-GISEL-NEXT:    s_endpgm
 ;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_i64:
+; GFX13-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_bc_i32:
 ; GFX13-SDAG:       ; %bb.0:
 ; GFX13-SDAG-NEXT:    s_clause 0x1
 ; GFX13-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
 ; GFX13-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT:    s_mov_b32 s4, 0x3ff
+; GFX13-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX13-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX13-SDAG-NEXT:    s_mov_b32 s5, 0x3ff
 ; GFX13-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-SDAG-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
 ; GFX13-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT:    v_permlanex16_b32 v1, v2, s0, s1 op_sel:[0,1]
-; GFX13-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX13-SDAG-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX13-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX13-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
 ; GFX13-SDAG-NEXT:    s_endpgm
 ;
-; GFX13-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_i64:
+; GFX13-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_bc_i32:
 ; GFX13-GISEL:       ; %bb.0:
 ; GFX13-GISEL-NEXT:    s_clause 0x1
 ; GFX13-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
 ; GFX13-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
 ; GFX13-GISEL-NEXT:    s_mov_b32 s4, 0x3ff
 ; GFX13-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-GISEL-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
 ; GFX13-GISEL-NEXT:    s_wait_kmcnt 0x0
-; GFX13-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX13-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX13-GISEL-NEXT:    v_permlanex16_b32 v1, v2, s0, s1 op_sel:[0,1]
-; GFX13-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
+; GFX13-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
 ; GFX13-GISEL-NEXT:    s_endpgm
   %tidx = call i32 @llvm.amdgcn.workitem.id.x()
-  %tidx_i64 = zext i32 %tidx to i64
-  %undef = freeze i64 poison
-  %v = call i64 @llvm.amdgcn.permlanex16.i64(i64 %undef, i64 %tidx_i64, i32 %src1, i32 %src2, i1 false, i1 true)
-  store i64 %v, ptr addrspace(1) %out
+  %undef = freeze i32 poison
+  %v = call i32 @llvm.amdgcn.permlanex16.i32(i32 %undef, i32 %tidx, i32 %src1, i32 %src2, i1 true, i1 true)
+  store i32 %v, ptr addrspace(1) %out
   ret void
 }
 
-define amdgpu_kernel void @v_permlanex16_b32_i_tid_bc_f64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_f64:
+define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_bc_f32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
+; GFX10-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_bc_f32:
 ; GFX10-SDAG:       ; %bb.0:
-; GFX10-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
 ; GFX10-SDAG-NEXT:    s_clause 0x1
 ; GFX10-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
 ; GFX10-SDAG-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
 ; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v1, s4
 ; GFX10-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_permlanex16_b32 v1, v1, s0, s1 op_sel:[0,1]
-; GFX10-SDAG-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX10-SDAG-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX10-SDAG-NEXT:    global_store_dword v2, v1, s[2:3]
 ; GFX10-SDAG-NEXT:    s_endpgm
 ;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_f64:
+; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_bc_f32:
 ; GFX10-GISEL:       ; %bb.0:
-; GFX10-GISEL-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
 ; GFX10-GISEL-NEXT:    s_clause 0x1
 ; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
 ; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX10-GISEL-NEXT:    v_permlanex16_b32 v1, v1, s0, s1 op_sel:[0,1]
-; GFX10-GISEL-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
+; GFX10-GISEL-NEXT:    global_store_dword v1, v0, s[2:3]
 ; GFX10-GISEL-NEXT:    s_endpgm
 ;
-; GFX11-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_f64:
+; GFX11-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_bc_f32:
 ; GFX11-SDAG:       ; %bb.0:
-; GFX11-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GFX11-SDAG-NEXT:    s_clause 0x1
 ; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
 ; GFX11-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
 ; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, 0
-; GFX11-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX11-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT:    v_permlanex16_b32 v1, v1, s0, s1 op_sel:[0,1]
-; GFX11-SDAG-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX11-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX11-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
-; GFX11-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_f64:
+; GFX11-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_bc_f32:
 ; GFX11-GISEL:       ; %bb.0:
-; GFX11-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GFX11-GISEL-NEXT:    s_clause 0x1
 ; GFX11-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
 ; GFX11-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-GISEL-NEXT:    v_mov_b32_e32 v2, 0
-; GFX11-GISEL-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX11-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX11-GISEL-NEXT:    v_permlanex16_b32 v1, v1, s0, s1 op_sel:[0,1]
-; GFX11-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
+; GFX11-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
 ; GFX11-GISEL-NEXT:    s_endpgm
 ;
-; GFX12-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_f64:
+; GFX12-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_bc_f32:
 ; GFX12-SDAG:       ; %bb.0:
-; GFX12-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GFX12-SDAG-NEXT:    s_clause 0x1
 ; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
 ; GFX12-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v2, 0
-; GFX12-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX12-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-NEXT:    v_permlanex16_b32 v1, v1, s0, s1 op_sel:[0,1]
-; GFX12-SDAG-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX12-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX12-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
 ; GFX12-SDAG-NEXT:    s_endpgm
 ;
-; GFX12-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_f64:
+; GFX12-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_bc_f32:
 ; GFX12-GISEL:       ; %bb.0:
-; GFX12-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GFX12-GISEL-NEXT:    s_clause 0x1
 ; GFX12-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
 ; GFX12-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-GISEL-NEXT:    v_mov_b32_e32 v2, 0
-; GFX12-GISEL-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX12-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX12-GISEL-NEXT:    v_permlanex16_b32 v1, v1, s0, s1 op_sel:[0,1]
-; GFX12-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
+; GFX12-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
 ; GFX12-GISEL-NEXT:    s_endpgm
 ;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_f64:
+; GFX13-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_bc_f32:
 ; GFX13-SDAG:       ; %bb.0:
-; GFX13-SDAG-NEXT:    s_mov_b32 s0, 0x3ff
-; GFX13-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX13-SDAG-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s0, v0 bitop3:0x40
 ; GFX13-SDAG-NEXT:    s_clause 0x1
 ; GFX13-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
 ; GFX13-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX13-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX13-SDAG-NEXT:    ; freeze: $sgpr4
+; GFX13-SDAG-NEXT:    s_mov_b32 s5, 0x3ff
+; GFX13-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
 ; GFX13-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT:    v_permlanex16_b32 v1, v1, s0, s1 op_sel:[0,1]
-; GFX13-SDAG-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX13-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX13-SDAG-NEXT:    global_store_b32 v2, v1, s[2:3]
 ; GFX13-SDAG-NEXT:    s_endpgm
 ;
-; GFX13-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_f64:
+; GFX13-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_bc_f32:
 ; GFX13-GISEL:       ; %bb.0:
-; GFX13-GISEL-NEXT:    s_mov_b32 s0, 0x3ff
-; GFX13-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX13-GISEL-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s0, v0 bitop3:0x40
 ; GFX13-GISEL-NEXT:    s_clause 0x1
 ; GFX13-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
 ; GFX13-GISEL-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-GISEL-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX13-GISEL-NEXT:    s_mov_b32 s4, 0x3ff
+; GFX13-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-GISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
 ; GFX13-GISEL-NEXT:    s_wait_kmcnt 0x0
-; GFX13-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX13-GISEL-NEXT:    v_permlanex16_b32 v1, v1, s0, s1 op_sel:[0,1]
-; GFX13-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
+; GFX13-GISEL-NEXT:    global_store_b32 v1, v0, s[2:3]
 ; GFX13-GISEL-NEXT:    s_endpgm
-  %tidx = call i32 @llvm.amdgcn.workitem.id.x()
-  %tidx_f32 = bitcast i32 %tidx to float
-  %tidx_f64 = fpext float %tidx_f32 to double
-  %undef = freeze double poison
-  %v = call double @llvm.amdgcn.permlanex16.f64(double %undef, double %tidx_f64, i32 %src1, i32 %src2, i1 false, i1 true)
-  store double %v, ptr addrspace(1) %out
-  ret void
-}
-
-define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_bc_i32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-LABEL: v_permlanex16_b32_i_tid_fi_bc_i32:
-; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_clause 0x1
-; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX10-NEXT:    global_store_dword v1, v0, s[2:3]
-; GFX10-NEXT:    s_endpgm
-;
-; GFX11-LABEL: v_permlanex16_b32_i_tid_fi_bc_i32:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
-; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX11-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX11-NEXT:    s_endpgm
-;
-; GFX12-LABEL: v_permlanex16_b32_i_tid_fi_bc_i32:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_clause 0x1
-; GFX12-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
-; GFX12-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX12-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX12-NEXT:    s_endpgm
-;
-; GFX13-LABEL: v_permlanex16_b32_i_tid_fi_bc_i32:
-; GFX13:       ; %bb.0:
-; GFX13-NEXT:    s_clause 0x1
-; GFX13-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-NEXT:    s_mov_b32 s4, 0x3ff
-; GFX13-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
-; GFX13-NEXT:    s_wait_kmcnt 0x0
-; GFX13-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX13-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX13-NEXT:    s_endpgm
-  %tidx = call i32 @llvm.amdgcn.workitem.id.x()
-  %undef = freeze i32 poison
-  %v = call i32 @llvm.amdgcn.permlanex16.i32(i32 %undef, i32 %tidx, i32 %src1, i32 %src2, i1 true, i1 true)
-  store i32 %v, ptr addrspace(1) %out
-  ret void
-}
-
-define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_bc_f32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-LABEL: v_permlanex16_b32_i_tid_fi_bc_f32:
-; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_clause 0x1
-; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX10-NEXT:    global_store_dword v1, v0, s[2:3]
-; GFX10-NEXT:    s_endpgm
-;
-; GFX11-LABEL: v_permlanex16_b32_i_tid_fi_bc_f32:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
-; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX11-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX11-NEXT:    s_endpgm
-;
-; GFX12-LABEL: v_permlanex16_b32_i_tid_fi_bc_f32:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_clause 0x1
-; GFX12-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
-; GFX12-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX12-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX12-NEXT:    s_endpgm
-;
-; GFX13-LABEL: v_permlanex16_b32_i_tid_fi_bc_f32:
-; GFX13:       ; %bb.0:
-; GFX13-NEXT:    s_clause 0x1
-; GFX13-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-NEXT:    s_mov_b32 s4, 0x3ff
-; GFX13-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
-; GFX13-NEXT:    s_wait_kmcnt 0x0
-; GFX13-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX13-NEXT:    global_store_b32 v1, v0, s[2:3]
-; GFX13-NEXT:    s_endpgm
   %tidx = call i32 @llvm.amdgcn.workitem.id.x()
   %tidx_f32 = bitcast i32 %tidx to float
   %undef = freeze float poison
@@ -11273,11 +12295,14 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_bc_i64(ptr addrspace(1) %o
 ; GFX10-SDAG-NEXT:    s_clause 0x1
 ; GFX10-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
 ; GFX10-SDAG-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v3, 0
+; GFX10-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, s5
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v1, s4
 ; GFX10-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_permlanex16_b32 v1, v2, s0, s1 op_sel:[1,1]
-; GFX10-SDAG-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX10-SDAG-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v2, v3, s0, s1 op_sel:[1,1]
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX10-SDAG-NEXT:    global_store_dwordx2 v3, v[1:2], s[2:3]
 ; GFX10-SDAG-NEXT:    s_endpgm
 ;
 ; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_bc_i64:
@@ -11297,13 +12322,14 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_bc_i64(ptr addrspace(1) %o
 ; GFX11-SDAG-NEXT:    s_clause 0x1
 ; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
 ; GFX11-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, 0
-; GFX11-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT:    v_permlanex16_b32 v1, v2, s0, s1 op_sel:[1,1]
-; GFX11-SDAG-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX11-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v2, v3, s0, s1 op_sel:[1,1]
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX11-SDAG-NEXT:    global_store_b64 v3, v[1:2], s[2:3]
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
 ; GFX11-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_bc_i64:
@@ -11325,13 +12351,14 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_bc_i64(ptr addrspace(1) %o
 ; GFX12-SDAG-NEXT:    s_clause 0x1
 ; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
 ; GFX12-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-SDAG-NEXT:    v_mov_b32_e32 v2, 0
-; GFX12-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-NEXT:    v_permlanex16_b32 v1, v2, s0, s1 op_sel:[1,1]
-; GFX12-SDAG-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX12-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v2, v3, s0, s1 op_sel:[1,1]
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX12-SDAG-NEXT:    global_store_b64 v3, v[1:2], s[2:3]
 ; GFX12-SDAG-NEXT:    s_endpgm
 ;
 ; GFX12-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_bc_i64:
@@ -11353,14 +12380,16 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_bc_i64(ptr addrspace(1) %o
 ; GFX13-SDAG-NEXT:    s_clause 0x1
 ; GFX13-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
 ; GFX13-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT:    s_mov_b32 s4, 0x3ff
-; GFX13-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-SDAG-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX13-SDAG-NEXT:    s_mov_b32 s5, 0x3ff
+; GFX13-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
 ; GFX13-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT:    v_permlanex16_b32 v1, v2, s0, s1 op_sel:[1,1]
+; GFX13-SDAG-NEXT:    v_permlanex16_b32 v2, v3, s0, s1 op_sel:[1,1]
 ; GFX13-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX13-SDAG-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX13-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-SDAG-NEXT:    v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX13-SDAG-NEXT:    global_store_b64 v3, v[1:2], s[2:3]
 ; GFX13-SDAG-NEXT:    s_endpgm
 ;
 ; GFX13-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_bc_i64:
@@ -11388,15 +12417,18 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_bc_i64(ptr addrspace(1) %o
 define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_bc_f64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
 ; GFX10-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_bc_f64:
 ; GFX10-SDAG:       ; %bb.0:
-; GFX10-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
 ; GFX10-SDAG-NEXT:    s_clause 0x1
 ; GFX10-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x30
 ; GFX10-SDAG-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v4, 0
+; GFX10-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v3, s5
+; GFX10-SDAG-NEXT:    v_mov_b32_e32 v2, s4
 ; GFX10-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT:    v_permlanex16_b32 v1, v1, s0, s1 op_sel:[1,1]
-; GFX10-SDAG-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX10-SDAG-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v3, v1, s0, s1 op_sel:[1,1]
+; GFX10-SDAG-NEXT:    v_permlanex16_b32 v2, v0, s0, s1 op_sel:[1,1]
+; GFX10-SDAG-NEXT:    global_store_dwordx2 v4, v[2:3], s[2:3]
 ; GFX10-SDAG-NEXT:    s_endpgm
 ;
 ; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_bc_f64:
@@ -11414,17 +12446,20 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_bc_f64(ptr addrspace(1) %o
 ;
 ; GFX11-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_bc_f64:
 ; GFX11-SDAG:       ; %bb.0:
-; GFX11-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GFX11-SDAG-NEXT:    s_clause 0x1
 ; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
 ; GFX11-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT:    v_mov_b32_e32 v4, 0
+; GFX11-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX11-SDAG-NEXT:    v_dual_mov_b32 v3, s5 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT:    v_mov_b32_e32 v2, s4
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX11-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
 ; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT:    v_permlanex16_b32 v1, v1, s0, s1 op_sel:[1,1]
-; GFX11-SDAG-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX11-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v3, v1, s0, s1 op_sel:[1,1]
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-SDAG-NEXT:    v_permlanex16_b32 v2, v0, s0, s1 op_sel:[1,1]
+; GFX11-SDAG-NEXT:    global_store_b64 v4, v[2:3], s[2:3]
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
 ; GFX11-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_bc_f64:
@@ -11444,17 +12479,20 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_bc_f64(ptr addrspace(1) %o
 ;
 ; GFX12-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_bc_f64:
 ; GFX12-SDAG:       ; %bb.0:
-; GFX12-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GFX12-SDAG-NEXT:    s_clause 0x1
 ; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30
 ; GFX12-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX12-SDAG-NEXT:    v_mov_b32_e32 v4, 0
+; GFX12-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v3, s5 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT:    v_mov_b32_e32 v2, s4
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-NEXT:    v_permlanex16_b32 v1, v1, s0, s1 op_sel:[1,1]
-; GFX12-SDAG-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX12-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v3, v1, s0, s1 op_sel:[1,1]
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT:    v_permlanex16_b32 v2, v0, s0, s1 op_sel:[1,1]
+; GFX12-SDAG-NEXT:    global_store_b64 v4, v[2:3], s[2:3]
 ; GFX12-SDAG-NEXT:    s_endpgm
 ;
 ; GFX12-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_bc_f64:
@@ -11474,18 +12512,20 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_bc_f64(ptr addrspace(1) %o
 ;
 ; GFX13-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_bc_f64:
 ; GFX13-SDAG:       ; %bb.0:
-; GFX13-SDAG-NEXT:    s_mov_b32 s0, 0x3ff
-; GFX13-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX13-SDAG-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s0, v0 bitop3:0x40
 ; GFX13-SDAG-NEXT:    s_clause 0x1
 ; GFX13-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x30 nv
 ; GFX13-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-SDAG-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX13-SDAG-NEXT:    s_mov_b32 s6, 0x3ff
+; GFX13-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v3, s5 :: v_dual_bitop2_b32 v0, s6, v0 bitop3:0x40
+; GFX13-SDAG-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v4, 0
 ; GFX13-SDAG-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
 ; GFX13-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX13-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT:    v_permlanex16_b32 v1, v1, s0, s1 op_sel:[1,1]
-; GFX13-SDAG-NEXT:    v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX13-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-SDAG-NEXT:    v_permlanex16_b32 v3, v1, s0, s1 op_sel:[1,1]
+; GFX13-SDAG-NEXT:    v_permlanex16_b32 v2, v0, s0, s1 op_sel:[1,1]
+; GFX13-SDAG-NEXT:    global_store_b64 v4, v[2:3], s[2:3]
 ; GFX13-SDAG-NEXT:    s_endpgm
 ;
 ; GFX13-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_bc_f64:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane16.var.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane16.var.ll
index bb0aff99a2a76..d8ea5ab194e2a 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane16.var.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane16.var.ll
@@ -426,11 +426,13 @@ define amdgpu_kernel void @v_permlane16var_b32_undef_tid(ptr addrspace(1) %out,
 ; GFX12-SDAG-NEXT:    s_load_b32 s2, s[4:5], 0x30
 ; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX12-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT:    ; freeze: $sgpr3
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-NEXT:    v_permlane16_var_b32 v0, v0, v1
-; GFX12-SDAG-NEXT:    global_store_b32 v2, v0, s[0:1]
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, s3
+; GFX12-SDAG-NEXT:    v_mov_b32_e32 v3, 0
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT:    v_permlane16_var_b32 v2, v0, v1
+; GFX12-SDAG-NEXT:    global_store_b32 v3, v2, s[0:1]
 ; GFX12-SDAG-NEXT:    s_endpgm
 ;
 ; GFX12-GISEL-LABEL: v_permlane16var_b32_undef_tid:
@@ -493,11 +495,13 @@ define amdgpu_kernel void @v_permlane16var_b32_i_tid_fi(ptr addrspace(1) %out, i
 ; GFX12-SDAG-NEXT:    s_load_b32 s2, s[4:5], 0x30
 ; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX12-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT:    ; freeze: $sgpr3
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-NEXT:    v_permlane16_var_b32 v0, v0, v1 op_sel:[1,0]
-; GFX12-SDAG-NEXT:    global_store_b32 v2, v0, s[0:1]
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, s3
+; GFX12-SDAG-NEXT:    v_mov_b32_e32 v3, 0
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT:    v_permlane16_var_b32 v2, v0, v1 op_sel:[1,0]
+; GFX12-SDAG-NEXT:    global_store_b32 v3, v2, s[0:1]
 ; GFX12-SDAG-NEXT:    s_endpgm
 ;
 ; GFX12-GISEL-LABEL: v_permlane16var_b32_i_tid_fi:
@@ -526,11 +530,13 @@ define amdgpu_kernel void @v_permlane16var_b32_i_tid_bc(ptr addrspace(1) %out, i
 ; GFX12-SDAG-NEXT:    s_load_b32 s2, s[4:5], 0x30
 ; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX12-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT:    ; freeze: $sgpr3
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-NEXT:    v_permlane16_var_b32 v0, v0, v1 op_sel:[0,1]
-; GFX12-SDAG-NEXT:    global_store_b32 v2, v0, s[0:1]
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, s3
+; GFX12-SDAG-NEXT:    v_mov_b32_e32 v3, 0
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT:    v_permlane16_var_b32 v2, v0, v1 op_sel:[0,1]
+; GFX12-SDAG-NEXT:    global_store_b32 v3, v2, s[0:1]
 ; GFX12-SDAG-NEXT:    s_endpgm
 ;
 ; GFX12-GISEL-LABEL: v_permlane16var_b32_i_tid_bc:
@@ -559,11 +565,13 @@ define amdgpu_kernel void @v_permlane16var_b32_i_tid_fi_bc(ptr addrspace(1) %out
 ; GFX12-SDAG-NEXT:    s_load_b32 s2, s[4:5], 0x30
 ; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX12-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT:    ; freeze: $sgpr3
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-NEXT:    v_permlane16_var_b32 v0, v0, v1 op_sel:[1,1]
-; GFX12-SDAG-NEXT:    global_store_b32 v2, v0, s[0:1]
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, s3
+; GFX12-SDAG-NEXT:    v_mov_b32_e32 v3, 0
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT:    v_permlane16_var_b32 v2, v0, v1 op_sel:[1,1]
+; GFX12-SDAG-NEXT:    global_store_b32 v3, v2, s[0:1]
 ; GFX12-SDAG-NEXT:    s_endpgm
 ;
 ; GFX12-GISEL-LABEL: v_permlane16var_b32_i_tid_fi_bc:
@@ -624,11 +632,13 @@ define amdgpu_kernel void @v_permlanex16var_b32_undef_tid(ptr addrspace(1) %out,
 ; GFX12-SDAG-NEXT:    s_load_b32 s2, s[4:5], 0x30
 ; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX12-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT:    ; freeze: $sgpr3
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-NEXT:    v_permlanex16_var_b32 v0, v0, v1
-; GFX12-SDAG-NEXT:    global_store_b32 v2, v0, s[0:1]
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, s3
+; GFX12-SDAG-NEXT:    v_mov_b32_e32 v3, 0
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT:    v_permlanex16_var_b32 v2, v0, v1
+; GFX12-SDAG-NEXT:    global_store_b32 v3, v2, s[0:1]
 ; GFX12-SDAG-NEXT:    s_endpgm
 ;
 ; GFX12-GISEL-LABEL: v_permlanex16var_b32_undef_tid:
@@ -691,11 +701,13 @@ define amdgpu_kernel void @v_permlanex16var_b32_i_tid_fi(ptr addrspace(1) %out,
 ; GFX12-SDAG-NEXT:    s_load_b32 s2, s[4:5], 0x30
 ; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX12-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT:    ; freeze: $sgpr3
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-NEXT:    v_permlanex16_var_b32 v0, v0, v1 op_sel:[1,0]
-; GFX12-SDAG-NEXT:    global_store_b32 v2, v0, s[0:1]
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, s3
+; GFX12-SDAG-NEXT:    v_mov_b32_e32 v3, 0
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT:    v_permlanex16_var_b32 v2, v0, v1 op_sel:[1,0]
+; GFX12-SDAG-NEXT:    global_store_b32 v3, v2, s[0:1]
 ; GFX12-SDAG-NEXT:    s_endpgm
 ;
 ; GFX12-GISEL-LABEL: v_permlanex16var_b32_i_tid_fi:
@@ -724,11 +736,13 @@ define amdgpu_kernel void @v_permlanex16var_b32_i_tid_bc(ptr addrspace(1) %out,
 ; GFX12-SDAG-NEXT:    s_load_b32 s2, s[4:5], 0x30
 ; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX12-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT:    ; freeze: $sgpr3
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-NEXT:    v_permlanex16_var_b32 v0, v0, v1 op_sel:[0,1]
-; GFX12-SDAG-NEXT:    global_store_b32 v2, v0, s[0:1]
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, s3
+; GFX12-SDAG-NEXT:    v_mov_b32_e32 v3, 0
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT:    v_permlanex16_var_b32 v2, v0, v1 op_sel:[0,1]
+; GFX12-SDAG-NEXT:    global_store_b32 v3, v2, s[0:1]
 ; GFX12-SDAG-NEXT:    s_endpgm
 ;
 ; GFX12-GISEL-LABEL: v_permlanex16var_b32_i_tid_bc:
@@ -757,11 +771,13 @@ define amdgpu_kernel void @v_permlanex16var_b32_i_tid_fi_bc(ptr addrspace(1) %ou
 ; GFX12-SDAG-NEXT:    s_load_b32 s2, s[4:5], 0x30
 ; GFX12-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX12-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT:    ; freeze: $sgpr3
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-NEXT:    v_permlanex16_var_b32 v0, v0, v1 op_sel:[1,1]
-; GFX12-SDAG-NEXT:    global_store_b32 v2, v0, s[0:1]
+; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, s3
+; GFX12-SDAG-NEXT:    v_mov_b32_e32 v3, 0
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT:    v_permlanex16_var_b32 v2, v0, v1 op_sel:[1,1]
+; GFX12-SDAG-NEXT:    global_store_b32 v3, v2, s[0:1]
 ; GFX12-SDAG-NEXT:    s_endpgm
 ;
 ; GFX12-GISEL-LABEL: v_permlanex16var_b32_i_tid_fi_bc:
diff --git a/llvm/test/CodeGen/AMDGPU/mad_64_32.ll b/llvm/test/CodeGen/AMDGPU/mad_64_32.ll
index 68767c5183c3a..e3dfbb8b6add3 100644
--- a/llvm/test/CodeGen/AMDGPU/mad_64_32.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad_64_32.ll
@@ -528,8 +528,10 @@ define i63 @mad_i64_i32_sextops_i31_i63(i31 %arg0, i31 %arg1, i63 %arg2) #0 {
 ; SI-LABEL: mad_i64_i32_sextops_i31_i63:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v4, v1
+; SI-NEXT:    v_mov_b32_e32 v5, v2
 ; SI-NEXT:    v_bfe_i32 v0, v0, 0, 31
-; SI-NEXT:    v_bfe_i32 v1, v1, 0, 31
+; SI-NEXT:    v_bfe_i32 v1, v4, 0, 31
 ; SI-NEXT:    v_mul_lo_u32 v4, v0, v1
 ; SI-NEXT:    v_mul_hi_i32 v1, v0, v1
 ; SI-NEXT:    v_add_i32_e32 v0, vcc, v4, v2
diff --git a/llvm/test/CodeGen/AMDGPU/mdt-preserving-crash.ll b/llvm/test/CodeGen/AMDGPU/mdt-preserving-crash.ll
index 93feacb33c11a..7b0bf9ad6a0b2 100644
--- a/llvm/test/CodeGen/AMDGPU/mdt-preserving-crash.ll
+++ b/llvm/test/CodeGen/AMDGPU/mdt-preserving-crash.ll
@@ -25,9 +25,10 @@ define protected amdgpu_kernel void @_RSENC_PRInit______________________________
 ; CHECK-NEXT:    s_and_b64 vcc, exec, s[4:5]
 ; CHECK-NEXT:    s_cbranch_vccnz .LBB0_11
 ; CHECK-NEXT:  ; %bb.2: ; %lor.lhs.false17
-; CHECK-NEXT:    s_cmp_eq_u32 s4, 0
+; CHECK-NEXT:    ; freeze: $sgpr4
 ; CHECK-NEXT:  .LBB0_3: ; %while.cond.i
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    s_cmp_eq_u32 s4, 0
 ; CHECK-NEXT:    s_cbranch_scc1 .LBB0_3
 ; CHECK-NEXT:  ; %bb.4: ; %if.end60
 ; CHECK-NEXT:    s_cbranch_execz .LBB0_10
diff --git a/llvm/test/CodeGen/AMDGPU/mul_int24.ll b/llvm/test/CodeGen/AMDGPU/mul_int24.ll
index 155909e6af0a3..935cc3ae572ba 100644
--- a/llvm/test/CodeGen/AMDGPU/mul_int24.ll
+++ b/llvm/test/CodeGen/AMDGPU/mul_int24.ll
@@ -497,13 +497,13 @@ define amdgpu_kernel void @test_smul24_i33(ptr addrspace(1) %out, i33 %a, i33 %b
 ; GFX9-LABEL: test_smul24_i33:
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x2c
-; GFX9-NEXT:    s_load_dword s7, s[4:5], 0x34
+; GFX9-NEXT:    s_load_dword s8, s[4:5], 0x34
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
 ; GFX9-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX9-NEXT:    s_mov_b32 s2, -1
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_bfe_i32 s4, s6, 0x180000
-; GFX9-NEXT:    s_bfe_i32 s6, s7, 0x180000
+; GFX9-NEXT:    s_bfe_i32 s6, s8, 0x180000
 ; GFX9-NEXT:    s_mul_hi_i32 s5, s4, s6
 ; GFX9-NEXT:    s_mul_i32 s4, s4, s6
 ; GFX9-NEXT:    s_lshl_b64 s[4:5], s[4:5], 31
@@ -599,13 +599,13 @@ define amdgpu_kernel void @test_smulhi24_i33(ptr addrspace(1) %out, i33 %a, i33
 ; GFX9-LABEL: test_smulhi24_i33:
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x2c
-; GFX9-NEXT:    s_load_dword s7, s[4:5], 0x34
+; GFX9-NEXT:    s_load_dword s8, s[4:5], 0x34
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
 ; GFX9-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX9-NEXT:    s_mov_b32 s2, -1
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_bfe_i32 s4, s6, 0x180000
-; GFX9-NEXT:    s_bfe_i32 s5, s7, 0x180000
+; GFX9-NEXT:    s_bfe_i32 s5, s8, 0x180000
 ; GFX9-NEXT:    s_mul_hi_i32 s4, s4, s5
 ; GFX9-NEXT:    s_and_b32 s4, s4, 1
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s4
diff --git a/llvm/test/CodeGen/AMDGPU/permute_i8.ll b/llvm/test/CodeGen/AMDGPU/permute_i8.ll
index 0e67c1c517b68..8cda8b0e97f7a 100644
--- a/llvm/test/CodeGen/AMDGPU/permute_i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/permute_i8.ll
@@ -3768,18 +3768,21 @@ define hidden void @any_extend_to_perm_wide_src(ptr addrspace(1) %in0, ptr addrs
 ; GFX10-NEXT:    global_load_ubyte v8, v[2:3], off
 ; GFX10-NEXT:    ; kill: killed $vgpr0 killed $vgpr1
 ; GFX10-NEXT:    ; kill: killed $vgpr2 killed $vgpr3
-; GFX10-NEXT:    s_lshl_b32 s4, s4, 8
-; GFX10-NEXT:    s_lshl_b32 s5, s4, 8
+; GFX10-NEXT:    ; freeze: $sgpr4
+; GFX10-NEXT:    ; freeze: $sgpr5
+; GFX10-NEXT:    ; freeze: $sgpr6
+; GFX10-NEXT:    s_lshl_b32 s5, s5, 8
+; GFX10-NEXT:    s_lshl_b32 s6, s6, 8
 ; GFX10-NEXT:    s_waitcnt vmcnt(2)
 ; GFX10-NEXT:    v_lshrrev_b64 v[0:1], 24, v[6:7]
 ; GFX10-NEXT:    s_waitcnt vmcnt(1)
 ; GFX10-NEXT:    v_lshrrev_b64 v[1:2], 16, v[12:13]
 ; GFX10-NEXT:    v_lshrrev_b64 v[2:3], 8, v[10:11]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_or_b32_sdwa v3, v8, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_or_b32_sdwa v3, v8, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX10-NEXT:    v_perm_b32 v0, v1, v0, 0xc0c0004
 ; GFX10-NEXT:    v_perm_b32 v1, s4, v2, 0xc0c0004
-; GFX10-NEXT:    v_or_b32_sdwa v2, v9, s5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT:    v_or_b32_sdwa v2, v9, s6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX10-NEXT:    v_lshl_or_b32 v0, v0, 16, v1
 ; GFX10-NEXT:    v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
 ; GFX10-NEXT:    global_store_dwordx2 v[4:5], v[0:1], off
@@ -3794,19 +3797,22 @@ define hidden void @any_extend_to_perm_wide_src(ptr addrspace(1) %in0, ptr addrs
 ; GFX9-NEXT:    ; kill: killed $vgpr0 killed $vgpr1
 ; GFX9-NEXT:    ; kill: killed $vgpr2 killed $vgpr3
 ; GFX9-NEXT:    s_mov_b32 s4, 0xc0c0004
+; GFX9-NEXT:    ; freeze: $sgpr6
+; GFX9-NEXT:    ; freeze: $sgpr7
 ; GFX9-NEXT:    v_mov_b32_e32 v14, 0xc0c0004
-; GFX9-NEXT:    s_lshl_b32 s5, s4, 8
-; GFX9-NEXT:    s_lshl_b32 s6, s4, 8
+; GFX9-NEXT:    s_lshl_b32 s6, s6, 8
+; GFX9-NEXT:    s_lshl_b32 s7, s7, 8
+; GFX9-NEXT:    ; freeze: $sgpr5
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    v_lshrrev_b64 v[0:1], 8, v[6:7]
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_lshrrev_b64 v[1:2], 24, v[10:11]
 ; GFX9-NEXT:    v_lshrrev_b64 v[2:3], 16, v[8:9]
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_or_b32_sdwa v3, v12, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v6, v13, s6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v3, v12, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v6, v13, s7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
 ; GFX9-NEXT:    v_perm_b32 v1, v2, v1, s4
-; GFX9-NEXT:    v_perm_b32 v0, s4, v0, v14
+; GFX9-NEXT:    v_perm_b32 v0, s5, v0, v14
 ; GFX9-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
 ; GFX9-NEXT:    v_or_b32_sdwa v1, v3, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
 ; GFX9-NEXT:    global_store_dwordx2 v[4:5], v[0:1], off
diff --git a/llvm/test/CodeGen/AMDGPU/promote-alloca-vector-dynamic-idx-bitcasts-llc.ll b/llvm/test/CodeGen/AMDGPU/promote-alloca-vector-dynamic-idx-bitcasts-llc.ll
index c73d31c368846..ddfe187cc4337 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-alloca-vector-dynamic-idx-bitcasts-llc.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-alloca-vector-dynamic-idx-bitcasts-llc.ll
@@ -6,46 +6,144 @@
 define amdgpu_kernel void @test_bitcast_llc_v128i8_v16i8(ptr addrspace(1) %out, i32 %idx) {
 ; GFX9-LABEL: test_bitcast_llc_v128i8_v16i8:
 ; GFX9:       ; %bb.0: ; %entry
-; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0
+; GFX9-NEXT:    s_load_dword s33, s[4:5], 0x8
+; GFX9-NEXT:    s_load_dwordx2 s[34:35], s[4:5], 0x0
+; GFX9-NEXT:    ; freeze: $sgpr30_sgpr31
+; GFX9-NEXT:    ; freeze: $sgpr28_sgpr29
+; GFX9-NEXT:    ; freeze: $sgpr26_sgpr27
+; GFX9-NEXT:    ; freeze: $sgpr24_sgpr25
+; GFX9-NEXT:    ; freeze: $sgpr22_sgpr23
+; GFX9-NEXT:    ; freeze: $sgpr20_sgpr21
+; GFX9-NEXT:    ; freeze: $sgpr18_sgpr19
+; GFX9-NEXT:    ; freeze: $sgpr16_sgpr17
+; GFX9-NEXT:    ; freeze: $sgpr14_sgpr15
+; GFX9-NEXT:    ; freeze: $sgpr12_sgpr13
+; GFX9-NEXT:    ; freeze: $sgpr10_sgpr11
+; GFX9-NEXT:    ; freeze: $sgpr8_sgpr9
+; GFX9-NEXT:    ; freeze: $sgpr6_sgpr7
+; GFX9-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX9-NEXT:    ; freeze: $sgpr2_sgpr3
+; GFX9-NEXT:    ; freeze: $sgpr0_sgpr1
+; GFX9-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
+; GFX9-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
+; GFX9-NEXT:    v_mov_b64_e32 v[4:5], s[4:5]
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_lshl_b32 s2, s2, 2
-; GFX9-NEXT:    s_set_gpr_idx_on s2, gpr_idx(SRC0)
-; GFX9-NEXT:    v_mov_b32_e32 v3, v3
-; GFX9-NEXT:    v_mov_b32_e32 v2, v2
-; GFX9-NEXT:    v_mov_b32_e32 v1, v1
-; GFX9-NEXT:    v_mov_b32_e32 v0, v0
+; GFX9-NEXT:    s_lshl_b32 s33, s33, 2
+; GFX9-NEXT:    v_mov_b64_e32 v[6:7], s[6:7]
+; GFX9-NEXT:    v_mov_b64_e32 v[8:9], s[8:9]
+; GFX9-NEXT:    v_mov_b64_e32 v[10:11], s[10:11]
+; GFX9-NEXT:    v_mov_b64_e32 v[12:13], s[12:13]
+; GFX9-NEXT:    v_mov_b64_e32 v[14:15], s[14:15]
+; GFX9-NEXT:    v_mov_b64_e32 v[16:17], s[16:17]
+; GFX9-NEXT:    v_mov_b64_e32 v[18:19], s[18:19]
+; GFX9-NEXT:    v_mov_b64_e32 v[20:21], s[20:21]
+; GFX9-NEXT:    v_mov_b64_e32 v[22:23], s[22:23]
+; GFX9-NEXT:    v_mov_b64_e32 v[24:25], s[24:25]
+; GFX9-NEXT:    v_mov_b64_e32 v[26:27], s[26:27]
+; GFX9-NEXT:    v_mov_b64_e32 v[28:29], s[28:29]
+; GFX9-NEXT:    v_mov_b64_e32 v[30:31], s[30:31]
+; GFX9-NEXT:    s_set_gpr_idx_on s33, gpr_idx(SRC0)
+; GFX9-NEXT:    v_mov_b32_e32 v35, v3
+; GFX9-NEXT:    v_mov_b32_e32 v34, v2
 ; GFX9-NEXT:    s_set_gpr_idx_off
-; GFX9-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX9-NEXT:    v_mov_b32_e32 v36, 0
+; GFX9-NEXT:    s_set_gpr_idx_on s33, gpr_idx(SRC0)
+; GFX9-NEXT:    v_mov_b32_e32 v33, v1
+; GFX9-NEXT:    v_mov_b32_e32 v32, v0
+; GFX9-NEXT:    s_set_gpr_idx_off
+; GFX9-NEXT:    global_store_dwordx4 v36, v[32:35], s[34:35]
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: test_bitcast_llc_v128i8_v16i8:
 ; GFX11:       ; %bb.0: ; %entry
 ; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x8
-; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
-; GFX11-NEXT:    v_mov_b32_e32 v4, 0
+; GFX11-NEXT:    s_load_b32 s33, s[4:5], 0x8
+; GFX11-NEXT:    s_load_b64 s[34:35], s[4:5], 0x0
+; GFX11-NEXT:    ; freeze: $sgpr30_sgpr31
+; GFX11-NEXT:    ; freeze: $sgpr28_sgpr29
+; GFX11-NEXT:    ; freeze: $sgpr26_sgpr27
+; GFX11-NEXT:    ; freeze: $sgpr24_sgpr25
+; GFX11-NEXT:    ; freeze: $sgpr22_sgpr23
+; GFX11-NEXT:    ; freeze: $sgpr20_sgpr21
+; GFX11-NEXT:    ; freeze: $sgpr18_sgpr19
+; GFX11-NEXT:    ; freeze: $sgpr16_sgpr17
+; GFX11-NEXT:    ; freeze: $sgpr14_sgpr15
+; GFX11-NEXT:    ; freeze: $sgpr12_sgpr13
+; GFX11-NEXT:    ; freeze: $sgpr10_sgpr11
+; GFX11-NEXT:    ; freeze: $sgpr8_sgpr9
+; GFX11-NEXT:    ; freeze: $sgpr6_sgpr7
+; GFX11-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX11-NEXT:    ; freeze: $sgpr2_sgpr3
+; GFX11-NEXT:    ; freeze: $sgpr0_sgpr1
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX11-NEXT:    v_dual_mov_b32 v30, s30 :: v_dual_mov_b32 v31, s31
+; GFX11-NEXT:    v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
+; GFX11-NEXT:    v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s7
+; GFX11-NEXT:    v_dual_mov_b32 v8, s8 :: v_dual_mov_b32 v9, s9
+; GFX11-NEXT:    v_dual_mov_b32 v10, s10 :: v_dual_mov_b32 v11, s11
+; GFX11-NEXT:    v_dual_mov_b32 v12, s12 :: v_dual_mov_b32 v13, s13
+; GFX11-NEXT:    v_dual_mov_b32 v14, s14 :: v_dual_mov_b32 v15, s15
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_lshl_b32 m0, s2, 2
-; GFX11-NEXT:    v_movrels_b32_e32 v3, v3
-; GFX11-NEXT:    v_movrels_b32_e32 v2, v2
-; GFX11-NEXT:    v_movrels_b32_e32 v1, v1
-; GFX11-NEXT:    v_movrels_b32_e32 v0, v0
-; GFX11-NEXT:    global_store_b128 v4, v[0:3], s[0:1]
+; GFX11-NEXT:    s_lshl_b32 m0, s33, 2
+; GFX11-NEXT:    v_dual_mov_b32 v16, s16 :: v_dual_mov_b32 v17, s17
+; GFX11-NEXT:    v_dual_mov_b32 v18, s18 :: v_dual_mov_b32 v19, s19
+; GFX11-NEXT:    v_dual_mov_b32 v20, s20 :: v_dual_mov_b32 v21, s21
+; GFX11-NEXT:    v_dual_mov_b32 v22, s22 :: v_dual_mov_b32 v23, s23
+; GFX11-NEXT:    v_dual_mov_b32 v24, s24 :: v_dual_mov_b32 v25, s25
+; GFX11-NEXT:    v_dual_mov_b32 v26, s26 :: v_dual_mov_b32 v27, s27
+; GFX11-NEXT:    v_dual_mov_b32 v28, s28 :: v_dual_mov_b32 v29, s29
+; GFX11-NEXT:    v_mov_b32_e32 v35, 0
+; GFX11-NEXT:    v_movrels_b32_e32 v34, v3
+; GFX11-NEXT:    v_movrels_b32_e32 v33, v2
+; GFX11-NEXT:    v_movrels_b32_e32 v32, v1
+; GFX11-NEXT:    v_movrels_b32_e32 v31, v0
+; GFX11-NEXT:    global_store_b128 v35, v[31:34], s[34:35]
 ; GFX11-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: test_bitcast_llc_v128i8_v16i8:
 ; GFX12:       ; %bb.0: ; %entry
-; GFX12-NEXT:    s_load_b96 s[0:2], s[4:5], 0x0
-; GFX12-NEXT:    v_mov_b32_e32 v4, 0
+; GFX12-NEXT:    s_load_b96 s[36:38], s[4:5], 0x0
+; GFX12-NEXT:    ; freeze: $sgpr30_sgpr31
+; GFX12-NEXT:    ; freeze: $sgpr28_sgpr29
+; GFX12-NEXT:    ; freeze: $sgpr26_sgpr27
+; GFX12-NEXT:    ; freeze: $sgpr24_sgpr25
+; GFX12-NEXT:    ; freeze: $sgpr22_sgpr23
+; GFX12-NEXT:    ; freeze: $sgpr20_sgpr21
+; GFX12-NEXT:    ; freeze: $sgpr18_sgpr19
+; GFX12-NEXT:    ; freeze: $sgpr16_sgpr17
+; GFX12-NEXT:    ; freeze: $sgpr14_sgpr15
+; GFX12-NEXT:    ; freeze: $sgpr12_sgpr13
+; GFX12-NEXT:    ; freeze: $sgpr10_sgpr11
+; GFX12-NEXT:    ; freeze: $sgpr8_sgpr9
+; GFX12-NEXT:    ; freeze: $sgpr6_sgpr7
+; GFX12-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX12-NEXT:    ; freeze: $sgpr2_sgpr3
+; GFX12-NEXT:    ; freeze: $sgpr0_sgpr1
+; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-NEXT:    v_dual_mov_b32 v30, s30 :: v_dual_mov_b32 v31, s31
+; GFX12-NEXT:    v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
+; GFX12-NEXT:    v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s7
+; GFX12-NEXT:    v_dual_mov_b32 v8, s8 :: v_dual_mov_b32 v9, s9
+; GFX12-NEXT:    v_dual_mov_b32 v10, s10 :: v_dual_mov_b32 v11, s11
+; GFX12-NEXT:    v_dual_mov_b32 v12, s12 :: v_dual_mov_b32 v13, s13
+; GFX12-NEXT:    v_dual_mov_b32 v14, s14 :: v_dual_mov_b32 v15, s15
+; GFX12-NEXT:    v_dual_mov_b32 v16, s16 :: v_dual_mov_b32 v17, s17
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_lshl_b32 m0, s2, 2
-; GFX12-NEXT:    v_movrels_b32_e32 v3, v3
-; GFX12-NEXT:    v_movrels_b32_e32 v2, v2
-; GFX12-NEXT:    v_movrels_b32_e32 v1, v1
-; GFX12-NEXT:    v_movrels_b32_e32 v0, v0
-; GFX12-NEXT:    global_store_b128 v4, v[0:3], s[0:1]
+; GFX12-NEXT:    s_lshl_b32 m0, s38, 2
+; GFX12-NEXT:    v_dual_mov_b32 v18, s18 :: v_dual_mov_b32 v19, s19
+; GFX12-NEXT:    v_dual_mov_b32 v20, s20 :: v_dual_mov_b32 v21, s21
+; GFX12-NEXT:    v_dual_mov_b32 v22, s22 :: v_dual_mov_b32 v23, s23
+; GFX12-NEXT:    v_dual_mov_b32 v24, s24 :: v_dual_mov_b32 v25, s25
+; GFX12-NEXT:    v_dual_mov_b32 v26, s26 :: v_dual_mov_b32 v27, s27
+; GFX12-NEXT:    v_dual_mov_b32 v28, s28 :: v_dual_mov_b32 v29, s29
+; GFX12-NEXT:    v_mov_b32_e32 v35, 0
+; GFX12-NEXT:    v_movrels_b32_e32 v34, v3
+; GFX12-NEXT:    v_movrels_b32_e32 v33, v2
+; GFX12-NEXT:    v_movrels_b32_e32 v32, v1
+; GFX12-NEXT:    v_movrels_b32_e32 v31, v0
+; GFX12-NEXT:    global_store_b128 v35, v[31:34], s[36:37]
 ; GFX12-NEXT:    s_endpgm
 entry:
   %alloca = freeze <128 x i8> poison
@@ -59,46 +157,144 @@ entry:
 define amdgpu_kernel void @test_bitcast_llc_v64i16_v8i16(ptr addrspace(1) %out, i32 %idx) {
 ; GFX9-LABEL: test_bitcast_llc_v64i16_v8i16:
 ; GFX9:       ; %bb.0: ; %entry
-; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0
+; GFX9-NEXT:    s_load_dword s33, s[4:5], 0x8
+; GFX9-NEXT:    s_load_dwordx2 s[34:35], s[4:5], 0x0
+; GFX9-NEXT:    ; freeze: $sgpr30_sgpr31
+; GFX9-NEXT:    ; freeze: $sgpr28_sgpr29
+; GFX9-NEXT:    ; freeze: $sgpr26_sgpr27
+; GFX9-NEXT:    ; freeze: $sgpr24_sgpr25
+; GFX9-NEXT:    ; freeze: $sgpr22_sgpr23
+; GFX9-NEXT:    ; freeze: $sgpr20_sgpr21
+; GFX9-NEXT:    ; freeze: $sgpr18_sgpr19
+; GFX9-NEXT:    ; freeze: $sgpr16_sgpr17
+; GFX9-NEXT:    ; freeze: $sgpr14_sgpr15
+; GFX9-NEXT:    ; freeze: $sgpr12_sgpr13
+; GFX9-NEXT:    ; freeze: $sgpr10_sgpr11
+; GFX9-NEXT:    ; freeze: $sgpr8_sgpr9
+; GFX9-NEXT:    ; freeze: $sgpr6_sgpr7
+; GFX9-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX9-NEXT:    ; freeze: $sgpr2_sgpr3
+; GFX9-NEXT:    ; freeze: $sgpr0_sgpr1
+; GFX9-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
+; GFX9-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
+; GFX9-NEXT:    v_mov_b64_e32 v[4:5], s[4:5]
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_lshl_b32 s2, s2, 2
-; GFX9-NEXT:    s_set_gpr_idx_on s2, gpr_idx(SRC0)
-; GFX9-NEXT:    v_mov_b32_e32 v3, v3
-; GFX9-NEXT:    v_mov_b32_e32 v2, v2
-; GFX9-NEXT:    v_mov_b32_e32 v1, v1
-; GFX9-NEXT:    v_mov_b32_e32 v0, v0
+; GFX9-NEXT:    s_lshl_b32 s33, s33, 2
+; GFX9-NEXT:    v_mov_b64_e32 v[6:7], s[6:7]
+; GFX9-NEXT:    v_mov_b64_e32 v[8:9], s[8:9]
+; GFX9-NEXT:    v_mov_b64_e32 v[10:11], s[10:11]
+; GFX9-NEXT:    v_mov_b64_e32 v[12:13], s[12:13]
+; GFX9-NEXT:    v_mov_b64_e32 v[14:15], s[14:15]
+; GFX9-NEXT:    v_mov_b64_e32 v[16:17], s[16:17]
+; GFX9-NEXT:    v_mov_b64_e32 v[18:19], s[18:19]
+; GFX9-NEXT:    v_mov_b64_e32 v[20:21], s[20:21]
+; GFX9-NEXT:    v_mov_b64_e32 v[22:23], s[22:23]
+; GFX9-NEXT:    v_mov_b64_e32 v[24:25], s[24:25]
+; GFX9-NEXT:    v_mov_b64_e32 v[26:27], s[26:27]
+; GFX9-NEXT:    v_mov_b64_e32 v[28:29], s[28:29]
+; GFX9-NEXT:    v_mov_b64_e32 v[30:31], s[30:31]
+; GFX9-NEXT:    s_set_gpr_idx_on s33, gpr_idx(SRC0)
+; GFX9-NEXT:    v_mov_b32_e32 v35, v3
+; GFX9-NEXT:    v_mov_b32_e32 v34, v2
+; GFX9-NEXT:    s_set_gpr_idx_off
+; GFX9-NEXT:    v_mov_b32_e32 v36, 0
+; GFX9-NEXT:    s_set_gpr_idx_on s33, gpr_idx(SRC0)
+; GFX9-NEXT:    v_mov_b32_e32 v33, v1
+; GFX9-NEXT:    v_mov_b32_e32 v32, v0
 ; GFX9-NEXT:    s_set_gpr_idx_off
-; GFX9-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX9-NEXT:    global_store_dwordx4 v36, v[32:35], s[34:35]
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: test_bitcast_llc_v64i16_v8i16:
 ; GFX11:       ; %bb.0: ; %entry
 ; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x8
-; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
-; GFX11-NEXT:    v_mov_b32_e32 v4, 0
+; GFX11-NEXT:    s_load_b32 s33, s[4:5], 0x8
+; GFX11-NEXT:    s_load_b64 s[34:35], s[4:5], 0x0
+; GFX11-NEXT:    ; freeze: $sgpr30_sgpr31
+; GFX11-NEXT:    ; freeze: $sgpr28_sgpr29
+; GFX11-NEXT:    ; freeze: $sgpr26_sgpr27
+; GFX11-NEXT:    ; freeze: $sgpr24_sgpr25
+; GFX11-NEXT:    ; freeze: $sgpr22_sgpr23
+; GFX11-NEXT:    ; freeze: $sgpr20_sgpr21
+; GFX11-NEXT:    ; freeze: $sgpr18_sgpr19
+; GFX11-NEXT:    ; freeze: $sgpr16_sgpr17
+; GFX11-NEXT:    ; freeze: $sgpr14_sgpr15
+; GFX11-NEXT:    ; freeze: $sgpr12_sgpr13
+; GFX11-NEXT:    ; freeze: $sgpr10_sgpr11
+; GFX11-NEXT:    ; freeze: $sgpr8_sgpr9
+; GFX11-NEXT:    ; freeze: $sgpr6_sgpr7
+; GFX11-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX11-NEXT:    ; freeze: $sgpr2_sgpr3
+; GFX11-NEXT:    ; freeze: $sgpr0_sgpr1
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX11-NEXT:    v_dual_mov_b32 v30, s30 :: v_dual_mov_b32 v31, s31
+; GFX11-NEXT:    v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
+; GFX11-NEXT:    v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s7
+; GFX11-NEXT:    v_dual_mov_b32 v8, s8 :: v_dual_mov_b32 v9, s9
+; GFX11-NEXT:    v_dual_mov_b32 v10, s10 :: v_dual_mov_b32 v11, s11
+; GFX11-NEXT:    v_dual_mov_b32 v12, s12 :: v_dual_mov_b32 v13, s13
+; GFX11-NEXT:    v_dual_mov_b32 v14, s14 :: v_dual_mov_b32 v15, s15
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_lshl_b32 m0, s2, 2
-; GFX11-NEXT:    v_movrels_b32_e32 v3, v3
-; GFX11-NEXT:    v_movrels_b32_e32 v2, v2
-; GFX11-NEXT:    v_movrels_b32_e32 v1, v1
-; GFX11-NEXT:    v_movrels_b32_e32 v0, v0
-; GFX11-NEXT:    global_store_b128 v4, v[0:3], s[0:1]
+; GFX11-NEXT:    s_lshl_b32 m0, s33, 2
+; GFX11-NEXT:    v_dual_mov_b32 v16, s16 :: v_dual_mov_b32 v17, s17
+; GFX11-NEXT:    v_dual_mov_b32 v18, s18 :: v_dual_mov_b32 v19, s19
+; GFX11-NEXT:    v_dual_mov_b32 v20, s20 :: v_dual_mov_b32 v21, s21
+; GFX11-NEXT:    v_dual_mov_b32 v22, s22 :: v_dual_mov_b32 v23, s23
+; GFX11-NEXT:    v_dual_mov_b32 v24, s24 :: v_dual_mov_b32 v25, s25
+; GFX11-NEXT:    v_dual_mov_b32 v26, s26 :: v_dual_mov_b32 v27, s27
+; GFX11-NEXT:    v_dual_mov_b32 v28, s28 :: v_dual_mov_b32 v29, s29
+; GFX11-NEXT:    v_mov_b32_e32 v35, 0
+; GFX11-NEXT:    v_movrels_b32_e32 v34, v3
+; GFX11-NEXT:    v_movrels_b32_e32 v33, v2
+; GFX11-NEXT:    v_movrels_b32_e32 v32, v1
+; GFX11-NEXT:    v_movrels_b32_e32 v31, v0
+; GFX11-NEXT:    global_store_b128 v35, v[31:34], s[34:35]
 ; GFX11-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: test_bitcast_llc_v64i16_v8i16:
 ; GFX12:       ; %bb.0: ; %entry
-; GFX12-NEXT:    s_load_b96 s[0:2], s[4:5], 0x0
-; GFX12-NEXT:    v_mov_b32_e32 v4, 0
+; GFX12-NEXT:    s_load_b96 s[36:38], s[4:5], 0x0
+; GFX12-NEXT:    ; freeze: $sgpr30_sgpr31
+; GFX12-NEXT:    ; freeze: $sgpr28_sgpr29
+; GFX12-NEXT:    ; freeze: $sgpr26_sgpr27
+; GFX12-NEXT:    ; freeze: $sgpr24_sgpr25
+; GFX12-NEXT:    ; freeze: $sgpr22_sgpr23
+; GFX12-NEXT:    ; freeze: $sgpr20_sgpr21
+; GFX12-NEXT:    ; freeze: $sgpr18_sgpr19
+; GFX12-NEXT:    ; freeze: $sgpr16_sgpr17
+; GFX12-NEXT:    ; freeze: $sgpr14_sgpr15
+; GFX12-NEXT:    ; freeze: $sgpr12_sgpr13
+; GFX12-NEXT:    ; freeze: $sgpr10_sgpr11
+; GFX12-NEXT:    ; freeze: $sgpr8_sgpr9
+; GFX12-NEXT:    ; freeze: $sgpr6_sgpr7
+; GFX12-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX12-NEXT:    ; freeze: $sgpr2_sgpr3
+; GFX12-NEXT:    ; freeze: $sgpr0_sgpr1
+; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-NEXT:    v_dual_mov_b32 v30, s30 :: v_dual_mov_b32 v31, s31
+; GFX12-NEXT:    v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
+; GFX12-NEXT:    v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s7
+; GFX12-NEXT:    v_dual_mov_b32 v8, s8 :: v_dual_mov_b32 v9, s9
+; GFX12-NEXT:    v_dual_mov_b32 v10, s10 :: v_dual_mov_b32 v11, s11
+; GFX12-NEXT:    v_dual_mov_b32 v12, s12 :: v_dual_mov_b32 v13, s13
+; GFX12-NEXT:    v_dual_mov_b32 v14, s14 :: v_dual_mov_b32 v15, s15
+; GFX12-NEXT:    v_dual_mov_b32 v16, s16 :: v_dual_mov_b32 v17, s17
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_lshl_b32 m0, s2, 2
-; GFX12-NEXT:    v_movrels_b32_e32 v3, v3
-; GFX12-NEXT:    v_movrels_b32_e32 v2, v2
-; GFX12-NEXT:    v_movrels_b32_e32 v1, v1
-; GFX12-NEXT:    v_movrels_b32_e32 v0, v0
-; GFX12-NEXT:    global_store_b128 v4, v[0:3], s[0:1]
+; GFX12-NEXT:    s_lshl_b32 m0, s38, 2
+; GFX12-NEXT:    v_dual_mov_b32 v18, s18 :: v_dual_mov_b32 v19, s19
+; GFX12-NEXT:    v_dual_mov_b32 v20, s20 :: v_dual_mov_b32 v21, s21
+; GFX12-NEXT:    v_dual_mov_b32 v22, s22 :: v_dual_mov_b32 v23, s23
+; GFX12-NEXT:    v_dual_mov_b32 v24, s24 :: v_dual_mov_b32 v25, s25
+; GFX12-NEXT:    v_dual_mov_b32 v26, s26 :: v_dual_mov_b32 v27, s27
+; GFX12-NEXT:    v_dual_mov_b32 v28, s28 :: v_dual_mov_b32 v29, s29
+; GFX12-NEXT:    v_mov_b32_e32 v35, 0
+; GFX12-NEXT:    v_movrels_b32_e32 v34, v3
+; GFX12-NEXT:    v_movrels_b32_e32 v33, v2
+; GFX12-NEXT:    v_movrels_b32_e32 v32, v1
+; GFX12-NEXT:    v_movrels_b32_e32 v31, v0
+; GFX12-NEXT:    global_store_b128 v35, v[31:34], s[36:37]
 ; GFX12-NEXT:    s_endpgm
 entry:
   %alloca = freeze <64 x i16> poison
@@ -112,46 +308,144 @@ entry:
 define amdgpu_kernel void @test_bitcast_llc_v32i32_v4i32(ptr addrspace(1) %out, i32 %idx) {
 ; GFX9-LABEL: test_bitcast_llc_v32i32_v4i32:
 ; GFX9:       ; %bb.0: ; %entry
-; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0
+; GFX9-NEXT:    s_load_dword s33, s[4:5], 0x8
+; GFX9-NEXT:    s_load_dwordx2 s[34:35], s[4:5], 0x0
+; GFX9-NEXT:    ; freeze: $sgpr30_sgpr31
+; GFX9-NEXT:    ; freeze: $sgpr28_sgpr29
+; GFX9-NEXT:    ; freeze: $sgpr26_sgpr27
+; GFX9-NEXT:    ; freeze: $sgpr24_sgpr25
+; GFX9-NEXT:    ; freeze: $sgpr22_sgpr23
+; GFX9-NEXT:    ; freeze: $sgpr20_sgpr21
+; GFX9-NEXT:    ; freeze: $sgpr18_sgpr19
+; GFX9-NEXT:    ; freeze: $sgpr16_sgpr17
+; GFX9-NEXT:    ; freeze: $sgpr14_sgpr15
+; GFX9-NEXT:    ; freeze: $sgpr12_sgpr13
+; GFX9-NEXT:    ; freeze: $sgpr10_sgpr11
+; GFX9-NEXT:    ; freeze: $sgpr8_sgpr9
+; GFX9-NEXT:    ; freeze: $sgpr6_sgpr7
+; GFX9-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX9-NEXT:    ; freeze: $sgpr2_sgpr3
+; GFX9-NEXT:    ; freeze: $sgpr0_sgpr1
+; GFX9-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
+; GFX9-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
+; GFX9-NEXT:    v_mov_b64_e32 v[4:5], s[4:5]
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_lshl_b32 s2, s2, 2
-; GFX9-NEXT:    s_set_gpr_idx_on s2, gpr_idx(SRC0)
-; GFX9-NEXT:    v_mov_b32_e32 v3, v3
-; GFX9-NEXT:    v_mov_b32_e32 v2, v2
-; GFX9-NEXT:    v_mov_b32_e32 v1, v1
-; GFX9-NEXT:    v_mov_b32_e32 v0, v0
+; GFX9-NEXT:    s_lshl_b32 s33, s33, 2
+; GFX9-NEXT:    v_mov_b64_e32 v[6:7], s[6:7]
+; GFX9-NEXT:    v_mov_b64_e32 v[8:9], s[8:9]
+; GFX9-NEXT:    v_mov_b64_e32 v[10:11], s[10:11]
+; GFX9-NEXT:    v_mov_b64_e32 v[12:13], s[12:13]
+; GFX9-NEXT:    v_mov_b64_e32 v[14:15], s[14:15]
+; GFX9-NEXT:    v_mov_b64_e32 v[16:17], s[16:17]
+; GFX9-NEXT:    v_mov_b64_e32 v[18:19], s[18:19]
+; GFX9-NEXT:    v_mov_b64_e32 v[20:21], s[20:21]
+; GFX9-NEXT:    v_mov_b64_e32 v[22:23], s[22:23]
+; GFX9-NEXT:    v_mov_b64_e32 v[24:25], s[24:25]
+; GFX9-NEXT:    v_mov_b64_e32 v[26:27], s[26:27]
+; GFX9-NEXT:    v_mov_b64_e32 v[28:29], s[28:29]
+; GFX9-NEXT:    v_mov_b64_e32 v[30:31], s[30:31]
+; GFX9-NEXT:    s_set_gpr_idx_on s33, gpr_idx(SRC0)
+; GFX9-NEXT:    v_mov_b32_e32 v35, v3
+; GFX9-NEXT:    v_mov_b32_e32 v34, v2
 ; GFX9-NEXT:    s_set_gpr_idx_off
-; GFX9-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX9-NEXT:    v_mov_b32_e32 v36, 0
+; GFX9-NEXT:    s_set_gpr_idx_on s33, gpr_idx(SRC0)
+; GFX9-NEXT:    v_mov_b32_e32 v33, v1
+; GFX9-NEXT:    v_mov_b32_e32 v32, v0
+; GFX9-NEXT:    s_set_gpr_idx_off
+; GFX9-NEXT:    global_store_dwordx4 v36, v[32:35], s[34:35]
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: test_bitcast_llc_v32i32_v4i32:
 ; GFX11:       ; %bb.0: ; %entry
 ; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x8
-; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
-; GFX11-NEXT:    v_mov_b32_e32 v4, 0
+; GFX11-NEXT:    s_load_b32 s33, s[4:5], 0x8
+; GFX11-NEXT:    s_load_b64 s[34:35], s[4:5], 0x0
+; GFX11-NEXT:    ; freeze: $sgpr30_sgpr31
+; GFX11-NEXT:    ; freeze: $sgpr28_sgpr29
+; GFX11-NEXT:    ; freeze: $sgpr26_sgpr27
+; GFX11-NEXT:    ; freeze: $sgpr24_sgpr25
+; GFX11-NEXT:    ; freeze: $sgpr22_sgpr23
+; GFX11-NEXT:    ; freeze: $sgpr20_sgpr21
+; GFX11-NEXT:    ; freeze: $sgpr18_sgpr19
+; GFX11-NEXT:    ; freeze: $sgpr16_sgpr17
+; GFX11-NEXT:    ; freeze: $sgpr14_sgpr15
+; GFX11-NEXT:    ; freeze: $sgpr12_sgpr13
+; GFX11-NEXT:    ; freeze: $sgpr10_sgpr11
+; GFX11-NEXT:    ; freeze: $sgpr8_sgpr9
+; GFX11-NEXT:    ; freeze: $sgpr6_sgpr7
+; GFX11-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX11-NEXT:    ; freeze: $sgpr2_sgpr3
+; GFX11-NEXT:    ; freeze: $sgpr0_sgpr1
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX11-NEXT:    v_dual_mov_b32 v30, s30 :: v_dual_mov_b32 v31, s31
+; GFX11-NEXT:    v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
+; GFX11-NEXT:    v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s7
+; GFX11-NEXT:    v_dual_mov_b32 v8, s8 :: v_dual_mov_b32 v9, s9
+; GFX11-NEXT:    v_dual_mov_b32 v10, s10 :: v_dual_mov_b32 v11, s11
+; GFX11-NEXT:    v_dual_mov_b32 v12, s12 :: v_dual_mov_b32 v13, s13
+; GFX11-NEXT:    v_dual_mov_b32 v14, s14 :: v_dual_mov_b32 v15, s15
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_lshl_b32 m0, s2, 2
-; GFX11-NEXT:    v_movrels_b32_e32 v3, v3
-; GFX11-NEXT:    v_movrels_b32_e32 v2, v2
-; GFX11-NEXT:    v_movrels_b32_e32 v1, v1
-; GFX11-NEXT:    v_movrels_b32_e32 v0, v0
-; GFX11-NEXT:    global_store_b128 v4, v[0:3], s[0:1]
+; GFX11-NEXT:    s_lshl_b32 m0, s33, 2
+; GFX11-NEXT:    v_dual_mov_b32 v16, s16 :: v_dual_mov_b32 v17, s17
+; GFX11-NEXT:    v_dual_mov_b32 v18, s18 :: v_dual_mov_b32 v19, s19
+; GFX11-NEXT:    v_dual_mov_b32 v20, s20 :: v_dual_mov_b32 v21, s21
+; GFX11-NEXT:    v_dual_mov_b32 v22, s22 :: v_dual_mov_b32 v23, s23
+; GFX11-NEXT:    v_dual_mov_b32 v24, s24 :: v_dual_mov_b32 v25, s25
+; GFX11-NEXT:    v_dual_mov_b32 v26, s26 :: v_dual_mov_b32 v27, s27
+; GFX11-NEXT:    v_dual_mov_b32 v28, s28 :: v_dual_mov_b32 v29, s29
+; GFX11-NEXT:    v_mov_b32_e32 v35, 0
+; GFX11-NEXT:    v_movrels_b32_e32 v34, v3
+; GFX11-NEXT:    v_movrels_b32_e32 v33, v2
+; GFX11-NEXT:    v_movrels_b32_e32 v32, v1
+; GFX11-NEXT:    v_movrels_b32_e32 v31, v0
+; GFX11-NEXT:    global_store_b128 v35, v[31:34], s[34:35]
 ; GFX11-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: test_bitcast_llc_v32i32_v4i32:
 ; GFX12:       ; %bb.0: ; %entry
-; GFX12-NEXT:    s_load_b96 s[0:2], s[4:5], 0x0
-; GFX12-NEXT:    v_mov_b32_e32 v4, 0
+; GFX12-NEXT:    s_load_b96 s[36:38], s[4:5], 0x0
+; GFX12-NEXT:    ; freeze: $sgpr30_sgpr31
+; GFX12-NEXT:    ; freeze: $sgpr28_sgpr29
+; GFX12-NEXT:    ; freeze: $sgpr26_sgpr27
+; GFX12-NEXT:    ; freeze: $sgpr24_sgpr25
+; GFX12-NEXT:    ; freeze: $sgpr22_sgpr23
+; GFX12-NEXT:    ; freeze: $sgpr20_sgpr21
+; GFX12-NEXT:    ; freeze: $sgpr18_sgpr19
+; GFX12-NEXT:    ; freeze: $sgpr16_sgpr17
+; GFX12-NEXT:    ; freeze: $sgpr14_sgpr15
+; GFX12-NEXT:    ; freeze: $sgpr12_sgpr13
+; GFX12-NEXT:    ; freeze: $sgpr10_sgpr11
+; GFX12-NEXT:    ; freeze: $sgpr8_sgpr9
+; GFX12-NEXT:    ; freeze: $sgpr6_sgpr7
+; GFX12-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX12-NEXT:    ; freeze: $sgpr2_sgpr3
+; GFX12-NEXT:    ; freeze: $sgpr0_sgpr1
+; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-NEXT:    v_dual_mov_b32 v30, s30 :: v_dual_mov_b32 v31, s31
+; GFX12-NEXT:    v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
+; GFX12-NEXT:    v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s7
+; GFX12-NEXT:    v_dual_mov_b32 v8, s8 :: v_dual_mov_b32 v9, s9
+; GFX12-NEXT:    v_dual_mov_b32 v10, s10 :: v_dual_mov_b32 v11, s11
+; GFX12-NEXT:    v_dual_mov_b32 v12, s12 :: v_dual_mov_b32 v13, s13
+; GFX12-NEXT:    v_dual_mov_b32 v14, s14 :: v_dual_mov_b32 v15, s15
+; GFX12-NEXT:    v_dual_mov_b32 v16, s16 :: v_dual_mov_b32 v17, s17
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_lshl_b32 m0, s2, 2
-; GFX12-NEXT:    v_movrels_b32_e32 v3, v3
-; GFX12-NEXT:    v_movrels_b32_e32 v2, v2
-; GFX12-NEXT:    v_movrels_b32_e32 v1, v1
-; GFX12-NEXT:    v_movrels_b32_e32 v0, v0
-; GFX12-NEXT:    global_store_b128 v4, v[0:3], s[0:1]
+; GFX12-NEXT:    s_lshl_b32 m0, s38, 2
+; GFX12-NEXT:    v_dual_mov_b32 v18, s18 :: v_dual_mov_b32 v19, s19
+; GFX12-NEXT:    v_dual_mov_b32 v20, s20 :: v_dual_mov_b32 v21, s21
+; GFX12-NEXT:    v_dual_mov_b32 v22, s22 :: v_dual_mov_b32 v23, s23
+; GFX12-NEXT:    v_dual_mov_b32 v24, s24 :: v_dual_mov_b32 v25, s25
+; GFX12-NEXT:    v_dual_mov_b32 v26, s26 :: v_dual_mov_b32 v27, s27
+; GFX12-NEXT:    v_dual_mov_b32 v28, s28 :: v_dual_mov_b32 v29, s29
+; GFX12-NEXT:    v_mov_b32_e32 v35, 0
+; GFX12-NEXT:    v_movrels_b32_e32 v34, v3
+; GFX12-NEXT:    v_movrels_b32_e32 v33, v2
+; GFX12-NEXT:    v_movrels_b32_e32 v32, v1
+; GFX12-NEXT:    v_movrels_b32_e32 v31, v0
+; GFX12-NEXT:    global_store_b128 v35, v[31:34], s[36:37]
 ; GFX12-NEXT:    s_endpgm
 entry:
   %alloca = freeze <32 x i32> poison
@@ -165,63 +459,161 @@ entry:
 define amdgpu_kernel void @test_bitcast_llc_v16i64_v4i256(ptr addrspace(1) %out, i32 %idx) {
 ; GFX9-LABEL: test_bitcast_llc_v16i64_v4i256:
 ; GFX9:       ; %bb.0: ; %entry
-; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX9-NEXT:    v_mov_b32_e32 v8, 0
+; GFX9-NEXT:    s_load_dword s33, s[4:5], 0x8
+; GFX9-NEXT:    s_load_dwordx2 s[34:35], s[4:5], 0x0
+; GFX9-NEXT:    ; freeze: $sgpr30_sgpr31
+; GFX9-NEXT:    ; freeze: $sgpr28_sgpr29
+; GFX9-NEXT:    ; freeze: $sgpr26_sgpr27
+; GFX9-NEXT:    ; freeze: $sgpr24_sgpr25
+; GFX9-NEXT:    ; freeze: $sgpr22_sgpr23
+; GFX9-NEXT:    ; freeze: $sgpr20_sgpr21
+; GFX9-NEXT:    ; freeze: $sgpr18_sgpr19
+; GFX9-NEXT:    ; freeze: $sgpr16_sgpr17
+; GFX9-NEXT:    ; freeze: $sgpr14_sgpr15
+; GFX9-NEXT:    ; freeze: $sgpr12_sgpr13
+; GFX9-NEXT:    ; freeze: $sgpr10_sgpr11
+; GFX9-NEXT:    ; freeze: $sgpr8_sgpr9
+; GFX9-NEXT:    ; freeze: $sgpr6_sgpr7
+; GFX9-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX9-NEXT:    ; freeze: $sgpr2_sgpr3
+; GFX9-NEXT:    ; freeze: $sgpr0_sgpr1
+; GFX9-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
+; GFX9-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
+; GFX9-NEXT:    v_mov_b64_e32 v[4:5], s[4:5]
+; GFX9-NEXT:    v_mov_b64_e32 v[6:7], s[6:7]
+; GFX9-NEXT:    v_mov_b64_e32 v[8:9], s[8:9]
+; GFX9-NEXT:    v_mov_b64_e32 v[10:11], s[10:11]
+; GFX9-NEXT:    v_mov_b64_e32 v[12:13], s[12:13]
+; GFX9-NEXT:    v_mov_b64_e32 v[14:15], s[14:15]
+; GFX9-NEXT:    v_mov_b64_e32 v[16:17], s[16:17]
+; GFX9-NEXT:    v_mov_b64_e32 v[18:19], s[18:19]
+; GFX9-NEXT:    v_mov_b64_e32 v[20:21], s[20:21]
+; GFX9-NEXT:    v_mov_b64_e32 v[22:23], s[22:23]
+; GFX9-NEXT:    v_mov_b64_e32 v[24:25], s[24:25]
+; GFX9-NEXT:    v_mov_b64_e32 v[26:27], s[26:27]
+; GFX9-NEXT:    v_mov_b64_e32 v[28:29], s[28:29]
+; GFX9-NEXT:    v_mov_b64_e32 v[30:31], s[30:31]
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_lshl_b32 s2, s2, 3
-; GFX9-NEXT:    s_set_gpr_idx_on s2, gpr_idx(SRC0)
-; GFX9-NEXT:    v_mov_b32_e32 v3, v3
-; GFX9-NEXT:    v_mov_b32_e32 v2, v2
-; GFX9-NEXT:    v_mov_b32_e32 v1, v1
-; GFX9-NEXT:    v_mov_b32_e32 v0, v0
-; GFX9-NEXT:    v_mov_b32_e32 v7, v7
-; GFX9-NEXT:    v_mov_b32_e32 v6, v6
-; GFX9-NEXT:    v_mov_b32_e32 v5, v5
-; GFX9-NEXT:    v_mov_b32_e32 v4, v4
+; GFX9-NEXT:    s_lshl_b32 s0, s33, 3
+; GFX9-NEXT:    s_set_gpr_idx_on s0, gpr_idx(SRC0)
+; GFX9-NEXT:    v_mov_b32_e32 v35, v3
+; GFX9-NEXT:    v_mov_b32_e32 v34, v2
+; GFX9-NEXT:    v_mov_b32_e32 v33, v1
+; GFX9-NEXT:    v_mov_b32_e32 v32, v0
+; GFX9-NEXT:    v_mov_b32_e32 v39, v7
+; GFX9-NEXT:    v_mov_b32_e32 v38, v6
+; GFX9-NEXT:    s_set_gpr_idx_off
+; GFX9-NEXT:    v_mov_b32_e32 v40, 0
+; GFX9-NEXT:    s_set_gpr_idx_on s0, gpr_idx(SRC0)
+; GFX9-NEXT:    v_mov_b32_e32 v37, v5
+; GFX9-NEXT:    v_mov_b32_e32 v36, v4
 ; GFX9-NEXT:    s_set_gpr_idx_off
-; GFX9-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
-; GFX9-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1]
+; GFX9-NEXT:    global_store_dwordx4 v40, v[36:39], s[34:35] offset:16
+; GFX9-NEXT:    global_store_dwordx4 v40, v[32:35], s[34:35]
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: test_bitcast_llc_v16i64_v4i256:
 ; GFX11:       ; %bb.0: ; %entry
 ; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x8
-; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
-; GFX11-NEXT:    v_mov_b32_e32 v8, 0
+; GFX11-NEXT:    s_load_b32 s33, s[4:5], 0x8
+; GFX11-NEXT:    s_load_b64 s[34:35], s[4:5], 0x0
+; GFX11-NEXT:    ; freeze: $sgpr30_sgpr31
+; GFX11-NEXT:    ; freeze: $sgpr28_sgpr29
+; GFX11-NEXT:    ; freeze: $sgpr26_sgpr27
+; GFX11-NEXT:    ; freeze: $sgpr24_sgpr25
+; GFX11-NEXT:    ; freeze: $sgpr22_sgpr23
+; GFX11-NEXT:    ; freeze: $sgpr20_sgpr21
+; GFX11-NEXT:    ; freeze: $sgpr18_sgpr19
+; GFX11-NEXT:    ; freeze: $sgpr16_sgpr17
+; GFX11-NEXT:    ; freeze: $sgpr14_sgpr15
+; GFX11-NEXT:    ; freeze: $sgpr12_sgpr13
+; GFX11-NEXT:    ; freeze: $sgpr10_sgpr11
+; GFX11-NEXT:    ; freeze: $sgpr8_sgpr9
+; GFX11-NEXT:    ; freeze: $sgpr6_sgpr7
+; GFX11-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX11-NEXT:    ; freeze: $sgpr2_sgpr3
+; GFX11-NEXT:    ; freeze: $sgpr0_sgpr1
+; GFX11-NEXT:    v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
+; GFX11-NEXT:    v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s7
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX11-NEXT:    v_dual_mov_b32 v8, s8 :: v_dual_mov_b32 v9, s9
+; GFX11-NEXT:    v_dual_mov_b32 v10, s10 :: v_dual_mov_b32 v11, s11
+; GFX11-NEXT:    v_dual_mov_b32 v12, s12 :: v_dual_mov_b32 v13, s13
+; GFX11-NEXT:    v_dual_mov_b32 v14, s14 :: v_dual_mov_b32 v15, s15
+; GFX11-NEXT:    v_dual_mov_b32 v16, s16 :: v_dual_mov_b32 v17, s17
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_lshl_b32 m0, s2, 3
-; GFX11-NEXT:    v_movrels_b32_e32 v3, v3
-; GFX11-NEXT:    v_movrels_b32_e32 v2, v2
-; GFX11-NEXT:    v_movrels_b32_e32 v1, v1
-; GFX11-NEXT:    v_movrels_b32_e32 v7, v7
-; GFX11-NEXT:    v_movrels_b32_e32 v6, v6
-; GFX11-NEXT:    v_movrels_b32_e32 v5, v5
-; GFX11-NEXT:    v_movrels_b32_e32 v4, v4
-; GFX11-NEXT:    v_movrels_b32_e32 v0, v0
+; GFX11-NEXT:    s_lshl_b32 m0, s33, 3
+; GFX11-NEXT:    v_dual_mov_b32 v18, s18 :: v_dual_mov_b32 v19, s19
+; GFX11-NEXT:    v_dual_mov_b32 v20, s20 :: v_dual_mov_b32 v21, s21
+; GFX11-NEXT:    v_dual_mov_b32 v22, s22 :: v_dual_mov_b32 v23, s23
+; GFX11-NEXT:    v_dual_mov_b32 v24, s24 :: v_dual_mov_b32 v25, s25
+; GFX11-NEXT:    v_dual_mov_b32 v26, s26 :: v_dual_mov_b32 v27, s27
+; GFX11-NEXT:    v_dual_mov_b32 v28, s28 :: v_dual_mov_b32 v29, s29
+; GFX11-NEXT:    v_dual_mov_b32 v30, s30 :: v_dual_mov_b32 v31, s31
+; GFX11-NEXT:    v_mov_b32_e32 v39, 0
+; GFX11-NEXT:    v_movrels_b32_e32 v38, v7
+; GFX11-NEXT:    v_movrels_b32_e32 v37, v6
+; GFX11-NEXT:    v_movrels_b32_e32 v36, v5
+; GFX11-NEXT:    v_movrels_b32_e32 v35, v4
+; GFX11-NEXT:    v_movrels_b32_e32 v34, v3
+; GFX11-NEXT:    v_movrels_b32_e32 v33, v2
+; GFX11-NEXT:    v_movrels_b32_e32 v32, v1
+; GFX11-NEXT:    v_movrels_b32_e32 v31, v0
 ; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    global_store_b128 v8, v[4:7], s[0:1] offset:16
-; GFX11-NEXT:    global_store_b128 v8, v[0:3], s[0:1]
+; GFX11-NEXT:    global_store_b128 v39, v[35:38], s[34:35] offset:16
+; GFX11-NEXT:    global_store_b128 v39, v[31:34], s[34:35]
 ; GFX11-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: test_bitcast_llc_v16i64_v4i256:
 ; GFX12:       ; %bb.0: ; %entry
-; GFX12-NEXT:    s_load_b96 s[0:2], s[4:5], 0x0
-; GFX12-NEXT:    v_mov_b32_e32 v8, 0
+; GFX12-NEXT:    s_load_b96 s[36:38], s[4:5], 0x0
+; GFX12-NEXT:    ; freeze: $sgpr30_sgpr31
+; GFX12-NEXT:    ; freeze: $sgpr28_sgpr29
+; GFX12-NEXT:    ; freeze: $sgpr26_sgpr27
+; GFX12-NEXT:    ; freeze: $sgpr24_sgpr25
+; GFX12-NEXT:    ; freeze: $sgpr22_sgpr23
+; GFX12-NEXT:    ; freeze: $sgpr20_sgpr21
+; GFX12-NEXT:    ; freeze: $sgpr18_sgpr19
+; GFX12-NEXT:    ; freeze: $sgpr16_sgpr17
+; GFX12-NEXT:    ; freeze: $sgpr14_sgpr15
+; GFX12-NEXT:    ; freeze: $sgpr12_sgpr13
+; GFX12-NEXT:    ; freeze: $sgpr10_sgpr11
+; GFX12-NEXT:    ; freeze: $sgpr8_sgpr9
+; GFX12-NEXT:    ; freeze: $sgpr6_sgpr7
+; GFX12-NEXT:    ; freeze: $sgpr4_sgpr5
+; GFX12-NEXT:    ; freeze: $sgpr2_sgpr3
+; GFX12-NEXT:    ; freeze: $sgpr0_sgpr1
+; GFX12-NEXT:    v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
+; GFX12-NEXT:    v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s7
+; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-NEXT:    v_dual_mov_b32 v8, s8 :: v_dual_mov_b32 v9, s9
+; GFX12-NEXT:    v_dual_mov_b32 v10, s10 :: v_dual_mov_b32 v11, s11
+; GFX12-NEXT:    v_dual_mov_b32 v12, s12 :: v_dual_mov_b32 v13, s13
+; GFX12-NEXT:    v_dual_mov_b32 v14, s14 :: v_dual_mov_b32 v15, s15
+; GFX12-NEXT:    v_dual_mov_b32 v16, s16 :: v_dual_mov_b32 v17, s17
+; GFX12-NEXT:    v_dual_mov_b32 v18, s18 :: v_dual_mov_b32 v19, s19
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_lshl_b32 m0, s2, 3
-; GFX12-NEXT:    v_movrels_b32_e32 v3, v3
-; GFX12-NEXT:    v_movrels_b32_e32 v2, v2
-; GFX12-NEXT:    v_movrels_b32_e32 v1, v1
-; GFX12-NEXT:    v_movrels_b32_e32 v7, v7
-; GFX12-NEXT:    v_movrels_b32_e32 v6, v6
-; GFX12-NEXT:    v_movrels_b32_e32 v5, v5
-; GFX12-NEXT:    v_movrels_b32_e32 v4, v4
-; GFX12-NEXT:    v_movrels_b32_e32 v0, v0
+; GFX12-NEXT:    s_lshl_b32 m0, s38, 3
+; GFX12-NEXT:    v_dual_mov_b32 v20, s20 :: v_dual_mov_b32 v21, s21
+; GFX12-NEXT:    v_dual_mov_b32 v22, s22 :: v_dual_mov_b32 v23, s23
+; GFX12-NEXT:    v_dual_mov_b32 v24, s24 :: v_dual_mov_b32 v25, s25
+; GFX12-NEXT:    v_dual_mov_b32 v26, s26 :: v_dual_mov_b32 v27, s27
+; GFX12-NEXT:    v_dual_mov_b32 v28, s28 :: v_dual_mov_b32 v29, s29
+; GFX12-NEXT:    v_dual_mov_b32 v30, s30 :: v_dual_mov_b32 v31, s31
+; GFX12-NEXT:    v_mov_b32_e32 v39, 0
+; GFX12-NEXT:    v_movrels_b32_e32 v38, v7
+; GFX12-NEXT:    v_movrels_b32_e32 v37, v6
+; GFX12-NEXT:    v_movrels_b32_e32 v36, v5
+; GFX12-NEXT:    v_movrels_b32_e32 v35, v4
+; GFX12-NEXT:    v_movrels_b32_e32 v34, v3
+; GFX12-NEXT:    v_movrels_b32_e32 v33, v2
+; GFX12-NEXT:    v_movrels_b32_e32 v32, v1
+; GFX12-NEXT:    v_movrels_b32_e32 v31, v0
 ; GFX12-NEXT:    s_clause 0x1
-; GFX12-NEXT:    global_store_b128 v8, v[4:7], s[0:1] offset:16
-; GFX12-NEXT:    global_store_b128 v8, v[0:3], s[0:1]
+; GFX12-NEXT:    global_store_b128 v39, v[35:38], s[36:37] offset:16
+; GFX12-NEXT:    global_store_b128 v39, v[31:34], s[36:37]
 ; GFX12-NEXT:    s_endpgm
 entry:
   %alloca = freeze <16 x i64> poison
diff --git a/llvm/test/CodeGen/AMDGPU/promote-alloca-vector-to-vector.ll b/llvm/test/CodeGen/AMDGPU/promote-alloca-vector-to-vector.ll
index cb44d2ba4bde6..aa5af3bee6015 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-alloca-vector-to-vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-alloca-vector-to-vector.ll
@@ -173,7 +173,10 @@ entry:
 ; OPT-LABEL: define i64 @ptr_alloca_bitcast
 
 ; GCN-NOT: buffer_
-; GCN: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN: ; freeze: $sgpr4_sgpr5
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mov_b32_e32 v0, s4
+; GCN-NEXT: v_mov_b32_e32 v1, s5
 ; GCN-NEXT: s_setpc_b64
 
 
diff --git a/llvm/test/CodeGen/AMDGPU/sad.ll b/llvm/test/CodeGen/AMDGPU/sad.ll
index 5e31f9e463355..5c84ae49446eb 100644
--- a/llvm/test/CodeGen/AMDGPU/sad.ll
+++ b/llvm/test/CodeGen/AMDGPU/sad.ll
@@ -1293,13 +1293,14 @@ define amdgpu_kernel void @v_sad_u32_i16_pat2(ptr addrspace(1) %out) {
 ; GFX12-5-SDAG-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-5-SDAG-NEXT:    global_load_u16 v3, v[0:1], off scope:SCOPE_SYS
 ; GFX12-5-SDAG-NEXT:    s_wait_loadcnt 0x0
-; GFX12-5-SDAG-NEXT:    v_and_b32_e32 v4, 0xffff, v0
-; GFX12-5-SDAG-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GFX12-5-SDAG-NEXT:    v_mov_b16_e32 v4.l, v0.l
 ; GFX12-5-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-5-SDAG-NEXT:    v_and_b32_e32 v4, 0xffff, v4
+; GFX12-5-SDAG-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; GFX12-5-SDAG-NEXT:    v_min_u32_e32 v5, v4, v2
 ; GFX12-5-SDAG-NEXT:    v_max_u32_e32 v2, v4, v2
+; GFX12-5-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-5-SDAG-NEXT:    v_dual_mov_b32 v4, 0 :: v_dual_sub_nc_u32 v2, v2, v5
-; GFX12-5-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-5-SDAG-NEXT:    v_add_nc_u32_e32 v2, v2, v3
 ; GFX12-5-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-5-SDAG-NEXT:    global_store_b16 v4, v2, s[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll b/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll
index c212792a25f9d..08e552ed291ce 100644
--- a/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll
+++ b/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll
@@ -1675,11 +1675,12 @@ bb11:                                             ; preds = %bb10, %bb2
 define void @crash_lshlrevb16_not_reg_op() {
 ; NOSDWA-LABEL: crash_lshlrevb16_not_reg_op:
 ; NOSDWA:       ; %bb.0: ; %bb0
+; NOSDWA-NEXT:    ; freeze: $sgpr4
 ; NOSDWA-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; NOSDWA-NEXT:    s_and_b64 vcc, exec, -1
 ; NOSDWA-NEXT:    s_bitset1_b32 s4, 8
 ; NOSDWA-NEXT:    s_and_b32 s6, s4, 0x1ff
 ; NOSDWA-NEXT:    s_mov_b64 s[4:5], 0
-; NOSDWA-NEXT:    s_and_b64 vcc, exec, -1
 ; NOSDWA-NEXT:  .LBB22_1: ; %bb1
 ; NOSDWA-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; NOSDWA-NEXT:    s_lshl_b32 s7, s4, 3
@@ -1697,11 +1698,12 @@ define void @crash_lshlrevb16_not_reg_op() {
 ;
 ; GFX89-LABEL: crash_lshlrevb16_not_reg_op:
 ; GFX89:       ; %bb.0: ; %bb0
+; GFX89-NEXT:    ; freeze: $sgpr4
 ; GFX89-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX89-NEXT:    s_and_b64 vcc, exec, -1
 ; GFX89-NEXT:    s_bitset1_b32 s4, 8
 ; GFX89-NEXT:    s_and_b32 s6, s4, 0x1ff
 ; GFX89-NEXT:    s_mov_b64 s[4:5], 0
-; GFX89-NEXT:    s_and_b64 vcc, exec, -1
 ; GFX89-NEXT:  .LBB22_1: ; %bb1
 ; GFX89-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX89-NEXT:    s_lshl_b32 s7, s4, 3
@@ -1719,11 +1721,12 @@ define void @crash_lshlrevb16_not_reg_op() {
 ;
 ; GFX9-LABEL: crash_lshlrevb16_not_reg_op:
 ; GFX9:       ; %bb.0: ; %bb0
+; GFX9-NEXT:    ; freeze: $sgpr4
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_and_b64 vcc, exec, -1
 ; GFX9-NEXT:    s_bitset1_b32 s4, 8
 ; GFX9-NEXT:    s_and_b32 s6, s4, 0x1ff
 ; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:    s_and_b64 vcc, exec, -1
 ; GFX9-NEXT:  .LBB22_1: ; %bb1
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_lshl_b32 s7, s4, 3
@@ -1741,9 +1744,10 @@ define void @crash_lshlrevb16_not_reg_op() {
 ;
 ; GFX10-LABEL: crash_lshlrevb16_not_reg_op:
 ; GFX10:       ; %bb.0: ; %bb0
+; GFX10-NEXT:    ; freeze: $sgpr4
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    s_bitset1_b32 s4, 8
 ; GFX10-NEXT:    s_mov_b32 vcc_lo, exec_lo
+; GFX10-NEXT:    s_bitset1_b32 s4, 8
 ; GFX10-NEXT:    s_and_b32 s6, s4, 0x1ff
 ; GFX10-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX10-NEXT:  .LBB22_1: ; %bb1
diff --git a/llvm/test/CodeGen/AMDGPU/select-undef.ll b/llvm/test/CodeGen/AMDGPU/select-undef.ll
index cbb621088222f..295a1836962c6 100644
--- a/llvm/test/CodeGen/AMDGPU/select-undef.ll
+++ b/llvm/test/CodeGen/AMDGPU/select-undef.ll
@@ -810,6 +810,7 @@ define i64 @poison_should_freeze(i1 %cond1, i32 %val, i16 %val2, i64 %a, i64 %b)
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GCN-NEXT:    v_mov_b32_e32 v7, 0x5040100
+; GCN-NEXT:    ; freeze: $sgpr4
 ; GCN-NEXT:    v_perm_b32 v2, v2, s4, v7
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc
@@ -831,6 +832,7 @@ define <8 x i16> @poison_should_freeze_v8i16(i1 %cond1, i32 %val, i16 %val2, <8
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GCN-NEXT:    v_mov_b32_e32 v11, 0x5040100
+; GCN-NEXT:    ; freeze: $sgpr4
 ; GCN-NEXT:    v_perm_b32 v2, v2, s4, v11
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc
diff --git a/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask.ll b/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask.ll
index d4a4f6ed9eefa..66e38163f3bbb 100644
--- a/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask.ll
+++ b/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask.ll
@@ -52,6 +52,7 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
   ; CHECK-NEXT:   undef [[S_MOV_B32_:%[0-9]+]].sub0:sgpr_128 = S_MOV_B32 0
   ; CHECK-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
   ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET undef %117:sgpr_128, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; CHECK-NEXT:   [[FREEZE:%[0-9]+]]:sreg_32 = FREEZE undef %170:sreg_32
   ; CHECK-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub1:sgpr_128 = COPY [[S_MOV_B32_]].sub0
   ; CHECK-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub2:sgpr_128 = COPY [[S_MOV_B32_]].sub0
   ; CHECK-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub3:sgpr_128 = COPY [[S_MOV_B32_]].sub0
@@ -67,8 +68,8 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
   ; CHECK-NEXT:   [[S_LOAD_DWORDX4_IMM1:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_1]], 64, 0 :: (invariant load (s128) from %ir.89, addrspace 4)
   ; CHECK-NEXT:   [[S_LOAD_DWORDX4_IMM2:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_2]], 64, 0 :: (invariant load (s128) from %ir.95, addrspace 4)
   ; CHECK-NEXT:   [[S_ADD_U32_3:%[0-9]+]].sub1:sreg_64 = S_ADDC_U32 undef %54:sreg_32, [[S_ASHR_I32_2]], implicit-def dead $scc, implicit $scc
-  ; CHECK-NEXT:   [[S_ASHR_I32_3:%[0-9]+]]:sreg_32_xm0 = S_ASHR_I32 undef %170:sreg_32, 31, implicit-def dead $scc
-  ; CHECK-NEXT:   undef [[S_ADD_U32_4:%[0-9]+]].sub0:sreg_64 = S_ADD_U32 [[COPY6]], undef %170:sreg_32, implicit-def $scc
+  ; CHECK-NEXT:   [[S_ASHR_I32_3:%[0-9]+]]:sreg_32_xm0 = S_ASHR_I32 [[FREEZE]], 31, implicit-def dead $scc
+  ; CHECK-NEXT:   undef [[S_ADD_U32_4:%[0-9]+]].sub0:sreg_64 = S_ADD_U32 [[COPY6]], [[FREEZE]], implicit-def $scc
   ; CHECK-NEXT:   [[S_ADD_U32_4:%[0-9]+]].sub1:sreg_64 = S_ADDC_U32 undef %54:sreg_32, [[S_ASHR_I32_3]], implicit-def dead $scc, implicit $scc
   ; CHECK-NEXT:   undef [[S_ADD_U32_5:%[0-9]+]].sub0:sreg_64 = S_ADD_U32 [[COPY7]].sub0, [[S_LSHL_B32_]], implicit-def $scc
   ; CHECK-NEXT:   [[S_LOAD_DWORDX4_IMM3:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_3]], 64, 0 :: (invariant load (s128) from %ir.101, addrspace 4)
@@ -77,7 +78,7 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
   ; CHECK-NEXT:   undef [[S_ADD_U32_6:%[0-9]+]].sub0:sreg_64 = S_ADD_U32 [[COPY7]].sub0, [[S_LSHL_B32_1]], implicit-def $scc
   ; CHECK-NEXT:   [[S_LOAD_DWORDX4_IMM5:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_5]], 0, 0 :: (invariant load (s128) from %ir.114, addrspace 4)
   ; CHECK-NEXT:   [[S_ADD_U32_6:%[0-9]+]].sub1:sreg_64 = S_ADDC_U32 undef %51:sreg_32, [[S_ASHR_I32_1]], implicit-def dead $scc, implicit $scc
-  ; CHECK-NEXT:   undef [[S_ADD_U32_7:%[0-9]+]].sub0:sreg_64 = S_ADD_U32 [[COPY7]].sub0, undef %170:sreg_32, implicit-def $scc
+  ; CHECK-NEXT:   undef [[S_ADD_U32_7:%[0-9]+]].sub0:sreg_64 = S_ADD_U32 [[COPY7]].sub0, [[FREEZE]], implicit-def $scc
   ; CHECK-NEXT:   [[S_ADD_U32_7:%[0-9]+]].sub1:sreg_64 = S_ADDC_U32 undef %51:sreg_32, [[S_ASHR_I32_3]], implicit-def dead $scc, implicit $scc
   ; CHECK-NEXT:   undef [[S_ADD_U32_8:%[0-9]+]].sub0:sreg_64 = S_ADD_U32 [[COPY7]].sub0, [[S_LSHL_B32_2]], implicit-def $scc
   ; CHECK-NEXT:   [[S_ADD_U32_8:%[0-9]+]].sub1:sreg_64 = S_ADDC_U32 undef %51:sreg_32, [[S_ASHR_I32_2]], implicit-def dead $scc, implicit $scc
diff --git a/llvm/test/CodeGen/AMDGPU/udiv64.ll b/llvm/test/CodeGen/AMDGPU/udiv64.ll
index 4c9f2d08ce8f1..89b32b6ce56f2 100644
--- a/llvm/test/CodeGen/AMDGPU/udiv64.ll
+++ b/llvm/test/CodeGen/AMDGPU/udiv64.ll
@@ -1801,7 +1801,8 @@ define i64 @v_test_udiv24_pow2_k_den_i64(i64 %x) {
 ; GCN-IR-LABEL: v_test_udiv24_pow2_k_den_i64:
 ; GCN-IR:       ; %bb.0: ; %_udiv-special-cases
 ; GCN-IR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IR-NEXT:    v_lshrrev_b32_e32 v4, 8, v1
+; GCN-IR-NEXT:    v_lshrrev_b32_e32 v0, 8, v1
+; GCN-IR-NEXT:    v_mov_b32_e32 v4, v0
 ; GCN-IR-NEXT:    v_ffbh_u32_e32 v0, v4
 ; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, 32, v0
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s4, 0
diff --git a/llvm/test/CodeGen/Hexagon/pr196966.ll b/llvm/test/CodeGen/Hexagon/pr196966.ll
index 8612395feb8e4..ff473b2c224fe 100644
--- a/llvm/test/CodeGen/Hexagon/pr196966.ll
+++ b/llvm/test/CodeGen/Hexagon/pr196966.ll
@@ -34,33 +34,36 @@ define void @pr196966(ptr %base) {
 ; CHECK-NEXT:     r0 = add(r29,#0)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     r6 = add(r16,#6)
+; CHECK-NEXT:     r3 = add(r16,#20)
+; CHECK-NEXT:     r8 = add(r16,#6)
+; CHECK-NEXT:     r7:6 = memd(r29+#0)
 ; CHECK-NEXT:     r1:0 = memd(r29+#24)
-; CHECK-NEXT:     r5:4 = memd(r29+#0)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     r5:4 = extractu(r5:4,#16,#32)
-; CHECK-NEXT:     p0 = bitsclr(r0,#4)
-; CHECK-NEXT:     r3:2 = memd(r29+#8)
+; CHECK-NEXT:     r7:6 = extractu(r7:6,#16,#32)
+; CHECK-NEXT:     r5:4 = memd(r29+#8)
+; CHECK-NEXT:     r17:16 = memd(r30+#-8)
 ; CHECK-NEXT:    }
+; CHECK-NEXT:    // freeze: $r5
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     p1 = bitsclr(r0,#32)
-; CHECK-NEXT:     p2 = !tstbit(r0,#13)
-; CHECK-NEXT:     r3 = add(r16,#20)
-; CHECK-NEXT:     if (!p0) memh(r0+#0) = r4
+; CHECK-NEXT:     p0 = bitsclr(r5,#4)
+; CHECK-NEXT:     p1 = bitsclr(r5,#32)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     p0 = !tstbit(r0,#15)
-; CHECK-NEXT:     if (!p1) memh(r3+#0) = r2.h
-; CHECK-NEXT:     if (!p2) memh(r6+#0) = r0.h
+; CHECK-NEXT:     p2 = !tstbit(r5,#13)
+; CHECK-NEXT:     p0 = !tstbit(r5,#15)
+; CHECK-NEXT:     if (!p0) memh(r2+#0) = r6
+; CHECK-NEXT:     if (!p1) memh(r3+#0) = r4.h
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     if (!p2) memh(r8+#0) = r0.h
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     r17:16 = memd(r30+#-8)
 ; CHECK-NEXT:     r31:30 = deallocframe(r30):raw
-; CHECK-NEXT:    } // 8-byte Folded Reload
+; CHECK-NEXT:     if (!p0) memh(r2+#0) = r1.h
+; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     jumpr r31
-; CHECK-NEXT:     if (!p0) memh(r0+#0) = r1.h
 ; CHECK-NEXT:    }
 entry:
   %mb = icmp ne ptr %base, null
diff --git a/llvm/test/CodeGen/PowerPC/sms-grp-order.ll b/llvm/test/CodeGen/PowerPC/sms-grp-order.ll
index 9760e3d83d77e..1533bf1d5ce8f 100644
--- a/llvm/test/CodeGen/PowerPC/sms-grp-order.ll
+++ b/llvm/test/CodeGen/PowerPC/sms-grp-order.ll
@@ -5,19 +5,20 @@
 define void @lame_encode_buffer_interleaved(ptr %arg0) local_unnamed_addr {
 ; CHECK-LABEL: lame_encode_buffer_interleaved:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li 4, 1
-; CHECK-NEXT:    rldic 4, 4, 62, 1
-; CHECK-NEXT:    mtctr 4
+; CHECK-NEXT:    li 5, 1
+; CHECK-NEXT:    # freeze: $x4
+; CHECK-NEXT:    rldic 5, 5, 62, 1
+; CHECK-NEXT:    mtctr 5
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1:
-; CHECK-NEXT:    lha 4, 0(3)
 ; CHECK-NEXT:    lha 5, 0(3)
-; CHECK-NEXT:    srawi 4, 4, 1
-; CHECK-NEXT:    addze 4, 4
+; CHECK-NEXT:    lha 6, 0(4)
 ; CHECK-NEXT:    srawi 5, 5, 1
 ; CHECK-NEXT:    addze 5, 5
-; CHECK-NEXT:    sth 4, 0(3)
+; CHECK-NEXT:    srawi 6, 6, 1
+; CHECK-NEXT:    addze 6, 6
 ; CHECK-NEXT:    sth 5, 0(3)
+; CHECK-NEXT:    sth 6, 0(4)
 ; CHECK-NEXT:    bdnz .LBB0_1
 ; CHECK-NEXT:  # %bb.2:
 ; CHECK-NEXT:    blr
diff --git a/llvm/test/CodeGen/PowerPC/widen-vec-correctly-be.ll b/llvm/test/CodeGen/PowerPC/widen-vec-correctly-be.ll
index c2d62cfc23e96..95c121b8c23be 100644
--- a/llvm/test/CodeGen/PowerPC/widen-vec-correctly-be.ll
+++ b/llvm/test/CodeGen/PowerPC/widen-vec-correctly-be.ll
@@ -11,15 +11,16 @@ define void @test() local_unnamed_addr #0 align 2 {
 ; CHECK-BE-NEXT:    lhz r3, 0(r3)
 ; CHECK-BE-NEXT:    vspltisw v2, -16
 ; CHECK-BE-NEXT:    addi r3, r3, 1
+; CHECK-BE-NEXT:    # freeze: $v4
 ; CHECK-BE-NEXT:    xxlxor vs0, vs0, vs0
+; CHECK-BE-NEXT:    vsrw v2, v2, v2
 ; CHECK-BE-NEXT:    sldi r3, r3, 48
 ; CHECK-BE-NEXT:    std r3, -32(r1)
 ; CHECK-BE-NEXT:    std r3, -24(r1)
 ; CHECK-BE-NEXT:    addi r3, r1, -32
 ; CHECK-BE-NEXT:    lxvw4x v3, 0, r3
 ; CHECK-BE-NEXT:    addi r3, r1, -16
-; CHECK-BE-NEXT:    vmrghh v3, v3, v2
-; CHECK-BE-NEXT:    vsrw v2, v2, v2
+; CHECK-BE-NEXT:    vmrghh v3, v3, v4
 ; CHECK-BE-NEXT:    xxsel vs0, v3, vs0, v2
 ; CHECK-BE-NEXT:    stxvw4x vs0, 0, r3
 ; CHECK-BE-NEXT:    lwz r3, -16(r1)
@@ -33,6 +34,7 @@ define void @test() local_unnamed_addr #0 align 2 {
 ; CHECK-P9-BE:       # %bb.0: # %bb
 ; CHECK-P9-BE-NEXT:    lhz r3, 0(r3)
 ; CHECK-P9-BE-NEXT:    vspltisw v3, -16
+; CHECK-P9-BE-NEXT:    # freeze: $v2
 ; CHECK-P9-BE-NEXT:    xxlxor vs2, vs2, vs2
 ; CHECK-P9-BE-NEXT:    addi r3, r3, 1
 ; CHECK-P9-BE-NEXT:    vsrw v3, v3, v3
diff --git a/llvm/test/CodeGen/RISCV/freeze-multi-use.ll b/llvm/test/CodeGen/RISCV/freeze-multi-use.ll
index 18b7644740660..202d37fcfbb59 100644
--- a/llvm/test/CodeGen/RISCV/freeze-multi-use.ll
+++ b/llvm/test/CodeGen/RISCV/freeze-multi-use.ll
@@ -8,14 +8,16 @@
 define i32 @add_twice(i32 %n) {
 ; RV32-LABEL: add_twice:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    add a0, a0, a0
-; RV32-NEXT:    add a0, a0, a0
+; RV32-NEXT:    # freeze: $x11
+; RV32-NEXT:    add a0, a0, a1
+; RV32-NEXT:    add a0, a0, a1
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: add_twice:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    add a0, a0, a0
-; RV64-NEXT:    addw a0, a0, a0
+; RV64-NEXT:    # freeze: $x11
+; RV64-NEXT:    add a0, a0, a1
+; RV64-NEXT:    addw a0, a0, a1
 ; RV64-NEXT:    ret
   %w = freeze i32 poison
   %a = add i32 %n, %w
@@ -30,8 +32,9 @@ define i32 @cross_block(i1 %c, i32 %x) {
 ; RV32-NEXT:    andi a0, a0, 1
 ; RV32-NEXT:    beqz a0, .LBB1_2
 ; RV32-NEXT:  # %bb.1: # %t
-; RV32-NEXT:    add a0, a0, a1
-; RV32-NEXT:    sub a0, a0, a0
+; RV32-NEXT:    # freeze: $x10
+; RV32-NEXT:    add a1, a0, a1
+; RV32-NEXT:    sub a0, a1, a0
 ; RV32-NEXT:    ret
 ; RV32-NEXT:  .LBB1_2: # %e
 ; RV32-NEXT:    mv a0, a1
@@ -42,8 +45,9 @@ define i32 @cross_block(i1 %c, i32 %x) {
 ; RV64-NEXT:    andi a0, a0, 1
 ; RV64-NEXT:    beqz a0, .LBB1_2
 ; RV64-NEXT:  # %bb.1: # %t
-; RV64-NEXT:    add a0, a0, a1
-; RV64-NEXT:    subw a0, a0, a0
+; RV64-NEXT:    # freeze: $x10
+; RV64-NEXT:    add a1, a0, a1
+; RV64-NEXT:    subw a0, a1, a0
 ; RV64-NEXT:    ret
 ; RV64-NEXT:  .LBB1_2: # %e
 ; RV64-NEXT:    mv a0, a1
@@ -66,16 +70,23 @@ define i32 @float_bits() {
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32-NEXT:    .cfi_offset ra, -4
+; RV32-NEXT:    .cfi_offset s0, -8
+; RV32-NEXT:    # freeze: $x8
+; RV32-NEXT:    mv a0, s0
+; RV32-NEXT:    mv a1, s0
 ; RV32-NEXT:    call __unordsf2
-; RV32-NEXT:    slli a1, a0, 1
-; RV32-NEXT:    srli a1, a1, 1
-; RV32-NEXT:    lui a2, 522240
+; RV32-NEXT:    slli s0, s0, 1
+; RV32-NEXT:    srli s0, s0, 1
+; RV32-NEXT:    lui a1, 522240
 ; RV32-NEXT:    snez a0, a0
-; RV32-NEXT:    sltu a1, a2, a1
+; RV32-NEXT:    sltu a1, a1, s0
 ; RV32-NEXT:    xor a0, a0, a1
 ; RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32-NEXT:    .cfi_restore ra
+; RV32-NEXT:    .cfi_restore s0
 ; RV32-NEXT:    addi sp, sp, 16
 ; RV32-NEXT:    .cfi_def_cfa_offset 0
 ; RV32-NEXT:    ret
@@ -85,16 +96,23 @@ define i32 @float_bits() {
 ; RV64-NEXT:    addi sp, sp, -16
 ; RV64-NEXT:    .cfi_def_cfa_offset 16
 ; RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
 ; RV64-NEXT:    .cfi_offset ra, -8
+; RV64-NEXT:    .cfi_offset s0, -16
+; RV64-NEXT:    # freeze: $x8
+; RV64-NEXT:    mv a0, s0
+; RV64-NEXT:    mv a1, s0
 ; RV64-NEXT:    call __unordsf2
-; RV64-NEXT:    slli a1, a0, 33
-; RV64-NEXT:    srli a1, a1, 33
-; RV64-NEXT:    lui a2, 522240
+; RV64-NEXT:    slli s0, s0, 33
+; RV64-NEXT:    srli s0, s0, 33
+; RV64-NEXT:    lui a1, 522240
 ; RV64-NEXT:    snez a0, a0
-; RV64-NEXT:    sltu a1, a2, a1
+; RV64-NEXT:    sltu a1, a1, s0
 ; RV64-NEXT:    xor a0, a0, a1
 ; RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
 ; RV64-NEXT:    .cfi_restore ra
+; RV64-NEXT:    .cfi_restore s0
 ; RV64-NEXT:    addi sp, sp, 16
 ; RV64-NEXT:    .cfi_def_cfa_offset 0
 ; RV64-NEXT:    ret
@@ -114,8 +132,13 @@ define i32 @across_call() nounwind {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32-NEXT:    # freeze: $x8
+; RV32-NEXT:    mv a0, s0
 ; RV32-NEXT:    call use
+; RV32-NEXT:    mv a0, s0
 ; RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32-NEXT:    addi sp, sp, 16
 ; RV32-NEXT:    ret
 ;
@@ -123,8 +146,13 @@ define i32 @across_call() nounwind {
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    addi sp, sp, -16
 ; RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
+; RV64-NEXT:    # freeze: $x8
+; RV64-NEXT:    mv a0, s0
 ; RV64-NEXT:    call use
+; RV64-NEXT:    mv a0, s0
 ; RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
 ; RV64-NEXT:    addi sp, sp, 16
 ; RV64-NEXT:    ret
   %w = freeze i32 poison
diff --git a/llvm/test/CodeGen/RISCV/rvv/freeze-multi-use.ll b/llvm/test/CodeGen/RISCV/rvv/freeze-multi-use.ll
index 71d79b5f18b4e..ee48dc88fd0b0 100644
--- a/llvm/test/CodeGen/RISCV/rvv/freeze-multi-use.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/freeze-multi-use.ll
@@ -7,13 +7,14 @@
 define i32 @partial_vector(i32 %x) {
 ; CHECK-LABEL: partial_vector:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    # freeze: $x10
 ; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; CHECK-NEXT:    vmv.v.x v8, a0
 ; CHECK-NEXT:    vadd.vx v8, v8, a0
 ; CHECK-NEXT:    vslidedown.vi v8, v8, 1
-; CHECK-NEXT:    vmv.x.s a0, v8
-; CHECK-NEXT:    add a1, a0, a0
-; CHECK-NEXT:    subw a0, a0, a1
+; CHECK-NEXT:    vmv.x.s a1, v8
+; CHECK-NEXT:    add a0, a0, a0
+; CHECK-NEXT:    subw a0, a1, a0
 ; CHECK-NEXT:    ret
   %v = insertelement <4 x i32> poison, i32 %x, i32 0
   %w = freeze <4 x i32> %v
@@ -29,11 +30,20 @@ define i32 @partial_vector(i32 %x) {
 define <4 x i32> @vec_across_call() nounwind {
 ; CHECK-LABEL: vec_across_call:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi sp, sp, -16
-; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    addi sp, sp, -48
+; CHECK-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    sub sp, sp, a0
+; CHECK-NEXT:    # freeze: $v8
+; CHECK-NEXT:    addi a0, sp, 32
+; CHECK-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
 ; CHECK-NEXT:    call usev
-; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    addi a0, sp, 32
+; CHECK-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 48
 ; CHECK-NEXT:    ret
   %w = freeze <4 x i32> poison
   call void @usev(<4 x i32> %w)
diff --git a/llvm/test/CodeGen/SystemZ/pr60413.ll b/llvm/test/CodeGen/SystemZ/pr60413.ll
index 97cf1050408dd..7958ce8b0fd66 100644
--- a/llvm/test/CodeGen/SystemZ/pr60413.ll
+++ b/llvm/test/CodeGen/SystemZ/pr60413.ll
@@ -21,8 +21,9 @@ define dso_local void @m() local_unnamed_addr #1 {
 ; CHECK-NEXT:    ic %r1, 6(%r2)
 ; CHECK-NEXT:    larl %r2, e
 ; CHECK-NEXT:    lb %r0, 3(%r2)
-; CHECK-NEXT:    vlvgp %v0, %r0, %r1
-; CHECK-NEXT:    vlvgp %v1, %r1, %r0
+; CHECK-NEXT:    # freeze: $r2l
+; CHECK-NEXT:    vlvgp %v0, %r2, %r1
+; CHECK-NEXT:    vlvgp %v1, %r1, %r2
 ; CHECK-NEXT:    vlvgf %v1, %r1, 0
 ; CHECK-NEXT:    vlvgf %v1, %r1, 2
 ; CHECK-NEXT:    vlvgp %v2, %r1, %r1
@@ -31,8 +32,8 @@ define dso_local void @m() local_unnamed_addr #1 {
 ; CHECK-NEXT:    chi %r1, 128
 ; CHECK-NEXT:    ipm %r1
 ; CHECK-NEXT:    risbg %r1, %r1, 63, 191, 36
-; CHECK-NEXT:    vlvgf %v0, %r0, 0
-; CHECK-NEXT:    vlvgf %v0, %r0, 2
+; CHECK-NEXT:    vlvgf %v0, %r2, 0
+; CHECK-NEXT:    vlvgf %v0, %r2, 2
 ; CHECK-NEXT:    vgbm %v3, 30583
 ; CHECK-NEXT:    vn %v0, %v0, %v3
 ; CHECK-NEXT:    vn %v1, %v1, %v3
@@ -74,6 +75,7 @@ define dso_local void @m() local_unnamed_addr #1 {
 ; CHECK-NEXT:    rosbg %r2, %r3, 32, 61, 2
 ; CHECK-NEXT:    rosbg %r2, %r5, 32, 62, 1
 ; CHECK-NEXT:    or %r2, %r14
+; CHECK-NEXT:    # freeze: $r3l
 ; CHECK-NEXT:    xilf %r3, 4294967295
 ; CHECK-NEXT:    or %r3, %r2
 ; CHECK-NEXT:    tmll %r3, 65535
diff --git a/llvm/test/CodeGen/X86/bit-manip-i256.ll b/llvm/test/CodeGen/X86/bit-manip-i256.ll
index 042b7d428ba5c..c6d8b26a7cfb3 100644
--- a/llvm/test/CodeGen/X86/bit-manip-i256.ll
+++ b/llvm/test/CodeGen/X86/bit-manip-i256.ll
@@ -2856,15 +2856,16 @@ define i256 @isolate_msb_i256_vector(<4 x i64> %v0, i256 %idx) nounwind {
 ;
 ; AVX512F-LABEL: isolate_msb_i256_vector:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovaps {{.*#+}} zmm1 = [0,0,0,9223372036854775808,0,0,0,0]
-; AVX512F-NEXT:    vmovups %zmm1, -{{[0-9]+}}(%rsp)
 ; AVX512F-NEXT:    vpermq {{.*#+}} ymm1 = ymm0[3,2,1,0]
+; AVX512F-NEXT:    vplzcntq %zmm1, %zmm2
+; AVX512F-NEXT:    vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2 # [0,64,128,192]
 ; AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    vplzcntq %zmm1, %zmm1
-; AVX512F-NEXT:    vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
+; AVX512F-NEXT:    # freeze: $ymm1
 ; AVX512F-NEXT:    kshiftlw $12, %k0, %k0
 ; AVX512F-NEXT:    kshiftrw $12, %k0, %k1
-; AVX512F-NEXT:    vpcompressq %zmm1, %zmm1 {%k1}
+; AVX512F-NEXT:    vpcompressq %zmm2, %zmm1 {%k1}
+; AVX512F-NEXT:    vmovaps {{.*#+}} zmm2 = [0,0,0,9223372036854775808,0,0,0,0]
+; AVX512F-NEXT:    vmovups %zmm2, -{{[0-9]+}}(%rsp)
 ; AVX512F-NEXT:    vmovd %xmm1, %ecx
 ; AVX512F-NEXT:    movl %ecx, %eax
 ; AVX512F-NEXT:    shrb $6, %al
@@ -2900,12 +2901,13 @@ define i256 @isolate_msb_i256_vector(<4 x i64> %v0, i256 %idx) nounwind {
 ; AVX512VL-NEXT:    vptestmq %ymm1, %ymm1, %k1
 ; AVX512VL-NEXT:    vplzcntq %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
-; AVX512VL-NEXT:    vpcompressq %ymm1, %ymm1 {%k1}
-; AVX512VL-NEXT:    vmovaps {{.*#+}} ymm2 = [0,0,0,9223372036854775808]
-; AVX512VL-NEXT:    vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT:    vxorps %xmm2, %xmm2, %xmm2
-; AVX512VL-NEXT:    vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT:    vmovd %xmm1, %ecx
+; AVX512VL-NEXT:    # freeze: $ymm2
+; AVX512VL-NEXT:    vpcompressq %ymm1, %ymm2 {%k1}
+; AVX512VL-NEXT:    vmovaps {{.*#+}} ymm1 = [0,0,0,9223372036854775808]
+; AVX512VL-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; AVX512VL-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT:    vmovd %xmm2, %ecx
 ; AVX512VL-NEXT:    movl %ecx, %eax
 ; AVX512VL-NEXT:    shrb $6, %al
 ; AVX512VL-NEXT:    movzbl %al, %edx
@@ -2939,12 +2941,13 @@ define i256 @isolate_msb_i256_vector(<4 x i64> %v0, i256 %idx) nounwind {
 ; AVX512VBMI-NEXT:    vptestmq %ymm1, %ymm1, %k1
 ; AVX512VBMI-NEXT:    vplzcntq %ymm1, %ymm1
 ; AVX512VBMI-NEXT:    vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
-; AVX512VBMI-NEXT:    vpcompressq %ymm1, %ymm1 {%k1}
-; AVX512VBMI-NEXT:    vmovaps {{.*#+}} ymm2 = [0,0,0,9223372036854775808]
-; AVX512VBMI-NEXT:    vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT:    vxorps %xmm2, %xmm2, %xmm2
-; AVX512VBMI-NEXT:    vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT:    vmovd %xmm1, %ecx
+; AVX512VBMI-NEXT:    # freeze: $ymm2
+; AVX512VBMI-NEXT:    vpcompressq %ymm1, %ymm2 {%k1}
+; AVX512VBMI-NEXT:    vmovaps {{.*#+}} ymm1 = [0,0,0,9223372036854775808]
+; AVX512VBMI-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; AVX512VBMI-NEXT:    vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT:    vmovd %xmm2, %ecx
 ; AVX512VBMI-NEXT:    movl %ecx, %eax
 ; AVX512VBMI-NEXT:    shrb $6, %al
 ; AVX512VBMI-NEXT:    movzbl %al, %edx
diff --git a/llvm/test/CodeGen/X86/combine-urem.ll b/llvm/test/CodeGen/X86/combine-urem.ll
index 34c7d3deea023..dd33f38cc4c75 100644
--- a/llvm/test/CodeGen/X86/combine-urem.ll
+++ b/llvm/test/CodeGen/X86/combine-urem.ll
@@ -63,16 +63,18 @@ define <4 x i32> @combine_vec_urem_by_negone(<4 x i32> %x) {
 define <4 x i32> @combine_vec_urem_undef_by_negone(<4 x i32> %in) {
 ; SSE-LABEL: combine_vec_urem_undef_by_negone:
 ; SSE:       # %bb.0:
+; SSE-NEXT:    # freeze: $xmm1
 ; SSE-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE-NEXT:    pandn %xmm0, %xmm0
+; SSE-NEXT:    pcmpeqd %xmm1, %xmm0
+; SSE-NEXT:    pandn %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: combine_vec_urem_undef_by_negone:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT:    vpandn %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    # freeze: $xmm0
+; AVX-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm1
+; AVX-NEXT:    vpandn %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    retq
   %x = call <4 x i32> @llvm.x86.sse2.pslli.d(<4 x i32> undef, i32 0)
   %y = urem <4 x i32> %x, <i32 -1, i32 -1, i32 -1, i32 -1>
diff --git a/llvm/test/CodeGen/X86/freeze-binary.ll b/llvm/test/CodeGen/X86/freeze-binary.ll
index e4b2dcf0a62dd..0d011746f753e 100644
--- a/llvm/test/CodeGen/X86/freeze-binary.ll
+++ b/llvm/test/CodeGen/X86/freeze-binary.ll
@@ -399,11 +399,13 @@ define i32 @freeze_shl_nsw(i32 %a0) nounwind {
 define i32 @freeze_shl_outofrange(i32 %a0) nounwind {
 ; X86-LABEL: freeze_shl_outofrange:
 ; X86:       # %bb.0:
+; X86-NEXT:    # freeze: $eax
 ; X86-NEXT:    shll $2, %eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: freeze_shl_outofrange:
 ; X64:       # %bb.0:
+; X64-NEXT:    # freeze: $eax
 ; X64-NEXT:    shll $2, %eax
 ; X64-NEXT:    retq
   %x = shl i32 %a0, 32
@@ -513,11 +515,13 @@ define i32 @freeze_ashr_exact_extra_use(i32 %a0, ptr %escape) nounwind {
 define i32 @freeze_ashr_outofrange(i32 %a0) nounwind {
 ; X86-LABEL: freeze_ashr_outofrange:
 ; X86:       # %bb.0:
+; X86-NEXT:    # freeze: $eax
 ; X86-NEXT:    sarl $3, %eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: freeze_ashr_outofrange:
 ; X64:       # %bb.0:
+; X64-NEXT:    # freeze: $eax
 ; X64-NEXT:    sarl $3, %eax
 ; X64-NEXT:    retq
   %x = ashr i32 %a0, 32
@@ -628,11 +632,13 @@ define i32 @freeze_lshr_exact_extra_use(i32 %a0, ptr %escape) nounwind {
 define i32 @freeze_lshr_outofrange(i32 %a0) nounwind {
 ; X86-LABEL: freeze_lshr_outofrange:
 ; X86:       # %bb.0:
+; X86-NEXT:    # freeze: $eax
 ; X86-NEXT:    shrl %eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: freeze_lshr_outofrange:
 ; X64:       # %bb.0:
+; X64-NEXT:    # freeze: $eax
 ; X64-NEXT:    shrl %eax
 ; X64-NEXT:    retq
   %x = lshr i32 %a0, 32
diff --git a/llvm/test/CodeGen/X86/freeze-multi-use.ll b/llvm/test/CodeGen/X86/freeze-multi-use.ll
index 08bee89cec667..07ae3e751c95b 100644
--- a/llvm/test/CodeGen/X86/freeze-multi-use.ll
+++ b/llvm/test/CodeGen/X86/freeze-multi-use.ll
@@ -7,9 +7,10 @@
 define i32 @add_twice(i32 %n) {
 ; CHECK-LABEL: add_twice:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    movl %edi, %eax
-; CHECK-NEXT:    addl %eax, %eax
-; CHECK-NEXT:    addl %eax, %eax
+; CHECK-NEXT:    # kill: def $edi killed $edi def $rdi
+; CHECK-NEXT:    # freeze: $ecx
+; CHECK-NEXT:    leal (%rdi,%rcx), %eax
+; CHECK-NEXT:    addl %ecx, %eax
 ; CHECK-NEXT:    retq
   %w = freeze i32 poison
   %a = add i32 %n, %w
@@ -21,8 +22,10 @@ define i32 @add_twice(i32 %n) {
 define i32 @ashr_mul(i32 %x) {
 ; CHECK-LABEL: ashr_mul:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    # freeze: $ecx
+; CHECK-NEXT:    movl %ecx, %eax
 ; CHECK-NEXT:    sarl $3, %eax
-; CHECK-NEXT:    imull %eax, %eax
+; CHECK-NEXT:    imull %ecx, %eax
 ; CHECK-NEXT:    retq
   %p = ashr i32 %x, 32
   %w = freeze i32 %p
@@ -35,14 +38,14 @@ define i32 @ashr_mul(i32 %x) {
 define i32 @cross_block(i1 %c, i32 %x) {
 ; CHECK-LABEL: cross_block:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl %esi, %eax
 ; CHECK-NEXT:    testb $1, %dil
 ; CHECK-NEXT:    je .LBB2_2
 ; CHECK-NEXT:  # %bb.1: # %t
-; CHECK-NEXT:    addl %esi, %eax
-; CHECK-NEXT:    subl %eax, %eax
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    # freeze: $ecx
+; CHECK-NEXT:    addl %ecx, %eax
+; CHECK-NEXT:    subl %ecx, %eax
 ; CHECK-NEXT:  .LBB2_2: # %e
-; CHECK-NEXT:    movl %esi, %eax
 ; CHECK-NEXT:    retq
 entry:
   %w = freeze i32 poison
@@ -59,9 +62,12 @@ e:
 define i32 @across_call() nounwind {
 ; CHECK-LABEL: across_call:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    # freeze: $ebx
+; CHECK-NEXT:    movl %ebx, %edi
 ; CHECK-NEXT:    callq use at PLT
-; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    movl %ebx, %eax
+; CHECK-NEXT:    popq %rbx
 ; CHECK-NEXT:    retq
   %w = freeze i32 poison
   call void @use(i32 %w)
@@ -72,9 +78,12 @@ define i32 @across_call() nounwind {
 define <4 x i32> @vec_across_call() nounwind {
 ; CHECK-LABEL: vec_across_call:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    subq $24, %rsp
+; CHECK-NEXT:    # freeze: $xmm0
+; CHECK-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
 ; CHECK-NEXT:    callq usev at PLT
-; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    addq $24, %rsp
 ; CHECK-NEXT:    retq
   %w = freeze <4 x i32> poison
   call void @usev(<4 x i32> %w)
diff --git a/llvm/test/CodeGen/X86/freeze-vector.ll b/llvm/test/CodeGen/X86/freeze-vector.ll
index 2327dac331088..6461c3dde3fbe 100644
--- a/llvm/test/CodeGen/X86/freeze-vector.ll
+++ b/llvm/test/CodeGen/X86/freeze-vector.ll
@@ -594,7 +594,8 @@ define void @freeze_buildvector_one_undef_elt(ptr %origin0, ptr %origin1, ptr %o
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X86-NEXT:    vpinsrd $1, (%edx), %xmm0, %xmm0
-; X86-NEXT:    vpinsrd $2, %eax, %xmm0, %xmm0
+; X86-NEXT:    # freeze: $edx
+; X86-NEXT:    vpinsrd $2, %edx, %xmm0, %xmm0
 ; X86-NEXT:    vpinsrd $3, (%ecx), %xmm0, %xmm0
 ; X86-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-NEXT:    vmovdqa %xmm0, (%eax)
@@ -605,6 +606,7 @@ define void @freeze_buildvector_one_undef_elt(ptr %origin0, ptr %origin1, ptr %o
 ; X64:       # %bb.0:
 ; X64-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X64-NEXT:    vpinsrd $1, (%rsi), %xmm0, %xmm0
+; X64-NEXT:    # freeze: $eax
 ; X64-NEXT:    vpinsrd $2, %eax, %xmm0, %xmm0
 ; X64-NEXT:    vpinsrd $3, (%rcx), %xmm0, %xmm0
 ; X64-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [7,7,7,7]
@@ -1125,13 +1127,15 @@ define <4 x i32> @freeze_poison_multi_use_with_zeros(ptr %p) nounwind {
 ; X86-LABEL: freeze_poison_multi_use_with_zeros:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl %eax, (%eax)
-; X86-NEXT:    vmovd %eax, %xmm0
+; X86-NEXT:    # freeze: $ecx
+; X86-NEXT:    movl %ecx, (%eax)
+; X86-NEXT:    vmovd %ecx, %xmm0
 ; X86-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,0,1,1]
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: freeze_poison_multi_use_with_zeros:
 ; X64:       # %bb.0:
+; X64-NEXT:    # freeze: $eax
 ; X64-NEXT:    movl %eax, (%rdi)
 ; X64-NEXT:    vmovd %eax, %xmm0
 ; X64-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,0,1,1]
@@ -1147,15 +1151,17 @@ define <4 x i32> @freeze_poison_multi_use_splat(ptr %p, i32 %x) nounwind {
 ; X86-LABEL: freeze_poison_multi_use_splat:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl %eax, (%eax)
-; X86-NEXT:    vmovd %eax, %xmm0
+; X86-NEXT:    # freeze: $ecx
+; X86-NEXT:    movl %ecx, (%eax)
+; X86-NEXT:    vmovd %ecx, %xmm0
 ; X86-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; X86-NEXT:    vpinsrd $2, %eax, %xmm0, %xmm0
-; X86-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
+; X86-NEXT:    vpinsrd $2, %ecx, %xmm0, %xmm0
+; X86-NEXT:    vpinsrd $3, %ecx, %xmm0, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: freeze_poison_multi_use_splat:
 ; X64:       # %bb.0:
+; X64-NEXT:    # freeze: $eax
 ; X64-NEXT:    movl %eax, (%rdi)
 ; X64-NEXT:    vmovd %eax, %xmm0
 ; X64-NEXT:    vpinsrd $1, %esi, %xmm0, %xmm0
@@ -1176,15 +1182,17 @@ define <8 x i32> @freeze_poison_multi_use_upper(ptr %p, i32 %x) nounwind {
 ; X86-LABEL: freeze_poison_multi_use_upper:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl %eax, (%eax)
+; X86-NEXT:    # freeze: $ecx
+; X86-NEXT:    movl %ecx, (%eax)
 ; X86-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X86-NEXT:    vmovd %eax, %xmm1
+; X86-NEXT:    vmovd %ecx, %xmm1
 ; X86-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; X86-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: freeze_poison_multi_use_upper:
 ; X64:       # %bb.0:
+; X64-NEXT:    # freeze: $eax
 ; X64-NEXT:    movl %eax, (%rdi)
 ; X64-NEXT:    vmovd %esi, %xmm0
 ; X64-NEXT:    vmovd %eax, %xmm1
diff --git a/llvm/test/CodeGen/X86/freeze.ll b/llvm/test/CodeGen/X86/freeze.ll
index 2609b2fafac54..384a7116b7bb8 100644
--- a/llvm/test/CodeGen/X86/freeze.ll
+++ b/llvm/test/CodeGen/X86/freeze.ll
@@ -6,7 +6,7 @@
 define i32 @freeze_int() {
 ; X86ASM-LABEL: freeze_int:
 ; X86ASM:       # %bb.0:
-; X86ASM-NEXT:    # implicit-def: $eax
+; X86ASM-NEXT:    # freeze: $eax
 ; X86ASM-NEXT:    imull %eax, %eax
 ; X86ASM-NEXT:    retq
   %y1 = freeze i32 undef
@@ -17,7 +17,7 @@ define i32 @freeze_int() {
 define void @freeze_int_asm() {
 ; X86ASM-LABEL: freeze_int_asm:
 ; X86ASM:       # %bb.0:
-; X86ASM-NEXT:    # implicit-def: $ecx
+; X86ASM-NEXT:    # freeze: $ecx
 ; X86ASM-NEXT:    imull %ecx, %ecx
 ; X86ASM-NEXT:    #APP
 ; X86ASM-NEXT:    # use %ecx
@@ -32,6 +32,7 @@ define void @freeze_int_asm() {
 define i5 @freeze_int2() {
 ; X86ASM-LABEL: freeze_int2:
 ; X86ASM:       # %bb.0:
+; X86ASM-NEXT:    # freeze: $al
 ; X86ASM-NEXT:    mulb %al
 ; X86ASM-NEXT:    retq
   %y1 = freeze i5 undef
@@ -42,7 +43,7 @@ define i5 @freeze_int2() {
 define float @freeze_float() {
 ; X86ASM-LABEL: freeze_float:
 ; X86ASM:       # %bb.0:
-; X86ASM-NEXT:    # implicit-def: $xmm0
+; X86ASM-NEXT:    # freeze: $xmm0
 ; X86ASM-NEXT:    addss %xmm0, %xmm0
 ; X86ASM-NEXT:    retq
   %y1 = freeze float undef
@@ -55,6 +56,7 @@ define half @freeze_half() {
 ; X86ASM:       # %bb.0:
 ; X86ASM-NEXT:    pushq %rax
 ; X86ASM-NEXT:    .cfi_def_cfa_offset 16
+; X86ASM-NEXT:    # freeze: $xmm0
 ; X86ASM-NEXT:    callq __extendhfsf2 at PLT
 ; X86ASM-NEXT:    addss %xmm0, %xmm0
 ; X86ASM-NEXT:    callq __truncsfhf2 at PLT
@@ -69,6 +71,7 @@ define half @freeze_half() {
 define <2 x i32> @freeze_ivec() {
 ; X86ASM-LABEL: freeze_ivec:
 ; X86ASM:       # %bb.0:
+; X86ASM-NEXT:    # freeze: $xmm0
 ; X86ASM-NEXT:    paddd %xmm0, %xmm0
 ; X86ASM-NEXT:    retq
   %y1 = freeze <2 x i32> undef
@@ -79,6 +82,7 @@ define <2 x i32> @freeze_ivec() {
 define ptr @freeze_ptr() {
 ; X86ASM-LABEL: freeze_ptr:
 ; X86ASM:       # %bb.0:
+; X86ASM-NEXT:    # freeze: $rax
 ; X86ASM-NEXT:    addq $4, %rax
 ; X86ASM-NEXT:    retq
   %y1 = freeze ptr undef
@@ -89,7 +93,9 @@ define ptr @freeze_ptr() {
 define i32 @freeze_struct() {
 ; X86ASM-LABEL: freeze_struct:
 ; X86ASM:       # %bb.0:
-; X86ASM-NEXT:    addl %eax, %eax
+; X86ASM-NEXT:    # freeze: $ecx
+; X86ASM-NEXT:    # freeze: $eax
+; X86ASM-NEXT:    addl %ecx, %eax
 ; X86ASM-NEXT:    retq
   %y1 = freeze %struct.T undef
   %v1 = extractvalue %struct.T %y1, 0
@@ -101,7 +107,9 @@ define i32 @freeze_struct() {
 define i32 @freeze_anonstruct() {
 ; X86ASM-LABEL: freeze_anonstruct:
 ; X86ASM:       # %bb.0:
-; X86ASM-NEXT:    addl %eax, %eax
+; X86ASM-NEXT:    # freeze: $ecx
+; X86ASM-NEXT:    # freeze: $eax
+; X86ASM-NEXT:    addl %ecx, %eax
 ; X86ASM-NEXT:    retq
   %y1 = freeze {i32, i32} undef
   %v1 = extractvalue {i32, i32} %y1, 0
@@ -113,8 +121,10 @@ define i32 @freeze_anonstruct() {
 define i32 @freeze_anonstruct2() {
 ; X86ASM-LABEL: freeze_anonstruct2:
 ; X86ASM:       # %bb.0:
+; X86ASM-NEXT:    # freeze: $ecx
+; X86ASM-NEXT:    # freeze: $ax
 ; X86ASM-NEXT:    movzwl %ax, %eax
-; X86ASM-NEXT:    addl %eax, %eax
+; X86ASM-NEXT:    addl %ecx, %eax
 ; X86ASM-NEXT:    retq
   %y1 = freeze {i32, i16} undef
   %v1 = extractvalue {i32, i16} %y1, 0
@@ -127,7 +137,9 @@ define i32 @freeze_anonstruct2() {
 define i64 @freeze_array() {
 ; X86ASM-LABEL: freeze_array:
 ; X86ASM:       # %bb.0:
-; X86ASM-NEXT:    addq %rax, %rax
+; X86ASM-NEXT:    # freeze: $rcx
+; X86ASM-NEXT:    # freeze: $rax
+; X86ASM-NEXT:    addq %rcx, %rax
 ; X86ASM-NEXT:    retq
   %y1 = freeze [2 x i64] undef
   %v1 = extractvalue [2 x i64] %y1, 0
@@ -207,6 +219,8 @@ entry:
 define i128 @freeze_i128() {
 ; X86ASM-LABEL: freeze_i128:
 ; X86ASM:       # %bb.0:
+; X86ASM-NEXT:    # freeze: $rax
+; X86ASM-NEXT:    # freeze: $rdx
 ; X86ASM-NEXT:    retq
   %y1 = freeze i128 poison
   ret i128 %y1
diff --git a/llvm/test/CodeGen/X86/pr169485.ll b/llvm/test/CodeGen/X86/pr169485.ll
index 63035fce02615..008aad367cdb0 100644
--- a/llvm/test/CodeGen/X86/pr169485.ll
+++ b/llvm/test/CodeGen/X86/pr169485.ll
@@ -9,8 +9,9 @@ define <8 x float> @widget(<2 x float> %arg) #0 {
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
 ; CHECK-NEXT:    callq *%rax
-; CHECK-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
-; CHECK-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,0],zero
+; CHECK-NEXT:    vmovaps (%rsp), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    # freeze: $xmm0
+; CHECK-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],zero
 ; CHECK-NEXT:    addq $24, %rsp
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/pr57402.ll b/llvm/test/CodeGen/X86/pr57402.ll
index 338229f51cf6e..3a1b5eb963900 100644
--- a/llvm/test/CodeGen/X86/pr57402.ll
+++ b/llvm/test/CodeGen/X86/pr57402.ll
@@ -4,25 +4,27 @@
 define void @PR57402() {
 ; CHECK-LABEL: PR57402:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    # freeze: $rcx
+; CHECK-NEXT:    movl %ecx, %eax
 ; CHECK-NEXT:    notl %eax
 ; CHECK-NEXT:    andl $-2, %eax
-; CHECK-NEXT:    leal 1(%rax,%rax,2), %ecx
-; CHECK-NEXT:    movswq %cx, %rsi
-; CHECK-NEXT:    xorl %edi, %edi
+; CHECK-NEXT:    leal 1(%rax,%rax,2), %esi
+; CHECK-NEXT:    movswq %si, %rdi
+; CHECK-NEXT:    xorl %r8d, %r8d
 ; CHECK-NEXT:    movq $-1, %rax
 ; CHECK-NEXT:    xorl %edx, %edx
-; CHECK-NEXT:    divq %rsi
-; CHECK-NEXT:    testb %dil, %dil
+; CHECK-NEXT:    divq %rdi
+; CHECK-NEXT:    testb %r8b, %r8b
 ; CHECK-NEXT:    jne .LBB0_4
 ; CHECK-NEXT:  # %bb.1: # %entry
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    testb %al, %al
 ; CHECK-NEXT:    jne .LBB0_4
 ; CHECK-NEXT:  # %bb.2: # %entry
-; CHECK-NEXT:    andl %ecx, %edx
+; CHECK-NEXT:    andl %esi, %edx
 ; CHECK-NEXT:    movswl %dx, %eax
-; CHECK-NEXT:    imull %eax, %eax
-; CHECK-NEXT:    testq %rax, %rax
+; CHECK-NEXT:    imull %eax, %ecx
+; CHECK-NEXT:    testq %rcx, %rcx
 ; CHECK-NEXT:    jne .LBB0_3
 ; CHECK-NEXT:  .LBB0_4: # %if.end
 ; CHECK-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/pr57554.ll b/llvm/test/CodeGen/X86/pr57554.ll
index 0eb40979c3552..8c7fee5afc28a 100644
--- a/llvm/test/CodeGen/X86/pr57554.ll
+++ b/llvm/test/CodeGen/X86/pr57554.ll
@@ -6,6 +6,7 @@
 define void @PR57554() {
 ; CHECK-LABEL: PR57554:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    # freeze: $al
 ; CHECK-NEXT:    movb %al, a(%rip)
 ; CHECK-NEXT:    retq
   %.fr13 = freeze i8 undef
diff --git a/llvm/test/CodeGen/X86/var-permute-256.ll b/llvm/test/CodeGen/X86/var-permute-256.ll
index 5155077f41df0..9024545cca9a8 100644
--- a/llvm/test/CodeGen/X86/var-permute-256.ll
+++ b/llvm/test/CodeGen/X86/var-permute-256.ll
@@ -1542,7 +1542,8 @@ define <16 x i16> @var_shuffle_v16i16_from_v8i16(<8 x i16> %v, <16 x i16> %indic
 ; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
 ; AVX2-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [514,514,514,514,514,514,514,514,514,514,514,514,514,514,514,514]
 ; AVX2-NEXT:    vpaddw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [256,256,256,256,256,256,256,256,256,256,256,256,256,256,256,256]
-; AVX2-NEXT:    vpshufb %ymm1, %ymm0, %ymm2
+; AVX2-NEXT:    # freeze: $xmm2
+; AVX2-NEXT:    vpshufb %ymm1, %ymm2, %ymm2
 ; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
 ; AVX2-NEXT:    vpshufb %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
@@ -1554,7 +1555,8 @@ define <16 x i16> @var_shuffle_v16i16_from_v8i16(<8 x i16> %v, <16 x i16> %indic
 ; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
 ; AVX512-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [514,514,514,514,514,514,514,514,514,514,514,514,514,514,514,514]
 ; AVX512-NEXT:    vpaddw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [256,256,256,256,256,256,256,256,256,256,256,256,256,256,256,256]
-; AVX512-NEXT:    vpshufb %ymm1, %ymm0, %ymm2
+; AVX512-NEXT:    # freeze: $xmm2
+; AVX512-NEXT:    vpshufb %ymm1, %ymm2, %ymm2
 ; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
 ; AVX512-NEXT:    vpshufb %ymm1, %ymm0, %ymm0
 ; AVX512-NEXT:    vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
@@ -1568,6 +1570,7 @@ define <16 x i16> @var_shuffle_v16i16_from_v8i16(<8 x i16> %v, <16 x i16> %indic
 ; AVX512VLF-NEXT:    vpaddw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [256,256,256,256,256,256,256,256,256,256,256,256,256,256,256,256]
 ; AVX512VLF-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
 ; AVX512VLF-NEXT:    vpshufb %ymm1, %ymm0, %ymm2
+; AVX512VLF-NEXT:    # freeze: $xmm0
 ; AVX512VLF-NEXT:    vpshufb %ymm1, %ymm0, %ymm3
 ; AVX512VLF-NEXT:    vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm0
 ; AVX512VLF-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm2 ^ (ymm0 & (ymm3 ^ ymm2))
diff --git a/llvm/test/CodeGen/X86/vector-compress-freeze.ll b/llvm/test/CodeGen/X86/vector-compress-freeze.ll
index 981557f9b56cf..ddf3b306069dd 100644
--- a/llvm/test/CodeGen/X86/vector-compress-freeze.ll
+++ b/llvm/test/CodeGen/X86/vector-compress-freeze.ll
@@ -24,8 +24,9 @@ define <16 x i32> @test_compress_freeze(<16 x i32> %a0, <16 x i32> %a1, <16 x i8
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpcmpgtd %zmm1, %zmm0, %k1
 ; CHECK-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
-; CHECK-NEXT:    vpcompressd %zmm0, %zmm0 {%k1}
-; CHECK-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0
+; CHECK-NEXT:    # freeze: $zmm1
+; CHECK-NEXT:    vpcompressd %zmm0, %zmm1 {%k1}
+; CHECK-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0
 ; CHECK-NEXT:    retq
   %cmp = icmp sgt <16 x i32> %a0, %a1
   %ext = zext <16 x i8> %a3 to <16 x i32>
diff --git a/llvm/test/CodeGen/X86/vector-compress.ll b/llvm/test/CodeGen/X86/vector-compress.ll
index e3b08871f4732..4ae8f55560743 100644
--- a/llvm/test/CodeGen/X86/vector-compress.ll
+++ b/llvm/test/CodeGen/X86/vector-compress.ll
@@ -612,10 +612,10 @@ define <16 x i32> @test_compress_v16i32(<16 x i32> %vec, <16 x i1> %mask, <16 x
 ; AVX2-NEXT:    vpextrd $1, %xmm3, %eax
 ; AVX2-NEXT:    vmovd %xmm3, %ecx
 ; AVX2-NEXT:    addl %eax, %ecx
-; AVX2-NEXT:    vpextrb $4, %xmm2, %edx
+; AVX2-NEXT:    vpextrb $4, %xmm2, %esi
 ; AVX2-NEXT:    andl $15, %ecx
 ; AVX2-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT:    andl $1, %edx
+; AVX2-NEXT:    andl $1, %esi
 ; AVX2-NEXT:    vpextrb $1, %xmm2, %eax
 ; AVX2-NEXT:    andl $1, %eax
 ; AVX2-NEXT:    vmovd %xmm2, %ecx
@@ -627,17 +627,17 @@ define <16 x i32> @test_compress_v16i32(<16 x i32> %vec, <16 x i1> %mask, <16 x
 ; AVX2-NEXT:    andl $1, %ecx
 ; AVX2-NEXT:    addq %rax, %rcx
 ; AVX2-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT:    vpextrb $3, %xmm2, %r9d
+; AVX2-NEXT:    vpextrb $3, %xmm2, %r8d
+; AVX2-NEXT:    andl $1, %r8d
+; AVX2-NEXT:    addq %rcx, %r8
+; AVX2-NEXT:    addq %r8, %rsi
+; AVX2-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT:    vpextrb $5, %xmm2, %r9d
 ; AVX2-NEXT:    andl $1, %r9d
-; AVX2-NEXT:    addq %rcx, %r9
-; AVX2-NEXT:    addq %r9, %rdx
-; AVX2-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT:    vpextrb $5, %xmm2, %edi
-; AVX2-NEXT:    andl $1, %edi
-; AVX2-NEXT:    addq %rdx, %rdi
+; AVX2-NEXT:    addq %rsi, %r9
 ; AVX2-NEXT:    vpextrb $6, %xmm2, %r10d
 ; AVX2-NEXT:    andl $1, %r10d
-; AVX2-NEXT:    addq %rdi, %r10
+; AVX2-NEXT:    addq %r9, %r10
 ; AVX2-NEXT:    vpextrb $7, %xmm2, %r11d
 ; AVX2-NEXT:    andl $1, %r11d
 ; AVX2-NEXT:    addq %r10, %r11
@@ -656,36 +656,38 @@ define <16 x i32> @test_compress_v16i32(<16 x i32> %vec, <16 x i1> %mask, <16 x
 ; AVX2-NEXT:    vpextrb $12, %xmm2, %r13d
 ; AVX2-NEXT:    andl $1, %r13d
 ; AVX2-NEXT:    addq %r12, %r13
-; AVX2-NEXT:    vpextrb $13, %xmm2, %edx
-; AVX2-NEXT:    andl $1, %edx
-; AVX2-NEXT:    addq %r13, %rdx
-; AVX2-NEXT:    vpextrb $14, %xmm2, %ecx
-; AVX2-NEXT:    andl $1, %ecx
-; AVX2-NEXT:    addq %rdx, %rcx
-; AVX2-NEXT:    vpextrb $15, %xmm2, %esi
+; AVX2-NEXT:    vpextrb $13, %xmm2, %ecx
+; AVX2-NEXT:    movq %rcx, %rsi
 ; AVX2-NEXT:    andl $1, %esi
-; AVX2-NEXT:    addq %rcx, %rsi
+; AVX2-NEXT:    addq %r13, %rsi
+; AVX2-NEXT:    vpextrb $14, %xmm2, %eax
+; AVX2-NEXT:    movq %rax, %rcx
+; AVX2-NEXT:    andl $1, %ecx
+; AVX2-NEXT:    addq %rsi, %rcx
+; AVX2-NEXT:    vpextrb $15, %xmm2, %edx
+; AVX2-NEXT:    andl $1, %edx
+; AVX2-NEXT:    addq %rcx, %rdx
 ; AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm2
 ; AVX2-NEXT:    vextractps $3, %xmm2, %eax
-; AVX2-NEXT:    cmpq $16, %rsi
-; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
-; AVX2-NEXT:    cmovbl 32(%rsp,%r8,4), %eax
+; AVX2-NEXT:    cmpq $16, %rdx
+; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
+; AVX2-NEXT:    cmovbl 32(%rsp,%rdi,4), %eax
 ; AVX2-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
 ; AVX2-NEXT:    vmovss %xmm0, {{[0-9]+}}(%rsp)
-; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
-; AVX2-NEXT:    vextractps $1, %xmm0, 32(%rsp,%r8,4)
-; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
-; AVX2-NEXT:    vextractps $2, %xmm0, 32(%rsp,%r8,4)
-; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
-; AVX2-NEXT:    vextractps $3, %xmm0, 32(%rsp,%r8,4)
-; AVX2-NEXT:    andl $15, %r9d
+; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
+; AVX2-NEXT:    vextractps $1, %xmm0, 32(%rsp,%rdi,4)
+; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
+; AVX2-NEXT:    vextractps $2, %xmm0, 32(%rsp,%rdi,4)
+; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
+; AVX2-NEXT:    vextractps $3, %xmm0, 32(%rsp,%rdi,4)
+; AVX2-NEXT:    andl $15, %r8d
 ; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX2-NEXT:    vmovss %xmm0, 32(%rsp,%r9,4)
+; AVX2-NEXT:    vmovss %xmm0, 32(%rsp,%r8,4)
 ; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
 ; AVX2-NEXT:    andl $15, %eax
 ; AVX2-NEXT:    vextractps $1, %xmm0, 32(%rsp,%rax,4)
-; AVX2-NEXT:    andl $15, %edi
-; AVX2-NEXT:    vextractps $2, %xmm0, 32(%rsp,%rdi,4)
+; AVX2-NEXT:    andl $15, %r9d
+; AVX2-NEXT:    vextractps $2, %xmm0, 32(%rsp,%r9,4)
 ; AVX2-NEXT:    andl $15, %r10d
 ; AVX2-NEXT:    vextractps $3, %xmm0, 32(%rsp,%r10,4)
 ; AVX2-NEXT:    andl $15, %r11d
@@ -700,13 +702,13 @@ define <16 x i32> @test_compress_v16i32(<16 x i32> %vec, <16 x i1> %mask, <16 x
 ; AVX2-NEXT:    vmovss %xmm2, 32(%rsp,%r12,4)
 ; AVX2-NEXT:    andl $15, %r13d
 ; AVX2-NEXT:    vextractps $1, %xmm2, 32(%rsp,%r13,4)
-; AVX2-NEXT:    andl $15, %edx
-; AVX2-NEXT:    vextractps $2, %xmm2, 32(%rsp,%rdx,4)
+; AVX2-NEXT:    andl $15, %esi
+; AVX2-NEXT:    vextractps $2, %xmm2, 32(%rsp,%rsi,4)
 ; AVX2-NEXT:    andl $15, %ecx
 ; AVX2-NEXT:    vextractps $3, %xmm2, 32(%rsp,%rcx,4)
-; AVX2-NEXT:    cmpq $15, %rsi
+; AVX2-NEXT:    cmpq $15, %rdx
 ; AVX2-NEXT:    movl $15, %eax
-; AVX2-NEXT:    cmovbq %rsi, %rax
+; AVX2-NEXT:    cmovbq %rdx, %rax
 ; AVX2-NEXT:    shll $2, %eax
 ; AVX2-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %ecx # 4-byte Reload
 ; AVX2-NEXT:    movl %ecx, 32(%rsp,%rax)
@@ -828,7 +830,8 @@ define <16 x float> @test_compress_v16f32(<16 x float> %vec, <16 x i1> %mask, <1
 ; AVX2-NEXT:    # kill: def $eax killed $eax killed $rax def $rax
 ; AVX2-NEXT:    andl $15, %eax
 ; AVX2-NEXT:    vextractps $3, %xmm1, (%rsp,%rax,4)
-; AVX2-NEXT:    vpextrb $12, %xmm2, %edx
+; AVX2-NEXT:    vpextrb $12, %xmm2, %eax
+; AVX2-NEXT:    movq %rax, %rdx
 ; AVX2-NEXT:    andl $1, %edx
 ; AVX2-NEXT:    addq %rcx, %rdx
 ; AVX2-NEXT:    # kill: def $ecx killed $ecx killed $rcx def $rcx
@@ -844,26 +847,27 @@ define <16 x float> @test_compress_v16f32(<16 x float> %vec, <16 x i1> %mask, <1
 ; AVX2-NEXT:    movl %eax, %ecx
 ; AVX2-NEXT:    andl $15, %ecx
 ; AVX2-NEXT:    vextractps $2, %xmm0, (%rsp,%rcx,4)
-; AVX2-NEXT:    vpextrb $14, %xmm2, %edx
-; AVX2-NEXT:    andl $1, %edx
-; AVX2-NEXT:    vpextrb $15, %xmm2, %ecx
-; AVX2-NEXT:    addq %rax, %rdx
+; AVX2-NEXT:    vpextrb $14, %xmm2, %ecx
 ; AVX2-NEXT:    andl $1, %ecx
-; AVX2-NEXT:    addq %rdx, %rcx
-; AVX2-NEXT:    # kill: def $edx killed $edx killed $rdx def $rdx
-; AVX2-NEXT:    andl $15, %edx
+; AVX2-NEXT:    vpextrb $15, %xmm2, %edx
+; AVX2-NEXT:    addq %rax, %rcx
+; AVX2-NEXT:    movq %rdx, %rax
+; AVX2-NEXT:    andl $1, %eax
+; AVX2-NEXT:    addq %rcx, %rax
+; AVX2-NEXT:    # kill: def $ecx killed $ecx killed $rcx def $rcx
+; AVX2-NEXT:    andl $15, %ecx
 ; AVX2-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; AVX2-NEXT:    vmovss %xmm0, (%rsp,%rdx,4)
-; AVX2-NEXT:    cmpq $16, %rcx
+; AVX2-NEXT:    vmovss %xmm0, (%rsp,%rcx,4)
+; AVX2-NEXT:    cmpq $16, %rax
 ; AVX2-NEXT:    jae .LBB9_2
 ; AVX2-NEXT:  # %bb.1:
 ; AVX2-NEXT:    vmovaps %xmm3, %xmm0
 ; AVX2-NEXT:  .LBB9_2:
-; AVX2-NEXT:    cmpq $15, %rcx
-; AVX2-NEXT:    movl $15, %eax
-; AVX2-NEXT:    cmovbq %rcx, %rax
-; AVX2-NEXT:    shll $2, %eax
-; AVX2-NEXT:    vmovss %xmm0, (%rsp,%rax)
+; AVX2-NEXT:    cmpq $15, %rax
+; AVX2-NEXT:    movl $15, %ecx
+; AVX2-NEXT:    cmovbq %rax, %rcx
+; AVX2-NEXT:    shll $2, %ecx
+; AVX2-NEXT:    vmovss %xmm0, (%rsp,%rcx)
 ; AVX2-NEXT:    vmovaps (%rsp), %ymm0
 ; AVX2-NEXT:    vmovaps {{[0-9]+}}(%rsp), %ymm1
 ; AVX2-NEXT:    movq %rbp, %rsp
@@ -918,19 +922,19 @@ define <8 x i64> @test_compress_v8i64(<8 x i64> %vec, <8 x i1> %mask, <8 x i64>
 ; AVX2-NEXT:    vmovd %xmm2, %esi
 ; AVX2-NEXT:    andl $1, %esi
 ; AVX2-NEXT:    addq %rsi, %rdx
-; AVX2-NEXT:    vpextrw $2, %xmm2, %r8d
+; AVX2-NEXT:    vpextrw $2, %xmm2, %edi
+; AVX2-NEXT:    andl $1, %edi
+; AVX2-NEXT:    addq %rdx, %rdi
+; AVX2-NEXT:    vpextrw $3, %xmm2, %r8d
 ; AVX2-NEXT:    andl $1, %r8d
-; AVX2-NEXT:    addq %rdx, %r8
-; AVX2-NEXT:    vpextrw $3, %xmm2, %r9d
+; AVX2-NEXT:    addq %rdi, %r8
+; AVX2-NEXT:    addq %r8, %rax
+; AVX2-NEXT:    vpextrw $5, %xmm2, %r9d
 ; AVX2-NEXT:    andl $1, %r9d
-; AVX2-NEXT:    addq %r8, %r9
-; AVX2-NEXT:    addq %r9, %rax
-; AVX2-NEXT:    vpextrw $5, %xmm2, %edi
-; AVX2-NEXT:    andl $1, %edi
-; AVX2-NEXT:    addq %rax, %rdi
+; AVX2-NEXT:    addq %rax, %r9
 ; AVX2-NEXT:    vpextrw $6, %xmm2, %r10d
 ; AVX2-NEXT:    andl $1, %r10d
-; AVX2-NEXT:    addq %rdi, %r10
+; AVX2-NEXT:    addq %r9, %r10
 ; AVX2-NEXT:    vpextrw $7, %xmm2, %r11d
 ; AVX2-NEXT:    andl $1, %r11d
 ; AVX2-NEXT:    addq %r10, %r11
@@ -942,13 +946,13 @@ define <8 x i64> @test_compress_v8i64(<8 x i64> %vec, <8 x i1> %mask, <8 x i64>
 ; AVX2-NEXT:    vpextrq $1, %xmm0, (%rsp,%rsi,8)
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, (%rsp,%rdx,8)
-; AVX2-NEXT:    vpextrq $1, %xmm0, (%rsp,%r8,8)
-; AVX2-NEXT:    andl $7, %r9d
-; AVX2-NEXT:    vmovq %xmm1, (%rsp,%r9,8)
+; AVX2-NEXT:    vpextrq $1, %xmm0, (%rsp,%rdi,8)
+; AVX2-NEXT:    andl $7, %r8d
+; AVX2-NEXT:    vmovq %xmm1, (%rsp,%r8,8)
 ; AVX2-NEXT:    andl $7, %eax
 ; AVX2-NEXT:    vpextrq $1, %xmm1, (%rsp,%rax,8)
-; AVX2-NEXT:    andl $7, %edi
-; AVX2-NEXT:    vmovq %xmm2, (%rsp,%rdi,8)
+; AVX2-NEXT:    andl $7, %r9d
+; AVX2-NEXT:    vmovq %xmm2, (%rsp,%r9,8)
 ; AVX2-NEXT:    andl $7, %r10d
 ; AVX2-NEXT:    vpextrq $1, %xmm2, (%rsp,%r10,8)
 ; AVX2-NEXT:    cmpq $7, %r11
@@ -1032,7 +1036,8 @@ define <8 x double> @test_compress_v8f64(<8 x double> %vec, <8 x i1> %mask, <8 x
 ; AVX2-NEXT:    # kill: def $eax killed $eax killed $rax def $rax
 ; AVX2-NEXT:    andl $7, %eax
 ; AVX2-NEXT:    vmovhpd %xmm1, (%rsp,%rax,8)
-; AVX2-NEXT:    vpextrw $6, %xmm2, %edx
+; AVX2-NEXT:    vpextrw $6, %xmm2, %eax
+; AVX2-NEXT:    movq %rax, %rdx
 ; AVX2-NEXT:    andl $1, %edx
 ; AVX2-NEXT:    addq %rcx, %rdx
 ; AVX2-NEXT:    # kill: def $ecx killed $ecx killed $rcx def $rcx
@@ -1103,14 +1108,14 @@ define <16 x i8> @test_compress_v16i8(<16 x i8> %vec, <16 x i1> %mask, <16 x i8>
 ; AVX2-NEXT:    subb %sil, %al
 ; AVX2-NEXT:    vpextrb $3, %xmm1, %ebp
 ; AVX2-NEXT:    subb %bpl, %al
-; AVX2-NEXT:    vpextrb $4, %xmm1, %r12d
-; AVX2-NEXT:    subb %r12b, %al
-; AVX2-NEXT:    vpextrb $5, %xmm1, %r15d
-; AVX2-NEXT:    vpextrb $6, %xmm1, %r14d
+; AVX2-NEXT:    vpextrb $4, %xmm1, %r15d
 ; AVX2-NEXT:    subb %r15b, %al
-; AVX2-NEXT:    subb %r14b, %al
-; AVX2-NEXT:    vpextrb $7, %xmm1, %ebx
+; AVX2-NEXT:    vpextrb $5, %xmm1, %ebx
+; AVX2-NEXT:    vpextrb $6, %xmm1, %r12d
 ; AVX2-NEXT:    subb %bl, %al
+; AVX2-NEXT:    subb %r12b, %al
+; AVX2-NEXT:    vpextrb $7, %xmm1, %r14d
+; AVX2-NEXT:    subb %r14b, %al
 ; AVX2-NEXT:    vpextrb $8, %xmm1, %r11d
 ; AVX2-NEXT:    subb %r11b, %al
 ; AVX2-NEXT:    vpextrb $9, %xmm1, %r10d
@@ -1146,28 +1151,28 @@ define <16 x i8> @test_compress_v16i8(<16 x i8> %vec, <16 x i1> %mask, <16 x i8>
 ; AVX2-NEXT:    andl $1, %ebp
 ; AVX2-NEXT:    addq %rsi, %rbp
 ; AVX2-NEXT:    vpextrb $4, %xmm0, -56(%rsp,%rbp)
-; AVX2-NEXT:    andl $1, %r12d
-; AVX2-NEXT:    addq %rbp, %r12
-; AVX2-NEXT:    movl %r12d, %eax
-; AVX2-NEXT:    andl $15, %eax
-; AVX2-NEXT:    vpextrb $5, %xmm0, -56(%rsp,%rax)
 ; AVX2-NEXT:    andl $1, %r15d
-; AVX2-NEXT:    addq %r12, %r15
-; AVX2-NEXT:    andl $1, %r14d
-; AVX2-NEXT:    addq %r15, %r14
-; AVX2-NEXT:    # kill: def $r15d killed $r15d killed $r15 def $r15
-; AVX2-NEXT:    andl $15, %r15d
-; AVX2-NEXT:    vpextrb $6, %xmm0, -56(%rsp,%r15)
-; AVX2-NEXT:    movl %r14d, %eax
+; AVX2-NEXT:    addq %rbp, %r15
+; AVX2-NEXT:    movl %r15d, %eax
 ; AVX2-NEXT:    andl $15, %eax
-; AVX2-NEXT:    vpextrb $7, %xmm0, -56(%rsp,%rax)
+; AVX2-NEXT:    vpextrb $5, %xmm0, -56(%rsp,%rax)
 ; AVX2-NEXT:    andl $1, %ebx
-; AVX2-NEXT:    addq %r14, %rbx
-; AVX2-NEXT:    andl $1, %r11d
-; AVX2-NEXT:    addq %rbx, %r11
+; AVX2-NEXT:    addq %r15, %rbx
+; AVX2-NEXT:    andl $1, %r12d
+; AVX2-NEXT:    addq %rbx, %r12
 ; AVX2-NEXT:    # kill: def $ebx killed $ebx killed $rbx def $rbx
 ; AVX2-NEXT:    andl $15, %ebx
-; AVX2-NEXT:    vpextrb $8, %xmm0, -56(%rsp,%rbx)
+; AVX2-NEXT:    vpextrb $6, %xmm0, -56(%rsp,%rbx)
+; AVX2-NEXT:    movl %r12d, %eax
+; AVX2-NEXT:    andl $15, %eax
+; AVX2-NEXT:    vpextrb $7, %xmm0, -56(%rsp,%rax)
+; AVX2-NEXT:    andl $1, %r14d
+; AVX2-NEXT:    addq %r12, %r14
+; AVX2-NEXT:    andl $1, %r11d
+; AVX2-NEXT:    addq %r14, %r11
+; AVX2-NEXT:    # kill: def $r14d killed $r14d killed $r14 def $r14
+; AVX2-NEXT:    andl $15, %r14d
+; AVX2-NEXT:    vpextrb $8, %xmm0, -56(%rsp,%r14)
 ; AVX2-NEXT:    movl %r11d, %eax
 ; AVX2-NEXT:    andl $15, %eax
 ; AVX2-NEXT:    vpextrb $9, %xmm0, -56(%rsp,%rax)
@@ -1183,36 +1188,36 @@ define <16 x i8> @test_compress_v16i8(<16 x i8> %vec, <16 x i1> %mask, <16 x i8>
 ; AVX2-NEXT:    vpextrb $11, %xmm0, -56(%rsp,%rax)
 ; AVX2-NEXT:    andl $1, %r8d
 ; AVX2-NEXT:    addq %r9, %r8
-; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; AVX2-NEXT:    andl $1, %ecx
-; AVX2-NEXT:    addq %r8, %rcx
+; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
+; AVX2-NEXT:    andl $1, %edi
+; AVX2-NEXT:    addq %r8, %rdi
 ; AVX2-NEXT:    # kill: def $r8d killed $r8d killed $r8 def $r8
 ; AVX2-NEXT:    andl $15, %r8d
 ; AVX2-NEXT:    vpextrb $12, %xmm0, -56(%rsp,%r8)
-; AVX2-NEXT:    movl %ecx, %eax
+; AVX2-NEXT:    movl %edi, %eax
 ; AVX2-NEXT:    andl $15, %eax
 ; AVX2-NEXT:    vpextrb $13, %xmm0, -56(%rsp,%rax)
-; AVX2-NEXT:    andl $1, %edx
-; AVX2-NEXT:    addq %rcx, %rdx
-; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; AVX2-NEXT:    andl $1, %ecx
-; AVX2-NEXT:    addq %rdx, %rcx
-; AVX2-NEXT:    # kill: def $edx killed $edx killed $rdx def $rdx
-; AVX2-NEXT:    andl $15, %edx
-; AVX2-NEXT:    vpextrb $14, %xmm0, -56(%rsp,%rdx)
-; AVX2-NEXT:    movl %ecx, %eax
-; AVX2-NEXT:    andl $15, %eax
-; AVX2-NEXT:    vpextrb $15, %xmm0, -56(%rsp,%rax)
+; AVX2-NEXT:    movq %rdx, %rsi
+; AVX2-NEXT:    andl $1, %esi
+; AVX2-NEXT:    addq %rdi, %rsi
 ; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
 ; AVX2-NEXT:    andl $1, %eax
-; AVX2-NEXT:    addq %rcx, %rax
-; AVX2-NEXT:    cmpq $15, %rax
-; AVX2-NEXT:    movq %rax, %rdx
+; AVX2-NEXT:    addq %rsi, %rax
+; AVX2-NEXT:    # kill: def $esi killed $esi killed $rsi def $rsi
+; AVX2-NEXT:    andl $15, %esi
+; AVX2-NEXT:    vpextrb $14, %xmm0, -56(%rsp,%rsi)
+; AVX2-NEXT:    movl %eax, %ecx
+; AVX2-NEXT:    andl $15, %ecx
+; AVX2-NEXT:    vpextrb $15, %xmm0, -56(%rsp,%rcx)
+; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; AVX2-NEXT:    andl $1, %ecx
+; AVX2-NEXT:    addq %rax, %rcx
+; AVX2-NEXT:    cmpq $15, %rcx
 ; AVX2-NEXT:    vpextrb $15, %xmm0, %eax
 ; AVX2-NEXT:    cmovbel {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Folded Reload
-; AVX2-NEXT:    movl $15, %ecx
-; AVX2-NEXT:    cmovbq %rdx, %rcx
-; AVX2-NEXT:    movb %al, -56(%rsp,%rcx)
+; AVX2-NEXT:    movl $15, %edx
+; AVX2-NEXT:    cmovbq %rcx, %rdx
+; AVX2-NEXT:    movb %al, -56(%rsp,%rdx)
 ; AVX2-NEXT:    vmovaps -{{[0-9]+}}(%rsp), %xmm0
 ; AVX2-NEXT:    popq %rbx
 ; AVX2-NEXT:    popq %r12
@@ -1258,7 +1263,6 @@ define <16 x i8> @test_compress_v16i8(<16 x i8> %vec, <16 x i1> %mask, <16 x i8>
 define <8 x i16> @test_compress_v8i16(<8 x i16> %vec, <8 x i1> %mask, <8 x i16> %passthru) nounwind {
 ; AVX2-LABEL: test_compress_v8i16:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    pushq %rbp
 ; AVX2-NEXT:    pushq %r14
 ; AVX2-NEXT:    pushq %rbx
 ; AVX2-NEXT:    vpsllw $15, %xmm1, %xmm1
@@ -1266,43 +1270,43 @@ define <8 x i16> @test_compress_v8i16(<8 x i16> %vec, <8 x i1> %mask, <8 x i16>
 ; AVX2-NEXT:    vmovaps %xmm2, -{{[0-9]+}}(%rsp)
 ; AVX2-NEXT:    vpextrw $1, %xmm1, %r9d
 ; AVX2-NEXT:    andl $1, %r9d
-; AVX2-NEXT:    vmovd %xmm1, %esi
-; AVX2-NEXT:    andl $1, %esi
-; AVX2-NEXT:    leaq (%rsi,%r9), %rcx
+; AVX2-NEXT:    vmovd %xmm1, %edx
+; AVX2-NEXT:    andl $1, %edx
+; AVX2-NEXT:    leaq (%rdx,%r9), %rcx
 ; AVX2-NEXT:    vpextrw $2, %xmm1, %r10d
-; AVX2-NEXT:    movl %r10d, %eax
+; AVX2-NEXT:    movq %r10, %rax
 ; AVX2-NEXT:    andl $1, %eax
 ; AVX2-NEXT:    addq %rcx, %rax
 ; AVX2-NEXT:    vpextrw $3, %xmm1, %r8d
-; AVX2-NEXT:    movl %r8d, %edx
-; AVX2-NEXT:    andl $1, %edx
-; AVX2-NEXT:    addq %rax, %rdx
+; AVX2-NEXT:    movq %r8, %rsi
+; AVX2-NEXT:    andl $1, %esi
+; AVX2-NEXT:    addq %rax, %rsi
 ; AVX2-NEXT:    vpextrw $4, %xmm1, %r11d
-; AVX2-NEXT:    movl %r11d, %edi
+; AVX2-NEXT:    movq %r11, %rdi
 ; AVX2-NEXT:    andl $1, %edi
-; AVX2-NEXT:    addq %rdx, %rdi
-; AVX2-NEXT:    movl %esi, %ebx
+; AVX2-NEXT:    addq %rsi, %rdi
+; AVX2-NEXT:    movl %edx, %ebx
 ; AVX2-NEXT:    addl %r9d, %ebx
-; AVX2-NEXT:    vpextrw $5, %xmm1, %ebp
+; AVX2-NEXT:    vpextrw $5, %xmm1, %r14d
 ; AVX2-NEXT:    andl $1, %r10d
 ; AVX2-NEXT:    andl $1, %r8d
 ; AVX2-NEXT:    addl %r10d, %r8d
-; AVX2-NEXT:    movl %ebp, %r9d
+; AVX2-NEXT:    movq %r14, %r9
 ; AVX2-NEXT:    andl $1, %r9d
 ; AVX2-NEXT:    addq %rdi, %r9
 ; AVX2-NEXT:    addl %ebx, %r8d
 ; AVX2-NEXT:    vpextrw $6, %xmm1, %ebx
 ; AVX2-NEXT:    andl $1, %r11d
-; AVX2-NEXT:    andl $1, %ebp
-; AVX2-NEXT:    addl %r11d, %ebp
-; AVX2-NEXT:    movl %ebx, %r10d
+; AVX2-NEXT:    andl $1, %r14d
+; AVX2-NEXT:    addl %r11d, %r14d
+; AVX2-NEXT:    movq %rbx, %r10
 ; AVX2-NEXT:    andl $1, %r10d
 ; AVX2-NEXT:    addq %r9, %r10
 ; AVX2-NEXT:    andl $1, %ebx
-; AVX2-NEXT:    addl %ebp, %ebx
+; AVX2-NEXT:    addl %r14d, %ebx
 ; AVX2-NEXT:    vpextrw $7, %xmm1, %r14d
 ; AVX2-NEXT:    addl %r8d, %ebx
-; AVX2-NEXT:    movl %r14d, %r8d
+; AVX2-NEXT:    movq %r14, %r8
 ; AVX2-NEXT:    andl $1, %r8d
 ; AVX2-NEXT:    addq %r10, %r8
 ; AVX2-NEXT:    andl $1, %r14d
@@ -1310,30 +1314,29 @@ define <8 x i16> @test_compress_v8i16(<8 x i16> %vec, <8 x i1> %mask, <8 x i16>
 ; AVX2-NEXT:    andl $7, %r14d
 ; AVX2-NEXT:    vpextrw $7, %xmm0, %r11d
 ; AVX2-NEXT:    cmpq $8, %r8
-; AVX2-NEXT:    cmovbw -16(%rsp,%r14,2), %r11w
+; AVX2-NEXT:    cmovbw -24(%rsp,%r14,2), %r11w
 ; AVX2-NEXT:    vpextrw $0, %xmm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT:    vpextrw $1, %xmm0, -16(%rsp,%rsi,2)
+; AVX2-NEXT:    vpextrw $1, %xmm0, -24(%rsp,%rdx,2)
 ; AVX2-NEXT:    andl $7, %ecx
-; AVX2-NEXT:    vpextrw $2, %xmm0, -16(%rsp,%rcx,2)
+; AVX2-NEXT:    vpextrw $2, %xmm0, -24(%rsp,%rcx,2)
 ; AVX2-NEXT:    andl $7, %eax
-; AVX2-NEXT:    vpextrw $3, %xmm0, -16(%rsp,%rax,2)
-; AVX2-NEXT:    andl $7, %edx
-; AVX2-NEXT:    vpextrw $4, %xmm0, -16(%rsp,%rdx,2)
+; AVX2-NEXT:    vpextrw $3, %xmm0, -24(%rsp,%rax,2)
+; AVX2-NEXT:    andl $7, %esi
+; AVX2-NEXT:    vpextrw $4, %xmm0, -24(%rsp,%rsi,2)
 ; AVX2-NEXT:    andl $7, %edi
-; AVX2-NEXT:    vpextrw $5, %xmm0, -16(%rsp,%rdi,2)
+; AVX2-NEXT:    vpextrw $5, %xmm0, -24(%rsp,%rdi,2)
 ; AVX2-NEXT:    andl $7, %r9d
-; AVX2-NEXT:    vpextrw $6, %xmm0, -16(%rsp,%r9,2)
+; AVX2-NEXT:    vpextrw $6, %xmm0, -24(%rsp,%r9,2)
 ; AVX2-NEXT:    andl $7, %r10d
-; AVX2-NEXT:    vpextrw $7, %xmm0, -16(%rsp,%r10,2)
+; AVX2-NEXT:    vpextrw $7, %xmm0, -24(%rsp,%r10,2)
 ; AVX2-NEXT:    cmpq $7, %r8
 ; AVX2-NEXT:    movl $7, %eax
 ; AVX2-NEXT:    cmovbq %r8, %rax
 ; AVX2-NEXT:    andl $7, %eax
-; AVX2-NEXT:    movw %r11w, -16(%rsp,%rax,2)
+; AVX2-NEXT:    movw %r11w, -24(%rsp,%rax,2)
 ; AVX2-NEXT:    vmovaps -{{[0-9]+}}(%rsp), %xmm0
 ; AVX2-NEXT:    popq %rbx
 ; AVX2-NEXT:    popq %r14
-; AVX2-NEXT:    popq %rbp
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: test_compress_v8i16:
@@ -1970,10 +1973,11 @@ define <16 x i16> @test_compress_v16i16(<16 x i16> %vec, <16 x i1> %mask, <16 x
 ; AVX2-NEXT:    vpextrw $1, %xmm2, %eax
 ; AVX2-NEXT:    vmovd %xmm2, %ecx
 ; AVX2-NEXT:    addl %eax, %ecx
-; AVX2-NEXT:    vpextrw $6, %xmm1, %edx
+; AVX2-NEXT:    vpextrw $6, %xmm1, %esi
 ; AVX2-NEXT:    andl $15, %ecx
 ; AVX2-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT:    andl $1, %edx
+; AVX2-NEXT:    movq %rsi, %r8
+; AVX2-NEXT:    andl $1, %r8d
 ; AVX2-NEXT:    vpextrw $1, %xmm1, %eax
 ; AVX2-NEXT:    andl $1, %eax
 ; AVX2-NEXT:    vmovd %xmm1, %ecx
@@ -1985,24 +1989,24 @@ define <16 x i16> @test_compress_v16i16(<16 x i16> %vec, <16 x i1> %mask, <16 x
 ; AVX2-NEXT:    andl $1, %ecx
 ; AVX2-NEXT:    addq %rax, %rcx
 ; AVX2-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT:    vpextrw $3, %xmm1, %r8d
-; AVX2-NEXT:    andl $1, %r8d
-; AVX2-NEXT:    addq %rcx, %r8
-; AVX2-NEXT:    vpextrw $4, %xmm1, %r10d
+; AVX2-NEXT:    vpextrw $3, %xmm1, %eax
+; AVX2-NEXT:    andl $1, %eax
+; AVX2-NEXT:    addq %rcx, %rax
+; AVX2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT:    vpextrw $4, %xmm1, %r9d
+; AVX2-NEXT:    andl $1, %r9d
+; AVX2-NEXT:    addq %rax, %r9
+; AVX2-NEXT:    vpextrw $5, %xmm1, %r10d
 ; AVX2-NEXT:    andl $1, %r10d
-; AVX2-NEXT:    addq %r8, %r10
-; AVX2-NEXT:    vpextrw $5, %xmm1, %r11d
+; AVX2-NEXT:    addq %r9, %r10
+; AVX2-NEXT:    addq %r10, %r8
+; AVX2-NEXT:    vpextrw $7, %xmm1, %r11d
 ; AVX2-NEXT:    andl $1, %r11d
-; AVX2-NEXT:    addq %r10, %r11
-; AVX2-NEXT:    addq %r11, %rdx
-; AVX2-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT:    vpextrw $7, %xmm1, %r9d
-; AVX2-NEXT:    andl $1, %r9d
-; AVX2-NEXT:    addq %rdx, %r9
+; AVX2-NEXT:    addq %r8, %r11
 ; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm1
 ; AVX2-NEXT:    vmovd %xmm1, %ebx
 ; AVX2-NEXT:    andl $1, %ebx
-; AVX2-NEXT:    addq %r9, %rbx
+; AVX2-NEXT:    addq %r11, %rbx
 ; AVX2-NEXT:    vpextrw $1, %xmm1, %r14d
 ; AVX2-NEXT:    andl $1, %r14d
 ; AVX2-NEXT:    addq %rbx, %r14
@@ -2015,18 +2019,20 @@ define <16 x i16> @test_compress_v16i16(<16 x i16> %vec, <16 x i1> %mask, <16 x
 ; AVX2-NEXT:    vpextrw $4, %xmm1, %r13d
 ; AVX2-NEXT:    andl $1, %r13d
 ; AVX2-NEXT:    addq %r12, %r13
-; AVX2-NEXT:    vpextrw $5, %xmm1, %edx
-; AVX2-NEXT:    andl $1, %edx
-; AVX2-NEXT:    addq %r13, %rdx
-; AVX2-NEXT:    vpextrw $6, %xmm1, %ecx
-; AVX2-NEXT:    andl $1, %ecx
-; AVX2-NEXT:    addq %rdx, %rcx
-; AVX2-NEXT:    vpextrw $7, %xmm1, %esi
+; AVX2-NEXT:    vpextrw $5, %xmm1, %ecx
+; AVX2-NEXT:    movq %rcx, %rsi
 ; AVX2-NEXT:    andl $1, %esi
-; AVX2-NEXT:    addq %rcx, %rsi
+; AVX2-NEXT:    addq %r13, %rsi
+; AVX2-NEXT:    vpextrw $6, %xmm1, %eax
+; AVX2-NEXT:    movq %rax, %rcx
+; AVX2-NEXT:    andl $1, %ecx
+; AVX2-NEXT:    addq %rsi, %rcx
+; AVX2-NEXT:    vpextrw $7, %xmm1, %edx
+; AVX2-NEXT:    andl $1, %edx
+; AVX2-NEXT:    addq %rcx, %rdx
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpextrw $7, %xmm1, %eax
-; AVX2-NEXT:    cmpq $16, %rsi
+; AVX2-NEXT:    cmpq $16, %rdx
 ; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
 ; AVX2-NEXT:    cmovbw (%rsp,%rdi,2), %ax
 ; AVX2-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
@@ -2037,16 +2043,16 @@ define <16 x i16> @test_compress_v16i16(<16 x i16> %vec, <16 x i1> %mask, <16 x
 ; AVX2-NEXT:    vpextrw $2, %xmm0, (%rsp,%rdi,2)
 ; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
 ; AVX2-NEXT:    vpextrw $3, %xmm0, (%rsp,%rdi,2)
-; AVX2-NEXT:    vpextrw $4, %xmm0, (%rsp,%r8,2)
-; AVX2-NEXT:    andl $15, %r10d
-; AVX2-NEXT:    vpextrw $5, %xmm0, (%rsp,%r10,2)
-; AVX2-NEXT:    andl $15, %r11d
-; AVX2-NEXT:    vpextrw $6, %xmm0, (%rsp,%r11,2)
 ; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; AVX2-NEXT:    andl $15, %eax
-; AVX2-NEXT:    vpextrw $7, %xmm0, (%rsp,%rax,2)
+; AVX2-NEXT:    vpextrw $4, %xmm0, (%rsp,%rax,2)
 ; AVX2-NEXT:    andl $15, %r9d
-; AVX2-NEXT:    vpextrw $0, %xmm1, (%rsp,%r9,2)
+; AVX2-NEXT:    vpextrw $5, %xmm0, (%rsp,%r9,2)
+; AVX2-NEXT:    andl $15, %r10d
+; AVX2-NEXT:    vpextrw $6, %xmm0, (%rsp,%r10,2)
+; AVX2-NEXT:    andl $15, %r8d
+; AVX2-NEXT:    vpextrw $7, %xmm0, (%rsp,%r8,2)
+; AVX2-NEXT:    andl $15, %r11d
+; AVX2-NEXT:    vpextrw $0, %xmm1, (%rsp,%r11,2)
 ; AVX2-NEXT:    andl $15, %ebx
 ; AVX2-NEXT:    vpextrw $1, %xmm1, (%rsp,%rbx,2)
 ; AVX2-NEXT:    andl $15, %r14d
@@ -2057,13 +2063,13 @@ define <16 x i16> @test_compress_v16i16(<16 x i16> %vec, <16 x i1> %mask, <16 x
 ; AVX2-NEXT:    vpextrw $4, %xmm1, (%rsp,%r12,2)
 ; AVX2-NEXT:    andl $15, %r13d
 ; AVX2-NEXT:    vpextrw $5, %xmm1, (%rsp,%r13,2)
-; AVX2-NEXT:    andl $15, %edx
-; AVX2-NEXT:    vpextrw $6, %xmm1, (%rsp,%rdx,2)
+; AVX2-NEXT:    andl $15, %esi
+; AVX2-NEXT:    vpextrw $6, %xmm1, (%rsp,%rsi,2)
 ; AVX2-NEXT:    andl $15, %ecx
 ; AVX2-NEXT:    vpextrw $7, %xmm1, (%rsp,%rcx,2)
-; AVX2-NEXT:    cmpq $15, %rsi
+; AVX2-NEXT:    cmpq $15, %rdx
 ; AVX2-NEXT:    movl $15, %eax
-; AVX2-NEXT:    cmovbq %rsi, %rax
+; AVX2-NEXT:    cmovbq %rdx, %rax
 ; AVX2-NEXT:    andl $15, %eax
 ; AVX2-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %ecx # 4-byte Reload
 ; AVX2-NEXT:    movw %cx, (%rsp,%rax,2)
@@ -3378,123 +3384,149 @@ define <32 x i16> @test_compress_v32i16(<32 x i16> %vec, <32 x i1> %mask, <32 x
 ; AVX2-NEXT:    andl $31, %ecx
 ; AVX2-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX2-NEXT:    vpextrb $4, %xmm2, %eax
-; AVX2-NEXT:    vpextrb $1, %xmm2, %ecx
+; AVX2-NEXT:    vpextrb $1, %xmm2, %edx
+; AVX2-NEXT:    andl $1, %eax
+; AVX2-NEXT:    movq %rax, %rcx
+; AVX2-NEXT:    movq %rdx, %rax
 ; AVX2-NEXT:    andl $1, %eax
-; AVX2-NEXT:    andl $1, %ecx
 ; AVX2-NEXT:    vmovd %xmm2, %edx
 ; AVX2-NEXT:    andl $1, %edx
 ; AVX2-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT:    addq %rdx, %rcx
-; AVX2-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT:    addq %rdx, %rax
+; AVX2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX2-NEXT:    vpextrb $2, %xmm2, %edx
 ; AVX2-NEXT:    andl $1, %edx
-; AVX2-NEXT:    addq %rcx, %rdx
+; AVX2-NEXT:    addq %rax, %rdx
 ; AVX2-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT:    vpextrb $3, %xmm2, %ecx
-; AVX2-NEXT:    andl $1, %ecx
-; AVX2-NEXT:    addq %rdx, %rcx
-; AVX2-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT:    addq %rcx, %rax
+; AVX2-NEXT:    vpextrb $3, %xmm2, %eax
+; AVX2-NEXT:    andl $1, %eax
+; AVX2-NEXT:    addq %rdx, %rax
 ; AVX2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT:    vpextrb $5, %xmm2, %ecx
-; AVX2-NEXT:    andl $1, %ecx
 ; AVX2-NEXT:    addq %rax, %rcx
 ; AVX2-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT:    vpextrb $5, %xmm2, %eax
+; AVX2-NEXT:    andl $1, %eax
+; AVX2-NEXT:    addq %rcx, %rax
+; AVX2-NEXT:    movq %rax, %rcx
+; AVX2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX2-NEXT:    vpextrb $6, %xmm2, %eax
 ; AVX2-NEXT:    andl $1, %eax
 ; AVX2-NEXT:    addq %rcx, %rax
+; AVX2-NEXT:    movq %rax, %rcx
+; AVX2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT:    vpextrb $7, %xmm2, %eax
+; AVX2-NEXT:    andl $1, %eax
+; AVX2-NEXT:    addq %rcx, %rax
+; AVX2-NEXT:    movq %rax, %rcx
 ; AVX2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT:    vpextrb $7, %xmm2, %ecx
-; AVX2-NEXT:    andl $1, %ecx
-; AVX2-NEXT:    addq %rax, %rcx
-; AVX2-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX2-NEXT:    vpextrb $8, %xmm2, %eax
 ; AVX2-NEXT:    andl $1, %eax
 ; AVX2-NEXT:    addq %rcx, %rax
+; AVX2-NEXT:    movq %rax, %rcx
+; AVX2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT:    vpextrb $9, %xmm2, %eax
+; AVX2-NEXT:    andl $1, %eax
+; AVX2-NEXT:    addq %rcx, %rax
+; AVX2-NEXT:    movq %rax, %rcx
 ; AVX2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT:    vpextrb $9, %xmm2, %ecx
-; AVX2-NEXT:    andl $1, %ecx
-; AVX2-NEXT:    addq %rax, %rcx
-; AVX2-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX2-NEXT:    vpextrb $10, %xmm2, %eax
 ; AVX2-NEXT:    andl $1, %eax
 ; AVX2-NEXT:    addq %rcx, %rax
+; AVX2-NEXT:    movq %rax, %rcx
+; AVX2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT:    vpextrb $11, %xmm2, %eax
+; AVX2-NEXT:    andl $1, %eax
+; AVX2-NEXT:    addq %rcx, %rax
+; AVX2-NEXT:    movq %rax, %rcx
 ; AVX2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT:    vpextrb $11, %xmm2, %ecx
-; AVX2-NEXT:    andl $1, %ecx
-; AVX2-NEXT:    addq %rax, %rcx
-; AVX2-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX2-NEXT:    vpextrb $12, %xmm2, %eax
 ; AVX2-NEXT:    andl $1, %eax
 ; AVX2-NEXT:    addq %rcx, %rax
+; AVX2-NEXT:    movq %rax, %rcx
+; AVX2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT:    vpextrb $13, %xmm2, %eax
+; AVX2-NEXT:    andl $1, %eax
+; AVX2-NEXT:    addq %rcx, %rax
+; AVX2-NEXT:    movq %rax, %rcx
 ; AVX2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT:    vpextrb $13, %xmm2, %ecx
-; AVX2-NEXT:    andl $1, %ecx
-; AVX2-NEXT:    addq %rax, %rcx
-; AVX2-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX2-NEXT:    vpextrb $14, %xmm2, %eax
 ; AVX2-NEXT:    andl $1, %eax
 ; AVX2-NEXT:    addq %rcx, %rax
+; AVX2-NEXT:    movq %rax, %rcx
+; AVX2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT:    vpextrb $15, %xmm2, %eax
+; AVX2-NEXT:    andl $1, %eax
+; AVX2-NEXT:    addq %rcx, %rax
+; AVX2-NEXT:    movq %rax, %rcx
 ; AVX2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT:    vpextrb $15, %xmm2, %ecx
-; AVX2-NEXT:    andl $1, %ecx
-; AVX2-NEXT:    addq %rax, %rcx
-; AVX2-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX2-NEXT:    vmovd %xmm3, %eax
 ; AVX2-NEXT:    andl $1, %eax
 ; AVX2-NEXT:    addq %rcx, %rax
+; AVX2-NEXT:    movq %rax, %rcx
 ; AVX2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT:    vpextrb $1, %xmm3, %ecx
-; AVX2-NEXT:    andl $1, %ecx
-; AVX2-NEXT:    addq %rax, %rcx
-; AVX2-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT:    vpextrb $2, %xmm3, %eax
+; AVX2-NEXT:    vpextrb $1, %xmm3, %eax
 ; AVX2-NEXT:    andl $1, %eax
 ; AVX2-NEXT:    addq %rcx, %rax
 ; AVX2-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT:    vpextrb $3, %xmm3, %ecx
-; AVX2-NEXT:    andl $1, %ecx
-; AVX2-NEXT:    addq %rax, %rcx
-; AVX2-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT:    vpextrb $2, %xmm3, %r13d
+; AVX2-NEXT:    andl $1, %r13d
+; AVX2-NEXT:    addq %rax, %r13
+; AVX2-NEXT:    movq %r13, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT:    vpextrb $3, %xmm3, %r12d
+; AVX2-NEXT:    andl $1, %r12d
+; AVX2-NEXT:    addq %r13, %r12
+; AVX2-NEXT:    movq %r12, %rax
+; AVX2-NEXT:    movq %r12, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; AVX2-NEXT:    vpextrb $4, %xmm3, %r15d
-; AVX2-NEXT:    andl $1, %r15d
-; AVX2-NEXT:    addq %rcx, %r15
+; AVX2-NEXT:    movq %r15, %r12
+; AVX2-NEXT:    andl $1, %r12d
+; AVX2-NEXT:    addq %rax, %r12
 ; AVX2-NEXT:    vpextrb $5, %xmm3, %r14d
+; AVX2-NEXT:    movq %r14, %r15
+; AVX2-NEXT:    andl $1, %r15d
+; AVX2-NEXT:    addq %r12, %r15
+; AVX2-NEXT:    vpextrb $6, %xmm3, %ebx
+; AVX2-NEXT:    movq %rbx, %r14
 ; AVX2-NEXT:    andl $1, %r14d
 ; AVX2-NEXT:    addq %r15, %r14
-; AVX2-NEXT:    vpextrb $6, %xmm3, %ebx
+; AVX2-NEXT:    vpextrb $7, %xmm3, %r11d
+; AVX2-NEXT:    movq %r11, %rbx
 ; AVX2-NEXT:    andl $1, %ebx
 ; AVX2-NEXT:    addq %r14, %rbx
-; AVX2-NEXT:    vpextrb $7, %xmm3, %r11d
+; AVX2-NEXT:    vpextrb $8, %xmm3, %r10d
+; AVX2-NEXT:    movq %r10, %r11
 ; AVX2-NEXT:    andl $1, %r11d
 ; AVX2-NEXT:    addq %rbx, %r11
-; AVX2-NEXT:    vpextrb $8, %xmm3, %r10d
+; AVX2-NEXT:    vpextrb $9, %xmm3, %r9d
+; AVX2-NEXT:    movq %r9, %r10
 ; AVX2-NEXT:    andl $1, %r10d
 ; AVX2-NEXT:    addq %r11, %r10
-; AVX2-NEXT:    vpextrb $9, %xmm3, %r9d
+; AVX2-NEXT:    vpextrb $10, %xmm3, %edi
+; AVX2-NEXT:    movq %rdi, %r9
 ; AVX2-NEXT:    andl $1, %r9d
 ; AVX2-NEXT:    addq %r10, %r9
-; AVX2-NEXT:    vpextrb $10, %xmm3, %r8d
-; AVX2-NEXT:    andl $1, %r8d
-; AVX2-NEXT:    addq %r9, %r8
-; AVX2-NEXT:    vpextrb $11, %xmm3, %edi
+; AVX2-NEXT:    vpextrb $11, %xmm3, %esi
+; AVX2-NEXT:    movq %rsi, %rdi
 ; AVX2-NEXT:    andl $1, %edi
-; AVX2-NEXT:    addq %r8, %rdi
-; AVX2-NEXT:    vpextrb $12, %xmm3, %esi
+; AVX2-NEXT:    addq %r9, %rdi
+; AVX2-NEXT:    vpextrb $12, %xmm3, %edx
+; AVX2-NEXT:    movq %rdx, %rsi
 ; AVX2-NEXT:    andl $1, %esi
 ; AVX2-NEXT:    addq %rdi, %rsi
-; AVX2-NEXT:    vpextrb $13, %xmm3, %edx
+; AVX2-NEXT:    vpextrb $13, %xmm3, %ecx
+; AVX2-NEXT:    movq %rcx, %rdx
 ; AVX2-NEXT:    andl $1, %edx
 ; AVX2-NEXT:    addq %rsi, %rdx
-; AVX2-NEXT:    vpextrb $14, %xmm3, %ecx
+; AVX2-NEXT:    vpextrb $14, %xmm3, %eax
+; AVX2-NEXT:    movq %rax, %rcx
 ; AVX2-NEXT:    andl $1, %ecx
 ; AVX2-NEXT:    addq %rdx, %rcx
-; AVX2-NEXT:    vpextrb $15, %xmm3, %r12d
-; AVX2-NEXT:    andl $1, %r12d
-; AVX2-NEXT:    addq %rcx, %r12
+; AVX2-NEXT:    vpextrb $15, %xmm3, %r8d
+; AVX2-NEXT:    andl $1, %r8d
+; AVX2-NEXT:    addq %rcx, %r8
 ; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; AVX2-NEXT:    vpextrw $7, %xmm2, %eax
-; AVX2-NEXT:    cmpq $32, %r12
+; AVX2-NEXT:    cmpq $32, %r8
 ; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Reload
 ; AVX2-NEXT:    cmovbw (%rsp,%r13,2), %ax
 ; AVX2-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
@@ -3557,20 +3589,20 @@ define <32 x i16> @test_compress_v32i16(<32 x i16> %vec, <32 x i1> %mask, <32 x
 ; AVX2-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
 ; AVX2-NEXT:    andl $31, %eax
 ; AVX2-NEXT:    vpextrw $4, %xmm1, (%rsp,%rax,2)
+; AVX2-NEXT:    andl $31, %r12d
+; AVX2-NEXT:    vpextrw $5, %xmm1, (%rsp,%r12,2)
 ; AVX2-NEXT:    andl $31, %r15d
-; AVX2-NEXT:    vpextrw $5, %xmm1, (%rsp,%r15,2)
+; AVX2-NEXT:    vpextrw $6, %xmm1, (%rsp,%r15,2)
 ; AVX2-NEXT:    andl $31, %r14d
-; AVX2-NEXT:    vpextrw $6, %xmm1, (%rsp,%r14,2)
+; AVX2-NEXT:    vpextrw $7, %xmm1, (%rsp,%r14,2)
 ; AVX2-NEXT:    andl $31, %ebx
-; AVX2-NEXT:    vpextrw $7, %xmm1, (%rsp,%rbx,2)
+; AVX2-NEXT:    vpextrw $0, %xmm2, (%rsp,%rbx,2)
 ; AVX2-NEXT:    andl $31, %r11d
-; AVX2-NEXT:    vpextrw $0, %xmm2, (%rsp,%r11,2)
+; AVX2-NEXT:    vpextrw $1, %xmm2, (%rsp,%r11,2)
 ; AVX2-NEXT:    andl $31, %r10d
-; AVX2-NEXT:    vpextrw $1, %xmm2, (%rsp,%r10,2)
+; AVX2-NEXT:    vpextrw $2, %xmm2, (%rsp,%r10,2)
 ; AVX2-NEXT:    andl $31, %r9d
-; AVX2-NEXT:    vpextrw $2, %xmm2, (%rsp,%r9,2)
-; AVX2-NEXT:    andl $31, %r8d
-; AVX2-NEXT:    vpextrw $3, %xmm2, (%rsp,%r8,2)
+; AVX2-NEXT:    vpextrw $3, %xmm2, (%rsp,%r9,2)
 ; AVX2-NEXT:    andl $31, %edi
 ; AVX2-NEXT:    vpextrw $4, %xmm2, (%rsp,%rdi,2)
 ; AVX2-NEXT:    andl $31, %esi
@@ -3579,9 +3611,9 @@ define <32 x i16> @test_compress_v32i16(<32 x i16> %vec, <32 x i1> %mask, <32 x
 ; AVX2-NEXT:    vpextrw $6, %xmm2, (%rsp,%rdx,2)
 ; AVX2-NEXT:    andl $31, %ecx
 ; AVX2-NEXT:    vpextrw $7, %xmm2, (%rsp,%rcx,2)
-; AVX2-NEXT:    cmpq $31, %r12
+; AVX2-NEXT:    cmpq $31, %r8
 ; AVX2-NEXT:    movl $31, %eax
-; AVX2-NEXT:    cmovbq %r12, %rax
+; AVX2-NEXT:    cmovbq %r8, %rax
 ; AVX2-NEXT:    addl %eax, %eax
 ; AVX2-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %ecx # 4-byte Reload
 ; AVX2-NEXT:    movw %cx, (%rsp,%rax)
@@ -4607,19 +4639,19 @@ define <8 x i64> @test_compress_knownbits_zext_v8i16_8i64(<8 x i16> %vec, <8 x
 ; AVX2-NEXT:    vmovd %xmm1, %esi
 ; AVX2-NEXT:    andl $1, %esi
 ; AVX2-NEXT:    addq %rsi, %rdx
-; AVX2-NEXT:    vpextrw $2, %xmm1, %r8d
+; AVX2-NEXT:    vpextrw $2, %xmm1, %edi
+; AVX2-NEXT:    andl $1, %edi
+; AVX2-NEXT:    addq %rdx, %rdi
+; AVX2-NEXT:    vpextrw $3, %xmm1, %r8d
 ; AVX2-NEXT:    andl $1, %r8d
-; AVX2-NEXT:    addq %rdx, %r8
-; AVX2-NEXT:    vpextrw $3, %xmm1, %r9d
+; AVX2-NEXT:    addq %rdi, %r8
+; AVX2-NEXT:    addq %r8, %rax
+; AVX2-NEXT:    vpextrw $5, %xmm1, %r9d
 ; AVX2-NEXT:    andl $1, %r9d
-; AVX2-NEXT:    addq %r8, %r9
-; AVX2-NEXT:    addq %r9, %rax
-; AVX2-NEXT:    vpextrw $5, %xmm1, %edi
-; AVX2-NEXT:    andl $1, %edi
-; AVX2-NEXT:    addq %rax, %rdi
+; AVX2-NEXT:    addq %rax, %r9
 ; AVX2-NEXT:    vpextrw $6, %xmm1, %r10d
 ; AVX2-NEXT:    andl $1, %r10d
-; AVX2-NEXT:    addq %rdi, %r10
+; AVX2-NEXT:    addq %r9, %r10
 ; AVX2-NEXT:    vpextrw $7, %xmm1, %r11d
 ; AVX2-NEXT:    andl $1, %r11d
 ; AVX2-NEXT:    addq %r10, %r11
@@ -4632,13 +4664,13 @@ define <8 x i64> @test_compress_knownbits_zext_v8i16_8i64(<8 x i16> %vec, <8 x
 ; AVX2-NEXT:    vpextrq $1, %xmm0, (%rsp,%rsi,8)
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, (%rsp,%rdx,8)
-; AVX2-NEXT:    vpextrq $1, %xmm0, (%rsp,%r8,8)
-; AVX2-NEXT:    andl $7, %r9d
-; AVX2-NEXT:    vmovq %xmm4, (%rsp,%r9,8)
+; AVX2-NEXT:    vpextrq $1, %xmm0, (%rsp,%rdi,8)
+; AVX2-NEXT:    andl $7, %r8d
+; AVX2-NEXT:    vmovq %xmm4, (%rsp,%r8,8)
 ; AVX2-NEXT:    andl $7, %eax
 ; AVX2-NEXT:    vpextrq $1, %xmm4, (%rsp,%rax,8)
-; AVX2-NEXT:    andl $7, %edi
-; AVX2-NEXT:    vmovq %xmm1, (%rsp,%rdi,8)
+; AVX2-NEXT:    andl $7, %r9d
+; AVX2-NEXT:    vmovq %xmm1, (%rsp,%r9,8)
 ; AVX2-NEXT:    andl $7, %r10d
 ; AVX2-NEXT:    vpextrq $1, %xmm1, (%rsp,%r10,8)
 ; AVX2-NEXT:    cmpq $7, %r11
@@ -4711,19 +4743,19 @@ define <8 x i64> @test_compress_knownbits_sext_v8i16_8i64(<8 x i16> %vec, <8 x i
 ; AVX2-NEXT:    vmovd %xmm1, %esi
 ; AVX2-NEXT:    andl $1, %esi
 ; AVX2-NEXT:    addq %rsi, %rdx
-; AVX2-NEXT:    vpextrw $2, %xmm1, %r8d
+; AVX2-NEXT:    vpextrw $2, %xmm1, %edi
+; AVX2-NEXT:    andl $1, %edi
+; AVX2-NEXT:    addq %rdx, %rdi
+; AVX2-NEXT:    vpextrw $3, %xmm1, %r8d
 ; AVX2-NEXT:    andl $1, %r8d
-; AVX2-NEXT:    addq %rdx, %r8
-; AVX2-NEXT:    vpextrw $3, %xmm1, %r9d
+; AVX2-NEXT:    addq %rdi, %r8
+; AVX2-NEXT:    addq %r8, %rax
+; AVX2-NEXT:    vpextrw $5, %xmm1, %r9d
 ; AVX2-NEXT:    andl $1, %r9d
-; AVX2-NEXT:    addq %r8, %r9
-; AVX2-NEXT:    addq %r9, %rax
-; AVX2-NEXT:    vpextrw $5, %xmm1, %edi
-; AVX2-NEXT:    andl $1, %edi
-; AVX2-NEXT:    addq %rax, %rdi
+; AVX2-NEXT:    addq %rax, %r9
 ; AVX2-NEXT:    vpextrw $6, %xmm1, %r10d
 ; AVX2-NEXT:    andl $1, %r10d
-; AVX2-NEXT:    addq %rdi, %r10
+; AVX2-NEXT:    addq %r9, %r10
 ; AVX2-NEXT:    vpextrw $7, %xmm1, %r11d
 ; AVX2-NEXT:    andl $1, %r11d
 ; AVX2-NEXT:    addq %r10, %r11
@@ -4736,13 +4768,13 @@ define <8 x i64> @test_compress_knownbits_sext_v8i16_8i64(<8 x i16> %vec, <8 x i
 ; AVX2-NEXT:    vpextrq $1, %xmm0, (%rsp,%rsi,8)
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, (%rsp,%rdx,8)
-; AVX2-NEXT:    vpextrq $1, %xmm0, (%rsp,%r8,8)
-; AVX2-NEXT:    andl $7, %r9d
-; AVX2-NEXT:    vmovq %xmm4, (%rsp,%r9,8)
+; AVX2-NEXT:    vpextrq $1, %xmm0, (%rsp,%rdi,8)
+; AVX2-NEXT:    andl $7, %r8d
+; AVX2-NEXT:    vmovq %xmm4, (%rsp,%r8,8)
 ; AVX2-NEXT:    andl $7, %eax
 ; AVX2-NEXT:    vpextrq $1, %xmm4, (%rsp,%rax,8)
-; AVX2-NEXT:    andl $7, %edi
-; AVX2-NEXT:    vmovq %xmm1, (%rsp,%rdi,8)
+; AVX2-NEXT:    andl $7, %r9d
+; AVX2-NEXT:    vmovq %xmm1, (%rsp,%r9,8)
 ; AVX2-NEXT:    andl $7, %r10d
 ; AVX2-NEXT:    vpextrq $1, %xmm1, (%rsp,%r10,8)
 ; AVX2-NEXT:    cmpq $7, %r11



More information about the llvm-branch-commits mailing list