[llvm-branch-commits] [llvm] [SelectionDAG] Select the FREEZE target opcode (PR #240476)
Mitch Briles via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Mon Nov 9 09:46:07 PST 2026
https://github.com/MitchBriles created https://github.com/llvm/llvm-project/pull/240476
Selecting freeze as a COPY is wrong if the value may be undefined. Now we can
select ISD::FREEZE as FREEZE instead. Targets that don't have physical registers
can keep selecting COPY since there is no regalloc to observe the difference.
Fixes #58321, #60429, #144780.
Assisted-By: Claude Opus 5.5 <noreply at anthropic.com>
>From e75dc4fcdd2a7c7b1697eba102bba3e61cb1d365 Mon Sep 17 00:00:00 2001
From: Mitch <mitchbriles at gmail.com>
Date: Fri, 23 Oct 2026 06:40:58 -0600
Subject: [PATCH 1/2] Precommit miscompiling FREEZE tests
Co-Authored-By: Claude Opus 5.5 <noreply at anthropic.com>
---
llvm/test/CodeGen/AArch64/freeze-multi-use.ll | 117 +++++++++++++++
llvm/test/CodeGen/AArch64/freeze.ll | 14 ++
llvm/test/CodeGen/AMDGPU/freeze-multi-use.ll | 94 ++++++++++++
llvm/test/CodeGen/RISCV/freeze-multi-use.ll | 137 ++++++++++++++++++
.../CodeGen/RISCV/rvv/freeze-multi-use.ll | 43 ++++++
llvm/test/CodeGen/X86/freeze-multi-use.ll | 85 +++++++++++
llvm/test/CodeGen/X86/freeze.ll | 8 +
7 files changed, 498 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/freeze-multi-use.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/freeze-multi-use.ll
create mode 100644 llvm/test/CodeGen/RISCV/freeze-multi-use.ll
create mode 100644 llvm/test/CodeGen/RISCV/rvv/freeze-multi-use.ll
create mode 100644 llvm/test/CodeGen/X86/freeze-multi-use.ll
diff --git a/llvm/test/CodeGen/AArch64/freeze-multi-use.ll b/llvm/test/CodeGen/AArch64/freeze-multi-use.ll
new file mode 100644
index 00000000000000..822ec50d0a5717
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/freeze-multi-use.ll
@@ -0,0 +1,117 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=aarch64-- < %s | FileCheck %s
+
+; Every use of a frozen value must observe the same value.
+
+; The multiply and the shift must read the same value.
+define i32 @ashr_mul(i32 %x) {
+; CHECK-LABEL: ashr_mul:
+; CHECK: // %bb.0:
+; CHECK-NEXT: asr w8, w8, #3
+; CHECK-NEXT: mul w0, w8, w8
+; CHECK-NEXT: ret
+ %p = ashr i32 %x, 32
+ %w = freeze i32 %p
+ %s = ashr i32 %w, 3
+ %r = mul i32 %w, %s
+ ret i32 %r
+}
+
+; Both halves of the funnel shift must use the same shift amount.
+define i8 @fshr(i32 %x) {
+; CHECK-LABEL: fshr:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mvn w8, w8
+; CHECK-NEXT: mov w9, #2 // =0x2
+; CHECK-NEXT: lsr w10, w0, w8
+; CHECK-NEXT: lsl w8, w9, w8
+; CHECK-NEXT: orr w0, w8, w10
+; CHECK-NEXT: ret
+ %w = freeze i32 poison
+ %f = call i32 @llvm.fshr.i32(i32 1, i32 %x, i32 %w)
+ %r = trunc i32 %f to i8
+ ret i8 %r
+}
+
+; The select condition and the zext must read the same bit.
+define i64 @select_zext(i64 %a, i64 %b) {
+; CHECK-LABEL: select_zext:
+; CHECK: // %bb.0:
+; CHECK-NEXT: tst w8, #0x1
+; CHECK-NEXT: csel x8, x0, x1, ne
+; CHECK-NEXT: and x9, x8, #0x1
+; CHECK-NEXT: add x0, x8, x9
+; CHECK-NEXT: ret
+ %w = freeze i1 poison
+ %s = select i1 %w, i64 %a, i64 %b
+ %z = zext i1 %w to i64
+ %r = add i64 %s, %z
+ ret i64 %r
+}
+
+; The fcmp and the bitcast must see the same float, so the result is 0.
+define i32 @float_bits() {
+; CHECK-LABEL: float_bits:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fcmp s0, s0
+; CHECK-NEXT: and w8, w8, #0x7fffffff
+; CHECK-NEXT: mov w9, #2139095040 // =0x7f800000
+; CHECK-NEXT: cset w10, vs
+; CHECK-NEXT: cmp w8, w9
+; CHECK-NEXT: cset w8, hi
+; CHECK-NEXT: eor w0, w10, w8
+; CHECK-NEXT: ret
+ %w = freeze float poison
+ %c = fcmp uno float %w, 0.0
+ %b = bitcast float %w to i32
+ %e = and i32 %b, 2147483647
+ %n = icmp ugt i32 %e, 2139095040
+ %x = xor i1 %c, %n
+ %r = zext i1 %x to i32
+ ret i32 %r
+}
+
+; The result must be element 0 of %x.
+define i32 @vector_lane_and_whole(<4 x i32> %x) {
+; CHECK-LABEL: vector_lane_and_whole:
+; CHECK: // %bb.0:
+; CHECK-NEXT: add v0.4s, v0.4s, v0.4s
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: sub w0, w8, w8
+; CHECK-NEXT: ret
+ %w = freeze <4 x i32> poison
+ %e0 = extractelement <4 x i32> %w, i32 0
+ %y = add <4 x i32> %w, %x
+ %e1 = extractelement <4 x i32> %y, i32 0
+ %r = sub i32 %e1, %e0
+ ret i32 %r
+}
+
+; The returned value must be the one passed to @use.
+define i32 @across_call() nounwind {
+; CHECK-LABEL: across_call:
+; CHECK: // %bb.0:
+; CHECK-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: bl use
+; CHECK-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+ %w = freeze i32 poison
+ call void @use(i32 %w)
+ ret i32 %w
+}
+
+; The returned value must be the one passed to @usev.
+define <4 x i32> @vec_across_call() nounwind {
+; CHECK-LABEL: vec_across_call:
+; CHECK: // %bb.0:
+; CHECK-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: bl usev
+; CHECK-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+ %w = freeze <4 x i32> poison
+ call void @usev(<4 x i32> %w)
+ ret <4 x i32> %w
+}
+
+declare void @use(i32)
+declare void @usev(<4 x i32>)
diff --git a/llvm/test/CodeGen/AArch64/freeze.ll b/llvm/test/CodeGen/AArch64/freeze.ll
index a660c52c567ede..fff659e1f21b9c 100644
--- a/llvm/test/CodeGen/AArch64/freeze.ll
+++ b/llvm/test/CodeGen/AArch64/freeze.ll
@@ -588,3 +588,17 @@ define i32 @freeze_ucmp(i32 %a0) nounwind {
%z = call i32 @llvm.ucmp.i32(i32 %y, i32 1)
ret i32 %z
}
+
+define i128 @freeze_i128() {
+; CHECK-SD-LABEL: freeze_i128:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: freeze_i128:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mov d0, v0.d[1]
+; CHECK-GI-NEXT: fmov x1, d0
+; CHECK-GI-NEXT: ret
+ %y1 = freeze i128 poison
+ ret i128 %y1
+}
diff --git a/llvm/test/CodeGen/AMDGPU/freeze-multi-use.ll b/llvm/test/CodeGen/AMDGPU/freeze-multi-use.ll
new file mode 100644
index 00000000000000..08422bf946c3b3
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/freeze-multi-use.ll
@@ -0,0 +1,94 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgpu11.00-amd-amdhsa < %s | FileCheck %s
+
+; Every use of a frozen value must observe the same value.
+
+; The multiply and the shift must read the same value.
+define i32 @ashr_mul(i32 %x) {
+; CHECK-LABEL: ashr_mul:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: s_ashr_i32 s0, s0, 3
+; CHECK-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; CHECK-NEXT: s_mul_i32 s0, s0, s0
+; CHECK-NEXT: v_mov_b32_e32 v0, s0
+; CHECK-NEXT: s_setpc_b64 s[30:31]
+ %p = ashr i32 %x, 32
+ %w = freeze i32 %p
+ %s = ashr i32 %w, 3
+ %r = mul i32 %w, %s
+ ret i32 %r
+}
+
+; The returned value must be the one passed to @use.
+define i32 @across_call() nounwind {
+; CHECK-LABEL: across_call:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: s_mov_b32 s0, s33
+; CHECK-NEXT: s_mov_b32 s33, s32
+; CHECK-NEXT: s_or_saveexec_b32 s1, -1
+; CHECK-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
+; CHECK-NEXT: s_mov_b32 exec_lo, s1
+; CHECK-NEXT: v_writelane_b32 v40, s0, 2
+; CHECK-NEXT: s_add_i32 s32, s32, 16
+; CHECK-NEXT: v_writelane_b32 v40, s30, 0
+; CHECK-NEXT: v_writelane_b32 v40, s31, 1
+; CHECK-NEXT: s_getpc_b64 s[0:1]
+; CHECK-NEXT: s_add_u32 s0, s0, use at gotpcrel32@lo+4
+; CHECK-NEXT: s_addc_u32 s1, s1, use at gotpcrel32@hi+12
+; CHECK-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: s_swappc_b64 s[30:31], s[0:1]
+; CHECK-NEXT: v_readlane_b32 s30, v40, 0
+; CHECK-NEXT: v_readlane_b32 s31, v40, 1
+; CHECK-NEXT: s_mov_b32 s32, s33
+; CHECK-NEXT: v_readlane_b32 s0, v40, 2
+; CHECK-NEXT: s_or_saveexec_b32 s1, -1
+; CHECK-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
+; CHECK-NEXT: s_mov_b32 exec_lo, s1
+; CHECK-NEXT: s_mov_b32 s33, s0
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: s_setpc_b64 s[30:31]
+ %w = freeze i32 poison
+ call void @use(i32 %w)
+ ret i32 %w
+}
+
+; The returned value must be the one passed to @usev.
+define <4 x i32> @vec_across_call() nounwind {
+; CHECK-LABEL: vec_across_call:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: s_mov_b32 s0, s33
+; CHECK-NEXT: s_mov_b32 s33, s32
+; CHECK-NEXT: s_or_saveexec_b32 s1, -1
+; CHECK-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
+; CHECK-NEXT: s_mov_b32 exec_lo, s1
+; CHECK-NEXT: v_writelane_b32 v40, s0, 2
+; CHECK-NEXT: s_add_i32 s32, s32, 16
+; CHECK-NEXT: v_writelane_b32 v40, s30, 0
+; CHECK-NEXT: v_writelane_b32 v40, s31, 1
+; CHECK-NEXT: s_getpc_b64 s[0:1]
+; CHECK-NEXT: s_add_u32 s0, s0, usev at gotpcrel32@lo+4
+; CHECK-NEXT: s_addc_u32 s1, s1, usev at gotpcrel32@hi+12
+; CHECK-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: s_swappc_b64 s[30:31], s[0:1]
+; CHECK-NEXT: v_readlane_b32 s30, v40, 0
+; CHECK-NEXT: v_readlane_b32 s31, v40, 1
+; CHECK-NEXT: s_mov_b32 s32, s33
+; CHECK-NEXT: v_readlane_b32 s0, v40, 2
+; CHECK-NEXT: s_or_saveexec_b32 s1, -1
+; CHECK-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
+; CHECK-NEXT: s_mov_b32 exec_lo, s1
+; CHECK-NEXT: s_mov_b32 s33, s0
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: s_setpc_b64 s[30:31]
+ %w = freeze <4 x i32> poison
+ call void @usev(<4 x i32> %w)
+ ret <4 x i32> %w
+}
+
+declare void @use(i32)
+declare void @usev(<4 x i32>)
diff --git a/llvm/test/CodeGen/RISCV/freeze-multi-use.ll b/llvm/test/CodeGen/RISCV/freeze-multi-use.ll
new file mode 100644
index 00000000000000..18b76447406603
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/freeze-multi-use.ll
@@ -0,0 +1,137 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=riscv32 < %s | FileCheck %s --check-prefixes=CHECK,RV32
+; RUN: llc -mtriple=riscv64 < %s | FileCheck %s --check-prefixes=CHECK,RV64
+
+; Every use of a frozen value must observe the same value.
+
+; The result must have the same parity as %n.
+define i32 @add_twice(i32 %n) {
+; RV32-LABEL: add_twice:
+; RV32: # %bb.0:
+; RV32-NEXT: add a0, a0, a0
+; RV32-NEXT: add a0, a0, a0
+; RV32-NEXT: ret
+;
+; RV64-LABEL: add_twice:
+; RV64: # %bb.0:
+; RV64-NEXT: add a0, a0, a0
+; RV64-NEXT: addw a0, a0, a0
+; RV64-NEXT: ret
+ %w = freeze i32 poison
+ %a = add i32 %n, %w
+ %r = add i32 %a, %w
+ ret i32 %r
+}
+
+; The result must be %x.
+define i32 @cross_block(i1 %c, i32 %x) {
+; RV32-LABEL: cross_block:
+; RV32: # %bb.0: # %entry
+; RV32-NEXT: andi a0, a0, 1
+; RV32-NEXT: beqz a0, .LBB1_2
+; RV32-NEXT: # %bb.1: # %t
+; RV32-NEXT: add a0, a0, a1
+; RV32-NEXT: sub a0, a0, a0
+; RV32-NEXT: ret
+; RV32-NEXT: .LBB1_2: # %e
+; RV32-NEXT: mv a0, a1
+; RV32-NEXT: ret
+;
+; RV64-LABEL: cross_block:
+; RV64: # %bb.0: # %entry
+; RV64-NEXT: andi a0, a0, 1
+; RV64-NEXT: beqz a0, .LBB1_2
+; RV64-NEXT: # %bb.1: # %t
+; RV64-NEXT: add a0, a0, a1
+; RV64-NEXT: subw a0, a0, a0
+; RV64-NEXT: ret
+; RV64-NEXT: .LBB1_2: # %e
+; RV64-NEXT: mv a0, a1
+; RV64-NEXT: ret
+entry:
+ %w = freeze i32 poison
+ %a = add i32 %w, %x
+ br i1 %c, label %t, label %e
+t:
+ %s = sub i32 %a, %w
+ ret i32 %s
+e:
+ ret i32 %x
+}
+
+; The fcmp and the bitcast must see the same float, so the result is 0.
+define i32 @float_bits() {
+; RV32-LABEL: float_bits:
+; RV32: # %bb.0:
+; RV32-NEXT: addi sp, sp, -16
+; RV32-NEXT: .cfi_def_cfa_offset 16
+; RV32-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32-NEXT: .cfi_offset ra, -4
+; RV32-NEXT: call __unordsf2
+; RV32-NEXT: slli a1, a0, 1
+; RV32-NEXT: srli a1, a1, 1
+; RV32-NEXT: lui a2, 522240
+; RV32-NEXT: snez a0, a0
+; RV32-NEXT: sltu a1, a2, a1
+; RV32-NEXT: xor a0, a0, a1
+; RV32-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32-NEXT: .cfi_restore ra
+; RV32-NEXT: addi sp, sp, 16
+; RV32-NEXT: .cfi_def_cfa_offset 0
+; RV32-NEXT: ret
+;
+; RV64-LABEL: float_bits:
+; RV64: # %bb.0:
+; RV64-NEXT: addi sp, sp, -16
+; RV64-NEXT: .cfi_def_cfa_offset 16
+; RV64-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
+; RV64-NEXT: .cfi_offset ra, -8
+; RV64-NEXT: call __unordsf2
+; RV64-NEXT: slli a1, a0, 33
+; RV64-NEXT: srli a1, a1, 33
+; RV64-NEXT: lui a2, 522240
+; RV64-NEXT: snez a0, a0
+; RV64-NEXT: sltu a1, a2, a1
+; RV64-NEXT: xor a0, a0, a1
+; RV64-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
+; RV64-NEXT: .cfi_restore ra
+; RV64-NEXT: addi sp, sp, 16
+; RV64-NEXT: .cfi_def_cfa_offset 0
+; RV64-NEXT: ret
+ %w = freeze float poison
+ %c = fcmp uno float %w, 0.0
+ %b = bitcast float %w to i32
+ %e = and i32 %b, 2147483647
+ %n = icmp ugt i32 %e, 2139095040
+ %x = xor i1 %c, %n
+ %r = zext i1 %x to i32
+ ret i32 %r
+}
+
+; The returned value must be the one passed to @use.
+define i32 @across_call() nounwind {
+; RV32-LABEL: across_call:
+; RV32: # %bb.0:
+; RV32-NEXT: addi sp, sp, -16
+; RV32-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32-NEXT: call use
+; RV32-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32-NEXT: addi sp, sp, 16
+; RV32-NEXT: ret
+;
+; RV64-LABEL: across_call:
+; RV64: # %bb.0:
+; RV64-NEXT: addi sp, sp, -16
+; RV64-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
+; RV64-NEXT: call use
+; RV64-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
+; RV64-NEXT: addi sp, sp, 16
+; RV64-NEXT: ret
+ %w = freeze i32 poison
+ call void @use(i32 %w)
+ ret i32 %w
+}
+
+declare void @use(i32)
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
diff --git a/llvm/test/CodeGen/RISCV/rvv/freeze-multi-use.ll b/llvm/test/CodeGen/RISCV/rvv/freeze-multi-use.ll
new file mode 100644
index 00000000000000..71d79b5f18b4ef
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/freeze-multi-use.ll
@@ -0,0 +1,43 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=riscv64 -mattr=+v < %s | FileCheck %s
+
+; Every use of a frozen value must observe the same value.
+
+; Element 1 is poison before the freeze. The result must be 0.
+define i32 @partial_vector(i32 %x) {
+; CHECK-LABEL: partial_vector:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT: vmv.v.x v8, a0
+; CHECK-NEXT: vadd.vx v8, v8, a0
+; CHECK-NEXT: vslidedown.vi v8, v8, 1
+; CHECK-NEXT: vmv.x.s a0, v8
+; CHECK-NEXT: add a1, a0, a0
+; CHECK-NEXT: subw a0, a0, a1
+; CHECK-NEXT: ret
+ %v = insertelement <4 x i32> poison, i32 %x, i32 0
+ %w = freeze <4 x i32> %v
+ %y = add <4 x i32> %w, %w
+ %e = extractelement <4 x i32> %w, i32 1
+ %e2 = extractelement <4 x i32> %y, i32 1
+ %r = sub i32 %e2, %e
+ %q = sub i32 %r, %e
+ ret i32 %q
+}
+
+; The returned value must be the one passed to @usev.
+define <4 x i32> @vec_across_call() nounwind {
+; CHECK-LABEL: vec_across_call:
+; CHECK: # %bb.0:
+; CHECK-NEXT: addi sp, sp, -16
+; CHECK-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT: call usev
+; CHECK-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT: addi sp, sp, 16
+; CHECK-NEXT: ret
+ %w = freeze <4 x i32> poison
+ call void @usev(<4 x i32> %w)
+ ret <4 x i32> %w
+}
+
+declare void @usev(<4 x i32>)
diff --git a/llvm/test/CodeGen/X86/freeze-multi-use.ll b/llvm/test/CodeGen/X86/freeze-multi-use.ll
new file mode 100644
index 00000000000000..08bee89cec6670
--- /dev/null
+++ b/llvm/test/CodeGen/X86/freeze-multi-use.ll
@@ -0,0 +1,85 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=x86_64-- < %s | FileCheck %s
+
+; Every use of a frozen value must observe the same value.
+
+; The result must have the same parity as %n.
+define i32 @add_twice(i32 %n) {
+; CHECK-LABEL: add_twice:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl %edi, %eax
+; CHECK-NEXT: addl %eax, %eax
+; CHECK-NEXT: addl %eax, %eax
+; CHECK-NEXT: retq
+ %w = freeze i32 poison
+ %a = add i32 %n, %w
+ %r = add i32 %a, %w
+ ret i32 %r
+}
+
+; The multiply and the shift must read the same value.
+define i32 @ashr_mul(i32 %x) {
+; CHECK-LABEL: ashr_mul:
+; CHECK: # %bb.0:
+; CHECK-NEXT: sarl $3, %eax
+; CHECK-NEXT: imull %eax, %eax
+; CHECK-NEXT: retq
+ %p = ashr i32 %x, 32
+ %w = freeze i32 %p
+ %s = ashr i32 %w, 3
+ %r = mul i32 %w, %s
+ ret i32 %r
+}
+
+; The result must be %x.
+define i32 @cross_block(i1 %c, i32 %x) {
+; CHECK-LABEL: cross_block:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: testb $1, %dil
+; CHECK-NEXT: je .LBB2_2
+; CHECK-NEXT: # %bb.1: # %t
+; CHECK-NEXT: addl %esi, %eax
+; CHECK-NEXT: subl %eax, %eax
+; CHECK-NEXT: retq
+; CHECK-NEXT: .LBB2_2: # %e
+; CHECK-NEXT: movl %esi, %eax
+; CHECK-NEXT: retq
+entry:
+ %w = freeze i32 poison
+ %a = add i32 %w, %x
+ br i1 %c, label %t, label %e
+t:
+ %s = sub i32 %a, %w
+ ret i32 %s
+e:
+ ret i32 %x
+}
+
+; The returned value must be the one passed to @use.
+define i32 @across_call() nounwind {
+; CHECK-LABEL: across_call:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pushq %rax
+; CHECK-NEXT: callq use at PLT
+; CHECK-NEXT: popq %rax
+; CHECK-NEXT: retq
+ %w = freeze i32 poison
+ call void @use(i32 %w)
+ ret i32 %w
+}
+
+; The returned value must be the one passed to @usev.
+define <4 x i32> @vec_across_call() nounwind {
+; CHECK-LABEL: vec_across_call:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pushq %rax
+; CHECK-NEXT: callq usev at PLT
+; CHECK-NEXT: popq %rax
+; CHECK-NEXT: retq
+ %w = freeze <4 x i32> poison
+ call void @usev(<4 x i32> %w)
+ ret <4 x i32> %w
+}
+
+declare void @use(i32)
+declare void @usev(<4 x i32>)
diff --git a/llvm/test/CodeGen/X86/freeze.ll b/llvm/test/CodeGen/X86/freeze.ll
index 5ea78f884e9803..2609b2fafac54c 100644
--- a/llvm/test/CodeGen/X86/freeze.ll
+++ b/llvm/test/CodeGen/X86/freeze.ll
@@ -203,3 +203,11 @@ entry:
%and = and i64 %sel2, %shl
ret i64 %and
}
+
+define i128 @freeze_i128() {
+; X86ASM-LABEL: freeze_i128:
+; X86ASM: # %bb.0:
+; X86ASM-NEXT: retq
+ %y1 = freeze i128 poison
+ ret i128 %y1
+}
>From d1309cba27c746d6cacc9dca203a6c8e90fb5e59 Mon Sep 17 00:00:00 2001
From: Mitch <mitchbriles at gmail.com>
Date: Fri, 23 Oct 2026 15:19:06 -0600
Subject: [PATCH 2/2] [SelectionDAG] Select the FREEZE target opcode
Selecting freeze as a COPY is wrong if the value may be undefined.
Now we can select ISD::FREEZE as FREEZE instead. Targets that don't
have physical registers can keep selecting COPY since there is no
regalloc to observe the difference.
Fixes #58321, #60429, #144780.
Assisted-By: Claude Opus 5.5 <noreply at anthropic.com>
---
llvm/docs/CodeGenerator.md | 4 +-
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 22 +-
.../CodeGen/SelectionDAG/SelectionDAGISel.cpp | 10 +-
.../AArch64/concat-vector-add-combine.ll | 1 +
llvm/test/CodeGen/AArch64/fma-fneg-combine.ll | 6 +-
llvm/test/CodeGen/AArch64/freeze-multi-use.ll | 51 +-
llvm/test/CodeGen/AArch64/freeze.ll | 279 +-
.../AArch64/neon-shuffle-vector-tbl.ll | 70 +-
llvm/test/CodeGen/AArch64/vector-compress.ll | 12 +-
.../AArch64/vector-extract-last-active.ll | 70 +-
.../AMDGPU/agpr-copy-no-free-registers.ll | 14 +-
llvm/test/CodeGen/AMDGPU/andorbitset.ll | 7 +-
.../test/CodeGen/AMDGPU/carryout-selection.ll | 40 +-
.../AMDGPU/divergence-driven-trunc-to-i1.ll | 48 +-
.../AMDGPU/float-to-arbitrary-fp-fp8-hw.ll | 22 +-
llvm/test/CodeGen/AMDGPU/fold-fabs.ll | 1 +
llvm/test/CodeGen/AMDGPU/freeze-multi-use.ll | 47 +-
llvm/test/CodeGen/AMDGPU/freeze.ll | 39 +-
.../CodeGen/AMDGPU/llvm.amdgcn.permlane.ll | 3806 +++++++++++------
.../AMDGPU/llvm.amdgcn.permlane16.var.ll | 80 +-
llvm/test/CodeGen/AMDGPU/mad_64_32.ll | 4 +-
.../CodeGen/AMDGPU/mdt-preserving-crash.ll | 3 +-
llvm/test/CodeGen/AMDGPU/mul_int24.ll | 8 +-
llvm/test/CodeGen/AMDGPU/permute_i8.ll | 24 +-
...-alloca-vector-dynamic-idx-bitcasts-llc.ll | 638 ++-
.../AMDGPU/promote-alloca-vector-to-vector.ll | 5 +-
llvm/test/CodeGen/AMDGPU/sad.ll | 7 +-
llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll | 12 +-
llvm/test/CodeGen/AMDGPU/select-undef.ll | 2 +
.../AMDGPU/splitkit-getsubrangeformask.ll | 7 +-
llvm/test/CodeGen/AMDGPU/udiv64.ll | 3 +-
llvm/test/CodeGen/Hexagon/pr196966.ll | 33 +-
llvm/test/CodeGen/PowerPC/sms-grp-order.ll | 15 +-
.../CodeGen/PowerPC/widen-vec-correctly-be.ll | 6 +-
llvm/test/CodeGen/RISCV/freeze-multi-use.ll | 60 +-
.../CodeGen/RISCV/rvv/freeze-multi-use.ll | 24 +-
llvm/test/CodeGen/SystemZ/pr60413.ll | 10 +-
llvm/test/CodeGen/X86/bit-manip-i256.ll | 37 +-
llvm/test/CodeGen/X86/combine-urem.ll | 12 +-
llvm/test/CodeGen/X86/freeze-binary.ll | 6 +
llvm/test/CodeGen/X86/freeze-multi-use.ll | 33 +-
llvm/test/CodeGen/X86/freeze-vector.ll | 26 +-
llvm/test/CodeGen/X86/freeze.ll | 28 +-
llvm/test/CodeGen/X86/pr169485.ll | 5 +-
llvm/test/CodeGen/X86/pr57402.ll | 18 +-
llvm/test/CodeGen/X86/pr57554.ll | 1 +
llvm/test/CodeGen/X86/var-permute-256.ll | 7 +-
.../CodeGen/X86/vector-compress-freeze.ll | 5 +-
llvm/test/CodeGen/X86/vector-compress.ll | 574 +--
49 files changed, 4014 insertions(+), 2228 deletions(-)
diff --git a/llvm/docs/CodeGenerator.md b/llvm/docs/CodeGenerator.md
index 227b16fe9b01db..f0cff99a17d1f1 100644
--- a/llvm/docs/CodeGenerator.md
+++ b/llvm/docs/CodeGenerator.md
@@ -481,7 +481,9 @@ register allocation, a `FREEZE` of a defined value becomes a `COPY`, and a
`FREEZE` of an undefined value stays until code emission, where it emits no
code.
-Selectors still lower `freeze` to a `COPY`. If the copied value is undefined,
+SelectionDAG selects `freeze` to `FREEZE`, except on targets that don't use
+physical registers for values, such as WebAssembly. FastISel and the GlobalISel
+selectors still lower it to a `COPY`. If the copied value is undefined,
different uses can observe different values, which violates the IR semantics of
`freeze`.
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 805c45b9a79285..8aa132f231a457 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -1063,19 +1063,30 @@ bool SDNodeKeyInfo::isEqual(const SDNodeKey &Key, const SDNode &N) {
// SelectionDAG Class
//===----------------------------------------------------------------------===//
+/// Return true if \p Opc with operand \p Op is a freeze of undef, before or
+/// after instruction selection.
+static bool isFreezeOfUndef(unsigned Opc, SDValue Op) {
+ if (Opc != ISD::FREEZE && Opc != ~(unsigned)TargetOpcode::FREEZE)
+ return false;
+ return Op.isUndef() || (Op.isMachineOpcode() &&
+ Op.getMachineOpcode() == TargetOpcode::IMPLICIT_DEF);
+}
+
/// doNotCSE - Return true if CSE should not be performed for this node.
static bool doNotCSE(SDNode *N) {
if (N->getValueType(0) == MVT::Glue)
return true; // Never CSE anything that produces a glue result.
+ // Each freeze of undef is a distinct value.
+ if (N->getNumOperands() == 1 &&
+ isFreezeOfUndef(N->getOpcode(), N->getOperand(0)))
+ return true;
+
switch (N->getOpcode()) {
default: break;
case ISD::HANDLENODE:
case ISD::EH_LABEL:
return true; // Never CSE these nodes.
- case ISD::FREEZE:
- // Each freeze of undef is a distinct value.
- return N->getOperand(0).isUndef();
}
// Check that remaining values produced are not flags.
@@ -7600,7 +7611,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
SDVTList VTs = getVTList(VT);
SDValue Ops[] = {N1};
// Don't CSE glue producing nodes or freezes of undef.
- if (VT != MVT::Glue && !(Opcode == ISD::FREEZE && N1.isUndef())) {
+ if (VT != MVT::Glue && !isFreezeOfUndef(Opcode, N1)) {
SDNodeKey ID(Opcode, VTs, Ops);
FoldingSetInsertToken InsertToken;
if (SDNode *E = lookupNode(ID, DL, InsertToken)) {
@@ -12721,7 +12732,8 @@ SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
SDVTList VTs, ArrayRef<SDValue> Ops) {
// If an identical node already exists, use it.
FoldingSetInsertToken InsertToken;
- if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
+ if (VTs.VTs[VTs.NumVTs - 1] != MVT::Glue &&
+ !(Ops.size() == 1 && isFreezeOfUndef(Opc, Ops[0]))) {
SDNodeKey ID(Opc, VTs, Ops);
AddNodeIDCustom(ID.Tail, N, Opc);
if (SDNode *ON = lookupNode(ID, SDLoc(N), InsertToken))
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index bf8a7523f0b2d7..cbe5f0f9cd94c5 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -2575,11 +2575,11 @@ void SelectionDAGISel::Select_RELOC_NONE(SDNode *N) {
}
void SelectionDAGISel::Select_FREEZE(SDNode *N) {
- // TODO: We don't have FREEZE pseudo-instruction in MachineInstr-level now.
- // If FREEZE instruction is added later, the code below must be changed as
- // well.
- CurDAG->SelectNodeTo(N, TargetOpcode::COPY, N->getValueType(0),
- N->getOperand(0));
+ // Without physical registers, different uses of an undefined value can't
+ // observe different values.
+ unsigned Opc =
+ TM.usesPhysRegsForValues() ? TargetOpcode::FREEZE : TargetOpcode::COPY;
+ CurDAG->SelectNodeTo(N, Opc, N->getValueType(0), N->getOperand(0));
}
void SelectionDAGISel::Select_ARITH_FENCE(SDNode *N) {
diff --git a/llvm/test/CodeGen/AArch64/concat-vector-add-combine.ll b/llvm/test/CodeGen/AArch64/concat-vector-add-combine.ll
index 171f74149c9059..e233238bee9057 100644
--- a/llvm/test/CodeGen/AArch64/concat-vector-add-combine.ll
+++ b/llvm/test/CodeGen/AArch64/concat-vector-add-combine.ll
@@ -94,6 +94,7 @@ define i32 @combine_undef_add_8xi32(i32 %a, i32 %b, i32 %c, i32 %d) local_unname
; CHECK-LABEL: combine_undef_add_8xi32:
; CHECK: // %bb.0:
; CHECK-NEXT: fmov s1, w0
+; CHECK-NEXT: // freeze: $w8
; CHECK-NEXT: dup v0.4s, w8
; CHECK-NEXT: mov v1.s[1], w1
; CHECK-NEXT: mov v1.s[2], w2
diff --git a/llvm/test/CodeGen/AArch64/fma-fneg-combine.ll b/llvm/test/CodeGen/AArch64/fma-fneg-combine.ll
index 936cc6a7d366fc..58117d5bcad9cc 100644
--- a/llvm/test/CodeGen/AArch64/fma-fneg-combine.ll
+++ b/llvm/test/CodeGen/AArch64/fma-fneg-combine.ll
@@ -15,7 +15,8 @@ define float @positive_case_fma(float %a0, float %a1, float %a2) {
define float @negative_case_fma(float %a0, float %a1) {
; CHECK-LABEL: negative_case_fma:
; CHECK: // %bb.0:
-; CHECK-NEXT: fnmadd s0, s0, s1, s0
+; CHECK-NEXT: // freeze: $s2
+; CHECK-NEXT: fnmadd s0, s0, s1, s2
; CHECK-NEXT: ret
%fma = call float @llvm.fma.f32(float %a0, float %a1, float poison)
%freeze = freeze float %fma
@@ -37,7 +38,8 @@ define float @positive_case_fmad(float %a0, float %a1, float %a2) {
define float @negative_case_fmad(float %a0, float %a1) {
; CHECK-LABEL: negative_case_fmad:
; CHECK: // %bb.0:
-; CHECK-NEXT: fnmadd s0, s0, s1, s0
+; CHECK-NEXT: // freeze: $s2
+; CHECK-NEXT: fnmadd s0, s0, s1, s2
; CHECK-NEXT: ret
%fmad = call float @llvm.fmuladd.f32(float %a0, float %a1, float poison)
%freeze = freeze float %fmad
diff --git a/llvm/test/CodeGen/AArch64/freeze-multi-use.ll b/llvm/test/CodeGen/AArch64/freeze-multi-use.ll
index 822ec50d0a5717..0fbf4c61db5bfa 100644
--- a/llvm/test/CodeGen/AArch64/freeze-multi-use.ll
+++ b/llvm/test/CodeGen/AArch64/freeze-multi-use.ll
@@ -7,8 +7,9 @@
define i32 @ashr_mul(i32 %x) {
; CHECK-LABEL: ashr_mul:
; CHECK: // %bb.0:
-; CHECK-NEXT: asr w8, w8, #3
-; CHECK-NEXT: mul w0, w8, w8
+; CHECK-NEXT: // freeze: $w0
+; CHECK-NEXT: asr w8, w0, #3
+; CHECK-NEXT: mul w0, w0, w8
; CHECK-NEXT: ret
%p = ashr i32 %x, 32
%w = freeze i32 %p
@@ -21,11 +22,12 @@ define i32 @ashr_mul(i32 %x) {
define i8 @fshr(i32 %x) {
; CHECK-LABEL: fshr:
; CHECK: // %bb.0:
-; CHECK-NEXT: mvn w8, w8
-; CHECK-NEXT: mov w9, #2 // =0x2
-; CHECK-NEXT: lsr w10, w0, w8
-; CHECK-NEXT: lsl w8, w9, w8
-; CHECK-NEXT: orr w0, w8, w10
+; CHECK-NEXT: // freeze: $w8
+; CHECK-NEXT: mvn w9, w8
+; CHECK-NEXT: mov w10, #2 // =0x2
+; CHECK-NEXT: lsr w8, w0, w8
+; CHECK-NEXT: lsl w9, w10, w9
+; CHECK-NEXT: orr w0, w9, w8
; CHECK-NEXT: ret
%w = freeze i32 poison
%f = call i32 @llvm.fshr.i32(i32 1, i32 %x, i32 %w)
@@ -37,10 +39,11 @@ define i8 @fshr(i32 %x) {
define i64 @select_zext(i64 %a, i64 %b) {
; CHECK-LABEL: select_zext:
; CHECK: // %bb.0:
+; CHECK-NEXT: // freeze: $w8
; CHECK-NEXT: tst w8, #0x1
-; CHECK-NEXT: csel x8, x0, x1, ne
-; CHECK-NEXT: and x9, x8, #0x1
-; CHECK-NEXT: add x0, x8, x9
+; CHECK-NEXT: csel x9, x0, x1, ne
+; CHECK-NEXT: and x8, x8, #0x1
+; CHECK-NEXT: add x0, x9, x8
; CHECK-NEXT: ret
%w = freeze i1 poison
%s = select i1 %w, i64 %a, i64 %b
@@ -53,9 +56,11 @@ define i64 @select_zext(i64 %a, i64 %b) {
define i32 @float_bits() {
; CHECK-LABEL: float_bits:
; CHECK: // %bb.0:
+; CHECK-NEXT: // freeze: $s0
+; CHECK-NEXT: fmov w8, s0
; CHECK-NEXT: fcmp s0, s0
-; CHECK-NEXT: and w8, w8, #0x7fffffff
; CHECK-NEXT: mov w9, #2139095040 // =0x7f800000
+; CHECK-NEXT: and w8, w8, #0x7fffffff
; CHECK-NEXT: cset w10, vs
; CHECK-NEXT: cmp w8, w9
; CHECK-NEXT: cset w8, hi
@@ -75,9 +80,11 @@ define i32 @float_bits() {
define i32 @vector_lane_and_whole(<4 x i32> %x) {
; CHECK-LABEL: vector_lane_and_whole:
; CHECK: // %bb.0:
-; CHECK-NEXT: add v0.4s, v0.4s, v0.4s
-; CHECK-NEXT: fmov w8, s0
-; CHECK-NEXT: sub w0, w8, w8
+; CHECK-NEXT: // freeze: $q1
+; CHECK-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-NEXT: fmov w8, s1
+; CHECK-NEXT: fmov w9, s0
+; CHECK-NEXT: sub w0, w9, w8
; CHECK-NEXT: ret
%w = freeze <4 x i32> poison
%e0 = extractelement <4 x i32> %w, i32 0
@@ -91,9 +98,12 @@ define i32 @vector_lane_and_whole(<4 x i32> %x) {
define i32 @across_call() nounwind {
; CHECK-LABEL: across_call:
; CHECK: // %bb.0:
-; CHECK-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: stp x30, x19, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEXT: // freeze: $w19
+; CHECK-NEXT: mov w0, w19
; CHECK-NEXT: bl use
-; CHECK-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: mov w0, w19
+; CHECK-NEXT: ldp x30, x19, [sp], #16 // 16-byte Folded Reload
; CHECK-NEXT: ret
%w = freeze i32 poison
call void @use(i32 %w)
@@ -104,9 +114,14 @@ define i32 @across_call() nounwind {
define <4 x i32> @vec_across_call() nounwind {
; CHECK-LABEL: vec_across_call:
; CHECK: // %bb.0:
-; CHECK-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: sub sp, sp, #32
+; CHECK-NEXT: str x30, [sp, #16] // 8-byte Spill
+; CHECK-NEXT: // freeze: $q0
+; CHECK-NEXT: str q0, [sp] // 16-byte Spill
; CHECK-NEXT: bl usev
-; CHECK-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ldr q0, [sp] // 16-byte Reload
+; CHECK-NEXT: ldr x30, [sp, #16] // 8-byte Reload
+; CHECK-NEXT: add sp, sp, #32
; CHECK-NEXT: ret
%w = freeze <4 x i32> poison
call void @usev(<4 x i32> %w)
diff --git a/llvm/test/CodeGen/AArch64/freeze.ll b/llvm/test/CodeGen/AArch64/freeze.ll
index fff659e1f21b9c..de3317d58a5f12 100644
--- a/llvm/test/CodeGen/AArch64/freeze.ll
+++ b/llvm/test/CodeGen/AArch64/freeze.ll
@@ -7,30 +7,48 @@
%struct.T = type { i32, i32 }
define i32 @freeze_int() {
-; CHECK-LABEL: freeze_int:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mul w0, w8, w8
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: freeze_int:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: // freeze: $w0
+; CHECK-SD-NEXT: mul w0, w0, w0
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: freeze_int:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mul w0, w8, w8
+; CHECK-GI-NEXT: ret
%y1 = freeze i32 undef
%t1 = mul i32 %y1, %y1
ret i32 %t1
}
define i5 @freeze_int2() {
-; CHECK-LABEL: freeze_int2:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mul w0, w8, w8
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: freeze_int2:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: // freeze: $w0
+; CHECK-SD-NEXT: mul w0, w0, w0
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: freeze_int2:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mul w0, w8, w8
+; CHECK-GI-NEXT: ret
%y1 = freeze i5 undef
%t1 = mul i5 %y1, %y1
ret i5 %t1
}
define float @freeze_float() {
-; CHECK-LABEL: freeze_float:
-; CHECK: // %bb.0:
-; CHECK-NEXT: fadd s0, s0, s0
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: freeze_float:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: // freeze: $s0
+; CHECK-SD-NEXT: fadd s0, s0, s0
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: freeze_float:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: fadd s0, s0, s0
+; CHECK-GI-NEXT: ret
%y1 = freeze float undef
%t1 = fadd float %y1, %y1
ret float %t1
@@ -39,6 +57,7 @@ define float @freeze_float() {
define <2 x i8> @freeze_v2i8() {
; CHECK-LABEL: freeze_v2i8:
; CHECK: // %bb.0:
+; CHECK-NEXT: // freeze: $d0
; CHECK-NEXT: add v0.2s, v0.2s, v0.2s
; CHECK-NEXT: ret
%y1 = freeze <2 x i8> undef
@@ -49,6 +68,7 @@ define <2 x i8> @freeze_v2i8() {
define <3 x i8> @freeze_v3i8() {
; CHECK-SD-LABEL: freeze_v3i8:
; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: // freeze: $d0
; CHECK-SD-NEXT: add v0.4h, v0.4h, v0.4h
; CHECK-SD-NEXT: umov w0, v0.h[0]
; CHECK-SD-NEXT: umov w1, v0.h[1]
@@ -76,6 +96,7 @@ define <3 x i8> @freeze_v3i8() {
define <4 x i8> @freeze_v4i8() {
; CHECK-SD-LABEL: freeze_v4i8:
; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: // freeze: $d0
; CHECK-SD-NEXT: add v0.4h, v0.4h, v0.4h
; CHECK-SD-NEXT: ret
;
@@ -98,20 +119,32 @@ define <4 x i8> @freeze_v4i8() {
}
define <8 x i8> @freeze_v8i8() {
-; CHECK-LABEL: freeze_v8i8:
-; CHECK: // %bb.0:
-; CHECK-NEXT: add v0.8b, v0.8b, v0.8b
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: freeze_v8i8:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: // freeze: $d0
+; CHECK-SD-NEXT: add v0.8b, v0.8b, v0.8b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: freeze_v8i8:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: add v0.8b, v0.8b, v0.8b
+; CHECK-GI-NEXT: ret
%y1 = freeze <8 x i8> undef
%t1 = add <8 x i8> %y1, %y1
ret <8 x i8> %t1
}
define <16 x i8> @freeze_v16i8() {
-; CHECK-LABEL: freeze_v16i8:
-; CHECK: // %bb.0:
-; CHECK-NEXT: add v0.16b, v0.16b, v0.16b
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: freeze_v16i8:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: // freeze: $q0
+; CHECK-SD-NEXT: add v0.16b, v0.16b, v0.16b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: freeze_v16i8:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: add v0.16b, v0.16b, v0.16b
+; CHECK-GI-NEXT: ret
%y1 = freeze <16 x i8> undef
%t1 = add <16 x i8> %y1, %y1
ret <16 x i8> %t1
@@ -120,8 +153,10 @@ define <16 x i8> @freeze_v16i8() {
define <32 x i8> @freeze_v32i8() {
; CHECK-SD-LABEL: freeze_v32i8:
; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: // freeze: $q1
+; CHECK-SD-NEXT: // freeze: $q0
; CHECK-SD-NEXT: add v0.16b, v0.16b, v0.16b
-; CHECK-SD-NEXT: add v1.16b, v0.16b, v0.16b
+; CHECK-SD-NEXT: add v1.16b, v1.16b, v1.16b
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: freeze_v32i8:
@@ -137,6 +172,7 @@ define <32 x i8> @freeze_v32i8() {
define <2 x i16> @freeze_v2i16() {
; CHECK-SD-LABEL: freeze_v2i16:
; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: // freeze: $d0
; CHECK-SD-NEXT: add v0.2s, v0.2s, v0.2s
; CHECK-SD-NEXT: ret
;
@@ -153,30 +189,48 @@ define <2 x i16> @freeze_v2i16() {
}
define <3 x i16> @freeze_v3i16() {
-; CHECK-LABEL: freeze_v3i16:
-; CHECK: // %bb.0:
-; CHECK-NEXT: add v0.4h, v0.4h, v0.4h
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: freeze_v3i16:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: // freeze: $d0
+; CHECK-SD-NEXT: add v0.4h, v0.4h, v0.4h
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: freeze_v3i16:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: add v0.4h, v0.4h, v0.4h
+; CHECK-GI-NEXT: ret
%y1 = freeze <3 x i16> undef
%t1 = add <3 x i16> %y1, %y1
ret <3 x i16> %t1
}
define <4 x i16> @freeze_v4i16() {
-; CHECK-LABEL: freeze_v4i16:
-; CHECK: // %bb.0:
-; CHECK-NEXT: add v0.4h, v0.4h, v0.4h
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: freeze_v4i16:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: // freeze: $d0
+; CHECK-SD-NEXT: add v0.4h, v0.4h, v0.4h
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: freeze_v4i16:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: add v0.4h, v0.4h, v0.4h
+; CHECK-GI-NEXT: ret
%y1 = freeze <4 x i16> undef
%t1 = add <4 x i16> %y1, %y1
ret <4 x i16> %t1
}
define <8 x i16> @freeze_v8i16() {
-; CHECK-LABEL: freeze_v8i16:
-; CHECK: // %bb.0:
-; CHECK-NEXT: add v0.8h, v0.8h, v0.8h
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: freeze_v8i16:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: // freeze: $q0
+; CHECK-SD-NEXT: add v0.8h, v0.8h, v0.8h
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: freeze_v8i16:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: add v0.8h, v0.8h, v0.8h
+; CHECK-GI-NEXT: ret
%y1 = freeze <8 x i16> undef
%t1 = add <8 x i16> %y1, %y1
ret <8 x i16> %t1
@@ -185,8 +239,10 @@ define <8 x i16> @freeze_v8i16() {
define <16 x i16> @freeze_v16i16() {
; CHECK-SD-LABEL: freeze_v16i16:
; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: // freeze: $q1
+; CHECK-SD-NEXT: // freeze: $q0
; CHECK-SD-NEXT: add v0.8h, v0.8h, v0.8h
-; CHECK-SD-NEXT: add v1.8h, v0.8h, v0.8h
+; CHECK-SD-NEXT: add v1.8h, v1.8h, v1.8h
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: freeze_v16i16:
@@ -200,30 +256,48 @@ define <16 x i16> @freeze_v16i16() {
}
define <2 x i32> @freeze_v2i32() {
-; CHECK-LABEL: freeze_v2i32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: add v0.2s, v0.2s, v0.2s
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: freeze_v2i32:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: // freeze: $d0
+; CHECK-SD-NEXT: add v0.2s, v0.2s, v0.2s
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: freeze_v2i32:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: add v0.2s, v0.2s, v0.2s
+; CHECK-GI-NEXT: ret
%y1 = freeze <2 x i32> undef
%t1 = add <2 x i32> %y1, %y1
ret <2 x i32> %t1
}
define <3 x i32> @freeze_v3i32() {
-; CHECK-LABEL: freeze_v3i32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: add v0.4s, v0.4s, v0.4s
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: freeze_v3i32:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: // freeze: $q0
+; CHECK-SD-NEXT: add v0.4s, v0.4s, v0.4s
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: freeze_v3i32:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: add v0.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT: ret
%y1 = freeze <3 x i32> undef
%t1 = add <3 x i32> %y1, %y1
ret <3 x i32> %t1
}
define <4 x i32> @freeze_v4i32() {
-; CHECK-LABEL: freeze_v4i32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: add v0.4s, v0.4s, v0.4s
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: freeze_v4i32:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: // freeze: $q0
+; CHECK-SD-NEXT: add v0.4s, v0.4s, v0.4s
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: freeze_v4i32:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: add v0.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT: ret
%y1 = freeze <4 x i32> undef
%t1 = add <4 x i32> %y1, %y1
ret <4 x i32> %t1
@@ -232,8 +306,10 @@ define <4 x i32> @freeze_v4i32() {
define <8 x i32> @freeze_v8i32() {
; CHECK-SD-LABEL: freeze_v8i32:
; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: // freeze: $q1
+; CHECK-SD-NEXT: // freeze: $q0
; CHECK-SD-NEXT: add v0.4s, v0.4s, v0.4s
-; CHECK-SD-NEXT: add v1.4s, v0.4s, v0.4s
+; CHECK-SD-NEXT: add v1.4s, v1.4s, v1.4s
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: freeze_v8i32:
@@ -247,10 +323,16 @@ define <8 x i32> @freeze_v8i32() {
}
define <2 x i64> @freeze_v2i64() {
-; CHECK-LABEL: freeze_v2i64:
-; CHECK: // %bb.0:
-; CHECK-NEXT: add v0.2d, v0.2d, v0.2d
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: freeze_v2i64:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: // freeze: $q0
+; CHECK-SD-NEXT: add v0.2d, v0.2d, v0.2d
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: freeze_v2i64:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: add v0.2d, v0.2d, v0.2d
+; CHECK-GI-NEXT: ret
%y1 = freeze <2 x i64> undef
%t1 = add <2 x i64> %y1, %y1
ret <2 x i64> %t1
@@ -259,10 +341,12 @@ define <2 x i64> @freeze_v2i64() {
define <3 x i64> @freeze_v3i64() {
; CHECK-SD-LABEL: freeze_v3i64:
; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: // freeze: $q0
; CHECK-SD-NEXT: add v0.2d, v0.2d, v0.2d
; CHECK-SD-NEXT: mov d1, v0.d[1]
-; CHECK-SD-NEXT: add d2, d0, d0
+; CHECK-SD-NEXT: // freeze: $q2
; CHECK-SD-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT: add d2, d2, d2
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: freeze_v3i64:
@@ -281,8 +365,10 @@ define <3 x i64> @freeze_v3i64() {
define <4 x i64> @freeze_v4i64() {
; CHECK-SD-LABEL: freeze_v4i64:
; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: // freeze: $q1
+; CHECK-SD-NEXT: // freeze: $q0
; CHECK-SD-NEXT: add v0.2d, v0.2d, v0.2d
-; CHECK-SD-NEXT: add v1.2d, v0.2d, v0.2d
+; CHECK-SD-NEXT: add v1.2d, v1.2d, v1.2d
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: freeze_v4i64:
@@ -300,7 +386,8 @@ define <2 x ptr> @freeze_v2p0() {
; CHECK-SD: // %bb.0:
; CHECK-SD-NEXT: mov w8, #4 // =0x4
; CHECK-SD-NEXT: dup v0.2d, x8
-; CHECK-SD-NEXT: add v0.2d, v0.2d, v0.2d
+; CHECK-SD-NEXT: // freeze: $q1
+; CHECK-SD-NEXT: add v0.2d, v1.2d, v0.2d
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: freeze_v2p0:
@@ -318,10 +405,12 @@ define <3 x ptr> @freeze_v3p0() {
; CHECK-SD-LABEL: freeze_v3p0:
; CHECK-SD: // %bb.0:
; CHECK-SD-NEXT: mov w8, #4 // =0x4
+; CHECK-SD-NEXT: // freeze: $q0
; CHECK-SD-NEXT: dup v2.2d, x8
; CHECK-SD-NEXT: add v0.2d, v0.2d, v2.2d
; CHECK-SD-NEXT: mov d1, v0.d[1]
-; CHECK-SD-NEXT: add d2, d0, d2
+; CHECK-SD-NEXT: // freeze: $q3
+; CHECK-SD-NEXT: add d2, d3, d2
; CHECK-SD-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-SD-NEXT: ret
;
@@ -343,9 +432,11 @@ define <4 x ptr> @freeze_v4p0() {
; CHECK-SD-LABEL: freeze_v4p0:
; CHECK-SD: // %bb.0:
; CHECK-SD-NEXT: mov w8, #4 // =0x4
+; CHECK-SD-NEXT: // freeze: $q2
+; CHECK-SD-NEXT: // freeze: $q0
; CHECK-SD-NEXT: dup v1.2d, x8
; CHECK-SD-NEXT: add v0.2d, v0.2d, v1.2d
-; CHECK-SD-NEXT: add v1.2d, v0.2d, v1.2d
+; CHECK-SD-NEXT: add v1.2d, v2.2d, v1.2d
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: freeze_v4p0:
@@ -361,20 +452,33 @@ define <4 x ptr> @freeze_v4p0() {
}
define ptr @freeze_ptr() {
-; CHECK-LABEL: freeze_ptr:
-; CHECK: // %bb.0:
-; CHECK-NEXT: add x0, x8, #4
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: freeze_ptr:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: // freeze: $x0
+; CHECK-SD-NEXT: add x0, x0, #4
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: freeze_ptr:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: add x0, x8, #4
+; CHECK-GI-NEXT: ret
%y1 = freeze ptr undef
%t1 = getelementptr i8, ptr %y1, i64 4
ret ptr %t1
}
define i32 @freeze_struct() {
-; CHECK-LABEL: freeze_struct:
-; CHECK: // %bb.0:
-; CHECK-NEXT: add w0, w8, w8
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: freeze_struct:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: // freeze: $w0
+; CHECK-SD-NEXT: // freeze: $w8
+; CHECK-SD-NEXT: add w0, w8, w0
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: freeze_struct:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: add w0, w8, w8
+; CHECK-GI-NEXT: ret
%y1 = freeze %struct.T undef
%v1 = extractvalue %struct.T %y1, 0
%v2 = extractvalue %struct.T %y1, 1
@@ -383,10 +487,17 @@ define i32 @freeze_struct() {
}
define i32 @freeze_anonstruct() {
-; CHECK-LABEL: freeze_anonstruct:
-; CHECK: // %bb.0:
-; CHECK-NEXT: add w0, w8, w8
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: freeze_anonstruct:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: // freeze: $w0
+; CHECK-SD-NEXT: // freeze: $w8
+; CHECK-SD-NEXT: add w0, w8, w0
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: freeze_anonstruct:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: add w0, w8, w8
+; CHECK-GI-NEXT: ret
%y1 = freeze {i32, i32} undef
%v1 = extractvalue {i32, i32} %y1, 0
%v2 = extractvalue {i32, i32} %y1, 1
@@ -395,10 +506,17 @@ define i32 @freeze_anonstruct() {
}
define i32 @freeze_anonstruct2() {
-; CHECK-LABEL: freeze_anonstruct2:
-; CHECK: // %bb.0:
-; CHECK-NEXT: add w0, w8, w8, uxth
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: freeze_anonstruct2:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: // freeze: $w0
+; CHECK-SD-NEXT: // freeze: $w8
+; CHECK-SD-NEXT: add w0, w8, w0, uxth
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: freeze_anonstruct2:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: add w0, w8, w8, uxth
+; CHECK-GI-NEXT: ret
%y1 = freeze {i32, i16} undef
%v1 = extractvalue {i32, i16} %y1, 0
%v2 = extractvalue {i32, i16} %y1, 1
@@ -408,10 +526,17 @@ define i32 @freeze_anonstruct2() {
}
define i64 @freeze_array() {
-; CHECK-LABEL: freeze_array:
-; CHECK: // %bb.0:
-; CHECK-NEXT: add x0, x8, x8
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: freeze_array:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: // freeze: $x0
+; CHECK-SD-NEXT: // freeze: $x8
+; CHECK-SD-NEXT: add x0, x8, x0
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: freeze_array:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: add x0, x8, x8
+; CHECK-GI-NEXT: ret
%y1 = freeze [2 x i64] undef
%v1 = extractvalue [2 x i64] %y1, 0
%v2 = extractvalue [2 x i64] %y1, 1
@@ -592,6 +717,8 @@ define i32 @freeze_ucmp(i32 %a0) nounwind {
define i128 @freeze_i128() {
; CHECK-SD-LABEL: freeze_i128:
; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: // freeze: $x0
+; CHECK-SD-NEXT: // freeze: $x1
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: freeze_i128:
diff --git a/llvm/test/CodeGen/AArch64/neon-shuffle-vector-tbl.ll b/llvm/test/CodeGen/AArch64/neon-shuffle-vector-tbl.ll
index 9315a5b831aa16..d35e4e7f2353c7 100644
--- a/llvm/test/CodeGen/AArch64/neon-shuffle-vector-tbl.ll
+++ b/llvm/test/CodeGen/AArch64/neon-shuffle-vector-tbl.ll
@@ -565,68 +565,68 @@ define <16 x i8> @no_shuffle_with_two_mask_sources(<16 x i8> %src, <16 x i8> %ma
; CHECK-SD: // %bb.0:
; CHECK-SD-NEXT: sub sp, sp, #16
; CHECK-SD-NEXT: .cfi_def_cfa_offset 16
-; CHECK-SD-NEXT: umov.b w8, v1[0]
-; CHECK-SD-NEXT: umov.b w10, v1[8]
-; CHECK-SD-NEXT: mov x11, sp
-; CHECK-SD-NEXT: umov.b w9, v2[1]
+; CHECK-SD-NEXT: umov.b w9, v1[0]
+; CHECK-SD-NEXT: umov.b w11, v1[8]
+; CHECK-SD-NEXT: mov x13, sp
+; CHECK-SD-NEXT: umov.b w10, v2[1]
; CHECK-SD-NEXT: umov.b w12, v2[9]
; CHECK-SD-NEXT: mov x14, sp
-; CHECK-SD-NEXT: mov x13, sp
+; CHECK-SD-NEXT: mov x8, sp
; CHECK-SD-NEXT: str q0, [sp]
; CHECK-SD-NEXT: umov.b w15, v2[11]
-; CHECK-SD-NEXT: bfxil x11, x8, #0, #4
-; CHECK-SD-NEXT: bfxil x14, x10, #0, #4
-; CHECK-SD-NEXT: umov.b w8, v1[2]
; CHECK-SD-NEXT: bfxil x13, x9, #0, #4
-; CHECK-SD-NEXT: mov x9, sp
-; CHECK-SD-NEXT: umov.b w10, v1[10]
-; CHECK-SD-NEXT: bfxil x9, x12, #0, #4
-; CHECK-SD-NEXT: ldr b0, [x11]
+; CHECK-SD-NEXT: bfxil x14, x11, #0, #4
+; CHECK-SD-NEXT: umov.b w9, v1[2]
+; CHECK-SD-NEXT: bfxil x8, x10, #0, #4
+; CHECK-SD-NEXT: mov x10, sp
+; CHECK-SD-NEXT: umov.b w11, v1[10]
+; CHECK-SD-NEXT: bfxil x10, x12, #0, #4
+; CHECK-SD-NEXT: ldr b0, [x13]
; CHECK-SD-NEXT: ldr b3, [x14]
-; CHECK-SD-NEXT: mov x11, sp
-; CHECK-SD-NEXT: umov.b w12, v2[3]
+; CHECK-SD-NEXT: mov x12, sp
+; CHECK-SD-NEXT: umov.b w13, v2[3]
; CHECK-SD-NEXT: mov x14, sp
-; CHECK-SD-NEXT: ld1.b { v0 }[1], [x13]
-; CHECK-SD-NEXT: ld1.b { v3 }[1], [x9]
-; CHECK-SD-NEXT: bfxil x11, x8, #0, #4
-; CHECK-SD-NEXT: bfxil x14, x10, #0, #4
+; CHECK-SD-NEXT: ld1.b { v0 }[1], [x8]
+; CHECK-SD-NEXT: ld1.b { v3 }[1], [x10]
+; CHECK-SD-NEXT: bfxil x12, x9, #0, #4
+; CHECK-SD-NEXT: bfxil x14, x11, #0, #4
; CHECK-SD-NEXT: mov x8, sp
; CHECK-SD-NEXT: umov.b w9, v1[4]
; CHECK-SD-NEXT: mov x10, sp
-; CHECK-SD-NEXT: umov.b w13, v1[12]
-; CHECK-SD-NEXT: ld1.b { v0 }[2], [x11]
+; CHECK-SD-NEXT: umov.b w11, v1[12]
+; CHECK-SD-NEXT: ld1.b { v0 }[2], [x12]
; CHECK-SD-NEXT: ld1.b { v3 }[2], [x14]
-; CHECK-SD-NEXT: bfxil x8, x12, #0, #4
+; CHECK-SD-NEXT: bfxil x8, x13, #0, #4
; CHECK-SD-NEXT: bfxil x10, x15, #0, #4
-; CHECK-SD-NEXT: mov x11, sp
-; CHECK-SD-NEXT: umov.b w12, v2[5]
+; CHECK-SD-NEXT: mov x12, sp
+; CHECK-SD-NEXT: umov.b w13, v2[5]
; CHECK-SD-NEXT: mov x14, sp
; CHECK-SD-NEXT: umov.b w15, v2[13]
-; CHECK-SD-NEXT: bfxil x11, x9, #0, #4
+; CHECK-SD-NEXT: bfxil x12, x9, #0, #4
; CHECK-SD-NEXT: ld1.b { v0 }[3], [x8]
; CHECK-SD-NEXT: ld1.b { v3 }[3], [x10]
-; CHECK-SD-NEXT: bfxil x14, x13, #0, #4
+; CHECK-SD-NEXT: bfxil x14, x11, #0, #4
; CHECK-SD-NEXT: umov.b w8, v1[6]
; CHECK-SD-NEXT: umov.b w9, v1[14]
; CHECK-SD-NEXT: mov x10, sp
-; CHECK-SD-NEXT: mov x13, sp
-; CHECK-SD-NEXT: bfxil x10, x12, #0, #4
-; CHECK-SD-NEXT: umov.b w12, v2[7]
-; CHECK-SD-NEXT: ld1.b { v0 }[4], [x11]
-; CHECK-SD-NEXT: ld1.b { v3 }[4], [x14]
-; CHECK-SD-NEXT: bfxil x13, x15, #0, #4
; CHECK-SD-NEXT: mov x11, sp
+; CHECK-SD-NEXT: bfxil x10, x13, #0, #4
+; CHECK-SD-NEXT: umov.b w13, v2[7]
+; CHECK-SD-NEXT: ld1.b { v0 }[4], [x12]
+; CHECK-SD-NEXT: ld1.b { v3 }[4], [x14]
+; CHECK-SD-NEXT: bfxil x11, x15, #0, #4
+; CHECK-SD-NEXT: mov x12, sp
; CHECK-SD-NEXT: mov x14, sp
; CHECK-SD-NEXT: umov.b w15, v2[15]
-; CHECK-SD-NEXT: bfxil x11, x8, #0, #4
+; CHECK-SD-NEXT: bfxil x12, x8, #0, #4
; CHECK-SD-NEXT: bfxil x14, x9, #0, #4
; CHECK-SD-NEXT: mov x8, sp
; CHECK-SD-NEXT: ld1.b { v0 }[5], [x10]
-; CHECK-SD-NEXT: ld1.b { v3 }[5], [x13]
+; CHECK-SD-NEXT: ld1.b { v3 }[5], [x11]
; CHECK-SD-NEXT: mov x9, sp
-; CHECK-SD-NEXT: bfxil x9, x12, #0, #4
+; CHECK-SD-NEXT: bfxil x9, x13, #0, #4
; CHECK-SD-NEXT: bfxil x8, x15, #0, #4
-; CHECK-SD-NEXT: ld1.b { v0 }[6], [x11]
+; CHECK-SD-NEXT: ld1.b { v0 }[6], [x12]
; CHECK-SD-NEXT: ld1.b { v3 }[6], [x14]
; CHECK-SD-NEXT: ld1.b { v0 }[7], [x9]
; CHECK-SD-NEXT: ld1.b { v3 }[7], [x8]
diff --git a/llvm/test/CodeGen/AArch64/vector-compress.ll b/llvm/test/CodeGen/AArch64/vector-compress.ll
index 56f6a1cd341d27..7a4d14b4ab6bd3 100644
--- a/llvm/test/CodeGen/AArch64/vector-compress.ll
+++ b/llvm/test/CodeGen/AArch64/vector-compress.ll
@@ -357,10 +357,10 @@ define <4 x i8> @test_compress_small(<4 x i8> %vec, <4 x i1> %mask) {
; CHECK-NEXT: umov.h w9, v1[0]
; CHECK-NEXT: umov.h w10, v1[1]
; CHECK-NEXT: umov.h w11, v1[2]
-; CHECK-NEXT: bfi x8, x9, #1, #1
; CHECK-NEXT: and x10, x10, #0x1
-; CHECK-NEXT: and x9, x9, #0x1
-; CHECK-NEXT: add x9, x9, x10
+; CHECK-NEXT: and x12, x9, #0x1
+; CHECK-NEXT: bfi x8, x9, #1, #1
+; CHECK-NEXT: add x9, x12, x10
; CHECK-NEXT: add x10, sp, #8
; CHECK-NEXT: and x11, x11, #0x1
; CHECK-NEXT: bfi x10, x9, #1, #2
@@ -390,10 +390,10 @@ define <4 x i4> @test_compress_illegal_element_type(<4 x i4> %vec, <4 x i1> %mas
; CHECK-NEXT: umov.h w9, v1[0]
; CHECK-NEXT: umov.h w10, v1[1]
; CHECK-NEXT: umov.h w11, v1[2]
-; CHECK-NEXT: bfi x8, x9, #1, #1
; CHECK-NEXT: and x10, x10, #0x1
-; CHECK-NEXT: and x9, x9, #0x1
-; CHECK-NEXT: add x9, x9, x10
+; CHECK-NEXT: and x12, x9, #0x1
+; CHECK-NEXT: bfi x8, x9, #1, #1
+; CHECK-NEXT: add x9, x12, x10
; CHECK-NEXT: add x10, sp, #8
; CHECK-NEXT: and x11, x11, #0x1
; CHECK-NEXT: bfi x10, x9, #1, #2
diff --git a/llvm/test/CodeGen/AArch64/vector-extract-last-active.ll b/llvm/test/CodeGen/AArch64/vector-extract-last-active.ll
index 21f306bb129d76..6a3012351d5616 100644
--- a/llvm/test/CodeGen/AArch64/vector-extract-last-active.ll
+++ b/llvm/test/CodeGen/AArch64/vector-extract-last-active.ll
@@ -11,16 +11,16 @@ define i8 @extract_last_i8(<16 x i8> %data, <16 x i8> %mask, i8 %passthru) {
; NEON-FIXED-NEXT: adrp x8, .LCPI0_0
; NEON-FIXED-NEXT: cmtst v1.16b, v1.16b, v1.16b
; NEON-FIXED-NEXT: ldr q3, [x8, :lo12:.LCPI0_0]
-; NEON-FIXED-NEXT: mov x9, sp
+; NEON-FIXED-NEXT: mov x8, sp
; NEON-FIXED-NEXT: str q0, [sp]
; NEON-FIXED-NEXT: bic v2.16b, v3.16b, v2.16b
; NEON-FIXED-NEXT: umaxv b1, v1.16b
; NEON-FIXED-NEXT: umaxv b2, v2.16b
-; NEON-FIXED-NEXT: fmov w8, s2
-; NEON-FIXED-NEXT: bfxil x9, x8, #0, #4
-; NEON-FIXED-NEXT: ldrb w8, [x9]
; NEON-FIXED-NEXT: fmov w9, s1
+; NEON-FIXED-NEXT: fmov w1, s2
; NEON-FIXED-NEXT: tst w9, #0x1
+; NEON-FIXED-NEXT: bfxil x8, x1, #0, #4
+; NEON-FIXED-NEXT: ldrb w8, [x8]
; NEON-FIXED-NEXT: csel w0, w8, w0, ne
; NEON-FIXED-NEXT: add sp, sp, #16
; NEON-FIXED-NEXT: ret
@@ -49,18 +49,18 @@ define i16 @extract_last_i16(<8 x i16> %data, <8 x i16> %mask, i16 %passthru) {
; NEON-FIXED-NEXT: .cfi_def_cfa_offset 16
; NEON-FIXED-NEXT: cmtst v1.8h, v1.8h, v1.8h
; NEON-FIXED-NEXT: adrp x8, .LCPI1_0
-; NEON-FIXED-NEXT: mov x9, sp
-; NEON-FIXED-NEXT: ldr d3, [x8, :lo12:.LCPI1_0]
; NEON-FIXED-NEXT: str q0, [sp]
+; NEON-FIXED-NEXT: ldr d3, [x8, :lo12:.LCPI1_0]
+; NEON-FIXED-NEXT: mov x8, sp
; NEON-FIXED-NEXT: xtn v2.8b, v1.8h
; NEON-FIXED-NEXT: umaxv h1, v1.8h
; NEON-FIXED-NEXT: and v2.8b, v2.8b, v3.8b
-; NEON-FIXED-NEXT: umaxv b2, v2.8b
-; NEON-FIXED-NEXT: fmov w8, s2
-; NEON-FIXED-NEXT: bfi x9, x8, #1, #3
-; NEON-FIXED-NEXT: ldrh w8, [x9]
; NEON-FIXED-NEXT: fmov w9, s1
+; NEON-FIXED-NEXT: umaxv b2, v2.8b
; NEON-FIXED-NEXT: tst w9, #0x1
+; NEON-FIXED-NEXT: fmov w1, s2
+; NEON-FIXED-NEXT: bfi x8, x1, #1, #3
+; NEON-FIXED-NEXT: ldrh w8, [x8]
; NEON-FIXED-NEXT: csel w0, w8, w0, ne
; NEON-FIXED-NEXT: add sp, sp, #16
; NEON-FIXED-NEXT: ret
@@ -89,18 +89,18 @@ define i32 @extract_last_i32(<4 x i32> %data, <4 x i32> %mask, i32 %passthru) {
; NEON-FIXED-NEXT: .cfi_def_cfa_offset 16
; NEON-FIXED-NEXT: cmtst v1.4s, v1.4s, v1.4s
; NEON-FIXED-NEXT: adrp x8, .LCPI2_0
-; NEON-FIXED-NEXT: mov x9, sp
-; NEON-FIXED-NEXT: ldr d3, [x8, :lo12:.LCPI2_0]
; NEON-FIXED-NEXT: str q0, [sp]
+; NEON-FIXED-NEXT: ldr d3, [x8, :lo12:.LCPI2_0]
+; NEON-FIXED-NEXT: mov x8, sp
; NEON-FIXED-NEXT: xtn v2.4h, v1.4s
; NEON-FIXED-NEXT: umaxv s1, v1.4s
; NEON-FIXED-NEXT: and v2.8b, v2.8b, v3.8b
-; NEON-FIXED-NEXT: umaxv h2, v2.4h
-; NEON-FIXED-NEXT: fmov w8, s2
-; NEON-FIXED-NEXT: bfi x9, x8, #2, #2
-; NEON-FIXED-NEXT: ldr w8, [x9]
; NEON-FIXED-NEXT: fmov w9, s1
+; NEON-FIXED-NEXT: umaxv h2, v2.4h
; NEON-FIXED-NEXT: tst w9, #0x1
+; NEON-FIXED-NEXT: fmov w1, s2
+; NEON-FIXED-NEXT: bfi x8, x1, #2, #2
+; NEON-FIXED-NEXT: ldr w8, [x8]
; NEON-FIXED-NEXT: csel w0, w8, w0, ne
; NEON-FIXED-NEXT: add sp, sp, #16
; NEON-FIXED-NEXT: ret
@@ -171,18 +171,18 @@ define half @extract_last_half(<8 x half> %data, <8 x i16> %mask, half %passthru
; NEON-FIXED-NEXT: .cfi_def_cfa_offset 16
; NEON-FIXED-NEXT: cmtst v1.8h, v1.8h, v1.8h
; NEON-FIXED-NEXT: adrp x8, .LCPI4_0
-; NEON-FIXED-NEXT: mov x9, sp
-; NEON-FIXED-NEXT: ldr d4, [x8, :lo12:.LCPI4_0]
; NEON-FIXED-NEXT: str q0, [sp]
+; NEON-FIXED-NEXT: ldr d4, [x8, :lo12:.LCPI4_0]
+; NEON-FIXED-NEXT: mov x8, sp
; NEON-FIXED-NEXT: // kill: def $h2 killed $h2 def $s2
; NEON-FIXED-NEXT: xtn v3.8b, v1.8h
; NEON-FIXED-NEXT: umaxv h1, v1.8h
; NEON-FIXED-NEXT: and v3.8b, v3.8b, v4.8b
; NEON-FIXED-NEXT: umaxv b3, v3.8b
-; NEON-FIXED-NEXT: fmov w8, s3
-; NEON-FIXED-NEXT: bfi x9, x8, #1, #3
+; NEON-FIXED-NEXT: fmov w0, s3
+; NEON-FIXED-NEXT: bfi x8, x0, #1, #3
+; NEON-FIXED-NEXT: ldr h0, [x8]
; NEON-FIXED-NEXT: fmov w8, s1
-; NEON-FIXED-NEXT: ldr h0, [x9]
; NEON-FIXED-NEXT: tst w8, #0x1
; NEON-FIXED-NEXT: fcsel s0, s0, s2, ne
; NEON-FIXED-NEXT: // kill: def $h0 killed $h0 killed $s0
@@ -214,18 +214,18 @@ define bfloat @extract_last_bfloat(<8 x bfloat> %data, <8 x i16> %mask, bfloat %
; NEON-FIXED-NEXT: .cfi_def_cfa_offset 16
; NEON-FIXED-NEXT: cmtst v1.8h, v1.8h, v1.8h
; NEON-FIXED-NEXT: adrp x8, .LCPI5_0
-; NEON-FIXED-NEXT: mov x9, sp
-; NEON-FIXED-NEXT: ldr d4, [x8, :lo12:.LCPI5_0]
; NEON-FIXED-NEXT: str q0, [sp]
+; NEON-FIXED-NEXT: ldr d4, [x8, :lo12:.LCPI5_0]
+; NEON-FIXED-NEXT: mov x8, sp
; NEON-FIXED-NEXT: // kill: def $h2 killed $h2 def $s2
; NEON-FIXED-NEXT: xtn v3.8b, v1.8h
; NEON-FIXED-NEXT: umaxv h1, v1.8h
; NEON-FIXED-NEXT: and v3.8b, v3.8b, v4.8b
; NEON-FIXED-NEXT: umaxv b3, v3.8b
-; NEON-FIXED-NEXT: fmov w8, s3
-; NEON-FIXED-NEXT: bfi x9, x8, #1, #3
+; NEON-FIXED-NEXT: fmov w0, s3
+; NEON-FIXED-NEXT: bfi x8, x0, #1, #3
+; NEON-FIXED-NEXT: ldr h0, [x8]
; NEON-FIXED-NEXT: fmov w8, s1
-; NEON-FIXED-NEXT: ldr h0, [x9]
; NEON-FIXED-NEXT: tst w8, #0x1
; NEON-FIXED-NEXT: fcsel s0, s0, s2, ne
; NEON-FIXED-NEXT: // kill: def $h0 killed $h0 killed $s0
@@ -257,17 +257,17 @@ define float @extract_last_float(<4 x float> %data, <4 x i32> %mask, float %pass
; NEON-FIXED-NEXT: .cfi_def_cfa_offset 16
; NEON-FIXED-NEXT: cmtst v1.4s, v1.4s, v1.4s
; NEON-FIXED-NEXT: adrp x8, .LCPI6_0
-; NEON-FIXED-NEXT: mov x9, sp
-; NEON-FIXED-NEXT: ldr d4, [x8, :lo12:.LCPI6_0]
; NEON-FIXED-NEXT: str q0, [sp]
+; NEON-FIXED-NEXT: ldr d4, [x8, :lo12:.LCPI6_0]
+; NEON-FIXED-NEXT: mov x8, sp
; NEON-FIXED-NEXT: xtn v3.4h, v1.4s
; NEON-FIXED-NEXT: umaxv s1, v1.4s
; NEON-FIXED-NEXT: and v3.8b, v3.8b, v4.8b
; NEON-FIXED-NEXT: umaxv h3, v3.4h
-; NEON-FIXED-NEXT: fmov w8, s3
-; NEON-FIXED-NEXT: bfi x9, x8, #2, #2
+; NEON-FIXED-NEXT: fmov w0, s3
+; NEON-FIXED-NEXT: bfi x8, x0, #2, #2
+; NEON-FIXED-NEXT: ldr s0, [x8]
; NEON-FIXED-NEXT: fmov w8, s1
-; NEON-FIXED-NEXT: ldr s0, [x9]
; NEON-FIXED-NEXT: tst w8, #0x1
; NEON-FIXED-NEXT: fcsel s0, s0, s2, ne
; NEON-FIXED-NEXT: add sp, sp, #16
@@ -441,7 +441,7 @@ define i32 @extract_last_active_v3i32(<3 x i32> %a, <3 x i1> %c) {
; NEON-FIXED-NEXT: .cfi_def_cfa_offset 16
; NEON-FIXED-NEXT: movi v1.2d, #0000000000000000
; NEON-FIXED-NEXT: adrp x8, .LCPI18_0
-; NEON-FIXED-NEXT: mov x11, sp
+; NEON-FIXED-NEXT: mov x10, sp
; NEON-FIXED-NEXT: ldr d3, [x8, :lo12:.LCPI18_0]
; NEON-FIXED-NEXT: str q0, [sp]
; NEON-FIXED-NEXT: mov v1.h[0], w0
@@ -456,9 +456,9 @@ define i32 @extract_last_active_v3i32(<3 x i32> %a, <3 x i1> %c) {
; NEON-FIXED-NEXT: orr w8, w9, w8, lsr #16
; NEON-FIXED-NEXT: tst w8, #0x1
; NEON-FIXED-NEXT: umaxv h2, v2.4h
-; NEON-FIXED-NEXT: fmov w10, s2
-; NEON-FIXED-NEXT: bfi x11, x10, #2, #2
-; NEON-FIXED-NEXT: ldr w9, [x11]
+; NEON-FIXED-NEXT: fmov w0, s2
+; NEON-FIXED-NEXT: bfi x10, x0, #2, #2
+; NEON-FIXED-NEXT: ldr w9, [x10]
; NEON-FIXED-NEXT: csinv w0, w9, wzr, ne
; NEON-FIXED-NEXT: add sp, sp, #16
; NEON-FIXED-NEXT: ret
diff --git a/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll b/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
index d37e7695ff7869..ace95d0caf4b8d 100644
--- a/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
+++ b/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
@@ -515,12 +515,12 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
; GFX908-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
; GFX908-NEXT: s_load_dword s7, s[8:9], 0x18
; GFX908-NEXT: s_mov_b32 s6, 0
-; GFX908-NEXT: s_mov_b32 s9, s6
+; GFX908-NEXT: ; freeze: $sgpr18_sgpr19
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: v_cvt_f32_u32_e32 v1, s3
; GFX908-NEXT: s_sub_i32 s8, 0, s3
; GFX908-NEXT: v_cvt_f32_f16_e32 v12, s7
-; GFX908-NEXT: s_mov_b64 s[18:19], 0
+; GFX908-NEXT: s_mov_b32 s9, s6
; GFX908-NEXT: v_rcp_f32_e32 v1, v1
; GFX908-NEXT: v_mov_b32_e32 v14, 0
; GFX908-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1
@@ -553,7 +553,8 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
; GFX908-NEXT: s_lshl_b64 s[12:13], s[4:5], 5
; GFX908-NEXT: s_lshl_b64 s[14:15], s[8:9], 5
; GFX908-NEXT: s_lshl_b64 s[16:17], s[2:3], 5
-; GFX908-NEXT: s_and_b64 s[0:1], exec, s[0:1]
+; GFX908-NEXT: s_and_b64 s[0:1], exec, s[18:19]
+; GFX908-NEXT: s_mov_b64 s[18:19], 0
; GFX908-NEXT: s_branch .LBB3_2
; GFX908-NEXT: .LBB3_1: ; in Loop: Header=BB3_2 Depth=1
; GFX908-NEXT: s_branch .LBB3_9
@@ -654,11 +655,11 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
; GFX90A-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
; GFX90A-NEXT: s_load_dword s7, s[8:9], 0x18
; GFX90A-NEXT: s_mov_b32 s6, 0
-; GFX90A-NEXT: s_mov_b32 s9, s6
+; GFX90A-NEXT: ; freeze: $sgpr18_sgpr19
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_cvt_f32_u32_e32 v0, s3
; GFX90A-NEXT: s_sub_i32 s8, 0, s3
-; GFX90A-NEXT: s_mov_b64 s[18:19], 0
+; GFX90A-NEXT: s_mov_b32 s9, s6
; GFX90A-NEXT: v_mov_b32_e32 v14, 0
; GFX90A-NEXT: v_rcp_f32_e32 v0, v0
; GFX90A-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
@@ -692,7 +693,8 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
; GFX90A-NEXT: s_lshl_b64 s[12:13], s[4:5], 5
; GFX90A-NEXT: s_lshl_b64 s[14:15], s[8:9], 5
; GFX90A-NEXT: s_lshl_b64 s[16:17], s[2:3], 5
-; GFX90A-NEXT: s_and_b64 s[0:1], exec, s[0:1]
+; GFX90A-NEXT: s_and_b64 s[0:1], exec, s[18:19]
+; GFX90A-NEXT: s_mov_b64 s[18:19], 0
; GFX90A-NEXT: s_branch .LBB3_2
; GFX90A-NEXT: .LBB3_1: ; in Loop: Header=BB3_2 Depth=1
; GFX90A-NEXT: s_branch .LBB3_9
diff --git a/llvm/test/CodeGen/AMDGPU/andorbitset.ll b/llvm/test/CodeGen/AMDGPU/andorbitset.ll
index 03499b184d565d..e7aa64ce127e2c 100644
--- a/llvm/test/CodeGen/AMDGPU/andorbitset.ll
+++ b/llvm/test/CodeGen/AMDGPU/andorbitset.ll
@@ -109,20 +109,21 @@ define amdgpu_kernel void @s_set_midbit(ptr addrspace(1) %out, i32 %in) {
define void @bitset_verifier_error_freeze_poison() local_unnamed_addr #0 {
; SI-LABEL: bitset_verifier_error_freeze_poison:
; SI: ; %bb.0: ; %bb
+; SI-NEXT: ; freeze: $sgpr8
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_getpc_b64 s[4:5]
; SI-NEXT: s_add_u32 s4, s4, gv at gotpcrel32@lo+4
; SI-NEXT: s_addc_u32 s5, s5, gv at gotpcrel32@hi+12
; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x0
+; SI-NEXT: s_and_b32 s9, s8, 0x7fffffff
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: v_mov_b32_e32 v0, s9
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_and_b32 s8, s4, 0x7fffffff
-; SI-NEXT: v_mov_b32_e32 v0, s8
; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v0, 0x3f7fbe77
-; SI-NEXT: v_cmp_ge_f32_e64 vcc, |s4|, v0
+; SI-NEXT: v_cmp_ge_f32_e64 vcc, |s8|, v0
; SI-NEXT: s_cbranch_vccnz .LBB6_2
; SI-NEXT: ; %bb.1: ; %bb5
; SI-NEXT: .LBB6_2: ; %bb6
diff --git a/llvm/test/CodeGen/AMDGPU/carryout-selection.ll b/llvm/test/CodeGen/AMDGPU/carryout-selection.ll
index 3f0618de15a328..a2f5fa25bf8221 100644
--- a/llvm/test/CodeGen/AMDGPU/carryout-selection.ll
+++ b/llvm/test/CodeGen/AMDGPU/carryout-selection.ll
@@ -4230,19 +4230,19 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GCN-ISEL-NEXT: [[S_CSELECT_B32_4:%[0-9]+]]:sreg_32 = S_CSELECT_B32 [[S_MOV_B32_10]], [[S_MOV_B32_9]], implicit $scc
; GCN-ISEL-NEXT: S_CMP_EQ_U32 [[S_SUB_C2]], [[COPY22]], implicit-def $scc
; GCN-ISEL-NEXT: [[S_CSELECT_B32_5:%[0-9]+]]:sreg_32 = S_CSELECT_B32 [[S_CSELECT_B32_4]], [[S_CSELECT_B32_3]], implicit $scc
-; GCN-ISEL-NEXT: [[COPY72:%[0-9]+]]:sreg_32 = COPY [[S_CSELECT_B32_5]]
+; GCN-ISEL-NEXT: [[FREEZE:%[0-9]+]]:sreg_32 = FREEZE [[S_CSELECT_B32_5]]
; GCN-ISEL-NEXT: [[REG_SEQUENCE22:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY69]], %subreg.sub0, [[COPY68]], %subreg.sub1
; GCN-ISEL-NEXT: [[S_MOV_B64_1:%[0-9]+]]:sreg_64 = S_MOV_B64 1
; GCN-ISEL-NEXT: [[S_ADD_U6:%[0-9]+]]:sreg_64 = S_ADD_U64_PSEUDO [[REG_SEQUENCE22]], [[S_MOV_B64_1]], implicit-def dead $scc
-; GCN-ISEL-NEXT: [[COPY73:%[0-9]+]]:sreg_32 = COPY [[S_ADD_U6]].sub0
+; GCN-ISEL-NEXT: [[COPY72:%[0-9]+]]:sreg_32 = COPY [[S_ADD_U6]].sub0
; GCN-ISEL-NEXT: [[S_MOV_B64_2:%[0-9]+]]:sreg_64 = S_MOV_B64 2
; GCN-ISEL-NEXT: [[S_ADD_U7:%[0-9]+]]:sreg_64 = S_ADD_U64_PSEUDO [[REG_SEQUENCE22]], [[S_MOV_B64_2]], implicit-def dead $scc
-; GCN-ISEL-NEXT: [[COPY74:%[0-9]+]]:sreg_32 = COPY [[S_ADD_U7]].sub0
-; GCN-ISEL-NEXT: S_CMP_LG_U32 [[COPY72]], [[S_MOV_B32_9]], implicit-def $scc
-; GCN-ISEL-NEXT: [[S_CSELECT_B32_6:%[0-9]+]]:sreg_32 = S_CSELECT_B32 [[COPY74]], [[COPY73]], implicit $scc
-; GCN-ISEL-NEXT: [[COPY75:%[0-9]+]]:sreg_32 = COPY [[S_ADD_U6]].sub1
-; GCN-ISEL-NEXT: [[COPY76:%[0-9]+]]:sreg_32 = COPY [[S_ADD_U7]].sub1
-; GCN-ISEL-NEXT: [[S_CSELECT_B32_7:%[0-9]+]]:sreg_32 = S_CSELECT_B32 [[COPY76]], [[COPY75]], implicit $scc
+; GCN-ISEL-NEXT: [[COPY73:%[0-9]+]]:sreg_32 = COPY [[S_ADD_U7]].sub0
+; GCN-ISEL-NEXT: S_CMP_LG_U32 [[FREEZE]], [[S_MOV_B32_9]], implicit-def $scc
+; GCN-ISEL-NEXT: [[S_CSELECT_B32_6:%[0-9]+]]:sreg_32 = S_CSELECT_B32 [[COPY73]], [[COPY72]], implicit $scc
+; GCN-ISEL-NEXT: [[COPY74:%[0-9]+]]:sreg_32 = COPY [[S_ADD_U6]].sub1
+; GCN-ISEL-NEXT: [[COPY75:%[0-9]+]]:sreg_32 = COPY [[S_ADD_U7]].sub1
+; GCN-ISEL-NEXT: [[S_CSELECT_B32_7:%[0-9]+]]:sreg_32 = S_CSELECT_B32 [[COPY75]], [[COPY74]], implicit $scc
; GCN-ISEL-NEXT: [[S_SUB_C4:%[0-9]+]]:sreg_32, [[S_SUB_C5:%[0-9]+]]:sreg_64_xexec = S_SUB_CO_PSEUDO [[COPY60]], [[S_ADD_I32_8]], [[S_USUBO1]], implicit-def dead $scc
; GCN-ISEL-NEXT: S_CMP_GE_U32 [[S_SUB_C4]], [[COPY22]], implicit-def $scc
; GCN-ISEL-NEXT: [[S_CSELECT_B32_8:%[0-9]+]]:sreg_32 = S_CSELECT_B32 [[S_MOV_B32_10]], [[S_MOV_B32_9]], implicit $scc
@@ -4250,25 +4250,25 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GCN-ISEL-NEXT: [[S_CSELECT_B32_9:%[0-9]+]]:sreg_32 = S_CSELECT_B32 [[S_MOV_B32_10]], [[S_MOV_B32_9]], implicit $scc
; GCN-ISEL-NEXT: S_CMP_EQ_U32 [[S_SUB_C4]], [[COPY22]], implicit-def $scc
; GCN-ISEL-NEXT: [[S_CSELECT_B32_10:%[0-9]+]]:sreg_32 = S_CSELECT_B32 [[S_CSELECT_B32_9]], [[S_CSELECT_B32_8]], implicit $scc
-; GCN-ISEL-NEXT: [[COPY77:%[0-9]+]]:sreg_32 = COPY [[S_CSELECT_B32_10]]
-; GCN-ISEL-NEXT: S_CMP_LG_U32 [[COPY77]], [[S_MOV_B32_9]], implicit-def $scc
+; GCN-ISEL-NEXT: [[FREEZE1:%[0-9]+]]:sreg_32 = FREEZE [[S_CSELECT_B32_10]]
+; GCN-ISEL-NEXT: S_CMP_LG_U32 [[FREEZE1]], [[S_MOV_B32_9]], implicit-def $scc
; GCN-ISEL-NEXT: [[S_CSELECT_B32_11:%[0-9]+]]:sreg_32 = S_CSELECT_B32 [[S_CSELECT_B32_7]], [[COPY68]], implicit $scc
; GCN-ISEL-NEXT: [[S_CSELECT_B32_12:%[0-9]+]]:sreg_32 = S_CSELECT_B32 [[S_CSELECT_B32_6]], [[COPY69]], implicit $scc
; GCN-ISEL-NEXT: [[REG_SEQUENCE23:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_CSELECT_B32_12]], %subreg.sub0, [[S_CSELECT_B32_11]], %subreg.sub1
-; GCN-ISEL-NEXT: [[COPY78:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE23]]
+; GCN-ISEL-NEXT: [[COPY76:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE23]]
; GCN-ISEL-NEXT: {{ $}}
; GCN-ISEL-NEXT: bb.3..split:
-; GCN-ISEL-NEXT: [[PHI:%[0-9]+]]:sreg_64 = PHI [[COPY78]], %bb.2, [[COPY20]], %bb.1
-; GCN-ISEL-NEXT: [[COPY79:%[0-9]+]]:sreg_32 = COPY [[COPY7]].sub1
-; GCN-ISEL-NEXT: [[COPY80:%[0-9]+]]:sreg_32 = COPY [[COPY7]].sub0
-; GCN-ISEL-NEXT: [[REG_SEQUENCE24:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY80]], %subreg.sub0, [[COPY79]], %subreg.sub1
-; GCN-ISEL-NEXT: [[COPY81:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE24]].sub1
-; GCN-ISEL-NEXT: [[COPY82:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE24]].sub0
+; GCN-ISEL-NEXT: [[PHI:%[0-9]+]]:sreg_64 = PHI [[COPY76]], %bb.2, [[COPY20]], %bb.1
+; GCN-ISEL-NEXT: [[COPY77:%[0-9]+]]:sreg_32 = COPY [[COPY7]].sub1
+; GCN-ISEL-NEXT: [[COPY78:%[0-9]+]]:sreg_32 = COPY [[COPY7]].sub0
+; GCN-ISEL-NEXT: [[REG_SEQUENCE24:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY78]], %subreg.sub0, [[COPY77]], %subreg.sub1
+; GCN-ISEL-NEXT: [[COPY79:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE24]].sub1
+; GCN-ISEL-NEXT: [[COPY80:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE24]].sub0
; GCN-ISEL-NEXT: [[S_MOV_B32_11:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
; GCN-ISEL-NEXT: [[S_MOV_B32_12:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
-; GCN-ISEL-NEXT: [[REG_SEQUENCE25:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY82]], %subreg.sub0, [[COPY81]], %subreg.sub1, [[S_MOV_B32_12]], %subreg.sub2, [[S_MOV_B32_11]], %subreg.sub3
-; GCN-ISEL-NEXT: [[COPY83:%[0-9]+]]:vreg_64 = COPY [[PHI]]
-; GCN-ISEL-NEXT: BUFFER_STORE_DWORDX2_OFFSET [[COPY83]], [[REG_SEQUENCE25]], 0, 0, 0, 0, implicit $exec :: (store (s64) into %ir.13, addrspace 1)
+; GCN-ISEL-NEXT: [[REG_SEQUENCE25:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY80]], %subreg.sub0, [[COPY79]], %subreg.sub1, [[S_MOV_B32_12]], %subreg.sub2, [[S_MOV_B32_11]], %subreg.sub3
+; GCN-ISEL-NEXT: [[COPY81:%[0-9]+]]:vreg_64 = COPY [[PHI]]
+; GCN-ISEL-NEXT: BUFFER_STORE_DWORDX2_OFFSET [[COPY81]], [[REG_SEQUENCE25]], 0, 0, 0, 0, implicit $exec :: (store (s64) into %ir.13, addrspace 1)
; GCN-ISEL-NEXT: S_ENDPGM 0
%result = udiv i64 %x, %y
store i64 %result, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/divergence-driven-trunc-to-i1.ll b/llvm/test/CodeGen/AMDGPU/divergence-driven-trunc-to-i1.ll
index 5589311e90d675..cd8d0a2606d2bf 100644
--- a/llvm/test/CodeGen/AMDGPU/divergence-driven-trunc-to-i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/divergence-driven-trunc-to-i1.ll
@@ -17,18 +17,18 @@ define amdgpu_kernel void @uniform_trunc_i16_to_i1(ptr addrspace(1) %out, i16 %x
; GCN-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 16
; GCN-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[S_LOAD_DWORD_IMM]], [[S_MOV_B32_2]], implicit-def dead $scc
; GCN-NEXT: [[S_LSHR_B32_:%[0-9]+]]:sreg_32 = S_LSHR_B32 [[S_LOAD_DWORD_IMM]], [[S_MOV_B32_2]], implicit-def dead $scc
- ; GCN-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY [[S_LSHR_B32_]]
- ; GCN-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 1, [[COPY3]], implicit-def dead $scc
+ ; GCN-NEXT: [[FREEZE:%[0-9]+]]:sreg_32 = FREEZE [[S_LSHR_B32_]]
+ ; GCN-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 1, [[FREEZE]], implicit-def dead $scc
; GCN-NEXT: S_CMP_EQ_U32 [[S_AND_B32_]], 1, implicit-def $scc
- ; GCN-NEXT: [[COPY4:%[0-9]+]]:sreg_64 = COPY $scc
+ ; GCN-NEXT: [[COPY3:%[0-9]+]]:sreg_64 = COPY $scc
; GCN-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; GCN-NEXT: S_CMP_LT_I32 [[S_LSHL_B32_]], [[S_MOV_B32_3]], implicit-def $scc
- ; GCN-NEXT: [[COPY5:%[0-9]+]]:sreg_64 = COPY $scc
- ; GCN-NEXT: [[S_OR_B64_:%[0-9]+]]:sreg_64 = S_OR_B64 [[COPY5]], [[COPY4]], implicit-def dead $scc
+ ; GCN-NEXT: [[COPY4:%[0-9]+]]:sreg_64 = COPY $scc
+ ; GCN-NEXT: [[S_OR_B64_:%[0-9]+]]:sreg_64 = S_OR_B64 [[COPY4]], [[COPY3]], implicit-def dead $scc
; GCN-NEXT: $scc = COPY [[S_OR_B64_]]
; GCN-NEXT: [[S_CSELECT_B32_:%[0-9]+]]:sreg_32 = S_CSELECT_B32 1, 0, implicit $scc
- ; GCN-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[S_CSELECT_B32_]]
- ; GCN-NEXT: BUFFER_STORE_BYTE_OFFSET [[COPY6]], [[REG_SEQUENCE]], 0, 0, 0, 0, implicit $exec :: (store (s8) into %ir.out.load, addrspace 1)
+ ; GCN-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[S_CSELECT_B32_]]
+ ; GCN-NEXT: BUFFER_STORE_BYTE_OFFSET [[COPY5]], [[REG_SEQUENCE]], 0, 0, 0, 0, implicit $exec :: (store (s8) into %ir.out.load, addrspace 1)
; GCN-NEXT: S_ENDPGM 0
%setcc = icmp slt i16 %x, 0
%select = select i1 %setcc, i1 true, i1 %z
@@ -71,20 +71,20 @@ define amdgpu_kernel void @uniform_trunc_i32_to_i1(ptr addrspace(1) %out, i32 %x
; GCN-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
; GCN-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[S_MOV_B32_1]], %subreg.sub2, [[S_MOV_B32_]], %subreg.sub3
- ; GCN-NEXT: [[COPY3:%[0-9]+]]:sreg_64 = COPY [[S_LOAD_DWORDX2_IMM1]]
- ; GCN-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY [[COPY3]].sub0
- ; GCN-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY [[COPY3]].sub1
- ; GCN-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 1, [[COPY5]], implicit-def dead $scc
+ ; GCN-NEXT: [[FREEZE:%[0-9]+]]:sreg_64 = FREEZE [[S_LOAD_DWORDX2_IMM1]]
+ ; GCN-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY [[FREEZE]].sub0
+ ; GCN-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY [[FREEZE]].sub1
+ ; GCN-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 1, [[COPY4]], implicit-def dead $scc
; GCN-NEXT: S_CMP_EQ_U32 [[S_AND_B32_]], 1, implicit-def $scc
- ; GCN-NEXT: [[COPY6:%[0-9]+]]:sreg_64 = COPY $scc
+ ; GCN-NEXT: [[COPY5:%[0-9]+]]:sreg_64 = COPY $scc
; GCN-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; GCN-NEXT: S_CMP_LT_I32 [[COPY4]], [[S_MOV_B32_2]], implicit-def $scc
- ; GCN-NEXT: [[COPY7:%[0-9]+]]:sreg_64 = COPY $scc
- ; GCN-NEXT: [[S_OR_B64_:%[0-9]+]]:sreg_64 = S_OR_B64 [[COPY7]], [[COPY6]], implicit-def dead $scc
+ ; GCN-NEXT: S_CMP_LT_I32 [[COPY3]], [[S_MOV_B32_2]], implicit-def $scc
+ ; GCN-NEXT: [[COPY6:%[0-9]+]]:sreg_64 = COPY $scc
+ ; GCN-NEXT: [[S_OR_B64_:%[0-9]+]]:sreg_64 = S_OR_B64 [[COPY6]], [[COPY5]], implicit-def dead $scc
; GCN-NEXT: $scc = COPY [[S_OR_B64_]]
; GCN-NEXT: [[S_CSELECT_B32_:%[0-9]+]]:sreg_32 = S_CSELECT_B32 1, 0, implicit $scc
- ; GCN-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_CSELECT_B32_]]
- ; GCN-NEXT: BUFFER_STORE_BYTE_OFFSET [[COPY8]], [[REG_SEQUENCE]], 0, 0, 0, 0, implicit $exec :: (store (s8) into %ir.out.load, addrspace 1)
+ ; GCN-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_CSELECT_B32_]]
+ ; GCN-NEXT: BUFFER_STORE_BYTE_OFFSET [[COPY7]], [[REG_SEQUENCE]], 0, 0, 0, 0, implicit $exec :: (store (s8) into %ir.out.load, addrspace 1)
; GCN-NEXT: S_ENDPGM 0
%setcc = icmp slt i32 %x, 0
%select = select i1 %setcc, i1 true, i1 %z
@@ -129,18 +129,18 @@ define amdgpu_kernel void @uniform_trunc_i64_to_i1(ptr addrspace(1) %out, i64 %x
; GCN-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
; GCN-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[S_MOV_B32_1]], %subreg.sub2, [[S_MOV_B32_]], %subreg.sub3
; GCN-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub3
- ; GCN-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORD_IMM]]
- ; GCN-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 1, [[COPY6]], implicit-def dead $scc
+ ; GCN-NEXT: [[FREEZE:%[0-9]+]]:sreg_32 = FREEZE [[S_LOAD_DWORD_IMM]]
+ ; GCN-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 1, [[FREEZE]], implicit-def dead $scc
; GCN-NEXT: S_CMP_EQ_U32 [[S_AND_B32_]], 1, implicit-def $scc
- ; GCN-NEXT: [[COPY7:%[0-9]+]]:sreg_64 = COPY $scc
+ ; GCN-NEXT: [[COPY6:%[0-9]+]]:sreg_64 = COPY $scc
; GCN-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; GCN-NEXT: S_CMP_LT_I32 [[COPY5]], [[S_MOV_B32_2]], implicit-def $scc
- ; GCN-NEXT: [[COPY8:%[0-9]+]]:sreg_64 = COPY $scc
- ; GCN-NEXT: [[S_OR_B64_:%[0-9]+]]:sreg_64 = S_OR_B64 [[COPY8]], [[COPY7]], implicit-def dead $scc
+ ; GCN-NEXT: [[COPY7:%[0-9]+]]:sreg_64 = COPY $scc
+ ; GCN-NEXT: [[S_OR_B64_:%[0-9]+]]:sreg_64 = S_OR_B64 [[COPY7]], [[COPY6]], implicit-def dead $scc
; GCN-NEXT: $scc = COPY [[S_OR_B64_]]
; GCN-NEXT: [[S_CSELECT_B32_:%[0-9]+]]:sreg_32 = S_CSELECT_B32 1, 0, implicit $scc
- ; GCN-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[S_CSELECT_B32_]]
- ; GCN-NEXT: BUFFER_STORE_BYTE_OFFSET [[COPY9]], [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec :: (store (s8) into %ir.2, addrspace 1)
+ ; GCN-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_CSELECT_B32_]]
+ ; GCN-NEXT: BUFFER_STORE_BYTE_OFFSET [[COPY8]], [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec :: (store (s8) into %ir.2, addrspace 1)
; GCN-NEXT: S_ENDPGM 0
%setcc = icmp slt i64 %x, 0
%select = select i1 %setcc, i1 true, i1 %z
diff --git a/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-hw.ll b/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-hw.ll
index 954d6a3f54cec8..d2298cc92c401d 100644
--- a/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-hw.ll
+++ b/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-hw.ll
@@ -1165,21 +1165,19 @@ define i8 @to_fp8_f64(double %x) {
; GFX950-NEXT: v_lshrrev_b64 v[12:13], v6, v[12:13]
; GFX950-NEXT: v_and_b32_e32 v6, v12, v14
; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GFX950-NEXT: s_movk_i32 s1, 0x80
; GFX950-NEXT: s_mov_b32 s0, 0
+; GFX950-NEXT: s_movk_i32 s1, 0x80
; GFX950-NEXT: v_cndmask_b32_e32 v6, 0, v6, vcc
; GFX950-NEXT: v_lshl_add_u64 v[4:5], v[8:9], 0, v[6:7]
; GFX950-NEXT: v_cmp_lt_i64_e32 vcc, 3, v[4:5]
-; GFX950-NEXT: v_and_b32_sdwa v8, v1, s1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX950-NEXT: v_lshrrev_b32_e32 v6, 17, v11
-; GFX950-NEXT: v_cndmask_b32_e64 v4, v4, 0, vcc
-; GFX950-NEXT: v_cndmask_b32_e64 v5, 0, 4, vcc
-; GFX950-NEXT: v_or3_b32 v9, v8, v5, v4
; GFX950-NEXT: v_and_b32_e32 v5, 0x1ffff, v11
+; GFX950-NEXT: v_lshrrev_b32_e32 v6, 17, v11
+; GFX950-NEXT: v_cndmask_b32_e64 v8, v4, 0, vcc
; GFX950-NEXT: v_mov_b32_e32 v4, v10
+; GFX950-NEXT: v_cndmask_b32_e64 v9, 0, 4, vcc
; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GFX950-NEXT: v_bfe_u32 v4, v11, 18, 2
-; GFX950-NEXT: s_nop 0
+; GFX950-NEXT: v_and_b32_sdwa v12, v1, s1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
; GFX950-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
; GFX950-NEXT: v_and_or_b32 v5, v4, 1, v5
; GFX950-NEXT: v_and_b32_e32 v6, v6, v5
@@ -1187,29 +1185,29 @@ define i8 @to_fp8_f64(double %x) {
; GFX950-NEXT: v_lshl_add_u64 v[4:5], v[4:5], 0, v[6:7]
; GFX950-NEXT: v_cmp_lt_i64_e32 vcc, 3, v[4:5]
; GFX950-NEXT: v_mov_b32_e32 v7, s0
-; GFX950-NEXT: s_nop 0
+; GFX950-NEXT: v_or3_b32 v8, v12, v9, v8
; GFX950-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
; GFX950-NEXT: v_lshl_add_u64 v[2:3], v[2:3], 0, v[6:7]
; GFX950-NEXT: v_lshl_add_u64 v[2:3], v[2:3], 0, 14
; GFX950-NEXT: v_lshlrev_b64 v[6:7], 2, v[2:3]
; GFX950-NEXT: v_cndmask_b32_e64 v4, v4, 0, vcc
-; GFX950-NEXT: v_or_b32_e32 v6, v6, v8
+; GFX950-NEXT: v_or_b32_e32 v6, v6, v12
; GFX950-NEXT: v_cndmask_b32_e64 v5, v5, 0, vcc
; GFX950-NEXT: v_or_b32_e32 v6, v6, v4
; GFX950-NEXT: v_cmp_gt_i64_e32 vcc, 1, v[2:3]
; GFX950-NEXT: v_cmp_eq_u64_e64 s[0:1], 30, v[2:3]
; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_cndmask_b32_e32 v6, v6, v9, vcc
+; GFX950-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc
; GFX950-NEXT: v_cmp_lt_i64_e32 vcc, 3, v[4:5]
; GFX950-NEXT: s_and_b64 s[0:1], s[0:1], vcc
; GFX950-NEXT: v_cmp_lt_i64_e32 vcc, 30, v[2:3]
-; GFX950-NEXT: v_or_b32_e32 v2, 0x7c, v8
+; GFX950-NEXT: v_or_b32_e32 v2, 0x7c, v12
; GFX950-NEXT: s_or_b64 vcc, vcc, s[0:1]
; GFX950-NEXT: v_cndmask_b32_e32 v3, v6, v2, vcc
; GFX950-NEXT: v_cmp_eq_f64_e32 vcc, 0, v[0:1]
; GFX950-NEXT: s_movk_i32 s0, 0x204
; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_cndmask_b32_e32 v3, v3, v8, vcc
+; GFX950-NEXT: v_cndmask_b32_e32 v3, v3, v12, vcc
; GFX950-NEXT: v_cmp_class_f64_e64 vcc, v[0:1], s0
; GFX950-NEXT: s_nop 1
; GFX950-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc
diff --git a/llvm/test/CodeGen/AMDGPU/fold-fabs.ll b/llvm/test/CodeGen/AMDGPU/fold-fabs.ll
index 0dbbae3c94f387..608afa627df767 100644
--- a/llvm/test/CodeGen/AMDGPU/fold-fabs.ll
+++ b/llvm/test/CodeGen/AMDGPU/fold-fabs.ll
@@ -70,6 +70,7 @@ exit:
define float @fold_abs_in_branch_undef(float %arg1, float %arg2) {
; GFX10-LABEL: fold_abs_in_branch_undef:
; GFX10: ; %bb.0: ; %entry
+; GFX10-NEXT: ; freeze: $sgpr4
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_f32_e64 v0, |s4|, |s4|
; GFX10-NEXT: v_cmp_lt_f32_e32 vcc_lo, 1.0, v0
diff --git a/llvm/test/CodeGen/AMDGPU/freeze-multi-use.ll b/llvm/test/CodeGen/AMDGPU/freeze-multi-use.ll
index 08422bf946c3b3..a0ce9aec62f12d 100644
--- a/llvm/test/CodeGen/AMDGPU/freeze-multi-use.ll
+++ b/llvm/test/CodeGen/AMDGPU/freeze-multi-use.ll
@@ -7,10 +7,11 @@
define i32 @ashr_mul(i32 %x) {
; CHECK-LABEL: ashr_mul:
; CHECK: ; %bb.0:
+; CHECK-NEXT: ; freeze: $sgpr0
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: s_ashr_i32 s0, s0, 3
+; CHECK-NEXT: s_ashr_i32 s1, s0, 3
; CHECK-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; CHECK-NEXT: s_mul_i32 s0, s0, s0
+; CHECK-NEXT: s_mul_i32 s0, s0, s1
; CHECK-NEXT: v_mov_b32_e32 v0, s0
; CHECK-NEXT: s_setpc_b64 s[30:31]
%p = ashr i32 %x, 32
@@ -30,20 +31,25 @@ define i32 @across_call() nounwind {
; CHECK-NEXT: s_or_saveexec_b32 s1, -1
; CHECK-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
; CHECK-NEXT: s_mov_b32 exec_lo, s1
-; CHECK-NEXT: v_writelane_b32 v40, s0, 2
+; CHECK-NEXT: v_writelane_b32 v40, s0, 3
; CHECK-NEXT: s_add_i32 s32, s32, 16
-; CHECK-NEXT: v_writelane_b32 v40, s30, 0
-; CHECK-NEXT: v_writelane_b32 v40, s31, 1
+; CHECK-NEXT: v_writelane_b32 v40, s34, 0
+; CHECK-NEXT: v_writelane_b32 v40, s30, 1
+; CHECK-NEXT: v_writelane_b32 v40, s31, 2
; CHECK-NEXT: s_getpc_b64 s[0:1]
; CHECK-NEXT: s_add_u32 s0, s0, use at gotpcrel32@lo+4
; CHECK-NEXT: s_addc_u32 s1, s1, use at gotpcrel32@hi+12
; CHECK-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; CHECK-NEXT: ; freeze: $sgpr34
+; CHECK-NEXT: v_mov_b32_e32 v0, s34
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: s_swappc_b64 s[30:31], s[0:1]
-; CHECK-NEXT: v_readlane_b32 s30, v40, 0
-; CHECK-NEXT: v_readlane_b32 s31, v40, 1
+; CHECK-NEXT: v_readlane_b32 s30, v40, 1
+; CHECK-NEXT: v_mov_b32_e32 v0, s34
+; CHECK-NEXT: v_readlane_b32 s31, v40, 2
+; CHECK-NEXT: v_readlane_b32 s34, v40, 0
; CHECK-NEXT: s_mov_b32 s32, s33
-; CHECK-NEXT: v_readlane_b32 s0, v40, 2
+; CHECK-NEXT: v_readlane_b32 s0, v40, 3
; CHECK-NEXT: s_or_saveexec_b32 s1, -1
; CHECK-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
; CHECK-NEXT: s_mov_b32 exec_lo, s1
@@ -65,20 +71,33 @@ define <4 x i32> @vec_across_call() nounwind {
; CHECK-NEXT: s_or_saveexec_b32 s1, -1
; CHECK-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
; CHECK-NEXT: s_mov_b32 exec_lo, s1
-; CHECK-NEXT: v_writelane_b32 v40, s0, 2
+; CHECK-NEXT: v_writelane_b32 v40, s0, 6
; CHECK-NEXT: s_add_i32 s32, s32, 16
-; CHECK-NEXT: v_writelane_b32 v40, s30, 0
-; CHECK-NEXT: v_writelane_b32 v40, s31, 1
+; CHECK-NEXT: v_writelane_b32 v40, s36, 0
+; CHECK-NEXT: v_writelane_b32 v40, s37, 1
+; CHECK-NEXT: v_writelane_b32 v40, s38, 2
+; CHECK-NEXT: v_writelane_b32 v40, s39, 3
+; CHECK-NEXT: v_writelane_b32 v40, s30, 4
+; CHECK-NEXT: v_writelane_b32 v40, s31, 5
; CHECK-NEXT: s_getpc_b64 s[0:1]
; CHECK-NEXT: s_add_u32 s0, s0, usev at gotpcrel32@lo+4
; CHECK-NEXT: s_addc_u32 s1, s1, usev at gotpcrel32@hi+12
; CHECK-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; CHECK-NEXT: ; freeze: $sgpr36_sgpr37_sgpr38_sgpr39
+; CHECK-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
+; CHECK-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: s_swappc_b64 s[30:31], s[0:1]
-; CHECK-NEXT: v_readlane_b32 s30, v40, 0
-; CHECK-NEXT: v_readlane_b32 s31, v40, 1
+; CHECK-NEXT: v_readlane_b32 s30, v40, 4
+; CHECK-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
+; CHECK-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
+; CHECK-NEXT: v_readlane_b32 s31, v40, 5
+; CHECK-NEXT: v_readlane_b32 s39, v40, 3
+; CHECK-NEXT: v_readlane_b32 s38, v40, 2
+; CHECK-NEXT: v_readlane_b32 s37, v40, 1
+; CHECK-NEXT: v_readlane_b32 s36, v40, 0
; CHECK-NEXT: s_mov_b32 s32, s33
-; CHECK-NEXT: v_readlane_b32 s0, v40, 2
+; CHECK-NEXT: v_readlane_b32 s0, v40, 6
; CHECK-NEXT: s_or_saveexec_b32 s1, -1
; CHECK-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
; CHECK-NEXT: s_mov_b32 exec_lo, s1
diff --git a/llvm/test/CodeGen/AMDGPU/freeze.ll b/llvm/test/CodeGen/AMDGPU/freeze.ll
index ae9c97cc8d139b..8f171d12b8bf9c 100644
--- a/llvm/test/CodeGen/AMDGPU/freeze.ll
+++ b/llvm/test/CodeGen/AMDGPU/freeze.ll
@@ -16908,9 +16908,11 @@ define double @freeze_fabs_double(float %a, double %b, double %c) {
; GFX6-SDAG-LABEL: freeze_fabs_double:
; GFX6-SDAG: ; %bb.0:
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-SDAG-NEXT: v_mov_b32_e32 v5, v0
-; GFX6-SDAG-NEXT: v_add_f64 v[0:1], |v[4:5]|, v[1:2]
-; GFX6-SDAG-NEXT: v_add_f64 v[2:3], |v[4:5]|, v[3:4]
+; GFX6-SDAG-NEXT: v_mov_b32_e32 v6, v0
+; GFX6-SDAG-NEXT: ; freeze: $sgpr4
+; GFX6-SDAG-NEXT: v_mov_b32_e32 v5, s4
+; GFX6-SDAG-NEXT: v_add_f64 v[0:1], |v[5:6]|, v[1:2]
+; GFX6-SDAG-NEXT: v_add_f64 v[2:3], |v[5:6]|, v[3:4]
; GFX6-SDAG-NEXT: v_add_f64 v[0:1], v[0:1], v[2:3]
; GFX6-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -16926,9 +16928,11 @@ define double @freeze_fabs_double(float %a, double %b, double %c) {
; GFX7-SDAG-LABEL: freeze_fabs_double:
; GFX7-SDAG: ; %bb.0:
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-SDAG-NEXT: v_mov_b32_e32 v5, v0
-; GFX7-SDAG-NEXT: v_add_f64 v[0:1], |v[4:5]|, v[1:2]
-; GFX7-SDAG-NEXT: v_add_f64 v[2:3], |v[4:5]|, v[3:4]
+; GFX7-SDAG-NEXT: v_mov_b32_e32 v6, v0
+; GFX7-SDAG-NEXT: ; freeze: $sgpr4
+; GFX7-SDAG-NEXT: v_mov_b32_e32 v5, s4
+; GFX7-SDAG-NEXT: v_add_f64 v[0:1], |v[5:6]|, v[1:2]
+; GFX7-SDAG-NEXT: v_add_f64 v[2:3], |v[5:6]|, v[3:4]
; GFX7-SDAG-NEXT: v_add_f64 v[0:1], v[0:1], v[2:3]
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -16944,9 +16948,11 @@ define double @freeze_fabs_double(float %a, double %b, double %c) {
; GFX8-SDAG-LABEL: freeze_fabs_double:
; GFX8-SDAG: ; %bb.0:
; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-SDAG-NEXT: v_mov_b32_e32 v5, v0
-; GFX8-SDAG-NEXT: v_add_f64 v[0:1], |v[4:5]|, v[1:2]
-; GFX8-SDAG-NEXT: v_add_f64 v[2:3], |v[4:5]|, v[3:4]
+; GFX8-SDAG-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-SDAG-NEXT: ; freeze: $sgpr4
+; GFX8-SDAG-NEXT: v_mov_b32_e32 v5, s4
+; GFX8-SDAG-NEXT: v_add_f64 v[0:1], |v[5:6]|, v[1:2]
+; GFX8-SDAG-NEXT: v_add_f64 v[2:3], |v[5:6]|, v[3:4]
; GFX8-SDAG-NEXT: v_add_f64 v[0:1], v[0:1], v[2:3]
; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -16971,9 +16977,11 @@ define double @freeze_fabs_double(float %a, double %b, double %c) {
; GFX10-SDAG-LABEL: freeze_fabs_double:
; GFX10-SDAG: ; %bb.0:
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v5, v0
-; GFX10-SDAG-NEXT: v_add_f64 v[0:1], |v[4:5]|, v[1:2]
-; GFX10-SDAG-NEXT: v_add_f64 v[2:3], |v[4:5]|, v[3:4]
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v6, v0
+; GFX10-SDAG-NEXT: ; freeze: $sgpr4
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v5, s4
+; GFX10-SDAG-NEXT: v_add_f64 v[0:1], |v[5:6]|, v[1:2]
+; GFX10-SDAG-NEXT: v_add_f64 v[2:3], |v[5:6]|, v[3:4]
; GFX10-SDAG-NEXT: v_add_f64 v[0:1], v[0:1], v[2:3]
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -16988,10 +16996,11 @@ define double @freeze_fabs_double(float %a, double %b, double %c) {
;
; GFX11-SDAG-LABEL: freeze_fabs_double:
; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: ; freeze: $sgpr0
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v5, v0
-; GFX11-SDAG-NEXT: v_add_f64 v[0:1], |v[4:5]|, v[1:2]
-; GFX11-SDAG-NEXT: v_add_f64 v[2:3], |v[4:5]|, v[3:4]
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v6, v0 :: v_dual_mov_b32 v5, s0
+; GFX11-SDAG-NEXT: v_add_f64 v[0:1], |v[5:6]|, v[1:2]
+; GFX11-SDAG-NEXT: v_add_f64 v[2:3], |v[5:6]|, v[3:4]
; GFX11-SDAG-NEXT: v_add_f64 v[0:1], v[0:1], v[2:3]
; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane.ll
index ae1f9ff278c300..7a9f4603a976b9 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane.ll
@@ -7349,369 +7349,499 @@ define amdgpu_kernel void @v_permlane16_b32_tid_tid_f64(ptr addrspace(1) %out, f
}
define amdgpu_kernel void @v_permlane16_b32_undef_tid_i32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-LABEL: v_permlane16_b32_undef_tid_i32:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_permlane16_b32 v0, v0, s0, s1
-; GFX10-NEXT: global_store_dword v1, v0, s[2:3]
-; GFX10-NEXT: s_endpgm
-;
-; GFX11-LABEL: v_permlane16_b32_undef_tid_i32:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_permlane16_b32 v0, v0, s0, s1
-; GFX11-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX11-NEXT: s_endpgm
-;
-; GFX12-LABEL: v_permlane16_b32_undef_tid_i32:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
-; GFX12-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_permlane16_b32 v0, v0, s0, s1
-; GFX12-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX12-NEXT: s_endpgm
-;
-; GFX13-LABEL: v_permlane16_b32_undef_tid_i32:
-; GFX13: ; %bb.0:
-; GFX13-NEXT: s_clause 0x1
-; GFX13-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-NEXT: s_mov_b32 s4, 0x3ff
-; GFX13-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
-; GFX13-NEXT: s_wait_kmcnt 0x0
-; GFX13-NEXT: v_permlane16_b32 v0, v0, s0, s1
-; GFX13-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX13-NEXT: s_endpgm
- %tidx = call i32 @llvm.amdgcn.workitem.id.x()
- %undef = freeze i32 poison
- %v = call i32 @llvm.amdgcn.permlane16.i32(i32 %undef, i32 %tidx, i32 %src1, i32 %src2, i1 false, i1 false)
- store i32 %v, ptr addrspace(1) %out
- ret void
-}
-
-define amdgpu_kernel void @v_permlane16_b32_undef_tid_f32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-LABEL: v_permlane16_b32_undef_tid_f32:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_permlane16_b32 v0, v0, s0, s1
-; GFX10-NEXT: global_store_dword v1, v0, s[2:3]
-; GFX10-NEXT: s_endpgm
-;
-; GFX11-LABEL: v_permlane16_b32_undef_tid_f32:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_permlane16_b32 v0, v0, s0, s1
-; GFX11-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX11-NEXT: s_endpgm
-;
-; GFX12-LABEL: v_permlane16_b32_undef_tid_f32:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
-; GFX12-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_permlane16_b32 v0, v0, s0, s1
-; GFX12-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX12-NEXT: s_endpgm
-;
-; GFX13-LABEL: v_permlane16_b32_undef_tid_f32:
-; GFX13: ; %bb.0:
-; GFX13-NEXT: s_clause 0x1
-; GFX13-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-NEXT: s_mov_b32 s4, 0x3ff
-; GFX13-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
-; GFX13-NEXT: s_wait_kmcnt 0x0
-; GFX13-NEXT: v_permlane16_b32 v0, v0, s0, s1
-; GFX13-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX13-NEXT: s_endpgm
- %tidx = call i32 @llvm.amdgcn.workitem.id.x()
- %tidx_f32 = bitcast i32 %tidx to float
- %undef = freeze float poison
- %v = call float @llvm.amdgcn.permlane16.f32(float %undef, float %tidx_f32, i32 %src1, i32 %src2, i1 false, i1 false)
- store float %v, ptr addrspace(1) %out
- ret void
-}
-
-define amdgpu_kernel void @v_permlane16_b32_undef_tid_i64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_undef_tid_i64:
+; GFX10-SDAG-LABEL: v_permlane16_b32_undef_tid_i32:
; GFX10-SDAG: ; %bb.0:
; GFX10-SDAG-NEXT: s_clause 0x1
; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT: ; freeze: $sgpr4
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s4
; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v2, s0, s1
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1
+; GFX10-SDAG-NEXT: global_store_dword v2, v1, s[2:3]
; GFX10-SDAG-NEXT: s_endpgm
;
-; GFX10-GISEL-LABEL: v_permlane16_b32_undef_tid_i64:
+; GFX10-GISEL-LABEL: v_permlane16_b32_undef_tid_i32:
; GFX10-GISEL: ; %bb.0:
; GFX10-GISEL-NEXT: s_clause 0x1
; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1
-; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v2, s0, s1
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[2:3]
; GFX10-GISEL-NEXT: s_endpgm
;
-; GFX11-SDAG-LABEL: v_permlane16_b32_undef_tid_i64:
+; GFX11-SDAG-LABEL: v_permlane16_b32_undef_tid_i32:
; GFX11-SDAG: ; %bb.0:
; GFX11-SDAG-NEXT: s_clause 0x1
; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
; GFX11-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT: ; freeze: $sgpr4
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT: v_permlane16_b32 v1, v2, s0, s1
-; GFX11-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1
-; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1
+; GFX11-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX11-GISEL-LABEL: v_permlane16_b32_undef_tid_i64:
+; GFX11-GISEL-LABEL: v_permlane16_b32_undef_tid_i32:
; GFX11-GISEL: ; %bb.0:
; GFX11-GISEL-NEXT: s_clause 0x1
; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
; GFX11-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1
-; GFX11-GISEL-NEXT: v_permlane16_b32 v1, v2, s0, s1
-; GFX11-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
; GFX11-GISEL-NEXT: s_endpgm
;
-; GFX12-SDAG-LABEL: v_permlane16_b32_undef_tid_i64:
+; GFX12-SDAG-LABEL: v_permlane16_b32_undef_tid_i32:
; GFX12-SDAG: ; %bb.0:
; GFX12-SDAG-NEXT: s_clause 0x1
; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
; GFX12-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX12-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT: ; freeze: $sgpr4
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-NEXT: v_permlane16_b32 v1, v2, s0, s1
-; GFX12-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1
-; GFX12-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1
+; GFX12-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
; GFX12-SDAG-NEXT: s_endpgm
;
-; GFX12-GISEL-LABEL: v_permlane16_b32_undef_tid_i64:
+; GFX12-GISEL-LABEL: v_permlane16_b32_undef_tid_i32:
; GFX12-GISEL: ; %bb.0:
; GFX12-GISEL-NEXT: s_clause 0x1
; GFX12-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
; GFX12-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1
-; GFX12-GISEL-NEXT: v_permlane16_b32 v1, v2, s0, s1
-; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlane16_b32_undef_tid_i64:
+; GFX13-SDAG-LABEL: v_permlane16_b32_undef_tid_i32:
; GFX13-SDAG: ; %bb.0:
; GFX13-SDAG-NEXT: s_clause 0x1
; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: s_mov_b32 s4, 0x3ff
+; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-SDAG-NEXT: ; freeze: $sgpr4
+; GFX13-SDAG-NEXT: s_mov_b32 s5, 0x3ff
; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v2, s0, s1
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1
+; GFX13-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
; GFX13-SDAG-NEXT: s_endpgm
;
-; GFX13-GISEL-LABEL: v_permlane16_b32_undef_tid_i64:
+; GFX13-GISEL-LABEL: v_permlane16_b32_undef_tid_i32:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_clause 0x1
; GFX13-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
; GFX13-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
; GFX13-GISEL-NEXT: s_mov_b32 s4, 0x3ff
; GFX13-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-GISEL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
; GFX13-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX13-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1
-; GFX13-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX13-GISEL-NEXT: v_permlane16_b32 v1, v2, s0, s1
-; GFX13-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
; GFX13-GISEL-NEXT: s_endpgm
%tidx = call i32 @llvm.amdgcn.workitem.id.x()
- %tidx_i64 = zext i32 %tidx to i64
- %undef = freeze i64 poison
- %v = call i64 @llvm.amdgcn.permlane16.i64(i64 %undef, i64 %tidx_i64, i32 %src1, i32 %src2, i1 false, i1 false)
- store i64 %v, ptr addrspace(1) %out
+ %undef = freeze i32 poison
+ %v = call i32 @llvm.amdgcn.permlane16.i32(i32 %undef, i32 %tidx, i32 %src1, i32 %src2, i1 false, i1 false)
+ store i32 %v, ptr addrspace(1) %out
ret void
}
-define amdgpu_kernel void @v_permlane16_b32_undef_tid_f64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_undef_tid_f64:
+define amdgpu_kernel void @v_permlane16_b32_undef_tid_f32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
+; GFX10-SDAG-LABEL: v_permlane16_b32_undef_tid_f32:
; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
; GFX10-SDAG-NEXT: s_clause 0x1
; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT: ; freeze: $sgpr4
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s4
; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v1, s0, s1
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1
+; GFX10-SDAG-NEXT: global_store_dword v2, v1, s[2:3]
; GFX10-SDAG-NEXT: s_endpgm
;
-; GFX10-GISEL-LABEL: v_permlane16_b32_undef_tid_f64:
+; GFX10-GISEL-LABEL: v_permlane16_b32_undef_tid_f32:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
; GFX10-GISEL-NEXT: s_clause 0x1
; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1
-; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v1, s0, s1
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[2:3]
; GFX10-GISEL-NEXT: s_endpgm
;
-; GFX11-SDAG-LABEL: v_permlane16_b32_undef_tid_f64:
+; GFX11-SDAG-LABEL: v_permlane16_b32_undef_tid_f32:
; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-SDAG-NEXT: s_clause 0x1
; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
; GFX11-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX11-SDAG-NEXT: ; freeze: $sgpr4
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT: v_permlane16_b32 v1, v1, s0, s1
-; GFX11-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1
-; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1
+; GFX11-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX11-GISEL-LABEL: v_permlane16_b32_undef_tid_f64:
+; GFX11-GISEL-LABEL: v_permlane16_b32_undef_tid_f32:
; GFX11-GISEL: ; %bb.0:
-; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-GISEL-NEXT: s_clause 0x1
; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
; GFX11-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX11-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1
-; GFX11-GISEL-NEXT: v_permlane16_b32 v1, v1, s0, s1
-; GFX11-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
; GFX11-GISEL-NEXT: s_endpgm
;
-; GFX12-SDAG-LABEL: v_permlane16_b32_undef_tid_f64:
+; GFX12-SDAG-LABEL: v_permlane16_b32_undef_tid_f32:
; GFX12-SDAG: ; %bb.0:
-; GFX12-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-SDAG-NEXT: s_clause 0x1
; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
; GFX12-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX12-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX12-SDAG-NEXT: ; freeze: $sgpr4
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-NEXT: v_permlane16_b32 v1, v1, s0, s1
-; GFX12-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1
-; GFX12-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1
+; GFX12-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
; GFX12-SDAG-NEXT: s_endpgm
;
-; GFX12-GISEL-LABEL: v_permlane16_b32_undef_tid_f64:
+; GFX12-GISEL-LABEL: v_permlane16_b32_undef_tid_f32:
; GFX12-GISEL: ; %bb.0:
-; GFX12-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-GISEL-NEXT: s_clause 0x1
; GFX12-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
; GFX12-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX12-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX12-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1
-; GFX12-GISEL-NEXT: v_permlane16_b32 v1, v1, s0, s1
-; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlane16_b32_undef_tid_f64:
+; GFX13-SDAG-LABEL: v_permlane16_b32_undef_tid_f32:
; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_mov_b32 s0, 0x3ff
-; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s0, v0 bitop3:0x40
; GFX13-SDAG-NEXT: s_clause 0x1
; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-SDAG-NEXT: ; freeze: $sgpr4
+; GFX13-SDAG-NEXT: s_mov_b32 s5, 0x3ff
+; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v1, s0, s1
-; GFX13-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1
+; GFX13-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
; GFX13-SDAG-NEXT: s_endpgm
;
-; GFX13-GISEL-LABEL: v_permlane16_b32_undef_tid_f64:
+; GFX13-GISEL-LABEL: v_permlane16_b32_undef_tid_f32:
; GFX13-GISEL: ; %bb.0:
-; GFX13-GISEL-NEXT: s_mov_b32 s0, 0x3ff
-; GFX13-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX13-GISEL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s0, v0 bitop3:0x40
; GFX13-GISEL-NEXT: s_clause 0x1
; GFX13-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
; GFX13-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX13-GISEL-NEXT: s_mov_b32 s4, 0x3ff
+; GFX13-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
; GFX13-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX13-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX13-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1
-; GFX13-GISEL-NEXT: v_permlane16_b32 v1, v1, s0, s1
-; GFX13-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
; GFX13-GISEL-NEXT: s_endpgm
%tidx = call i32 @llvm.amdgcn.workitem.id.x()
%tidx_f32 = bitcast i32 %tidx to float
- %tidx_f64 = fpext float %tidx_f32 to double
- %undef = freeze double poison
- %v = call double @llvm.amdgcn.permlane16.f64(double %undef, double %tidx_f64, i32 %src1, i32 %src2, i1 false, i1 false)
- store double %v, ptr addrspace(1) %out
+ %undef = freeze float poison
+ %v = call float @llvm.amdgcn.permlane16.f32(float %undef, float %tidx_f32, i32 %src1, i32 %src2, i1 false, i1 false)
+ store float %v, ptr addrspace(1) %out
ret void
}
-define amdgpu_kernel void @v_permlane16_b32_i_tid_i32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_i_tid_i32:
+define amdgpu_kernel void @v_permlane16_b32_undef_tid_i64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
+; GFX10-SDAG-LABEL: v_permlane16_b32_undef_tid_i64:
; GFX10-SDAG: ; %bb.0:
; GFX10-SDAG-NEXT: s_clause 0x1
; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, 0x3039
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v3, 0
+; GFX10-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, s5
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s4
+; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_permlane16_b32 v2, v3, s0, s1
+; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1
+; GFX10-SDAG-NEXT: global_store_dwordx2 v3, v[1:2], s[2:3]
+; GFX10-SDAG-NEXT: s_endpgm
+;
+; GFX10-GISEL-LABEL: v_permlane16_b32_undef_tid_i64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1
+; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v2, s0, s1
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-GISEL-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: v_permlane16_b32_undef_tid_i64:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_clause 0x1
+; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_permlane16_b32 v2, v3, s0, s1
+; GFX11-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1
+; GFX11-SDAG-NEXT: global_store_b64 v3, v[1:2], s[2:3]
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: v_permlane16_b32_undef_tid_i64:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1
+; GFX11-GISEL-NEXT: v_permlane16_b32 v1, v2, s0, s1
+; GFX11-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: v_permlane16_b32_undef_tid_i64:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_clause 0x1
+; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_permlane16_b32 v2, v3, s0, s1
+; GFX12-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1
+; GFX12-SDAG-NEXT: global_store_b64 v3, v[1:2], s[2:3]
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: v_permlane16_b32_undef_tid_i64:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_clause 0x1
+; GFX12-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1
+; GFX12-GISEL-NEXT: v_permlane16_b32 v1, v2, s0, s1
+; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-GISEL-NEXT: s_endpgm
+;
+; GFX13-SDAG-LABEL: v_permlane16_b32_undef_tid_i64:
+; GFX13-SDAG: ; %bb.0:
+; GFX13-SDAG-NEXT: s_clause 0x1
+; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX13-SDAG-NEXT: s_mov_b32 s5, 0x3ff
+; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
+; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX13-SDAG-NEXT: v_permlane16_b32 v2, v3, s0, s1
+; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1
+; GFX13-SDAG-NEXT: global_store_b64 v3, v[1:2], s[2:3]
+; GFX13-SDAG-NEXT: s_endpgm
+;
+; GFX13-GISEL-LABEL: v_permlane16_b32_undef_tid_i64:
+; GFX13-GISEL: ; %bb.0:
+; GFX13-GISEL-NEXT: s_clause 0x1
+; GFX13-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-GISEL-NEXT: s_mov_b32 s4, 0x3ff
+; GFX13-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-GISEL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX13-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1
+; GFX13-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX13-GISEL-NEXT: v_permlane16_b32 v1, v2, s0, s1
+; GFX13-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-GISEL-NEXT: s_endpgm
+ %tidx = call i32 @llvm.amdgcn.workitem.id.x()
+ %tidx_i64 = zext i32 %tidx to i64
+ %undef = freeze i64 poison
+ %v = call i64 @llvm.amdgcn.permlane16.i64(i64 %undef, i64 %tidx_i64, i32 %src1, i32 %src2, i1 false, i1 false)
+ store i64 %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane16_b32_undef_tid_f64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
+; GFX10-SDAG-LABEL: v_permlane16_b32_undef_tid_f64:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_clause 0x1
+; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; GFX10-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v3, s5
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, s4
+; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_permlane16_b32 v3, v1, s0, s1
+; GFX10-SDAG-NEXT: v_permlane16_b32 v2, v0, s0, s1
+; GFX10-SDAG-NEXT: global_store_dwordx2 v4, v[2:3], s[2:3]
+; GFX10-SDAG-NEXT: s_endpgm
+;
+; GFX10-GISEL-LABEL: v_permlane16_b32_undef_tid_f64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1
+; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v1, s0, s1
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-GISEL-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: v_permlane16_b32_undef_tid_f64:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_clause 0x1
+; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; GFX11-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v3, s5 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, s4
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_permlane16_b32 v3, v1, s0, s1
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_permlane16_b32 v2, v0, s0, s1
+; GFX11-SDAG-NEXT: global_store_b64 v4, v[2:3], s[2:3]
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: v_permlane16_b32_undef_tid_f64:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1
+; GFX11-GISEL-NEXT: v_permlane16_b32 v1, v1, s0, s1
+; GFX11-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: v_permlane16_b32_undef_tid_f64:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_clause 0x1
+; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; GFX12-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v3, s5 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, s4
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_permlane16_b32 v3, v1, s0, s1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_permlane16_b32 v2, v0, s0, s1
+; GFX12-SDAG-NEXT: global_store_b64 v4, v[2:3], s[2:3]
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: v_permlane16_b32_undef_tid_f64:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-NEXT: s_clause 0x1
+; GFX12-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1
+; GFX12-GISEL-NEXT: v_permlane16_b32 v1, v1, s0, s1
+; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-GISEL-NEXT: s_endpgm
+;
+; GFX13-SDAG-LABEL: v_permlane16_b32_undef_tid_f64:
+; GFX13-SDAG: ; %bb.0:
+; GFX13-SDAG-NEXT: s_clause 0x1
+; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX13-SDAG-NEXT: s_mov_b32 s6, 0x3ff
+; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v3, s5 :: v_dual_bitop2_b32 v0, s6, v0 bitop3:0x40
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v4, 0
+; GFX13-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-SDAG-NEXT: v_permlane16_b32 v3, v1, s0, s1
+; GFX13-SDAG-NEXT: v_permlane16_b32 v2, v0, s0, s1
+; GFX13-SDAG-NEXT: global_store_b64 v4, v[2:3], s[2:3]
+; GFX13-SDAG-NEXT: s_endpgm
+;
+; GFX13-GISEL-LABEL: v_permlane16_b32_undef_tid_f64:
+; GFX13-GISEL: ; %bb.0:
+; GFX13-GISEL-NEXT: s_mov_b32 s0, 0x3ff
+; GFX13-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX13-GISEL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s0, v0 bitop3:0x40
+; GFX13-GISEL-NEXT: s_clause 0x1
+; GFX13-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX13-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX13-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1
+; GFX13-GISEL-NEXT: v_permlane16_b32 v1, v1, s0, s1
+; GFX13-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-GISEL-NEXT: s_endpgm
+ %tidx = call i32 @llvm.amdgcn.workitem.id.x()
+ %tidx_f32 = bitcast i32 %tidx to float
+ %tidx_f64 = fpext float %tidx_f32 to double
+ %undef = freeze double poison
+ %v = call double @llvm.amdgcn.permlane16.f64(double %undef, double %tidx_f64, i32 %src1, i32 %src2, i1 false, i1 false)
+ store double %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane16_b32_i_tid_i32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
+; GFX10-SDAG-LABEL: v_permlane16_b32_i_tid_i32:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_clause 0x1
+; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, 0x3039
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1
; GFX10-SDAG-NEXT: global_store_dword v2, v1, s[2:3]
@@ -8186,53 +8316,109 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_f64(ptr addrspace(1) %out, i32
}
define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_i32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-LABEL: v_permlane16_b32_i_tid_fi_i32:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX10-NEXT: global_store_dword v1, v0, s[2:3]
-; GFX10-NEXT: s_endpgm
+; GFX10-SDAG-LABEL: v_permlane16_b32_i_tid_fi_i32:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_clause 0x1
+; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT: ; freeze: $sgpr4
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s4
+; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX10-SDAG-NEXT: global_store_dword v2, v1, s[2:3]
+; GFX10-SDAG-NEXT: s_endpgm
;
-; GFX11-LABEL: v_permlane16_b32_i_tid_fi_i32:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX11-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX11-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_i_tid_fi_i32:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[2:3]
+; GFX10-GISEL-NEXT: s_endpgm
;
-; GFX12-LABEL: v_permlane16_b32_i_tid_fi_i32:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
-; GFX12-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX12-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX12-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: v_permlane16_b32_i_tid_fi_i32:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_clause 0x1
+; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: ; freeze: $sgpr4
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX11-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX13-LABEL: v_permlane16_b32_i_tid_fi_i32:
-; GFX13: ; %bb.0:
-; GFX13-NEXT: s_clause 0x1
-; GFX13-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-NEXT: s_mov_b32 s4, 0x3ff
-; GFX13-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
-; GFX13-NEXT: s_wait_kmcnt 0x0
-; GFX13-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX13-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX13-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: v_permlane16_b32_i_tid_fi_i32:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX11-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: v_permlane16_b32_i_tid_fi_i32:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_clause 0x1
+; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-SDAG-NEXT: ; freeze: $sgpr4
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX12-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: v_permlane16_b32_i_tid_fi_i32:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_clause 0x1
+; GFX12-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX12-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
+; GFX12-GISEL-NEXT: s_endpgm
+;
+; GFX13-SDAG-LABEL: v_permlane16_b32_i_tid_fi_i32:
+; GFX13-SDAG: ; %bb.0:
+; GFX13-SDAG-NEXT: s_clause 0x1
+; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-SDAG-NEXT: ; freeze: $sgpr4
+; GFX13-SDAG-NEXT: s_mov_b32 s5, 0x3ff
+; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
+; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX13-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
+; GFX13-SDAG-NEXT: s_endpgm
+;
+; GFX13-GISEL-LABEL: v_permlane16_b32_i_tid_fi_i32:
+; GFX13-GISEL: ; %bb.0:
+; GFX13-GISEL-NEXT: s_clause 0x1
+; GFX13-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-GISEL-NEXT: s_mov_b32 s4, 0x3ff
+; GFX13-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX13-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX13-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
+; GFX13-GISEL-NEXT: s_endpgm
%tidx = call i32 @llvm.amdgcn.workitem.id.x()
%undef = freeze i32 poison
%v = call i32 @llvm.amdgcn.permlane16.i32(i32 %undef, i32 %tidx, i32 %src1, i32 %src2, i1 true, i1 false)
@@ -8241,53 +8427,109 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_i32(ptr addrspace(1) %out,
}
define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_f32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-LABEL: v_permlane16_b32_i_tid_fi_f32:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX10-NEXT: global_store_dword v1, v0, s[2:3]
-; GFX10-NEXT: s_endpgm
+; GFX10-SDAG-LABEL: v_permlane16_b32_i_tid_fi_f32:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_clause 0x1
+; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT: ; freeze: $sgpr4
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s4
+; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX10-SDAG-NEXT: global_store_dword v2, v1, s[2:3]
+; GFX10-SDAG-NEXT: s_endpgm
;
-; GFX11-LABEL: v_permlane16_b32_i_tid_fi_f32:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX11-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX11-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_i_tid_fi_f32:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[2:3]
+; GFX10-GISEL-NEXT: s_endpgm
;
-; GFX12-LABEL: v_permlane16_b32_i_tid_fi_f32:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
-; GFX12-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX12-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX12-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: v_permlane16_b32_i_tid_fi_f32:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_clause 0x1
+; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: ; freeze: $sgpr4
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX11-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX13-LABEL: v_permlane16_b32_i_tid_fi_f32:
-; GFX13: ; %bb.0:
-; GFX13-NEXT: s_clause 0x1
-; GFX13-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-NEXT: s_mov_b32 s4, 0x3ff
-; GFX13-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
-; GFX13-NEXT: s_wait_kmcnt 0x0
-; GFX13-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX13-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX13-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: v_permlane16_b32_i_tid_fi_f32:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX11-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: v_permlane16_b32_i_tid_fi_f32:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_clause 0x1
+; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-SDAG-NEXT: ; freeze: $sgpr4
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX12-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: v_permlane16_b32_i_tid_fi_f32:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_clause 0x1
+; GFX12-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX12-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
+; GFX12-GISEL-NEXT: s_endpgm
+;
+; GFX13-SDAG-LABEL: v_permlane16_b32_i_tid_fi_f32:
+; GFX13-SDAG: ; %bb.0:
+; GFX13-SDAG-NEXT: s_clause 0x1
+; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-SDAG-NEXT: ; freeze: $sgpr4
+; GFX13-SDAG-NEXT: s_mov_b32 s5, 0x3ff
+; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
+; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX13-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
+; GFX13-SDAG-NEXT: s_endpgm
+;
+; GFX13-GISEL-LABEL: v_permlane16_b32_i_tid_fi_f32:
+; GFX13-GISEL: ; %bb.0:
+; GFX13-GISEL-NEXT: s_clause 0x1
+; GFX13-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-GISEL-NEXT: s_mov_b32 s4, 0x3ff
+; GFX13-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX13-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX13-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
+; GFX13-GISEL-NEXT: s_endpgm
%tidx = call i32 @llvm.amdgcn.workitem.id.x()
%tidx_f32 = bitcast i32 %tidx to float
%undef = freeze float poison
@@ -8302,11 +8544,14 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_i64(ptr addrspace(1) %out,
; GFX10-SDAG-NEXT: s_clause 0x1
; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v3, 0
+; GFX10-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, s5
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s4
; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v2, s0, s1 op_sel:[1,0]
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-SDAG-NEXT: v_permlane16_b32 v2, v3, s0, s1 op_sel:[1,0]
+; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX10-SDAG-NEXT: global_store_dwordx2 v3, v[1:2], s[2:3]
; GFX10-SDAG-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_permlane16_b32_i_tid_fi_i64:
@@ -8326,13 +8571,14 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_i64(ptr addrspace(1) %out,
; GFX11-SDAG-NEXT: s_clause 0x1
; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
; GFX11-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT: v_permlane16_b32 v1, v2, s0, s1 op_sel:[1,0]
-; GFX11-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-SDAG-NEXT: v_permlane16_b32 v2, v3, s0, s1 op_sel:[1,0]
+; GFX11-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX11-SDAG-NEXT: global_store_b64 v3, v[1:2], s[2:3]
; GFX11-SDAG-NEXT: s_endpgm
;
; GFX11-GISEL-LABEL: v_permlane16_b32_i_tid_fi_i64:
@@ -8354,13 +8600,14 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_i64(ptr addrspace(1) %out,
; GFX12-SDAG-NEXT: s_clause 0x1
; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
; GFX12-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX12-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-NEXT: v_permlane16_b32 v1, v2, s0, s1 op_sel:[1,0]
-; GFX12-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX12-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-SDAG-NEXT: v_permlane16_b32 v2, v3, s0, s1 op_sel:[1,0]
+; GFX12-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX12-SDAG-NEXT: global_store_b64 v3, v[1:2], s[2:3]
; GFX12-SDAG-NEXT: s_endpgm
;
; GFX12-GISEL-LABEL: v_permlane16_b32_i_tid_fi_i64:
@@ -8382,14 +8629,16 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_i64(ptr addrspace(1) %out,
; GFX13-SDAG-NEXT: s_clause 0x1
; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: s_mov_b32 s4, 0x3ff
-; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX13-SDAG-NEXT: s_mov_b32 s5, 0x3ff
+; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v2, s0, s1 op_sel:[1,0]
+; GFX13-SDAG-NEXT: v_permlane16_b32 v2, v3, s0, s1 op_sel:[1,0]
; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX13-SDAG-NEXT: global_store_b64 v3, v[1:2], s[2:3]
; GFX13-SDAG-NEXT: s_endpgm
;
; GFX13-GISEL-LABEL: v_permlane16_b32_i_tid_fi_i64:
@@ -8417,15 +8666,18 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_i64(ptr addrspace(1) %out,
define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_f64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
; GFX10-SDAG-LABEL: v_permlane16_b32_i_tid_fi_f64:
; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
; GFX10-SDAG-NEXT: s_clause 0x1
; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; GFX10-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v3, s5
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, s4
; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v1, s0, s1 op_sel:[1,0]
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-SDAG-NEXT: v_permlane16_b32 v3, v1, s0, s1 op_sel:[1,0]
+; GFX10-SDAG-NEXT: v_permlane16_b32 v2, v0, s0, s1 op_sel:[1,0]
+; GFX10-SDAG-NEXT: global_store_dwordx2 v4, v[2:3], s[2:3]
; GFX10-SDAG-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_permlane16_b32_i_tid_fi_f64:
@@ -8443,17 +8695,20 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_f64(ptr addrspace(1) %out,
;
; GFX11-SDAG-LABEL: v_permlane16_b32_i_tid_fi_f64:
; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-SDAG-NEXT: s_clause 0x1
; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
; GFX11-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; GFX11-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v3, s5 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, s4
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT: v_permlane16_b32 v1, v1, s0, s1 op_sel:[1,0]
-; GFX11-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-SDAG-NEXT: v_permlane16_b32 v3, v1, s0, s1 op_sel:[1,0]
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_permlane16_b32 v2, v0, s0, s1 op_sel:[1,0]
+; GFX11-SDAG-NEXT: global_store_b64 v4, v[2:3], s[2:3]
; GFX11-SDAG-NEXT: s_endpgm
;
; GFX11-GISEL-LABEL: v_permlane16_b32_i_tid_fi_f64:
@@ -8473,17 +8728,20 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_f64(ptr addrspace(1) %out,
;
; GFX12-SDAG-LABEL: v_permlane16_b32_i_tid_fi_f64:
; GFX12-SDAG: ; %bb.0:
-; GFX12-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-SDAG-NEXT: s_clause 0x1
; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
; GFX12-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; GFX12-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v3, s5 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, s4
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-NEXT: v_permlane16_b32 v1, v1, s0, s1 op_sel:[1,0]
-; GFX12-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX12-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-SDAG-NEXT: v_permlane16_b32 v3, v1, s0, s1 op_sel:[1,0]
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_permlane16_b32 v2, v0, s0, s1 op_sel:[1,0]
+; GFX12-SDAG-NEXT: global_store_b64 v4, v[2:3], s[2:3]
; GFX12-SDAG-NEXT: s_endpgm
;
; GFX12-GISEL-LABEL: v_permlane16_b32_i_tid_fi_f64:
@@ -8503,18 +8761,20 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_f64(ptr addrspace(1) %out,
;
; GFX13-SDAG-LABEL: v_permlane16_b32_i_tid_fi_f64:
; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_mov_b32 s0, 0x3ff
-; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s0, v0 bitop3:0x40
; GFX13-SDAG-NEXT: s_clause 0x1
; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX13-SDAG-NEXT: s_mov_b32 s6, 0x3ff
+; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v3, s5 :: v_dual_bitop2_b32 v0, s6, v0 bitop3:0x40
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v4, 0
; GFX13-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v1, s0, s1 op_sel:[1,0]
-; GFX13-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-SDAG-NEXT: v_permlane16_b32 v3, v1, s0, s1 op_sel:[1,0]
+; GFX13-SDAG-NEXT: v_permlane16_b32 v2, v0, s0, s1 op_sel:[1,0]
+; GFX13-SDAG-NEXT: global_store_b64 v4, v[2:3], s[2:3]
; GFX13-SDAG-NEXT: s_endpgm
;
; GFX13-GISEL-LABEL: v_permlane16_b32_i_tid_fi_f64:
@@ -8542,53 +8802,109 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_f64(ptr addrspace(1) %out,
}
define amdgpu_kernel void @v_permlane16_b32_i_tid_bc_i32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-LABEL: v_permlane16_b32_i_tid_bc_i32:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX10-NEXT: global_store_dword v1, v0, s[2:3]
-; GFX10-NEXT: s_endpgm
-;
-; GFX11-LABEL: v_permlane16_b32_i_tid_bc_i32:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX11-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX11-NEXT: s_endpgm
+; GFX10-SDAG-LABEL: v_permlane16_b32_i_tid_bc_i32:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_clause 0x1
+; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT: ; freeze: $sgpr4
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s4
+; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX10-SDAG-NEXT: global_store_dword v2, v1, s[2:3]
+; GFX10-SDAG-NEXT: s_endpgm
;
-; GFX12-LABEL: v_permlane16_b32_i_tid_bc_i32:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
-; GFX12-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX12-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX12-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_i_tid_bc_i32:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[2:3]
+; GFX10-GISEL-NEXT: s_endpgm
;
-; GFX13-LABEL: v_permlane16_b32_i_tid_bc_i32:
-; GFX13: ; %bb.0:
-; GFX13-NEXT: s_clause 0x1
-; GFX13-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-NEXT: s_mov_b32 s4, 0x3ff
-; GFX13-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
-; GFX13-NEXT: s_wait_kmcnt 0x0
-; GFX13-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX13-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX13-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: v_permlane16_b32_i_tid_bc_i32:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_clause 0x1
+; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: ; freeze: $sgpr4
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX11-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: v_permlane16_b32_i_tid_bc_i32:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX11-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: v_permlane16_b32_i_tid_bc_i32:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_clause 0x1
+; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-SDAG-NEXT: ; freeze: $sgpr4
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX12-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: v_permlane16_b32_i_tid_bc_i32:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_clause 0x1
+; GFX12-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX12-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
+; GFX12-GISEL-NEXT: s_endpgm
+;
+; GFX13-SDAG-LABEL: v_permlane16_b32_i_tid_bc_i32:
+; GFX13-SDAG: ; %bb.0:
+; GFX13-SDAG-NEXT: s_clause 0x1
+; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-SDAG-NEXT: ; freeze: $sgpr4
+; GFX13-SDAG-NEXT: s_mov_b32 s5, 0x3ff
+; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
+; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX13-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
+; GFX13-SDAG-NEXT: s_endpgm
+;
+; GFX13-GISEL-LABEL: v_permlane16_b32_i_tid_bc_i32:
+; GFX13-GISEL: ; %bb.0:
+; GFX13-GISEL-NEXT: s_clause 0x1
+; GFX13-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-GISEL-NEXT: s_mov_b32 s4, 0x3ff
+; GFX13-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX13-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX13-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
+; GFX13-GISEL-NEXT: s_endpgm
%tidx = call i32 @llvm.amdgcn.workitem.id.x()
%undef = freeze i32 poison
%v = call i32 @llvm.amdgcn.permlane16.i32(i32 %undef, i32 %tidx, i32 %src1, i32 %src2, i1 false, i1 true)
@@ -8597,53 +8913,109 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_bc_i32(ptr addrspace(1) %out,
}
define amdgpu_kernel void @v_permlane16_b32_i_tid_bc_f32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-LABEL: v_permlane16_b32_i_tid_bc_f32:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX10-NEXT: global_store_dword v1, v0, s[2:3]
-; GFX10-NEXT: s_endpgm
+; GFX10-SDAG-LABEL: v_permlane16_b32_i_tid_bc_f32:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_clause 0x1
+; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT: ; freeze: $sgpr4
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s4
+; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX10-SDAG-NEXT: global_store_dword v2, v1, s[2:3]
+; GFX10-SDAG-NEXT: s_endpgm
;
-; GFX11-LABEL: v_permlane16_b32_i_tid_bc_f32:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX11-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX11-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_i_tid_bc_f32:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[2:3]
+; GFX10-GISEL-NEXT: s_endpgm
;
-; GFX12-LABEL: v_permlane16_b32_i_tid_bc_f32:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
-; GFX12-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX12-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX12-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: v_permlane16_b32_i_tid_bc_f32:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_clause 0x1
+; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: ; freeze: $sgpr4
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX11-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX13-LABEL: v_permlane16_b32_i_tid_bc_f32:
-; GFX13: ; %bb.0:
-; GFX13-NEXT: s_clause 0x1
-; GFX13-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-NEXT: s_mov_b32 s4, 0x3ff
-; GFX13-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
-; GFX13-NEXT: s_wait_kmcnt 0x0
-; GFX13-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX13-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX13-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: v_permlane16_b32_i_tid_bc_f32:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX11-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: v_permlane16_b32_i_tid_bc_f32:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_clause 0x1
+; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-SDAG-NEXT: ; freeze: $sgpr4
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX12-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: v_permlane16_b32_i_tid_bc_f32:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_clause 0x1
+; GFX12-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX12-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
+; GFX12-GISEL-NEXT: s_endpgm
+;
+; GFX13-SDAG-LABEL: v_permlane16_b32_i_tid_bc_f32:
+; GFX13-SDAG: ; %bb.0:
+; GFX13-SDAG-NEXT: s_clause 0x1
+; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-SDAG-NEXT: ; freeze: $sgpr4
+; GFX13-SDAG-NEXT: s_mov_b32 s5, 0x3ff
+; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
+; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX13-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
+; GFX13-SDAG-NEXT: s_endpgm
+;
+; GFX13-GISEL-LABEL: v_permlane16_b32_i_tid_bc_f32:
+; GFX13-GISEL: ; %bb.0:
+; GFX13-GISEL-NEXT: s_clause 0x1
+; GFX13-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-GISEL-NEXT: s_mov_b32 s4, 0x3ff
+; GFX13-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX13-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX13-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
+; GFX13-GISEL-NEXT: s_endpgm
%tidx = call i32 @llvm.amdgcn.workitem.id.x()
%tidx_f32 = bitcast i32 %tidx to float
%undef = freeze float poison
@@ -8658,11 +9030,14 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_bc_i64(ptr addrspace(1) %out,
; GFX10-SDAG-NEXT: s_clause 0x1
; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v3, 0
+; GFX10-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, s5
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s4
; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v2, s0, s1 op_sel:[0,1]
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-SDAG-NEXT: v_permlane16_b32 v2, v3, s0, s1 op_sel:[0,1]
+; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX10-SDAG-NEXT: global_store_dwordx2 v3, v[1:2], s[2:3]
; GFX10-SDAG-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_permlane16_b32_i_tid_bc_i64:
@@ -8682,13 +9057,14 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_bc_i64(ptr addrspace(1) %out,
; GFX11-SDAG-NEXT: s_clause 0x1
; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
; GFX11-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT: v_permlane16_b32 v1, v2, s0, s1 op_sel:[0,1]
-; GFX11-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-SDAG-NEXT: v_permlane16_b32 v2, v3, s0, s1 op_sel:[0,1]
+; GFX11-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX11-SDAG-NEXT: global_store_b64 v3, v[1:2], s[2:3]
; GFX11-SDAG-NEXT: s_endpgm
;
; GFX11-GISEL-LABEL: v_permlane16_b32_i_tid_bc_i64:
@@ -8710,13 +9086,14 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_bc_i64(ptr addrspace(1) %out,
; GFX12-SDAG-NEXT: s_clause 0x1
; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
; GFX12-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX12-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-NEXT: v_permlane16_b32 v1, v2, s0, s1 op_sel:[0,1]
-; GFX12-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX12-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-SDAG-NEXT: v_permlane16_b32 v2, v3, s0, s1 op_sel:[0,1]
+; GFX12-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX12-SDAG-NEXT: global_store_b64 v3, v[1:2], s[2:3]
; GFX12-SDAG-NEXT: s_endpgm
;
; GFX12-GISEL-LABEL: v_permlane16_b32_i_tid_bc_i64:
@@ -8738,14 +9115,16 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_bc_i64(ptr addrspace(1) %out,
; GFX13-SDAG-NEXT: s_clause 0x1
; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: s_mov_b32 s4, 0x3ff
-; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX13-SDAG-NEXT: s_mov_b32 s5, 0x3ff
+; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v2, s0, s1 op_sel:[0,1]
+; GFX13-SDAG-NEXT: v_permlane16_b32 v2, v3, s0, s1 op_sel:[0,1]
; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX13-SDAG-NEXT: global_store_b64 v3, v[1:2], s[2:3]
; GFX13-SDAG-NEXT: s_endpgm
;
; GFX13-GISEL-LABEL: v_permlane16_b32_i_tid_bc_i64:
@@ -8773,15 +9152,18 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_bc_i64(ptr addrspace(1) %out,
define amdgpu_kernel void @v_permlane16_b32_i_tid_bc_f64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
; GFX10-SDAG-LABEL: v_permlane16_b32_i_tid_bc_f64:
; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
; GFX10-SDAG-NEXT: s_clause 0x1
; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; GFX10-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v3, s5
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, s4
; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v1, s0, s1 op_sel:[0,1]
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-SDAG-NEXT: v_permlane16_b32 v3, v1, s0, s1 op_sel:[0,1]
+; GFX10-SDAG-NEXT: v_permlane16_b32 v2, v0, s0, s1 op_sel:[0,1]
+; GFX10-SDAG-NEXT: global_store_dwordx2 v4, v[2:3], s[2:3]
; GFX10-SDAG-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_permlane16_b32_i_tid_bc_f64:
@@ -8799,17 +9181,20 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_bc_f64(ptr addrspace(1) %out,
;
; GFX11-SDAG-LABEL: v_permlane16_b32_i_tid_bc_f64:
; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-SDAG-NEXT: s_clause 0x1
; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
; GFX11-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; GFX11-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v3, s5 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, s4
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT: v_permlane16_b32 v1, v1, s0, s1 op_sel:[0,1]
-; GFX11-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-SDAG-NEXT: v_permlane16_b32 v3, v1, s0, s1 op_sel:[0,1]
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_permlane16_b32 v2, v0, s0, s1 op_sel:[0,1]
+; GFX11-SDAG-NEXT: global_store_b64 v4, v[2:3], s[2:3]
; GFX11-SDAG-NEXT: s_endpgm
;
; GFX11-GISEL-LABEL: v_permlane16_b32_i_tid_bc_f64:
@@ -8829,17 +9214,20 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_bc_f64(ptr addrspace(1) %out,
;
; GFX12-SDAG-LABEL: v_permlane16_b32_i_tid_bc_f64:
; GFX12-SDAG: ; %bb.0:
-; GFX12-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-SDAG-NEXT: s_clause 0x1
; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
; GFX12-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; GFX12-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v3, s5 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, s4
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-NEXT: v_permlane16_b32 v1, v1, s0, s1 op_sel:[0,1]
-; GFX12-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX12-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-SDAG-NEXT: v_permlane16_b32 v3, v1, s0, s1 op_sel:[0,1]
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_permlane16_b32 v2, v0, s0, s1 op_sel:[0,1]
+; GFX12-SDAG-NEXT: global_store_b64 v4, v[2:3], s[2:3]
; GFX12-SDAG-NEXT: s_endpgm
;
; GFX12-GISEL-LABEL: v_permlane16_b32_i_tid_bc_f64:
@@ -8859,18 +9247,20 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_bc_f64(ptr addrspace(1) %out,
;
; GFX13-SDAG-LABEL: v_permlane16_b32_i_tid_bc_f64:
; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_mov_b32 s0, 0x3ff
-; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s0, v0 bitop3:0x40
; GFX13-SDAG-NEXT: s_clause 0x1
; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX13-SDAG-NEXT: s_mov_b32 s6, 0x3ff
+; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v3, s5 :: v_dual_bitop2_b32 v0, s6, v0 bitop3:0x40
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v4, 0
; GFX13-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v1, s0, s1 op_sel:[0,1]
-; GFX13-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-SDAG-NEXT: v_permlane16_b32 v3, v1, s0, s1 op_sel:[0,1]
+; GFX13-SDAG-NEXT: v_permlane16_b32 v2, v0, s0, s1 op_sel:[0,1]
+; GFX13-SDAG-NEXT: global_store_b64 v4, v[2:3], s[2:3]
; GFX13-SDAG-NEXT: s_endpgm
;
; GFX13-GISEL-LABEL: v_permlane16_b32_i_tid_bc_f64:
@@ -8898,53 +9288,109 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_bc_f64(ptr addrspace(1) %out,
}
define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_bc_i32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-LABEL: v_permlane16_b32_i_tid_fi_bc_i32:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX10-NEXT: global_store_dword v1, v0, s[2:3]
-; GFX10-NEXT: s_endpgm
+; GFX10-SDAG-LABEL: v_permlane16_b32_i_tid_fi_bc_i32:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_clause 0x1
+; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT: ; freeze: $sgpr4
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s4
+; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX10-SDAG-NEXT: global_store_dword v2, v1, s[2:3]
+; GFX10-SDAG-NEXT: s_endpgm
;
-; GFX11-LABEL: v_permlane16_b32_i_tid_fi_bc_i32:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX11-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX11-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_i_tid_fi_bc_i32:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
+; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[2:3]
+; GFX10-GISEL-NEXT: s_endpgm
;
-; GFX12-LABEL: v_permlane16_b32_i_tid_fi_bc_i32:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
-; GFX12-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX12-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX12-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: v_permlane16_b32_i_tid_fi_bc_i32:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_clause 0x1
+; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: ; freeze: $sgpr4
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX11-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX13-LABEL: v_permlane16_b32_i_tid_fi_bc_i32:
-; GFX13: ; %bb.0:
-; GFX13-NEXT: s_clause 0x1
-; GFX13-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-NEXT: s_mov_b32 s4, 0x3ff
-; GFX13-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
-; GFX13-NEXT: s_wait_kmcnt 0x0
-; GFX13-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX13-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX13-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: v_permlane16_b32_i_tid_fi_bc_i32:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
+; GFX11-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: v_permlane16_b32_i_tid_fi_bc_i32:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_clause 0x1
+; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-SDAG-NEXT: ; freeze: $sgpr4
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX12-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: v_permlane16_b32_i_tid_fi_bc_i32:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_clause 0x1
+; GFX12-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
+; GFX12-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
+; GFX12-GISEL-NEXT: s_endpgm
+;
+; GFX13-SDAG-LABEL: v_permlane16_b32_i_tid_fi_bc_i32:
+; GFX13-SDAG: ; %bb.0:
+; GFX13-SDAG-NEXT: s_clause 0x1
+; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-SDAG-NEXT: ; freeze: $sgpr4
+; GFX13-SDAG-NEXT: s_mov_b32 s5, 0x3ff
+; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
+; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX13-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
+; GFX13-SDAG-NEXT: s_endpgm
+;
+; GFX13-GISEL-LABEL: v_permlane16_b32_i_tid_fi_bc_i32:
+; GFX13-GISEL: ; %bb.0:
+; GFX13-GISEL-NEXT: s_clause 0x1
+; GFX13-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-GISEL-NEXT: s_mov_b32 s4, 0x3ff
+; GFX13-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX13-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
+; GFX13-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
+; GFX13-GISEL-NEXT: s_endpgm
%tidx = call i32 @llvm.amdgcn.workitem.id.x()
%undef = freeze i32 poison
%v = call i32 @llvm.amdgcn.permlane16.i32(i32 %undef, i32 %tidx, i32 %src1, i32 %src2, i1 true, i1 true)
@@ -8953,53 +9399,109 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_bc_i32(ptr addrspace(1) %ou
}
define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_bc_f32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-LABEL: v_permlane16_b32_i_tid_fi_bc_f32:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX10-NEXT: global_store_dword v1, v0, s[2:3]
-; GFX10-NEXT: s_endpgm
+; GFX10-SDAG-LABEL: v_permlane16_b32_i_tid_fi_bc_f32:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_clause 0x1
+; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT: ; freeze: $sgpr4
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s4
+; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX10-SDAG-NEXT: global_store_dword v2, v1, s[2:3]
+; GFX10-SDAG-NEXT: s_endpgm
;
-; GFX11-LABEL: v_permlane16_b32_i_tid_fi_bc_f32:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX11-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX11-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlane16_b32_i_tid_fi_bc_f32:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
+; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[2:3]
+; GFX10-GISEL-NEXT: s_endpgm
;
-; GFX12-LABEL: v_permlane16_b32_i_tid_fi_bc_f32:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
-; GFX12-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX12-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX12-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: v_permlane16_b32_i_tid_fi_bc_f32:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_clause 0x1
+; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: ; freeze: $sgpr4
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX11-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX13-LABEL: v_permlane16_b32_i_tid_fi_bc_f32:
-; GFX13: ; %bb.0:
-; GFX13-NEXT: s_clause 0x1
-; GFX13-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-NEXT: s_mov_b32 s4, 0x3ff
-; GFX13-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
-; GFX13-NEXT: s_wait_kmcnt 0x0
-; GFX13-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX13-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX13-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: v_permlane16_b32_i_tid_fi_bc_f32:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
+; GFX11-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: v_permlane16_b32_i_tid_fi_bc_f32:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_clause 0x1
+; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-SDAG-NEXT: ; freeze: $sgpr4
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX12-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: v_permlane16_b32_i_tid_fi_bc_f32:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_clause 0x1
+; GFX12-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
+; GFX12-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
+; GFX12-GISEL-NEXT: s_endpgm
+;
+; GFX13-SDAG-LABEL: v_permlane16_b32_i_tid_fi_bc_f32:
+; GFX13-SDAG: ; %bb.0:
+; GFX13-SDAG-NEXT: s_clause 0x1
+; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-SDAG-NEXT: ; freeze: $sgpr4
+; GFX13-SDAG-NEXT: s_mov_b32 s5, 0x3ff
+; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
+; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX13-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
+; GFX13-SDAG-NEXT: s_endpgm
+;
+; GFX13-GISEL-LABEL: v_permlane16_b32_i_tid_fi_bc_f32:
+; GFX13-GISEL: ; %bb.0:
+; GFX13-GISEL-NEXT: s_clause 0x1
+; GFX13-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-GISEL-NEXT: s_mov_b32 s4, 0x3ff
+; GFX13-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX13-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
+; GFX13-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
+; GFX13-GISEL-NEXT: s_endpgm
%tidx = call i32 @llvm.amdgcn.workitem.id.x()
%tidx_f32 = bitcast i32 %tidx to float
%undef = freeze float poison
@@ -9014,11 +9516,14 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_bc_i64(ptr addrspace(1) %ou
; GFX10-SDAG-NEXT: s_clause 0x1
; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v3, 0
+; GFX10-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, s5
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s4
; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v2, s0, s1 op_sel:[1,1]
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-SDAG-NEXT: v_permlane16_b32 v2, v3, s0, s1 op_sel:[1,1]
+; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX10-SDAG-NEXT: global_store_dwordx2 v3, v[1:2], s[2:3]
; GFX10-SDAG-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_permlane16_b32_i_tid_fi_bc_i64:
@@ -9038,13 +9543,14 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_bc_i64(ptr addrspace(1) %ou
; GFX11-SDAG-NEXT: s_clause 0x1
; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
; GFX11-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT: v_permlane16_b32 v1, v2, s0, s1 op_sel:[1,1]
-; GFX11-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-SDAG-NEXT: v_permlane16_b32 v2, v3, s0, s1 op_sel:[1,1]
+; GFX11-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX11-SDAG-NEXT: global_store_b64 v3, v[1:2], s[2:3]
; GFX11-SDAG-NEXT: s_endpgm
;
; GFX11-GISEL-LABEL: v_permlane16_b32_i_tid_fi_bc_i64:
@@ -9066,13 +9572,14 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_bc_i64(ptr addrspace(1) %ou
; GFX12-SDAG-NEXT: s_clause 0x1
; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
; GFX12-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX12-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-NEXT: v_permlane16_b32 v1, v2, s0, s1 op_sel:[1,1]
-; GFX12-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX12-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-SDAG-NEXT: v_permlane16_b32 v2, v3, s0, s1 op_sel:[1,1]
+; GFX12-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX12-SDAG-NEXT: global_store_b64 v3, v[1:2], s[2:3]
; GFX12-SDAG-NEXT: s_endpgm
;
; GFX12-GISEL-LABEL: v_permlane16_b32_i_tid_fi_bc_i64:
@@ -9094,14 +9601,16 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_bc_i64(ptr addrspace(1) %ou
; GFX13-SDAG-NEXT: s_clause 0x1
; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: s_mov_b32 s4, 0x3ff
-; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX13-SDAG-NEXT: s_mov_b32 s5, 0x3ff
+; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v2, s0, s1 op_sel:[1,1]
+; GFX13-SDAG-NEXT: v_permlane16_b32 v2, v3, s0, s1 op_sel:[1,1]
; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX13-SDAG-NEXT: global_store_b64 v3, v[1:2], s[2:3]
; GFX13-SDAG-NEXT: s_endpgm
;
; GFX13-GISEL-LABEL: v_permlane16_b32_i_tid_fi_bc_i64:
@@ -9129,15 +9638,18 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_bc_i64(ptr addrspace(1) %ou
define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_bc_f64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
; GFX10-SDAG-LABEL: v_permlane16_b32_i_tid_fi_bc_f64:
; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
; GFX10-SDAG-NEXT: s_clause 0x1
; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; GFX10-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v3, s5
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, s4
; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v1, s0, s1 op_sel:[1,1]
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-SDAG-NEXT: v_permlane16_b32 v3, v1, s0, s1 op_sel:[1,1]
+; GFX10-SDAG-NEXT: v_permlane16_b32 v2, v0, s0, s1 op_sel:[1,1]
+; GFX10-SDAG-NEXT: global_store_dwordx2 v4, v[2:3], s[2:3]
; GFX10-SDAG-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_permlane16_b32_i_tid_fi_bc_f64:
@@ -9155,17 +9667,20 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_bc_f64(ptr addrspace(1) %ou
;
; GFX11-SDAG-LABEL: v_permlane16_b32_i_tid_fi_bc_f64:
; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-SDAG-NEXT: s_clause 0x1
; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
; GFX11-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; GFX11-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v3, s5 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, s4
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT: v_permlane16_b32 v1, v1, s0, s1 op_sel:[1,1]
-; GFX11-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-SDAG-NEXT: v_permlane16_b32 v3, v1, s0, s1 op_sel:[1,1]
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_permlane16_b32 v2, v0, s0, s1 op_sel:[1,1]
+; GFX11-SDAG-NEXT: global_store_b64 v4, v[2:3], s[2:3]
; GFX11-SDAG-NEXT: s_endpgm
;
; GFX11-GISEL-LABEL: v_permlane16_b32_i_tid_fi_bc_f64:
@@ -9185,17 +9700,20 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_bc_f64(ptr addrspace(1) %ou
;
; GFX12-SDAG-LABEL: v_permlane16_b32_i_tid_fi_bc_f64:
; GFX12-SDAG: ; %bb.0:
-; GFX12-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-SDAG-NEXT: s_clause 0x1
; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
; GFX12-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; GFX12-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v3, s5 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, s4
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-NEXT: v_permlane16_b32 v1, v1, s0, s1 op_sel:[1,1]
-; GFX12-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX12-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-SDAG-NEXT: v_permlane16_b32 v3, v1, s0, s1 op_sel:[1,1]
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_permlane16_b32 v2, v0, s0, s1 op_sel:[1,1]
+; GFX12-SDAG-NEXT: global_store_b64 v4, v[2:3], s[2:3]
; GFX12-SDAG-NEXT: s_endpgm
;
; GFX12-GISEL-LABEL: v_permlane16_b32_i_tid_fi_bc_f64:
@@ -9215,18 +9733,20 @@ define amdgpu_kernel void @v_permlane16_b32_i_tid_fi_bc_f64(ptr addrspace(1) %ou
;
; GFX13-SDAG-LABEL: v_permlane16_b32_i_tid_fi_bc_f64:
; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_mov_b32 s0, 0x3ff
-; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s0, v0 bitop3:0x40
; GFX13-SDAG-NEXT: s_clause 0x1
; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX13-SDAG-NEXT: s_mov_b32 s6, 0x3ff
+; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v3, s5 :: v_dual_bitop2_b32 v0, s6, v0 bitop3:0x40
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v4, 0
; GFX13-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlane16_b32 v1, v1, s0, s1 op_sel:[1,1]
-; GFX13-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-SDAG-NEXT: v_permlane16_b32 v3, v1, s0, s1 op_sel:[1,1]
+; GFX13-SDAG-NEXT: v_permlane16_b32 v2, v0, s0, s1 op_sel:[1,1]
+; GFX13-SDAG-NEXT: global_store_b64 v4, v[2:3], s[2:3]
; GFX13-SDAG-NEXT: s_endpgm
;
; GFX13-GISEL-LABEL: v_permlane16_b32_i_tid_fi_bc_f64:
@@ -9608,358 +10128,488 @@ define amdgpu_kernel void @v_permlanex16_b32_tid_tid_f64(ptr addrspace(1) %out,
}
define amdgpu_kernel void @v_permlanex16_b32_undef_tid_i32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-LABEL: v_permlanex16_b32_undef_tid_i32:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_permlanex16_b32 v0, v0, s0, s1
-; GFX10-NEXT: global_store_dword v1, v0, s[2:3]
-; GFX10-NEXT: s_endpgm
-;
-; GFX11-LABEL: v_permlanex16_b32_undef_tid_i32:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_permlanex16_b32 v0, v0, s0, s1
-; GFX11-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX11-NEXT: s_endpgm
-;
-; GFX12-LABEL: v_permlanex16_b32_undef_tid_i32:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
-; GFX12-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_permlanex16_b32 v0, v0, s0, s1
-; GFX12-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX12-NEXT: s_endpgm
-;
-; GFX13-LABEL: v_permlanex16_b32_undef_tid_i32:
-; GFX13: ; %bb.0:
-; GFX13-NEXT: s_clause 0x1
-; GFX13-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-NEXT: s_mov_b32 s4, 0x3ff
-; GFX13-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
-; GFX13-NEXT: s_wait_kmcnt 0x0
-; GFX13-NEXT: v_permlanex16_b32 v0, v0, s0, s1
-; GFX13-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX13-NEXT: s_endpgm
- %tidx = call i32 @llvm.amdgcn.workitem.id.x()
- %undef = freeze i32 poison
- %v = call i32 @llvm.amdgcn.permlanex16.i32(i32 %undef, i32 %tidx, i32 %src1, i32 %src2, i1 false, i1 false)
- store i32 %v, ptr addrspace(1) %out
- ret void
-}
-
-define amdgpu_kernel void @v_permlanex16_b32_undef_tid_f32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-LABEL: v_permlanex16_b32_undef_tid_f32:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_permlanex16_b32 v0, v0, s0, s1
-; GFX10-NEXT: global_store_dword v1, v0, s[2:3]
-; GFX10-NEXT: s_endpgm
-;
-; GFX11-LABEL: v_permlanex16_b32_undef_tid_f32:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_permlanex16_b32 v0, v0, s0, s1
-; GFX11-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX11-NEXT: s_endpgm
-;
-; GFX12-LABEL: v_permlanex16_b32_undef_tid_f32:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
-; GFX12-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_permlanex16_b32 v0, v0, s0, s1
-; GFX12-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX12-NEXT: s_endpgm
-;
-; GFX13-LABEL: v_permlanex16_b32_undef_tid_f32:
-; GFX13: ; %bb.0:
-; GFX13-NEXT: s_clause 0x1
-; GFX13-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-NEXT: s_mov_b32 s4, 0x3ff
-; GFX13-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
-; GFX13-NEXT: s_wait_kmcnt 0x0
-; GFX13-NEXT: v_permlanex16_b32 v0, v0, s0, s1
-; GFX13-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX13-NEXT: s_endpgm
- %tidx = call i32 @llvm.amdgcn.workitem.id.x()
- %tidx_f32 = bitcast i32 %tidx to float
- %undef = freeze float poison
- %v = call float @llvm.amdgcn.permlanex16.f32(float %undef, float %tidx_f32, i32 %src1, i32 %src2, i1 false, i1 false)
- store float %v, ptr addrspace(1) %out
- ret void
-}
-
-define amdgpu_kernel void @v_permlanex16_b32_undef_tid_i64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_undef_tid_i64:
+; GFX10-SDAG-LABEL: v_permlanex16_b32_undef_tid_i32:
; GFX10-SDAG: ; %bb.0:
; GFX10-SDAG-NEXT: s_clause 0x1
; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT: ; freeze: $sgpr4
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s4
; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v2, s0, s1
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1
+; GFX10-SDAG-NEXT: global_store_dword v2, v1, s[2:3]
; GFX10-SDAG-NEXT: s_endpgm
;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_undef_tid_i64:
+; GFX10-GISEL-LABEL: v_permlanex16_b32_undef_tid_i32:
; GFX10-GISEL: ; %bb.0:
; GFX10-GISEL-NEXT: s_clause 0x1
; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v2, s0, s1
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[2:3]
; GFX10-GISEL-NEXT: s_endpgm
;
-; GFX11-SDAG-LABEL: v_permlanex16_b32_undef_tid_i64:
+; GFX11-SDAG-LABEL: v_permlanex16_b32_undef_tid_i32:
; GFX11-SDAG: ; %bb.0:
; GFX11-SDAG-NEXT: s_clause 0x1
; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
; GFX11-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT: ; freeze: $sgpr4
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT: v_permlanex16_b32 v1, v2, s0, s1
-; GFX11-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1
-; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1
+; GFX11-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX11-GISEL-LABEL: v_permlanex16_b32_undef_tid_i64:
+; GFX11-GISEL-LABEL: v_permlanex16_b32_undef_tid_i32:
; GFX11-GISEL: ; %bb.0:
; GFX11-GISEL-NEXT: s_clause 0x1
; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
; GFX11-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1
-; GFX11-GISEL-NEXT: v_permlanex16_b32 v1, v2, s0, s1
-; GFX11-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
; GFX11-GISEL-NEXT: s_endpgm
;
-; GFX12-SDAG-LABEL: v_permlanex16_b32_undef_tid_i64:
+; GFX12-SDAG-LABEL: v_permlanex16_b32_undef_tid_i32:
; GFX12-SDAG: ; %bb.0:
; GFX12-SDAG-NEXT: s_clause 0x1
; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
; GFX12-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX12-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT: ; freeze: $sgpr4
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-NEXT: v_permlanex16_b32 v1, v2, s0, s1
-; GFX12-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1
-; GFX12-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1
+; GFX12-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
; GFX12-SDAG-NEXT: s_endpgm
;
-; GFX12-GISEL-LABEL: v_permlanex16_b32_undef_tid_i64:
+; GFX12-GISEL-LABEL: v_permlanex16_b32_undef_tid_i32:
; GFX12-GISEL: ; %bb.0:
; GFX12-GISEL-NEXT: s_clause 0x1
; GFX12-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
; GFX12-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1
-; GFX12-GISEL-NEXT: v_permlanex16_b32 v1, v2, s0, s1
-; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_undef_tid_i64:
+; GFX13-SDAG-LABEL: v_permlanex16_b32_undef_tid_i32:
; GFX13-SDAG: ; %bb.0:
; GFX13-SDAG-NEXT: s_clause 0x1
; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: s_mov_b32 s4, 0x3ff
+; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-SDAG-NEXT: ; freeze: $sgpr4
+; GFX13-SDAG-NEXT: s_mov_b32 s5, 0x3ff
; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v2, s0, s1
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1
+; GFX13-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
; GFX13-SDAG-NEXT: s_endpgm
;
-; GFX13-GISEL-LABEL: v_permlanex16_b32_undef_tid_i64:
+; GFX13-GISEL-LABEL: v_permlanex16_b32_undef_tid_i32:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_clause 0x1
; GFX13-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
; GFX13-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
; GFX13-GISEL-NEXT: s_mov_b32 s4, 0x3ff
; GFX13-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-GISEL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
; GFX13-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX13-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1
-; GFX13-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX13-GISEL-NEXT: v_permlanex16_b32 v1, v2, s0, s1
-; GFX13-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
; GFX13-GISEL-NEXT: s_endpgm
%tidx = call i32 @llvm.amdgcn.workitem.id.x()
- %tidx_i64 = zext i32 %tidx to i64
- %undef = freeze i64 poison
- %v = call i64 @llvm.amdgcn.permlanex16.i64(i64 %undef, i64 %tidx_i64, i32 %src1, i32 %src2, i1 false, i1 false)
- store i64 %v, ptr addrspace(1) %out
+ %undef = freeze i32 poison
+ %v = call i32 @llvm.amdgcn.permlanex16.i32(i32 %undef, i32 %tidx, i32 %src1, i32 %src2, i1 false, i1 false)
+ store i32 %v, ptr addrspace(1) %out
ret void
}
-define amdgpu_kernel void @v_permlanex16_b32_undef_tid_f64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_undef_tid_f64:
+define amdgpu_kernel void @v_permlanex16_b32_undef_tid_f32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
+; GFX10-SDAG-LABEL: v_permlanex16_b32_undef_tid_f32:
; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
; GFX10-SDAG-NEXT: s_clause 0x1
; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT: ; freeze: $sgpr4
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s4
; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v1, s0, s1
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1
+; GFX10-SDAG-NEXT: global_store_dword v2, v1, s[2:3]
; GFX10-SDAG-NEXT: s_endpgm
;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_undef_tid_f64:
+; GFX10-GISEL-LABEL: v_permlanex16_b32_undef_tid_f32:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
; GFX10-GISEL-NEXT: s_clause 0x1
; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, s0, s1
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[2:3]
; GFX10-GISEL-NEXT: s_endpgm
;
-; GFX11-SDAG-LABEL: v_permlanex16_b32_undef_tid_f64:
+; GFX11-SDAG-LABEL: v_permlanex16_b32_undef_tid_f32:
; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-SDAG-NEXT: s_clause 0x1
; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
; GFX11-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX11-SDAG-NEXT: ; freeze: $sgpr4
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT: v_permlanex16_b32 v1, v1, s0, s1
-; GFX11-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1
-; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1
+; GFX11-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX11-GISEL-LABEL: v_permlanex16_b32_undef_tid_f64:
+; GFX11-GISEL-LABEL: v_permlanex16_b32_undef_tid_f32:
; GFX11-GISEL: ; %bb.0:
-; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-GISEL-NEXT: s_clause 0x1
; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
; GFX11-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX11-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1
-; GFX11-GISEL-NEXT: v_permlanex16_b32 v1, v1, s0, s1
-; GFX11-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
; GFX11-GISEL-NEXT: s_endpgm
;
-; GFX12-SDAG-LABEL: v_permlanex16_b32_undef_tid_f64:
+; GFX12-SDAG-LABEL: v_permlanex16_b32_undef_tid_f32:
; GFX12-SDAG: ; %bb.0:
-; GFX12-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-SDAG-NEXT: s_clause 0x1
; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
; GFX12-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX12-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX12-SDAG-NEXT: ; freeze: $sgpr4
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-NEXT: v_permlanex16_b32 v1, v1, s0, s1
-; GFX12-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1
-; GFX12-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1
+; GFX12-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
; GFX12-SDAG-NEXT: s_endpgm
;
-; GFX12-GISEL-LABEL: v_permlanex16_b32_undef_tid_f64:
+; GFX12-GISEL-LABEL: v_permlanex16_b32_undef_tid_f32:
; GFX12-GISEL: ; %bb.0:
-; GFX12-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-GISEL-NEXT: s_clause 0x1
; GFX12-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
; GFX12-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX12-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX12-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1
-; GFX12-GISEL-NEXT: v_permlanex16_b32 v1, v1, s0, s1
-; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_undef_tid_f64:
+; GFX13-SDAG-LABEL: v_permlanex16_b32_undef_tid_f32:
; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_mov_b32 s0, 0x3ff
-; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s0, v0 bitop3:0x40
; GFX13-SDAG-NEXT: s_clause 0x1
; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-SDAG-NEXT: ; freeze: $sgpr4
+; GFX13-SDAG-NEXT: s_mov_b32 s5, 0x3ff
+; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v1, s0, s1
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1
+; GFX13-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
; GFX13-SDAG-NEXT: s_endpgm
;
-; GFX13-GISEL-LABEL: v_permlanex16_b32_undef_tid_f64:
+; GFX13-GISEL-LABEL: v_permlanex16_b32_undef_tid_f32:
; GFX13-GISEL: ; %bb.0:
-; GFX13-GISEL-NEXT: s_mov_b32 s0, 0x3ff
-; GFX13-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX13-GISEL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s0, v0 bitop3:0x40
; GFX13-GISEL-NEXT: s_clause 0x1
; GFX13-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
; GFX13-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX13-GISEL-NEXT: s_mov_b32 s4, 0x3ff
+; GFX13-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
; GFX13-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX13-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX13-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1
-; GFX13-GISEL-NEXT: v_permlanex16_b32 v1, v1, s0, s1
-; GFX13-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
; GFX13-GISEL-NEXT: s_endpgm
%tidx = call i32 @llvm.amdgcn.workitem.id.x()
%tidx_f32 = bitcast i32 %tidx to float
- %tidx_f64 = fpext float %tidx_f32 to double
- %undef = freeze double poison
- %v = call double @llvm.amdgcn.permlanex16.f64(double %undef, double %tidx_f64, i32 %src1, i32 %src2, i1 false, i1 false)
- store double %v, ptr addrspace(1) %out
+ %undef = freeze float poison
+ %v = call float @llvm.amdgcn.permlanex16.f32(float %undef, float %tidx_f32, i32 %src1, i32 %src2, i1 false, i1 false)
+ store float %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlanex16_b32_undef_tid_i64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
+; GFX10-SDAG-LABEL: v_permlanex16_b32_undef_tid_i64:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_clause 0x1
+; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v3, 0
+; GFX10-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, s5
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s4
+; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_permlanex16_b32 v2, v3, s0, s1
+; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1
+; GFX10-SDAG-NEXT: global_store_dwordx2 v3, v[1:2], s[2:3]
+; GFX10-SDAG-NEXT: s_endpgm
+;
+; GFX10-GISEL-LABEL: v_permlanex16_b32_undef_tid_i64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v2, s0, s1
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-GISEL-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: v_permlanex16_b32_undef_tid_i64:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_clause 0x1
+; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_permlanex16_b32 v2, v3, s0, s1
+; GFX11-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1
+; GFX11-SDAG-NEXT: global_store_b64 v3, v[1:2], s[2:3]
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: v_permlanex16_b32_undef_tid_i64:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1
+; GFX11-GISEL-NEXT: v_permlanex16_b32 v1, v2, s0, s1
+; GFX11-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: v_permlanex16_b32_undef_tid_i64:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_clause 0x1
+; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_permlanex16_b32 v2, v3, s0, s1
+; GFX12-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1
+; GFX12-SDAG-NEXT: global_store_b64 v3, v[1:2], s[2:3]
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: v_permlanex16_b32_undef_tid_i64:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_clause 0x1
+; GFX12-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1
+; GFX12-GISEL-NEXT: v_permlanex16_b32 v1, v2, s0, s1
+; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-GISEL-NEXT: s_endpgm
+;
+; GFX13-SDAG-LABEL: v_permlanex16_b32_undef_tid_i64:
+; GFX13-SDAG: ; %bb.0:
+; GFX13-SDAG-NEXT: s_clause 0x1
+; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX13-SDAG-NEXT: s_mov_b32 s5, 0x3ff
+; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
+; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX13-SDAG-NEXT: v_permlanex16_b32 v2, v3, s0, s1
+; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1
+; GFX13-SDAG-NEXT: global_store_b64 v3, v[1:2], s[2:3]
+; GFX13-SDAG-NEXT: s_endpgm
+;
+; GFX13-GISEL-LABEL: v_permlanex16_b32_undef_tid_i64:
+; GFX13-GISEL: ; %bb.0:
+; GFX13-GISEL-NEXT: s_clause 0x1
+; GFX13-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-GISEL-NEXT: s_mov_b32 s4, 0x3ff
+; GFX13-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-GISEL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX13-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1
+; GFX13-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX13-GISEL-NEXT: v_permlanex16_b32 v1, v2, s0, s1
+; GFX13-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-GISEL-NEXT: s_endpgm
+ %tidx = call i32 @llvm.amdgcn.workitem.id.x()
+ %tidx_i64 = zext i32 %tidx to i64
+ %undef = freeze i64 poison
+ %v = call i64 @llvm.amdgcn.permlanex16.i64(i64 %undef, i64 %tidx_i64, i32 %src1, i32 %src2, i1 false, i1 false)
+ store i64 %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlanex16_b32_undef_tid_f64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
+; GFX10-SDAG-LABEL: v_permlanex16_b32_undef_tid_f64:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_clause 0x1
+; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; GFX10-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v3, s5
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, s4
+; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_permlanex16_b32 v3, v1, s0, s1
+; GFX10-SDAG-NEXT: v_permlanex16_b32 v2, v0, s0, s1
+; GFX10-SDAG-NEXT: global_store_dwordx2 v4, v[2:3], s[2:3]
+; GFX10-SDAG-NEXT: s_endpgm
+;
+; GFX10-GISEL-LABEL: v_permlanex16_b32_undef_tid_f64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, s0, s1
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-GISEL-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: v_permlanex16_b32_undef_tid_f64:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_clause 0x1
+; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; GFX11-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v3, s5 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, s4
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_permlanex16_b32 v3, v1, s0, s1
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_permlanex16_b32 v2, v0, s0, s1
+; GFX11-SDAG-NEXT: global_store_b64 v4, v[2:3], s[2:3]
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: v_permlanex16_b32_undef_tid_f64:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1
+; GFX11-GISEL-NEXT: v_permlanex16_b32 v1, v1, s0, s1
+; GFX11-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: v_permlanex16_b32_undef_tid_f64:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_clause 0x1
+; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; GFX12-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v3, s5 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, s4
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_permlanex16_b32 v3, v1, s0, s1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_permlanex16_b32 v2, v0, s0, s1
+; GFX12-SDAG-NEXT: global_store_b64 v4, v[2:3], s[2:3]
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: v_permlanex16_b32_undef_tid_f64:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-NEXT: s_clause 0x1
+; GFX12-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1
+; GFX12-GISEL-NEXT: v_permlanex16_b32 v1, v1, s0, s1
+; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-GISEL-NEXT: s_endpgm
+;
+; GFX13-SDAG-LABEL: v_permlanex16_b32_undef_tid_f64:
+; GFX13-SDAG: ; %bb.0:
+; GFX13-SDAG-NEXT: s_clause 0x1
+; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX13-SDAG-NEXT: s_mov_b32 s6, 0x3ff
+; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v3, s5 :: v_dual_bitop2_b32 v0, s6, v0 bitop3:0x40
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v4, 0
+; GFX13-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-SDAG-NEXT: v_permlanex16_b32 v3, v1, s0, s1
+; GFX13-SDAG-NEXT: v_permlanex16_b32 v2, v0, s0, s1
+; GFX13-SDAG-NEXT: global_store_b64 v4, v[2:3], s[2:3]
+; GFX13-SDAG-NEXT: s_endpgm
+;
+; GFX13-GISEL-LABEL: v_permlanex16_b32_undef_tid_f64:
+; GFX13-GISEL: ; %bb.0:
+; GFX13-GISEL-NEXT: s_mov_b32 s0, 0x3ff
+; GFX13-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX13-GISEL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s0, v0 bitop3:0x40
+; GFX13-GISEL-NEXT: s_clause 0x1
+; GFX13-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX13-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX13-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1
+; GFX13-GISEL-NEXT: v_permlanex16_b32 v1, v1, s0, s1
+; GFX13-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-GISEL-NEXT: s_endpgm
+ %tidx = call i32 @llvm.amdgcn.workitem.id.x()
+ %tidx_f32 = bitcast i32 %tidx to float
+ %tidx_f64 = fpext float %tidx_f32 to double
+ %undef = freeze double poison
+ %v = call double @llvm.amdgcn.permlanex16.f64(double %undef, double %tidx_f64, i32 %src1, i32 %src2, i1 false, i1 false)
+ store double %v, ptr addrspace(1) %out
ret void
}
@@ -10445,53 +11095,109 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_f64(ptr addrspace(1) %out, i3
}
define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_i32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-LABEL: v_permlanex16_b32_i_tid_fi_i32:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX10-NEXT: global_store_dword v1, v0, s[2:3]
-; GFX10-NEXT: s_endpgm
+; GFX10-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_i32:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_clause 0x1
+; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT: ; freeze: $sgpr4
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s4
+; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX10-SDAG-NEXT: global_store_dword v2, v1, s[2:3]
+; GFX10-SDAG-NEXT: s_endpgm
;
-; GFX11-LABEL: v_permlanex16_b32_i_tid_fi_i32:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX11-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX11-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_i32:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[2:3]
+; GFX10-GISEL-NEXT: s_endpgm
;
-; GFX12-LABEL: v_permlanex16_b32_i_tid_fi_i32:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
-; GFX12-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX12-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX12-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_i32:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_clause 0x1
+; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: ; freeze: $sgpr4
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX11-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX13-LABEL: v_permlanex16_b32_i_tid_fi_i32:
-; GFX13: ; %bb.0:
-; GFX13-NEXT: s_clause 0x1
-; GFX13-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-NEXT: s_mov_b32 s4, 0x3ff
-; GFX13-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
-; GFX13-NEXT: s_wait_kmcnt 0x0
-; GFX13-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX13-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX13-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_i32:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX11-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_i32:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_clause 0x1
+; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-SDAG-NEXT: ; freeze: $sgpr4
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX12-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_i32:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_clause 0x1
+; GFX12-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX12-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
+; GFX12-GISEL-NEXT: s_endpgm
+;
+; GFX13-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_i32:
+; GFX13-SDAG: ; %bb.0:
+; GFX13-SDAG-NEXT: s_clause 0x1
+; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-SDAG-NEXT: ; freeze: $sgpr4
+; GFX13-SDAG-NEXT: s_mov_b32 s5, 0x3ff
+; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
+; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX13-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
+; GFX13-SDAG-NEXT: s_endpgm
+;
+; GFX13-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_i32:
+; GFX13-GISEL: ; %bb.0:
+; GFX13-GISEL-NEXT: s_clause 0x1
+; GFX13-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-GISEL-NEXT: s_mov_b32 s4, 0x3ff
+; GFX13-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX13-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX13-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
+; GFX13-GISEL-NEXT: s_endpgm
%tidx = call i32 @llvm.amdgcn.workitem.id.x()
%undef = freeze i32 poison
%v = call i32 @llvm.amdgcn.permlanex16.i32(i32 %undef, i32 %tidx, i32 %src1, i32 %src2, i1 true, i1 false)
@@ -10500,101 +11206,647 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_i32(ptr addrspace(1) %out,
}
define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_f32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-LABEL: v_permlanex16_b32_i_tid_fi_f32:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX10-NEXT: global_store_dword v1, v0, s[2:3]
-; GFX10-NEXT: s_endpgm
+; GFX10-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_f32:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_clause 0x1
+; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT: ; freeze: $sgpr4
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s4
+; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX10-SDAG-NEXT: global_store_dword v2, v1, s[2:3]
+; GFX10-SDAG-NEXT: s_endpgm
;
-; GFX11-LABEL: v_permlanex16_b32_i_tid_fi_f32:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX11-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX11-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_f32:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[2:3]
+; GFX10-GISEL-NEXT: s_endpgm
;
-; GFX12-LABEL: v_permlanex16_b32_i_tid_fi_f32:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
-; GFX12-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX12-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX12-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_f32:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_clause 0x1
+; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: ; freeze: $sgpr4
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX11-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_f32:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX11-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_f32:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_clause 0x1
+; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-SDAG-NEXT: ; freeze: $sgpr4
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX12-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_f32:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_clause 0x1
+; GFX12-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX12-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
+; GFX12-GISEL-NEXT: s_endpgm
+;
+; GFX13-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_f32:
+; GFX13-SDAG: ; %bb.0:
+; GFX13-SDAG-NEXT: s_clause 0x1
+; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-SDAG-NEXT: ; freeze: $sgpr4
+; GFX13-SDAG-NEXT: s_mov_b32 s5, 0x3ff
+; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
+; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX13-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
+; GFX13-SDAG-NEXT: s_endpgm
+;
+; GFX13-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_f32:
+; GFX13-GISEL: ; %bb.0:
+; GFX13-GISEL-NEXT: s_clause 0x1
+; GFX13-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-GISEL-NEXT: s_mov_b32 s4, 0x3ff
+; GFX13-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX13-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX13-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
+; GFX13-GISEL-NEXT: s_endpgm
+ %tidx = call i32 @llvm.amdgcn.workitem.id.x()
+ %tidx_f32 = bitcast i32 %tidx to float
+ %undef = freeze float poison
+ %v = call float @llvm.amdgcn.permlanex16.f32(float %undef, float %tidx_f32, i32 %src1, i32 %src2, i1 true, i1 false)
+ store float %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_i64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
+; GFX10-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_i64:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_clause 0x1
+; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v3, 0
+; GFX10-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, s5
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s4
+; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_permlanex16_b32 v2, v3, s0, s1 op_sel:[1,0]
+; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX10-SDAG-NEXT: global_store_dwordx2 v3, v[1:2], s[2:3]
+; GFX10-SDAG-NEXT: s_endpgm
+;
+; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_i64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[1,0]
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-GISEL-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_i64:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_clause 0x1
+; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_permlanex16_b32 v2, v3, s0, s1 op_sel:[1,0]
+; GFX11-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX11-SDAG-NEXT: global_store_b64 v3, v[1:2], s[2:3]
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_i64:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX11-GISEL-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[1,0]
+; GFX11-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_i64:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_clause 0x1
+; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_permlanex16_b32 v2, v3, s0, s1 op_sel:[1,0]
+; GFX12-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX12-SDAG-NEXT: global_store_b64 v3, v[1:2], s[2:3]
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_i64:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_clause 0x1
+; GFX12-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX12-GISEL-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[1,0]
+; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-GISEL-NEXT: s_endpgm
+;
+; GFX13-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_i64:
+; GFX13-SDAG: ; %bb.0:
+; GFX13-SDAG-NEXT: s_clause 0x1
+; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX13-SDAG-NEXT: s_mov_b32 s5, 0x3ff
+; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
+; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX13-SDAG-NEXT: v_permlanex16_b32 v2, v3, s0, s1 op_sel:[1,0]
+; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,0]
+; GFX13-SDAG-NEXT: global_store_b64 v3, v[1:2], s[2:3]
+; GFX13-SDAG-NEXT: s_endpgm
+;
+; GFX13-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_i64:
+; GFX13-GISEL: ; %bb.0:
+; GFX13-GISEL-NEXT: s_clause 0x1
+; GFX13-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-GISEL-NEXT: s_mov_b32 s4, 0x3ff
+; GFX13-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-GISEL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX13-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX13-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX13-GISEL-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[1,0]
+; GFX13-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-GISEL-NEXT: s_endpgm
+ %tidx = call i32 @llvm.amdgcn.workitem.id.x()
+ %tidx_i64 = zext i32 %tidx to i64
+ %undef = freeze i64 poison
+ %v = call i64 @llvm.amdgcn.permlanex16.i64(i64 %undef, i64 %tidx_i64, i32 %src1, i32 %src2, i1 true, i1 false)
+ store i64 %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_f64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
+; GFX10-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_f64:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_clause 0x1
+; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; GFX10-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v3, s5
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, s4
+; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_permlanex16_b32 v3, v1, s0, s1 op_sel:[1,0]
+; GFX10-SDAG-NEXT: v_permlanex16_b32 v2, v0, s0, s1 op_sel:[1,0]
+; GFX10-SDAG-NEXT: global_store_dwordx2 v4, v[2:3], s[2:3]
+; GFX10-SDAG-NEXT: s_endpgm
+;
+; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_f64:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[1,0]
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-GISEL-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_f64:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_clause 0x1
+; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; GFX11-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v3, s5 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, s4
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_permlanex16_b32 v3, v1, s0, s1 op_sel:[1,0]
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_permlanex16_b32 v2, v0, s0, s1 op_sel:[1,0]
+; GFX11-SDAG-NEXT: global_store_b64 v4, v[2:3], s[2:3]
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_f64:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX11-GISEL-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[1,0]
+; GFX11-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_f64:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_clause 0x1
+; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; GFX12-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v3, s5 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, s4
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_permlanex16_b32 v3, v1, s0, s1 op_sel:[1,0]
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_permlanex16_b32 v2, v0, s0, s1 op_sel:[1,0]
+; GFX12-SDAG-NEXT: global_store_b64 v4, v[2:3], s[2:3]
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_f64:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-NEXT: s_clause 0x1
+; GFX12-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX12-GISEL-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[1,0]
+; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-GISEL-NEXT: s_endpgm
+;
+; GFX13-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_f64:
+; GFX13-SDAG: ; %bb.0:
+; GFX13-SDAG-NEXT: s_clause 0x1
+; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX13-SDAG-NEXT: s_mov_b32 s6, 0x3ff
+; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v3, s5 :: v_dual_bitop2_b32 v0, s6, v0 bitop3:0x40
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v4, 0
+; GFX13-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-SDAG-NEXT: v_permlanex16_b32 v3, v1, s0, s1 op_sel:[1,0]
+; GFX13-SDAG-NEXT: v_permlanex16_b32 v2, v0, s0, s1 op_sel:[1,0]
+; GFX13-SDAG-NEXT: global_store_b64 v4, v[2:3], s[2:3]
+; GFX13-SDAG-NEXT: s_endpgm
+;
+; GFX13-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_f64:
+; GFX13-GISEL: ; %bb.0:
+; GFX13-GISEL-NEXT: s_mov_b32 s0, 0x3ff
+; GFX13-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX13-GISEL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s0, v0 bitop3:0x40
+; GFX13-GISEL-NEXT: s_clause 0x1
+; GFX13-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX13-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX13-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX13-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX13-GISEL-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[1,0]
+; GFX13-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-GISEL-NEXT: s_endpgm
+ %tidx = call i32 @llvm.amdgcn.workitem.id.x()
+ %tidx_f32 = bitcast i32 %tidx to float
+ %tidx_f64 = fpext float %tidx_f32 to double
+ %undef = freeze double poison
+ %v = call double @llvm.amdgcn.permlanex16.f64(double %undef, double %tidx_f64, i32 %src1, i32 %src2, i1 true, i1 false)
+ store double %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlanex16_b32_i_tid_bc_i32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
+; GFX10-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_i32:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_clause 0x1
+; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT: ; freeze: $sgpr4
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s4
+; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX10-SDAG-NEXT: global_store_dword v2, v1, s[2:3]
+; GFX10-SDAG-NEXT: s_endpgm
+;
+; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_i32:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[2:3]
+; GFX10-GISEL-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_i32:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_clause 0x1
+; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: ; freeze: $sgpr4
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX11-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_i32:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX11-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_i32:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_clause 0x1
+; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-SDAG-NEXT: ; freeze: $sgpr4
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX12-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_i32:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_clause 0x1
+; GFX12-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX12-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
+; GFX12-GISEL-NEXT: s_endpgm
+;
+; GFX13-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_i32:
+; GFX13-SDAG: ; %bb.0:
+; GFX13-SDAG-NEXT: s_clause 0x1
+; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-SDAG-NEXT: ; freeze: $sgpr4
+; GFX13-SDAG-NEXT: s_mov_b32 s5, 0x3ff
+; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
+; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX13-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
+; GFX13-SDAG-NEXT: s_endpgm
+;
+; GFX13-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_i32:
+; GFX13-GISEL: ; %bb.0:
+; GFX13-GISEL-NEXT: s_clause 0x1
+; GFX13-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-GISEL-NEXT: s_mov_b32 s4, 0x3ff
+; GFX13-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX13-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX13-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
+; GFX13-GISEL-NEXT: s_endpgm
+ %tidx = call i32 @llvm.amdgcn.workitem.id.x()
+ %undef = freeze i32 poison
+ %v = call i32 @llvm.amdgcn.permlanex16.i32(i32 %undef, i32 %tidx, i32 %src1, i32 %src2, i1 false, i1 true)
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlanex16_b32_i_tid_bc_f32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
+; GFX10-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_f32:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_clause 0x1
+; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT: ; freeze: $sgpr4
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s4
+; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX10-SDAG-NEXT: global_store_dword v2, v1, s[2:3]
+; GFX10-SDAG-NEXT: s_endpgm
+;
+; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_f32:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[2:3]
+; GFX10-GISEL-NEXT: s_endpgm
+;
+; GFX11-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_f32:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_clause 0x1
+; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: ; freeze: $sgpr4
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX11-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_f32:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX11-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX11-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_f32:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_clause 0x1
+; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-SDAG-NEXT: ; freeze: $sgpr4
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX12-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_f32:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_clause 0x1
+; GFX12-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
+; GFX12-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX12-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX12-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
+; GFX12-GISEL-NEXT: s_endpgm
+;
+; GFX13-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_f32:
+; GFX13-SDAG: ; %bb.0:
+; GFX13-SDAG-NEXT: s_clause 0x1
+; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-SDAG-NEXT: ; freeze: $sgpr4
+; GFX13-SDAG-NEXT: s_mov_b32 s5, 0x3ff
+; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
+; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX13-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
+; GFX13-SDAG-NEXT: s_endpgm
;
-; GFX13-LABEL: v_permlanex16_b32_i_tid_fi_f32:
-; GFX13: ; %bb.0:
-; GFX13-NEXT: s_clause 0x1
-; GFX13-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-NEXT: s_mov_b32 s4, 0x3ff
-; GFX13-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
-; GFX13-NEXT: s_wait_kmcnt 0x0
-; GFX13-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX13-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX13-NEXT: s_endpgm
+; GFX13-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_f32:
+; GFX13-GISEL: ; %bb.0:
+; GFX13-GISEL-NEXT: s_clause 0x1
+; GFX13-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX13-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-GISEL-NEXT: s_mov_b32 s4, 0x3ff
+; GFX13-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX13-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX13-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
+; GFX13-GISEL-NEXT: s_endpgm
%tidx = call i32 @llvm.amdgcn.workitem.id.x()
%tidx_f32 = bitcast i32 %tidx to float
%undef = freeze float poison
- %v = call float @llvm.amdgcn.permlanex16.f32(float %undef, float %tidx_f32, i32 %src1, i32 %src2, i1 true, i1 false)
+ %v = call float @llvm.amdgcn.permlanex16.f32(float %undef, float %tidx_f32, i32 %src1, i32 %src2, i1 false, i1 true)
store float %v, ptr addrspace(1) %out
ret void
}
-define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_i64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_i64:
+define amdgpu_kernel void @v_permlanex16_b32_i_tid_bc_i64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
+; GFX10-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_i64:
; GFX10-SDAG: ; %bb.0:
; GFX10-SDAG-NEXT: s_clause 0x1
; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v3, 0
+; GFX10-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, s5
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s4
; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[1,0]
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-SDAG-NEXT: v_permlanex16_b32 v2, v3, s0, s1 op_sel:[0,1]
+; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX10-SDAG-NEXT: global_store_dwordx2 v3, v[1:2], s[2:3]
; GFX10-SDAG-NEXT: s_endpgm
;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_i64:
+; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_i64:
; GFX10-GISEL: ; %bb.0:
; GFX10-GISEL-NEXT: s_clause 0x1
; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[1,0]
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[0,1]
; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
; GFX10-GISEL-NEXT: s_endpgm
;
-; GFX11-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_i64:
+; GFX11-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_i64:
; GFX11-SDAG: ; %bb.0:
; GFX11-SDAG-NEXT: s_clause 0x1
; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
; GFX11-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[1,0]
-; GFX11-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-SDAG-NEXT: v_permlanex16_b32 v2, v3, s0, s1 op_sel:[0,1]
+; GFX11-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX11-SDAG-NEXT: global_store_b64 v3, v[1:2], s[2:3]
; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX11-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_i64:
+; GFX11-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_i64:
; GFX11-GISEL: ; %bb.0:
; GFX11-GISEL-NEXT: s_clause 0x1
; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
@@ -10603,26 +11855,27 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_i64(ptr addrspace(1) %out,
; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX11-GISEL-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[1,0]
+; GFX11-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX11-GISEL-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[0,1]
; GFX11-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
; GFX11-GISEL-NEXT: s_endpgm
;
-; GFX12-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_i64:
+; GFX12-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_i64:
; GFX12-SDAG: ; %bb.0:
; GFX12-SDAG-NEXT: s_clause 0x1
; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
; GFX12-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX12-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[1,0]
-; GFX12-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX12-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-SDAG-NEXT: v_permlanex16_b32 v2, v3, s0, s1 op_sel:[0,1]
+; GFX12-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX12-SDAG-NEXT: global_store_b64 v3, v[1:2], s[2:3]
; GFX12-SDAG-NEXT: s_endpgm
;
-; GFX12-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_i64:
+; GFX12-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_i64:
; GFX12-GISEL: ; %bb.0:
; GFX12-GISEL-NEXT: s_clause 0x1
; GFX12-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
@@ -10631,27 +11884,29 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_i64(ptr addrspace(1) %out,
; GFX12-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX12-GISEL-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[1,0]
+; GFX12-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX12-GISEL-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[0,1]
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_i64:
+; GFX13-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_i64:
; GFX13-SDAG: ; %bb.0:
; GFX13-SDAG-NEXT: s_clause 0x1
; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: s_mov_b32 s4, 0x3ff
-; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX13-SDAG-NEXT: s_mov_b32 s5, 0x3ff
+; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[1,0]
+; GFX13-SDAG-NEXT: v_permlanex16_b32 v2, v3, s0, s1 op_sel:[0,1]
; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1 op_sel:[0,1]
+; GFX13-SDAG-NEXT: global_store_b64 v3, v[1:2], s[2:3]
; GFX13-SDAG-NEXT: s_endpgm
;
-; GFX13-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_i64:
+; GFX13-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_i64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_clause 0x1
; GFX13-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
@@ -10660,34 +11915,37 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_i64(ptr addrspace(1) %out,
; GFX13-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX13-GISEL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
; GFX13-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX13-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
+; GFX13-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
; GFX13-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX13-GISEL-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[1,0]
+; GFX13-GISEL-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[0,1]
; GFX13-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
; GFX13-GISEL-NEXT: s_endpgm
%tidx = call i32 @llvm.amdgcn.workitem.id.x()
%tidx_i64 = zext i32 %tidx to i64
%undef = freeze i64 poison
- %v = call i64 @llvm.amdgcn.permlanex16.i64(i64 %undef, i64 %tidx_i64, i32 %src1, i32 %src2, i1 true, i1 false)
+ %v = call i64 @llvm.amdgcn.permlanex16.i64(i64 %undef, i64 %tidx_i64, i32 %src1, i32 %src2, i1 false, i1 true)
store i64 %v, ptr addrspace(1) %out
ret void
}
-define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_f64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_f64:
+define amdgpu_kernel void @v_permlanex16_b32_i_tid_bc_f64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
+; GFX10-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_f64:
; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
; GFX10-SDAG-NEXT: s_clause 0x1
; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; GFX10-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v3, s5
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, s4
; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[1,0]
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-SDAG-NEXT: v_permlanex16_b32 v3, v1, s0, s1 op_sel:[0,1]
+; GFX10-SDAG-NEXT: v_permlanex16_b32 v2, v0, s0, s1 op_sel:[0,1]
+; GFX10-SDAG-NEXT: global_store_dwordx2 v4, v[2:3], s[2:3]
; GFX10-SDAG-NEXT: s_endpgm
;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_f64:
+; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_f64:
; GFX10-GISEL: ; %bb.0:
; GFX10-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
; GFX10-GISEL-NEXT: s_clause 0x1
@@ -10695,27 +11953,30 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_f64(ptr addrspace(1) %out,
; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[1,0]
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[0,1]
; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
; GFX10-GISEL-NEXT: s_endpgm
;
-; GFX11-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_f64:
+; GFX11-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_f64:
; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-SDAG-NEXT: s_clause 0x1
; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
; GFX11-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; GFX11-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v3, s5 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, s4
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[1,0]
-; GFX11-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-SDAG-NEXT: v_permlanex16_b32 v3, v1, s0, s1 op_sel:[0,1]
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_permlanex16_b32 v2, v0, s0, s1 op_sel:[0,1]
+; GFX11-SDAG-NEXT: global_store_b64 v4, v[2:3], s[2:3]
; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX11-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_f64:
+; GFX11-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_f64:
; GFX11-GISEL: ; %bb.0:
; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-GISEL-NEXT: s_clause 0x1
@@ -10725,27 +11986,30 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_f64(ptr addrspace(1) %out,
; GFX11-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX11-GISEL-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[1,0]
+; GFX11-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX11-GISEL-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[0,1]
; GFX11-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
; GFX11-GISEL-NEXT: s_endpgm
;
-; GFX12-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_f64:
+; GFX12-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_f64:
; GFX12-SDAG: ; %bb.0:
-; GFX12-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-SDAG-NEXT: s_clause 0x1
; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
; GFX12-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; GFX12-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v3, s5 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, s4
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[1,0]
-; GFX12-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX12-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-SDAG-NEXT: v_permlanex16_b32 v3, v1, s0, s1 op_sel:[0,1]
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_permlanex16_b32 v2, v0, s0, s1 op_sel:[0,1]
+; GFX12-SDAG-NEXT: global_store_b64 v4, v[2:3], s[2:3]
; GFX12-SDAG-NEXT: s_endpgm
;
-; GFX12-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_f64:
+; GFX12-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_f64:
; GFX12-GISEL: ; %bb.0:
; GFX12-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-GISEL-NEXT: s_clause 0x1
@@ -10755,28 +12019,30 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_f64(ptr addrspace(1) %out,
; GFX12-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX12-GISEL-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[1,0]
+; GFX12-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX12-GISEL-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[0,1]
; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_f64:
+; GFX13-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_f64:
; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_mov_b32 s0, 0x3ff
-; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s0, v0 bitop3:0x40
; GFX13-SDAG-NEXT: s_clause 0x1
; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX13-SDAG-NEXT: s_mov_b32 s6, 0x3ff
+; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v3, s5 :: v_dual_bitop2_b32 v0, s6, v0 bitop3:0x40
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v4, 0
; GFX13-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[1,0]
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-SDAG-NEXT: v_permlanex16_b32 v3, v1, s0, s1 op_sel:[0,1]
+; GFX13-SDAG-NEXT: v_permlanex16_b32 v2, v0, s0, s1 op_sel:[0,1]
+; GFX13-SDAG-NEXT: global_store_b64 v4, v[2:3], s[2:3]
; GFX13-SDAG-NEXT: s_endpgm
;
-; GFX13-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_f64:
+; GFX13-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_f64:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_mov_b32 s0, 0x3ff
; GFX13-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
@@ -10787,478 +12053,234 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_f64(ptr addrspace(1) %out,
; GFX13-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
; GFX13-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX13-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,0]
-; GFX13-GISEL-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[1,0]
+; GFX13-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
+; GFX13-GISEL-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[0,1]
; GFX13-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
; GFX13-GISEL-NEXT: s_endpgm
%tidx = call i32 @llvm.amdgcn.workitem.id.x()
%tidx_f32 = bitcast i32 %tidx to float
%tidx_f64 = fpext float %tidx_f32 to double
%undef = freeze double poison
- %v = call double @llvm.amdgcn.permlanex16.f64(double %undef, double %tidx_f64, i32 %src1, i32 %src2, i1 true, i1 false)
+ %v = call double @llvm.amdgcn.permlanex16.f64(double %undef, double %tidx_f64, i32 %src1, i32 %src2, i1 false, i1 true)
store double %v, ptr addrspace(1) %out
ret void
}
-define amdgpu_kernel void @v_permlanex16_b32_i_tid_bc_i32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-LABEL: v_permlanex16_b32_i_tid_bc_i32:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX10-NEXT: global_store_dword v1, v0, s[2:3]
-; GFX10-NEXT: s_endpgm
-;
-; GFX11-LABEL: v_permlanex16_b32_i_tid_bc_i32:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX11-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX11-NEXT: s_endpgm
-;
-; GFX12-LABEL: v_permlanex16_b32_i_tid_bc_i32:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
-; GFX12-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX12-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX12-NEXT: s_endpgm
-;
-; GFX13-LABEL: v_permlanex16_b32_i_tid_bc_i32:
-; GFX13: ; %bb.0:
-; GFX13-NEXT: s_clause 0x1
-; GFX13-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-NEXT: s_mov_b32 s4, 0x3ff
-; GFX13-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
-; GFX13-NEXT: s_wait_kmcnt 0x0
-; GFX13-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX13-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX13-NEXT: s_endpgm
- %tidx = call i32 @llvm.amdgcn.workitem.id.x()
- %undef = freeze i32 poison
- %v = call i32 @llvm.amdgcn.permlanex16.i32(i32 %undef, i32 %tidx, i32 %src1, i32 %src2, i1 false, i1 true)
- store i32 %v, ptr addrspace(1) %out
- ret void
-}
-
-define amdgpu_kernel void @v_permlanex16_b32_i_tid_bc_f32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-LABEL: v_permlanex16_b32_i_tid_bc_f32:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX10-NEXT: global_store_dword v1, v0, s[2:3]
-; GFX10-NEXT: s_endpgm
-;
-; GFX11-LABEL: v_permlanex16_b32_i_tid_bc_f32:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX11-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX11-NEXT: s_endpgm
-;
-; GFX12-LABEL: v_permlanex16_b32_i_tid_bc_f32:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
-; GFX12-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX12-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX12-NEXT: s_endpgm
-;
-; GFX13-LABEL: v_permlanex16_b32_i_tid_bc_f32:
-; GFX13: ; %bb.0:
-; GFX13-NEXT: s_clause 0x1
-; GFX13-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-NEXT: s_mov_b32 s4, 0x3ff
-; GFX13-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
-; GFX13-NEXT: s_wait_kmcnt 0x0
-; GFX13-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX13-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX13-NEXT: s_endpgm
- %tidx = call i32 @llvm.amdgcn.workitem.id.x()
- %tidx_f32 = bitcast i32 %tidx to float
- %undef = freeze float poison
- %v = call float @llvm.amdgcn.permlanex16.f32(float %undef, float %tidx_f32, i32 %src1, i32 %src2, i1 false, i1 true)
- store float %v, ptr addrspace(1) %out
- ret void
-}
-
-define amdgpu_kernel void @v_permlanex16_b32_i_tid_bc_i64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_i64:
+define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_bc_i32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
+; GFX10-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_bc_i32:
; GFX10-SDAG: ; %bb.0:
; GFX10-SDAG-NEXT: s_clause 0x1
; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT: ; freeze: $sgpr4
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s4
; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[0,1]
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX10-SDAG-NEXT: global_store_dword v2, v1, s[2:3]
; GFX10-SDAG-NEXT: s_endpgm
;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_i64:
+; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_bc_i32:
; GFX10-GISEL: ; %bb.0:
; GFX10-GISEL-NEXT: s_clause 0x1
; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[0,1]
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
+; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[2:3]
; GFX10-GISEL-NEXT: s_endpgm
;
-; GFX11-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_i64:
+; GFX11-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_bc_i32:
; GFX11-SDAG: ; %bb.0:
; GFX11-SDAG-NEXT: s_clause 0x1
; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
; GFX11-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT: ; freeze: $sgpr4
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[0,1]
-; GFX11-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX11-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX11-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_i64:
+; GFX11-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_bc_i32:
; GFX11-GISEL: ; %bb.0:
; GFX11-GISEL-NEXT: s_clause 0x1
; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
; GFX11-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX11-GISEL-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[0,1]
-; GFX11-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
+; GFX11-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
; GFX11-GISEL-NEXT: s_endpgm
;
-; GFX12-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_i64:
+; GFX12-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_bc_i32:
; GFX12-SDAG: ; %bb.0:
; GFX12-SDAG-NEXT: s_clause 0x1
; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
; GFX12-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX12-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT: ; freeze: $sgpr4
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[0,1]
-; GFX12-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX12-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX12-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
; GFX12-SDAG-NEXT: s_endpgm
;
-; GFX12-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_i64:
+; GFX12-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_bc_i32:
; GFX12-GISEL: ; %bb.0:
; GFX12-GISEL-NEXT: s_clause 0x1
; GFX12-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
; GFX12-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX12-GISEL-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[0,1]
-; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
+; GFX12-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_i64:
+; GFX13-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_bc_i32:
; GFX13-SDAG: ; %bb.0:
; GFX13-SDAG-NEXT: s_clause 0x1
; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: s_mov_b32 s4, 0x3ff
+; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-SDAG-NEXT: ; freeze: $sgpr4
+; GFX13-SDAG-NEXT: s_mov_b32 s5, 0x3ff
; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[0,1]
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX13-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
; GFX13-SDAG-NEXT: s_endpgm
;
-; GFX13-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_i64:
+; GFX13-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_bc_i32:
; GFX13-GISEL: ; %bb.0:
; GFX13-GISEL-NEXT: s_clause 0x1
; GFX13-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
; GFX13-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
; GFX13-GISEL-NEXT: s_mov_b32 s4, 0x3ff
; GFX13-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-GISEL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
; GFX13-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX13-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX13-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX13-GISEL-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[0,1]
-; GFX13-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
+; GFX13-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
; GFX13-GISEL-NEXT: s_endpgm
%tidx = call i32 @llvm.amdgcn.workitem.id.x()
- %tidx_i64 = zext i32 %tidx to i64
- %undef = freeze i64 poison
- %v = call i64 @llvm.amdgcn.permlanex16.i64(i64 %undef, i64 %tidx_i64, i32 %src1, i32 %src2, i1 false, i1 true)
- store i64 %v, ptr addrspace(1) %out
+ %undef = freeze i32 poison
+ %v = call i32 @llvm.amdgcn.permlanex16.i32(i32 %undef, i32 %tidx, i32 %src1, i32 %src2, i1 true, i1 true)
+ store i32 %v, ptr addrspace(1) %out
ret void
}
-define amdgpu_kernel void @v_permlanex16_b32_i_tid_bc_f64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_f64:
+define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_bc_f32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
+; GFX10-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_bc_f32:
; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
; GFX10-SDAG-NEXT: s_clause 0x1
; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT: ; freeze: $sgpr4
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s4
; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[0,1]
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX10-SDAG-NEXT: global_store_dword v2, v1, s[2:3]
; GFX10-SDAG-NEXT: s_endpgm
;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_f64:
+; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_bc_f32:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
; GFX10-GISEL-NEXT: s_clause 0x1
; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[0,1]
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
+; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[2:3]
; GFX10-GISEL-NEXT: s_endpgm
;
-; GFX11-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_f64:
+; GFX11-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_bc_f32:
; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-SDAG-NEXT: s_clause 0x1
; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
; GFX11-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX11-SDAG-NEXT: ; freeze: $sgpr4
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[0,1]
-; GFX11-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX11-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX11-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_f64:
+; GFX11-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_bc_f32:
; GFX11-GISEL: ; %bb.0:
-; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-GISEL-NEXT: s_clause 0x1
; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
; GFX11-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX11-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX11-GISEL-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[0,1]
-; GFX11-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
+; GFX11-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
; GFX11-GISEL-NEXT: s_endpgm
;
-; GFX12-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_f64:
+; GFX12-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_bc_f32:
; GFX12-SDAG: ; %bb.0:
-; GFX12-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-SDAG-NEXT: s_clause 0x1
; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
; GFX12-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX12-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX12-SDAG-NEXT: ; freeze: $sgpr4
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[0,1]
-; GFX12-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX12-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX12-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
; GFX12-SDAG-NEXT: s_endpgm
;
-; GFX12-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_f64:
+; GFX12-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_bc_f32:
; GFX12-GISEL: ; %bb.0:
-; GFX12-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-GISEL-NEXT: s_clause 0x1
; GFX12-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
; GFX12-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX12-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX12-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX12-GISEL-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[0,1]
-; GFX12-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
+; GFX12-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
; GFX12-GISEL-NEXT: s_endpgm
;
-; GFX13-SDAG-LABEL: v_permlanex16_b32_i_tid_bc_f64:
+; GFX13-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_bc_f32:
; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_mov_b32 s0, 0x3ff
-; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s0, v0 bitop3:0x40
; GFX13-SDAG-NEXT: s_clause 0x1
; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX13-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX13-SDAG-NEXT: ; freeze: $sgpr4
+; GFX13-SDAG-NEXT: s_mov_b32 s5, 0x3ff
+; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[0,1]
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX13-SDAG-NEXT: global_store_b32 v2, v1, s[2:3]
; GFX13-SDAG-NEXT: s_endpgm
;
-; GFX13-GISEL-LABEL: v_permlanex16_b32_i_tid_bc_f64:
+; GFX13-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_bc_f32:
; GFX13-GISEL: ; %bb.0:
-; GFX13-GISEL-NEXT: s_mov_b32 s0, 0x3ff
-; GFX13-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX13-GISEL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s0, v0 bitop3:0x40
; GFX13-GISEL-NEXT: s_clause 0x1
; GFX13-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
; GFX13-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-GISEL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX13-GISEL-NEXT: s_mov_b32 s4, 0x3ff
+; GFX13-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
; GFX13-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX13-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[0,1]
-; GFX13-GISEL-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[0,1]
-; GFX13-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
+; GFX13-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
; GFX13-GISEL-NEXT: s_endpgm
- %tidx = call i32 @llvm.amdgcn.workitem.id.x()
- %tidx_f32 = bitcast i32 %tidx to float
- %tidx_f64 = fpext float %tidx_f32 to double
- %undef = freeze double poison
- %v = call double @llvm.amdgcn.permlanex16.f64(double %undef, double %tidx_f64, i32 %src1, i32 %src2, i1 false, i1 true)
- store double %v, ptr addrspace(1) %out
- ret void
-}
-
-define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_bc_i32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-LABEL: v_permlanex16_b32_i_tid_fi_bc_i32:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX10-NEXT: global_store_dword v1, v0, s[2:3]
-; GFX10-NEXT: s_endpgm
-;
-; GFX11-LABEL: v_permlanex16_b32_i_tid_fi_bc_i32:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX11-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX11-NEXT: s_endpgm
-;
-; GFX12-LABEL: v_permlanex16_b32_i_tid_fi_bc_i32:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
-; GFX12-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX12-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX12-NEXT: s_endpgm
-;
-; GFX13-LABEL: v_permlanex16_b32_i_tid_fi_bc_i32:
-; GFX13: ; %bb.0:
-; GFX13-NEXT: s_clause 0x1
-; GFX13-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-NEXT: s_mov_b32 s4, 0x3ff
-; GFX13-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
-; GFX13-NEXT: s_wait_kmcnt 0x0
-; GFX13-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX13-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX13-NEXT: s_endpgm
- %tidx = call i32 @llvm.amdgcn.workitem.id.x()
- %undef = freeze i32 poison
- %v = call i32 @llvm.amdgcn.permlanex16.i32(i32 %undef, i32 %tidx, i32 %src1, i32 %src2, i1 true, i1 true)
- store i32 %v, ptr addrspace(1) %out
- ret void
-}
-
-define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_bc_f32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX10-LABEL: v_permlanex16_b32_i_tid_fi_bc_f32:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX10-NEXT: global_store_dword v1, v0, s[2:3]
-; GFX10-NEXT: s_endpgm
-;
-; GFX11-LABEL: v_permlanex16_b32_i_tid_fi_bc_f32:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX11-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX11-NEXT: s_endpgm
-;
-; GFX12-LABEL: v_permlanex16_b32_i_tid_fi_bc_f32:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
-; GFX12-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX12-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX12-NEXT: s_endpgm
-;
-; GFX13-LABEL: v_permlanex16_b32_i_tid_fi_bc_f32:
-; GFX13: ; %bb.0:
-; GFX13-NEXT: s_clause 0x1
-; GFX13-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX13-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-NEXT: s_mov_b32 s4, 0x3ff
-; GFX13-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
-; GFX13-NEXT: s_wait_kmcnt 0x0
-; GFX13-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX13-NEXT: global_store_b32 v1, v0, s[2:3]
-; GFX13-NEXT: s_endpgm
%tidx = call i32 @llvm.amdgcn.workitem.id.x()
%tidx_f32 = bitcast i32 %tidx to float
%undef = freeze float poison
@@ -11273,11 +12295,14 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_bc_i64(ptr addrspace(1) %o
; GFX10-SDAG-NEXT: s_clause 0x1
; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v3, 0
+; GFX10-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, s5
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s4
; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[1,1]
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-SDAG-NEXT: v_permlanex16_b32 v2, v3, s0, s1 op_sel:[1,1]
+; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX10-SDAG-NEXT: global_store_dwordx2 v3, v[1:2], s[2:3]
; GFX10-SDAG-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_bc_i64:
@@ -11297,13 +12322,14 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_bc_i64(ptr addrspace(1) %o
; GFX11-SDAG-NEXT: s_clause 0x1
; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
; GFX11-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[1,1]
-; GFX11-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-SDAG-NEXT: v_permlanex16_b32 v2, v3, s0, s1 op_sel:[1,1]
+; GFX11-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX11-SDAG-NEXT: global_store_b64 v3, v[1:2], s[2:3]
; GFX11-SDAG-NEXT: s_endpgm
;
; GFX11-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_bc_i64:
@@ -11325,13 +12351,14 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_bc_i64(ptr addrspace(1) %o
; GFX12-SDAG-NEXT: s_clause 0x1
; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
; GFX12-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX12-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[1,1]
-; GFX12-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX12-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-SDAG-NEXT: v_permlanex16_b32 v2, v3, s0, s1 op_sel:[1,1]
+; GFX12-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX12-SDAG-NEXT: global_store_b64 v3, v[1:2], s[2:3]
; GFX12-SDAG-NEXT: s_endpgm
;
; GFX12-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_bc_i64:
@@ -11353,14 +12380,16 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_bc_i64(ptr addrspace(1) %o
; GFX13-SDAG-NEXT: s_clause 0x1
; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX13-SDAG-NEXT: s_mov_b32 s4, 0x3ff
-; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s5
+; GFX13-SDAG-NEXT: s_mov_b32 s5, 0x3ff
+; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_bitop2_b32 v0, s5, v0 bitop3:0x40
; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v2, s0, s1 op_sel:[1,1]
+; GFX13-SDAG-NEXT: v_permlanex16_b32 v2, v3, s0, s1 op_sel:[1,1]
; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v0, s0, s1 op_sel:[1,1]
+; GFX13-SDAG-NEXT: global_store_b64 v3, v[1:2], s[2:3]
; GFX13-SDAG-NEXT: s_endpgm
;
; GFX13-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_bc_i64:
@@ -11388,15 +12417,18 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_bc_i64(ptr addrspace(1) %o
define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_bc_f64(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
; GFX10-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_bc_f64:
; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
; GFX10-SDAG-NEXT: s_clause 0x1
; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; GFX10-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v3, s5
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, s4
; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[1,1]
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX10-SDAG-NEXT: v_permlanex16_b32 v3, v1, s0, s1 op_sel:[1,1]
+; GFX10-SDAG-NEXT: v_permlanex16_b32 v2, v0, s0, s1 op_sel:[1,1]
+; GFX10-SDAG-NEXT: global_store_dwordx2 v4, v[2:3], s[2:3]
; GFX10-SDAG-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_bc_f64:
@@ -11414,17 +12446,20 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_bc_f64(ptr addrspace(1) %o
;
; GFX11-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_bc_f64:
; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-SDAG-NEXT: s_clause 0x1
; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
; GFX11-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; GFX11-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v3, s5 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, s4
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[1,1]
-; GFX11-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX11-SDAG-NEXT: v_permlanex16_b32 v3, v1, s0, s1 op_sel:[1,1]
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_permlanex16_b32 v2, v0, s0, s1 op_sel:[1,1]
+; GFX11-SDAG-NEXT: global_store_b64 v4, v[2:3], s[2:3]
; GFX11-SDAG-NEXT: s_endpgm
;
; GFX11-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_bc_f64:
@@ -11444,17 +12479,20 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_bc_f64(ptr addrspace(1) %o
;
; GFX12-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_bc_f64:
; GFX12-SDAG: ; %bb.0:
-; GFX12-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-SDAG-NEXT: s_clause 0x1
; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30
; GFX12-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; GFX12-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v3, s5 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, s4
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[1,1]
-; GFX12-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX12-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-SDAG-NEXT: v_permlanex16_b32 v3, v1, s0, s1 op_sel:[1,1]
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_permlanex16_b32 v2, v0, s0, s1 op_sel:[1,1]
+; GFX12-SDAG-NEXT: global_store_b64 v4, v[2:3], s[2:3]
; GFX12-SDAG-NEXT: s_endpgm
;
; GFX12-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_bc_f64:
@@ -11474,18 +12512,20 @@ define amdgpu_kernel void @v_permlanex16_b32_i_tid_fi_bc_f64(ptr addrspace(1) %o
;
; GFX13-SDAG-LABEL: v_permlanex16_b32_i_tid_fi_bc_f64:
; GFX13-SDAG: ; %bb.0:
-; GFX13-SDAG-NEXT: s_mov_b32 s0, 0x3ff
-; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s0, v0 bitop3:0x40
; GFX13-SDAG-NEXT: s_clause 0x1
; GFX13-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
; GFX13-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX13-SDAG-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX13-SDAG-NEXT: s_mov_b32 s6, 0x3ff
+; GFX13-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v3, s5 :: v_dual_bitop2_b32 v0, s6, v0 bitop3:0x40
+; GFX13-SDAG-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v4, 0
; GFX13-SDAG-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
; GFX13-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX13-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v1, v1, s0, s1 op_sel:[1,1]
-; GFX13-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1 op_sel:[1,1]
-; GFX13-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX13-SDAG-NEXT: v_permlanex16_b32 v3, v1, s0, s1 op_sel:[1,1]
+; GFX13-SDAG-NEXT: v_permlanex16_b32 v2, v0, s0, s1 op_sel:[1,1]
+; GFX13-SDAG-NEXT: global_store_b64 v4, v[2:3], s[2:3]
; GFX13-SDAG-NEXT: s_endpgm
;
; GFX13-GISEL-LABEL: v_permlanex16_b32_i_tid_fi_bc_f64:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane16.var.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane16.var.ll
index bb0aff99a2a761..d8ea5ab194e2af 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane16.var.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane16.var.ll
@@ -426,11 +426,13 @@ define amdgpu_kernel void @v_permlane16var_b32_undef_tid(ptr addrspace(1) %out,
; GFX12-SDAG-NEXT: s_load_b32 s2, s[4:5], 0x30
; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX12-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT: ; freeze: $sgpr3
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-NEXT: v_permlane16_var_b32 v0, v0, v1
-; GFX12-SDAG-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, s3
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v3, 0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_permlane16_var_b32 v2, v0, v1
+; GFX12-SDAG-NEXT: global_store_b32 v3, v2, s[0:1]
; GFX12-SDAG-NEXT: s_endpgm
;
; GFX12-GISEL-LABEL: v_permlane16var_b32_undef_tid:
@@ -493,11 +495,13 @@ define amdgpu_kernel void @v_permlane16var_b32_i_tid_fi(ptr addrspace(1) %out, i
; GFX12-SDAG-NEXT: s_load_b32 s2, s[4:5], 0x30
; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX12-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT: ; freeze: $sgpr3
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-NEXT: v_permlane16_var_b32 v0, v0, v1 op_sel:[1,0]
-; GFX12-SDAG-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, s3
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v3, 0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_permlane16_var_b32 v2, v0, v1 op_sel:[1,0]
+; GFX12-SDAG-NEXT: global_store_b32 v3, v2, s[0:1]
; GFX12-SDAG-NEXT: s_endpgm
;
; GFX12-GISEL-LABEL: v_permlane16var_b32_i_tid_fi:
@@ -526,11 +530,13 @@ define amdgpu_kernel void @v_permlane16var_b32_i_tid_bc(ptr addrspace(1) %out, i
; GFX12-SDAG-NEXT: s_load_b32 s2, s[4:5], 0x30
; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX12-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT: ; freeze: $sgpr3
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-NEXT: v_permlane16_var_b32 v0, v0, v1 op_sel:[0,1]
-; GFX12-SDAG-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, s3
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v3, 0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_permlane16_var_b32 v2, v0, v1 op_sel:[0,1]
+; GFX12-SDAG-NEXT: global_store_b32 v3, v2, s[0:1]
; GFX12-SDAG-NEXT: s_endpgm
;
; GFX12-GISEL-LABEL: v_permlane16var_b32_i_tid_bc:
@@ -559,11 +565,13 @@ define amdgpu_kernel void @v_permlane16var_b32_i_tid_fi_bc(ptr addrspace(1) %out
; GFX12-SDAG-NEXT: s_load_b32 s2, s[4:5], 0x30
; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX12-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT: ; freeze: $sgpr3
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-NEXT: v_permlane16_var_b32 v0, v0, v1 op_sel:[1,1]
-; GFX12-SDAG-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, s3
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v3, 0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_permlane16_var_b32 v2, v0, v1 op_sel:[1,1]
+; GFX12-SDAG-NEXT: global_store_b32 v3, v2, s[0:1]
; GFX12-SDAG-NEXT: s_endpgm
;
; GFX12-GISEL-LABEL: v_permlane16var_b32_i_tid_fi_bc:
@@ -624,11 +632,13 @@ define amdgpu_kernel void @v_permlanex16var_b32_undef_tid(ptr addrspace(1) %out,
; GFX12-SDAG-NEXT: s_load_b32 s2, s[4:5], 0x30
; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX12-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT: ; freeze: $sgpr3
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-NEXT: v_permlanex16_var_b32 v0, v0, v1
-; GFX12-SDAG-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, s3
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v3, 0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_permlanex16_var_b32 v2, v0, v1
+; GFX12-SDAG-NEXT: global_store_b32 v3, v2, s[0:1]
; GFX12-SDAG-NEXT: s_endpgm
;
; GFX12-GISEL-LABEL: v_permlanex16var_b32_undef_tid:
@@ -691,11 +701,13 @@ define amdgpu_kernel void @v_permlanex16var_b32_i_tid_fi(ptr addrspace(1) %out,
; GFX12-SDAG-NEXT: s_load_b32 s2, s[4:5], 0x30
; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX12-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT: ; freeze: $sgpr3
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-NEXT: v_permlanex16_var_b32 v0, v0, v1 op_sel:[1,0]
-; GFX12-SDAG-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, s3
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v3, 0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_permlanex16_var_b32 v2, v0, v1 op_sel:[1,0]
+; GFX12-SDAG-NEXT: global_store_b32 v3, v2, s[0:1]
; GFX12-SDAG-NEXT: s_endpgm
;
; GFX12-GISEL-LABEL: v_permlanex16var_b32_i_tid_fi:
@@ -724,11 +736,13 @@ define amdgpu_kernel void @v_permlanex16var_b32_i_tid_bc(ptr addrspace(1) %out,
; GFX12-SDAG-NEXT: s_load_b32 s2, s[4:5], 0x30
; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX12-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT: ; freeze: $sgpr3
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-NEXT: v_permlanex16_var_b32 v0, v0, v1 op_sel:[0,1]
-; GFX12-SDAG-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, s3
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v3, 0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_permlanex16_var_b32 v2, v0, v1 op_sel:[0,1]
+; GFX12-SDAG-NEXT: global_store_b32 v3, v2, s[0:1]
; GFX12-SDAG-NEXT: s_endpgm
;
; GFX12-GISEL-LABEL: v_permlanex16var_b32_i_tid_bc:
@@ -757,11 +771,13 @@ define amdgpu_kernel void @v_permlanex16var_b32_i_tid_fi_bc(ptr addrspace(1) %ou
; GFX12-SDAG-NEXT: s_load_b32 s2, s[4:5], 0x30
; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX12-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-NEXT: ; freeze: $sgpr3
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-NEXT: v_permlanex16_var_b32 v0, v0, v1 op_sel:[1,1]
-; GFX12-SDAG-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, s3
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v3, 0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_permlanex16_var_b32 v2, v0, v1 op_sel:[1,1]
+; GFX12-SDAG-NEXT: global_store_b32 v3, v2, s[0:1]
; GFX12-SDAG-NEXT: s_endpgm
;
; GFX12-GISEL-LABEL: v_permlanex16var_b32_i_tid_fi_bc:
diff --git a/llvm/test/CodeGen/AMDGPU/mad_64_32.ll b/llvm/test/CodeGen/AMDGPU/mad_64_32.ll
index 68767c5183c3ab..e3dfbb8b6add35 100644
--- a/llvm/test/CodeGen/AMDGPU/mad_64_32.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad_64_32.ll
@@ -528,8 +528,10 @@ define i63 @mad_i64_i32_sextops_i31_i63(i31 %arg0, i31 %arg1, i63 %arg2) #0 {
; SI-LABEL: mad_i64_i32_sextops_i31_i63:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: v_mov_b32_e32 v4, v1
+; SI-NEXT: v_mov_b32_e32 v5, v2
; SI-NEXT: v_bfe_i32 v0, v0, 0, 31
-; SI-NEXT: v_bfe_i32 v1, v1, 0, 31
+; SI-NEXT: v_bfe_i32 v1, v4, 0, 31
; SI-NEXT: v_mul_lo_u32 v4, v0, v1
; SI-NEXT: v_mul_hi_i32 v1, v0, v1
; SI-NEXT: v_add_i32_e32 v0, vcc, v4, v2
diff --git a/llvm/test/CodeGen/AMDGPU/mdt-preserving-crash.ll b/llvm/test/CodeGen/AMDGPU/mdt-preserving-crash.ll
index 93feacb33c11af..7b0bf9ad6a0b2b 100644
--- a/llvm/test/CodeGen/AMDGPU/mdt-preserving-crash.ll
+++ b/llvm/test/CodeGen/AMDGPU/mdt-preserving-crash.ll
@@ -25,9 +25,10 @@ define protected amdgpu_kernel void @_RSENC_PRInit______________________________
; CHECK-NEXT: s_and_b64 vcc, exec, s[4:5]
; CHECK-NEXT: s_cbranch_vccnz .LBB0_11
; CHECK-NEXT: ; %bb.2: ; %lor.lhs.false17
-; CHECK-NEXT: s_cmp_eq_u32 s4, 0
+; CHECK-NEXT: ; freeze: $sgpr4
; CHECK-NEXT: .LBB0_3: ; %while.cond.i
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: s_cmp_eq_u32 s4, 0
; CHECK-NEXT: s_cbranch_scc1 .LBB0_3
; CHECK-NEXT: ; %bb.4: ; %if.end60
; CHECK-NEXT: s_cbranch_execz .LBB0_10
diff --git a/llvm/test/CodeGen/AMDGPU/mul_int24.ll b/llvm/test/CodeGen/AMDGPU/mul_int24.ll
index 155909e6af0a39..935cc3ae572ba1 100644
--- a/llvm/test/CodeGen/AMDGPU/mul_int24.ll
+++ b/llvm/test/CodeGen/AMDGPU/mul_int24.ll
@@ -497,13 +497,13 @@ define amdgpu_kernel void @test_smul24_i33(ptr addrspace(1) %out, i33 %a, i33 %b
; GFX9-LABEL: test_smul24_i33:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dword s7, s[4:5], 0x34
+; GFX9-NEXT: s_load_dword s8, s[4:5], 0x34
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_mov_b32 s3, 0xf000
; GFX9-NEXT: s_mov_b32 s2, -1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_bfe_i32 s4, s6, 0x180000
-; GFX9-NEXT: s_bfe_i32 s6, s7, 0x180000
+; GFX9-NEXT: s_bfe_i32 s6, s8, 0x180000
; GFX9-NEXT: s_mul_hi_i32 s5, s4, s6
; GFX9-NEXT: s_mul_i32 s4, s4, s6
; GFX9-NEXT: s_lshl_b64 s[4:5], s[4:5], 31
@@ -599,13 +599,13 @@ define amdgpu_kernel void @test_smulhi24_i33(ptr addrspace(1) %out, i33 %a, i33
; GFX9-LABEL: test_smulhi24_i33:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dword s7, s[4:5], 0x34
+; GFX9-NEXT: s_load_dword s8, s[4:5], 0x34
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_mov_b32 s3, 0xf000
; GFX9-NEXT: s_mov_b32 s2, -1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_bfe_i32 s4, s6, 0x180000
-; GFX9-NEXT: s_bfe_i32 s5, s7, 0x180000
+; GFX9-NEXT: s_bfe_i32 s5, s8, 0x180000
; GFX9-NEXT: s_mul_hi_i32 s4, s4, s5
; GFX9-NEXT: s_and_b32 s4, s4, 1
; GFX9-NEXT: v_mov_b32_e32 v0, s4
diff --git a/llvm/test/CodeGen/AMDGPU/permute_i8.ll b/llvm/test/CodeGen/AMDGPU/permute_i8.ll
index 0e67c1c517b684..8cda8b0e97f7a6 100644
--- a/llvm/test/CodeGen/AMDGPU/permute_i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/permute_i8.ll
@@ -3768,18 +3768,21 @@ define hidden void @any_extend_to_perm_wide_src(ptr addrspace(1) %in0, ptr addrs
; GFX10-NEXT: global_load_ubyte v8, v[2:3], off
; GFX10-NEXT: ; kill: killed $vgpr0 killed $vgpr1
; GFX10-NEXT: ; kill: killed $vgpr2 killed $vgpr3
-; GFX10-NEXT: s_lshl_b32 s4, s4, 8
-; GFX10-NEXT: s_lshl_b32 s5, s4, 8
+; GFX10-NEXT: ; freeze: $sgpr4
+; GFX10-NEXT: ; freeze: $sgpr5
+; GFX10-NEXT: ; freeze: $sgpr6
+; GFX10-NEXT: s_lshl_b32 s5, s5, 8
+; GFX10-NEXT: s_lshl_b32 s6, s6, 8
; GFX10-NEXT: s_waitcnt vmcnt(2)
; GFX10-NEXT: v_lshrrev_b64 v[0:1], 24, v[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(1)
; GFX10-NEXT: v_lshrrev_b64 v[1:2], 16, v[12:13]
; GFX10-NEXT: v_lshrrev_b64 v[2:3], 8, v[10:11]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_or_b32_sdwa v3, v8, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v3, v8, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX10-NEXT: v_perm_b32 v0, v1, v0, 0xc0c0004
; GFX10-NEXT: v_perm_b32 v1, s4, v2, 0xc0c0004
-; GFX10-NEXT: v_or_b32_sdwa v2, v9, s5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v2, v9, s6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-NEXT: v_lshl_or_b32 v0, v0, 16, v1
; GFX10-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: global_store_dwordx2 v[4:5], v[0:1], off
@@ -3794,19 +3797,22 @@ define hidden void @any_extend_to_perm_wide_src(ptr addrspace(1) %in0, ptr addrs
; GFX9-NEXT: ; kill: killed $vgpr0 killed $vgpr1
; GFX9-NEXT: ; kill: killed $vgpr2 killed $vgpr3
; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
+; GFX9-NEXT: ; freeze: $sgpr6
+; GFX9-NEXT: ; freeze: $sgpr7
; GFX9-NEXT: v_mov_b32_e32 v14, 0xc0c0004
-; GFX9-NEXT: s_lshl_b32 s5, s4, 8
-; GFX9-NEXT: s_lshl_b32 s6, s4, 8
+; GFX9-NEXT: s_lshl_b32 s6, s6, 8
+; GFX9-NEXT: s_lshl_b32 s7, s7, 8
+; GFX9-NEXT: ; freeze: $sgpr5
; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_lshrrev_b64 v[0:1], 8, v[6:7]
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshrrev_b64 v[1:2], 24, v[10:11]
; GFX9-NEXT: v_lshrrev_b64 v[2:3], 16, v[8:9]
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_or_b32_sdwa v3, v12, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v6, v13, s6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v3, v12, s6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v6, v13, s7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_perm_b32 v1, v2, v1, s4
-; GFX9-NEXT: v_perm_b32 v0, s4, v0, v14
+; GFX9-NEXT: v_perm_b32 v0, s5, v0, v14
; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX9-NEXT: v_or_b32_sdwa v1, v3, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_dwordx2 v[4:5], v[0:1], off
diff --git a/llvm/test/CodeGen/AMDGPU/promote-alloca-vector-dynamic-idx-bitcasts-llc.ll b/llvm/test/CodeGen/AMDGPU/promote-alloca-vector-dynamic-idx-bitcasts-llc.ll
index c73d31c368846b..ddfe187cc43377 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-alloca-vector-dynamic-idx-bitcasts-llc.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-alloca-vector-dynamic-idx-bitcasts-llc.ll
@@ -6,46 +6,144 @@
define amdgpu_kernel void @test_bitcast_llc_v128i8_v16i8(ptr addrspace(1) %out, i32 %idx) {
; GFX9-LABEL: test_bitcast_llc_v128i8_v16i8:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: s_load_dword s33, s[4:5], 0x8
+; GFX9-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x0
+; GFX9-NEXT: ; freeze: $sgpr30_sgpr31
+; GFX9-NEXT: ; freeze: $sgpr28_sgpr29
+; GFX9-NEXT: ; freeze: $sgpr26_sgpr27
+; GFX9-NEXT: ; freeze: $sgpr24_sgpr25
+; GFX9-NEXT: ; freeze: $sgpr22_sgpr23
+; GFX9-NEXT: ; freeze: $sgpr20_sgpr21
+; GFX9-NEXT: ; freeze: $sgpr18_sgpr19
+; GFX9-NEXT: ; freeze: $sgpr16_sgpr17
+; GFX9-NEXT: ; freeze: $sgpr14_sgpr15
+; GFX9-NEXT: ; freeze: $sgpr12_sgpr13
+; GFX9-NEXT: ; freeze: $sgpr10_sgpr11
+; GFX9-NEXT: ; freeze: $sgpr8_sgpr9
+; GFX9-NEXT: ; freeze: $sgpr6_sgpr7
+; GFX9-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX9-NEXT: ; freeze: $sgpr2_sgpr3
+; GFX9-NEXT: ; freeze: $sgpr0_sgpr1
+; GFX9-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX9-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX9-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshl_b32 s2, s2, 2
-; GFX9-NEXT: s_set_gpr_idx_on s2, gpr_idx(SRC0)
-; GFX9-NEXT: v_mov_b32_e32 v3, v3
-; GFX9-NEXT: v_mov_b32_e32 v2, v2
-; GFX9-NEXT: v_mov_b32_e32 v1, v1
-; GFX9-NEXT: v_mov_b32_e32 v0, v0
+; GFX9-NEXT: s_lshl_b32 s33, s33, 2
+; GFX9-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX9-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX9-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX9-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX9-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX9-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
+; GFX9-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; GFX9-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
+; GFX9-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
+; GFX9-NEXT: v_mov_b64_e32 v[24:25], s[24:25]
+; GFX9-NEXT: v_mov_b64_e32 v[26:27], s[26:27]
+; GFX9-NEXT: v_mov_b64_e32 v[28:29], s[28:29]
+; GFX9-NEXT: v_mov_b64_e32 v[30:31], s[30:31]
+; GFX9-NEXT: s_set_gpr_idx_on s33, gpr_idx(SRC0)
+; GFX9-NEXT: v_mov_b32_e32 v35, v3
+; GFX9-NEXT: v_mov_b32_e32 v34, v2
; GFX9-NEXT: s_set_gpr_idx_off
-; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX9-NEXT: v_mov_b32_e32 v36, 0
+; GFX9-NEXT: s_set_gpr_idx_on s33, gpr_idx(SRC0)
+; GFX9-NEXT: v_mov_b32_e32 v33, v1
+; GFX9-NEXT: v_mov_b32_e32 v32, v0
+; GFX9-NEXT: s_set_gpr_idx_off
+; GFX9-NEXT: global_store_dwordx4 v36, v[32:35], s[34:35]
; GFX9-NEXT: s_endpgm
;
; GFX11-LABEL: test_bitcast_llc_v128i8_v16i8:
; GFX11: ; %bb.0: ; %entry
; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX11-NEXT: v_mov_b32_e32 v4, 0
+; GFX11-NEXT: s_load_b32 s33, s[4:5], 0x8
+; GFX11-NEXT: s_load_b64 s[34:35], s[4:5], 0x0
+; GFX11-NEXT: ; freeze: $sgpr30_sgpr31
+; GFX11-NEXT: ; freeze: $sgpr28_sgpr29
+; GFX11-NEXT: ; freeze: $sgpr26_sgpr27
+; GFX11-NEXT: ; freeze: $sgpr24_sgpr25
+; GFX11-NEXT: ; freeze: $sgpr22_sgpr23
+; GFX11-NEXT: ; freeze: $sgpr20_sgpr21
+; GFX11-NEXT: ; freeze: $sgpr18_sgpr19
+; GFX11-NEXT: ; freeze: $sgpr16_sgpr17
+; GFX11-NEXT: ; freeze: $sgpr14_sgpr15
+; GFX11-NEXT: ; freeze: $sgpr12_sgpr13
+; GFX11-NEXT: ; freeze: $sgpr10_sgpr11
+; GFX11-NEXT: ; freeze: $sgpr8_sgpr9
+; GFX11-NEXT: ; freeze: $sgpr6_sgpr7
+; GFX11-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX11-NEXT: ; freeze: $sgpr2_sgpr3
+; GFX11-NEXT: ; freeze: $sgpr0_sgpr1
+; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX11-NEXT: v_dual_mov_b32 v30, s30 :: v_dual_mov_b32 v31, s31
+; GFX11-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
+; GFX11-NEXT: v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s7
+; GFX11-NEXT: v_dual_mov_b32 v8, s8 :: v_dual_mov_b32 v9, s9
+; GFX11-NEXT: v_dual_mov_b32 v10, s10 :: v_dual_mov_b32 v11, s11
+; GFX11-NEXT: v_dual_mov_b32 v12, s12 :: v_dual_mov_b32 v13, s13
+; GFX11-NEXT: v_dual_mov_b32 v14, s14 :: v_dual_mov_b32 v15, s15
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b32 m0, s2, 2
-; GFX11-NEXT: v_movrels_b32_e32 v3, v3
-; GFX11-NEXT: v_movrels_b32_e32 v2, v2
-; GFX11-NEXT: v_movrels_b32_e32 v1, v1
-; GFX11-NEXT: v_movrels_b32_e32 v0, v0
-; GFX11-NEXT: global_store_b128 v4, v[0:3], s[0:1]
+; GFX11-NEXT: s_lshl_b32 m0, s33, 2
+; GFX11-NEXT: v_dual_mov_b32 v16, s16 :: v_dual_mov_b32 v17, s17
+; GFX11-NEXT: v_dual_mov_b32 v18, s18 :: v_dual_mov_b32 v19, s19
+; GFX11-NEXT: v_dual_mov_b32 v20, s20 :: v_dual_mov_b32 v21, s21
+; GFX11-NEXT: v_dual_mov_b32 v22, s22 :: v_dual_mov_b32 v23, s23
+; GFX11-NEXT: v_dual_mov_b32 v24, s24 :: v_dual_mov_b32 v25, s25
+; GFX11-NEXT: v_dual_mov_b32 v26, s26 :: v_dual_mov_b32 v27, s27
+; GFX11-NEXT: v_dual_mov_b32 v28, s28 :: v_dual_mov_b32 v29, s29
+; GFX11-NEXT: v_mov_b32_e32 v35, 0
+; GFX11-NEXT: v_movrels_b32_e32 v34, v3
+; GFX11-NEXT: v_movrels_b32_e32 v33, v2
+; GFX11-NEXT: v_movrels_b32_e32 v32, v1
+; GFX11-NEXT: v_movrels_b32_e32 v31, v0
+; GFX11-NEXT: global_store_b128 v35, v[31:34], s[34:35]
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: test_bitcast_llc_v128i8_v16i8:
; GFX12: ; %bb.0: ; %entry
-; GFX12-NEXT: s_load_b96 s[0:2], s[4:5], 0x0
-; GFX12-NEXT: v_mov_b32_e32 v4, 0
+; GFX12-NEXT: s_load_b96 s[36:38], s[4:5], 0x0
+; GFX12-NEXT: ; freeze: $sgpr30_sgpr31
+; GFX12-NEXT: ; freeze: $sgpr28_sgpr29
+; GFX12-NEXT: ; freeze: $sgpr26_sgpr27
+; GFX12-NEXT: ; freeze: $sgpr24_sgpr25
+; GFX12-NEXT: ; freeze: $sgpr22_sgpr23
+; GFX12-NEXT: ; freeze: $sgpr20_sgpr21
+; GFX12-NEXT: ; freeze: $sgpr18_sgpr19
+; GFX12-NEXT: ; freeze: $sgpr16_sgpr17
+; GFX12-NEXT: ; freeze: $sgpr14_sgpr15
+; GFX12-NEXT: ; freeze: $sgpr12_sgpr13
+; GFX12-NEXT: ; freeze: $sgpr10_sgpr11
+; GFX12-NEXT: ; freeze: $sgpr8_sgpr9
+; GFX12-NEXT: ; freeze: $sgpr6_sgpr7
+; GFX12-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX12-NEXT: ; freeze: $sgpr2_sgpr3
+; GFX12-NEXT: ; freeze: $sgpr0_sgpr1
+; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-NEXT: v_dual_mov_b32 v30, s30 :: v_dual_mov_b32 v31, s31
+; GFX12-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
+; GFX12-NEXT: v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s7
+; GFX12-NEXT: v_dual_mov_b32 v8, s8 :: v_dual_mov_b32 v9, s9
+; GFX12-NEXT: v_dual_mov_b32 v10, s10 :: v_dual_mov_b32 v11, s11
+; GFX12-NEXT: v_dual_mov_b32 v12, s12 :: v_dual_mov_b32 v13, s13
+; GFX12-NEXT: v_dual_mov_b32 v14, s14 :: v_dual_mov_b32 v15, s15
+; GFX12-NEXT: v_dual_mov_b32 v16, s16 :: v_dual_mov_b32 v17, s17
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_lshl_b32 m0, s2, 2
-; GFX12-NEXT: v_movrels_b32_e32 v3, v3
-; GFX12-NEXT: v_movrels_b32_e32 v2, v2
-; GFX12-NEXT: v_movrels_b32_e32 v1, v1
-; GFX12-NEXT: v_movrels_b32_e32 v0, v0
-; GFX12-NEXT: global_store_b128 v4, v[0:3], s[0:1]
+; GFX12-NEXT: s_lshl_b32 m0, s38, 2
+; GFX12-NEXT: v_dual_mov_b32 v18, s18 :: v_dual_mov_b32 v19, s19
+; GFX12-NEXT: v_dual_mov_b32 v20, s20 :: v_dual_mov_b32 v21, s21
+; GFX12-NEXT: v_dual_mov_b32 v22, s22 :: v_dual_mov_b32 v23, s23
+; GFX12-NEXT: v_dual_mov_b32 v24, s24 :: v_dual_mov_b32 v25, s25
+; GFX12-NEXT: v_dual_mov_b32 v26, s26 :: v_dual_mov_b32 v27, s27
+; GFX12-NEXT: v_dual_mov_b32 v28, s28 :: v_dual_mov_b32 v29, s29
+; GFX12-NEXT: v_mov_b32_e32 v35, 0
+; GFX12-NEXT: v_movrels_b32_e32 v34, v3
+; GFX12-NEXT: v_movrels_b32_e32 v33, v2
+; GFX12-NEXT: v_movrels_b32_e32 v32, v1
+; GFX12-NEXT: v_movrels_b32_e32 v31, v0
+; GFX12-NEXT: global_store_b128 v35, v[31:34], s[36:37]
; GFX12-NEXT: s_endpgm
entry:
%alloca = freeze <128 x i8> poison
@@ -59,46 +157,144 @@ entry:
define amdgpu_kernel void @test_bitcast_llc_v64i16_v8i16(ptr addrspace(1) %out, i32 %idx) {
; GFX9-LABEL: test_bitcast_llc_v64i16_v8i16:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: s_load_dword s33, s[4:5], 0x8
+; GFX9-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x0
+; GFX9-NEXT: ; freeze: $sgpr30_sgpr31
+; GFX9-NEXT: ; freeze: $sgpr28_sgpr29
+; GFX9-NEXT: ; freeze: $sgpr26_sgpr27
+; GFX9-NEXT: ; freeze: $sgpr24_sgpr25
+; GFX9-NEXT: ; freeze: $sgpr22_sgpr23
+; GFX9-NEXT: ; freeze: $sgpr20_sgpr21
+; GFX9-NEXT: ; freeze: $sgpr18_sgpr19
+; GFX9-NEXT: ; freeze: $sgpr16_sgpr17
+; GFX9-NEXT: ; freeze: $sgpr14_sgpr15
+; GFX9-NEXT: ; freeze: $sgpr12_sgpr13
+; GFX9-NEXT: ; freeze: $sgpr10_sgpr11
+; GFX9-NEXT: ; freeze: $sgpr8_sgpr9
+; GFX9-NEXT: ; freeze: $sgpr6_sgpr7
+; GFX9-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX9-NEXT: ; freeze: $sgpr2_sgpr3
+; GFX9-NEXT: ; freeze: $sgpr0_sgpr1
+; GFX9-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX9-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX9-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshl_b32 s2, s2, 2
-; GFX9-NEXT: s_set_gpr_idx_on s2, gpr_idx(SRC0)
-; GFX9-NEXT: v_mov_b32_e32 v3, v3
-; GFX9-NEXT: v_mov_b32_e32 v2, v2
-; GFX9-NEXT: v_mov_b32_e32 v1, v1
-; GFX9-NEXT: v_mov_b32_e32 v0, v0
+; GFX9-NEXT: s_lshl_b32 s33, s33, 2
+; GFX9-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX9-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX9-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX9-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX9-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX9-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
+; GFX9-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; GFX9-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
+; GFX9-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
+; GFX9-NEXT: v_mov_b64_e32 v[24:25], s[24:25]
+; GFX9-NEXT: v_mov_b64_e32 v[26:27], s[26:27]
+; GFX9-NEXT: v_mov_b64_e32 v[28:29], s[28:29]
+; GFX9-NEXT: v_mov_b64_e32 v[30:31], s[30:31]
+; GFX9-NEXT: s_set_gpr_idx_on s33, gpr_idx(SRC0)
+; GFX9-NEXT: v_mov_b32_e32 v35, v3
+; GFX9-NEXT: v_mov_b32_e32 v34, v2
+; GFX9-NEXT: s_set_gpr_idx_off
+; GFX9-NEXT: v_mov_b32_e32 v36, 0
+; GFX9-NEXT: s_set_gpr_idx_on s33, gpr_idx(SRC0)
+; GFX9-NEXT: v_mov_b32_e32 v33, v1
+; GFX9-NEXT: v_mov_b32_e32 v32, v0
; GFX9-NEXT: s_set_gpr_idx_off
-; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX9-NEXT: global_store_dwordx4 v36, v[32:35], s[34:35]
; GFX9-NEXT: s_endpgm
;
; GFX11-LABEL: test_bitcast_llc_v64i16_v8i16:
; GFX11: ; %bb.0: ; %entry
; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX11-NEXT: v_mov_b32_e32 v4, 0
+; GFX11-NEXT: s_load_b32 s33, s[4:5], 0x8
+; GFX11-NEXT: s_load_b64 s[34:35], s[4:5], 0x0
+; GFX11-NEXT: ; freeze: $sgpr30_sgpr31
+; GFX11-NEXT: ; freeze: $sgpr28_sgpr29
+; GFX11-NEXT: ; freeze: $sgpr26_sgpr27
+; GFX11-NEXT: ; freeze: $sgpr24_sgpr25
+; GFX11-NEXT: ; freeze: $sgpr22_sgpr23
+; GFX11-NEXT: ; freeze: $sgpr20_sgpr21
+; GFX11-NEXT: ; freeze: $sgpr18_sgpr19
+; GFX11-NEXT: ; freeze: $sgpr16_sgpr17
+; GFX11-NEXT: ; freeze: $sgpr14_sgpr15
+; GFX11-NEXT: ; freeze: $sgpr12_sgpr13
+; GFX11-NEXT: ; freeze: $sgpr10_sgpr11
+; GFX11-NEXT: ; freeze: $sgpr8_sgpr9
+; GFX11-NEXT: ; freeze: $sgpr6_sgpr7
+; GFX11-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX11-NEXT: ; freeze: $sgpr2_sgpr3
+; GFX11-NEXT: ; freeze: $sgpr0_sgpr1
+; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX11-NEXT: v_dual_mov_b32 v30, s30 :: v_dual_mov_b32 v31, s31
+; GFX11-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
+; GFX11-NEXT: v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s7
+; GFX11-NEXT: v_dual_mov_b32 v8, s8 :: v_dual_mov_b32 v9, s9
+; GFX11-NEXT: v_dual_mov_b32 v10, s10 :: v_dual_mov_b32 v11, s11
+; GFX11-NEXT: v_dual_mov_b32 v12, s12 :: v_dual_mov_b32 v13, s13
+; GFX11-NEXT: v_dual_mov_b32 v14, s14 :: v_dual_mov_b32 v15, s15
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b32 m0, s2, 2
-; GFX11-NEXT: v_movrels_b32_e32 v3, v3
-; GFX11-NEXT: v_movrels_b32_e32 v2, v2
-; GFX11-NEXT: v_movrels_b32_e32 v1, v1
-; GFX11-NEXT: v_movrels_b32_e32 v0, v0
-; GFX11-NEXT: global_store_b128 v4, v[0:3], s[0:1]
+; GFX11-NEXT: s_lshl_b32 m0, s33, 2
+; GFX11-NEXT: v_dual_mov_b32 v16, s16 :: v_dual_mov_b32 v17, s17
+; GFX11-NEXT: v_dual_mov_b32 v18, s18 :: v_dual_mov_b32 v19, s19
+; GFX11-NEXT: v_dual_mov_b32 v20, s20 :: v_dual_mov_b32 v21, s21
+; GFX11-NEXT: v_dual_mov_b32 v22, s22 :: v_dual_mov_b32 v23, s23
+; GFX11-NEXT: v_dual_mov_b32 v24, s24 :: v_dual_mov_b32 v25, s25
+; GFX11-NEXT: v_dual_mov_b32 v26, s26 :: v_dual_mov_b32 v27, s27
+; GFX11-NEXT: v_dual_mov_b32 v28, s28 :: v_dual_mov_b32 v29, s29
+; GFX11-NEXT: v_mov_b32_e32 v35, 0
+; GFX11-NEXT: v_movrels_b32_e32 v34, v3
+; GFX11-NEXT: v_movrels_b32_e32 v33, v2
+; GFX11-NEXT: v_movrels_b32_e32 v32, v1
+; GFX11-NEXT: v_movrels_b32_e32 v31, v0
+; GFX11-NEXT: global_store_b128 v35, v[31:34], s[34:35]
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: test_bitcast_llc_v64i16_v8i16:
; GFX12: ; %bb.0: ; %entry
-; GFX12-NEXT: s_load_b96 s[0:2], s[4:5], 0x0
-; GFX12-NEXT: v_mov_b32_e32 v4, 0
+; GFX12-NEXT: s_load_b96 s[36:38], s[4:5], 0x0
+; GFX12-NEXT: ; freeze: $sgpr30_sgpr31
+; GFX12-NEXT: ; freeze: $sgpr28_sgpr29
+; GFX12-NEXT: ; freeze: $sgpr26_sgpr27
+; GFX12-NEXT: ; freeze: $sgpr24_sgpr25
+; GFX12-NEXT: ; freeze: $sgpr22_sgpr23
+; GFX12-NEXT: ; freeze: $sgpr20_sgpr21
+; GFX12-NEXT: ; freeze: $sgpr18_sgpr19
+; GFX12-NEXT: ; freeze: $sgpr16_sgpr17
+; GFX12-NEXT: ; freeze: $sgpr14_sgpr15
+; GFX12-NEXT: ; freeze: $sgpr12_sgpr13
+; GFX12-NEXT: ; freeze: $sgpr10_sgpr11
+; GFX12-NEXT: ; freeze: $sgpr8_sgpr9
+; GFX12-NEXT: ; freeze: $sgpr6_sgpr7
+; GFX12-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX12-NEXT: ; freeze: $sgpr2_sgpr3
+; GFX12-NEXT: ; freeze: $sgpr0_sgpr1
+; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-NEXT: v_dual_mov_b32 v30, s30 :: v_dual_mov_b32 v31, s31
+; GFX12-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
+; GFX12-NEXT: v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s7
+; GFX12-NEXT: v_dual_mov_b32 v8, s8 :: v_dual_mov_b32 v9, s9
+; GFX12-NEXT: v_dual_mov_b32 v10, s10 :: v_dual_mov_b32 v11, s11
+; GFX12-NEXT: v_dual_mov_b32 v12, s12 :: v_dual_mov_b32 v13, s13
+; GFX12-NEXT: v_dual_mov_b32 v14, s14 :: v_dual_mov_b32 v15, s15
+; GFX12-NEXT: v_dual_mov_b32 v16, s16 :: v_dual_mov_b32 v17, s17
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_lshl_b32 m0, s2, 2
-; GFX12-NEXT: v_movrels_b32_e32 v3, v3
-; GFX12-NEXT: v_movrels_b32_e32 v2, v2
-; GFX12-NEXT: v_movrels_b32_e32 v1, v1
-; GFX12-NEXT: v_movrels_b32_e32 v0, v0
-; GFX12-NEXT: global_store_b128 v4, v[0:3], s[0:1]
+; GFX12-NEXT: s_lshl_b32 m0, s38, 2
+; GFX12-NEXT: v_dual_mov_b32 v18, s18 :: v_dual_mov_b32 v19, s19
+; GFX12-NEXT: v_dual_mov_b32 v20, s20 :: v_dual_mov_b32 v21, s21
+; GFX12-NEXT: v_dual_mov_b32 v22, s22 :: v_dual_mov_b32 v23, s23
+; GFX12-NEXT: v_dual_mov_b32 v24, s24 :: v_dual_mov_b32 v25, s25
+; GFX12-NEXT: v_dual_mov_b32 v26, s26 :: v_dual_mov_b32 v27, s27
+; GFX12-NEXT: v_dual_mov_b32 v28, s28 :: v_dual_mov_b32 v29, s29
+; GFX12-NEXT: v_mov_b32_e32 v35, 0
+; GFX12-NEXT: v_movrels_b32_e32 v34, v3
+; GFX12-NEXT: v_movrels_b32_e32 v33, v2
+; GFX12-NEXT: v_movrels_b32_e32 v32, v1
+; GFX12-NEXT: v_movrels_b32_e32 v31, v0
+; GFX12-NEXT: global_store_b128 v35, v[31:34], s[36:37]
; GFX12-NEXT: s_endpgm
entry:
%alloca = freeze <64 x i16> poison
@@ -112,46 +308,144 @@ entry:
define amdgpu_kernel void @test_bitcast_llc_v32i32_v4i32(ptr addrspace(1) %out, i32 %idx) {
; GFX9-LABEL: test_bitcast_llc_v32i32_v4i32:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: s_load_dword s33, s[4:5], 0x8
+; GFX9-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x0
+; GFX9-NEXT: ; freeze: $sgpr30_sgpr31
+; GFX9-NEXT: ; freeze: $sgpr28_sgpr29
+; GFX9-NEXT: ; freeze: $sgpr26_sgpr27
+; GFX9-NEXT: ; freeze: $sgpr24_sgpr25
+; GFX9-NEXT: ; freeze: $sgpr22_sgpr23
+; GFX9-NEXT: ; freeze: $sgpr20_sgpr21
+; GFX9-NEXT: ; freeze: $sgpr18_sgpr19
+; GFX9-NEXT: ; freeze: $sgpr16_sgpr17
+; GFX9-NEXT: ; freeze: $sgpr14_sgpr15
+; GFX9-NEXT: ; freeze: $sgpr12_sgpr13
+; GFX9-NEXT: ; freeze: $sgpr10_sgpr11
+; GFX9-NEXT: ; freeze: $sgpr8_sgpr9
+; GFX9-NEXT: ; freeze: $sgpr6_sgpr7
+; GFX9-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX9-NEXT: ; freeze: $sgpr2_sgpr3
+; GFX9-NEXT: ; freeze: $sgpr0_sgpr1
+; GFX9-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX9-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX9-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshl_b32 s2, s2, 2
-; GFX9-NEXT: s_set_gpr_idx_on s2, gpr_idx(SRC0)
-; GFX9-NEXT: v_mov_b32_e32 v3, v3
-; GFX9-NEXT: v_mov_b32_e32 v2, v2
-; GFX9-NEXT: v_mov_b32_e32 v1, v1
-; GFX9-NEXT: v_mov_b32_e32 v0, v0
+; GFX9-NEXT: s_lshl_b32 s33, s33, 2
+; GFX9-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX9-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX9-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX9-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX9-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX9-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
+; GFX9-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; GFX9-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
+; GFX9-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
+; GFX9-NEXT: v_mov_b64_e32 v[24:25], s[24:25]
+; GFX9-NEXT: v_mov_b64_e32 v[26:27], s[26:27]
+; GFX9-NEXT: v_mov_b64_e32 v[28:29], s[28:29]
+; GFX9-NEXT: v_mov_b64_e32 v[30:31], s[30:31]
+; GFX9-NEXT: s_set_gpr_idx_on s33, gpr_idx(SRC0)
+; GFX9-NEXT: v_mov_b32_e32 v35, v3
+; GFX9-NEXT: v_mov_b32_e32 v34, v2
; GFX9-NEXT: s_set_gpr_idx_off
-; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX9-NEXT: v_mov_b32_e32 v36, 0
+; GFX9-NEXT: s_set_gpr_idx_on s33, gpr_idx(SRC0)
+; GFX9-NEXT: v_mov_b32_e32 v33, v1
+; GFX9-NEXT: v_mov_b32_e32 v32, v0
+; GFX9-NEXT: s_set_gpr_idx_off
+; GFX9-NEXT: global_store_dwordx4 v36, v[32:35], s[34:35]
; GFX9-NEXT: s_endpgm
;
; GFX11-LABEL: test_bitcast_llc_v32i32_v4i32:
; GFX11: ; %bb.0: ; %entry
; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX11-NEXT: v_mov_b32_e32 v4, 0
+; GFX11-NEXT: s_load_b32 s33, s[4:5], 0x8
+; GFX11-NEXT: s_load_b64 s[34:35], s[4:5], 0x0
+; GFX11-NEXT: ; freeze: $sgpr30_sgpr31
+; GFX11-NEXT: ; freeze: $sgpr28_sgpr29
+; GFX11-NEXT: ; freeze: $sgpr26_sgpr27
+; GFX11-NEXT: ; freeze: $sgpr24_sgpr25
+; GFX11-NEXT: ; freeze: $sgpr22_sgpr23
+; GFX11-NEXT: ; freeze: $sgpr20_sgpr21
+; GFX11-NEXT: ; freeze: $sgpr18_sgpr19
+; GFX11-NEXT: ; freeze: $sgpr16_sgpr17
+; GFX11-NEXT: ; freeze: $sgpr14_sgpr15
+; GFX11-NEXT: ; freeze: $sgpr12_sgpr13
+; GFX11-NEXT: ; freeze: $sgpr10_sgpr11
+; GFX11-NEXT: ; freeze: $sgpr8_sgpr9
+; GFX11-NEXT: ; freeze: $sgpr6_sgpr7
+; GFX11-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX11-NEXT: ; freeze: $sgpr2_sgpr3
+; GFX11-NEXT: ; freeze: $sgpr0_sgpr1
+; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX11-NEXT: v_dual_mov_b32 v30, s30 :: v_dual_mov_b32 v31, s31
+; GFX11-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
+; GFX11-NEXT: v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s7
+; GFX11-NEXT: v_dual_mov_b32 v8, s8 :: v_dual_mov_b32 v9, s9
+; GFX11-NEXT: v_dual_mov_b32 v10, s10 :: v_dual_mov_b32 v11, s11
+; GFX11-NEXT: v_dual_mov_b32 v12, s12 :: v_dual_mov_b32 v13, s13
+; GFX11-NEXT: v_dual_mov_b32 v14, s14 :: v_dual_mov_b32 v15, s15
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b32 m0, s2, 2
-; GFX11-NEXT: v_movrels_b32_e32 v3, v3
-; GFX11-NEXT: v_movrels_b32_e32 v2, v2
-; GFX11-NEXT: v_movrels_b32_e32 v1, v1
-; GFX11-NEXT: v_movrels_b32_e32 v0, v0
-; GFX11-NEXT: global_store_b128 v4, v[0:3], s[0:1]
+; GFX11-NEXT: s_lshl_b32 m0, s33, 2
+; GFX11-NEXT: v_dual_mov_b32 v16, s16 :: v_dual_mov_b32 v17, s17
+; GFX11-NEXT: v_dual_mov_b32 v18, s18 :: v_dual_mov_b32 v19, s19
+; GFX11-NEXT: v_dual_mov_b32 v20, s20 :: v_dual_mov_b32 v21, s21
+; GFX11-NEXT: v_dual_mov_b32 v22, s22 :: v_dual_mov_b32 v23, s23
+; GFX11-NEXT: v_dual_mov_b32 v24, s24 :: v_dual_mov_b32 v25, s25
+; GFX11-NEXT: v_dual_mov_b32 v26, s26 :: v_dual_mov_b32 v27, s27
+; GFX11-NEXT: v_dual_mov_b32 v28, s28 :: v_dual_mov_b32 v29, s29
+; GFX11-NEXT: v_mov_b32_e32 v35, 0
+; GFX11-NEXT: v_movrels_b32_e32 v34, v3
+; GFX11-NEXT: v_movrels_b32_e32 v33, v2
+; GFX11-NEXT: v_movrels_b32_e32 v32, v1
+; GFX11-NEXT: v_movrels_b32_e32 v31, v0
+; GFX11-NEXT: global_store_b128 v35, v[31:34], s[34:35]
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: test_bitcast_llc_v32i32_v4i32:
; GFX12: ; %bb.0: ; %entry
-; GFX12-NEXT: s_load_b96 s[0:2], s[4:5], 0x0
-; GFX12-NEXT: v_mov_b32_e32 v4, 0
+; GFX12-NEXT: s_load_b96 s[36:38], s[4:5], 0x0
+; GFX12-NEXT: ; freeze: $sgpr30_sgpr31
+; GFX12-NEXT: ; freeze: $sgpr28_sgpr29
+; GFX12-NEXT: ; freeze: $sgpr26_sgpr27
+; GFX12-NEXT: ; freeze: $sgpr24_sgpr25
+; GFX12-NEXT: ; freeze: $sgpr22_sgpr23
+; GFX12-NEXT: ; freeze: $sgpr20_sgpr21
+; GFX12-NEXT: ; freeze: $sgpr18_sgpr19
+; GFX12-NEXT: ; freeze: $sgpr16_sgpr17
+; GFX12-NEXT: ; freeze: $sgpr14_sgpr15
+; GFX12-NEXT: ; freeze: $sgpr12_sgpr13
+; GFX12-NEXT: ; freeze: $sgpr10_sgpr11
+; GFX12-NEXT: ; freeze: $sgpr8_sgpr9
+; GFX12-NEXT: ; freeze: $sgpr6_sgpr7
+; GFX12-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX12-NEXT: ; freeze: $sgpr2_sgpr3
+; GFX12-NEXT: ; freeze: $sgpr0_sgpr1
+; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-NEXT: v_dual_mov_b32 v30, s30 :: v_dual_mov_b32 v31, s31
+; GFX12-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
+; GFX12-NEXT: v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s7
+; GFX12-NEXT: v_dual_mov_b32 v8, s8 :: v_dual_mov_b32 v9, s9
+; GFX12-NEXT: v_dual_mov_b32 v10, s10 :: v_dual_mov_b32 v11, s11
+; GFX12-NEXT: v_dual_mov_b32 v12, s12 :: v_dual_mov_b32 v13, s13
+; GFX12-NEXT: v_dual_mov_b32 v14, s14 :: v_dual_mov_b32 v15, s15
+; GFX12-NEXT: v_dual_mov_b32 v16, s16 :: v_dual_mov_b32 v17, s17
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_lshl_b32 m0, s2, 2
-; GFX12-NEXT: v_movrels_b32_e32 v3, v3
-; GFX12-NEXT: v_movrels_b32_e32 v2, v2
-; GFX12-NEXT: v_movrels_b32_e32 v1, v1
-; GFX12-NEXT: v_movrels_b32_e32 v0, v0
-; GFX12-NEXT: global_store_b128 v4, v[0:3], s[0:1]
+; GFX12-NEXT: s_lshl_b32 m0, s38, 2
+; GFX12-NEXT: v_dual_mov_b32 v18, s18 :: v_dual_mov_b32 v19, s19
+; GFX12-NEXT: v_dual_mov_b32 v20, s20 :: v_dual_mov_b32 v21, s21
+; GFX12-NEXT: v_dual_mov_b32 v22, s22 :: v_dual_mov_b32 v23, s23
+; GFX12-NEXT: v_dual_mov_b32 v24, s24 :: v_dual_mov_b32 v25, s25
+; GFX12-NEXT: v_dual_mov_b32 v26, s26 :: v_dual_mov_b32 v27, s27
+; GFX12-NEXT: v_dual_mov_b32 v28, s28 :: v_dual_mov_b32 v29, s29
+; GFX12-NEXT: v_mov_b32_e32 v35, 0
+; GFX12-NEXT: v_movrels_b32_e32 v34, v3
+; GFX12-NEXT: v_movrels_b32_e32 v33, v2
+; GFX12-NEXT: v_movrels_b32_e32 v32, v1
+; GFX12-NEXT: v_movrels_b32_e32 v31, v0
+; GFX12-NEXT: global_store_b128 v35, v[31:34], s[36:37]
; GFX12-NEXT: s_endpgm
entry:
%alloca = freeze <32 x i32> poison
@@ -165,63 +459,161 @@ entry:
define amdgpu_kernel void @test_bitcast_llc_v16i64_v4i256(ptr addrspace(1) %out, i32 %idx) {
; GFX9-LABEL: test_bitcast_llc_v16i64_v4i256:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX9-NEXT: v_mov_b32_e32 v8, 0
+; GFX9-NEXT: s_load_dword s33, s[4:5], 0x8
+; GFX9-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x0
+; GFX9-NEXT: ; freeze: $sgpr30_sgpr31
+; GFX9-NEXT: ; freeze: $sgpr28_sgpr29
+; GFX9-NEXT: ; freeze: $sgpr26_sgpr27
+; GFX9-NEXT: ; freeze: $sgpr24_sgpr25
+; GFX9-NEXT: ; freeze: $sgpr22_sgpr23
+; GFX9-NEXT: ; freeze: $sgpr20_sgpr21
+; GFX9-NEXT: ; freeze: $sgpr18_sgpr19
+; GFX9-NEXT: ; freeze: $sgpr16_sgpr17
+; GFX9-NEXT: ; freeze: $sgpr14_sgpr15
+; GFX9-NEXT: ; freeze: $sgpr12_sgpr13
+; GFX9-NEXT: ; freeze: $sgpr10_sgpr11
+; GFX9-NEXT: ; freeze: $sgpr8_sgpr9
+; GFX9-NEXT: ; freeze: $sgpr6_sgpr7
+; GFX9-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX9-NEXT: ; freeze: $sgpr2_sgpr3
+; GFX9-NEXT: ; freeze: $sgpr0_sgpr1
+; GFX9-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX9-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX9-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX9-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX9-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
+; GFX9-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX9-NEXT: v_mov_b64_e32 v[12:13], s[12:13]
+; GFX9-NEXT: v_mov_b64_e32 v[14:15], s[14:15]
+; GFX9-NEXT: v_mov_b64_e32 v[16:17], s[16:17]
+; GFX9-NEXT: v_mov_b64_e32 v[18:19], s[18:19]
+; GFX9-NEXT: v_mov_b64_e32 v[20:21], s[20:21]
+; GFX9-NEXT: v_mov_b64_e32 v[22:23], s[22:23]
+; GFX9-NEXT: v_mov_b64_e32 v[24:25], s[24:25]
+; GFX9-NEXT: v_mov_b64_e32 v[26:27], s[26:27]
+; GFX9-NEXT: v_mov_b64_e32 v[28:29], s[28:29]
+; GFX9-NEXT: v_mov_b64_e32 v[30:31], s[30:31]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshl_b32 s2, s2, 3
-; GFX9-NEXT: s_set_gpr_idx_on s2, gpr_idx(SRC0)
-; GFX9-NEXT: v_mov_b32_e32 v3, v3
-; GFX9-NEXT: v_mov_b32_e32 v2, v2
-; GFX9-NEXT: v_mov_b32_e32 v1, v1
-; GFX9-NEXT: v_mov_b32_e32 v0, v0
-; GFX9-NEXT: v_mov_b32_e32 v7, v7
-; GFX9-NEXT: v_mov_b32_e32 v6, v6
-; GFX9-NEXT: v_mov_b32_e32 v5, v5
-; GFX9-NEXT: v_mov_b32_e32 v4, v4
+; GFX9-NEXT: s_lshl_b32 s0, s33, 3
+; GFX9-NEXT: s_set_gpr_idx_on s0, gpr_idx(SRC0)
+; GFX9-NEXT: v_mov_b32_e32 v35, v3
+; GFX9-NEXT: v_mov_b32_e32 v34, v2
+; GFX9-NEXT: v_mov_b32_e32 v33, v1
+; GFX9-NEXT: v_mov_b32_e32 v32, v0
+; GFX9-NEXT: v_mov_b32_e32 v39, v7
+; GFX9-NEXT: v_mov_b32_e32 v38, v6
+; GFX9-NEXT: s_set_gpr_idx_off
+; GFX9-NEXT: v_mov_b32_e32 v40, 0
+; GFX9-NEXT: s_set_gpr_idx_on s0, gpr_idx(SRC0)
+; GFX9-NEXT: v_mov_b32_e32 v37, v5
+; GFX9-NEXT: v_mov_b32_e32 v36, v4
; GFX9-NEXT: s_set_gpr_idx_off
-; GFX9-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
-; GFX9-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
+; GFX9-NEXT: global_store_dwordx4 v40, v[36:39], s[34:35] offset:16
+; GFX9-NEXT: global_store_dwordx4 v40, v[32:35], s[34:35]
; GFX9-NEXT: s_endpgm
;
; GFX11-LABEL: test_bitcast_llc_v16i64_v4i256:
; GFX11: ; %bb.0: ; %entry
; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX11-NEXT: v_mov_b32_e32 v8, 0
+; GFX11-NEXT: s_load_b32 s33, s[4:5], 0x8
+; GFX11-NEXT: s_load_b64 s[34:35], s[4:5], 0x0
+; GFX11-NEXT: ; freeze: $sgpr30_sgpr31
+; GFX11-NEXT: ; freeze: $sgpr28_sgpr29
+; GFX11-NEXT: ; freeze: $sgpr26_sgpr27
+; GFX11-NEXT: ; freeze: $sgpr24_sgpr25
+; GFX11-NEXT: ; freeze: $sgpr22_sgpr23
+; GFX11-NEXT: ; freeze: $sgpr20_sgpr21
+; GFX11-NEXT: ; freeze: $sgpr18_sgpr19
+; GFX11-NEXT: ; freeze: $sgpr16_sgpr17
+; GFX11-NEXT: ; freeze: $sgpr14_sgpr15
+; GFX11-NEXT: ; freeze: $sgpr12_sgpr13
+; GFX11-NEXT: ; freeze: $sgpr10_sgpr11
+; GFX11-NEXT: ; freeze: $sgpr8_sgpr9
+; GFX11-NEXT: ; freeze: $sgpr6_sgpr7
+; GFX11-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX11-NEXT: ; freeze: $sgpr2_sgpr3
+; GFX11-NEXT: ; freeze: $sgpr0_sgpr1
+; GFX11-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
+; GFX11-NEXT: v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s7
+; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX11-NEXT: v_dual_mov_b32 v8, s8 :: v_dual_mov_b32 v9, s9
+; GFX11-NEXT: v_dual_mov_b32 v10, s10 :: v_dual_mov_b32 v11, s11
+; GFX11-NEXT: v_dual_mov_b32 v12, s12 :: v_dual_mov_b32 v13, s13
+; GFX11-NEXT: v_dual_mov_b32 v14, s14 :: v_dual_mov_b32 v15, s15
+; GFX11-NEXT: v_dual_mov_b32 v16, s16 :: v_dual_mov_b32 v17, s17
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b32 m0, s2, 3
-; GFX11-NEXT: v_movrels_b32_e32 v3, v3
-; GFX11-NEXT: v_movrels_b32_e32 v2, v2
-; GFX11-NEXT: v_movrels_b32_e32 v1, v1
-; GFX11-NEXT: v_movrels_b32_e32 v7, v7
-; GFX11-NEXT: v_movrels_b32_e32 v6, v6
-; GFX11-NEXT: v_movrels_b32_e32 v5, v5
-; GFX11-NEXT: v_movrels_b32_e32 v4, v4
-; GFX11-NEXT: v_movrels_b32_e32 v0, v0
+; GFX11-NEXT: s_lshl_b32 m0, s33, 3
+; GFX11-NEXT: v_dual_mov_b32 v18, s18 :: v_dual_mov_b32 v19, s19
+; GFX11-NEXT: v_dual_mov_b32 v20, s20 :: v_dual_mov_b32 v21, s21
+; GFX11-NEXT: v_dual_mov_b32 v22, s22 :: v_dual_mov_b32 v23, s23
+; GFX11-NEXT: v_dual_mov_b32 v24, s24 :: v_dual_mov_b32 v25, s25
+; GFX11-NEXT: v_dual_mov_b32 v26, s26 :: v_dual_mov_b32 v27, s27
+; GFX11-NEXT: v_dual_mov_b32 v28, s28 :: v_dual_mov_b32 v29, s29
+; GFX11-NEXT: v_dual_mov_b32 v30, s30 :: v_dual_mov_b32 v31, s31
+; GFX11-NEXT: v_mov_b32_e32 v39, 0
+; GFX11-NEXT: v_movrels_b32_e32 v38, v7
+; GFX11-NEXT: v_movrels_b32_e32 v37, v6
+; GFX11-NEXT: v_movrels_b32_e32 v36, v5
+; GFX11-NEXT: v_movrels_b32_e32 v35, v4
+; GFX11-NEXT: v_movrels_b32_e32 v34, v3
+; GFX11-NEXT: v_movrels_b32_e32 v33, v2
+; GFX11-NEXT: v_movrels_b32_e32 v32, v1
+; GFX11-NEXT: v_movrels_b32_e32 v31, v0
; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: global_store_b128 v8, v[4:7], s[0:1] offset:16
-; GFX11-NEXT: global_store_b128 v8, v[0:3], s[0:1]
+; GFX11-NEXT: global_store_b128 v39, v[35:38], s[34:35] offset:16
+; GFX11-NEXT: global_store_b128 v39, v[31:34], s[34:35]
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: test_bitcast_llc_v16i64_v4i256:
; GFX12: ; %bb.0: ; %entry
-; GFX12-NEXT: s_load_b96 s[0:2], s[4:5], 0x0
-; GFX12-NEXT: v_mov_b32_e32 v8, 0
+; GFX12-NEXT: s_load_b96 s[36:38], s[4:5], 0x0
+; GFX12-NEXT: ; freeze: $sgpr30_sgpr31
+; GFX12-NEXT: ; freeze: $sgpr28_sgpr29
+; GFX12-NEXT: ; freeze: $sgpr26_sgpr27
+; GFX12-NEXT: ; freeze: $sgpr24_sgpr25
+; GFX12-NEXT: ; freeze: $sgpr22_sgpr23
+; GFX12-NEXT: ; freeze: $sgpr20_sgpr21
+; GFX12-NEXT: ; freeze: $sgpr18_sgpr19
+; GFX12-NEXT: ; freeze: $sgpr16_sgpr17
+; GFX12-NEXT: ; freeze: $sgpr14_sgpr15
+; GFX12-NEXT: ; freeze: $sgpr12_sgpr13
+; GFX12-NEXT: ; freeze: $sgpr10_sgpr11
+; GFX12-NEXT: ; freeze: $sgpr8_sgpr9
+; GFX12-NEXT: ; freeze: $sgpr6_sgpr7
+; GFX12-NEXT: ; freeze: $sgpr4_sgpr5
+; GFX12-NEXT: ; freeze: $sgpr2_sgpr3
+; GFX12-NEXT: ; freeze: $sgpr0_sgpr1
+; GFX12-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
+; GFX12-NEXT: v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s7
+; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-NEXT: v_dual_mov_b32 v8, s8 :: v_dual_mov_b32 v9, s9
+; GFX12-NEXT: v_dual_mov_b32 v10, s10 :: v_dual_mov_b32 v11, s11
+; GFX12-NEXT: v_dual_mov_b32 v12, s12 :: v_dual_mov_b32 v13, s13
+; GFX12-NEXT: v_dual_mov_b32 v14, s14 :: v_dual_mov_b32 v15, s15
+; GFX12-NEXT: v_dual_mov_b32 v16, s16 :: v_dual_mov_b32 v17, s17
+; GFX12-NEXT: v_dual_mov_b32 v18, s18 :: v_dual_mov_b32 v19, s19
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_lshl_b32 m0, s2, 3
-; GFX12-NEXT: v_movrels_b32_e32 v3, v3
-; GFX12-NEXT: v_movrels_b32_e32 v2, v2
-; GFX12-NEXT: v_movrels_b32_e32 v1, v1
-; GFX12-NEXT: v_movrels_b32_e32 v7, v7
-; GFX12-NEXT: v_movrels_b32_e32 v6, v6
-; GFX12-NEXT: v_movrels_b32_e32 v5, v5
-; GFX12-NEXT: v_movrels_b32_e32 v4, v4
-; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: s_lshl_b32 m0, s38, 3
+; GFX12-NEXT: v_dual_mov_b32 v20, s20 :: v_dual_mov_b32 v21, s21
+; GFX12-NEXT: v_dual_mov_b32 v22, s22 :: v_dual_mov_b32 v23, s23
+; GFX12-NEXT: v_dual_mov_b32 v24, s24 :: v_dual_mov_b32 v25, s25
+; GFX12-NEXT: v_dual_mov_b32 v26, s26 :: v_dual_mov_b32 v27, s27
+; GFX12-NEXT: v_dual_mov_b32 v28, s28 :: v_dual_mov_b32 v29, s29
+; GFX12-NEXT: v_dual_mov_b32 v30, s30 :: v_dual_mov_b32 v31, s31
+; GFX12-NEXT: v_mov_b32_e32 v39, 0
+; GFX12-NEXT: v_movrels_b32_e32 v38, v7
+; GFX12-NEXT: v_movrels_b32_e32 v37, v6
+; GFX12-NEXT: v_movrels_b32_e32 v36, v5
+; GFX12-NEXT: v_movrels_b32_e32 v35, v4
+; GFX12-NEXT: v_movrels_b32_e32 v34, v3
+; GFX12-NEXT: v_movrels_b32_e32 v33, v2
+; GFX12-NEXT: v_movrels_b32_e32 v32, v1
+; GFX12-NEXT: v_movrels_b32_e32 v31, v0
; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: global_store_b128 v8, v[4:7], s[0:1] offset:16
-; GFX12-NEXT: global_store_b128 v8, v[0:3], s[0:1]
+; GFX12-NEXT: global_store_b128 v39, v[35:38], s[36:37] offset:16
+; GFX12-NEXT: global_store_b128 v39, v[31:34], s[36:37]
; GFX12-NEXT: s_endpgm
entry:
%alloca = freeze <16 x i64> poison
diff --git a/llvm/test/CodeGen/AMDGPU/promote-alloca-vector-to-vector.ll b/llvm/test/CodeGen/AMDGPU/promote-alloca-vector-to-vector.ll
index cb44d2ba4bde65..aa5af3bee60159 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-alloca-vector-to-vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-alloca-vector-to-vector.ll
@@ -173,7 +173,10 @@ entry:
; OPT-LABEL: define i64 @ptr_alloca_bitcast
; GCN-NOT: buffer_
-; GCN: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN: ; freeze: $sgpr4_sgpr5
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mov_b32_e32 v0, s4
+; GCN-NEXT: v_mov_b32_e32 v1, s5
; GCN-NEXT: s_setpc_b64
diff --git a/llvm/test/CodeGen/AMDGPU/sad.ll b/llvm/test/CodeGen/AMDGPU/sad.ll
index 5e31f9e463355f..5c84ae49446eb1 100644
--- a/llvm/test/CodeGen/AMDGPU/sad.ll
+++ b/llvm/test/CodeGen/AMDGPU/sad.ll
@@ -1293,13 +1293,14 @@ define amdgpu_kernel void @v_sad_u32_i16_pat2(ptr addrspace(1) %out) {
; GFX12-5-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX12-5-SDAG-NEXT: global_load_u16 v3, v[0:1], off scope:SCOPE_SYS
; GFX12-5-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX12-5-SDAG-NEXT: v_and_b32_e32 v4, 0xffff, v0
-; GFX12-5-SDAG-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-5-SDAG-NEXT: v_mov_b16_e32 v4.l, v0.l
; GFX12-5-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-5-SDAG-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX12-5-SDAG-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX12-5-SDAG-NEXT: v_min_u32_e32 v5, v4, v2
; GFX12-5-SDAG-NEXT: v_max_u32_e32 v2, v4, v2
+; GFX12-5-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-5-SDAG-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_sub_nc_u32 v2, v2, v5
-; GFX12-5-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-5-SDAG-NEXT: v_add_nc_u32_e32 v2, v2, v3
; GFX12-5-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX12-5-SDAG-NEXT: global_store_b16 v4, v2, s[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll b/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll
index c212792a25f9d3..08e552ed291ce1 100644
--- a/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll
+++ b/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll
@@ -1675,11 +1675,12 @@ bb11: ; preds = %bb10, %bb2
define void @crash_lshlrevb16_not_reg_op() {
; NOSDWA-LABEL: crash_lshlrevb16_not_reg_op:
; NOSDWA: ; %bb.0: ; %bb0
+; NOSDWA-NEXT: ; freeze: $sgpr4
; NOSDWA-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; NOSDWA-NEXT: s_and_b64 vcc, exec, -1
; NOSDWA-NEXT: s_bitset1_b32 s4, 8
; NOSDWA-NEXT: s_and_b32 s6, s4, 0x1ff
; NOSDWA-NEXT: s_mov_b64 s[4:5], 0
-; NOSDWA-NEXT: s_and_b64 vcc, exec, -1
; NOSDWA-NEXT: .LBB22_1: ; %bb1
; NOSDWA-NEXT: ; =>This Inner Loop Header: Depth=1
; NOSDWA-NEXT: s_lshl_b32 s7, s4, 3
@@ -1697,11 +1698,12 @@ define void @crash_lshlrevb16_not_reg_op() {
;
; GFX89-LABEL: crash_lshlrevb16_not_reg_op:
; GFX89: ; %bb.0: ; %bb0
+; GFX89-NEXT: ; freeze: $sgpr4
; GFX89-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX89-NEXT: s_and_b64 vcc, exec, -1
; GFX89-NEXT: s_bitset1_b32 s4, 8
; GFX89-NEXT: s_and_b32 s6, s4, 0x1ff
; GFX89-NEXT: s_mov_b64 s[4:5], 0
-; GFX89-NEXT: s_and_b64 vcc, exec, -1
; GFX89-NEXT: .LBB22_1: ; %bb1
; GFX89-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX89-NEXT: s_lshl_b32 s7, s4, 3
@@ -1719,11 +1721,12 @@ define void @crash_lshlrevb16_not_reg_op() {
;
; GFX9-LABEL: crash_lshlrevb16_not_reg_op:
; GFX9: ; %bb.0: ; %bb0
+; GFX9-NEXT: ; freeze: $sgpr4
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_and_b64 vcc, exec, -1
; GFX9-NEXT: s_bitset1_b32 s4, 8
; GFX9-NEXT: s_and_b32 s6, s4, 0x1ff
; GFX9-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-NEXT: s_and_b64 vcc, exec, -1
; GFX9-NEXT: .LBB22_1: ; %bb1
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_lshl_b32 s7, s4, 3
@@ -1741,9 +1744,10 @@ define void @crash_lshlrevb16_not_reg_op() {
;
; GFX10-LABEL: crash_lshlrevb16_not_reg_op:
; GFX10: ; %bb.0: ; %bb0
+; GFX10-NEXT: ; freeze: $sgpr4
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_bitset1_b32 s4, 8
; GFX10-NEXT: s_mov_b32 vcc_lo, exec_lo
+; GFX10-NEXT: s_bitset1_b32 s4, 8
; GFX10-NEXT: s_and_b32 s6, s4, 0x1ff
; GFX10-NEXT: s_mov_b64 s[4:5], 0
; GFX10-NEXT: .LBB22_1: ; %bb1
diff --git a/llvm/test/CodeGen/AMDGPU/select-undef.ll b/llvm/test/CodeGen/AMDGPU/select-undef.ll
index cbb621088222ff..295a1836962c66 100644
--- a/llvm/test/CodeGen/AMDGPU/select-undef.ll
+++ b/llvm/test/CodeGen/AMDGPU/select-undef.ll
@@ -810,6 +810,7 @@ define i64 @poison_should_freeze(i1 %cond1, i32 %val, i16 %val2, i64 %a, i64 %b)
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_and_b32_e32 v0, 1, v0
; GCN-NEXT: v_mov_b32_e32 v7, 0x5040100
+; GCN-NEXT: ; freeze: $sgpr4
; GCN-NEXT: v_perm_b32 v2, v2, s4, v7
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
; GCN-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
@@ -831,6 +832,7 @@ define <8 x i16> @poison_should_freeze_v8i16(i1 %cond1, i32 %val, i16 %val2, <8
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_and_b32_e32 v0, 1, v0
; GCN-NEXT: v_mov_b32_e32 v11, 0x5040100
+; GCN-NEXT: ; freeze: $sgpr4
; GCN-NEXT: v_perm_b32 v2, v2, s4, v11
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
; GCN-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
diff --git a/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask.ll b/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask.ll
index d4a4f6ed9eefa0..66e38163f3bbb4 100644
--- a/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask.ll
+++ b/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask.ll
@@ -52,6 +52,7 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
; CHECK-NEXT: undef [[S_MOV_B32_:%[0-9]+]].sub0:sgpr_128 = S_MOV_B32 0
; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
; CHECK-NEXT: [[BUFFER_LOAD_DWORD_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET undef %117:sgpr_128, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+ ; CHECK-NEXT: [[FREEZE:%[0-9]+]]:sreg_32 = FREEZE undef %170:sreg_32
; CHECK-NEXT: [[S_MOV_B32_:%[0-9]+]].sub1:sgpr_128 = COPY [[S_MOV_B32_]].sub0
; CHECK-NEXT: [[S_MOV_B32_:%[0-9]+]].sub2:sgpr_128 = COPY [[S_MOV_B32_]].sub0
; CHECK-NEXT: [[S_MOV_B32_:%[0-9]+]].sub3:sgpr_128 = COPY [[S_MOV_B32_]].sub0
@@ -67,8 +68,8 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM1:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_1]], 64, 0 :: (invariant load (s128) from %ir.89, addrspace 4)
; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM2:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_2]], 64, 0 :: (invariant load (s128) from %ir.95, addrspace 4)
; CHECK-NEXT: [[S_ADD_U32_3:%[0-9]+]].sub1:sreg_64 = S_ADDC_U32 undef %54:sreg_32, [[S_ASHR_I32_2]], implicit-def dead $scc, implicit $scc
- ; CHECK-NEXT: [[S_ASHR_I32_3:%[0-9]+]]:sreg_32_xm0 = S_ASHR_I32 undef %170:sreg_32, 31, implicit-def dead $scc
- ; CHECK-NEXT: undef [[S_ADD_U32_4:%[0-9]+]].sub0:sreg_64 = S_ADD_U32 [[COPY6]], undef %170:sreg_32, implicit-def $scc
+ ; CHECK-NEXT: [[S_ASHR_I32_3:%[0-9]+]]:sreg_32_xm0 = S_ASHR_I32 [[FREEZE]], 31, implicit-def dead $scc
+ ; CHECK-NEXT: undef [[S_ADD_U32_4:%[0-9]+]].sub0:sreg_64 = S_ADD_U32 [[COPY6]], [[FREEZE]], implicit-def $scc
; CHECK-NEXT: [[S_ADD_U32_4:%[0-9]+]].sub1:sreg_64 = S_ADDC_U32 undef %54:sreg_32, [[S_ASHR_I32_3]], implicit-def dead $scc, implicit $scc
; CHECK-NEXT: undef [[S_ADD_U32_5:%[0-9]+]].sub0:sreg_64 = S_ADD_U32 [[COPY7]].sub0, [[S_LSHL_B32_]], implicit-def $scc
; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM3:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_3]], 64, 0 :: (invariant load (s128) from %ir.101, addrspace 4)
@@ -77,7 +78,7 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
; CHECK-NEXT: undef [[S_ADD_U32_6:%[0-9]+]].sub0:sreg_64 = S_ADD_U32 [[COPY7]].sub0, [[S_LSHL_B32_1]], implicit-def $scc
; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM5:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_5]], 0, 0 :: (invariant load (s128) from %ir.114, addrspace 4)
; CHECK-NEXT: [[S_ADD_U32_6:%[0-9]+]].sub1:sreg_64 = S_ADDC_U32 undef %51:sreg_32, [[S_ASHR_I32_1]], implicit-def dead $scc, implicit $scc
- ; CHECK-NEXT: undef [[S_ADD_U32_7:%[0-9]+]].sub0:sreg_64 = S_ADD_U32 [[COPY7]].sub0, undef %170:sreg_32, implicit-def $scc
+ ; CHECK-NEXT: undef [[S_ADD_U32_7:%[0-9]+]].sub0:sreg_64 = S_ADD_U32 [[COPY7]].sub0, [[FREEZE]], implicit-def $scc
; CHECK-NEXT: [[S_ADD_U32_7:%[0-9]+]].sub1:sreg_64 = S_ADDC_U32 undef %51:sreg_32, [[S_ASHR_I32_3]], implicit-def dead $scc, implicit $scc
; CHECK-NEXT: undef [[S_ADD_U32_8:%[0-9]+]].sub0:sreg_64 = S_ADD_U32 [[COPY7]].sub0, [[S_LSHL_B32_2]], implicit-def $scc
; CHECK-NEXT: [[S_ADD_U32_8:%[0-9]+]].sub1:sreg_64 = S_ADDC_U32 undef %51:sreg_32, [[S_ASHR_I32_2]], implicit-def dead $scc, implicit $scc
diff --git a/llvm/test/CodeGen/AMDGPU/udiv64.ll b/llvm/test/CodeGen/AMDGPU/udiv64.ll
index 4c9f2d08ce8f11..89b32b6ce56f2b 100644
--- a/llvm/test/CodeGen/AMDGPU/udiv64.ll
+++ b/llvm/test/CodeGen/AMDGPU/udiv64.ll
@@ -1801,7 +1801,8 @@ define i64 @v_test_udiv24_pow2_k_den_i64(i64 %x) {
; GCN-IR-LABEL: v_test_udiv24_pow2_k_den_i64:
; GCN-IR: ; %bb.0: ; %_udiv-special-cases
; GCN-IR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IR-NEXT: v_lshrrev_b32_e32 v4, 8, v1
+; GCN-IR-NEXT: v_lshrrev_b32_e32 v0, 8, v1
+; GCN-IR-NEXT: v_mov_b32_e32 v4, v0
; GCN-IR-NEXT: v_ffbh_u32_e32 v0, v4
; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, 32, v0
; GCN-IR-NEXT: s_flbit_i32_b32 s4, 0
diff --git a/llvm/test/CodeGen/Hexagon/pr196966.ll b/llvm/test/CodeGen/Hexagon/pr196966.ll
index 8612395feb8e40..ff473b2c224fe5 100644
--- a/llvm/test/CodeGen/Hexagon/pr196966.ll
+++ b/llvm/test/CodeGen/Hexagon/pr196966.ll
@@ -34,33 +34,36 @@ define void @pr196966(ptr %base) {
; CHECK-NEXT: r0 = add(r29,#0)
; CHECK-NEXT: }
; CHECK-NEXT: {
-; CHECK-NEXT: r6 = add(r16,#6)
+; CHECK-NEXT: r3 = add(r16,#20)
+; CHECK-NEXT: r8 = add(r16,#6)
+; CHECK-NEXT: r7:6 = memd(r29+#0)
; CHECK-NEXT: r1:0 = memd(r29+#24)
-; CHECK-NEXT: r5:4 = memd(r29+#0)
; CHECK-NEXT: }
; CHECK-NEXT: {
-; CHECK-NEXT: r5:4 = extractu(r5:4,#16,#32)
-; CHECK-NEXT: p0 = bitsclr(r0,#4)
-; CHECK-NEXT: r3:2 = memd(r29+#8)
+; CHECK-NEXT: r7:6 = extractu(r7:6,#16,#32)
+; CHECK-NEXT: r5:4 = memd(r29+#8)
+; CHECK-NEXT: r17:16 = memd(r30+#-8)
; CHECK-NEXT: }
+; CHECK-NEXT: // freeze: $r5
; CHECK-NEXT: {
-; CHECK-NEXT: p1 = bitsclr(r0,#32)
-; CHECK-NEXT: p2 = !tstbit(r0,#13)
-; CHECK-NEXT: r3 = add(r16,#20)
-; CHECK-NEXT: if (!p0) memh(r0+#0) = r4
+; CHECK-NEXT: p0 = bitsclr(r5,#4)
+; CHECK-NEXT: p1 = bitsclr(r5,#32)
; CHECK-NEXT: }
; CHECK-NEXT: {
-; CHECK-NEXT: p0 = !tstbit(r0,#15)
-; CHECK-NEXT: if (!p1) memh(r3+#0) = r2.h
-; CHECK-NEXT: if (!p2) memh(r6+#0) = r0.h
+; CHECK-NEXT: p2 = !tstbit(r5,#13)
+; CHECK-NEXT: p0 = !tstbit(r5,#15)
+; CHECK-NEXT: if (!p0) memh(r2+#0) = r6
+; CHECK-NEXT: if (!p1) memh(r3+#0) = r4.h
+; CHECK-NEXT: }
+; CHECK-NEXT: {
+; CHECK-NEXT: if (!p2) memh(r8+#0) = r0.h
; CHECK-NEXT: }
; CHECK-NEXT: {
-; CHECK-NEXT: r17:16 = memd(r30+#-8)
; CHECK-NEXT: r31:30 = deallocframe(r30):raw
-; CHECK-NEXT: } // 8-byte Folded Reload
+; CHECK-NEXT: if (!p0) memh(r2+#0) = r1.h
+; CHECK-NEXT: }
; CHECK-NEXT: {
; CHECK-NEXT: jumpr r31
-; CHECK-NEXT: if (!p0) memh(r0+#0) = r1.h
; CHECK-NEXT: }
entry:
%mb = icmp ne ptr %base, null
diff --git a/llvm/test/CodeGen/PowerPC/sms-grp-order.ll b/llvm/test/CodeGen/PowerPC/sms-grp-order.ll
index 9760e3d83d77e6..1533bf1d5ce8f0 100644
--- a/llvm/test/CodeGen/PowerPC/sms-grp-order.ll
+++ b/llvm/test/CodeGen/PowerPC/sms-grp-order.ll
@@ -5,19 +5,20 @@
define void @lame_encode_buffer_interleaved(ptr %arg0) local_unnamed_addr {
; CHECK-LABEL: lame_encode_buffer_interleaved:
; CHECK: # %bb.0:
-; CHECK-NEXT: li 4, 1
-; CHECK-NEXT: rldic 4, 4, 62, 1
-; CHECK-NEXT: mtctr 4
+; CHECK-NEXT: li 5, 1
+; CHECK-NEXT: # freeze: $x4
+; CHECK-NEXT: rldic 5, 5, 62, 1
+; CHECK-NEXT: mtctr 5
; CHECK-NEXT: .p2align 4
; CHECK-NEXT: .LBB0_1:
-; CHECK-NEXT: lha 4, 0(3)
; CHECK-NEXT: lha 5, 0(3)
-; CHECK-NEXT: srawi 4, 4, 1
-; CHECK-NEXT: addze 4, 4
+; CHECK-NEXT: lha 6, 0(4)
; CHECK-NEXT: srawi 5, 5, 1
; CHECK-NEXT: addze 5, 5
-; CHECK-NEXT: sth 4, 0(3)
+; CHECK-NEXT: srawi 6, 6, 1
+; CHECK-NEXT: addze 6, 6
; CHECK-NEXT: sth 5, 0(3)
+; CHECK-NEXT: sth 6, 0(4)
; CHECK-NEXT: bdnz .LBB0_1
; CHECK-NEXT: # %bb.2:
; CHECK-NEXT: blr
diff --git a/llvm/test/CodeGen/PowerPC/widen-vec-correctly-be.ll b/llvm/test/CodeGen/PowerPC/widen-vec-correctly-be.ll
index c2d62cfc23e964..95c121b8c23bec 100644
--- a/llvm/test/CodeGen/PowerPC/widen-vec-correctly-be.ll
+++ b/llvm/test/CodeGen/PowerPC/widen-vec-correctly-be.ll
@@ -11,15 +11,16 @@ define void @test() local_unnamed_addr #0 align 2 {
; CHECK-BE-NEXT: lhz r3, 0(r3)
; CHECK-BE-NEXT: vspltisw v2, -16
; CHECK-BE-NEXT: addi r3, r3, 1
+; CHECK-BE-NEXT: # freeze: $v4
; CHECK-BE-NEXT: xxlxor vs0, vs0, vs0
+; CHECK-BE-NEXT: vsrw v2, v2, v2
; CHECK-BE-NEXT: sldi r3, r3, 48
; CHECK-BE-NEXT: std r3, -32(r1)
; CHECK-BE-NEXT: std r3, -24(r1)
; CHECK-BE-NEXT: addi r3, r1, -32
; CHECK-BE-NEXT: lxvw4x v3, 0, r3
; CHECK-BE-NEXT: addi r3, r1, -16
-; CHECK-BE-NEXT: vmrghh v3, v3, v2
-; CHECK-BE-NEXT: vsrw v2, v2, v2
+; CHECK-BE-NEXT: vmrghh v3, v3, v4
; CHECK-BE-NEXT: xxsel vs0, v3, vs0, v2
; CHECK-BE-NEXT: stxvw4x vs0, 0, r3
; CHECK-BE-NEXT: lwz r3, -16(r1)
@@ -33,6 +34,7 @@ define void @test() local_unnamed_addr #0 align 2 {
; CHECK-P9-BE: # %bb.0: # %bb
; CHECK-P9-BE-NEXT: lhz r3, 0(r3)
; CHECK-P9-BE-NEXT: vspltisw v3, -16
+; CHECK-P9-BE-NEXT: # freeze: $v2
; CHECK-P9-BE-NEXT: xxlxor vs2, vs2, vs2
; CHECK-P9-BE-NEXT: addi r3, r3, 1
; CHECK-P9-BE-NEXT: vsrw v3, v3, v3
diff --git a/llvm/test/CodeGen/RISCV/freeze-multi-use.ll b/llvm/test/CodeGen/RISCV/freeze-multi-use.ll
index 18b76447406603..202d37fcfbb59c 100644
--- a/llvm/test/CodeGen/RISCV/freeze-multi-use.ll
+++ b/llvm/test/CodeGen/RISCV/freeze-multi-use.ll
@@ -8,14 +8,16 @@
define i32 @add_twice(i32 %n) {
; RV32-LABEL: add_twice:
; RV32: # %bb.0:
-; RV32-NEXT: add a0, a0, a0
-; RV32-NEXT: add a0, a0, a0
+; RV32-NEXT: # freeze: $x11
+; RV32-NEXT: add a0, a0, a1
+; RV32-NEXT: add a0, a0, a1
; RV32-NEXT: ret
;
; RV64-LABEL: add_twice:
; RV64: # %bb.0:
-; RV64-NEXT: add a0, a0, a0
-; RV64-NEXT: addw a0, a0, a0
+; RV64-NEXT: # freeze: $x11
+; RV64-NEXT: add a0, a0, a1
+; RV64-NEXT: addw a0, a0, a1
; RV64-NEXT: ret
%w = freeze i32 poison
%a = add i32 %n, %w
@@ -30,8 +32,9 @@ define i32 @cross_block(i1 %c, i32 %x) {
; RV32-NEXT: andi a0, a0, 1
; RV32-NEXT: beqz a0, .LBB1_2
; RV32-NEXT: # %bb.1: # %t
-; RV32-NEXT: add a0, a0, a1
-; RV32-NEXT: sub a0, a0, a0
+; RV32-NEXT: # freeze: $x10
+; RV32-NEXT: add a1, a0, a1
+; RV32-NEXT: sub a0, a1, a0
; RV32-NEXT: ret
; RV32-NEXT: .LBB1_2: # %e
; RV32-NEXT: mv a0, a1
@@ -42,8 +45,9 @@ define i32 @cross_block(i1 %c, i32 %x) {
; RV64-NEXT: andi a0, a0, 1
; RV64-NEXT: beqz a0, .LBB1_2
; RV64-NEXT: # %bb.1: # %t
-; RV64-NEXT: add a0, a0, a1
-; RV64-NEXT: subw a0, a0, a0
+; RV64-NEXT: # freeze: $x10
+; RV64-NEXT: add a1, a0, a1
+; RV64-NEXT: subw a0, a1, a0
; RV64-NEXT: ret
; RV64-NEXT: .LBB1_2: # %e
; RV64-NEXT: mv a0, a1
@@ -66,16 +70,23 @@ define i32 @float_bits() {
; RV32-NEXT: addi sp, sp, -16
; RV32-NEXT: .cfi_def_cfa_offset 16
; RV32-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32-NEXT: sw s0, 8(sp) # 4-byte Folded Spill
; RV32-NEXT: .cfi_offset ra, -4
+; RV32-NEXT: .cfi_offset s0, -8
+; RV32-NEXT: # freeze: $x8
+; RV32-NEXT: mv a0, s0
+; RV32-NEXT: mv a1, s0
; RV32-NEXT: call __unordsf2
-; RV32-NEXT: slli a1, a0, 1
-; RV32-NEXT: srli a1, a1, 1
-; RV32-NEXT: lui a2, 522240
+; RV32-NEXT: slli s0, s0, 1
+; RV32-NEXT: srli s0, s0, 1
+; RV32-NEXT: lui a1, 522240
; RV32-NEXT: snez a0, a0
-; RV32-NEXT: sltu a1, a2, a1
+; RV32-NEXT: sltu a1, a1, s0
; RV32-NEXT: xor a0, a0, a1
; RV32-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32-NEXT: lw s0, 8(sp) # 4-byte Folded Reload
; RV32-NEXT: .cfi_restore ra
+; RV32-NEXT: .cfi_restore s0
; RV32-NEXT: addi sp, sp, 16
; RV32-NEXT: .cfi_def_cfa_offset 0
; RV32-NEXT: ret
@@ -85,16 +96,23 @@ define i32 @float_bits() {
; RV64-NEXT: addi sp, sp, -16
; RV64-NEXT: .cfi_def_cfa_offset 16
; RV64-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
+; RV64-NEXT: sd s0, 0(sp) # 8-byte Folded Spill
; RV64-NEXT: .cfi_offset ra, -8
+; RV64-NEXT: .cfi_offset s0, -16
+; RV64-NEXT: # freeze: $x8
+; RV64-NEXT: mv a0, s0
+; RV64-NEXT: mv a1, s0
; RV64-NEXT: call __unordsf2
-; RV64-NEXT: slli a1, a0, 33
-; RV64-NEXT: srli a1, a1, 33
-; RV64-NEXT: lui a2, 522240
+; RV64-NEXT: slli s0, s0, 33
+; RV64-NEXT: srli s0, s0, 33
+; RV64-NEXT: lui a1, 522240
; RV64-NEXT: snez a0, a0
-; RV64-NEXT: sltu a1, a2, a1
+; RV64-NEXT: sltu a1, a1, s0
; RV64-NEXT: xor a0, a0, a1
; RV64-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
+; RV64-NEXT: ld s0, 0(sp) # 8-byte Folded Reload
; RV64-NEXT: .cfi_restore ra
+; RV64-NEXT: .cfi_restore s0
; RV64-NEXT: addi sp, sp, 16
; RV64-NEXT: .cfi_def_cfa_offset 0
; RV64-NEXT: ret
@@ -114,8 +132,13 @@ define i32 @across_call() nounwind {
; RV32: # %bb.0:
; RV32-NEXT: addi sp, sp, -16
; RV32-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32-NEXT: sw s0, 8(sp) # 4-byte Folded Spill
+; RV32-NEXT: # freeze: $x8
+; RV32-NEXT: mv a0, s0
; RV32-NEXT: call use
+; RV32-NEXT: mv a0, s0
; RV32-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32-NEXT: lw s0, 8(sp) # 4-byte Folded Reload
; RV32-NEXT: addi sp, sp, 16
; RV32-NEXT: ret
;
@@ -123,8 +146,13 @@ define i32 @across_call() nounwind {
; RV64: # %bb.0:
; RV64-NEXT: addi sp, sp, -16
; RV64-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
+; RV64-NEXT: sd s0, 0(sp) # 8-byte Folded Spill
+; RV64-NEXT: # freeze: $x8
+; RV64-NEXT: mv a0, s0
; RV64-NEXT: call use
+; RV64-NEXT: mv a0, s0
; RV64-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
+; RV64-NEXT: ld s0, 0(sp) # 8-byte Folded Reload
; RV64-NEXT: addi sp, sp, 16
; RV64-NEXT: ret
%w = freeze i32 poison
diff --git a/llvm/test/CodeGen/RISCV/rvv/freeze-multi-use.ll b/llvm/test/CodeGen/RISCV/rvv/freeze-multi-use.ll
index 71d79b5f18b4ef..ee48dc88fd0b0c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/freeze-multi-use.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/freeze-multi-use.ll
@@ -7,13 +7,14 @@
define i32 @partial_vector(i32 %x) {
; CHECK-LABEL: partial_vector:
; CHECK: # %bb.0:
+; CHECK-NEXT: # freeze: $x10
; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma
; CHECK-NEXT: vmv.v.x v8, a0
; CHECK-NEXT: vadd.vx v8, v8, a0
; CHECK-NEXT: vslidedown.vi v8, v8, 1
-; CHECK-NEXT: vmv.x.s a0, v8
-; CHECK-NEXT: add a1, a0, a0
-; CHECK-NEXT: subw a0, a0, a1
+; CHECK-NEXT: vmv.x.s a1, v8
+; CHECK-NEXT: add a0, a0, a0
+; CHECK-NEXT: subw a0, a1, a0
; CHECK-NEXT: ret
%v = insertelement <4 x i32> poison, i32 %x, i32 0
%w = freeze <4 x i32> %v
@@ -29,11 +30,20 @@ define i32 @partial_vector(i32 %x) {
define <4 x i32> @vec_across_call() nounwind {
; CHECK-LABEL: vec_across_call:
; CHECK: # %bb.0:
-; CHECK-NEXT: addi sp, sp, -16
-; CHECK-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT: addi sp, sp, -48
+; CHECK-NEXT: sd ra, 40(sp) # 8-byte Folded Spill
+; CHECK-NEXT: csrr a0, vlenb
+; CHECK-NEXT: sub sp, sp, a0
+; CHECK-NEXT: # freeze: $v8
+; CHECK-NEXT: addi a0, sp, 32
+; CHECK-NEXT: vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
; CHECK-NEXT: call usev
-; CHECK-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
-; CHECK-NEXT: addi sp, sp, 16
+; CHECK-NEXT: addi a0, sp, 32
+; CHECK-NEXT: vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
+; CHECK-NEXT: csrr a0, vlenb
+; CHECK-NEXT: add sp, sp, a0
+; CHECK-NEXT: ld ra, 40(sp) # 8-byte Folded Reload
+; CHECK-NEXT: addi sp, sp, 48
; CHECK-NEXT: ret
%w = freeze <4 x i32> poison
call void @usev(<4 x i32> %w)
diff --git a/llvm/test/CodeGen/SystemZ/pr60413.ll b/llvm/test/CodeGen/SystemZ/pr60413.ll
index 97cf1050408dd5..7958ce8b0fd66f 100644
--- a/llvm/test/CodeGen/SystemZ/pr60413.ll
+++ b/llvm/test/CodeGen/SystemZ/pr60413.ll
@@ -21,8 +21,9 @@ define dso_local void @m() local_unnamed_addr #1 {
; CHECK-NEXT: ic %r1, 6(%r2)
; CHECK-NEXT: larl %r2, e
; CHECK-NEXT: lb %r0, 3(%r2)
-; CHECK-NEXT: vlvgp %v0, %r0, %r1
-; CHECK-NEXT: vlvgp %v1, %r1, %r0
+; CHECK-NEXT: # freeze: $r2l
+; CHECK-NEXT: vlvgp %v0, %r2, %r1
+; CHECK-NEXT: vlvgp %v1, %r1, %r2
; CHECK-NEXT: vlvgf %v1, %r1, 0
; CHECK-NEXT: vlvgf %v1, %r1, 2
; CHECK-NEXT: vlvgp %v2, %r1, %r1
@@ -31,8 +32,8 @@ define dso_local void @m() local_unnamed_addr #1 {
; CHECK-NEXT: chi %r1, 128
; CHECK-NEXT: ipm %r1
; CHECK-NEXT: risbg %r1, %r1, 63, 191, 36
-; CHECK-NEXT: vlvgf %v0, %r0, 0
-; CHECK-NEXT: vlvgf %v0, %r0, 2
+; CHECK-NEXT: vlvgf %v0, %r2, 0
+; CHECK-NEXT: vlvgf %v0, %r2, 2
; CHECK-NEXT: vgbm %v3, 30583
; CHECK-NEXT: vn %v0, %v0, %v3
; CHECK-NEXT: vn %v1, %v1, %v3
@@ -74,6 +75,7 @@ define dso_local void @m() local_unnamed_addr #1 {
; CHECK-NEXT: rosbg %r2, %r3, 32, 61, 2
; CHECK-NEXT: rosbg %r2, %r5, 32, 62, 1
; CHECK-NEXT: or %r2, %r14
+; CHECK-NEXT: # freeze: $r3l
; CHECK-NEXT: xilf %r3, 4294967295
; CHECK-NEXT: or %r3, %r2
; CHECK-NEXT: tmll %r3, 65535
diff --git a/llvm/test/CodeGen/X86/bit-manip-i256.ll b/llvm/test/CodeGen/X86/bit-manip-i256.ll
index 042b7d428ba5cc..c6d8b26a7cfb3e 100644
--- a/llvm/test/CodeGen/X86/bit-manip-i256.ll
+++ b/llvm/test/CodeGen/X86/bit-manip-i256.ll
@@ -2856,15 +2856,16 @@ define i256 @isolate_msb_i256_vector(<4 x i64> %v0, i256 %idx) nounwind {
;
; AVX512F-LABEL: isolate_msb_i256_vector:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vmovaps {{.*#+}} zmm1 = [0,0,0,9223372036854775808,0,0,0,0]
-; AVX512F-NEXT: vmovups %zmm1, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: vpermq {{.*#+}} ymm1 = ymm0[3,2,1,0]
+; AVX512F-NEXT: vplzcntq %zmm1, %zmm2
+; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2 # [0,64,128,192]
; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k0
-; AVX512F-NEXT: vplzcntq %zmm1, %zmm1
-; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
+; AVX512F-NEXT: # freeze: $ymm1
; AVX512F-NEXT: kshiftlw $12, %k0, %k0
; AVX512F-NEXT: kshiftrw $12, %k0, %k1
-; AVX512F-NEXT: vpcompressq %zmm1, %zmm1 {%k1}
+; AVX512F-NEXT: vpcompressq %zmm2, %zmm1 {%k1}
+; AVX512F-NEXT: vmovaps {{.*#+}} zmm2 = [0,0,0,9223372036854775808,0,0,0,0]
+; AVX512F-NEXT: vmovups %zmm2, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: vmovd %xmm1, %ecx
; AVX512F-NEXT: movl %ecx, %eax
; AVX512F-NEXT: shrb $6, %al
@@ -2900,12 +2901,13 @@ define i256 @isolate_msb_i256_vector(<4 x i64> %v0, i256 %idx) nounwind {
; AVX512VL-NEXT: vptestmq %ymm1, %ymm1, %k1
; AVX512VL-NEXT: vplzcntq %ymm1, %ymm1
; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
-; AVX512VL-NEXT: vpcompressq %ymm1, %ymm1 {%k1}
-; AVX512VL-NEXT: vmovaps {{.*#+}} ymm2 = [0,0,0,9223372036854775808]
-; AVX512VL-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: vxorps %xmm2, %xmm2, %xmm2
-; AVX512VL-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: vmovd %xmm1, %ecx
+; AVX512VL-NEXT: # freeze: $ymm2
+; AVX512VL-NEXT: vpcompressq %ymm1, %ymm2 {%k1}
+; AVX512VL-NEXT: vmovaps {{.*#+}} ymm1 = [0,0,0,9223372036854775808]
+; AVX512VL-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; AVX512VL-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: vmovd %xmm2, %ecx
; AVX512VL-NEXT: movl %ecx, %eax
; AVX512VL-NEXT: shrb $6, %al
; AVX512VL-NEXT: movzbl %al, %edx
@@ -2939,12 +2941,13 @@ define i256 @isolate_msb_i256_vector(<4 x i64> %v0, i256 %idx) nounwind {
; AVX512VBMI-NEXT: vptestmq %ymm1, %ymm1, %k1
; AVX512VBMI-NEXT: vplzcntq %ymm1, %ymm1
; AVX512VBMI-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
-; AVX512VBMI-NEXT: vpcompressq %ymm1, %ymm1 {%k1}
-; AVX512VBMI-NEXT: vmovaps {{.*#+}} ymm2 = [0,0,0,9223372036854775808]
-; AVX512VBMI-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: vxorps %xmm2, %xmm2, %xmm2
-; AVX512VBMI-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: vmovd %xmm1, %ecx
+; AVX512VBMI-NEXT: # freeze: $ymm2
+; AVX512VBMI-NEXT: vpcompressq %ymm1, %ymm2 {%k1}
+; AVX512VBMI-NEXT: vmovaps {{.*#+}} ymm1 = [0,0,0,9223372036854775808]
+; AVX512VBMI-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; AVX512VBMI-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: vmovd %xmm2, %ecx
; AVX512VBMI-NEXT: movl %ecx, %eax
; AVX512VBMI-NEXT: shrb $6, %al
; AVX512VBMI-NEXT: movzbl %al, %edx
diff --git a/llvm/test/CodeGen/X86/combine-urem.ll b/llvm/test/CodeGen/X86/combine-urem.ll
index 34c7d3deea0237..dd33f38cc4c75a 100644
--- a/llvm/test/CodeGen/X86/combine-urem.ll
+++ b/llvm/test/CodeGen/X86/combine-urem.ll
@@ -63,16 +63,18 @@ define <4 x i32> @combine_vec_urem_by_negone(<4 x i32> %x) {
define <4 x i32> @combine_vec_urem_undef_by_negone(<4 x i32> %in) {
; SSE-LABEL: combine_vec_urem_undef_by_negone:
; SSE: # %bb.0:
+; SSE-NEXT: # freeze: $xmm1
; SSE-NEXT: pcmpeqd %xmm0, %xmm0
-; SSE-NEXT: pcmpeqd %xmm0, %xmm0
-; SSE-NEXT: pandn %xmm0, %xmm0
+; SSE-NEXT: pcmpeqd %xmm1, %xmm0
+; SSE-NEXT: pandn %xmm1, %xmm0
; SSE-NEXT: retq
;
; AVX-LABEL: combine_vec_urem_undef_by_negone:
; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vpandn %xmm0, %xmm0, %xmm0
+; AVX-NEXT: # freeze: $xmm0
+; AVX-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1
+; AVX-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm1
+; AVX-NEXT: vpandn %xmm0, %xmm1, %xmm0
; AVX-NEXT: retq
%x = call <4 x i32> @llvm.x86.sse2.pslli.d(<4 x i32> undef, i32 0)
%y = urem <4 x i32> %x, <i32 -1, i32 -1, i32 -1, i32 -1>
diff --git a/llvm/test/CodeGen/X86/freeze-binary.ll b/llvm/test/CodeGen/X86/freeze-binary.ll
index e4b2dcf0a62dd5..0d011746f753ed 100644
--- a/llvm/test/CodeGen/X86/freeze-binary.ll
+++ b/llvm/test/CodeGen/X86/freeze-binary.ll
@@ -399,11 +399,13 @@ define i32 @freeze_shl_nsw(i32 %a0) nounwind {
define i32 @freeze_shl_outofrange(i32 %a0) nounwind {
; X86-LABEL: freeze_shl_outofrange:
; X86: # %bb.0:
+; X86-NEXT: # freeze: $eax
; X86-NEXT: shll $2, %eax
; X86-NEXT: retl
;
; X64-LABEL: freeze_shl_outofrange:
; X64: # %bb.0:
+; X64-NEXT: # freeze: $eax
; X64-NEXT: shll $2, %eax
; X64-NEXT: retq
%x = shl i32 %a0, 32
@@ -513,11 +515,13 @@ define i32 @freeze_ashr_exact_extra_use(i32 %a0, ptr %escape) nounwind {
define i32 @freeze_ashr_outofrange(i32 %a0) nounwind {
; X86-LABEL: freeze_ashr_outofrange:
; X86: # %bb.0:
+; X86-NEXT: # freeze: $eax
; X86-NEXT: sarl $3, %eax
; X86-NEXT: retl
;
; X64-LABEL: freeze_ashr_outofrange:
; X64: # %bb.0:
+; X64-NEXT: # freeze: $eax
; X64-NEXT: sarl $3, %eax
; X64-NEXT: retq
%x = ashr i32 %a0, 32
@@ -628,11 +632,13 @@ define i32 @freeze_lshr_exact_extra_use(i32 %a0, ptr %escape) nounwind {
define i32 @freeze_lshr_outofrange(i32 %a0) nounwind {
; X86-LABEL: freeze_lshr_outofrange:
; X86: # %bb.0:
+; X86-NEXT: # freeze: $eax
; X86-NEXT: shrl %eax
; X86-NEXT: retl
;
; X64-LABEL: freeze_lshr_outofrange:
; X64: # %bb.0:
+; X64-NEXT: # freeze: $eax
; X64-NEXT: shrl %eax
; X64-NEXT: retq
%x = lshr i32 %a0, 32
diff --git a/llvm/test/CodeGen/X86/freeze-multi-use.ll b/llvm/test/CodeGen/X86/freeze-multi-use.ll
index 08bee89cec6670..07ae3e751c95bb 100644
--- a/llvm/test/CodeGen/X86/freeze-multi-use.ll
+++ b/llvm/test/CodeGen/X86/freeze-multi-use.ll
@@ -7,9 +7,10 @@
define i32 @add_twice(i32 %n) {
; CHECK-LABEL: add_twice:
; CHECK: # %bb.0:
-; CHECK-NEXT: movl %edi, %eax
-; CHECK-NEXT: addl %eax, %eax
-; CHECK-NEXT: addl %eax, %eax
+; CHECK-NEXT: # kill: def $edi killed $edi def $rdi
+; CHECK-NEXT: # freeze: $ecx
+; CHECK-NEXT: leal (%rdi,%rcx), %eax
+; CHECK-NEXT: addl %ecx, %eax
; CHECK-NEXT: retq
%w = freeze i32 poison
%a = add i32 %n, %w
@@ -21,8 +22,10 @@ define i32 @add_twice(i32 %n) {
define i32 @ashr_mul(i32 %x) {
; CHECK-LABEL: ashr_mul:
; CHECK: # %bb.0:
+; CHECK-NEXT: # freeze: $ecx
+; CHECK-NEXT: movl %ecx, %eax
; CHECK-NEXT: sarl $3, %eax
-; CHECK-NEXT: imull %eax, %eax
+; CHECK-NEXT: imull %ecx, %eax
; CHECK-NEXT: retq
%p = ashr i32 %x, 32
%w = freeze i32 %p
@@ -35,14 +38,14 @@ define i32 @ashr_mul(i32 %x) {
define i32 @cross_block(i1 %c, i32 %x) {
; CHECK-LABEL: cross_block:
; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movl %esi, %eax
; CHECK-NEXT: testb $1, %dil
; CHECK-NEXT: je .LBB2_2
; CHECK-NEXT: # %bb.1: # %t
-; CHECK-NEXT: addl %esi, %eax
-; CHECK-NEXT: subl %eax, %eax
-; CHECK-NEXT: retq
+; CHECK-NEXT: # freeze: $ecx
+; CHECK-NEXT: addl %ecx, %eax
+; CHECK-NEXT: subl %ecx, %eax
; CHECK-NEXT: .LBB2_2: # %e
-; CHECK-NEXT: movl %esi, %eax
; CHECK-NEXT: retq
entry:
%w = freeze i32 poison
@@ -59,9 +62,12 @@ e:
define i32 @across_call() nounwind {
; CHECK-LABEL: across_call:
; CHECK: # %bb.0:
-; CHECK-NEXT: pushq %rax
+; CHECK-NEXT: pushq %rbx
+; CHECK-NEXT: # freeze: $ebx
+; CHECK-NEXT: movl %ebx, %edi
; CHECK-NEXT: callq use at PLT
-; CHECK-NEXT: popq %rax
+; CHECK-NEXT: movl %ebx, %eax
+; CHECK-NEXT: popq %rbx
; CHECK-NEXT: retq
%w = freeze i32 poison
call void @use(i32 %w)
@@ -72,9 +78,12 @@ define i32 @across_call() nounwind {
define <4 x i32> @vec_across_call() nounwind {
; CHECK-LABEL: vec_across_call:
; CHECK: # %bb.0:
-; CHECK-NEXT: pushq %rax
+; CHECK-NEXT: subq $24, %rsp
+; CHECK-NEXT: # freeze: $xmm0
+; CHECK-NEXT: movaps %xmm0, (%rsp) # 16-byte Spill
; CHECK-NEXT: callq usev at PLT
-; CHECK-NEXT: popq %rax
+; CHECK-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload
+; CHECK-NEXT: addq $24, %rsp
; CHECK-NEXT: retq
%w = freeze <4 x i32> poison
call void @usev(<4 x i32> %w)
diff --git a/llvm/test/CodeGen/X86/freeze-vector.ll b/llvm/test/CodeGen/X86/freeze-vector.ll
index 2327dac331088d..6461c3dde3fbea 100644
--- a/llvm/test/CodeGen/X86/freeze-vector.ll
+++ b/llvm/test/CodeGen/X86/freeze-vector.ll
@@ -594,7 +594,8 @@ define void @freeze_buildvector_one_undef_elt(ptr %origin0, ptr %origin1, ptr %o
; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
; X86-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
; X86-NEXT: vpinsrd $1, (%edx), %xmm0, %xmm0
-; X86-NEXT: vpinsrd $2, %eax, %xmm0, %xmm0
+; X86-NEXT: # freeze: $edx
+; X86-NEXT: vpinsrd $2, %edx, %xmm0, %xmm0
; X86-NEXT: vpinsrd $3, (%ecx), %xmm0, %xmm0
; X86-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
; X86-NEXT: vmovdqa %xmm0, (%eax)
@@ -605,6 +606,7 @@ define void @freeze_buildvector_one_undef_elt(ptr %origin0, ptr %origin1, ptr %o
; X64: # %bb.0:
; X64-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
; X64-NEXT: vpinsrd $1, (%rsi), %xmm0, %xmm0
+; X64-NEXT: # freeze: $eax
; X64-NEXT: vpinsrd $2, %eax, %xmm0, %xmm0
; X64-NEXT: vpinsrd $3, (%rcx), %xmm0, %xmm0
; X64-NEXT: vpbroadcastd {{.*#+}} xmm1 = [7,7,7,7]
@@ -1125,13 +1127,15 @@ define <4 x i32> @freeze_poison_multi_use_with_zeros(ptr %p) nounwind {
; X86-LABEL: freeze_poison_multi_use_with_zeros:
; X86: # %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl %eax, (%eax)
-; X86-NEXT: vmovd %eax, %xmm0
+; X86-NEXT: # freeze: $ecx
+; X86-NEXT: movl %ecx, (%eax)
+; X86-NEXT: vmovd %ecx, %xmm0
; X86-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,0,1,1]
; X86-NEXT: retl
;
; X64-LABEL: freeze_poison_multi_use_with_zeros:
; X64: # %bb.0:
+; X64-NEXT: # freeze: $eax
; X64-NEXT: movl %eax, (%rdi)
; X64-NEXT: vmovd %eax, %xmm0
; X64-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,0,1,1]
@@ -1147,15 +1151,17 @@ define <4 x i32> @freeze_poison_multi_use_splat(ptr %p, i32 %x) nounwind {
; X86-LABEL: freeze_poison_multi_use_splat:
; X86: # %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl %eax, (%eax)
-; X86-NEXT: vmovd %eax, %xmm0
+; X86-NEXT: # freeze: $ecx
+; X86-NEXT: movl %ecx, (%eax)
+; X86-NEXT: vmovd %ecx, %xmm0
; X86-NEXT: vpinsrd $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; X86-NEXT: vpinsrd $2, %eax, %xmm0, %xmm0
-; X86-NEXT: vpinsrd $3, %eax, %xmm0, %xmm0
+; X86-NEXT: vpinsrd $2, %ecx, %xmm0, %xmm0
+; X86-NEXT: vpinsrd $3, %ecx, %xmm0, %xmm0
; X86-NEXT: retl
;
; X64-LABEL: freeze_poison_multi_use_splat:
; X64: # %bb.0:
+; X64-NEXT: # freeze: $eax
; X64-NEXT: movl %eax, (%rdi)
; X64-NEXT: vmovd %eax, %xmm0
; X64-NEXT: vpinsrd $1, %esi, %xmm0, %xmm0
@@ -1176,15 +1182,17 @@ define <8 x i32> @freeze_poison_multi_use_upper(ptr %p, i32 %x) nounwind {
; X86-LABEL: freeze_poison_multi_use_upper:
; X86: # %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl %eax, (%eax)
+; X86-NEXT: # freeze: $ecx
+; X86-NEXT: movl %ecx, (%eax)
; X86-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X86-NEXT: vmovd %eax, %xmm1
+; X86-NEXT: vmovd %ecx, %xmm1
; X86-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
; X86-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
; X86-NEXT: retl
;
; X64-LABEL: freeze_poison_multi_use_upper:
; X64: # %bb.0:
+; X64-NEXT: # freeze: $eax
; X64-NEXT: movl %eax, (%rdi)
; X64-NEXT: vmovd %esi, %xmm0
; X64-NEXT: vmovd %eax, %xmm1
diff --git a/llvm/test/CodeGen/X86/freeze.ll b/llvm/test/CodeGen/X86/freeze.ll
index 2609b2fafac54c..384a7116b7bb83 100644
--- a/llvm/test/CodeGen/X86/freeze.ll
+++ b/llvm/test/CodeGen/X86/freeze.ll
@@ -6,7 +6,7 @@
define i32 @freeze_int() {
; X86ASM-LABEL: freeze_int:
; X86ASM: # %bb.0:
-; X86ASM-NEXT: # implicit-def: $eax
+; X86ASM-NEXT: # freeze: $eax
; X86ASM-NEXT: imull %eax, %eax
; X86ASM-NEXT: retq
%y1 = freeze i32 undef
@@ -17,7 +17,7 @@ define i32 @freeze_int() {
define void @freeze_int_asm() {
; X86ASM-LABEL: freeze_int_asm:
; X86ASM: # %bb.0:
-; X86ASM-NEXT: # implicit-def: $ecx
+; X86ASM-NEXT: # freeze: $ecx
; X86ASM-NEXT: imull %ecx, %ecx
; X86ASM-NEXT: #APP
; X86ASM-NEXT: # use %ecx
@@ -32,6 +32,7 @@ define void @freeze_int_asm() {
define i5 @freeze_int2() {
; X86ASM-LABEL: freeze_int2:
; X86ASM: # %bb.0:
+; X86ASM-NEXT: # freeze: $al
; X86ASM-NEXT: mulb %al
; X86ASM-NEXT: retq
%y1 = freeze i5 undef
@@ -42,7 +43,7 @@ define i5 @freeze_int2() {
define float @freeze_float() {
; X86ASM-LABEL: freeze_float:
; X86ASM: # %bb.0:
-; X86ASM-NEXT: # implicit-def: $xmm0
+; X86ASM-NEXT: # freeze: $xmm0
; X86ASM-NEXT: addss %xmm0, %xmm0
; X86ASM-NEXT: retq
%y1 = freeze float undef
@@ -55,6 +56,7 @@ define half @freeze_half() {
; X86ASM: # %bb.0:
; X86ASM-NEXT: pushq %rax
; X86ASM-NEXT: .cfi_def_cfa_offset 16
+; X86ASM-NEXT: # freeze: $xmm0
; X86ASM-NEXT: callq __extendhfsf2 at PLT
; X86ASM-NEXT: addss %xmm0, %xmm0
; X86ASM-NEXT: callq __truncsfhf2 at PLT
@@ -69,6 +71,7 @@ define half @freeze_half() {
define <2 x i32> @freeze_ivec() {
; X86ASM-LABEL: freeze_ivec:
; X86ASM: # %bb.0:
+; X86ASM-NEXT: # freeze: $xmm0
; X86ASM-NEXT: paddd %xmm0, %xmm0
; X86ASM-NEXT: retq
%y1 = freeze <2 x i32> undef
@@ -79,6 +82,7 @@ define <2 x i32> @freeze_ivec() {
define ptr @freeze_ptr() {
; X86ASM-LABEL: freeze_ptr:
; X86ASM: # %bb.0:
+; X86ASM-NEXT: # freeze: $rax
; X86ASM-NEXT: addq $4, %rax
; X86ASM-NEXT: retq
%y1 = freeze ptr undef
@@ -89,7 +93,9 @@ define ptr @freeze_ptr() {
define i32 @freeze_struct() {
; X86ASM-LABEL: freeze_struct:
; X86ASM: # %bb.0:
-; X86ASM-NEXT: addl %eax, %eax
+; X86ASM-NEXT: # freeze: $ecx
+; X86ASM-NEXT: # freeze: $eax
+; X86ASM-NEXT: addl %ecx, %eax
; X86ASM-NEXT: retq
%y1 = freeze %struct.T undef
%v1 = extractvalue %struct.T %y1, 0
@@ -101,7 +107,9 @@ define i32 @freeze_struct() {
define i32 @freeze_anonstruct() {
; X86ASM-LABEL: freeze_anonstruct:
; X86ASM: # %bb.0:
-; X86ASM-NEXT: addl %eax, %eax
+; X86ASM-NEXT: # freeze: $ecx
+; X86ASM-NEXT: # freeze: $eax
+; X86ASM-NEXT: addl %ecx, %eax
; X86ASM-NEXT: retq
%y1 = freeze {i32, i32} undef
%v1 = extractvalue {i32, i32} %y1, 0
@@ -113,8 +121,10 @@ define i32 @freeze_anonstruct() {
define i32 @freeze_anonstruct2() {
; X86ASM-LABEL: freeze_anonstruct2:
; X86ASM: # %bb.0:
+; X86ASM-NEXT: # freeze: $ecx
+; X86ASM-NEXT: # freeze: $ax
; X86ASM-NEXT: movzwl %ax, %eax
-; X86ASM-NEXT: addl %eax, %eax
+; X86ASM-NEXT: addl %ecx, %eax
; X86ASM-NEXT: retq
%y1 = freeze {i32, i16} undef
%v1 = extractvalue {i32, i16} %y1, 0
@@ -127,7 +137,9 @@ define i32 @freeze_anonstruct2() {
define i64 @freeze_array() {
; X86ASM-LABEL: freeze_array:
; X86ASM: # %bb.0:
-; X86ASM-NEXT: addq %rax, %rax
+; X86ASM-NEXT: # freeze: $rcx
+; X86ASM-NEXT: # freeze: $rax
+; X86ASM-NEXT: addq %rcx, %rax
; X86ASM-NEXT: retq
%y1 = freeze [2 x i64] undef
%v1 = extractvalue [2 x i64] %y1, 0
@@ -207,6 +219,8 @@ entry:
define i128 @freeze_i128() {
; X86ASM-LABEL: freeze_i128:
; X86ASM: # %bb.0:
+; X86ASM-NEXT: # freeze: $rax
+; X86ASM-NEXT: # freeze: $rdx
; X86ASM-NEXT: retq
%y1 = freeze i128 poison
ret i128 %y1
diff --git a/llvm/test/CodeGen/X86/pr169485.ll b/llvm/test/CodeGen/X86/pr169485.ll
index 63035fce026159..008aad367cdb0b 100644
--- a/llvm/test/CodeGen/X86/pr169485.ll
+++ b/llvm/test/CodeGen/X86/pr169485.ll
@@ -9,8 +9,9 @@ define <8 x float> @widget(<2 x float> %arg) #0 {
; CHECK-NEXT: xorl %eax, %eax
; CHECK-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill
; CHECK-NEXT: callq *%rax
-; CHECK-NEXT: vmovaps (%rsp), %xmm0 # 16-byte Reload
-; CHECK-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,0],zero
+; CHECK-NEXT: vmovaps (%rsp), %xmm1 # 16-byte Reload
+; CHECK-NEXT: # freeze: $xmm0
+; CHECK-NEXT: vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],zero
; CHECK-NEXT: addq $24, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 8
; CHECK-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/pr57402.ll b/llvm/test/CodeGen/X86/pr57402.ll
index 338229f51cf6e2..3a1b5eb9639002 100644
--- a/llvm/test/CodeGen/X86/pr57402.ll
+++ b/llvm/test/CodeGen/X86/pr57402.ll
@@ -4,25 +4,27 @@
define void @PR57402() {
; CHECK-LABEL: PR57402:
; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: # freeze: $rcx
+; CHECK-NEXT: movl %ecx, %eax
; CHECK-NEXT: notl %eax
; CHECK-NEXT: andl $-2, %eax
-; CHECK-NEXT: leal 1(%rax,%rax,2), %ecx
-; CHECK-NEXT: movswq %cx, %rsi
-; CHECK-NEXT: xorl %edi, %edi
+; CHECK-NEXT: leal 1(%rax,%rax,2), %esi
+; CHECK-NEXT: movswq %si, %rdi
+; CHECK-NEXT: xorl %r8d, %r8d
; CHECK-NEXT: movq $-1, %rax
; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divq %rsi
-; CHECK-NEXT: testb %dil, %dil
+; CHECK-NEXT: divq %rdi
+; CHECK-NEXT: testb %r8b, %r8b
; CHECK-NEXT: jne .LBB0_4
; CHECK-NEXT: # %bb.1: # %entry
; CHECK-NEXT: xorl %eax, %eax
; CHECK-NEXT: testb %al, %al
; CHECK-NEXT: jne .LBB0_4
; CHECK-NEXT: # %bb.2: # %entry
-; CHECK-NEXT: andl %ecx, %edx
+; CHECK-NEXT: andl %esi, %edx
; CHECK-NEXT: movswl %dx, %eax
-; CHECK-NEXT: imull %eax, %eax
-; CHECK-NEXT: testq %rax, %rax
+; CHECK-NEXT: imull %eax, %ecx
+; CHECK-NEXT: testq %rcx, %rcx
; CHECK-NEXT: jne .LBB0_3
; CHECK-NEXT: .LBB0_4: # %if.end
; CHECK-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/pr57554.ll b/llvm/test/CodeGen/X86/pr57554.ll
index 0eb40979c35520..8c7fee5afc28a2 100644
--- a/llvm/test/CodeGen/X86/pr57554.ll
+++ b/llvm/test/CodeGen/X86/pr57554.ll
@@ -6,6 +6,7 @@
define void @PR57554() {
; CHECK-LABEL: PR57554:
; CHECK: # %bb.0:
+; CHECK-NEXT: # freeze: $al
; CHECK-NEXT: movb %al, a(%rip)
; CHECK-NEXT: retq
%.fr13 = freeze i8 undef
diff --git a/llvm/test/CodeGen/X86/var-permute-256.ll b/llvm/test/CodeGen/X86/var-permute-256.ll
index 5155077f41df09..9024545cca9a8c 100644
--- a/llvm/test/CodeGen/X86/var-permute-256.ll
+++ b/llvm/test/CodeGen/X86/var-permute-256.ll
@@ -1542,7 +1542,8 @@ define <16 x i16> @var_shuffle_v16i16_from_v8i16(<8 x i16> %v, <16 x i16> %indic
; AVX2-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
; AVX2-NEXT: vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [514,514,514,514,514,514,514,514,514,514,514,514,514,514,514,514]
; AVX2-NEXT: vpaddw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [256,256,256,256,256,256,256,256,256,256,256,256,256,256,256,256]
-; AVX2-NEXT: vpshufb %ymm1, %ymm0, %ymm2
+; AVX2-NEXT: # freeze: $xmm2
+; AVX2-NEXT: vpshufb %ymm1, %ymm2, %ymm2
; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
; AVX2-NEXT: vpshufb %ymm1, %ymm0, %ymm0
; AVX2-NEXT: vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
@@ -1554,7 +1555,8 @@ define <16 x i16> @var_shuffle_v16i16_from_v8i16(<8 x i16> %v, <16 x i16> %indic
; AVX512-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
; AVX512-NEXT: vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [514,514,514,514,514,514,514,514,514,514,514,514,514,514,514,514]
; AVX512-NEXT: vpaddw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [256,256,256,256,256,256,256,256,256,256,256,256,256,256,256,256]
-; AVX512-NEXT: vpshufb %ymm1, %ymm0, %ymm2
+; AVX512-NEXT: # freeze: $xmm2
+; AVX512-NEXT: vpshufb %ymm1, %ymm2, %ymm2
; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
; AVX512-NEXT: vpshufb %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
@@ -1568,6 +1570,7 @@ define <16 x i16> @var_shuffle_v16i16_from_v8i16(<8 x i16> %v, <16 x i16> %indic
; AVX512VLF-NEXT: vpaddw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [256,256,256,256,256,256,256,256,256,256,256,256,256,256,256,256]
; AVX512VLF-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
; AVX512VLF-NEXT: vpshufb %ymm1, %ymm0, %ymm2
+; AVX512VLF-NEXT: # freeze: $xmm0
; AVX512VLF-NEXT: vpshufb %ymm1, %ymm0, %ymm3
; AVX512VLF-NEXT: vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm0
; AVX512VLF-NEXT: vpternlogq {{.*#+}} ymm0 = ymm2 ^ (ymm0 & (ymm3 ^ ymm2))
diff --git a/llvm/test/CodeGen/X86/vector-compress-freeze.ll b/llvm/test/CodeGen/X86/vector-compress-freeze.ll
index 981557f9b56cfb..ddf3b306069dda 100644
--- a/llvm/test/CodeGen/X86/vector-compress-freeze.ll
+++ b/llvm/test/CodeGen/X86/vector-compress-freeze.ll
@@ -24,8 +24,9 @@ define <16 x i32> @test_compress_freeze(<16 x i32> %a0, <16 x i32> %a1, <16 x i8
; CHECK: # %bb.0:
; CHECK-NEXT: vpcmpgtd %zmm1, %zmm0, %k1
; CHECK-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
-; CHECK-NEXT: vpcompressd %zmm0, %zmm0 {%k1}
-; CHECK-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0
+; CHECK-NEXT: # freeze: $zmm1
+; CHECK-NEXT: vpcompressd %zmm0, %zmm1 {%k1}
+; CHECK-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0
; CHECK-NEXT: retq
%cmp = icmp sgt <16 x i32> %a0, %a1
%ext = zext <16 x i8> %a3 to <16 x i32>
diff --git a/llvm/test/CodeGen/X86/vector-compress.ll b/llvm/test/CodeGen/X86/vector-compress.ll
index e3b08871f47328..4ae8f55560743f 100644
--- a/llvm/test/CodeGen/X86/vector-compress.ll
+++ b/llvm/test/CodeGen/X86/vector-compress.ll
@@ -612,10 +612,10 @@ define <16 x i32> @test_compress_v16i32(<16 x i32> %vec, <16 x i1> %mask, <16 x
; AVX2-NEXT: vpextrd $1, %xmm3, %eax
; AVX2-NEXT: vmovd %xmm3, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: vpextrb $4, %xmm2, %edx
+; AVX2-NEXT: vpextrb $4, %xmm2, %esi
; AVX2-NEXT: andl $15, %ecx
; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: andl $1, %edx
+; AVX2-NEXT: andl $1, %esi
; AVX2-NEXT: vpextrb $1, %xmm2, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: vmovd %xmm2, %ecx
@@ -627,17 +627,17 @@ define <16 x i32> @test_compress_v16i32(<16 x i32> %vec, <16 x i1> %mask, <16 x
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: vpextrb $3, %xmm2, %r9d
+; AVX2-NEXT: vpextrb $3, %xmm2, %r8d
+; AVX2-NEXT: andl $1, %r8d
+; AVX2-NEXT: addq %rcx, %r8
+; AVX2-NEXT: addq %r8, %rsi
+; AVX2-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: vpextrb $5, %xmm2, %r9d
; AVX2-NEXT: andl $1, %r9d
-; AVX2-NEXT: addq %rcx, %r9
-; AVX2-NEXT: addq %r9, %rdx
-; AVX2-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: vpextrb $5, %xmm2, %edi
-; AVX2-NEXT: andl $1, %edi
-; AVX2-NEXT: addq %rdx, %rdi
+; AVX2-NEXT: addq %rsi, %r9
; AVX2-NEXT: vpextrb $6, %xmm2, %r10d
; AVX2-NEXT: andl $1, %r10d
-; AVX2-NEXT: addq %rdi, %r10
+; AVX2-NEXT: addq %r9, %r10
; AVX2-NEXT: vpextrb $7, %xmm2, %r11d
; AVX2-NEXT: andl $1, %r11d
; AVX2-NEXT: addq %r10, %r11
@@ -656,36 +656,38 @@ define <16 x i32> @test_compress_v16i32(<16 x i32> %vec, <16 x i1> %mask, <16 x
; AVX2-NEXT: vpextrb $12, %xmm2, %r13d
; AVX2-NEXT: andl $1, %r13d
; AVX2-NEXT: addq %r12, %r13
-; AVX2-NEXT: vpextrb $13, %xmm2, %edx
-; AVX2-NEXT: andl $1, %edx
-; AVX2-NEXT: addq %r13, %rdx
-; AVX2-NEXT: vpextrb $14, %xmm2, %ecx
-; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rdx, %rcx
-; AVX2-NEXT: vpextrb $15, %xmm2, %esi
+; AVX2-NEXT: vpextrb $13, %xmm2, %ecx
+; AVX2-NEXT: movq %rcx, %rsi
; AVX2-NEXT: andl $1, %esi
-; AVX2-NEXT: addq %rcx, %rsi
+; AVX2-NEXT: addq %r13, %rsi
+; AVX2-NEXT: vpextrb $14, %xmm2, %eax
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: andl $1, %ecx
+; AVX2-NEXT: addq %rsi, %rcx
+; AVX2-NEXT: vpextrb $15, %xmm2, %edx
+; AVX2-NEXT: andl $1, %edx
+; AVX2-NEXT: addq %rcx, %rdx
; AVX2-NEXT: vextractf128 $1, %ymm1, %xmm2
; AVX2-NEXT: vextractps $3, %xmm2, %eax
-; AVX2-NEXT: cmpq $16, %rsi
-; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
-; AVX2-NEXT: cmovbl 32(%rsp,%r8,4), %eax
+; AVX2-NEXT: cmpq $16, %rdx
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
+; AVX2-NEXT: cmovbl 32(%rsp,%rdi,4), %eax
; AVX2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; AVX2-NEXT: vmovss %xmm0, {{[0-9]+}}(%rsp)
-; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
-; AVX2-NEXT: vextractps $1, %xmm0, 32(%rsp,%r8,4)
-; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
-; AVX2-NEXT: vextractps $2, %xmm0, 32(%rsp,%r8,4)
-; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
-; AVX2-NEXT: vextractps $3, %xmm0, 32(%rsp,%r8,4)
-; AVX2-NEXT: andl $15, %r9d
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
+; AVX2-NEXT: vextractps $1, %xmm0, 32(%rsp,%rdi,4)
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
+; AVX2-NEXT: vextractps $2, %xmm0, 32(%rsp,%rdi,4)
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
+; AVX2-NEXT: vextractps $3, %xmm0, 32(%rsp,%rdi,4)
+; AVX2-NEXT: andl $15, %r8d
; AVX2-NEXT: vextractf128 $1, %ymm0, %xmm0
-; AVX2-NEXT: vmovss %xmm0, 32(%rsp,%r9,4)
+; AVX2-NEXT: vmovss %xmm0, 32(%rsp,%r8,4)
; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; AVX2-NEXT: andl $15, %eax
; AVX2-NEXT: vextractps $1, %xmm0, 32(%rsp,%rax,4)
-; AVX2-NEXT: andl $15, %edi
-; AVX2-NEXT: vextractps $2, %xmm0, 32(%rsp,%rdi,4)
+; AVX2-NEXT: andl $15, %r9d
+; AVX2-NEXT: vextractps $2, %xmm0, 32(%rsp,%r9,4)
; AVX2-NEXT: andl $15, %r10d
; AVX2-NEXT: vextractps $3, %xmm0, 32(%rsp,%r10,4)
; AVX2-NEXT: andl $15, %r11d
@@ -700,13 +702,13 @@ define <16 x i32> @test_compress_v16i32(<16 x i32> %vec, <16 x i1> %mask, <16 x
; AVX2-NEXT: vmovss %xmm2, 32(%rsp,%r12,4)
; AVX2-NEXT: andl $15, %r13d
; AVX2-NEXT: vextractps $1, %xmm2, 32(%rsp,%r13,4)
-; AVX2-NEXT: andl $15, %edx
-; AVX2-NEXT: vextractps $2, %xmm2, 32(%rsp,%rdx,4)
+; AVX2-NEXT: andl $15, %esi
+; AVX2-NEXT: vextractps $2, %xmm2, 32(%rsp,%rsi,4)
; AVX2-NEXT: andl $15, %ecx
; AVX2-NEXT: vextractps $3, %xmm2, 32(%rsp,%rcx,4)
-; AVX2-NEXT: cmpq $15, %rsi
+; AVX2-NEXT: cmpq $15, %rdx
; AVX2-NEXT: movl $15, %eax
-; AVX2-NEXT: cmovbq %rsi, %rax
+; AVX2-NEXT: cmovbq %rdx, %rax
; AVX2-NEXT: shll $2, %eax
; AVX2-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %ecx # 4-byte Reload
; AVX2-NEXT: movl %ecx, 32(%rsp,%rax)
@@ -828,7 +830,8 @@ define <16 x float> @test_compress_v16f32(<16 x float> %vec, <16 x i1> %mask, <1
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $15, %eax
; AVX2-NEXT: vextractps $3, %xmm1, (%rsp,%rax,4)
-; AVX2-NEXT: vpextrb $12, %xmm2, %edx
+; AVX2-NEXT: vpextrb $12, %xmm2, %eax
+; AVX2-NEXT: movq %rax, %rdx
; AVX2-NEXT: andl $1, %edx
; AVX2-NEXT: addq %rcx, %rdx
; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
@@ -844,26 +847,27 @@ define <16 x float> @test_compress_v16f32(<16 x float> %vec, <16 x i1> %mask, <1
; AVX2-NEXT: movl %eax, %ecx
; AVX2-NEXT: andl $15, %ecx
; AVX2-NEXT: vextractps $2, %xmm0, (%rsp,%rcx,4)
-; AVX2-NEXT: vpextrb $14, %xmm2, %edx
-; AVX2-NEXT: andl $1, %edx
-; AVX2-NEXT: vpextrb $15, %xmm2, %ecx
-; AVX2-NEXT: addq %rax, %rdx
+; AVX2-NEXT: vpextrb $14, %xmm2, %ecx
; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rdx, %rcx
-; AVX2-NEXT: # kill: def $edx killed $edx killed $rdx def $rdx
-; AVX2-NEXT: andl $15, %edx
+; AVX2-NEXT: vpextrb $15, %xmm2, %edx
+; AVX2-NEXT: addq %rax, %rcx
+; AVX2-NEXT: movq %rdx, %rax
+; AVX2-NEXT: andl $1, %eax
+; AVX2-NEXT: addq %rcx, %rax
+; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
+; AVX2-NEXT: andl $15, %ecx
; AVX2-NEXT: vshufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; AVX2-NEXT: vmovss %xmm0, (%rsp,%rdx,4)
-; AVX2-NEXT: cmpq $16, %rcx
+; AVX2-NEXT: vmovss %xmm0, (%rsp,%rcx,4)
+; AVX2-NEXT: cmpq $16, %rax
; AVX2-NEXT: jae .LBB9_2
; AVX2-NEXT: # %bb.1:
; AVX2-NEXT: vmovaps %xmm3, %xmm0
; AVX2-NEXT: .LBB9_2:
-; AVX2-NEXT: cmpq $15, %rcx
-; AVX2-NEXT: movl $15, %eax
-; AVX2-NEXT: cmovbq %rcx, %rax
-; AVX2-NEXT: shll $2, %eax
-; AVX2-NEXT: vmovss %xmm0, (%rsp,%rax)
+; AVX2-NEXT: cmpq $15, %rax
+; AVX2-NEXT: movl $15, %ecx
+; AVX2-NEXT: cmovbq %rax, %rcx
+; AVX2-NEXT: shll $2, %ecx
+; AVX2-NEXT: vmovss %xmm0, (%rsp,%rcx)
; AVX2-NEXT: vmovaps (%rsp), %ymm0
; AVX2-NEXT: vmovaps {{[0-9]+}}(%rsp), %ymm1
; AVX2-NEXT: movq %rbp, %rsp
@@ -918,19 +922,19 @@ define <8 x i64> @test_compress_v8i64(<8 x i64> %vec, <8 x i1> %mask, <8 x i64>
; AVX2-NEXT: vmovd %xmm2, %esi
; AVX2-NEXT: andl $1, %esi
; AVX2-NEXT: addq %rsi, %rdx
-; AVX2-NEXT: vpextrw $2, %xmm2, %r8d
+; AVX2-NEXT: vpextrw $2, %xmm2, %edi
+; AVX2-NEXT: andl $1, %edi
+; AVX2-NEXT: addq %rdx, %rdi
+; AVX2-NEXT: vpextrw $3, %xmm2, %r8d
; AVX2-NEXT: andl $1, %r8d
-; AVX2-NEXT: addq %rdx, %r8
-; AVX2-NEXT: vpextrw $3, %xmm2, %r9d
+; AVX2-NEXT: addq %rdi, %r8
+; AVX2-NEXT: addq %r8, %rax
+; AVX2-NEXT: vpextrw $5, %xmm2, %r9d
; AVX2-NEXT: andl $1, %r9d
-; AVX2-NEXT: addq %r8, %r9
-; AVX2-NEXT: addq %r9, %rax
-; AVX2-NEXT: vpextrw $5, %xmm2, %edi
-; AVX2-NEXT: andl $1, %edi
-; AVX2-NEXT: addq %rax, %rdi
+; AVX2-NEXT: addq %rax, %r9
; AVX2-NEXT: vpextrw $6, %xmm2, %r10d
; AVX2-NEXT: andl $1, %r10d
-; AVX2-NEXT: addq %rdi, %r10
+; AVX2-NEXT: addq %r9, %r10
; AVX2-NEXT: vpextrw $7, %xmm2, %r11d
; AVX2-NEXT: andl $1, %r11d
; AVX2-NEXT: addq %r10, %r11
@@ -942,13 +946,13 @@ define <8 x i64> @test_compress_v8i64(<8 x i64> %vec, <8 x i1> %mask, <8 x i64>
; AVX2-NEXT: vpextrq $1, %xmm0, (%rsp,%rsi,8)
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0
; AVX2-NEXT: vmovq %xmm0, (%rsp,%rdx,8)
-; AVX2-NEXT: vpextrq $1, %xmm0, (%rsp,%r8,8)
-; AVX2-NEXT: andl $7, %r9d
-; AVX2-NEXT: vmovq %xmm1, (%rsp,%r9,8)
+; AVX2-NEXT: vpextrq $1, %xmm0, (%rsp,%rdi,8)
+; AVX2-NEXT: andl $7, %r8d
+; AVX2-NEXT: vmovq %xmm1, (%rsp,%r8,8)
; AVX2-NEXT: andl $7, %eax
; AVX2-NEXT: vpextrq $1, %xmm1, (%rsp,%rax,8)
-; AVX2-NEXT: andl $7, %edi
-; AVX2-NEXT: vmovq %xmm2, (%rsp,%rdi,8)
+; AVX2-NEXT: andl $7, %r9d
+; AVX2-NEXT: vmovq %xmm2, (%rsp,%r9,8)
; AVX2-NEXT: andl $7, %r10d
; AVX2-NEXT: vpextrq $1, %xmm2, (%rsp,%r10,8)
; AVX2-NEXT: cmpq $7, %r11
@@ -1032,7 +1036,8 @@ define <8 x double> @test_compress_v8f64(<8 x double> %vec, <8 x i1> %mask, <8 x
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $7, %eax
; AVX2-NEXT: vmovhpd %xmm1, (%rsp,%rax,8)
-; AVX2-NEXT: vpextrw $6, %xmm2, %edx
+; AVX2-NEXT: vpextrw $6, %xmm2, %eax
+; AVX2-NEXT: movq %rax, %rdx
; AVX2-NEXT: andl $1, %edx
; AVX2-NEXT: addq %rcx, %rdx
; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
@@ -1103,14 +1108,14 @@ define <16 x i8> @test_compress_v16i8(<16 x i8> %vec, <16 x i1> %mask, <16 x i8>
; AVX2-NEXT: subb %sil, %al
; AVX2-NEXT: vpextrb $3, %xmm1, %ebp
; AVX2-NEXT: subb %bpl, %al
-; AVX2-NEXT: vpextrb $4, %xmm1, %r12d
-; AVX2-NEXT: subb %r12b, %al
-; AVX2-NEXT: vpextrb $5, %xmm1, %r15d
-; AVX2-NEXT: vpextrb $6, %xmm1, %r14d
+; AVX2-NEXT: vpextrb $4, %xmm1, %r15d
; AVX2-NEXT: subb %r15b, %al
-; AVX2-NEXT: subb %r14b, %al
-; AVX2-NEXT: vpextrb $7, %xmm1, %ebx
+; AVX2-NEXT: vpextrb $5, %xmm1, %ebx
+; AVX2-NEXT: vpextrb $6, %xmm1, %r12d
; AVX2-NEXT: subb %bl, %al
+; AVX2-NEXT: subb %r12b, %al
+; AVX2-NEXT: vpextrb $7, %xmm1, %r14d
+; AVX2-NEXT: subb %r14b, %al
; AVX2-NEXT: vpextrb $8, %xmm1, %r11d
; AVX2-NEXT: subb %r11b, %al
; AVX2-NEXT: vpextrb $9, %xmm1, %r10d
@@ -1146,28 +1151,28 @@ define <16 x i8> @test_compress_v16i8(<16 x i8> %vec, <16 x i1> %mask, <16 x i8>
; AVX2-NEXT: andl $1, %ebp
; AVX2-NEXT: addq %rsi, %rbp
; AVX2-NEXT: vpextrb $4, %xmm0, -56(%rsp,%rbp)
-; AVX2-NEXT: andl $1, %r12d
-; AVX2-NEXT: addq %rbp, %r12
-; AVX2-NEXT: movl %r12d, %eax
-; AVX2-NEXT: andl $15, %eax
-; AVX2-NEXT: vpextrb $5, %xmm0, -56(%rsp,%rax)
; AVX2-NEXT: andl $1, %r15d
-; AVX2-NEXT: addq %r12, %r15
-; AVX2-NEXT: andl $1, %r14d
-; AVX2-NEXT: addq %r15, %r14
-; AVX2-NEXT: # kill: def $r15d killed $r15d killed $r15 def $r15
-; AVX2-NEXT: andl $15, %r15d
-; AVX2-NEXT: vpextrb $6, %xmm0, -56(%rsp,%r15)
-; AVX2-NEXT: movl %r14d, %eax
+; AVX2-NEXT: addq %rbp, %r15
+; AVX2-NEXT: movl %r15d, %eax
; AVX2-NEXT: andl $15, %eax
-; AVX2-NEXT: vpextrb $7, %xmm0, -56(%rsp,%rax)
+; AVX2-NEXT: vpextrb $5, %xmm0, -56(%rsp,%rax)
; AVX2-NEXT: andl $1, %ebx
-; AVX2-NEXT: addq %r14, %rbx
-; AVX2-NEXT: andl $1, %r11d
-; AVX2-NEXT: addq %rbx, %r11
+; AVX2-NEXT: addq %r15, %rbx
+; AVX2-NEXT: andl $1, %r12d
+; AVX2-NEXT: addq %rbx, %r12
; AVX2-NEXT: # kill: def $ebx killed $ebx killed $rbx def $rbx
; AVX2-NEXT: andl $15, %ebx
-; AVX2-NEXT: vpextrb $8, %xmm0, -56(%rsp,%rbx)
+; AVX2-NEXT: vpextrb $6, %xmm0, -56(%rsp,%rbx)
+; AVX2-NEXT: movl %r12d, %eax
+; AVX2-NEXT: andl $15, %eax
+; AVX2-NEXT: vpextrb $7, %xmm0, -56(%rsp,%rax)
+; AVX2-NEXT: andl $1, %r14d
+; AVX2-NEXT: addq %r12, %r14
+; AVX2-NEXT: andl $1, %r11d
+; AVX2-NEXT: addq %r14, %r11
+; AVX2-NEXT: # kill: def $r14d killed $r14d killed $r14 def $r14
+; AVX2-NEXT: andl $15, %r14d
+; AVX2-NEXT: vpextrb $8, %xmm0, -56(%rsp,%r14)
; AVX2-NEXT: movl %r11d, %eax
; AVX2-NEXT: andl $15, %eax
; AVX2-NEXT: vpextrb $9, %xmm0, -56(%rsp,%rax)
@@ -1183,36 +1188,36 @@ define <16 x i8> @test_compress_v16i8(<16 x i8> %vec, <16 x i1> %mask, <16 x i8>
; AVX2-NEXT: vpextrb $11, %xmm0, -56(%rsp,%rax)
; AVX2-NEXT: andl $1, %r8d
; AVX2-NEXT: addq %r9, %r8
-; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %r8, %rcx
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
+; AVX2-NEXT: andl $1, %edi
+; AVX2-NEXT: addq %r8, %rdi
; AVX2-NEXT: # kill: def $r8d killed $r8d killed $r8 def $r8
; AVX2-NEXT: andl $15, %r8d
; AVX2-NEXT: vpextrb $12, %xmm0, -56(%rsp,%r8)
-; AVX2-NEXT: movl %ecx, %eax
+; AVX2-NEXT: movl %edi, %eax
; AVX2-NEXT: andl $15, %eax
; AVX2-NEXT: vpextrb $13, %xmm0, -56(%rsp,%rax)
-; AVX2-NEXT: andl $1, %edx
-; AVX2-NEXT: addq %rcx, %rdx
-; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rdx, %rcx
-; AVX2-NEXT: # kill: def $edx killed $edx killed $rdx def $rdx
-; AVX2-NEXT: andl $15, %edx
-; AVX2-NEXT: vpextrb $14, %xmm0, -56(%rsp,%rdx)
-; AVX2-NEXT: movl %ecx, %eax
-; AVX2-NEXT: andl $15, %eax
-; AVX2-NEXT: vpextrb $15, %xmm0, -56(%rsp,%rax)
+; AVX2-NEXT: movq %rdx, %rsi
+; AVX2-NEXT: andl $1, %esi
+; AVX2-NEXT: addq %rdi, %rsi
; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; AVX2-NEXT: andl $1, %eax
-; AVX2-NEXT: addq %rcx, %rax
-; AVX2-NEXT: cmpq $15, %rax
-; AVX2-NEXT: movq %rax, %rdx
+; AVX2-NEXT: addq %rsi, %rax
+; AVX2-NEXT: # kill: def $esi killed $esi killed $rsi def $rsi
+; AVX2-NEXT: andl $15, %esi
+; AVX2-NEXT: vpextrb $14, %xmm0, -56(%rsp,%rsi)
+; AVX2-NEXT: movl %eax, %ecx
+; AVX2-NEXT: andl $15, %ecx
+; AVX2-NEXT: vpextrb $15, %xmm0, -56(%rsp,%rcx)
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; AVX2-NEXT: andl $1, %ecx
+; AVX2-NEXT: addq %rax, %rcx
+; AVX2-NEXT: cmpq $15, %rcx
; AVX2-NEXT: vpextrb $15, %xmm0, %eax
; AVX2-NEXT: cmovbel {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Folded Reload
-; AVX2-NEXT: movl $15, %ecx
-; AVX2-NEXT: cmovbq %rdx, %rcx
-; AVX2-NEXT: movb %al, -56(%rsp,%rcx)
+; AVX2-NEXT: movl $15, %edx
+; AVX2-NEXT: cmovbq %rcx, %rdx
+; AVX2-NEXT: movb %al, -56(%rsp,%rdx)
; AVX2-NEXT: vmovaps -{{[0-9]+}}(%rsp), %xmm0
; AVX2-NEXT: popq %rbx
; AVX2-NEXT: popq %r12
@@ -1258,7 +1263,6 @@ define <16 x i8> @test_compress_v16i8(<16 x i8> %vec, <16 x i1> %mask, <16 x i8>
define <8 x i16> @test_compress_v8i16(<8 x i16> %vec, <8 x i1> %mask, <8 x i16> %passthru) nounwind {
; AVX2-LABEL: test_compress_v8i16:
; AVX2: # %bb.0:
-; AVX2-NEXT: pushq %rbp
; AVX2-NEXT: pushq %r14
; AVX2-NEXT: pushq %rbx
; AVX2-NEXT: vpsllw $15, %xmm1, %xmm1
@@ -1266,43 +1270,43 @@ define <8 x i16> @test_compress_v8i16(<8 x i16> %vec, <8 x i1> %mask, <8 x i16>
; AVX2-NEXT: vmovaps %xmm2, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vpextrw $1, %xmm1, %r9d
; AVX2-NEXT: andl $1, %r9d
-; AVX2-NEXT: vmovd %xmm1, %esi
-; AVX2-NEXT: andl $1, %esi
-; AVX2-NEXT: leaq (%rsi,%r9), %rcx
+; AVX2-NEXT: vmovd %xmm1, %edx
+; AVX2-NEXT: andl $1, %edx
+; AVX2-NEXT: leaq (%rdx,%r9), %rcx
; AVX2-NEXT: vpextrw $2, %xmm1, %r10d
-; AVX2-NEXT: movl %r10d, %eax
+; AVX2-NEXT: movq %r10, %rax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
; AVX2-NEXT: vpextrw $3, %xmm1, %r8d
-; AVX2-NEXT: movl %r8d, %edx
-; AVX2-NEXT: andl $1, %edx
-; AVX2-NEXT: addq %rax, %rdx
+; AVX2-NEXT: movq %r8, %rsi
+; AVX2-NEXT: andl $1, %esi
+; AVX2-NEXT: addq %rax, %rsi
; AVX2-NEXT: vpextrw $4, %xmm1, %r11d
-; AVX2-NEXT: movl %r11d, %edi
+; AVX2-NEXT: movq %r11, %rdi
; AVX2-NEXT: andl $1, %edi
-; AVX2-NEXT: addq %rdx, %rdi
-; AVX2-NEXT: movl %esi, %ebx
+; AVX2-NEXT: addq %rsi, %rdi
+; AVX2-NEXT: movl %edx, %ebx
; AVX2-NEXT: addl %r9d, %ebx
-; AVX2-NEXT: vpextrw $5, %xmm1, %ebp
+; AVX2-NEXT: vpextrw $5, %xmm1, %r14d
; AVX2-NEXT: andl $1, %r10d
; AVX2-NEXT: andl $1, %r8d
; AVX2-NEXT: addl %r10d, %r8d
-; AVX2-NEXT: movl %ebp, %r9d
+; AVX2-NEXT: movq %r14, %r9
; AVX2-NEXT: andl $1, %r9d
; AVX2-NEXT: addq %rdi, %r9
; AVX2-NEXT: addl %ebx, %r8d
; AVX2-NEXT: vpextrw $6, %xmm1, %ebx
; AVX2-NEXT: andl $1, %r11d
-; AVX2-NEXT: andl $1, %ebp
-; AVX2-NEXT: addl %r11d, %ebp
-; AVX2-NEXT: movl %ebx, %r10d
+; AVX2-NEXT: andl $1, %r14d
+; AVX2-NEXT: addl %r11d, %r14d
+; AVX2-NEXT: movq %rbx, %r10
; AVX2-NEXT: andl $1, %r10d
; AVX2-NEXT: addq %r9, %r10
; AVX2-NEXT: andl $1, %ebx
-; AVX2-NEXT: addl %ebp, %ebx
+; AVX2-NEXT: addl %r14d, %ebx
; AVX2-NEXT: vpextrw $7, %xmm1, %r14d
; AVX2-NEXT: addl %r8d, %ebx
-; AVX2-NEXT: movl %r14d, %r8d
+; AVX2-NEXT: movq %r14, %r8
; AVX2-NEXT: andl $1, %r8d
; AVX2-NEXT: addq %r10, %r8
; AVX2-NEXT: andl $1, %r14d
@@ -1310,30 +1314,29 @@ define <8 x i16> @test_compress_v8i16(<8 x i16> %vec, <8 x i1> %mask, <8 x i16>
; AVX2-NEXT: andl $7, %r14d
; AVX2-NEXT: vpextrw $7, %xmm0, %r11d
; AVX2-NEXT: cmpq $8, %r8
-; AVX2-NEXT: cmovbw -16(%rsp,%r14,2), %r11w
+; AVX2-NEXT: cmovbw -24(%rsp,%r14,2), %r11w
; AVX2-NEXT: vpextrw $0, %xmm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vpextrw $1, %xmm0, -16(%rsp,%rsi,2)
+; AVX2-NEXT: vpextrw $1, %xmm0, -24(%rsp,%rdx,2)
; AVX2-NEXT: andl $7, %ecx
-; AVX2-NEXT: vpextrw $2, %xmm0, -16(%rsp,%rcx,2)
+; AVX2-NEXT: vpextrw $2, %xmm0, -24(%rsp,%rcx,2)
; AVX2-NEXT: andl $7, %eax
-; AVX2-NEXT: vpextrw $3, %xmm0, -16(%rsp,%rax,2)
-; AVX2-NEXT: andl $7, %edx
-; AVX2-NEXT: vpextrw $4, %xmm0, -16(%rsp,%rdx,2)
+; AVX2-NEXT: vpextrw $3, %xmm0, -24(%rsp,%rax,2)
+; AVX2-NEXT: andl $7, %esi
+; AVX2-NEXT: vpextrw $4, %xmm0, -24(%rsp,%rsi,2)
; AVX2-NEXT: andl $7, %edi
-; AVX2-NEXT: vpextrw $5, %xmm0, -16(%rsp,%rdi,2)
+; AVX2-NEXT: vpextrw $5, %xmm0, -24(%rsp,%rdi,2)
; AVX2-NEXT: andl $7, %r9d
-; AVX2-NEXT: vpextrw $6, %xmm0, -16(%rsp,%r9,2)
+; AVX2-NEXT: vpextrw $6, %xmm0, -24(%rsp,%r9,2)
; AVX2-NEXT: andl $7, %r10d
-; AVX2-NEXT: vpextrw $7, %xmm0, -16(%rsp,%r10,2)
+; AVX2-NEXT: vpextrw $7, %xmm0, -24(%rsp,%r10,2)
; AVX2-NEXT: cmpq $7, %r8
; AVX2-NEXT: movl $7, %eax
; AVX2-NEXT: cmovbq %r8, %rax
; AVX2-NEXT: andl $7, %eax
-; AVX2-NEXT: movw %r11w, -16(%rsp,%rax,2)
+; AVX2-NEXT: movw %r11w, -24(%rsp,%rax,2)
; AVX2-NEXT: vmovaps -{{[0-9]+}}(%rsp), %xmm0
; AVX2-NEXT: popq %rbx
; AVX2-NEXT: popq %r14
-; AVX2-NEXT: popq %rbp
; AVX2-NEXT: retq
;
; AVX512F-LABEL: test_compress_v8i16:
@@ -1970,10 +1973,11 @@ define <16 x i16> @test_compress_v16i16(<16 x i16> %vec, <16 x i1> %mask, <16 x
; AVX2-NEXT: vpextrw $1, %xmm2, %eax
; AVX2-NEXT: vmovd %xmm2, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: vpextrw $6, %xmm1, %edx
+; AVX2-NEXT: vpextrw $6, %xmm1, %esi
; AVX2-NEXT: andl $15, %ecx
; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: andl $1, %edx
+; AVX2-NEXT: movq %rsi, %r8
+; AVX2-NEXT: andl $1, %r8d
; AVX2-NEXT: vpextrw $1, %xmm1, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: vmovd %xmm1, %ecx
@@ -1985,24 +1989,24 @@ define <16 x i16> @test_compress_v16i16(<16 x i16> %vec, <16 x i1> %mask, <16 x
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: vpextrw $3, %xmm1, %r8d
-; AVX2-NEXT: andl $1, %r8d
-; AVX2-NEXT: addq %rcx, %r8
-; AVX2-NEXT: vpextrw $4, %xmm1, %r10d
+; AVX2-NEXT: vpextrw $3, %xmm1, %eax
+; AVX2-NEXT: andl $1, %eax
+; AVX2-NEXT: addq %rcx, %rax
+; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: vpextrw $4, %xmm1, %r9d
+; AVX2-NEXT: andl $1, %r9d
+; AVX2-NEXT: addq %rax, %r9
+; AVX2-NEXT: vpextrw $5, %xmm1, %r10d
; AVX2-NEXT: andl $1, %r10d
-; AVX2-NEXT: addq %r8, %r10
-; AVX2-NEXT: vpextrw $5, %xmm1, %r11d
+; AVX2-NEXT: addq %r9, %r10
+; AVX2-NEXT: addq %r10, %r8
+; AVX2-NEXT: vpextrw $7, %xmm1, %r11d
; AVX2-NEXT: andl $1, %r11d
-; AVX2-NEXT: addq %r10, %r11
-; AVX2-NEXT: addq %r11, %rdx
-; AVX2-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: vpextrw $7, %xmm1, %r9d
-; AVX2-NEXT: andl $1, %r9d
-; AVX2-NEXT: addq %rdx, %r9
+; AVX2-NEXT: addq %r8, %r11
; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm1
; AVX2-NEXT: vmovd %xmm1, %ebx
; AVX2-NEXT: andl $1, %ebx
-; AVX2-NEXT: addq %r9, %rbx
+; AVX2-NEXT: addq %r11, %rbx
; AVX2-NEXT: vpextrw $1, %xmm1, %r14d
; AVX2-NEXT: andl $1, %r14d
; AVX2-NEXT: addq %rbx, %r14
@@ -2015,18 +2019,20 @@ define <16 x i16> @test_compress_v16i16(<16 x i16> %vec, <16 x i1> %mask, <16 x
; AVX2-NEXT: vpextrw $4, %xmm1, %r13d
; AVX2-NEXT: andl $1, %r13d
; AVX2-NEXT: addq %r12, %r13
-; AVX2-NEXT: vpextrw $5, %xmm1, %edx
-; AVX2-NEXT: andl $1, %edx
-; AVX2-NEXT: addq %r13, %rdx
-; AVX2-NEXT: vpextrw $6, %xmm1, %ecx
-; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rdx, %rcx
-; AVX2-NEXT: vpextrw $7, %xmm1, %esi
+; AVX2-NEXT: vpextrw $5, %xmm1, %ecx
+; AVX2-NEXT: movq %rcx, %rsi
; AVX2-NEXT: andl $1, %esi
-; AVX2-NEXT: addq %rcx, %rsi
+; AVX2-NEXT: addq %r13, %rsi
+; AVX2-NEXT: vpextrw $6, %xmm1, %eax
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: andl $1, %ecx
+; AVX2-NEXT: addq %rsi, %rcx
+; AVX2-NEXT: vpextrw $7, %xmm1, %edx
+; AVX2-NEXT: andl $1, %edx
+; AVX2-NEXT: addq %rcx, %rdx
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX2-NEXT: vpextrw $7, %xmm1, %eax
-; AVX2-NEXT: cmpq $16, %rsi
+; AVX2-NEXT: cmpq $16, %rdx
; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
; AVX2-NEXT: cmovbw (%rsp,%rdi,2), %ax
; AVX2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
@@ -2037,16 +2043,16 @@ define <16 x i16> @test_compress_v16i16(<16 x i16> %vec, <16 x i1> %mask, <16 x
; AVX2-NEXT: vpextrw $2, %xmm0, (%rsp,%rdi,2)
; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
; AVX2-NEXT: vpextrw $3, %xmm0, (%rsp,%rdi,2)
-; AVX2-NEXT: vpextrw $4, %xmm0, (%rsp,%r8,2)
-; AVX2-NEXT: andl $15, %r10d
-; AVX2-NEXT: vpextrw $5, %xmm0, (%rsp,%r10,2)
-; AVX2-NEXT: andl $15, %r11d
-; AVX2-NEXT: vpextrw $6, %xmm0, (%rsp,%r11,2)
; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; AVX2-NEXT: andl $15, %eax
-; AVX2-NEXT: vpextrw $7, %xmm0, (%rsp,%rax,2)
+; AVX2-NEXT: vpextrw $4, %xmm0, (%rsp,%rax,2)
; AVX2-NEXT: andl $15, %r9d
-; AVX2-NEXT: vpextrw $0, %xmm1, (%rsp,%r9,2)
+; AVX2-NEXT: vpextrw $5, %xmm0, (%rsp,%r9,2)
+; AVX2-NEXT: andl $15, %r10d
+; AVX2-NEXT: vpextrw $6, %xmm0, (%rsp,%r10,2)
+; AVX2-NEXT: andl $15, %r8d
+; AVX2-NEXT: vpextrw $7, %xmm0, (%rsp,%r8,2)
+; AVX2-NEXT: andl $15, %r11d
+; AVX2-NEXT: vpextrw $0, %xmm1, (%rsp,%r11,2)
; AVX2-NEXT: andl $15, %ebx
; AVX2-NEXT: vpextrw $1, %xmm1, (%rsp,%rbx,2)
; AVX2-NEXT: andl $15, %r14d
@@ -2057,13 +2063,13 @@ define <16 x i16> @test_compress_v16i16(<16 x i16> %vec, <16 x i1> %mask, <16 x
; AVX2-NEXT: vpextrw $4, %xmm1, (%rsp,%r12,2)
; AVX2-NEXT: andl $15, %r13d
; AVX2-NEXT: vpextrw $5, %xmm1, (%rsp,%r13,2)
-; AVX2-NEXT: andl $15, %edx
-; AVX2-NEXT: vpextrw $6, %xmm1, (%rsp,%rdx,2)
+; AVX2-NEXT: andl $15, %esi
+; AVX2-NEXT: vpextrw $6, %xmm1, (%rsp,%rsi,2)
; AVX2-NEXT: andl $15, %ecx
; AVX2-NEXT: vpextrw $7, %xmm1, (%rsp,%rcx,2)
-; AVX2-NEXT: cmpq $15, %rsi
+; AVX2-NEXT: cmpq $15, %rdx
; AVX2-NEXT: movl $15, %eax
-; AVX2-NEXT: cmovbq %rsi, %rax
+; AVX2-NEXT: cmovbq %rdx, %rax
; AVX2-NEXT: andl $15, %eax
; AVX2-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %ecx # 4-byte Reload
; AVX2-NEXT: movw %cx, (%rsp,%rax,2)
@@ -3378,123 +3384,149 @@ define <32 x i16> @test_compress_v32i16(<32 x i16> %vec, <32 x i1> %mask, <32 x
; AVX2-NEXT: andl $31, %ecx
; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: vpextrb $4, %xmm2, %eax
-; AVX2-NEXT: vpextrb $1, %xmm2, %ecx
+; AVX2-NEXT: vpextrb $1, %xmm2, %edx
+; AVX2-NEXT: andl $1, %eax
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: movq %rdx, %rax
; AVX2-NEXT: andl $1, %eax
-; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: vmovd %xmm2, %edx
; AVX2-NEXT: andl $1, %edx
; AVX2-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: addq %rdx, %rcx
-; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: addq %rdx, %rax
+; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: vpextrb $2, %xmm2, %edx
; AVX2-NEXT: andl $1, %edx
-; AVX2-NEXT: addq %rcx, %rdx
+; AVX2-NEXT: addq %rax, %rdx
; AVX2-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: vpextrb $3, %xmm2, %ecx
-; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rdx, %rcx
-; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: addq %rcx, %rax
+; AVX2-NEXT: vpextrb $3, %xmm2, %eax
+; AVX2-NEXT: andl $1, %eax
+; AVX2-NEXT: addq %rdx, %rax
; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: vpextrb $5, %xmm2, %ecx
-; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: vpextrb $5, %xmm2, %eax
+; AVX2-NEXT: andl $1, %eax
+; AVX2-NEXT: addq %rcx, %rax
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: vpextrb $6, %xmm2, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: vpextrb $7, %xmm2, %eax
+; AVX2-NEXT: andl $1, %eax
+; AVX2-NEXT: addq %rcx, %rax
+; AVX2-NEXT: movq %rax, %rcx
; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: vpextrb $7, %xmm2, %ecx
-; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rax, %rcx
-; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: vpextrb $8, %xmm2, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: vpextrb $9, %xmm2, %eax
+; AVX2-NEXT: andl $1, %eax
+; AVX2-NEXT: addq %rcx, %rax
+; AVX2-NEXT: movq %rax, %rcx
; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: vpextrb $9, %xmm2, %ecx
-; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rax, %rcx
-; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: vpextrb $10, %xmm2, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: vpextrb $11, %xmm2, %eax
+; AVX2-NEXT: andl $1, %eax
+; AVX2-NEXT: addq %rcx, %rax
+; AVX2-NEXT: movq %rax, %rcx
; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: vpextrb $11, %xmm2, %ecx
-; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rax, %rcx
-; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: vpextrb $12, %xmm2, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: vpextrb $13, %xmm2, %eax
+; AVX2-NEXT: andl $1, %eax
+; AVX2-NEXT: addq %rcx, %rax
+; AVX2-NEXT: movq %rax, %rcx
; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: vpextrb $13, %xmm2, %ecx
-; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rax, %rcx
-; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: vpextrb $14, %xmm2, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: vpextrb $15, %xmm2, %eax
+; AVX2-NEXT: andl $1, %eax
+; AVX2-NEXT: addq %rcx, %rax
+; AVX2-NEXT: movq %rax, %rcx
; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: vpextrb $15, %xmm2, %ecx
-; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rax, %rcx
-; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: vmovd %xmm3, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
+; AVX2-NEXT: movq %rax, %rcx
; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: vpextrb $1, %xmm3, %ecx
-; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rax, %rcx
-; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: vpextrb $2, %xmm3, %eax
+; AVX2-NEXT: vpextrb $1, %xmm3, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: vpextrb $3, %xmm3, %ecx
-; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rax, %rcx
-; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: vpextrb $2, %xmm3, %r13d
+; AVX2-NEXT: andl $1, %r13d
+; AVX2-NEXT: addq %rax, %r13
+; AVX2-NEXT: movq %r13, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: vpextrb $3, %xmm3, %r12d
+; AVX2-NEXT: andl $1, %r12d
+; AVX2-NEXT: addq %r13, %r12
+; AVX2-NEXT: movq %r12, %rax
+; AVX2-NEXT: movq %r12, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: vpextrb $4, %xmm3, %r15d
-; AVX2-NEXT: andl $1, %r15d
-; AVX2-NEXT: addq %rcx, %r15
+; AVX2-NEXT: movq %r15, %r12
+; AVX2-NEXT: andl $1, %r12d
+; AVX2-NEXT: addq %rax, %r12
; AVX2-NEXT: vpextrb $5, %xmm3, %r14d
+; AVX2-NEXT: movq %r14, %r15
+; AVX2-NEXT: andl $1, %r15d
+; AVX2-NEXT: addq %r12, %r15
+; AVX2-NEXT: vpextrb $6, %xmm3, %ebx
+; AVX2-NEXT: movq %rbx, %r14
; AVX2-NEXT: andl $1, %r14d
; AVX2-NEXT: addq %r15, %r14
-; AVX2-NEXT: vpextrb $6, %xmm3, %ebx
+; AVX2-NEXT: vpextrb $7, %xmm3, %r11d
+; AVX2-NEXT: movq %r11, %rbx
; AVX2-NEXT: andl $1, %ebx
; AVX2-NEXT: addq %r14, %rbx
-; AVX2-NEXT: vpextrb $7, %xmm3, %r11d
+; AVX2-NEXT: vpextrb $8, %xmm3, %r10d
+; AVX2-NEXT: movq %r10, %r11
; AVX2-NEXT: andl $1, %r11d
; AVX2-NEXT: addq %rbx, %r11
-; AVX2-NEXT: vpextrb $8, %xmm3, %r10d
+; AVX2-NEXT: vpextrb $9, %xmm3, %r9d
+; AVX2-NEXT: movq %r9, %r10
; AVX2-NEXT: andl $1, %r10d
; AVX2-NEXT: addq %r11, %r10
-; AVX2-NEXT: vpextrb $9, %xmm3, %r9d
+; AVX2-NEXT: vpextrb $10, %xmm3, %edi
+; AVX2-NEXT: movq %rdi, %r9
; AVX2-NEXT: andl $1, %r9d
; AVX2-NEXT: addq %r10, %r9
-; AVX2-NEXT: vpextrb $10, %xmm3, %r8d
-; AVX2-NEXT: andl $1, %r8d
-; AVX2-NEXT: addq %r9, %r8
-; AVX2-NEXT: vpextrb $11, %xmm3, %edi
+; AVX2-NEXT: vpextrb $11, %xmm3, %esi
+; AVX2-NEXT: movq %rsi, %rdi
; AVX2-NEXT: andl $1, %edi
-; AVX2-NEXT: addq %r8, %rdi
-; AVX2-NEXT: vpextrb $12, %xmm3, %esi
+; AVX2-NEXT: addq %r9, %rdi
+; AVX2-NEXT: vpextrb $12, %xmm3, %edx
+; AVX2-NEXT: movq %rdx, %rsi
; AVX2-NEXT: andl $1, %esi
; AVX2-NEXT: addq %rdi, %rsi
-; AVX2-NEXT: vpextrb $13, %xmm3, %edx
+; AVX2-NEXT: vpextrb $13, %xmm3, %ecx
+; AVX2-NEXT: movq %rcx, %rdx
; AVX2-NEXT: andl $1, %edx
; AVX2-NEXT: addq %rsi, %rdx
-; AVX2-NEXT: vpextrb $14, %xmm3, %ecx
+; AVX2-NEXT: vpextrb $14, %xmm3, %eax
+; AVX2-NEXT: movq %rax, %rcx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rdx, %rcx
-; AVX2-NEXT: vpextrb $15, %xmm3, %r12d
-; AVX2-NEXT: andl $1, %r12d
-; AVX2-NEXT: addq %rcx, %r12
+; AVX2-NEXT: vpextrb $15, %xmm3, %r8d
+; AVX2-NEXT: andl $1, %r8d
+; AVX2-NEXT: addq %rcx, %r8
; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
; AVX2-NEXT: vpextrw $7, %xmm2, %eax
-; AVX2-NEXT: cmpq $32, %r12
+; AVX2-NEXT: cmpq $32, %r8
; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Reload
; AVX2-NEXT: cmovbw (%rsp,%r13,2), %ax
; AVX2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
@@ -3557,20 +3589,20 @@ define <32 x i16> @test_compress_v32i16(<32 x i16> %vec, <32 x i1> %mask, <32 x
; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; AVX2-NEXT: andl $31, %eax
; AVX2-NEXT: vpextrw $4, %xmm1, (%rsp,%rax,2)
+; AVX2-NEXT: andl $31, %r12d
+; AVX2-NEXT: vpextrw $5, %xmm1, (%rsp,%r12,2)
; AVX2-NEXT: andl $31, %r15d
-; AVX2-NEXT: vpextrw $5, %xmm1, (%rsp,%r15,2)
+; AVX2-NEXT: vpextrw $6, %xmm1, (%rsp,%r15,2)
; AVX2-NEXT: andl $31, %r14d
-; AVX2-NEXT: vpextrw $6, %xmm1, (%rsp,%r14,2)
+; AVX2-NEXT: vpextrw $7, %xmm1, (%rsp,%r14,2)
; AVX2-NEXT: andl $31, %ebx
-; AVX2-NEXT: vpextrw $7, %xmm1, (%rsp,%rbx,2)
+; AVX2-NEXT: vpextrw $0, %xmm2, (%rsp,%rbx,2)
; AVX2-NEXT: andl $31, %r11d
-; AVX2-NEXT: vpextrw $0, %xmm2, (%rsp,%r11,2)
+; AVX2-NEXT: vpextrw $1, %xmm2, (%rsp,%r11,2)
; AVX2-NEXT: andl $31, %r10d
-; AVX2-NEXT: vpextrw $1, %xmm2, (%rsp,%r10,2)
+; AVX2-NEXT: vpextrw $2, %xmm2, (%rsp,%r10,2)
; AVX2-NEXT: andl $31, %r9d
-; AVX2-NEXT: vpextrw $2, %xmm2, (%rsp,%r9,2)
-; AVX2-NEXT: andl $31, %r8d
-; AVX2-NEXT: vpextrw $3, %xmm2, (%rsp,%r8,2)
+; AVX2-NEXT: vpextrw $3, %xmm2, (%rsp,%r9,2)
; AVX2-NEXT: andl $31, %edi
; AVX2-NEXT: vpextrw $4, %xmm2, (%rsp,%rdi,2)
; AVX2-NEXT: andl $31, %esi
@@ -3579,9 +3611,9 @@ define <32 x i16> @test_compress_v32i16(<32 x i16> %vec, <32 x i1> %mask, <32 x
; AVX2-NEXT: vpextrw $6, %xmm2, (%rsp,%rdx,2)
; AVX2-NEXT: andl $31, %ecx
; AVX2-NEXT: vpextrw $7, %xmm2, (%rsp,%rcx,2)
-; AVX2-NEXT: cmpq $31, %r12
+; AVX2-NEXT: cmpq $31, %r8
; AVX2-NEXT: movl $31, %eax
-; AVX2-NEXT: cmovbq %r12, %rax
+; AVX2-NEXT: cmovbq %r8, %rax
; AVX2-NEXT: addl %eax, %eax
; AVX2-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %ecx # 4-byte Reload
; AVX2-NEXT: movw %cx, (%rsp,%rax)
@@ -4607,19 +4639,19 @@ define <8 x i64> @test_compress_knownbits_zext_v8i16_8i64(<8 x i16> %vec, <8 x
; AVX2-NEXT: vmovd %xmm1, %esi
; AVX2-NEXT: andl $1, %esi
; AVX2-NEXT: addq %rsi, %rdx
-; AVX2-NEXT: vpextrw $2, %xmm1, %r8d
+; AVX2-NEXT: vpextrw $2, %xmm1, %edi
+; AVX2-NEXT: andl $1, %edi
+; AVX2-NEXT: addq %rdx, %rdi
+; AVX2-NEXT: vpextrw $3, %xmm1, %r8d
; AVX2-NEXT: andl $1, %r8d
-; AVX2-NEXT: addq %rdx, %r8
-; AVX2-NEXT: vpextrw $3, %xmm1, %r9d
+; AVX2-NEXT: addq %rdi, %r8
+; AVX2-NEXT: addq %r8, %rax
+; AVX2-NEXT: vpextrw $5, %xmm1, %r9d
; AVX2-NEXT: andl $1, %r9d
-; AVX2-NEXT: addq %r8, %r9
-; AVX2-NEXT: addq %r9, %rax
-; AVX2-NEXT: vpextrw $5, %xmm1, %edi
-; AVX2-NEXT: andl $1, %edi
-; AVX2-NEXT: addq %rax, %rdi
+; AVX2-NEXT: addq %rax, %r9
; AVX2-NEXT: vpextrw $6, %xmm1, %r10d
; AVX2-NEXT: andl $1, %r10d
-; AVX2-NEXT: addq %rdi, %r10
+; AVX2-NEXT: addq %r9, %r10
; AVX2-NEXT: vpextrw $7, %xmm1, %r11d
; AVX2-NEXT: andl $1, %r11d
; AVX2-NEXT: addq %r10, %r11
@@ -4632,13 +4664,13 @@ define <8 x i64> @test_compress_knownbits_zext_v8i16_8i64(<8 x i16> %vec, <8 x
; AVX2-NEXT: vpextrq $1, %xmm0, (%rsp,%rsi,8)
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0
; AVX2-NEXT: vmovq %xmm0, (%rsp,%rdx,8)
-; AVX2-NEXT: vpextrq $1, %xmm0, (%rsp,%r8,8)
-; AVX2-NEXT: andl $7, %r9d
-; AVX2-NEXT: vmovq %xmm4, (%rsp,%r9,8)
+; AVX2-NEXT: vpextrq $1, %xmm0, (%rsp,%rdi,8)
+; AVX2-NEXT: andl $7, %r8d
+; AVX2-NEXT: vmovq %xmm4, (%rsp,%r8,8)
; AVX2-NEXT: andl $7, %eax
; AVX2-NEXT: vpextrq $1, %xmm4, (%rsp,%rax,8)
-; AVX2-NEXT: andl $7, %edi
-; AVX2-NEXT: vmovq %xmm1, (%rsp,%rdi,8)
+; AVX2-NEXT: andl $7, %r9d
+; AVX2-NEXT: vmovq %xmm1, (%rsp,%r9,8)
; AVX2-NEXT: andl $7, %r10d
; AVX2-NEXT: vpextrq $1, %xmm1, (%rsp,%r10,8)
; AVX2-NEXT: cmpq $7, %r11
@@ -4711,19 +4743,19 @@ define <8 x i64> @test_compress_knownbits_sext_v8i16_8i64(<8 x i16> %vec, <8 x i
; AVX2-NEXT: vmovd %xmm1, %esi
; AVX2-NEXT: andl $1, %esi
; AVX2-NEXT: addq %rsi, %rdx
-; AVX2-NEXT: vpextrw $2, %xmm1, %r8d
+; AVX2-NEXT: vpextrw $2, %xmm1, %edi
+; AVX2-NEXT: andl $1, %edi
+; AVX2-NEXT: addq %rdx, %rdi
+; AVX2-NEXT: vpextrw $3, %xmm1, %r8d
; AVX2-NEXT: andl $1, %r8d
-; AVX2-NEXT: addq %rdx, %r8
-; AVX2-NEXT: vpextrw $3, %xmm1, %r9d
+; AVX2-NEXT: addq %rdi, %r8
+; AVX2-NEXT: addq %r8, %rax
+; AVX2-NEXT: vpextrw $5, %xmm1, %r9d
; AVX2-NEXT: andl $1, %r9d
-; AVX2-NEXT: addq %r8, %r9
-; AVX2-NEXT: addq %r9, %rax
-; AVX2-NEXT: vpextrw $5, %xmm1, %edi
-; AVX2-NEXT: andl $1, %edi
-; AVX2-NEXT: addq %rax, %rdi
+; AVX2-NEXT: addq %rax, %r9
; AVX2-NEXT: vpextrw $6, %xmm1, %r10d
; AVX2-NEXT: andl $1, %r10d
-; AVX2-NEXT: addq %rdi, %r10
+; AVX2-NEXT: addq %r9, %r10
; AVX2-NEXT: vpextrw $7, %xmm1, %r11d
; AVX2-NEXT: andl $1, %r11d
; AVX2-NEXT: addq %r10, %r11
@@ -4736,13 +4768,13 @@ define <8 x i64> @test_compress_knownbits_sext_v8i16_8i64(<8 x i16> %vec, <8 x i
; AVX2-NEXT: vpextrq $1, %xmm0, (%rsp,%rsi,8)
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0
; AVX2-NEXT: vmovq %xmm0, (%rsp,%rdx,8)
-; AVX2-NEXT: vpextrq $1, %xmm0, (%rsp,%r8,8)
-; AVX2-NEXT: andl $7, %r9d
-; AVX2-NEXT: vmovq %xmm4, (%rsp,%r9,8)
+; AVX2-NEXT: vpextrq $1, %xmm0, (%rsp,%rdi,8)
+; AVX2-NEXT: andl $7, %r8d
+; AVX2-NEXT: vmovq %xmm4, (%rsp,%r8,8)
; AVX2-NEXT: andl $7, %eax
; AVX2-NEXT: vpextrq $1, %xmm4, (%rsp,%rax,8)
-; AVX2-NEXT: andl $7, %edi
-; AVX2-NEXT: vmovq %xmm1, (%rsp,%rdi,8)
+; AVX2-NEXT: andl $7, %r9d
+; AVX2-NEXT: vmovq %xmm1, (%rsp,%r9,8)
; AVX2-NEXT: andl $7, %r10d
; AVX2-NEXT: vpextrq $1, %xmm1, (%rsp,%r10,8)
; AVX2-NEXT: cmpq $7, %r11
More information about the llvm-branch-commits
mailing list