[llvm] [AMDGPU] Re-implement splat scalar handling for packed 64-bit ops (PR #216896)
Changpeng Fang via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 18 14:46:47 PDT 2026
https://github.com/changpeng updated https://github.com/llvm/llvm-project/pull/216896
>From cd414e6d7bd6e9ec600cd60232aadab47c4de214 Mon Sep 17 00:00:00 2001
From: Changpeng Fang <changpeng.fang at amd.com>
Date: Mon, 17 Aug 2026 14:56:53 -0700
Subject: [PATCH 1/2] [AMDGPU] Re-implement splat scalar handling for packed
64-bit ops
In instruction selection, we identify the splat scalar pattern and set opsel
to 0. The legalizer is responsible for moving SGPRs to VGPRs if opsel is not 0.
A future optimization could save SGPR copies for upper lanes since hardware only
reads the first SGPR.
This new implementation fixes the issue when an uniform value has to be computed
by a VALU-only instruction, and thus the result is in a VGPR. This will lead to
the missing data in the upper lane with the previous approach.
---
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 6 +-
.../packed-fp64-uniform-vgpr-splat-operand.ll | 123 ++++++++++++++++++
llvm/test/CodeGen/AMDGPU/packed-fp64.ll | 40 +++---
llvm/test/CodeGen/AMDGPU/packed-u64.ll | 1 +
llvm/test/CodeGen/AMDGPU/pk-lshl-add-u64.ll | 7 +-
5 files changed, 159 insertions(+), 18 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/packed-fp64-uniform-vgpr-splat-operand.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index 40345819a3f8d..744bde043e16c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -3699,12 +3699,16 @@ bool AMDGPUDAGToDAGISel::SelectVOP3PMods(SDValue In, SDValue &Src,
CurDAG->getTargetConstant(RC->getID(), SL, MVT::i32), Lo,
CurDAG->getTargetConstant(TRI->getSubRegFromChannel(0, NumRegs), SL,
MVT::i32),
- (!HasOpSel && Lo->isDivergent()) ? Lo : Undef,
+ // For packed 64-bit ops without OPSEL support, a later pass will
+ // optimize the splat sgpr patterns to save registers.
+ HasOpSel ? Undef : Lo,
CurDAG->getTargetConstant(
TRI->getSubRegFromChannel(NumRegs, NumRegs), SL, MVT::i32)};
Src = SDValue(CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, SL,
Src.getValueType(), Ops), 0);
+ // Check that both op_sel_0 and op_sel_1 are zero.
+ assert(!(Mods & (SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1)));
}
SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
return true;
diff --git a/llvm/test/CodeGen/AMDGPU/packed-fp64-uniform-vgpr-splat-operand.ll b/llvm/test/CodeGen/AMDGPU/packed-fp64-uniform-vgpr-splat-operand.ll
new file mode 100644
index 0000000000000..f5e5ff70fb6fa
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/packed-fp64-uniform-vgpr-splat-operand.ll
@@ -0,0 +1,123 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.51 < %s | FileCheck -enable-var-scope -check-prefixes=GFX1251,GFX1251-SDAG %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.51 < %s | FileCheck -enable-var-scope -check-prefixes=GFX1251,GFX1251-GISEL %s
+
+define <2 x double> @test_v2f64_fadd(<2 x double> %vec, double inreg %a) {
+; GFX1251-SDAG-LABEL: test_v2f64_fadd:
+; GFX1251-SDAG: ; %bb.0: ; %entry
+; GFX1251-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1251-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1251-SDAG-NEXT: v_mul_f64_e64 v[4:5], 0x40240000, s[0:1]
+; GFX1251-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[6:7], v[4:5]
+; GFX1251-SDAG-NEXT: v_pk_add_f64 v[0:3], v[0:3], v[4:7]
+; GFX1251-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1251-GISEL-LABEL: test_v2f64_fadd:
+; GFX1251-GISEL: ; %bb.0: ; %entry
+; GFX1251-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1251-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1251-GISEL-NEXT: v_mul_f64_e64 v[4:5], 0x40240000, s[0:1]
+; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s0, v4
+; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s1, v5
+; GFX1251-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
+; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[2:3]
+; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1251-GISEL-NEXT: v_pk_add_f64 v[0:3], v[0:3], v[4:7]
+; GFX1251-GISEL-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ ; mul0 is uniform but in vgpr
+ %mul0 = fmul double %a, 10.0
+
+ %splat.insert = insertelement <2 x double> poison, double %mul0, i64 0
+ %splat = shufflevector <2 x double> %splat.insert, <2 x double> poison, <2 x i32> zeroinitializer
+
+ %add = fadd contract <2 x double> %vec, %splat
+
+ ret <2 x double> %add
+}
+
+define <2 x double> @test_v2f64_fmul_fadd(<2 x double> %vec, double inreg %a) {
+; GFX1251-SDAG-LABEL: test_v2f64_fmul_fadd:
+; GFX1251-SDAG: ; %bb.0: ; %entry
+; GFX1251-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1251-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1251-SDAG-NEXT: v_mul_f64_e64 v[4:5], 0x40240000, s[0:1]
+; GFX1251-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[6:7], v[4:5]
+; GFX1251-SDAG-NEXT: v_pk_fma_f64 v[0:3], v[0:3], v[4:7], v[4:7]
+; GFX1251-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1251-GISEL-LABEL: test_v2f64_fmul_fadd:
+; GFX1251-GISEL: ; %bb.0: ; %entry
+; GFX1251-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1251-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1251-GISEL-NEXT: v_mul_f64_e64 v[4:5], 0x40240000, s[0:1]
+; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s0, v4
+; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s1, v5
+; GFX1251-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
+; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[2:3]
+; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1251-GISEL-NEXT: v_pk_fma_f64 v[0:3], v[0:3], v[4:7], v[4:7]
+; GFX1251-GISEL-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ ; mul0 is uniform but in vgpr
+ %mul0 = fmul double %a, 10.0
+
+ %splat.insert = insertelement <2 x double> poison, double %mul0, i64 0
+ %splat = shufflevector <2 x double> %splat.insert, <2 x double> poison, <2 x i32> zeroinitializer
+
+ %mul = fmul contract <2 x double> %vec, %splat
+ %add = fadd contract <2 x double> %mul, %splat
+
+ ret <2 x double> %add
+}
+
+define <2 x double> @test_v2f64_fmul_fmul(<2 x double> %vec, double inreg %a) {
+; GFX1251-SDAG-LABEL: test_v2f64_fmul_fmul:
+; GFX1251-SDAG: ; %bb.0: ; %entry
+; GFX1251-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1251-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1251-SDAG-NEXT: v_mul_f64_e64 v[4:5], 0x40240000, s[0:1]
+; GFX1251-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[6:7], v[4:5]
+; GFX1251-SDAG-NEXT: v_pk_add_f64 v[0:3], v[0:3], v[4:7]
+; GFX1251-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1251-SDAG-NEXT: v_pk_max_num_f64 v[0:3], v[0:3], v[4:7]
+; GFX1251-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1251-GISEL-LABEL: test_v2f64_fmul_fmul:
+; GFX1251-GISEL: ; %bb.0: ; %entry
+; GFX1251-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1251-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1251-GISEL-NEXT: v_mul_f64_e64 v[8:9], 0x40240000, s[0:1]
+; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s0, v8
+; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s1, v9
+; GFX1251-GISEL-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
+; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[2:3]
+; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1251-GISEL-NEXT: v_pk_add_f64 v[0:3], v[0:3], v[4:7]
+; GFX1251-GISEL-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[8:9]
+; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1251-GISEL-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[8:9]
+; GFX1251-GISEL-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ ; mul0 is uniform but in vgpr
+ %mul0 = fmul double %a, 10.0
+
+ %splat.insert = insertelement <2 x double> poison, double %mul0, i64 0
+ %splat = shufflevector <2 x double> %splat.insert, <2 x double> poison, <2 x i32> zeroinitializer
+
+ %add = fadd contract <2 x double> %vec, %splat
+ %max = call nnan <2 x double> @llvm.maxnum.v2f64(<2 x double> %add, <2 x double> %splat)
+
+ ret <2 x double> %max
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX1251: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/packed-fp64.ll b/llvm/test/CodeGen/AMDGPU/packed-fp64.ll
index c9bcd710adb97..e0374fded2c88 100644
--- a/llvm/test/CodeGen/AMDGPU/packed-fp64.ll
+++ b/llvm/test/CodeGen/AMDGPU/packed-fp64.ll
@@ -482,6 +482,7 @@ define amdgpu_kernel void @fadd_v2_v_v_sgpr_splat(ptr addrspace(1) %a, double %v
; GFX1251-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1251-SDAG-NEXT: global_load_b128 v[0:3], v4, s[0:1] scale_offset
; GFX1251-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
+; GFX1251-SDAG-NEXT: s_mov_b64 s[6:7], s[2:3]
; GFX1251-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1251-SDAG-NEXT: v_pk_add_f64 v[0:3], v[0:3], s[4:7]
; GFX1251-SDAG-NEXT: global_store_b128 v4, v[0:3], s[0:1] scale_offset
@@ -683,6 +684,7 @@ define amdgpu_kernel void @fadd_v2_v_fneg(ptr addrspace(1) %a, double %x) {
; GFX1251-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1251-SDAG-NEXT: global_load_b128 v[0:3], v4, s[0:1] scale_offset
; GFX1251-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
+; GFX1251-SDAG-NEXT: s_mov_b64 s[6:7], s[2:3]
; GFX1251-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1251-SDAG-NEXT: v_pk_add_f64 v[0:3], v[0:3], s[4:7] neg_lo:[0,1] neg_hi:[0,1]
; GFX1251-SDAG-NEXT: global_store_b128 v4, v[0:3], s[0:1] scale_offset
@@ -733,6 +735,7 @@ define amdgpu_kernel void @fadd_v2_v_fneg_lo(ptr addrspace(1) %a, double %x) {
; GFX1251-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1251-SDAG-NEXT: global_load_b128 v[0:3], v4, s[0:1] scale_offset
; GFX1251-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
+; GFX1251-SDAG-NEXT: s_mov_b64 s[6:7], s[2:3]
; GFX1251-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1251-SDAG-NEXT: v_pk_add_f64 v[0:3], v[0:3], s[4:7] neg_lo:[0,1]
; GFX1251-SDAG-NEXT: global_store_b128 v4, v[0:3], s[0:1] scale_offset
@@ -783,6 +786,7 @@ define amdgpu_kernel void @fadd_v2_v_fneg_hi(ptr addrspace(1) %a, double %x) {
; GFX1251-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1251-SDAG-NEXT: global_load_b128 v[0:3], v4, s[0:1] scale_offset
; GFX1251-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
+; GFX1251-SDAG-NEXT: s_mov_b64 s[6:7], s[2:3]
; GFX1251-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1251-SDAG-NEXT: v_pk_add_f64 v[0:3], v[0:3], s[4:7] neg_hi:[0,1]
; GFX1251-SDAG-NEXT: global_store_b128 v4, v[0:3], s[0:1] scale_offset
@@ -1480,6 +1484,7 @@ define amdgpu_kernel void @fmul_v2_v_fneg(ptr addrspace(1) %a, double %x) {
; GFX1251-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1251-SDAG-NEXT: global_load_b128 v[0:3], v4, s[0:1] scale_offset
; GFX1251-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
+; GFX1251-SDAG-NEXT: s_mov_b64 s[6:7], s[2:3]
; GFX1251-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1251-SDAG-NEXT: v_pk_mul_f64 v[0:3], v[0:3], s[4:7] neg_lo:[0,1] neg_hi:[0,1]
; GFX1251-SDAG-NEXT: global_store_b128 v4, v[0:3], s[0:1] scale_offset
@@ -1917,14 +1922,16 @@ define amdgpu_kernel void @fma_v2_v_imm(ptr addrspace(1) %a) {
; GFX1251-SDAG-NEXT: v_nop
; GFX1251-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1251-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX1251-SDAG-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
; GFX1251-SDAG-NEXT: v_and_b32_e32 v4, 0x3ff, v0
; GFX1251-SDAG-NEXT: s_mov_b64 s[0:1], 0x4059000000000000
+; GFX1251-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1251-SDAG-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX1251-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1251-SDAG-NEXT: global_load_b128 v[0:3], v4, s[2:3] scale_offset
+; GFX1251-SDAG-NEXT: global_load_b128 v[0:3], v4, s[6:7] scale_offset
; GFX1251-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1251-SDAG-NEXT: v_pk_fma_f64 v[0:3], v[0:3], s[0:3], 0x40690000
-; GFX1251-SDAG-NEXT: global_store_b128 v4, v[0:3], s[2:3] scale_offset
+; GFX1251-SDAG-NEXT: global_store_b128 v4, v[0:3], s[6:7] scale_offset
; GFX1251-SDAG-NEXT: s_endpgm
;
; GFX1251-GISEL-LABEL: fma_v2_v_imm:
@@ -2070,6 +2077,7 @@ define amdgpu_kernel void @fma_v2_v_fneg(ptr addrspace(1) %a, double %x) {
; GFX1251-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1251-SDAG-NEXT: global_load_b128 v[0:3], v4, s[0:1] scale_offset
; GFX1251-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
+; GFX1251-SDAG-NEXT: s_mov_b64 s[6:7], s[2:3]
; GFX1251-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1251-SDAG-NEXT: v_pk_fma_f64 v[0:3], v[0:3], s[4:7], s[4:7] neg_lo:[0,1,1] neg_hi:[0,1,1]
; GFX1251-SDAG-NEXT: global_store_b128 v4, v[0:3], s[0:1] scale_offset
@@ -2742,18 +2750,18 @@ define amdgpu_kernel void @fma_v2_s_imm_imm(ptr addrspace(1) %a) {
; GFX1251-SDAG-NEXT: v_nop
; GFX1251-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1251-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX1251-SDAG-NEXT: s_wait_xcnt 0x0
-; GFX1251-SDAG-NEXT: s_mov_b64 s[4:5], 0x4069000000000000
+; GFX1251-SDAG-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
+; GFX1251-SDAG-NEXT: s_mov_b64 s[2:3], 0x4069000000000000
; GFX1251-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[4:5]
-; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
; GFX1251-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1251-SDAG-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
+; GFX1251-SDAG-NEXT: s_load_b64 s[0:1], s[6:7], 0x0
; GFX1251-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1251-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1251-SDAG-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX1251-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1251-SDAG-NEXT: v_pk_fma_f64 v[2:5], 0x40590000, s[0:3], v[2:5]
-; GFX1251-SDAG-NEXT: global_store_b128 v0, v[2:5], s[2:3] scale_offset
+; GFX1251-SDAG-NEXT: global_store_b128 v0, v[2:5], s[6:7] scale_offset
; GFX1251-SDAG-NEXT: s_endpgm
;
; GFX1251-GISEL-LABEL: fma_v2_s_imm_imm:
@@ -2806,17 +2814,19 @@ define amdgpu_kernel void @fma_v2_imm_imm_s(ptr addrspace(1) %a) {
; GFX1251-SDAG-NEXT: v_nop
; GFX1251-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1251-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX1251-SDAG-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
; GFX1251-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1251-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1251-SDAG-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
+; GFX1251-SDAG-NEXT: s_load_b64 s[0:1], s[6:7], 0x0
; GFX1251-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX1251-SDAG-NEXT: s_mov_b64 s[0:1], 0x4059000000000000
-; GFX1251-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1251-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1251-SDAG-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[4:5], v[2:3]
+; GFX1251-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1251-SDAG-NEXT: v_pk_fma_f64 v[2:5], 0x40690000, s[0:3], v[2:5]
-; GFX1251-SDAG-NEXT: global_store_b128 v0, v[2:5], s[2:3] scale_offset
+; GFX1251-SDAG-NEXT: global_store_b128 v0, v[2:5], s[6:7] scale_offset
; GFX1251-SDAG-NEXT: s_endpgm
;
; GFX1251-GISEL-LABEL: fma_v2_imm_imm_s:
diff --git a/llvm/test/CodeGen/AMDGPU/packed-u64.ll b/llvm/test/CodeGen/AMDGPU/packed-u64.ll
index 71bc83860ffb2..ee86ba72093f7 100644
--- a/llvm/test/CodeGen/AMDGPU/packed-u64.ll
+++ b/llvm/test/CodeGen/AMDGPU/packed-u64.ll
@@ -482,6 +482,7 @@ define amdgpu_kernel void @add_v2_v_v_sgpr_splat(ptr addrspace(1) %a, i64 %v) {
; GFX1251-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1251-SDAG-NEXT: global_load_b128 v[0:3], v4, s[0:1] scale_offset
; GFX1251-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
+; GFX1251-SDAG-NEXT: s_mov_b64 s[6:7], s[2:3]
; GFX1251-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1251-SDAG-NEXT: v_pk_add_nc_u64 v[0:3], v[0:3], s[4:7]
; GFX1251-SDAG-NEXT: global_store_b128 v4, v[0:3], s[0:1] scale_offset
diff --git a/llvm/test/CodeGen/AMDGPU/pk-lshl-add-u64.ll b/llvm/test/CodeGen/AMDGPU/pk-lshl-add-u64.ll
index 8af990338f3d3..427465f5a5b35 100644
--- a/llvm/test/CodeGen/AMDGPU/pk-lshl-add-u64.ll
+++ b/llvm/test/CodeGen/AMDGPU/pk-lshl-add-u64.ll
@@ -261,8 +261,10 @@ define <2 x i64> @pk_lshl_add_u64_v1_sgpr_splat(<2 x i64> %v, i64 inreg %a) {
; GFX1251-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1251-NEXT: s_wait_kmcnt 0x0
; GFX1251-NEXT: s_mov_b32 s2, 1
-; GFX1251-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1251-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1251-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s2
+; GFX1251-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX1251-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX1251-NEXT: v_pk_lshl_add_u64 v[0:3], v[0:3], v[4:5], s[0:3]
; GFX1251-NEXT: s_set_pc_i64 s[30:31]
%s1 = insertelement <2 x i64> poison, i64 %a, i32 0
@@ -296,7 +298,8 @@ define <2 x i64> @pk_lshl_add_u64_sgpr_splat_v1(i64 inreg %v, <2 x i64> %a) {
; GFX1251-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1251-NEXT: s_wait_kmcnt 0x0
; GFX1251-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
-; GFX1251-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1251-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1251-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX1251-NEXT: v_pk_add_nc_u64 v[0:3], s[0:3], v[0:3]
; GFX1251-NEXT: s_set_pc_i64 s[30:31]
%s1 = insertelement <2 x i64> poison, i64 %v, i32 0
>From dc5828aef055e80bcb2b0d2db00660e0972b4a59 Mon Sep 17 00:00:00 2001
From: Changpeng Fang <changpeng.fang at amd.com>
Date: Tue, 18 Aug 2026 14:46:36 -0700
Subject: [PATCH 2/2] Trigger CI rerun
More information about the llvm-commits
mailing list