[llvm] b15daae - [AMDGPU] Add scalar_to_vector pattern for v2bf16 (#225303)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 24 23:15:09 PDT 2026
Author: Changpeng Fang
Date: 2026-09-24T23:14:58-07:00
New Revision: b15daae148e77f1ad2e20990094ddbeb8cc598ac
URL: https://github.com/llvm/llvm-project/commit/b15daae148e77f1ad2e20990094ddbeb8cc598ac
DIFF: https://github.com/llvm/llvm-project/commit/b15daae148e77f1ad2e20990094ddbeb8cc598ac.diff
LOG: [AMDGPU] Add scalar_to_vector pattern for v2bf16 (#225303)
A target DAGCombine rewrites `v2bf16 = scalar_to_vector` into bitcast(any_extend),
but it only runs at -O1 and above, so at -O0 the node reaches ISel and fails to select.
Add the missing pattern, mirroring the v2f16 one, for both fake-true16 and real-true16.
Fixes: ROCM-31415
Added:
llvm/test/CodeGen/AMDGPU/scalar_to_vector_v2bf16.ll
Modified:
llvm/lib/Target/AMDGPU/SIInstructions.td
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index eaece88e98525..3cf20ffb1fcf4 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -4254,6 +4254,11 @@ def : GCNPat <
(v4f16 (scalar_to_vector f16:$src0)),
(INSERT_SUBREG (IMPLICIT_DEF), $src0, sub0)
>;
+
+def : GCNPat <
+ (v2bf16 (DivergentUnaryFrag<scalar_to_vector> (bf16 VGPR_32:$src0))),
+ (COPY VGPR_32:$src0)
+>;
}
let True16Predicate = UseRealTrue16Insts in {
@@ -4276,8 +4281,20 @@ def : GCNPat <
(v4f16 (scalar_to_vector f16:$src0)),
(REG_SEQUENCE VReg_64, $src0, lo16, (i16 (IMPLICIT_DEF)), hi16, (i32 (IMPLICIT_DEF)), sub1)
>;
+
+def : GCNPat <
+ (v2bf16 (DivergentUnaryFrag<scalar_to_vector> (bf16 VGPR_16:$src0))),
+ (REG_SEQUENCE VGPR_32, VGPR_16:$src0, lo16, (i16 (IMPLICIT_DEF)), hi16)
+>;
}
+// A bf16 value can only be in the low 16 bits of an SGPR, regardless of
+// Real-True16 or Fake-True16.
+def : GCNPat <
+ (v2bf16 (UniformUnaryFrag<scalar_to_vector> (bf16 SReg_32:$src0))),
+ (COPY SReg_32:$src0)
+>;
+
def : GCNPat <
(i64 (int_amdgcn_mov_dpp i64:$src, timm:$dpp_ctrl, timm:$row_mask,
timm:$bank_mask, timm:$bound_ctrl)),
diff --git a/llvm/test/CodeGen/AMDGPU/scalar_to_vector_v2bf16.ll b/llvm/test/CodeGen/AMDGPU/scalar_to_vector_v2bf16.ll
new file mode 100644
index 0000000000000..970edb0a08835
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/scalar_to_vector_v2bf16.ll
@@ -0,0 +1,227 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+
+; RUN: llc -O0 -mtriple=amdgpu12.50 -mattr=-real-true16 < %s | FileCheck -check-prefix=GFX1250-FAKE16 %s
+; RUN: llc -O0 -mtriple=amdgpu12.50 -mattr=+real-true16 < %s | FileCheck -check-prefix=GFX1250-TRUE16 %s
+
+define <2 x bfloat> @scalar_to_vector_v2bf16(bfloat %x) {
+; GFX1250-FAKE16-LABEL: scalar_to_vector_v2bf16:
+; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-TRUE16-LABEL: scalar_to_vector_v2bf16:
+; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: ; implicit-def: $sgpr0
+; GFX1250-TRUE16-NEXT: ; implicit-def: $sgpr1
+; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.h, s0
+; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
+ %ins = insertelement <2 x bfloat> poison, bfloat %x, i32 0
+ ret <2 x bfloat> %ins
+}
+
+define <4 x bfloat> @scalar_to_vector_v4bf16(bfloat %x) {
+; GFX1250-FAKE16-LABEL: scalar_to_vector_v4bf16:
+; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX1250-FAKE16-NEXT: ; implicit-def: $sgpr0
+; GFX1250-FAKE16-NEXT: ; implicit-def: $sgpr1
+; GFX1250-FAKE16-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v3, s0
+; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v0, v2
+; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v1, v3
+; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-TRUE16-LABEL: scalar_to_vector_v4bf16:
+; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: ; implicit-def: $sgpr0
+; GFX1250-TRUE16-NEXT: ; implicit-def: $sgpr1
+; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v2.h, s0
+; GFX1250-TRUE16-NEXT: ; implicit-def: $sgpr0
+; GFX1250-TRUE16-NEXT: ; implicit-def: $sgpr1
+; GFX1250-TRUE16-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v3, s0
+; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v0, v2
+; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v1, v3
+; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
+ %ins = insertelement <4 x bfloat> poison, bfloat %x, i32 0
+ ret <4 x bfloat> %ins
+}
+
+define bfloat @fadd_bf16(bfloat %x, bfloat %y) {
+; GFX1250-FAKE16-LABEL: fadd_bf16:
+; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-FAKE16-NEXT: v_pk_add_bf16 v0, v0, v1
+; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-TRUE16-LABEL: fadd_bf16:
+; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: ; implicit-def: $sgpr0
+; GFX1250-TRUE16-NEXT: ; implicit-def: $sgpr1
+; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v1.h, s0
+; GFX1250-TRUE16-NEXT: ; implicit-def: $sgpr0
+; GFX1250-TRUE16-NEXT: ; implicit-def: $sgpr1
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.h, s0
+; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v0, v0, v1
+; GFX1250-TRUE16-NEXT: ; kill: def $vgpr0_lo16 killed $vgpr0_lo16 killed $vgpr0 killed $exec
+; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
+ %r = fadd bfloat %x, %y
+ ret bfloat %r
+}
+
+define bfloat @fsub_bf16(bfloat %x, bfloat %y) {
+; GFX1250-FAKE16-LABEL: fsub_bf16:
+; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-FAKE16-NEXT: v_pk_add_bf16 v0, v0, v1 neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-TRUE16-LABEL: fsub_bf16:
+; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: ; implicit-def: $sgpr0
+; GFX1250-TRUE16-NEXT: ; implicit-def: $sgpr1
+; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v1.h, s0
+; GFX1250-TRUE16-NEXT: ; implicit-def: $sgpr0
+; GFX1250-TRUE16-NEXT: ; implicit-def: $sgpr1
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.h, s0
+; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v0, v0, v1 neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-TRUE16-NEXT: ; kill: def $vgpr0_lo16 killed $vgpr0_lo16 killed $vgpr0 killed $exec
+; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
+ %r = fsub bfloat %x, %y
+ ret bfloat %r
+}
+
+define bfloat @fmul_bf16(bfloat %x, bfloat %y) {
+; GFX1250-FAKE16-LABEL: fmul_bf16:
+; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-FAKE16-NEXT: v_pk_mul_bf16 v0, v0, v1
+; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-TRUE16-LABEL: fmul_bf16:
+; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: ; implicit-def: $sgpr0
+; GFX1250-TRUE16-NEXT: ; implicit-def: $sgpr1
+; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v1.h, s0
+; GFX1250-TRUE16-NEXT: ; implicit-def: $sgpr0
+; GFX1250-TRUE16-NEXT: ; implicit-def: $sgpr1
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.h, s0
+; GFX1250-TRUE16-NEXT: v_pk_mul_bf16 v0, v0, v1
+; GFX1250-TRUE16-NEXT: ; kill: def $vgpr0_lo16 killed $vgpr0_lo16 killed $vgpr0 killed $exec
+; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
+ %r = fmul bfloat %x, %y
+ ret bfloat %r
+}
+
+define bfloat @fmaxnum_bf16(bfloat %x, bfloat %y) {
+; GFX1250-FAKE16-LABEL: fmaxnum_bf16:
+; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-FAKE16-NEXT: v_pk_max_num_bf16 v0, v0, v1
+; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-TRUE16-LABEL: fmaxnum_bf16:
+; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: ; implicit-def: $sgpr0
+; GFX1250-TRUE16-NEXT: ; implicit-def: $sgpr1
+; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v1.h, s0
+; GFX1250-TRUE16-NEXT: ; implicit-def: $sgpr0
+; GFX1250-TRUE16-NEXT: ; implicit-def: $sgpr1
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.h, s0
+; GFX1250-TRUE16-NEXT: v_pk_max_num_bf16 v0, v0, v1
+; GFX1250-TRUE16-NEXT: ; kill: def $vgpr0_lo16 killed $vgpr0_lo16 killed $vgpr0 killed $exec
+; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
+ %r = call bfloat @llvm.maxnum.bf16(bfloat %x, bfloat %y)
+ ret bfloat %r
+}
+
+define bfloat @fminnum_bf16(bfloat %x, bfloat %y) {
+; GFX1250-FAKE16-LABEL: fminnum_bf16:
+; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-FAKE16-NEXT: v_pk_min_num_bf16 v0, v0, v1
+; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-TRUE16-LABEL: fminnum_bf16:
+; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: ; implicit-def: $sgpr0
+; GFX1250-TRUE16-NEXT: ; implicit-def: $sgpr1
+; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v1.h, s0
+; GFX1250-TRUE16-NEXT: ; implicit-def: $sgpr0
+; GFX1250-TRUE16-NEXT: ; implicit-def: $sgpr1
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.h, s0
+; GFX1250-TRUE16-NEXT: v_pk_min_num_bf16 v0, v0, v1
+; GFX1250-TRUE16-NEXT: ; kill: def $vgpr0_lo16 killed $vgpr0_lo16 killed $vgpr0 killed $exec
+; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
+ %r = call bfloat @llvm.minnum.bf16(bfloat %x, bfloat %y)
+ ret bfloat %r
+}
+
+define bfloat @fma_bf16(bfloat %x, bfloat %y, bfloat %z) {
+; GFX1250-FAKE16-LABEL: fma_bf16:
+; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-FAKE16-NEXT: v_pk_fma_bf16 v0, v0, v1, v2
+; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-TRUE16-LABEL: fma_bf16:
+; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: ; implicit-def: $sgpr0
+; GFX1250-TRUE16-NEXT: ; implicit-def: $sgpr1
+; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v2.h, s0
+; GFX1250-TRUE16-NEXT: ; implicit-def: $sgpr0
+; GFX1250-TRUE16-NEXT: ; implicit-def: $sgpr1
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v1.h, s0
+; GFX1250-TRUE16-NEXT: ; implicit-def: $sgpr0
+; GFX1250-TRUE16-NEXT: ; implicit-def: $sgpr1
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.h, s0
+; GFX1250-TRUE16-NEXT: v_pk_fma_bf16 v0, v0, v1, v2
+; GFX1250-TRUE16-NEXT: ; kill: def $vgpr0_lo16 killed $vgpr0_lo16 killed $vgpr0 killed $exec
+; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
+ %r = call bfloat @llvm.fma.bf16(bfloat %x, bfloat %y, bfloat %z)
+ ret bfloat %r
+}
+
+define <2 x bfloat> @uniform_scalar_to_vector_v2bf16(bfloat inreg %x) {
+; GFX1250-FAKE16-LABEL: uniform_scalar_to_vector_v2bf16:
+; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-TRUE16-LABEL: uniform_scalar_to_vector_v2bf16:
+; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
+ %ins = insertelement <2 x bfloat> poison, bfloat %x, i32 0
+ ret <2 x bfloat> %ins
+}
More information about the llvm-commits
mailing list