[llvm] 02b5beb - [AMDGPU][GISel] RegBankLegalize rules for CVT Scale and SR BF16-specific intrinsics (#214018)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 4 15:48:25 PDT 2026
Author: Chinmay Deshpande
Date: 2026-08-04T15:48:20-07:00
New Revision: 02b5beb57bb4bbc4139853f6db5ec4b90a70f656
URL: https://github.com/llvm/llvm-project/commit/02b5beb57bb4bbc4139853f6db5ec4b90a70f656
DIFF: https://github.com/llvm/llvm-project/commit/02b5beb57bb4bbc4139853f6db5ec4b90a70f656.diff
LOG: [AMDGPU][GISel] RegBankLegalize rules for CVT Scale and SR BF16-specific intrinsics (#214018)
Added:
Modified:
llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scale.pk.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.pk.gfx950.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.pk.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.pk16.gfx1250.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.pk8.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.pk.gfx1250.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.pk.gfx950.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.pk.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.pk16.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.sr.pk.bf16.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
index 66a5e1bb378be..dfb80f53eb70d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
@@ -2044,15 +2044,23 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
.Uni(S32, {{UniInVgprS32}, {IntrId, Vgpr16, Vgpr32, Vgpr32}})
.Div(S32, {{Vgpr32}, {IntrId, Vgpr16, Vgpr32, Vgpr32}});
- addRulesForIOpcs({amdgcn_cvt_sr_pk_f16_f32}, Standard)
+ addRulesForIOpcs({amdgcn_cvt_sr_pk_f16_f32, amdgcn_cvt_sr_pk_bf16_f32},
+ Standard)
.Uni(V2S16, {{UniInVgprV2S16}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
.Div(V2S16, {{VgprV2S16}, {IntrId, Vgpr32, Vgpr32, Vgpr32}});
- addRulesForIOpcs({amdgcn_cvt_scalef32_sr_fp8_f16})
- .Any({{DivS32}, {{Vgpr32}, {IntrId, Vgpr32, Vgpr16, Vgpr32, Vgpr32}}});
+ addRulesForIOpcs(
+ {amdgcn_cvt_scalef32_sr_fp8_f16, amdgcn_cvt_scalef32_sr_bf8_f16,
+ amdgcn_cvt_scalef32_sr_fp8_bf16, amdgcn_cvt_scalef32_sr_bf8_bf16})
+ .Any({{DivS32}, {{Vgpr32}, {IntrId, Vgpr32, Vgpr16, Vgpr32, Vgpr32}}})
+ .Any({{UniS32},
+ {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr16, Vgpr32, Vgpr32}}});
- addRulesForIOpcs({amdgcn_cvt_scalef32_sr_fp8_f32})
- .Any({{DivS32}, {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32, Vgpr32}}});
+ addRulesForIOpcs(
+ {amdgcn_cvt_scalef32_sr_fp8_f32, amdgcn_cvt_scalef32_sr_bf8_f32})
+ .Any({{DivS32}, {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32, Vgpr32}}})
+ .Any({{UniS32},
+ {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Vgpr32, Vgpr32}}});
addRulesForIOpcs({amdgcn_cubesc, amdgcn_cubetc, amdgcn_cubema, amdgcn_cubeid,
amdgcn_fma_legacy},
@@ -2098,14 +2106,20 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
amdgcn_cvt_scalef32_sr_pk32_bf6_bf16,
amdgcn_cvt_scalef32_sr_pk32_fp6_bf16},
Standard)
- .Any({{DivV6S32}, {{VgprV6S32}, {IntrId, VgprV32S16, Vgpr32, Vgpr32}}});
+ .Any({{DivV6S32}, {{VgprV6S32}, {IntrId, VgprV32S16, Vgpr32, Vgpr32}}})
+ .Any({{UniV6S32},
+ {{UniInVgprV6S32}, {IntrId, VgprV32S16, Vgpr32, Vgpr32}}});
addRulesForIOpcs({amdgcn_cvt_scalef32_sr_pk32_bf6_f32,
amdgcn_cvt_scalef32_sr_pk32_fp6_f32},
Standard)
- .Any({{DivV6S32}, {{VgprV6S32}, {IntrId, VgprV32S32, Vgpr32, Vgpr32}}});
+ .Any({{DivV6S32}, {{VgprV6S32}, {IntrId, VgprV32S32, Vgpr32, Vgpr32}}})
+ .Any({{UniV6S32},
+ {{UniInVgprV6S32}, {IntrId, VgprV32S32, Vgpr32, Vgpr32}}});
- addRulesForIOpcs({amdgcn_cvt_scalef32_sr_pk_fp4_f16}, Standard)
+ addRulesForIOpcs(
+ {amdgcn_cvt_scalef32_sr_pk_fp4_f16, amdgcn_cvt_scalef32_sr_pk_fp4_bf16},
+ Standard)
.Uni(S32, {{UniInVgprS32}, {IntrId, Vgpr32, VgprV2S16, Vgpr32, Vgpr32}})
.Div(S32, {{Vgpr32}, {IntrId, Vgpr32, VgprV2S16, Vgpr32, Vgpr32}});
@@ -2131,7 +2145,8 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
.Uni(S32, {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32}});
addRulesForIOpcs(
- {amdgcn_cvt_scalef32_pk16_bf6_f16, amdgcn_cvt_scalef32_pk16_fp6_f16},
+ {amdgcn_cvt_scalef32_pk16_bf6_f16, amdgcn_cvt_scalef32_pk16_fp6_f16,
+ amdgcn_cvt_scalef32_pk16_bf6_bf16, amdgcn_cvt_scalef32_pk16_fp6_bf16},
Standard)
.Any({{DivV3S32}, {{VgprV3S32}, {IntrId, VgprV16S16, Vgpr32}}})
.Any({{UniV3S32}, {{UniInVgprV3S32}, {IntrId, VgprV16S16, Vgpr32}}});
@@ -2143,7 +2158,8 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
.Any({{UniV3S32}, {{UniInVgprV3S32}, {IntrId, VgprV16S32, Vgpr32}}});
addRulesForIOpcs(
- {amdgcn_cvt_scalef32_pk8_bf8_f16, amdgcn_cvt_scalef32_pk8_fp8_f16},
+ {amdgcn_cvt_scalef32_pk8_bf8_f16, amdgcn_cvt_scalef32_pk8_fp8_f16,
+ amdgcn_cvt_scalef32_pk8_bf8_bf16, amdgcn_cvt_scalef32_pk8_fp8_bf16},
Standard)
.Any({{DivV2S32}, {{VgprV2S32}, {IntrId, VgprV8S16, Vgpr32}}})
.Any({{UniV2S32}, {{UniInVgprV2S32}, {IntrId, VgprV8S16, Vgpr32}}});
@@ -2154,7 +2170,9 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
.Any({{DivV2S32}, {{VgprV2S32}, {IntrId, VgprV8S32, Vgpr32}}})
.Any({{UniV2S32}, {{UniInVgprV2S32}, {IntrId, VgprV8S32, Vgpr32}}});
- addRulesForIOpcs({amdgcn_cvt_scalef32_pk8_fp4_f16}, Standard)
+ addRulesForIOpcs(
+ {amdgcn_cvt_scalef32_pk8_fp4_f16, amdgcn_cvt_scalef32_pk8_fp4_bf16},
+ Standard)
.Div(S32, {{Vgpr32}, {IntrId, VgprV8S16, Vgpr32}})
.Uni(S32, {{UniInVgprS32}, {IntrId, VgprV8S16, Vgpr32}});
@@ -2163,7 +2181,9 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
.Uni(S32, {{UniInVgprS32}, {IntrId, VgprV8S32, Vgpr32}});
addRulesForIOpcs({amdgcn_cvt_scalef32_sr_pk16_bf6_f16,
- amdgcn_cvt_scalef32_sr_pk16_fp6_f16},
+ amdgcn_cvt_scalef32_sr_pk16_fp6_f16,
+ amdgcn_cvt_scalef32_sr_pk16_bf6_bf16,
+ amdgcn_cvt_scalef32_sr_pk16_fp6_bf16},
Standard)
.Any({{DivV3S32}, {{VgprV3S32}, {IntrId, VgprV16S16, Vgpr32, Vgpr32}}})
.Any({{UniV3S32},
@@ -2176,9 +2196,11 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
.Any({{UniV3S32},
{{UniInVgprV3S32}, {IntrId, VgprV16S32, Vgpr32, Vgpr32}}});
- addRulesForIOpcs(
- {amdgcn_cvt_scalef32_sr_pk8_bf8_f16, amdgcn_cvt_scalef32_sr_pk8_fp8_f16},
- Standard)
+ addRulesForIOpcs({amdgcn_cvt_scalef32_sr_pk8_bf8_f16,
+ amdgcn_cvt_scalef32_sr_pk8_fp8_f16,
+ amdgcn_cvt_scalef32_sr_pk8_bf8_bf16,
+ amdgcn_cvt_scalef32_sr_pk8_fp8_bf16},
+ Standard)
.Any({{DivV2S32}, {{VgprV2S32}, {IntrId, VgprV8S16, Vgpr32, Vgpr32}}})
.Any({{UniV2S32},
{{UniInVgprV2S32}, {IntrId, VgprV8S16, Vgpr32, Vgpr32}}});
@@ -2190,7 +2212,9 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
.Any({{UniV2S32},
{{UniInVgprV2S32}, {IntrId, VgprV8S32, Vgpr32, Vgpr32}}});
- addRulesForIOpcs({amdgcn_cvt_scalef32_sr_pk8_fp4_f16}, Standard)
+ addRulesForIOpcs(
+ {amdgcn_cvt_scalef32_sr_pk8_fp4_f16, amdgcn_cvt_scalef32_sr_pk8_fp4_bf16},
+ Standard)
.Div(S32, {{Vgpr32}, {IntrId, VgprV8S16, Vgpr32, Vgpr32}})
.Uni(S32, {{UniInVgprS32}, {IntrId, VgprV8S16, Vgpr32, Vgpr32}});
@@ -2199,7 +2223,9 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
.Uni(S32, {{UniInVgprS32}, {IntrId, VgprV8S32, Vgpr32, Vgpr32}});
addRulesForIOpcs(
- {amdgcn_cvt_scale_pk16_f16_bf6, amdgcn_cvt_scale_pk16_f16_fp6}, Standard)
+ {amdgcn_cvt_scale_pk16_f16_bf6, amdgcn_cvt_scale_pk16_f16_fp6,
+ amdgcn_cvt_scale_pk16_bf16_bf6, amdgcn_cvt_scale_pk16_bf16_fp6},
+ Standard)
.Any({{DivV16S16}, {{VgprV16S16}, {IntrId, VgprV3S32, Vgpr32}}})
.Any({{UniV16S16}, {{UniInVgprV16S16}, {IntrId, VgprV3S32, Vgpr32}}});
@@ -2208,12 +2234,15 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
.Any({{DivV16S32}, {{VgprV16S32}, {IntrId, VgprV3S32, Vgpr32}}})
.Any({{UniV16S32}, {{UniInVgprV16S32}, {IntrId, VgprV3S32, Vgpr32}}});
- addRulesForIOpcs({amdgcn_cvt_scale_pk8_f16_bf8, amdgcn_cvt_scale_pk8_f16_fp8},
+ addRulesForIOpcs({amdgcn_cvt_scale_pk8_f16_bf8, amdgcn_cvt_scale_pk8_f16_fp8,
+ amdgcn_cvt_scale_pk8_bf16_bf8,
+ amdgcn_cvt_scale_pk8_bf16_fp8},
Standard)
.Any({{DivV8S16}, {{VgprV8S16}, {IntrId, VgprV2S32, Vgpr32}}})
.Any({{UniV8S16}, {{UniInVgprV8S16}, {IntrId, VgprV2S32, Vgpr32}}});
- addRulesForIOpcs({amdgcn_cvt_scale_pk8_f16_fp4}, Standard)
+ addRulesForIOpcs(
+ {amdgcn_cvt_scale_pk8_f16_fp4, amdgcn_cvt_scale_pk8_bf16_fp4}, Standard)
.Any({{DivV8S16}, {{VgprV8S16}, {IntrId, Vgpr32, Vgpr32}}})
.Any({{UniV8S16}, {{UniInVgprV8S16}, {IntrId, Vgpr32, Vgpr32}}});
@@ -2227,7 +2256,8 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
.Any({{UniV8S32}, {{UniInVgprV8S32}, {IntrId, Vgpr32, Vgpr32}}});
addRulesForIOpcs(
- {amdgcn_cvt_scalef32_pk32_bf6_f16, amdgcn_cvt_scalef32_pk32_fp6_f16},
+ {amdgcn_cvt_scalef32_pk32_bf6_f16, amdgcn_cvt_scalef32_pk32_fp6_f16,
+ amdgcn_cvt_scalef32_pk32_bf6_bf16, amdgcn_cvt_scalef32_pk32_fp6_bf16},
Standard)
.Any({{DivV6S32}, {{VgprV6S32}, {IntrId, VgprV32S16, Vgpr32}}})
.Any({{UniV6S32}, {{UniInVgprV6S32}, {IntrId, VgprV32S16, Vgpr32}}});
@@ -2252,7 +2282,8 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
.Any({{UniV2S32}, {{UniInVgprV2S32}, {IntrId, Vgpr32, Vgpr32}}});
addRulesForIOpcs(
- {amdgcn_cvt_scalef32_pk_fp8_f16, amdgcn_cvt_scalef32_pk_bf8_f16},
+ {amdgcn_cvt_scalef32_pk_fp8_f16, amdgcn_cvt_scalef32_pk_bf8_f16,
+ amdgcn_cvt_scalef32_pk_fp8_bf16, amdgcn_cvt_scalef32_pk_bf8_bf16},
Standard)
.Div(V2S16, {{VgprV2S16}, {IntrId, VgprV2S16, VgprV2S16, Vgpr32}})
.Uni(V2S16, {{UniInVgprV2S16}, {IntrId, VgprV2S16, VgprV2S16, Vgpr32}});
@@ -2265,10 +2296,11 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
.Div(S32, {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32, Vgpr32}})
.Uni(S32, {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Vgpr32, Vgpr32}});
- addRulesForIOpcs({amdgcn_cvt_scalef32_pk_f16_fp4,
- amdgcn_cvt_scalef32_pk_f16_fp8,
- amdgcn_cvt_scalef32_pk_f16_bf8},
- Standard)
+ addRulesForIOpcs(
+ {amdgcn_cvt_scalef32_pk_f16_fp4, amdgcn_cvt_scalef32_pk_f16_fp8,
+ amdgcn_cvt_scalef32_pk_f16_bf8, amdgcn_cvt_scalef32_pk_bf16_fp4,
+ amdgcn_cvt_scalef32_pk_bf16_fp8, amdgcn_cvt_scalef32_pk_bf16_bf8},
+ Standard)
.Div(V2S16, {{VgprV2S16}, {IntrId, Vgpr32, Vgpr32}})
.Uni(V2S16, {{UniInVgprV2S16}, {IntrId, Vgpr32, Vgpr32}});
@@ -2279,12 +2311,15 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
.Any({{UniV32S32}, {{UniInVgprV32S32}, {IntrId, VgprV6S32, Vgpr32}}});
addRulesForIOpcs(
- {amdgcn_cvt_scalef32_pk32_f16_fp6, amdgcn_cvt_scalef32_pk32_f16_bf6},
+ {amdgcn_cvt_scalef32_pk32_f16_fp6, amdgcn_cvt_scalef32_pk32_f16_bf6,
+ amdgcn_cvt_scalef32_pk32_bf16_fp6, amdgcn_cvt_scalef32_pk32_bf16_bf6},
Standard)
.Any({{DivV32S16}, {{VgprV32S16}, {IntrId, VgprV6S32, Vgpr32}}})
.Any({{UniV32S16}, {{UniInVgprV32S16}, {IntrId, VgprV6S32, Vgpr32}}});
- addRulesForIOpcs({amdgcn_cvt_scalef32_pk_fp4_f16}, Standard)
+ addRulesForIOpcs(
+ {amdgcn_cvt_scalef32_pk_fp4_f16, amdgcn_cvt_scalef32_pk_fp4_bf16},
+ Standard)
.Div(S32, {{Vgpr32}, {IntrId, Vgpr32, VgprV2S16, Vgpr32}})
.Uni(S32, {{UniInVgprS32}, {IntrId, Vgpr32, VgprV2S16, Vgpr32}});
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scale.pk.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scale.pk.ll
index c2efbbe512523..1312e87a5b182 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scale.pk.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scale.pk.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.50 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL %s
declare <8 x half> @llvm.amdgcn.cvt.scale.pk8.f16.fp8(<2 x i32> %src, i32 %scale, i32 %scale_sel)
declare <8 x bfloat> @llvm.amdgcn.cvt.scale.pk8.bf16.fp8(<2 x i32> %src, i32 %scale, i32 %scale_sel)
@@ -123,30 +123,104 @@ define amdgpu_ps void @test_cvt_scale_pk8_f16_bf8_ss(<2 x i32> inreg %src, i32 i
}
define amdgpu_ps void @test_cvt_scale_pk8_bf16_fp8_vv(<2 x i32> %src, i32 %scale, ptr addrspace(1) %out) {
-; GFX1250-LABEL: test_cvt_scale_pk8_bf16_fp8_vv:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_dual_mov_b32 v9, v4 :: v_dual_mov_b32 v8, v3
-; GFX1250-NEXT: v_cvt_scale_pk8_bf16_fp8 v[4:7], v[0:1], v2 scale_sel:1
-; GFX1250-NEXT: global_store_b128 v[8:9], v[4:7], off
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: test_cvt_scale_pk8_bf16_fp8_vv:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v9, v4 :: v_dual_mov_b32 v8, v3
+; GFX1250-SDAG-NEXT: v_cvt_scale_pk8_bf16_fp8 v[4:7], v[0:1], v2 scale_sel:1
+; GFX1250-SDAG-NEXT: global_store_b128 v[8:9], v[4:7], off
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: test_cvt_scale_pk8_bf16_fp8_vv:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4
+; GFX1250-GISEL-NEXT: v_cvt_scale_pk8_bf16_fp8 v[4:7], v[0:1], v2 scale_sel:1
+; GFX1250-GISEL-NEXT: global_store_b128 v[8:9], v[4:7], off
+; GFX1250-GISEL-NEXT: s_endpgm
+ %cvt = tail call <8 x bfloat> @llvm.amdgcn.cvt.scale.pk8.bf16.fp8(<2 x i32> %src, i32 %scale, i32 1)
+ store <8 x bfloat> %cvt, ptr addrspace(1) %out, align 8
+ ret void
+}
+
+define amdgpu_ps void @test_cvt_scale_pk8_bf16_fp8_ss(<2 x i32> inreg %src, i32 inreg %scale, ptr addrspace(1) %out) {
+; GFX1250-SDAG-LABEL: test_cvt_scale_pk8_bf16_fp8_ss:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v6, s0 :: v_dual_mov_b32 v7, s1
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_cvt_scale_pk8_bf16_fp8 v[2:5], v[6:7], s2 scale_sel:1
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: test_cvt_scale_pk8_bf16_fp8_ss:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_cvt_scale_pk8_bf16_fp8 v[2:5], v[6:7], s2 scale_sel:1
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-GISEL-NEXT: s_endpgm
%cvt = tail call <8 x bfloat> @llvm.amdgcn.cvt.scale.pk8.bf16.fp8(<2 x i32> %src, i32 %scale, i32 1)
store <8 x bfloat> %cvt, ptr addrspace(1) %out, align 8
ret void
}
define amdgpu_ps void @test_cvt_scale_pk8_bf16_bf8_vv(<2 x i32> %src, i32 %scale, ptr addrspace(1) %out) {
-; GFX1250-LABEL: test_cvt_scale_pk8_bf16_bf8_vv:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_dual_mov_b32 v9, v4 :: v_dual_mov_b32 v8, v3
-; GFX1250-NEXT: v_cvt_scale_pk8_bf16_bf8 v[4:7], v[0:1], v2 scale_sel:2
-; GFX1250-NEXT: global_store_b128 v[8:9], v[4:7], off
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: test_cvt_scale_pk8_bf16_bf8_vv:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v9, v4 :: v_dual_mov_b32 v8, v3
+; GFX1250-SDAG-NEXT: v_cvt_scale_pk8_bf16_bf8 v[4:7], v[0:1], v2 scale_sel:2
+; GFX1250-SDAG-NEXT: global_store_b128 v[8:9], v[4:7], off
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: test_cvt_scale_pk8_bf16_bf8_vv:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4
+; GFX1250-GISEL-NEXT: v_cvt_scale_pk8_bf16_bf8 v[4:7], v[0:1], v2 scale_sel:2
+; GFX1250-GISEL-NEXT: global_store_b128 v[8:9], v[4:7], off
+; GFX1250-GISEL-NEXT: s_endpgm
+ %cvt = tail call <8 x bfloat> @llvm.amdgcn.cvt.scale.pk8.bf16.bf8(<2 x i32> %src, i32 %scale, i32 2)
+ store <8 x bfloat> %cvt, ptr addrspace(1) %out, align 8
+ ret void
+}
+
+define amdgpu_ps void @test_cvt_scale_pk8_bf16_bf8_ss(<2 x i32> inreg %src, i32 inreg %scale, ptr addrspace(1) %out) {
+; GFX1250-SDAG-LABEL: test_cvt_scale_pk8_bf16_bf8_ss:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v6, s0 :: v_dual_mov_b32 v7, s1
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_cvt_scale_pk8_bf16_bf8 v[2:5], v[6:7], s2 scale_sel:2
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: test_cvt_scale_pk8_bf16_bf8_ss:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_cvt_scale_pk8_bf16_bf8 v[2:5], v[6:7], s2 scale_sel:2
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-GISEL-NEXT: s_endpgm
%cvt = tail call <8 x bfloat> @llvm.amdgcn.cvt.scale.pk8.bf16.bf8(<2 x i32> %src, i32 %scale, i32 2)
store <8 x bfloat> %cvt, ptr addrspace(1) %out, align 8
ret void
@@ -196,6 +270,22 @@ define amdgpu_ps void @test_cvt_scale_pk8_bf16_fp4_vv(i32 %src, i32 %scale, ptr
ret void
}
+define amdgpu_ps void @test_cvt_scale_pk8_bf16_fp4_ss(i32 inreg %src, i32 inreg %scale, ptr addrspace(1) %out) {
+; GFX1250-LABEL: test_cvt_scale_pk8_bf16_fp4_ss:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: v_mov_b32_e32 v2, s0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cvt_scale_pk8_bf16_fp4 v[2:5], v2, s1 scale_sel:4
+; GFX1250-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-NEXT: s_endpgm
+ %cvt = tail call <8 x bfloat> @llvm.amdgcn.cvt.scale.pk8.bf16.fp4(i32 %src, i32 %scale, i32 4)
+ store <8 x bfloat> %cvt, ptr addrspace(1) %out, align 16
+ ret void
+}
+
define amdgpu_ps void @test_cvt_scale_pk8_f32_fp8_vv(<2 x i32> %src, i32 %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_cvt_scale_pk8_f32_fp8_vv:
; GFX1250-SDAG: ; %bb.0:
@@ -543,35 +633,75 @@ define amdgpu_ps void @test_cvt_scale_pk16_f16_fp6_sl(<3 x i32> inreg %src, ptr
}
define amdgpu_ps void @test_cvt_scale_pk16_bf16_fp6_vv(<3 x i32> %src, i32 %scale, ptr addrspace(1) %out) {
-; GFX1250-LABEL: test_cvt_scale_pk16_bf16_fp6_vv:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_cvt_scale_pk16_bf16_fp6 v[6:13], v[0:2], v3 scale_sel:2
-; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: global_store_b128 v[4:5], v[10:13], off offset:16
-; GFX1250-NEXT: global_store_b128 v[4:5], v[6:9], off
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: test_cvt_scale_pk16_bf16_fp6_vv:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_cvt_scale_pk16_bf16_fp6 v[6:13], v[0:2], v3 scale_sel:2
+; GFX1250-SDAG-NEXT: s_clause 0x1
+; GFX1250-SDAG-NEXT: global_store_b128 v[4:5], v[10:13], off offset:16
+; GFX1250-SDAG-NEXT: global_store_b128 v[4:5], v[6:9], off
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: test_cvt_scale_pk16_bf16_fp6_vv:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_cvt_scale_pk16_bf16_fp6 v[6:13], v[0:2], v3 scale_sel:2
+; GFX1250-GISEL-NEXT: s_clause 0x1
+; GFX1250-GISEL-NEXT: global_store_b128 v[4:5], v[6:9], off
+; GFX1250-GISEL-NEXT: global_store_b128 v[4:5], v[10:13], off offset:16
+; GFX1250-GISEL-NEXT: s_endpgm
%cvt = tail call <16 x bfloat> @llvm.amdgcn.cvt.scale.pk16.bf16.fp6(<3 x i32> %src, i32 %scale, i32 2)
store <16 x bfloat> %cvt, ptr addrspace(1) %out, align 8
ret void
}
define amdgpu_ps void @test_cvt_scale_pk16_bf16_fp6_sl(<3 x i32> inreg %src, ptr addrspace(1) %out) {
-; GFX1250-LABEL: test_cvt_scale_pk16_bf16_fp6_sl:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_dual_mov_b32 v10, s0 :: v_dual_mov_b32 v11, s1
-; GFX1250-NEXT: v_mov_b32_e32 v12, s2
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cvt_scale_pk16_bf16_fp6 v[2:9], v[10:12], 0x64 scale_sel:3
-; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16
-; GFX1250-NEXT: global_store_b128 v[0:1], v[2:5], off
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: test_cvt_scale_pk16_bf16_fp6_sl:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v10, s0 :: v_dual_mov_b32 v11, s1
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v12, s2
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_cvt_scale_pk16_bf16_fp6 v[2:9], v[10:12], 0x64 scale_sel:3
+; GFX1250-SDAG-NEXT: s_clause 0x1
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: test_cvt_scale_pk16_bf16_fp6_sl:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v12, s2 :: v_dual_mov_b32 v11, s1
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v10, s0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_cvt_scale_pk16_bf16_fp6 v[2:9], v[10:12], 0x64 scale_sel:3
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v2
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s1, v3
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s2, v4
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s3, v5
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s4, v6
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s5, v7
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s6, v8
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s7, v9
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7]
+; GFX1250-GISEL-NEXT: s_clause 0x1
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16
+; GFX1250-GISEL-NEXT: s_endpgm
%cvt = tail call <16 x bfloat> @llvm.amdgcn.cvt.scale.pk16.bf16.fp6(<3 x i32> %src, i32 100, i32 3)
store <16 x bfloat> %cvt, ptr addrspace(1) %out, align 8
ret void
@@ -701,35 +831,75 @@ define amdgpu_ps void @test_cvt_scale_pk16_f16_bf6_sl(<3 x i32> inreg %src, ptr
}
define amdgpu_ps void @test_cvt_scale_pk16_bf16_bf6_vv(<3 x i32> %src, i32 %scale, ptr addrspace(1) %out) {
-; GFX1250-LABEL: test_cvt_scale_pk16_bf16_bf6_vv:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_cvt_scale_pk16_bf16_bf6 v[6:13], v[0:2], v3 scale_sel:6
-; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: global_store_b128 v[4:5], v[10:13], off offset:16
-; GFX1250-NEXT: global_store_b128 v[4:5], v[6:9], off
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: test_cvt_scale_pk16_bf16_bf6_vv:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_cvt_scale_pk16_bf16_bf6 v[6:13], v[0:2], v3 scale_sel:6
+; GFX1250-SDAG-NEXT: s_clause 0x1
+; GFX1250-SDAG-NEXT: global_store_b128 v[4:5], v[10:13], off offset:16
+; GFX1250-SDAG-NEXT: global_store_b128 v[4:5], v[6:9], off
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: test_cvt_scale_pk16_bf16_bf6_vv:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_cvt_scale_pk16_bf16_bf6 v[6:13], v[0:2], v3 scale_sel:6
+; GFX1250-GISEL-NEXT: s_clause 0x1
+; GFX1250-GISEL-NEXT: global_store_b128 v[4:5], v[6:9], off
+; GFX1250-GISEL-NEXT: global_store_b128 v[4:5], v[10:13], off offset:16
+; GFX1250-GISEL-NEXT: s_endpgm
%cvt = tail call <16 x bfloat> @llvm.amdgcn.cvt.scale.pk16.bf16.bf6(<3 x i32> %src, i32 %scale, i32 6)
store <16 x bfloat> %cvt, ptr addrspace(1) %out, align 8
ret void
}
define amdgpu_ps void @test_cvt_scale_pk16_bf16_bf6_sl(<3 x i32> inreg %src, ptr addrspace(1) %out) {
-; GFX1250-LABEL: test_cvt_scale_pk16_bf16_bf6_sl:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_dual_mov_b32 v10, s0 :: v_dual_mov_b32 v11, s1
-; GFX1250-NEXT: v_mov_b32_e32 v12, s2
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cvt_scale_pk16_bf16_bf6 v[2:9], v[10:12], 0x64 scale_sel:8
-; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16
-; GFX1250-NEXT: global_store_b128 v[0:1], v[2:5], off
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: test_cvt_scale_pk16_bf16_bf6_sl:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v10, s0 :: v_dual_mov_b32 v11, s1
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v12, s2
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_cvt_scale_pk16_bf16_bf6 v[2:9], v[10:12], 0x64 scale_sel:8
+; GFX1250-SDAG-NEXT: s_clause 0x1
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: test_cvt_scale_pk16_bf16_bf6_sl:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v12, s2 :: v_dual_mov_b32 v11, s1
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v10, s0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_cvt_scale_pk16_bf16_bf6 v[2:9], v[10:12], 0x64 scale_sel:8
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v2
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s1, v3
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s2, v4
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s3, v5
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s4, v6
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s5, v7
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s6, v8
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s7, v9
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7]
+; GFX1250-GISEL-NEXT: s_clause 0x1
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16
+; GFX1250-GISEL-NEXT: s_endpgm
%cvt = tail call <16 x bfloat> @llvm.amdgcn.cvt.scale.pk16.bf16.bf6(<3 x i32> %src, i32 100, i32 8)
store <16 x bfloat> %cvt, ptr addrspace(1) %out, align 8
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.pk.gfx950.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.pk.gfx950.ll
index 49545e2af4ef2..b58b7ae2fc582 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.pk.gfx950.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.pk.gfx950.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
; RUN: llc -global-isel=0 -mtriple=amdgpu9.50 < %s | FileCheck --check-prefixes=GCN,GFX950-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.50 < %s | FileCheck --check-prefixes=GCN,GFX950-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.50 < %s | FileCheck --check-prefixes=GCN,GFX950-GISEL %s
declare <6 x i32> @llvm.amdgcn.cvt.scalef32.2xpk16.bf6.f32(<16 x float> %src0, <16 x float> %src1, float %scale)
declare <6 x i32> @llvm.amdgcn.cvt.scalef32.2xpk16.fp6.f32(<16 x float> %src0, <16 x float> %src1, float %scale)
@@ -682,12 +682,19 @@ define <2 x i16> @test_cvt_scalef32_pk_fp8_bf16_imm1(<2 x i16> %old, float %scal
}
define <2 x i16> @test_cvt_scalef32_pk_fp8_bf16_imm2(<2 x i16> %old, float %scale) {
-; GCN-LABEL: test_cvt_scalef32_pk_fp8_bf16_imm2:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: s_mov_b32 s0, 0x40004080
-; GCN-NEXT: v_cvt_scalef32_pk_fp8_bf16 v0, s0, v1
-; GCN-NEXT: s_setpc_b64 s[30:31]
+; GFX950-SDAG-LABEL: test_cvt_scalef32_pk_fp8_bf16_imm2:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: s_mov_b32 s0, 0x40004080
+; GFX950-SDAG-NEXT: v_cvt_scalef32_pk_fp8_bf16 v0, s0, v1
+; GFX950-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-GISEL-LABEL: test_cvt_scalef32_pk_fp8_bf16_imm2:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, 0x40004080
+; GFX950-GISEL-NEXT: v_cvt_scalef32_pk_fp8_bf16 v0, v2, v1
+; GFX950-GISEL-NEXT: s_setpc_b64 s[30:31]
%ret = tail call <2 x i16> @llvm.amdgcn.cvt.scalef32.pk.fp8.bf16(<2 x i16> %old, <2 x bfloat> <bfloat 4.0, bfloat 2.0>, float %scale, i1 false)
ret <2 x i16> %ret
}
@@ -771,12 +778,19 @@ define <2 x i16> @test_cvt_scalef32_pk_bf8_bf16_imm1(<2 x i16> %old, float %scal
}
define <2 x i16> @test_cvt_scalef32_pk_bf8_bf16_imm2(<2 x i16> %old, float %scale) {
-; GCN-LABEL: test_cvt_scalef32_pk_bf8_bf16_imm2:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: s_mov_b32 s0, 0x40004080
-; GCN-NEXT: v_cvt_scalef32_pk_bf8_bf16 v0, s0, v1
-; GCN-NEXT: s_setpc_b64 s[30:31]
+; GFX950-SDAG-LABEL: test_cvt_scalef32_pk_bf8_bf16_imm2:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: s_mov_b32 s0, 0x40004080
+; GFX950-SDAG-NEXT: v_cvt_scalef32_pk_bf8_bf16 v0, s0, v1
+; GFX950-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-GISEL-LABEL: test_cvt_scalef32_pk_bf8_bf16_imm2:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, 0x40004080
+; GFX950-GISEL-NEXT: v_cvt_scalef32_pk_bf8_bf16 v0, v2, v1
+; GFX950-GISEL-NEXT: s_setpc_b64 s[30:31]
%ret = tail call <2 x i16> @llvm.amdgcn.cvt.scalef32.pk.bf8.bf16(<2 x i16> %old, <2 x bfloat> <bfloat 4.0, bfloat 2.0>, float %scale, i1 false)
ret <2 x i16> %ret
}
@@ -1103,35 +1117,60 @@ define <32 x half> @test_cvt_scalef32_pk32_f16_fp6_sl(<6 x i32> inreg %src) {
}
define <32 x bfloat> @test_cvt_scalef32_pk32_bf16_fp6_vv(<6 x i32> %src, float %scale) {
-; GCN-LABEL: test_cvt_scalef32_pk32_bf16_fp6_vv:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v22, v6
-; GCN-NEXT: v_mov_b32_e32 v21, v5
-; GCN-NEXT: v_mov_b32_e32 v20, v4
-; GCN-NEXT: v_mov_b32_e32 v19, v3
-; GCN-NEXT: v_mov_b32_e32 v18, v2
-; GCN-NEXT: v_mov_b32_e32 v17, v1
-; GCN-NEXT: v_mov_b32_e32 v16, v0
-; GCN-NEXT: v_cvt_scalef32_pk32_bf16_fp6 v[0:15], v[16:21], v22
-; GCN-NEXT: s_setpc_b64 s[30:31]
+; GFX950-SDAG-LABEL: test_cvt_scalef32_pk32_bf16_fp6_vv:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v22, v6
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v21, v5
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v20, v4
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v19, v3
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v18, v2
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v17, v1
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v16, v0
+; GFX950-SDAG-NEXT: v_cvt_scalef32_pk32_bf16_fp6 v[0:15], v[16:21], v22
+; GFX950-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-GISEL-LABEL: test_cvt_scalef32_pk32_bf16_fp6_vv:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v16, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v17, v1
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v18, v2
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v19, v3
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v20, v4
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v21, v5
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v22, v6
+; GFX950-GISEL-NEXT: v_cvt_scalef32_pk32_bf16_fp6 v[0:15], v[16:21], v22
+; GFX950-GISEL-NEXT: s_setpc_b64 s[30:31]
%ret = tail call <32 x bfloat> @llvm.amdgcn.cvt.scalef32.pk32.bf16.fp6(<6 x i32> %src, float %scale)
ret <32 x bfloat> %ret
}
define <32 x bfloat> @test_cvt_scalef32_pk32_bf16_fp6_sl(<6 x i32> inreg %src) {
-; GCN-LABEL: test_cvt_scalef32_pk32_bf16_fp6_sl:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v16, s0
-; GCN-NEXT: v_mov_b32_e32 v17, s1
-; GCN-NEXT: v_mov_b32_e32 v18, s2
-; GCN-NEXT: v_mov_b32_e32 v19, s3
-; GCN-NEXT: v_mov_b32_e32 v20, s16
-; GCN-NEXT: v_mov_b32_e32 v21, s17
-; GCN-NEXT: s_mov_b32 s0, 0x42c80000
-; GCN-NEXT: v_cvt_scalef32_pk32_bf16_fp6 v[0:15], v[16:21], s0
-; GCN-NEXT: s_setpc_b64 s[30:31]
+; GFX950-SDAG-LABEL: test_cvt_scalef32_pk32_bf16_fp6_sl:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v16, s0
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v17, s1
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v18, s2
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v19, s3
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v20, s16
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v21, s17
+; GFX950-SDAG-NEXT: s_mov_b32 s0, 0x42c80000
+; GFX950-SDAG-NEXT: v_cvt_scalef32_pk32_bf16_fp6 v[0:15], v[16:21], s0
+; GFX950-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-GISEL-LABEL: test_cvt_scalef32_pk32_bf16_fp6_sl:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: s_mov_b32 s4, s16
+; GFX950-GISEL-NEXT: s_mov_b32 s5, s17
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[4:5]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v22, 0x42c80000
+; GFX950-GISEL-NEXT: v_cvt_scalef32_pk32_bf16_fp6 v[0:15], v[16:21], v22
+; GFX950-GISEL-NEXT: s_setpc_b64 s[30:31]
%ret = tail call <32 x bfloat> @llvm.amdgcn.cvt.scalef32.pk32.bf16.fp6(<6 x i32> %src, float 100.0)
ret <32 x bfloat> %ret
}
@@ -1196,35 +1235,60 @@ define <32 x half> @test_cvt_scalef32_pk32_f16_bf6_sl(<6 x i32> inreg %src) {
}
define <32 x bfloat> @test_cvt_scalef32_pk32_bf16_bf6_vv(<6 x i32> %src, float %scale) {
-; GCN-LABEL: test_cvt_scalef32_pk32_bf16_bf6_vv:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v22, v6
-; GCN-NEXT: v_mov_b32_e32 v21, v5
-; GCN-NEXT: v_mov_b32_e32 v20, v4
-; GCN-NEXT: v_mov_b32_e32 v19, v3
-; GCN-NEXT: v_mov_b32_e32 v18, v2
-; GCN-NEXT: v_mov_b32_e32 v17, v1
-; GCN-NEXT: v_mov_b32_e32 v16, v0
-; GCN-NEXT: v_cvt_scalef32_pk32_bf16_bf6 v[0:15], v[16:21], v22
-; GCN-NEXT: s_setpc_b64 s[30:31]
+; GFX950-SDAG-LABEL: test_cvt_scalef32_pk32_bf16_bf6_vv:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v22, v6
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v21, v5
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v20, v4
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v19, v3
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v18, v2
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v17, v1
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v16, v0
+; GFX950-SDAG-NEXT: v_cvt_scalef32_pk32_bf16_bf6 v[0:15], v[16:21], v22
+; GFX950-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-GISEL-LABEL: test_cvt_scalef32_pk32_bf16_bf6_vv:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v16, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v17, v1
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v18, v2
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v19, v3
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v20, v4
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v21, v5
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v22, v6
+; GFX950-GISEL-NEXT: v_cvt_scalef32_pk32_bf16_bf6 v[0:15], v[16:21], v22
+; GFX950-GISEL-NEXT: s_setpc_b64 s[30:31]
%ret = tail call <32 x bfloat> @llvm.amdgcn.cvt.scalef32.pk32.bf16.bf6(<6 x i32> %src, float %scale)
ret <32 x bfloat> %ret
}
define <32 x bfloat> @test_cvt_scalef32_pk32_bf16_bf6_sl(<6 x i32> inreg %src) {
-; GCN-LABEL: test_cvt_scalef32_pk32_bf16_bf6_sl:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v16, s0
-; GCN-NEXT: v_mov_b32_e32 v17, s1
-; GCN-NEXT: v_mov_b32_e32 v18, s2
-; GCN-NEXT: v_mov_b32_e32 v19, s3
-; GCN-NEXT: v_mov_b32_e32 v20, s16
-; GCN-NEXT: v_mov_b32_e32 v21, s17
-; GCN-NEXT: s_mov_b32 s0, 0x42c80000
-; GCN-NEXT: v_cvt_scalef32_pk32_bf16_bf6 v[0:15], v[16:21], s0
-; GCN-NEXT: s_setpc_b64 s[30:31]
+; GFX950-SDAG-LABEL: test_cvt_scalef32_pk32_bf16_bf6_sl:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v16, s0
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v17, s1
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v18, s2
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v19, s3
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v20, s16
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v21, s17
+; GFX950-SDAG-NEXT: s_mov_b32 s0, 0x42c80000
+; GFX950-SDAG-NEXT: v_cvt_scalef32_pk32_bf16_bf6 v[0:15], v[16:21], s0
+; GFX950-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-GISEL-LABEL: test_cvt_scalef32_pk32_bf16_bf6_sl:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: s_mov_b32 s4, s16
+; GFX950-GISEL-NEXT: s_mov_b32 s5, s17
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[4:5]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v22, 0x42c80000
+; GFX950-GISEL-NEXT: v_cvt_scalef32_pk32_bf16_bf6 v[0:15], v[16:21], v22
+; GFX950-GISEL-NEXT: s_setpc_b64 s[30:31]
%ret = tail call <32 x bfloat> @llvm.amdgcn.cvt.scalef32.pk32.bf16.bf6(<6 x i32> %src, float 100.0)
ret <32 x bfloat> %ret
}
@@ -1446,13 +1510,21 @@ define i32 @test_cvt_scalef32_fp4_bf16_imm1(float %scale, i32 %old) {
}
define i32 @test_cvt_scalef32_fp4_bf16_imm2(float %scale, i32 %old) {
-; GCN-LABEL: test_cvt_scalef32_fp4_bf16_imm2:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: s_mov_b32 s0, 0x40004080
-; GCN-NEXT: v_cvt_scalef32_pk_fp4_bf16 v1, s0, v0
-; GCN-NEXT: v_mov_b32_e32 v0, v1
-; GCN-NEXT: s_setpc_b64 s[30:31]
+; GFX950-SDAG-LABEL: test_cvt_scalef32_fp4_bf16_imm2:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: s_mov_b32 s0, 0x40004080
+; GFX950-SDAG-NEXT: v_cvt_scalef32_pk_fp4_bf16 v1, s0, v0
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, v1
+; GFX950-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-GISEL-LABEL: test_cvt_scalef32_fp4_bf16_imm2:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, 0x40004080
+; GFX950-GISEL-NEXT: v_cvt_scalef32_pk_fp4_bf16 v1, v2, v0
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, v1
+; GFX950-GISEL-NEXT: s_setpc_b64 s[30:31]
%ret = tail call i32 @llvm.amdgcn.cvt.scalef32.pk.fp4.bf16(i32 %old, <2 x bfloat> <bfloat 4.0, bfloat 2.0>, float %scale, i32 0)
ret i32 %ret
}
@@ -2210,6 +2282,18 @@ define <2 x i16> @test_cvt_scalef32_pk_fp8_bf16_word0_inreg_src(<2 x i16> %old,
ret <2 x i16> %ret
}
+define <2 x i16> @test_cvt_scalef32_pk_fp8_bf16_word0_ss(<2 x i16> inreg %old, <2 x bfloat> inreg %src, float inreg %scale) {
+; GCN-LABEL: test_cvt_scalef32_pk_fp8_bf16_word0_ss:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: v_mov_b32_e32 v1, s2
+; GCN-NEXT: v_cvt_scalef32_pk_fp8_bf16 v0, s1, v1
+; GCN-NEXT: s_setpc_b64 s[30:31]
+ %ret = tail call <2 x i16> @llvm.amdgcn.cvt.scalef32.pk.fp8.bf16(<2 x i16> %old, <2 x bfloat> %src, float %scale, i1 false)
+ ret <2 x i16> %ret
+}
+
define <2 x i16> @test_cvt_scalef32_pk_fp8_bf16_word1_inreg_src(<2 x i16> %old, <2 x bfloat> inreg %src, float %scale) {
; GCN-LABEL: test_cvt_scalef32_pk_fp8_bf16_word1_inreg_src:
; GCN: ; %bb.0:
@@ -2262,6 +2346,18 @@ define <2 x i16> @test_cvt_scalef32_pk_bf8_bf16_word0_inreg_src(<2 x i16> %old,
ret <2 x i16> %ret
}
+define <2 x i16> @test_cvt_scalef32_pk_bf8_bf16_word0_ss(<2 x i16> inreg %old, <2 x bfloat> inreg %src, float inreg %scale) {
+; GCN-LABEL: test_cvt_scalef32_pk_bf8_bf16_word0_ss:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: v_mov_b32_e32 v1, s2
+; GCN-NEXT: v_cvt_scalef32_pk_bf8_bf16 v0, s1, v1
+; GCN-NEXT: s_setpc_b64 s[30:31]
+ %ret = tail call <2 x i16> @llvm.amdgcn.cvt.scalef32.pk.bf8.bf16(<2 x i16> %old, <2 x bfloat> %src, float %scale, i1 false)
+ ret <2 x i16> %ret
+}
+
define <2 x i16> @test_cvt_scalef32_pk_bf8_bf16_word1_inreg_src(<2 x i16> %old, <2 x bfloat> inreg %src, float %scale) {
; GCN-LABEL: test_cvt_scalef32_pk_bf8_bf16_word1_inreg_src:
; GCN: ; %bb.0:
@@ -2437,6 +2533,17 @@ define <2 x bfloat> @test_cvt_scale_bf16_fp4_byte0_inreg_src(i32 inreg %src, flo
ret <2 x bfloat> %ret
}
+define <2 x bfloat> @test_cvt_scale_bf16_fp4_byte0_ss(i32 inreg %src, float inreg %scale) {
+; GCN-LABEL: test_cvt_scale_bf16_fp4_byte0_ss:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mov_b32_e32 v0, s1
+; GCN-NEXT: v_cvt_scalef32_pk_bf16_fp4 v0, s0, v0
+; GCN-NEXT: s_setpc_b64 s[30:31]
+ %ret = tail call <2 x bfloat> @llvm.amdgcn.cvt.scalef32.pk.bf16.fp4(i32 %src, float %scale, i32 0)
+ ret <2 x bfloat> %ret
+}
+
define <2 x bfloat> @test_cvt_scale_bf16_fp4_byte1_inreg_src(i32 inreg %src, float %scale) {
; GCN-LABEL: test_cvt_scale_bf16_fp4_byte1_inreg_src:
; GCN: ; %bb.0:
@@ -2665,35 +2772,59 @@ define <32 x half> @test_cvt_scalef32_pk32_f16_fp6_sl_inreg_src(<6 x i32> inreg
}
define <32 x bfloat> @test_cvt_scalef32_pk32_bf16_fp6_vv_inreg_src(<6 x i32> inreg %src, float %scale) {
-; GCN-LABEL: test_cvt_scalef32_pk32_bf16_fp6_vv_inreg_src:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v16, v0
-; GCN-NEXT: v_mov_b32_e32 v18, s0
-; GCN-NEXT: v_mov_b32_e32 v19, s1
-; GCN-NEXT: v_mov_b32_e32 v20, s2
-; GCN-NEXT: v_mov_b32_e32 v21, s3
-; GCN-NEXT: v_mov_b32_e32 v22, s16
-; GCN-NEXT: v_mov_b32_e32 v23, s17
-; GCN-NEXT: v_cvt_scalef32_pk32_bf16_fp6 v[0:15], v[18:23], v16
-; GCN-NEXT: s_setpc_b64 s[30:31]
+; GFX950-SDAG-LABEL: test_cvt_scalef32_pk32_bf16_fp6_vv_inreg_src:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v16, v0
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v18, s0
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v19, s1
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v20, s2
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v21, s3
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v22, s16
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v23, s17
+; GFX950-SDAG-NEXT: v_cvt_scalef32_pk32_bf16_fp6 v[0:15], v[18:23], v16
+; GFX950-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-GISEL-LABEL: test_cvt_scalef32_pk32_bf16_fp6_vv_inreg_src:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: s_mov_b32 s4, s16
+; GFX950-GISEL-NEXT: s_mov_b32 s5, s17
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v16, v0
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[22:23], s[4:5]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[2:3]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[0:1]
+; GFX950-GISEL-NEXT: v_cvt_scalef32_pk32_bf16_fp6 v[0:15], v[18:23], v16
+; GFX950-GISEL-NEXT: s_setpc_b64 s[30:31]
%ret = tail call <32 x bfloat> @llvm.amdgcn.cvt.scalef32.pk32.bf16.fp6(<6 x i32> %src, float %scale)
ret <32 x bfloat> %ret
}
define <32 x bfloat> @test_cvt_scalef32_pk32_bf16_fp6_sl_inreg_src(<6 x i32> inreg inreg %src) {
-; GCN-LABEL: test_cvt_scalef32_pk32_bf16_fp6_sl_inreg_src:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v16, s0
-; GCN-NEXT: v_mov_b32_e32 v17, s1
-; GCN-NEXT: v_mov_b32_e32 v18, s2
-; GCN-NEXT: v_mov_b32_e32 v19, s3
-; GCN-NEXT: v_mov_b32_e32 v20, s16
-; GCN-NEXT: v_mov_b32_e32 v21, s17
-; GCN-NEXT: s_mov_b32 s0, 0x42c80000
-; GCN-NEXT: v_cvt_scalef32_pk32_bf16_fp6 v[0:15], v[16:21], s0
-; GCN-NEXT: s_setpc_b64 s[30:31]
+; GFX950-SDAG-LABEL: test_cvt_scalef32_pk32_bf16_fp6_sl_inreg_src:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v16, s0
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v17, s1
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v18, s2
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v19, s3
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v20, s16
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v21, s17
+; GFX950-SDAG-NEXT: s_mov_b32 s0, 0x42c80000
+; GFX950-SDAG-NEXT: v_cvt_scalef32_pk32_bf16_fp6 v[0:15], v[16:21], s0
+; GFX950-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-GISEL-LABEL: test_cvt_scalef32_pk32_bf16_fp6_sl_inreg_src:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: s_mov_b32 s4, s16
+; GFX950-GISEL-NEXT: s_mov_b32 s5, s17
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[4:5]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v22, 0x42c80000
+; GFX950-GISEL-NEXT: v_cvt_scalef32_pk32_bf16_fp6 v[0:15], v[16:21], v22
+; GFX950-GISEL-NEXT: s_setpc_b64 s[30:31]
%ret = tail call <32 x bfloat> @llvm.amdgcn.cvt.scalef32.pk32.bf16.fp6(<6 x i32> %src, float 100.0)
ret <32 x bfloat> %ret
}
@@ -2784,35 +2915,59 @@ define <32 x half> @test_cvt_scalef32_pk32_f16_bf6_sl_inreg_src(<6 x i32> inreg
}
define <32 x bfloat> @test_cvt_scalef32_pk32_bf16_bf6_vv_inreg_src(<6 x i32> inreg %src, float %scale) {
-; GCN-LABEL: test_cvt_scalef32_pk32_bf16_bf6_vv_inreg_src:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v16, v0
-; GCN-NEXT: v_mov_b32_e32 v18, s0
-; GCN-NEXT: v_mov_b32_e32 v19, s1
-; GCN-NEXT: v_mov_b32_e32 v20, s2
-; GCN-NEXT: v_mov_b32_e32 v21, s3
-; GCN-NEXT: v_mov_b32_e32 v22, s16
-; GCN-NEXT: v_mov_b32_e32 v23, s17
-; GCN-NEXT: v_cvt_scalef32_pk32_bf16_bf6 v[0:15], v[18:23], v16
-; GCN-NEXT: s_setpc_b64 s[30:31]
+; GFX950-SDAG-LABEL: test_cvt_scalef32_pk32_bf16_bf6_vv_inreg_src:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v16, v0
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v18, s0
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v19, s1
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v20, s2
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v21, s3
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v22, s16
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v23, s17
+; GFX950-SDAG-NEXT: v_cvt_scalef32_pk32_bf16_bf6 v[0:15], v[18:23], v16
+; GFX950-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-GISEL-LABEL: test_cvt_scalef32_pk32_bf16_bf6_vv_inreg_src:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: s_mov_b32 s4, s16
+; GFX950-GISEL-NEXT: s_mov_b32 s5, s17
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v16, v0
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[22:23], s[4:5]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[2:3]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[0:1]
+; GFX950-GISEL-NEXT: v_cvt_scalef32_pk32_bf16_bf6 v[0:15], v[18:23], v16
+; GFX950-GISEL-NEXT: s_setpc_b64 s[30:31]
%ret = tail call <32 x bfloat> @llvm.amdgcn.cvt.scalef32.pk32.bf16.bf6(<6 x i32> %src, float %scale)
ret <32 x bfloat> %ret
}
define <32 x bfloat> @test_cvt_scalef32_pk32_bf16_bf6_sl_inreg_src(<6 x i32> inreg inreg %src) {
-; GCN-LABEL: test_cvt_scalef32_pk32_bf16_bf6_sl_inreg_src:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v16, s0
-; GCN-NEXT: v_mov_b32_e32 v17, s1
-; GCN-NEXT: v_mov_b32_e32 v18, s2
-; GCN-NEXT: v_mov_b32_e32 v19, s3
-; GCN-NEXT: v_mov_b32_e32 v20, s16
-; GCN-NEXT: v_mov_b32_e32 v21, s17
-; GCN-NEXT: s_mov_b32 s0, 0x42c80000
-; GCN-NEXT: v_cvt_scalef32_pk32_bf16_bf6 v[0:15], v[16:21], s0
-; GCN-NEXT: s_setpc_b64 s[30:31]
+; GFX950-SDAG-LABEL: test_cvt_scalef32_pk32_bf16_bf6_sl_inreg_src:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v16, s0
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v17, s1
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v18, s2
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v19, s3
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v20, s16
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v21, s17
+; GFX950-SDAG-NEXT: s_mov_b32 s0, 0x42c80000
+; GFX950-SDAG-NEXT: v_cvt_scalef32_pk32_bf16_bf6 v[0:15], v[16:21], s0
+; GFX950-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-GISEL-LABEL: test_cvt_scalef32_pk32_bf16_bf6_sl_inreg_src:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: s_mov_b32 s4, s16
+; GFX950-GISEL-NEXT: s_mov_b32 s5, s17
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[4:5]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v22, 0x42c80000
+; GFX950-GISEL-NEXT: v_cvt_scalef32_pk32_bf16_bf6 v[0:15], v[16:21], v22
+; GFX950-GISEL-NEXT: s_setpc_b64 s[30:31]
%ret = tail call <32 x bfloat> @llvm.amdgcn.cvt.scalef32.pk32.bf16.bf6(<6 x i32> %src, float 100.0)
ret <32 x bfloat> %ret
}
@@ -2889,6 +3044,17 @@ define <2 x bfloat> @test_cvt_scalef32_pk_bf16_fp8_word0_inreg_src(i32 inreg %sr
ret <2 x bfloat> %ret
}
+define <2 x bfloat> @test_cvt_scalef32_pk_bf16_fp8_word0_ss(i32 inreg %src, float inreg %scale) {
+; GCN-LABEL: test_cvt_scalef32_pk_bf16_fp8_word0_ss:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mov_b32_e32 v0, s1
+; GCN-NEXT: v_cvt_scalef32_pk_bf16_fp8 v0, s0, v0
+; GCN-NEXT: s_setpc_b64 s[30:31]
+ %ret = tail call <2 x bfloat> @llvm.amdgcn.cvt.scalef32.pk.bf16.fp8(i32 %src, float %scale, i1 false)
+ ret <2 x bfloat> %ret
+}
+
define <2 x bfloat> @test_cvt_scalef32_pk_bf16_fp8_word1_inreg_src(i32 inreg %src, float %scale) {
; GCN-LABEL: test_cvt_scalef32_pk_bf16_fp8_word1_inreg_src:
; GCN: ; %bb.0:
@@ -2909,6 +3075,17 @@ define <2 x bfloat> @test_cvt_scalef32_pk_bf16_bf8_word0_inreg_src(i32 inreg %sr
ret <2 x bfloat> %ret
}
+define <2 x bfloat> @test_cvt_scalef32_pk_bf16_bf8_word0_ss(i32 inreg %src, float inreg %scale) {
+; GCN-LABEL: test_cvt_scalef32_pk_bf16_bf8_word0_ss:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mov_b32_e32 v0, s1
+; GCN-NEXT: v_cvt_scalef32_pk_bf16_bf8 v0, s0, v0
+; GCN-NEXT: s_setpc_b64 s[30:31]
+ %ret = tail call <2 x bfloat> @llvm.amdgcn.cvt.scalef32.pk.bf16.bf8(i32 %src, float %scale, i1 false)
+ ret <2 x bfloat> %ret
+}
+
define <2 x bfloat> @test_cvt_scalef32_pk_bf16_bf8_word1_inreg_src(i32 inreg %src, float %scale) {
; GCN-LABEL: test_cvt_scalef32_pk_bf16_bf8_word1_inreg_src:
; GCN: ; %bb.0:
@@ -2989,6 +3166,18 @@ define i32 @test_cvt_scalef32_fp4_bf16_byte0_inreg_src(<2 x bfloat> inreg %src0,
ret i32 %ret
}
+define i32 @test_cvt_scalef32_fp4_bf16_byte0_ss(<2 x bfloat> inreg %src0, float inreg %scale, i32 inreg %old) {
+; GCN-LABEL: test_cvt_scalef32_fp4_bf16_byte0_ss:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mov_b32_e32 v0, s2
+; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: v_cvt_scalef32_pk_fp4_bf16 v0, s0, v1
+; GCN-NEXT: s_setpc_b64 s[30:31]
+ %ret = tail call i32 @llvm.amdgcn.cvt.scalef32.pk.fp4.bf16(i32 %old, <2 x bfloat> %src0, float %scale, i32 0)
+ ret i32 %ret
+}
+
define i32 @test_cvt_scalef32_fp4_bf16_byte1_inreg_src(<2 x bfloat> inreg %src0, float %scale, i32 %old) {
; GCN-LABEL: test_cvt_scalef32_fp4_bf16_byte1_inreg_src:
; GCN: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.pk.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.pk.ll
index 9bb3faea51c08..fa690fd1492a4 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.pk.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.pk.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
; RUN: llc -global-isel=0 -mtriple=amdgpu9.50 < %s | FileCheck -check-prefix=GFX950-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.50 < %s | FileCheck -check-prefix=GFX950-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.50 < %s | FileCheck -check-prefix=GFX950-GISEL %s
declare <6 x i32> @llvm.amdgcn.cvt.scalef32.pk32.bf6.bf16(<32 x bfloat> %src, float %scale)
declare <6 x i32> @llvm.amdgcn.cvt.scalef32.pk32.bf6.f16(<32 x half> %src, float %scale)
@@ -19,11 +19,11 @@ define amdgpu_ps void @test_scalef32_pk32_bf6_bf16_vv(<32 x bfloat> %src, float
;
; GFX950-GISEL-LABEL: test_scalef32_pk32_bf6_bf16_vv:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v25, v18
; GFX950-GISEL-NEXT: v_mov_b32_e32 v24, v17
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v25, v18
; GFX950-GISEL-NEXT: v_cvt_scalef32_pk32_bf6_bf16 v[18:23], v[0:15], v16
-; GFX950-GISEL-NEXT: global_store_dwordx2 v[24:25], v[22:23], off offset:16
; GFX950-GISEL-NEXT: global_store_dwordx4 v[24:25], v[18:21], off
+; GFX950-GISEL-NEXT: global_store_dwordx2 v[24:25], v[22:23], off offset:16
; GFX950-GISEL-NEXT: s_endpgm
%cvt = tail call <6 x i32> @llvm.amdgcn.cvt.scalef32.pk32.bf6.bf16(<32 x bfloat> %src, float %scale)
store <6 x i32> %cvt, ptr addrspace(1) %out, align 8
@@ -57,26 +57,29 @@ define amdgpu_ps void @test_scalef32_pk32_bf6_bf16_sl(<32 x bfloat> inreg %src,
;
; GFX950-GISEL-LABEL: test_scalef32_pk32_bf6_bf16_sl:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, s0
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, s1
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, s2
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, s3
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v6, s4
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v7, s5
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v8, s6
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v9, s7
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v10, s8
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v11, s9
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v12, s10
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v13, s11
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v14, s12
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v15, s13
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v16, s14
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v17, s15
-; GFX950-GISEL-NEXT: s_mov_b32 s0, 0x42c80000
-; GFX950-GISEL-NEXT: v_cvt_scalef32_pk32_bf6_bf16 v[18:23], v[2:17], s0
-; GFX950-GISEL-NEXT: global_store_dwordx2 v[0:1], v[22:23], off offset:16
-; GFX950-GISEL-NEXT: global_store_dwordx4 v[0:1], v[18:21], off
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[14:15]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[12:13]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[10:11]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[8:9]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v24, 0x42c80000
+; GFX950-GISEL-NEXT: v_cvt_scalef32_pk32_bf6_bf16 v[18:23], v[2:17], v24
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s0, v18
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s1, v19
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s2, v20
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s3, v21
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s4, v22
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s5, v23
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-GISEL-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
+; GFX950-GISEL-NEXT: s_nop 1
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[4:5]
+; GFX950-GISEL-NEXT: global_store_dwordx2 v[0:1], v[2:3], off offset:16
; GFX950-GISEL-NEXT: s_endpgm
%cvt = tail call <6 x i32> @llvm.amdgcn.cvt.scalef32.pk32.bf6.bf16(<32 x bfloat> %src, float 100.0)
store <6 x i32> %cvt, ptr addrspace(1) %out, align 8
@@ -174,11 +177,11 @@ define amdgpu_ps void @test_scalef32_pk32_fp6_bf16_vv(<32 x bfloat> %src, float
;
; GFX950-GISEL-LABEL: test_scalef32_pk32_fp6_bf16_vv:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v25, v18
; GFX950-GISEL-NEXT: v_mov_b32_e32 v24, v17
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v25, v18
; GFX950-GISEL-NEXT: v_cvt_scalef32_pk32_fp6_bf16 v[18:23], v[0:15], v16
-; GFX950-GISEL-NEXT: global_store_dwordx2 v[24:25], v[22:23], off offset:16
; GFX950-GISEL-NEXT: global_store_dwordx4 v[24:25], v[18:21], off
+; GFX950-GISEL-NEXT: global_store_dwordx2 v[24:25], v[22:23], off offset:16
; GFX950-GISEL-NEXT: s_endpgm
%cvt = tail call <6 x i32> @llvm.amdgcn.cvt.scalef32.pk32.fp6.bf16(<32 x bfloat> %src, float %scale)
store <6 x i32> %cvt, ptr addrspace(1) %out, align 8
@@ -212,26 +215,29 @@ define amdgpu_ps void @test_scalef32_pk32_fp6_bf16_sl(<32 x bfloat> inreg %src,
;
; GFX950-GISEL-LABEL: test_scalef32_pk32_fp6_bf16_sl:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, s0
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, s1
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, s2
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, s3
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v6, s4
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v7, s5
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v8, s6
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v9, s7
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v10, s8
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v11, s9
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v12, s10
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v13, s11
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v14, s12
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v15, s13
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v16, s14
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v17, s15
-; GFX950-GISEL-NEXT: s_mov_b32 s0, 0x42c80000
-; GFX950-GISEL-NEXT: v_cvt_scalef32_pk32_fp6_bf16 v[18:23], v[2:17], s0
-; GFX950-GISEL-NEXT: global_store_dwordx2 v[0:1], v[22:23], off offset:16
-; GFX950-GISEL-NEXT: global_store_dwordx4 v[0:1], v[18:21], off
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[14:15]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[12:13]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[10:11]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[8:9]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v24, 0x42c80000
+; GFX950-GISEL-NEXT: v_cvt_scalef32_pk32_fp6_bf16 v[18:23], v[2:17], v24
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s0, v18
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s1, v19
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s2, v20
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s3, v21
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s4, v22
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s5, v23
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-GISEL-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
+; GFX950-GISEL-NEXT: s_nop 1
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[4:5]
+; GFX950-GISEL-NEXT: global_store_dwordx2 v[0:1], v[2:3], off offset:16
; GFX950-GISEL-NEXT: s_endpgm
%cvt = tail call <6 x i32> @llvm.amdgcn.cvt.scalef32.pk32.fp6.bf16(<32 x bfloat> %src, float 100.0)
store <6 x i32> %cvt, ptr addrspace(1) %out, align 8
@@ -346,26 +352,18 @@ define amdgpu_ps void @test_scalef32_pk32_bf6_bf16_vv_inreg_src(<32 x bfloat> in
; GFX950-GISEL-LABEL: test_scalef32_pk32_bf6_bf16_vv_inreg_src:
; GFX950-GISEL: ; %bb.0:
; GFX950-GISEL-NEXT: v_mov_b32_e32 v25, v2
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, s0
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, s1
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, s2
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, s3
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v6, s4
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v7, s5
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v8, s6
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v9, s7
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v10, s8
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v11, s9
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v12, s10
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v13, s11
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v14, s12
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v15, s13
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v16, s14
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v17, s15
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[14:15]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[12:13]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[10:11]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[8:9]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-GISEL-NEXT: v_mov_b32_e32 v24, v1
; GFX950-GISEL-NEXT: v_cvt_scalef32_pk32_bf6_bf16 v[18:23], v[2:17], v0
-; GFX950-GISEL-NEXT: global_store_dwordx2 v[24:25], v[22:23], off offset:16
; GFX950-GISEL-NEXT: global_store_dwordx4 v[24:25], v[18:21], off
+; GFX950-GISEL-NEXT: global_store_dwordx2 v[24:25], v[22:23], off offset:16
; GFX950-GISEL-NEXT: s_endpgm
%cvt = tail call <6 x i32> @llvm.amdgcn.cvt.scalef32.pk32.bf6.bf16(<32 x bfloat> %src, float %scale)
store <6 x i32> %cvt, ptr addrspace(1) %out, align 8
@@ -399,26 +397,29 @@ define amdgpu_ps void @test_scalef32_pk32_bf6_bf16_sl_inreg_src(<32 x bfloat> in
;
; GFX950-GISEL-LABEL: test_scalef32_pk32_bf6_bf16_sl_inreg_src:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, s0
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, s1
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, s2
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, s3
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v6, s4
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v7, s5
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v8, s6
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v9, s7
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v10, s8
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v11, s9
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v12, s10
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v13, s11
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v14, s12
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v15, s13
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v16, s14
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v17, s15
-; GFX950-GISEL-NEXT: s_mov_b32 s0, 0x42c80000
-; GFX950-GISEL-NEXT: v_cvt_scalef32_pk32_bf6_bf16 v[18:23], v[2:17], s0
-; GFX950-GISEL-NEXT: global_store_dwordx2 v[0:1], v[22:23], off offset:16
-; GFX950-GISEL-NEXT: global_store_dwordx4 v[0:1], v[18:21], off
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[14:15]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[12:13]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[10:11]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[8:9]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v24, 0x42c80000
+; GFX950-GISEL-NEXT: v_cvt_scalef32_pk32_bf6_bf16 v[18:23], v[2:17], v24
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s0, v18
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s1, v19
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s2, v20
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s3, v21
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s4, v22
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s5, v23
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-GISEL-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
+; GFX950-GISEL-NEXT: s_nop 1
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[4:5]
+; GFX950-GISEL-NEXT: global_store_dwordx2 v[0:1], v[2:3], off offset:16
; GFX950-GISEL-NEXT: s_endpgm
%cvt = tail call <6 x i32> @llvm.amdgcn.cvt.scalef32.pk32.bf6.bf16(<32 x bfloat> %src, float 100.0)
store <6 x i32> %cvt, ptr addrspace(1) %out, align 8
@@ -611,26 +612,18 @@ define amdgpu_ps void @test_scalef32_pk32_fp6_bf16_vv_inreg_src(<32 x bfloat> in
; GFX950-GISEL-LABEL: test_scalef32_pk32_fp6_bf16_vv_inreg_src:
; GFX950-GISEL: ; %bb.0:
; GFX950-GISEL-NEXT: v_mov_b32_e32 v25, v2
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, s0
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, s1
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, s2
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, s3
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v6, s4
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v7, s5
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v8, s6
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v9, s7
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v10, s8
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v11, s9
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v12, s10
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v13, s11
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v14, s12
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v15, s13
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v16, s14
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v17, s15
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[14:15]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[12:13]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[10:11]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[8:9]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-GISEL-NEXT: v_mov_b32_e32 v24, v1
; GFX950-GISEL-NEXT: v_cvt_scalef32_pk32_fp6_bf16 v[18:23], v[2:17], v0
-; GFX950-GISEL-NEXT: global_store_dwordx2 v[24:25], v[22:23], off offset:16
; GFX950-GISEL-NEXT: global_store_dwordx4 v[24:25], v[18:21], off
+; GFX950-GISEL-NEXT: global_store_dwordx2 v[24:25], v[22:23], off offset:16
; GFX950-GISEL-NEXT: s_endpgm
%cvt = tail call <6 x i32> @llvm.amdgcn.cvt.scalef32.pk32.fp6.bf16(<32 x bfloat> %src, float %scale)
store <6 x i32> %cvt, ptr addrspace(1) %out, align 8
@@ -664,26 +657,29 @@ define amdgpu_ps void @test_scalef32_pk32_fp6_bf16_sl_inreg_src(<32 x bfloat> in
;
; GFX950-GISEL-LABEL: test_scalef32_pk32_fp6_bf16_sl_inreg_src:
; GFX950-GISEL: ; %bb.0:
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, s0
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, s1
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, s2
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, s3
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v6, s4
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v7, s5
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v8, s6
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v9, s7
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v10, s8
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v11, s9
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v12, s10
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v13, s11
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v14, s12
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v15, s13
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v16, s14
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v17, s15
-; GFX950-GISEL-NEXT: s_mov_b32 s0, 0x42c80000
-; GFX950-GISEL-NEXT: v_cvt_scalef32_pk32_fp6_bf16 v[18:23], v[2:17], s0
-; GFX950-GISEL-NEXT: global_store_dwordx2 v[0:1], v[22:23], off offset:16
-; GFX950-GISEL-NEXT: global_store_dwordx4 v[0:1], v[18:21], off
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[14:15]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[12:13]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[10:11]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[8:9]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v24, 0x42c80000
+; GFX950-GISEL-NEXT: v_cvt_scalef32_pk32_fp6_bf16 v[18:23], v[2:17], v24
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s0, v18
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s1, v19
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s2, v20
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s3, v21
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s4, v22
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s5, v23
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-GISEL-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
+; GFX950-GISEL-NEXT: s_nop 1
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[4:5]
+; GFX950-GISEL-NEXT: global_store_dwordx2 v[0:1], v[2:3], off offset:16
; GFX950-GISEL-NEXT: s_endpgm
%cvt = tail call <6 x i32> @llvm.amdgcn.cvt.scalef32.pk32.fp6.bf16(<32 x bfloat> %src, float 100.0)
store <6 x i32> %cvt, ptr addrspace(1) %out, align 8
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.pk16.gfx1250.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.pk16.gfx1250.ll
index 62365a9d9cf70..b2e27415b95e3 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.pk16.gfx1250.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.pk16.gfx1250.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50 < %s | FileCheck -check-prefix=GFX1250-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.50 < %s | FileCheck -check-prefix=GFX1250-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 < %s | FileCheck -check-prefix=GFX1250-GISEL %s
declare <3 x i32> @llvm.amdgcn.cvt.scalef32.pk16.bf6.f32(<16 x float> %src, float %scale)
declare <3 x i32> @llvm.amdgcn.cvt.scalef32.pk16.fp6.f32(<16 x float> %src, float %scale)
@@ -240,7 +240,7 @@ define amdgpu_ps void @test_scalef32_pk16_bf6_bf16_vv(<16 x bfloat> %src, float
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v15, v10 :: v_dual_mov_b32 v14, v9
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v14, v9 :: v_dual_mov_b32 v15, v10
; GFX1250-GISEL-NEXT: v_cvt_scalef32_pk16_bf6_bf16 v[10:12], v[0:7], v8
; GFX1250-GISEL-NEXT: global_store_b96 v[14:15], v[10:12], off
; GFX1250-GISEL-NEXT: s_endpgm
@@ -269,10 +269,10 @@ define amdgpu_ps void @test_scalef32_pk16_bf6_bf16_sl(<16 x bfloat> inreg %src,
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, s4 :: v_dual_mov_b32 v7, s5
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, s6 :: v_dual_mov_b32 v9, s7
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_cvt_scalef32_pk16_bf6_bf16 v[10:12], v[2:9], 0x42c80000
; GFX1250-GISEL-NEXT: global_store_b96 v[0:1], v[10:12], off
@@ -389,7 +389,7 @@ define amdgpu_ps void @test_scalef32_pk16_fp6_bf16_vv(<16 x bfloat> %src, float
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v15, v10 :: v_dual_mov_b32 v14, v9
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v14, v9 :: v_dual_mov_b32 v15, v10
; GFX1250-GISEL-NEXT: v_cvt_scalef32_pk16_fp6_bf16 v[10:12], v[0:7], v8
; GFX1250-GISEL-NEXT: global_store_b96 v[14:15], v[10:12], off
; GFX1250-GISEL-NEXT: s_endpgm
@@ -418,10 +418,10 @@ define amdgpu_ps void @test_scalef32_pk16_fp6_bf16_sl(<16 x bfloat> inreg %src,
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, s4 :: v_dual_mov_b32 v7, s5
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, s6 :: v_dual_mov_b32 v9, s7
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_cvt_scalef32_pk16_fp6_bf16 v[10:12], v[2:9], 0x42c80000
; GFX1250-GISEL-NEXT: global_store_b96 v[0:1], v[10:12], off
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.pk8.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.pk8.ll
index c0f5e3e2d4cd7..ed661babb487e 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.pk8.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.pk8.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50 < %s | FileCheck -check-prefix=GFX1250-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.50 < %s | FileCheck -check-prefix=GFX1250-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 < %s | FileCheck -check-prefix=GFX1250-GISEL %s
declare <2 x i32> @llvm.amdgcn.cvt.scalef32.pk8.fp8.bf16(<8 x bfloat> %src, float %scale)
declare <2 x i32> @llvm.amdgcn.cvt.scalef32.pk8.bf8.bf16(<8 x bfloat> %src, float %scale)
@@ -28,9 +28,9 @@ define amdgpu_ps void @test_scalef32_pk8_fp8_bf16_vv(<8 x bfloat> %src, float %s
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v7, v6 :: v_dual_mov_b32 v6, v5
-; GFX1250-GISEL-NEXT: v_cvt_scalef32_pk8_fp8_bf16 v[8:9], v[0:3], v4
-; GFX1250-GISEL-NEXT: global_store_b64 v[6:7], v[8:9], off
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v5 :: v_dual_mov_b32 v9, v6
+; GFX1250-GISEL-NEXT: v_cvt_scalef32_pk8_fp8_bf16 v[6:7], v[0:3], v4
+; GFX1250-GISEL-NEXT: global_store_b64 v[8:9], v[6:7], off
; GFX1250-GISEL-NEXT: s_endpgm
%cvt = tail call <2 x i32> @llvm.amdgcn.cvt.scalef32.pk8.fp8.bf16(<8 x bfloat> %src, float %scale)
store <2 x i32> %cvt, ptr addrspace(1) %out, align 8
@@ -55,8 +55,8 @@ define amdgpu_ps void @test_scalef32_pk8_fp8_bf16_sl(<8 x bfloat> inreg %src, pt
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_cvt_scalef32_pk8_fp8_bf16 v[6:7], v[2:5], 0x42c80000
; GFX1250-GISEL-NEXT: global_store_b64 v[0:1], v[6:7], off
@@ -82,9 +82,9 @@ define amdgpu_ps void @test_scalef32_pk8_bf8_bf16_vv(<8 x bfloat> %src, float %s
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v7, v6 :: v_dual_mov_b32 v6, v5
-; GFX1250-GISEL-NEXT: v_cvt_scalef32_pk8_bf8_bf16 v[8:9], v[0:3], v4
-; GFX1250-GISEL-NEXT: global_store_b64 v[6:7], v[8:9], off
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v5 :: v_dual_mov_b32 v9, v6
+; GFX1250-GISEL-NEXT: v_cvt_scalef32_pk8_bf8_bf16 v[6:7], v[0:3], v4
+; GFX1250-GISEL-NEXT: global_store_b64 v[8:9], v[6:7], off
; GFX1250-GISEL-NEXT: s_endpgm
%cvt = tail call <2 x i32> @llvm.amdgcn.cvt.scalef32.pk8.bf8.bf16(<8 x bfloat> %src, float %scale)
store <2 x i32> %cvt, ptr addrspace(1) %out, align 8
@@ -109,8 +109,8 @@ define amdgpu_ps void @test_scalef32_pk8_bf8_bf16_sl(<8 x bfloat> inreg %src, pt
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_cvt_scalef32_pk8_bf8_bf16 v[6:7], v[2:5], 0x42c80000
; GFX1250-GISEL-NEXT: global_store_b64 v[0:1], v[6:7], off
@@ -658,9 +658,9 @@ define amdgpu_ps void @test_scalef32_pk8_fp4_bf16_vv(<8 x bfloat> %src, float %s
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v7, v6 :: v_dual_mov_b32 v6, v5
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v5 :: v_dual_mov_b32 v9, v6
; GFX1250-GISEL-NEXT: v_cvt_scalef32_pk8_fp4_bf16 v5, v[0:3], v4
-; GFX1250-GISEL-NEXT: global_store_b32 v[6:7], v5, off
+; GFX1250-GISEL-NEXT: global_store_b32 v[8:9], v5, off
; GFX1250-GISEL-NEXT: s_endpgm
%cvt = tail call i32 @llvm.amdgcn.cvt.scalef32.pk8.fp4.bf16(<8 x bfloat> %src, float %scale)
store i32 %cvt, ptr addrspace(1) %out, align 4
@@ -685,8 +685,8 @@ define amdgpu_ps void @test_scalef32_pk8_fp4_bf16_sl(<8 x bfloat> inreg %src, pt
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_cvt_scalef32_pk8_fp4_bf16 v6, v[2:5], 0x42c80000
; GFX1250-GISEL-NEXT: global_store_b32 v[0:1], v6, off
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.ll
index 3fce59470be28..d5a6320c83194 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
; RUN: llc -global-isel=0 -mtriple=amdgpu9.50 -o - %s | FileCheck -check-prefix=GFX950 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.50 -o - %s | FileCheck -check-prefix=GFX950 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.50 -o - %s | FileCheck -check-prefix=GFX950 %s
declare i32 @llvm.amdgcn.cvt.scalef32.sr.bf8.bf16(i32 %old, bfloat %src, i32 %seed, float %scale, i32 %dst_sel)
declare i32 @llvm.amdgcn.cvt.scalef32.sr.bf8.f16(i32 %old, half %src, i32 %seed, float %scale, i32 %dst_sel)
@@ -344,3 +344,87 @@ define amdgpu_ps void @test_cvt_scalef32_sr_fp8_f32_dst_sel_3(ptr addrspace(1) %
store i32 %cvt, ptr addrspace(1) %out, align 4
ret void
}
+
+define amdgpu_ps void @test_cvt_scalef32_sr_bf8_bf16_uni(ptr addrspace(1) %out, i32 inreg %old, bfloat inreg %src, i32 inreg %seed, float inreg %scale) {
+; GFX950-LABEL: test_cvt_scalef32_sr_bf8_bf16_uni:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: v_mov_b32_e32 v2, s0
+; GFX950-NEXT: v_mov_b32_e32 v3, s2
+; GFX950-NEXT: v_mov_b32_e32 v4, s3
+; GFX950-NEXT: v_cvt_scalef32_sr_bf8_bf16 v2, s1, v3, v4
+; GFX950-NEXT: global_store_dword v[0:1], v2, off
+; GFX950-NEXT: s_endpgm
+ %cvt = call i32 @llvm.amdgcn.cvt.scalef32.sr.bf8.bf16(i32 %old, bfloat %src, i32 %seed, float %scale, i32 0)
+ store i32 %cvt, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_ps void @test_cvt_scalef32_sr_bf8_f16_uni(ptr addrspace(1) %out, i32 inreg %old, half inreg %src, i32 inreg %seed, float inreg %scale) {
+; GFX950-LABEL: test_cvt_scalef32_sr_bf8_f16_uni:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: v_mov_b32_e32 v2, s0
+; GFX950-NEXT: v_mov_b32_e32 v3, s2
+; GFX950-NEXT: v_mov_b32_e32 v4, s3
+; GFX950-NEXT: v_cvt_scalef32_sr_bf8_f16 v2, s1, v3, v4
+; GFX950-NEXT: global_store_dword v[0:1], v2, off
+; GFX950-NEXT: s_endpgm
+ %cvt = call i32 @llvm.amdgcn.cvt.scalef32.sr.bf8.f16(i32 %old, half %src, i32 %seed, float %scale, i32 0)
+ store i32 %cvt, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_ps void @test_cvt_scalef32_sr_bf8_f32_uni(ptr addrspace(1) %out, i32 inreg %old, float inreg %src, i32 inreg %seed, float inreg %scale) {
+; GFX950-LABEL: test_cvt_scalef32_sr_bf8_f32_uni:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: v_mov_b32_e32 v2, s0
+; GFX950-NEXT: v_mov_b32_e32 v3, s2
+; GFX950-NEXT: v_mov_b32_e32 v4, s3
+; GFX950-NEXT: v_cvt_scalef32_sr_bf8_f32 v2, s1, v3, v4
+; GFX950-NEXT: global_store_dword v[0:1], v2, off
+; GFX950-NEXT: s_endpgm
+ %cvt = call i32 @llvm.amdgcn.cvt.scalef32.sr.bf8.f32(i32 %old, float %src, i32 %seed, float %scale, i32 0)
+ store i32 %cvt, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_ps void @test_cvt_scalef32_sr_fp8_bf16_uni(ptr addrspace(1) %out, i32 inreg %old, bfloat inreg %src, i32 inreg %seed, float inreg %scale) {
+; GFX950-LABEL: test_cvt_scalef32_sr_fp8_bf16_uni:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: v_mov_b32_e32 v2, s0
+; GFX950-NEXT: v_mov_b32_e32 v3, s2
+; GFX950-NEXT: v_mov_b32_e32 v4, s3
+; GFX950-NEXT: v_cvt_scalef32_sr_fp8_bf16 v2, s1, v3, v4
+; GFX950-NEXT: global_store_dword v[0:1], v2, off
+; GFX950-NEXT: s_endpgm
+ %cvt = call i32 @llvm.amdgcn.cvt.scalef32.sr.fp8.bf16(i32 %old, bfloat %src, i32 %seed, float %scale, i32 0)
+ store i32 %cvt, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_ps void @test_cvt_scalef32_sr_fp8_f16_uni(ptr addrspace(1) %out, i32 inreg %old, half inreg %src, i32 inreg %seed, float inreg %scale) {
+; GFX950-LABEL: test_cvt_scalef32_sr_fp8_f16_uni:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: v_mov_b32_e32 v2, s0
+; GFX950-NEXT: v_mov_b32_e32 v3, s2
+; GFX950-NEXT: v_mov_b32_e32 v4, s3
+; GFX950-NEXT: v_cvt_scalef32_sr_fp8_f16 v2, s1, v3, v4
+; GFX950-NEXT: global_store_dword v[0:1], v2, off
+; GFX950-NEXT: s_endpgm
+ %cvt = call i32 @llvm.amdgcn.cvt.scalef32.sr.fp8.f16(i32 %old, half %src, i32 %seed, float %scale, i32 0)
+ store i32 %cvt, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_ps void @test_cvt_scalef32_sr_fp8_f32_uni(ptr addrspace(1) %out, i32 inreg %old, float inreg %src, i32 inreg %seed, float inreg %scale) {
+; GFX950-LABEL: test_cvt_scalef32_sr_fp8_f32_uni:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: v_mov_b32_e32 v2, s0
+; GFX950-NEXT: v_mov_b32_e32 v3, s2
+; GFX950-NEXT: v_mov_b32_e32 v4, s3
+; GFX950-NEXT: v_cvt_scalef32_sr_fp8_f32 v2, s1, v3, v4
+; GFX950-NEXT: global_store_dword v[0:1], v2, off
+; GFX950-NEXT: s_endpgm
+ %cvt = call i32 @llvm.amdgcn.cvt.scalef32.sr.fp8.f32(i32 %old, float %src, i32 %seed, float %scale, i32 0)
+ store i32 %cvt, ptr addrspace(1) %out, align 4
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.pk.gfx1250.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.pk.gfx1250.ll
index e8647f669997f..8e4877a2f96db 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.pk.gfx1250.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.pk.gfx1250.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50 < %s | FileCheck -check-prefix=GFX1250-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.50 < %s | FileCheck -check-prefix=GFX1250-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 < %s | FileCheck -check-prefix=GFX1250-GISEL %s
declare <2 x i32> @llvm.amdgcn.cvt.scalef32.sr.pk8.fp8.bf16(<8 x bfloat> %src, i32 %sr, float %scale)
declare <2 x i32> @llvm.amdgcn.cvt.scalef32.sr.pk8.bf8.bf16(<8 x bfloat> %src, i32 %sr, float %scale)
@@ -53,8 +53,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_fp8_bf16_sl(<8 x bfloat> inreg %src,
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_cvt_scalef32_sr_pk8_fp8_bf16 v[6:7], v[2:5], s4, 0x42c80000
; GFX1250-GISEL-NEXT: global_store_b64 v[0:1], v[6:7], off
@@ -105,8 +105,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_bf8_bf16_sl(<8 x bfloat> inreg %src,
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_cvt_scalef32_sr_pk8_bf8_bf16 v[6:7], v[2:5], s4, 0x42c80000
; GFX1250-GISEL-NEXT: global_store_b64 v[0:1], v[6:7], off
@@ -667,8 +667,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_fp4_bf16_sl(<8 x bfloat> inreg %src,
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_cvt_scalef32_sr_pk8_fp4_bf16 v6, v[2:5], s4, 0x42c80000
; GFX1250-GISEL-NEXT: global_store_b32 v[0:1], v6, off
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.pk.gfx950.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.pk.gfx950.ll
index efa708a16d27e..cf0e16675346a 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.pk.gfx950.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.pk.gfx950.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
; RUN: llc -global-isel=0 -mtriple=amdgpu9.50 < %s | FileCheck -check-prefixes=GFX950,GFX950-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.50 < %s | FileCheck -check-prefixes=GFX950,GFX950-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.50 < %s | FileCheck -check-prefixes=GFX950,GFX950-GISEL %s
declare i32 @llvm.amdgcn.cvt.scalef32.sr.pk.fp4.f16(i32 %old, <2 x half> %src, i32 %seed, float %scale, i32 %dst_sel)
declare i32 @llvm.amdgcn.cvt.scalef32.sr.pk.fp4.bf16(i32 %old, <2 x bfloat> %src, i32 %seed, float %scale, i32 %dst_sel)
@@ -188,6 +188,20 @@ define amdgpu_ps void @test_scalef32_sr_pk_fp4_f16_uni(ptr addrspace(1) %out, i3
ret void
}
+define amdgpu_ps void @test_scalef32_sr_pk_fp4_bf16_uni(ptr addrspace(1) %out, i32 inreg %old, <2 x bfloat> inreg %src, i32 inreg %seed, float inreg %scale) {
+; GFX950-LABEL: test_scalef32_sr_pk_fp4_bf16_uni:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: v_mov_b32_e32 v2, s0
+; GFX950-NEXT: v_mov_b32_e32 v3, s2
+; GFX950-NEXT: v_mov_b32_e32 v4, s3
+; GFX950-NEXT: v_cvt_scalef32_sr_pk_fp4_bf16 v5, s1, v3, v4
+; GFX950-NEXT: global_store_dword v[0:1], v5, off
+; GFX950-NEXT: s_endpgm
+ %cvt = tail call i32 @llvm.amdgcn.cvt.scalef32.sr.pk.fp4.bf16(i32 %old, <2 x bfloat> %src, i32 %seed, float %scale, i32 0)
+ store i32 %cvt, ptr addrspace(1) %out, align 4
+ ret void
+}
+
define amdgpu_ps void @test_scalef32_sr_pk_fp4_f32_uni(ptr addrspace(1) %out, i32 inreg %old, <2 x float> inreg %src, i32 inreg %seed, float inreg %scale) {
; GFX950-SDAG-LABEL: test_scalef32_sr_pk_fp4_f32_uni:
; GFX950-SDAG: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.pk.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.pk.ll
index 0ee8435b88a77..86cd846beb5ab 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.pk.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.pk.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
-; RUN: llc -global-isel=0 -mtriple=amdgpu9.50 < %s | FileCheck -check-prefix=GCN,GFX950-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.50 < %s | FileCheck -check-prefix=GCN,GFX950-GISEL %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu9.50 < %s | FileCheck -check-prefixes=GCN,GFX950-SDAG %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.50 < %s | FileCheck -check-prefixes=GCN,GFX950-GISEL %s
declare <6 x i32> @llvm.amdgcn.cvt.scalef32.sr.pk32.bf6.bf16(<32 x bfloat> %src, i32 %sr, float %scale)
declare <6 x i32> @llvm.amdgcn.cvt.scalef32.sr.pk32.bf6.f16(<32 x half> %src, i32 %sr, float %scale)
@@ -29,29 +29,56 @@ define amdgpu_ps void @test_scalef32_sr_pk32_bf6_bf16_vv(<32 x bfloat> %src, i32
}
define amdgpu_ps void @test_scalef32_sr_pk32_bf6_bf16_sl(<32 x bfloat> inreg %src, i32 inreg %sr, ptr addrspace(1) %out) {
-; GCN-LABEL: test_scalef32_sr_pk32_bf6_bf16_sl:
-; GCN: ; %bb.0:
-; GCN-NEXT: v_mov_b32_e32 v2, s0
-; GCN-NEXT: v_mov_b32_e32 v3, s1
-; GCN-NEXT: v_mov_b32_e32 v4, s2
-; GCN-NEXT: v_mov_b32_e32 v5, s3
-; GCN-NEXT: v_mov_b32_e32 v6, s4
-; GCN-NEXT: v_mov_b32_e32 v7, s5
-; GCN-NEXT: v_mov_b32_e32 v8, s6
-; GCN-NEXT: v_mov_b32_e32 v9, s7
-; GCN-NEXT: v_mov_b32_e32 v10, s8
-; GCN-NEXT: v_mov_b32_e32 v11, s9
-; GCN-NEXT: v_mov_b32_e32 v12, s10
-; GCN-NEXT: v_mov_b32_e32 v13, s11
-; GCN-NEXT: v_mov_b32_e32 v14, s12
-; GCN-NEXT: v_mov_b32_e32 v15, s13
-; GCN-NEXT: v_mov_b32_e32 v16, s14
-; GCN-NEXT: v_mov_b32_e32 v17, s15
-; GCN-NEXT: v_mov_b32_e32 v24, 0x42c80000
-; GCN-NEXT: v_cvt_scalef32_sr_pk32_bf6_bf16 v[18:23], v[2:17], s16, v24
-; GCN-NEXT: global_store_dwordx2 v[0:1], v[22:23], off offset:16
-; GCN-NEXT: global_store_dwordx4 v[0:1], v[18:21], off
-; GCN-NEXT: s_endpgm
+; GFX950-SDAG-LABEL: test_scalef32_sr_pk32_bf6_bf16_sl:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, s0
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, s1
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v4, s2
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v5, s3
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v6, s4
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v7, s5
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v8, s6
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v9, s7
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v10, s8
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v11, s9
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v12, s10
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v13, s11
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v14, s12
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v15, s13
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v16, s14
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v17, s15
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v24, 0x42c80000
+; GFX950-SDAG-NEXT: v_cvt_scalef32_sr_pk32_bf6_bf16 v[18:23], v[2:17], s16, v24
+; GFX950-SDAG-NEXT: global_store_dwordx2 v[0:1], v[22:23], off offset:16
+; GFX950-SDAG-NEXT: global_store_dwordx4 v[0:1], v[18:21], off
+; GFX950-SDAG-NEXT: s_endpgm
+;
+; GFX950-GISEL-LABEL: test_scalef32_sr_pk32_bf6_bf16_sl:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[14:15]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[12:13]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[10:11]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[8:9]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v24, 0x42c80000
+; GFX950-GISEL-NEXT: v_cvt_scalef32_sr_pk32_bf6_bf16 v[18:23], v[2:17], s16, v24
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s0, v18
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s1, v19
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s2, v20
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s3, v21
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s4, v22
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s5, v23
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-GISEL-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
+; GFX950-GISEL-NEXT: s_nop 1
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[4:5]
+; GFX950-GISEL-NEXT: global_store_dwordx2 v[0:1], v[2:3], off offset:16
+; GFX950-GISEL-NEXT: s_endpgm
%cvt = tail call <6 x i32> @llvm.amdgcn.cvt.scalef32.sr.pk32.bf6.bf16(<32 x bfloat> %src, i32 %sr, float 100.0)
store <6 x i32> %cvt, ptr addrspace(1) %out, align 8
ret void
@@ -77,29 +104,56 @@ define amdgpu_ps void @test_scalef32_sr_pk32_bf6_f16_vv(<32 x half> %src, i32 %s
}
define amdgpu_ps void @test_scalef32_sr_pk32_bf6_f16_sl(<32 x half> inreg %src, i32 inreg %sr, ptr addrspace(1) %out) {
-; GCN-LABEL: test_scalef32_sr_pk32_bf6_f16_sl:
-; GCN: ; %bb.0:
-; GCN-NEXT: v_mov_b32_e32 v2, s0
-; GCN-NEXT: v_mov_b32_e32 v3, s1
-; GCN-NEXT: v_mov_b32_e32 v4, s2
-; GCN-NEXT: v_mov_b32_e32 v5, s3
-; GCN-NEXT: v_mov_b32_e32 v6, s4
-; GCN-NEXT: v_mov_b32_e32 v7, s5
-; GCN-NEXT: v_mov_b32_e32 v8, s6
-; GCN-NEXT: v_mov_b32_e32 v9, s7
-; GCN-NEXT: v_mov_b32_e32 v10, s8
-; GCN-NEXT: v_mov_b32_e32 v11, s9
-; GCN-NEXT: v_mov_b32_e32 v12, s10
-; GCN-NEXT: v_mov_b32_e32 v13, s11
-; GCN-NEXT: v_mov_b32_e32 v14, s12
-; GCN-NEXT: v_mov_b32_e32 v15, s13
-; GCN-NEXT: v_mov_b32_e32 v16, s14
-; GCN-NEXT: v_mov_b32_e32 v17, s15
-; GCN-NEXT: v_mov_b32_e32 v24, 0x42c80000
-; GCN-NEXT: v_cvt_scalef32_sr_pk32_bf6_f16 v[18:23], v[2:17], s16, v24
-; GCN-NEXT: global_store_dwordx2 v[0:1], v[22:23], off offset:16
-; GCN-NEXT: global_store_dwordx4 v[0:1], v[18:21], off
-; GCN-NEXT: s_endpgm
+; GFX950-SDAG-LABEL: test_scalef32_sr_pk32_bf6_f16_sl:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, s0
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, s1
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v4, s2
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v5, s3
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v6, s4
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v7, s5
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v8, s6
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v9, s7
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v10, s8
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v11, s9
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v12, s10
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v13, s11
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v14, s12
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v15, s13
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v16, s14
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v17, s15
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v24, 0x42c80000
+; GFX950-SDAG-NEXT: v_cvt_scalef32_sr_pk32_bf6_f16 v[18:23], v[2:17], s16, v24
+; GFX950-SDAG-NEXT: global_store_dwordx2 v[0:1], v[22:23], off offset:16
+; GFX950-SDAG-NEXT: global_store_dwordx4 v[0:1], v[18:21], off
+; GFX950-SDAG-NEXT: s_endpgm
+;
+; GFX950-GISEL-LABEL: test_scalef32_sr_pk32_bf6_f16_sl:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[14:15]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[12:13]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[10:11]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[8:9]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v24, 0x42c80000
+; GFX950-GISEL-NEXT: v_cvt_scalef32_sr_pk32_bf6_f16 v[18:23], v[2:17], s16, v24
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s0, v18
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s1, v19
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s2, v20
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s3, v21
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s4, v22
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s5, v23
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-GISEL-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
+; GFX950-GISEL-NEXT: s_nop 1
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[4:5]
+; GFX950-GISEL-NEXT: global_store_dwordx2 v[0:1], v[2:3], off offset:16
+; GFX950-GISEL-NEXT: s_endpgm
%cvt = tail call <6 x i32> @llvm.amdgcn.cvt.scalef32.sr.pk32.bf6.f16(<32 x half> %src, i32 %sr, float 100.0)
store <6 x i32> %cvt, ptr addrspace(1) %out, align 8
ret void
@@ -125,29 +179,56 @@ define amdgpu_ps void @test_scalef32_sr_pk32_fp6_bf16_vv(<32 x bfloat> %src, i32
}
define amdgpu_ps void @test_scalef32_sr_pk32_fp6_bf16_sl(<32 x bfloat> inreg %src, i32 inreg %sr, ptr addrspace(1) %out) {
-; GCN-LABEL: test_scalef32_sr_pk32_fp6_bf16_sl:
-; GCN: ; %bb.0:
-; GCN-NEXT: v_mov_b32_e32 v2, s0
-; GCN-NEXT: v_mov_b32_e32 v3, s1
-; GCN-NEXT: v_mov_b32_e32 v4, s2
-; GCN-NEXT: v_mov_b32_e32 v5, s3
-; GCN-NEXT: v_mov_b32_e32 v6, s4
-; GCN-NEXT: v_mov_b32_e32 v7, s5
-; GCN-NEXT: v_mov_b32_e32 v8, s6
-; GCN-NEXT: v_mov_b32_e32 v9, s7
-; GCN-NEXT: v_mov_b32_e32 v10, s8
-; GCN-NEXT: v_mov_b32_e32 v11, s9
-; GCN-NEXT: v_mov_b32_e32 v12, s10
-; GCN-NEXT: v_mov_b32_e32 v13, s11
-; GCN-NEXT: v_mov_b32_e32 v14, s12
-; GCN-NEXT: v_mov_b32_e32 v15, s13
-; GCN-NEXT: v_mov_b32_e32 v16, s14
-; GCN-NEXT: v_mov_b32_e32 v17, s15
-; GCN-NEXT: v_mov_b32_e32 v24, 0x42c80000
-; GCN-NEXT: v_cvt_scalef32_sr_pk32_fp6_bf16 v[18:23], v[2:17], s16, v24
-; GCN-NEXT: global_store_dwordx2 v[0:1], v[22:23], off offset:16
-; GCN-NEXT: global_store_dwordx4 v[0:1], v[18:21], off
-; GCN-NEXT: s_endpgm
+; GFX950-SDAG-LABEL: test_scalef32_sr_pk32_fp6_bf16_sl:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, s0
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, s1
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v4, s2
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v5, s3
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v6, s4
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v7, s5
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v8, s6
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v9, s7
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v10, s8
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v11, s9
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v12, s10
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v13, s11
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v14, s12
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v15, s13
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v16, s14
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v17, s15
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v24, 0x42c80000
+; GFX950-SDAG-NEXT: v_cvt_scalef32_sr_pk32_fp6_bf16 v[18:23], v[2:17], s16, v24
+; GFX950-SDAG-NEXT: global_store_dwordx2 v[0:1], v[22:23], off offset:16
+; GFX950-SDAG-NEXT: global_store_dwordx4 v[0:1], v[18:21], off
+; GFX950-SDAG-NEXT: s_endpgm
+;
+; GFX950-GISEL-LABEL: test_scalef32_sr_pk32_fp6_bf16_sl:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[14:15]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[12:13]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[10:11]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[8:9]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v24, 0x42c80000
+; GFX950-GISEL-NEXT: v_cvt_scalef32_sr_pk32_fp6_bf16 v[18:23], v[2:17], s16, v24
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s0, v18
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s1, v19
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s2, v20
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s3, v21
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s4, v22
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s5, v23
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-GISEL-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
+; GFX950-GISEL-NEXT: s_nop 1
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[4:5]
+; GFX950-GISEL-NEXT: global_store_dwordx2 v[0:1], v[2:3], off offset:16
+; GFX950-GISEL-NEXT: s_endpgm
%cvt = tail call <6 x i32> @llvm.amdgcn.cvt.scalef32.sr.pk32.fp6.bf16(<32 x bfloat> %src, i32 %sr, float 100.0)
store <6 x i32> %cvt, ptr addrspace(1) %out, align 8
ret void
@@ -173,29 +254,56 @@ define amdgpu_ps void @test_scalef32_sr_pk32_fp6_f16_vv(<32 x half> %src, i32 %s
}
define amdgpu_ps void @test_scalef32_sr_pk32_fp6_f16_sl(<32 x half> inreg %src, i32 inreg %sr, ptr addrspace(1) %out) {
-; GCN-LABEL: test_scalef32_sr_pk32_fp6_f16_sl:
-; GCN: ; %bb.0:
-; GCN-NEXT: v_mov_b32_e32 v2, s0
-; GCN-NEXT: v_mov_b32_e32 v3, s1
-; GCN-NEXT: v_mov_b32_e32 v4, s2
-; GCN-NEXT: v_mov_b32_e32 v5, s3
-; GCN-NEXT: v_mov_b32_e32 v6, s4
-; GCN-NEXT: v_mov_b32_e32 v7, s5
-; GCN-NEXT: v_mov_b32_e32 v8, s6
-; GCN-NEXT: v_mov_b32_e32 v9, s7
-; GCN-NEXT: v_mov_b32_e32 v10, s8
-; GCN-NEXT: v_mov_b32_e32 v11, s9
-; GCN-NEXT: v_mov_b32_e32 v12, s10
-; GCN-NEXT: v_mov_b32_e32 v13, s11
-; GCN-NEXT: v_mov_b32_e32 v14, s12
-; GCN-NEXT: v_mov_b32_e32 v15, s13
-; GCN-NEXT: v_mov_b32_e32 v16, s14
-; GCN-NEXT: v_mov_b32_e32 v17, s15
-; GCN-NEXT: v_mov_b32_e32 v24, 0x42c80000
-; GCN-NEXT: v_cvt_scalef32_sr_pk32_fp6_f16 v[18:23], v[2:17], s16, v24
-; GCN-NEXT: global_store_dwordx2 v[0:1], v[22:23], off offset:16
-; GCN-NEXT: global_store_dwordx4 v[0:1], v[18:21], off
-; GCN-NEXT: s_endpgm
+; GFX950-SDAG-LABEL: test_scalef32_sr_pk32_fp6_f16_sl:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, s0
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, s1
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v4, s2
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v5, s3
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v6, s4
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v7, s5
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v8, s6
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v9, s7
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v10, s8
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v11, s9
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v12, s10
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v13, s11
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v14, s12
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v15, s13
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v16, s14
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v17, s15
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v24, 0x42c80000
+; GFX950-SDAG-NEXT: v_cvt_scalef32_sr_pk32_fp6_f16 v[18:23], v[2:17], s16, v24
+; GFX950-SDAG-NEXT: global_store_dwordx2 v[0:1], v[22:23], off offset:16
+; GFX950-SDAG-NEXT: global_store_dwordx4 v[0:1], v[18:21], off
+; GFX950-SDAG-NEXT: s_endpgm
+;
+; GFX950-GISEL-LABEL: test_scalef32_sr_pk32_fp6_f16_sl:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[14:15]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[12:13]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[10:11]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[8:9]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v24, 0x42c80000
+; GFX950-GISEL-NEXT: v_cvt_scalef32_sr_pk32_fp6_f16 v[18:23], v[2:17], s16, v24
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s0, v18
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s1, v19
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s2, v20
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s3, v21
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s4, v22
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s5, v23
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-GISEL-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
+; GFX950-GISEL-NEXT: s_nop 1
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[4:5]
+; GFX950-GISEL-NEXT: global_store_dwordx2 v[0:1], v[2:3], off offset:16
+; GFX950-GISEL-NEXT: s_endpgm
%cvt = tail call <6 x i32> @llvm.amdgcn.cvt.scalef32.sr.pk32.fp6.f16(<32 x half> %src, i32 %sr, float 100.0)
store <6 x i32> %cvt, ptr addrspace(1) %out, align 8
ret void
@@ -221,45 +329,80 @@ define amdgpu_ps void @test_scalef32_sr_pk32_bf6_f32_vv(<32 x float> %src, i32 %
}
define amdgpu_ps void @test_scalef32_sr_pk32_bf6_f32_sl(<32 x float> inreg %src, i32 inreg %sr, ptr addrspace(1) %out) {
-; GCN-LABEL: test_scalef32_sr_pk32_bf6_f32_sl:
-; GCN: ; %bb.0:
-; GCN-NEXT: v_mov_b32_e32 v2, s0
-; GCN-NEXT: v_mov_b32_e32 v3, s1
-; GCN-NEXT: v_mov_b32_e32 v4, s2
-; GCN-NEXT: v_mov_b32_e32 v5, s3
-; GCN-NEXT: v_mov_b32_e32 v6, s4
-; GCN-NEXT: v_mov_b32_e32 v7, s5
-; GCN-NEXT: v_mov_b32_e32 v8, s6
-; GCN-NEXT: v_mov_b32_e32 v9, s7
-; GCN-NEXT: v_mov_b32_e32 v10, s8
-; GCN-NEXT: v_mov_b32_e32 v11, s9
-; GCN-NEXT: v_mov_b32_e32 v12, s10
-; GCN-NEXT: v_mov_b32_e32 v13, s11
-; GCN-NEXT: v_mov_b32_e32 v14, s12
-; GCN-NEXT: v_mov_b32_e32 v15, s13
-; GCN-NEXT: v_mov_b32_e32 v16, s14
-; GCN-NEXT: v_mov_b32_e32 v17, s15
-; GCN-NEXT: v_mov_b32_e32 v18, s16
-; GCN-NEXT: v_mov_b32_e32 v19, s17
-; GCN-NEXT: v_mov_b32_e32 v20, s18
-; GCN-NEXT: v_mov_b32_e32 v21, s19
-; GCN-NEXT: v_mov_b32_e32 v22, s20
-; GCN-NEXT: v_mov_b32_e32 v23, s21
-; GCN-NEXT: v_mov_b32_e32 v24, s22
-; GCN-NEXT: v_mov_b32_e32 v25, s23
-; GCN-NEXT: v_mov_b32_e32 v26, s24
-; GCN-NEXT: v_mov_b32_e32 v27, s25
-; GCN-NEXT: v_mov_b32_e32 v28, s26
-; GCN-NEXT: v_mov_b32_e32 v29, s27
-; GCN-NEXT: v_mov_b32_e32 v30, s28
-; GCN-NEXT: v_mov_b32_e32 v31, s29
-; GCN-NEXT: v_mov_b32_e32 v32, s30
-; GCN-NEXT: v_mov_b32_e32 v33, s31
-; GCN-NEXT: v_mov_b32_e32 v40, 0x42c80000
-; GCN-NEXT: v_cvt_scalef32_sr_pk32_bf6_f32 v[34:39], v[2:33], s32, v40
-; GCN-NEXT: global_store_dwordx2 v[0:1], v[38:39], off offset:16
-; GCN-NEXT: global_store_dwordx4 v[0:1], v[34:37], off
-; GCN-NEXT: s_endpgm
+; GFX950-SDAG-LABEL: test_scalef32_sr_pk32_bf6_f32_sl:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, s0
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, s1
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v4, s2
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v5, s3
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v6, s4
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v7, s5
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v8, s6
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v9, s7
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v10, s8
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v11, s9
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v12, s10
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v13, s11
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v14, s12
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v15, s13
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v16, s14
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v17, s15
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v18, s16
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v19, s17
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v20, s18
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v21, s19
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v22, s20
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v23, s21
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v24, s22
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v25, s23
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v26, s24
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v27, s25
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v28, s26
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v29, s27
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v30, s28
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v31, s29
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v32, s30
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v33, s31
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v40, 0x42c80000
+; GFX950-SDAG-NEXT: v_cvt_scalef32_sr_pk32_bf6_f32 v[34:39], v[2:33], s32, v40
+; GFX950-SDAG-NEXT: global_store_dwordx2 v[0:1], v[38:39], off offset:16
+; GFX950-SDAG-NEXT: global_store_dwordx4 v[0:1], v[34:37], off
+; GFX950-SDAG-NEXT: s_endpgm
+;
+; GFX950-GISEL-LABEL: test_scalef32_sr_pk32_bf6_f32_sl:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[32:33], s[30:31]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[30:31], s[28:29]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[28:29], s[26:27]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[26:27], s[24:25]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[24:25], s[22:23]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[22:23], s[20:21]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[18:19]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[16:17]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[14:15]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[12:13]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[10:11]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[8:9]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v40, 0x42c80000
+; GFX950-GISEL-NEXT: v_cvt_scalef32_sr_pk32_bf6_f32 v[34:39], v[2:33], s32, v40
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s0, v34
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s1, v35
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s2, v36
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s3, v37
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s4, v38
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s5, v39
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-GISEL-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
+; GFX950-GISEL-NEXT: s_nop 1
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[4:5]
+; GFX950-GISEL-NEXT: global_store_dwordx2 v[0:1], v[2:3], off offset:16
+; GFX950-GISEL-NEXT: s_endpgm
%cvt = tail call <6 x i32> @llvm.amdgcn.cvt.scalef32.sr.pk32.bf6.f32(<32 x float> %src, i32 %sr, float 100.0)
store <6 x i32> %cvt, ptr addrspace(1) %out, align 8
ret void
@@ -285,46 +428,83 @@ define amdgpu_ps void @test_scalef32_sr_pk32_fp6_f32_vv(<32 x float> %src, i32 %
}
define amdgpu_ps void @test_scalef32_sr_pk32_fp6_f32_sl(<32 x float> inreg %src, i32 inreg %sr, ptr addrspace(1) %out) {
-; GCN-LABEL: test_scalef32_sr_pk32_fp6_f32_sl:
-; GCN: ; %bb.0:
-; GCN-NEXT: v_mov_b32_e32 v2, s0
-; GCN-NEXT: v_mov_b32_e32 v3, s1
-; GCN-NEXT: v_mov_b32_e32 v4, s2
-; GCN-NEXT: v_mov_b32_e32 v5, s3
-; GCN-NEXT: v_mov_b32_e32 v6, s4
-; GCN-NEXT: v_mov_b32_e32 v7, s5
-; GCN-NEXT: v_mov_b32_e32 v8, s6
-; GCN-NEXT: v_mov_b32_e32 v9, s7
-; GCN-NEXT: v_mov_b32_e32 v10, s8
-; GCN-NEXT: v_mov_b32_e32 v11, s9
-; GCN-NEXT: v_mov_b32_e32 v12, s10
-; GCN-NEXT: v_mov_b32_e32 v13, s11
-; GCN-NEXT: v_mov_b32_e32 v14, s12
-; GCN-NEXT: v_mov_b32_e32 v15, s13
-; GCN-NEXT: v_mov_b32_e32 v16, s14
-; GCN-NEXT: v_mov_b32_e32 v17, s15
-; GCN-NEXT: v_mov_b32_e32 v18, s16
-; GCN-NEXT: v_mov_b32_e32 v19, s17
-; GCN-NEXT: v_mov_b32_e32 v20, s18
-; GCN-NEXT: v_mov_b32_e32 v21, s19
-; GCN-NEXT: v_mov_b32_e32 v22, s20
-; GCN-NEXT: v_mov_b32_e32 v23, s21
-; GCN-NEXT: v_mov_b32_e32 v24, s22
-; GCN-NEXT: v_mov_b32_e32 v25, s23
-; GCN-NEXT: v_mov_b32_e32 v26, s24
-; GCN-NEXT: v_mov_b32_e32 v27, s25
-; GCN-NEXT: v_mov_b32_e32 v28, s26
-; GCN-NEXT: v_mov_b32_e32 v29, s27
-; GCN-NEXT: v_mov_b32_e32 v30, s28
-; GCN-NEXT: v_mov_b32_e32 v31, s29
-; GCN-NEXT: v_mov_b32_e32 v32, s30
-; GCN-NEXT: v_mov_b32_e32 v33, s31
-; GCN-NEXT: v_mov_b32_e32 v40, 0x42c80000
-; GCN-NEXT: v_cvt_scalef32_sr_pk32_fp6_f32 v[34:39], v[2:33], s32, v40
-; GCN-NEXT: global_store_dwordx2 v[0:1], v[38:39], off offset:16
-; GCN-NEXT: global_store_dwordx4 v[0:1], v[34:37], off
-; GCN-NEXT: s_endpgm
+; GFX950-SDAG-LABEL: test_scalef32_sr_pk32_fp6_f32_sl:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, s0
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, s1
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v4, s2
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v5, s3
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v6, s4
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v7, s5
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v8, s6
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v9, s7
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v10, s8
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v11, s9
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v12, s10
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v13, s11
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v14, s12
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v15, s13
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v16, s14
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v17, s15
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v18, s16
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v19, s17
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v20, s18
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v21, s19
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v22, s20
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v23, s21
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v24, s22
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v25, s23
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v26, s24
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v27, s25
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v28, s26
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v29, s27
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v30, s28
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v31, s29
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v32, s30
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v33, s31
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v40, 0x42c80000
+; GFX950-SDAG-NEXT: v_cvt_scalef32_sr_pk32_fp6_f32 v[34:39], v[2:33], s32, v40
+; GFX950-SDAG-NEXT: global_store_dwordx2 v[0:1], v[38:39], off offset:16
+; GFX950-SDAG-NEXT: global_store_dwordx4 v[0:1], v[34:37], off
+; GFX950-SDAG-NEXT: s_endpgm
+;
+; GFX950-GISEL-LABEL: test_scalef32_sr_pk32_fp6_f32_sl:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[32:33], s[30:31]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[30:31], s[28:29]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[28:29], s[26:27]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[26:27], s[24:25]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[24:25], s[22:23]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[22:23], s[20:21]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[18:19]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[16:17]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[14:15]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[12:13]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[10:11]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[8:9]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v40, 0x42c80000
+; GFX950-GISEL-NEXT: v_cvt_scalef32_sr_pk32_fp6_f32 v[34:39], v[2:33], s32, v40
+; GFX950-GISEL-NEXT: s_nop 0
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s0, v34
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s1, v35
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s2, v36
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s3, v37
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s4, v38
+; GFX950-GISEL-NEXT: v_readfirstlane_b32 s5, v39
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-GISEL-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
+; GFX950-GISEL-NEXT: s_nop 1
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[4:5]
+; GFX950-GISEL-NEXT: global_store_dwordx2 v[0:1], v[2:3], off offset:16
+; GFX950-GISEL-NEXT: s_endpgm
%cvt = tail call <6 x i32> @llvm.amdgcn.cvt.scalef32.sr.pk32.fp6.f32(<32 x float> %src, i32 %sr, float 100.0)
store <6 x i32> %cvt, ptr addrspace(1) %out, align 8
ret void
}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GCN: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.pk16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.pk16.ll
index 4d7fa3799d173..16d8d6d963f10 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.pk16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.pk16.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.50 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL %s
declare <3 x i32> @llvm.amdgcn.cvt.scalef32.sr.pk16.bf6.bf16(<16 x bfloat> %src, i32 %sr, float %scale)
declare <3 x i32> @llvm.amdgcn.cvt.scalef32.sr.pk16.bf6.f16(<16 x half> %src, i32 %sr, float %scale)
@@ -24,19 +24,33 @@ define amdgpu_ps void @test_scalef32_sr_pk16_bf6_bf16_vv(<16 x bfloat> %src, i32
}
define amdgpu_ps void @test_scalef32_sr_pk16_bf6_bf16_sl(<16 x bfloat> inreg %src, i32 inreg %sr, ptr addrspace(1) %out) {
-; GFX1250-LABEL: test_scalef32_sr_pk16_bf6_bf16_sl:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
-; GFX1250-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
-; GFX1250-NEXT: v_dual_mov_b32 v6, s4 :: v_dual_mov_b32 v7, s5
-; GFX1250-NEXT: v_dual_mov_b32 v8, s6 :: v_dual_mov_b32 v9, s7
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cvt_scalef32_sr_pk16_bf6_bf16 v[10:12], v[2:9], s8, 0x42c80000
-; GFX1250-NEXT: global_store_b96 v[0:1], v[10:12], off
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: test_scalef32_sr_pk16_bf6_bf16_sl:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v6, s4 :: v_dual_mov_b32 v7, s5
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v8, s6 :: v_dual_mov_b32 v9, s7
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_cvt_scalef32_sr_pk16_bf6_bf16 v[10:12], v[2:9], s8, 0x42c80000
+; GFX1250-SDAG-NEXT: global_store_b96 v[0:1], v[10:12], off
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: test_scalef32_sr_pk16_bf6_bf16_sl:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_cvt_scalef32_sr_pk16_bf6_bf16 v[10:12], v[2:9], s8, 0x42c80000
+; GFX1250-GISEL-NEXT: global_store_b96 v[0:1], v[10:12], off
+; GFX1250-GISEL-NEXT: s_endpgm
%cvt = tail call <3 x i32> @llvm.amdgcn.cvt.scalef32.sr.pk16.bf6.bf16(<16 x bfloat> %src, i32 %sr, float 100.0)
store <3 x i32> %cvt, ptr addrspace(1) %out, align 8
ret void
@@ -137,19 +151,33 @@ define amdgpu_ps void @test_scalef32_sr_pk16_fp6_bf16_vv(<16 x bfloat> %src, i32
}
define amdgpu_ps void @test_scalef32_sr_pk16_fp6_bf16_sl(<16 x bfloat> inreg %src, i32 inreg %sr, ptr addrspace(1) %out) {
-; GFX1250-LABEL: test_scalef32_sr_pk16_fp6_bf16_sl:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
-; GFX1250-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
-; GFX1250-NEXT: v_dual_mov_b32 v6, s4 :: v_dual_mov_b32 v7, s5
-; GFX1250-NEXT: v_dual_mov_b32 v8, s6 :: v_dual_mov_b32 v9, s7
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cvt_scalef32_sr_pk16_fp6_bf16 v[10:12], v[2:9], s8, 0x42c80000
-; GFX1250-NEXT: global_store_b96 v[0:1], v[10:12], off
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: test_scalef32_sr_pk16_fp6_bf16_sl:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v6, s4 :: v_dual_mov_b32 v7, s5
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v8, s6 :: v_dual_mov_b32 v9, s7
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_cvt_scalef32_sr_pk16_fp6_bf16 v[10:12], v[2:9], s8, 0x42c80000
+; GFX1250-SDAG-NEXT: global_store_b96 v[0:1], v[10:12], off
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: test_scalef32_sr_pk16_fp6_bf16_sl:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_cvt_scalef32_sr_pk16_fp6_bf16 v[10:12], v[2:9], s8, 0x42c80000
+; GFX1250-GISEL-NEXT: global_store_b96 v[0:1], v[10:12], off
+; GFX1250-GISEL-NEXT: s_endpgm
%cvt = tail call <3 x i32> @llvm.amdgcn.cvt.scalef32.sr.pk16.fp6.bf16(<16 x bfloat> %src, i32 %sr, float 100.0)
store <3 x i32> %cvt, ptr addrspace(1) %out, align 8
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.sr.pk.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.sr.pk.bf16.ll
index df1fd7bed883d..e3024f0e1328b 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.sr.pk.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.sr.pk.bf16.ll
@@ -1,12 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50 < %s | FileCheck -check-prefix=GCN %s
-; xUN: llc -global-isel=1 -mtriple=amdgpu12.50 < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 < %s | FileCheck -check-prefix=GCN %s
; RUN: not --crash llc -global-isel=0 -mtriple=amdgpu9.50 < %s 2>&1 | FileCheck -check-prefix=ERR %s
; ERR: LLVM ERROR: Cannot select: intrinsic %llvm.amdgcn.cvt.sr.pk.bf16.f32
-; FIXME: GlobalISel does not work with bf16
-
declare <2 x bfloat> @llvm.amdgcn.cvt.sr.pk.bf16.f32(float, float, i32) #0
define amdgpu_ps float @cvt_sr_pk_bf16_f32_vvv(float %src0, float %src1, i32 %src2) #1 {
More information about the llvm-commits
mailing list