[llvm] Reland "[NVPTX] Add folding for cvt.rn.bf16x2.f32" (PR #116417)
Alex MacLean via llvm-commits
llvm-commits at lists.llvm.org
Fri Dec 6 10:34:40 PST 2024
https://github.com/AlexMaclean updated https://github.com/llvm/llvm-project/pull/116417
>From 2880f551039e74815aabd6a9b65a90897b20e93e Mon Sep 17 00:00:00 2001
From: Alex MacLean <amaclean at nvidia.com>
Date: Thu, 14 Nov 2024 13:26:23 -0800
Subject: [PATCH 1/5] [NVPTX] Add folding for cvt.rn.bf16x2.f32 (#116109)
---
llvm/lib/Target/NVPTX/NVPTXInstrInfo.td | 14 +
llvm/test/CodeGen/NVPTX/bf16-instructions.ll | 54 ++--
.../NVPTX/bf16x2-instructions-approx.ll | 1 +
llvm/test/CodeGen/NVPTX/convert-sm80.ll | 280 +++++++++++++-----
4 files changed, 236 insertions(+), 113 deletions(-)
diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
index b8a55851e8d3e1..abc8fcb582020f 100644
--- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
+++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
@@ -727,6 +727,20 @@ let hasSideEffects = false in {
def CVT_f16x2_e5m2x2 : CVT_f16x2_fp8<"e5m2">;
}
+def fpround_oneuse : PatFrag<(ops node:$a), (fpround node:$a), [{
+ return N->hasOneUse();
+}]>;
+
+def : Pat<(v2bf16 (build_vector (bf16 (fpround_oneuse Float32Regs:$a)),
+ (bf16 (fpround_oneuse Float32Regs:$b)))),
+ (CVT_bf16x2_f32 Float32Regs:$a, Float32Regs:$b, CvtRN)>,
+ Requires<[hasPTX<70>, hasSM<80>, hasBF16Math]>;
+
+def : Pat<(v2f16 (build_vector (f16 (fpround_oneuse Float32Regs:$a)),
+ (f16 (fpround_oneuse Float32Regs:$b)))),
+ (CVT_f16x2_f32 Float32Regs:$a, Float32Regs:$b, CvtRN)>,
+ Requires<[hasPTX<70>, hasSM<80>, useFP16Math]>;
+
//-----------------------------------
// Selection instructions (selp)
//-----------------------------------
diff --git a/llvm/test/CodeGen/NVPTX/bf16-instructions.ll b/llvm/test/CodeGen/NVPTX/bf16-instructions.ll
index 80815b3ca37c05..ded3019e173b4c 100644
--- a/llvm/test/CodeGen/NVPTX/bf16-instructions.ll
+++ b/llvm/test/CodeGen/NVPTX/bf16-instructions.ll
@@ -204,7 +204,7 @@ define <2 x bfloat> @test_faddx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
;
; SM80-LABEL: test_faddx2(
; SM80: {
-; SM80-NEXT: .reg .b16 %rs<7>;
+; SM80-NEXT: .reg .b16 %rs<5>;
; SM80-NEXT: .reg .b32 %r<4>;
; SM80-NEXT: .reg .f32 %f<7>;
; SM80-EMPTY:
@@ -216,18 +216,16 @@ define <2 x bfloat> @test_faddx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM80-NEXT: mov.b32 {%rs3, %rs4}, %r1;
; SM80-NEXT: cvt.f32.bf16 %f2, %rs4;
; SM80-NEXT: add.rn.f32 %f3, %f2, %f1;
-; SM80-NEXT: cvt.rn.bf16.f32 %rs5, %f3;
; SM80-NEXT: cvt.f32.bf16 %f4, %rs1;
; SM80-NEXT: cvt.f32.bf16 %f5, %rs3;
; SM80-NEXT: add.rn.f32 %f6, %f5, %f4;
-; SM80-NEXT: cvt.rn.bf16.f32 %rs6, %f6;
-; SM80-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM80-NEXT: cvt.rn.bf16x2.f32 %r3, %f6, %f3;
; SM80-NEXT: st.param.b32 [func_retval0], %r3;
; SM80-NEXT: ret;
;
; SM80-FTZ-LABEL: test_faddx2(
; SM80-FTZ: {
-; SM80-FTZ-NEXT: .reg .b16 %rs<7>;
+; SM80-FTZ-NEXT: .reg .b16 %rs<5>;
; SM80-FTZ-NEXT: .reg .b32 %r<4>;
; SM80-FTZ-NEXT: .reg .f32 %f<7>;
; SM80-FTZ-EMPTY:
@@ -239,12 +237,10 @@ define <2 x bfloat> @test_faddx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM80-FTZ-NEXT: mov.b32 {%rs3, %rs4}, %r1;
; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f2, %rs4;
; SM80-FTZ-NEXT: add.rn.ftz.f32 %f3, %f2, %f1;
-; SM80-FTZ-NEXT: cvt.rn.bf16.f32 %rs5, %f3;
; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f4, %rs1;
; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f5, %rs3;
; SM80-FTZ-NEXT: add.rn.ftz.f32 %f6, %f5, %f4;
-; SM80-FTZ-NEXT: cvt.rn.bf16.f32 %rs6, %f6;
-; SM80-FTZ-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM80-FTZ-NEXT: cvt.rn.bf16x2.f32 %r3, %f6, %f3;
; SM80-FTZ-NEXT: st.param.b32 [func_retval0], %r3;
; SM80-FTZ-NEXT: ret;
;
@@ -311,7 +307,7 @@ define <2 x bfloat> @test_fsubx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
;
; SM80-LABEL: test_fsubx2(
; SM80: {
-; SM80-NEXT: .reg .b16 %rs<7>;
+; SM80-NEXT: .reg .b16 %rs<5>;
; SM80-NEXT: .reg .b32 %r<4>;
; SM80-NEXT: .reg .f32 %f<7>;
; SM80-EMPTY:
@@ -323,18 +319,16 @@ define <2 x bfloat> @test_fsubx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM80-NEXT: mov.b32 {%rs3, %rs4}, %r1;
; SM80-NEXT: cvt.f32.bf16 %f2, %rs4;
; SM80-NEXT: sub.rn.f32 %f3, %f2, %f1;
-; SM80-NEXT: cvt.rn.bf16.f32 %rs5, %f3;
; SM80-NEXT: cvt.f32.bf16 %f4, %rs1;
; SM80-NEXT: cvt.f32.bf16 %f5, %rs3;
; SM80-NEXT: sub.rn.f32 %f6, %f5, %f4;
-; SM80-NEXT: cvt.rn.bf16.f32 %rs6, %f6;
-; SM80-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM80-NEXT: cvt.rn.bf16x2.f32 %r3, %f6, %f3;
; SM80-NEXT: st.param.b32 [func_retval0], %r3;
; SM80-NEXT: ret;
;
; SM80-FTZ-LABEL: test_fsubx2(
; SM80-FTZ: {
-; SM80-FTZ-NEXT: .reg .b16 %rs<7>;
+; SM80-FTZ-NEXT: .reg .b16 %rs<5>;
; SM80-FTZ-NEXT: .reg .b32 %r<4>;
; SM80-FTZ-NEXT: .reg .f32 %f<7>;
; SM80-FTZ-EMPTY:
@@ -346,12 +340,10 @@ define <2 x bfloat> @test_fsubx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM80-FTZ-NEXT: mov.b32 {%rs3, %rs4}, %r1;
; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f2, %rs4;
; SM80-FTZ-NEXT: sub.rn.ftz.f32 %f3, %f2, %f1;
-; SM80-FTZ-NEXT: cvt.rn.bf16.f32 %rs5, %f3;
; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f4, %rs1;
; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f5, %rs3;
; SM80-FTZ-NEXT: sub.rn.ftz.f32 %f6, %f5, %f4;
-; SM80-FTZ-NEXT: cvt.rn.bf16.f32 %rs6, %f6;
-; SM80-FTZ-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM80-FTZ-NEXT: cvt.rn.bf16x2.f32 %r3, %f6, %f3;
; SM80-FTZ-NEXT: st.param.b32 [func_retval0], %r3;
; SM80-FTZ-NEXT: ret;
;
@@ -418,7 +410,7 @@ define <2 x bfloat> @test_fmulx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
;
; SM80-LABEL: test_fmulx2(
; SM80: {
-; SM80-NEXT: .reg .b16 %rs<7>;
+; SM80-NEXT: .reg .b16 %rs<5>;
; SM80-NEXT: .reg .b32 %r<4>;
; SM80-NEXT: .reg .f32 %f<7>;
; SM80-EMPTY:
@@ -430,18 +422,16 @@ define <2 x bfloat> @test_fmulx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM80-NEXT: mov.b32 {%rs3, %rs4}, %r1;
; SM80-NEXT: cvt.f32.bf16 %f2, %rs4;
; SM80-NEXT: mul.rn.f32 %f3, %f2, %f1;
-; SM80-NEXT: cvt.rn.bf16.f32 %rs5, %f3;
; SM80-NEXT: cvt.f32.bf16 %f4, %rs1;
; SM80-NEXT: cvt.f32.bf16 %f5, %rs3;
; SM80-NEXT: mul.rn.f32 %f6, %f5, %f4;
-; SM80-NEXT: cvt.rn.bf16.f32 %rs6, %f6;
-; SM80-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM80-NEXT: cvt.rn.bf16x2.f32 %r3, %f6, %f3;
; SM80-NEXT: st.param.b32 [func_retval0], %r3;
; SM80-NEXT: ret;
;
; SM80-FTZ-LABEL: test_fmulx2(
; SM80-FTZ: {
-; SM80-FTZ-NEXT: .reg .b16 %rs<7>;
+; SM80-FTZ-NEXT: .reg .b16 %rs<5>;
; SM80-FTZ-NEXT: .reg .b32 %r<4>;
; SM80-FTZ-NEXT: .reg .f32 %f<7>;
; SM80-FTZ-EMPTY:
@@ -453,12 +443,10 @@ define <2 x bfloat> @test_fmulx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM80-FTZ-NEXT: mov.b32 {%rs3, %rs4}, %r1;
; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f2, %rs4;
; SM80-FTZ-NEXT: mul.rn.ftz.f32 %f3, %f2, %f1;
-; SM80-FTZ-NEXT: cvt.rn.bf16.f32 %rs5, %f3;
; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f4, %rs1;
; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f5, %rs3;
; SM80-FTZ-NEXT: mul.rn.ftz.f32 %f6, %f5, %f4;
-; SM80-FTZ-NEXT: cvt.rn.bf16.f32 %rs6, %f6;
-; SM80-FTZ-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM80-FTZ-NEXT: cvt.rn.bf16x2.f32 %r3, %f6, %f3;
; SM80-FTZ-NEXT: st.param.b32 [func_retval0], %r3;
; SM80-FTZ-NEXT: ret;
;
@@ -525,7 +513,7 @@ define <2 x bfloat> @test_fdiv(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
;
; SM80-LABEL: test_fdiv(
; SM80: {
-; SM80-NEXT: .reg .b16 %rs<7>;
+; SM80-NEXT: .reg .b16 %rs<5>;
; SM80-NEXT: .reg .b32 %r<4>;
; SM80-NEXT: .reg .f32 %f<7>;
; SM80-EMPTY:
@@ -537,18 +525,16 @@ define <2 x bfloat> @test_fdiv(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM80-NEXT: mov.b32 {%rs3, %rs4}, %r1;
; SM80-NEXT: cvt.f32.bf16 %f2, %rs4;
; SM80-NEXT: div.rn.f32 %f3, %f2, %f1;
-; SM80-NEXT: cvt.rn.bf16.f32 %rs5, %f3;
; SM80-NEXT: cvt.f32.bf16 %f4, %rs1;
; SM80-NEXT: cvt.f32.bf16 %f5, %rs3;
; SM80-NEXT: div.rn.f32 %f6, %f5, %f4;
-; SM80-NEXT: cvt.rn.bf16.f32 %rs6, %f6;
-; SM80-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM80-NEXT: cvt.rn.bf16x2.f32 %r3, %f6, %f3;
; SM80-NEXT: st.param.b32 [func_retval0], %r3;
; SM80-NEXT: ret;
;
; SM80-FTZ-LABEL: test_fdiv(
; SM80-FTZ: {
-; SM80-FTZ-NEXT: .reg .b16 %rs<7>;
+; SM80-FTZ-NEXT: .reg .b16 %rs<5>;
; SM80-FTZ-NEXT: .reg .b32 %r<4>;
; SM80-FTZ-NEXT: .reg .f32 %f<7>;
; SM80-FTZ-EMPTY:
@@ -560,18 +546,16 @@ define <2 x bfloat> @test_fdiv(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM80-FTZ-NEXT: mov.b32 {%rs3, %rs4}, %r1;
; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f2, %rs4;
; SM80-FTZ-NEXT: div.rn.ftz.f32 %f3, %f2, %f1;
-; SM80-FTZ-NEXT: cvt.rn.bf16.f32 %rs5, %f3;
; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f4, %rs1;
; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f5, %rs3;
; SM80-FTZ-NEXT: div.rn.ftz.f32 %f6, %f5, %f4;
-; SM80-FTZ-NEXT: cvt.rn.bf16.f32 %rs6, %f6;
-; SM80-FTZ-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM80-FTZ-NEXT: cvt.rn.bf16x2.f32 %r3, %f6, %f3;
; SM80-FTZ-NEXT: st.param.b32 [func_retval0], %r3;
; SM80-FTZ-NEXT: ret;
;
; SM90-LABEL: test_fdiv(
; SM90: {
-; SM90-NEXT: .reg .b16 %rs<7>;
+; SM90-NEXT: .reg .b16 %rs<5>;
; SM90-NEXT: .reg .b32 %r<4>;
; SM90-NEXT: .reg .f32 %f<7>;
; SM90-EMPTY:
@@ -583,12 +567,10 @@ define <2 x bfloat> @test_fdiv(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r1;
; SM90-NEXT: cvt.f32.bf16 %f2, %rs4;
; SM90-NEXT: div.rn.f32 %f3, %f2, %f1;
-; SM90-NEXT: cvt.rn.bf16.f32 %rs5, %f3;
; SM90-NEXT: cvt.f32.bf16 %f4, %rs1;
; SM90-NEXT: cvt.f32.bf16 %f5, %rs3;
; SM90-NEXT: div.rn.f32 %f6, %f5, %f4;
-; SM90-NEXT: cvt.rn.bf16.f32 %rs6, %f6;
-; SM90-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM90-NEXT: cvt.rn.bf16x2.f32 %r3, %f6, %f3;
; SM90-NEXT: st.param.b32 [func_retval0], %r3;
; SM90-NEXT: ret;
%r = fdiv <2 x bfloat> %a, %b
diff --git a/llvm/test/CodeGen/NVPTX/bf16x2-instructions-approx.ll b/llvm/test/CodeGen/NVPTX/bf16x2-instructions-approx.ll
index 3e54aaf5580729..9625bf7e5481f4 100644
--- a/llvm/test/CodeGen/NVPTX/bf16x2-instructions-approx.ll
+++ b/llvm/test/CodeGen/NVPTX/bf16x2-instructions-approx.ll
@@ -7,6 +7,7 @@ target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
declare <2 x bfloat> @llvm.sin.f16(<2 x bfloat> %a) #0
declare <2 x bfloat> @llvm.cos.f16(<2 x bfloat> %a) #0
+
define <2 x bfloat> @test_sin(<2 x bfloat> %a) #0 #1 {
; CHECK-LABEL: test_sin(
; CHECK: {
diff --git a/llvm/test/CodeGen/NVPTX/convert-sm80.ll b/llvm/test/CodeGen/NVPTX/convert-sm80.ll
index 4e30cebfe90251..3023f0e3064770 100644
--- a/llvm/test/CodeGen/NVPTX/convert-sm80.ll
+++ b/llvm/test/CodeGen/NVPTX/convert-sm80.ll
@@ -1,41 +1,70 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc < %s -march=nvptx64 -mcpu=sm_80 -mattr=+ptx70 | FileCheck %s
; RUN: %if ptxas-11.0 %{ llc < %s -march=nvptx64 -mcpu=sm_80 -mattr=+ptx70 | %ptxas-verify -arch=sm_80 %}
-; CHECK-LABEL: cvt_rn_bf16x2_f32
define <2 x bfloat> @cvt_rn_bf16x2_f32(float %f1, float %f2) {
-
-; CHECK: cvt.rn.bf16x2.f32
- %val = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rn(float %f1, float %f2);
-
-ret <2 x bfloat> %val
+; CHECK-LABEL: cvt_rn_bf16x2_f32(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .f32 %f<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.f32 %f1, [cvt_rn_bf16x2_f32_param_0];
+; CHECK-NEXT: ld.param.f32 %f2, [cvt_rn_bf16x2_f32_param_1];
+; CHECK-NEXT: cvt.rn.bf16x2.f32 %r1, %f1, %f2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %val = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rn(float %f1, float %f2)
+ ret <2 x bfloat> %val
}
-; CHECK-LABEL: cvt_rn_relu_bf16x2_f32
define <2 x bfloat> @cvt_rn_relu_bf16x2_f32(float %f1, float %f2) {
-
-; CHECK: cvt.rn.relu.bf16x2.f32
-%val = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rn.relu(float %f1, float %f2);
-
-ret <2 x bfloat> %val
+; CHECK-LABEL: cvt_rn_relu_bf16x2_f32(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .f32 %f<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.f32 %f1, [cvt_rn_relu_bf16x2_f32_param_0];
+; CHECK-NEXT: ld.param.f32 %f2, [cvt_rn_relu_bf16x2_f32_param_1];
+; CHECK-NEXT: cvt.rn.relu.bf16x2.f32 %r1, %f1, %f2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %val = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rn.relu(float %f1, float %f2)
+ ret <2 x bfloat> %val
}
-; CHECK-LABEL: cvt_rz_bf16x2_f32
define <2 x bfloat> @cvt_rz_bf16x2_f32(float %f1, float %f2) {
-
-; CHECK: cvt.rz.bf16x2.f32
- %val = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %f1, float %f2);
-
-ret <2 x bfloat> %val
+; CHECK-LABEL: cvt_rz_bf16x2_f32(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .f32 %f<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.f32 %f1, [cvt_rz_bf16x2_f32_param_0];
+; CHECK-NEXT: ld.param.f32 %f2, [cvt_rz_bf16x2_f32_param_1];
+; CHECK-NEXT: cvt.rz.bf16x2.f32 %r1, %f1, %f2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %val = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %f1, float %f2)
+ ret <2 x bfloat> %val
}
-; CHECK-LABEL: cvt_rz_relu_bf16x2_f32
define <2 x bfloat> @cvt_rz_relu_bf16x2_f32(float %f1, float %f2) {
-
-; CHECK: cvt.rz.relu.bf16x2.f32
-%val = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz.relu(float %f1, float %f2);
-
-ret <2 x bfloat> %val
+; CHECK-LABEL: cvt_rz_relu_bf16x2_f32(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .f32 %f<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.f32 %f1, [cvt_rz_relu_bf16x2_f32_param_0];
+; CHECK-NEXT: ld.param.f32 %f2, [cvt_rz_relu_bf16x2_f32_param_1];
+; CHECK-NEXT: cvt.rz.relu.bf16x2.f32 %r1, %f1, %f2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %val = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz.relu(float %f1, float %f2)
+ ret <2 x bfloat> %val
}
declare <2 x bfloat> @llvm.nvvm.ff2bf16x2.rn(float, float)
@@ -43,40 +72,68 @@ declare <2 x bfloat> @llvm.nvvm.ff2bf16x2.rn.relu(float, float)
declare <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float, float)
declare <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz.relu(float, float)
-; CHECK-LABEL: cvt_rn_f16x2_f32
define <2 x half> @cvt_rn_f16x2_f32(float %f1, float %f2) {
-
-; CHECK: cvt.rn.f16x2.f32
- %val = call <2 x half> @llvm.nvvm.ff2f16x2.rn(float %f1, float %f2);
-
-ret <2 x half> %val
+; CHECK-LABEL: cvt_rn_f16x2_f32(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .f32 %f<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.f32 %f1, [cvt_rn_f16x2_f32_param_0];
+; CHECK-NEXT: ld.param.f32 %f2, [cvt_rn_f16x2_f32_param_1];
+; CHECK-NEXT: cvt.rn.f16x2.f32 %r1, %f1, %f2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %val = call <2 x half> @llvm.nvvm.ff2f16x2.rn(float %f1, float %f2)
+ ret <2 x half> %val
}
-; CHECK-LABEL: cvt_rn_relu_f16x2_f32
define <2 x half> @cvt_rn_relu_f16x2_f32(float %f1, float %f2) {
-
-; CHECK: cvt.rn.relu.f16x2.f32
-%val = call <2 x half> @llvm.nvvm.ff2f16x2.rn.relu(float %f1, float %f2);
-
-ret <2 x half> %val
+; CHECK-LABEL: cvt_rn_relu_f16x2_f32(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .f32 %f<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.f32 %f1, [cvt_rn_relu_f16x2_f32_param_0];
+; CHECK-NEXT: ld.param.f32 %f2, [cvt_rn_relu_f16x2_f32_param_1];
+; CHECK-NEXT: cvt.rn.relu.f16x2.f32 %r1, %f1, %f2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %val = call <2 x half> @llvm.nvvm.ff2f16x2.rn.relu(float %f1, float %f2)
+ ret <2 x half> %val
}
-; CHECK-LABEL: cvt_rz_f16x2_f32
define <2 x half> @cvt_rz_f16x2_f32(float %f1, float %f2) {
-
-; CHECK: cvt.rz.f16x2.f32
- %val = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %f1, float %f2);
-
-ret <2 x half> %val
+; CHECK-LABEL: cvt_rz_f16x2_f32(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .f32 %f<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.f32 %f1, [cvt_rz_f16x2_f32_param_0];
+; CHECK-NEXT: ld.param.f32 %f2, [cvt_rz_f16x2_f32_param_1];
+; CHECK-NEXT: cvt.rz.f16x2.f32 %r1, %f1, %f2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %val = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %f1, float %f2)
+ ret <2 x half> %val
}
-; CHECK-LABEL: cvt_rz_relu_f16x2_f32
define <2 x half> @cvt_rz_relu_f16x2_f32(float %f1, float %f2) {
-
-; CHECK: cvt.rz.relu.f16x2.f32
-%val = call <2 x half> @llvm.nvvm.ff2f16x2.rz.relu(float %f1, float %f2);
-
-ret <2 x half> %val
+; CHECK-LABEL: cvt_rz_relu_f16x2_f32(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .f32 %f<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.f32 %f1, [cvt_rz_relu_f16x2_f32_param_0];
+; CHECK-NEXT: ld.param.f32 %f2, [cvt_rz_relu_f16x2_f32_param_1];
+; CHECK-NEXT: cvt.rz.relu.f16x2.f32 %r1, %f1, %f2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %val = call <2 x half> @llvm.nvvm.ff2f16x2.rz.relu(float %f1, float %f2)
+ ret <2 x half> %val
}
declare <2 x half> @llvm.nvvm.ff2f16x2.rn(float, float)
@@ -84,40 +141,64 @@ declare <2 x half> @llvm.nvvm.ff2f16x2.rn.relu(float, float)
declare <2 x half> @llvm.nvvm.ff2f16x2.rz(float, float)
declare <2 x half> @llvm.nvvm.ff2f16x2.rz.relu(float, float)
-; CHECK-LABEL: cvt_rn_bf16_f32
define bfloat @cvt_rn_bf16_f32(float %f1) {
-
-; CHECK: cvt.rn.bf16.f32
- %val = call bfloat @llvm.nvvm.f2bf16.rn(float %f1);
-
-ret bfloat %val
+; CHECK-LABEL: cvt_rn_bf16_f32(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .f32 %f<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.f32 %f1, [cvt_rn_bf16_f32_param_0];
+; CHECK-NEXT: cvt.rn.bf16.f32 %rs1, %f1;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %val = call bfloat @llvm.nvvm.f2bf16.rn(float %f1)
+ ret bfloat %val
}
-; CHECK-LABEL: cvt_rn_relu_bf16_f32
define bfloat @cvt_rn_relu_bf16_f32(float %f1) {
-
-; CHECK: cvt.rn.relu.bf16.f32
-%val = call bfloat @llvm.nvvm.f2bf16.rn.relu(float %f1);
-
-ret bfloat %val
+; CHECK-LABEL: cvt_rn_relu_bf16_f32(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .f32 %f<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.f32 %f1, [cvt_rn_relu_bf16_f32_param_0];
+; CHECK-NEXT: cvt.rn.relu.bf16.f32 %rs1, %f1;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %val = call bfloat @llvm.nvvm.f2bf16.rn.relu(float %f1)
+ ret bfloat %val
}
-; CHECK-LABEL: cvt_rz_bf16_f32
define bfloat @cvt_rz_bf16_f32(float %f1) {
-
-; CHECK: cvt.rz.bf16.f32
- %val = call bfloat @llvm.nvvm.f2bf16.rz(float %f1);
-
-ret bfloat %val
+; CHECK-LABEL: cvt_rz_bf16_f32(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .f32 %f<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.f32 %f1, [cvt_rz_bf16_f32_param_0];
+; CHECK-NEXT: cvt.rz.bf16.f32 %rs1, %f1;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %val = call bfloat @llvm.nvvm.f2bf16.rz(float %f1)
+ ret bfloat %val
}
-; CHECK-LABEL: cvt_rz_relu_bf16_f32
define bfloat @cvt_rz_relu_bf16_f32(float %f1) {
-
-; CHECK: cvt.rz.relu.bf16.f32
-%val = call bfloat @llvm.nvvm.f2bf16.rz.relu(float %f1);
-
-ret bfloat %val
+; CHECK-LABEL: cvt_rz_relu_bf16_f32(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .f32 %f<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.f32 %f1, [cvt_rz_relu_bf16_f32_param_0];
+; CHECK-NEXT: cvt.rz.relu.bf16.f32 %rs1, %f1;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %val = call bfloat @llvm.nvvm.f2bf16.rz.relu(float %f1)
+ ret bfloat %val
}
declare bfloat @llvm.nvvm.f2bf16.rn(float)
@@ -125,13 +206,58 @@ declare bfloat @llvm.nvvm.f2bf16.rn.relu(float)
declare bfloat @llvm.nvvm.f2bf16.rz(float)
declare bfloat @llvm.nvvm.f2bf16.rz.relu(float)
-; CHECK-LABEL: cvt_rna_tf32_f32
define i32 @cvt_rna_tf32_f32(float %f1) {
+; CHECK-LABEL: cvt_rna_tf32_f32(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .f32 %f<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.f32 %f1, [cvt_rna_tf32_f32_param_0];
+; CHECK-NEXT: cvt.rna.tf32.f32 %r1, %f1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %val = call i32 @llvm.nvvm.f2tf32.rna(float %f1)
+ ret i32 %val
+}
-; CHECK: cvt.rna.tf32.f32
- %val = call i32 @llvm.nvvm.f2tf32.rna(float %f1);
+declare i32 @llvm.nvvm.f2tf32.rna(float)
-ret i32 %val
+
+define <2 x bfloat> @fold_ff2bf16x2(float %a, float %b) {
+; CHECK-LABEL: fold_ff2bf16x2(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .f32 %f<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.f32 %f1, [fold_ff2bf16x2_param_0];
+; CHECK-NEXT: ld.param.f32 %f2, [fold_ff2bf16x2_param_1];
+; CHECK-NEXT: cvt.rn.bf16x2.f32 %r1, %f1, %f2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %ah = fptrunc float %a to bfloat
+ %bh = fptrunc float %b to bfloat
+ %v0 = insertelement <2 x bfloat> poison, bfloat %ah, i64 0
+ %v1 = insertelement <2 x bfloat> %v0, bfloat %bh, i64 1
+ ret <2 x bfloat> %v1
}
-declare i32 @llvm.nvvm.f2tf32.rna(float)
+define <2 x half> @fold_ff2f16x2(float %a, float %b) {
+; CHECK-LABEL: fold_ff2f16x2(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .f32 %f<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.f32 %f1, [fold_ff2f16x2_param_0];
+; CHECK-NEXT: ld.param.f32 %f2, [fold_ff2f16x2_param_1];
+; CHECK-NEXT: cvt.rn.f16x2.f32 %r1, %f1, %f2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %ah = fptrunc float %a to half
+ %bh = fptrunc float %b to half
+ %v0 = insertelement <2 x half> poison, half %ah, i64 0
+ %v1 = insertelement <2 x half> %v0, half %bh, i64 1
+ ret <2 x half> %v1
+}
>From 00fe344688218aed4aa9ce56f7713ae3828aaa0a Mon Sep 17 00:00:00 2001
From: Alex Maclean <amaclean at nvidia.com>
Date: Fri, 15 Nov 2024 18:18:30 +0000
Subject: [PATCH 2/5] fixup - flip operand order
---
llvm/lib/Target/NVPTX/NVPTXInstrInfo.td | 4 +-
llvm/test/CodeGen/NVPTX/bf16-instructions.ll | 72 +++++++++----------
.../NVPTX/bf16x2-instructions-approx.ll | 1 -
llvm/test/CodeGen/NVPTX/convert-sm80.ll | 4 +-
4 files changed, 40 insertions(+), 41 deletions(-)
diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
index abc8fcb582020f..031669ea41357b 100644
--- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
+++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
@@ -733,12 +733,12 @@ def fpround_oneuse : PatFrag<(ops node:$a), (fpround node:$a), [{
def : Pat<(v2bf16 (build_vector (bf16 (fpround_oneuse Float32Regs:$a)),
(bf16 (fpround_oneuse Float32Regs:$b)))),
- (CVT_bf16x2_f32 Float32Regs:$a, Float32Regs:$b, CvtRN)>,
+ (CVT_bf16x2_f32 Float32Regs:$b, Float32Regs:$a, CvtRN)>,
Requires<[hasPTX<70>, hasSM<80>, hasBF16Math]>;
def : Pat<(v2f16 (build_vector (f16 (fpround_oneuse Float32Regs:$a)),
(f16 (fpround_oneuse Float32Regs:$b)))),
- (CVT_f16x2_f32 Float32Regs:$a, Float32Regs:$b, CvtRN)>,
+ (CVT_f16x2_f32 Float32Regs:$b, Float32Regs:$a, CvtRN)>,
Requires<[hasPTX<70>, hasSM<80>, useFP16Math]>;
//-----------------------------------
diff --git a/llvm/test/CodeGen/NVPTX/bf16-instructions.ll b/llvm/test/CodeGen/NVPTX/bf16-instructions.ll
index ded3019e173b4c..eee31be80e9826 100644
--- a/llvm/test/CodeGen/NVPTX/bf16-instructions.ll
+++ b/llvm/test/CodeGen/NVPTX/bf16-instructions.ll
@@ -212,12 +212,12 @@ define <2 x bfloat> @test_faddx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM80-NEXT: ld.param.b32 %r1, [test_faddx2_param_0];
; SM80-NEXT: ld.param.b32 %r2, [test_faddx2_param_1];
; SM80-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM80-NEXT: cvt.f32.bf16 %f1, %rs2;
+; SM80-NEXT: cvt.f32.bf16 %f1, %rs1;
; SM80-NEXT: mov.b32 {%rs3, %rs4}, %r1;
-; SM80-NEXT: cvt.f32.bf16 %f2, %rs4;
+; SM80-NEXT: cvt.f32.bf16 %f2, %rs3;
; SM80-NEXT: add.rn.f32 %f3, %f2, %f1;
-; SM80-NEXT: cvt.f32.bf16 %f4, %rs1;
-; SM80-NEXT: cvt.f32.bf16 %f5, %rs3;
+; SM80-NEXT: cvt.f32.bf16 %f4, %rs2;
+; SM80-NEXT: cvt.f32.bf16 %f5, %rs4;
; SM80-NEXT: add.rn.f32 %f6, %f5, %f4;
; SM80-NEXT: cvt.rn.bf16x2.f32 %r3, %f6, %f3;
; SM80-NEXT: st.param.b32 [func_retval0], %r3;
@@ -233,12 +233,12 @@ define <2 x bfloat> @test_faddx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM80-FTZ-NEXT: ld.param.b32 %r1, [test_faddx2_param_0];
; SM80-FTZ-NEXT: ld.param.b32 %r2, [test_faddx2_param_1];
; SM80-FTZ-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f1, %rs2;
+; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f1, %rs1;
; SM80-FTZ-NEXT: mov.b32 {%rs3, %rs4}, %r1;
-; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f2, %rs4;
+; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f2, %rs3;
; SM80-FTZ-NEXT: add.rn.ftz.f32 %f3, %f2, %f1;
-; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f4, %rs1;
-; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f5, %rs3;
+; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f4, %rs2;
+; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f5, %rs4;
; SM80-FTZ-NEXT: add.rn.ftz.f32 %f6, %f5, %f4;
; SM80-FTZ-NEXT: cvt.rn.bf16x2.f32 %r3, %f6, %f3;
; SM80-FTZ-NEXT: st.param.b32 [func_retval0], %r3;
@@ -315,12 +315,12 @@ define <2 x bfloat> @test_fsubx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM80-NEXT: ld.param.b32 %r1, [test_fsubx2_param_0];
; SM80-NEXT: ld.param.b32 %r2, [test_fsubx2_param_1];
; SM80-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM80-NEXT: cvt.f32.bf16 %f1, %rs2;
+; SM80-NEXT: cvt.f32.bf16 %f1, %rs1;
; SM80-NEXT: mov.b32 {%rs3, %rs4}, %r1;
-; SM80-NEXT: cvt.f32.bf16 %f2, %rs4;
+; SM80-NEXT: cvt.f32.bf16 %f2, %rs3;
; SM80-NEXT: sub.rn.f32 %f3, %f2, %f1;
-; SM80-NEXT: cvt.f32.bf16 %f4, %rs1;
-; SM80-NEXT: cvt.f32.bf16 %f5, %rs3;
+; SM80-NEXT: cvt.f32.bf16 %f4, %rs2;
+; SM80-NEXT: cvt.f32.bf16 %f5, %rs4;
; SM80-NEXT: sub.rn.f32 %f6, %f5, %f4;
; SM80-NEXT: cvt.rn.bf16x2.f32 %r3, %f6, %f3;
; SM80-NEXT: st.param.b32 [func_retval0], %r3;
@@ -336,12 +336,12 @@ define <2 x bfloat> @test_fsubx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM80-FTZ-NEXT: ld.param.b32 %r1, [test_fsubx2_param_0];
; SM80-FTZ-NEXT: ld.param.b32 %r2, [test_fsubx2_param_1];
; SM80-FTZ-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f1, %rs2;
+; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f1, %rs1;
; SM80-FTZ-NEXT: mov.b32 {%rs3, %rs4}, %r1;
-; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f2, %rs4;
+; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f2, %rs3;
; SM80-FTZ-NEXT: sub.rn.ftz.f32 %f3, %f2, %f1;
-; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f4, %rs1;
-; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f5, %rs3;
+; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f4, %rs2;
+; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f5, %rs4;
; SM80-FTZ-NEXT: sub.rn.ftz.f32 %f6, %f5, %f4;
; SM80-FTZ-NEXT: cvt.rn.bf16x2.f32 %r3, %f6, %f3;
; SM80-FTZ-NEXT: st.param.b32 [func_retval0], %r3;
@@ -418,12 +418,12 @@ define <2 x bfloat> @test_fmulx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM80-NEXT: ld.param.b32 %r1, [test_fmulx2_param_0];
; SM80-NEXT: ld.param.b32 %r2, [test_fmulx2_param_1];
; SM80-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM80-NEXT: cvt.f32.bf16 %f1, %rs2;
+; SM80-NEXT: cvt.f32.bf16 %f1, %rs1;
; SM80-NEXT: mov.b32 {%rs3, %rs4}, %r1;
-; SM80-NEXT: cvt.f32.bf16 %f2, %rs4;
+; SM80-NEXT: cvt.f32.bf16 %f2, %rs3;
; SM80-NEXT: mul.rn.f32 %f3, %f2, %f1;
-; SM80-NEXT: cvt.f32.bf16 %f4, %rs1;
-; SM80-NEXT: cvt.f32.bf16 %f5, %rs3;
+; SM80-NEXT: cvt.f32.bf16 %f4, %rs2;
+; SM80-NEXT: cvt.f32.bf16 %f5, %rs4;
; SM80-NEXT: mul.rn.f32 %f6, %f5, %f4;
; SM80-NEXT: cvt.rn.bf16x2.f32 %r3, %f6, %f3;
; SM80-NEXT: st.param.b32 [func_retval0], %r3;
@@ -439,12 +439,12 @@ define <2 x bfloat> @test_fmulx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM80-FTZ-NEXT: ld.param.b32 %r1, [test_fmulx2_param_0];
; SM80-FTZ-NEXT: ld.param.b32 %r2, [test_fmulx2_param_1];
; SM80-FTZ-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f1, %rs2;
+; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f1, %rs1;
; SM80-FTZ-NEXT: mov.b32 {%rs3, %rs4}, %r1;
-; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f2, %rs4;
+; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f2, %rs3;
; SM80-FTZ-NEXT: mul.rn.ftz.f32 %f3, %f2, %f1;
-; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f4, %rs1;
-; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f5, %rs3;
+; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f4, %rs2;
+; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f5, %rs4;
; SM80-FTZ-NEXT: mul.rn.ftz.f32 %f6, %f5, %f4;
; SM80-FTZ-NEXT: cvt.rn.bf16x2.f32 %r3, %f6, %f3;
; SM80-FTZ-NEXT: st.param.b32 [func_retval0], %r3;
@@ -521,12 +521,12 @@ define <2 x bfloat> @test_fdiv(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM80-NEXT: ld.param.b32 %r1, [test_fdiv_param_0];
; SM80-NEXT: ld.param.b32 %r2, [test_fdiv_param_1];
; SM80-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM80-NEXT: cvt.f32.bf16 %f1, %rs2;
+; SM80-NEXT: cvt.f32.bf16 %f1, %rs1;
; SM80-NEXT: mov.b32 {%rs3, %rs4}, %r1;
-; SM80-NEXT: cvt.f32.bf16 %f2, %rs4;
+; SM80-NEXT: cvt.f32.bf16 %f2, %rs3;
; SM80-NEXT: div.rn.f32 %f3, %f2, %f1;
-; SM80-NEXT: cvt.f32.bf16 %f4, %rs1;
-; SM80-NEXT: cvt.f32.bf16 %f5, %rs3;
+; SM80-NEXT: cvt.f32.bf16 %f4, %rs2;
+; SM80-NEXT: cvt.f32.bf16 %f5, %rs4;
; SM80-NEXT: div.rn.f32 %f6, %f5, %f4;
; SM80-NEXT: cvt.rn.bf16x2.f32 %r3, %f6, %f3;
; SM80-NEXT: st.param.b32 [func_retval0], %r3;
@@ -542,12 +542,12 @@ define <2 x bfloat> @test_fdiv(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM80-FTZ-NEXT: ld.param.b32 %r1, [test_fdiv_param_0];
; SM80-FTZ-NEXT: ld.param.b32 %r2, [test_fdiv_param_1];
; SM80-FTZ-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f1, %rs2;
+; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f1, %rs1;
; SM80-FTZ-NEXT: mov.b32 {%rs3, %rs4}, %r1;
-; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f2, %rs4;
+; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f2, %rs3;
; SM80-FTZ-NEXT: div.rn.ftz.f32 %f3, %f2, %f1;
-; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f4, %rs1;
-; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f5, %rs3;
+; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f4, %rs2;
+; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f5, %rs4;
; SM80-FTZ-NEXT: div.rn.ftz.f32 %f6, %f5, %f4;
; SM80-FTZ-NEXT: cvt.rn.bf16x2.f32 %r3, %f6, %f3;
; SM80-FTZ-NEXT: st.param.b32 [func_retval0], %r3;
@@ -563,12 +563,12 @@ define <2 x bfloat> @test_fdiv(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM90-NEXT: ld.param.b32 %r1, [test_fdiv_param_0];
; SM90-NEXT: ld.param.b32 %r2, [test_fdiv_param_1];
; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM90-NEXT: cvt.f32.bf16 %f1, %rs2;
+; SM90-NEXT: cvt.f32.bf16 %f1, %rs1;
; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r1;
-; SM90-NEXT: cvt.f32.bf16 %f2, %rs4;
+; SM90-NEXT: cvt.f32.bf16 %f2, %rs3;
; SM90-NEXT: div.rn.f32 %f3, %f2, %f1;
-; SM90-NEXT: cvt.f32.bf16 %f4, %rs1;
-; SM90-NEXT: cvt.f32.bf16 %f5, %rs3;
+; SM90-NEXT: cvt.f32.bf16 %f4, %rs2;
+; SM90-NEXT: cvt.f32.bf16 %f5, %rs4;
; SM90-NEXT: div.rn.f32 %f6, %f5, %f4;
; SM90-NEXT: cvt.rn.bf16x2.f32 %r3, %f6, %f3;
; SM90-NEXT: st.param.b32 [func_retval0], %r3;
diff --git a/llvm/test/CodeGen/NVPTX/bf16x2-instructions-approx.ll b/llvm/test/CodeGen/NVPTX/bf16x2-instructions-approx.ll
index 9625bf7e5481f4..3e54aaf5580729 100644
--- a/llvm/test/CodeGen/NVPTX/bf16x2-instructions-approx.ll
+++ b/llvm/test/CodeGen/NVPTX/bf16x2-instructions-approx.ll
@@ -7,7 +7,6 @@ target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
declare <2 x bfloat> @llvm.sin.f16(<2 x bfloat> %a) #0
declare <2 x bfloat> @llvm.cos.f16(<2 x bfloat> %a) #0
-
define <2 x bfloat> @test_sin(<2 x bfloat> %a) #0 #1 {
; CHECK-LABEL: test_sin(
; CHECK: {
diff --git a/llvm/test/CodeGen/NVPTX/convert-sm80.ll b/llvm/test/CodeGen/NVPTX/convert-sm80.ll
index 3023f0e3064770..b739148393ced6 100644
--- a/llvm/test/CodeGen/NVPTX/convert-sm80.ll
+++ b/llvm/test/CodeGen/NVPTX/convert-sm80.ll
@@ -233,7 +233,7 @@ define <2 x bfloat> @fold_ff2bf16x2(float %a, float %b) {
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.f32 %f1, [fold_ff2bf16x2_param_0];
; CHECK-NEXT: ld.param.f32 %f2, [fold_ff2bf16x2_param_1];
-; CHECK-NEXT: cvt.rn.bf16x2.f32 %r1, %f1, %f2;
+; CHECK-NEXT: cvt.rn.bf16x2.f32 %r1, %f2, %f1;
; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
; CHECK-NEXT: ret;
%ah = fptrunc float %a to bfloat
@@ -252,7 +252,7 @@ define <2 x half> @fold_ff2f16x2(float %a, float %b) {
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.f32 %f1, [fold_ff2f16x2_param_0];
; CHECK-NEXT: ld.param.f32 %f2, [fold_ff2f16x2_param_1];
-; CHECK-NEXT: cvt.rn.f16x2.f32 %r1, %f1, %f2;
+; CHECK-NEXT: cvt.rn.f16x2.f32 %r1, %f2, %f1;
; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
; CHECK-NEXT: ret;
%ah = fptrunc float %a to half
>From 2ed3c54ed25731fdeef8a77e8b85c5f246331058 Mon Sep 17 00:00:00 2001
From: Alex Maclean <amaclean at nvidia.com>
Date: Sun, 17 Nov 2024 18:50:33 +0000
Subject: [PATCH 3/5] address comments
---
llvm/lib/Target/NVPTX/NVPTXInstrInfo.td | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
index 031669ea41357b..66684dbff6b335 100644
--- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
+++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
@@ -731,14 +731,14 @@ def fpround_oneuse : PatFrag<(ops node:$a), (fpround node:$a), [{
return N->hasOneUse();
}]>;
-def : Pat<(v2bf16 (build_vector (bf16 (fpround_oneuse Float32Regs:$a)),
- (bf16 (fpround_oneuse Float32Regs:$b)))),
- (CVT_bf16x2_f32 Float32Regs:$b, Float32Regs:$a, CvtRN)>,
+def : Pat<(v2bf16 (build_vector (bf16 (fpround_oneuse Float32Regs:$lo)),
+ (bf16 (fpround_oneuse Float32Regs:$hi)))),
+ (CVT_bf16x2_f32 Float32Regs:$hi, Float32Regs:$lo, CvtRN)>,
Requires<[hasPTX<70>, hasSM<80>, hasBF16Math]>;
-def : Pat<(v2f16 (build_vector (f16 (fpround_oneuse Float32Regs:$a)),
- (f16 (fpround_oneuse Float32Regs:$b)))),
- (CVT_f16x2_f32 Float32Regs:$b, Float32Regs:$a, CvtRN)>,
+def : Pat<(v2f16 (build_vector (f16 (fpround_oneuse Float32Regs:$lo)),
+ (f16 (fpround_oneuse Float32Regs:$hi)))),
+ (CVT_f16x2_f32 Float32Regs:$hi, Float32Regs:$lo, CvtRN)>,
Requires<[hasPTX<70>, hasSM<80>, useFP16Math]>;
//-----------------------------------
>From e6718b429525e1ffa13a10cdefdf59fb1c6606f1 Mon Sep 17 00:00:00 2001
From: Alex Maclean <amaclean at nvidia.com>
Date: Mon, 18 Nov 2024 22:36:55 +0000
Subject: [PATCH 4/5] rebase tests
---
.../NVPTX/bf16x2-instructions-approx.ll | 20 +--
.../test/CodeGen/NVPTX/bf16x2-instructions.ll | 147 +++++++-----------
llvm/test/CodeGen/NVPTX/fma-relu-contract.ll | 28 ++--
.../CodeGen/NVPTX/fma-relu-fma-intrinsic.ll | 20 +--
.../NVPTX/fma-relu-instruction-flag.ll | 48 +++---
5 files changed, 108 insertions(+), 155 deletions(-)
diff --git a/llvm/test/CodeGen/NVPTX/bf16x2-instructions-approx.ll b/llvm/test/CodeGen/NVPTX/bf16x2-instructions-approx.ll
index 3e54aaf5580729..b317697210850a 100644
--- a/llvm/test/CodeGen/NVPTX/bf16x2-instructions-approx.ll
+++ b/llvm/test/CodeGen/NVPTX/bf16x2-instructions-approx.ll
@@ -10,20 +10,18 @@ declare <2 x bfloat> @llvm.cos.f16(<2 x bfloat> %a) #0
define <2 x bfloat> @test_sin(<2 x bfloat> %a) #0 #1 {
; CHECK-LABEL: test_sin(
; CHECK: {
-; CHECK-NEXT: .reg .b16 %rs<5>;
+; CHECK-NEXT: .reg .b16 %rs<3>;
; CHECK-NEXT: .reg .b32 %r<3>;
; CHECK-NEXT: .reg .f32 %f<5>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b32 %r1, [test_sin_param_0];
; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; CHECK-NEXT: cvt.f32.bf16 %f1, %rs2;
+; CHECK-NEXT: cvt.f32.bf16 %f1, %rs1;
; CHECK-NEXT: sin.approx.f32 %f2, %f1;
-; CHECK-NEXT: cvt.rn.bf16.f32 %rs3, %f2;
-; CHECK-NEXT: cvt.f32.bf16 %f3, %rs1;
+; CHECK-NEXT: cvt.f32.bf16 %f3, %rs2;
; CHECK-NEXT: sin.approx.f32 %f4, %f3;
-; CHECK-NEXT: cvt.rn.bf16.f32 %rs4, %f4;
-; CHECK-NEXT: mov.b32 %r2, {%rs4, %rs3};
+; CHECK-NEXT: cvt.rn.bf16x2.f32 %r2, %f4, %f2;
; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
; CHECK-NEXT: ret;
%r = call <2 x bfloat> @llvm.sin.f16(<2 x bfloat> %a)
@@ -33,20 +31,18 @@ define <2 x bfloat> @test_sin(<2 x bfloat> %a) #0 #1 {
define <2 x bfloat> @test_cos(<2 x bfloat> %a) #0 #1 {
; CHECK-LABEL: test_cos(
; CHECK: {
-; CHECK-NEXT: .reg .b16 %rs<5>;
+; CHECK-NEXT: .reg .b16 %rs<3>;
; CHECK-NEXT: .reg .b32 %r<3>;
; CHECK-NEXT: .reg .f32 %f<5>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b32 %r1, [test_cos_param_0];
; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; CHECK-NEXT: cvt.f32.bf16 %f1, %rs2;
+; CHECK-NEXT: cvt.f32.bf16 %f1, %rs1;
; CHECK-NEXT: cos.approx.f32 %f2, %f1;
-; CHECK-NEXT: cvt.rn.bf16.f32 %rs3, %f2;
-; CHECK-NEXT: cvt.f32.bf16 %f3, %rs1;
+; CHECK-NEXT: cvt.f32.bf16 %f3, %rs2;
; CHECK-NEXT: cos.approx.f32 %f4, %f3;
-; CHECK-NEXT: cvt.rn.bf16.f32 %rs4, %f4;
-; CHECK-NEXT: mov.b32 %r2, {%rs4, %rs3};
+; CHECK-NEXT: cvt.rn.bf16x2.f32 %r2, %f4, %f2;
; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
; CHECK-NEXT: ret;
%r = call <2 x bfloat> @llvm.cos.f16(<2 x bfloat> %a)
diff --git a/llvm/test/CodeGen/NVPTX/bf16x2-instructions.ll b/llvm/test/CodeGen/NVPTX/bf16x2-instructions.ll
index e545d4c1177915..7f29676686a26d 100644
--- a/llvm/test/CodeGen/NVPTX/bf16x2-instructions.ll
+++ b/llvm/test/CodeGen/NVPTX/bf16x2-instructions.ll
@@ -22,20 +22,18 @@ define <2 x bfloat> @test_ret_const() #0 {
define <2 x bfloat> @test_fadd_imm_0(<2 x bfloat> %a) #0 {
; SM80-LABEL: test_fadd_imm_0(
; SM80: {
-; SM80-NEXT: .reg .b16 %rs<5>;
+; SM80-NEXT: .reg .b16 %rs<3>;
; SM80-NEXT: .reg .b32 %r<3>;
; SM80-NEXT: .reg .f32 %f<5>;
; SM80-EMPTY:
; SM80-NEXT: // %bb.0:
; SM80-NEXT: ld.param.b32 %r1, [test_fadd_imm_0_param_0];
; SM80-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; SM80-NEXT: cvt.f32.bf16 %f1, %rs2;
-; SM80-NEXT: add.rn.f32 %f2, %f1, 0f40000000;
-; SM80-NEXT: cvt.rn.bf16.f32 %rs3, %f2;
-; SM80-NEXT: cvt.f32.bf16 %f3, %rs1;
-; SM80-NEXT: add.rn.f32 %f4, %f3, 0f3F800000;
-; SM80-NEXT: cvt.rn.bf16.f32 %rs4, %f4;
-; SM80-NEXT: mov.b32 %r2, {%rs4, %rs3};
+; SM80-NEXT: cvt.f32.bf16 %f1, %rs1;
+; SM80-NEXT: add.rn.f32 %f2, %f1, 0f3F800000;
+; SM80-NEXT: cvt.f32.bf16 %f3, %rs2;
+; SM80-NEXT: add.rn.f32 %f4, %f3, 0f40000000;
+; SM80-NEXT: cvt.rn.bf16x2.f32 %r2, %f4, %f2;
; SM80-NEXT: st.param.b32 [func_retval0], %r2;
; SM80-NEXT: ret;
;
@@ -84,7 +82,7 @@ define bfloat @test_fadd_imm_1(bfloat %a) #0 {
define <2 x bfloat> @test_fsubx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM80-LABEL: test_fsubx2(
; SM80: {
-; SM80-NEXT: .reg .b16 %rs<7>;
+; SM80-NEXT: .reg .b16 %rs<5>;
; SM80-NEXT: .reg .b32 %r<4>;
; SM80-NEXT: .reg .f32 %f<7>;
; SM80-EMPTY:
@@ -92,16 +90,14 @@ define <2 x bfloat> @test_fsubx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM80-NEXT: ld.param.b32 %r1, [test_fsubx2_param_0];
; SM80-NEXT: ld.param.b32 %r2, [test_fsubx2_param_1];
; SM80-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM80-NEXT: cvt.f32.bf16 %f1, %rs2;
+; SM80-NEXT: cvt.f32.bf16 %f1, %rs1;
; SM80-NEXT: mov.b32 {%rs3, %rs4}, %r1;
-; SM80-NEXT: cvt.f32.bf16 %f2, %rs4;
+; SM80-NEXT: cvt.f32.bf16 %f2, %rs3;
; SM80-NEXT: sub.rn.f32 %f3, %f2, %f1;
-; SM80-NEXT: cvt.rn.bf16.f32 %rs5, %f3;
-; SM80-NEXT: cvt.f32.bf16 %f4, %rs1;
-; SM80-NEXT: cvt.f32.bf16 %f5, %rs3;
+; SM80-NEXT: cvt.f32.bf16 %f4, %rs2;
+; SM80-NEXT: cvt.f32.bf16 %f5, %rs4;
; SM80-NEXT: sub.rn.f32 %f6, %f5, %f4;
-; SM80-NEXT: cvt.rn.bf16.f32 %rs6, %f6;
-; SM80-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM80-NEXT: cvt.rn.bf16x2.f32 %r3, %f6, %f3;
; SM80-NEXT: st.param.b32 [func_retval0], %r3;
; SM80-NEXT: ret;
;
@@ -122,7 +118,7 @@ define <2 x bfloat> @test_fsubx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
define <2 x bfloat> @test_fmulx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM80-LABEL: test_fmulx2(
; SM80: {
-; SM80-NEXT: .reg .b16 %rs<7>;
+; SM80-NEXT: .reg .b16 %rs<5>;
; SM80-NEXT: .reg .b32 %r<4>;
; SM80-NEXT: .reg .f32 %f<7>;
; SM80-EMPTY:
@@ -130,16 +126,14 @@ define <2 x bfloat> @test_fmulx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM80-NEXT: ld.param.b32 %r1, [test_fmulx2_param_0];
; SM80-NEXT: ld.param.b32 %r2, [test_fmulx2_param_1];
; SM80-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM80-NEXT: cvt.f32.bf16 %f1, %rs2;
+; SM80-NEXT: cvt.f32.bf16 %f1, %rs1;
; SM80-NEXT: mov.b32 {%rs3, %rs4}, %r1;
-; SM80-NEXT: cvt.f32.bf16 %f2, %rs4;
+; SM80-NEXT: cvt.f32.bf16 %f2, %rs3;
; SM80-NEXT: mul.rn.f32 %f3, %f2, %f1;
-; SM80-NEXT: cvt.rn.bf16.f32 %rs5, %f3;
-; SM80-NEXT: cvt.f32.bf16 %f4, %rs1;
-; SM80-NEXT: cvt.f32.bf16 %f5, %rs3;
+; SM80-NEXT: cvt.f32.bf16 %f4, %rs2;
+; SM80-NEXT: cvt.f32.bf16 %f5, %rs4;
; SM80-NEXT: mul.rn.f32 %f6, %f5, %f4;
-; SM80-NEXT: cvt.rn.bf16.f32 %rs6, %f6;
-; SM80-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM80-NEXT: cvt.rn.bf16x2.f32 %r3, %f6, %f3;
; SM80-NEXT: st.param.b32 [func_retval0], %r3;
; SM80-NEXT: ret;
;
@@ -160,7 +154,7 @@ define <2 x bfloat> @test_fmulx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
define <2 x bfloat> @test_fdiv(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; CHECK-LABEL: test_fdiv(
; CHECK: {
-; CHECK-NEXT: .reg .b16 %rs<7>;
+; CHECK-NEXT: .reg .b16 %rs<5>;
; CHECK-NEXT: .reg .b32 %r<4>;
; CHECK-NEXT: .reg .f32 %f<7>;
; CHECK-EMPTY:
@@ -168,16 +162,14 @@ define <2 x bfloat> @test_fdiv(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; CHECK-NEXT: ld.param.b32 %r1, [test_fdiv_param_0];
; CHECK-NEXT: ld.param.b32 %r2, [test_fdiv_param_1];
; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; CHECK-NEXT: cvt.f32.bf16 %f1, %rs2;
+; CHECK-NEXT: cvt.f32.bf16 %f1, %rs1;
; CHECK-NEXT: mov.b32 {%rs3, %rs4}, %r1;
-; CHECK-NEXT: cvt.f32.bf16 %f2, %rs4;
+; CHECK-NEXT: cvt.f32.bf16 %f2, %rs3;
; CHECK-NEXT: div.rn.f32 %f3, %f2, %f1;
-; CHECK-NEXT: cvt.rn.bf16.f32 %rs5, %f3;
-; CHECK-NEXT: cvt.f32.bf16 %f4, %rs1;
-; CHECK-NEXT: cvt.f32.bf16 %f5, %rs3;
+; CHECK-NEXT: cvt.f32.bf16 %f4, %rs2;
+; CHECK-NEXT: cvt.f32.bf16 %f5, %rs4;
; CHECK-NEXT: div.rn.f32 %f6, %f5, %f4;
-; CHECK-NEXT: cvt.rn.bf16.f32 %rs6, %f6;
-; CHECK-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; CHECK-NEXT: cvt.rn.bf16x2.f32 %r3, %f6, %f3;
; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
; CHECK-NEXT: ret;
%r = fdiv <2 x bfloat> %a, %b
@@ -418,15 +410,12 @@ define <2 x bfloat> @test_select_cc_bf16_f32(<2 x bfloat> %a, <2 x bfloat> %b,
define <2 x bfloat> @test_fptrunc_2xfloat(<2 x float> %a) #0 {
; CHECK-LABEL: test_fptrunc_2xfloat(
; CHECK: {
-; CHECK-NEXT: .reg .b16 %rs<3>;
; CHECK-NEXT: .reg .b32 %r<2>;
; CHECK-NEXT: .reg .f32 %f<3>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.v2.f32 {%f1, %f2}, [test_fptrunc_2xfloat_param_0];
-; CHECK-NEXT: cvt.rn.bf16.f32 %rs1, %f2;
-; CHECK-NEXT: cvt.rn.bf16.f32 %rs2, %f1;
-; CHECK-NEXT: mov.b32 %r1, {%rs2, %rs1};
+; CHECK-NEXT: cvt.rn.bf16x2.f32 %r1, %f2, %f1;
; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
; CHECK-NEXT: ret;
%r = fptrunc <2 x float> %a to <2 x bfloat>
@@ -503,20 +492,18 @@ declare <2 x bfloat> @llvm.fmuladd.f16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bf
define <2 x bfloat> @test_sqrt(<2 x bfloat> %a) #0 {
; CHECK-LABEL: test_sqrt(
; CHECK: {
-; CHECK-NEXT: .reg .b16 %rs<5>;
+; CHECK-NEXT: .reg .b16 %rs<3>;
; CHECK-NEXT: .reg .b32 %r<3>;
; CHECK-NEXT: .reg .f32 %f<5>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b32 %r1, [test_sqrt_param_0];
; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; CHECK-NEXT: cvt.f32.bf16 %f1, %rs2;
+; CHECK-NEXT: cvt.f32.bf16 %f1, %rs1;
; CHECK-NEXT: sqrt.rn.f32 %f2, %f1;
-; CHECK-NEXT: cvt.rn.bf16.f32 %rs3, %f2;
-; CHECK-NEXT: cvt.f32.bf16 %f3, %rs1;
+; CHECK-NEXT: cvt.f32.bf16 %f3, %rs2;
; CHECK-NEXT: sqrt.rn.f32 %f4, %f3;
-; CHECK-NEXT: cvt.rn.bf16.f32 %rs4, %f4;
-; CHECK-NEXT: mov.b32 %r2, {%rs4, %rs3};
+; CHECK-NEXT: cvt.rn.bf16x2.f32 %r2, %f4, %f2;
; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
; CHECK-NEXT: ret;
%r = call <2 x bfloat> @llvm.sqrt.f16(<2 x bfloat> %a)
@@ -556,7 +543,7 @@ define <2 x bfloat> @test_fabs(<2 x bfloat> %a) #0 {
define <2 x bfloat> @test_fabs_add(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM80-LABEL: test_fabs_add(
; SM80: {
-; SM80-NEXT: .reg .b16 %rs<11>;
+; SM80-NEXT: .reg .b16 %rs<7>;
; SM80-NEXT: .reg .b32 %r<6>;
; SM80-NEXT: .reg .f32 %f<11>;
; SM80-EMPTY:
@@ -564,25 +551,21 @@ define <2 x bfloat> @test_fabs_add(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM80-NEXT: ld.param.b32 %r1, [test_fabs_add_param_1];
; SM80-NEXT: ld.param.b32 %r2, [test_fabs_add_param_0];
; SM80-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM80-NEXT: cvt.f32.bf16 %f1, %rs2;
+; SM80-NEXT: cvt.f32.bf16 %f1, %rs1;
; SM80-NEXT: add.rn.f32 %f2, %f1, %f1;
-; SM80-NEXT: cvt.rn.bf16.f32 %rs3, %f2;
-; SM80-NEXT: cvt.f32.bf16 %f3, %rs1;
+; SM80-NEXT: cvt.f32.bf16 %f3, %rs2;
; SM80-NEXT: add.rn.f32 %f4, %f3, %f3;
-; SM80-NEXT: cvt.rn.bf16.f32 %rs4, %f4;
-; SM80-NEXT: mov.b32 %r3, {%rs4, %rs3};
+; SM80-NEXT: cvt.rn.bf16x2.f32 %r3, %f4, %f2;
; SM80-NEXT: abs.bf16x2 %r4, %r3;
-; SM80-NEXT: mov.b32 {%rs5, %rs6}, %r4;
-; SM80-NEXT: cvt.f32.bf16 %f5, %rs6;
-; SM80-NEXT: mov.b32 {%rs7, %rs8}, %r1;
-; SM80-NEXT: cvt.f32.bf16 %f6, %rs8;
+; SM80-NEXT: mov.b32 {%rs3, %rs4}, %r4;
+; SM80-NEXT: cvt.f32.bf16 %f5, %rs3;
+; SM80-NEXT: mov.b32 {%rs5, %rs6}, %r1;
+; SM80-NEXT: cvt.f32.bf16 %f6, %rs5;
; SM80-NEXT: add.rn.f32 %f7, %f5, %f6;
-; SM80-NEXT: cvt.rn.bf16.f32 %rs9, %f7;
-; SM80-NEXT: cvt.f32.bf16 %f8, %rs5;
-; SM80-NEXT: cvt.f32.bf16 %f9, %rs7;
+; SM80-NEXT: cvt.f32.bf16 %f8, %rs4;
+; SM80-NEXT: cvt.f32.bf16 %f9, %rs6;
; SM80-NEXT: add.rn.f32 %f10, %f8, %f9;
-; SM80-NEXT: cvt.rn.bf16.f32 %rs10, %f10;
-; SM80-NEXT: mov.b32 %r5, {%rs10, %rs9};
+; SM80-NEXT: cvt.rn.bf16x2.f32 %r5, %f10, %f7;
; SM80-NEXT: st.param.b32 [func_retval0], %r5;
; SM80-NEXT: ret;
;
@@ -637,20 +620,18 @@ define <2 x bfloat> @test_maxnum(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
define <2 x bfloat> @test_floor(<2 x bfloat> %a) #0 {
; SM80-LABEL: test_floor(
; SM80: {
-; SM80-NEXT: .reg .b16 %rs<5>;
+; SM80-NEXT: .reg .b16 %rs<3>;
; SM80-NEXT: .reg .b32 %r<3>;
; SM80-NEXT: .reg .f32 %f<5>;
; SM80-EMPTY:
; SM80-NEXT: // %bb.0:
; SM80-NEXT: ld.param.b32 %r1, [test_floor_param_0];
; SM80-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; SM80-NEXT: cvt.f32.bf16 %f1, %rs2;
+; SM80-NEXT: cvt.f32.bf16 %f1, %rs1;
; SM80-NEXT: cvt.rmi.f32.f32 %f2, %f1;
-; SM80-NEXT: cvt.rn.bf16.f32 %rs3, %f2;
-; SM80-NEXT: cvt.f32.bf16 %f3, %rs1;
+; SM80-NEXT: cvt.f32.bf16 %f3, %rs2;
; SM80-NEXT: cvt.rmi.f32.f32 %f4, %f3;
-; SM80-NEXT: cvt.rn.bf16.f32 %rs4, %f4;
-; SM80-NEXT: mov.b32 %r2, {%rs4, %rs3};
+; SM80-NEXT: cvt.rn.bf16x2.f32 %r2, %f4, %f2;
; SM80-NEXT: st.param.b32 [func_retval0], %r2;
; SM80-NEXT: ret;
;
@@ -674,20 +655,18 @@ define <2 x bfloat> @test_floor(<2 x bfloat> %a) #0 {
define <2 x bfloat> @test_ceil(<2 x bfloat> %a) #0 {
; SM80-LABEL: test_ceil(
; SM80: {
-; SM80-NEXT: .reg .b16 %rs<5>;
+; SM80-NEXT: .reg .b16 %rs<3>;
; SM80-NEXT: .reg .b32 %r<3>;
; SM80-NEXT: .reg .f32 %f<5>;
; SM80-EMPTY:
; SM80-NEXT: // %bb.0:
; SM80-NEXT: ld.param.b32 %r1, [test_ceil_param_0];
; SM80-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; SM80-NEXT: cvt.f32.bf16 %f1, %rs2;
+; SM80-NEXT: cvt.f32.bf16 %f1, %rs1;
; SM80-NEXT: cvt.rpi.f32.f32 %f2, %f1;
-; SM80-NEXT: cvt.rn.bf16.f32 %rs3, %f2;
-; SM80-NEXT: cvt.f32.bf16 %f3, %rs1;
+; SM80-NEXT: cvt.f32.bf16 %f3, %rs2;
; SM80-NEXT: cvt.rpi.f32.f32 %f4, %f3;
-; SM80-NEXT: cvt.rn.bf16.f32 %rs4, %f4;
-; SM80-NEXT: mov.b32 %r2, {%rs4, %rs3};
+; SM80-NEXT: cvt.rn.bf16x2.f32 %r2, %f4, %f2;
; SM80-NEXT: st.param.b32 [func_retval0], %r2;
; SM80-NEXT: ret;
;
@@ -711,20 +690,18 @@ define <2 x bfloat> @test_ceil(<2 x bfloat> %a) #0 {
define <2 x bfloat> @test_trunc(<2 x bfloat> %a) #0 {
; SM80-LABEL: test_trunc(
; SM80: {
-; SM80-NEXT: .reg .b16 %rs<5>;
+; SM80-NEXT: .reg .b16 %rs<3>;
; SM80-NEXT: .reg .b32 %r<3>;
; SM80-NEXT: .reg .f32 %f<5>;
; SM80-EMPTY:
; SM80-NEXT: // %bb.0:
; SM80-NEXT: ld.param.b32 %r1, [test_trunc_param_0];
; SM80-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; SM80-NEXT: cvt.f32.bf16 %f1, %rs2;
+; SM80-NEXT: cvt.f32.bf16 %f1, %rs1;
; SM80-NEXT: cvt.rzi.f32.f32 %f2, %f1;
-; SM80-NEXT: cvt.rn.bf16.f32 %rs3, %f2;
-; SM80-NEXT: cvt.f32.bf16 %f3, %rs1;
+; SM80-NEXT: cvt.f32.bf16 %f3, %rs2;
; SM80-NEXT: cvt.rzi.f32.f32 %f4, %f3;
-; SM80-NEXT: cvt.rn.bf16.f32 %rs4, %f4;
-; SM80-NEXT: mov.b32 %r2, {%rs4, %rs3};
+; SM80-NEXT: cvt.rn.bf16x2.f32 %r2, %f4, %f2;
; SM80-NEXT: st.param.b32 [func_retval0], %r2;
; SM80-NEXT: ret;
;
@@ -748,20 +725,18 @@ define <2 x bfloat> @test_trunc(<2 x bfloat> %a) #0 {
define <2 x bfloat> @test_rint(<2 x bfloat> %a) #0 {
; SM80-LABEL: test_rint(
; SM80: {
-; SM80-NEXT: .reg .b16 %rs<5>;
+; SM80-NEXT: .reg .b16 %rs<3>;
; SM80-NEXT: .reg .b32 %r<3>;
; SM80-NEXT: .reg .f32 %f<5>;
; SM80-EMPTY:
; SM80-NEXT: // %bb.0:
; SM80-NEXT: ld.param.b32 %r1, [test_rint_param_0];
; SM80-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; SM80-NEXT: cvt.f32.bf16 %f1, %rs2;
+; SM80-NEXT: cvt.f32.bf16 %f1, %rs1;
; SM80-NEXT: cvt.rni.f32.f32 %f2, %f1;
-; SM80-NEXT: cvt.rn.bf16.f32 %rs3, %f2;
-; SM80-NEXT: cvt.f32.bf16 %f3, %rs1;
+; SM80-NEXT: cvt.f32.bf16 %f3, %rs2;
; SM80-NEXT: cvt.rni.f32.f32 %f4, %f3;
-; SM80-NEXT: cvt.rn.bf16.f32 %rs4, %f4;
-; SM80-NEXT: mov.b32 %r2, {%rs4, %rs3};
+; SM80-NEXT: cvt.rn.bf16x2.f32 %r2, %f4, %f2;
; SM80-NEXT: st.param.b32 [func_retval0], %r2;
; SM80-NEXT: ret;
;
@@ -786,14 +761,14 @@ define <2 x bfloat> @test_round(<2 x bfloat> %a) #0 {
; CHECK-LABEL: test_round(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<5>;
-; CHECK-NEXT: .reg .b16 %rs<5>;
+; CHECK-NEXT: .reg .b16 %rs<3>;
; CHECK-NEXT: .reg .b32 %r<9>;
; CHECK-NEXT: .reg .f32 %f<17>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b32 %r1, [test_round_param_0];
; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; CHECK-NEXT: cvt.f32.bf16 %f1, %rs2;
+; CHECK-NEXT: cvt.f32.bf16 %f1, %rs1;
; CHECK-NEXT: mov.b32 %r2, %f1;
; CHECK-NEXT: and.b32 %r3, %r2, -2147483648;
; CHECK-NEXT: or.b32 %r4, %r3, 1056964608;
@@ -806,8 +781,7 @@ define <2 x bfloat> @test_round(<2 x bfloat> %a) #0 {
; CHECK-NEXT: cvt.rzi.f32.f32 %f7, %f1;
; CHECK-NEXT: setp.lt.f32 %p2, %f5, 0f3F000000;
; CHECK-NEXT: selp.f32 %f8, %f7, %f6, %p2;
-; CHECK-NEXT: cvt.rn.bf16.f32 %rs3, %f8;
-; CHECK-NEXT: cvt.f32.bf16 %f9, %rs1;
+; CHECK-NEXT: cvt.f32.bf16 %f9, %rs2;
; CHECK-NEXT: mov.b32 %r5, %f9;
; CHECK-NEXT: and.b32 %r6, %r5, -2147483648;
; CHECK-NEXT: or.b32 %r7, %r6, 1056964608;
@@ -820,8 +794,7 @@ define <2 x bfloat> @test_round(<2 x bfloat> %a) #0 {
; CHECK-NEXT: cvt.rzi.f32.f32 %f15, %f9;
; CHECK-NEXT: setp.lt.f32 %p4, %f13, 0f3F000000;
; CHECK-NEXT: selp.f32 %f16, %f15, %f14, %p4;
-; CHECK-NEXT: cvt.rn.bf16.f32 %rs4, %f16;
-; CHECK-NEXT: mov.b32 %r8, {%rs4, %rs3};
+; CHECK-NEXT: cvt.rn.bf16x2.f32 %r8, %f16, %f8;
; CHECK-NEXT: st.param.b32 [func_retval0], %r8;
; CHECK-NEXT: ret;
%r = call <2 x bfloat> @llvm.round.f16(<2 x bfloat> %a)
diff --git a/llvm/test/CodeGen/NVPTX/fma-relu-contract.ll b/llvm/test/CodeGen/NVPTX/fma-relu-contract.ll
index 8cc4548f6e85e0..40771784d1b28b 100644
--- a/llvm/test/CodeGen/NVPTX/fma-relu-contract.ll
+++ b/llvm/test/CodeGen/NVPTX/fma-relu-contract.ll
@@ -959,7 +959,7 @@ define <2 x bfloat> @fma_bf16x2_expanded_no_nans(<2 x bfloat> %a, <2 x bfloat> %
define <2 x bfloat> @fma_bf16x2_expanded_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c) #0 {
; CHECK-LABEL: fma_bf16x2_expanded_no_nans_multiple_uses_of_fma(
; CHECK: {
-; CHECK-NEXT: .reg .b16 %rs<15>;
+; CHECK-NEXT: .reg .b16 %rs<13>;
; CHECK-NEXT: .reg .b32 %r<20>;
; CHECK-NEXT: .reg .f32 %f<11>;
; CHECK-EMPTY:
@@ -971,40 +971,38 @@ define <2 x bfloat> @fma_bf16x2_expanded_no_nans_multiple_uses_of_fma(<2 x bfloa
; CHECK-NEXT: mov.b32 %r5, 0;
; CHECK-NEXT: max.bf16x2 %r6, %r4, %r5;
; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r4;
-; CHECK-NEXT: cvt.u32.u16 %r7, %rs1;
+; CHECK-NEXT: cvt.u32.u16 %r7, %rs2;
; CHECK-NEXT: shl.b32 %r8, %r7, 16;
; CHECK-NEXT: mov.b32 %f1, %r8;
; CHECK-NEXT: add.f32 %f2, %f1, 0f40E00000;
; CHECK-NEXT: cvt.rn.bf16.f32 %rs4, %f2;
-; CHECK-NEXT: cvt.u32.u16 %r9, %rs2;
+; CHECK-NEXT: cvt.u32.u16 %r9, %rs1;
; CHECK-NEXT: shl.b32 %r10, %r9, 16;
; CHECK-NEXT: mov.b32 %f3, %r10;
; CHECK-NEXT: add.f32 %f4, %f3, 0f40E00000;
; CHECK-NEXT: cvt.rn.bf16.f32 %rs6, %f4;
; CHECK-NEXT: mov.b32 {%rs7, %rs8}, %r6;
-; CHECK-NEXT: cvt.u32.u16 %r11, %rs8;
+; CHECK-NEXT: cvt.u32.u16 %r11, %rs7;
; CHECK-NEXT: shl.b32 %r12, %r11, 16;
; CHECK-NEXT: mov.b32 %f5, %r12;
; CHECK-NEXT: cvt.u32.u16 %r13, %rs6;
; CHECK-NEXT: shl.b32 %r14, %r13, 16;
; CHECK-NEXT: mov.b32 %f6, %r14;
; CHECK-NEXT: add.f32 %f7, %f5, %f6;
-; CHECK-NEXT: cvt.rn.bf16.f32 %rs11, %f7;
-; CHECK-NEXT: cvt.u32.u16 %r15, %rs7;
+; CHECK-NEXT: cvt.u32.u16 %r15, %rs8;
; CHECK-NEXT: shl.b32 %r16, %r15, 16;
; CHECK-NEXT: mov.b32 %f8, %r16;
; CHECK-NEXT: cvt.u32.u16 %r17, %rs4;
; CHECK-NEXT: shl.b32 %r18, %r17, 16;
; CHECK-NEXT: mov.b32 %f9, %r18;
; CHECK-NEXT: add.f32 %f10, %f8, %f9;
-; CHECK-NEXT: cvt.rn.bf16.f32 %rs14, %f10;
-; CHECK-NEXT: mov.b32 %r19, {%rs14, %rs11};
+; CHECK-NEXT: cvt.rn.bf16x2.f32 %r19, %f10, %f7;
; CHECK-NEXT: st.param.b32 [func_retval0], %r19;
; CHECK-NEXT: ret;
;
; CHECK-FTZ-LABEL: fma_bf16x2_expanded_no_nans_multiple_uses_of_fma(
; CHECK-FTZ: {
-; CHECK-FTZ-NEXT: .reg .b16 %rs<15>;
+; CHECK-FTZ-NEXT: .reg .b16 %rs<13>;
; CHECK-FTZ-NEXT: .reg .b32 %r<20>;
; CHECK-FTZ-NEXT: .reg .f32 %f<11>;
; CHECK-FTZ-EMPTY:
@@ -1016,34 +1014,32 @@ define <2 x bfloat> @fma_bf16x2_expanded_no_nans_multiple_uses_of_fma(<2 x bfloa
; CHECK-FTZ-NEXT: mov.b32 %r5, 0;
; CHECK-FTZ-NEXT: max.bf16x2 %r6, %r4, %r5;
; CHECK-FTZ-NEXT: mov.b32 {%rs1, %rs2}, %r4;
-; CHECK-FTZ-NEXT: cvt.u32.u16 %r7, %rs1;
+; CHECK-FTZ-NEXT: cvt.u32.u16 %r7, %rs2;
; CHECK-FTZ-NEXT: shl.b32 %r8, %r7, 16;
; CHECK-FTZ-NEXT: mov.b32 %f1, %r8;
; CHECK-FTZ-NEXT: add.ftz.f32 %f2, %f1, 0f40E00000;
; CHECK-FTZ-NEXT: cvt.rn.bf16.f32 %rs4, %f2;
-; CHECK-FTZ-NEXT: cvt.u32.u16 %r9, %rs2;
+; CHECK-FTZ-NEXT: cvt.u32.u16 %r9, %rs1;
; CHECK-FTZ-NEXT: shl.b32 %r10, %r9, 16;
; CHECK-FTZ-NEXT: mov.b32 %f3, %r10;
; CHECK-FTZ-NEXT: add.ftz.f32 %f4, %f3, 0f40E00000;
; CHECK-FTZ-NEXT: cvt.rn.bf16.f32 %rs6, %f4;
; CHECK-FTZ-NEXT: mov.b32 {%rs7, %rs8}, %r6;
-; CHECK-FTZ-NEXT: cvt.u32.u16 %r11, %rs8;
+; CHECK-FTZ-NEXT: cvt.u32.u16 %r11, %rs7;
; CHECK-FTZ-NEXT: shl.b32 %r12, %r11, 16;
; CHECK-FTZ-NEXT: mov.b32 %f5, %r12;
; CHECK-FTZ-NEXT: cvt.u32.u16 %r13, %rs6;
; CHECK-FTZ-NEXT: shl.b32 %r14, %r13, 16;
; CHECK-FTZ-NEXT: mov.b32 %f6, %r14;
; CHECK-FTZ-NEXT: add.ftz.f32 %f7, %f5, %f6;
-; CHECK-FTZ-NEXT: cvt.rn.bf16.f32 %rs11, %f7;
-; CHECK-FTZ-NEXT: cvt.u32.u16 %r15, %rs7;
+; CHECK-FTZ-NEXT: cvt.u32.u16 %r15, %rs8;
; CHECK-FTZ-NEXT: shl.b32 %r16, %r15, 16;
; CHECK-FTZ-NEXT: mov.b32 %f8, %r16;
; CHECK-FTZ-NEXT: cvt.u32.u16 %r17, %rs4;
; CHECK-FTZ-NEXT: shl.b32 %r18, %r17, 16;
; CHECK-FTZ-NEXT: mov.b32 %f9, %r18;
; CHECK-FTZ-NEXT: add.ftz.f32 %f10, %f8, %f9;
-; CHECK-FTZ-NEXT: cvt.rn.bf16.f32 %rs14, %f10;
-; CHECK-FTZ-NEXT: mov.b32 %r19, {%rs14, %rs11};
+; CHECK-FTZ-NEXT: cvt.rn.bf16x2.f32 %r19, %f10, %f7;
; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r19;
; CHECK-FTZ-NEXT: ret;
;
diff --git a/llvm/test/CodeGen/NVPTX/fma-relu-fma-intrinsic.ll b/llvm/test/CodeGen/NVPTX/fma-relu-fma-intrinsic.ll
index 16219aa9da0950..4d61ce1478953a 100644
--- a/llvm/test/CodeGen/NVPTX/fma-relu-fma-intrinsic.ll
+++ b/llvm/test/CodeGen/NVPTX/fma-relu-fma-intrinsic.ll
@@ -642,7 +642,7 @@ define <2 x bfloat> @fma_bf16x2_no_nans(<2 x bfloat> %a, <2 x bfloat> %b, <2 x b
define <2 x bfloat> @fma_bf16x2_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c) #0 {
; CHECK-LABEL: fma_bf16x2_no_nans_multiple_uses_of_fma(
; CHECK: {
-; CHECK-NEXT: .reg .b16 %rs<11>;
+; CHECK-NEXT: .reg .b16 %rs<9>;
; CHECK-NEXT: .reg .b32 %r<14>;
; CHECK-NEXT: .reg .f32 %f<9>;
; CHECK-EMPTY:
@@ -652,12 +652,12 @@ define <2 x bfloat> @fma_bf16x2_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2
; CHECK-NEXT: ld.param.b32 %r3, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_0];
; CHECK-NEXT: fma.rn.bf16x2 %r4, %r3, %r2, %r1;
; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r4;
-; CHECK-NEXT: cvt.u32.u16 %r5, %rs1;
+; CHECK-NEXT: cvt.u32.u16 %r5, %rs2;
; CHECK-NEXT: shl.b32 %r6, %r5, 16;
; CHECK-NEXT: mov.b32 %f1, %r6;
; CHECK-NEXT: add.f32 %f2, %f1, 0f40E00000;
; CHECK-NEXT: cvt.rn.bf16.f32 %rs4, %f2;
-; CHECK-NEXT: cvt.u32.u16 %r7, %rs2;
+; CHECK-NEXT: cvt.u32.u16 %r7, %rs1;
; CHECK-NEXT: shl.b32 %r8, %r7, 16;
; CHECK-NEXT: mov.b32 %f3, %r8;
; CHECK-NEXT: add.f32 %f4, %f3, 0f40E00000;
@@ -666,19 +666,17 @@ define <2 x bfloat> @fma_bf16x2_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2
; CHECK-NEXT: shl.b32 %r10, %r9, 16;
; CHECK-NEXT: mov.b32 %f5, %r10;
; CHECK-NEXT: add.f32 %f6, %f5, %f3;
-; CHECK-NEXT: cvt.rn.bf16.f32 %rs8, %f6;
; CHECK-NEXT: cvt.u32.u16 %r11, %rs4;
; CHECK-NEXT: shl.b32 %r12, %r11, 16;
; CHECK-NEXT: mov.b32 %f7, %r12;
; CHECK-NEXT: add.f32 %f8, %f7, %f1;
-; CHECK-NEXT: cvt.rn.bf16.f32 %rs10, %f8;
-; CHECK-NEXT: mov.b32 %r13, {%rs10, %rs8};
+; CHECK-NEXT: cvt.rn.bf16x2.f32 %r13, %f8, %f6;
; CHECK-NEXT: st.param.b32 [func_retval0], %r13;
; CHECK-NEXT: ret;
;
; CHECK-FTZ-LABEL: fma_bf16x2_no_nans_multiple_uses_of_fma(
; CHECK-FTZ: {
-; CHECK-FTZ-NEXT: .reg .b16 %rs<11>;
+; CHECK-FTZ-NEXT: .reg .b16 %rs<9>;
; CHECK-FTZ-NEXT: .reg .b32 %r<14>;
; CHECK-FTZ-NEXT: .reg .f32 %f<9>;
; CHECK-FTZ-EMPTY:
@@ -688,12 +686,12 @@ define <2 x bfloat> @fma_bf16x2_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2
; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_0];
; CHECK-FTZ-NEXT: fma.rn.bf16x2 %r4, %r3, %r2, %r1;
; CHECK-FTZ-NEXT: mov.b32 {%rs1, %rs2}, %r4;
-; CHECK-FTZ-NEXT: cvt.u32.u16 %r5, %rs1;
+; CHECK-FTZ-NEXT: cvt.u32.u16 %r5, %rs2;
; CHECK-FTZ-NEXT: shl.b32 %r6, %r5, 16;
; CHECK-FTZ-NEXT: mov.b32 %f1, %r6;
; CHECK-FTZ-NEXT: add.ftz.f32 %f2, %f1, 0f40E00000;
; CHECK-FTZ-NEXT: cvt.rn.bf16.f32 %rs4, %f2;
-; CHECK-FTZ-NEXT: cvt.u32.u16 %r7, %rs2;
+; CHECK-FTZ-NEXT: cvt.u32.u16 %r7, %rs1;
; CHECK-FTZ-NEXT: shl.b32 %r8, %r7, 16;
; CHECK-FTZ-NEXT: mov.b32 %f3, %r8;
; CHECK-FTZ-NEXT: add.ftz.f32 %f4, %f3, 0f40E00000;
@@ -702,13 +700,11 @@ define <2 x bfloat> @fma_bf16x2_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2
; CHECK-FTZ-NEXT: shl.b32 %r10, %r9, 16;
; CHECK-FTZ-NEXT: mov.b32 %f5, %r10;
; CHECK-FTZ-NEXT: add.ftz.f32 %f6, %f5, %f3;
-; CHECK-FTZ-NEXT: cvt.rn.bf16.f32 %rs8, %f6;
; CHECK-FTZ-NEXT: cvt.u32.u16 %r11, %rs4;
; CHECK-FTZ-NEXT: shl.b32 %r12, %r11, 16;
; CHECK-FTZ-NEXT: mov.b32 %f7, %r12;
; CHECK-FTZ-NEXT: add.ftz.f32 %f8, %f7, %f1;
-; CHECK-FTZ-NEXT: cvt.rn.bf16.f32 %rs10, %f8;
-; CHECK-FTZ-NEXT: mov.b32 %r13, {%rs10, %rs8};
+; CHECK-FTZ-NEXT: cvt.rn.bf16x2.f32 %r13, %f8, %f6;
; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r13;
; CHECK-FTZ-NEXT: ret;
;
diff --git a/llvm/test/CodeGen/NVPTX/fma-relu-instruction-flag.ll b/llvm/test/CodeGen/NVPTX/fma-relu-instruction-flag.ll
index af21bada7783be..043c0fcca63710 100644
--- a/llvm/test/CodeGen/NVPTX/fma-relu-instruction-flag.ll
+++ b/llvm/test/CodeGen/NVPTX/fma-relu-instruction-flag.ll
@@ -694,7 +694,7 @@ define <2 x bfloat> @fma_bf16x2_expanded_no_nans(<2 x bfloat> %a, <2 x bfloat> %
define <2 x bfloat> @fma_bf16x2_expanded_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c) {
; CHECK-LABEL: fma_bf16x2_expanded_no_nans_multiple_uses_of_fma(
; CHECK: {
-; CHECK-NEXT: .reg .b16 %rs<15>;
+; CHECK-NEXT: .reg .b16 %rs<13>;
; CHECK-NEXT: .reg .b32 %r<20>;
; CHECK-NEXT: .reg .f32 %f<11>;
; CHECK-EMPTY:
@@ -706,40 +706,38 @@ define <2 x bfloat> @fma_bf16x2_expanded_no_nans_multiple_uses_of_fma(<2 x bfloa
; CHECK-NEXT: mov.b32 %r5, 0;
; CHECK-NEXT: max.bf16x2 %r6, %r4, %r5;
; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r4;
-; CHECK-NEXT: cvt.u32.u16 %r7, %rs1;
+; CHECK-NEXT: cvt.u32.u16 %r7, %rs2;
; CHECK-NEXT: shl.b32 %r8, %r7, 16;
; CHECK-NEXT: mov.b32 %f1, %r8;
; CHECK-NEXT: add.rn.f32 %f2, %f1, 0f40E00000;
; CHECK-NEXT: cvt.rn.bf16.f32 %rs4, %f2;
-; CHECK-NEXT: cvt.u32.u16 %r9, %rs2;
+; CHECK-NEXT: cvt.u32.u16 %r9, %rs1;
; CHECK-NEXT: shl.b32 %r10, %r9, 16;
; CHECK-NEXT: mov.b32 %f3, %r10;
; CHECK-NEXT: add.rn.f32 %f4, %f3, 0f40E00000;
; CHECK-NEXT: cvt.rn.bf16.f32 %rs6, %f4;
; CHECK-NEXT: mov.b32 {%rs7, %rs8}, %r6;
-; CHECK-NEXT: cvt.u32.u16 %r11, %rs8;
+; CHECK-NEXT: cvt.u32.u16 %r11, %rs7;
; CHECK-NEXT: shl.b32 %r12, %r11, 16;
; CHECK-NEXT: mov.b32 %f5, %r12;
; CHECK-NEXT: cvt.u32.u16 %r13, %rs6;
; CHECK-NEXT: shl.b32 %r14, %r13, 16;
; CHECK-NEXT: mov.b32 %f6, %r14;
; CHECK-NEXT: add.rn.f32 %f7, %f5, %f6;
-; CHECK-NEXT: cvt.rn.bf16.f32 %rs11, %f7;
-; CHECK-NEXT: cvt.u32.u16 %r15, %rs7;
+; CHECK-NEXT: cvt.u32.u16 %r15, %rs8;
; CHECK-NEXT: shl.b32 %r16, %r15, 16;
; CHECK-NEXT: mov.b32 %f8, %r16;
; CHECK-NEXT: cvt.u32.u16 %r17, %rs4;
; CHECK-NEXT: shl.b32 %r18, %r17, 16;
; CHECK-NEXT: mov.b32 %f9, %r18;
; CHECK-NEXT: add.rn.f32 %f10, %f8, %f9;
-; CHECK-NEXT: cvt.rn.bf16.f32 %rs14, %f10;
-; CHECK-NEXT: mov.b32 %r19, {%rs14, %rs11};
+; CHECK-NEXT: cvt.rn.bf16x2.f32 %r19, %f10, %f7;
; CHECK-NEXT: st.param.b32 [func_retval0], %r19;
; CHECK-NEXT: ret;
;
; CHECK-FTZ-LABEL: fma_bf16x2_expanded_no_nans_multiple_uses_of_fma(
; CHECK-FTZ: {
-; CHECK-FTZ-NEXT: .reg .b16 %rs<15>;
+; CHECK-FTZ-NEXT: .reg .b16 %rs<13>;
; CHECK-FTZ-NEXT: .reg .b32 %r<20>;
; CHECK-FTZ-NEXT: .reg .f32 %f<11>;
; CHECK-FTZ-EMPTY:
@@ -751,34 +749,32 @@ define <2 x bfloat> @fma_bf16x2_expanded_no_nans_multiple_uses_of_fma(<2 x bfloa
; CHECK-FTZ-NEXT: mov.b32 %r5, 0;
; CHECK-FTZ-NEXT: max.bf16x2 %r6, %r4, %r5;
; CHECK-FTZ-NEXT: mov.b32 {%rs1, %rs2}, %r4;
-; CHECK-FTZ-NEXT: cvt.u32.u16 %r7, %rs1;
+; CHECK-FTZ-NEXT: cvt.u32.u16 %r7, %rs2;
; CHECK-FTZ-NEXT: shl.b32 %r8, %r7, 16;
; CHECK-FTZ-NEXT: mov.b32 %f1, %r8;
; CHECK-FTZ-NEXT: add.rn.ftz.f32 %f2, %f1, 0f40E00000;
; CHECK-FTZ-NEXT: cvt.rn.bf16.f32 %rs4, %f2;
-; CHECK-FTZ-NEXT: cvt.u32.u16 %r9, %rs2;
+; CHECK-FTZ-NEXT: cvt.u32.u16 %r9, %rs1;
; CHECK-FTZ-NEXT: shl.b32 %r10, %r9, 16;
; CHECK-FTZ-NEXT: mov.b32 %f3, %r10;
; CHECK-FTZ-NEXT: add.rn.ftz.f32 %f4, %f3, 0f40E00000;
; CHECK-FTZ-NEXT: cvt.rn.bf16.f32 %rs6, %f4;
; CHECK-FTZ-NEXT: mov.b32 {%rs7, %rs8}, %r6;
-; CHECK-FTZ-NEXT: cvt.u32.u16 %r11, %rs8;
+; CHECK-FTZ-NEXT: cvt.u32.u16 %r11, %rs7;
; CHECK-FTZ-NEXT: shl.b32 %r12, %r11, 16;
; CHECK-FTZ-NEXT: mov.b32 %f5, %r12;
; CHECK-FTZ-NEXT: cvt.u32.u16 %r13, %rs6;
; CHECK-FTZ-NEXT: shl.b32 %r14, %r13, 16;
; CHECK-FTZ-NEXT: mov.b32 %f6, %r14;
; CHECK-FTZ-NEXT: add.rn.ftz.f32 %f7, %f5, %f6;
-; CHECK-FTZ-NEXT: cvt.rn.bf16.f32 %rs11, %f7;
-; CHECK-FTZ-NEXT: cvt.u32.u16 %r15, %rs7;
+; CHECK-FTZ-NEXT: cvt.u32.u16 %r15, %rs8;
; CHECK-FTZ-NEXT: shl.b32 %r16, %r15, 16;
; CHECK-FTZ-NEXT: mov.b32 %f8, %r16;
; CHECK-FTZ-NEXT: cvt.u32.u16 %r17, %rs4;
; CHECK-FTZ-NEXT: shl.b32 %r18, %r17, 16;
; CHECK-FTZ-NEXT: mov.b32 %f9, %r18;
; CHECK-FTZ-NEXT: add.rn.ftz.f32 %f10, %f8, %f9;
-; CHECK-FTZ-NEXT: cvt.rn.bf16.f32 %rs14, %f10;
-; CHECK-FTZ-NEXT: mov.b32 %r19, {%rs14, %rs11};
+; CHECK-FTZ-NEXT: cvt.rn.bf16x2.f32 %r19, %f10, %f7;
; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r19;
; CHECK-FTZ-NEXT: ret;
;
@@ -1637,7 +1633,7 @@ define <2 x bfloat> @fma_bf16x2_no_nans(<2 x bfloat> %a, <2 x bfloat> %b, <2 x b
define <2 x bfloat> @fma_bf16x2_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c) {
; CHECK-LABEL: fma_bf16x2_no_nans_multiple_uses_of_fma(
; CHECK: {
-; CHECK-NEXT: .reg .b16 %rs<11>;
+; CHECK-NEXT: .reg .b16 %rs<9>;
; CHECK-NEXT: .reg .b32 %r<14>;
; CHECK-NEXT: .reg .f32 %f<9>;
; CHECK-EMPTY:
@@ -1647,12 +1643,12 @@ define <2 x bfloat> @fma_bf16x2_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2
; CHECK-NEXT: ld.param.b32 %r3, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_0];
; CHECK-NEXT: fma.rn.bf16x2 %r4, %r3, %r2, %r1;
; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r4;
-; CHECK-NEXT: cvt.u32.u16 %r5, %rs1;
+; CHECK-NEXT: cvt.u32.u16 %r5, %rs2;
; CHECK-NEXT: shl.b32 %r6, %r5, 16;
; CHECK-NEXT: mov.b32 %f1, %r6;
; CHECK-NEXT: add.rn.f32 %f2, %f1, 0f40E00000;
; CHECK-NEXT: cvt.rn.bf16.f32 %rs4, %f2;
-; CHECK-NEXT: cvt.u32.u16 %r7, %rs2;
+; CHECK-NEXT: cvt.u32.u16 %r7, %rs1;
; CHECK-NEXT: shl.b32 %r8, %r7, 16;
; CHECK-NEXT: mov.b32 %f3, %r8;
; CHECK-NEXT: add.rn.f32 %f4, %f3, 0f40E00000;
@@ -1661,19 +1657,17 @@ define <2 x bfloat> @fma_bf16x2_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2
; CHECK-NEXT: shl.b32 %r10, %r9, 16;
; CHECK-NEXT: mov.b32 %f5, %r10;
; CHECK-NEXT: add.rn.f32 %f6, %f5, %f3;
-; CHECK-NEXT: cvt.rn.bf16.f32 %rs8, %f6;
; CHECK-NEXT: cvt.u32.u16 %r11, %rs4;
; CHECK-NEXT: shl.b32 %r12, %r11, 16;
; CHECK-NEXT: mov.b32 %f7, %r12;
; CHECK-NEXT: add.rn.f32 %f8, %f7, %f1;
-; CHECK-NEXT: cvt.rn.bf16.f32 %rs10, %f8;
-; CHECK-NEXT: mov.b32 %r13, {%rs10, %rs8};
+; CHECK-NEXT: cvt.rn.bf16x2.f32 %r13, %f8, %f6;
; CHECK-NEXT: st.param.b32 [func_retval0], %r13;
; CHECK-NEXT: ret;
;
; CHECK-FTZ-LABEL: fma_bf16x2_no_nans_multiple_uses_of_fma(
; CHECK-FTZ: {
-; CHECK-FTZ-NEXT: .reg .b16 %rs<11>;
+; CHECK-FTZ-NEXT: .reg .b16 %rs<9>;
; CHECK-FTZ-NEXT: .reg .b32 %r<14>;
; CHECK-FTZ-NEXT: .reg .f32 %f<9>;
; CHECK-FTZ-EMPTY:
@@ -1683,12 +1677,12 @@ define <2 x bfloat> @fma_bf16x2_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2
; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_0];
; CHECK-FTZ-NEXT: fma.rn.bf16x2 %r4, %r3, %r2, %r1;
; CHECK-FTZ-NEXT: mov.b32 {%rs1, %rs2}, %r4;
-; CHECK-FTZ-NEXT: cvt.u32.u16 %r5, %rs1;
+; CHECK-FTZ-NEXT: cvt.u32.u16 %r5, %rs2;
; CHECK-FTZ-NEXT: shl.b32 %r6, %r5, 16;
; CHECK-FTZ-NEXT: mov.b32 %f1, %r6;
; CHECK-FTZ-NEXT: add.rn.ftz.f32 %f2, %f1, 0f40E00000;
; CHECK-FTZ-NEXT: cvt.rn.bf16.f32 %rs4, %f2;
-; CHECK-FTZ-NEXT: cvt.u32.u16 %r7, %rs2;
+; CHECK-FTZ-NEXT: cvt.u32.u16 %r7, %rs1;
; CHECK-FTZ-NEXT: shl.b32 %r8, %r7, 16;
; CHECK-FTZ-NEXT: mov.b32 %f3, %r8;
; CHECK-FTZ-NEXT: add.rn.ftz.f32 %f4, %f3, 0f40E00000;
@@ -1697,13 +1691,11 @@ define <2 x bfloat> @fma_bf16x2_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2
; CHECK-FTZ-NEXT: shl.b32 %r10, %r9, 16;
; CHECK-FTZ-NEXT: mov.b32 %f5, %r10;
; CHECK-FTZ-NEXT: add.rn.ftz.f32 %f6, %f5, %f3;
-; CHECK-FTZ-NEXT: cvt.rn.bf16.f32 %rs8, %f6;
; CHECK-FTZ-NEXT: cvt.u32.u16 %r11, %rs4;
; CHECK-FTZ-NEXT: shl.b32 %r12, %r11, 16;
; CHECK-FTZ-NEXT: mov.b32 %f7, %r12;
; CHECK-FTZ-NEXT: add.rn.ftz.f32 %f8, %f7, %f1;
-; CHECK-FTZ-NEXT: cvt.rn.bf16.f32 %rs10, %f8;
-; CHECK-FTZ-NEXT: mov.b32 %r13, {%rs10, %rs8};
+; CHECK-FTZ-NEXT: cvt.rn.bf16x2.f32 %r13, %f8, %f6;
; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r13;
; CHECK-FTZ-NEXT: ret;
;
>From 07b204589ea4554b26f5aa76d83f5bf61f72be76 Mon Sep 17 00:00:00 2001
From: Alex Maclean <amaclean at nvidia.com>
Date: Mon, 18 Nov 2024 22:39:56 +0000
Subject: [PATCH 5/5] address comments
---
llvm/test/CodeGen/NVPTX/convert-sm80.ll | 20 ++++++++++----------
1 file changed, 10 insertions(+), 10 deletions(-)
diff --git a/llvm/test/CodeGen/NVPTX/convert-sm80.ll b/llvm/test/CodeGen/NVPTX/convert-sm80.ll
index b739148393ced6..1106a062f1d565 100644
--- a/llvm/test/CodeGen/NVPTX/convert-sm80.ll
+++ b/llvm/test/CodeGen/NVPTX/convert-sm80.ll
@@ -224,7 +224,7 @@ define i32 @cvt_rna_tf32_f32(float %f1) {
declare i32 @llvm.nvvm.f2tf32.rna(float)
-define <2 x bfloat> @fold_ff2bf16x2(float %a, float %b) {
+define <2 x bfloat> @fold_ff2bf16x2(float %lo, float %hi) {
; CHECK-LABEL: fold_ff2bf16x2(
; CHECK: {
; CHECK-NEXT: .reg .b32 %r<2>;
@@ -236,14 +236,14 @@ define <2 x bfloat> @fold_ff2bf16x2(float %a, float %b) {
; CHECK-NEXT: cvt.rn.bf16x2.f32 %r1, %f2, %f1;
; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
; CHECK-NEXT: ret;
- %ah = fptrunc float %a to bfloat
- %bh = fptrunc float %b to bfloat
- %v0 = insertelement <2 x bfloat> poison, bfloat %ah, i64 0
- %v1 = insertelement <2 x bfloat> %v0, bfloat %bh, i64 1
+ %loh = fptrunc float %lo to bfloat
+ %hih = fptrunc float %hi to bfloat
+ %v0 = insertelement <2 x bfloat> poison, bfloat %loh, i64 0
+ %v1 = insertelement <2 x bfloat> %v0, bfloat %hih, i64 1
ret <2 x bfloat> %v1
}
-define <2 x half> @fold_ff2f16x2(float %a, float %b) {
+define <2 x half> @fold_ff2f16x2(float %lo, float %hi) {
; CHECK-LABEL: fold_ff2f16x2(
; CHECK: {
; CHECK-NEXT: .reg .b32 %r<2>;
@@ -255,9 +255,9 @@ define <2 x half> @fold_ff2f16x2(float %a, float %b) {
; CHECK-NEXT: cvt.rn.f16x2.f32 %r1, %f2, %f1;
; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
; CHECK-NEXT: ret;
- %ah = fptrunc float %a to half
- %bh = fptrunc float %b to half
- %v0 = insertelement <2 x half> poison, half %ah, i64 0
- %v1 = insertelement <2 x half> %v0, half %bh, i64 1
+ %loh = fptrunc float %lo to half
+ %hih = fptrunc float %hi to half
+ %v0 = insertelement <2 x half> poison, half %loh, i64 0
+ %v1 = insertelement <2 x half> %v0, half %hih, i64 1
ret <2 x half> %v1
}
More information about the llvm-commits
mailing list